diff --git a/.bazelignore b/.bazelignore deleted file mode 120000 index 8fdc75e7..00000000 --- a/.bazelignore +++ /dev/null @@ -1 +0,0 @@ -examples/.bazelignore \ No newline at end of file diff --git a/.bazelignore b/.bazelignore new file mode 100644 index 00000000..e0df23f9 --- /dev/null +++ b/.bazelignore @@ -0,0 +1,4 @@ +.* +.zig-cache +.devenv +venv/ diff --git a/.bazelrc b/.bazelrc deleted file mode 120000 index b9f681ed..00000000 --- a/.bazelrc +++ /dev/null @@ -1 +0,0 @@ -examples/.bazelrc \ No newline at end of file diff --git a/.bazelrc b/.bazelrc new file mode 100644 index 00000000..09c50737 --- /dev/null +++ b/.bazelrc @@ -0,0 +1,147 @@ +# Faster hash function +startup --digest_function=blake3 + +# Needed by LLVM and/or XLA +common --experimental_repo_remote_exec + +# Self explanatory +common --enable_platform_specific_config + +# Ensure host tools are built optimized +common --host_compilation_mode=opt +common --@rules_zig//zig/settings:host_mode=release_safe + +# Suppress all warnings, LLVM and friends are too noisy +common --copt='-w' +common --host_copt='-w' + +# Enable C++17 +common --cxxopt='-std=c++17' +common --host_cxxopt='-std=c++17' +common --enable_runfiles + +# Ensure rules_python doesn't rely on the system's Python for stage-1 bootstrap +common --@rules_python//python/config_settings:bootstrap_impl=script + +# We don't to check the built files in the sandbox or repositories +common --experimental_check_output_files=false + +# Expose Zig's shared cache directories to sandboxed actions when the +# selected sandbox implementation supports host path mounts. +# +# These flags are consumed by linux-sandbox, and by Bazel's Docker sandbox. +# They are ignored by processwrapper-sandbox and macOS sandbox-exec, which use +# a symlinked execroot rather than bind mounts. +build:linux --sandbox_add_mount_pair=/tmp +build:linux --sandbox_writable_path=/tmp +build:linux --sandbox_add_mount_pair=/var/tmp +build:linux --sandbox_writable_path=/var/tmp + +# Ensure workers are sandboxed +build --worker_sandboxing + +# Build in debug for now, by default +common --compilation_mode=dbg + +# Various cc options +common --experimental_cc_shared_library +common --incompatible_use_cc_configure_from_rules_cc +common --incompatible_autoload_externally=+@rules_cc,+@rules_python,+@rules_shell,+@rules_java + +common --experimental_platform_in_output_dir + +# Faster sandbox speeds +build --experimental_inmemory_sandbox_stashes + +# clang by default +common --compiler=clang + +# Allow empty globs since some legacy rules +common --incompatible_disallow_empty_glob=false + +# Use the C linker to link Zig code +common --@rules_zig//zig/settings:use_cc_common_link=true + +common --@rules_zig//zig/settings:use_standalone_translate_c=True + +common --@llvm//config:experimental_stub_libgcc_s=true + +common --macos_minimum_os=13.0 + +# Enable stamping +common --stamp=true +common --workspace_status_command=./tools/workspace_status.sh + +common --@rules_cc//cc/toolchains/args/archiver_flags:use_libtool_on_macos=false +common --repo_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1 +common --repo_env=BAZEL_NO_APPLE_CPP_TOOLCHAIN=1 +common --xcode_version_config=//:disable_xcode + +# Required for protobuf upb on Linux +common:linux --copt=-fPIC + +# Prefer prebuilt protoc (finally!) +common --@com_google_protobuf//bazel/toolchains:prefer_prebuilt_protoc=true + +# Print test errors in the console +test --test_output=errors + +common:silent --noshow_progress +common:silent --ui_event_filters=,+error,+fail +common:silent --show_result=0 +common:silent --logging=0 + +# --config=debug will used optmized backend, but all the frontend zig code will be compiled in debug mode. +common:debug --compilation_mode=opt +common:debug --@rules_zig//zig/settings:mode=debug +common:debug --strategy=ZigBuildLib=local +common:debug --strategy=ZigBuildObj=local +common:debug --strategy=ZigBuildTestLib=local + +# --config=alldebug build everything in debug mode, including the backend. +common:alldebug --compilation_mode=dbg +common:alldebug --@rules_zig//zig/settings:mode=debug +common:alldebug --strategy=local + +common:release --compilation_mode=opt +common:release --copt=-Ofast +common:release --strip=always +common:release --@rules_zig//zig/settings:mode=release_safe +common:release --@rules_zig//zig/settings:zigopt=-fllvm + +# --config=native builds everything optimized for the local CPU +common:native --copt=-march=native +common:native --@rules_zig//zig/settings:zigopt=-mcpu=native + +# Remote execution and caching with BuildBuddy +common:remote --bes_results_url=https://app.buildbuddy.io/invocation/ +common:remote --bes_backend=grpcs://remote.buildbuddy.io +common:remote --remote_cache=grpcs://remote.buildbuddy.io +common:remote --experimental_remote_downloader=grpcs://remote.buildbuddy.io +common:remote --remote_executor=grpcs://remote.buildbuddy.io +common:remote --remote_download_toplevel +common:remote --remote_download_outputs=minimal +common:remote --nobuild_runfile_links +common:remote --remote_timeout=3600 +common:remote --experimental_remote_cache_chunking +common:remote --noexperimental_throttle_remote_action_building +common:remote --remote_cache_compression +# Re-runs actions instead of failing on transient lost remote/disk cache inputs. +common:remote --rewind_lost_inputs +# Falls back to local downloads if the remote asset downloader is unavailable. +common:remote --experimental_remote_downloader_local_fallback +common:remote --grpc_keepalive_time=30s +common:remote --jobs=2000 +common:remote --extra_execution_platforms=@zml//:rbe_platform +common:remote --experimental_repo_remote_exec=true + +# Runtime profiling helpers for `bazel run`. +run:nsys --@zml//platforms:cuda=true +run:nsys --run_under=//tools:nsys + +run:rocprofv3 --@zml//platforms:rocm=true +run:rocprofv3 --run_under=//tools:rocprofv3 + +run:neuron-profile --@zml//platforms:neuron=true +run:neuron-profile --@zml//platforms:cpu=false +run:neuron-profile --run_under='//tools/neuron:profile /tmp/zml-neuron-profile' diff --git a/.bazelrc-examples b/.bazelrc-examples deleted file mode 100644 index f96c2ab6..00000000 --- a/.bazelrc-examples +++ /dev/null @@ -1,2 +0,0 @@ -common --override_module=zml=../ -common --registry=file://%workspace%/../third_party/ diff --git a/.bazelrc-zml b/.bazelrc-zml deleted file mode 100644 index 4ffe30c1..00000000 --- a/.bazelrc-zml +++ /dev/null @@ -1,2 +0,0 @@ -common --override_module=zml=./ -common --registry=file://%workspace%/third_party/ diff --git a/.bazelversion b/.bazelversion deleted file mode 120000 index 8ee3d828..00000000 --- a/.bazelversion +++ /dev/null @@ -1 +0,0 @@ -examples/.bazelversion \ No newline at end of file diff --git a/.bazelversion b/.bazelversion new file mode 100644 index 00000000..b13d146a --- /dev/null +++ b/.bazelversion @@ -0,0 +1 @@ +9.3.0 diff --git a/examples/.bazelignore b/.editorconfig similarity index 100% rename from examples/.bazelignore rename to .editorconfig diff --git a/.github/ISSUE_TEMPLATE/llmd--bug-report.md b/.github/ISSUE_TEMPLATE/llmd--bug-report.md new file mode 100644 index 00000000..b9629993 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/llmd--bug-report.md @@ -0,0 +1,34 @@ +--- +name: 'LLMD: Bug report' +about: If you hit a bug in LLMD, let us know +title: "[LLMD]" +labels: LLMD +assignees: '' +type: Bug + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior: +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Configuration:** + - Model: [e.g. `google/gemma-4-31B-it`] + - OS: [e.g. `Ubuntu 24.04`] + - Platform: [`cuda` / `rocm` / `oneapi` / `metal` / `tpu`] + - Accelerator: [e.g. `AMD MI300X`] + - Driver version: [e.g. `595.71.05`] + - Version: [e.g. 0.1.2 or docker image `sha256@`] + - Command line: [e.g. `docker run ...`] + +**Additional context** +Add any other context about the problem here. diff --git a/.github/runs-on.yml b/.github/runs-on.yml index 384121f7..aefeccc9 100644 --- a/.github/runs-on.yml +++ b/.github/runs-on.yml @@ -3,9 +3,30 @@ images: platform: "linux" arch: "amd64" ami: "ami-04a92520784b93e73" - preinstall: | - #!/bin/bash - wget https://github.com/jbdalido/bazel-remote/releases/download/test/bazel-remote -O /usr/local/bin/bzlcache - chmod +x /usr/local/bin/bzlcache - echo "Run bazel cache" - systemd-run --property=Restart=always /usr/local/bin/bzlcache --max_size=1000 --grpc_address 127.0.0.1:15501 --dir=/tmp/cache --s3.region eu-west-3 --s3.bucket runs-on-s3bucketcache-jycgicewbssh --s3.auth_method iam_role --s3.prefix cache --s3.endpoint=s3.eu-west-3.amazonaws.com --storage_mode uncompressed + + ubuntu24-arm64: + platform: "linux" + arch: "arm64" + ami: "ami-035c36ba717c3d79f" + +runners: + large-runner-amd64: + cpu: [16] + family: ["c8", "c7"] + image: ubuntu24-amd64 + volume: 80gb + extras: ["s3-cache"] + + large-runner-arm64: + cpu: [16] + family: ["c8g", "c7g"] + image: ubuntu24-arm64 + volume: 80gb + extras: ["s3-cache"] + + small-runner-amd64: + cpu: [4] + family: ["c8", "c7"] + image: ubuntu24-amd64 + volume: 80gb + extras: ["s3-cache"] diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 00000000..31bc46e8 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,189 @@ +name: ci +on: + push: + branches: + - master + tags: + - "*" + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ !contains(github.ref, 'master')}} + +jobs: + ci: + name: ci-${{ matrix.target }}-${{ matrix.arch }} + strategy: + fail-fast: false + matrix: + include: + - target: cpu + arch: amd64 + bazel_common_opts: "" + run_tests: true + runner_labels: + - runs-on=${{ github.run_id }}-${{ github.run_attempt }}-ci-cpu-amd64 + - runner=large-runner-amd64 + + - target: neuron + arch: amd64 + bazel_common_opts: "--@zml//platforms:neuron=true --@zml//platforms:cpu=false" + run_tests: false + runner_labels: + - runs-on=${{ github.run_id }}-${{ github.run_attempt }}-ci-neuron-amd64 + - runner=large-runner-amd64 + + - target: tpu + arch: amd64 + bazel_common_opts: "--@zml//platforms:tpu=true --@zml//platforms:cpu=false" + run_tests: false + runner_labels: + - runs-on=${{ github.run_id }}-${{ github.run_attempt }}-ci-tpu-amd64 + - runner=large-runner-amd64 + + - target: cuda + arch: amd64 + bazel_common_opts: "--@zml//platforms:cuda=true --@zml//platforms:cpu=false" + run_tests: true + runner_labels: ["self-hosted", "linux", "x64", "platform=cuda"] + + - target: cuda + arch: arm64 + bazel_common_opts: "--@zml//platforms:cuda=true --@zml//platforms:cpu=false" + run_tests: false + runner_labels: + - runs-on=${{ github.run_id }}-${{ github.run_attempt }}-ci-cuda-arm64 + - runner=large-runner-arm64 + + - target: rocm + arch: amd64 + bazel_common_opts: "--@zml//platforms:rocm=true --@zml//platforms:cpu=false" + run_tests: true + runner_labels: ["self-hosted", "linux", "x64", "platform=rocm"] + + - target: oneapi + arch: amd64 + bazel_common_opts: "--@zml//platforms:oneapi=true --@zml//platforms:cpu=false" + run_tests: false + runner_labels: + - runs-on=${{ github.run_id }}-${{ github.run_attempt }}-ci-oneapi-amd64 + - runner=large-runner-amd64 + + - target: metal + arch: arm64 + bazel_common_opts: "--@zml//platforms:metal=true --@zml//platforms:cpu=false" + bazel_build_opts: "--platforms=@zml//platforms:macos_arm64" + run_tests: false + runner_labels: + - runs-on=${{ github.run_id }}-${{ github.run_attempt }}-ci-metal-arm64 + - runner=large-runner-amd64 + + - target: furiosa + arch: amd64 + bazel_common_opts: "--@zml//platforms:furiosa=true --@zml//platforms:cpu=false" + run_tests: false + runner_labels: + - runs-on=${{ github.run_id }}-${{ github.run_attempt }}-ci-furiosa-amd64 + - runner=large-runner-amd64 + + runs-on: ${{ matrix.runner_labels }} + steps: + - uses: runs-on/action@17b59092eb37ebfdbb034460892887c09100eaa3 # v2.5.0 + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Ensure self hosted cache permissions + if: ${{ contains(matrix.runner_labels, 'self-hosted') }} + run: mkdir -p /tmp/zig-cache && mkdir -p /tmp/zml && sudo chmod -R 777 /tmp/zig-cache && sudo chmod -R 777 /tmp/zml + + - name: Setup Bazel + uses: bazel-contrib/setup-bazel@104e3eea9447cdaa4b00d0fb46191a3985d26ab3 # 0.19.1 + with: + bazelisk-version: 1.29.0 + bazelisk-cache: true + cache-save: false + bazelrc: | + startup --experimental_remote_repo_contents_cache + common --verbose_failures + common --keep_going + common --test_output=streamed + common --color=yes + common --show_timestamps + common --config=remote + common --strategy=TestRunner=local + ${{ format('common:remote --remote_header=x-buildbuddy-api-key={0}', secrets.BUILDBUDDY_CI_API_KEY) || '' }} + + - name: Capture self hosted storage usage before + id: storage_before + if: ${{ contains(matrix.runner_labels, 'self-hosted') }} + continue-on-error: true + shell: bash + run: &measure-self-hosted-storage | + output_base=$(bazel info output_base) || output_base="" + bazel_size="unavailable" + if [[ -n "$output_base" ]]; then + bazel_size=$(du -sh --si -- "$output_base" | cut -f1) || bazel_size="unavailable" + fi + disks=$(df -H --local --output=source,size,used,avail,pcent,target | + awk '$1 ~ /^\/dev\// && $1 !~ /^\/dev\/loop/ && !seen[$1]++ { + printf "%s%s: %s / %s used (%s), %s free", separator, $6, $3, $2, $5, $4 + separator = "; " + } END {if (separator == "") printf "none"}') || disks="unavailable" + { + printf 'output_base=%s\n' "$output_base" + printf 'bazel_size=%s\n' "$bazel_size" + printf 'disks=%s\n' "$disks" + } >> "$GITHUB_OUTPUT" + + - run: bazel mod graph + - run: bazel query ${{ matrix.bazel_common_opts }} --output=graph //... + - run: bazel cquery ${{ matrix.bazel_common_opts }} ${{ matrix.bazel_build_opts }} //... + - run: bazel build ${{ matrix.bazel_common_opts }} ${{ matrix.bazel_build_opts }} //... + - if: ${{ matrix.run_tests }} + run: bazel test ${{ matrix.bazel_common_opts }} ${{ matrix.bazel_build_opts }} //... + + - name: Capture self hosted storage usage after + id: storage_after + if: ${{ always() && contains(matrix.runner_labels, 'self-hosted') }} + continue-on-error: true + shell: bash + run: *measure-self-hosted-storage + + - name: Report self hosted storage usage + if: ${{ always() && contains(matrix.runner_labels, 'self-hosted') }} + env: + STORAGE_JOB: ${{ github.job }}-${{ matrix.target }}-${{ matrix.arch }} + OUTPUT_BASE: ${{ steps.storage_after.outputs.output_base || steps.storage_before.outputs.output_base || 'unavailable' }} + SIZE_BEFORE: ${{ steps.storage_before.outputs.bazel_size || 'unavailable' }} + SIZE_AFTER: ${{ steps.storage_after.outputs.bazel_size || 'unavailable' }} + DISKS_BEFORE: ${{ steps.storage_before.outputs.disks || 'unavailable' }} + DISKS_AFTER: ${{ steps.storage_after.outputs.disks || 'unavailable' }} + run: | + message=$(printf 'Runner: %s\nBazel output base: %s\nBazel directory size: %s -> %s\nDisks before: %s\nDisks after: %s' \ + "$RUNNER_NAME" "$OUTPUT_BASE" "$SIZE_BEFORE" "$SIZE_AFTER" "$DISKS_BEFORE" "$DISKS_AFTER") + message=${message//'%'/'%25'} + message=${message//$'\r'/'%0D'} + message=${message//$'\n'/'%0A'} + printf '::notice title=Disk usage - %s::%s\n' "$STORAGE_JOB" "$message" + + lint: + runs-on: + - runs-on=${{ github.run_id }}-${{ github.run_attempt }}-lint + - runner=small-runner-amd64 + steps: + - uses: runs-on/action@17b59092eb37ebfdbb034460892887c09100eaa3 # v2.5.0 + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Setup Zig + uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2.2.1 + with: + version: 0.17.0 + mirror: https://mirror.zml.ai/zig + + - name: Zig Format + run: zig fmt --check $(git ls-files --exclude-standard -- ':!third_party/*' '*.zig' | xargs) + + - name: No autofix in Zig code + run: bash -c "! git grep -l '// autofix' -- '*.zig'" diff --git a/.github/workflows/commit.yaml b/.github/workflows/commit.yaml deleted file mode 100644 index 3ea52fb3..00000000 --- a/.github/workflows/commit.yaml +++ /dev/null @@ -1,44 +0,0 @@ -name: commit -on: - push: - branches: - - master - tags: - - '*' - pull_request: - -env: - ZIG_VERSION: 0.13.0 - BAZELISK_VERSION: 1.20.0 - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: ${{ !contains(github.ref, 'master')}} - -jobs: - test: - runs-on: ["runs-on","runner=4cpu-linux-x64", "image=ubuntu24-amd64", "run-id=${{ github.run_id }}"] - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Setup Zig - uses: goto-bus-stop/setup-zig@v2 - with: - version: ${{ env.ZIG_VERSION }} - - - name: Format - run: zig fmt --check *.zig zml/ examples/ async/ mlir/ pjrt/ - - - name: No Autofix - run: bash -c 'grep "// autofix" zml/**/*.zig ; [ 1 -eq $? ]' - - - name: Setup Bazel - run: | - wget https://github.com/bazelbuild/bazelisk/releases/download/v${{ env.BAZELISK_VERSION }}/bazelisk-linux-amd64 -O ~/bin/bazel - chmod +x ~/bin/bazel - - - name: Tests - run: | - export PATH=$PATH:~/bin - bazel test --remote_cache=grpc://127.0.0.1:15501 //zml:test --test_output=streamed diff --git a/.gitignore b/.gitignore index 6add214f..0ec34365 100644 --- a/.gitignore +++ b/.gitignore @@ -23,8 +23,22 @@ zig-out/ # Data files *.wav *.ecdc + +# Python slope .venv +venv # Editor specific ## Neovim .nvim.local.lua + +# Devenv +.devenv* +devenv.local.nix +devenv.local.yaml + +# direnv +.direnv + +# pre-commit +.pre-commit-config.yaml diff --git a/.helix/languages.toml b/.helix/languages.toml new file mode 100644 index 00000000..529cdcf3 --- /dev/null +++ b/.helix/languages.toml @@ -0,0 +1,19 @@ +[[language]] +auto-format = true +file-types = [ "zig" ] +language-servers = [ "zls" ] +name = "zig" +roots = [ "build.zig", "build.zig.zon", "zls.build.json" ] +scope = "source.zig" + +[language.indent] +tab-width = 4 +unit = " " + +[language-server.zls] +command = "sh" +args = [ + "-c", + "exec \"$(git rev-parse --show-toplevel)/tools/zls.sh\" \"$@\"", + "zls" +] diff --git a/.nvim.lua b/.nvim.lua deleted file mode 120000 index 2a8b1ccb..00000000 --- a/.nvim.lua +++ /dev/null @@ -1 +0,0 @@ -examples/.nvim.lua \ No newline at end of file diff --git a/.nvim.lua b/.nvim.lua new file mode 100644 index 00000000..9982de80 --- /dev/null +++ b/.nvim.lua @@ -0,0 +1,66 @@ +-- Get the current working directory (CWD) +local cwd = vim.fn.getcwd() + +-- Load the local config file if it exists +local f, err = loadfile(".nvim.local.lua") +if not err then + local ok, err_f = pcall(f) + if not ok then + vim.print(err_f) + end +else + vim.lsp.config("zls", { + cmd = { cwd .. "/tools/zls.sh" }, + }) + vim.lsp.enable("zls") + + function runFixAll() + local params = vim.lsp.util.make_range_params() + params.context = { + only = { "source.fixAll" }, + diagnostics = {}, + } + + -- Send synchronous request (timeout 2 seconds) + local responses = vim.lsp.buf_request_sync(0, "textDocument/codeAction", params, 1000) + if not responses then + return + end + + for _, resp in pairs(responses) do + for _, action in ipairs(resp.result or {}) do + -- If action has edits, apply them + if action.edit then + vim.lsp.util.apply_workspace_edit(action.edit, "utf-16") + end + -- If the action has a command, execute it + if action.command then + vim.lsp.buf.execute_command(action.command) + end + end + end + end + + vim.api.nvim_create_autocmd("BufWritePre", { + pattern = { "*.zig", "*.zon" }, + callback = runFixAll, + }) + + local ok, conform = pcall(require, "conform") + + if ok then + conform.formatters_by_ft.bzl = { "buildifier" } + conform.formatters.buildifier = { + command = cwd .. "/tools/buildifier.sh", + } + + vim.api.nvim_create_autocmd("FileType", { + pattern = "bzl", + callback = function() + vim.keymap.set("n", "f", function() + conform.format() + end, { remap = false, silent = true }) + end, + }) + end +end diff --git a/.vscode b/.vscode deleted file mode 120000 index fb3442e0..00000000 --- a/.vscode +++ /dev/null @@ -1 +0,0 @@ -examples/.vscode \ No newline at end of file diff --git a/examples/.vscode/extensions.json b/.vscode/extensions.json similarity index 100% rename from examples/.vscode/extensions.json rename to .vscode/extensions.json diff --git a/examples/.vscode/settings.json b/.vscode/settings.json similarity index 59% rename from examples/.vscode/settings.json rename to .vscode/settings.json index f7b2c3e7..772705fb 100644 --- a/examples/.vscode/settings.json +++ b/.vscode/settings.json @@ -9,19 +9,24 @@ "**/bazel-*": true }, "files.watcherExclude": { - "**/bazel-*": true, + "**/bazel-*": true }, // "files.trimTrailingWhitespace": true, "files.insertFinalNewline": true, "bazel.buildifierFixOnFormat": true, "bazel.queriesShareServer": true, - "bazel.pathsToIgnore": [ - "." - ], + "bazel.pathsToIgnore": ["."], "bazel.buildifierExecutable": "@buildifier_prebuilt//:buildifier", + "C_Cpp.formatting": "clangFormat", + "C_Cpp.clang_format_style": "Microsoft", "zig.formattingProvider": "zls", - "zig.initialSetupDone": true, - "zig.path": "${workspaceFolder}/tools/zig.sh", - "zig.zls.globalCachePath": null, + "zig.path": "x", "zig.zls.path": "${workspaceFolder}/tools/zls.sh", + "[zig]": { + "editor.suggest.insertMode": "replace", + "editor.stickyScroll.defaultModel": "foldingProviderModel", + "editor.codeActionsOnSave": { + "source.fixAll": "explicit" + } + } } diff --git a/.zed/settings.json b/.zed/settings.json new file mode 100644 index 00000000..55eff7c8 --- /dev/null +++ b/.zed/settings.json @@ -0,0 +1,41 @@ +// Folder-specific settings +// +// For a full list of overridable settings, and general information on folder-specific settings, +// see the documentation: https://zed.dev/docs/configuring-zed#settings-files +{ + "languages": { + "Zig": { + "formatter": "language_server", + "code_actions_on_format": { + "source.fixAll": true, + "source.organizeImports": true + } + }, + "YAML": { + "tab_size": 2 + } + }, + "file_scan_exclusions": [ + "**/.git", + "**/.svn", + "**/.hg", + "**/.jj", + "**/CVS", + "**/.DS_Store", + "**/Thumbs.db", + "**/.classpath", + "**/.settings", + "**/bazel-*" + ], + "tab_size": 4, + "inlay_hints": { + "enabled": true + }, + "lsp": { + "zls": { + "binary": { + "path": "tools/zls.sh" + } + } + } +} diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..fe0f5d9c --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,35 @@ +# Repository Guidelines + +## Project Structure & Module Organization + +ZML is a Zig project built with Bazel. Core library code lives in `zml/`, with IO under `zml/io/`, tokenizers under `zml/tokenizer/`, attention implementations under `zml/attention/`, and MoE support under `zml/moe/`. Shared Zig utilities are in `stdx/`; FFI and C helpers are in `ffi/` and `upb/`. Bazel rules and workspace helpers live in `bazel/` and `third_party/`. Runnable examples are in `examples/`, while user-facing documentation is in `docs/`. Keep tests close to implementation files unless a package already has a separate test file. + +## Build, Test, and Development Commands + +- `bazel build //zml`: builds the main ZML library. +- `bazel test //zml:test`: runs the core Zig test target. +- `bazel test //stdx:test`: runs shared utility tests. +- `bazel test //zml/tokenizer:test`: runs tokenizer tests. +- `bazel run //examples/mnist`: runs the quick MNIST smoke test. +- `bazel run //examples/llm -- --model=hf://... --prompt="..."`: runs the LLM example. +- `bazel build //...` and `bazel test //...`: use before broad or shared changes. +- `./tools/buildifier.sh`: formats Bazel/Starlark files. +- `echo "$(bazel info output_base)/$(bazel cquery --output=files "$(bazel cquery "filter('zig_toolchain', deps(//zml))" 2>/dev/null | cut -d' ' -f1 | head -n1)" 2>/dev/null | rg "/lib\$")/std"`: find the current Zig standard library used by Bazel. + +Append platform flags when relevant, for example `--@zml//platforms:cuda=true`, `--@zml//platforms:rocm=true`, `--@zml//platforms:tpu=true`, or `--@zml//platforms:cpu=false`. + +## Coding Style & Naming Conventions + +Follow the Zig style guide and format Zig code with `zig fmt`; CI checks `zig fmt --check` outside `third_party/`. Use four-space indentation. Prefer `const x: Foo = .{ .bar = 1 };`, `pub fn method(self: Foo)`, and module imports such as `const foo = @import("foo.zig"); foo.bar()`. Use `PascalCase` for types and `lowerCamelCase` for functions, fields, and locals. Add comments only for non-obvious invariants, ownership, tensor shape semantics, or platform behavior. + +## Testing Guidelines + +Tests are generally inline Zig `test "..." {}` blocks near the code they cover. Use descriptive test names and narrow Bazel targets first, then broaden when touching shared APIs. For tensor or graph changes, cover shape/tag inference and runtime behavior when practical. CI builds all targets and runs tests on CPU, CUDA, and ROCm where available. + +## Commit & Pull Request Guidelines + +Recent history uses scoped, imperative commit subjects such as `zml/tensor: add Tensor.onMemory()` or `workspace: use latest version of upstreamable rules_zig`. Keep subjects concise and mention the affected area first. Pull requests should include a short description, relevant linked issue, platform impact if any, and the exact `bazel build` or `bazel test` commands run. Include screenshots only for docs or UI-visible changes. + +## Quirks + +If during build, you encounter an `AccessDenied` error, the solution is not to use the `--repo_env` bazel argument to override the zig cache. The solution is to run `sudo chmod -R 777`on `/tmp/zig-cache` and `/var/tmp/zig-cache`. This is due to the fact that all users share the same zig cache directory, and bazel runs as a different user than the one who created the cache. diff --git a/BUILD.bazel b/BUILD.bazel index e69de29b..734a3792 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -0,0 +1,64 @@ +load("@@apple_support+//xcode:xcode_config.bzl", "xcode_config") +load("@rules_zig//zig/translate-c:toolchain.bzl", "translate_c_toolchain") +load("@rules_zig//zig/zls:zls_completion.bzl", "zls_completion") + +# Dummy xcode_config so we don't need to build xcode_locator in the repository rule. +xcode_config( + name = "disable_xcode", + visibility = ["//visibility:public"], +) + +alias( + name = "translate-c", + actual = "@translate-c//:translate-c", + visibility = ["//visibility:public"], +) + +translate_c_toolchain( + name = "translate_c_toolchain_impl", + runtime_modules = [ + "@translate-c//:c_builtins", + "@translate-c//:helpers", + ], + translate_c = "@translate-c//:translate-c", +) + +toolchain( + name = "translate_c_toolchain", + target_settings = ["@rules_zig//zig/config/use_standalone_translate_c:enabled"], + toolchain = ":translate_c_toolchain_impl", + toolchain_type = "@rules_zig//zig/translate-c:toolchain_type", + visibility = ["//visibility:public"], +) + +zls_completion( + name = "completion", + visibility = ["//visibility:public"], + deps = [ + "//bin/zml-smi", + "//examples/llm", + "//examples/mnist", + "//examples/sharding", + "//examples/io:playground", + "//stdx", + "//zml", + "//zml/tokenizer/iree", + ], +) + +platform( + name = "rbe_platform", + constraint_values = [ + "@platforms//cpu:x86_64", + "@platforms//os:linux", + # ubuntu:22.04 has GNU C Library (Ubuntu GLIBC 2.35-0ubuntu3.10) + # But apparently not in this docker://ubuntu:22.04 image? + "@llvm//constraints/libc:gnu.2.28", + "@bazel_tools//tools/cpp:clang", + ], + exec_properties = { + "container-image": "docker://mirror.gcr.io/library/ubuntu:22.04", + "OSFamily": "Linux", + }, + visibility = ["//visibility:public"], +) diff --git a/MODULE.bazel b/MODULE.bazel index e9a8b7c9..e9c4c9d3 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -2,78 +2,420 @@ module( name = "zml", ) -new_git_repository = use_repo_rule("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository") +bazel_dep(name = "abseil-cpp", version = "20260526.0", repo_name = "com_google_absl") -bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "hermetic_cc_toolchain", version = "3.1.0") -bazel_dep(name = "patchelf", version = "0.18.0") -bazel_dep(name = "platforms", version = "0.0.10") -bazel_dep(name = "rules_cc", version = "0.0.9") -bazel_dep(name = "rules_pkg", version = "1.0.1") -bazel_dep(name = "rules_proto", version = "6.0.2") +# Work around abseil-cpp#2071 until the BCR release includes the Clang BMI2 fix. +single_version_override( + module_name = "abseil-cpp", + patch_strip = 1, + patches = [ + "//:third_party/abseil-cpp/bmi2intrin.patch", + ], + version = "20260526.0", +) -bazel_dep(name = "buildifier_prebuilt", version = "6.4.0", dev_dependency = True) +bazel_dep(name = "bazel_lib", version = "3.7.2") +bazel_dep(name = "aspect_rules_py", version = "1.8.4") +bazel_dep(name = "bazel_skylib", version = "1.9.0") -bazel_dep(name = "aspect_bazel_lib", version = "2.8.1.1") -bazel_lib_toolchains = use_extension("@aspect_bazel_lib//lib:extensions.bzl", "toolchains", dev_dependency = True) -use_repo(bazel_lib_toolchains, "jq_toolchains") +## Needed by iree +bazel_dep(name = "libbacktrace", version = "1.0.0-20250926-7939218") +bazel_dep(name = "patchelf", version = "0.18.0.bcr.1") +bazel_dep(name = "pcre2", version = "10.45") +bazel_dep(name = "platforms", version = "1.1.0") +bazel_dep(name = "protobuf", version = "35.0", repo_name = "com_google_protobuf") +bazel_dep(name = "pybind11_bazel", version = "3.0.1") +bazel_dep(name = "re2", version = "2025-11-05", repo_name = "com_googlesource_code_re2") +bazel_dep(name = "rules_cc", version = "0.2.19") +bazel_dep(name = "rules_distroless", version = "0.8.0") -toolchains = use_extension("@hermetic_cc_toolchain//toolchain:ext.bzl", "toolchains") -use_repo(toolchains, "zig_sdk") +# Once the rules_distroless can parse unspaced apt version constraints +# Update the rules_distroless version and remove the patch +# https://github.com/bazel-contrib/rules_distroless/pull/229 +single_version_override( + module_name = "rules_distroless", + patch_strip = 1, + patches = [ + "//:third_party/rules_distroless/accept-nospace-version-constraints.patch", + "//:third_party/rules_distroless/flat-repositories.patch", + ], + version = "0.8.0", +) -bazel_dep(name = "rules_zig", version = "20240913.0-1957d05") +bazel_dep(name = "rules_license", version = "1.0.0") +bazel_dep(name = "rules_oci", version = "2.2.7") +bazel_dep(name = "rules_proto", version = "7.1.0") +bazel_dep(name = "rules_python", version = "1.9.0") +bazel_dep(name = "rules_shell", version = "0.6.1") + +# repo_name = None prevents rules_swift from eagerly auto-registering toolchains. +bazel_dep(name = "rules_swift", version = "3.6.1", repo_name = None) +bazel_dep(name = "rules_zig", version = "0.17.0") + +bazel_dep(name = "tar.bzl", version = "0.10.4") +bazel_dep(name = "llvm", version = "0.8.24") +bazel_dep(name = "libxml2", version = "2.15.3") +bazel_dep(name = "xz", version = "5.4.5.bcr.9", repo_name = "llvm_xz") +bazel_dep(name = "zlib-ng", version = "2.3.3", repo_name = "llvm_zlib") +bazel_dep(name = "zstd", version = "1.5.7.bcr.1", repo_name = "llvm_zstd") +bazel_dep(name = "with_cfg.bzl", version = "0.14.4") +bazel_dep(name = "highway", version = "1.4.0") +bazel_dep(name = "grpc", version = "1.80.0") +single_version_override( + module_name = "grpc", + patch_strip = 1, + patches = [ + "//:third_party/grpc/protobuf-35-upb-load.patch", + "//:third_party/grpc/grpc++_unsecure-visibility.patch", + "//:third_party/grpc/plugin-flags.patch", + ], + version = "1.80.0", +) + +single_version_override( + module_name = "envoy_api", + version = "0.0.0-20250128-4de3c74", +) + +# Added to support bazel 9+ since an old version is pulled by GRPC. +bazel_dep(name = "rules_foreign_cc", version = "0.15.1") + +bazel_dep(name = "buildifier_prebuilt", version = "8.5.1", dev_dependency = True) + +# BSDTar toolchains +tar_toolchains = use_extension("@tar.bzl//tar:extensions.bzl", "toolchains") +use_repo( + tar_toolchains, + "bsd_tar_toolchains_darwin_amd64", + "bsd_tar_toolchains_darwin_arm64", + "bsd_tar_toolchains_linux_amd64", + "bsd_tar_toolchains_linux_arm64", + "bsd_tar_toolchains_windows_amd64", + "bsd_tar_toolchains_windows_arm64", +) +### + +### Zig toolchain zig = use_extension("@rules_zig//zig:extensions.bzl", "zig") zig.index(file = "//bazel:zig_index.json") -zig.toolchain(zig_version = "0.14.0-dev.363+c3faae6bf") +zig.toolchain( + # Ensure Zig actions are run locally only + extra_exec_compatible_with = ["//platforms:local_execution"], + zig_version = "0.17.0", +) zig.mirrors(urls = [ - "https://mirror.zml.ai/zig", + "https://pkg.machengine.org/zig", + "https://zigmirror.hryx.net/zig", + "https://zig.linus.dev/zig", + "https://zig.squirl.dev", + "https://zig.florent.dev", + "https://zig.mirror.mschae23.de/zig", + "https://zigmirror.meox.dev", + "https://ziglang.freetls.fastly.net", + "https://zig.tilok.dev", + "https://zig-mirror.tsimnet.eu/zig", + "https://zig.karearl.com/zig", + "https://pkg.earth/zig", + "https://fs.liujiacai.net/zigbuilds", + "https://ziglang.org/builds/", ]) use_repo(zig, "zig_toolchains") register_toolchains("@rules_zig//zig/target:all") + register_toolchains("@zig_toolchains//:all") -register_toolchains( - "@zig_sdk//toolchain:linux_amd64_gnu.2.31", - "@zig_sdk//toolchain:linux_arm64_gnu.2.31", + +register_toolchains("//:translate_c_toolchain") + +# Add the local execution platform to the set of eligible execution platforms +# This is used so that Zig toolchains that are only declared as compatible with +# the constraints described by this platform be selected and matched with only +# this platform only. +# This ensures Zig actions only run locally because they're slower remotely +# due to the Zig compilation model which concentrates one action for the whole +# build graph. +register_execution_platforms("//platforms:local_execution_platform") + +register_toolchains("//third_party/zls:toolchain") + +### + +### LLVM Bootstrapped toolchains +register_toolchains("@llvm//toolchain:all") +### + +uv = use_extension("@rules_python//python/uv:uv.bzl", "uv", dev_dependency = True) +uv.configure(version = "0.9.30") + +python = use_extension("@rules_python//python/extensions:python.bzl", "python") +python.toolchain(python_version = "3.12") +use_repo(python, "python_3_12", "python_versions") + +pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip") + +pip_neuron = use_extension("@rules_python//python/extensions:pip.bzl", "pip") +pip_neuron.parse( + download_only = True, + extra_pip_args = [ + "--abi=cp312", + "--implementation=cp", + "--python-version=312", + "--platform=linux_x86_64", + "--platform=manylinux2014_x86_64", + "--platform=manylinux_2_27_x86_64", + "--platform=manylinux_2_28_x86_64", + "--platform=manylinux_2_29_x86_64", + ], + hub_name = "neuron_py_deps", + python_version = "3.12", + requirements_lock = "//platforms/neuron:requirements.lock.txt", + target_platforms = ["linux_x86_64"], +) +use_repo(pip_neuron, "neuron_py_deps") + +# For downloading models +pip.parse( + download_only = True, + hub_name = "huggingface_hub", + python_version = "3.12", + requirements_lock = "//tools/hf:requirements.lock.txt", +) +use_repo(pip, "huggingface_hub") + +pip.parse( + download_only = True, + # fsspec[gcs] -> gcsfs -> fsspec is a circular dependency, install them together. + experimental_requirement_cycles = { + "fsspec": [ + "fsspec", + "gcsfs", + ], + }, + hub_name = "xprof", + python_version = "3.12", + requirements_lock = "//tools/xprof:requirements.lock.txt", +) +use_repo(pip, "xprof") + +cpu = use_extension("//platforms/cpu:cpu.bzl", "cpu_plugin") +use_repo(cpu, "libzml_cpu_darwin_amd64", "libzml_cpu_darwin_arm64", "libzml_cpu_linux_amd64") + +cuda = use_extension("//platforms/cuda:cuda.bzl", "cuda_packages") +use_repo(cuda, "cuda_nvml_dev_linux_sbsa", "cuda_nvml_dev_linux_x86_64", "cuda_nvtx_linux_sbsa", "cuda_nvtx_linux_x86_64", "libzml_cuda_linux_amd64", "libzml_cuda_linux_arm64", "zlib1g_linux_amd64", "zlib1g_linux_arm64") + +rocm = use_extension("//platforms/rocm:rocm.bzl", "rocm_packages") +use_repo( + rocm, + "amdrocm-amdsmi", + "amdrocm-base", + "amdrocm-blas", + "amdrocm-dnn", + "amdrocm-fft", + "amdrocm-llvm", + "amdrocm-math-common", + "amdrocm-profiler-base", + "amdrocm-rccl", + "amdrocm-runtime", + "amdrocm-solver", + "amdrocm-sparse", + "amdrocm-sysdeps", + "libatomic1", + "libdrm-common", + "libzml_rocm", ) -cpu = use_extension("//runtimes/cpu:cpu.bzl", "cpu_pjrt_plugin") -use_repo(cpu, "libpjrt_cpu_darwin_arm64", "libpjrt_cpu_linux_amd64") +oneapi = use_extension("//platforms/oneapi:oneapi.bzl", "oneapi_packages") +use_repo(oneapi, "libzml_oneapi") -cuda = use_extension("//runtimes/cuda:cuda.bzl", "cuda_packages") -use_repo(cuda, "libpjrt_cuda") +furiosa = use_extension("//platforms/furiosa:furiosa.bzl", "furiosa_packages") +use_repo(furiosa, "libzml_furiosa") -rocm = use_extension("//runtimes/rocm:rocm.bzl", "rocm_packages") -use_repo(rocm, "libpjrt_rocm") +tpu = use_extension("//platforms/tpu:tpu.bzl", "tpu_packages") +use_repo(tpu, "libzml_tpu") -tpu = use_extension("//runtimes/tpu:tpu.bzl", "tpu_packages") -use_repo(tpu, "libpjrt_tpu") +neuron = use_extension("//platforms/neuron:neuron.bzl", "neuron_packages") +use_repo(neuron, "aws-neuronx-collectives", "aws-neuronx-runtime-lib", "aws-neuronx-tools", "libgcc_s1", "libgomp1", "libstdcpp6", "libzml_neuron") -zls = use_extension("//third_party/zls:zls.bzl", "repo") -use_repo(zls, "zls_aarch64-macos", "zls_x86_64-linux") +http_archive = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -register_toolchains("//third_party/zls:all") +http_archive( + name = "sevenzip_darwin_amd64", + build_file_content = """exports_files(["7zz"])""", + sha256 = "26aa75bc262bb10bf0805617b95569c3035c2c590a99f7db55c7e9607b2685e0", + urls = ["https://www.7-zip.org/a/7z2501-mac.tar.xz"], +) -bazel_dep(name = "libxev", version = "20240910.0-a2d9b31") -bazel_dep(name = "llvm-raw", version = "20240919.0-94c024a") +# Host tools for the portable AEA extraction path. +http_archive( + name = "sevenzip_darwin_arm64", + build_file_content = """exports_files(["7zz"])""", + sha256 = "26aa75bc262bb10bf0805617b95569c3035c2c590a99f7db55c7e9607b2685e0", + urls = ["https://www.7-zip.org/a/7z2501-mac.tar.xz"], +) -llvm = use_extension("@llvm-raw//utils/bazel:extension.bzl", "llvm") -llvm.configure( - targets = [ - "AArch64", - "X86", - "NVPTX", +http_archive( + name = "sevenzip_linux_amd64", + build_file_content = """exports_files(["7zz"])""", + sha256 = "4ca3b7c6f2f67866b92622818b58233dc70367be2f36b498eb0bdeaaa44b53f4", + urls = ["https://www.7-zip.org/a/7z2501-linux-x64.tar.xz"], +) + +http_archive( + name = "ipsw_linux_amd64", + build_file_content = """exports_files(["ipsw"])""", + sha256 = "923c1caa44d3adaf2b792725c6209e879a13ab68af462ae5adfe1d611e2f3653", + urls = ["https://github.com/blacktop/ipsw/releases/download/v3.1.729/ipsw_3.1.729_linux_x86_64.tar.gz"], +) + +http_archive( + name = "sevenzip_linux_arm64", + build_file_content = """exports_files(["7zz"])""", + sha256 = "39c5140f02ce4436599303c59a149f654cb1bbc47cdc105a942120d747ae040d", + urls = ["https://www.7-zip.org/a/7z2501-linux-arm64.tar.xz"], +) + +http_archive( + name = "ipsw_linux_arm64", + build_file_content = """exports_files(["ipsw"])""", + sha256 = "fdef33729a0cb1d6be0cbffe677a97fb167e6af07fda3eafd03137ce607e92b4", + urls = ["https://github.com/blacktop/ipsw/releases/download/v3.1.729/ipsw_3.1.729_linux_arm64.tar.gz"], +) + +http_aea_archive = use_repo_rule("//bazel:http_aea_archive.bzl", "http_aea_archive") + +http_aea_archive( + name = "metal_toolchain", + archive_decryption_key = "v7VbaT39JmQPp5p4N3HoVPAKcZ5vyhY+nnjMRD79z5E=", + build_file = "//platforms/metal:metal_toolchain.BUILD.bazel", + canonical_id = "com.apple.MobileAsset.MetalToolchain-17B54", + includes = ["Metal.xctoolchain/*"], + sha256 = "84f1cec3984878294777349e794f7bff0407e501a728bd600dc8605fa36abf6f", + urls = [ + "https://updates.cdn-apple.com/2025/mobileassets/022-20354/192ADD28-31EE-43C8-972D-E5BED5FB1845/com_apple_MobileAsset_MetalToolchain/637B0E26-E00A-488B-BA3B-CB50A8ECFD74.aar", ], ) + +metal = use_extension("//platforms/metal:metal.bzl", "metal_packages") +use_repo(metal, "libzml_metal") + +non_module_deps = use_extension("//:third_party/non_module_deps.bzl", "non_module_deps") +use_repo( + non_module_deps, + "arocc", + "cloud_accelerator_diagnostics", + "com_google_sentencepiece", + "cuda_tile", + "cute_ir", + "flashinfer_cutlass_moe_linux_amd64", + "flashinfer_cutlass_moe_linux_arm64", + "flydsl", + "iree", + "libvaxis", + "linenoise", + "mnist", + "org_swig_swig", + "stb", + "translate-c", + "uucode", + "xla", + "zigimg", + "zio", + "zls", + "zls_diffz", + "zls_known_folders", + "zls_lsp", +) + +xla = use_extension("//third_party/xla:xla.bzl", "xla") +use_repo( + xla, + "com_github_grpc_grpc", + "eigen_archive", + "farmhash_archive", + "farmhash_gpu_archive", + "highwayhash", + "hwloc", + "llvm-raw", + "local_config_cuda", + "local_config_remote_execution", + "local_config_rocm", + "local_config_sycl", + "local_config_tensorrt", + "ml_dtypes_py", + "python_version_repo", + "rules_ml_toolchain", + "shardy", + "snappy", + "stablehlo", + "sycl_configure", + "sycl_configure_ext", + "triton", + "tsl", +) + +llvm = use_extension("//third_party/xla:llvm.bzl", "llvm") +llvm.configure(targets = [ + "X86", + "NVPTX", + "AMDGPU", + "AArch64", +]) use_repo(llvm, "llvm-project") -bazel_dep(name = "stablehlo", version = "20240917.0-78c753a") -bazel_dep(name = "xla", version = "20240919.0-1b18dd6") +apt = use_extension("@rules_distroless//apt:extensions.bzl", "apt") +apt.install( + name = "apt_cpu", + lock = "//platforms/cpu:packages.lock.json", + manifest = "//platforms/cpu:packages.yaml", +) +use_repo(apt, "apt_cpu") -tsl = use_extension("@xla//:tsl.bzl", "tsl") -use_repo(tsl, "tsl") +apt.install( + name = "apt_cuda", + lock = "//platforms/cuda:packages.lock.json", + manifest = "//platforms/cuda:packages.yaml", +) +use_repo(apt, "apt_cuda") -bazel_dep(name = "zigcoro", version = "20240829.0-fc1db29") -bazel_dep(name = "sentencepiece", version = "20240618.0-d7ace0a") -bazel_dep(name = "zig-protobuf", version = "20240722.0-c644d11") -bazel_dep(name = "zig-yaml", version = "20240903.0-83d5fdf") +apt.install( + name = "apt_rocm", + lock = "//platforms/rocm:packages.lock.json", + manifest = "//platforms/rocm:packages.yaml", +) +use_repo(apt, "apt_rocm") + +apt.install( + name = "apt_oneapi", + lock = "//platforms/oneapi:packages.lock.json", + manifest = "//platforms/oneapi:packages.yaml", + resolve_transitive = False, +) +use_repo(apt, "apt_oneapi") + +apt.install( + name = "apt_neuron", + lock = "//platforms/neuron:packages.lock.json", + manifest = "//platforms/neuron:packages.yaml", +) +use_repo(apt, "apt_neuron") + +oci = use_extension("@rules_oci//oci:extensions.bzl", "oci") +oci.pull( + name = "distroless_cc_debian12", + digest = "sha256:eccec5274132c1be0ce5d2c8e6fe41033e64af5e987ccee9007826e4c012069d", + image = "gcr.io/distroless/cc-debian12", + platforms = ["linux/amd64"], +) +oci.pull( + name = "debian_12", + digest = "sha256:bc960ef50e6feed90686c593361df158517556ed1d2d98e5d1df3724024e0f49", + image = "mirror.gcr.io/library/debian:12", + platforms = ["linux/amd64"], +) +use_repo( + oci, + "debian_12", + "debian_12_linux_amd64", + "distroless_cc_debian12", + "distroless_cc_debian12_linux_amd64", +) diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index ee6947a7..1e746605 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -1,1305 +1,2150 @@ { - "lockFileVersion": 11, + "lockFileVersion": 28, "registryFileHashes": { - "https://bazel-registry.zml.ai/modules/abseil-cpp/20210324.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/abseil-cpp/20211102.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/abseil-cpp/20230125.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/abseil-cpp/20230802.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/abseil-cpp/20240116.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/apple_support/1.11.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/apple_support/1.5.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/aspect_bazel_lib/2.8.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_features/1.1.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_features/1.11.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_features/1.3.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_features/1.4.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_skylib/1.0.3/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_skylib/1.1.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_skylib/1.2.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_skylib/1.2.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_skylib/1.3.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_skylib/1.4.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_skylib/1.4.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_skylib/1.5.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_skylib/1.6.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_skylib/1.7.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/buildifier_prebuilt/6.4.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/buildozer/7.1.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/google_benchmark/1.8.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/googletest/1.11.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/googletest/1.14.0.bcr.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/googletest/1.14.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/hermetic_cc_toolchain/3.1.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/libpfm/4.11.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/nlohmann_json/3.6.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/patchelf/0.18.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/platforms/0.0.10/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/platforms/0.0.4/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/platforms/0.0.5/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/platforms/0.0.6/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/platforms/0.0.7/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/platforms/0.0.8/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/platforms/0.0.9/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/protobuf/21.7/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/protobuf/3.19.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/protobuf/3.19.6/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/pybind11_bazel/2.11.1.bzl.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/pybind11_bazel/2.11.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/re2/2023-09-01/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/re2/2024-02-01/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_apple/3.2.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_cc/0.0.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_cc/0.0.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_cc/0.0.6/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_cc/0.0.8/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_cc/0.0.9/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_foreign_cc/0.9.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_java/4.0.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_java/6.3.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_java/7.3.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_java/7.6.5/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_jvm_external/4.4.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_jvm_external/5.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_license/0.0.3/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_license/0.0.4/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_license/0.0.7/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_license/0.0.8/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_pkg/0.7.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_pkg/0.9.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_pkg/1.0.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_proto/4.0.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_proto/5.3.0-21.7/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_proto/6.0.0-rc1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_proto/6.0.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_python/0.10.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_python/0.22.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_python/0.25.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_python/0.29.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_python/0.31.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_python/0.4.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_swift/1.15.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/stardoc/0.5.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/stardoc/0.5.3/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/stardoc/0.6.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/zlib/1.2.11/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/zlib/1.2.12/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/zlib/1.2.13/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/zlib/1.3.1.bcr.3/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/zstd/1.5.6/MODULE.bazel": "not found", "https://bcr.bazel.build/bazel_registry.json": "8a28e4aff06ee60aed2a8c281907fb8bcbf3b753c91fb5a5c57da3215d5b3497", - "https://bcr.bazel.build/modules/abseil-cpp/20210324.2/MODULE.bazel": "7cd0312e064fde87c8d1cd79ba06c876bd23630c83466e9500321be55c96ace2", - "https://bcr.bazel.build/modules/abseil-cpp/20211102.0/MODULE.bazel": "70390338f7a5106231d20620712f7cccb659cd0e9d073d1991c038eb9fc57589", - "https://bcr.bazel.build/modules/abseil-cpp/20230125.1/MODULE.bazel": "89047429cb0207707b2dface14ba7f8df85273d484c2572755be4bab7ce9c3a0", - "https://bcr.bazel.build/modules/abseil-cpp/20230802.0/MODULE.bazel": "d253ae36a8bd9ee3c5955384096ccb6baf16a1b1e93e858370da0a3b94f77c16", - "https://bcr.bazel.build/modules/abseil-cpp/20240116.0/MODULE.bazel": "98dc378d64c12a4e4741ad3362f87fb737ee6a0886b2d90c3cdbb4d93ea3e0bf", - "https://bcr.bazel.build/modules/abseil-cpp/20240116.0/source.json": "251927429731bd37831facad38ddfc83b8f6f680f819e1d4d35e811b30479d5b", + "https://bcr.bazel.build/modules/abseil-cpp/20260526.0/MODULE.bazel": "b4a76a8d70ba08aa0253fd5e374ca5a5fa6b15aba985a7bbe4467c138954830f", + "https://bcr.bazel.build/modules/abseil-cpp/20260526.0/source.json": "a39a26fec1b7037bdd7ea6c25cc098b99ae80af10ff0b6ec8c4bfe0df694f451", + "https://bcr.bazel.build/modules/abseil-py/2.1.0/MODULE.bazel": "5ebe5bf853769c65707e5c28f216798f7a4b1042015e6a36e6d03094d94bec8a", + "https://bcr.bazel.build/modules/abseil-py/2.1.0/source.json": "0e8fc4f088ce07099c1cd6594c20c7ddbb48b4b3c0849b7d94ba94be88ff042b", "https://bcr.bazel.build/modules/apple_support/1.11.1/MODULE.bazel": "1843d7cd8a58369a444fc6000e7304425fba600ff641592161d9f15b179fb896", - "https://bcr.bazel.build/modules/apple_support/1.11.1/source.json": "c797537544f4b76d46d12f488456529d7372b6ccff249c82fca50ce0a665957f", - "https://bcr.bazel.build/modules/apple_support/1.5.0/MODULE.bazel": "50341a62efbc483e8a2a6aec30994a58749bd7b885e18dd96aa8c33031e558ef", + "https://bcr.bazel.build/modules/apple_support/1.13.0/MODULE.bazel": "7c8cdea7e031b7f9f67f0b497adf6d2c6a2675e9304ca93a9af6ed84eef5a524", + "https://bcr.bazel.build/modules/apple_support/1.15.1/MODULE.bazel": "a0556fefca0b1bb2de8567b8827518f94db6a6e7e7d632b4c48dc5f865bc7c85", + "https://bcr.bazel.build/modules/apple_support/1.17.1/MODULE.bazel": "655c922ab1209978a94ef6ca7d9d43e940cd97d9c172fb55f94d91ac53f8610b", + "https://bcr.bazel.build/modules/apple_support/1.21.0/MODULE.bazel": "ac1824ed5edf17dee2fdd4927ada30c9f8c3b520be1b5fd02a5da15bc10bff3e", + "https://bcr.bazel.build/modules/apple_support/1.21.1/MODULE.bazel": "5809fa3efab15d1f3c3c635af6974044bac8a4919c62238cce06acee8a8c11f1", + "https://bcr.bazel.build/modules/apple_support/1.22.1/MODULE.bazel": "90bd1a660590f3ceffbdf524e37483094b29352d85317060b2327fff8f3f4458", + "https://bcr.bazel.build/modules/apple_support/1.23.1/MODULE.bazel": "53763fed456a968cf919b3240427cf3a9d5481ec5466abc9d5dc51bc70087442", + "https://bcr.bazel.build/modules/apple_support/1.24.1/MODULE.bazel": "f46e8ddad60aef170ee92b2f3d00ef66c147ceafea68b6877cb45bd91737f5f8", + "https://bcr.bazel.build/modules/apple_support/1.24.2/MODULE.bazel": "0e62471818affb9f0b26f128831d5c40b074d32e6dda5a0d3852847215a41ca4", + "https://bcr.bazel.build/modules/apple_support/2.3.0/MODULE.bazel": "d48f824ae8eeea5f837eb3038cef3615075d996d3e82eb9187192c2820605a81", + "https://bcr.bazel.build/modules/apple_support/2.5.2/MODULE.bazel": "bb1b7081e390a8d3885f642cb0073dfcf8e7a36525e079fa8f5835af7079c0fe", + "https://bcr.bazel.build/modules/apple_support/2.5.4/MODULE.bazel": "964e9268b37edb64e9a0fa525e5b5e535ba56bf63cadc68375f92e69fcb734f6", + "https://bcr.bazel.build/modules/apple_support/2.8.0/MODULE.bazel": "c45f5176057092afba0be6c48a9ea02101666802f8be536947e099e94757f8ed", + "https://bcr.bazel.build/modules/apple_support/2.8.0/source.json": "b7f6d87669c206adaa7d82cc44733349c63e341b80a0e049496ebd91d78c3472", + "https://bcr.bazel.build/modules/aspect_bazel_lib/1.31.2/MODULE.bazel": "7bee702b4862612f29333590f4b658a5832d433d6f8e4395f090e8f4e85d442f", + "https://bcr.bazel.build/modules/aspect_bazel_lib/1.38.0/MODULE.bazel": "6307fec451ba9962c1c969eb516ebfe1e46528f7fa92e1c9ac8646bef4cdaa3f", + "https://bcr.bazel.build/modules/aspect_bazel_lib/1.40.3/MODULE.bazel": "668e6bcb4d957fc0e284316dba546b705c8d43c857f87119619ee83c4555b859", + "https://bcr.bazel.build/modules/aspect_bazel_lib/2.14.0/MODULE.bazel": "2b31ffcc9bdc8295b2167e07a757dbbc9ac8906e7028e5170a3708cecaac119f", + "https://bcr.bazel.build/modules/aspect_bazel_lib/2.19.3/MODULE.bazel": "253d739ba126f62a5767d832765b12b59e9f8d2bc88cc1572f4a73e46eb298ca", + "https://bcr.bazel.build/modules/aspect_bazel_lib/2.22.5/MODULE.bazel": "004ba890363d05372a97248c37205ae64b6fa31047629cd2c0895a9d0c7779e8", + "https://bcr.bazel.build/modules/aspect_bazel_lib/2.22.5/source.json": "ac2c3213df8f985785f1d0aeb7f0f73d5324e6e67d593d9b9470fb74a25d4a9b", + "https://bcr.bazel.build/modules/aspect_bazel_lib/2.7.2/MODULE.bazel": "780d1a6522b28f5edb7ea09630748720721dfe27690d65a2d33aa7509de77e07", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.8.1/MODULE.bazel": "812d2dd42f65dca362152101fbec418029cc8fd34cbad1a2fde905383d705838", + "https://bcr.bazel.build/modules/aspect_rules_js/1.33.1/MODULE.bazel": "db3e7f16e471cf6827059d03af7c21859e7a0d2bc65429a3a11f005d46fc501b", + "https://bcr.bazel.build/modules/aspect_rules_js/1.39.0/MODULE.bazel": "aece421d479e3c31dc3e5f6d49a12acc2700457c03c556650ec7a0ff23fc0d95", + "https://bcr.bazel.build/modules/aspect_rules_lint/0.12.0/MODULE.bazel": "e767c5dbfeb254ec03275a7701b5cfde2c4d2873676804bc7cb27ddff3728fed", + "https://bcr.bazel.build/modules/aspect_rules_py/1.8.4/MODULE.bazel": "4ab4a78e9d65cf15e755807f7ae504655e1771cefd7c3343ad34dbbb625b9564", + "https://bcr.bazel.build/modules/aspect_rules_py/1.8.4/source.json": "7cd80ad811f0c2b3aad3c878506ae602fce2fe53b78487ecfbcc8bcde07d220e", + "https://bcr.bazel.build/modules/aspect_tools_telemetry/0.3.3/MODULE.bazel": "37c764292861c2f70314efa9846bb6dbb44fc0308903b3285da6528305450183", + "https://bcr.bazel.build/modules/aspect_tools_telemetry/0.3.3/source.json": "605086bbc197743a0d360f7ddc550a1d4dfa0441bc807236e17170f636153348", + "https://bcr.bazel.build/modules/bazel_features/0.1.0/MODULE.bazel": "47011d645b0f949f42ee67f2e8775188a9cf4a0a1528aa2fa4952f2fd00906fd", + "https://bcr.bazel.build/modules/bazel_features/1.1.0/MODULE.bazel": "cfd42ff3b815a5f39554d97182657f8c4b9719568eb7fded2b9135f084bf760b", "https://bcr.bazel.build/modules/bazel_features/1.1.1/MODULE.bazel": "27b8c79ef57efe08efccbd9dd6ef70d61b4798320b8d3c134fd571f78963dbcd", + "https://bcr.bazel.build/modules/bazel_features/1.10.0/MODULE.bazel": "f75e8807570484a99be90abcd52b5e1f390362c258bcb73106f4544957a48101", "https://bcr.bazel.build/modules/bazel_features/1.11.0/MODULE.bazel": "f9382337dd5a474c3b7d334c2f83e50b6eaedc284253334cf823044a26de03e8", - "https://bcr.bazel.build/modules/bazel_features/1.11.0/source.json": "c9320aa53cd1c441d24bd6b716da087ad7e4ff0d9742a9884587596edfe53015", + "https://bcr.bazel.build/modules/bazel_features/1.13.0/MODULE.bazel": "c14c33c7c3c730612bdbe14ebbb5e61936b6f11322ea95a6e91cd1ba962f94df", + "https://bcr.bazel.build/modules/bazel_features/1.15.0/MODULE.bazel": "d38ff6e517149dc509406aca0db3ad1efdd890a85e049585b7234d04238e2a4d", + "https://bcr.bazel.build/modules/bazel_features/1.17.0/MODULE.bazel": "039de32d21b816b47bd42c778e0454217e9c9caac4a3cf8e15c7231ee3ddee4d", + "https://bcr.bazel.build/modules/bazel_features/1.18.0/MODULE.bazel": "1be0ae2557ab3a72a57aeb31b29be347bcdc5d2b1eb1e70f39e3851a7e97041a", + "https://bcr.bazel.build/modules/bazel_features/1.19.0/MODULE.bazel": "59adcdf28230d220f0067b1f435b8537dd033bfff8db21335ef9217919c7fb58", + "https://bcr.bazel.build/modules/bazel_features/1.20.0/MODULE.bazel": "8b85300b9c8594752e0721a37210e34879d23adc219ed9dc8f4104a4a1750920", + "https://bcr.bazel.build/modules/bazel_features/1.21.0/MODULE.bazel": "675642261665d8eea09989aa3b8afb5c37627f1be178382c320d1b46afba5e3b", + "https://bcr.bazel.build/modules/bazel_features/1.23.0/MODULE.bazel": "fd1ac84bc4e97a5a0816b7fd7d4d4f6d837b0047cf4cbd81652d616af3a6591a", + "https://bcr.bazel.build/modules/bazel_features/1.24.0/MODULE.bazel": "4796b4c25b47053e9bbffa792b3792d07e228ff66cd0405faef56a978708acd4", + "https://bcr.bazel.build/modules/bazel_features/1.27.0/MODULE.bazel": "621eeee06c4458a9121d1f104efb80f39d34deff4984e778359c60eaf1a8cb65", + "https://bcr.bazel.build/modules/bazel_features/1.28.0/MODULE.bazel": "4b4200e6cbf8fa335b2c3f43e1d6ef3e240319c33d43d60cc0fbd4b87ece299d", "https://bcr.bazel.build/modules/bazel_features/1.3.0/MODULE.bazel": "cdcafe83ec318cda34e02948e81d790aab8df7a929cec6f6969f13a489ccecd9", + "https://bcr.bazel.build/modules/bazel_features/1.30.0/MODULE.bazel": "a14b62d05969a293b80257e72e597c2da7f717e1e69fa8b339703ed6731bec87", + "https://bcr.bazel.build/modules/bazel_features/1.32.0/MODULE.bazel": "095d67022a58cb20f7e20e1aefecfa65257a222c18a938e2914fd257b5f1ccdc", + "https://bcr.bazel.build/modules/bazel_features/1.33.0/MODULE.bazel": "8b8dc9d2a4c88609409c3191165bccec0e4cb044cd7a72ccbe826583303459f6", + "https://bcr.bazel.build/modules/bazel_features/1.34.0/MODULE.bazel": "e8475ad7c8965542e0c7aac8af68eb48c4af904be3d614b6aa6274c092c2ea1e", + "https://bcr.bazel.build/modules/bazel_features/1.36.0/MODULE.bazel": "596cb62090b039caf1cad1d52a8bc35cf188ca9a4e279a828005e7ee49a1bec3", + "https://bcr.bazel.build/modules/bazel_features/1.38.0/MODULE.bazel": "f9b8a9c890ebd216b4049fd12a31d3c2602e3403c7af636b04fbbd7453edc9c9", + "https://bcr.bazel.build/modules/bazel_features/1.39.0/MODULE.bazel": "28739425c1fc283c91931619749c832b555e60bcd1010b40d8441ce0a5cf726d", "https://bcr.bazel.build/modules/bazel_features/1.4.1/MODULE.bazel": "e45b6bb2350aff3e442ae1111c555e27eac1d915e77775f6fdc4b351b758b5d7", + "https://bcr.bazel.build/modules/bazel_features/1.42.0/MODULE.bazel": "e8ca15cb2639c5f12183db6dcb678735555d0cdd739b32a0418b6532b5e565f8", + "https://bcr.bazel.build/modules/bazel_features/1.42.1/MODULE.bazel": "275a59b5406ff18c01739860aa70ad7ccb3cfb474579411decca11c93b951080", + "https://bcr.bazel.build/modules/bazel_features/1.43.0/MODULE.bazel": "defa2226f06ba20550d6548c3a2ea2a7929634437a52973869c20c225450eb91", + "https://bcr.bazel.build/modules/bazel_features/1.47.0/MODULE.bazel": "e34df3cb35b1684cfa69923a61ae3803595babd3942cd306a488d51400886b30", + "https://bcr.bazel.build/modules/bazel_features/1.50.0/MODULE.bazel": "2083ef9c7a469f520890483ccf8e0189d6e71e2117e7752e15e6554433d5ae3e", + "https://bcr.bazel.build/modules/bazel_features/1.50.0/source.json": "e0ee3debde2789ff56e4452e612d126925ba9ab64d4bde79c67f099d2902df9b", + "https://bcr.bazel.build/modules/bazel_features/1.9.0/MODULE.bazel": "885151d58d90d8d9c811eb75e3288c11f850e1d6b481a8c9f766adee4712358b", + "https://bcr.bazel.build/modules/bazel_features/1.9.1/MODULE.bazel": "8f679097876a9b609ad1f60249c49d68bfab783dd9be012faf9d82547b14815a", + "https://bcr.bazel.build/modules/bazel_lib/3.0.0-beta.1/MODULE.bazel": "407729e232f611c3270005b016b437005daa7b1505826798ea584169a476e878", + "https://bcr.bazel.build/modules/bazel_lib/3.0.0-rc.0/MODULE.bazel": "d6e00979a98ac14ada5e31c8794708b41434d461e7e7ca39b59b765e6d233b18", + "https://bcr.bazel.build/modules/bazel_lib/3.0.0/MODULE.bazel": "22b70b80ac89ad3f3772526cd9feee2fa412c2b01933fea7ed13238a448d370d", + "https://bcr.bazel.build/modules/bazel_lib/3.1.0/MODULE.bazel": "6809765c14e3c766a9b9286c7b0ec56ed87a73326e48fe01749f0c0fdcfe3287", + "https://bcr.bazel.build/modules/bazel_lib/3.2.2/MODULE.bazel": "e2c890c8a515d6bca9c66d47718aa9e44b458fde64ec7204b8030bf2d349058c", + "https://bcr.bazel.build/modules/bazel_lib/3.7.2/MODULE.bazel": "cbc920fff00a55b2fd663d40850560a6fb17fd5ef3a9f3d6cbf0d32908dd8a40", + "https://bcr.bazel.build/modules/bazel_lib/3.7.2/source.json": "7a9c2b06a113c4962d329bb68befdb90d0447788a19aabaa32b476a0458e7d28", "https://bcr.bazel.build/modules/bazel_skylib/1.0.3/MODULE.bazel": "bcb0fd896384802d1ad283b4e4eb4d718eebd8cb820b0a2c3a347fb971afd9d8", "https://bcr.bazel.build/modules/bazel_skylib/1.1.1/MODULE.bazel": "1add3e7d93ff2e6998f9e118022c84d163917d912f5afafb3058e3d2f1545b5e", "https://bcr.bazel.build/modules/bazel_skylib/1.2.0/MODULE.bazel": "44fe84260e454ed94ad326352a698422dbe372b21a1ac9f3eab76eb531223686", "https://bcr.bazel.build/modules/bazel_skylib/1.2.1/MODULE.bazel": "f35baf9da0efe45fa3da1696ae906eea3d615ad41e2e3def4aeb4e8bc0ef9a7a", "https://bcr.bazel.build/modules/bazel_skylib/1.3.0/MODULE.bazel": "20228b92868bf5cfc41bda7afc8a8ba2a543201851de39d990ec957b513579c5", + "https://bcr.bazel.build/modules/bazel_skylib/1.4.0/MODULE.bazel": "2ab127ef8d56a739a99bb2ce00ec4c7d1ecc7977d4370c0ca6efd0d8f03d6d99", "https://bcr.bazel.build/modules/bazel_skylib/1.4.1/MODULE.bazel": "a0dcb779424be33100dcae821e9e27e4f2901d9dfd5333efe5ac6a8d7ab75e1d", "https://bcr.bazel.build/modules/bazel_skylib/1.4.2/MODULE.bazel": "3bd40978e7a1fac911d5989e6b09d8f64921865a45822d8b09e815eaa726a651", "https://bcr.bazel.build/modules/bazel_skylib/1.5.0/MODULE.bazel": "32880f5e2945ce6a03d1fbd588e9198c0a959bb42297b2cfaf1685b7bc32e138", "https://bcr.bazel.build/modules/bazel_skylib/1.6.1/MODULE.bazel": "8fdee2dbaace6c252131c00e1de4b165dc65af02ea278476187765e1a617b917", + "https://bcr.bazel.build/modules/bazel_skylib/1.7.0/MODULE.bazel": "0db596f4563de7938de764cc8deeabec291f55e8ec15299718b93c4423e9796d", "https://bcr.bazel.build/modules/bazel_skylib/1.7.1/MODULE.bazel": "3120d80c5861aa616222ec015332e5f8d3171e062e3e804a2a0253e1be26e59b", - "https://bcr.bazel.build/modules/bazel_skylib/1.7.1/source.json": "f121b43eeefc7c29efbd51b83d08631e2347297c95aac9764a701f2a6a2bb953", - "https://bcr.bazel.build/modules/buildifier_prebuilt/6.4.0/MODULE.bazel": "37389c6b5a40c59410b4226d3bb54b08637f393d66e2fa57925c6fcf68e64bf4", - "https://bcr.bazel.build/modules/buildifier_prebuilt/6.4.0/source.json": "83eb01b197ed0b392f797860c9da5ed1bf95f4d0ded994d694a3d44731275916", - "https://bcr.bazel.build/modules/buildozer/7.1.2/MODULE.bazel": "2e8dd40ede9c454042645fd8d8d0cd1527966aa5c919de86661e62953cd73d84", - "https://bcr.bazel.build/modules/buildozer/7.1.2/source.json": "c9028a501d2db85793a6996205c8de120944f50a0d570438fcae0457a5f9d1f8", - "https://bcr.bazel.build/modules/google_benchmark/1.8.2/MODULE.bazel": "a70cf1bba851000ba93b58ae2f6d76490a9feb74192e57ab8e8ff13c34ec50cb", + "https://bcr.bazel.build/modules/bazel_skylib/1.8.0/MODULE.bazel": "2fb3fb53675f6adfc1ca5bfbd5cfb655ae350fba4706d924a8ec7e3ba945671c", + "https://bcr.bazel.build/modules/bazel_skylib/1.8.1/MODULE.bazel": "88ade7293becda963e0e3ea33e7d54d3425127e0a326e0d17da085a5f1f03ff6", + "https://bcr.bazel.build/modules/bazel_skylib/1.8.2/MODULE.bazel": "69ad6927098316848b34a9142bcc975e018ba27f08c4ff403f50c1b6e646ca67", + "https://bcr.bazel.build/modules/bazel_skylib/1.9.0/MODULE.bazel": "72997b29dfd95c3fa0d0c48322d05590418edef451f8db8db5509c57875fb4b7", + "https://bcr.bazel.build/modules/bazel_skylib/1.9.0/source.json": "7ad77c1e8c1b84222d9b3f3cae016a76639435744c19330b0b37c0a3c9da7dc0", + "https://bcr.bazel.build/modules/bazel_worker_api/0.0.8/MODULE.bazel": "396c1ef53835aafe3d42ce6619080531ee770648303731f16cfaa33fa056bf0c", + "https://bcr.bazel.build/modules/bazel_worker_api/0.0.8/source.json": "abaf8ac9d2ab2f47bda9af4c0c080ff7907378888e1f4bc62a0539dd13ba61e8", + "https://bcr.bazel.build/modules/bazel_worker_java/0.0.8/MODULE.bazel": "e76479eae70bd4e8f5f4c2dfc5d03ab971cfb18750246c7b3f3454c5c2ee6629", + "https://bcr.bazel.build/modules/bazel_worker_java/0.0.8/source.json": "9395c4679444bc47bf7e51a710366a4480aa371c6f6bed01868e2fabcf11acec", + "https://bcr.bazel.build/modules/boringssl/0.0.0-20230215-5c22014/MODULE.bazel": "4b03dc0d04375fa0271174badcd202ed249870c8e895b26664fd7298abea7282", + "https://bcr.bazel.build/modules/boringssl/0.0.0-20240530-2db0eb3/MODULE.bazel": "d0405b762c5e87cd445b7015f2b8da5400ef9a8dbca0bfefa6c1cea79d528a97", + "https://bcr.bazel.build/modules/boringssl/0.20240913.0/MODULE.bazel": "fcaa7503a5213290831a91ed1eb538551cf11ac0bc3a6ad92d0fef92c5bd25fb", + "https://bcr.bazel.build/modules/boringssl/0.20241024.0/MODULE.bazel": "b540cff73d948cb79cb0bc108d7cef391d2098a25adabfda5043e4ef548dbc87", + "https://bcr.bazel.build/modules/boringssl/0.20241024.0/source.json": "d843092e682b84188c043ac742965d7f96e04c846c7e338187e03238674909a9", + "https://bcr.bazel.build/modules/buildifier_prebuilt/8.5.1/MODULE.bazel": "77f2a1958d1d07376dd3ce3ae16540f2c1b01921c1fd21930827271260e75a66", + "https://bcr.bazel.build/modules/buildifier_prebuilt/8.5.1/source.json": "ae9f3d9dc7bec033976cf47165a78788bebad2b5c272241063ae31bad8664fd4", + "https://bcr.bazel.build/modules/buildozer/8.5.1/MODULE.bazel": "a35d9561b3fc5b18797c330793e99e3b834a473d5fbd3d7d7634aafc9bdb6f8f", + "https://bcr.bazel.build/modules/buildozer/8.5.1/source.json": "e3386e6ff4529f2442800dee47ad28d3e6487f36a1f75ae39ae56c70f0cd2fbd", + "https://bcr.bazel.build/modules/c-ares/1.19.1/MODULE.bazel": "73bca21720772370ff91cc8e88bbbaf14897720c6473e87c1ddc0f848284c313", + "https://bcr.bazel.build/modules/c-ares/1.19.1/source.json": "56bfa95b01e4e0012e90eaa9f1bab823c48c3af4454286b889b9cc74f5098da1", + "https://bcr.bazel.build/modules/cel-spec/0.15.0/MODULE.bazel": "e1eed53d233acbdcf024b4b0bc1528116d92c29713251b5154078ab1348cb600", + "https://bcr.bazel.build/modules/cel-spec/0.24.0/MODULE.bazel": "e310c7aff8490ed689ccafd32729b77a660b9547f5a5ba9b20e967011c324b36", + "https://bcr.bazel.build/modules/cel-spec/0.24.0/source.json": "522d08bc22524e07863276dd0f038f446a83166e91281dcfc07d5b8433c8d89e", + "https://bcr.bazel.build/modules/civetweb/1.16/MODULE.bazel": "46a38f9daeb57392e3827fce7d40926be0c802bd23cdd6bfd3a96c804de42fae", + "https://bcr.bazel.build/modules/civetweb/1.16/source.json": "ba8b9585adb8355cb51b999d57172fd05e7a762c56b8d4bac6db42c99de3beb7", + "https://bcr.bazel.build/modules/curl/8.7.1/MODULE.bazel": "088221c35a2939c555e6e47cb31a81c15f8b59f4daa8009b1e9271a502d33485", + "https://bcr.bazel.build/modules/curl/8.8.0/MODULE.bazel": "7da3b3e79b0b4ee8f8c95d640bc6ad7b430ce66ef6e9c9d2bc29b3b5ef85f6fe", + "https://bcr.bazel.build/modules/curl/8.8.0/source.json": "d7d138b6878cf38891692fee0649ace35357fd549b425614d571786f054374d4", + "https://bcr.bazel.build/modules/cython/3.0.11-1/MODULE.bazel": "868b3f5c956c3657420d2302004c6bb92606bfa47e314bab7f2ba0630c7c966c", + "https://bcr.bazel.build/modules/cython/3.0.11-1/source.json": "da318be900b8ca9c3d1018839d3bebc5a8e1645620d0848fa2c696d4ecf7c296", + "https://bcr.bazel.build/modules/envoy_api/0.0.0-20250128-4de3c74/MODULE.bazel": "1fe72489212c530086e3ffb0e018b2bfef4663200ca03571570f9f006bef1d75", + "https://bcr.bazel.build/modules/envoy_api/0.0.0-20250128-4de3c74/source.json": "028519164a2e24563f4b43d810fdedc702daed90e71e7042d45ba82ad807b46f", + "https://bcr.bazel.build/modules/gawk/5.3.2.bcr.1/MODULE.bazel": "cdf8cbe5ee750db04b78878c9633cc76e80dcf4416cbe982ac3a9222f80713c8", + "https://bcr.bazel.build/modules/gawk/5.3.2.bcr.3/MODULE.bazel": "f1b7bb2dd53e8f2ef984b39485ec8a44e9076dda5c4b8efd2fb4c6a6e856a31d", + "https://bcr.bazel.build/modules/gawk/5.3.2.bcr.3/source.json": "ebe931bfe362e4b41e59ee00a528db6074157ff2ced92eb9e970acab2e1089c9", + "https://bcr.bazel.build/modules/gazelle/0.27.0/MODULE.bazel": "3446abd608295de6d90b4a8a118ed64a9ce11dcb3dda2dc3290a22056bd20996", + "https://bcr.bazel.build/modules/gazelle/0.30.0/MODULE.bazel": "f888a1effe338491f35f0e0e85003b47bb9d8295ccba73c37e07702d8d31c65b", + "https://bcr.bazel.build/modules/gazelle/0.32.0/MODULE.bazel": "b499f58a5d0d3537f3cf5b76d8ada18242f64ec474d8391247438bf04f58c7b8", + "https://bcr.bazel.build/modules/gazelle/0.33.0/MODULE.bazel": "a13a0f279b462b784fb8dd52a4074526c4a2afe70e114c7d09066097a46b3350", + "https://bcr.bazel.build/modules/gazelle/0.34.0/MODULE.bazel": "abdd8ce4d70978933209db92e436deb3a8b737859e9354fb5fd11fb5c2004c8a", + "https://bcr.bazel.build/modules/gazelle/0.36.0/MODULE.bazel": "e375d5d6e9a6ca59b0cb38b0540bc9a05b6aa926d322f2de268ad267a2ee74c0", + "https://bcr.bazel.build/modules/gazelle/0.37.0/MODULE.bazel": "d1327ba0907d0275ed5103bfbbb13518f6c04955b402213319d0d6c0ce9839d4", + "https://bcr.bazel.build/modules/gazelle/0.39.1/MODULE.bazel": "1fa3fefad240e535066fd0e6950dfccd627d36dc699ee0034645e51dbde3980f", + "https://bcr.bazel.build/modules/gazelle/0.45.0/MODULE.bazel": "ecd19ebe9f8e024e1ccffb6d997cc893a974bcc581f1ae08f386bdd448b10687", + "https://bcr.bazel.build/modules/gazelle/0.47.0/MODULE.bazel": "b61bb007c4efad134aa30ee7f4a8e2a39b22aa5685f005edaa022fbd1de43ebc", + "https://bcr.bazel.build/modules/gazelle/0.51.3/MODULE.bazel": "618a729142f66de1e2cb776d026413763be5b80d5e3d29ffb9d3d90c5defde90", + "https://bcr.bazel.build/modules/gazelle/0.51.3/source.json": "fbe5312a01fb4a2a58caff4a0d40dcf384b6f0bda75e85546663360da1d7538a", + "https://bcr.bazel.build/modules/google_benchmark/1.8.4/MODULE.bazel": "c6d54a11dcf64ee63545f42561eda3fd94c1b5f5ebe1357011de63ae33739d5e", + "https://bcr.bazel.build/modules/google_benchmark/1.8.4/source.json": "84590f7bc5a1fd99e1ef274ee16bb41c214f705e62847b42e705010dfa81fe53", + "https://bcr.bazel.build/modules/googleapis-cc/1.0.0/MODULE.bazel": "cf01757e7590c56140a4b81638ff2b3e7074769e6271720bbf738fcda25b6fc2", + "https://bcr.bazel.build/modules/googleapis-cc/1.0.0/source.json": "ab0e3a2ee9968a8848f59872fbbfa3e1f768597d71d2229e6caa319d357967c7", + "https://bcr.bazel.build/modules/googleapis-go/1.0.0/MODULE.bazel": "0a207a4c49da28c5cc1f7b3aeb23c2f7828c85c14aa8d9db0e30357a8d2250ed", + "https://bcr.bazel.build/modules/googleapis-go/1.0.0/source.json": "ef189be4e7853e1ebc6123fe20b71822bf9896bd1f8eed8f68505c4585f72a48", + "https://bcr.bazel.build/modules/googleapis-grpc-cc/1.0.0/MODULE.bazel": "3553358a9d8d96026c9e28d9fb6c268574950d0be7fa9b4c0aeaf3c37c73f2d3", + "https://bcr.bazel.build/modules/googleapis-grpc-cc/1.0.0/source.json": "fa7b79043b3c82bf74f1f2fa45af289e19b247375868d0752db2c114a1c7366c", + "https://bcr.bazel.build/modules/googleapis-java/1.0.0/MODULE.bazel": "d633989337d069b5a95e6101777319681d7a4af4677e36801f11839d6512095c", + "https://bcr.bazel.build/modules/googleapis-java/1.0.0/source.json": "ee59e2de37e4b531172870ac0296afa38f1ea004105ee21b2793c31a9d0ddccd", + "https://bcr.bazel.build/modules/googleapis-python/1.0.0/MODULE.bazel": "0ccd1614a914fb524b3ac267f9c97f9a5cd5412b027f0176b81a725882ec42ff", + "https://bcr.bazel.build/modules/googleapis-python/1.0.0/source.json": "24364f075ec5e6d5e0cfc5d651bd833b119e0cea1a45d51330c93461e3586e42", + "https://bcr.bazel.build/modules/googleapis-rules-registry/1.0.0/MODULE.bazel": "97c6a4d413b373d4cc97065da3de1b2166e22cbbb5f4cc9f05760bfa83619e24", + "https://bcr.bazel.build/modules/googleapis-rules-registry/1.0.0/source.json": "cf611c836a60e98e2e2ab2de8004f119e9f06878dcf4ea2d95a437b1b7a89fe9", + "https://bcr.bazel.build/modules/googleapis/0.0.0-20240326-1c8d509c5/MODULE.bazel": "a4b7e46393c1cdcc5a00e6f85524467c48c565256b22b5fae20f84ab4a999a68", + "https://bcr.bazel.build/modules/googleapis/0.0.0-20240819-fe8ba054a/MODULE.bazel": "117b7c7be7327ed5d6c482274533f2dbd78631313f607094d4625c28203cacdf", + "https://bcr.bazel.build/modules/googleapis/0.0.0-20241220-5e258e33.bcr.1/MODULE.bazel": "ee6c30f82ecd476e61f019fb1151aaab380ea419958ff274ef2f0efca7969f5c", + "https://bcr.bazel.build/modules/googleapis/0.0.0-20251003-2193a2bf/MODULE.bazel": "cc9e5ed294ed9ebf42cdbbdddd2df29048519e3797004df1e3f369f31ff4f2d4", + "https://bcr.bazel.build/modules/googleapis/0.0.0-20251003-2193a2bf/source.json": "21558a194c519e27262cca9cf031bb166a666a8b7fb89993b700c828f8dc0857", "https://bcr.bazel.build/modules/googletest/1.11.0/MODULE.bazel": "3a83f095183f66345ca86aa13c58b59f9f94a2f81999c093d4eeaa2d262d12f4", "https://bcr.bazel.build/modules/googletest/1.14.0.bcr.1/MODULE.bazel": "22c31a561553727960057361aa33bf20fb2e98584bc4fec007906e27053f80c6", - "https://bcr.bazel.build/modules/googletest/1.14.0.bcr.1/source.json": "41e9e129f80d8c8bf103a7acc337b76e54fad1214ac0a7084bf24f4cd924b8b4", - "https://bcr.bazel.build/modules/googletest/1.14.0/MODULE.bazel": "cfbcbf3e6eac06ef9d85900f64424708cc08687d1b527f0ef65aa7517af8118f", - "https://bcr.bazel.build/modules/hermetic_cc_toolchain/3.1.0/MODULE.bazel": "ea4b3a25a9417a7db57a8a2f9ebdee91d679823c6274b482b817ed128d81c594", - "https://bcr.bazel.build/modules/hermetic_cc_toolchain/3.1.0/source.json": "9d1df0459caefdf41052d360469922a73e219f67c8ce4da0628cc604469822b9", + "https://bcr.bazel.build/modules/googletest/1.15.2/MODULE.bazel": "6de1edc1d26cafb0ea1a6ab3f4d4192d91a312fd2d360b63adaa213cd00b2108", + "https://bcr.bazel.build/modules/googletest/1.17.0.bcr.2/MODULE.bazel": "827f54f492a3ce549c940106d73de332c2b30cebd0c20c0bc5d786aba7f116cb", + "https://bcr.bazel.build/modules/googletest/1.17.0.bcr.2/source.json": "3664514073a819992320ffbce5825e4238459df344d8b01748af2208f8d2e1eb", + "https://bcr.bazel.build/modules/googletest/1.17.0/MODULE.bazel": "dbec758171594a705933a29fcf69293d2468c49ec1f2ebca65c36f504d72df46", + "https://bcr.bazel.build/modules/grpc-java/1.62.2/MODULE.bazel": "99b8771e8c7cacb130170fed2a10c9e8fed26334a93e73b42d2953250885a158", + "https://bcr.bazel.build/modules/grpc-java/1.66.0/MODULE.bazel": "86ff26209fac846adb89db11f3714b3dc0090fb2fb81575673cc74880cda4e7e", + "https://bcr.bazel.build/modules/grpc-proto/0.0.0-20240627-ec30f58/MODULE.bazel": "88de79051e668a04726e9ea94a481ec6f1692086735fd6f488ab908b3b909238", + "https://bcr.bazel.build/modules/grpc/1.80.0/MODULE.bazel": "35ef253a0a31055c8a2495640516224647545b51673b0ccb956b116ee92ef554", + "https://bcr.bazel.build/modules/grpc/1.80.0/source.json": "55bf4cdcf278a450185ff93bef843a0d7db0a83d0b9761d9c1dee91faa9cdf46", + "https://bcr.bazel.build/modules/highway/1.4.0/MODULE.bazel": "22d48d947bda0c685d6174e62f9b6690089fefb87dfff4943cd420410ae82f68", + "https://bcr.bazel.build/modules/highway/1.4.0/source.json": "ae87bf2499d4902a15d760e14b7a1e94dc5dd0564bec256b9b0ed46c0e835f89", + "https://bcr.bazel.build/modules/jq.bzl/0.1.0/MODULE.bazel": "2ce69b1af49952cd4121a9c3055faa679e748ce774c7f1fda9657f936cae902f", + "https://bcr.bazel.build/modules/jq.bzl/0.1.0/source.json": "746bf13cac0860f091df5e4911d0c593971cd8796b5ad4e809b2f8e133eee3d5", + "https://bcr.bazel.build/modules/jsoncpp/1.9.5/MODULE.bazel": "31271aedc59e815656f5736f282bb7509a97c7ecb43e927ac1a37966e0578075", + "https://bcr.bazel.build/modules/jsoncpp/1.9.6.bcr.2/MODULE.bazel": "64f508885f907ac2518039b3d0c5c1703a8f6137d9f5d635067ba93d33c2670a", + "https://bcr.bazel.build/modules/jsoncpp/1.9.6.bcr.2/source.json": "13199fa0a267ca46814e9e6ab313a71890c8f1822e57376fdc23a2d2cd384051", + "https://bcr.bazel.build/modules/jsoncpp/1.9.6/MODULE.bazel": "2f8d20d3b7d54143213c4dfc3d98225c42de7d666011528dc8fe91591e2e17b0", + "https://bcr.bazel.build/modules/libbacktrace/1.0.0-20250926-7939218/MODULE.bazel": "4cd5b9355f1c2533c30f01904b71ac1b1a9bf61c43c81ba9972a5bf8f216476b", + "https://bcr.bazel.build/modules/libbacktrace/1.0.0-20250926-7939218/source.json": "83239989d09748f63dee4bdd430d261e217a012670e066e2238925a303a65d89", "https://bcr.bazel.build/modules/libpfm/4.11.0/MODULE.bazel": "45061ff025b301940f1e30d2c16bea596c25b176c8b6b3087e92615adbd52902", + "https://bcr.bazel.build/modules/libpfm/4.11.0/source.json": "caaffb3ac2b59b8aac456917a4ecf3167d40478ee79f15ab7a877ec9273937c9", + "https://bcr.bazel.build/modules/libxml2/2.15.3/MODULE.bazel": "a6bffafd8aa8b0c13eaa4bf25d55be188ec8548549395f03e98c39bab264adaa", + "https://bcr.bazel.build/modules/libxml2/2.15.3/source.json": "fd7c420cb14981416c9fadd0db555bd1e249355e0e5b63b2ce15b92996b5fbfb", + "https://bcr.bazel.build/modules/llvm/0.8.24/MODULE.bazel": "e03e030b1d0b44bb05c52665c2f43de9260af9152f666262e11e25f6c072aae3", + "https://bcr.bazel.build/modules/llvm/0.8.24/source.json": "924ed48858c1c081698f1992c7b489434375de901c47b7046e203459840cb544", + "https://bcr.bazel.build/modules/mbedtls/3.6.0/MODULE.bazel": "8e380e4698107c5f8766264d4df92e36766248447858db28187151d884995a09", + "https://bcr.bazel.build/modules/mbedtls/3.6.0/source.json": "1dbe7eb5258050afcc3806b9d43050f71c6f539ce0175535c670df606790b30c", + "https://bcr.bazel.build/modules/nlohmann_json/3.11.3/MODULE.bazel": "87023db2f55fc3a9949c7b08dc711fae4d4be339a80a99d04453c4bb3998eefc", + "https://bcr.bazel.build/modules/nlohmann_json/3.12.0.bcr.1/MODULE.bazel": "a1c8bb07b5b91d971727c635f449d05623ac9608f6fe4f5f04254ea12f08e349", + "https://bcr.bazel.build/modules/nlohmann_json/3.12.0.bcr.1/source.json": "93f82a5ae985eb935c539bfee95e04767187818189241ac956f3ccadbdb8fb02", "https://bcr.bazel.build/modules/nlohmann_json/3.6.1/MODULE.bazel": "6f7b417dcc794d9add9e556673ad25cb3ba835224290f4f848f8e2db1e1fca74", - "https://bcr.bazel.build/modules/nlohmann_json/3.6.1/source.json": "f448c6e8963fdfa7eb831457df83ad63d3d6355018f6574fb017e8169deb43a9", - "https://bcr.bazel.build/modules/patchelf/0.18.0/MODULE.bazel": "15a6beff7e828d585c5bd0f9f93589df117b5594e9d19e43096c77de58b9ae5f", - "https://bcr.bazel.build/modules/patchelf/0.18.0/source.json": "57caf6bcaa5ba515c6fb1c2eacee00735afbeb1ffacb34a57553fb139c8e4333", + "https://bcr.bazel.build/modules/opencensus-cpp/0.0.0-20230502-50eb5de.bcr.2/MODULE.bazel": "cc18734138dd18c912c6ce2a59186db28f85d8058c99c9f21b46ca3e0aba0ebe", + "https://bcr.bazel.build/modules/opencensus-cpp/0.0.0-20230502-50eb5de.bcr.2/source.json": "7c135f9d42bb3b045669c3c6ab3bb3c208e00b46aca4422eea64c29811a5b240", + "https://bcr.bazel.build/modules/opencensus-proto/0.4.1.bcr.2/MODULE.bazel": "789706a714855f92c5c8cfcf1ef32bbb64dcd3b7c9066756ad7986ec59709d29", + "https://bcr.bazel.build/modules/opencensus-proto/0.4.1.bcr.2/source.json": "aadf3f53e08b72376506b7c4ea3d167010c9efb160d7d6e1e304ed646bac1b36", + "https://bcr.bazel.build/modules/opencensus-proto/0.4.1/MODULE.bazel": "4a2e8b4d0b544002502474d611a5a183aa282251e14f6a01afe841c0c1b10372", + "https://bcr.bazel.build/modules/openssl/3.3.1.bcr.1/MODULE.bazel": "49c0c07e8fb87b480bccb842cfee1b32617f11dac590f732573c69058699a3d1", + "https://bcr.bazel.build/modules/openssl/3.3.1.bcr.1/source.json": "0c0872e048bbea052a9c541fb47019481a19201ba5555a71d762ad591bf94e1f", + "https://bcr.bazel.build/modules/opentelemetry-cpp/1.19.0/MODULE.bazel": "3455326c08b28415648a3d60d8e3c811847ebdbe64474f75b25878f25585aea1", + "https://bcr.bazel.build/modules/opentelemetry-cpp/1.19.0/source.json": "4e48137e4c3ecb99401ff99876df8fa330598d7da051869bec643446e8a8ff95", + "https://bcr.bazel.build/modules/opentelemetry-proto/1.4.0.bcr.1/MODULE.bazel": "5ceaf25e11170d22eded4c8032728b4a3f273765fccda32f9e94f463755c4167", + "https://bcr.bazel.build/modules/opentelemetry-proto/1.5.0/MODULE.bazel": "7543d91a53b98e7b5b37c5a0865b93bff12c1ee022b1e322cd236b968894b030", + "https://bcr.bazel.build/modules/opentelemetry-proto/1.5.0/source.json": "046b721ce203e88cdaad44d7dd17a86b7200eab9388b663b234e72e13ff7b143", + "https://bcr.bazel.build/modules/opentracing-cpp/1.6.0/MODULE.bazel": "b3925269f63561b8b880ae7cf62ccf81f6ece55b62cd791eda9925147ae116ec", + "https://bcr.bazel.build/modules/opentracing-cpp/1.6.0/source.json": "da1cb1add160f5e5074b7272e9db6fd8f1b3336c15032cd0a653af9d2f484aed", + "https://bcr.bazel.build/modules/package_metadata/0.0.2/MODULE.bazel": "fb8d25550742674d63d7b250063d4580ca530499f045d70748b1b142081ebb92", + "https://bcr.bazel.build/modules/package_metadata/0.0.3/MODULE.bazel": "77890552ecea9e284b5424c9de827a58099348763a4359e975c359a83d4faa83", + "https://bcr.bazel.build/modules/package_metadata/0.0.5/MODULE.bazel": "ef4f9439e3270fdd6b9fd4dbc3d2f29d13888e44c529a1b243f7a31dfbc2e8e4", + "https://bcr.bazel.build/modules/package_metadata/0.0.7/MODULE.bazel": "7adb03933fc8401f495800cf4eafcff0edc6da0ff55c7db223ef69d19f689486", + "https://bcr.bazel.build/modules/package_metadata/0.0.7/source.json": "50639625e937b56115012674c797cca7a05a96b4878c87d803c13dc2b31de8a0", + "https://bcr.bazel.build/modules/patchelf/0.18.0.bcr.1/MODULE.bazel": "35320668481400503dea7c75313473b9ffd6dea4e7ba583b96a896199b0088d5", + "https://bcr.bazel.build/modules/patchelf/0.18.0.bcr.1/source.json": "d877d271ef5d531407885322054c99c7f0617a9a85ff99c0a70f4ced7f13e13a", + "https://bcr.bazel.build/modules/pcre2/10.45/MODULE.bazel": "9bdc838aa2e21be030d7181385e45a5c87b14575bb7ff35d10b5be3d56913bed", + "https://bcr.bazel.build/modules/pcre2/10.45/source.json": "c0b10fd1afbfc978aad9a83e84533880b6c533026fa930d73bb95df34291bc4b", "https://bcr.bazel.build/modules/platforms/0.0.10/MODULE.bazel": "8cb8efaf200bdeb2150d93e162c40f388529a25852b332cec879373771e48ed5", - "https://bcr.bazel.build/modules/platforms/0.0.10/source.json": "f22828ff4cf021a6b577f1bf6341cb9dcd7965092a439f64fc1bb3b7a5ae4bd5", + "https://bcr.bazel.build/modules/platforms/0.0.11/MODULE.bazel": "0daefc49732e227caa8bfa834d65dc52e8cc18a2faf80df25e8caea151a9413f", "https://bcr.bazel.build/modules/platforms/0.0.4/MODULE.bazel": "9b328e31ee156f53f3c416a64f8491f7eb731742655a47c9eec4703a71644aee", "https://bcr.bazel.build/modules/platforms/0.0.5/MODULE.bazel": "5733b54ea419d5eaf7997054bb55f6a1d0b5ff8aedf0176fef9eea44f3acda37", "https://bcr.bazel.build/modules/platforms/0.0.6/MODULE.bazel": "ad6eeef431dc52aefd2d77ed20a4b353f8ebf0f4ecdd26a807d2da5aa8cd0615", "https://bcr.bazel.build/modules/platforms/0.0.7/MODULE.bazel": "72fd4a0ede9ee5c021f6a8dd92b503e089f46c227ba2813ff183b71616034814", "https://bcr.bazel.build/modules/platforms/0.0.8/MODULE.bazel": "9f142c03e348f6d263719f5074b21ef3adf0b139ee4c5133e2aa35664da9eb2d", "https://bcr.bazel.build/modules/platforms/0.0.9/MODULE.bazel": "4a87a60c927b56ddd67db50c89acaa62f4ce2a1d2149ccb63ffd871d5ce29ebc", + "https://bcr.bazel.build/modules/platforms/1.0.0/MODULE.bazel": "f05feb42b48f1b3c225e4ccf351f367be0371411a803198ec34a389fb22aa580", + "https://bcr.bazel.build/modules/platforms/1.1.0/MODULE.bazel": "1c0c09f5bdcf4b3f924720d2478a3711cb39f4977019ca5988685e5b7e18b3d2", + "https://bcr.bazel.build/modules/platforms/1.1.0/source.json": "fcf351c47596c939140ab0d333dfdd08ed1ea6ce33c2fe70c12493a301cf1344", + "https://bcr.bazel.build/modules/prometheus-cpp/1.3.0.bcr.1/MODULE.bazel": "116ad46e97c1d2aeb020fe2899a342a7e703574ce7c0faf7e4810f938c974a9a", + "https://bcr.bazel.build/modules/prometheus-cpp/1.3.0.bcr.1/source.json": "e813cce2d450708cfcb26e309c5172583a7440776edf354e83e6788c768e5cca", + "https://bcr.bazel.build/modules/prometheus-cpp/1.3.0/MODULE.bazel": "ce82e086bbc0b60267e970f6a54b2ca6d0f22d3eb6633e00e2cc2899c700f3d8", "https://bcr.bazel.build/modules/protobuf/21.7/MODULE.bazel": "a5a29bb89544f9b97edce05642fac225a808b5b7be74038ea3640fae2f8e66a7", - "https://bcr.bazel.build/modules/protobuf/21.7/source.json": "bbe500720421e582ff2d18b0802464205138c06056f443184de39fbb8187b09b", + "https://bcr.bazel.build/modules/protobuf/23.1/MODULE.bazel": "88b393b3eb4101d18129e5db51847cd40a5517a53e81216144a8c32dfeeca52a", + "https://bcr.bazel.build/modules/protobuf/24.4/MODULE.bazel": "7bc7ce5f2abf36b3b7b7c8218d3acdebb9426aeb35c2257c96445756f970eb12", + "https://bcr.bazel.build/modules/protobuf/25.6/MODULE.bazel": "fc0ae073b47c7ede88b825ff79e64f1c058967c7a87a86cdf4abecd9e0516625", + "https://bcr.bazel.build/modules/protobuf/26.0/MODULE.bazel": "8402da964092af40097f4a205eec2a33fd4a7748dc43632b7d1629bfd9a2b856", + "https://bcr.bazel.build/modules/protobuf/27.0-rc2/MODULE.bazel": "b2b0dbafd57b6bec0ca9b251da02e628c357dab53a097570aa7d79d020f107cf", + "https://bcr.bazel.build/modules/protobuf/27.0/MODULE.bazel": "7873b60be88844a0a1d8f80b9d5d20cfbd8495a689b8763e76c6372998d3f64c", + "https://bcr.bazel.build/modules/protobuf/27.1/MODULE.bazel": "703a7b614728bb06647f965264967a8ef1c39e09e8f167b3ca0bb1fd80449c0d", + "https://bcr.bazel.build/modules/protobuf/27.2/MODULE.bazel": "32450b50673882e4c8c3d10a83f3bc82161b213ed2f80d17e38bece8f165c295", + "https://bcr.bazel.build/modules/protobuf/29.0-rc2.bcr.1/MODULE.bazel": "52f4126f63a2f0bbf36b99c2a87648f08467a4eaf92ba726bc7d6a500bbf770c", + "https://bcr.bazel.build/modules/protobuf/29.0-rc2/MODULE.bazel": "6241d35983510143049943fc0d57937937122baf1b287862f9dc8590fc4c37df", + "https://bcr.bazel.build/modules/protobuf/29.0-rc3/MODULE.bazel": "33c2dfa286578573afc55a7acaea3cada4122b9631007c594bf0729f41c8de92", + "https://bcr.bazel.build/modules/protobuf/29.0/MODULE.bazel": "319dc8bf4c679ff87e71b1ccfb5a6e90a6dbc4693501d471f48662ac46d04e4e", + "https://bcr.bazel.build/modules/protobuf/29.1/MODULE.bazel": "557c3457560ff49e122ed76c0bc3397a64af9574691cb8201b4e46d4ab2ecb95", + "https://bcr.bazel.build/modules/protobuf/29.3/MODULE.bazel": "77480eea5fb5541903e49683f24dc3e09f4a79e0eea247414887bb9fc0066e94", "https://bcr.bazel.build/modules/protobuf/3.19.0/MODULE.bazel": "6b5fbb433f760a99a22b18b6850ed5784ef0e9928a72668b66e4d7ccd47db9b0", + "https://bcr.bazel.build/modules/protobuf/3.19.2/MODULE.bazel": "532ffe5f2186b69fdde039efe6df13ba726ff338c6bc82275ad433013fa10573", "https://bcr.bazel.build/modules/protobuf/3.19.6/MODULE.bazel": "9233edc5e1f2ee276a60de3eaa47ac4132302ef9643238f23128fea53ea12858", - "https://bcr.bazel.build/modules/pybind11_bazel/2.11.1.bzl.1/MODULE.bazel": "1ef2994c097ee88f8f7ae8fbf991aaefb0603b2540fe575eca14943bc9f220a6", - "https://bcr.bazel.build/modules/pybind11_bazel/2.11.1.bzl.1/source.json": "d8f5b2ebc186b90c6112f92f43d86359dda2c3649c754d9680fd668f8dd28a79", + "https://bcr.bazel.build/modules/protobuf/30.0/MODULE.bazel": "0e736de5d52ad7824113f47e65256a26ee74b689ba859c5447a0663e5a075409", + "https://bcr.bazel.build/modules/protobuf/31.1/MODULE.bazel": "379a389bb330b7b8c1cdf331cc90bf3e13de5614799b3b52cdb7c6f389f6b38e", + "https://bcr.bazel.build/modules/protobuf/32.1/MODULE.bazel": "89cd2866a9cb07fee9ff74c41ceace11554f32e0d849de4e23ac55515cfada4d", + "https://bcr.bazel.build/modules/protobuf/33.4/MODULE.bazel": "114775b816b38b6d0ca620450d6b02550c60ceedfdc8d9a229833b34a223dc42", + "https://bcr.bazel.build/modules/protobuf/34.0.bcr.1/MODULE.bazel": "74e541b0ba877813da786a11707d4e394433c157841d5111a36be0d44b907931", + "https://bcr.bazel.build/modules/protobuf/35.0/MODULE.bazel": "ee97170c013d94c366bbb5fe8a97b0650477cbc00e5e0f6a2c297484bc34d81b", + "https://bcr.bazel.build/modules/protobuf/35.0/source.json": "2162e80adef862606ba85ae42e0df85eb519376fd6ab49caddee6fb776a1281e", + "https://bcr.bazel.build/modules/protoc-gen-validate/1.0.4.bcr.2/MODULE.bazel": "c4bd2c850211ff5b7dadf9d2d0496c1c922fdedc303c775b01dfd3b3efc907ed", + "https://bcr.bazel.build/modules/protoc-gen-validate/1.0.4/MODULE.bazel": "b8913c154b16177990f6126d2d2477d187f9ddc568e95ee3e2d50fc65d2c494a", + "https://bcr.bazel.build/modules/protoc-gen-validate/1.2.1.bcr.1/MODULE.bazel": "4bf09676b62fa587ae07e073420a76ec8766dcce7545e5f8c68cfa8e484b5120", + "https://bcr.bazel.build/modules/protoc-gen-validate/1.2.1.bcr.1/source.json": "c19071ebc4b53b5f1cfab9c66eefaf6e4179eb8a998970d07b1077687e777f29", "https://bcr.bazel.build/modules/pybind11_bazel/2.11.1/MODULE.bazel": "88af1c246226d87e65be78ed49ecd1e6f5e98648558c14ce99176da041dc378e", + "https://bcr.bazel.build/modules/pybind11_bazel/2.12.0/MODULE.bazel": "e6f4c20442eaa7c90d7190d8dc539d0ab422f95c65a57cc59562170c58ae3d34", + "https://bcr.bazel.build/modules/pybind11_bazel/2.13.6/MODULE.bazel": "2d746fda559464b253b2b2e6073cb51643a2ac79009ca02100ebbc44b4548656", + "https://bcr.bazel.build/modules/pybind11_bazel/3.0.1/MODULE.bazel": "67899804c3dbdf169d107ce7c00e60a38a46d097c7ff3460bdebe192cbc77f56", + "https://bcr.bazel.build/modules/pybind11_bazel/3.0.1/source.json": "5bcb3ae55c0d5a159bd910154d7ee8873bdf61dd7b13cd0a2bb456011ec706ae", + "https://bcr.bazel.build/modules/rapidjson/1.1.0.bcr.20241007/MODULE.bazel": "82fbcb2e42f9e0040e76ccc74c06c3e46dfd33c64ca359293f8b84df0e6dff4c", + "https://bcr.bazel.build/modules/rapidjson/1.1.0.bcr.20241007/source.json": "5c42389ad0e21fc06b95ad7c0b730008271624a2fa3292e0eab5f30e15adeee3", + "https://bcr.bazel.build/modules/re2/2021-09-01/MODULE.bazel": "bcb6b96f3b071e6fe2d8bed9cc8ada137a105f9d2c5912e91d27528b3d123833", "https://bcr.bazel.build/modules/re2/2023-09-01/MODULE.bazel": "cb3d511531b16cfc78a225a9e2136007a48cf8a677e4264baeab57fe78a80206", - "https://bcr.bazel.build/modules/re2/2024-02-01/MODULE.bazel": "5ed922cb8b6c110e30969695e73bd0d3159576bf17ce8ee2443a7d07bf500551", - "https://bcr.bazel.build/modules/re2/2024-02-01/source.json": "0b108d5e9836cb66815fbbb70e4395eab52dc29dd2c39a9785f01d6661a8f37e", - "https://bcr.bazel.build/modules/rules_apple/3.2.1/MODULE.bazel": "55f19b572fdb4a4bd5a1c2231b60c663f8b4e3146769062b2bdeba72226c74b7", - "https://bcr.bazel.build/modules/rules_apple/3.2.1/source.json": "b7e8aa85c5fcd9dfec8ea62642b7e93ca161a03b2c403ae05872813f5dfeb7be", + "https://bcr.bazel.build/modules/re2/2024-05-01/MODULE.bazel": "55a3f059538f381107824e7d00df5df6d061ba1fb80e874e4909c0f0549e8f3e", + "https://bcr.bazel.build/modules/re2/2024-07-02.bcr.1/MODULE.bazel": "b4963dda9b31080be1905ef085ecd7dd6cd47c05c79b9cdf83ade83ab2ab271a", + "https://bcr.bazel.build/modules/re2/2024-07-02/MODULE.bazel": "0eadc4395959969297cbcf31a249ff457f2f1d456228c67719480205aa306daa", + "https://bcr.bazel.build/modules/re2/2025-08-12.bcr.1/MODULE.bazel": "e09b434b122bfb786a69179f9b325e35cb1856c3f56a7a81dd61609260ed46e1", + "https://bcr.bazel.build/modules/re2/2025-11-05/MODULE.bazel": "d97e240364f132017500a3af816493af7efd4be40aab2bb403ed449cc96f70a3", + "https://bcr.bazel.build/modules/re2/2025-11-05/source.json": "7df1394aabda1c9bc188a302f5d54b1c657924edd04ebc57d2be29dbd7efd141", + "https://bcr.bazel.build/modules/rules_android/0.1.1/MODULE.bazel": "48809ab0091b07ad0182defb787c4c5328bd3a278938415c00a7b69b50c4d3a8", + "https://bcr.bazel.build/modules/rules_android/0.7.1/MODULE.bazel": "a806fc382a774252f228a40e3b11b9fcc6276f8778c7fb33e9f72937c6258363", + "https://bcr.bazel.build/modules/rules_android/0.7.1/source.json": "151440aed3f0f73a00d4ed5cec5d31f63a6fef9b95d8fab1eb1810150fa525f2", + "https://bcr.bazel.build/modules/rules_apple/3.13.0/MODULE.bazel": "b4559a2c6281ca3165275bb36c1f0ac74666632adc5bdb680e366de7ce845f43", + "https://bcr.bazel.build/modules/rules_apple/3.16.0/MODULE.bazel": "0d1caf0b8375942ce98ea944be754a18874041e4e0459401d925577624d3a54a", + "https://bcr.bazel.build/modules/rules_apple/4.1.0/MODULE.bazel": "76e10fd4a48038d3fc7c5dc6e63b7063bbf5304a2e3bd42edda6ec660eebea68", + "https://bcr.bazel.build/modules/rules_apple/4.1.0/source.json": "8ee81e1708756f81b343a5eb2b2f0b953f1d25c4ab3d4a68dc02754872e80715", + "https://bcr.bazel.build/modules/rules_buf/0.1.1/MODULE.bazel": "6189aec18a4f7caff599ad41b851ab7645d4f1e114aa6431acf9b0666eb92162", "https://bcr.bazel.build/modules/rules_cc/0.0.1/MODULE.bazel": "cb2aa0747f84c6c3a78dad4e2049c154f08ab9d166b1273835a8174940365647", + "https://bcr.bazel.build/modules/rules_cc/0.0.10/MODULE.bazel": "ec1705118f7eaedd6e118508d3d26deba2a4e76476ada7e0e3965211be012002", + "https://bcr.bazel.build/modules/rules_cc/0.0.13/MODULE.bazel": "0e8529ed7b323dad0775ff924d2ae5af7640b23553dfcd4d34344c7e7a867191", + "https://bcr.bazel.build/modules/rules_cc/0.0.14/MODULE.bazel": "5e343a3aac88b8d7af3b1b6d2093b55c347b8eefc2e7d1442f7a02dc8fea48ac", + "https://bcr.bazel.build/modules/rules_cc/0.0.15/MODULE.bazel": "6704c35f7b4a72502ee81f61bf88706b54f06b3cbe5558ac17e2e14666cd5dcc", + "https://bcr.bazel.build/modules/rules_cc/0.0.16/MODULE.bazel": "7661303b8fc1b4d7f532e54e9d6565771fea666fbdf839e0a86affcd02defe87", + "https://bcr.bazel.build/modules/rules_cc/0.0.17/MODULE.bazel": "2ae1d8f4238ec67d7185d8861cb0a2cdf4bc608697c331b95bf990e69b62e64a", "https://bcr.bazel.build/modules/rules_cc/0.0.2/MODULE.bazel": "6915987c90970493ab97393024c156ea8fb9f3bea953b2f3ec05c34f19b5695c", - "https://bcr.bazel.build/modules/rules_cc/0.0.6/MODULE.bazel": "abf360251023dfe3efcef65ab9d56beefa8394d4176dd29529750e1c57eaa33f", + "https://bcr.bazel.build/modules/rules_cc/0.0.5/MODULE.bazel": "be41f87587998fe8890cd82ea4e848ed8eb799e053c224f78f3ff7fe1a1d9b74", "https://bcr.bazel.build/modules/rules_cc/0.0.8/MODULE.bazel": "964c85c82cfeb6f3855e6a07054fdb159aced38e99a5eecf7bce9d53990afa3e", "https://bcr.bazel.build/modules/rules_cc/0.0.9/MODULE.bazel": "836e76439f354b89afe6a911a7adf59a6b2518fafb174483ad78a2a2fde7b1c5", - "https://bcr.bazel.build/modules/rules_cc/0.0.9/source.json": "1f1ba6fea244b616de4a554a0f4983c91a9301640c8fe0dd1d410254115c8430", + "https://bcr.bazel.build/modules/rules_cc/0.1.1/MODULE.bazel": "2f0222a6f229f0bf44cd711dc13c858dad98c62d52bd51d8fc3a764a83125513", + "https://bcr.bazel.build/modules/rules_cc/0.1.2/MODULE.bazel": "557ddc3a96858ec0d465a87c0a931054d7dcfd6583af2c7ed3baf494407fd8d0", + "https://bcr.bazel.build/modules/rules_cc/0.1.4/MODULE.bazel": "bb03a452a7527ac25a7518fb86a946ef63df860b9657d8323a0c50f8504fb0b9", + "https://bcr.bazel.build/modules/rules_cc/0.1.5/MODULE.bazel": "88dfc9361e8b5ae1008ac38f7cdfd45ad738e4fa676a3ad67d19204f045a1fd8", + "https://bcr.bazel.build/modules/rules_cc/0.2.13/MODULE.bazel": "eecdd666eda6be16a8d9dc15e44b5c75133405e820f620a234acc4b1fdc5aa37", + "https://bcr.bazel.build/modules/rules_cc/0.2.14/MODULE.bazel": "353c99ed148887ee89c54a17d4100ae7e7e436593d104b668476019023b58df8", + "https://bcr.bazel.build/modules/rules_cc/0.2.15/MODULE.bazel": "6a0a4a75a57aa6dc888300d848053a58c6b12a29f89d4304e1c41448514ec6e8", + "https://bcr.bazel.build/modules/rules_cc/0.2.16/MODULE.bazel": "9242fa89f950c6ef7702801ab53922e99c69b02310c39fb6e62b2bd30df2a1d4", + "https://bcr.bazel.build/modules/rules_cc/0.2.17/MODULE.bazel": "1849602c86cb60da8613d2de887f9566a6d354a6df6d7009f9d04a14402f9a84", + "https://bcr.bazel.build/modules/rules_cc/0.2.18/MODULE.bazel": "4460ec36adc8f722a6a2a4ac9374cb91f2acebadaa93fc37966129afb3dece87", + "https://bcr.bazel.build/modules/rules_cc/0.2.19/MODULE.bazel": "d5e0f05b63273281a16654eb6b1a8742a75ec153ac8b4f0419949d6e401e46f0", + "https://bcr.bazel.build/modules/rules_cc/0.2.20/MODULE.bazel": "f5c07bce5ddcb99be21a0812ff5aadb439e688b7449c6542152363b2fd859c1a", + "https://bcr.bazel.build/modules/rules_cc/0.2.25/MODULE.bazel": "4a3d4f3606d3b3190be495dbc497acca552807d1d5540661ef0d1658472882e3", + "https://bcr.bazel.build/modules/rules_cc/0.2.25/source.json": "8df28a9a97b878fe45f0a4f4adf019869dda51a60ee2f6e15a0c79656a25058d", + "https://bcr.bazel.build/modules/rules_cc/0.2.4/MODULE.bazel": "1ff1223dfd24f3ecf8f028446d4a27608aa43c3f41e346d22838a4223980b8cc", + "https://bcr.bazel.build/modules/rules_cc/0.2.8/MODULE.bazel": "f1df20f0bf22c28192a794f29b501ee2018fa37a3862a1a2132ae2940a23a642", + "https://bcr.bazel.build/modules/rules_cc/0.2.9/MODULE.bazel": "34263f1dca62ea664265438cef714d7db124c03e1ed55ebb4f1dc860164308d1", + "https://bcr.bazel.build/modules/rules_distroless/0.8.0/MODULE.bazel": "a3e473943b685190e20668f5149f62a116fdcbeba66c7b9778f71d2aed4533c1", + "https://bcr.bazel.build/modules/rules_distroless/0.8.0/source.json": "a6931e20d97a13675693adb8b6cd3a0e6eb5218db43dfc83b75566ddc4158efc", + "https://bcr.bazel.build/modules/rules_foreign_cc/0.10.1/MODULE.bazel": "b9527010e5fef060af92b6724edb3691970a5b1f76f74b21d39f7d433641be60", + "https://bcr.bazel.build/modules/rules_foreign_cc/0.15.1/MODULE.bazel": "c2c60d26c79fda484acb95cdbec46e89d6b28b4845cb277160ce1e0c8622bb88", + "https://bcr.bazel.build/modules/rules_foreign_cc/0.15.1/source.json": "a161811a63ba8a859086da3b7ff3ad04f2e9c255d7727b41087103fc0eb22f55", "https://bcr.bazel.build/modules/rules_foreign_cc/0.9.0/MODULE.bazel": "c9e8c682bf75b0e7c704166d79b599f93b72cfca5ad7477df596947891feeef6", + "https://bcr.bazel.build/modules/rules_fuzzing/0.5.2/MODULE.bazel": "40c97d1144356f52905566c55811f13b299453a14ac7769dfba2ac38192337a8", + "https://bcr.bazel.build/modules/rules_go/0.33.0/MODULE.bazel": "a2b11b64cd24bf94f57454f53288a5dacfe6cb86453eee7761b7637728c1910c", + "https://bcr.bazel.build/modules/rules_go/0.38.1/MODULE.bazel": "fb8e73dd3b6fc4ff9d260ceacd830114891d49904f5bda1c16bc147bcc254f71", + "https://bcr.bazel.build/modules/rules_go/0.39.1/MODULE.bazel": "d34fb2a249403a5f4339c754f1e63dc9e5ad70b47c5e97faee1441fc6636cd61", + "https://bcr.bazel.build/modules/rules_go/0.41.0/MODULE.bazel": "55861d8e8bb0e62cbd2896f60ff303f62ffcb0eddb74ecb0e5c0cbe36fc292c8", + "https://bcr.bazel.build/modules/rules_go/0.42.0/MODULE.bazel": "8cfa875b9aa8c6fce2b2e5925e73c1388173ea3c32a0db4d2b4804b453c14270", + "https://bcr.bazel.build/modules/rules_go/0.45.1/MODULE.bazel": "6d7884f0edf890024eba8ab31a621faa98714df0ec9d512389519f0edff0281a", + "https://bcr.bazel.build/modules/rules_go/0.46.0/MODULE.bazel": "3477df8bdcc49e698b9d25f734c4f3a9f5931ff34ee48a2c662be168f5f2d3fd", + "https://bcr.bazel.build/modules/rules_go/0.48.0/MODULE.bazel": "d00ebcae0908ee3f5e6d53f68677a303d6d59a77beef879598700049c3980a03", + "https://bcr.bazel.build/modules/rules_go/0.49.0/MODULE.bazel": "61cfc1ba17123356d1b12b6c50f6e0162b2cc7fd6f51753c12471e973a0f72a5", + "https://bcr.bazel.build/modules/rules_go/0.50.1/MODULE.bazel": "b91a308dc5782bb0a8021ad4330c81fea5bda77f96b9e4c117b9b9c8f6665ee0", + "https://bcr.bazel.build/modules/rules_go/0.53.0/MODULE.bazel": "a4ed760d3ac0dbc0d7b967631a9a3fd9100d28f7d9fcf214b4df87d4bfff5f9a", + "https://bcr.bazel.build/modules/rules_go/0.59.0/MODULE.bazel": "b7e43e7414a3139a7547d1b4909b29085fbe5182b6c58cbe1ed4c6272815aeae", + "https://bcr.bazel.build/modules/rules_go/0.62.0/MODULE.bazel": "8ee616065c3d2b2f7ac0880108316ce8d0c332b3a30aad24e95c0bc124ec853e", + "https://bcr.bazel.build/modules/rules_go/0.62.0/source.json": "36d781c558eb7d3bd49dc5e190455714f174232372f15207ab200b4348bda3e6", "https://bcr.bazel.build/modules/rules_java/4.0.0/MODULE.bazel": "5a78a7ae82cd1a33cef56dc578c7d2a46ed0dca12643ee45edbb8417899e6f74", + "https://bcr.bazel.build/modules/rules_java/5.3.5/MODULE.bazel": "a4ec4f2db570171e3e5eb753276ee4b389bae16b96207e9d3230895c99644b86", + "https://bcr.bazel.build/modules/rules_java/5.5.0/MODULE.bazel": "486ad1aa15cdc881af632b4b1448b0136c76025a1fe1ad1b65c5899376b83a50", + "https://bcr.bazel.build/modules/rules_java/6.0.0/MODULE.bazel": "8a43b7df601a7ec1af61d79345c17b31ea1fedc6711fd4abfd013ea612978e39", "https://bcr.bazel.build/modules/rules_java/6.3.0/MODULE.bazel": "a97c7678c19f236a956ad260d59c86e10a463badb7eb2eda787490f4c969b963", + "https://bcr.bazel.build/modules/rules_java/6.4.0/MODULE.bazel": "e986a9fe25aeaa84ac17ca093ef13a4637f6107375f64667a15999f77db6c8f6", + "https://bcr.bazel.build/modules/rules_java/6.5.2/MODULE.bazel": "1d440d262d0e08453fa0c4d8f699ba81609ed0e9a9a0f02cd10b3e7942e61e31", + "https://bcr.bazel.build/modules/rules_java/7.1.0/MODULE.bazel": "30d9135a2b6561c761bd67bd4990da591e6bdc128790ce3e7afd6a3558b2fb64", + "https://bcr.bazel.build/modules/rules_java/7.10.0/MODULE.bazel": "530c3beb3067e870561739f1144329a21c851ff771cd752a49e06e3dc9c2e71a", + "https://bcr.bazel.build/modules/rules_java/7.12.2/MODULE.bazel": "579c505165ee757a4280ef83cda0150eea193eed3bef50b1004ba88b99da6de6", + "https://bcr.bazel.build/modules/rules_java/7.2.0/MODULE.bazel": "06c0334c9be61e6cef2c8c84a7800cef502063269a5af25ceb100b192453d4ab", "https://bcr.bazel.build/modules/rules_java/7.3.2/MODULE.bazel": "50dece891cfdf1741ea230d001aa9c14398062f2b7c066470accace78e412bc2", + "https://bcr.bazel.build/modules/rules_java/7.4.0/MODULE.bazel": "a592852f8a3dd539e82ee6542013bf2cadfc4c6946be8941e189d224500a8934", + "https://bcr.bazel.build/modules/rules_java/7.6.1/MODULE.bazel": "2f14b7e8a1aa2f67ae92bc69d1ec0fa8d9f827c4e17ff5e5f02e91caa3b2d0fe", "https://bcr.bazel.build/modules/rules_java/7.6.5/MODULE.bazel": "481164be5e02e4cab6e77a36927683263be56b7e36fef918b458d7a8a1ebadb1", - "https://bcr.bazel.build/modules/rules_java/7.6.5/source.json": "a805b889531d1690e3c72a7a7e47a870d00323186a9904b36af83aa3d053ee8d", + "https://bcr.bazel.build/modules/rules_java/8.3.2/MODULE.bazel": "7336d5511ad5af0b8615fdc7477535a2e4e723a357b6713af439fe8cf0195017", + "https://bcr.bazel.build/modules/rules_java/8.5.1/MODULE.bazel": "d8a9e38cc5228881f7055a6079f6f7821a073df3744d441978e7a43e20226939", + "https://bcr.bazel.build/modules/rules_java/8.6.0/MODULE.bazel": "9c064c434606d75a086f15ade5edb514308cccd1544c2b2a89bbac4310e41c71", + "https://bcr.bazel.build/modules/rules_java/8.6.1/MODULE.bazel": "f4808e2ab5b0197f094cabce9f4b006a27766beb6a9975931da07099560ca9c2", + "https://bcr.bazel.build/modules/rules_java/8.8.0/MODULE.bazel": "de589d0880911ac007abd521b9f0ddcd8b0dbd05c8553e6f8124a050b83acf7d", + "https://bcr.bazel.build/modules/rules_java/8.9.0/MODULE.bazel": "e17c876cb53dcd817b7b7f0d2985b710610169729e8c371b2221cacdcd3dce4a", + "https://bcr.bazel.build/modules/rules_java/9.1.0/MODULE.bazel": "ee63f27e36a3fada80342869361182f120a9819c74320e8e65b1e04ba0cd7a9d", + "https://bcr.bazel.build/modules/rules_java/9.3.0/MODULE.bazel": "f657c72d65ac449caae9abf2e68e66c0d36f9416848c4c4903d0b3234229e7f2", + "https://bcr.bazel.build/modules/rules_java/9.3.0/source.json": "59ae7e662c3c7042b88bbb42ad12483523e234c65ebe4c51611baa43e85cb248", "https://bcr.bazel.build/modules/rules_jvm_external/4.4.2/MODULE.bazel": "a56b85e418c83eb1839819f0b515c431010160383306d13ec21959ac412d2fe7", + "https://bcr.bazel.build/modules/rules_jvm_external/5.1/MODULE.bazel": "33f6f999e03183f7d088c9be518a63467dfd0be94a11d0055fe2d210f89aa909", "https://bcr.bazel.build/modules/rules_jvm_external/5.2/MODULE.bazel": "d9351ba35217ad0de03816ef3ed63f89d411349353077348a45348b096615036", - "https://bcr.bazel.build/modules/rules_jvm_external/5.2/source.json": "10572111995bc349ce31c78f74b3c147f6b3233975c7fa5eff9211f6db0d34d9", + "https://bcr.bazel.build/modules/rules_jvm_external/5.3/MODULE.bazel": "bf93870767689637164657731849fb887ad086739bd5d360d90007a581d5527d", + "https://bcr.bazel.build/modules/rules_jvm_external/6.0/MODULE.bazel": "37c93a5a78d32e895d52f86a8d0416176e915daabd029ccb5594db422e87c495", + "https://bcr.bazel.build/modules/rules_jvm_external/6.1/MODULE.bazel": "75b5fec090dbd46cf9b7d8ea08cf84a0472d92ba3585b476f44c326eda8059c4", + "https://bcr.bazel.build/modules/rules_jvm_external/6.2/MODULE.bazel": "36a6e52487a855f33cb960724eb56547fa87e2c98a0474c3acad94339d7f8e99", + "https://bcr.bazel.build/modules/rules_jvm_external/6.3/MODULE.bazel": "c998e060b85f71e00de5ec552019347c8bca255062c990ac02d051bb80a38df0", + "https://bcr.bazel.build/modules/rules_jvm_external/6.7/MODULE.bazel": "e717beabc4d091ecb2c803c2d341b88590e9116b8bf7947915eeb33aab4f96dd", + "https://bcr.bazel.build/modules/rules_jvm_external/6.9/MODULE.bazel": "07c5db05527db7744a54fcffd653e1550d40e0540207a7f7e6d0a4de5bef8274", + "https://bcr.bazel.build/modules/rules_jvm_external/6.9/source.json": "b12970214f3cc144b26610caeb101fa622d910f1ab3d98f0bae1058edbd00bd4", + "https://bcr.bazel.build/modules/rules_kotlin/1.9.0/MODULE.bazel": "ef85697305025e5a61f395d4eaede272a5393cee479ace6686dba707de804d59", + "https://bcr.bazel.build/modules/rules_kotlin/1.9.5/MODULE.bazel": "043a16a572f610558ec2030db3ff0c9938574e7dd9f58bded1bb07c0192ef025", + "https://bcr.bazel.build/modules/rules_kotlin/1.9.6/MODULE.bazel": "d269a01a18ee74d0335450b10f62c9ed81f2321d7958a2934e44272fe82dcef3", + "https://bcr.bazel.build/modules/rules_kotlin/2.3.20/MODULE.bazel": "3443d53d275e14fecfebd0b491f01d06ea3883c04a1b3336e7ae9d5ec9066bef", + "https://bcr.bazel.build/modules/rules_kotlin/2.3.20/source.json": "5a5553cffea43f2c5156c8ad0de4a14ad95413ceb39cd4d08f50e2aea86927e8", "https://bcr.bazel.build/modules/rules_license/0.0.3/MODULE.bazel": "627e9ab0247f7d1e05736b59dbb1b6871373de5ad31c3011880b4133cafd4bd0", - "https://bcr.bazel.build/modules/rules_license/0.0.4/MODULE.bazel": "6a88dd22800cf1f9f79ba32cacad0d3a423ed28efa2c2ed5582eaa78dd3ac1e5", "https://bcr.bazel.build/modules/rules_license/0.0.7/MODULE.bazel": "088fbeb0b6a419005b89cf93fe62d9517c0a2b8bb56af3244af65ecfe37e7d5d", "https://bcr.bazel.build/modules/rules_license/0.0.8/MODULE.bazel": "5669c6fe49b5134dbf534db681ad3d67a2d49cfc197e4a95f1ca2fd7f3aebe96", - "https://bcr.bazel.build/modules/rules_license/0.0.8/source.json": "ccfd3964cd0cd1739202efb8dbf9a06baab490e61e174b2ad4790f9c4e610beb", + "https://bcr.bazel.build/modules/rules_license/1.0.0/MODULE.bazel": "a7fda60eefdf3d8c827262ba499957e4df06f659330bbe6cdbdb975b768bb65c", + "https://bcr.bazel.build/modules/rules_license/1.0.0/source.json": "a52c89e54cc311196e478f8382df91c15f7a2bfdf4c6cd0e2675cc2ff0b56efb", + "https://bcr.bazel.build/modules/rules_nodejs/5.8.2/MODULE.bazel": "6bc03c8f37f69401b888023bf511cb6ee4781433b0cb56236b2e55a21e3a026a", + "https://bcr.bazel.build/modules/rules_oci/2.2.7/MODULE.bazel": "f6150e4b224d459f7f6523ef65967464ca4efdd266c7fbf2f5a2a51011957e0c", + "https://bcr.bazel.build/modules/rules_oci/2.2.7/source.json": "b099f02af330f47f19dc67fc9300ef6e1937a8c86882690db0e7a2fcea8c7f6b", + "https://bcr.bazel.build/modules/rules_perl/0.2.4/MODULE.bazel": "5f5af7be4bf5fb88d91af7469518f0fd2161718aefc606188f7cd51f436ca938", + "https://bcr.bazel.build/modules/rules_perl/0.2.4/source.json": "574317d6b3c7e4843fe611b76f15e62a1889949f5570702e1ee4ad335ea3c339", "https://bcr.bazel.build/modules/rules_pkg/0.7.0/MODULE.bazel": "df99f03fc7934a4737122518bb87e667e62d780b610910f0447665a7e2be62dc", - "https://bcr.bazel.build/modules/rules_pkg/0.9.1/MODULE.bazel": "af00144208c4be503bc920d043ba3284fb37385b3f6160b4a4daf4df80b4b823", "https://bcr.bazel.build/modules/rules_pkg/1.0.1/MODULE.bazel": "5b1df97dbc29623bccdf2b0dcd0f5cb08e2f2c9050aab1092fd39a41e82686ff", "https://bcr.bazel.build/modules/rules_pkg/1.0.1/source.json": "bd82e5d7b9ce2d31e380dd9f50c111d678c3bdaca190cb76b0e1c71b05e1ba8a", "https://bcr.bazel.build/modules/rules_proto/4.0.0/MODULE.bazel": "a7a7b6ce9bee418c1a760b3d84f83a299ad6952f9903c67f19e4edd964894e06", "https://bcr.bazel.build/modules/rules_proto/5.3.0-21.7/MODULE.bazel": "e8dff86b0971688790ae75528fe1813f71809b5afd57facb44dad9e8eca631b7", "https://bcr.bazel.build/modules/rules_proto/6.0.0-rc1/MODULE.bazel": "1e5b502e2e1a9e825eef74476a5a1ee524a92297085015a052510b09a1a09483", + "https://bcr.bazel.build/modules/rules_proto/6.0.0/MODULE.bazel": "b531d7f09f58dce456cd61b4579ce8c86b38544da75184eadaf0a7cb7966453f", "https://bcr.bazel.build/modules/rules_proto/6.0.2/MODULE.bazel": "ce916b775a62b90b61888052a416ccdda405212b6aaeb39522f7dc53431a5e73", - "https://bcr.bazel.build/modules/rules_proto/6.0.2/source.json": "17a2e195f56cb28d6bbf763e49973d13890487c6945311ed141e196fb660426d", + "https://bcr.bazel.build/modules/rules_proto/7.0.2/MODULE.bazel": "bf81793bd6d2ad89a37a40693e56c61b0ee30f7a7fdbaf3eabbf5f39de47dea2", + "https://bcr.bazel.build/modules/rules_proto/7.1.0/MODULE.bazel": "002d62d9108f75bb807cd56245d45648f38275cb3a99dcd45dfb864c5d74cb96", + "https://bcr.bazel.build/modules/rules_proto/7.1.0/source.json": "39f89066c12c24097854e8f57ab8558929f9c8d474d34b2c00ac04630ad8940e", "https://bcr.bazel.build/modules/rules_python/0.10.2/MODULE.bazel": "cc82bc96f2997baa545ab3ce73f196d040ffb8756fd2d66125a530031cd90e5f", + "https://bcr.bazel.build/modules/rules_python/0.20.0/MODULE.bazel": "bfe14d17f20e3fe900b9588f526f52c967a6f281e47a1d6b988679bd15082286", + "https://bcr.bazel.build/modules/rules_python/0.22.0/MODULE.bazel": "b8057bafa11a9e0f4b08fc3b7cd7bee0dcbccea209ac6fc9a3ff051cd03e19e9", "https://bcr.bazel.build/modules/rules_python/0.22.1/MODULE.bazel": "26114f0c0b5e93018c0c066d6673f1a2c3737c7e90af95eff30cfee38d0bbac7", + "https://bcr.bazel.build/modules/rules_python/0.23.1/MODULE.bazel": "49ffccf0511cb8414de28321f5fcf2a31312b47c40cc21577144b7447f2bf300", "https://bcr.bazel.build/modules/rules_python/0.25.0/MODULE.bazel": "72f1506841c920a1afec76975b35312410eea3aa7b63267436bfb1dd91d2d382", - "https://bcr.bazel.build/modules/rules_python/0.29.0/MODULE.bazel": "2ac8cd70524b4b9ec49a0b8284c79e4cd86199296f82f6e0d5da3f783d660c82", + "https://bcr.bazel.build/modules/rules_python/0.28.0/MODULE.bazel": "cba2573d870babc976664a912539b320cbaa7114cd3e8f053c720171cde331ed", "https://bcr.bazel.build/modules/rules_python/0.31.0/MODULE.bazel": "93a43dc47ee570e6ec9f5779b2e64c1476a6ce921c48cc9a1678a91dd5f8fd58", - "https://bcr.bazel.build/modules/rules_python/0.31.0/source.json": "a41c836d4065888eef4377f2f27b6eea0fedb9b5adb1bab1970437373fe90dc7", + "https://bcr.bazel.build/modules/rules_python/0.33.2/MODULE.bazel": "3e036c4ad8d804a4dad897d333d8dce200d943df4827cb849840055be8d2e937", + "https://bcr.bazel.build/modules/rules_python/0.34.0/MODULE.bazel": "1d623d026e075b78c9fde483a889cda7996f5da4f36dffb24c246ab30f06513a", + "https://bcr.bazel.build/modules/rules_python/0.35.0/MODULE.bazel": "c3657951764cdcdb5a7370d5e885fad5e8c1583320aad18d46f9f110d2c22755", + "https://bcr.bazel.build/modules/rules_python/0.37.1/MODULE.bazel": "3faeb2d9fa0a81f8980643ee33f212308f4d93eea4b9ce6f36d0b742e71e9500", + "https://bcr.bazel.build/modules/rules_python/0.37.2/MODULE.bazel": "b5ffde91410745750b6c13be1c5dc4555ef5bc50562af4a89fd77807fdde626a", "https://bcr.bazel.build/modules/rules_python/0.4.0/MODULE.bazel": "9208ee05fd48bf09ac60ed269791cf17fb343db56c8226a720fbb1cdf467166c", - "https://bcr.bazel.build/modules/rules_swift/1.15.1/MODULE.bazel": "7031d3e5221b54dfee3901d221f10f54a8b0f2a64d4616de7879bcf83fe8d85d", - "https://bcr.bazel.build/modules/rules_swift/1.15.1/source.json": "6472a4607d0a2f39eeb7daae78054b995aba593caad99aa64561f631fcf93a63", + "https://bcr.bazel.build/modules/rules_python/0.40.0/MODULE.bazel": "9d1a3cd88ed7d8e39583d9ffe56ae8a244f67783ae89b60caafc9f5cf318ada7", + "https://bcr.bazel.build/modules/rules_python/1.0.0/MODULE.bazel": "898a3d999c22caa585eb062b600f88654bf92efb204fa346fb55f6f8edffca43", + "https://bcr.bazel.build/modules/rules_python/1.2.0/MODULE.bazel": "5aeeb48b2a6c19d668b48adf2b8a2b209a6310c230db0ce77450f148a89846e4", + "https://bcr.bazel.build/modules/rules_python/1.3.0/MODULE.bazel": "8361d57eafb67c09b75bf4bbe6be360e1b8f4f18118ab48037f2bd50aa2ccb13", + "https://bcr.bazel.build/modules/rules_python/1.4.1/MODULE.bazel": "8991ad45bdc25018301d6b7e1d3626afc3c8af8aaf4bc04f23d0b99c938b73a6", + "https://bcr.bazel.build/modules/rules_python/1.5.1/MODULE.bazel": "acfe65880942d44a69129d4c5c3122d57baaf3edf58ae5a6bd4edea114906bf5", + "https://bcr.bazel.build/modules/rules_python/1.6.0/MODULE.bazel": "7e04ad8f8d5bea40451cf80b1bd8262552aa73f841415d20db96b7241bd027d8", + "https://bcr.bazel.build/modules/rules_python/1.6.3/MODULE.bazel": "a7b80c42cb3de5ee2a5fa1abc119684593704fcd2fec83165ebe615dec76574f", + "https://bcr.bazel.build/modules/rules_python/1.7.0/MODULE.bazel": "d01f995ecd137abf30238ad9ce97f8fc3ac57289c8b24bd0bf53324d937a14f8", + "https://bcr.bazel.build/modules/rules_python/1.8.0/MODULE.bazel": "c151c025dbcc93d8f62ab68ecc313c9176a868a0e6386981bf2a12aec77cbe7b", + "https://bcr.bazel.build/modules/rules_python/1.9.0/MODULE.bazel": "afc3a05f29f09f2d3ee95ad99a145250dab41a2b2d8d6f82cc91936b3213282c", + "https://bcr.bazel.build/modules/rules_python/1.9.0/source.json": "3921ea0b65298d51aead5b9e4a82203d7be9b5918619b58b53f1c259f4e63169", + "https://bcr.bazel.build/modules/rules_robolectric/4.14.1.2/MODULE.bazel": "d44fec647d0aeb67b9f3b980cf68ba634976f3ae7ccd6c07d790b59b87a4f251", + "https://bcr.bazel.build/modules/rules_robolectric/4.14.1.2/source.json": "37c10335f2361c337c5c1f34ed36d2da70534c23088062b33a8bdaab68aa9dea", + "https://bcr.bazel.build/modules/rules_rust/0.51.0/MODULE.bazel": "2b6d1617ac8503bfdcc0e4520c20539d4bba3a691100bee01afe193ceb0310f9", + "https://bcr.bazel.build/modules/rules_rust/0.69.0/MODULE.bazel": "4326fec48f2fef0d514de46346f7f77e200c82936dd08b91c9ef039fbdad5c10", + "https://bcr.bazel.build/modules/rules_rust/0.69.0/source.json": "0d094307d690cc18b3ab003998697be8070a206f65592c5c8476999796f11c4b", + "https://bcr.bazel.build/modules/rules_shell/0.2.0/MODULE.bazel": "fda8a652ab3c7d8fee214de05e7a9916d8b28082234e8d2c0094505c5268ed3c", + "https://bcr.bazel.build/modules/rules_shell/0.3.0/MODULE.bazel": "de4402cd12f4cc8fda2354fce179fdb068c0b9ca1ec2d2b17b3e21b24c1a937b", + "https://bcr.bazel.build/modules/rules_shell/0.4.0/MODULE.bazel": "0f8f11bb3cd11755f0b48c1de0bbcf62b4b34421023aa41a2fc74ef68d9584f0", + "https://bcr.bazel.build/modules/rules_shell/0.4.1/MODULE.bazel": "00e501db01bbf4e3e1dd1595959092c2fadf2087b2852d3f553b5370f5633592", + "https://bcr.bazel.build/modules/rules_shell/0.6.1/MODULE.bazel": "72e76b0eea4e81611ef5452aa82b3da34caca0c8b7b5c0c9584338aa93bae26b", + "https://bcr.bazel.build/modules/rules_shell/0.8.0/MODULE.bazel": "f6a89f1d6a669a26f28fe814503857055d76306b79cfc11d12399af08d0b80ae", + "https://bcr.bazel.build/modules/rules_shell/0.8.0/source.json": "eb53cc815bc503c6683c5fe12d943f98883f81fc22f51403ec8a95610cba4195", + "https://bcr.bazel.build/modules/rules_swift/1.16.0/MODULE.bazel": "4a09f199545a60d09895e8281362b1ff3bb08bbde69c6fc87aff5b92fcc916ca", + "https://bcr.bazel.build/modules/rules_swift/2.1.1/MODULE.bazel": "494900a80f944fc7aa61500c2073d9729dff0b764f0e89b824eb746959bc1046", + "https://bcr.bazel.build/modules/rules_swift/2.4.0/MODULE.bazel": "1639617eb1ede28d774d967a738b4a68b0accb40650beadb57c21846beab5efd", + "https://bcr.bazel.build/modules/rules_swift/2.5.0/MODULE.bazel": "6d2fec72be0ad2e1fc2eab536cf9d3fb149037c6d2aa678078f3512c06a819e1", + "https://bcr.bazel.build/modules/rules_swift/3.1.2/MODULE.bazel": "72c8f5cf9d26427cee6c76c8e3853eb46ce6b0412a081b2b6db6e8ad56267400", + "https://bcr.bazel.build/modules/rules_swift/3.6.1/MODULE.bazel": "5a68bb05b6b8044f8a7bd7184d56150189e7eca05522ce525517c234fed50ed8", + "https://bcr.bazel.build/modules/rules_swift/3.6.1/source.json": "9af2e521ffdb41961b6b9ac05293bd6519ee2fb52849a2be658a20b0221cb8db", + "https://bcr.bazel.build/modules/rules_zig/0.17.0/MODULE.bazel": "2c1c08adc986cfdb5fee97e21982cac846f8a20a6a8054ae9cab8b02721c2011", + "https://bcr.bazel.build/modules/rules_zig/0.17.0/source.json": "488f71f57c596ed9b7635ebeb28066b23c6fc1802c30749d06bdad261d7b04a2", + "https://bcr.bazel.build/modules/stardoc/0.5.0/MODULE.bazel": "f9f1f46ba8d9c3362648eea571c6f9100680efc44913618811b58cc9c02cd678", "https://bcr.bazel.build/modules/stardoc/0.5.1/MODULE.bazel": "1a05d92974d0c122f5ccf09291442580317cdd859f07a8655f1db9a60374f9f8", "https://bcr.bazel.build/modules/stardoc/0.5.3/MODULE.bazel": "c7f6948dae6999bf0db32c1858ae345f112cacf98f174c7a8bb707e41b974f1c", + "https://bcr.bazel.build/modules/stardoc/0.5.4/MODULE.bazel": "6569966df04610b8520957cb8e97cf2e9faac2c0309657c537ab51c16c18a2a4", + "https://bcr.bazel.build/modules/stardoc/0.5.6/MODULE.bazel": "c43dabc564990eeab55e25ed61c07a1aadafe9ece96a4efabb3f8bf9063b71ef", "https://bcr.bazel.build/modules/stardoc/0.6.2/MODULE.bazel": "7060193196395f5dd668eda046ccbeacebfd98efc77fed418dbe2b82ffaa39fd", - "https://bcr.bazel.build/modules/stardoc/0.6.2/source.json": "d2ff8063b63b4a85e65fe595c4290f99717434fa9f95b4748a79a7d04dfed349", + "https://bcr.bazel.build/modules/stardoc/0.7.0/MODULE.bazel": "05e3d6d30c099b6770e97da986c53bd31844d7f13d41412480ea265ac9e8079c", + "https://bcr.bazel.build/modules/stardoc/0.7.1/MODULE.bazel": "3548faea4ee5dda5580f9af150e79d0f6aea934fc60c1cc50f4efdd9420759e7", + "https://bcr.bazel.build/modules/stardoc/0.7.2/MODULE.bazel": "fc152419aa2ea0f51c29583fab1e8c99ddefd5b3778421845606ee628629e0e5", + "https://bcr.bazel.build/modules/stardoc/0.7.2/source.json": "58b029e5e901d6802967754adf0a9056747e8176f017cfe3607c0851f4d42216", + "https://bcr.bazel.build/modules/swift_argument_parser/1.3.1.1/MODULE.bazel": "5e463fbfba7b1701d957555ed45097d7f984211330106ccd1352c6e0af0dcf91", + "https://bcr.bazel.build/modules/swift_argument_parser/1.3.1.2/MODULE.bazel": "75aab2373a4bbe2a1260b9bf2a1ebbdbf872d3bd36f80bff058dccd82e89422f", + "https://bcr.bazel.build/modules/swift_argument_parser/1.3.1.2/source.json": "5fba48bbe0ba48761f9e9f75f92876cafb5d07c0ce059cc7a8027416de94a05b", + "https://bcr.bazel.build/modules/tar.bzl/0.10.4/MODULE.bazel": "e8f9ff79199e8d9eaad7f1b0a77ad74b30bb82d794b87d8ca942bead5de83ae9", + "https://bcr.bazel.build/modules/tar.bzl/0.10.4/source.json": "20143442376c03426f6135292ba02d825cb75308aa47e6bf42dd4cc5a435c2ff", + "https://bcr.bazel.build/modules/tar.bzl/0.2.1/MODULE.bazel": "52d1c00a80a8cc67acbd01649e83d8dd6a9dc426a6c0b754a04fe8c219c76468", + "https://bcr.bazel.build/modules/tar.bzl/0.5.1/MODULE.bazel": "7c2eb3dcfc53b0f3d6f9acdfd911ca803eaf92aadf54f8ca6e4c1f3aee288351", + "https://bcr.bazel.build/modules/tar.bzl/0.5.5/MODULE.bazel": "4bfab9bbc7a1966c2c5f7371f5848f5e2d27c465951b4435adc9aaf00ed681da", + "https://bcr.bazel.build/modules/tar.bzl/0.6.0/MODULE.bazel": "a3584b4edcfafcabd9b0ef9819808f05b372957bbdff41601429d5fd0aac2e7c", "https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "7298990c00040a0e2f121f6c32544bab27d4452f80d9ce51349b1a28f3005c43", - "https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/source.json": "f1ef7d3f9e0e26d4b23d1c39b5f5de71f584dd7d1b4ef83d9bbba6ec7a6a6459", + "https://bcr.bazel.build/modules/upb/0.0.0-20230516-61a97ef/MODULE.bazel": "c0df5e35ad55e264160417fd0875932ee3c9dda63d9fccace35ac62f45e1b6f9", + "https://bcr.bazel.build/modules/windows_support/0.4.1/MODULE.bazel": "2287f95ac65c48ea05197f626a9b5711ea3c88cea61f2a5782882bdd55fdeb51", + "https://bcr.bazel.build/modules/windows_support/0.4.1/source.json": "bdb72e6717b00135e0a4a8512b3f6b626df0baea0bde74bf7312c7484f791287", + "https://bcr.bazel.build/modules/with_cfg.bzl/0.12.0/MODULE.bazel": "b573395fe63aef4299ba095173e2f62ccfee5ad9bbf7acaa95dba73af9fc2b38", + "https://bcr.bazel.build/modules/with_cfg.bzl/0.14.1/MODULE.bazel": "aa0ef3f6c67dd35db7ac38c76785dc02f48b53f34e16b20487e70f51b5324d0a", + "https://bcr.bazel.build/modules/with_cfg.bzl/0.14.4/MODULE.bazel": "be2276f83878525d8f228b9ec777daee5eb7baba6100dad45081510d0cec35e3", + "https://bcr.bazel.build/modules/with_cfg.bzl/0.14.4/source.json": "63d661a0c144ca74fd8b0aa61e458c7627554c7d4a8e0a4d1ce00975437c776a", + "https://bcr.bazel.build/modules/xds/0.0.0-20240423-555b57e/MODULE.bazel": "cea509976a77e34131411684ef05a1d6ad194dd71a8d5816643bc5b0af16dc0f", + "https://bcr.bazel.build/modules/xds/0.0.0-20251210-ee656c7/MODULE.bazel": "03e1c50b16fc2403de5dd98ff8e2dc0b404865740434b8705c7c0853241f943d", + "https://bcr.bazel.build/modules/xds/0.0.0-20251210-ee656c7/source.json": "69b87f3bff38ba29399c012f3e3fc0f239ac4e5c1ae66ebd9481be6d8461da55", + "https://bcr.bazel.build/modules/xz/5.4.5.bcr.9/MODULE.bazel": "ff7cf8a5334a96b7b9ffcbe73418d45929ce125efebac39c1ee7d14bd2ae7a9b", + "https://bcr.bazel.build/modules/xz/5.4.5.bcr.9/source.json": "abf61f3c1630eab7b81722f710060b6f31e7c160511bcbfaad74bb4385883051", + "https://bcr.bazel.build/modules/yq.bzl/0.1.1/MODULE.bazel": "9039681f9bcb8958ee2c87ffc74bdafba9f4369096a2b5634b88abc0eaefa072", + "https://bcr.bazel.build/modules/yq.bzl/0.3.1/MODULE.bazel": "9bcb7151b3cd4681b89d350530eaf7b45e32a44dda94843b8932b0cb1cd4594a", + "https://bcr.bazel.build/modules/yq.bzl/0.3.1/source.json": "f0b0f204a2a6b0e34b4c9541efe8c04f2ef1af65948daa784eccea738b21dbd2", + "https://bcr.bazel.build/modules/zlib-ng/2.3.3.bcr.2/MODULE.bazel": "86130924115bf8dfc4aba286fb49a2ec8d32430a21771db9ccf5381acf0d10cc", + "https://bcr.bazel.build/modules/zlib-ng/2.3.3.bcr.2/source.json": "7aa4112eb2fbb99757e97cf0b7a4e2bdb3464b2b3cfb351bc807ffcb34bd349f", + "https://bcr.bazel.build/modules/zlib-ng/2.3.3/MODULE.bazel": "0afa781a5f354b4fd811a5c9a086e111daf50e09d8e98ab1cf7c46eeacc33aac", "https://bcr.bazel.build/modules/zlib/1.2.11/MODULE.bazel": "07b389abc85fdbca459b69e2ec656ae5622873af3f845e1c9d80fe179f3effa0", "https://bcr.bazel.build/modules/zlib/1.2.12/MODULE.bazel": "3b1a8834ada2a883674be8cbd36ede1b6ec481477ada359cd2d3ddc562340b27", - "https://bcr.bazel.build/modules/zlib/1.2.13/MODULE.bazel": "aa6deb1b83c18ffecd940c4119aff9567cd0a671d7bba756741cb2ef043a29d5", - "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.3/MODULE.bazel": "af322bc08976524477c79d1e45e241b6efbeb918c497e8840b8ab116802dda79", - "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.3/source.json": "2be409ac3c7601245958cd4fcdff4288be79ed23bd690b4b951f500d54ee6e7d", - "https://bcr.bazel.build/modules/zstd/1.5.6/MODULE.bazel": "471ebe7d3cdd8c6469390fcf623eb4779ff55fbee0a87f1dc57a1def468b96d4", - "https://bcr.bazel.build/modules/zstd/1.5.6/source.json": "02010c3333fc89b44fe861db049968decb6e688411f7f9d4f6791d74f9adfb51" + "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.1/MODULE.bazel": "6a9fe6e3fc865715a7be9823ce694ceb01e364c35f7a846bf0d2b34762bc066b", + "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.5/MODULE.bazel": "eec517b5bbe5492629466e11dae908d043364302283de25581e3eb944326c4ca", + "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.5/source.json": "22bc55c47af97246cfc093d0acf683a7869377de362b5d1c552c2c2e16b7a806", + "https://bcr.bazel.build/modules/zlib/1.3.1/MODULE.bazel": "751c9940dcfe869f5f7274e1295422a34623555916eb98c174c1e945594bf198", + "https://bcr.bazel.build/modules/zstd/1.5.7.bcr.1/MODULE.bazel": "c5977176dd8555be7a9d598512ae0cae11831259c06f00a5e5e0037d5db4e3f5", + "https://bcr.bazel.build/modules/zstd/1.5.7.bcr.1/source.json": "aa95e0b5aac9d80195b9047d223beaf26b1948be55d49f0e803e180e9ccc6e75" }, "selectedYankedVersions": {}, "moduleExtensions": { - "@@apple_support~//crosstool:setup.bzl%apple_cc_configure_extension": { + "@@aspect_rules_py+//py:extensions.bzl%py_tools": { "general": { - "bzlTransitiveDigest": "hDHJiBbKme6a+N8oiSQcVnU1v5B7tHMjJaAzS6GFfPc=", - "usagesDigest": "atH8xayh8CVhGZG9cm/kh7fV7XwOYgQy6Zhe1AzBH3g=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, + "bzlTransitiveDigest": "6o8lyMtImPc3Si+zcGss6WH8IY80mTWhC0MC3r0ra8Y=", + "usagesDigest": "9BiP3ASHlGl4doGhzXJwK17taNzv/q6x/IZ4Hb6LxNY=", + "recordedInputs": [ + "REPO_MAPPING:aspect_rules_py+,aspect_tools_telemetry_report aspect_tools_telemetry++telemetry+aspect_tools_telemetry_report", + "REPO_MAPPING:aspect_rules_py+,bazel_skylib bazel_skylib+", + "REPO_MAPPING:aspect_rules_py+,bazel_tools bazel_tools" + ], "generatedRepoSpecs": { - "local_config_apple_cc": { - "bzlFile": "@@apple_support~//crosstool:setup.bzl", - "ruleClassName": "_apple_cc_autoconf", - "attributes": {} + "rules_py_tools.darwin_amd64": { + "repoRuleId": "@@aspect_rules_py+//py/private/toolchain:repo.bzl%prebuilt_tool_repo", + "attributes": { + "platform": "darwin_amd64" + } }, - "local_config_apple_cc_toolchains": { - "bzlFile": "@@apple_support~//crosstool:setup.bzl", - "ruleClassName": "_apple_cc_autoconf_toolchains", - "attributes": {} + "rules_py_tools.darwin_arm64": { + "repoRuleId": "@@aspect_rules_py+//py/private/toolchain:repo.bzl%prebuilt_tool_repo", + "attributes": { + "platform": "darwin_arm64" + } + }, + "rules_py_tools.linux_amd64": { + "repoRuleId": "@@aspect_rules_py+//py/private/toolchain:repo.bzl%prebuilt_tool_repo", + "attributes": { + "platform": "linux_amd64" + } + }, + "rules_py_tools.linux_arm64": { + "repoRuleId": "@@aspect_rules_py+//py/private/toolchain:repo.bzl%prebuilt_tool_repo", + "attributes": { + "platform": "linux_arm64" + } + }, + "rules_py_tools": { + "repoRuleId": "@@aspect_rules_py+//py/private/toolchain:repo.bzl%toolchains_repo", + "attributes": { + "user_repository_name": "rules_py_tools" + } + }, + "rules_py_pex_2_3_1": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", + "attributes": { + "urls": [ + "https://files.pythonhosted.org/packages/e7/d0/fbda2a4d41d62d86ce53f5ae4fbaaee8c34070f75bb7ca009090510ae874/pex-2.3.1-py2.py3-none-any.whl" + ], + "sha256": "64692a5bf6f298403aab930d22f0d836ae4736c5bc820e262e9092fe8c56f830", + "downloaded_file_path": "pex-2.3.1-py2.py3-none-any.whl" + } } - }, - "recordedRepoMappingEntries": [ - [ - "apple_support~", - "bazel_tools", - "bazel_tools" - ] - ] + } } }, - "@@aspect_bazel_lib~//lib:extensions.bzl%toolchains": { + "@@aspect_tools_telemetry+//:extension.bzl%telemetry": { "general": { - "bzlTransitiveDigest": "p8GR5EeSAP5RvoQ0821nQQ9nelAyTrNoUULFFBUdGlU=", - "usagesDigest": "JTRHwJCDIFLoz4veHZ+G6HnnCkU60EauNsYpX/JQlYc=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, + "bzlTransitiveDigest": "MePriaXmQNSqUfE+YQvEtzBc2bU1mjITINIffFiZYgo=", + "usagesDigest": "pyph4lSDn5CHIn4s2mtKJTkRaLV/o2zIFgnrpufr/8o=", + "recordedInputs": [ + "REPO_MAPPING:aspect_tools_telemetry+,bazel_lib bazel_lib+", + "REPO_MAPPING:aspect_tools_telemetry+,bazel_skylib bazel_skylib+" + ], "generatedRepoSpecs": { - "expand_template_windows_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:expand_template_toolchain.bzl", - "ruleClassName": "expand_template_platform_repo", + "aspect_tools_telemetry_report": { + "repoRuleId": "@@aspect_tools_telemetry+//:extension.bzl%tel_repository", "attributes": { - "platform": "windows_amd64" + "deps": { + "aspect_rules_py": "1.8.4", + "aspect_tools_telemetry": "0.3.3" + } + } + } + } + } + }, + "@@rules_android+//rules/android_sdk_repository:rule.bzl%android_sdk_repository_extension": { + "general": { + "bzlTransitiveDigest": "aLDGLKolYFwLsBssOQfwOs+bo3Aof0yKkW0HQRPX3ms=", + "usagesDigest": "iEGI2aNDMkHt9LXCdViLNUUOslpiVj2DrevWWXZEFnU=", + "recordedInputs": [], + "generatedRepoSpecs": { + "androidsdk": { + "repoRuleId": "@@rules_android+//rules/android_sdk_repository:rule.bzl%_android_sdk_repository", + "attributes": {} + } + } + } + }, + "@@rules_oci+//oci:extensions.bzl%oci": { + "general": { + "bzlTransitiveDigest": "cbfE3+b9HnO8wGMMbdSzaHKjcON7SL8gb4fKTNiVHug=", + "usagesDigest": "JLbamplho20rviuebatMGqnQnaoj32ewCzLuRVusGoU=", + "recordedInputs": [ + "REPO_MAPPING:aspect_bazel_lib+,bazel_tools bazel_tools", + "REPO_MAPPING:bazel_features+,bazel_tools bazel_tools", + "REPO_MAPPING:rules_oci+,aspect_bazel_lib aspect_bazel_lib+", + "REPO_MAPPING:rules_oci+,bazel_features bazel_features+", + "REPO_MAPPING:rules_oci+,bazel_skylib bazel_skylib+" + ], + "generatedRepoSpecs": { + "distroless_cc_debian12_linux_amd64": { + "repoRuleId": "@@rules_oci+//oci/private:pull.bzl%oci_pull", + "attributes": { + "www_authenticate_challenges": {}, + "scheme": "https", + "registry": "gcr.io", + "repository": "distroless/cc-debian12", + "identifier": "sha256:eccec5274132c1be0ce5d2c8e6fe41033e64af5e987ccee9007826e4c012069d", + "platform": "linux/amd64", + "target_name": "distroless_cc_debian12_linux_amd64", + "bazel_tags": [] } }, - "copy_to_directory_windows_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_to_directory_toolchain.bzl", - "ruleClassName": "copy_to_directory_platform_repo", + "distroless_cc_debian12": { + "repoRuleId": "@@rules_oci+//oci/private:pull.bzl%oci_alias", "attributes": { - "platform": "windows_amd64" + "target_name": "distroless_cc_debian12", + "www_authenticate_challenges": {}, + "scheme": "https", + "registry": "gcr.io", + "repository": "distroless/cc-debian12", + "identifier": "sha256:eccec5274132c1be0ce5d2c8e6fe41033e64af5e987ccee9007826e4c012069d", + "platforms": { + "@@platforms//cpu:x86_64": "@distroless_cc_debian12_linux_amd64" + }, + "bzlmod_repository": "distroless_cc_debian12", + "reproducible": true } }, - "jq_darwin_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:jq_toolchain.bzl", - "ruleClassName": "jq_platform_repo", + "debian_12_linux_amd64": { + "repoRuleId": "@@rules_oci+//oci/private:pull.bzl%oci_pull", + "attributes": { + "www_authenticate_challenges": {}, + "scheme": "https", + "registry": "mirror.gcr.io", + "repository": "library/debian", + "identifier": "sha256:bc960ef50e6feed90686c593361df158517556ed1d2d98e5d1df3724024e0f49", + "platform": "linux/amd64", + "target_name": "debian_12_linux_amd64", + "bazel_tags": [] + } + }, + "debian_12": { + "repoRuleId": "@@rules_oci+//oci/private:pull.bzl%oci_alias", + "attributes": { + "target_name": "debian_12", + "www_authenticate_challenges": {}, + "scheme": "https", + "registry": "mirror.gcr.io", + "repository": "library/debian", + "identifier": "sha256:bc960ef50e6feed90686c593361df158517556ed1d2d98e5d1df3724024e0f49", + "platforms": { + "@@platforms//cpu:x86_64": "@debian_12_linux_amd64" + }, + "bzlmod_repository": "debian_12", + "reproducible": true + } + }, + "oci_crane_darwin_amd64": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", "attributes": { "platform": "darwin_amd64", - "version": "1.7" + "crane_version": "v0.18.0" } }, - "copy_to_directory_freebsd_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_to_directory_toolchain.bzl", - "ruleClassName": "copy_to_directory_platform_repo", - "attributes": { - "platform": "freebsd_amd64" - } - }, - "expand_template_linux_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:expand_template_toolchain.bzl", - "ruleClassName": "expand_template_platform_repo", - "attributes": { - "platform": "linux_amd64" - } - }, - "jq_linux_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:jq_toolchain.bzl", - "ruleClassName": "jq_platform_repo", - "attributes": { - "platform": "linux_arm64", - "version": "1.7" - } - }, - "coreutils_darwin_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:coreutils_toolchain.bzl", - "ruleClassName": "coreutils_platform_repo", + "oci_crane_darwin_arm64": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", "attributes": { "platform": "darwin_arm64", - "version": "0.0.26" + "crane_version": "v0.18.0" } }, - "copy_to_directory_linux_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_to_directory_toolchain.bzl", - "ruleClassName": "copy_to_directory_platform_repo", - "attributes": { - "platform": "linux_arm64" - } - }, - "bsd_tar_linux_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:tar_toolchain.bzl", - "ruleClassName": "bsdtar_binary_repo", - "attributes": { - "platform": "linux_arm64" - } - }, - "copy_directory_darwin_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_directory_toolchain.bzl", - "ruleClassName": "copy_directory_platform_repo", - "attributes": { - "platform": "darwin_amd64" - } - }, - "coreutils_darwin_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:coreutils_toolchain.bzl", - "ruleClassName": "coreutils_platform_repo", - "attributes": { - "platform": "darwin_amd64", - "version": "0.0.26" - } - }, - "coreutils_linux_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:coreutils_toolchain.bzl", - "ruleClassName": "coreutils_platform_repo", + "oci_crane_linux_arm64": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", "attributes": { "platform": "linux_arm64", - "version": "0.0.26" + "crane_version": "v0.18.0" } }, - "zstd_linux_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:zstd_toolchain.bzl", - "ruleClassName": "zstd_binary_repo", + "oci_crane_linux_armv6": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", "attributes": { - "platform": "linux_arm64" + "platform": "linux_armv6", + "crane_version": "v0.18.0" } }, - "yq_linux_s390x": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:yq_toolchain.bzl", - "ruleClassName": "yq_platform_repo", + "oci_crane_linux_i386": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", + "attributes": { + "platform": "linux_i386", + "crane_version": "v0.18.0" + } + }, + "oci_crane_linux_s390x": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", "attributes": { "platform": "linux_s390x", - "version": "4.25.2" + "crane_version": "v0.18.0" } }, - "yq": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:yq_toolchain.bzl", - "ruleClassName": "yq_host_alias_repo", - "attributes": {} + "oci_crane_linux_amd64": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", + "attributes": { + "platform": "linux_amd64", + "crane_version": "v0.18.0" + } }, - "expand_template_darwin_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:expand_template_toolchain.bzl", - "ruleClassName": "expand_template_platform_repo", + "oci_crane_windows_armv6": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", + "attributes": { + "platform": "windows_armv6", + "crane_version": "v0.18.0" + } + }, + "oci_crane_windows_amd64": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", + "attributes": { + "platform": "windows_amd64", + "crane_version": "v0.18.0" + } + }, + "oci_crane_toolchains": { + "repoRuleId": "@@rules_oci+//oci/private:toolchains_repo.bzl%toolchains_repo", + "attributes": { + "toolchain_type": "@rules_oci//oci:crane_toolchain_type", + "toolchain": "@oci_crane_{platform}//:crane_toolchain" + } + }, + "oci_regctl_darwin_amd64": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", "attributes": { "platform": "darwin_amd64" } }, - "copy_directory_linux_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_directory_toolchain.bzl", - "ruleClassName": "copy_directory_platform_repo", + "oci_regctl_darwin_arm64": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", + "attributes": { + "platform": "darwin_arm64" + } + }, + "oci_regctl_linux_arm64": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", + "attributes": { + "platform": "linux_arm64" + } + }, + "oci_regctl_linux_s390x": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", + "attributes": { + "platform": "linux_s390x" + } + }, + "oci_regctl_linux_amd64": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", "attributes": { "platform": "linux_amd64" } }, - "jq_darwin_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:jq_toolchain.bzl", - "ruleClassName": "jq_platform_repo", + "oci_regctl_windows_amd64": { + "repoRuleId": "@@rules_oci+//oci:repositories.bzl%regctl_repositories", "attributes": { - "platform": "darwin_arm64", - "version": "1.7" + "platform": "windows_amd64" } }, + "oci_regctl_toolchains": { + "repoRuleId": "@@rules_oci+//oci/private:toolchains_repo.bzl%toolchains_repo", + "attributes": { + "toolchain_type": "@rules_oci//oci:regctl_toolchain_type", + "toolchain": "@oci_regctl_{platform}//:regctl_toolchain" + } + } + }, + "moduleExtensionMetadata": { + "explicitRootModuleDirectDeps": [ + "distroless_cc_debian12", + "distroless_cc_debian12_linux_amd64", + "debian_12", + "debian_12_linux_amd64" + ], + "explicitRootModuleDirectDevDeps": [], + "useAllRepos": "NO", + "reproducible": false + } + } + }, + "@@rules_python+//python/uv:uv.bzl%uv": { + "general": { + "bzlTransitiveDigest": "ZtcPw94gqLP8+ocBKipnnyLIivxEC/OUYen7Kx7hnEA=", + "usagesDigest": "Ucfum/vsVnlkhtglTERDLTj+bBeG7dcpbV9h0LP9HOQ=", + "recordedInputs": [ + "REPO_MAPPING:rules_python+,bazel_tools bazel_tools", + "REPO_MAPPING:rules_python+,platforms platforms" + ], + "generatedRepoSpecs": { + "uv_0_9_30_aarch64_apple_darwin": { + "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", + "attributes": { + "version": "0.9.30", + "platform": "aarch64-apple-darwin", + "urls": [ + "https://github.com/astral-sh/uv/releases/download/0.9.30/uv-aarch64-apple-darwin.tar.gz" + ], + "sha256": "03a5d9ec7f7d588446b2ec226d13ff6300055e55365eca8f3fab39f342b0e805" + } + }, + "uv_0_9_30_aarch64_unknown_linux_gnu": { + "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", + "attributes": { + "version": "0.9.30", + "platform": "aarch64-unknown-linux-gnu", + "urls": [ + "https://github.com/astral-sh/uv/releases/download/0.9.30/uv-aarch64-unknown-linux-gnu.tar.gz" + ], + "sha256": "6aadf3c71600d594e16dabf382cc15282ead4c5ca768599b6bcb43c5004d9aa8" + } + }, + "uv_0_9_30_powerpc64_unknown_linux_gnu": { + "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", + "attributes": { + "version": "0.9.30", + "platform": "powerpc64-unknown-linux-gnu", + "urls": [ + "https://github.com/astral-sh/uv/releases/download/0.9.30/uv-powerpc64-unknown-linux-gnu.tar.gz" + ], + "sha256": "ac4cd1a021462885932f6023b005a4835cca4c72bb60dec186ee2be4b60dca6f" + } + }, + "uv_0_9_30_powerpc64le_unknown_linux_gnu": { + "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", + "attributes": { + "version": "0.9.30", + "platform": "powerpc64le-unknown-linux-gnu", + "urls": [ + "https://github.com/astral-sh/uv/releases/download/0.9.30/uv-powerpc64le-unknown-linux-gnu.tar.gz" + ], + "sha256": "73b8cbc560c6b2fa205358365d4e174abdf50cfcf57dc36a447572c56eba5ae4" + } + }, + "uv_0_9_30_s390x_unknown_linux_gnu": { + "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", + "attributes": { + "version": "0.9.30", + "platform": "s390x-unknown-linux-gnu", + "urls": [ + "https://github.com/astral-sh/uv/releases/download/0.9.30/uv-s390x-unknown-linux-gnu.tar.gz" + ], + "sha256": "b35975bb9e5c2c418b428d0316cc6e3c7a6eff710c69212be14005c192f54516" + } + }, + "uv_0_9_30_x86_64_apple_darwin": { + "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", + "attributes": { + "version": "0.9.30", + "platform": "x86_64-apple-darwin", + "urls": [ + "https://github.com/astral-sh/uv/releases/download/0.9.30/uv-x86_64-apple-darwin.tar.gz" + ], + "sha256": "ce069bf750567e9a4a31d6e285d1eae75d444d8a281409b641235903943b7681" + } + }, + "uv_0_9_30_x86_64_pc_windows_msvc": { + "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", + "attributes": { + "version": "0.9.30", + "platform": "x86_64-pc-windows-msvc", + "urls": [ + "https://github.com/astral-sh/uv/releases/download/0.9.30/uv-x86_64-pc-windows-msvc.zip" + ], + "sha256": "875981be7908295937dee09532bb66d576986d4f223259e171b0c767c885897a" + } + }, + "uv_0_9_30_x86_64_unknown_linux_gnu": { + "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", + "attributes": { + "version": "0.9.30", + "platform": "x86_64-unknown-linux-gnu", + "urls": [ + "https://github.com/astral-sh/uv/releases/download/0.9.30/uv-x86_64-unknown-linux-gnu.tar.gz" + ], + "sha256": "8b3762374972daa7a74bbc6896cc73229ca69a07403dd9f9ea3805a51ffd7582" + } + }, + "uv": { + "repoRuleId": "@@rules_python+//python/uv/private:uv_toolchains_repo.bzl%uv_toolchains_repo", + "attributes": { + "toolchain_type": "'@@rules_python+//python/uv:uv_toolchain_type'", + "toolchain_names": [ + "0_9_30_aarch64_apple_darwin", + "0_9_30_aarch64_unknown_linux_gnu", + "0_9_30_powerpc64_unknown_linux_gnu", + "0_9_30_powerpc64le_unknown_linux_gnu", + "0_9_30_s390x_unknown_linux_gnu", + "0_9_30_x86_64_apple_darwin", + "0_9_30_x86_64_pc_windows_msvc", + "0_9_30_x86_64_unknown_linux_gnu" + ], + "toolchain_implementations": { + "0_9_30_aarch64_apple_darwin": "@uv_0_9_30_aarch64_apple_darwin//:uv_toolchain", + "0_9_30_aarch64_unknown_linux_gnu": "@uv_0_9_30_aarch64_unknown_linux_gnu//:uv_toolchain", + "0_9_30_powerpc64_unknown_linux_gnu": "@uv_0_9_30_powerpc64_unknown_linux_gnu//:uv_toolchain", + "0_9_30_powerpc64le_unknown_linux_gnu": "@uv_0_9_30_powerpc64le_unknown_linux_gnu//:uv_toolchain", + "0_9_30_s390x_unknown_linux_gnu": "@uv_0_9_30_s390x_unknown_linux_gnu//:uv_toolchain", + "0_9_30_x86_64_apple_darwin": "@uv_0_9_30_x86_64_apple_darwin//:uv_toolchain", + "0_9_30_x86_64_pc_windows_msvc": "@uv_0_9_30_x86_64_pc_windows_msvc//:uv_toolchain", + "0_9_30_x86_64_unknown_linux_gnu": "@uv_0_9_30_x86_64_unknown_linux_gnu//:uv_toolchain" + }, + "toolchain_compatible_with": { + "0_9_30_aarch64_apple_darwin": [ + "'@@platforms//os:macos'", + "'@@platforms//cpu:aarch64'" + ], + "0_9_30_aarch64_unknown_linux_gnu": [ + "'@@platforms//os:linux'", + "'@@platforms//cpu:aarch64'" + ], + "0_9_30_powerpc64_unknown_linux_gnu": [ + "'@@platforms//os:linux'", + "'@@platforms//cpu:ppc'" + ], + "0_9_30_powerpc64le_unknown_linux_gnu": [ + "'@@platforms//os:linux'", + "'@@platforms//cpu:ppc64le'" + ], + "0_9_30_s390x_unknown_linux_gnu": [ + "'@@platforms//os:linux'", + "'@@platforms//cpu:s390x'" + ], + "0_9_30_x86_64_apple_darwin": [ + "'@@platforms//os:macos'", + "'@@platforms//cpu:x86_64'" + ], + "0_9_30_x86_64_pc_windows_msvc": [ + "'@@platforms//os:windows'", + "'@@platforms//cpu:x86_64'" + ], + "0_9_30_x86_64_unknown_linux_gnu": [ + "'@@platforms//os:linux'", + "'@@platforms//cpu:x86_64'" + ] + }, + "toolchain_target_settings": {} + } + } + } + } + }, + "@@rules_zig+//zig:extensions.bzl%zig": { + "general": { + "bzlTransitiveDigest": "hPjDj28qKZsSFAE45ZtQPcgDkeRXAcwxiuAQyeEF7ng=", + "usagesDigest": "FDLiNUluzC0WBatQ/5WFpxzyQP4ubA/DUgTwOzYLGw4=", + "recordedInputs": [ + "REPO_MAPPING:rules_zig+,bazel_skylib bazel_skylib+", + "REPO_MAPPING:rules_zig+,bazel_tools bazel_tools", + "FILE:@@//bazel/zig_index.json 387d1ab130896456fb092fb08bfaf0c57ccba854121955c752052600d1643d37", + "FILE:@@rules_zig+//zig/private/versions.json a33370258a702099b35362d4220fe8f02c869c2b845abcbf0aa78200abab097a" + ], + "generatedRepoSpecs": { + "zig_0.17.0_aarch64-linux": { + "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", + "attributes": { + "url": "https://ziglang.org/download/0.17.0/zig-aarch64-linux-0.17.0.tar.xz", + "mirrors": [ + "https://pkg.machengine.org/zig", + "https://zigmirror.hryx.net/zig", + "https://zig.linus.dev/zig", + "https://zig.squirl.dev", + "https://zig.florent.dev", + "https://zig.mirror.mschae23.de/zig", + "https://zigmirror.meox.dev", + "https://ziglang.freetls.fastly.net", + "https://zig.tilok.dev", + "https://zig-mirror.tsimnet.eu/zig", + "https://zig.karearl.com/zig", + "https://pkg.earth/zig", + "https://fs.liujiacai.net/zigbuilds", + "https://ziglang.org/builds/", + "https://pkg.machengine.org/zig", + "https://zigmirror.hryx.net/zig", + "https://zig.linus.dev/zig", + "https://zig.squirl.dev", + "https://zig.florent.dev", + "https://zig.mirror.mschae23.de/zig", + "https://zigmirror.meox.dev", + "https://ziglang.freetls.fastly.net", + "https://zig.tilok.dev", + "https://zig-mirror.tsimnet.eu/zig", + "https://zig.karearl.com/zig", + "https://pkg.earth/zig", + "https://fs.liujiacai.net/zigbuilds" + ], + "sha256": "9e8d11661d4ae3bd57702a3832781e23ad151dde5798e16a5ccd503f65234ff8", + "zig_version": "0.17.0", + "platform": "aarch64-linux" + } + }, + "zig_0.17.0_aarch64-macos": { + "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", + "attributes": { + "url": "https://ziglang.org/download/0.17.0/zig-aarch64-macos-0.17.0.tar.xz", + "mirrors": [ + "https://pkg.machengine.org/zig", + "https://zigmirror.hryx.net/zig", + "https://zig.linus.dev/zig", + "https://zig.squirl.dev", + "https://zig.florent.dev", + "https://zig.mirror.mschae23.de/zig", + "https://zigmirror.meox.dev", + "https://ziglang.freetls.fastly.net", + "https://zig.tilok.dev", + "https://zig-mirror.tsimnet.eu/zig", + "https://zig.karearl.com/zig", + "https://pkg.earth/zig", + "https://fs.liujiacai.net/zigbuilds", + "https://ziglang.org/builds/", + "https://pkg.machengine.org/zig", + "https://zigmirror.hryx.net/zig", + "https://zig.linus.dev/zig", + "https://zig.squirl.dev", + "https://zig.florent.dev", + "https://zig.mirror.mschae23.de/zig", + "https://zigmirror.meox.dev", + "https://ziglang.freetls.fastly.net", + "https://zig.tilok.dev", + "https://zig-mirror.tsimnet.eu/zig", + "https://zig.karearl.com/zig", + "https://pkg.earth/zig", + "https://fs.liujiacai.net/zigbuilds" + ], + "sha256": "b607e9b9234790a008116ae5bdb71c6243b84b9fb42a53a9e70fde41c06c536a", + "zig_version": "0.17.0", + "platform": "aarch64-macos" + } + }, + "zig_0.17.0_aarch64-windows": { + "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", + "attributes": { + "url": "https://ziglang.org/download/0.17.0/zig-aarch64-windows-0.17.0.zip", + "mirrors": [ + "https://pkg.machengine.org/zig", + "https://zigmirror.hryx.net/zig", + "https://zig.linus.dev/zig", + "https://zig.squirl.dev", + "https://zig.florent.dev", + "https://zig.mirror.mschae23.de/zig", + "https://zigmirror.meox.dev", + "https://ziglang.freetls.fastly.net", + "https://zig.tilok.dev", + "https://zig-mirror.tsimnet.eu/zig", + "https://zig.karearl.com/zig", + "https://pkg.earth/zig", + "https://fs.liujiacai.net/zigbuilds", + "https://ziglang.org/builds/", + "https://pkg.machengine.org/zig", + "https://zigmirror.hryx.net/zig", + "https://zig.linus.dev/zig", + "https://zig.squirl.dev", + "https://zig.florent.dev", + "https://zig.mirror.mschae23.de/zig", + "https://zigmirror.meox.dev", + "https://ziglang.freetls.fastly.net", + "https://zig.tilok.dev", + "https://zig-mirror.tsimnet.eu/zig", + "https://zig.karearl.com/zig", + "https://pkg.earth/zig", + "https://fs.liujiacai.net/zigbuilds" + ], + "sha256": "0a59d91fa1cb40cf068e9b0954434ce973500c7a2ea749f1e01af62cdab52d26", + "zig_version": "0.17.0", + "platform": "aarch64-windows" + } + }, + "zig_0.17.0_x86_64-linux": { + "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", + "attributes": { + "url": "https://ziglang.org/download/0.17.0/zig-x86_64-linux-0.17.0.tar.xz", + "mirrors": [ + "https://pkg.machengine.org/zig", + "https://zigmirror.hryx.net/zig", + "https://zig.linus.dev/zig", + "https://zig.squirl.dev", + "https://zig.florent.dev", + "https://zig.mirror.mschae23.de/zig", + "https://zigmirror.meox.dev", + "https://ziglang.freetls.fastly.net", + "https://zig.tilok.dev", + "https://zig-mirror.tsimnet.eu/zig", + "https://zig.karearl.com/zig", + "https://pkg.earth/zig", + "https://fs.liujiacai.net/zigbuilds", + "https://ziglang.org/builds/", + "https://pkg.machengine.org/zig", + "https://zigmirror.hryx.net/zig", + "https://zig.linus.dev/zig", + "https://zig.squirl.dev", + "https://zig.florent.dev", + "https://zig.mirror.mschae23.de/zig", + "https://zigmirror.meox.dev", + "https://ziglang.freetls.fastly.net", + "https://zig.tilok.dev", + "https://zig-mirror.tsimnet.eu/zig", + "https://zig.karearl.com/zig", + "https://pkg.earth/zig", + "https://fs.liujiacai.net/zigbuilds" + ], + "sha256": "1cbe9df9f27e6b78d14ccbca43b6703a404ef79ef1c463de901d7f088d4e2026", + "zig_version": "0.17.0", + "platform": "x86_64-linux" + } + }, + "zig_0.17.0_x86_64-macos": { + "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", + "attributes": { + "url": "https://ziglang.org/download/0.17.0/zig-x86_64-macos-0.17.0.tar.xz", + "mirrors": [ + "https://pkg.machengine.org/zig", + "https://zigmirror.hryx.net/zig", + "https://zig.linus.dev/zig", + "https://zig.squirl.dev", + "https://zig.florent.dev", + "https://zig.mirror.mschae23.de/zig", + "https://zigmirror.meox.dev", + "https://ziglang.freetls.fastly.net", + "https://zig.tilok.dev", + "https://zig-mirror.tsimnet.eu/zig", + "https://zig.karearl.com/zig", + "https://pkg.earth/zig", + "https://fs.liujiacai.net/zigbuilds", + "https://ziglang.org/builds/", + "https://pkg.machengine.org/zig", + "https://zigmirror.hryx.net/zig", + "https://zig.linus.dev/zig", + "https://zig.squirl.dev", + "https://zig.florent.dev", + "https://zig.mirror.mschae23.de/zig", + "https://zigmirror.meox.dev", + "https://ziglang.freetls.fastly.net", + "https://zig.tilok.dev", + "https://zig-mirror.tsimnet.eu/zig", + "https://zig.karearl.com/zig", + "https://pkg.earth/zig", + "https://fs.liujiacai.net/zigbuilds" + ], + "sha256": "4f9a1c5269aa17ebda5e6d3c2b89d6cbf36f7d2b22a0306e9ab98f25f95529c6", + "zig_version": "0.17.0", + "platform": "x86_64-macos" + } + }, + "zig_0.17.0_x86_64-windows": { + "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", + "attributes": { + "url": "https://ziglang.org/download/0.17.0/zig-x86_64-windows-0.17.0.zip", + "mirrors": [ + "https://pkg.machengine.org/zig", + "https://zigmirror.hryx.net/zig", + "https://zig.linus.dev/zig", + "https://zig.squirl.dev", + "https://zig.florent.dev", + "https://zig.mirror.mschae23.de/zig", + "https://zigmirror.meox.dev", + "https://ziglang.freetls.fastly.net", + "https://zig.tilok.dev", + "https://zig-mirror.tsimnet.eu/zig", + "https://zig.karearl.com/zig", + "https://pkg.earth/zig", + "https://fs.liujiacai.net/zigbuilds", + "https://ziglang.org/builds/", + "https://pkg.machengine.org/zig", + "https://zigmirror.hryx.net/zig", + "https://zig.linus.dev/zig", + "https://zig.squirl.dev", + "https://zig.florent.dev", + "https://zig.mirror.mschae23.de/zig", + "https://zigmirror.meox.dev", + "https://ziglang.freetls.fastly.net", + "https://zig.tilok.dev", + "https://zig-mirror.tsimnet.eu/zig", + "https://zig.karearl.com/zig", + "https://pkg.earth/zig", + "https://fs.liujiacai.net/zigbuilds" + ], + "sha256": "b5663f69581dcf391293fbf16c06cb80d81d806545ce618b4d0bab7f0eb8c428", + "zig_version": "0.17.0", + "platform": "x86_64-windows" + } + }, + "zig_toolchains": { + "repoRuleId": "@@rules_zig+//zig/private/repo:toolchains_repo.bzl%toolchains_repo", + "attributes": { + "names": [ + "zig_0.17.0_aarch64-linux", + "zig_0.17.0_aarch64-macos", + "zig_0.17.0_aarch64-windows", + "zig_0.17.0_x86_64-linux", + "zig_0.17.0_x86_64-macos", + "zig_0.17.0_x86_64-windows" + ], + "labels": [ + "@zig_0.17.0_aarch64-linux//:zig_toolchain", + "@zig_0.17.0_aarch64-macos//:zig_toolchain", + "@zig_0.17.0_aarch64-windows//:zig_toolchain", + "@zig_0.17.0_x86_64-linux//:zig_toolchain", + "@zig_0.17.0_x86_64-macos//:zig_toolchain", + "@zig_0.17.0_x86_64-windows//:zig_toolchain" + ], + "zig_versions": [ + "0.17.0", + "0.17.0", + "0.17.0", + "0.17.0", + "0.17.0", + "0.17.0" + ], + "exec_lengths": [ + 3, + 3, + 3, + 3, + 3, + 3 + ], + "exec_constraints": [ + "@platforms//os:linux", + "@platforms//cpu:aarch64", + "'@@//platforms:local_execution'", + "@platforms//os:macos", + "@platforms//cpu:aarch64", + "'@@//platforms:local_execution'", + "@platforms//os:windows", + "@platforms//cpu:aarch64", + "'@@//platforms:local_execution'", + "@platforms//os:linux", + "@platforms//cpu:x86_64", + "'@@//platforms:local_execution'", + "@platforms//os:macos", + "@platforms//cpu:x86_64", + "'@@//platforms:local_execution'", + "@platforms//os:windows", + "@platforms//cpu:x86_64", + "'@@//platforms:local_execution'" + ], + "target_compatible_lengths": [ + 0, + 0, + 0, + 0, + 0, + 0 + ], + "target_compatible_constraints": [], + "target_settings_lengths": [ + 0, + 0, + 0, + 0, + 0, + 0 + ], + "target_settings": [] + } + } + } + } + }, + "@@rules_zig+//zig/zls:extensions.bzl%zls": { + "general": { + "bzlTransitiveDigest": "8UokkNRK4sShFNMJ3aDabiCS4Q/ldgChV/NYNszfmCg=", + "usagesDigest": "kOrP52crJGijpHNzo4S7msXvbGpfmzvkE4tuhpRWTsI=", + "recordedInputs": [ + "REPO_MAPPING:rules_zig+,bazel_skylib bazel_skylib+", + "REPO_MAPPING:rules_zig+,bazel_tools bazel_tools", + "FILE:@@rules_zig+//zig/zls/private/versions.json 127803289dfd79b5255db5f74be1d9a2f790f03292d6147009af702d888aba69" + ], + "generatedRepoSpecs": { + "zls_toolchains": { + "repoRuleId": "@@rules_zig+//zig/zls/private/repo:toolchains_repo.bzl%toolchains_repo", + "attributes": { + "names": [], + "labels": [], + "zig_versions": [], + "exec_lengths": [], + "exec_constraints": [] + } + } + } + } + }, + "@@yq.bzl+//yq:extensions.bzl%yq": { + "general": { + "bzlTransitiveDigest": "iWlctm1cJYZLESuQEBygCQc6adq+oIRMb4Dmk0rYxkA=", + "usagesDigest": "1CP5kLHwnlFZ3obmnV0eEOp+80JltkcSimeCHXe8/+E=", + "recordedInputs": [], + "generatedRepoSpecs": { "yq_darwin_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:yq_toolchain.bzl", - "ruleClassName": "yq_platform_repo", + "repoRuleId": "@@yq.bzl+//yq/toolchain:platforms.bzl%yq_platform_repo", "attributes": { "platform": "darwin_amd64", - "version": "4.25.2" - } - }, - "copy_directory_linux_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_directory_toolchain.bzl", - "ruleClassName": "copy_directory_platform_repo", - "attributes": { - "platform": "linux_arm64" - } - }, - "expand_template_toolchains": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:expand_template_toolchain.bzl", - "ruleClassName": "expand_template_toolchains_repo", - "attributes": { - "user_repository_name": "expand_template" - } - }, - "bats_assert": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "sha256": "98ca3b685f8b8993e48ec057565e6e2abcc541034ed5b0e81f191505682037fd", - "urls": [ - "https://github.com/bats-core/bats-assert/archive/v2.1.0.tar.gz" - ], - "strip_prefix": "bats-assert-2.1.0", - "build_file_content": "load(\"@aspect_bazel_lib//lib:copy_to_directory.bzl\", \"copy_to_directory\")\n\ncopy_to_directory(\n name = \"assert\",\n hardlink = \"on\",\n srcs = glob([\n \"src/**\",\n \"load.bash\",\n ]),\n out = \"bats-assert\",\n visibility = [\"//visibility:public\"]\n)\n" - } - }, - "copy_to_directory_darwin_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_to_directory_toolchain.bzl", - "ruleClassName": "copy_to_directory_platform_repo", - "attributes": { - "platform": "darwin_amd64" - } - }, - "zstd_darwin_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:zstd_toolchain.bzl", - "ruleClassName": "zstd_binary_repo", - "attributes": { - "platform": "darwin_arm64" - } - }, - "bsd_tar_linux_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:tar_toolchain.bzl", - "ruleClassName": "bsdtar_binary_repo", - "attributes": { - "platform": "linux_amd64" - } - }, - "yq_toolchains": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:yq_toolchain.bzl", - "ruleClassName": "yq_toolchains_repo", - "attributes": { - "user_repository_name": "yq" - } - }, - "zstd_linux_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:zstd_toolchain.bzl", - "ruleClassName": "zstd_binary_repo", - "attributes": { - "platform": "linux_amd64" - } - }, - "bats_support": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "sha256": "7815237aafeb42ddcc1b8c698fc5808026d33317d8701d5ec2396e9634e2918f", - "urls": [ - "https://github.com/bats-core/bats-support/archive/v0.3.0.tar.gz" - ], - "strip_prefix": "bats-support-0.3.0", - "build_file_content": "load(\"@aspect_bazel_lib//lib:copy_to_directory.bzl\", \"copy_to_directory\")\n\ncopy_to_directory(\n name = \"support\",\n hardlink = \"on\",\n srcs = glob([\n \"src/**\",\n \"load.bash\",\n ]),\n out = \"bats-support\",\n visibility = [\"//visibility:public\"]\n)\n" - } - }, - "bsd_tar_windows_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:tar_toolchain.bzl", - "ruleClassName": "bsdtar_binary_repo", - "attributes": { - "platform": "windows_amd64" - } - }, - "jq": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:jq_toolchain.bzl", - "ruleClassName": "jq_host_alias_repo", - "attributes": {} - }, - "expand_template_darwin_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:expand_template_toolchain.bzl", - "ruleClassName": "expand_template_platform_repo", - "attributes": { - "platform": "darwin_arm64" - } - }, - "bsd_tar_darwin_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:tar_toolchain.bzl", - "ruleClassName": "bsdtar_binary_repo", - "attributes": { - "platform": "darwin_arm64" - } - }, - "copy_to_directory_linux_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_to_directory_toolchain.bzl", - "ruleClassName": "copy_to_directory_platform_repo", - "attributes": { - "platform": "linux_amd64" - } - }, - "coreutils_linux_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:coreutils_toolchain.bzl", - "ruleClassName": "coreutils_platform_repo", - "attributes": { - "platform": "linux_amd64", - "version": "0.0.26" - } - }, - "copy_directory_toolchains": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_directory_toolchain.bzl", - "ruleClassName": "copy_directory_toolchains_repo", - "attributes": { - "user_repository_name": "copy_directory" - } - }, - "yq_linux_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:yq_toolchain.bzl", - "ruleClassName": "yq_platform_repo", - "attributes": { - "platform": "linux_amd64", - "version": "4.25.2" - } - }, - "copy_to_directory_darwin_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_to_directory_toolchain.bzl", - "ruleClassName": "copy_to_directory_platform_repo", - "attributes": { - "platform": "darwin_arm64" - } - }, - "coreutils_toolchains": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:coreutils_toolchain.bzl", - "ruleClassName": "coreutils_toolchains_repo", - "attributes": { - "user_repository_name": "coreutils" - } - }, - "copy_directory_freebsd_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_directory_toolchain.bzl", - "ruleClassName": "copy_directory_platform_repo", - "attributes": { - "platform": "freebsd_amd64" - } - }, - "zstd_darwin_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:zstd_toolchain.bzl", - "ruleClassName": "zstd_binary_repo", - "attributes": { - "platform": "darwin_amd64" - } - }, - "zstd_toolchains": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:zstd_toolchain.bzl", - "ruleClassName": "zstd_toolchains_repo", - "attributes": { - "user_repository_name": "zstd" - } - }, - "bats_file": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "sha256": "9b69043241f3af1c2d251f89b4fcafa5df3f05e97b89db18d7c9bdf5731bb27a", - "urls": [ - "https://github.com/bats-core/bats-file/archive/v0.4.0.tar.gz" - ], - "strip_prefix": "bats-file-0.4.0", - "build_file_content": "load(\"@aspect_bazel_lib//lib:copy_to_directory.bzl\", \"copy_to_directory\")\n\ncopy_to_directory(\n name = \"file\",\n hardlink = \"on\",\n srcs = glob([\n \"src/**\",\n \"load.bash\",\n ]),\n out = \"bats-file\",\n visibility = [\"//visibility:public\"]\n)\n" - } - }, - "expand_template_linux_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:expand_template_toolchain.bzl", - "ruleClassName": "expand_template_platform_repo", - "attributes": { - "platform": "linux_arm64" - } - }, - "jq_linux_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:jq_toolchain.bzl", - "ruleClassName": "jq_platform_repo", - "attributes": { - "platform": "linux_amd64", - "version": "1.7" - } - }, - "bsd_tar_darwin_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:tar_toolchain.bzl", - "ruleClassName": "bsdtar_binary_repo", - "attributes": { - "platform": "darwin_amd64" - } - }, - "bsd_tar_toolchains": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:tar_toolchain.bzl", - "ruleClassName": "tar_toolchains_repo", - "attributes": { - "user_repository_name": "bsd_tar" - } - }, - "bats_toolchains": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "sha256": "a1a9f7875aa4b6a9480ca384d5865f1ccf1b0b1faead6b47aa47d79709a5c5fd", - "urls": [ - "https://github.com/bats-core/bats-core/archive/v1.10.0.tar.gz" - ], - "strip_prefix": "bats-core-1.10.0", - "build_file_content": "load(\"@local_config_platform//:constraints.bzl\", \"HOST_CONSTRAINTS\")\nload(\"@aspect_bazel_lib//lib/private:bats_toolchain.bzl\", \"bats_toolchain\")\nload(\"@aspect_bazel_lib//lib:copy_to_directory.bzl\", \"copy_to_directory\")\n\ncopy_to_directory(\n name = \"core\",\n hardlink = \"on\",\n srcs = glob([\n \"lib/**\",\n \"libexec/**\"\n ]) + [\"bin/bats\"],\n out = \"bats-core\",\n)\n\nbats_toolchain(\n name = \"toolchain\",\n core = \":core\",\n libraries = [\"@bats_support//:support\", \"@bats_assert//:assert\", \"@bats_file//:file\"]\n)\n\ntoolchain(\n name = \"bats_toolchain\",\n exec_compatible_with = HOST_CONSTRAINTS,\n toolchain = \":toolchain\",\n toolchain_type = \"@aspect_bazel_lib//lib:bats_toolchain_type\",\n)\n" - } - }, - "yq_windows_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:yq_toolchain.bzl", - "ruleClassName": "yq_platform_repo", - "attributes": { - "platform": "windows_amd64", - "version": "4.25.2" - } - }, - "jq_windows_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:jq_toolchain.bzl", - "ruleClassName": "jq_platform_repo", - "attributes": { - "platform": "windows_amd64", - "version": "1.7" - } - }, - "expand_template_freebsd_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:expand_template_toolchain.bzl", - "ruleClassName": "expand_template_platform_repo", - "attributes": { - "platform": "freebsd_amd64" - } - }, - "yq_linux_ppc64le": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:yq_toolchain.bzl", - "ruleClassName": "yq_platform_repo", - "attributes": { - "platform": "linux_ppc64le", - "version": "4.25.2" - } - }, - "copy_to_directory_toolchains": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_to_directory_toolchain.bzl", - "ruleClassName": "copy_to_directory_toolchains_repo", - "attributes": { - "user_repository_name": "copy_to_directory" - } - }, - "jq_toolchains": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:jq_toolchain.bzl", - "ruleClassName": "jq_toolchains_repo", - "attributes": { - "user_repository_name": "jq" - } - }, - "copy_directory_darwin_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_directory_toolchain.bzl", - "ruleClassName": "copy_directory_platform_repo", - "attributes": { - "platform": "darwin_arm64" - } - }, - "copy_directory_windows_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_directory_toolchain.bzl", - "ruleClassName": "copy_directory_platform_repo", - "attributes": { - "platform": "windows_amd64" + "version": "4.45.1" } }, "yq_darwin_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:yq_toolchain.bzl", - "ruleClassName": "yq_platform_repo", + "repoRuleId": "@@yq.bzl+//yq/toolchain:platforms.bzl%yq_platform_repo", "attributes": { "platform": "darwin_arm64", - "version": "4.25.2" + "version": "4.45.1" } }, - "coreutils_windows_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:coreutils_toolchain.bzl", - "ruleClassName": "coreutils_platform_repo", + "yq_linux_amd64": { + "repoRuleId": "@@yq.bzl+//yq/toolchain:platforms.bzl%yq_platform_repo", "attributes": { - "platform": "windows_amd64", - "version": "0.0.26" + "platform": "linux_amd64", + "version": "4.45.1" } }, "yq_linux_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:yq_toolchain.bzl", - "ruleClassName": "yq_platform_repo", + "repoRuleId": "@@yq.bzl+//yq/toolchain:platforms.bzl%yq_platform_repo", "attributes": { "platform": "linux_arm64", - "version": "4.25.2" + "version": "4.45.1" + } + }, + "yq_linux_s390x": { + "repoRuleId": "@@yq.bzl+//yq/toolchain:platforms.bzl%yq_platform_repo", + "attributes": { + "platform": "linux_s390x", + "version": "4.45.1" + } + }, + "yq_linux_riscv64": { + "repoRuleId": "@@yq.bzl+//yq/toolchain:platforms.bzl%yq_platform_repo", + "attributes": { + "platform": "linux_riscv64", + "version": "4.45.1" + } + }, + "yq_linux_ppc64le": { + "repoRuleId": "@@yq.bzl+//yq/toolchain:platforms.bzl%yq_platform_repo", + "attributes": { + "platform": "linux_ppc64le", + "version": "4.45.1" + } + }, + "yq_windows_amd64": { + "repoRuleId": "@@yq.bzl+//yq/toolchain:platforms.bzl%yq_platform_repo", + "attributes": { + "platform": "windows_amd64", + "version": "4.45.1" + } + }, + "yq_toolchains": { + "repoRuleId": "@@yq.bzl+//yq/toolchain:toolchain.bzl%yq_toolchains_repo", + "attributes": { + "user_repository_name": "yq" } } - }, - "recordedRepoMappingEntries": [ - [ - "aspect_bazel_lib~", - "aspect_bazel_lib", - "aspect_bazel_lib~" - ], - [ - "aspect_bazel_lib~", - "bazel_skylib", - "bazel_skylib~" - ], - [ - "aspect_bazel_lib~", - "bazel_tools", - "bazel_tools" - ] + } + } + } + }, + "facts": { + "@@rules_go+//go:extensions.bzl%go_sdk": { + "1.21.1": { + "aix_ppc64": [ + "go1.21.1.aix-ppc64.tar.gz", + "ad5c160e2ff7989f818572f2391ca000d587569ff68d8591a695382a863719d0" + ], + "darwin_amd64": [ + "go1.21.1.darwin-amd64.tar.gz", + "809f5b0ef4f7dcdd5f51e9630a5b2e5a1006f22a047126d61560cdc365678a19" + ], + "darwin_arm64": [ + "go1.21.1.darwin-arm64.tar.gz", + "ffd40391a1e995855488b008ad9326ff8c2e81803a6e80894401003bae47fcf1" + ], + "dragonfly_amd64": [ + "go1.21.1.dragonfly-amd64.tar.gz", + "646f5db8ba479032aea07c597ccd187bc75c161f18e7a1785c62ab8cb103cb3e" + ], + "freebsd_386": [ + "go1.21.1.freebsd-386.tar.gz", + "9919a9a4dc82371aba3da5b7c830bcb6249fc1502cd26d959eb340a60e41ee01" + ], + "freebsd_amd64": [ + "go1.21.1.freebsd-amd64.tar.gz", + "2571f10f6047e04d87c1f5986a05e5e8f7b511faf98803ef12b66d563845d2a1" + ], + "freebsd_arm64": [ + "go1.21.1.freebsd-arm64.tar.gz", + "4946118e72deccf194a9bfcfedbafe52e45b5fe9441fd3acfced760b27a20f9a" + ], + "freebsd_armv6l": [ + "go1.21.1.freebsd-arm.tar.gz", + "6d4dab314ed6adff68a74e3bce516e36fd89c7bf4523c2c105f96e04933fc982" + ], + "freebsd_riscv64": [ + "go1.21.1.freebsd-riscv64.tar.gz", + "591bd28e2dac581facb20c1529f8c29590d7eaaccbcdc93d12a234461b7bfe89" + ], + "illumos_amd64": [ + "go1.21.1.illumos-amd64.tar.gz", + "829548b591675df5f113317ac011bfc3381da7a96bd6a1c748694370818b4497" + ], + "linux_386": [ + "go1.21.1.linux-386.tar.gz", + "b93850666cdadbd696a986cf7b03111fe99db8c34a9aaa113d7c96d0081e1901" + ], + "linux_amd64": [ + "go1.21.1.linux-amd64.tar.gz", + "b3075ae1ce5dab85f89bc7905d1632de23ca196bd8336afd93fa97434cfa55ae" + ], + "linux_arm64": [ + "go1.21.1.linux-arm64.tar.gz", + "7da1a3936a928fd0b2602ed4f3ef535b8cd1990f1503b8d3e1acc0fa0759c967" + ], + "linux_armv6l": [ + "go1.21.1.linux-armv6l.tar.gz", + "f3716a43f59ae69999841d6007b42c9e286e8d8ce470656fb3e70d7be2d7ca85" + ], + "linux_loong64": [ + "go1.21.1.linux-loong64.tar.gz", + "2273cc4ed398d8b341aee01fd62f3845045b2f439b84b28720a7f3d3552f5d41" + ], + "linux_mips": [ + "go1.21.1.linux-mips.tar.gz", + "2ba529210bae7ba7b8c48b0084f713b41e8c109ae75637820fd5b72be9f3faff" + ], + "linux_mips64": [ + "go1.21.1.linux-mips64.tar.gz", + "f2cef18856b17cc5c81cc8b0a35b1ca08d2b04d4b88adc07d190560cb464e2f6" + ], + "linux_mips64le": [ + "go1.21.1.linux-mips64le.tar.gz", + "3aa007a41f533b50eae2491bbd29926ada09357367a8aa05e7e50ec50c78acf9" + ], + "linux_mipsle": [ + "go1.21.1.linux-mipsle.tar.gz", + "c0059f49c3ea31bebe972f47d460d49addb78641081a7000e79e730464bcce95" + ], + "linux_ppc64": [ + "go1.21.1.linux-ppc64.tar.gz", + "6871fd3e6dae3897112dc4a98988406461ef1018f4a17c87689d977eba8dbf20" + ], + "linux_ppc64le": [ + "go1.21.1.linux-ppc64le.tar.gz", + "eddf018206f8a5589bda75252b72716d26611efebabdca5d0083ec15e9e41ab7" + ], + "linux_riscv64": [ + "go1.21.1.linux-riscv64.tar.gz", + "fac64ed26e003f49f1d77f6d2c4cf951422aecbce12232d9ec1bf4585fc44ee1" + ], + "linux_s390x": [ + "go1.21.1.linux-s390x.tar.gz", + "a83b3e8eb4dbf76294e773055eb51397510ff4d612a247bad9903560267bba6d" + ], + "netbsd_386": [ + "go1.21.1.netbsd-386.tar.gz", + "c919f2ba4755e6aebeea6e7f19abf1c5ed13d37c2930867d10a86adfd0686eba" + ], + "netbsd_amd64": [ + "go1.21.1.netbsd-amd64.tar.gz", + "b2dd9913063caa2b5f960e8ebc6c225293b4155e9ebd88d43afd55bb81ca8931" + ], + "netbsd_arm64": [ + "go1.21.1.netbsd-arm64.tar.gz", + "5d245ac932d421389c1bfcedc38733367ae407dbdc162a6e7bfca5677fb15269" + ], + "netbsd_armv6l": [ + "go1.21.1.netbsd-arm.tar.gz", + "4d1756a9845917b07c8ec4999b8e019a2bd2a8b3d5d16da5da0ef3ce84d2139c" + ], + "openbsd_386": [ + "go1.21.1.openbsd-386.tar.gz", + "ce094e551227522aaf3111a94a7f115c5a2b4f862c42d349026503544a40fb96" + ], + "openbsd_amd64": [ + "go1.21.1.openbsd-amd64.tar.gz", + "321c31c42131db2bd0e9d67dd98450f55b281e8e6d8eb681594c39e6688ccbb2" + ], + "openbsd_arm64": [ + "go1.21.1.openbsd-arm64.tar.gz", + "4d65bcb383895be8d6c05507f2b410e15448b0610e1f5fb23ece9ff4e045f7a8" + ], + "openbsd_armv6l": [ + "go1.21.1.openbsd-arm.tar.gz", + "481254b99b9861b65c9eaab137af15311fca618f9f17f2bd872795368c726666" + ], + "plan9_386": [ + "go1.21.1.plan9-386.tar.gz", + "93ea25cbb8063cd4d5917b8ca58a72c26b6cb9e6c1d785e30867ae8ad8290484" + ], + "plan9_amd64": [ + "go1.21.1.plan9-amd64.tar.gz", + "14d61633059651d12cc02b2f53739016f1b07063a4140147e44a196413950553" + ], + "plan9_armv6l": [ + "go1.21.1.plan9-arm.tar.gz", + "adfacbcfb5e7415802187fb6aa8b5397d589fdedcd5d0eac3b86352da7390603" + ], + "solaris_amd64": [ + "go1.21.1.solaris-amd64.tar.gz", + "b86a223e52e41de3ff180f90f1d7e705d30f96271f9a81005e20f25be2658990" + ], + "windows_386": [ + "go1.21.1.windows-386.zip", + "170256c820f466f29d64876f25f4dfa4029ed9902a0a9095d8bd603aecf4d83b" + ], + "windows_amd64": [ + "go1.21.1.windows-amd64.zip", + "10a4f5b63215d11d1770453733dbcbf024f3f74872f84e28d7ea59f0250316c6" + ], + "windows_arm64": [ + "go1.21.1.windows-arm64.zip", + "41135ce6e0ced4bc1e459cb96bd4090c9dc2062e24179c3f337d855af9b560ef" + ], + "windows_armv6l": [ + "go1.21.1.windows-arm.zip", + "d7a2472999a80dbf6bbcb6783a2b4299916055473263117295022f18e7fad487" ] - } - }, - "@@buildifier_prebuilt~//:defs.bzl%buildifier_prebuilt_deps_extension": { - "general": { - "bzlTransitiveDigest": "RaNT6gZicQa6HpONHOm8ejwF7zVAk4fIUHrbuHme7z4=", - "usagesDigest": "nThSTPRdiQbhDFl8FRM2nsKJftWMtPBQHrp/mdk716w=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "buildozer_darwin_amd64": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "urls": [ - "https://github.com/bazelbuild/buildtools/releases/download/v6.4.0/buildozer-darwin-amd64" - ], - "downloaded_file_path": "buildozer", - "executable": true, - "sha256": "d29e347ecd6b5673d72cb1a8de05bf1b06178dd229ff5eb67fad5100c840cc8e" - } - }, - "buildifier_linux_amd64": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "urls": [ - "https://github.com/bazelbuild/buildtools/releases/download/v6.4.0/buildifier-linux-amd64" - ], - "downloaded_file_path": "buildifier", - "executable": true, - "sha256": "be63db12899f48600bad94051123b1fd7b5251e7661b9168582ce52396132e92" - } - }, - "buildozer_darwin_arm64": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "urls": [ - "https://github.com/bazelbuild/buildtools/releases/download/v6.4.0/buildozer-darwin-arm64" - ], - "downloaded_file_path": "buildozer", - "executable": true, - "sha256": "9b9e71bdbec5e7223871e913b65d12f6d8fa026684daf991f00e52ed36a6978d" - } - }, - "buildozer_linux_amd64": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "urls": [ - "https://github.com/bazelbuild/buildtools/releases/download/v6.4.0/buildozer-linux-amd64" - ], - "downloaded_file_path": "buildozer", - "executable": true, - "sha256": "8dfd6345da4e9042daa738d7fdf34f699c5dfce4632f7207956fceedd8494119" - } - }, - "buildozer_windows_amd64": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "urls": [ - "https://github.com/bazelbuild/buildtools/releases/download/v6.4.0/buildozer-windows-amd64.exe" - ], - "downloaded_file_path": "buildozer.exe", - "executable": true, - "sha256": "e7f05bf847f7c3689dd28926460ce6e1097ae97380ac8e6ae7147b7b706ba19b" - } - }, - "buildozer_linux_arm64": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "urls": [ - "https://github.com/bazelbuild/buildtools/releases/download/v6.4.0/buildozer-linux-arm64" - ], - "downloaded_file_path": "buildozer", - "executable": true, - "sha256": "6559558fded658c8fa7432a9d011f7c4dcbac6b738feae73d2d5c352e5f605fa" - } - }, - "buildifier_windows_amd64": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "urls": [ - "https://github.com/bazelbuild/buildtools/releases/download/v6.4.0/buildifier-windows-amd64.exe" - ], - "downloaded_file_path": "buildifier.exe", - "executable": true, - "sha256": "da8372f35e34b65fb6d997844d041013bb841e55f58b54d596d35e49680fe13c" - } - }, - "buildifier_prebuilt_toolchains": { - "bzlFile": "@@buildifier_prebuilt~//:defs.bzl", - "ruleClassName": "_buildifier_toolchain_setup", - "attributes": { - "assets_json": "[{\"arch\":\"amd64\",\"name\":\"buildifier\",\"platform\":\"darwin\",\"sha256\":\"eeb47b2de27f60efe549348b183fac24eae80f1479e8b06cac0799c486df5bed\",\"version\":\"v6.4.0\"},{\"arch\":\"arm64\",\"name\":\"buildifier\",\"platform\":\"darwin\",\"sha256\":\"fa07ba0d20165917ca4cc7609f9b19a8a4392898148b7babdf6bb2a7dd963f05\",\"version\":\"v6.4.0\"},{\"arch\":\"amd64\",\"name\":\"buildifier\",\"platform\":\"linux\",\"sha256\":\"be63db12899f48600bad94051123b1fd7b5251e7661b9168582ce52396132e92\",\"version\":\"v6.4.0\"},{\"arch\":\"arm64\",\"name\":\"buildifier\",\"platform\":\"linux\",\"sha256\":\"18540fc10f86190f87485eb86963e603e41fa022f88a2d1b0cf52ff252b5e1dd\",\"version\":\"v6.4.0\"},{\"arch\":\"amd64\",\"name\":\"buildifier\",\"platform\":\"windows\",\"sha256\":\"da8372f35e34b65fb6d997844d041013bb841e55f58b54d596d35e49680fe13c\",\"version\":\"v6.4.0\"},{\"arch\":\"amd64\",\"name\":\"buildozer\",\"platform\":\"darwin\",\"sha256\":\"d29e347ecd6b5673d72cb1a8de05bf1b06178dd229ff5eb67fad5100c840cc8e\",\"version\":\"v6.4.0\"},{\"arch\":\"arm64\",\"name\":\"buildozer\",\"platform\":\"darwin\",\"sha256\":\"9b9e71bdbec5e7223871e913b65d12f6d8fa026684daf991f00e52ed36a6978d\",\"version\":\"v6.4.0\"},{\"arch\":\"amd64\",\"name\":\"buildozer\",\"platform\":\"linux\",\"sha256\":\"8dfd6345da4e9042daa738d7fdf34f699c5dfce4632f7207956fceedd8494119\",\"version\":\"v6.4.0\"},{\"arch\":\"arm64\",\"name\":\"buildozer\",\"platform\":\"linux\",\"sha256\":\"6559558fded658c8fa7432a9d011f7c4dcbac6b738feae73d2d5c352e5f605fa\",\"version\":\"v6.4.0\"},{\"arch\":\"amd64\",\"name\":\"buildozer\",\"platform\":\"windows\",\"sha256\":\"e7f05bf847f7c3689dd28926460ce6e1097ae97380ac8e6ae7147b7b706ba19b\",\"version\":\"v6.4.0\"}]" - } - }, - "buildifier_darwin_amd64": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "urls": [ - "https://github.com/bazelbuild/buildtools/releases/download/v6.4.0/buildifier-darwin-amd64" - ], - "downloaded_file_path": "buildifier", - "executable": true, - "sha256": "eeb47b2de27f60efe549348b183fac24eae80f1479e8b06cac0799c486df5bed" - } - }, - "buildifier_darwin_arm64": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "urls": [ - "https://github.com/bazelbuild/buildtools/releases/download/v6.4.0/buildifier-darwin-arm64" - ], - "downloaded_file_path": "buildifier", - "executable": true, - "sha256": "fa07ba0d20165917ca4cc7609f9b19a8a4392898148b7babdf6bb2a7dd963f05" - } - }, - "buildifier_linux_arm64": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "urls": [ - "https://github.com/bazelbuild/buildtools/releases/download/v6.4.0/buildifier-linux-arm64" - ], - "downloaded_file_path": "buildifier", - "executable": true, - "sha256": "18540fc10f86190f87485eb86963e603e41fa022f88a2d1b0cf52ff252b5e1dd" - } - } - }, - "recordedRepoMappingEntries": [ - [ - "buildifier_prebuilt~", - "bazel_skylib", - "bazel_skylib~" - ], - [ - "buildifier_prebuilt~", - "bazel_tools", - "bazel_tools" - ] + }, + "1.22.0": { + "aix_ppc64": [ + "go1.22.0.aix-ppc64.tar.gz", + "190e105fc4133a8b5bb1492f368fa89aa4b729270441120714be7ee82e871ebc" + ], + "darwin_amd64": [ + "go1.22.0.darwin-amd64.tar.gz", + "ebca81df938d2d1047cc992be6c6c759543cf309d401b86af38a6aed3d4090f4" + ], + "darwin_arm64": [ + "go1.22.0.darwin-arm64.tar.gz", + "bf8e388b09134164717cd52d3285a4ab3b68691b80515212da0e9f56f518fb1e" + ], + "dragonfly_amd64": [ + "go1.22.0.dragonfly-amd64.tar.gz", + "357ab446200effa26c73ccaf3e8551426428950bf2490859fb296a09e53228b1" + ], + "freebsd_386": [ + "go1.22.0.freebsd-386.tar.gz", + "b8065da37783e8b9e7086365a54d74537e832c92311b61101a66989ab2458d8e" + ], + "freebsd_amd64": [ + "go1.22.0.freebsd-amd64.tar.gz", + "50f421c7f217083ac94aab1e09400cb9c2fea7d337679ec11f1638a11460da30" + ], + "freebsd_arm64": [ + "go1.22.0.freebsd-arm64.tar.gz", + "e23385e5c640787fa02cd58f2301ea09e162c4d99f8ca9fa6d52766f428a933d" + ], + "freebsd_armv6l": [ + "go1.22.0.freebsd-arm.tar.gz", + "c9c8b305f90903536f4981bad9f029828c2483b3216ca1783777344fbe603f2d" + ], + "freebsd_riscv64": [ + "go1.22.0.freebsd-riscv64.tar.gz", + "c8f94d1de6024546194d58e7b9370dc7ea06176aad94a675b0062c25c40cb645" + ], + "illumos_amd64": [ + "go1.22.0.illumos-amd64.tar.gz", + "d6792f11ad6ee5fc42d2fe51e1f1683471aa2ee4f20e3ad1be22a4afdbd38e7f" + ], + "linux_386": [ + "go1.22.0.linux-386.tar.gz", + "1e209c4abde069067ac9afb341c8003db6a210f8173c77777f02d3a524313da3" + ], + "linux_amd64": [ + "go1.22.0.linux-amd64.tar.gz", + "f6c8a87aa03b92c4b0bf3d558e28ea03006eb29db78917daec5cfb6ec1046265" + ], + "linux_arm64": [ + "go1.22.0.linux-arm64.tar.gz", + "6a63fef0e050146f275bf02a0896badfe77c11b6f05499bb647e7bd613a45a10" + ], + "linux_armv6l": [ + "go1.22.0.linux-armv6l.tar.gz", + "0525f92f79df7ed5877147bce7b955f159f3962711b69faac66bc7121d36dcc4" + ], + "linux_loong64": [ + "go1.22.0.linux-loong64.tar.gz", + "b4b1d425cc113608452a32768469b6e34e538fd072bde9f508a75c8dbbdb843f" + ], + "linux_mips": [ + "go1.22.0.linux-mips.tar.gz", + "ec0f9194df750c78492c02e4c70ffe6f3864f2511e47c894415320db752719f8" + ], + "linux_mips64": [ + "go1.22.0.linux-mips64.tar.gz", + "47e938d215e4968ab42afb7307792e5e64184a717e8f176d0de7c411af96d63d" + ], + "linux_mips64le": [ + "go1.22.0.linux-mips64le.tar.gz", + "c162a1a516b8bc8839fc0c0283ad90f6b511e5aca3da4939ed4800f124b9e72c" + ], + "linux_mipsle": [ + "go1.22.0.linux-mipsle.tar.gz", + "6ce3e9a06e3a8ca0312dc1f85046b6914c19852eba5393c5cbf26acf698f8977" + ], + "linux_ppc64": [ + "go1.22.0.linux-ppc64.tar.gz", + "5ae23bf460731eb078f5700b01a50a303308b9f7174a2994892e28bf061f7c85" + ], + "linux_ppc64le": [ + "go1.22.0.linux-ppc64le.tar.gz", + "0e57f421df9449066f00155ce98a5be93744b3d81b00ee4c2c9b511be2a31d93" + ], + "linux_riscv64": [ + "go1.22.0.linux-riscv64.tar.gz", + "afe9cedcdbd6fdff27c57efd30aa5ce0f666f471fed5fa96cd4fb38d6b577086" + ], + "linux_s390x": [ + "go1.22.0.linux-s390x.tar.gz", + "2e546a3583ba7bd3988f8f476245698f6a93dfa9fe206a8ca8f85c1ceecb2446" + ], + "netbsd_386": [ + "go1.22.0.netbsd-386.tar.gz", + "9b7e2dbd43a169bff18bf533a6c2f462eebe04126ab01c138d0d669c046e1658" + ], + "netbsd_amd64": [ + "go1.22.0.netbsd-amd64.tar.gz", + "b11995c271d2256dfe85cf54882ca3655e18c49c4d7db0502bff9977767894e7" + ], + "netbsd_arm64": [ + "go1.22.0.netbsd-arm64.tar.gz", + "499918ccfedde78264d194244d251bf41d95cf606cc0adad13b36b8103cb974f" + ], + "netbsd_armv6l": [ + "go1.22.0.netbsd-arm.tar.gz", + "b57a3aa9c862300ec0ee8609ce5e0f430f132044f351677fd34711a504081872" + ], + "openbsd_386": [ + "go1.22.0.openbsd-386.tar.gz", + "379e0829104c18a139d78b33378e6bd3ab2e0792f6c43b4c38e7f20d8d77b99d" + ], + "openbsd_amd64": [ + "go1.22.0.openbsd-amd64.tar.gz", + "ceb0c97ffc3bfaf74e1df843cb8571d7fc3173a08432f0f42112495df6a31520" + ], + "openbsd_arm64": [ + "go1.22.0.openbsd-arm64.tar.gz", + "358801cab7122ab50d7d92727644f26d818e9e973403f09e85c2e935a625db75" + ], + "openbsd_armv6l": [ + "go1.22.0.openbsd-arm.tar.gz", + "8af5aea3df539bc95ed412c0a176fe84baced70ea1dd29f4aa82d0e9ce27fd9f" + ], + "plan9_386": [ + "go1.22.0.plan9-386.tar.gz", + "fa42c545c9025c45ca9af176dc13a0f4af0cc26bacff6fcb35bb4a170ac538e8" + ], + "plan9_amd64": [ + "go1.22.0.plan9-amd64.tar.gz", + "d8cf64f37a1dfd8e190c5a303c43ab2d49324868f098d88a3106072d137a5a0b" + ], + "plan9_armv6l": [ + "go1.22.0.plan9-arm.tar.gz", + "86fd6165f0cbb47ad551094f74b3e5a6c5e09de858d8b99de72d978d41be6e2a" + ], + "solaris_amd64": [ + "go1.22.0.solaris-amd64.tar.gz", + "a6c12651768d3a74f16104502b4b7bef513ea6b646d99990a28d934c261d1689" + ], + "windows_386": [ + "go1.22.0.windows-386.zip", + "553d44928509965cbda02a45b35ab01cf8b925534bc526a34e2d9dc7794b57e8" + ], + "windows_amd64": [ + "go1.22.0.windows-amd64.zip", + "78b3158fe3aa358e0b6c9f26ecd338f9a11441e88bc434ae2e9f0ca2b0cc4dd3" + ], + "windows_arm64": [ + "go1.22.0.windows-arm64.zip", + "31a61e41d06a3bb2189a303f5f3e777ca4b454eff439f0a67bc2b166330021f4" + ], + "windows_armv6l": [ + "go1.22.0.windows-arm.zip", + "495c7dfaea4e2bf48643662bb622e4ce6378d6d9840015238ad4b8792b99ddbf" ] - } - }, - "@@hermetic_cc_toolchain~//toolchain:ext.bzl%toolchains": { - "general": { - "bzlTransitiveDigest": "YrUOAIuZsxXEx+Q8Bh4BooEu9qcrrOJRxaoC368wqAs=", - "usagesDigest": "EoOqmoW16SGIu0l1zbP8oSGHkhceATca/trgf094fVo=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "zig_sdk": { - "bzlFile": "@@hermetic_cc_toolchain~//toolchain:defs.bzl", - "ruleClassName": "zig_repository", - "attributes": { - "version": "0.12.0", - "url_formats": [ - "https://mirror.bazel.build/ziglang.org/download/{version}/zig-{host_platform}-{version}.{_ext}", - "https://ziglang.org/download/{version}/zig-{host_platform}-{version}.{_ext}" - ], - "host_platform_sha256": { - "linux-aarch64": "754f1029484079b7e0ca3b913a0a2f2a6afd5a28990cb224fe8845e72f09de63", - "linux-x86_64": "c7ae866b8a76a568e2d5cfd31fe89cdb629bdd161fdd5018b29a4a0a17045cad", - "macos-aarch64": "294e224c14fd0822cfb15a35cf39aa14bd9967867999bf8bdfe3db7ddec2a27f", - "macos-x86_64": "4d411bf413e7667821324da248e8589278180dbc197f4f282b7dbb599a689311", - "windows-aarch64": "04c6b92689241ca7a8a59b5f12d2ca2820c09d5043c3c4808b7e93e41c7bf97b", - "windows-x86_64": "2199eb4c2000ddb1fba85ba78f1fcf9c1fb8b3e57658f6a627a8e513131893f5" - }, - "host_platform_ext": { - "linux-aarch64": "tar.xz", - "linux-x86_64": "tar.xz", - "macos-aarch64": "tar.xz", - "macos-x86_64": "tar.xz", - "windows-x86_64": "zip" - } - } - } - }, - "recordedRepoMappingEntries": [ - [ - "hermetic_cc_toolchain~", - "bazel_tools", - "bazel_tools" - ], - [ - "hermetic_cc_toolchain~", - "hermetic_cc_toolchain", - "hermetic_cc_toolchain~" - ] + }, + "1.22.4": { + "aix_ppc64": [ + "go1.22.4.aix-ppc64.tar.gz", + "b9647fa9fc83a0cc5d4f092a19eaeaecf45f063a5aa7d4962fde65aeb7ae6ce1" + ], + "darwin_amd64": [ + "go1.22.4.darwin-amd64.tar.gz", + "c95967f50aa4ace34af0c236cbdb49a9a3e80ee2ad09d85775cb4462a5c19ed3" + ], + "darwin_arm64": [ + "go1.22.4.darwin-arm64.tar.gz", + "242b78dc4c8f3d5435d28a0d2cec9b4c1aa999b601fb8aa59fb4e5a1364bf827" + ], + "dragonfly_amd64": [ + "go1.22.4.dragonfly-amd64.tar.gz", + "f2fbb51af4719d3616efb482d6ed2b96579b474156f85a7ddc6f126764feec4b" + ], + "freebsd_386": [ + "go1.22.4.freebsd-386.tar.gz", + "7c54884bb9f274884651d41e61d1bc12738863ad1497e97ea19ad0e9aa6bf7b5" + ], + "freebsd_amd64": [ + "go1.22.4.freebsd-amd64.tar.gz", + "88d44500e1701dd35797619774d6dd51bf60f45a8338b0a82ddc018e4e63fb78" + ], + "freebsd_arm64": [ + "go1.22.4.freebsd-arm64.tar.gz", + "726dc093cf020277be45debf03c3b02b43c2efb3e2a5d4fba8f52579d65327dc" + ], + "freebsd_armv6l": [ + "go1.22.4.freebsd-arm.tar.gz", + "3d9efe47db142a22679aba46b1772e3900b0d87ae13bd2b3bc80dbf2ac0b2cd6" + ], + "freebsd_riscv64": [ + "go1.22.4.freebsd-riscv64.tar.gz", + "5f6b67e5e32f1d6ccb2d4dcb44934a5e2e870a877ba7443d86ec43cfc28afa71" + ], + "illumos_amd64": [ + "go1.22.4.illumos-amd64.tar.gz", + "d56ecc2f85b6418a21ef83879594d0c42ab4f65391a676bb12254870e6690d63" + ], + "linux_386": [ + "go1.22.4.linux-386.tar.gz", + "47a2a8d249a91eb8605c33bceec63aedda0441a43eac47b4721e3975ff916cec" + ], + "linux_amd64": [ + "go1.22.4.linux-amd64.tar.gz", + "ba79d4526102575196273416239cca418a651e049c2b099f3159db85e7bade7d" + ], + "linux_arm64": [ + "go1.22.4.linux-arm64.tar.gz", + "a8e177c354d2e4a1b61020aca3562e27ea3e8f8247eca3170e3fa1e0c2f9e771" + ], + "linux_armv6l": [ + "go1.22.4.linux-armv6l.tar.gz", + "e2b143fbacbc9cbd448e9ef41ac3981f0488ce849af1cf37e2341d09670661de" + ], + "linux_loong64": [ + "go1.22.4.linux-loong64.tar.gz", + "e2ff9436e4b34bf6926b06d97916e26d67a909a2effec17967245900f0816f1d" + ], + "linux_mips": [ + "go1.22.4.linux-mips.tar.gz", + "73f0dcc60458c4770593b05a7bc01cc0d31fc98f948c0c2334812c7a1f2fc3f1" + ], + "linux_mips64": [ + "go1.22.4.linux-mips64.tar.gz", + "417af97fc2630a647052375768be4c38adcc5af946352ea5b28613ea81ca5d45" + ], + "linux_mips64le": [ + "go1.22.4.linux-mips64le.tar.gz", + "7486e2d7dd8c98eb44df815ace35a7fe7f30b7c02326e3741bd934077508139b" + ], + "linux_mipsle": [ + "go1.22.4.linux-mipsle.tar.gz", + "69479c8aad301e459a8365b40cad1074a0dbba5defb9291669f94809c4c4be6e" + ], + "linux_ppc64": [ + "go1.22.4.linux-ppc64.tar.gz", + "dd238847e65bc3e2745caca475a5db6522a2fcf85cf6c38fc36a06642b19efd7" + ], + "linux_ppc64le": [ + "go1.22.4.linux-ppc64le.tar.gz", + "a3e5834657ef92523f570f798fed42f1f87bc18222a16815ec76b84169649ec4" + ], + "linux_riscv64": [ + "go1.22.4.linux-riscv64.tar.gz", + "56a827ff7dc6245bcd7a1e9288dffaa1d8b0fd7468562264c1523daf3b4f1b4a" + ], + "linux_s390x": [ + "go1.22.4.linux-s390x.tar.gz", + "7590c3e278e2dc6040aae0a39da3ca1eb2e3921673a7304cc34d588c45889eec" + ], + "netbsd_386": [ + "go1.22.4.netbsd-386.tar.gz", + "ddd2eebe34471a2502de6c5dad04ab27c9fc80cbde7a9ad5b3c66ecec4504e1d" + ], + "netbsd_amd64": [ + "go1.22.4.netbsd-amd64.tar.gz", + "33af79f6f935f6fbacc5d23876450b3567b79348fc065beef8e64081127dd234" + ], + "netbsd_arm64": [ + "go1.22.4.netbsd-arm64.tar.gz", + "c9a2971dec9f6d320c6f2b049b2353c6d0a2d35e87b8a4b2d78a2f0d62545f8e" + ], + "netbsd_armv6l": [ + "go1.22.4.netbsd-arm.tar.gz", + "fa3550ebd5375a70b3bcd342b5a71f4bd271dcbbfaf4eabefa2144ab5d8924b6" + ], + "openbsd_386": [ + "go1.22.4.openbsd-386.tar.gz", + "d21af022331bfdc2b5b161d616c3a1a4573d33cf7a30416ee509a8f3641deb47" + ], + "openbsd_amd64": [ + "go1.22.4.openbsd-amd64.tar.gz", + "72c0094c43f7e5722ec49c2a3e9dfa7a1123ac43a5f3a63eecf3e3795d3ff0ae" + ], + "openbsd_arm64": [ + "go1.22.4.openbsd-arm64.tar.gz", + "a7ab8d4e0b02bf06ed144ba42c61c0e93ee00f2b433415dfd4ad4b6e79f31650" + ], + "openbsd_armv6l": [ + "go1.22.4.openbsd-arm.tar.gz", + "1096831ea3c5ea3ca57d14251d9eda3786889531eb40d7d6775dcaa324d4b065" + ], + "openbsd_ppc64": [ + "go1.22.4.openbsd-ppc64.tar.gz", + "9716327c8a628358798898dc5148c49dbbeb5196bf2cbf088e550721a6e4f60b" + ], + "plan9_386": [ + "go1.22.4.plan9-386.tar.gz", + "a8dd4503c95c32a502a616ab78870a19889c9325fe9bd31eb16dd69346e4bfa8" + ], + "plan9_amd64": [ + "go1.22.4.plan9-amd64.tar.gz", + "5423a25808d76fe5aca8607a2e5ac5673abf45446b168cb5e9d8519ee9fe39a1" + ], + "plan9_armv6l": [ + "go1.22.4.plan9-arm.tar.gz", + "6af939ad583f5c85c09c53728ab7d38c3cc2b39167562d6c18a07c5c6608b370" + ], + "solaris_amd64": [ + "go1.22.4.solaris-amd64.tar.gz", + "e8cabe69c03085725afdb32a6f9998191a3e55a747b270d835fd05000d56abba" + ], + "windows_386": [ + "go1.22.4.windows-386.zip", + "aca4e2c37278a10f1c70dd0df142f7d66b50334fcee48978d409202d308d6d25" + ], + "windows_amd64": [ + "go1.22.4.windows-amd64.zip", + "26321c4d945a0035d8a5bc4a1965b0df401ff8ceac66ce2daadabf9030419a98" + ], + "windows_arm64": [ + "go1.22.4.windows-arm64.zip", + "8a2daa9ea28cbdafddc6171aefed384f4e5b6e714fb52116fe9ed25a132f37ed" + ], + "windows_armv6l": [ + "go1.22.4.windows-arm.zip", + "5fcd0671a49cecf39b41021621ee1b6e7aa1370f37122b72e80d4fd4185833b6" ] - } - }, - "@@platforms//host:extension.bzl%host_platform": { - "general": { - "bzlTransitiveDigest": "WewbYICdNVp22bzUQafEVMzMIpBnNjJ3zqKlUOCDIGc=", - "usagesDigest": "V1R2Y2oMxKNfx2WCWpSCaUV1WefW1o8HZGm3v1vHgY4=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "host_platform": { - "bzlFile": "@@platforms//host:extension.bzl", - "ruleClassName": "host_platform_repo", - "attributes": {} - } - }, - "recordedRepoMappingEntries": [] - } - }, - "@@rules_python~//python/extensions:python.bzl%python": { - "general": { - "bzlTransitiveDigest": "uFBdNKDewkX8DPVvKoPEcW0b3dXtksiE7SShRkuCA3E=", - "usagesDigest": "fQEsnAYwqRJT7/lTBAe+NllONXk6f/Tc57oiPxLG8SI=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": { - "RULES_PYTHON_BZLMOD_DEBUG": null - }, - "generatedRepoSpecs": { - "python_3_11_s390x-unknown-linux-gnu": { - "bzlFile": "@@rules_python~//python:repositories.bzl", - "ruleClassName": "python_repository", - "attributes": { - "sha256": "49520e3ff494708020f306e30b0964f079170be83e956be4504f850557378a22", - "patches": [], - "platform": "s390x-unknown-linux-gnu", - "python_version": "3.11.7", - "release_filename": "20240107/cpython-3.11.7+20240107-s390x-unknown-linux-gnu-install_only.tar.gz", - "urls": [ - "https://github.com/indygreg/python-build-standalone/releases/download/20240107/cpython-3.11.7+20240107-s390x-unknown-linux-gnu-install_only.tar.gz" - ], - "distutils_content": "", - "strip_prefix": "python", - "coverage_tool": "", - "ignore_root_user_error": false - } - }, - "python_3_11_host": { - "bzlFile": "@@rules_python~//python/private:toolchains_repo.bzl", - "ruleClassName": "host_toolchain", - "attributes": { - "python_version": "3.11.7", - "user_repository_name": "python_3_11", - "platforms": [ - "aarch64-apple-darwin", - "aarch64-unknown-linux-gnu", - "ppc64le-unknown-linux-gnu", - "s390x-unknown-linux-gnu", - "x86_64-apple-darwin", - "x86_64-pc-windows-msvc", - "x86_64-unknown-linux-gnu" - ] - } - }, - "python_3_11": { - "bzlFile": "@@rules_python~//python/private:toolchains_repo.bzl", - "ruleClassName": "toolchain_aliases", - "attributes": { - "python_version": "3.11.7", - "user_repository_name": "python_3_11", - "platforms": [ - "aarch64-apple-darwin", - "aarch64-unknown-linux-gnu", - "ppc64le-unknown-linux-gnu", - "s390x-unknown-linux-gnu", - "x86_64-apple-darwin", - "x86_64-pc-windows-msvc", - "x86_64-unknown-linux-gnu" - ] - } - }, - "python_3_11_aarch64-unknown-linux-gnu": { - "bzlFile": "@@rules_python~//python:repositories.bzl", - "ruleClassName": "python_repository", - "attributes": { - "sha256": "b102eaf865eb715aa98a8a2ef19037b6cc3ae7dfd4a632802650f29de635aa13", - "patches": [], - "platform": "aarch64-unknown-linux-gnu", - "python_version": "3.11.7", - "release_filename": "20240107/cpython-3.11.7+20240107-aarch64-unknown-linux-gnu-install_only.tar.gz", - "urls": [ - "https://github.com/indygreg/python-build-standalone/releases/download/20240107/cpython-3.11.7+20240107-aarch64-unknown-linux-gnu-install_only.tar.gz" - ], - "distutils_content": "", - "strip_prefix": "python", - "coverage_tool": "", - "ignore_root_user_error": false - } - }, - "python_3_11_aarch64-apple-darwin": { - "bzlFile": "@@rules_python~//python:repositories.bzl", - "ruleClassName": "python_repository", - "attributes": { - "sha256": "b042c966920cf8465385ca3522986b12d745151a72c060991088977ca36d3883", - "patches": [], - "platform": "aarch64-apple-darwin", - "python_version": "3.11.7", - "release_filename": "20240107/cpython-3.11.7+20240107-aarch64-apple-darwin-install_only.tar.gz", - "urls": [ - "https://github.com/indygreg/python-build-standalone/releases/download/20240107/cpython-3.11.7+20240107-aarch64-apple-darwin-install_only.tar.gz" - ], - "distutils_content": "", - "strip_prefix": "python", - "coverage_tool": "", - "ignore_root_user_error": false - } - }, - "python_3_11_ppc64le-unknown-linux-gnu": { - "bzlFile": "@@rules_python~//python:repositories.bzl", - "ruleClassName": "python_repository", - "attributes": { - "sha256": "b44e1b74afe75c7b19143413632c4386708ae229117f8f950c2094e9681d34c7", - "patches": [], - "platform": "ppc64le-unknown-linux-gnu", - "python_version": "3.11.7", - "release_filename": "20240107/cpython-3.11.7+20240107-ppc64le-unknown-linux-gnu-install_only.tar.gz", - "urls": [ - "https://github.com/indygreg/python-build-standalone/releases/download/20240107/cpython-3.11.7+20240107-ppc64le-unknown-linux-gnu-install_only.tar.gz" - ], - "distutils_content": "", - "strip_prefix": "python", - "coverage_tool": "", - "ignore_root_user_error": false - } - }, - "python_3_11_x86_64-apple-darwin": { - "bzlFile": "@@rules_python~//python:repositories.bzl", - "ruleClassName": "python_repository", - "attributes": { - "sha256": "a0e615eef1fafdc742da0008425a9030b7ea68a4ae4e73ac557ef27b112836d4", - "patches": [], - "platform": "x86_64-apple-darwin", - "python_version": "3.11.7", - "release_filename": "20240107/cpython-3.11.7+20240107-x86_64-apple-darwin-install_only.tar.gz", - "urls": [ - "https://github.com/indygreg/python-build-standalone/releases/download/20240107/cpython-3.11.7+20240107-x86_64-apple-darwin-install_only.tar.gz" - ], - "distutils_content": "", - "strip_prefix": "python", - "coverage_tool": "", - "ignore_root_user_error": false - } - }, - "pythons_hub": { - "bzlFile": "@@rules_python~//python/private/bzlmod:pythons_hub.bzl", - "ruleClassName": "hub_repo", - "attributes": { - "default_python_version": "3.11", - "toolchain_prefixes": [ - "_0000_python_3_11_" - ], - "toolchain_python_versions": [ - "3.11" - ], - "toolchain_set_python_version_constraints": [ - "False" - ], - "toolchain_user_repository_names": [ - "python_3_11" - ] - } - }, - "python_versions": { - "bzlFile": "@@rules_python~//python/private:toolchains_repo.bzl", - "ruleClassName": "multi_toolchain_aliases", - "attributes": { - "python_versions": { - "3.11": "python_3_11" - } - } - }, - "python_3_11_x86_64-pc-windows-msvc": { - "bzlFile": "@@rules_python~//python:repositories.bzl", - "ruleClassName": "python_repository", - "attributes": { - "sha256": "67077e6fa918e4f4fd60ba169820b00be7c390c497bf9bc9cab2c255ea8e6f3e", - "patches": [], - "platform": "x86_64-pc-windows-msvc", - "python_version": "3.11.7", - "release_filename": "20240107/cpython-3.11.7+20240107-x86_64-pc-windows-msvc-shared-install_only.tar.gz", - "urls": [ - "https://github.com/indygreg/python-build-standalone/releases/download/20240107/cpython-3.11.7+20240107-x86_64-pc-windows-msvc-shared-install_only.tar.gz" - ], - "distutils_content": "", - "strip_prefix": "python", - "coverage_tool": "", - "ignore_root_user_error": false - } - }, - "python_3_11_x86_64-unknown-linux-gnu": { - "bzlFile": "@@rules_python~//python:repositories.bzl", - "ruleClassName": "python_repository", - "attributes": { - "sha256": "4a51ce60007a6facf64e5495f4cf322e311ba9f39a8cd3f3e4c026eae488e140", - "patches": [], - "platform": "x86_64-unknown-linux-gnu", - "python_version": "3.11.7", - "release_filename": "20240107/cpython-3.11.7+20240107-x86_64-unknown-linux-gnu-install_only.tar.gz", - "urls": [ - "https://github.com/indygreg/python-build-standalone/releases/download/20240107/cpython-3.11.7+20240107-x86_64-unknown-linux-gnu-install_only.tar.gz" - ], - "distutils_content": "", - "strip_prefix": "python", - "coverage_tool": "", - "ignore_root_user_error": false - } - } - }, - "recordedRepoMappingEntries": [ - [ - "rules_python~", - "bazel_skylib", - "bazel_skylib~" - ], - [ - "rules_python~", - "bazel_tools", - "bazel_tools" - ] + }, + "1.24.6": { + "aix_ppc64": [ + "go1.24.6.aix-ppc64.tar.gz", + "5cafb2927af95969fc3d0884be1ad4df54f19103d86dda928d9f76494a5ae6a6" + ], + "darwin_amd64": [ + "go1.24.6.darwin-amd64.tar.gz", + "4a8d7a32052f223e71faab424a69430455b27b3fff5f4e651f9d97c3e51a8746" + ], + "darwin_arm64": [ + "go1.24.6.darwin-arm64.tar.gz", + "4e29202c49573b953be7cc3500e1f8d9e66ddd12faa8cf0939a4951411e09a2a" + ], + "dragonfly_amd64": [ + "go1.24.6.dragonfly-amd64.tar.gz", + "c6e21bf8347f7a1c653dd4136c8c27a858515e98501d2843023a4bb3f1f7fb63" + ], + "freebsd_386": [ + "go1.24.6.freebsd-386.tar.gz", + "9cd74ad74f3ad833e92529f2fd9b0d7d9ffaab46307eccadb0afcf9a1ba09553" + ], + "freebsd_amd64": [ + "go1.24.6.freebsd-amd64.tar.gz", + "4983e2b10ae1f754e4eb07e1e589691c7e1d0dc428a92c16bd0e2ba03cc23ed9" + ], + "freebsd_arm": [ + "go1.24.6.freebsd-arm.tar.gz", + "a8da621d8282a91ee17b257a46f2606391c019cc1a7d7be628638792ca8033ad" + ], + "freebsd_arm64": [ + "go1.24.6.freebsd-arm64.tar.gz", + "76a75ad5125217c268029c0ad9c7295cc7f6042fe9cba4bebf9a89f7f42ad8af" + ], + "freebsd_riscv64": [ + "go1.24.6.freebsd-riscv64.tar.gz", + "ac206417d8460662f26d46dc2ad0488b2f9e22039946069ba4b48a0cb646e8b0" + ], + "illumos_amd64": [ + "go1.24.6.illumos-amd64.tar.gz", + "c54199c46ec823c857c52335f859b493593433d71fe479b9e7f95d868a144ae2" + ], + "linux_386": [ + "go1.24.6.linux-386.tar.gz", + "bb5bf69d75e7edbc93339824753a1a4655a928451a2c5e13ff90959ad69e065b" + ], + "linux_amd64": [ + "go1.24.6.linux-amd64.tar.gz", + "bbca37cc395c974ffa4893ee35819ad23ebb27426df87af92e93a9ec66ef8712" + ], + "linux_arm64": [ + "go1.24.6.linux-arm64.tar.gz", + "124ea6033a8bf98aa9fbab53e58d134905262d45a022af3a90b73320f3c3afd5" + ], + "linux_armv6l": [ + "go1.24.6.linux-armv6l.tar.gz", + "7feb4d25f5e72f94fda81c99d4adb6630dfa2c35211e0819417d53af6e71809e" + ], + "linux_loong64": [ + "go1.24.6.linux-loong64.tar.gz", + "8424d9c52b254255d2720769366af73802747e4eaf339c6e86bb52e5a1809fcb" + ], + "linux_mips": [ + "go1.24.6.linux-mips.tar.gz", + "791964a6ce65f604a2099b7e61f71128cc7a3a4fcee858218d2ce99033fc2b09" + ], + "linux_mips64": [ + "go1.24.6.linux-mips64.tar.gz", + "2f5282fdf2eaca44515728d518caec888b0dc4948effecb8e7f2ddff1ff9aa9e" + ], + "linux_mips64le": [ + "go1.24.6.linux-mips64le.tar.gz", + "64ddc6b28907b4ab997f2d81c226be403511a59a6100536560af26ccfb10b4a4" + ], + "linux_mipsle": [ + "go1.24.6.linux-mipsle.tar.gz", + "5ca12d2d4939dea977100e76885f4e08cd3ef3b73823853527b58ecac4598fd3" + ], + "linux_ppc64": [ + "go1.24.6.linux-ppc64.tar.gz", + "9a4a61ffbba7a58b3ef97a6d3b2b394168f613f9ca6092016981a88c78a57f86" + ], + "linux_ppc64le": [ + "go1.24.6.linux-ppc64le.tar.gz", + "63fc9559a3d6dfd63aa902f714375b879bbc848466181c035c122489b9646e27" + ], + "linux_riscv64": [ + "go1.24.6.linux-riscv64.tar.gz", + "e92c19ff15a9004fe43e1f62d433555250ef9fe3fabee3dc29ebd4802c2b8021" + ], + "linux_s390x": [ + "go1.24.6.linux-s390x.tar.gz", + "4cde28d9ffb6eef86bf8dac6852a45db335009f67e60bee3e477dd0ba0ff9704" + ], + "netbsd_386": [ + "go1.24.6.netbsd-386.tar.gz", + "d238045676104d2aaad9da53c4ba3f489cf81b26d3315d2caad2b6e240664423" + ], + "netbsd_amd64": [ + "go1.24.6.netbsd-amd64.tar.gz", + "9f8a9245cf146f1ebdac6785f71646c3dd0edf07dcdb8e5a8df85ec6447e5308" + ], + "netbsd_arm": [ + "go1.24.6.netbsd-arm.tar.gz", + "2359c8af6b2d6979910060478a57fd216824682e0908ea1a4412ea8dc4a55db7" + ], + "netbsd_arm64": [ + "go1.24.6.netbsd-arm64.tar.gz", + "99101158c796997ec85023d395c133fa90c782bd5faa56f8c3824817d28b0ee8" + ], + "openbsd_386": [ + "go1.24.6.openbsd-386.tar.gz", + "b6193420489f24d5560b87ccb38caf52cecb308d892d99bf21d1ef0d2c99de04" + ], + "openbsd_amd64": [ + "go1.24.6.openbsd-amd64.tar.gz", + "8dc0d94afbc65be9cafebe543b88a841dcf3429e50f668ea714200ef9f4489bd" + ], + "openbsd_arm": [ + "go1.24.6.openbsd-arm.tar.gz", + "c29563d14d8e4b5fa89f130afac40abba7a1ee1e87fcc759a95f1e9b5831bf68" + ], + "openbsd_arm64": [ + "go1.24.6.openbsd-arm64.tar.gz", + "3d97bdc5e1997cccfaa398242c52e8a97d3b626d988a65248bbdbf2f1ab188bf" + ], + "openbsd_ppc64": [ + "go1.24.6.openbsd-ppc64.tar.gz", + "54cf517933db0bff58f3e527e7fa194f505d562c71b1fb0050d3365fcdda4914" + ], + "openbsd_riscv64": [ + "go1.24.6.openbsd-riscv64.tar.gz", + "f7bdd8f882ecbc01ff6c1782ce3e29298445dd99b3dd4299423257df307c7b0d" + ], + "plan9_386": [ + "go1.24.6.plan9-386.tar.gz", + "9bc0a801dc0d8d0705ac9493219b926b2e3c0105b8921c6c2fce69d14c4b9007" + ], + "plan9_amd64": [ + "go1.24.6.plan9-amd64.tar.gz", + "7133c28acc277d3c5e8317fd9454b3a0902138c384466f818644b504b8b89ffe" + ], + "plan9_arm": [ + "go1.24.6.plan9-arm.tar.gz", + "c4e2aa8aa4353f75fc083ca728b73917467cef88c6f5f917ca420578672c6d58" + ], + "solaris_amd64": [ + "go1.24.6.solaris-amd64.tar.gz", + "cfcb4f1f7e987c6dee893c4401546516cc271c1ac7d7fb37a32b25343dc7df32" + ], + "windows_386": [ + "go1.24.6.windows-386.zip", + "39b4d31b933d2f7c8913e70fb0fffef27252e441c386eac5c13632cbb28dfb6e" + ], + "windows_amd64": [ + "go1.24.6.windows-amd64.zip", + "4fbc8af2cfca9e5059019b5150a426eb78e1e57718bf08f0e52b1c942a2782bf" + ], + "windows_arm64": [ + "go1.24.6.windows-arm64.zip", + "45c41b237d00e92e4cf8adce11b4c5258048b47a92bfbb1f4ef3b928d6fcb0b2" ] - } - }, - "@@rules_python~//python/private/bzlmod:internal_deps.bzl%internal_deps": { - "general": { - "bzlTransitiveDigest": "ZKHEOaFX4G/WohTzMTV688TfCsSfLvbwBC2ma87N24w=", - "usagesDigest": "4Fj9JSpEDoJSLPRSbvSTol2bTL7baZjuA3k9U7kG/1k=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "pypi__wheel": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/b8/8b/31273bf66016be6ad22bb7345c37ff350276cfd46e389a0c2ac5da9d9073/wheel-0.41.2-py3-none-any.whl", - "sha256": "75909db2664838d015e3d9139004ee16711748a52c8f336b52882266540215d8", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:defs.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude in /python/pip_install/tools/bazel.py\n # to avoid non-determinism following pip install's behavior.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/* *\",\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__click": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/00/2e/d53fa4befbf2cfa713304affc7ca780ce4fc1fd8710527771b58311a3229/click-8.1.7-py3-none-any.whl", - "sha256": "ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:defs.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude in /python/pip_install/tools/bazel.py\n # to avoid non-determinism following pip install's behavior.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/* *\",\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__importlib_metadata": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/cc/37/db7ba97e676af155f5fcb1a35466f446eadc9104e25b83366e8088c9c926/importlib_metadata-6.8.0-py3-none-any.whl", - "sha256": "3ebb78df84a805d7698245025b975d9d67053cd94c79245ba4b3eb694abe68bb", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:defs.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude in /python/pip_install/tools/bazel.py\n # to avoid non-determinism following pip install's behavior.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/* *\",\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__pyproject_hooks": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/d5/ea/9ae603de7fbb3df820b23a70f6aff92bf8c7770043254ad8d2dc9d6bcba4/pyproject_hooks-1.0.0-py3-none-any.whl", - "sha256": "283c11acd6b928d2f6a7c73fa0d01cb2bdc5f07c57a2eeb6e83d5e56b97976f8", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:defs.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude in /python/pip_install/tools/bazel.py\n # to avoid non-determinism following pip install's behavior.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/* *\",\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__pep517": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/ee/2f/ef63e64e9429111e73d3d6cbee80591672d16f2725e648ebc52096f3d323/pep517-0.13.0-py3-none-any.whl", - "sha256": "4ba4446d80aed5b5eac6509ade100bff3e7943a8489de249654a5ae9b33ee35b", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:defs.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude in /python/pip_install/tools/bazel.py\n # to avoid non-determinism following pip install's behavior.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/* *\",\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__packaging": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/ab/c3/57f0601a2d4fe15de7a553c00adbc901425661bf048f2a22dfc500caf121/packaging-23.1-py3-none-any.whl", - "sha256": "994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:defs.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude in /python/pip_install/tools/bazel.py\n # to avoid non-determinism following pip install's behavior.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/* *\",\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__pip_tools": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/e8/df/47e6267c6b5cdae867adbdd84b437393e6202ce4322de0a5e0b92960e1d6/pip_tools-7.3.0-py3-none-any.whl", - "sha256": "8717693288720a8c6ebd07149c93ab0be1fced0b5191df9e9decd3263e20d85e", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:defs.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude in /python/pip_install/tools/bazel.py\n # to avoid non-determinism following pip install's behavior.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/* *\",\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__setuptools": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/4f/ab/0bcfebdfc3bfa8554b2b2c97a555569c4c1ebc74ea288741ea8326c51906/setuptools-68.1.2-py3-none-any.whl", - "sha256": "3d8083eed2d13afc9426f227b24fd1659489ec107c0e86cec2ffdde5c92e790b", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:defs.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude in /python/pip_install/tools/bazel.py\n # to avoid non-determinism following pip install's behavior.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/* *\",\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__zipp": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/8c/08/d3006317aefe25ea79d3b76c9650afabaf6d63d1c8443b236e7405447503/zipp-3.16.2-py3-none-any.whl", - "sha256": "679e51dd4403591b2d6838a48de3d283f3d188412a9782faadf845f298736ba0", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:defs.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude in /python/pip_install/tools/bazel.py\n # to avoid non-determinism following pip install's behavior.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/* *\",\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__colorama": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", - "sha256": "4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:defs.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude in /python/pip_install/tools/bazel.py\n # to avoid non-determinism following pip install's behavior.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/* *\",\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__build": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/58/91/17b00d5fac63d3dca605f1b8269ba3c65e98059e1fd99d00283e42a454f0/build-0.10.0-py3-none-any.whl", - "sha256": "af266720050a66c893a6096a2f410989eeac74ff9a68ba194b3f6473e8e26171", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:defs.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude in /python/pip_install/tools/bazel.py\n # to avoid non-determinism following pip install's behavior.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/* *\",\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "rules_python_internal": { - "bzlFile": "@@rules_python~//python/private:internal_config_repo.bzl", - "ruleClassName": "internal_config_repo", - "attributes": {} - }, - "pypi__pip": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/50/c2/e06851e8cc28dcad7c155f4753da8833ac06a5c704c109313b8d5a62968a/pip-23.2.1-py3-none-any.whl", - "sha256": "7ccf472345f20d35bdc9d1841ff5f313260c2c33fe417f48c30ac46cccabf5be", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:defs.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude in /python/pip_install/tools/bazel.py\n # to avoid non-determinism following pip install's behavior.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/* *\",\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__installer": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/e5/ca/1172b6638d52f2d6caa2dd262ec4c811ba59eee96d54a7701930726bce18/installer-0.7.0-py3-none-any.whl", - "sha256": "05d1933f0a5ba7d8d6296bb6d5018e7c94fa473ceb10cf198a92ccea19c27b53", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:defs.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude in /python/pip_install/tools/bazel.py\n # to avoid non-determinism following pip install's behavior.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/* *\",\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__more_itertools": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/5a/cb/6dce742ea14e47d6f565589e859ad225f2a5de576d7696e0623b784e226b/more_itertools-10.1.0-py3-none-any.whl", - "sha256": "64e0735fcfdc6f3464ea133afe8ea4483b1c5fe3a3d69852e6503b43a0b222e6", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:defs.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude in /python/pip_install/tools/bazel.py\n # to avoid non-determinism following pip install's behavior.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/* *\",\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__tomli": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/97/75/10a9ebee3fd790d20926a90a2547f0bf78f371b2f13aa822c759680ca7b9/tomli-2.0.1-py3-none-any.whl", - "sha256": "939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:defs.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude in /python/pip_install/tools/bazel.py\n # to avoid non-determinism following pip install's behavior.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/* *\",\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - } - }, - "recordedRepoMappingEntries": [ - [ - "rules_python~", - "bazel_tools", - "bazel_tools" - ] + }, + "1.25.0": { + "aix_ppc64": [ + "go1.25.0.aix-ppc64.tar.gz", + "e5234a7dac67bc86c528fe9752fc9d63557918627707a733ab4cac1a6faed2d4" + ], + "darwin_amd64": [ + "go1.25.0.darwin-amd64.tar.gz", + "5bd60e823037062c2307c71e8111809865116714d6f6b410597cf5075dfd80ef" + ], + "darwin_arm64": [ + "go1.25.0.darwin-arm64.tar.gz", + "544932844156d8172f7a28f77f2ac9c15a23046698b6243f633b0a0b00c0749c" + ], + "dragonfly_amd64": [ + "go1.25.0.dragonfly-amd64.tar.gz", + "5ed3cf9a810a1483822538674f1336c06b51aa1b94d6d545a1a0319a48177120" + ], + "freebsd_386": [ + "go1.25.0.freebsd-386.tar.gz", + "abea5d5c6697e6b5c224731f2158fe87c602996a2a233ac0c4730cd57bf8374e" + ], + "freebsd_amd64": [ + "go1.25.0.freebsd-amd64.tar.gz", + "86e6fe0a29698d7601c4442052dac48bd58d532c51cccb8f1917df648138730b" + ], + "freebsd_arm": [ + "go1.25.0.freebsd-arm.tar.gz", + "d90b78e41921f72f30e8bbc81d9dec2cff7ff384a33d8d8debb24053e4336bfe" + ], + "freebsd_arm64": [ + "go1.25.0.freebsd-arm64.tar.gz", + "451d0da1affd886bfb291b7c63a6018527b269505db21ce6e14724f22ab0662e" + ], + "freebsd_riscv64": [ + "go1.25.0.freebsd-riscv64.tar.gz", + "7b565f76bd8bda46549eeaaefe0e53b251e644c230577290c0f66b1ecdb3cdbe" + ], + "illumos_amd64": [ + "go1.25.0.illumos-amd64.tar.gz", + "b1e1fdaab1ad25aa1c08d7a36c97d45d74b98b89c3f78c6d2145f77face54a2c" + ], + "linux_386": [ + "go1.25.0.linux-386.tar.gz", + "8c602dd9d99bc9453b3995d20ce4baf382cc50855900a0ece5de9929df4a993a" + ], + "linux_amd64": [ + "go1.25.0.linux-amd64.tar.gz", + "2852af0cb20a13139b3448992e69b868e50ed0f8a1e5940ee1de9e19a123b613" + ], + "linux_arm64": [ + "go1.25.0.linux-arm64.tar.gz", + "05de75d6994a2783699815ee553bd5a9327d8b79991de36e38b66862782f54ae" + ], + "linux_armv6l": [ + "go1.25.0.linux-armv6l.tar.gz", + "a5a8f8198fcf00e1e485b8ecef9ee020778bf32a408a4e8873371bfce458cd09" + ], + "linux_loong64": [ + "go1.25.0.linux-loong64.tar.gz", + "cab86b1cf761b1cb3bac86a8877cfc92e7b036fc0d3084123d77013d61432afc" + ], + "linux_mips": [ + "go1.25.0.linux-mips.tar.gz", + "d66b6fb74c3d91b9829dc95ec10ca1f047ef5e89332152f92e136cf0e2da5be1" + ], + "linux_mips64": [ + "go1.25.0.linux-mips64.tar.gz", + "4082e4381a8661bc2a839ff94ba3daf4f6cde20f8fb771b5b3d4762dc84198a2" + ], + "linux_mips64le": [ + "go1.25.0.linux-mips64le.tar.gz", + "70002c299ec7f7175ac2ef673b1b347eecfa54ae11f34416a6053c17f855afcc" + ], + "linux_mipsle": [ + "go1.25.0.linux-mipsle.tar.gz", + "b00a3a39eff099f6df9f1c7355bf28e4589d0586f42d7d4a394efb763d145a73" + ], + "linux_ppc64": [ + "go1.25.0.linux-ppc64.tar.gz", + "df166f33bd98160662560a72ff0b4ba731f969a80f088922bddcf566a88c1ec1" + ], + "linux_ppc64le": [ + "go1.25.0.linux-ppc64le.tar.gz", + "0f18a89e7576cf2c5fa0b487a1635d9bcbf843df5f110e9982c64df52a983ad0" + ], + "linux_riscv64": [ + "go1.25.0.linux-riscv64.tar.gz", + "c018ff74a2c48d55c8ca9b07c8e24163558ffec8bea08b326d6336905d956b67" + ], + "linux_s390x": [ + "go1.25.0.linux-s390x.tar.gz", + "34e5a2e19f2292fbaf8783e3a241e6e49689276aef6510a8060ea5ef54eee408" + ], + "netbsd_386": [ + "go1.25.0.netbsd-386.tar.gz", + "f8586cdb7aa855657609a5c5f6dbf523efa00c2bbd7c76d3936bec80aa6c0aba" + ], + "netbsd_amd64": [ + "go1.25.0.netbsd-amd64.tar.gz", + "ae8dc1469385b86a157a423bb56304ba45730de8a897615874f57dd096db2c2a" + ], + "netbsd_arm": [ + "go1.25.0.netbsd-arm.tar.gz", + "1ff7e4cc764425fc9dd6825eaee79d02b3c7cafffbb3691687c8d672ade76cb7" + ], + "netbsd_arm64": [ + "go1.25.0.netbsd-arm64.tar.gz", + "e1b310739f26724216aa6d7d7208c4031f9ff54c9b5b9a796ddc8bebcb4a5f16" + ], + "openbsd_386": [ + "go1.25.0.openbsd-386.tar.gz", + "4802a9b20e533da91adb84aab42e94aa56cfe3e5475d0550bed3385b182e69d8" + ], + "openbsd_amd64": [ + "go1.25.0.openbsd-amd64.tar.gz", + "c016cd984bebe317b19a4f297c4f50def120dc9788490540c89f28e42f1dabe1" + ], + "openbsd_arm": [ + "go1.25.0.openbsd-arm.tar.gz", + "a1e31d0bf22172ddde42edf5ec811ef81be43433df0948ece52fecb247ccfd8d" + ], + "openbsd_arm64": [ + "go1.25.0.openbsd-arm64.tar.gz", + "343ea8edd8c218196e15a859c6072d0dd3246fbbb168481ab665eb4c4140458d" + ], + "openbsd_ppc64": [ + "go1.25.0.openbsd-ppc64.tar.gz", + "694c14da1bcaeb5e3332d49bdc2b6d155067648f8fe1540c5de8f3cf8e157154" + ], + "openbsd_riscv64": [ + "go1.25.0.openbsd-riscv64.tar.gz", + "aa510ad25cf54c06cd9c70b6d80ded69cb20188ac6e1735655eef29ff7e7885f" + ], + "plan9_386": [ + "go1.25.0.plan9-386.tar.gz", + "46f8cef02086cf04bf186c5912776b56535178d4cb319cd19c9fdbdd29231986" + ], + "plan9_amd64": [ + "go1.25.0.plan9-amd64.tar.gz", + "29b34391d84095e44608a228f63f2f88113a37b74a79781353ec043dfbcb427b" + ], + "plan9_arm": [ + "go1.25.0.plan9-arm.tar.gz", + "0a047107d13ebe7943aaa6d54b1d7bbd2e45e68ce449b52915a818da715799c2" + ], + "solaris_amd64": [ + "go1.25.0.solaris-amd64.tar.gz", + "9977f9e4351984364a3b2b78f8b88bfd1d339812356d5237678514594b7d3611" + ], + "windows_386": [ + "go1.25.0.windows-386.zip", + "df9f39db82a803af0db639e3613a36681ab7a42866b1384b3f3a1045663961a7" + ], + "windows_amd64": [ + "go1.25.0.windows-amd64.zip", + "89efb4f9b30812eee083cc1770fdd2913c14d301064f6454851428f9707d190b" + ], + "windows_arm64": [ + "go1.25.0.windows-arm64.zip", + "27bab004c72b3d7bd05a69b6ec0fc54a309b4b78cc569dd963d8b3ec28bfdb8c" ] } } - } + }, + "factsVersions": {} } diff --git a/README.md b/README.md index b4bd3a49..d9e68b81 100644 --- a/README.md +++ b/README.md @@ -1,201 +1,140 @@
- + Website | Getting Started - | Documentation + | Documentation | Discord | Contributing
[ZML]: https://zml.ai/ [Getting Started]: #getting-started -[Documentation]: https://docs.zml.ai +[Documentation]: ./docs/README.md [Contributing]: ./CONTRIBUTING.md [Discord]: https://discord.gg/6y72SN2E7H -# Bonjour 👋 +# About -At ZML, we are creating exciting AI products on top of our high-performance -AI inference stack. Our stack is built for production, using the amazing -[Zig](https://ziglang.org) language, [MLIR](https://mlir.llvm.org), and the -power of [Bazel](https://bazel.build). +ZML is a production inference stack, purpose-built to decouple AI workloads from proprietary hardware. -
-
Take me straight to getting started or give me a taste 🥐!
-
+Any model, many hardwares, one codebase, peak performance. ---- - -  - -# We're happy to share! -We're very happy to share our inference stack with the World and hope it allows -you, too, to build cool and exciting AI projects. - -To give you a glimpse of what you can do with ZML, here is an early demo: - -
- -It shows a prototype running a LLaMA2 model sharded on 1 NVIDIA RTX 4090, 1 AMD -6800XT, and 1 Google Cloud TPU v2. All accelerators were hosted in different -locations, with activations being passed over a VPN. - -All processes used the same model code, cross-compiled on a Mac, and copied onto -the servers. - -For more inspiration, see also the examples below or check out the -[examples](./examples) folder. - - - -# Getting started +Compiled directly to NVIDIA, AMD, INTEL, TPU, Trainium for peak hardware performance on any accelerator. No rewriting. +It is built using the +[Zig](https://ziglang.org) language, [MLIR](https://mlir.llvm.org), and [Bazel](https://bazel.build). +# Getting Started ## Prerequisites We use `bazel` to build ZML and its dependencies. The only prerequisite is -`bazel`, which we recommend to download through `bazelisk`, a version manager -for `bazel`. - -**Please note: If you do not wish to install `bazel`** system-wide, we provide -[examples/bazel.sh](examples/bazel.sh) which downloads it to your home folder -and runs it. - -**Install Bazel** (recommended): - -
+`bazel`, which we recommend installing through `bazelisk`. ### macOS - -``` +```bash brew install bazelisk ``` -
- -
### Linux - -``` -curl -L -o /usr/local/bin/bazel 'https://github.com/bazelbuild/bazelisk/releases/download/v1.20.0/bazelisk-linux-amd64' +```bash +curl -L -o /usr/local/bin/bazel 'https://github.com/bazelbuild/bazelisk/releases/download/v1.28.0/bazelisk-linux-amd64' chmod +x /usr/local/bin/bazel ``` -
+## 30-Second Smoke Test -## Run a pre-packaged model +Run the MNIST example: -We have implemented a variety of example models in ZML. See our reference -implementations in the -[examples](https://github.com/zml/zml/tree/master/examples/) folder. - -### MNIST - -The [classic](https://en.wikipedia.org/wiki/MNIST_database) handwritten digits -recognition task. The model is tasked to recognize a handwritten digit, which -has been converted to a 28x28 pixel monochrome image. `Bazel` will download a -pre-trained model, and the test dataset. The program will load the model, -compile it, and classify a randomly picked example from the test dataset. - -On the command line: - -``` -cd examples -bazel run -c opt //mnist - -# or -./bazel.sh run -c opt //mnist +```bash +bazel run //examples/mnist ``` +This downloads a small pretrained MNIST model, compiles it, loads the weights, and +classifies a random handwritten digit. +## LLM Quickstart -### TinyLlama, Stories 15M +The main LLM example is [`//examples/llm`](./examples/llm). It currently supports: -Our LLM examples start with a small model trained specifically on children's -history books. This model has been trained by [Andrej -Karpathy](https://x.com/karpathy); you can read more about it on his -[GitHub](https://github.com/karpathy/llama2.c). +- Llama 3.1 / 3.2 +- Qwen 3.5 +- LFM 2.5 -``` -cd examples -bazel run -c opt //llama:TinyLlama-Stories-15M -bazel run -c opt //llama:TinyLlama-Stories-15M -- --prompt="Once upon a time, there was a cute little dragon" +Authenticate with Hugging Face if you want to load gated repos such as Meta +Llama: + +```bash +bazel run //tools/hf -- auth login ``` -### OpenLLama 3B +Alternatively, set the `HF_TOKEN` environment variable. -``` -cd examples -bazel run -c opt //llama:OpenLLaMA-3B -bazel run -c opt //llama:OpenLLaMA-3B -- --prompt="Once upon a time," +Then run a prompt directly: + +```bash +bazel run //examples/llm -- --model=hf://meta-llama/Llama-3.2-1B-Instruct --prompt="What is the capital of France?" ``` -### Meta Llama 3 8B +Open the interactive chat loop by omitting `--prompt`: -This model has restrictions, see -[here](https://huggingface.co/meta-llama/Meta-Llama-3-8B). It **requires -approval from Meta on Huggingface**, which can take a few hours to get granted. - -While waiting, you can already generate an access token to log into HuggingFace -from `bazel`; see [here](./docs/huggingface-access-token.md). - -Once you've been granted access, you're ready to download a gated model like -`Meta-Llama-3-8b`! - -``` -# requires token in $HOME/.cache/huggingface/token, as created by the -# `huggingface-cli login` command, or the `HUGGINGFACE_TOKEN` environment variable. -cd examples -bazel run -c opt //llama:Llama-3.1-8B-Instruct -bazel run -c opt //llama:Llama-3.1-8B-Instruct -- --prompt="Once upon a time," +```bash +bazel run //examples/llm -- --model=hf://meta-llama/Llama-3.2-1B-Instruct ``` +You can also load from: + +- a local directory: `--model=/var/models/meta-llama/Llama-3.2-1B-Instruct` +- S3: `--model=s3://bucket/path/to/model` ## Running Models on GPU / TPU -You can compile models for accelerator runtimes by appending one or more of the -following arguments to the command line when compiling / running a model: +Append one or more platform flags when compiling or running: -- NVIDIA CUDA: `--@zml//runtimes:cuda=true` -- AMD RoCM: `--@zml//runtimes:rocm=true` -- Google TPU: `--@zml//runtimes:tpu=true` -- **AVOID CPU:** `--@zml//runtimes:cpu=false` +- NVIDIA CUDA: `--@zml//platforms:cuda=true` +- AMD RoCM: `--@zml//platforms:rocm=true` +- Intel OneAPI: `--@zml//platforms:oneapi=true` +- Google TPU: `--@zml//platforms:tpu=true` +- AWS Trainium / Inferentia 2: `--@zml//platforms:neuron=true` +- Disable CPU compilation: `--@zml//platforms:cpu=false` -The latter, avoiding compilation for CPU, cuts down compilation time. +Example on CUDA: -So, to run the OpenLLama model from above on your host sporting an NVIDIA GPU, -run the following: - -``` -cd examples -bazel run -c opt //llama:OpenLLaMA-3B \ - --@zml//runtimes:cuda=true \ - -- --prompt="Once upon a time," +```bash +bazel run //examples/llm --@zml//platforms:cuda=true -- --model=hf://meta-llama/Llama-3.2-1B-Instruct --prompt="Write a haiku about Zig" ``` +Example on ROCm: + +```bash +bazel run //examples/llm --@zml//platforms:rocm=true -- --model=hf://meta-llama/Llama-3.2-1B-Instruct --prompt="Write a haiku about Zig" +``` + +Example on Intel OneAPI: + +```bash +bazel run //examples/llm --@zml//platforms:cpu=false --@zml//platforms:oneapi=true -- --model=hf://meta-llama/Llama-3.2-1B-Instruct --prompt="Write a haiku about Zig" +``` ## Run Tests -``` +```bash bazel test //zml:test ``` +# Examples -# A taste of ZML - - - -## MNIST +- [`examples/llm`](./examples/llm): unified LLM CLI for Llama, Qwen, and LFM +- [`examples/mnist`](./examples/mnist): smallest end-to-end model run +- [`examples/sharding`](./examples/sharding): logical mesh, partitioners, shard-local execution, profiler output +- [`examples/io`](./examples/io): inspect and load local, `hf://`, `https://`, and `s3://` repositories through the VFS layer +- [`examples/benchmark`](./examples/benchmark): measure loading and execution performance +# A Taste Of ZML ```zig -const std = @import("std"); -const zml = @import("zml"); - -/// Model definition const Mnist = struct { fc1: Layer, fc2: Layer, @@ -204,65 +143,84 @@ const Mnist = struct { weight: zml.Tensor, bias: zml.Tensor, + pub fn init(store: zml.io.TensorStore.View) Layer { + return .{ + .weight = store.createReplicatedTensor("weight", .{ .d_out, .d }), + .bias = store.createReplicatedTensor("bias", .{.d_out}), + }; + } + pub fn forward(self: Layer, input: zml.Tensor) zml.Tensor { - return self.weight.matmul(input).add(self.bias).relu(); + return self.weight.dot(input, .d).add(self.bias).relu().withTags(.{.d}); } }; + pub fn init(store: zml.io.TensorStore.View) Mnist { + return .{ + .fc1 = .init(store.withPrefix("fc1")), + .fc2 = .init(store.withPrefix("fc2")), + }; + } + + pub fn load( + self: *const Mnist, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + store: *const zml.io.TensorStore, + meshes: []const *const zml.Mesh, + ) !zml.Bufferized(Mnist) { + return zml.io.load(Mnist, self, allocator, io, platform, store, .{ + .meshes = meshes, + .parallelism = 1, + .dma_chunks = 1, + .dma_chunk_size = 16 * 1024 * 1024, + }); + } + + pub fn unloadBuffers(self: *zml.Bufferized(Mnist)) void { + self.fc1.weight.deinit(); + self.fc1.bias.deinit(); + self.fc2.weight.deinit(); + self.fc2.bias.deinit(); + } + /// just two linear layers + relu activation pub fn forward(self: Mnist, input: zml.Tensor) zml.Tensor { - std.log.info("Compiling for target: {s}", .{@tagName(input.getContext().target())}); - var x = input.flattenAll().convert(.f32); + var x = input.flatten().convert(.f32).withTags(.{.d}); const layers: []const Layer = &.{ self.fc1, self.fc2 }; for (layers) |layer| { - x = zml.call(layer, .forward, .{x}); + x = layer.forward(x); } - return x.argMax(0, .u8).indices; + return x.argMax(0).indices.convert(.u8); } }; ``` +For a full walkthrough, see: +- [Getting Started](./docs/tutorials/getting_started.md) +- [Writing your first model](./docs/tutorials/write_first_model.md) +- [ZML Concepts](./docs/learn/concepts.md) +- [Deploying on a server](./docs/howtos/deploy_on_server.md) -## Tagged Tensors - -```zig -const Sdpa = struct { - pub fn forward(_: Sdpa, ctx: *zml.Context, q_: zml.Tensor, k_: zml.Tensor, v_: zml.Tensor) zml.Tensor { - const q = q_.withTags(.{ .b, .h, .q, .hd }); - const k = k_.withTags(.{ .b, .h, .k, .hd }); - const v = v_.withTags(.{ .b, .h, .k, .hd }); - const attn_mask = zml.nn.causalAttnMask(ctx, .{ .q = q.dim(.q), .k = k.dim(.k) }, q.dtype(), null); - return zml.nn.sdpa(ctx, q, k, v, .{ .attn_mask = attn_mask }); - } -}; -``` - - - - -# Where to go next: - -You might want to check out more [examples](./examples), read through the -[documentation directly on GitHub](./docs/README.md), or, for the full rendering -experience, browse the -[online documentation with included API reference](https://docs.zml.ai). - +# Where To Go Next +- Run more examples in [`./examples`](./examples) +- Read the example-specific notes in [`examples/llm/README.md`](./examples/llm/README.md) +- Learn tagged dimensions in [`working_with_tensors.md`](./docs/tutorials/working_with_tensors.md) +- Start building a model with [`write_first_model.md`](./docs/tutorials/write_first_model.md) +- Explore deployment in [`deploy_on_server.md`](./docs/howtos/deploy_on_server.md) # Contributing See [here][Contributing]. - - # License ZML is licensed under the [Apache 2.0 license](./LICENSE). - - -# Thanks to our contributors +# Thanks To Our Contributors diff --git a/async/async.zig b/async/async.zig deleted file mode 100644 index 19166532..00000000 --- a/async/async.zig +++ /dev/null @@ -1,403 +0,0 @@ -const std = @import("std"); -const xev = @import("xev"); -const libcoro = @import("libcoro"); -const aio = libcoro.asyncio; - -/// Normalize from a real tuple to a generic tuple. This is needed because -/// real tuples are reifed tuples are not the same. -fn NormalizedTuple(comptime T: type) type { - const ti = @typeInfo(T).Struct; - var types: [ti.fields.len]type = undefined; - inline for (ti.fields, 0..) |field, i| { - types[i] = field.type; - } - return std.meta.Tuple(&types); -} - -pub fn FnSignature(comptime func: anytype, comptime argsT: ?type) type { - return struct { - pub const FuncT = if (@TypeOf(func) == type) func else @TypeOf(func); - pub const ArgsT = blk: { - if (@typeInfo(FuncT).Fn.params.len == 0) { - break :blk @TypeOf(.{}); - } - break :blk argsT orelse std.meta.ArgsTuple(FuncT); - }; - pub const ReturnT = @TypeOf(@call(.auto, func, @as(ArgsT, undefined))); - pub const ReturnPayloadT = blk: { - break :blk switch (@typeInfo(ReturnT)) { - .ErrorUnion => |u| u.payload, - else => ReturnT, - }; - }; - pub const ReturnErrorSet: ?type = blk: { - break :blk switch (@typeInfo(ReturnT)) { - .ErrorUnion => |u| u.error_set, - else => null, - }; - }; - }; -} - -pub fn Frame(comptime func: anytype) type { - const Signature = FnSignature(func, null); - return FrameEx(func, Signature.ArgsT); -} - -pub fn FrameEx(comptime func: anytype, comptime argsT: type) type { - return FrameExx(func, argsT); -} - -fn FrameExx(comptime func: anytype, comptime argsT: type) type { - return struct { - const Self = @This(); - const Signature = FnSignature(func, argsT); - const FrameT = libcoro.FrameT(func, .{ .ArgsT = Signature.ArgsT }); - - inner: FrameT, - - pub fn await_(self: *Self) Signature.ReturnT { - defer { - self.inner.deinit(); - self.* = undefined; - } - return libcoro.xawait(self.inner); - } - - fn from(other: anytype) !Self { - return .{ .inner = FrameT.wrap(other.frame()) }; - } - }; -} - -pub fn async_(comptime func: anytype, args: anytype) !FrameEx(func, @TypeOf(args)) { - const frame = try aio.xasync(func, args, null); - return FrameEx(func, @TypeOf(args)).from(frame); -} - -pub fn call(comptime func: anytype, args: std.meta.ArgsTuple(@TypeOf(func))) @TypeOf(callGeneric(func, args)) { - return callGeneric(func, args); -} - -pub fn callGeneric(comptime func: anytype, args: anytype) FnSignature(func, @TypeOf(args)).ReturnT { - const Signature = FnSignature(func, @TypeOf(args)); - - const TaskT = struct { - const Self = @This(); - - _task: xev.ThreadPool.Task = .{ .callback = &Self.run }, - - notif: Notification, - args: *const Signature.ArgsT, - result: Signature.ReturnT = undefined, - - pub fn run(task_: *xev.ThreadPool.Task) void { - const task: *Self = @alignCast(@fieldParentPtr("_task", task_)); - task.result = @call(.auto, func, task.args.*); - task.notif.notify() catch @panic("Unable to notify"); - } - }; - - var newtask: TaskT = .{ - .notif = Notification.init() catch @panic("Notification.init failed"), - .args = &args, - }; - // NOTE(Corentin): Disable this to avoid crash with epoll backend - //defer newtask.notif.deinit(); - - AsyncThread.current.thread_pool.schedule(xev.ThreadPool.Batch.from(&newtask._task)); - newtask.notif.wait() catch @panic("Unable to wait for notification"); - return newtask.result; -} - -pub fn tick() void { - AsyncThread.current.executor.exec.tick(); -} - -pub fn sleep(ms: u64) !void { - try aio.sleep(AsyncThread.current.executor, ms); -} - -pub const Notification = struct { - inner: aio.AsyncNotification, - - pub fn init() !Notification { - return .{ - .inner = aio.AsyncNotification.init(AsyncThread.current.executor, try xev.Async.init()), - }; - } - - pub fn notify(self: *Notification) !void { - try self.inner.notif.notify(); - } - - pub fn wait(self: *Notification) !void { - try self.inner.wait(); - } - - pub fn deinit(self: *Notification) void { - self.inner.notif.deinit(); - self.* = undefined; - } -}; - -pub const AsyncThread = struct { - threadlocal var current: AsyncThread = undefined; - - executor: *aio.Executor, - loop: *xev.Loop, - thread_pool: *xev.ThreadPool, - - pub fn main(allocator: std.mem.Allocator, comptime func: anytype, args: anytype) !FnSignature(func, NormalizedTuple(@TypeOf(args))).ReturnPayloadT { - const Signature = FnSignature(func, NormalizedTuple(@TypeOf(args))); - - var thread_pool = xev.ThreadPool.init(.{}); - defer { - thread_pool.shutdown(); - thread_pool.deinit(); - } - - var loop = try xev.Loop.init(.{ - .thread_pool = &thread_pool, - }); - defer loop.deinit(); - - var executor = aio.Executor.init(&loop); - - AsyncThread.current = .{ - .executor = &executor, - .loop = &loop, - .thread_pool = &thread_pool, - }; - - aio.initEnv(.{ - .stack_allocator = allocator, - .default_stack_size = 16 * 1024 * 1024, - }); - - if (Signature.ReturnErrorSet) |_| { - return try aio.run(&executor, func, args, null); - } else { - return aio.run(&executor, func, args, null); - } - } -}; - -pub fn StdIn() !File { - return File.init(std.io.getStdIn()) catch @panic("Unable to open stdin"); -} - -pub fn StdOut() File { - return File.init(std.io.getStdOut()) catch @panic("Unable to open stdout"); -} - -pub fn StdErr() File { - return File.init(std.io.getStdErr()) catch @panic("Unable to open stderr"); -} - -pub const File = struct { - pub const SeekError = FnSignature(File.seekTo, null).ReturnErrorSet.? || FnSignature(File.seekBy, null).ReturnErrorSet.?; - pub const GetSeekPosError = SeekError || FnSignature(File.stat, null).ReturnErrorSet.?; - pub const Reader = std.io.GenericReader(File, FnSignature(File.read, null).ReturnErrorSet.?, File.read); - pub const Writer = std.io.GenericWriter(File, FnSignature(File.write, null).ReturnErrorSet.?, File.write); - pub const SeekableStream = std.io.SeekableStream( - File, - SeekError, - GetSeekPosError, - seekTo, - seekBy, - getPos, - getEndPos, - ); - - inner: aio.File, - - fn asFile(self: File) std.fs.File { - return .{ .handle = self.inner.file.fd }; - } - - pub fn init(file_: std.fs.File) !File { - return .{ .inner = aio.File.init(AsyncThread.current.executor, try xev.File.init(file_)) }; - } - - pub fn fromFd(fd: std.fs.File.Handle) !File { - return .{ .inner = aio.File.init(AsyncThread.current.executor, try xev.File.initFd(fd)) }; - } - - pub fn open(path: []const u8, flags: std.fs.File.OpenFlags) !File { - return init(try call(std.fs.Dir.openFile, .{ std.fs.cwd(), path, flags })); - } - - pub fn read(self: File, buf: []u8) !usize { - // NOTE(Corentin): Early return is required to avoid error with xev on Linux with io_uring backend. - if (buf.len == 0) return 0; - - return self.inner.read(.{ .slice = buf }) catch |err| switch (err) { - // NOTE(Corentin): read shouldn't return an error on EOF, but a read length of 0 instead. This is to be iso with std.fs.File. - error.EOF => 0, - else => err, - }; - } - - pub fn pread(self: File, buf: []u8, offset: u64) !usize { - // NOTE(Corentin): Early return is required to avoid error with xev on Linux with io_uring backend. - if (buf.len == 0) return 0; - - return self.inner.pread(.{ .slice = buf }, offset) catch |err| switch (err) { - // NOTE(Corentin): pread shouldn't return an error on EOF, but a read length of 0 instead. This is to be iso with std.fs.File. - error.EOF => 0, - else => err, - }; - } - - pub fn write(self: File, buf: []const u8) !usize { - return self.inner.write(.{ .slice = buf }); - } - - pub fn pwrite(self: File, buf: []const u8, offset: u64) !usize { - return self.inner.pwrite(.{ .slice = buf }, offset); - } - - pub fn close(self: File) !void { - return self.inner.close(); - } - - pub fn reader(self: File) Reader { - return .{ .context = self }; - } - - pub fn seekableStream(file: File) SeekableStream { - return .{ .context = file }; - } - - pub fn writer(self: File) Writer { - return .{ .context = self }; - } - - pub fn stat(self: File) !std.fs.File.Stat { - return try call(std.fs.File.stat, .{self.asFile()}); - } - - pub fn seekBy(self: File, offset: i64) !void { - try call(std.fs.File.seekBy, .{ self.asFile(), offset }); - } - - pub fn seekTo(self: File, offset: u64) !void { - try call(std.fs.File.seekTo, .{ self.asFile(), offset }); - } - - pub fn getPos(self: File) !u64 { - return try call(std.fs.File.getPos, .{self.asFile()}); - } - - pub fn getEndPos(self: File) !u64 { - return try call(std.fs.File.getEndPos, .{self.asFile()}); - } -}; - -pub const Socket = struct { - pub const TCP = struct { - pub const Reader = std.io.GenericReader(TCP, FnSignature(TCP.read, null).ReturnErrorSet.?, TCP.read); - pub const Writer = std.io.GenericWriter(TCP, FnSignature(TCP.write, null).ReturnErrorSet.?, TCP.write); - - inner: aio.TCP, - - pub fn init(addr: std.net.Address) !TCP { - return .{ .inner = aio.TCP.init(AsyncThread.current.executor, try xev.TCP.init(addr)) }; - } - - pub fn deinit(self: *TCP) void { - self.inner.shutdown(); - } - - pub fn connect(self: *TCP, addr: std.net.Address) !void { - return self.inner.connect(addr); - } - - pub fn read(self: *TCP, buf: []u8) !usize { - return self.inner.read(.{ .slice = buf }); - } - - pub fn write(self: *TCP, buf: []const u8) !usize { - return self.inner.write(.{ .slice = buf }); - } - - pub fn close(self: *TCP) !void { - defer self.* = undefined; - return self.inner.close(); - } - - pub fn reader(self: File) Reader { - return .{ .context = self }; - } - - pub fn writer(self: File) Writer { - return .{ .context = self }; - } - }; - - pub const UDP = struct { - pub const Reader = std.io.GenericReader(UDP, FnSignature(UDP.read, null).ReturnErrorSet.?, UDP.read); - pub const WriterContext = struct { - file: UDP, - addr: std.net.Address, - }; - pub const Writer = std.io.GenericWriter(WriterContext, FnSignature(UDP.write, null).ReturnErrorSet.?, struct { - fn call(self: WriterContext, buf: []const u8) !usize { - return self.file.write(self.addr, buf); - } - }.call); - - inner: aio.UDP, - - pub fn init(addr: std.net.Address) !UDP { - return .{ .inner = aio.UDP.init(AsyncThread.current.executor, try xev.UDP.init(addr)) }; - } - - pub fn read(self: UDP, buf: []u8) !usize { - return self.inner.read(.{ .slice = buf }); - } - - pub fn write(self: UDP, addr: std.net.Address, buf: []const u8) !usize { - return self.inner.write(addr, .{ .slice = buf }); - } - - pub fn close(self: *UDP) !void { - defer self.* = undefined; - return self.inner.close(); - } - - pub fn reader(self: File) Reader { - return .{ .context = self }; - } - - pub fn writer(self: File, addr: std.net.Address) Writer { - return .{ - .context = .{ - .file = self, - .addr = addr, - }, - }; - } - }; -}; - -pub const Mutex = struct { - const VoidChannel = libcoro.Channel(void, .{ .capacity = 1 }); - - inner: VoidChannel, - - pub fn init() Mutex { - return .{ .inner = VoidChannel.init(&AsyncThread.current.executor.exec) }; - } - - pub fn lock(self: *Mutex) !void { - try self.inner.send({}); - } - - pub fn unlock(self: *Mutex) void { - _ = self.inner.recv(); - } -}; - -pub const inCoro = libcoro.inCoro; diff --git a/examples/bazel.sh b/bazel.sh similarity index 96% rename from examples/bazel.sh rename to bazel.sh index 345a7ad5..c00dd89c 100755 --- a/examples/bazel.sh +++ b/bazel.sh @@ -1,5 +1,5 @@ #!/bin/bash -BAZELISK_VERSION=v1.20.0 +BAZELISK_VERSION=v1.25.0 case $OSTYPE in "darwin"*) diff --git a/bazel/BUILD.bazel b/bazel/BUILD.bazel index e69de29b..eb2b2ab2 100644 --- a/bazel/BUILD.bazel +++ b/bazel/BUILD.bazel @@ -0,0 +1,25 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library") + +cc_library( + name = "retain_runfiles_init", + srcs = ["runfiles_init.c"], + # Retain the constructor even when its object is in a static archive. + # Mach-O adds an underscore to the exported C symbol. + linkopts = select({ + "@platforms//os:macos": ["-Wl,-u,__bazel_runfiles_init"], + "//conditions:default": ["-Wl,-u,_bazel_runfiles_init"], + }), +) + +zig_library( + name = "bazel", + main = "bazel.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + ":retain_runfiles_init", + "//stdx", + "@rules_zig//zig/runfiles", + ], +) diff --git a/bazel/bazel.zig b/bazel/bazel.zig new file mode 100644 index 00000000..67513227 --- /dev/null +++ b/bazel/bazel.zig @@ -0,0 +1,27 @@ +const std = @import("std"); + +const bazel_runfiles = @import("runfiles"); +const stdx = @import("stdx"); + +var runfiles_global: bazel_runfiles.Runfiles = undefined; + +pub export fn _bazel_runfiles_init_impl(argc: c_int, argv: [*c][*:0]u8, envp: [*:null]?[*:0]u8) callconv(.c) void { + _ = argc; + _ = envp; + var threaded: std.Io.Threaded = .init_single_threaded; + runfiles_global = bazel_runfiles.Runfiles.create(.{ + .allocator = std.heap.c_allocator, + .io = threaded.io(), + .argv0 = std.mem.span(argv[0]), + .directory = if (std.c.getenv("RUNFILES_DIR")) |directory| std.mem.span(directory) else null, + .manifest = if (std.c.getenv("RUNFILES_MANIFEST_FILE")) |manifest| std.mem.span(manifest) else null, + }) catch { + std.debug.panic("Unable to find runfiles", .{}); + } orelse { + std.debug.panic("Unable to initialize runfiles", .{}); + }; +} + +pub fn runfiles(source_repository: []const u8) !bazel_runfiles.Runfiles.WithSourceRepo { + return runfiles_global.withSourceRepo(source_repository); +} diff --git a/bazel/cc_import.bzl b/bazel/cc_import.bzl index 92cb0144..fc6ef727 100644 --- a/bazel/cc_import.bzl +++ b/bazel/cc_import.bzl @@ -1,4 +1,5 @@ -load("@bazel_tools//tools/build_defs/cc:cc_import.bzl", _cc_import = "cc_import") +load("@rules_cc//cc:defs.bzl", "cc_library", _cc_import = "cc_import") +load("@rules_cc//cc/common:cc_info.bzl", "CcInfo") load(":patchelf.bzl", "patchelf") def _cc_import_runfiles_impl(ctx): @@ -36,61 +37,44 @@ _cc_import_runfiles = rule( ) def cc_import( - name, - static_library = None, - pic_static_library = None, - shared_library = None, - interface_library = None, - data = None, - deps = None, - visibility = None, - soname = None, - add_needed = None, - remove_needed = None, - replace_needed = None, - **kwargs): - if shared_library and (soname or add_needed or remove_needed or replace_needed): - patched_name = "{}_patchelf".format(name) + name, + static_library = None, + pic_static_library = None, + shared_library = None, + interface_library = None, + data = None, + deps = [], + visibility = None, + soname = None, + add_needed = None, + remove_needed = None, + replace_needed = None, + rename_dynamic_symbols = None, + **kwargs): + if shared_library and (soname or add_needed or remove_needed or replace_needed or rename_dynamic_symbols): + patched_name = "{}.patchelf".format(name) patchelf( name = patched_name, - shared_library = shared_library, + src = shared_library, soname = soname, add_needed = add_needed, remove_needed = remove_needed, replace_needed = replace_needed, + rename_dynamic_symbols = rename_dynamic_symbols, ) shared_library = ":" + patched_name - if data: - _cc_import( - name = name + "_no_runfiles", - static_library = static_library, - pic_static_library = pic_static_library, - shared_library = shared_library, - interface_library = interface_library, - data = data, - deps = deps, - **kwargs - ) - _cc_import_runfiles( - name = name, - src = ":{}_no_runfiles".format(name), - static_library = static_library, - pic_static_library = pic_static_library, - shared_library = shared_library, - interface_library = interface_library, - data = data, - deps = deps, - visibility = visibility, - ) - else: - _cc_import( - name = name, - static_library = static_library, - pic_static_library = pic_static_library, - shared_library = shared_library, - interface_library = interface_library, - data = data, - deps = deps, - visibility = visibility, - **kwargs - ) + _cc_import( + name = name + ".cc_import", + static_library = static_library, + pic_static_library = pic_static_library, + shared_library = shared_library, + interface_library = interface_library, + visibility = visibility, + ) + cc_library( + name = name, + data = data, + deps = deps + [name + ".cc_import"], + visibility = visibility, + **kwargs + ) diff --git a/bazel/dpkg.bzl b/bazel/dpkg.bzl new file mode 100644 index 00000000..343f188a --- /dev/null +++ b/bazel/dpkg.bzl @@ -0,0 +1,29 @@ +def _packages_to_dict(txt): + packages = {} + current_pkg = {} + for line in txt.splitlines(): + if line == "": + if current_pkg: + pkg_name = current_pkg["Package"] + pkg_version = current_pkg["Version"] + if pkg_name not in packages: + packages[pkg_name] = {} + packages[pkg_name][pkg_version] = struct(**current_pkg) + current_pkg = {} + continue + if line.startswith(" "): + current_pkg[key] += line + continue + split = line.split(": ", 1) + key = split[0] + value = len(split) > 1 and split[1] or "" + current_pkg[key] = value + return packages + +def _read_packages(mctx, label): + data = mctx.read(Label(label)) + return _packages_to_dict(data) + +dpkg = struct( + read_packages = _read_packages, +) diff --git a/bazel/http_aea_archive.bzl b/bazel/http_aea_archive.bzl new file mode 100644 index 00000000..439a6a41 --- /dev/null +++ b/bazel/http_aea_archive.bzl @@ -0,0 +1,164 @@ +"""Repository rule for downloading and extracting Apple Encrypted Archive assets on macOS and Linux AMD64/ARM64.""" + +load("@bazel_lib//lib:repo_utils.bzl", "repo_utils") + +def _fail_result(step, result): + if result.return_code != 0: + fail("%s failed with exit code %d\nstdout:\n%s\nstderr:\n%s" % ( + step, + result.return_code, + result.stdout, + result.stderr, + )) + +def _execute(rctx, step, argv, quiet = True): + result = rctx.execute([str(arg) for arg in argv], quiet = quiet) + _fail_result(step, result) + return result + +def _host_tools(rctx): + platform = repo_utils.platform(rctx) + if platform not in ["darwin_amd64", "darwin_arm64", "linux_amd64", "linux_arm64"]: + fail("http_aea_archive supports macOS and Linux AMD64/ARM64 repository hosts; got %s/%s" % (rctx.os.name, rctx.os.arch)) + ipsw = Label("@ipsw_{}//:ipsw".format(platform)) if platform.startswith("linux_") else None + sevenzip = Label("@sevenzip_{}//:7zz".format(platform)) + return (ipsw, sevenzip) + +def _single_restore_dmg(restore_dirs): + dmgs = [] + entries = [] + for restore_dir in restore_dirs: + if not restore_dir.exists: + continue + for entry in restore_dir.readdir(): + entries.append(str(entry)) + if not entry.is_dir and entry.basename.endswith(".dmg"): + dmgs.append(entry) + + if len(dmgs) != 1: + fail("expected exactly one DMG under AssetData/Restore, got %d; entries: %s" % ( + len(dmgs), + ", ".join(sorted(entries)), + )) + + return dmgs[0] + +def _linux_restore_dirs(rctx, ipsw, aar): + # Create an explicit empty config. It avoids loading the user's ipsw configuration. + rctx.file("ipsw-config.yaml", "{}\n") + _execute(rctx, "ipsw ota extract", [ + rctx.path(ipsw), + "--config", + rctx.path("ipsw-config.yaml"), + "ota", + "extract", + aar, + "--key-val", + rctx.attr.archive_decryption_key, + "--pattern", + "AssetData/Restore/.*\\.dmg$", # pattern to match DMG files under AssetData/Restore + "--output", + rctx.path("ipsw-output"), + "--confirm", + "--no-color", + ]) + output = rctx.path("ipsw-output") + restore_dirs = [] + if output.exists: + # ipsw prefixes extracted paths with an asset-specific directory. + for entry in output.readdir(): + if entry.is_dir: + restore_dirs.append(entry.get_child("AssetData", "Restore")) + return restore_dirs + +def _http_aea_archive_impl(rctx): + ipsw, sevenzip = _host_tools(rctx) + + urls = [] + if rctx.attr.url: + urls.append(rctx.attr.url) + urls.extend(rctx.attr.urls) + if not urls: + fail("http_aea_archive requires url or urls") + + if not rctx.attr.sha256 and not rctx.attr.integrity: + fail("http_aea_archive requires sha256 or integrity for reproducible downloads") + if not rctx.attr.archive_decryption_key: + fail("archive_decryption_key is required") + + aar = rctx.path("asset.aar") # path to the downloaded Apple Encrypted Archive (AAR) file + download_kwargs = { + "canonical_id": rctx.attr.canonical_id, + "integrity": rctx.attr.integrity, + "output": aar, + "sha256": rctx.attr.sha256, + "url": urls, + } + if rctx.attr.netrc: + download_kwargs["auth"] = rctx.use_netrc( + rctx.read_netrc(rctx.attr.netrc), + urls, + rctx.attr.auth_patterns, + ) + + rctx.download(**download_kwargs) + + if ipsw: + restore_dirs = _linux_restore_dirs(rctx, ipsw, aar) + else: + _execute(rctx, "aa patch", [ + "/usr/bin/aa", + "patch", + "-i", + aar, + "-key-value", + "base64:%s" % rctx.attr.archive_decryption_key, + "-src", + "/var/empty", + "-dst", + rctx.path("."), + ]) + restore_dirs = [rctx.path("AssetData/Restore")] + dmg = _single_restore_dmg(restore_dirs) + + extract_args = [ + rctx.path(sevenzip), + "x", + "-y", + dmg, + ] + extract_args.extend(rctx.attr.includes) + _execute(rctx, "7z extract dmg", extract_args) + + # Central cleanup of temporary files and directories after extraction. + # If absent, rctx.delete will handle it gracefully. + rctx.delete("asset.aar") + rctx.delete("AssetData") + rctx.delete("ipsw-output") + rctx.delete("ipsw-config.yaml") + + rctx.template("BUILD.bazel", rctx.attr.build_file) + + return None + +http_aea_archive = repository_rule( + implementation = _http_aea_archive_impl, + attrs = { + "url": attr.string(), + "urls": attr.string_list(), + "archive_decryption_key": attr.string(mandatory = True), + "sha256": attr.string(), + "integrity": attr.string(), + "canonical_id": attr.string(), + "netrc": attr.string(), + "auth_patterns": attr.string_dict(), + "build_file": attr.label( + allow_single_file = True, + ), + "includes": attr.string_list( + default = [], + doc = "Patterns to pass to 7z for selective extraction from the DMG. If empty, extracts everything.", + ), + }, + doc = "Downloads and extracts an AEA-wrapped AppleArchive asset on macOS and Linux AMD64/ARM64.", +) diff --git a/bazel/http_deb_archive.bzl b/bazel/http_deb_archive.bzl index f847e6f3..968001d6 100644 --- a/bazel/http_deb_archive.bzl +++ b/bazel/http_deb_archive.bzl @@ -1,45 +1,253 @@ +load("@bazel_lib//lib:repo_utils.bzl", "repo_utils") +load( + "@bazel_tools//tools/build_defs/repo:cache.bzl", + "DEFAULT_CANONICAL_ID_ENV", + "get_default_canonical_id", +) load( "@bazel_tools//tools/build_defs/repo:utils.bzl", "get_auth", "patch", + "update_attrs", "workspace_and_buildfile", ) -def _http_deb_archive_impl(rctx): - if rctx.attr.build_file and rctx.attr.build_file_content: - fail("Only one of build_file and build_file_content can be provided.") - download_info = rctx.download_and_extract( - url = rctx.attr.urls, - output = "tmp", - sha256 = rctx.attr.sha256, - type = "deb", - stripPrefix = "", - canonical_id = " ".join(rctx.attr.urls), - auth = get_auth(rctx, rctx.attr.urls), +def _get_source_urls(rctx): + if not rctx.attr.url and not rctx.attr.urls: + fail("At least one of url and urls must be provided") + + source_urls = [] + if rctx.attr.urls: + source_urls = rctx.attr.urls + if rctx.attr.url: + source_urls = [rctx.attr.url] + source_urls + return source_urls + +def _prefixed_pattern(strip_prefix, pattern): + if not strip_prefix: + return pattern + if not pattern: + return strip_prefix + if pattern == strip_prefix or pattern.startswith(strip_prefix + "/"): + return pattern + return strip_prefix + "/" + pattern + +def _downloaded_archive_name(rctx, source_urls): + if rctx.attr.type: + return ".downloaded.archive." + rctx.attr.type + + basename = source_urls[0].split("?", 1)[0].split("/")[-1] + if basename: + return ".downloaded." + basename + return ".downloaded.archive" + +def _placeholder_dir_from_exclude(pattern): + if pattern.endswith("/*"): + return pattern[:-2].strip("/") + return None + +def _ensure_placeholder_dirs(rctx, add_prefix): + for exclude in rctx.attr.excludes: + directory = _placeholder_dir_from_exclude(exclude) + if not directory: + continue + if add_prefix: + directory = add_prefix + "/" + directory + keep_file = directory + "/.http_deb_archive.keep" + rctx.file(keep_file, "") + rctx.delete(keep_file) + +def _host_bsdtar_label(rctx): + platform = repo_utils.platform(rctx) + binary = "tar.exe" if platform.startswith("windows_") else "tar" + return Label("@bsd_tar_toolchains_{}//:{}".format(platform, binary)) + +def _update_integrity_attrs(rctx, attrs, archive_info, remote_files_info, remote_patches_info): + integrity_override = {} + if not rctx.attr.sha256 and not rctx.attr.integrity: + integrity_override["integrity"] = archive_info.integrity + + remote_file_integrity = {path: info.integrity for path, info in remote_files_info.items()} + if rctx.attr.remote_file_integrity != remote_file_integrity: + integrity_override["remote_file_integrity"] = remote_file_integrity + + # TODO(cerisier): Remove this "if" when we no longer support bazel < 8.5.0 + if remote_patches_info: + remote_patch_integrity = {url: info.integrity for url, info in remote_patches_info.items()} + if rctx.attr.remote_patches != remote_patch_integrity: + integrity_override["remote_patches"] = remote_patch_integrity + + if not integrity_override: + return rctx.repo_metadata(reproducible = True) + + return rctx.repo_metadata( + attrs_for_reproducibility = update_attrs(rctx.attr, attrs.keys(), integrity_override), ) +# TODO(cerisier): Remove when we no longer support bazel < 8.5.0 +def symlink_files(ctx): + # type: (repository_ctx) -> None + """Utility function for symlinking local files. + + This is intended to be used in the implementation function of a repository rule. It assumes the + parameter `files` is present in `ctx.attr`. + + Existing files will be overwritten. + + Args: + ctx: The repository context of the repository rule calling this utility + function. + """ + for path, label in ctx.attr.files.items(): + src_path = ctx.path(label) + + # On Windows `ctx.symlink` may be implemented as a copy, so the file MUST be watched + ctx.watch(src_path) + if not src_path.exists: + fail("Input %s does not exist" % label) + if ctx.path(path).exists: + ctx.delete(path) + ctx.symlink(src_path, path) + +# TODO(cerisier): Remove when we no longer support bazel < 8.5.0 +def download_remote_files(ctx, auth = None): + """Utility function for downloading remote files. + + This rule is intended to be used in the implementation function of + a repository rule. It assumes the parameters `remote_file_urls` and + `remote_file_integrity` to be present in `ctx.attr`. + + Existing files will be overwritten. + + Args: + ctx: The repository context of the repository rule calling this utility + function. + auth: An optional dict specifying authentication information for some of the URLs. + + Returns: + dict mapping file paths to a download info. + """ + pending = { + path: ctx.download( + remote_file_urls, + path, + canonical_id = ctx.attr.canonical_id, + auth = get_auth(ctx, remote_file_urls) if auth == None else auth, + integrity = ctx.attr.remote_file_integrity.get(path, ""), + block = False, + # Overlaid files may be shell scripts. + executable = True, + ) + for path, remote_file_urls in ctx.attr.remote_file_urls.items() + } + + # Wait until the requests are done + return {path: token.wait() for path, token in pending.items()} + +def _http_deb_archive_impl(rctx): + source_urls = _get_source_urls(rctx) + + if rctx.attr.build_file and rctx.attr.build_file_content: + fail("Only one of build_file and build_file_content can be provided.") + + archive = _downloaded_archive_name(rctx, source_urls) + archive_info = rctx.download( + source_urls, + archive, + rctx.attr.sha256, + canonical_id = rctx.attr.canonical_id or get_default_canonical_id(rctx, source_urls), + auth = get_auth(rctx, source_urls), + integrity = rctx.attr.integrity, + ) + + host_bsdtar = _host_bsdtar_label(rctx) + res = rctx.execute([ + rctx.path(host_bsdtar), + "-xf", + archive, + "--include=data.tar.*", + ]) for ext in ["gz", "xz", "zst"]: - data = "tmp/data.tar.{}".format(ext) + data = "data.tar.{}".format(ext) if rctx.path(data).exists: - rctx.extract( - archive = data, - output = "", - stripPrefix = rctx.attr.strip_prefix, - ) - rctx.delete("tmp") + archive = data break + + strip_prefix = rctx.attr.strip_prefix.strip("/") + includes = [_prefixed_pattern(strip_prefix, include) for include in rctx.attr.includes] + excludes = [_prefixed_pattern(strip_prefix, exclude) for exclude in rctx.attr.excludes] + + if strip_prefix and not includes: + includes = [strip_prefix, strip_prefix + "/*"] + + args = [] + for include in includes: + args.extend(["--include", include]) + for exclude in excludes: + args.extend(["--exclude", exclude]) + if strip_prefix: + args.extend(["--strip-components", str(len(strip_prefix.split("/")))]) + args.extend(rctx.attr.bsdtar_extra_args) + + add_prefix = rctx.attr.add_prefix.strip("/") + keep_file = None + if add_prefix: + keep_file = add_prefix + "/.http_deb_archive.keep" + rctx.file(keep_file, "") + args.extend(["-C", add_prefix]) + + args.extend(["-xf", archive]) + res = rctx.execute([rctx.path(host_bsdtar)] + args) + if res.return_code != 0: + fail("Failed to extract archive: {}\n{}".format(res.stderr, res.stdout)) + + if keep_file: + rctx.delete(keep_file) + _ensure_placeholder_dirs(rctx, add_prefix) + rctx.delete(archive) + workspace_and_buildfile(rctx) - patch(rctx) + remote_files_info = download_remote_files(rctx) + remote_patches_info = patch(rctx) + symlink_files(rctx) + + return _update_integrity_attrs(rctx, _http_deb_archive_attrs, archive_info, remote_files_info, remote_patches_info) + +_http_deb_archive_attrs = { + "url": attr.string(), + "urls": attr.string_list(), + "sha256": attr.string(), + "integrity": attr.string(), + "netrc": attr.string(), + "auth_patterns": attr.string_dict(), + "canonical_id": attr.string(), + "strip_prefix": attr.string(), + "add_prefix": attr.string(default = ""), + "files": attr.string_keyed_label_dict(default = {}), + "type": attr.string(), + "patches": attr.label_list(default = []), + "remote_file_urls": attr.string_list_dict(default = {}), + "remote_file_integrity": attr.string_dict(default = {}), + "remote_module_file_urls": attr.string_list(default = []), + "remote_module_file_integrity": attr.string(default = ""), + "remote_patches": attr.string_dict(default = {}), + "remote_patch_strip": attr.int(default = 0), + "patch_tool": attr.string(default = ""), + "patch_args": attr.string_list(default = []), + "patch_strip": attr.int(default = 0), + "patch_cmds": attr.string_list(default = []), + "patch_cmds_win": attr.string_list(default = []), + "build_file": attr.label(allow_single_file = True), + "build_file_content": attr.string(), + "workspace_file": attr.label(), + "workspace_file_content": attr.string(), + "includes": attr.string_list(default = []), + "excludes": attr.string_list(default = []), + "bsdtar_extra_args": attr.string_list(default = []), +} http_deb_archive = repository_rule( - _http_deb_archive_impl, - attrs = { - "urls": attr.string_list(mandatory = True), - "sha256": attr.string(mandatory = True), - "strip_prefix": attr.string(), - "build_file": attr.label(allow_single_file = True), - "build_file_content": attr.string(), - "workspace_file": attr.label(allow_single_file = True), - "workspace_file_content": attr.string(), - }, + implementation = _http_deb_archive_impl, + attrs = _http_deb_archive_attrs, + environ = [DEFAULT_CANONICAL_ID_ENV], ) diff --git a/bazel/huggingface.bzl b/bazel/huggingface.bzl index 11eb0a7a..dbe1545d 100644 --- a/bazel/huggingface.bzl +++ b/bazel/huggingface.bzl @@ -82,8 +82,11 @@ def _huggingface_repository_impl(rctx): continue if (not includes or _glob(rctx, entry["path"], includes)): tpl = RAW_FILE_URL_REMPLATE - if ("lfs" in entry): + is_lfs = "lfs" in entry + sha256 = "" + if (is_lfs): tpl = LFS_FILE_URL_TEMPLATE + sha256 = entry["lfs"]["oid"] url = tpl.format( model = rctx.attr.model, commit = rctx.attr.commit, @@ -94,6 +97,7 @@ def _huggingface_repository_impl(rctx): output = entry["path"], canonical_id = entry["oid"], headers = headers, + sha256 = sha256, block = False, )) diff --git a/bazel/patchelf.bzl b/bazel/patchelf.bzl index 244dbf68..bafaa6ec 100644 --- a/bazel/patchelf.bzl +++ b/bazel/patchelf.bzl @@ -1,56 +1,78 @@ -def _render_kv(e): - return e - def _patchelf_impl(ctx): - output_name = ctx.file.shared_library.basename - if ctx.attr.soname: - output_name = ctx.attr.soname - output = ctx.actions.declare_file("{}/{}".format(ctx.attr.name, output_name)) + output_name = ctx.attr.soname or ctx.file.src.basename + output = ctx.actions.declare_file("{}/{}".format(ctx.label.name, output_name)) - commands = [ - "set -e", - 'cp -f "$2" "$3"', - 'chmod +w "$3"', + inputs = [ctx.file.src] + args = ctx.actions.args() + + # --- soname ----------------------------------------------------------- + if ctx.attr.soname: + args.add("--set-soname", ctx.attr.soname) + + # --- DT_NEEDED -------------------------------------------------------- + args.add_all(ctx.attr.remove_needed, before_each = "--remove-needed") + args.add_all(ctx.attr.add_needed, before_each = "--add-needed") + for old, new in ctx.attr.replace_needed.items(): + args.add("--replace-needed") + args.add(old) + args.add(new) + + # --- RPATH / RUNPATH -------------------------------------------------- + # patchelf only accepts one rpath operation per invocation. + rpath_ops = [ + bool(ctx.attr.set_rpath), + bool(ctx.attr.add_rpath), + ctx.attr.remove_rpath, ] + if len([op for op in rpath_ops if op]) > 1: + fail("patchelf: set_rpath, add_rpath and remove_rpath are mutually exclusive") - if ctx.attr.soname: - commands.append('"$1" --set-soname "{}" "$3"'.format(ctx.attr.soname)) - if ctx.attr.remove_needed: - for v in ctx.attr.remove_needed: - commands.append('"$1" --remove-needed "{}" "$3"'.format(v)) - if ctx.attr.add_needed: - for v in ctx.attr.add_needed: - commands.append('"$1" --add-needed "{}" "$3"'.format(v)) + if ctx.attr.set_rpath: + args.add("--force-rpath") + args.add("--set-rpath", ctx.attr.set_rpath) + elif ctx.attr.add_rpath: + args.add("--force-rpath") + args.add("--add-rpath", ":".join(ctx.attr.add_rpath)) + elif ctx.attr.remove_rpath: + args.add("--remove-rpath") - if ctx.attr.replace_needed: - for k, v in ctx.attr.replace_needed.items(): - commands.append('"$1" --replace-needed "{}" "{}" "$3"'.format(k, v)) + # --- dynamic symbol renames ------------------------------------------ + if ctx.attr.rename_dynamic_symbols: + symbols = ctx.actions.declare_file(ctx.label.name + ".symbols") + ctx.actions.write(symbols, "".join([ + "{} {}\n".format(old, new) + for old, new in ctx.attr.rename_dynamic_symbols.items() + ])) + inputs.append(symbols) + args.add("--rename-dynamic-symbols", symbols) - ctx.actions.run_shell( - inputs = [ctx.file.shared_library], + args.add("--output", output) + args.add(ctx.file.src) + + ctx.actions.run( + executable = ctx.executable._patchelf, + inputs = inputs, outputs = [output], - arguments = [ctx.executable._patchelf.path, ctx.file.shared_library.path, output.path], - command = "\n".join(commands), - tools = [ctx.executable._patchelf], + arguments = [args], + mnemonic = "Patchelf", + progress_message = "Patching ELF %{output}", ) - - return [ - DefaultInfo( - files = depset([output]), - ), - ] + return [DefaultInfo(files = depset([output]))] patchelf = rule( implementation = _patchelf_impl, attrs = { - "shared_library": attr.label(allow_single_file = True, mandatory = True), - "soname": attr.string(), + "src": attr.label(allow_single_file = True, mandatory = True), + "soname": attr.string(doc = "Set DT_SONAME; also used as the output file name."), "add_needed": attr.string_list(), "remove_needed": attr.string_list(), - "replace_needed": attr.string_dict(), + "replace_needed": attr.string_dict(doc = "old -> new DT_NEEDED replacements."), + "rename_dynamic_symbols": attr.string_dict(doc = "old -> new symbol renames."), + "set_rpath": attr.string(), + "add_rpath": attr.string_list(doc = "Entries appended to the existing RPATH (joined with ':')."), + "remove_rpath": attr.bool(default = False, doc = "Strip the DT_RPATH/DT_RUNPATH entry entirely."), "_patchelf": attr.label( default = "@patchelf", - allow_single_file = True, executable = True, cfg = "exec", ), diff --git a/bazel/runfiles.bzl b/bazel/runfiles.bzl new file mode 100644 index 00000000..df5041b5 --- /dev/null +++ b/bazel/runfiles.bzl @@ -0,0 +1,17 @@ +def _runfiles_to_default(ctx): + files = depset([], transitive = [ + dep[DefaultInfo].default_runfiles.files + for dep in ctx.attr.deps + ]) + return [ + DefaultInfo( + files = files, + ), + ] + +runfiles_to_default = rule( + implementation = _runfiles_to_default, + attrs = { + "deps": attr.label_list(providers = [DefaultInfo]), # We expect DefaultInfo from dependencies + }, +) diff --git a/bazel/runfiles_init.c b/bazel/runfiles_init.c new file mode 100644 index 00000000..770a4b91 --- /dev/null +++ b/bazel/runfiles_init.c @@ -0,0 +1,15 @@ +extern void _bazel_runfiles_init_impl(int argc, char **argv, char **envp); + +// Preserve the existing startup ABI. Registration happens in C; +// runfiles discovery and initialization remain in Zig. +// glibc and Darwin's dyld pass argc/argv/envp to constructors; musl does not. +// The original Zig initializer already made this assumption to read argv[0]. +// +// Let C emit the platform's constructor registration. Zig's incremental ELF +// linker does not yet handle .init_array's section type, flags, and size correctly. +// TODO: Revert this workaround once Zig issue #37453 is fixed: +// https://codeberg.org/ziglang/zig/issues/37453 +__attribute__((constructor)) +void _bazel_runfiles_init(int argc, char **argv, char **envp) { + _bazel_runfiles_init_impl(argc, argv, envp); +} diff --git a/bazel/simple_repository.bzl b/bazel/simple_repository.bzl new file mode 100644 index 00000000..9fd07136 --- /dev/null +++ b/bazel/simple_repository.bzl @@ -0,0 +1,23 @@ +_UNSET = "_UNSET" + +def _simple_repository_impl(rctx): + if (rctx.attr.build_file == None) == (rctx.attr.build_file_content == _UNSET): + fail("exactly one of `build_file` and `build_file_content` must be specified") + + if rctx.attr.build_file != None: + # Remove any existing BUILD.bazel in the repository to ensure + # the symlink to the defined build_file doesn't fail. + rctx.delete("BUILD.bazel") + rctx.symlink(rctx.attr.build_file, "BUILD.bazel") + else: + rctx.file("BUILD.bazel", rctx.attr.build_file_content) + +simple_repository = repository_rule( + implementation = _simple_repository_impl, + attrs = { + "build_file": attr.label(allow_single_file = True), + "build_file_content": attr.string(default = _UNSET), + }, + doc = "Makes an empty repository from just one BUILD.bazel file.", + local = True, +) diff --git a/bazel/stamp/BUILD.bazel b/bazel/stamp/BUILD.bazel new file mode 100644 index 00000000..387f189c --- /dev/null +++ b/bazel/stamp/BUILD.bazel @@ -0,0 +1,37 @@ +load("@bazel_lib//lib:run_binary.bzl", "run_binary") +load("@rules_cc//cc:cc_binary.bzl", "cc_binary") +load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library") + +run_binary( + name = "stamp_cc_src", + outs = ["stamp.c"], + args = [ + "$(location stamp.c)", + "bazel-out/stable-status.txt", + "bazel-out/volatile-status.txt", + ], + stamp = 1, + tool = ":genstamp", + visibility = ["//visibility:private"], +) + +cc_binary( + name = "genstamp", + srcs = ["genstamp.c"], + visibility = ["//visibility:private"], +) + +cc_library( + name = "cc", + srcs = [":stamp_cc_src"], + visibility = ["//visibility:public"], +) + +zig_library( + name = "zig", + import_name = "bazel/stamp", + main = "stamp.zig", + visibility = ["//visibility:public"], + deps = [":cc"], +) diff --git a/bazel/stamp/genstamp.c b/bazel/stamp/genstamp.c new file mode 100644 index 00000000..4fba8e6f --- /dev/null +++ b/bazel/stamp/genstamp.c @@ -0,0 +1,121 @@ +#include +#include +#include + +static void write_escaped(FILE *out, const char *value) { + for (const unsigned char *p = (const unsigned char *)value; *p != '\0'; ++p) { + switch (*p) { + case '\\': + fputs("\\\\", out); + break; + case '"': + fputs("\\\"", out); + break; + case '\n': + fputs("\\n", out); + break; + case '\r': + fputs("\\r", out); + break; + case '\t': + fputs("\\t", out); + break; + default: + fputc(*p, out); + break; + } + } +} + +static int process_stream(FILE *out, FILE *in, const char *path) { + char *line = NULL; + size_t cap = 0; + ssize_t len; + + while ((len = getline(&line, &cap, in)) != -1) { + while (len > 0 && (line[len - 1] == '\n' || line[len - 1] == '\r')) { + line[--len] = '\0'; + } + + if (len == 0) { + continue; + } + + char *space = strchr(line, ' '); + if (space == NULL) { + continue; + } + + *space = '\0'; + const char *name = line; + const char *value = space + 1; + + if (*name == '\0') { + continue; + } + + fputs("const char ", out); + fputs(name, out); + fputs("[] = \"", out); + write_escaped(out, value); + fputs("\";\n", out); + } + + if (ferror(in)) { + fprintf(stderr, "genstamp: failed while reading %s\n", path); + free(line); + return 1; + } + + free(line); + return 0; +} + +int main(int argc, char **argv) { + if (argc < 3) { + fprintf(stderr, "usage: %s [ ...]\n", argv[0]); + return 1; + } + + const char *output_path = argv[1]; + FILE *out = fopen(output_path, "w"); + if (out == NULL) { + fprintf(stderr, "genstamp: failed to open %s for writing\n", output_path); + return 1; + } + + for (int i = 2; i < argc; ++i) { + const char *path = argv[i]; + FILE *in = fopen(path, "r"); + if (in == NULL) { + fprintf(stderr, "genstamp: failed to open %s\n", path); + fclose(out); + return 1; + } + + int rc = process_stream(out, in, path); + if (fclose(in) != 0) { + fprintf(stderr, "genstamp: failed to close %s\n", path); + fclose(out); + return 1; + } + + if (rc != 0) { + fclose(out); + return rc; + } + } + + if (fflush(out) != 0) { + fprintf(stderr, "genstamp: failed to flush %s\n", output_path); + fclose(out); + return 1; + } + + if (fclose(out) != 0) { + fprintf(stderr, "genstamp: failed to close %s\n", output_path); + return 1; + } + + return 0; +} \ No newline at end of file diff --git a/bazel/stamp/stamp.zig b/bazel/stamp/stamp.zig new file mode 100644 index 00000000..0e60e2c7 --- /dev/null +++ b/bazel/stamp/stamp.zig @@ -0,0 +1,5 @@ +const std = @import("std"); + +pub fn stamp(comptime name: []const u8) []const u8 { + return std.mem.span(@extern([*c]const u8, .{ .name = name })); +} diff --git a/bazel/swig.bzl b/bazel/swig.bzl new file mode 100644 index 00000000..cd4f994f --- /dev/null +++ b/bazel/swig.bzl @@ -0,0 +1,153 @@ +load("@rules_cc//cc:action_names.bzl", "C_COMPILE_ACTION_NAME") +load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("@rules_cc//cc/common:cc_common.bzl", "cc_common") +load("@rules_cc//cc/common:cc_info.bzl", "CcInfo") +load("@rules_cc//cc:find_cc_toolchain.bzl", "find_cc_toolchain", "use_cc_toolchain") + +def _swig_cc_library_impl(ctx): + args = ctx.actions.args() + + if ctx.attr.cpp: + args.add("-c++") + + args.add("-std=c++17") + args.add("-c") + args.add("-O") + args.add("-module", ctx.attr.module) + args.add_joined("-features", ctx.attr.enabled_features, join_with = ",") + + if ctx.attr.defines: + args.add_all(ctx.attr.defines, format_each = "-D%s") + + cc_toolchain = find_cc_toolchain(ctx) + if (cc_toolchain): + args.add_all(cc_toolchain.built_in_include_directories, format_each = "-I%s") + + feature_configuration = cc_common.configure_features( + ctx = ctx, + cc_toolchain = cc_toolchain, + requested_features = ctx.features, + unsupported_features = ctx.disabled_features, + ) + c_compile_variables = cc_common.create_compile_variables( + feature_configuration = feature_configuration, + cc_toolchain = cc_toolchain, + user_compile_flags = ctx.fragments.cpp.copts + ctx.fragments.cpp.conlyopts, + ) + cc_compile_command_line = cc_common.get_memory_inefficient_command_line( + feature_configuration = feature_configuration, + action_name = C_COMPILE_ACTION_NAME, + variables = c_compile_variables, + ) + for arg in cc_compile_command_line: + if (arg.startswith("-I") or arg.startswith("-D")): + args.add(arg) + + cc_info = cc_common.merge_cc_infos(direct_cc_infos = [dep[CcInfo] for dep in ctx.attr.deps]) + args.add_all(cc_info.compilation_context.defines, format_each = "-D%s") + args.add_all(cc_info.compilation_context.local_defines, format_each = "-D%s") + args.add_all(cc_info.compilation_context.framework_includes, format_each = "-I%s") + args.add_all(cc_info.compilation_context.includes, format_each = "-I%s") + args.add_all(cc_info.compilation_context.quote_includes, format_each = "-I%s") + args.add_all(cc_info.compilation_context.system_includes, format_each = "-I%s") + + output_cpp = ctx.actions.declare_file("%s.cpp" % ctx.attr.module) + output_h = ctx.actions.declare_file("%s.h" % ctx.attr.module) + args.add("-outdir", output_h.dirname) + + outputs = [ + output_cpp, + output_h, + ] + args.add("-o", output_cpp) + args.add("-w-305") + args.add("-w-524") + args.add(ctx.file.interface) + + inputs = depset(ctx.attr.srcs, transitive = [ + ctx.attr.interface.files, + cc_info.compilation_context.headers, + ctx.attr._swig_lib.files, + ]) + + ctx.actions.run( + inputs = inputs, + outputs = outputs, + executable = ctx.executable._swig, + arguments = [args], + env = { + "SWIG_LIB": ctx.files._swig_lib[0].dirname, + }, + mnemonic = "SwigC", + ) + + return [ + DefaultInfo( + files = depset(outputs), + ), + OutputGroupInfo( + hdrs = depset([output_h]), + srcs = depset([output_cpp]), + ), + ] + +_swig_cc_library = rule( + _swig_cc_library_impl, + attrs = { + "interface": attr.label( + mandatory = True, + allow_single_file = True, + ), + "srcs": attr.label_list( + allow_files = True, + ), + "deps": attr.label_list( + providers = [CcInfo], + ), + "defines": attr.string_list(), + "enabled_features": attr.string_list(), + "module": attr.string( + mandatory = True, + ), + "cpp": attr.bool( + default = True, + ), + "intgosize": attr.int( + default = 64, + ), + "_swig": attr.label( + default = "@org_swig_swig//:swig", + cfg = "exec", + executable = True, + ), + "_swig_lib": attr.label( + default = "@org_swig_swig//:lib", + allow_files = True, + ), + }, + toolchains = use_cc_toolchain(), + fragments = ["cpp"], +) + +def swig_cc_library(name, deps = [], **kwargs): + _swig_cc_library( + name = "{}.swig".format(name), + deps = deps, + **kwargs + ) + native.filegroup( + name = "{}.hdrs".format(name), + srcs = [":{}.swig".format(name)], + output_group = "hdrs", + ) + native.filegroup( + name = "{}.srcs".format(name), + srcs = [":{}.swig".format(name)], + output_group = "srcs", + ) + cc_library( + name = name, + hdrs = [":{}.hdrs".format(name)], + srcs = [":{}.srcs".format(name)], + deps = deps, + ) diff --git a/bazel/zig.bzl b/bazel/zig.bzl deleted file mode 100644 index f6de39a9..00000000 --- a/bazel/zig.bzl +++ /dev/null @@ -1,39 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "BINARY_KIND", "zig_binary") - -def zig_cc_binary(name, args = None, env = None, data = [], deps = [], visibility = None, **kwargs): - zig_binary( - name = "{}_lib".format(name), - kind = BINARY_KIND.static_lib, - deps = deps + [ - "@rules_zig//zig/lib:libc", - ], - **kwargs - ) - native.cc_binary( - name = name, - args = args, - env = env, - data = data, - deps = [":{}_lib".format(name)], - visibility = visibility, - ) - -def zig_cc_test(name, env = None, data = [], deps = [], test_runner = None, visibility = None, **kwargs): - zig_binary( - name = "{}_test_lib".format(name), - kind = BINARY_KIND.test_lib, - test_runner = test_runner, - data = data, - deps = deps + [ - "@rules_zig//zig/lib:libc", - ], - **kwargs - ) - native.cc_test( - name = name, - env = env, - data = data, - deps = [":{}_test_lib".format(name)], - visibility = visibility, - linkstatic = True, - ) diff --git a/bazel/zig_index.json b/bazel/zig_index.json index 4c37b610..b8cf9b9e 100644 --- a/bazel/zig_index.json +++ b/bazel/zig_index.json @@ -1,73 +1,936 @@ { + "0.17.0": { + "version": "0.17.0", + "date": "2026-10-01", + "docs": "https://ziglang.org/documentation/0.17.0/", + "stdDocs": "https://ziglang.org/documentation/0.17.0/std/", + "notes": "https://ziglang.org/download/0.17.0/release-notes.html", + "src": { + "tarball": "https://ziglang.org/download/0.17.0/zig-0.17.0.tar.xz", + "shasum": "b6c7f1728f043700d6529bac980800792f824256a9d2f1839b3d62beed0b8abd", + "size": "23049960" + }, + "bootstrap": { + "tarball": "https://ziglang.org/download/0.17.0/zig-bootstrap-0.17.0.tar.xz", + "shasum": "1e9e9b8e3c753b35dfb1d9ea48f097579fadbd3e5c1e724ab3fc5bce05477b71", + "size": "57030388" + }, + "x86_64-macos": { + "tarball": "https://ziglang.org/download/0.17.0/zig-x86_64-macos-0.17.0.tar.xz", + "shasum": "4f9a1c5269aa17ebda5e6d3c2b89d6cbf36f7d2b22a0306e9ab98f25f95529c6", + "size": "59317572" + }, + "aarch64-macos": { + "tarball": "https://ziglang.org/download/0.17.0/zig-aarch64-macos-0.17.0.tar.xz", + "shasum": "b607e9b9234790a008116ae5bdb71c6243b84b9fb42a53a9e70fde41c06c536a", + "size": "53985220" + }, + "x86_64-linux": { + "tarball": "https://ziglang.org/download/0.17.0/zig-x86_64-linux-0.17.0.tar.xz", + "shasum": "1cbe9df9f27e6b78d14ccbca43b6703a404ef79ef1c463de901d7f088d4e2026", + "size": "57332648" + }, + "aarch64-linux": { + "tarball": "https://ziglang.org/download/0.17.0/zig-aarch64-linux-0.17.0.tar.xz", + "shasum": "9e8d11661d4ae3bd57702a3832781e23ad151dde5798e16a5ccd503f65234ff8", + "size": "52877280" + }, + "arm-linux": { + "tarball": "https://ziglang.org/download/0.17.0/zig-arm-linux-0.17.0.tar.xz", + "shasum": "53f0045cdef7ba06da70a12b2ef654e4c5283b0139bc441aa7d84f364b46de43", + "size": "53841896" + }, + "riscv64-linux": { + "tarball": "https://ziglang.org/download/0.17.0/zig-riscv64-linux-0.17.0.tar.xz", + "shasum": "18ff032ac6cecf746a84259f6463264f13160ff88bc3b80722ce06f9479b82ed", + "size": "57116976" + }, + "powerpc64le-linux": { + "tarball": "https://ziglang.org/download/0.17.0/zig-powerpc64le-linux-0.17.0.tar.xz", + "shasum": "731e2887e66fff41fdc34f7fd9d8afa814d86d7426f5827805f8854b2977684b", + "size": "56990076" + }, + "x86-linux": { + "tarball": "https://ziglang.org/download/0.17.0/zig-x86-linux-0.17.0.tar.xz", + "shasum": "55e39e175cd5b3098afc29ec47b2590134f5b874175369fb9ae7b8e836d7531a", + "size": "59823964" + }, + "loongarch64-linux": { + "tarball": "https://ziglang.org/download/0.17.0/zig-loongarch64-linux-0.17.0.tar.xz", + "shasum": "5a6000b866201c3b3820b563fcfd29641185393e7d35039e50235db418d8367c", + "size": "54257324" + }, + "s390x-linux": { + "tarball": "https://ziglang.org/download/0.17.0/zig-s390x-linux-0.17.0.tar.xz", + "shasum": "13905f11989b65566c544d191d4bc5bd5d6f858e303208cf1d2f090e5d91b512", + "size": "57136832" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/download/0.17.0/zig-x86_64-windows-0.17.0.zip", + "shasum": "b5663f69581dcf391293fbf16c06cb80d81d806545ce618b4d0bab7f0eb8c428", + "size": "100268137" + }, + "aarch64-windows": { + "tarball": "https://ziglang.org/download/0.17.0/zig-aarch64-windows-0.17.0.zip", + "shasum": "0a59d91fa1cb40cf068e9b0954434ce973500c7a2ea749f1e01af62cdab52d26", + "size": "95992853" + }, + "x86-windows": { + "tarball": "https://ziglang.org/download/0.17.0/zig-x86-windows-0.17.0.zip", + "shasum": "c7b83507549c7bcd42cd58c4910b47037f4150ed7778e11b3684f130620969d1", + "size": "101891323" + }, + "aarch64-freebsd": { + "tarball": "https://ziglang.org/download/0.17.0/zig-aarch64-freebsd-0.17.0.tar.xz", + "shasum": "3cf53d07b3587a26727b196a6eb7918704b907d951520daa68e4e1820fc6f48e", + "size": "52860244" + }, + "arm-freebsd": { + "tarball": "https://ziglang.org/download/0.17.0/zig-arm-freebsd-0.17.0.tar.xz", + "shasum": "e0c3ecf21c186a48d3b8e8d0b4459f9dc67ff8024445401506a7e970782da5f8", + "size": "54273908" + }, + "powerpc64le-freebsd": { + "tarball": "https://ziglang.org/download/0.17.0/zig-powerpc64le-freebsd-0.17.0.tar.xz", + "shasum": "94e127c2a4beb5151a8c58e8af4b82dc3649ed95f343ffbbfa381e2afba50234", + "size": "57027652" + }, + "riscv64-freebsd": { + "tarball": "https://ziglang.org/download/0.17.0/zig-riscv64-freebsd-0.17.0.tar.xz", + "shasum": "4ab73eb01a0cb910dfa23d8efc88d38dcdd5ebb2174537cf531738ba520354f1", + "size": "57199160" + }, + "x86_64-freebsd": { + "tarball": "https://ziglang.org/download/0.17.0/zig-x86_64-freebsd-0.17.0.tar.xz", + "shasum": "a314f1cc92d098bd39caf9e5607a0de2a8fd0d00064776cb3fd5b3bee38a67c9", + "size": "57468884" + }, + "aarch64-netbsd": { + "tarball": "https://ziglang.org/download/0.17.0/zig-aarch64-netbsd-0.17.0.tar.xz", + "shasum": "421fefe3910dedc8b8a6ae9a8905ea9200ecb0760499bcf0f56af7a5ba4930b5", + "size": "52769844" + }, + "arm-netbsd": { + "tarball": "https://ziglang.org/download/0.17.0/zig-arm-netbsd-0.17.0.tar.xz", + "shasum": "2537d66f6dbd57438355d100180d09f19640c98895d4052e4e2994a02cac320a", + "size": "55345456" + }, + "riscv64-netbsd": { + "tarball": "https://ziglang.org/download/0.17.0/zig-riscv64-netbsd-0.17.0.tar.xz", + "shasum": "687c92430ab3e7a5329e941e95f0ec4368d8d8ac0a4048f61c2dd714e675ca12", + "size": "57157568" + }, + "x86-netbsd": { + "tarball": "https://ziglang.org/download/0.17.0/zig-x86-netbsd-0.17.0.tar.xz", + "shasum": "2025c508390cc78d9c469675d2a0602f2a975c92ec197af51faf09a565b27ee3", + "size": "60380332" + }, + "x86_64-netbsd": { + "tarball": "https://ziglang.org/download/0.17.0/zig-x86_64-netbsd-0.17.0.tar.xz", + "shasum": "10f71cc89efbafc6e22447f23c44047d53c5d2061c82b1c32efd1ef55ea2d9b6", + "size": "57363300" + }, + "aarch64-openbsd": { + "tarball": "https://ziglang.org/download/0.17.0/zig-aarch64-openbsd-0.17.0.tar.xz", + "shasum": "1cf6c33a0cdd9a58a0b40c7ee86d8de9c190ed7bd8d47ad61687908955ecac10", + "size": "53265892" + }, + "arm-openbsd": { + "tarball": "https://ziglang.org/download/0.17.0/zig-arm-openbsd-0.17.0.tar.xz", + "shasum": "068d5200b027541c2b0a2e16efc3fb367f1fae78f9c3954e57512868c9a0ecc2", + "size": "54024240" + }, + "riscv64-openbsd": { + "tarball": "https://ziglang.org/download/0.17.0/zig-riscv64-openbsd-0.17.0.tar.xz", + "shasum": "4858839a9e4d96fb50d6e4894d10270f5377a087b2929d238b1a12b8293e274a", + "size": "57490512" + }, + "x86_64-openbsd": { + "tarball": "https://ziglang.org/download/0.17.0/zig-x86_64-openbsd-0.17.0.tar.xz", + "shasum": "5e9606fc322ef4c346c199e162d26e4c293358a36ea3ca72e877d42ef71e24be", + "size": "58612444" + } + }, + "0.17.0-dev.1936+5a625d5f3": { + "version": "0.17.0-dev.1936+5a625d5f3", + "docs": "https://ziglang.org/documentation/master/", + "stdDocs": "https://ziglang.org/documentation/master/std/", + "x86_64-macos": { + "tarball": "https://ziglang.org/builds/zig-x86_64-macos-0.17.0-dev.1936+5a625d5f3.tar.xz", + "shasum": "8ab9f0d015abb75c0c9c039f7191711a7a2fc72c984458f553926725794791dd", + "size": "59162280" + }, + "aarch64-macos": { + "tarball": "https://ziglang.org/builds/zig-aarch64-macos-0.17.0-dev.1936+5a625d5f3.tar.xz", + "shasum": "e08d4485c96d2d31a8ce3ae80a58badec202920b675149ad6b0c3b974ec7c005", + "size": "53867184" + }, + "x86_64-linux": { + "tarball": "https://ziglang.org/builds/zig-x86_64-linux-0.17.0-dev.1936+5a625d5f3.tar.xz", + "shasum": "38620c06ca57b175f7dab8f65c8001f9a67b0ff1987c344d4e64ccb72b57174f", + "size": "57180268" + }, + "aarch64-linux": { + "tarball": "https://ziglang.org/builds/zig-aarch64-linux-0.17.0-dev.1936+5a625d5f3.tar.xz", + "shasum": "6c559ff4bca3e84fbbf7b6ad8521ce70bb3e460371b3d8803112353fb8ce70cc", + "size": "52741876" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/builds/zig-x86_64-windows-0.17.0-dev.1936+5a625d5f3.zip", + "shasum": "e0d62a00e067de41a0a34af2af73694e00692e045ca244748965a7b3729633a1", + "size": "100793891" + }, + "aarch64-windows": { + "tarball": "https://ziglang.org/builds/zig-aarch64-windows-0.17.0-dev.1936+5a625d5f3.zip", + "shasum": "e75f340afee0772fe4daed61d92b999534c8460601175b66fe890703cf752850", + "size": "96526396" + } + }, + "0.17.0-dev.1509+bb296ab9b": { + "version": "0.17.0-dev.1509+bb296ab9b", + "date": "2026-07-29", + "docs": "https://ziglang.org/documentation/master/", + "stdDocs": "https://ziglang.org/documentation/master/std/", + "x86_64-macos": { + "tarball": "https://ziglang.org/builds/zig-x86_64-macos-0.17.0-dev.1509+bb296ab9b.tar.xz", + "shasum": "087aad38fead6c0ba7a52b5db3c405e63708c4765eda643d4144b9e4933d45e5", + "size": "59064872" + }, + "aarch64-macos": { + "tarball": "https://ziglang.org/builds/zig-aarch64-macos-0.17.0-dev.1509+bb296ab9b.tar.xz", + "shasum": "24bd83c1d435b8ab6192f58afcc22f4e5252077a938ffbc35eaa4ca97c5be709", + "size": "53773824" + }, + "x86_64-linux": { + "tarball": "https://ziglang.org/builds/zig-x86_64-linux-0.17.0-dev.1509+bb296ab9b.tar.xz", + "shasum": "48cc865b8b410ec84eaa97e50c2bd7a657871802ce3aaaf04dd1da2294d4b28a", + "size": "57056988" + }, + "aarch64-linux": { + "tarball": "https://ziglang.org/builds/zig-aarch64-linux-0.17.0-dev.1509+bb296ab9b.tar.xz", + "shasum": "7f17e09b675cfe39805c5b551b251151f7750b9df8a168538007890c1c00dc8e", + "size": "52649416" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/builds/zig-x86_64-windows-0.17.0-dev.1509+bb296ab9b.zip", + "shasum": "a4f537761ca1f5274c9bb0e7da598c6e82205a4ea78f329690a26eade555d7e8", + "size": "100743514" + }, + "aarch64-windows": { + "tarball": "https://ziglang.org/builds/zig-aarch64-windows-0.17.0-dev.1509+bb296ab9b.zip", + "shasum": "1fa7a2b5f5442628287fead0f7451b8cb93ed2ca5dd6c79f8026fc8ca83580d2", + "size": "96467368" + } + }, "master": { - "version": "0.14.0-dev.363+c3faae6bf", - "date": "2024-07-18", + "version": "0.16.0-dev.3153+d6f43caad", + "date": "2026-04-12", "docs": "https://ziglang.org/documentation/master/", "stdDocs": "https://ziglang.org/documentation/master/std/", "src": { - "tarball": "https://ziglang.org/builds/zig-0.14.0-dev.363+c3faae6bf.tar.xz", - "shasum": "55b48780575ee86668fe9e8119abcc0831d3ce93f5f848b7a9b610155c1a865e", - "size": "17272356" + "tarball": "https://ziglang.org/builds/zig-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "cb598f754d00a71af7586d437f490056b084eb6e54587fc2bb9f55148757fd49", + "size": "22477252" }, "bootstrap": { - "tarball": "https://ziglang.org/builds/zig-bootstrap-0.14.0-dev.363+c3faae6bf.tar.xz", - "shasum": "65cd278494293ff953561194607c4b7e9b96908aa903c7a8421f4e8014a6345b", - "size": "46507328" + "tarball": "https://ziglang.org/builds/zig-bootstrap-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "a2ccc95aaed569b3c25d8693f01187144f09e08a2a9b828bad3e5f2c19a4c2bc", + "size": "55240588" }, "x86_64-macos": { - "tarball": "https://ziglang.org/builds/zig-macos-x86_64-0.14.0-dev.363+c3faae6bf.tar.xz", - "shasum": "256b09afa6a4e0cd5c4f8497ef3625ba9b01de2b75f57b728337bc1de4681c9c", - "size": "48937384" + "tarball": "https://ziglang.org/builds/zig-x86_64-macos-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "0251a8414f98513a33a7a7bd1f6cc13fb59f391168a742eee47b8c685e9827a0", + "size": "57381556" }, "aarch64-macos": { - "tarball": "https://ziglang.org/builds/zig-macos-aarch64-0.14.0-dev.363+c3faae6bf.tar.xz", - "shasum": "cd9f563150b1adb7306912b5acff9b00e39ef283075a42b95186f39bda656862", - "size": "44960052" + "tarball": "https://ziglang.org/builds/zig-aarch64-macos-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "9121e3f03515a3434ef9f362f8c541540968316d33ed6485b2d8a31dc7332fe6", + "size": "52217412" }, "x86_64-linux": { - "tarball": "https://ziglang.org/builds/zig-linux-x86_64-0.14.0-dev.363+c3faae6bf.tar.xz", - "shasum": "98ce531beaac0e683713ec1843023b8aa81a318686472ff13f2c075f0362bf0a", - "size": "47164832" + "tarball": "https://ziglang.org/builds/zig-x86_64-linux-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "afb3016a94a8b5608d364e58ec7b01111af775a020d5f93f69eaebde2053fe77", + "size": "55445192" }, "aarch64-linux": { - "tarball": "https://ziglang.org/builds/zig-linux-aarch64-0.14.0-dev.363+c3faae6bf.tar.xz", - "shasum": "81e1c06740c017ad8aa3df451c544da17a2e23440c1e695954b8c4b612243af0", - "size": "43190732" + "tarball": "https://ziglang.org/builds/zig-aarch64-linux-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "e46037f7747492b405a42d40bb9bf3ca3f28c71cf9c1ccdf165a4de74cb484bb", + "size": "51193908" }, - "armv7a-linux": { - "tarball": "https://ziglang.org/builds/zig-linux-armv7a-0.14.0-dev.363+c3faae6bf.tar.xz", - "shasum": "cda6c3f2b51355c3f117814c13962c9fd65d9ba02f5981d45ea83c135b9019b6", - "size": "44096460" + "arm-linux": { + "tarball": "https://ziglang.org/builds/zig-arm-linux-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "aec322b170a88c998c1c7b59c731e808543310008960b1654caab569567e7c95", + "size": "52093456" }, "riscv64-linux": { - "tarball": "https://ziglang.org/builds/zig-linux-riscv64-0.14.0-dev.363+c3faae6bf.tar.xz", - "shasum": "aa488f1763ff65a910c8f6bdedc5ff8b9a05225d8b29a1446e3c0e0cb7bff683", - "size": "45637416" + "tarball": "https://ziglang.org/builds/zig-riscv64-linux-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "ad83132048c075edd1031a1e2fc9b29921e342f6b1d58142923e60e203a5b53a", + "size": "55303468" }, "powerpc64le-linux": { - "tarball": "https://ziglang.org/builds/zig-linux-powerpc64le-0.14.0-dev.363+c3faae6bf.tar.xz", - "shasum": "8e9f715b53edf8d8b8c0d9dcc173a1faafdb3b659cebe4435d19d2b2ef24eda9", - "size": "46654844" + "tarball": "https://ziglang.org/builds/zig-powerpc64le-linux-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "1866a073fdcd0773397d345b21f98221a20126f7e1cec298c16ff0f4ca3fb78f", + "size": "55241328" }, "x86-linux": { - "tarball": "https://ziglang.org/builds/zig-linux-x86-0.14.0-dev.363+c3faae6bf.tar.xz", - "shasum": "4633ce74826903cf019f390956726c434a5b032f06931aa085e0b22194b55bb5", - "size": "52144844" + "tarball": "https://ziglang.org/builds/zig-x86-linux-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "6572c8768de14c0a0bbaf1a0401e19849681840594744a875ade93e4dc040edd", + "size": "58171580" + }, + "loongarch64-linux": { + "tarball": "https://ziglang.org/builds/zig-loongarch64-linux-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "9bdf0f91187ede6b418b898d178521c4ce2065171b8bcfaa68cae0adf1d0a8f0", + "size": "52511048" + }, + "s390x-linux": { + "tarball": "https://ziglang.org/builds/zig-s390x-linux-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "c893299d0e486337390262ef480615d4f573da158f73a647eb104e10a2811f0d", + "size": "54926720" }, "x86_64-windows": { - "tarball": "https://ziglang.org/builds/zig-windows-x86_64-0.14.0-dev.363+c3faae6bf.zip", - "shasum": "44a26238f1757723f54e9b5d4d08b508be8ebdfad9442218d0f2a3ae61f032a5", - "size": "79857623" + "tarball": "https://ziglang.org/builds/zig-x86_64-windows-0.16.0-dev.3153+d6f43caad.zip", + "shasum": "ab31039adca06ae87c95c11b3ff8ed82808335866d58bc6e508cd3174d2d7e3d", + "size": "97965399" }, "aarch64-windows": { - "tarball": "https://ziglang.org/builds/zig-windows-aarch64-0.14.0-dev.363+c3faae6bf.zip", - "shasum": "8214699c48b5753d127f9632fbb78432fdfa1dd5715ff73106469b339bb641ef", - "size": "75829556" + "tarball": "https://ziglang.org/builds/zig-aarch64-windows-0.16.0-dev.3153+d6f43caad.zip", + "shasum": "b25c7046c111f5a4e614fae1a1b33c8db73fbc3de4c135acabb5740455a3b6fc", + "size": "93857928" }, "x86-windows": { - "tarball": "https://ziglang.org/builds/zig-windows-x86-0.14.0-dev.363+c3faae6bf.zip", - "shasum": "86be63cb2017f23371da631980b27ab6aa4a43b5d4ed96fd08f1311584d779a9", - "size": "83972311" + "tarball": "https://ziglang.org/builds/zig-x86-windows-0.16.0-dev.3153+d6f43caad.zip", + "shasum": "51d579299617bd5fae4d0333d6f2b06a8078c1a6f2f6db2dd08289236fad70d4", + "size": "99767726" + }, + "aarch64-freebsd": { + "tarball": "https://ziglang.org/builds/zig-aarch64-freebsd-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "fe2a97ed29641cc214d58fa2c054fb3ad3a7120b7b43241e6b833c0978187bb6", + "size": "51125424" + }, + "arm-freebsd": { + "tarball": "https://ziglang.org/builds/zig-arm-freebsd-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "0aa627f73f0bc9c100394d512a375188afb8d33168409050d07b0c8b87adbc0b", + "size": "52702132" + }, + "powerpc64le-freebsd": { + "tarball": "https://ziglang.org/builds/zig-powerpc64le-freebsd-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "18500e3b2e742c07541b2d35377e3dc770758ea41e9a3bf09535eacad5266583", + "size": "55254720" + }, + "riscv64-freebsd": { + "tarball": "https://ziglang.org/builds/zig-riscv64-freebsd-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "593c19b8feb76a2e66c6d49f3a96390eed22fc4e2c780a96429d202bf4912afc", + "size": "55393696" + }, + "x86_64-freebsd": { + "tarball": "https://ziglang.org/builds/zig-x86_64-freebsd-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "b043fa6731e205f208d2e50fa84d1d79c7060e04688a4f5e3bff6dace337d5f8", + "size": "55587512" + }, + "aarch64-netbsd": { + "tarball": "https://ziglang.org/builds/zig-aarch64-netbsd-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "c1625244d6fe4008feb27d8bc7bf49739da98d04bc961295124a4e8732cf3b18", + "size": "51092376" + }, + "arm-netbsd": { + "tarball": "https://ziglang.org/builds/zig-arm-netbsd-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "582116aeee90ea3d38ede6901f93138e543a07c6dc32d2645bf0b4e6fee813c6", + "size": "53732592" + }, + "x86-netbsd": { + "tarball": "https://ziglang.org/builds/zig-x86-netbsd-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "eba2eb8e8efe0748820fcedb4066233a96406f97c3fa7b326e220f7c3a9e07f3", + "size": "58765376" + }, + "x86_64-netbsd": { + "tarball": "https://ziglang.org/builds/zig-x86_64-netbsd-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "281d172d159326167a75f2bc87fbe18afe4e37a26b5a1440cd65c8eef952626a", + "size": "55515836" + }, + "aarch64-openbsd": { + "tarball": "https://ziglang.org/builds/zig-aarch64-openbsd-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "1d750bb3ed57ff4a83b1411a07df7e38f86a0a611123fcbede1ed84a48937585", + "size": "51551144" + }, + "arm-openbsd": { + "tarball": "https://ziglang.org/builds/zig-arm-openbsd-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "2bd936eff1c2fadd5aa842e30f35f8509ee07f980bcb42a407940b1ae8fe5f29", + "size": "52270724" + }, + "riscv64-openbsd": { + "tarball": "https://ziglang.org/builds/zig-riscv64-openbsd-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "c2067369e07e991f2e1cfe8f657c9185d726487a4ab4cbc1fde9768c8fae363d", + "size": "55713660" + }, + "x86_64-openbsd": { + "tarball": "https://ziglang.org/builds/zig-x86_64-openbsd-0.16.0-dev.3153+d6f43caad.tar.xz", + "shasum": "01122836baaa88627de2e82addf7cdcdcfb3ba104ae9d051d92e36aaad4f4e27", + "size": "56927908" + } + }, + "0.16.0": { + "version": "0.16.0", + "date": "2026-04-13", + "docs": "https://ziglang.org/documentation/0.16.0/", + "stdDocs": "https://ziglang.org/documentation/0.16.0/std/", + "notes": "https://ziglang.org/download/0.16.0/release-notes.html", + "src": { + "tarball": "https://ziglang.org/download/0.16.0/zig-0.16.0.tar.xz", + "shasum": "43186959edc87d5c7a1be7b7d2a25efffd22ce5807c7af99067f86f99641bfdf", + "size": "22503260" + }, + "bootstrap": { + "tarball": "https://ziglang.org/download/0.16.0/zig-bootstrap-0.16.0.tar.xz", + "shasum": "2a8266a4205772ef40838c8cbdf14875855a515ff3adf89b49c2d2ae93613d10", + "size": "55245980" + }, + "x86_64-macos": { + "tarball": "https://ziglang.org/download/0.16.0/zig-x86_64-macos-0.16.0.tar.xz", + "shasum": "0387557ed1877bc6a2e1802c8391953baddba76081876301c522f52977b52ba7", + "size": "57396836" + }, + "aarch64-macos": { + "tarball": "https://ziglang.org/download/0.16.0/zig-aarch64-macos-0.16.0.tar.xz", + "shasum": "b23d70deaa879b5c2d486ed3316f7eaa53e84acf6fc9cc747de152450d401489", + "size": "52238004" + }, + "x86_64-linux": { + "tarball": "https://ziglang.org/download/0.16.0/zig-x86_64-linux-0.16.0.tar.xz", + "shasum": "70e49664a74374b48b51e6f3fdfbf437f6395d42509050588bd49abe52ba3d00", + "size": "55478392" + }, + "aarch64-linux": { + "tarball": "https://ziglang.org/download/0.16.0/zig-aarch64-linux-0.16.0.tar.xz", + "shasum": "ea4b09bfb22ec6f6c6ceac57ab63efb6b46e17ab08d21f69f3a48b38e1534f17", + "size": "51211944" + }, + "arm-linux": { + "tarball": "https://ziglang.org/download/0.16.0/zig-arm-linux-0.16.0.tar.xz", + "shasum": "f85116bf2f9189bb6ae280c7f92f03b89c2551a88e17881c0c2df86bf4e42c50", + "size": "52122000" + }, + "riscv64-linux": { + "tarball": "https://ziglang.org/download/0.16.0/zig-riscv64-linux-0.16.0.tar.xz", + "shasum": "bc069b0f2f568f54bafbdfc1d65b12fd386ed6a652044a37aee6a4f72f14076e", + "size": "55313616" + }, + "powerpc64le-linux": { + "tarball": "https://ziglang.org/download/0.16.0/zig-powerpc64le-linux-0.16.0.tar.xz", + "shasum": "18800b45c08bf40b335ca5ab79aea70aca287ca969036e938155772becaeebeb", + "size": "55242728" + }, + "x86-linux": { + "tarball": "https://ziglang.org/download/0.16.0/zig-x86-linux-0.16.0.tar.xz", + "shasum": "4e34e279a9f856358de420490b531974c3d37f8f3707eef9f0342e92c14c301f", + "size": "58196012" + }, + "loongarch64-linux": { + "tarball": "https://ziglang.org/download/0.16.0/zig-loongarch64-linux-0.16.0.tar.xz", + "shasum": "2503be8ecc5965f1f7962471d267d9f83fcb3cc2f7ff78ac34093b9722bbea93", + "size": "52515080" + }, + "s390x-linux": { + "tarball": "https://ziglang.org/download/0.16.0/zig-s390x-linux-0.16.0.tar.xz", + "shasum": "14db4884b31414bd637879036c1e6e335f3ba44ccac59df104163728a0cab30e", + "size": "54995144" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/download/0.16.0/zig-x86_64-windows-0.16.0.zip", + "shasum": "68659eb5f1e4eb1437a722f1dd889c5a322c9954607f5edcf337bc3684a75a7e", + "size": "97217739" + }, + "aarch64-windows": { + "tarball": "https://ziglang.org/download/0.16.0/zig-aarch64-windows-0.16.0.zip", + "shasum": "aee38316ee4111717900f45dd3130145c39289e105541d737eb8c5ed653c78ef", + "size": "93109828" + }, + "x86-windows": { + "tarball": "https://ziglang.org/download/0.16.0/zig-x86-windows-0.16.0.zip", + "shasum": "8aee7e8a8deb998ba96cb95d89aa5fcdf32933fdc67de51d280d9e4d7396f1a0", + "size": "99015159" + }, + "aarch64-freebsd": { + "tarball": "https://ziglang.org/download/0.16.0/zig-aarch64-freebsd-0.16.0.tar.xz", + "shasum": "0a441f50696a34cfb9f9a0b3c510fd49a56b18b364e03c26525cb7999959c969", + "size": "51137568" + }, + "arm-freebsd": { + "tarball": "https://ziglang.org/download/0.16.0/zig-arm-freebsd-0.16.0.tar.xz", + "shasum": "78b2a7bfa74f504075dd4b5c00418347a21b893f0eae580cf4e56c553cf181f6", + "size": "52741336" + }, + "powerpc64le-freebsd": { + "tarball": "https://ziglang.org/download/0.16.0/zig-powerpc64le-freebsd-0.16.0.tar.xz", + "shasum": "46b3fa0239be42669d86ce8394f8de2d608a4f51e3e65cb8222bfd543b95e310", + "size": "55232224" + }, + "riscv64-freebsd": { + "tarball": "https://ziglang.org/download/0.16.0/zig-riscv64-freebsd-0.16.0.tar.xz", + "shasum": "aefbaea44f32bde07118f5522b7bb21de9967a6d09183e3e37b1f5be2c78f948", + "size": "55416052" + }, + "x86_64-freebsd": { + "tarball": "https://ziglang.org/download/0.16.0/zig-x86_64-freebsd-0.16.0.tar.xz", + "shasum": "390379bfaf6b89b001c6bb4035b8fcbc5c12193fd9fd5048c3a0fe39e5d1cf72", + "size": "55591888" + }, + "aarch64-netbsd": { + "tarball": "https://ziglang.org/download/0.16.0/zig-aarch64-netbsd-0.16.0.tar.xz", + "shasum": "4767425d72c779275ebbc5a6319f5a3c5d109664d3d8c880125d550e87d9d46a", + "size": "51104512" + }, + "arm-netbsd": { + "tarball": "https://ziglang.org/download/0.16.0/zig-arm-netbsd-0.16.0.tar.xz", + "shasum": "07aaa403c18e9fa33687a3eb3109bb484a96aece95a4f6792a911d2eefac07c5", + "size": "53772868" + }, + "x86-netbsd": { + "tarball": "https://ziglang.org/download/0.16.0/zig-x86-netbsd-0.16.0.tar.xz", + "shasum": "aa36c02caad52d92e6354baec16dd866642e64b7c30e4d515806c8296d65586f", + "size": "58792260" + }, + "x86_64-netbsd": { + "tarball": "https://ziglang.org/download/0.16.0/zig-x86_64-netbsd-0.16.0.tar.xz", + "shasum": "e235ca96f63034a009aca831c69394a1c14498a8e06ff5f358fc3fb4bed73dfb", + "size": "55533332" + }, + "aarch64-openbsd": { + "tarball": "https://ziglang.org/download/0.16.0/zig-aarch64-openbsd-0.16.0.tar.xz", + "shasum": "f3c868ccc1bc30e2d2b68128186b2153d86c3efc41190f4170a43024a3cd7809", + "size": "51564888" + }, + "arm-openbsd": { + "tarball": "https://ziglang.org/download/0.16.0/zig-arm-openbsd-0.16.0.tar.xz", + "shasum": "33786d6487aaeca9f300e5a932d5ee5365d9254be0add44b3af8f6db8441ca2f", + "size": "52301840" + }, + "riscv64-openbsd": { + "tarball": "https://ziglang.org/download/0.16.0/zig-riscv64-openbsd-0.16.0.tar.xz", + "shasum": "ffb74aefa8184a4b886f8c0a9105db00ac9562d502330f4e764a7a49171e4f8e", + "size": "55734756" + }, + "x86_64-openbsd": { + "tarball": "https://ziglang.org/download/0.16.0/zig-x86_64-openbsd-0.16.0.tar.xz", + "shasum": "25b72946e8b2dde5c93bc03cbebeb17e4d90eea8040acce8bb78cefdeee97337", + "size": "56962688" + } + }, + "0.15.2": { + "version": "0.15.2", + "date": "2025-10-11", + "docs": "https://ziglang.org/documentation/0.15.2/", + "stdDocs": "https://ziglang.org/documentation/0.15.2/std/", + "src": { + "tarball": "https://ziglang.org/download/0.15.2/zig-0.15.2.tar.xz", + "shasum": "d9b30c7aa983fcff5eed2084d54ae83eaafe7ff3a84d8fb754d854165a6e521c", + "size": "21366268" + }, + "bootstrap": { + "tarball": "https://ziglang.org/download/0.15.2/zig-bootstrap-0.15.2.tar.xz", + "shasum": "a6845459501df3c3264ebc587b02a7094ad14f4f3f7287c48f04457e784d0d85", + "size": "52718988" + }, + "x86_64-macos": { + "tarball": "https://ziglang.org/download/0.15.2/zig-x86_64-macos-0.15.2.tar.xz", + "shasum": "375b6909fc1495d16fc2c7db9538f707456bfc3373b14ee83fdd3e22b3d43f7f", + "size": "55800460" + }, + "aarch64-macos": { + "tarball": "https://ziglang.org/download/0.15.2/zig-aarch64-macos-0.15.2.tar.xz", + "shasum": "3cc2bab367e185cdfb27501c4b30b1b0653c28d9f73df8dc91488e66ece5fa6b", + "size": "50635984" + }, + "x86_64-linux": { + "tarball": "https://ziglang.org/download/0.15.2/zig-x86_64-linux-0.15.2.tar.xz", + "shasum": "02aa270f183da276e5b5920b1dac44a63f1a49e55050ebde3aecc9eb82f93239", + "size": "53733924" + }, + "aarch64-linux": { + "tarball": "https://ziglang.org/download/0.15.2/zig-aarch64-linux-0.15.2.tar.xz", + "shasum": "958ed7d1e00d0ea76590d27666efbf7a932281b3d7ba0c6b01b0ff26498f667f", + "size": "49471996" + }, + "arm-linux": { + "tarball": "https://ziglang.org/download/0.15.2/zig-arm-linux-0.15.2.tar.xz", + "shasum": "7d8401495065dae45d6249c68d5faf10508f8203c86362ccb698aeaafc66b7cd", + "size": "50454216" + }, + "riscv64-linux": { + "tarball": "https://ziglang.org/download/0.15.2/zig-riscv64-linux-0.15.2.tar.xz", + "shasum": "493512bdca485be3c6a9b0f69dcb4cbe4587f3af8e1be282fdd827108ba39930", + "size": "53613656" + }, + "powerpc64le-linux": { + "tarball": "https://ziglang.org/download/0.15.2/zig-powerpc64le-linux-0.15.2.tar.xz", + "shasum": "e182c5f8d30fc7f97d17d2ffef1488826aa3afaa51e5f0dbe14c597a98b45778", + "size": "53570632" + }, + "x86-linux": { + "tarball": "https://ziglang.org/download/0.15.2/zig-x86-linux-0.15.2.tar.xz", + "shasum": "4c6e23f39daa305e274197bfdff0d56ffd1750fc1de226ae10505c0eff52d7a5", + "size": "56334032" + }, + "loongarch64-linux": { + "tarball": "https://ziglang.org/download/0.15.2/zig-loongarch64-linux-0.15.2.tar.xz", + "shasum": "95d6a03e0415dbe30ae9c00d58b5446fd681ebf3d4044a8b707e1784c8074a61", + "size": "50800332" + }, + "s390x-linux": { + "tarball": "https://ziglang.org/download/0.15.2/zig-s390x-linux-0.15.2.tar.xz", + "shasum": "cd4b295f43b1b7ca3d603368caea82dbd00fce6691affb7527e607c0b6eafe8e", + "size": "53486436" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/download/0.15.2/zig-x86_64-windows-0.15.2.zip", + "shasum": "3a0ed1e8799a2f8ce2a6e6290a9ff22e6906f8227865911fb7ddedc3cc14cb0c", + "size": "92614574" + }, + "aarch64-windows": { + "tarball": "https://ziglang.org/download/0.15.2/zig-aarch64-windows-0.15.2.zip", + "shasum": "b926465f8872bf983422257cd9ec248bb2b270996fbe8d57872cca13b56fc370", + "size": "88464585" + }, + "x86-windows": { + "tarball": "https://ziglang.org/download/0.15.2/zig-x86-windows-0.15.2.zip", + "shasum": "7a6dfc00f4cc09ec46d3e10eb06f42538e92b6285e34debea7462edaf371da98", + "size": "94516930" + }, + "aarch64-freebsd": { + "tarball": "https://ziglang.org/download/0.15.2/zig-aarch64-freebsd-0.15.2.tar.xz", + "shasum": "c62efd319f86663eb7747709dfca259205edba8eaee98efc96a51ce40a9437de", + "size": "49363332" + }, + "arm-freebsd": { + "tarball": "https://ziglang.org/download/0.15.2/zig-arm-freebsd-0.15.2.tar.xz", + "shasum": "1d5d06428b2d4d31cf4e99008602f2096a9c652dd1025820a4aa66326fbf3dc4", + "size": "50900824" + }, + "powerpc64-freebsd": { + "tarball": "https://ziglang.org/download/0.15.2/zig-powerpc64-freebsd-0.15.2.tar.xz", + "shasum": "2bdb5a38069e667936d3378dfa332d67c21bb454cb17bea3c3b03faaa02a74af", + "size": "52099240" + }, + "powerpc64le-freebsd": { + "tarball": "https://ziglang.org/download/0.15.2/zig-powerpc64le-freebsd-0.15.2.tar.xz", + "shasum": "940894eb954c2077d0643f4deba8113239436fd0dcbc88160163b1b23fe3f035", + "size": "53459116" + }, + "riscv64-freebsd": { + "tarball": "https://ziglang.org/download/0.15.2/zig-riscv64-freebsd-0.15.2.tar.xz", + "shasum": "9aecdcf32fb63944c11389c10993aecd5e443a82d456693cc5ad230b5fe584ad", + "size": "53658752" + }, + "x86_64-freebsd": { + "tarball": "https://ziglang.org/download/0.15.2/zig-x86_64-freebsd-0.15.2.tar.xz", + "shasum": "5509ff57cd3f219165caed0da10221739af82742b9edfcda3f7bfaf4da7212dd", + "size": "53776208" + }, + "aarch64-netbsd": { + "tarball": "https://ziglang.org/download/0.15.2/zig-aarch64-netbsd-0.15.2.tar.xz", + "shasum": "357c1e62df6581ec1726546f9a827973233cb31ac371d2b0e92456b306a0c6c0", + "size": "49365256" + }, + "arm-netbsd": { + "tarball": "https://ziglang.org/download/0.15.2/zig-arm-netbsd-0.15.2.tar.xz", + "shasum": "1c43871791d49dbabd224da307b995076e493e34c36734cb21c4f6cbc208f8c3", + "size": "52021696" + }, + "x86-netbsd": { + "tarball": "https://ziglang.org/download/0.15.2/zig-x86-netbsd-0.15.2.tar.xz", + "shasum": "7f18fc4720426a6ca77bbfb7b970c31a61c37ce4a867b53830762958e921cdde", + "size": "56893368" + }, + "x86_64-netbsd": { + "tarball": "https://ziglang.org/download/0.15.2/zig-x86_64-netbsd-0.15.2.tar.xz", + "shasum": "11a3b9f4b66bece34a79c0e98e91136baf0ff9795ddf60e442ca6c8ad73b9204", + "size": "53778132" + } + }, + "0.15.1": { + "date": "2025-08-19", + "docs": "https://ziglang.org/documentation/0.15.1/", + "stdDocs": "https://ziglang.org/documentation/0.15.1/std/", + "notes": "https://ziglang.org/download/0.15.1/release-notes.html", + "src": { + "tarball": "https://ziglang.org/download/0.15.1/zig-0.15.1.tar.xz", + "shasum": "816c0303ab313f59766ce2097658c9fff7fafd1504f61f80f9507cd11652865f", + "size": "21359884" + }, + "bootstrap": { + "tarball": "https://ziglang.org/download/0.15.1/zig-bootstrap-0.15.1.tar.xz", + "shasum": "4c0cfbcf12da144955761ca43f89e3c74956bce978694fc1d0a63555f5c0a199", + "size": "52711548" + }, + "x86_64-macos": { + "tarball": "https://ziglang.org/download/0.15.1/zig-x86_64-macos-0.15.1.tar.xz", + "shasum": "9919392e0287cccc106dfbcbb46c7c1c3fa05d919567bb58d7eb16bca4116184", + "size": "55791880" + }, + "aarch64-macos": { + "tarball": "https://ziglang.org/download/0.15.1/zig-aarch64-macos-0.15.1.tar.xz", + "shasum": "c4bd624d901c1268f2deb9d8eb2d86a2f8b97bafa3f118025344242da2c54d7b", + "size": "50644996" + }, + "x86_64-linux": { + "tarball": "https://ziglang.org/download/0.15.1/zig-x86_64-linux-0.15.1.tar.xz", + "shasum": "c61c5da6edeea14ca51ecd5e4520c6f4189ef5250383db33d01848293bfafe05", + "size": "53734456" + }, + "aarch64-linux": { + "tarball": "https://ziglang.org/download/0.15.1/zig-aarch64-linux-0.15.1.tar.xz", + "shasum": "bb4a8d2ad735e7fba764c497ddf4243cb129fece4148da3222a7046d3f1f19fe", + "size": "49493872" + }, + "arm-linux": { + "tarball": "https://ziglang.org/download/0.15.1/zig-arm-linux-0.15.1.tar.xz", + "shasum": "3f4bf3b06b67d14e3f38be30798488c1abe3cf5b33de570cd0e87bbf09b978ad", + "size": "50477464" + }, + "riscv64-linux": { + "tarball": "https://ziglang.org/download/0.15.1/zig-riscv64-linux-0.15.1.tar.xz", + "shasum": "7ca7a3e621436fb31d66a253132fc39574a13d2a1b4d8458af4f2e7c6e4374fe", + "size": "53597792" + }, + "powerpc64le-linux": { + "tarball": "https://ziglang.org/download/0.15.1/zig-powerpc64le-linux-0.15.1.tar.xz", + "shasum": "339e2106496be70b614e32d444298216e676c36d08f5cd7bee3dd1dbd4567fd7", + "size": "53566944" + }, + "x86-linux": { + "tarball": "https://ziglang.org/download/0.15.1/zig-x86-linux-0.15.1.tar.xz", + "shasum": "dff166f25fdd06e8341d831a71211b5ba7411463a6b264bdefa8868438690b6a", + "size": "56311228" + }, + "loongarch64-linux": { + "tarball": "https://ziglang.org/download/0.15.1/zig-loongarch64-linux-0.15.1.tar.xz", + "shasum": "0af18a012f3c4cffbef29ab5f42021484d92517f921a1380faacc89c50c1f89d", + "size": "50794376" + }, + "s390x-linux": { + "tarball": "https://ziglang.org/download/0.15.1/zig-s390x-linux-0.15.1.tar.xz", + "shasum": "bcd13e5c88cf2d0da7100a48572195560069a94402c9bec3b316398204aa27e2", + "size": "53508068" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/download/0.15.1/zig-x86_64-windows-0.15.1.zip", + "shasum": "91e69e887ca8c943ce9a515df3af013d95a66a190a3df3f89221277ebad29e34", + "size": "92612958" + }, + "aarch64-windows": { + "tarball": "https://ziglang.org/download/0.15.1/zig-aarch64-windows-0.15.1.zip", + "shasum": "1f1bf16228b0ffcc882b713dc5e11a6db4219cb30997e13c72e8e723c2104ec6", + "size": "88458549" + }, + "x86-windows": { + "tarball": "https://ziglang.org/download/0.15.1/zig-x86-windows-0.15.1.zip", + "shasum": "fb1c07cffbb43615d3158ab8b8f5db5da1d48875eca99e1d7a8a0064ff63fc5b", + "size": "94516052" + }, + "aarch64-freebsd": { + "tarball": "https://ziglang.org/download/0.15.1/zig-aarch64-freebsd-0.15.1.tar.xz", + "shasum": "4d9d25c775828d49ea037b2284310c295d951793da8ebe94827a54fed4cca3ce", + "size": "49358464" + }, + "arm-freebsd": { + "tarball": "https://ziglang.org/download/0.15.1/zig-arm-freebsd-0.15.1.tar.xz", + "shasum": "9707f3a5f7e1a3d99c40db9a74de1acc61016a197ad289c2ad964f93cb213a18", + "size": "50904124" + }, + "powerpc64-freebsd": { + "tarball": "https://ziglang.org/download/0.15.1/zig-powerpc64-freebsd-0.15.1.tar.xz", + "shasum": "79448884372db04e62f77a46b92245fce805063e69534729537f75cb4681e7e3", + "size": "52099020" + }, + "powerpc64le-freebsd": { + "tarball": "https://ziglang.org/download/0.15.1/zig-powerpc64le-freebsd-0.15.1.tar.xz", + "shasum": "f18ee12ba9c98a20b8d2ad0410c679e7aa5591bc7917f169fd6b377833d2c7ad", + "size": "53480976" + }, + "riscv64-freebsd": { + "tarball": "https://ziglang.org/download/0.15.1/zig-riscv64-freebsd-0.15.1.tar.xz", + "shasum": "ee9f864a6fd8b57c1f4fdbb11daa06578746a6f8253afe3f5ddb5a76f2eddd2d", + "size": "53677800" + }, + "x86_64-freebsd": { + "tarball": "https://ziglang.org/download/0.15.1/zig-x86_64-freebsd-0.15.1.tar.xz", + "shasum": "9714f8ac3d3dc908b1599837c6167f857c1efaa930f0cfa840699458de7c3cd0", + "size": "53782112" + }, + "aarch64-netbsd": { + "tarball": "https://ziglang.org/download/0.15.1/zig-aarch64-netbsd-0.15.1.tar.xz", + "shasum": "b2a528399777583b85b89c54ccd45488af7709d6dd29a27323ec2a229db40910", + "size": "49368000" + }, + "arm-netbsd": { + "tarball": "https://ziglang.org/download/0.15.1/zig-arm-netbsd-0.15.1.tar.xz", + "shasum": "93dc70109cbf5d2e022d20dfb56211978c4ea3c0b1e67aaabff947d8d1583aab", + "size": "52028184" + }, + "x86-netbsd": { + "tarball": "https://ziglang.org/download/0.15.1/zig-x86-netbsd-0.15.1.tar.xz", + "shasum": "a91b26051822ff17f3143f859b87dce5b4a13e90928bd6daa6f07a895d3410f0", + "size": "56881864" + }, + "x86_64-netbsd": { + "tarball": "https://ziglang.org/download/0.15.1/zig-x86_64-netbsd-0.15.1.tar.xz", + "shasum": "6d7ba6eca5b4434351ebdb971b7303c9934514f9bb8481852251dbd5b52b03d6", + "size": "53794836" + } + }, + "0.14.1": { + "date": "2025-05-21", + "docs": "https://ziglang.org/documentation/0.14.1/", + "stdDocs": "https://ziglang.org/documentation/0.14.1/std/", + "src": { + "tarball": "https://ziglang.org/download/0.14.1/zig-0.14.1.tar.xz", + "shasum": "237f8abcc8c3fd68c70c66cdbf63dce4fb5ad4a2e6225ac925e3d5b4c388f203", + "size": "17787696" + }, + "bootstrap": { + "tarball": "https://ziglang.org/download/0.14.1/zig-bootstrap-0.14.1.tar.xz", + "shasum": "89b2fce50bfbb1eee29c382193d22c6eb0c7da3a96b5ba6d05e0af2945b3ca3d", + "size": "48041028" + }, + "x86_64-macos": { + "tarball": "https://ziglang.org/download/0.14.1/zig-x86_64-macos-0.14.1.tar.xz", + "shasum": "b0f8bdfb9035783db58dd6c19d7dea89892acc3814421853e5752fe4573e5f43", + "size": "51044512" + }, + "aarch64-macos": { + "tarball": "https://ziglang.org/download/0.14.1/zig-aarch64-macos-0.14.1.tar.xz", + "shasum": "39f3dc5e79c22088ce878edc821dedb4ca5a1cd9f5ef915e9b3cc3053e8faefa", + "size": "45903552" + }, + "x86_64-linux": { + "tarball": "https://ziglang.org/download/0.14.1/zig-x86_64-linux-0.14.1.tar.xz", + "shasum": "24aeeec8af16c381934a6cd7d95c807a8cb2cf7df9fa40d359aa884195c4716c", + "size": "49086504" + }, + "aarch64-linux": { + "tarball": "https://ziglang.org/download/0.14.1/zig-aarch64-linux-0.14.1.tar.xz", + "shasum": "f7a654acc967864f7a050ddacfaa778c7504a0eca8d2b678839c21eea47c992b", + "size": "44954692" + }, + "armv7a-linux": { + "tarball": "https://ziglang.org/download/0.14.1/zig-armv7a-linux-0.14.1.tar.xz", + "shasum": "1b34d9ecfaeb3b360e86c0bc233e1a8a2bbed2d40f2d4f20c12bde2128714324", + "size": "46137456" + }, + "riscv64-linux": { + "tarball": "https://ziglang.org/download/0.14.1/zig-riscv64-linux-0.14.1.tar.xz", + "shasum": "005f214f74dbafb7b4d8bd305f4e9d25048f711d9ec6fa7b3d4fca177e11b882", + "size": "48094380" + }, + "powerpc64le-linux": { + "tarball": "https://ziglang.org/download/0.14.1/zig-powerpc64le-linux-0.14.1.tar.xz", + "shasum": "15523e748efd9224ae164482e5cc1f8c11b035246b9749fb5b00758484f384a1", + "size": "48720956" + }, + "x86-linux": { + "tarball": "https://ziglang.org/download/0.14.1/zig-x86-linux-0.14.1.tar.xz", + "shasum": "4bce6347fa112247443cb0952c19e560d1f90b910506cf895fd07a7b8d1c4a76", + "size": "51643520" + }, + "loongarch64-linux": { + "tarball": "https://ziglang.org/download/0.14.1/zig-loongarch64-linux-0.14.1.tar.xz", + "shasum": "fdc344789be6becbe220bf5ad035156e653cc148f4da270188cbac87729f17e0", + "size": "45825404" + }, + "s390x-linux": { + "tarball": "https://ziglang.org/download/0.14.1/zig-s390x-linux-0.14.1.tar.xz", + "shasum": "8e1bb63532ba22725f0a4a45e5c2920a8af508d8ca81e6007c330eec2b7033a6", + "size": "55602032" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/download/0.14.1/zig-x86_64-windows-0.14.1.zip", + "shasum": "554f5378228923ffd558eac35e21af020c73789d87afeabf4bfd16f2e6feed2c", + "size": "82229343" + }, + "aarch64-windows": { + "tarball": "https://ziglang.org/download/0.14.1/zig-aarch64-windows-0.14.1.zip", + "shasum": "b5aac0ccc40dd91e8311b1f257717d8e3903b5fefb8f659de6d65a840ad1d0e7", + "size": "78125379" + }, + "x86-windows": { + "tarball": "https://ziglang.org/download/0.14.1/zig-x86-windows-0.14.1.zip", + "shasum": "3ee730c2a5523570dc4dc1b724f3e4f30174ebc1fa109ca472a719586a473b18", + "size": "83983932" + } + }, + "0.14.0": { + "date": "2025-03-05", + "docs": "https://ziglang.org/documentation/0.14.0/", + "stdDocs": "https://ziglang.org/documentation/0.14.0/std/", + "notes": "https://ziglang.org/download/0.14.0/release-notes.html", + "src": { + "tarball": "https://ziglang.org/download/0.14.0/zig-0.14.0.tar.xz", + "shasum": "c76638c03eb204c4432ae092f6fa07c208567e110fbd4d862d131a7332584046", + "size": "17772188" + }, + "bootstrap": { + "tarball": "https://ziglang.org/download/0.14.0/zig-bootstrap-0.14.0.tar.xz", + "shasum": "bf3fcb22be0b83f4791748adb567d3304779d66d7bf9b1bd557ef6c2e0232807", + "size": "48029040" + }, + "x86_64-macos": { + "tarball": "https://ziglang.org/download/0.14.0/zig-macos-x86_64-0.14.0.tar.xz", + "shasum": "685816166f21f0b8d6fc7aa6a36e91396dcd82ca6556dfbe3e329deffc01fec3", + "size": "51039964" + }, + "aarch64-macos": { + "tarball": "https://ziglang.org/download/0.14.0/zig-macos-aarch64-0.14.0.tar.xz", + "shasum": "b71e4b7c4b4be9953657877f7f9e6f7ee89114c716da7c070f4a238220e95d7e", + "size": "45902412" + }, + "x86_64-linux": { + "tarball": "https://ziglang.org/download/0.14.0/zig-linux-x86_64-0.14.0.tar.xz", + "shasum": "473ec26806133cf4d1918caf1a410f8403a13d979726a9045b421b685031a982", + "size": "49091960" + }, + "aarch64-linux": { + "tarball": "https://ziglang.org/download/0.14.0/zig-linux-aarch64-0.14.0.tar.xz", + "shasum": "ab64e3ea277f6fc5f3d723dcd95d9ce1ab282c8ed0f431b4de880d30df891e4f", + "size": "44922728" + }, + "armv7a-linux": { + "tarball": "https://ziglang.org/download/0.14.0/zig-linux-armv7a-0.14.0.tar.xz", + "shasum": "a67dbfa9bdf769228ec994f2098698c619f930883ca5ef638f50eee2d7788d10", + "size": "46112980" + }, + "riscv64-linux": { + "tarball": "https://ziglang.org/download/0.14.0/zig-linux-riscv64-0.14.0.tar.xz", + "shasum": "a2b14d3de326d3fd095548ef38bf5a67b15dadd62fbcc90836d63cc4355f8ef7", + "size": "48069188" + }, + "powerpc64le-linux": { + "tarball": "https://ziglang.org/download/0.14.0/zig-linux-powerpc64le-0.14.0.tar.xz", + "shasum": "3eabd60876ebc2748de8eb57b4b8cfa78861ba9bf7c6dd83f4e3e1d271d7c45e", + "size": "48707620" + }, + "x86-linux": { + "tarball": "https://ziglang.org/download/0.14.0/zig-linux-x86-0.14.0.tar.xz", + "shasum": "55d1ba21de5109686ffa675b9cc1dd66930093c202995a637ce3e397816e4c08", + "size": "51621460" + }, + "loongarch64-linux": { + "tarball": "https://ziglang.org/download/0.14.0/zig-linux-loongarch64-0.14.0.tar.xz", + "shasum": "31a2f07df55f8f528b92d540db9aae6c0b38643c34dc1ac33a0111d855e996ae", + "size": "45821860" + }, + "x86_64-windows": { + "tarball": "https://ziglang.org/download/0.14.0/zig-windows-x86_64-0.14.0.zip", + "shasum": "f53e5f9011ba20bbc3e0e6d0a9441b31eb227a97bac0e7d24172f1b8b27b4371", + "size": "82219809" + }, + "aarch64-windows": { + "tarball": "https://ziglang.org/download/0.14.0/zig-windows-aarch64-0.14.0.zip", + "shasum": "03e984383ebb8f85293557cfa9f48ee8698e7c400239570c9ff1aef3bffaf046", + "size": "78113283" + }, + "x86-windows": { + "tarball": "https://ziglang.org/download/0.14.0/zig-windows-x86-0.14.0.zip", + "shasum": "1a867d808cf4fa9184358395d94441390b6b24ee8d00d356ca11ea7cbfd3a4ec", + "size": "83970029" } }, "0.13.0": { diff --git a/bazel/zig_options.bzl b/bazel/zig_options.bzl new file mode 100644 index 00000000..85938a0c --- /dev/null +++ b/bazel/zig_options.bzl @@ -0,0 +1,170 @@ +"""Build settings for Zig options.""" + +load("@bazel_skylib//lib:types.bzl", "types") +load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") +load("@rules_zig//zig:defs.bzl", "zig_library") + +ZigOption = provider( + doc = "A Zig option's configured value and allowed string values.", + fields = { + "name": "The flag's target name, or this option's target name for explicit values.", + "module_name": "The Bazel module declaring the flag, or empty when unavailable.", + "value": "The configured build setting or explicit value.", + "type": "The Zig integer type, or None for other flag types.", + "allowed_values": "Allowed values for string flags; an empty list allows any string. None for other flag types.", + "enum_name": "Exported enum type name for a constrained string flag; empty keeps the option as a string.", + "nullable": "Whether a string or enum emits null for an empty value.", + }, +) + +_FlagInfo = provider(fields = ["allowed_values"]) + +def _flag_info_impl(_target, ctx): + return [_FlagInfo(allowed_values = getattr(ctx.rule.attr, "values", []))] + +_flag_info = aspect(implementation = _flag_info_impl) + +def _zig_option_impl(ctx): + flag = ctx.attr.flag + if ctx.attr.value_source == "flag": + if flag == None: + fail("flag must resolve to a build setting") + value = flag[BuildSettingInfo].value + else: + value = getattr(ctx.attr, ctx.attr.value_source) + if not (types.is_bool(value) or types.is_int(value) or types.is_string(value)): + fail("Zig options require an integer, boolean, or string value") + allowed_values = None + if types.is_string(value): + allowed_values = flag[_FlagInfo].allowed_values if flag else [] + if ctx.attr.type and not types.is_int(value): + fail("type is only supported for integer options") + if ctx.attr.nullable and not types.is_string(value): + fail("nullable is only supported for string options") + if ctx.attr.enum_name and not allowed_values: + fail("enum_name requires a string flag with nonempty values") + return [ZigOption( + name = flag.label.name if flag else ctx.label.name, + module_name = "", + value = value, + type = (ctx.attr.type or "usize") if types.is_int(value) else None, + allowed_values = allowed_values, + enum_name = ctx.attr.enum_name, + nullable = ctx.attr.nullable, + )] + +def _zig_string(value): + # JSON and Zig differ in their control-character and Unicode escapes. + escaped = [] + for char in value.elems(): + part = json.encode(char)[1:-1] + if part.startswith("\\u"): + part = "\\u{" + part[2:] + "}" + escaped.append(part.replace("\\b", "\\x08").replace("\\f", "\\x0c")) + return '"' + "".join(escaped) + '"' + +def _zig_options_impl(ctx): + lines = [] + names = {} + for setting in ctx.attr.settings: + option = setting[ZigOption] + name = option.name + if name in names: + fail("Duplicate Zig declaration: {}".format(name)) + names[name] = True + value = option.value + if types.is_bool(value): + zig_type = "bool" + literal = "true" if value else "false" + elif types.is_int(value): + zig_type = option.type + literal = str(value) + elif option.enum_name: + enum_values = [v for v in option.allowed_values if not option.nullable or v != ""] + zig_type = "enum { " + ", ".join(["@" + _zig_string(v) for v in enum_values]) + " }" + if option.enum_name in names: + fail("Duplicate Zig declaration: {}".format(option.enum_name)) + names[option.enum_name] = True + lines.append("pub const @{} = {};".format(_zig_string(option.enum_name), zig_type)) + zig_type = "@" + _zig_string(option.enum_name) + if option.nullable: + zig_type = "?" + zig_type + literal = "null" if option.nullable and not value else ".@" + _zig_string(value) + else: + zig_type = "?[]const u8" if option.nullable else "[]const u8" + literal = "null" if option.nullable and not value else _zig_string(value) + lines.append("pub const @{}: {} = {};".format(_zig_string(name), zig_type, literal)) + ctx.actions.write(ctx.outputs.out, "\n".join(lines) + "\n") + return [DefaultInfo(files = depset([ctx.outputs.out]))] + +_zig_options_source = rule( + implementation = _zig_options_impl, + attrs = { + "out": attr.output(mandatory = True), + "settings": attr.label_list( + mandatory = True, + allow_files = False, + providers = [ZigOption], + ), + }, +) + +def zig_options(name, settings, **kwargs): + """Generate an options module and expose it as a zig_library. + + Args: + name: Library target name; the generated source is .zig. + settings: zig_option targets; flag names or explicit option names become quoted Zig constants. + **kwargs: Attributes forwarded to zig_library, except main which is generated. + """ + if "main" in kwargs: + fail("zig_options generates its own main source") + source_attrs = {key: kwargs[key] for key in ["testonly", "tags", "compatible_with", "restricted_to", "target_compatible_with"] if key in kwargs} + _zig_options_source( + name = name + "_source", + settings = settings, + out = name + ".zig", + visibility = ["//visibility:private"], + **source_attrs + ) + zig_library(name = name, main = ":" + name + "_source", **kwargs) + +_zig_option = rule( + implementation = _zig_option_impl, + attrs = { + "flag": attr.label( + providers = [BuildSettingInfo], + aspects = [_flag_info], + doc = "Boolean, integer, or string build setting. An aspect discovers allowed string values.", + ), + "bool": attr.bool(), + "string": attr.string(), + "int": attr.int(), + "value_source": attr.string(mandatory = True, values = ["flag", "bool", "string", "int"]), + "type": attr.string(doc = "Zig integer type; defaults to usize. Only valid for integer options."), + "nullable": attr.bool(doc = "Emit an optional string or enum; an empty value becomes null. String flags with values must include an empty string to allow null; it is excluded from the enum."), + "enum_name": attr.string(doc = "Emit a named Zig enum instead of a string; requires nonempty values. Defaults to emitting a string."), + }, + doc = "Expose a build setting or explicit value as a Zig option, named after the flag or this target respectively.", +) + +def zig_option(name, flag = None, bool = None, string = None, int = None, **kwargs): + """Expose exactly one build setting or explicit value as a Zig option. + + Args: + name: Target name; also the generated Zig constant name for explicit values. + flag: Boolean, integer, or string build setting to read. + bool: Explicit boolean value, which may use select(). + string: Explicit string value, which may use select(). + int: Explicit integer value, which may use select(). + **kwargs: Attributes forwarded to the rule, including type, nullable, and enum_name. + """ + sources = {key: value for key, value in { + "flag": flag, + "bool": bool, + "string": string, + "int": int, + }.items() if value != None} + if len(sources) != 1: + fail("zig_option requires exactly one of flag, bool, string, or int") + _zig_option(name = name, value_source = sources.keys()[0], **dict(sources, **kwargs)) diff --git a/bazel/zig_proto_library.bzl b/bazel/zig_proto_library.bzl index 99c08311..fa82d877 100644 --- a/bazel/zig_proto_library.bzl +++ b/bazel/zig_proto_library.bzl @@ -1,5 +1,6 @@ """Starlark implementation of zig_proto_library""" +load("@com_google_protobuf//bazel/common:proto_info.bzl", "ProtoInfo") load("@rules_proto//proto:defs.bzl", "proto_common") load( "@rules_zig//zig/private/providers:zig_module_info.bzl", @@ -38,7 +39,6 @@ def zig_proto_library_aspect_impl(target, ctx): pb_zig_name = proto_src.basename[:-len(".proto")] + ".pb.zig" zig_src = ctx.actions.declare_file(pb_zig_name, sibling = proto_src) - ctx.actions.run proto_common.compile( ctx.actions, proto_info = proto_info, @@ -71,10 +71,9 @@ def get_import_name(target, proto_src): name = str(target.label) # special handling of builtin types - if "com_google_protobuf//:" in name: - name = "google_protobuf_" + proto_src.basename - else: - name = name.rsplit("//")[-1] + if "com_google_protobuf" in name: + return "google_protobuf_" + proto_src.basename.replace(".", "_") + name = name.rsplit("//")[-1] return name.replace(".", "_").replace(":", "_").replace("/", "_") diff --git a/bazel/zig_srcs.bzl b/bazel/zig_srcs.bzl new file mode 100644 index 00000000..96bebfab --- /dev/null +++ b/bazel/zig_srcs.bzl @@ -0,0 +1,36 @@ +load("@tar.bzl//tar:mtree.bzl", "mtree_spec") +load("@tar.bzl//tar:tar.bzl", "tar") +load("@rules_zig//zig:defs.bzl", "zig_static_library") + +def zig_srcs(name, zig_bin = "", zig_lib = ""): + """For a given zig_library, recursively extract all zig sources into a tarball. + + This also includes the files translated from C headers. + It's also possible to pass zig_lib instead of zig_bin in which case, + The rule takes care of creating an intermediary binary from the lib. + """ + if zig_bin == "": + zig_bin = "{}_bin".format(name) + zig_static_library( + name = zig_bin, + tags = ["manual"], + deps = [zig_lib], + ) + + native.filegroup( + name = "{}_files".format(name), + srcs = [zig_bin], + tags = ["manual"], + output_group = "srcs", + ) + mtree_spec( + name = "{}_mtree".format(name), + srcs = [":{}_files".format(name)], + tags = ["manual"], + ) + tar( + name = name, + srcs = ["{}_files".format(name)], + args = [], + mtree = "{}_mtree".format(name), + ) diff --git a/bin/zml-smi/BUILD.bazel b/bin/zml-smi/BUILD.bazel new file mode 100644 index 00000000..077f5b34 --- /dev/null +++ b/bin/zml-smi/BUILD.bazel @@ -0,0 +1,222 @@ +load("@bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") +load("@bazel_lib//lib:transitions.bzl", "platform_transition_filegroup") +load("@bazel_lib//tools/release:hashes.bzl", "hashes") +load("@bazel_skylib//rules:run_binary.bzl", "run_binary") +load("@rules_oci//oci:defs.bzl", "oci_image", "oci_load") +load("@rules_shell//shell:sh_binary.bzl", "sh_binary") +load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library") +load("@tar.bzl//tar:mtree.bzl", "mtree_spec") +load("@tar.bzl//tar:tar.bzl", "tar") + +# zig libraries + +# smi + +run_binary( + name = "libnrt.stripped", + srcs = ["@aws-neuronx-runtime-lib//:libnrt.patchelf"], + outs = ["libnrt.so.1"], + args = [ + "--strip-debug", + "-o", + "$(location libnrt.so.1)", + "$(location @aws-neuronx-runtime-lib//:libnrt.patchelf)", + ], + target_compatible_with = ["@platforms//os:linux"], + tool = "@llvm//tools:llvm-strip", +) + +run_binary( + name = "libncfw.stripped", + srcs = ["@aws-neuronx-runtime-lib//:libncfw.patchelf"], + outs = ["libncfw.so.2"], + args = [ + "--strip-debug", + "-o", + "$(location libncfw.so.2)", + "$(location @aws-neuronx-runtime-lib//:libncfw.patchelf)", + ], + target_compatible_with = ["@platforms//os:linux"], + tool = "@llvm//tools:llvm-strip", +) + +copy_to_directory( + name = "sandbox", + srcs = select({ + "@platforms//os:macos": [], + "@llvm//platforms/config:linux_x86_64": [ + ":libncfw.stripped", + ":libnrt.stripped", + "//platforms/rocm:zmlxrocm", + "@amdrocm-amdsmi//:libamd_smi", + "@amdrocm-sysdeps//:system_libs", + "@libzml_rocm//:amdgpu_ids", + ], + "@llvm//platforms/config:linux_aarch64": [], + }), + include_external_repositories = ["**"], + replace_prefixes = { + "amdgpu.ids": "share/libdrm/amdgpu.ids", + "platforms/rocm": "lib", + "libnrt.so.1": "lib/libnrt.so.1", + "libncfw.so.2": "lib/libncfw.so.2", + }, +) + +run_binary( + name = "zml-smi.stripped", + srcs = [":zml-smi"], + outs = ["zml-smi.stripped"], + args = [ + "--strip-all", + "-o", + "$(location zml-smi.stripped)", + "$(location :zml-smi)", + ], + tool = "@llvm//tools:llvm-strip", +) + +zig_binary( + name = "zml-smi", + data = [":sandbox"], + main = "main.zig", + visibility = ["//visibility:public"], + deps = [ + "//bin/zml-smi/api", + "//bin/zml-smi/collector", + "//bin/zml-smi/csv", + "//bin/zml-smi/info", + "//bin/zml-smi/json", + "//bin/zml-smi/prometheus", + "//bin/zml-smi/tui", + "//bin/zml-smi/utils:sysfs", + "//stdx", + "@libvaxis//:vaxis", + "@rules_zig//zig/runfiles", + ] + select({ + "@platforms//os:macos": ["//bin/zml-smi/platforms/macos"], + "@platforms//os:linux": ["//bin/zml-smi/platforms/linux"], + }) + select({ + "@llvm//platforms/config:linux_x86_64": [ + "//bin/zml-smi/platforms/amdsmi", + "//bin/zml-smi/platforms/neuron", + "//bin/zml-smi/platforms/nvml", + "//bin/zml-smi/platforms/oneapi", + "//bin/zml-smi/platforms/tpu", + "//platforms/neuron:libnrt_headers", + ], + "@llvm//platforms/config:linux_aarch64": [ + "//bin/zml-smi/platforms/nvml", + ], + "//conditions:default": [], + }), +) + +# docker image + +mtree_spec( + name = "mtree_spec", + srcs = [ + ":zml-smi", + ":zml-smi.stripped", + ], + tags = ["manual"], +) + +# Use this genrule until mtree_mutate is fixed +genrule( + name = "mtree", + srcs = [":mtree_spec"], + outs = ["mtree.txt"], + cmd = """sed -e '/type=dir/d' -e '/zml-smi\\/zml-smi /d' -e 's#^bin/##g' -e 's#zml-smi\\.stripped#zml-smi#' -e '/stamp.c/d' $< > $@""", + tags = ["manual"], +) + +# mtree_mutate( +# name = "mtree", +# mtree = ":mtree_spec", +# strip_prefix = "bin", +# ) + +# Release archives, one per platform, correctly named for distribution +# Build one: bazel build --config=release //bin/zml-smi:zml-smi-linux-amd64 +# Build all: bazel build --config=release //bin/zml-smi:archives + +_RELEASE_PLATFORMS = [ + ("linux", "amd64"), + ("linux", "arm64"), + ("macos", "arm64"), +] + +[tar( + name = "archive_" + os + "_" + arch, + srcs = [ + ":zml-smi", + ":zml-smi.stripped", + ], + out = "zml-smi-" + os + "-" + arch + ".tar.zst", + args = [ + "--options", + ",".join([ + "zstd:compression-level=22", + "zstd:threads=16", + ]), + ], + compress = "zstd", + mtree = ":mtree", + tags = ["manual"], +) for os, arch in _RELEASE_PLATFORMS] + +[platform_transition_filegroup( + name = "zml-smi-" + os + "-" + arch, + srcs = [":archive_" + os + "_" + arch], + tags = ["manual"], + target_platform = "//platforms:" + os + "_" + arch, +) for os, arch in _RELEASE_PLATFORMS] + +[hashes( + name = "zml-smi-" + os + "-" + arch + ".sha256", + src = ":zml-smi-" + os + "-" + arch, + tags = ["manual"], +) for os, arch in _RELEASE_PLATFORMS] + +sh_binary( + name = "release", + srcs = ["scripts/release.sh"], + tags = ["manual"], +) + +filegroup( + name = "archives", + srcs = [":zml-smi-" + os + "-" + arch for os, arch in _RELEASE_PLATFORMS] + + [":zml-smi-" + os + "-" + arch + ".sha256" for os, arch in _RELEASE_PLATFORMS], + tags = ["manual"], +) + +oci_image( + name = "image_", + base = "@distroless_cc_debian12", + entrypoint = ["/zml-smi/zml-smi"], + tags = ["manual"], + target_compatible_with = [ + "@platforms//os:linux", + "@platforms//cpu:x86_64", + ], + tars = [ + ":archive_linux_amd64", + ], +) + +platform_transition_filegroup( + name = "image", + srcs = [":image_"], + tags = ["manual"], + target_platform = "//platforms:linux_amd64", +) + +oci_load( + name = "load", + image = ":image", + repo_tags = ["zmlai/zml-smi:latest"], + tags = ["manual"], +) diff --git a/bin/zml-smi/api/BUILD.bazel b/bin/zml-smi/api/BUILD.bazel new file mode 100644 index 00000000..1a09f597 --- /dev/null +++ b/bin/zml-smi/api/BUILD.bazel @@ -0,0 +1,21 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "api", + import_name = "zml-smi/api", + main = "api.zig", + srcs = [ + "client.zig", + "server.zig", + ], + visibility = ["//visibility:public"], + deps = [ + "//bin/zml-smi/collector", + "//bin/zml-smi/info", + "//bin/zml-smi/json", + "//bin/zml-smi/utils:double_buffer", + ] + select({ + "@platforms//os:macos": ["//bin/zml-smi/platforms/macos"], + "//conditions:default": ["//bin/zml-smi/platforms/linux"], + }), +) diff --git a/bin/zml-smi/api/api.zig b/bin/zml-smi/api/api.zig new file mode 100644 index 00000000..fb39ddd8 --- /dev/null +++ b/bin/zml-smi/api/api.zig @@ -0,0 +1,2 @@ +pub const Server = @import("server.zig").Server; +pub const addRemotes = @import("client.zig").addRemotes; diff --git a/bin/zml-smi/api/client.zig b/bin/zml-smi/api/client.zig new file mode 100644 index 00000000..ddec6d2d --- /dev/null +++ b/bin/zml-smi/api/client.zig @@ -0,0 +1,138 @@ +const std = @import("std"); +const smi_info = @import("zml-smi/info"); +const DeviceInfo = smi_info.device_info.DeviceInfo; +const pi = smi_info.process_info; +const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); +const Collector = @import("zml-smi/collector").Collector; + +fn setRemote(device: *DeviceInfo) void { + inline for (@typeInfo(DeviceInfo).@"union".field_names) |field_name| { + if (device.* == @field(smi_info.device_info.Target, field_name)) { + @field(device, field_name).values[0].remote = true; + @field(device, field_name).values[1].remote = true; + } + } +} + +pub fn addRemotes(collector: *Collector, hosts: []const u8) !void { + var it = std.mem.splitScalar(u8, hosts, ','); + while (it.next()) |host| { + addHost(collector, host) catch |err| { + std.log.err("{s}: {s}", .{ host, @errorName(err) }); + }; + } +} + +fn addHost(collector: *Collector, host: []const u8) !void { + const body = try httpGet(collector.gpa, collector.io, host); + defer collector.gpa.free(body); + + const parsed = try std.json.parseFromSlice(std.json.Value, collector.gpa, body, .{}); + defer parsed.deinit(); + + const root = parsed.value.object; + const json_devices = if (root.get("devices")) |v| v.array.items else &.{}; + + const dev_offset: u16 = @intCast(collector.device_infos.items.len); + + var devices: std.ArrayList(*DeviceInfo) = .empty; + for (json_devices) |item| { + var device = std.json.parseFromValueLeaky(DeviceInfo, collector.arena, item, .{ .ignore_unknown_fields = true }) catch continue; + setRemote(&device); + const info = try collector.addDevice(device); + try devices.append(collector.arena, info); + } + + if (devices.items.len == 0) { + return; + } + + const processes = try collector.createProcessList(); + const str_arenas: [2]*std.heap.ArenaAllocator = .{ try collector.createPollArena(), try collector.createPollArena() }; + + const json_processes = if (root.get("processes")) |v| v.array.items else &.{}; + const back_idx: usize = 1 - processes.current.load(.acquire); + const back = processes.back(); + for (json_processes) |item| { + var proc = std.json.parseFromValueLeaky(pi.ProcessInfo, str_arenas[back_idx].allocator(), item, .{ .ignore_unknown_fields = true }) catch continue; + proc.device_idx += dev_offset; + proc.remote = true; + back.append(collector.gpa, proc) catch continue; + } + processes.swap(); + + const url = try collector.arena.dupe(u8, host); + const poll_arena = try collector.createPollArena(); + try collector.spawnPoll(pollOnce, .{ poll_arena, collector.gpa, collector.io, url, devices.items, processes, dev_offset, str_arenas }); +} + +fn pollOnce(poll_arena: *std.heap.ArenaAllocator, gpa: std.mem.Allocator, io: std.Io, host: []const u8, devices: []const *DeviceInfo, processes: *ProcessDoubleBuffer, dev_offset: u16, str_arenas: [2]*std.heap.ArenaAllocator) void { + _ = poll_arena.reset(.retain_capacity); + + const body = httpGet(gpa, io, host) catch return; + defer gpa.free(body); + + const parsed = std.json.parseFromSlice(std.json.Value, gpa, body, .{}) catch return; + defer parsed.deinit(); + + const root = parsed.value.object; + const json_devices = if (root.get("devices")) |v| v.array.items else &.{}; + + for (json_devices, 0..) |item, i| { + if (i >= devices.len) { + std.log.warn("{s}: remote has more devices than expected ({d} > {d})", .{ host, json_devices.len, devices.len }); + break; + } + var device = std.json.parseFromValueLeaky(DeviceInfo, poll_arena.allocator(), item, .{ .ignore_unknown_fields = true }) catch continue; + setRemote(&device); + + inline for (@typeInfo(DeviceInfo).@"union".field_names) |field_name| { + if (devices[i].* == @field(smi_info.device_info.Target, field_name)) { + @field(devices[i], field_name).back().* = @field(device, field_name).front().*; + @field(devices[i], field_name).swap(); + } + } + } + + const json_processes = if (root.get("processes")) |v| v.array.items else &.{}; + const back_idx: usize = 1 - processes.current.load(.acquire); + _ = str_arenas[back_idx].reset(.retain_capacity); + const alloc = str_arenas[back_idx].allocator(); + const back = processes.back(); + back.clearRetainingCapacity(); + for (json_processes) |item| { + var proc = std.json.parseFromValueLeaky(pi.ProcessInfo, alloc, item, .{ .ignore_unknown_fields = true }) catch continue; + proc.device_idx += dev_offset; + proc.remote = true; + back.append(gpa, proc) catch continue; + } + processes.swap(); +} + +fn httpGet(allocator: std.mem.Allocator, io: std.Io, host: []const u8) ![]const u8 { + if (!std.mem.startsWith(u8, host, "http://") and !std.mem.startsWith(u8, host, "https://")) { + std.log.err("remote host must start with http:// or https://: {s}", .{host}); + return error.InvalidUrl; + } + + const url = try std.fmt.allocPrint(allocator, "{s}/metrics", .{host}); + defer allocator.free(url); + + var client: std.http.Client = .{ .allocator = allocator, .io = io }; + defer client.deinit(); + + var aw: std.Io.Writer.Allocating = .init(allocator); + errdefer aw.deinit(); + + const result = try client.fetch(.{ + .location = .{ .url = url }, + .response_writer = &aw.writer, + }); + + if (result.status != .ok) { + aw.deinit(); + return error.HttpError; + } + + return try aw.toOwnedSlice(); +} diff --git a/bin/zml-smi/api/server.zig b/bin/zml-smi/api/server.zig new file mode 100644 index 00000000..e05aa7fe --- /dev/null +++ b/bin/zml-smi/api/server.zig @@ -0,0 +1,105 @@ +const std = @import("std"); +const builtin = @import("builtin"); +const smi_info = @import("zml-smi/info"); +const DeviceInfo = smi_info.device_info.DeviceInfo; +const ProcessInfo = smi_info.process_info.ProcessInfo; +const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(ProcessInfo)); +const json = @import("zml-smi/json"); +const ProcessEnricher = if (builtin.os.tag == .macos) + @import("zml-smi/platforms/macos").process.ProcessEnricher +else + @import("zml-smi/platforms/linux").process.ProcessEnricher; + +pub const Server = struct { + allocator: std.mem.Allocator, + io: std.Io, + devices: []const *DeviceInfo, + process_lists: []const *const ProcessDoubleBuffer, + enricher: *ProcessEnricher, + tcp: std.Io.net.Server, + connection_group: std.Io.Group = .init, + + pub fn init(allocator: std.mem.Allocator, io: std.Io, port: u16, devices: []const *DeviceInfo, process_lists: []const *const ProcessDoubleBuffer, enricher: *ProcessEnricher) !Server { + const address: std.Io.net.IpAddress = .{ .ip4 = .{ + .bytes = .{ 0, 0, 0, 0 }, + .port = port, + } }; + var tcp = try address.listen(io, .{ .reuse_address = true }); + errdefer tcp.deinit(io); + + return .{ + .allocator = allocator, + .io = io, + .devices = devices, + .process_lists = process_lists, + .enricher = enricher, + .tcp = tcp, + }; + } + + pub fn run(allocator: std.mem.Allocator, io: std.Io, port: u16, devices: []const *DeviceInfo, process_lists: []const *const ProcessDoubleBuffer, enricher: *ProcessEnricher) void { + var self = init(allocator, io, port, devices, process_lists, enricher) catch |err| { + std.log.err("api server failed to start: {s}", .{@errorName(err)}); + return; + }; + defer self.deinit(); + self.serve(); + } + + pub fn deinit(self: *Server) void { + self.connection_group.await(self.io) catch {}; + self.tcp.deinit(self.io); + } + + pub fn serve(self: *Server) void { + while (true) { + const stream = self.tcp.accept(self.io) catch break; + self.connection_group.concurrent(self.io, Server.onConnection, .{ self, stream }) catch break; + } + } + + fn onConnection(self: *Server, stream: std.Io.net.Stream) std.Io.Cancelable!void { + defer stream.close(self.io); + self.handleConn(stream) catch {}; + } + + fn handleConn(self: *Server, stream: std.Io.net.Stream) !void { + var read_buf: [8192]u8 = undefined; + var reader = stream.reader(self.io, &read_buf); + + var write_buf: [4096]u8 = undefined; + var writer = stream.writer(self.io, &write_buf); + + var http_server: std.http.Server = .init(&reader.interface, &writer.interface); + + var procs: std.ArrayList(ProcessInfo) = .empty; + defer procs.deinit(self.allocator); + + while (true) { + var request = http_server.receiveHead() catch |err| switch (err) { + error.HttpConnectionClosing => return, + else => return err, + }; + + procs.clearRetainingCapacity(); + for (self.process_lists) |pl| { + procs.appendSlice(self.allocator, pl.front().items) catch {}; + } + self.enricher.enrich(self.io, procs.items); + + var body_buf: [4096]u8 = undefined; + var body_writer = try request.respondStreaming(&body_buf, .{ + .respond_options = .{ + .status = .ok, + .extra_headers = &.{ + .{ .name = "content-type", .value = "application/json" }, + .{ .name = "access-control-allow-origin", .value = "*" }, + }, + }, + }); + + try json.write(&body_writer.writer, self.devices, procs.items); + try body_writer.end(); + } + } +}; diff --git a/bin/zml-smi/collector/BUILD.bazel b/bin/zml-smi/collector/BUILD.bazel new file mode 100644 index 00000000..0bf5fa8e --- /dev/null +++ b/bin/zml-smi/collector/BUILD.bazel @@ -0,0 +1,13 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "collector", + import_name = "zml-smi/collector", + main = "collector.zig", + srcs = ["worker.zig"], + visibility = ["//visibility:public"], + deps = [ + "//bin/zml-smi/info", + "//bin/zml-smi/utils:double_buffer", + ], +) diff --git a/bin/zml-smi/collector/collector.zig b/bin/zml-smi/collector/collector.zig new file mode 100644 index 00000000..d98b204d --- /dev/null +++ b/bin/zml-smi/collector/collector.zig @@ -0,0 +1,93 @@ +const std = @import("std"); +const smi_info = @import("zml-smi/info"); +const DeviceInfo = smi_info.device_info.DeviceInfo; +const pi = smi_info.process_info; +const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); +const Worker = @import("worker.zig").Worker; + +pub const Collector = struct { + device_infos: std.ArrayList(*DeviceInfo) = .empty, + process_lists: std.ArrayList(*ProcessDoubleBuffer) = .empty, + poll_arenas: std.ArrayList(*std.heap.ArenaAllocator) = .empty, + arena: std.mem.Allocator, + gpa: std.mem.Allocator, + worker: Worker, + io: std.Io, + poll_only: bool, + + pub fn init(arena: std.mem.Allocator, gpa: std.mem.Allocator, io: std.Io, options: struct { poll_interval_ms: u16, poll_only: bool = false }) Collector { + return .{ + .arena = arena, + .gpa = gpa, + .io = io, + .worker = .{ .poll_interval_ms = options.poll_interval_ms }, + .poll_only = options.poll_only, + }; + } + + pub fn addDevice(self: *Collector, initial: DeviceInfo) !*DeviceInfo { + const info = try self.arena.create(DeviceInfo); + info.* = initial; + try self.device_infos.append(self.arena, info); + + return info; + } + + pub fn createProcessList(self: *Collector) !*ProcessDoubleBuffer { + const list = try self.arena.create(ProcessDoubleBuffer); + list.* = .{ .values = .{ .empty, .empty } }; + try self.process_lists.append(self.arena, list); + + return list; + } + + pub fn createPollArena(self: *Collector) !*std.heap.ArenaAllocator { + const poll_arena = try self.arena.create(std.heap.ArenaAllocator); + poll_arena.* = std.heap.ArenaAllocator.init(self.gpa); + try self.poll_arenas.append(self.arena, poll_arena); + + return poll_arena; + } + + pub fn spawnPoll(self: *Collector, comptime pollOnce: anytype, args: std.meta.ArgsTuple(@TypeOf(pollOnce))) !void { + @call(.auto, pollOnce, args); + if (self.poll_only) return; + + const Args = std.meta.ArgsTuple(@TypeOf(pollOnce)); + try self.worker.spawn(self.io, struct { + fn f(io: std.Io, w: *const Worker, a: Args) void { + const interval: std.Io.Duration = .fromMilliseconds(w.poll_interval_ms); + while (w.isRunning()) { + const start: std.Io.Timestamp = .now(io, .awake); + + @call(.auto, pollOnce, a); + + const elapsed = start.untilNow(io, .awake); + if (elapsed.nanoseconds < interval.nanoseconds) { + io.sleep(.fromNanoseconds(interval.nanoseconds - elapsed.nanoseconds), .awake) catch {}; + } + } + } + }.f, .{ self.io, &self.worker, args }); + } + + pub fn deinit(self: *Collector) void { + self.worker.shutdown(self.io); + + for (self.device_infos.items) |info| { + self.arena.destroy(info); + } + self.device_infos.deinit(self.arena); + + for (self.process_lists.items) |list| { + list.values[0].deinit(self.gpa); + list.values[1].deinit(self.gpa); + } + self.process_lists.deinit(self.arena); + + for (self.poll_arenas.items) |poll_arena| { + poll_arena.deinit(); + } + self.poll_arenas.deinit(self.arena); + } +}; diff --git a/bin/zml-smi/collector/worker.zig b/bin/zml-smi/collector/worker.zig new file mode 100644 index 00000000..125440c5 --- /dev/null +++ b/bin/zml-smi/collector/worker.zig @@ -0,0 +1,25 @@ +const std = @import("std"); + +pub const Worker = struct { + poll_interval_ms: u16, + should_stop: std.atomic.Value(bool) = .init(false), + group: std.Io.Group = .init, + + pub fn shutdown(self: *Worker, io: std.Io) void { + self.should_stop.store(true, .release); + self.group.await(io) catch {}; + } + + pub fn isRunning(self: *const Worker) bool { + return !self.should_stop.load(.acquire); + } + + pub fn spawn( + self: *Worker, + io: std.Io, + comptime runFn: anytype, + args: std.meta.ArgsTuple(@TypeOf(runFn)), + ) !void { + try self.group.concurrent(io, runFn, args); + } +}; diff --git a/async/BUILD.bazel b/bin/zml-smi/csv/BUILD.bazel similarity index 55% rename from async/BUILD.bazel rename to bin/zml-smi/csv/BUILD.bazel index d7ac4a59..a0cb3422 100644 --- a/async/BUILD.bazel +++ b/bin/zml-smi/csv/BUILD.bazel @@ -1,11 +1,11 @@ load("@rules_zig//zig:defs.bzl", "zig_library") zig_library( - name = "async", - main = "async.zig", + name = "csv", + import_name = "zml-smi/csv", + main = "csv.zig", visibility = ["//visibility:public"], deps = [ - "@zigcoro//:libcoro", - "@libxev//:xev", + "//bin/zml-smi/info", ], ) diff --git a/bin/zml-smi/csv/csv.zig b/bin/zml-smi/csv/csv.zig new file mode 100644 index 00000000..dbd80dd9 --- /dev/null +++ b/bin/zml-smi/csv/csv.zig @@ -0,0 +1,46 @@ +const std = @import("std"); +const smi_info = @import("zml-smi/info"); +const DeviceInfo = smi_info.device_info.DeviceInfo; + +pub fn write(writer: *std.Io.Writer, devices: []const *DeviceInfo) !void { + const info = @typeInfo(DeviceInfo).@"union"; + inline for (info.field_names, info.field_types) |type_name, DeviceType| { + const tag = @field(smi_info.Target, type_name); + var header_printed = false; + + for (devices, 0..) |dev, i| { + switch (dev.*) { + tag => |*db| { + if (!header_printed) { + try writer.writeAll("index,type"); + inline for (@typeInfo(DeviceType.Value).@"struct".field_names) |field_name| { + try writer.writeAll("," ++ field_name); + } + try writer.writeAll("\n"); + + header_printed = true; + } + + const val = db.front().*; + + try writer.print("{d},{s}", .{ i, type_name }); + inline for (@typeInfo(DeviceType.Value).@"struct".field_names) |field_name| { + try writer.writeAll(","); + try writeValue(writer, @field(val, field_name)); + } + try writer.writeAll("\n"); + }, + else => {}, + } + } + } +} + +fn writeValue(writer: *std.Io.Writer, field: anytype) !void { + if (@typeInfo(@TypeOf(field)) == .optional) { + if (field) |v| return writeValue(writer, v); + } else switch (@typeInfo(@TypeOf(field))) { + .pointer => return writer.writeAll(field), + else => return writer.print("{any}", .{field}), + } +} diff --git a/bin/zml-smi/info/BUILD.bazel b/bin/zml-smi/info/BUILD.bazel new file mode 100644 index 00000000..d035b161 --- /dev/null +++ b/bin/zml-smi/info/BUILD.bazel @@ -0,0 +1,15 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "info", + import_name = "zml-smi/info", + main = "info.zig", + srcs = [ + "device_info.zig", + "host_info.zig", + "poll_metrics.zig", + "process_info.zig", + ], + visibility = ["//visibility:public"], + deps = ["//bin/zml-smi/utils:double_buffer"], +) diff --git a/bin/zml-smi/info/device_info.zig b/bin/zml-smi/info/device_info.zig new file mode 100644 index 00000000..4b01c728 --- /dev/null +++ b/bin/zml-smi/info/device_info.zig @@ -0,0 +1,95 @@ +const std = @import("std"); +const DoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer; + +pub const Target = enum { + cuda, + rocm, + oneapi, + neuron, + tpu, +}; + +pub const DeviceInfo = union(Target) { + cuda: DoubleBuffer(GpuInfo), + rocm: DoubleBuffer(GpuInfo), + oneapi: DoubleBuffer(GpuInfo), + neuron: DoubleBuffer(NeuronInfo), + tpu: DoubleBuffer(TpuInfo), + + pub fn isRemote(self: *const DeviceInfo) bool { + switch (self.*) { + inline else => |*db| return db.front().remote, + } + } +}; + +pub const GpuInfo = struct { + remote: bool = false, + name: ?[]const u8 = null, + driver_version: ?[]const u8 = null, + cuda_driver_version: ?[]const u8 = null, + + // Utilization + util_percent: ?u64 = null, + encoder_util_percent: ?u64 = null, + decoder_util_percent: ?u64 = null, + + // Power + power_mw: ?u64 = null, + power_limit_mw: ?u64 = null, + // Thermal + temperature: ?u64 = null, + fan_speed_percent: ?u64 = null, + + // Clocks + clock_graphics_mhz: ?u64 = null, + clock_sm_mhz: ?u64 = null, + clock_soc_mhz: ?u64 = null, + clock_mem_mhz: ?u64 = null, + clock_graphics_max_mhz: ?u64 = null, + clock_mem_max_mhz: ?u64 = null, + + // Memory + mem_used_bytes: ?u64 = null, + mem_total_bytes: ?u64 = null, + mem_bus_width: ?u64 = null, + + // PCIe + pcie_tx_kbps: ?u64 = null, + pcie_rx_kbps: ?u64 = null, + pcie_bandwidth_mbps: ?u64 = null, + pcie_link_gen: ?u64 = null, + pcie_link_width: ?u64 = null, +}; + +pub const NeuronInfo = struct { + remote: bool = false, + name: ?[]const u8 = null, + driver_version: ?[]const u8 = null, + + util_percent: ?u64 = null, + mem_used_bytes: ?u64 = null, + mem_total_bytes: ?u64 = null, + + // Neuron per-core HBM breakdown (bytes) + nc_tensors: ?u64 = null, + nc_constants: ?u64 = null, + nc_model_code: ?u64 = null, + nc_shared_scratchpad: ?u64 = null, + nc_nonshared_scratchpad: ?u64 = null, + nc_runtime: ?u64 = null, + nc_driver: ?u64 = null, + nc_dma_rings: ?u64 = null, + nc_collectives: ?u64 = null, + nc_notifications: ?u64 = null, + nc_uncategorized: ?u64 = null, +}; + +pub const TpuInfo = struct { + remote: bool = false, + name: ?[]const u8 = null, + + util_percent: ?u64 = null, + mem_used_bytes: ?u64 = null, + mem_total_bytes: ?u64 = null, +}; diff --git a/bin/zml-smi/info/host_info.zig b/bin/zml-smi/info/host_info.zig new file mode 100644 index 00000000..f33f6fd8 --- /dev/null +++ b/bin/zml-smi/info/host_info.zig @@ -0,0 +1,16 @@ +const DoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer; + +pub const HostInfo = DoubleBuffer(HostData); + +pub const HostData = struct { + hostname: ?[]const u8 = null, + kernel: ?[]const u8 = null, + cpu_name: ?[]const u8 = null, + cpu_cores: ?u64 = null, + mem_total_kib: ?u64 = null, + mem_available_kib: ?u64 = null, + load_1: ?f32 = null, + load_5: ?f32 = null, + load_15: ?f32 = null, + uptime_seconds: ?u64 = null, +}; diff --git a/bin/zml-smi/info/info.zig b/bin/zml-smi/info/info.zig new file mode 100644 index 00000000..42e1b1b3 --- /dev/null +++ b/bin/zml-smi/info/info.zig @@ -0,0 +1,9 @@ +const std = @import("std"); + +pub const device_info = @import("device_info.zig"); +pub const host_info = @import("host_info.zig"); +pub const poll_metrics = @import("poll_metrics.zig"); +pub const process_info = @import("process_info.zig"); + +pub const Target = device_info.Target; +pub const Targets = std.EnumSet(Target); diff --git a/bin/zml-smi/info/poll_metrics.zig b/bin/zml-smi/info/poll_metrics.zig new file mode 100644 index 00000000..62ed94a7 --- /dev/null +++ b/bin/zml-smi/info/poll_metrics.zig @@ -0,0 +1,20 @@ +const std = @import("std"); + +pub fn apply(comptime table: anytype, dest: anytype, src: anytype) void { + inline for (table) |m| { + @field(dest, m.field) = m.query(src) catch null; + } +} + +pub fn poll(comptime DB: type, comptime Dev: type, comptime table: anytype) fn (?*std.heap.ArenaAllocator, DB, Dev) void { + return struct { + fn f(arena: ?*std.heap.ArenaAllocator, db: DB, dev: Dev) void { + if (arena) |a| _ = a.reset(.retain_capacity); + + const back = db.back(); + back.* = db.front().*; + apply(table, back, dev); + db.swap(); + } + }.f; +} diff --git a/bin/zml-smi/info/process_info.zig b/bin/zml-smi/info/process_info.zig new file mode 100644 index 00000000..73504656 --- /dev/null +++ b/bin/zml-smi/info/process_info.zig @@ -0,0 +1,12 @@ +pub const ProcessInfo = struct { + pid: u32 = 0, + device_idx: u16 = 0, + dev_util_percent: ?u16 = null, + dev_mem_kib: ?u64 = null, + uid: u32 = 0, + username: []const u8 = "", + comm: []const u8 = "", + cpu_percent: u16 = 0, // *10, e.g. 125 = 12.5% + rss_kib: u64 = 0, + remote: bool = false, +}; diff --git a/bin/zml-smi/json/BUILD.bazel b/bin/zml-smi/json/BUILD.bazel new file mode 100644 index 00000000..decc8ca1 --- /dev/null +++ b/bin/zml-smi/json/BUILD.bazel @@ -0,0 +1,11 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "json", + import_name = "zml-smi/json", + main = "json.zig", + visibility = ["//visibility:public"], + deps = [ + "//bin/zml-smi/info", + ], +) diff --git a/bin/zml-smi/json/json.zig b/bin/zml-smi/json/json.zig new file mode 100644 index 00000000..c32a16cf --- /dev/null +++ b/bin/zml-smi/json/json.zig @@ -0,0 +1,15 @@ +const std = @import("std"); +const smi_info = @import("zml-smi/info"); +const DeviceInfo = smi_info.device_info.DeviceInfo; +const ProcessInfo = smi_info.process_info.ProcessInfo; + +pub fn write(writer: *std.Io.Writer, devices: []const *DeviceInfo, processes: []const ProcessInfo) !void { + var jw: std.json.Stringify = .{ .writer = writer }; + try jw.write(Response{ .devices = devices, .processes = processes }); + try writer.writeAll("\n"); +} + +const Response = struct { + devices: []const *DeviceInfo, + processes: []const ProcessInfo, +}; diff --git a/bin/zml-smi/main.zig b/bin/zml-smi/main.zig new file mode 100644 index 00000000..f066dd34 --- /dev/null +++ b/bin/zml-smi/main.zig @@ -0,0 +1,241 @@ +const std = @import("std"); +const builtin = @import("builtin"); + +const stdx = @import("stdx"); + +pub const std_options: std.Options = .{ + .log_scope_levels = &.{ + .{ .scope = .vaxis, .level = .warn }, + }, +}; + +const Collector = @import("zml-smi/collector").Collector; +const smi_info = @import("zml-smi/info"); +const HostInfo = smi_info.host_info.HostInfo; +const platform = if (builtin.os.tag == .macos) + @import("zml-smi/platforms/macos") +else + @import("zml-smi/platforms/linux"); +const ProcessEnricher = platform.process.ProcessEnricher; +const host = platform.metrics; +const c = @import("c"); +const csv = @import("zml-smi/csv"); +const json = @import("zml-smi/json"); +const api = @import("zml-smi/api"); +const prometheus = @import("zml-smi/prometheus"); +const sysfs = @import("zml-smi/sysfs"); +const smi_tui = @import("zml-smi/tui"); +const tui = smi_tui.top; +const static_print = smi_tui.print; +const data = smi_tui.data; + +const CliArgs = struct { + top: bool = false, + csv: bool = false, + json: bool = false, + api: bool = false, + api_port: u16 = 9090, + remotes: ?[]const u8 = null, + prometheus_listen: ?[]const u8 = null, + tui_refresh_rate: u16 = 100, + poll_interval: u16 = 500, + + pub const help = + \\ zml-smi [--top] [--csv] [--json] [--prometheus-listen HOST:PORT] + \\ [--tui-refresh-rate MS] [--poll-interval MS] + \\ + \\ --top Interactive TUI mode + \\ --csv Output device metrics as CSV + \\ --json Output device metrics as JSON + \\ --prometheus-listen Expose metrics as Prometheus endpoint on HOST:PORT + \\ --tui-refresh-rate TUI refresh rate in ms (default: 100) + \\ --poll-interval Device polling interval in ms (default: 500) + \\ + ; + + // [--api] [--port PORT] [--remotes HOST,...] + // \\ --api Expose device metrics as HTTP JSON endpoint + // \\ --api-port API server port (default: 9090) + // \\ --remotes Add devices from remote hosts (comma-separated URLs) +}; + +const device_backends = switch (builtin.os.tag) { + .macos => .{}, + .linux => switch (builtin.cpu.arch) { + .x86_64 => .{ + .{ .cuda, @import("zml-smi/platforms/nvml") }, + .{ .rocm, @import("zml-smi/platforms/amdsmi") }, + .{ .oneapi, @import("zml-smi/platforms/oneapi") }, + .{ .neuron, @import("zml-smi/platforms/neuron") }, + .{ .tpu, @import("zml-smi/platforms/tpu") }, + }, + .aarch64 => .{ + .{ .cuda, @import("zml-smi/platforms/nvml") }, + }, + else => unreachable, + }, + else => unreachable, +}; + +pub fn main(init: std.process.Init) !void { + const gpa = init.gpa; + const arena = init.arena.allocator(); + const io = init.io; + const args = stdx.flags.parse(init.minimal.args, CliArgs); + + const targets = detect(gpa, io); + + var collector: Collector = .init(arena, gpa, io, .{ + .poll_interval_ms = args.poll_interval, + .poll_only = !args.top and args.prometheus_listen == null and !args.api, + }); + defer collector.deinit(); + + var host_info: HostInfo = .{ .values = .{ .{}, .{} } }; + try host.init(&collector, &host_info); + + var enricher: ProcessEnricher = try .init(gpa, io); + defer enricher.deinit(); + + if (args.api) { + std.log.info("serving api metrics on port {d}", .{args.api_port}); + } + + if (args.prometheus_listen) |listen| { + std.log.info("serving prometheus metrics on {s}", .{listen}); + } + + inline for (device_backends) |entry| { + const target, const backend = entry; + if (targets.contains(target)) { + backend.start(&collector) catch |err| { + std.log.err("{s} skipped: {s}", .{ @tagName(target), @errorName(err) }); + }; + } + } + + var api_group: std.Io.Group = .init; + + if (args.remotes) |hosts| { + try api.addRemotes(&collector, hosts); + } + + var state = try data.SystemState.init(.{ + .devices = collector.device_infos.items, + .host = &host_info, + .targets = targets, + .tui_refresh_rate = args.tui_refresh_rate, + .process_lists = collector.process_lists.items, + .enricher = &enricher, + .gpa = gpa, + .arena = arena, + .io = io, + }); + defer state.deinit(arena); + + if (args.prometheus_listen) |listen| { + try api_group.concurrent(io, prometheus.Server.run, .{ io, listen, collector.device_infos.items, &host_info }); + } + + if (args.api) { + try api_group.concurrent(io, api.Server.run, .{ gpa, io, args.api_port, collector.device_infos.items, collector.process_lists.items, &enricher }); + } + + if (args.prometheus_listen != null or args.api) { + try api_group.await(io); + } else if (args.csv) { + var csv_buf: [4096]u8 = undefined; + var csv_writer = std.Io.File.stdout().writer(io, &csv_buf); + + try csv.write(&csv_writer.interface, collector.device_infos.items); + try csv_writer.flush(); + } else if (args.json) { + var procs: std.ArrayList(smi_info.process_info.ProcessInfo) = .empty; + defer procs.deinit(gpa); + for (collector.process_lists.items) |pl| { + try procs.appendSlice(gpa, pl.front().items); + } + enricher.enrich(io, procs.items); + + var json_buf: [4096]u8 = undefined; + var json_writer = std.Io.File.stdout().writer(io, &json_buf); + + try json.write(&json_writer.interface, collector.device_infos.items, procs.items); + try json_writer.flush(); + } else if (args.top) { + try tui.run(gpa, io, init.environ_map, &state); + } else { + try static_print.run(arena, io, init.environ_map, &state); + } +} + +fn detect(allocator: std.mem.Allocator, io: std.Io) smi_info.Targets { + if (comptime builtin.os.tag == .macos) { + return .{}; + } + + var targets: smi_info.Targets = .{}; + + switch (builtin.os.tag) { + .macos => {}, + .linux => { + switch (builtin.cpu.arch) { + .x86_64 => { + if (hasDevice(io, "/dev/nvidiactl")) { + targets.insert(.cuda); + } + + if (hasDevice(io, "/dev/kfd")) { + targets.insert(.rocm); + } + + if (hasDrmRenderDevice(allocator, io, "0x8086")) { + targets.insert(.oneapi); + } + + if (hasDevice(io, c.NEURON_DEVICE_PREFIX ++ "0")) { + targets.insert(.neuron); + } + + if (hasDevice(io, "/dev/accel0") or hasDevice(io, "/dev/vfio/0")) { + targets.insert(.tpu); + } + }, + .aarch64 => { + if (hasDevice(io, "/dev/nvidiactl")) { + targets.insert(.cuda); + } + }, + else => unreachable, + } + }, + else => unreachable, + } + + return targets; +} + +fn hasDevice(io: std.Io, path: []const u8) bool { + std.Io.Dir.accessAbsolute(io, path, .{ .read = true }) catch return false; + return true; +} + +fn hasDrmRenderDevice(allocator: std.mem.Allocator, io: std.Io, comptime pci_vendor_id: []const u8) bool { + var dir = std.Io.Dir.openDirAbsolute(io, "/dev/dri", .{ .iterate = true }) catch return false; + defer dir.close(io); + + var it = dir.iterate(); + while (it.next(io) catch null) |entry| { + if (!std.mem.startsWith(u8, entry.name, "renderD")) continue; + + var vendor_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const vendor_path = std.fmt.bufPrint(&vendor_path_buf, "/sys/class/drm/{s}/device/vendor", .{entry.name}) catch continue; + const vendor = sysfs.readString(allocator, io, vendor_path) catch continue; + defer allocator.free(vendor); + if (std.mem.eql(u8, std.mem.trim(u8, vendor, &std.ascii.whitespace), pci_vendor_id)) { + return true; + } + } + + return false; +} diff --git a/bin/zml-smi/platforms/amdsmi/BUILD.bazel b/bin/zml-smi/platforms/amdsmi/BUILD.bazel new file mode 100644 index 00000000..dac6b935 --- /dev/null +++ b/bin/zml-smi/platforms/amdsmi/BUILD.bazel @@ -0,0 +1,22 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "amdsmi", + srcs = [ + "amdsmi.zig", + "process.zig", + ], + import_name = "zml-smi/platforms/amdsmi", + main = "metrics.zig", + target_compatible_with = ["@platforms//os:linux"], + visibility = ["//visibility:public"], + deps = [ + "//bin/zml-smi/collector", + "//bin/zml-smi/info", + "//bin/zml-smi/utils:double_buffer", + "//bin/zml-smi/utils:dynlib", + "//bin/zml-smi/utils:sandbox", + "//stdx", + "@amdrocm-amdsmi//:amdsmi", + ], +) diff --git a/bin/zml-smi/platforms/amdsmi/amdsmi.zig b/bin/zml-smi/platforms/amdsmi/amdsmi.zig new file mode 100644 index 00000000..2d89bc7c --- /dev/null +++ b/bin/zml-smi/platforms/amdsmi/amdsmi.zig @@ -0,0 +1,295 @@ +const std = @import("std"); +const c = @import("c"); +const stdx = @import("stdx"); +const DynLib = @import("zml-smi/dynlib"); +const sandbox = @import("zml-smi/sandbox"); + +const AmdSmi = @This(); + +pub const Error = ReturnError || error{AmdSmiUnavailable}; +pub const Handle = c.amdsmi_processor_handle; +pub const ProcInfo = c.amdsmi_proc_info_t; + +lib: Fns, +gpu_handles: []c.amdsmi_processor_handle, + +const Fns = struct { + amdsmi_init: *const @TypeOf(c.amdsmi_init), + amdsmi_get_socket_handles: *const @TypeOf(c.amdsmi_get_socket_handles), + amdsmi_get_processor_handles: *const @TypeOf(c.amdsmi_get_processor_handles), + amdsmi_get_gpu_asic_info: *const @TypeOf(c.amdsmi_get_gpu_asic_info), + amdsmi_get_gpu_metrics_info: *const @TypeOf(c.amdsmi_get_gpu_metrics_info), + amdsmi_get_power_info: *const @TypeOf(c.amdsmi_get_power_info), + amdsmi_get_temp_metric: *const @TypeOf(c.amdsmi_get_temp_metric), + amdsmi_get_gpu_fan_speed: *const @TypeOf(c.amdsmi_get_gpu_fan_speed), + amdsmi_get_gpu_activity: *const @TypeOf(c.amdsmi_get_gpu_activity), + amdsmi_get_clock_info: *const @TypeOf(c.amdsmi_get_clock_info), + amdsmi_get_gpu_memory_total: *const @TypeOf(c.amdsmi_get_gpu_memory_total), + amdsmi_get_gpu_memory_usage: *const @TypeOf(c.amdsmi_get_gpu_memory_usage), + amdsmi_get_pcie_info: *const @TypeOf(c.amdsmi_get_pcie_info), + amdsmi_get_gpu_bdf_id: *const @TypeOf(c.amdsmi_get_gpu_bdf_id), + amdsmi_get_gpu_process_list: *const @TypeOf(c.amdsmi_get_gpu_process_list), + amdsmi_get_gpu_driver_info: *const @TypeOf(c.amdsmi_get_gpu_driver_info), +}; + +pub fn init(allocator: std.mem.Allocator) !AmdSmi { + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const sandbox_path = sandbox.path(&path_buf) orelse return error.AmdSmiUnavailable; + + var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libamd_smi.so.27" }); + + var dynlib: std.DynLib = .{ .inner = .{ + .handle = std.c.dlopen(path, .{ .LAZY = true, .GLOBAL = true, .NODELETE = true }) orelse { + if (std.c.dlerror()) |err| std.log.err("amdsmi: dlopen: {s}", .{err}); + return error.AmdSmiUnavailable; + }, + } }; + const fns = DynLib.lookupStruct(&dynlib, Fns) catch return error.AmdSmiUnavailable; + + try check(fns.amdsmi_init(c.AMDSMI_INIT_AMD_GPUS)); + + var socket_count: u32 = 0; + try check(fns.amdsmi_get_socket_handles(&socket_count, null)); + + const sockets = try allocator.alloc(c.amdsmi_socket_handle, socket_count); + defer allocator.free(sockets); + try check(fns.amdsmi_get_socket_handles(&socket_count, @ptrCast(sockets.ptr))); + + var handle_list: std.ArrayList(c.amdsmi_processor_handle) = .empty; + defer handle_list.deinit(allocator); + + for (sockets[0..socket_count]) |socket| { + var proc_count: u32 = 0; + try check(fns.amdsmi_get_processor_handles(socket, &proc_count, null)); + + const procs = try allocator.alloc(c.amdsmi_processor_handle, proc_count); + defer allocator.free(procs); + + try check(fns.amdsmi_get_processor_handles(socket, &proc_count, @ptrCast(procs.ptr))); + + for (procs[0..proc_count]) |proc| { + try handle_list.append(allocator, proc); + } + } + + return .{ .lib = fns, .gpu_handles = try handle_list.toOwnedSlice(allocator) }; +} + +pub fn handleByIndex(self: AmdSmi, device_id: u32) Error!Handle { + if (device_id >= self.gpu_handles.len) { + return error.not_found; + } + return self.gpu_handles[device_id]; +} + +pub fn deviceCount(self: AmdSmi) u32 { + return @intCast(self.gpu_handles.len); +} + +pub const name_buf_len = c.AMDSMI_MAX_STRING_LENGTH; + +pub fn name(self: AmdSmi, handle: Handle, buf: *[c.AMDSMI_MAX_STRING_LENGTH]u8) Error![:0]const u8 { + var info: c.amdsmi_asic_info_t = undefined; + try check(self.lib.amdsmi_get_gpu_asic_info(handle, &info)); + @memcpy(buf, &info.market_name); + return std.mem.span(@as([*c]const u8, @ptrCast(buf))); +} + +pub fn powerUsage(self: AmdSmi, handle: Handle) Error!u32 { + var metrics: c.amdsmi_gpu_metrics_t = undefined; + try check(self.lib.amdsmi_get_gpu_metrics_info(handle, &metrics)); + const unsupported16 = std.math.maxInt(u16); + if (metrics.current_socket_power != unsupported16) { + return metrics.current_socket_power; + } + if (metrics.average_socket_power != unsupported16) { + return metrics.average_socket_power; + } + return error.not_supported; +} + +pub fn powerLimit(self: AmdSmi, handle: Handle) Error!u32 { + var info: c.amdsmi_power_info_t = undefined; + try check(self.lib.amdsmi_get_power_info(handle, &info)); + return info.power_limit; +} + +pub fn temperature(self: AmdSmi, handle: Handle) Error!i64 { + var temp: i64 = 0; + try check(self.lib.amdsmi_get_temp_metric(handle, c.AMDSMI_TEMPERATURE_TYPE_EDGE, c.AMDSMI_TEMP_CURRENT, &temp)); + return temp; +} + +pub fn fanSpeed(self: AmdSmi, handle: Handle) Error!i64 { + var speed: i64 = 0; + try check(self.lib.amdsmi_get_gpu_fan_speed(handle, @as(u32, 0), &speed)); + return speed; +} + +pub fn gpuUtil(self: AmdSmi, handle: Handle) Error!u32 { + var usage: c.amdsmi_engine_usage_t = undefined; + try check(self.lib.amdsmi_get_gpu_activity(handle, &usage)); + return usage.gfx_activity; +} + +pub fn clockGraphics(self: AmdSmi, handle: Handle) Error!u32 { + var info: c.amdsmi_clk_info_t = undefined; + try check(self.lib.amdsmi_get_clock_info(handle, c.AMDSMI_CLK_TYPE_SYS, &info)); + return info.clk; +} + +pub fn maxClockGraphics(self: AmdSmi, handle: Handle) Error!u32 { + var info: c.amdsmi_clk_info_t = undefined; + try check(self.lib.amdsmi_get_clock_info(handle, c.AMDSMI_CLK_TYPE_SYS, &info)); + return info.max_clk; +} + +pub fn clockMem(self: AmdSmi, handle: Handle) Error!u32 { + var info: c.amdsmi_clk_info_t = undefined; + try check(self.lib.amdsmi_get_clock_info(handle, c.AMDSMI_CLK_TYPE_MEM, &info)); + return info.clk; +} + +pub fn maxClockMem(self: AmdSmi, handle: Handle) Error!u32 { + var info: c.amdsmi_clk_info_t = undefined; + try check(self.lib.amdsmi_get_clock_info(handle, c.AMDSMI_CLK_TYPE_MEM, &info)); + return info.max_clk; +} + +pub fn clockSoc(self: AmdSmi, handle: Handle) Error!u32 { + var info: c.amdsmi_clk_info_t = undefined; + try check(self.lib.amdsmi_get_clock_info(handle, c.AMDSMI_CLK_TYPE_SOC, &info)); + return info.clk; +} + +pub fn memTotal(self: AmdSmi, handle: Handle) Error!u64 { + var total: u64 = 0; + try check(self.lib.amdsmi_get_gpu_memory_total(handle, c.AMDSMI_MEM_TYPE_VRAM, &total)); + return total; +} + +pub fn memUsed(self: AmdSmi, handle: Handle) Error!u64 { + var used: u64 = 0; + try check(self.lib.amdsmi_get_gpu_memory_usage(handle, c.AMDSMI_MEM_TYPE_VRAM, &used)); + return used; +} + +pub fn pcieWidth(self: AmdSmi, handle: Handle) Error!u32 { + var info: c.amdsmi_pcie_info_t = undefined; + try check(self.lib.amdsmi_get_pcie_info(handle, &info)); + return @intCast(info.pcie_metric.pcie_width); +} + +pub fn pcieBandwidth(self: AmdSmi, handle: Handle) Error!u32 { + var info: c.amdsmi_pcie_info_t = undefined; + try check(self.lib.amdsmi_get_pcie_info(handle, &info)); + return info.pcie_metric.pcie_bandwidth; +} + +pub fn pcieLinkGen(self: AmdSmi, handle: Handle) Error!u32 { + var info: c.amdsmi_pcie_info_t = undefined; + try check(self.lib.amdsmi_get_pcie_info(handle, &info)); + return info.pcie_static.pcie_interface_version; +} + +pub fn bdfId(self: AmdSmi, handle: Handle) Error!u64 { + var bdf_id: u64 = 0; + try check(self.lib.amdsmi_get_gpu_bdf_id(handle, &bdf_id)); + return bdf_id; +} + +pub const driver_version_buf_len = @typeInfo(@TypeOf(@as(c.amdsmi_driver_info_t, undefined).driver_version)).array.len; + +pub fn driverVersion(self: AmdSmi, handle: Handle, buf: *[driver_version_buf_len]u8) Error![:0]const u8 { + var info: c.amdsmi_driver_info_t = std.mem.zeroes(c.amdsmi_driver_info_t); + try check(self.lib.amdsmi_get_gpu_driver_info(handle, &info)); + @memcpy(buf, &info.driver_version); + return std.mem.span(@as([*c]const u8, @ptrCast(buf))); +} + +pub fn processList(self: AmdSmi, allocator: std.mem.Allocator, handle: Handle) (Error || error{OutOfMemory})![]const ProcInfo { + var count: u32 = 0; + try check(self.lib.amdsmi_get_gpu_process_list(handle, &count, null)); + if (count == 0) { + return &.{}; + } + const procs = try allocator.alloc(ProcInfo, count); + errdefer allocator.free(procs); + try check(self.lib.amdsmi_get_gpu_process_list(handle, &count, @ptrCast(procs.ptr))); + return procs; +} + +// Error handling + +pub const ReturnError = error{ + inval, + not_supported, + not_yet_implemented, + fail_load_module, + fail_load_symbol, + drm_error, + api_failed, + timeout, + retry, + no_perm, + interrupt, + io_error, + address_fault, + file_error, + out_of_resources, + internal_exception, + input_out_of_bounds, + init_error, + refcount_overflow, + directory_not_found, + busy, + not_found, + not_init, + no_slot, + driver_not_loaded, + no_data, + insufficient_size, + unexpected_size, + unexpected_data, + unknown_error, +}; + +fn check(ret: c.amdsmi_status_t) Error!void { + if (ret == c.AMDSMI_STATUS_SUCCESS) { + return; + } + + return switch (ret) { + c.AMDSMI_STATUS_INVAL => error.inval, + c.AMDSMI_STATUS_NOT_SUPPORTED => error.not_supported, + c.AMDSMI_STATUS_NOT_YET_IMPLEMENTED => error.not_yet_implemented, + c.AMDSMI_STATUS_FAIL_LOAD_MODULE => error.fail_load_module, + c.AMDSMI_STATUS_FAIL_LOAD_SYMBOL => error.fail_load_symbol, + c.AMDSMI_STATUS_DRM_ERROR => error.drm_error, + c.AMDSMI_STATUS_API_FAILED => error.api_failed, + c.AMDSMI_STATUS_TIMEOUT => error.timeout, + c.AMDSMI_STATUS_RETRY => error.retry, + c.AMDSMI_STATUS_NO_PERM => error.no_perm, + c.AMDSMI_STATUS_INTERRUPT => error.interrupt, + c.AMDSMI_STATUS_IO => error.io_error, + c.AMDSMI_STATUS_ADDRESS_FAULT => error.address_fault, + c.AMDSMI_STATUS_FILE_ERROR => error.file_error, + c.AMDSMI_STATUS_OUT_OF_RESOURCES => error.out_of_resources, + c.AMDSMI_STATUS_INTERNAL_EXCEPTION => error.internal_exception, + c.AMDSMI_STATUS_INPUT_OUT_OF_BOUNDS => error.input_out_of_bounds, + c.AMDSMI_STATUS_INIT_ERROR => error.init_error, + c.AMDSMI_STATUS_REFCOUNT_OVERFLOW => error.refcount_overflow, + c.AMDSMI_STATUS_DIRECTORY_NOT_FOUND => error.directory_not_found, + c.AMDSMI_STATUS_BUSY => error.busy, + c.AMDSMI_STATUS_NOT_FOUND => error.not_found, + c.AMDSMI_STATUS_NOT_INIT => error.not_init, + c.AMDSMI_STATUS_NO_SLOT => error.no_slot, + c.AMDSMI_STATUS_DRIVER_NOT_LOADED => error.driver_not_loaded, + c.AMDSMI_STATUS_NO_DATA => error.no_data, + c.AMDSMI_STATUS_INSUFFICIENT_SIZE => error.insufficient_size, + c.AMDSMI_STATUS_UNEXPECTED_SIZE => error.unexpected_size, + c.AMDSMI_STATUS_UNEXPECTED_DATA => error.unexpected_data, + else => error.unknown_error, + }; +} diff --git a/bin/zml-smi/platforms/amdsmi/metrics.zig b/bin/zml-smi/platforms/amdsmi/metrics.zig new file mode 100644 index 00000000..8aa0487c --- /dev/null +++ b/bin/zml-smi/platforms/amdsmi/metrics.zig @@ -0,0 +1,151 @@ +const std = @import("std"); +const AmdSmi = @import("amdsmi.zig"); +const device_info = @import("zml-smi/info").device_info; +const DeviceInfo = device_info.DeviceInfo; +const GpuInfo = device_info.GpuInfo; +const DoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer; +const Collector = @import("zml-smi/collector").Collector; +const poll_metrics = @import("zml-smi/info").poll_metrics; +const process = @import("process.zig"); + +pub fn start(collector: *Collector) !void { + const amdsmi = try collector.arena.create(AmdSmi); + amdsmi.* = try AmdSmi.init(collector.arena); + + const count = amdsmi.deviceCount(); + const dev_offset: u16 = @intCast(collector.device_infos.items.len); + + for (0..count) |i| { + const dev = Device.open(amdsmi, @intCast(i)) catch continue; + const initial: GpuInfo = .{ + .name = dev.name(collector.arena) catch null, + .driver_version = dev.driverVersion(collector.arena) catch null, + }; + const info = try collector.addDevice(.{ .rocm = .{ .values = .{ initial, initial } } }); + try collector.spawnPoll(pollOnce, .{ null, &info.rocm, dev }); + } + + const processes = try collector.createProcessList(); + try process.init(collector, processes, amdsmi, dev_offset); +} + +const pollOnce = poll_metrics.poll(*DoubleBuffer(GpuInfo), Device, metrics); + +const Device = struct { + amdsmi: *const AmdSmi, + handle: AmdSmi.Handle, + + pub fn open(amdsmi: *const AmdSmi, index: u32) !Device { + return .{ .amdsmi = amdsmi, .handle = try amdsmi.handleByIndex(index) }; + } + + fn name(self: Device, arena: std.mem.Allocator) ![]const u8 { + var buf: [AmdSmi.name_buf_len]u8 = undefined; + const slice = try self.amdsmi.name(self.handle, &buf); + + return try arena.dupe(u8, slice); + } + + fn driverVersion(self: Device, arena: std.mem.Allocator) ![]const u8 { + var buf: [AmdSmi.driver_version_buf_len]u8 = undefined; + const slice = try self.amdsmi.driverVersion(self.handle, &buf); + return try arena.dupe(u8, slice); + } + + // Power + pub fn powerUsage(self: Device) !u64 { + const pw = try self.amdsmi.powerUsage(self.handle); + return @as(u64, pw) * 1000; + } + + pub fn powerLimit(self: Device) !u64 { + const limit = try self.amdsmi.powerLimit(self.handle); + if (limit == std.math.maxInt(u32)) { + return error.not_supported; + } + + return @as(u64, limit) / 1000; + } + + // Thermal + pub fn temperature(self: Device) !u64 { + const temp = try self.amdsmi.temperature(self.handle); + return @intCast(temp); + } + + pub fn fanSpeed(self: Device) !u64 { + const speed = try self.amdsmi.fanSpeed(self.handle); + return @intCast(@divTrunc(speed * 100, 255)); + } + + // Utilization + pub fn gpuUtil(self: Device) !u64 { + return try self.amdsmi.gpuUtil(self.handle); + } + + // Clocks + pub fn clockGraphics(self: Device) !u64 { + return try self.amdsmi.clockGraphics(self.handle); + } + + pub fn clockSoc(self: Device) !u64 { + return try self.amdsmi.clockSoc(self.handle); + } + + pub fn clockMem(self: Device) !u64 { + return try self.amdsmi.clockMem(self.handle); + } + + pub fn maxClockGraphics(self: Device) !u64 { + return try self.amdsmi.maxClockGraphics(self.handle); + } + + pub fn maxClockMem(self: Device) !u64 { + return try self.amdsmi.maxClockMem(self.handle); + } + + // Memory + pub fn memUsed(self: Device) !u64 { + return self.amdsmi.memUsed(self.handle); + } + + pub fn memTotal(self: Device) !u64 { + return self.amdsmi.memTotal(self.handle); + } + + // PCIe + pub fn pcieBandwidth(self: Device) !u64 { + const bw = try self.amdsmi.pcieBandwidth(self.handle); + if (bw == std.math.maxInt(u32)) { + return error.not_supported; + } + + return bw; + } + + pub fn pcieLinkGen(self: Device) !u64 { + return try self.amdsmi.pcieLinkGen(self.handle); + } + + pub fn pcieLinkWidth(self: Device) !u64 { + return try self.amdsmi.pcieWidth(self.handle); + } +}; + +const metrics = .{ + .{ .field = "power_mw", .query = Device.powerUsage }, + .{ .field = "power_limit_mw", .query = Device.powerLimit }, + .{ .field = "temperature", .query = Device.temperature }, + .{ .field = "fan_speed_percent", .query = Device.fanSpeed }, + .{ .field = "util_percent", .query = Device.gpuUtil }, + .{ .field = "clock_graphics_mhz", .query = Device.clockGraphics }, + .{ .field = "clock_soc_mhz", .query = Device.clockSoc }, + .{ .field = "clock_mem_mhz", .query = Device.clockMem }, + .{ .field = "clock_graphics_max_mhz", .query = Device.maxClockGraphics }, + .{ .field = "clock_mem_max_mhz", .query = Device.maxClockMem }, + .{ .field = "mem_used_bytes", .query = Device.memUsed }, + .{ .field = "mem_total_bytes", .query = Device.memTotal }, + .{ .field = "pcie_bandwidth_mbps", .query = Device.pcieBandwidth }, + .{ .field = "pcie_link_gen", .query = Device.pcieLinkGen }, + .{ .field = "pcie_link_width", .query = Device.pcieLinkWidth }, +}; diff --git a/bin/zml-smi/platforms/amdsmi/process.zig b/bin/zml-smi/platforms/amdsmi/process.zig new file mode 100644 index 00000000..8d1fdd01 --- /dev/null +++ b/bin/zml-smi/platforms/amdsmi/process.zig @@ -0,0 +1,139 @@ +const std = @import("std"); +const c = std.c; +const AmdSmi = @import("amdsmi.zig"); +const pi = @import("zml-smi/info").process_info; +const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); +const Collector = @import("zml-smi/collector").Collector; + +const bdf_len = "0000:00:00.0".len; + +pub fn init(collector: *Collector, list: *ProcessDoubleBuffer, amdsmi: *const AmdSmi, dev_offset: u16) !void { + const device_count = amdsmi.deviceCount(); + const pci_slots = try collector.arena.alloc(?[bdf_len]u8, device_count); + + for (pci_slots, 0..) |*slot, i| { + const handle = amdsmi.handleByIndex(@intCast(i)) catch { + slot.* = null; + continue; + }; + + slot.* = formatBdf(amdsmi.bdfId(handle) catch { + slot.* = null; + continue; + }); + } + + try collector.spawnPoll(pollOnce, .{ collector.gpa, collector.io, list, amdsmi, dev_offset, device_count, pci_slots }); +} + +fn pollOnce(allocator: std.mem.Allocator, io: std.Io, list: *ProcessDoubleBuffer, amdsmi: *const AmdSmi, dev_offset: u16, device_count: u32, pci_slots: []?[bdf_len]u8) void { + const back = list.back(); + + back.clearRetainingCapacity(); + + for (0..device_count) |dev_idx| { + const handle = amdsmi.handleByIndex(@intCast(dev_idx)) catch continue; + + const procs = blk: { + const saved = c.dup(c.STDERR_FILENO); + const devnull = c.open("/dev/null", .{ .ACCMODE = .WRONLY }); + + if (devnull >= 0) { + if (saved >= 0) { + _ = c.dup2(devnull, c.STDERR_FILENO); + } + + _ = c.close(devnull); + } + + defer if (saved >= 0) { + _ = c.dup2(saved, c.STDERR_FILENO); + _ = c.close(saved); + }; + + break :blk amdsmi.processList(allocator, handle) catch continue; + }; + defer { + if (procs.len > 0) { + allocator.free(procs); + } + } + + const pci_slot = if (dev_idx < pci_slots.len) &(pci_slots[dev_idx] orelse continue) else continue; + + for (procs) |proc| { + back.append(allocator, .{ + .pid = proc.pid, + .device_idx = @intCast(dev_idx + dev_offset), + .dev_util_percent = @intCast(proc.engine_usage.gfx + proc.engine_usage.enc), + .dev_mem_kib = readProcessGpuVram(io, proc.pid, pci_slot), + }) catch break; + } + } + + list.swap(); +} + +/// Reads per-GPU VRAM from /proc//fdinfo/, filtered by PCI slot. +/// amdsmi reports global VRAM across all GPUs; fdinfo gives per-GPU values. +fn readProcessGpuVram(io: std.Io, pid: u32, pci_slot: *const [bdf_len]u8) u64 { + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const dir_path = std.fmt.bufPrint(&path_buf, "/proc/{d}/fdinfo", .{pid}) catch return 0; + + var dir = std.Io.Dir.openDirAbsolute(io, dir_path, .{ .iterate = true }) catch return 0; + defer dir.close(io); + + var total_vram_kib: u64 = 0; + var it = dir.iterate(); + while (it.next(io) catch null) |entry| { + var file_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const file_path = std.fmt.bufPrint(&file_buf, "{s}/{s}", .{ dir_path, entry.name }) catch continue; + + total_vram_kib += parseFdinfoVram(io, file_path, pci_slot); + } + + return total_vram_kib; +} + +fn parseFdinfoVram(io: std.Io, path: []const u8, pci_slot: *const [bdf_len]u8) u64 { + var file = std.Io.Dir.openFile(.cwd(), io, path, .{ .mode = .read_only }) catch return 0; + defer file.close(io); + + var read_buf: [4096]u8 = undefined; + var reader = file.reader(io, &read_buf); + + var found_pci = false; + var vram_kib: ?u64 = null; + + while (true) { + const line = reader.interface.takeDelimiterExclusive('\n') catch break; + reader.interface.toss(1); + + if (std.mem.indexOf(u8, line, pci_slot) != null) found_pci = true; + + // Support both new and old fdinfo keys (see amdgpu_fdinfo.c) + inline for ([_][]const u8{ "drm-memory-vram:\t", "vram mem:\t" }) |key| { + if (std.mem.startsWith(u8, line, key)) { + const rest = line[key.len..]; + const end = std.mem.indexOfAny(u8, rest, " \t\n") orelse rest.len; + vram_kib = std.fmt.parseInt(u64, rest[0..end], 10) catch 0; + } + } + + if (found_pci and vram_kib != null) return vram_kib.?; + } + + return 0; +} + +fn formatBdf(bdf_id: u64) [bdf_len]u8 { + var buf: [bdf_len]u8 = undefined; + _ = std.fmt.bufPrint(&buf, "{x:0>4}:{x:0>2}:{x:0>2}.{x}", .{ + (bdf_id >> 16) & 0xFFFF, + (bdf_id >> 8) & 0xFF, + (bdf_id >> 3) & 0x1F, + bdf_id & 0x7, + }) catch unreachable; + + return buf; +} diff --git a/bin/zml-smi/platforms/linux/BUILD.bazel b/bin/zml-smi/platforms/linux/BUILD.bazel new file mode 100644 index 00000000..57bb96d7 --- /dev/null +++ b/bin/zml-smi/platforms/linux/BUILD.bazel @@ -0,0 +1,17 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "linux", + import_name = "zml-smi/platforms/linux", + main = "linux.zig", + srcs = [ + "metrics.zig", + "process.zig", + ], + visibility = ["//visibility:public"], + deps = [ + "//bin/zml-smi/collector", + "//bin/zml-smi/info", + "//bin/zml-smi/utils:sysfs", + ], +) diff --git a/bin/zml-smi/platforms/linux/linux.zig b/bin/zml-smi/platforms/linux/linux.zig new file mode 100644 index 00000000..ffab3ec4 --- /dev/null +++ b/bin/zml-smi/platforms/linux/linux.zig @@ -0,0 +1,2 @@ +pub const metrics = @import("metrics.zig"); +pub const process = @import("process.zig"); diff --git a/bin/zml-smi/platforms/linux/metrics.zig b/bin/zml-smi/platforms/linux/metrics.zig new file mode 100644 index 00000000..5a69663a --- /dev/null +++ b/bin/zml-smi/platforms/linux/metrics.zig @@ -0,0 +1,97 @@ +const std = @import("std"); +const sysfs = @import("zml-smi/sysfs"); +const host_info = @import("zml-smi/info").host_info; +const HostInfo = host_info.HostInfo; +const HostData = host_info.HostData; +const poll_metrics = @import("zml-smi/info").poll_metrics; +const Collector = @import("zml-smi/collector").Collector; + +pub fn init(collector: *Collector, info: *HostInfo) !void { + const poll_arena = try collector.createPollArena(); + const host: Host = .{ .io = collector.io, .arena = poll_arena }; + + // read once metrics + var initial = info.front().*; + initial.hostname = host.hostname(collector.arena) catch null; + initial.kernel = host.kernel(collector.arena) catch null; + initial.cpu_name = host.cpuName(collector.arena) catch null; + initial.cpu_cores = host.cpuCores() catch null; + initial.mem_total_kib = host.memTotal() catch null; + info.back().* = initial; + info.swap(); + + try collector.spawnPoll(pollOnce, .{ poll_arena, info, host }); +} + +const pollOnce = poll_metrics.poll(*HostInfo, Host, metrics); + +const Host = struct { + io: std.Io, + arena: *std.heap.ArenaAllocator, + + pub fn hostname(self: Host, allocator: std.mem.Allocator) ![]const u8 { + return sysfs.readString(allocator, self.io, "/proc/sys/kernel/hostname"); + } + + pub fn kernel(self: Host, allocator: std.mem.Allocator) ![]const u8 { + return sysfs.readString(allocator, self.io, "/proc/sys/kernel/osrelease"); + } + + pub fn cpuName(self: Host, allocator: std.mem.Allocator) ![]const u8 { + return sysfs.readFieldString(allocator, self.io, "/proc/cpuinfo", "model name"); + } + + pub fn cpuCores(self: Host) !u64 { + const data = try sysfs.readString(self.arena.allocator(), self.io, "/sys/devices/system/cpu/present"); + const dash = std.mem.indexOfScalar(u8, data, '-') orelse return 1; + + return (try std.fmt.parseInt(u64, data[dash + 1 ..], 10)) + 1; + } + + pub fn memTotal(self: Host) !u64 { + return sysfs.readFieldInt(self.arena.allocator(), self.io, "/proc/meminfo", "MemTotal"); + } + + pub fn memAvailable(self: Host) !u64 { + return sysfs.readFieldInt(self.arena.allocator(), self.io, "/proc/meminfo", "MemAvailable"); + } + + fn parseLoadAvg(self: Host, comptime n: usize) !f32 { + const data = try sysfs.readString(self.arena.allocator(), self.io, "/proc/loadavg"); + var iter = std.mem.splitScalar(u8, data, ' '); + + var i: usize = 0; + while (iter.next()) |tok| : (i += 1) { + if (i == n) return std.fmt.parseFloat(f32, tok); + } + + return error.NotFound; + } + + pub fn load1(self: Host) !f32 { + return self.parseLoadAvg(0); + } + + pub fn load5(self: Host) !f32 { + return self.parseLoadAvg(1); + } + + pub fn load15(self: Host) !f32 { + return self.parseLoadAvg(2); + } + + pub fn uptime(self: Host) !u64 { + const data = try sysfs.readString(self.arena.allocator(), self.io, "/proc/uptime"); + const dot = std.mem.indexOfScalar(u8, data, '.') orelse data.len; + + return std.fmt.parseInt(u64, data[0..dot], 10); + } +}; + +const metrics = .{ + .{ .field = "mem_available_kib", .query = Host.memAvailable }, + .{ .field = "load_1", .query = Host.load1 }, + .{ .field = "load_5", .query = Host.load5 }, + .{ .field = "load_15", .query = Host.load15 }, + .{ .field = "uptime_seconds", .query = Host.uptime }, +}; diff --git a/bin/zml-smi/platforms/linux/process.zig b/bin/zml-smi/platforms/linux/process.zig new file mode 100644 index 00000000..c57d1725 --- /dev/null +++ b/bin/zml-smi/platforms/linux/process.zig @@ -0,0 +1,118 @@ +const std = @import("std"); +const sysfs = @import("zml-smi/sysfs"); +const pi = @import("zml-smi/info").process_info; + +const ProcessInfo = pi.ProcessInfo; + +pub const ProcessEnricher = struct { + gpa: std.mem.Allocator, + arena: std.heap.ArenaAllocator, + prev_ticks: std.AutoHashMapUnmanaged(u32, u64) = .{}, + prev_total_ticks: u64 = 0, + + pub fn init(gpa: std.mem.Allocator, io: std.Io) !ProcessEnricher { + var arena = std.heap.ArenaAllocator.init(gpa); + errdefer arena.deinit(); + + const prev_total_ticks = readTotalCpuTicks(arena.allocator(), io); + var enricher: ProcessEnricher = .{ + .gpa = gpa, + .arena = arena, + .prev_total_ticks = prev_total_ticks, + .prev_ticks = .{}, + }; + + try enricher.prev_ticks.ensureTotalCapacity(gpa, 64); + + return enricher; + } + + pub fn deinit(self: *ProcessEnricher) void { + self.prev_ticks.deinit(self.gpa); + self.arena.deinit(); + } + + pub fn enrich(self: *ProcessEnricher, io: std.Io, procs: []ProcessInfo) void { + _ = self.arena.reset(.retain_capacity); + const alloc = self.arena.allocator(); + + const curr_total = readTotalCpuTicks(alloc, io); + const delta_total = curr_total -| self.prev_total_ticks; + + var curr_ticks: std.AutoHashMapUnmanaged(u32, u64) = .{}; + curr_ticks.ensureTotalCapacity(self.gpa, @intCast(procs.len)) catch return; + + for (procs) |*info| { + if (info.remote) continue; + // Read Uid, VmRSS from /proc/{pid}/status + const status_path = std.fmt.allocPrint(alloc, "/proc/{d}/status", .{info.pid}) catch continue; + const uid: u32 = @intCast(sysfs.readFieldInt(alloc, io, status_path, "Uid") catch 0); + info.uid = uid; + info.username = lookupUsername(alloc, uid) catch continue; + info.rss_kib = sysfs.readFieldInt(alloc, io, status_path, "VmRSS") catch 0; + + // Read cmdline + const cmdline_path = std.fmt.allocPrint(alloc, "/proc/{d}/cmdline", .{info.pid}) catch continue; + info.comm = readCmdline(alloc, io, cmdline_path) catch continue; + + // CPU% delta + const ticks = readProcTicks(alloc, io, info.pid) orelse continue; + if (delta_total > 0) { + if (self.prev_ticks.get(info.pid)) |prev| { + const delta_proc = ticks -| prev; + info.cpu_percent = @intCast(@min(delta_proc *| 1000 / delta_total, std.math.maxInt(u16))); + } + } + + curr_ticks.getOrPutAssumeCapacity(info.pid).value_ptr.* = ticks; + } + + self.prev_ticks.deinit(self.gpa); + self.prev_ticks = curr_ticks; + self.prev_total_ticks = curr_total; + } +}; + +fn lookupUsername(allocator: std.mem.Allocator, uid: u32) ![]const u8 { + const pw = std.c.getpwuid(uid) orelse return error.NoUser; + const name = std.mem.span(pw.name orelse return error.NoName); + + return try allocator.dupe(u8, name); +} + +/// Extract utime + stime from /proc/{pid}/stat. +fn readProcTicks(allocator: std.mem.Allocator, io: std.Io, pid: u32) ?u64 { + const path = std.fmt.allocPrint(allocator, "/proc/{d}/stat", .{pid}) catch return null; + const data = sysfs.readFirstLine(allocator, io, path) catch return null; + + // Fields after ") ": state=0, ppid=1, ..., utime=11, stime=12 + const rest = data[(std.mem.lastIndexOfScalar(u8, data, ')') orelse return null) + 2 ..]; + + return sysfs.nthTokenInt(rest, 11) + sysfs.nthTokenInt(rest, 12); +} + +fn readCmdline(allocator: std.mem.Allocator, io: std.Io, path: []const u8) ![]const u8 { + const data = try sysfs.readFirstLine(allocator, io, path); + + for (data) |*c| { + if (c.* == 0) { + c.* = ' '; + } + } + + return std.mem.trimEnd(u8, data, &std.ascii.whitespace); +} + +fn readTotalCpuTicks(allocator: std.mem.Allocator, io: std.Io) u64 { + const data = sysfs.readFirstLine(allocator, io, "/proc/stat") catch return 0; + + var total: u64 = 0; + var iter = std.mem.tokenizeAny(u8, data, " \t"); + + _ = iter.next(); // skip "cpu" + while (iter.next()) |tok| { + total += std.fmt.parseInt(u64, tok, 10) catch 0; + } + + return total; +} diff --git a/bin/zml-smi/platforms/macos/BUILD.bazel b/bin/zml-smi/platforms/macos/BUILD.bazel new file mode 100644 index 00000000..a92e298e --- /dev/null +++ b/bin/zml-smi/platforms/macos/BUILD.bazel @@ -0,0 +1,16 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "macos", + import_name = "zml-smi/platforms/macos", + main = "macos.zig", + srcs = [ + "metrics.zig", + "process.zig", + ], + visibility = ["//visibility:public"], + deps = [ + "//bin/zml-smi/collector", + "//bin/zml-smi/info", + ], +) diff --git a/bin/zml-smi/platforms/macos/macos.zig b/bin/zml-smi/platforms/macos/macos.zig new file mode 100644 index 00000000..ffab3ec4 --- /dev/null +++ b/bin/zml-smi/platforms/macos/macos.zig @@ -0,0 +1,2 @@ +pub const metrics = @import("metrics.zig"); +pub const process = @import("process.zig"); diff --git a/bin/zml-smi/platforms/macos/metrics.zig b/bin/zml-smi/platforms/macos/metrics.zig new file mode 100644 index 00000000..616168cd --- /dev/null +++ b/bin/zml-smi/platforms/macos/metrics.zig @@ -0,0 +1,161 @@ +const std = @import("std"); +const host_info = @import("zml-smi/info").host_info; +const HostInfo = host_info.HostInfo; +const poll_metrics = @import("zml-smi/info").poll_metrics; +const Collector = @import("zml-smi/collector").Collector; + +const c = std.c; + +const vm_statistics64 = extern struct { + free_count: c_uint, + active_count: c_uint, + inactive_count: c_uint, + wire_count: c_uint, + zero_fill_count: u64, + reactivations: u64, + pageins: u64, + pageouts: u64, + faults: u64, + cow_faults: u64, + lookups: u64, + hits: u64, + purges: u64, + purgeable_count: c_uint, + speculative_count: c_uint, + decompressions: u64, + compressions: u64, + swapins: u64, + swapouts: u64, + compressor_page_count: c_uint, + throttled_count: c_uint, + external_page_count: c_uint, + internal_page_count: c_uint, + total_uncompressed_pages_in_compressor: u64, +}; + +const HOST_VM_INFO64: c_int = 4; +const HOST_VM_INFO64_COUNT: c_uint = @sizeOf(vm_statistics64) / @sizeOf(c_int); +const KERN_SUCCESS: c_int = 0; + +extern "c" fn host_statistics64(host: c.mach_port_t, flavor: c_int, info: *anyopaque, count: *c_uint) c.kern_return_t; +extern "c" fn getloadavg(loadavg: *[3]f64, nelem: c_int) c_int; + +pub fn init(collector: *Collector, info: *HostInfo) !void { + const poll_arena = try collector.createPollArena(); + const host: Host = .{}; + + var initial = info.front().*; + initial.hostname = sysctlString(collector.arena, "kern.hostname") catch null; + initial.kernel = sysctlString(collector.arena, "kern.osrelease") catch null; + initial.cpu_name = sysctlString(collector.arena, "machdep.cpu.brand_string") catch null; + initial.cpu_cores = host.cpuCores() catch null; + initial.mem_total_kib = host.memTotal() catch null; + info.back().* = initial; + info.swap(); + + try collector.spawnPoll(pollOnce, .{ poll_arena, info, host }); +} + +const pollOnce = poll_metrics.poll(*HostInfo, Host, metrics); + +const Host = struct { + pub fn cpuCores(_: Host) !u64 { + return @intCast(try sysctlU32("hw.logicalcpu")); + } + + pub fn memTotal(_: Host) !u64 { + return (try sysctlU64("hw.memsize")) / 1024; + } + + pub fn memAvailable(_: Host) !u64 { + var vm_stat: vm_statistics64 = undefined; + var count: c_uint = HOST_VM_INFO64_COUNT; + + if (host_statistics64(c.mach_host_self(), HOST_VM_INFO64, &vm_stat, &count) != KERN_SUCCESS) { + return error.HostStatsFailed; + } + + var page_size: u64 = 0; + if (c._host_page_size(c.mach_host_self(), &page_size) != KERN_SUCCESS) { + return error.PageSizeFailed; + } + + const available_pages: u64 = @as(u64, vm_stat.free_count) + + @as(u64, vm_stat.inactive_count) + + @as(u64, vm_stat.purgeable_count); + + return (available_pages * page_size) / 1024; + } + + fn loadAvg() ![3]f64 { + var avg: [3]f64 = undefined; + if (getloadavg(&avg, 3) != 3) { + return error.LoadAvgFailed; + } + + return avg; + } + + pub fn load1(_: Host) !f32 { + return @floatCast((try loadAvg())[0]); + } + + pub fn load5(_: Host) !f32 { + return @floatCast((try loadAvg())[1]); + } + + pub fn load15(_: Host) !f32 { + return @floatCast((try loadAvg())[2]); + } + + pub fn uptime(_: Host) !u64 { + var boottime: c.timeval = undefined; + var len: usize = @sizeOf(c.timeval); + if (c.sysctlbyname("kern.boottime", std.mem.asBytes(&boottime), &len, null, 0) != 0) { + return error.SysctlFailed; + } + + var now: c.timeval = undefined; + _ = c.gettimeofday(&now, null); + + return @intCast(now.sec - boottime.sec); + } +}; + +fn sysctlString(allocator: std.mem.Allocator, name: [*:0]const u8) ![]const u8 { + var buf: [256]u8 = undefined; + var len: usize = buf.len; + if (c.sysctlbyname(name, &buf, &len, null, 0) != 0) { + return error.SysctlFailed; + } + + return allocator.dupe(u8, std.mem.span(@as([*:0]const u8, @ptrCast(buf[0..len])))); +} + +fn sysctlU64(name: [*:0]const u8) !u64 { + var value: u64 = 0; + var len: usize = @sizeOf(u64); + if (c.sysctlbyname(name, std.mem.asBytes(&value), &len, null, 0) != 0) { + return error.SysctlFailed; + } + + return value; +} + +fn sysctlU32(name: [*:0]const u8) !u32 { + var value: u32 = 0; + var len: usize = @sizeOf(u32); + if (c.sysctlbyname(name, std.mem.asBytes(&value), &len, null, 0) != 0) { + return error.SysctlFailed; + } + + return value; +} + +const metrics = .{ + .{ .field = "mem_available_kib", .query = Host.memAvailable }, + .{ .field = "load_1", .query = Host.load1 }, + .{ .field = "load_5", .query = Host.load5 }, + .{ .field = "load_15", .query = Host.load15 }, + .{ .field = "uptime_seconds", .query = Host.uptime }, +}; diff --git a/bin/zml-smi/platforms/macos/process.zig b/bin/zml-smi/platforms/macos/process.zig new file mode 100644 index 00000000..a48e9831 --- /dev/null +++ b/bin/zml-smi/platforms/macos/process.zig @@ -0,0 +1,12 @@ +const std = @import("std"); +const pi = @import("zml-smi/info").process_info; + +pub const ProcessEnricher = struct { + pub fn init(_: std.mem.Allocator, _: std.Io) !ProcessEnricher { + return .{}; + } + + pub fn deinit(_: *ProcessEnricher) void {} + + pub fn enrich(_: *ProcessEnricher, _: std.Io, _: []pi.ProcessInfo) void {} +}; diff --git a/bin/zml-smi/platforms/neuron/BUILD.bazel b/bin/zml-smi/platforms/neuron/BUILD.bazel new file mode 100644 index 00000000..c7129ac7 --- /dev/null +++ b/bin/zml-smi/platforms/neuron/BUILD.bazel @@ -0,0 +1,22 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "neuron", + import_name = "zml-smi/platforms/neuron", + main = "metrics.zig", + srcs = [ + "nrt.zig", + "process.zig", + ], + visibility = ["//visibility:public"], + deps = [ + "//bin/zml-smi/collector", + "//bin/zml-smi/info", + "//bin/zml-smi/utils:double_buffer", + "//bin/zml-smi/utils:dynlib", + "//bin/zml-smi/utils:sandbox", + "//bin/zml-smi/utils:sysfs", + "//platforms/neuron:libnrt_headers", + "//stdx", + ], +) diff --git a/bin/zml-smi/platforms/neuron/metrics.zig b/bin/zml-smi/platforms/neuron/metrics.zig new file mode 100644 index 00000000..0b5a5ddd --- /dev/null +++ b/bin/zml-smi/platforms/neuron/metrics.zig @@ -0,0 +1,201 @@ +const std = @import("std"); +const sysfs = @import("zml-smi/sysfs"); +const DoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer; +const device_info = @import("zml-smi/info").device_info; +const DeviceInfo = device_info.DeviceInfo; +const NeuronInfo = device_info.NeuronInfo; +const Collector = @import("zml-smi/collector").Collector; +const poll_metrics = @import("zml-smi/info").poll_metrics; +const Nrt = @import("nrt.zig"); +const process = @import("process.zig"); + +const base_path = "/sys/devices/virtual/neuron_device"; + +pub const UtilAtomic = std.atomic.Value(u64); + +pub fn start(collector: *Collector) !void { + const nrt = try collector.arena.create(Nrt); + nrt.* = try Nrt.init(collector.arena, collector.io); + + if (nrt.handles.len == 0) { + return; + } + + const total_nc = nrt.totalNcCount() catch return; + const nc_per_device = total_nc / @as(u32, @intCast(nrt.handles.len)); + + const total_cores = @as(usize, nrt.handles.len) * nc_per_device; + const util_buf = try collector.arena.alloc(UtilAtomic, total_cores); + for (util_buf) |*u| u.* = .init(0); + + const GiB: u64 = 1024 * 1024 * 1024; + var neuron_infos: std.ArrayList(*DeviceInfo) = .empty; + const dev_offset: u8 = @intCast(collector.device_infos.items.len); + + var ver_buf: [Nrt.version_buf_len]u8 = undefined; + const nrt_ver: ?[]const u8 = if (nrt.version(&ver_buf)) |v| + collector.arena.dupe(u8, v) catch null + else |_| + null; + + var flat_core: usize = 0; + for (nrt.handles, nrt.device_indexes) |dev_handle, device_idx| { + const device_type = Nrt.deviceType(dev_handle); + const mem_per_device: u64 = switch (device_type) { + .inf1 => 8 * GiB, + .inf2_trn1 => 32 * GiB, + .trn2 => 96 * GiB, + .trn3 => 144 * GiB, + else => 0, + }; + + for (0..nc_per_device) |ci| { + const poll_arena = try collector.createPollArena(); + + const dev: Device = .{ + .io = collector.io, + .arena = poll_arena, + .device_idx = @intCast(device_idx), + .core_idx = @intCast(ci), + .mem_per_device = mem_per_device, + .nc_per_device = nc_per_device, + .util_src = &util_buf[flat_core], + }; + + const initial: NeuronInfo = .{ + .name = dev.name(collector.arena) catch null, + .driver_version = nrt_ver, + }; + const info = try collector.addDevice(.{ .neuron = .{ .values = .{ initial, initial } } }); + try neuron_infos.append(collector.arena, info); + try collector.spawnPoll(pollOnce, .{ poll_arena, &info.neuron, dev }); + flat_core += 1; + } + } + + if (neuron_infos.items.len > 0) { + const processes = try collector.createProcessList(); + const warmup = collector.poll_only; + try process.init(collector, processes, nrt, nc_per_device, util_buf, dev_offset, warmup); + + if (warmup) { + for (neuron_infos.items, util_buf) |info, *u| { + const back = info.neuron.back(); + back.* = info.neuron.front().*; + back.util_percent = u.load(.acquire); + info.neuron.swap(); + } + } + } +} + +const pollOnce = poll_metrics.poll(*DoubleBuffer(NeuronInfo), Device, metrics); + +const Device = struct { + io: std.Io, + arena: *std.heap.ArenaAllocator, + device_idx: u32, + core_idx: u32, + mem_per_device: u64 = 0, + nc_per_device: u32 = 1, + util_src: *const UtilAtomic, + + pub fn utilPercent(self: Device) !u64 { + return self.util_src.load(.acquire); + } + + fn devicePath(self: Device, sub_path: []const u8) ![]const u8 { + return std.fmt.allocPrint(self.arena.allocator(), base_path ++ "/neuron{d}/{s}", .{ self.device_idx, sub_path }); + } + + fn corePath(self: Device, sub_path: []const u8) ![]const u8 { + return std.fmt.allocPrint(self.arena.allocator(), base_path ++ "/neuron{d}/neuron_core{d}/{s}", .{ self.device_idx, self.core_idx, sub_path }); + } + + fn name(self: Device, allocator: std.mem.Allocator) ![]const u8 { + return sysfs.readString(allocator, self.io, try self.devicePath("info/architecture/device_name")); + } + + fn readCoreMem(self: Device, comptime subdir: []const u8) !u64 { + return sysfs.readInt(self.arena.allocator(), self.io, try self.corePath("stats/memory_usage/device_mem/" ++ subdir ++ "/total")); + } + + pub fn memTotal(self: Device) !u64 { + if (self.mem_per_device == 0) { + return error.NrtUnavailable; + } + return self.mem_per_device; + } + + // Sysfs-based metrics — RAM is shared across cores, so sum all cores on the device. + pub fn memUsed(self: Device) !u64 { + var total: u64 = 0; + for (0..self.nc_per_device) |ci| { + const path = try std.fmt.allocPrint(self.arena.allocator(), base_path ++ "/neuron{d}/neuron_core{d}/stats/memory_usage/device_mem/total", .{ self.device_idx, ci }); + total += try sysfs.readInt(self.arena.allocator(), self.io, path); + } + + return total; + } + + pub fn tensors(self: Device) !u64 { + return self.readCoreMem("tensors"); + } + + pub fn constants(self: Device) !u64 { + return self.readCoreMem("constants"); + } + + pub fn modelCode(self: Device) !u64 { + return self.readCoreMem("model_code"); + } + + pub fn sharedScratchpad(self: Device) !u64 { + return self.readCoreMem("model_shared_scratchpad"); + } + + pub fn nonsharedScratchpad(self: Device) !u64 { + return self.readCoreMem("nonshared_scratchpad"); + } + + pub fn runtime(self: Device) !u64 { + return self.readCoreMem("runtime_memory"); + } + + pub fn driver(self: Device) !u64 { + return self.readCoreMem("driver_memory"); + } + + pub fn dmaRings(self: Device) !u64 { + return self.readCoreMem("dma_rings"); + } + + pub fn collectives(self: Device) !u64 { + return self.readCoreMem("collectives"); + } + + pub fn notifications(self: Device) !u64 { + return self.readCoreMem("notifications"); + } + + pub fn uncategorized(self: Device) !u64 { + return self.readCoreMem("uncategorized"); + } +}; + +const metrics = .{ + .{ .field = "util_percent", .query = Device.utilPercent }, + .{ .field = "mem_total_bytes", .query = Device.memTotal }, + .{ .field = "mem_used_bytes", .query = Device.memUsed }, + .{ .field = "nc_tensors", .query = Device.tensors }, + .{ .field = "nc_constants", .query = Device.constants }, + .{ .field = "nc_model_code", .query = Device.modelCode }, + .{ .field = "nc_shared_scratchpad", .query = Device.sharedScratchpad }, + .{ .field = "nc_nonshared_scratchpad", .query = Device.nonsharedScratchpad }, + .{ .field = "nc_runtime", .query = Device.runtime }, + .{ .field = "nc_driver", .query = Device.driver }, + .{ .field = "nc_dma_rings", .query = Device.dmaRings }, + .{ .field = "nc_collectives", .query = Device.collectives }, + .{ .field = "nc_notifications", .query = Device.notifications }, + .{ .field = "nc_uncategorized", .query = Device.uncategorized }, +}; diff --git a/bin/zml-smi/platforms/neuron/nrt.zig b/bin/zml-smi/platforms/neuron/nrt.zig new file mode 100644 index 00000000..cd77b3c0 --- /dev/null +++ b/bin/zml-smi/platforms/neuron/nrt.zig @@ -0,0 +1,230 @@ +const std = @import("std"); +const c = @import("c"); +const elf = std.elf; +const stdx = @import("stdx"); +const DynLib = @import("zml-smi/dynlib"); +const sandbox = @import("zml-smi/sandbox"); + +const Nrt = @This(); + +pub const Error = error{ nrt_error, NrtUnavailable }; +pub const AppInfo = c.struct_neuron_app_info; +pub const DeviceType = enum(c_int) { + inf1 = 1, + inf2_trn1 = 2, + trn2 = 3, + trn3 = 4, + _, +}; + +lib: Fns, +private_lib: PrivateFns, +handles: []const *c.ndl_device_t, +device_indexes: []const c_int, + +const Fns = struct { + nrt_get_total_nc_count: *const @TypeOf(c.nrt_get_total_nc_count), + nrt_get_version: *const @TypeOf(c.nrt_get_version), +}; + +const PrivateFns = struct { + nds_open: *const @TypeOf(c.nds_open), + ndl_available_devices: *const @TypeOf(c.ndl_available_devices), + ndl_open_device: *const @TypeOf(c.ndl_open_device), + ndl_close_device: *const @TypeOf(c.ndl_close_device), + ndl_get_all_apps_info: *const @TypeOf(c.ndl_get_all_apps_info), + nds_close: *const @TypeOf(c.nds_close), + nds_get_nc_counter: *const @TypeOf(c.nds_get_nc_counter), +}; + +pub fn init(allocator: std.mem.Allocator, io: std.Io) !Nrt { + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const sandbox_path = sandbox.path(&path_buf) orelse { + std.log.err("neuron: sandbox path unavailable", .{}); + return error.NrtUnavailable; + }; + + var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const path = stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libnrt.so.1" }) catch { + std.log.err("neuron: failed to construct libnrt.so.1 path", .{}); + return error.NrtUnavailable; + }; + + const dl_handle = std.c.dlopen(path, .{ .LAZY = true, .GLOBAL = true, .NODELETE = true }) orelse { + if (std.c.dlerror()) |err| std.log.err("neuron: dlopen: {s}", .{err}); + return error.NrtUnavailable; + }; + var dynlib: std.DynLib = .{ .inner = .{ .handle = dl_handle } }; + + const fns = DynLib.lookupStruct(&dynlib, Fns) catch return error.NrtUnavailable; + const private_fns = resolveFromElf(dl_handle, io, "nrt_init") catch |err| { + std.log.err("neuron: ELF resolution failed: {s}", .{@errorName(err)}); + return error.NrtUnavailable; + }; + + var dev_index_buf: [c.MAX_NEURON_DEVICE_COUNT]c_int = undefined; + const count: usize = @intCast(@max(0, private_fns.ndl_available_devices(&dev_index_buf, c.MAX_NEURON_DEVICE_COUNT))); + + var handle_list: std.ArrayList(*c.ndl_device_t) = .empty; + var index_list: std.ArrayList(c_int) = .empty; + + for (dev_index_buf[0..count]) |device_idx| { + var dev: ?*c.ndl_device_t = null; + var t: c.struct_ndl_device_init_param = .{ + .initialize_device = false, + .num_dram_regions = 0, + .map_hbm = false, + .skip_copy_bufs = true, + }; + + if (private_fns.ndl_open_device(device_idx, &t, &dev) == 0) { + if (dev) |d| { + errdefer _ = private_fns.ndl_close_device(d); + try handle_list.append(allocator, d); + try index_list.append(allocator, device_idx); + } + } + } + + return .{ + .lib = fns, + .private_lib = private_fns, + .handles = try handle_list.toOwnedSlice(allocator), + .device_indexes = try index_list.toOwnedSlice(allocator), + }; +} + +pub fn deviceCount(self: Nrt) u32 { + return @intCast(self.handles.len); +} + +pub fn allAppsInfo(self: Nrt, dev: *c.ndl_device_t) Error!struct { ptr: ?[*]AppInfo, count: usize } { + var info: ?[*]AppInfo = null; + var count: usize = 0; + if (self.private_lib.ndl_get_all_apps_info(dev, &info, &count, c.APP_INFO_ALL) != 0) { + return error.nrt_error; + } + return .{ .ptr = info, .count = count }; +} + +pub fn ndsOpen(self: Nrt, dev: *c.ndl_device_t, pid: c.pid_t) Error!*c.nds_instance_t { + var inst: ?*c.nds_instance_t = null; + if (self.private_lib.nds_open(dev, pid, &inst) != 0) { + return error.nrt_error; + } + return inst orelse error.nrt_error; +} + +pub fn ndsClose(self: Nrt, inst: *c.nds_instance_t) void { + _ = self.private_lib.nds_close(inst); +} + +pub fn ncCounter(self: Nrt, inst: *c.nds_instance_t, pnc_index: c_int, counter_index: u32) Error!u64 { + var value: u64 = 0; + if (self.private_lib.nds_get_nc_counter(inst, pnc_index, counter_index, &value) != 0) { + return error.nrt_error; + } + return value; +} + +pub const version_buf_len = c.RT_VERSION_DETAIL_LEN; + +pub fn version(self: Nrt, buf: *[version_buf_len]u8) Error![:0]const u8 { + var ver: c.nrt_version_t = std.mem.zeroes(c.nrt_version_t); + if (self.lib.nrt_get_version(&ver, @sizeOf(c.nrt_version_t)) != 0) { + return error.nrt_error; + } + @memcpy(buf, &ver.rt_detail); + return std.mem.span(@as([*c]const u8, @ptrCast(buf))); +} + +pub fn totalNcCount(self: Nrt) Error!u32 { + var count: u32 = 0; + if (self.lib.nrt_get_total_nc_count(&count) != 0) { + return error.nrt_error; + } + return count; +} + +pub fn hbmSize(dev: *c.ndl_device_t) usize { + return dev.hbm_size; +} + +pub fn deviceType(dev: *c.ndl_device_t) DeviceType { + return @fromBackingInt(@intCast(dev.device_type)); +} + +fn resolveFromElf(handle: *anyopaque, io: std.Io, comptime known_sym: [:0]const u8) !PrivateFns { + const field_names = comptime std.meta.fieldNames(PrivateFns); + + const sym_addr = std.c.dlsym(handle, known_sym) orelse + return error.SymbolNotFound; + + var dl_info: DlInfo = undefined; + if (dladdr(sym_addr, &dl_info) == 0) return error.DladdrFailed; + + const elf_base = @intFromPtr(dl_info.dli_fbase); + const elf_path = std.mem.span(dl_info.dli_fname); + + const file = try std.Io.Dir.openFileAbsolute(io, elf_path, .{}); + defer file.close(io); + + const stat = try file.stat(io); + const mmap = try std.posix.mmap( + null, + @intCast(stat.size), + .{ .READ = true }, + .{ .TYPE = .PRIVATE }, + file.handle, + 0, + ); + defer std.posix.munmap(mmap); + + var result: PrivateFns = undefined; + var remaining: usize = field_names.len; + + const ehdr: *const elf.ElfN.Ehdr = @ptrCast(@alignCast(mmap)); + const shdrs: []const elf.ElfN.Shdr = @as([*]const elf.ElfN.Shdr, @ptrCast(@alignCast(mmap[ehdr.shoff..])))[0..ehdr.shnum]; + + for (shdrs) |sh| { + if (sh.type != .SYMTAB) { + continue; + } + + const strtab_sh = shdrs[sh.link]; + const strtab: []const u8 = mmap[strtab_sh.offset..][0..strtab_sh.size]; + + for (std.mem.bytesAsSlice(elf.ElfN.Sym, mmap[sh.offset..][0..sh.size])) |s| { + if (s.value == 0) { + continue; + } + + const sym_name = std.mem.span(@as([*c]const u8, @ptrCast(strtab[@as(usize, s.name)..]))); + + inline for (field_names) |field_name| { + if (std.mem.eql(u8, sym_name, field_name)) { + @field(result, field_name) = @ptrFromInt(elf_base + @as(usize, s.value)); + remaining -= 1; + } + } + + if (remaining == 0) { + return result; + } + } + } + + if (remaining > 0) { + return error.SymbolResolutionFailed; + } + return result; +} + +const DlInfo = extern struct { + dli_fname: [*:0]const u8, + dli_fbase: *anyopaque, + dli_sname: [*c]const u8, + dli_saddr: ?*anyopaque, +}; + +extern fn dladdr(addr: *anyopaque, info: *DlInfo) c_int; diff --git a/bin/zml-smi/platforms/neuron/process.zig b/bin/zml-smi/platforms/neuron/process.zig new file mode 100644 index 00000000..2dbf363c --- /dev/null +++ b/bin/zml-smi/platforms/neuron/process.zig @@ -0,0 +1,108 @@ +const std = @import("std"); +const c = @import("c"); +const Nrt = @import("nrt.zig"); +const smi_info = @import("zml-smi/info"); +const pi = smi_info.process_info; +const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); +const Collector = @import("zml-smi/collector").Collector; +const UtilAtomic = @import("metrics.zig").UtilAtomic; + +const DeltaState = struct { + prev_total_us: u64 = 0, + prev_timestamp: ?std.Io.Timestamp = null, +}; + +pub fn init(collector: *Collector, list: *ProcessDoubleBuffer, nrt: *const Nrt, nc_per_device: u32, util_buf: []UtilAtomic, dev_offset: u16, warmup: bool) !void { + if (nrt.handles.len == 0) return; + const delta_states = try collector.arena.alloc(DeltaState, util_buf.len); + @memset(delta_states, .{}); + const args: std.meta.ArgsTuple(@TypeOf(pollOnce)) = .{ collector.gpa, collector.io, list, nrt, nc_per_device, util_buf, dev_offset, delta_states }; + if (warmup) { + @call(.auto, pollOnce, args); + collector.io.sleep(.fromMilliseconds(500), .awake) catch {}; + @call(.auto, pollOnce, args); + collector.io.sleep(.fromMilliseconds(500), .awake) catch {}; + } + + try collector.spawnPoll(pollOnce, args); +} + +fn pollOnce(allocator: std.mem.Allocator, io: std.Io, list: *ProcessDoubleBuffer, nrt: *const Nrt, nc_per_device: u32, util_buf: []UtilAtomic, dev_offset: u16, delta_states: []DeltaState) void { + const back = list.back(); + + back.clearRetainingCapacity(); + + for (nrt.handles, 0..) |handle, dev_i| { + const apps_result = nrt.allAppsInfo(handle) catch continue; + defer { + if (apps_result.ptr) |ptr| { + std.c.free(@ptrCast(ptr)); + } + } + + const apps = if (apps_result.ptr) |ptr| ptr[0..apps_result.count] else continue; + + var total_us_per_core: [c.MAX_NC_PER_DEVICE]u64 = @splat(0); + + for (apps) |*app| { + if (app.pid <= 0) { + continue; + } + + const nds = nrt.ndsOpen(handle, app.pid) catch continue; + defer nrt.ndsClose(nds); + + for (0..nc_per_device) |ci| { + const time_in_use = nrt.ncCounter(nds, @intCast(ci), c.NDS_NC_COUNTER_TIME_IN_USE) catch continue; + total_us_per_core[ci] += time_in_use; + + if (time_in_use == 0) { + continue; + } + + const dev_idx: u32 = @intCast(dev_i * nc_per_device + ci); + + const pid = std.math.cast(u32, app.pid) orelse continue; + var info: pi.ProcessInfo = .{ + .pid = pid, + .device_idx = @intCast(dev_idx + dev_offset), + .dev_mem_kib = @intCast(app.device_mem_size / 1024), + }; + + if (dev_idx < util_buf.len) { + info.dev_util_percent = @intCast(util_buf[dev_idx].load(.acquire)); + } + + if (info.dev_mem_kib != null and info.dev_mem_kib.? > 0) + back.append(allocator, info) catch break; + } + } + + const now: std.Io.Timestamp = .now(io, .awake); + + for (0..nc_per_device) |ci| { + const dev_idx = dev_i * nc_per_device + ci; + if (dev_idx >= delta_states.len) { + break; + } + + const ds = &delta_states[dev_idx]; + const prev_us = ds.prev_total_us; + ds.prev_total_us = total_us_per_core[ci]; + + if (ds.prev_timestamp) |prev| { + const elapsed = prev.untilNow(io, .awake); + const elapsed_us: u64 = @intCast(@divFloor(elapsed.nanoseconds, std.time.ns_per_us)); + + if (elapsed_us > 0) { + const delta_us = total_us_per_core[ci] -| prev_us; + util_buf[dev_idx].store(@min(100, delta_us * 100 / elapsed_us), .release); + } + } + + ds.prev_timestamp = now; + } + } + + list.swap(); +} diff --git a/bin/zml-smi/platforms/nvml/BUILD.bazel b/bin/zml-smi/platforms/nvml/BUILD.bazel new file mode 100644 index 00000000..0ab0fae1 --- /dev/null +++ b/bin/zml-smi/platforms/nvml/BUILD.bazel @@ -0,0 +1,22 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "nvml", + srcs = [ + "nvml.zig", + "process.zig", + ], + import_name = "zml-smi/platforms/nvml", + main = "metrics.zig", + target_compatible_with = ["@platforms//os:linux"], + visibility = ["//visibility:public"], + deps = [ + "//bin/zml-smi/collector", + "//bin/zml-smi/info", + "//bin/zml-smi/utils:double_buffer", + "//bin/zml-smi/utils:dynlib", + ] + select({ + "@llvm//platforms/config:linux_x86_64": ["@cuda_nvml_dev_linux_x86_64//:nvml"], + "@llvm//platforms/config:linux_aarch64": ["@cuda_nvml_dev_linux_sbsa//:nvml"], + }), +) diff --git a/bin/zml-smi/platforms/nvml/metrics.zig b/bin/zml-smi/platforms/nvml/metrics.zig new file mode 100644 index 00000000..213dd253 --- /dev/null +++ b/bin/zml-smi/platforms/nvml/metrics.zig @@ -0,0 +1,222 @@ +const std = @import("std"); +const Nvml = @import("nvml.zig"); +const device_info = @import("zml-smi/info").device_info; +const GpuInfo = device_info.GpuInfo; +const poll_metrics = @import("zml-smi/info").poll_metrics; +const DoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer; +const Collector = @import("zml-smi/collector").Collector; +const process = @import("process.zig"); + +const pcie_every: u8 = 4; + +const Slot = struct { + info: *DoubleBuffer(GpuInfo), + dev: Device, +}; + +const Ctx = struct { + gpa: std.mem.Allocator, + nvml: *const Nvml, + slots: []Slot, + processes: *process.ProcessDoubleBuffer, + last_seen_ts: []u64, + dev_offset: u16, + device_count: u32, + tick: u8 = 0, +}; + +pub fn start(collector: *Collector) !void { + const nvml = try collector.arena.create(Nvml); + nvml.* = try Nvml.init(); + + const count = try nvml.deviceCount(); + const dev_offset: u16 = @intCast(collector.device_infos.items.len); + + var slots: std.ArrayList(Slot) = try .initCapacity(collector.arena, count); + for (0..count) |i| { + const dev = Device.open(nvml, @intCast(i)) catch continue; + const initial: GpuInfo = .{ + .name = dev.name(collector.arena) catch null, + .driver_version = dev.driverVersion(collector.arena) catch null, + .cuda_driver_version = dev.cudaDriverVersion(collector.arena) catch null, + }; + const info = try collector.addDevice(.{ .cuda = .{ .values = .{ initial, initial } } }); + slots.appendAssumeCapacity(.{ .info = &info.cuda, .dev = dev }); + } + + const last_seen_ts = try collector.arena.alloc(u64, count); + @memset(last_seen_ts, 0); + + const ctx = try collector.arena.create(Ctx); + ctx.* = .{ + .gpa = collector.gpa, + .nvml = nvml, + .slots = slots.items, + .processes = try collector.createProcessList(), + .last_seen_ts = last_seen_ts, + .dev_offset = dev_offset, + .device_count = count, + }; + try collector.spawnPoll(pollOnce, .{ctx}); +} + +fn pollOnce(ctx: *Ctx) void { + const pcie = ctx.tick % pcie_every == 0; + ctx.tick +%= 1; + + for (ctx.slots) |slot| { + pollDevice(slot.info, slot.dev, pcie); + } + process.pollOnce(ctx.gpa, ctx.processes, ctx.nvml, ctx.dev_offset, ctx.device_count, ctx.last_seen_ts); +} + +fn pollDevice(db: *DoubleBuffer(GpuInfo), dev: Device, pcie: bool) void { + const back = db.back(); + back.* = db.front().*; + poll_metrics.apply(metrics, back, dev); + if (pcie) poll_metrics.apply(pcie_metrics, back, dev); + db.swap(); +} + +const Device = struct { + nvml: *const Nvml, + handle: Nvml.Handle, + + pub fn open(nvml: *const Nvml, index: u32) !Device { + return .{ .nvml = nvml, .handle = try nvml.handleByIndex(index) }; + } + + fn name(self: Device, arena: std.mem.Allocator) ![]const u8 { + var buf: [Nvml.name_buf_len]u8 = undefined; + const slice = try self.nvml.name(self.handle, &buf); + + return try arena.dupe(u8, slice); + } + + fn driverVersion(self: Device, arena: std.mem.Allocator) ![]const u8 { + var buf: [Nvml.driver_version_buf_size]u8 = undefined; + const slice = try self.nvml.driverVersion(&buf); + + return try arena.dupe(u8, slice); + } + + fn cudaDriverVersion(self: Device, arena: std.mem.Allocator) ![]const u8 { + const v = try self.nvml.cudaDriverVersion(); + return try std.fmt.allocPrint(arena, "{}.{}", .{ + Nvml.cudaDriverVersionMajor(v), + Nvml.cudaDriverVersionMinor(v), + }); + } + + // Power + pub fn powerUsage(self: Device) !u64 { + return @intCast(try self.nvml.powerUsage(self.handle)); + } + + pub fn powerLimit(self: Device) !u64 { + return @intCast(try self.nvml.powerLimit(self.handle)); + } + + // Thermal + pub fn temperature(self: Device) !u64 { + return @intCast(try self.nvml.temperature(self.handle)); + } + + pub fn fanSpeed(self: Device) !u64 { + return @intCast(try self.nvml.fanSpeed(self.handle)); + } + + // Utilization + pub fn gpuUtil(self: Device) !u64 { + return @intCast(try self.nvml.utilizationGpu(self.handle)); + } + + pub fn encoderUtil(self: Device) !u64 { + return @intCast(try self.nvml.encoderUtil(self.handle)); + } + + pub fn decoderUtil(self: Device) !u64 { + return @intCast(try self.nvml.decoderUtil(self.handle)); + } + + // Clocks + pub fn clockGraphics(self: Device) !u64 { + return @intCast(try self.nvml.clockGraphics(self.handle)); + } + + pub fn clockSm(self: Device) !u64 { + return @intCast(try self.nvml.clockSm(self.handle)); + } + + pub fn clockMem(self: Device) !u64 { + return @intCast(try self.nvml.clockMem(self.handle)); + } + + pub fn maxClockGraphics(self: Device) !u64 { + return @intCast(try self.nvml.maxClockGraphics(self.handle)); + } + + pub fn maxClockMem(self: Device) !u64 { + return @intCast(try self.nvml.maxClockMem(self.handle)); + } + + // Memory + pub fn memUsed(self: Device) !u64 { + return self.nvml.memUsed(self.handle); + } + + pub fn memTotal(self: Device) !u64 { + return self.nvml.memTotal(self.handle); + } + + pub fn memBusWidth(self: Device) !u64 { + return @intCast(try self.nvml.memBusWidth(self.handle)); + } + + // PCIe + pub fn pcieTx(self: Device) !u64 { + return @intCast(try self.nvml.pcieTxKBps(self.handle)); + } + + pub fn pcieRx(self: Device) !u64 { + return @intCast(try self.nvml.pcieRxKBps(self.handle)); + } + + pub fn pcieBandwidth(self: Device) !u64 { + return @intCast(try self.nvml.pcieSpeed(self.handle)); + } + + pub fn pcieLinkGen(self: Device) !u64 { + return @intCast(try self.nvml.pcieLinkGen(self.handle)); + } + + pub fn pcieLinkWidth(self: Device) !u64 { + return @intCast(try self.nvml.pcieLinkWidth(self.handle)); + } +}; + +const metrics = .{ + .{ .field = "power_mw", .query = Device.powerUsage }, + .{ .field = "power_limit_mw", .query = Device.powerLimit }, + .{ .field = "temperature", .query = Device.temperature }, + .{ .field = "fan_speed_percent", .query = Device.fanSpeed }, + .{ .field = "util_percent", .query = Device.gpuUtil }, + .{ .field = "encoder_util_percent", .query = Device.encoderUtil }, + .{ .field = "decoder_util_percent", .query = Device.decoderUtil }, + .{ .field = "clock_graphics_mhz", .query = Device.clockGraphics }, + .{ .field = "clock_sm_mhz", .query = Device.clockSm }, + .{ .field = "clock_mem_mhz", .query = Device.clockMem }, + .{ .field = "clock_graphics_max_mhz", .query = Device.maxClockGraphics }, + .{ .field = "clock_mem_max_mhz", .query = Device.maxClockMem }, + .{ .field = "mem_used_bytes", .query = Device.memUsed }, + .{ .field = "mem_total_bytes", .query = Device.memTotal }, + .{ .field = "mem_bus_width", .query = Device.memBusWidth }, + .{ .field = "pcie_bandwidth_mbps", .query = Device.pcieBandwidth }, + .{ .field = "pcie_link_gen", .query = Device.pcieLinkGen }, + .{ .field = "pcie_link_width", .query = Device.pcieLinkWidth }, +}; + +const pcie_metrics = .{ + .{ .field = "pcie_tx_kbps", .query = Device.pcieTx }, + .{ .field = "pcie_rx_kbps", .query = Device.pcieRx }, +}; diff --git a/bin/zml-smi/platforms/nvml/nvml.zig b/bin/zml-smi/platforms/nvml/nvml.zig new file mode 100644 index 00000000..e2b36982 --- /dev/null +++ b/bin/zml-smi/platforms/nvml/nvml.zig @@ -0,0 +1,349 @@ +const std = @import("std"); +const c = @import("c"); +const DynLib = @import("zml-smi/dynlib"); + +const Nvml = @This(); + +pub const driver_version_buf_size = c.NVML_SYSTEM_DRIVER_VERSION_BUFFER_SIZE; +pub const Error = ReturnError || error{NvmlUnavailable}; +pub const Handle = c.nvmlDevice_t; +pub const ProcessInfo_t = c.nvmlProcessInfo_t; + +lib: Fns, + +const Fns = struct { + nvmlInit_v2: *const @TypeOf(c.nvmlInit_v2), + nvmlDeviceGetHandleByIndex_v2: *const @TypeOf(c.nvmlDeviceGetHandleByIndex_v2), + nvmlDeviceGetCount_v2: *const @TypeOf(c.nvmlDeviceGetCount_v2), + nvmlDeviceGetName: *const @TypeOf(c.nvmlDeviceGetName), + nvmlDeviceGetPowerUsage: *const @TypeOf(c.nvmlDeviceGetPowerUsage), + nvmlDeviceGetTemperature: *const @TypeOf(c.nvmlDeviceGetTemperature), + nvmlDeviceGetUtilizationRates: *const @TypeOf(c.nvmlDeviceGetUtilizationRates), + nvmlDeviceGetClockInfo: *const @TypeOf(c.nvmlDeviceGetClockInfo), + nvmlDeviceGetMaxClockInfo: *const @TypeOf(c.nvmlDeviceGetMaxClockInfo), + nvmlDeviceGetMemoryInfo: *const @TypeOf(c.nvmlDeviceGetMemoryInfo), + nvmlDeviceGetFanSpeed: *const @TypeOf(c.nvmlDeviceGetFanSpeed), + nvmlDeviceGetEnforcedPowerLimit: *const @TypeOf(c.nvmlDeviceGetEnforcedPowerLimit), + nvmlDeviceGetPcieThroughput: *const @TypeOf(c.nvmlDeviceGetPcieThroughput), + nvmlDeviceGetEncoderUtilization: *const @TypeOf(c.nvmlDeviceGetEncoderUtilization), + nvmlDeviceGetDecoderUtilization: *const @TypeOf(c.nvmlDeviceGetDecoderUtilization), + nvmlDeviceGetCurrPcieLinkGeneration: *const @TypeOf(c.nvmlDeviceGetCurrPcieLinkGeneration), + nvmlDeviceGetCurrPcieLinkWidth: *const @TypeOf(c.nvmlDeviceGetCurrPcieLinkWidth), + nvmlDeviceGetMemoryBusWidth: *const @TypeOf(c.nvmlDeviceGetMemoryBusWidth), + nvmlDeviceGetPcieSpeed: *const @TypeOf(c.nvmlDeviceGetPcieSpeed), + nvmlDeviceGetComputeRunningProcesses_v3: *const @TypeOf(c.nvmlDeviceGetComputeRunningProcesses_v3), + nvmlDeviceGetGraphicsRunningProcesses_v3: *const @TypeOf(c.nvmlDeviceGetGraphicsRunningProcesses_v3), + nvmlDeviceGetProcessUtilization: *const @TypeOf(c.nvmlDeviceGetProcessUtilization), + nvmlSystemGetCudaDriverVersion_v2: *const @TypeOf(c.nvmlSystemGetCudaDriverVersion_v2), + nvmlSystemGetDriverVersion: *const @TypeOf(c.nvmlSystemGetDriverVersion), +}; + +pub fn init() Error!Nvml { + var dynlib: std.DynLib = .{ .inner = .{ + .handle = std.c.dlopen("libnvidia-ml.so.1", .{ .LAZY = true, .GLOBAL = true, .NODELETE = true }) orelse + std.c.dlopen("libnvidia-ml.so", .{ .LAZY = true, .GLOBAL = true, .NODELETE = true }) orelse { + if (std.c.dlerror()) |err| std.log.err("nvml: dlopen: {s}", .{err}); + return error.NvmlUnavailable; + }, + } }; + const fns = DynLib.lookupStruct(&dynlib, Fns) catch return error.NvmlUnavailable; + try check(fns.nvmlInit_v2()); + return .{ .lib = fns }; +} + +pub fn handleByIndex(self: Nvml, device_id: u32) Error!c.nvmlDevice_t { + var handle: c.nvmlDevice_t = undefined; + try check(self.lib.nvmlDeviceGetHandleByIndex_v2(device_id, &handle)); + return handle; +} + +pub fn deviceCount(self: Nvml) Error!u32 { + var count: c_uint = 0; + try check(self.lib.nvmlDeviceGetCount_v2(&count)); + return @intCast(count); +} + +pub const name_buf_len = c.NVML_DEVICE_NAME_V2_BUFFER_SIZE; + +pub fn name(self: Nvml, handle: c.nvmlDevice_t, buf: *[c.NVML_DEVICE_NAME_V2_BUFFER_SIZE]u8) Error![:0]const u8 { + try check(self.lib.nvmlDeviceGetName(handle, buf, buf.len)); + return std.mem.span(@as([*c]const u8, @ptrCast(buf))); +} + +pub fn powerUsage(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var power_mw: c_uint = 0; + try check(self.lib.nvmlDeviceGetPowerUsage(handle, &power_mw)); + return power_mw; +} + +pub fn temperature(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var temp: c_uint = 0; + try check(self.lib.nvmlDeviceGetTemperature(handle, c.NVML_TEMPERATURE_GPU, &temp)); + return temp; +} + +pub fn utilizationGpu(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var util: c.nvmlUtilization_t = undefined; + try check(self.lib.nvmlDeviceGetUtilizationRates(handle, &util)); + return util.gpu; +} + +pub fn clockGraphics(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var clock: c_uint = 0; + try check(self.lib.nvmlDeviceGetClockInfo(handle, c.NVML_CLOCK_GRAPHICS, &clock)); + return clock; +} + +pub fn clockSm(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var clock: c_uint = 0; + try check(self.lib.nvmlDeviceGetClockInfo(handle, c.NVML_CLOCK_SM, &clock)); + return clock; +} + +pub fn clockMem(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var clock: c_uint = 0; + try check(self.lib.nvmlDeviceGetClockInfo(handle, c.NVML_CLOCK_MEM, &clock)); + return clock; +} + +pub fn maxClockGraphics(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var clock: c_uint = 0; + try check(self.lib.nvmlDeviceGetMaxClockInfo(handle, c.NVML_CLOCK_GRAPHICS, &clock)); + return clock; +} + +pub fn maxClockMem(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var clock: c_uint = 0; + try check(self.lib.nvmlDeviceGetMaxClockInfo(handle, c.NVML_CLOCK_MEM, &clock)); + return clock; +} + +pub fn memTotal(self: Nvml, handle: c.nvmlDevice_t) Error!u64 { + var mem: c.nvmlMemory_t = undefined; + try check(self.lib.nvmlDeviceGetMemoryInfo(handle, &mem)); + return @intCast(mem.total); +} + +pub fn memUsed(self: Nvml, handle: c.nvmlDevice_t) Error!u64 { + var mem: c.nvmlMemory_t = undefined; + try check(self.lib.nvmlDeviceGetMemoryInfo(handle, &mem)); + return @intCast(mem.used); +} + +pub fn fanSpeed(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var speed: c_uint = 0; + try check(self.lib.nvmlDeviceGetFanSpeed(handle, &speed)); + return speed; +} + +pub fn powerLimit(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var limit: c_uint = 0; + try check(self.lib.nvmlDeviceGetEnforcedPowerLimit(handle, &limit)); + return limit; +} + +pub fn pcieTxKBps(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var value: c_uint = 0; + try check(self.lib.nvmlDeviceGetPcieThroughput(handle, c.NVML_PCIE_UTIL_TX_BYTES, &value)); + return value; +} + +pub fn pcieRxKBps(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var value: c_uint = 0; + try check(self.lib.nvmlDeviceGetPcieThroughput(handle, c.NVML_PCIE_UTIL_RX_BYTES, &value)); + return value; +} + +pub fn encoderUtil(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var util: c_uint = 0; + var sampling: c_uint = 0; + try check(self.lib.nvmlDeviceGetEncoderUtilization(handle, &util, &sampling)); + return util; +} + +pub fn decoderUtil(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var util: c_uint = 0; + var sampling: c_uint = 0; + try check(self.lib.nvmlDeviceGetDecoderUtilization(handle, &util, &sampling)); + return util; +} + +pub fn pcieLinkGen(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var gen: c_uint = 0; + try check(self.lib.nvmlDeviceGetCurrPcieLinkGeneration(handle, &gen)); + return gen; +} + +pub fn pcieLinkWidth(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var width: c_uint = 0; + try check(self.lib.nvmlDeviceGetCurrPcieLinkWidth(handle, &width)); + return width; +} + +pub fn pcieSpeed(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var speed: c_uint = 0; + try check(self.lib.nvmlDeviceGetPcieSpeed(handle, &speed)); + return speed; +} + +pub fn driverVersion(self: Nvml, buf: *[c.NVML_SYSTEM_DRIVER_VERSION_BUFFER_SIZE]u8) Error![:0]const u8 { + try check(self.lib.nvmlSystemGetDriverVersion(buf, buf.len)); + return std.mem.span(@as([*c]const u8, @ptrCast(buf))); +} + +pub fn cudaDriverVersionMajor(v: c_int) c_int { + return @divTrunc(v, 1000); +} + +pub fn cudaDriverVersionMinor(v: c_int) c_int { + return @divTrunc(@mod(v, 1000), 10); +} + +pub fn cudaDriverVersion(self: Nvml) Error!c_int { + var version: c_int = 0; + try check(self.lib.nvmlSystemGetCudaDriverVersion_v2(&version)); + return version; +} + +pub fn memBusWidth(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { + var width: c_uint = 0; + try check(self.lib.nvmlDeviceGetMemoryBusWidth(handle, &width)); + return width; +} + +const max_query = 4096; + +fn sizedCount(count: c_uint) ?c_uint { + if (count == 0 or count > max_query) return null; + return count; +} + +pub fn computeRunningProcesses(self: Nvml, allocator: std.mem.Allocator, handle: c.nvmlDevice_t) (Error || error{OutOfMemory})![]c.nvmlProcessInfo_t { + var count: c_uint = 0; + const ret = self.lib.nvmlDeviceGetComputeRunningProcesses_v3(handle, &count, null); + if (ret != c.NVML_ERROR_INSUFFICIENT_SIZE) { + if (ret == c.NVML_SUCCESS) { + return &.{}; + } + try check(ret); + } + count = sizedCount(count) orelse return &.{}; + const infos = try allocator.alloc(c.nvmlProcessInfo_t, count); + errdefer allocator.free(infos); + try check(self.lib.nvmlDeviceGetComputeRunningProcesses_v3(handle, &count, @ptrCast(infos.ptr))); + return infos; +} + +pub fn graphicsRunningProcesses(self: Nvml, allocator: std.mem.Allocator, handle: c.nvmlDevice_t) (Error || error{OutOfMemory})![]const c.nvmlProcessInfo_t { + var count: c_uint = 0; + const ret = self.lib.nvmlDeviceGetGraphicsRunningProcesses_v3(handle, &count, null); + + if (ret != c.NVML_ERROR_INSUFFICIENT_SIZE) { + if (ret == c.NVML_SUCCESS) { + return &.{}; + } + try check(ret); + } + + count = sizedCount(count) orelse return &.{}; + const infos = try allocator.alloc(c.nvmlProcessInfo_t, count); + errdefer allocator.free(infos); + + try check(self.lib.nvmlDeviceGetGraphicsRunningProcesses_v3(handle, &count, @ptrCast(infos.ptr))); + return infos; +} + +pub fn processUtilization(self: Nvml, allocator: std.mem.Allocator, handle: c.nvmlDevice_t, last_seen: u64) (Error || error{OutOfMemory})![]const c.nvmlProcessUtilizationSample_t { + var count: c_uint = 0; + const ret = self.lib.nvmlDeviceGetProcessUtilization(handle, null, &count, last_seen); + + if (ret != c.NVML_ERROR_INSUFFICIENT_SIZE) { + if (ret == c.NVML_SUCCESS) { + return &.{}; + } + try check(ret); + } + + count = sizedCount(count) orelse return &.{}; + const samples = try allocator.alloc(c.nvmlProcessUtilizationSample_t, count); + errdefer allocator.free(samples); + + try check(self.lib.nvmlDeviceGetProcessUtilization(handle, @ptrCast(samples.ptr), &count, last_seen)); + return samples; +} + +// Error handling + +pub const ReturnError = error{ + error_uninitialized, + error_invalid_argument, + error_not_supported, + error_no_permission, + error_already_initialized, + error_not_found, + error_insufficient_size, + error_insufficient_power, + error_driver_not_loaded, + error_timeout, + error_irq_issue, + error_library_not_found, + error_function_not_found, + error_corrupted_inforom, + error_gpu_is_lost, + error_reset_required, + error_operating_system, + error_lib_rm_version_mismatch, + error_in_use, + error_memory, + error_no_data, + error_vgpu_ecc_not_supported, + error_insufficient_resources, + error_freq_not_supported, + error_argument_version_mismatch, + error_deprecated, + error_not_ready, + error_gpu_not_found, + error_invalid_state, + error_reset_type_not_supported, + error_unknown, +}; + +fn check(ret: c_uint) Error!void { + if (ret == c.NVML_SUCCESS) { + return; + } + return switch (ret) { + c.NVML_ERROR_UNINITIALIZED => error.error_uninitialized, + c.NVML_ERROR_INVALID_ARGUMENT => error.error_invalid_argument, + c.NVML_ERROR_NOT_SUPPORTED => error.error_not_supported, + c.NVML_ERROR_NO_PERMISSION => error.error_no_permission, + c.NVML_ERROR_ALREADY_INITIALIZED => error.error_already_initialized, + c.NVML_ERROR_NOT_FOUND => error.error_not_found, + c.NVML_ERROR_INSUFFICIENT_SIZE => error.error_insufficient_size, + c.NVML_ERROR_INSUFFICIENT_POWER => error.error_insufficient_power, + c.NVML_ERROR_DRIVER_NOT_LOADED => error.error_driver_not_loaded, + c.NVML_ERROR_TIMEOUT => error.error_timeout, + c.NVML_ERROR_IRQ_ISSUE => error.error_irq_issue, + c.NVML_ERROR_LIBRARY_NOT_FOUND => error.error_library_not_found, + c.NVML_ERROR_FUNCTION_NOT_FOUND => error.error_function_not_found, + c.NVML_ERROR_CORRUPTED_INFOROM => error.error_corrupted_inforom, + c.NVML_ERROR_GPU_IS_LOST => error.error_gpu_is_lost, + c.NVML_ERROR_RESET_REQUIRED => error.error_reset_required, + c.NVML_ERROR_OPERATING_SYSTEM => error.error_operating_system, + c.NVML_ERROR_LIB_RM_VERSION_MISMATCH => error.error_lib_rm_version_mismatch, + c.NVML_ERROR_IN_USE => error.error_in_use, + c.NVML_ERROR_MEMORY => error.error_memory, + c.NVML_ERROR_NO_DATA => error.error_no_data, + c.NVML_ERROR_VGPU_ECC_NOT_SUPPORTED => error.error_vgpu_ecc_not_supported, + c.NVML_ERROR_INSUFFICIENT_RESOURCES => error.error_insufficient_resources, + c.NVML_ERROR_FREQ_NOT_SUPPORTED => error.error_freq_not_supported, + c.NVML_ERROR_ARGUMENT_VERSION_MISMATCH => error.error_argument_version_mismatch, + c.NVML_ERROR_DEPRECATED => error.error_deprecated, + c.NVML_ERROR_NOT_READY => error.error_not_ready, + c.NVML_ERROR_GPU_NOT_FOUND => error.error_gpu_not_found, + c.NVML_ERROR_INVALID_STATE => error.error_invalid_state, + c.NVML_ERROR_RESET_TYPE_NOT_SUPPORTED => error.error_reset_type_not_supported, + c.NVML_ERROR_UNKNOWN => error.error_unknown, + else => error.error_unknown, + }; +} diff --git a/bin/zml-smi/platforms/nvml/process.zig b/bin/zml-smi/platforms/nvml/process.zig new file mode 100644 index 00000000..5e18cdcb --- /dev/null +++ b/bin/zml-smi/platforms/nvml/process.zig @@ -0,0 +1,54 @@ +const std = @import("std"); +const Nvml = @import("nvml.zig"); +const pi = @import("zml-smi/info").process_info; + +pub const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); + +pub fn pollOnce(allocator: std.mem.Allocator, list: *ProcessDoubleBuffer, nvml: *const Nvml, dev_offset: u16, device_count: u32, last_seen_ts: []u64) void { + const back = list.back(); + + back.clearRetainingCapacity(); + + for (0..device_count) |dev_idx| { + const handle = nvml.handleByIndex(@intCast(dev_idx)) catch continue; + const idx: u16 = @intCast(dev_idx + dev_offset); + + const compute = nvml.computeRunningProcesses(allocator, handle) catch &.{}; + defer if (compute.len > 0) allocator.free(compute); + collectFromQuery(allocator, back, idx, compute); + + const graphics = nvml.graphicsRunningProcesses(allocator, handle) catch &.{}; + defer if (graphics.len > 0) allocator.free(graphics); + collectFromQuery(allocator, back, idx, graphics); + + const last_ts = last_seen_ts[dev_idx]; + const utils = nvml.processUtilization(allocator, handle, last_ts) catch &.{}; + defer if (utils.len > 0) allocator.free(utils); + + for (utils) |sample| { + if (sample.smUtil > 100 or sample.timeStamp <= last_ts) { + continue; + } + + last_seen_ts[dev_idx] = @max(last_seen_ts[dev_idx], sample.timeStamp); + + for (back.items) |*entry| { + if (entry.pid == sample.pid and entry.device_idx == idx) { + entry.dev_util_percent = @intCast(sample.smUtil); + } + } + } + } + + list.swap(); +} + +fn collectFromQuery(allocator: std.mem.Allocator, back: *std.ArrayList(pi.ProcessInfo), dev_idx: u16, procs: []const Nvml.ProcessInfo_t) void { + for (procs) |gp| { + back.append(allocator, .{ + .pid = gp.pid, + .device_idx = dev_idx, + .dev_mem_kib = @intCast(gp.usedGpuMemory / 1024), + }) catch return; + } +} diff --git a/bin/zml-smi/platforms/oneapi/BUILD.bazel b/bin/zml-smi/platforms/oneapi/BUILD.bazel new file mode 100644 index 00000000..c0b27d84 --- /dev/null +++ b/bin/zml-smi/platforms/oneapi/BUILD.bazel @@ -0,0 +1,24 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "oneapi", + import_name = "zml-smi/platforms/oneapi", + main = "metrics.zig", + srcs = [ + "monitor.zig", + "oneapi.zig", + "process.zig", + ], + visibility = ["//visibility:public"], + deps = [ + "//bin/zml-smi/collector", + "//bin/zml-smi/info", + "//bin/zml-smi/utils:double_buffer", + "//bin/zml-smi/utils:sysfs", + ], +) + +zig_test( + name = "test", + deps = [":oneapi"], +) diff --git a/bin/zml-smi/platforms/oneapi/metrics.zig b/bin/zml-smi/platforms/oneapi/metrics.zig new file mode 100644 index 00000000..ae2a86a3 --- /dev/null +++ b/bin/zml-smi/platforms/oneapi/metrics.zig @@ -0,0 +1,134 @@ +const std = @import("std"); +const OneApi = @import("oneapi.zig"); +const device_info = @import("zml-smi/info").device_info; +const GpuInfo = device_info.GpuInfo; +const DoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer; +const Collector = @import("zml-smi/collector").Collector; +const poll_metrics = @import("zml-smi/info").poll_metrics; +const process = @import("process.zig"); + +pub fn start(collector: *Collector) !void { + const oneapi = try collector.arena.create(OneApi); + oneapi.* = try OneApi.init(collector.arena, collector.io); + + const count = oneapi.monitor.devices.len; + const dev_offset: u16 = @intCast(collector.device_infos.items.len); + + for (0..count) |i| { + const poll_arena = try collector.createPollArena(); + const dev = Device.open(oneapi, i, poll_arena) catch continue; + const initial: GpuInfo = .{ + .name = dev.name(collector.arena) catch null, + .driver_version = dev.driverVersion(collector.arena) catch null, + .power_limit_mw = dev.powerLimit() catch null, + .clock_graphics_max_mhz = dev.maxClockGraphics() catch null, + .mem_total_bytes = dev.memTotal() catch null, + .pcie_link_gen = dev.pcieLinkGen() catch null, + .pcie_link_width = dev.pcieLinkWidth() catch null, + .pcie_bandwidth_mbps = dev.pcieBandwidth() catch null, + }; + const info = try collector.addDevice(.{ .oneapi = .{ .values = .{ initial, initial } } }); + + try collector.spawnPoll(pollOnce, .{ poll_arena, &info.oneapi, dev }); + } + + const processes = try collector.createProcessList(); + try process.init(collector, processes, oneapi, dev_offset); +} + +const pollOnce = poll_metrics.poll(*DoubleBuffer(GpuInfo), Device, metrics); + +const Device = struct { + oneapi: *OneApi, + arena: *std.heap.ArenaAllocator, + handle: OneApi.Handle, + + fn open(oneapi: *OneApi, index: usize, arena: *std.heap.ArenaAllocator) !Device { + return .{ + .oneapi = oneapi, + .arena = arena, + .handle = try oneapi.handleByIndex(index), + }; + } + + fn name(self: Device, arena: std.mem.Allocator) ![]const u8 { + return self.oneapi.name(arena, self.handle); + } + + fn driverVersion(self: Device, arena: std.mem.Allocator) ![]const u8 { + return self.oneapi.driverVersion(arena); + } + + pub fn powerUsage(self: Device) !u64 { + return self.oneapi.powerUsage(self.arena.allocator(), self.handle); + } + + pub fn temperature(self: Device) !u64 { + return self.oneapi.temperature(self.arena.allocator(), self.handle); + } + + pub fn fanSpeed(self: Device) !u64 { + return self.oneapi.fanSpeed(self.arena.allocator(), self.handle); + } + + pub fn gpuUtil(self: Device) !u64 { + return self.oneapi.gpuUtil(self.arena.allocator(), self.handle); + } + + pub fn encoderUtil(self: Device) !u64 { + return self.oneapi.encoderUtil(self.arena.allocator(), self.handle); + } + + pub fn decoderUtil(self: Device) !u64 { + return self.oneapi.decoderUtil(self.arena.allocator(), self.handle); + } + + pub fn powerLimit(self: Device) !u64 { + return self.oneapi.powerLimit(self.arena.allocator(), self.handle); + } + + pub fn clockGraphics(self: Device) !u64 { + return self.oneapi.clockGraphics(self.arena.allocator(), self.handle); + } + + pub fn maxClockGraphics(self: Device) !u64 { + return self.oneapi.maxClockGraphics(self.arena.allocator(), self.handle); + } + + pub fn memUsed(self: Device) !u64 { + return self.oneapi.memUsed(self.arena.allocator(), self.handle); + } + + pub fn memTotal(self: Device) !u64 { + return self.oneapi.memTotal(self.arena.allocator(), self.handle); + } + + pub fn pcieLinkGen(self: Device) !u64 { + return self.oneapi.pcieLinkGen(self.arena.allocator(), self.handle); + } + + pub fn pcieLinkWidth(self: Device) !u64 { + return self.oneapi.pcieLinkWidth(self.arena.allocator(), self.handle); + } + + pub fn pcieBandwidth(self: Device) !u64 { + return self.oneapi.pcieBandwidth(self.arena.allocator(), self.handle); + } +}; + +const metrics = .{ + .{ .field = "power_mw", .query = Device.powerUsage }, + .{ .field = "power_limit_mw", .query = Device.powerLimit }, + .{ .field = "temperature", .query = Device.temperature }, + .{ .field = "fan_speed_percent", .query = Device.fanSpeed }, + .{ .field = "util_percent", .query = Device.gpuUtil }, + .{ .field = "encoder_util_percent", .query = Device.encoderUtil }, + .{ .field = "decoder_util_percent", .query = Device.decoderUtil }, + .{ .field = "clock_graphics_mhz", .query = Device.clockGraphics }, + .{ .field = "clock_graphics_max_mhz", .query = Device.maxClockGraphics }, + .{ .field = "mem_used_bytes", .query = Device.memUsed }, + .{ .field = "mem_total_bytes", .query = Device.memTotal }, + .{ .field = "pcie_bandwidth_mbps", .query = Device.pcieBandwidth }, + .{ .field = "pcie_link_gen", .query = Device.pcieLinkGen }, + .{ .field = "pcie_link_width", .query = Device.pcieLinkWidth }, +}; diff --git a/bin/zml-smi/platforms/oneapi/monitor.zig b/bin/zml-smi/platforms/oneapi/monitor.zig new file mode 100644 index 00000000..dd5f97a9 --- /dev/null +++ b/bin/zml-smi/platforms/oneapi/monitor.zig @@ -0,0 +1,877 @@ +const std = @import("std"); +const smi_sysfs = @import("zml-smi/sysfs"); +const Monitor = @This(); + +pub const bus_device_identifier_len = "0000:00:00.0".len; +pub const Handle = enum(u32) { _ }; + +const Device = struct { + render_idx: usize, + drm_path: []const u8, + dev_path: []const u8, + pcie_path: []const u8, + hwmon_path: ?[]const u8, + bus_device_identifier: ?[bus_device_identifier_len]u8, + activity_prev: ?EngineSample = null, + encoder_prev: ?EngineSample = null, + decoder_prev: ?EngineSample = null, + energy_prev: ?EnergySample = null, +}; + +const EnergySample = struct { + micro_joules: u64, + timestamp_ns: u64, +}; + +pub const EngineSample = struct { + engine_ns: u64, + timestamp_ns: u64, +}; + +const DeviceSample = struct { + mem_kib: u64 = 0, + engine: ?EngineSample = null, + encoder: ?EngineSample = null, + decoder: ?EngineSample = null, +}; + +pub const ProcessSample = struct { + pid: u32, + device_idx: u16, + mem_kib: ?u64 = null, + engine: ?EngineSample = null, + encoder: ?EngineSample = null, + decoder: ?EngineSample = null, +}; + +pub const ProcessUsage = std.AutoHashMapUnmanaged(u64, ProcessSample); +pub const ProcessPrevious = std.AutoHashMapUnmanaged(u64, EngineSample); +const DeviceUsage = std.AutoHashMapUnmanaged(u16, DeviceSample); + +allocator: std.mem.Allocator, +io: std.Io, +devices: []Device, + +pub fn init(allocator: std.mem.Allocator, io: std.Io) !Monitor { + var monitor: Monitor = .{ + .allocator = allocator, + .io = io, + .devices = try discoverIntelDevices(allocator, io), + }; + try monitor.seedDeviceMetrics(); + return monitor; +} + +pub fn deviceId(self: *const Monitor, allocator: std.mem.Allocator, handle: Handle) ![]const u8 { + const dev = try self.device(handle); + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buf, "{s}/device", .{dev.dev_path}); + const raw = try smi_sysfs.readString(allocator, self.io, path); + return std.mem.trim(u8, raw, &std.ascii.whitespace); +} + +inline fn seedDeviceMetrics(self: *Monitor) !void { + var device_usage = collectDeviceUsage(self.allocator, self.io, self.devices) catch DeviceUsage{}; + defer device_usage.deinit(self.allocator); + + for (self.devices, 0..) |*dev, idx| { + dev.energy_prev = readEnergy(self.allocator, self.io, dev) catch null; + if (device_usage.get(@intCast(idx))) |sample| { + dev.activity_prev = sample.engine; + dev.encoder_prev = sample.encoder; + dev.decoder_prev = sample.decoder; + } + } +} + +pub fn saveProcessPrevious(allocator: std.mem.Allocator, previous: *ProcessPrevious, usage: *const ProcessUsage) !void { + previous.clearRetainingCapacity(); + try previous.ensureTotalCapacity(allocator, @intCast(usage.count())); + + var it = usage.iterator(); + while (it.next()) |entry| { + if (entry.value_ptr.engine) |engine| { + previous.putAssumeCapacity(entry.key_ptr.*, engine); + } + } +} + +inline fn discoverIntelDevices(allocator: std.mem.Allocator, io: std.Io) ![]Device { + var devices: std.ArrayList(Device) = .empty; + errdefer devices.deinit(allocator); + + var dir = std.Io.Dir.openDirAbsolute(io, "/dev/dri", .{ .iterate = true }) catch |err| switch (err) { + error.FileNotFound => return &.{}, + else => |e| return e, + }; + defer dir.close(io); + + var it = dir.iterate(); + while (it.next(io) catch null) |entry| { + const render_idx = parseRenderIndex(entry.name) orelse continue; + const dev = openDevice(allocator, io, render_idx) catch continue; + try devices.append(allocator, dev); + } + + std.mem.sort(Device, devices.items, {}, struct { + fn lessThan(_: void, lhs: Device, rhs: Device) bool { + return lhs.render_idx < rhs.render_idx; + } + }.lessThan); + return devices.toOwnedSlice(allocator); +} + +inline fn openDevice(allocator: std.mem.Allocator, io: std.Io, render_idx: usize) !Device { + const drm_path = try std.fmt.allocPrint(allocator, "/sys/class/drm/renderD{d}", .{render_idx}); + errdefer allocator.free(drm_path); + + const dev_path = try std.fmt.allocPrint(allocator, "{s}/device", .{drm_path}); + errdefer allocator.free(dev_path); + + var vendor_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const vendor_path = try std.fmt.bufPrint(&vendor_path_buf, "{s}/vendor", .{dev_path}); + const vendor = try smi_sysfs.readString(allocator, io, vendor_path); + if (!std.mem.eql(u8, std.mem.trim(u8, vendor, &std.ascii.whitespace), "0x8086")) { + return error.not_intel; + } + + const pcie_path = try findPcieLinkPath(allocator, io, dev_path); + errdefer if (pcie_path.ptr != dev_path.ptr) allocator.free(pcie_path); + + return .{ + .render_idx = render_idx, + .drm_path = drm_path, + .dev_path = dev_path, + .pcie_path = pcie_path, + .hwmon_path = findHwmon(allocator, io, dev_path) catch null, + .bus_device_identifier = readBusDeviceId(allocator, io, dev_path) catch null, + }; +} + +pub fn device(self: *const Monitor, handle: Handle) !*const Device { + const idx = @backingInt(handle); + if (idx >= self.devices.len) return error.NotFound; + return &self.devices[idx]; +} + +pub fn readTemperature(allocator: std.mem.Allocator, io: std.Io, dev: *const Device) !u64 { + const hwmon = dev.hwmon_path orelse return error.NotFound; + const milli_c = readHwmonInput(allocator, io, hwmon, "temp", "pkg") catch + readHwmonInput(allocator, io, hwmon, "temp", "gpu") catch + readNumberedInput(allocator, io, hwmon, "temp", 2) catch + try readNumberedInput(allocator, io, hwmon, "temp", 1); + return (milli_c + 500) / 1000; +} + +pub fn readEnergy(allocator: std.mem.Allocator, io: std.Io, dev: *const Device) !EnergySample { + const hwmon = dev.hwmon_path orelse return error.NotFound; + const micro_joules = readHwmonInput(allocator, io, hwmon, "energy", "card") catch + readHwmonInput(allocator, io, hwmon, "energy", "pkg") catch + try readNumberedInput(allocator, io, hwmon, "energy", 1); + return .{ + .micro_joules = micro_joules, + .timestamp_ns = @intCast(std.Io.Timestamp.now(io, .awake).nanoseconds), + }; +} +pub fn readPowerLimit(allocator: std.mem.Allocator, io: std.Io, dev: *const Device) !u64 { + const hwmon = dev.hwmon_path orelse return error.NotFound; + const microwatts = readNumberedSuffix(allocator, io, hwmon, "power", 1, "cap") catch + try readNumberedSuffix(allocator, io, hwmon, "power", 1, "max"); + return microwatts / 1000; +} + +pub fn readFanSpeed(allocator: std.mem.Allocator, io: std.Io, dev: *const Device) !u64 { + const hwmon = dev.hwmon_path orelse return error.NotFound; + return readPwmFanSpeed(allocator, io, hwmon) catch readRpmFanSpeedPercent(allocator, io, hwmon); +} + +pub fn readClockGraphics(allocator: std.mem.Allocator, io: std.Io, dev: *const Device) !u64 { + return readFreq(allocator, io, dev, "act_freq") catch readFreq(allocator, io, dev, "cur_freq"); +} + +pub fn readMaxClockGraphics(allocator: std.mem.Allocator, io: std.Io, dev: *const Device) !u64 { + return readFreq(allocator, io, dev, "max_freq"); +} + +pub fn readMemTotal(allocator: std.mem.Allocator, io: std.Io, dev: *const Device) !u64 { + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buf, "{s}/resource", .{dev.dev_path}); + return readLargestResource(allocator, io, path); +} + +pub fn readPcieLinkGen(allocator: std.mem.Allocator, io: std.Io, dev: *const Device) !u64 { + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buf, "{s}/current_link_speed", .{dev.pcie_path}); + const raw = try smi_sysfs.readString(allocator, io, path); + return pcieGenFromSpeed(raw); +} + +pub fn readPcieLinkWidth(allocator: std.mem.Allocator, io: std.Io, dev: *const Device) !u64 { + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buf, "{s}/current_link_width", .{dev.pcie_path}); + return smi_sysfs.readInt(allocator, io, path); +} + +pub fn readPcieBandwidth(allocator: std.mem.Allocator, io: std.Io, dev: *const Device) !u64 { + const gen = try readPcieLinkGen(allocator, io, dev); + const width = try readPcieLinkWidth(allocator, io, dev); + return pcieBandwidthFromLink(gen, width) orelse error.NotFound; +} + +pub fn collectDeviceUsage(allocator: std.mem.Allocator, io: std.Io, devices: []const Device) !DeviceUsage { + var processes = try collectProcessUsage(allocator, io, devices); + errdefer processes.deinit(allocator); + + var result: DeviceUsage = .{}; + errdefer result.deinit(allocator); + + var it = processes.iterator(); + while (it.next()) |entry| { + const sample = entry.value_ptr.*; + const gop = try result.getOrPut(allocator, sample.device_idx); + if (!gop.found_existing) gop.value_ptr.* = .{}; + if (sample.mem_kib) |mem| gop.value_ptr.mem_kib += mem; + addEngineSample(&gop.value_ptr.engine, sample.engine); + addEngineSample(&gop.value_ptr.encoder, sample.encoder); + addEngineSample(&gop.value_ptr.decoder, sample.decoder); + } + + return result; +} + +pub fn collectProcessUsage(allocator: std.mem.Allocator, io: std.Io, devices: []const Device) !ProcessUsage { + var result: ProcessUsage = .{}; + errdefer result.deinit(allocator); + + var seen_device_clients: std.AutoHashMapUnmanaged(struct { u16, u64 }, void) = .{}; + defer seen_device_clients.deinit(allocator); + + var proc_dir = try std.Io.Dir.openDirAbsolute(io, "/proc", .{ .iterate = true }); + defer proc_dir.close(io); + + const now_ns: u64 = @intCast(std.Io.Timestamp.now(io, .awake).nanoseconds); + var proc_it = proc_dir.iterate(); + while (proc_it.next(io) catch null) |proc_entry| { + const pid = std.fmt.parseInt(u32, proc_entry.name, 10) catch continue; + + var fdinfo_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const fdinfo_path = std.fmt.bufPrint(&fdinfo_path_buf, "/proc/{d}/fdinfo", .{pid}) catch continue; + var fdinfo_dir = std.Io.Dir.openDirAbsolute(io, fdinfo_path, .{ .iterate = true }) catch continue; + defer fdinfo_dir.close(io); + + var fd_it = fdinfo_dir.iterate(); + while (fd_it.next(io) catch null) |fd_entry| { + var file_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const file_path = std.fmt.bufPrint(&file_path_buf, "{s}/{s}", .{ fdinfo_path, fd_entry.name }) catch continue; + const parsed = parseFdinfoFile(io, file_path, now_ns) catch continue; + const bus_device_identifier = parsed.bus_device_identifier orelse continue; + const dev_idx = matchDevice(devices, bus_device_identifier) orelse continue; + + // Skip fd if DRM client has been seen for this device. + // Summing after dup() / fork() over-reports memory and compute. + if (parsed.drm_client_id) |cid| { + if ((try seen_device_clients.getOrPut(allocator, .{ dev_idx, cid })).found_existing) continue; + } + + const key = processKey(pid, dev_idx); + const gop = try result.getOrPut(allocator, key); + if (!gop.found_existing) { + gop.value_ptr.* = .{ .pid = pid, .device_idx = dev_idx }; + } + mergeProcessSample(gop.value_ptr, parsed.sample); + } + } + + return result; +} + +pub fn processUtil(previous: ?EngineSample, current: ?EngineSample) ?u16 { + const prev = previous orelse return null; + const cur = current orelse return null; + if (cur.timestamp_ns <= prev.timestamp_ns or cur.engine_ns < prev.engine_ns) return null; + const engine_delta = cur.engine_ns - prev.engine_ns; + const time_delta = cur.timestamp_ns - prev.timestamp_ns; + if (time_delta == 0) return null; + const pct = @min(engine_delta * 100 / time_delta, 100); + return @intCast(if (pct == 0 and engine_delta > 0) 0 else pct); +} + +pub fn powerMilliwatts(previous: ?EnergySample, current: ?EnergySample) ?u64 { + const prev = previous orelse return null; + const cur = current orelse return null; + return milliwattsFromEnergyDelta(prev.micro_joules, cur.micro_joules, prev.timestamp_ns, cur.timestamp_ns); +} + +inline fn milliwattsFromEnergyDelta(prev_uj: u64, cur_uj: u64, prev_ns: u64, cur_ns: u64) ?u64 { + if (cur_uj < prev_uj or cur_ns <= prev_ns) return null; + const energy_delta = cur_uj - prev_uj; + const time_delta = cur_ns - prev_ns; + if (time_delta == 0) return null; + return energy_delta * 1_000_000 / time_delta; +} + +inline fn readFreq(allocator: std.mem.Allocator, io: std.Io, dev: *const Device, file: []const u8) !u64 { + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buf, "{s}/tile0/gt0/freq0/{s}", .{ dev.dev_path, file }); + return smi_sysfs.readInt(allocator, io, path); +} + +inline fn parseRenderIndex(name: []const u8) ?usize { + if (!std.mem.startsWith(u8, name, "renderD")) return null; + return std.fmt.parseInt(usize, name["renderD".len..], 10) catch null; +} + +inline fn findHwmon(allocator: std.mem.Allocator, io: std.Io, dev_path: []const u8) !?[]const u8 { + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const hwmon_root = try std.fmt.bufPrint(&path_buf, "{s}/hwmon", .{dev_path}); + var dir = std.Io.Dir.openDirAbsolute(io, hwmon_root, .{ .iterate = true }) catch return null; + defer dir.close(io); + + var it = dir.iterate(); + while (it.next(io) catch null) |entry| { + if (!std.mem.startsWith(u8, entry.name, "hwmon")) continue; + const candidate = try std.fmt.allocPrint(allocator, "{s}/{s}", .{ hwmon_root, entry.name }); + var name_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const name_path = try std.fmt.bufPrint(&name_buf, "{s}/name", .{candidate}); + const name = smi_sysfs.readString(allocator, io, name_path) catch return candidate; + const trimmed = std.mem.trim(u8, name, &std.ascii.whitespace); + if (std.mem.eql(u8, trimmed, "xe") or std.mem.eql(u8, trimmed, "i915")) return candidate; + } + return null; +} + +inline fn readBusDeviceId(allocator: std.mem.Allocator, io: std.Io, dev_path: []const u8) ![bus_device_identifier_len]u8 { + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buf, "{s}/uevent", .{dev_path}); + const raw = try smi_sysfs.readFieldString(allocator, io, path, "PCI_SLOT_NAME="); + const trimmed = std.mem.trim(u8, raw, &std.ascii.whitespace); + if (trimmed.len != bus_device_identifier_len) return error.NotFound; + var out: [bus_device_identifier_len]u8 = undefined; + @memcpy(&out, trimmed); + return out; +} + +inline fn findPcieLinkPath(allocator: std.mem.Allocator, io: std.Io, dev_path: []const u8) ![]const u8 { + var candidate = dev_path; + var best_path: ?[]const u8 = null; + + var best_bandwidth: u64 = 0; + var depth: usize = 0; + while (depth < 8) : (depth += 1) { + if (!try isIntelPciDevice(allocator, io, candidate)) break; + + const bandwidth = readMaxPcieBandwidth(allocator, io, candidate) catch 0; + if (bandwidth > best_bandwidth) { + best_path = candidate; + best_bandwidth = bandwidth; + } + + const parent = try std.fmt.allocPrint(allocator, "{s}/..", .{candidate}); + candidate = parent; + } + + return best_path orelse dev_path; +} + +inline fn isIntelPciDevice(allocator: std.mem.Allocator, io: std.Io, dev_path: []const u8) !bool { + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const vendor_path = try std.fmt.bufPrint(&path_buf, "{s}/vendor", .{dev_path}); + const vendor_raw = smi_sysfs.readString(allocator, io, vendor_path) catch return false; + + const vendor = std.mem.trim(u8, vendor_raw, &std.ascii.whitespace); + return std.mem.eql(u8, vendor, "0x8086"); +} + +inline fn readMaxPcieBandwidth(allocator: std.mem.Allocator, io: std.Io, dev_path: []const u8) !u64 { + var speed_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const speed_path = try std.fmt.bufPrint(&speed_path_buf, "{s}/max_link_speed", .{dev_path}); + const raw_speed = try smi_sysfs.readString(allocator, io, speed_path); + + var width_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const width_path = try std.fmt.bufPrint(&width_path_buf, "{s}/max_link_width", .{dev_path}); + const width = try smi_sysfs.readInt(allocator, io, width_path); + const gen = try pcieGenFromSpeed(raw_speed); + return pcieBandwidthFromLink(gen, width) orelse error.NotFound; +} + +inline fn readHwmonInput(allocator: std.mem.Allocator, io: std.Io, hwmon: []const u8, comptime prefix: []const u8, label: []const u8) !u64 { + for (1..16) |idx| { + var label_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const label_path = try std.fmt.bufPrint(&label_path_buf, "{s}/{s}{d}_label", .{ hwmon, prefix, idx }); + const raw = smi_sysfs.readString(allocator, io, label_path) catch continue; + const trimmed = std.mem.trim(u8, raw, &std.ascii.whitespace); + if (!std.ascii.eqlIgnoreCase(trimmed, label)) continue; + return readNumberedInput(allocator, io, hwmon, prefix, idx); + } + return error.NotFound; +} + +inline fn readNumberedInput(allocator: std.mem.Allocator, io: std.Io, hwmon: []const u8, comptime prefix: []const u8, idx: usize) !u64 { + return readNumberedSuffix(allocator, io, hwmon, prefix, idx, "input"); +} + +inline fn readNumberedSuffix(allocator: std.mem.Allocator, io: std.Io, hwmon: []const u8, comptime prefix: []const u8, idx: usize, suffix: []const u8) !u64 { + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buf, "{s}/{s}{d}_{s}", .{ hwmon, prefix, idx, suffix }); + return smi_sysfs.readInt(allocator, io, path); +} + +inline fn readNumberedRaw(allocator: std.mem.Allocator, io: std.Io, hwmon: []const u8, comptime prefix: []const u8, idx: usize) !u64 { + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buf, "{s}/{s}{d}", .{ hwmon, prefix, idx }); + return smi_sysfs.readInt(allocator, io, path); +} + +inline fn readPwmFanSpeed(allocator: std.mem.Allocator, io: std.Io, hwmon: []const u8) !u64 { + for (1..16) |idx| { + const pwm = readNumberedRaw(allocator, io, hwmon, "pwm", idx) catch continue; + return pwmToPercent(pwm); + } + return error.NotFound; +} + +inline fn readRpmFanSpeedPercent(allocator: std.mem.Allocator, io: std.Io, hwmon: []const u8) !u64 { + for (1..16) |idx| { + const input = readNumberedSuffix(allocator, io, hwmon, "fan", idx, "input") catch continue; + const max = readNumberedSuffix(allocator, io, hwmon, "fan", idx, "max") catch continue; + if (max == 0) continue; + return @min(input * 100 / max, 100); + } + return error.NotFound; +} + +inline fn pwmToPercent(pwm: u64) u64 { + return @min((@min(pwm, 255) * 100 + 127) / 255, 100); +} + +inline fn readLargestResource(allocator: std.mem.Allocator, io: std.Io, path: []const u8) !u64 { + var file = try std.Io.Dir.openFile(.cwd(), io, path, .{ .mode = .read_only }); + defer file.close(io); + + var read_buf: [4096]u8 = undefined; + var reader = file.reader(io, &read_buf); + var line_writer: std.Io.Writer.Allocating = .init(allocator); + defer line_writer.deinit(); + + var largest: u64 = 0; + while (true) { + line_writer.clearRetainingCapacity(); + _ = reader.interface.streamDelimiter(&line_writer.writer, '\n') catch |err| switch (err) { + error.EndOfStream => break, + else => |e| return e, + }; + reader.interface.toss(1); + largest = @max(largest, resourceLineSize(line_writer.written()) orelse 0); + } + if (largest == 0) return error.NotFound; + return largest; +} + +inline fn resourceLineSize(line: []const u8) ?u64 { + var it = std.mem.tokenizeAny(u8, line, " \t"); + const base = parseHex(it.next() orelse return null) orelse return null; + const end = parseHex(it.next() orelse return null) orelse return null; + if (end < base) return null; + const size = end - base + 1; + return if (size >= 256 * 1024 * 1024) size else null; +} + +inline fn parseHex(raw: []const u8) ?u64 { + const trimmed = std.mem.trim(u8, raw, &std.ascii.whitespace); + const digits = if (std.mem.startsWith(u8, trimmed, "0x")) trimmed[2..] else trimmed; + return std.fmt.parseInt(u64, digits, 16) catch null; +} + +inline fn pcieGenFromSpeed(raw: []const u8) !u64 { + const trimmed = std.mem.trim(u8, raw, &std.ascii.whitespace); + if (std.mem.startsWith(u8, trimmed, "2.5")) return 1; + if (std.mem.startsWith(u8, trimmed, "5.0") or std.mem.startsWith(u8, trimmed, "5 ")) return 2; + if (std.mem.startsWith(u8, trimmed, "8.0") or std.mem.startsWith(u8, trimmed, "8 ")) return 3; + if (std.mem.startsWith(u8, trimmed, "16.0") or std.mem.startsWith(u8, trimmed, "16 ")) return 4; + if (std.mem.startsWith(u8, trimmed, "32.0") or std.mem.startsWith(u8, trimmed, "32 ")) return 5; + if (std.mem.startsWith(u8, trimmed, "64.0") or std.mem.startsWith(u8, trimmed, "64 ")) return 6; + return error.NotFound; +} + +inline fn pcieBandwidthFromLink(gen: u64, width: u64) ?u64 { + const lane_mb_s: u64 = switch (gen) { + 1 => 250, + 2 => 500, + 3 => 985, + 4 => 1969, + 5 => 3938, + 6 => 7563, + else => return null, + }; + return lane_mb_s * width; +} + +const FdinfoSample = struct { + mem_kib: ?u64 = null, + engine: ?EngineSample = null, + encoder: ?EngineSample = null, + decoder: ?EngineSample = null, +}; + +const ParsedFdinfo = struct { + bus_device_identifier: ?[bus_device_identifier_len]u8 = null, + drm_client_id: ?u64 = null, + sample: FdinfoSample = .{}, + resident_vram_kib: u64 = 0, + total_vram_kib: u64 = 0, + resident_memory_kib: u64 = 0, + total_memory_kib: u64 = 0, + cycle_counter: u64 = 0, + total_cycle_counter: u64 = 0, + engine_time_seen: bool = false, +}; + +inline fn parseFdinfoFile(io: std.Io, path: []const u8, timestamp_ns: u64) !ParsedFdinfo { + var read_buf: [8192]u8 = undefined; + const data = std.Io.Dir.readFile(.cwd(), io, path, &read_buf) catch return error.FileUnavailable; + + var parsed: ParsedFdinfo = .{}; + + var lines = std.mem.tokenizeScalar(u8, data, '\n'); + while (lines.next()) |line| { + parseFdinfoLine(&parsed, line, timestamp_ns); + } + + finishFdinfo(&parsed); + return parsed; +} + +inline fn finishFdinfo(self: *ParsedFdinfo) void { + if (self.resident_vram_kib > 0) { + self.sample.mem_kib = self.resident_vram_kib; + } else if (self.total_vram_kib > 0) { + self.sample.mem_kib = self.total_vram_kib; + } else if (self.resident_memory_kib > 0) { + self.sample.mem_kib = self.resident_memory_kib; + } else if (self.total_memory_kib > 0) { + self.sample.mem_kib = self.total_memory_kib; + } + + if (!self.engine_time_seen and self.total_cycle_counter > 0) { + self.sample.engine = .{ + .engine_ns = self.cycle_counter, + .timestamp_ns = self.total_cycle_counter, + }; + } +} + +inline fn parseFdinfoLine(parsed: *ParsedFdinfo, line: []const u8, timestamp_ns: u64) void { + if (std.mem.startsWith(u8, line, "drm-pdev:")) { + parsed.bus_device_identifier = parseBdf(line["drm-pdev:".len..]); + return; + } + + if (std.mem.startsWith(u8, line, "drm-client-id:")) { + parsed.drm_client_id = firstInt(line["drm-client-id:".len..]); + return; + } + + if (std.mem.startsWith(u8, line, "vram mem:")) { + parsed.total_vram_kib += memoryKiB(line["vram mem:".len..]); + return; + } + + if (std.mem.startsWith(u8, line, "drm-cycles-")) { + const colon = std.mem.indexOfScalar(u8, line, ':') orelse return; + parsed.cycle_counter += firstInt(line[colon + 1 ..]); + return; + } + + if (std.mem.startsWith(u8, line, "drm-total-cycles-")) { + const colon = std.mem.indexOfScalar(u8, line, ':') orelse return; + parsed.total_cycle_counter = @max(parsed.total_cycle_counter, firstInt(line[colon + 1 ..])); + return; + } + + if (std.mem.startsWith(u8, line, "drm-engine-capacity-")) { + return; + } + + if (std.mem.startsWith(u8, line, "drm-engine-")) { + const colon = std.mem.indexOfScalar(u8, line, ':') orelse return; + const engine_label = line["drm-engine-".len..colon]; + const value = firstInt(line[colon + 1 ..]); + parsed.engine_time_seen = true; + addEngineTime(&parsed.sample.engine, value, timestamp_ns); + if (containsAsciiIgnoreCase(engine_label, "encode")) { + addEngineTime(&parsed.sample.encoder, value, timestamp_ns); + } else if (containsAsciiIgnoreCase(engine_label, "decode")) { + addEngineTime(&parsed.sample.decoder, value, timestamp_ns); + } + return; + } + + if (std.mem.startsWith(u8, line, "drm-")) { + const colon = std.mem.indexOfScalar(u8, line, ':') orelse return; + const key = line[0..colon]; + const value = memoryKiB(line[colon + 1 ..]); + if (value == 0) return; + + if (std.mem.startsWith(u8, key, "drm-resident-vram")) { + parsed.resident_vram_kib += value; + return; + } + if (std.mem.startsWith(u8, key, "drm-total-vram") or std.mem.startsWith(u8, key, "drm-memory-vram")) { + parsed.total_vram_kib += value; + return; + } + if (std.mem.startsWith(u8, key, "drm-resident-")) { + parsed.resident_memory_kib += value; + return; + } + if (std.mem.startsWith(u8, key, "drm-total-")) { + parsed.total_memory_kib += value; + return; + } + if (std.mem.eql(u8, key, "drm-resident-memory")) { + parsed.resident_memory_kib += value; + return; + } + if (std.mem.eql(u8, key, "drm-total-memory")) { + parsed.total_memory_kib += value; + return; + } + } +} + +fn mergeProcessSample(dst: *ProcessSample, src: FdinfoSample) void { + if (src.mem_kib) |mem| dst.mem_kib = (dst.mem_kib orelse 0) + mem; + addEngineSample(&dst.engine, src.engine); + addEngineSample(&dst.encoder, src.encoder); + addEngineSample(&dst.decoder, src.decoder); +} + +fn addEngineSample(dst: *?EngineSample, src: ?EngineSample) void { + if (src) |engine| addEngineTime(dst, engine.engine_ns, engine.timestamp_ns); +} + +fn addEngineTime(dst: *?EngineSample, engine_ns: u64, timestamp_ns: u64) void { + const current = dst.* orelse EngineSample{ .engine_ns = 0, .timestamp_ns = timestamp_ns }; + dst.* = .{ + .engine_ns = current.engine_ns + engine_ns, + .timestamp_ns = timestamp_ns, + }; +} + +fn matchDevice(devices: []const Device, bus_device_identifier: [bus_device_identifier_len]u8) ?u16 { + for (devices, 0..) |dev, idx| { + if (dev.bus_device_identifier) |device_bus_device_identifier| { + if (std.mem.eql(u8, &device_bus_device_identifier, &bus_device_identifier)) return @intCast(idx); + } + } + return null; +} + +fn processKey(pid: u32, device_idx: u16) u64 { + return (@as(u64, device_idx) << 32) | pid; +} + +pub const BdfParts = struct { + domain: u32, + bus: u32, + device: u32, + function: u32, +}; + +pub fn formatBdf(parts: BdfParts) [bus_device_identifier_len]u8 { + var buf: [bus_device_identifier_len]u8 = undefined; + _ = std.fmt.bufPrint(&buf, "{x:0>4}:{x:0>2}:{x:0>2}.{x}", .{ + parts.domain, + parts.bus, + parts.device, + parts.function, + }) catch unreachable; + return buf; +} + +inline fn parseBdf(raw: []const u8) ?[bus_device_identifier_len]u8 { + const trimmed = std.mem.trim(u8, raw, " \t\n"); + var out: [bus_device_identifier_len]u8 = undefined; + if (trimmed.len == bus_device_identifier_len) { + @memcpy(&out, trimmed[0..bus_device_identifier_len]); + return out; + } + if (trimmed.len == "00:00.0".len) { + _ = std.fmt.bufPrint(&out, "0000:{s}", .{trimmed}) catch return null; + return out; + } + return null; +} + +inline fn firstInt(raw: []const u8) u64 { + var iter = std.mem.tokenizeAny(u8, raw, " \t"); + return std.fmt.parseInt(u64, iter.next() orelse return 0, 10) catch 0; +} + +inline fn memoryKiB(raw: []const u8) u64 { + var iter = std.mem.tokenizeAny(u8, raw, " \t"); + const value = std.fmt.parseInt(u64, iter.next() orelse return 0, 10) catch return 0; + const unit = iter.next() orelse return value; + if (std.ascii.eqlIgnoreCase(unit, "KiB")) return value; + if (std.ascii.eqlIgnoreCase(unit, "MiB")) return value * 1024; + if (std.ascii.eqlIgnoreCase(unit, "GiB")) return value * 1024 * 1024; + if (std.ascii.eqlIgnoreCase(unit, "B")) return value / 1024; + return value; +} + +inline fn containsAsciiIgnoreCase(haystack: []const u8, needle: []const u8) bool { + if (needle.len == 0) return true; + if (needle.len > haystack.len) return false; + for (0..haystack.len - needle.len + 1) |i| { + if (std.ascii.eqlIgnoreCase(haystack[i .. i + needle.len], needle)) return true; + } + return false; +} + +test "oneAPI fdinfo parser" { + var parsed: ParsedFdinfo = .{}; + parseFdinfoLine(&parsed, "drm-pdev:\t0000:03:00.0", 10); + parseFdinfoLine(&parsed, "drm-client-id:\t42", 10); + parseFdinfoLine(&parsed, "drm-total-memory:\t2048 KiB", 10); + parseFdinfoLine(&parsed, "drm-engine-render:\t100 ns", 10); + parseFdinfoLine(&parsed, "drm-engine-compute:\t50 ns", 10); + finishFdinfo(&parsed); + + try std.testing.expect(parsed.bus_device_identifier != null); + try std.testing.expectEqualSlices(u8, "0000:03:00.0", &parsed.bus_device_identifier.?); + try std.testing.expectEqual(@as(?u64, 42), parsed.drm_client_id); + try std.testing.expectEqual(@as(?u64, 2048), parsed.sample.mem_kib); + try std.testing.expectEqual(@as(u64, 150), parsed.sample.engine.?.engine_ns); + try std.testing.expectEqual(@as(?EngineSample, null), parsed.sample.encoder); + try std.testing.expectEqual(@as(?EngineSample, null), parsed.sample.decoder); +} + +test "oneAPI fdinfo parser classifies explicit media engines" { + var parsed: ParsedFdinfo = .{}; + parseFdinfoLine(&parsed, "drm-engine-video-decode:\t100 ns", 10); + parseFdinfoLine(&parsed, "drm-engine-media-encode:\t50 ns", 10); + parseFdinfoLine(&parsed, "drm-engine-VIDEO-DECODE1:\t25 ns", 10); + finishFdinfo(&parsed); + + try std.testing.expectEqual(@as(u64, 175), parsed.sample.engine.?.engine_ns); + try std.testing.expectEqual(@as(u64, 50), parsed.sample.encoder.?.engine_ns); + try std.testing.expectEqual(@as(u64, 125), parsed.sample.decoder.?.engine_ns); +} + +test "oneAPI fdinfo parser leaves generic video engines unclassified" { + var parsed: ParsedFdinfo = .{}; + parseFdinfoLine(&parsed, "drm-engine-video:\t100 ns", 10); + parseFdinfoLine(&parsed, "drm-engine-vcs0:\t50 ns", 10); + parseFdinfoLine(&parsed, "drm-engine-vecs0:\t25 ns", 10); + finishFdinfo(&parsed); + + try std.testing.expectEqual(@as(u64, 175), parsed.sample.engine.?.engine_ns); + try std.testing.expectEqual(@as(?EngineSample, null), parsed.sample.encoder); + try std.testing.expectEqual(@as(?EngineSample, null), parsed.sample.decoder); +} + +test "oneAPI fdinfo parser cycle counters" { + var parsed: ParsedFdinfo = .{}; + parseFdinfoLine(&parsed, "drm-pdev:\t0000:03:00.0", 10); + parseFdinfoLine(&parsed, "drm-cycles-rcs:\t10", 10); + parseFdinfoLine(&parsed, "drm-total-cycles-rcs:\t100", 10); + parseFdinfoLine(&parsed, "drm-engine-capacity-vcs:\t2", 10); + parseFdinfoLine(&parsed, "drm-cycles-ccs:\t30", 10); + parseFdinfoLine(&parsed, "drm-total-cycles-ccs:\t100", 10); + finishFdinfo(&parsed); + + try std.testing.expectEqual(@as(u64, 40), parsed.sample.engine.?.engine_ns); + try std.testing.expectEqual(@as(u64, 100), parsed.sample.engine.?.timestamp_ns); +} + +test "oneAPI fdinfo parser xe vram regions" { + var parsed: ParsedFdinfo = .{}; + parseFdinfoLine(&parsed, "drm-pdev:\t0000:03:00.0", 10); + parseFdinfoLine(&parsed, "drm-total-system:\t184 MiB", 10); + parseFdinfoLine(&parsed, "drm-total-vram0:\t4096 KiB", 10); + parseFdinfoLine(&parsed, "drm-resident-vram0:\t3072 KiB", 10); + finishFdinfo(&parsed); + + try std.testing.expectEqual(@as(?u64, 3072), parsed.sample.mem_kib); +} + +test "oneAPI fdinfo parser falls back to total vram" { + var parsed: ParsedFdinfo = .{}; + parseFdinfoLine(&parsed, "drm-total-vram0:\t2 MiB", 10); + finishFdinfo(&parsed); + + try std.testing.expectEqual(@as(?u64, 2048), parsed.sample.mem_kib); +} + +test "oneAPI DRM clients dedup per device" { + var seen_device_clients: std.AutoHashMapUnmanaged(struct { u16, u64 }, void) = .{}; + defer seen_device_clients.deinit(std.testing.allocator); + + try std.testing.expectEqual(false, (try seen_device_clients.getOrPut(std.testing.allocator, .{ 0, 42 })).found_existing); + try std.testing.expectEqual(true, (try seen_device_clients.getOrPut(std.testing.allocator, .{ 0, 42 })).found_existing); + try std.testing.expectEqual(false, (try seen_device_clients.getOrPut(std.testing.allocator, .{ 1, 42 })).found_existing); + try std.testing.expectEqual(false, (try seen_device_clients.getOrPut(std.testing.allocator, .{ 0, 43 })).found_existing); +} + +test "oneAPI process utilization delta" { + try std.testing.expectEqual(@as(?u16, 50), processUtil(.{ .engine_ns = 100, .timestamp_ns = 1000 }, .{ .engine_ns = 600, .timestamp_ns = 2000 })); + try std.testing.expectEqual(@as(?u16, 1), processUtil(.{ .engine_ns = 100, .timestamp_ns = 1000 }, .{ .engine_ns = 101, .timestamp_ns = 2000 })); + try std.testing.expectEqual(@as(?u16, null), processUtil(null, .{ .engine_ns = 600, .timestamp_ns = 2000 })); +} + +test "oneAPI PWM fan speed conversion" { + try std.testing.expectEqual(@as(u64, 0), pwmToPercent(0)); + try std.testing.expectEqual(@as(u64, 50), pwmToPercent(128)); + try std.testing.expectEqual(@as(u64, 100), pwmToPercent(255)); + try std.testing.expectEqual(@as(u64, 100), pwmToPercent(300)); +} + +test "oneAPI process previous keeps engine samples" { + var usage: ProcessUsage = .{}; + defer usage.deinit(std.testing.allocator); + + const engine_key = processKey(10, 1); + const mem_only_key = processKey(20, 1); + try usage.put(std.testing.allocator, engine_key, .{ + .pid = 10, + .device_idx = 1, + .engine = .{ .engine_ns = 100, .timestamp_ns = 1000 }, + }); + try usage.put(std.testing.allocator, mem_only_key, .{ + .pid = 20, + .device_idx = 1, + .mem_kib = 2048, + }); + + var previous: ProcessPrevious = .{}; + defer previous.deinit(std.testing.allocator); + + try saveProcessPrevious(std.testing.allocator, &previous, &usage); + + try std.testing.expectEqual(@as(u32, 1), previous.count()); + try std.testing.expectEqual(@as(?EngineSample, .{ .engine_ns = 100, .timestamp_ns = 1000 }), previous.get(engine_key)); + try std.testing.expectEqual(@as(?EngineSample, null), previous.get(mem_only_key)); +} + +test "oneAPI power delta from hwmon energy" { + try std.testing.expectEqual(@as(?u64, 20_000), milliwattsFromEnergyDelta(1_000, 11_000, 1_000, 501_000)); + try std.testing.expectEqual(@as(?u64, null), milliwattsFromEnergyDelta(11_000, 1_000, 1_000, 501_000)); + try std.testing.expectEqual(@as(?u64, null), milliwattsFromEnergyDelta(1_000, 11_000, 501_000, 1_000)); +} + +test "oneAPI resource line chooses large memory BARs" { + try std.testing.expectEqual(@as(?u64, 34_359_738_368), resourceLineSize("0x0000048800000000 0x0000048fffffffff 0x000000000014220c")); + try std.testing.expectEqual(@as(?u64, null), resourceLineSize("0x00000000f6c00000 0x00000000f6dfffff 0x0000000000046200")); +} + +test "oneAPI PCIe speed maps to generation" { + try std.testing.expectEqual(@as(u64, 1), try pcieGenFromSpeed("2.5 GT/s PCIe")); + try std.testing.expectEqual(@as(u64, 4), try pcieGenFromSpeed("16.0 GT/s PCIe")); + try std.testing.expectError(error.NotFound, pcieGenFromSpeed("Unknown")); +} + +test "oneAPI PCIe bandwidth derives from generation and width" { + try std.testing.expectEqual(@as(?u64, 250), pcieBandwidthFromLink(1, 1)); + try std.testing.expectEqual(@as(?u64, 63_008), pcieBandwidthFromLink(5, 16)); + try std.testing.expectEqual(@as(?u64, null), pcieBandwidthFromLink(0, 16)); +} diff --git a/bin/zml-smi/platforms/oneapi/oneapi.zig b/bin/zml-smi/platforms/oneapi/oneapi.zig new file mode 100644 index 00000000..2e1e5deb --- /dev/null +++ b/bin/zml-smi/platforms/oneapi/oneapi.zig @@ -0,0 +1,248 @@ +const std = @import("std"); +const Monitor = @import("monitor.zig"); +const smi_sysfs = @import("zml-smi/sysfs"); + +const OneApi = @This(); + +pub const Handle = Monitor.Handle; + +monitor: Monitor, +io: std.Io, + +pub fn init(allocator: std.mem.Allocator, io: std.Io) !OneApi { + return .{ + .monitor = try Monitor.init(allocator, io), + .io = io, + }; +} + +pub fn handleByIndex(self: *const OneApi, device_id: usize) !Handle { + if (device_id >= self.monitor.devices.len) return error.NotFound; + return @fromBackingInt(@intCast(@as(u32, @intCast(device_id)))); +} + +pub fn name(self: *const OneApi, allocator: std.mem.Allocator, handle: Handle) ![]const u8 { + if (self.monitor.device(handle)) |dev| { + if (dev.bus_device_identifier) |bdf| { + if (try lspciName(allocator, self.io, bdf[0..])) |display_name| { + return display_name; + } + } + } else |_| {} + + const id = try self.monitor.deviceId(allocator, handle); + return try std.fmt.allocPrint(allocator, "Intel GPU {s}", .{id}); +} + +fn lspciName(allocator: std.mem.Allocator, io: std.Io, bdf: []const u8) !?[]const u8 { + const result = std.process.run(allocator, io, .{ + .argv = &.{ "lspci", "-Dnn", "-s", bdf }, + .stdout_limit = .limited(4096), + .stderr_limit = .limited(1024), + .reserve_amount = 256, + }) catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + else => return null, + }; + defer allocator.free(result.stdout); + defer allocator.free(result.stderr); + + switch (result.term) { + .exited => |code| if (code != 0) return null, + else => return null, + } + + return parseLspciName(allocator, result.stdout); +} + +fn parseLspciName(allocator: std.mem.Allocator, output: []const u8) !?[]const u8 { + var lines = std.mem.tokenizeScalar(u8, output, '\n'); + while (lines.next()) |line| { + const trimmed = std.mem.trim(u8, line, &std.ascii.whitespace); + const header_end = std.mem.indexOf(u8, trimmed, "]: ") orelse continue; + const header = trimmed[0 .. header_end + 1]; + if (!containsAsciiIgnoreCase(header, "VGA") and !containsAsciiIgnoreCase(header, "Display")) continue; + + const name_start = header_end + "]: ".len; + const name_end = std.mem.lastIndexOf(u8, trimmed[name_start..], " [8086:") orelse + std.mem.indexOf(u8, trimmed[name_start..], " (rev ") orelse + trimmed[name_start..].len; + const raw_name = std.mem.trim(u8, trimmed[name_start .. name_start + name_end], &std.ascii.whitespace); + if (raw_name.len == 0 or std.mem.startsWith(u8, raw_name, "Intel Corporation Device ")) return null; + return try allocator.dupe(u8, raw_name); + } + return null; +} + +fn containsAsciiIgnoreCase(haystack: []const u8, needle: []const u8) bool { + if (needle.len == 0) return true; + if (needle.len > haystack.len) return false; + for (0..haystack.len - needle.len + 1) |idx| { + if (std.ascii.eqlIgnoreCase(haystack[idx .. idx + needle.len], needle)) return true; + } + return false; +} + +test "parse lspci display name" { + const output = + \\0000:84:00.0 VGA compatible controller [0300]: Intel Corporation Battlemage G31 [Arc Pro B70] [8086:e223] (rev 04) + ; + const name_ = (try parseLspciName(std.testing.allocator, output)).?; + defer std.testing.allocator.free(name_); + try std.testing.expectEqualStrings("Intel Corporation Battlemage G31 [Arc Pro B70]", name_); +} + +test "parse lspci display name skips generic device ids" { + const output = + \\0000:84:00.0 Display controller [0380]: Intel Corporation Device e223 [8086:e223] (rev 04) + ; + try std.testing.expect(try parseLspciName(std.testing.allocator, output) == null); +} + +pub fn driverVersion(self: *const OneApi, allocator: std.mem.Allocator) ![]const u8 { + const version = smi_sysfs.readString(allocator, self.io, "/sys/module/xe/version") catch + smi_sysfs.readString(allocator, self.io, "/sys/module/xe/srcversion") catch + smi_sysfs.readString(allocator, self.io, "/sys/module/i915/version") catch + try smi_sysfs.readString(allocator, self.io, "/sys/module/i915/srcversion"); + const trimmed = std.mem.trim(u8, version, &std.ascii.whitespace); + if (trimmed.len == 0) return error.NotFound; + return trimmed; +} + +pub fn powerUsage(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { + const dev = block: { + const idx = @backingInt(handle); + if (idx >= self.monitor.devices.len) return error.NotFound; + break :block &self.monitor.devices[idx]; + }; + const current = Monitor.readEnergy(allocator, self.io, dev) catch |err| { + dev.energy_prev = null; + return err; + }; + const power = Monitor.powerMilliwatts(dev.energy_prev, current) orelse { + dev.energy_prev = current; + return error.NotFound; + }; + dev.energy_prev = current; + return power; +} + +pub fn powerLimit(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { + const dev = try self.monitor.device(handle); + return Monitor.readPowerLimit(allocator, self.io, dev); +} + +pub fn temperature(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { + const dev = try self.monitor.device(handle); + return Monitor.readTemperature(allocator, self.io, dev); +} + +pub fn fanSpeed(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { + const dev = try self.monitor.device(handle); + return Monitor.readFanSpeed(allocator, self.io, dev); +} + +pub fn gpuUtil(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { + const dev = block: { + const idx = @backingInt(handle); + if (idx >= self.monitor.devices.len) return error.NotFound; + break :block &self.monitor.devices[idx]; + }; + + var usage = Monitor.collectDeviceUsage(allocator, self.io, self.monitor.devices) catch { + dev.activity_prev = null; + return 0; + }; + defer usage.deinit(allocator); + + const dev_idx: u16 = @intCast(@backingInt(handle)); + const current = if (usage.get(dev_idx)) |sample| sample.engine else null; + const util = Monitor.processUtil(dev.activity_prev, current) orelse 0; + dev.activity_prev = current; + return util; +} + +pub fn encoderUtil(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { + const dev = block: { + const idx = @backingInt(handle); + if (idx >= self.monitor.devices.len) return error.NotFound; + break :block &self.monitor.devices[idx]; + }; + + var usage = Monitor.collectDeviceUsage(allocator, self.io, self.monitor.devices) catch |err| { + dev.encoder_prev = null; + return err; + }; + defer usage.deinit(allocator); + + const dev_idx: u16 = @intCast(@backingInt(handle)); + const current = if (usage.get(dev_idx)) |sample| sample.encoder else null; + const util = Monitor.processUtil(dev.encoder_prev, current) orelse { + dev.encoder_prev = current; + return error.NotFound; + }; + dev.encoder_prev = current; + return util; +} + +pub fn decoderUtil(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { + const dev = block: { + const idx = @backingInt(handle); + if (idx >= self.monitor.devices.len) return error.NotFound; + break :block &self.monitor.devices[idx]; + }; + + var usage = Monitor.collectDeviceUsage(allocator, self.io, self.monitor.devices) catch |err| { + dev.decoder_prev = null; + return err; + }; + defer usage.deinit(allocator); + + const dev_idx: u16 = @intCast(@backingInt(handle)); + const current = if (usage.get(dev_idx)) |sample| sample.decoder else null; + const util = Monitor.processUtil(dev.decoder_prev, current) orelse { + dev.decoder_prev = current; + return error.NotFound; + }; + dev.decoder_prev = current; + return util; +} + +pub fn clockGraphics(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { + const dev = try self.monitor.device(handle); + return Monitor.readClockGraphics(allocator, self.io, dev); +} + +pub fn maxClockGraphics(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { + const dev = try self.monitor.device(handle); + return Monitor.readMaxClockGraphics(allocator, self.io, dev); +} + +pub fn memUsed(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { + const dev_idx: u16 = @intCast(@backingInt(handle)); + + var usage = Monitor.collectDeviceUsage(allocator, self.io, self.monitor.devices) catch return 0; + defer usage.deinit(allocator); + + return if (usage.get(dev_idx)) |sample| sample.mem_kib * 1024 else 0; +} + +pub fn memTotal(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { + const dev = try self.monitor.device(handle); + return Monitor.readMemTotal(allocator, self.io, dev); +} + +pub fn pcieLinkGen(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { + const dev = try self.monitor.device(handle); + return Monitor.readPcieLinkGen(allocator, self.io, dev); +} + +pub fn pcieLinkWidth(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { + const dev = try self.monitor.device(handle); + return Monitor.readPcieLinkWidth(allocator, self.io, dev); +} + +pub fn pcieBandwidth(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { + const dev = try self.monitor.device(handle); + return Monitor.readPcieBandwidth(allocator, self.io, dev); +} diff --git a/bin/zml-smi/platforms/oneapi/process.zig b/bin/zml-smi/platforms/oneapi/process.zig new file mode 100644 index 00000000..c4d0c719 --- /dev/null +++ b/bin/zml-smi/platforms/oneapi/process.zig @@ -0,0 +1,45 @@ +const std = @import("std"); +const OneApi = @import("oneapi.zig"); +const Monitor = @import("monitor.zig"); +const pi = @import("zml-smi/info").process_info; +const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); +const Collector = @import("zml-smi/collector").Collector; + +pub fn init(collector: *Collector, list: *ProcessDoubleBuffer, oneapi: *OneApi, dev_offset: u16) !void { + const state = try collector.arena.create(Monitor.ProcessPrevious); + state.* = .{}; + + const previous_usage: ?Monitor.ProcessUsage = Monitor.collectProcessUsage(collector.gpa, collector.io, oneapi.monitor.devices) catch null; + if (previous_usage) |usage_value| { + var usage = usage_value; + defer usage.deinit(collector.gpa); + Monitor.saveProcessPrevious(collector.arena, state, &usage) catch {}; + } + + try collector.spawnPoll(pollOnce, .{ collector.gpa, collector.arena, collector.io, list, oneapi, dev_offset, state }); +} + +fn pollOnce(allocator: std.mem.Allocator, arena: std.mem.Allocator, io: std.Io, list: *ProcessDoubleBuffer, oneapi: *OneApi, dev_offset: u16, state: *Monitor.ProcessPrevious) void { + const back = list.back(); + back.clearRetainingCapacity(); + + var usage = Monitor.collectProcessUsage(allocator, io, oneapi.monitor.devices) catch { + list.swap(); + return; + }; + defer usage.deinit(allocator); + + var it = usage.iterator(); + while (it.next()) |entry| { + const sample = entry.value_ptr.*; + back.append(allocator, .{ + .pid = sample.pid, + .device_idx = @intCast(sample.device_idx + dev_offset), + .dev_mem_kib = sample.mem_kib, + .dev_util_percent = Monitor.processUtil(state.get(entry.key_ptr.*), sample.engine), + }) catch break; + } + + Monitor.saveProcessPrevious(arena, state, &usage) catch {}; + list.swap(); +} diff --git a/bin/zml-smi/platforms/tpu/BUILD.bazel b/bin/zml-smi/platforms/tpu/BUILD.bazel new file mode 100644 index 00000000..a65377b3 --- /dev/null +++ b/bin/zml-smi/platforms/tpu/BUILD.bazel @@ -0,0 +1,30 @@ +load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library") + +cc_library( + name = "tpuinfo", + srcs = ["tpuinfo.cc"], + hdrs = ["tpuinfo.h"], + deps = [ + "@cloud_accelerator_diagnostics//:tpu_metric_service_cc_grpc", + "@grpc//:grpc++_unsecure", + ], +) + +zig_library( + name = "tpu", + import_name = "zml-smi/platforms/tpu", + main = "metrics.zig", + srcs = [ + "process.zig", + "tpuinfo.zig", + ], + visibility = ["//visibility:public"], + deps = [ + ":tpuinfo", + "//bin/zml-smi/collector", + "//bin/zml-smi/info", + "//bin/zml-smi/utils:double_buffer", + "//bin/zml-smi/utils:sysfs", + ], +) diff --git a/bin/zml-smi/platforms/tpu/metrics.zig b/bin/zml-smi/platforms/tpu/metrics.zig new file mode 100644 index 00000000..9410b3f8 --- /dev/null +++ b/bin/zml-smi/platforms/tpu/metrics.zig @@ -0,0 +1,168 @@ +const std = @import("std"); +const sysfs = @import("zml-smi/sysfs"); +const tpuinfo = @import("tpuinfo.zig"); +const device_info = @import("zml-smi/info").device_info; +const DeviceInfo = device_info.DeviceInfo; +const TpuInfo = device_info.TpuInfo; +const Collector = @import("zml-smi/collector").Collector; +const tpu_process = @import("process.zig"); + +const address = "localhost:8431"; + +pub fn start(collector: *Collector) !void { + const chip = scanPciChips(collector.arena, collector.io) orelse return error.TpuUnavailable; + const device_count = chip.chip_count * chip.devices_per_chip; + var tpu_infos: std.ArrayList(*DeviceInfo) = .empty; + const dev_offset: u8 = @intCast(collector.device_infos.items.len); + + for (0..device_count) |_| { + const chip_name = try collector.arena.dupe(u8, chip.name); + const initial: TpuInfo = .{ .name = chip_name }; + const info = try collector.addDevice(.{ .tpu = .{ .values = .{ initial, initial } } }); + try tpu_infos.append(collector.arena, info); + } + + try collector.spawnPoll(pollAllDevices, .{tpu_infos.items}); + + if (tpu_infos.items.len > 0) { + const processes = try collector.createProcessList(); + try tpu_process.init(collector, chip.devices_per_chip, tpu_infos.items, processes, dev_offset); + } +} + +const MetricKind = enum { int, double }; + +fn pollAllDevices(devs: []*DeviceInfo) void { + var mem_used: [tpuinfo.max_devices]?u64 = undefined; + var mem_total: [tpuinfo.max_devices]?u64 = undefined; + var util: [tpuinfo.max_devices]?u64 = undefined; + + for (devs, 0..) |info, i| { + const front = info.tpu.front(); + mem_used[i] = front.mem_used_bytes; + mem_total[i] = front.mem_total_bytes; + util[i] = front.util_percent; + } + + queryRaw("tpu.runtime.hbm.memory.usage.bytes", .int, mem_used[0..devs.len]); + queryRaw("tpu.runtime.hbm.memory.total.bytes", .int, mem_total[0..devs.len]); + queryRaw("tpu.runtime.tensorcore.dutycycle.percent", .double, util[0..devs.len]); + + for (devs, 0..) |info, i| { + const back = info.tpu.back(); + back.* = info.tpu.front().*; + back.mem_used_bytes = mem_used[i]; + back.mem_total_bytes = mem_total[i]; + back.util_percent = util[i]; + info.tpu.swap(); + } +} + +fn queryRaw(metric_name: [:0]const u8, comptime kind: MetricKind, out: []?u64) void { + var ids: [tpuinfo.max_devices]c_longlong = undefined; + switch (kind) { + .int => { + var vals: [tpuinfo.max_devices]c_longlong = undefined; + const n = tpuinfo.queryInt(address, metric_name, &ids, &vals) catch { + @memset(out, null); + return; + }; + + for (ids[0..n], vals[0..n]) |id, val| { + if (id >= 0 and id < out.len) out[@intCast(id)] = @intCast(val); + } + }, + .double => { + var vals: [tpuinfo.max_devices]f64 = undefined; + const n = tpuinfo.queryDouble(address, metric_name, &ids, &vals) catch { + @memset(out, null); + return; + }; + + for (ids[0..n], vals[0..n]) |id, val| { + if (id >= 0 and id < out.len and std.math.isFinite(val)) + out[@intCast(id)] = @intFromFloat(@round(val)); + } + }, + } +} + +// -- Discover available chips +// We need to scan /sys/bus/pci/devices for Google TPU chips. +// From: https://github.com/AI-Hypercomputer/cloud-accelerator-diagnostics/blob/c94f463dd3bf99eaecf4f5af46f87cc2a5ff1b22/tpu_info/tpu_info/device.py#L43-L92 + +const ChipInfo = struct { + name: []const u8, + devices_per_chip: u32, + chip_count: u32, +}; + +const google_pci_vendor_id = "0x1ae0"; +const pci_base = "/sys/bus/pci/devices"; + +const ChipEntry = struct { + device_id: []const u8, + subsystem_id: ?[]const u8 = null, + name: []const u8, + devices_per_chip: u32, +}; + +const chip_table = [_]ChipEntry{ + .{ .device_id = "0x0027", .subsystem_id = "0x004e", .name = "TPU v2", .devices_per_chip = 2 }, + .{ .device_id = "0x0027", .subsystem_id = "0x004f", .name = "TPU v3", .devices_per_chip = 2 }, + .{ .device_id = "0x005e", .name = "TPU v4", .devices_per_chip = 1 }, + .{ .device_id = "0x0063", .name = "TPU v5e", .devices_per_chip = 1 }, + .{ .device_id = "0x0062", .name = "TPU v5p", .devices_per_chip = 1 }, + .{ .device_id = "0x006f", .name = "TPU v6e", .devices_per_chip = 1 }, + .{ .device_id = "0x0076", .name = "TPU7x", .devices_per_chip = 2 }, +}; + +fn scanPciChips(allocator: std.mem.Allocator, io: std.Io) ?ChipInfo { + var pci_dir = std.Io.Dir.openDir(.cwd(), io, pci_base, .{ .iterate = true }) catch return null; + defer pci_dir.close(io); + + var it = pci_dir.iterate(); + var chip_count: u32 = 0; + var result: ChipInfo = undefined; + + while (it.next(io) catch null) |entry| { + const vendor = readSysfs(allocator, io, entry.name, "vendor") catch continue; + if (!std.mem.eql(u8, vendor, google_pci_vendor_id)) { + continue; + } + + const device_raw = readSysfs(allocator, io, entry.name, "device") catch continue; + const subsystem_raw: ?[]const u8 = readSysfs(allocator, io, entry.name, "subsystem_device") catch null; + + for (chip_table) |chip| { + const dev_match = std.mem.eql(u8, device_raw, chip.device_id); + const sub_match = if (chip.subsystem_id) |expected_sub| + if (subsystem_raw) |actual_sub| std.mem.eql(u8, actual_sub, expected_sub) else false + else + true; + + if (dev_match and sub_match) { + result = .{ + .name = chip.name, + .devices_per_chip = chip.devices_per_chip, + .chip_count = undefined, + }; + + chip_count += 1; + break; + } + } + } + + if (chip_count > 0) { + result.chip_count = chip_count; + return result; + } + + return null; +} + +fn readSysfs(allocator: std.mem.Allocator, io: std.Io, slot: []const u8, file: []const u8) ![]const u8 { + const path = try std.fmt.allocPrint(allocator, pci_base ++ "/{s}/{s}", .{ slot, file }); + return sysfs.readString(allocator, io, path); +} diff --git a/bin/zml-smi/platforms/tpu/process.zig b/bin/zml-smi/platforms/tpu/process.zig new file mode 100644 index 00000000..cca8f6a1 --- /dev/null +++ b/bin/zml-smi/platforms/tpu/process.zig @@ -0,0 +1,73 @@ +const std = @import("std"); +const smi_info = @import("zml-smi/info"); +const pi = smi_info.process_info; +const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); +const DeviceInfo = smi_info.device_info.DeviceInfo; +const Collector = @import("zml-smi/collector").Collector; + +pub fn init(collector: *Collector, devices_per_chip: u32, device_infos: []*DeviceInfo, list: *ProcessDoubleBuffer, dev_offset: u16) !void { + try collector.spawnPoll(pollOnce, .{ collector.gpa, collector.io, list, devices_per_chip, device_infos, dev_offset }); +} + +fn pollOnce(allocator: std.mem.Allocator, io: std.Io, list: *ProcessDoubleBuffer, devices_per_chip: u32, device_infos: []*DeviceInfo, dev_offset: u16) void { + scan(allocator, io, list.back(), devices_per_chip, device_infos, dev_offset); + list.swap(); +} + +fn scan(allocator: std.mem.Allocator, io: std.Io, back: *std.ArrayList(pi.ProcessInfo), devices_per_chip: u32, infos: []*DeviceInfo, dev_offset: u16) void { + back.clearRetainingCapacity(); + + var proc_dir = std.Io.Dir.openDirAbsolute(io, "/proc", .{ .iterate = true }) catch return; + defer proc_dir.close(io); + + var proc_it = proc_dir.iterate(); + while (proc_it.next(io) catch null) |proc_entry| { + const pid = std.fmt.parseInt(u32, proc_entry.name, 10) catch continue; + + var fd_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const fd_sub = std.fmt.bufPrint(&fd_path_buf, "{d}/fd", .{pid}) catch continue; + var fd_dir = proc_dir.openDir(io, fd_sub, .{ .iterate = true }) catch continue; + defer fd_dir.close(io); + + var fd_it = fd_dir.iterate(); + while (fd_it.next(io) catch null) |fd_entry| { + var link_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const len = fd_dir.readLink(io, fd_entry.name, &link_buf) catch continue; + const target = link_buf[0..len]; + + if (parseChipIndex(target)) |chip_idx| { + const base: u16 = @intCast(chip_idx * devices_per_chip); + for (0..devices_per_chip) |d| { + const local_idx: u16 = @intCast(base + d); + var info: pi.ProcessInfo = .{ .pid = pid, .device_idx = local_idx + dev_offset }; + + if (local_idx < infos.len) { + const tpu = infos[local_idx].tpu.front().*; + + if (tpu.mem_used_bytes) |mem| { + info.dev_mem_kib = @intCast(mem / 1024); + } + + if (tpu.util_percent) |util| { + info.dev_util_percent = @intCast(util); + } + } + + back.append(allocator, info) catch return; + } + } + } + } +} + +fn parseChipIndex(target: []const u8) ?u32 { + if (std.mem.startsWith(u8, target, "/dev/accel")) { + return std.fmt.parseInt(u32, target["/dev/accel".len..], 10) catch null; + } + + if (std.mem.startsWith(u8, target, "/dev/vfio/")) { + return std.fmt.parseInt(u32, target["/dev/vfio/".len..], 10) catch null; + } + + return null; +} diff --git a/bin/zml-smi/platforms/tpu/tpuinfo.cc b/bin/zml-smi/platforms/tpu/tpuinfo.cc new file mode 100644 index 00000000..a1e3fbda --- /dev/null +++ b/bin/zml-smi/platforms/tpu/tpuinfo.cc @@ -0,0 +1,84 @@ +#include "tpuinfo.h" + +#include +#include +#include +#include + +#include + +#include "tpu_metric_service.grpc.pb.h" +#include "tpu_metric_service.pb.h" + +using tpu::monitoring::runtime::MetricRequest; +using tpu::monitoring::runtime::MetricResponse; +using tpu::monitoring::runtime::RuntimeMetricService; + +static MetricResponse FetchMetric(const char* address, const char* metric_name, + bool* ok) { + auto channel = + grpc::CreateChannel(address, grpc::InsecureChannelCredentials()); + auto stub = RuntimeMetricService::NewStub(channel); + + grpc::ClientContext ctx; + ctx.set_deadline(std::chrono::system_clock::now() + std::chrono::seconds(1)); + + MetricRequest req; + req.set_metric_name(metric_name); + + MetricResponse resp; + auto status = stub->GetRuntimeMetric(&ctx, req, &resp); + *ok = status.ok(); + + return resp; +} + +extern "C" int tpu_query_int(const char* address, const char* metric_name, + long long* device_ids, long long* values, + int max_n) { + bool ok; + auto resp = FetchMetric(address, metric_name, &ok); + if (!ok) return -1; + + std::map sorted; + for (const auto& m : resp.metric().metrics()) { + int id = static_cast(m.attribute().value().int_attr()); + sorted[id] = m.gauge().as_int(); + } + + int i = 0; + for (const auto& [id, val] : sorted) { + if (i >= max_n) break; + + device_ids[i] = id; + values[i] = val; + ++i; + } + + return i; +} + +extern "C" int tpu_query_double(const char* address, const char* metric_name, + long long* device_ids, double* values, + int max_n) { + bool ok; + auto resp = FetchMetric(address, metric_name, &ok); + if (!ok) return -1; + + std::map sorted; + for (const auto& m : resp.metric().metrics()) { + int id = static_cast(m.attribute().value().int_attr()); + sorted[id] = m.gauge().as_double(); + } + + int i = 0; + for (const auto& [id, val] : sorted) { + if (i >= max_n) break; + + device_ids[i] = id; + values[i] = val; + ++i; + } + + return i; +} diff --git a/bin/zml-smi/platforms/tpu/tpuinfo.h b/bin/zml-smi/platforms/tpu/tpuinfo.h new file mode 100644 index 00000000..f2dc7db6 --- /dev/null +++ b/bin/zml-smi/platforms/tpu/tpuinfo.h @@ -0,0 +1,20 @@ +#ifndef TPUINFO_H_ +#define TPUINFO_H_ + +#define TPUINFO_MAX_DEVICES 64 + +#ifdef __cplusplus +extern "C" { +#endif + +int tpu_query_int(const char* address, const char* metric_name, + long long* device_ids, long long* values, int max_n); + +int tpu_query_double(const char* address, const char* metric_name, + long long* device_ids, double* values, int max_n); + +#ifdef __cplusplus +} +#endif + +#endif /* TPUINFO_H_ */ diff --git a/bin/zml-smi/platforms/tpu/tpuinfo.zig b/bin/zml-smi/platforms/tpu/tpuinfo.zig new file mode 100644 index 00000000..1baf62df --- /dev/null +++ b/bin/zml-smi/platforms/tpu/tpuinfo.zig @@ -0,0 +1,37 @@ +const std = @import("std"); +const c = @import("c"); + +pub const max_devices: usize = c.TPUINFO_MAX_DEVICES; + +pub const Error = error{ + TpuUnavailable, + QueryFailed, +}; + +pub fn queryInt( + address: [:0]const u8, + metric_name: [:0]const u8, + device_ids: []c_longlong, + values: []c_longlong, +) Error!u32 { + const n = c.tpu_query_int(address.ptr, metric_name.ptr, device_ids.ptr, values.ptr, @intCast(device_ids.len)); + if (n < 0) { + return error.QueryFailed; + } + + return @intCast(n); +} + +pub fn queryDouble( + address: [:0]const u8, + metric_name: [:0]const u8, + device_ids: []c_longlong, + values: []f64, +) Error!u32 { + const n = c.tpu_query_double(address.ptr, metric_name.ptr, device_ids.ptr, values.ptr, @intCast(device_ids.len)); + if (n < 0) { + return error.QueryFailed; + } + + return @intCast(n); +} diff --git a/bin/zml-smi/prometheus/BUILD.bazel b/bin/zml-smi/prometheus/BUILD.bazel new file mode 100644 index 00000000..c00a26b5 --- /dev/null +++ b/bin/zml-smi/prometheus/BUILD.bazel @@ -0,0 +1,16 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "prometheus", + import_name = "zml-smi/prometheus", + main = "prometheus.zig", + srcs = [ + "server.zig", + "exposition.zig", + ], + visibility = ["//visibility:public"], + deps = [ + "//bin/zml-smi/info", + "//bin/zml-smi/utils:double_buffer", + ], +) diff --git a/bin/zml-smi/prometheus/exposition.zig b/bin/zml-smi/prometheus/exposition.zig new file mode 100644 index 00000000..54d48cb6 --- /dev/null +++ b/bin/zml-smi/prometheus/exposition.zig @@ -0,0 +1,159 @@ +const std = @import("std"); +const smi_info = @import("zml-smi/info"); +const DeviceInfo = smi_info.device_info.DeviceInfo; +const HostInfo = smi_info.host_info.HostInfo; + +const device_metrics = .{ + // Shared across all device types + .{ .field = "util_percent", .metric = "zml_device_utilization_percent", .help = "Device utilization percentage" }, + .{ .field = "mem_used_bytes", .metric = "zml_device_memory_used_bytes", .help = "Device memory used in bytes" }, + .{ .field = "mem_total_bytes", .metric = "zml_device_memory_total_bytes", .help = "Device memory total in bytes" }, + // GPU (CUDA/ROCm/oneAPI) + .{ .field = "encoder_util_percent", .metric = "zml_device_encoder_utilization_percent", .help = "Encoder utilization percentage" }, + .{ .field = "decoder_util_percent", .metric = "zml_device_decoder_utilization_percent", .help = "Decoder utilization percentage" }, + .{ .field = "power_mw", .metric = "zml_device_power_watts", .help = "Power draw in watts", .fmt = fmtMilliwattsAsWatts }, + .{ .field = "power_limit_mw", .metric = "zml_device_power_limit_watts", .help = "Power limit in watts", .fmt = fmtMilliwattsAsWatts }, + .{ .field = "temperature", .metric = "zml_device_temperature_celsius", .help = "Device temperature in celsius" }, + .{ .field = "fan_speed_percent", .metric = "zml_device_fan_speed_percent", .help = "Fan speed percentage" }, + .{ .field = "clock_graphics_mhz", .metric = "zml_device_clock_graphics_mhz", .help = "Graphics clock frequency in MHz" }, + .{ .field = "clock_sm_mhz", .metric = "zml_device_clock_sm_mhz", .help = "SM clock frequency in MHz" }, + .{ .field = "clock_mem_mhz", .metric = "zml_device_clock_memory_mhz", .help = "Memory clock frequency in MHz" }, + .{ .field = "pcie_tx_kbps", .metric = "zml_device_pcie_tx_bytes_per_second", .help = "PCIe TX throughput in bytes per second", .fmt = fmtScaled(128) }, + .{ .field = "pcie_rx_kbps", .metric = "zml_device_pcie_rx_bytes_per_second", .help = "PCIe RX throughput in bytes per second", .fmt = fmtScaled(128) }, + // Neuron + .{ .field = "nc_tensors", .metric = "zml_device_neuron_tensors_bytes", .help = "Neuron core tensor memory in bytes" }, + .{ .field = "nc_constants", .metric = "zml_device_neuron_constants_bytes", .help = "Neuron core constants memory in bytes" }, + .{ .field = "nc_model_code", .metric = "zml_device_neuron_model_code_bytes", .help = "Neuron core model code memory in bytes" }, + .{ .field = "nc_shared_scratchpad", .metric = "zml_device_neuron_shared_scratchpad_bytes", .help = "Neuron core shared scratchpad memory in bytes" }, + .{ .field = "nc_nonshared_scratchpad", .metric = "zml_device_neuron_nonshared_scratchpad_bytes", .help = "Neuron core non-shared scratchpad memory in bytes" }, + .{ .field = "nc_runtime", .metric = "zml_device_neuron_runtime_bytes", .help = "Neuron core runtime memory in bytes" }, + .{ .field = "nc_driver", .metric = "zml_device_neuron_driver_bytes", .help = "Neuron core driver memory in bytes" }, + .{ .field = "nc_dma_rings", .metric = "zml_device_neuron_dma_rings_bytes", .help = "Neuron core DMA rings memory in bytes" }, + .{ .field = "nc_collectives", .metric = "zml_device_neuron_collectives_bytes", .help = "Neuron core collectives memory in bytes" }, + .{ .field = "nc_notifications", .metric = "zml_device_neuron_notifications_bytes", .help = "Neuron core notifications memory in bytes" }, + .{ .field = "nc_uncategorized", .metric = "zml_device_neuron_uncategorized_bytes", .help = "Neuron core uncategorized memory in bytes" }, +}; + +const host_metrics = .{ + .{ .field = "cpu_cores", .metric = "zml_host_cpu_cores", .help = "Number of CPU cores" }, + .{ .field = "mem_total_kib", .metric = "zml_host_memory_total_bytes", .help = "Total host memory in bytes", .fmt = fmtScaled(1024) }, + .{ .field = "mem_available_kib", .metric = "zml_host_memory_available_bytes", .help = "Available host memory in bytes", .fmt = fmtScaled(1024) }, + .{ .field = "load_1", .metric = "zml_host_load_1", .help = "1-minute load average" }, + .{ .field = "load_5", .metric = "zml_host_load_5", .help = "5-minute load average" }, + .{ .field = "load_15", .metric = "zml_host_load_15", .help = "15-minute load average" }, + .{ .field = "uptime_seconds", .metric = "zml_host_uptime_seconds", .help = "System uptime in seconds" }, +}; + +pub fn write(writer: *std.Io.Writer, devices: []const *DeviceInfo, host: *const HostInfo) !void { + try writer.writeAll("# HELP zml_device_info Device metadata\n"); + try writer.writeAll("# TYPE zml_device_info gauge\n"); + for (devices, 0..) |dev, i| { + inline for (@typeInfo(DeviceInfo).@"union".field_names) |type_name| { + const tag = @field(smi_info.device_info.Target, type_name); + + switch (dev.*) { + tag => |*db| { + const val = db.front().*; + + try writer.print("zml_device_info{{device=\"{d}\",type=\"{s}\"", .{ i, type_name }); + + inline for (.{ + .{ "name", "name" }, + .{ "driver_version", "driver" }, + .{ "cuda_driver_version", "cuda_version" }, + }) |label| { + if (@hasField(@TypeOf(val), label[0])) { + if (@field(val, label[0])) |v| { + try writer.print(",{s}=\"", .{label[1]}); + try std.zig.stringEscape(v, writer); + try writer.writeAll("\""); + } + } + } + + try writer.writeAll("} 1\n"); + }, + else => {}, + } + } + } + try writer.writeAll("\n"); + + try writeAllDeviceMetrics(writer, device_metrics, devices); + try writeHostMetrics(writer, host.front().*); +} + +fn writeAllDeviceMetrics( + writer: *std.Io.Writer, + comptime metrics: anytype, + devices: []const *DeviceInfo, +) !void { + inline for (metrics) |m| { + var header_written = false; + + for (devices, 0..) |dev, i| { + switch (dev.*) { + inline else => |*db| { + const val = db.front().*; + if (@hasField(@TypeOf(val), m.field)) { + if (@field(val, m.field)) |raw| { + if (!header_written) { + try writer.print("# HELP {s} {s}\n# TYPE {s} gauge\n", .{ m.metric, m.help, m.metric }); + header_written = true; + } + + const device_type = @tagName(std.meta.activeTag(dev.*)); + try writer.print("{s}{{device=\"{d}\",type=\"{s}\",name=\"", .{ m.metric, i, device_type }); + try std.zig.stringEscape(val.name orelse "", writer); + try writer.writeAll("\"} "); + try fmtValue(m, writer, raw); + try writer.writeAll("\n"); + } + } + }, + } + } + + if (header_written) { + try writer.writeAll("\n"); + } + } +} + +fn writeHostMetrics(writer: *std.Io.Writer, val: anytype) !void { + inline for (host_metrics) |m| { + if (@field(val, m.field)) |raw| { + try writer.print("# HELP {s} {s}\n# TYPE {s} gauge\n{s} ", .{ m.metric, m.help, m.metric, m.metric }); + try fmtValue(m, writer, raw); + try writer.writeAll("\n\n"); + } + } +} + +fn fmtValue(comptime m: anytype, writer: *std.Io.Writer, raw: anytype) !void { + if (@hasField(@TypeOf(m), "fmt")) { + try m.fmt(writer, raw); + } else { + try writeValue(writer, raw); + } +} + +fn writeValue(writer: *std.Io.Writer, val: anytype) std.Io.Writer.Error!void { + switch (@typeInfo(@TypeOf(val))) { + .float, .comptime_float => try writer.print("{d:.2}", .{val}), + .pointer => try writer.writeAll(val), + else => try writer.print("{d}", .{val}), + } +} + +fn fmtMilliwattsAsWatts(writer: *std.Io.Writer, val: u64) std.Io.Writer.Error!void { + try writer.print("{d}.{d:0>3}", .{ val / 1000, val % 1000 }); +} + +fn fmtScaled(comptime scale: u64) fn (*std.Io.Writer, u64) std.Io.Writer.Error!void { + return struct { + fn f(writer: *std.Io.Writer, val: u64) std.Io.Writer.Error!void { + try writer.print("{d}", .{val * scale}); + } + }.f; +} diff --git a/bin/zml-smi/prometheus/prometheus.zig b/bin/zml-smi/prometheus/prometheus.zig new file mode 100644 index 00000000..cfa2fc28 --- /dev/null +++ b/bin/zml-smi/prometheus/prometheus.zig @@ -0,0 +1,2 @@ +pub const Server = @import("server.zig").Server; +pub const exposition = @import("exposition.zig"); diff --git a/bin/zml-smi/prometheus/server.zig b/bin/zml-smi/prometheus/server.zig new file mode 100644 index 00000000..9517ecac --- /dev/null +++ b/bin/zml-smi/prometheus/server.zig @@ -0,0 +1,87 @@ +const std = @import("std"); +const smi_info = @import("zml-smi/info"); +const DeviceInfo = smi_info.device_info.DeviceInfo; +const HostInfo = smi_info.host_info.HostInfo; +const exposition = @import("exposition.zig"); + +pub const Server = struct { + io: std.Io, + devices: []const *DeviceInfo, + host: *const HostInfo, + tcp: std.Io.net.Server, + connection_group: std.Io.Group = .init, + + pub fn init(io: std.Io, listen: []const u8, devices: []const *DeviceInfo, host: *const HostInfo) !Server { + const address = try std.Io.net.IpAddress.parseLiteral(listen); + var tcp = try address.listen(io, .{ .reuse_address = true }); + errdefer tcp.deinit(io); + + return .{ + .io = io, + .devices = devices, + .host = host, + .tcp = tcp, + }; + } + + pub fn run(io: std.Io, listen: []const u8, devices: []const *DeviceInfo, host_info: *const HostInfo) void { + var self = init(io, listen, devices, host_info) catch |err| { + std.log.err("prometheus server failed to start: {s}", .{@errorName(err)}); + return; + }; + defer self.deinit(); + self.serve(); + } + + pub fn deinit(self: *Server) void { + self.connection_group.await(self.io) catch {}; + self.tcp.deinit(self.io); + } + + pub fn serve(self: *Server) void { + while (true) { + const stream = self.tcp.accept(self.io) catch break; + self.connection_group.concurrent(self.io, Server.onConnection, .{ self, stream }) catch break; + } + } + + fn onConnection(self: *Server, stream: std.Io.net.Stream) std.Io.Cancelable!void { + defer stream.close(self.io); + self.handleConn(stream) catch {}; + } + + fn handleConn(self: *Server, stream: std.Io.net.Stream) !void { + var read_buf: [8192]u8 = undefined; + var reader = stream.reader(self.io, &read_buf); + + var write_buf: [4096]u8 = undefined; + var writer = stream.writer(self.io, &write_buf); + + var http_server: std.http.Server = .init(&reader.interface, &writer.interface); + + while (true) { + var request = http_server.receiveHead() catch |err| switch (err) { + error.HttpConnectionClosing => return, + else => return err, + }; + + if (request.head.method != .GET) { + try request.respond("Method Not Allowed\n", .{ .status = .method_not_allowed }); + continue; + } + + var body_buf: [4096]u8 = undefined; + var body_writer = try request.respondStreaming(&body_buf, .{ + .respond_options = .{ + .status = .ok, + .extra_headers = &.{ + .{ .name = "content-type", .value = "text/plain" }, + }, + }, + }); + + try exposition.write(&body_writer.writer, self.devices, self.host); + try body_writer.end(); + } + } +}; diff --git a/bin/zml-smi/scripts/release.sh b/bin/zml-smi/scripts/release.sh new file mode 100755 index 00000000..d1e89902 --- /dev/null +++ b/bin/zml-smi/scripts/release.sh @@ -0,0 +1,395 @@ +#!/usr/bin/env bash +set -euo pipefail + +# --- UI --- + +if [ -t 1 ]; then + DIM='\033[2m' + RESET='\033[0m' + BLUE='\033[1;38;2;140;180;255m' + GREEN='\033[1;32m' + RED='\033[1;31m' + YELLOW='\033[1;33m' + CYAN='\033[1;36m' + WHITE='\033[1;37m' +else + DIM='' RESET='' BLUE='' GREEN='' RED='' YELLOW='' CYAN='' WHITE='' +fi + +info() { printf "${BLUE} │${RESET} %b\n" "$*"; } +success() { printf "${GREEN} ✓${RESET} %b\n" "$*"; } +warn() { printf "${YELLOW} !${RESET} %b\n" "$*"; } +fail() { printf "${RED} ✗ %b${RESET}\n" "$*" >&2; exit 1; } +step() { printf "\n${WHITE} ▸ %b${RESET}\n" "$*"; } + +confirm() { + if [ "$YES" = true ]; then + return 0 + fi + printf "${WHITE} ? %b ${DIM}[y/N]${RESET} " "$*" + local answer + read -r answer + case "$answer" in + y|Y|yes|Yes) return 0 ;; + *) return 1 ;; + esac +} + +usage() { + printf >&2 " + ${WHITE}Usage${RESET} + bazel run //bin/zml-smi:release -- [flags] + + ${WHITE}Arguments${RESET} + ${CYAN}version${RESET} Version string, e.g. ${DIM}v1.0.0${RESET} (tagged as ${DIM}zml-smi-v1.0.0${RESET}) + + ${WHITE}Flags${RESET} + ${CYAN}--dry-run${RESET} Build only — skip tagging, uploading, and publishing + ${CYAN}--verify${RESET} Verify uploaded files by comparing sha256 against the CDN + ${CYAN}--publish${RESET} Push git tag and create a GitHub release + ${CYAN}-y, --yes${RESET} Skip all confirmation prompts + ${CYAN}-h, --help${RESET} Show this help + + ${WHITE}Environment${RESET} + ${CYAN}BASE_URL${RESET} Public base URL serving the bucket + ${CYAN}RCLONE_REMOTE${RESET} Rclone remote name ${DIM}(default: r2)${RESET} + + ${DIM}If rclone is not configured, you can set the remote via env vars:${RESET} + ${DIM} RCLONE_CONFIG_R2_TYPE=s3${RESET} + ${DIM} RCLONE_CONFIG_R2_PROVIDER=Cloudflare${RESET} + ${DIM} RCLONE_CONFIG_R2_ACCESS_KEY_ID=your_access_key${RESET} + ${DIM} RCLONE_CONFIG_R2_SECRET_ACCESS_KEY=your_secret_key${RESET} + ${DIM} RCLONE_CONFIG_R2_ENDPOINT=https://.r2.cloudflarestorage.com${RESET} + + ${WHITE}Examples${RESET} + ${DIM}# Interactive, local only${RESET} + bazel run //bin/zml-smi:release -- 0.4.0 + + ${DIM}# Full release, non-interactive${RESET} + bazel run //bin/zml-smi:release -- --publish --verify --yes 0.4.0 + +" + exit 0 +} + +# --- Checks --- + +check_env() { + if [ -z "${BUILD_WORKSPACE_DIRECTORY:-}" ]; then + fail "Must be run via 'bazel run //bin/zml-smi:release -- '" + fi + if [ -z "${BASE_URL:-}" ]; then + fail "Missing required environment variable: ${CYAN}BASE_URL" + fi +} + +# --- Steps --- + +changelog() { + local prev_tag + prev_tag="$(git describe --tags --abbrev=0 --match 'zml-smi-*' 2>/dev/null || true)" + local range="${prev_tag:+${prev_tag}..HEAD}" + + git log ${range:-HEAD} --pretty=format:"- %s" -- bin/zml-smi/ | grep -vi WIP || true +} + +tag() { + step "Tagging ${CYAN}${TAG}${WHITE}" + + # Cache changelog before tagging, otherwise the new tag makes the range empty + CACHED_CHANGELOG="$(changelog)" + + if [ -z "$CACHED_CHANGELOG" ]; then + warn "No changes since last release" + fi + + if git rev-parse "$TAG" &>/dev/null; then + warn "Tag ${TAG} already exists, skipping" + return + fi + + if [ "$DRY_RUN" = true ]; then + warn "Dry run — skipping tag creation" + elif confirm "Create tag ${CYAN}${TAG}${RESET}${WHITE}?"; then + git tag -a "$TAG" -m "$(printf '%s\n\n%s' "$TAG" "$CACHED_CHANGELOG")" + success "Created tag ${CYAN}${TAG}" + else + warn "Skipped tagging" + return + fi + printf "\n" + + printf "${DIM}" + echo "$CACHED_CHANGELOG" + printf "${RESET}" +} + +build() { + step "Building release archives" + + if ! confirm "Build all platforms?"; then + warn "Skipped build" + return 1 + fi + + printf "${DIM}%s${RESET}\n" "────────────────────────────────────────" + + bazel build --config=release //bin/zml-smi:archives + + printf "${DIM}%s${RESET}\n" "────────────────────────────────────────" + FILES=() + while IFS= read -r line; do + FILES+=("$line") + done < <(bazel cquery --config=release --output=files //bin/zml-smi:archives 2>/dev/null) + success "Built ${CYAN}${#FILES[@]}${RESET}${GREEN} artifacts" +} + +upload() { + local rclone_remote="${RCLONE_REMOTE:-r2}" + local install_url="${BASE_URL}/zml-smi/${VERSION}" + + step "Patching install script" + + INSTALL_SCRIPT="$(mktemp)" + sed -e "s|^BASE_URL=.*|BASE_URL=\"${install_url}\"|" \ + -e "s|^VERSION=.*|VERSION=\"${VERSION}\"|" \ + "${BUILD_WORKSPACE_DIRECTORY}/bin/zml-smi/scripts/remote-install.sh" > "$INSTALL_SCRIPT" + success "BASE_URL=${CYAN}${install_url}" + + if [ "$DRY_RUN" = true ]; then + step "Uploading to ${CYAN}${rclone_remote}" + warn "Dry run — skipping upload" + return + fi + + step "Uploading to ${CYAN}${rclone_remote}" + + if ! confirm "Upload artifacts?"; then + warn "Skipped upload" + return + fi + + for f in "${FILES[@]}"; do + if [ ! -f "$f" ]; then + warn "SKIP $(basename "$f") ${DIM}(not built for this host)${RESET}" + continue + fi + local name remote_name + name="$(basename "$f")" + remote_name="${name/zml-smi-/zml-smi-${VERSION}-}" + rclone copyto "$f" "${rclone_remote}:zml-smi/${VERSION}/${remote_name}" + success "${remote_name}" + done + + rclone copyto "$INSTALL_SCRIPT" "${rclone_remote}:zml-smi/${VERSION}/install.sh" + success "zml-smi/${VERSION}/install.sh" + + if confirm "Set ${CYAN}${VERSION}${RESET}${WHITE} as the default install version?"; then + rclone copyto "$INSTALL_SCRIPT" "${rclone_remote}:zml-smi/install.sh" + success "zml-smi/install.sh" + DEFAULT_INSTALL=true + else + warn "Skipped default install script" + DEFAULT_INSTALL=false + fi +} + +verify() { + local install_url="${BASE_URL}/zml-smi/${VERSION}" + + step "Verifying uploads" + + for f in "${FILES[@]}"; do + if [ ! -f "$f" ]; then + continue + fi + local name remote_name + name="$(basename "$f")" + remote_name="${name/zml-smi-/zml-smi-${VERSION}-}" + local local_sha remote_sha + local_sha="$(shasum -a 256 "$f" | awk '{print $1}')" + remote_sha="$(curl -fsSL "${install_url}/${remote_name}" | shasum -a 256 | awk '{print $1}')" + if [ "$local_sha" = "$remote_sha" ]; then + success "${remote_name}" + else + fail "${remote_name} — sha256 mismatch (local: ${local_sha}, remote: ${remote_sha})" + fi + done + + local local_sha remote_sha + local_sha="$(shasum -a 256 "$INSTALL_SCRIPT" | awk '{print $1}')" + remote_sha="$(curl -fsSL "${install_url}/install.sh" | shasum -a 256 | awk '{print $1}')" + if [ "$local_sha" = "$remote_sha" ]; then + success "install.sh" + else + fail "install.sh — sha256 mismatch (local: ${local_sha}, remote: ${remote_sha})" + fi +} + +release_notes() { + local install_url="${BASE_URL}/zml-smi/${VERSION}" + local log="$CACHED_CHANGELOG" + + local downloads="" + for f in "${FILES[@]}"; do + [ -f "$f" ] || continue + local name remote_name + name="$(basename "$f")" + [[ "$name" == *.sha256 ]] && continue + local sha_file="" + for sf in "${FILES[@]}"; do + if [ "$(basename "$sf")" = "${name}.sha256" ]; then + sha_file="$sf" + break + fi + done + if [ -z "$sha_file" ] || [ ! -f "$sha_file" ]; then + fail "Missing sha256 sidecar for ${name}" + fi + local sha + sha="$(awk '{print $1}' "$sha_file")" + local actual + actual="$(shasum -a 256 "$f" | awk '{print $1}')" + if [ "$sha" != "$actual" ]; then + fail "${name} sha256 mismatch — sidecar: ${sha}, actual: ${actual}" + fi + remote_name="${name/zml-smi-/zml-smi-${VERSION}-}" + downloads="${downloads}| [${remote_name}](${install_url}/${remote_name}) | \`${sha}\` |"$'\n' + done + + cat </dev/null && [ -w "$INSTALL_DIR" ] && + mkdir -p "$BIN_DIR" 2>/dev/null && [ -w "$BIN_DIR" ]; then + return + fi + + warn "Cannot write to /opt or /usr/local/bin — falling back to ~/.local" + INSTALL_DIR="${HOME}/.local/zml-smi" + BIN_DIR="${HOME}/.local/bin" +} + +if [ -t 1 ]; then + DIM='\033[2m' + RESET='\033[0m' + BLUE='\033[1;38;2;140;180;255m' + GREEN='\033[1;32m' + RED='\033[1;31m' + YELLOW='\033[1;33m' + CYAN='\033[1;36m' + WHITE='\033[1;37m' +else + DIM='' RESET='' BLUE='' GREEN='' RED='' YELLOW='' CYAN='' WHITE='' +fi + +info() { printf "${BLUE} │${RESET} %b\n" "$*"; } +success() { printf "${GREEN} ✓${RESET} %b\n" "$*"; } +warn() { printf "${YELLOW} !${RESET} %b\n" "$*" >&2; } +fail() { printf "${RED} ✗ %b${RESET}\n" "$*" >&2; exit 1; } +step() { printf "\n${WHITE} ▸ %b${RESET}\n" "$*"; } + +print_logo() { + printf "\n" + printf "${BLUE} █████╗███╗ ███╗██╗ ${RESET}\n" + printf "${BLUE} ╚═██╔╝████╗ ████║██║ ${RESET}\n" + printf "${BLUE} ██╔╝ ██╔████╔██║██║ ${RESET}\n" + printf "${BLUE} █████╗██║ ╚═╝ ██║████╗${RESET}\n" + printf "${BLUE} ╚════╝╚═╝ ╚═╝╚═══╝${RESET}\n" +} + +detect_platform() { + step "Detecting platform" + + OS="$(uname -s)" + ARCH="$(uname -m)" + + case "$OS" in + Linux) OS_LABEL="linux" ;; + Darwin) OS_LABEL="macos" ;; + MINGW*|MSYS*|CYGWIN*) + fail "Windows is not supported. Use WSL2 instead." ;; + *) + fail "Unsupported operating system: ${OS}" ;; + esac + + case "$ARCH" in + x86_64|amd64) ARCH_LABEL="amd64" ;; + aarch64|arm64) ARCH_LABEL="arm64" ;; + *) + fail "Unsupported architecture: ${ARCH}" ;; + esac + + if [ "$OS_LABEL" = "macos" ] && [ "$ARCH_LABEL" = "amd64" ]; then + fail "macOS x86_64 is not supported. Only Apple Silicon (arm64) is supported." + fi + + success "${OS_LABEL} ${ARCH_LABEL}" + + DOWNLOAD_URL="${BASE_URL}/${BINARY_NAME}-${VERSION}-${OS_LABEL}-${ARCH_LABEL}.tar.zst" +} + +check_deps() { + step "Checking dependencies" + local missing=() + + if command -v curl &>/dev/null; then + DOWNLOADER="curl" + success "curl" + elif command -v wget &>/dev/null; then + DOWNLOADER="wget" + success "wget" + else + missing+=("curl or wget") + fi + + if command -v zstd &>/dev/null; then + success "zstd" + else + missing+=("zstd") + fi + + if command -v tar &>/dev/null; then + success "tar" + else + missing+=("tar") + fi + + if [ ${#missing[@]} -gt 0 ]; then + fail "Missing required tools: ${missing[*]}" + fi +} + +download_and_install() { + step "Downloading & extracting" + + rm -rf "$INSTALL_DIR" + mkdir -p "$INSTALL_DIR" + + if [ "$DOWNLOADER" = "curl" ]; then + if ! curl -fsSL "$DOWNLOAD_URL" | tar --zstd -xf - --strip-components=1 -C "$INSTALL_DIR"; then + fail "Download failed — is this OS/arch supported?" + fi + else + if ! wget -q -O- "$DOWNLOAD_URL" | tar --zstd -xf - --strip-components=1 -C "$INSTALL_DIR"; then + fail "Download failed — is this OS/arch supported?" + fi + fi + + chmod 755 "${INSTALL_DIR}/${BINARY_NAME}" + success "Installed to ${INSTALL_DIR}/" + + mkdir -p "$BIN_DIR" + ln -sf "${INSTALL_DIR}/${BINARY_NAME}" "${BIN_DIR}/${BINARY_NAME}" + success "Symlink at ${BIN_DIR}/${BINARY_NAME}" +} + +check_path() { + if ! echo "$PATH" | tr ':' '\n' | grep -qx "$BIN_DIR"; then + printf "\n" + warn "${BIN_DIR} is not in your PATH" + step "Add to your shell profile:" + info "${CYAN}export PATH=\"${BIN_DIR}:\$PATH\"" + printf "\n" + fi +} + +main() { + print_logo + resolve_dirs + detect_platform + check_deps + + info "Install to: ${CYAN}${INSTALL_DIR}${RESET}" + info "Bin dir: ${CYAN}${BIN_DIR}${RESET}" + + download_and_install + check_path + + printf "\n" + printf " ${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}\n" + printf " ${GREEN} Installation complete!${RESET}\n" + printf " ${DIM} Run '${BINARY_NAME}' to get started.${RESET}\n" + printf " ${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}\n" + printf "\n" +} + +main diff --git a/bin/zml-smi/tui/BUILD.bazel b/bin/zml-smi/tui/BUILD.bazel new file mode 100644 index 00000000..29f4824c --- /dev/null +++ b/bin/zml-smi/tui/BUILD.bazel @@ -0,0 +1,59 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "tui", + import_name = "zml-smi/tui", + main = "tui.zig", + srcs = [ + "data.zig", + "image_cache.zig", + "lib/compose.zig", + "lib/image.zig", + "lib/ui.zig", + "lib/utils.zig", + "pages/detail.zig", + "pages/detail/detail_common.zig", + "pages/detail/gpu_detail.zig", + "pages/detail/neuron_detail.zig", + "pages/detail/tpu_detail.zig", + "pages/overview.zig", + "print.zig", + "theme.zig", + "top.zig", + "widgets/braille_chart.zig", + "widgets/chart.zig", + "widgets/column_layout.zig", + "widgets/device_card.zig", + "widgets/gauge.zig", + "widgets/info_lines.zig", + "widgets/logo.zig", + "widgets/metric_card.zig", + "widgets/process_table.zig", + "widgets/status_line.zig", + "widgets/titled_border.zig", + ], + extra_srcs = [ + "assets/amd.png", + "assets/intel.png", + "assets/logo.png", + "assets/neuron.png", + "assets/nvidia.png", + "assets/tpu.png", + ], + visibility = ["//visibility:public"], + deps = [ + "//bazel/stamp:zig", + "//bin/zml-smi/info", + "//bin/zml-smi/utils:double_buffer", + "//zml:logo", + "@libvaxis//:vaxis", + ] + select({ + "@platforms//os:macos": ["//bin/zml-smi/platforms/macos"], + "//conditions:default": ["//bin/zml-smi/platforms/linux"], + }), +) + +zig_test( + name = "test", + deps = [":tui"], +) diff --git a/bin/zml-smi/tui/assets/amd.png b/bin/zml-smi/tui/assets/amd.png new file mode 100644 index 00000000..1de65ece Binary files /dev/null and b/bin/zml-smi/tui/assets/amd.png differ diff --git a/bin/zml-smi/tui/assets/intel.png b/bin/zml-smi/tui/assets/intel.png new file mode 100644 index 00000000..1f71fcd7 Binary files /dev/null and b/bin/zml-smi/tui/assets/intel.png differ diff --git a/bin/zml-smi/tui/assets/logo.png b/bin/zml-smi/tui/assets/logo.png new file mode 100644 index 00000000..77ceba76 Binary files /dev/null and b/bin/zml-smi/tui/assets/logo.png differ diff --git a/bin/zml-smi/tui/assets/neuron.png b/bin/zml-smi/tui/assets/neuron.png new file mode 100644 index 00000000..ea57a753 Binary files /dev/null and b/bin/zml-smi/tui/assets/neuron.png differ diff --git a/bin/zml-smi/tui/assets/nvidia.png b/bin/zml-smi/tui/assets/nvidia.png new file mode 100644 index 00000000..6c2a474c Binary files /dev/null and b/bin/zml-smi/tui/assets/nvidia.png differ diff --git a/bin/zml-smi/tui/assets/tpu.png b/bin/zml-smi/tui/assets/tpu.png new file mode 100644 index 00000000..97a6ae3c Binary files /dev/null and b/bin/zml-smi/tui/assets/tpu.png differ diff --git a/bin/zml-smi/tui/data.zig b/bin/zml-smi/tui/data.zig new file mode 100644 index 00000000..bc6171b7 --- /dev/null +++ b/bin/zml-smi/tui/data.zig @@ -0,0 +1,155 @@ +const std = @import("std"); +const builtin = @import("builtin"); +const smi_info = @import("zml-smi/info"); +const di = smi_info.device_info; +pub const DeviceInfo = di.DeviceInfo; +pub const GpuInfo = di.GpuInfo; +pub const NeuronInfo = di.NeuronInfo; +pub const TpuInfo = di.TpuInfo; +pub const Target = di.Target; +pub const Targets = smi_info.Targets; +const hi = smi_info.host_info; +pub const HostInfo = hi.HostInfo; +pub const HostData = hi.HostData; +pub const pi = smi_info.process_info; +pub const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); +pub const ProcessEnricher = if (builtin.os.tag == .macos) + @import("zml-smi/platforms/macos").process.ProcessEnricher +else + @import("zml-smi/platforms/linux").process.ProcessEnricher; + +pub const history_len: usize = 500; + +pub fn RingBuffer(comptime T: type, comptime capacity: usize) type { + return struct { + const Self = @This(); + + buf: [capacity]T = @splat(0), + head: usize = 0, + len: usize = 0, + + pub fn push(self: *Self, value: T) void { + self.buf[self.head] = value; + self.head = (self.head + 1) % capacity; + if (self.len < capacity) { + self.len += 1; + } + } + + pub fn sliceLast(self: *const Self, arena: std.mem.Allocator, n: usize) std.mem.Allocator.Error![]const T { + const count = @min(n, self.len); + if (count == 0) { + return &.{}; + } + const out = try arena.alloc(T, count); + const start = (self.head + capacity - count) % capacity; + for (0..count) |i| { + out[i] = self.buf[(start + i) % capacity]; + } + return out; + } + }; +} + +pub const HistoryBuffers = struct { + util: []RingBuffer(u64, history_len) = &.{}, + mem_util: []RingBuffer(u64, history_len) = &.{}, + temp: []RingBuffer(u64, history_len) = &.{}, + power: []RingBuffer(u64, history_len) = &.{}, + + pub fn init(allocator: std.mem.Allocator, count: usize) !HistoryBuffers { + const h: HistoryBuffers = .{ + .util = try allocator.alloc(RingBuffer(u64, history_len), count), + .mem_util = try allocator.alloc(RingBuffer(u64, history_len), count), + .temp = try allocator.alloc(RingBuffer(u64, history_len), count), + .power = try allocator.alloc(RingBuffer(u64, history_len), count), + }; + + @memset(h.util, .{}); + @memset(h.mem_util, .{}); + @memset(h.temp, .{}); + @memset(h.power, .{}); + + return h; + } + + pub fn deinit(self: *HistoryBuffers, allocator: std.mem.Allocator) void { + allocator.free(self.util); + allocator.free(self.mem_util); + allocator.free(self.temp); + allocator.free(self.power); + } +}; + +pub const SystemState = struct { + devices: []*DeviceInfo, + host: *HostInfo, + history: HistoryBuffers = .{}, + targets: Targets, + process_lists: []*ProcessDoubleBuffer, + enricher: *ProcessEnricher, + gpa: std.mem.Allocator, + arena: std.mem.Allocator, + io: std.Io, + tui_refresh_rate: u16, + + pub const Config = struct { + devices: []*DeviceInfo, + host: *HostInfo, + targets: Targets, + tui_refresh_rate: u16, + process_lists: []*ProcessDoubleBuffer, + enricher: *ProcessEnricher, + gpa: std.mem.Allocator, + arena: std.mem.Allocator, + io: std.Io, + }; + + pub fn init(cfg: Config) !SystemState { + return .{ + .devices = cfg.devices, + .host = cfg.host, + .history = try HistoryBuffers.init(cfg.arena, cfg.devices.len), + .targets = cfg.targets, + .tui_refresh_rate = cfg.tui_refresh_rate, + .process_lists = cfg.process_lists, + .enricher = cfg.enricher, + .gpa = cfg.gpa, + .arena = cfg.arena, + .io = cfg.io, + }; + } + + pub fn deinit(self: *SystemState, allocator: std.mem.Allocator) void { + self.history.deinit(allocator); + } + + pub fn deviceCount(self: *const SystemState) usize { + return self.devices.len; + } + + pub fn recordHistory(self: *SystemState) void { + for (0..self.deviceCount()) |i| { + switch (self.devices[i].*) { + .cuda, .rocm, .oneapi => |*sv| { + const gpu = sv.front().*; + self.history.util[i].push(gpu.util_percent orelse 0); + const used = gpu.mem_used_bytes orelse 0; + const total = gpu.mem_total_bytes orelse 0; + self.history.mem_util[i].push(if (total > 0) used * 100 / total else 0); + self.history.temp[i].push(gpu.temperature orelse 0); + self.history.power[i].push(gpu.power_mw orelse 0); + }, + inline else => |*sv| { + const info = sv.front().*; + self.history.util[i].push(info.util_percent orelse 0); + const used = info.mem_used_bytes orelse 0; + const total = info.mem_total_bytes orelse 0; + self.history.mem_util[i].push(if (total > 0) used * 100 / total else 0); + self.history.temp[i].push(0); + self.history.power[i].push(0); + }, + } + } + } +}; diff --git a/bin/zml-smi/tui/image_cache.zig b/bin/zml-smi/tui/image_cache.zig new file mode 100644 index 00000000..1aea4778 --- /dev/null +++ b/bin/zml-smi/tui/image_cache.zig @@ -0,0 +1,37 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); + +const ImageCache = @This(); + +pub var global: ImageCache = .{}; + +map: std.StringHashMapUnmanaged(vaxis.Image) = .empty, + +pub fn deinit(self: *ImageCache, allocator: std.mem.Allocator) void { + self.map.deinit(allocator); +} + +pub fn load( + self: *ImageCache, + vx: *vaxis.Vaxis, + allocator: std.mem.Allocator, + writer: *std.Io.Writer, + key: []const u8, + data: []const u8, +) void { + const image = vx.loadImage(allocator, writer, .{ .mem = data }) catch return; + self.map.put(allocator, key, image) catch return; +} + +pub fn loadAll(self: *ImageCache, vx: *vaxis.Vaxis, allocator: std.mem.Allocator, writer: *std.Io.Writer) void { + self.load(vx, allocator, writer, "logo", @embedFile("assets/logo.png")); + self.load(vx, allocator, writer, "gpu_cuda", @embedFile("assets/nvidia.png")); + self.load(vx, allocator, writer, "gpu_rocm", @embedFile("assets/amd.png")); + self.load(vx, allocator, writer, "gpu_oneapi", @embedFile("assets/intel.png")); + self.load(vx, allocator, writer, "gpu_neuron", @embedFile("assets/neuron.png")); + self.load(vx, allocator, writer, "gpu_tpu", @embedFile("assets/tpu.png")); +} + +pub fn get(self: *const ImageCache, key: []const u8) ?vaxis.Image { + return self.map.get(key); +} diff --git a/bin/zml-smi/tui/lib/compose.zig b/bin/zml-smi/tui/lib/compose.zig new file mode 100644 index 00000000..e3438762 --- /dev/null +++ b/bin/zml-smi/tui/lib/compose.zig @@ -0,0 +1,46 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; + +// ── SurfaceBuilder ───────────────────────────────────────────────── + +pub const SurfaceBuilder = struct { + arena: std.mem.Allocator, + children: std.ArrayList(vxfw.SubSurface), + + pub fn add(self: *SurfaceBuilder, row: i17, col: i17, surface: vxfw.Surface) !void { + try self.addZ(row, col, surface, 0); + } + + pub fn addZ(self: *SurfaceBuilder, row: i17, col: i17, surface: vxfw.Surface, z_index: u8) !void { + try self.children.append(self.arena, .{ .origin = .{ .row = row, .col = col }, .surface = surface, .z_index = z_index }); + } + + pub fn finish(self: *SurfaceBuilder, size: vxfw.Size, wgt: vxfw.Widget) vxfw.Surface { + return .{ .size = size, .widget = wgt, .buffer = &.{}, .children = self.children.items }; + } +}; + +pub fn surfaceBuilder(arena: std.mem.Allocator) SurfaceBuilder { + return .{ .arena = arena, .children = .empty }; +} + +// ── Fluent widget wrappers ───────────────────────────────────────── + +pub fn center(arena: std.mem.Allocator, child: vxfw.Widget) std.mem.Allocator.Error!vxfw.Widget { + const c = try arena.create(vxfw.Center); + c.* = .{ .child = child }; + return c.widget(); +} + +pub fn sized(arena: std.mem.Allocator, child: vxfw.Widget, size: vxfw.Size) std.mem.Allocator.Error!vxfw.Widget { + const s = try arena.create(vxfw.SizedBox); + s.* = .{ .child = child, .size = size }; + return s.widget(); +} + +pub fn pad(arena: std.mem.Allocator, child: vxfw.Widget, padding: vxfw.Padding.PadValues) std.mem.Allocator.Error!vxfw.Widget { + const p = try arena.create(vxfw.Padding); + p.* = .{ .child = child, .padding = padding }; + return p.widget(); +} diff --git a/bin/zml-smi/tui/lib/image.zig b/bin/zml-smi/tui/lib/image.zig new file mode 100644 index 00000000..a365ac4d --- /dev/null +++ b/bin/zml-smi/tui/lib/image.zig @@ -0,0 +1,84 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; + +const ui = @import("ui.zig"); +const Allocator = std.mem.Allocator; + +const Image = @This(); + +/// The loaded image to display. +image: vaxis.Image, +/// Desired height in cell rows. Width is computed to maintain aspect ratio. +/// If null, uses max constraint height (or 1 if unbounded). +rows: ?u16 = null, + +pub fn draw(self: *const Image, ctx: vxfw.DrawContext) Allocator.Error!vxfw.Surface { + const target_rows = self.rows orelse ctx.max.height orelse 1; + if (target_rows == 0 or self.image.width == 0 or self.image.height == 0) { + return .{ + .size = .{ .width = 0, .height = 0 }, + .widget = ui.widget(self), + .buffer = &.{}, + .children = &.{}, + }; + } + + // Compute cell columns from image aspect ratio and terminal cell pixel size + const cell_h: u32 = if (ctx.cell_size.height > 0) ctx.cell_size.height else 20; + const cell_w: u32 = if (ctx.cell_size.width > 0) ctx.cell_size.width else 10; + const height_px = @as(u32, target_rows) * cell_h; + const scale_f = @as(f64, @floatFromInt(height_px)) / @as(f64, @floatFromInt(self.image.height)); + const width_px: u32 = @intFromFloat(@as(f64, @floatFromInt(self.image.width)) * scale_f); + const cols: u16 = @intCast((width_px + cell_w - 1) / cell_w); + + const size: vxfw.Size = .{ .width = cols, .height = target_rows }; + var surf = try vxfw.Surface.init(ctx.arena, ui.widget(self), size); + surf.writeCell(0, 0, .{ + .image = .{ + .img_id = self.image.id, + .options = .{ + // Emit both rows and cols so that target bounds are unambiguous. + // Rows alone lets some terminals fall back to native pixel size. + .size = .{ .rows = target_rows, .cols = cols }, + }, + }, + }); + return surf; +} + +test Image { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + vxfw.DrawContext.init(.unicode); + + const img: Image = .{ + .image = .{ .id = 1, .width = 100, .height = 50 }, + .rows = 5, + }; + const w = ui.widget(&img); + + const draw_ctx: vxfw.DrawContext = .{ + .arena = arena.allocator(), + .min = .{}, + .max = .{ .width = 40, .height = 10 }, + .cell_size = .{ .width = 10, .height = 20 }, + }; + + const surface = try w.draw(draw_ctx); + try std.testing.expectEqual(@as(u16, 5), surface.size.height); + // 5 rows * 20px = 100px height. scale = 100/50 = 2.0. width = 100*2 = 200px. cols = 200/10 = 20 + try std.testing.expectEqual(@as(u16, 20), surface.size.width); + // Image cell should be at (0, 0) + const cell = surface.buffer[0]; + try std.testing.expect(cell.image != null); + try std.testing.expectEqual(@as(u32, 1), cell.image.?.img_id); + // Both rows and cols must be emitted so terminals don't fall back to native pixel size. + const size = cell.image.?.options.size.?; + try std.testing.expectEqual(@as(?u16, 5), size.rows); + try std.testing.expectEqual(@as(?u16, 20), size.cols); +} + +test "refAllDecls" { + std.testing.refAllDecls(@This()); +} diff --git a/bin/zml-smi/tui/lib/ui.zig b/bin/zml-smi/tui/lib/ui.zig new file mode 100644 index 00000000..6008d1e0 --- /dev/null +++ b/bin/zml-smi/tui/lib/ui.zig @@ -0,0 +1,87 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; + +/// Creates a vxfw.Widget from any pointer whose type has a `draw` method +/// (and optionally a `handleEvent` method). +pub fn widget(ptr: anytype) vxfw.Widget { + const Ptr = @TypeOf(ptr); + const T = @typeInfo(Ptr).pointer.child; + return .{ + .userdata = @ptrCast(@constCast(ptr)), + .drawFn = struct { + fn f(p: *anyopaque, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { + const self: *T = @ptrCast(@alignCast(p)); + return self.draw(ctx); + } + }.f, + .eventHandler = if (@hasDecl(T, "handleEvent")) struct { + fn f(p: *anyopaque, ctx: *vxfw.EventContext, event: vxfw.Event) anyerror!void { + const self: *T = @ptrCast(@alignCast(p)); + return self.handleEvent(ctx, event); + } + }.f else null, + }; +} + +/// Creates a vxfw.Widget that calls a specific draw function (for non-standard widget patterns +/// like MetricCard.drawContent or Chart.drawContent). +pub fn drawWidget(ptr: anytype, comptime drawFn: anytype) vxfw.Widget { + const Ptr = @TypeOf(ptr); + const T = @typeInfo(Ptr).pointer.child; + return .{ + .userdata = @ptrCast(@constCast(ptr)), + .drawFn = struct { + fn f(p: *anyopaque, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { + const self: *T = @ptrCast(@alignCast(p)); + return @call(.auto, drawFn, .{ self, ctx }); + } + }.f, + }; +} + +// ── DrawContext constraint helpers ────────────────────────────────── + +/// min = max = {w, h} — exact size. +pub fn fixedSize(ctx: vxfw.DrawContext, w: u16, h: u16) vxfw.DrawContext { + return ctx.withConstraints(.{ .width = w, .height = h }, .{ .width = w, .height = h }); +} + +/// Width fixed (min = max = w), height unbounded. +pub fn fixedWidth(ctx: vxfw.DrawContext, w: u16) vxfw.DrawContext { + return ctx.withConstraints(.{ .width = w }, .{ .width = w, .height = null }); +} + +/// max = {w, h}, min = default — bounded but unconstrained from below. +pub fn maxSize(ctx: vxfw.DrawContext, w: u16, h: u16) vxfw.DrawContext { + return ctx.withConstraints(.{}, .{ .width = w, .height = h }); +} + +// ── Drawing helpers ──────────────────────────────────────────────── + +/// Draw a single-line RichText with softwrap=false, overflow=clip, max width. +pub fn drawRichLine(ctx: vxfw.DrawContext, segments: []const vaxis.Cell.Segment, max_w: u16) std.mem.Allocator.Error!vxfw.Surface { + const rich: vxfw.RichText = .{ + .text = segments, + .softwrap = false, + .overflow = .clip, + }; + return rich.draw(ctx.withConstraints(.{}, .{ .width = max_w, .height = 1 })); +} + +pub const ImageCellSize = struct { + cols: u16, + rows: u16, +}; + +/// Compute the cell dimensions for an image scaled to a target row count, +/// maintaining aspect ratio based on the terminal's cell pixel size. +pub fn imageCellSize(img: vaxis.Image, target_rows: u16, cell_size: vxfw.Size) ImageCellSize { + const cell_h: u32 = if (cell_size.height > 0) cell_size.height else 20; + const cell_w: u32 = if (cell_size.width > 0) cell_size.width else 10; + const height_px = @as(u32, target_rows) * cell_h; + const scale_f = @as(f64, @floatFromInt(height_px)) / @as(f64, @floatFromInt(img.height)); + const width_px: u32 = @intFromFloat(@as(f64, @floatFromInt(img.width)) * scale_f); + const img_cols: u16 = @intCast((width_px + cell_w - 1) / cell_w); + return .{ .cols = img_cols, .rows = target_rows }; +} diff --git a/bin/zml-smi/tui/lib/utils.zig b/bin/zml-smi/tui/lib/utils.zig new file mode 100644 index 00000000..993703c9 --- /dev/null +++ b/bin/zml-smi/tui/lib/utils.zig @@ -0,0 +1,30 @@ +const std = @import("std"); + +pub fn bytesToMb(val: ?u64) u64 { + return if (val) |v| v / (1024 * 1024) else 0; +} + +pub fn repeatStr(arena: std.mem.Allocator, char: []const u8, count: u16) std.mem.Allocator.Error![]const u8 { + const buf = try arena.alloc(u8, char.len * count); + for (0..count) |i| { + @memcpy(buf[i * char.len ..][0..char.len], char); + } + return buf; +} + +/// Maps values from [min_val, max_val] to [0, 100], clamped. +pub fn normalizeRange(arena: std.mem.Allocator, raw: []const u64, min_val: u64, max_val: u64) std.mem.Allocator.Error![]u8 { + const result = try arena.alloc(u8, raw.len); + const range = max_val -| min_val; + for (raw, 0..) |v, i| { + result[i] = if (v <= min_val) + 0 + else if (v >= max_val) + 100 + else if (range > 0) + @intCast((v - min_val) * 100 / range) + else + 0; + } + return result; +} diff --git a/bin/zml-smi/tui/pages/detail.zig b/bin/zml-smi/tui/pages/detail.zig new file mode 100644 index 00000000..86d4b7a3 --- /dev/null +++ b/bin/zml-smi/tui/pages/detail.zig @@ -0,0 +1,28 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; +const data = @import("../data.zig"); +const ui = @import("../lib/ui.zig"); +const ProcessTable = @import("../widgets/process_table.zig"); +const gpu_detail = @import("detail/gpu_detail.zig"); +const neuron_detail = @import("detail/neuron_detail.zig"); +const tpu_detail = @import("detail/tpu_detail.zig"); + +const Detail = @This(); + +state: *const data.SystemState, +device_id: u16, +process_table: *ProcessTable = undefined, + +pub fn draw(self: *const Detail, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { + const w = ctx.max.width orelse 80; + const id = self.device_id; + const dev = self.state.devices[id]; + const wgt = ui.widget(self); + + return switch (dev.*) { + .cuda, .rocm, .oneapi => |*sv| try gpu_detail.draw(self.state, self.process_table, ctx, w, id, sv.front().*, wgt), + .neuron => |*sv| try neuron_detail.draw(self.state, self.process_table, ctx, w, id, sv.front().*, wgt), + .tpu => |*sv| try tpu_detail.draw(self.state, self.process_table, ctx, w, id, sv.front().*, wgt), + }; +} diff --git a/bin/zml-smi/tui/pages/detail/detail_common.zig b/bin/zml-smi/tui/pages/detail/detail_common.zig new file mode 100644 index 00000000..93e8addb --- /dev/null +++ b/bin/zml-smi/tui/pages/detail/detail_common.zig @@ -0,0 +1,55 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; +const data = @import("../../data.zig"); +const theme = @import("../../theme.zig"); +const utils = @import("../../lib/utils.zig"); +const ui = @import("../../lib/ui.zig"); +const compose = @import("../../lib/compose.zig"); +const Chart = @import("../../widgets/chart.zig"); +const ColumnLayout = @import("../../widgets/column_layout.zig"); + +pub fn headerText(arena: std.mem.Allocator, id: u16, name: []const u8) std.mem.Allocator.Error!vxfw.Text { + return .{ + .text = try std.fmt.allocPrint(arena, "Device {d}: {s}", .{ id, name }), + .style = theme.header_style, + .softwrap = false, + .overflow = .clip, + }; +} + +/// Creates a Chart pre-filled with history data and standard defaults. +pub fn historyChart( + ctx: vxfw.DrawContext, + state: *const data.SystemState, + id: u16, + history: anytype, + opts: struct { + title: []const u8, + value_label: []const u8, + info_line: ?[]const u8 = null, + y_min: u32 = 0, + y_max: u32 = 100, + y_unit: []const u8 = "%", + }, +) std.mem.Allocator.Error!Chart { + return .{ + .title = opts.title, + .data = try utils.normalizeRange(ctx.arena, try history[id].sliceLast(ctx.arena, data.history_len), opts.y_min, opts.y_max), + .value_label = opts.value_label, + .info_line = opts.info_line, + .y_min = opts.y_min, + .y_max = opts.y_max, + .y_unit = opts.y_unit, + .chart_height = 8, + .tui_refresh_rate = state.tui_refresh_rate, + }; +} + +/// Draws a page-level layout with standard margin (top=1, left/right=2) inside a full-width surface. +pub fn pageFrame(ctx: vxfw.DrawContext, layout: ColumnLayout, w: u16, parent_widget: vxfw.Widget) std.mem.Allocator.Error!vxfw.Surface { + const padded = try compose.pad(ctx.arena, ui.widget(&layout), .{ .top = 1, .left = 2, .right = 2 }); + var surf = try padded.draw(ui.fixedWidth(ctx, w)); + surf.widget = parent_widget; + return surf; +} diff --git a/bin/zml-smi/tui/pages/detail/gpu_detail.zig b/bin/zml-smi/tui/pages/detail/gpu_detail.zig new file mode 100644 index 00000000..7884d6f8 --- /dev/null +++ b/bin/zml-smi/tui/pages/detail/gpu_detail.zig @@ -0,0 +1,160 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; +const data = @import("../../data.zig"); +const utils = @import("../../lib/utils.zig"); +const ui = @import("../../lib/ui.zig"); +const Chart = @import("../../widgets/chart.zig"); +const ColumnLayout = @import("../../widgets/column_layout.zig"); +const MetricCard = @import("../../widgets/metric_card.zig"); +const ProcessTable = @import("../../widgets/process_table.zig"); +const common = @import("detail_common.zig"); + +pub fn draw( + state: *const data.SystemState, + process_table: *ProcessTable, + ctx: vxfw.DrawContext, + w: u16, + id: u16, + gpu: data.GpuInfo, + parent_widget: vxfw.Widget, +) std.mem.Allocator.Error!vxfw.Surface { + // ── Header ────────────────────────────────────────────── + const name_suffix: []const u8 = blk: { + const drv = gpu.driver_version orelse ""; + const cuda = gpu.cuda_driver_version orelse ""; + break :blk if (drv.len > 0 and cuda.len > 0) + try std.fmt.allocPrint(ctx.arena, " ({s}, CUDA {s})", .{ drv, cuda }) + else if (drv.len > 0) + try std.fmt.allocPrint(ctx.arena, " ({s})", .{drv}) + else if (cuda.len > 0) + try std.fmt.allocPrint(ctx.arena, " (CUDA {s})", .{cuda}) + else + @as([]const u8, ""); + }; + const header = try common.headerText(ctx.arena, id, try std.fmt.allocPrint(ctx.arena, "{s}{s}", .{ gpu.name orelse "Unknown", name_suffix })); + + // ── GPU Utilization + Memory charts ───────────────────── + const util_chart = try common.historyChart(ctx, state, id, state.history.util, .{ + .title = "GPU Utilization", + .value_label = try std.fmt.allocPrint(ctx.arena, "{d}%", .{gpu.util_percent orelse 0}), + .info_line = if (gpu.encoder_util_percent != null or gpu.decoder_util_percent != null) + try std.fmt.allocPrint(ctx.arena, "Enc {d}% Dec {d}%", .{ + gpu.encoder_util_percent orelse 0, gpu.decoder_util_percent orelse 0, + }) + else + null, + }); + const mem_chart = try common.historyChart(ctx, state, id, state.history.mem_util, .{ + .title = "Memory", + .value_label = try std.fmt.allocPrint(ctx.arena, "{d}%", .{if (gpu.mem_total_bytes) |total| if (total > 0) (gpu.mem_used_bytes orelse 0) * 100 / total else 0 else 0}), + .info_line = if (gpu.mem_bus_width) |bw| + try std.fmt.allocPrint(ctx.arena, "VRAM {d} / {d} MB ({d}-bit)", .{ + utils.bytesToMb(gpu.mem_used_bytes), utils.bytesToMb(gpu.mem_total_bytes), bw, + }) + else + try std.fmt.allocPrint(ctx.arena, "VRAM {d} / {d} MB", .{ + utils.bytesToMb(gpu.mem_used_bytes), utils.bytesToMb(gpu.mem_total_bytes), + }), + }); + + // ── Temperature + Power charts ────────────────────────── + const temp_chart = try common.historyChart(ctx, state, id, state.history.temp, .{ + .title = "Temperature", + .value_label = try std.fmt.allocPrint(ctx.arena, "{d}\u{00b0}C", .{gpu.temperature orelse 0}), + .info_line = try std.fmt.allocPrint(ctx.arena, "Fan {d}%", .{gpu.fan_speed_percent orelse 0}), + .y_min = 20, + .y_max = 100, + .y_unit = "\u{00b0}C", + }); + + const power_limit_mw = gpu.power_limit_mw orelse 0; + const power_limit_w = power_limit_mw / 1000; + const power_raw = try state.history.power[id].sliceLast(ctx.arena, data.history_len); + const power_watts = try ctx.arena.alloc(u64, power_raw.len); + for (power_raw, 0..) |p, j| { + power_watts[j] = p / 1000; + } + const y_max = if (power_limit_w > 0) power_limit_w else 1; + const power_chart: Chart = .{ + .title = "Power", + .data = try utils.normalizeRange(ctx.arena, power_watts, 0, y_max), + .value_label = try std.fmt.allocPrint(ctx.arena, "{d}W", .{(gpu.power_mw orelse 0) / 1000}), + .info_line = if (power_limit_w > 0) + try std.fmt.allocPrint(ctx.arena, "Limit {d}W", .{power_limit_w}) + else + "N/A", + .y_min = 0, + .y_max = @intCast(@min(y_max, std.math.maxInt(u32))), + .y_unit = "W", + .chart_height = 8, + .tui_refresh_rate = state.tui_refresh_rate, + }; + + const charts_flow: ColumnLayout = .{ + .children = &.{ ui.widget(&util_chart), ui.widget(&mem_chart), ui.widget(&temp_chart), ui.widget(&power_chart) }, + .min_child_width = 60, + .gap = 1, + }; + + // ── Clocks + PCIe ─────────────────────────────────────── + var clock_lines: std.ArrayList(MetricCard.LineEntry) = .empty; + try clock_lines.append(ctx.arena, .{ .label = "Graphics ", .value = try formatClockPair(ctx.arena, gpu.clock_graphics_mhz, gpu.clock_graphics_max_mhz) }); + if (gpu.clock_sm_mhz) |sm| { + try clock_lines.append(ctx.arena, .{ .label = "SM ", .value = try std.fmt.allocPrint(ctx.arena, " {d} MHz", .{sm}) }); + } + if (gpu.clock_soc_mhz) |soc| { + try clock_lines.append(ctx.arena, .{ .label = "SOC ", .value = try std.fmt.allocPrint(ctx.arena, " {d} MHz", .{soc}) }); + } + try clock_lines.append(ctx.arena, .{ .label = "Memory ", .value = try formatClockPair(ctx.arena, gpu.clock_mem_mhz, gpu.clock_mem_max_mhz) }); + const clocks_card: MetricCard = .{ .title = "Clocks", .lines = clock_lines.items }; + + var pcie_lines: std.ArrayList(MetricCard.LineEntry) = .empty; + try pcie_lines.append(ctx.arena, .{ .label = "Link ", .value = try formatPcieLink(ctx.arena, gpu.pcie_link_gen, gpu.pcie_link_width) }); + if (gpu.pcie_tx_kbps != null or gpu.pcie_rx_kbps != null) { + try pcie_lines.append(ctx.arena, .{ .label = "TX ", .value = try formatBandwidth(ctx.arena, gpu.pcie_tx_kbps orelse 0) }); + try pcie_lines.append(ctx.arena, .{ .label = "RX ", .value = try formatBandwidth(ctx.arena, gpu.pcie_rx_kbps orelse 0) }); + } + if (gpu.pcie_bandwidth_mbps) |bw| { + try pcie_lines.append(ctx.arena, .{ .label = "BW ", .value = try std.fmt.allocPrint(ctx.arena, " {d} Mb/s", .{bw}) }); + } + const pcie_card: MetricCard = .{ .title = "PCIe", .lines = pcie_lines.items }; + + const cards_flow: ColumnLayout = .{ + .children = &.{ ui.widget(&clocks_card), ui.widget(&pcie_card) }, + .min_child_width = 30, + }; + + // ── Compose vertical layout ───────────────────────────── + return common.pageFrame(ctx, .{ + .children = &.{ + header.widget(), + ui.widget(&charts_flow), + ui.widget(&cards_flow), + ui.widget(process_table), + }, + .gap = 1, + }, w, parent_widget); +} + +fn formatClockPair(arena: std.mem.Allocator, current: ?u64, max: ?u64) std.mem.Allocator.Error![]const u8 { + if (current == null and max == null) return " N/A"; + return std.fmt.allocPrint(arena, " {d} / {d} MHz", .{ current orelse 0, max orelse 0 }); +} + +fn formatPcieLink(arena: std.mem.Allocator, gen: ?u64, width: ?u64) std.mem.Allocator.Error![]const u8 { + if (gen == null and width == null) return " N/A"; + return std.fmt.allocPrint(arena, " Gen{d} x{d}", .{ gen orelse 0, width orelse 0 }); +} + +fn formatBandwidth(arena: std.mem.Allocator, kbps: u64) std.mem.Allocator.Error![]const u8 { + if (kbps >= 1_000_000) { + const gbps = @as(f64, @floatFromInt(kbps)) / 1_000_000.0; + return std.fmt.allocPrint(arena, " {d:.1} GB/s", .{gbps}); + } else if (kbps >= 1_000) { + const mbps = @as(f64, @floatFromInt(kbps)) / 1_000.0; + return std.fmt.allocPrint(arena, " {d:.1} MB/s", .{mbps}); + } else { + return std.fmt.allocPrint(arena, " {d} KB/s", .{kbps}); + } +} diff --git a/bin/zml-smi/tui/pages/detail/neuron_detail.zig b/bin/zml-smi/tui/pages/detail/neuron_detail.zig new file mode 100644 index 00000000..e0dc3bcd --- /dev/null +++ b/bin/zml-smi/tui/pages/detail/neuron_detail.zig @@ -0,0 +1,83 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; +const data = @import("../../data.zig"); +const utils = @import("../../lib/utils.zig"); +const ui = @import("../../lib/ui.zig"); +const ColumnLayout = @import("../../widgets/column_layout.zig"); +const MetricCard = @import("../../widgets/metric_card.zig"); +const ProcessTable = @import("../../widgets/process_table.zig"); +const common = @import("detail_common.zig"); + +pub fn draw( + state: *const data.SystemState, + process_table: *ProcessTable, + ctx: vxfw.DrawContext, + w: u16, + id: u16, + nc: data.NeuronInfo, + parent_widget: vxfw.Widget, +) std.mem.Allocator.Error!vxfw.Surface { + // ── Header ────────────────────────────────────────────── + const name_suffix: []const u8 = if (nc.driver_version) |v| + try std.fmt.allocPrint(ctx.arena, " ({s})", .{v}) + else + ""; + const header = try common.headerText(ctx.arena, id, try std.fmt.allocPrint(ctx.arena, "{s}{s}", .{ nc.name orelse "Unknown", name_suffix })); + + // ── Utilization + Memory charts ───────────────────────── + const util_chart = try common.historyChart(ctx, state, id, state.history.util, .{ + .title = "Core Utilization", + .value_label = try std.fmt.allocPrint(ctx.arena, "{d}%", .{nc.util_percent orelse 0}), + }); + + const mem_used = nc.mem_used_bytes orelse 0; + const mem_total = nc.mem_total_bytes orelse 0; + const mem_chart = try common.historyChart(ctx, state, id, state.history.mem_util, .{ + .title = "Memory", + .value_label = try std.fmt.allocPrint(ctx.arena, "{d}%", .{if (mem_total > 0) mem_used * 100 / mem_total else 0}), + .info_line = try std.fmt.allocPrint(ctx.arena, "HBM {d} / {d} MB", .{ + utils.bytesToMb(nc.mem_used_bytes), utils.bytesToMb(nc.mem_total_bytes), + }), + }); + + const charts_flow: ColumnLayout = .{ + .children = &.{ ui.widget(&util_chart), ui.widget(&mem_chart) }, + .min_child_width = 40, + .gap = 1, + }; + + // ── HBM Breakdown ─────────────────────────────────────── + const fmt_mb = struct { + fn f(arena: std.mem.Allocator, val: ?u64) std.mem.Allocator.Error![]const u8 { + return std.fmt.allocPrint(arena, " {d} MB", .{utils.bytesToMb(val)}); + } + }.f; + const hbm_card: MetricCard = .{ + .title = "HBM Breakdown", + .lines = &.{ + .{ .label = "Tensors ", .value = try fmt_mb(ctx.arena, nc.nc_tensors) }, + .{ .label = "Constants ", .value = try fmt_mb(ctx.arena, nc.nc_constants) }, + .{ .label = "Model Code ", .value = try fmt_mb(ctx.arena, nc.nc_model_code) }, + .{ .label = "Shared Scratch", .value = try fmt_mb(ctx.arena, nc.nc_shared_scratchpad) }, + .{ .label = "Nonshared ", .value = try fmt_mb(ctx.arena, nc.nc_nonshared_scratchpad) }, + .{ .label = "Runtime ", .value = try fmt_mb(ctx.arena, nc.nc_runtime) }, + .{ .label = "Driver ", .value = try fmt_mb(ctx.arena, nc.nc_driver) }, + .{ .label = "DMA Rings ", .value = try fmt_mb(ctx.arena, nc.nc_dma_rings) }, + .{ .label = "Collectives ", .value = try fmt_mb(ctx.arena, nc.nc_collectives) }, + .{ .label = "Notifications ", .value = try fmt_mb(ctx.arena, nc.nc_notifications) }, + .{ .label = "Uncategorized ", .value = try fmt_mb(ctx.arena, nc.nc_uncategorized) }, + }, + }; + + // ── Compose vertical layout ───────────────────────────── + return common.pageFrame(ctx, .{ + .children = &.{ + header.widget(), + ui.widget(&charts_flow), + ui.widget(&hbm_card), + ui.widget(process_table), + }, + .gap = 1, + }, w, parent_widget); +} diff --git a/bin/zml-smi/tui/pages/detail/tpu_detail.zig b/bin/zml-smi/tui/pages/detail/tpu_detail.zig new file mode 100644 index 00000000..ca6d3b66 --- /dev/null +++ b/bin/zml-smi/tui/pages/detail/tpu_detail.zig @@ -0,0 +1,54 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; +const data = @import("../../data.zig"); +const utils = @import("../../lib/utils.zig"); +const ui = @import("../../lib/ui.zig"); +const ColumnLayout = @import("../../widgets/column_layout.zig"); +const ProcessTable = @import("../../widgets/process_table.zig"); +const common = @import("detail_common.zig"); + +pub fn draw( + state: *const data.SystemState, + process_table: *ProcessTable, + ctx: vxfw.DrawContext, + w: u16, + id: u16, + tp: data.TpuInfo, + parent_widget: vxfw.Widget, +) std.mem.Allocator.Error!vxfw.Surface { + // ── Header ────────────────────────────────────────────── + const header = try common.headerText(ctx.arena, id, tp.name orelse "Unknown"); + + // ── Utilization + Memory charts ───────────────────────── + const util_chart = try common.historyChart(ctx, state, id, state.history.util, .{ + .title = "Duty Cycle", + .value_label = try std.fmt.allocPrint(ctx.arena, "{d}%", .{tp.util_percent orelse 0}), + }); + + const mem_used = tp.mem_used_bytes orelse 0; + const mem_total = tp.mem_total_bytes orelse 0; + const mem_chart = try common.historyChart(ctx, state, id, state.history.mem_util, .{ + .title = "Memory", + .value_label = try std.fmt.allocPrint(ctx.arena, "{d}%", .{if (mem_total > 0) mem_used * 100 / mem_total else 0}), + .info_line = try std.fmt.allocPrint(ctx.arena, "HBM {d} / {d} MB", .{ + utils.bytesToMb(tp.mem_used_bytes), utils.bytesToMb(tp.mem_total_bytes), + }), + }); + + const charts_flow: ColumnLayout = .{ + .children = &.{ ui.widget(&util_chart), ui.widget(&mem_chart) }, + .min_child_width = 40, + .gap = 1, + }; + + // ── Compose vertical layout ───────────────────────────── + return common.pageFrame(ctx, .{ + .children = &.{ + header.widget(), + ui.widget(&charts_flow), + ui.widget(process_table), + }, + .gap = 1, + }, w, parent_widget); +} diff --git a/bin/zml-smi/tui/pages/overview.zig b/bin/zml-smi/tui/pages/overview.zig new file mode 100644 index 00000000..e91253bd --- /dev/null +++ b/bin/zml-smi/tui/pages/overview.zig @@ -0,0 +1,119 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; +const data = @import("../data.zig"); +const ui = @import("../lib/ui.zig"); +const compose = @import("../lib/compose.zig"); +const image_cache = @import("../image_cache.zig"); +const ColumnLayout = @import("../widgets/column_layout.zig"); +const common = @import("detail/detail_common.zig"); +const DeviceCard = @import("../widgets/device_card.zig"); +const Logo = @import("../widgets/logo.zig"); +const InfoLines = @import("../widgets/info_lines.zig"); +const ProcessTable = @import("../widgets/process_table.zig"); + +const Overview = @This(); + +const two_col_breakpoint: u16 = 140; +const narrow_breakpoint: u16 = 80; +const max_info_width: u16 = 50; +/// Total width of the wide banner: logo box (incl. left margin) + info lines + right margin. +pub const host_line_width: u16 = (Logo.logo_width + 6) + max_info_width + 2; + +state: *const data.SystemState, +device_cards: []DeviceCard = &.{}, +process_table: ?*ProcessTable = null, // maybe null if we decide not to print the process table in print mode +viewing_device: *?u16 = undefined, +use_braille: bool = false, + +pub fn init(allocator: std.mem.Allocator, state: *const data.SystemState, process_table: ?*ProcessTable, viewing_device: *?u16) !Overview { + const count = state.deviceCount(); + const cards = try allocator.alloc(DeviceCard, count); + for (cards, 0..) |*card, i| { + card.* = .{ + .device_id = @intCast(i), + .state = state, + }; + } + return .{ + .state = state, + .device_cards = cards, + .process_table = process_table, + .viewing_device = viewing_device, + }; +} + +pub fn deinit(self: *Overview, allocator: std.mem.Allocator) void { + allocator.free(self.device_cards); +} + +fn drawNarrowBanner(self: *const Overview, ctx: vxfw.DrawContext, content_w: u16) !vxfw.Surface { + const logo: Logo = .{ .image = image_cache.global.get("logo") }; + const info_lines: InfoLines = .{ .state = self.state }; + + const children = [2]vxfw.Widget{ + try compose.sized(ctx.arena, try compose.center(ctx.arena, ui.widget(&logo)), .{ .width = content_w, .height = Logo.logo_height }), + ui.widget(&info_lines), + }; + const layout: ColumnLayout = .{ .children = &children, .gap = 1 }; + return ui.widget(&layout).draw(ui.fixedWidth(ctx, content_w)); +} + +fn drawWideBanner(self: *const Overview, ctx: vxfw.DrawContext, content_w: u16) !vxfw.Surface { + const logo: Logo = .{ .image = image_cache.global.get("logo") }; + const info_lines: InfoLines = .{ .state = self.state }; + + const logo_box_w = Logo.logo_width + 6; // +4 centering + 2 absorbed page margin + const info_max_w = @min(content_w -| logo_box_w, max_info_width); + const banner_h = @max(Logo.logo_height, InfoLines.entry_count); + + const flex_items = [2]vxfw.FlexItem{ + .{ .widget = try compose.sized(ctx.arena, try compose.center(ctx.arena, ui.widget(&logo)), .{ .width = logo_box_w, .height = banner_h }), .flex = 0 }, + .{ .widget = try compose.sized(ctx.arena, try compose.center(ctx.arena, ui.widget(&info_lines)), .{ .width = info_max_w, .height = banner_h }), .flex = 0 }, + }; + const sized_banner = try compose.sized(ctx.arena, (vxfw.FlexRow{ .children = &flex_items }).widget(), .{ .width = content_w, .height = banner_h }); + return sized_banner.draw(ui.fixedWidth(ctx, content_w)); +} + +pub fn draw(self: *Overview, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { + const w = ctx.max.width orelse 80; + const narrow = w < narrow_breakpoint; + const content_w: u16 = w -| 4; + const content_ctx = ui.fixedWidth(ctx, content_w); + + var sb = compose.surfaceBuilder(ctx.arena); + var row: i17 = 1; // top margin + + // ── Banner (Logo + Info Lines) ───────────────────────── + const banner_surf = try if (narrow) self.drawNarrowBanner(ctx, content_w) else self.drawWideBanner(ctx, content_w + 2); + try sb.add(row, if (narrow) @as(i17, 2) else 0, banner_surf); + row += @intCast(banner_surf.size.height + 1); + + // ── Device Grid ───────────────────────────────────────── + for (self.device_cards) |*card| { + card.use_braille = self.use_braille; + card.viewing_device = self.viewing_device; + } + const card_widgets = try ctx.arena.alloc(vxfw.Widget, self.device_cards.len); + for (self.device_cards, card_widgets) |*card, *cw| { + cw.* = ui.widget(card); + } + const device_grid: ColumnLayout = .{ + .children = card_widgets, + .min_child_width = two_col_breakpoint / 2, + .col_gap = 2, + .gap = 1, + }; + const grid_surf = try ui.widget(&device_grid).draw(content_ctx); + try sb.add(row, 2, grid_surf); + row += @intCast(grid_surf.size.height + 1); + + // ── Process Table ──────────────────────────────────────── + if (self.process_table) |pt| { + const pt_surf = try ui.widget(pt).draw(content_ctx); + try sb.add(row, 2, pt_surf); + row += @intCast(pt_surf.size.height); + } + + return sb.finish(.{ .width = w, .height = @intCast(row) }, ui.widget(self)); +} diff --git a/bin/zml-smi/tui/print.zig b/bin/zml-smi/tui/print.zig new file mode 100644 index 00000000..238534cd --- /dev/null +++ b/bin/zml-smi/tui/print.zig @@ -0,0 +1,91 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; +const data = @import("data.zig"); +const ui = @import("lib/ui.zig"); +const image_cache = @import("image_cache.zig"); +const Overview = @import("pages/overview.zig"); + +fn silentResize(vx: *vaxis.Vaxis, allocator: std.mem.Allocator, winsize: vaxis.Winsize) !void { + vx.screen.deinit(allocator); + vx.screen = try vaxis.Screen.init(allocator, winsize); + vx.screen.width_method = vx.caps.unicode; + vx.screen_last.deinit(allocator); + vx.screen_last = try vaxis.AllocatingScreen.init(allocator, winsize.cols, winsize.rows); +} + +pub fn run(allocator: std.mem.Allocator, io: std.Io, env_map: *std.process.Environ.Map, state: *data.SystemState) !void { + var tty_buffer: [4096]u8 = undefined; + var tty = try vaxis.Tty.init(io, &tty_buffer); + defer tty.deinit(); + + var vx = try vaxis.init(io, allocator, env_map, .{}); + defer { + // Reset terminal modes without clearing screen content. + vx.resetModes(tty.writer()) catch {}; + var buf: [256]u8 = undefined; + var discarding: std.Io.Writer.Discarding = .init(&buf); + vx.deinit(allocator, &discarding.writer); + } + + const ws = try tty.getWinsize(); + try silentResize(&vx, allocator, ws); + + { + const EventLoop = vaxis.Loop(vxfw.Event); + var loop: EventLoop = .init(io, &tty, &vx); + try loop.start(); + defer loop.stop(); + + // queryTerminal sends cursor home (\x1b[H) for capability detection. + // Save/restore cursor so we stay at the current position. + const writer = tty.writer(); + try writer.writeAll("\x1b7"); // DECSC: save cursor + try writer.flush(); + try vx.queryTerminal(writer, .fromSeconds(1)); + try writer.writeAll("\x1b8"); // DECRC: restore cursor + try writer.flush(); + } + + image_cache.global.loadAll(&vx, allocator, tty.writer()); + defer image_cache.global.deinit(allocator); + + vxfw.DrawContext.init(vx.screen.width_method); + + var arena = std.heap.ArenaAllocator.init(allocator); + defer arena.deinit(); + + const content_w: u16 = @min(@as(u16, @intCast(vx.screen.width)), Overview.host_line_width + 2); + + var viewing_device: ?u16 = null; + var overview = try Overview.init(allocator, state, null, &viewing_device); + defer overview.deinit(allocator); + + const draw_ctx: vxfw.DrawContext = .{ + .arena = arena.allocator(), + .min = .{ .width = content_w }, + .max = .{ .width = content_w, .height = null }, + .cell_size = .{ + .width = if (vx.screen.width > 0) vx.screen.width_pix / vx.screen.width else 0, + .height = if (vx.screen.height > 0) vx.screen.height_pix / vx.screen.height else 0, + }, + }; + + const surface = try overview.draw(draw_ctx); + + try silentResize(&vx, allocator, .{ + .rows = surface.size.height, + .cols = content_w, + .x_pixel = ws.x_pixel, + .y_pixel = ws.y_pixel, + }); + + const win = vx.window(); + const root_win = win.child(.{ + .width = surface.size.width, + .height = surface.size.height, + }); + surface.render(root_win, ui.widget(&overview)); + + try vx.prettyPrint(tty.writer()); +} diff --git a/bin/zml-smi/tui/theme.zig b/bin/zml-smi/tui/theme.zig new file mode 100644 index 00000000..e037f67f --- /dev/null +++ b/bin/zml-smi/tui/theme.zig @@ -0,0 +1,50 @@ +const vaxis = @import("vaxis"); + +const Color = vaxis.Cell.Color; +const Style = vaxis.Cell.Style; + +/// Returns a color on a green → yellow → red gradient based on percentage (0-100). +pub fn colorForPercent(pct: u8) Color { + if (pct <= 50) { + // Green (#00cc66) → Yellow (#ffcc00) + const t: u16 = @as(u16, pct) * 255 / 50; + return .{ .rgb = .{ @intCast(t), 204, @intCast(102 -| (t * 102 / 255)) } }; + } else if (pct <= 80) { + // Yellow (#ffcc00) → Orange (#ff6600) + const t: u16 = (@as(u16, pct) - 50) * 255 / 30; + return .{ .rgb = .{ 255, @intCast(204 -| (t * 102 / 255)), 0 } }; + } else { + // Orange (#ff6600) → Red (#ff2200) + const t: u16 = (@as(u16, pct) - 80) * 255 / 20; + return .{ .rgb = .{ 255, @intCast(102 -| (t * 68 / 255)), 0 } }; + } +} + +/// Returns a color for temperature (20-100°C range). +pub fn colorForTemp(temp_c: u16) Color { + if (temp_c <= 20) { + return colorForPercent(0); + } + if (temp_c >= 100) { + return colorForPercent(100); + } + const pct: u8 = @intCast((@as(u32, temp_c) - 20) * 100 / 80); + return colorForPercent(pct); +} + +// ── Style Constants ────────────────────────────────────────────── + +pub const accent: Color = .{ .rgb = .{ 120, 180, 255 } }; +pub const accent_secondary: Color = .{ .rgb = .{ 200, 160, 255 } }; +pub const dim: Color = .{ .rgb = .{ 100, 100, 120 } }; +pub const text_primary: Color = .{ .rgb = .{ 230, 230, 240 } }; +pub const text_secondary: Color = .{ .rgb = .{ 160, 160, 175 } }; +pub const surface_border: Color = .{ .rgb = .{ 70, 70, 90 } }; +pub const gauge_empty: Color = .{ .rgb = .{ 50, 50, 65 } }; + +pub const header_style: Style = .{ .bold = true, .fg = accent }; +pub const title_style: Style = .{ .bold = true, .fg = accent_secondary }; +pub const label_style: Style = .{ .fg = text_secondary }; +pub const value_style: Style = .{ .bold = true, .fg = text_primary }; +pub const border_style: Style = .{ .fg = surface_border }; +pub const dim_style: Style = .{ .fg = dim }; diff --git a/bin/zml-smi/tui/top.zig b/bin/zml-smi/tui/top.zig new file mode 100644 index 00000000..74b3796b --- /dev/null +++ b/bin/zml-smi/tui/top.zig @@ -0,0 +1,168 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; + +const data = @import("data.zig"); +const ui = @import("lib/ui.zig"); +const compose = @import("lib/compose.zig"); +const image_cache = @import("image_cache.zig"); +const Logo = @import("widgets/logo.zig"); +const ProcessTable = @import("widgets/process_table.zig"); +const Overview = @import("pages/overview.zig"); +const Detail = @import("pages/detail.zig"); +const StatusLine = @import("widgets/status_line.zig"); + +const min_width: u16 = 45; +const min_height: u16 = 24; + +const ScrollState = struct { + row: i17 = 0, + col: i17 = 0, + + pub fn reset(self: *ScrollState) void { + self.row = 0; + self.col = 0; + } + + pub fn scrollBy(self: *ScrollState, dr: i17, dc: i17) void { + self.row = @max(0, self.row + dr); + self.col = @max(0, self.col + dc); + } + + pub fn clamp(self: *ScrollState, content_h: u16, viewport_h: u16, content_w: u16, viewport_w: u16) void { + const max_row: i17 = @max(0, @as(i17, content_h) - @as(i17, viewport_h)); + const max_col: i17 = @max(0, @as(i17, content_w) - @as(i17, viewport_w)); + self.row = std.math.clamp(self.row, 0, max_row); + self.col = std.math.clamp(self.col, 0, max_col); + } +}; + +const Model = struct { + allocator: std.mem.Allocator, + state: *data.SystemState, + viewing_device: ?u16 = null, + prev_viewing_device: ?u16 = null, + scroll: ScrollState = .{}, + vx: *vaxis.Vaxis, + tty: *vaxis.Tty, + process_table: ProcessTable = .{}, + overview: Overview = undefined, + + fn deinit(self: *Model) void { + self.process_table.deinit(self.allocator); + self.overview.deinit(self.allocator); + image_cache.global.deinit(self.allocator); + } + + pub fn handleEvent(self: *Model, ctx: *vxfw.EventContext, event: vxfw.Event) anyerror!void { + switch (event) { + .init => { + self.overview = try Overview.init(self.allocator, self.state, &self.process_table, &self.viewing_device); + self.overview.use_braille = true; + image_cache.global.loadAll(self.vx, self.allocator, self.tty.writer()); + try ctx.tick(1000, ui.widget(self)); + }, + .tick => { + self.state.recordHistory(); + try ctx.tick(self.state.tui_refresh_rate, ui.widget(self)); + ctx.redraw = true; + }, + .key_press => |key| { + if (key.matches('q', .{}) or key.matches('c', .{ .ctrl = true })) { + ctx.quit = true; + return; + } + if (self.viewing_device != null) { + if (key.matches(vaxis.Key.escape, .{}) or key.matches(vaxis.Key.backspace, .{})) { + self.viewing_device = null; + return ctx.consumeAndRedraw(); + } + } else { + if (key.matches('v', .{})) { + self.overview.use_braille = !self.overview.use_braille; + return ctx.consumeAndRedraw(); + } + } + if (key.matches(vaxis.Key.down, .{})) { + self.scroll.scrollBy(1, 0); + } else if (key.matches(vaxis.Key.up, .{})) { + self.scroll.scrollBy(-1, 0); + } else if (key.matches(vaxis.Key.right, .{})) { + self.scroll.scrollBy(0, 1); + } else if (key.matches(vaxis.Key.left, .{})) { + self.scroll.scrollBy(0, -1); + } else return; + return ctx.consumeAndRedraw(); + }, + .mouse => |mouse| { + switch (mouse.button) { + .wheel_up => self.scroll.scrollBy(-3, 0), + .wheel_down => self.scroll.scrollBy(3, 0), + .wheel_left => self.scroll.scrollBy(0, 3), + .wheel_right => self.scroll.scrollBy(0, -3), + else => return, + } + return ctx.consumeAndRedraw(); + }, + else => {}, + } + } + + pub fn draw(self: *Model, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { + const screen = ctx.max.size(); + const content_w = @max(screen.width, min_width); + + if (self.viewing_device != self.prev_viewing_device) { + self.scroll.reset(); + self.process_table.resetScroll(); + self.prev_viewing_device = self.viewing_device; + } + + self.process_table.prepare(self.state, self.viewing_device); + + var content_surf: vxfw.Surface = undefined; + + if (self.viewing_device) |dev_id| { + const detail: Detail = .{ + .state = self.state, + .device_id = dev_id, + .process_table = &self.process_table, + }; + content_surf = try detail.draw(ui.fixedWidth(ctx, content_w)); + } else { + content_surf = try self.overview.draw(ui.fixedWidth(ctx, content_w)); + } + + content_surf.size.height = @max(content_surf.size.height, min_height); + + const content_h = screen.height -| 1; + self.scroll.clamp(content_surf.size.height, content_h, content_surf.size.width, screen.width); + + const status_line: StatusLine = .{ + .viewing_device = self.viewing_device, + .use_braille = self.overview.use_braille, + }; + const status_surf = try status_line.draw(ui.fixedSize(ctx, screen.width, 1)); + + var sb = compose.surfaceBuilder(ctx.arena); + try sb.add(-self.scroll.row, -self.scroll.col, content_surf); + try sb.addZ(@intCast(content_h), 0, status_surf, 1); + return sb.finish(screen, ui.widget(self)); + } +}; + +pub fn run(allocator: std.mem.Allocator, io: std.Io, env_map: *std.process.Environ.Map, state: *data.SystemState) !void { + var tty_buffer: [4096]u8 = undefined; + var app = try vxfw.App.init(io, allocator, env_map, &tty_buffer); + defer app.deinit(); + + var model: Model = .{ + .allocator = allocator, + .state = state, + .vx = &app.vx, + .tty = &app.tty, + }; + defer model.deinit(); + + try app.run(ui.widget(&model), .{}); +} diff --git a/bin/zml-smi/tui/tui.zig b/bin/zml-smi/tui/tui.zig new file mode 100644 index 00000000..015e3ce2 --- /dev/null +++ b/bin/zml-smi/tui/tui.zig @@ -0,0 +1,3 @@ +pub const data = @import("data.zig"); +pub const print = @import("print.zig"); +pub const top = @import("top.zig"); diff --git a/bin/zml-smi/tui/widgets/braille_chart.zig b/bin/zml-smi/tui/widgets/braille_chart.zig new file mode 100644 index 00000000..498739d1 --- /dev/null +++ b/bin/zml-smi/tui/widgets/braille_chart.zig @@ -0,0 +1,76 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; +const theme = @import("../theme.zig"); + +const ui = @import("../lib/ui.zig"); + +const BrailleChart = @This(); + +values: []const u8, // 0-100 +height: u16 = 2, + +pub const chart_bg: vaxis.Cell.Color = .{ .rgb = .{ 30, 30, 40 } }; + +/// Braille dot bit positions: [row_in_cell (0-3)][col_in_cell (0-1)] +/// Each braille character is a 2x4 dot grid encoded as U+2800 + pattern byte. +pub const dot_bits = [4][2]u8{ + .{ 0x01, 0x08 }, // row 0 + .{ 0x02, 0x10 }, // row 1 + .{ 0x04, 0x20 }, // row 2 + .{ 0x40, 0x80 }, // row 3 +}; + +pub fn draw(self: *const BrailleChart, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { + const w = ctx.max.width orelse 40; + var surface = try vxfw.Surface.init(ctx.arena, ui.widget(self), .{ .width = w, .height = self.height }); + try self.renderTo(ctx.arena, &surface, 0, 0, w); + return surface; +} + +/// Render braille data into an existing surface at the given offset. +pub fn renderTo(self: *const BrailleChart, arena: std.mem.Allocator, surface: *vxfw.Surface, col_offset: u16, row_offset: u16, w: u16) std.mem.Allocator.Error!void { + const h = self.height; + const dot_rows: u32 = @as(u32, h) * 4; + const dot_cols: u32 = @as(u32, w) * 2; + const data_len: u32 = @intCast(self.values.len); + + for (0..w) |cell_col| { + var max_val: u8 = 0; + for (0..2) |dx| { + const dot_x: u32 = @as(u32, @intCast(cell_col)) * 2 + @as(u32, @intCast(dx)); + if (dot_x + data_len >= dot_cols) { + max_val = @max(max_val, self.values[@intCast(dot_x + data_len - dot_cols)]); + } + } + const color = theme.colorForPercent(max_val); + + for (0..h) |cell_row| { + var pattern: u8 = 0; + for (0..2) |dx| { + const dot_x: u32 = @as(u32, @intCast(cell_col)) * 2 + @as(u32, @intCast(dx)); + if (dot_x + data_len >= dot_cols) { + const val = self.values[@intCast(dot_x + data_len - dot_cols)]; + const fill: u32 = @as(u32, val) * dot_rows / 100; + const threshold: u32 = dot_rows - fill; + for (0..4) |dr| { + const abs_row: u32 = @as(u32, @intCast(cell_row)) * 4 + @as(u32, @intCast(dr)); + if (abs_row >= threshold) { + pattern |= dot_bits[dr][dx]; + } + } + } + } + + const buf = try arena.alloc(u8, 3); + buf[0] = 0xE2; + buf[1] = 0xA0 + (pattern >> 6); + buf[2] = 0x80 + (pattern & 0x3F); + + surface.writeCell(col_offset + @as(u16, @intCast(cell_col)), row_offset + @as(u16, @intCast(cell_row)), .{ + .char = .{ .grapheme = buf, .width = 1 }, + .style = .{ .fg = color, .bg = chart_bg }, + }); + } + } +} diff --git a/bin/zml-smi/tui/widgets/chart.zig b/bin/zml-smi/tui/widgets/chart.zig new file mode 100644 index 00000000..5421cd1f --- /dev/null +++ b/bin/zml-smi/tui/widgets/chart.zig @@ -0,0 +1,141 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; +const theme = @import("../theme.zig"); +const ui = @import("../lib/ui.zig"); +const TitledBorder = @import("titled_border.zig"); +const BrailleChart = @import("braille_chart.zig"); + +const Chart = @This(); + +// Layout constants for rows below the chart data area +const x_axis_h: u16 = 1; // x-axis line +const time_labels_h: u16 = 1; // time labels below x-axis +const info_line_h: u16 = 1; // info text below time labels + +title: []const u8, +data: []const u8, // 0-100 normalized +value_label: ?[]const u8 = null, +info_line: ?[]const u8 = null, +y_min: u32 = 0, +y_max: u32 = 100, +y_unit: []const u8 = "%", +chart_height: u16 = 6, +tui_refresh_rate: u16, + +pub fn draw(self: *const Chart, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { + const tb: TitledBorder = .{ + .child = ui.drawWidget(self, drawContent), + .title = self.title, + .value_label = self.value_label, + }; + return tb.draw(ctx); +} + +fn drawContent(self: *const Chart, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { + const max_w = ctx.max.width orelse 40; + const content_h: u16 = self.chart_height + x_axis_h + time_labels_h + info_line_h; + + // ── Y-axis label widths ────────────────────────────────── + const y_labels = [3]struct { row: u16, val: u32 }{ + .{ .row = 0, .val = self.y_max }, + .{ .row = self.chart_height / 2, .val = (self.y_min + self.y_max) / 2 }, + .{ .row = self.chart_height, .val = self.y_min }, + }; + var y_strs: [3][]const u8 = undefined; + var y_str_widths: [3]u16 = undefined; + var y_label_w: u16 = 0; + for (&y_strs, &y_str_widths, y_labels) |*s, *sw, lbl| { + s.* = try std.fmt.allocPrint(ctx.arena, "{d}{s}", .{ lbl.val, self.y_unit }); + sw.* = @intCast(ctx.stringWidth(s.*)); + y_label_w = @max(y_label_w, sw.*); + } + + // ── Column layout ──────────────────────────────────────── + const chart_start: u16 = y_label_w + 1; + const chart_w: u16 = if (max_w > chart_start) max_w - chart_start else 0; + + var surface = try vxfw.Surface.init(ctx.arena, ui.drawWidget(self, drawContent), .{ .width = max_w, .height = content_h }); + if (chart_w == 0) { + return surface; + } + + // ── Y-axis labels + axis line ──────────────────────────── + for (y_strs, y_str_widths, y_labels) |text, sw, lbl| { + writeStr(&surface, y_label_w - sw, lbl.row, text, theme.dim_style); + } + for (0..self.chart_height + 1) |r| { + const row: u16 = @intCast(r); + const ch: []const u8 = if (row == self.chart_height) + "\u{2514}" + else + "\u{2502}"; + surface.writeCell(y_label_w, row, .{ + .char = .{ .grapheme = ch, .width = 1 }, + .style = theme.dim_style, + }); + } + + // ── Braille data area ──────────────────────────────────── + const braille: BrailleChart = .{ .values = self.data, .height = self.chart_height }; + try braille.renderTo(ctx.arena, &surface, chart_start, 0, chart_w); + + // ── X-axis line ────────────────────────────────────────── + const tick_mid: u16 = chart_w / 2; + const tick_right: u16 = chart_w -| 1; + for (0..chart_w) |c| { + const col: u16 = @intCast(c); + const is_tick = (col == 0 or col == tick_mid or col == tick_right); + surface.writeCell(chart_start + col, self.chart_height, .{ + .char = .{ .grapheme = if (is_tick) "\u{252C}" else "\u{2500}", .width = 1 }, + .style = theme.dim_style, + }); + } + + // ── X-axis time labels ─────────────────────────────────── + const total_ms: u32 = @as(u32, chart_w) * 2 * @as(u32, self.tui_refresh_rate); + const total_s: u16 = @intCast(std.math.divCeil(u32, total_ms, 1000) catch unreachable); + const mid_s: u16 = @intCast(std.math.divCeil(u32, total_ms / 2, 1000) catch unreachable); + const label_row = self.chart_height + x_axis_h; + const left_str = try fmtTime(ctx.arena, total_s); + const mid_str = try fmtTime(ctx.arena, mid_s); + + writeStr(&surface, chart_start, label_row, left_str, theme.dim_style); + + const mid_w: u16 = @intCast(ctx.stringWidth(mid_str)); + writeStr(&surface, chart_start + tick_mid -| (mid_w / 2), label_row, mid_str, theme.dim_style); + + const right_str: []const u8 = "now"; + const right_w: u16 = @intCast(ctx.stringWidth(right_str)); + writeStr(&surface, chart_start + tick_right + 1 -| right_w, label_row, right_str, theme.dim_style); + + // ── Info line ──────────────────────────────────────────── + if (self.info_line) |info| { + writeStr(&surface, chart_start, self.chart_height + x_axis_h + time_labels_h, info, theme.dim_style); + } + + return surface; +} + +fn writeStr(surface: *vxfw.Surface, col: u16, row: u16, text: []const u8, style: vaxis.Cell.Style) void { + var c = col; + var i: usize = 0; + while (i < text.len) { + const len = std.unicode.utf8ByteSequenceLength(text[i]) catch 1; + const end = @min(i + len, text.len); + surface.writeCell(c, row, .{ + .char = .{ .grapheme = text[i..end], .width = 1 }, + .style = style, + }); + c += 1; + i = end; + } +} + +fn fmtTime(arena: std.mem.Allocator, seconds: u16) std.mem.Allocator.Error![]const u8 { + if (seconds >= 120) { + return std.fmt.allocPrint(arena, "-{d}m", .{seconds / 60}); + } else { + return std.fmt.allocPrint(arena, "-{d}s", .{seconds}); + } +} diff --git a/bin/zml-smi/tui/widgets/column_layout.zig b/bin/zml-smi/tui/widgets/column_layout.zig new file mode 100644 index 00000000..f307d40c --- /dev/null +++ b/bin/zml-smi/tui/widgets/column_layout.zig @@ -0,0 +1,64 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; +const ui = @import("../lib/ui.zig"); +const compose = @import("../lib/compose.zig"); + +const ColumnLayout = @This(); + +children: []const vxfw.Widget, +gap: u16 = 0, +col_gap: u16 = 0, +/// When set, children wrap into a grid with this minimum column width. +/// When null (default), behaves as a simple vertical stack. +min_child_width: ?u16 = null, + +pub fn draw(self: *const ColumnLayout, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { + const n = self.children.len; + if (n == 0) { + return vxfw.Surface.init(ctx.arena, ui.widget(self), ctx.min); + } + + const available_w = ctx.max.width orelse 80; + const cols: u16 = if (self.min_child_width) |mcw| + @intCast(@max(1, @min(n, available_w / @max(mcw, 1)))) + else + 1; + + var sb = compose.surfaceBuilder(ctx.arena); + var row_y: u16 = 0; + var max_width: u16 = 0; + var i: usize = 0; + + while (i < n) { + if (i > 0) { + row_y += self.gap; + } + + const row_end = @min(i + cols, n); + var row_h: u16 = 0; + + for (i..row_end) |j| { + const col_idx: u16 = @intCast(j - i); + + const col_x, const w = if (cols > 1) blk: { + const total_col_gaps = @as(u16, cols - 1) * self.col_gap; + const usable_w = available_w -| total_col_gaps; + const child_w = usable_w / cols; + const x = col_idx * (child_w + self.col_gap); + break :blk .{ x, if (col_idx == cols - 1) available_w - x else child_w }; + } else .{ 0, available_w }; + + const surf = try self.children[j].draw(ui.fixedWidth(ctx, w)); + try sb.add(row_y, col_x, surf); + max_width = @max(max_width, col_x + surf.size.width); + row_h = @max(row_h, surf.size.height); + } + + row_y += row_h; + i = row_end; + } + + const result_w = if (ctx.max.width) |mw| @min(@max(max_width, ctx.min.width), mw) else @max(max_width, ctx.min.width); + return sb.finish(.{ .width = result_w, .height = row_y }, ui.widget(self)); +} diff --git a/bin/zml-smi/tui/widgets/device_card.zig b/bin/zml-smi/tui/widgets/device_card.zig new file mode 100644 index 00000000..c7f7ac38 --- /dev/null +++ b/bin/zml-smi/tui/widgets/device_card.zig @@ -0,0 +1,148 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; +const data = @import("../data.zig"); +const theme = @import("../theme.zig"); +const utils = @import("../lib/utils.zig"); +const ui = @import("../lib/ui.zig"); +const image_cache = @import("../image_cache.zig"); +const MetricCard = @import("metric_card.zig"); + +const DeviceCard = @This(); + +device_id: u16 = 0, +highlighted: bool = false, + +state: *const data.SystemState = undefined, +use_braille: bool = false, +viewing_device: *?u16 = undefined, + +pub fn handleEvent(self: *DeviceCard, ctx: *vxfw.EventContext, event: vxfw.Event) anyerror!void { + switch (event) { + .mouse => |mouse| { + if (mouse.type == .press and mouse.button == .left) { + self.viewing_device.* = self.device_id; + return ctx.consumeAndRedraw(); + } + }, + .mouse_enter => { + self.highlighted = true; + try ctx.setMouseShape(.pointer); + return ctx.consumeAndRedraw(); + }, + .mouse_leave => { + self.highlighted = false; + try ctx.setMouseShape(.default); + ctx.redraw = true; + }, + else => {}, + } +} + +pub fn draw(self: *DeviceCard, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { + const i = self.device_id; + const dev = self.state.devices[i]; + const target: data.Target = std.meta.activeTag(dev.*); + const cf = switch (dev.*) { + inline else => |*sv| blk: { + const f = sv.front().*; + break :blk .{ + .name = f.name orelse "Unknown", + .util_percent = @as(u8, @intCast(@min(f.util_percent orelse 0, 100))), + .mem_used = f.mem_used_bytes orelse 0, + .mem_total = f.mem_total_bytes orelse 0, + }; + }, + }; + const mem_pct: u8 = if (cf.mem_total > 0) @intCast(@min(cf.mem_used * 100 / cf.mem_total, 100)) else 0; + + const device_label, const util_label = switch (target) { + .cuda, .rocm, .oneapi => .{ "GPU", "GPU" }, + .neuron => .{ "NC", "Core" }, + .tpu => .{ "TPU", "Duty" }, + }; + const driver_ver: []const u8 = switch (dev.*) { + .cuda => |*sv| blk: { + const gpu = sv.front(); + const drv = gpu.driver_version orelse ""; + const cuda = gpu.cuda_driver_version orelse ""; + break :blk if (drv.len > 0 and cuda.len > 0) + try std.fmt.allocPrint(ctx.arena, " ({s}, CUDA {s})", .{ drv, cuda }) + else if (drv.len > 0) + try std.fmt.allocPrint(ctx.arena, " ({s})", .{drv}) + else if (cuda.len > 0) + try std.fmt.allocPrint(ctx.arena, " (CUDA {s})", .{cuda}) + else + @as([]const u8, ""); + }, + .rocm, .oneapi => |*sv| if (sv.front().driver_version) |v| try std.fmt.allocPrint(ctx.arena, " ({s})", .{v}) else "", + .neuron => |*sv| if (sv.front().driver_version) |v| try std.fmt.allocPrint(ctx.arena, " ({s})", .{v}) else "", + else => "", + }; + const card_title = try std.fmt.allocPrint(ctx.arena, "{s} {d}: {s}{s}", .{ device_label, i, cf.name, driver_ver }); + + const dev_suffix = switch (dev.*) { + .cuda, .rocm, .oneapi => |*sv| blk: { + const gpu = sv.front().*; + const power = (gpu.power_mw orelse 0) / 1000; + const power_limit = (gpu.power_limit_mw orelse 0) / 1000; + + const power_limit_str = try std.fmt.allocPrint(ctx.arena, "{d}", .{power_limit}); + const power_str = try std.fmt.allocPrint(ctx.arena, "{d}", .{power}); + + const pad = try utils.repeatStr(ctx.arena, " ", @intCast(power_limit_str.len -| power_str.len)); + break :blk try std.fmt.allocPrint(ctx.arena, "{d}\u{00b0}C | {s}{s} W", .{ + gpu.temperature orelse 0, pad, power_str, + }); + }, + else => "", + }; + + const mem_used_mb = utils.bytesToMb(cf.mem_used); + const mem_total_mb = utils.bytesToMb(cf.mem_total); + const mem_total_str = try std.fmt.allocPrint(ctx.arena, "{d}", .{mem_total_mb}); + const mem_used_str = try std.fmt.allocPrint(ctx.arena, "{d}", .{mem_used_mb}); + + const mem_pad = try utils.repeatStr(ctx.arena, " ", @intCast(mem_total_str.len -| mem_used_str.len)); + const mem_suffix = try std.fmt.allocPrint(ctx.arena, "{s}{s} / {s} MB", .{ + mem_pad, mem_used_str, mem_total_str, + }); + + var card: MetricCard = .{ + .title = card_title, + .title_image = image_cache.global.get(switch (dev.*) { + .cuda => "gpu_cuda", + .rocm => "gpu_rocm", + .oneapi => "gpu_oneapi", + .neuron => "gpu_neuron", + .tpu => "gpu_tpu", + }), + .title_right = if (self.state.devices[i].isRemote()) "remote" else null, + .cell_size = ctx.cell_size, + .highlighted = self.highlighted, + }; + + const util_hist = try self.state.history.util[i].sliceLast(ctx.arena, data.history_len); + const mem_hist = try self.state.history.mem_util[i].sliceLast(ctx.arena, data.history_len); + const util_chart_data = try utils.normalizeRange(ctx.arena, util_hist, 0, 100); + const mem_chart_data = try utils.normalizeRange(ctx.arena, mem_hist, 0, 100); + + const chart_entries = [2]MetricCard.ChartEntry{ + .{ .label = util_label, .value = cf.util_percent, .suffix = dev_suffix, .data = util_chart_data }, + .{ .label = "MEM", .value = mem_pct, .suffix = mem_suffix, .data = mem_chart_data }, + }; + const gauge_entries = [2]MetricCard.GaugeEntry{ + .{ .label = util_label, .value = cf.util_percent, .suffix = dev_suffix }, + .{ .label = "MEM", .value = mem_pct, .suffix = mem_suffix }, + }; + + if (self.use_braille) { + card.charts = &chart_entries; + } else { + card.gauges = &gauge_entries; + } + + var card_surf = try card.draw(ctx); + card_surf.widget = ui.widget(self); + return card_surf; +} diff --git a/bin/zml-smi/tui/widgets/gauge.zig b/bin/zml-smi/tui/widgets/gauge.zig new file mode 100644 index 00000000..865a45f5 --- /dev/null +++ b/bin/zml-smi/tui/widgets/gauge.zig @@ -0,0 +1,103 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; +const theme = @import("../theme.zig"); +const utils = @import("../lib/utils.zig"); +const ui = @import("../lib/ui.zig"); + +const RichText = vxfw.RichText; + +const Gauge = @This(); + +value: u8, // 0-100 +label: ?[]const u8 = null, +suffix: ?[]const u8 = null, +/// Minimum label space to reserve (for aligning gauges with different label widths). +label_reserve: u16 = 0, +/// Minimum suffix space to reserve (for aligning gauges with different suffix widths). +suffix_reserve: u16 = 0, + +pub const sep_w: u16 = 3; // " " before suffix + +pub fn draw(self: *const Gauge, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { + const max_w = ctx.max.width orelse 30; + const width: u16 = @max(ctx.min.width, max_w); + + // Label width (use reserve for alignment if set) + var label_w: u16 = 0; + if (self.label) |label| { + label_w = @intCast(ctx.stringWidth(label)); + label_w += 1; // trailing space + } + const effective_label_w = @max(label_w, self.label_reserve); + + // Percentage text: " XX%" + const pct_str = try std.fmt.allocPrint(ctx.arena, "{d:>3}%", .{self.value}); + const pct_width: u16 = @intCast(ctx.stringWidth(pct_str)); + + // Suffix area (text + separator); use reserve for bar calculation, actual for rendering + const actual_suffix_area: u16 = if (self.suffix) |s| @as(u16, @intCast(ctx.stringWidth(s))) + sep_w else 0; + const effective_suffix_area = @max(actual_suffix_area, self.suffix_reserve); + + // Reserved: label + "[" + "]" + pct + suffix_area + const reserved = effective_label_w + pct_width + 2 + effective_suffix_area; // 2 for [ ] + const bar_width = width -| reserved; + const filled: u16 = @intCast(@as(u32, bar_width) * @as(u32, @min(self.value, 100)) / 100); + const empty_count = bar_width - filled; + + var spans: std.ArrayList(vaxis.Cell.Segment) = .empty; + + // Label (padded to effective_label_w for alignment) + if (self.label) |label| { + try spans.append(ctx.arena, .{ .text = label, .style = theme.label_style }); + const pad = effective_label_w -| label_w; + const pad_str = try utils.repeatStr(ctx.arena, " ", pad + 1); // +1 for trailing space + try spans.append(ctx.arena, .{ .text = pad_str, .style = theme.label_style }); + } + + // Opening bracket + try spans.append(ctx.arena, .{ .text = "[", .style = theme.dim_style }); + + // Filled blocks — per-block gradient color + for (0..filled) |i| { + const pos_pct: u8 = if (bar_width > 1) + @intCast(@as(u32, @intCast(i)) * 100 / (@as(u32, bar_width) - 1)) + else + 0; + try spans.append(ctx.arena, .{ + .text = "\u{2588}", + .style = .{ .fg = theme.colorForPercent(pos_pct) }, + }); + } + + // Empty blocks + if (empty_count > 0) { + const empty_str = try utils.repeatStr(ctx.arena, "\u{2591}", empty_count); + try spans.append(ctx.arena, .{ .text = empty_str, .style = .{ .fg = theme.gauge_empty } }); + } + + // Closing bracket (no space — pct% collides with bar) + try spans.append(ctx.arena, .{ .text = "]", .style = theme.dim_style }); + + // Percentage text + try spans.append(ctx.arena, .{ .text = pct_str, .style = .{ .bold = true, .fg = theme.colorForPercent(self.value) } }); + + // Suffix (right-aligned) + if (self.suffix) |suffix| { + const suffix_text_w: u16 = @intCast(ctx.stringWidth(suffix)); + const pad = effective_suffix_area -| suffix_text_w; + if (pad > 0) { + const pad_str = try utils.repeatStr(ctx.arena, " ", pad); + try spans.append(ctx.arena, .{ .text = pad_str, .style = theme.dim_style }); + } + try spans.append(ctx.arena, .{ .text = suffix, .style = theme.dim_style }); + } + + const rich: RichText = .{ + .text = spans.items, + .softwrap = false, + .overflow = .clip, + .width_basis = .parent, + }; + return rich.draw(ui.fixedSize(ctx, width, 1)); +} diff --git a/bin/zml-smi/tui/widgets/info_lines.zig b/bin/zml-smi/tui/widgets/info_lines.zig new file mode 100644 index 00000000..fcf1ae0d --- /dev/null +++ b/bin/zml-smi/tui/widgets/info_lines.zig @@ -0,0 +1,112 @@ +const std = @import("std"); + +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; + +const data = @import("../data.zig"); +const compose = @import("../lib/compose.zig"); +const ui = @import("../lib/ui.zig"); +const theme = @import("../theme.zig"); +const stamp = @import("bazel/stamp"); + +const InfoLines = @This(); + +state: *const data.SystemState, + +const labels = [_][]const u8{ + "zml-smi ", + "Hostname ", + "Kernel ", + "CPU ", + "Cores ", + "Memory ", + "Uptime ", + "Load ", + "Devices ", +}; + +pub const entry_count: u16 = labels.len; + +const Entry = struct { + label: []const u8, + value: []const u8, +}; + +pub fn draw(self: *const InfoLines, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { + const w = ctx.max.width orelse 80; + const host = self.state.host.front().*; + + const hostname_str = try std.fmt.allocPrint(ctx.arena, "{s}", .{host.hostname orelse "Unknown"}); + const kernel_str = try std.fmt.allocPrint(ctx.arena, "{s}", .{host.kernel orelse "Unknown"}); + const cpu_str = try std.fmt.allocPrint(ctx.arena, "{s}", .{host.cpu_name orelse "Unknown"}); + const uptime_str = try formatUptime(ctx.arena, host.uptime_seconds orelse 0); + const cores_str = try std.fmt.allocPrint(ctx.arena, "{d}", .{host.cpu_cores orelse 0}); + const load_str = try std.fmt.allocPrint(ctx.arena, "{d:.2} / {d:.2} / {d:.2}", .{ + host.load_1 orelse 0, host.load_5 orelse 0, host.load_15 orelse 0, + }); + const total_kib = host.mem_total_kib orelse 0; + const avail_kib = host.mem_available_kib orelse 0; + const used_gb = (total_kib -| avail_kib) / (1024 * 1024); + const total_gb = total_kib / (1024 * 1024); + const mem_str = try std.fmt.allocPrint(ctx.arena, "{d} / {d} GB", .{ used_gb, total_gb }); + const devices_str = try std.fmt.allocPrint(ctx.arena, "{d}", .{self.state.deviceCount()}); + const values = [entry_count][]const u8{ + stamp.stamp("STABLE_ZML_SMI_VERSION"), + hostname_str, + kernel_str, + cpu_str, + cores_str, + mem_str, + uptime_str, + load_str, + devices_str, + }; + var entries: [entry_count]Entry = undefined; + for (&entries, labels, values) |*e, label, value| { + e.* = .{ .label = label, .value = value }; + } + + var sb = compose.surfaceBuilder(ctx.arena); + + for (entries, 0..) |entry, i| { + if (w == 0) { + continue; + } + + const total_len = entry.label.len + entry.value.len; + const segments = if (total_len < w) blk: { + const pad_len = w - total_len; + const pad = try ctx.arena.alloc(u8, pad_len); + @memset(pad, '-'); + pad[pad_len - 1] = ' '; + break :blk try ctx.arena.dupe(vaxis.Cell.Segment, &.{ + .{ .text = entry.label, .style = theme.header_style }, + .{ .text = pad, .style = theme.dim_style }, + .{ .text = entry.value, .style = theme.value_style }, + }); + } else try ctx.arena.dupe(vaxis.Cell.Segment, &.{ + .{ .text = entry.label, .style = theme.header_style }, + .{ .text = entry.value, .style = theme.value_style }, + }); + + const text_surf = try ui.drawRichLine(ctx, segments, w); + + try sb.add(@intCast(i), 0, text_surf); + } + + const total_h: u16 = @intCast(entries.len); + return sb.finish(.{ .width = w, .height = total_h }, ui.widget(self)); +} + +fn formatUptime(arena: std.mem.Allocator, seconds: u64) std.mem.Allocator.Error![]const u8 { + const days = seconds / 86400; + const hours = (seconds % 86400) / 3600; + const mins = (seconds % 3600) / 60; + if (days > 0) { + return std.fmt.allocPrint(arena, "{d}d {d}h {d}m", .{ days, hours, mins }); + } else if (hours > 0) { + return std.fmt.allocPrint(arena, "{d}h {d}m", .{ hours, mins }); + } else { + return std.fmt.allocPrint(arena, "{d}m", .{mins}); + } +} diff --git a/bin/zml-smi/tui/widgets/logo.zig b/bin/zml-smi/tui/widgets/logo.zig new file mode 100644 index 00000000..e6970485 --- /dev/null +++ b/bin/zml-smi/tui/widgets/logo.zig @@ -0,0 +1,63 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; +const ui = @import("../lib/ui.zig"); +const compose = @import("../lib/compose.zig"); +const Image = @import("../lib/image.zig"); +const zml_logo = @import("zml/logo"); + +const Logo = @This(); + +style: vaxis.Cell.Style = .{}, +image: ?vaxis.Image = null, + +pub const logo_width: u16 = 28; +pub const logo_height: u16 = @intCast(zml_logo.zml_art_blocks[0].rows.len); + +pub fn draw(self: *const Logo, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { + if (self.image) |img| { + const img_w: Image = .{ .image = img, .rows = logo_height }; + return img_w.draw(ctx); + } + + var sb = compose.surfaceBuilder(ctx.arena); + for (0..logo_height, 0..) |logo_row, row| { + try sb.add(@intCast(row), 0, try self.drawLogoRow(ctx, logo_row)); + } + return sb.finish(.{ .width = logo_width, .height = logo_height }, ui.widget(self)); +} + +fn drawLogoRow(self: *const Logo, ctx: vxfw.DrawContext, row_index: usize) std.mem.Allocator.Error!vxfw.Surface { + var segments: std.ArrayList(vaxis.Cell.Segment) = .empty; + for (zml_logo.zml_art_blocks[0..]) |block| { + const row = block.rows[row_index]; + if (row.text.len == 0) continue; + + var view: std.unicode.Utf8View = .initUnchecked(row.text); + var iter = view.iterator(); + while (iter.nextCodepointSlice()) |codepoint| { + const shiny = zml_logo.isShineGlyph(codepoint); + try segments.append(ctx.arena, .{ + .text = codepoint, + .style = .{ + .fg = switch (if (shiny) row.shine else row.color) { + .reset => .default, + .cyan => .{ .rgb = .{ 0, 255, 255 } }, + .sky => .{ .rgb = .{ 0, 221, 255 } }, + .blue => .{ .rgb = .{ 135, 143, 255 } }, + .violet => .{ .rgb = .{ 215, 90, 219 } }, + .pink => .{ .rgb = .{ 242, 48, 174 } }, + .shine_cyan => .{ .rgb = .{ 140, 255, 255 } }, + .shine_sky => .{ .rgb = .{ 128, 244, 255 } }, + .shine_blue => .{ .rgb = .{ 205, 210, 255 } }, + .shine_violet => .{ .rgb = .{ 255, 170, 255 } }, + .shine_pink => .{ .rgb = .{ 255, 150, 220 } }, + }, + .bg = self.style.bg, + .bold = shiny, + }, + }); + } + } + return ui.drawRichLine(ctx, segments.items, logo_width); +} diff --git a/bin/zml-smi/tui/widgets/metric_card.zig b/bin/zml-smi/tui/widgets/metric_card.zig new file mode 100644 index 00000000..6ceb9069 --- /dev/null +++ b/bin/zml-smi/tui/widgets/metric_card.zig @@ -0,0 +1,165 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; +const theme = @import("../theme.zig"); +const ui = @import("../lib/ui.zig"); +const compose = @import("../lib/compose.zig"); +const Gauge = @import("gauge.zig"); +const BrailleChart = @import("braille_chart.zig"); +const TitledBorder = @import("titled_border.zig"); + +const MetricCard = @This(); + +title: []const u8, +title_image: ?vaxis.Image = null, +title_right: ?[]const u8 = null, +cell_size: vxfw.Size = .{}, +/// Lines of "label: value" pairs to display. +lines: []const LineEntry = &.{}, +/// Optional gauge bars. +gauges: []const GaugeEntry = &.{}, +/// Optional braille mini-charts. +charts: []const ChartEntry = &.{}, +/// Whether this card is highlighted (e.g. hovered). +highlighted: bool = false, + +pub const LineEntry = struct { + label: []const u8, + value: []const u8, + style: vaxis.Cell.Style = theme.value_style, +}; + +pub const GaugeEntry = struct { + label: []const u8, + value: u8, + suffix: ?[]const u8 = null, +}; + +pub const ChartEntry = struct { + label: []const u8, + value: u8, + suffix: ?[]const u8 = null, + data: []const u8, // history values 0-100 + chart_height: u16 = 3, +}; + +pub fn draw(self: *const MetricCard, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { + const bstyle: vaxis.Cell.Style = if (self.highlighted) + .{ .fg = theme.accent, .bold = true } + else + theme.border_style; + const tb: TitledBorder = .{ + .child = ui.drawWidget(self, drawContent), + .title = self.title, + .title_image = self.title_image, + .value_label = self.title_right, + .cell_size = self.cell_size, + .border_style = bstyle, + }; + return tb.draw(ctx); +} + +fn maxFieldWidth(ctx: vxfw.DrawContext, entries: anytype, comptime field: []const u8) u16 { + var max_w: u16 = 0; + for (entries) |entry| { + const raw = @field(entry, field); + const value: []const u8 = if (@typeInfo(@TypeOf(raw)) == .optional) raw orelse continue else raw; + max_w = @max(max_w, @as(u16, @intCast(ctx.stringWidth(value)))); + } + return max_w; +} + +fn drawContent(self: *const MetricCard, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { + const inner_w = ctx.max.width orelse 40; + + var sb = compose.surfaceBuilder(ctx.arena); + var row: i17 = 0; + + // Charts (gauge-like layout: label [braille] pct% suffix, braille spans multiple rows) + { + const max_chart_label_w = maxFieldWidth(ctx, self.charts, "label") + 1; // +1 trailing space + const max_chart_suffix_w = maxFieldWidth(ctx, self.charts, "suffix") + Gauge.sep_w; + + for (self.charts, 0..) |chart, ci| { + if (ci > 0) { + row += 1; + } + + const ch_h = chart.chart_height; + + const pct_str = try std.fmt.allocPrint(ctx.arena, "{d:>3}%", .{chart.value}); + const pct_w: u16 = @intCast(ctx.stringWidth(pct_str)); + + const reserved = max_chart_label_w + pct_w + 1 + max_chart_suffix_w; + const braille_w: u16 = if (reserved < inner_w) inner_w - reserved else 0; + const text_row: i17 = row + @as(i17, @intCast(ch_h / 2)); + + // Label (vertically centered, col 0) + const label_surf = try ui.drawRichLine(ctx, &.{ + .{ .text = chart.label, .style = theme.label_style }, + }, max_chart_label_w); + try sb.add(text_row, 0, label_surf); + + // Braille chart (all rows, after label) + if (braille_w > 0) { + const braille: BrailleChart = .{ .values = chart.data, .height = ch_h }; + const braille_surf = try braille.draw(ui.fixedSize(ctx, braille_w, ch_h)); + try sb.add(row, @intCast(max_chart_label_w), braille_surf); + } + + // Pct + suffix (vertically centered, after braille) + const right_col: i17 = @intCast(max_chart_label_w + braille_w); + const pct_surf = try ui.drawRichLine(ctx, &.{ + .{ .text = " ", .style = .{} }, + .{ .text = pct_str, .style = .{ .bold = true, .fg = theme.colorForPercent(chart.value) } }, + }, pct_w + 1); + try sb.add(text_row, right_col, pct_surf); + + if (chart.suffix) |suffix| { + const suffix_text_w: u16 = @intCast(ctx.stringWidth(suffix)); + const suffix_col: i17 = @intCast(inner_w -| suffix_text_w); + const suffix_surf = try ui.drawRichLine(ctx, &.{ + .{ .text = suffix, .style = theme.dim_style }, + }, suffix_text_w); + try sb.add(text_row, suffix_col, suffix_surf); + } + + row += @intCast(ch_h); + } + } + + // Gap between charts and gauges + if (self.charts.len > 0 and self.gauges.len > 0) { + row += 1; + } + + // Gauges (with 1-row gap between consecutive gauges) + const max_label_w = maxFieldWidth(ctx, self.gauges, "label") + 1; // +1 trailing space + const max_suffix_w = maxFieldWidth(ctx, self.gauges, "suffix") + Gauge.sep_w; + for (self.gauges, 0..) |ge, gi| { + const gauge: Gauge = .{ + .value = ge.value, + .label = ge.label, + .suffix = ge.suffix, + .label_reserve = max_label_w, + .suffix_reserve = max_suffix_w, + }; + const gauge_surf = try gauge.draw(ui.fixedSize(ctx, inner_w, 1)); + try sb.add(row, 0, gauge_surf); + row += 1; + if (gi + 1 < self.gauges.len) row += 1; // gap between gauges + } + + // Text lines + for (self.lines) |line| { + const text_surf = try ui.drawRichLine(ctx, &.{ + .{ .text = line.label, .style = theme.label_style }, + .{ .text = line.value, .style = line.style }, + }, inner_w); + try sb.add(row, 0, text_surf); + row += 1; + } + + const content_h: u16 = @intCast(@max(row, 1)); + return sb.finish(.{ .width = inner_w, .height = content_h }, ui.drawWidget(self, drawContent)); +} diff --git a/bin/zml-smi/tui/widgets/process_table.zig b/bin/zml-smi/tui/widgets/process_table.zig new file mode 100644 index 00000000..58b20d51 --- /dev/null +++ b/bin/zml-smi/tui/widgets/process_table.zig @@ -0,0 +1,183 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; +const data = @import("../data.zig"); +const theme = @import("../theme.zig"); +const ui = @import("../lib/ui.zig"); +const compose = @import("../lib/compose.zig"); +const TitledBorder = @import("titled_border.zig"); +const ColumnLayout = @import("column_layout.zig"); +const pi = @import("zml-smi/info").process_info; +const ProcessInfo = pi.ProcessInfo; + +const RichText = vxfw.RichText; +const Segment = vaxis.Cell.Segment; + +const ProcessTable = @This(); + +const max_visible_rows: u16 = 20; + +scroll_bars: vxfw.ScrollBars = .{ + .scroll_view = .{ + .children = .{ .builder = .{ .userdata = undefined, .buildFn = buildRow } }, + .draw_cursor = false, + }, + .draw_horizontal_scrollbar = false, +}, +merged: std.ArrayList(ProcessInfo) = .empty, + +pub fn deinit(self: *ProcessTable, allocator: std.mem.Allocator) void { + self.merged.deinit(allocator); +} + +pub fn prepare(self: *ProcessTable, state: *data.SystemState, device_id: ?u16) void { + self.scroll_bars.scroll_view.children.builder.userdata = self; + self.merged.clearRetainingCapacity(); + + for (state.process_lists) |pl| { + const pl_items = pl.front().items; + if (device_id) |did| { + for (pl_items) |entry| { + if (entry.device_idx != did) { + continue; + } + self.merged.append(state.gpa, entry) catch break; + } + } else { + self.merged.appendSlice(state.gpa, pl_items) catch break; + } + } + + // Enrich with host data (uid, username, cpu%, rss, cmdline) + state.enricher.enrich(state.io, self.merged.items); + + // Sort by dev_mem descending + std.sort.insertion(ProcessInfo, self.merged.items, {}, struct { + fn cmp(_: void, a: ProcessInfo, b: ProcessInfo) bool { + return (a.dev_mem_kib orelse 0) > (b.dev_mem_kib orelse 0); + } + }.cmp); + + const count: u32 = @intCast(self.merged.items.len); + self.scroll_bars.estimated_content_height = count; + self.scroll_bars.scroll_view.item_count = count; +} + +pub fn resetScroll(self: *ProcessTable) void { + self.scroll_bars.scroll_view.scroll = .{}; + self.scroll_bars.scroll_view.cursor = 0; +} + +pub fn draw(self: *ProcessTable, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { + const w = ctx.max.width orelse 80; + const row_count = self.merged.items.len; + + var header_rt: RichText = .{ .text = &header_segs, .softwrap = false, .overflow = .clip }; + const empty_rt: RichText = .{ + .text = &.{.{ .text = " No process data available", .style = theme.dim_style }}, + .softwrap = false, + .overflow = .clip, + }; + + var children: std.ArrayList(vxfw.Widget) = .empty; + try children.append(ctx.arena, header_rt.widget()); + + if (row_count == 0) { + try children.append(ctx.arena, try compose.sized( + ctx.arena, + try compose.pad(ctx.arena, empty_rt.widget(), .{ .top = 1 }), + .{ .width = w, .height = 2 }, + )); + } else { + const scroll_h: u16 = @min(@as(u16, @intCast(row_count)), max_visible_rows); + try children.append(ctx.arena, try compose.sized(ctx.arena, self.scroll_bars.widget(), .{ .width = w, .height = scroll_h })); + } + + const content: ColumnLayout = .{ .children = children.items }; + const tb: TitledBorder = .{ + .child = ui.widget(&content), + .title = "Processes", + }; + return tb.draw(ui.fixedWidth(ctx, w)); +} + +/// Builds only data rows (no header) for the scroll view. +fn buildRow(ptr: *const anyopaque, idx: usize, _: usize) ?vxfw.Widget { + const self: *ProcessTable = @ptrCast(@alignCast(@constCast(ptr))); + if (idx >= self.merged.items.len) { + return null; + } + return .{ .userdata = @ptrCast(&self.merged.items[idx]), .drawFn = drawProcessRow }; +} + +// Column format strings — shared between header (comptime) and row (runtime). +const col_fmt = struct { + const pid = " {s: >7} "; + const user = "{s: <10} "; + const dev = "{s: <5} "; + const util = "{s: >6} "; + const mem = "{s: >8} "; + const cpu = "{s: >6} "; + const host_mem = "{s: >8} "; +}; + +const header_segs = blk: { + const s = theme.label_style; + break :blk [_]Segment{ + .{ .style = s, .text = std.fmt.comptimePrint(col_fmt.pid, .{"PID"}) }, + .{ .style = s, .text = std.fmt.comptimePrint(col_fmt.user, .{"USER"}) }, + .{ .style = s, .text = std.fmt.comptimePrint(col_fmt.dev, .{"DEV"}) }, + .{ .style = s, .text = std.fmt.comptimePrint(col_fmt.util, .{"UTIL"}) }, + .{ .style = s, .text = std.fmt.comptimePrint(col_fmt.mem, .{"DEV MEM"}) }, + .{ .style = s, .text = std.fmt.comptimePrint(col_fmt.cpu, .{"CPU"}) }, + .{ .style = s, .text = std.fmt.comptimePrint(col_fmt.host_mem, .{"HOST MEM"}) }, + .{ .style = s, .text = "Command" }, + }; +}; + +/// Format a value into a column: col(arena, col_fmt.pid, "{d}", .{pid}) +fn col(arena: std.mem.Allocator, comptime column: []const u8, comptime fmt: []const u8, args: anytype) std.mem.Allocator.Error![]const u8 { + return std.fmt.allocPrint(arena, column, .{try std.fmt.allocPrint(arena, fmt, args)}); +} + +fn drawProcessRow(ptr: *anyopaque, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { + const info: *const ProcessInfo = @ptrCast(@alignCast(ptr)); + const val: vaxis.Style = .{ .fg = theme.text_primary }; + const dim: vaxis.Style = .{ .fg = theme.dim }; + + const segs = try ctx.arena.alloc(Segment, 8); + segs[0] = .{ .style = val, .text = try col(ctx.arena, col_fmt.pid, "{d}", .{info.pid}) }; + segs[1] = .{ .style = val, .text = try col(ctx.arena, col_fmt.user, "{s}", .{info.username[0..@min(info.username.len, 10)]}) }; + segs[2] = .{ .style = val, .text = try col(ctx.arena, col_fmt.dev, "{d}", .{info.device_idx}) }; + segs[3] = if (info.dev_util_percent) |util| + .{ .style = val, .text = try col(ctx.arena, col_fmt.util, "{d}%", .{util}) } + else + .{ .style = dim, .text = try col(ctx.arena, col_fmt.util, "-", .{}) }; + segs[4] = if (info.dev_mem_kib) |mem| + .{ .style = val, .text = try col(ctx.arena, col_fmt.mem, "{s}", .{try fmtMem(ctx.arena, mem)}) } + else + .{ .style = dim, .text = try col(ctx.arena, col_fmt.mem, "-", .{}) }; + segs[5] = .{ .style = val, .text = try col(ctx.arena, col_fmt.cpu, "{d}.{d}%", .{ info.cpu_percent / 10, info.cpu_percent % 10 }) }; + segs[6] = .{ .style = val, .text = try col(ctx.arena, col_fmt.host_mem, "{s}", .{try fmtMem(ctx.arena, info.rss_kib)}) }; + segs[7] = .{ .style = val, .text = info.comm }; + + const rich: RichText = .{ .text = segs, .softwrap = false, .overflow = .clip }; + return rich.draw(ctx.withConstraints( + .{ .width = ctx.min.width, .height = 1 }, + .{ .width = ctx.max.width, .height = 1 }, + )); +} + +fn fmtMem(arena: std.mem.Allocator, kib: u64) std.mem.Allocator.Error![]const u8 { + if (kib >= 1024 * 1024) { + const whole = kib / (1024 * 1024); + const frac = (kib % (1024 * 1024)) * 10 / (1024 * 1024); + return std.fmt.allocPrint(arena, "{d}.{d}G", .{ whole, frac }); + } else if (kib >= 1024) { + const whole = kib / 1024; + const frac = (kib % 1024) * 10 / 1024; + return std.fmt.allocPrint(arena, "{d}.{d}M", .{ whole, frac }); + } else { + return std.fmt.allocPrint(arena, "{d}K", .{kib}); + } +} diff --git a/bin/zml-smi/tui/widgets/status_line.zig b/bin/zml-smi/tui/widgets/status_line.zig new file mode 100644 index 00000000..fb32541e --- /dev/null +++ b/bin/zml-smi/tui/widgets/status_line.zig @@ -0,0 +1,77 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; +const theme = @import("../theme.zig"); +const ui = @import("../lib/ui.zig"); + +const StatusLine = @This(); + +viewing_device: ?u16, +use_braille: bool, + +const Binding = struct { + key: []const u8, + desc: []const u8, +}; + +pub fn draw(self: *const StatusLine, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { + const w = ctx.max.width orelse 80; + + const key_style: vaxis.Style = .{ .bold = true, .fg = .{ .rgb = .{ 30, 30, 40 } }, .bg = theme.accent }; + const desc_style: vaxis.Style = .{ .fg = theme.text_secondary }; + const sep_style: vaxis.Style = .{ .fg = theme.dim }; + const bg_style: vaxis.Style = .{ .bg = .{ .rgb = .{ 30, 30, 45 } } }; + + const chart_label = if (self.use_braille) "gauges" else "charts"; + + var bindings_buf: [8]Binding = undefined; + var n: usize = 0; + + if (self.viewing_device != null) { + bindings_buf[n] = .{ .key = "Esc", .desc = "back" }; + n += 1; + } else { + bindings_buf[n] = .{ .key = "v", .desc = chart_label }; + n += 1; + bindings_buf[n] = .{ .key = "click", .desc = "device details" }; + n += 1; + } + bindings_buf[n] = .{ .key = "q", .desc = "quit" }; + n += 1; + + const bindings = bindings_buf[0..n]; + + // Build segments: " key desc | key desc | ... " + var segments: std.ArrayList(vaxis.Cell.Segment) = .empty; + + // Leading space + try segments.append(ctx.arena, .{ .text = " ", .style = bg_style }); + + for (bindings, 0..) |b, i| { + try segments.append(ctx.arena, .{ .text = " ", .style = bg_style }); + try segments.append(ctx.arena, .{ .text = b.key, .style = key_style }); + try segments.append(ctx.arena, .{ .text = " ", .style = bg_style }); + try segments.append(ctx.arena, .{ .text = b.desc, .style = desc_style }); + if (i + 1 < bindings.len) { + try segments.append(ctx.arena, .{ .text = " \u{2502}", .style = sep_style }); + } + } + + const rich: vxfw.RichText = .{ + .text = segments.items, + .softwrap = false, + .overflow = .clip, + }; + const surf = try rich.draw(ui.fixedSize(ctx, w, 1)); + + // Fill background across full width + if (surf.buffer.len > 0) { + for (surf.buffer) |*cell| { + if (cell.style.bg == .default) { + cell.style.bg = bg_style.bg; + } + } + } + + return surf; +} diff --git a/bin/zml-smi/tui/widgets/titled_border.zig b/bin/zml-smi/tui/widgets/titled_border.zig new file mode 100644 index 00000000..d9c16171 --- /dev/null +++ b/bin/zml-smi/tui/widgets/titled_border.zig @@ -0,0 +1,112 @@ +const std = @import("std"); +const vaxis = @import("vaxis"); +const vxfw = vaxis.vxfw; +const theme = @import("../theme.zig"); +const ui = @import("../lib/ui.zig"); +const Image = @import("../lib/image.zig"); + +const Text = vxfw.Text; + +const TitledBorder = @This(); + +child: vxfw.Widget, +title: []const u8 = "", +title_style: vaxis.Cell.Style = theme.title_style, +title_image: ?vaxis.Image = null, +cell_size: vxfw.Size = .{}, +value_label: ?[]const u8 = null, +value_label_style: vaxis.Cell.Style = theme.value_style, +border_style: vaxis.Cell.Style = theme.border_style, +padding: vxfw.Padding.PadValues = .{ .left = 1, .right = 1, .top = 1, .bottom = 1 }, + +pub fn draw(self: *const TitledBorder, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { + const pad: vxfw.Padding = .{ + .child = self.child, + .padding = self.padding, + }; + const border: vxfw.Border = .{ + .child = pad.widget(), + .style = self.border_style, + }; + var surf = try border.draw(ctx); + + // Count overlay children needed + var overlay_count: usize = 0; + var title_col: u16 = 2; + var has_image = false; + + if (self.title_image) |img| { + const icon = ui.imageCellSize(img, 1, self.cell_size); + has_image = true; + title_col += icon.cols; + overlay_count += 1; + } + if (self.title.len > 0) { + overlay_count += 1; + } + if (self.value_label != null) { + overlay_count += 1; + } + + if (overlay_count == 0) { + return surf; + } + + const new_children = try ctx.arena.alloc(vxfw.SubSurface, surf.children.len + overlay_count); + @memcpy(new_children[0..surf.children.len], surf.children); + var idx = surf.children.len; + + // Image icon overlay + if (has_image) { + const img = self.title_image.?; + const img_widget: Image = .{ .image = img, .rows = 1 }; + const img_surf = try img_widget.draw(ctx); + new_children[idx] = .{ + .origin = .{ .row = 0, .col = 2 }, + .surface = img_surf, + .z_index = 1, + }; + idx += 1; + } + + // Title text overlay (top-left) + if (self.title.len > 0) { + const title_text = try std.fmt.allocPrint(ctx.arena, " {s} ", .{self.title}); + const title_widget: Text = .{ + .text = title_text, + .style = self.title_style, + .softwrap = false, + .overflow = .clip, + }; + const max_title_w = surf.size.width -| (title_col + 1); + const title_surf = try title_widget.draw(ui.maxSize(ctx, max_title_w, 1)); + new_children[idx] = .{ + .origin = .{ .row = 0, .col = @intCast(title_col) }, + .surface = title_surf, + .z_index = 1, + }; + idx += 1; + } + + // Value label overlay (top-right) + if (self.value_label) |vlabel| { + const label_text = try std.fmt.allocPrint(ctx.arena, " {s} ", .{vlabel}); + const label_widget: Text = .{ + .text = label_text, + .style = self.value_label_style, + .softwrap = false, + .overflow = .clip, + }; + const right = surf.size.width -| 1; + const lw: u16 = @intCast(@min(ctx.stringWidth(label_text), right -| 2)); + const label_surf = try label_widget.draw(ui.maxSize(ctx, lw, 1)); + new_children[idx] = .{ + .origin = .{ .row = 0, .col = @intCast(right -| lw) }, + .surface = label_surf, + .z_index = 1, + }; + } + + surf.children = new_children; + return surf; +} diff --git a/bin/zml-smi/utils/BUILD.bazel b/bin/zml-smi/utils/BUILD.bazel new file mode 100644 index 00000000..a154556c --- /dev/null +++ b/bin/zml-smi/utils/BUILD.bazel @@ -0,0 +1,30 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "double_buffer", + import_name = "zml-smi/double_buffer", + main = "double_buffer.zig", + visibility = ["//visibility:public"], +) + +zig_library( + name = "dynlib", + import_name = "zml-smi/dynlib", + main = "dynlib.zig", + visibility = ["//visibility:public"], +) + +zig_library( + name = "sandbox", + import_name = "zml-smi/sandbox", + main = "sandbox.zig", + visibility = ["//visibility:public"], + deps = ["@rules_zig//zig/runfiles"], +) + +zig_library( + name = "sysfs", + import_name = "zml-smi/sysfs", + main = "sysfs.zig", + visibility = ["//visibility:public"], +) diff --git a/bin/zml-smi/utils/double_buffer.zig b/bin/zml-smi/utils/double_buffer.zig new file mode 100644 index 00000000..add3037b --- /dev/null +++ b/bin/zml-smi/utils/double_buffer.zig @@ -0,0 +1,32 @@ +const std = @import("std"); + +pub fn DoubleBuffer(comptime T: type) type { + return struct { + const Self = @This(); + pub const Value = T; + + values: [2]T, + current: std.atomic.Value(u8) = std.atomic.Value(u8).init(0), + + pub fn front(self: *const Self) *const T { + return &self.values[self.current.load(.acquire)]; + } + + pub fn back(self: *Self) *T { + return &self.values[1 - self.current.load(.acquire)]; + } + + pub fn swap(self: *Self) void { + self.current.store(1 - self.current.load(.acquire), .release); + } + + pub fn jsonStringify(self: *const Self, jw: *std.json.Stringify) !void { + try jw.write(self.front().*); + } + + pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) !Self { + const val = try std.json.innerParseFromValue(T, allocator, source, options); + return .{ .values = .{ val, val } }; + } + }; +} diff --git a/bin/zml-smi/utils/dynlib.zig b/bin/zml-smi/utils/dynlib.zig new file mode 100644 index 00000000..4a478107 --- /dev/null +++ b/bin/zml-smi/utils/dynlib.zig @@ -0,0 +1,10 @@ +const std = @import("std"); + +pub fn lookupStruct(dynlib: *std.DynLib, comptime VTable: type) !VTable { + var result: VTable = undefined; + inline for (comptime std.meta.fieldNames(VTable), comptime std.meta.fieldTypes(VTable)) |field_name, Field| { + @field(result, field_name) = dynlib.lookup(Field, field_name) orelse return error.SymbolResolutionFailed; + } + + return result; +} diff --git a/bin/zml-smi/utils/sandbox.zig b/bin/zml-smi/utils/sandbox.zig new file mode 100644 index 00000000..d4e268bd --- /dev/null +++ b/bin/zml-smi/utils/sandbox.zig @@ -0,0 +1,44 @@ +const std = @import("std"); +const builtin = @import("builtin"); +const bazel_runfiles = @import("runfiles"); +const bazel_builtin = @import("bazel_builtin"); + +var runfiles_global: bazel_runfiles.Runfiles = undefined; + +const init_array_section = switch (builtin.object_format) { + .macho => "__DATA,__mod_init_func", + .elf => ".init_array", + else => "", +}; + +export const _ align(@alignOf(usize)) linksection(init_array_section) = &struct { + fn call(argc: c_int, argv: [*c][*:0]u8, envp: [*:null]?[*:0]u8) callconv(.c) void { + _ = argc; + _ = argv; + _ = envp; + var threaded: std.Io.Threaded = .init_single_threaded; + const io = threaded.io(); + + var exe_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const exe_size = std.process.executablePath(io, &exe_path_buf) catch + std.debug.panic("Unable to get executable path", .{}); + + runfiles_global = bazel_runfiles.Runfiles.create(.{ + .allocator = std.heap.c_allocator, + .io = io, + .argv0 = exe_path_buf[0..exe_size], + .directory = if (std.c.getenv("RUNFILES_DIR")) |directory| std.mem.span(directory) else null, + .manifest = if (std.c.getenv("RUNFILES_MANIFEST_FILE")) |manifest| std.mem.span(manifest) else null, + }) catch { + std.debug.panic("Unable to find runfiles", .{}); + } orelse { + std.debug.panic("Unable to initialize runfiles", .{}); + }; + } +}.call; + +pub fn path(buf: *[std.Io.Dir.max_path_bytes]u8) ?[]const u8 { + const with_repo = runfiles_global.withSourceRepo(bazel_builtin.current_repository); + + return with_repo.rlocation("zml/bin/zml-smi/sandbox", buf) catch null; +} diff --git a/bin/zml-smi/utils/sysfs.zig b/bin/zml-smi/utils/sysfs.zig new file mode 100644 index 00000000..636883b2 --- /dev/null +++ b/bin/zml-smi/utils/sysfs.zig @@ -0,0 +1,70 @@ +const std = @import("std"); + +pub fn readInt(allocator: std.mem.Allocator, io: std.Io, path: []const u8) !u64 { + const data = try readFirstLine(allocator, io, path); + return std.fmt.parseInt(u64, data, 10); +} + +pub fn readString(allocator: std.mem.Allocator, io: std.Io, path: []const u8) ![]const u8 { + return readFirstLine(allocator, io, path); +} + +pub fn readFieldInt(allocator: std.mem.Allocator, io: std.Io, path: []const u8, comptime key: []const u8) !u64 { + return nthTokenInt(try findField(allocator, io, path, key), 0); +} + +pub fn nthTokenInt(data: []const u8, n: usize) u64 { + var i: usize = 0; + + var iter = std.mem.tokenizeAny(u8, data, " \t"); + while (iter.next()) |tok| : (i += 1) { + if (i == n) { + return std.fmt.parseInt(u64, tok, 10) catch 0; + } + } + + return 0; +} + +pub fn readFieldString(allocator: std.mem.Allocator, io: std.Io, path: []const u8, comptime key: []const u8) ![]const u8 { + const value = try findField(allocator, io, path, key); + return std.mem.trimEnd(u8, value, &std.ascii.whitespace); +} + +fn findField(allocator: std.mem.Allocator, io: std.Io, path: []const u8, comptime key: []const u8) ![]const u8 { + var file = try std.Io.Dir.openFile(.cwd(), io, path, .{ .mode = .read_only }); + defer file.close(io); + + var read_buf: [4096]u8 = undefined; + var reader = file.reader(io, &read_buf); + + var line_writer: std.Io.Writer.Allocating = .init(allocator); + errdefer line_writer.deinit(); + + while (true) { + line_writer.clearRetainingCapacity(); + _ = reader.interface.streamDelimiter(&line_writer.writer, '\n') catch return error.NotFound; + reader.interface.toss(1); + + const line = line_writer.written(); + if (std.mem.startsWith(u8, line, key)) { + const owned = line_writer.toOwnedSlice() catch return error.NotFound; + return std.mem.trimStart(u8, owned[key.len..], " :\t"); + } + } +} + +pub fn readFirstLine(allocator: std.mem.Allocator, io: std.Io, path: []const u8) ![]u8 { + var file = try std.Io.Dir.openFile(.cwd(), io, path, .{ .mode = .read_only }); + defer file.close(io); + + var read_buf: [4096]u8 = undefined; + var reader = file.reader(io, &read_buf); + var writer: std.Io.Writer.Allocating = .init(allocator); + errdefer writer.deinit(); + _ = reader.interface.streamDelimiter(&writer.writer, '\n') catch |err| switch (err) { + error.EndOfStream => 0, + else => |e| return e, + }; + return writer.toOwnedSlice(); +} diff --git a/build.zig b/build.zig deleted file mode 120000 index 0468ff14..00000000 --- a/build.zig +++ /dev/null @@ -1 +0,0 @@ -examples/build.zig \ No newline at end of file diff --git a/build.zig b/build.zig new file mode 100644 index 00000000..e69de29b diff --git a/devenv.lock b/devenv.lock new file mode 100644 index 00000000..6c3be332 --- /dev/null +++ b/devenv.lock @@ -0,0 +1,65 @@ +{ + "nodes": { + "devenv": { + "locked": { + "dir": "src/modules", + "lastModified": 1793199252, + "narHash": "sha256-LRgdAdMHL3uIX+kix/oyMzG+gMxzSYOzLrrRaoZthog=", + "owner": "cachix", + "repo": "devenv", + "rev": "0d6c556e5f862c14a75a543e2ee4fdb7acc005ad", + "type": "github" + }, + "original": { + "dir": "src/modules", + "owner": "cachix", + "repo": "devenv", + "type": "github" + } + }, + "nixpkgs": { + "inputs": { + "nixpkgs-src": "nixpkgs-src" + }, + "locked": { + "lastModified": 1793043970, + "narHash": "sha256-m14odYTdkBUEkJOiEYs2zjGjn8fcjvdCJsm45+k084U=", + "owner": "cachix", + "repo": "devenv-nixpkgs", + "rev": "d97234e58d4e90e3bf77c082fc3b546e51200938", + "type": "github" + }, + "original": { + "owner": "cachix", + "ref": "rolling", + "repo": "devenv-nixpkgs", + "type": "github" + } + }, + "nixpkgs-src": { + "flake": false, + "locked": { + "lastModified": 1792812153, + "narHash": "sha256-T2+XBfNbf9pecLUWS4+RtuVNZ3W8K5taG5QdV71mWlk=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "674a2028661e550a97400c1cc992b9cca9a668a1", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "devenv": "devenv", + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} \ No newline at end of file diff --git a/devenv.nix b/devenv.nix new file mode 100644 index 00000000..cafd0f90 --- /dev/null +++ b/devenv.nix @@ -0,0 +1,20 @@ +{ + pkgs, + config, + ... +}: { + packages = with pkgs; [ + bazelisk + python3 + uv + zig + ]; + env.ZLS_CMD = "${config.devenv.root}/tools/zls.sh"; + scripts.bazel.exec = ''bazelisk "$@"''; + + enterShell = pkgs.lib.optionalString pkgs.stdenv.isDarwin '' + # Use the "real" XCode rather than Nixpkg's one. This does assume you have it installed. + export DEVELOPER_DIR="/Applications/Xcode.app/Contents/Developer" + export SDKROOT="/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk"; + ''; +} diff --git a/devenv.yaml b/devenv.yaml new file mode 100644 index 00000000..116a2adb --- /dev/null +++ b/devenv.yaml @@ -0,0 +1,15 @@ +# yaml-language-server: $schema=https://devenv.sh/devenv.schema.json +inputs: + nixpkgs: + url: github:cachix/devenv-nixpkgs/rolling + +# If you're using non-OSS software, you can set allowUnfree to true. +# allowUnfree: true + +# If you're willing to use a package that's vulnerable +# permittedInsecurePackages: +# - "openssl-1.1.1w" + +# If you have more than one devenv you can merge them +#imports: +# - ./backend diff --git a/docs/README.md b/docs/README.md index a24259ac..03369120 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,7 +1,7 @@ ### Greetings, Programs! -If you want run our example models, please head over to +If you want to run our example models, please head over to [Getting Started](./tutorials/getting_started.md). Ready to write some code? Try starting with [your first model in ZML](./tutorials/write_first_model.md), or familiarize yourself with the high-level [ZML concepts](./learn/concepts.md) first. @@ -17,9 +17,9 @@ Ready to write some code? Try starting with [your first model in ZML](./tutorial ## How to ? - [HuggingFace Authentication](./howtos/huggingface_access_token.md) - [Port Pytorch models to ZML](./howtos/howto_torch2zml.md) -- [Add Weights Files to your projects](./howtos/add_weights.md) - [Cross Compile and Deploy Models on a Server](./howtos/deploy_on_server.md) - [Dockerize Models](./howtos/dockerize_models.md) +- [Profile ZML Programs](./howtos/profiling.md) ## Learn more... @@ -28,4 +28,3 @@ Ready to write some code? Try starting with [your first model in ZML](./tutorial ## Contribute - [Style Guide](./misc/style_guide.md) - diff --git a/docs/howtos/add_weights.md b/docs/howtos/add_weights.md deleted file mode 100644 index 825ab652..00000000 --- a/docs/howtos/add_weights.md +++ /dev/null @@ -1,77 +0,0 @@ - -# Adding Weights Files - -Our [first model](../tutorials/write_first_model.md) did not need any weights files. -We just created weights and bias at runtime. - -But real-world models typically need weights files, and maybe some other -supporting files. - -We recommend, for easy deployments, you upload those files. In many instances, -you will use a site like [🤗 Hugging Face](https://huggingface.co). - -We also recommend to add a `weights.bzl` file to your project root directory, so -you don't "pollute" your build file with long URLs and SHAs: - -```python -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file") - -def _weights_impl(mctx): - http_file( - name = "com_github_zml_cdn_mnist", - downloaded_file_path = "mnist.pt", - sha256 = "d8a25252e28915e147720c19223721f0f53e3317493727ca754a2dd672450ba9", - url = "https://github.com/ggerganov/ggml/raw/18703ad600cc68dbdb04d57434c876989a841d12/examples/mnist/models/mnist/mnist_model.state_dict", - ) - - http_file( - name = "com_github_zml_cdn_mnist_data", - downloaded_file_path = "mnist.ylc", - sha256 = "0fa7898d509279e482958e8ce81c8e77db3f2f8254e26661ceb7762c4d494ce7", - url = "https://github.com/ggerganov/ggml/raw/18703ad600cc68dbdb04d57434c876989a841d12/examples/mnist/models/mnist/t10k-images.idx3-ubyte", - ) - - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -weights = module_extension( - implementation = _weights_impl, -) -``` - -The above `weights.bzl` shows how we load files for MNIST: - -- `mnist.pt` (model weights) -- `mnist.ylc` (dataset for picking sample images) - -Then, in your `BUILD.bazel`, you can refer to the files you defined above, in -the following way: - -```python -zig_cc_binary( - name = "mnist", - args = [ - "$(location @com_github_zml_cdn_mnist//file)", - "$(location @com_github_zml_cdn_mnist_data//file)", - ], - data = [ - "@com_github_zml_cdn_mnist//file", - "@com_github_zml_cdn_mnist_data//file", - ], - main = "mnist.zig", - deps = [ - "//async", - "//zml", - ], -) -``` - -See how: - -- we use `data = [ ... ]` to reference the files in `weights.bzl` -- we use `args = [ ... ]` to pass the files as command-line arguments to the - MNIST executable at runtime, automatically. - diff --git a/docs/howtos/deploy_on_server.md b/docs/howtos/deploy_on_server.md index 36b27671..a8a6452e 100644 --- a/docs/howtos/deploy_on_server.md +++ b/docs/howtos/deploy_on_server.md @@ -14,17 +14,21 @@ information on how to compile a model. You can compile models for accelerator runtimes by appending one or more of the following arguments to the command line when compiling / running a model: -- NVIDIA CUDA: `--@zml//runtimes:cuda=true` -- AMD RoCM: `--@zml//runtimes:rocm=true` -- Google TPU: `--@zml//runtimes:tpu=true` -- **AVOID CPU:** `--@zml//runtimes:cpu=false` +- NVIDIA CUDA: `--@zml//platforms:cuda=true` +- AMD RoCM: `--@zml//platforms:rocm=true` +- Google TPU: `--@zml//platforms:tpu=true` +- AWS Trainium/Inferentia 2: `--@zml//platforms:neuron=true` +- **AVOID CPU:** `--@zml//platforms:cpu=false` -So, to run the OpenLLama model from above **on your development machine** +So, to run the Llama model from above **on your development machine** housing an NVIDIA GPU, run the following: ``` -cd examples -bazel run -c opt //llama:OpenLLaMA-3B --@zml//runtimes:cuda=true +bazel run --config=release //examples/llm \ + --@zml//platforms:cuda=true \ + -- \ + --model=hf://meta-llama/Llama-3.2-1B-Instruct \ + --prompt="What is the capital of France?" ``` @@ -37,14 +41,13 @@ architectures: - Linux ARM64: `--platforms=@zml//platforms:linux_arm64` - MacOS ARM64: `--platforms=@zml//platforms:macos_arm64` -As an example, here is how you build above OpenLLama for CUDA on Linux X86_64: +As an example, here is how you build above Llama for CUDA on Linux X86_64: ``` -cd examples -bazel build -c opt //llama:OpenLLaMA-3B \ - --@zml//runtimes:cuda=true \ - --@zml//runtimes:cpu=false \ - --platforms=@zml//platforms:linux_amd64 +bazel build --config=release //examples/llm \ + --@zml//platforms:cuda=true \ + --@zml//platforms:cpu=false \ + --platforms=@zml//platforms:linux_amd64 ``` ### Creating the TAR @@ -58,7 +61,8 @@ If not present already, add an "archive" target to the model's `BUILD.bazel`, like this: ```python -load("@aspect_bazel_lib//lib:tar.bzl", "mtree_spec", "tar") +load("@tar.bzl//tar:mtree.bzl", "mtree_spec") +load("@tar.bzl//tar:tar.bzl", "tar") # Manifest, required for building the tar archive mtree_spec( @@ -82,16 +86,15 @@ tar( ... and then build the TAR archive: ``` -# cd examples -bazel build -c opt //mnist:archive \ - --@zml//runtimes:cuda=true \ - --@zml//runtimes:cpu=false \ +bazel build --config=release //examples/mnist:archive \ + --@zml//platforms:cuda=true \ + --@zml//platforms:cpu=false \ --platforms=@zml//platforms:linux_amd64 ``` -Note the `//mnist:archive` notation. +Note the `//examples/mnist:archive` notation. -The resulting tar file will be in `bazel-bin/mnist/archive.tar.zst`. +The resulting tar file will be in `bazel-bin/examples/mnist/archive.tar.zst`. ### Run it on the server @@ -100,37 +103,35 @@ and run it: ```bash # on your machine -scp bazel-bin/mnist/archive.tar.zst destination-server: +scp bazel-bin/examples/mnist/archive.tar.zst destination-server: ssh destination-server # to enter the server # ... on the server tar xvf archive.tar.zst -./mnist \ - 'mnist.runfiles/_main~_repo_rules~com_github_ggerganov_ggml_mnist/file/mnist.pt' \ - 'mnist.runfiles/_main~_repo_rules~com_github_ggerganov_ggml_mnist_data/file/mnist.ylc' +./examples/mnist/mnist \ + $PWD/examples/mnist/mnist.runfiles/+non_module_deps+mnist/mnist.safetensors \ + $PWD/examples/mnist/mnist.runfiles/+non_module_deps+mnist/t10k-images.idx3-ubyte ``` The easiest way to figure out the commandline arguments of an example model is to consult the model's `BUILD.bazel` and check out its `args` section. It will -reference e.g. weights files that are defined either in the same `BUILD.bazel` -file or in a `weights.bzl` file. +reference e.g. weights files that are defined in the same `BUILD.bazel`. You can also consult the console output when running your model locally: ```bash -bazel run //mnist +bazel run //examples/mnist -INFO: Analyzed target //mnist:mnist (0 packages loaded, 0 targets configured). +INFO: Analyzed target //examples/mnist:mnist (0 packages loaded, 0 targets configured). INFO: Found 1 target... -Target //mnist:mnist up-to-date: - bazel-bin/mnist/mnist -INFO: Elapsed time: 0.302s, Critical Path: 0.00s -INFO: 3 processes: 3 internal. -INFO: Build completed successfully, 3 total actions -INFO: Running command line: bazel-bin/mnist/mnist ../_main~_repo_rules~com_github_ggerganov_ggml_mnist/file/mnist.pt ../_main~_repo_rules~com_github_ggerganov_ggml_mnist_data/file/mnist.ylc -# ... +Target //examples/mnist:mnist up-to-date: + bazel-bin/examples/mnist/mnist +INFO: Elapsed time: 0.114s, Critical Path: 0.00s +INFO: 1 process: 1 internal. +INFO: Build completed successfully, 1 total action +INFO: Running command line: bazel-bin/examples/mnist/mnist ../+non_module_deps+mnist/mnist.safetensors ../+non_module_deps+mnist/t10k-images.idx3-ubyte. +# .. ``` You see the command line right up there. On the server, you just need to replace -`../` with the 'runfiles' directory of your TAR. - +`../` with the absolute path to the file. diff --git a/docs/howtos/dockerize_models.md b/docs/howtos/dockerize_models.md index 318cccd2..7414b985 100644 --- a/docs/howtos/dockerize_models.md +++ b/docs/howtos/dockerize_models.md @@ -7,11 +7,11 @@ just have to append a few lines to your model's `BUILD.bazel`. Here is how it's done. **Note:** This walkthrough will work with your installed container runtime, no -matter if it's **Docker or e.g. Podman.** Also, we'll create images in the +matter if it's **Docker or e.g. Podman.** Also, we'll create images in the [OCI](https://github.com/opencontainers/image-spec) open image format. Let's try containerizing our [first model](../tutorials/write_first_model.md), as it -doesn't need any additional weights files. We'll see [down below](#adding-weights-and-data) +doesn't need any additional weights files. We'll see [down below](#adding-weights-and-data) how to add those. We'll also see how to add GPU/TPU support for our container there. @@ -38,15 +38,16 @@ We need to add a few "imports" at the beginning of our `BUILD.bazel` so we can use their rules to define our 5 additional targets: ```python -load("@aspect_bazel_lib//lib:tar.bzl", "mtree_spec", "tar") -load("@aspect_bazel_lib//lib:transitions.bzl", "platform_transition_filegroup") +load("@bazel_lib//lib:transitions.bzl", "platform_transition_filegroup") load("@rules_oci//oci:defs.bzl", "oci_image", "oci_load", "oci_push") +load("@rules_zig//zig:defs.bzl", "zig_binary") +load("@tar.bzl//tar:mtree.bzl", "mtree_spec") +load("@tar.bzl//tar:tar.bzl", "tar") -zig_cc_binary( +zig_binary( name = "simple_layer", main = "main.zig", deps = [ - "@zml//async", "@zml//zml", ], ) @@ -56,7 +57,7 @@ zig_cc_binary( ### 1. The Manifest To get started, let's make bazel generate a manifest that will be used when -creating the TAR archive. +creating the TAR archive. ```python # Manifest created from the simple_layer binary and friends @@ -117,7 +118,7 @@ See how we use string interpolation to fill in the folder name for the container's entrypoint? -Next, we use a transition rule to force the container to be built for +Next, we use a transition rule to force the container to be built for Linux X86_64: ```python @@ -132,14 +133,13 @@ platform_transition_filegroup( And that's almost it! You can already build the image: ``` -# cd examples -bazel build -c opt //simple_layer:image +bazel build --config=release //examples/simple_layer:image -INFO: Analyzed target //simple_layer:image (1 packages loaded, 8 targets configured). +INFO: Analyzed target //examples/simple_layer:image (0 packages loaded, 0 targets configured). INFO: Found 1 target... -Target //simple_layer:image up-to-date: - bazel-out/k8-dbg-ST-f832ad0148ae/bin/simple_layer/image_ -INFO: Elapsed time: 0.279s, Critical Path: 0.00s +Target //examples/simple_layer:image up-to-date: + bazel-out/linux_amd64-opt/bin/examples/simple_layer/image_ +INFO: Elapsed time: 0.285s, Critical Path: 0.00s INFO: 1 process: 1 internal. INFO: Build completed successfully, 1 total action ``` @@ -149,10 +149,10 @@ INFO: Build completed successfully, 1 total action ### 4. The Load -While inspecting the image is surely interesting, we usually want to load the +While inspecting the image is surely interesting, we usually want to load the image so we can run it. -There is a bazel rule for that: `oci_load`. When we append the following lines +There is a bazel rule for that: `oci_load`. When we append the following lines to `BUILD.bazel`: ```python @@ -168,7 +168,7 @@ oci_load( ... then we can load the image and run it with the following commands: ``` -bazel run -c opt //simple_layer:load +bazel run --config=release //examples/simple_layer:load docker run --rm distroless/simple_layer:latest ``` @@ -193,7 +193,7 @@ This will push the `simple_layer` image with the tag `latest` (you can add more) to the docker registry: ``` -bazel run -c opt //simple_layer:push +bazel run --config=release //examples/simple_layer:push ``` When dealing with maybe a public and a private container registry - or if you @@ -201,7 +201,7 @@ just want to try it out **right now**, you can always override the repository on the command line: ``` -bazel run -c opt //simple_layer:push -- --repository my.server.com/org/image +bazel run --config=release //examples/simple_layer:push -- --repository my.server.com/org/image ``` @@ -215,22 +215,23 @@ We'll use the [MNIST example](https://github.com/zml/zml/tree/master/examples/mnist) to illustrate how to build Docker images that also contain data files. -You can `bazel run -c opt //mnist:push -- --repository -index.docker.io/my_org/zml_mnist` in the `./examples` folder if you want to try -it out. +You can `bazel run --config=release //examples/mnist:push -- --repository +index.docker.io/my_org/zml_mnist` if you want to try +it out. **Note: Please add one more of the following parameters to specify all the platforms your containerized model should support.** -- NVIDIA CUDA: `--@zml//runtimes:cuda=true` -- AMD RoCM: `--@zml//runtimes:rocm=true` -- Google TPU: `--@zml//runtimes:tpu=true` -- **AVOID CPU:** `--@zml//runtimes:cpu=false` +- NVIDIA CUDA: `--@zml//platforms:cuda=true` +- AMD RoCM: `--@zml//platforms:rocm=true` +- Google TPU: `--@zml//platforms:tpu=true` +- AWS Trainium/Inferentia 2: `--@zml//platforms:neuron=true` +- **AVOID CPU:** `--@zml//platforms:cpu=false` **Example:** ``` -bazel run //mnist:push -c opt --@zml//runtimes:cuda=true -- --repository index.docker.io/my_org/zml_mnist +bazel run //examples/mnist:push --config=release --@zml//platforms:cuda=true -- --repository index.docker.io/my_org/zml_mnist ``` @@ -243,26 +244,27 @@ same. Let's start with creating the manifest and archive: ```python -load("@aspect_bazel_lib//lib:expand_template.bzl", "expand_template") -load("@aspect_bazel_lib//lib:tar.bzl", "mtree_spec", "tar") -load("@aspect_bazel_lib//lib:transitions.bzl", "platform_transition_filegroup") +load("@bazel_lib//lib:transitions.bzl", "platform_transition_filegroup") load("@rules_oci//oci:defs.bzl", "oci_image", "oci_load", "oci_push") -load("@zml//bazel:zig.bzl", "zig_cc_binary") +load("@rules_zig//zig:defs.bzl", "zig_binary") +load("@tar.bzl//tar:mtree.bzl", "mtree_spec") +load("@tar.bzl//tar:tar.bzl", "tar") # The executable -zig_cc_binary( +zig_binary( name = "mnist", args = [ - "$(location @com_github_ggerganov_ggml_mnist//file)", - "$(location @com_github_ggerganov_ggml_mnist_data//file)", + "$(location @mnist//:mnist.safetensors)", + "$(location @mnist//:t10k-images.idx3-ubyte)", ], data = [ - "@com_github_ggerganov_ggml_mnist//file", - "@com_github_ggerganov_ggml_mnist_data//file", + "@mnist//:mnist.safetensors", + "@mnist//:t10k-images.idx3-ubyte", ], main = "mnist.zig", + visibility = ["//visibility:public"], deps = [ - "@zml//async", + "@zml//bazel", "@zml//zml", ], ) @@ -295,22 +297,24 @@ simple string interpolation will not be enough. For this reason, we use the `expand_template` rule, like this: ```python +load("@bazel_lib//lib:expand_template.bzl", "expand_template") + # A convenience template for creating the "command line" for the entrypoint expand_template( name = "entrypoint", data = [ ":mnist", - "@com_github_ggerganov_ggml_mnist//file", - "@com_github_ggerganov_ggml_mnist_data//file", + "@mnist//:mnist.safetensors", + "@mnist//:t10k-images.idx3-ubyte", ], substitutions = { - ":model": "$(rlocationpath @com_github_ggerganov_ggml_mnist//file)", - ":data": "$(rlocationpath @com_github_ggerganov_ggml_mnist_data//file)", + ":model": "$(rlocationpath @mnist//:mnist.safetensors)", + ":data": "$(rlocationpath @mnist//:t10k-images.idx3-ubyte)", }, template = [ "./{}/mnist".format(package_name()), - "./{}/mnist.runfiles/:model".format(package_name()), - "./{}/mnist.runfiles/:data".format(package_name()), + "/{}/mnist.runfiles/:model".format(package_name()), + "/{}/mnist.runfiles/:data".format(package_name()), ], ) ``` @@ -336,7 +340,7 @@ oci_image( name = "image_", base = "@distroless_cc_debian12", # the entrypoint comes from the expand_template rule `entrypoint` above - entrypoint = ":entrypoint", + entrypoint = ":entrypoint", tars = [":archive"], ) @@ -371,6 +375,5 @@ And that's it! With one simple bazel command, you can push a neatly packaged MNIST model, including weights and dataset, to the docker registry: ``` -bazel run //mnist:push --@zml//runtimes:cuda=true -- --repository index.docker.io/my_org/zml_mnist +bazel run //examples/mnist:push --@zml//platforms:cuda=true -- --repository index.docker.io/my_org/zml_mnist ``` - diff --git a/docs/howtos/howto_torch2zml.md b/docs/howtos/howto_torch2zml.md index ef16c0ef..3d246f11 100644 --- a/docs/howtos/howto_torch2zml.md +++ b/docs/howtos/howto_torch2zml.md @@ -67,14 +67,15 @@ package, so the simplest way to use it, is to copy it next to your python script. Then wrap the model/pipeline in a `zml_utils.ActivationCollector`. The collector wraps the given model, and returns the original results AND the activations in a dict of `torch.Tensor` when it's being called. After that, you -can save those activations to a `.pt` file. +can save those activations to a `.safetensors` file. ```python import torch import transformers +from safetensors.torch import save_file import zml_utils -model_path = "meta-llama/Meta-Llama-3-8B" +model_path = "meta-llama/Llama-3.2-1B-Instruct" pipeline = transformers.pipeline( "text-generation", @@ -97,8 +98,8 @@ if output: print(output) # Save activations to a file. -filename = model_path.split("/")[-1] + ".activations.pt" -torch.save(activations, filename) +filename = model_path.split("/")[-1] + ".activations.safetensors" +save_file(activations, filename) print(f"Saved {len(activations)} activations to {filename}") ``` @@ -118,45 +119,35 @@ model. Put the following in `my_project/torch2zml.zig`. const std = @import("std"); const log = std.log; -const asynk = @import("async"); const zml = @import("zml"); -pub fn main() !void { - var gpa = std.heap.GeneralPurposeAllocator(.{}){}; - defer _ = gpa.deinit(); - try asynk.AsyncThread.main(gpa.allocator(), asyncMain, .{}); -} - -pub fn asyncMain() !void { - var gpa = std.heap.GeneralPurposeAllocator(.{}){}; - defer _ = gpa.deinit(); - const allocator = gpa.allocator(); - - const args = try std.process.argsAlloc(allocator); - defer std.process.argsFree(allocator, args); +pub fn main(init: std.process.Init) !void { + const allocator = init.gpa; + const io = init.io; + const args = try init.minimal.args.toSlice(allocator); + defer allocator.free(args); const model_path, const activations_path = args[1..3].*; - const activations = try zml.aio.torch.open(allocator, activations_path); - defer activations.deinit(); - log.info("Found {} activations in {s}", .{ activations.buffers.count(), activations_path }); + var activations_registry = try zml.safetensors.TensorRegistry.fromPath(allocator, io, activations_path); + defer activations_registry.deinit(); + log.info("Found {} activations in {s}", .{ activations_registry.tensors.count(), activations_path }); - const model_weights = try zml.aio.detectFormatAndOpen(allocator, model_path); - defer model_weights.deinit(); - log.info("Found {} model layers in {s}", .{ model_weights.buffers.count(), activations_path }); + var model_registry = try zml.safetensors.TensorRegistry.fromPath(allocator, io, model_path); + defer model_registry.deinit(); + log.info("Found {} activations in {s}", .{ model_registry.tensors.count(), model_path }); } ``` -And add a `zig_cc_binary` target in `my_project/BUILD.bazel`: +And add a `zig_binary` target in `my_project/BUILD.bazel`: ```python -load("@zml//bazel:zig.bzl", "zig_cc_binary") +load("@rules_zig//zig:defs.bzl", "zig_binary") -zig_cc_binary( +zig_binary( name = "torch2zml", main = "torch2zml.zig", deps = [ - "@zml//async", "@zml//zml", ], ) @@ -166,19 +157,16 @@ Now check that the weights can be loaded correctly using the bazel CLI. ```bash bazel build //my_project:torch2zml -./bazel-bin/my_project/torch2zml /path/to/my/model.safetensors.index.json ./my_project/Meta-Llama-3-8B.activations.pt +./bazel-bin/my_project/torch2zml /path/to/my/model.safetensors.index.json ./my_project/Llama-3.2-1B-Instruct.activations.safetensors -info: Found 1108 activations in /Users/guw/Documents/zml/models/torch2zml/Meta-Llama-3-8B.activations.pt -debug(zml_io): Loading shard: model-00004-of-00004.safetensors -debug(zml_io): Loading shard: model-00001-of-00004.safetensors -debug(zml_io): Loading shard: model-00002-of-00004.safetensors -debug(zml_io): Loading shard: model-00003-of-00004.safetensors -info: Found 291 model layers in /Users/guw/Documents/zml/models/torch2zml/Meta-Llama-3-8B.activations.pt +info: Found 564 activations in /Users/corendos/dev/zml/python/temp/Llama-3.2-1B-Instruct.activations.safetensors +debug(zml/safetensors): Added new metadata key=format with value=pt +info: Found 146 activations in /Users/corendos/models/Llama-3.2-1B-Instruct/model.safetensors ``` -## Loading an individual layer +## Initializing an individual layer -In the above Zig code, the `model_weights` struct is a wrapper around a flat +In the above Zig code, the `model_registry` struct is a wrapper around a flat dictionary, containing an entry for each tensor in the model (similar to a "state dict"). Manipulating a dictionary is generally not very convenient, so let's convert it to a Zig struct. @@ -196,26 +184,70 @@ const Mlp = struct { The `zml.nn.Linear` is the equivalent of `torch.nn.Linear` and is defined by its `weight` and optional `bias` tensors. -To create such a struct from our `model_weights` dictionary, we can use the -`zml.aio.populateModelWithPrefix` helper: +To create such a struct from our `model_registry` dictionary, we first need to +convert it to a `TensorStore`: ```zig -pub fn asyncMain() !void { +pub fn main(init: std.process.Init) !void { ... - const mlp_shape = try zml.aio.populateModelWithPrefix(Mlp, allocator, model_weights, "model.layers.0.mlp"); - log.info("layer.0.mlp: {}", .{mlp_shape}); + var model_store: zml.io.TensorStore = .fromRegistry(allocator, &model_registry, &.{}); + defer model_store.deinit(); +} +``` + +Then, we can initialize our MLP: + +```zig +pub fn main(init: std.process.Init) !void { + ... + + const mlp_view = model_store.view().withPrefix("model.layers.0.mlp"); + + const mlp: Mlp = .{ + .up_proj = .init( + mlp_view.createReplicatedTensor("up_proj.weight", .{ .dout, .d }), + mlp_view.maybeCreateReplicatedTensor("up_proj.bias", .{.dout}), + .d, + ), + .gate_proj = .init( + mlp_view.createReplicatedTensor("gate_proj.weight", .{ .dout, .d }), + mlp_view.maybeCreateReplicatedTensor("gate_proj.bias", .{.dout}), + .d, + ), + .down_proj = .init( + mlp_view.createReplicatedTensor("down_proj.weight", .{ .d, .dout }), + mlp_view.maybeCreateReplicatedTensor("down_proj.bias", .{.d}), + .dout, + ), + }; } ``` Build and run, using previous commands. -Typical errors are of the form _"Layer not found: ..."_. This is typically due -to the naming of layers in Zig not matching the naming in the file. +Typical errors are due to the naming of layers in Zig not matching the naming +in the file. Double-check everything and don't hesitate to print more things, e.g. in the Python script. Alternatively, Huggingface's web-interface allows to peek into `.safetensor` files. +## Loading an individual layer + +Currently, the layer is initialized but the weights are still on the disk. +ZML provides a convenience to automatically load weights from an initialized model: + +```zig +pub fn main(init: std.process.Init) !void { + ... + + var platform: *zml.Platform = try .auto(allocator, io, .{}); + defer platform.deinit(allocator); + var mlp_weights = try zml.io.load(Mlp, &mlp, allocator, io, platform, &model_store, .auto); +} +``` + + ## Testing an individual layer Finally, we are going to write the actual math code for our `MLP` layer. @@ -226,39 +258,45 @@ const Mlp = struct { gate_proj: zml.nn.Linear, down_proj: zml.nn.Linear, - pub fn forward(self: Mlp, x: Tensor) Tensor { - const proj = zml.call(self.up_proj, .forward, .{x}); - var output = zml.call(self.gate_proj, .forward, .{x}); + pub fn forward(self: Mlp, x: zml.Tensor) zml.Tensor { + const proj = self.up_proj.forward(x, x.dtype()); + var output = self.gate_proj.forward(x, x.dtype()); output = output.silu().mul(proj); - return zml.call(self.down_proj, .forward, .{output}); + return self.down_proj.forward(output, x.dtype()); } }; ``` -Note that we use `zml.call` instead of directly calling -`self.up_proj.forward(x)`. Calling `forward` directly results in the same -computation happening at runtime; but going through `zml.call` allows ZML to -generate an MLIR representation that is closer to the Zig code and therefore -easier to read. - -We can test the MLP layer with the `zml.testing.testLayer` utility: +Our MLP layer is now almost ready to be tested with the `zml.testing.testLayer` utility. +As you may have noticed, ZML makes heavy-use of tagged tensors. In the MLP, we expect +to do a dot product on the `.d` dimension. But the activations we load from the +activations files are not tagged. One solution could be to tag the tensors before saving +them to file, but a much simpler solution is to intercept the inputs before providing +them to the layer we want to test, using a wrapper. ```zig -pub fn asyncMain() !void { - ... - - var ctx = try zml.Context.init(); - defer ctx.deinit(); - const platform = ctx.autoPlatform(); - const mlp_weights = try zml.aio.loadModelBuffers(Mlp, mlp_shape, model_weights, allocator, platform); +pub const Wrapper = struct { + mlp: Mlp, - zml.testing.testLayer(platform, activations, "model.layers.0.mlp", mlp_shape, mlp_weights, 1e-3); + pub fn forward(wrapper: Wrapper, input: zml.Tensor) zml.Tensor { + const tagged_input = input.withTags(.{.b, .s, .d}); + return wrapper.mlp.forward(tagged_input); + } +}; + +pub fn main(init: std.process.Init) !void { + ... + + var activations_store: zml.io.TensorStore = .fromRegistry(allocator, &activations_registry, &.{}); + defer activations_store.deinit(); + + try zml.testing.testLayer(allocator, io, platform, Wrapper{ .mlp = mlp }, .forward, activations_store.view(), "model.model.layers.0.mlp", .{ .mlp = mlp_weights }, &.{}, .{}); } ``` During this phase, you have three kinds of errors that can appear: -* Zig compilation errors: we've all have been there, learning a new language +* Zig compilation errors: we've all been there, learning a new language can be tough. Normally, the compiler should help you figure out what's wrong. You can also check [ZML concepts](../learn/concepts.md) that explains types used by ZML. @@ -293,5 +331,5 @@ During this phase, you have three kinds of errors that can appear: "C,W,H" of tensors, you can port this to actual tensor attributes using `x.withTags(.{.c, .w, .h})`, and use those tags (eg `.c`) to refer to axes instead of offsets. E.g. in Pytorch: `x.sum(0) # reduce over channel axis` - becomes `x.sum(.c)`. More on this topic in + becomes `x.sum(.c)`. More on this topic in ["Working with tensors"](../tutorials/working_with_tensors.md). diff --git a/docs/howtos/huggingface_access_token.md b/docs/howtos/huggingface_access_token.md index 99967bb7..cc11ffbb 100644 --- a/docs/howtos/huggingface_access_token.md +++ b/docs/howtos/huggingface_access_token.md @@ -1,24 +1,21 @@ +# Running Gated Huggingface Models with Token Authentication -# Huggingface Token Authentication +Some models have restrictions and may require some sort of approval or agreement +process, which, by consequence, **requires token-authentication with Huggingface**. -Some models have restrictions and may require some sort of approval or -agreement process, which, by consequence, **requires token-authentication with -Huggingface**. +The easiest way might be to use the `hf auth login` command. -Here is how you can generate a **"read-only public repositories"** access token -to log into your account on Huggingface, directly from `bazel`, in order to -download models. +Alternatively, here is how you can generate a **"read-only public repositories"** +access token to log into your account on Huggingface, directly from `bazel`, in order to download models. * log in at [https://huggingface.co/settings/tokens](https://huggingface.co/settings/tokens). * click on "Create new token" -* give the token a name, eg `zml_public_repos` -* under _Repositories_, grant the following permission: "Read access to - contents of all public gated repos you can access". -* at the bottom, click on "Create token". +* give the token a name, eg `zml_public_repos`, +* under _Repositories_, grant the following permission: "Read access to contents of all public gated repos you can access". +* at the bottom click on "Create token". * copy the token by clicking `Copy`. **You won't be able to see it again.** * the token looks something like `hf_abCdEfGhijKlM`. -* store the token on your machine (replace the placeholder with your actual - token): +* store the token on your machine (replace the placeholder with your actual token): You can use the `HUGGINGFACE_TOKEN` environment variable to store the token or use its standard location: @@ -26,16 +23,12 @@ its standard location: mkdir -p $HOME/.cache/huggingface/; echo > "$HOME/.cache/huggingface/token" ``` -Now you're ready to download a gated model like `Meta-Llama-3-8b`! +Now you're ready to download a gated model like `Meta-Llama-3.2-1b`! **Example:** ``` # requires token in $HOME/.cache/huggingface/token, as created by the -# `huggingface-cli login` command, or the `HUGGINGFACE_TOKEN` environment variable. -cd examples -bazel run -c opt //llama:Meta-Llama-3-8b -bazel run -c opt //llama:Meta-Llama-3-8b -- --promt="Once upon a time," +# `hf auth login` command, or the `HUGGINGFACE_TOKEN` environment variable. +bazel run --config=release //examples/llm -- --model=hf://meta-llama/Llama-3.2-1B-Instruct --prompt="What is the capital of France?" ``` - - diff --git a/docs/howtos/profiling.md b/docs/howtos/profiling.md new file mode 100644 index 00000000..f98a7dc2 --- /dev/null +++ b/docs/howtos/profiling.md @@ -0,0 +1,375 @@ +# Profiling ZML Programs + +ZML has two profiling layers: + +- host tracing through `zml.tracer.scope(...)` +- backend profiling through `zml.Platform.profiler(...)` + +The host tracing path is always based on `TraceMe`. Depending on the target +platform, the tracing bridge also emits device-visible annotations: + +- CUDA on Linux: NVTX ranges +- ROCm on Linux: ROCTx ranges +- macOS: `os_signpost` intervals for Instruments + +When backend profiling is enabled, ZML writes both the raw `XSpace` protobuf +and a streamed Perfetto / trace-viewer JSON file. + +## Profiling architecture + +ZML profiling is split into four layers: + +1. `zml.tracer.scope(...)` in Zig marks lexical regions of interest +2. `zml/profiling/traceme_lib.cc` bridges those scopes into platform-specific + tracing backends +3. `zml.Platform.profiler(...)` manages a profiling session and collects profile + data from PJRT +4. `tools/xspace_to_perfetto` converts the final `XSpace` protobuf into a + streamed trace-event JSON file + +The important architectural split is: + +- `zml.tracer.scope(...)` is for instrumentation inside your code +- `zml.Platform.profiler(...)` is for session lifecycle and artifact generation + +You can use scopes without a profiler session, but you only get exported +`profiling.xplane.pb` and `profiling.trace.json` when a profiler session is +started. + +### Scope instrumentation path + +`zml.tracer.scope(...)` is the lowest-level API you use directly in Zig code. +Each scope creates a `TraceMe` region, and the C bridge adds extra +platform-visible annotations when supported: + +- CUDA/Linux: NVTX ranges +- ROCm/Linux: ROCTx ranges +- macOS: `os_signpost` intervals + +Metadata passed to `zml.tracer.scope(...)` is encoded into the `TraceMe` scope +name. That is why the API only accepts small scalar or string-like fields: it is +designed to produce compact trace labels rather than structured protobuf +payloads. + +### Profiler session path + +`zml.Platform.profiler(...)` builds a `tensorflow.ProfileOptions` protobuf from +`zml.Platform.ProfilerOptions` and asks the active PJRT backend for a profiler +session. + +At `start()` time, ZML does two distinct things: + +- it tries to start the backend PJRT profiler session +- it configures local host-side `TraceMeRecorder` capture when enabled + +At `stop()` time, ZML: + +1. stops the PJRT profiler session if one exists +2. collects the backend `XSpace` protobuf +3. merges locally recorded host `TraceMe` events into the host plane +4. writes the final protobuf to disk +5. streams trace JSON from that final protobuf + +This separation is important because host scopes and backend profiler output are +not produced by the same subsystem. ZML joins them only at the end of the +session. + +### Export path + +The final export path is intentionally one-way: + +- source data format: `XSpace` protobuf +- derived export format: trace-event JSON for Perfetto / trace viewer + +ZML keeps `profiling.xplane.pb` as the source of truth and generates +`profiling.trace.json` from it afterwards. The converter is implemented in Zig +and streams JSON directly from the protobuf instead of materializing the whole +JSON document in memory first. + +## The generated artifacts + +By default, a profiling session is written under: + +```text +/tmp/xprof/plugins/profile/profiling/ +``` + +The relevant files are: + +- `profiling.xplane.pb`: raw `XSpace` protobuf +- `profiling.trace.json`: Perfetto / trace-viewer JSON export + +These paths come from `zml/profiling/profiler.zig` and can be changed through +`zml.Platform.ProfilerOptions`. + +## Add trace scopes in Zig + +Use `zml.tracer.scope(name, metadata)` to bracket the code you want to inspect. + +```zig +const zml = @import("zml"); + +var trace = try zml.tracer.scope("llm.compile", .{ + .seqlen = args.seqlen, + .interactive = args.prompt == null, +}); +defer trace.end(); + +try models.LoadedModel.compile(...); +``` + +Use `.{}` when there is no metadata: + +```zig +var trace = try zml.tracer.scope("llm.load_buffers", .{}); +defer trace.end(); +``` + +Metadata is encoded into the `TraceMe` scope name, so keep it small and +high-signal. + +## Record a ZML profile from code + +If you want ZML to collect a backend profile and export `profiling.trace.json`, +create a profiler from the current platform: + +```zig +const platform: *zml.Platform = try .auto(allocator, io, .{}); + +var profiler = try platform.profiler(allocator, io, .defaults); +defer profiler.deinit(); + +try profiler.start(); +defer { + if ((profiler.stop() catch unreachable)) |profile| { + std.log.info("Profile dumped: {s} and {s}", .{ + profile.protobuf_path, + profile.perfetto_path, + }); + } +} +``` + +This is the normal ZML flow: + +1. start the PJRT backend profiler +2. record local `TraceMe` scopes +3. merge the local host trace into the backend `XSpace` +4. write `profiling.xplane.pb` +5. stream `profiling.trace.json` + +If the backend profiler is unavailable, ZML still keeps the local host trace +path and logs a warning. + +## CUDA on Linux with Nsight Systems + +ZML provides a Bazel run config for `nsys`: + +```bash +bazel run --config=nsys //examples/benchmark:benchmark +``` + +Or for a real model: + +```bash +bazel run --config=nsys //examples/llm -- \ + --model=/path/to/model \ + --prompt="Write a story about a cat" +``` + +The wrapper target is `//tools:nsys`. By default it runs: + +```text +sudo -E nsys profile ... +``` + +with CUDA, cuBLAS, cuSPARSE, cuDNN, NVTX, syscall, and OS runtime tracing +enabled. + +If you need to install Nsight Systems manually, use NVIDIA's official download +page: . For the server +or target machine where `nsys` will run, choose the `Linux CLI only` package. +For the client machine where you open and inspect reports, choose the full host +package for your OS. NVIDIA's installation guide is here: +. + +Useful overrides: + +- `ZML_PROFILE_NO_SUDO=1`: run `nsys` without `sudo` +- `ZML_NSYS_ARGS='...'`: replace the default `nsys` arguments + +Example: + +```bash +ZML_PROFILE_NO_SUDO=1 bazel run --config=nsys //examples/benchmark:benchmark +``` + +### Important interaction with `zml.profiler` + +The `nsys` wrapper sets: + +```text +SKIP_PJRT_PROFILER=true +``` + +That tells `zml/profiling/profiler.zig` to skip nested PJRT backend profiling. This is +intentional. Under `nsys`, you normally want: + +- the external Nsight session +- local `TraceMe` / NVTX annotations from your scopes + +and not a second backend profile session layered on top. + +## ROCm on Linux with `rocprofv3` + +ZML also provides a Bazel run config for ROCm: + +```bash +bazel run --config=rocprofv3 //examples/benchmark:benchmark +``` + +Or: + +```bash +bazel run --config=rocprofv3 //examples/llm -- \ + --model=/path/to/model \ + --prompt="Write a story about a cat" +``` + +The wrapper target is `//tools:rocprofv3`. + +This path is intentionally sandboxed inside the repository's ROCm packaging: + +- it launches the Bazel-runfiles copy of `rocprofv3` +- it passes `--rocm-root` pointing at the sandboxed ROCm runtime + +So it does not depend on a host-installed `rocprofv3` layout. + +The default wrapper enables system, runtime, HIP, kernel, memory copy, +scratch-memory, HSA core, and marker tracing, and writes `pftrace` output. + +Useful overrides: + +- `ZML_ROCPROFV3_ARGS='...'`: replace the default `rocprofv3` arguments + +### Important interaction with `zml.profiler` + +Like the `nsys` wrapper, the `rocprofv3` wrapper sets: + +```text +SKIP_PJRT_PROFILER=true +``` + +That means: + +- the external `rocprofv3` session owns backend/device profiling +- ZML still records local host `TraceMe` scopes +- ROCTx ranges from `zml.tracer.scope(...)` stay visible as device annotations + +## Neuron Profiling + +Use `--config=neuron-profile` with the command you already run. The wrapper +prints the profile directory when the program exits. + +```bash +NEURON_RT_VISIBLE_CORES=0,1 \ +bazel run --config=neuron-profile //examples/llm -- \ + --model=/path/to/model \ + --prompt="Explain mixture-of-experts routing in one paragraph." \ + --seqlen=128 \ + --backend=nki \ + --topk=1 +``` + +The default profile root is `/tmp/zml-neuron-profile`. Each run contains: + +```text +/tmp/zml-neuron-profile// + compile/ + execution/ +``` + +Start the Neuron Explorer server in one terminal: + +```bash +bazel run //tools/neuron:server +``` + +Open . When working through SSH, forward ports `3001` +and `3002`. + +With the server running, ingest the run from another terminal: + +```bash +bazel run //tools/neuron:ingest -- /tmp/zml-neuron-profile/ +``` + +Create a quick summary from one run when you want a file artifact: + +```bash +bazel run //tools/neuron:summary-json -- /tmp/zml-neuron-profile/ +bazel run //tools/neuron:summary-txt -- /tmp/zml-neuron-profile/ +bazel run //tools/neuron:summary-perfetto -- /tmp/zml-neuron-profile/ +``` + +Each summary command writes beside the run and prints the output path: +`summary.json`, `summary.txt`, or `summary.pftrace`. + +## macOS Instruments + +On macOS, `zml.tracer.scope(...)` emits `os_signpost` intervals. These show up +in Instruments, especially in the `Points of Interest` template. + +Build the binary first: + +```bash +bazel build //examples/llm:llm +``` + +Then either launch it from the Instruments UI or use `xctrace`: + +```bash +xctrace record \ + --template 'Points of Interest' \ + --output /tmp/llm.trace \ + --launch -- \ + /home/you/zml/bazel-bin/examples/llm/llm \ + --model=/path/to/model \ + --prompt="Write a story about a cat" +``` + +Use full Xcode for `xctrace`. Command Line Tools alone are not enough. + +## Open the exported trace + +The `profiling.trace.json` file can be opened in Perfetto or in tools that still +understand the trace-event JSON format. + +Typical flow: + +1. run a ZML program with `zml.Platform.profiler(...)` +2. wait for `profiling.trace.json` +3. open the JSON in Perfetto + +The converter is implemented in Zig and streams JSON directly from `XSpace`, +which keeps large traces much more manageable than the old in-memory path. + +## Platform summary + +- Plain ZML profiling: backend `XSpace` profile plus merged local host trace +- CUDA + `--config=nsys`: external Nsight session plus local `TraceMe` / NVTX +- ROCm + `--config=rocprofv3`: external `rocprofv3` session plus local `TraceMe` / ROCTx +- macOS Instruments: local `TraceMe` scopes exposed as `os_signpost` + +## Troubleshooting + +- If you run under `--config=nsys`, `--config=rocprofv3`, or + `--config=neuron-profile`, PJRT profiling is skipped on purpose because the + wrappers set `SKIP_PJRT_PROFILER=true`. +- If you do not see device annotations, first check whether you are on a + supported platform: + - CUDA/Linux for NVTX + - ROCm/Linux for ROCTx + - macOS for Instruments signposts +- If you only need host scopes, `zml.tracer.scope(...)` works independently of + whether backend profiling succeeds. diff --git a/docs/huggingface-access-token.md b/docs/huggingface-access-token.md deleted file mode 100644 index 94514162..00000000 --- a/docs/huggingface-access-token.md +++ /dev/null @@ -1,37 +0,0 @@ -# Running Gated Huggingface Models with Token Authentication - -Some models have restrictions and may require some sort of approval or agreement -process, which, by consequence, **requires token-authentication with Huggingface**. - -The easiest way might be to use the `huggingface-cli login` command. - -Alternatively, here is how you can generate a **"read-only public repositories"** -access token to log into your account on Huggingface, directly from `bazel`, in order to download models. - -* log in at [https://huggingface.co/settings/tokens](https://huggingface.co/settings/tokens). -* click on "Create new token" -* give the token a name, eg `zml_public_repos`, -* under _Repositories_, grant the following permission: "Read access to contents of all public gated repos you can access". -* at the bottom click on "Create token". -* copy the token by clicking `Copy`. **You won't be able to see it again.** -* the token looks something like `hf_abCdEfGhijKlM`. -* store the token on your machine (replace the placeholder with your actual token): - -You can use the `HUGGINGFACE_TOKEN` environment variable to store the token or use -its standard location: -``` -mkdir -p $HOME/.cache/huggingface/; echo > "$HOME/.cache/huggingface/token" -``` - -Now you're ready to download a gated model like `Meta-Llama-3-8b`! - -**Example:** - -``` -# requires token in $HOME/.cache/huggingface/token, as created by the -# `huggingface-cli login` command, or the `HUGGINGFACE_TOKEN` environment variable. -cd examples -bazel run -c opt //llama:Meta-Llama-3-8b -bazel run -c opt //llama:Meta-Llama-3-8b -- --promt="Once upon a time," -``` - diff --git a/docs/learn/concepts.md b/docs/learn/concepts.md index 93e0d9bc..1a3cea05 100644 --- a/docs/learn/concepts.md +++ b/docs/learn/concepts.md @@ -8,8 +8,8 @@ particulary Neural Networks (NN). The lifecycle of a model is implemented in the following steps: -1. Open the model file and read the shapes of the weights, but leave the - weights on the disk. +1. Open the model file and read the shapes of the weights, but don't load the weights + yet. 2. Using the loaded shapes and optional metadata, instantiate a model struct with `Tensor`s, representing the shape and layout of each layer of the NN. @@ -20,28 +20,21 @@ The lifecycle of a model is implemented in the following steps: 4. Load the model weights from disk, onto the accelerator memory. -5. Bind the model weights to the executable. +5. Load some user inputs, and copy them to the accelerator. -6. Load some user inputs, and copy them to the accelerator. +6. Call the executable using the weights and the user inputs. -7. Call the executable on the user inputs. - -8. Fetch the returned model output from accelerator into host memory, and +7. Fetch the returned model output from accelerator into host memory, and finally present it to the user. -9. When all user inputs have been processed, free the executable resources and - the associated weights. +8. When all user inputs have been processed, free the executable resources, weights and + inputs. **Some details:** Note that the compilation and weight loading steps are both bottlenecks to your -model startup time, but they can be done in parallel. **ZML provides -asynchronous primitives** to make that easy. - -The **compilation can be cached** across runs, and if you're always using the -same model architecture with the same shapes, it's possible to by-pass it -entirely. +model startup time, but they can be done in parallel thanks to Zig's `std.Io` interface. The accelerator is typically a GPU, but can be another chip, or even the CPU itself, churning vector instructions. @@ -51,7 +44,7 @@ itself, churning vector instructions. In ZML, we leverage Zig's static type system to differentiate between a few concepts, hence we not only have a `Tensor` to work with, like other ML -frameworks, but also `Buffer`, `HostBuffer`, and `Shape`. +frameworks, but also `Buffer`, `Slice`, and `Shape`. Let's explain all that. @@ -65,25 +58,19 @@ Let's explain all that. `Shape` struct can also represent a regular number, aka a scalar: `Shape.init(.{}, .i32)` represents a 32-bit signed integer. -* `HostBuffer`: _is_ a multi-dimensional array, whose memory is allocated **on - the CPU**. - - points to the slice of memory containing the array - - typically owns the underlying memory - but has a flag to remember when it - doesn't. +* `Slice`: _is_ the combination of a `Shape` and raw bytes (bytes that are **on the CPU**). + - can own the underlying memory - but can also accomodate non-owned memory. * `Buffer`: _is_ a multi-dimension array, whose memory is allocated **on an accelerator**. - contains a handle that the ZML runtime can use to convert it into a physical address, but there is no guarantee this address is visible from the CPU. - - can be created by loading weights from disk directly to the device via - `zml.aio.loadBuffers` - - can be created by calling `HostBuffer.toDevice(accelerator)`. + - can be created from a `Slice` by calling `Buffer.fromSlice(...)`. * `Tensor`: is a mathematical object representing an intermediary result of a - computation. - - is basically a `Shape` with an attached MLIR value representing the - mathematical operation that produced this `Tensor`. + computation or an input to an executable (including weights). + - is basically a `Shape` with an attached MLIR value when in the context of a compilation. ## The model struct @@ -91,7 +78,7 @@ Let's explain all that. The model struct is the Zig code that describes your Neural Network (NN). Let's look a the following model architecture: -![Multilayer perceptrons](https://zml.ai/docs-assets/perceptron.png) +![Multilayer perceptrons](https://raw.githubusercontent.com/zml/zml.github.io/refs/heads/main/docs-assets/perceptron.png) This is how we can describe it in a Zig struct: @@ -137,25 +124,22 @@ struct of `Model` but with a `Buffer` replacing every `Tensor`. Let's look at the model life cycle again, but this time annotated with the corresponding types. -1. Open the model file and read the shapes of the weights -> `zml.HostBuffer` - (using memory mapping, no actual copies happen yet) +1. Open the model file and read the shapes of the weights -> `zml.Shape` 2. Instantiate a model struct -> `Model` struct (with `zml.Tensor` inside) 3. Compile the model struct and its `forward` function into an executable. `foward` is a `Tensor -> Tensor` function, executable is a - `zml.Exe(Model.forward)` + `zml.Executable` 4. Load the model weights from disk, onto accelerator memory -> `zml.Bufferized(Model)` struct (with `zml.Buffer` inside) -5. Bind the model weights to the executable `zml.ExeWithWeight(Model.forward)` +5. Load some user inputs (custom struct), encode them into arrays of numbers + (`zml.Slice`), and copy them to the accelerator (`zml.Buffer`). -6. Load some user inputs (custom struct), encode them into arrays of numbers - (`zml.HostBuffer`), and copy them to the accelerator (`zml.Buffer`). - -7. Call the executable on the user inputs. `module.call` accepts `zml.Buffer` +6. Call the executable on the user inputs. `exe.call` accepts `zml.Buffer` arguments and returns `zml.Buffer` -8. Return the model output (`zml.Buffer`) to the host (`zml.HostBuffer`), +7. Return the model output (`zml.Buffer`) to the host (`zml.Slice`), decode it (custom struct) and finally return to the user. diff --git a/docs/misc/style_guide.md b/docs/misc/style_guide.md index 165555ce..a112377c 100644 --- a/docs/misc/style_guide.md +++ b/docs/misc/style_guide.md @@ -1,7 +1,7 @@ # ZML Style Guide -We prefer to keep it simple and adhere to the [Zig Style Guide](https://ziglang.org/documentation/0.13.0/#Style-Guide). +We prefer to keep it simple and adhere to the [Zig Style Guide](https://ziglang.org/documentation/0.15.1/#Style-Guide). We use ZLS to auto-format code. diff --git a/docs/tutorials/getting_started.md b/docs/tutorials/getting_started.md index dcb8678d..ababeb15 100644 --- a/docs/tutorials/getting_started.md +++ b/docs/tutorials/getting_started.md @@ -21,14 +21,14 @@ through `bazelisk`, a version manager for `bazel`. **macOs:** ``` - brew install bazelisk +brew install bazelisk ``` **Linux:** ``` - curl -L -o /usr/local/bin/bazel 'https://github.com/bazelbuild/bazelisk/releases/download/v1.20.0/bazelisk-linux-amd64' - chmod +x /usr/local/bin/bazel +curl -L -o /usr/local/bin/bazel 'https://github.com/bazelbuild/bazelisk/releases/download/v1.28.1/bazelisk-linux-amd64' +chmod +x /usr/local/bin/bazel ``` @@ -49,8 +49,7 @@ compile it, and classify a randomly picked example from the test dataset. On the command line: ``` -cd examples -bazel run -c opt //mnist +bazel run --config=release //examples/mnist ``` ### Llama @@ -59,47 +58,39 @@ Llama is a family of "Large Language Models", trained to generate text, based on the beginning of a sentence/book/article. This "beginning" is generally referred to as the "prompt". -#### TinyLlama, Stories 15M - -To start, you can use a small model trained specifically on children's history -books. This model has been trained by [Andrej Karpathy](https://x.com/karpathy); -you can read more about it on his -[Github](https://github.com/karpathy/llama2.c). - -``` -cd examples -bazel run -c opt //llama:TinyLlama-Stories-15M -bazel run -c opt //llama:TinyLlama-Stories-15M -- --prompt="Once upon a time, there was a cute little dragon" -``` - -#### OpenLLama 3B - -``` -cd examples -bazel run -c opt //llama:OpenLLaMA-3B -bazel run -c opt //llama:OpenLLaMA-3B -- --prompt="Once upon a time," -``` - -#### Meta Llama 3 8B +#### Meta Llama 3.1 8B This model has restrictions, see -[here](https://huggingface.co/meta-llama/Meta-Llama-3-8B): it **requires +[here](https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct). It **requires approval from Meta on Huggingface**, which can take a few hours to get granted. While waiting for approval, you can already [generate your Huggingface access token](../howtos/huggingface_access_token.md). Once you've been granted access, you're ready to download a gated model like -`Meta-Llama-3-8b`! +`Meta-Llama-3.1-8B-Instruct`! ``` -# requires token in $HOME/.cache/huggingface/token, as created by the -# `huggingface-cli login` command, or the `HUGGINGFACE_TOKEN` environment variable. -cd examples -bazel run -c opt //llama:Meta-Llama-3-8b -bazel run -c opt //llama:Meta-Llama-3-8b -- --promt="Once upon a time," +# Authenticate on Huggingface. +bazel run //tools/hf -- auth login +# Run the model, loading directly from HF. +bazel run --config=release //examples/llm -- --model=hf://meta-llama/Llama-3.1-8B-Instruct --prompt="What is the capital of France?" ``` +You can also try `Llama-3.1-70B-Instruct` if you have enough memory. + +### Meta Llama 3.2 1B + +Like the 8B model above, this model also requires approval. See +[here](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct) for access requirements. + +``` +bazel run //tools/hf -- auth login +bazel run --config=release //examples/llm -- --model=hf://meta-llama/Llama-3.2-1B-Instruct --prompt="What is the capital of France?" +``` + +For a larger 3.2 model, you can also try `Llama-3.2-3B-Instruct`. + ## Run Tests @@ -112,22 +103,24 @@ bazel test //zml:test You can compile models for accelerator runtimes by appending one or more of the following arguments to the command line when compiling or running a model: -- NVIDIA CUDA: `--@zml//runtimes:cuda=true` -- AMD RoCM: `--@zml//runtimes:rocm=true` -- Google TPU: `--@zml//runtimes:tpu=true` -- **AVOID CPU:** `--@zml//runtimes:cpu=false` +- NVIDIA CUDA: `--@zml//platforms:cuda=true` +- AMD RoCM: `--@zml//platforms:rocm=true` +- Google TPU: `--@zml//platforms:tpu=true` +- AWS Trainium/Inferentia 2: `--@zml//platforms:neuron=true` +- **AVOID CPU:** `--@zml//platforms:cpu=false` The latter, avoiding compilation for CPU, cuts down compilation time. -So, to run the OpenLLama model from above on your host sporting an NVIDIA GPU, +So, to run the Llama model from above on your host sporting an NVIDIA GPU, run the following: ``` -cd examples -bazel run -c opt //llama:OpenLLaMA-3B \ - --@zml//runtimes:cuda=true \ - -- --prompt="Once upon a time," +bazel run --config=release //examples/llm \ + --@zml//platforms:cuda=true \ + -- \ + --model=hf://meta-llama/Llama-3.1-8B-Instruct \ + --prompt="What is the capital of France?" ``` @@ -143,4 +136,3 @@ You might also want to check out the [documentation](../README.md), start [writing your first model](../tutorials/write_first_model.md), or read about more high-level [ZML concepts](../learn/concepts.md). - diff --git a/docs/tutorials/write_first_model.md b/docs/tutorials/write_first_model.md index 0bd93022..312c9afc 100644 --- a/docs/tutorials/write_first_model.md +++ b/docs/tutorials/write_first_model.md @@ -8,7 +8,7 @@ - add some Bazel to integrate our code with ZML - no weights files or anything external is required for this example -The reason we're doing our excercise in the `examples` folder is because it's +The reason we're doing our exercise in the `examples` folder is because it's especially prepared for new ZML projects. It contains everything needed for ZML development. From `bazel` configs to `vscode` settings, and `neovim` LSP support. The `examples` folder serves as a cookiecutter ZML project example, @@ -42,50 +42,39 @@ touch simple_layer/main.zig touch simple_layer/BUILD.bazel ``` -By the way, you can access the complete source code of this walkthrough here: - -- [main.zig](https://github.com/zml/zml/tree/master/examples/simple_layer/main.zig) -- [BUILD.bazel](https://github.com/zml/zml/tree/master/examples/simple_layer/BUILD.bazel) - - - ## The high-level Overview Before firing up our editor, let's quickly talk about a few basic ZML fundamentals. - + In ZML, we describe a _Module_, which represents our AI model, as a Zig `struct`. That struct can contain Tensor fields that are used for computation, e.g. weights and biases. In the _forward_ function of a Module, we describe the -computation by calling tensor operations like _mul_, _add_, _dotGeneral_, -_conv2D_, etc., or even nested Modules. +computation by calling tensor operations like _mul_, _add_, _dot_, +_conv2D_, etc., or even nested Modules. ZML creates an MLIR representation of the computation when we compile the -Module. For compilation, only the _Shapes_ of all tensors must be known. No -actual tensor data is needed at this step. This is important for large models: -we can compile them while the actual weight data is being fetched from disk. +_Module_. For compilation, only the _Tensors_ are required. No actual tensor data +is needed at this step. This is important for large models: we can compile them +while the actual weight data is being fetched from disk. -To accomplish this, ZML uses a _BufferStore_. The _BufferStore_ knows how to -only load shapes and when to load actual tensor data. In our example, we will -fake the _BufferStore_ a bit: we won't load from disk; we'll use float arrays -instead. +To accomplish this, ZML uses a _TensorStore_. The _TensorStore_ loads everything +that is required to make _Tensors_ and later materialize them to _Buffers_. In our +example though, we won't even use a _TensorStore_ as it's optional. We will directly +create tensors using their shape and data type. -After compilation is done (and the _BufferStore_ has finished loading weights), -we can send the weights from the _BufferStore_ to our computation device. That -produces an _executable_ module which we can call with different _inputs_. - -In our example, we then copy the result from the computation device to CPU -memory and print it. +After compilation is done, we get what we call an _Executable_. From this _Executable_ +we can create _Args_ and _Results_, two structs that will store respectively the inputs +and outputs of a computation done with an _Executable_. **So the steps for us are:** - describe the computation as ZML _Module_, using tensor operations -- create a _BufferStore_ that provides _Shapes_ and data of weights and bias - (ca. 5 lines of code). -- compile the _Module_ **asynchronously** -- make the compiled _Module_ send the weights (and bias) to the computation - device utilizing the _BufferStore_, producing an _executable_ module -- prepare input tensor and call the _executable_ module. +- initialize the _Module_ +- compile the _Module_ to produce an _Executable_ +- make a "bufferized" version of the _Module_, containing the actual data on the + computation device +- prepare the _Args_ and _Results_ of a computation and call the _Executable_ - get the result back to CPU memory and print it If you like to read more about the underlying concepts of the above, please see @@ -99,16 +88,9 @@ Let's start by writing some Zig code, importing ZML and often-used modules: ```zig const std = @import("std"); const zml = @import("zml"); -const asynk = @import("async"); - -// shortcut to the async_ function in the asynk module -const async_ = asynk.async_; ``` -You will use above lines probably in all ZML projects. Also, note that **ZML is -async** and comes with its own async runtime, thanks to -[zigcoro](https://github.com/rsepassi/zigcoro). - +You will use above lines probably in all ZML projects. ### Defining our Model @@ -140,204 +122,139 @@ There are more things to observe: - more advanced use-cases are passing in / returning structs or tuples, like `struct { Tensor, Tensor }` as an example for a tuple of two tensors. You can see such use-cases, for example in the - [Llama Model](https://github.com/zml/zml/tree/master/examples/llama) + [LLM CLI](https://github.com/zml/zml/tree/master/examples/llm) - in the model, tensors may be optional. As is the case with `bias`. ### Adding a main() function -ZML code is async. Hence, We need to provide an async main function. It works -like this: - ```zig -pub fn main() !void { - var gpa = std.heap.GeneralPurposeAllocator(.{}){}; - defer _ = gpa.deinit(); - try asynk.AsyncThread.main(gpa.allocator(), asyncMain, .{}); -} +pub fn main(init: std.process.Init) !void { + const allocator = init.gpa; + const io = init.io; - -pub fn asyncMain() !void { - // ... -``` - -The above `main()` function only creates an allocator and an async main thread -that executes our `asyncMain()` function by calling it with no (`.{}`) -arguments. - -So, let's start with the async main function: - -```zig -pub fn asyncMain() !void { - // Short lived allocations - var gpa = std.heap.GeneralPurposeAllocator(.{}){}; - defer _ = gpa.deinit(); - const allocator = gpa.allocator(); - - // Arena allocator for BufferStore etc. - var arena_state = std.heap.ArenaAllocator.init(allocator); - defer arena_state.deinit(); - const arena = arena_state.allocator(); - - // Create ZML context - var context = try zml.Context.init(); - defer context.deinit(); - - const platform = context.autoPlatform(); + var platform: *zml.Platform = try .auto(allocator, io, .{}); + defer platform.deinit(allocator, io); + ... } ``` -This is boilerplate code that provides a general-purpose allocator and, for -convenience, an arena allocator that we will use later. The advantage of arena -allocators is that you don't need to deallocate individual allocations; you -simply call `.deinit()` to deinitialize the entire arena instead! +The above `main()` function takes advantage of Zig's "fat main" to avoid having to write +boilerplate code for allocations and io. -We also initialize the ZML context `context` and get our CPU `platform` -automatically. +We also get our ZML CPU platform `platform` automatically. +### Initializing the Module -### The BufferStore - -Next, we need to set up the concrete weight and bias tensors for our model. -Typically, we would load them from disk. But since our example works without -stored weights, we are going to create a BufferStore manually, containing -_HostBuffers_ (buffers on the CPU) for both the `weight` and the `bias` tensor. - -A BufferStore basically contains a dictionary with string keys that match the -name of the struct fields of our `Layer` struct. So, let's create this -dictionary: +Next, we need to initialize the module so that we can compile it. ```zig -// Our weights and bias to use -var weights = [3]f16{ 2.0, 2.0, 2.0 }; -var bias = [3]f16{ 1.0, 2.0, 3.0 }; -const input_shape = zml.Shape.init(.{3}, .f16); - -// We manually produce a BufferStore. You would not normally do that. -// A BufferStore is usually created by loading model data from a file. -var buffers: zml.aio.BufferStore.Buffers = .{}; -try buffers.put(arena, "weight", zml.HostBuffer.fromArray(&weights)); -try buffers.put(arena, "bias", zml.HostBuffer.fromArray(&bias)); - -// the actual BufferStore -var bs: zml.aio.BufferStore = .{ - .arena = arena_state, - .buffers = buffers, +const layer: Layer = .{ + .weight = zml.Tensor.init(.{3}, .f16), + .bias = zml.Tensor.init(.{3}, .f16), }; ``` -Our weights are `{2.0, 2.0, 2.0}`, and our bias is just `{1.0, 2.0, 3.0}`. The -shape of the weight and bias tensors is `{3}`, and because of that, the **shape -of the input tensor** is also going to be `{3}`! - -Note that `zml.Shape` always takes the data type associated with the tensor. In -our example, that is `f16`, expressed as the enum value `.f16`. - +As you can see, it's pretty simple. You can create _Tensors_ from their shape and data type. +Alternatively, you are free to add an arbitrary `.init()` function to your _Module_ if the +initialization code is complex. ### Compiling our Module for the accelerator We're only going to use the CPU for our simple model, but we need to compile the -`forward()` function nonetheless. This compilation is usually done -asynchronously. That means, we can continue doing other things while the module -is compiling: +`forward()` function nonetheless: ```zig -// A clone of our model, consisting of shapes. We only need shapes for compiling. -// We use the BufferStore to infer the shapes. -const model_shapes = try zml.aio.populateModel(Layer, allocator, bs); +// Our computation require an input tensor +const input: zml.Tensor = .init(.{3}, .f16); -// Start compiling. This uses the inferred shapes from the BufferStore. -// The shape of the input tensor, we have to pass in manually. -var compilation = try async_( - zml.compileModel, - .{ allocator, model_shapes, .forward, .{input_shape}, platform }, -); - -// Produce a bufferized weights struct from the fake BufferStore. -// This is like the inferred shapes, but with actual values. -// We will need to send those to the computation device later. -var model_weights = try zml.aio.loadBuffers(Layer, .{}, bs, arena, platform); -defer zml.aio.unloadBuffers(&model_weights); // for good practice - -// Wait for compilation to finish -const compiled = try compilation.await_(); +var executable = try platform.compile(allocator, io, layer, .forward, .{input}, .{}); +defer executable.deinit(); ``` -Compiling is happening in the background via the `async_` function. We call -`async_` with the `zml.compileModel` function and its arguments -separately. The arguments themselves are basically the shapes of the weights in -the BufferStore, the `.forward` function name in order to compile -`Layer.forward`, the shape of the input tensor(s), and the platform for which to -compile (we used auto platform). +You might wonder what this `sharding` variable is for ? ZML supports sharding tensors across multiple devices, +and the `compile()` function needs to know how the tensors are sharded in order to compile the module correctly. +For this simple example, we just replicate the tensors across all devices for simplicity. +### Creating the "bufferized" Model -### Creating the Executable Model - -Now that we have compiled the module utilizing the shapes, we turn it into an -executable. +Now that we have compiled the module, we can take care of loading the actual tensor data. +In our case, we'll create the _Buffers_ from the raw data directly (but keep in mind that ZML +provides ways to have extra-low loading times). ```zig -// pass the model weights to the compiled module to create an executable module -var executable = try compiled.prepare(arena, model_weights); -defer executable.deinit(); + +const weight_slice: zml.Slice = .init(layer.weight.shape(), std.mem.sliceAsBytes(&[3]f16{1.0, 2.0, 3.0})); +const bias_slice: zml.Slice = .init(layer.bias.?.shape(), std.mem.sliceAsBytes(&[3]f16{1.0, 1.0, 1.0})); +var layer_buffers: zml.Bufferized(Layer) = .{ + .weight = try zml.Buffer.fromSlice(io, platform, weight_slice, .replicated), + .bias = try zml.Buffer.fromSlice(io, platform, bias_slice, .replicated), +}; +defer layer_buffers.weight.deinit(); +defer layer_buffers.bias.?.deinit(); ``` ### Calling / running the Model -The executable can now be invoked with an input of our choice. +Before being able to run our _Executable_ there is still two steps: +- create the input buffer +- create and fill the _Args_ and _Results_ structs -To create the `input`, we directly use `zml.Buffer` by calling -`zml.Buffer.fromArray()`. It's important to note that `Buffer`s reside in -_accelerator_ (or _device_) memory, which is precisely where the input needs to -be for the executable to process it on the device. - -For clarity, let's recap the distinction: `HostBuffer`s are located in standard -_host_ memory, which is accessible by the CPU. When we initialized the weights, -we used `HostBuffers` to set up the `BufferStore`. This is because the -`BufferStore` typically loads weights from disk into `HostBuffer`s, and then -converts them into `Buffer`s when we call `loadBuffers()`. - -However, for inputs, we bypass the `BufferStore` and create `Buffer`s directly -in device memory. +To create the `input` we already know how to do it, as we did it for the _Module_ +buffers. +To create the _Args_ and _Results_, the _Executable_ exposes two conveniences. ```zig -// prepare an input buffer -// Here, we use zml.HostBuffer.fromSlice to show how you would create a -// HostBuffer with a specific shape from an array. -// For situations where e.g. you have an [4]f16 array but need a .{2, 2} input -// shape. -var input = [3]f16{ 5.0, 5.0, 5.0 }; -var input_buffer = try zml.Buffer.from( - platform, - zml.HostBuffer.fromSlice(input_shape, &input), -); +// create the input buffer +const input_slice: zml.Slice = .init(input.shape(), std.mem.sliceAsBytes(&[3]f16{ 5.0, 5.0, 5.0 })); +var input_buffer: zml.Buffer = try .fromSlice(io, platform, input_slice, .replicated); defer input_buffer.deinit(); -// call our executable module -var result: zml.Buffer = executable.call(.{input_buffer}); +// create the Args and Results structs +var args = try executable.args(allocator); +defer args.deinit(allocator); + +var results = try executable.results(allocator); +defer results.deinit(allocator); + +// fill the Args +args.set(.{ layer_buffers, input_buffer }); + +// call our executable +executable.call(io, args, &results, .{}); + +// Retrieve the resulting buffer +var result = results.get(zml.Buffer); defer result.deinit(); // fetch the result buffer to CPU memory -const cpu_result = try result.toHostAlloc(arena); +const result_slice = try result.toSliceAlloc(allocator, io); +defer result_slice.free(allocator); + std.debug.print( "\n\nThe result of {d} * {d} + {d} = {d}\n", - .{ &weights, &input, &bias, cpu_result.items(f16) }, + .{ weight_slice, input_slice, bias_slice, result_slice }, ); ``` Note that the result of a computation is usually residing in the memory of the -computation device, so with `.toHostAlloc()` we bring it back to CPU memory in -the form of a `HostBuffer`. After that, we can print it. +computation device, so with `.toSliceAlloc()` we bring it back to CPU memory in +the form of a `Slice`. After that, we can print it. -In order to print it, we need to tell the host buffer how to interpret the -memory. We do that by calling `.items(f16)`, making it cast the memory to `f16` -items. +`call(io, args, &results, .{})` enqueues execution and returns immediately. Use +`call(io, args, &results, .{ .wait = true })` to wait for execution to finish before returning. +Here, `toSliceAlloc()` waits for the result before reading it. + +Raw calls leave input buffer handles owned by the caller, including inputs donated +with `reuseBuffer()`. Deinit their original handles after calling. Runners and +`eval` destroy donated input handles automatically. With runners, use output +destinations pointing to those buffers to replace them with the returned outputs. And that's it! Now, let's have a look at building and actually running this example! @@ -351,26 +268,24 @@ As mentioned already, ZML uses Bazel; so to build our model, we just need to create a simple `BUILD.bazel` file, next to the `main.zig` file, like this: ```python -load("@zml//bazel:zig.bzl", "zig_cc_binary") +load("@rules_zig//zig:defs.bzl", "zig_binary") -zig_cc_binary( +zig_binary( name = "simple_layer", main = "main.zig", deps = [ - "@zml//async", "@zml//zml", ], ) ``` -To produce an executable, we import `zig_cc_binary` from the zig rules, and -pass it a name and the zig file we just wrote. The dependencies in `deps` are -what's needed for a basic ZML executable and correlate with our imports at the +To produce an executable, we import `zig_binary` from the zig rules, and +pass it a name and the zig file we just wrote. The dependency in `deps` is +what's needed for a basic ZML executable and correlate with our import at the top of the Zig file: ```zig const zml = @import("zml"); -const asynk = @import("async"); ``` @@ -379,48 +294,48 @@ const asynk = @import("async"); With everything in place now, running the model is easy: ``` -# run release (-c opt) -cd examples -bazel run -c opt //simple_layer +# run release (--config=release) +bazel run --config=release //examples/simple_layer # compile and run debug version -bazel run //simple_layer +bazel run //examples/simple_layer ``` And voila! Here's the output: ``` -bazel run -c opt //simple_layer -INFO: Analyzed target //simple_layer:simple_layer (0 packages loaded, 0 targets configured). +bazel run --config=release //examples/simple_layer +INFO: Analyzed target //examples/simple_layer:simple_layer (0 packages loaded, 0 targets configured). INFO: Found 1 target... -Target //simple_layer:simple_layer up-to-date: - bazel-bin/simple_layer/simple_layer +Target //examples/simple_layer:simple_layer up-to-date: + bazel-bin/examples/simple_layer/simple_layer INFO: Elapsed time: 0.120s, Critical Path: 0.00s INFO: 1 process: 1 internal. INFO: Build completed successfully, 1 total action -INFO: Running command line: bazel-bin/simple_layer/simple_layer -info(pjrt): Loaded library: libpjrt_cpu.dylib -info(zml_module): Compiling main.Layer.forward with { Shape({3}, dtype=.f16) } +INFO: Running command line: bazel-bin/examples/simple_layer/simple_layer +info(pjrt): Loaded library: libzml_cpu.dylib +debug(zml/module): +******** ZML generated MLIR ******** +module @zml attributes {mhlo.num_partitions = 4 : i32, mhlo.num_replicas = 1 : i32} { + sdy.mesh @replicated = <["bus"=4]> + func.func @main(%arg0: tensor<3xf16> {sdy.sharding = #sdy.sharding<@replicated, [{}], replicated={"bus"}>}, %arg1: tensor<3xf16> {sdy.sharding = #sdy.sharding<@replicated, [{}], replicated={"bus"}>}, %arg2: tensor<3xf16> {sdy.sharding = #sdy.sharding<@replicated, [{}], replicated={"bus"}>}) -> (tensor<3xf16> {sdy.sharding = #sdy.sharding<@replicated, [{}], replicated={"bus"}>}) { + %0 = stablehlo.multiply %arg1, %arg2 : tensor<3xf16> + %1 = stablehlo.add %0, %arg0 : tensor<3xf16> + return %1 : tensor<3xf16> + } +} -The result of { 2, 2, 2 } * { 5, 5, 5 } + { 1, 2, 3 } = { 11, 12, 13 } + + +The result of {1,2,3} * {5,5,5} + {1,1,1} = {6,11,16} ``` ---- - -You can access the complete source code of this walkthrough here: - -- [main.zig](https://github.com/zml/zml/tree/master/examples/simple_layer/main.zig) -- [BUILD.bazel](https://github.com/zml/zml/tree/master/examples/simple_layer/BUILD.bazel) - ## The complete example ```zig const std = @import("std"); const zml = @import("zml"); -const asynk = @import("async"); - -const async_ = asynk.async_; /// Model definition const Layer = struct { @@ -436,99 +351,68 @@ const Layer = struct { } }; -pub fn main() !void { - var gpa = std.heap.GeneralPurposeAllocator(.{}){}; - defer _ = gpa.deinit(); - try asynk.AsyncThread.main(gpa.allocator(), asyncMain, .{}); -} +pub fn main(init: std.process.Init) !void { + const allocator = init.gpa; + const io = init.io; -pub fn asyncMain() !void { - // Short lived allocations - var gpa = std.heap.GeneralPurposeAllocator(.{}){}; - defer _ = gpa.deinit(); - const allocator = gpa.allocator(); + var platform: *zml.Platform = try .auto(allocator, io, .{}); + defer platform.deinit(allocator, io); - // Arena allocator for BufferStore etc. - var arena_state = std.heap.ArenaAllocator.init(allocator); - defer arena_state.deinit(); - const arena = arena_state.allocator(); - - // Create ZML context - var context = try zml.Context.init(); - defer context.deinit(); - - const platform = context.autoPlatform(); - - // Our weights and bias to use - var weights = [3]f16{ 2.0, 2.0, 2.0 }; - var bias = [3]f16{ 1.0, 2.0, 3.0 }; - const input_shape = zml.Shape.init(.{3}, .f16); - - // We manually produce a BufferStore. You would not normally do that. - // A BufferStore is usually created by loading model data from a file. - var buffers: zml.aio.BufferStore.Buffers = .{}; - try buffers.put(arena, "weight", zml.HostBuffer.fromArray(&weights)); - try buffers.put(arena, "bias", zml.HostBuffer.fromArray(&bias)); - - // the actual BufferStore - const bs: zml.aio.BufferStore = .{ - .arena = arena_state, - .buffers = buffers, + const layer: Layer = .{ + .weight = zml.Tensor.init(.{3}, .f16), + .bias = zml.Tensor.init(.{3}, .f16), }; - // A clone of our model, consisting of shapes. We only need shapes for - // compiling. We use the BufferStore to infer the shapes. - const model_shapes = try zml.aio.populateModel(Layer, allocator, bs); + // Our computation require an input tensor + const input: zml.Tensor = .init(.{3}, .f16); - // Start compiling. This uses the inferred shapes from the BufferStore. - // The shape of the input tensor, we have to pass in manually. - var compilation = try async_( - zml.compileModel, - .{ allocator, model_shapes, .forward, .{input_shape}, platform }, - ); - - // Produce a bufferized weights struct from the fake BufferStore. - // This is like the inferred shapes, but with actual values. - // We will need to send those to the computation device later. - var model_weights = try zml.aio.loadBuffers(Layer, .{}, bs, arena, platform); - defer zml.aio.unloadBuffers(&model_weights); // for good practice - - // Wait for compilation to finish - const compiled = try compilation.await_(); - - // pass the model weights to the compiled module to create an executable - // module - var executable = try compiled.prepare(arena, model_weights); + var executable = try platform.compile(allocator, io, layer, .forward, .{input}, .{}); defer executable.deinit(); - // prepare an input buffer - // Here, we use zml.HostBuffer.fromSlice to show how you would create a - // HostBuffer with a specific shape from an array. - // For situations where e.g. you have an [4]f16 array but need a .{2, 2} - // input shape. - var input = [3]f16{ 5.0, 5.0, 5.0 }; - var input_buffer = try zml.Buffer.from( - platform, - zml.HostBuffer.fromSlice(input_shape, &input), - ); + const weight_slice: zml.Slice = .init(layer.weight.shape(), std.mem.sliceAsBytes(&[3]f16{ 1.0, 2.0, 3.0 })); + const bias_slice: zml.Slice = .init(layer.bias.?.shape(), std.mem.sliceAsBytes(&[3]f16{ 1.0, 1.0, 1.0 })); + var layer_buffers: zml.Bufferized(Layer) = .{ + .weight = try zml.Buffer.fromSlice(io, platform, weight_slice, .replicated), + .bias = try zml.Buffer.fromSlice(io, platform, bias_slice, .replicated), + }; + defer layer_buffers.weight.deinit(); + defer layer_buffers.bias.?.deinit(); + + // create the input buffer + const input_slice: zml.Slice = .init(input.shape(), std.mem.sliceAsBytes(&[3]f16{ 5.0, 5.0, 5.0 })); + var input_buffer: zml.Buffer = try .fromSlice(io, platform, input_slice, .replicated); defer input_buffer.deinit(); - // call our executable module - var result: zml.Buffer = executable.call(.{input_buffer}); + // create the Args and Results structs + var args = try executable.args(allocator); + defer args.deinit(allocator); + + var results = try executable.results(allocator); + defer results.deinit(allocator); + + // fill the Args + args.set(.{ layer_buffers, input_buffer }); + + // call our executable + executable.call(io, args, &results, .{}); + + // Retrieve the resulting buffer + var result = results.get(zml.Buffer); defer result.deinit(); - // fetch the result to CPU memory - const cpu_result = try result.toHostAlloc(arena); + // fetch the result buffer to CPU memory + const result_slice = try result.toSliceAlloc(allocator, io); + defer result_slice.free(allocator); + std.debug.print( "\n\nThe result of {d} * {d} + {d} = {d}\n", - .{ &weights, &input, &bias, cpu_result.items(f16) }, + .{ weight_slice, input_slice, bias_slice, result_slice }, ); } ``` ## Where to go from here -- [Add some weights files to your model](../howtos/add_weights.md) - [Run the model on GPU](../tutorials/getting_started.md) - [Deploy the model on a server](../howtos/deploy_on_server.md) - [Dockerize this model](../howtos/dockerize_models.md) diff --git a/dsl/common/BUILD.bazel b/dsl/common/BUILD.bazel new file mode 100644 index 00000000..1476dcfd --- /dev/null +++ b/dsl/common/BUILD.bazel @@ -0,0 +1,21 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "builder", + srcs = [ + "args.zig", + "control_flow.zig", + ], + import_name = "dsl/common/builder", + main = "builder.zig", + visibility = ["//visibility:public"], + deps = [ + "//mlir", + "//mlir/dialects/scf", + ], +) + +zig_test( + name = "test", + deps = [":builder"], +) diff --git a/dsl/common/args.zig b/dsl/common/args.zig new file mode 100644 index 00000000..84d57d49 --- /dev/null +++ b/dsl/common/args.zig @@ -0,0 +1,4 @@ +pub fn NamedArgs(comptime Spec: type, comptime ValueT: type) type { + const field_names = @typeInfo(Spec).@"struct".field_names; + return @Struct(.auto, null, field_names, &@splat(ValueT), &@splat(.{})); +} diff --git a/dsl/common/builder.zig b/dsl/common/builder.zig new file mode 100644 index 00000000..1addd259 --- /dev/null +++ b/dsl/common/builder.zig @@ -0,0 +1,35 @@ +const std = @import("std"); + +const mlir = @import("mlir"); +const scf = @import("mlir/dialects/scf"); + +pub const control_flow = @import("control_flow.zig"); +pub const args = @import("args.zig"); + +pub const arity = control_flow.arity; +pub const BlockStack = control_flow.BlockStack; +pub const forBounds = control_flow.forBounds; +pub const ForOpts = scf.ForOpts; +pub const openFor = control_flow.openFor; +pub const openIf = control_flow.openIf; +pub const openIfElse = control_flow.openIfElse; +pub const openWhile = control_flow.openWhile; +pub const innerValues = control_flow.innerValues; +pub const resultTypes = control_flow.resultTypes; +pub const ForScope = control_flow.ForScope; +pub const IfOnlyScope = control_flow.IfOnlyScope; +pub const IfScope = control_flow.IfScope; +pub const WhileScope = control_flow.WhileScope; + +pub const NamedArgs = args.NamedArgs; + +/// Verifies `module`, logging it when it fails. +pub fn verify(module: *mlir.Module, comptime target: []const u8) error{InvalidMlir}!void { + if (module.operation().verify()) return; + std.log.err(target ++ ": the module does not verify:\n{f}", .{module.operation().fmt(.{ .debug_info = false })}); + return error.InvalidMlir; +} + +test { + std.testing.refAllDecls(@This()); +} diff --git a/dsl/common/control_flow.zig b/dsl/common/control_flow.zig new file mode 100644 index 00000000..e557e24b --- /dev/null +++ b/dsl/common/control_flow.zig @@ -0,0 +1,364 @@ +//! `scf` scope wrappers, generic over the frontend's `BuilderT` and `ValueT`, +//! and `BlockStack`, the blocks a builder emits into. +//! +//! `BuilderT` must expose `ctx`, `loc()`, `pushBlock`, `popBlock` and +//! `currentBlock`. The openers also call its `lift` on literal inits and its +//! `scalarTy` on dtype result types, and `forBounds` its `constantLike` on +//! literal bounds. +//! `ValueT` must have an `inner: *const mlir.Value` field and a `kernel: ?*BuilderT` field. + +const std = @import("std"); +const mlir = @import("mlir"); +const scf = @import("mlir/dialects/scf"); + +/// The blocks a builder emits into: the function's entry block at the bottom, +/// the innermost scope's on top. +pub const BlockStack = struct { + blocks: std.ArrayList(*mlir.Block) = .empty, + + pub fn push(self: *BlockStack, allocator: std.mem.Allocator, block: *mlir.Block) void { + self.blocks.append(allocator, block) catch @panic("BlockStack.push: out of memory"); + } + + /// Closes the innermost scope. + pub fn pop(self: *BlockStack) void { + if (self.blocks.items.len < 2) @panic("BlockStack.pop: no scope open"); // the entry block always stays + _ = self.blocks.pop(); + } + + /// Drops every block, keeping the memory for the next function. + pub fn clear(self: *BlockStack) void { + self.blocks.clearRetainingCapacity(); + } + + /// Returns the top block. + pub fn current(self: *const BlockStack) *mlir.Block { + return self.blocks.getLastOrNull() orelse @panic("BlockStack.current: no block, call declareArgs first"); + } + + /// Replaces the top block with `block`. + pub fn replaceTop(self: *BlockStack, block: *mlir.Block) void { + if (self.blocks.items.len == 0) @panic("BlockStack.replaceTop: no block, call declareArgs first"); + self.blocks.items[self.blocks.items.len - 1] = block; + } +}; + +/// The length of `T`, a tuple like `.{ v1, v2 }` or an array. +pub fn arity(comptime T: type, comptime what: []const u8) comptime_int { + return switch (@typeInfo(T)) { + .array => |a| a.len, + .@"struct" => |s| if (s.is_tuple) s.field_names.len else @compileError(what ++ " must be a tuple like `.{ v1, v2 }` or an array"), + else => @compileError(what ++ " must be a tuple like `.{ v1, v2 }` or an array"), + }; +} + +/// The MLIR values of `values`, `N` `ValueT`s. +pub fn innerValues(comptime ValueT: type, comptime N: usize, values: anytype, comptime what: []const u8) [N]*const mlir.Value { + if (arity(@TypeOf(values), what) != N) + @compileError(what ++ ": the arity must match the scope's"); + var buf: [N]*const mlir.Value = undefined; + inline for (0..N) |i| { + if (@TypeOf(values[i]) != ValueT) @compileError(what ++ ": every element must be a Value"); + buf[i] = values[i].inner; + } + return buf; +} + +/// `values` as `ValueT`s: values as they are, anything else through `kernel.lift`. +fn liftValues(kernel: anytype, comptime ValueT: type, values: anytype, comptime what: []const u8) [arity(@TypeOf(values), what)]ValueT { + var out: [arity(@TypeOf(values), what)]ValueT = undefined; + inline for (0..out.len) |i| out[i] = if (@TypeOf(values[i]) == ValueT) values[i] else kernel.lift(values[i]); + return out; +} + +/// The types `types` names: `*const mlir.Type`s, `ValueT`s for their types, +/// or dtypes (`.f32`) for `kernel.scalarTy`. +pub fn resultTypes(kernel: anytype, comptime ValueT: type, types: anytype, comptime what: []const u8) [arity(@TypeOf(types), what)]*const mlir.Type { + var out: [arity(@TypeOf(types), what)]*const mlir.Type = undefined; + inline for (0..out.len) |i| { + const T = @TypeOf(types[i]); + out[i] = if (T == *const mlir.Type) types[i] else if (T == ValueT) types[i].inner.type_() else kernel.scalarTy(types[i]); + } + return out; +} + +fn scfYield(comptime ValueT: type, comptime N: usize, kernel: anytype, block: *mlir.Block, values: anytype, comptime what: []const u8) void { + const buf = innerValues(ValueT, N, values, what); + _ = scf.yield(kernel.ctx, &buf, kernel.loc()).appendTo(block); +} + +pub fn ForScope(comptime BuilderT: type, comptime ValueT: type, comptime N: usize) type { + return struct { + kernel: *BuilderT, + body: *mlir.Block, + lb_inner: *const mlir.Value, + ub_inner: *const mlir.Value, + step_inner: *const mlir.Value, + inits_inner: [N]*const mlir.Value, + iv: ValueT, + carried: [N]ValueT, + results: [N]ValueT = undefined, + opts: scf.ForOpts = .{}, + + const Self = @This(); + + pub fn yield(self: *Self, values: anytype) void { + if (N == 0) @compileError("ForScope.yield: the loop carries nothing, end it with close()"); + self.end(values); + } + + pub fn close(self: *Self) void { + if (N != 0) @compileError("ForScope.close: the loop carries values, end it with yield()"); + self.end(.{}); + } + + fn end(self: *Self, values: anytype) void { + const k = self.kernel; + scfYield(ValueT, N, k, self.body, values, "ForScope.yield"); + k.popBlock(); + const for_op = scf.for_( + k.ctx, + self.lb_inner, + self.ub_inner, + self.step_inner, + &self.inits_inner, + self.body, + self.opts, + k.loc(), + ); + _ = for_op.appendTo(k.currentBlock()); + for (0..N) |i| self.results[i] = .{ .inner = for_op.result(i), .kernel = k }; + } + }; +} + +pub fn IfOnlyScope(comptime BuilderT: type, comptime ValueT: type) type { + return struct { + kernel: *BuilderT, + cond_inner: *const mlir.Value, + then_block: *mlir.Block, + + const Self = @This(); + + pub fn close(self: *Self) void { + const k = self.kernel; + scfYield(ValueT, 0, k, self.then_block, .{}, "IfOnlyScope.close"); + k.popBlock(); + const if_op = scf.if_(k.ctx, self.cond_inner, &.{}, self.then_block, .{}, k.loc()); + _ = if_op.appendTo(k.currentBlock()); + } + }; +} + +pub fn IfScope(comptime BuilderT: type, comptime ValueT: type, comptime N: usize) type { + return struct { + kernel: *BuilderT, + cond_inner: *const mlir.Value, + then_block: *mlir.Block, + else_block: *mlir.Block, + result_types: [N]*const mlir.Type, + results: [N]ValueT = undefined, + + const Self = @This(); + + pub fn yieldThen(self: *Self, values: anytype) void { + if (N == 0) @compileError("IfScope.yieldThen: the if has no results, switch branches with elseBranch()"); + self.endThen(values); + } + + pub fn yieldElse(self: *Self, values: anytype) void { + if (N == 0) @compileError("IfScope.yieldElse: the if has no results, end it with close()"); + self.endElse(values); + } + + pub fn elseBranch(self: *Self) void { + if (N != 0) @compileError("IfScope.elseBranch: the if has results, switch branches with yieldThen()"); + self.endThen(.{}); + } + + pub fn close(self: *Self) void { + if (N != 0) @compileError("IfScope.close: the if has results, end it with yieldElse()"); + self.endElse(.{}); + } + + fn endThen(self: *Self, values: anytype) void { + const k = self.kernel; + scfYield(ValueT, N, k, self.then_block, values, "IfScope.yieldThen"); + k.popBlock(); + k.pushBlock(self.else_block); + } + + fn endElse(self: *Self, values: anytype) void { + const k = self.kernel; + scfYield(ValueT, N, k, self.else_block, values, "IfScope.yieldElse"); + k.popBlock(); + const if_op = scf.if_(k.ctx, self.cond_inner, &self.result_types, self.then_block, .{ .else_block = self.else_block }, k.loc()); + _ = if_op.appendTo(k.currentBlock()); + for (0..N) |i| self.results[i] = .{ .inner = if_op.result(i), .kernel = k }; + } + }; +} + +pub fn WhileScope(comptime BuilderT: type, comptime ValueT: type, comptime N: usize, comptime M: usize) type { + return struct { + kernel: *BuilderT, + before_block: *mlir.Block, + after_block: *mlir.Block, + inits_inner: [N]*const mlir.Value, + after_types: [M]*const mlir.Type, + before_carried: [N]ValueT, + after_carried: [M]ValueT = undefined, + results: [M]ValueT = undefined, + + const Self = @This(); + + pub fn yieldBefore(self: *Self, cond: ValueT, forwarded: anytype) void { + const k = self.kernel; + const buf = innerValues(ValueT, M, forwarded, "WhileScope.yieldBefore"); + _ = scf.condition(k.ctx, cond.inner, &buf, k.loc()).appendTo(self.before_block); + k.popBlock(); + k.pushBlock(self.after_block); + for (0..M) |i| self.after_carried[i] = .{ .inner = self.after_block.argument(i), .kernel = k }; + } + + pub fn yieldAfter(self: *Self, values: anytype) void { + const k = self.kernel; + scfYield(ValueT, N, k, self.after_block, values, "WhileScope.yieldAfter"); + k.popBlock(); + const w = scf.while_( + k.ctx, + &self.inits_inner, + &self.after_types, + self.before_block, + self.after_block, + k.loc(), + ); + _ = w.appendTo(k.currentBlock()); + for (0..M) |i| self.results[i] = .{ .inner = w.result(i), .kernel = k }; + } + }; +} + +/// `openFor`'s bounds as values. A bound that isn't a value takes the type of +/// the first one that is (`kernel.constantLike`); with none, `lower` becomes +/// `defaultLower(kernel, lower)` and the others take its type. +pub fn forBounds( + kernel: anytype, + comptime ValueT: type, + lower: anytype, + upper: anytype, + step: anytype, + comptime defaultLower: anytype, +) [3]ValueT { + const bounds: [3]ValueT = if (@TypeOf(lower) != ValueT and @TypeOf(upper) != ValueT and @TypeOf(step) != ValueT) blk: { + const lb = defaultLower(kernel, lower); + break :blk .{ lb, kernel.constantLike(upper, lb), kernel.constantLike(step, lb) }; + } else blk: { + const first: ValueT = if (@TypeOf(lower) == ValueT) lower else if (@TypeOf(upper) == ValueT) upper else step; + break :blk .{ kernel.constantLike(lower, first), kernel.constantLike(upper, first), kernel.constantLike(step, first) }; + }; + const t = bounds[0].inner.type_(); + if (!bounds[1].inner.type_().eql(t) or !bounds[2].inner.type_().eql(t)) + std.debug.panic("openFor: bounds of types {f}, {f} and {f}", .{ t, bounds[1].inner.type_(), bounds[2].inner.type_() }); + return bounds; +} + +/// Opens `for iv in [bounds[0], bounds[1]) step bounds[2]` carrying `inits`, +/// values or literals `kernel.lift` takes. +pub fn openFor( + kernel: anytype, + comptime ValueT: type, + bounds: [3]ValueT, + inits: anytype, + opts: scf.ForOpts, +) ForScope(@TypeOf(kernel.*), ValueT, arity(@TypeOf(inits), "openFor: inits")) { + const N = comptime arity(@TypeOf(inits), "openFor: inits"); + const values = liftValues(kernel, ValueT, inits, "openFor: inits"); + var types: [N + 1]*const mlir.Type = undefined; + types[0] = bounds[0].inner.type_(); + var inits_inner: [N]*const mlir.Value = undefined; + for (values, 0..) |v, i| { + types[i + 1] = v.inner.type_(); + inits_inner[i] = v.inner; + } + const locs: [N + 1]*const mlir.Location = @splat(kernel.loc()); + const body = mlir.Block.init(&types, &locs); + kernel.pushBlock(body); + var carried: [N]ValueT = undefined; + for (&carried, 0..) |*c, i| c.* = .{ .inner = body.argument(i + 1), .kernel = kernel }; + return .{ + .kernel = kernel, + .body = body, + .lb_inner = bounds[0].inner, + .ub_inner = bounds[1].inner, + .step_inner = bounds[2].inner, + .inits_inner = inits_inner, + .iv = .{ .inner = body.argument(0), .kernel = kernel }, + .carried = carried, + .opts = opts, + }; +} + +pub fn openIf(kernel: anytype, comptime ValueT: type, cond: ValueT) IfOnlyScope(@TypeOf(kernel.*), ValueT) { + const then_block = mlir.Block.init(&.{}, &.{}); + kernel.pushBlock(then_block); + return .{ .kernel = kernel, .cond_inner = cond.inner, .then_block = then_block }; +} + +/// Opens an `if` with an `else`, of results of `result_types` (see `resultTypes`). +pub fn openIfElse( + kernel: anytype, + comptime ValueT: type, + cond: ValueT, + result_types: anytype, +) IfScope(@TypeOf(kernel.*), ValueT, arity(@TypeOf(result_types), "openIfElse: result_types")) { + const then_block = mlir.Block.init(&.{}, &.{}); + const else_block = mlir.Block.init(&.{}, &.{}); + kernel.pushBlock(then_block); + return .{ + .kernel = kernel, + .cond_inner = cond.inner, + .then_block = then_block, + .else_block = else_block, + .result_types = resultTypes(kernel, ValueT, result_types, "openIfElse: result_types"), + }; +} + +/// Opens an `scf.while` carrying `inits` into its condition region, which +/// forwards values of `after_types` (see `resultTypes`) to its body. +pub fn openWhile( + kernel: anytype, + comptime ValueT: type, + inits: anytype, + after_types: anytype, +) WhileScope(@TypeOf(kernel.*), ValueT, arity(@TypeOf(inits), "openWhile: inits"), arity(@TypeOf(after_types), "openWhile: after_types")) { + const N = comptime arity(@TypeOf(inits), "openWhile: inits"); + const M = comptime arity(@TypeOf(after_types), "openWhile: after_types"); + const values = liftValues(kernel, ValueT, inits, "openWhile: inits"); + var before_types: [N]*const mlir.Type = undefined; + var inits_inner: [N]*const mlir.Value = undefined; + for (values, 0..) |v, i| { + before_types[i] = v.inner.type_(); + inits_inner[i] = v.inner; + } + const before_locs: [N]*const mlir.Location = @splat(kernel.loc()); + const before_block = mlir.Block.init(&before_types, &before_locs); + const after_types_ = resultTypes(kernel, ValueT, after_types, "openWhile: after_types"); + const after_locs: [M]*const mlir.Location = @splat(kernel.loc()); + const after_block = mlir.Block.init(&after_types_, &after_locs); + kernel.pushBlock(before_block); + var before_carried: [N]ValueT = undefined; + for (&before_carried, 0..) |*c, i| c.* = .{ .inner = before_block.argument(i), .kernel = kernel }; + return .{ + .kernel = kernel, + .before_block = before_block, + .after_block = after_block, + .inits_inner = inits_inner, + .after_types = after_types_, + .before_carried = before_carried, + }; +} + +test { + std.testing.refAllDecls(@This()); +} diff --git a/dsl/cuda_tile/BUILD.bazel b/dsl/cuda_tile/BUILD.bazel new file mode 100644 index 00000000..527c53e4 --- /dev/null +++ b/dsl/cuda_tile/BUILD.bazel @@ -0,0 +1,24 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "builder", + srcs = [ + "control_flow.zig", + "dtype.zig", + ], + import_name = "dsl/cuda_tile/builder", + main = "builder.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//dsl/common:builder", + "//mlir", + "//mlir/dialects/cuda_tile", + "//stdx", + ], +) + +zig_test( + name = "test", + deps = [":builder"], +) diff --git a/dsl/cuda_tile/README.md b/dsl/cuda_tile/README.md new file mode 100644 index 00000000..d8c5fdb8 --- /dev/null +++ b/dsl/cuda_tile/README.md @@ -0,0 +1,137 @@ +# CUDA Tile IR kernels in Zig + +`dsl/cuda_tile` is a Zig DSL that emits NVIDIA's `cuda_tile` MLIR dialect +(CUDA Tile IR). It is the third instance of the `dsl/triton` / +`dsl/mosaic_tpu` pattern: a `Builder` that owns an MLIR module and appends +ops to a block stack, a `Value` with fluent arithmetic, and a +`zml.kernel.cuda_tile.Kernel` wrapper that turns a `run` function into a +`stablehlo.custom_call` XLA compiles with `tileiras`. + +The dialect binding lives in `mlir/dialects/cuda_tile`: one function per op +(all 100 of them, in `Ops.td` order), every type through the dialect's C API, +every enum attribute through its C getter. + +## A kernel + +```zig +const zml = @import("zml"); +const cut = zml.kernel.cuda_tile; + +const Cfg = struct { n: i64 }; + +pub const AddOne = cut.Kernel(Cfg, .{ + .name = "add_one", + .inputs = &.{"x"}, + .outputs = &.{"out"}, + .run = run, +}); + +fn run(b: *cut.Builder, cfg: Cfg) cut.FinishError!void { + // Every parameter is a rank-0 tile>; views are built inside. + const a = try b.declareArgsOpts(.{ + .x = .{ .ptr = .f32 }, + .out = .{ .ptr = .f32 }, + }, .{ .hints = &.{.{ .arch = .sm_120, .num_worker_warps_per_cta = 4 }} }); + + // pointer -> tensor_view -> partition_view; the view pads the ragged edge. + const vx = b.partitionView(b.tensorView(a.x, &.{cfg.n}, &.{1}), &.{128}, .{}); + const vo = b.partitionView(b.tensorView(a.out, &.{cfg.n}, &.{1}), &.{128}, .{}); + + const bid = b.tileBlockId(); + _ = b.store(b.load(vx, &.{bid.x}).add(1.0), vo, &.{bid.x}); +} + +// In a model: +const out = AddOne.call(.{ .x = x }, .{ .out = x.shape() }, .{ + .cfg = .{ .n = 1024 }, + .grid = .{ 8, 1, 1 }, +}).out; +``` + +Put each scope's body in a Zig `{ ... }` block right after `openFor`, `openIf` +or `openLoop`, ending with its `yield` (or `close()`). `openIfElse` gets one +block per branch, ending with `yieldThen`/`yieldElse` (`elseBranch()`/`close()` +without results). + +## What is different from the Triton DSL + +* **Rank 0 is the scalar.** Every value is a `tile<...>`; `entry` parameters + must be rank 0, so buffers arrive as `tile>`. +* **Memory is a view stack.** `tensorView` → `partitionView` (or + `stridedView`, `gatherScatterView`) → tile-space `load`/`store`. A partial + edge tile reads the view's `padding` and its store is masked; there is no + mask tile. The pointer-tile arm (`loadPtr`/`storePtr` with `offset`) exists + for gathers but cannot use TMA. +* **Every load and store returns a token.** The dialect has no barrier or + fence; `loadOpts`/`storeOpts` take and return tokens, `load`/`store` drop + them. +* **Float ops carry a rounding mode.** `addf` defaults to `nearest_even`; + `addfOpts` sets it. Integer ops carry none. +* **Loop bodies end with `continue`, `if` bodies with `yield`, and `loop` + exits with `break`** — `ForScope`, `IfScope`, `LoopScope` do this for you. +* **No `num_warps`/`num_stages`.** Warp and CTA tuning is + `optimization_hints` on the entry, per architecture (`Opts.hints`), and + `allow_tma`/`latency` on view loads and stores (`LoadOpts.hints`). +* **Tile dims are powers of two**, at most 2^24 elements; `tileTy` and + `partitionView` check this in Zig. +* **Some conversions are one-way.** `ftof` to `f8e8m0fnu` (the MX scale + type) rounds toward zero by default (`CastOpts.rounding = .positive_inf` + is the other legal choice); an integer reaches it through `f32`, which + `cast`/`to` do for you. `ftoi` always truncates toward zero. `tanh` takes + only `approx`/`full`. `Value.cdiv` is integer-only. +* **`i4` is a tile element type, not a pointee.** Declare a 4-bit buffer as + `.i8` and `unpack` it in the kernel. +* **`atomicRedView` is relaxed-only** on an unpadded view and takes no mask; + use the pointer atomics (`atomicRmw`, `atomicCas`) for masked updates. +* **Reductions over several tiles at once** (argmax, an online softmax's + (max, sum)) go through `reduceMulti`/`scanMulti`. +* **A scalar argument is a pointer**: `.{ .n = .{ .ptr = .i32 } }` then + `b.loadPtr(a.n)` gives the rank-0 tile; XLA passes device addresses only. + +## The custom call + +`ops.cudaTile` emits `__gpu$xla.gpu.cuda_tile` with a printed dictionary: +`name`, `kernel_type = "cuda_tile"`, `ir` (the textual module), `grid_x/y/z`, +optional `ir_version`. XLA parses the module, renames the entry, checks the +argument count against the operands + result leaves, scrubs locations, +serializes bytecode, runs `tileiras`, and launches over `grid` tile blocks with +a `(1,1,1)` block and no shared memory — cuda-tile's launch ABI. Every operand +and result is one raw device pointer in the default layout, operands first; +XLA rejects an entry whose parameters are not all `tile>`. + +To see what XLA received, run with `--xla_dump_to=`: it writes +`.cuda_tile.mlir` (post-rename) and `.tilebc` (the exact bytes +handed to `tileiras`). + +## Finding what exists + +There is one function per op and no hidden API, so grep is the index: + +```sh +# every op of the dialect, in Ops.td order (115 constructors) +grep -n '^pub fn ' mlir/dialects/cuda_tile/cuda_tile.zig + +# the types, enums and attribute helpers they take +grep -n '^pub const ' mlir/dialects/cuda_tile/cuda_tile.zig + +# the DSL: Value's fluent methods (add, mul, lt, to, sum, reshape, ...) +awk '/^pub const Value = struct/,/^};/' dsl/cuda_tile/builder.zig | grep 'pub fn ' + +# the DSL: everything on Builder (views, load/store, math, casts, reductions, +# atomics, control flow) -- the `*Opts` variant of a call takes the options +awk '/^pub const Builder = struct/,0' dsl/cuda_tile/builder.zig | grep ' pub fn ' + +# what each option struct holds, and its defaults +grep -n '^pub const [A-Za-z]*Opts = struct' -A 8 dsl/cuda_tile/builder.zig \ + dsl/cuda_tile/control_flow.zig + +# for / if / loop scopes and their terminators +grep -n 'pub fn ' dsl/cuda_tile/control_flow.zig + +# element types +grep -n 'pub const DType = enum' -A 30 dsl/cuda_tile/dtype.zig +``` + +Every op also has a test: `grep -n '^test "' dsl/cuda_tile/builder.zig` +lists them, and each one prints IR that round-trips through the verifier, so +they double as usage examples. diff --git a/dsl/cuda_tile/builder.zig b/dsl/cuda_tile/builder.zig new file mode 100644 index 00000000..a8414acc --- /dev/null +++ b/dsl/cuda_tile/builder.zig @@ -0,0 +1,2218 @@ +const std = @import("std"); + +const common = @import("dsl/common/builder"); +const cuda_tile = @import("mlir/dialects/cuda_tile"); +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +const cf = @import("control_flow.zig"); +const dtypes = @import("dtype.zig"); +pub const DType = dtypes.DType; +const isFloatDtype = dtypes.isFloatDtype; +const dtypeBitwidth = dtypes.dtypeBitwidth; +const intBitwidth = dtypes.intBitwidth; + +pub const RoundingMode = cuda_tile.RoundingMode; +pub const Signedness = cuda_tile.Signedness; +pub const IntegerOverflow = cuda_tile.IntegerOverflow; +pub const ComparisonPredicate = cuda_tile.ComparisonPredicate; +pub const ComparisonOrdering = cuda_tile.ComparisonOrdering; +pub const AtomicRMWMode = cuda_tile.AtomicRMWMode; +pub const MemoryScope = cuda_tile.MemoryScope; +pub const MemoryOrdering = cuda_tile.MemoryOrdering; +pub const PaddingValue = cuda_tile.PaddingValue; +pub const Arch = cuda_tile.Arch; +pub const EntryHint = cuda_tile.EntryHint; +pub const LoadStoreHint = cuda_tile.LoadStoreHint; +pub const PtrLoadStoreHint = cuda_tile.PtrLoadStoreHint; + +test { + std.testing.refAllDecls(@This()); + std.testing.refAllDecls(Builder); + std.testing.refAllDecls(Value); +} + +pub fn registerDialects(registry: *mlir.DialectRegistry) void { + inline for (.{cuda_tile}) |dialect| dialect.registerDialect(registry); +} + +pub const FinishError = error{InvalidMlir} || std.mem.Allocator.Error || std.Io.Writer.Error; + +pub const Value = struct { + inner: *const mlir.Value, + kernel: ?*Builder = null, + + pub const Shape = stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK); + + pub fn type_(self: Value) *const mlir.Type { + return self.inner.type_(); + } + + fn kern(self: Value) *Builder { + return self.kernel orelse @panic("Value has no owning kernel; use Builder.* helpers instead"); + } + + pub fn tile(self: Value) *const cuda_tile.TileType { + return self.type_().isA(cuda_tile.TileType) orelse std.debug.panic("Value {f} is not a tile", .{self.type_()}); + } + + pub fn isTile(self: Value) bool { + return self.type_().isA(cuda_tile.TileType) != null; + } + + pub fn elemType(self: Value) *const mlir.Type { + return self.tile().elementType(); + } + + pub fn rank(self: Value) usize { + return self.tile().rank(); + } + + pub fn isScalar(self: Value) bool { + return self.rank() == 0; + } + + pub fn dim(self: Value, i: usize) i64 { + return self.tile().dimension(i); + } + + pub fn shape(self: Value) Value.Shape { + var out: Value.Shape = .empty; + const t = self.tile(); + for (0..t.rank()) |i| out.appendAssumeCapacity(t.dimension(i)); + return out; + } + + pub fn isFloatElem(self: Value) bool { + const et = self.elemType(); + inline for (comptime std.meta.fieldNames(mlir.FloatTypes)) |f_name| { + if (et.isA(mlir.FloatType(@field(mlir.FloatTypes, f_name))) != null) return true; + } + return false; + } + + pub fn isIntElem(self: Value) bool { + return self.elemType().isA(mlir.IntegerType) != null; + } + + pub fn isPtrElem(self: Value) bool { + return self.elemType().isA(cuda_tile.PointerType) != null; + } + + pub fn add(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return if (l.isFloatElem()) k.addf(l, r) else k.addi(l, r); + } + + pub fn sub(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return if (l.isFloatElem()) k.subf(l, r) else k.subi(l, r); + } + + pub fn mul(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return if (l.isFloatElem()) k.mulf(l, r) else k.muli(l, r); + } + + pub fn div(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return if (l.isFloatElem()) k.divf(l, r) else k.divi(l, r, .signed); + } + + pub fn rem(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return if (l.isFloatElem()) k.remf(l, r) else k.remi(l, r, .signed); + } + + pub fn cdiv(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + if (l.isFloatElem()) @panic("Value.cdiv is integer-only; use div (and ceil) on floats"); + return k.diviOpts(l, r, .{ .signedness = .signed, .rounding = .positive_inf }); + } + + pub fn neg(self: Value) Value { + const k = self.kern(); + return if (self.isFloatElem()) k.negf(self) else k.negi(self); + } + + pub fn abs(self: Value) Value { + const k = self.kern(); + return if (self.isFloatElem()) k.absf(self) else k.absi(self); + } + + pub fn bitAnd(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return k.andi(l, r); + } + + pub fn bitOr(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return k.ori(l, r); + } + + pub fn bitXor(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return k.xori(l, r); + } + + pub fn shl(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return k.shli(l, r); + } + + pub fn shr(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return k.shri(l, r, .signed); + } + + pub fn minimum(self: Value, rhs: anytype) Value { + return self.kern().minimum(self, rhs); + } + + pub fn maximum(self: Value, rhs: anytype) Value { + return self.kern().maximum(self, rhs); + } + + fn compare(self: Value, rhs: anytype, pred: ComparisonPredicate) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return if (l.isFloatElem()) k.cmpf(pred, l, r) else k.cmpi(pred, l, r, .signed); + } + + pub fn lt(self: Value, rhs: anytype) Value { + return self.compare(rhs, .less_than); + } + pub fn le(self: Value, rhs: anytype) Value { + return self.compare(rhs, .less_than_or_equal); + } + pub fn gt(self: Value, rhs: anytype) Value { + return self.compare(rhs, .greater_than); + } + pub fn ge(self: Value, rhs: anytype) Value { + return self.compare(rhs, .greater_than_or_equal); + } + pub fn eq(self: Value, rhs: anytype) Value { + return self.compare(rhs, .equal); + } + pub fn ne(self: Value, rhs: anytype) Value { + return self.compare(rhs, .not_equal); + } + + pub fn to(self: Value, dtype: DType) Value { + return self.kern().cast(self, dtype); + } + + pub fn bitCast(self: Value, dtype: DType) Value { + return self.kern().bitCast(self, dtype); + } + + pub fn reshape(self: Value, shape_: []const i64) Value { + return self.kern().reshape(self, shape_); + } + + pub fn broadcastTo(self: Value, shape_: []const i64) Value { + return self.kern().broadcastTo(self, shape_); + } + + pub fn expandDims(self: Value, axis: usize) Value { + return self.kern().expandDims(self, axis); + } + + pub fn permute(self: Value, order: []const i32) Value { + return self.kern().permute(self, order); + } + + pub fn sum(self: Value, axis: usize) Value { + return self.kern().sum(self, axis); + } + + pub fn max(self: Value, axis: usize) Value { + return self.kern().max(self, axis); + } + + pub fn min(self: Value, axis: usize) Value { + return self.kern().min(self, axis); + } + + pub fn cumsum(self: Value, axis: usize) Value { + return self.kern().cumsum(self, axis); + } + + pub fn offset(self: Value, off: anytype) Value { + const k = self.kern(); + const o: Value = if (@TypeOf(off) == Value) off else k.lift(off); + const p, const o2 = k.coerceShapes(self, o); + return k.offset(p, o2); + } +}; + +pub const Loaded = struct { + tile: Value, + token: Value, +}; + +pub const BlockId = struct { x: Value, y: Value, z: Value }; + +pub const ArgSpec = struct { + name: []const u8, + kind: Kind, + + /// XLA passes one raw device pointer per custom-call argument, so every + /// parameter is a rank-0 `tile>` and views are built inside the + /// kernel. A scalar reaches the kernel through a pointer (`loadPtr` on a + /// rank-0 pointer tile) or as a comptime constant in `cfg`. + pub const Kind = union(enum) { + ptr: DType, + ptr_opts: PtrOpts, + }; + + /// `div_by` becomes a `cuda_tile.assume #cuda_tile.div_by` at the top + /// of the entry, and `arg(i)` returns the refined value. XLA guarantees + /// only 16 bytes on the pointer it passes. + pub const PtrOpts = struct { + dtype: DType, + div_by: ?u32 = 16, + }; +}; + +pub const Opts = struct { + hints: []const EntryHint = &.{}, +}; + +pub const FloatOpts = struct { + rounding: RoundingMode = .nearest_even, + flush_to_zero: bool = false, +}; + +pub const MinMaxOpts = struct { + propagate_nan: bool = false, + flush_to_zero: bool = false, +}; + +pub const IntOpts = struct { + overflow: IntegerOverflow = .none, +}; + +pub const DivIOpts = struct { + signedness: Signedness = .signed, + rounding: ?RoundingMode = null, +}; + +/// `rounding` null picks the target's legal default: `nearest_even`, except +/// `f8e8m0fnu`, which the dialect only lets you reach with `zero` or +/// `positive_inf`. +pub const CastOpts = struct { + rounding: ?RoundingMode = null, + signedness: Signedness = .signed, +}; + +/// `ftoi` admits exactly one rounding mode (`nearest_int_to_zero`), so only +/// the signedness is a choice. +pub const FToIOpts = struct { + signedness: Signedness = .signed, +}; + +pub const MmaOpts = struct { + /// 13.3 only. + fast_acc: bool = false, +}; + +pub const PartitionOpts = struct { + padding: ?PaddingValue = .zero, + dim_map: []const i32 = &.{}, +}; + +pub const LoadOpts = struct { + ordering: MemoryOrdering = .weak, + scope: ?MemoryScope = null, + token: ?Value = null, + hints: []const LoadStoreHint = &.{}, +}; + +pub const StoreOpts = LoadOpts; + +/// The pointer arm takes `latency` hints only; `allow_tma` is a view-op hint. +pub const LoadPtrOpts = struct { + mask: ?Value = null, + padding: ?Value = null, + ordering: MemoryOrdering = .weak, + scope: ?MemoryScope = null, + token: ?Value = null, + hints: []const PtrLoadStoreHint = &.{}, +}; + +pub const StorePtrOpts = struct { + mask: ?Value = null, + ordering: MemoryOrdering = .weak, + scope: ?MemoryScope = null, + token: ?Value = null, + hints: []const PtrLoadStoreHint = &.{}, +}; + +pub const AtomicOpts = struct { + ordering: MemoryOrdering = .acq_rel, + scope: MemoryScope = .device, + mask: ?Value = null, + token: ?Value = null, +}; + +/// `atomic_red_view_tko` is relaxed-only, scoped to the block or device, and +/// has no mask operand — the view's padding decides what is touched. +pub const RedViewOpts = struct { + scope: MemoryScope = .device, + token: ?Value = null, +}; + +pub const AllocaOpts = struct { + alignment: i64 = 16, + global: bool = false, +}; + +pub const GlobalOpts = struct { + alignment: ?i64 = null, + constant: bool = false, + visibility: ?cuda_tile.SymbolVisibility = null, +}; + +pub const ScanOpts = struct { + reverse: bool = false, +}; + +pub fn ForScope(comptime N: usize) type { + return cf.ForScope(Builder, Value, N); +} + +pub const ForOpts = cf.ForOpts; + +pub const IfOnlyScope = cf.IfOnlyScope(Builder, Value); + +pub fn IfScope(comptime N: usize) type { + return cf.IfScope(Builder, Value, N); +} + +pub fn LoopScope(comptime N: usize) type { + return cf.LoopScope(Builder, Value, N); +} + +/// A custom reduction: `combine(builder, element, accumulator, ctx)` runs on +/// rank-0 tiles of `src`'s element type; `identity` is the element attribute +/// the dialect requires (see `Builder.identityAttr`). +pub fn ReduceArgs(comptime CtxT: type) type { + return struct { + src: Value, + axis: usize, + identity: *const mlir.Attribute, + combine: *const fn (*Builder, Value, Value, CtxT) Value, + }; +} + +pub fn ScanArgs(comptime CtxT: type) type { + return struct { + src: Value, + axis: usize, + reverse: bool = false, + identity: *const mlir.Attribute, + combine: *const fn (*Builder, Value, Value, CtxT) Value, + }; +} + +/// A reduction over several same-shaped tiles at once (argmax, an online +/// softmax's (max, sum)): `combine(builder, elements, accumulators, ctx)` +/// gets one rank-0 tile per source in each slice and returns the new +/// accumulators, one per source; `identities` is one element attribute per +/// source. +pub fn ReduceMultiArgs(comptime CtxT: type) type { + return struct { + srcs: []const Value, + axis: usize, + identities: []const *const mlir.Attribute, + combine: *const fn (*Builder, []const Value, []const Value, CtxT) []const Value, + }; +} + +pub fn ScanMultiArgs(comptime CtxT: type) type { + return struct { + srcs: []const Value, + axis: usize, + reverse: bool = false, + identities: []const *const mlir.Attribute, + combine: *const fn (*Builder, []const Value, []const Value, CtxT) []const Value, + }; +} + +pub const Builder = struct { + allocator: std.mem.Allocator, + arena: std.heap.ArenaAllocator, + ctx: *mlir.Context, + module: *mlir.Module, + name: []const u8, + func_op: ?*mlir.Operation, + entry_block: ?*mlir.Block, + block_stack: common.BlockStack = .{}, + ct_module: ?*mlir.Operation = null, + ct_body: ?*mlir.Block = null, + refined_args: []?Value = &.{}, + + pub fn init( + allocator: std.mem.Allocator, + ctx: *mlir.Context, + name: []const u8, + args: []const ArgSpec, + ) !Builder { + return initOpts(allocator, ctx, name, args, .{}); + } + + pub fn initOpts( + allocator: std.mem.Allocator, + ctx: *mlir.Context, + name: []const u8, + args: []const ArgSpec, + opts: Opts, + ) !Builder { + var b = try open(allocator, ctx, name); + errdefer b.deinit(); + try b.declareArgsLowOpts(args, opts); + return b; + } + + pub fn open( + allocator: std.mem.Allocator, + ctx: *mlir.Context, + name: []const u8, + ) !Builder { + const unknown_loc: *const mlir.Location = .unknown(ctx); + const module: *mlir.Module = .init(unknown_loc); + errdefer module.deinit(); + + var arena: std.heap.ArenaAllocator = .init(allocator); + errdefer arena.deinit(); + + return .{ + .allocator = allocator, + .arena = arena, + .ctx = ctx, + .module = module, + .name = name, + .func_op = null, + .entry_block = null, + .block_stack = .{}, + }; + } + + pub fn declareArgs(self: *Builder, spec: anytype) !common.NamedArgs(@TypeOf(spec), Value) { + return self.declareArgsOpts(spec, .{}); + } + + pub fn declareArgsOpts(self: *Builder, spec: anytype, opts: Opts) !common.NamedArgs(@TypeOf(spec), Value) { + const Spec = @TypeOf(spec); + const field_names = @typeInfo(Spec).@"struct".field_names; + + var arg_specs: [field_names.len]ArgSpec = undefined; + inline for (0.., field_names) |i, field_name| { + const raw = @field(spec, field_name); + const kind: ArgSpec.Kind = if (@TypeOf(raw) == ArgSpec.Kind) raw else blk: { + const variant = @typeInfo(@TypeOf(raw)).@"struct".field_names[0]; + const tag = @field(std.meta.Tag(ArgSpec.Kind), variant); + const inner = @field(raw, variant); + break :blk switch (tag) { + .ptr => .{ .ptr = inner }, + .ptr_opts => .{ .ptr_opts = .{ + .dtype = inner.dtype, + .div_by = if (@hasField(@TypeOf(inner), "div_by")) inner.div_by else 16, + } }, + }; + }; + arg_specs[i] = .{ .name = field_name, .kind = kind }; + } + + try self.declareArgsLowOpts(&arg_specs, opts); + + var named: common.NamedArgs(Spec, Value) = undefined; + inline for (0.., field_names) |i, field_name| { + @field(named, field_name) = self.arg(i); + } + return named; + } + + fn declareArgsLowOpts(self: *Builder, args: []const ArgSpec, opts: Opts) !void { + std.debug.assert(self.entry_block == null); + const ctx = self.ctx; + const unknown_loc: *const mlir.Location = .unknown(ctx); + const scratch = self.arena.allocator(); + + const arg_types = try scratch.alloc(*const mlir.Type, args.len); + const arg_locs = try scratch.alloc(*const mlir.Location, args.len); + for (arg_types, arg_locs, args) |*ty, *a_loc, a| { + ty.* = switch (a.kind) { + .ptr => |dt| self.ptrTy(dt), + .ptr_opts => |p| self.ptrTy(p.dtype), + }; + a_loc.* = unknown_loc; + } + + const entry = mlir.Block.init(arg_types, arg_locs); + const ct_body = mlir.Block.init(&.{}, &.{}); + const ct_module = cuda_tile.module(ctx, self.name, ct_body, unknown_loc); + _ = ct_module.appendTo(self.module.body()); + + const func_op = cuda_tile.entry(ctx, .{ + .name = self.name, + .block = entry, + .optimization_hints = if (opts.hints.len > 0) cuda_tile.optimizationHintsAttr(ctx, opts.hints) else null, + .location = unknown_loc, + }); + _ = func_op.appendTo(ct_body); + + self.func_op = func_op; + self.entry_block = entry; + self.block_stack.push(self.arena.allocator(), entry); + self.ct_module = ct_module; + self.ct_body = ct_body; + + self.refined_args = try scratch.alloc(?Value, args.len); + for (self.refined_args, args, 0..) |*refined, a, i| { + refined.* = switch (a.kind) { + .ptr => self.assumeDivBy(self.rawArg(i), 16), + .ptr_opts => |p| if (p.div_by) |n| self.assumeDivBy(self.rawArg(i), n) else null, + }; + } + } + + pub fn deinit(self: *Builder) void { + self.module.deinit(); + self.arena.deinit(); + } + + fn rawArg(self: *Builder, i: usize) Value { + const eb = self.entry_block orelse @panic("Builder.arg called before declareArgs"); + return .{ .inner = eb.argument(i), .kernel = self }; + } + + /// Parameter `i`, refined by its `assume` when one was declared. The + /// owner is re-stamped: `initOpts` fills `refined_args` through a Builder + /// that is then moved out of its frame. + pub fn arg(self: *Builder, i: usize) Value { + if (i < self.refined_args.len) { + if (self.refined_args[i]) |v| return .{ .inner = v.inner, .kernel = self }; + } + return self.rawArg(i); + } + + pub fn pushBlock(self: *Builder, b: *mlir.Block) void { + self.block_stack.push(self.arena.allocator(), b); + } + + pub fn popBlock(self: *Builder) void { + self.block_stack.pop(); + } + + pub fn currentBlock(self: *Builder) *mlir.Block { + return self.block_stack.current(); + } + + fn innerSlice(self: *Builder, values: []const Value) []const *const mlir.Value { + const out = self.arena.allocator().alloc(*const mlir.Value, values.len) catch @panic("Builder.innerSlice OOM"); + for (values, 0..) |v, i| out[i] = v.inner; + return out; + } + + fn innerOpt(v: ?Value) ?*const mlir.Value { + return if (v) |x| x.inner else null; + } + + pub fn emit(self: *Builder, op: *mlir.Operation) Value { + _ = op.appendTo(self.currentBlock()); + return .{ .inner = op.result(0), .kernel = self }; + } + + pub fn emitMulti(self: *Builder, op: *mlir.Operation, n: usize) []Value { + _ = op.appendTo(self.currentBlock()); + const out = self.arena.allocator().alloc(Value, n) catch @panic("Builder.emitMulti OOM"); + for (0..n) |i| out[i] = .{ .inner = op.result(i), .kernel = self }; + return out; + } + + pub fn yield(self: *Builder, values: anytype) []const Value { + const T = @TypeOf(values); + const info = @typeInfo(T); + if (info != .@"struct" or !info.@"struct".is_tuple) + @compileError("Builder.yield expects a tuple literal like `.{ v1, v2 }`"); + const n = info.@"struct".field_names.len; + const out = self.arena.allocator().alloc(Value, n) catch @panic("Builder.yield OOM"); + inline for (info.@"struct".field_names, 0..) |f, i| { + if (@FieldType(T, f) != Value) + @compileError("Builder.yield: every tuple element must be a Value; got " ++ @typeName(@FieldType(T, f))); + out[i] = @field(values, f); + } + return out; + } + + pub fn loc(self: *const Builder) *const mlir.Location { + return .unknown(self.ctx); + } + + /// `tile`; `&.{}` for a scalar. + pub fn tileTy(self: *const Builder, shape: []const i64, dtype: DType) *const mlir.Type { + if (!dtypes.tileLegal(shape)) { + std.debug.panic("Builder.tileTy: {any} is not a legal tile shape (power-of-two dims, at most 2^24 elements)", .{shape}); + } + return cuda_tile.tileType(self.ctx, shape, dtype.toMlir(self.ctx)); + } + + pub fn scalarTy(self: *const Builder, dtype: DType) *const mlir.Type { + return self.tileTy(&.{}, dtype); + } + + /// `tile>` — the type of a buffer parameter. + pub fn ptrTy(self: *const Builder, dtype: DType) *const mlir.Type { + return self.ptrTileTy(&.{}, dtype); + } + + /// `tile>`. `i4` is a tile element type but not a pointee + /// (Types.td: CudaTile_NumberType excludes Int4), so a 4-bit buffer is + /// declared as `.i8` and `unpack`ed in the kernel. + pub fn ptrTileTy(self: *const Builder, shape: []const i64, dtype: DType) *const mlir.Type { + if (dtype == .i4) @panic("Builder: i4 is a tile element type only, never a pointee; declare the buffer as .i8 and unpack in-kernel"); + return cuda_tile.tileType(self.ctx, shape, cuda_tile.pointerType(self.ctx, dtype.toMlir(self.ctx))); + } + + pub fn tokenTy(self: *const Builder) *const mlir.Type { + return cuda_tile.tokenType(self.ctx); + } + + fn tileLike(self: *const Builder, v: Value, elem: *const mlir.Type) *const mlir.Type { + return cuda_tile.tileType(self.ctx, v.shape().constSlice(), elem); + } + + fn mlirElemToDType(self: *const Builder, elem: *const mlir.Type) DType { + return dtypes.mlirElemToDType(self.ctx, elem); + } + + fn pointee(v: Value) *const mlir.Type { + const p = v.elemType().isA(cuda_tile.PointerType) orelse std.debug.panic("expected a pointer tile, got {f}", .{v.type_()}); + return p.pointee(); + } + + fn elemAttr(self: *Builder, dtype: DType, value: anytype) *const mlir.Attribute { + const elem = dtype.toMlir(self.ctx); + const T = @TypeOf(value); + if (isFloatDtype(dtype)) { + const v: f64 = switch (@typeInfo(T)) { + .comptime_int, .int => @floatFromInt(value), + .comptime_float, .float => @floatCast(value), + else => @compileError("Builder: unsupported scalar " ++ @typeName(T)), + }; + return cuda_tile.floatElemAttr(self.ctx, elem, v); + } + const v: i128 = switch (@typeInfo(T)) { + .comptime_int, .int => value, + .bool => @intFromBool(value), + .comptime_float, .float => @intFromFloat(value), + else => @compileError("Builder: unsupported scalar " ++ @typeName(T)), + }; + return cuda_tile.intElemAttr(elem, intBits(v, @intCast(intBitwidth(dtype)), @tagName(dtype))); + } + + /// `value` as the `i64` an integer constant of `bits` bits is built from. + /// It may be signed or an unsigned bit pattern, so it must lie in + /// [-2^(bits-1), 2^bits). + fn intBits(value: i128, bits: u7, dtype: []const u8) i64 { + if (value < -(@as(i128, 1) << (bits - 1)) or value >= @as(i128, 1) << bits) + std.debug.panic("Builder: {d} does not fit {s}", .{ value, dtype }); + return @truncate(value); + } + + fn splat(self: *Builder, tile_ty: *const mlir.Type, dtype: DType, value: anytype) Value { + return self.emit(cuda_tile.constant(self.ctx, cuda_tile.splatAttr(tile_ty, self.elemAttr(dtype, value)), tile_ty, self.loc())); + } + + /// `value` as a rank-0 constant of `dtype`; a Value passes through. An + /// integer is signed or an unsigned bit pattern of `dtype`'s width; + /// anything else panics. + pub fn constant(self: *Builder, dtype: DType, value: anytype) Value { + if (@TypeOf(value) == Value) return value; + return self.splat(self.scalarTy(dtype), dtype, value); + } + + pub fn full(self: *Builder, shape: []const i64, value: anytype, dtype: DType) Value { + return self.splat(self.tileTy(shape, dtype), dtype, value); + } + + pub fn zeros(self: *Builder, shape: []const i64, dtype: DType) Value { + return self.full(shape, 0, dtype); + } + + pub fn ones(self: *Builder, shape: []const i64, dtype: DType) Value { + return self.full(shape, 1, dtype); + } + + /// A dense constant from a slice of Zig scalars in row-major order. MLIR + /// stores every element in whole bytes except `i1`, which it bit-packs; + /// the slice must hold exactly that many bytes (use `full` for a splat). + pub fn dense(self: *Builder, shape: []const i64, dtype: DType, values: anytype) Value { + checkDenseBytes("Builder.dense", shape, dtype, values); + const ty = self.tileTy(shape, dtype); + return self.emit(cuda_tile.constant(self.ctx, cuda_tile.denseAttr(ty, values), ty, self.loc())); + } + + fn checkDenseBytes(comptime who: []const u8, shape: []const i64, dtype: DType, values: anytype) void { + var elems: usize = 1; + for (shape) |d| elems *= @intCast(d); + const bits = dtypeBitwidth(dtype); + const expected: usize = if (bits == 1) (elems + 7) / 8 else ((bits + 7) / 8) * elems; + const got = std.mem.sliceAsBytes(values).len; + if (got != expected) { + std.debug.panic("{s}: {d} bytes given for {any} x {s}, {d} expected", .{ who, got, shape, @tagName(dtype), expected }); + } + } + + pub fn iota(self: *Builder, n: i64, dtype: DType) Value { + return self.emit(cuda_tile.iota(self.ctx, self.tileTy(&.{n}, dtype), self.loc())); + } + + /// The element attribute `reduce`/`scan` need as an identity. + pub fn identityAttr(self: *Builder, dtype: DType, value: anytype) *const mlir.Attribute { + return self.elemAttr(dtype, value); + } + + /// Lift a Zig scalar to a rank-0 constant. Dtype follows the source type; + /// unsigned ints are bit-cast. Use `constant` for an explicit dtype. + pub fn lift(self: *Builder, value: anytype) Value { + const T = @TypeOf(value); + if (T == Value) return value; + return switch (@typeInfo(T)) { + .comptime_int => blk: { + if (value >= std.math.minInt(i32) and value <= std.math.maxInt(i32)) + break :blk self.constant(.i32, value); + break :blk self.constant(.i64, value); + }, + .int => |info| if (info.bits > 32) self.constant(.i64, value) else self.constant(.i32, value), + .bool => self.constant(.i1, value), + .comptime_float => self.constant(.f32, value), + .float => |info| switch (info.bits) { + 16 => self.constant(.f16, value), + 32 => self.constant(.f32, value), + 64 => self.constant(.f64, value), + else => @compileError("Builder.lift: unsupported float bitwidth"), + }, + else => @compileError("Builder.lift: unsupported type " ++ @typeName(T)), + }; + } + + fn liftMatching(self: *Builder, value: anytype, ref_elem: *const mlir.Type) Value { + const T = @TypeOf(value); + if (T == Value) return value; + return switch (@typeInfo(T)) { + .comptime_int, .comptime_float => self.constant(self.mlirElemToDType(ref_elem), value), + else => self.lift(value), + }; + } + + /// `value` as a constant of `like`'s element type; a Value passes through. + pub fn constantLike(self: *Builder, value: anytype, like: Value) Value { + if (@TypeOf(value) == Value) return value; + return self.constMatching(value, like.type_()); + } + + fn i32Bound(self: *Builder, value: anytype) Value { + return self.constant(.i32, value); + } + + fn constMatching(self: *Builder, value: anytype, ref: *const mlir.Type) Value { + const elem = if (ref.isA(cuda_tile.TileType)) |t| t.elementType() else ref; + return self.constant(self.mlirElemToDType(elem), value); + } + + pub fn broadcastLike(self: *Builder, value: anytype, ref: Value) Value { + const v = self.liftMatching(value, ref.elemType()); + return self.expandTo(v, ref.shape().constSlice()); + } + + fn expandTo(self: *Builder, v: Value, target: []const i64) Value { + const cur = v.shape(); + if (std.mem.eql(i64, cur.constSlice(), target)) return v; + var aligned = v; + if (cur.len != target.len) { + std.debug.assert(cur.len <= target.len); + var padded: Value.Shape = .empty; + for (0..target.len - cur.len) |_| padded.appendAssumeCapacity(1); + padded.appendSliceAssumeCapacity(cur.constSlice()); + aligned = self.reshape(v, padded.constSlice()); + } + if (std.mem.eql(i64, aligned.shape().constSlice(), target)) return aligned; + return self.broadcastTo(aligned, target); + } + + fn coerceShapes(self: *Builder, a: Value, b: Value) struct { Value, Value } { + const a_sh = a.shape(); + const b_sh = b.shape(); + if (std.mem.eql(i64, a_sh.constSlice(), b_sh.constSlice())) return .{ a, b }; + const n = @max(a_sh.len, b_sh.len); + var target: Value.Shape = .empty; + for (0..n) |i| { + const ai: i64 = if (i + a_sh.len >= n) a_sh.get(i + a_sh.len - n) else 1; + const bi: i64 = if (i + b_sh.len >= n) b_sh.get(i + b_sh.len - n) else 1; + if (ai == bi or bi == 1) { + target.appendAssumeCapacity(ai); + } else if (ai == 1) { + target.appendAssumeCapacity(bi); + } else { + std.debug.panic("Builder.coerce: incompatible shapes {any} vs {any} at axis {d}", .{ a_sh.constSlice(), b_sh.constSlice(), i }); + } + } + return .{ self.expandTo(a, target.constSlice()), self.expandTo(b, target.constSlice()) }; + } + + pub fn coerce(self: *Builder, a: anytype, b: anytype) struct { Value, Value } { + const a_ref: ?*const mlir.Type = if (@TypeOf(b) == Value) b.elemType() else null; + const b_ref: ?*const mlir.Type = if (@TypeOf(a) == Value) a.elemType() else null; + var av = if (a_ref) |t| self.liftMatching(a, t) else self.lift(a); + var bv = if (b_ref) |t| self.liftMatching(b, t) else self.lift(b); + + if (av.isIntElem() and bv.isIntElem()) { + const a_dt = self.mlirElemToDType(av.elemType()); + const b_dt = self.mlirElemToDType(bv.elemType()); + const aw = intBitwidth(a_dt); + const bw = intBitwidth(b_dt); + if (aw < bw) av = av.to(b_dt); + if (bw < aw) bv = bv.to(a_dt); + } + + return self.coerceShapes(av, bv); + } + + pub fn tileBlockId(self: *Builder) BlockId { + const r = self.emitMulti(cuda_tile.get_tile_block_id(self.ctx, self.loc()), 3); + return .{ .x = r[0], .y = r[1], .z = r[2] }; + } + + pub fn numTileBlocks(self: *Builder) BlockId { + const r = self.emitMulti(cuda_tile.get_num_tile_blocks(self.ctx, self.loc()), 3); + return .{ .x = r[0], .y = r[1], .z = r[2] }; + } + + pub fn assumeDivBy(self: *Builder, v: Value, n: u64) Value { + return self.emit(cuda_tile.assume(self.ctx, v.inner, cuda_tile.divByAttr(self.ctx, n, .{}), self.loc())); + } + + pub fn assumeDivByEvery(self: *Builder, v: Value, n: u64, every: i64, along: i64) Value { + return self.emit(cuda_tile.assume(self.ctx, v.inner, cuda_tile.divByAttr(self.ctx, n, .{ .every = every, .along = along }), self.loc())); + } + + pub fn assumeSameElements(self: *Builder, v: Value, values: []const i64) Value { + return self.emit(cuda_tile.assume(self.ctx, v.inner, cuda_tile.sameElementsAttr(self.ctx, values), self.loc())); + } + + pub fn assumeBounded(self: *Builder, v: Value, lb: ?i64, ub: ?i64) Value { + return self.emit(cuda_tile.assume(self.ctx, v.inner, cuda_tile.boundedAttr(self.ctx, .{ .lb = lb, .ub = ub }), self.loc())); + } + + pub fn tensorView(self: *Builder, base: Value, shape: []const i64, strides: []const i64) Value { + const ty = cuda_tile.tensorViewType(self.ctx, pointee(base), shape, strides); + return self.emit(cuda_tile.make_tensor_view(self.ctx, base.inner, &.{}, &.{}, ty, self.loc())); + } + + pub const Dim = union(enum) { + static: i64, + dynamic: Value, + }; + + pub fn tensorViewDyn(self: *Builder, base: Value, shape: []const Dim, strides: []const Dim) Value { + const scratch = self.arena.allocator(); + var static_shape: Value.Shape = .empty; + var static_strides: Value.Shape = .empty; + var dyn_shape = std.ArrayList(*const mlir.Value).empty; + var dyn_strides = std.ArrayList(*const mlir.Value).empty; + for (shape) |d| switch (d) { + .static => |s| static_shape.appendAssumeCapacity(s), + .dynamic => |v| { + static_shape.appendAssumeCapacity(cuda_tile.TensorViewType.dynamic()); + dyn_shape.append(scratch, v.inner) catch @panic("OOM"); + }, + }; + for (strides) |d| switch (d) { + .static => |s| static_strides.appendAssumeCapacity(s), + .dynamic => |v| { + static_strides.appendAssumeCapacity(cuda_tile.TensorViewType.dynamic()); + dyn_strides.append(scratch, v.inner) catch @panic("OOM"); + }, + }; + const ty = cuda_tile.tensorViewType(self.ctx, pointee(base), static_shape.constSlice(), static_strides.constSlice()); + return self.emit(cuda_tile.make_tensor_view(self.ctx, base.inner, dyn_shape.items, dyn_strides.items, ty, self.loc())); + } + + fn i32Slice(self: *Builder, values: []const i64) []const i32 { + const out = self.arena.allocator().alloc(i32, values.len) catch @panic("OOM"); + for (values, 0..) |v, i| out[i] = @intCast(v); + return out; + } + + /// Tiles `tv` into `tile`-shaped pieces addressed by tile-space indices. + /// `padding` fills the ragged edge on load; stores past the edge are + /// masked. That is why this DSL has no mask tile. + pub fn partitionView(self: *Builder, tv: Value, tile: []const i64, opts: PartitionOpts) Value { + if (!dtypes.tileLegal(tile)) std.debug.panic("Builder.partitionView: {any} is not a legal tile shape", .{tile}); + const ty: *const mlir.Type = @ptrCast(cuda_tile.PartitionViewType.get(self.ctx, .{ .tile_shape = self.i32Slice(tile), .tensor_view = tv.type_(), .dim_map = opts.dim_map, .padding = opts.padding })); + return self.emit(cuda_tile.make_partition_view(self.ctx, tv.inner, ty, self.loc())); + } + + /// 13.3: like `partitionView`, stepping by `traversal_strides` elements + /// per index instead of a whole tile. + pub fn stridedView(self: *Builder, tv: Value, tile: []const i64, traversal_strides: []const i64, opts: PartitionOpts) Value { + if (!dtypes.tileLegal(tile)) std.debug.panic("Builder.stridedView: {any} is not a legal tile shape", .{tile}); + const ty: *const mlir.Type = @ptrCast(cuda_tile.StridedViewType.get(self.ctx, .{ + .tile_shape = self.i32Slice(tile), + .traversal_strides = self.i32Slice(traversal_strides), + .tensor_view = tv.type_(), + .dim_map = opts.dim_map, + .padding = opts.padding, + })); + return self.emit(cuda_tile.make_strided_view(self.ctx, tv.inner, ty, self.loc())); + } + + /// 13.3: a view whose `sparse_dim` is indexed by a rank-1 tile of + /// `tile[sparse_dim]` indices (every other index stays rank 0). + pub fn gatherScatterView(self: *Builder, tv: Value, tile: []const i64, sparse_dim: u32, padding: ?PaddingValue) Value { + if (!dtypes.tileLegal(tile)) std.debug.panic("Builder.gatherScatterView: {any} is not a legal tile shape", .{tile}); + const ty = cuda_tile.gatherScatterViewType(self.ctx, tile, tv.type_(), sparse_dim, .{ .padding = padding }); + return self.emit(cuda_tile.make_gather_scatter_view(self.ctx, tv.inner, ty, self.loc())); + } + + fn i32Scalars(self: *Builder, n: usize) []const *const mlir.Type { + const out = self.arena.allocator().alloc(*const mlir.Type, n) catch @panic("OOM"); + for (out) |*t| t.* = self.scalarTy(.i32); + return out; + } + + pub fn indexSpaceShape(self: *Builder, view: Value) []Value { + const n = cuda_tile.indexRankOfView(view.type_()); + return self.emitMulti(cuda_tile.get_index_space_shape(self.ctx, view.inner, self.i32Scalars(n), self.loc()), n); + } + + pub fn tensorShape(self: *Builder, tv: Value) []Value { + const t = tv.type_().isA(cuda_tile.TensorViewType) orelse std.debug.panic("expected a tensor_view, got {f}", .{tv.type_()}); + const n = t.rank(); + return self.emitMulti(cuda_tile.get_tensor_shape(self.ctx, tv.inner, self.i32Scalars(n), self.loc()), n); + } + + fn memOpts(self: *Builder, ordering: MemoryOrdering, scope: ?MemoryScope, token: ?Value, hints: anytype) cuda_tile.MemOpts { + return .{ + .ordering = ordering, + .scope = scope, + .token = innerOpt(token), + .optimization_hints = if (hints.len > 0) cuda_tile.optimizationHintsAttr(self.ctx, hints) else null, + }; + } + + pub fn load(self: *Builder, view: Value, index: []const Value) Value { + return self.loadOpts(view, index, .{}).tile; + } + + pub fn loadOpts(self: *Builder, view: Value, index: []const Value, opts: LoadOpts) Loaded { + const expected = cuda_tile.indexRankOfView(view.type_()); + if (index.len != expected) std.debug.panic("Builder.load: view takes {d} indices, got {d}", .{ expected, index.len }); + const r = self.emitMulti(cuda_tile.load_view_tko(self.ctx, view.inner, self.innerSlice(index), self.memOpts(opts.ordering, opts.scope, opts.token, opts.hints), self.loc()), 2); + return .{ .tile = r[0], .token = r[1] }; + } + + pub fn store(self: *Builder, tile: Value, view: Value, index: []const Value) Value { + return self.storeOpts(tile, view, index, .{}); + } + + pub fn storeOpts(self: *Builder, tile: Value, view: Value, index: []const Value, opts: StoreOpts) Value { + const expected = cuda_tile.indexRankOfView(view.type_()); + if (index.len != expected) std.debug.panic("Builder.store: view takes {d} indices, got {d}", .{ expected, index.len }); + return self.emit(cuda_tile.store_view_tko(self.ctx, tile.inner, view.inner, self.innerSlice(index), self.memOpts(opts.ordering, opts.scope, opts.token, opts.hints), self.loc())); + } + + /// Gather through a tile of pointers, dropping the token. This arm cannot + /// use TMA; prefer views for dense access. + pub fn loadPtr(self: *Builder, ptrs: Value) Value { + return self.loadPtrOpts(ptrs, .{}).tile; + } + + pub fn loadPtrOpts(self: *Builder, ptrs: Value, opts: LoadPtrOpts) Loaded { + const result_ty = self.tileLike(ptrs, pointee(ptrs)); + const mask: ?Value = if (opts.mask) |m| self.expandTo(m, ptrs.shape().constSlice()) else null; + const r = self.emitMulti(cuda_tile.load_ptr_tko(self.ctx, ptrs.inner, result_ty, .{ + .mask = innerOpt(mask), + .padding = innerOpt(opts.padding), + .mem = self.memOpts(opts.ordering, opts.scope, opts.token, opts.hints), + }, self.loc()), 2); + return .{ .tile = r[0], .token = r[1] }; + } + + pub fn storePtr(self: *Builder, ptrs: Value, value: Value) Value { + return self.storePtrOpts(ptrs, value, .{}); + } + + pub fn storePtrOpts(self: *Builder, ptrs: Value, value: Value, opts: StorePtrOpts) Value { + const mask: ?Value = if (opts.mask) |m| self.expandTo(m, ptrs.shape().constSlice()) else null; + return self.emit(cuda_tile.store_ptr_tko(self.ctx, ptrs.inner, value.inner, .{ + .mask = innerOpt(mask), + .mem = self.memOpts(opts.ordering, opts.scope, opts.token, opts.hints), + }, self.loc())); + } + + pub fn makeToken(self: *Builder) Value { + return self.emit(cuda_tile.make_token(self.ctx, self.loc())); + } + + pub fn joinTokens(self: *Builder, tokens: []const Value) Value { + return self.emit(cuda_tile.join_tokens(self.ctx, self.innerSlice(tokens), self.loc())); + } + + pub fn print(self: *Builder, str: []const u8, args: []const Value) Value { + return self.printOpts(str, args, null); + } + + pub fn printOpts(self: *Builder, str: []const u8, args: []const Value, token: ?Value) Value { + return self.emit(cuda_tile.print_tko(self.ctx, str, self.innerSlice(args), .{ .token = innerOpt(token) }, self.loc())); + } + + fn atomicOpts(opts: AtomicOpts) cuda_tile.AtomicOpts { + return .{ .ordering = opts.ordering, .scope = opts.scope, .mask = innerOpt(opts.mask), .token = innerOpt(opts.token) }; + } + + pub fn atomicCas(self: *Builder, ptrs: Value, cmp: Value, val: Value, opts: AtomicOpts) Loaded { + const r = self.emitMulti(cuda_tile.atomic_cas_tko(self.ctx, ptrs.inner, cmp.inner, val.inner, atomicOpts(opts), self.loc()), 2); + return .{ .tile = r[0], .token = r[1] }; + } + + pub fn atomicRmw(self: *Builder, mode: AtomicRMWMode, ptrs: Value, operand: Value, opts: AtomicOpts) Loaded { + const r = self.emitMulti(cuda_tile.atomic_rmw_tko(self.ctx, ptrs.inner, mode, operand.inner, atomicOpts(opts), self.loc()), 2); + return .{ .tile = r[0], .token = r[1] }; + } + + /// 13.3: an atomic reduction into an unpadded tiled view; returns the + /// token. `xchg` is not a reduction and the dialect rejects it. + pub fn atomicRedView(self: *Builder, mode: AtomicRMWMode, view: Value, index: []const Value, value: Value, opts: RedViewOpts) Value { + return self.emit(cuda_tile.atomic_red_view_tko(self.ctx, view.inner, self.innerSlice(index), mode, value.inner, .{ + .scope = opts.scope, + .token = innerOpt(opts.token), + }, self.loc())); + } + + /// 13.3: `num_elem` elements of `dtype`, as a rank-0 pointer tile. + pub fn alloca(self: *Builder, dtype: DType, num_elem: i64, opts: AllocaOpts) Value { + return self.emit(cuda_tile.alloca(self.ctx, num_elem, opts.alignment, opts.global, self.ptrTy(dtype), self.loc())); + } + + pub fn global(self: *Builder, name: []const u8, shape: []const i64, dtype: DType, value: anytype, opts: GlobalOpts) void { + const ty = self.tileTy(shape, dtype); + const attr = cuda_tile.splatAttr(ty, self.elemAttr(dtype, value)); + self.appendGlobal(cuda_tile.global(self.ctx, name, attr, .{ .alignment = opts.alignment, .constant = opts.constant, .visibility = opts.visibility }, self.loc())); + } + + /// A module-level dense buffer from a slice of Zig scalars (see `dense` + /// for the byte-count rule). + pub fn globalDense(self: *Builder, name: []const u8, shape: []const i64, dtype: DType, values: anytype, opts: GlobalOpts) void { + checkDenseBytes("Builder.globalDense", shape, dtype, values); + const ty = self.tileTy(shape, dtype); + self.appendGlobal(cuda_tile.global(self.ctx, name, cuda_tile.denseAttr(ty, values), .{ .alignment = opts.alignment, .constant = opts.constant, .visibility = opts.visibility }, self.loc())); + } + + fn appendGlobal(self: *Builder, op: *mlir.Operation) void { + const body = self.ct_body orelse @panic("Builder.global called before declareArgs"); + const first = body.firstOperation() orelse { + body.appendOwnedOperation(op); + return; + }; + body.insertOwnedOperationBefore(first, op); + } + + pub fn getGlobal(self: *Builder, name: []const u8, dtype: DType) Value { + return self.emit(cuda_tile.get_global(self.ctx, name, self.ptrTy(dtype), self.loc())); + } + + pub fn addf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.addfOpts(lhs, rhs, .{}); + } + pub fn addfOpts(self: *Builder, lhs: Value, rhs: Value, opts: FloatOpts) Value { + return self.emit(cuda_tile.addf(self.ctx, lhs.inner, rhs.inner, opts.rounding, opts.flush_to_zero, self.loc())); + } + pub fn subf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.subfOpts(lhs, rhs, .{}); + } + pub fn subfOpts(self: *Builder, lhs: Value, rhs: Value, opts: FloatOpts) Value { + return self.emit(cuda_tile.subf(self.ctx, lhs.inner, rhs.inner, opts.rounding, opts.flush_to_zero, self.loc())); + } + pub fn mulf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.mulfOpts(lhs, rhs, .{}); + } + pub fn mulfOpts(self: *Builder, lhs: Value, rhs: Value, opts: FloatOpts) Value { + return self.emit(cuda_tile.mulf(self.ctx, lhs.inner, rhs.inner, opts.rounding, opts.flush_to_zero, self.loc())); + } + pub fn divf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.divfOpts(lhs, rhs, .{}); + } + pub fn divfOpts(self: *Builder, lhs: Value, rhs: Value, opts: FloatOpts) Value { + return self.emit(cuda_tile.divf(self.ctx, lhs.inner, rhs.inner, opts.rounding, opts.flush_to_zero, self.loc())); + } + pub fn remf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(cuda_tile.remf(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn negf(self: *Builder, x: Value) Value { + return self.emit(cuda_tile.negf(self.ctx, x.inner, self.loc())); + } + pub fn absf(self: *Builder, x: Value) Value { + return self.emit(cuda_tile.absf(self.ctx, x.inner, self.loc())); + } + pub fn fma(self: *Builder, a: Value, b: Value, acc: Value) Value { + return self.fmaOpts(a, b, acc, .{}); + } + pub fn fmaOpts(self: *Builder, a: Value, b: Value, acc: Value, opts: FloatOpts) Value { + return self.emit(cuda_tile.fma(self.ctx, a.inner, b.inner, acc.inner, opts.rounding, opts.flush_to_zero, self.loc())); + } + pub fn maxf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.maxfOpts(lhs, rhs, .{}); + } + pub fn maxfOpts(self: *Builder, lhs: Value, rhs: Value, opts: MinMaxOpts) Value { + return self.emit(cuda_tile.maxf(self.ctx, lhs.inner, rhs.inner, opts.propagate_nan, opts.flush_to_zero, self.loc())); + } + pub fn minf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.minfOpts(lhs, rhs, .{}); + } + pub fn minfOpts(self: *Builder, lhs: Value, rhs: Value, opts: MinMaxOpts) Value { + return self.emit(cuda_tile.minf(self.ctx, lhs.inner, rhs.inner, opts.propagate_nan, opts.flush_to_zero, self.loc())); + } + pub fn pow(self: *Builder, x: Value, exponent: Value) Value { + return self.emit(cuda_tile.pow(self.ctx, x.inner, exponent.inner, self.loc())); + } + pub fn atan2(self: *Builder, x: Value, y: Value) Value { + return self.emit(cuda_tile.atan2(self.ctx, x.inner, y.inner, self.loc())); + } + pub fn exp(self: *Builder, x: Value) Value { + return self.emit(cuda_tile.exp(self.ctx, x.inner, .{}, self.loc())); + } + /// 13.3: `exp` with an explicit rounding mode. + pub fn expRound(self: *Builder, x: Value, rounding: RoundingMode) Value { + return self.emit(cuda_tile.exp(self.ctx, x.inner, .{ .rounding = rounding }, self.loc())); + } + pub fn exp2(self: *Builder, x: Value) Value { + return self.exp2Opts(x, false); + } + pub fn exp2Opts(self: *Builder, x: Value, flush_to_zero: bool) Value { + return self.emit(cuda_tile.exp2(self.ctx, x.inner, flush_to_zero, self.loc())); + } + pub fn log(self: *Builder, x: Value) Value { + return self.emit(cuda_tile.log(self.ctx, x.inner, self.loc())); + } + pub fn log2(self: *Builder, x: Value) Value { + return self.emit(cuda_tile.log2(self.ctx, x.inner, self.loc())); + } + pub fn sqrt(self: *Builder, x: Value) Value { + return self.sqrtOpts(x, .{}); + } + pub fn sqrtOpts(self: *Builder, x: Value, opts: FloatOpts) Value { + return self.emit(cuda_tile.sqrt(self.ctx, x.inner, opts.rounding, opts.flush_to_zero, self.loc())); + } + pub fn rsqrt(self: *Builder, x: Value) Value { + return self.rsqrtOpts(x, false); + } + pub fn rsqrtOpts(self: *Builder, x: Value, flush_to_zero: bool) Value { + return self.emit(cuda_tile.rsqrt(self.ctx, x.inner, flush_to_zero, self.loc())); + } + pub fn sin(self: *Builder, x: Value) Value { + return self.emit(cuda_tile.sin(self.ctx, x.inner, self.loc())); + } + pub fn cos(self: *Builder, x: Value) Value { + return self.emit(cuda_tile.cos(self.ctx, x.inner, self.loc())); + } + pub fn tan(self: *Builder, x: Value) Value { + return self.emit(cuda_tile.tan(self.ctx, x.inner, self.loc())); + } + pub fn sinh(self: *Builder, x: Value) Value { + return self.emit(cuda_tile.sinh(self.ctx, x.inner, self.loc())); + } + pub fn cosh(self: *Builder, x: Value) Value { + return self.emit(cuda_tile.cosh(self.ctx, x.inner, self.loc())); + } + pub fn tanh(self: *Builder, x: Value) Value { + return self.emit(cuda_tile.tanh(self.ctx, x.inner, .{}, self.loc())); + } + /// 13.2: `tanh` with an explicit rounding mode. + pub fn tanhRound(self: *Builder, x: Value, rounding: RoundingMode) Value { + return self.emit(cuda_tile.tanh(self.ctx, x.inner, .{ .rounding = rounding }, self.loc())); + } + pub fn ceil(self: *Builder, x: Value) Value { + return self.emit(cuda_tile.ceil(self.ctx, x.inner, self.loc())); + } + pub fn floor(self: *Builder, x: Value) Value { + return self.emit(cuda_tile.floor(self.ctx, x.inner, self.loc())); + } + + pub fn cmpf(self: *Builder, predicate: ComparisonPredicate, lhs: Value, rhs: Value) Value { + return self.cmpfOpts(predicate, lhs, rhs, .ordered); + } + pub fn cmpfOpts(self: *Builder, predicate: ComparisonPredicate, lhs: Value, rhs: Value, ordering: ComparisonOrdering) Value { + return self.emit(cuda_tile.cmpf(self.ctx, predicate, ordering, lhs.inner, rhs.inner, self.loc())); + } + + pub fn mmaf(self: *Builder, a: Value, b: Value, acc: Value) Value { + return self.mmafOpts(a, b, acc, .{}); + } + pub fn mmafOpts(self: *Builder, a: Value, b: Value, acc: Value, opts: MmaOpts) Value { + return self.emit(cuda_tile.mmaf(self.ctx, a.inner, b.inner, acc.inner, opts.fast_acc, self.loc())); + } + /// 13.3: block-scaled MMA. Scales are plain tiles in logical layout — + /// `a_scale` is `[M, K/V]`, `b_scale` is `[K/V, N]` — never pre-swizzled. + pub fn mmafScaled(self: *Builder, a: Value, b: Value, acc: Value, a_scale: Value, b_scale: Value) Value { + return self.emit(cuda_tile.mmaf_scaled(self.ctx, a.inner, b.inner, acc.inner, a_scale.inner, b_scale.inner, self.loc())); + } + pub fn mmai(self: *Builder, a: Value, b: Value, acc: Value, a_signedness: Signedness, b_signedness: Signedness) Value { + return self.emit(cuda_tile.mmai(self.ctx, a.inner, b.inner, acc.inner, a_signedness, b_signedness, self.loc())); + } + + pub fn addi(self: *Builder, lhs: Value, rhs: Value) Value { + return self.addiOpts(lhs, rhs, .{}); + } + pub fn addiOpts(self: *Builder, lhs: Value, rhs: Value, opts: IntOpts) Value { + return self.emit(cuda_tile.addi(self.ctx, lhs.inner, rhs.inner, opts.overflow, self.loc())); + } + pub fn subi(self: *Builder, lhs: Value, rhs: Value) Value { + return self.subiOpts(lhs, rhs, .{}); + } + pub fn subiOpts(self: *Builder, lhs: Value, rhs: Value, opts: IntOpts) Value { + return self.emit(cuda_tile.subi(self.ctx, lhs.inner, rhs.inner, opts.overflow, self.loc())); + } + pub fn muli(self: *Builder, lhs: Value, rhs: Value) Value { + return self.muliOpts(lhs, rhs, .{}); + } + pub fn muliOpts(self: *Builder, lhs: Value, rhs: Value, opts: IntOpts) Value { + return self.emit(cuda_tile.muli(self.ctx, lhs.inner, rhs.inner, opts.overflow, self.loc())); + } + pub fn mulhii(self: *Builder, x: Value, y: Value) Value { + return self.emit(cuda_tile.mulhii(self.ctx, x.inner, y.inner, self.loc())); + } + pub fn divi(self: *Builder, lhs: Value, rhs: Value, signedness: Signedness) Value { + return self.diviOpts(lhs, rhs, .{ .signedness = signedness }); + } + pub fn diviOpts(self: *Builder, lhs: Value, rhs: Value, opts: DivIOpts) Value { + return self.emit(cuda_tile.divi(self.ctx, lhs.inner, rhs.inner, opts.signedness, .{ .rounding = opts.rounding }, self.loc())); + } + pub fn remi(self: *Builder, lhs: Value, rhs: Value, signedness: Signedness) Value { + return self.emit(cuda_tile.remi(self.ctx, lhs.inner, rhs.inner, signedness, self.loc())); + } + pub fn maxi(self: *Builder, lhs: Value, rhs: Value, signedness: Signedness) Value { + return self.emit(cuda_tile.maxi(self.ctx, lhs.inner, rhs.inner, signedness, self.loc())); + } + pub fn mini(self: *Builder, lhs: Value, rhs: Value, signedness: Signedness) Value { + return self.emit(cuda_tile.mini(self.ctx, lhs.inner, rhs.inner, signedness, self.loc())); + } + pub fn negi(self: *Builder, x: Value) Value { + return self.emit(cuda_tile.negi(self.ctx, x.inner, .none, self.loc())); + } + pub fn absi(self: *Builder, x: Value) Value { + return self.emit(cuda_tile.absi(self.ctx, x.inner, self.loc())); + } + pub fn shli(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(cuda_tile.shli(self.ctx, lhs.inner, rhs.inner, .none, self.loc())); + } + pub fn shri(self: *Builder, lhs: Value, rhs: Value, signedness: Signedness) Value { + return self.emit(cuda_tile.shri(self.ctx, lhs.inner, rhs.inner, signedness, self.loc())); + } + pub fn andi(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(cuda_tile.andi(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn ori(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(cuda_tile.ori(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn xori(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(cuda_tile.xori(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn cmpi(self: *Builder, predicate: ComparisonPredicate, lhs: Value, rhs: Value, signedness: Signedness) Value { + return self.emit(cuda_tile.cmpi(self.ctx, predicate, signedness, lhs.inner, rhs.inner, self.loc())); + } + + pub fn maximum(self: *Builder, a: anytype, b: anytype) Value { + const l, const r = self.coerce(a, b); + return if (l.isFloatElem()) self.maxf(l, r) else self.maxi(l, r, .signed); + } + + pub fn minimum(self: *Builder, a: anytype, b: anytype) Value { + const l, const r = self.coerce(a, b); + return if (l.isFloatElem()) self.minf(l, r) else self.mini(l, r, .signed); + } + + pub fn select(self: *Builder, cond: Value, t: Value, f: Value) Value { + return self.emit(cuda_tile.select(self.ctx, cond.inner, t.inner, f.inner, self.loc())); + } + + pub fn where(self: *Builder, cond: Value, x: anytype, y: anytype) Value { + var xv, var yv = self.coerce(x, y); + var cv = cond; + cv, xv = self.coerceShapes(cv, xv); + xv, yv = self.coerceShapes(xv, yv); + cv, _ = self.coerceShapes(cv, xv); + return self.select(cv, xv, yv); + } + + pub fn bitCast(self: *Builder, src: Value, dtype: DType) Value { + return self.emit(cuda_tile.bitcast(self.ctx, src.inner, self.tileLike(src, dtype.toMlir(self.ctx)), self.loc())); + } + pub fn exti(self: *Builder, src: Value, dtype: DType, signedness: Signedness) Value { + return self.emit(cuda_tile.exti(self.ctx, src.inner, signedness, self.tileLike(src, dtype.toMlir(self.ctx)), self.loc())); + } + pub fn trunci(self: *Builder, src: Value, dtype: DType) Value { + return self.emit(cuda_tile.trunci(self.ctx, src.inner, self.tileLike(src, dtype.toMlir(self.ctx)), .none, self.loc())); + } + pub fn ftof(self: *Builder, src: Value, dtype: DType) Value { + return self.ftofOpts(src, dtype, .{}); + } + pub fn ftofOpts(self: *Builder, src: Value, dtype: DType, opts: CastOpts) Value { + const rounding: RoundingMode = opts.rounding orelse (if (dtype == .f8e8m0fnu) .zero else .nearest_even); + return self.emit(cuda_tile.ftof(self.ctx, src.inner, self.tileLike(src, dtype.toMlir(self.ctx)), .{ .rounding = rounding }, self.loc())); + } + /// Float -> integer, truncating toward zero (the only rounding `ftoi` + /// admits). + pub fn ftoi(self: *Builder, src: Value, dtype: DType) Value { + return self.ftoiOpts(src, dtype, .{}); + } + pub fn ftoiOpts(self: *Builder, src: Value, dtype: DType, opts: FToIOpts) Value { + return self.emit(cuda_tile.ftoi(self.ctx, src.inner, opts.signedness, .nearest_int_to_zero, self.tileLike(src, dtype.toMlir(self.ctx)), self.loc())); + } + /// Integer -> float. `f8e8m0fnu` is not a legal target; go through `f32`. + pub fn itof(self: *Builder, src: Value, dtype: DType) Value { + return self.itofOpts(src, dtype, .{}); + } + pub fn itofOpts(self: *Builder, src: Value, dtype: DType, opts: CastOpts) Value { + if (dtype == .f8e8m0fnu) @panic("Builder.itof: the dialect refuses integer -> f8e8m0fnu; convert to .f32 first (Builder.cast does)"); + return self.emit(cuda_tile.itof(self.ctx, src.inner, opts.signedness, opts.rounding orelse .nearest_even, self.tileLike(src, dtype.toMlir(self.ctx)), self.loc())); + } + pub fn intToPtr(self: *Builder, src: Value, pointee_dtype: DType) Value { + return self.emit(cuda_tile.int_to_ptr(self.ctx, src.inner, self.ptrTileTy(src.shape().constSlice(), pointee_dtype), self.loc())); + } + pub fn ptrToInt(self: *Builder, src: Value) Value { + return self.emit(cuda_tile.ptr_to_int(self.ctx, src.inner, self.tileLike(src, DType.i64.toMlir(self.ctx)), self.loc())); + } + pub fn ptrToPtr(self: *Builder, src: Value, pointee_dtype: DType) Value { + return self.emit(cuda_tile.ptr_to_ptr(self.ctx, src.inner, self.ptrTileTy(src.shape().constSlice(), pointee_dtype), self.loc())); + } + /// 13.3: a sub-byte tile packed into `tile`. + pub fn pack(self: *Builder, src: Value, shape: []const i64) Value { + return self.emit(cuda_tile.pack(self.ctx, src.inner, self.tileTy(shape, .i8), self.loc())); + } + /// 13.3: an i8 tile unpacked into `tile` of a sub-byte `dtype`. + pub fn unpack(self: *Builder, src: Value, shape: []const i64, dtype: DType) Value { + return self.emit(cuda_tile.unpack(self.ctx, src.inner, self.tileTy(shape, dtype), self.loc())); + } + + /// Numeric cast with auto-dispatch: ftof / ftoi / itof / exti / trunci / + /// bitcast by element kind and width. Integers are signed; i1 is not. + pub fn cast(self: *Builder, src: Value, dtype: DType) Value { + const cur_elem = src.elemType(); + const tgt_elem = dtype.toMlir(self.ctx); + if (cur_elem.eql(tgt_elem)) return src; + const cur_dtype = self.mlirElemToDType(cur_elem); + const cur_is_float = src.isFloatElem(); + const tgt_is_float = isFloatDtype(dtype); + if (cur_is_float and tgt_is_float) return self.ftof(src, dtype); + if (cur_is_float) return self.ftoi(src, dtype); + if (tgt_is_float) { + const signedness: Signedness = if (cur_dtype == .i1) .unsigned else .signed; + if (dtype == .f8e8m0fnu) return self.ftof(self.itofOpts(src, .f32, .{ .signedness = signedness }), dtype); + return self.itofOpts(src, dtype, .{ .signedness = signedness }); + } + const cur_bw = dtypeBitwidth(cur_dtype); + const tgt_bw = dtypeBitwidth(dtype); + if (tgt_bw > cur_bw) return self.exti(src, dtype, if (cur_dtype == .i1) .unsigned else .signed); + if (tgt_bw < cur_bw) return self.trunci(src, dtype); + return self.bitCast(src, dtype); + } + + pub fn reshape(self: *Builder, src: Value, shape: []const i64) Value { + return self.emit(cuda_tile.reshape(self.ctx, src.inner, self.tileTyOf(shape, src.elemType()), self.loc())); + } + + fn tileTyOf(self: *const Builder, shape: []const i64, elem: *const mlir.Type) *const mlir.Type { + if (!dtypes.tileLegal(shape)) std.debug.panic("Builder: {any} is not a legal tile shape", .{shape}); + return cuda_tile.tileType(self.ctx, shape, elem); + } + + pub fn broadcastTo(self: *Builder, src: Value, shape: []const i64) Value { + return self.emit(cuda_tile.broadcast(self.ctx, src.inner, self.tileTyOf(shape, src.elemType()), self.loc())); + } + + pub fn expandDims(self: *Builder, src: Value, axis: usize) Value { + const in_shape = src.shape(); + var out: Value.Shape = .empty; + for (0..in_shape.len + 1) |i| { + if (i == axis) out.appendAssumeCapacity(1) else out.appendAssumeCapacity(in_shape.get(if (i < axis) i else i - 1)); + } + return self.reshape(src, out.constSlice()); + } + + pub fn cat(self: *Builder, lhs: Value, rhs: Value, dim: usize) Value { + var out = lhs.shape(); + out.set(dim, out.get(dim) + rhs.dim(dim)); + return self.emit(cuda_tile.cat(self.ctx, lhs.inner, rhs.inner, @intCast(dim), self.tileTyOf(out.constSlice(), lhs.elemType()), self.loc())); + } + + pub fn extract(self: *Builder, src: Value, indices: []const Value, shape: []const i64) Value { + return self.emit(cuda_tile.extract(self.ctx, src.inner, self.innerSlice(indices), self.tileTyOf(shape, src.elemType()), self.loc())); + } + + pub fn permute(self: *Builder, src: Value, order: []const i32) Value { + const src_shape = src.shape(); + std.debug.assert(order.len == src_shape.len); + var out: Value.Shape = .empty; + for (order) |i| out.appendAssumeCapacity(src_shape.get(@intCast(i))); + return self.emit(cuda_tile.permute(self.ctx, src.inner, order, self.tileTyOf(out.constSlice(), src.elemType()), self.loc())); + } + + pub fn transpose(self: *Builder, src: Value) Value { + std.debug.assert(src.rank() == 2); + return self.permute(src, &.{ 1, 0 }); + } + + pub fn offset(self: *Builder, ptrs: Value, off: Value) Value { + return self.emit(cuda_tile.offset(self.ctx, ptrs.inner, off.inner, self.loc())); + } + + fn reducedShape(src: Value, axis: usize) Value.Shape { + if (axis >= src.rank()) std.debug.panic("Builder.reduce: axis {d} out of range for rank {d}", .{ axis, src.rank() }); + var out: Value.Shape = .empty; + const s = src.shape(); + for (0..s.len) |i| if (i != axis) out.appendAssumeCapacity(s.get(i)); + return out; + } + + fn checkAxis(src: Value, axis: usize) void { + if (axis >= src.rank()) std.debug.panic("Builder.scan: axis {d} out of range for rank {d}", .{ axis, src.rank() }); + } + + fn combineBody(self: *Builder, ctx: anytype, elem: *const mlir.Type, combine: anytype) *mlir.Block { + const scalar = cuda_tile.tileType(self.ctx, &.{}, elem); + const body = mlir.Block.init(&.{ scalar, scalar }, &.{ self.loc(), self.loc() }); + self.pushBlock(body); + const element: Value = .{ .inner = body.argument(0), .kernel = self }; + const acc: Value = .{ .inner = body.argument(1), .kernel = self }; + const combined: Value = combine(self, element, acc, ctx); + _ = cuda_tile.yield(self.ctx, &.{combined.inner}, self.loc()).appendTo(body); + self.popBlock(); + return body; + } + + pub fn reduce(self: *Builder, ctx: anytype, args: ReduceArgs(@TypeOf(ctx))) Value { + const result_ty = self.tileTyOf(reducedShape(args.src, args.axis).constSlice(), args.src.elemType()); + const body = self.combineBody(ctx, args.src.elemType(), args.combine); + return self.emit(cuda_tile.reduce(self.ctx, &.{args.src.inner}, @intCast(args.axis), &.{args.identity}, body, &.{result_ty}, self.loc())); + } + + pub fn scan(self: *Builder, ctx: anytype, args: ScanArgs(@TypeOf(ctx))) Value { + checkAxis(args.src, args.axis); + const body = self.combineBody(ctx, args.src.elemType(), args.combine); + return self.emit(cuda_tile.scan(self.ctx, &.{args.src.inner}, @intCast(args.axis), args.reverse, &.{args.identity}, body, &.{args.src.type_()}, self.loc())); + } + + /// The `2N`-argument body of a variadic `reduce`/`scan`: arguments come in + /// (element, accumulator) pairs, one pair per source. + fn combineBodyMulti(self: *Builder, ctx: anytype, srcs: []const Value, combine: anytype) *mlir.Block { + const scratch = self.arena.allocator(); + const n = srcs.len; + const types = scratch.alloc(*const mlir.Type, 2 * n) catch @panic("OOM"); + const locs = scratch.alloc(*const mlir.Location, 2 * n) catch @panic("OOM"); + for (srcs, 0..) |s, i| { + const scalar = cuda_tile.tileType(self.ctx, &.{}, s.elemType()); + types[2 * i] = scalar; + types[2 * i + 1] = scalar; + locs[2 * i] = self.loc(); + locs[2 * i + 1] = self.loc(); + } + const body = mlir.Block.init(types, locs); + self.pushBlock(body); + const elements = scratch.alloc(Value, n) catch @panic("OOM"); + const accs = scratch.alloc(Value, n) catch @panic("OOM"); + for (0..n) |i| { + elements[i] = .{ .inner = body.argument(2 * i), .kernel = self }; + accs[i] = .{ .inner = body.argument(2 * i + 1), .kernel = self }; + } + const out: []const Value = combine(self, elements, accs, ctx); + if (out.len != n) std.debug.panic("Builder.reduceMulti: combine returned {d} values for {d} sources", .{ out.len, n }); + _ = cuda_tile.yield(self.ctx, self.innerSlice(out), self.loc()).appendTo(body); + self.popBlock(); + return body; + } + + pub fn reduceMulti(self: *Builder, ctx: anytype, args: ReduceMultiArgs(@TypeOf(ctx))) []Value { + std.debug.assert(args.srcs.len == args.identities.len); + const result_types = self.arena.allocator().alloc(*const mlir.Type, args.srcs.len) catch @panic("OOM"); + for (args.srcs, 0..) |s, i| result_types[i] = self.tileTyOf(reducedShape(s, args.axis).constSlice(), s.elemType()); + const body = self.combineBodyMulti(ctx, args.srcs, args.combine); + return self.emitMulti(cuda_tile.reduce(self.ctx, self.innerSlice(args.srcs), @intCast(args.axis), args.identities, body, result_types, self.loc()), args.srcs.len); + } + + pub fn scanMulti(self: *Builder, ctx: anytype, args: ScanMultiArgs(@TypeOf(ctx))) []Value { + std.debug.assert(args.srcs.len == args.identities.len); + const result_types = self.arena.allocator().alloc(*const mlir.Type, args.srcs.len) catch @panic("OOM"); + for (args.srcs, 0..) |s, i| { + checkAxis(s, args.axis); + result_types[i] = s.type_(); + } + const body = self.combineBodyMulti(ctx, args.srcs, args.combine); + return self.emitMulti(cuda_tile.scan(self.ctx, self.innerSlice(args.srcs), @intCast(args.axis), args.reverse, args.identities, body, result_types, self.loc()), args.srcs.len); + } + + const ReduceKind = enum { sum, max, min, prod }; + + fn combineKind(k: *Builder, element: Value, acc: Value, kind: ReduceKind) Value { + const is_float = element.isFloatElem(); + return switch (kind) { + .sum => if (is_float) k.addf(element, acc) else k.addi(element, acc), + .prod => if (is_float) k.mulf(element, acc) else k.muli(element, acc), + .max => if (is_float) k.maxf(element, acc) else k.maxi(element, acc, .signed), + .min => if (is_float) k.minf(element, acc) else k.mini(element, acc, .signed), + }; + } + + fn kindIdentity(self: *Builder, src: Value, kind: ReduceKind) *const mlir.Attribute { + const dt = self.mlirElemToDType(src.elemType()); + if (isFloatDtype(dt)) { + return self.identityAttr(dt, switch (kind) { + .sum => @as(f64, 0), + .prod => @as(f64, 1), + .max => -std.math.inf(f64), + .min => std.math.inf(f64), + }); + } + const bits: u32 = intBitwidth(dt); + const min_int: i64 = if (bits == 64) std.math.minInt(i64) else -(@as(i64, 1) << @intCast(bits - 1)); + const max_int: i64 = if (bits == 64) std.math.maxInt(i64) else (@as(i64, 1) << @intCast(bits - 1)) - 1; + return self.identityAttr(dt, switch (kind) { + .sum => @as(i64, 0), + .prod => @as(i64, 1), + .max => min_int, + .min => max_int, + }); + } + + fn reduceKind(self: *Builder, src: Value, axis: usize, comptime kind: ReduceKind) Value { + const combine = struct { + fn c(k: *Builder, element: Value, acc: Value, _: void) Value { + return combineKind(k, element, acc, kind); + } + }.c; + return self.reduce({}, .{ .src = src, .axis = axis, .identity = self.kindIdentity(src, kind), .combine = combine }); + } + + pub fn sum(self: *Builder, src: Value, axis: usize) Value { + return self.reduceKind(src, axis, .sum); + } + pub fn max(self: *Builder, src: Value, axis: usize) Value { + return self.reduceKind(src, axis, .max); + } + pub fn min(self: *Builder, src: Value, axis: usize) Value { + return self.reduceKind(src, axis, .min); + } + pub fn prod(self: *Builder, src: Value, axis: usize) Value { + return self.reduceKind(src, axis, .prod); + } + + fn scanKind(self: *Builder, src: Value, axis: usize, opts: ScanOpts, comptime kind: ReduceKind) Value { + const combine = struct { + fn c(k: *Builder, element: Value, acc: Value, _: void) Value { + return combineKind(k, element, acc, kind); + } + }.c; + return self.scan({}, .{ .src = src, .axis = axis, .reverse = opts.reverse, .identity = self.kindIdentity(src, kind), .combine = combine }); + } + + pub fn cumsum(self: *Builder, src: Value, axis: usize) Value { + return self.scanKind(src, axis, .{}, .sum); + } + pub fn cumsumOpts(self: *Builder, src: Value, axis: usize, opts: ScanOpts) Value { + return self.scanKind(src, axis, opts, .sum); + } + pub fn cumprod(self: *Builder, src: Value, axis: usize) Value { + return self.scanKind(src, axis, .{}, .prod); + } + pub fn cummax(self: *Builder, src: Value, axis: usize) Value { + return self.scanKind(src, axis, .{}, .max); + } + pub fn cummin(self: *Builder, src: Value, axis: usize) Value { + return self.scanKind(src, axis, .{}, .min); + } + + /// `for iv in [lower, upper) step` carrying `inits`; close with + /// `scope.yield(.{...})` (`scope.close()` without inits), then read + /// `scope.results`. + pub fn openFor( + self: *Builder, + lower: anytype, + upper: anytype, + step: anytype, + inits: anytype, + ) ForScope(common.arity(@TypeOf(inits), "openFor: inits")) { + return self.openForOpts(lower, upper, step, inits, .{}); + } + + /// `openFor` with `cuda_tile.for` options. + pub fn openForOpts( + self: *Builder, + lower: anytype, + upper: anytype, + step: anytype, + inits: anytype, + opts: ForOpts, + ) ForScope(common.arity(@TypeOf(inits), "openFor: inits")) { + const N = comptime common.arity(@TypeOf(inits), "openFor: inits"); + + // Literals take the type of the first bound that is a Value, else + // `i32`; all three must agree (`AllTypesMatch` on the op). + const lb_v, const ub_v, const step_v = common.forBounds(self, Value, lower, upper, step, i32Bound); + + var block_types: [N + 1]*const mlir.Type = undefined; + block_types[0] = lb_v.type_(); + var inits_inner: [N]*const mlir.Value = undefined; + inline for (0..N) |i| { + const v: Value = if (@TypeOf(inits[i]) == Value) inits[i] else self.lift(inits[i]); + block_types[i + 1] = v.type_(); + inits_inner[i] = v.inner; + } + const block_locs: [N + 1]*const mlir.Location = @splat(self.loc()); + const body = mlir.Block.init(&block_types, &block_locs); + self.pushBlock(body); + + var carried: [N]Value = undefined; + for (0..N) |i| carried[i] = .{ .inner = body.argument(i + 1), .kernel = self }; + + return .{ + .kernel = self, + .body = body, + .lb_inner = lb_v.inner, + .ub_inner = ub_v.inner, + .step_inner = step_v.inner, + .inits_inner = inits_inner, + .iv = .{ .inner = body.argument(0), .kernel = self }, + .carried = carried, + .opts = opts, + }; + } + + pub fn openIf(self: *Builder, cond: Value) IfOnlyScope { + const then_block = mlir.Block.init(&.{}, &.{}); + self.pushBlock(then_block); + return .{ .kernel = self, .cond_inner = cond.inner, .then_block = then_block }; + } + + pub fn openIfElse(self: *Builder, cond: Value, result_types: anytype) IfScope(common.arity(@TypeOf(result_types), "openIfElse: result_types")) { + const then_block = mlir.Block.init(&.{}, &.{}); + const else_block = mlir.Block.init(&.{}, &.{}); + self.pushBlock(then_block); + return .{ + .kernel = self, + .cond_inner = cond.inner, + .then_block = then_block, + .else_block = else_block, + .result_types = common.resultTypes(self, Value, result_types, "openIfElse: result_types"), + }; + } + + /// An unstructured loop carrying `inits` (`scope.close()` ends a body that + /// carries nothing, in place of `scope.yield`); leave with `scope.breakWith` + /// inside an `if`, and close the body with `scope.yield`. + pub fn openLoop(self: *Builder, inits: anytype) LoopScope(common.arity(@TypeOf(inits), "openLoop: inits")) { + const N = comptime common.arity(@TypeOf(inits), "openLoop: inits"); + var block_types: [N]*const mlir.Type = undefined; + var inits_inner: [N]*const mlir.Value = undefined; + inline for (0..N) |i| { + const v: Value = if (@TypeOf(inits[i]) == Value) inits[i] else self.lift(inits[i]); + block_types[i] = v.type_(); + inits_inner[i] = v.inner; + } + const block_locs: [N]*const mlir.Location = @splat(self.loc()); + const body = mlir.Block.init(&block_types, &block_locs); + self.pushBlock(body); + + var carried: [N]Value = undefined; + for (0..N) |i| carried[i] = .{ .inner = body.argument(i), .kernel = self }; + + return .{ + .kernel = self, + .body = body, + .inits_inner = inits_inner, + .result_types = block_types, + .carried = carried, + }; + } + + pub fn deviceAssert(self: *Builder, condition: Value, message: []const u8) void { + _ = cuda_tile.assert_(self.ctx, condition.inner, message, self.loc()).appendTo(self.currentBlock()); + } + + /// Append `return`, verify the module, and serialize it to a NUL-terminated + /// string owned by the kernel's allocator. `results` must be empty: an + /// `entry` returns nothing; outputs reach the host through pointers. + pub fn finish(self: *Builder, results: []const Value) FinishError![:0]const u8 { + std.debug.assert(results.len == 0); + const current = self.currentBlock(); + if (current.terminator() == null) { + _ = cuda_tile.return_(self.ctx, &.{}, self.loc()).appendTo(current); + } + + try common.verify(self.module, "dsl/cuda_tile"); + + var al: std.Io.Writer.Allocating = .init(self.allocator); + defer al.deinit(); + + try al.writer.print("{f}", .{self.module.operation()}); + + return try self.allocator.dupeSentinel(u8, al.written(), 0); + } +}; + +fn testContext() !*mlir.Context { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + registerDialects(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + return ctx; +} + +fn expectRoundTrip(ctx: *mlir.Context, ir: [:0]const u8) !void { + const parsed = try mlir.Module.parse(ctx, ir); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "empty entry round-trips" { + const ctx = try testContext(); + defer ctx.deinit(); + + var b = try Builder.open(std.testing.allocator, ctx, "empty"); + defer b.deinit(); + _ = try b.declareArgs(.{ .p = .{ .ptr = .f32 } }); + + const ir = try b.finish(&.{}); + defer std.testing.allocator.free(ir); + + try std.testing.expect(std.mem.find(u8, ir, "cuda_tile.module @empty") != null); + try std.testing.expect(std.mem.find(u8, ir, "entry @empty(") != null); + try std.testing.expect(std.mem.find(u8, ir, "assume") != null); + try std.testing.expect(std.mem.find(u8, ir, "div_by<16>") != null); + try std.testing.expect(std.mem.find(u8, ir, "cuda_tile.entry") == null); + try std.testing.expect(std.mem.find(u8, ir, "return") != null); + try expectRoundTrip(ctx, ir); +} + +test "add_one over pointer tiles matches the XLA milestone kernel" { + const ctx = try testContext(); + defer ctx.deinit(); + + var b = try Builder.open(std.testing.allocator, ctx, "add_one"); + defer b.deinit(); + const a = try b.declareArgsOpts(.{ + .in = .{ .ptr_opts = .{ .dtype = .f32, .div_by = null } }, + .out = .{ .ptr_opts = .{ .dtype = .f32, .div_by = null } }, + }, .{}); + + const offsets = b.iota(128, .i32); + const loaded = b.loadPtrOpts(a.in.offset(offsets), .{}); + const sum = loaded.tile.add(1.0); + _ = b.storePtrOpts(a.out.offset(offsets), sum, .{ .token = loaded.token }); + + const ir = try b.finish(&.{}); + defer std.testing.allocator.free(ir); + + try std.testing.expect(std.mem.find(u8, ir, "iota : tile<128xi32>") != null); + try std.testing.expect(std.mem.find(u8, ir, "reshape") != null); + try std.testing.expect(std.mem.find(u8, ir, "broadcast") != null); + try std.testing.expect(std.mem.find(u8, ir, "offset") != null); + try std.testing.expect(std.mem.find(u8, ir, "load_ptr_tko weak") != null); + try std.testing.expect(std.mem.find(u8, ir, "addf") != null); + try std.testing.expect(std.mem.find(u8, ir, "store_ptr_tko weak") != null); + try std.testing.expect(std.mem.find(u8, ir, "token = ") != null); + try expectRoundTrip(ctx, ir); +} + +test "vector_add over partition views" { + const ctx = try testContext(); + defer ctx.deinit(); + + var b = try Builder.open(std.testing.allocator, ctx, "vector_add"); + defer b.deinit(); + const a = try b.declareArgsOpts(.{ + .x = .{ .ptr = .f32 }, + .y = .{ .ptr = .f32 }, + .out = .{ .ptr = .f32 }, + }, .{ .hints = &.{.{ .arch = .sm_120, .num_worker_warps_per_cta = 4 }} }); + + const shape = [_]i64{ 8192, 128 }; + const strides = [_]i64{ 128, 1 }; + const tile = [_]i64{ 64, 64 }; + const vx = b.partitionView(b.tensorView(a.x, &shape, &strides), &tile, .{}); + const vy = b.partitionView(b.tensorView(a.y, &shape, &strides), &tile, .{}); + const vo = b.partitionView(b.tensorView(a.out, &shape, &strides), &tile, .{}); + + const bid = b.tileBlockId(); + const idx = [_]Value{ bid.x, bid.y }; + const xt = b.load(vx, &idx); + const yt = b.load(vy, &idx); + _ = b.store(xt.add(yt), vo, &idx); + + const ir = try b.finish(&.{}); + defer std.testing.allocator.free(ir); + + try std.testing.expect(std.mem.find(u8, ir, "optimization_hints=") != null); + try std.testing.expect(std.mem.find(u8, ir, "make_tensor_view") != null); + try std.testing.expect(std.mem.find(u8, ir, "partition_view") != null or std.mem.find(u8, ir, "addf") != null); + try std.testing.expect(std.mem.find(u8, ir, "store_view_tko weak") != null); + try expectRoundTrip(ctx, ir); +} + +test "gemm with a for loop and mmaf" { + const ctx = try testContext(); + defer ctx.deinit(); + + var b = try Builder.open(std.testing.allocator, ctx, "gemm"); + defer b.deinit(); + const a = try b.declareArgs(.{ .a = .{ .ptr = .bf16 }, .bm = .{ .ptr = .bf16 }, .c = .{ .ptr = .f32 } }); + + const m: i64 = 1024; + const n: i64 = 1024; + const k: i64 = 512; + const pa = b.partitionView(b.tensorView(a.a, &.{ m, k }, &.{ k, 1 }), &.{ 128, 64 }, .{}); + const pb = b.partitionView(b.tensorView(a.bm, &.{ k, n }, &.{ n, 1 }), &.{ 64, 128 }, .{}); + const pc = b.partitionView(b.tensorView(a.c, &.{ m, n }, &.{ n, 1 }), &.{ 128, 128 }, .{}); + + const bid = b.tileBlockId(); + const acc0 = b.zeros(&.{ 128, 128 }, .f32); + + var loop = b.openFor(0, @divExact(k, 64), 1, .{acc0}); + { + const kt = loop.iv; + const acc = loop.carried[0]; + const at = b.load(pa, &.{ bid.x, kt }); + const bt = b.load(pb, &.{ kt, bid.y }); + loop.yield(.{b.mmaf(at, bt, acc)}); + } + _ = b.store(loop.results[0], pc, &.{ bid.x, bid.y }); + + const ir = try b.finish(&.{}); + defer std.testing.allocator.free(ir); + + try std.testing.expect(std.mem.find(u8, ir, "for ") != null); + try std.testing.expect(std.mem.find(u8, ir, "iter_values") != null); + try std.testing.expect(std.mem.find(u8, ir, "mmaf") != null); + try std.testing.expect(std.mem.find(u8, ir, "continue ") != null); + try expectRoundTrip(ctx, ir); +} + +test "if, if-else, loop with break, reduce, scan, cast, select" { + const ctx = try testContext(); + defer ctx.deinit(); + + var b = try Builder.open(std.testing.allocator, ctx, "control"); + defer b.deinit(); + // A scalar reaches the kernel through a rank-0 pointer. + const a = try b.declareArgs(.{ .x = .{ .ptr = .f32 }, .n = .{ .ptr = .i32 } }); + const n = b.loadPtr(a.n); + try std.testing.expect(n.isScalar()); + + const v = b.partitionView(b.tensorView(a.x, &.{ 256, 64 }, &.{ 64, 1 }), &.{ 8, 64 }, .{}); + const bid = b.tileBlockId(); + const t = b.load(v, &.{ bid.x, bid.y }); + + // Reductions and scans over the tile. + const row_sum = b.sum(t, 1); + const col_max = b.max(t, 0); + const running = b.cumsum(t, 1); + try std.testing.expectEqual(@as(usize, 1), row_sum.rank()); + try std.testing.expectEqual(@as(i64, 8), row_sum.dim(0)); + try std.testing.expectEqual(@as(i64, 64), col_max.dim(0)); + try std.testing.expectEqual(@as(usize, 2), running.rank()); + + // A variadic reduce: argmax over axis 1 as (max, index). + const cols = b.broadcastTo(b.reshape(b.iota(64, .i32), &.{ 1, 64 }), &.{ 8, 64 }); + const argmax = b.reduceMulti({}, .{ + .srcs = &.{ t, cols }, + .axis = 1, + .identities = &.{ b.identityAttr(.f32, -std.math.inf(f64)), b.identityAttr(.i32, 0) }, + .combine = struct { + fn c(k: *Builder, elements: []const Value, accs: []const Value, _: void) []const Value { + const better = elements[0].gt(accs[0]); + return k.yield(.{ k.select(better, elements[0], accs[0]), k.select(better, elements[1], accs[1]) }); + } + }.c, + }); + try std.testing.expectEqual(@as(usize, 2), argmax.len); + try std.testing.expectEqual(@as(i64, 8), argmax[1].dim(0)); + + // Casts and select. + const as_i32 = t.to(.i32); + const back = as_i32.to(.f32); + const picked = b.where(t.gt(0.0), back, t.neg()); + + // if / if-else carrying a value. + const is_first = bid.x.eq(0); + var only = b.openIf(is_first); + { + _ = b.store(picked, v, &.{ bid.x, bid.y }); + only.close(); + } + + var branch = b.openIfElse(is_first, .{b.tileTy(&.{ 8, 64 }, .f32)}); + { + branch.yieldThen(.{picked}); + } + { + branch.yieldElse(.{t}); + } + const chosen = branch.results[0]; + + // loop: count up to n, break when reached. + var lp = b.openLoop(.{b.constant(.i32, 0)}); + { + const i = lp.carried[0]; + var done = b.openIf(i.ge(n)); + { + lp.breakWith(.{i}); + done.close(); + } + lp.yield(.{i.add(1)}); + } + _ = lp.results[0]; + + // a loop carrying nothing ends with close(). + var once = b.openLoop(.{}); + { + var first = b.openIf(is_first); + { + once.breakWith(.{}); + first.close(); + } + once.close(); + } + + // for with an early `continue` inside an `if`, and i64 bounds from a Value. + var skip = b.openFor(b.constant(.i64, 0), 16, 1, .{b.constant(.i64, 0)}); + { + const acc = skip.carried[0]; + var odd = b.openIf(skip.iv.bitAnd(1).eq(1)); + { + skip.continueWith(.{acc}); + odd.close(); + } + skip.yield(.{acc.add(skip.iv)}); + } + try std.testing.expect(skip.results[0].isScalar()); + + _ = b.store(chosen, v, &.{ bid.x, bid.y }); + + const ir = try b.finish(&.{}); + defer std.testing.allocator.free(ir); + + try std.testing.expect(std.mem.find(u8, ir, "reduce ") != null); + try std.testing.expect(std.mem.find(u8, ir, "scan ") != null); + try std.testing.expect(std.mem.find(u8, ir, "ftoi") != null); + try std.testing.expect(std.mem.find(u8, ir, "itof") != null); + try std.testing.expect(std.mem.find(u8, ir, "select") != null); + try std.testing.expect(std.mem.find(u8, ir, "loop ") != null); + try std.testing.expect(std.mem.find(u8, ir, "break ") != null); + try std.testing.expect(std.mem.find(u8, ir, "yield") != null); + try expectRoundTrip(ctx, ir); +} + +test "atomics, print, globals, alloca, dynamic views, strided view, tokens" { + const ctx = try testContext(); + defer ctx.deinit(); + + var b = try Builder.open(std.testing.allocator, ctx, "misc"); + defer b.deinit(); + const a = try b.declareArgs(.{ .x = .{ .ptr = .f32 }, .cnt = .{ .ptr = .i32 }, .rows = .{ .ptr = .i32 } }); + b.global("lut", &.{16}, .f32, 1.5, .{ .constant = true }); + + const rows = b.loadPtr(a.rows); + const tv = b.tensorViewDyn(a.x, &.{ .{ .dynamic = rows }, .{ .static = 64 } }, &.{ .{ .static = 64 }, .{ .static = 1 } }); + const extents = b.tensorShape(tv); + try std.testing.expectEqual(@as(usize, 2), extents.len); + const v = b.partitionView(tv, &.{ 8, 64 }, .{ .padding = .nan }); + const tiles = b.indexSpaceShape(v); + try std.testing.expectEqual(@as(usize, 2), tiles.len); + const sv = b.stridedView(tv, &.{ 8, 64 }, &.{ 2, 1 }, .{}); + _ = b.load(sv, &.{ b.constant(.i32, 0), b.constant(.i32, 0) }); + + const bid = b.tileBlockId(); + const loaded = b.loadOpts(v, &.{ bid.x, bid.y }, .{ .hints = &.{.{ .arch = .sm_120, .allow_tma = true }} }); + const tok = b.print("tile %f\n", &.{loaded.tile}); + + const counter = a.cnt.offset(b.iota(8, .i32)); + const old = b.atomicRmw(.add, counter, b.full(&.{8}, 1, .i32), .{ .token = tok }); + const cas = b.atomicCas(counter, old.tile, b.full(&.{8}, 0, .i32), .{ .token = old.token }); + const joined = b.joinTokens(&.{ loaded.token, cas.token, b.makeToken() }); + // Atomic reductions refuse a padded view; the default options must verify. + const unpadded = b.partitionView(tv, &.{ 8, 64 }, .{ .padding = null }); + _ = b.atomicRedView(.addf, unpadded, &.{ bid.x, bid.y }, loaded.tile, .{ .token = joined }); + _ = b.atomicRedView(.addf, unpadded, &.{ bid.x, bid.y }, loaded.tile, .{}); + + // A gather/scatter view: the sparse index is a rank-1 tile of row ids. + const gsv = b.gatherScatterView(tv, &.{ 8, 64 }, 0, .zero); + const gathered = b.load(gsv, &.{ b.iota(8, .i32), b.constant(.i32, 0) }); + try std.testing.expectEqual(@as(i64, 8), gathered.dim(0)); + try std.testing.expectEqual(@as(i64, 64), gathered.dim(1)); + _ = b.store(gathered, gsv, &.{ b.iota(8, .i32), b.constant(.i32, 0) }); + + const lut = b.getGlobal("lut", .f32); + const scratch = b.alloca(.f32, 64, .{}); + _ = b.loadPtr(lut.offset(b.iota(16, .i32))); + _ = b.storePtr(scratch.offset(b.iota(64, .i32)), b.zeros(&.{64}, .f32)); + + const ir = try b.finish(&.{}); + defer std.testing.allocator.free(ir); + + try std.testing.expect(std.mem.find(u8, ir, "global @lut") != null); + try std.testing.expect(std.mem.find(u8, ir, "get_global @lut") != null); + try std.testing.expect(std.mem.find(u8, ir, "tensor_view") != null); + try expectRoundTrip(ctx, ir); +} diff --git a/dsl/cuda_tile/control_flow.zig b/dsl/cuda_tile/control_flow.zig new file mode 100644 index 00000000..26979e0a --- /dev/null +++ b/dsl/cuda_tile/control_flow.zig @@ -0,0 +1,185 @@ +//! `cuda_tile` scope wrappers, like `dsl/common/control_flow.zig` but with +//! the dialect's terminators: `continue` ends a `for`/`loop` body, `yield` an +//! `if`, `break` leaves a `loop`. An already-terminated branch is left alone. + +const common = @import("dsl/common/builder"); +const std = @import("std"); +const cuda_tile = @import("mlir/dialects/cuda_tile"); +const mlir = @import("mlir"); + +fn terminate(comptime ValueT: type, comptime N: usize, kernel: anytype, block: *mlir.Block, comptime mnemonic: enum { yield, continue_, break_ }, values: anytype, comptime what: []const u8) void { + if (block.terminator() != null) return; + const buf = common.innerValues(ValueT, N, values, what); + const op = switch (mnemonic) { + .yield => cuda_tile.yield(kernel.ctx, &buf, kernel.loc()), + .continue_ => cuda_tile.continue_(kernel.ctx, &buf, kernel.loc()), + .break_ => cuda_tile.break_(kernel.ctx, &buf, kernel.loc()), + }; + _ = op.appendTo(block); +} + +pub const ForOpts = struct { + /// Compare the induction variable unsigned. + unsigned_cmp: bool = false, +}; + +/// `for %iv in (lb to ub, step) iter_values(...)`; `yield` closes it, or +/// `close()` when it carries nothing. +pub fn ForScope(comptime BuilderT: type, comptime ValueT: type, comptime N: usize) type { + return struct { + kernel: *BuilderT, + body: *mlir.Block, + lb_inner: *const mlir.Value, + ub_inner: *const mlir.Value, + step_inner: *const mlir.Value, + inits_inner: [N]*const mlir.Value, + iv: ValueT, + carried: [N]ValueT, + results: [N]ValueT = undefined, + opts: ForOpts = .{}, + + const Self = @This(); + + /// `continue` in the current block, inside a zero-result `if`: skips + /// the rest of the iteration and terminates that branch. + pub fn continueWith(self: *Self, values: anytype) void { + const k = self.kernel; + terminate(ValueT, N, k, k.currentBlock(), .continue_, values, "ForScope.continueWith"); + } + + pub fn yield(self: *Self, values: anytype) void { + if (N == 0) @compileError("ForScope.yield: the loop carries nothing, end it with close()"); + self.end(values); + } + + pub fn close(self: *Self) void { + if (N != 0) @compileError("ForScope.close: the loop carries values, end it with yield()"); + self.end(.{}); + } + + fn end(self: *Self, values: anytype) void { + const k = self.kernel; + terminate(ValueT, N, k, self.body, .continue_, values, "ForScope.yield"); + k.popBlock(); + const for_op = cuda_tile.for_(k.ctx, self.lb_inner, self.ub_inner, self.step_inner, &self.inits_inner, self.body, self.opts.unsigned_cmp, k.loc()); + _ = for_op.appendTo(k.currentBlock()); + for (0..N) |i| self.results[i] = .{ .inner = for_op.result(i), .kernel = k }; + } + }; +} + +pub fn IfOnlyScope(comptime BuilderT: type, comptime ValueT: type) type { + return struct { + kernel: *BuilderT, + cond_inner: *const mlir.Value, + then_block: *mlir.Block, + + const Self = @This(); + + pub fn close(self: *Self) void { + const k = self.kernel; + terminate(ValueT, 0, k, self.then_block, .yield, .{}, "IfOnlyScope.close"); + k.popBlock(); + _ = cuda_tile.if_(k.ctx, self.cond_inner, &.{}, self.then_block, .{}, k.loc()).appendTo(k.currentBlock()); + } + }; +} + +pub fn IfScope(comptime BuilderT: type, comptime ValueT: type, comptime N: usize) type { + return struct { + kernel: *BuilderT, + cond_inner: *const mlir.Value, + then_block: *mlir.Block, + else_block: *mlir.Block, + result_types: [N]*const mlir.Type, + results: [N]ValueT = undefined, + + const Self = @This(); + + pub fn yieldThen(self: *Self, values: anytype) void { + if (N == 0) @compileError("IfScope.yieldThen: the if has no results, switch branches with elseBranch()"); + self.endThen(values); + } + + pub fn yieldElse(self: *Self, values: anytype) void { + if (N == 0) @compileError("IfScope.yieldElse: the if has no results, end it with close()"); + self.endElse(values); + } + + pub fn elseBranch(self: *Self) void { + if (N != 0) @compileError("IfScope.elseBranch: the if has results, switch branches with yieldThen()"); + self.endThen(.{}); + } + + pub fn close(self: *Self) void { + if (N != 0) @compileError("IfScope.close: the if has results, end it with yieldElse()"); + self.endElse(.{}); + } + + fn endThen(self: *Self, values: anytype) void { + const k = self.kernel; + terminate(ValueT, N, k, self.then_block, .yield, values, "IfScope.yieldThen"); + k.popBlock(); + k.pushBlock(self.else_block); + } + + fn endElse(self: *Self, values: anytype) void { + const k = self.kernel; + terminate(ValueT, N, k, self.else_block, .yield, values, "IfScope.yieldElse"); + k.popBlock(); + const if_op = cuda_tile.if_(k.ctx, self.cond_inner, &self.result_types, self.then_block, .{ .else_block = self.else_block }, k.loc()); + _ = if_op.appendTo(k.currentBlock()); + for (0..N) |i| self.results[i] = .{ .inner = if_op.result(i), .kernel = k }; + } + }; +} + +/// `loop iter_values(...)`: the body ends with `continue` (`yield`, or +/// `close()` when it carries nothing) and leaves through `break` (`breakWith`, +/// inside an `if`). `results` are the broken. +pub fn LoopScope(comptime BuilderT: type, comptime ValueT: type, comptime N: usize) type { + return struct { + kernel: *BuilderT, + body: *mlir.Block, + inits_inner: [N]*const mlir.Value, + result_types: [N]*const mlir.Type, + carried: [N]ValueT, + results: [N]ValueT = undefined, + + const Self = @This(); + + /// `break` in the current block; terminates the enclosing `if` branch. + pub fn breakWith(self: *Self, values: anytype) void { + const k = self.kernel; + terminate(ValueT, N, k, k.currentBlock(), .break_, values, "LoopScope.breakWith"); + } + + pub fn continueWith(self: *Self, values: anytype) void { + const k = self.kernel; + terminate(ValueT, N, k, k.currentBlock(), .continue_, values, "LoopScope.continueWith"); + } + + pub fn yield(self: *Self, values: anytype) void { + if (N == 0) @compileError("LoopScope.yield: the loop carries nothing, end it with close()"); + self.end(values); + } + + pub fn close(self: *Self) void { + if (N != 0) @compileError("LoopScope.close: the loop carries values, end it with yield()"); + self.end(.{}); + } + + fn end(self: *Self, values: anytype) void { + const k = self.kernel; + terminate(ValueT, N, k, self.body, .continue_, values, "LoopScope.yield"); + k.popBlock(); + const loop_op = cuda_tile.loop(k.ctx, &self.inits_inner, &self.result_types, self.body, k.loc()); + _ = loop_op.appendTo(k.currentBlock()); + for (0..N) |i| self.results[i] = .{ .inner = loop_op.result(i), .kernel = k }; + } + }; +} + +test { + std.testing.refAllDecls(@This()); +} diff --git a/dsl/cuda_tile/dtype.zig b/dsl/cuda_tile/dtype.zig new file mode 100644 index 00000000..a9193839 --- /dev/null +++ b/dsl/cuda_tile/dtype.zig @@ -0,0 +1,103 @@ +const std = @import("std"); +const mlir = @import("mlir"); + +/// Every element type `cuda_tile.tile` accepts (Types.td: +/// CudaTile_TileElementType). `i4`, `f4e2m1fn` need 13.3; `f8e8m0fnu` needs +/// 13.2. `i4` is a tile element only — a pointer cannot point at it. +pub const DType = enum { + i1, + i4, + i8, + i16, + i32, + i64, + f16, + bf16, + f32, + tf32, + f64, + f8e4m3fn, + f8e5m2, + f8e8m0fnu, + f4e2m1fn, + + pub fn toMlir(self: DType, ctx: *mlir.Context) *const mlir.Type { + return switch (self) { + .i1 => .int(ctx, .i1), + .i4 => .int(ctx, .i4), + .i8 => .int(ctx, .i8), + .i16 => .int(ctx, .i16), + .i32 => .int(ctx, .i32), + .i64 => .int(ctx, .i64), + .f16 => .float(ctx, .f16), + .bf16 => .float(ctx, .bf16), + .f32 => .float(ctx, .f32), + .tf32 => .float(ctx, .tf32), + .f64 => .float(ctx, .f64), + .f8e4m3fn => .float(ctx, .f8e4m3fn), + .f8e5m2 => .float(ctx, .f8e5m2), + .f8e8m0fnu => .float(ctx, .f8e8m0fnu), + .f4e2m1fn => .float(ctx, .f4e2m1fn), + }; + } +}; + +pub fn isFloatDtype(dt: DType) bool { + return switch (dt) { + .f16, .bf16, .f32, .tf32, .f64, .f8e4m3fn, .f8e5m2, .f8e8m0fnu, .f4e2m1fn => true, + else => false, + }; +} + +pub fn dtypeBitwidth(dt: DType) usize { + return switch (dt) { + .i1 => 1, + .i4, .f4e2m1fn => 4, + .i8, .f8e4m3fn, .f8e5m2, .f8e8m0fnu => 8, + .i16, .f16, .bf16 => 16, + .i32, .f32, .tf32 => 32, + .i64, .f64 => 64, + }; +} + +pub fn intBitwidth(dt: DType) u32 { + return switch (dt) { + .i1 => 1, + .i4 => 4, + .i8 => 8, + .i16 => 16, + .i32 => 32, + .i64 => 64, + else => std.debug.panic("intBitwidth: not an integer DType: {s}", .{@tagName(dt)}), + }; +} + +pub fn mlirElemToDType(ctx: *mlir.Context, elem: *const mlir.Type) DType { + inline for (comptime std.meta.fieldNames(DType)) |f_name| { + const dt = @field(DType, f_name); + if (elem.eql(dt.toMlir(ctx))) return dt; + } + @panic("element type not a recognized CUDA Tile IR DType"); +} + +/// SharedVerifiers.h verifyTileSize: every dimension a power of two and the +/// tile at most 2^24 elements. Checked in Zig so the failure names the shape. +pub fn tileLegal(shape: []const i64) bool { + var elems: i64 = 1; + for (shape) |d| { + if (d <= 0 or (d & (d - 1)) != 0) return false; + elems *= d; + } + return elems <= (1 << 24); +} + +test { + std.testing.refAllDecls(@This()); +} + +test "tileLegal" { + try std.testing.expect(tileLegal(&.{})); + try std.testing.expect(tileLegal(&.{ 64, 128 })); + try std.testing.expect(!tileLegal(&.{3})); + try std.testing.expect(!tileLegal(&.{ 4096, 4096, 2 })); +} diff --git a/dsl/cute/BUILD.bazel b/dsl/cute/BUILD.bazel new file mode 100644 index 00000000..ef5fc6d9 --- /dev/null +++ b/dsl/cute/BUILD.bazel @@ -0,0 +1,34 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "builder", + srcs = [ + "dtype.zig", + ], + import_name = "dsl/cute/builder", + main = "builder.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//dsl/common:builder", + "//mlir", + "//mlir/dialects/arith", + "//mlir/dialects/cf", + "//mlir/dialects/cuda", + "//mlir/dialects/cute_ir", + "//mlir/dialects/cute_nvgpu", + "//mlir/dialects/func", + "//mlir/dialects/gpu", + "//mlir/dialects/llvm", + "//mlir/dialects/math", + "//mlir/dialects/nvvm", + "//mlir/dialects/scf", + "//mlir/dialects/vector", + "//stdx", + ], +) + +zig_test( + name = "test", + deps = [":builder"], +) diff --git a/dsl/cute/README.md b/dsl/cute/README.md new file mode 100644 index 00000000..946c542d --- /dev/null +++ b/dsl/cute/README.md @@ -0,0 +1,186 @@ +# CuTe DSL kernels in Zig + +`dsl/cute` is a Zig front end for NVIDIA's CuTe DSL, shaped after the +Python `cutlass.cute` API: a kernel takes tensors, indexes them with +coordinates, and reads `thread_idx()` and friends from `cute.arch`. It is the +fourth instance of the `dsl/triton` / `dsl/mosaic_tpu` / +`dsl/cuda_tile` pattern: a `Builder` that owns an MLIR module and appends +ops to a block stack, a `Value` with fluent scalar arithmetic, and a +`zml.kernel.cute.Kernel` wrapper that turns a `run` function into a +`stablehlo.custom_call` XLA compiles in process with `libcute_compiler.so`. + +The dialect bindings live in `mlir/dialects/cute_ir` (`cute`), +`mlir/dialects/cute_nvgpu`, and `mlir/dialects/nvvm` and `mlir/dialects/cuda` +for NVIDIA's `nvvm` and `cuda` (operation builders under `.ops`), with types +and attributes through each dialect's C API. + +## A kernel + +The Python notebook's `naive_elementwise_add_kernel`, in Zig: + +```zig +const zml = @import("zml"); +const cute = zml.kernel.cute; + +const Cfg = struct { m: i64, n: i64 }; + +pub const NaiveElementwiseAdd = cute.Kernel(Cfg, .{ + .name = "naive_elementwise_add_kernel", + .inputs = &.{ "gA", "gB" }, + .outputs = &.{"gC"}, + .run = run, +}); + +fn run(b: *cute.Builder, cfg: Cfg) cute.FinishError!void { + // A `tensor` argument is XLA's device pointer composed with a static + // layout; the Python kernel takes `cute.Tensor` the same way. + const t = try b.declareArgs(.{ + .gA = .{ .tensor = .{ .dtype = .f32, .shape = &.{ cfg.m, cfg.n } } }, + .gB = .{ .tensor = .{ .dtype = .f32, .shape = &.{ cfg.m, cfg.n } } }, + .gC = .{ .tensor = .{ .dtype = .f32, .shape = &.{ cfg.m, cfg.n } } }, + }); + + const tidx = b.threadIdx().x; + const bidx = b.blockIdx().x; + const bdim = b.blockDim().x; + const thread_idx = bidx.mul(bdim).add(tidx); + + const n = t.gA.dim(1); + const ni = thread_idx.rem(n); + const mi = thread_idx.div(n); + + // gC[mi, ni] = gA[mi, ni] + gB[mi, ni] + t.gC.set(.{ mi, ni }, t.gA.get(.{ mi, ni }).add(t.gB.get(.{ mi, ni }))); +} + +// In a model (`kernel.launch(grid=..., block=...)`): +const c = NaiveElementwiseAdd.call(.{ .gA = a, .gB = b }, .{ .gC = a.shape() }, .{ + .cfg = .{ .m = 64, .n = 128 }, + .grid = .{ 32, 1, 1 }, + .block = .{ 256, 1, 1 }, +}).gC; +``` + +## Python to Zig + +Put each control-flow region in a Zig `{ ... }` block immediately after +`openFor`, `openIf`, or `openWhile`, with its corresponding `yield` call (or +`close()` for a scope without values) inside the block. For `openIfElse`, use +separate blocks ending in `yieldThen` and `yieldElse` (`elseBranch()` and +`close()` when it has no results); likewise, give `openWhile` separate +`yieldBefore` and `yieldAfter` blocks. Keep the scope variable outside these +blocks so its `.results` remain accessible afterward. The braces make the +generated control flow visible and keep region-local values from escaping +their scope; the builder's `open` and `yield` calls still create and close +the actual IR regions. + +| `cutlass.cute` | `dsl/cute` | +|--------------------------------------------|-------------------------------------------------| +| `gA: cute.Tensor` kernel argument | `.gA = .{ .tensor = .{ .dtype, .shape } }` | +| `gA[mi, ni]` / `gC[mi, ni] = v` | `gA.get(.{ mi, ni })` / `gC.set(.{ mi, ni }, v)`| +| `cute.make_layout(shape, stride=...)` | `b.makeLayout(shape, .{ .stride = ... })` | +| `cute.make_tensor(ptr, layout)` | `b.makeTensor(ptr, layout)` | +| `cute.arch.thread_idx()` → `(x, y, z)` | `b.threadIdx().x` | +| `block_idx()`, `block_dim()`, `grid_dim()` | `blockIdx()`, `blockDim()`, `gridDim()` | +| `cute.arch.alloc_smem(T, n, align)` | `b.allocSmem(.f32, n, align)` | +| `cute.arch.sync_threads()` | `b.syncThreads()` | +| `cute.arch.shuffle_sync_bfly(v, lane)` | `b.shuffleXor(v, lane)` | +| `cute.arch.griddepcontrol_launch_dependents()` | `b.launchDependents()` | +| `x.to(Float32)` | `x.to(.f32)` | +| `if cond:` / `for i in range(...)` | `b.openIf(cond)` / `b.openFor(0, n, 1, .{})` | +| `cpasync.make_tiled_tma_atom(...)` | `b.makeTiledTmaAtom(...)` | +| `cpasync.tma_partition(...)` | `b.tmaPartition(...)` | +| `cute.copy(atom, src, dst)` | `b.copy(atom, src, dst, .{})` | +| `cute.make_tiled_mma(atom)` | `b.makeTiledMma(atom, result_type)` | +| `tiled_mma.make_fragment_A/B/C(...)` | `b.mmaMakeFragment(...)` | +| `cute.gemm(...)` | `b.gemm(...)` | +| `cute.arch.alloc_tmem(...)` | `b.allocTmem(...)` | +| `cute.arch.retrieve_tmem_ptr(...)` | `b.retrieveTmemPtr(...)` | + +* **Layouts accept static and dynamic extents.** `makeLayout` takes Zig slices + for a tracked static `Layout`, or nested tuples of integers and SSA `Value`s + for a `View`. Dynamic shape and stride leaves retain their integer widths. + The tracked `Tensor` keeps its shape and stride in Zig; use `makeTensorView` + with dynamic layouts. Integers stay static in coordinate types. +* **Stride defaults differ by origin.** `makeLayout` defaults to CuTe's + compact left-most stride like `cute.make_layout`; a `tensor` argument + defaults to XLA's row-major layout, which is what the buffer holds. +* **Scalars are `i32` and `f32` by default**, like `Int32`/`Float32`; + `constant(dtype, v)` picks another type and `coerce` lifts literals to the other + operand's type. +* **`nvvm` and `cuda` are the compiler's.** The context registers NVIDIA's + `nvvm` and `cuda` dialects (`nvvm.ops`, `cuda.ops`, from the + `mlir/dialects/nvvm` and `mlir/dialects/cuda` packages) and upstream's + `llvm`, so every operation is + verified as it is built; nothing is left unregistered. +* **The module is just the kernel.** `finish` prints a module holding only + `func.func @name(...)`; the CuTe compiler makes every public function a + kernel entry, so there is no `gpu.module` or host launch function to write. +* **Generic form.** `finish` and `finishProgram` print the generic operation + form: NVIDIA's compiler parses with its own MLIR revision, whose custom + assembly differs from ours (`llvm.load`/`llvm.store` alignment, for one). +* **Explicit result types:** Zig covers local tiling, TMA partitioning, tiled + copies, tensor-memory fragments, and SM100 block-scaled MMA. CuTe derives + their layouts in the compiler, so callers spell out the resulting CuTe type + with `View` instead of relying on Python's dynamic type objects. +* **TMA tilers follow Python.** `makeTiledTmaAtom` accepts a shape tuple, tile, + or layout, composes it with the GMEM identity layout, and removes the SMEM + stage mode when present. The lower-level `makeTiledTmaLoadAtom` and + `makeTiledTmaStoreAtom` take an already composed coordinate map. `copy` + makes store atoms executable automatically; loads require `tma_bar_ptr`. +* **Typed result types.** Operations whose result type CuTe derives + (`localTile`, `slice`, `tmaPartition`, ...) take an `*mlir.Type` assembled + with `layoutSpec`, `layoutType`, `memrefType`, `coordTensorType`, + `smemDescViewType` or `tiledCopyType`; nothing is parsed from text. +* **Architecture sections.** `Builder` groups TMA (`cute.nvgpu.cpasync`, + SM90+) and tcgen05 / tensor memory (`cute.nvgpu.tcgen05`, SM100) in their + own sections; mbarriers, fences and bulk-copy groups sit with `cute.arch`. + Each doc comment names its minimum architecture when it is above SM80. + +## Programs with a host launch + +Kernels that need launch-time CuTe objects (TMA descriptors, tiled MMA atoms, +scheduler parameters) use `zml.kernel.cute.Program` instead of `Kernel`. Its +`run` callback emits one `gpu.module` with the `cuda.kernel`s +(`b.beginFunction(name, .cuda_kernel)`), then a public host `func.func` +(`b.beginFunction(name, .host)`) that builds those objects from the XLA +buffers and calls `b.launchEx`. The custom call carries no grid or block: +XLA runs the host function with the real buffers and launches (or records in +a command buffer) whatever it launched. `CallOpts.scalars` are passed to the +host function after the buffers. `zml/kernels/moe/cute/persistent_mxfp4.zig` +is the reference: a warp-specialized SM100 block-scaled GEMM whose host +function builds four TMA tensor maps and a PDL launch. + +## The custom call + +`ops.cute` emits `__gpu$xla.gpu.cute` with a printed dictionary: `name`, +`kernel_type = "cute"`, `ir` (the textual module), `grid` and `block` +(3-element arrays), optional `zeroed_outputs`. XLA compiles it in process +(`lib/libcute_compiler.so`, which pulls in `lib/cutlass_ir.so`, both from the +sandbox pointed at by `--xla_gpu_cuda_data_dir`), takes the kernel's +dynamic shared memory requirement from the compiler's metadata, and launches +the cubin over `grid` × `block`. Every operand and result is one raw device +pointer in the default layout, operands first; the count is not checked +against the kernel signature, so `inputs`/`outputs` must match `declareArgs`. +A module that carries its own `gpu.module` and host launch still works; +it then owns the launch configuration and `grid`/`block` must be left out. + +To see what XLA received, run with `--xla_dump_to=`: it writes +`.cute.mlir`, `.cute.json` and `.cubin`. + +## Finding what exists + +```sh +# the DSL: Value's fluent methods (add, mul, lt, to, exp, ...) +awk '/^pub const Value = struct/,/^};/' dsl/cute/builder.zig | grep 'pub fn ' + +# the DSL: everything on Tensor and Builder +awk '/^pub const Tensor = struct/,/^};/' dsl/cute/builder.zig | grep 'pub fn ' +awk '/^pub const Builder = struct/,0' dsl/cute/builder.zig | grep ' pub fn ' + +# every bound op of the dialect +grep -n '^pub fn ' mlir/dialects/cute_ir/cute_ir.zig mlir/dialects/cute_nvgpu/cute_nvgpu.zig +``` + +`grep -n '^test "' dsl/cute/builder.zig` lists the tests; each prints a +kernel that round-trips through the verifier. diff --git a/dsl/cute/builder.zig b/dsl/cute/builder.zig new file mode 100644 index 00000000..8f791f80 --- /dev/null +++ b/dsl/cute/builder.zig @@ -0,0 +1,4953 @@ +const std = @import("std"); + +const arith = @import("mlir/dialects/arith"); +const cf = @import("mlir/dialects/cf"); +const common = @import("dsl/common/builder"); +const cuda = @import("mlir/dialects/cuda"); +const cute_ir = @import("mlir/dialects/cute_ir"); +const cute_nvgpu = @import("mlir/dialects/cute_nvgpu"); +const func = @import("mlir/dialects/func"); +const gpu = @import("mlir/dialects/gpu"); +const llvm = @import("mlir/dialects/llvm"); +const math = @import("mlir/dialects/math"); +const nvvm = @import("mlir/dialects/nvvm"); +const scf = @import("mlir/dialects/scf"); +const vector = @import("mlir/dialects/vector"); +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +/// `CUtensorMapDataType`. +pub const TensorMapDataType = enum(u32) { + uint8 = 0, + uint16 = 1, + uint32 = 2, + int32 = 3, + uint64 = 4, + int64 = 5, + float16 = 6, + float32 = 7, + float64 = 8, + bfloat16 = 9, + float32_ftz = 10, + tfloat32 = 11, + tfloat32_ftz = 12, + @"16u4_align8b" = 13, + @"16u4_align16b" = 14, + @"16u6_align16b" = 15, +}; + +/// `CUtensorMapSwizzle`. +pub const TensorMapSwizzle = enum(u32) { none = 0, @"32b" = 1, @"64b" = 2, @"128b" = 3, @"128b_atom_32b" = 4 }; +/// `CUtensorMapL2promotion`. +pub const TensorMapL2Promotion = enum(u32) { none = 0, @"64b" = 1, @"128b" = 2, @"256b" = 3 }; +/// `CUtensorMapInterleave`. +pub const TensorMapInterleave = enum(u32) { none = 0, @"16b" = 1, @"32b" = 2 }; +/// `CUtensorMapFloatOOBfill`. +pub const TensorMapOobFill = enum(u32) { none = 0, nan_request_zero_fma = 1 }; + +pub const TensorMapSpec = struct { + data_type: TensorMapDataType, + rank: u32, + global_dims: [5]u64 = @splat(1), + /// Byte strides of dimensions 1..rank-1. + global_strides: [4]u64 = @splat(0), + box_dims: [5]u32 = @splat(1), + element_strides: [5]u32 = @splat(1), + interleave: TensorMapInterleave = .none, + swizzle: TensorMapSwizzle, + l2_promotion: TensorMapL2Promotion = .none, + oob_fill: TensorMapOobFill = .none, +}; + +const dtypes = @import("dtype.zig"); +pub const DType = dtypes.DType; +const isFloatDtype = dtypes.isFloatDtype; +const dtypeBitwidth = dtypes.dtypeBitwidth; + +test { + std.testing.refAllDecls(@This()); + std.testing.refAllDecls(Builder); + std.testing.refAllDecls(Value); + std.testing.refAllDecls(Tensor); + std.testing.refAllDecls(View); + std.testing.refAllDecls(Atom); +} + +/// `nvvm` and `cuda` are NVIDIA's (`nvvm.ops`, `cuda.ops`, handles `cute_nvvm` +/// and `cuda`); `llvm` is upstream's, whose types and operations NVIDIA's +/// compiler reads as its own. +pub fn registerDialects(registry: *mlir.DialectRegistry) void { + inline for (.{ func, gpu, cute_ir, cute_nvgpu, nvvm, cuda, llvm, arith, scf, math, cf, vector }) |dialect| dialect.registerDialect(registry); +} + +pub const FinishError = error{InvalidMlir} || std.mem.Allocator.Error || std.Io.Writer.Error; + +pub const MAX_RANK = 8; +pub const Dims = stdx.BoundedArray(i64, MAX_RANK); + +pub const MemorySpace = cute_ir.MemorySpace; + +/// A CuTe algebra expression assembled from Zig values. Kernel authors use +/// `Builder.expr`, `Builder.basis`, and `Builder.layoutSpec`; the DSL builds +/// the algebra attribute of the kind each use needs. +pub const AlgebraExpr = union(enum) { + static: i64, + /// A dynamic leaf known to be a multiple of this, 1 when nothing is known. + dynamic: i64, + /// A dynamic integer with an explicit bit width and divisibility. + dynamic_int: struct { width: u8, divisible_by: i64 = 1 }, + all, + basis: ScaledBasis, + tuple: []const AlgebraExpr, +}; + +pub const AlgebraToken = enum { + /// CuTe's slice placeholder (`_`). + all, + /// A dynamic algebra leaf (`?`). + dynamic, +}; + +pub const ScaledBasis = struct { + numerator: i64, + denominator: i64 = 1, + modes: [MAX_RANK]u8 = @splat(0), + mode_count: u8, +}; + +/// A dynamic CuTe integer whose runtime value is known to be divisible by a +/// static amount. CuTe carries this fact in dependent coordinate-tensor types +/// after tiling and slicing, for example `?{div=128}`. +pub const ConstrainedDynamic = struct { + divisible_by: u64, +}; + +pub const LayoutSpec = struct { + shape: AlgebraExpr, + stride: AlgebraExpr, +}; + +/// A scalar SSA value (`Int32`, `Float32`, ... in the Python DSL). +pub const Value = struct { + inner: *const mlir.Value, + kernel: ?*Builder = null, + + pub fn type_(self: Value) *const mlir.Type { + return self.inner.type_(); + } + + fn kern(self: Value) *Builder { + return self.kernel orelse @panic("Value has no owning kernel; use Builder.* helpers instead"); + } + + pub fn dtype(self: Value) DType { + return dtypes.mlirToDType(self.kern().ctx, self.type_()); + } + + pub fn isFloat(self: Value) bool { + const t = self.type_(); + inline for (@typeInfo(mlir.FloatTypes).@"enum".field_names) |f| { + if (t.isA(mlir.FloatType(@field(mlir.FloatTypes, f))) != null) return true; + } + return false; + } + + pub fn isInt(self: Value) bool { + return self.type_().isA(mlir.IntegerType) != null; + } + + pub fn isPtr(self: Value) bool { + return self.type_().isA(cute_ir.PtrType) != null; + } + + pub fn add(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return k.emit(if (l.isFloat()) arith.addf(k.ctx, l.inner, r.inner, k.loc()) else arith.addi(k.ctx, l.inner, r.inner, k.loc())); + } + + pub fn sub(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return k.emit(if (l.isFloat()) arith.subf(k.ctx, l.inner, r.inner, k.loc()) else arith.subi(k.ctx, l.inner, r.inner, k.loc())); + } + + pub fn mul(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return k.emit(if (l.isFloat()) arith.mulf(k.ctx, l.inner, r.inner, k.loc()) else arith.muli(k.ctx, l.inner, r.inner, k.loc())); + } + + /// Signed division on integers (Python `//` on Int32). + pub fn div(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return k.emit(if (l.isFloat()) arith.divf(k.ctx, l.inner, r.inner, k.loc()) else arith.divsi(k.ctx, l.inner, r.inner, k.loc())); + } + + pub fn rem(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return k.emit(if (l.isFloat()) arith.remf(k.ctx, l.inner, r.inner, k.loc()) else arith.remsi(k.ctx, l.inner, r.inner, k.loc())); + } + + pub fn cdiv(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + if (l.isFloat()) @panic("Value.cdiv is integer-only"); + return k.emit(arith.ceildivsi(k.ctx, l.inner, r.inner, k.loc())); + } + + pub fn neg(self: Value) Value { + const k = self.kern(); + if (self.isFloat()) return k.emit(arith.negf(k.ctx, self.inner, k.loc())); + return k.constant(self.dtype(), 0).sub(self); + } + + pub fn abs(self: Value) Value { + const k = self.kern(); + return k.emit(if (self.isFloat()) math.absf(k.ctx, self.inner, k.loc()) else math.absi(k.ctx, self.inner, k.loc())); + } + + pub fn minimum(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return k.emit(if (l.isFloat()) arith.minimumf(k.ctx, l.inner, r.inner, k.loc()) else arith.minsi(k.ctx, l.inner, r.inner, k.loc())); + } + + pub fn maximum(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return k.emit(if (l.isFloat()) arith.maximumf(k.ctx, l.inner, r.inner, k.loc()) else arith.maxsi(k.ctx, l.inner, r.inner, k.loc())); + } + + /// Float max that ignores NaN operands (`arith.maxnumf`, a single `max.f32`), unlike + /// the NaN-propagating `maximum`. + pub fn maxnum(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return k.emit(arith.maxnumf(k.ctx, l.inner, r.inner, k.loc())); + } + + pub fn bitAnd(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return k.emit(arith.andi(k.ctx, l.inner, r.inner, k.loc())); + } + + pub fn bitOr(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return k.emit(arith.ori(k.ctx, l.inner, r.inner, k.loc())); + } + + pub fn bitXor(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return k.emit(arith.xori(k.ctx, l.inner, r.inner, k.loc())); + } + + pub fn shl(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return k.emit(arith.shli(k.ctx, l.inner, r.inner, k.loc())); + } + + pub fn shr(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return k.emit(arith.shrsi(k.ctx, l.inner, r.inner, k.loc())); + } + + pub fn shrLogical(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return k.emit(arith.shrui(k.ctx, l.inner, r.inner, k.loc())); + } + + pub fn bitCast(self: Value, dt: DType) Value { + const k = self.kern(); + if (dtypeBitwidth(self.dtype()) != dtypeBitwidth(dt)) @panic("Value.bitCast requires equal bit widths"); + return k.emit(arith.bitcast(k.ctx, self.inner, dt.toMlir(k.ctx), k.loc())); + } + + pub fn lt(self: Value, rhs: anytype) Value { + return self.kern().cmp(self, rhs, .slt, .olt); + } + + pub fn le(self: Value, rhs: anytype) Value { + return self.kern().cmp(self, rhs, .sle, .ole); + } + + pub fn gt(self: Value, rhs: anytype) Value { + return self.kern().cmp(self, rhs, .sgt, .ogt); + } + + pub fn ge(self: Value, rhs: anytype) Value { + return self.kern().cmp(self, rhs, .sge, .oge); + } + + pub fn eq(self: Value, rhs: anytype) Value { + return self.kern().cmp(self, rhs, .eq, .oeq); + } + + pub fn ne(self: Value, rhs: anytype) Value { + return self.kern().cmp(self, rhs, .ne, .une); + } + + pub fn exp(self: Value) Value { + const k = self.kern(); + return k.emit(math.exp(k.ctx, self.inner, k.loc())); + } + + pub fn log(self: Value) Value { + const k = self.kern(); + return k.emit(math.log(k.ctx, self.inner, k.loc())); + } + + pub fn log2(self: Value) Value { + const k = self.kern(); + return k.emit(math.log2(k.ctx, self.inner, k.loc())); + } + + pub fn exp2(self: Value) Value { + const k = self.kern(); + return k.emit(math.exp2(k.ctx, self.inner, k.loc())); + } + + pub fn ceil(self: Value) Value { + const k = self.kern(); + if (!self.isFloat()) @panic("Value.ceil is float-only"); + return k.emit(math.ceil(k.ctx, self.inner, k.loc())); + } + + pub fn sqrt(self: Value) Value { + const k = self.kern(); + return k.emit(math.sqrt(k.ctx, self.inner, k.loc())); + } + + pub fn rsqrt(self: Value) Value { + const k = self.kern(); + return k.emit(math.rsqrt(k.ctx, self.inner, k.loc())); + } + + pub fn tanh(self: Value) Value { + const k = self.kern(); + return k.emit(math.tanh(k.ctx, self.inner, k.loc())); + } + + /// Python `x.to(T)`: int <-> int is signed, int <-> float goes through + /// `sitofp`/`fptosi`, same-width floats through `convertf`. + pub fn to(self: Value, dt: DType) Value { + const k = self.kern(); + const src = self.dtype(); + if (src == dt) return self; + const ty = dt.toMlir(k.ctx); + const src_f = isFloatDtype(src); + const dst_f = isFloatDtype(dt); + const sw = dtypeBitwidth(src); + const dw = dtypeBitwidth(dt); + if (src_f and dst_f) { + if (sw == dw) return k.emit(arith.convertf(k.ctx, self.inner, ty, .{}, k.loc())); + return k.emit(if (dw > sw) arith.extf(k.ctx, self.inner, ty, k.loc()) else arith.truncf(k.ctx, self.inner, ty, k.loc())); + } + if (src_f) return k.emit(arith.fptosi(k.ctx, self.inner, ty, k.loc())); + if (dst_f) return k.emit(arith.sitofp(k.ctx, self.inner, ty, k.loc())); + if (sw == dw) return self; + if (src == .i1) return k.emit(arith.extui(k.ctx, self.inner, ty, k.loc())); + return k.emit(if (dw > sw) arith.extsi(k.ctx, self.inner, ty, k.loc()) else arith.trunci(k.ctx, self.inner, ty, k.loc())); + } +}; + +/// `cute.Layout`: a static shape and stride, carried in Zig so coordinates +/// and element counts need no MLIR queries. +pub const Layout = struct { + inner: *const mlir.Value, + shape: Dims, + stride: Dims, + + pub fn rank(self: Layout) usize { + return self.shape.len; + } + + pub fn size(self: Layout) i64 { + var n: i64 = 1; + for (self.shape.constSlice()) |d| n *= d; + return n; + } + + pub fn dim(self: Layout, i: usize) i64 { + return self.shape.get(i); + } +}; + +/// `cute.Tensor`: an engine (pointer) composed with a layout. `get`/`set` +/// are the Python `t[coord]` / `t[coord] = v`. +pub const Tensor = struct { + inner: *const mlir.Value, + kernel: *Builder, + layout: Layout, + dtype: DType, + space: MemorySpace, + + pub fn shape(self: Tensor) []const i64 { + return self.layout.shape.constSlice(); + } + + pub fn rank(self: Tensor) usize { + return self.layout.rank(); + } + + pub fn size(self: Tensor) i64 { + return self.layout.size(); + } + + pub fn dim(self: Tensor, i: usize) i64 { + return self.layout.dim(i); + } + + /// `t[coord]`: `coord` is a tuple of `Value`s and ints, one per mode. + pub fn get(self: Tensor, coord: anytype) Value { + const k = self.kernel; + const c = k.makeCoord(coord); + return k.emit(cute_ir.memref_load(k.ctx, self.inner, c, self.dtype.toMlir(k.ctx), k.loc())); + } + + /// `t[coord] = value`; the value is converted to the element type. + pub fn set(self: Tensor, coord: anytype, value_: anytype) void { + const k = self.kernel; + const c = k.makeCoord(coord); + const v = k.constant(self.dtype, value_).to(self.dtype); + _ = cute_ir.memref_store(k.ctx, self.inner, c, v.inner, k.loc()).appendTo(k.currentBlock()); + } + + /// Erase Zig's tracked layout while retaining the MLIR memref. CuTe + /// layout algebra derives the layouts returned by tiling/partitioning; + /// those values are represented by `View` because Zig cannot recompute + /// their dependent types. + pub fn view(self: Tensor) View { + return .{ .inner = self.inner, .kernel = self.kernel }; + } + + pub fn value(self: Tensor) Value { + return .{ .inner = self.inner, .kernel = self.kernel }; + } +}; + +/// A CuTe tensor or pointer with a layout derived by the CuTe compiler. +pub const View = struct { + inner: *const mlir.Value, + kernel: *Builder, + + pub fn type_(self: View) *const mlir.Type { + return self.inner.type_(); + } + + pub fn value(self: View) Value { + return .{ .inner = self.inner, .kernel = self.kernel }; + } + + pub fn get(self: View, coord: anytype, result_dtype: DType) Value { + const c = self.kernel.makeCoord(coord); + return self.kernel.emit(cute_ir.memref_load(self.kernel.ctx, self.inner, c, result_dtype.toMlir(self.kernel.ctx), self.kernel.loc())); + } + + pub fn set(self: View, coord: anytype, value_: Value) void { + self.kernel.emitVoid(cute_ir.memref_store(self.kernel.ctx, self.inner, self.kernel.makeCoord(coord), value_.inner, self.kernel.loc())); + } +}; + +/// A copy/MMA atom or a tiled atom. Atom fields are immutable SSA values; +/// `setField` returns the atom carrying the updated runtime field. +pub const Atom = struct { + inner: *const mlir.Value, + kernel: *Builder, + + pub fn type_(self: Atom) *const mlir.Type { + return self.inner.type_(); + } + + pub fn value(self: Atom) Value { + return .{ .inner = self.inner, .kernel = self.kernel }; + } + + pub fn setField(self: Atom, field: []const u8, value_: anytype) Atom { + const T = @TypeOf(value_); + const new_value = if (T == Value or T == Tensor or T == View or T == Atom or T == Layout) + self.kernel.asValue(value_) + else + self.kernel.lift(value_); + const result = self.kernel.emit(cute_nvgpu.atom_set_value( + self.kernel.ctx, + self.inner, + new_value.inner, + self.type_(), + .string(self.kernel.ctx, field), + self.kernel.loc(), + )); + return .{ .inner = result.inner, .kernel = self.kernel }; + } +}; + +pub const TmaDescriptor = struct { + atom: Atom, + tensor: View, +}; + +pub const TmaPartition = struct { + smem: View, + targets: []const View, +}; + +/// Element encoding of a TMA transfer when it differs from the tensor dtype. +pub const TmaFormat = enum { + default, + /// Packed 4-bit elements, two per byte in shared memory (NVFP4 operands + /// of the `mxf4nvf4` MMA). + u4, + /// 4-bit elements expanded to one per byte in shared memory (FP4 operands + /// of the `mxf8f6f4` MMA). + u4_unpack_u8, + u16, +}; + +/// CuTe's 64-bit TMA `.L2::cache_hint` policies for SM90. +pub const CacheHintSm90 = enum(u64) { + evict_normal = 0x1000000000000000, + evict_first = 0x12F0000000000000, + evict_last = 0x14F0000000000000, +}; + +/// CuTe's 64-bit TMA `.L2::cache_hint` policies for SM100. +pub const CacheHintSm100 = enum(u64) { + evict_normal = 0x1000000000000000, + evict_first = 0x12F0000000000000, + evict_last = 0x14F0000000000000, +}; + +pub const CtaGroup = enum(u8) { + one = 1, + two = 2, + + fn toNvvm(self: CtaGroup) nvvm.CtaGroup { + return switch (self) { + .one => .cta_1, + .two => .cta_2, + }; + } +}; +pub const MBarrierScope = nvvm.MBarrierScope; +pub const OperandMajorMode = enum(u1) { k = 0, mn = 1 }; + +/// Number of CTAs cooperating on one instruction, as in CuTe's CtaGroup.value. +pub fn ctaCount(group: CtaGroup) u8 { + return @backingInt(group); +} + +pub const Tcgen05SmemSwizzle = enum(u3) { + none = 0, + swizzle_128b_atom_32b = 1, + swizzle_128b = 2, + swizzle_64b = 4, + swizzle_32b = 6, +}; + +/// CuTe's packed SM100 shared-memory descriptor. All offsets are in bytes. +pub const Tcgen05SmemDesc = struct { + pub const Options = struct { + leading_byte_offset: u32 = 0, + stride_byte_offset: u32 = 0, + version: u2 = 1, + base_offset: u3 = 0, + layout: Tcgen05SmemSwizzle = .none, + }; + + pub fn build(b: *Builder, start_address: Value, opts: Options) Value { + std.debug.assert(opts.leading_byte_offset % 16 == 0 and opts.leading_byte_offset < 0x40000); + std.debug.assert(opts.stride_byte_offset % 16 == 0 and opts.stride_byte_offset < 0x40000); + const fields = (@as(u64, opts.leading_byte_offset >> 4) << 16) | + (@as(u64, opts.stride_byte_offset >> 4) << 32) | + (@as(u64, opts.version) << 46) | + (@as(u64, opts.base_offset) << 49) | + (@as(u64, @backingInt(opts.layout)) << 61); + const address = if (start_address.isInt()) start_address else b.ptrToInt(start_address, .i32); + return address.to(.i64).shrLogical(4).bitAnd(0x3fff).bitOr(b.constant(.i64, @as(i64, @bitCast(fields)))); + } +}; + +/// Dense SM100 instruction descriptor; the MMA kind disambiguates dtype encodings. +pub const Tcgen05InstrDesc = struct { + pub const Options = struct { + a_dtype: DType, + b_dtype: DType, + c_dtype: DType = .f32, + m_dim: u16, + n_dim: u16, + a_major: OperandMajorMode = .k, + b_major: OperandMajorMode = .k, + a_negate: bool = false, + b_negate: bool = false, + }; + + pub fn build(comptime opts: Options) u32 { + comptime std.debug.assert(opts.m_dim > 0 and opts.m_dim <= 256 and opts.m_dim % 16 == 0); + comptime std.debug.assert(opts.n_dim > 0 and opts.n_dim <= 256 and opts.n_dim % 8 == 0); + const half_inputs = (opts.a_dtype == .f16 or opts.a_dtype == .bf16) and (opts.b_dtype == .f16 or opts.b_dtype == .bf16); + const fp8_inputs = (opts.a_dtype == .f8e4m3fn or opts.a_dtype == .f8e5m2) and (opts.b_dtype == .f8e4m3fn or opts.b_dtype == .f8e5m2); + const valid = (half_inputs and opts.c_dtype == .f32) or + (opts.a_dtype == .f16 and opts.b_dtype == .f16 and opts.c_dtype == .f16) or + (opts.a_dtype == .f32 and opts.b_dtype == .f32 and opts.c_dtype == .f32) or + (fp8_inputs and (opts.c_dtype == .f16 or opts.c_dtype == .f32)) or + (opts.a_dtype == .i8 and opts.b_dtype == .i8 and opts.c_dtype == .i32); + if (!valid) @compileError("unsupported dense tcgen05 dtype combination"); + const c: u32 = switch (opts.c_dtype) { + .f16 => 0, + .f32 => 1, + .i32 => 2, + else => @compileError("unsupported tcgen05 accumulator dtype"), + }; + return (c << 4) | (inputFormat(opts.a_dtype) << 7) | (inputFormat(opts.b_dtype) << 10) | + (@as(u32, @intFromBool(opts.a_negate)) << 13) | + (@as(u32, @intFromBool(opts.b_negate)) << 14) | + (@as(u32, @backingInt(opts.a_major)) << 15) | + (@as(u32, @backingInt(opts.b_major)) << 16) | + (@as(u32, opts.n_dim >> 3) << 17) | (@as(u32, opts.m_dim >> 4) << 24); + } + + fn inputFormat(comptime dtype: DType) u32 { + return switch (dtype) { + .f16, .f8e4m3fn => 0, + .bf16, .f8e5m2, .i8 => 1, + .f32 => 2, + .f4e2m1fn => 5, + else => @compileError("unsupported tcgen05 input dtype"), + }; + } +}; + +/// Typed description of a non-multicast tiled TMA atom: the element type, +/// bits moved per copy, basis of one box, and coordinate layout of the tensor. +pub const TmaConfig = struct { + dtype: DType, + copy_bits: u32, + global_basis: LayoutSpec, + coordinate_layout: LayoutSpec, + coordinate_rank: u8, + format: TmaFormat = .default, + /// Load completion group: with `.two`, each CTA's load completes on + /// the mbarrier of the pair's even CTA. Ignored by TMA stores. + cta_group: CtaGroup = .one, + mode: enum { tiled, gather4 } = .tiled, +}; + +pub const CopyBulkTensorTileG2SOp = struct { cta_group: CtaGroup = .one }; +pub const CopyBulkTensor2DGather4G2SOp = struct { cta_group: CtaGroup = .one }; +pub const CopyBulkTensorTileS2GOp = struct {}; + +pub const BlockScaledMma = struct { + atom: Atom, + tiled: Atom, +}; + +/// SS, K-major SM100 block-scaled MMA instruction with FP4 weights as A. +/// The instruction `kind` fixes B, the scales and K. +pub const BlockScaledMmaConfig = struct { + kind: Kind = .mxfp4_mxfp8, + /// M is 128 for one CTA, or 128 or 256 for a CTA pair. + m: u16 = 128, + n: u16, + cta_group: CtaGroup = .one, + + pub const Kind = enum { + /// MXFP4 weights x MXFP8 activations: E8M0 scales every 32 + /// elements, K = 32 (PTX `kind::mxf8f6f4`). + mxfp4_mxfp8, + /// NVFP4 x NVFP4: E4M3 scales every 16 elements, K = 64 (PTX + /// `kind::mxf4nvf4.block16`). + nvfp4_nvfp4, + }; + + pub fn k(self: BlockScaledMmaConfig) u16 { + return switch (self.kind) { + .mxfp4_mxfp8 => 32, + .nvfp4_nvfp4 => 64, + }; + } + + fn bType(self: BlockScaledMmaConfig) DType { + return switch (self.kind) { + .mxfp4_mxfp8 => .f8e4m3fn, + .nvfp4_nvfp4 => .f4e2m1fn, + }; + } + + fn scaleType(self: BlockScaledMmaConfig) DType { + return switch (self.kind) { + .mxfp4_mxfp8 => .f8e8m0fnu, + .nvfp4_nvfp4 => .f8e4m3fn, + }; + } + + fn vecSize(self: BlockScaledMmaConfig) u8 { + return switch (self.kind) { + .mxfp4_mxfp8 => 32, + .nvfp4_nvfp4 => 16, + }; + } + + fn validate(self: BlockScaledMmaConfig) void { + const m_ok = self.m == 128 or (self.cta_group == .two and self.m == 256); + const n_multiple: u16 = if (self.cta_group == .two) 16 else 8; + if (!m_ok or self.n < n_multiple or self.n > 256 or self.n % n_multiple != 0) { + std.debug.panic("unsupported SM100 block-scaled MMA shape {d}x{d}x{d} on {d} CTAs", .{ self.m, self.n, self.k(), ctaCount(self.cta_group) }); + } + } +}; + +fn tmaLoadKind(config: TmaConfig) cute_nvgpu.TiledTmaLoad { + return switch (config.cta_group) { + .one => .sm_90, + .two => .sm_100_2sm, + }; +} + +fn tmaDataFormat(format: TmaFormat, dtype: DType) cute_nvgpu.TmaDataFormat { + return switch (format) { + .u4 => .U4, + .u4_unpack_u8 => .U4_UNPACK_U8, + .u16 => .U16, + .default => switch (dtype) { + .f8e4m3fn => .U8, + .f32 => .F32_RN, + else => @panic("TMA format must be specified for this dtype"), + }, + }; +} + +fn tmaFormatAttribute(b: *Builder, config: TmaConfig) ?*const mlir.Attribute { + if (config.format == .default) return null; + const attr = cute_nvgpu.TmaDataFormatAttr.get(b.ctx, .{ .value = tmaDataFormat(config.format, config.dtype) }) catch @panic("invalid TMA format"); + return attr.attribute(); +} + +pub const Index3 = struct { x: Value, y: Value, z: Value }; + +pub const LaunchConfig = struct { + inner: *const mlir.Value, + kernel: *Builder, + + pub fn value(self: LaunchConfig) Value { + return .{ .inner = self.inner, .kernel = self.kernel }; + } +}; + +pub const ArgSpec = struct { + name: []const u8, + kind: Kind, + + /// XLA passes one raw device pointer per custom-call argument. A `tensor` + /// argument is that pointer composed with a static layout at the top of + /// the kernel, the way the Python DSL passes `cute.Tensor` arguments. + pub const Kind = union(enum) { + ptr: DType, + tensor: TensorSpec, + /// A type constructed through a dialect/type API. + mlir_type: MlirTypeSpec, + }; + + pub const MlirTypeSpec = struct { + type_: *const mlir.Type, + grid_constant: bool = false, + cuda_grid_constant: bool = false, + }; + + pub const TensorSpec = struct { + dtype: DType, + shape: []const i64, + /// Defaults to XLA's row-major layout. + stride: ?[]const i64 = null, + alignment: u64 = 16, + }; +}; + +pub const LayoutOpts = struct { + /// Defaults to CuTe's compact left-most stride, like `cute.make_layout`. + stride: ?[]const i64 = null, +}; + +pub const IfOnlyScope = common.IfOnlyScope(Builder, Value); + +pub fn IfScope(comptime N: usize) type { + return common.IfScope(Builder, Value, N); +} + +pub fn ForScope(comptime N: usize) type { + return common.ForScope(Builder, Value, N); +} + +pub fn WhileScope(comptime N: usize, comptime M: usize) type { + return common.WhileScope(Builder, Value, N, M); +} + +pub const Builder = struct { + pub const FunctionKind = enum { func_only_kernel, cuda_kernel, host }; + + allocator: std.mem.Allocator, + arena: std.heap.ArenaAllocator, + ctx: *mlir.Context, + module: *mlir.Module, + gpu_body: ?*mlir.Block = null, + name: []const u8, + function_kind: FunctionKind = .func_only_kernel, + func_op: ?*mlir.Operation = null, + entry_block: ?*mlir.Block = null, + block_stack: common.BlockStack = .{}, + args: []const ArgSpec = &.{}, + tensors: []?Tensor = &.{}, + + pub fn open(allocator: std.mem.Allocator, ctx: *mlir.Context, name: []const u8) !Builder { + const module: *mlir.Module = .init(.unknown(ctx)); + errdefer module.deinit(); + return .{ + .allocator = allocator, + .arena = .init(allocator), + .ctx = ctx, + .module = module, + .name = name, + }; + } + + /// Open a complete CuTe program. Unlike `open`, which emits the compact + /// func.func-only ABI consumed directly by XLA, a program owns a + /// `gpu.module` for one or more `cuda.kernel` operations and can append a + /// host `func.func` that constructs CuTe launch objects and calls + /// `cuda.launch_ex`. + pub fn openProgram(allocator: std.mem.Allocator, ctx: *mlir.Context, name: []const u8) !Builder { + var self = try open(allocator, ctx, name); + self.module.operation().setAttribute(gpu.containerModule(ctx)); + const body = mlir.Block.init(&.{}, &.{}); + const gpu_module = gpu.module(ctx, "kernels", body, self.loc()); + _ = gpu_module.appendTo(self.module.body()); + self.gpu_body = body; + self.function_kind = .cuda_kernel; + return self; + } + + /// Start another function in a complete program after `endFunction`. + pub fn beginFunction(self: *Builder, name: []const u8, kind: FunctionKind) void { + std.debug.assert(self.entry_block == null and self.func_op == null); + self.name = name; + self.function_kind = kind; + } + + pub fn deinit(self: *Builder) void { + self.module.deinit(); + self.arena.deinit(); + } + + /// One field per kernel argument: `.{ .a = .{ .tensor = .{ .dtype = .f32, .shape = &.{n} } } }` + /// yields a `Tensor`, `.{ .p = .{ .ptr = .f32 } }` a pointer `Value`. + pub fn declareArgs(self: *Builder, spec: anytype) FinishError!ArgsOf(@TypeOf(spec)) { + const Spec = @TypeOf(spec); + const fields = @typeInfo(Spec).@"struct".field_names; + + const arg_specs = try self.arena.allocator().alloc(ArgSpec, fields.len); + inline for (fields, 0..) |f, i| { + const raw = @field(spec, f); + const variant = @typeInfo(@TypeOf(raw)).@"struct".field_names[0]; + const inner = @field(raw, variant); + arg_specs[i] = .{ .name = f, .kind = switch (@field(std.meta.Tag(ArgSpec.Kind), variant)) { + .ptr => .{ .ptr = inner }, + .tensor => .{ .tensor = .{ + .dtype = inner.dtype, + .shape = inner.shape, + .stride = if (@hasField(@TypeOf(inner), "stride")) inner.stride else null, + .alignment = if (@hasField(@TypeOf(inner), "alignment")) inner.alignment else 16, + } }, + .mlir_type => .{ .mlir_type = .{ + .type_ = inner.type_, + .grid_constant = if (@hasField(@TypeOf(inner), "grid_constant")) inner.grid_constant else false, + .cuda_grid_constant = if (@hasField(@TypeOf(inner), "cuda_grid_constant")) inner.cuda_grid_constant else false, + } }, + } }; + } + try self.declareArgsLow(arg_specs); + + var named: ArgsOf(Spec) = undefined; + inline for (fields, 0..) |f, i| { + @field(named, f) = switch (@FieldType(ArgsOf(Spec), f)) { + Tensor => self.tensors[i].?, + else => self.arg(i), + }; + } + return named; + } + + fn ArgsOf(comptime Spec: type) type { + const in = @typeInfo(Spec).@"struct".field_names; + comptime var names: [in.len][]const u8 = undefined; + comptime var types: [in.len]type = undefined; + inline for (in, 0..) |f, i| { + names[i] = f; + types[i] = if (@hasField(@FieldType(Spec, f), "tensor")) Tensor else Value; + } + return @Struct(.auto, null, &names, &types, &@splat(.{})); + } + + pub fn declareArgsLow(self: *Builder, args: []const ArgSpec) FinishError!void { + std.debug.assert(self.entry_block == null); + const ctx = self.ctx; + const scratch = self.arena.allocator(); + + const arg_types = try scratch.alloc(*const mlir.Type, args.len); + const arg_locs = try scratch.alloc(*const mlir.Location, args.len); + for (arg_types, arg_locs, args) |*ty, *l, a| { + ty.* = switch (a.kind) { + .ptr => |dt| self.ptrTy(dt, .gmem, 16), + .tensor => |t| self.ptrTy(t.dtype, .gmem, t.alignment), + .mlir_type => |typed| typed.type_, + } catch return error.InvalidMlir; + l.* = self.loc(); + } + + const entry = mlir.Block.init(arg_types, arg_locs); + const empty_dict: *const mlir.Attribute = .dict(ctx, &.{}); + const arg_attrs = try scratch.alloc(*const mlir.Attribute, args.len); + var any_arg_attr = false; + for (arg_attrs, args) |*attr, a| { + attr.* = switch (a.kind) { + .mlir_type => |typed| if (typed.cuda_grid_constant) .dict(ctx, &.{.named(ctx, "cuda.grid_constant", .unit(ctx))}) else if (typed.grid_constant) .dict(ctx, &.{cute_nvgpu.gridConstant(ctx)}) else empty_dict, + else => empty_dict, + }; + if (attr.* != empty_dict) any_arg_attr = true; + } + + const func_op = switch (self.function_kind) { + .func_only_kernel => func.func(ctx, .{ + .name = self.name, + .block = entry, + .args_attributes = if (any_arg_attr) arg_attrs else null, + .results = &.{}, + .location = self.loc(), + .verify = false, + .extra_attributes = &.{ + cute_ir.kernel(ctx), + gpu.kernel(ctx), + }, + }), + .cuda_kernel => cuda.kernel(ctx, .{ + .name = self.name, + .function_type = .function(ctx, arg_types, &.{}), + .body = entry, + .arg_attrs = .array(ctx, arg_attrs), + .extra_attributes = &.{ + cute_ir.kernel(ctx), + gpu.kernel(ctx), + }, + .location = self.loc(), + }), + .host => func.func(ctx, .{ + .name = self.name, + .block = entry, + .args_attributes = if (any_arg_attr) arg_attrs else null, + .results = &.{.int(ctx, .i32)}, + .location = self.loc(), + .verify = false, + .extra_attributes = &.{llvm.emitCInterface(ctx)}, + }), + }; + _ = func_op.appendTo(if (self.function_kind == .cuda_kernel) + self.gpu_body orelse @panic("cuda_kernel requires Builder.openProgram") + else + self.module.body()); + + self.func_op = func_op; + self.entry_block = entry; + self.block_stack.push(self.arena.allocator(), entry); + self.args = args; + + self.tensors = try scratch.alloc(?Tensor, args.len); + for (self.tensors, args, 0..) |*t, a, i| { + t.* = switch (a.kind) { + .ptr => null, + .tensor => |spec| self.makeTensor(self.arg(i), self.makeLayout(spec.shape, .{ + .stride = spec.stride orelse self.rowMajor(spec.shape), + })), + .mlir_type => null, + }; + } + } + + pub fn arg(self: *Builder, i: usize) Value { + const eb = self.entry_block orelse @panic("Builder.arg called before declareArgs"); + return .{ .inner = eb.argument(i), .kernel = self }; + } + + pub fn pushBlock(self: *Builder, b: *mlir.Block) void { + self.block_stack.push(self.arena.allocator(), b); + } + + pub fn popBlock(self: *Builder) void { + self.block_stack.pop(); + } + + pub fn currentBlock(self: *Builder) *mlir.Block { + return self.block_stack.current(); + } + + pub fn emit(self: *Builder, op: *mlir.Operation) Value { + _ = op.appendTo(self.currentBlock()); + return .{ .inner = op.result(0), .kernel = self }; + } + + /// Append an operation which does not return an SSA value. + pub fn emitVoid(self: *Builder, op: *mlir.Operation) void { + std.debug.assert(op.numResults() == 0); + _ = op.appendTo(self.currentBlock()); + } + + /// Append a multi-result operation and return all of its SSA values. + /// + /// Most of the small CuTe kernels only need one-result operations, but + /// TMA descriptor construction and TMA partitioning return an atom and a + /// coordinate tensor together. Keeping both results from one operation + /// is required: rebuilding either value changes the descriptor identity. + pub fn emitResults(self: *Builder, comptime N: usize, op: *mlir.Operation) [N]Value { + std.debug.assert(op.numResults() == N); + _ = op.appendTo(self.currentBlock()); + var result: [N]Value = undefined; + for (&result, 0..) |*value, i| value.* = .{ .inner = op.result(i), .kernel = self }; + return result; + } + + /// Convert any DSL handle into a scalar wrapper without changing its + /// MLIR type. This keeps the low-level generated bindings usable while + /// callers work with `Tensor`, `View`, and `Atom` handles. + pub fn asValue(self: *Builder, value_: anytype) Value { + return switch (@TypeOf(value_)) { + Value => value_, + Tensor => value_.value(), + View => value_.value(), + Atom => value_.value(), + LaunchConfig => value_.value(), + Layout => .{ .inner = value_.inner, .kernel = self }, + else => @compileError("expected a CuTe DSL handle, got " ++ @typeName(@TypeOf(value_))), + }; + } + + fn innerSlice(self: *Builder, values: anytype) []const *const mlir.Value { + const out = self.arena.allocator().alloc(*const mlir.Value, values.len) catch @panic("OOM"); + switch (@typeInfo(@TypeOf(values))) { + .@"struct" => { + inline for (values, 0..) |value, i| out[i] = self.asValue(value).inner; + }, + else => { + for (values, out) |value, *dst| dst.* = self.asValue(value).inner; + }, + } + return out; + } + + pub fn loc(self: *const Builder) *const mlir.Location { + return .unknown(self.ctx); + } + + /// `#cuda.dev_max_shared_memory_optin`, the device's opt-in maximum of + /// shared memory per block, for a kernel's `cu_attrs`. + pub fn devMaxSharedMemoryOptin(self: *Builder) *const mlir.Attribute { + return (cuda.ops.DevMaxSharedMemoryOptinAttr.get(self.ctx, .{}) catch @panic("invalid CUDA attribute")).attribute(); + } + + pub fn setFunctionAttribute(self: *Builder, attribute: mlir.NamedAttribute) void { + const function = self.func_op orelse @panic("setFunctionAttribute called before declareArgs"); + function.setAttribute(attribute); + } + + // ==================== cuda host launch ==================== + + /// `@kernels::@`, a kernel of the nested GPU module. + fn kernelSymbol(self: *Builder, kernel_name: []const u8) *const mlir.Attribute { + return .symbolRef(self.ctx, "kernels", &.{.init(self.ctx, kernel_name)}); + } + + pub fn kernelSmemSize(self: *Builder, kernel_name: []const u8) Value { + return self.emit(cute_ir.kernel_smem_size( + self.ctx, + .int(self.ctx, .i64), + self.kernelSymbol(kernel_name), + self.loc(), + )); + } + + /// Convert the stream placeholder used by a generated host entry into a + /// CUDA stream. The native CuTe host bridge substitutes XLA's current + /// stream when it executes the wrapper. + pub fn cudaStream(self: *Builder) Value { + return self.emit(cuda.cast(self.ctx, cuda.streamType(self.ctx), self.constant(.i64, 0).inner, self.loc())); + } + + pub const LaunchOptions = struct { + grid: [3]Value, + block: [3]Value, + dynamic_smem: Value, + stream: Value, + cluster: ?[3]Value = null, + cooperative: bool = false, + use_pdl: bool = false, + }; + + /// Build the mutable CUDA launch configuration used by `cuda.launch_ex`. + pub fn makeLaunchConfig(self: *Builder, opts: LaunchOptions) LaunchConfig { + const config = self.emit(cuda.launch_cfg_create(self.ctx, .{ + .max_attrs = 17, + .block = .{ opts.block[0].inner, opts.block[1].inner, opts.block[2].inner }, + .dynamic_smem = opts.dynamic_smem.inner, + .grid = .{ opts.grid[0].inner, opts.grid[1].inner, opts.grid[2].inner }, + .stream = opts.stream.inner, + .location = self.loc(), + })); + if (opts.cluster) |cluster| { + self.emitVoid(cuda.launch_cfg_cluster_dim(self.ctx, config.inner, .{ cluster[0].inner, cluster[1].inner, cluster[2].inner }, self.loc())); + } + self.emitVoid(cuda.launch_cfg_cooperative(self.ctx, config.inner, self.constant(.i32, @intFromBool(opts.cooperative)).inner, self.loc())); + self.emitVoid(cuda.launch_cfg_programmatic_stream_serialization_allowed(self.ctx, config.inner, self.constant(.i32, @intFromBool(opts.use_pdl)).inner, self.loc())); + return .{ .inner = config.inner, .kernel = self }; + } + + /// Request a CUDA kernel launch from a generated host function. The host + /// bridge records this request; XLA performs or command-buffer-records the + /// actual launch on its stream. + pub fn launchEx(self: *Builder, kernel_name: []const u8, config: LaunchConfig, arguments: anytype) Value { + const values = self.innerSlice(arguments); + return self.emit(cuda.launch_ex(self.ctx, .{ + .config = config.inner, + .inputs = values, + .callee = self.kernelSymbol(kernel_name), + .assume_kernel_attr = cuda.assumeKernelAttr(self.ctx, true), + .location = self.loc(), + })); + } + + /// `!cuda.tensor_map`: the kernel argument type of an explicitly encoded map. + pub fn tensorMapType(self: *Builder) *const mlir.Type { + return (cuda.ops.TensorMapType.get(self.ctx, .{}) catch unreachable).type_(); + } + + /// A host stack array of `values` (u64 -> i64 slots, u32 -> i32 slots): a `cuuint64_t*` / + /// `cuuint32_t*` argument of `cuTensorMapEncodeTiled`. + fn tensorMapHostArray(self: *Builder, comptime T: type, values: []const T) Value { + const ctx = self.ctx; + const elem: *const mlir.Type = if (T == u64) .int(ctx, .i64) else .int(ctx, .i32); + const ptr_type = llvm.pointerType(ctx, llvm.NVPTXAddressSpace.generic); + const array = self.emit(mlir.Operation.make(ctx, "llvm.alloca", .{ + .operands = .{ .flat = &.{self.constant(.i32, @as(i32, @intCast(@max(values.len, 1)))).inner} }, + .results = .{ .flat = &.{ptr_type} }, + .attributes = &.{ .named(ctx, "elem_type", .typeAttr(elem)), .named(ctx, "alignment", .int(ctx, .i64, 8)) }, + .location = self.loc(), + })); + for (values, 0..) |v, i| { + const slot = self.emit(mlir.Operation.make(ctx, "llvm.getelementptr", .{ + .operands = .{ .flat = &.{array.inner} }, + .results = .{ .flat = &.{ptr_type} }, + .attributes = &.{ .named(ctx, "elem_type", .typeAttr(elem)), .named(ctx, "rawConstantIndices", .denseArray(ctx, .i32, &[_]i32{@intCast(i)})) }, + .location = self.loc(), + })); + const value = if (T == u64) self.constant(.i64, @as(i64, @intCast(v))) else self.constant(.i32, @as(i32, @intCast(v))); + self.emitVoid(llvm.store(ctx, value.inner, slot.inner, .{ .alignment = @sizeOf(T) }, self.loc())); + } + return array; + } + + /// Encode a tiled map in a host function and return both the map and CUDA status. + /// The caller supplies the final device address and handles the returned status. + pub fn encodeTensorMap(self: *Builder, spec: TensorMapSpec, address: Value) struct { map: Value, result: Value } { + const ctx = self.ctx; + // The `cuda` dialect's address operand is a `ptr` dialect pointer, which zml does not register. + ctx.setAllowUnregisteredDialects(true); + const addr = self.emit(cuda.cast(ctx, (mlir.Type.parse(self.ctx, "!ptr.ptr<#ptr.generic_space>") catch @panic("invalid tensor-map IR")), address.inner, self.loc())); + const results = self.emitResults(2, cuda.ops.tensor_map_encode_tiled( + ctx, + self.constant(.i32, @as(i32, @intCast(@backingInt(spec.data_type)))).inner, + self.constant(.i32, @as(i32, @intCast(spec.rank))).inner, + addr.inner, + self.tensorMapHostArray(u64, spec.global_dims[0..spec.rank]).inner, + self.tensorMapHostArray(u64, spec.global_strides[0 .. spec.rank - 1]).inner, + self.tensorMapHostArray(u32, spec.box_dims[0..spec.rank]).inner, + self.tensorMapHostArray(u32, spec.element_strides[0..spec.rank]).inner, + self.constant(.i32, @as(i32, @intCast(@backingInt(spec.interleave)))).inner, + self.constant(.i32, @as(i32, @intCast(@backingInt(spec.swizzle)))).inner, + self.constant(.i32, @as(i32, @intCast(@backingInt(spec.l2_promotion)))).inner, + self.constant(.i32, @as(i32, @intCast(@backingInt(spec.oob_fill)))).inner, + cuda.resultType(ctx), + self.tensorMapType(), + self.loc(), + )); + return .{ .map = results[1], .result = results[0] }; + } + + /// Device side: the generic address (i64) of the `!cuda.tensor_map` kernel argument `arg` + /// (`&tensor_map` for a `__grid_constant__` parameter). + pub fn tensorMapAddress(self: *Builder, descriptor: Value) Value { + const ctx = self.ctx; + const llvm_ptr = self.emit(cuda.cast(ctx, llvm.pointerType(ctx, llvm.NVPTXAddressSpace.generic), descriptor.inner, self.loc())); + return self.emit(mlir.Operation.make(ctx, "llvm.ptrtoint", .{ + .operands = .{ .flat = &.{llvm_ptr.inner} }, + .results = .{ .flat = &.{DType.i64.toMlir(ctx)} }, + .location = self.loc(), + })); + } + + pub fn cudaResultStatus(self: *Builder, result: Value) Value { + return self.emit(cuda.cast(self.ctx, .int(self.ctx, .i32), result.inner, self.loc())); + } + + pub fn returnHostStatus(self: *Builder, status: Value) void { + self.emitVoid(func.returns(self.ctx, &.{status.inner}, self.loc())); + } + + /// Finish the current function but retain the surrounding module so a + /// host launch function can be emitted next. + pub fn endFunction(self: *Builder, block: ?[3]i32) void { + const current = self.currentBlock(); + if (current.terminator() == null) { + const terminator = if (self.function_kind == .cuda_kernel) + cuda.return_(self.ctx, &.{}, self.loc()) + else + func.returns(self.ctx, &.{}, self.loc()); + _ = terminator.appendTo(current); + } + if (block) |dims| { + const function = self.func_op orelse @panic("endFunction before declareArgs"); + function.setAttribute(nvvm.reqntid(self.ctx, dims)); + } + self.func_op = null; + self.entry_block = null; + self.block_stack.clear(); + self.args = &.{}; + self.tensors = &.{}; + } + + pub fn finishProgram(self: *Builder) FinishError![:0]const u8 { + std.debug.assert(self.entry_block == null and self.func_op == null); + try common.verify(self.module, "dsl/cute"); + return self.renderModule(); + } + + fn renderModule(self: *Builder) FinishError![:0]const u8 { + var printed: std.Io.Writer.Allocating = .init(self.allocator); + defer printed.deinit(); + // Always the generic form: NVIDIA's compiler parses the module with its + // own MLIR revision, whose custom assembly differs from ours (e.g. + // `cute.make_composed_layout`, or `llvm.load`/`llvm.store` printing + // their alignment as a `` property). + try printed.writer.print("{f}", .{self.module.operation().fmt(.{ .print_generic_op_form = true })}); + return try self.allocator.dupeSentinel(u8, printed.written(), 0); + } + + // ==================== types ==================== + + pub fn ptrTy(self: *Builder, dt: DType, space: MemorySpace, alignment: u64) !*const mlir.Type { + return cute_ir.pointerType(self.ctx, space, alignment, .{ .element_type = dt.toMlir(self.ctx) }); + } + + pub fn swizzledPtrTy(self: *Builder, dt: DType, space: MemorySpace, alignment: u64, swizzle: Swizzle) !*const mlir.Type { + return (try cute_ir.PtrType.get(self.ctx, .{ + .valueType = dt.toMlir(self.ctx), + .addressSpace = space, + .alignment = alignment, + .swizzle = (try self.swizzleAttr(swizzle)).attribute(), + })).type_(); + } + + fn swizzleAttr(self: *Builder, swizzle: Swizzle) !*const cute_ir.SwizzleAttr { + return cute_ir.SwizzleAttr.get(self.ctx, .{ + .bits = std.math.cast(u32, swizzle.bits) orelse return error.InvalidMlir, + .base = std.math.cast(u32, swizzle.base) orelse return error.InvalidMlir, + .shift = std.math.cast(i32, swizzle.shift) orelse return error.InvalidMlir, + }); + } + + pub fn intToPtr(self: *Builder, address: anytype, dt: DType, space: MemorySpace, alignment: u64) Value { + const ptr_type = self.ptrTy(dt, space, alignment) catch @panic("invalid CuTe pointer type"); + return self.emit(cute_ir.inttoptr(self.ctx, self.lift(address).inner, ptr_type, self.loc())); + } + + /// `cute.ptrtoint`: the integer address of a pointer (`i32` for shared + /// memory, `i64` for global memory), for hand-written PTX operands. + pub fn ptrToInt(self: *Builder, pointer: anytype, dt: DType) Value { + const ptr = self.asValue(pointer); + return self.emit(if (ptr.isPtr()) cute_ir.ptrtoint(self.ctx, ptr.inner, dt.toMlir(self.ctx), self.loc()) else llvm.ptrtoint(self.ctx, ptr.inner, dt.toMlir(self.ctx), self.loc())); + } + + /// Reinterpret a pointer without changing its address. This is the typed + /// counterpart of Python CuTe's `recast_ptr`; packed MXFP4 buffers use it + /// to expose their byte storage to a scalar correctness implementation. + pub fn recastPointer(self: *Builder, pointer: anytype, dt: DType, space: MemorySpace, alignment: u64) Value { + const result_type = self.ptrTy(dt, space, alignment) catch @panic("invalid recast pointer type"); + return self.emit(cute_ir.recast_iter(self.ctx, self.asValue(pointer).inner, result_type, self.loc())); + } + + pub fn recastSwizzledPointer(self: *Builder, pointer: anytype, dt: DType, space: MemorySpace, alignment: u64, swizzle: Swizzle) Value { + const result_type = self.swizzledPtrTy(dt, space, alignment, swizzle) catch @panic("invalid swizzled pointer type"); + return self.emit(cute_ir.recast_iter(self.ctx, self.asValue(pointer).inner, result_type, self.loc())); + } + + /// The scalar of a single extent, otherwise the tuple of `dims`. + fn dimsExpr(self: *Builder, dims: []const i64) AlgebraExpr { + if (dims.len == 1) return .{ .static = dims[0] }; + const elements = self.arena.allocator().alloc(AlgebraExpr, dims.len) catch @panic("OOM"); + for (dims, elements) |d, *e| e.* = .{ .static = d }; + return .{ .tuple = elements }; + } + + /// Build a CuTe algebra expression from integers, nested Zig tuples, + /// `AlgebraToken`s, `ConstrainedDynamic`s and `ScaledBasis` values. + pub fn expr(self: *Builder, value: anytype) AlgebraExpr { + const T = @TypeOf(value); + if (T == AlgebraExpr) return value; + if (T == AlgebraToken) return switch (value) { + .all => .all, + .dynamic => .{ .dynamic = 1 }, + }; + if (T == ScaledBasis) return .{ .basis = value }; + if (T == ConstrainedDynamic) { + if (value.divisible_by == 0) @panic("CuTe constrained dynamic divisor must be positive"); + return .{ .dynamic = @intCast(value.divisible_by) }; + } + switch (@typeInfo(T)) { + .comptime_int, .int => return .{ .static = @intCast(value) }, + .@"struct" => |info| { + const elements = self.arena.allocator().alloc(AlgebraExpr, info.field_names.len) catch @panic("OOM"); + inline for (info.field_names, 0..) |field, i| elements[i] = self.expr(@field(value, field)); + return .{ .tuple = elements }; + }, + else => @compileError("unsupported CuTe algebra value " ++ @typeName(T)), + } + } + + /// The algebra attribute `A` (`cute_ir.IntTupleAttr`, `ShapeAttr`, ...) + /// of `e`. + fn algebraAttr(self: *Builder, comptime A: type, e: AlgebraExpr) *const A { + const result = switch (e) { + .static => |v| if (@hasDecl(A, "getStatic")) A.getStatic(self.ctx, v) else std.debug.panic("{s} has no integer leaf", .{@typeName(A)}), + .dynamic => |divisibility| if (@hasDecl(A, "getDynamic")) A.getDynamic(self.ctx, 32, divisibility) else std.debug.panic("{s} has no dynamic leaf", .{@typeName(A)}), + .dynamic_int => |v| if (@hasDecl(A, "getDynamic")) A.getDynamic(self.ctx, v.width, v.divisible_by) else std.debug.panic("{s} has no dynamic leaf", .{@typeName(A)}), + .all => if (@hasDecl(A, "getUnderscore")) A.getUnderscore(self.ctx) else std.debug.panic("{s} has no `_` leaf", .{@typeName(A)}), + .basis => |b| if (@hasDecl(A, "getBasis")) blk: { + const modes = self.arena.allocator().alloc(i32, b.mode_count) catch @panic("OOM"); + for (modes, b.modes[0..b.mode_count]) |*mode, m| mode.* = m; + break :blk A.getBasis(self.ctx, b.numerator, b.denominator, modes); + } else std.debug.panic("{s} has no scaled-basis leaf", .{@typeName(A)}), + .tuple => |elements| blk: { + const attrs = self.arena.allocator().alloc(*const A, elements.len) catch @panic("OOM"); + for (elements, attrs) |element, *attr| attr.* = self.algebraAttr(A, element); + break :blk A.getTuple(self.ctx, attrs); + }, + }; + return result catch std.debug.panic("invalid CuTe algebra for {s}", .{@typeName(A)}); + } + + fn layoutAttr(self: *Builder, spec: LayoutSpec) *const cute_ir.LayoutAttr { + return cute_ir.LayoutAttr.get(self.ctx, .{ + .shape = self.algebraAttr(cute_ir.ShapeAttr, spec.shape), + .stride = self.algebraAttr(cute_ir.StrideAttr, spec.stride), + }) catch @panic("invalid CuTe layout"); + } + + fn intTupleType(self: *Builder, e: AlgebraExpr) *const mlir.Type { + return (cute_ir.IntTupleType.get(self.ctx, .{ .attr = self.algebraAttr(cute_ir.IntTupleAttr, e).attribute() }) catch @panic("invalid CuTe integer tuple")).type_(); + } + + /// `numerator/denominator` along `modes`, the mode path outermost first like + /// Python's `ScaledBasis(value, modes)`. CuTe prints it innermost first: + /// `basis(1, 2, .{ 0, 1 })` is `1/2@1@0`. + pub fn basis(self: *Builder, numerator: i64, denominator: i64, modes: anytype) ScaledBasis { + _ = self; + if (denominator <= 0) @panic("CuTe scaled-basis denominator must be positive"); + const fields = @typeInfo(@TypeOf(modes)).@"struct".field_names; + if (fields.len == 0 or fields.len > MAX_RANK) @panic("CuTe scaled basis needs 1..MAX_RANK modes"); + var result: ScaledBasis = .{ .numerator = numerator, .denominator = denominator, .mode_count = fields.len }; + inline for (fields, 0..) |field, i| result.modes[i] = @intCast(@field(modes, field)); + return result; + } + + pub fn layoutSpec(self: *Builder, shape: anytype, stride: anytype) LayoutSpec { + return .{ .shape = self.expr(shape), .stride = self.expr(stride) }; + } + + pub fn layoutType(self: *Builder, spec: LayoutSpec) *const mlir.Type { + return (cute_ir.LayoutType.get(self.ctx, .{ .attr = self.layoutAttr(spec).attribute() }) catch @panic("invalid CuTe layout")).type_(); + } + + pub fn composedLayoutType(self: *Builder, swizzle: Swizzle, offset: anytype, outer: LayoutSpec) *const mlir.Type { + const attr = cute_ir.ComposedLayoutAttr.get(self.ctx, .{ + .inner = .{ .swizzle = self.swizzleAttr(swizzle) catch @panic("invalid CuTe swizzle") }, + .offset = self.algebraAttr(cute_ir.IntTupleAttr, self.expr(offset)), + .outer = self.layoutAttr(outer), + }) catch @panic("invalid CuTe composed layout"); + return (cute_ir.ComposedLayoutType.get(self.ctx, .{ .attr = attr.attribute() }) catch @panic("invalid CuTe composed layout")).type_(); + } + + pub fn memrefType(self: *Builder, dtype: DType, space: MemorySpace, alignment: u64, layout_type: *const mlir.Type) *const mlir.Type { + const pointer = self.ptrTy(dtype, space, alignment) catch @panic("invalid CuTe pointer type"); + return (cute_ir.MemRefType.get(self.ctx, .{ .ptr = pointer, .layout = layout_type }) catch @panic("invalid CuTe memref type")).type_(); + } + + pub fn smemDescType(self: *Builder) *const mlir.Type { + return (cute_nvgpu.SmemDescType.get(self.ctx, .{}) catch @panic("invalid shared-memory descriptor")).type_(); + } + + /// `!cute_nvgpu.smem_desc_view` of a shared-memory descriptor over `layout`. + pub fn smemDescViewType(self: *Builder, layout: LayoutSpec) *const mlir.Type { + return (cute_nvgpu.SmemDescViewType.get(self.ctx, .{ + .desc = self.smemDescType(), + .layout = self.layoutAttr(layout).attribute(), + }) catch @panic("invalid shared-memory descriptor view")).type_(); + } + + /// `!cute.tiled_copy` of `copy_atom` over `layout_copy_tv`, tiled by the + /// modes of `tiler_mn`. + pub fn tiledCopyType(self: *Builder, copy_atom: *const mlir.Type, layout_copy_tv: LayoutSpec, tiler_mn: []const LayoutSpec) *const mlir.Type { + return (cute_ir.TiledCopyType.get(self.ctx, .{ + .copyAtom = copy_atom, + .layoutCopyTv = self.layoutAttr(layout_copy_tv).attribute(), + .tilerMn = self.tileAttr(tiler_mn).attribute(), + }) catch @panic("invalid tiled copy")).type_(); + } + + pub fn memrefTypeFromPointer(self: *Builder, pointer_type: *const mlir.Type, layout_type: *const mlir.Type) *const mlir.Type { + return (cute_ir.MemRefType.get(self.ctx, .{ .ptr = pointer_type, .layout = layout_type }) catch @panic("invalid CuTe memref type")).type_(); + } + + /// `cute.static` of an integer tuple such as `128` or `.{ 0, 4 }`. + pub fn staticIntTuple(self: *Builder, value: anytype) View { + const result = self.emit(cute_ir.static(self.ctx, self.intTupleType(self.expr(value)), self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn staticLayout(self: *Builder, spec: LayoutSpec) View { + const result = self.emit(cute_ir.static(self.ctx, self.layoutType(spec), self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + pub const Swizzle = struct { bits: i64, base: i64, shift: i64 }; + + pub fn staticSwizzle(self: *Builder, spec: Swizzle) View { + const attr = self.swizzleAttr(spec) catch @panic("invalid CuTe swizzle"); + const ty = (cute_ir.SwizzleType.get(self.ctx, .{ .attr = attr.attribute() }) catch @panic("invalid CuTe swizzle")).type_(); + const result = self.emit(cute_ir.static(self.ctx, ty, self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + /// Construct `swizzle o offset o outer_layout` through the CuTe dialect. + pub fn makeComposedLayout(self: *Builder, swizzle: Swizzle, offset: anytype, outer: LayoutSpec) View { + const inner = self.staticSwizzle(swizzle); + const offset_value = self.emit(cute_ir.make_int_tuple(self.ctx, &.{}, self.intTupleType(self.expr(offset)), self.loc())); + const outer_value = self.staticLayout(outer); + const result_ty = self.composedLayoutType(swizzle, offset, outer); + const result = self.emit(cute_ir.make_composed_layout(self.ctx, inner.inner, offset_value.inner, outer_value.inner, result_ty, self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + /// Compose a raw pointer and any CuTe layout value. The memref result type + /// is obtained through the dialect C API rather than parsed from assembly. + pub fn makeTensorView(self: *Builder, pointer: anytype, layout: anytype) View { + const p = self.asValue(pointer); + const l = self.asValue(layout); + const result_ty = (cute_ir.MemRefType.get(self.ctx, .{ .ptr = p.type_(), .layout = l.type_() }) catch @panic("invalid CuTe tensor view")).type_(); + const result = self.emit(cute_ir.make_view(self.ctx, p.inner, result_ty, .{ .layout = l.inner }, self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn coordTensorType(self: *Builder, rank: usize, layout: LayoutSpec) *const mlir.Type { + if (rank == 0 or rank > MAX_RANK) @panic("coordTensorType rank must be in 1..MAX_RANK"); + const zeros: [MAX_RANK]i64 = @splat(0); + return self.coordTensorTypeWithTuple(self.dimsExpr(zeros[0..rank]), layout); + } + + /// Coordinate-tensor type with an explicit arithmetic-tuple expression, + /// such as `.{ 0, ConstrainedDynamic{ .divisible_by = 128 }, AlgebraToken.dynamic }`. + /// This represents the divisibility information produced by CuTe tiling + /// without requiring kernel code to spell textual MLIR types. + pub fn coordTensorTypeWithTuple(self: *Builder, tuple: anytype, layout: LayoutSpec) *const mlir.Type { + return (cute_ir.CoordTensorType.get(self.ctx, .{ .arithTuple = self.intTupleType(self.expr(tuple)), .layout = self.layoutType(layout) }) catch @panic("invalid coordinate tensor")).type_(); + } + + fn rowMajor(self: *Builder, shape: []const i64) []const i64 { + const s = self.arena.allocator().alloc(i64, shape.len) catch @panic("OOM"); + var acc: i64 = 1; + var i = shape.len; + while (i > 0) : (i -= 1) { + s[i - 1] = acc; + acc *= shape[i - 1]; + } + return s; + } + + fn leftMost(self: *Builder, shape: []const i64) []const i64 { + const s = self.arena.allocator().alloc(i64, shape.len) catch @panic("OOM"); + var acc: i64 = 1; + for (shape, 0..) |d, i| { + s[i] = acc; + acc *= d; + } + return s; + } + + // ==================== cute.core ==================== + + fn layoutResult(comptime Shape: type) type { + return switch (@typeInfo(Shape)) { + .pointer => Layout, + else => View, + }; + } + + /// `cute.make_layout(shape, stride=...)`. Slices keep the tracked static + /// Layout API; nested tuples accept mixed integer and SSA shape/stride leaves. + pub fn makeLayout(self: *Builder, shape: anytype, opts: anytype) layoutResult(@TypeOf(shape)) { + if (comptime layoutResult(@TypeOf(shape)) == Layout) return self.makeStaticLayout(shape, .{ + .stride = if (@hasField(@TypeOf(opts), "stride")) opts.stride else null, + }); + var shape_values: stdx.BoundedArray(*const mlir.Value, MAX_RANK) = .empty; + var stride_values: stdx.BoundedArray(*const mlir.Value, MAX_RANK) = .empty; + const shape_expr = self.coordExpr(&shape_values, shape); + const stride_expr = if (@hasField(@TypeOf(opts), "stride") and @TypeOf(opts.stride) != @TypeOf(null)) self.coordExpr(&stride_values, opts.stride) else blk: { + var stride: AlgebraExpr = .{ .static = 1 }; + var stride_value: ?Value = null; + var index: usize = 0; + break :blk self.compactLayoutStride(shape_expr, shape_values.constSlice(), &index, &stride, &stride_value, &stride_values); + }; + const shape_value = self.emit(cute_ir.make_shape(self.ctx, shape_values.constSlice(), self.shapeType(shape_expr), self.loc())); + const stride_type = (cute_ir.StrideType.get(self.ctx, .{ .attr = self.algebraAttr(cute_ir.StrideAttr, stride_expr).attribute() }) catch @panic("invalid stride")).type_(); + const stride_value = self.emit(cute_ir.make_stride(self.ctx, stride_values.constSlice(), stride_type, self.loc())); + const result = self.emit(cute_ir.make_layout(self.ctx, self.layoutType(.{ .shape = shape_expr, .stride = stride_expr }), .{ .shape = shape_value.inner, .stride = stride_value.inner }, self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + fn compactLayoutStride(self: *Builder, shape: AlgebraExpr, values: []const *const mlir.Value, index: *usize, stride: *AlgebraExpr, stride_value: *?Value, dynamic: *stdx.BoundedArray(*const mlir.Value, MAX_RANK)) AlgebraExpr { + if (shape == .tuple) { + const elements = self.arena.allocator().alloc(AlgebraExpr, shape.tuple.len) catch @panic("OOM"); + for (shape.tuple, elements) |extent, *element| element.* = self.compactLayoutStride(extent, values, index, stride, stride_value, dynamic); + return .{ .tuple = elements }; + } + if (shape == .static and shape.static == 1) return .{ .static = 0 }; + const result = stride.*; + if (stride_value.*) |v| dynamic.append(v.inner) catch @panic("too many stride leaves"); + if (shape == .static and stride.* == .static) { + stride.static *= shape.static; + } else { + const extent = if (shape == .dynamic or shape == .dynamic_int) blk: { + const v: Value = .{ .inner = values[index.*], .kernel = self }; + index.* += 1; + break :blk v; + } else self.lift(shape.static); + stride_value.* = (stride_value.* orelse self.lift(stride.static)).mul(extent); + stride.* = dynamicExpr(stride_value.*.?); + } + return result; + } + + fn makeStaticLayout(self: *Builder, shape: []const i64, opts: LayoutOpts) Layout { + if (shape.len == 0 or shape.len > MAX_RANK) std.debug.panic("makeLayout: rank {d} not in 1..{d}", .{ shape.len, MAX_RANK }); + const stride = opts.stride orelse self.leftMost(shape); + if (stride.len != shape.len) std.debug.panic("makeLayout: stride rank {d} != shape rank {d}", .{ stride.len, shape.len }); + const layout_ty = self.layoutType(.{ .shape = self.dimsExpr(shape), .stride = self.dimsExpr(stride) }); + const l = self.emit(cute_ir.static(self.ctx, layout_ty, self.loc())); + return .{ .inner = l.inner, .shape = Dims.fromSlice(shape) catch unreachable, .stride = Dims.fromSlice(stride) catch unreachable }; + } + + /// `cute.make_tensor(ptr, layout)`. + pub fn makeTensor(self: *Builder, ptr: Value, layout: Layout) Tensor { + const pt = ptr.type_().isA(cute_ir.PtrType) orelse std.debug.panic("makeTensor: {f} is not a !cute.ptr", .{ptr.type_()}); + const elem = pt.valueType() orelse @panic("makeTensor: untyped pointer"); + const memref_ty = (cute_ir.MemRefType.get(self.ctx, .{ .ptr = ptr.type_(), .layout = layout.inner.type_() }) catch @panic("bad memref")).type_(); + const v = self.emit(cute_ir.make_view(self.ctx, ptr.inner, memref_ty, .{ .layout = layout.inner }, self.loc())); + return .{ .inner = v.inner, .kernel = self, .layout = layout, .dtype = dtypes.mlirToDType(self.ctx, elem), .space = pt.addressSpace() }; + } + + /// The algebra of a coordinate whose `Value` leaves are dynamic and + /// appended to `dyn`. A one-element tuple is its element. + fn coordExpr(self: *Builder, dyn: *stdx.BoundedArray(*const mlir.Value, MAX_RANK), coord: anytype) AlgebraExpr { + const T = @TypeOf(coord); + if (T == Value) { + dyn.append(coord.inner) catch @panic("makeCoord: too many dynamic leaves"); + return dynamicExpr(coord); + } + if (T == AlgebraToken) return switch (coord) { + .all => .all, + .dynamic => @panic("makeCoord: AlgebraToken.dynamic needs an SSA value"), + }; + switch (@typeInfo(T)) { + .comptime_int, .int => return .{ .static = @intCast(coord) }, + .@"struct" => |info| { + if (info.field_names.len == 0) @compileError("makeCoord: empty tuple"); + if (info.field_names.len == 1) return self.coordExpr(dyn, @field(coord, info.field_names[0])); + const elements = self.arena.allocator().alloc(AlgebraExpr, info.field_names.len) catch @panic("OOM"); + inline for (info.field_names, 0..) |field, i| elements[i] = self.coordExpr(dyn, @field(coord, field)); + return .{ .tuple = elements }; + }, + else => @compileError("makeCoord: unsupported coordinate leaf " ++ @typeName(T)), + } + } + + fn dynamicExpr(value: Value) AlgebraExpr { + std.debug.assert(value.isInt()); + const width = dtypeBitwidth(value.dtype()); + return if (width == 32) .{ .dynamic = 1 } else .{ .dynamic_int = .{ .width = @intCast(width) } }; + } + + /// A `!cute.coord` from nested tuples of `Value`s, integers, and `.all` + /// placeholders. Integer leaves stay static in the dependent CuTe type. + pub fn makeCoord(self: *Builder, coord: anytype) *const mlir.Value { + if (@TypeOf(coord) == Value) return self.makeCoord(.{coord}); + var dyn: stdx.BoundedArray(*const mlir.Value, MAX_RANK) = .empty; + const attr = self.algebraAttr(cute_ir.CoordAttr, self.coordExpr(&dyn, coord)); + const ty = (cute_ir.CoordType.get(self.ctx, .{ .attr = attr.attribute() }) catch @panic("bad coord")).type_(); + return self.emit(cute_ir.make_coord(self.ctx, dyn.constSlice(), ty, self.loc())).inner; + } + + /// A dynamic/static CuTe integer tuple, used for pointer arithmetic in + /// tensor and shared memory. It accepts the same nested Zig values as a + /// coordinate, but preserves integer-tuple semantics. + pub fn makeIntTuple(self: *Builder, values: anytype) View { + var dyn: stdx.BoundedArray(*const mlir.Value, MAX_RANK) = .empty; + const ty = self.intTupleType(self.coordExpr(&dyn, values)); + const value = self.emit(cute_ir.make_int_tuple(self.ctx, dyn.constSlice(), ty, self.loc())); + return .{ .inner = value.inner, .kernel = self }; + } + + /// A one-element integer tuple holding a value known to be a multiple of + /// `divisor`. Ops that derive alignment from an offset, such as + /// `cute.add_offset`, need that to keep the operand's alignment. + pub fn makeIntTupleDivBy(self: *Builder, value: anytype, comptime divisor: u32) View { + const constrained = self.assumeDivBy(value, divisor); + const tuple = self.emit(cute_ir.make_int_tuple(self.ctx, &.{constrained.inner}, self.intTupleType(.{ .dynamic = divisor }), self.loc())); + return .{ .inner = tuple.inner, .kernel = self }; + } + + /// Refine an integer SSA value with the divisibility information CuTe + /// derives while slicing a tiled coordinate tensor. + pub fn assumeDivBy(self: *Builder, value: anytype, comptime divisor: u32) Value { + if (divisor == 0) @compileError("assumeDivBy divisor must be non-zero"); + const constrained = cute_ir.constrainedIntType(self.ctx, .i32, divisor) catch @panic("invalid CuTe constrained integer"); + return self.emit(cute_ir.assume(self.ctx, self.lift(value).to(.i32).inner, constrained, self.loc())); + } + + /// Build a coordinate tensor from a runtime origin and a static layout. + /// This is the direct form of the `get_iter`/`make_arith_tuple_iter`/ + /// `make_view` sequence emitted by the Python CuTe front end after a tile + /// is selected from a larger coordinate tensor. + pub fn makeCoordTensor(self: *Builder, origin: anytype, layout: anytype) View { + const tuple = self.makeIntTuple(origin); + const layout_value = self.asValue(layout); + const iterator_type = (cute_ir.ArithTupleIteratorType.get(self.ctx, .{ .arithTuple = tuple.type_() }) catch @panic("invalid arithmetic tuple iterator")).type_(); + const iterator = self.emit(cute_ir.make_arith_tuple_iter(self.ctx, iterator_type, .{ .value_ = tuple.inner }, self.loc())); + const result_type = (cute_ir.CoordTensorType.get(self.ctx, .{ .arithTuple = tuple.type_(), .layout = layout_value.type_() }) catch @panic("invalid coordinate tensor")).type_(); + const result = self.emit(cute_ir.make_view(self.ctx, iterator.inner, result_type, .{ .layout = layout_value.inner }, self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + /// Rebuild a coordinate tensor from the arithmetic-tuple iterator of an + /// existing tensor. CuTe's Python front end emits this sequence after it + /// slices a TMA-partitioned coordinate tensor: + /// + /// get_iter -> deref_arith_tuple_iter -> get_leaves -> make_int_tuple + /// -> make_arith_tuple_iter -> make_view + /// + /// Keeping that sequence explicit matters for TMA. The iterator carries + /// the descriptor coordinates selected by `slice`; constructing a fresh + /// tuple from the scheduler indices would lose CuTe's coordinate algebra. + pub fn rebuildCoordTensorIterator( + self: *Builder, + comptime leaf_count: usize, + input: anytype, + tuple: anytype, + comptime leaf_divisibility: [leaf_count]u32, + layout: anytype, + ) View { + if (leaf_count == 0 or leaf_count > MAX_RANK) @panic("invalid coordinate tuple leaf count"); + const source = self.asValue(input); + const source_type = source.type_().isA(cute_ir.CoordTensorType) orelse + std.debug.panic("rebuildCoordTensorIterator: expected coordinate tensor, got {f}", .{source.type_()}); + const source_tuple_type = source_type.arithTuple(); + const source_iterator_type = (cute_ir.ArithTupleIteratorType.get(self.ctx, .{ .arithTuple = source_tuple_type }) catch + @panic("invalid source arithmetic tuple iterator")).type_(); + const source_iterator = self.emit(cute_ir.get_iter(self.ctx, source.inner, source_iterator_type, self.loc())); + const source_tuple = self.emit(cute_ir.deref_arith_tuple_iter( + self.ctx, + source_iterator.inner, + source_tuple_type, + self.loc(), + )); + + const leaf_types = self.arena.allocator().alloc(*const mlir.Type, leaf_count) catch @panic("OOM"); + // 0 marks a leaf the producer knows to be statically zero, 1 a leaf + // with no known divisibility. + inline for (leaf_divisibility, 0..) |divisor, i| { + leaf_types[i] = self.intTupleType(if (divisor == 0) .{ .static = 0 } else .{ .dynamic = divisor }); + } + const leaves = self.emitResults(leaf_count, cute_ir.get_leaves( + self.ctx, + source_tuple.inner, + leaf_types, + self.loc(), + )); + + const rebuilt_tuple_type = self.intTupleType(self.expr(tuple)); + const rebuilt_tuple = self.emit(cute_ir.make_int_tuple( + self.ctx, + self.innerSlice(leaves[0..]), + rebuilt_tuple_type, + self.loc(), + )); + const rebuilt_iterator_type = (cute_ir.ArithTupleIteratorType.get(self.ctx, .{ .arithTuple = rebuilt_tuple_type }) catch + @panic("invalid rebuilt arithmetic tuple iterator")).type_(); + const rebuilt_iterator = self.emit(cute_ir.make_arith_tuple_iter(self.ctx, rebuilt_iterator_type, .{ .value_ = rebuilt_tuple.inner }, self.loc())); + const layout_value = self.asValue(layout); + const result_type = (cute_ir.CoordTensorType.get(self.ctx, .{ + .arithTuple = rebuilt_tuple_type, + .layout = layout_value.type_(), + }) catch @panic("invalid rebuilt coordinate tensor")).type_(); + const result = self.emit(cute_ir.make_view(self.ctx, rebuilt_iterator.inner, result_type, .{ .layout = layout_value.inner }, self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + /// `!cute.shape` of a CuTe algebra value such as `.{ .{ 128, 16 }, 1, 1, 2 }` + /// or `.{ AlgebraToken.dynamic, AlgebraToken.dynamic }`. + pub fn shapeType(self: *Builder, shape: anytype) *const mlir.Type { + return (cute_ir.ShapeType.get(self.ctx, .{ .attr = self.algebraAttr(cute_ir.ShapeAttr, self.expr(shape)).attribute() }) catch @panic("invalid CuTe shape")).type_(); + } + + /// Static `cute.make_shape` of a CuTe algebra value. + pub fn makeShape(self: *Builder, shape: anytype) View { + const value = self.emit(cute_ir.make_shape(self.ctx, &.{}, self.shapeType(shape), self.loc())); + return .{ .inner = value.inner, .kernel = self }; + } + + /// Dynamic `cute.make_shape`. The result type describes which leaves are + /// runtime values (for example `!cute.shape<"(?,?,?)">`). Keeping these + /// leaves dynamic is significant to CuTe's TMA layout algebra when a + /// runtime extent happens to be one. + pub fn makeShapeValues(self: *Builder, values: anytype, result_type: *const mlir.Type) View { + const value = self.emit(cute_ir.make_shape(self.ctx, self.innerSlice(values), result_type, self.loc())); + return .{ .inner = value.inner, .kernel = self }; + } + + pub fn tileToShape(self: *Builder, input: anytype, shape: anytype, order: View, result_type: *const mlir.Type) View { + const result = self.emit(cute_ir.tile_to_shape( + self.ctx, + self.asValue(input).inner, + self.asValue(shape).inner, + order.inner, + result_type, + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + /// Static `cute.make_tile` from mode extents. + pub fn makeTile(self: *Builder, shape: []const i64) View { + const modes = self.arena.allocator().alloc(LayoutSpec, shape.len) catch @panic("OOM"); + for (shape, modes) |extent, *mode| mode.* = .{ .shape = .{ .static = extent }, .stride = .{ .static = 1 } }; + const value = self.emit(cute_ir.make_tile(self.ctx, &.{}, self.tileType(modes), self.loc())); + return .{ .inner = value.inner, .kernel = self }; + } + + /// `!cute.tile` whose modes are `layouts`, such as `[512:1;1:0;4:1]`. + pub fn tileType(self: *Builder, layouts: []const LayoutSpec) *const mlir.Type { + return (cute_ir.TileType.get(self.ctx, .{ .attr = self.tileAttr(layouts).attribute() }) catch @panic("invalid CuTe tile")).type_(); + } + + fn tileAttr(self: *Builder, layouts: []const LayoutSpec) *const cute_ir.TileAttr { + const modes = self.arena.allocator().alloc(*const cute_ir.TileAttr, layouts.len) catch @panic("OOM"); + for (layouts, modes) |layout, *mode| mode.* = cute_ir.TileAttr.getLayout(self.ctx, self.layoutAttr(layout)) catch @panic("invalid CuTe tile mode"); + return cute_ir.TileAttr.getTuple(self.ctx, modes) catch @panic("invalid CuTe tile"); + } + + /// Construct a target atom whose complete architecture payload is + /// encoded in `atom_type` (for example `!cute_nvgpu.sm100.mma_bs<...>`). + pub fn makeAtom(self: *Builder, atom_type: *const mlir.Type, values: anytype) Atom { + const result = self.emit(cute_ir.make_atom(self.ctx, self.innerSlice(values), atom_type, self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn makeTiledMma(self: *Builder, atom: Atom, result_type: *const mlir.Type) Atom { + const result = self.emit(cute_ir.make_tiled_mma(self.ctx, atom.inner, result_type, self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + /// Allocate shared memory with a type assembled by the Zig DSL. + pub fn allocSmemView(self: *Builder, dtype: DType, alignment: u64, layout_type: *const mlir.Type) View { + const memref_type = self.memrefType(dtype, .smem, alignment, layout_type); + const result = self.emit(cute_ir.memref_alloc_smem(self.ctx, memref_type, self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + /// Allocate one compiler-visible shared-memory storage object. The field + /// descriptions use the CuTe host compiler's `name:size:offset` format; + /// they let its shared-memory sizing/layout pass reproduce a Python + /// `SharedStorage` struct while Zig keeps typed pointers to its fields. + pub fn allocSmemStorage( + self: *Builder, + dtype: DType, + alignment: u64, + layout_type: *const mlir.Type, + partition_id: i32, + fields: []const []const u8, + ) View { + const field_attrs = self.arena.allocator().alloc(*const mlir.Attribute, fields.len) catch @panic("OOM"); + for (fields, field_attrs) |field, *attr| attr.* = .string(self.ctx, field); + const memref_type = self.memrefType(dtype, .smem, alignment, layout_type); + const result = self.emit(mlir.Operation.make(self.ctx, "cute.memref.alloca", .{ + .results = .{ .flat = &.{memref_type} }, + .attributes = &.{ + .named(self.ctx, "smem.partition_id", .int(self.ctx, .i32, partition_id)), + .named(self.ctx, "smem.struct_fields", .array(self.ctx, field_attrs)), + }, + .location = self.loc(), + })); + return .{ .inner = result.inner, .kernel = self }; + } + + /// Allocate a register-memory tensor whose layout is represented by a + /// normal CuTe layout value. + pub fn allocRmem(self: *Builder, dtype: DType, alignment: u64, layout: anytype) View { + const layout_value = self.asValue(layout); + const memref_type = self.memrefType(dtype, .rmem, alignment, layout_value.type_()); + const result = self.emit(cute_ir.memref_alloca(self.ctx, memref_type, .{ .layout = layout_value.inner }, self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn makeView(self: *Builder, pointer: anytype, layout: anytype, result_type: *const mlir.Type) View { + const result = self.emit(cute_ir.make_view(self.ctx, self.asValue(pointer).inner, result_type, .{ .layout = self.asValue(layout).inner }, self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn makeViewNoLayout(self: *Builder, pointer: anytype, result_type: *const mlir.Type) View { + const result = self.emit(cute_ir.make_view(self.ctx, self.asValue(pointer).inner, result_type, .{}, self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + /// `projection`, like Python's `proj`, selects the modes of `tile` that + /// apply, such as `b.expr(.{ 1, AlgebraToken.all, 1 })`. + pub fn localTile(self: *Builder, input: anytype, tile: anytype, coord: anytype, result_type: ?*const mlir.Type, projection: ?AlgebraExpr) View { + if (result_type == null) return self.inferView("cute.local_tile", &.{ self.asValue(input).inner, self.asValue(tile).inner, if (@TypeOf(coord) == View) coord.inner else self.makeCoord(coord) }, if (projection) |p| &.{.named(self.ctx, "proj", self.algebraAttr(cute_ir.CoordAttr, p).attribute())} else &.{}); + const coord_value = if (@TypeOf(coord) == View) coord.inner else self.makeCoord(coord); + const result = self.emit(cute_ir.local_tile( + self.ctx, + self.asValue(input).inner, + self.asValue(tile).inner, + coord_value, + result_type.?, + .{ .proj = if (projection) |p| self.algebraAttr(cute_ir.CoordAttr, p).attribute() else null }, + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + // A null result type on the algebra operations below requests dialect inference. + fn inferView(self: *Builder, name: []const u8, operands: []const *const mlir.Value, attrs: []const mlir.NamedAttribute) View { + const value = self.emit(mlir.Operation.make(self.ctx, name, .{ .operands = .{ .flat = operands }, .attributes = attrs, .result_type_inference = true, .location = self.loc() })); + return .{ .inner = value.inner, .kernel = self }; + } + + pub fn localPartition(self: *Builder, input: anytype, tiler: anytype, index: anytype, result_type: *const mlir.Type) View { + const result = self.emit(cute_ir.local_partition( + self.ctx, + self.asValue(input).inner, + self.asValue(tiler).inner, + self.makeCoord(index), + result_type, + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn slice(self: *Builder, input: anytype, coord: anytype, result_type: ?*const mlir.Type) View { + if (result_type == null) return self.inferView("cute.slice", &.{ self.asValue(input).inner, self.makeCoord(coord) }, &.{}); + const result = self.emit(cute_ir.slice( + self.ctx, + self.asValue(input).inner, + self.makeCoord(coord), + result_type.?, + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + /// Rebuild a tensor with the same iterator and a new layout. This is the + /// tensor form of CuTe's `group_modes`: grouping changes only the layout, + /// while the backing pointer or arithmetic-tuple iterator stays intact. + /// The local CUTLASS schema exposes `cute.group_modes` only for layouts, + /// so expressing the equivalent algebra this way also remains portable. + pub fn withLayout(self: *Builder, input: anytype, layout: anytype) View { + const value = self.asValue(input); + const layout_value = self.asValue(layout); + const input_type = value.type_(); + const result_type: *const mlir.Type = if (input_type.isA(cute_ir.MemRefType)) |memref| + (cute_ir.MemRefType.get(self.ctx, .{ .ptr = memref.ptr_(), .layout = layout_value.type_() }) catch @panic("invalid regrouped memref")).type_() + else if (input_type.isA(cute_ir.CoordTensorType)) |coords| + (cute_ir.CoordTensorType.get(self.ctx, .{ .arithTuple = coords.arithTuple(), .layout = layout_value.type_() }) catch @panic("invalid regrouped coordinate tensor")).type_() + else if (input_type.isA(cute_nvgpu.SmemDescViewType)) |desc| + (cute_nvgpu.SmemDescViewType.get(self.ctx, .{ .desc = desc.desc(), .layout = layout_value.type_().isA(cute_ir.LayoutType).?.attr() }) catch @panic("invalid regrouped descriptor")).type_() + else + std.debug.panic("withLayout: expected memref, coordinate tensor or shared-memory descriptor view, got {f}", .{input_type}); + + const iterator_type: *const mlir.Type = if (input_type.isA(cute_ir.MemRefType)) |memref| + memref.ptr_() + else if (input_type.isA(cute_ir.CoordTensorType)) |coords| + (cute_ir.ArithTupleIteratorType.get(self.ctx, .{ .arithTuple = coords.arithTuple() }) catch @panic("invalid arithmetic tuple iterator")).type_() + else if (input_type.isA(cute_nvgpu.SmemDescViewType)) |desc| + desc.desc() + else + unreachable; + const iterator = self.emit(cute_ir.get_iter(self.ctx, value.inner, iterator_type, self.loc())); + const result = self.emit(cute_ir.make_view(self.ctx, iterator.inner, result_type, .{ .layout = layout_value.inner }, self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn groupModes(self: *Builder, input: anytype, begin: u32, end: u32, result_type: ?*const mlir.Type) View { + if (result_type == null) return self.inferView("cute.group_modes", &.{self.asValue(input).inner}, &.{ .named(self.ctx, "begin", .int(self.ctx, .i32, begin)), .named(self.ctx, "end", .int(self.ctx, .i32, end)) }); + const result = self.emit(cute_ir.group_modes( + self.ctx, + self.asValue(input).inner, + result_type.?, + .int(self.ctx, .i32, begin), + .int(self.ctx, .i32, end), + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn tiledDivide(self: *Builder, input: anytype, tiler: anytype, result_type: *const mlir.Type) View { + const result = self.emit(cute_ir.tiled_divide( + self.ctx, + self.asValue(input).inner, + self.asValue(tiler).inner, + result_type, + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn flatDivide(self: *Builder, input: anytype, tiler: anytype, result_type: ?*const mlir.Type) View { + if (result_type == null) return self.inferView("cute.flat_divide", &.{ self.asValue(input).inner, self.asValue(tiler).inner }, &.{}); + const result = self.emit(cute_ir.flat_divide( + self.ctx, + self.asValue(input).inner, + self.asValue(tiler).inner, + result_type.?, + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn coalesce(self: *Builder, input: anytype, target_profile: ?View, result_type: *const mlir.Type) View { + const result = self.emit(cute_ir.coalesce( + self.ctx, + self.asValue(input).inner, + result_type, + .{ .target_profile = if (target_profile) |target| target.inner else null }, + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn filterZeros(self: *Builder, input: anytype, target_profile: ?View, result_type: ?*const mlir.Type) View { + if (result_type == null) return self.inferView("cute.filter_zeros", if (target_profile) |target| &.{ self.asValue(input).inner, target.inner } else &.{self.asValue(input).inner}, &.{}); + const result = self.emit(cute_ir.filter_zeros( + self.ctx, + self.asValue(input).inner, + result_type.?, + .{ .target_profile = if (target_profile) |target| target.inner else null }, + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn getLayout(self: *Builder, input: anytype, result_type: ?*const mlir.Type) View { + if (result_type == null) return self.inferView("cute.get_layout", &.{self.asValue(input).inner}, &.{}); + const result = self.emit(cute_ir.get_layout(self.ctx, self.asValue(input).inner, result_type.?, self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + /// `cute.get_iter`: the iterator of `input`, of type `result_type`. + pub fn getIter(self: *Builder, input: anytype, result_type: ?*const mlir.Type) View { + if (result_type == null) return self.inferView("cute.get_iter", &.{self.asValue(input).inner}, &.{}); + const result = self.emit(cute_ir.get_iter(self.ctx, self.asValue(input).inner, result_type.?, self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn getShape(self: *Builder, input: anytype, result_type: ?*const mlir.Type) View { + if (result_type == null) return self.inferView("cute.get_shape", &.{self.asValue(input).inner}, &.{}); + const result = self.emit(cute_ir.get_shape(self.ctx, self.asValue(input).inner, result_type.?, self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn addOffset(self: *Builder, input: anytype, offset: anytype, result_type: ?*const mlir.Type) View { + if (result_type == null) return self.inferView("cute.add_offset", &.{ self.asValue(input).inner, self.asValue(offset).inner }, &.{}); + const result = self.emit(cute_ir.add_offset( + self.ctx, + self.asValue(input).inner, + self.asValue(offset).inner, + result_type.?, + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn recastIter(self: *Builder, input: anytype, result_type: *const mlir.Type) View { + const result = self.emit(cute_ir.recast_iter(self.ctx, self.asValue(input).inner, result_type, self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn appendToRank(self: *Builder, input: anytype, element: anytype, rank: u32, result_type: *const mlir.Type) View { + const result = self.emit(cute_ir.append_to_rank( + self.ctx, + self.asValue(input).inner, + self.asValue(element).inner, + result_type, + .int(self.ctx, .i32, rank), + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn makeTiledCopy(self: *Builder, atom: Atom, result_type: *const mlir.Type) Atom { + const result = self.emit(cute_ir.make_tiled_copy(self.ctx, atom.inner, result_type, self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn tiledCopyPartition(self: *Builder, tiled: Atom, input: anytype, coord: anytype, destination: bool, result_type: *const mlir.Type) View { + const op = if (destination) + cute_ir.tiled_copy_partition_D(self.ctx, tiled.inner, self.asValue(input).inner, self.makeCoord(coord), result_type, self.loc()) + else + cute_ir.tiled_copy_partition_S(self.ctx, tiled.inner, self.asValue(input).inner, self.makeCoord(coord), result_type, self.loc()); + const result = self.emit(op); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn tiledCopyRetile(self: *Builder, tiled: Atom, input: anytype, result_type: ?*const mlir.Type) View { + if (result_type == null) return self.inferView("cute.tiled.copy.retile", &.{ tiled.inner, self.asValue(input).inner }, &.{}); + const result = self.emit(cute_ir.tiled_copy_retile( + self.ctx, + tiled.inner, + self.asValue(input).inner, + result_type.?, + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn tiledMmaPartition(self: *Builder, tiled: Atom, input: anytype, coord: anytype, operand_id: u2, result_type: ?*const mlir.Type) View { + if (result_type == null) return self.inferView("cute.tiled.mma.partition", &.{ tiled.inner, self.asValue(input).inner, self.makeCoord(coord) }, &.{.named(self.ctx, "operand_id", .int(self.ctx, .i32, operand_id))}); + const result = self.emit(cute_ir.tiled_mma_partition( + self.ctx, + tiled.inner, + self.asValue(input).inner, + self.makeCoord(coord), + result_type.?, + .int(self.ctx, .i32, operand_id), + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn mmaMakeFragment(self: *Builder, tiled: Atom, input: anytype, operand_id: u2, result_type: ?*const mlir.Type) View { + if (result_type == null) return self.inferView("cute.mma.make_fragment", &.{ tiled.inner, self.asValue(input).inner }, &.{.named(self.ctx, "operand_id", .int(self.ctx, .i32, operand_id))}); + const result = self.emit(cute_ir.mma_make_fragment( + self.ctx, + tiled.inner, + self.asValue(input).inner, + result_type.?, + .int(self.ctx, .i32, operand_id), + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + pub const CopyOptions = struct { pred: ?Value = null, tma_bar_ptr: ?Value = null, cache_policy: ?Value = null }; + + pub fn copy(self: *Builder, atom: Atom, src: anytype, dst: anytype, opts: CopyOptions) void { + var exec = atom; + if (atom.type_().isA(cute_nvgpu.CopyAtomNonExecTiledTmaStoreType)) |ty| { + const exec_type = (cute_nvgpu.CopyAtomTmaStoreType.get(self.ctx, .{ + .valType = ty.valType(), + .copyBits = ty.copyBits(), + .mode = .tiled, + .gStride = self.emptyStrideType(), + .tmaGbasis = ty.tmaGbasis(), + }) catch @panic("invalid executable TMA store atom")).type_(); + exec = self.makeExecTma(atom, exec_type); + } + if (opts.tma_bar_ptr) |mbar| { + if (atom.type_().isA(cute_nvgpu.CopyAtomNonExec2DGather4TmaLoadType)) |ty| { + const exec_type = (cute_nvgpu.CopyAtomTmaLoadType.get(self.ctx, .{ + .valType = ty.valType(), + .copyBits = ty.copyBits(), + .mode = .gather4, + .numCta = if (ty.kind() == .sm_100_2sm) 2 else 1, + .gStride = self.emptyStrideType(), + .tmaGbasis = ty.tmaGbasis(), + }) catch @panic("invalid executable gather atom")).type_(); + exec = self.makeExecTma(atom, exec_type); + } else if (atom.type_().isA(cute_nvgpu.CopyAtomNonExecTiledTmaLoadType)) |ty| { + const exec_type = (cute_nvgpu.CopyAtomTmaLoadType.get(self.ctx, .{ + .valType = ty.valType(), + .copyBits = ty.copyBits(), + .mode = .tiled, + .numCta = if (ty.kind() == .sm_100_2sm) 2 else 1, + .gStride = self.emptyStrideType(), + .tmaGbasis = ty.tmaGbasis(), + }) catch @panic("invalid executable TMA atom")).type_(); + exec = self.makeExecTma(atom, exec_type); + } + exec = self.setTmaBarrier(exec, mbar); + } + if (opts.cache_policy) |policy| { + const field = if (exec.type_().isA(cute_nvgpu.CopyAtomTmaLoadType) != null) + (cute_nvgpu.AtomCopyFieldTmaLoadAttr.get(self.ctx, .{ .value = .cache_policy }) catch unreachable).attribute() + else if (exec.type_().isA(cute_nvgpu.CopyAtomTmaStoreType) != null) + (cute_nvgpu.AtomCopyFieldTmaStoreAttr.get(self.ctx, .{ .value = .cache_policy }) catch unreachable).attribute() + else + @panic("cache_policy requires an executable TMA copy atom"); + const updated = self.emit(cute_nvgpu.atom_set_value(self.ctx, exec.inner, policy.to(.i64).inner, exec.type_(), field, self.loc())); + exec = .{ .inner = updated.inner, .kernel = self }; + } + self.emitVoid(cute_ir.copy(self.ctx, exec.inner, self.innerSlice(src), self.innerSlice(dst), .{ .pred = if (opts.pred) |p| p.inner else null }, self.loc())); + } + + pub fn gemm(self: *Builder, tiled: Atom, d: anytype, a: anytype, rhs: anytype, c: anytype) void { + self.emitVoid(cute_ir.gemm( + self.ctx, + tiled.inner, + self.asValue(d).inner, + self.innerSlice(a), + self.innerSlice(rhs), + self.asValue(c).inner, + self.loc(), + )); + } + + pub fn vectorExtract(self: *Builder, source: Value, lane: i64, dtype: DType) Value { + return self.emit(vector.extract( + self.ctx, + source.inner, + &.{lane}, + &.{}, + dtype.toMlir(self.ctx), + self.loc(), + )); + } + + // ==================== cute_nvgpu.cpasync: TMA (SM90+) ==================== + // The host function builds tiled TMA atoms (Python `make_tiled_tma_atom`); + // the kernel receives them as grid-constant arguments and makes them + // executable before issuing copies. + + pub const TmaAtomOptions = struct { gmem_coord_tensor: ?View = null }; + + fn inferValue(self: *Builder, name: []const u8, operands: []const *const mlir.Value, attributes: []const mlir.NamedAttribute) Value { + return self.emit(mlir.Operation.make(self.ctx, name, .{ + .operands = .{ .flat = operands }, + .attributes = attributes, + .result_type_inference = true, + .location = self.loc(), + })); + } + + fn valueShape(self: *Builder, value: Value) Value { + if (value.type_().isA(cute_ir.ShapeType) != null) return value; + return self.inferValue("cute.get_shape", &.{value.inner}, &.{}); + } + + fn shapeRank(shape: Value) usize { + return shape.type_().isA(cute_ir.ShapeType).?.attr().isA(cute_ir.ShapeAttr).?.rank(); + } + + /// Common CuTe TMA atom constructor. Gather uses the index tensor's layout + /// at construction and passes data/index tensors together to partition/copy. + pub fn makeTiledTmaAtom(self: *Builder, op: anytype, gmem: anytype, smem_layout: anytype, cta_tiler: anytype, config: TmaConfig, opts: TmaAtomOptions) TmaDescriptor { + const Tiler = @TypeOf(cta_tiler); + const tiler = if (Tiler == Value or Tiler == View or Tiler == Layout) + self.asValue(cta_tiler) + else if (@typeInfo(Tiler) == .pointer) + self.makeShape(self.dimsExpr(cta_tiler)).value() + else + self.makeShape(cta_tiler).value(); + var smem = self.asValue(smem_layout); + const tiler_rank = shapeRank(self.valueShape(tiler)); + const smem_rank = shapeRank(self.valueShape(smem)); + std.debug.assert(smem_rank == tiler_rank or smem_rank == tiler_rank + 1); + if (smem_rank == tiler_rank + 1) { + const modes = self.arena.allocator().alloc(i32, tiler_rank) catch @panic("OOM"); + for (modes, 0..) |*mode, i| mode.* = @intCast(i); + smem = self.inferValue("cute.select", &.{smem.inner}, &.{.named(self.ctx, "mode", .denseArray(self.ctx, .i32, modes))}); + } + const shape = self.valueShape(self.asValue(gmem)); + const identity = self.inferValue("cute.make_identity_layout", &.{shape.inner}, &.{}); + const cta_map = self.inferValue("cute.composition", &.{ identity.inner, tiler.inner }, &.{}); + var cfg = config; + cfg.mode = .tiled; + if (@TypeOf(op) == CopyBulkTensorTileS2GOp) return self.makeTiledTmaStoreAtom(gmem, smem, cta_map, cfg); + cfg.cta_group = op.cta_group; + if (@TypeOf(op) == CopyBulkTensorTileG2SOp) return self.makeTiledTmaLoadAtom(gmem, smem, cta_map, cfg); + if (@TypeOf(op) != CopyBulkTensor2DGather4G2SOp) @compileError("unsupported TMA copy operation"); + cfg.mode = .gather4; + const indices = opts.gmem_coord_tensor orelse @panic("gather4 requires gmem_coord_tensor"); + const index_type = indices.type_().isA(cute_ir.MemRefType) orelse @panic("gather4 indices must be a tensor"); + const layout = self.emit(cute_ir.get_layout(self.ctx, indices.inner, index_type.layout(), self.loc())); + const kind: cute_nvgpu.GatherScatterTmaLoad = if (cfg.cta_group == .two) .sm_100_2sm else .sm_100; + const results = self.emitResults(2, cute_nvgpu.atom_make_non_exec_2d_gather4_tma_load(self.ctx, self.asValue(gmem).inner, layout.inner, smem.inner, cta_map.inner, self.tmaLoadAtomType(cfg), self.coordTensorType(cfg.coordinate_rank, cfg.coordinate_layout), (cute_nvgpu.GatherScatterTmaLoadAttr.get(self.ctx, .{ .value = kind }) catch unreachable).attribute(), .{ .num_multicast = .int(self.ctx, .i32, 1), .tma_format = tmaFormatAttribute(self, cfg) }, self.loc())); + return .{ .atom = .{ .inner = results[0].inner, .kernel = self }, .tensor = .{ .inner = results[1].inner, .kernel = self } }; + } + + /// Build the host-visible TMA descriptor and its coordinate tensor, whose + /// types are assembled from `config`. + pub fn makeTiledTmaLoadAtom(self: *Builder, gmem: anytype, smem_layout: anytype, cta_map: anytype, config: TmaConfig) TmaDescriptor { + std.debug.assert(config.mode == .tiled); + const results = self.emitResults(2, cute_nvgpu.atom_make_non_exec_tiled_tma_load( + self.ctx, + self.asValue(gmem).inner, + self.asValue(smem_layout).inner, + self.asValue(cta_map).inner, + self.tmaLoadAtomType(config), + self.coordTensorType(config.coordinate_rank, config.coordinate_layout), + (cute_nvgpu.TiledTmaLoadAttr.get(self.ctx, .{ .value = tmaLoadKind(config) }) catch @panic("invalid tiled TMA load")).attribute(), + .{ .num_multicast = .int(self.ctx, .i32, 1), .tma_format = tmaFormatAttribute(self, config) }, + self.loc(), + )); + return .{ + .atom = .{ .inner = results[0].inner, .kernel = self }, + .tensor = .{ .inner = results[1].inner, .kernel = self }, + }; + } + + /// Host-visible tiled TMA store atom and its coordinate tensor. + pub fn makeTiledTmaStoreAtom(self: *Builder, gmem: anytype, smem_layout: anytype, cta_map: anytype, config: TmaConfig) TmaDescriptor { + const results = self.emitResults(2, cute_nvgpu.atom_make_non_exec_tiled_tma_store( + self.ctx, + self.asValue(gmem).inner, + self.asValue(smem_layout).inner, + self.asValue(cta_map).inner, + self.tmaStoreAtomType(config), + self.coordTensorType(config.coordinate_rank, config.coordinate_layout), + .{ .tma_format = tmaFormatAttribute(self, config) }, + self.loc(), + )); + return .{ + .atom = .{ .inner = results[0].inner, .kernel = self }, + .tensor = .{ .inner = results[1].inner, .kernel = self }, + }; + } + + /// Host-side tiled TMA load atom type described by `config`. + pub fn tmaLoadAtomType(self: *Builder, config: TmaConfig) *const mlir.Type { + if (config.mode == .gather4) return (cute_nvgpu.CopyAtomNonExec2DGather4TmaLoadType.get(self.ctx, .{ + .kind = if (config.cta_group == .two) .sm_100_2sm else .sm_100, + .valType = config.dtype.toMlir(self.ctx), + .copyBits = @intCast(config.copy_bits), + .tmaGbasis = self.layoutType(config.global_basis), + .tmaFormat = if (config.format == .default) null else tmaDataFormat(config.format, config.dtype), + }) catch @panic("invalid gather TMA atom")).type_(); + return (cute_nvgpu.CopyAtomNonExecTiledTmaLoadType.get(self.ctx, .{ + .kind = tmaLoadKind(config), + .valType = config.dtype.toMlir(self.ctx), + .copyBits = @intCast(config.copy_bits), + .tmaGbasis = self.layoutType(config.global_basis), + .tmaFormat = tmaDataFormat(config.format, config.dtype), + }) catch @panic("invalid TMA load atom")).type_(); + } + + /// Executable device-side form of a tiled TMA load atom. `make_exec_tma` + /// preserves the tensor-map payload and adds the launch mode used by + /// `copy`. + pub fn tmaLoadExecAtomType(self: *Builder, config: TmaConfig) *const mlir.Type { + return (cute_nvgpu.CopyAtomTmaLoadType.get(self.ctx, .{ + .valType = config.dtype.toMlir(self.ctx), + .copyBits = @intCast(config.copy_bits), + .mode = if (config.mode == .gather4) .gather4 else .tiled, + .numCta = ctaCount(config.cta_group), + .gStride = self.emptyStrideType(), + .tmaGbasis = self.layoutType(config.global_basis), + }) catch @panic("invalid TMA load atom")).type_(); + } + + /// Host-side tiled TMA store atom type described by `config`. + pub fn tmaStoreAtomType(self: *Builder, config: TmaConfig) *const mlir.Type { + return (cute_nvgpu.CopyAtomNonExecTiledTmaStoreType.get(self.ctx, .{ + .valType = config.dtype.toMlir(self.ctx), + .copyBits = @intCast(config.copy_bits), + .tmaGbasis = self.layoutType(config.global_basis), + .tmaFormat = tmaDataFormat(config.format, config.dtype), + }) catch @panic("invalid TMA store atom")).type_(); + } + + /// Executable device-side form of a tiled TMA store atom. + pub fn tmaStoreExecAtomType(self: *Builder, config: TmaConfig) *const mlir.Type { + return (cute_nvgpu.CopyAtomTmaStoreType.get(self.ctx, .{ + .valType = config.dtype.toMlir(self.ctx), + .copyBits = @intCast(config.copy_bits), + .mode = .tiled, + .gStride = self.emptyStrideType(), + .tmaGbasis = self.layoutType(config.global_basis), + }) catch @panic("invalid TMA store atom")).type_(); + } + + /// The `()` global stride of tiled TMA atoms. + fn emptyStrideType(self: *Builder) *const mlir.Type { + return (cute_ir.StrideType.get(self.ctx, .{ .attr = self.algebraAttr(cute_ir.StrideAttr, .{ .tuple = &.{} }).attribute() }) catch @panic("invalid CuTe stride")).type_(); + } + + /// Generic address (i64) of the tensor map in an executable tiled TMA atom. + pub fn tmaDescriptorAddress(self: *Builder, descriptor: Atom) Value { + const ptr = self.emit(cute_nvgpu.get_tma_desc_addr(self.ctx, descriptor.inner, (cute_ir.PtrType.get(self.ctx, .{ .valueType = (cute_nvgpu.TmaDescriptorTiledType.get(self.ctx, .{}) catch unreachable).type_(), .addressSpace = .generic, .alignment = 64 }) catch unreachable).type_(), self.loc())); + return self.ptrToInt(ptr, .i64); + } + + pub fn prefetchTmaDesc(self: *Builder, descriptor: Atom) void { + self.emitVoid(cute_nvgpu.prefetch_tma_desc(self.ctx, descriptor.inner, self.loc())); + } + + pub fn makeExecTma(self: *Builder, descriptor: Atom, result_type: *const mlir.Type) Atom { + const result = self.emit(cute_nvgpu.atom_make_exec_tma(self.ctx, descriptor.inner, result_type, .{}, self.loc())); + return .{ .inner = result.inner, .kernel = self }; + } + + /// Attach the mbarrier that an executable TMA load atom completes. + pub fn setTmaBarrier(self: *Builder, atom: Atom, mbar: Value) Atom { + const result = self.emit(cute_nvgpu.atom_set_value( + self.ctx, + atom.inner, + mbar.inner, + atom.type_(), + (cute_nvgpu.AtomCopyFieldTmaLoadAttr.get(self.ctx, .{ .value = .tma_bar }) catch @panic("invalid TMA load field")).attribute(), + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + /// Partition `smem` and `targets` for the TMA atom `descriptor`; the + /// dependent result types are built by the caller. + pub fn tmaPartition( + self: *Builder, + descriptor: Atom, + cta_coord: anytype, + cta_layout: anytype, + smem: anytype, + targets: anytype, + smem_type: *const mlir.Type, + target_types: []const *const mlir.Type, + ) TmaPartition { + if (targets.len != target_types.len) @panic("tmaPartition: target count does not match result type count"); + const views = self.arena.allocator().alloc(View, target_types.len) catch @panic("OOM"); + const op = cute_nvgpu.atom_tma_partition( + self.ctx, + descriptor.inner, + self.makeCoord(cta_coord), + self.asValue(cta_layout).inner, + self.asValue(smem).inner, + self.innerSlice(targets), + smem_type, + target_types, + self.loc(), + ); + _ = op.appendTo(self.currentBlock()); + for (views, 0..) |*view, i| view.* = .{ .inner = op.result(i + 1), .kernel = self }; + return .{ .smem = .{ .inner = op.result(0), .kernel = self }, .targets = views }; + } + + // ==================== cute_nvgpu.tcgen05 (SM100) ==================== + + /// Tiled MMA type of the block-scaled atom described by `cfg`. + pub fn blockScaledMmaType(self: *Builder, cfg: BlockScaledMmaConfig) *const mlir.Type { + return (cute_ir.TiledMmaType.get(self.ctx, .{ + .mmaAtom = self.blockScaledAtomType(cfg), + .atomLayoutMNK = self.layoutAttr(self.layoutSpec(.{ 1, 1, 1 }, .{ 0, 0, 0 })).attribute(), + .permutationMNK = self.algebraAttr(cute_ir.TileAttr, self.expr(.{ AlgebraToken.all, AlgebraToken.all, AlgebraToken.all })).attribute(), + }) catch @panic("invalid block-scaled tiled MMA")).type_(); + } + + fn blockScaledAtomType(self: *Builder, cfg: BlockScaledMmaConfig) *const mlir.Type { + cfg.validate(); + return (cute_nvgpu.MmaAtomSM100UMMABlockScaledType.get(self.ctx, .{ + .shapeMnk = self.algebraAttr(cute_ir.ShapeAttr, self.expr(.{ cfg.m, cfg.n, cfg.k() })).attribute(), + .numCta = ctaCount(cfg.cta_group), + .aMajor = .k, + .bMajor = .k, + .aType = DType.f4e2m1fn.toMlir(self.ctx), + .bType = cfg.bType().toMlir(self.ctx), + .cType = DType.f32.toMlir(self.ctx), + .sfType = cfg.scaleType().toMlir(self.ctx), + .aFragKind = .smem_desc, + .vecSize = cfg.vecSize(), + }) catch @panic("invalid block-scaled MMA atom")).type_(); + } + + /// Block-scaled atom and its tiled MMA. Both scale-factor pointers start + /// at `scale_ptr` and are patched after tensor-memory allocation. + pub fn makeBlockScaledMma(self: *Builder, cfg: BlockScaledMmaConfig, scale_ptr: Value) BlockScaledMma { + const disabled = self.constant(.i1, false); + const atom = self.makeAtom(self.blockScaledAtomType(cfg), .{ disabled, disabled, disabled, scale_ptr, scale_ptr }); + const tiled = self.emit(cute_ir.make_tiled_mma(self.ctx, atom.inner, self.blockScaledMmaType(cfg), self.loc())); + return .{ .atom = atom, .tiled = .{ .inner = tiled.inner, .kernel = self } }; + } + + /// Enable (or disable) accumulation into C for the next block-scaled MMA. + pub fn setMmaAccumulate(self: *Builder, mma: Atom, enabled: Value) Atom { + const result = self.emit(cute_nvgpu.atom_set_value( + self.ctx, + mma.inner, + enabled.inner, + mma.type_(), + (cute_nvgpu.AtomMmaFieldSM100BlockScaledAttr.get(self.ctx, .{ .value = .accum_c }) catch @panic("invalid block-scaled MMA field")).attribute(), + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + pub const Tcgen05CommitOptions = struct { + /// Must match the group of the operations being tracked. + cta_group: CtaGroup = .one, + /// Arrive at the same barrier offset in each CTA whose rank bit is set. + /// Without a mask, arrive only on the issuing CTA's barrier. + multicast_mask: ?Value = null, + }; + + /// Track this thread's preceding tcgen05 MMA/copy operations of the + /// selected group, signaling one arrival per destination on completion. + /// The same elected thread that issues those operations must commit them. + /// For `.two`, either CTA in the pair can be the issuer. All tcgen05 + /// operations in a kernel must use the same group. + pub fn tcgen05Commit(self: *Builder, mbar: Value, opts: Tcgen05CommitOptions) void { + self.emitVoid(nvvm.tcgen05_commit( + self.ctx, + self.sharedBarrierPtr(mbar).inner, + opts.cta_group.toNvvm(), + if (opts.multicast_mask) |mask| mask.to(.i16).inner else null, + self.loc(), + )); + } + + /// Wait until asynchronous tensor-memory loads have reached registers. + pub fn fenceTmemLoad(self: *Builder) void { + self.emitVoid(nvvm.tcgen05_wait(self.ctx, .load, self.loc())); + } + + pub const TmemOptions = struct { cta_group: CtaGroup = .one, exclusive: bool = false }; + + pub fn allocTmem(self: *Builder, columns: u16, holding_ptr: anytype, opts: TmemOptions) void { + self.emitVoid(cute_nvgpu.arch_sm100_alloc_tmem( + self.ctx, + self.constant(.i32, columns).inner, + self.asValue(holding_ptr).inner, + .{ .is_two_cta = if (opts.cta_group == .two) .unit(self.ctx) else null, .exclusive = if (opts.exclusive) .unit(self.ctx) else null }, + self.loc(), + )); + } + + pub fn retrieveTmemPtr(self: *Builder, holding_ptr: anytype, dtype: DType, alignment: u64) Value { + const ptr_type = self.ptrTy(dtype, .tmem, alignment) catch @panic("invalid tensor-memory pointer type"); + return self.emit(cute_nvgpu.arch_sm100_retrieve_tmem_ptr(self.ctx, self.asValue(holding_ptr).inner, ptr_type, self.loc())); + } + + pub fn relinquishTmemAllocPermit(self: *Builder, opts: TmemOptions) void { + self.emitVoid(cute_nvgpu.arch_sm100_relinquish_tmem_alloc_permit(self.ctx, .{ .is_two_cta = if (opts.cta_group == .two) .unit(self.ctx) else null }, self.loc())); + } + + pub fn deallocTmem(self: *Builder, pointer: anytype, columns: u16, opts: TmemOptions) void { + self.emitVoid(cute_nvgpu.arch_sm100_dealloc_tmem( + self.ctx, + self.asValue(pointer).inner, + self.constant(.i32, columns).inner, + .{ .is_two_cta = if (opts.cta_group == .two) .unit(self.ctx) else null, .exclusive = if (opts.exclusive) .unit(self.ctx) else null }, + self.loc(), + )); + } + + pub const TmemLoadOptions = struct { + num_dp: u16, + num_b: u16, + num_rep: u16, + pack_16: bool = false, + }; + + pub fn tmemLoad(self: *Builder, src: anytype, result_type: *const mlir.Type, opts: TmemLoadOptions) View { + const result = self.emit(cute_nvgpu.arch_copy_SM100_tmem_load( + self.ctx, + self.asValue(src).inner, + result_type, + .int(self.ctx, .i32, opts.num_dp), + .int(self.ctx, .i32, opts.num_b), + .int(self.ctx, .i32, opts.num_rep), + .{ .pack_16 = if (opts.pack_16) .boolean(self.ctx, true) else null }, + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + /// Direct tensor-memory load into an SSA vector. Higher-level tiled-copy + /// helpers remain preferable when their dependent layouts are available. + pub fn tmemLoadVector(self: *Builder, src: anytype, lanes: i64, opts: TmemLoadOptions) Value { + // The NVVM-facing operation transports raw 32-bit register words. + // Callers reinterpret each lane as the accumulator element type. + const result_type = mlir.Type.vector(&.{lanes}, DType.i32.toMlir(self.ctx)); + return self.emit(cute_nvgpu.arch_copy_SM100_tmem_load( + self.ctx, + self.asValue(src).inner, + result_type, + .int(self.ctx, .i32, opts.num_dp), + .int(self.ctx, .i32, opts.num_b), + .int(self.ctx, .i32, opts.num_rep), + .{ .pack_16 = if (opts.pack_16) .boolean(self.ctx, true) else null }, + self.loc(), + )); + } + + pub const S2tCopyConfig = struct { + num_dp: i32, + num_bit: i32, + num_cta: i32 = 1, + broadcast: cute_nvgpu.CopyS2TBroadcast = .none, + }; + + /// `!cute_nvgpu.atom.s2t_copy`, a `tcgen05.cp` atom moving `dtype`. + pub fn s2tCopyAtomType(self: *Builder, dtype: DType, cfg: S2tCopyConfig) *const mlir.Type { + return (cute_nvgpu.CopyAtomSM100CopyS2TType.get(self.ctx, .{ + .valType = dtype.toMlir(self.ctx), + .numDp = cfg.num_dp, + .numBit = cfg.num_bit, + .numCta = cfg.num_cta, + .broadcast = cfg.broadcast, + }) catch @panic("invalid S2T copy atom")).type_(); + } + + pub fn makeS2tCopy(self: *Builder, atom: Atom, tmem: anytype, result_type: *const mlir.Type) Atom { + const result = self.emit(cute_nvgpu.atom_make_s2t_copy( + self.ctx, + atom.inner, + self.asValue(tmem).inner, + result_type, + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn makeTmemCopy(self: *Builder, atom: Atom, tmem: anytype, result_type: *const mlir.Type) Atom { + const result = self.emit(cute_nvgpu.atom_make_tmem_copy( + self.ctx, + atom.inner, + self.asValue(tmem).inner, + result_type, + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + pub fn s2tSmemDescriptor(self: *Builder, atom: Atom, input: anytype, result_type: *const mlir.Type) View { + const result = self.emit(cute_nvgpu.atom_get_copy_s2t_smem_desc_view( + self.ctx, + atom.inner, + self.asValue(input).inner, + result_type, + self.loc(), + )); + return .{ .inner = result.inner, .kernel = self }; + } + + /// Raw SM100 TMA load. The descriptor address, destinations and + /// coordinates come from the executable atom and partition operations. + pub fn tmaLoad( + self: *Builder, + descriptor_addr: Value, + data_addr: Value, + barrier_addr: Value, + coords: []const Value, + multicast_mask: ?Value, + offsets: []const Value, + cache_policy: ?Value, + mode: *const mlir.Attribute, + num_cta: u8, + ) void { + self.emitVoid(cute_nvgpu.arch_copy_SM100_tma_load( + self.ctx, + descriptor_addr.inner, + data_addr.inner, + barrier_addr.inner, + self.innerSlice(coords), + self.innerSlice(offsets), + mode, + .int(self.ctx, .i32, num_cta), + .{ .multicast_mask = if (multicast_mask) |v| v.inner else null, .cache_policy = if (cache_policy) |v| v.inner else null }, + self.loc(), + )); + } + + /// Raw SM100 TMA store. + pub fn tmaStore( + self: *Builder, + descriptor_addr: Value, + data_addr: Value, + coords: []const Value, + cache_policy: ?Value, + mode: *const mlir.Attribute, + ) void { + self.emitVoid(cute_nvgpu.arch_copy_SM100_tma_store( + self.ctx, + descriptor_addr.inner, + data_addr.inner, + self.innerSlice(coords), + mode, + .{ .cache_policy = if (cache_policy) |v| v.inner else null }, + self.loc(), + )); + } + + // ==================== cute.arch ==================== + + fn sreg(self: *Builder, comptime reg: []const u8) Index3 { + var out: [3]Value = undefined; + inline for (.{ "x", "y", "z" }, 0..) |axis, i| { + out[i] = self.emit(nvvm.read_sreg(self.ctx, reg ++ "." ++ axis, self.loc())); + } + return .{ .x = out[0], .y = out[1], .z = out[2] }; + } + + pub fn threadIdx(self: *Builder) Index3 { + return self.sreg("tid"); + } + + pub fn blockIdx(self: *Builder) Index3 { + return self.sreg("ctaid"); + } + + pub fn blockDim(self: *Builder) Index3 { + return self.sreg("ntid"); + } + + pub fn gridDim(self: *Builder) Index3 { + return self.sreg("nctaid"); + } + + /// Mark a scalar as identical in every lane of its warp. CuTe uses this + /// fact when lowering role branches that contain warp-collective TMA and + /// tensor-memory operations. + pub fn makeWarpUniform(self: *Builder, value: anytype) Value { + const scalar = self.asValue(value); + // The CuTe Python frontend legalizes predicates through an i32: + // extui i1 -> i32; make_warp_uniform i32; cmpi ne 0 + // `arch.make_warp_uniform` on i1 parses, but the native SM100 + // pipeline does not lower it all the way to NVVM. + const uniform_input = if (scalar.dtype() == .i1) scalar.to(.i32) else scalar; + const uniform = self.emit(cute_nvgpu.arch_make_warp_uniform( + self.ctx, + uniform_input.inner, + uniform_input.type_(), + self.loc(), + )); + return if (scalar.dtype() == .i1) uniform.ne(0) else uniform; + } + + /// `cute.arch.sync_threads()`. + pub fn syncThreads(self: *Builder) void { + _ = nvvm.barrier(self.ctx, self.loc()).appendTo(self.currentBlock()); + } + + /// `bar.sync id, participants` (`barrier.cta.sync.aligned`). + /// `id` is an integer or an i32 Value in [0, 15]. `participants` must be + /// a positive warp multiple, and every participating thread must execute + /// the same barrier id. The aligned form requires CTA-uniform control flow. + pub fn barrierCtaSyncAligned(self: *Builder, id: anytype, participants: anytype) void { + if (@TypeOf(participants) == ?Value) { + if (participants) |count| return self.barrierCtaSyncAligned(id, count); + return self.barrierCtaSyncAligned(id, null); + } + if (@TypeOf(participants) != Value and @TypeOf(participants) != @TypeOf(null)) { + std.debug.assert(participants > 0 and participants <= 1024 and @mod(participants, 32) == 0); + } + if (@TypeOf(id) == Value) { + std.debug.assert(id.dtype() == .i32); + } else { + const immediate: i32 = id; + std.debug.assert(immediate >= 0 and immediate < 16); + } + // The LLVM intrinsic preserves convergence and both operands across + // the embedded compiler's NVVM dialect version boundary. + if (@TypeOf(participants) == @TypeOf(null)) { + _ = self.callIntrinsic("llvm.nvvm.barrier.cta.sync.aligned", &.{self.lift(id).to(.i32)}, null); + } else { + _ = self.callIntrinsic("llvm.nvvm.barrier.cta.sync.aligned.count", &.{ self.lift(id).to(.i32), self.lift(participants).to(.i32) }, null); + } + } + + pub const BarrierOptions = struct { barrier_id: ?Value = null, number_of_threads: ?Value = null }; + + pub fn barrier(self: *Builder, opts: BarrierOptions) void { + self.barrierCtaSyncAligned(opts.barrier_id orelse self.constant(.i32, 0), opts.number_of_threads); + } + + /// Build the CuTe first-class divisor used by CUTLASS's persistent tile + /// scheduler. Its arithmetic is lowered to the same multiply/shift + /// sequence as Python CuTe DSL rather than an integer div/rem pair. + pub fn fastDivmodCreate(self: *Builder, divisor_: anytype) Value { + const divisor = self.lift(divisor_).to(.i32); + return self.emit(cute_ir.fast_divmod_create_divisor( + self.ctx, + divisor.inner, + (cute_ir.FastDivmodDivisorType.get(self.ctx, .{ .width = 32, .isPow2 = false }) catch @panic("invalid fast divmod divisor")).type_(), + self.loc(), + )); + } + + /// Return `{ quotient, remainder }` for a CUTLASS FastDivmod divisor. + pub fn fastDivmod(self: *Builder, dividend_: anytype, divisor: Value) [2]Value { + const dividend = self.lift(dividend_).to(.i32); + return self.emitResults(2, cute_ir.fast_divmod_compute( + self.ctx, + dividend.inner, + divisor.inner, + DType.i32.toMlir(self.ctx), + DType.i32.toMlir(self.ctx), + self.loc(), + )); + } + + /// Elect one lane from the active warp. Used by TMA issue, tensor-memory + /// allocation, and tcgen05 commit operations. + pub fn electSync(self: *Builder) Value { + return self.emit(nvvm.elect_sync(self.ctx, self.loc())); + } + + /// `cute.arch.shuffle_sync_bfly`: exchange a scalar with the lane at + /// `lane_id ^ offset` in the current warp. + /// Warp-wide maximum of a non-negative float, as one `redux.sync` instead + /// of a shuffle reduction: IEEE bit patterns of non-negative floats order + /// like unsigned integers. + pub fn warpMaxNonNegative(self: *Builder, value: Value) Value { + const dtype = value.dtype(); + if (dtypeBitwidth(dtype) != 32) @panic("warpMaxNonNegative requires a 32-bit scalar"); + const bits = if (dtype == .i32) value else value.bitCast(.i32); + const reduced = self.emit(llvm.inline_asm( + self.ctx, + "redux.sync.max.u32 $0, $1, 0xffffffff;", + &.{bits.inner}, + .{ .constraints = "=r,r", .result_type = .int(self.ctx, .i32) }, + self.loc(), + )); + return if (dtype == .i32) reduced else reduced.bitCast(dtype); + } + + pub fn shuffleXor(self: *Builder, value: Value, offset: i32) Value { + const dtype = value.dtype(); + if (dtypeBitwidth(dtype) != 32) @panic("shuffleXor currently requires a 32-bit scalar"); + const bits = if (dtype == .i32) value else value.bitCast(.i32); + const shuffled = self.emit(llvm.inline_asm( + self.ctx, + "shfl.sync.bfly.b32 $0, $1, $2, 31, 0xffffffff;", + &.{ bits.inner, self.constant(.i32, offset).inner }, + .{ .constraints = "=r,r,r", .result_type = .int(self.ctx, .i32) }, + self.loc(), + )); + return if (dtype == .i32) shuffled else shuffled.bitCast(dtype); + } + + /// `cute.arch.shuffle_sync`: read a scalar from lane `lane` of the + /// current warp. Every lane must participate. + pub fn shuffleIdx(self: *Builder, value: Value, lane: anytype) Value { + const dtype = value.dtype(); + if (dtypeBitwidth(dtype) != 32) @panic("shuffleIdx currently requires a 32-bit scalar"); + const bits = if (dtype == .i32) value else value.bitCast(.i32); + const shuffled = self.emit(llvm.inline_asm( + self.ctx, + "shfl.sync.idx.b32 $0, $1, $2, 31, 0xffffffff;", + &.{ bits.inner, self.constant(.i32, lane).inner }, + .{ .constraints = "=r,r,r", .result_type = .int(self.ctx, .i32) }, + self.loc(), + )); + return if (dtype == .i32) shuffled else shuffled.bitCast(dtype); + } + + /// Four FP32 values rounded to FP8 E4M3 (saturating) and packed into one + /// 32-bit word, `a` in the lowest byte. + pub fn packFp8x4(self: *Builder, a: Value, b: Value, c: Value, d: Value) Value { + return self.emit(llvm.inline_asm( + self.ctx, + "{ .reg .b16 lo, hi; cvt.rn.satfinite.e4m3x2.f32 lo, $2, $1; cvt.rn.satfinite.e4m3x2.f32 hi, $4, $3; mov.b32 $0, {lo, hi}; }", + &.{ a.to(.f32).inner, b.to(.f32).inner, c.to(.f32).inner, d.to(.f32).inner }, + .{ .constraints = "=r,f,f,f,f", .result_type = .int(self.ctx, .i32) }, + self.loc(), + )); + } + + /// Eight FP32 values rounded to FP4 E2M1 (saturating, nearest-even) and + /// packed into one 32-bit word, `values[0]` in the lowest nibble: the + /// packed NVFP4 storage order. + pub fn packFp4x8(self: *Builder, values: [8]Value) Value { + var operands: [8]*const mlir.Value = undefined; + for (&operands, values) |*operand, value| operand.* = value.to(.f32).inner; + return self.emit(llvm.inline_asm( + self.ctx, + "{ .reg .b8 b0, b1, b2, b3; cvt.rn.satfinite.e2m1x2.f32 b0, $2, $1; cvt.rn.satfinite.e2m1x2.f32 b1, $4, $3; " ++ + "cvt.rn.satfinite.e2m1x2.f32 b2, $6, $5; cvt.rn.satfinite.e2m1x2.f32 b3, $8, $7; mov.b32 $0, {b0, b1, b2, b3}; }", + &operands, + .{ .constraints = "=r,f,f,f,f,f,f,f,f", .result_type = .int(self.ctx, .i32) }, + self.loc(), + )); + } + + /// Two FP32 values rounded to FP4 E2M1 (saturating, nearest-even) in the + /// low byte of a 32-bit word, `a` in the low nibble. + pub fn packFp4x2(self: *Builder, a: Value, b: Value) Value { + return self.emit(llvm.inline_asm( + self.ctx, + "{ .reg .b8 lo; cvt.rn.satfinite.e2m1x2.f32 lo, $2, $1; cvt.u32.u8 $0, lo; }", + &.{ a.to(.f32).inner, b.to(.f32).inner }, + .{ .constraints = "=r,f,f", .result_type = .int(self.ctx, .i32) }, + self.loc(), + )); + } + + /// The FP8 E4M3 byte at the low end of `bits` (an i32), decoded to FP32. + pub fn e4m3ToF32(self: *Builder, bits: Value) Value { + return self.emit(llvm.inline_asm( + self.ctx, + "{ .reg .b16 lo, hl, hh; .reg .b32 pair; cvt.u16.u32 lo, $1; cvt.rn.f16x2.e4m3x2 pair, lo; mov.b32 {hl, hh}, pair; cvt.f32.f16 $0, hl; }", + &.{bits.inner}, + .{ .constraints = "=f,r", .result_type = DType.f32.toMlir(self.ctx) }, + self.loc(), + )); + } + + /// One-operand FP32 PTX instruction, e.g. `ex2.approx.ftz.f32`. + pub fn unaryF32(self: *Builder, comptime instruction: []const u8, value: Value) Value { + return self.emit(llvm.inline_asm( + self.ctx, + instruction ++ " $0, $1;", + &.{value.to(.f32).inner}, + .{ .constraints = "=f,f", .result_type = DType.f32.toMlir(self.ctx) }, + self.loc(), + )); + } + + /// Signal programmatic dependent launches after this grid has consumed + /// its inputs. The caller must also launch the producer/consumer chain + /// with PDL enabled. + pub fn launchDependents(self: *Builder) void { + _ = llvm.inline_asm(self.ctx, "griddepcontrol.launch_dependents;", &.{}, .{ .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); + } + + /// Wait until the preceding grid has made its programmatic dependency + /// available. This is the consumer-side half of a PDL launch chain. + pub fn waitForDependency(self: *Builder) void { + _ = llvm.inline_asm(self.ctx, "griddepcontrol.wait;", &.{}, .{ .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); + } + + /// `cute.arch.alloc_smem(dtype, size_in_elems, alignment)`: a static + /// shared-memory pointer, sized at compile time. + pub fn allocSmem(self: *Builder, dt: DType, size_in_elems: i64, alignment: ?u64) Value { + const al = alignment orelse @max(1, dtypeBitwidth(dt) / 8); + const ty = self.ptrTy(dt, .smem, al) catch @panic("bad smem pointer"); + return self.emit(cute_nvgpu.arch_alloc_smem(self.ctx, ty, .int(self.ctx, .i32, size_in_elems), self.loc())); + } + + // mbarriers, async-proxy fences and bulk-copy groups. + + /// mbarrier operands are `!llvm.ptr<3>` for NVVM. A barrier is either a CuTe + /// shared-memory pointer or a raw `i32` shared-memory address. + fn sharedBarrierPtr(self: *Builder, mbar: Value) Value { + if (mbar.isInt()) return self.addressPtr(mbar, .shared); + return self.emit(mlir.unrealized_conversion_cast(self.ctx, &.{mbar.inner}, &.{llvm.pointerType(self.ctx, NVPTXAddressSpace.shared)}, self.loc())); + } + + /// Initialize a shared-memory mbarrier expecting `arrivals` per phase. + pub fn mbarrierInit(self: *Builder, mbar: Value, arrivals: u32) void { + self.emitVoid(nvvm.mbarrier_init(self.ctx, self.sharedBarrierPtr(mbar).inner, self.constant(.i32, arrivals).inner, self.loc())); + } + + /// Arrive and expect `bytes` of asynchronous transactions (SM90+), as a + /// TMA producer does before issuing its copies. + pub fn mbarrierArriveExpectTx(self: *Builder, mbar: Value, bytes: anytype) void { + self.emitVoid(nvvm.mbarrier_arrive_expect_tx(self.ctx, self.sharedBarrierPtr(mbar).inner, self.lift(bytes).to(.i32).inner, .cta, self.loc())); + } + + pub const MBarrierArriveOptions = struct { + peer_cta_rank_in_cluster: ?Value = null, + arrive_count: ?Value = null, + scope: MBarrierScope = .cta, + relaxed: bool = false, + }; + + /// Arrive locally or at the same offset in a peer CTA. Local arrivals + /// return the opaque state token; remote arrivals have no state result. + pub fn mbarrierArrive(self: *Builder, mbar: Value, opts: MBarrierArriveOptions) ?Value { + var ptr = self.sharedBarrierPtr(mbar); + if (opts.peer_cta_rank_in_cluster) |rank| { + ptr = self.emit(nvvm.mapa(self.ctx, ptr.inner, rank.inner, llvm.pointerType(self.ctx, NVPTXAddressSpace.shared_cluster), self.loc())); + } + const op = nvvm.ops.mbarrier_arrive(self.ctx, ptr.inner, .{ + .count = if (opts.arrive_count) |count| count.to(.i32).inner else null, + .scope = (nvvm.ops.MemScopeKindAttr.get(self.ctx, .{ .value = switch (opts.scope) { + .cta => .cta, + .cluster => .cluster, + } }) catch unreachable).attribute(), + .relaxed = .boolean(self.ctx, opts.relaxed), + .res_type = if (opts.peer_cta_rank_in_cluster == null) mlir.Type.int(self.ctx, .i64) else null, + }, self.loc()); + if (opts.peer_cta_rank_in_cluster == null) return self.emit(op); + self.emitVoid(op); + return null; + } + + pub const MBarrierTryWaitOptions = struct { + scope: MBarrierScope = .cta, + time_limit: ?Value = null, + }; + + /// Try an acquire wait for phase `phase` (i32 parity, 0 or 1). May suspend + /// briefly and return false before completion. `scope` selects which + /// threads this wait can synchronize with; the barrier remains local. + pub fn mbarrierTryWait(self: *Builder, mbar: Value, phase: Value, opts: MBarrierTryWaitOptions) Value { + const ptr = self.sharedBarrierPtr(mbar); + if (opts.time_limit) |ticks| { + return self.emit(nvvm.mbarrier_try_wait_parity_timelimit(self.ctx, ptr.inner, phase.inner, ticks.to(.i32).inner, opts.scope, self.loc())); + } + return self.emit(nvvm.mbarrier_wait_parity(self.ctx, ptr.inner, phase.inner, .@"try", opts.scope, self.loc())); + } + + pub const MBarrierWaitOptions = struct { + scope: MBarrierScope = .cta, + /// Hardware suspend hint in nanoseconds per attempt, not a total timeout. + time_limit: u32 = 10_000_000, + /// Probe before suspending, as in CuTe's pipeline consumer wait. + test_first: bool = true, + }; + + fn mbarrierWaitLoop(self: *Builder, mbar: Value, phase: Value, opts: MBarrierWaitOptions) void { + const ptr = self.sharedBarrierPtr(mbar); + const ticks = self.constant(.i32, opts.time_limit); + switch (opts.scope) { + // Preserve the operation recognized by CuTe's pipeline compiler. + .cta => self.emitVoid(nvvm.mbarrier_try_wait_parity(self.ctx, ptr.inner, phase.inner, ticks.inner, self.loc())), + .cluster => { + var pending = self.openWhile(.{}, .{}); + const ready = self.mbarrierTryWait(mbar, phase, .{ .scope = opts.scope, .time_limit = ticks }); + pending.yieldBefore(ready.eq(false), .{}); + pending.yieldAfter(.{}); + }, + } + } + + /// Acquire a pipeline phase at `scope`: test without suspending, then + /// block only when the phase is still outstanding. The initial test matters when + /// a producer reaches a full ring and must yield to the consumer that + /// recycles its next stage. + pub fn mbarrierWait(self: *Builder, mbar: Value, phase: Value, opts: MBarrierWaitOptions) void { + if (opts.test_first) { + const ready = self.mbarrierTestWait(mbar, phase, .{ .scope = opts.scope }); + var pending = self.openIf(ready.eq(false)); + self.mbarrierWaitLoop(mbar, phase, opts); + pending.close(); + } else { + self.mbarrierWaitLoop(mbar, phase, opts); + } + } + + /// Make mbarrier initialization visible to the async proxy (SM90+). + pub fn fenceMbarrierInit(self: *Builder) void { + self.emitVoid(nvvm.fence_mbarrier_init(self.ctx, self.loc())); + } + + /// Publish ordinary shared-memory writes to the asynchronous (TMA) proxy (SM90+). + pub fn fenceProxyAsyncShared(self: *Builder) void { + self.emitVoid(nvvm.fence_proxy(self.ctx, .@"async.shared", .cta, self.loc())); + } + + /// Close the current group of bulk asynchronous copies, e.g. TMA stores (SM90+). + pub fn cpAsyncBulkCommitGroup(self: *Builder) void { + self.emitVoid(nvvm.cp_async_bulk_commit_group(self.ctx, self.loc())); + } + + /// Wait until at most `pending` bulk copy groups are in flight; with + /// `read`, only until their sources have been read (SM90+). + pub fn cpAsyncBulkWaitGroup(self: *Builder, pending: u32, read: bool) void { + self.emitVoid(nvvm.cp_async_bulk_wait_group(self.ctx, pending, read, self.loc())); + } + + // ==================== raw-address SM90/SM100 primitives ==================== + // Hand-scheduled kernels address shared memory with `i32` offsets in the + // shared window, global memory with `i64` addresses and tensor memory with + // `i32` TMEM addresses (lane << 16 | column). These helpers emit the NVVM / + // LLVM operations for them directly, using inline PTX where needed. + + pub const NVPTXAddressSpace = llvm.NVPTXAddressSpace; + + /// Typed LLVM pointer (`llvm.inttoptr`) for an integer address in `space`. + pub fn addressPtr(self: *Builder, address: Value, comptime space: NVPTXAddressSpace) Value { + return self.emit(llvm.inttoptr(self.ctx, address.inner, llvm.pointerType(self.ctx, space), self.loc())); + } + + fn wordsType(self: *Builder, comptime n: usize) *const mlir.Type { + const word = mlir.Type.int(self.ctx, .i32); + return if (n == 1) word else mlir.Type.vector(&.{n}, word); + } + + fn splitWords(self: *Builder, comptime n: usize, v: Value) [n]Value { + var out: [n]Value = undefined; + if (n == 1) { + out[0] = v; + } else { + for (0..n) |i| out[i] = self.vectorExtract(v, @intCast(i), .i32); + } + return out; + } + + fn joinWords(self: *Builder, comptime n: usize, words: [n]Value) Value { + if (n == 1) return words[0]; + var inner: [n]*const mlir.Value = undefined; + for (words, &inner) |w, *x| x.* = w.inner; + return self.emit(vector.from_elements(self.ctx, &inner, self.wordsType(n), self.loc())); + } + + pub const LoadOptions = struct { + /// Read-only for the whole kernel (global: non-coherent `ld.global.nc`). + invariant: bool = false, + }; + + /// Load `n` (1, 2 or 4) consecutive 32-bit words from a typed pointer. + pub fn loadWordsPtr(self: *Builder, comptime n: usize, ptr: Value, opts: LoadOptions) [n]Value { + const v = self.emit(llvm.load(self.ctx, ptr.inner, self.wordsType(n), .{ .alignment = 4 * n, .invariant = opts.invariant }, self.loc())); + return self.splitWords(n, v); + } + + /// Load `n` (1, 2 or 4) consecutive 32-bit words at an integer address. + pub fn loadWords(self: *Builder, comptime n: usize, address: Value, comptime space: NVPTXAddressSpace, opts: LoadOptions) [n]Value { + return self.loadWordsPtr(n, self.addressPtr(address, space), opts); + } + + /// Store `n` (1, 2 or 4) consecutive 32-bit words through a typed pointer. + pub fn storeWordsPtr(self: *Builder, comptime n: usize, ptr: Value, words: [n]Value) void { + self.emitVoid(llvm.store(self.ctx, self.joinWords(n, words).inner, ptr.inner, .{ .alignment = 4 * n }, self.loc())); + } + + /// Store `n` (1, 2 or 4) consecutive 32-bit words at an integer address. + pub fn storeWords(self: *Builder, comptime n: usize, address: Value, comptime space: NVPTXAddressSpace, words: [n]Value) void { + self.storeWordsPtr(n, self.addressPtr(address, space), words); + } + + /// Store the low 16 bits of an `i32`. + pub fn storeHalf(self: *Builder, address: Value, comptime space: NVPTXAddressSpace, word: Value) void { + self.emitVoid(llvm.store(self.ctx, word.to(.i16).inner, self.addressPtr(address, space).inner, .{ .alignment = 2 }, self.loc())); + } + + /// `bar.red.or.pred`: named barrier over `threads` threads that also returns whether + /// `predicate` (i1) is true in any of them. + pub fn barrierReduceOr(self: *Builder, id: i32, threads: i32, predicate: Value) Value { + return self.callIntrinsic("llvm.nvvm.barrier.cta.red.or.aligned.count", &.{ self.constant(.i32, id), self.constant(.i32, threads), predicate }, mlir.Type.int(self.ctx, .i1)).?; + } + + /// `bar.warp.sync 0xffffffff`. + pub fn syncWarp(self: *Builder) void { + self.emitVoid(nvvm.bar_warp_sync(self.ctx, self.constant(.i32, -1).inner, self.loc())); + } + + /// Non-blocking acquire test of mbarrier phase `parity` (i32, 0 or 1). + pub fn mbarrierTestWait(self: *Builder, mbar: Value, parity: Value, opts: MBarrierTryWaitOptions) Value { + std.debug.assert(opts.time_limit == null); + return self.emit(nvvm.mbarrier_wait_parity(self.ctx, self.sharedBarrierPtr(mbar).inner, parity.inner, .@"test", opts.scope, self.loc())); + } + + /// Call an LLVM intrinsic by name (e.g. an `llvm.nvvm.*` intrinsic the NVVM dialect of + /// the CuTe compiler does not model). + pub fn callIntrinsic(self: *Builder, comptime name: []const u8, operands: []const Value, result: ?*const mlir.Type) ?Value { + const alloc = self.arena.allocator(); + const inner = alloc.alloc(*const mlir.Value, operands.len) catch @panic("OOM"); + for (operands, inner) |o, *x| x.* = o.inner; + const op = llvm.call_intrinsic(self.ctx, name, inner, .{ .result_type = result }, self.loc()); + if (result == null) { + self.emitVoid(op); + return null; + } + return self.emit(op); + } + + /// 16-byte `cp.async.cg` global -> shared. With `valid` (i1) false, the destination is + /// zero-filled and `src` is not read (src-size 0). + pub fn cpAsync16(self: *Builder, dst: Value, src: Value, valid: ?Value) void { + const d = self.addressPtr(dst, .shared); + const g = self.addressPtr(src, .global); + if (valid) |v| { + _ = self.callIntrinsic("llvm.nvvm.cp.async.cg.shared.global.16.s", &.{ d, g, self.select(v, self.constant(.i32, 16), self.constant(.i32, 0)) }, null); + } else { + _ = self.callIntrinsic("llvm.nvvm.cp.async.cg.shared.global.16", &.{ d, g }, null); + } + } + + /// Arrive on `barrier` once this thread's prior cp.async copies have landed; with + /// `noinc`, the arrival is counted in the barrier's expected count. + pub fn cpAsyncMbarrierArrive(self: *Builder, mbar: Value, comptime noinc: bool) void { + _ = self.callIntrinsic(if (noinc) "llvm.nvvm.cp.async.mbarrier.arrive.noinc.shared" else "llvm.nvvm.cp.async.mbarrier.arrive.shared", &.{self.sharedBarrierPtr(mbar)}, null); + } + + /// Wait until all of this thread's cp.async copies have landed. + pub fn cpAsyncWaitAll(self: *Builder) void { + self.emitVoid(nvvm.cp_async_commit_group(self.ctx, self.loc())); + self.emitVoid(nvvm.cp_async_wait_group(self.ctx, 0, self.loc())); + } + + /// Prefetch the global line of `address` (i64) into L2 with evict-last priority. + pub fn prefetchL2EvictLast(self: *Builder, address: Value) void { + _ = self.callIntrinsic("llvm.nvvm.prefetch.global.L2.evict.last", &.{self.addressPtr(address, .global)}, null); + } + + /// Warp-wide TMEM allocation of `columns`; the base address is written to `holder` (i32 smem). + pub fn tmemAllocAt(self: *Builder, holder: Value, columns: u32) void { + self.emitVoid(nvvm.tcgen05_alloc(self.ctx, self.addressPtr(holder, .shared).inner, self.constant(.i32, columns).inner, self.loc())); + } + + pub fn tmemDeallocAt(self: *Builder, taddr: Value, columns: u32) void { + self.emitVoid(nvvm.tcgen05_dealloc(self.ctx, self.addressPtr(taddr, .tensor).inner, self.constant(.i32, columns).inner, self.loc())); + } + + pub const Tcgen05Fence = nvvm.Tcgen05FenceKind; + + /// `tcgen05.fence::{before,after}_thread_sync`. + pub fn tcgen05Fence(self: *Builder, comptime kind: Tcgen05Fence) void { + self.emitVoid(nvvm.tcgen05_fence(self.ctx, kind, self.loc())); + } + + /// `tcgen05.wait::st`. + pub fn fenceTmemStore(self: *Builder) void { + self.emitVoid(nvvm.tcgen05_wait(self.ctx, .store, self.loc())); + } + + /// `tcgen05.mma.cta_group::1.kind::f16`: D (TMEM at `d`) = A . B (+ D when + /// `accumulate`, i1), with A and B given by shared-memory descriptors (i64) and the + /// instruction descriptor `idesc` (i32). + pub fn tcgen05MmaF16(self: *Builder, d: Value, adesc: Value, bdesc: Value, idesc: Value, accumulate: Value) void { + self.emitVoid(nvvm.tcgen05_mma(self.ctx, self.addressPtr(d, .tensor).inner, adesc.inner, bdesc.inner, idesc.inner, accumulate.inner, .f16, .cta_1, self.loc())); + } + + /// `tcgen05.ld.32x32b.x{n}`: thread i of the warp gets TMEM lane (warp base + i), + /// `n` consecutive 32-bit columns from `taddr`. + pub fn tmemLoad32x32b(self: *Builder, comptime n: usize, taddr: Value) [n]Value { + const v = self.emit(nvvm.tcgen05_ld(self.ctx, self.addressPtr(taddr, .tensor).inner, self.wordsType(n), .shape_32x32b, self.loc())); + return self.splitWords(n, v); + } + + /// `tcgen05.st.32x32b.x{n}` (see `tmemLoad32x32b`). + pub fn tmemStore32x32b(self: *Builder, comptime n: usize, taddr: Value, words: [n]Value) void { + self.emitVoid(nvvm.tcgen05_st(self.ctx, self.addressPtr(taddr, .tensor).inner, self.joinWords(n, words).inner, .shape_32x32b, self.loc())); + } + + /// Rank of this CTA in its cluster. + pub fn clusterCtaRank(self: *Builder) Value { + return self.emit(nvvm.read_sreg(self.ctx, "cluster.ctarank", self.loc())); + } + + /// The same shared-memory location (i32 address) in CTA `rank` of the cluster, as a + /// `shared::cluster` pointer. + pub fn mapaShared(self: *Builder, address: Value, rank: Value) Value { + return self.emit(nvvm.mapa(self.ctx, self.addressPtr(address, .shared).inner, rank.inner, llvm.pointerType(self.ctx, NVPTXAddressSpace.shared_cluster), self.loc())); + } + + /// Full cluster barrier (arrive.release + wait.acquire, aligned). + pub fn clusterSync(self: *Builder) void { + self.emitVoid(nvvm.cluster_arrive(self.ctx, true, self.loc())); + self.emitVoid(nvvm.cluster_wait(self.ctx, true, self.loc())); + } + + /// Warp-wide maximum of an f32 (`redux.sync.max.f32`, SM100). + pub fn warpMaxF32(self: *Builder, value: Value) Value { + return self.callIntrinsic("llvm.nvvm.redux.sync.fmax", &.{ value, self.constant(.i32, -1) }, mlir.Type.float(self.ctx, .f32)).?; + } + + /// Fast approximate 2^x (`ex2.approx.ftz.f32`). + pub fn exp2Approx(self: *Builder, value: Value) Value { + return self.emit(nvvm.ex2(self.ctx, value.inner, true, self.loc())); + } + + /// Fast approximate 1/x (`rcp.approx.ftz.f32`). + pub fn rcpApprox(self: *Builder, value: Value) Value { + return self.emit(nvvm.rcp_approx_ftz(self.ctx, value.inner, self.loc())); + } + + /// `vector<2 x dt>`, e.g. a bf16x2 pair packed in 32 bits. + fn pairType(self: *Builder, comptime dt: DType) *const mlir.Type { + return mlir.Type.vector(&.{2}, dt.toMlir(self.ctx)); + } + + fn bitcastTo(self: *Builder, value: Value, ty: *const mlir.Type) Value { + return self.emit(llvm.bitcast(self.ctx, value.inner, ty, self.loc())); + } + + /// Two FP8 e4m3 (low 16 bits of `pair`, i32) -> bf16x2 packed in an i32. + pub fn e4m3x2ToBf16x2(self: *Builder, pair: Value) Value { + const v = self.callIntrinsic("llvm.nvvm.e4m3x2.to.bf16x2.rn", &.{pair.to(.i16)}, self.pairType(.bf16)).?; + return self.bitcastTo(v, mlir.Type.int(self.ctx, .i32)); + } + + /// Two FP4 e2m1 (the low byte of `pair`, the even element in the low nibble) + /// -> bf16x2 packed in an i32. + pub fn e2m1x2ToBf16x2(self: *Builder, pair: Value) Value { + const v = self.callIntrinsic("llvm.nvvm.e2m1x2.to.bf16x2.rn", &.{pair.to(.i16)}, self.pairType(.bf16)).?; + return self.bitcastTo(v, mlir.Type.int(self.ctx, .i32)); + } + + /// Eight FP4 e2m1 (one word, even elements in the low nibbles) -> four bf16x2 packed + /// in i32s. The pairs are read as the low bytes of the 16-bit halves of `word` and + /// `word >> 8`, which ptxas maps to operand selectors (one shift per word). + pub fn e2m1x8ToBf16x2x4(self: *Builder, word: Value) [4]Value { + const halves = self.pairType(.i16); + const even = self.bitcastTo(word, halves); + const odd = self.bitcastTo(word.shrLogical(8), halves); + const pairs = [4]Value{ + self.vectorExtract(even, 0, .i16), self.vectorExtract(odd, 0, .i16), + self.vectorExtract(even, 1, .i16), self.vectorExtract(odd, 1, .i16), + }; + var out: [4]Value = undefined; + for (pairs, &out) |pair, *o| o.* = self.e2m1x2ToBf16x2(pair); + return out; + } + + /// bf16x2 * bf16x2 (round to nearest), both packed in i32. + pub fn mulBf16x2(self: *Builder, a: Value, b: Value) Value { + const ty = self.pairType(.bf16); + const p = self.emit(llvm.fmul(self.ctx, self.bitcastTo(a, ty).inner, self.bitcastTo(b, ty).inner, ty, self.loc())); + return self.bitcastTo(p, mlir.Type.int(self.ctx, .i32)); + } + + /// Two f32 -> a packed pair of `dt` (bf16 or f16) in an i32, `lo` in the low half. + pub fn packHalf2(self: *Builder, lo: Value, hi: Value, comptime dt: DType) Value { + const pair = self.emit(vector.from_elements(self.ctx, &.{ lo.to(dt).inner, hi.to(dt).inner }, self.pairType(dt), self.loc())); + return self.bitcastTo(pair, mlir.Type.int(self.ctx, .i32)); + } + + /// A packed pair of `dt` (bf16 or f16) in an i32 -> two f32 (low half first). + pub fn unpackHalf2(self: *Builder, word: Value, comptime dt: DType) [2]Value { + const pair = self.bitcastTo(word, self.pairType(dt)); + return .{ self.vectorExtract(pair, 0, dt).to(.f32), self.vectorExtract(pair, 1, dt).to(.f32) }; + } + + /// f32 -> bf16 bits (round to nearest even) in the low 16 bits of an i32. + pub fn f32ToBf16Bits(self: *Builder, value: Value) Value { + return self.emit(arith.extui(self.ctx, value.to(.bf16).bitCast(.i16).inner, mlir.Type.int(self.ctx, .i32), self.loc())); + } + + /// An i32 equal to `value` at run time that the compiler cannot constant-fold + /// (scaled by `%nctaid.z`, which is 1 for 2-D grids): e.g. a loop bound that must + /// keep a loop rolled (small code, fewer instruction-cache misses). + pub fn opaqueI32(self: *Builder, value: i32) Value { + return self.gridDim().z.mul(value); + } + + /// SM100 `tile::gather4` TMA load from a tiled 2-D descriptor address `desc` (i64). + /// Gather four rows `r` (i32) at column `col` into 128-byte-aligned shared `dst` (i32), + /// completing their bytes on an 8-byte-aligned local mbarrier `bar` (i32). + /// The descriptor must have row box extent 1 and no interleaving. `col` and `r` + /// are element coordinates; row length and out-of-bounds fill come from the + /// descriptor (zero or OOB-NaN). Arm the barrier for all four rows' bytes before waiting. + pub fn tmaGather4(self: *Builder, desc: Value, dst: Value, bar: Value, col: Value, r: [4]Value, policy: CacheHintSm100) void { + std.debug.assert(desc.dtype() == .i64 and dst.dtype() == .i32 and bar.dtype() == .i32 and col.dtype() == .i32); + for (r) |row| std.debug.assert(row.dtype() == .i32); + _ = llvm.inline_asm(self.ctx, "cp.async.bulk.tensor.2d.shared::cta.global.tile::gather4.mbarrier::complete_tx::bytes.cta_group::1.L2::cache_hint [$0], [$1, {$2, $3, $4, $5, $6}], [$7], $8;", &.{ dst.inner, desc.inner, col.inner, r[0].inner, r[1].inner, r[2].inner, r[3].inner, bar.inner, self.constant(.i64, policy).inner }, .{ .constraints = "r,l,r,r,r,r,r,r,l,~{memory}", .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); + } + + pub const BulkPrefetchOptions = struct { l2_cache_hint: ?Value = null }; + + /// Best-effort byte-range prefetch into L2; size must be a multiple of 16. + pub fn cpAsyncBulkPrefetch(self: *Builder, addr: Value, bytes: anytype, opts: BulkPrefetchOptions) void { + if (@TypeOf(bytes) != Value) std.debug.assert(bytes > 0 and bytes % 16 == 0); + const address = if (addr.isInt()) self.addressPtr(addr, .global) else self.llvmPointer(addr, .global); + self.emitVoid(nvvm.ops.cp_async_bulk_prefetch(self.ctx, address.inner, self.lift(bytes).to(.i32).inner, .{ + .l2CacheHint = if (opts.l2_cache_hint) |hint| hint.to(.i64).inner else null, + }, self.loc())); + } + + pub fn prefetchL1(self: *Builder, addr: Value) void { + const address = if (addr.isInt()) addr else self.ptrToInt(addr, .i64); + _ = llvm.inline_asm(self.ctx, "prefetch.global.L1 [$0];", &.{address.inner}, .{ .constraints = "l", .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); + } + + pub fn prefetchTensorMap(self: *Builder, desc: Value) void { + const address = if (desc.isInt()) desc else self.ptrToInt(desc, .i64); + _ = llvm.inline_asm(self.ctx, "prefetch.tensormap [$0];", &.{address.inner}, .{ .constraints = "l", .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); + } + + pub const OperandSource = union(enum) { smem: Value, tmem: Value }; + pub const Tcgen05MmaWsOptions = struct { + collector_b_buffer: ?nvvm.ops.Tcgen05MMACollectorBBuffer = null, + collector_op: ?nvvm.ops.Tcgen05MMACollectorOp = null, + col_b_zero_mask: ?Value = null, + }; + + /// Single-thread, asynchronous weight-stationary MMA, as in Python primitives. + /// The caller elects the issuer and commits/waits before consuming D. + pub fn tcgen05MmaWs(self: *Builder, kind: nvvm.Tcgen05MmaKind, d: Value, a: OperandSource, bdesc: Value, idesc: anytype, enable_input_d: Value, opts: Tcgen05MmaWsOptions) void { + std.debug.assert(kind == .f16 or kind == .tf32 or kind == .f8f6f4 or kind == .i8); + const adesc = switch (a) { + .smem => |desc| self.smemDescriptorToInt(desc), + .tmem => |ptr| if (ptr.isInt()) self.addressPtr(ptr, .tensor) else self.llvmPointer(ptr, .tensor), + }; + const dest = if (d.isInt()) self.addressPtr(d, .tensor) else self.llvmPointer(d, .tensor); + self.emitVoid(nvvm.ops.tcgen05_mma_ws(self.ctx, dest.inner, adesc.inner, self.smemDescriptorToInt(bdesc).inner, self.lift(idesc).to(.i32).inner, enable_input_d.to(.i1).inner, (nvvm.ops.Tcgen05MMAKindAttr.get(self.ctx, .{ .value = kind }) catch unreachable).attribute(), .{ + .collectorBBuffer = if (opts.collector_b_buffer) |v| (nvvm.ops.Tcgen05MMACollectorBBufferAttr.get(self.ctx, .{ .value = v }) catch unreachable).attribute() else null, + .collectorOp = if (opts.collector_op) |v| (nvvm.ops.Tcgen05MMACollectorOpAttr.get(self.ctx, .{ .value = v }) catch unreachable).attribute() else null, + .zeroColMask = if (opts.col_b_zero_mask) |v| v.to(.i64).inner else null, + }, self.loc())); + } + + pub fn makeUmmaSmemDesc(self: *Builder, src: Value, layout: LayoutSpec, major: OperandMajorMode, next_src: ?Value) Value { + return self.emit(cute_nvgpu.make_umma_smem_desc(self.ctx, src.inner, self.smemDescType(), self.layoutAttr(layout).attribute(), (cute_nvgpu.MajorModeAttr.get(self.ctx, .{ .value = switch (major) { + .k => .k, + .mn => .mn, + } }) catch unreachable).attribute(), .{ .nextSrc = if (next_src) |v| v.inner else null }, self.loc())); + } + + pub fn intToSmemDescriptor(self: *Builder, value: anytype) Value { + return self.emit(mlir.unrealized_conversion_cast(self.ctx, &.{self.lift(value).to(.i64).inner}, &.{self.smemDescType()}, self.loc())); + } + + pub fn smemDescriptorToInt(self: *Builder, desc: Value) Value { + if (desc.isInt()) return desc.to(.i64); + return self.emit(mlir.unrealized_conversion_cast(self.ctx, &.{desc.inner}, &.{mlir.Type.int(self.ctx, .i64)}, self.loc())); + } + + pub fn setMaxRegisterIncrease(self: *Builder, comptime count: u32) void { + self.setMaxRegister(count, .increase); + } + + pub fn setMaxRegisterDecrease(self: *Builder, comptime count: u32) void { + self.setMaxRegister(count, .decrease); + } + + fn setMaxRegister(self: *Builder, comptime count: u32, comptime action: enum { increase, decrease }) void { + comptime std.debug.assert(count >= 24 and count <= 256 and count % 8 == 0); + const text = std.fmt.comptimePrint("setmaxnreg.{s}.sync.aligned.u32 {d};", .{ if (action == .increase) "inc" else "dec", count }); + _ = llvm.inline_asm(self.ctx, text, &.{}, .{ .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); + } + + /// Specialized busy wait for hand-scheduled kernels; never hardware-suspends. + pub fn mbarrierSpin(self: *Builder, bar: Value, parity: Value, scope: MBarrierScope) void { + std.debug.assert(bar.dtype() == .i32 and parity.dtype() == .i32); + _ = llvm.inline_asm(self.ctx, switch (scope) { + .cta => "{ .reg .pred p; spin${:uid}: mbarrier.test_wait.parity.acquire.cta.shared::cta.b64 p, [$0], $1; @!p bra spin${:uid}; }", + .cluster => "{ .reg .pred p; spin${:uid}: mbarrier.test_wait.parity.acquire.cluster.shared::cta.b64 p, [$0], $1; @!p bra spin${:uid}; }", + }, &.{ bar.inner, parity.inner }, .{ .constraints = "r,r,~{memory}", .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); + } + + /// Convert a CuTe or LLVM pointer to the LLVM pointer required by NVVM. + fn llvmPointer(self: *Builder, ptr: Value, comptime space: NVPTXAddressSpace) Value { + if (!ptr.isPtr()) return ptr; + return self.emit(mlir.unrealized_conversion_cast(self.ctx, &.{ptr.inner}, &.{llvm.pointerType(self.ctx, space)}, self.loc())); + } + + /// Map a local shared pointer/address to a typed pointer in peer CTA memory. + pub fn mapDsmemPtr(self: *Builder, smem: Value, peer_cta_rank: Value) Value { + const ptr = if (smem.isInt()) self.addressPtr(smem, .shared) else self.llvmPointer(smem, .shared); + const remote = self.emit(nvvm.mapa(self.ctx, ptr.inner, peer_cta_rank.to(.i32).inner, llvm.pointerType(self.ctx, NVPTXAddressSpace.shared_cluster), self.loc())); + if (smem.type_().isA(cute_ir.PtrType)) |ty| { + std.debug.assert(ty.addressSpace() == .smem); + const result_type = (cute_ir.PtrType.get(self.ctx, .{ + .valueType = ty.valueType(), + .addressSpace = .dsmem, + .alignment = ty.alignment(), + .swizzle = ty.swizzle(), + .bitlayout = ty.bitlayout(), + }) catch @panic("invalid DSMEM pointer")).type_(); + return self.emit(cute_ir.inttoptr(self.ctx, self.ptrToInt(remote, .i32).inner, result_type, self.loc())); + } + return remote; + } + + /// Store one, two, or four i32 words in a peer CTA, completing bytes on its + /// mbarrier. Both inputs name local shared memory; both are mapped to the peer. + pub fn storeAsyncDsmem(self: *Builder, smem: Value, value: anytype, mbar: Value, peer_cta_rank: Value) void { + const n = if (@TypeOf(value) == Value) 1 else common.arity(@TypeOf(value), "storeAsyncDsmem: value"); + comptime std.debug.assert(n == 1 or n == 2 or n == 4); + if (smem.type_().isA(cute_ir.PtrType)) |ty| { + std.debug.assert(ty.addressSpace() == .smem and ty.alignment() % (n * 4) == 0); + } + if (mbar.type_().isA(cute_ir.PtrType)) |ty| { + std.debug.assert(ty.addressSpace() == .smem and ty.alignment() % 8 == 0); + } + const packed_value = if (@TypeOf(value) == Value) value else blk: { + var words: [n]Value = undefined; + inline for (0..n) |i| words[i] = self.lift(value[i]).to(.i32); + break :blk self.joinWords(n, words); + }; + self.emitVoid(nvvm.ops.store_async(self.ctx, self.llvmPointer(self.mapDsmemPtr(smem, peer_cta_rank), .shared_cluster).inner, packed_value.inner, self.llvmPointer(self.mapDsmemPtr(mbar, peer_cta_rank), .shared_cluster).inner, self.loc())); + } + + /// Explicit aligned variants retain the convergence requirement of raw PTX. + pub fn barrierClusterArriveRelaxedAligned(self: *Builder) void { + _ = llvm.inline_asm(self.ctx, "barrier.cluster.arrive.relaxed.aligned;", &.{}, .{ .constraints = "~{memory}", .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); + } + + pub fn barrierClusterWaitAligned(self: *Builder) void { + self.emitVoid(nvvm.cluster_wait(self.ctx, true, self.loc())); + } + + pub const ClusterArriveOptions = struct { aligned: bool = false }; + + pub fn clusterArriveRelaxed(self: *Builder, opts: ClusterArriveOptions) void { + if (opts.aligned) return self.barrierClusterArriveRelaxedAligned(); + _ = llvm.inline_asm(self.ctx, "barrier.cluster.arrive.relaxed;", &.{}, .{ .constraints = "~{memory}", .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); + } + + pub fn clusterWait(self: *Builder) void { + self.emitVoid(nvvm.cluster_wait(self.ctx, false, self.loc())); + } + + /// A null mask selects CuTe's FULL_MASK; integers and SSA masks also work. + pub fn voteAnySync(self: *Builder, pred: Value, mask: anytype) Value { + const active = if (@TypeOf(mask) == @TypeOf(null)) self.constant(.i32, -1) else self.lift(mask).to(.i32); + return self.callIntrinsic("llvm.nvvm.vote.any.sync", &.{ active, pred }, mlir.Type.int(self.ctx, .i1)).?; + } + + /// A null mask selects CuTe's FULL_MASK. + pub fn voteBallotSync(self: *Builder, pred: Value, mask: anytype) Value { + const active = if (@TypeOf(mask) == @TypeOf(null)) self.constant(.i32, -1) else self.lift(mask).to(.i32); + return self.callIntrinsic("llvm.nvvm.vote.ballot.sync", &.{ active, pred }, mlir.Type.int(self.ctx, .i32)).?; + } + + pub const FmaxOptions = struct { abs: bool = false, nan: bool = false, ftz: bool = false }; + + pub fn fmax(self: *Builder, x: Value, y: Value, opts: FmaxOptions) Value { + if (!opts.abs and !opts.nan and !opts.ftz) return self.callIntrinsic("llvm.nvvm.fmax.f", &.{ x, y }, mlir.Type.float(self.ctx, .f32)).?; + return self.emit(nvvm.ops.fmax(self.ctx, x.inner, y.inner, mlir.Type.float(self.ctx, .f32), .{ + .abs = if (opts.abs) .unit(self.ctx) else null, + .nan = if (opts.nan) .unit(self.ctx) else null, + .ftz = if (opts.ftz) .unit(self.ctx) else null, + }, self.loc())); + } + + fn vec2(self: *Builder, x: [2]Value) Value { + const ty = mlir.Type.vector(&.{2}, mlir.Type.float(self.ctx, .f32)); + return self.emit(vector.from_elements(self.ctx, &.{ x[0].inner, x[1].inner }, ty, self.loc())); + } + + fn unvec2(self: *Builder, v: Value) [2]Value { + return .{ self.vectorExtract(v, 0, .f32), self.vectorExtract(v, 1, .f32) }; + } + + pub const PackedF32x2Options = struct { rnd: nvvm.ops.FPRoundingMode = .rn, ftz: bool = false }; + + fn packedF32x2Attrs(self: *Builder, opts: PackedF32x2Options) struct { rnd: ?*const mlir.Attribute, ftz: ?*const mlir.Attribute } { + return .{ + .rnd = (nvvm.ops.FPRoundingModeAttr.get(self.ctx, .{ .value = opts.rnd }) catch unreachable).attribute(), + .ftz = if (opts.ftz) .unit(self.ctx) else null, + }; + } + + pub fn fmaPackedF32x2(self: *Builder, a: [2]Value, b: [2]Value, c: [2]Value, opts: PackedF32x2Options) [2]Value { + return self.unvec2(self.emit(nvvm.ops.fma_packed_f32x2(self.ctx, self.vec2(a).inner, self.vec2(b).inner, self.vec2(c).inner, mlir.Type.vector(&.{2}, mlir.Type.float(self.ctx, .f32)), .{ .rnd = self.packedF32x2Attrs(opts).rnd, .ftz = self.packedF32x2Attrs(opts).ftz }, self.loc()))); + } + + pub fn addPackedF32x2(self: *Builder, a: [2]Value, b: [2]Value, opts: PackedF32x2Options) [2]Value { + return self.unvec2(self.emit(nvvm.ops.add_packed_f32x2(self.ctx, self.vec2(a).inner, self.vec2(b).inner, mlir.Type.vector(&.{2}, mlir.Type.float(self.ctx, .f32)), .{ .rnd = self.packedF32x2Attrs(opts).rnd, .ftz = self.packedF32x2Attrs(opts).ftz }, self.loc()))); + } + + pub fn mulPackedF32x2(self: *Builder, a: [2]Value, b: [2]Value, opts: PackedF32x2Options) [2]Value { + return self.unvec2(self.emit(nvvm.ops.mul_packed_f32x2(self.ctx, self.vec2(a).inner, self.vec2(b).inner, mlir.Type.vector(&.{2}, mlir.Type.float(self.ctx, .f32)), .{ .rnd = self.packedF32x2Attrs(opts).rnd, .ftz = self.packedF32x2Attrs(opts).ftz }, self.loc()))); + } + + pub const StoreOptions = struct { + /// Address space for raw integer addresses; typed pointers retain theirs. + space: NVPTXAddressSpace = .global, + level1_eviction_priority: nvvm.ops.EvictKind = .evict_normal, + }; + + /// Scalar/vector store. Eight i32 words with no_allocate keep STG.E.256. + pub fn store(self: *Builder, addr: Value, value: anytype, comptime opts: StoreOptions) void { + if (@TypeOf(value) != Value) { + const n = common.arity(@TypeOf(value), "store: value"); + comptime std.debug.assert(n == 1 or n == 2 or n == 4 or n == 8); + var words: [n]Value = undefined; + inline for (0..n) |i| words[i] = self.lift(value[i]).to(.i32); + const is_global = if (addr.type_().isA(cute_ir.PtrType)) |ty| ty.addressSpace() == .gmem else addr.isInt() and opts.space == .global; + if (n == 8 and is_global and opts.level1_eviction_priority == .evict_no_allocate) { + const address = if (addr.isInt()) addr else self.ptrToInt(addr, .i64); + var args: [9]*const mlir.Value = undefined; + args[0] = address.inner; + for (words, 0..) |word, i| args[i + 1] = word.inner; + _ = llvm.inline_asm(self.ctx, "st.global.L1::no_allocate.v8.b32 [$0], {$1, $2, $3, $4, $5, $6, $7, $8};", &args, .{ .constraints = "l,r,r,r,r,r,r,r,r,~{memory}", .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); + return; + } + self.store(addr, self.joinWords(n, words), opts); + return; + } + const ptr = if (addr.isInt()) self.addressPtr(addr, opts.space) else if (addr.type_().isA(cute_ir.PtrType)) |ty| blk: { + const space: NVPTXAddressSpace = @fromBackingInt(@intCast(@backingInt(ty.addressSpace()))); + break :blk self.emit(mlir.unrealized_conversion_cast(self.ctx, &.{addr.inner}, &.{llvm.pointerType(self.ctx, space)}, self.loc())); + } else addr; + self.emitVoid(nvvm.ops.store_ext(self.ctx, value.inner, ptr.inner, .{ + .evict = if (opts.level1_eviction_priority == .evict_normal) null else (nvvm.ops.EvictKindAttr.get(self.ctx, .{ .value = opts.level1_eviction_priority }) catch unreachable).attribute(), + }, self.loc())); + } + + pub fn fma(self: *Builder, x: Value, y: Value, z: Value) Value { + return self.callIntrinsic("llvm.fma.f32", &.{ x, y, z }, mlir.Type.float(self.ctx, .f32)).?; + } + + pub fn clock(self: *Builder) Value { + return self.emit(llvm.inline_asm(self.ctx, "mov.u32 $0, %clock;", &.{}, .{ .result_type = mlir.Type.int(self.ctx, .i32), .constraints = "=r", .has_side_effects = true }, self.loc())); + } + + pub fn globalTimer(self: *Builder) Value { + return self.emit(llvm.inline_asm(self.ctx, "mov.u64 $0, %globaltimer;", &.{}, .{ .result_type = mlir.Type.int(self.ctx, .i64), .constraints = "=l", .has_side_effects = true }, self.loc())); + } + + // ==================== scalars ==================== + + /// `value` as a constant of `dt`; a Value passes through and enums use + /// their backing integers. An integer is signed or an unsigned bit + /// pattern of `dt`'s width; anything else panics. + pub fn constant(self: *Builder, dt: DType, value: anytype) Value { + if (@TypeOf(value) == Value) return value; + if (@typeInfo(@TypeOf(value)) == .@"enum") return self.constant(dt, @backingInt(value)); + const ctx = self.ctx; + const T = @TypeOf(value); + if (isFloatDtype(dt)) { + const f: f64 = switch (@typeInfo(T)) { + .comptime_int, .comptime_float => value, + .int => @floatFromInt(value), + .bool => @floatFromInt(@intFromBool(value)), + .float => @floatCast(value), + else => @compileError("Builder.constant: unsupported value type " ++ @typeName(T)), + }; + return self.emit(switch (dt) { + inline .f16, .bf16, .f32, .f64, .f4e2m1fn, .f8e4m3fn, .f8e5m2, .f8e8m0fnu => |ft| arith.constant_float(ctx, f, @field(mlir.FloatTypes, @tagName(ft)), self.loc()), + else => unreachable, + }); + } + const i: i128 = switch (@typeInfo(T)) { + .comptime_int, .int => value, + .bool => @intFromBool(value), + .comptime_float, .float => @intFromFloat(value), + else => @compileError("Builder.constant: unsupported value type " ++ @typeName(T)), + }; + return self.emit(arith.constant_int(ctx, intBits(i, @intCast(dtypeBitwidth(dt)), @tagName(dt)), dt.toMlir(ctx), self.loc())); + } + + /// `value` as the `i64` an integer constant of `bits` bits is built from. + /// It may be signed or an unsigned bit pattern, so it must lie in + /// [-2^(bits-1), 2^bits). + fn intBits(value: i128, bits: u7, dtype: []const u8) i64 { + if (value < -(@as(i128, 1) << (bits - 1)) or value >= @as(i128, 1) << bits) + std.debug.panic("Builder.constant: {d} does not fit {s}", .{ value, dtype }); + return @truncate(value); + } + + fn fitsInt(i: i64, dt: DType) bool { + if (dt == .i1) return i == 0 or i == 1; + const bits = dtypeBitwidth(dt); + if (bits >= 64) return true; + const shift: u6 = @intCast(bits - 1); + return i >= -(@as(i64, 1) << shift) and i < (@as(i64, 1) << shift); + } + + fn literalFits(value: anytype, dt: DType) bool { + return switch (@typeInfo(@TypeOf(value))) { + .comptime_int, .int => fitsInt(@intCast(value), dt), + .bool => !isFloatDtype(dt), + else => isFloatDtype(dt), + }; + } + + /// A Zig scalar as a constant: ints become `i32` when they fit, `i64` + /// otherwise; floats `f32`, like the Python DSL's `Int32`/`Float32`. + pub fn lift(self: *Builder, value: anytype) Value { + const T = @TypeOf(value); + if (T == Value) return value; + return switch (@typeInfo(T)) { + .comptime_int, .int => if (literalFits(value, .i32)) self.constant(.i32, value) else self.constant(.i64, value), + .bool => self.constant(.i1, value), + .comptime_float => self.constant(.f32, value), + .float => |info| switch (info.bits) { + 16 => self.constant(.f16, value), + 32 => self.constant(.f32, value), + 64 => self.constant(.f64, value), + else => @compileError("Builder.lift: unsupported float bitwidth"), + }, + else => @compileError("Builder.lift: unsupported type " ++ @typeName(T)), + }; + } + + /// `value` as a constant of `like`'s element type; a Value passes through. + pub fn constantLike(self: *Builder, value: anytype, like: Value) Value { + return self.constant(like.dtype(), value); + } + + fn liftLike(self: *Builder, value: anytype, ref: Value) Value { + const dt = ref.dtype(); + return if (literalFits(value, dt)) self.constant(dt, value) else self.lift(value); + } + + /// Lift a literal to the other operand's dtype when it fits, else to its + /// own width; then widen the narrower integer. + pub fn coerce(self: *Builder, a: anytype, b: anytype) struct { Value, Value } { + var av: Value = if (@TypeOf(a) == Value) a else if (@TypeOf(b) == Value) self.liftLike(a, b) else self.lift(a); + var bv: Value = if (@TypeOf(b) == Value) b else self.liftLike(b, av); + if (av.isInt() and bv.isInt()) { + const aw = dtypeBitwidth(av.dtype()); + const bw = dtypeBitwidth(bv.dtype()); + if (aw < bw) av = av.to(bv.dtype()); + if (bw < aw) bv = bv.to(av.dtype()); + } + return .{ av, bv }; + } + + fn cmp(self: *Builder, a: Value, b: anytype, ip: arith.CmpIPredicate, fp: arith.CmpFPredicate) Value { + const l, const r = self.coerce(a, b); + return self.emit(if (l.isFloat()) arith.cmpf(self.ctx, fp, l.inner, r.inner, self.loc()) else arith.cmpi(self.ctx, ip, l.inner, r.inner, self.loc())); + } + + pub fn select(self: *Builder, cond: Value, t: anytype, f: anytype) Value { + const tv, const fv = self.coerce(t, f); + return self.emit(arith.select(self.ctx, cond.inner, tv.inner, fv.inner, self.loc())); + } + + // ==================== scf ==================== + + pub fn scalarTy(self: *const Builder, dtype: DType) *const mlir.Type { + return dtype.toMlir(self.ctx); + } + + pub fn openIf(self: *Builder, cond: Value) IfOnlyScope { + return common.openIf(self, Value, cond); + } + + pub fn openIfElse(self: *Builder, cond: Value, result_types: anytype) IfScope(common.arity(@TypeOf(result_types), "openIfElse: result_types")) { + return common.openIfElse(self, Value, cond, result_types); + } + + /// `for iv in range(lower, upper, step)` with loop-carried `inits`. Literal + /// bounds take the type of a Value bound, else `lower` is lifted (`i32` + /// when it fits). + pub fn openFor( + self: *Builder, + lower: anytype, + upper: anytype, + step: anytype, + inits: anytype, + ) ForScope(common.arity(@TypeOf(inits), "openFor: inits")) { + return self.openForOpts(lower, upper, step, inits, .{}); + } + + /// `openFor` with `scf.for` options. + pub fn openForOpts( + self: *Builder, + lower: anytype, + upper: anytype, + step: anytype, + inits: anytype, + opts: common.ForOpts, + ) ForScope(common.arity(@TypeOf(inits), "openFor: inits")) { + return common.openFor(self, Value, common.forBounds(self, Value, lower, upper, step, Builder.lift), inits, opts); + } + + /// `scf.while` with explicit loop-carried values. The before region sees + /// `inits`, calls `yieldBefore(condition, forwarded)`, then the after + /// region sees `forwarded` and calls `yieldAfter(next_inits)`. This is the + /// control-flow shape emitted by the Python CuTe persistent scheduler. + pub fn openWhile( + self: *Builder, + inits: anytype, + after_types: anytype, + ) WhileScope(common.arity(@TypeOf(inits), "openWhile: inits"), common.arity(@TypeOf(after_types), "openWhile: after_types")) { + return common.openWhile(self, Value, inits, after_types); + } + + // ==================== module ==================== + + /// The module the CuTe compiler takes: the verified kernel as a public + /// `func.func`, which the compiler turns into the kernel entry. Grid and + /// block travel in the custom call; `block` is also pinned as + /// `nvvm.reqntid`. + pub fn finish(self: *Builder, block: [3]i32) FinishError![:0]const u8 { + const current = self.currentBlock(); + if (current.terminator() == null) { + _ = func.returns(self.ctx, &.{}, self.loc()).appendTo(current); + } + const func_op = self.func_op orelse return error.InvalidMlir; + func_op.setAttribute(nvvm.reqntid(self.ctx, block)); + try common.verify(self.module, "dsl/cute"); + + return self.renderModule(); + } +}; + +fn testContext() !*mlir.Context { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + registerDialects(registry); + func.registerExtensions(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + return ctx; +} + +fn expectContains(haystack: []const u8, needle: []const u8) !void { + if (std.mem.indexOf(u8, haystack, needle) == null) { + std.debug.print("missing {s} in:\n{s}\n", .{ needle, haystack }); + return error.TestExpectedContains; + } +} + +test "naive elementwise add matches the Python notebook kernel" { + const ctx = try testContext(); + defer ctx.deinit(); + + var b = try Builder.open(std.testing.allocator, ctx, "naive_elementwise_add_kernel"); + defer b.deinit(); + const a = try b.declareArgs(.{ + .gA = .{ .tensor = .{ .dtype = .f16, .shape = &.{ 16, 8 } } }, + .gB = .{ .tensor = .{ .dtype = .f16, .shape = &.{ 16, 8 } } }, + .gC = .{ .tensor = .{ .dtype = .f16, .shape = &.{ 16, 8 } } }, + }); + + const tidx = b.threadIdx().x; + const bidx = b.blockIdx().x; + const bdim = b.blockDim().x; + const thread_idx = bidx.mul(bdim).add(tidx); + const n = a.gA.dim(1); + const ni = thread_idx.rem(n); + const mi = thread_idx.div(n); + a.gC.set(.{ mi, ni }, a.gA.get(.{ mi, ni }).add(a.gB.get(.{ mi, ni }))); + + // The module round-trips through the local dialects. + const ir = try b.finish(.{ 128, 1, 1 }); + defer std.testing.allocator.free(ir); + try expectContains(ir, "\"func.func\"() <{function_type = (!cute.ptr>, !cute.ptr>, !cute.ptr>) -> (), sym_name = \"naive_elementwise_add_kernel\"}>"); + try expectContains(ir, "}) {cute.kernel, gpu.kernel, nvvm.reqntid = array} : () -> ()"); + try expectContains(ir, "\"nvvm.read.ptx.sreg.tid.x\"() : () -> i32"); + try expectContains(ir, "!cute.layout<\"(16,8):(8,1)\">"); + try expectContains(ir, "!cute.coord<\"(?,?)\">"); + try expectContains(ir, "cute.memref.load"); + try expectContains(ir, "cute.memref.store"); + try expectContains(ir, "arith.addf"); + try expectContains(ir, "\"func.return\"() : () -> ()\n })"); + try std.testing.expect(std.mem.indexOf(u8, ir, "gpu.module") == null); + const parsed = try mlir.Module.parse(ctx, ir); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "guard, shared memory, sync, layouts, for loop" { + const ctx = try testContext(); + defer ctx.deinit(); + + var b = try Builder.open(std.testing.allocator, ctx, "reverse_blocks"); + defer b.deinit(); + const a = try b.declareArgs(.{ + .x = .{ .ptr = .f32 }, + .out = .{ .tensor = .{ .dtype = .f32, .shape = &.{4096} } }, + }); + const gx = b.makeTensor(a.x, b.makeLayout(&.{4096}, .{})); + const smem = b.makeTensor(b.allocSmem(.f32, 128, 16), b.makeLayout(&.{128}, .{})); + + const tid = b.threadIdx().x; + const i = b.blockIdx().x.mul(128).add(tid); + smem.set(.{tid}, gx.get(.{i})); + b.syncThreads(); + b.barrierCtaSyncAligned(2, 128); + b.launchDependents(); + b.waitForDependency(); + const j = b.constant(.i32, 127).sub(tid); + var guard = b.openIf(i.lt(4096)); + { + var acc = b.openFor(0, 4, 1, .{b.constant(.f32, 0.0)}); + { + acc.yield(.{acc.carried[0].add(smem.get(.{j}).mul(acc.iv.to(.f32)))}); + } + a.out.set(.{i}, acc.results[0]); + guard.close(); + } + + const kernel = try b.finish(.{ 128, 1, 1 }); + defer std.testing.allocator.free(kernel); + try expectContains(kernel, "cute_nvgpu.arch.alloc_smem"); + try expectContains(kernel, "!cute.ptr>"); + try expectContains(kernel, "\"nvvm.barrier\"() <{operandSegmentSizes = array}> : () -> ()"); + try expectContains(kernel, "llvm.nvvm.barrier.cta.sync.aligned.count"); + try expectContains(kernel, "value = 2 : i32"); + try expectContains(kernel, "value = 128 : i32"); + try expectContains(kernel, "griddepcontrol.launch_dependents;"); + try expectContains(kernel, "griddepcontrol.wait;"); + try expectContains(kernel, "scf.if"); + try expectContains(kernel, "scf.for"); + try expectContains(kernel, "!cute.layout<\"4096:1\">"); + const parsed = try mlir.Module.parse(ctx, kernel); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "left-most default stride and static coordinate leaves" { + const ctx = try testContext(); + defer ctx.deinit(); + + var b = try Builder.open(std.testing.allocator, ctx, "k"); + defer b.deinit(); + const a = try b.declareArgs(.{ .p = .{ .ptr = .f32 } }); + const t = b.makeTensor(a.p, b.makeLayout(&.{ 4, 4 }, .{})); + try std.testing.expectEqualSlices(i64, &.{ 1, 4 }, t.layout.stride.constSlice()); + try std.testing.expectEqual(@as(i64, 16), t.size()); + const v = t.get(.{ b.threadIdx().x, 3 }); + try std.testing.expect(v.isFloat()); + const kernel = try b.finish(.{ 1, 1, 1 }); + defer std.testing.allocator.free(kernel); + try expectContains(kernel, "!cute.layout<\"(4,4):(1,4)\">"); + try expectContains(kernel, "!cute.coord<\"(?,3)\">"); + const parsed = try mlir.Module.parse(ctx, kernel); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "casts, select, if-else, integer widening" { + const ctx = try testContext(); + defer ctx.deinit(); + + var b = try Builder.open(std.testing.allocator, ctx, "k"); + defer b.deinit(); + const a = try b.declareArgs(.{ .p = .{ .tensor = .{ .dtype = .f16, .shape = &.{8} } } }); + const tid = b.threadIdx().x; + + // A literal that does not fit i32 widens the i32 Value instead of truncating. + const big: u32 = 3_000_000_000; + const wide = tid.add(big); + try std.testing.expectEqual(DType.i64, wide.dtype()); + const bound = tid.lt(@as(i64, 5_000_000_000)); + try std.testing.expectEqual(DType.i1, bound.dtype()); + + const f = tid.to(.f32); + const shuffled = b.shuffleXor(f, 16); + try std.testing.expectEqual(DType.f16, f.to(.f16).dtype()); + try std.testing.expectEqual(DType.f64, f.to(.f64).dtype()); + try std.testing.expectEqual(DType.bf16, f.to(.f16).to(.bf16).dtype()); + try std.testing.expectEqual(DType.i32, f.to(.i32).dtype()); + try std.testing.expectEqual(DType.i8, tid.to(.i8).dtype()); + try std.testing.expectEqual(DType.i32, bound.to(.i32).dtype()); + + const picked = b.select(bound, shuffled, 2.0); + var scope = b.openIfElse(tid.lt(4), .{.f16}); + { + scope.yieldThen(.{picked.to(.f16)}); + } + { + scope.yieldElse(.{b.constant(.f16, 1)}); + } + a.p.set(.{tid}, scope.results[0]); + + const kernel = try b.finish(.{ 1, 1, 1 }); + defer std.testing.allocator.free(kernel); + try expectContains(kernel, "arith.extsi"); + try expectContains(kernel, "\"arith.constant\"() <{value = 3000000000 : i64}> : () -> i64"); + try expectContains(kernel, "arith.sitofp"); + try expectContains(kernel, "arith.truncf"); + try expectContains(kernel, "arith.extf"); + try expectContains(kernel, "arith.convertf"); + try expectContains(kernel, "arith.fptosi"); + try expectContains(kernel, "arith.trunci"); + try expectContains(kernel, "arith.extui"); + try expectContains(kernel, "arith.select"); + try expectContains(kernel, "shfl.sync.bfly.b32"); + try expectContains(kernel, "\"scf.if\""); + try expectContains(kernel, "}, {"); + const parsed = try mlir.Module.parse(ctx, kernel); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "complete program emits a Zig-built CUDA host launch" { + const ctx = try testContext(); + defer ctx.deinit(); + + var b = try Builder.openProgram(std.testing.allocator, ctx, "program_add"); + defer b.deinit(); + const device = try b.declareArgs(.{ + .input = .{ .tensor = .{ .dtype = .f32, .shape = &.{128} } }, + .output = .{ .tensor = .{ .dtype = .f32, .shape = &.{128} } }, + }); + const tid = b.threadIdx().x; + device.output.set(.{tid}, device.input.get(.{tid}).add(1.0)); + b.endFunction(.{ 128, 1, 1 }); + + b.beginFunction("program_add", .host); + _ = try b.declareArgs(.{ + .input = .{ .ptr = DType.f32 }, + .output = .{ .ptr = DType.f32 }, + }); + const one = b.constant(.i32, 1); + const config = b.makeLaunchConfig(.{ + .grid = .{ one, one, one }, + .block = .{ b.constant(.i32, 128), one, one }, + .dynamic_smem = b.kernelSmemSize("program_add"), + .stream = b.cudaStream(), + .use_pdl = true, + }); + const launched = b.launchEx("program_add", config, .{ b.arg(0), b.arg(1) }); + b.returnHostStatus(b.cudaResultStatus(launched)); + b.endFunction(null); + + const ir = try b.finishProgram(); + defer std.testing.allocator.free(ir); + try expectContains(ir, "\"gpu.module\""); + try expectContains(ir, "cuda.kernel"); + try expectContains(ir, "sym_name = \"program_add\""); + try expectContains(ir, "cuda.launch_ex"); +} + +test "SM100 block-scaled MMA atoms use the CuTe DSL instruction shapes" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + registerDialects(registry); + func.registerExtensions(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + + var b = try Builder.open(std.testing.allocator, ctx, "sm100_blockscaled_atoms"); + defer b.deinit(); + const tma: TmaConfig = .{ + .dtype = .f8e4m3fn, + .copy_bits = 16384, + .global_basis = b.layoutSpec(.{ 128, 16, 1 }, .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}), b.basis(1, 1, .{2}) }), + .coordinate_layout = b.layoutSpec(.{ 16, 128, 8 }, .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}), b.basis(1, 1, .{2}) }), + .coordinate_rank = 3, + }; + var pair_tma = tma; + pair_tma.cta_group = .two; + const args = try b.declareArgs(.{ + .single = .{ .mlir_type = .{ .type_ = b.tmaLoadAtomType(tma), .grid_constant = true } }, + .pair = .{ .mlir_type = .{ .type_ = b.tmaLoadAtomType(pair_tma), .grid_constant = true } }, + }); + _ = b.makeExecTma(.{ .inner = args.single.inner, .kernel = &b }, b.tmaLoadExecAtomType(tma)); + _ = b.makeExecTma(.{ .inner = args.pair.inner, .kernel = &b }, b.tmaLoadExecAtomType(pair_tma)); + + const sf_ptr = b.intToPtr(0, .f8e8m0fnu, .tmem, 1); + for ([_]CtaGroup{ .one, .two }) |group| { + _ = b.makeBlockScaledMma(.{ .n = 16, .cta_group = group }, sf_ptr); + _ = b.makeBlockScaledMma(.{ .n = 128, .cta_group = group }, sf_ptr); + } + _ = b.makeBlockScaledMma(.{ .m = 256, .n = 16, .cta_group = .two }, sf_ptr); + _ = b.makeBlockScaledMma(.{ .m = 256, .n = 128, .cta_group = .two }, sf_ptr); + const nvfp4_sf_ptr = b.intToPtr(0, .f8e4m3fn, .tmem, 1); + _ = b.makeBlockScaledMma(.{ .kind = .nvfp4_nvfp4, .n = 16 }, nvfp4_sf_ptr); + + const ir = try b.finish(.{ 192, 1, 1 }); + defer std.testing.allocator.free(ir); + try expectContains(ir, "sm100.mma_bs<128x16x32, num_cta = 1"); + try expectContains(ir, "sm100.mma_bs<128x128x32, num_cta = 1"); + try expectContains(ir, "sm100.mma_bs<128x16x32, num_cta = 2"); + try expectContains(ir, "sm100.mma_bs<128x128x32, num_cta = 2"); + try expectContains(ir, "sm100.mma_bs<256x16x32, num_cta = 2"); + try expectContains(ir, "sm100.mma_bs<256x128x32, num_cta = 2"); + try expectContains(ir, "sm100.mma_bs<128x16x64, num_cta = 1, ab_major = (k, k), elem_type = (f4E2M1FN, f4E2M1FN, f32), sf_type = f8E4M3FN, frag_kind = ss, vec_size = 16>"); + try expectContains(ir, "non_exec_tiled_tma_load"); + try expectContains(ir, "value = 3 : i32"); + try expectContains(ir, "value = 32 : i32"); + try expectContains(ir, "!llvm.ptr<7>"); + try expectContains(ir, "#nvvm.mem_scope"); + try expectContains(ir, "#nvvm.mem_scope"); + + const parsed = try mlir.Module.parse(ctx, ir); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "mbarrier waits preserve acquire scope through polling and blocking retries" { + const ctx = try testContext(); + defer ctx.deinit(); + for ([_]MBarrierScope{ .cta, .cluster }) |scope| { + var b = try Builder.open(std.testing.allocator, ctx, "barrier_waits"); + defer b.deinit(); + _ = try b.declareArgs(.{}); + + const barrier = b.allocSmem(.i64, 1, 8); + const phase = b.constant(.i32, 0); + _ = b.mbarrierTestWait(barrier, phase, .{ .scope = scope }); + _ = b.mbarrierTryWait(barrier, phase, .{ .scope = scope }); + b.mbarrierWait(barrier, phase, .{ .scope = scope, .time_limit = 1234, .test_first = false }); + b.mbarrierWait(barrier, phase, .{ .scope = scope }); + + const ir = try b.finish(.{ 32, 1, 1 }); + defer std.testing.allocator.free(ir); + try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, ir, "#nvvm.mbar_wait")); + try std.testing.expectEqual(@as(usize, 1), std.mem.count(u8, ir, "#nvvm.mbar_wait")); + try expectContains(ir, "value = 1234 : i32"); + try expectContains(ir, "\"scf.if\""); + switch (scope) { + .cta => { + try std.testing.expectEqual(@as(usize, 3), std.mem.count(u8, ir, "#nvvm.mbar_scope")); + try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, ir, "\"nvvm.mbarrier.try_wait.parity\"")); + try std.testing.expectEqual(@as(usize, 0), std.mem.count(u8, ir, "\"scf.while\"")); + }, + .cluster => { + try std.testing.expectEqual(@as(usize, 5), std.mem.count(u8, ir, "#nvvm.mbar_scope")); + try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, ir, "#nvvm.mem_order")); + try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, ir, "\"nvvm.mbarrier.try_wait.parity.timelimit\"")); + try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, ir, "\"scf.while\"")); + try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, ir, "\"scf.condition\"")); + try std.testing.expectEqual(@as(usize, 0), std.mem.count(u8, ir, "#nvvm.mbar_scope")); + }, + } + + const parsed = try mlir.Module.parse(ctx, ir); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); + } +} + +test "SM100 raw primitives preserve operand types and round-trip through MLIR" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = try Builder.open(std.testing.allocator, ctx, "sm100_raw_primitives"); + defer b.deinit(); + const cfg: TmaConfig = .{ + .dtype = .f8e4m3fn, + .copy_bits = 1024, + .global_basis = b.layoutSpec(.{ 128, 1 }, .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }), + .coordinate_layout = b.layoutSpec(.{ 1024, 16 }, .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }), + .coordinate_rank = 2, + }; + const a = try b.declareArgs(.{ + .gmem = .{ .ptr = DType.i32 }, + .tma = .{ .mlir_type = .{ .type_ = b.tmaLoadAtomType(cfg), .grid_constant = true } }, + }); + const atom = b.makeExecTma(.{ .inner = a.tma.inner, .kernel = &b }, b.tmaLoadExecAtomType(cfg)); + const desc = b.tmaDescriptorAddress(atom); + const dst = b.constant(.i32, 1024); + const bar = b.constant(.i32, 0); + const zero = b.constant(.i32, 0); + const gmem = b.ptrToInt(a.gmem, .i64); + const pred = b.threadIdx().x.eq(0); + b.tmaGather4(desc, dst, bar, zero, .{ zero, zero.add(1), zero.add(2), zero.add(3) }, .evict_last); + b.cpAsyncBulkPrefetch(gmem, b.constant(.i32, 128), .{}); + b.prefetchL1(gmem); + b.prefetchTensorMap(desc); + const swizzled = Tcgen05SmemDesc.build(&b, dst, .{ .leading_byte_offset = 16, .stride_byte_offset = 1024, .layout = .swizzle_128b }); + const interleaved = Tcgen05SmemDesc.build(&b, dst, .{ .leading_byte_offset = 1024, .stride_byte_offset = 128 }); + b.tcgen05MmaWs(.f16, zero, .{ .tmem = dst }, swizzled, Tcgen05InstrDesc.build(.{ .a_dtype = .bf16, .b_dtype = .bf16, .m_dim = 64, .n_dim = 128, .a_major = .k, .b_major = .k }), pred, .{}); + b.tcgen05MmaWs(.f16, zero, .{ .smem = interleaved }, swizzled, Tcgen05InstrDesc.build(.{ .a_dtype = .bf16, .b_dtype = .bf16, .m_dim = 64, .n_dim = 256, .a_major = .k, .b_major = .mn }), pred, .{}); + b.setMaxRegisterIncrease(128); + b.barrierCtaSyncAligned(0, 512); + b.setMaxRegisterDecrease(128); + b.barrierCtaSyncAligned(b.threadIdx().x.div(128), 128); + b.mbarrierSpin(bar, zero, .cta); + b.mbarrierSpin(bar, zero, .cluster); + const state = b.mbarrierArrive(bar, .{}).?; + const remote = b.mapDsmemPtr(dst, zero); + _ = remote; + _ = b.mbarrierArrive(bar, .{ .peer_cta_rank_in_cluster = zero, .arrive_count = b.constant(.i32, 1), .scope = .cluster }); + b.storeAsyncDsmem(dst, .{ zero, zero }, bar, zero); + b.storeAsyncDsmem(dst, .{ zero, zero, zero, zero }, bar, zero); + b.barrierClusterArriveRelaxedAligned(); + b.barrierClusterWaitAligned(); + const any = b.voteAnySync(pred, null); + const mask = b.voteBallotSync(pred, null); + const x = b.constant(.f32, 1.0); + const y = b.constant(.f32, 2.0); + const maximum = b.fmax(x, y, .{}); + const sum = b.addPackedF32x2(.{ x, y }, .{ y, x }, .{}); + const product = b.mulPackedF32x2(sum, .{ x, y }, .{}); + const fused = b.fmaPackedF32x2(product, sum, .{ x, y }, .{}); + const scalar = b.fma(maximum, fused[0], fused[1]); + b.store(gmem, .{ scalar.bitCast(.i32), maximum.bitCast(.i32), mask, any.to(.i32), state.to(.i32), b.clock(), b.globalTimer().to(.i32), zero }, .{ .level1_eviction_priority = .evict_no_allocate }); + + const ir = try b.finish(.{ 512, 1, 1 }); + defer std.testing.allocator.free(ir); + for ([_][]const u8{ + "cute_nvgpu.get_tma_desc_addr", "tile::gather4", "nvvm.cp.async.bulk.prefetch", + "prefetch.global.L1", "prefetch.tensormap", "nvvm.tcgen05.mma.ws", + "nvvm.tcgen05.mma.ws", "setmaxnreg.inc", "setmaxnreg.dec", + "llvm.nvvm.barrier.cta.sync.aligned.count", "mbarrier.test_wait.parity.acquire.cta", "mbarrier.test_wait.parity.acquire.cluster", + "nvvm.mapa", "#nvvm.mem_scope", "nvvm.store.async", + "vector<4xi32>", "barrier.cluster.arrive.relaxed.aligned", "nvvm.cluster.wait", + "llvm.nvvm.vote.any.sync", "llvm.nvvm.vote.ballot.sync", "llvm.nvvm.fmax.f", + "nvvm.add.packed.f32x2", "nvvm.mul.packed.f32x2", "nvvm.fma.packed.f32x2", + "llvm.fma.f32", "st.global.L1::no_allocate.v8.b32", "%clock", + "%globaltimer", + }) |op| try expectContains(ir, op); + const parsed = try mlir.Module.parse(ctx, ir); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "SM100 BF16 weight-stationary descriptor encodes supported shapes and major modes" { + try std.testing.expectEqual(@as(u32, 0x04200490), Tcgen05InstrDesc.build(.{ .a_dtype = .bf16, .b_dtype = .bf16, .m_dim = 64, .n_dim = 128, .a_major = .k, .b_major = .k })); + try std.testing.expectEqual(@as(u32, 0x04410490), Tcgen05InstrDesc.build(.{ .a_dtype = .bf16, .b_dtype = .bf16, .m_dim = 64, .n_dim = 256, .a_major = .k, .b_major = .mn })); + try std.testing.expectEqual(@as(u32, 0x08118490), Tcgen05InstrDesc.build(.{ .a_dtype = .bf16, .b_dtype = .bf16, .m_dim = 128, .n_dim = 64, .a_major = .mn, .b_major = .mn })); + try std.testing.expectEqual(@as(u32, 0x02100490), Tcgen05InstrDesc.build(.{ .a_dtype = .bf16, .b_dtype = .bf16, .m_dim = 32, .n_dim = 64, .a_major = .k, .b_major = .k })); +} + +test "SM100 shared descriptors preserve nonzero swizzle bases and boundary offsets" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = try Builder.open(std.testing.allocator, ctx, "smem_descriptors"); + defer b.deinit(); + const args = try b.declareArgs(.{ .out = .{ .ptr = DType.i64 } }); + const addr = b.constant(.i32, 128); + const fields = [_]u64{ + 0x4000404000010000, // sw128, base 0, leading 16 bytes, stride 1024 bytes + 0x4002404000010000, // sw128, pattern at byte 128: base 1 + 0x400e404000010000, // sw128, pattern at byte 896: base 7 + 0x00007fff3fff0000, // unswizzled, largest representable offsets + }; + const descs = [_]Value{ + Tcgen05SmemDesc.build(&b, addr, .{ .leading_byte_offset = 16, .stride_byte_offset = 1024, .base_offset = 0, .layout = .swizzle_128b }), + Tcgen05SmemDesc.build(&b, addr, .{ .leading_byte_offset = 16, .stride_byte_offset = 1024, .base_offset = 1, .layout = .swizzle_128b }), + Tcgen05SmemDesc.build(&b, addr, .{ .leading_byte_offset = 16, .stride_byte_offset = 1024, .base_offset = 7, .layout = .swizzle_128b }), + Tcgen05SmemDesc.build(&b, addr, .{ .leading_byte_offset = 0x3fff0, .stride_byte_offset = 0x3fff0 }), + }; + const out = b.ptrToInt(args.out, .i64); + for (descs, 0..) |desc, i| b.storeWords(2, out.add(@as(i64, @intCast(i * 8))), .global, .{ desc.to(.i32), desc.shrLogical(32).to(.i32) }); + const ir = try b.finish(.{ 32, 1, 1 }); + defer std.testing.allocator.free(ir); + for (fields) |bits| { + const expected = try std.fmt.allocPrint(std.testing.allocator, "value = {d} : i64", .{bits}); + defer std.testing.allocator.free(expected); + try expectContains(ir, expected); + } + const parsed = try mlir.Module.parse(ctx, ir); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "makeLayout preserves dynamic singleton and nested shape and stride leaves" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = try Builder.open(std.testing.allocator, ctx, "mixed_layout"); + defer b.deinit(); + _ = try b.declareArgs(.{}); + const n = b.blockIdx().x.add(1); + const singleton = b.constant(.i32, 1); + const compact = b.makeLayout(.{ .{ 4, n }, singleton }, .{}); + const explicit = b.makeLayout(.{ n, 16 }, .{ .stride = .{ 1, n } }); + const static_nested = b.makeLayout(.{ .{ 4, 8 }, 16 }, .{}); + _ = compact; + _ = explicit; + _ = static_nested; + const ir = try b.finish(.{ 32, 1, 1 }); + defer std.testing.allocator.free(ir); + try expectContains(ir, "((4,?),?):((1,4),?)"); + try expectContains(ir, "(?,16):(1,?)"); + try expectContains(ir, "((4,8),16):((1,4),32)"); + const parsed = try mlir.Module.parse(ctx, ir); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "makeLayout preserves 64-bit shape and stride values" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = try Builder.open(std.testing.allocator, ctx, "wide_layout"); + defer b.deinit(); + const args = try b.declareArgs(.{ .extent = .{ .mlir_type = .{ .type_ = DType.i64.toMlir(ctx) } } }); + const wide = args.extent.add(@as(i64, 1) << 32); + const explicit = b.makeLayout(.{ 8, 2 }, .{ .stride = .{ 1, wide } }); + const wide_leaf: AlgebraExpr = .{ .dynamic_int = .{ .width = 64 } }; + try std.testing.expect(explicit.type_().eql(b.layoutType(.{ + .shape = b.expr(.{ 8, 2 }), + .stride = .{ .tuple = &.{ .{ .static = 1 }, wide_leaf } }, + }))); + _ = b.makeLayout(.{ .{ 4, wide }, 1, 8 }, .{ .stride = null }); + const ir = try b.finish(.{ 32, 1, 1 }); + defer std.testing.allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "arith.trunci") == null); + try expectContains(ir, "((4,?{i64}),1,8):((1,4),0,?{i64})"); + try expectContains(ir, "value = 4294967296 : i64"); + const parsed = try mlir.Module.parse(ctx, ir); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "TMA constructors normalize tilers and staged layouts and copy executes store atoms" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = try Builder.open(std.testing.allocator, ctx, "tma_stores"); + defer b.deinit(); + const cfg: TmaConfig = .{ + .dtype = .f32, + .copy_bits = 2048, + .global_basis = b.layoutSpec(.{ 16, 4 }, .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }), + .coordinate_layout = b.layoutSpec(.{ 128, 64 }, .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }), + .coordinate_rank = 2, + }; + const args = try b.declareArgs(.{ + .data = .{ .tensor = .{ .dtype = .f32, .shape = &.{ 128, 64 }, .stride = &.{ 1, 128 } } }, + }); + const staged = b.makeLayout(.{ 16, 4, 2 }, .{ .stride = .{ 1, 16, 64 } }); + const store = b.makeTiledTmaAtom(CopyBulkTensorTileS2GOp{}, args.data, staged, .{ 16, 4 }, cfg, .{}); + const smem = b.makeTensorView(b.intToPtr(0, .f32, .smem, 128), b.makeLayout(.{ 16, 4 }, .{})); + const tile = b.makeCoordTensor(.{ 0, 0 }, b.staticLayout(cfg.global_basis)); + b.copy(store.atom, .{smem}, .{tile}, .{}); + b.copy(store.atom, .{smem}, .{tile}, .{ .cache_policy = b.constant(.i64, CacheHintSm90.evict_last) }); + const swizzled = b.makeComposedLayout(.{ .bits = 1, .base = 4, .shift = 3 }, 0, b.layoutSpec(.{ 16, 4, 2 }, .{ 1, 16, 64 })); + _ = b.makeTiledTmaAtom(CopyBulkTensorTileG2SOp{}, args.data, swizzled, b.makeTile(&.{ 16, 4 }), cfg, .{}); + _ = b.makeTiledTmaAtom(CopyBulkTensorTileG2SOp{}, args.data, staged, b.makeLayout(.{ 16, 4 }, .{ .stride = .{ 1, 128 } }), cfg, .{}); + const ir = try b.finish(.{ 32, 1, 1 }); + defer std.testing.allocator.free(ir); + try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, ir, "\"cute_nvgpu.atom.make_exec_tma\"")); + try std.testing.expectEqual(@as(usize, 3), std.mem.count(u8, ir, "\"cute.select\"")); + try expectContains(ir, "(16,4):(1@0,1@1)"); + try expectContains(ir, "tma_store"); + const parsed = try mlir.Module.parse(ctx, ir); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "CuTe options preserve runtime masks, timed try waits, descriptors and store widths" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = try Builder.open(std.testing.allocator, ctx, "cute_options"); + defer b.deinit(); + const a = try b.declareArgs(.{ .out = .{ .ptr = DType.i32 } }); + const tid = b.threadIdx().x; + const mbar = b.allocSmem(.i64, 1, 8); + b.barrier(.{}); + b.barrier(.{ .barrier_id = tid.div(128), .number_of_threads = b.blockDim().x }); + b.tcgen05Commit(mbar, .{ .cta_group = .two, .multicast_mask = b.clusterCtaRank().shl(1).bitOr(1) }); + _ = b.mbarrierTryWait(mbar, b.constant(.i32, 0), .{ .time_limit = tid.add(100), .scope = .cluster }); + const state = b.mbarrierArrive(mbar, .{ .relaxed = true }).?; + b.storeAsyncDsmem(b.constant(.i32, 128), tid, mbar, b.clusterCtaRank().bitXor(1)); + b.clusterArriveRelaxed(.{}); + b.clusterWait(); + b.clusterArriveRelaxed(.{ .aligned = true }); + b.barrierClusterWaitAligned(); + _ = b.voteAnySync(tid.eq(0), tid.bitOr(1)); + _ = b.voteBallotSync(tid.eq(0), 0xff); + const x = tid.to(.f32); + const y = b.fmax(x, b.constant(.f32, 0), .{ .nan = true, .ftz = true, .abs = true }); + const pair = b.addPackedF32x2(.{ x, y }, .{ y, x }, .{ .rnd = .rz, .ftz = true }); + b.store(a.out, state.to(.i32), .{}); + b.store(a.out, .{ pair[0].bitCast(.i32), pair[1].bitCast(.i32) }, .{}); + const desc = b.makeUmmaSmemDesc(b.intToPtr(0, .bf16, .smem, 16), b.layoutSpec(.{ 64, 16 }, .{ 16, 1 }), .k, null); + const raw_desc = b.smemDescriptorToInt(desc); + _ = b.intToSmemDescriptor(raw_desc); + const mapped = b.mapDsmemPtr(b.intToPtr(128, .i32, .smem, 16), b.clusterCtaRank()); + const mapped_ty = mapped.type_().isA(cute_ir.PtrType).?; + try std.testing.expectEqual(MemorySpace.dsmem, mapped_ty.addressSpace()); + try std.testing.expectEqual(@as(u64, 16), mapped_ty.alignment()); + b.store(mapped, tid, .{}); + const swizzles = [_]Tcgen05SmemSwizzle{ .none, .swizzle_32b, .swizzle_64b, .swizzle_128b, .swizzle_128b_atom_32b }; + for (swizzles) |layout| _ = Tcgen05SmemDesc.build(&b, tid.mul(16), .{ .layout = layout }); + const idesc = b.constant(.i32, Tcgen05InstrDesc.build(.{ .a_dtype = .bf16, .b_dtype = .bf16, .m_dim = 64, .n_dim = 128 })); + b.tcgen05MmaWs(.f16, tid, .{ .tmem = tid }, desc, idesc, tid.ne(0), .{ .collector_b_buffer = .b2, .collector_op = .use, .col_b_zero_mask = state }); + b.cpAsyncBulkPrefetch(a.out, 128, .{ .l2_cache_hint = state }); + const ir = try b.finish(.{ 128, 1, 1 }); + defer std.testing.allocator.free(ir); + try expectContains(ir, "nvvm.mbarrier.try_wait.parity.timelimit"); + try std.testing.expectEqual(@as(usize, 0), std.mem.count(u8, ir, "scf.while")); + try expectContains(ir, "nvvm.store.ext"); + try expectContains(ir, "vector<2xi32>"); + try expectContains(ir, "#nvvm.fp_rnd_mode"); + try expectContains(ir, "cute_nvgpu.make_umma_smem_desc"); + const parsed = try mlir.Module.parse(ctx, ir); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "gather4 uses CuTe TMA atom construction, partitioning and copy options" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = try Builder.openProgram(std.testing.allocator, ctx, "gather4_copy"); + defer b.deinit(); + const cfg: TmaConfig = .{ + .dtype = .f32, + .copy_bits = 2048, + .global_basis = b.layoutSpec(.{ 16, 1 }, .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }), + .coordinate_layout = b.layoutSpec(.{ 128, 64 }, .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }), + .coordinate_rank = 2, + .mode = .gather4, + }; + var pair = cfg; + pair.cta_group = .two; + const args = try b.declareArgs(.{ + .indices = .{ .tensor = .{ .dtype = .i32, .shape = &.{ 128, 64 }, .stride = &.{ 0, 1 } } }, + .atom = .{ .mlir_type = .{ .type_ = b.tmaLoadAtomType(cfg), .grid_constant = true } }, + .pair = .{ .mlir_type = .{ .type_ = b.tmaLoadAtomType(pair), .grid_constant = true } }, + }); + const smem_layout = b.makeLayout(.{ 16, 4 }, .{ .stride = .{ 1, 16 } }); + const smem = b.makeTensorView(b.intToPtr(0, .f32, .smem, 128), smem_layout); + const coords = b.makeCoordTensor(.{ 0, 0 }, b.staticLayout(cfg.coordinate_layout)); + const atom: Atom = .{ .inner = args.atom.inner, .kernel = &b }; + const partitioned = b.tmaPartition(atom, 0, b.makeLayout(1, .{}), smem, .{ coords, args.indices.view() }, smem.type_(), &.{ coords.type_(), args.indices.inner.type_() }); + const mbar = b.allocSmem(.i64, 1, 8); + b.copy(atom, .{ partitioned.targets[0], partitioned.targets[1] }, .{partitioned.smem}, .{ + .tma_bar_ptr = mbar, + .cache_policy = b.constant(.i64, CacheHintSm100.evict_last), + }); + b.copy(.{ .inner = args.pair.inner, .kernel = &b }, .{ partitioned.targets[0], partitioned.targets[1] }, .{partitioned.smem}, .{ .tma_bar_ptr = mbar }); + b.endFunction(.{ 32, 1, 1 }); + + b.beginFunction("gather4_copy", .host); + const host = try b.declareArgs(.{ + .data = .{ .tensor = .{ .dtype = .f32, .shape = &.{ 128, 64 }, .stride = &.{ 1, 128 } } }, + .indices = .{ .tensor = .{ .dtype = .i32, .shape = &.{ 128, 64 }, .stride = &.{ 0, 1 } } }, + }); + const host_layout = b.makeLayout(.{ 16, 4 }, .{ .stride = .{ 1, 16 } }); + const tiler = b.makeTile(&.{ 16, 4 }); + _ = b.makeTiledTmaAtom(CopyBulkTensor2DGather4G2SOp{}, host.data, host_layout, tiler, cfg, .{ .gmem_coord_tensor = host.indices.view() }); + _ = b.makeTiledTmaAtom(CopyBulkTensor2DGather4G2SOp{ .cta_group = .two }, host.data, host_layout, tiler, pair, .{ .gmem_coord_tensor = host.indices.view() }); + b.returnHostStatus(b.constant(.i32, 0)); + b.endFunction(null); + const ir = try b.finishProgram(); + defer std.testing.allocator.free(ir); + try expectContains(ir, "atom.make_non_exec_2d_gather4_tma_load"); + try expectContains(ir, "atom.tma_partition"); + try expectContains(ir, "mode = gather4, num_cta = 1"); + try expectContains(ir, "mode = gather4, num_cta = 2"); + try expectContains(ir, "non_exec_2d_gather4_tma_load", .{structFields(b, n, elem)}) catch @panic("OOM")) catch @panic("invalid inline assembly result type")); + const packed_ = b.emit(llvm.inline_asm(b.ctx, text, inner, .{ .result_type = struct_type, .constraints = constraints, .has_side_effects = side_effects }, b.loc())); + const out = b.arena.allocator().alloc(Value, n) catch @panic("OOM"); + for (out, 0..) |*o, i| { + o.* = b.emit(mlir.Operation.make(b.ctx, "llvm.extractvalue", .{ + .operands = .{ .flat = &.{packed_.inner} }, + .results = .{ .flat = &.{elem} }, + .attributes = &.{.named(b.ctx, "position", .denseArray(b.ctx, .i64, &[_]i64{@intCast(i)}))}, + .location = b.loc(), + })); + } + return out; +} + +pub fn structFields(b: *Builder, n: usize, elem: *const mlir.Type) []const u8 { + var w: std.Io.Writer.Allocating = .init(b.arena.allocator()); + for (0..n) |i| w.writer.print("{s}{f}", .{ if (i == 0) "" else ", ", elem }) catch @panic("OOM"); + return w.written(); +} + +pub fn ldAcq(b: *Builder, ptr: Value) Value { + return asmValue(b, "ld.acquire.gpu.global.b32 $0, [$1];", &.{ptr}, .i32, "=r,l,~{memory}", true); +} + +pub fn ldAcqSysU64(b: *Builder, ptr: Value) Value { + return asmValue(b, "ld.acquire.sys.global.b64 $0, [$1];", &.{ptr}, .i64, "=l,l,~{memory}", true); +} + +pub fn ldAcqSysS32(b: *Builder, ptr: Value) Value { + return asmValue(b, "ld.acquire.sys.global.s32 $0, [$1];", &.{ptr}, .i32, "=r,l,~{memory}", true); +} + +pub fn ldAcqSysU32(b: *Builder, ptr: Value) Value { + return asmValue(b, "ld.acquire.sys.global.u32 $0, [$1];", &.{ptr}, .i32, "=r,l,~{memory}", true); +} + +pub fn ldAcqGpuU64(b: *Builder, ptr: Value) Value { + return asmValue(b, "ld.acquire.gpu.global.u64 $0, [$1];", &.{ptr}, .i64, "=l,l,~{memory}", true); +} + +pub fn atomicAddU32(b: *Builder, ptr: Value, value: anytype) Value { + return asmValue(b, "atom.global.add.u32 $0, [$1], $2;", &.{ ptr, liftAs(b, value, .i32) }, .i32, "=r,l,r", true); +} + +pub fn atomicAddU64(b: *Builder, ptr: Value, value: Value) Value { + return asmValue(b, "atom.global.add.u64 $0, [$1], $2;", &.{ ptr, value }, .i64, "=l,l,l", true); +} + +pub fn atomicAddSys(b: *Builder, ptr: Value, value: Value) Value { + return asmValue(b, "atom.sys.global.add.u64 $0, [$1], $2;", &.{ ptr, value }, .i64, "=l,l,l", true); +} + +pub fn atomicAddRel(b: *Builder, ptr: Value, value: anytype) Value { + return asmValue(b, "atom.release.gpu.global.add.u32 $0, [$1], $2;", &.{ ptr, liftAs(b, value, .i32) }, .i32, "=r,l,r,~{memory}", true); +} + +pub fn stRelSys(b: *Builder, ptr: Value, value: Value) void { + asmVoid(b, "st.release.sys.global.u64 [$0], $1;", &.{ ptr, value }, "l,l,~{memory}"); +} + +pub fn redAddS32(b: *Builder, ptr: Value, value: anytype) void { + asmVoid(b, "red.gpu.global.add.s32 [$0], $1;", &.{ ptr, liftAs(b, value, .i32) }, "l,r"); +} + +pub fn redAddU32(b: *Builder, ptr: Value, value: anytype) void { + asmVoid(b, "red.gpu.global.add.u32 [$0], $1;", &.{ ptr, liftAs(b, value, .i32) }, "l,r"); +} + +pub fn redXorRel(b: *Builder, ptr: Value, value: Value) void { + asmVoid(b, "red.release.gpu.global.xor.b64 [$0], $1;", &.{ ptr, value }, "l,l,~{memory}"); +} + +pub fn redAddRel(b: *Builder, ptr: Value, value: anytype) void { + asmVoid(b, "red.release.gpu.global.add.u32 [$0], $1;", &.{ ptr, liftAs(b, value, .i32) }, "l,r,~{memory}"); +} + +pub fn redAddRelSys(b: *Builder, ptr: Value, value: anytype) void { + asmVoid(b, "red.release.sys.global.add.s32 [$0], $1;", &.{ ptr, liftAs(b, value, .i32) }, "l,r,~{memory}"); +} + +pub fn atomicAddBlockShared(b: *Builder, smem: Value, value: anytype) Value { + return asmValue(b, "atom.shared.cta.add.u32 $0, [$1], $2;", &.{ smem, liftAs(b, value, .i32) }, .i32, "=r,r,r,~{memory}", true); +} + +pub fn fenceAcqRelCta(b: *Builder) void { + asmVoid(b, "fence.acq_rel.cta;", &.{}, "~{memory}"); +} + +pub fn liftAs(b: *Builder, value: anytype, dt: DType) Value { + if (@TypeOf(value) == Value) return value; + return b.constant(dt, value); +} + +/// `cvta`-style widening of an `i32` shared-memory address to the 64-bit operand +/// `__cvta_generic_to_shared` gives DeepGEMM's `"l"` constraints. +pub fn zext64(b: *Builder, v: Value) Value { + if (v.dtype() == .i64) return v; + return b.emit(arith.extui(b.ctx, v.inner, DType.i64.toMlir(b.ctx), b.loc())); +} + +pub fn i16Of(b: *Builder, v: Value) Value { + return if (v.dtype() == .i16) v else b.emit(arith.trunci(b.ctx, v.inner, DType.i16.toMlir(b.ctx), b.loc())); +} + +pub fn cst32(b: *Builder, v: anytype) Value { + return b.constant(.i32, @as(i32, @bitCast(@as(u32, @truncate(@as(u64, @bitCast(@as(i64, v)))))))); +} + +pub fn cst64(b: *Builder, v: u64) Value { + return b.constant(.i64, @as(i64, @bitCast(v))); +} + +pub fn getSmIdx(b: *Builder) Value { + return asmValue(b, "mov.u32 $0, %smid;", &.{}, .i32, "=r", false); +} + +pub fn getLaneIdx(b: *Builder) Value { + return asmValue(b, "mov.u32 $0, %laneid;", &.{}, .i32, "=r", false); +} + +/// `ptx::get_grid_idx`: the 64-bit `%gridid`, unique per launch. +pub fn getGridIdx(b: *Builder) Value { + return asmValue(b, "mov.u64 $0, %gridid;", &.{}, .i64, "=l", true); +} + +pub fn blockRankInCluster(b: *Builder) Value { + return asmValue(b, "mov.u32 $0, %cluster_ctarank;\n", &.{}, .i32, "=r", true); +} + +/// `clock64()` (`mov.u64 %rd, %clock64;`). +pub fn clock64(b: *Builder) Value { + return asmValue(b, "mov.u64 $0, %clock64;", &.{}, .i64, "=l", true); +} + +/// Keep the original election sequence until typed election matches its small-batch performance. +pub fn electOneSync(b: *Builder) Value { + const pred = asmValue(b, + \\{ + \\.reg .b32 %rx; + \\.reg .pred %px; + \\ mov.u32 $0, 0; + \\ elect.sync %rx|%px, $1; + \\@%px mov.s32 $0, 1; + \\} + , &.{cst32(b, 0xFFFFFFFF)}, .i32, "=r,r", true); + return pred.ne(0); +} + +/// `asm volatile("trap;")` (`DG_TRAP_ONLY_DEVICE_ASSERT`, and this port's timeout). +pub fn trap(b: *Builder) void { + asmVoid(b, "trap;", &.{}, "~{memory}"); +} + +/// `ptx::sync_unaligned(num_threads, barrier_idx)`: `barrier.sync id, n` (threads of a warp may +/// reach it at different program points). +pub fn syncUnaligned(b: *Builder, num_threads: anytype, barrier_idx: anytype) void { + asmVoid(b, "barrier.sync $0, $1;", &.{ liftAs(b, barrier_idx, .i32), liftAs(b, num_threads, .i32) }, "r,r,~{memory}"); +} + +/// `cute::cluster_arrive`. +pub fn clusterArrive(b: *Builder) void { + asmVoid(b, "barrier.cluster.arrive.aligned;\n", &.{}, "~{memory}"); +} + +/// `cudaGridDependencySynchronize()` (`griddepcontrol.wait;`, PDL). +pub fn gridDependencySynchronize(b: *Builder) void { + asmVoid(b, "griddepcontrol.wait;", &.{}, "~{memory}"); +} + +/// `cutlass::arch::warpgroup_reg_alloc`. +pub fn warpgroupRegAlloc(b: *Builder, reg_count: u32) void { + asmVoid(b, std.fmt.allocPrint(b.arena.allocator(), "setmaxnreg.inc.sync.aligned.u32 {d};\n", .{reg_count}) catch @panic("OOM"), &.{}, ""); +} + +/// `cutlass::arch::warpgroup_reg_dealloc`. +pub fn warpgroupRegDealloc(b: *Builder, reg_count: u32) void { + asmVoid(b, std.fmt.allocPrint(b.arena.allocator(), "setmaxnreg.dec.sync.aligned.u32 {d};\n", .{reg_count}) catch @panic("OOM"), &.{}, ""); +} + +/// `ptx::ld_volatile(const uint64_t*)`. +pub fn ldVolatileU64(b: *Builder, ptr: Value) Value { + return asmValue(b, "ld.volatile.global.b64 $0, [$1];", &.{ptr}, .i64, "=l,l", true); +} + +/// `ptx::ld_volatile(const uint32_t*)`. +pub fn ldVolatileU32(b: *Builder, ptr: Value) Value { + return asmValue(b, "ld.volatile.global.b32 $0, [$1];", &.{ptr}, .i32, "=r,l", true); +} + +/// `ptx::ld_shared(const uint32_t*)`. +pub fn ldSharedU32(b: *Builder, smem: Value) Value { + return asmValue(b, "ld.shared.u32 $0, [$1];", &.{zext64(b, smem)}, .i32, "=r,l", true); +} + +/// `ptx::ld_shared(const float2*)`. +pub fn ldSharedF32x2(b: *Builder, smem: Value) [2]Value { + return asmValues(b, 2, "ld.shared.v2.f32 {$0, $1}, [$2];", &.{zext64(b, smem)}, .f32, "=f,=f,l", true); +} + +/// `ptx::ld_shared(const float4*)`. +pub fn ldSharedF32x4(b: *Builder, smem: Value) [4]Value { + return asmValues(b, 4, "ld.shared.v4.f32 {$0, $1, $2, $3}, [$4];", &.{zext64(b, smem)}, .f32, "=f,=f,=f,=f,l", true); +} + +/// `ptx::ld_shared(const uint4*)`. +pub fn ldSharedU32x4(b: *Builder, smem: Value) [4]Value { + return asmValues(b, 4, "ld.shared.v4.u32 {$0, $1, $2, $3}, [$4];", &.{zext64(b, smem)}, .i32, "=r,=r,=r,=r,l", true); +} + +/// `ptx::st_shared(const void*, x, y, z, w)`. +pub fn stSharedU32x4(b: *Builder, smem: Value, w: [4]Value) void { + asmVoid(b, "st.shared.v4.u32 [$0], {$1, $2, $3, $4};", &.{ zext64(b, smem), w[0], w[1], w[2], w[3] }, "l,r,r,r,r"); +} + +/// `ptx::st_shared(const uint32_t*, uint32_t)`. +pub fn stSharedU32(b: *Builder, smem: Value, value: Value) void { + asmVoid(b, "st.shared.u32 [$0], $1;", &.{ zext64(b, smem), value }, "l,r"); +} + +/// `ptx::st_shared(const float2*, float2)`. +pub fn stSharedF32x2(b: *Builder, smem: Value, v: [2]Value) void { + asmVoid(b, "st.shared.v2.f32 [$0], {$1, $2};", &.{ zext64(b, smem), v[0], v[1] }, "l,f,f"); +} + +/// `ptx::st_shared_bulk(smem, num_bytes)`: zero `num_bytes` (a multiple of 8) of shared memory. +pub fn stSharedBulk(b: *Builder, smem: Value, num_bytes: u32) void { + std.debug.assert(num_bytes % 8 == 0); + asmVoid(b, "st.bulk.weak.shared::cta [$0], $1, 0;", &.{ zext64(b, smem), b.constant(.i64, num_bytes) }, "l,l,~{memory}"); +} + +/// `ptx::mapa_shared(ptr, dst_cta_idx)`: this CTA's shared address in cluster CTA `cta`. +pub fn mapaShared(b: *Builder, smem: Value, dst_cta_idx: Value) Value { + return asmValue(b, "mapa.shared::cluster.u32 $0, $1, $2;", &.{ smem, dst_cta_idx }, .i32, "=r,r,r", true); +} + +/// `ptx::st_async_cluster(dst, src, dst_cta_idx, barrier)` for `words.len / 4` 16-byte +/// chunks: `st.async` into CTA `dst_cta_idx`'s shared memory, completing the bytes on its +/// mbarrier at `barrier` (both local addresses, mapped with `mapa`). +pub fn stAsyncCluster(b: *Builder, dst: Value, words: []const Value, dst_cta_idx: Value, barrier: Value) void { + std.debug.assert(words.len % 4 == 0); + const mapped_barrier = mapaShared(b, barrier, dst_cta_idx); + const mapped_dst = mapaShared(b, dst, dst_cta_idx); + var i: usize = 0; + while (i < words.len) : (i += 4) { + asmVoid( + b, + "st.async.shared::cluster.mbarrier::complete_tx::bytes.u32.v4 [$0], {$1, $2, $3, $4}, [$5];", + &.{ mapped_dst.add(@as(i32, @intCast(i * 4))), words[i], words[i + 1], words[i + 2], words[i + 3], mapped_barrier }, + "r,r,r,r,r,r,~{memory}", + ); + } +} + +/// `ptx::SM100_U8x4_STSM_T::copy(src, smem)`: `stmatrix.m16n8.x1.trans.b8`. +pub fn stmatrixU8x4T(b: *Builder, src: Value, smem: Value) void { + asmVoid(b, "stmatrix.sync.aligned.m16n8.x1.trans.shared.b8 [$0], {$1};\n", &.{ zext64(b, smem), src }, "l,r"); +} + +/// `ptx::SM90_U32x4_STSM_T::copy(s0, s1, s2, s3, smem)`: `stmatrix.x4.m8n8.trans.b16`. +pub fn stmatrixU32x4T(b: *Builder, src: [4]Value, smem: Value) void { + asmVoid(b, "stmatrix.sync.aligned.x4.m8n8.shared.b16.trans [$0], {$1, $2, $3, $4};\n", &.{ zext64(b, smem), src[0], src[1], src[2], src[3] }, "l,r,r,r,r"); +} + +/// `ClusterBarrier::arrive()` on this CTA's barrier. +pub fn mbarrierArrive(b: *Builder, bar: Value) void { + asmVoid(b, "{\n\tmbarrier.arrive.shared::cta.b64 _, [$0];\n\t}", &.{bar}, "r,~{memory}"); +} + +/// `ClusterBarrier::arrive(cta_id, pred = true)`: arrive on the barrier at the same offset of +/// cluster CTA `cta_id`. +pub fn mbarrierArriveCluster(b: *Builder, bar: Value, cta_id: Value) void { + asmVoid( + b, + "{\n\t.reg .b32 remAddr32;\n\tmapa.shared::cluster.u32 remAddr32, $0, $1;\n\tmbarrier.arrive.shared::cluster.b64 _, [remAddr32];\n\t}", + &.{ bar, cta_id }, + "r,r,~{memory}", + ); +} + +/// `ClusterTransactionBarrier::arrive_and_expect_tx(bytes, cta_id, pred)`: on cluster CTA +/// `cta_id`'s barrier, only when `pred == 1`. +pub fn mbarrierArriveAndExpectTxCluster(b: *Builder, bar: Value, transaction_bytes: anytype, cta_id: Value, pred: Value) void { + asmVoid( + b, + "{\n\t.reg .pred p;\n\t.reg .b32 remAddr32;\n\tsetp.eq.u32 p, $2, 1;\n\t@p mapa.shared::cluster.u32 remAddr32, $0, $1;\n\t@p mbarrier.arrive.expect_tx.shared::cluster.b64 _, [remAddr32], $3;\n\t}", + &.{ bar, cta_id, pred, liftAs(b, transaction_bytes, .i32) }, + "r,r,r,r,~{memory}", + ); +} + +/// `cute::prefetch_tma_descriptor(desc)` (generic descriptor address). +pub fn prefetchTmaDescriptor(b: *Builder, desc: Value) void { + asmVoid(b, "prefetch.tensormap [$0];", &.{desc}, "l,~{memory}"); +} + +/// `SM100_TMA_2SM_LOAD_2D::copy(desc, mbar, cache_hint, smem, c0, c1)`: issued by both CTAs of +/// the pair; the barrier address is masked so the bytes complete on the leader's barrier. +pub fn tma2SmLoad2d(b: *Builder, desc: Value, mbar: Value, cache_hint: u64, smem: Value, c0: Value, c1: Value) void { + asmVoid( + b, + "cp.async.bulk.tensor.2d.cta_group::2.shared::cluster.global.mbarrier::complete_tx::bytes.L2::cache_hint [$0], [$1, {$3, $4}], [$2], $5;", + &.{ smem, desc, mbar.bitAnd(cst32(b, sm100_mma_peer_bit_mask)), c0, c1, cst64(b, cache_hint) }, + "r,l,r,r,r,l,~{memory}", + ); +} + +/// `SM100_TMA_2SM_LOAD_4D::copy(desc, mbar, cache_hint, smem, c0, c1, c2, c3)`. +pub fn tma2SmLoad4d(b: *Builder, desc: Value, mbar: Value, cache_hint: u64, smem: Value, c: [4]Value) void { + asmVoid( + b, + "cp.async.bulk.tensor.4d.cta_group::2.shared::cluster.global.mbarrier::complete_tx::bytes.L2::cache_hint [$0], [$1, {$3, $4, $5, $6}], [$2], $7;", + &.{ smem, desc, mbar.bitAnd(cst32(b, sm100_mma_peer_bit_mask)), c[0], c[1], c[2], c[3], cst64(b, cache_hint) }, + "r,l,r,r,r,r,r,l,~{memory}", + ); +} + +/// `ptx::tma_load_1d(dst, src, mbar, num_bytes, hint = EVICT_FIRST)`: non-tensor bulk copy of +/// `num_bytes` from a global (possibly peer) address into this CTA's shared memory. +pub fn tmaLoad1d(b: *Builder, dst: Value, src: Value, mbar: Value, num_bytes: anytype, cache_hint: u64) void { + asmVoid( + b, + "cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes.L2::cache_hint [$0], [$1], $2, [$3], $4;\n", + &.{ dst, src, liftAs(b, num_bytes, .i32), mbar, cst64(b, cache_hint) }, + "r,l,r,r,l,~{memory}", + ); +} + +/// `ptx::tma_store_1d(dst, src, num_bytes, hint = EVICT_NORMAL)`. +pub fn tmaStore1d(b: *Builder, dst: Value, src: Value, num_bytes: anytype, cache_hint: u64) void { + asmVoid( + b, + "cp.async.bulk.global.shared::cta.bulk_group.L2::cache_hint [$0], [$1], $2, $3;\n", + &.{ dst, src, liftAs(b, num_bytes, .i32), cst64(b, cache_hint) }, + "l,r,r,l,~{memory}", + ); +} + +/// `cute::SM90_TMA_STORE_2D::copy(desc, smem, c0, c1)`. +pub fn tmaStore2d(b: *Builder, desc: Value, smem: Value, c0: Value, c1: Value) void { + asmVoid(b, "cp.async.bulk.tensor.2d.global.shared::cta.bulk_group [$0, {$2, $3}], [$1];", &.{ desc, smem, c0, c1 }, "l,r,r,r,~{memory}"); +} + +/// `cute::TMEM::Allocator2Sm::allocate(num_columns, dst)`: write the base column to shared `dst`. +pub fn tmemAllocate2Sm(b: *Builder, num_columns: u32, dst: Value) void { + asmVoid(b, "tcgen05.alloc.cta_group::2.sync.aligned.shared::cta.b32 [$0], $1;", &.{ dst, cst32(b, num_columns) }, "r,r,~{memory}"); +} + +/// `cute::TMEM::Allocator2Sm::free(tmem_ptr, num_columns)`. +pub fn tmemFree2Sm(b: *Builder, tmem_ptr: Value, num_columns: u32) void { + asmVoid(b, "{\n\ttcgen05.dealloc.cta_group::2.sync.aligned.b32 $0, $1; \n\t}", &.{ tmem_ptr, cst32(b, num_columns) }, "r,r,~{memory}"); +} + +/// `ptx::tcgen05_before_thread_sync`. +pub fn tcgen05BeforeThreadSync(b: *Builder) void { + asmVoid(b, "tcgen05.fence::before_thread_sync;", &.{}, "~{memory}"); +} + +/// `ptx::tcgen05_after_thread_sync`. +pub fn tcgen05AfterThreadSync(b: *Builder) void { + asmVoid(b, "tcgen05.fence::after_thread_sync;", &.{}, "~{memory}"); +} + +/// `cutlass::arch::fence_view_async_tmem_load` (`tcgen05.wait::ld`). +pub fn fenceViewAsyncTmemLoad(b: *Builder) void { + asmVoid(b, "{\n\ttcgen05.wait::ld.sync.aligned; \n}", &.{}, "~{memory}"); +} + +/// `fence_view_async_tmem_load` with the values of earlier TMEM loads threaded through it, so +/// LLVM cannot move their uses above the wait (the PTX is the same single instruction). +pub fn fenceViewAsyncTmemLoadValues(b: *Builder, comptime n: usize, values: [n]Value) [n]Value { + comptime var constraints: []const u8 = ""; + inline for (0..n) |i| constraints = constraints ++ (if (i == 0) "" else ",") ++ "=r"; + inline for (0..n) |i| constraints = constraints ++ std.fmt.comptimePrint(",{d}", .{i}); + constraints = constraints ++ ",~{memory}"; + return asmValues(b, n, "{\n\ttcgen05.wait::ld.sync.aligned; \n}", &values, .i32, constraints, true); +} + +/// `fenceViewAsyncTmemLoadValues` for a run-time number of values (the epilogue's store blocks +/// hold `STORE_BLOCK_M / ATOM_M * 8` words): the same single `tcgen05.wait::ld`, every value +/// tied through it. +pub fn fenceViewAsyncTmemLoadSlice(b: *Builder, values: []const Value) []Value { + const n = values.len; + const a = b.arena.allocator(); + var w: std.Io.Writer.Allocating = .init(a); + for (0..n) |i| w.writer.print("{s}=r", .{if (i == 0) "" else ","}) catch @panic("OOM"); + for (0..n) |i| w.writer.print(",{d}", .{i}) catch @panic("OOM"); + w.writer.writeAll(",~{memory}") catch @panic("OOM"); + return asmValuesSlice(b, n, "{\n\ttcgen05.wait::ld.sync.aligned; \n}", values, .i32, w.written(), true); +} + +/// `cute::SM100_TMEM_LOAD_16dp256b1x::copy(src, d0, d1, d2, d3)`: lane t of the warp gets +/// rows `base + t/4` (d0, d1) and `base + 8 + t/4` (d2, d3), columns `2(t%4)`, `2(t%4)+1`. +pub fn tmemLoad16dp256b1x(b: *Builder, taddr: Value) [4]Value { + return asmValues(b, 4, "tcgen05.ld.sync.aligned.16x256b.x1.b32{$0, $1, $2, $3},[$4];\n", &.{taddr}, .i32, "=r,=r,=r,=r,r", true); +} + +/// `cute::SM100_UTCCP_4x32dp128bit_2cta::copy(src_desc, dst_taddr)`. +pub fn utccp4x32dp128bit2cta(b: *Builder, src_desc: Value, dst_taddr: Value) void { + asmVoid(b, "tcgen05.cp.cta_group::2.32x128b.warpx4 [$0], $1;", &.{ dst_taddr, src_desc }, "r,l"); +} + +/// `ptx::SM100_MMA_MXF8F6F4_2x1SM_SS::fma(desc_a, desc_b, tmem_c, scale_c, desc, tmem_sfa, +/// tmem_sfb)`: `desc` is the 64-bit runtime descriptor (`instr << 32`), `scale_c` (i32) +/// nonzero accumulates. +pub fn mmaMxf8f6f42x1SmSs(b: *Builder, desc_a: Value, desc_b: Value, tmem_c: Value, scale_c: Value, desc: Value, tmem_sfa: Value, tmem_sfb: Value) void { + const idesc = b.emit(arith.trunci(b.ctx, desc.shrLogical(32).inner, DType.i32.toMlir(b.ctx), b.loc())); + asmVoid( + b, + "{\n\t.reg .pred p;\n\tsetp.ne.b32 p, $4, 0;\n\ttcgen05.mma.cta_group::2.kind::mxf8f6f4.block_scale [$0], $1, $2, $3, [$5], [$6], p; \n\t}\n", + &.{ tmem_c, desc_a, desc_b, idesc, scale_c, tmem_sfa, tmem_sfb }, + "r,l,l,r,r,r,r", + ); +} + +/// `cutlass::arch::umma_arrive_multicast_2x1SM(barrier, cta_mask)`: inside `elect_one_sync`, +/// `tcgen05.commit` arriving on `barrier` in every CTA of `cta_mask`. +pub fn ummaArriveMulticast2x1Sm(b: *Builder, bar: Value, cta_mask: u16) void { + var elected = b.openIf(electOneSync(b)); + { + asmVoid( + b, + "{\n\ttcgen05.commit.cta_group::2.mbarrier::arrive::one.shared::cluster.multicast::cluster.b64 [$0], $1; \n\t}", + &.{ bar, b.constant(.i16, @as(i16, @bitCast(cta_mask))) }, + "r,h,~{memory}", + ); + elected.close(); + } +} + +pub fn bits32(v: Value) Value { + return if (v.dtype() == .i32) v else v.bitCast(.i32); +} + +pub fn fromBits32(v: Value, dt: DType) Value { + return if (dt == .i32) v else v.bitCast(dt); +} + +/// `__shfl_sync(0xffffffff, v, src_lane)` / `ptx::exchange` (32-bit). +pub fn shflSync(b: *Builder, v: Value, src_lane: anytype) Value { + const r = asmValue(b, "shfl.sync.idx.b32 $0, $1, $2, 31, -1;", &.{ bits32(v), liftAs(b, src_lane, .i32) }, .i32, "=r,r,r", true); + return fromBits32(r, v.dtype()); +} + +/// `__shfl_xor_sync(0xffffffff, v, lane_mask)`. +pub fn shflXorSync(b: *Builder, v: Value, lane_mask: anytype) Value { + const r = asmValue(b, "shfl.sync.bfly.b32 $0, $1, $2, 31, -1;", &.{ bits32(v), liftAs(b, lane_mask, .i32) }, .i32, "=r,r,r", true); + return fromBits32(r, v.dtype()); +} + +/// `__shfl_up_sync(0xffffffff, v, delta)`. +pub fn shflUpSync(b: *Builder, v: Value, delta: anytype) Value { + const r = asmValue(b, "shfl.sync.up.b32 $0, $1, $2, 0, -1;", &.{ bits32(v), liftAs(b, delta, .i32) }, .i32, "=r,r,r", true); + return fromBits32(r, v.dtype()); +} + +/// `__reduce_add_sync(0xffffffff, v)` (u32). +pub fn reduceAddSync(b: *Builder, v: Value) Value { + return asmValue(b, "redux.sync.add.u32 $0, $1, $2;", &.{ v, cst32(b, 0xFFFFFFFF) }, .i32, "=r,r,r", true); +} + +/// `__reduce_min_sync(0xffffffff, v)` (u32). +pub fn reduceMinSync(b: *Builder, v: Value) Value { + return asmValue(b, "redux.sync.min.u32 $0, $1, $2;", &.{ v, cst32(b, 0xFFFFFFFF) }, .i32, "=r,r,r", true); +} + +/// `__all_sync(0xffffffff, pred)` (`pred` i1; returns i1). +pub fn allSync(b: *Builder, pred: Value) Value { + const r = asmValue(b, "{ .reg .pred p, q; setp.ne.u32 p, $1, 0; vote.sync.all.pred q, p, $2; selp.u32 $0, 1, 0, q; }", &.{ pred.to(.i32), cst32(b, 0xFFFFFFFF) }, .i32, "=r,r,r", true); + return r.ne(0); +} + +pub fn popc(b: *Builder, v: Value) Value { + return b.callIntrinsic("llvm.ctpop.i32", &.{v}, DType.i32.toMlir(b.ctx)).?; +} + +/// `__ffs(v)`: 1-based index of the lowest set bit, 0 for 0 (nvcc: `brev` + `bfind.shiftamt`). +pub fn ffs(b: *Builder, v: Value) Value { + return asmValue(b, "{ .reg .b32 r; brev.b32 r, $1; bfind.shiftamt.u32 $0, r; }", &.{v}, .i32, "=r,r", false).add(1); +} + +/// `__fns(mask, base, offset)`: position of the `offset`-th set bit of `mask` from `base`. +pub fn fns(b: *Builder, mask: Value, base: anytype, offset: Value) Value { + return asmValue(b, "fns.b32 $0, $1, $2, $3;", &.{ mask, liftAs(b, base, .i32), offset }, .i32, "=r,r,r,r", false); +} + +/// `fminf(a, b)` (`min.f32`, NaN-ignoring; `kClampAfterSilu`). +pub fn fminf(b: *Builder, x: Value, y: Value) Value { + return asmValue(b, "min.f32 $0, $1, $2;", &.{ x, y }, .f32, "=f,f,f", false); +} + +/// `fabsf(x)` (`abs.f32`). +pub fn fabsf(b: *Builder, x: Value) Value { + return asmValue(b, "abs.f32 $0, $1;", &.{x}, .f32, "=f,f", false); +} + +/// `math::fast_rcp(x)`. +pub fn fastRcp(b: *Builder, x: Value) Value { + return asmValue(b, "rcp.approx.ftz.f32 $0, $1;", &.{x}, .f32, "=f,f", true); +} + +/// `__expf(x)` without `--use_fast_math`: `ex2.approx.f32(x * log2(e))` (non-FTZ). +pub fn expfFast(b: *Builder, x: Value) Value { + return asmValue(b, "{ .reg .f32 t; mul.f32 t, $1, 0f3FB8AA3B; ex2.approx.f32 $0, t; }", &.{x}, .f32, "=f,f", false); +} + +/// `__expf(-x)`: nvcc folds the negation into the constant (`mul.f32 t, x, -log2(e)`). +pub fn expfFastNeg(b: *Builder, x: Value) Value { + return asmValue(b, "{ .reg .f32 t; mul.f32 t, $1, 0fBFB8AA3B; ex2.approx.f32 $0, t; }", &.{x}, .f32, "=f,f", false); +} + +/// `__float22bfloat162_rn({lo, hi})`: bf16x2 word, `lo` in the low half. +pub fn float22bfloat162Rn(b: *Builder, lo: Value, hi: Value) Value { + return asmValue(b, "{ cvt.rn.bf16x2.f32 $0, $2, $1;}\n", &.{ lo, hi }, .i32, "=r,f,f", false); +} + +/// `__hmin2(a, b)` on bf16x2 words. +pub fn hmin2Bf16(b: *Builder, x: Value, y: Value) Value { + return asmValue(b, "{ min.bf16x2 $0,$1,$2;\n}", &.{ x, y }, .i32, "=r,r,r", false); +} + +/// `__hmax2(a, b)` on bf16x2 words. +pub fn hmax2Bf16(b: *Builder, x: Value, y: Value) Value { + return asmValue(b, "{ max.bf16x2 $0,$1,$2;\n}", &.{ x, y }, .i32, "=r,r,r", false); +} + +/// `__bfloat1622float2(word)`: `{low, high}` f32 (exact). +pub fn bfloat1622float2(b: *Builder, word: Value) [2]Value { + return asmValues(b, 2, "{ .reg .b16 l, h; mov.b32 {l, h}, $2; cvt.f32.bf16 $0, l; cvt.f32.bf16 $1, h; }", &.{word}, .f32, "=f,=f,r", false); +} + +/// `ptx::accumulate(float2& a, nv_bfloat162 b)`: `a.x += bf16(b.x)`, `a.y += bf16(b.y)` with +/// `add.rn.f32.bf16` (one rounding, the bf16 widened exactly). +pub fn accumulateBf16x2(b: *Builder, acc: [2]Value, bf16x2: Value) [2]Value { + const lo = i16Of(b, bf16x2); + const hi = i16Of(b, bf16x2.shrLogical(16)); + return .{ + asmValue(b, "add.rn.f32.bf16 $0, $1, $0;\n", &.{ lo, acc[0] }, .f32, "=f,h,0", false), + asmValue(b, "add.rn.f32.bf16 $0, $1, $0;\n", &.{ hi, acc[1] }, .f32, "=f,h,0", false), + }; +} +/// IEEE operations used when staging quantized inputs; no FTZ or contraction. +pub fn divRn(b: *Builder, x: Value, y: Value) Value { + return asmValue(b, "div.rn.f32 $0, $1, $2;", &.{ x, y }, .f32, "=f,f,f", false); +} +pub fn mulRn(b: *Builder, x: Value, y: Value) Value { + return asmValue(b, "mul.rn.f32 $0, $1, $2;", &.{ x, y }, .f32, "=f,f,f", false); +} + +test "canonical primitives preserve aligned barriers and runtime transaction counts" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = try Builder.open(std.testing.allocator, ctx, "compatibility_primitives"); + defer b.deinit(); + _ = try b.declareArgs(.{}); + const bar = b.constant(.i32, 0); + const phase = b.constant(.i32, 0); + const bytes = b.threadIdx().x.add(16); + b.barrierCtaSyncAligned(1, 32); + b.barrierClusterArriveRelaxedAligned(); + b.barrierClusterWaitAligned(); + b.mbarrierInit(bar, 1); + b.fenceMbarrierInit(); + b.mbarrierArriveExpectTx(bar, bytes); + b.mbarrierArriveExpectTx(bar, 16); + b.mbarrierWait(bar, phase, .{ .test_first = false }); + _ = phase.bitXor(1); + b.fenceProxyAsyncShared(); + b.fenceProxyAsyncShared(); + b.cpAsyncBulkCommitGroup(); + b.cpAsyncBulkWaitGroup(0, false); + _ = b.voteBallotSync(bytes.ne(0), null); + _ = b.fmax(b.constant(.f32, 1), b.constant(.f32, 2), .{}); + const ir = try b.finish(.{ 32, 1, 1 }); + defer std.testing.allocator.free(ir); + try expectContains(ir, "llvm.nvvm.barrier.cta.sync.aligned.count"); + try expectContains(ir, "barrier.cluster.arrive.relaxed.aligned"); + try expectContains(ir, "nvvm.mbarrier.arrive.expect_tx"); + const parsed = try mlir.Module.parse(ctx, ir); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} diff --git a/dsl/cute/dtype.zig b/dsl/cute/dtype.zig new file mode 100644 index 00000000..ae98cb08 --- /dev/null +++ b/dsl/cute/dtype.zig @@ -0,0 +1,67 @@ +const std = @import("std"); +const mlir = @import("mlir"); + +/// Element types a `!cute.ptr` can point at. +pub const DType = enum { + i1, + i8, + i16, + i32, + i64, + f16, + bf16, + f32, + f64, + f4e2m1fn, + f8e4m3fn, + f8e5m2, + f8e8m0fnu, + + pub fn toMlir(self: DType, ctx: *mlir.Context) *const mlir.Type { + return switch (self) { + .i1 => .int(ctx, .i1), + .i8 => .int(ctx, .i8), + .i16 => .int(ctx, .i16), + .i32 => .int(ctx, .i32), + .i64 => .int(ctx, .i64), + .f16 => .float(ctx, .f16), + .bf16 => .float(ctx, .bf16), + .f32 => .float(ctx, .f32), + .f64 => .float(ctx, .f64), + .f4e2m1fn => .float(ctx, .f4e2m1fn), + .f8e4m3fn => .float(ctx, .f8e4m3fn), + .f8e5m2 => .float(ctx, .f8e5m2), + .f8e8m0fnu => .float(ctx, .f8e8m0fnu), + }; + } +}; + +pub fn isFloatDtype(dt: DType) bool { + return switch (dt) { + .f16, .bf16, .f32, .f64, .f4e2m1fn, .f8e4m3fn, .f8e5m2, .f8e8m0fnu => true, + else => false, + }; +} + +pub fn dtypeBitwidth(dt: DType) usize { + return switch (dt) { + .i1 => 1, + .f4e2m1fn => 4, + .i8, .f8e4m3fn, .f8e5m2, .f8e8m0fnu => 8, + .i16, .f16, .bf16 => 16, + .i32, .f32 => 32, + .i64, .f64 => 64, + }; +} + +pub fn mlirToDType(ctx: *mlir.Context, t: *const mlir.Type) DType { + inline for (@typeInfo(DType).@"enum".field_names) |f| { + const dt = @field(DType, f); + if (t.eql(dt.toMlir(ctx))) return dt; + } + std.debug.panic("type {f} is not a CuTe DType", .{t}); +} + +test { + std.testing.refAllDecls(@This()); +} diff --git a/dsl/fly/BUILD.bazel b/dsl/fly/BUILD.bazel new file mode 100644 index 00000000..c8bdf854 --- /dev/null +++ b/dsl/fly/BUILD.bazel @@ -0,0 +1,35 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "builder", + srcs = [ + "dtype.zig", + "layout.zig", + ], + import_name = "dsl/fly/builder", + main = "builder.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//dsl/common:builder", + "//mlir", + "//mlir/dialects/arith", + "//mlir/dialects/fly", + "//mlir/dialects/fly_rocdl", + "//mlir/dialects/func", + "//mlir/dialects/gpu", + "//mlir/dialects/llvm", + "//mlir/dialects/math", + "//mlir/dialects/memref", + "//mlir/dialects/rocdl", + "//mlir/dialects/scf", + "//mlir/dialects/ub", + "//mlir/dialects/vector", + "//stdx", + ], +) + +zig_test( + name = "test", + deps = [":builder"], +) diff --git a/dsl/fly/PORTING.md b/dsl/fly/PORTING.md new file mode 100644 index 00000000..f7cb19f3 --- /dev/null +++ b/dsl/fly/PORTING.md @@ -0,0 +1,104 @@ +# Porting a FlyDSL Python kernel to Zig + +The Zig DSL builds Fly IR directly. Work from the revision and patches in +[the FlyDSL repository rule](../../third_party/flydsl/repo.bzl), reading +Python sources with `git -C show :`. Check the +[ROCm plugin pin](../../platforms/rocm/rocm.bzl) as well: the emitter and +runtime must agree on the dialect version. FlyDSL HEAD may differ from both. + +## Cheat sheet + +| Python (`flydsl.expr`) | Zig | +|---|---| +| `@flyc.kernel def k(A: Tensor, ...)` | `fly.Kernel(Cfg, .{ .inputs = &.{"a"}, .outputs = &.{"c"}, .run = run })`; `const a = K.args(b)` | +| kernel arg `tiled_copy: fx.TiledCopy` | build it inside `run` (static values cannot be arguments) | +| `fx.thread_idx.x`, `fx.block_idx.x` | `b.threadId(.x)`, `b.blockId(.x)` (`i32`) | +| `A.shape.unpack()` | `a.a.shapeStatic().at(i).leaf.s` | +| `fx.make_layout((8,16),(16,1))` | `fly.L(.{ 8, 16 }, .{ 16, 1 })` (comptime) or `b.static(layout)` | +| `fx.make_ordered_layout((8,16),(1,0))` | `fly.ordered(.{ 8, 16 }, .{ 1, 0 })` | +| `fx.make_tile(64, 8)` / tuple divisor `(64, 8)` | `fly.tile(.{ 64, 8 })` / `.{ 64, 8 }` | +| `fx.make_int_tuple((m, tid))`, coords | `b.intTuple(.{ m, tid })` (`Value`s become `?` leaves) | +| `t[None, None, bx, by]` | `t.slice(.{ null, null, bx, by })` | +| `t[(0, None), None, None]` | `t.slice(.{ .{ 0, null }, null, null })` | +| `t[i]` (no `None`) / `t[i] = v` | `t.at(i)` / `t.set(i, v)` | +| `fx.flat_divide(t, tile)` etc. | `t.flatDivide(tile)`, `.logicalDivide`, `.zippedDivide`, `.tiledDivide` | +| `fx.raked_product(a, b)` etc. | `a.rakedProduct(b)`, `.logicalProduct`, ... | +| `fx.right_inverse(l)`, `fx.composition(l, r)` | `l.rightInverse()`, `l.composition(r)` | +| `fx.get_shape(l)`, `fx.size(l)` (emitted) | `l.emitShape()`, `l.emitSize()` | +| `fx.size(t.shape).unpack()` (host int) | `t.sizeStatic()` | +| `fx.select(tup, [0])`, `fx.get(tup, [1])` | `tup.selectModes(&.{0})`, `tup.get(&.{1})` | +| `fx.make_view((0,0), fx.make_identity_layout((M,N)))` | `b.identity(.{ m, n })` | +| `fx.make_view(ptr, layout)` | `ptr.view(layout)` | +| `fx.make_copy_atom(fx.UniversalCopy128b(), fx.Float32)` | `b.copyAtom(.{ .universal = 128 }, .f32)` | +| `fx.rocdl.BufferCopy32b()` / `BufferCopyLDS128b()` | `.{ .buffer_copy = 32 }` / `.{ .buffer_copy_lds = 128 }` | +| `fx.make_mma_atom(fx.rocdl.MFMA(16,16,4,fx.Float32))` | `b.mmaAtom((try fly.fly_rocdl.MmaOpCDNA3MFMAType.get(b.ctx, .{ .m = 16, .n = 16, .k = 4, ... })).type_())` | +| `fx.make_tiled_mma(atom, layout)` | `b.tiledMma(atom, layout, null)` | +| `fx.make_tiled_copy_tv(atom, thr, val)` | `b.tiledCopyTV(atom, thr, val)` | +| `fx.make_tiled_copy_A(atom, tiled_mma)` | `b.tiledCopyA(atom, tiled_mma)` | +| `tiled_copy.tile_mn` | `tc.tileMN()` | +| `tiled_mma.tile_size_mnk` etc. | `@import("mlir/dialects/fly").expect(tm.value.type_(), .tiled_mma).tileSizeMNK()` | +| `tiled_copy.get_slice(tid).partition_S(t)` | `tc.getSlice(tid).partitionS(t)` | +| `thr_copy.retile(t)` | `thr.retile(t)` | +| `tiled_mma.make_fragment_A(t)` | `tm.makeFragmentA(t)` | +| `fx.make_fragment_like(t[, dtype=fx.Boolean])` | `t.makeFragmentLike(null)` / `t.makeFragmentLike(.i1)` | +| `fx.make_rmem_tensor(8, fx.Float16)` | `b.rmemTensor(fly.L(8, 1), .f16)` | +| `fx.copy(atom, src, dst, pred=p)` | `b.copy(atom, src, dst, .{ .pred = p })` | +| `fx.copy_atom_call(atom, src, dst)` | `b.copyAtomCall(atom, src, dst)` | +| `fx.gemm(atom, d, a, b, c)` | `b.gemm(atom, d, a, bb, c)` | +| `t.load()`, `t.store(v)`, `t.fill(0)` | same names | +| `fx.elem_less(coord, (M, N))` | `coord.elemLess(.{ m, n })` | +| `fx.rocdl.make_buffer_tensor(t)` | `b.bufferTensor(t)` | +| `fx.SharedAllocator().allocate(fx.Array[f32, n, 16])` | `b.sharedArray(.f32, n, 16)` (returns an `n:1` tensor) | +| `gpu.barrier()` | `b.barrier()` | +| `x.shuffle_xor(off, 64)` | `x.shuffleXor(off, 64)` | +| `v.reduce(ReductionOp.ADD)` | `v.reduce(.add)` | +| `v.to(fx.Float32)` | `v.to(.f32)` | +| `x * rrms` (vector × scalar) | `x.mul(rrms)` | +| `fmath.rsqrt(x, fastmath=fast)` | `b.setFastMath(true); b.rsqrt(x)` | +| `lane < RED_SLOTS`, `cond.select(a, b)` | `lane.lt(n)`, `cond.select(a, b)` | +| `if tid == 0: ...` | `var s = b.openIf(tid.eq(0)); ...; s.close();` | +| `for i in range_constexpr(n)` | a plain Zig `for (0..n)` (unrolled at emit time) | +| `for i in range(lo, hi)` with carried values | `b.openFor(lo, hi, 1, .{ init })` + `.yield(.{ ... })` | +| `launch(grid=..., block=(256,1,1))` | `K.call(inputs, outputs, .{ .grid = ..., .threads = 256 })` | + +## Method + +1. Read the kernel and its helpers at the pin; note every `fx.*` primitive. +2. Write the Zig `run` top to bottom, preserving operation order and numeric + conversions. For matrix loops, use SSA copy and MMA calls as described + under [control flow and register tensors](README.md#control-flow-and-register-tensors): + `b.copyAtomLoad` returns vectors, `b.mmaAtomCall` returns the accumulator, + and `b.copyAtomStore` writes vectors. Carry accumulator values through + `openFor`; do not put mutable register tensors across control-flow regions. +3. `bazel test //dsl/fly:test` (or a `K.emit` test): the module must + verify and re-parse. A panic from `fly.: cannot create operation` + lists the operand types; compare them with the Python trace. +4. Run on the ROCm box against a host reference + (`bazel test //zml:test --@zml//platforms:rocm=true ...`). + +## Things that differ + +- Python config (`N`, `dtype_str`, `const_expr(...)` branches) becomes the + kernel `Config`; branch on it with plain `if` in `run`. +- Python lists of vectors across an unrolled loop are Zig arrays of `Value`. + `v.insert(i, x)` returns a new vector; `v.extract(i)` reads an element. + `v.bitCast(dtype)` preserves total bits and adjusts vector length. +- Keep unsigned semantics explicit for packed bytes and offsets with + `toUnsigned`, `divUnsigned`, `remUnsigned` and `cmpUnsigned`. +- `v.shuffleVector(rhs, indices)` rearranges vector elements within a thread; + `v.shuffle(.up, offset, width)` and `shuffleXor` exchange values across lanes. +- Runtime `while` loops use `b.openWhile(inits, after_types)`, `yieldBefore` + and `yieldAfter`; use a plain Zig `while` only for loops unrolled at emit time. +- Python `Int32(x)` / `Float32(x)` constants are `b.constant(.i32, x)`; most + arithmetic methods also take a literal directly (`tid.add(256)`). +- Cached scalars from LDS are read with `t.at(i)`; there is no implicit load. +- bf16 outputs use `arith.truncf`; the Python's manual round-to-nearest bit + trick for pre-gfx950 parts is not reproduced. + +## Model kernels + +The [MoE kernels](../../zml/kernels/moe/fly/) include native routing and +MXFP4 GEMV/MFMA implementations. llmd's sparse MLA kernel +(`llmd/kernels/mla/fly/sparse_mla.zig`, downstream) uses SSA +matrix accumulators through attention loops. These are examples of the +builder APIs with model tensor layouts and launch wrappers. diff --git a/dsl/fly/README.md b/dsl/fly/README.md new file mode 100644 index 00000000..687c99dc --- /dev/null +++ b/dsl/fly/README.md @@ -0,0 +1,255 @@ +# ZML FlyDSL DSL — Author's Guide + +A Zig builder for AMD's FlyDSL (`fly` / `fly_rocdl` MLIR dialects) that reads +like `flydsl.expr` Python while staying in pure Zig. Kernels are compiled and +launched by ZML's ROCm PJRT plugin through the `__gpu$xla.gpu.fly` custom +call; ZML never links a GPU compiler. + +Two layers define the surface: + +- **Low layer** — `dsl/fly/builder.zig`: `Builder`, `Value`, `TiledCopy`, + `TiledMma`, the comptime layout literals in `layout.zig`, and `DType`. + `Builder.open` takes an `*mlir.Context` from `zml.kernel.fly.newContext()`. +- **High layer** — `zml.kernel.fly.Kernel(Config, spec)`: names the kernel's + inputs and outputs, hands `run` its arguments by name (`K.args(b)`), and + `call`s it from a ZML model. + +The dialect bindings construct typed MLIR operations. Layout-transforming +operations use FlyDSL's `inferReturnTypes`; static layout literals use typed +attribute constructors and accessors. + +## Skeleton of a kernel + +```zig +const zml = @import("zml"); +const fly = zml.kernel.fly; + +pub const Cfg = struct { + thr: fly.Layout = fly.ordered(.{ 8, 16 }, .{ 1, 0 }), // 128 threads + val: fly.Layout = fly.ordered(.{ 1, 4 }, .{ 0, 1 }), // f32x4 along N +}; + +pub const VectorAdd = fly.Kernel(Cfg, .{ + .name = "vector_add", + .inputs = &.{ "a", "b" }, + .outputs = &.{"c"}, + .run = run, +}); + +fn run(b: *fly.Builder, cfg: Cfg) fly.FinishError!void { + const a = VectorAdd.args(b); // .a/.b/.c: row-major !fly.memref views + const shape = a.a.shapeStatic(); + const m = shape.at(0).leaf.s; + const n = shape.at(1).leaf.s; + + const atom = b.copyAtom(.{ .universal = 128 }, .f32); + const tc = b.tiledCopyTV(atom, cfg.thr, cfg.val); // fx.make_tiled_copy_tv + const tid = b.threadId(.x); + const bx = b.blockId(.x); + const by = b.blockId(.y); + + const tile = tc.tileMN(); + const gA = a.a.flatDivide(tile).slice(.{ null, null, bx, by }); // A[None, None, bx, by] + const gB = a.b.flatDivide(tile).slice(.{ null, null, bx, by }); + const gC = a.c.flatDivide(tile).slice(.{ null, null, bx, by }); + const cC = b.identity(.{ m, n }).flatDivide(tile).slice(.{ null, null, bx, by }); + + const thr = tc.getSlice(tid); + const tgA = thr.partitionS(gA); + const tgB = thr.partitionS(gB); + const tgC = thr.partitionD(gC); + const tcC = thr.partitionS(cC).slice(.{ .{ 0, null }, null, null }); + + const rA = tgA.makeFragmentLike(null); + const rB = tgB.makeFragmentLike(null); + const rC = tgC.makeFragmentLike(null); + const pC = tcC.makeFragmentLike(.i1); + for (0..@intCast(pC.sizeStatic())) |i| pC.set(i, tcC.at(i).elemLess(.{ m, n })); + + b.copy(atom, tgA, rA, .{ .pred = pC }); + b.copy(atom, tgB, rB, .{ .pred = pC }); + rC.store(rA.load().add(rB.load())); + b.copy(atom, rC, tgC, .{ .pred = pC }); +} + +// In a model: +const c = VectorAdd.call(.{ .a = a, .b = bt }, .{ .c = a.shape() }, .{ + .cfg = .{}, + .grid = .{ grid_m, grid_n, 1 }, + .threads = 128, +}).c; +``` + +Complete vector-add and tiled-MMA kernels are the tests in `builder.zig` +and `zml/kernel/fly/test.zig`. + +Model kernels live beside their consumers: + +- [MoE routing and MXFP4 projections](../../zml/kernels/moe/fly/moe.zig). +- Sparse MLA, downstream in llmd (`llmd/kernels/mla/fly/sparse_mla.zig`). + +Fly is a separate backend. The [MoE selector](../../zml/kernels/moe/moe.zig) and +llmd's sparse-MLA selector select by platform, +architecture and dtype. The current native kernels target gfx942; their +callers check shape support before routing or launching kernels and use +Triton for unsupported contracts. + +## The ABI + +`Kernel.emit(allocator, cfg, shapes)` takes shapes in input-then-output order +and produces + +```mlir +module attributes {gpu.container_module} { + gpu.module @zml_fly_kernels { + gpu.func @name(%a: !fly.ptr, ..., %c: !fly.ptr) kernel { ... gpu.return } + } +} +``` + +with one `!fly.ptr` per input, then per output. The plugin checks only the +argument **count**, so the pointer element type is on you: `DType.storageElem` +maps predicates to `i8` because that is how XLA stores them. Static DSL values +(layouts, tiles, atoms) must be built inside the body, never passed as +arguments. `Kernel.call` bridges each non-scalar argument to a row-major +`!fly.memref` view of its `zml.Shape`. Rank-zero arguments remain pointers; +use `ptrLoad` and `ptrStore` for their scalar values. + +Launch geometry lives in `CallOpts`: `grid`, `threads` (threads per block, +converted to wavefronts using the device's wave size — 64 on CDNA, 32 on +RDNA — so it must be a multiple of that), optional `waves_per_eu` (sets the +LLVM `amdgpu-waves-per-eu` attribute to `N, N` when positive), +`shared_mem_bytes` (only for `fly.get_dyn_shared`; static LDS from +`sharedArray` needs none) and `zeroed_args` (indices into inputs-then-outputs; +ignored inside HIP graphs, so zero what you need yourself). + +`output_operand_aliases` maps output names to input names for buffers that +the kernel reads and writes. Aliasing preserves the input-then-output ABI; +it does not initialize output memory. + +## Layouts, tiles, coordinates + +`layout.zig` holds comptime literals. `toAttr` turns one into the matching +fly attribute through the dialect's own constructors, and reading a type back +goes through its accessors — neither direction goes through text: + +| Zig | MLIR | +|---|---| +| `fly.it(.{ .{ 2, 4 }, 8 })` | `!fly.int_tuple<((2,4),8)>` | +| `fly.L(.{ 4, 8 }, .{ 1, 4 })` | `!fly.layout<(4,8):(1,4)>` | +| `fly.rowMajor(.{ 4, 8 })` / `fly.colMajor` | `(4,8):(8,1)` / `(4,8):(1,4)` | +| `fly.ordered(.{ 8, 16 }, .{ 1, 0 })` | `fx.make_ordered_layout((8,16),(1,0))`, folded | +| `fly.tile(.{ 128, 64 })`, `fly.tile(.{ null, 8 })` | `!fly.tile<[128\|64]>`, `[*\|8]` | + +`b.static(literal)` materializes one as `fly.static`. Wherever the Python +takes a tuple, the Zig takes a tuple literal: `t.flatDivide(.{ 64, 8 })` is a +tile, `t.slice(.{ null, bid })` is `t[None, bid]`, `b.intTuple(.{ m, tid })` +mixes static ints, `null` (`*`) and dynamic `Value`s (`?` leaves, passed as +operands). `v.shapeStatic()` / `v.sizeStatic()` / `v.elemDType()` read the +inferred type back on the host; `v.emitShape()` / `v.emitSize()` emit ops. + +## Atoms and tiled ops + +- `b.copyAtom(.{ .universal = 128 }, .f32)`, `.{ .buffer_copy = 32 }`, + `.{ .buffer_copy_lds = 128 }`. +- `b.mmaAtom(mma_op)` takes an MLIR type. Generated MMA type constructors + return a typed wrapper; handle their error and pass the wrapper's + `.type_()` result. +- `b.tiledCopyTV(atom, thr, val)`, `b.tiledCopy(atom, layout_tv, tile)`, + `b.tiledCopyA/B/C(copy_atom, tiled_mma)`; `tc.getSlice(tid).partitionS/D`, + `.retile`. +- `b.tiledMma(atom, fly.L(.{ 2, 2, 1 }, .{ 1, 2, 0 }), null)`; + `tm.makeFragmentA/B/C(t)`, `tm.getSlice(tid).partitionA/B/C`. +- `b.copy(atom, src, dst, .{ .pred = p })`, `b.copyAtomCall(atom, src, dst)`, + `b.gemm(atom, d, a, b, c)`. +- `b.mmaAtomCall(atom, a, b, c)` takes SSA operands and returns the updated + accumulator. Operand vectors must match the atom's per-lane fragment + types and layout; this call performs no tiled partitioning. +- `b.bufferTensor(t)` is `fx.rocdl.make_buffer_tensor`: the same tensor over + a CDNA raw-buffer descriptor, for bounds-checked `buffer_copy` atoms. +- Derived layouts come off the type: `@import("mlir/dialects/fly").expect(tm.value.type_(), .tiled_mma)` + then `.tileSizeMNK()`, `.tiledThrValLayoutA()`, ... + +## Memory and scalars + +`t.load()` / `t.store(vec)` move a whole register tensor as a `vector`; +`t.at(coord)` / `t.set(coord, v)` are element accesses (an int tuple for a +coordinate tensor); `t.fill(0)` splats. `b.rmemTensor(layout, dtype)` is +`fx.make_rmem_tensor`; `b.sharedArray(.f32, n, 16)` is a static LDS array +(`fx.SharedAllocator().allocate(fx.Array[...])`). + +Scalars and vectors: `add/sub/mul/div/rem` and `lt(x)` (`cmp` for integers, +`cmpf` for floats; `cmpUnsigned`) broadcast a scalar against a vector; +`to(dtype)`, `cond.select(a, b)`, `v.reduce(.add)`, `v.shuffleXor(off, 64)`, +`b.rsqrt(x)`, `b.constant(.f32, 1.0)`, `b.threadId(.x)` / `b.blockId(.x)` +(already `i32`), `b.barrier()`. + +Integer types are signless in MLIR. `div`, `rem`, `cmp` and `to` use signed +integer semantics, except that `to` treats predicates as unsigned. +`divUnsigned`, `remUnsigned`, `cmpUnsigned` and `toUnsigned` make unsigned +operations explicit. `bitAnd`, `bitOr`, `bitXor`, `shl` and `shrU` operate +on integer bits. + +`splat` creates a vector, including when its length is one. Numeric casts +preserve vector shape. `bitCast` preserves total bits and adjusts vector +length to the destination element width; scalar bitcasts require equal +widths. Use `extract` to obtain a scalar, `insert` to replace an element and +`shuffleVector` to select elements from two vectors. Lane `shuffle` +supports `xor`, `up`, `down` and `idx` modes with a literal or an `i32` +`Value` offset. + +`maximum` propagates NaNs; `maxNum` selects the numeric operand when only +one operand is NaN. `neg`, `Builder.log` and `Builder.fma` provide floating +operations. Fast math is off by default; `setFastMath(true)` tags subsequent +float operations `fastmath` and permits relaxed numerical semantics. + +`Builder.perm` emits AMD byte permutation. `Builder.cvtPkF32Fp8` decodes +two FP8 bytes from the low or high half of an `i32` word using the CDNA3 +packed conversion instruction. Architecture selection belongs in the +backend before these operations are emitted. + +`Builder.ptrAtomicAdd` adds an `i32` counter and returns its old value. +Use `workgroup` scope for LDS pointers and `agent` scope for global +pointers. The ordering is monotonic; callers provide barriers or kernel +dependencies before consuming completed counters. + +Control flow uses `dsl/common`: `b.openFor(lo, hi, step, .{ inits })` +with `.yield(.{ ... })` (`.close()` without inits), `b.openIf(cond)` with +`.close()`, `b.openIfElse(cond, .{ types })` with `.yieldThen` / `.yieldElse` +(`.elseBranch()` / `.close()` without types). Types are dtypes, Values for +their types, or MLIR types; tuples can also be arrays. +`openWhile` has separate condition and body regions: `before_carried` +feeds `yieldBefore`, `after_carried` feeds `yieldAfter`, and `results` +contains the values returned when the condition becomes false. + +## Control flow and register tensors + +The published ROCm plugin lowers `scf` to `cf` **before** running the Fly +passes on a native `gpu.func` entry, and Fly's register-alloca promotion +only rewrites a single-block body. Kernels with runtime control flow must +keep their register state in SSA values: + +- `copyAtomLoad` returns a vector and `copyAtomStore` stores a vector. +- `mmaAtomCall` returns an accumulator without allocating register memory. +- Pass accumulators through `openFor` initial values, `carried`, `yield` + and `results`. Branches and while loops also return SSA values through + their scope objects. + +Put each scope's body in a Zig `{ ... }` block right after `openFor`, +`openIf` or `openWhile`, ending with its `yield` (or `close()` for a scope +without values). `openIfElse` and `openWhile` get one block per region, ending +with `yieldThen`/`yieldElse` (`elseBranch()`/`close()` without results) and +`yieldBefore`/`yieldAfter`. The scope variable stays outside, so its `results` +stay readable after the blocks. + +The MoE and sparse-MLA kernels linked above use this path with the published +runtime. Straight-line kernels can still use `rmemTensor`, +`makeFragmentLike`, `fill` and `gemm`. + +## Debugging + +- A failing `inferReturnTypes` panics with the op name and every operand type + (`fly.make_tiled_copy: cannot create operation ... operand 1: !fly.layout<...>`). +- `Kernel.emit` returns the module text; `bazel test //dsl/fly:test` + builds the example kernels on any host, no GPU needed. +- On the device side, `--xla_dump_to=` dumps the module the plugin saw. diff --git a/dsl/fly/builder.zig b/dsl/fly/builder.zig new file mode 100644 index 00000000..a083d943 --- /dev/null +++ b/dsl/fly/builder.zig @@ -0,0 +1,1978 @@ +const std = @import("std"); + +const arith = @import("mlir/dialects/arith"); +const common = @import("dsl/common/builder"); +const fly = @import("mlir/dialects/fly"); +const func = @import("mlir/dialects/func"); +const gpu = @import("mlir/dialects/gpu"); +const llvm = @import("mlir/dialects/llvm"); +const math = @import("mlir/dialects/math"); +const memref = @import("mlir/dialects/memref"); +const rocdl = @import("mlir/dialects/rocdl"); +const scf = @import("mlir/dialects/scf"); +const ub = @import("mlir/dialects/ub"); +const vector = @import("mlir/dialects/vector"); +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +pub const layout = @import("layout.zig"); +pub const dtypes = @import("dtype.zig"); + +pub const DType = dtypes.DType; +pub const fly_rocdl = @import("mlir/dialects/fly_rocdl"); +pub const IntTuple = layout.IntTuple; +pub const Layout = layout.Layout; +pub const Tile = layout.Tile; +pub const it = layout.it; +pub const L = layout.L; +pub const rowMajor = layout.rowMajor; +pub const colMajor = layout.colMajor; +pub const ordered = layout.ordered; +pub const tile = layout.tile; + +pub fn registerDialects(registry: *mlir.DialectRegistry) void { + inline for (.{ func, arith, scf, math, vector, memref, llvm, gpu, rocdl, ub, fly, fly_rocdl }) |dialect| dialect.registerDialect(registry); +} + +pub const FinishError = error{ InvalidMlir, OutOfMemory } || std.Io.Writer.Error; + +pub const Arch = enum { gfx942, gfx950 }; + +pub const Dim = gpu.Dim; + +pub const gpu_module_name = "zml_fly_kernels"; + +test { + std.testing.refAllDecls(@This()); + std.testing.refAllDecls(Builder); + std.testing.refAllDecls(Value); +} + +// ============================================================================= +// Value +// ============================================================================= + +pub const Value = struct { + inner: *const mlir.Value, + kernel: ?*Builder = null, + + pub const Kind = fly.TypeKind; + + pub fn type_(self: Value) *const mlir.Type { + return self.inner.type_(); + } + + fn kern(self: Value) *Builder { + return self.kernel orelse @panic("fly.Value has no owning Builder; use Builder.* helpers instead"); + } + + /// Derived, never stored: control_flow.zig builds Values with bare + /// `.{ .inner, .kernel }` literals. + pub fn kind(self: Value) Kind { + return fly.typeKind(self.type_()); + } + + pub fn shapeStatic(self: Value) IntTuple { + return self.kern().readIntTuple(fly.layoutLikeShape(self.type_())); + } + + /// Host value, not an op; see `emitSize`. + pub fn sizeStatic(self: Value) i64 { + return self.shapeStatic().product() orelse std.debug.panic("fly: `{f}` has a dynamic shape", .{self.type_()}); + } + + pub fn intTupleStatic(self: Value) IntTuple { + return self.kern().readIntTuple(fly.expect(self.type_(), .int_tuple)); + } + + pub fn elemDType(self: Value) DType { + const t = fly.elemType(self.type_()); + return dtypes.fromMlir(self.kern().ctx, t) orelse std.debug.panic("fly: unsupported element type `{f}`", .{t}); + } + + fn isI64(self: Value) bool { + const int_ty = self.type_().isA(mlir.IntegerType) orelse return false; + return int_ty.width() == 64; + } + + fn isFloatElem(self: Value) bool { + return self.scalarElemType().isFloat(); + } + + /// The element type of a vector, or the type itself. + fn scalarElemType(self: Value) *const mlir.Type { + if (self.type_().isA(mlir.VectorType)) |vec| return vec.shaped().elementType(); + return self.type_(); + } + + fn scalarDType(self: Value) DType { + const t = self.scalarElemType(); + return dtypes.fromMlir(self.kern().ctx, t) orelse std.debug.panic("fly: unsupported scalar type `{f}`", .{t}); + } + + // ------------------------------------------------------- layout algebra + // Every op below takes a Value or a comptime Layout/Tile/tuple literal, + // and FlyDSL infers the result type. + + fn unary(self: Value, comptime build: anytype) Value { + const k = self.kern(); + return k.emit(build(k.ctx, self.inner, k.loc())); + } + + fn binary(self: Value, comptime build: anytype, rhs: anytype) Value { + const k = self.kern(); + const r = k.toStatic(rhs); + return k.emit(build(k.ctx, self.inner, r.inner, k.loc())); + } + + pub fn flatDivide(self: Value, tiler: anytype) Value { + return self.binary(fly.ops.flat_divide, tiler); + } + pub fn logicalDivide(self: Value, tiler: anytype) Value { + return self.binary(fly.ops.logical_divide, tiler); + } + pub fn zippedDivide(self: Value, tiler: anytype) Value { + return self.binary(fly.ops.zipped_divide, tiler); + } + pub fn tiledDivide(self: Value, tiler: anytype) Value { + return self.binary(fly.ops.tiled_divide, tiler); + } + pub fn logicalProduct(self: Value, tiler: anytype) Value { + return self.binary(fly.ops.logical_product, tiler); + } + pub fn zippedProduct(self: Value, tiler: anytype) Value { + return self.binary(fly.ops.zipped_product, tiler); + } + pub fn tiledProduct(self: Value, tiler: anytype) Value { + return self.binary(fly.ops.tiled_product, tiler); + } + pub fn flatProduct(self: Value, tiler: anytype) Value { + return self.binary(fly.ops.flat_product, tiler); + } + pub fn blockedProduct(self: Value, tiler: anytype) Value { + return self.binary(fly.ops.blocked_product, tiler); + } + pub fn rakedProduct(self: Value, tiler: anytype) Value { + return self.binary(fly.ops.raked_product, tiler); + } + pub fn composition(self: Value, inner: anytype) Value { + return self.binary(fly.ops.composition, inner); + } + pub fn rightInverse(self: Value) Value { + return self.unary(fly.ops.right_inverse); + } + pub fn leftInverse(self: Value) Value { + return self.unary(fly.ops.left_inverse); + } + pub fn coalesce(self: Value) Value { + const k = self.kern(); + return k.emit(fly.ops.coalesce(k.ctx, self.inner, .{}, k.loc())); + } + pub fn complement(self: Value) Value { + const k = self.kern(); + return k.emit(fly.ops.complement(k.ctx, self.inner, .{}, k.loc())); + } + + /// `t[coord]` with a `null` somewhere; `coord` is a tuple literal + /// (`.{ null, null, bx, by }`, nesting allowed) or an int-tuple Value. + pub fn slice(self: Value, coord: anytype) Value { + const k = self.kern(); + const c = k.intTuple(coord); + return k.emit(fly.ops.slice(k.ctx, self.inner, c.inner, k.loc())); + } + pub fn dice(self: Value, coord: anytype) Value { + const k = self.kern(); + const c = k.intTuple(coord); + return k.emit(fly.ops.dice(k.ctx, self.inner, c.inner, k.loc())); + } + + /// The sub-tuple / sub-layout at a mode path. + pub fn get(self: Value, mode: []const i32) Value { + const k = self.kern(); + return k.emit(fly.ops.get(k.ctx, self.inner, .denseArray(k.ctx, .i32, mode), k.loc())); + } + pub fn selectModes(self: Value, indices: []const i32) Value { + const k = self.kern(); + return k.emit(fly.ops.select(k.ctx, self.inner, .denseArray(k.ctx, .i32, indices), k.loc())); + } + pub fn take(self: Value, begin: i32, end: i32) Value { + const k = self.kern(); + return k.emit(fly.ops.take(k.ctx, self.inner, .int(k.ctx, .i32, begin), .int(k.ctx, .i32, end), k.loc())); + } + pub fn group(self: Value, begin: i32, end: i32) Value { + const k = self.kern(); + return k.emit(fly.ops.group(k.ctx, self.inner, .int(k.ctx, .i32, begin), .int(k.ctx, .i32, end), k.loc())); + } + + /// Emits an op; `shapeStatic` reads the type instead. + pub fn emitShape(self: Value) Value { + return self.unary(fly.ops.get_shape); + } + pub fn emitStride(self: Value) Value { + return self.unary(fly.ops.get_stride); + } + pub fn emitLayout(self: Value) Value { + return self.unary(fly.ops.get_layout); + } + pub fn emitIter(self: Value) Value { + return self.unary(fly.ops.get_iter); + } + pub fn emitSize(self: Value) Value { + return self.unary(fly.ops.size); + } + pub fn cosize(self: Value) Value { + return self.unary(fly.ops.cosize); + } + pub fn productEach(self: Value) Value { + return self.unary(fly.ops.int_tuple_product_each); + } + + /// A register tensor shaped like `self`. + pub fn makeFragmentLike(self: Value, dtype: ?DType) Value { + const k = self.kern(); + const dtype_attr: ?*const mlir.Attribute = if (dtype) |d| .typeAttr(d.toMlir(k.ctx)) else null; + return k.emit(fly.ops.make_fragment_like(k.ctx, self.inner, .{ .dtype = dtype_attr }, k.loc())); + } + + pub fn elemLess(self: Value, rhs: anytype) Value { + const k = self.kern(); + const r = k.intTuple(rhs); + return k.emit(fly.ops.elem_less(k.ctx, self.inner, r.inner, .int(k.ctx, .i1), k.loc())); + } + + // ---------------------------------------------------------------- tensor access + + /// `t[coord]` with no `null` (a coord tensor yields an int tuple). + pub fn at(self: Value, coord: anytype) Value { + const k = self.kern(); + const c = k.intTuple(coord); + return k.emit(fly.ops.memref_load(k.ctx, self.inner, c.inner, k.loc())); + } + + pub fn set(self: Value, coord: anytype, value: Value) void { + const k = self.kern(); + const c = k.intTuple(coord); + k.emitNone(fly.ops.memref_store(k.ctx, value.inner, self.inner, c.inner, k.loc())); + } + + /// The whole register tensor as a `vector`. + pub fn load(self: Value) Value { + return self.unary(fly.ops.memref_load_vec); + } + + pub fn store(self: Value, vector_: Value) void { + const k = self.kern(); + k.emitNone(fly.ops.memref_store_vec(k.ctx, vector_.inner, self.inner, k.loc())); + } + + /// Store a splat vector of the tensor's static size. + pub fn fill(self: Value, value: anytype) void { + const k = self.kern(); + const n = self.sizeStatic(); + const dt = self.elemDType(); + const cst = k.constant(dt, value); + const vec_ty = mlir.Type.vector(&.{n}, dt.toMlir(k.ctx)); + const vec = k.emit(vector.broadcast(k.ctx, cst.inner, vec_ty, k.loc())); + self.store(vec); + } + + // ---------------------------------------------------------------- pointers + + pub fn addOffset(self: Value, offset: anytype) Value { + const k = self.kern(); + const o = k.intTuple(offset); + return k.emit(fly.ops.add_offset(k.ctx, self.inner, o.inner, k.loc())); + } + + /// Scalar load. + pub fn ptrLoad(self: Value) Value { + const k = self.kern(); + const elem = self.elemDType().toMlir(k.ctx); + return k.emit(fly.ops.ptr_load(k.ctx, self.inner, elem, k.loc())); + } + + pub fn ptrStore(self: Value, value: Value) void { + const k = self.kern(); + k.emitNone(fly.ops.ptr_store(k.ctx, value.inner, self.inner, k.loc())); + } + + /// A tensor over this pointer. + pub fn view(self: Value, lay: anytype) Value { + return self.binary(fly.ops.make_view, lay); + } + + // ---------------------------------------------------------------- arithmetic + + /// Broadcast a scalar against a vector operand. + fn coerceArith(self: Value, rhs: anytype) struct { Value, Value } { + const k = self.kern(); + const r: Value = if (@TypeOf(rhs) == Value) rhs else k.constantLike(rhs, self); + if (self.type_().eql(r.type_())) return .{ self, r }; + if (self.type_().isA(mlir.VectorType) != null and r.type_().isA(mlir.VectorType) == null) + return .{ self, k.emit(vector.broadcast(k.ctx, r.inner, self.type_(), k.loc())) }; + if (self.type_().isA(mlir.VectorType) == null and r.type_().isA(mlir.VectorType) != null) + return .{ k.emit(vector.broadcast(k.ctx, self.inner, r.type_(), k.loc())), r }; + std.debug.panic("fly: cannot combine {f} with {f}", .{ self.type_(), r.type_() }); + } + + pub fn add(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = self.coerceArith(rhs); + return k.emitFast(if (l.isFloatElem()) arith.addf(k.ctx, l.inner, r.inner, k.loc()) else arith.addi(k.ctx, l.inner, r.inner, k.loc())); + } + pub fn sub(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = self.coerceArith(rhs); + return k.emitFast(if (l.isFloatElem()) arith.subf(k.ctx, l.inner, r.inner, k.loc()) else arith.subi(k.ctx, l.inner, r.inner, k.loc())); + } + pub fn mul(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = self.coerceArith(rhs); + return k.emitFast(if (l.isFloatElem()) arith.mulf(k.ctx, l.inner, r.inner, k.loc()) else arith.muli(k.ctx, l.inner, r.inner, k.loc())); + } + pub fn div(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = self.coerceArith(rhs); + return k.emitFast(if (l.isFloatElem()) arith.divf(k.ctx, l.inner, r.inner, k.loc()) else arith.divsi(k.ctx, l.inner, r.inner, k.loc())); + } + /// Signed. + pub fn rem(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = self.coerceArith(rhs); + return k.emit(arith.remsi(k.ctx, l.inner, r.inner, k.loc())); + } + + pub fn bitAnd(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = self.coerceArith(rhs); + return k.emit(arith.andi(k.ctx, l.inner, r.inner, k.loc())); + } + pub fn bitOr(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = self.coerceArith(rhs); + return k.emit(arith.ori(k.ctx, l.inner, r.inner, k.loc())); + } + pub fn bitXor(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = self.coerceArith(rhs); + return k.emit(arith.xori(k.ctx, l.inner, r.inner, k.loc())); + } + pub fn shl(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = self.coerceArith(rhs); + return k.emit(arith.shli(k.ctx, l.inner, r.inner, k.loc())); + } + pub fn shrU(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = self.coerceArith(rhs); + return k.emit(arith.shrui(k.ctx, l.inner, r.inner, k.loc())); + } + pub fn divUnsigned(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = self.coerceArith(rhs); + return k.emit(arith.divui(k.ctx, l.inner, r.inner, k.loc())); + } + pub fn remUnsigned(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = self.coerceArith(rhs); + return k.emit(arith.remui(k.ctx, l.inner, r.inner, k.loc())); + } + pub fn cmpUnsigned(self: Value, pred: Cmp, rhs: anytype) Value { + const k = self.kern(); + const l, const r = self.coerceArith(rhs); + const p: arith.CmpIPredicate = switch (pred) { + .eq => .eq, + .ne => .ne, + .lt => .ult, + .le => .ule, + .gt => .ugt, + .ge => .uge, + }; + return k.emit(arith.cmpi(k.ctx, p, l.inner, r.inner, k.loc())); + } + + pub fn neg(self: Value) Value { + const k = self.kern(); + return k.emitFast(arith.negf(k.ctx, self.inner, k.loc())); + } + /// Propagates NaNs; use `maxNum` to select the numeric operand instead. + pub fn maximum(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = self.coerceArith(rhs); + return k.emitFast(arith.maximumf(k.ctx, l.inner, r.inner, k.loc())); + } + pub fn maxNum(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = self.coerceArith(rhs); + return k.emitFast(arith.maxnumf(k.ctx, l.inner, r.inner, k.loc())); + } + + /// Interpret integer operands and results as unsigned. + pub fn toUnsigned(self: Value, dtype: DType) Value { + const k = self.kern(); + const src = self.scalarDType(); + if (src == dtype) return self; + if (src.isFloat() and dtype.isFloat()) return self.to(dtype); + const result = self.withElemType(dtype.toMlir(k.ctx)); + const op = if (src.isFloat()) + arith.fptoui(k.ctx, self.inner, result, k.loc()) + else if (dtype.isFloat()) + arith.uitofp(k.ctx, self.inner, result, k.loc()) + else if (dtype.bitWidth() > src.bitWidth()) + arith.extui(k.ctx, self.inner, result, k.loc()) + else + arith.trunci(k.ctx, self.inner, result, k.loc()); + return k.emit(op); + } + + pub fn splat(self: Value, n: i64) Value { + const k = self.kern(); + std.debug.assert(self.type_().isA(mlir.VectorType) == null and n > 0); + return k.emit(vector.broadcast(k.ctx, self.inner, .vector(&.{n}, self.type_()), k.loc())); + } + + /// Preserve total bits, adjusting vector length to the destination element. + pub fn bitCast(self: Value, dtype: DType) Value { + const k = self.kern(); + const src = self.scalarDType(); + if (self.type_().isA(mlir.VectorType)) |vec| { + const n = @divExact(vec.dimension(0) * @as(i64, src.bitWidth()), dtype.bitWidth()); + return k.emit(vector.bitcast(k.ctx, self.inner, .vector(&.{n}, dtype.toMlir(k.ctx)), k.loc())); + } + std.debug.assert(src.bitWidth() == dtype.bitWidth()); + return k.emit(arith.bitcast(k.ctx, self.inner, dtype.toMlir(k.ctx), k.loc())); + } + + pub fn insert(self: Value, index: i64, value: Value) Value { + const k = self.kern(); + return k.emit(vector.insert(k.ctx, value.inner, self.inner, &.{index}, &.{}, k.loc())); + } + + pub fn shuffleVector(self: Value, rhs: Value, indices: []const i64) Value { + const k = self.kern(); + return k.emit(vector.shuffle(k.ctx, self.inner, rhs.inner, indices, .vector(&.{@intCast(indices.len)}, self.scalarElemType()), k.loc())); + } + + fn withElemType(self: Value, elem: *const mlir.Type) *const mlir.Type { + if (self.type_().isA(mlir.VectorType)) |vec| return .vector(&.{vec.dimension(0)}, elem); + return elem; + } + + pub fn toI32(self: Value) Value { + const k = self.kern(); + return k.emit(arith.index_cast(k.ctx, self.inner, .int(k.ctx, .i32), k.loc())); + } + + /// 1 for a scalar. + pub fn vectorLen(self: Value) i64 { + const vec = self.type_().isA(mlir.VectorType) orelse return 1; + return vec.dimension(0); + } + + /// Numeric conversion, scalar or vector. `i1` is treated as unsigned, so a + /// predicate converts to 1, not -1; equal-width float pairs (f16 <-> bf16, + /// the fp8 spellings) go through `arith.convertf`, which is the only op + /// that accepts them. + pub fn to(self: Value, dtype: DType) Value { + const k = self.kern(); + const src = self.scalarDType(); + if (src == dtype) return self; + const result = self.withElemType(dtype.toMlir(k.ctx)); + const unsigned_src = src == .i1; + const op = if (src.isFloat() and dtype.isFloat()) + (if (dtype.bitWidth() > src.bitWidth()) + arith.extf(k.ctx, self.inner, result, k.loc()) + else if (dtype.bitWidth() < src.bitWidth()) + arith.truncf(k.ctx, self.inner, result, k.loc()) + else + arith.convertf(k.ctx, self.inner, result, .{}, k.loc())) + else if (src.isFloat()) + arith.fptosi(k.ctx, self.inner, result, k.loc()) + else if (dtype.isFloat()) + (if (unsigned_src) arith.uitofp(k.ctx, self.inner, result, k.loc()) else arith.sitofp(k.ctx, self.inner, result, k.loc())) + else if (dtype.bitWidth() > src.bitWidth()) + (if (unsigned_src) arith.extui(k.ctx, self.inner, result, k.loc()) else arith.extsi(k.ctx, self.inner, result, k.loc())) + else + arith.trunci(k.ctx, self.inner, result, k.loc()); + return k.emit(op); + } + + pub const Cmp = enum { eq, ne, lt, le, gt, ge }; + + /// Signed integer comparison against a Value or an integer literal; a + /// scalar broadcasts against a vector. + pub fn cmp(self: Value, pred: Cmp, rhs: anytype) Value { + const k = self.kern(); + const l, const r = self.coerceArith(rhs); + const p: arith.CmpIPredicate = switch (pred) { + .eq => .eq, + .ne => .ne, + .lt => .slt, + .le => .sle, + .gt => .sgt, + .ge => .sge, + }; + return k.emit(arith.cmpi(k.ctx, p, l.inner, r.inner, k.loc())); + } + + /// Ordered. + pub fn cmpf(self: Value, pred: Cmp, rhs: Value) Value { + const k = self.kern(); + const p: arith.CmpFPredicate = switch (pred) { + .eq => .oeq, + .ne => .one, + .lt => .olt, + .le => .ole, + .gt => .ogt, + .ge => .oge, + }; + return k.emit(arith.cmpf(k.ctx, p, self.inner, rhs.inner, k.loc())); + } + + pub fn lt(self: Value, rhs: anytype) Value { + return self.compare(.lt, rhs); + } + + pub fn le(self: Value, rhs: anytype) Value { + return self.compare(.le, rhs); + } + + pub fn gt(self: Value, rhs: anytype) Value { + return self.compare(.gt, rhs); + } + + pub fn ge(self: Value, rhs: anytype) Value { + return self.compare(.ge, rhs); + } + + pub fn eq(self: Value, rhs: anytype) Value { + return self.compare(.eq, rhs); + } + + pub fn ne(self: Value, rhs: anytype) Value { + return self.compare(.ne, rhs); + } + + /// `cmpf` for floats, `cmp` (signed) for integers, `index` included. + fn compare(self: Value, pred: Cmp, rhs: anytype) Value { + if (!self.isFloatElem()) return self.cmp(pred, rhs); + const l, const r = self.coerceArith(rhs); + return l.cmpf(pred, r); + } + + pub fn select(self: Value, a: Value, b: Value) Value { + const k = self.kern(); + return k.emit(arith.select(k.ctx, self.inner, a.inner, b.inner, k.loc())); + } + + /// Lane `k` reads lane `k ^ offset`, within `width` lanes. + pub fn shuffleXor(self: Value, offset: i32, width: i32) Value { + return self.shuffle(.xor, offset, width); + } + + pub const ShuffleMode = gpu.ShuffleMode; + + pub fn shuffle(self: Value, mode: ShuffleMode, offset: anytype, width: i32) Value { + const k = self.kern(); + const off: Value = if (@TypeOf(offset) == Value) offset else k.constant(.i32, offset); + const wid = k.constant(.i32, width); + return k.emit(gpu.shuffle(k.ctx, self.inner, off.inner, wid.inner, mode, k.loc())); + } + + pub const ReduceKind = enum { add, mul, minf, maxf, minsi, maxsi, and_, or_, xor }; + + /// Reduce a vector to a scalar. + pub fn reduce(self: Value, reduce_kind: ReduceKind) Value { + const k = self.kern(); + const result = self.scalarElemType(); + const ck: vector.CombiningKind = switch (reduce_kind) { + .add => .add, + .mul => .mul, + .minf => .minnumf, + .maxf => .maxnumf, + .minsi => .minsi, + .maxsi => .maxsi, + .and_ => .@"and", + .or_ => .@"or", + .xor => .xor, + }; + const op = vector.reduction(k.ctx, ck, self.inner, result, .{}, k.loc()); + return k.emitFast(op); + } + + /// Static position. + pub fn extract(self: Value, i: i64) Value { + const k = self.kern(); + return k.emit(vector.extract(k.ctx, self.inner, &.{i}, &.{}, self.scalarElemType(), k.loc())); + } +}; + +// ============================================================================= +// Tiled copy / MMA handles +// ============================================================================= + +/// A `!fly.tiled_copy` plus the tile it was built with. +pub const TiledCopy = struct { + value: Value, + tile: Value, + + pub fn getSlice(self: TiledCopy, thr_idx: Value) ThrCopy { + return .{ .tiled_copy = self.value, .thr = self.value.kern().intTuple(thr_idx) }; + } + + pub fn tileMN(self: TiledCopy) Value { + return self.tile; + } +}; + +/// Per-thread slice of a TiledCopy. +pub const ThrCopy = struct { + tiled_copy: Value, + thr: Value, + + pub fn partitionS(self: ThrCopy, src: Value) Value { + const k = self.tiled_copy.kern(); + return k.emit(fly.ops.tiled_copy_partition_src(k.ctx, self.tiled_copy.inner, src.inner, self.thr.inner, k.loc())); + } + + pub fn partitionD(self: ThrCopy, dst: Value) Value { + const k = self.tiled_copy.kern(); + return k.emit(fly.ops.tiled_copy_partition_dst(k.ctx, self.tiled_copy.inner, dst.inner, self.thr.inner, k.loc())); + } + + pub fn retile(self: ThrCopy, t: Value) Value { + const k = self.tiled_copy.kern(); + return k.emit(fly.ops.tiled_copy_retile(k.ctx, self.tiled_copy.inner, t.inner, k.loc())); + } +}; + +pub const TiledMma = struct { + value: Value, + + pub fn getSlice(self: TiledMma, thr_idx: Value) ThrMma { + return .{ .tiled_mma = self.value, .thr = self.value.kern().intTuple(thr_idx) }; + } + + pub const FragmentOpts = struct { + stages: ?i32 = null, + }; + + fn makeFragment(self: TiledMma, operand: fly.MmaOperand, input: Value, opts: FragmentOpts) Value { + const k = self.value.kern(); + const operand_id = Builder.must(fly.attributes.mmaOperandAttr(k.ctx, operand)); + const stages: ?*const mlir.Attribute = if (opts.stages) |st| .int(k.ctx, .i32, st) else null; + return k.emit(fly.ops.mma_make_fragment(k.ctx, self.value.inner, input.inner, operand_id, .{ .stages = stages }, k.loc())); + } + + /// Per-thread register fragment for the A operand. + pub fn makeFragmentA(self: TiledMma, a: Value) Value { + return self.makeFragment(.a, a, .{}); + } + pub fn makeFragmentB(self: TiledMma, b: Value) Value { + return self.makeFragment(.b, b, .{}); + } + pub fn makeFragmentC(self: TiledMma, c: Value) Value { + return self.makeFragment(.c, c, .{}); + } + pub fn makeFragmentAOpts(self: TiledMma, a: Value, opts: FragmentOpts) Value { + return self.makeFragment(.a, a, opts); + } + pub fn makeFragmentBOpts(self: TiledMma, b: Value, opts: FragmentOpts) Value { + return self.makeFragment(.b, b, opts); + } +}; + +pub const ThrMma = struct { + tiled_mma: Value, + thr: Value, + + fn partition(self: ThrMma, operand: fly.MmaOperand, t: Value) Value { + const k = self.tiled_mma.kern(); + const operand_id = Builder.must(fly.attributes.mmaOperandAttr(k.ctx, operand)); + return k.emit(fly.ops.tiled_mma_partition(k.ctx, self.tiled_mma.inner, t.inner, self.thr.inner, operand_id, k.loc())); + } + + pub fn partitionA(self: ThrMma, a: Value) Value { + return self.partition(.a, a); + } + pub fn partitionB(self: ThrMma, b: Value) Value { + return self.partition(.b, b); + } + pub fn partitionC(self: ThrMma, c: Value) Value { + return self.partition(.c, c); + } +}; + +// ============================================================================= +// Builder +// ============================================================================= + +pub const Builder = struct { + allocator: std.mem.Allocator, + arena: std.heap.ArenaAllocator, + ctx: *mlir.Context, + /// `builtin.module attributes {gpu.container_module}`. + module: *mlir.Module, + gpu_module: *mlir.Operation, + gpu_body: *mlir.Block, + name: []const u8, + func_op: ?*mlir.Operation = null, + entry_block: ?*mlir.Block = null, + block_stack: common.BlockStack = .{}, + arg_specs: []const ArgSpec = &.{}, + /// The `!fly.memref` view for tensor arguments, the raw `!fly.ptr` otherwise. + arg_values: []Value = &.{}, + target: Arch = .gfx942, + fast_math: bool = false, + + pub const ArgSpec = struct { + name: []const u8, + dtype: DType, + space: fly.AddressSpace = .global, + /// Row-major tensor argument; null for a bare pointer. + dims: ?[]const i64 = null, + }; + + pub fn open(allocator: std.mem.Allocator, ctx: *mlir.Context, name: []const u8) !Builder { + const loc_: *const mlir.Location = .unknown(ctx); + const module: *mlir.Module = .init(loc_); + errdefer module.deinit(); + module.operation().setAttribute(gpu.containerModule(ctx)); + + const gpu_body = mlir.Block.init(&.{}, &.{}); + const gpu_module = gpu.module(ctx, gpu_module_name, gpu_body, loc_); + _ = gpu_module.appendTo(module.body()); + + var arena: std.heap.ArenaAllocator = .init(allocator); + errdefer arena.deinit(); + + return .{ + .allocator = allocator, + .arena = arena, + .ctx = ctx, + .module = module, + .gpu_module = gpu_module, + .gpu_body = gpu_body, + .name = name, + }; + } + + pub fn deinit(self: *Builder) void { + self.module.deinit(); + self.arena.deinit(); + } + + /// Arguments by name. Each field is `.{ .ptr = dtype }` or + /// `.{ .tensor = .{ .dtype = ..., .dims = &.{ ... } } }`. + pub fn declareArgs(self: *Builder, spec: anytype) !common.NamedArgs(@TypeOf(spec), Value) { + const Spec = @TypeOf(spec); + const fields = @typeInfo(Spec).@"struct".field_names; + var specs: [fields.len]ArgSpec = undefined; + inline for (fields, 0..) |f, i| { + const raw = @field(spec, f); + const variant = @typeInfo(@TypeOf(raw)).@"struct".field_names[0]; + const inner = @field(raw, variant); + specs[i] = if (comptime std.mem.eql(u8, variant, "ptr")) + .{ .name = f, .dtype = inner } + else if (comptime std.mem.eql(u8, variant, "tensor")) + .{ .name = f, .dtype = inner.dtype, .dims = inner.dims, .space = if (@hasField(@TypeOf(inner), "space")) inner.space else .global } + else + @compileError("fly.declareArgs: argument `" ++ f ++ "` must be `.{ .ptr = dtype }` or `.{ .tensor = .{ .dtype, .dims } }`"); + } + try self.declareArgsLow(&specs); + var named: common.NamedArgs(Spec, Value) = undefined; + inline for (fields, 0..) |f, i| @field(named, f) = self.arg(i); + return named; + } + + /// One `!fly.ptr` per spec, in order; tensor + /// arguments are bridged to `!fly.memref` views. + pub fn declareArgsLow(self: *Builder, specs: []const ArgSpec) !void { + std.debug.assert(self.entry_block == null); + const ctx = self.ctx; + const scratch = self.arena.allocator(); + self.arg_specs = try scratch.dupe(ArgSpec, specs); + + const arg_types = try scratch.alloc(*const mlir.Type, specs.len); + const arg_locs = try scratch.alloc(*const mlir.Location, specs.len); + for (arg_types, arg_locs, specs) |*ty_, *a_loc, a| { + ty_.* = self.ptrType(a.dtype.storageElem(), self.addressSpaceAttr(a.space)); + a_loc.* = self.loc().named(ctx, a.name); + } + + const entry = mlir.Block.init(arg_types, arg_locs); + const func_op = gpu.func(ctx, .{ + .name = self.name, + .args = arg_types, + .block = entry, + .location = self.loc(), + }); + _ = func_op.appendTo(self.gpu_body); + self.func_op = func_op; + self.entry_block = entry; + self.block_stack.push(self.arena.allocator(), entry); + + // The view keeps the storage element type: recasting a predicate + // argument back to `!fly.ptr` would make a copy atom's element + // count `bits / 1` while it still moves `bits / 8` bytes. + self.arg_values = try scratch.alloc(Value, specs.len); + for (specs, 0..) |a, i| { + const raw = self.rawArg(i); + self.arg_values[i] = if (a.dims) |dims| raw.view(self.staticRowMajor(dims)) else raw; + } + } + + pub fn rawArg(self: *Builder, i: usize) Value { + const eb = self.entry_block orelse @panic("Builder.rawArg called before declareArgs"); + return .{ .inner = eb.argument(i), .kernel = self }; + } + + /// A tensor view when the spec has dims, else the raw pointer. + pub fn arg(self: *Builder, i: usize) Value { + return self.arg_values[i]; + } + + pub fn pushBlock(self: *Builder, b: *mlir.Block) void { + self.block_stack.push(self.arena.allocator(), b); + } + + pub fn popBlock(self: *Builder) void { + self.block_stack.pop(); + } + + pub fn currentBlock(self: *Builder) *mlir.Block { + return self.block_stack.current(); + } + + pub fn loc(self: *const Builder) *const mlir.Location { + return .unknown(self.ctx); + } + + pub fn emit(self: *Builder, op: *mlir.Operation) Value { + _ = op.appendTo(self.currentBlock()); + return .{ .inner = op.result(0), .kernel = self }; + } + + pub fn emitNone(self: *Builder, op: *mlir.Operation) void { + _ = op.appendTo(self.currentBlock()); + } + + // ---------------------------------------------------------------- types + + /// Arena-allocated. + pub fn readIntTuple(self: *Builder, tuple_ty: *const fly.types.IntTupleType) IntTuple { + if (tuple_ty.isLeaf()) { + return switch (tuple_ty.leaf()) { + .static => |v| IntTuple.static(v), + .dynamic, .basis => IntTuple.dyn, + .none => IntTuple.star, + }; + } + const n = tuple_ty.numElements(); + const elems = self.alloc(IntTuple, n); + for (0..n) |i| elems[i] = self.readIntTuple(tuple_ty.element(i)); + return .{ .tup = elems }; + } + + fn alloc(self: *Builder, comptime T: type, n: usize) []T { + return self.arena.allocator().alloc(T, n) catch @panic("fly: OOM"); + } + + /// The shape and stride of a static `!fly.layout`, as host literals. + pub fn readLayout(self: *Builder, layout_ty: *const mlir.Type) Layout { + const lay = fly.expect(layout_ty, .layout); + const shape = lay.shape(); + const stride = lay.stride(); + if (!shape.isStatic() or !stride.isStatic()) { + std.debug.panic("fly: `{f}` is not static", .{layout_ty}); + } + return .{ .shape = self.readIntTuple(shape), .stride = self.readIntTuple(stride) }; + } + + /// A fly type or attribute is rejected only on malformed input, which is + /// a bug in the kernel, not a runtime condition. + pub fn must(result: anytype) @typeInfo(@TypeOf(result)).error_union.payload { + return result catch |err| std.debug.panic("fly: cannot build type ({})", .{err}); + } + + pub fn scalarTy(self: *const Builder, dtype: DType) *const mlir.Type { + return dtype.toMlir(self.ctx); + } + + pub fn intTupleType(self: *Builder, t: IntTuple) *const mlir.Type { + return must(fly.types.IntTupleType.get(self.ctx, .{ .attr = must(t.toAttr(self.ctx)) })).type_(); + } + + pub fn layoutType(self: *Builder, lay: Layout) *const mlir.Type { + return must(fly.types.LayoutType.get(self.ctx, .{ .attr = must(lay.toAttr(self.ctx)) })).type_(); + } + + pub fn tileType(self: *Builder, t: Tile) *const mlir.Type { + return must(fly.types.TileType.get(self.ctx, .{ .attr = must(t.toAttr(self.ctx)) })).type_(); + } + + pub fn ptrType(self: *Builder, dtype: DType, space: *const mlir.Attribute) *const mlir.Type { + return must(fly.types.PointerType.get(self.ctx, .{ .elemTy = dtype.toMlir(self.ctx), .addressSpace = space })).type_(); + } + + fn addressSpaceAttr(self: *Builder, space: fly.AddressSpace) *const mlir.Attribute { + return must(fly.attributes.AddressSpaceAttr.get(self.ctx, .{ .addressSpace = space })).attribute(); + } + + /// `!fly.memref`. + pub fn memRefType(self: *Builder, dtype: DType, space: fly.AddressSpace, lay: Layout) *const mlir.Type { + return must(fly.types.MemRefType.get(self.ctx, .{ + .elemTy = dtype.toMlir(self.ctx), + .addressSpace = self.addressSpaceAttr(space), + .layout = must(lay.toAttr(self.ctx)).attribute(), + })).type_(); + } + + // ---------------------------------------------------------------- statics + + /// `fly.static` of an already-built type. + pub fn staticTy(self: *Builder, t: *const mlir.Type) Value { + return self.emit(fly.ops.static(self.ctx, t, self.loc())); + } + + pub fn static(self: *Builder, v: anytype) Value { + const T = @TypeOf(v); + return self.staticTy(if (T == Layout) + self.layoutType(v) + else if (T == Tile) + self.tileType(v) + else if (T == IntTuple) + self.intTupleType(v) + else + @compileError("fly.static: expected a Layout, Tile or IntTuple, got " ++ @typeName(T))); + } + + pub fn staticRowMajor(self: *Builder, dims: []const i64) Value { + if (dims.len == 1) return self.static(Layout{ .shape = .static(dims[0]), .stride = .static(1) }); + const shape = self.alloc(IntTuple, dims.len); + const stride = self.alloc(IntTuple, dims.len); + var acc: i64 = 1; + var i: usize = dims.len; + while (i > 0) { + i -= 1; + shape[i] = .static(dims[i]); + stride[i] = .static(acc); + acc *= dims[i]; + } + return self.static(Layout{ .shape = .{ .tup = shape }, .stride = .{ .tup = stride } }); + } + + /// A Value passes through; a layout literal becomes a `fly.static`, and a + /// bare tuple literal (`.{ 64, 8 }`) is a tile. + pub fn toStatic(self: *Builder, v: anytype) Value { + const T = @TypeOf(v); + if (T == Value) return v; + if (T == Layout or T == Tile or T == IntTuple) return self.static(v); + if (@typeInfo(T) == .@"struct" and @typeInfo(T).@"struct".is_tuple) return self.static(tile(v)); + @compileError("fly: expected a Value, Layout, Tile, IntTuple or tuple literal, got " ++ @typeName(T)); + } + + /// A (possibly nested) int tuple from integers (static leaves), `null` + /// (`*`), `IntTuple` literals and `Value`s (dynamic leaves, passed as + /// operands). An `!fly.int_tuple` Value passes through. + pub fn intTuple(self: *Builder, elems: anytype) Value { + if (@TypeOf(elems) == Value and elems.kind() == .int_tuple) return elems; + var operands: stdx.BoundedArray(*const mlir.Value, 32) = .empty; + const spec = self.intTupleSpec(&operands, elems); + const t = self.intTupleType(spec); + return self.emit(fly.ops.make_int_tuple(self.ctx, operands.constSlice(), t, self.loc())); + } + + /// `intTuple`, spelled for coordinates. + pub const coord = intTuple; + + /// The literal for `intTuple`, collecting each dynamic leaf as an operand. + fn intTupleSpec(self: *Builder, operands: *stdx.BoundedArray(*const mlir.Value, 32), v: anytype) IntTuple { + const T = @TypeOf(v); + if (T == Value) { + if (v.kind() == .int_tuple) { + std.debug.panic("fly.intTuple: nested !fly.int_tuple values are not supported; pass scalars", .{}); + } + operands.appendAssumeCapacity(v.inner); + return if (v.isI64()) .dynamic(64, 1) else .dyn; + } + if (T == IntTuple) return v; + if (T == @TypeOf(null)) return .star; + switch (@typeInfo(T)) { + .comptime_int, .int => return .static(@intCast(v)), + .@"struct" => |s| { + if (!s.is_tuple) @compileError("fly.intTuple: expected an integer, null, Value, IntTuple or tuple, got " ++ @typeName(T)); + const elems = self.alloc(IntTuple, s.field_names.len); + inline for (s.field_names, 0..) |f, i| elems[i] = self.intTupleSpec(operands, @field(v, f)); + return .{ .tup = elems }; + }, + else => @compileError("fly.intTuple: expected an integer, null, Value, IntTuple or tuple, got " ++ @typeName(T)), + } + } + + // ---------------------------------------------------------------- scalars & ids + + /// `value` as a constant of `dtype`; a Value passes through. An integer is + /// signed or an unsigned bit pattern of `dtype`'s width; anything else + /// panics. Only the representation `dtype` needs is computed: a float + /// constant outside i64 range (inf, floatMax) must not be forced through + /// `@intFromFloat`, which traps. + pub fn constant(self: *Builder, dtype: DType, value: anytype) Value { + if (@TypeOf(value) == Value) return value; + const is_float_lit = switch (@typeInfo(@TypeOf(value))) { + .float, .comptime_float => true, + else => false, + }; + return switch (dtype) { + .f16, .bf16, .f32, .f64 => blk: { + const f: f64 = if (is_float_lit) @floatCast(value) else @floatFromInt(value); + break :blk self.emit(switch (dtype) { + .f16 => arith.constant_float(self.ctx, f, .f16, self.loc()), + .bf16 => arith.constant_float(self.ctx, f, .bf16, self.loc()), + .f32 => arith.constant_float(self.ctx, f, .f32, self.loc()), + else => arith.constant_float(self.ctx, f, .f64, self.loc()), + }); + }, + .i1, .i8, .i16, .i32, .i64 => blk: { + const i: i128 = if (is_float_lit) @intFromFloat(value) else value; + break :blk self.emit(arith.constant_int(self.ctx, intBits(i, @intCast(dtype.bitWidth()), @tagName(dtype)), dtype.toMlir(self.ctx), self.loc())); + }, + else => std.debug.panic("fly.constant: no constant for {s}", .{@tagName(dtype)}), + }; + } + + /// `value` as the `i64` an integer constant of `bits` bits is built from. + /// It may be signed or an unsigned bit pattern, so it must lie in + /// [-2^(bits-1), 2^bits). + fn intBits(value: i128, bits: u7, dtype: []const u8) i64 { + if (value < -(@as(i128, 1) << (bits - 1)) or value >= @as(i128, 1) << bits) + std.debug.panic("Builder.constant: {d} does not fit {s}", .{ value, dtype }); + return @truncate(value); + } + + /// A Zig scalar as a constant: a comptime int is `i32`, or `i64` when it + /// doesn't fit; a runtime int is `i64` past 32 bits, else `i32`; a float + /// keeps its width, and a comptime float is `f32`. A Value passes through. + pub fn lift(self: *Builder, value: anytype) Value { + const T = @TypeOf(value); + if (T == Value) return value; + return switch (@typeInfo(T)) { + .comptime_int => self.constant(if (value >= std.math.minInt(i32) and value <= std.math.maxInt(i32)) .i32 else .i64, value), + .int => |info| self.constant(if (info.bits > 32) .i64 else .i32, value), + .comptime_float => self.constant(.f32, value), + .float => |info| switch (info.bits) { + 16 => self.constant(.f16, value), + 32 => self.constant(.f32, value), + 64 => self.constant(.f64, value), + else => @compileError("Builder.lift: unsupported float bitwidth"), + }, + else => @compileError("Builder.lift: unsupported type " ++ @typeName(T)), + }; + } + + pub fn constIndex(self: *Builder, value: i64) Value { + return self.emit(arith.constant_index(self.ctx, value, self.loc())); + } + + fn idI32(self: *Builder, op: *mlir.Operation) Value { + return self.emit(op).toI32(); + } + + /// Already cast to `i32`: `fly.make_int_tuple` takes only i32/i64. + pub fn threadId(self: *Builder, dim: Dim) Value { + return self.idI32(gpu.thread_id(self.ctx, dim, self.loc())); + } + pub fn blockId(self: *Builder, dim: Dim) Value { + return self.idI32(gpu.block_id(self.ctx, dim, self.loc())); + } + pub fn blockDim(self: *Builder, dim: Dim) Value { + return self.idI32(gpu.block_dim(self.ctx, dim, self.loc())); + } + pub fn gridDim(self: *Builder, dim: Dim) Value { + return self.idI32(gpu.grid_dim(self.ctx, dim, self.loc())); + } + + pub fn barrier(self: *Builder) void { + self.emitNone(gpu.barrier(self.ctx, self.loc())); + } + + /// Tag float arithmetic emitted from now on with `fastmath`. + pub fn setFastMath(self: *Builder, on: bool) void { + self.fast_math = on; + } + + pub fn emitFast(self: *Builder, op: *mlir.Operation) Value { + if (self.fast_math and op.numResults() > 0 and (Value{ .inner = op.result(0), .kernel = self }).isFloatElem()) { + op.setAttribute(arith.fastmath(self.ctx, .fast) catch unreachable); + } + return self.emit(op); + } + + pub fn rsqrt(self: *Builder, x: Value) Value { + return self.emitFast(math.rsqrt(self.ctx, x.inner, self.loc())); + } + pub fn sqrt(self: *Builder, x: Value) Value { + return self.emitFast(math.sqrt(self.ctx, x.inner, self.loc())); + } + pub fn exp(self: *Builder, x: Value) Value { + return self.emitFast(math.exp(self.ctx, x.inner, self.loc())); + } + + pub fn log(self: *Builder, x: Value) Value { + return self.emitFast(math.log(self.ctx, x.inner, self.loc())); + } + pub fn fma(self: *Builder, a: Value, b: Value, c: Value) Value { + return self.emitFast(math.fma(self.ctx, a.inner, b.inner, c.inner, self.loc())); + } + + /// A matrix atom with SSA operands; unlike `gemm`, this needs no register + /// allocas and can carry its accumulator through control flow. + pub fn mmaAtomCall(self: *Builder, atom: Value, a: Value, b: Value, c: Value) Value { + return self.emit(fly.ops.mma_atom_call_ssa(self.ctx, atom.inner, &.{a.inner}, &.{b.inner}, c.inner, &.{c.type_()}, .{}, self.loc())); + } + + /// AMD byte permutation. ROCDL has no corresponding operation at our pin. + pub fn perm(self: *Builder, hi: Value, lo: Value, selector: Value) Value { + return self.emit(llvm.inline_asm( + self.ctx, + "v_perm_b32 $0, $1, $2, $3", + &.{ hi.inner, lo.inner, selector.inner }, + .{ .constraints = "=v,v,v,v", .result_type = .int(self.ctx, .i32) }, + self.loc(), + )); + } + + /// CDNA3 packed FP8 conversion, selecting two bytes from the low/high word. + pub fn cvtPkF32Fp8(self: *Builder, bits: Value, high: bool) Value { + return self.emit(rocdl.cvtPkF32Fp8(self.ctx, bits.inner, high, self.loc())); + } + + pub const AtomicScope = enum { workgroup, agent }; + + /// Monotonic i32 addition to a global or shared pointer. Callers provide + /// barriers or kernel dependencies before reading the completed counters. + pub fn ptrAtomicAdd(self: *Builder, ptr: Value, value: Value, scope: AtomicScope) Value { + std.debug.assert(ptr.elemDType() == .i32 and value.type_().eql(DType.i32.toMlir(self.ctx))); + const space: i32 = switch (scope) { + .workgroup => 3, + .agent => 1, + }; + const llvm_ptr = self.emit(fly.ops.to_llvm_ptr(self.ctx, ptr.inner, .int(self.ctx, .i32, space), self.loc())); + return self.emit(llvm.atomicrmw(self.ctx, llvm_ptr.inner, value.inner, .add, .monotonic, .{ .syncscope = @tagName(scope), .alignment = 4 }, self.loc())); + } + + /// Static LDS, as an `n:1` tensor in shared memory. Each call is its own + /// `fly.make_ptr {allocBytes, allocAlign}` leaf, which the plugin lowers + /// to one LLVM global, so `shared_mem_bytes` may stay 0. + pub fn sharedArray(self: *Builder, dtype: DType, n: i64, alignment: i32) Value { + const bytes: i64 = @divExact(@as(i64, dtype.bitWidth()) * n, 8); + const raw_ty = must(fly.types.PointerType.get(self.ctx, .{ + .elemTy = .int(self.ctx, .i8), + .addressSpace = self.addressSpaceAttr(.shared), + .alignment = must(fly.attributes.AlignAttr.get(self.ctx, .{ .alignment = alignment })), + })).type_(); + const dict: *const mlir.Attribute = .dict(self.ctx, &.{ + .named(self.ctx, "allocBytes", .int(self.ctx, .i64, bytes)), + .named(self.ctx, "allocAlign", .int(self.ctx, .i64, alignment)), + }); + const raw = self.emit(fly.ops.make_ptr(self.ctx, &.{}, raw_ty, .{ .dict_attrs = dict }, self.loc())); + const typed = self.recast(raw, dtype); + return typed.view(self.static(Layout{ .shape = .static(n), .stride = .static(1) })); + } + + /// A single atom-sized copy. + pub fn copyAtomCall(self: *Builder, atom: Value, src: Value, dst: Value) void { + self.emitNone(fly.ops.copy_atom_call(self.ctx, atom.inner, src.inner, dst.inner, .{}, self.loc())); + } + + /// One atom-sized load straight into a `vector`. + /// + /// Required over an rmem tensor + `copyAtomCall` + `load` in any kernel + /// with control flow: the plugin lowers `scf` to `cf` before the Fly + /// passes, and Fly's register-alloca promotion only handles one block. + pub fn copyAtomLoad(self: *Builder, atom: Value, src: Value) Value { + const n = src.sizeStatic(); + const dt = src.elemDType(); + const vec_ty = mlir.Type.vector(&.{n}, dt.toMlir(self.ctx)); + return self.emit(fly.ops.copy_atom_call_ssa(self.ctx, atom.inner, src.inner, &.{vec_ty}, .{}, self.loc())); + } + + /// One atom-sized store of a `vector`; see `copyAtomLoad`. + pub fn copyAtomStore(self: *Builder, atom: Value, vec: Value, dst: Value) void { + self.emitNone(fly.ops.copy_atom_call_ssa(self.ctx, atom.inner, vec.inner, &.{}, .{ .dst = dst.inner }, self.loc())); + } + + // ---------------------------------------------------------------- tensors + + pub fn makeView(self: *Builder, iter: Value, lay: anytype) Value { + _ = self; + return iter.view(lay); + } + + /// A coordinate tensor: value == logical coord. + pub fn identity(self: *Builder, shape: anytype) Value { + const shp = self.intTuple(shape); + const lay = self.emit(fly.ops.make_identity_layout(self.ctx, shp.inner, self.loc())); + const rank_ = shp.intTupleStatic().rank(); + const zero: IntTuple = if (rank_ == 1) .static(0) else blk: { + const zeros = self.alloc(IntTuple, rank_); + for (zeros) |*z| z.* = .static(0); + break :blk .{ .tup = zeros }; + }; + const base_ty = self.intTupleType(zero); + const base = self.emit(fly.ops.make_int_tuple(self.ctx, &.{}, base_ty, self.loc())); + return base.view(lay); + } + + /// A register tensor; see `copyAtomLoad` before using one in a kernel + /// with control flow. + pub fn rmemTensor(self: *Builder, comptime lay: Layout, dtype: DType) Value { + const lay_v = self.static(lay); + const t = self.memRefType(dtype, .register, lay); + return self.emit(fly.ops.memref_alloca(self.ctx, lay_v.inner, t, self.loc())); + } + + /// `rmemTensor` with a runtime `Layout`. + pub fn rmemTensorRuntime(self: *Builder, lay: Layout, dtype: DType) Value { + const lay_v = self.static(lay); + const t = self.memRefType(dtype, .register, lay); + return self.emit(fly.ops.memref_alloca(self.ctx, lay_v.inner, t, self.loc())); + } + + /// Reinterpret a pointer as another element type, keeping the address + /// space, alignment and swizzle. + pub fn recast(self: *Builder, ptr: Value, dtype: DType) Value { + const t = must(fly.ptrWithElem(self.ctx, ptr.type_(), dtype.toMlir(self.ctx))).type_(); + return self.emit(fly.ops.recast_iter(self.ctx, ptr.inner, t, self.loc())); + } + + // ---------------------------------------------------------------- atoms + + /// The copy-op type an atom wraps, by bit size. + pub const CopyOp = union(enum) { + universal: i32, + buffer_copy: i32, + buffer_copy_lds: i32, + + fn type_(self: CopyOp, ctx: *mlir.Context) *const mlir.Type { + return switch (self) { + .universal => |b| must(fly.types.CopyOpUniversalCopyType.get(ctx, .{ .bitSize = b })).type_(), + .buffer_copy => |b| must(fly_rocdl.CopyOpCDNA3BufferCopyType.get(ctx, .{ .bitSize = b })).type_(), + .buffer_copy_lds => |b| must(fly_rocdl.CopyOpCDNA3BufferCopyLDSType.get(ctx, .{ .bitSize = b })).type_(), + }; + } + }; + + pub fn copyAtom(self: *Builder, op: CopyOp, dtype: DType) Value { + const bits: i32 = @intCast(dtype.bitWidth()); + const t = must(fly.types.CopyAtomType.get(self.ctx, .{ .copyOp = op.type_(self.ctx), .valBits = bits })).type_(); + return self.emit(fly.ops.make_copy_atom(self.ctx, &.{}, t, .int(self.ctx, .i32, bits), self.loc())); + } + + /// `op` is an MMA op type, such as a `fly_rocdl.MmaOpCDNA3MFMAType`. + pub fn mmaAtom(self: *Builder, op: *const mlir.Type) Value { + const t = must(fly.types.MmaAtomType.get(self.ctx, .{ .mmaOp = op })).type_(); + return self.emit(fly.ops.make_mma_atom(self.ctx, t, self.loc())); + } + + pub fn tiledCopy(self: *Builder, atom: Value, layout_tv: anytype, tile_mn: anytype) TiledCopy { + const tv = self.toStatic(layout_tv); + const tl = self.toStatic(tile_mn); + const v = self.emit(fly.ops.make_tiled_copy(self.ctx, atom.inner, tv.inner, tl.inner, self.loc())); + return .{ .value = v, .tile = tl }; + } + + /// raked product -> right inverse -> composition -> make_tiled_copy, + /// with the tiler read off the inferred product shape. + pub fn tiledCopyTV(self: *Builder, atom: Value, thr: Layout, val: Layout) TiledCopy { + const thr_size = thr.size() orelse @panic("fly.tiledCopyTV: thr_layout must be static"); + const val_size = val.size() orelse @panic("fly.tiledCopyTV: val_layout must be static"); + const layout_mn = self.static(thr).rakedProduct(self.static(val)); + const tmp = self.static(Layout{ + .shape = .{ .tup = &.{ .static(thr_size), .static(val_size) } }, + .stride = .{ .tup = &.{ .static(1), .static(thr_size) } }, + }); + const layout_tv = layout_mn.rightInverse().composition(tmp); + const tiler = layout_mn.emitShape().productEach(); + // The tiler as a tile literal: one leaf per mode, `[a|b]`. + const shape = tiler.intTupleStatic(); + const tile_modes = self.alloc(Tile, shape.rank()); + for (tile_modes, 0..) |*m, i| m.* = .{ .leaf = switch (shape.at(i)) { + .leaf => |l| l, + else => std.debug.panic("fly.tiledCopyTV: tiler mode {d} is not a leaf", .{i}), + } }; + return self.tiledCopy(atom, layout_tv, Tile{ .modes = tile_modes }); + } + + /// A TiledCopy matched to an MMA operand: the TV layout and tile size + /// come off the `!fly.tiled_mma` type. + pub fn tiledCopyFor(self: *Builder, operand: fly.MmaOperand, copy_atom: Value, tm: TiledMma) TiledCopy { + const tm_ty = fly.expect(tm.value.type_(), .tiled_mma); + const layout_tv = self.staticTy((switch (operand) { + .a => tm_ty.tiledThrValLayoutA(), + .b => tm_ty.tiledThrValLayoutB(), + .c => tm_ty.tiledThrValLayoutC(), + .d => @panic("fly.tiledCopyFor: no D operand copy"), + }).type_()); + const tile_size = self.staticTy(tm_ty.tileSizeMNK().type_()); + const modes: [2]i32 = switch (operand) { + .a => .{ 0, 2 }, + .b => .{ 1, 2 }, + .c => .{ 0, 1 }, + .d => unreachable, + }; + const one = self.intTuple(1); + const tile_modes = self.alloc(Tile, modes.len); + for (modes, 0..) |m, i| { + const lay = self.emit(fly.ops.make_layout(self.ctx, tile_size.selectModes(&.{m}).inner, one.inner, self.loc())); + tile_modes[i] = .{ .layout = self.readLayout(lay.type_()) }; + } + return self.tiledCopy(copy_atom, layout_tv, Tile{ .modes = tile_modes }); + } + + pub fn tiledCopyA(self: *Builder, copy_atom: Value, tm: TiledMma) TiledCopy { + return self.tiledCopyFor(.a, copy_atom, tm); + } + pub fn tiledCopyB(self: *Builder, copy_atom: Value, tm: TiledMma) TiledCopy { + return self.tiledCopyFor(.b, copy_atom, tm); + } + pub fn tiledCopyC(self: *Builder, copy_atom: Value, tm: TiledMma) TiledCopy { + return self.tiledCopyFor(.c, copy_atom, tm); + } + + /// The same tensor over a CDNA buffer resource + /// (`!fly.ptr`), for hardware bounds-checked + /// accesses and `buffer_copy` atoms. Descriptor size is the maximum. + pub fn bufferTensor(self: *Builder, t: Value) Value { + const ptr = t.emitIter(); + const lay = t.emitLayout(); + const c0 = self.constant(.i16, 0); + const nrec = self.constant(.i64, 0xFFFFFFFF); + // (7 << 12) | (4 << 15): the CDNA descriptor flags FlyDSL and XLA use. + const flags = self.constant(.i32, 0x27000); + const elem = t.elemDType(); + const buf_ty = self.ptrType(elem, (must(fly_rocdl.BufferDescAddressAttr.get(self.ctx))).attribute()); + const buf_ptr = self.emit(fly.ops.make_ptr(self.ctx, &.{ ptr.inner, c0.inner, nrec.inner, flags.inner }, buf_ty, .{}, self.loc())); + return buf_ptr.view(lay); + } + + pub fn tiledMma(self: *Builder, atom: Value, atom_layout: anytype, permutation: ?Value) TiledMma { + const al = self.toStatic(atom_layout); + const v = self.emit(fly.ops.make_tiled_mma(self.ctx, atom.inner, al.inner, .{ .permutation = if (permutation) |p| p.inner else null }, self.loc())); + return .{ .value = v }; + } + + pub const CopyOpts = struct { + pred: ?Value = null, + }; + + pub fn copy(self: *Builder, atom: Value, src: Value, dst: Value, opts: CopyOpts) void { + self.emitNone(fly.ops.copy(self.ctx, atom.inner, src.inner, dst.inner, .{ .pred = if (opts.pred) |p| p.inner else null }, self.loc())); + } + + pub fn gemm(self: *Builder, atom: Value, d: Value, a: Value, b: Value, c: Value) void { + self.emitNone(fly.ops.gemm(self.ctx, atom.inner, d.inner, &.{a.inner}, &.{b.inner}, c.inner, .{}, self.loc())); + } + + // ---------------------------------------------------------------- control flow + + pub fn ForScope(comptime N: usize) type { + return common.ForScope(Builder, Value, N); + } + pub const IfOnlyScope = common.IfOnlyScope(Builder, Value); + pub fn IfScope(comptime N: usize) type { + return common.IfScope(Builder, Value, N); + } + pub fn WhileScope(comptime N: usize, comptime M: usize) type { + return common.WhileScope(Builder, Value, N, M); + } + + /// Loop-carried `inits`. Literal bounds take the type of a Value bound, + /// else they are `index`. + pub fn openFor( + self: *Builder, + lower: anytype, + upper: anytype, + step: anytype, + inits: anytype, + ) ForScope(common.arity(@TypeOf(inits), "openFor: inits")) { + return self.openForOpts(lower, upper, step, inits, .{}); + } + + /// `openFor` with `scf.for` options. + pub fn openForOpts( + self: *Builder, + lower: anytype, + upper: anytype, + step: anytype, + inits: anytype, + opts: common.ForOpts, + ) ForScope(common.arity(@TypeOf(inits), "openFor: inits")) { + return common.openFor(self, Value, common.forBounds(self, Value, lower, upper, step, Builder.constIndex), inits, opts); + } + + /// `value` as a scalar constant of `like`'s element type (`index` for an + /// `index` Value); a Value passes through. + pub fn constantLike(self: *Builder, value: anytype, like: Value) Value { + if (@TypeOf(value) == Value) return value; + if (like.type_().isA(mlir.IndexType) != null) { + const i: i128 = switch (@typeInfo(@TypeOf(value))) { + .float, .comptime_float => @intFromFloat(value), + else => value, + }; + return self.constIndex(intBits(i, 64, "index")); + } + return self.constant(like.scalarDType(), value); + } + + pub fn openIf(self: *Builder, cond: Value) IfOnlyScope { + return common.openIf(self, Value, cond); + } + + pub fn openIfElse(self: *Builder, cond: Value, result_types: anytype) IfScope(common.arity(@TypeOf(result_types), "openIfElse: result_types")) { + return common.openIfElse(self, Value, cond, result_types); + } + + pub fn openWhile( + self: *Builder, + inits: anytype, + after_types: anytype, + ) WhileScope(common.arity(@TypeOf(inits), "openWhile: inits"), common.arity(@TypeOf(after_types), "openWhile: after_types")) { + return common.openWhile(self, Value, inits, after_types); + } + + // ---------------------------------------------------------------- finalization + + /// Append `gpu.return`, verify, and print the module (no debug info: XLA + /// scrubs locations). Owned by the builder's allocator. + pub fn finish(self: *Builder) FinishError![:0]const u8 { + _ = gpu.return_(self.ctx, &.{}, self.loc()).appendTo(self.currentBlock()); + + try common.verify(self.module, "dsl/fly"); + + var al: std.Io.Writer.Allocating = .init(self.allocator); + defer al.deinit(); + try al.writer.print("{f}", .{self.module.operation().fmt(.{ .debug_info = false })}); + return try self.allocator.dupeSentinel(u8, al.written(), 0); + } +}; + +// ============================================================================= +// Tests +// ============================================================================= + +fn testContext() !*mlir.Context { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + registerDialects(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + return ctx; +} + +/// C = A + B over (M, N) f32, 128-bit vectorized and predicated on the +/// ragged edge. +pub fn emitVectorAdd(b: *Builder, thr_layout: Layout, val_layout: Layout, a_in: Value, b_in: Value, c_out: Value) void { + const shape = a_in.shapeStatic(); + const m = shape.at(0).leaf.s; + const n = shape.at(1).leaf.s; + + const atom = b.copyAtom(.{ .universal = 128 }, .f32); + const tc = b.tiledCopyTV(atom, thr_layout, val_layout); + + const tid = b.threadId(.x); + const bx = b.blockId(.x); + const by = b.blockId(.y); + + const idC = b.identity(.{ m, n }); + const tile_mn = tc.tileMN(); + + const gA = a_in.flatDivide(tile_mn).slice(.{ null, null, bx, by }); + const gB = b_in.flatDivide(tile_mn).slice(.{ null, null, bx, by }); + const gC = c_out.flatDivide(tile_mn).slice(.{ null, null, bx, by }); + const cC = idC.flatDivide(tile_mn).slice(.{ null, null, bx, by }); + + const thr = tc.getSlice(tid); + const tgA = thr.partitionS(gA); + const tgB = thr.partitionS(gB); + const tgC = thr.partitionD(gC); + const tcC = thr.partitionS(cC).slice(.{ .{ 0, null }, null, null }); + + const rA = tgA.makeFragmentLike(null); + const rB = tgB.makeFragmentLike(null); + const rC = tgC.makeFragmentLike(null); + const pC = tcC.makeFragmentLike(.i1); + + for (0..@intCast(pC.sizeStatic())) |i| { + pC.set(i, tcC.at(i).elemLess(.{ m, n })); + } + + b.copy(atom, tgA, rA, .{ .pred = pC }); + b.copy(atom, tgB, rB, .{ .pred = pC }); + rC.store(rA.load().add(rB.load())); + b.copy(atom, rC, tgC, .{ .pred = pC }); +} + +test "vectorAdd builds, verifies and re-parses" { + const ctx = try testContext(); + defer ctx.deinit(); + + var b = try Builder.open(std.testing.allocator, ctx, "vector_add"); + defer b.deinit(); + const a = try b.declareArgs(.{ + .a = .{ .tensor = .{ .dtype = .f32, .dims = &.{ 100, 1000 } } }, + .b = .{ .tensor = .{ .dtype = .f32, .dims = &.{ 100, 1000 } } }, + .c = .{ .tensor = .{ .dtype = .f32, .dims = &.{ 100, 1000 } } }, + }); + emitVectorAdd(&b, ordered(.{ 8, 16 }, .{ 1, 0 }), ordered(.{ 1, 4 }, .{ 0, 1 }), a.a, a.b, a.c); + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + + const expected = [_][]const u8{ + "module attributes {gpu.container_module}", + "gpu.module @zml_fly_kernels", + "gpu.func @vector_add(%arg0: !fly.ptr, %arg1: !fly.ptr, %arg2: !fly.ptr) kernel", + "fly.static : !fly.layout<(100,1000):(1000,1)>", + "fly.make_copy_atom {valBits = 32 : i32} : !fly.copy_atom, 32>", + "fly.raked_product", + "fly.static : !fly.tile<[8|64]>", + "fly.make_tiled_copy", + "gpu.thread_id x", + "fly.tiled_copy.partition_src", + "fly.tiled_copy.partition_dst", + "fly.make_fragment_like", + "fly.elem_less", + "fly.copy(", + "fly.memref.load_vec", + "arith.addf", + "fly.memref.store_vec", + "gpu.return", + }; + for (expected) |needle| { + if (std.mem.indexOf(u8, ir, needle) == null) { + std.debug.print("missing `{s}` in:\n{s}\n", .{ needle, ir }); + return error.TestUnexpectedResult; + } + } + + const reparsed = try mlir.Module.parse(ctx, ir); + defer reparsed.deinit(); + try std.testing.expect(reparsed.operation().verify()); +} + +/// The matrix atom a device family provides. CDNA has MFMA over f32; RDNA has +/// WMMA, whose verifier requires M=N=K=16 and rejects f32 operands, so the +/// operand type and the K tile change with the family, not just the mnemonic. +pub const MmaFlavor = enum { + cdna3_mfma, + gfx11_wmma, + gfx120x_wmma, + + /// MFMA takes f32 operands at 16x16x4; both WMMAs take f16 at 16x16x16. + pub fn atomType(self: MmaFlavor, ctx: *mlir.Context) *const mlir.Type { + const operand = self.operandDType().toMlir(ctx); + const acc: *const mlir.Type = .float(ctx, .f32); + return switch (self) { + .cdna3_mfma => (fly_rocdl.MmaOpCDNA3MFMAType.get(ctx, .{ + .m = 16, + .n = 16, + .k = 4, + .elemTyA = operand, + .elemTyB = operand, + .elemTyAcc = acc, + }) catch unreachable).type_(), + inline .gfx11_wmma, .gfx120x_wmma => |f| blk: { + const T = if (f == .gfx11_wmma) fly_rocdl.MmaOpGFX11WMMAType else fly_rocdl.MmaOpGFX120XWMMAType; + break :blk (T.get(ctx, .{ + .m = 16, + .n = 16, + .k = 16, + .elemTyA = operand, + .elemTyB = operand, + .elemTyAcc = acc, + }) catch unreachable).type_(); + }, + }; + } + + /// A and B operand type. The accumulator is f32 either way. + pub fn operandDType(self: MmaFlavor) DType { + return switch (self) { + .cdna3_mfma => .f32, + .gfx11_wmma, .gfx120x_wmma => .f16, + }; + } + + /// Threads one atom occupies: MFMA runs on a wave64, WMMA on a wave32. + /// Checked against the dialect's own `mma_atom.thr_layout` below. + pub fn atomThreads(self: MmaFlavor) i32 { + return switch (self) { + .cdna3_mfma => 64, + .gfx11_wmma, .gfx120x_wmma => 32, + }; + } + + /// `emitTiledMma` lays the atom out (2,2,1), so a block is four of them. + pub fn blockThreads(self: MmaFlavor) i32 { + return 4 * self.atomThreads(); + } + + /// The block tile's K, a multiple of the atom's K. + pub fn blockK(self: MmaFlavor) i64 { + return switch (self) { + .cdna3_mfma => 8, + .gfx11_wmma, .gfx120x_wmma => 16, + }; + } + + /// Buffer copies are CDNA-only: FlyROCDL declares no RDNA copy atom, so + /// RDNA moves through the core universal copy over plain tensors. + fn hasBufferCopy(self: MmaFlavor) bool { + return self == .cdna3_mfma; + } +}; + +/// A two-mode tile from extents known only at emit time. +fn tile2(b: *Builder, x: i64, y: i64) Tile { + const modes = b.alloc(Tile, 2); + modes[0] = .{ .leaf = .{ .s = x } }; + modes[1] = .{ .leaf = .{ .s = y } }; + return .{ .modes = modes }; +} + +/// C = A @ B^T in one block of 256 threads, tiled (2,2,1) over the family's +/// matrix atom. +pub fn emitTiledMma(b: *Builder, flavor: MmaFlavor, a_in: Value, b_in: Value, c_out: Value) void { + const block_m = 64; + const block_n = 64; + const block_k = flavor.blockK(); + + const tid = b.threadId(.x); + const bid = b.blockId(.x); + + const A = if (flavor.hasBufferCopy()) b.bufferTensor(a_in) else a_in; + const B = if (flavor.hasBufferCopy()) b.bufferTensor(b_in) else b_in; + const C = if (flavor.hasBufferCopy()) b.bufferTensor(c_out) else c_out; + + const bA = A.zippedDivide(tile2(b, block_m, block_k)).slice(.{ null, bid }); + const bB = B.zippedDivide(tile2(b, block_n, block_k)).slice(.{ null, bid }); + const bC = C.zippedDivide(tile2(b, block_m, block_n)).slice(.{ null, bid }); + + const mma_atom = b.mmaAtom(flavor.atomType(b.ctx)); + const tiled_mma = b.tiledMma(mma_atom, L(.{ 2, 2, 1 }, .{ 1, 2, 0 }), null); + + // A and B move at the operand type, C at the f32 accumulator type. One + // element per instruction: a WMMA fragment's values are strided in memory, + // so a wider copy would move the wrong neighbours. + const copy_ab: Builder.CopyOp = if (flavor.hasBufferCopy()) .{ .buffer_copy = 32 } else .{ .universal = @intCast(flavor.operandDType().bitWidth()) }; + const copy_c: Builder.CopyOp = if (flavor.hasBufferCopy()) .{ .buffer_copy = 32 } else .{ .universal = @intCast(c_out.elemDType().bitWidth()) }; + const atom_ab = b.copyAtom(copy_ab, flavor.operandDType()); + const atom_c = b.copyAtom(copy_c, c_out.elemDType()); + + const tiled_copy_a = b.tiledCopyA(atom_ab, tiled_mma); + const tiled_copy_b = b.tiledCopyB(atom_ab, tiled_mma); + const tiled_copy_c = b.tiledCopyC(atom_c, tiled_mma); + + const thr_copy_a = tiled_copy_a.getSlice(tid); + const thr_copy_b = tiled_copy_b.getSlice(tid); + const thr_copy_c = tiled_copy_c.getSlice(tid); + + const copy_src_a = thr_copy_a.partitionS(bA); + const copy_src_b = thr_copy_b.partitionS(bB); + const copy_dst_c = thr_copy_c.partitionS(bC); + + const frag_a = tiled_mma.makeFragmentA(bA); + const frag_b = tiled_mma.makeFragmentB(bB); + const frag_c = tiled_mma.makeFragmentC(bC); + + const copy_frag_a = thr_copy_a.retile(frag_a); + const copy_frag_b = thr_copy_b.retile(frag_b); + const copy_frag_c = thr_copy_c.retile(frag_c); + + b.copy(atom_ab, copy_src_a, copy_frag_a, .{}); + b.copy(atom_ab, copy_src_b, copy_frag_b, .{}); + + frag_c.fill(0); + b.gemm(mma_atom, frag_c, frag_a, frag_b, frag_c); + + b.copy(atom_c, copy_frag_c, copy_dst_c, .{}); +} + +test "tiledMma builds, verifies and re-parses" { + const ctx = try testContext(); + defer ctx.deinit(); + + var b = try Builder.open(std.testing.allocator, ctx, "tiled_mma"); + defer b.deinit(); + const a = try b.declareArgs(.{ + .a = .{ .tensor = .{ .dtype = .f32, .dims = &.{ 64, 8 } } }, + .b = .{ .tensor = .{ .dtype = .f32, .dims = &.{ 64, 8 } } }, + .c = .{ .tensor = .{ .dtype = .f32, .dims = &.{ 64, 64 } } }, + }); + emitTiledMma(&b, .cdna3_mfma, a.a, a.b, a.c); + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + + const expected = [_][]const u8{ + "!fly.ptr", + "fly.make_mma_atom : !fly.mma_atom f32>>", + "fly.make_tiled_mma", + "fly.static : !fly.int_tuple<(32,32,4)>", + "fly.mma.make_fragment", + "fly.tiled_copy.retile", + "vector.broadcast", + "fly.gemm(", + }; + for (expected) |needle| { + if (std.mem.indexOf(u8, ir, needle) == null) { + std.debug.print("missing `{s}` in:\n{s}\n", .{ needle, ir }); + return error.TestUnexpectedResult; + } + } + const reparsed = try mlir.Module.parse(ctx, ir); + defer reparsed.deinit(); + try std.testing.expect(reparsed.operation().verify()); +} + +test "MmaFlavor.atomThreads matches the dialect" { + const ctx = try testContext(); + defer ctx.deinit(); + inline for (comptime std.enums.values(MmaFlavor)) |flavor| { + const atom = try fly.types.MmaAtomType.get(ctx, .{ .mmaOp = flavor.atomType(ctx) }); + // `mma_atom.thr_layout` is the wavefront an atom occupies; the launch + // geometry is derived from it, so a wrong table silently corrupts C. + const threads = switch (atom.thrLayout().shape().leaf()) { + .static => |v| v, + else => return error.TestUnexpectedResult, + }; + try std.testing.expectEqual(@as(i64, flavor.atomThreads()), threads); + } +} + +test "tiledMma builds for the RDNA WMMA atom" { + const ctx = try testContext(); + defer ctx.deinit(); + + var b = try Builder.open(std.testing.allocator, ctx, "tiled_mma_wmma"); + defer b.deinit(); + const k = MmaFlavor.gfx11_wmma.blockK(); + const a = try b.declareArgs(.{ + .a = .{ .tensor = .{ .dtype = .f16, .dims = &.{ 64, k } } }, + .b = .{ .tensor = .{ .dtype = .f16, .dims = &.{ 64, k } } }, + .c = .{ .tensor = .{ .dtype = .f32, .dims = &.{ 64, 64 } } }, + }); + emitTiledMma(&b, .gfx11_wmma, a.a, a.b, a.c); + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + + // No buffer descriptor: FlyROCDL declares no RDNA copy atom, so this path + // moves through the core universal copy over plain tensors. + if (std.mem.indexOf(u8, ir, "buffer_desc") != null) { + std.debug.print("unexpected buffer descriptor on the WMMA path:\n{s}\n", .{ir}); + return error.TestUnexpectedResult; + } + for ([_][]const u8{ + "!fly_rocdl.gfx11.wmma<16x16x16, (f16, f16) -> f32", + "!fly.copy_atom, 16>", + "fly.gemm(", + }) |needle| { + if (std.mem.indexOf(u8, ir, needle) == null) { + std.debug.print("missing `{s}` in:\n{s}\n", .{ needle, ir }); + return error.TestUnexpectedResult; + } + } + + const reparsed = try mlir.Module.parse(ctx, ir); + defer reparsed.deinit(); + try std.testing.expect(reparsed.operation().verify()); +} + +test "scalar conversions and constants" { + const ctx = try testContext(); + defer ctx.deinit(); + + var b = try Builder.open(std.testing.allocator, ctx, "conv"); + defer b.deinit(); + const a = try b.declareArgs(.{ + .x = .{ .tensor = .{ .dtype = .f16, .dims = &.{16} } }, + .p = .{ .tensor = .{ .dtype = .i1, .dims = &.{16} } }, + }); + + // A predicate is unsigned: `true` must convert to 1, never -1. + const pred = b.constant(.i32, 3).lt(7); + _ = pred.to(.f32); + _ = pred.to(.i32); + // Equal-width float pairs are neither ext nor trunc. + _ = b.constant(.f16, 1.0).to(.bf16); + // A float constant outside i64 range must not go through @intFromFloat. + _ = b.constant(.f32, -std.math.inf(f32)); + _ = b.constant(.f32, std.math.floatMax(f32)); + // A `.bool` argument keeps its storage element type end to end. + try std.testing.expectEqual(DType.i8, a.p.elemDType()); + try std.testing.expectEqual(DType.f16, a.x.elemDType()); + // Literal inits are lifted, and an integer constant may be a bit pattern. + var loop = b.openFor(0, 4, 1, .{ 0, 0.5 }); + { + loop.yield(.{ loop.carried[0].bitXor(b.constant(.i32, 0xd4d0ccc8)), loop.carried[1].mul(2.0) }); + } + + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + for ([_][]const u8{ + "arith.uitofp", + "arith.extui", + "arith.convertf", + "arith.constant 0xFF800000 : f32", + "!fly.memref", + "-> (i32, f32)", + "arith.constant -724513592 : i32", + }) |needle| { + if (std.mem.indexOf(u8, ir, needle) == null) { + std.debug.print("missing `{s}` in:\n{s}\n", .{ needle, ir }); + return error.TestUnexpectedResult; + } + } + if (std.mem.indexOf(u8, ir, "arith.sitofp") != null or std.mem.indexOf(u8, ir, "arith.extsi") != null) { + std.debug.print("i1 was sign-extended:\n{s}\n", .{ir}); + return error.TestUnexpectedResult; + } +} + +test "shuffles and fast math carry their attributes" { + const ctx = try testContext(); + defer ctx.deinit(); + + var b = try Builder.open(std.testing.allocator, ctx, "shuffle"); + defer b.deinit(); + _ = try b.declareArgs(.{ + .x = .{ .tensor = .{ .dtype = .f32, .dims = &.{16} } }, + }); + + b.setFastMath(true); + const x = b.constant(.f32, 1.0); + _ = x.shuffleXor(1, 64).add(x.shuffle(.idx, 3, 64)); + + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + for ([_][]const u8{ + "gpu.shuffle xor", + "gpu.shuffle idx", + "arith.addf %shuffleResult, %shuffleResult_1 fastmath", + }) |needle| { + if (std.mem.indexOf(u8, ir, needle) == null) { + std.debug.print("missing `{s}` in:\n{s}\n", .{ needle, ir }); + return error.TestUnexpectedResult; + } + } +} + +test "static queries read inferred types" { + const ctx = try testContext(); + defer ctx.deinit(); + + var b = try Builder.open(std.testing.allocator, ctx, "q"); + defer b.deinit(); + const a = try b.declareArgs(.{ .x = .{ .tensor = .{ .dtype = .bf16, .dims = &.{ 64, 32 } } } }); + try std.testing.expectEqual(@as(i64, 64 * 32), a.x.sizeStatic()); + try std.testing.expectEqual(DType.bf16, a.x.elemDType()); + try std.testing.expectEqual(Value.Kind.memref, a.x.kind()); + const tiled = a.x.flatDivide(tile(.{ 16, 8 })); + try std.testing.expectEqual(@as(i64, 64 * 32), tiled.sizeStatic()); + try std.testing.expectEqual(@as(usize, 4), tiled.shapeStatic().rank()); +} + +test "atomic addition builds, verifies and re-parses" { + const ctx = try testContext(); + defer ctx.deinit(); + + var b = try Builder.open(std.testing.allocator, ctx, "atomic_add"); + defer b.deinit(); + _ = try b.declareArgs(.{}); + const counter = b.sharedArray(.i32, 1, 4); + _ = b.ptrAtomicAdd(counter.emitIter(), b.constant(.i32, 1), .workgroup); + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + + // LLVM's current custom assembly stores alignment in the property dictionary. + for ([_][]const u8{ + "llvm.atomicrmw add", + "syncscope(\"workgroup\") monotonic ", + ": !llvm.ptr<3>, i32", + }) |needle| { + if (std.mem.indexOf(u8, ir, needle) == null) { + std.debug.print("IR missing `{s}`:\n{s}\n", .{ needle, ir }); + return error.TestUnexpectedResult; + } + } + const reparsed = try mlir.Module.parse(ctx, ir); + defer reparsed.deinit(); + try std.testing.expect(reparsed.operation().verify()); +} + +test "SSA matrix atom builds, verifies and re-parses" { + const ctx = try testContext(); + defer ctx.deinit(); + + var b = try Builder.open(std.testing.allocator, ctx, "mma_atom_ssa"); + defer b.deinit(); + _ = try b.declareArgs(.{}); + const atom_type = try fly_rocdl.MmaOpCDNA3MFMAType.get(ctx, .{ + .m = 16, + .n = 16, + .k = 16, + .elemTyA = .float(ctx, .bf16), + .elemTyB = .float(ctx, .bf16), + .elemTyAcc = .float(ctx, .f32), + }); + const atom = b.mmaAtom(atom_type.type_()); + const lhs = b.constant(.bf16, 0).splat(4); + const rhs = b.constant(.bf16, 0).splat(4); + const acc = b.constant(.f32, 0).splat(4); + const result = b.mmaAtomCall(atom, lhs, rhs, acc); + try std.testing.expect(result.type_().eql(acc.type_())); + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + + const reparsed = try mlir.Module.parse(ctx, ir); + defer reparsed.deinit(); + try std.testing.expect(reparsed.operation().verify()); +} + +test "comparisons take index ivs and vectors" { + const ctx = try testContext(); + defer ctx.deinit(); + + var b = try Builder.open(std.testing.allocator, ctx, "compare"); + defer b.deinit(); + _ = try b.declareArgs(.{}); + + // Literal bounds make an `index` loop. + var loop = b.openFor(0, 4, 1, .{}); + { + _ = loop.iv.lt(2); + loop.close(); + } + // A literal or a scalar broadcasts against a vector. + const v = b.constant(.f32, 1.0).splat(4); + _ = v.lt(0.5); + _ = v.ge(b.constant(.f32, 2.0)); + _ = b.constant(.i32, 3).splat(4).eq(3); + + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + for ([_][2][]const u8{ + .{ "arith.cmpi slt", ": index" }, + .{ "arith.cmpf olt", ": vector<4xf32>" }, + .{ "arith.cmpf oge", ": vector<4xf32>" }, + .{ "arith.cmpi eq", ": vector<4xi32>" }, + }) |needles| { + var lines = std.mem.splitScalar(u8, ir, '\n'); + const found = while (lines.next()) |line| { + if (std.mem.indexOf(u8, line, needles[0]) != null and std.mem.endsWith(u8, line, needles[1])) break true; + } else false; + if (!found) { + std.debug.print("no `{s} ... {s}` in:\n{s}\n", .{ needles[0], needles[1], ir }); + return error.TestUnexpectedResult; + } + } +} diff --git a/dsl/fly/dtype.zig b/dsl/fly/dtype.zig new file mode 100644 index 00000000..ecc9f070 --- /dev/null +++ b/dsl/fly/dtype.zig @@ -0,0 +1,84 @@ +const std = @import("std"); +const mlir = @import("mlir"); + +pub const DType = enum { + i1, + i8, + i16, + i32, + i64, + f16, + bf16, + f32, + f64, + f8e4m3fn, + f8e4m3fnuz, + f8e5m2, + f8e5m2fnuz, + + pub fn toMlir(self: DType, ctx: *mlir.Context) *const mlir.Type { + return switch (self) { + .i1 => .int(ctx, .i1), + .i8 => .int(ctx, .i8), + .i16 => .int(ctx, .i16), + .i32 => .int(ctx, .i32), + .i64 => .int(ctx, .i64), + .f16 => .float(ctx, .f16), + .bf16 => .float(ctx, .bf16), + .f32 => .float(ctx, .f32), + .f64 => .float(ctx, .f64), + .f8e4m3fn => .float(ctx, .f8e4m3fn), + .f8e4m3fnuz => .float(ctx, .f8e4m3fnuz), + .f8e5m2 => .float(ctx, .f8e5m2), + .f8e5m2fnuz => .float(ctx, .f8e5m2fnuz), + }; + } + + pub fn bitWidth(self: DType) u32 { + return switch (self) { + .i1 => 1, + .i8, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz => 8, + .i16, .f16, .bf16 => 16, + .i32, .f32 => 32, + .i64, .f64 => 64, + }; + } + + pub fn isFloat(self: DType) bool { + return switch (self) { + .f16, .bf16, .f32, .f64, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz => true, + else => false, + }; + } + + /// The element type a kernel argument is stored with on device: + /// predicates live in one byte. A copy atom over `!fly.ptr` would + /// claim `bits / 1` elements while moving `bits / 8` bytes, so an + /// argument keeps its storage type and the body converts explicitly. + pub fn storageElem(self: DType) DType { + return switch (self) { + .i1 => .i8, + else => self, + }; + } +}; + +/// Inverse of `toMlir`. +pub fn fromMlir(ctx: *mlir.Context, ty: *const mlir.Type) ?DType { + inline for (comptime std.enums.values(DType)) |dt| { + if (ty.eql(dt.toMlir(ctx))) return dt; + } + return null; +} + +test "dtype types round-trip" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + inline for (comptime std.enums.values(DType)) |dt| { + try std.testing.expectEqual(dt, fromMlir(ctx, dt.toMlir(ctx)).?); + } + try std.testing.expectEqual(DType.i8, DType.i1.storageElem()); + try std.testing.expectEqual(DType.f8e4m3fnuz, DType.f8e4m3fnuz.storageElem()); +} diff --git a/dsl/fly/layout.zig b/dsl/fly/layout.zig new file mode 100644 index 00000000..06dd64c6 --- /dev/null +++ b/dsl/fly/layout.zig @@ -0,0 +1,544 @@ +const std = @import("std"); + +const fly = @import("mlir/dialects/fly"); +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +const attributes = fly.attributes; + +/// Enough for any tuple or tile a kernel writes by hand. +const max_modes = 32; + +/// A dynamic integer leaf: `?`, `?{i64}`, `?{div=8}`, `?{i64 div=8}`. +pub const Dyn = struct { + width: u8 = 32, + div: u32 = 1, +}; + +/// Static, dynamic, or the `*` wildcard. +pub const Leaf = union(enum) { + s: i64, + d: Dyn, + none, + + pub fn print(self: Leaf, w: *std.Io.Writer) std.Io.Writer.Error!void { + switch (self) { + .s => |v| try w.print("{d}", .{v}), + .none => try w.writeAll("*"), + .d => |dy| { + try w.writeAll("?"); + if (dy.width != 32 or dy.div != 1) { + try w.writeAll("{"); + if (dy.width != 32) try w.print("i{d}", .{dy.width}); + if (dy.div != 1) { + if (dy.width != 32) try w.writeAll(" "); + try w.print("div={d}", .{dy.div}); + } + try w.writeAll("}"); + } + }, + } + } +}; + +fn leafAttr(leaf: Leaf, ctx: *mlir.Context) mlir.Error!*const attributes.IntAttr { + return switch (leaf) { + .s => |v| .getStatic(ctx, @intCast(v)), + .d => |dy| .getDynamic(ctx, dy.width, @intCast(dy.div)), + .none => .getNone(ctx), + }; +} + +/// A scaled basis element: `vE0`, `2E1E2`. +pub const Basis = struct { + value: Leaf, + modes: []const i32, +}; + +pub const IntTuple = union(enum) { + leaf: Leaf, + basis: Basis, + tup: []const IntTuple, + + pub const dyn: IntTuple = .{ .leaf = .{ .d = .{} } }; + pub const star: IntTuple = .{ .leaf = .none }; + + pub fn static(v: i64) IntTuple { + return .{ .leaf = .{ .s = v } }; + } + + pub fn dynamic(width: u8, div: u32) IntTuple { + return .{ .leaf = .{ .d = .{ .width = width, .div = div } } }; + } + + /// 1 for a leaf. + pub fn rank(self: IntTuple) usize { + return switch (self) { + .tup => |t| t.len, + else => 1, + }; + } + + pub fn isLeaf(self: IntTuple) bool { + return self != .tup; + } + + pub fn at(self: IntTuple, i: usize) IntTuple { + return switch (self) { + .tup => |t| t[i], + else => if (i == 0) self else @panic("fly.IntTuple.at: index out of range on a leaf"), + }; + } + + /// Null if any leaf is dynamic or `*`. + pub fn product(self: IntTuple) ?i64 { + switch (self) { + .leaf => |l| return switch (l) { + .s => |v| v, + else => null, + }, + .basis => return null, + .tup => |t| { + var p: i64 = 1; + for (t) |e| p *= e.product() orelse return null; + return p; + }, + } + } + + pub fn isStatic(self: IntTuple) bool { + return switch (self) { + .leaf => |l| l == .s, + .basis => |b| b.value == .s, + .tup => |t| for (t) |e| { + if (!e.isStatic()) break false; + } else true, + }; + } + + /// At any depth. + pub fn leafCount(self: IntTuple) usize { + return switch (self) { + .tup => |t| blk: { + var n: usize = 0; + for (t) |e| n += e.leafCount(); + break :blk n; + }, + else => 1, + }; + } + + pub fn print(self: IntTuple, w: *std.Io.Writer) std.Io.Writer.Error!void { + switch (self) { + .leaf => |l| try l.print(w), + .basis => |b| { + try b.value.print(w); + for (b.modes) |m| try w.print("E{d}", .{m}); + }, + .tup => |t| { + try w.writeAll("("); + for (t, 0..) |e, i| { + if (i > 0) try w.writeAll(","); + try e.print(w); + } + try w.writeAll(")"); + }, + } + } + + pub fn format(self: IntTuple, w: *std.Io.Writer) std.Io.Writer.Error!void { + try self.print(w); + } + + pub fn toAttr(self: IntTuple, ctx: *mlir.Context) mlir.Error!*const attributes.IntTupleAttr { + return switch (self) { + .leaf => |l| .get(ctx, .{ .value = (try leafAttr(l, ctx)).attribute() }), + .basis => |b| .getBasis(ctx, try leafAttr(b.value, ctx), b.modes), + .tup => |elems| blk: { + var buf: stdx.BoundedArray(*const attributes.IntTupleAttr, max_modes) = .empty; + for (elems) |e| buf.appendAssumeCapacity(try e.toAttr(ctx)); + break :blk .getTuple(ctx, buf.constSlice()); + }, + }; + } + + pub fn eql(a: IntTuple, b: IntTuple) bool { + if (std.meta.activeTag(a) != std.meta.activeTag(b)) return false; + return switch (a) { + .leaf => |l| std.meta.eql(l, b.leaf), + .basis => |x| std.meta.eql(x.value, b.basis.value) and std.mem.eql(i32, x.modes, b.basis.modes), + .tup => |t| t.len == b.tup.len and for (t, b.tup) |x, y| { + if (!x.eql(y)) break false; + } else true, + }; + } +}; + +/// Integers become static leaves, tuples nest, `null` is `*`, an `IntTuple` +/// passes through: `it(.{ .{ 2, 4 }, 8 })` is `((2,4),8)`. +pub fn it(comptime v: anytype) IntTuple { + return comptime itImpl(v); +} + +fn itImpl(comptime v: anytype) IntTuple { + const T = @TypeOf(v); + if (T == IntTuple) return v; + if (T == Leaf) return .{ .leaf = v }; + if (T == @TypeOf(null)) return IntTuple.star; + switch (@typeInfo(T)) { + .comptime_int, .int => return IntTuple.static(v), + .@"struct" => |s| { + if (!s.is_tuple) @compileError("fly.it: expected an integer, null, IntTuple or tuple, got " ++ @typeName(T)); + var elems: [s.field_names.len]IntTuple = undefined; + for (s.field_names, 0..) |f, i| elems[i] = itImpl(@field(v, f)); + const frozen = elems; + return .{ .tup = &frozen }; + }, + else => @compileError("fly.it: expected an integer, null, IntTuple or tuple, got " ++ @typeName(T)), + } +} + +/// `!fly.layout`. +pub const Layout = struct { + shape: IntTuple, + stride: IntTuple, + + pub fn rank(self: Layout) usize { + return self.shape.rank(); + } + + /// Null if dynamic. + pub fn size(self: Layout) ?i64 { + return self.shape.product(); + } + + pub fn print(self: Layout, w: *std.Io.Writer) std.Io.Writer.Error!void { + try self.shape.print(w); + try w.writeAll(":"); + try self.stride.print(w); + } + + pub fn format(self: Layout, w: *std.Io.Writer) std.Io.Writer.Error!void { + try self.print(w); + } + + pub fn toAttr(self: Layout, ctx: *mlir.Context) mlir.Error!*const attributes.LayoutAttr { + return .get(ctx, .{ .shape = try self.shape.toAttr(ctx), .stride = try self.stride.toAttr(ctx) }); + } + + pub fn eql(a: Layout, b: Layout) bool { + return a.shape.eql(b.shape) and a.stride.eql(b.stride); + } +}; + +/// Two congruent tuples: `L(.{ 4, 8 }, .{ 1, 4 })` is `(4,8):(1,4)`. +pub fn L(comptime shape: anytype, comptime stride: anytype) Layout { + return comptime blk: { + const s = it(shape); + const d = it(stride); + checkCongruent(s, d); + break :blk .{ .shape = s, .stride = d }; + }; +} + +fn checkCongruent(comptime a: IntTuple, comptime b: IntTuple) void { + if (a.isLeaf() != b.isLeaf() or a.rank() != b.rank()) { + @compileError("fly.L: shape and stride are not congruent"); + } + if (a == .tup) { + for (a.tup, b.tup) |x, y| checkCongruent(x, y); + } +} + +/// Last mode contiguous: `rowMajor(.{ 4, 8 })` is `(4,8):(8,1)`. +pub fn rowMajor(comptime shape: anytype) Layout { + return comptime blk: { + const s = it(shape); + const n = s.leafCount(); + var order: [n]i32 = undefined; + for (0..n) |i| order[i] = @intCast(n - 1 - i); + break :blk orderedFlat(s, &order); + }; +} + +/// First mode contiguous. +pub fn colMajor(comptime shape: anytype) Layout { + return comptime blk: { + const s = it(shape); + const n = s.leafCount(); + var order: [n]i32 = undefined; + for (0..n) |i| order[i] = @intCast(i); + break :blk orderedFlat(s, &order); + }; +} + +/// `make_ordered_layout` folded at comptime: the smallest order value gets +/// stride 1, the next the product of the previous extents, and so on. +/// `ordered(.{ 8, 16 }, .{ 1, 0 })` is `(8,16):(16,1)`. +pub fn ordered(comptime shape: anytype, comptime order: anytype) Layout { + return comptime blk: { + const s = it(shape); + const o = it(order); + checkCongruent(s, o); + const n = s.leafCount(); + var flat_order: [n]i32 = undefined; + var idx: usize = 0; + flattenOrder(o, &flat_order, &idx); + break :blk orderedFlat(s, &flat_order); + }; +} + +fn flattenOrder(comptime o: IntTuple, out: []i32, idx: *usize) void { + switch (o) { + .leaf => |l| { + out[idx.*] = @intCast(l.s); + idx.* += 1; + }, + .tup => |t| for (t) |e| flattenOrder(e, out, idx), + .basis => @compileError("fly.ordered: basis elements are not allowed in an order"), + } +} + +fn orderedFlat(comptime shape: IntTuple, comptime order: []const i32) Layout { + comptime { + const n = shape.leafCount(); + var extents: [n]i64 = undefined; + var idx: usize = 0; + flattenStatic(shape, &extents, &idx); + var strides: [n]i64 = undefined; + var assigned: [n]bool = @splat(false); + var acc: i64 = 1; + for (0..n) |_| { + var best: ?usize = null; + for (0..n) |i| { + if (assigned[i]) continue; + if (best == null or order[i] < order[best.?]) best = i; + } + strides[best.?] = acc; + acc *= extents[best.?]; + assigned[best.?] = true; + } + var pos: usize = 0; + return .{ .shape = shape, .stride = unflatten(shape, &strides, &pos) }; + } +} + +fn flattenStatic(comptime t: IntTuple, out: []i64, idx: *usize) void { + switch (t) { + .leaf => |l| switch (l) { + .s => |v| { + out[idx.*] = v; + idx.* += 1; + }, + else => @compileError("fly: a folded layout needs a fully static shape"), + }, + .basis => @compileError("fly: a folded layout needs a plain integer shape"), + .tup => |elems| for (elems) |e| flattenStatic(e, out, idx), + } +} + +fn unflatten(comptime like: IntTuple, values: []const i64, pos: *usize) IntTuple { + switch (like) { + .tup => |elems| { + var out: [elems.len]IntTuple = undefined; + for (elems, 0..) |e, i| out[i] = unflatten(e, values, pos); + const frozen = out; + return .{ .tup = &frozen }; + }, + else => { + const v = values[pos.*]; + pos.* += 1; + return IntTuple.static(v); + }, + } +} + +/// A leaf (`8` or `*`), a layout, or a nested tile. +pub const Tile = union(enum) { + leaf: Leaf, + layout: Layout, + modes: []const Tile, + + pub fn print(self: Tile, w: *std.Io.Writer) std.Io.Writer.Error!void { + switch (self) { + .leaf => |l| try l.print(w), + .layout => |l| try l.print(w), + .modes => |m| { + try w.writeAll("["); + for (m, 0..) |e, i| { + if (i > 0) try w.writeAll("|"); + try e.print(w); + } + try w.writeAll("]"); + }, + } + } + + pub fn format(self: Tile, w: *std.Io.Writer) std.Io.Writer.Error!void { + try self.print(w); + } + + /// One mode: an int, a layout, or a nested tile. + fn modeAttr(self: Tile, ctx: *mlir.Context) mlir.Error!*const mlir.Attribute { + return switch (self) { + .leaf => |l| (try leafAttr(l, ctx)).attribute(), + .layout => |l| (try l.toAttr(ctx)).attribute(), + .modes => (try self.toAttr(ctx)).attribute(), + }; + } + + pub fn toAttr(self: Tile, ctx: *mlir.Context) mlir.Error!*const attributes.TileAttr { + const modes = switch (self) { + .modes => |m| m, + else => return .get(ctx, .{ .value = try self.modeAttr(ctx) }), + }; + var buf: stdx.BoundedArray(*const mlir.Attribute, max_modes) = .empty; + for (modes) |m| buf.appendAssumeCapacity(try m.modeAttr(ctx)); + return .getModes(ctx, buf.constSlice()); + } +}; + +/// `tile(.{ 128, 64 })` is `[128|64]`; `tile(.{ null, 8 })` is `[*|8]`. +pub fn tile(comptime v: anytype) Tile { + return comptime tileImpl(v); +} + +fn tileImpl(comptime v: anytype) Tile { + const T = @TypeOf(v); + if (T == Tile) return v; + if (T == Layout) return .{ .layout = v }; + if (T == Leaf) return .{ .leaf = v }; + if (T == @TypeOf(null)) return .{ .leaf = .none }; + switch (@typeInfo(T)) { + .comptime_int, .int => return .{ .leaf = .{ .s = v } }, + .@"struct" => |s| { + if (!s.is_tuple) @compileError("fly.tile: expected an integer, null, Layout, Tile or tuple, got " ++ @typeName(T)); + var elems: [s.field_names.len]Tile = undefined; + for (s.field_names, 0..) |f, i| elems[i] = tileImpl(@field(v, f)); + const frozen = elems; + return .{ .modes = &frozen }; + }, + else => @compileError("fly.tile: expected an integer, null, Layout, Tile or tuple, got " ++ @typeName(T)), + } +} + +/// `S`. +pub const Swizzle = struct { + mask: i32, + base: i32, + shift: i32, + + pub const trivial: Swizzle = .{ .mask = 0, .base = 0, .shift = 0 }; + + pub fn isTrivial(self: Swizzle) bool { + return self.mask == 0 and self.base == 0 and self.shift == 0; + } + + pub fn print(self: Swizzle, w: *std.Io.Writer) std.Io.Writer.Error!void { + try w.print("S<{d},{d},{d}>", .{ self.mask, self.base, self.shift }); + } + + pub fn format(self: Swizzle, w: *std.Io.Writer) std.Io.Writer.Error!void { + try self.print(w); + } + + pub fn toAttr(self: Swizzle, ctx: *mlir.Context) mlir.Error!*const attributes.SwizzleAttr { + return .get(ctx, .{ .mask = self.mask, .base = self.base, .shift = self.shift }); + } +}; + +// ----------------------------------------------------------------------------- +// Tests +// ----------------------------------------------------------------------------- + +fn expectPrints(expected: []const u8, v: anytype) !void { + var buf: [1024]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print("{f}", .{v}); + try std.testing.expectEqualStrings(expected, w.buffered()); +} + +test "int tuple literals print in fly syntax" { + try expectPrints("42", it(42)); + try expectPrints("((2,4),8)", it(.{ .{ 2, 4 }, 8 })); + try expectPrints("(?,8)", it(.{ IntTuple.dyn, 8 })); + try expectPrints("?{i64 div=8}", IntTuple.dynamic(64, 8)); + try expectPrints("?{div=4}", IntTuple.dynamic(32, 4)); + try expectPrints("?{i64}", IntTuple.dynamic(64, 1)); + try expectPrints("(*,8)", it(.{ null, 8 })); + try expectPrints("(1,(2,3))", it(.{ 1, .{ 2, 3 } })); +} + +test "layout literals" { + try expectPrints("(4,8):(1,4)", L(.{ 4, 8 }, .{ 1, 4 })); + try expectPrints("((2,4),8):((1,2),8)", L(.{ .{ 2, 4 }, 8 }, .{ .{ 1, 2 }, 8 })); + try expectPrints("(4,8):(8,1)", rowMajor(.{ 4, 8 })); + try expectPrints("(4,8):(1,4)", colMajor(.{ 4, 8 })); + try expectPrints("((2,4),8):((32,8),1)", rowMajor(.{ .{ 2, 4 }, 8 })); + try expectPrints("(8,16):(16,1)", ordered(.{ 8, 16 }, .{ 1, 0 })); + try expectPrints("(1,4):(1,1)", ordered(.{ 1, 4 }, .{ 0, 1 })); + try expectPrints("(4,2,8):(2,1,8)", ordered(.{ 4, 2, 8 }, .{ 1, 0, 2 })); +} + +test "tiles and swizzles" { + try expectPrints("[128|64]", tile(.{ 128, 64 })); + try expectPrints("[*|8]", tile(.{ null, 8 })); + try expectPrints("[(4,8):(1,4)|16]", tile(.{ L(.{ 4, 8 }, .{ 1, 4 }), 16 })); + try expectPrints("[[2|3]|4]", tile(.{ .{ 2, 3 }, 4 })); + try expectPrints("S<3,3,3>", Swizzle{ .mask = 3, .base = 3, .shift = 3 }); +} + +test "static queries" { + try std.testing.expectEqual(@as(?i64, 64), it(.{ .{ 2, 4 }, 8 }).product()); + try std.testing.expectEqual(@as(?i64, null), it(.{ IntTuple.dyn, 8 }).product()); + try std.testing.expectEqual(@as(usize, 2), it(.{ .{ 2, 4 }, 8 }).rank()); + try std.testing.expectEqual(@as(usize, 3), it(.{ .{ 2, 4 }, 8 }).leafCount()); + try std.testing.expect(it(.{ 2, 4 }).eql(it(.{ 2, 4 }))); + try std.testing.expect(!it(.{ 2, 4 }).eql(it(.{ 4, 2 }))); +} + +fn testContext() !*mlir.Context { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + @import("builder.zig").registerDialects(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + return ctx; +} + +/// The type built from the host model must be the type the printed literal +/// parses to: the two paths may never drift. +fn expectBuilt(ctx: *mlir.Context, comptime T: type, comptime fmt: []const u8, value: anytype) !void { + var buf: [512]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print(fmt, .{value}); + const parsed = try mlir.Type.parse(ctx, w.buffered()); + const built = try T.get(ctx, .{ .attr = try value.toAttr(ctx) }); + if (!parsed.eql(built.type_())) { + std.debug.print("parsed `{f}` but built `{f}`\n", .{ parsed, built }); + return error.TestUnexpectedResult; + } +} + +test "the host model builds the types its literals print" { + const ctx = try testContext(); + defer ctx.deinit(); + const IntTupleType = fly.types.IntTupleType; + const LayoutType = fly.types.LayoutType; + const TileType = fly.types.TileType; + + try expectBuilt(ctx, IntTupleType, "!fly.int_tuple<{f}>", it(42)); + try expectBuilt(ctx, IntTupleType, "!fly.int_tuple<{f}>", it(.{ .{ 2, 4 }, 8 })); + try expectBuilt(ctx, IntTupleType, "!fly.int_tuple<{f}>", it(.{ IntTuple.dyn, 8 })); + try expectBuilt(ctx, IntTupleType, "!fly.int_tuple<{f}>", IntTuple.dynamic(64, 8)); + try expectBuilt(ctx, IntTupleType, "!fly.int_tuple<{f}>", it(.{ null, 8 })); + try expectBuilt(ctx, IntTupleType, "!fly.int_tuple<{f}>", IntTuple{ .basis = .{ .value = .{ .s = 2 }, .modes = &.{ 1, 2 } } }); + + try expectBuilt(ctx, LayoutType, "!fly.layout<{f}>", L(.{ 4, 8 }, .{ 1, 4 })); + try expectBuilt(ctx, LayoutType, "!fly.layout<{f}>", rowMajor(.{ .{ 2, 4 }, 8 })); + + try expectBuilt(ctx, TileType, "!fly.tile<{f}>", tile(.{ 128, 64 })); + try expectBuilt(ctx, TileType, "!fly.tile<{f}>", tile(.{ null, 8 })); + try expectBuilt(ctx, TileType, "!fly.tile<{f}>", tile(.{ L(.{ 4, 8 }, .{ 1, 4 }), 16 })); + try expectBuilt(ctx, TileType, "!fly.tile<{f}>", tile(.{ .{ 2, 3 }, 4 })); +} diff --git a/dsl/metal/BUILD.bazel b/dsl/metal/BUILD.bazel new file mode 100644 index 00000000..d68355c0 --- /dev/null +++ b/dsl/metal/BUILD.bazel @@ -0,0 +1,25 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "builder", + srcs = ["types.zig"], + import_name = "dsl/metal/builder", + main = "builder.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//dsl/common:builder", + "//mlir", + "//mlir/dialects/air", + "//mlir/dialects/arith", + "//mlir/dialects/cf", + "//mlir/dialects/func", + "//mlir/dialects/llvm", + "//mlir/dialects/vector", + ], +) + +zig_test( + name = "test", + deps = [":builder"], +) diff --git a/dsl/metal/README.md b/dsl/metal/README.md new file mode 100644 index 00000000..1cbc40ce --- /dev/null +++ b/dsl/metal/README.md @@ -0,0 +1,150 @@ +# ZML Metal DSL + +A Zig builder for Metal kernels that reads like MSL. It emits the `air` +dialect (`mlir/dialects/air`) and the parts of `arith`, `cf`, `vector` and +`llvm` that MSL code compiles to: values, locals in memory, blocks and +branches. `zml.kernel.metal.Kernel` wraps a kernel into +a `__gpu$xla.gpu.air` custom call, and XLA does the rest: it lowers the module to +Apple's AIR, runs the Metal frontend's optimization pipeline +(`air-opt -O2 -disable-loop-unrolling`) and builds a metallib. Apple's +backend compiles that metallib the way it compiles an MSL kernel. + +## Skeleton of a kernel + +```zig +const metal = @import("zml").kernel.metal; + +pub const Axpy = struct { + pub const Cfg = struct {}; + pub const Kernel = metal.Kernel(Cfg, .{ + .name = "axpy", + .inputs = &.{ "a", "b" }, + .outputs = &.{"out"}, + .constants = &.{ "scale", "n" }, + .run = run, + }); + + fn run(b: *metal.Builder, _: Cfg) metal.FinishError!void { + const a = try b.declareArgs(.{ + .a = .{ .buffer = .{ .type = .float, .const_ = true } }, + .b = .{ .buffer = .{ .type = .float, .const_ = true } }, + .out = .{ .buffer = .{ .type = .float } }, + .scale = .{ .constant = .float }, + .n = .{ .constant = .uint }, + .tid = .{ .thread_position_in_grid = .uint }, + }); + b.returnIf(a.tid.ge(a.n)); + a.out.store(a.tid, b.fma(a.a.load(a.tid), a.scale, a.b.load(a.tid))); + } +}; + +// In a model: +const out = Axpy.Kernel.call(.{ .a = x, .b = y }, .{ .out = x.shape() }, .{ + .cfg = .{}, + .grid = .{ 4, 1, 1 }, + .threadgroup = .{ 256, 1, 1 }, + .constants = .{ .scale = .of(@as(f32, 0.5)), .n = .of(@as(u32, 1000)) }, +}).out; +``` + +`declareArgs` declares the kernel parameters in order, which is the `[[buffer(n)]]` +order. `inputs` and `outputs` bind buffers to tensors by name. An output named +like an input updates that input's buffer in place. `constants` bind +`constant T &` arguments by value. `Constant.of` takes a scalar, or a host +`extern struct` laid out like the MSL struct. `CallOpts.threadgroup_memory` +sizes `[[threadgroup(0)]]`. + +Function constants have no runtime form. A kernel takes them through its `Cfg`, +so each configuration emits its own specialized kernel. + +## Arguments + +| MSL | `ArgSpec` | +|---|---| +| `device T *p`, `const device T *p` | `.buffer = .{ .type = T, .const_ = true }` | +| `constant T &x` | `.constant = T`, read wherever it is used, as C++ reads a reference | +| `constant S &s` | `.constant_struct = S`, read with `s.field("name")` | +| `threadgroup T *p [[threadgroup(0)]]` | `.threadgroup = T` | +| `uint3 tid [[thread_position_in_grid]]` and the other builtins | `.thread_position_in_grid = .uint3`, ... | + +## Values + +A `Value` is a scalar or vector (`Type`, spelled as in MSL: `.float4`, +`.ushort`), a pointer, a private array, a `simdgroup_matrix` or a constant +struct. Operations follow C++: integer promotion and the usual arithmetic +conversions pick the result type, signed arithmetic is `nsw`, and float math +carries the kernel's fast-math flags. A Zig number is the MSL literal it +spells, an `int` (a `long` when it does not fit) or a `float`, and a runtime +Zig integer keeps its width and signedness; the usual conversions follow, so +`h.mul(0.5)` on a `half` is a `float` multiply, as `h * 0.5` is in MSL. A +scalar next to a vector takes its element type. Stores, assignments and the +functions of one type (`max`, `min`, `clamp`, `fma`) convert a number to their +type. `0.5h` is `b.constant(.half, 0.5)`. + +| MSL | DSL | +|---|---| +| `a + b`, `a * b`, `a < b`, `a & b` | `a.add(b)`, `a.mul(b)`, `a.lt(b)`, `a.bitAnd(b)` | +| `(float) x`, `float4(v)` | `x.to(.float)`, `v.to(.float4)` | +| `as_type(x)` | `x.asType(.uint)` | +| `v.x`, `v[i]`, `v[i] = e`, `v.y = e` | `v.x()`, `v.at(i)`, `v = v.insert(i, e)`, `v = v.insert(1, e)`: a subscript takes the index's type (a literal is an `int`) | +| `p[i]`, `p[i] = v`, `p + i` | `p.load(i)`, `p.store(i, v)`, `p.add(i)` | +| `(threadgroup float4 *) p` | `p.to(.float4)` | +| `T x;`, `T x = e;`, `x = e` | `const x = b.local(.T, null)`, `b.local(.T, e)`, `x.assign(e)` | +| `device T *p = e;` that the code moves | `const p = b.localPointer(e)`, `p.assign(p.add(i))` | +| `float s[8];`, `float s[8] = {};` | `const s = b.array(.float, 8)`, `b.zeroedArray(.float, 8)`, then `s.load(i)` and `s.store(i, v)` | +| `&x` of a local | `x.address()` | +| `float4(a, b, c, d)`, `1.0f` | `b.vec(.float4, .{ a, b, c, d })`, `b.constant(.float, 1)` | +| `c ? x : y` | `b.ternary(c, x, y)`, a select of evaluated operands; `var t = b.openIfElse(c, .{.T})`, `x`, `t.yieldThen(.{x})`, `y`, `t.yieldElse(.{y})`, `t.results[0]` | +| `exp(x)`, `fast::exp(x)`, `precise::tanh(x)` | `b.exp(x)`, `b.fast().exp(x)`, `b.precise().tanh(x)` | +| `fma`, `dot`, `max`, `clamp` | `b.fma`, `b.dot`, `b.max`, `b.clamp` | +| `simd_sum(x)`, `simd_shuffle_xor(x, m)` | `b.simdSum(x)`, `b.simdShuffleXor(x, m)` | +| `threadgroup_barrier(mem_flags::mem_threadgroup)` | `b.threadgroupBarrier(.mem_threadgroup)` | +| `simdgroup_load(m, p, ld, origin, transpose)`, `simdgroup_store(m, p, ld)` | `const m = b.simdgroupLoad(p, ld, .{ .origin = ..., .transpose = true })`, `b.simdgroupStore(m, p, ld, .{})` | +| `simdgroup_multiply_accumulate(d, a, b, c)` | `const d = b.simdgroupMultiplyAccumulate(a, b, c)` | +| `m.thread_elements()` read, assigned | `m.threadElements()`; `const m = b.matrix(.T, elements)` | + +## Control flow + +| MSL | DSL | +|---|---| +| `for (T i = lower; i < upper; i += s)`, `++i` | `var l = b.openFor(lower, upper, s)`, `l.iv`, `l.close()`; a step of 1 is `++i` | +| `#pragma unroll`, `#pragma clang loop unroll(full)` | `b.openForOpts(..., .{ .unroll = .enable })`, `.{ .unroll = .full }` | +| `while (c) { ... }`, `for (init; c; inc)`, `for (;;)` | `var l = b.openLoop(.{})`, `l.body(c)`, the body, `l.next()`, the increment, `l.close()` | +| `break;` | `l.breakLoop()` | +| `if (c) { ... }` | `var s = b.openIf(c)`, `s.close()` | +| `if (c) { ... } else { ... }` | `var s = b.openIfElse(c, .{})`, `s.elseBranch()`, `s.close()` | +| `if (a \|\| (b && !x))` | `var c = b.condition()`, `c.or_(a)`, `c.and_(b)`, `var s = c.openIfNot(x)` | +| `x = a && b && c` | `var l = b.openAnd()`, `l.term(a)`, `l.term(b)`, `l.close(c)` | +| `return;` | `b.return_()`, or `b.returnIf(c)` | + +`condition`, `openAnd`, `openOr` and a ternary `openIfElse` skip a term or an +operand, so use them only when one must not run, such as a load past the end +of a buffer. Combine comparisons with `bitAnd` and `bitOr`, and pick between +values that are safe to compute with `b.ternary`. + +Put each `if` or loop body in a Zig `{ ... }` block right after the opener, +one block per region, each ending with the call that ends the region: +`elseBranch()` or `yieldThen`, then `close()` or `yieldElse` for an `if`; +`body(c)`, `next()`, then `close()` for `openLoop`. The scope variable stays +outside, so its `results` stay readable after the blocks. + +A value is evaluated where it is written; a Zig `const` holds it, and an MSL +function is a Zig function that emits its body. A value a loop or branch +changes is a local, which lives in memory until the pipeline promotes it. +The terms of `&&` and `||` run only when the ones before leave the result +open, and an `openIfElse(c, .{T})` operand only when `c` selects it, each in +its own block. Loops follow C++: every +loop is `mustprogress`, and its condition and increment follow C++'s +conversions, so a `short` loop variable compares as an `int` and `++i` steps +it as a `short`. + +A `simdgroup_matrix` is a value of its 64 elements, of which a lane holds the +first two; the load, store and multiply-accumulate call the +`air.simdgroup_matrix_8x8_*` builtins. + +## Checking a port + +Fast math lets Apple's backend reassociate and contract float math, so a port +computes what its MSL kernel computes up to rounding, not always to the bit. +Compare its outputs with the MSL kernel's on the same inputs, and time both +with the same grid, threadgroup size and threadgroup memory. diff --git a/dsl/metal/builder.zig b/dsl/metal/builder.zig new file mode 100644 index 00000000..dea3d48a --- /dev/null +++ b/dsl/metal/builder.zig @@ -0,0 +1,2320 @@ +const std = @import("std"); + +const air = @import("mlir/dialects/air"); +const arith = @import("mlir/dialects/arith"); +const cf = @import("mlir/dialects/cf"); +const common = @import("dsl/common/builder"); +const func = @import("mlir/dialects/func"); +const llvm = @import("mlir/dialects/llvm"); +const mlir = @import("mlir"); +const vector = @import("mlir/dialects/vector"); + +pub const types = @import("types.zig"); + +pub const Scalar = types.Scalar; +pub const Type = types.Type; +pub const Struct = types.Struct; +pub const StructField = types.StructField; +pub const Builtin = air.Builtin; + +pub fn registerDialects(registry: *mlir.DialectRegistry) void { + inline for (.{ air, arith, cf, func, llvm, vector }) |dialect| dialect.registerDialect(registry); +} + +pub const FinishError = error{InvalidMlir} || std.mem.Allocator.Error || std.Io.Writer.Error; + +test { + std.testing.refAllDecls(@This()); + std.testing.refAllDecls(Builder); + std.testing.refAllDecls(Value); +} + +pub const AddressSpace = air.AddressSpace; + +/// `mem_flags`: `.mem_threadgroup`, or `.{ .device = true, .threadgroup = true }` +/// for `mem_device | mem_threadgroup`. +pub const MemFlags = packed struct(u32) { + device: bool = false, + threadgroup: bool = false, + texture: bool = false, + _padding: u29 = 0, + + pub const mem_none: MemFlags = .{}; + pub const mem_device: MemFlags = .{ .device = true }; + pub const mem_threadgroup: MemFlags = .{ .threadgroup = true }; + pub const mem_texture: MemFlags = .{ .texture = true }; +}; + +/// A kernel argument, as `Builder.declareArgs` takes it. Device and constant +/// arguments take the next `[[buffer(n)]]`, threadgroup arguments the next +/// `[[threadgroup(n)]]`, in declaration order. +pub const ArgSpec = union(enum) { + /// `device T *p`, or `const device T *p` with `.const_`. + buffer: Buffer, + /// `constant T &x`; the argument is its value. + constant: Type, + /// `constant S &s`; `Value.field` reads a field. + constant_struct: Struct, + /// `threadgroup T *p`. + threadgroup: Type, + thread_position_in_grid: Type, + threadgroup_position_in_grid: Type, + thread_position_in_threadgroup: Type, + threads_per_threadgroup: Type, + threadgroups_per_grid: Type, + threads_per_grid: Type, + thread_index_in_simdgroup: Type, + simdgroup_index_in_threadgroup: Type, + threads_per_simdgroup: Type, + simdgroups_per_threadgroup: Type, + thread_index_in_threadgroup: Type, + + pub const Buffer = struct { + type: Type, + const_: bool = false, + /// `__restrict__`. + restrict: bool = false, + }; +}; + +/// `raw` as an `ArgSpec`: an `ArgSpec`, or a literal of one of its variants +/// such as `.{ .buffer = .{ .type = .float } }`. +fn argSpec(raw: anytype) ArgSpec { + if (@TypeOf(raw) == ArgSpec) return raw; + const variant = comptime literalVariant(@TypeOf(raw), "ArgSpec"); + const inner = @field(raw, variant); + return switch (@field(std.meta.Tag(ArgSpec), variant)) { + .buffer => blk: { + comptime knownFields(@TypeOf(inner), ArgSpec.Buffer, "ArgSpec.buffer"); + break :blk .{ .buffer = .{ + .type = typeOf(inner.type), + .const_ = if (@hasField(@TypeOf(inner), "const_")) inner.const_ else false, + .restrict = if (@hasField(@TypeOf(inner), "restrict")) inner.restrict else false, + } }; + }, + .constant_struct => .{ .constant_struct = inner }, + inline else => |tag| @unionInit(ArgSpec, @tagName(tag), typeOf(inner)), + }; +} + +/// The variant a spec literal names: one, as a union literal does. +fn literalVariant(Literal: type, what: []const u8) []const u8 { + const names = @typeInfo(Literal).@"struct".field_names; + if (names.len != 1) @compileError("dsl/metal: " ++ what ++ " literals name one variant, not " ++ std.fmt.comptimePrint("{d}", .{names.len})); + return names[0]; +} + +/// Rejects a field of a spec literal that `Options` does not have. +fn knownFields(Literal: type, Options: type, what: []const u8) void { + for (@typeInfo(Literal).@"struct".field_names) |name| { + if (!@hasField(Options, name)) @compileError("dsl/metal: " ++ what ++ " has no field " ++ name); + } +} + +/// A `Type`, or the name of one (`.float`). +fn typeOf(t: anytype) Type { + return if (@TypeOf(t) == @EnumLiteral()) @field(Type, @tagName(t)) else t; +} + +/// A declared device, constant or threadgroup argument, for binding. +pub const Argument = struct { + name: []const u8, + kind: enum { buffer, constant, threadgroup }, + /// The `[[buffer(n)]]` or `[[threadgroup(n)]]` index. + index: u32, + /// The size of a constant argument. + size: u64 = 0, +}; + +/// A pointer: its address space and pointee type. +pub const Pointer = struct { + space: AddressSpace, + pointee: Type, +}; + +/// An SSA value with its MSL type. Operators follow C++: operands take the +/// usual arithmetic conversions, integers narrower than `int` are promoted, +/// and signedness picks the operation (`uint / int` divides unsigned). +pub const Value = struct { + inner: *const mlir.Value, + kernel: ?*Builder = null, + kind: Kind = undefined, + + pub const Kind = union(enum) { + value: Type, + pointer: Pointer, + /// A `constant S &` argument. + constant_struct: Struct, + /// A `constant T &` argument or a local: an lvalue, read where it is + /// used as C++ reads it. + reference: Pointer, + /// A `simdgroup_matrix`: its 64 elements, of which a lane + /// holds the first two. + matrix: Scalar, + /// A pointer local. + pointer_reference: Pointer, + }; + + /// The value itself, an lvalue read. + pub fn rvalue(self: Value) Value { + return switch (self.kind) { + .reference => |p| (Value{ .inner = self.inner, .kernel = self.kernel, .kind = .{ .pointer = p } }).load(0), + .pointer_reference => |p| blk: { + const k = self.kern(); + break :blk k.emit(llvm.load(k.ctx, self.inner, llvm.pointerType(k.ctx, p.space), .{ .alignment = 8 }, k.loc()), .{ .pointer = p }); + }, + else => self, + }; + } + + /// `&x` of a local. + pub fn address(reference: Value) Value { + return switch (reference.kind) { + .reference => |p| .{ .inner = reference.inner, .kernel = reference.kernel, .kind = .{ .pointer = p } }, + else => std.debug.panic("dsl/metal: the address of a {t}", .{reference.kind}), + }; + } + + /// `x = value` of a local, converting `value` to its type. + pub fn assign(self: Value, value: anytype) void { + const k = self.kern(); + switch (self.kind) { + .reference => |p| { + if (p.space != .thread) std.debug.panic("dsl/metal: assigning to a {t} reference", .{p.space}); + (Value{ .inner = self.inner, .kernel = k, .kind = .{ .pointer = p } }).store(0, value); + }, + .pointer_reference => |p| { + const v = k.lift(value); + if (v.kind != .pointer or v.kind.pointer.space != p.space) std.debug.panic("dsl/metal: assigning a {t} to a {t} pointer", .{ v.kind, p.space }); + _ = llvm.store(k.ctx, v.inner, self.inner, .{ .alignment = 8 }, k.loc()).appendTo(k.currentBlock()); + }, + else => std.debug.panic("dsl/metal: assigning to a {t}", .{self.kind}), + } + } + + fn kern(self: Value) *Builder { + return self.kernel orelse @panic("dsl/metal: value has no builder"); + } + + /// The MSL type of a scalar or vector value. + pub fn type_(self: Value) Type { + return switch (self.kind) { + .value => |t| t, + .reference => |p| p.pointee, + else => std.debug.panic("dsl/metal: expected a scalar or vector, got a {t}", .{self.kind}), + }; + } + + pub fn pointer(self: Value) Pointer { + return switch (self.kind) { + .pointer => |p| p, + else => std.debug.panic("dsl/metal: expected a pointer, got a {t}", .{self.kind}), + }; + } + + pub fn add(reference: Value, rhs: anytype) Value { + const self = reference.rvalue(); + if (self.kind == .pointer) return self.kern().offset(self, self.kern().promoted(rhs), false); + return self.kern().arithmetic(.add, self, rhs); + } + pub fn sub(reference: Value, rhs: anytype) Value { + const self = reference.rvalue(); + if (self.kind == .pointer) return self.kern().offset(self, self.kern().promoted(rhs), true); + return self.kern().arithmetic(.sub, self, rhs); + } + pub fn mul(reference: Value, rhs: anytype) Value { + const self = reference.rvalue(); + return self.kern().arithmetic(.mul, self, rhs); + } + pub fn div(reference: Value, rhs: anytype) Value { + const self = reference.rvalue(); + return self.kern().arithmetic(.div, self, rhs); + } + pub fn rem(reference: Value, rhs: anytype) Value { + const self = reference.rvalue(); + return self.kern().arithmetic(.rem, self, rhs); + } + pub fn bitAnd(reference: Value, rhs: anytype) Value { + const self = reference.rvalue(); + return self.kern().arithmetic(.bit_and, self, rhs); + } + pub fn bitOr(reference: Value, rhs: anytype) Value { + const self = reference.rvalue(); + return self.kern().arithmetic(.bit_or, self, rhs); + } + pub fn bitXor(reference: Value, rhs: anytype) Value { + const self = reference.rvalue(); + return self.kern().arithmetic(.bit_xor, self, rhs); + } + pub fn shl(reference: Value, rhs: anytype) Value { + const self = reference.rvalue(); + return self.kern().shift(.left, self, rhs); + } + pub fn shr(reference: Value, rhs: anytype) Value { + const self = reference.rvalue(); + return self.kern().shift(.right, self, rhs); + } + pub fn neg(reference: Value) Value { + const self = reference.rvalue(); + return self.kern().negate(self); + } + /// `~x`. + pub fn bitNot(reference: Value) Value { + const self = reference.rvalue(); + const k = self.kern(); + const v = k.convert(self, self.type_().withScalar(self.type_().scalar.promoted())); + return v.bitXor(k.constant(v.type_(), -1)); + } + + pub fn lt(reference: Value, rhs: anytype) Value { + const self = reference.rvalue(); + return self.kern().compare(.lt, self, rhs); + } + pub fn le(reference: Value, rhs: anytype) Value { + const self = reference.rvalue(); + return self.kern().compare(.le, self, rhs); + } + pub fn gt(reference: Value, rhs: anytype) Value { + const self = reference.rvalue(); + return self.kern().compare(.gt, self, rhs); + } + pub fn ge(reference: Value, rhs: anytype) Value { + const self = reference.rvalue(); + return self.kern().compare(.ge, self, rhs); + } + pub fn eq(reference: Value, rhs: anytype) Value { + const self = reference.rvalue(); + return self.kern().compare(.eq, self, rhs); + } + pub fn ne(reference: Value, rhs: anytype) Value { + const self = reference.rvalue(); + return self.kern().compare(.ne, self, rhs); + } + + /// `T(x)`, `(T) x` or `static_cast(x)` on a scalar or vector; on a + /// pointer, `(space T *) p`. + pub fn to(reference: Value, t: Type) Value { + const self = reference.rvalue(); + const k = self.kern(); + return switch (self.kind) { + .pointer => |p| .{ .inner = self.inner, .kernel = k, .kind = .{ .pointer = .{ .space = p.space, .pointee = t } } }, + else => k.convert(self, t), + }; + } + + /// `as_type(x)`: the same bits as another type of the same size. + pub fn asType(reference: Value, t: Type) Value { + const self = reference.rvalue(); + const k = self.kern(); + const from = self.type_(); + if (from.size() != t.size() or from.len == 3 or t.len == 3) + std.debug.panic("dsl/metal: as_type<{s}> of a {s}", .{ t.name(), from.name() }); + if (from.eql(t)) return self; + return k.emit(llvm.bitcast(k.ctx, self.inner, t.mlirType(k.ctx), k.loc()), .{ .value = t }); + } + + /// `v.x`. + pub fn x(reference: Value) Value { + const self = reference.rvalue(); + return self.at(0); + } + pub fn y(reference: Value) Value { + const self = reference.rvalue(); + return self.at(1); + } + pub fn z(reference: Value) Value { + const self = reference.rvalue(); + return self.at(2); + } + pub fn w(reference: Value) Value { + const self = reference.rvalue(); + return self.at(3); + } + + /// `v[i]` of a vector; the index keeps its type, a literal is an `int`. + pub fn at(reference: Value, i: anytype) Value { + const self = reference.rvalue(); + const k = self.kern(); + const t = self.type_(); + if (!t.isVector()) std.debug.panic("dsl/metal: indexing a {s}", .{t.name()}); + const element = t.element(); + return k.emit(llvm.extractelement(k.ctx, self.inner, k.subscript(i).inner, element.mlirType(k.ctx), k.loc()), .{ .value = element }); + } + + /// The vector with `v[i] = element`, `i` typed as `at` types it; `v.y = + /// element` is `insert(@as(i64, 1), element)`. + pub fn insert(reference: Value, i: anytype, element: anytype) Value { + const self = reference.rvalue(); + const k = self.kern(); + const t = self.type_(); + const e = k.convert(k.lift(element), t.element()); + return k.emit(llvm.insertelement(k.ctx, self.inner, e.inner, k.subscript(i).inner, k.loc()), .{ .value = t }); + } + + /// `p[i]`. + pub fn load(reference: Value, i: anytype) Value { + const self = reference.rvalue(); + const k = self.kern(); + const p = self.pointer(); + const at_ = k.offset(self, i, false); + return k.loadMemory(at_.inner, p.pointee); + } + + /// `p[i] = value`, converting `value` to the pointee type. + pub fn store(reference: Value, i: anytype, value: anytype) void { + const self = reference.rvalue(); + const k = self.kern(); + const p = self.pointer(); + var v = k.convert(k.lift(value), p.pointee); + if (p.pointee.scalar == .bool) v = k.emit(arith.extui(k.ctx, v.inner, p.pointee.memory().mlirType(k.ctx), k.loc()), .{ .value = p.pointee.memory() }); + const at_ = k.offset(self, i, false); + _ = llvm.store(k.ctx, v.inner, at_.inner, .{ .alignment = v.type_().alignment() }, k.loc()).appendTo(k.currentBlock()); + } + + /// `s.name` of a constant struct argument. + pub fn field(reference: Value, comptime name: []const u8) Value { + const self = reference.rvalue(); + const k = self.kern(); + const s = switch (self.kind) { + .constant_struct => |s| s, + else => std.debug.panic("dsl/metal: .{s} of a {t}", .{ name, self.kind }), + }; + const index = s.fieldIndex(name); + const f = s.fields[index]; + if (f.count != 0) std.debug.panic("dsl/metal: struct array field {s}", .{name}); + const gep = llvm.getelementptr(k.ctx, self.inner, &.{ .{ .constant = 0 }, .{ .constant = @intCast(index) } }, k.structType(s), k.loc()); + _ = gep.appendTo(k.currentBlock()); + return k.loadMemory(gep.result(0), f.type); + } + + /// `m.thread_elements()` of a `simdgroup_matrix`: the lane's two + /// elements. + pub fn threadElements(m: Value) Value { + const k = m.kern(); + const t: Type = .vec(Builder.matrixScalar(m), 2); + return k.emit(llvm.shufflevector(k.ctx, m.inner, m.inner, t.mlirType(k.ctx), &.{ 0, 1 }, k.loc()), .{ .value = t }); + } +}; + +/// A loop: `while (c)` and `for (init; c; inc)` branch to the body with +/// `body(c)`, `for (;;)` has its body right away; `next()` ends the body +/// and starts the increment, and `close()` ends the loop. +/// +/// A `break` and the end of the body meet in a latch that reads whether to +/// leave from a local, as clang's cleanup does: Apple's compiler then keeps +/// the loop's test at the top instead of rotating the loop, which is faster +/// (paged attention's decode loop by 2%). +pub const LoopScope = struct { + kernel: *Builder, + cond_block: *mlir.Block, + end_block: *mlir.Block, + loop_annotation: *const llvm.LoopAnnotationAttr, + /// Where a `break` goes after setting `leave`; null until the first one. + latch: ?*mlir.Block = null, + leave: Value = undefined, + exits: bool = false, + + /// Leaves the loop when `cond` is false. + pub fn body(self: *LoopScope, cond: Value) void { + const k = self.kernel; + const body_block = mlir.Block.init(&.{}, &.{}); + k.condBranch(cond, body_block, self.end_block); + self.exits = true; + k.startBlock(body_block); + } + + /// `break;`. + pub fn breakLoop(self: *LoopScope) void { + if (self.kernel.insert == null) return; + self.toLatch(true); + self.exits = true; + } + + fn toLatch(self: *LoopScope, leave: bool) void { + const k = self.kernel; + if (self.latch == null) { + self.latch = mlir.Block.init(&.{}, &.{}); + self.leave = k.local(.bool, null); + } + self.leave.assign(leave); + _ = k.branch(self.latch.?); + } + + /// Ends the body: the increment of `for (...; ...; inc)` follows. + pub fn next(self: *LoopScope) void { + const k = self.kernel; + const inc = mlir.Block.init(&.{}, &.{}); + if (self.latch) |latch| { + if (k.insert != null) self.toLatch(false); + k.startBlock(latch); + k.condBranch(self.leave, self.end_block, inc); + self.latch = null; + } else _ = k.branch(inc); + k.startBlock(inc); + } + + pub fn close(self: *LoopScope) void { + const k = self.kernel; + if (self.latch != null) self.next(); + k.branchLoop(self.cond_block, self.loop_annotation); + if (self.exits) k.startBlock(self.end_block) else self.end_block.deinit(); + } +}; + +/// `for (T i = lower; i < upper; i += step)`: `i` is the local `iv`, the +/// condition and the increment follow C++'s conversions, and a step of 1 is +/// `++i`. +pub const ForScope = struct { + loop: LoopScope, + iv: Value, + step: Step, + + const Step = union(enum) { one, value: Value, int: i64 }; + + pub fn breakLoop(self: *ForScope) void { + self.loop.breakLoop(); + } + + pub fn close(self: *ForScope) void { + const k = self.loop.kernel; + self.loop.next(); + if (k.insert != null) switch (self.step) { + .one => self.iv.assign(k.increment(self.iv)), + .value => |v| { + const s = v.rvalue(); + self.iv.assign(self.iv.add(s)); + }, + .int => |n| self.iv.assign(self.iv.add(k.constant(if (std.math.cast(i32, n) != null) .int else .long, n))), + }; + self.loop.close(); + } +}; + +/// `if (c) { ... }`, or `if (c) { ... } else { ... }` with `elseBranch()` +/// between the two. +pub const IfScope = struct { + kernel: *Builder, + else_block: ?*mlir.Block, + end_block: *mlir.Block, + joins: bool = false, + + pub fn elseBranch(self: *IfScope) void { + const k = self.kernel; + const else_block = self.else_block orelse @panic("dsl/metal: elseBranch of an if without else (open it with openIfElse(c, .{})), or a second one"); + if (k.branch(self.end_block)) self.joins = true; + k.startBlock(else_block); + self.else_block = null; + } + + pub fn close(self: *IfScope) void { + const k = self.kernel; + if (k.branch(self.end_block)) self.joins = true; + if (self.joins) k.startBlock(self.end_block) else self.end_block.deinit(); + } +}; + +/// A condition of `&&`, `||` and `!` terms, short-circuited: a term's code +/// runs only when the terms before it leave the outcome open. `and_(x)` is +/// `x && ...`, `or_(x)` is `x || ...`; the last term opens what the condition +/// guards. A group of `&&` terms ends a `||` chain and the +/// other way around, as in `a || (b && c)`. +pub const Condition = struct { + kernel: *Builder, + true_block: *mlir.Block, + false_block: *mlir.Block, + + pub fn and_(self: *Condition, x: Value) void { + self.term(x, false, .@"and"); + } + pub fn andNot(self: *Condition, x: Value) void { + self.term(x, true, .@"and"); + } + pub fn or_(self: *Condition, x: Value) void { + self.term(x, false, .@"or"); + } + pub fn orNot(self: *Condition, x: Value) void { + self.term(x, true, .@"or"); + } + + fn term(self: *Condition, x: Value, negate: bool, comptime op: enum { @"and", @"or" }) void { + const k = self.kernel; + const next = mlir.Block.init(&.{}, &.{}); + const on_true, const on_false = switch (op) { + .@"and" => .{ next, self.false_block }, + .@"or" => .{ self.true_block, next }, + }; + if (negate) k.condBranch(x, on_false, on_true) else k.condBranch(x, on_true, on_false); + k.startBlock(next); + } + + fn last(self: *Condition, x: Value, negate: bool) void { + if (negate) self.kernel.condBranch(x, self.false_block, self.true_block) else self.kernel.condBranch(x, self.true_block, self.false_block); + } + + /// `if (... x) { ... }`. + pub fn openIf(self: *Condition, x: Value) IfScope { + return self.opened(x, false, false); + } + /// `if (... !x) { ... }`. + pub fn openIfNot(self: *Condition, x: Value) IfScope { + return self.opened(x, true, false); + } + /// `if (... x) { ... } else { ... }`; a chain has no result types. + pub fn openIfElse(self: *Condition, x: Value, result_types: anytype) IfScope { + noResults(result_types); + return self.opened(x, false, true); + } + /// `if (... !x) { ... } else { ... }`. + pub fn openIfElseNot(self: *Condition, x: Value, result_types: anytype) IfScope { + noResults(result_types); + return self.opened(x, true, true); + } + + fn noResults(result_types: anytype) void { + if (common.arity(@TypeOf(result_types), "Condition.openIfElse: result_types") != 0) + @compileError("dsl/metal: a condition chain opens no `c ? a : b`"); + } + + fn opened(self: *Condition, x: Value, negate: bool, with_else: bool) IfScope { + const k = self.kernel; + self.last(x, negate); + k.startBlock(self.true_block); + if (!with_else) return .{ .kernel = k, .else_block = null, .end_block = self.false_block, .joins = true }; + return .{ .kernel = k, .else_block = self.false_block, .end_block = mlir.Block.init(&.{}, &.{}) }; + } +}; + +/// `a && b && ...` or `a || b || ...` as a bool, short-circuited: a +/// term's code runs only when the terms before it leave the result open. +/// `term(x)` adds a term followed by more, `close(x)` the last one. +pub const Logical = struct { + kernel: *Builder, + op: enum { @"and", @"or" }, + end_block: *mlir.Block, + + pub fn term(self: *Logical, x: Value) void { + const k = self.kernel; + const next = mlir.Block.init(&.{}, &.{}); + const c = k.convert(x, .bool); + const decided: []const *const mlir.Value = &.{k.constant(.bool, self.op == .@"or").inner}; + _ = switch (self.op) { + .@"and" => cf.cond_br(k.ctx, c.inner, next, &.{}, self.end_block, decided, .{}, k.loc()), + .@"or" => cf.cond_br(k.ctx, c.inner, self.end_block, decided, next, &.{}, .{}, k.loc()), + }.appendTo(k.currentBlock()); + k.insert = null; + k.startBlock(next); + } + + pub fn close(self: *Logical, x: Value) Value { + const k = self.kernel; + const c = k.convert(x, .bool); + _ = cf.br(k.ctx, self.end_block, &.{c.inner}, k.loc()).appendTo(k.currentBlock()); + k.insert = null; + k.startBlock(self.end_block); + return .{ .inner = self.end_block.argument(0), .kernel = k, .kind = .{ .value = .bool } }; + } +}; + +/// `c ? a : b` evaluating one operand: the true operand's code, then +/// `yieldThen(.{a})`, the false operand's code, then `yieldElse(.{b})`; +/// `results[0]` is the value. +pub const TernaryScope = struct { + kernel: *Builder, + type: Type, + else_block: *mlir.Block, + end_block: *mlir.Block, + results: [1]Value = undefined, + + pub fn yieldThen(self: *TernaryScope, values: anytype) void { + const k = self.kernel; + self.branchEnd(values, "yieldThen"); + k.startBlock(self.else_block); + } + + pub fn yieldElse(self: *TernaryScope, values: anytype) void { + const k = self.kernel; + self.branchEnd(values, "yieldElse"); + k.startBlock(self.end_block); + self.results = .{.{ .inner = self.end_block.argument(0), .kernel = k, .kind = .{ .value = self.type } }}; + } + + fn branchEnd(self: *TernaryScope, values: anytype, comptime what: []const u8) void { + if (common.arity(@TypeOf(values), "TernaryScope." ++ what) != 1) @compileError("TernaryScope." ++ what ++ " takes one value"); + const k = self.kernel; + const v = k.convert(k.lift(values[0]), self.type); + _ = cf.br(k.ctx, self.end_block, &.{v.inner}, k.loc()).appendTo(k.currentBlock()); + k.insert = null; + } +}; + +/// What `Builder.openIfElse` opens for `n` result types. +fn IfElseScope(comptime n: usize) type { + return switch (n) { + 0 => IfScope, + 1 => TernaryScope, + else => @compileError("dsl/metal: openIfElse has one result at most, as `c ? a : b`"), + }; +} + +/// A loop pragma: `full` is `#pragma clang loop unroll(full)`, `enable` +/// `#pragma unroll`. +pub const Unroll = enum { full, enable }; + +pub const LoopOpts = struct { + unroll: ?Unroll = null, +}; + +/// MSL's `metal::fast` and `metal::precise` functions. +pub const MathMode = enum { default, fast, precise }; + +/// `metal::fast::` or `metal::precise::`. +pub const Namespace = struct { + b: *Builder, + mode: MathMode, + + pub fn exp(self: Namespace, x: Value) Value { + return self.b.floatMath("exp", self.mode, .{x}); + } + pub fn exp2(self: Namespace, x: Value) Value { + return self.b.floatMath("exp2", self.mode, .{x}); + } + pub fn log(self: Namespace, x: Value) Value { + return self.b.floatMath("log", self.mode, .{x}); + } + pub fn log2(self: Namespace, x: Value) Value { + return self.b.floatMath("log2", self.mode, .{x}); + } + pub fn tanh(self: Namespace, x: Value) Value { + return self.b.floatMath("tanh", self.mode, .{x}); + } + pub fn sqrt(self: Namespace, x: Value) Value { + return self.b.floatMath("sqrt", self.mode, .{x}); + } + pub fn rsqrt(self: Namespace, x: Value) Value { + return self.b.floatMath("rsqrt", self.mode, .{x}); + } + pub fn sin(self: Namespace, x: Value) Value { + return self.b.floatMath("sin", self.mode, .{x}); + } + pub fn cos(self: Namespace, x: Value) Value { + return self.b.floatMath("cos", self.mode, .{x}); + } + pub fn max(self: Namespace, a: Value, b: anytype) Value { + return self.b.floatMath("fmax", self.mode, .{ a, b }); + } + pub fn min(self: Namespace, a: Value, b: anytype) Value { + return self.b.floatMath("fmin", self.mode, .{ a, b }); + } +}; + +pub const Builder = struct { + allocator: std.mem.Allocator, + arena: std.heap.ArenaAllocator, + ctx: *mlir.Context, + module: *mlir.Module, + name: []const u8, + fast_math: bool, + func_op: ?*mlir.Operation = null, + entry_block: ?*mlir.Block = null, + /// The kernel's body; blocks join it in the order they start. + body: ?*mlir.Region = null, + /// The block code goes to; null after a `break` or a `return`. + insert: ?*mlir.Block = null, + arguments: std.ArrayList(Argument) = .empty, + + pub const Opts = struct { + /// MSL's default; `-fno-fast-math` otherwise. + fast_math: bool = true, + }; + + pub fn open(allocator: std.mem.Allocator, ctx: *mlir.Context, name: []const u8, opts: Opts) Builder { + const module: *mlir.Module = .init(.unknown(ctx)); + return .{ + .allocator = allocator, + .arena = .init(allocator), + .ctx = ctx, + .module = module, + .name = name, + .fast_math = opts.fast_math, + }; + } + + pub fn deinit(self: *Builder) void { + self.module.deinit(); + self.arena.deinit(); + } + + /// Declares the kernel's arguments, in order, from a struct literal of + /// `ArgSpec`s, and returns them by name. + pub fn declareArgs(self: *Builder, spec: anytype) FinishError!common.NamedArgs(@TypeOf(spec), Value) { + const ctx = self.ctx; + const location = self.loc(); + const fields = @typeInfo(@TypeOf(spec)).@"struct".field_names; + const arena = self.arena.allocator(); + var arg_types: [fields.len]*const mlir.Type = undefined; + var arg_attrs: [fields.len]*const mlir.Attribute = undefined; + var kinds: [fields.len]Value.Kind = undefined; + var buffers: u32 = 0; + var threadgroups: u32 = 0; + inline for (fields, 0..) |name, i| { + const arg = argSpec(@field(spec, name)); + const opts: air.ArgumentOpts = .{ .name = name }; + switch (arg) { + .buffer => |buffer| { + arg_types[i] = llvm.pointerType(ctx, AddressSpace.device); + arg_attrs[i] = air.bufferArgumentAttr(ctx, .{ + .locationIndex = buffers, + .addressSpace = .device, + .access = if (buffer.const_) .read else .read_write, + .elementType = buffer.type.memory().mlirType(ctx), + .typeName = buffer.type.name(), + .typeSize = buffer.type.size(), + .typeAlignment = buffer.type.alignment(), + .noalias_ = buffer.restrict, + }, opts) catch return error.InvalidMlir; + kinds[i] = .{ .pointer = .{ .space = .device, .pointee = buffer.type } }; + try self.arguments.append(arena, .{ .name = name, .kind = .buffer, .index = buffers }); + buffers += 1; + }, + .constant => |t| { + arg_types[i] = llvm.pointerType(ctx, AddressSpace.constant); + arg_attrs[i] = air.bufferArgumentAttr(ctx, .{ + .locationIndex = buffers, + .addressSpace = .constant, + .access = .read, + .elementType = t.memory().mlirType(ctx), + .typeName = t.name(), + .typeSize = t.size(), + .typeAlignment = t.alignment(), + .bufferSize = t.size(), + }, opts) catch return error.InvalidMlir; + kinds[i] = .{ .reference = .{ .space = .constant, .pointee = t } }; + try self.arguments.append(arena, .{ .name = name, .kind = .constant, .index = buffers, .size = t.size() }); + buffers += 1; + }, + .constant_struct => |s| { + var struct_fields = try arena.alloc(*const air.StructFieldAttr, s.fields.len); + for (s.fields, 0..) |f, j| struct_fields[j] = air.StructFieldAttr.get(ctx, .{ + .offset = s.offset(j), + .size = f.type.size(), + .count = f.count, + .typeName = f.type.name(), + .name = f.name, + }) catch return error.InvalidMlir; + arg_types[i] = llvm.pointerType(ctx, AddressSpace.constant); + arg_attrs[i] = air.bufferArgumentAttr(ctx, .{ + .locationIndex = buffers, + .addressSpace = .constant, + .access = .read, + .elementType = self.structType(s), + .typeName = s.name, + .typeSize = s.size(), + .typeAlignment = s.alignment(), + .bufferSize = s.size(), + .fields = struct_fields, + }, opts) catch return error.InvalidMlir; + kinds[i] = .{ .constant_struct = s }; + try self.arguments.append(arena, .{ .name = name, .kind = .constant, .index = buffers, .size = s.size() }); + buffers += 1; + }, + .threadgroup => |t| { + arg_types[i] = llvm.pointerType(ctx, AddressSpace.threadgroup); + arg_attrs[i] = air.bufferArgumentAttr(ctx, .{ + .locationIndex = threadgroups, + .addressSpace = .threadgroup, + .access = .read_write, + .elementType = t.memory().mlirType(ctx), + .typeName = t.name(), + .typeSize = t.size(), + .typeAlignment = t.alignment(), + }, opts) catch return error.InvalidMlir; + kinds[i] = .{ .pointer = .{ .space = .threadgroup, .pointee = t } }; + try self.arguments.append(arena, .{ .name = name, .kind = .threadgroup, .index = threadgroups }); + threadgroups += 1; + }, + inline else => |t, builtin| { + arg_types[i] = t.mlirType(ctx); + arg_attrs[i] = air.builtinArgumentAttr(ctx, @field(air.Builtin, @tagName(builtin)), opts) catch return error.InvalidMlir; + kinds[i] = .{ .value = t }; + }, + } + } + var locs: [fields.len]*const mlir.Location = @splat(location); + const block = mlir.Block.init(&arg_types, &locs); + self.entry_block = block; + self.func_op = air.kernel(ctx, .{ + .name = self.name, + .block = block, + .args_attributes = &arg_attrs, + .location = location, + .verify = false, + }).appendTo(self.module.body()); + self.body = block.parentRegion(); + self.insert = block; + + var out: common.NamedArgs(@TypeOf(spec), Value) = undefined; + inline for (fields, 0..) |name, i| @field(out, name) = .{ .inner = block.argument(i), .kernel = self, .kind = kinds[i] }; + return out; + } + + pub fn loc(self: *const Builder) *const mlir.Location { + return .unknown(self.ctx); + } + + /// The block code goes to: a fresh, unreachable one after a `break` or + /// a `return`. + pub fn currentBlock(self: *Builder) *mlir.Block { + if (self.insert == null) self.startBlock(mlir.Block.init(&.{}, &.{})); + return self.insert.?; + } + + fn startBlock(self: *Builder, block: *mlir.Block) void { + self.body.?.appendOwnedBlock(block); + self.insert = block; + } + + /// Branches to `dest` unless the code here is unreachable; true if it did. + fn branch(self: *Builder, dest: *mlir.Block) bool { + const block = self.insert orelse return false; + _ = cf.br(self.ctx, dest, &.{}, self.loc()).appendTo(block); + self.insert = null; + return true; + } + + fn condBranch(self: *Builder, cond: Value, on_true: *mlir.Block, on_false: *mlir.Block) void { + const c = self.convert(cond, .bool); + _ = cf.cond_br(self.ctx, c.inner, on_true, &.{}, on_false, &.{}, .{}, self.loc()).appendTo(self.currentBlock()); + self.insert = null; + } + + /// The back edge of a loop, which carries its `llvm.loop` metadata. + fn branchLoop(self: *Builder, dest: *mlir.Block, annotation: *const llvm.LoopAnnotationAttr) void { + const block = self.insert orelse return; + _ = llvm.br(self.ctx, dest, &.{}, .{ .loop_annotation = annotation }, self.loc()).appendTo(block); + self.insert = null; + } + + /// A thread-memory slot of `t`, at the top of the entry block. + fn allocate(self: *Builder, t: *const mlir.Type, alignment: u64) *const mlir.Value { + const ctx = self.ctx; + const entry = self.entry_block.?; + const size = arith.constant_int(ctx, 1, .int(ctx, .i32), self.loc()); + const alloca = llvm.alloca(ctx, size.result(0), llvm.pointerType(ctx, AddressSpace.thread), t, .{ .alignment = alignment }, self.loc()); + if (entry.firstOperation()) |first| entry.insertOwnedOperationBefore(first, alloca) else _ = alloca.appendTo(entry); + entry.insertOwnedOperationBefore(alloca, size); + return alloca.result(0); + } + + /// `T x;`, or `T x = init;`: a local, read where it is used and written by + /// `assign`. + pub fn local(self: *Builder, t: Type, init: anytype) Value { + const memory = t.memory(); + const x: Value = .{ .inner = self.allocate(memory.mlirType(self.ctx), memory.alignment()), .kernel = self, .kind = .{ .reference = .{ .space = .thread, .pointee = t } } }; + if (@TypeOf(init) != @TypeOf(null)) x.assign(init); + return x; + } + + /// `T *p = init;`: a pointer local. + pub fn localPointer(self: *Builder, init: Value) Value { + const p: Pointer = switch (init.kind) { + .pointer, .pointer_reference => |q| q, + else => std.debug.panic("dsl/metal: a pointer local of a {t}", .{init.kind}), + }; + const x: Value = .{ .inner = self.allocate(llvm.pointerType(self.ctx, p.space), 8), .kernel = self, .kind = .{ .pointer_reference = p } }; + x.assign(init); + return x; + } + + /// A `simdgroup_matrix` whose lane holds `elements`, two values + /// converted to `T`: what assigning `m.thread_elements()` makes. + pub fn matrix(self: *Builder, scalar: Scalar, elements: anytype) Value { + const v = self.convert(self.lift(elements), .vec(scalar, 2)); + var mask: [64]i32 = @splat(-1); + mask[0..2].* = .{ 0, 1 }; + return self.emit(llvm.shufflevector(self.ctx, v.inner, v.inner, self.matrixType(scalar), &mask, self.loc()), .{ .matrix = scalar }); + } + + /// `++i`: a narrow integer steps in its own type, as C++ steps it. + fn increment(self: *Builder, i: Value) Value { + const v = i.rvalue(); + const t = v.type_(); + if (t.scalar.isFloat() or t.scalar.bits() >= 32) return v.add(1); + return self.emit(arith.addi(self.ctx, v.inner, self.constant(t, 1).inner, self.loc()), .{ .value = t }); + } + + fn emit(self: *Builder, op: *mlir.Operation, kind: Value.Kind) Value { + _ = op.appendTo(self.currentBlock()); + return .{ .inner = op.result(0), .kernel = self, .kind = kind }; + } + + fn fastmath(self: *const Builder) air.FloatOpts { + return .{ .fastmath = if (self.fast_math) .fast else .none }; + } + + fn setFastmath(self: *Builder, op: *mlir.Operation) void { + if (!self.fast_math) return; + op.setAttribute(arith.fastmath(self.ctx, .fast) catch unreachable); + } + + fn structType(self: *Builder, s: Struct) *const mlir.Type { + var fields: [32]*const mlir.Type = undefined; + for (s.fields, 0..) |f, i| { + const t = f.type.memory().mlirType(self.ctx); + fields[i] = if (f.count != 0) llvm.arrayType(t, f.count) else t; + } + var name: std.ArrayList(u8) = .empty; + name.print(self.arena.allocator(), "struct.{s}", .{s.name}) catch @panic("dsl/metal: out of memory"); + return llvm.identifiedStructType(self.ctx, name.items, fields[0..s.fields.len]) orelse + std.debug.panic("dsl/metal: two structs named {s}", .{s.name}); + } + + /// The MSL type of a Zig number, as `lift` gives it. + fn literalType(value: anytype) Type { + return switch (@typeInfo(@TypeOf(value))) { + .comptime_int => if (value >= std.math.minInt(i32) and value <= std.math.maxInt(i32)) .int else .long, + .comptime_float => .float, + .int => |info| switch (info.bits) { + 8 => if (info.signedness == .signed) .char else .uchar, + 16 => if (info.signedness == .signed) .short else .ushort, + 32 => if (info.signedness == .signed) .int else .uint, + 64 => if (info.signedness == .signed) .long else .ulong, + else => @compileError("dsl/metal: no MSL type for " ++ @typeName(@TypeOf(value))), + }, + else => @compileError("dsl/metal: expected a literal, got " ++ @typeName(@TypeOf(value))), + }; + } + + /// A Zig value as an MSL literal: integers are `int` (`long` when they do + /// not fit), floats are `float`. + pub fn lift(self: *Builder, value: anytype) Value { + const T = @TypeOf(value); + if (T == Value) return value.rvalue(); + return switch (@typeInfo(T)) { + .comptime_int, .comptime_float, .int => self.constant(literalType(value), value), + .bool => self.constant(.bool, value), + .float => |info| self.constant(switch (info.bits) { + 16 => Type.half, + 32 => Type.float, + else => @compileError("dsl/metal: no MSL type for " ++ @typeName(T)), + }, value), + else => @compileError("dsl/metal: expected a Value or a number, got " ++ @typeName(T)), + }; + } + + /// A constant of type `t`, splat across a vector. + pub fn constant(self: *Builder, t: Type, value: anytype) Value { + const ctx = self.ctx; + const ty = t.mlirType(ctx); + const op = if (t.scalar.isFloat()) blk: { + const v: f64 = switch (@typeInfo(@TypeOf(value))) { + .int, .comptime_int => @floatFromInt(value), + .bool => @floatFromInt(@intFromBool(value)), + else => @floatCast(value), + }; + if (t.isVector()) break :blk arith.constant_dense_splat_f(ctx, v, ty, self.loc()); + break :blk switch (t.scalar) { + .half => arith.constant_float(ctx, v, .f16, self.loc()), + .bfloat => arith.constant_float(ctx, v, .bf16, self.loc()), + else => arith.constant_float(ctx, v, .f32, self.loc()), + }; + } else blk: { + const v: i64 = switch (@typeInfo(@TypeOf(value))) { + .bool => @intFromBool(value), + .int, .comptime_int => if (@TypeOf(value) == u64) @bitCast(value) else value, + else => @intFromFloat(value), + }; + if (t.isVector()) break :blk arith.constant_dense_splat_i(ctx, v, ty, self.loc()); + break :blk arith.constant_int(ctx, v, ty, self.loc()); + }; + return self.emit(op, .{ .value = t }); + } + + /// `T(a, b, ...)` from scalars, or `T(x)` splat from one scalar. + pub fn vec(self: *Builder, t: Type, elements: anytype) Value { + const fields = @typeInfo(@TypeOf(elements)).@"struct".field_names; + if (fields.len == 1) return self.convert(self.lift(@field(elements, fields[0])), t); + if (fields.len != t.len) std.debug.panic("dsl/metal: {d} elements of a {s}", .{ fields.len, t.name() }); + var inner: [fields.len]*const mlir.Value = undefined; + inline for (fields, 0..) |name, i| inner[i] = self.convert(self.lift(@field(elements, name)), t.element()).inner; + return self.emit(vector.from_elements(self.ctx, &inner, t.mlirType(self.ctx), self.loc()), .{ .value = t }); + } + + fn splat(self: *Builder, scalar: Value, t: Type) Value { + return self.emit(vector.broadcast(self.ctx, scalar.inner, t.mlirType(self.ctx), self.loc()), .{ .value = t }); + } + + /// The conversion of `v` to `to`, as C++ converts: integer widths extend + /// by the source's signedness or truncate, integers and floats convert + /// through `air.convert`, and a scalar converts to a vector by splatting. + pub fn convert(self: *Builder, reference: Value, to: Type) Value { + const v = reference.rvalue(); + const ctx = self.ctx; + const from = v.type_(); + if (from.eql(to)) return v; + if (from.len != to.len) { + if (!from.isVector()) return self.splat(self.convert(v, to.element()), to); + std.debug.panic("dsl/metal: cannot convert a {s} to a {s}", .{ from.name(), to.name() }); + } + const fs = from.scalar; + const ts = to.scalar; + const ty = to.mlirType(ctx); + const kind: Value.Kind = .{ .value = to }; + if (ts == .bool) return v.ne(self.constant(from, 0)); + if (fs == .bool) { + if (ts.isFloat()) return self.convert(self.convert(v, to.withScalar(.int)), to); + return self.emit(arith.extui(ctx, v.inner, ty, self.loc()), kind); + } + if (fs.isFloat() and ts.isFloat()) { + if (!from.isVector() and (fs == .float or ts == .float)) { + const op = if (ts == .float) arith.extf(ctx, v.inner, ty, self.loc()) else arith.truncf(ctx, v.inner, ty, self.loc()); + return self.emit(op, kind); + } + return self.emit(air.convert(ctx, v.inner, ty, self.fastmath(), self.loc()), kind); + } + if (fs.isFloat()) { + const op = if (ts.isUnsigned()) air.convert_u(ctx, v.inner, ty, .{}, self.loc()) else air.convert_s(ctx, v.inner, ty, .{}, self.loc()); + return self.emit(op, kind); + } + if (ts.isFloat()) { + const op = if (fs.isUnsigned()) air.convert_u(ctx, v.inner, ty, self.fastmath(), self.loc()) else air.convert_s(ctx, v.inner, ty, self.fastmath(), self.loc()); + return self.emit(op, kind); + } + if (fs.bits() == ts.bits()) return .{ .inner = v.inner, .kernel = self, .kind = kind }; + if (fs.bits() > ts.bits()) return self.emit(arith.trunci(ctx, v.inner, ty, self.loc()), kind); + const op = if (fs.isUnsigned()) arith.extui(ctx, v.inner, ty, self.loc()) else arith.extsi(ctx, v.inner, ty, self.loc()); + return self.emit(op, kind); + } + + /// The operands of a binary operator after the usual arithmetic + /// conversions; a scalar operand of a vector takes its element type. + fn usualConversions(self: *Builder, a: Value, b_: anytype) [2]Value { + const b = self.lift(b_); + const ta = a.type_(); + const tb = b.type_(); + const converted: Type = if (ta.isVector() and tb.isVector()) blk: { + if (!ta.eql(tb)) std.debug.panic("dsl/metal: {s} and {s} operands", .{ ta.name(), tb.name() }); + break :blk ta; + } else if (ta.isVector()) ta else if (tb.isVector()) tb else .{ .scalar = types.commonScalar(ta.scalar, tb.scalar) }; + return .{ self.convert(a, converted), self.convert(b, converted) }; + } + + const ArithmeticOp = enum { add, sub, mul, div, rem, bit_and, bit_or, bit_xor }; + + fn arithmetic(self: *Builder, comptime op: ArithmeticOp, a: Value, b: anytype) Value { + const ctx = self.ctx; + const l, const r = self.usualConversions(a, b); + const t = l.type_(); + const loc_ = self.loc(); + if (t.scalar.isFloat()) { + const result = switch (op) { + .add => arith.addf(ctx, l.inner, r.inner, loc_), + .sub => arith.subf(ctx, l.inner, r.inner, loc_), + .mul => arith.mulf(ctx, l.inner, r.inner, loc_), + .div => arith.divf(ctx, l.inner, r.inner, loc_), + else => std.debug.panic("dsl/metal: {t} of floats", .{op}), + }; + self.setFastmath(result); + return self.emit(result, .{ .value = t }); + } + const unsigned = t.scalar.isUnsigned(); + const result = switch (op) { + .add => arith.addi(ctx, l.inner, r.inner, loc_), + .sub => arith.subi(ctx, l.inner, r.inner, loc_), + .mul => arith.muli(ctx, l.inner, r.inner, loc_), + .div => if (unsigned) arith.divui(ctx, l.inner, r.inner, loc_) else arith.divsi(ctx, l.inner, r.inner, loc_), + .rem => if (unsigned) arith.remui(ctx, l.inner, r.inner, loc_) else arith.remsi(ctx, l.inner, r.inner, loc_), + .bit_and => arith.andi(ctx, l.inner, r.inner, loc_), + .bit_or => arith.ori(ctx, l.inner, r.inner, loc_), + .bit_xor => arith.xori(ctx, l.inner, r.inner, loc_), + }; + // Signed overflow is undefined in C++. + if (!unsigned and (op == .add or op == .sub or op == .mul)) { + result.setAttribute(arith.overflowFlags(ctx, .{ .nsw = true }) catch unreachable); + } + return self.emit(result, .{ .value = t }); + } + + /// `a << b` or `a >> b`. MSL shifts by the amount modulo the width: + /// clang casts the promoted amount to `a`'s promoted type as unsigned, + /// then masks it, which folds away for a number within the width. + fn shift(self: *Builder, comptime direction: enum { left, right }, a: Value, b: anytype) Value { + const t = a.type_().withScalar(a.type_().scalar.promoted()); + const l = self.convert(a, t); + const mask = t.scalar.bits() - 1; + const r = if (@TypeOf(b) == Value) blk: { + const rb = b.rvalue(); + const amount = self.shiftAmount(self.convert(rb, rb.type_().withScalar(rb.type_().scalar.promoted())), t); + break :blk self.emit(arith.andi(self.ctx, amount.inner, self.constant(t, mask).inner, self.loc()), .{ .value = t }); + } else blk: { + if (b < 0 or b > mask) std.debug.panic("dsl/metal: a shift of a {s} by {d}, past its width", .{ t.name(), b }); + const rb = self.lift(b); + break :blk self.convert(self.convert(rb, rb.type_().withScalar(rb.type_().scalar.promoted())), t); + }; + const loc_ = self.loc(); + const op = switch (direction) { + .left => arith.shli(self.ctx, l.inner, r.inner, loc_), + .right => if (t.scalar.isUnsigned()) arith.shrui(self.ctx, l.inner, r.inner, loc_) else arith.shrsi(self.ctx, l.inner, r.inner, loc_), + }; + return self.emit(op, .{ .value = t }); + } + + /// A promoted shift amount `v` as the shifted type `t`: a scalar of a + /// vector converts and splats, as C++ converts it; otherwise the cast is + /// unsigned, as clang casts it. + fn shiftAmount(self: *Builder, v: Value, t: Type) Value { + const from = v.type_(); + if (from.len != t.len) return self.convert(v, t); + if (from.scalar.bits() == t.scalar.bits()) return .{ .inner = v.inner, .kernel = self, .kind = .{ .value = t } }; + const ty = t.mlirType(self.ctx); + const op = if (from.scalar.bits() > t.scalar.bits()) arith.trunci(self.ctx, v.inner, ty, self.loc()) else arith.extui(self.ctx, v.inner, ty, self.loc()); + return self.emit(op, .{ .value = t }); + } + + fn negate(self: *Builder, a: Value) Value { + const t = a.type_(); + if (t.scalar.isFloat()) { + const op = arith.negf(self.ctx, a.inner, self.loc()); + self.setFastmath(op); + return self.emit(op, .{ .value = t }); + } + const x = self.convert(a, t.withScalar(t.scalar.promoted())); + return self.constant(x.type_(), 0).sub(x); + } + + const Comparison = enum { lt, le, gt, ge, eq, ne }; + + fn compare(self: *Builder, comptime op: Comparison, a: Value, b: anytype) Value { + const l, const r = self.usualConversions(a, b); + const t = l.type_(); + const result_type = t.withScalar(.bool); + if (t.scalar.isFloat()) { + const predicate: arith.CmpFPredicate = switch (op) { + .lt => .olt, + .le => .ole, + .gt => .ogt, + .ge => .oge, + .eq => .oeq, + .ne => .une, + }; + const result = arith.cmpf(self.ctx, predicate, l.inner, r.inner, self.loc()); + self.setFastmath(result); + return self.emit(result, .{ .value = result_type }); + } + const unsigned = t.scalar.isUnsigned(); + const predicate: arith.CmpIPredicate = switch (op) { + .lt => if (unsigned) .ult else .slt, + .le => if (unsigned) .ule else .sle, + .gt => if (unsigned) .ugt else .sgt, + .ge => if (unsigned) .uge else .sge, + .eq => .eq, + .ne => .ne, + }; + return self.emit(arith.cmpi(self.ctx, predicate, l.inner, r.inner, self.loc()), .{ .value = result_type }); + } + + /// `i` of `p + i` and `p - i`, promoted as C++ promotes an integer + /// operand; `p[i]` and a vector subscript take `i` as it is. + fn promoted(self: *Builder, i: anytype) Value { + const v = self.subscript(i); + const t = v.type_(); + return self.convert(v, t.withScalar(t.scalar.promoted())); + } + + /// A vector subscript, of its own type as clang keeps it: a literal is an + /// `int`, a `ushort` an `i16`, a `bool` an `i1`. + fn subscript(self: *Builder, i: anytype) Value { + const v = self.lift(i); + const t = v.type_(); + if (t.isVector() or t.scalar.isFloat()) std.debug.panic("dsl/metal: an index of a {s}", .{t.name()}); + return v; + } + + /// A `t` read from `p`, where it is a `t.memory()`. + fn loadMemory(self: *Builder, p: *const mlir.Value, t: Type) Value { + const memory = t.memory(); + const x = self.emit(llvm.load(self.ctx, p, memory.mlirType(self.ctx), .{ .alignment = memory.alignment() }, self.loc()), .{ .value = memory }); + if (t.scalar != .bool) return x; + return self.emit(arith.trunci(self.ctx, x.inner, t.mlirType(self.ctx), self.loc()), .{ .value = t }); + } + + /// `p + i` (`p - i` when `negative`) in elements of the pointee type. + fn offset(self: *Builder, p: Value, i: anytype, negative: bool) Value { + if (@TypeOf(i) == comptime_int and i == 0) return p; + const pointee = switch (p.kind) { + .pointer => |ptr| ptr.pointee, + else => std.debug.panic("dsl/metal: pointer arithmetic on a {t}", .{p.kind}), + }; + var index = self.lift(i); + if (index.type_().scalar.isFloat() or index.type_().isVector()) std.debug.panic("dsl/metal: pointer offset by a {s}", .{index.type_().name()}); + index = self.convert(index, if (index.type_().scalar.isUnsigned()) .ulong else .long); + if (negative) index = self.constant(.long, 0).sub(index); + const gep = llvm.getelementptr(self.ctx, p.inner, &.{.{ .value = index.inner }}, pointee.memory().mlirType(self.ctx), self.loc()); + return self.emit(gep, p.kind); + } + + /// `cond ? a : b` of evaluated operands, a select; + /// `openIfElse(cond, .{T})` evaluates one operand. + pub fn ternary(self: *Builder, cond: Value, a: anytype, b: anytype) Value { + const l, const r = self.usualConversions(self.lift(a), b); + const c = self.convert(cond, if (l.type_().isVector() and cond.type_().isVector()) l.type_().withScalar(.bool) else .bool); + return self.emit(arith.select(self.ctx, c.inner, l.inner, r.inner, self.loc()), l.kind); + } + + fn openTernary(self: *Builder, cond: Value, t: Type) TernaryScope { + const then_block = mlir.Block.init(&.{}, &.{}); + const else_block = mlir.Block.init(&.{}, &.{}); + var loc_ = [1]*const mlir.Location{self.loc()}; + const end_block = mlir.Block.init(&.{t.mlirType(self.ctx)}, &loc_); + self.condBranch(cond, then_block, else_block); + self.startBlock(then_block); + return .{ .kernel = self, .type = t, .else_block = else_block, .end_block = end_block }; + } + + /// `T a[n] = {};`, zeroed as clang zeroes it. + pub fn zeroedArray(self: *Builder, t: Type, n: u32) Value { + const a = self.array(t, n); + const len = self.constant(.long, @as(i64, n) * @as(i64, @intCast(t.size()))); + const operands = [_]*const mlir.Value{ a.inner, self.constant(.char, 0).inner, len.inner, self.constant(.bool, false).inner }; + _ = llvm.call_intrinsic(self.ctx, "llvm.memset.p0.i64", &operands, .{}, self.loc()).appendTo(self.currentBlock()); + return a; + } + + /// `T a[n];` in thread memory, as a pointer to its first element. + pub fn array(self: *Builder, t: Type, n: u32) Value { + return .{ .inner = self.allocate(llvm.arrayType(t.memory().mlirType(self.ctx), n), t.alignment()), .kernel = self, .kind = .{ .pointer = .{ .space = .thread, .pointee = t } } }; + } + + fn floatMath(self: *Builder, comptime name: []const u8, mode: MathMode, args: anytype) Value { + const x = self.lift(args[0]); + const t = x.type_(); + if (!t.scalar.isFloat() or t.scalar == .bfloat) + std.debug.panic("dsl/metal: no {s} of a {s}", .{ name, t.name() }); + const use_fast = switch (mode) { + .default => self.fast_math and t.scalar == .float, + .fast => true, + .precise => false, + }; + if (use_fast and t.scalar != .float) std.debug.panic("dsl/metal: fast::{s} of a {s}", .{ name, t.name() }); + const opts = self.fastmath(); + const loc_ = self.loc(); + const op = switch (args.len) { + 1 => if (use_fast) @field(air, "fast_" ++ name)(self.ctx, x.inner, opts, loc_) else @field(air, name)(self.ctx, x.inner, opts, loc_), + 2 => blk: { + const y = self.convert(self.lift(args[1]), t); + break :blk if (use_fast) @field(air, "fast_" ++ name)(self.ctx, x.inner, y.inner, opts, loc_) else @field(air, name)(self.ctx, x.inner, y.inner, opts, loc_); + }, + 3 => blk: { + const y = self.convert(self.lift(args[1]), t); + const z = self.convert(self.lift(args[2]), t); + break :blk if (use_fast) @field(air, "fast_" ++ name)(self.ctx, x.inner, y.inner, z.inner, opts, loc_) else @field(air, name)(self.ctx, x.inner, y.inner, z.inner, opts, loc_); + }, + else => unreachable, + }; + return self.emit(op, .{ .value = t }); + } + + /// `metal::fast::f`: `b.fast().exp2(x)`. + pub fn fast(self: *Builder) Namespace { + return .{ .b = self, .mode = .fast }; + } + + /// `metal::precise::f`: `b.precise().tanh(x)`. + pub fn precise(self: *Builder) Namespace { + return .{ .b = self, .mode = .precise }; + } + + pub fn exp(self: *Builder, x: Value) Value { + return self.floatMath("exp", .default, .{x}); + } + pub fn exp2(self: *Builder, x: Value) Value { + return self.floatMath("exp2", .default, .{x}); + } + pub fn log(self: *Builder, x: Value) Value { + return self.floatMath("log", .default, .{x}); + } + pub fn log2(self: *Builder, x: Value) Value { + return self.floatMath("log2", .default, .{x}); + } + pub fn tanh(self: *Builder, x: Value) Value { + return self.floatMath("tanh", .default, .{x}); + } + pub fn sqrt(self: *Builder, x: Value) Value { + return self.floatMath("sqrt", .default, .{x}); + } + pub fn rsqrt(self: *Builder, x: Value) Value { + return self.floatMath("rsqrt", .default, .{x}); + } + pub fn ceil(self: *Builder, x: Value) Value { + return self.floatMath("ceil", .default, .{x}); + } + pub fn floor(self: *Builder, x: Value) Value { + return self.floatMath("floor", .default, .{x}); + } + pub fn round(self: *Builder, x: Value) Value { + return self.floatMath("round", .default, .{x}); + } + pub fn trunc(self: *Builder, x: Value) Value { + return self.floatMath("trunc", .default, .{x}); + } + pub fn sin(self: *Builder, x: Value) Value { + return self.floatMath("sin", .default, .{x}); + } + pub fn cos(self: *Builder, x: Value) Value { + return self.floatMath("cos", .default, .{x}); + } + pub fn pow(self: *Builder, x: Value, y: anytype) Value { + return self.floatMath("pow", .default, .{ x, y }); + } + pub fn fmod(self: *Builder, x: Value, y: anytype) Value { + return self.floatMath("fmod", .default, .{ x, y }); + } + + /// `fma(a, b, c)` of floats. + pub fn fma(self: *Builder, a_: Value, b: anytype, c: anytype) Value { + const a = a_.rvalue(); + const t = a.type_(); + if (!t.scalar.isFloat() or t.scalar == .bfloat) std.debug.panic("dsl/metal: fma of a {s}", .{t.name()}); + const y = self.convert(self.lift(b), t); + const z = self.convert(self.lift(c), t); + return self.emit(air.fma(self.ctx, a.inner, y.inner, z.inner, self.fastmath(), self.loc()), .{ .value = t }); + } + + /// `dot(a, b)` of float vectors. + pub fn dot(self: *Builder, a_: Value, b_: Value) Value { + const a = a_.rvalue(); + const b = b_.rvalue(); + const t = a.type_(); + if (!t.eql(b.type_()) or !t.isVector()) std.debug.panic("dsl/metal: dot of {s} and {s}", .{ t.name(), b.type_().name() }); + return self.emit(air.dot(self.ctx, a.inner, b.inner, self.fastmath(), self.loc()), .{ .value = t.element() }); + } + + /// `abs(x)`. + pub fn abs(self: *Builder, x_: Value) Value { + const x = x_.rvalue(); + const t = x.type_(); + if (t.scalar.isFloat()) return self.floatMath("fabs", .default, .{x}); + if (t.scalar.isUnsigned()) return x; + return self.emit(air.abs_s(self.ctx, x.inner, self.loc()), .{ .value = t }); + } + + /// The operands of an MSL function taking several values of one type, + /// read, with literals converted to the other operand's type. + fn sameType(self: *Builder, a: anytype, b: anytype) [2]Value { + if (@TypeOf(a) != Value) { + const r = self.lift(b); + return .{ self.convert(self.lift(a), r.type_()), r }; + } + const l = a.rvalue(); + const r = if (@TypeOf(b) == Value) b.rvalue() else self.convert(self.lift(b), l.type_()); + if (!l.type_().eql(r.type_())) std.debug.panic("dsl/metal: {s} and {s} arguments", .{ l.type_().name(), r.type_().name() }); + return .{ l, r }; + } + + /// `max(a, b)`. + pub fn max(self: *Builder, a: anytype, b: anytype) Value { + const l, const r = self.sameType(a, b); + const t = l.type_(); + if (t.scalar.isFloat()) return self.floatMath("fmax", .default, .{ l, r }); + const op = if (t.scalar.isUnsigned()) air.max_u(self.ctx, l.inner, r.inner, self.loc()) else air.max_s(self.ctx, l.inner, r.inner, self.loc()); + return self.emit(op, .{ .value = t }); + } + + /// `min(a, b)`. + pub fn min(self: *Builder, a: anytype, b: anytype) Value { + const l, const r = self.sameType(a, b); + const t = l.type_(); + if (t.scalar.isFloat()) return self.floatMath("fmin", .default, .{ l, r }); + const op = if (t.scalar.isUnsigned()) air.min_u(self.ctx, l.inner, r.inner, self.loc()) else air.min_s(self.ctx, l.inner, r.inner, self.loc()); + return self.emit(op, .{ .value = t }); + } + + /// `clamp(x, lo, hi)`. + pub fn clamp(self: *Builder, x_: Value, lo: anytype, hi: anytype) Value { + const x = x_.rvalue(); + const t = x.type_(); + const l = self.convert(self.lift(lo), t); + const h = self.convert(self.lift(hi), t); + if (t.scalar.isFloat()) return self.floatMath("clamp", .default, .{ x, l, h }); + const op = if (t.scalar.isUnsigned()) air.clamp_u(self.ctx, x.inner, l.inner, h.inner, self.loc()) else air.clamp_s(self.ctx, x.inner, l.inner, h.inner, self.loc()); + return self.emit(op, .{ .value = t }); + } + + fn simdType(x: Value, comptime name: []const u8) Type { + const t = x.type_(); + switch (t.scalar) { + .half, .float, .char, .uchar, .short, .ushort, .int, .uint => {}, + else => std.debug.panic("dsl/metal: no {s} of a {s}", .{ name, t.name() }), + } + return t; + } + + fn simdReduce(self: *Builder, comptime name: []const u8, x_: Value) Value { + const x = x_.rvalue(); + const t = simdType(x, name); + const op = if (t.scalar.isFloat()) + @field(air, name)(self.ctx, x.inner, self.fastmath(), self.loc()) + else if (t.scalar.isUnsigned()) + @field(air, name ++ "_u")(self.ctx, x.inner, self.loc()) + else + @field(air, name ++ "_s")(self.ctx, x.inner, self.loc()); + return self.emit(op, .{ .value = t }); + } + + /// A function moving `x` across lanes; `x` is frozen, so that a poison + /// lane does not poison the others. + fn shuffle(self: *Builder, comptime name: []const u8, x_: Value, lane: anytype) Value { + const read = x_.rvalue(); + const t = simdType(read, name); + const x = self.emit(llvm.freeze(self.ctx, read.inner, self.loc()), .{ .value = t }); + const l = self.convert(self.lift(lane), .ushort); + const op = if (t.scalar.isFloat()) + @field(air, name)(self.ctx, x.inner, l.inner, self.fastmath(), self.loc()) + else if (t.scalar.isUnsigned()) + @field(air, name ++ "_u")(self.ctx, x.inner, l.inner, self.loc()) + else + @field(air, name ++ "_s")(self.ctx, x.inner, l.inner, self.loc()); + return self.emit(op, .{ .value = t }); + } + + pub fn simdSum(self: *Builder, x: Value) Value { + return self.simdReduce("simd_sum", x); + } + pub fn simdMax(self: *Builder, x: Value) Value { + return self.simdReduce("simd_max", x); + } + pub fn simdMin(self: *Builder, x: Value) Value { + return self.simdReduce("simd_min", x); + } + pub fn simdPrefixExclusiveSum(self: *Builder, x_: Value) Value { + const x = x_.rvalue(); + const t = x.type_(); + if (!t.scalar.isFloat() or t.scalar == .bfloat) std.debug.panic("dsl/metal: simd_prefix_exclusive_sum of a {s}", .{t.name()}); + return self.emit(air.simd_prefix_exclusive_sum(self.ctx, x.inner, self.fastmath(), self.loc()), .{ .value = t }); + } + /// `simd_shuffle(x, lane)`. + pub fn simdShuffle(self: *Builder, x: Value, lane: anytype) Value { + return self.shuffle("simd_shuffle", x, lane); + } + /// `simd_shuffle_xor(x, mask)`. + pub fn simdShuffleXor(self: *Builder, x: Value, mask: anytype) Value { + return self.shuffle("simd_shuffle_xor", x, mask); + } + /// `simd_shuffle_down(x, delta)`. + pub fn simdShuffleDown(self: *Builder, x: Value, delta: anytype) Value { + return self.shuffle("simd_shuffle_down", x, delta); + } + /// `simd_broadcast(x, lane)`. + pub fn simdBroadcast(self: *Builder, x: Value, lane: anytype) Value { + return self.shuffle("simd_broadcast", x, lane); + } + pub fn simdAny(self: *Builder, x: Value) Value { + return self.emit(air.simd_any(self.ctx, self.convert(x, .bool).inner, self.loc()), .{ .value = .bool }); + } + pub fn simdAll(self: *Builder, x: Value) Value { + return self.emit(air.simd_all(self.ctx, self.convert(x, .bool).inner, self.loc()), .{ .value = .bool }); + } + + /// `threadgroup_barrier(flags)`. + pub fn threadgroupBarrier(self: *Builder, flags: MemFlags) void { + _ = air.wg_barrier(self.ctx, @bitCast(flags), self.loc()).appendTo(self.currentBlock()); + } + + /// `simdgroup_barrier(flags)`. + pub fn simdgroupBarrier(self: *Builder, flags: MemFlags) void { + _ = air.simdgroup_barrier(self.ctx, @bitCast(flags), self.loc()).appendTo(self.currentBlock()); + } + + pub const MatrixOpts = struct { + /// `matrix_origin`. + origin: [2]i64 = .{ 0, 0 }, + /// `transpose_matrix`. + transpose: bool = false, + }; + + fn matrixType(self: *Builder, scalar: Scalar) *const mlir.Type { + return .vector(&.{64}, scalar.mlirType(self.ctx)); + } + + fn matrixScalar(m: Value) Scalar { + return switch (m.kind) { + .matrix => |s| s, + else => std.debug.panic("dsl/metal: expected a simdgroup_matrix, got a {t}", .{m.kind}), + }; + } + + /// The extents, strides and origin a matrix load or store passes the + /// builtin, from its `elements_per_row`, `matrix_origin` and + /// `transpose_matrix`. + fn accessLayout(self: *Builder, elements_per_row: anytype, opts: MatrixOpts) [3]*const mlir.Value { + const ulong2: Type = .vec(.ulong, 2); + const epr = self.convert(self.lift(elements_per_row), .ulong); + const eight = self.constant(.ulong, 8); + const one = self.constant(.ulong, 1); + const o0 = self.constant(.ulong, opts.origin[0]); + const o1 = self.constant(.ulong, opts.origin[1]); + const layout: [3]Value = if (opts.transpose) + .{ self.vec(ulong2, .{ eight, epr }), self.vec(ulong2, .{ epr, one }), self.vec(ulong2, .{ o1, o0 }) } + else + .{ self.vec(ulong2, .{ epr, eight }), self.vec(ulong2, .{ one, epr }), self.vec(ulong2, .{ o0, o1 }) }; + return .{ layout[0].inner, layout[1].inner, layout[2].inner }; + } + + /// `simdgroup_load(m, src, elements_per_row, matrix_origin, + /// transpose_matrix)`: the matrix `m` it loads. + pub fn simdgroupLoad(self: *Builder, src_: Value, elements_per_row: anytype, opts: MatrixOpts) Value { + const src = src_.rvalue(); + const t = src.pointer().pointee.scalar; + const layout = self.accessLayout(elements_per_row, opts); + return self.emit(air.simdgroup_matrix_8x8_load(self.ctx, src.inner, layout[0], layout[1], layout[2], self.matrixType(t), self.fastmath(), self.loc()), .{ .matrix = t }); + } + + /// `simdgroup_store(m, dst, elements_per_row, matrix_origin, + /// transpose_matrix)`. + pub fn simdgroupStore(self: *Builder, m: Value, dst_: Value, elements_per_row: anytype, opts: MatrixOpts) void { + const t = matrixScalar(m); + const dst = dst_.rvalue(); + const p = dst.pointer(); + if (p.pointee.scalar != t) std.debug.panic("dsl/metal: simdgroup_store to a {s} pointer", .{p.pointee.name()}); + const layout = self.accessLayout(elements_per_row, opts); + _ = air.simdgroup_matrix_8x8_store(self.ctx, m.inner, dst.inner, layout[0], layout[1], layout[2], self.loc()).appendTo(self.currentBlock()); + } + + /// `simdgroup_multiply_accumulate(d, a, b, c)`: the matrix `d = a * b + c`, + /// of `c`'s type. + pub fn simdgroupMultiplyAccumulate(self: *Builder, a: Value, b: Value, c: Value) Value { + _ = .{ matrixScalar(a), matrixScalar(b) }; + const t = matrixScalar(c); + return self.emit(air.simdgroup_matrix_8x8_multiply_accumulate(self.ctx, a.inner, b.inner, c.inner, self.matrixType(t), self.fastmath(), self.loc()), .{ .matrix = t }); + } + + /// The `llvm.loop` metadata of an MSL loop: C++ loops must make progress. + fn loopAnnotation(self: *Builder, opts: LoopOpts) *const llvm.LoopAnnotationAttr { + const unroll: ?*const llvm.LoopUnrollAttr = if (opts.unroll) |u| switch (u) { + .full => llvm.LoopUnrollAttr.get(self.ctx, .{ .full = true }), + .enable => llvm.LoopUnrollAttr.get(self.ctx, .{ .disable = false }), + } else null; + return llvm.LoopAnnotationAttr.get(self.ctx, .{ .unroll = unroll, .mustProgress = true }); + } + + /// The type `i < upper` compares in, for a loop variable `i` of type `t`: the + /// loop runs in `t`, which a short or char variable may be narrower than. + fn loopComparison(t: Type, upper: Type) Scalar { + const compared = types.commonScalar(t.scalar, upper.scalar); + const narrow = t.scalar.bits() < 32 and compared == .int; + if (t.scalar.isFloat() or t.isVector() or (compared.bits() != t.scalar.bits() and !narrow)) + std.debug.panic("dsl/metal: a {s} loop variable compared to a {s}", .{ t.name(), upper.name() }); + return compared; + } + + /// `for (T i = lower; i < upper; i += step)`, with `T` the type of + /// `lower`; `close` ends the body. + pub fn openFor(self: *Builder, lower: anytype, upper: anytype, step: anytype) ForScope { + return self.openForOpts(lower, upper, step, .{}); + } + + /// `openFor` under a loop pragma. + pub fn openForOpts(self: *Builder, lower: anytype, upper: anytype, step: anytype, opts: LoopOpts) ForScope { + const i = self.local(if (@TypeOf(lower) == Value) lower.type_() else literalType(lower), lower); + var loop = self.openLoop(opts); + _ = loopComparison(i.type_(), if (@TypeOf(upper) == Value) upper.type_() else literalType(upper)); + loop.body(i.lt(upper)); + const step_: ForScope.Step = switch (@typeInfo(@TypeOf(step))) { + .comptime_int, .int => if (step == 1) .one else .{ .int = step }, + else => .{ .value = step }, + }; + return .{ .loop = loop, .iv = i, .step = step_ }; + } + + /// A loop: `while (c)`, `for (init; c; inc)` or `for (;;)`. + pub fn openLoop(self: *Builder, opts: LoopOpts) LoopScope { + const cond_block = mlir.Block.init(&.{}, &.{}); + _ = self.branch(cond_block); + self.startBlock(cond_block); + return .{ .kernel = self, .cond_block = cond_block, .end_block = mlir.Block.init(&.{}, &.{}), .loop_annotation = self.loopAnnotation(opts) }; + } + + /// `if (cond) { ... }`. + pub fn openIf(self: *Builder, cond: Value) IfScope { + var c = self.condition(); + return c.openIf(cond); + } + + /// `if (cond) { ... } else { ... }` with no result types; with one, `T`, + /// `cond ? a : b` of type `T` evaluating one operand. + pub fn openIfElse(self: *Builder, cond: Value, result_types: anytype) IfElseScope(common.arity(@TypeOf(result_types), "openIfElse: result_types")) { + if (comptime common.arity(@TypeOf(result_types), "openIfElse: result_types") == 0) { + var c = self.condition(); + return c.openIfElse(cond, .{}); + } + return self.openTernary(cond, typeOf(result_types[0])); + } + + /// A condition of `&&`, `||` and `!` terms. + pub fn condition(self: *Builder) Condition { + return .{ .kernel = self, .true_block = mlir.Block.init(&.{}, &.{}), .false_block = mlir.Block.init(&.{}, &.{}) }; + } + + /// `a && b && ...` as a bool. + pub fn openAnd(self: *Builder) Logical { + return self.logicalChain(.@"and"); + } + + /// `a || b || ...` as a bool. + pub fn openOr(self: *Builder) Logical { + return self.logicalChain(.@"or"); + } + + fn logicalChain(self: *Builder, op: @FieldType(Logical, "op")) Logical { + var loc_ = [1]*const mlir.Location{self.loc()}; + return .{ .kernel = self, .op = op, .end_block = mlir.Block.init(&.{Type.bool.mlirType(self.ctx)}, &loc_) }; + } + + /// `return;`. + pub fn return_(self: *Builder) void { + const block = self.insert orelse return; + _ = func.returns(self.ctx, &.{}, self.loc()).appendTo(block); + self.insert = null; + } + + /// `if (cond) return;`. + pub fn returnIf(self: *Builder, cond: Value) void { + var scope = self.openIf(cond); + { + self.return_(); + scope.close(); + } + } + + /// Terminates the kernel and returns its module, verified. + pub fn finish(self: *Builder) FinishError![:0]const u8 { + self.return_(); + if (self.fast_math) air.enableFastMath(self.ctx, self.module.operation()); + try common.verify(self.module, "dsl/metal"); + var writer: std.Io.Writer.Allocating = .init(self.allocator); + defer writer.deinit(); + try writer.writer.print("{f}", .{self.module.operation()}); + return try self.allocator.dupeSentinel(u8, writer.written(), 0); + } +}; + +fn testContext() !*mlir.Context { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + registerDialects(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + return ctx; +} + +fn expectContains(ir: []const u8, needles: []const []const u8) !void { + for (needles) |needle| { + if (std.mem.find(u8, ir, needle) == null) { + std.debug.print("missing `{s}` in:\n{s}\n", .{ needle, ir }); + return error.TestExpectedEqual; + } + } +} + +test "arguments and C++ arithmetic" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = Builder.open(std.testing.allocator, ctx, "k", .{}); + defer b.deinit(); + const a = try b.declareArgs(.{ + .x = .{ .buffer = .{ .type = .bfloat, .const_ = true, .restrict = true } }, + .y = .{ .buffer = .{ .type = .float4 } }, + .n = .{ .constant = .int }, + .shmem = .{ .threadgroup = .half }, + .gid = .{ .threadgroup_position_in_grid = .uint3 }, + .lane = .{ .thread_index_in_simdgroup = .ushort }, + }); + // `gid.z / n` divides unsigned, `lane * 2` promotes to int. + const q = a.gid.z().div(a.n); + try std.testing.expect(q.type_().eql(.uint)); + const twice = a.lane.mul(2); + try std.testing.expect(twice.type_().eql(.int)); + const v = a.x.load(q).to(.float).mul(0.5); + try std.testing.expect(v.type_().eql(.float)); + a.y.store(twice, b.vec(.float4, .{v})); + a.shmem.store(a.lane, v); + try std.testing.expectEqual(4, b.arguments.items.len); + try std.testing.expectEqual(1, b.arguments.items[1].index); + try std.testing.expectEqual(4, b.arguments.items[2].size); + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + try expectContains(ir, &.{ "air.fast_math_enable", "air.kernel", "noalias = true" }); + try expectContains(ir, &.{ "arith.divui", "overflow", "arith.extf", "arith.mulf", "fastmath", "arith.truncf", "llvm.getelementptr", "#air.builtin", "air.arg_name = \"lane\"" }); +} + +test "conversions follow Apple's compiler" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = Builder.open(std.testing.allocator, ctx, "k", .{}); + defer b.deinit(); + const a = try b.declareArgs(.{ + .h = .{ .buffer = .{ .type = .half4, .const_ = true } }, + .o = .{ .buffer = .{ .type = .bfloat4 } }, + .i = .{ .buffer = .{ .type = .int } }, + .tid = .{ .thread_position_in_grid = .uint }, + }); + const f4 = a.h.load(a.tid).to(.float4); + a.o.store(a.tid, f4.to(.bfloat4)); + const n = a.i.load(a.tid); + a.i.store(0, n.to(.float).add(n.to(.half).to(.float)).to(.uint).to(.short)); + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + try expectContains(ir, &.{ "air.convert", ": vector<4xf16> to vector<4xf32>", ": vector<4xf32> to vector<4xbf16>", "air.convert.s", ": i32 to f32", ": i32 to f16", "arith.extf", "air.convert.u", "arith.trunci" }); +} + +test "math, SIMD-group and barrier functions" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = Builder.open(std.testing.allocator, ctx, "k", .{}); + defer b.deinit(); + const a = try b.declareArgs(.{ + .x = .{ .buffer = .{ .type = .float } }, + .v = .{ .buffer = .{ .type = .float4 } }, + .lane = .{ .thread_index_in_simdgroup = .ushort }, + }); + const x = a.x.load(a.lane); + const v = a.v.load(a.lane); + const m = b.simdMax(b.max(x, b.fast().exp2(x))); + const s = b.simdSum(b.exp(x.sub(m))); + const t = b.precise().tanh(b.fma(s, 2.0, b.dot(v, v))); + const r = b.simdShuffleXor(t, 8); + b.simdgroupBarrier(.mem_threadgroup); + b.threadgroupBarrier(.{ .device = true, .threadgroup = true }); + a.x.store(a.lane, b.ternary(r.gt(0), b.min(a.lane.to(.int), 3).to(.float), r)); + a.x.store(1, b.ternary(r.gt(1), 1.0, 2.0)); + a.v.store(a.lane, b.simdSum(v)); + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + try expectContains(ir, &.{ "air.fast_exp2", "air.fast_fmax", "air.simd_max", "air.fast_exp", "air.simd_sum", "air.dot", "air.fma", "air.tanh", "air.simd_shuffle_xor", "air.simdgroup.barrier ", "air.wg.barrier ", "air.min.s", "arith.select", ": vector<4xf32>" }); + try std.testing.expect(std.mem.find(u8, ir, "cf.cond_br") == null); +} + +test "loops, ifs and early returns" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = Builder.open(std.testing.allocator, ctx, "k", .{}); + defer b.deinit(); + const a = try b.declareArgs(.{ + .x = .{ .buffer = .{ .type = .float } }, + .n = .{ .constant = .int }, + .tid = .{ .thread_position_in_grid = .uint }, + }); + var ret = b.openIf(a.tid.ge(1024)); + { + a.x.store(0, 0.0); + b.return_(); + ret.close(); + } + b.returnIf(a.n.le(0)); + + // float acc = 0; for (int i = 0; i < n; ++i) { if (x[i] < 0) break; acc += x[i]; } + const acc = b.local(.float, 0.0); + var loop = b.openFor(0, a.n, 1); + { + var neg = b.openIf(a.x.load(loop.iv).lt(0.0)); + { + loop.breakLoop(); + neg.close(); + } + acc.assign(acc.add(a.x.load(loop.iv))); + loop.close(); + } + + // int lo = 0, hi = n; while (lo < hi) { int mid = (lo + hi + 1) / 2; ... } + const lo = b.local(.int, 0); + const hi = b.local(.int, a.n); + var search = b.openLoop(.{}); + { + search.body(lo.lt(hi)); + } + { + const mid = b.local(.int, lo.add(hi).add(1).div(2)); + var up = b.openIfElse(a.x.load(mid).le(acc), .{}); + { + lo.assign(mid); + up.elseBranch(); + } + { + hi.assign(mid.sub(1)); + up.close(); + } + search.close(); + } + + // bool m = lo > 0 || (n > 3 && acc < 1); + var m = b.openOr(); + m.term(lo.gt(0)); + var c = b.condition(); + c.and_(a.n.gt(3)); + var only = c.openIfNot(m.close(acc.lt(1.0))); + { + a.x.store(1, b.ternary(lo.gt(0), acc, 2.0)); + only.close(); + } + + const scratch = b.array(.float, 8); + scratch.store(a.tid.rem(8), acc); + a.x.store(2, scratch.load(3)); + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + try expectContains(ir, &.{ "cf.cond_br", "llvm.br", "loop_annotation", "mustProgress = true", "llvm.alloca", "!llvm.array<8 x f32>", "arith.remui" }); + try std.testing.expect(std.mem.find(u8, ir, "scf.") == null); + // The two early returns and the last one, each where it is. + try std.testing.expectEqual(3, std.mem.count(u8, ir, "\n return\n")); + // Every local is allocated in the entry block, before any branch. + try std.testing.expect(std.mem.findLast(u8, ir, "llvm.alloca").? < std.mem.find(u8, ir, "cf.cond_br").?); +} + +test "a return inside a loop and in both branches of an if" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = Builder.open(std.testing.allocator, ctx, "k", .{}); + defer b.deinit(); + const a = try b.declareArgs(.{ + .x = .{ .buffer = .{ .type = .float } }, + .n = .{ .constant = .int }, + }); + // for (;;) { if (n <= 0) break; if (x[0] < 0) return; x[0] -= 1; } + var forever = b.openLoop(.{}); + { + var stop = b.openIf(a.n.le(0)); + { + forever.breakLoop(); + stop.close(); + } + b.returnIf(a.x.load(0).lt(0.0)); + a.x.store(0, a.x.load(0).sub(1.0)); + forever.close(); + } + // if (n > 1) { x[1] = 1; return; } else { x[1] = 2; return; } + var both = b.openIfElse(a.n.gt(1), .{}); + { + a.x.store(1, 1.0); + b.return_(); + both.elseBranch(); + } + { + a.x.store(1, 2.0); + b.return_(); + both.close(); + } + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + // The loop's return and one per branch; `finish` adds none after them. + try std.testing.expectEqual(3, std.mem.count(u8, ir, "\n return\n")); + try expectContains(ir, &.{"loop_annotation"}); +} + +test "a break and the end of the body meet in the latch" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = Builder.open(std.testing.allocator, ctx, "k", .{}); + defer b.deinit(); + const a = try b.declareArgs(.{ + .x = .{ .buffer = .{ .type = .float } }, + .n = .{ .constant = .int }, + }); + // for (int i = 0; i < n; ++i) { if (x[i] < 0) break; x[i] = 1; } + var loop = b.openFor(0, a.n, 1); + { + var neg = b.openIf(a.x.load(loop.iv).lt(0.0)); + { + loop.breakLoop(); + neg.close(); + } + a.x.store(loop.iv, 1.0); + loop.close(); + } + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + // `i`, and the flag the break and the end of the body set for the latch. + try std.testing.expectEqual(2, std.mem.count(u8, ir, "llvm.alloca")); + // `i < n`, `x[i] < 0`, and the latch's choice between leaving and `++i`. + try std.testing.expectEqual(3, std.mem.count(u8, ir, "cf.cond_br")); +} + +test "simdgroup matrices" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = Builder.open(std.testing.allocator, ctx, "k", .{}); + defer b.deinit(); + const a = try b.declareArgs(.{ + .o = .{ .buffer = .{ .type = .float2 } }, + .s = .{ .threadgroup = .bfloat }, + .f = .{ .constant = .int }, + .lane = .{ .thread_index_in_simdgroup = .uint }, + }); + const k = b.simdgroupLoad(a.s.add(8), 40, .{ .transpose = true }); + const q = b.simdgroupLoad(a.s, a.f, .{}); + const acc = b.simdgroupMultiplyAccumulate(q, k, b.matrix(.float, a.o.load(a.lane))); + a.o.store(a.lane, acc.threadElements()); + b.simdgroupStore(acc, a.s.to(.float), 8, .{}); + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + try expectContains(ir, &.{ "air.simdgroup_matrix_8x8_load", "-> vector<64xbf16>", "air.simdgroup_matrix_8x8_multiply_accumulate", "vector<64xbf16>, vector<64xbf16>, vector<64xf32> -> vector<64xf32>", "air.simdgroup_matrix_8x8_store" }); + // `b.matrix` widens two elements to a matrix, `threadElements` takes them back. + try std.testing.expectEqual(2, std.mem.count(u8, ir, "llvm.shufflevector")); + try expectContains(ir, &.{"[0, 1] : vector<64xf32>"}); + try std.testing.expect(std.mem.find(u8, ir, "llvm.alloca") == null); +} + +test "constant struct arguments" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = Builder.open(std.testing.allocator, ctx, "k", .{ .fast_math = false }); + defer b.deinit(); + const Args: Struct = .{ .name = "args_t", .fields = &.{ + .{ .name = "num_heads", .type = .int }, + .{ .name = "scale", .type = .float }, + } }; + const a = try b.declareArgs(.{ + .args = .{ .constant_struct = Args }, + .o = .{ .buffer = .{ .type = .float } }, + }); + a.o.store(0, a.args.field("scale").mul(a.args.field("num_heads"))); + try std.testing.expectEqual(8, b.arguments.items[0].size); + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + try expectContains(ir, &.{ "", "typeName = \"args_t\"", "!llvm.struct<\"struct.args_t\", (i32, f32)>" }); + try std.testing.expect(std.mem.find(u8, ir, "fastmath") == null); +} + +fn expectCount(ir: []const u8, needle: []const u8, expected: usize) !void { + const n = std.mem.count(u8, ir, needle); + if (n != expected) { + std.debug.print("`{s}` {d} times, not {d}, in:\n{s}\n", .{ needle, n, expected, ir }); + return error.TestExpectedEqual; + } +} + +test "subscripts index with the index's type, swizzles with an int" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = Builder.open(std.testing.allocator, ctx, "k", .{}); + defer b.deinit(); + const a = try b.declareArgs(.{ + .v = .{ .buffer = .{ .type = .float4 } }, + .o = .{ .buffer = .{ .type = .float } }, + .n = .{ .constant = .long }, + .gid = .{ .threadgroup_position_in_grid = .uint3 }, + .lane = .{ .thread_index_in_simdgroup = .ushort }, + }); + // float4 x = v[0]; + // o[gid.z] = x.x + x.y + x.z + x.w + x[2] + x[lane] + x[(char) lane] + x[n]; + const x = b.local(.float4, a.v.load(0)); + const sum = x.x().add(x.y()).add(x.z()).add(x.w()).add(x.at(2)).add(x.at(a.lane)).add(x.at(a.lane.to(.char))).add(x.at(a.n)); + a.o.store(a.gid.z(), sum); + // x[lane] = 1.0f; x[2] = 2.0f; x.y = 3.0f; v[0] = x; + x.assign(x.insert(a.lane, 1.0)); + x.assign(x.insert(2, 2.0)); + x.assign(x.insert(1, 3.0)); + a.v.store(0, x); + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + // `i32` for the swizzles and literals, `i16 %lane`, `i8`, `i64 %n`. + try expectCount(ir, ": i32] : vector<4xf32>", 7); + try expectCount(ir, ": i16] : vector<4xf32>", 2); + try expectCount(ir, ": i8] : vector<4xf32>", 1); + try expectCount(ir, ": i64] : vector<4xf32>", 1); + try expectCount(ir, ": i32] : vector<3xi32>", 1); +} + +test "a pointer local" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = Builder.open(std.testing.allocator, ctx, "k", .{}); + defer b.deinit(); + const a = try b.declareArgs(.{ + .o = .{ .buffer = .{ .type = .float } }, + .lane = .{ .thread_index_in_simdgroup = .ushort }, + }); + // device float *p = o + 4; p += lane; *p = 1.0f; + const p = b.localPointer(a.o.add(4)); + p.assign(p.add(a.lane)); + p.store(0, 1.0); + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + try expectCount(ir, "llvm.alloca", 1); + try expectCount(ir, "llvm.getelementptr", 2); +} + +test "integer max, min, abs and clamp read locals and constant arguments" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = Builder.open(std.testing.allocator, ctx, "k", .{}); + defer b.deinit(); + const a = try b.declareArgs(.{ + .o = .{ .buffer = .{ .type = .int } }, + .w = .{ .buffer = .{ .type = .uint } }, + .f = .{ .buffer = .{ .type = .float } }, + .n = .{ .constant = .int }, + .u = .{ .constant = .uint }, + }); + // int m = n; o[0] = max(m, n) + min(n, m) + min(m, 7) + abs(m) + clamp(m, 0, 7); + const m = b.local(.int, a.n); + a.o.store(0, b.max(m, a.n).add(b.min(a.n, m)).add(b.min(m, 7)).add(b.abs(m)).add(b.clamp(m, 0, 7))); + // w[0] = max(u, 3u) + min(3u, u) + abs(u) + clamp(u, 1u, 2u); + a.w.store(0, b.max(a.u, 3).add(b.min(3, a.u)).add(b.abs(a.u)).add(b.clamp(a.u, 1, 2))); + // const float x = f[0]; f[1] = clamp(x, 0.0f, 1.0f) + fabs(x) + max(x, x); + const x = a.f.load(0); + a.f.store(1, b.clamp(x, 0.0, 1.0).add(b.abs(x)).add(b.max(x, x))); + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + try expectContains(ir, &.{ "air.max.s", "air.min.s", "air.abs.s", "air.clamp.s", "air.max.u", "air.min.u", "air.clamp.u", "air.fast_clamp", "air.fast_fabs", "air.fast_fmax" }); + // abs of a uint is the value itself. + try std.testing.expect(std.mem.find(u8, ir, "air.abs.u") == null); +} + +test "char and uchar take the same functions as wider integers" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = Builder.open(std.testing.allocator, ctx, "k", .{}); + defer b.deinit(); + const a = try b.declareArgs(.{ + .c = .{ .buffer = .{ .type = .char } }, + .u = .{ .buffer = .{ .type = .uchar } }, + .v = .{ .buffer = .{ .type = .char2 } }, + .f = .{ .buffer = .{ .type = .float } }, + .w = .{ .buffer = .{ .type = .float2 } }, + .lane = .{ .thread_index_in_simdgroup = .ushort }, + }); + const c = a.c.load(a.lane); + const u = a.u.load(a.lane); + const v = a.v.load(a.lane); + // f[0] = float(c) + float(u); c[0] = char(f[1]); u[0] = uchar(f[2]); + a.f.store(0, c.to(.float).add(u.to(.float))); + a.c.store(0, a.f.load(1).to(.char)); + a.u.store(0, a.f.load(2).to(.uchar)); + // w[0] = float2(v); v[0] = char2(w[1]); + a.w.store(0, v.to(.float2)); + a.v.store(0, a.w.load(1).to(.char2)); + // c[1] = max(c, c); u[1] = min(u, u); c[2] = abs(c); c[3] = clamp(c, 1, 5); v[1] = max(v, v); + a.c.store(1, b.max(c, c)); + a.u.store(1, b.min(u, u)); + a.c.store(2, b.abs(c)); + a.c.store(3, b.clamp(c, 1, 5)); + a.v.store(1, b.max(v, v)); + // c[4] = simd_shuffle(c, lane); c[5] = simd_sum(c); u[2] = simd_max(u); + a.c.store(4, b.simdShuffle(c, a.lane)); + a.c.store(5, b.simdSum(c)); + a.u.store(2, b.simdMax(u)); + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + try expectContains(ir, &.{ + "air.convert.s", ": i8 to f32", "air.convert.u", ": f32 to i8", + ": vector<2xi8> to vector<2xf32>", ": vector<2xf32> to vector<2xi8>", "air.max.s", "air.min.u", + "air.abs.s", "air.clamp.s", "air.simd_shuffle.s", "air.simd_sum.s", + "air.simd_max.u", + }); +} + +test "bitwise operators and shifts" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = Builder.open(std.testing.allocator, ctx, "k", .{}); + defer b.deinit(); + const a = try b.declareArgs(.{ + .o = .{ .buffer = .{ .type = .int } }, + .w = .{ .buffer = .{ .type = .uint } }, + .l = .{ .buffer = .{ .type = .long } }, + .n = .{ .constant = .int }, + .lane = .{ .thread_index_in_simdgroup = .ushort }, + }); + // int s = n; o[0] = (n << s) ^ (~(n | 3) & -s); w[0] = w[1] >> s; o[1] = lane >> 1; + // l[0] = l[1] << n; + const s = b.local(.int, a.n); + a.o.store(0, a.n.shl(s).bitXor(a.n.bitOr(3).bitNot().bitAnd(s.neg()))); + a.w.store(0, a.w.load(1).shr(s)); + a.o.store(1, a.lane.shr(1)); + a.l.store(0, a.l.load(1).shl(a.n)); + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + try expectContains(ir, &.{ "arith.shli", "arith.shrui", "arith.shrsi", "arith.xori", "arith.ori", "arith.subi" }); + // clang masks an amount to the width (`and i32 %s, 31`), after casting it + // as unsigned (`zext i32 %n to i64`, `and i64 ..., 63`); a number's mask + // folds away. + try expectCount(ir, "arith.andi", 4); + try expectContains(ir, &.{ "arith.constant 31 : i32", "arith.constant 63 : i64" }); + var lines = std.mem.splitScalar(u8, ir, '\n'); + const unsigned_cast = while (lines.next()) |line| { + if (std.mem.find(u8, line, "arith.extui") != null and std.mem.find(u8, line, "i32 to i64") != null) break true; + } else false; + try std.testing.expect(unsigned_cast); +} + +test "a shift reads its amount once" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = Builder.open(std.testing.allocator, ctx, "k", .{}); + defer b.deinit(); + const a = try b.declareArgs(.{ + .o = .{ .buffer = .{ .type = .int } }, + .n = .{ .constant = .int }, + }); + // int s = n; *o = s << s; + const s = b.local(.int, a.n); + a.o.store(0, s.shl(s)); + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + // n, and s twice. + try expectCount(ir, "llvm.load", 3); +} + +test "math functions" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = Builder.open(std.testing.allocator, ctx, "k", .{}); + defer b.deinit(); + const a = try b.declareArgs(.{ + .x = .{ .buffer = .{ .type = .float } }, + .h = .{ .buffer = .{ .type = .half } }, + }); + const v = a.x.load(0); + var sum = b.local(.float, 0.0); + for ([_]Value{ + b.exp2(v), b.log(v), b.log2(v), b.tanh(v), b.sqrt(v), + b.rsqrt(v), b.ceil(v), b.floor(v), b.round(v), b.trunc(v), + b.sin(v), b.cos(v), b.pow(v, 2.0), b.fmod(v, 3.0), + }) |term| sum.assign(sum.add(term)); + const fast_ = b.fast(); + for ([_]Value{ + fast_.exp(v), fast_.log(v), fast_.log2(v), fast_.sqrt(v), fast_.rsqrt(v), + fast_.sin(v), fast_.cos(v), fast_.tanh(v), fast_.max(v, 1.0), fast_.min(v, 0.5), + }) |term| sum.assign(sum.add(term)); + const precise_ = b.precise(); + for ([_]Value{ + precise_.exp(v), precise_.exp2(v), precise_.log(v), precise_.log2(v), precise_.sqrt(v), + precise_.rsqrt(v), precise_.sin(v), precise_.cos(v), precise_.max(v, 1.0), precise_.min(v, 0.5), + }) |term| sum.assign(sum.add(term)); + a.x.store(1, sum); + // half has no fast:: functions: exp of a half is precise. + a.h.store(0, b.exp(a.h.load(1))); + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + try expectContains(ir, &.{ + "air.fast_exp2", "air.fast_log", "air.fast_log2", "air.fast_tanh", "air.fast_sqrt", "air.fast_rsqrt", + "air.fast_ceil", "air.fast_floor", "air.fast_round", "air.fast_trunc", "air.fast_sin", "air.fast_cos", + "air.fast_pow", "air.fast_fmod", "air.fast_exp", "air.fast_fmax", "air.fast_fmin", + }); + try expectContains(ir, &.{ "air.exp ", "air.exp2 ", "air.log ", "air.log2 ", "air.sqrt ", "air.rsqrt ", "air.sin ", "air.cos ", "air.fmax ", "air.fmin " }); + try expectContains(ir, &.{"f16"}); +} + +test "SIMD-group functions" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = Builder.open(std.testing.allocator, ctx, "k", .{}); + defer b.deinit(); + const a = try b.declareArgs(.{ + .x = .{ .buffer = .{ .type = .float } }, + .i = .{ .buffer = .{ .type = .int } }, + .u = .{ .buffer = .{ .type = .uint } }, + .lane = .{ .thread_index_in_simdgroup = .ushort }, + }); + const f = a.x.load(a.lane); + const n = a.i.load(a.lane); + const m = a.u.load(a.lane); + a.x.store(0, b.simdMin(f).add(b.simdPrefixExclusiveSum(f)).add(b.simdShuffle(f, a.lane)).add(b.simdShuffleDown(f, 1)).add(b.simdBroadcast(f, 0))); + a.i.store(0, b.simdMin(n).add(b.simdShuffle(n, 3)).add(b.simdShuffleDown(n, a.lane)).add(b.simdBroadcast(n, 0))); + a.u.store(0, b.simdMin(m).add(b.simdShuffleXor(m, 1))); + a.i.store(1, b.ternary(b.simdAny(f.gt(0.0)), 1, 0).add(b.ternary(b.simdAll(n.ne(0)), 2, 0))); + b.threadgroupBarrier(.mem_device); + b.simdgroupBarrier(.mem_texture); + b.threadgroupBarrier(.mem_none); + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + try expectContains(ir, &.{ + "air.simd_min ", "air.simd_min.s", "air.simd_min.u", "air.simd_prefix_exclusive_sum", + "air.simd_shuffle ", "air.simd_shuffle.s", "air.simd_shuffle_down ", "air.simd_shuffle_down.s", + "air.simd_broadcast ", "air.simd_broadcast.s", "air.simd_shuffle_xor.u", "air.simd_any", + "air.simd_all", "llvm.freeze", + }); + try expectContains(ir, &.{ "air.wg.barrier ", "air.simdgroup.barrier ", "air.wg.barrier " }); +} + +test "pragmas, conditions and casts" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = Builder.open(std.testing.allocator, ctx, "k", .{}); + defer b.deinit(); + const a = try b.declareArgs(.{ + .o = .{ .buffer = .{ .type = .float } }, + .n = .{ .constant = .int }, + .lane = .{ .thread_index_in_simdgroup = .ushort }, + }); + + const r = a.o.load(a.lane).mul(2.0); + // float acc[4] = {}; + const acc = b.zeroedArray(.float, 4); + // #pragma clang loop unroll(full) + var full = b.openForOpts(0, 4, 1, .{ .unroll = .full }); + { + acc.store(full.iv, r); + full.close(); + } + // #pragma unroll + var enabled = b.openForOpts(0, a.n, 2, .{ .unroll = .enable }); + { + acc.store(0, acc.load(0).add(enabled.iv.to(.float))); + enabled.close(); + } + // for (int i = 0; i < 4; i += 1) {} + const i = b.local(.int, 0); + var counted = b.openLoop(.{}); + { + counted.body(i.lt(4)); + } + { + counted.next(); + i.assign(i.add(1)); + counted.close(); + } + // const uint bits = as_type(acc[0]); + const bits = acc.load(0).asType(.uint); + // float t = r; *&t = 2.0f; + const t = b.local(.float, r); + t.address().store(0, 2.0); + // bool ok = n > 0 && bits != 0; + var both = b.openAnd(); + both.term(a.n.gt(0)); + const ok = both.close(bits.ne(0)); + // if (!(n > 1) || ok) { ... } else { ... } + var c1 = b.condition(); + c1.orNot(a.n.gt(1)); + var s1 = c1.openIfElse(ok, .{}); + { + a.o.store(1, 1.0); + s1.elseBranch(); + } + { + a.o.store(1, 2.0); + s1.close(); + } + // if (!ok && !(n == 3)) { ... } else { ... } + var c2 = b.condition(); + c2.andNot(ok); + var s2 = c2.openIfElseNot(a.n.eq(3), .{}); + { + a.o.store(2, 1.0); + s2.elseBranch(); + } + { + s2.close(); + } + // o[lane] = n > 2 ? fast::exp(t) : t; + var choice = b.openIfElse(a.n.gt(2), .{.float}); + { + choice.yieldThen(.{b.fast().exp(t)}); + } + { + choice.yieldElse(.{t}); + } + a.o.store(a.lane, choice.results[0]); + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + try expectContains(ir, &.{ "llvm.memset.p0.i64", "full = true", "disable = false", "llvm.bitcast" }); +} diff --git a/dsl/metal/types.zig b/dsl/metal/types.zig new file mode 100644 index 00000000..9c3072bb --- /dev/null +++ b/dsl/metal/types.zig @@ -0,0 +1,291 @@ +const std = @import("std"); + +const mlir = @import("mlir"); + +/// An MSL scalar type. +pub const Scalar = enum { + bool, + char, + uchar, + short, + ushort, + int, + uint, + long, + ulong, + half, + bfloat, + float, + + pub fn isFloat(self: Scalar) bool { + return switch (self) { + .half, .bfloat, .float => true, + else => false, + }; + } + + pub fn isUnsigned(self: Scalar) bool { + return switch (self) { + .bool, .uchar, .ushort, .uint, .ulong => true, + else => false, + }; + } + + pub fn bits(self: Scalar) u16 { + return switch (self) { + .bool => 1, + .char, .uchar => 8, + .short, .ushort, .half, .bfloat => 16, + .int, .uint, .float => 32, + .long, .ulong => 64, + }; + } + + /// The size in memory: a bool takes a byte. + pub fn size(self: Scalar) u64 { + return if (self == .bool) 1 else self.bits() / 8; + } + + /// The C++ integer conversion rank. + fn rank(self: Scalar) u8 { + return switch (self) { + .bool => 0, + .char, .uchar => 1, + .short, .ushort => 2, + .int, .uint => 3, + .long, .ulong => 4, + else => unreachable, + }; + } + + fn unsignedOf(self: Scalar) Scalar { + return switch (self) { + .char => .uchar, + .short => .ushort, + .int => .uint, + .long => .ulong, + else => self, + }; + } + + /// C++ integral promotion: types narrower than int become int. + pub fn promoted(self: Scalar) Scalar { + return switch (self) { + .bool, .char, .uchar, .short, .ushort => .int, + else => self, + }; + } + + pub fn mlirType(self: Scalar, ctx: *mlir.Context) *const mlir.Type { + return switch (self) { + .bool => .int(ctx, .i1), + .char, .uchar => .int(ctx, .i8), + .short, .ushort => .int(ctx, .i16), + .int, .uint => .int(ctx, .i32), + .long, .ulong => .int(ctx, .i64), + .half => .float(ctx, .f16), + .bfloat => .float(ctx, .bf16), + .float => .float(ctx, .f32), + }; + } +}; + +/// The C++ usual arithmetic conversions of two scalar operand types. +pub fn commonScalar(a: Scalar, b: Scalar) Scalar { + if (a.isFloat() or b.isFloat()) { + if (!b.isFloat()) return a; + if (!a.isFloat()) return b; + if (a == b) return a; + if (a == .float or b == .float) return .float; + std.debug.panic("dsl/metal: no common type of {t} and {t}", .{ a, b }); + } + const x = a.promoted(); + const y = b.promoted(); + if (x == y) return x; + if (x.isUnsigned() == y.isUnsigned()) return if (x.rank() >= y.rank()) x else y; + const u = if (x.isUnsigned()) x else y; + const s = if (x.isUnsigned()) y else x; + if (u.rank() >= s.rank()) return u; + if (s.bits() > u.bits()) return s; + return s.unsignedOf(); +} + +/// An MSL scalar or vector type: `float`, `uint3`, `bfloat4`. +pub const Type = struct { + scalar: Scalar, + /// 1 for a scalar, else the vector length (2, 3 or 4). + len: u8 = 1, + + pub const @"bool": Type = .{ .scalar = .bool }; + pub const char: Type = .{ .scalar = .char }; + pub const uchar: Type = .{ .scalar = .uchar }; + pub const short: Type = .{ .scalar = .short }; + pub const ushort: Type = .{ .scalar = .ushort }; + pub const int: Type = .{ .scalar = .int }; + pub const uint: Type = .{ .scalar = .uint }; + pub const long: Type = .{ .scalar = .long }; + pub const ulong: Type = .{ .scalar = .ulong }; + pub const half: Type = .{ .scalar = .half }; + pub const bfloat: Type = .{ .scalar = .bfloat }; + pub const float: Type = .{ .scalar = .float }; + pub const bool2: Type = .vec(.bool, 2); + pub const bool3: Type = .vec(.bool, 3); + pub const bool4: Type = .vec(.bool, 4); + pub const char2: Type = .vec(.char, 2); + pub const char3: Type = .vec(.char, 3); + pub const char4: Type = .vec(.char, 4); + pub const uchar2: Type = .vec(.uchar, 2); + pub const uchar3: Type = .vec(.uchar, 3); + pub const uchar4: Type = .vec(.uchar, 4); + pub const short2: Type = .vec(.short, 2); + pub const short3: Type = .vec(.short, 3); + pub const short4: Type = .vec(.short, 4); + pub const ushort2: Type = .vec(.ushort, 2); + pub const ushort3: Type = .vec(.ushort, 3); + pub const ushort4: Type = .vec(.ushort, 4); + pub const int2: Type = .vec(.int, 2); + pub const int3: Type = .vec(.int, 3); + pub const int4: Type = .vec(.int, 4); + pub const uint2: Type = .vec(.uint, 2); + pub const uint3: Type = .vec(.uint, 3); + pub const uint4: Type = .vec(.uint, 4); + pub const long2: Type = .vec(.long, 2); + pub const long3: Type = .vec(.long, 3); + pub const long4: Type = .vec(.long, 4); + pub const ulong2: Type = .vec(.ulong, 2); + pub const ulong3: Type = .vec(.ulong, 3); + pub const ulong4: Type = .vec(.ulong, 4); + pub const half2: Type = .vec(.half, 2); + pub const half3: Type = .vec(.half, 3); + pub const half4: Type = .vec(.half, 4); + pub const bfloat2: Type = .vec(.bfloat, 2); + pub const bfloat3: Type = .vec(.bfloat, 3); + pub const bfloat4: Type = .vec(.bfloat, 4); + pub const float2: Type = .vec(.float, 2); + pub const float3: Type = .vec(.float, 3); + pub const float4: Type = .vec(.float, 4); + + /// `vec`. + pub fn vec(scalar: Scalar, len: u8) Type { + std.debug.assert(len >= 2 and len <= 4); + return .{ .scalar = scalar, .len = len }; + } + + pub fn isVector(self: Type) bool { + return self.len > 1; + } + + pub fn element(self: Type) Type { + return .{ .scalar = self.scalar }; + } + + /// The same shape with another element type. + pub fn withScalar(self: Type, scalar: Scalar) Type { + return .{ .scalar = scalar, .len = self.len }; + } + + /// The type in memory, where a bool is a byte, as clang stores it. + pub fn memory(self: Type) Type { + return if (self.scalar == .bool) self.withScalar(.uchar) else self; + } + + /// `sizeof`: a 3-element vector takes the space of 4. + pub fn size(self: Type) u64 { + const len: u64 = if (self.len == 3) 4 else self.len; + return len * self.scalar.size(); + } + + /// `alignof`, which is the size for MSL scalars and vectors. + pub fn alignment(self: Type) u64 { + return self.size(); + } + + pub fn eql(self: Type, other: Type) bool { + return self.scalar == other.scalar and self.len == other.len; + } + + /// The MSL spelling, `uint3`. + pub fn name(self: Type) []const u8 { + const names = comptime blk: { + const scalars = @typeInfo(Scalar).@"enum".field_names; + var table: [scalars.len][4][]const u8 = undefined; + for (scalars, 0..) |scalar, i| { + table[i] = .{ scalar, scalar ++ "2", scalar ++ "3", scalar ++ "4" }; + } + break :blk table; + }; + return names[@backingInt(self.scalar)][self.len - 1]; + } + + pub fn mlirType(self: Type, ctx: *mlir.Context) *const mlir.Type { + const scalar = self.scalar.mlirType(ctx); + return if (self.isVector()) .vector(&.{self.len}, scalar) else scalar; + } +}; + +/// A field of a constant buffer struct: `float s[3]` is +/// `.{ .name = "s", .type = .float, .count = 3 }`. +pub const StructField = struct { + name: []const u8, + type: Type, + /// The array length, zero for a non-array field. + count: u32 = 0, +}; + +/// A struct passed by constant reference, laid out as MSL lays it out. +pub const Struct = struct { + name: []const u8, + fields: []const StructField, + + pub fn offset(self: Struct, index: usize) u64 { + var end: u64 = 0; + for (self.fields[0..index]) |field| { + end = std.mem.alignForward(u64, end, field.type.alignment()); + end += field.type.size() * @max(field.count, 1); + } + return std.mem.alignForward(u64, end, self.fields[index].type.alignment()); + } + + pub fn alignment(self: Struct) u64 { + var a: u64 = 1; + for (self.fields) |field| a = @max(a, field.type.alignment()); + return a; + } + + pub fn size(self: Struct) u64 { + const last = self.fields.len - 1; + const end = self.offset(last) + self.fields[last].type.size() * @max(self.fields[last].count, 1); + return std.mem.alignForward(u64, end, self.alignment()); + } + + pub fn fieldIndex(self: Struct, field_name: []const u8) usize { + for (self.fields, 0..) |field, i| if (std.mem.eql(u8, field.name, field_name)) return i; + std.debug.panic("dsl/metal: struct {s} has no field {s}", .{ self.name, field_name }); + } +}; + +test commonScalar { + try std.testing.expectEqual(Scalar.uint, commonScalar(.uint, .int)); + try std.testing.expectEqual(Scalar.int, commonScalar(.ushort, .short)); + try std.testing.expectEqual(Scalar.long, commonScalar(.long, .uint)); + try std.testing.expectEqual(Scalar.ulong, commonScalar(.ulong, .int)); + try std.testing.expectEqual(Scalar.float, commonScalar(.int, .float)); + try std.testing.expectEqual(Scalar.half, commonScalar(.half, .uint)); + try std.testing.expectEqual(Scalar.float, commonScalar(.half, .float)); +} + +test Struct { + const args: Struct = .{ .name = "A", .fields = &.{ + .{ .name = "n", .type = .int }, + .{ .name = "s", .type = .float, .count = 3 }, + .{ .name = "v", .type = .uint2 }, + .{ .name = "t", .type = .short }, + } }; + try std.testing.expectEqual(0, args.offset(0)); + try std.testing.expectEqual(4, args.offset(1)); + try std.testing.expectEqual(16, args.offset(2)); + try std.testing.expectEqual(24, args.offset(3)); + try std.testing.expectEqual(32, args.size()); + try std.testing.expectEqualStrings("bfloat4", Type.bfloat4.name()); + try std.testing.expectEqual(16, Type.float3.size()); +} diff --git a/dsl/mosaic_tpu/BUILD.bazel b/dsl/mosaic_tpu/BUILD.bazel new file mode 100644 index 00000000..15510258 --- /dev/null +++ b/dsl/mosaic_tpu/BUILD.bazel @@ -0,0 +1,32 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "builder", + srcs = [ + "dtype.zig", + "pipeline.zig", + ], + import_name = "dsl/mosaic_tpu/builder", + main = "builder.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//dsl/common:builder", + "//mlir", + "//mlir/dialects/affine", + "//mlir/dialects/arith", + "//mlir/dialects/cf", + "//mlir/dialects/func", + "//mlir/dialects/math", + "//mlir/dialects/memref", + "//mlir/dialects/mosaic_tpu", + "//mlir/dialects/scf", + "//mlir/dialects/vector", + "//stdx", + ], +) + +zig_test( + name = "test", + deps = [":builder"], +) diff --git a/dsl/mosaic_tpu/README.md b/dsl/mosaic_tpu/README.md new file mode 100644 index 00000000..cb1e0bab --- /dev/null +++ b/dsl/mosaic_tpu/README.md @@ -0,0 +1,846 @@ +# ZML Mosaic TPU DSL — Author's Guide + +A Zig builder for Mosaic TPU IR (the `tpu` dialect in XLA) that tries +to read like `pallas.tpu` Python while staying in pure Zig. Used by +`zml/kernels/moe/mosaic_tpu/`, `zml/kernels/paged_attention/mosaic_tpu/` +and `platforms/tpu/`, and by anything else that wants to emit a TPU kernel +op-for-op against what `pallas_call` produces. + +The DSL builds `tpu` operations and attributes through the bindings in +`mlir/dialects/mosaic_tpu`. +A Mosaic kernel is a pure IR generator: it builds its MLIR in a throwaway +context, canonicalizes it, serializes it to a string, and hands that to a +`stablehlo.custom_call` targeting `tpu_custom_call` — it never touches ZML's +compilation context (which carries only `func`/`stablehlo`/`sdy`). The dialect and +its C API come from the pinned XLA repository. + +Two layers define the surface: + +- **Low layer** — `dsl/mosaic_tpu/builder.zig`: the IR builder primitives. + `Builder.open(allocator, ctx, name)` → `b.declareArgsOpts(spec, results, opts)` + → body via `b.*` helpers → `b.finishOpts(results, .{ .canonicalize = true })` + for the final IR string. Pass `Builder.open` / `Builder.init` a + `zml.kernel.mosaic_tpu.newContext()` (a throwaway context with the dialects + the DSL emits), `defer ctx.deinit()` once you have the string — not a + `Compilation`'s `mlir_ctx`. +- **High layer** — `zml.kernel.mosaic_tpu.Kernel(Config, spec)`: the + declarative kernel form, mirroring the Triton DSL. Bundles a config + type, a typed spec literal (name + named `inputs` / `outputs` + a typed + `run` function pointer), and produces a kernel type. Exposes + `.emit(allocator, cfg)` (IR string; manages its own context) and + `.call(inputs, outputs, opts)` (IR string + `stablehlo.custom_call` + targeting `tpu_custom_call` in one shot — the preferred form for + production model code). + +## Table of contents + +1. [Skeleton of a kernel](#skeleton-of-a-kernel) +2. [Argument specs](#argument-specs) +3. [Pallas `pallas_call` metadata](#pallas-pallas_call-metadata) +4. [The `Value` type](#the-value-type) +5. [Fluent methods on `Value`](#fluent-methods-on-value) +6. [Polymorphic scalars](#polymorphic-scalars) +7. [Constants and constant hoisting](#constants-and-constant-hoisting) +8. [Loads and stores](#loads-and-stores) +9. [Vector and shape helpers](#vector-and-shape-helpers) +10. [Reductions, scans, sorts](#reductions-scans-sorts) +11. [Compute primitives](#compute-primitives) +12. [Memref ops](#memref-ops) +13. [DMAs, semaphores, and barriers](#dmas-semaphores-and-barriers) +14. [Control flow](#control-flow) +15. [Escape hatches](#escape-hatches) +16. [Pallas Python ↔ Zig DSL cheat-sheet](#pallas-python--zig-dsl-cheat-sheet) +17. [Pitfalls](#pitfalls) + +--- + +## Skeleton of a kernel + +```zig +const mlir = @import("mlir"); +const mtt = @import("zml").kernel.mosaic_tpu; + +pub const MyKernel = struct { + pub const Cfg = struct { /* shape/dtype/grid knobs */ }; + pub const Kernel = mtt.Kernel(Cfg, .{ + .name = "my_kernel", + .inputs = &.{ "x", "cu_seqlens" }, + .outputs = &.{"y"}, + .run = run, + }); + fn run(b: *mtt.Builder, cfg: Cfg) mtt.FinishError!void { + _ = cfg; + const a = try b.declareArgsOpts(.{ + // Grid program-ids — i32 scalars. `pl.program_id(0)`, `pl.program_id(1)`. + .i_b = .{ .scalar = .i32 }, + + // Inputs: VMEM memrefs. + .x = .{ .ref = .{ .shape = &.{ 128, 256 }, .dtype = .bf16 } }, + + // SMEM scalar-prefetch. + .cu_seqlens = .{ .ref = .{ .shape = &.{4}, .dtype = .i32, .memory_space = .smem } }, + + // Outputs. + .y = .{ .ref = .{ .shape = &.{ 128, 256 }, .dtype = .bf16, .role = .output } }, + }, &.{}, .{ + .dimension_semantics = &.{.arbitrary}, + .iteration_bounds = &.{ /* grid */ 4 }, + .pallas_window_params = true, + }); + _ = a; + + // ... emit body via b.* helpers; reach args through `a.`. + } +}; +``` + +A kernel is a `struct {…}` wrapper bundling three decls: + +1. **`pub const Cfg`** — config type, an inline `struct { ... }` of + comptime/runtime parameters. +2. **`pub const Kernel = mtt.Kernel(Cfg, .{ .name, .inputs, .outputs, .run })`** — + the spec literal that turns into a kernel type. `inputs` / `outputs` are + slices of `[:0]const u8` field names; the factory generates `Inputs` + (struct of `Tensor`), `Outputs` (struct of `Shape`), and `Results` + (struct of `Tensor`) types from those names. The `run` field is a typed + function pointer — wrong signature is a compile error here. +3. **`fn run(b, cfg)`** — the kernel body, mirroring the Triton form. First + call `try b.declareArgsOpts(.{…}, &.{}, opts)` to create the `func.func` + op and bind a `NamedArgs(Spec)` struct of `Value`s. The factory has + already opened the builder; it'll terminate, verify, run + `canonicalize,cse,canonicalize`, and serialize after `run` returns — + no manual `b.finish(...)` call. + +The names in `spec.inputs` / `spec.outputs` are the **call-site** names +(used to build the `Inputs` / `Outputs` structs) and don't have to match +the names in `b.declareArgsOpts(...)` (which are the **MLIR-level** arg +names — what shows up in the `func.func` signature). By convention they +often do — pick whatever reads best at the boundary. + +The arg spec inside `b.declareArgsOpts(...)` is a struct literal — its +variant *tags* (`.scalar`, `.iter`, `.ref`, `.sem`) are comptime-known per +call site, but the *payload values* (shapes, dtypes, memory spaces, +windows) can be runtime values from a config. + +> **Why the wrapper struct.** `Cfg` has to be a *named* type so the typed +> `run: *const fn (*Builder, Cfg) FinishError!void` field in the spec can +> reference it. Wrapping the kernel in `pub const MyKernel = struct {...}` +> keeps `Cfg`, `Kernel`, and `run` scoped together and exposes them all to +> callers (`MyKernel.Cfg`, `MyKernel.Kernel`, plus everything the inner +> Kernel re-exports — `MyKernel.Kernel.Config`, `.Inputs`, `.Outputs`, +> `.Results`, `.CallOpts`). +> +> Single-kernel files (e.g. `zml/kernels/paged_attention/mosaic_tpu/ragged_attention.zig`) +> can skip the wrapper and put `pub const Cfg` / `pub const Kernel` at +> module scope directly — the wrapper is just convenient when several +> kernels share a file. + +### Two ways to use a kernel + +**Offline / tooling** — get the MLIR string and print or diff it (no MLIR +context needed; `emit` spins up and tears down its own): + +```zig +const ir = try MyKernel.Kernel.emit(allocator, .{ /* cfg */ }); +defer allocator.free(ir); +try stdout.print("{s}\n", .{ir}); +``` + +**Inside a ZML model** — emit IR and drop a `stablehlo.custom_call` +(targeting `tpu_custom_call`) into the model in one shot: + +```zig +const out = MyKernel.Kernel.call( + .{ .x = x_tensor, .cu_seqlens = cu }, // Inputs (by name) + .{ .y = x_tensor.shape() }, // Outputs (Shapes) + .{ + .cfg = .{ /* cfg */ }, + .extras = .{ .vmem_limit_bytes = 1 << 20 }, + }, +).y; +``` + +`Kernel.call` takes a `mosaic_tpu.CallExtras` via `opts.extras` for +TPU-specific knobs (`vmem_limit_bytes`, `disable_bounds_checks`, +`disable_semaphore_checks`, `has_communication`, plus +`additional_attributes` and `output_operand_aliases` for in-place +updates). + +--- + +## Argument specs + +Each field passed to `b.declareArgs(...)` / `b.declareArgsOpts(...)` is an +`ArgSpec.Kind` tagged-union literal. The four kinds mirror what a Mosaic +`func.func` can carry: + +| Kind literal | MLIR type | Use case | +|--------------------------------------------------------|--------------------------------------|-----------------------------------------| +| `.{ .scalar = .i32 }` | plain `i32` | `program_id`, `num_programs`, sizes | +| `.{ .iter = .{ .semantics = .parallel } }` | `index` | Loop / grid iter index with optional `tpu.dimension_semantics` | +| `.{ .ref = .{ .shape = &.{...}, .dtype = .bf16 } }` | `memref<… x T, #tpu.memory_space<...>>` | Pallas-style operand | +| `.{ .sem_array = .{ .shape = &.{2}, .kind = .dma } }` | `memref<…x!tpu.(dma_)semaphore, semaphore_mem>` | Scratch sem pool (e.g. DMA double-buffer) | +| `.{ .sem = .regular }` / `.{ .sem = .dma }` | `!tpu.semaphore` / `!tpu.dma_semaphore` | Single semaphore handle | + +### Roles + +`RefSpec.role` mirrors what Pallas `pallas_call` calls each operand: + +| Role | Where it appears | `window_params` / `transform_N` | +|--------------------|-----------------------------------------------------|---------------------------------| +| `.input` (default) | Inputs declared via `in_specs` | one entry per input | +| `.output` | Outputs declared via `out_specs` | one entry per output | +| `.scalar_prefetch` | Leading SMEM refs (Pallas's `scalar_prefetch`) | none — but appended to every `transform_N` body's args | +| `.scratch` | Trailing scratch refs (kv_bufs, l/m/acc, sem_array) | none | + +The DSL filters `window_params` to only `.input`/`.output` entries (matching +Pallas's `len(grid_mapping.block_mappings)`), and re-emits the +`scalar_prefetch` refs as trailing args on every emitted `transform_N` stub. + +The default `memory_space` for a `.ref` is `.vmem`. Override with `.smem`, +`.cmem`, `.hbm`, `.semaphore_mem`, `.vmem_shared`, or `.any` per the +`MemorySpace` enum. Iteration semantics (`.parallel`, `.arbitrary`, +`.core_parallel`, `.subcore_parallel`) attach `tpu.dimension_semantics` to +the iter arg. + +**Runtime payloads:** the variant tags are fixed per kernel, but shape / +dtype / memory_space values can be runtime — typical for kernels +parameterized by a `Config` struct. + +--- + +## Pallas `pallas_call` metadata + +The wrapper attributes Pallas attaches to the `func.func` are exposed through +`Builder.Opts`: + +| Field | Type | Meaning | +|--------------------------|---------------------------|--------------------------------------------------------------| +| `.@"noinline"` | `bool` (default false) | `no_inline` attribute on the function. | +| `.core_type` | `?CoreType` (default `.tc`) | `tpu.core_type = #tpu.core_type<…>`. | +| `.dimension_semantics` | `[]const DimensionSemantics` | `dimension_semantics = [...]` — one entry per grid dim. | +| `.scalar_prefetch` | `i64` (default 0) | `scalar_prefetch = N : i64`. | +| `.scratch_operands` | `i64` (default 0) | `scratch_operands = N : i64`. | +| `.iteration_bounds` | `?[]const i64` | `iteration_bounds = array` — the grid shape. | +| `.pallas_window_params` | `bool` (default false) | Auto-emit `window_params = [...]` and trailing `transform_N` stub funcs. | + +Per-arg `window_params` behavior follows +`jax/_src/pallas/mosaic/lowering.py:1014–1131` exactly. With +`pallas_window_params = true`: + +- **`.hbm` / `.any` / `.semaphore_mem`** memory spaces ⇒ empty `{}` window + entry, **no `transform_N`**. Anything else gets a populated dict. +- **VMEM trivial-window** (no `RefSpec.window`, or `window.block_shape == ref.shape`) + ⇒ auto-force `pipeline_mode = synchronous` (matching Pallas's + `Buffered(1)` injection). +- **Otherwise** ⇒ emit only attributes the user explicitly set on + `RefSpec.window`. +- **`window_bounds`** = `block_shape` (defaults to operand shape). +- **`transform_N` body** returns `block_rank` i32 zeros, mirroring a trivial + index map. + +Override per ref-arg via `RefSpec.window`: + +```zig +.x = .{ .ref = .{ + .shape = &.{ 256, 512 }, + .dtype = .bf16, + .window = mtt.ArgSpec.WindowSpec{ + .block_shape = &.{ 128, 512 }, + .pipeline_mode = .double_buffered, + .revisit_mode = .any, + .element_window = .{ .pad_low = &.{ 0, 0 }, .pad_high = &.{ 0, 0 } }, + // Custom transform_N return — by default the body returns + // `block_rank` zeros (a trivial index_map). Override per slot. + .transform_returns = &.{ + .{ .program_id = 1 }, // q_blk_idx + .{ .program_id = 0 }, // heads_blk_idx + .zero, + }, + }, +} }, +``` + +The DSL treats the window as **non-trivial** (so it does *not* auto-force +`pipeline_mode = synchronous` for VMEM) when `transform_returns` is non-null +and contains anything other than all `.zero`s. This matches Pallas's +`bm.has_trivial_window()` check. + +`PipelineMode` is `{ synchronous, double_buffered }` (matching Pallas's +`Buffered(1)` / `Buffered(2)`). `RevisitMode` is `{ immediate, any }`. + +> **Anonymous-struct gotcha.** Anonymous-struct literals (`.{ ... }`) don't +> coerce through the optional-pointer wrapping that Zig 0.16 expects for +> `RefSpec.window`. Always write the type explicitly: +> `.window = mtt.ArgSpec.WindowSpec{ ... }`. + +--- + +## The `Value` type + +```zig +pub const Value = struct { + inner: *const mlir.Value, + kernel: ?*Builder = null, + ... +}; +``` + +- `inner` is the underlying MLIR SSA handle; reach for it when calling raw + `tpu.*` / `arith.*` / `vector.*` constructors. +- `kernel` is the owning `Builder` pointer; all fluent methods route through + it. The DSL populates `kernel` automatically for every value it creates + (`emit`, `arg`, loop-region args, etc.). If you construct a `Value` by hand + (`.{ .inner = raw }`), the fluent methods will panic — use `Builder.*` + static helpers instead. + +Introspection methods always available: + +| Method | Returns | Notes | +|-------------------|----------------------------------|------------------------------------------------| +| `type_()` | `*const mlir.Type` | Full MLIR type (scalar, vector, or memref). | +| `elemType()` | `*const mlir.Type` | Element type; same as `type_()` for scalars. | +| `isVector()` | `bool` | `true` for `vector<…>`. | +| `isMemRef()` | `bool` | `true` for `memref<…>`. | +| `isShaped()` | `bool` | Vector or memref. | +| `rank()` | `usize` | `0` for scalars. | +| `dim(i)` | `i64` | Traps on scalars. | +| `shape()` | `Shape` (BoundedArray, MAX_RANK) | Use `.constSlice()` to feed `[]const i64`. | +| `isFloatElem()` | `bool` | True for any float element type. | +| `isIntElem()` | `bool` | True for any integer element type. | + +--- + +## Fluent methods on `Value` + +Every fluent method returns a new `Value`. They dispatch to +`arith.addi` / `arith.addf` / … based on the element type. The RHS is +`anytype` — see [Polymorphic scalars](#polymorphic-scalars). + +### Arithmetic + +| Method | Int op | Float op | +|----------------|--------------|------------------| +| `.add(rhs)` | `arith.addi` | `arith.addf` | +| `.sub(rhs)` | `arith.subi` | `arith.subf` | +| `.mul(rhs)` | `arith.muli` | `arith.mulf` | +| `.div(rhs)` | `arith.divsi`| `arith.divf` | +| `.rem(rhs)` | `arith.remsi`| `arith.remf` | + +For Python-style floor-division (`a // b` matches `jax.numpy.floor_divide`'s +sign-correcting expansion — `divsi` plus a `(sign(a) != sign(b)) && (a % b != +0)` correction), use `b.divFloor(lhs, rhs)`. `divsi` alone truncates toward +zero, which differs for mixed-sign operands. +| `.bitAnd(rhs)` | `arith.andi` | — | +| `.bitOr(rhs)` | `arith.ori` | — | +| `.minimum(rhs)`| `arith.minsi`| `arith.minnumf` | +| `.maximum(rhs)`| `arith.maxsi`| `arith.maxnumf` | + +### Comparisons + +All return i1 (scalar) or `vector<… x i1>`. Signed integer predicates by +default; ordered float predicates for floats. + +| Method | Int predicate | Float predicate | +|------------|---------------|-----------------| +| `.lt(rhs)` | `.slt` | `.olt` | +| `.le(rhs)` | `.sle` | `.ole` | +| `.gt(rhs)` | `.sgt` | `.ogt` | +| `.ge(rhs)` | `.sge` | `.oge` | +| `.eq(rhs)` | `.eq` | `.oeq` | +| `.ne(rhs)` | `.ne` | `.one` | + +### Casts + +| Method | Effect | +|-----------------------|-----------------------------------------------------| +| `.to(dtype)` | Numeric cast preserving shape; auto-dispatches int↔float. | +| `.bitCast(dtype)` | `tpu.bitcast` — same-bitwidth element-type swap. | +| `.abs()` | `math.absf` (float) / `math.absi` (int). | + +`.to(...)` chooses the right MLIR op based on the source vs target kind: + +| Source kind | Target kind | MLIR op | +|-------------|----------------|--------------------| +| int | wider int | `arith.extsi` | +| int | narrower int | `arith.trunci` | +| int | float | `arith.sitofp` | +| float | int | `arith.fptosi` | +| float | wider float | `arith.extf` | +| float | narrower float | `arith.truncf` | +| same type | same type | no-op | + +For `index`-typed values use `b.indexCast(v, target_ty)` or `b.toIndex(v)` +explicitly — `.to(...)` is for numeric types. + +--- + +## Polymorphic scalars + +Most `Builder.*` helpers and every fluent method's RHS take `anytype`. The +lift rule: + +- `Value` → pass through. +- `comptime_int` / `comptime_float` → a constant whose element type matches + the LHS operand. `v_i64.mul(16)` lifts `16` to i64, `v_f32.add(1.0)` lifts + `1.0` to f32. +- Runtime `i8/i16/i32`, `i64`, `f16/f32/f64`, etc. → a constant that + preserves the source Zig width. + +If you need to force a specific width for a runtime variable, cast at the +call site: `pid.mul(@as(i32, @intCast(block_size)))`. + +### When do you need `b.lift` / `b.constant` / `b.constIndex`? + +| Helper | Effect | +|--------------------------------|----------------------------------------------------------| +| `b.lift(value)` | Wrap a Zig scalar; dtype inferred from source. | +| `b.constant(dtype, value)` | Wrap a Zig scalar as a specific `DType`. | +| `b.constantLike(value, like)` | Wrap a Zig scalar as `like`'s element type. | +| `b.constIndex(value)` | `arith.constant N : index` — for memref offsets. | + +Reach for these when the DSL surface demands an existing `Value` and there's +no binop sibling to match against — typically loop seeds for +`openWhile(.{inits}, ...)`, `openFor(0, N, 1, .{init})`, or `var x: Value = +b.constant(.i32, 0);` that a later branch may reassign. + +If the constant flows through any `.add` / `.mul` / `splat` etc., skip the +lift and let auto-lift do it. + +--- + +## Constants and constant hoisting + +The DSL provides splat-style constructors for the dense-splat fast-path that +Pallas/Mosaic uses: + +| Helper | Emits | +|----------------------------------------|---------------------------------------------| +| `b.zeros(shape, dtype)` | `arith.constant dense<0.0> : vector<…>` | +| `b.ones(shape, dtype)` | Ones vector via dense splat. | +| `b.full(shape, value, dtype)` | `arith.constant dense : vector<…>` | +| `b.splat(value, shape, dtype)` | Same as `full` for comptime values; `vector.broadcast` for runtime Values. | + +These emit a single `arith.constant` (instead of a scalar-const + +`vector.broadcast` pair) so the IR diff vs Pallas stays empty for splat-only +constants. + +--- + +## Loads and stores + +Mosaic separates **vector** loads/stores from **scalar** memref loads/stores. +Helpers infer everything from the memref's type — there is no `result_type` +argument. + +### Vector + +| Helper | Pallas / Mosaic equivalent | +|---------------------------------------------------------|-------------------------------------------------------| +| `b.refLoad(ref)` | `vector.load ref[0, …, 0] : memref<…>, vector<…>` | +| `b.vectorLoadAt(ref, indices)` | `vector.load ref[indices...]` with full ref shape. | +| `b.vectorLoadShape(ref, indices, shape)` | Sliced `vector.load` — `ref[i, j, :, k:k+BLOCK]`. | +| `b.tpuVectorLoad(ref, indices, .{ .strides, .mask })` | `tpu.vector_load` — sublane stride + mask form. | +| `b.refStore(ref, value)` | Full-shape `tpu.vector_store ref[0, …]`. | +| `b.vectorStoreAt(ref, value, indices)` | `tpu.vector_store ref[indices…]`. | +| `b.refStoreOpts(ref, value, indices, .{ .strides, .mask, .add })` | Strided / masked / atomic-add store. | +| `b.stridedLoad(ref, indices, strides)` | `tpu.strided_load` — result shape = `ref.shape / strides` (per-dim ceil). | +| `b.stridedLoadShape(ref, indices, strides, result_shape)` | `tpu.strided_load` with explicit result shape — for cropped loads (e.g. half-rows). | +| `b.stridedStore(ref, value, indices, strides)` | `tpu.strided_store`. | + +> **`refLoad` vs `tpuVectorLoad`.** Pallas's default emit is +> `vector.load`, which is what `refLoad` / `vectorLoadAt` produce. Reach for +> `tpuVectorLoad` only when you need sublane stride or a per-element mask +> (the `tpu.vector_load` form). Same on the store side. + +`indices` must be `index`-typed Values (use `b.toIndex(v)` to convert from +i32). If you pass `&.{}`, the helper pads with a cached `arith.constant 0 : +index` to match the ref's rank — the same SSA value is reused per block, so +no `%c0` duplication shows up in the IR. + +### Scalar (SMEM) + +| Helper | MLIR op | +|-------------------------------------------------|------------------------------------------| +| `b.scalarLoad(ref, indices)` | `memref.load` | +| `b.scalarStore(ref, value, indices)` | `memref.store` | + +Use these for SMEM args (e.g. `cu_seqlens`) where Pallas emits scalar +`memref.load`. + +--- + +## Vector and shape helpers + +| Helper | Effect | +|-----------------------------------------------------|----------------------------------------------------------| +| `b.broadcastTo(src, shape)` | `vector.broadcast` — leading-dim broadcast or rank-prepend. | +| `b.shapeCast(src, new_shape)` | `vector.shape_cast` — reshape preserving total lanes. | +| `b.reshape(src, new_shape)` | `tpu.reshape` (TPU-flavored reshape; differs from `shapeCast`). | +| `b.repeat(src, dimension, times)` | `tpu.concatenate` of `times` copies, as `pltpu.repeat`. | +| `b.concatenate(sources, dimension)` | `tpu.concatenate`. | +| `b.transpose(src, permutation)` | `tpu.transpose`. | +| `b.broadcastInSublanes(src, lane)` | `tpu.broadcast_in_sublanes`. | +| `b.rotate(src, .{ .amount, .dimension, .stride, .stride_dimension })` | `tpu.dynamic_rotate` with a constant i32 amount. | +| `b.vectorExtract(src, position)` | `vector.extract` — scalar / lower-rank slice. | +| `b.iota(shape, dtype, dimensions)` | `tpu.iota` over a chosen dim. | +| `b.arange(n, dtype)` | Pallas-style 1-D iota: `<1xN>` `tpu.iota` + `vector.shape_cast` to ``. Mirrors `_iota_lowering_rule`. | + +> **Use `arange`, not `iota(&.{N})`.** The TPU layout pass requires vectors +> to be 2-D with the lane dim trailing; a bare 1-D `tpu.iota` is rejected. +> `arange` emits the `<1xN>` + `shape_cast` pair Pallas's lowering produces. + +> **`shapeCast` vs `reshape`.** `shapeCast` emits the upstream +> `vector.shape_cast` Pallas uses everywhere; `reshape` emits TPU's +> `tpu.reshape`. Stick to `shapeCast` when matching Pallas IR. + +### Auto-broadcast + +The fluent binary ops and comparisons (`add` … `maximum`, `lt` … `ne`), +`b.where`, `b.maximum` and `b.minimum` splat a scalar operand against a +vector one (`vector.broadcast`) and widen the narrower integer side. Nothing +else broadcasts: Mosaic kernels run inside the TPU's tile semantics and the +layout pass is strict about `vector<…>` ranks/shapes, so build rank changes +and size-1 expansions explicitly with `shapeCast` + `broadcastTo`: + +```zig +// b_k[:, None] broadcast to +const bk_2d = k.shapeCast(b_k, &.{ BK, 1 }); +const bk_bc = k.broadcastTo(bk_2d, &.{ BK, BV }); +``` + +For a vector made from a scalar outside a binary op, use +`b.broadcastTo(scalar, shape)` (it routes through `vector.broadcast`) or +`b.full / b.splat` for comptime constants. + +--- + +## Reductions, scans, sorts + +### Vector-dialect reductions + +| Helper | Effect | +|-----------------------------------------------------|----------------------------------------------------------| +| `b.multiReduction(kind, src, acc, axes)` | `vector.multi_reduction` — multi-D reduce along `axes`. | +| `b.reduceToScalar(kind, src, acc)` | 1-D vec → scalar via the Pallas idiom: `shape_cast→<1xN>`, `multi_reduction[1]`, `extract[0]`. | +| `b.vectorReductionFlat(kind, src)` | `vector.reduction` — bare 1-D reduce. Layout pass may reject; prefer `reduceToScalar`. | + +`kind` is a `CombiningKind` (`.add`, `.mul`, `.minsi`, `.maxsi`, `.minnumf`, +`.maxnumf`, `.and_`, `.or_`, `.xor`). + +> **Pallas's reduce-to-scalar idiom.** Pallas never emits a bare +> `vector.reduction` for the L2-norm / sum-to-scalar pattern — it always +> goes through `multi_reduction` after a leading-dim `shape_cast`. The +> `reduceToScalar` wrapper produces exactly that shape, so kernels stay +> byte-equivalent to `pallas_call`'s output. See +> `jax/_src/pallas/mosaic/lowering.py:reduce_lowering_rule`. + +### TPU-dialect reductions / scan / sort + +| Helper | MLIR op | +|-----------------------------------------------------|----------------------------------------------------------| +| `b.allReduce(input, kind, dim)` | `tpu.all_reduce` — keeps `dim` as size 1. | +| `b.reduceIndex(input, kind, axis)` | `tpu.reduce_index` — drop axis, replace with `index`. | +| `b.scan(input, kind, mask)` | `tpu.scan` along the last dimension; optional rank-one mask. | +| `b.sort(keys, values, mask, .{ .descending })` | `tpu.sort` — returns `[3]Value` (sorted_keys, indices, sorted_values). | + +--- + +## Compute primitives + +### Math (scalar / elementwise) + +| Helper | MLIR op | +|------------------------------------------------|--------------------| +| `b.exp(x)` | `math.exp` | +| `b.exp2(x)` | `exp(x * ln2)` — matches Pallas's `_exp2_lowering_rule` (forward-compat). | +| `b.mathExp2(x)` | Raw `math.exp2` — only legal on cloud TPUs after 2025-07-26. | +| `b.log(x)` / `b.log2(x)` | `math.log` / `math.log2` | +| `b.sqrt(x)` / `b.rsqrt(x)` | `math.sqrt` / `math.rsqrt` | +| `b.sin(x)` / `b.cos(x)` / `b.tanh(x)` / `b.erf(x)` | `math.*` | +| `b.absf(x)` / `b.absi(x)` / `b.abs(x)` | `math.absf` / `math.absi` (auto) | +| `b.floor(x)` / `b.ceil(x)` | `math.floor` / `math.ceil` | +| `b.powf(x, y)` | `math.powf` | +| `b.fma(a, b, c)` | `math.fma` | +| `b.reciprocal(x)` / `b.reciprocalOpts(x, .{ .approximate })` | `tpu.reciprocal` | + +### Matmul + +```zig +// y = lhs @ rhs + acc +const y = k.matmul(lhs, rhs, acc); +const y2 = k.matmulOpts(lhs, rhs, acc, .{ + .transpose_lhs = false, + .transpose_rhs = true, + .precision = .bf16, +}); +``` + +`matmul` lowers to `tpu.matmul`; result type is taken from `acc`. Use +`matmulOpts` to set transposes, `tpu.contract_precision`, or a custom +`#tpu.dot_dimension_numbers<…>` attribute. + +### Selects / where + +| Helper | MLIR op | +|---------------------------------|--------------------------| +| `b.select(cond, t, f)` | `arith.select` | +| `b.where(cond, x, y)` | Alias of `select`. | + +### Masks + +| Helper | Effect | +|----------------------------------------------------------|-----------------------------------------| +| `b.createMask(low_bounds, high_bounds, shape)` | `vector.create_mask`. | +| `b.rangeMask(limits, shape)` | `vector.create_mask` from `0..limit`. | + +--- + +## Memref ops + +For carving sub-references out of a larger memref or reinterpreting an +existing one: + +| Helper | MLIR op | +|-----------------------------------------------------------------|--------------------------| +| `b.memRefSlice(mem_ref, offsets, sizes, strides, result_shape)` | `memref.subview`. | +| `b.memRefSqueeze(mem_ref, result_shape)` | `memref.collapse_shape` style. | +| `b.memRefReshape(mem_ref, result_shape)` | `tpu.memref_reshape`. | +| `b.memRefBitcast(mem_ref, dtype)` | `tpu.memref_bitcast` — same shape, new element dtype. | +| `b.memRefBitcastShape(mem_ref, result_shape, dtype)` | `tpu.memref_bitcast` with explicit result shape — for the bf16↔i32 packing trick (`<64x128xbf16>` → `<32x128xi32>`). | +| `b.reinterpretCast(mem_ref, result_shape, dtype)` | `memref.reinterpret_cast`. | +| `b.assumeLayout(src)` | `tpu.assume_layout`. | +| `b.eraseMemRefLayout(mem_ref)` | `memref.cast` to no-layout. | + +--- + +## DMAs, semaphores, and barriers + +For multi-core / async DMA orchestration: + +| Helper | MLIR op | +|------------------------------------------------------------------------------|------------------------| +| `b.semAlloc(.regular)` / `b.semAlloc(.dma)` | `tpu.sem_alloc`. | +| `b.semBarrier()` | `tpu.sem_barrier`. | +| `b.semRead(sem)` | `tpu.sem_read`. | +| `b.semWait(sem, amount)` | `tpu.sem_wait`. | +| `b.semSignal(sem, amount)` / `b.semSignalOpts(sem, amount, .{ .device_id, .core_id, .subcore_id })` | `tpu.sem_signal`. | +| `b.barrier(barrier_id)` | `tpu.barrier`. | +| `b.enqueueDma(src, dst, target_sem, .{ .source_semaphore, .device_id, .core_id, .subcore_id, .priority, .strict_ordering })` | `tpu.enqueue_dma`. | +| `b.waitDma2(sem, src, dst, .{ .device_id, .core_id, .strict_ordering })` | `tpu.wait_dma2`. | +| `b.deviceId()` | `tpu.device_id`. | +| `b.delay(cycles)` | `tpu.delay`. | +| `b.traceStart(level, message)` / `b.traceStop()` | `tpu.trace_start` / `tpu.trace_stop` — Pallas's einsum boundaries. | +| `b.dotDimensionNumbers(lhs_c, rhs_c, lhs_nc, rhs_nc, output_order, lhs_b, rhs_b)` | Build a `#tpu.dot_dimension_numbers<...>` attr for `matmulOpts`. | + +--- + +## Control flow + +SCF regions are built with **scope-based** builders — `b.openFor(...)`, +`b.openIf(...)` / `b.openIfElse(...)`, `b.openWhile(...)` return a typed +scope value. You emit body ops into a bare `{}` block, terminate with +`yield` / `yieldThen` / `yieldAfter` (or `close()` for a scope without +values), and read results from `scope.results` afterward. No `ctx` struct, +no function literal — lexical scope carries captures. + +### `openFor` — scf.for with iter_args + +```zig +var loop = k.openFor(c0_i32, T, c1_i32, .{ b_h_init }); +{ + const t = loop.iv; + const b_h = loop.carried[0]; + + // ... body ops ... + + loop.yield(.{ b_h_updated }); +} +const final_b_h = loop.results[0]; +``` + +- `lower` / `upper` / `step` are `anytype`: pass Values, comptime ints, or + runtime Zig ints. A bound that isn't a Value takes the type of the first one + that is; with none, all three are `index`. Pass `b.constant(.i32, 0)` as + `lower` for an `i32` counter (matching Pallas's `scf.for ... : i32`). +- `inits` is a tuple `.{v1, v2, ...}` or an array. Arity is comptime — + `carried`, `results`, and the `yield(...)` values (also a tuple or an + array) all have the same length. +- Empty loops: `b.openFor(0, N, 1, .{})` + `loop.close()`. +- `b.openForOpts(lower, upper, step, inits, .{ .unsigned_cmp = true })` sets + the loop's options. + +### `openIf` — scf.if without else (side-effects only) + +```zig +var i = k.openIf(cond); +{ + // then body — store, increment, etc. + i.close(); +} +``` + +Use for conditional side-effects (no results). The else block is auto-built +empty. + +### `openIfElse` — scf.if with else and optional results + +```zig +var i = k.openIfElse(cond, .{.f32}); +{ + i.yieldThen(.{ x }); +} +{ + i.yieldElse(.{ y }); +} +const r = i.results[0]; +``` + +- `result_types` is a tuple or array of dtypes (`.f32`), Values (for their + types) or `*const mlir.Type`s such as `k.vectorTy(shape, dtype)` / + `k.memRefTy(shape, dtype, mem_space)`. Pass `.{}` for an if/else with no + results, then end its branches with `elseBranch()` and `close()`. + +### `openWhile` — scf.while + +```zig +var w = k.openWhile(.{ i0 }, .{.i32}); +{ + const s = w.before_carried[0]; + w.yieldBefore(s.lt(10), .{ s }); +} +{ + const s = w.after_carried[0]; + w.yieldAfter(.{ s.add(1) }); +} +const r = w.results[0]; +``` + +- `inits` — a tuple or array of `Value`s; before-region arg types = their types. +- `after_types` — a tuple or array of dtypes (`.f32`), Values (for their + types) or `*const mlir.Type`s; after-region arg types + the scf.while's + result types. +- `yieldBefore(cond, forwarded)` — `forwarded` arity must match `after_types`. +- `yieldAfter(values)` — arity must match `inits`. + +### General tips + +- Scopes stack naturally: `openFor { openIf { openFor { ... } } }` all just + push/pop the kernel's current-block stack. +- Scopes without values end with `close()`: `loop.close()`, `i.close()`, and + `elseBranch()` then `close()` for an if/else. +- After any `yield*`, the scope's `.results` holds the scf op's results as + a `[N]Value`. + +--- + +## Escape hatches + +Everything the DSL doesn't expose is still reachable — `Builder` is a +book-keeping layer, not a walled garden: + +- `v.inner` — the raw `*const mlir.Value`; feed it to any `tpu.*` / + `arith.*` / `vector.*` / `memref.*` / `scf.*` / `math.*` constructor + directly. +- `b.ctx` — the owning `*mlir.Context`. +- `b.currentBlock()` — the active `mlir.Block` (respects `pushBlock` / + `popBlock`). +- `b.emit(op)` / `b.emitMulti(op, n)` — if you've built an `mlir.Operation` + by hand, use these so the result comes back as a `Value` with its + `kernel` field populated. +- `b.scalarTy(dtype)` / `b.vectorTy(shape, dtype)` / + `b.memRefTy(shape, dtype, mem_space)` — type constructors used by raw + ops and by non-scalar `openIfElse.result_types` / `openWhile.after_types`. + +When you drop to raw ops, remember to still **append them to the current +block** — `b.emit(...)` does that for you; `_ = op.appendTo(b.currentBlock())` +is the manual form. + +--- + +## Pallas Python ↔ Zig DSL cheat-sheet + +| Python | Zig | +|-------------------------------------------------------|--------------------------------------------------------------| +| `pl.program_id(0)` | `a.i_v` (declared as `.{ .scalar = .i32 }`) | +| `cu_seqlens[i]` | `k.scalarLoad(a.cu_seqlens, &.{ k.toIndex(i) })` | +| `lax.div(a, b)` | `k.divsi(a, b)` | +| `lax.rem(a, b)` | `k.remsi(a, b)` | +| `jnp.arange(N, dtype=jnp.int32)` | `k.arange(N, .i32)` | +| `o_k < KEY_DIM` | `k.cmpi(.slt, o_k, dim_splat)` / `o_k.lt(dim_splat)` | +| `mask_k[:, None] & mask_v[None, :]` | `k.andi(k.broadcastTo(k.shapeCast(mask_k, &.{BK,1}), &.{BK,BV}), k.broadcastTo(mask_v, &.{BK,BV}))` | +| `jnp.where(mask, x, 0.0)` | `k.select(mask, x, k.zeros(shape, dtype))` | +| `q_ref[0, t, h, :].astype(jnp.float32)` | `k.vectorLoadShape(a.q, &.{c0, t_idx, h_idx, c0}, &.{1,1,1,BK})` then `k.shapeCast(v, &.{BK})` then `.to(.f32)` | +| `q_ref[0, t, h, :] = b_o.astype(o_ref.dtype)` | `k.vectorStoreAt(a.q, k.shapeCast(b_o, &.{1,1,1,BK}), &.{c0, t_idx, h_idx, c0})` | +| `jnp.sqrt(jnp.sum(b_q * b_q) + 1e-6)` | `k.sqrt(k.addf(k.reduceToScalar(.add, k.mulf(b_q, b_q), cst_zero_1), cst_eps))` | +| `jnp.exp(b_g)` | `k.exp(b_g)` | +| `jnp.sum(b_h * b_k[:, None], axis=0)` | `k.multiReduction(.add, k.mulf(b_h, bk_bc), cst_zero_v, &.{0})` | +| `b_h * jnp.exp(b_g)` (broadcast to ) | `k.mulf(b_h, k.broadcastTo(exp_g, &.{BK, BV}))` | +| `b_beta * (b_v - predicted_v)` | `k.mulf(k.broadcastTo(b_beta, &.{BV}), k.subf(b_v, predicted_v))` | +| `lax.fori_loop(0, T, body, init)` | `var loop = k.openFor(c0_i32, T, c1_i32, .{init}); { …; loop.yield(.{…}); }` | +| `if cond: …` (side-effects only) | `var i = k.openIf(cond); { …; i.close(); }` | +| `if cond: x = a else: x = b` | `var i = k.openIfElse(cond, .{ty}); { …; i.yieldThen(.{a}); }; { …; i.yieldElse(.{b}); }` | +| `while cond:` | `var w = k.openWhile(.{inits}, .{after_types}); { …; w.yieldBefore(cond, .{…}); }; { …; w.yieldAfter(.{…}); }` | +| `pltpu.MemorySpace.SMEM` | `.memory_space = .smem` on the `RefSpec` | +| `pl.BlockSpec(memory_space=…)` (trivial window) | Default `RefSpec` — auto-`synchronous` on VMEM | +| `pl.BlockSpec(block_shape=(BM, BK), …)` | `.window = .{ .block_shape = &.{BM, BK} }` | +| `pl.BlockSpec(..., pipeline_mode=Buffered(2))` | `.window = .{ .pipeline_mode = .double_buffered }` | + +--- + +## Pitfalls + +1. **TPU vectors are 2-D minimum.** A bare `tpu.iota : vector` or + `vector.reduction` on `vector` is rejected by the layout pass. + Use `b.arange(N, dtype)` (emits the `<1xN>` iota + `shape_cast`) and + `b.reduceToScalar(...)` (emits the `shape_cast→<1xN>` + + `multi_reduction[1]` + `extract[0]` triple). Match Pallas's emit pattern + verbatim and the layout pass is happy. + +2. **No implicit auto-broadcast.** Unlike the Triton DSL, Mosaic won't + silently `expand_dims` / `broadcast` a rank-mismatched RHS. Build the + broadcasts explicitly (`shapeCast` + `broadcastTo`). The Pallas Python + DSL emits the same explicit broadcast pair — mirroring it keeps the IR + diff empty. + +3. **`vector.load` indices must be `index`-typed.** Convert from i32 with + `k.toIndex(v)` (emits `arith.index_cast`). JAX does no CSE inside + `pallas_call` lowering, so re-emitting the same `index_cast` per use is + the byte-equivalent thing to do. + +4. **Re-emit derived offsets per use.** Pallas re-runs its index_map + computation per operand load/store rather than CSE-ing intermediate + `arith.muli` / `arith.addi`. To match its IR exactly, do the same: emit + `iv_off = i_v * BV` again at every site that needs it. The MLIR + canonicalizer collapses duplicates downstream. + +5. **`refLoad` vs `tpuVectorLoad`.** Pallas's default `ref[...]` lowers to + `vector.load` (not `tpu.vector_load`). Use `refLoad` / `vectorLoadAt` / + `vectorLoadShape` for byte-equivalence; reach for `tpuVectorLoad` only + when you need `tpu.vector_load`-specific features (sublane stride or + per-element mask). + +6. **`shapeCast` vs `reshape`.** `shapeCast` emits the upstream + `vector.shape_cast` Pallas uses everywhere; `reshape` emits TPU's + `tpu.reshape` (different op, different layout semantics). Stick with + `shapeCast` when matching Pallas IR. + +7. **SMEM args take scalar `memref.load`, not `vector.load`.** Use + `k.scalarLoad(a.cu_seqlens, ...)` for SMEM operands (typically the + scalar-prefetch path). VMEM args go through the vector-load helpers. + +8. **`pallas_window_params = true` only adds metadata.** It does not change + the kernel body — it just appends `iteration_bounds`, `window_params`, + and the `transform_N` stub funcs to the `func.func` so downstream passes + that consume `window_params` (DMA scheduling, double-buffering) work. + For non-trivial windows or non-VMEM operands, attach a `RefSpec.window` + to override the auto-`synchronous` default. + +9. **`finish` verifies before serializing.** A verifier failure here almost + always means an operand-type mismatch or a missing block terminator — + `scf.yield` operand count must match the loop's result types, and + `func.return` arity must match the declared result types. + +The removed low-level `tpu.relayout` helper has no replacement: upstream deleted +the unused operation and its vector-layout attribute. `Builder.rotate` retains +its static-amount API and emits `tpu.dynamic_rotate` with an i32 constant. diff --git a/dsl/mosaic_tpu/builder.zig b/dsl/mosaic_tpu/builder.zig new file mode 100644 index 00000000..cffcd658 --- /dev/null +++ b/dsl/mosaic_tpu/builder.zig @@ -0,0 +1,2248 @@ +//! ZML Mosaic TPU DSL — a Zig builder for Mosaic TPU IR (the `tpu` dialect). +//! +//! Args are memrefs (`!memref`) plus optional scalars, +//! iteration-index, and semaphore args. The dialect must be registered into +//! the MLIR context — see `mlir/dialects/mosaic_tpu`. The entire Mosaic stack is +//! backed by the Mosaic dialect in the pinned XLA repository. + +const std = @import("std"); + +const affine = @import("mlir/dialects/affine"); +const arith = @import("mlir/dialects/arith"); +const cf = @import("mlir/dialects/cf"); +const func = @import("mlir/dialects/func"); +const math = @import("mlir/dialects/math"); +const memref = @import("mlir/dialects/memref"); +const scf = @import("mlir/dialects/scf"); +const vector = @import("mlir/dialects/vector"); +/// `vector`-dialect enums re-exported for convenience. +/// Used by `vectorReduction` / `multiReduction`. +pub const CombiningKind = vector.CombiningKind; +const common = @import("dsl/common/builder"); +const mlir = @import("mlir"); +const stdx = @import("stdx"); +const mosaic_tpu = @import("mlir/dialects/mosaic_tpu"); +/// `tpu`-dialect enums re-exported for convenience. +pub const MemorySpace = mosaic_tpu.MemorySpace; +pub const ReductionKind = mosaic_tpu.ReductionKind; +pub const ContractPrecision = mosaic_tpu.ContractPrecision; +pub const RoundingMode = mosaic_tpu.RoundingMode; +pub const DimensionSemantics = mosaic_tpu.DimensionSemantics; +pub const CoreType = mosaic_tpu.CoreType; +pub const PipelineMode = mosaic_tpu.PipelineMode; +pub const RevisitMode = mosaic_tpu.RevisitMode; + +const dtypes = @import("dtype.zig"); +pub const DType = dtypes.DType; +const isFloatDtype = dtypes.isFloatDtype; +const dtypeBitwidth = dtypes.dtypeBitwidth; +const intBitwidth = dtypes.intBitwidth; + +pub const pipeline = @import("pipeline.zig"); + +// ============================================================================= +// Value — a handle to an MLIR value inside the kernel body. +// ============================================================================= + +/// MLIR SSA value with an optional back-pointer to its `Builder` for fluent ops. +pub const Value = struct { + inner: *const mlir.Value, + kernel: ?*Builder = null, + + pub const Shape = stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK); + + pub fn type_(self: Value) *const mlir.Type { + return self.inner.type_(); + } + + fn kern(self: Value) *Builder { + return self.kernel orelse @panic("Value has no owning kernel; use Builder.* helpers instead"); + } + + /// Element type of a vector / memref value, or the value's own type if + /// scalar. + pub fn elemType(self: Value) *const mlir.Type { + if (self.type_().isA(mlir.ShapedType)) |shaped| return shaped.elementType(); + return self.type_(); + } + + /// True when this value is a vector (rank ≥ 1). + pub fn isVector(self: Value) bool { + return self.type_().isA(mlir.VectorType) != null; + } + + /// True when this value is a memref. + pub fn isMemRef(self: Value) bool { + return self.type_().isA(mlir.MemRefType) != null; + } + + /// True when the value is shaped (vector / memref / tensor). + pub fn isShaped(self: Value) bool { + return self.type_().isA(mlir.ShapedType) != null; + } + + /// Rank of this value. Scalars have rank 0. + pub fn rank(self: Value) usize { + if (self.type_().isA(mlir.ShapedType)) |shaped| return shaped.rank(); + return 0; + } + + /// `i`-th dim size of a shaped value; traps on scalar. + pub fn dim(self: Value, i: usize) i64 { + return self.type_().isA(mlir.ShapedType).?.dimension(i); + } + + /// Shape as a stack-allocated BoundedArray. Use `.constSlice()` for `[]const i64`. + pub fn shape(self: Value) Shape { + var out: Shape = .empty; + if (self.type_().isA(mlir.ShapedType)) |shaped| { + const r = shaped.rank(); + for (0..r) |i| out.appendAssumeCapacity(shaped.dimension(i)); + } + return out; + } + + /// True if the element type is a floating-point type (f16/bf16/f32/f64/fp8/…). + pub fn isFloatElem(self: Value) bool { + const et = self.elemType(); + inline for (comptime std.meta.fieldNames(mlir.FloatTypes)) |field_name| { + if (et.isA(mlir.FloatType(@field(mlir.FloatTypes, field_name))) != null) return true; + } + return false; + } + + /// True if the element type is an integer type. + pub fn isIntElem(self: Value) bool { + return self.elemType().isA(mlir.IntegerType) != null; + } + + // -------- fluent arithmetic (int/float auto-dispatch from elem type) -------- + + pub fn add(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return if (l.isFloatElem()) k.addf(l, r) else k.addi(l, r); + } + + pub fn sub(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return if (l.isFloatElem()) k.subf(l, r) else k.subi(l, r); + } + + pub fn mul(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return if (l.isFloatElem()) k.mulf(l, r) else k.muli(l, r); + } + + pub fn div(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return if (l.isFloatElem()) k.divf(l, r) else k.divsi(l, r); + } + + pub fn rem(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return if (l.isFloatElem()) k.remf(l, r) else k.remsi(l, r); + } + + pub fn bitAnd(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return k.andi(l, r); + } + + pub fn bitOr(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return k.ori(l, r); + } + + pub fn minimum(self: Value, rhs: anytype) Value { + return self.kern().minimum(self, rhs); + } + pub fn maximum(self: Value, rhs: anytype) Value { + return self.kern().maximum(self, rhs); + } + + // -------- fluent comparisons (int predicates default; float uses cmpf) -------- + + pub fn lt(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return if (l.isFloatElem()) k.cmpf(.olt, l, r) else k.cmpi(.slt, l, r); + } + pub fn le(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return if (l.isFloatElem()) k.cmpf(.ole, l, r) else k.cmpi(.sle, l, r); + } + pub fn gt(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return if (l.isFloatElem()) k.cmpf(.ogt, l, r) else k.cmpi(.sgt, l, r); + } + pub fn ge(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return if (l.isFloatElem()) k.cmpf(.oge, l, r) else k.cmpi(.sge, l, r); + } + pub fn eq(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return if (l.isFloatElem()) k.cmpf(.oeq, l, r) else k.cmpi(.eq, l, r); + } + pub fn ne(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.coerce(self, rhs); + return if (l.isFloatElem()) k.cmpf(.one, l, r) else k.cmpi(.ne, l, r); + } + + // -------- casts -------- + + /// Numeric cast preserving shape (vector or scalar). Auto-dispatches on + /// source / destination float-vs-int. + pub fn to(self: Value, dtype: DType) Value { + return self.kern().cast(self, dtype); + } + + /// `tpu.bitcast` — same-bitwidth element-type swap. + pub fn bitCast(self: Value, dtype: DType) Value { + return self.kern().bitCast(self, dtype); + } + + /// Absolute value (float→absf, int→absi). + pub fn abs(self: Value) Value { + return self.kern().abs(self); + } + + /// Best-effort fold to an `i64` for `arith.constant` (and `addi`/`muli` of folded + /// operands). Returns `null` for anything else; the caller decides what to do. + /// TODO(raph): check if there is no better way + pub fn asConstantInt(self: Value) ?i64 { + if (self.inner.isA(mlir.OpResult) == null) return null; + + const op = self.inner.owner(); + const op_name = op.name(); + + if (std.mem.eql(u8, op_name, "arith.constant")) { + const attr = op.attributeByName("value") orelse return null; + const iattr = attr.isA(mlir.IntegerAttribute) orelse return null; + return iattr.value(i64); + } + + if (std.mem.eql(u8, op_name, "arith.muli")) { + const lhs: Value = .{ .inner = op.operand(0), .kernel = self.kernel }; + const rhs: Value = .{ .inner = op.operand(1), .kernel = self.kernel }; + const lv = lhs.asConstantInt(); + const rv = rhs.asConstantInt(); + if (lv) |l| if (rv) |r| return l *% r; + // One side null, the other known: still folds if the known side is 0. + if (lv) |l| if (l == 0) return 0; + if (rv) |r| if (r == 0) return 0; + return null; + } + + if (std.mem.eql(u8, op_name, "arith.addi")) { + const lhs: Value = .{ .inner = op.operand(0), .kernel = self.kernel }; + const rhs: Value = .{ .inner = op.operand(1), .kernel = self.kernel }; + const lv = lhs.asConstantInt() orelse return null; + const rv = rhs.asConstantInt() orelse return null; + return lv +% rv; + } + + return null; + } +}; + +// ============================================================================= +// Arg spec — describes one function parameter. +// ============================================================================= + +/// Per-arg specification for building `func.func`'s signature. +pub const ArgSpec = struct { + name: []const u8, + kind: Kind, + + pub const Kind = union(enum) { + /// Plain scalar — typically used for grid sizes / dynamic dims passed + /// in by the runtime. `.{ .scalar = .i32 }`. + scalar: DType, + /// Loop / grid iteration index — `index` type with optional + /// `tpu.dimension_semantics` annotation. + /// `.{ .iter = .{ .semantics = .parallel } }`. + iter: IterOpts, + /// Memref (the bread-and-butter kernel arg). The arg is typed as + /// `memref>`. + /// `.{ .ref = .{ .shape = &.{128, 256}, .dtype = .bf16, .memory_space = .vmem } }`. + ref: RefSpec, + /// `memref` — the + /// scratch-array form Pallas uses for semaphore pools. Pallas + /// classifies it as `.scratch`, so it never appears in + /// `window_params` / `transform_N`. + sem_array: SemArraySpec, + /// `!tpu.semaphore` or `!tpu.dma_semaphore`. + /// `.{ .sem = .regular }` / `.{ .sem = .dma }`. + sem: SemKind, + }; + + pub const IterOpts = struct { + semantics: ?DimensionSemantics = null, + }; + + pub const RefSpec = struct { + shape: []const i64, + dtype: DType, + memory_space: MemorySpace = .vmem, + /// Emit `{sc.persistent}` on this memref argument. Sparse-core Pallas + /// kernels mark persistent HBM operands this way. + persistent: bool = false, + /// Block window. `null` ⇒ trivial window (block == operand shape); + /// VMEM trivial windows auto-get `pipeline_mode = synchronous`. + window: ?WindowSpec = null, + /// Pallas role of this operand. Only `.input`/`.output` get a + /// `window_params` entry and a `transform_N` stub. + /// `.scalar_prefetch` refs become trailing args of every + /// `transform_N` body (Pallas's index_map sees them). + /// `.scratch` refs are not visible to transforms or window_params. + role: Role = .input, + }; + + pub const Role = enum { input, output, scratch, scalar_prefetch }; + + pub const WindowSpec = struct { + /// `null` ⇒ use the operand's full shape (trivial window). + /// Determines `window_bounds` and the `transform_N` arity. + block_shape: ?[]const i64 = null, + /// `#tpu.pipeline_mode<…>`. `null` ⇒ auto-`synchronous` for VMEM trivial windows, omitted otherwise. + pipeline_mode: ?PipelineMode = null, + /// `#tpu.revisit_mode<…>`. `null` ⇒ omit. + revisit_mode: ?RevisitMode = null, + /// `#tpu.element_window<[pad_low], [pad_high]>`. `null` ⇒ omit. + element_window: ?ElementWindowSpec = null, + /// What `transform_N` returns. `null` ⇒ all-zero `block_rank` i32s + /// (matching a trivial index_map). One entry per result; length + /// must match `block_shape.len` (or operand rank if `block_shape` + /// is `null`). + transform_returns: ?[]const TransformReturn = null, + }; + + /// One return slot of a `transform_N` body. Mirrors what + /// `lower_jaxpr_to_transform_func` produces — typically a select of + /// program-ids, scalar-prefetch loads, or constants. We only encode + /// the most common forms; drop to a custom transform body for anything + /// fancier. + pub const TransformReturn = union(enum) { + /// Return `arith.constant 0 : i32`. + zero, + /// Return the Nth program-id (`%argN`). + program_id: usize, + }; + + pub const ElementWindowSpec = struct { + pad_low: []const i64, + pad_high: []const i64, + }; + + pub const SemKind = enum { regular, dma }; + + pub const SemArraySpec = struct { + shape: []const i64, + kind: SemKind = .dma, + }; +}; + +pub const FinishError = error{ InvalidMlir, MlirUnexpected } || std.mem.Allocator.Error || std.Io.Writer.Error; + +// ============================================================================= +// Named-param structs. +// ============================================================================= + +pub const VectorLoadOpts = struct { + /// Per-dim load stride. Pass `&.{}` for unit-strided. + strides: []const i32 = &.{}, + /// Optional element-wise mask. Must be a `vector<...xi1>` matching the + /// load result shape. + mask: ?Value = null, +}; + +pub const VectorStoreOpts = struct { + strides: []const i32 = &.{}, + mask: ?Value = null, + /// If true emits `tpu.vector_store ... add = true` (atomic-add semantics + /// where supported). + add: bool = false, +}; + +pub const MatmulOpts = struct { + transpose_lhs: bool = false, + transpose_rhs: bool = false, + transpose_lhs_hint: bool = false, + precision: ?ContractPrecision = null, + /// Optional `#tpu.dot_dimension_numbers<...>` attribute. When omitted the + /// canonicalizer derives one. Build with `mosaic_tpu.dotDimensionNumbersAttr`. + dimension_numbers: ?*const mlir.Attribute = null, +}; + +pub const ReduceOpts = struct { + /// Reduction dim. Defaults to 0. + dim: i64 = 0, +}; + +pub const SortOpts = struct { + descending: bool = false, +}; + +pub const SemSignalOpts = struct { + device_id: ?Value = null, + core_id: ?Value = null, + subcore_id: ?Value = null, +}; + +pub const EnqueueDmaOpts = struct { + source_semaphore: ?Value = null, + device_id: ?Value = null, + core_id: ?Value = null, + subcore_id: ?Value = null, + priority: i32 = 0, + strict_ordering: bool = false, +}; + +pub const WaitDma2Opts = struct { + device_id: ?Value = null, + core_id: ?Value = null, + strict_ordering: bool = false, +}; + +pub const ReciprocalOpts = struct { + approx: bool = false, + full_range: bool = false, +}; + +pub const RotateOpts = struct { + amount: i32, + dimension: i32, + stride: ?i32 = null, + stride_dimension: ?i32 = null, +}; + +pub const CastOpts = struct { + /// Forwarded to `tpu.fptosi` / `tpu.fptoui` / etc. + rounding: ?RoundingMode = null, + /// If true, perform a same-bitwidth `tpu.bitcast` instead of a numeric cast. + bitcast: bool = false, +}; + +pub const MinMaxOpts = struct { + /// IEEE NaN-propagation toggle. Default is non-propagating (maxnumf/minnumf). + propagate_nan: bool = false, +}; + +// ============================================================================= +// Builder — the main DSL builder. +// ============================================================================= + +pub fn registerDialects(registry: *mlir.DialectRegistry) void { + inline for (.{ func, mosaic_tpu, arith, scf, math, cf, memref, vector, affine }) |dialect| dialect.registerDialect(registry); +} + +/// The main DSL builder. Create with `init`, populate the body with helpers, +/// then call `finish` to get the IR string. +pub const Builder = struct { + allocator: std.mem.Allocator, + arena: std.heap.ArenaAllocator, + ctx: *mlir.Context, + module: *mlir.Module, + /// Name of the kernel — used when `declareArgs` creates the `func.func`. + name: []const u8, + /// `func.func` op for this kernel. `null` between `open` and the first + /// `declareArgs` call; populated thereafter. + func_op: ?*mlir.Operation, + /// Entry block for the `func.func`. `null` between `open` and the first + /// `declareArgs` call; populated thereafter. + entry_block: ?*mlir.Block, + block_stack: common.BlockStack = .{}, + /// Cached `arith.constant 0 : index` Value (and the block it lives + /// in) so repeat no-index loads / stores share an SSA value. See + /// `zeroIndex` for the lookup logic. + zero_index: ?Value = null, + zero_index_block: ?*mlir.Block = null, + /// Snapshot of the kernel's ref-args saved at `declareArgsLowOpts` + /// time. `finish` walks this list to emit `transform_N` stubs when + /// `pallas_window_params=true`. Allocated in `arena`. + ref_args: []RefArgSnapshot = &.{}, + /// SMEM scalar-prefetch ref types — appended to every `transform_N` + /// body's argument list. Empty when no `.scalar_prefetch` refs were + /// declared. + prefetch_args: []const *const mlir.Type = &.{}, + /// Number of grid program-ids (i32) at the head of every `transform_N` + /// body. Equals `Opts.dimension_semantics.len`. + grid_dim_count: usize = 0, + /// Whether to emit Pallas-style `transform_N` stubs at `finish` time. + /// Set by `declareArgsLowOpts` from `Opts.pallas_window_params`. + emit_transform_stubs: bool = false, + /// Optional prefix for generated transform function names. + transform_name_prefix: ?[]const u8 = null, + /// Whether generated transform functions carry `{sc.is_transform_indices}`. + sc_transform_indices: bool = false, + + /// Per-ref-arg snapshot for `finish`-time `transform_N` stub emission. + /// `skipped` = ANY/HBM/SEMAPHORE memory space (empty window, no transform). + pub const RefArgSnapshot = struct { + block_rank: i64, + skipped: bool, + role: ArgSpec.Role, + transform_returns: ?[]const ArgSpec.TransformReturn = null, + }; + + /// Snapshot of the SMEM scalar-prefetch ref types (after the program-ids) + /// — repeated as trailing args of every `transform_N` body to mirror + /// Pallas's `lower_jaxpr_to_transform_func`. + pub const PrefetchSnapshot = struct { + /// Pre-built MLIR memref types (cached pointers from declareArgs time). + types: []const *const mlir.Type, + }; + + /// Builder-level options. `noinline` is a Zig keyword — write `.{ .@"noinline" = true }`. + pub const Opts = struct { + @"noinline": bool = false, + /// `tpu.core_type` on the `func.func`. `null` skips the attr. + core_type: ?CoreType = .tc, + /// One entry per grid dim. Empty list emits an empty array attr + /// (matching Pallas's `dimension_semantics = []`). + dimension_semantics: []const DimensionSemantics = &.{}, + /// Number of leading scalar-prefetch operands. + scalar_prefetch: i64 = 0, + /// Number of trailing scratch refs (memrefs holding kernel-local + /// scratch state). + scratch_operands: i64 = 0, + /// Pallas's `iteration_bounds = array` — the grid shape + /// (one entry per `dimension_semantics`). `null` omits the attr. + iteration_bounds: ?[]const i64 = null, + /// Emit `scalar_prefetch = N : i64`. Sparse-core kernels with no + /// scalar prefetches omit the zero-valued attr. + emit_scalar_prefetch_attr: bool = true, + /// When `true`, auto-emit `window_params = [...]` and trailing + /// `transform_N` stub funcs at `finish` time. Per-arg behavior + /// matches `lowering.py:1014–1131`: ANY/HBM/SEMAPHORE refs get + /// an empty `{}` and no transform; VMEM trivial windows + /// auto-force `pipeline_mode = synchronous`. Override per-arg + /// via `RefSpec.window`. + pallas_window_params: bool = false, + /// Include HBM refs in Pallas-style `window_params`. Tensor-core + /// Pallas lowering skips HBM; sparse-core lowering includes it. + window_hbm_refs: bool = false, + /// Optional prefix for generated transform function names. + transform_name_prefix: ?[]const u8 = null, + /// Add `{sc.is_transform_indices}` to generated transform stubs. + sc_transform_indices: bool = false, + }; + + /// Create a sub-module containing a single public `func.func` of the + /// given name, signature, and result types. + pub fn init( + allocator: std.mem.Allocator, + ctx: *mlir.Context, + name: []const u8, + args: []const ArgSpec, + result_types: []const *const mlir.Type, + ) !Builder { + return initOpts(allocator, ctx, name, args, result_types, .{}); + } + + pub fn initOpts( + allocator: std.mem.Allocator, + ctx: *mlir.Context, + name: []const u8, + args: []const ArgSpec, + result_types: []const *const mlir.Type, + opts: Opts, + ) !Builder { + var b = try open(allocator, ctx, name); + errdefer b.deinit(); + try b.declareArgsLowOpts(args, result_types, opts); + return b; + } + + /// Create an empty `Builder` with no `func.func` yet. + pub fn open( + allocator: std.mem.Allocator, + ctx: *mlir.Context, + name: []const u8, + ) !Builder { + const unknown_loc: *const mlir.Location = .unknown(ctx); + const module: *mlir.Module = .init(unknown_loc); + errdefer module.deinit(); + + var arena: std.heap.ArenaAllocator = .init(allocator); + errdefer arena.deinit(); + + return .{ + .allocator = allocator, + .arena = arena, + .ctx = ctx, + .module = module, + .name = name, + .func_op = null, + .entry_block = null, + .block_stack = .{}, + }; + } + + /// Declare the kernel's args from a named-field struct literal and + /// return a `NamedArgs(Spec)`. + pub fn declareArgs(self: *Builder, spec: anytype) !common.NamedArgs(@TypeOf(spec), Value) { + return self.declareArgsOpts(spec, &.{}, .{}); + } + + pub fn declareArgsOpts( + self: *Builder, + spec: anytype, + result_types: []const *const mlir.Type, + opts: Opts, + ) !common.NamedArgs(@TypeOf(spec), Value) { + const Spec = @TypeOf(spec); + const field_names = @typeInfo(Spec).@"struct".field_names; + + var arg_specs: [field_names.len]ArgSpec = undefined; + inline for (field_names, 0..) |field_name, i| { + const raw = @field(spec, field_name); + const kind: ArgSpec.Kind = if (@TypeOf(raw) == ArgSpec.Kind) raw else blk: { + const variant = @typeInfo(@TypeOf(raw)).@"struct".field_names[0]; + const tag = @field(std.meta.Tag(ArgSpec.Kind), variant); + const inner = @field(raw, variant); + break :blk switch (tag) { + .scalar => .{ .scalar = inner }, + .iter => .{ .iter = .{ + .semantics = if (@hasField(@TypeOf(inner), "semantics")) inner.semantics else null, + } }, + .ref => .{ .ref = .{ + .shape = inner.shape, + .dtype = inner.dtype, + .memory_space = if (@hasField(@TypeOf(inner), "memory_space")) inner.memory_space else .vmem, + .persistent = if (@hasField(@TypeOf(inner), "persistent")) inner.persistent else false, + .window = if (@hasField(@TypeOf(inner), "window")) inner.window else null, + .role = if (@hasField(@TypeOf(inner), "role")) inner.role else .input, + } }, + .sem_array => .{ .sem_array = .{ + .shape = inner.shape, + .kind = if (@hasField(@TypeOf(inner), "kind")) inner.kind else .dma, + } }, + .sem => .{ .sem = inner }, + }; + }; + arg_specs[i] = .{ .name = field_name, .kind = kind }; + } + + try self.declareArgsLowOpts(&arg_specs, result_types, opts); + + var named: common.NamedArgs(Spec, Value) = undefined; + inline for (field_names, 0..) |field_name, i| { + @field(named, field_name) = self.arg(i); + } + return named; + } + + fn declareArgsLowOpts( + self: *Builder, + args: []const ArgSpec, + result_types: []const *const mlir.Type, + opts: Opts, + ) !void { + std.debug.assert(self.entry_block == null); + const ctx = self.ctx; + const unknown_loc: *const mlir.Location = .unknown(ctx); + const scratch = self.arena.allocator(); + + const arg_types = try scratch.alloc(*const mlir.Type, args.len); + const arg_locs = try scratch.alloc(*const mlir.Location, args.len); + for (args, 0..) |a, i| { + arg_types[i] = switch (a.kind) { + .scalar => |dt| dt.toMlir(ctx), + .iter => .index(ctx), + .ref => |r| .memRef( + r.dtype.toMlir(ctx), + r.shape, + null, + r.memory_space.attribute(ctx), + ), + .sem => |s| switch (s) { + .regular => mosaic_tpu.semaphoreType(ctx), + .dma => mosaic_tpu.dmaSemaphoreType(ctx), + }, + .sem_array => |sa| .memRef( + switch (sa.kind) { + .regular => mosaic_tpu.semaphoreType(ctx), + .dma => mosaic_tpu.dmaSemaphoreType(ctx), + }, + sa.shape, + null, + MemorySpace.semaphore_mem.attribute(ctx), + ), + }; + arg_locs[i] = unknown_loc.named(ctx, a.name); + } + + const entry = mlir.Block.init(arg_types, arg_locs); + + // Build per-arg attribute dicts. + const arg_attrs = try scratch.alloc(*const mlir.Attribute, args.len); + var any_arg_attr = false; + for (args, 0..) |a, i| { + const empty_dict: *const mlir.Attribute = .dict(ctx, &.{}); + switch (a.kind) { + .iter => |io| { + if (io.semantics) |sem| { + arg_attrs[i] = .dict(ctx, &.{ + mosaic_tpu.dimensionSemanticsNamedAttr(ctx, sem), + }); + any_arg_attr = true; + } else { + arg_attrs[i] = empty_dict; + } + }, + .ref => |r| { + if (r.persistent) { + arg_attrs[i] = .dict(ctx, &.{ + .named(ctx, "sc.persistent", .unit(ctx)), + }); + any_arg_attr = true; + } else { + arg_attrs[i] = empty_dict; + } + }, + else => arg_attrs[i] = empty_dict, + } + } + + // Kernel-level func.func attributes. MLIR sorts attrs alphabetically on print. + var extra_attrs: stdx.BoundedArray(mlir.NamedAttribute, 12) = .empty; + const dim_sem_buf = try scratch.alloc(*const mlir.Attribute, opts.dimension_semantics.len); + for (opts.dimension_semantics, 0..) |s, i| dim_sem_buf[i] = s.attribute(ctx); + extra_attrs.appendAssumeCapacity(.named(ctx, "dimension_semantics", .array(ctx, dim_sem_buf))); + if (opts.iteration_bounds) |bounds| { + extra_attrs.appendAssumeCapacity(.named(ctx, "iteration_bounds", .denseArray(ctx, .i64, bounds))); + } + if (opts.emit_scalar_prefetch_attr) { + extra_attrs.appendAssumeCapacity(.named(ctx, "scalar_prefetch", .int(ctx, .i64, opts.scalar_prefetch))); + } + extra_attrs.appendAssumeCapacity(.named(ctx, "scratch_operands", .int(ctx, .i64, opts.scratch_operands))); + if (opts.core_type) |ct| { + extra_attrs.appendAssumeCapacity(mosaic_tpu.coreTypeNamedAttr(ctx, ct)); + } + + // Snapshot the ref args for `finish`-time stub emission. Only + // `.input` / `.output` refs participate in `window_params` and + // `transform_N` — `.scalar_prefetch` and `.scratch` are ignored + // (they're tracked separately for transform-arg expansion). + const arena_alloc = self.arena.allocator(); + var ref_args_buf: std.ArrayList(RefArgSnapshot) = .empty; + var prefetch_types_buf: std.ArrayList(*const mlir.Type) = .empty; + for (args, 0..) |a, i| switch (a.kind) { + .ref => |r| { + if (r.role == .input or r.role == .output) { + const block_shape = if (r.window) |w| + (w.block_shape orelse r.shape) + else + r.shape; + const skipped = isSkippedWindowSpace(r.memory_space) and !(opts.window_hbm_refs and r.memory_space == .hbm); + try ref_args_buf.append(arena_alloc, .{ + .block_rank = @intCast(block_shape.len), + .skipped = skipped, + .role = r.role, + .transform_returns = if (r.window) |w| w.transform_returns else null, + }); + } else if (r.role == .scalar_prefetch) { + try prefetch_types_buf.append(arena_alloc, arg_types[i]); + } + }, + else => {}, + }; + self.ref_args = try arena_alloc.dupe(RefArgSnapshot, ref_args_buf.items); + self.prefetch_args = try arena_alloc.dupe(*const mlir.Type, prefetch_types_buf.items); + self.grid_dim_count = opts.dimension_semantics.len; + self.emit_transform_stubs = opts.pallas_window_params; + self.transform_name_prefix = opts.transform_name_prefix; + self.sc_transform_indices = opts.sc_transform_indices; + + // window_params: one entry per `.input` / `.output` ref. + // ANY/HBM/SEMAPHORE → empty dict, no transform_N. + // Otherwise → {transform_indices, window_bounds, [pipeline_mode], [revisit_mode], [window_kind]}. + // VMEM trivial-window → auto-force pipeline_mode = synchronous. + if (opts.pallas_window_params) { + var idx: usize = 0; + const wp_buf = try scratch.alloc(*const mlir.Attribute, self.ref_args.len); + for (args) |a| switch (a.kind) { + .ref => |r| { + if (r.role != .input and r.role != .output) continue; + if (isSkippedWindowSpace(r.memory_space) and !(opts.window_hbm_refs and r.memory_space == .hbm)) { + wp_buf[idx] = .dict(ctx, &.{}); + idx += 1; + continue; + } + + const w = r.window orelse ArgSpec.WindowSpec{}; + const block_shape = w.block_shape orelse r.shape; + // `bm.has_trivial_window()` in pallas — block_shape == + // operand_shape AND the index_map is the identity + // (returns all zeros). + const has_trivial_index_map = if (w.transform_returns) |trs| blk: { + for (trs) |tr| { + if (tr != .zero) break :blk false; + } + break :blk true; + } else true; + const trivial = std.mem.eql(i64, block_shape, r.shape) and has_trivial_index_map; + + const resolved_pipeline: ?PipelineMode = + w.pipeline_mode orelse blk: { + // VMEM + trivial window ⇒ Buffered(1) ⇒ synchronous. + if (r.memory_space == .vmem and trivial) break :blk .synchronous; + break :blk null; + }; + + const sym_name = if (opts.transform_name_prefix) |prefix| + try std.fmt.allocPrint(scratch, "{s}_transform_{d}", .{ prefix, idx }) + else + try std.fmt.allocPrint(scratch, "transform_{d}", .{idx}); + + var inner: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + if (resolved_pipeline) |pm| { + inner.appendAssumeCapacity(.named(ctx, "pipeline_mode", pm.attribute(ctx))); + } + if (w.revisit_mode) |rm| { + inner.appendAssumeCapacity(.named(ctx, "revisit_mode", rm.attribute(ctx))); + } + inner.appendAssumeCapacity(.named(ctx, "transform_indices", .flatSymbolRef(ctx, sym_name))); + inner.appendAssumeCapacity(.named(ctx, "window_bounds", .denseArray(ctx, .i64, block_shape))); + if (w.element_window) |ew| { + inner.appendAssumeCapacity(.named(ctx, "window_kind", mosaic_tpu.elementWindowAttr(ctx, ew.pad_low, ew.pad_high))); + } + wp_buf[idx] = .dict(ctx, inner.constSlice()); + idx += 1; + }, + else => {}, + }; + extra_attrs.appendAssumeCapacity(.named(ctx, "window_params", .array(ctx, wp_buf))); + } + + const func_op = func.func(ctx, .{ + .name = self.name, + .args = arg_types, + .args_attributes = if (any_arg_attr) arg_attrs else null, + .results = result_types, + .block = entry, + .location = unknown_loc, + .no_inline = opts.@"noinline", + // Body is empty at declare time — verify only at `finish`. + .verify = false, + .extra_attributes = extra_attrs.constSlice(), + }); + _ = func_op.appendTo(self.module.body()); + + self.func_op = func_op; + self.entry_block = entry; + self.block_stack.push(self.arena.allocator(), entry); + } + + pub fn deinit(self: *Builder) void { + self.module.deinit(); + self.arena.deinit(); + } + + /// The i-th entry block argument. + pub fn arg(self: *Builder, i: usize) Value { + const eb = self.entry_block orelse @panic("Builder.arg called before declareArgs"); + return .{ .inner = eb.argument(i), .kernel = self }; + } + + pub fn pushBlock(self: *Builder, b: *mlir.Block) void { + self.block_stack.push(self.arena.allocator(), b); + } + + pub fn popBlock(self: *Builder) void { + self.block_stack.pop(); + } + + pub fn currentBlock(self: *Builder) *mlir.Block { + return self.block_stack.current(); + } + + /// Unpack a slice of `Value`s into an arena-allocated slice of raw MLIR + /// value pointers. + fn innerSlice(self: *Builder, values: []const Value) []const *const mlir.Value { + const out = self.arena.allocator().alloc(*const mlir.Value, values.len) catch @panic("Builder.innerSlice OOM"); + for (values, 0..) |v, i| out[i] = v.inner; + return out; + } + + /// Emit `op` into the current block and return its first result as a Value. + pub fn emit(self: *Builder, op: *mlir.Operation) Value { + _ = op.appendTo(self.currentBlock()); + return .{ .inner = op.result(0), .kernel = self }; + } + + pub fn emitMulti(self: *Builder, op: *mlir.Operation, n: usize) []Value { + _ = op.appendTo(self.currentBlock()); + const out = self.arena.allocator().alloc(Value, n) catch @panic("Builder.emitMulti OOM"); + for (0..n) |i| out[i] = .{ .inner = op.result(i), .kernel = self }; + return out; + } + + pub fn yield(self: *Builder, values: anytype) []const Value { + const T = @TypeOf(values); + const info = @typeInfo(T); + if (info != .@"struct" or !info.@"struct".is_tuple) + @compileError("Builder.yield expects a tuple literal like `.{ v1, v2 }`"); + const n = info.@"struct".field_names.len; + const out = self.arena.allocator().alloc(Value, n) catch @panic("Builder.yield OOM"); + inline for (info.@"struct".field_names, info.@"struct".field_types, 0..) |field_name, Field, i| { + if (Field != Value) + @compileError("Builder.yield: every tuple element must be a Value; got " ++ @typeName(Field)); + out[i] = @field(values, field_name); + } + return out; + } + + pub fn loc(self: *const Builder) *const mlir.Location { + return .unknown(self.ctx); + } + + // ==================== type helpers ==================== + + pub fn scalarTy(self: *const Builder, dtype: DType) *const mlir.Type { + return dtype.toMlir(self.ctx); + } + + pub fn vectorTy(self: *const Builder, shape: []const i64, dtype: DType) *const mlir.Type { + return .vector(shape, dtype.toMlir(self.ctx)); + } + + pub fn memRefTy(self: *const Builder, shape: []const i64, dtype: DType, memory_space: MemorySpace) *const mlir.Type { + return .memRef(dtype.toMlir(self.ctx), shape, null, memory_space.attribute(self.ctx)); + } + + fn mlirElemToDType(self: *const Builder, elem: *const mlir.Type) DType { + return dtypes.mlirElemToDType(self.ctx, elem); + } + + /// Replace `src`'s element type with `out_dtype`, preserving rank/shape. + fn swapElem(self: *const Builder, src: Value, out_dtype: DType) *const mlir.Type { + const out_elem = out_dtype.toMlir(self.ctx); + if (src.isVector()) return .vector(src.shape().constSlice(), out_elem); + if (src.isMemRef()) { + const mr = src.type_().isA(mlir.MemRefType).?; + return .memRef(out_elem, src.shape().constSlice(), null, mr.memorySpace()); + } + return out_elem; + } + + // ==================== constants ==================== + + fn emitInt(self: *Builder, dtype: DType, value: i64) Value { + return self.emit(arith.constant_int(self.ctx, value, dtype.toMlir(self.ctx), self.loc())); + } + + fn emitFloat(self: *Builder, dtype: DType, value: f64) Value { + return switch (dtype) { + .f16 => self.emit(arith.constant_float(self.ctx, value, .f16, self.loc())), + .bf16 => self.emit(arith.constant_float(self.ctx, value, .bf16, self.loc())), + .f32 => self.emit(arith.constant_float(self.ctx, value, .f32, self.loc())), + .f64 => self.emit(arith.constant_float(self.ctx, value, .f64, self.loc())), + // fp8 has no direct arith.constant; emit f32 then truncf. + else => self.arithTruncf(self.emitFloat(.f32, value), dtype), + }; + } + + /// Constant `index`-typed value (for memref / loop bounds). + pub fn constIndex(self: *Builder, value: i64) Value { + return self.emit(arith.constant_index(self.ctx, value, self.loc())); + } + + /// `openFor`'s default lower bound, an `index` constant. + fn indexBound(self: *Builder, value: anytype) Value { + return self.constIndex(intBits(value, 64, "index")); + } + + /// Lift a Zig scalar (or pass-through Value) to a DSL constant. + pub fn lift(self: *Builder, value: anytype) Value { + const T = @TypeOf(value); + if (T == Value) return value; + return switch (@typeInfo(T)) { + .comptime_int => blk: { + if (value >= std.math.minInt(i32) and value <= std.math.maxInt(i32)) + break :blk self.emitInt(.i32, value); + if (value >= std.math.minInt(i64) and value <= std.math.maxInt(i64)) + break :blk self.emitInt(.i64, value); + if (value >= 0 and value <= std.math.maxInt(u64)) + break :blk self.emitInt(.i64, @bitCast(@as(u64, value))); + @compileError("Builder.lift: integer literal out of 64-bit range"); + }, + .int => |info| blk: { + const signed = info.signedness == .signed; + if (info.bits > 32) { + const v: i64 = if (signed) @intCast(value) else @bitCast(@as(u64, @intCast(value))); + break :blk self.emitInt(.i64, v); + } + const v32: i32 = if (signed) @intCast(value) else @bitCast(@as(u32, @intCast(value))); + break :blk self.emitInt(.i32, v32); + }, + .comptime_float => self.emitFloat(.f32, @floatCast(value)), + .float => |info| switch (info.bits) { + 16 => self.emitFloat(.f16, @floatCast(value)), + 32 => self.emitFloat(.f32, @floatCast(value)), + 64 => self.emitFloat(.f64, value), + else => @compileError("Builder.lift: unsupported float bitwidth"), + }, + else => @compileError("Builder.lift: unsupported type " ++ @typeName(T)), + }; + } + + /// `value` as a constant of `dtype`; a Value passes through. An integer is + /// signed or an unsigned bit pattern of `dtype`'s width; anything else + /// panics. + pub fn constant(self: *Builder, dtype: DType, value: anytype) Value { + const T = @TypeOf(value); + if (T == Value) return value; + if (isFloatDtype(dtype)) { + const v_f64: f64 = switch (@typeInfo(T)) { + .comptime_int, .int => @floatFromInt(value), + .comptime_float, .float => @floatCast(value), + else => @compileError("Builder.constant: unsupported scalar " ++ @typeName(T)), + }; + return self.emitFloat(dtype, v_f64); + } + const v: i128 = switch (@typeInfo(T)) { + .comptime_int, .int => value, + .comptime_float, .float => @intFromFloat(value), + else => @compileError("Builder.constant: unsupported scalar " ++ @typeName(T)), + }; + return self.emitInt(dtype, intBits(v, @intCast(intBitwidth(dtype)), @tagName(dtype))); + } + + /// `value` as the `i64` an integer constant of `bits` bits is built from. + /// It may be signed or an unsigned bit pattern, so it must lie in + /// [-2^(bits-1), 2^bits). + fn intBits(value: i128, bits: u7, dtype: []const u8) i64 { + if (value < -(@as(i128, 1) << (bits - 1)) or value >= @as(i128, 1) << bits) + std.debug.panic("Builder.constant: {d} does not fit {s}", .{ value, dtype }); + return @truncate(value); + } + + /// `value` as a constant of `like`'s element type; a Value passes through. + pub fn constantLike(self: *Builder, value: anytype, like: Value) Value { + if (@TypeOf(value) == Value) return value; + return self.constMatching(value, like.elemType()); + } + + /// Build an arith.constant of the given MLIR element type from a Zig + /// numeric. Used for matching the type of an existing reference value. + fn constMatching(self: *Builder, value: anytype, elem: *const mlir.Type) Value { + if (elem.eql(.index(self.ctx))) return self.constIndex(intBits(value, 64, "index")); + return self.constant(self.mlirElemToDType(elem), value); + } + + fn liftMatching(self: *Builder, value: anytype, ref_elem: *const mlir.Type) Value { + const T = @TypeOf(value); + if (T == Value) return value; + return switch (@typeInfo(T)) { + .comptime_int, .comptime_float => self.constMatching(value, ref_elem), + else => self.lift(value), + }; + } + + /// Splat a scalar to a vector. Zig scalars take the dense-splat fast path + /// (`arith.constant dense`); a `Value` falls back to `vector.broadcast`. + pub fn splat(self: *Builder, value: anytype, shape: []const i64, dtype: DType) Value { + const T = @TypeOf(value); + const ty = mlir.Type.vector(shape, dtype.toMlir(self.ctx)); + if (T == Value) return self.emit(vector.broadcast(self.ctx, value.inner, ty, self.loc())); + if (isFloatDtype(dtype)) { + const v_f64: f64 = switch (@typeInfo(T)) { + .comptime_int, .int => @floatFromInt(value), + .comptime_float, .float => @floatCast(value), + else => @compileError("Builder.splat: unsupported scalar " ++ @typeName(T)), + }; + return self.emit(arith.constant_dense_splat_f(self.ctx, v_f64, ty, self.loc())); + } + const v_i64: i64 = switch (@typeInfo(T)) { + .comptime_int => @intCast(value), + .int => |info| if (info.signedness == .signed) @intCast(value) else @bitCast(@as(u64, @intCast(value))), + .comptime_float, .float => @intFromFloat(value), + else => @compileError("Builder.splat: unsupported scalar " ++ @typeName(T)), + }; + return self.emit(arith.constant_dense_splat_i(self.ctx, v_i64, ty, self.loc())); + } + + /// Zero-valued vector of `shape x dtype`. + pub fn zeros(self: *Builder, shape: []const i64, dtype: DType) Value { + return self.splat(@as(i64, 0), shape, dtype); + } + + pub fn ones(self: *Builder, shape: []const i64, dtype: DType) Value { + return self.splat(@as(i64, 1), shape, dtype); + } + + pub fn full(self: *Builder, shape: []const i64, value: anytype, dtype: DType) Value { + return self.splat(value, shape, dtype); + } + + /// Lift scalars and broadcast scalar → vector to match the other operand. + pub fn coerce(self: *Builder, a: anytype, b: anytype) struct { Value, Value } { + const a_ref: ?*const mlir.Type = if (@TypeOf(b) == Value) b.elemType() else null; + const b_ref: ?*const mlir.Type = if (@TypeOf(a) == Value) a.elemType() else null; + var av = if (a_ref) |t| self.liftMatching(a, t) else self.lift(a); + var bv = if (b_ref) |t| self.liftMatching(b, t) else self.lift(b); + + // integer-width promotion (int family only). + if (av.isIntElem() and bv.isIntElem()) { + const a_dt = self.mlirElemToDType(av.elemType()); + const b_dt = self.mlirElemToDType(bv.elemType()); + const aw = intBitwidth(a_dt); + const bw = intBitwidth(b_dt); + if (aw < bw) av = av.to(b_dt); + if (bw < aw) bv = bv.to(a_dt); + } + + // scalar↔vector splat. + if (av.isVector() and !bv.isVector()) { + const dt = self.mlirElemToDType(av.elemType()); + bv = self.splat(bv, av.shape().constSlice(), dt); + } + if (!av.isVector() and bv.isVector()) { + const dt = self.mlirElemToDType(bv.elemType()); + av = self.splat(av, bv.shape().constSlice(), dt); + } + + return .{ av, bv }; + } + + // ==================== arith: binary ==================== + + pub fn addi(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.addi(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn subi(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.subi(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn muli(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.muli(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn addf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.addf(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn subf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.subf(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn mulf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.mulf(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn divf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.divf(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn divsi(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.divsi(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn divui(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.divui(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + /// Mirrors `jax.numpy.floor_divide` (`jax/_src/numpy/ufuncs.py:2487`): + /// truncating `divsi` plus the sign-correcting expansion + /// `q - 1 if (sign(a) != sign(b) and rem != 0) else q`. + /// `lhs` must be a vector of i32. `rhs` is `anytype` — passing a + /// scalar comptime int produces the dense<...> splat Pallas emits. + /// JAX's MLIR canonicalizer does not fold the expansion away even + /// when one operand is a constant, so this helper is needed for + /// IR-equivalent floor-div with `pallas_call`. + pub fn divFloor(self: *Builder, lhs: Value, rhs: anytype) Value { + const shape = lhs.shape().constSlice(); + const rhs_v: Value = if (@TypeOf(rhs) == Value) rhs else self.splat(rhs, shape, .i32); + const zero_v = self.zeros(shape, .i32); + const one_v = self.ones(shape, .i32); + const q = self.divsi(lhs, rhs_v); + const sign_a = self.subi( + self.extui(self.cmpi(.sgt, lhs, zero_v), .i32), + self.extui(self.cmpi(.slt, lhs, zero_v), .i32), + ); + const sign_b = self.subi( + self.extui(self.cmpi(.sgt, rhs_v, zero_v), .i32), + self.extui(self.cmpi(.slt, rhs_v, zero_v), .i32), + ); + const sign_diff = self.cmpi(.ne, sign_a, sign_b); + const rem_nz = self.cmpi(.ne, self.remsi(lhs, rhs_v), zero_v); + const cond = self.andi(sign_diff, rem_nz); + return self.select(cond, self.subi(q, one_v), q); + } + + pub fn remsi(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.remsi(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn remui(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.remui(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn remf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.remf(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn andi(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.andi(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn ori(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.ori(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn xori(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.xori(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn shli(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.shli(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn shrsi(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.shrsi(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn shrui(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.shrui(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn maxsi(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.maxsi(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn minsi(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.minsi(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn maximumf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.maximumf(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn minimumf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.minimumf(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn maxnumf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.maxnumf(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn minnumf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.minnumf(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn negf(self: *Builder, src: Value) Value { + return self.emit(arith.negf(self.ctx, src.inner, self.loc())); + } + pub fn cmpi(self: *Builder, predicate: arith.CmpIPredicate, lhs: Value, rhs: Value) Value { + return self.emit(arith.cmpi(self.ctx, predicate, lhs.inner, rhs.inner, self.loc())); + } + pub fn cmpf(self: *Builder, predicate: arith.CmpFPredicate, lhs: Value, rhs: Value) Value { + return self.emit(arith.cmpf(self.ctx, predicate, lhs.inner, rhs.inner, self.loc())); + } + pub fn select(self: *Builder, cond: Value, t: Value, f: Value) Value { + return self.emit(arith.select(self.ctx, cond.inner, t.inner, f.inner, self.loc())); + } + pub fn where(self: *Builder, condition: Value, x: Value, y: Value) Value { + const xv, const yv = self.coerce(x, y); + const c = if (condition.isVector() == xv.isVector()) condition else blk: { + // Splat a scalar predicate to match a vector x/y. + if (xv.isVector()) { + break :blk self.splat(condition, xv.shape().constSlice(), .i1); + } + break :blk condition; + }; + return self.select(c, xv, yv); + } + + /// Elementwise max with auto-broadcast scalar→vector. Use `maxOpts` + /// for NaN-propagation control. + pub fn maximum(self: *Builder, a: anytype, b: anytype) Value { + return self.maximumOpts(a, b, .{}); + } + pub fn maximumOpts(self: *Builder, a: anytype, b: anytype, opts: MinMaxOpts) Value { + const l, const r = self.coerce(a, b); + if (l.isFloatElem()) { + return if (opts.propagate_nan) self.maximumf(l, r) else self.maxnumf(l, r); + } + return self.maxsi(l, r); + } + pub fn minimum(self: *Builder, a: anytype, b: anytype) Value { + return self.minimumOpts(a, b, .{}); + } + pub fn minimumOpts(self: *Builder, a: anytype, b: anytype, opts: MinMaxOpts) Value { + const l, const r = self.coerce(a, b); + if (l.isFloatElem()) { + return if (opts.propagate_nan) self.minimumf(l, r) else self.minnumf(l, r); + } + return self.minsi(l, r); + } + + // ==================== arith casts ==================== + + pub fn extsi(self: *Builder, src: Value, dt: DType) Value { + return self.emit(arith.extsi(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); + } + pub fn extui(self: *Builder, src: Value, dt: DType) Value { + return self.emit(arith.extui(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); + } + pub fn arithExtf(self: *Builder, src: Value, dt: DType) Value { + return self.emit(arith.extf(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); + } + pub fn trunci(self: *Builder, src: Value, dt: DType) Value { + return self.emit(arith.trunci(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); + } + pub fn arithTruncf(self: *Builder, src: Value, dt: DType) Value { + return self.emit(arith.truncf(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); + } + pub fn arithSitofp(self: *Builder, src: Value, dt: DType) Value { + return self.emit(arith.sitofp(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); + } + pub fn arithUitofp(self: *Builder, src: Value, dt: DType) Value { + return self.emit(arith.uitofp(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); + } + pub fn arithFptosi(self: *Builder, src: Value, dt: DType) Value { + return self.emit(arith.fptosi(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); + } + pub fn arithFptoui(self: *Builder, src: Value, dt: DType) Value { + return self.emit(arith.fptoui(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); + } + pub fn arithBitcast(self: *Builder, src: Value, dt: DType) Value { + return self.emit(arith.bitcast(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); + } + + /// `arith.index_cast` — cast between integer and `index` types. + pub fn indexCast(self: *Builder, src: Value, target: *const mlir.Type) Value { + return self.emit(arith.index_cast(self.ctx, src.inner, target, self.loc())); + } + + /// Cast a scalar integer (i32 / i64 / …) to the `index` type. Common + /// pattern for using i32 program-ids and scalar-prefetch values as + /// memref offsets. + pub fn toIndex(self: *Builder, src: Value) Value { + return self.indexCast(src, .index(self.ctx)); + } + + // ==================== TPU casts ==================== + + /// `tpu.bitcast` — same-bitwidth element-type swap. Shape preserved. + pub fn bitCast(self: *Builder, src: Value, dt: DType) Value { + return self.emit(mosaic_tpu.bitcast(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); + } + pub fn extf(self: *Builder, src: Value, dt: DType) Value { + return self.emit(mosaic_tpu.extf(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); + } + pub fn truncf(self: *Builder, src: Value, dt: DType) Value { + return self.emit(mosaic_tpu.truncf(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); + } + pub fn sitofp(self: *Builder, src: Value, dt: DType) Value { + return self.emit(mosaic_tpu.sitofp(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); + } + pub fn uitofp(self: *Builder, src: Value, dt: DType) Value { + return self.emit(mosaic_tpu.uitofp(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); + } + pub fn fptosi(self: *Builder, src: Value, dt: DType) Value { + return self.emit(mosaic_tpu.fptosi(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); + } + pub fn fptoui(self: *Builder, src: Value, dt: DType) Value { + return self.emit(mosaic_tpu.fptoui(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); + } + + /// Numeric cast with auto-dispatch. Routes through `arith.*` ops for the + /// common float / int conversions (which don't carry a rounding-mode + /// attribute). The `tpu.*` cast variants (`tpuTruncf`, `tpuFptosi`, …) + /// are exposed separately for callers that need rounding-mode control — + /// pass `opts.rounding` to opt in. + pub fn cast(self: *Builder, src: Value, dt: DType) Value { + return self.castOpts(src, dt, .{}); + } + pub fn castOpts(self: *Builder, src: Value, dt: DType, opts: CastOpts) Value { + if (opts.bitcast) return self.bitCast(src, dt); + const cur_elem = src.elemType(); + const tgt_elem = dt.toMlir(self.ctx); + if (cur_elem.eql(tgt_elem)) return src; + const cur_dtype = self.mlirElemToDType(cur_elem); + const cur_is_float = src.isFloatElem(); + const tgt_is_float = isFloatDtype(dt); + if (cur_is_float and tgt_is_float) { + const cur_bw = dtypeBitwidth(cur_dtype); + const tgt_bw = dtypeBitwidth(dt); + if (tgt_bw > cur_bw) return self.arithExtf(src, dt); + return self.arithTruncf(src, dt); + } + if (cur_is_float and !tgt_is_float) return self.arithFptosi(src, dt); + if (!cur_is_float and tgt_is_float) { + if (cur_dtype == .i1) return self.arithUitofp(src, dt); + return self.arithSitofp(src, dt); + } + const cur_bw = dtypeBitwidth(cur_dtype); + const tgt_bw = dtypeBitwidth(dt); + if (tgt_bw > cur_bw) return self.extsi(src, dt); + if (tgt_bw < cur_bw) return self.trunci(src, dt); + return self.arithBitcast(src, dt); + } + + // ==================== math dialect ==================== + + pub fn exp(self: *Builder, x: Value) Value { + return self.emit(math.exp(self.ctx, x.inner, self.loc())); + } + /// `exp2(x)` → `exp(x * ln(2))`. Matches Pallas's TPU lowering rule + /// (`jax/_src/pallas/mosaic/lowering.py:_exp2_lowering_rule`), which + /// expands `lax.exp2_p` into the same `x * ln2` pattern in forward-compat + /// mode. Materializes as an `arith.constant` (matching x's element type) + /// + `arith.mulf x, ln2` + `math.exp`. For the raw `math.exp2` op + /// (post-2025-07-26 cloud TPUs), use `mathExp2`. + pub fn exp2(self: *Builder, x: Value) Value { + const ln2 = self.constant(self.mlirElemToDType(x.elemType()), @as(f64, 0.6931471805599453)); + const ln2_bc = if (x.isShaped()) self.broadcastTo(ln2, x.shape().constSlice()) else ln2; + return self.exp(self.mulf(x, ln2_bc)); + } + pub fn mathExp2(self: *Builder, x: Value) Value { + return self.emit(math.exp2(self.ctx, x.inner, self.loc())); + } + pub fn log(self: *Builder, x: Value) Value { + return self.emit(math.log(self.ctx, x.inner, self.loc())); + } + pub fn log1p(self: *Builder, x: Value) Value { + return self.emit(math.log1p(self.ctx, x.inner, self.loc())); + } + pub fn log2(self: *Builder, x: Value) Value { + return self.emit(math.log2(self.ctx, x.inner, self.loc())); + } + pub fn sqrt(self: *Builder, x: Value) Value { + return self.emit(math.sqrt(self.ctx, x.inner, self.loc())); + } + pub fn rsqrt(self: *Builder, x: Value) Value { + return self.emit(math.rsqrt(self.ctx, x.inner, self.loc())); + } + pub fn sin(self: *Builder, x: Value) Value { + return self.emit(math.sin(self.ctx, x.inner, self.loc())); + } + pub fn cos(self: *Builder, x: Value) Value { + return self.emit(math.cos(self.ctx, x.inner, self.loc())); + } + pub fn tanh(self: *Builder, x: Value) Value { + return self.emit(math.tanh(self.ctx, x.inner, self.loc())); + } + pub fn erf(self: *Builder, x: Value) Value { + return self.emit(math.erf(self.ctx, x.inner, self.loc())); + } + pub fn absf(self: *Builder, x: Value) Value { + return self.emit(math.absf(self.ctx, x.inner, self.loc())); + } + pub fn absi(self: *Builder, x: Value) Value { + return self.emit(math.absi(self.ctx, x.inner, self.loc())); + } + pub fn abs(self: *Builder, x: Value) Value { + if (x.isFloatElem()) return self.absf(x); + return self.absi(x); + } + pub fn floor(self: *Builder, x: Value) Value { + return self.emit(math.floor(self.ctx, x.inner, self.loc())); + } + pub fn ceil(self: *Builder, x: Value) Value { + return self.emit(math.ceil(self.ctx, x.inner, self.loc())); + } + pub fn powf(self: *Builder, x: Value, y: Value) Value { + return self.emit(math.powf(self.ctx, x.inner, y.inner, self.loc())); + } + pub fn fma(self: *Builder, a: Value, b: Value, c_: Value) Value { + return self.emit(math.fma(self.ctx, a.inner, b.inner, c_.inner, self.loc())); + } + + // ==================== TPU memory: vector_load / vector_store / scalar load/store ==================== + + /// Build the result vector type for a full-shape `tpu.vector_load` from + /// the input ref's shape + element type. + fn refVectorType(self: *Builder, ref: Value) *const mlir.Type { + _ = self; + const mr = ref.type_().isA(mlir.MemRefType) orelse @panic("Builder.refVectorType: not a memref"); + var s: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; + for (0..mr.rank()) |i| s.appendAssumeCapacity(mr.dimension(i)); + return .vector(s.constSlice(), mr.elementType()); + } + + /// Full-shape `vector.load`. Use `tpuVectorLoad` for sublane stride / masking. + pub fn refLoad(self: *Builder, ref: Value) Value { + const result_ty = self.refVectorType(ref); + const idx_inner = self.indicesOrZero(ref, &.{}); + return self.emit(vector.load(self.ctx, ref.inner, idx_inner, result_ty, .{}, self.loc())); + } + + /// `vector.load(ref, indices)` — explicit `index`-typed offsets. + pub fn vectorLoadAt(self: *Builder, ref: Value, indices: []const Value) Value { + const result_ty = self.refVectorType(ref); + const idx_inner = self.indicesOrZero(ref, indices); + return self.emit(vector.load(self.ctx, ref.inner, idx_inner, result_ty, .{}, self.loc())); + } + + /// `tpu.vector_load` — sublane stride + optional element-wise mask. + pub fn tpuVectorLoad(self: *Builder, ref: Value, indices: []const Value, opts: VectorLoadOpts) Value { + const ctx = self.ctx; + const result_ty = self.refVectorType(ref); + const idx_inner = self.indicesOrZero(ref, indices); + const mask_inner: ?*const mlir.Value = if (opts.mask) |m| m.inner else null; + return self.emit(mosaic_tpu.vector_load(ctx, ref.inner, idx_inner, result_ty, .{ .strides = opts.strides, .mask = mask_inner }, self.loc())); + } + + /// `tpu.vector_load` with an explicit result vector shape. + pub fn tpuVectorLoadShape(self: *Builder, ref: Value, indices: []const Value, shape: []const i64, opts: VectorLoadOpts) Value { + const ctx = self.ctx; + const result_ty = mlir.Type.vector(shape, ref.elemType()); + const idx_inner = self.indicesOrZero(ref, indices); + const mask_inner: ?*const mlir.Value = if (opts.mask) |m| m.inner else null; + return self.emit(mosaic_tpu.vector_load(ctx, ref.inner, idx_inner, result_ty, .{ .strides = opts.strides, .mask = mask_inner }, self.loc())); + } + + /// `tpu.vector_store(value, ref)` — full-shape vector store. + pub fn refStore(self: *Builder, ref: Value, value: Value) void { + self.refStoreOpts(ref, value, &.{}, .{}); + } + + pub fn vectorStoreAt(self: *Builder, ref: Value, value: Value, indices: []const Value) void { + self.refStoreOpts(ref, value, indices, .{}); + } + + pub fn refStoreOpts(self: *Builder, ref: Value, value: Value, indices: []const Value, opts: VectorStoreOpts) void { + const idx_inner = self.indicesOrZero(ref, indices); + const mask_inner: ?*const mlir.Value = if (opts.mask) |m| m.inner else null; + _ = mosaic_tpu.vector_store(self.ctx, value.inner, ref.inner, idx_inner, .{ .strides = opts.strides, .add = opts.add, .mask = mask_inner }, self.loc()).appendTo(self.currentBlock()); + } + + /// `memref.load` — scalar load. Indices must be `index`-typed. + pub fn scalarLoad(self: *Builder, ref: Value, indices: []const Value) Value { + const idx_inner = self.indicesOrZero(ref, indices); + return self.emit(memref.load(self.ctx, ref.inner, idx_inner, .{}, self.loc())); + } + + /// `memref.store` — scalar store. + pub fn scalarStore(self: *Builder, ref: Value, value: Value, indices: []const Value) void { + const idx_inner = self.indicesOrZero(ref, indices); + _ = memref.store(self.ctx, value.inner, ref.inner, idx_inner, .{}, self.loc()).appendTo(self.currentBlock()); + } + + /// Returns a `*const mlir.Value` slice of length `ref.rank()`, padding with + /// a cached `arith.constant 0 : index` when `indices` is empty. + fn indicesOrZero(self: *Builder, ref: Value, indices: []const Value) []const *const mlir.Value { + const r = ref.rank(); + if (indices.len == r) return self.innerSlice(indices); + std.debug.assert(indices.len == 0); + const zero = self.zeroIndex(); + const out = self.arena.allocator().alloc(*const mlir.Value, r) catch @panic("Builder.indicesOrZero OOM"); + for (out) |*p| p.* = zero.inner; + return out; + } + + /// Return a cached `arith.constant 0 : index` Value, lazily emitting + /// it the first time it's needed. The cache is invalidated whenever + /// the insertion block changes — `index` constants are scoped to the + /// block they're emitted in. + fn zeroIndex(self: *Builder) Value { + const block = self.currentBlock(); + if (self.zero_index) |z| { + if (self.zero_index_block == block) return z; + } + const z = self.constIndex(0); + self.zero_index = z; + self.zero_index_block = block; + return z; + } + + /// `tpu.strided_load` — multi-dim strided load. Result shape is derived + /// from `ref.shape / strides` (per-dim ceil divide). + pub fn stridedLoad(self: *Builder, ref: Value, indices: []const Value, strides: []const i32) Value { + const mr = ref.type_().isA(mlir.MemRefType) orelse @panic("stridedLoad: not a memref"); + var s: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; + for (0..mr.rank()) |i| { + const stride: i64 = if (i < strides.len) strides[i] else 1; + const dim = mr.dimension(i); + s.appendAssumeCapacity(@divTrunc(dim + stride - 1, stride)); + } + const result_ty = mlir.Type.vector(s.constSlice(), mr.elementType()); + return self.emit(mosaic_tpu.strided_load(self.ctx, ref.inner, self.innerSlice(indices), strides, result_ty, self.loc())); + } + + /// `tpu.strided_load` with an explicit result shape — when you know the + /// load shape doesn't match the simple `ref.shape / strides` form (e.g. + /// inner-dim crop). + pub fn stridedLoadShape(self: *Builder, ref: Value, indices: []const Value, strides: []const i32, result_shape: []const i64) Value { + const mr = ref.type_().isA(mlir.MemRefType) orelse @panic("stridedLoadShape: not a memref"); + const result_ty = mlir.Type.vector(result_shape, mr.elementType()); + return self.emit(mosaic_tpu.strided_load(self.ctx, ref.inner, self.innerSlice(indices), strides, result_ty, self.loc())); + } + + /// `tpu.strided_store` — multi-dim strided store. + pub fn stridedStore(self: *Builder, ref: Value, value: Value, indices: []const Value, strides: []const i32) void { + _ = mosaic_tpu.strided_store(self.ctx, value.inner, ref.inner, self.innerSlice(indices), strides, self.loc()).appendTo(self.currentBlock()); + } + + // ==================== TPU compute: matmul / iota / reciprocal ==================== + + /// `tpu.matmul(lhs, rhs, acc)`. Result type is taken from `acc`. Use + /// `matmulOpts` to set transposes / precision / dimension_numbers. + pub fn matmul(self: *Builder, lhs: Value, rhs: Value, acc: Value) Value { + return self.matmulOpts(lhs, rhs, acc, .{}); + } + + pub fn matmulOpts(self: *Builder, lhs: Value, rhs: Value, acc: Value, opts: MatmulOpts) Value { + return self.emit(mosaic_tpu.matmul( + self.ctx, + lhs.inner, + rhs.inner, + acc.inner, + acc.type_(), + .{ + .transpose_lhs = opts.transpose_lhs, + .transpose_rhs = opts.transpose_rhs, + .transpose_lhs_hint = opts.transpose_lhs_hint, + .precision = opts.precision, + .dimension_numbers = opts.dimension_numbers, + }, + self.loc(), + )); + } + + /// `tpu.iota` over the given shape / dimensions. Pass `null` for + /// `dimensions` to use the default. + pub fn iota(self: *Builder, shape: []const i64, dtype: DType, dimensions: ?[]const i32) Value { + const ty = mlir.Type.vector(shape, dtype.toMlir(self.ctx)); + return self.emit(mosaic_tpu.iota(self.ctx, ty, .{ .dimensions = dimensions }, self.loc())); + } + + /// 1-D `arange(0, n)`. TPU layout pass requires rank ≥ 2 with the lane dim + /// trailing, so this emits a `<1xN>` `tpu.iota` then `shape_cast` to ``. + pub fn arange(self: *Builder, n: i64, dtype: DType) Value { + const elem = dtype.toMlir(self.ctx); + const ty_2d = .vector(&.{ 1, n }, elem); + const iota_2d = self.emit(mosaic_tpu.iota(self.ctx, ty_2d, .{ .dimensions = &.{1} }, self.loc())); + return self.shapeCast(iota_2d, &.{n}); + } + + /// `tpu.reciprocal(x)` with default options. + pub fn reciprocal(self: *Builder, x: Value) Value { + return self.reciprocalOpts(x, .{}); + } + + pub fn reciprocalOpts(self: *Builder, x: Value, opts: ReciprocalOpts) Value { + return self.emit(mosaic_tpu.reciprocal(self.ctx, x.inner, .{ + .approx = opts.approx, + .full_range = opts.full_range, + }, self.loc())); + } + + // ==================== Vector dialect helpers ==================== + // + // The `tpu.*` shape ops (`reshape`, `transpose`, + // `broadcastInSublanes`) are Mosaic-specific layout-aware variants. + + /// `vector.broadcast` to `shape`. For Zig scalar literals prefer `splat` (dense-splat fast path). + pub fn broadcastTo(self: *Builder, src: Value, shape: []const i64) Value { + const ty = mlir.Type.vector(shape, src.elemType()); + return self.emit(vector.broadcast(self.ctx, src.inner, ty, self.loc())); + } + + /// `vector.shape_cast` — reshape preserving total lanes. `reshape` emits `tpu.reshape` instead. + pub fn shapeCast(self: *Builder, src: Value, new_shape: []const i64) Value { + const ty = mlir.Type.vector(new_shape, src.elemType()); + return self.emit(vector.shape_cast(self.ctx, src.inner, ty, self.loc())); + } + + /// `vector.extract` — extract a scalar (or lower-rank vector) at a + /// static position. Result shape = `src.shape[position.len..]`. + pub fn vectorExtract(self: *Builder, src: Value, position: []const i64) Value { + const sh = src.shape(); + var out: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; + for (position.len..sh.len) |i| out.appendAssumeCapacity(sh.get(i)); + const ty: *const mlir.Type = if (out.len == 0) + src.elemType() + else + .vector(out.constSlice(), src.elemType()); + return self.emit(vector.extract(self.ctx, src.inner, position, &.{}, ty, self.loc())); + } + + /// `vector.extract_strided_slice` — statically slice a vector. + pub fn vectorExtractStridedSlice(self: *Builder, src: Value, offsets: []const i64, sizes: []const i64, strides: []const i64) Value { + const ty = mlir.Type.vector(sizes, src.elemType()); + return self.emit(vector.extract_strided_slice(self.ctx, src.inner, .{ + .offsets = offsets, + .sizes = sizes, + .strides = strides, + }, ty, self.loc())); + } + + /// `vector.reduction ` — reduce a 1-D vector to a scalar. + /// Emits the bare upstream op; on real TPU the layout pass may reject + /// 1-D vectors, so prefer `reduceToScalar` for kernel-level reductions. + pub fn vectorReductionFlat(self: *Builder, kind: CombiningKind, src: Value) Value { + return self.emit(vector.reduction(self.ctx, kind, src.inner, src.elemType(), .{}, self.loc())); + } + + /// 1-D vector → scalar. TPU vectors are 2-D, so this adds a leading + /// singleton before reducing: shape_cast→<1xN>, multi_reduction[1]→<1>, extract[0]. + /// `acc` is the reduction identity (e.g. `k.zeros(&.{1}, .f32)` for sum). + pub fn reduceToScalar(self: *Builder, kind: CombiningKind, src: Value, acc: Value) Value { + std.debug.assert(src.rank() == 1); + const n = src.shape().get(0); + const src_2d = self.shapeCast(src, &.{ 1, n }); + const reduced = self.multiReduction(kind, src_2d, acc, &.{1}); + return self.vectorExtract(reduced, &.{0}); + } + + /// `vector.multi_reduction ` — reduce a multi-D vector along + /// the given axes with an explicit accumulator. Result rank = + /// `src.rank - axes.len`. + pub fn multiReduction( + self: *Builder, + kind: CombiningKind, + src: Value, + acc: Value, + axes: []const i64, + ) Value { + const sh = src.shape(); + var out: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; + outer: for (0..sh.len) |i| { + for (axes) |a| if (a == @as(i64, @intCast(i))) continue :outer; + out.appendAssumeCapacity(sh.get(i)); + } + const ty: *const mlir.Type = if (out.len == 0) + src.elemType() + else + .vector(out.constSlice(), src.elemType()); + return self.emit(vector.multi_reduction(self.ctx, kind, src.inner, acc.inner, axes, ty, self.loc())); + } + + /// `vector.load` with an explicit result shape — for `ref[i, j, :, k:k+BLOCK]`-style sliced loads. + pub fn vectorLoadShape(self: *Builder, ref: Value, indices: []const Value, shape: []const i64) Value { + const idx_inner = self.indicesOrZero(ref, indices); + const ty = mlir.Type.vector(shape, ref.elemType()); + return self.emit(vector.load(self.ctx, ref.inner, idx_inner, ty, .{}, self.loc())); + } + + // ==================== TPU shape ==================== + + pub fn reshape(self: *Builder, src: Value, new_shape: []const i64) Value { + const ty = mlir.Type.vector(new_shape, src.elemType()); + return self.emit(mosaic_tpu.reshape(self.ctx, src.inner, ty, self.loc())); + } + + pub fn repeat(self: *Builder, src: Value, dimension: i32, times: i32) Value { + const in_shape = src.shape(); + var out: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; + for (0..in_shape.len) |i| { + const d = in_shape.get(i); + out.appendAssumeCapacity(if (i == @as(usize, @intCast(dimension))) d * times else d); + } + if (times == 1) return src; + const ty = mlir.Type.vector(out.constSlice(), src.elemType()); + const sources = self.arena.allocator().alloc(*const mlir.Value, @intCast(times)) catch @panic("OOM"); + @memset(sources, src.inner); + return self.emit(mosaic_tpu.concatenate(self.ctx, sources, dimension, ty, self.loc())); + } + + pub fn concatenate(self: *Builder, sources: []const Value, dimension: i32) Value { + std.debug.assert(sources.len > 0); + const ax: usize = @intCast(dimension); + var sum: i64 = 0; + for (sources) |s| sum += s.dim(ax); + const head = sources[0].shape(); + var out: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; + for (0..head.len) |i| out.appendAssumeCapacity(if (i == ax) sum else head.get(i)); + const ty = mlir.Type.vector(out.constSlice(), sources[0].elemType()); + return self.emit(mosaic_tpu.concatenate(self.ctx, self.innerSlice(sources), dimension, ty, self.loc())); + } + + pub fn transpose(self: *Builder, src: Value, permutation: []const i64) Value { + const in_shape = src.shape(); + std.debug.assert(permutation.len == in_shape.len); + var out: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; + for (permutation) |p| out.appendAssumeCapacity(in_shape.get(@intCast(p))); + const ty = mlir.Type.vector(out.constSlice(), src.elemType()); + return self.emit(mosaic_tpu.transpose(self.ctx, src.inner, permutation, ty, self.loc())); + } + + pub fn broadcastInSublanes(self: *Builder, src: Value, lane: i32) Value { + // Result shape = src shape (the op replicates within sublanes). + return self.emit(mosaic_tpu.broadcast_in_sublanes(self.ctx, src.inner, lane, src.type_(), self.loc())); + } + + pub fn rotate(self: *Builder, value: Value, opts: RotateOpts) Value { + const amount = self.constant(.i32, opts.amount); + return self.emit(mosaic_tpu.dynamic_rotate(self.ctx, value.inner, amount.inner, .{ + .dimension = opts.dimension, + .stride = opts.stride, + .stride_dimension = opts.stride_dimension, + }, self.loc())); + } + + // ==================== TPU reductions / scan / sort ==================== + + /// `tpu.all_reduce(input, dim, kind)` over a single dim. + pub fn allReduce(self: *Builder, input: Value, kind: ReductionKind, dim: i64) Value { + const in_shape = input.shape(); + var out: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; + for (0..in_shape.len) |i| out.appendAssumeCapacity(if (i == @as(usize, @intCast(dim))) 1 else in_shape.get(i)); + const ty = mlir.Type.vector(out.constSlice(), input.elemType()); + return self.emit(mosaic_tpu.all_reduce(self.ctx, input.inner, dim, kind, ty, self.loc())); + } + + /// `tpu.reduce_index(input, kind, axis)` — drop axis, replace with + /// `index`-valued indices (integer). + pub fn reduceIndex(self: *Builder, input: Value, kind: ReductionKind, axis: i32) Value { + const in_shape = input.shape(); + var out: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; + for (in_shape.constSlice(), 0..) |d, i| { + if (@as(i32, @intCast(i)) != axis) out.appendAssumeCapacity(d); + } + const ty = mlir.Type.vector(out.constSlice(), .int(self.ctx, .i32)); + return self.emit(mosaic_tpu.reduce_index(self.ctx, input.inner, axis, kind, ty, self.loc())); + } + + /// Scan the last dimension; a mask must be rank one with that dimension + /// as its length. The result has the same shape as the input. + pub fn scan(self: *Builder, input: Value, kind: ReductionKind, mask: ?Value) Value { + const m_inner: ?*const mlir.Value = if (mask) |m| m.inner else null; + return self.emit(mosaic_tpu.scan(self.ctx, input.inner, kind, input.type_(), .{ .mask = m_inner }, self.loc())); + } + + /// `tpu.sort(keys, values, mask?)` — returns (output_mask, sorted_keys, sorted_values). + pub fn sort(self: *Builder, keys: Value, values: Value, mask: ?Value, opts: SortOpts) [3]Value { + const m_inner: ?*const mlir.Value = if (mask) |m| m.inner else null; + const out_mask_ty = if (mask) |m| m.type_() else blk: { + // Default: vector. + break :blk .vector(keys.shape().constSlice(), .integer(self.ctx, .i1)); + }; + const op = mosaic_tpu.sort( + self.ctx, + keys.inner, + values.inner, + out_mask_ty, + keys.type_(), + values.type_(), + .{ .descending = opts.descending, .mask = m_inner }, + self.loc(), + ); + _ = op.appendTo(self.currentBlock()); + return .{ + .{ .inner = op.result(0), .kernel = self }, + .{ .inner = op.result(1), .kernel = self }, + .{ .inner = op.result(2), .kernel = self }, + }; + } + + // ==================== TPU memref-shape ops ==================== + + pub fn memRefSlice( + self: *Builder, + mem_ref: Value, + base_idx: []const Value, + result_shape: []const i64, + dynamic_sizes: []const Value, + ) Value { + const mr = mem_ref.type_().isA(mlir.MemRefType) orelse @panic("memRefSlice: not a memref"); + const result_ty = mlir.Type.memRef(mr.elementType(), result_shape, null, mr.memorySpace()); + return self.emit(mosaic_tpu.memref_slice(self.ctx, mem_ref.inner, .{ + .base_idx = self.innerSlice(base_idx), + .dynamic_sizes = self.innerSlice(dynamic_sizes), + }, result_ty, self.loc())); + } + + pub fn memRefSqueeze(self: *Builder, mem_ref: Value, result_shape: []const i64) Value { + const mr = mem_ref.type_().isA(mlir.MemRefType) orelse @panic("memRefSqueeze: not a memref"); + const result_ty = mlir.Type.memRef(mr.elementType(), result_shape, null, mr.memorySpace()); + return self.emit(mosaic_tpu.memref_squeeze(self.ctx, mem_ref.inner, result_ty, self.loc())); + } + + pub fn memRefReshape(self: *Builder, mem_ref: Value, result_shape: []const i64) Value { + const mr = mem_ref.type_().isA(mlir.MemRefType) orelse @panic("memRefReshape: not a memref"); + const result_ty = mlir.Type.memRef(mr.elementType(), result_shape, null, mr.memorySpace()); + return self.emit(mosaic_tpu.memref_reshape(self.ctx, mem_ref.inner, result_ty, self.loc())); + } + + pub fn memRefAlloca(self: *Builder, shape: []const i64, dtype: DType, memory_space: MemorySpace) Value { + return self.emit(memref.alloca(self.ctx, self.memRefTy(shape, dtype, memory_space), &.{}, &.{}, .{}, self.loc())); + } + + pub fn memRefBitcast(self: *Builder, mem_ref: Value, dt: DType) Value { + const mr = mem_ref.type_().isA(mlir.MemRefType) orelse @panic("memRefBitcast: not a memref"); + var s: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; + for (0..mr.rank()) |i| s.appendAssumeCapacity(mr.dimension(i)); + const result_ty = .memRef(dt.toMlir(self.ctx), s.constSlice(), null, mr.memorySpace()); + return self.emit(mosaic_tpu.memref_bitcast(self.ctx, mem_ref.inner, result_ty, self.loc())); + } + + /// `tpu.memref_bitcast` with an explicit result shape — needed for the + /// `bf16 → i32` packing trick in ragged-paged-attention's + /// `strided_load_kv` path (`<64x128xbf16>` → `<32x128xi32>`). + pub fn memRefBitcastShape(self: *Builder, mem_ref: Value, result_shape: []const i64, dt: DType) Value { + const mr = mem_ref.type_().isA(mlir.MemRefType) orelse @panic("memRefBitcastShape: not a memref"); + const result_ty = mlir.Type.memRef(dt.toMlir(self.ctx), result_shape, null, mr.memorySpace()); + return self.emit(mosaic_tpu.memref_bitcast(self.ctx, mem_ref.inner, result_ty, self.loc())); + } + + pub fn reinterpretCast(self: *Builder, mem_ref: Value, result_shape: []const i64, dt: DType) Value { + const mr = mem_ref.type_().isA(mlir.MemRefType) orelse @panic("reinterpretCast: not a memref"); + const result_ty = mlir.Type.memRef(dt.toMlir(self.ctx), result_shape, null, mr.memorySpace()); + return self.emit(mosaic_tpu.reinterpret_cast(self.ctx, mem_ref.inner, result_ty, self.loc())); + } + + pub fn assumeLayout(self: *Builder, src: Value) Value { + return self.emit(mosaic_tpu.assume_layout(self.ctx, src.inner, src.type_(), self.loc())); + } + + pub fn eraseMemRefLayout(self: *Builder, mem_ref: Value) Value { + const mr = mem_ref.type_().isA(mlir.MemRefType) orelse @panic("eraseMemRefLayout: not a memref"); + var s: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; + for (0..mr.rank()) |i| s.appendAssumeCapacity(mr.dimension(i)); + // Result drops the layout but keeps shape, element type, memory space. + const result_ty = .memRef(mr.elementType(), s.constSlice(), null, mr.memorySpace()); + return self.emit(mosaic_tpu.erase_memref_layout(self.ctx, mem_ref.inner, result_ty, self.loc())); + } + + // ==================== Mask building ==================== + + /// `tpu.create_mask` — 2-sided mask `[low, high)` per dim. + pub fn createMask(self: *Builder, low_bounds: []const Value, high_bounds: []const Value, shape: []const i64) Value { + const ty = mlir.Type.vector(shape, .integer(self.ctx, .i1)); + return self.emit(mosaic_tpu.create_mask(self.ctx, self.innerSlice(low_bounds), self.innerSlice(high_bounds), ty, self.loc())); + } + + /// Convenience: 1-sided range mask `(0 <= idx < limit)` — emits two + /// `index` constants and a `tpu.create_mask`. + pub fn rangeMask(self: *Builder, limits: []const Value, shape: []const i64) Value { + const zero = self.constIndex(0); + const lows = self.arena.allocator().alloc(Value, limits.len) catch @panic("rangeMask OOM"); + for (lows) |*l| l.* = zero; + return self.createMask(lows, limits, shape); + } + + // ==================== Semaphores / DMA / barriers ==================== + + pub fn semAlloc(self: *Builder, kind: ArgSpec.SemKind) Value { + const elem = switch (kind) { + .regular => mosaic_tpu.semaphoreType(self.ctx), + .dma => mosaic_tpu.dmaSemaphoreType(self.ctx), + }; + const ty = mlir.Type.memRef(elem, &.{}, null, MemorySpace.semaphore_mem.attribute(self.ctx)); + return self.emit(mosaic_tpu.sem_alloc(self.ctx, ty, self.loc())); + } + + pub fn semBarrier(self: *Builder) Value { + const ty = mlir.Type.memRef(mosaic_tpu.semaphoreType(self.ctx), &.{}, null, MemorySpace.semaphore_mem.attribute(self.ctx)); + return self.emit(mosaic_tpu.sem_barrier(self.ctx, ty, self.loc())); + } + + pub fn semRead(self: *Builder, semaphore: Value) Value { + return self.emit(mosaic_tpu.sem_read(self.ctx, semaphore.inner, self.loc())); + } + + pub fn semWait(self: *Builder, semaphore: Value, amount: anytype) void { + const a = if (@TypeOf(amount) == Value) amount else self.lift(amount); + _ = mosaic_tpu.sem_wait(self.ctx, semaphore.inner, a.inner, self.loc()).appendTo(self.currentBlock()); + } + + pub fn semSignal(self: *Builder, semaphore: Value, amount: anytype) void { + self.semSignalOpts(semaphore, amount, .{}); + } + + pub fn semSignalOpts(self: *Builder, semaphore: Value, amount: anytype, opts: SemSignalOpts) void { + const a = if (@TypeOf(amount) == Value) amount else self.lift(amount); + const dev: ?*const mlir.Value = if (opts.device_id) |d| d.inner else null; + const core: ?*const mlir.Value = if (opts.core_id) |c_| c_.inner else null; + const subcore: ?*const mlir.Value = if (opts.subcore_id) |sc| sc.inner else null; + _ = mosaic_tpu.sem_signal(self.ctx, semaphore.inner, a.inner, .{ + .device_id = dev, + .core_id = core, + .subcore_id = subcore, + }, self.loc()).appendTo(self.currentBlock()); + } + + pub fn barrier(self: *Builder, barrier_id: Value) void { + _ = mosaic_tpu.barrier(self.ctx, barrier_id.inner, self.loc()).appendTo(self.currentBlock()); + } + + pub fn enqueueDma(self: *Builder, source: Value, target: Value, target_semaphore: Value, opts: EnqueueDmaOpts) void { + const src_sem: ?*const mlir.Value = if (opts.source_semaphore) |s| s.inner else null; + const dev: ?*const mlir.Value = if (opts.device_id) |d| d.inner else null; + const core: ?*const mlir.Value = if (opts.core_id) |c_| c_.inner else null; + const subcore: ?*const mlir.Value = if (opts.subcore_id) |sc| sc.inner else null; + _ = mosaic_tpu.enqueue_dma(self.ctx, source.inner, target.inner, target_semaphore.inner, .{ + .source_semaphore = src_sem, + .device_id = dev, + .core_id = core, + .subcore_id = subcore, + .priority = opts.priority, + .strict_ordering = opts.strict_ordering, + }, self.loc()).appendTo(self.currentBlock()); + } + + pub fn waitDma2(self: *Builder, semaphore: Value, src: Value, dst: Value, opts: WaitDma2Opts) void { + const dev: ?*const mlir.Value = if (opts.device_id) |d| d.inner else null; + const core: ?*const mlir.Value = if (opts.core_id) |c_| c_.inner else null; + _ = mosaic_tpu.wait_dma2(self.ctx, semaphore.inner, src.inner, dst.inner, .{ + .device_id = dev, + .core_id = core, + .strict_ordering = opts.strict_ordering, + }, self.loc()).appendTo(self.currentBlock()); + } + + pub fn deviceId(self: *Builder) Value { + return self.emit(mosaic_tpu.device_id(self.ctx, self.loc())); + } + + /// `tpu.trace_start` — pair with `traceStop`. Used by Pallas to mark + /// einsum boundaries (e.g. `"nd,md->nm"`). + pub fn traceStart(self: *Builder, level: i32, message: []const u8) void { + _ = mosaic_tpu.trace_start(self.ctx, level, message, self.loc()).appendTo(self.currentBlock()); + } + + pub fn traceStop(self: *Builder) void { + _ = mosaic_tpu.trace_stop(self.ctx, self.loc()).appendTo(self.currentBlock()); + } + + /// Build a `#tpu.dot_dimension_numbers<...>` attribute for `matmulOpts`. + pub fn dotDimensionNumbers( + self: *Builder, + lhs_contracting: []const i64, + rhs_contracting: []const i64, + lhs_non_contracting: []const i64, + rhs_non_contracting: []const i64, + output_dim_order: []const i64, + lhs_batch: []const i64, + rhs_batch: []const i64, + ) *const mlir.Attribute { + return mosaic_tpu.dotDimensionNumbersAttr( + self.ctx, + lhs_contracting, + rhs_contracting, + lhs_non_contracting, + rhs_non_contracting, + output_dim_order, + lhs_batch, + rhs_batch, + ); + } + + pub fn delay(self: *Builder, cycles: anytype) void { + const c_ = if (@TypeOf(cycles) == Value) cycles else self.lift(cycles); + _ = mosaic_tpu.delay(self.ctx, c_.inner, self.loc()).appendTo(self.currentBlock()); + } + + pub fn assumeMultiple(self: *Builder, src: Value, multiple: i32) Value { + return self.emit(mosaic_tpu.assume_multiple(self.ctx, src.inner, multiple, self.loc())); + } + + pub fn openRegion(self: *Builder) *mlir.Block { + const block = mlir.Block.init(&.{}, &.{}); + self.pushBlock(block); + return block; + } + + pub fn closeRegion(self: *Builder, block: *mlir.Block) void { + _ = mosaic_tpu.yield(self.ctx, &.{}, self.loc()).appendTo(block); + self.popBlock(); + _ = mosaic_tpu.region(self.ctx, block, &.{}, self.loc()).appendTo(self.currentBlock()); + } + + // ==================== SCF regions ==================== + + /// Open a scoped `scf.for`. Literal bounds take the type of a Value bound, + /// else they are `index`. `inits` is `.{v1, v2, ...}` or an array. + pub fn openFor( + self: *Builder, + lower: anytype, + upper: anytype, + step: anytype, + inits: anytype, + ) ForScope(common.arity(@TypeOf(inits), "openFor: inits")) { + return self.openForOpts(lower, upper, step, inits, .{}); + } + + /// `openFor` with `scf.for` options. + pub fn openForOpts( + self: *Builder, + lower: anytype, + upper: anytype, + step: anytype, + inits: anytype, + opts: common.ForOpts, + ) ForScope(common.arity(@TypeOf(inits), "openFor: inits")) { + return common.openFor(self, Value, common.forBounds(self, Value, lower, upper, step, indexBound), inits, opts); + } + + pub fn openIf(self: *Builder, cond: Value) IfOnlyScope { + return common.openIf(self, Value, cond); + } + + pub fn openIfElse(self: *Builder, cond: Value, result_types: anytype) IfScope(common.arity(@TypeOf(result_types), "openIfElse: result_types")) { + return common.openIfElse(self, Value, cond, result_types); + } + + pub fn openWhile( + self: *Builder, + inits: anytype, + after_types: anytype, + ) WhileScope(common.arity(@TypeOf(inits), "openWhile: inits"), common.arity(@TypeOf(after_types), "openWhile: after_types")) { + return common.openWhile(self, Value, inits, after_types); + } + + // ==================== finalization ==================== + + /// Append `func.return` with the given results, verify the module, and + /// serialize to a NUL-terminated MLIR string owned by the kernel's + /// allocator. Caller must `defer allocator.free(ir)`. + pub fn finish(self: *Builder, results: []const Value) FinishError![:0]const u8 { + return self.finishOpts(results, .{}); + } + + pub const FinishOpts = struct { + /// Run `canonicalize,cse` on the module before serializing — lets + /// the user code stay one-line-per-Pallas-line and lean on MLIR + /// to dedupe duplicated constants / identical subexpressions, the + /// same way Pallas does (its lowering doesn't dedupe either; the + /// IR you read in `debug=True` is post-canonicalize). + canonicalize: bool = false, + }; + + pub fn finishOpts(self: *Builder, results: []const Value, opts: FinishOpts) FinishError![:0]const u8 { + const current = self.currentBlock(); + _ = func.returns(self.ctx, self.innerSlice(results), self.loc()).appendTo(current); + + if (self.emit_transform_stubs) self.appendTransformStubs(); + + try common.verify(self.module, "dsl/mosaic_tpu"); + + if (opts.canonicalize) { + const pm = mlir.PassManager.init(self.ctx); + defer pm.deinit(); + // `canonicalize` folds `x * 1`, `x + 0`, etc.; `cse` then + // dedupes identical constants and subexpressions. Same + // pipeline `zml/Compiler.zig` runs on every compiled module. + const opm = pm.asOpPassManager(); + inline for (.{ "canonicalize", "cse", "canonicalize" }) |pass| { + try opm.addPipeline(pass); + } + try pm.runOnOp(self.module.operation()); + } + + var al: std.Io.Writer.Allocating = .init(self.allocator); + defer al.deinit(); + try al.writer.print("{f}", .{self.module.operation().fmt(.{ + .debug_info = false, + })}); + + return try self.allocator.dupeSentinel(u8, al.written(), 0); + } + + /// Emit `transform_N` stub funcs — one per non-skipped i/o ref-arg. + /// Each takes `(i32 ×grid_dim_count, prefetch_refs...)` and returns + /// `block_rank` i32 values per `RefArgSnapshot.transform_returns` + /// (default: all zeros, matching a trivial index_map). + fn appendTransformStubs(self: *Builder) void { + const ctx = self.ctx; + const unknown_loc: *const mlir.Location = .unknown(ctx); + const i32_ty: *const mlir.Type = .int(ctx, .i32); + + const arena_alloc = self.arena.allocator(); + const total_args = self.grid_dim_count + self.prefetch_args.len; + const arg_types = arena_alloc.alloc(*const mlir.Type, total_args) catch @panic("OOM"); + const arg_locs = arena_alloc.alloc(*const mlir.Location, total_args) catch @panic("OOM"); + for (0..self.grid_dim_count) |i| { + arg_types[i] = i32_ty; + arg_locs[i] = unknown_loc; + } + for (self.prefetch_args, 0..) |ty, i| { + arg_types[self.grid_dim_count + i] = ty; + arg_locs[self.grid_dim_count + i] = unknown_loc; + } + + for (self.ref_args, 0..) |ra, idx| { + if (ra.skipped) continue; + + const rank: usize = @intCast(ra.block_rank); + const result_types = arena_alloc.alloc(*const mlir.Type, rank) catch @panic("OOM"); + for (result_types) |*r| r.* = i32_ty; + + const block = mlir.Block.init(arg_types, arg_locs); + + const c0 = arith.constant_int(ctx, 0, i32_ty, unknown_loc); + _ = c0.appendTo(block); + + const ret_vals = arena_alloc.alloc(*const mlir.Value, rank) catch @panic("OOM"); + const returns = ra.transform_returns; + for (ret_vals, 0..) |*v, i| { + if (returns) |r_list| { + if (i < r_list.len) { + v.* = switch (r_list[i]) { + .zero => c0.result(0), + .program_id => |pid| block.argument(pid), + }; + continue; + } + } + v.* = c0.result(0); + } + _ = func.returns(ctx, ret_vals, unknown_loc).appendTo(block); + + const name = if (self.transform_name_prefix) |prefix| + std.fmt.allocPrint(arena_alloc, "{s}_transform_{d}", .{ prefix, idx }) catch @panic("OOM") + else blk: { + var name_buf: [32]u8 = undefined; + break :blk arena_alloc.dupe(u8, std.fmt.bufPrint(&name_buf, "transform_{d}", .{idx}) catch unreachable) catch @panic("OOM"); + }; + + var extra_attrs: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (self.sc_transform_indices) { + extra_attrs.appendAssumeCapacity(.named(ctx, "sc.is_transform_indices", .unit(ctx))); + } + + const stub_op = func.func(ctx, .{ + .name = name, + .args = arg_types, + .results = result_types, + .block = block, + .location = unknown_loc, + .verify = false, + .extra_attributes = extra_attrs.constSlice(), + }); + _ = stub_op.appendTo(self.module.body()); + } + } +}; + +/// Memory spaces that don't support windowing (empty window_params, no transform_N). +fn isSkippedWindowSpace(ms: MemorySpace) bool { + return switch (ms) { + .hbm, .any, .semaphore_mem => true, + else => false, + }; +} + +// ==================== scope-type aliases pinned to Mosaic's Builder ==================== + +pub fn ForScope(comptime N: usize) type { + return common.ForScope(Builder, Value, N); +} + +pub const IfOnlyScope = common.IfOnlyScope(Builder, Value); + +pub fn IfScope(comptime N: usize) type { + return common.IfScope(Builder, Value, N); +} + +pub fn WhileScope(comptime N: usize, comptime M: usize) type { + return common.WhileScope(Builder, Value, N, M); +} + +test { + std.testing.refAllDecls(@This()); +} + +test "static rotate uses the current dynamic rotate operation" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + registerDialects(registry); + func.registerExtensions(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + var b = try Builder.init(std.testing.allocator, ctx, "rotate", &.{}, &.{}); + defer b.deinit(); + const input = b.zeros(&.{ 8, 128 }, .f32); + const rotated = b.rotate(input, .{ .amount = 3, .dimension = 1 }); + try std.testing.expectEqualStrings("tpu.dynamic_rotate", rotated.inner.owner().name()); + const amount: Value = .{ .inner = rotated.inner.owner().operand(1), .kernel = &b }; + try std.testing.expectEqual(@as(?i64, 3), amount.asConstantInt()); + const ir = try b.finish(&.{}); + defer std.testing.allocator.free(ir); +} diff --git a/dsl/mosaic_tpu/dtype.zig b/dsl/mosaic_tpu/dtype.zig new file mode 100644 index 00000000..549c69da --- /dev/null +++ b/dsl/mosaic_tpu/dtype.zig @@ -0,0 +1,72 @@ +const std = @import("std"); +const mlir = @import("mlir"); + +pub const DType = enum { + i1, + i8, + i16, + i32, + i64, + f16, + bf16, + f32, + f64, + f8e4m3fn, + f8e5m2, + + pub fn toMlir(self: DType, ctx: *mlir.Context) *const mlir.Type { + return switch (self) { + .i1 => .int(ctx, .i1), + .i8 => .int(ctx, .i8), + .i16 => .int(ctx, .i16), + .i32 => .int(ctx, .i32), + .i64 => .int(ctx, .i64), + .f16 => .float(ctx, .f16), + .bf16 => .float(ctx, .bf16), + .f32 => .float(ctx, .f32), + .f64 => .float(ctx, .f64), + .f8e4m3fn => .float(ctx, .f8e4m3fn), + .f8e5m2 => .float(ctx, .f8e5m2), + }; + } +}; + +pub fn isFloatDtype(dt: DType) bool { + return switch (dt) { + .f16, .bf16, .f32, .f64, .f8e4m3fn, .f8e5m2 => true, + else => false, + }; +} + +pub fn dtypeBitwidth(dt: DType) usize { + return switch (dt) { + .i1 => 1, + .i8, .f8e4m3fn, .f8e5m2 => 8, + .i16, .f16, .bf16 => 16, + .i32, .f32 => 32, + .i64, .f64 => 64, + }; +} + +pub fn intBitwidth(dt: DType) u32 { + return switch (dt) { + .i1 => 1, + .i8 => 8, + .i16 => 16, + .i32 => 32, + .i64 => 64, + else => std.debug.panic("intBitwidth: not an integer DType: {s}", .{@tagName(dt)}), + }; +} + +pub fn mlirElemToDType(ctx: *mlir.Context, elem: *const mlir.Type) DType { + inline for (comptime std.meta.fieldNames(DType)) |field_name| { + const dt = @field(DType, field_name); + if (elem.eql(dt.toMlir(ctx))) return dt; + } + @panic("element type not a recognized Mosaic-TPU DType"); +} + +test { + std.testing.refAllDecls(@This()); +} diff --git a/dsl/mosaic_tpu/pipeline.zig b/dsl/mosaic_tpu/pipeline.zig new file mode 100644 index 00000000..01264373 --- /dev/null +++ b/dsl/mosaic_tpu/pipeline.zig @@ -0,0 +1,1147 @@ +//! Zig port of `pltpu.emit_pipeline` (jax/_src/pallas/mosaic/pipeline.py) — +//! emits lowered `tpu`-dialect IR byte-equivalent to `pallas_call`. +//! +//! `.blocked`/`.squeezed`/one `.bounded_slice` block dim; trivial-windowing +//! inputs peeled as sync copy; `trace_scopes`. Unsupported features panic. + +const std = @import("std"); +const func = @import("mlir/dialects/func"); +const memref = @import("mlir/dialects/memref"); +const mlir = @import("mlir"); +const scf = @import("mlir/dialects/scf"); +const builder = @import("builder.zig"); + +const mosaic_tpu = @import("mlir/dialects/mosaic_tpu"); + +const Builder = builder.Builder; +const Value = builder.Value; +const DType = builder.DType; +const MemorySpace = builder.MemorySpace; +const FinishError = builder.FinishError; + +/// `ShapedType::kDynamic` — sentinel for a `?` dim in a memref shape. +const kDyn: i64 = std.math.minInt(i64); + +/// One entry of `pl.BlockSpec.block_shape`. +pub const BlockDim = union(enum) { + /// `pl.Blocked(n)` — windowed dim of size `n`. + blocked: i64, + /// `pl.Squeezed()` / `None` — dim dropped in the window. + squeezed, + /// `pl.BoundedSlice(n)` — index_map returns `.slice`; `n` is the max. + bounded_slice: i64, +}; + +/// One entry returned by `pl.BlockSpec.index_map`. +pub const BlockIndex = union(enum) { + /// `pl.ds(idx*block, block)` — for `.blocked`/`.squeezed` dims. + scalar: Value, + /// `pl.ds(start, size)` — for `.bounded_slice` dims. + slice: struct { start: Value, size: Value }, +}; + +/// `pl.BlockSpec.index_map`. `ctx` is opaque user data (Zig has no closures). +pub const IndexMapFn = *const fn (b: *Builder, indices: []const Value, ctx: ?*anyopaque) []const BlockIndex; + +/// Working window for one pipeline operand. Body must re-emit +/// `memref_slice`/`squeeze` at each use site. `slot == null` ⇒ trivial-windowing; +/// `buf == null` ⇒ no operand. +pub const RefWindow = struct { buf: ?Value = null, slot: ?Value = null }; + +/// The pipeline body — `pallas_call`'s inner kernel. +pub const BodyFn = *const fn (b: *Builder, grid_indices: []const Value, refs: []const RefWindow, scratches: []const Value, ctx: ?*anyopaque) FinishError!void; + +// Optional overrides for a buffered reference's DMA operations. +// Unspecified operations retain the standard pipeline behavior. +pub const BufferedRefStrategy = struct { + ctx: ?*anyopaque = null, + copy_in: ?*const fn (br: *BufferedRef, b: *Builder, indices: []const Value, slot_cumulative: Value, static_zero_start: bool, ctx: ?*anyopaque) void = null, + wait_in: ?*const fn (br: *BufferedRef, b: *Builder, indices: []const Value, ctx: ?*anyopaque) void = null, + copy_out: ?*const fn (br: *BufferedRef, b: *Builder, indices: []const Value, ctx: ?*anyopaque) void = null, + wait_out: ?*const fn (br: *BufferedRef, b: *Builder, indices: []const Value, ctx: ?*anyopaque) void = null, +}; + +/// `pl.BlockSpec` for one pipeline operand. `null` slot ↔ `None` spec. +pub const PipeSpec = struct { + full_shape: []const i64, + dtype: DType, + /// Optional HBM source/result view to materialize at each DMA use site. + source_reshape_shape: ?[]const i64 = null, + /// `null` ⇒ trivial windowing (the whole array). + block_shape: ?[]const BlockDim = null, + /// `null` only allowed with trivial windowing. + index_map: ?IndexMapFn = null, + index_map_ctx: ?*anyopaque = null, + /// Only `.vmem` supported. + memory_space: MemorySpace = .vmem, + + buffer_count: ?usize = null, + /// Tile that `_create_bounded_slice` rounds the DMA size up to. + bounded_slice_tiling: i32 = 1, + /// Pallas's `disable_bounds_checks`: skip OOB clamp, emit only `tpu.assume_multiple`. + bounded_slice_skip_clamp: bool = false, + + /// Optional custom DMA behavior for this operand. + transfer_strategy: ?BufferedRefStrategy = null, + /// Override the role infered from whether this spec appears + /// in `in_specs` or `out_specs` + buffer_type: ?BufferType = null, +}; + +pub const PipelineOpts = struct { + body: BodyFn, + body_ctx: ?*anyopaque = null, + /// Dynamic (`i32`) grid bounds. Comptime: pass `b.constant(.i32, N)`. + grid: []const Value, + /// One per HBM input ref (slot `null` <-> `None` spec). + in_specs: []const ?PipeSpec, + out_specs: []const ?PipeSpec, + trace_scopes: bool = true, +}; + +/// Emits `pltpu.emit_pipeline(body, grid=..., in_specs=..., out_specs=...)(*refs, scratches=...)`. +/// `refs` = `in_refs ++ out_refs`. Emits into `b.currentBlock()`. +pub fn emitPipeline( + b: *Builder, + refs: []const ?Value, + scratches: []const Value, + opts: PipelineOpts, +) FinishError!void { + std.debug.assert(opts.grid.len >= 1); + if (opts.grid.len > 4) @panic("emitPipeline: grids with rank > 4 unsupported"); + const n_in = opts.in_specs.len; + std.debug.assert(refs.len == n_in + opts.out_specs.len); + + const a = b.arena.allocator(); + + var brefs: std.ArrayList(BufferedRef) = try .initCapacity(a, opts.in_specs.len + opts.out_specs.len); + for (opts.in_specs, 0..) |maybe_spec, i| { + if (maybe_spec) |spec| { + std.debug.assert(refs[i] != null); + brefs.appendAssumeCapacity(BufferedRef.create(b, spec, .input, refs[i].?, i)); + } + } + for (opts.out_specs, 0..) |maybe_spec, j| { + if (maybe_spec) |spec| { + std.debug.assert(refs[n_in + j] != null); + brefs.appendAssumeCapacity(BufferedRef.create(b, spec, .output, refs[n_in + j].?, n_in + j)); + } + } + + // Whole pipeline lives in a `tpu.region`. + const region_block = mlir.Block.init(&.{}, &.{}); + b.pushBlock(region_block); + + for (brefs.items) |*br| br.allocate(b); + + const num_steps = gridSize(b, opts.grid); + const c0_i32 = b.constant(.i32, 0); + const c1_i32 = b.constant(.i32, 1); + { + var when = b.openIf(b.cmpi(.sgt, num_steps, c0_i32)); + { + for (brefs.items) |*br| if (br.is_trivial and (br.buffer_type == .input or br.buffer_type == .input_output)) br.syncCopyIn(b); + + const num_stages = blk: { + var m: usize = 2; + for (brefs.items) |br| if (br.buffer_count > m) { + m = br.buffer_count; + }; + break :blk m; + }; + var sched0 = Scheduler.init(b, c0_i32, opts.grid, num_steps, num_stages, opts.trace_scopes); + for (0..num_stages - 1) |step| { + if (opts.trace_scopes) { + var buf: [32]u8 = undefined; + b.traceStart(10, std.fmt.bufPrint(&buf, "ep_initialize_{d}", .{step}) catch unreachable); + } + for (brefs.items) |*br| sched0.initializeStep(b, br, step); + if (opts.trace_scopes) b.traceStop(); + } + + // iter_args: each bref's slots in spec order (see appendSlotInits), then grid indices. + var inits: std.ArrayList(*const mlir.Value) = try .initCapacity(a, brefs.items.len * 4 + opts.grid.len); + for (brefs.items) |*br| br.appendSlotInits(b, &inits); + for (0..opts.grid.len) |_| inits.appendAssumeCapacity(c0_i32.inner); + const n_slots = inits.items.len - opts.grid.len; + + const i32_ty = b.scalarTy(.i32); + const block_types = a.alloc(*const mlir.Type, inits.items.len + 1) catch unreachable; + const block_locs = a.alloc(*const mlir.Location, inits.items.len + 1) catch unreachable; + for (block_types) |*t| t.* = i32_ty; + for (block_locs) |*l| l.* = b.loc(); + const for_body = mlir.Block.init(block_types, block_locs); + b.pushBlock(for_body); + + const iv: Value = .{ .inner = for_body.argument(0), .kernel = b }; + { + var cur: usize = 1; + for (brefs.items) |*br| cur = br.bindSlotsAt(b, for_body, cur); + std.debug.assert(cur == n_slots + 1); + } + var grid_indices_buf: [4]Value = undefined; + for (0..opts.grid.len) |i| { + grid_indices_buf[i] = .{ .inner = for_body.argument(@intCast(n_slots + 1 + i)), .kernel = b }; + } + + var sched = Scheduler.init(b, iv, opts.grid, num_steps, num_stages, opts.trace_scopes); + for (0..opts.grid.len) |i| sched.indices_storage[i] = grid_indices_buf[i]; + sched.recomputeDerivedIndices(b); + + for (brefs.items) |*br| { + if (br.buffer_type == .input or br.buffer_type == .input_output) + sched.copyIn(b, br); + } + for (brefs.items) |*br| { + if (br.buffer_type == .input or br.buffer_type == .input_output) + sched.waitIn(b, br); + } + // Emit slot `rem` ops BEFORE the body; body re-emits slice/squeeze per use. + var current_refs = a.alloc(RefWindow, refs.len) catch unreachable; + for (current_refs) |*r| r.* = .{}; + for (brefs.items) |*br| { + if (br.is_trivial or !br.isBuffered()) { + current_refs[br.spec_index] = .{ .buf = br.window_ref }; + } else { + const active = switch (br.buffer_type) { + .input => br.wait_in_slot.?, + .output, .input_output => br.copy_out_slot.?, + }; + current_refs[br.spec_index] = .{ .buf = br.window_ref, .slot = b.remui(active, b.constant(.i32, br.buffer_count)) }; + } + } + + if (opts.trace_scopes) b.traceStart(10, "ep_run_kernel"); + try opts.body(b, sched.indices_storage[0..opts.grid.len], current_refs, scratches, opts.body_ctx); + if (opts.trace_scopes) b.traceStop(); + + for (brefs.items) |*br| { + if (br.buffer_type == .output or br.buffer_type == .input_output) + sched.copyOut(b, br); + } + for (brefs.items) |*br| { + if (br.buffer_type == .output or br.buffer_type == .input_output) + sched.waitOut(b, br); + } + + for (brefs.items) |*br| { + if (br.buffer_type == .input or br.buffer_type == .input_output) + sched.advanceSlots(b, br); + } + var yields: std.ArrayList(*const mlir.Value) = try .initCapacity(a, brefs.items.len * 4 + opts.grid.len); + for (brefs.items) |*br| br.appendSlotYields(b, &yields); + const next_indices = nextIndexND(b, sched.indices(), opts.grid); + for (next_indices) |idx| yields.appendAssumeCapacity(idx.inner); + _ = scf.yield(b.ctx, yields.items, b.loc()).appendTo(for_body); + b.popBlock(); + + const for_op = scf.for_(b.ctx, c0_i32.inner, num_steps.inner, c1_i32.inner, inits.items, for_body, .{}, b.loc()); + _ = for_op.appendTo(b.currentBlock()); + var k: usize = 0; + for (brefs.items) |*br| k = br.readbackSlots(b, for_op, k); + var final_grid_indices_buf: [4]Value = undefined; + for (0..opts.grid.len) |i| { + final_grid_indices_buf[i] = .{ .inner = for_op.result(@intCast(n_slots + i)), .kernel = b }; + } + + // prev_index BEFORE Scheduler.init so common arithmetic CSEs with `step`. + const final_idx = prevIndexND(b, final_grid_indices_buf[0..opts.grid.len], opts.grid); + var sched_fin = Scheduler.init(b, b.subi(num_steps, c1_i32), opts.grid, num_steps, num_stages, opts.trace_scopes); + for (0..opts.grid.len) |i| sched_fin.indices_storage[i] = final_idx[i]; + sched_fin.recomputeDerivedIndices(b); + for (brefs.items) |*br| sched_fin.finalize(b, br); + for (brefs.items) |*br| if (br.is_trivial and (br.buffer_type == .output or br.buffer_type == .input_output)) br.syncCopyOut(b); + + when.close(); + } + } + + _ = mosaic_tpu.yield(b.ctx, &.{}, b.loc()).appendTo(region_block); + b.popBlock(); + _ = mosaic_tpu.region(b.ctx, region_block, &.{}, b.loc()).appendTo(b.currentBlock()); +} + +pub const BufferType = enum { input, output, input_output }; + +/// VMEM double-buffer + DMA semaphores for one pipeline operand. +pub const BufferedRef = struct { + spec: PipeSpec, + buffer_type: BufferType, + buffer_count: usize, + is_trivial: bool, + /// Index into `refs` / `current_refs`. + spec_index: usize, + src: Value, + /// Block shape with `.squeezed` dims removed. + block_compact: []const i64, + /// Indices into `spec.block_shape` parallel to `block_compact`. + block_keep: []const usize, + + /// `memref` (or full shape if trivial). + window_ref: Value = undefined, + sem_recvs: ?Value = null, + sem_sends: ?Value = null, + + // Slot iter_arg state — see appendSlotInits for init values. + copy_in_slot: ?Value = null, + wait_in_slot: ?Value = null, + copy_out_slot: ?Value = null, + wait_out_slot: ?Value = null, + slot: ?Value = null, + + fn create(b: *Builder, spec: PipeSpec, bt: BufferType, src: Value, spec_index: usize) BufferedRef { + const a = b.arena.allocator(); + const buffer_type = spec.buffer_type orelse bt; + if (spec.memory_space != .vmem) @panic("emitPipeline: only VMEM operands supported"); + const trivial = specHasTrivialWindowing(spec); + var bc: usize = if (spec.buffer_count) |c| c else 2; + if (spec.buffer_count == null and trivial) bc = 1; + if (buffer_type != .input and bc > 2) @panic("emitPipeline: output and input_output buffer_count > 2 unsupported"); + + const block_len = if (spec.block_shape) |bs| bs.len else 0; + var keep = std.ArrayList(usize).initCapacity(a, block_len) catch unreachable; + var compact = std.ArrayList(i64).initCapacity(a, block_len) catch unreachable; + if (spec.block_shape) |bs| { + for (bs, 0..) |bd, di| switch (bd) { + .blocked => |n| { + keep.appendAssumeCapacity(di); + compact.appendAssumeCapacity(n); + }, + .bounded_slice => |n| { + keep.appendAssumeCapacity(di); + compact.appendAssumeCapacity(n); + }, + .squeezed => {}, + }; + } + return .{ + .spec = spec, + .buffer_type = buffer_type, + .buffer_count = bc, + .is_trivial = trivial, + .spec_index = spec_index, + .src = src, + .block_compact = compact.items, + .block_keep = keep.items, + }; + } + + fn isBuffered(self: *const BufferedRef) bool { + return self.buffer_count > 0; + } + + fn allocate(self: *BufferedRef, b: *Builder) void { + const a = b.arena.allocator(); + const vmem = MemorySpace.vmem.attribute(b.ctx); + const elem = self.spec.dtype.toMlir(b.ctx); + if (self.is_trivial) { + const ty = mlir.Type.memRef(elem, self.spec.full_shape, null, vmem); + self.window_ref = memRefAlloca(b, ty); + } else { + if (self.block_compact.len == 1) @panic("emitPipeline: 1-D flat block buffers unsupported (TODO)"); + var shape = a.alloc(i64, self.block_compact.len + 1) catch unreachable; + shape[0] = @intCast(self.buffer_count); + @memcpy(shape[1..], self.block_compact); + const ty = mlir.Type.memRef(elem, shape, null, vmem); + self.window_ref = memRefAlloca(b, ty); + } + if (!self.is_trivial) { + const sem_arr_ty = dmaSemRefTy(b, self.buffer_count); + if (self.buffer_type == .input or self.buffer_type == .input_output) self.sem_recvs = semAllocOp(b, sem_arr_ty); + if (self.buffer_type == .output or self.buffer_type == .input_output) self.sem_sends = semAllocOp(b, sem_arr_ty); + } + } + + fn appendSlotInits(self: *BufferedRef, b: *Builder, out: *std.ArrayList(*const mlir.Value)) void { + const c0 = b.constant(.i32, 0).inner; + switch (self.buffer_type) { + .input => { + if (self.is_trivial) { + out.appendAssumeCapacity(c0); + } else { + out.appendAssumeCapacity((self.copy_in_slot orelse b.constant(.i32, 0)).inner); + out.appendAssumeCapacity(c0); + } + }, + .output => { + out.appendAssumeCapacity(c0); + out.appendAssumeCapacity(c0); + }, + .input_output => { + out.appendAssumeCapacity( + (self.copy_in_slot orelse b.constant(.i32, 0)).inner, + ); + out.appendAssumeCapacity(c0); + out.appendAssumeCapacity(c0); + out.appendAssumeCapacity(c0); + }, + } + } + + fn nSlots(self: *const BufferedRef) usize { + return switch (self.buffer_type) { + .input => if (self.is_trivial) 1 else 2, + .output => 2, + .input_output => 4, + }; + } + + fn bindSlotsAt(self: *BufferedRef, b: *Builder, body: *mlir.Block, start: usize) usize { + var k = start; + const arg = struct { + fn f(b_: *Builder, body_: *mlir.Block, i: *usize) Value { + const v: Value = .{ .inner = body_.argument(@intCast(i.*)), .kernel = b_ }; + i.* += 1; + return v; + } + }.f; + switch (self.buffer_type) { + .input => { + if (self.is_trivial) { + self.slot = arg(b, body, &k); + } else { + self.copy_in_slot = arg(b, body, &k); + self.wait_in_slot = arg(b, body, &k); + } + }, + .output => { + self.copy_out_slot = arg(b, body, &k); + self.wait_out_slot = arg(b, body, &k); + }, + .input_output => { + self.copy_in_slot = arg(b, body, &k); + self.wait_in_slot = arg(b, body, &k); + self.copy_out_slot = arg(b, body, &k); + self.wait_out_slot = arg(b, body, &k); + }, + } + return k; + } + + fn appendSlotYields(self: *BufferedRef, b: *Builder, out: *std.ArrayList(*const mlir.Value)) void { + _ = b; + switch (self.buffer_type) { + .input => { + if (self.is_trivial) { + out.appendAssumeCapacity(self.slot.?.inner); + } else { + out.appendAssumeCapacity(self.copy_in_slot.?.inner); + out.appendAssumeCapacity(self.wait_in_slot.?.inner); + } + }, + .output => { + out.appendAssumeCapacity(self.copy_out_slot.?.inner); + out.appendAssumeCapacity(self.wait_out_slot.?.inner); + }, + .input_output => { + out.appendAssumeCapacity(self.copy_in_slot.?.inner); + out.appendAssumeCapacity(self.wait_in_slot.?.inner); + out.appendAssumeCapacity(self.copy_out_slot.?.inner); + out.appendAssumeCapacity(self.wait_out_slot.?.inner); + }, + } + } + + fn readbackSlots(self: *BufferedRef, b: *Builder, for_op: *mlir.Operation, start: usize) usize { + var k = start; + const take = struct { + fn f(b_: *Builder, op: *mlir.Operation, i: *usize) Value { + const v: Value = .{ .inner = op.result(@intCast(i.*)), .kernel = b_ }; + i.* += 1; + return v; + } + }.f; + switch (self.buffer_type) { + .input => { + if (self.is_trivial) { + self.slot = take(b, for_op, &k); + } else { + self.copy_in_slot = take(b, for_op, &k); + self.wait_in_slot = take(b, for_op, &k); + } + }, + .output => { + self.copy_out_slot = take(b, for_op, &k); + self.wait_out_slot = take(b, for_op, &k); + }, + .input_output => { + self.copy_in_slot = take(b, for_op, &k); + self.wait_in_slot = take(b, for_op, &k); + self.copy_out_slot = take(b, for_op, &k); + self.wait_out_slot = take(b, for_op, &k); + }, + } + return k; + } + + fn computeIndex(self: *const BufferedRef, b: *Builder, indices: []const Value) []const BlockIndex { + if (self.spec.index_map) |im| return im(b, indices, self.spec.index_map_ctx); + @panic("emitPipeline: index_map required for non-trivial windowing"); + } + + /// Per `block_shape` dim: `{base, size?}`. `size == null` ⇒ static-full. + /// `static_zero_start` enables prologue shortcut (Python-int 0 start folds). + const DmaSlice = struct { base: []Value, size: []?Value }; + fn getDmaSlice(self: *const BufferedRef, b: *Builder, indices: []const Value, static_zero_start: bool) DmaSlice { + const a = b.arena.allocator(); + const bs = self.spec.block_shape.?; + const bidx = self.computeIndex(b, indices); + std.debug.assert(bidx.len == bs.len); + var base = a.alloc(Value, bs.len) catch unreachable; + var size = a.alloc(?Value, bs.len) catch unreachable; + for (bs, 0..) |bd, d| { + switch (bd) { + .squeezed => { + base[d] = switch (bidx[d]) { + .scalar => |v| v, + .slice => @panic("emitPipeline: squeezed dim must get a scalar index"), + }; + size[d] = null; + }, + .blocked => |n| { + const idx = switch (bidx[d]) { + .scalar => |v| v, + .slice => @panic("emitPipeline: blocked dim must get a scalar index"), + }; + base[d] = if (n == 1) idx else idx.mul(@as(i32, @intCast(n))); + const dim_size = self.spec.full_shape[d]; + if (@mod(dim_size, n) == 0) { + size[d] = null; + } else { + const tiling: i64 = if (d == bs.len - 1) 128 else if (bs.len >= 2 and d == bs.len - 2) 8 else 1; + const clamped = @min(n, dim_size); + const sz_const = @divFloor(clamped + tiling - 1, tiling) * tiling; + size[d] = b.emit(mosaic_tpu.assume_multiple(b.ctx, b.constant(.i32, sz_const).inner, @as(i32, @intCast(tiling)), b.loc())); + } + }, + .bounded_slice => { + const sl = switch (bidx[d]) { + .scalar => @panic("emitPipeline: bounded_slice dim must get a `.slice` index from index_map"), + .slice => |s| s, + }; + const tiling: i32 = self.spec.bounded_slice_tiling; + const dim_size = self.spec.full_shape[d]; + base[d] = sl.start; + if (self.spec.bounded_slice_skip_clamp) { + size[d] = b.emit(mosaic_tpu.assume_multiple(b.ctx, sl.size.inner, tiling, b.loc())); + continue; + } + // Prologue shortcut: Python-int 0 start ⇒ `_round_up_to_nearest_multiple` folds. + if (static_zero_start and sl.start.asConstantInt() == @as(?i64, 0)) { + const is_oob = b.cmpi(.sgt, sl.size, b.constant(.i32, dim_size)); + const sz = b.select(is_oob, b.constant(.i32, dim_size), sl.size); + size[d] = b.emit(mosaic_tpu.assume_multiple(b.ctx, sz.inner, tiling, b.loc())); + continue; + } + const is_oob = b.cmpi(.sgt, b.addi(sl.start, sl.size), b.constant(.i32, dim_size)); + const rounded = blk: { + if (tiling == 1) break :blk b.subi(b.constant(.i32, dim_size + 1), sl.start); + // lax sign-fixup for `jnp.remainder`. + const m = b.lift(tiling); + const c0 = b.constant(.i32, 0); + const rem_arg = b.subi(b.constant(.i32, dim_size), sl.start); + const r = b.remsi(rem_arg, m); + const r_nz = b.cmpi(.ne, r, c0); + const r_neg = b.cmpi(.slt, r, c0); + const fixup = b.andi(r_neg, r_nz); + const r_plus = b.addi(r, m); + const floormod = b.select(fixup, r_plus, r); + const round_down = b.subi(rem_arg, floormod); + break :blk b.addi(round_down, m); + }; + const sz = b.select(is_oob, rounded, sl.size); + size[d] = b.emit(mosaic_tpu.assume_multiple(b.ctx, sz.inner, tiling, b.loc())); + }, + } + } + return .{ .base = base, .size = size }; + } + + fn sliceSrc(self: *const BufferedRef, b: *Builder, dma: DmaSlice) Value { + const a = b.arena.allocator(); + const bs = self.spec.block_shape.?; + const src = if (self.spec.source_reshape_shape) |shape| b.memRefReshape(self.src, shape) else self.src; + var base = a.alloc(Value, bs.len) catch unreachable; + var dyn = std.ArrayList(Value).initCapacity(a, bs.len) catch unreachable; + var result_shape = a.alloc(i64, bs.len) catch unreachable; + for (bs, 0..) |bd, d| { + base[d] = dma.base[d]; + if (dma.size[d]) |sz| { + result_shape[d] = kDyn; + dyn.appendAssumeCapacity(sz); + } else { + result_shape[d] = switch (bd) { + .squeezed => 1, + .blocked => |n| n, + .bounded_slice => unreachable, + }; + } + } + const sliced = b.memRefSlice(src, base, result_shape, dyn.items); + if (self.block_keep.len == bs.len) return sliced; + + const squeezed_shape = a.alloc(i64, self.block_compact.len) catch unreachable; + for (self.block_keep, 0..) |sd, ci| squeezed_shape[ci] = result_shape[sd]; + return b.memRefSqueeze(sliced, squeezed_shape); + } + + /// Slices `alloca[slot, 0…]` (non-squeezed dims) then squeezes the slot dim. + fn sliceWindowAt(self: *const BufferedRef, b: *Builder, slot: Value, dma: DmaSlice) Value { + const a = b.arena.allocator(); + const bs = self.spec.block_shape.?; + const c0 = b.constant(.i32, 0); + const nd = 1 + self.block_compact.len; + var base = a.alloc(Value, nd) catch unreachable; + var result_shape = a.alloc(i64, nd) catch unreachable; + var dyn = std.ArrayList(Value).initCapacity(a, self.block_keep.len) catch unreachable; + base[0] = slot; + result_shape[0] = 1; + for (self.block_keep, 0..) |sd, ci| { + base[ci + 1] = c0; + if (dma.size[sd]) |sz| { + result_shape[ci + 1] = kDyn; + dyn.appendAssumeCapacity(sz); + } else { + result_shape[ci + 1] = switch (bs[sd]) { + .blocked => |n| n, + else => unreachable, + }; + } + } + const sliced = b.memRefSlice(self.window_ref, base, result_shape, dyn.items); + const squeezed_shape = a.alloc(i64, self.block_compact.len) catch unreachable; + @memcpy(squeezed_shape, result_shape[1..]); + return b.memRefSqueeze(sliced, squeezed_shape); + } + + fn semAt(b: *Builder, sem_array: Value, i: Value) Value { + const sliced = b.memRefSlice(sem_array, &.{i}, &.{1}, &.{}); + return b.memRefSqueeze(sliced, &.{}); + } + + fn currentRef(self: *BufferedRef, b: *Builder) Value { + if (self.is_trivial or !self.isBuffered()) return self.window_ref; + const slot = switch (self.buffer_type) { + .output => b.remui(self.copy_out_slot.?, b.constant(.i32, self.buffer_count)), + .input => b.remui(self.wait_in_slot.?, b.constant(.i32, self.buffer_count)), + }; + const a = b.arena.allocator(); + const nd = 1 + self.block_compact.len; + var base = a.alloc(Value, nd) catch unreachable; + var result_shape = a.alloc(i64, nd) catch unreachable; + const c0 = b.constant(.i32, 0); + base[0] = slot; + result_shape[0] = 1; + for (self.block_compact, 0..) |n, ci| { + base[ci + 1] = c0; + result_shape[ci + 1] = n; + } + const sliced = b.memRefSlice(self.window_ref, base, result_shape, &.{}); + return b.memRefSqueeze(sliced, self.block_compact); + } + + fn syncCopyIn(self: *BufferedRef, b: *Builder) void { + const blk = mlir.Block.init(&.{}, &.{}); + b.pushBlock(blk); + const sem = semAllocOp(b, dmaSemRefTy(b, null)); + b.enqueueDma(self.src, self.window_ref, sem, .{}); + b.waitDma2(sem, self.src, self.window_ref, .{}); + _ = mosaic_tpu.yield(b.ctx, &.{}, b.loc()).appendTo(blk); + b.popBlock(); + _ = mosaic_tpu.region(b.ctx, blk, &.{}, b.loc()).appendTo(b.currentBlock()); + } + + fn syncCopyOut(self: *BufferedRef, b: *Builder) void { + const blk = mlir.Block.init(&.{}, &.{}); + b.pushBlock(blk); + const sem = semAllocOp(b, dmaSemRefTy(b, null)); + b.enqueueDma(self.window_ref, self.src, sem, .{}); + b.waitDma2(sem, self.window_ref, self.src, .{}); + _ = mosaic_tpu.yield(b.ctx, &.{}, b.loc()).appendTo(blk); + b.popBlock(); + _ = mosaic_tpu.region(b.ctx, blk, &.{}, b.loc()).appendTo(b.currentBlock()); + } + + // Op order mirrors pipeline.py: copy_in/copy_out/wait_out take slot rem + // before get_dma_slice; wait_in does it after. + + fn defaultCopyIn(self: *BufferedRef, b: *Builder, indices: []const Value, slot_cumulative: Value, static_zero_start: bool) void { + const slot = b.remui(slot_cumulative, b.constant(.i32, self.buffer_count)); + const dma = self.getDmaSlice(b, indices, static_zero_start); + const src_sl = self.sliceSrc(b, dma); + const dst_sl = self.sliceWindowAt(b, slot, dma); + const sem = semAt(b, self.sem_recvs.?, slot); + b.enqueueDma(src_sl, dst_sl, sem, .{}); + } + + fn defaultWaitIn(self: *BufferedRef, b: *Builder, indices: []const Value) void { + const dma = self.getDmaSlice(b, indices, false); + const slot = b.remui(self.wait_in_slot.?, b.constant(.i32, self.buffer_count)); + const src_sl = self.sliceSrc(b, dma); + const dst_sl = self.sliceWindowAt(b, slot, dma); + const sem = semAt(b, self.sem_recvs.?, slot); + b.waitDma2(sem, src_sl, dst_sl, .{}); + } + + fn defaultCopyOut(self: *BufferedRef, b: *Builder, indices: []const Value) void { + if (self.buffer_count == 1) @panic("emitPipeline: single-buffered output sync_copy_out unsupported (TODO)"); + const slot = b.remui(self.copy_out_slot.?, b.constant(.i32, self.buffer_count)); + const dma = self.getDmaSlice(b, indices, false); + // Pallas evaluates `src` (VMEM window) before `dst` (HBM). + const src_sl = self.sliceWindowAt(b, slot, dma); + const dst_sl = self.sliceSrc(b, dma); + const sem = semAt(b, self.sem_sends.?, slot); + b.enqueueDma(src_sl, dst_sl, sem, .{}); + } + + fn defaultWaitOut(self: *BufferedRef, b: *Builder, indices: []const Value) void { + if (self.buffer_count <= 1) return; + const slot = b.remui(self.wait_out_slot.?, b.constant(.i32, self.buffer_count)); + const dma = self.getDmaSlice(b, indices, false); + const src_sl = self.sliceWindowAt(b, slot, dma); + const dst_sl = self.sliceSrc(b, dma); + const sem = semAt(b, self.sem_sends.?, slot); + b.waitDma2(sem, src_sl, dst_sl, .{}); + } + + fn copyIn(self: *BufferedRef, b: *Builder, indices: []const Value, slot_cumulative: Value, static_zero_start: bool) void { + if (self.is_trivial) return; + + if (self.spec.transfer_strategy) |strategy| { + if (strategy.copy_in) |customCopyIn| { + customCopyIn(self, b, indices, slot_cumulative, static_zero_start, strategy.ctx); + return; + } + } + + self.defaultCopyIn(b, indices, slot_cumulative, static_zero_start); + } + + fn waitIn(self: *BufferedRef, b: *Builder, indices: []const Value) void { + if (self.spec.transfer_strategy) |strategy| { + if (strategy.wait_in) |customWaitIn| { + customWaitIn(self, b, indices, strategy.ctx); + return; + } + } + self.defaultWaitIn(b, indices); + } + + fn copyOut(self: *BufferedRef, b: *Builder, indices: []const Value) void { + if (self.spec.transfer_strategy) |strategy| { + if (strategy.copy_out) |customCopyOut| { + customCopyOut(self, b, indices, strategy.ctx); + return; + } + } + self.defaultCopyOut(b, indices); + } + + fn waitOut(self: *BufferedRef, b: *Builder, indices: []const Value) void { + if (self.spec.transfer_strategy) |strategy| { + if (strategy.wait_out) |customWaitOut| { + customWaitOut(self, b, indices, strategy.ctx); + return; + } + } + self.defaultWaitOut(b, indices); + } + + pub fn sourceRef(self: *const BufferedRef) Value { + return self.src; + } + + pub fn windowRef(self: *const BufferedRef) Value { + return self.window_ref; + } + + pub fn copyInSlot(self: *const BufferedRef, b: *Builder, slot_cumulative: Value) Value { + return b.remui(slot_cumulative, b.constant(.i32, self.buffer_count)); + } + + pub fn waitInSlot(self: *const BufferedRef, b: *Builder) Value { + return b.remui(self.wait_in_slot.?, b.constant(.i32, self.buffer_count)); + } + + pub fn receiveSemaphoreAt(self: *const BufferedRef, b: *Builder, slot: Value) Value { + return semAt(b, self.sem_recvs.?, slot); + } + + pub fn copyOutSlot(self: *const BufferedRef, b: *Builder) Value { + const buffer_count: i32 = @intCast(self.buffer_count); + return b.remui(self.copy_out_slot.?, b.lift(buffer_count)); + } + + pub fn waitOutSlot(self: *const BufferedRef, b: *Builder) Value { + const buffer_count: i32 = @intCast(self.buffer_count); + return b.remui(self.wait_out_slot.?, b.lift(buffer_count)); + } + + pub fn sendSemaphoreAt( + self: *const BufferedRef, + b: *Builder, + slot: Value, + ) Value { + return semAt(b, self.sem_sends.?, slot); + } +}; + +/// Prologue/loop/epilogue copy+wait schedule. +const Scheduler = struct { + step: Value, + grid: []const Value, + num_steps: Value, + num_stages: usize, + trace_scopes: bool, + first_step: Value, + last_step: Value, + indices_storage: [4]Value = undefined, + prev_storage: [4]Value = undefined, + next_storage: [4]Value = undefined, + fetch_storage: [8][4]Value = undefined, + fetch_len: usize, + indices_len: usize, + + fn init(b: *Builder, step: Value, grid: []const Value, num_steps: Value, num_stages: usize, trace_scopes: bool) Scheduler { + if (num_stages + 1 > 8) @panic("emitPipeline: buffer_count > 7 unsupported"); + const c0 = b.constant(.i32, 0); + const c1 = b.constant(.i32, 1); + var s: Scheduler = .{ + .step = step, + .grid = grid, + .num_steps = num_steps, + .num_stages = num_stages, + .trace_scopes = trace_scopes, + .first_step = b.cmpi(.eq, step, c0), + .last_step = b.cmpi(.eq, step, b.subi(num_steps, c1)), + .fetch_len = num_stages + 1, + .indices_len = grid.len, + }; + for (0..grid.len) |i| s.indices_storage[i] = c0; + s.recomputeDerivedIndices(b); + return s; + } + + fn indices(self: *Scheduler) []const Value { + return self.indices_storage[0..self.indices_len]; + } + fn prevIndices(self: *Scheduler) []const Value { + return self.prev_storage[0..self.indices_len]; + } + fn nextIndices(self: *Scheduler) []const Value { + return self.next_storage[0..self.indices_len]; + } + fn fetchIndices(self: *Scheduler, lookahead: usize) []const Value { + std.debug.assert(lookahead < self.fetch_len); + return self.fetch_storage[lookahead][0..self.indices_len]; + } + + fn recomputeDerivedIndices(self: *Scheduler, b: *Builder) void { + const prev = prevIndexND(b, self.indices(), self.grid); + const next = nextIndexND(b, self.indices(), self.grid); + for (0..self.indices_len) |i| { + self.prev_storage[i] = prev[i]; + self.next_storage[i] = next[i]; + self.fetch_storage[0][i] = self.indices_storage[i]; + self.fetch_storage[1][i] = next[i]; + } + var fetch = next; + var lookahead: usize = 2; + while (lookahead < self.fetch_len) : (lookahead += 1) { + fetch = nextIndexND(b, fetch, self.grid); + for (0..self.indices_len) |i| self.fetch_storage[lookahead][i] = fetch[i]; + } + } + + fn namedScope(self: *Scheduler, b: *Builder, name: []const u8) void { + if (self.trace_scopes) b.traceStart(10, name); + } + fn endScope(self: *Scheduler, b: *Builder) void { + if (self.trace_scopes) b.traceStop(); + } + + /// `~out_of_fetch` as `step < num_steps + (1 - buffer_count)`. + fn notOutOfFetch(self: *Scheduler, b: *Builder, br: *const BufferedRef) Value { + std.debug.assert(br.isBuffered()); + const off: i32 = 1 - @as(i32, @intCast(br.buffer_count)); + return b.cmpi(.slt, self.step, b.addi(self.num_steps, b.lift(off))); + } + + fn notFirstStep(self: *Scheduler, b: *Builder) Value { + return b.cmpi(.ne, self.step, b.constant(.i32, 0)); + } + + fn indicesDiffer(b: *Builder, br: *const BufferedRef, ai: []const Value, ci: []const Value) Value { + const xa = br.computeIndex(b, ai); + const xc = br.computeIndex(b, ci); + std.debug.assert(xa.len == xc.len and xa.len > 0); + var acc: ?Value = null; + for (xa, xc) |x, y| { + const d: Value = switch (x) { + .scalar => |xv| b.cmpi(.ne, xv, y.scalar), + .slice => |xs| b.ori(b.cmpi(.ne, xs.start, y.slice.start), b.cmpi(.ne, xs.size, y.slice.size)), + }; + acc = if (acc) |p| b.ori(p, d) else d; + } + return acc.?; + } + + fn hasChanged(self: *Scheduler, b: *Builder, br: *const BufferedRef) Value { + std.debug.assert(br.isBuffered() and !br.is_trivial); + return indicesDiffer(b, br, self.indices(), self.prevIndices()); + } + fn willChangeCurrent(self: *Scheduler, b: *Builder, br: *const BufferedRef) Value { + std.debug.assert(br.isBuffered() and !br.is_trivial); + return indicesDiffer(b, br, self.indices(), self.nextIndices()); + } + fn willChangeFetch(self: *Scheduler, b: *Builder, br: *const BufferedRef) Value { + std.debug.assert(br.isBuffered() and !br.is_trivial); + if (br.buffer_count < 2) return self.hasChanged(b, br); + return indicesDiffer(b, br, self.fetchIndices(br.buffer_count - 2), self.fetchIndices(br.buffer_count - 1)); + } + + fn advance(b: *Builder, slot: Value, pred: Value) Value { + return b.select(pred, b.addi(slot, b.constant(.i32, 1)), slot); + } + + fn initializeStep(self: *Scheduler, b: *Builder, br: *BufferedRef, step: usize) void { + if ((br.buffer_type != .input and br.buffer_type != .input_output) or !br.isBuffered() or br.is_trivial) return; + if (step + 1 >= br.buffer_count) return; + if (br.copy_in_slot == null) br.copy_in_slot = b.constant(.i32, 0); + + const pred = if (step == 0) self.first_step else blk: { + const block_changed = indicesDiffer(b, br, self.fetchIndices(step), self.fetchIndices(step - 1)); + break :blk b.andi(self.first_step, block_changed); + }; + const fetch_indices = self.fetchIndices(step); + if (step == 0) { + // static_zero_start=true: Python-int 0 grid indices enable folds in getDmaSlice. + br.copyIn(b, fetch_indices, br.copy_in_slot.?, true); + } else { + var when = b.openIf(pred); + { + br.copyIn(b, fetch_indices, br.copy_in_slot.?, false); + when.close(); + } + } + br.copy_in_slot = advance(b, br.copy_in_slot.?, pred); + } + + fn copyIn(self: *Scheduler, b: *Builder, br: *BufferedRef) void { + if (br.is_trivial) return; + var pred = b.andi(self.willChangeFetch(b, br), self.notOutOfFetch(b, br)); + if (br.isBuffered() and br.buffer_count < 2) pred = b.ori(pred, self.first_step); + if (br.buffer_type != .input and br.buffer_type != .input_output) return; + { + var when = b.openIf(pred); + { + self.namedScope(b, "ep_copy_in"); + br.copyIn(b, self.fetchIndices(br.buffer_count - 1), br.copy_in_slot.?, false); + self.endScope(b); + when.close(); + } + } + br.copy_in_slot = advance(b, br.copy_in_slot.?, pred); + } + + fn waitIn(self: *Scheduler, b: *Builder, br: *BufferedRef) void { + if (br.is_trivial) return; + const pred = b.ori(self.hasChanged(b, br), self.first_step); + { + var when = b.openIf(pred); + { + self.namedScope(b, "ep_wait_in"); + if (br.buffer_type == .input or br.buffer_type == .input_output) br.waitIn(b, self.indices()); + self.endScope(b); + when.close(); + } + } + } + + fn copyOut(self: *Scheduler, b: *Builder, br: *BufferedRef) void { + if (br.is_trivial) return; + const pred = b.ori(self.willChangeCurrent(b, br), self.last_step); + { + var when = b.openIf(pred); + { + self.namedScope(b, "ep_copy_out"); + if (br.buffer_type == .output or br.buffer_type == .input_output) br.copyOut(b, self.indices()); + self.endScope(b); + when.close(); + } + } + br.copy_out_slot = advance(b, br.copy_out_slot.?, pred); + } + + fn waitOut(self: *Scheduler, b: *Builder, br: *BufferedRef) void { + if (br.is_trivial) return; + const pred = b.andi(self.hasChanged(b, br), self.notFirstStep(b)); + { + var when = b.openIf(pred); + { + self.namedScope(b, "ep_wait_out"); + // copy_out was issued the previous iteration. + if (br.buffer_type == .output or br.buffer_type == .input_output) br.waitOut(b, self.prevIndices()); + self.endScope(b); + when.close(); + } + } + br.wait_out_slot = advance(b, br.wait_out_slot.?, pred); + } + + fn finalize(self: *Scheduler, b: *Builder, br: *BufferedRef) void { + if (br.is_trivial or (br.buffer_type != .output and br.buffer_type != .input_output)) return; + var when = b.openIf(self.last_step); + { + self.namedScope(b, "ep_finalize"); + br.waitOut(b, self.indices()); + self.endScope(b); + when.close(); + } + } + + fn advanceSlots(self: *Scheduler, b: *Builder, br: *BufferedRef) void { + if (br.buffer_type != .input and br.buffer_type != .input_output) return; + if (br.is_trivial) { + switch (br.buffer_type) { + .input => br.slot = advance(b, br.slot.?, self.last_step), + .input_output => br.wait_in_slot = advance(b, br.wait_in_slot.?, self.last_step), + .output => unreachable, + } + } else { + const pred = b.ori(self.willChangeCurrent(b, br), self.last_step); + br.wait_in_slot = advance(b, br.wait_in_slot.?, pred); + } + } +}; + +pub fn memRefAlloca(b: *Builder, ty: *const mlir.Type) Value { + return b.emit(memref.alloca(b.ctx, ty, &.{}, &.{}, .{}, b.loc())); +} + +pub fn semAllocOp(b: *Builder, ty: *const mlir.Type) Value { + return b.emit(mosaic_tpu.sem_alloc(b.ctx, ty, b.loc())); +} + +/// `pl.SemaphoreType.DMA((n,))`, or rank-0 for `n == null`. +pub fn dmaSemRefTy(b: *Builder, n: ?usize) *const mlir.Type { + const shape: []const i64 = if (n) |m| &.{@intCast(m)} else &.{}; + return mlir.Type.memRef(mosaic_tpu.dmaSemaphoreType(b.ctx), shape, null, MemorySpace.semaphore_mem.attribute(b.ctx)); +} + +fn gridSize(b: *Builder, grid: []const Value) Value { + var acc = grid[0]; + for (grid[1..]) |g| acc = b.muli(acc, g); + return acc; +} + +fn filterIndices(b: *Builder, indices: []const Value, grid: []const Value) [4]Value { + var out: [4]Value = undefined; + const c0 = b.constant(.i32, 0); + for (indices, grid, 0..) |i, g, pos| { + out[pos] = if (g.asConstantInt() == @as(?i64, 1)) c0 else i; + } + return out; +} + +fn nextIndexND(b: *Builder, indices: []const Value, grid: []const Value) []const Value { + const a = b.arena.allocator(); + const out = a.alloc(Value, indices.len) catch @panic("nextIndexND OOM"); + const c0 = b.constant(.i32, 0); + var carry: Value = b.cmpi(.eq, c0, c0); + var rev: usize = 0; + while (rev < indices.len) : (rev += 1) { + const pos = indices.len - 1 - rev; + const inc = b.select(carry, b.addi(indices[pos], b.constant(.i32, 1)), indices[pos]); + carry = b.cmpi(.eq, inc, grid[pos]); + out[pos] = b.select(carry, c0, inc); + } + const filtered = filterIndices(b, out, grid); + for (0..indices.len) |i| out[i] = filtered[i]; + return out; +} + +fn prevIndexND(b: *Builder, indices: []const Value, grid: []const Value) []const Value { + const a = b.arena.allocator(); + const out = a.alloc(Value, indices.len) catch @panic("prevIndexND OOM"); + const c0 = b.constant(.i32, 0); + var borrow: Value = b.cmpi(.eq, c0, c0); + const cm1 = b.constant(.i32, -1); + var rev: usize = 0; + while (rev < indices.len) : (rev += 1) { + const pos = indices.len - 1 - rev; + const dec = b.select(borrow, b.subi(indices[pos], b.constant(.i32, 1)), indices[pos]); + borrow = b.cmpi(.eq, dec, cm1); + out[pos] = b.select(borrow, b.subi(grid[pos], b.constant(.i32, 1)), dec); + } + const filtered = filterIndices(b, out, grid); + for (0..indices.len) |i| out[i] = filtered[i]; + return out; +} + +/// `null` block_shape, or every dim is `.blocked(full_dim)`. +fn specHasTrivialWindowing(spec: PipeSpec) bool { + const bs = spec.block_shape orelse return true; + if (bs.len != spec.full_shape.len) return false; + for (bs, spec.full_shape) |bd, fs| switch (bd) { + .blocked => |n| if (n != fs) return false, + else => return false, + }; + return true; +} + +test { + std.testing.refAllDecls(@This()); +} + +test "trivial input-output ref is copied in and out" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + builder.registerDialects(registry); + func.registerExtensions(registry); + + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + + var b = try Builder.init(std.testing.allocator, ctx, "trivial_input_output", &.{ + .{ + .name = "state", + .kind = .{ .ref = .{ + .shape = &.{ 8, 128 }, + .dtype = .bf16, + .memory_space = .hbm, + } }, + }, + }, &.{}); + defer b.deinit(); + + const noOpBody = struct { + fn call( + _: *Builder, + _: []const Value, + _: []const RefWindow, + _: []const Value, + _: ?*anyopaque, + ) FinishError!void {} + }.call; + + try emitPipeline(&b, &.{b.arg(0)}, &.{}, .{ + .body = noOpBody, + .grid = &.{b.constant(.i32, 1)}, + .in_specs = &.{.{ + .full_shape = &.{ 8, 128 }, + .dtype = .bf16, + .buffer_type = .input_output, + }}, + .out_specs = &.{}, + .trace_scopes = false, + }); + + const ir = try b.finish(&.{}); + defer std.testing.allocator.free(ir); + + // One enqueue/wait pair for copy_in, one for copy_out. + try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, ir, "tpu.enqueue_dma")); + try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, ir, "tpu.wait_dma")); +} diff --git a/dsl/tcl/BUILD.bazel b/dsl/tcl/BUILD.bazel new file mode 100644 index 00000000..096a3a67 --- /dev/null +++ b/dsl/tcl/BUILD.bazel @@ -0,0 +1,24 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "builder", + srcs = [ + "dtype.zig", + ], + import_name = "dsl/tcl/builder", + main = "builder.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//dsl/common:builder", + "//mlir", + "//mlir/dialects/arith", + "//mlir/dialects/func", + "//mlir/dialects/tcl", + ], +) + +zig_test( + name = "test", + deps = [":builder"], +) diff --git a/dsl/tcl/README.md b/dsl/tcl/README.md new file mode 100644 index 00000000..b01b7189 --- /dev/null +++ b/dsl/tcl/README.md @@ -0,0 +1,170 @@ +# TCL kernels in Zig + +`dsl/tcl` is a Zig front end for Furiosa's TCL (Tensor Contraction +Language), shaped after the Python `furiosa.tcl` DSL: a kernel declares named +axes and typed tensors, opens tensor operations that `fetch`, `contract`, run +VE instructions and `commit`, and returns tensors. The `Builder` emits the +native `tcl` MLIR dialect (`mlir/dialects/tcl`), and +`zml.kernel.tcl.Kernel` turns it into a `furiosa.tcl` custom call that the +Furiosa plugin exports to TCL source and compiles with `furiosa-tcc` as one +native unit. + +## A kernel + +`furiosa.tcl.examples.softmax`, in Zig: + +```zig +const zml = @import("zml"); +const tcl = zml.kernel.tcl; + +const Cfg = struct { b: i64, s: i64, d: i64 }; + +pub const Softmax = tcl.Kernel(Cfg, .{ + .name = "softmax_kernel", + .inputs = &.{"input"}, + .outputs = &.{"out"}, + .run = run, +}); + +fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { + const B = b.axis("B", cfg.b); + const S = b.axis("S", cfg.s); + const D = b.axis("D", cfg.d); + const t = try b.declareArgs(.{ .input = .{ .dtype = .bf16, .axes = &.{ B, S, D } } }); + + const op = b.tensorOperation(.{}); + const x = op.fetch(t.input, .{ .typecast_to = .f32 }); + const exped = x.subf(x.reduce(&.{D}, .maxf)).exp(); + const out = try op.commit(exped.divf(exped.reduce(&.{D}, .addf)), .{ .dtype = .bf16 }); + b.ret(&.{out}); +} + +// In a model: +const y = Softmax.call(.{ .input = x }, .{ .out = x.shape() }, .{ + .cfg = .{ .b = 2, .s = 16, .d = 1024 }, +}).out; +``` + +The plugin compiles it to the same TCL the Python DSL prints: + +``` +def softmax_kernel(B: Axis = 2, D: Axis = 1024, S: Axis = 16, v0: [B, S, D]/bf16) -> [B, S, D]/bf16: + v1: [B, S, D]/bf16 = tk.ReduceByVe( + %0 = read(v0, typecast_to=f32) + %1 = ve.exec(max_f t[D] %0) + %2 = ve.exec(%0 - %1) + %3 = ve.exec(exp %2) + %4 = ve.exec(+ t[D] %3) + %5 = ve.exec(%3 / %4) + write.to([B, S, D]/bf16) + ) + return v1 +``` + +## Python to Zig + +| `furiosa.tcl` | `dsl/tcl` | +|------------------------------------------------|-------------------------------------------------------| +| `x: tcl.Tensor[tcl.bf16, [B, S, D]]` | `.x = .{ .dtype = .bf16, .axes = &.{ B, S, D } }` | +| `x: tcl.Dram[tcl.bf16, [tcl.Broadcast], [M, N]]` | `.x = .{ .dtype = .bf16, .axes = &.{ M, N }, .dram = .{} }` | +| composite axis `C: Axis = (A * B)` | `b.axis("C", b.expr(.mul, A, B))` | +| `@tcl.kernel(config={...}, auto_config={...})` | `b.compilerConfig(.{ ... })`, `b.autoConfig(.{ ... })` | +| `@tcl.tensor_operation` | `const op = b.tensorOperation(.{})` | +| `fetch.typecast(x, tcl.f32)` | `op.fetch(x, .{ .typecast_to = .f32 })` | +| `fetch.broadcast(x, ...)`, `fetch.table_lookup` | `.broadcast_to = &.{...}`, `.table_lookup = table` | +| `fetch.pad(x, axis=L, left=1, right=1)` | `.pad = &.{.{ .axis = L, .left = 1, .right = 1 }}` | +| `fetch.slide(x, axis=L, frame_axis=Lf, window_axis=Lw)` | `.slide = &.{.{ .axis = L, .frame_axis = Lf, .window_axis = Lw }}` | +| `operation.contract(lhs, rhs, to=[B, M, N])` | `op.contract(lhs, rhs, &.{ B, M, N })` | +| `operation.subf(a, b)`, `operation.exp(a)` | `a.subf(b)`, `a.exp()`, or `op.ve(.subf, .{ a, b })` | +| `operation.reduce(x, axes=[D], op=tcl.Maxf)` | `x.reduce(&.{D}, .maxf)` | +| `operation.where(x > 0, a, b)` | `op.where(x, .gt, 0, a, b)` | +| `operation.to_fp(x, 15)`, `to_fxp(x, 15)` | `x.toFp(15)`, `x.toFxp(15)` | +| `compiler_hints=tcl.context(layout={Chip: ...})` | `.context = .{ .layout = .{ .chip = .broadcast } }` | +| `commit.typecast(v, tcl.bf16)` / `commit.permute` | `op.commit(v, .{ .dtype = .bf16, .axes = ... })` | +| `tcl.reshape`, `transmute`, `concat`, `slice` | `b.reshape`, `b.transmute`, `b.concat`, `b.slice` | +| `tcl.gather`, `scatter`, `arange`, `vector`, `all_gather` | `b.gather`, `b.scatter`, `b.arange`, `b.vector`, `b.allGather` | +| `for i in tcl.range(0, n):` (carried `acc`) | `var loop = b.openFor(n, .{acc})` ... `try loop.yield(.{new_acc})`, then `loop.results` | +| `x[i]`, `out[i] = v`, `tcl.scratchpad()` | `b.indexRead(x, loop.iv)`, `b.indexWrite(out, loop.iv, v)`, `b.scratchpad` | +| `core.reduce_max_i32(t)`, `core.eval_expr(...)` | `b.reduceMaxI32(t)`, `b.symExpr(.div, n, 64)` | +| `return out` | `b.ret(&.{out})` | + +* **Axes are static.** `b.axis(name, size)` declares `name: Axis = size`, + with an integer or an `Expr` over other axes; axis names start with an + uppercase letter. `furiosa-tcc` needs every axis bound. Kernel inputs and + outputs must match the XLA shapes axis by axis (row-major), which `call` + checks. +* **Loops.** Put the body in a Zig `{ ... }` block right after `openFor`, + ending with `try loop.yield(...)`. `loop` stays outside, so `loop.results` + stays readable after the block. +* **DRAM tensors.** A `.dram` argument comes back as its `as_logical` view; + `b.asDram(t, .{...})` returns a result in a DRAM mapping. +* **Multi-chip kernels.** A kernel that places data or work on chips (a + chip-split DRAM tensor, a `.context`, `allGather`) is compiled by the plugin + for all the chips of its SPMD program (2 or 4), and runs as one launch + across the partitions; other kernels stay per chip. Its arguments and + results are DRAM tensors: `Dram{ .chip = &.{.{ .axis = C }} }` splits axis + `C`, sized to the chip count, across chips (each partition holds its share), + the default `.broadcast` keeps a copy per chip. +* **Convolutions** pad and slide the input in its read: a slide replaces its + axis by `window_axis, frame_axis`, then `contract` with the filter. On its + own (without a slide) the padding is dropped. +* **The mainstream rule.** As in the Python DSL, instructions are pipelined: + `commit` must take the last instruction, there is at most one contraction + and it comes before VE instructions. Reads are hoisted in front. +* **VE registers are f32 or i32**, decided by the instruction. Scalar + operands become `f32`/`i32` constants (`x.mulf(0.5)`). +* **Tactics are guessed** like `@tcl.tensor_operation` (`EinsumByDpe`, + `ReduceByVe`, `Interleaving`, ...); `b.tensorOperation(.{ .tactic = ... })` + overrides it. +* **Tensors passed directly** to an instruction are read without modifiers. +* **Constant outputs.** `furiosa-tcc` 2026.3 fails ("No producer") on a + function returning two input-independent tensors, such as `arange` and + `vector`. +* **Loops** run on the device. `x[i]` and `acc[i] = v` address the outermost + axis, and the body must index something with `i`: that bounds the trip + count. A dynamic trip count must be computed in the kernel (`reduceMaxI32`, + `symExpr`); one read straight from an argument adds a CPU node the plugin + rejects. + +## Good practices on RNGD + +The TCL CPU interpreter (`furiosa.torch.TclModule`) accepts several programs +that compute garbage on the hardware: check kernels on RNGD. + +* **One VE stream per tensor operation.** Each instruction takes the previous + result and one side operand (a read or a constant). A select whose branches + are both constants must test the operation's first read. +* **Outer broadcasts** (`[B, H, V] * [B, H, K]` to `[B, H, V, K]`, tactic + `EinsumByVe`) need a shared leading axis, and the read with more axes first. +* **Contraction epilogues** read their extra tensors before the contraction + and combine the contraction result first: `contract(q, k).addf(bias)`. +* **Conversions.** `typecast_to` converts floats only; `toFp(n)` of an i32 + yields `x / 2^31` whatever `n`. +* **Update caches in place.** `scatter(rows, idx, axis, .{ .init = cache })` + on a kernel argument shares its buffer with the result (a whole-cache copy + otherwise), so return it as an aliased output. +* **Few axes, few operations.** tcc's tactic search grows with the number and + extents of axes, and more than linearly with the number of operations in a + function: merge axes before contractions (`[N, H, G*Q, C]`, not + `[N, H, G, Q, P, C]`) and set `.lowering_mode = "Heuristic"` above a few + tokens. +* **Prefer one-shot graphs to hand-written online softmax.** Furiosa's + production attention gathers the context, contracts, masks with a select, + runs the softmax as one `ReduceByVe` and contracts again; gathers take + `valid_length` to skip padding, and padding indices are spread + (`idx | (pos % n)`) rather than all pointing at one row. +* **Furiosa's kernels** in the SDK (`furiosa/kernels/`) are the reference for + layouts, tile sizes and compiler configs: `common/attention.py` (attention + and decode/prefill configs), `common/topk.py` and + `common/blockwise_moe_index.py` (routing on the device), + `qwen3_moe/optimized/` (blockwise MoE for prefill, activation-stationary MoE + for decode), `exaone_moe/k_exaone_w4fa16kv16/flash_attention.py` (chunked + softmax over a dynamic trip count). + +## The custom call + +`ops.tcl` emits `stablehlo.custom_call @furiosa.tcl` with the printed module +as its backend config. The plugin parses it, checks the function signature +against the operand and result shapes, exports TCL source and compiles it +with the rest of the program's units. With `--xla_dump_to=`, +`*.furiosa-unit-sources.txt` holds each unit's TCL. diff --git a/dsl/tcl/builder.zig b/dsl/tcl/builder.zig new file mode 100644 index 00000000..f86d338b --- /dev/null +++ b/dsl/tcl/builder.zig @@ -0,0 +1,1429 @@ +const std = @import("std"); + +const arith = @import("mlir/dialects/arith"); +const common = @import("dsl/common/builder"); +const func = @import("mlir/dialects/func"); +const mlir = @import("mlir"); +const tcl = @import("mlir/dialects/tcl"); + +pub const dtypes = @import("dtype.zig"); + +pub const DType = dtypes.DType; +pub const Tactic = tcl.Tactic; +pub const ReduceMode = tcl.ReduceMode; +pub const Predicate = tcl.Predicate; + +pub fn registerDialects(registry: *mlir.DialectRegistry) void { + inline for (.{ func, arith, tcl }) |dialect| dialect.registerDialect(registry); +} + +pub const FinishError = error{ InvalidMlir, OutOfMemory } || std.Io.Writer.Error; + +test { + std.testing.refAllDecls(@This()); +} + +/// A named TCL axis with its extent, from `Builder.axis`. +pub const Axis = struct { + name: []const u8, + size: i64, +}; + +/// A TCL expression over axes and integers, for composite axes +/// (`C: Axis = (A * B)`) and DRAM mappings (`D @ Dp`, `F / 2`, `Broadcast`). +/// `Builder.expr` makes the nodes. +pub const Expr = union(enum) { + int: i64, + axis: Axis, + broadcast, + node: *const Node, + + pub const Kind = enum { add, sub, mul, div, rem, exact_div, stride, modulo, padding, resize, pair }; + pub const Node = struct { kind: Kind, lhs: Expr, rhs: Expr }; + + /// The value of integer arithmetic over axes; null for mappings. + pub fn value(self: Expr) ?i64 { + return switch (self) { + .int => |n| n, + .axis => |a| a.size, + .broadcast => null, + .node => |n| { + const l = n.lhs.value() orelse return null; + const r = n.rhs.value() orelse return null; + return switch (n.kind) { + .add => l + r, + .sub => l - r, + .mul => l * r, + .div, .exact_div => if (r == 0) null else @divFloor(l, r), + .rem => if (r == 0) null else @mod(l, r), + else => null, + }; + }, + }; + } +}; + +/// `dram(chip | inner)/T`: a tensor in an explicit DRAM mapping, Python's +/// `tcl.Dram[T, chip, inner, tcl.original[...]]`. `chip` is `.broadcast` +/// (a copy per chip) or the axes split across chips; `inner` defaults to the +/// tensor's other axes. +pub const Dram = struct { + chip: []const Expr = &.{.broadcast}, + inner: ?[]const Expr = null, + original: []const Axis = &.{}, + + fn shards(self: Dram) i64 { + var n: i64 = 1; + for (self.chip) |e| if (e == .axis) { + n *= e.axis.size; + }; + return n; + } + + fn isChipAxis(self: Dram, a: Axis) bool { + for (self.chip) |e| if (e == .axis and std.mem.eql(u8, e.axis.name, a.name)) return true; + return false; + } +}; + +/// `@ context(operator = {...}, heuristic_hint = {...})`, Python's +/// `tcl.context(layout=..., heuristic_hint=...)`. +pub const Context = struct { + layout: ?Layout = null, + heuristic_hint: []const Hint = &.{}, + + /// `{Chip: ..., Cluster: ..., Split: [...]}`: `chip` and `cluster` are an + /// `.axis` or `.broadcast`. + pub const Layout = struct { + chip: Expr, + cluster: ?Expr = null, + split: []const Axis = &.{}, + }; + + pub const Hint = struct { + name: []const u8, + value: union(enum) { int: i64, float: f64, boolean: bool, axis: Axis }, + }; +}; + +/// The VE (vector engine) instructions, named like `furiosa.tcl.operation`. +pub const VeOp = enum { + exp, + neg_exp, + sqrt, + tanh, + sigmoid, + erf, + log, + sin, + cos, + negf, + absf, + addi, + addi_sat, + subi, + subi_sat, + mul_q31, + muli, + shl, + shl_sat, + shr_logical, + shr_arith, + shr_arith_round, + bit_andi, + bit_ori, + bit_xori, + mini, + maxi, + absmini, + absmaxi, + addf, + subf, + mulf, + divf, + bit_andf, + bit_orf, + bit_xorf, + minf, + maxf, + absminf, + absmaxf, + reinterpret_as_float, + reinterpret_as_int, + to_float, + to_int, + + pub fn arity(self: VeOp) usize { + return switch (self) { + .exp, .neg_exp, .sqrt, .tanh, .sigmoid, .erf, .log, .sin, .cos, .negf, .absf, .reinterpret_as_float, .reinterpret_as_int, .to_float, .to_int => 1, + else => 2, + }; + } + + /// VE registers are f32 or i32, according to the instruction. + pub fn result(self: VeOp) DType { + return switch (self) { + .exp, .neg_exp, .sqrt, .tanh, .sigmoid, .erf, .log, .sin, .cos, .negf, .absf, .addf, .subf, .mulf, .divf, .bit_andf, .bit_orf, .bit_xorf, .minf, .maxf, .absminf, .absmaxf, .reinterpret_as_float, .to_float => .f32, + else => .i32, + }; + } +}; + +/// A graph-level tensor: a kernel argument, or the result of a tensor or +/// graph operation (`x: [A, B]/bf16` in TCL). +pub const Tensor = struct { + inner: *const mlir.Value, + dtype: DType, + axes: []const Axis, + /// Chips a DRAM tensor is split across: each SPMD partition holds + /// 1/shards of it. + shards: i64 = 1, +}; + +/// A value inside a tensor operation (`%N` in TCL): the result of a read, +/// the contraction or a VE instruction. +pub const Value = struct { + op: *TensorOperation, + inner: *const mlir.Value, + dtype: DType, + axes: []const Axis, + + pub fn unary(self: Value, comptime opcode: VeOp) Value { + return self.op.ve(opcode, .{self}); + } + + /// `rhs` is a `Value`, a `Tensor` (read on use) or a scalar literal. + pub fn binary(self: Value, comptime opcode: VeOp, rhs: anytype) Value { + return self.op.ve(opcode, .{ self, rhs }); + } + + pub fn reduce(self: Value, axes: []const Axis, mode: ReduceMode) Value { + return self.op.reduce(self, axes, mode); + } + + pub fn cumsum(self: Value, axes: []const Axis) Value { + return self.op.cumsum(self, axes); + } + + pub fn addf(self: Value, rhs: anytype) Value { + return self.binary(.addf, rhs); + } + pub fn subf(self: Value, rhs: anytype) Value { + return self.binary(.subf, rhs); + } + pub fn mulf(self: Value, rhs: anytype) Value { + return self.binary(.mulf, rhs); + } + pub fn divf(self: Value, rhs: anytype) Value { + return self.binary(.divf, rhs); + } + pub fn maxf(self: Value, rhs: anytype) Value { + return self.binary(.maxf, rhs); + } + pub fn minf(self: Value, rhs: anytype) Value { + return self.binary(.minf, rhs); + } + pub fn addi(self: Value, rhs: anytype) Value { + return self.binary(.addi, rhs); + } + pub fn subi(self: Value, rhs: anytype) Value { + return self.binary(.subi, rhs); + } + pub fn muli(self: Value, rhs: anytype) Value { + return self.binary(.muli, rhs); + } + pub fn exp(self: Value) Value { + return self.unary(.exp); + } + pub fn sqrt(self: Value) Value { + return self.unary(.sqrt); + } + pub fn tanh(self: Value) Value { + return self.unary(.tanh); + } + pub fn sigmoid(self: Value) Value { + return self.unary(.sigmoid); + } + pub fn log(self: Value) Value { + return self.unary(.log); + } + pub fn negf(self: Value) Value { + return self.unary(.negf); + } + pub fn toFloat(self: Value) Value { + return self.unary(.to_float); + } + pub fn toInt(self: Value) Value { + return self.unary(.to_int); + } + pub fn toFp(self: Value, int_width: u5) Value { + return self.op.toFp(self, int_width); + } + pub fn toFxp(self: Value, int_width: u5) Value { + return self.op.toFxp(self, int_width); + } +}; + +/// One TCL tensor operation, `tk.(...)`: reads, at most one +/// contraction, VE instructions, then `commit` writes its result tensor. +/// Like `@tcl.tensor_operation`, every instruction after the reads is +/// pipelined: `commit` takes the last one. +pub const TensorOperation = struct { + pub const Options = struct { + /// Guessed from the instructions like the Python DSL when null. + tactic: ?Tactic = null, + context: ?Context = null, + }; + + pub const FetchOptions = struct { + typecast_to: ?DType = null, + broadcast_to: ?[]const Axis = null, + subtraction: ?Tensor = null, + table_lookup: ?Tensor = null, + /// Padding along an axis, applied by a `slide` over it. + pad: []const Pad = &.{}, + /// Sliding windows: each replaces its axis by `window_axis, frame_axis`. + slide: []const Slide = &.{}, + }; + + pub const Slide = struct { + axis: Axis, + frame_axis: Axis, + window_axis: Axis, + /// The window before dilation; `window_axis.size` by default. + undilated_window: ?i64 = null, + stride: i64 = 1, + dilation: i64 = 1, + }; + + pub const Pad = struct { + axis: Axis, + left: i64 = 0, + right: i64 = 0, + fill: f64 = 0, + }; + + pub const CommitOptions = struct { + dtype: ?DType = null, + /// A permutation of the committed value's axes. + axes: ?[]const Axis = null, + }; + + b: *Builder, + tactic: ?Tactic, + context: ?Context, + body: *mlir.Block, + inputs: std.ArrayList(Tensor) = .empty, + plain_reads: std.ArrayList(struct { tensor: *const mlir.Value, value: Value }) = .empty, + first_compute: ?*mlir.Operation = null, + last: ?*const mlir.Value = null, + has_dpe: bool = false, + has_ve: bool = false, + has_reduce: bool = false, + has_select: bool = false, + + /// `read(t, typecast_to=..., broadcast_to=..., ...)`. + pub fn fetch(self: *TensorOperation, t: Tensor, opts: FetchOptions) Value { + const b = self.b; + const ctx = b.ctx; + var operands: [3]*const mlir.Value = .{ self.input(t), undefined, undefined }; + var n: usize = 1; + var fields: [6]mlir.NamedAttribute = undefined; + var nfields: usize = 0; + var dtype = t.dtype; + var axes = t.axes; + if (opts.subtraction) |s| { + operands[n] = self.input(s); + fields[nfields] = .named(ctx, "subtraction", .int(ctx, .i64, n)); + n += 1; + nfields += 1; + } + if (opts.table_lookup) |table| { + operands[n] = self.input(table); + fields[nfields] = .named(ctx, "table_lookup", .int(ctx, .i64, n)); + dtype = table.dtype; + n += 1; + nfields += 1; + } + if (opts.typecast_to) |to| { + fields[nfields] = .named(ctx, "typecast_to", .typeAttr(to.toMlir(ctx))); + dtype = to; + nfields += 1; + } + if (opts.pad.len > 0) { + const pads = b.arena.allocator().alloc(mlir.NamedAttribute, opts.pad.len) catch @panic("OOM"); + for (pads, opts.pad) |*dst, p| dst.* = .named(ctx, p.axis.name, .array(ctx, &.{ + .int(ctx, .i64, p.left), + .int(ctx, .i64, p.right), + .float(ctx, .f64, p.fill), + })); + fields[nfields] = .named(ctx, "pad", .dict(ctx, pads)); + nfields += 1; + } + if (opts.slide.len > 0) { + const slides = b.arena.allocator().alloc(mlir.NamedAttribute, opts.slide.len) catch @panic("OOM"); + var slid: std.ArrayList(Axis) = .empty; + slid.appendSlice(b.arena.allocator(), axes) catch @panic("OOM"); + for (slides, opts.slide) |*dst, sl| { + dst.* = .named(ctx, sl.axis.name, .dict(ctx, &.{ + .named(ctx, "undilated_window", .int(ctx, .i64, sl.undilated_window orelse sl.window_axis.size)), + .named(ctx, "frame_axis", b.axisAttr(sl.frame_axis)), + .named(ctx, "window_axis", b.axisAttr(sl.window_axis)), + .named(ctx, "stride", .int(ctx, .i64, sl.stride)), + .named(ctx, "dilation", .int(ctx, .i64, sl.dilation)), + })); + const at = for (slid.items, 0..) |a, i| { + if (std.mem.eql(u8, a.name, sl.axis.name)) break i; + } else std.debug.panic("tcl: {s}: slide axis {s} is not read", .{ b.name, sl.axis.name }); + slid.items[at] = sl.window_axis; + slid.insert(b.arena.allocator(), at + 1, sl.frame_axis) catch @panic("OOM"); + } + fields[nfields] = .named(ctx, "slide", .dict(ctx, slides)); + nfields += 1; + axes = slid.items; + } + if (opts.broadcast_to) |to| { + fields[nfields] = .named(ctx, "broadcast_to", b.axesAttr(to)); + axes = b.dupeAxes(to); + nfields += 1; + } + const options = if (nfields == 0) null else tcl.readOptionsAttr(ctx, fields[0..nfields]) catch + std.debug.panic("tcl: invalid read options for {s}", .{b.name}); + const op = tcl.read(ctx, operands[0..n], b.logicalType(dtype, axes), .{ .options = options }, b.loc()); + return self.emit(op, dtype, axes, true); + } + + /// `dpe.exec(lhs, rhs -> [to])`: shared axes missing from `to` are + /// contracted. Accumulates in f32 (float inputs) or i32. + pub fn contract(self: *TensorOperation, lhs: anytype, rhs: anytype, to: []const Axis) Value { + if (self.has_dpe or self.has_ve) std.debug.panic("tcl: one contraction per tensor operation, before VE instructions", .{}); + const l = self.lift(lhs, .f32); + const r = self.lift(rhs, .f32); + const dtype: DType = if (l.dtype.isFloat()) .f32 else .i32; + self.has_dpe = true; + const op = tcl.dpe(self.b.ctx, l.inner, r.inner, self.b.logicalType(dtype, to), self.b.loc()); + return self.emit(op, dtype, self.b.dupeAxes(to), false); + } + + /// `ve.exec(...)` of a unary or binary instruction. Operands are + /// `Value`s, `Tensor`s (read on use) or scalar literals. + pub fn ve(self: *TensorOperation, comptime opcode: VeOp, operands: anytype) Value { + if (comptime operands.len != opcode.arity()) @compileError("tcl: wrong operand count for " ++ @tagName(opcode)); + return self.veNamed(@tagName(opcode), opcode.result(), operands); + } + + /// `ve.exec(x to_f)`: Q-format fixed point to f32, Python's + /// `operation.to_fp`. + pub fn toFp(self: *TensorOperation, x: anytype, int_width: u5) Value { + var buf: [8]u8 = undefined; + return self.veNamed(std.fmt.bufPrint(&buf, "to_f{d}", .{int_width}) catch unreachable, .f32, .{x}); + } + + /// `ve.exec(x to_i)`: f32 to Q-format fixed point, Python's + /// `operation.to_fxp`. + pub fn toFxp(self: *TensorOperation, x: anytype, int_width: u5) Value { + var buf: [8]u8 = undefined; + return self.veNamed(std.fmt.bufPrint(&buf, "to_i{d}", .{int_width}) catch unreachable, .i32, .{x}); + } + + fn veNamed(self: *TensorOperation, opcode: []const u8, scalar: DType, operands: anytype) Value { + var values: [2]*const mlir.Value = undefined; + var axes: []const Axis = &.{}; + inline for (operands, 0..) |o, i| { + const v = self.lift(o, scalar); + values[i] = v.inner; + axes = self.b.unionAxes(axes, v.axes); + } + self.has_ve = true; + const ctx = self.b.ctx; + const code = tcl.veOpcodeAttr(ctx, opcode) catch std.debug.panic("tcl: unknown VE instruction {s}", .{opcode}); + const op = tcl.ve(ctx, values[0..operands.len], self.b.logicalType(scalar, axes), code, self.b.loc()); + return self.emit(op, scalar, axes, false); + } + + /// `ve.exec( t[axes] x)`: removes `axes`. + pub fn reduce(self: *TensorOperation, x: anytype, axes: []const Axis, mode: ReduceMode) Value { + if (mode == .cumsum) return self.cumsum(x, axes); + const v = self.lift(x, .f32); + var kept: std.ArrayList(Axis) = .empty; + for (v.axes) |a| { + if (!containsAxis(axes, a)) kept.append(self.b.arena.allocator(), a) catch @panic("OOM"); + } + self.has_ve = true; + self.has_reduce = true; + const op = tcl.ve_reduce(self.b.ctx, v.inner, self.b.logicalType(v.dtype, kept.items), mode, self.b.axisAttrs(axes), self.b.loc()); + return self.emit(op, v.dtype, kept.items, false); + } + + /// `cumsum t[axes] x`, like `operation.cumsum`: the running sum of an i32 + /// value along `axes`, flattened in that order. + pub fn cumsum(self: *TensorOperation, x: anytype, axes: []const Axis) Value { + const v = self.lift(x, .i32); + self.has_ve = true; + self.has_reduce = true; + const op = tcl.ve_reduce(self.b.ctx, v.inner, self.b.logicalType(v.dtype, v.axes), .cumsum, self.b.axisAttrs(axes), self.b.loc()); + return self.emit(op, v.dtype, v.axes, false); + } + + /// `ve.exec(if cond threshold { yes } else { no })`, like + /// `operation.where(cond threshold, yes, no)`. + pub fn where(self: *TensorOperation, cond: anytype, pred: Predicate, threshold: anytype, yes: anytype, no: anytype) Value { + const ctx = self.b.ctx; + const c = self.lift(cond, .f32); + const float = isFloatOperand(yes) or isFloatOperand(no); + const scalar: DType = if (float) .f32 else .i32; + const t = self.lift(yes, scalar); + const f = self.lift(no, scalar); + const threshold_attr: *const mlir.Attribute = if (c.dtype.isFloat()) + .float(ctx, .f32, threshold) + else + .int(ctx, .i32, threshold); + const axes = self.b.unionAxes(self.b.unionAxes(c.axes, t.axes), f.axes); + self.has_ve = true; + self.has_select = true; + const op = tcl.ve_select(ctx, c.inner, t.inner, f.inner, self.b.logicalType(scalar, axes), pred, threshold_attr, self.b.loc()); + return self.emit(op, scalar, axes, false); + } + + /// `write.to(...)`: closes the tensor operation and returns its result. + pub fn commit(self: *TensorOperation, v: Value, opts: CommitOptions) FinishError!Tensor { + const b = self.b; + if (self.last != v.inner) std.debug.panic("tcl: {s}: commit must take the last instruction", .{b.name}); + const axes = if (opts.axes) |a| b.dupeAxes(a) else v.axes; + if (!sameAxisSet(axes, v.axes)) std.debug.panic("tcl: {s}: commit can only permute axes", .{b.name}); + const dtype = opts.dtype orelse v.dtype; + _ = tcl.write(b.ctx, v.inner, b.loc()).appendTo(self.body); + + const inputs = try b.arena.allocator().alloc(*const mlir.Value, self.inputs.items.len); + for (inputs, self.inputs.items) |*dst, t| dst.* = t.inner; + const context = if (self.context) |c| b.contextAttr(c) else null; + const kernel = tcl.kernel(b.ctx, inputs, b.logicalType(dtype, axes), self.tactic orelse self.guessTactic(axes), self.body, .{ .context = context }, b.loc()); + try b.append(kernel); + return .{ .inner = kernel.result(0), .dtype = dtype, .axes = axes }; + } + + /// Python's `BuilderHistory.guess_tactic_kind`. + fn guessTactic(self: *const TensorOperation, output: []const Axis) Tactic { + if (self.has_dpe) return .einsum_by_dpe; + if (!self.has_ve) return .tensor_operation; + if (self.has_reduce) return .reduce_by_ve; + const in = self.inputs.items; + if (in.len > 0 and in[0].axes.len < output.len) return .einsum_by_ve; + if (!self.has_select and in.len == 2 and sameAxisSet(in[0].axes, in[1].axes)) return .interleaving; + return .elementwise; + } + + const Operand = struct { inner: *const mlir.Value, dtype: DType, axes: []const Axis }; + + fn lift(self: *TensorOperation, x: anytype, scalar: DType) Operand { + const T = @TypeOf(x); + if (T == Value) return .{ .inner = x.inner, .dtype = x.dtype, .axes = x.axes }; + if (T == Tensor) { + const r = self.plainRead(x); + return .{ .inner = r.inner, .dtype = r.dtype, .axes = r.axes }; + } + const ctx = self.b.ctx; + const op = switch (@typeInfo(T)) { + .comptime_int, .int => if (scalar == .f32) + arith.constant_float(ctx, @floatFromInt(x), .f32, self.b.loc()) + else + arith.constant_int(ctx, x, .int(ctx, .i32), self.b.loc()), + .comptime_float, .float => blk: { + if (scalar != .f32) std.debug.panic("tcl: float literal for an integer VE instruction", .{}); + break :blk arith.constant_float(ctx, x, .f32, self.b.loc()); + }, + else => @compileError("tcl: expected a Value, a Tensor or a scalar, got " ++ @typeName(T)), + }; + _ = op.appendTo(self.body); + return .{ .inner = op.result(0), .dtype = scalar, .axes = &.{} }; + } + + /// Registers `t` as a kernel operand. + fn input(self: *TensorOperation, t: Tensor) *const mlir.Value { + if (t.inner.type_().isA(tcl.LogicalType) == null) + std.debug.panic("tcl: {s}: tensor operations read logical tensors, not DRAM ones", .{self.b.name}); + for (self.inputs.items) |known| { + if (known.inner == t.inner) return t.inner; + } + self.inputs.append(self.b.arena.allocator(), t) catch @panic("OOM"); + return t.inner; + } + + fn plainRead(self: *TensorOperation, t: Tensor) Value { + for (self.plain_reads.items) |r| if (r.tensor == t.inner) return r.value; + const r = self.fetch(t, .{}); + self.plain_reads.append(self.b.arena.allocator(), .{ .tensor = t.inner, .value = r }) catch @panic("OOM"); + return r; + } + + /// Reads go before every other instruction, as TCL lists them. + fn emit(self: *TensorOperation, op: *mlir.Operation, dtype: DType, axes: []const Axis, is_read: bool) Value { + if (is_read and self.first_compute != null) { + self.body.insertOwnedOperationBefore(self.first_compute.?, op); + } else { + _ = op.appendTo(self.body); + if (!is_read and self.first_compute == null) self.first_compute = op; + self.last = op.result(0); + } + return .{ .op = self, .inner = op.result(0), .dtype = dtype, .axes = axes }; + } +}; + +fn isFloatOperand(x: anytype) bool { + return switch (@typeInfo(@TypeOf(x))) { + .comptime_float, .float => true, + .comptime_int, .int => false, + else => x.dtype.isFloat(), + }; +} + +fn containsAxis(axes: []const Axis, a: Axis) bool { + for (axes) |x| if (std.mem.eql(u8, x.name, a.name)) return true; + return false; +} + +fn sameAxisSet(a: []const Axis, b: []const Axis) bool { + if (a.len != b.len) return false; + for (a) |x| if (!containsAxis(b, x)) return false; + return true; +} + +/// Builds one TCL function, `def name(axes..., args...) -> results:`, as a +/// `func.func` of the native `tcl` dialect. +pub const Builder = struct { + allocator: std.mem.Allocator, + arena: std.heap.ArenaAllocator, + ctx: *mlir.Context, + module: *mlir.Module, + name: []const u8, + axes: std.ArrayList(Declared) = .empty, + compiler_config: ?*const mlir.Attribute = null, + auto_config: ?*const mlir.Attribute = null, + fusible: bool = false, + func_op: ?*mlir.Operation = null, + entry: ?*mlir.Block = null, + args: []const Tensor = &.{}, + results: ?[]const Tensor = null, + /// Arguments `update` writes back into, by argument number. + updated: std.ArrayList(usize) = .empty, + /// Blocks being built inside the entry block (loop bodies), innermost last. + block_stack: common.BlockStack = .{}, + + pub fn open(allocator: std.mem.Allocator, ctx: *mlir.Context, name: []const u8) !Builder { + _ = tcl.symbolAttr(ctx, name) catch std.debug.panic("tcl: `{s}` is not a TCL function name", .{name}); + return .{ + .allocator = allocator, + .arena = .init(allocator), + .ctx = ctx, + .module = .init(.unknown(ctx)), + .name = name, + }; + } + + pub fn deinit(self: *Builder) void { + self.module.deinit(); + self.arena.deinit(); + } + + pub fn loc(self: *const Builder) *const mlir.Location { + return .unknown(self.ctx); + } + + const Declared = struct { axis: Axis, def: ?Expr }; + + /// `A: Axis = size`, where `size` is an integer or an `Expr` over other + /// axes (`b.axis("C", b.expr(.mul, A, B))`). Axis names start with an + /// uppercase letter. + pub fn axis(self: *Builder, name: []const u8, size: anytype) Axis { + const def: ?Expr, const n: i64 = if (comptime @TypeOf(size) == Expr) + .{ size, size.value() orelse std.debug.panic("tcl: axis {s} needs an integer size", .{name}) } + else + .{ null, size }; + if (name.len == 0 or !std.ascii.isUpper(name[0]) or n <= 0) + std.debug.panic("tcl: invalid axis {s} = {d}", .{ name, n }); + _ = tcl.symbolAttr(self.ctx, name) catch std.debug.panic("tcl: invalid axis name {s}", .{name}); + for (self.axes.items) |d| if (std.mem.eql(u8, d.axis.name, name)) { + if (d.axis.size != n) std.debug.panic("tcl: axis {s} redeclared as {d} (was {d})", .{ name, n, d.axis.size }); + return d.axis; + }; + const a: Axis = .{ .name = self.arena.allocator().dupe(u8, name) catch @panic("OOM"), .size = n }; + self.axes.append(self.arena.allocator(), .{ .axis = a, .def = def }) catch @panic("OOM"); + return a; + } + + /// An `Expr` node: `lhs` and `rhs` are axes, integers or expressions. + pub fn expr(self: *Builder, kind: Expr.Kind, lhs: anytype, rhs: anytype) Expr { + const node = self.arena.allocator().create(Expr.Node) catch @panic("OOM"); + node.* = .{ .kind = kind, .lhs = toExpr(lhs), .rhs = toExpr(rhs) }; + return .{ .node = node }; + } + + fn toExpr(x: anytype) Expr { + return switch (@TypeOf(x)) { + Expr => x, + Axis => .{ .axis = x }, + else => .{ .int = x }, + }; + } + + /// `@compiler_config({...})`, Python's `@tcl.kernel(config=...)`: an + /// anonymous struct written out as JSON, `.{ .lowering_mode = "Heuristic" }`. + pub fn compilerConfig(self: *Builder, config: anytype) void { + self.compiler_config = tcl.configAttr(self.ctx, self.jsonFields(config)) catch + std.debug.panic("tcl: {s}: invalid compiler config", .{self.name}); + } + + /// Lets the plugin compile the function into the unit of the work around + /// it instead of a unit of its own; the unit's compiler config applies. + pub fn fuse(self: *Builder) void { + self.fusible = true; + } + + /// `@auto_config({...})`, Python's `@tcl.kernel(auto_config=...)`. + pub fn autoConfig(self: *Builder, config: anytype) void { + self.auto_config = tcl.configAttr(self.ctx, self.jsonFields(config)) catch + std.debug.panic("tcl: {s}: invalid auto config", .{self.name}); + } + + /// One field per argument: `.{ .x = .{ .dtype = .bf16, .axes = &.{ M, K } } }`. + /// With `.dram = Dram{...}` (or `.{}`) the argument is DRAM-mapped and the + /// field is its `as_logical` view. + pub fn declareArgs(self: *Builder, spec: anytype) FinishError!ArgsOf(@TypeOf(spec)) { + std.debug.assert(self.entry == null); + const fields = @typeInfo(@TypeOf(spec)).@"struct".field_names; + const scratch = self.arena.allocator(); + var types: [fields.len]*const mlir.Type = undefined; + var locs: [fields.len]*const mlir.Location = undefined; + const args = try scratch.alloc(Tensor, fields.len); + inline for (fields, 0..) |f, i| { + const s = @field(spec, f); + const axes = self.dupeAxes(s.axes); + const dram: ?Dram = if (@hasField(@TypeOf(s), "dram")) toDram(s.dram) else null; + types[i] = if (dram) |d| self.mappedType(s.dtype, axes, d) else self.logicalType(s.dtype, axes); + locs[i] = self.loc(); + args[i] = .{ .inner = undefined, .dtype = s.dtype, .axes = axes, .shards = if (dram) |d| d.shards() else 1 }; + } + const entry = mlir.Block.init(&types, &locs); + // In place from the start: the kernel verifier walks to its inputs' + // regions. `finish` sets the function type. + const func_op = func.func(self.ctx, .{ .name = self.name, .block = entry, .results = &.{}, .location = self.loc(), .visibility = null, .verify = false }); + _ = func_op.appendTo(self.module.body()); + self.func_op = func_op; + self.entry = entry; + self.block_stack.push(self.arena.allocator(), entry); + var named: ArgsOf(@TypeOf(spec)) = undefined; + inline for (fields, 0..) |f, i| { + args[i].inner = entry.argument(i); + if (@hasField(@TypeOf(@field(spec, f)), "dram")) + args[i].inner = try self.graphOp(.as_logical, &.{entry.argument(i)}, self.logicalType(args[i].dtype, args[i].axes), &.{}, null); + @field(named, f) = args[i]; + } + self.args = args; + return named; + } + + fn toDram(d: anytype) Dram { + const D = @TypeOf(d); + if (D == Dram) return d; + return .{ + .chip = if (@hasField(D, "chip")) d.chip else &.{.broadcast}, + .inner = if (@hasField(D, "inner")) d.inner else null, + .original = if (@hasField(D, "original")) d.original else &.{}, + }; + } + + fn ArgsOf(comptime Spec: type) type { + const in = @typeInfo(Spec).@"struct".field_names; + comptime var names: [in.len][]const u8 = undefined; + inline for (in, 0..) |f, i| names[i] = f; + return @Struct(.auto, null, &names, &@splat(Tensor), &@splat(.{})); + } + + /// Opens a `tk.(...)` tensor operation; `commit` closes it. + pub fn tensorOperation(self: *Builder, opts: TensorOperation.Options) *TensorOperation { + _ = self.entryBlock(); + const op = self.arena.allocator().create(TensorOperation) catch @panic("OOM"); + op.* = .{ .b = self, .tactic = opts.tactic, .context = opts.context, .body = mlir.Block.init(&.{}, &.{}) }; + return op; + } + + pub const GraphOptions = struct { context: ?Context = null }; + + /// `reshape(t)` to `to`, same element count. + pub fn reshape(self: *Builder, t: Tensor, to: []const Axis, opts: GraphOptions) FinishError!Tensor { + if (elements(t.axes) != elements(to)) std.debug.panic("tcl: reshape changes the element count", .{}); + return self.graph(.reshape, &.{t}, t.dtype, to, &.{}, opts.context); + } + + /// `transmute(t)`: the same bits as `dtype` over `to`. + pub fn transmute(self: *Builder, t: Tensor, dtype: DType, to: []const Axis, opts: GraphOptions) FinishError!Tensor { + return self.graph(.transmute, &.{t}, dtype, to, &.{}, opts.context); + } + + /// `concat([inputs])` along the one axis of `to` the inputs lack. + pub fn concat(self: *Builder, inputs: []const Tensor, to: []const Axis, opts: GraphOptions) FinishError!Tensor { + return self.graph(.concat, inputs, inputs[0].dtype, to, &.{}, opts.context); + } + + /// `slice(t, axis, offset)`: the window of `to` starting at `offset` along `axis`. + pub fn slice(self: *Builder, t: Tensor, along: Axis, offset: i64, to: []const Axis, opts: GraphOptions) FinishError!Tensor { + return self.graph(.slice, &.{t}, t.dtype, to, &.{ + .named(self.ctx, "axis", self.axisAttr(along)), + .named(self.ctx, "offset", .int(self.ctx, .i64, offset)), + }, opts.context); + } + + pub const GatherOptions = struct { + batch_axis: ?Axis = null, + valid_length: ?Tensor = null, + context: ?Context = null, + }; + + /// `gather(table, indices, axis)`: rows of `table` along `axis`. + pub fn gather(self: *Builder, table: Tensor, indices: Tensor, along: Axis, to: []const Axis, opts: GatherOptions) FinishError!Tensor { + var options: [2]mlir.NamedAttribute = .{ .named(self.ctx, "axis", self.axisAttr(along)), undefined }; + if (opts.batch_axis) |a| options[1] = .named(self.ctx, "batch_axis", self.axisAttr(a)); + var inputs: [3]Tensor = .{ table, indices, opts.valid_length orelse undefined }; + const n_inputs = @as(usize, 2) + @intFromBool(opts.valid_length != null); + return self.graph(.gather, inputs[0..n_inputs], table.dtype, to, options[0 .. @as(usize, 1) + @intFromBool(opts.batch_axis != null)], opts.context); + } + + pub const ScatterOptions = struct { + /// The tensor scattered into; zeros otherwise. + init: ?Tensor = null, + batch_axis: ?Axis = null, + context: ?Context = null, + }; + + /// `scatter(updates, indices, axis, init=...)`: `updates` written at + /// `indices` along `axis` of the result. + pub fn scatter(self: *Builder, updates: Tensor, indices: Tensor, along: Axis, to: []const Axis, opts: ScatterOptions) FinishError!Tensor { + var options: [2]mlir.NamedAttribute = .{ .named(self.ctx, "axis", self.axisAttr(along)), undefined }; + if (opts.batch_axis) |a| options[1] = .named(self.ctx, "batch_axis", self.axisAttr(a)); + var inputs: [3]Tensor = .{ updates, indices, opts.init orelse undefined }; + const n_inputs = @as(usize, 2) + @intFromBool(opts.init != null); + return self.graph(.scatter, inputs[0..n_inputs], updates.dtype, to, options[0 .. @as(usize, 1) + @intFromBool(opts.batch_axis != null)], opts.context); + } + + pub const ArangeOptions = struct { + start: ?i64 = null, + step: ?i64 = null, + context: ?Context = null, + }; + + /// `arange(start=, end, step=)` over the one axis of `to`. + pub fn arange(self: *Builder, end: i64, dtype: DType, to: []const Axis, opts: ArangeOptions) FinishError!Tensor { + var options: [3]mlir.NamedAttribute = undefined; + var n: usize = 0; + if (opts.start) |v| { + options[n] = .named(self.ctx, "start", .int(self.ctx, .i64, v)); + n += 1; + } + options[n] = .named(self.ctx, "end", .int(self.ctx, .i64, end)); + n += 1; + if (opts.step) |v| { + options[n] = .named(self.ctx, "step", .int(self.ctx, .i64, v)); + n += 1; + } + return self.graph(.arange, &.{}, dtype, to, options[0..n], opts.context); + } + + pub const VectorOptions = struct { + /// Repeats `values` cyclically along this axis. + repeat_to: ?Axis = null, + context: ?Context = null, + }; + + /// `vector([values], repeat_to=)`: a constant tensor. + pub fn vector(self: *Builder, values: []const f64, dtype: DType, to: []const Axis, opts: VectorOptions) FinishError!Tensor { + const attrs = try self.arena.allocator().alloc(*const mlir.Attribute, values.len); + for (attrs, values) |*dst, v| dst.* = if (dtype.isFloat()) .float(self.ctx, .f64, v) else .int(self.ctx, .i64, @as(i64, @intFromFloat(v))); + var options: [2]mlir.NamedAttribute = .{ .named(self.ctx, "values", .array(self.ctx, attrs)), undefined }; + if (opts.repeat_to) |a| options[1] = .named(self.ctx, "repeat_to", self.axisAttr(a)); + return self.graph(.vector, &.{}, dtype, to, options[0 .. @as(usize, 1) + @intFromBool(opts.repeat_to != null)], opts.context); + } + + /// `all_gather(t, axis)` across chips. + pub fn allGather(self: *Builder, t: Tensor, along: Axis, opts: GraphOptions) FinishError!Tensor { + return self.graph(.all_gather, &.{t}, t.dtype, t.axes, &.{ + .named(self.ctx, "axis", self.axisAttr(along)), + }, opts.context); + } + + /// `as_dram(t)`: `t` in the DRAM mapping `dram`, to return it as such. + pub fn asDram(self: *Builder, t: Tensor, dram: Dram) FinishError!Tensor { + const out = try self.graphOp(.as_dram, &.{t.inner}, self.mappedType(t.dtype, t.axes, dram), &.{}, null); + return .{ .inner = out, .dtype = t.dtype, .axes = t.axes, .shards = dram.shards() }; + } + + /// Any `tcl.graph.*` operation; `options` are its TCL keyword fields. + pub fn graph(self: *Builder, comptime op: tcl.GraphOp, inputs: []const Tensor, dtype: DType, to: []const Axis, options: []const mlir.NamedAttribute, context: ?Context) FinishError!Tensor { + const values = try self.arena.allocator().alloc(*const mlir.Value, inputs.len); + for (values, inputs) |*v, t| v.* = t.inner; + const axes = self.dupeAxes(to); + const out = try self.graphOp(op, values, self.logicalType(dtype, axes), options, context); + return .{ .inner = out, .dtype = dtype, .axes = axes }; + } + + fn graphOp(self: *Builder, comptime op: tcl.GraphOp, inputs: []const *const mlir.Value, output: *const mlir.Type, options: []const mlir.NamedAttribute, context: ?Context) FinishError!*const mlir.Value { + const g = tcl.graph(self.ctx, op, inputs, output, .{ .options = options, .context = if (context) |c| self.contextAttr(c) else null }, self.loc()); + try self.append(g); + return g.result(0); + } + + /// `for iv in 0..limit (acc = init, ...):`, closed by `yield`. + pub fn ForScope(comptime N: usize) type { + return struct { + b: *Builder, + op: *mlir.Operation, + body: *mlir.Block, + /// The loop index, a scalar i32 tensor for `indexRead`, + /// `indexWrite` and `symExpr`. + iv: Tensor, + carried: [N]Tensor, + results: [N]Tensor = undefined, + + const Self = @This(); + + /// `yield values;` closes the loop and sets `results`. + pub fn yield(self: *Self, values: [N]Tensor) FinishError!void { + const b = self.b; + var inner: [N]*const mlir.Value = undefined; + for (&inner, values) |*v, t| v.* = t.inner; + _ = tcl.yield(b.ctx, &inner, b.loc()).appendTo(self.body); + b.popBlock(); + if (!self.op.verify()) { + std.log.err("tcl: {s}: invalid loop:\n{f}", .{ b.name, self.op.fmt(.{ .print_generic_op_form = true }) }); + return error.InvalidMlir; + } + for (&self.results, self.carried, 0..) |*r, c, i| r.* = .{ .inner = self.op.result(i), .dtype = c.dtype, .axes = c.axes, .shards = c.shards }; + } + }; + } + + /// Opens a device loop from 0 to `limit`: an `Axis`, an integer, or a + /// scalar i32 `Tensor` computed in the function (`reduceMaxI32`, + /// `symExpr`). The body must index a tensor with `iv`, which bounds the + /// trip count. `inits` is a tuple of the accumulators' initial tensors. + pub fn openFor(self: *Builder, limit: anytype, inits: anytype) ForScope(@typeInfo(@TypeOf(inits)).@"struct".field_names.len) { + const N = @typeInfo(@TypeOf(inits)).@"struct".field_names.len; + var types: [N + 1]*const mlir.Type = undefined; + var locs: [N + 1]*const mlir.Location = @splat(self.loc()); + var values: [N]*const mlir.Value = undefined; + var carried: [N]Tensor = undefined; + types[0] = self.logicalType(.i32, &.{}); + inline for (0..N) |i| { + carried[i] = inits[i]; + values[i] = inits[i].inner; + types[i + 1] = inits[i].inner.type_(); + } + const body = mlir.Block.init(&types, &locs); + const tensor_limit: ?*const mlir.Value, const bound: ?*const mlir.Attribute = switch (@TypeOf(limit)) { + Tensor => .{ limit.inner, null }, + Axis => .{ null, self.axisAttr(limit) }, + else => .{ null, .int(self.ctx, .i64, limit) }, + }; + // Attached before its body is built: the index operations' verifiers + // look for their loop. + const op = tcl.loop(self.ctx, &values, types[1..], body, .{ .limit = tensor_limit, .bound = bound }, self.loc()); + _ = op.appendTo(self.currentBlock()); + self.pushBlock(body); + for (&carried, 0..) |*c, i| c.inner = body.argument(i + 1); + return .{ .b = self, .op = op, .body = body, .iv = .{ .inner = body.argument(0), .dtype = .i32, .axes = &.{} }, .carried = carried }; + } + + /// `t[i]`: the slice of `t`'s outermost axis at loop index `i`. + pub fn indexRead(self: *Builder, t: Tensor, i: Tensor) FinishError!Tensor { + return self.graph(.index_read, &.{ t, i }, t.dtype, t.axes[1..], &.{}, null); + } + + /// `acc[i] = v`, in place: `acc` must have no other use; use the result. + pub fn indexWrite(self: *Builder, acc: Tensor, i: Tensor, v: Tensor) FinishError!Tensor { + return self.graph(.index_write, &.{ acc, i, v }, acc.dtype, acc.axes, &.{}, null); + } + + /// `scratchpad()`: an uninitialized tensor, typically a loop's output that + /// `indexWrite` fills. + pub fn scratchpad(self: *Builder, dtype: DType, to: []const Axis) FinishError!Tensor { + return self.graph(.scratchpad, &.{}, dtype, to, &.{}, null); + } + + /// `full(value)`: a constant tensor, like `tcl.full`. + pub fn full(self: *Builder, value: f64, dtype: DType, to: []const Axis) FinishError!Tensor { + const v: *const mlir.Attribute = if (dtype.isFloat()) .float(self.ctx, .f64, value) else .int(self.ctx, .i64, @as(i64, @intFromFloat(value))); + return self.graph(.full, &.{}, dtype, to, &.{.named(self.ctx, "value", v)}, null); + } + + /// `reduce_max_i32(t)`: the maximum of an i32 tensor, as a scalar. + pub fn reduceMaxI32(self: *Builder, t: Tensor) FinishError!Tensor { + return self.graph(.reduce_max_i32, &.{t}, .i32, &.{}, &.{}, null); + } + + /// `sym_expr(lhs rhs)`: scalar i32 arithmetic on the device, over + /// scalar tensors (a loop index, `reduceMaxI32`) and integers. + pub fn symExpr(self: *Builder, kind: Expr.Kind, lhs: anytype, rhs: anytype) FinishError!Tensor { + var inputs: [2]Tensor = undefined; + var n: usize = 0; + const l = self.symOperand(lhs, &inputs, &n); + const r = self.symOperand(rhs, &inputs, &n); + const e = tcl.exprAttr(self.ctx, @tagName(kind), &.{ l, r }) catch + std.debug.panic("tcl: {s}: invalid {s} expression", .{ self.name, @tagName(kind) }); + return self.graph(.sym_expr, inputs[0..n], .i32, &.{}, &.{.named(self.ctx, "expr", e)}, null); + } + + fn symOperand(self: *Builder, x: anytype, inputs: *[2]Tensor, n: *usize) *const mlir.Attribute { + if (@TypeOf(x) != Tensor) return .int(self.ctx, .i64, x); + inputs[n.*] = x; + n.* += 1; + return tcl.exprAttr(self.ctx, "arg", &.{.int(self.ctx, .i64, @as(i64, @intCast(n.* - 1)))}) catch unreachable; + } + + /// `copy(src=value, dst=argument)`: `value` written back into the + /// DRAM-mapped argument `index` in place. Return the result like any + /// other: the caller sees it as an output aliasing that argument. + pub fn update(self: *Builder, index: usize, value: Tensor) FinishError!Tensor { + const arg = self.entryBlock().argument(index); + const back = try self.graphOp(.as_dram, &.{value.inner}, arg.type_(), &.{}, null); + try self.append(tcl.copy(self.ctx, back, arg, self.loc())); + self.updated.append(self.arena.allocator(), index) catch @panic("OOM"); + return .{ .inner = arg, .dtype = value.dtype, .axes = value.axes, .shards = self.args[index].shards }; + } + + /// The argument each result updates in place, or null for results the + /// function returns. + pub fn updatedArgument(self: *const Builder, result: Tensor) ?usize { + for (self.updated.items) |i| if (self.entryBlock().argument(i) == result.inner) return i; + return null; + } + + /// `return results...`. + pub fn ret(self: *Builder, results: []const Tensor) void { + self.results = self.arena.allocator().dupe(Tensor, results) catch @panic("OOM"); + } + + /// The verified module: one `func.func` carrying its axes in `tcl.axes`. + pub fn finish(self: *Builder) FinishError![:0]const u8 { + const ctx = self.ctx; + const f = self.func_op orelse return error.InvalidMlir; + const results = self.results orelse return error.InvalidMlir; + const scratch = self.arena.allocator(); + var values: std.ArrayList(*const mlir.Value) = .empty; + var result_types: std.ArrayList(*const mlir.Type) = .empty; + const updated_inputs = try scratch.alloc(i64, results.len); + for (updated_inputs, results) |*u, r| { + if (self.updatedArgument(r)) |i| { + u.* = @intCast(i); + continue; + } + u.* = -1; + try values.append(scratch, r.inner); + try result_types.append(scratch, r.inner.type_()); + } + _ = func.returns(ctx, values.items, self.loc()).appendTo(self.entry.?); + + const entry = self.entry.?; + const arg_types = try scratch.alloc(*const mlir.Type, entry.numArguments()); + for (arg_types, 0..) |*t, i| t.* = entry.argument(i).type_(); + f.setAttributeByName("function_type", .typeAttr(.function(ctx, arg_types, result_types.items))); + + const axes = try scratch.alloc(mlir.NamedAttribute, self.axes.items.len); + for (axes, self.axes.items) |*dst, d| { + dst.* = .named(ctx, d.axis.name, if (d.def) |e| self.exprAttr(e) else .int(ctx, .i64, d.axis.size)); + } + tcl.annotateFunction(ctx, f, axes, .{ + .updated_inputs = if (self.updated.items.len > 0) updated_inputs else null, + .compiler_config = self.compiler_config, + .auto_config = self.auto_config, + .fusible = self.fusible, + }); + try common.verify(self.module, "dsl/tcl"); + + var printed: std.Io.Writer.Allocating = .init(self.allocator); + defer printed.deinit(); + try printed.writer.print("{f}", .{self.module.operation()}); + return try self.allocator.dupeSentinel(u8, printed.written(), 0); + } + + fn append(self: *Builder, op: *mlir.Operation) FinishError!void { + _ = op.appendTo(self.currentBlock()); + if (!op.verify()) { + std.log.err("tcl: {s}: invalid operation:\n{f}", .{ self.name, op.fmt(.{ .print_generic_op_form = true }) }); + return error.InvalidMlir; + } + } + + fn entryBlock(self: *const Builder) *mlir.Block { + return self.entry orelse std.debug.panic("tcl: {s}: declareArgs first", .{self.name}); + } + + pub fn pushBlock(self: *Builder, block: *mlir.Block) void { + self.block_stack.push(self.arena.allocator(), block); + } + + pub fn popBlock(self: *Builder) void { + self.block_stack.pop(); + } + + pub fn currentBlock(self: *const Builder) *mlir.Block { + return self.block_stack.current(); + } + + pub fn axisAttr(self: *const Builder, a: Axis) *const mlir.Attribute { + return tcl.symbolAttr(self.ctx, a.name) catch unreachable; + } + + fn axisAttrs(self: *Builder, axes: []const Axis) []const *const mlir.Attribute { + const attrs = self.arena.allocator().alloc(*const mlir.Attribute, axes.len) catch @panic("OOM"); + for (attrs, axes) |*dst, a| dst.* = self.axisAttr(a); + return attrs; + } + + fn axesAttr(self: *Builder, axes: []const Axis) *const mlir.Attribute { + return .array(self.ctx, self.axisAttrs(axes)); + } + + pub fn exprAttr(self: *Builder, e: Expr) *const mlir.Attribute { + return switch (e) { + .int => |n| .int(self.ctx, .i64, n), + .axis => |a| self.axisAttr(a), + .broadcast => tcl.exprAttr(self.ctx, "broadcast", &.{}) catch unreachable, + .node => |n| tcl.exprAttr(self.ctx, @tagName(n.kind), &.{ self.exprAttr(n.lhs), self.exprAttr(n.rhs) }) catch + std.debug.panic("tcl: {s}: invalid {s} expression", .{ self.name, @tagName(n.kind) }), + }; + } + + fn exprAttrs(self: *Builder, exprs: []const Expr) []const *const mlir.Attribute { + const attrs = self.arena.allocator().alloc(*const mlir.Attribute, exprs.len) catch @panic("OOM"); + for (attrs, exprs) |*dst, e| dst.* = self.exprAttr(e); + return attrs; + } + + pub fn contextAttr(self: *Builder, c: Context) *const mlir.Attribute { + const ctx = self.ctx; + var fields: [2]mlir.NamedAttribute = undefined; + var n: usize = 0; + if (c.layout) |l| { + var layout: [3]mlir.NamedAttribute = undefined; + var m: usize = 0; + layout[m] = .named(ctx, "Chip", self.exprAttr(l.chip)); + m += 1; + if (l.cluster) |cl| { + layout[m] = .named(ctx, "Cluster", self.exprAttr(cl)); + m += 1; + } + if (l.split.len > 0) { + layout[m] = .named(ctx, "Split", self.axesAttr(l.split)); + m += 1; + } + fields[n] = .named(ctx, "operator", .dict(ctx, layout[0..m])); + n += 1; + } + if (c.heuristic_hint.len > 0) { + const hints = self.arena.allocator().alloc(mlir.NamedAttribute, c.heuristic_hint.len) catch @panic("OOM"); + for (hints, c.heuristic_hint) |*dst, h| dst.* = .named(ctx, h.name, switch (h.value) { + .int => |v| .int(ctx, .i64, v), + .float => |v| .float(ctx, .f64, v), + .boolean => |v| .boolean(ctx, v), + .axis => |a| self.axisAttr(a), + }); + fields[n] = .named(ctx, "heuristic_hint", .dict(ctx, hints)); + n += 1; + } + return tcl.contextAttr(ctx, fields[0..n]) catch std.debug.panic("tcl: {s}: invalid context", .{self.name}); + } + + fn jsonFields(self: *Builder, config: anytype) []const mlir.NamedAttribute { + const fields = @typeInfo(@TypeOf(config)).@"struct".field_names; + const out = self.arena.allocator().alloc(mlir.NamedAttribute, fields.len) catch @panic("OOM"); + inline for (fields, 0..) |f, i| out[i] = .named(self.ctx, f, self.json(@field(config, f))); + return out; + } + + fn json(self: *Builder, v: anytype) *const mlir.Attribute { + const T = @TypeOf(v); + return switch (@typeInfo(T)) { + .bool => .boolean(self.ctx, v), + .comptime_int, .int => .int(self.ctx, .i64, v), + .comptime_float, .float => .float(self.ctx, .f64, v), + .enum_literal, .@"enum" => .string(self.ctx, @tagName(v)), + .@"struct" => |info| if (info.is_tuple) blk: { + var items: [info.field_names.len]*const mlir.Attribute = undefined; + inline for (info.field_names, 0..) |f, i| items[i] = self.json(@field(v, f)); + break :blk .array(self.ctx, &items); + } else .dict(self.ctx, self.jsonFields(v)), + .pointer => if (std.meta.Elem(T) == u8) .string(self.ctx, v) else blk: { + const items = self.arena.allocator().alloc(*const mlir.Attribute, v.len) catch @panic("OOM"); + for (items, v) |*dst, x| dst.* = self.json(x); + break :blk .array(self.ctx, items); + }, + else => @compileError("tcl: config values are JSON: bool, numbers, strings, structs, tuples"), + }; + } + + fn mappedType(self: *Builder, dtype: DType, axes: []const Axis, dram: Dram) *const mlir.Type { + for (axes) |a| _ = self.axis(a.name, a.size); + const inner = if (dram.inner) |e| self.exprAttrs(e) else blk: { + var rest: std.ArrayList(Axis) = .empty; + for (axes) |a| if (!dram.isChipAxis(a)) rest.append(self.arena.allocator(), a) catch @panic("OOM"); + break :blk self.axisAttrs(rest.items); + }; + const mapping = tcl.dramMappingAttr(self.ctx, self.exprAttrs(dram.chip), inner, self.axisAttrs(dram.original)) catch + std.debug.panic("tcl: {s}: invalid DRAM mapping", .{self.name}); + return tcl.mappedType(self.ctx, dtype.toMlir(self.ctx), mapping) catch unreachable; + } + + pub fn logicalType(self: *Builder, dtype: DType, axes: []const Axis) *const mlir.Type { + for (axes) |a| _ = self.axis(a.name, a.size); + return tcl.logicalType(self.ctx, dtype.toMlir(self.ctx), self.axisAttrs(axes)) catch + std.debug.panic("tcl: {s}: invalid tensor type (repeated axis?)", .{self.name}); + } + + fn dupeAxes(self: *Builder, axes: []const Axis) []const Axis { + return self.arena.allocator().dupe(Axis, axes) catch @panic("OOM"); + } + + fn unionAxes(self: *Builder, a: []const Axis, b: []const Axis) []const Axis { + var out: std.ArrayList(Axis) = .empty; + const long, const short = if (a.len >= b.len) .{ a, b } else .{ b, a }; + out.appendSlice(self.arena.allocator(), long) catch @panic("OOM"); + for (short) |x| if (!containsAxis(out.items, x)) out.append(self.arena.allocator(), x) catch @panic("OOM"); + return out.items; + } +}; + +fn elements(axes: []const Axis) i64 { + var n: i64 = 1; + for (axes) |a| n *= a.size; + return n; +} + +fn testContext() !*mlir.Context { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + registerDialects(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + return ctx; +} + +fn expectContains(haystack: []const u8, needle: []const u8) !void { + if (std.mem.indexOf(u8, haystack, needle) == null) { + std.debug.print("missing {s} in:\n{s}\n", .{ needle, haystack }); + return error.TestExpectedContains; + } +} + +test "softmax matches furiosa.tcl.examples.softmax" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = try Builder.open(std.testing.allocator, ctx, "softmax_kernel"); + defer b.deinit(); + + const B = b.axis("B", 2); + const S = b.axis("S", 16); + const D = b.axis("D", 1024); + const t = try b.declareArgs(.{ .input = .{ .dtype = .bf16, .axes = &.{ B, S, D } } }); + + const op = b.tensorOperation(.{}); + const x = op.fetch(t.input, .{ .typecast_to = .f32 }); + const centered = x.subf(x.reduce(&.{D}, .maxf)); + const exped = centered.exp(); + const out = try op.commit(exped.divf(exped.reduce(&.{D}, .addf)), .{ .dtype = .bf16 }); + b.ret(&.{out}); + + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + try expectContains(ir, "tcl.axes = {B = 2 : i64, D = 1024 : i64, S = 16 : i64}"); + try expectContains(ir, "options = #tcl.read_options<{typecast_to = f32}>"); + try expectContains(ir, "mode = #tcl.reduce_mode<\"Maxf\">"); + try expectContains(ir, "opcode = #tcl.ve_opcode<\"divf\">"); + try expectContains(ir, "tactic = #tcl.tactic<\"ReduceByVe\">"); + try expectContains(ir, "-> !tcl.logical, #tcl.symbol<\"S\">, #tcl.symbol<\"D\">]>"); + const parsed = try mlir.Module.parse(ctx, ir); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "batched matmul, scalar operands and graph reshape" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = try Builder.open(std.testing.allocator, ctx, "bmm_scaled"); + defer b.deinit(); + + const B = b.axis("B", 2); + const M = b.axis("M", 128); + const K = b.axis("K", 64); + const N = b.axis("N", 256); + const t = try b.declareArgs(.{ + .lhs = .{ .dtype = .bf16, .axes = &.{ B, M, K } }, + .rhs = .{ .dtype = .bf16, .axes = &.{ B, K, N } }, + }); + + const mm = b.tensorOperation(.{}); + const acc = mm.contract(t.lhs, t.rhs, &.{ B, M, N }); + const prod = try mm.commit(acc.mulf(0.5), .{ .dtype = .bf16 }); + const flat = try b.reshape(prod, &.{ B, b.axis("MN", 128 * 256) }, .{}); + b.ret(&.{ prod, flat }); + + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + try expectContains(ir, "tactic = #tcl.tactic<\"EinsumByDpe\">"); + try expectContains(ir, "\"tcl.dpe\""); + try expectContains(ir, "arith.constant 5.000000e-01 : f32"); + try expectContains(ir, "\"tcl.graph.reshape\""); +} + +test "conv1d matches furiosa.kernels.convolution" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = try Builder.open(std.testing.allocator, ctx, "conv"); + defer b.deinit(); + + const N = b.axis("N", 2); + const C = b.axis("C", 64); + const L = b.axis("L", 128); + const K = b.axis("K", 128); + const Lw = b.axis("Lw", 3); + const Lf = b.axis("Lf", 128); + const t = try b.declareArgs(.{ + .inp = .{ .dtype = .bf16, .axes = &.{ N, C, L } }, + .flt = .{ .dtype = .bf16, .axes = &.{ K, C, Lw } }, + .bias = .{ .dtype = .bf16, .axes = &.{K} }, + }); + + const op = b.tensorOperation(.{}); + const windows = op.fetch(t.inp, .{ + .pad = &.{.{ .axis = L, .left = 1, .right = 1 }}, + .slide = &.{.{ .axis = L, .frame_axis = Lf, .window_axis = Lw }}, + }); + try std.testing.expectEqualStrings("Lw", windows.axes[2].name); + const out = try op.commit(op.contract(windows, t.flt, &.{ N, K, Lf }).addf(t.bias), .{ .dtype = .bf16 }); + b.ret(&.{out}); + + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + try expectContains(ir, "slide = {L = {dilation = 1 : i64, frame_axis = #tcl.symbol<\"Lf\">, stride = 1 : i64, undilated_window = 3 : i64, window_axis = #tcl.symbol<\"Lw\">}}"); + try expectContains(ir, "tactic = #tcl.tactic<\"EinsumByDpe\">"); +} + +test "composite axes, DRAM, fixed point, context, config and graph ops" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = try Builder.open(std.testing.allocator, ctx, "features"); + defer b.deinit(); + + const M = b.axis("M", 64); + const N = b.axis("N", 128); + const G = b.axis("G", 8); + const MN = b.axis("MN", b.expr(.mul, M, N)); + try std.testing.expectEqual(64 * 128, MN.size); + b.compilerConfig(.{ .lowering_mode = "Heuristic", .max_num_partitioning_axes = 1 }); + const t = try b.declareArgs(.{ + .x = .{ .dtype = .f32, .axes = &.{ M, N }, .dram = .{} }, + .idx = .{ .dtype = .i32, .axes = &.{G} }, + }); + + const op = b.tensorOperation(.{ .context = .{ + .layout = .{ .chip = .broadcast, .split = &.{M} }, + .heuristic_hint = &.{.{ .name = "max_num_partitioning_axes", .value = .{ .int = 2 } }}, + } }); + const sum = try op.commit(op.fetch(t.x, .{}).reduce(&.{N}, .addf).toFxp(15).toFp(15), .{}); + + const rows = try b.gather(t.x, t.idx, M, &.{ G, N }, .{}); + const back = try b.scatter(rows, t.idx, M, &.{ M, N }, .{ .init = t.x }); + const iota = try b.arange(N.size, .i32, &.{N}, .{}); + const pattern = try b.vector(&.{ 1, 2 }, .f32, &.{N}, .{ .repeat_to = N }); + const bits = try b.transmute(back, .i32, &.{ M, N }, .{}); + const flat = try b.asDram(try b.reshape(bits, &.{MN}, .{}), .{}); + b.ret(&.{ sum, iota, pattern, flat }); + + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + try expectContains(ir, "MN = #tcl.expr<\"mul\", [#tcl.symbol<\"M\">, #tcl.symbol<\"N\">]>"); + try expectContains(ir, "tcl.compiler_config = #tcl.config<{lowering_mode = \"Heuristic\", max_num_partitioning_axes = 1 : i64}>"); + try expectContains(ir, "%arg0: !tcl.mapped], [#tcl.symbol<\"M\">, #tcl.symbol<\"N\">], []>>"); + try expectContains(ir, "\"tcl.graph.as_logical\"(%arg0)"); + try expectContains(ir, "opcode = #tcl.ve_opcode<\"to_i15\">"); + try expectContains(ir, "operator = {Chip = #tcl.expr<\"broadcast\", []>, Split = [#tcl.symbol<\"M\">]}"); + inline for (.{ "gather", "scatter", "arange", "vector", "transmute", "reshape", "as_dram" }) |g| + try expectContains(ir, "\"tcl.graph." ++ g ++ "\""); +} + +test "scatter written back into its DRAM argument in place" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = try Builder.open(std.testing.allocator, ctx, "write_rows"); + defer b.deinit(); + + const M = b.axis("M", 64); + const N = b.axis("N", 128); + const G = b.axis("G", 8); + const t = try b.declareArgs(.{ + .cache = .{ .dtype = .bf16, .axes = &.{ M, N }, .dram = .{} }, + .rows = .{ .dtype = .bf16, .axes = &.{ G, N } }, + .idx = .{ .dtype = .i32, .axes = &.{G} }, + }); + const written = try b.update(0, try b.scatter(t.rows, t.idx, M, &.{ M, N }, .{ .init = t.cache })); + try std.testing.expectEqual(@as(?usize, 0), b.updatedArgument(written)); + b.ret(&.{written}); + + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + try expectContains(ir, "\"tcl.graph.copy\""); + try expectContains(ir, "tcl.updated_inputs = array"); +} + +test "loop over a device-computed trip count with index reads and writes" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = try Builder.open(std.testing.allocator, ctx, "prefix"); + defer b.deinit(); + + const T = b.axis("T", 8); + const D = b.axis("D", 64); + const B = b.axis("B", 4); + const t = try b.declareArgs(.{ + .x = .{ .dtype = .f32, .axes = &.{ T, D } }, + .s = .{ .dtype = .f32, .axes = &.{D} }, + .lens = .{ .dtype = .i32, .axes = &.{B} }, + }); + // Running sums of the rows up to the longest length. + const n = try b.symExpr(.add, try b.reduceMaxI32(t.lens), 0); + var loop = b.openFor(n, .{ t.s, try b.scratchpad(.f32, &.{ T, D }) }); + { + const op = b.tensorOperation(.{}); + const sum = try op.commit(op.fetch(try b.indexRead(t.x, loop.iv), .{}).addf(op.fetch(loop.carried[0], .{})), .{}); + try loop.yield(.{ sum, try b.indexWrite(loop.carried[1], loop.iv, sum) }); + } + b.ret(&loop.results); + + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + try expectContains(ir, "\"tcl.graph.for\""); + try expectContains(ir, "operandSegmentSizes = array"); + try expectContains(ir, "\"tcl.graph.index_write\""); + try expectContains(ir, "#tcl.expr<\"add\", [#tcl.expr<\"arg\", [0]>, 0]>"); +} + +test "multi-chip kernel over chip-split DRAM tensors" { + const ctx = try testContext(); + defer ctx.deinit(); + var b = try Builder.open(std.testing.allocator, ctx, "split_exp"); + defer b.deinit(); + + const C = b.axis("C", 2); + const A = b.axis("A", 1024); + const t = try b.declareArgs(.{ .x = .{ .dtype = .f32, .axes = &.{ C, A }, .dram = Dram{ .chip = &.{.{ .axis = C }} } } }); + try std.testing.expectEqual(2, t.x.shards); + + const op = b.tensorOperation(.{ .context = .{ .layout = .{ .chip = .{ .axis = C } } } }); + const local = try op.commit(op.fetch(t.x, .{}).exp(), .{}); + const all = try b.allGather(local, C, .{}); + b.ret(&.{ try b.asDram(local, .{ .chip = &.{.{ .axis = C }} }), try b.asDram(all, .{}) }); + + const ir = try b.finish(); + defer std.testing.allocator.free(ir); + try expectContains(ir, "%arg0: !tcl.mapped], [#tcl.symbol<\"A\">], []>>"); + try expectContains(ir, "operator = {Chip = #tcl.symbol<\"C\">}"); + try expectContains(ir, "\"tcl.graph.all_gather\""); +} diff --git a/dsl/tcl/dtype.zig b/dsl/tcl/dtype.zig new file mode 100644 index 00000000..9faf1e18 --- /dev/null +++ b/dsl/tcl/dtype.zig @@ -0,0 +1,57 @@ +const std = @import("std"); + +const mlir = @import("mlir"); + +/// TCL element types, named as in TCL (`[A]/f8_e4`). +pub const DType = enum { + bool, + i4, + i8, + i16, + i32, + i64, + u8, + u16, + u32, + u64, + f16, + bf16, + f32, + f64, + f4_e2, + f8_e4, + f8_e5, + + pub fn toMlir(self: DType, ctx: *mlir.Context) *const mlir.Type { + return switch (self) { + .bool => .int(ctx, .i1), + .i4 => .int(ctx, .i4), + .i8 => .int(ctx, .i8), + .i16 => .int(ctx, .i16), + .i32 => .int(ctx, .i32), + .i64 => .int(ctx, .i64), + .u8 => .int(ctx, .u8), + .u16 => .int(ctx, .u16), + .u32 => .int(ctx, .u32), + .u64 => .int(ctx, .u64), + .f16 => .float(ctx, .f16), + .bf16 => .float(ctx, .bf16), + .f32 => .float(ctx, .f32), + .f64 => .float(ctx, .f64), + .f4_e2 => .float(ctx, .f4e2m1fn), + .f8_e4 => .float(ctx, .f8e4m3fn), + .f8_e5 => .float(ctx, .f8e5m2), + }; + } + + pub fn isFloat(self: DType) bool { + return switch (self) { + .f16, .bf16, .f32, .f64, .f4_e2, .f8_e4, .f8_e5 => true, + else => false, + }; + } +}; + +test { + std.testing.refAllDecls(@This()); +} diff --git a/dsl/triton/BUILD.bazel b/dsl/triton/BUILD.bazel new file mode 100644 index 00000000..6699694a --- /dev/null +++ b/dsl/triton/BUILD.bazel @@ -0,0 +1,29 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "builder", + srcs = [ + "dtype.zig", + ], + import_name = "dsl/triton/builder", + main = "builder.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//dsl/common:builder", + "//mlir", + "//mlir/dialects/arith", + "//mlir/dialects/cf", + "//mlir/dialects/func", + "//mlir/dialects/llvm", + "//mlir/dialects/math", + "//mlir/dialects/scf", + "//mlir/dialects/ttir", + "//stdx", + ], +) + +zig_test( + name = "test", + deps = [":builder"], +) diff --git a/dsl/triton/PORTING.md b/dsl/triton/PORTING.md new file mode 100644 index 00000000..0d328710 --- /dev/null +++ b/dsl/triton/PORTING.md @@ -0,0 +1,758 @@ +# Porting Triton kernels from Python to the Zig DSL + +This guide covers the day-to-day workflow for **adding a new Triton kernel +to ZML** (port from a Python `@triton.jit` source) and **fixing an existing +Zig kernel that diverges from Python** (using the comparison harness in +`examples/triton_emitter/`). + +> **The harness is not in this repository.** `examples/triton_emitter/` and +> its scripts (`run.sh`, `dump_python_ir.py`, `kernels_zig.zig`, +> `dump_via_xla.zig`, `compare_ir.py`) are not part of ZML; the steps that +> use them describe the comparison it runs. Writing the kernel and its +> caller (Steps 1.2 and 1.3) and the porting bugs of Phase 3 don't need it. + +## Goals (in priority order) + +1. **Lowered code matches.** The Zig version produces **the same LLVM IR + + PTX** as the Python version. TTIR/TTGIR may differ cosmetically (op + ordering, naming) — what matters is what actually runs on the GPU. + +2. **Source code looks alike.** A Python kernel and its Zig port should be + *visually transparent* side by side: same overall structure, same + variable names, same comment placement, same line ordering. Someone + reading both should be able to follow them in lockstep. + + Visual fidelity matters because: + - It makes future ports trivial — patterns transfer one-to-one. + - It makes upstream-Triton patches easy to apply (the Python diff + translates almost mechanically to a Zig diff). + - It catches porting bugs before they reach the comparison harness: + when the Zig code reads differently from Python, that's where the + real-vs-cosmetic divergence usually hides. + + Concretely: if a Python source line is + ```python + y_ptr_offset = (row.to(tl.int64) * y_row_stride) + (row_g_id.to(tl.int64) * group_size) + ``` + the Zig port should be one line, named the same way: + ```zig + const y_ptr_offset = row.to(.i64).mul(y_row_stride).add(row_g_id.to(.i64).mul(group_size)); + ``` + Not split across three temporaries with renamed locals — even if the + resulting LLIR is identical. Goal #1 is a hard requirement; goal #2 + is what makes this codebase maintainable. + +For the DSL surface (`zml.kernel.triton.Kernel`, `Builder`, `Value`, etc.), see +[`README.md`](./README.md). This file is about the *process*, not the API. + +--- + +## Prerequisites + +- CUDA or ROCm GPU (XLA's Triton backend is GPU-only). +- The Python venv at `examples/triton_emitter/.venv` activated, or a + system-level Python with `triton`, `torch`, `numpy` installed. +- Bazel + the standard ZML build deps. + +One-time: + +```sh +cd examples/triton_emitter +python -m venv .venv +source .venv/bin/activate +pip install -r requirements.txt +``` + +--- + +## Phase 1 — Initial port + +Start by copying the Python kernel verbatim into the Zig DSL form. The +goal is a kernel that *compiles*, *verifies*, **and reads as similarly +to the Python source as possible**. + +> **Author hint:** open the Python file in a left pane and the Zig file +> in a right pane. Translate **line by line**, top to bottom. Keep +> variable names, comment placement, and intermediate-binding granularity +> identical wherever the DSL allows. Don't refactor, don't rename, don't +> "improve" — that's a separate task you do after the LLIR matches. + +### Step 1.1 — Drop the Python source next to the Zig file + +For comparison purposes, the Python source has to live under +`examples/triton_emitter/kernels_py/`. The dump script auto-discovers +every `@triton.jit` function in that directory. + +If your kernel's already in `vllm/` or another upstream tree, copy the +relevant `@triton.jit` definition (and any `@triton.jit` helpers it +calls) into a single `.py` file there. Strip away launch wrappers, +autotune decorators, and any code that isn't actually inside the kernel +function. + +### Step 1.2 — Sketch the Zig declaration + +Inside `zml/kernels/moe/triton/` or `zml/kernels/paged_attention/triton/` +(or whichever production file fits), add: + +```zig +const tri = zml.kernel.triton; + +pub const MyKernel = struct { + pub const Cfg = struct { + // One field per `tl.constexpr` parameter, named as in Python. + BLOCK_SIZE: usize, + // ... + }; + pub const Kernel = tri.Kernel(Cfg, .{ + .name = "my_kernel", // EXACTLY matches the Python `def my_kernel` + // Call-site names, in `tt.func` argument order: inputs, then outputs. + .inputs = &.{"x_ptr"}, + .outputs = &.{"out_ptr"}, + .run = run, + }); + + fn run(b: *tri.Builder, cfg: Cfg) tri.FinishError!void { + const a = try b.declareArgs(.{ + // One field per *runtime* parameter (pointers, dims that + // arrive as 0-d tensors). Use the same names + order Python + // uses; the README's "Argument specs" table maps the + // tagged-union variants. + .x_ptr = .{ .ptr = .f32 }, + // ... + }); + + // Body — translate Python line by line. Cheat-sheet in the + // README's "Python Triton ↔ Zig DSL cheat-sheet" section is + // your best friend here. + } +}; +``` + +### Step 1.3 — Wire the production caller + +Inside the same `.zig` directory, find the function that launches your +kernel (e.g. `pagedAttention2d`, `fusedExpertsImpl`). Replace the +TTIR-emit + `ops.triton(...)` two-step with a single +`MyKernel.Kernel.call(inputs, outputs, opts)` — see the +[README](./README.md#calling-kernels-from-a-zml-model--kkernelcall) for the +exact shape. + +If this is a brand-new kernel with no existing caller, write the caller +fresh. + +### Step 1.4 — Wire the comparison harness + +Add the kernel to `examples/triton_emitter/kernels_zig.zig`: + +```zig +withConfig(moe.MyKernel, .{ .BLOCK_SIZE = 64, /* ... */ }), +``` + +If the kernel takes runtime tensor inputs, also add a `forward` struct + +`KERNELS` entry to `examples/triton_emitter/dump_via_xla.zig` so XLA's +codegen pipeline can lower it. The tensor sizes are dummies — XLA +doesn't launch the kernel, it just runs the lowering passes. + +For the Python side, the auto-discovery in `dump_python_ir.py` works for +simple kernels (only `*_ptr` runtime args + `tl.constexpr` params). +**If the signature mixes runtime args and constexpr in the middle of the +parameter list**, write an override function and add it to `_OVERRIDES` +(follow `_per_token_group_quant_fp8` / `_kernel_unified_attention_2d_ptr` +as templates). Use the same constexpr values + tensor shapes you used in +the Zig `withConfig(...)` call. + +### Step 1.5 — Build it + +```sh +bazel build //examples/triton_emitter:dump_zig_ir # Zig side compiles + verifies +bazel build //zml:test # production side compiles +``` + +If the Zig DSL `b.finish()` rejects your IR, it logs the module. Mixed +integer widths and scalar/tensor operands are not the cause: the fluent ops +widen the narrower integer side (`v_i32.mul(v_i64)` is an i64 multiply), and +`addPtr` splats a scalar pointer or offset against a tensor partner. A common +cause is an **`scf.yield` type mismatch** — the values you pass to +`loop.yield(...)` / `i.yieldThen(...)` must have the types of the loop's +`inits` / the if's `result_types` (their arity is checked at compile time). + +### Step 1.6 — First dump + +```sh +bash examples/triton_emitter/run.sh +``` + +This runs the full pipeline (Python TTIR → XLA → per-stage IR; same on +the Zig side; diff each stage). At the end you get a per-kernel +verdict: + +``` +my_kernel + ttir +30/-30 + ttgir +30/-30 + llir +50/-50 + ptx +40/-40 +``` + +Don't worry yet about cosmetic diffs — go through Phase 2 to find +*real* divergences first. + +--- + +## Phase 2 — The comparison pipeline + +The harness has six steps (see `run.sh`), output appears under +`examples/triton_emitter/`: + +| Step | Output | What it is | +|------|------------------------------|-----------------------------------------------| +| 1 | `py_ir/*.ttir`/`.ttgir`/... | Python's per-stage IR (Triton's own pipeline) | +| 2 | `zig_ir/*.ttir` | Just the TTIR your DSL emits. | +| 3 | `xla_dump_py/` | XLA's per-pass dumps lowering Python's TTIR. | +| 4 | `xla_dump_zig/` | XLA's per-pass dumps lowering the Zig TTIR. | +| 5 | `xla_py/*.{ttir,ttgir,llir,ptx}` + `xla_zig/*.{...}` | Per-stage IR extracted by `extract_xla_dump.py`. Both sides go through XLA, not Triton-the-Python-frontend, so the comparison is apples-to-apples. | +| 6 | stdout | `compare_ir.py` diff summary per kernel. | + +### Useful flags + +```sh +# Just one kernel (faster iteration): +# (use ZIG_IR/PY_IR filter — but note dump_zig_ir's CLI expects --kernel=NAME). +bash run.sh # full run +python compare_ir.py --left xla_py --right xla_zig --kernel my_kernel +python compare_ir.py --left xla_py --right xla_zig --kernel my_kernel --stage ttir --show-diffs + +# Skip the slow XLA-lowering steps if only the TTIR changed: +python dump_python_ir.py --out-dir py_ir +bazel run //examples/triton_emitter:dump_zig_ir -- --out-dir=$PWD/zig_ir +diff py_ir/my_kernel.ttir zig_ir/my_kernel.ttir +``` + +### Reading the verdict + +``` +my_kernel + ttir +30/-30 <- TTIR diverges 30 lines + ttgir +30/-30 <- so does TTGIR (same shape) + llir ✓ <- but LLIR converges! cosmetic-only TTIR diff + ptx ✓ <- and PTX matches +``` + +This kernel is **good** — final code is identical, the TTIR diff is just +op-naming/ordering cosmetics that XLA's lowering canonicalizes away. + +``` +my_kernel + ttir +5/-5 + ttgir +5/-5 + llir +60/-58 <- LLIR diverges → real semantic difference + ptx +50/-48 +``` + +This kernel needs work: a real divergence appeared somewhere between +TTIR and LLIR. Drill into the LLIR diff to find which op got lowered +differently. + +--- + +## Phase 3 — Fixing divergences + +Workflow: pick the *smallest* diverging stage's diff and locate the +op-by-op mismatch. Most fixes fall into one of the categories below. + +### Step 3.1 — `--show-diffs` to locate the divergence + +```sh +python compare_ir.py --left xla_py --right xla_zig --kernel my_kernel --stage ttir --show-diffs +``` + +Read the diff carefully. The first diverging op tells you what to look +at; subsequent diffs are usually downstream consequences. + +### Step 3.2 — Common porting bugs + +These are the gotchas we've hit while porting kernels. If your diff +shows any of these patterns, fix the corresponding Zig DSL call. + +#### 3.2.1 — `arith.maxnumf` vs `arith.maximumf` (and `min*`) + +**Symptom:** +``` +- %v43 = arith.maxnumf %v37, %v14 : f32 (Python) ++ %v43 = arith.maximumf %v37, %v13 : f32 (Zig) +``` + +**Fix:** **Use `b.maximum` / `Value.maximum` / `b.minimum` / +`Value.minimum`** — they default to `maxnumf`/`minnumf` and match +Python's `tl.max`/`tl.min`/`tl.maximum`/`tl.minimum`. **Don't reach +for `b.maximumf` / `b.minimumf` directly** — those are the IEEE-754 +NaN-propagating escape hatches (`arith.maximumf`/`minimumf`), exposed +for the rare case where you genuinely want NaN propagation. The naming +is unfortunate (the `f` suffix reads like "the float version") — it's +actually "the NaN-propagating version." See README's "Reductions and +scans" gotcha table for the full mapping. + +#### 3.2.2 — 2-operand vs 3-operand `tt.load` + +**Symptom:** +``` +- %v33 = tt.load %v32, %v30, %v9 : tensor<128x!tt.ptr> (Python — has `other`) ++ %v33 = tt.load %v32, %v30 : tensor<128x!tt.ptr> (Zig — no `other`) +``` + +**Fix:** Match Python's exact form. The DSL has only one masked-load +API (`b.loadOpts`); whether it emits 2-operand or 3-operand `tt.load` +depends entirely on whether you pass `.other`: + +```zig +// Python: tl.load(ptr, mask=mask, other=0.0) → 3-operand +const zero = b.zeros(&.{BLOCK}, dtype); +const x = b.loadOpts(ptr, .{ .mask = mask, .other = zero }); + +// Python: tl.load(ptr, mask=mask) → 2-operand +const x = b.loadOpts(ptr, .{ .mask = mask }); +``` + +The two forms are *semantically* close but not identical (out-of-bounds +lanes are undefined for 2-operand, zero for 3-operand). Always copy +Python's form, even if you suspect the difference is harmless — XLA's +canonicalization may or may not erase it. + +#### 3.2.3 — `i32` vs `i64` width in pointer offsets + +**Symptom:** +``` +- %v26 = tt.addptr %v6, %v16 : !tt.ptr, i32 (Python — uses i32) ++ %v25 = tt.addptr %v6, %v16 : !tt.ptr, i64 (Zig — uses i64) +``` + +**Fix:** Python's pointer arithmetic preserves the original integer +width. `tl.program_id(0)` is i32; if Python writes +`y_s_ptr += tl.program_id(0)` (no cast), the offset stays i32. In Zig, +if you converted `pid` to i64 early to use it elsewhere, pass the raw +`b.programId(.x)` (i32) to that one `addPtr` site: + +```zig +// raw i32 program_id for one site, i64 elsewhere +const pid_i32 = b.programId(.x); +const g_id = pid_i32.to(.i64); +// ... +const y_s_ptr_shifted = a.y_s_ptr.addPtr(pid_i32); // i32, not g_id +``` + +#### 3.2.4 — Mask comparison width + +**Symptom:** +``` +- %v30 = arith.cmpi slt, %v28, %v29 : tensor<128xi64> (Python — i64 vs i64) ++ %v28 = arith.trunci %v10 : i64 to i32 ++ %v29 = tt.splat %v28 : i32 -> tensor<128xi32> ++ %v30 = arith.cmpi slt, %v26, %v29 : tensor<128xi32> (Zig — trunc + splat + i32 cmp) +``` + +**Fix:** Match Python's promotion direction. Python implicitly promotes +the narrower operand to the wider one. If `cols` is i32 and +`group_size` is i64, Python emits `extsi cols i32→i64` then compares +at i64. The fluent ops promote the same way, so compare the two as they +are; a `trunci` like the one above comes from narrowing one side by hand: + +```zig +const cols = b.arange(0, BLOCK, .i32); +const mask = cols.lt(group_size); // extsi cols to i64, no trunc +``` + +#### 3.2.5 — Op emit order + +**Symptom:** TTIR diff shows the same ops in a different order: +``` +- %v18 = arith.divsi %v17, %v15 : i64 +- %v19 = arith.remsi %v17, %v15 : i64 +- %v20 = arith.muli %v18, %v13 : i64 +- %v21 = arith.muli %v19, %v11 : i64 +- %v22 = arith.addi %v20, %v21 : i64 ++ %v17 = arith.divsi %v16, %v14 : i64 ++ %v18 = arith.muli %v17, %v12 : i64 ++ %v19 = arith.remsi %v16, %v14 : i64 ++ %v20 = arith.muli %v19, %v10 : i64 ++ %v21 = arith.addi %v18, %v20 : i64 +``` + +This usually doesn't affect LLIR/PTX (compiler reorders), but if it +does (or if you want a clean TTIR diff), match Python's source order +op-by-op. Bind every intermediate to a named local in the same order +Python evaluates: + +```zig +// Python: y_ptr += (g_id // groups_per_row) * y_row_stride + (g_id % groups_per_row) * group_size +// eval order: divsi, remsi, muli, muli, addi +const row = g_id.div(groups_per_row); // divsi +const row_g_id = g_id.rem(groups_per_row); // remsi +const row_off = row.mul(y_row_stride); // muli +const grp_off = row_g_id.mul(group_size); // muli +const y_ptr_shifted = a.y_ptr.addPtr(row_off.add(grp_off)); // addi + addptr +``` + +#### 3.2.6 — `arith.maxnumf` (and `splat`) vs `tt.splat` of a runtime value + +**Symptom:** +``` +- %splat = tt.splat %scalar : i32 -> tensor<128xi32> (Python — runtime splat) ++ %const = arith.constant dense<...> : tensor<128xi32> (Zig — constant pool) +``` + +**Fix:** `b.splat(comptime_literal, shape)` materializes a constant +dense; `b.splat(runtime_value, shape)` emits `tt.splat`. If Python +uses a runtime value (loaded scalar, function arg) and you accidentally +hardcoded the constant, switch to the runtime path. + +#### 3.2.7 — `tl.constexpr` divisibility hints + +**Symptom:** TTIR header differs in `tt.divisibility = N : i32` attrs. + +**Fix:** Default `.{ .ptr = .dtype }` carries `tt.divisibility = 16`. +For scalars, switch to `.scalar_opts`: + +```zig +.n_elements = .{ .scalar_opts = .{ .dtype = .i32, .divisibility = 16 } }, +``` + +This matches what Python's JIT runtime auto-attaches when the runtime +value is a multiple of 16. + +#### 3.2.8 — Reductions over rank-1 tensors + +**Symptom:** Python emits a `tt.reshape allow_reorder` before the +`tt.reduce`; Zig doesn't. + +**Fix:** Use the `axis = null` (default) form: `b.max(x)`, not +`b.maxOpts(x, .{ .axis = 0 })`. The `axis = null` path inserts the +reshape; explicit-axis paths skip it. + +#### 3.2.9 — `tl.dot` precision and accumulator + +**Symptom:** TTIR shows different `tt.dot` `inputPrecision` attribute or +different `acc` type. + +**Fix:** Match Python's `tl.dot(a, b, acc, input_precision="...")` +explicitly: + +```zig +const new_acc = b.dotOpts(a_val, b_val, acc, .{ + .input_precision = .tf32, // matches `input_precision="tf32"` + .max_num_imprecise_acc = 0, +}); +``` + +Default `input_precision` differs across Triton versions — always be +explicit. + +#### 3.2.10 — `iter_args` order in `scf.for` + +**Symptom:** TTIR diff shows the loop signature reordered: +``` +- %v152:3 = scf.for ... iter_args(%v149 = %v131, %v150 = %v143, %v151 = %v37) +- -> (tensor<...!tt.ptr>, tensor<...!tt.ptr>, tensor<...xf32>) ++ %v152:3 = scf.for ... iter_args(%v149 = %v37, %v150 = %v131, %v151 = %v143) ++ -> (tensor<...xf32>, tensor<...!tt.ptr>, tensor<...!tt.ptr>) +``` + +**Fix:** Python's `scf.for` lowering captures `iter_args` in the order +they're **declared** above the loop, not the order they're mutated inside. +For `fused_moe_kernel`: + +```python +a_ptrs = ... # declared 1st +b_ptrs = ... # declared 2nd +accumulator = tl.zeros(...) # declared 3rd +for k in range(...): + accumulator += tl.dot(a, b) # mutated 1st + a_ptrs += BLOCK_SIZE_K # mutated 2nd + b_ptrs += BLOCK_SIZE_K # mutated 3rd +``` + +Python's `iter_args` order = declaration order = `(a_ptrs, b_ptrs, acc)`, +**not** mutation order. The DSL `openFor` follows the order you pass — so +match Python's declaration order: + +```zig +var loop = b.openFor(0, num_k_iters, 1, .{ a_ptrs_init, b_ptrs_init, acc_init }); +{ + const a_ptrs = loop.carried[0]; + const b_ptrs = loop.carried[1]; + const acc = loop.carried[2]; + // ... + loop.yield(.{ new_a_ptrs, new_b_ptrs, new_acc }); +} +const accumulator = loop.results[2]; +``` + +The wrong order survives all the way to PTX as different register-block +layouts. + +#### 3.2.11 — Operand order in commutative `arith.muli` + +**Symptom:** TTIR diff shows `arith.muli` operands swapped: +``` +- %v137 = arith.muli %v136, %v135 : tensor<1x64xi64> (Python: stride on LHS) ++ %v137 = arith.muli %v135, %v136 : tensor<1x64xi64> (Zig: offs on LHS) +``` + +**Fix:** `arith.muli` is commutative semantically, but the operand order +affects downstream layout decisions in TTGIR. Match Python's source order +exactly. For `stride * offs[None, :]`, Python emits `arith.muli %stride_splat, +%offs`: + +```zig +// Python: stride_cm * offs_token[:, None] — stride on LHS +const cm_col = stride_cm.mul(offs_token_col); +// NOT: offs_token_col.mul(stride_cm) — offs on LHS, wrong order +``` + +Same applies to `arith.addi`, `arith.subi`, `arith.andi`, etc. + +#### 3.2.12 — Two-step `addPtr` for `c_ptr + cm[:, None] + cn[None, :]` + +**Symptom:** Python emits two `tt.addptr` ops; the Zig port collapses +them into one big addptr with a pre-merged 2D offset. +``` +Python: + %ptr_col = tt.addptr %c_ptr_splat_Mx1, %cm_col_Mx1 : Mx1x!tt.ptr, Mx1xi64 + %ptr_2d = tt.broadcast %ptr_col → MxNx!tt.ptr + %final_ptr = tt.addptr %ptr_2d, %cn_off_2d : MxNx!tt.ptr, MxNxi64 +Zig: + %offset = arith.addi %cm_term, %cn_term : MxNxi64 + %final_ptr = tt.addptr %c_ptr_splat, %offset : MxNx!tt.ptr, MxNxi64 +``` + +**Fix:** Python evaluates `c_ptr + cm[:, None] + cn[None, :]` left-to-right, +so it emits two `tt.addptr` ops. Match by splitting: + +```zig +// First addptr: scalar c_ptr + (Mx1) cm offset → Mx1 ptr tensor +const cm_col = stride_cm.mul(b.expandDims(offs_token, 1)); +const c_ptrs_col = b.splat(c_ptr, &.{ block_size_m, 1 }).addPtr(cm_col); + +// Second addptr: broadcast to MxN, add cn offset +const cn_row = b.expandDims(offs_cn, 0); +const c_ptrs_2d = b.broadcastTo(c_ptrs_col, &.{ block_size_m, block_size_n }); +const c_ptrs = c_ptrs_2d.addPtr(b.broadcastTo(cn_row, &.{ block_size_m, block_size_n })); +``` + +This pattern shows up wherever Python writes `ptr + a[:, None] + b[None, :]` +(both A and B ptrs in fused_moe, the C-output store, etc.). + +#### 3.2.13 — `extsi` placement relative to sibling `expand_dims`/`mul` + +**Symptom:** Same set of ops, different positions: +``` +- %v134 = expand_dims %offs_k_i32 (axis=1) (Python: offs_k expand) +- %v135 = expand_dims %offs_bn_i64 (axis=0) (Python: offs_bn expand) +- %v136 = splat %stride_bn → 1x64xi64 (Python: bn stride) +- %v137 = muli %v135, %v136 (Python: bn_row mul) +- %v138 = extsi %v134 → 32x1xi64 (Python: extsi DEFERRED) ++ %v135 = extsi %v134 → 32x1xi64 (Zig: extsi EAGER) ++ ... (rest in different order) +``` + +**Fix:** When Python writes `offs_k[:, None]` and uses it later in an +`addi` against an `i64`-typed tensor, the implicit `extsi` happens at the +**use site** (where the auto-promotion fires), not at the `expand_dims`. +Mirror this — keep the chain at i32 until just before it joins the i64 +expression: + +```zig +// Wrong: extsi happens immediately after expand_dims +const offs_k_col = b.expandDims(offs_k_i32, 1).to(.i64); // extsi NOW +const offs_bn_row = b.expandDims(offs_bn, 0); +const bn_row = offs_bn_row.mul(stride_bn); + +// Right: extsi deferred to after offs_bn's mul +const offs_k_col_i32 = b.expandDims(offs_k_i32, 1); // 32x1xi32 +const offs_bn_row = b.expandDims(offs_bn, 0); +const bn_row = offs_bn_row.mul(stride_bn); // 1x64xi64 +const offs_k_col = offs_k_col_i32.to(.i64); // extsi HERE +``` + +Same idea for placing a `muli` before an `extsi` on a sibling load: + +```zig +// Python emits muli before the extsi at the cmpi: +const num_tokens_post_padded_i32 = b.load(...); // load +const pid_m_block = pid_m.mul(block_size_m); // muli FIRST +const num_tokens_post_padded = num_tokens_post_padded_i32.to(.i64); // extsi +const out_of_range = pid_m_block.ge(num_tokens_post_padded); +``` + +### Step 3.3 — Iterate + +After each fix: + +```sh +bazel build //examples/triton_emitter:dump_zig_ir +bash examples/triton_emitter/run.sh +``` + +Re-read the diff. The line counts should drop, and ideally the LLIR/PTX +columns flip to `✓`. If not, drill again — there's another mismatch +hiding behind the first. + +### Step 3.4 — Don't be a perfectionist about TTIR (but do read every line) + +You only need **LLIR + PTX** to match. If TTIR/TTGIR have +5 cosmetic +lines but LLIR is byte-identical, ship it. The XLA canonicalization + +CSE passes erase a lot of low-level differences before LLIR. + +A kernel like `count_and_sort_expert_tokens_kernel` (TTIR/TTGIR +8/-8, +LLIR/PTX ✓) is a clean port. Don't keep grinding it. + +**But** — when LLIR is **not** ✓ and the diff is small (tens of lines), +do **not** label the rest "scheduling noise" without a name-normalized +diff. Most lines in a small LLIR diff are downstream SSA renumbering +caused by 1–4 high-leverage TTIR-level fixes (typically the patterns in +3.2.10–3.2.13). Run: + +```sh +diff \ + <(sed -E 's/%[a-zA-Z_.][a-zA-Z0-9_.]*[0-9]+/%V/g; s/%[0-9]+/%V/g; s/^[0-9]+:/V:/g' xla_py/foo.llir) \ + <(sed -E 's/%[a-zA-Z_.][a-zA-Z0-9_.]*[0-9]+/%V/g; s/%[0-9]+/%V/g; s/^[0-9]+:/V:/g' xla_zig/foo.llir) +``` + +If this normalized diff is small (~tens of lines), every remaining line +is pointing at a structural fix. The "+100 LLIR is just XLA scheduling" +intuition has been wrong every time so far — `fused_moe_kernel` went from +LLIR +100 → ✓ via four TTIR-level fixes (iter_args order, muli operand +order, two-step addPtr, extsi placement) that were all visible in the ++100-line diff once read carefully. + +### Step 3.5 — Visual-transparency pass + +Once the LLIR matches, do one pass over the Zig source with the Python +source open beside it. Check (in order): + +- **Variable names.** Python `groups_per_row` → Zig `groups_per_row`, + not `gp_row` or `gpr`. Same for arg names — `y_ptr` not `output_ptr`. +- **Line order.** Each Python line should correspond to roughly one + Zig line, in the same order. If a single Python expression became + three Zig temporaries, ask: does the DSL force the split? If not, + collapse them onto one chained `.add(...).mul(...)` line. +- **Granularity of locals.** If Python writes + `_absmax = tl.maximum(tl.max(tl.abs(y)), eps)` as a one-liner, the + Zig port should also be one line: `const absmax = b.max(b.absf(y)).maximum(eps);`. +- **Comments.** Carry over Python's comments verbatim where they apply. + If Python writes `# Map the program id to the row of X and Y it + should compute.` above its block, paste the same comment above the + Zig block. They serve as anchors for future side-by-side reads. +- **Helper functions.** If Python factors something into a + `@triton.jit` helper (e.g. `find_seq_idx`), the Zig port should have + a matching `fn findSeqIdx(b: *Builder, ...) Value` helper, not inline + the body. Same arguments, same name (snake_case → camelCase). +- **Control flow shape.** Python `for k in range(...)` → `b.openFor`, + Python `if ... return` → `b.returnIf` / `b.openReturnIf`. Don't + flatten an `if` ladder into a `select` chain (or vice versa) just + because the DSL allows it. + +If a transparency fix changes the LLIR diff, **the LLIR-match goal +wins** — back out the visual change and leave a comment explaining the +divergence. Goal #1 always beats goal #2. + +A useful sanity check: open both files side by side in your editor and +scroll both at the same rate. If the cursors stay roughly aligned, the +port reads transparently. If you have to scroll one twice as fast as +the other, there's structural drift to clean up. + +--- + +## Phase 4 — Production wiring (post-port) + +Once the LLIR matches *and* the visual-transparency pass is done: + +1. **Side-by-side review.** Pull up Python and Zig in adjacent panes + one more time. Anything that still reads differently between them + either has a justification (DSL constraint, language constraint — + leave a brief comment explaining why) or needs another pass. +2. **Remove any `// TODO match python` comments** from the kernel body. +3. **Verify the production caller passes the right cfg** — the dump + tool's config and the production caller's config can differ (the + dump tool uses small dimensions; production uses real ones). + Production behavior depends on the runtime cfg, so make sure + `makeFooConfig(...)` in the caller produces sensible values. +4. **Run the model end-to-end** if possible. The comparison harness + only validates IR equivalence — runtime correctness still needs a + real test (mismatched grid, missing strides, etc. won't show up in + the diff but will crash at execution time). + +--- + +## Cheat-sheet: when to look at which file + +| Problem | File | +|---------------------------------------------------|----------------------------------------------| +| "How do I express `tl.X(...)` in Zig?" | `dsl/triton/README.md` (Python ↔ Zig table) | +| "What does `b.declareArgs(...)` accept?" | `dsl/triton/README.md` (Argument specs) | +| "How do I call this kernel from `forward`?" | `dsl/triton/README.md` (`K.call(...)`) | +| "My kernel diverges from Python at LLIR" | This file (Phase 3) | +| "How do I add a kernel to the comparison harness?"| Phase 1 above + `examples/triton_emitter/` | +| "I want to inspect raw TTIR ops" | `mlir/dialects/ttir/ttir.zig` (escape hatch) | + +--- + +## When you're stuck + +If a divergence resists all the standard fixes: + +1. **Print both TTIRs side by side.** The visual structure usually + reveals which control-flow region or loop body is misaligned. +2. **Check if the Python kernel uses any non-`@triton.jit` helpers** — + if it imports something from `triton.language.extra` or has a custom + `@triton.jit` helper, that helper might also need a port (or a + `_SKIP` entry if it's pure-comptime). +3. **Verify the Python override args match the Zig config.** Mismatched + constexpr values (e.g. `BLOCK_SIZE=64` in Zig vs `1024` auto-default + in Python) generate completely different IR. +4. **Compare against the `zml/kernels/moe/triton/` MoE kernels** — they + went through this loop and are reasonable references for the patterns + above. + +--- + +## Before "fixing" the DSL + +Every time a divergence makes you suspect the DSL itself is buggy and you +reach for `dsl/triton/builder.zig` — **verify against the Triton sources +first** in a Triton checkout. The DSL exists to mirror what Python's +frontend emits; the source of truth is the Python trace, not what the rule +"looks like" or what feels symmetric. + +Concretely: + +- If you think `Value.cdiv` is wrong, read + `triton/python/triton/language/standard.py` for the definition of + `tl.cdiv` and compare. Don't assume the desugaring — check. +- If you think a binop dispatches to the wrong arith op + (`maxnumf` vs `maximumf`, `divsi` vs `divui`, …), read + `triton/python/triton/language/semantic.py` and find the cast / + promotion rule that actually fires. +- If you think the type-promotion is wrong (i32 stays i32, doesn't + auto-extend to i64, …), read `semantic.py:cast()` and the binop + promotion helpers. + +A second pitfall: **the local kernel may define its own helper that +shadows the stdlib name.** `unified_attention.py` defines a private +`cdiv_fn(x, y) = (x + y - 1) // y` — left-to-right Python parses this +as `((x + y) - 1)`, so it emits `addi` then `subi`. Triton's *stdlib* +`tl.cdiv` is `(x + (div - 1)) // div` and emits `subi` then `addi`. +These desugar identically when the divisor is comptime (Triton folds +`div - 1` at trace time), but differ when the divisor is a runtime +tensor. If your divergence is just an `add↔sub` reorder, check whether +the Python kernel actually uses `tl.cdiv` or a private `cdiv`-shaped +helper, and either keep the DSL aligned with `tl.cdiv` and write a +local Zig helper for the kernel-private form, or the other way around. + +In short: when a "bug" in the DSL would also affect kernels that +already match Python, the bug is almost certainly elsewhere. + +**And if you do change the DSL**, run `examples/triton_emitter/run.sh` +afterwards and confirm no kernel that previously matched (or was +codegen-equivalent) regressed at LLIR/PTX. A DSL edit that fixes one +kernel by introducing a divergence in another is not a fix — it's a +trade. Always check the full table before committing the change. diff --git a/dsl/triton/README.md b/dsl/triton/README.md new file mode 100644 index 00000000..939a1b07 --- /dev/null +++ b/dsl/triton/README.md @@ -0,0 +1,1116 @@ +# ZML Triton DSL — Author's Guide + +A Zig builder for Triton IR (TTIR) that tries to read like `triton.language` +Python while staying in pure Zig. Used by `zml/kernels/moe/triton/`, +`zml/kernels/paged_attention/triton/`, and by anything else that wants to call +Triton kernels from a ZML model. + +Two layers define the surface: + +- **Low layer** — `dsl/triton/builder.zig`: the IR builder primitives. + `Builder`, `Value`, `DType`, `ArgSpec`. Use these directly when you need + full control (custom op insertion, dynamic-arity arg lists, escape hatches). + `Builder.open` / `Builder.init` take an `*mlir.Context` — pass them a + `zml.kernel.triton.newContext()` (a throwaway context with the dialects the + DSL emits), `defer ctx.deinit()` once you have the IR string. Not a + `zml.Compiler`'s `mlir_ctx`: that one carries only `func`/`stablehlo`/`sdy` + (a registered `llvm` dialect makes XLA:TPU choke). +- **High layer** — `zml.kernel.triton.Kernel(Config, spec)`: the declarative + kernel form. Bundles a config type, a typed spec literal (name + named + `inputs` / `outputs` + a typed `run` function pointer), and produces a + kernel type. Exposes `.emit(allocator, cfg)` (TTIR string; manages its own + throwaway context) and `.call(inputs, outputs, opts)` (TTIR string + + `stablehlo.custom_call` in one shot — the preferred form for production + model code). Inputs and outputs are by-name structs generated from the + spec, so missing / typo'd / extra arguments are compile errors at the callsite. + +## Table of contents + +1. [Skeleton of a kernel](#skeleton-of-a-kernel) +2. [`zml.kernel.triton.Kernel(Config, spec)` — the declarative form](#zmlkerneltritonkernelconfig-spec--the-declarative-form) +3. [Calling kernels from a ZML model — `K.Kernel.call(...)`](#calling-kernels-from-a-zml-model--kkernelcall) +4. [Argument specs](#argument-specs) +5. [The `Value` type](#the-value-type) +6. [Fluent methods on `Value`](#fluent-methods-on-value) +7. [Polymorphic scalars](#polymorphic-scalars) +8. [Comptime vs runtime widths](#comptime-vs-runtime-widths) +9. [Shape helpers](#shape-helpers) +10. [Loads and stores](#loads-and-stores) +11. [Reductions and scans](#reductions-and-scans) +12. [Control flow](#control-flow) +13. [Escape hatches](#escape-hatches) +14. [Lower layer — `Builder.open`](#lower-layer--builderopen) +15. [Python Triton ↔ Zig DSL cheat-sheet](#python-triton--zig-dsl-cheat-sheet) +16. [Pitfalls](#pitfalls) + +--- + +## Skeleton of a kernel + +```zig +const tri = @import("zml").kernel.triton; +const zml = @import("zml"); + +pub const MyKernel = struct { + pub const Cfg = struct { BLOCK: i32 = 64 }; + pub const Kernel = tri.Kernel(Cfg, .{ + .name = "my_kernel", + .inputs = &.{ "x_ptr", "y_ptr", "n" }, + .outputs = &.{"out"}, + .run = run, + }); + fn run(b: *tri.Builder, cfg: Cfg) tri.FinishError!void { + const a = try b.declareArgs(.{ + .x_ptr = .{ .ptr = .f32 }, + .y_ptr = .{ .ptr = .f32 }, + .n = .{ .scalar = .i32 }, + .out_ptr = .{ .ptr = .f32 }, + }); + + const pid = b.programId(.x); + const offs = b.arange(0, cfg.BLOCK, .i32).add(pid.mul(cfg.BLOCK)); + const mask = offs.lt(a.n); + + const x = b.loadOpts(a.x_ptr.addPtr(offs), .{ .mask = mask }); + const y = x.mul(2.0); + b.storeOpts(a.out_ptr.addPtr(offs), y, .{ .mask = mask }); + } +}; +``` + +A kernel is a `struct {…}` wrapper bundling three decls: + +1. **`pub const Cfg`** — config type, an inline `struct { ... }` of + comptime/runtime parameters. Use field defaults for sensible defaults. +2. **`pub const Kernel = tri.Kernel(Cfg, .{ .name, .inputs, .outputs, .run })`** — + the spec literal that turns into a kernel type. `inputs` / `outputs` are + slices of `[:0]const u8` field names; the factory generates `Inputs` + (struct of `Tensor`), `Outputs` (struct of `Shape`), and `Results` + (struct of `Tensor`) types from those names. The `run` field is a typed + function pointer — wrong signature is a compile error here. +3. **`fn run(b, cfg)`** — the kernel body. First call `try b.declareArgs(.{…})` + to create the `tt.func` op and bind a struct of `Value`s named after the + spec's fields. The rest of `run` builds the body via `b.*` helpers. No + string return, no allocator, no manual `tt.return` — the factory + terminates, verifies, and serializes for you. + +The names in `spec.inputs` / `spec.outputs` are the **call-site** names +(used to build the `Inputs` / `Outputs` structs) and don't have to match +the names in `b.declareArgs(...)` (which are the **TTIR-level** arg names). +By convention they often do — pick whatever reads best at the boundary. + +The arg spec inside `b.declareArgs(...)` is a struct literal — its variant +*tags* (`.ptr`, `.scalar`, etc.) are comptime-known per call site, but the +*payload values* (`cfg.input_dtype`, `cfg.output_dtype`, …) can be runtime +values. This is what makes the same form work for both comptime-pinned +configs (offline tools, simple kernels) and runtime configs (production +models where dtypes/sizes come from `Tensor.dim()` / `Options`). + +> **Why the wrapper struct.** `Cfg` has to be a *named* type so the typed +> `run: *const fn (*Builder, Cfg) FinishError!void` field in the spec can +> reference it. Wrapping the kernel in `pub const MyKernel = struct {...}` +> keeps `Cfg`, `Kernel`, and `run` scoped together and exposes them all to +> callers (`MyKernel.Cfg`, `MyKernel.Kernel`, plus everything the inner +> Kernel re-exports — `MyKernel.Kernel.Config`, `.Inputs`, `.Outputs`, +> `.Results`, `.CallOpts`). + +### Two ways to use a kernel + +**Offline / tooling** — get the TTIR string (no MLIR context needed; `emit` +spins up and tears down its own): + +```zig +const ttir = try MyKernel.Kernel.emit(allocator, .{ .BLOCK = 1024 }); +defer allocator.free(ttir); +``` + +**Inside a ZML `forward(...)`** — emit TTIR and drop a `stablehlo.custom_call` +into the model in one shot: + +```zig +const out = MyKernel.Kernel.call( + .{ .x_ptr = x_tensor, .y_ptr = y_tensor, .n = n_tensor }, // Inputs + .{ .out = x_tensor.shape() }, // Outputs (Shapes) + .{ + .cfg = .{ .BLOCK = 1024 }, + .grid = .{ ceilDiv(n, 1024), 1, 1 }, + .num_warps = 4, + .num_stages = 2, + }, +).out; +``` + +`Kernel.call` returns a `Results` struct (one `Tensor` per output name); +read fields by name (`out.out`, `align_outs.sorted_token_ids`, …). + +--- + +## `zml.kernel.triton.Kernel(Config, spec)` — the declarative form + +`zml.kernel.triton.Kernel(Config, spec)` is a comptime function returning a +kernel type. `Config` is the per-call config type (used both for the +generated `Inputs`/`Outputs`/`CallOpts` and for the typed `run` signature); +`spec` is a `Spec(Config)` literal: + +| Field | Type | Meaning | +|-------------|---------------------------------------------------|---------------------------------------------------------------------| +| `.name` | `[:0]const u8` | Kernel name (used as the `tt.func` symbol). | +| `.inputs` | `[]const [:0]const u8` | Field names for the call-site `Inputs` struct (fields are `Tensor`). Order must match the `tt.func` arg order for input pointers. | +| `.outputs` | `[]const [:0]const u8` | Field names for the `Outputs` (struct of `Shape`) and `Results` (struct of `Tensor`). Order must match the `tt.func` arg order for output pointers — XLA's lowering passes inputs first, then outputs. | +| `.run` | `*const fn (*Builder, Config) FinishError!void` | Typed function pointer to the kernel body. Wrong signature → compile error here, not at first call. | + +Wrong arity, missing field, typo'd name on the call site → compile error +at the call site (because `Inputs` / `Outputs` are real struct types, not +tuples). + +`run` is responsible for declaring the kernel's args (via +`b.declareArgs(...)`) and emitting the body. It emits ops via `b.*` helpers +and never returns a string; `Kernel.emit(...)` does the +terminate-verify-serialize dance after `run` returns. + +--- + +## Calling kernels from a ZML model — `K.Kernel.call(...)` + +Inside a model's `forward(...)`, drop in a Triton kernel alongside regular +ZML ops with `K.Kernel.call(...)`: + +```zig +pub fn forward(self: Model, x: zml.Tensor, y: zml.Tensor) zml.Tensor { + const x_pre = x.mul(2.0); // stablehlo + + const out = MyKernel.Kernel.call( + .{ .x_ptr = x_pre, .y_ptr = y, .n = n_tensor }, // Inputs (by name) + .{ .out = x_pre.shape() }, // Outputs (Shapes) + .{ + .cfg = .{ .BLOCK = 1024 }, // runtime config + .grid = .{ ceilDiv(x.dim(0), 1024), 1, 1 }, + .num_warps = 4, + .num_stages = 2, + }, + ).out; // .out: Tensor + + return out.relu(); // stablehlo +} +``` + +`call` is a thin wrapper over `Kernel.emit(...)` + `zml.ops.triton(...)`. +It must be called from inside a `Compilation` (i.e., during +`zml.Compiler.compile(...)`) — `emit` builds the TTIR string in its own +throwaway context; `zml.ops.triton(...)` drops the `stablehlo.custom_call` +into the model's context. For offline tools, `Kernel.emit(allocator, cfg)` +gives you just the TTIR string. + +Signature: `Kernel.call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results`. + +| Positional arg | Type | Meaning | +|----------------|------------------|-------------------------------------------------------------------------| +| `inputs` | `Kernel.Inputs` | Struct of `Tensor`, one field per name in `spec.inputs`. | +| `outputs` | `Kernel.Outputs` | Struct of `Shape`, one field per name in `spec.outputs`. | + +`Kernel.CallOpts` (typed struct, all fields named): + +| Field | Type | Meaning | +|---------------------------|-----------------------|------------------------------------------| +| `.cfg` | `Kernel.Config` | Per-call kernel config. | +| `.grid` | `[3]i32` | Triton launch grid `(grid_x, grid_y, grid_z)`. | +| `.num_warps` | `i32` | Triton launch param. | +| `.num_stages` | `i32` | Triton launch param. | +| `.output_operand_aliases` | `?zml.ops.CustomCallOutputOperandAliases(Inputs, Outputs)` (default `null`) | For in-place updates: each output field names the input it aliases — `.{ .out = .x_ptr }`. | +| `.debug` | `bool` (default false)| Emit `tt.debug = true` on the custom call. | + +Returns `Kernel.Results` — a struct of `Tensor` named like `spec.outputs`. +Read by name: `out.out`, `attn_output.segm_max`, etc. + +The `cfg` value can be a comptime literal *or* a runtime expression built +from `Tensor.dim()` / `Options` fields. The kernel TTIR is regenerated per +call (cached by the MLIR canonicalizer / CSE pass at the XLA layer). + +> **Why named `Inputs` / `Outputs`.** The previous form took +> `anytype` tuples and matched by position; misordering 27 args (typical +> in `unified_attention.zig`) was a silent corruption hazard. With +> generated structs (`Inputs = StructOf(spec.inputs, Tensor)`, +> `Outputs = StructOf(spec.outputs, Shape)`), Zig refuses unknown fields, +> missing fields, and wrong-typed values at the callsite. + +--- + +## Argument specs + +Each field passed to `b.declareArgs(...)` is an `ArgSpec.Kind` tagged-union +literal. + +The four arg kinds mirror what TTIR's `tt.func` can carry: + +| Kind literal | MLIR type | Use case | +|---------------------------------------------------------|--------------------------|---------------------------------| +| `.{ .ptr = .f32 }` | `!tt.ptr` | Pointers (default divisibility) | +| `.{ .scalar = .i32 }` | plain `i32` | Strides, sizes, flags | +| `.{ .tensor = .{ &.{64, 128}, .f32 } }` | `tensor<64x128 x f32>` | Pre-built tensor input (rare) | +| `.{ .ptr_opts = .{ .dtype = .f32, .divisibility = 16 } }` | `!tt.ptr` with override | Custom `address_space`/divisibility | +| `.{ .scalar_opts = .{ .dtype = .i32, .divisibility = 16 } }` | plain `i32` with `tt.divisibility = 16 : i32` | Scalar with specialization hint | + +`.ptr` gets a default `tt.divisibility = 16` hint (matching Python Triton's +default). Use `.ptr_opts` to change that (or set `divisibility = null` to +suppress). `.scalar` carries no extra hint; switch to `.scalar_opts` when +you want to declare the same `tt.divisibility` Python's JIT auto-attaches +to int args whose runtime value is a multiple of 16. + +**Runtime dtypes:** the variant *tags* (`.ptr`, `.scalar`, etc.) must be +fixed per kernel — every call to `b.declareArgs(...)` for a given kernel +uses the same struct shape. The variant *payloads* (the dtype enum, the +divisibility int) can be runtime values from `cfg`: +`.a_ptr = .{ .ptr = cfg.a_dtype }` works fine even when `cfg.a_dtype` is a +runtime field. + +--- + +## The `Value` type + +```zig +pub const Value = struct { + inner: *const mlir.Value, + kernel: ?*Builder = null, + ... +}; +``` + +- `inner` is the underlying MLIR SSA handle; reach for it when you need to call + into raw `ttir.*` / `arith.*` constructors. +- `kernel` is the owning `Builder` pointer; all fluent methods route through it. + The DSL populates `kernel` automatically for every value it creates (`emit`, + `arg`, loop-region args, reduction-region args, etc.). If you construct a + `Value` by hand (`.{ .inner = raw }`), the fluent methods will panic — use + `Builder.*` static helpers instead. + +Introspection methods always available: + +| Method | Returns | Notes | +|-------------------|-----------------------------------|-----------------------------------------| +| `type_()` | `*const mlir.Type` | Full MLIR type (scalar or tensor). | +| `elemType()` | `*const mlir.Type` | Element type; same as `type_()` for scalars. | +| `isTensor()` | `bool` | `true` for ranked tensors. | +| `rank()` | `usize` | `0` for scalars. | +| `dim(i)` | `i64` | Traps on scalars. | +| `shape()` | `Shape` (BoundedArray, `MAX_RANK`)| Use `.constSlice()` to feed `[]const i64`. | +| `isFloatElem()` | `bool` | True for any float element type. | +| `isIntElem()` | `bool` | True for any integer element type. | + +--- + +## Fluent methods on `Value` + +Every fluent method returns a new `Value`. They dispatch to +`arith.addi` / `arith.addf` / ... based on the element type. The RHS is +`anytype` — see [Polymorphic scalars](#polymorphic-scalars). + +### Arithmetic + +| Method | Int op | Float op | Notes | +|----------------|--------------|--------------|------------------------------------| +| `.add(rhs)` | `arith.addi` | `arith.addf` | | +| `.sub(rhs)` | `arith.subi` | `arith.subf` | | +| `.mul(rhs)` | `arith.muli` | `arith.mulf` | | +| `.div(rhs)` | `arith.divsi`| `arith.divf` | Signed integer divide. | +| `.rem(rhs)` | `arith.remsi`| `arith.remf` | Signed integer remainder. | +| `.cdiv(rhs)` | `arith.ceildivsi` | — | Int only — for `tl.cdiv`. | +| `.minimum(rhs)`| `arith.minsi`| `arith.minnumf` | Matches `tl.minimum` (non-NaN-prop). | +| `.maximum(rhs)`| `arith.maxsi`| `arith.maxnumf` | Matches `tl.maximum` (non-NaN-prop). | +| `.bitAnd(rhs)` | `arith.andi` | — | Also handles i1 masks. | +| `.bitOr(rhs)` | `arith.ori` | — | | + +### Comparisons + +All return i1 (scalar) or `tensor<... x i1>`. Signed integer predicates by +default; `.olt` / `.oge` / ... for floats. + +| Method | Int predicate | Float predicate | +|--------------|---------------|-----------------| +| `.lt(rhs)` | `.slt` | `.olt` | +| `.le(rhs)` | `.sle` | `.ole` | +| `.gt(rhs)` | `.sgt` | `.ogt` | +| `.ge(rhs)` | `.sge` | `.oge` | +| `.eq(rhs)` | `.eq` | `.oeq` | +| `.ne(rhs)` | `.ne` | `.one` | + +### Casts — `.to(dtype)` + +`v.to(.f32)` chooses the right MLIR op based on the current vs target kind: + +| Source kind | Target kind | MLIR op | +|-------------|------------------|-------------------------------| +| int | wider int | `arith.extsi` | +| int | narrower int | `arith.trunci` | +| int | float | `arith.sitofp` | +| **i1** | **float** | **`arith.uitofp`** (i1 is unsigned bool; sign-extending `1` would land at `-1.0`) | +| float | int | `arith.fptosi` | +| float | wider float | `arith.extf` (no rounding) | +| float | narrower float | fp8 involved or custom rounding → `tt.fp_to_fp`; otherwise `arith.truncf` | +| same type | same type | no-op (returns `self`) | + +Triton's verifier rejects `tt.fp_to_fp` with a rounding attribute when the +target is not strictly narrower than the source (e.g. `bf16 → f32`, or +same-width `bf16 ↔ f16`). The DSL routes strict upcasts through +`arith.extf` to match Python Triton's own lowering; only genuine +downcasts carry the `rtne` rounding attribute. + +For narrowing FP→FP, the dispatch follows +`triton/python/triton/language/semantic.py:cast()`: standard FP narrowing +between fp64 / fp32 / fp16 / bf16 lowers to `arith.truncf` (rank-independent +— scalar and tensor both), while `tt.fp_to_fp` is reserved for cases where +fp8 is on either side or the caller requested a non-default +`fp_downcast_rounding`. + +### Constants + +There is no constant cache and no hoisting: every lifted literal emits its +own `arith.constant` where it is used (see Pitfall 5). `zeros(shape, dtype)`, +`ones(shape, dtype)`, `full(shape, value, dtype)` and `splat` of a literal +emit a scalar `arith.constant` and a `tt.splat`; `splat(Value, shape)` emits +the `tt.splat` alone. + +### Shape / pointer helpers + +| Method | Effect | +|------------------------------|-------------------------------------------------------------| +| `v.expandDims(axis)` / `b.expandDims(v, axis)` | `tt.expand_dims` — insert a size-1 axis at `axis`. Mirrors `v[:, None]` (`axis=1`) / `v[None, :]` (`axis=0`). | +| `b.broadcastTo(v, shape)` | `tt.broadcast` — explicit broadcast (size-1 dims → target). Rarely needed; auto-broadcast handles it. | +| `.splatTo(shape)` | `tt.splat` this scalar to the given shape. | +| `.addPtr(offset)` | `tt.addptr`; `offset` can be a Value or a comptime int. | + +> **Auto-broadcast is everywhere.** Mirrors Triton's +> `semantic.broadcast_impl_value` (`triton/python/triton/language/semantic.py:724`): +> rank-unification (prepend size-1 axes) + size-1 → size-N expansion. +> Applied transparently at: +> - every fluent binop (`.add`, `.mul`, `.lt`, `.bitAnd`, …) +> - `.addPtr(offset)` (either side may be scalar or rank-mismatched) +> - `b.where(cond, x, y)` / `b.select(...)` +> - `mask` and `other` of `b.loadOpts` / `b.storeOpts` +> +> So `v[:, None] * stride + w[None, :] * other_stride` is just: +> +> ```zig +> v.expandDims(1).mul(stride).add(w.expandDims(0).mul(other_stride)) +> ``` +> +> No `broadcastTo`, no full-shape `tt.splat` — the broadcast materializes +> implicitly at the `addi`. This matches Python's emit pattern op-for-op +> (Python's frontend never emits an eager `tt.broadcast` followed by a +> `arith.muli` either; it emits `expand_dims` + `splat + muli`). +> +> Reach for `b.broadcastTo` only when the consumer is a low-level op that +> bypasses the DSL's auto-broadcast — direct `arith.*` / raw `ttir.*` +> escape-hatch paths. + +--- + +## Polymorphic scalars + +Every fluent method's RHS (and every `b.splat` / `b.lift` input) is `anytype`. +The lift rule: + +- `Value` → pass through. +- `comptime_int` / `comptime_float` → a constant whose element type matches + the LHS operand. `v_i64.mul(16)` lifts `16` to i64, `v_f32.add(1.0)` lifts + `1.0` to f32, and so on. +- Runtime `i8/i16/i32`, `i64`, `f16/f32/f64`, etc. → a constant that preserves + the source Zig width. i8/i16/i32 (signed or unsigned) → i32 const; i64/u64 → + i64 const; f16 → f16; f32 → f32; f64 → f64. Unsigned ints are bit-cast + (bit pattern preserved), so `@as(u32, 0xFFFF_FFFF)` becomes an i32 with the + same all-ones bit pattern. + +This is why the common Python idiom `offs < 64` works in Zig as +`offs.lt(64)` without any wrapper — as long as the comptime scalar can be +represented in `offs`'s element type. + +If you need to force a specific width for a runtime variable, cast at the call +site: `pid.mul(@as(i32, @intCast(block_size)))`. + +### When is `@as` actually needed? (Rule of thumb) + +Auto-lift covers most cases. `@as(...)` around a binop rhs is **redundant** +in all of these — drop it: + +| Pattern | Auto-lifts to… | +|------------------------------------------|---------------------------------------| +| `v_i64.eq(0)` / `.div(16)` / `.add(-1)` | comptime_int → match lhs elem type | +| `v_f32.mul(1.0 / config.fp8_max)` | already-f32 expression → f32 const | +| `v_i64.lt(NUM_QUERY_HEADS)` (Zig i64) | runtime i64 → i64 const | +| `v_i32.add(pad_term)` (Zig i32) | runtime i32 → i32 const | +| `b.splat(-1, shape)` / `b.openFor(0, N, 1, ...)` | comptime → matches elem / the first Value bound | +| `b.constant(.i32, 0)` | constant takes anytype — wrap nothing | + +`@as(...)` **is** required exactly when you're narrowing a runtime Zig int +to a smaller width that `lift` won't pick on its own. The idiom is +`@as(i32, @intCast(X_i64))` — you need the `@as` because `@intCast` has no +target type to infer, and without the narrowing `lift` would emit an i64 +constant, which widens the i32 lhs and the result to i64: + +```zig +// BLOCK_Q is a Zig i64 (from config). lhs is i32. Must narrow. +q_start_raw.div(@as(i32, @intCast(BLOCK_Q))).add(seq_idx) +``` + +The alternative — `q_start_raw.to(.i64).div(BLOCK_Q).to(.i32)` — mirrors +Python's `.to(tl.int64)` style but costs two extra Value-level casts; prefer +the narrowing `@as` or a pre-declared `const FOO_I32: i32 = @intCast(FOO);` +when the same width cast recurs. + +### When do you need `b.lift` / `b.constant`? + +Almost never. The DSL auto-lifts rhs for every fluent binop, `b.splat`, +`b.full`, `b.openFor` bounds, `.addPtr` offsets, etc. Reach for `b.lift` / +`b.constant` **only** when the DSL surface demands an existing `Value` and +there's no binop sibling to match against — typically: + +- Loop seeds for `openWhile(.{inits}, ...)` where you start with a raw + comptime literal: `const left_init = b.constant(.i32, 0);`. +- Mutable `var x: Value = b.constant(.i32, 0);` that a later branch may + reassign — the var needs an initial Value even if the comptime path would + work on its own. + +If the constant is about to flow through any `.add` / `.mul` / `.splatTo` +etc., skip the lift and let auto-lift do it. And if a stride is hardcoded +to 1, **delete it**: `x.mul(1)` is a no-op, the DSL isn't Python `constexpr` +where such names are required. + +### Scalar ↔ tensor auto-broadcast + +Fluent arithmetic and comparison ops (`.add`, `.sub`, `.mul`, `.div`, `.rem`, +`.cdiv`, `.bitAnd`, `.bitOr`, `.minimum`, `.maximum`, `.lt`, `.le`, `.gt`, +`.ge`, `.eq`, `.ne`) are symmetric: if one operand is a tensor and the other a +scalar, the scalar is splatted to match. Either direction works: + +```zig +const offs = iv.add(cols); // scalar iv + tensor cols → tensor +const offs = cols.add(iv); // tensor cols + scalar iv → tensor +``` + +This mirrors Triton/NumPy's `i + cols`. **Size-1 and rank-unification +broadcasting also work** (mirroring Python's +`semantic.broadcast_impl_value`): a `(M, 1)` and a `(1, N)` operand +broadcast to `(M, N)` automatically, and a `(M,)` lined up against a +`(M, N)` is first `expand_dims`'d to `(1, M)` and then broadcast. So +`offs_token[:, None] + offs_k[None, :]` is just: + +```zig +const offs = b.expandDims(offs_token, 1).add(b.expandDims(offs_k, 0)); +``` + +with no `b.broadcastTo` calls needed. `b.where` / `b.select` and the +`mask` / `other` operands of `b.loadOpts` / `b.storeOpts` also auto-broadcast, +so the only time `b.broadcastTo` is needed is when the consumer bypasses +the DSL (raw `arith.*` / `ttir.*` escape-hatch paths). + +`.addPtr` auto-splats in **both directions**, matching Triton's +`ptr + offset` semantics: a scalar pointer is splatted to match a tensor +offset (`x_ptr.addPtr(tensor_offs)`), and a scalar offset is splatted to +match a tensor pointer (`a_ptrs.addPtr(BLOCK_K)`). Pre-splatting via +`b.splat` is no longer required for the loop-stride case. + +### When is `scalar.splatTo(shape)` redundant? + +Because fluent binops (`.add` / `.mul` / …) already auto-splat scalars +against tensor siblings, and `.addPtr` auto-splats either side (scalar +pointer or scalar offset) against a tensor partner, a pre-emptive +`scalar.splatTo(shape)` is usually noise: + +```zig +// redundant +pid_n.mul(N).splatTo(shape).add(b.arange(0, N, .i64)).rem(n_block.splatTo(shape)) +// equivalent — scalars auto-splat against the tensor sibling +pid_n.mul(N).add(b.arange(0, N, .i64)).rem(n_block) +``` + +Keep the explicit `splatTo` only when the scalar feeds into a raw +`arith.*` / `ttir.*` escape-hatch op that doesn't go through the DSL's +auto-broadcast. + +--- + +## Comptime vs runtime widths + +TTIR has no implicit int-width conversion — `arith.muli(i32, i64)` is a verifier +error. The DSL mirrors Python and **auto-promotes** integer binops when the +two runtime operands have different widths: it `extsi`s the narrower side +to the wider, matching `semantic.integer_promote_impl`. So +`v_i32.mul(v_i64)` works without a manual `.to(.i64)` — it emits exactly +the same `extsi` + `muli` pair Python would. + +For the rare case where you want explicit control, you can still cast +yourself; the auto-promotion is a no-op when widths already agree. + +Recurring example from `zml/kernels/moe/triton/moe.zig`: + +```zig +// pid (i32) vs block (i64 because it came from a usize config) +const block_i32: i32 = @intCast(block); +const token_start_init = pid.mul(block_i32); +// alternative: pid.to(.i64).mul(block) +``` + +For pointer math it is often cleaner to promote everything to i64 up front: + +```zig +const pid = b.programId(.x).to(.i64); +const offs = b.arange(0, BLOCK, .i64); +``` + +--- + +## Shape helpers + +| Helper | Effect | +|----------------------------------------|--------------------------------------------------| +| `b.arange(start, end, dtype)` | `tl.arange(start, end).to(dtype)`. Pass `.i32` for no-op cast. | +| `b.zeros(shape, dtype)` | `tl.zeros(shape, dtype=dtype)`. | +| `b.ones(shape, dtype)` | Ones tensor. | +| `b.splat(value, shape)` | `tl.splat`; accepts Value or comptime scalar. | +| `b.expandDims(v, axis)` / `v.expandDims(axis)` | `tt.expand_dims` — insert a size-1 axis. Mirrors `v[:, None]` (`axis=1`) / `v[None, :]` (`axis=0`). | +| `b.mask2d(cond_m, cond_n, _, _)` | `cond_m[:, None] & cond_n[None, :]` — emits the `(m,1) & (1,n)` form; auto-broadcast at the next consumer. | +| `b.lift(value)` | Wrap a Zig scalar as a DSL Value; dtype inferred from source. | +| `b.constant(dtype, value)` | Wrap a Zig scalar as a DSL Value of a specific `DType`. | +| `b.constantLike(value, like)` | Wrap a Zig scalar as a DSL Value of `like`'s element type. | + +--- + +## Loads and stores + +Load helpers infer everything from the pointer's type — scalar `!tt.ptr` +loads a scalar `T`, `tensor<... x !tt.ptr>` loads a `tensor<... x T>`. +No `result_type` / `shape` / `dtype` arguments required. + +| Helper | Python equivalent | +|-------------------------------------------------|-------------------------------------------| +| `b.load(ptr)` | `tl.load(ptr)` — scalar or tensor. | +| `b.loadOpts(ptr, .{ .mask, .other, ... })` | `tl.load(ptrs, mask=..., other=..., cache_modifier=..., eviction_policy=..., volatile=...)`. | +| `b.store(ptr, value)` | `tl.store(ptr, value)`. | +| `b.storeOpts(ptr, value, .{ .mask, ... })` | `tl.store(ptr, value, mask=..., cache_modifier=..., eviction_policy=...)`. | + +`LoadOpts` / `StoreOpts` fields (`.mask`, `.other`, `.cache_modifier`, +`.eviction_policy`, `.@"volatile"`) take DSL-level `Value`s / enum literals — +no raw MLIR types needed. + +> **Match Python's emit form for masked loads.** `tl.load(ptr, mask=m, +> other=0.0)` lowers to a 3-operand `tt.load` op. To match it byte-for-byte +> in the Zig DSL, pass an explicit `.other`: +> +> ```zig +> const zero = b.zeros(&.{BLOCK}, dtype); +> const x = b.loadOpts(ptr, .{ .mask = mask, .other = zero }); +> ``` +> +> Calling `loadOpts` with `.mask` only (no `.other`) emits a 2-operand +> `tt.load` — semantically equivalent (out-of-bounds lanes are undefined, +> not zero), but the TTIR/TTGIR diff vs Python will not be empty. Triton's +> `tl.load(ptr, mask=...)` without `other=` lowers to the 2-operand form, +> so you only need the `.other` operand when the Python source explicitly +> passes `other=`. + +### Result-type inference + +Most helpers that produce a new Value infer their result type from the inputs, +so user code never constructs `*const mlir.Type` manually: + +| Helper | How the result type is derived | +|-----------------------------------------------------|---------------------------------------------| +| `b.load(ptr)` / `b.loadOpts(ptr, opts)` | From `ptr.type_()` (scalar or tensor of ptrs). | +| `b.dot(a, b, acc)` / `b.dotOpts(a, b, acc, opts)` | `acc.type_()`. | +| `b.dotScaled(...)` / `b.dotScaledOpts(...)` | `acc.type_()`. | +| `b.reshape(src, new_shape)` / `b.reshapeOpts(...)` | `new_shape` + `src` element type. | +| `b.gather(src, indices, axis)` | `indices` shape + `src` element type. | +| `b.histogram(src, num_bins)` / `b.histogramOpts(...)` | `tensor`. | +| `b.fpToFp(src, out_dtype)` / `b.fpToFpOpts(...)` | `src` shape + `out_dtype`. | +| `b.convertf(src, out_dtype)` / `b.convertfOpts(...)`| `src` shape + `out_dtype`. | +| `b.scalingTruncf(src, scale, out_dtype)` / …Opts | `src` shape + `out_dtype`. | +| `b.descriptorLoad(desc, indices, shape, dtype)` | `shape` + `dtype`. | + +Rule of thumb: you only supply things the op needs beyond what inputs carry — +a `new_shape`, a `num_bins`, an `out_dtype`, a TMA tile descriptor. The DSL +never asks you to restate information it can read off a `Value`. + +`b.scalarTy(dtype)` / `b.tensorTy(shape, dtype)` are still available but only +needed for tensor result types in `openIfElse` / `openWhile` (scalar ones are +plain dtypes) and by the raw `externElementwise` / `inlineAsmElementwise` +escape hatches. + +### `fn X / fn XOpts` pattern + +Every Builder method whose Python Triton counterpart has named keyword-only +arguments comes in two flavours: + +- `b.X(non_named_args...)` — defaults applied; matches `tl.X(args...)` in + Python, which in Python would use the zero-arg defaults. +- `b.XOpts(non_named_args..., struct { named... })` — full named-param set + as a typed Zig struct literal with field defaults matching Triton. + +Pairs in place today: `load` / `loadOpts`, `store` / `storeOpts`, +`dot` / `dotOpts`, `reshape` / `reshapeOpts`, +`atomicRmw` / `atomicRmwOpts`, `atomicCas` / `atomicCasOpts`, +`fpToFp` / `fpToFpOpts`, `clampf` / `clampfOpts`, +`cast` / `castOpts`, `cat` / `catOpts`, +`convertf` / `convertfOpts`, `scalingTruncf` / `scalingTruncfOpts`, +`dotScaled` / `dotScaledOpts`, +`inlineAsmElementwise` / `inlineAsmElementwiseOpts`, +`externElementwise` / `externElementwiseOpts`, +`descriptorLoad` / `descriptorLoadOpts`, `histogram` / `histogramOpts`, +`devicePrint` / `devicePrintOpts`, +`deviceAssert` / `deviceAssertOpts`, +`maximum` / `maximumOpts`, `minimum` / `minimumOpts`, +`sum` / `sumOpts`, `max` / `maxOpts`, `min` / `minOpts`, +`cumsum` / `cumsumOpts`, `cumprod` / `cumprodOpts`, +`openFor` / `openForOpts` (`tl.range(..., num_stages=...)`). + +DSL-only pairs (no Python kwarg counterpart — `.inline_return` is about IR +block layout, see the `returnIf` / `returnIfOpts` section under Control flow): +`returnIf` / `returnIfOpts`, `openReturnIf` / `openReturnIfOpts`. + +Ops with no kwargs in Python (pure positional): `gather(src, indices, axis)`, +`trans(src, order)`, `permute(src, order)`, `cat(lhs, rhs)` (dim=0 default), +`split(src)`, `join(lhs, rhs)`, `item(src)`, `broadcast(a, b)` (symmetric), +`broadcastTo(value, shape)`, `where(cond, x, y)`, `zeros(shape, dtype)`, +`full(shape, value, dtype)`. + +--- + +## Reductions and scans + +```zig +const absmax = b.max(abs_y); // tl.max(abs_y) — axis=None (all dims) +const total = b.sum(padded_counts); // tl.sum(padded_counts) +const cumul = b.cumsum(x); // tl.cumsum(x, axis=0) + +// Keyword arguments → the *Opts variant with a typed struct literal. +const axis0_max = b.maxOpts(abs_y, .{ .axis = 0 }); // tl.max(x, axis=0) +const absmax_kd = b.maxOpts(abs_y, .{ .axis = 0, .keep_dims = true }); // tl.max(x, 0, keep_dims=True) +const rev_cum = b.cumsumOpts(x, .{ .reverse = true }); // tl.cumsum(x, 0, reverse=True) +``` + +Under the hood these build the required `tt.reduce` / `tt.scan` region with a +matching `arith.addf` / `arith.addi` / `maxnumf` / `maxsi` combine function — +no more hand-rolled combiner closures for the common cases. + +> **Gotcha — `maxnumf` vs `maximumf` (and `minnumf` vs `minimumf`):** +> Triton's `tl.max`, `tl.min`, `tl.maximum`, `tl.minimum` all default to +> the **non-NaN-propagating** form (`arith.maxnumf` / `arith.minnumf`). +> The DSL fluent forms match Python: +> +> | Python | DSL | MLIR op | +> |-------------------------------------|--------------------------------------|-------------------| +> | `tl.max(x)` | `b.max(x)` / `x.max()` | `arith.maxnumf` | +> | `tl.min(x)` | `b.min(x)` / `x.min()` | `arith.minnumf` | +> | `tl.maximum(a, b)` | `a.maximum(b)` / `b.maximum(a, b)` | `arith.maxnumf` | +> | `tl.minimum(a, b)` | `a.minimum(b)` / `b.minimum(a, b)` | `arith.minnumf` | +> | `tl.maximum(a, b, propagate_nan=NAN_ALL)` | `a.maximumOpts(b, .{ .propagate_nan = .all })` | `arith.maximumf` | +> +> The lower-level builders **`b.maximumf(a, b)` / `b.minimumf(a, b)` skip +> Python's defaults** and emit IEEE-754 NaN-propagating ops directly. Reach +> for them only when you genuinely want NaN propagation; otherwise stick +> with `b.maximum` / `b.minimum` (or the fluent `Value.maximum` / +> `Value.minimum`). Mismatched ops here cause subtle TTIR/LLIR diffs vs +> Python and are a frequent porting bug. + +When `.keep_dims = true`, the DSL inserts a trailing +`tt.expand_dims` / `splat` so the result shape matches the input rank with a +size-1 reduced axis (matching Python Triton). For `axis = null` (the default), +the reduction goes through a `tt.reshape allow_reorder` first — even on rank-1 +input — so the layout-assignment pass picks the same reduction-friendly +layout Python's frontend gets. + +For custom combiners, use the low-level `b.reduce(ctx, .{ .src, .axis, .elem, +.result, .combine })` and `b.scan(ctx, .{ .src, .axis, .elem, .result, +.combine, .reverse })` — they take a `fn(*Builder, Value, Value, CtxT) Value` +closure for `combine` and give you full control. + +--- + +## Control flow + +SCF regions are built with **scope-based** builders — `b.openFor(...)`, +`b.openIf(...)` / `b.openIfElse(...)`, `b.openWhile(...)` return a typed scope +value. You emit body ops into a bare `{}` block, terminate with `yield` / +`yieldThen` / `yieldAfter` (or `close()` for a scope without values), and read +results from `scope.results` afterward. +No `ctx` struct, no function literal, no `kk`/`k` split — lexical scope +carries captures because everything runs in the same function. + +### `openFor` — scf.for with iter_args + +```zig +var loop = b.openFor(0, N, BLOCK, .{acc0}); +{ + const iv = loop.iv; + const acc = loop.carried[0]; + const offs = iv.add(cols); + const tile = b.loadOpts(x_ptr.addPtr(offs), .{ .mask = offs.lt(N) }); + const partial = tile.sum(); + loop.yield(.{ acc.add(partial) }); +} +const total = loop.results[0]; +``` + +- `lower`/`upper`/`step` are `anytype`: pass Values, comptime ints, or runtime + Zig ints. A bound that isn't a Value takes the type of the first one that is; + with none, `lower` is lifted (`i32` when it fits) and the others follow it. + Pass a typed constant (`b.constant(.i64, 0)`) to pick another width. +- `inits` is a tuple `.{v1, v2, ...}` or an array. Arity is comptime — + `carried`, `results`, and the `yield(...)` values (also a tuple or an array) + all have the same length. +- Empty loops: `b.openFor(0, N, S, .{})` + `loop.close()` (no iter_args). +- `b.openForOpts(lower, upper, step, inits, .{ .num_stages = 3 })` sets the + loop's options (`tt.num_stages`, `.unsigned_cmp`). + +### `openIf` — scf.if without else (side-effects only) + +```zig +var i = b.openIf(cond); +{ + // then body + i.close(); // closes scope, builds scf.if with empty else +} +``` + +Use for conditional side-effects (no results). There is no else branch — the +else block is auto-built empty. + +### `openIfElse` — scf.if with else and optional results + +```zig +var i = b.openIfElse(cond, .{.f32}); // result types +{ + // then body + i.yieldThen(.{ x }); +} +{ + // else body + i.yieldElse(.{ y }); +} +const r = i.results[0]; +``` + +- `result_types` is a tuple or array of dtypes (`.f32`), Values (for their + types) or `*const mlir.Type`s such as `b.tensorTy(shape, dtype)`. Pass + `.{}` for an if/else with no results but distinct then/else bodies, then + end them with `elseBranch()` and `close()`. +- `yieldThen` swaps to the else block; `yieldElse` builds the `scf.if` op. +- Both branches must yield arity-matching tuples. + +### `openWhile` — scf.while + +```zig +var w = b.openWhile(.{ i0 }, .{.i32}); +{ + // before region (condition) + const s = w.before_carried[0]; + w.yieldBefore(s.lt(10), .{ s }); // scf.condition: cond + values forwarded to after +} +{ + // after region (body) + const s = w.after_carried[0]; + w.yieldAfter(.{ s.add(1) }); // values forwarded back to before +} +const r = w.results[0]; +``` + +- `inits` — a tuple or array of `Value`s; before-region arg types = their types. +- `after_types` — a tuple or array of dtypes (`.f32`), Values (for their + types) or `*const mlir.Type`s; after-region arg types + the scf.while's + result types. +- `yieldBefore(cond, forwarded)` — `forwarded` arity must match `after_types`. +- `yieldAfter(values)` — arity must match `inits`. + +### `returnIf` / `returnIfOpts` — early `tt.return` via cf.cond_br + +```zig +// Python: if pid_m * BLOCK >= num_tokens_post_padded: return +const out_of_range = pid_m.mul(block).ge(num_tokens); +b.returnIf(out_of_range); +// ... ops here live in a fresh fall-through block ... +``` + +Lowers to: + +```mlir +cf.cond_br %out_of_range, ^ret, ^cont +^ret: + tt.return +^cont: + // subsequent ops +``` + +- Kernels return nothing, so neither form takes return values. +- Works **only at the top-level `tt.func` body** — don't call inside an + `scf.if`/`scf.for`/`scf.while` region (those must stay structured; cf + branches can't escape their parent scf region). +- After `returnIf`, the kernel's current block is swapped to a fresh + fall-through block — any subsequent `b.*` calls emit into that block, and + `b.finish()` appends the final `tt.return` there. +- Use this to match Python Triton's `if cond: return` lowering exactly. + Python emits `cf.cond_br` for early-return idioms while using `scf.for` / + `scf.if` for structured loops and value-carrying branches; our DSL does + the same. + +**`returnIfOpts(cond, .{ .inline_return = true })` — match Python's +per-`return` block layout.** By default `returnIf` routes every return path +(every early `returnIf` / `openReturnIf` plus the natural fall-through from +`b.finish`) through one **shared exit block** holding the single `tt.return`. +Python's frontend does the opposite: each `if cond: return` (and the natural +exit) gets its **own** `tt.return`-only block, and LLVM's SimplifyCFG later +merges them into a single `common.ret`. The two are semantically identical +but differ in the function-exit block's label and position — which renumbers +every `$L__BB` label downstream and blows up the PTX diff. When the Python source contains *any* literal `return` statement (it then +has ≥2 `ret` blocks in LLVM — the explicit one(s) plus the implicit +function-end one — which SimplifyCFG merges into `common.ret`), pass +`.{ .inline_return = true }` on *every* return site, including each +`openReturnIfOpts` (see below). Each one +then emits `tt.return` inline and SimplifyCFG reproduces Python's +`common.ret`. The default shared exit block emits exactly one `tt.return`, +so the merge never fires and the function-exit block lands somewhere else. + +### `openReturnIf` — early `tt.return` with side-effects in the taken branch + +```zig +// Python: if off_experts == -1: write_zeros_to_output(...); return +var scope = b.openReturnIf(is_dead); // or openReturnIfOpts, see `returnIfOpts` above +{ + writeZerosToOutput(k, c_ptr, ...); // ops here emit into ^ret + scope.close(); // closes ^ret with tt.return (or cf.br ^exit) +} +// ... ops here emit into ^cont (fall-through) ... +``` + +Lowers to the same `cf.cond_br ^ret, ^cont` shape as `returnIf`, but the +taken branch can carry arbitrary side-effects between the `cf.cond_br` and +the `tt.return`. Use this instead of the +`openIf(cond){body}close; returnIf(cond)` pair — that pattern +duplicates the predicate (emits both `scf.if` and `cf.cond_br` on the same +boolean) and drifts from Triton's reference IR. + +`openReturnIfOpts(cond, .{ .inline_return = true })` makes `close` end +`^ret` with `tt.return` directly instead of `cf.br ^exit`. Use it on every +`returnIf` / `openReturnIf` of a kernel when matching a Python source that has +literal `return` statements. + +Same restrictions as `returnIf`: top-level `tt.func` body only. + +### When to use `cf` (returnIf / openReturnIf) vs `scf` (openIf / openIfElse / openFor) + +| Python source | Emits | DSL builder | +|----------------------------------------|-------|---------------------------------| +| `if cond: return` | `cf` | `b.returnIf(cond)` | +| `if cond: ; return` | `cf` | `var s = b.openReturnIf(cond); {...; s.close()}` | +| `if cond: ` | `scf` | `b.openIf(cond) {...; close()}` | +| `if cond: x=A else: x=B` (value-carrying) | `scf` | `b.openIfElse(cond, .{ty}) {...; yieldThen(.{a})} {...; yieldElse(.{b})}` | +| `for i in range(...)` | `scf` | `b.openFor(lo, hi, step, .{inits})` | +| `while cond:` | `scf` | `b.openWhile(.{inits}, .{after_types})` | + +Rule of thumb: **`scf` for structured, value-carrying control. `cf` for +early-exits.** Both lower to identical PTX after the +`convert-scf-to-cf` pass — the difference only matters at the TTIR layer +(pattern-matching passes that run before lowering). + +### `reduce` / `scan` — custom combiners (still callback-based) + +```zig +b.reduce(ctx, .{ .src, .axis, .elem, .result, .combine = fn (*Builder, Value, Value, Ctx) Value }); +b.scan (ctx, .{ .src, .axis, .reverse = false, .elem, .result, .combine }); +``` + +These stay on the callback pattern because the combine fn is a tiny pure +function (`(a, b) -> a ⊕ b`) where a `ctx` struct is a cleaner fit than a +scoped builder. Reach for `b.sum` / `b.max` / `b.cumsum` / etc. first — the +built-in reductions cover the common cases. + +### General tips + +- Scopes stack naturally: `openFor { openIf { openFor { ... } } }` all just + push/pop the kernel's current-block stack (same for `openIfElse` / + `openWhile`). +- Scopes without values end with `close()`: `loop.close()`, `i.close()`, and + `elseBranch()` then `close()` for an if/else. +- After any `yield*`, the scope's `.results` field holds the scf op's results + as a `[N]Value`. + +--- + +## Escape hatches + +Everything the DSL doesn't expose is still reachable — `Builder` is a +book-keeping layer, not a walled garden: + +- `v.inner` — the raw `*const mlir.Value`; feed it to any `ttir.*` / + `arith.*` / `math.*` / `scf.*` constructor directly. +- `b.ctx` — the owning `*mlir.Context`. +- `b.currentBlock()` — the active `mlir.Block` (respects `pushBlock` / + `popBlock`). +- `b.emit(op)` / `b.emitMulti(op, n)` — if you've built an `mlir.Operation` + by hand, use these so the result comes back as a `Value` with its `kernel` + field populated (you'll need that for any subsequent fluent calls). + +When you drop to raw ops, remember to still **append them to the current +block** — `b.emit(...)` does that for you; `_ = op.appendTo(b.currentBlock())` +is the manual form. + +--- + +## Lower layer — `Builder.open` + +The declarative `zml.kernel.triton.Kernel(Config, spec)` form covers most +cases. For dynamic-arity arg lists, or escape-hatch scenarios where you want +to drive the IR construction yourself, use the `Builder` directly: + +```zig +const Builder = tri.Builder; + +pub fn fusedMoeKernel( + allocator: std.mem.Allocator, + ctx: *mlir.Context, + config: GenerationConfig, +) ![:0]const u8 { + var b = try Builder.open(allocator, ctx, "fused_moe_kernel"); + defer b.deinit(); + const a = try b.declareArgs(.{ + .a_ptr = .{ .ptr = config.a_dtype }, + .b_ptr = .{ .ptr = config.b_dtype }, + // ... + }); + + // ... emit body via b.* helpers; same DSL as inside `Kernel(...)` ... + + return b.finish(); +} +``` + +- `Builder.open(allocator, ctx, name)` creates an empty builder, and + `b.declareArgs(spec)` creates the `tt.func` and returns its arguments as + the same `NamedArgs(Spec)` struct as inside `Kernel(...)` — exactly what + `zml.kernel.triton.Kernel(...)` does internally. + `b.declareArgsOpts(spec, .{ .@"noinline" = true })` sets the function's + options. +- `b.finish()` terminates with `tt.return`, verifies, and serializes to a + `[:0]const u8` owned by `allocator`. `b.deinit()` frees the module. + +For dynamic-arg-count construction, `Builder.init(allocator, ctx, name, +args)` takes a runtime `[]const ArgSpec` instead of a comptime struct +literal; you grab block args with `b.arg(i)`. + +Pass these constructors a `zml.kernel.triton.newContext()`, not a +`Compilation`'s `mlir_ctx`; `defer ctx.deinit()` once you have the IR +string. `Kernel.emit` does this for you, which is why it takes no context. + +Prefer `zml.kernel.triton.Kernel(Config, spec)` for the common case. + +--- + +## Python Triton ↔ Zig DSL cheat-sheet + +| Python | Zig | +|-----------------------------------------------|--------------------------------------------------------------------| +| `tl.program_id(0)` | `b.programId(.x)` | +| `tl.num_programs(0)` | `b.numPrograms(.x)` | +| `tl.arange(0, N)` | `b.arange(0, N, .i32)` | +| `x.to(tl.int64)` | `x.to(.i64)` | +| `a + b`, `a * b`, `a // b` | `a.add(b)`, `a.mul(b)`, `a.div(b)` | +| `a < b` | `a.lt(b)` | +| `mask & other` | `mask.bitAnd(other)` | +| `tl.where(c, a, b)` | `b.where(c, a, b)` (alias of `b.select`) | +| `vec[:, None]` | `vec.expandDims(1)` | +| `vec[None, :]` | `vec.expandDims(0)` | +| `m[:, None] & (c[None, :] < N)` | `b.mask2d(m, c.lt(N), m_sz, n_sz)` | +| `ptr + offs` | `ptr.addPtr(offs)` (auto-splats scalar↔tensor either way) | +| `tl.load(ptr)` | `b.load(ptrs)` | +| `tl.load(ptr, mask=m)` | `b.loadOpts(ptrs, .{ .mask = m })` | +| `tl.load(ptr, mask=m, other=o)` | `b.loadOpts(ptrs, .{ .mask = m, .other = o })` | +| `tl.store(ptr, v)` | `b.store(ptr, v)` | +| `tl.store(ptr, v, mask=mask)` | `b.storeOpts(ptr, v, .{ .mask = mask })` | +| `tl.zeros((M, N), dtype=tl.float32)` | `b.zeros(&.{ M, N }, .f32)` | +| `tl.full((M, N), 3, tl.int32)` | `b.full(&.{ M, N }, 3, .i32)` | +| `tl.sum(x)` | `b.sum(x)` — axis=None default | +| `tl.sum(x, axis=0)` | `b.sumOpts(x, .{ .axis = 0 })` | +| `tl.sum(x, axis=0, keep_dims=True)` | `b.sumOpts(x, .{ .axis = 0, .keep_dims = true })` | +| `tl.max(x)` / `tl.min(x)` | `b.max(x)` / `b.min(x)` | +| `tl.max(x, axis=0)` | `b.maxOpts(x, .{ .axis = 0 })` | +| `tl.maximum(a, b)` / `tl.minimum(a, b)` | `a.maximum(b)` / `a.minimum(b)` or `b.maximum(a, b)` | +| `tl.cumsum(x)` / `tl.cumprod(x)` | `b.cumsum(x)` / `b.cumprod(x)` | +| `tl.cumsum(x, axis=0, reverse=True)` | `b.cumsumOpts(x, .{ .reverse = true })` | +| `tl.dot(a, b, acc=acc)` | `b.dot(a, b, acc)` — defaults to `input_precision = tf32` (NVIDIA) | +| `tl.dot(a, b, acc, input_precision="ieee")` | `b.dotOpts(a, b, acc, .{ .input_precision = .ieee })` | +| `tl.clamp(x, mn, mx)` | `b.clampf(x, mn, mx)` | +| `tl.clamp(x, mn, mx, propagate_nan=NAN_ALL)` | `b.clampfOpts(x, mn, mx, .{ .propagate_nan = .all })` | +| `tl.atomic_add(p, v)` | `b.atomicRmw(.add, p, v)` | +| `tl.atomic_add(p, v, mask=m, sem="relaxed")` | `b.atomicRmwOpts(.add, p, v, .{ .mask = m, .sem = .relaxed })` | +| `tl.histogram(x, num_bins)` | `b.histogram(x, num_bins)` | +| `tl.histogram(x, num_bins, mask=m)` | `b.histogramOpts(x, num_bins, .{ .mask = m })` | +| `tl.reshape(x, shape)` | `b.reshape(x, &.{ ...shape })` | +| `tl.reshape(x, shape, can_reorder=True)` | `b.reshapeOpts(x, &.{ ...shape }, .{ .can_reorder = true })` | +| `tl.cast(x, tl.bfloat16)` | `b.cast(x, .bf16)` (or fluent `x.to(.bf16)`) | +| `tl.cast(x, tl.bf16, fp_downcast_rounding="rtne")` | `b.castOpts(x, .bf16, .{ .fp_downcast_rounding = .rtne })` | +| `tl.cast(x, tl.int32, bitcast=True)` | `b.castOpts(x, .i32, .{ .bitcast = true })` | +| `src[indices]` (gather) | `b.gather(src, indices, axis)` | +| `tl.trans(x, dims)` / `tl.permute(x, dims)` | `b.trans(x, order)` / `b.permute(x, order)` | +| `tl.cat(a, b)` | `b.cat(a, b)` (dim=0 default) | +| `tl.cat(a, b, can_reorder=True)` | `b.catOpts(a, b, .{ .can_reorder = true })` | +| `tl.join(a, b)` / `tl.split(x)` | `b.join(a, b)` / `b.split(x)` | +| `tl.item(x)` | `b.item(x)` | +| `tl.abs(x)` | `b.abs(x)` (auto-dispatch) or fluent `x.abs()` | +| `tl.broadcast(a, b)` (symmetric) | `b.broadcast(a, b)` → `.{ Value, Value }` | +| `tl.broadcast_to(v, shape)` | `b.broadcastTo(v, &.{ ...shape })` | +| `tl.umulhi(a, b)` | `b.umulhi(a, b)` | +| `tl.sqrt_rn(x)` / `tl.div_rn(x, y)` | `b.sqrtRn(x)` / `b.divRn(x, y)` | +| `tl.device_print("dbg", args...)` | `b.devicePrint("dbg", args, is_signed)` | +| `tl.device_assert(cond, "msg")` | `b.deviceAssert(cond, "msg")` | +| `tl.inline_asm_elementwise(...)` | `b.inlineAsmElementwise(...)` | +| `tl.extern_elementwise(...)` | `b.externElementwise(srcs, shape, dtype, lib, path, sym)` | +| `for i in range(0, N, S): ...` | `var loop = b.openFor(0, N, S, .{inits...}); { ...; loop.yield(.{...}); }` | +| `while cond: ...` | `var w = b.openWhile(.{inits}, .{after_types}); { ...; w.yieldBefore(cond, .{...}); }; { ...; w.yieldAfter(.{...}); }` | +| `if cond: ...` (side-effects, no else) | `var i = b.openIf(cond); { ...; i.close(); }` | +| `if cond: ... else: ...` | `var i = b.openIfElse(cond, .{result_types}); { ...; i.yieldThen(.{...}); }; { ...; i.yieldElse(.{...}); }` | +| `if cond: return` (early exit) | `b.returnIf(cond);` — emits `cf.cond_br` + `tt.return`, continues in fall-through block (`b.returnIfOpts(cond, .{ .inline_return = true })` when matching a Python source that has literal `return`s) | +| `if cond: ; return` (early exit, with side-effects) | `var s = b.openReturnIf(cond); { ...; s.close(); }` — single `cf.cond_br`; body emits into `^ret` before the `tt.return` (`b.openReturnIfOpts(cond, .{ .inline_return = true })` to match, alongside `returnIfOpts`) | + +--- + +## Pitfalls + +1. **Mixed integer widths widen.** `v_i32.mul(v_i64)` is an i64 multiply + with an `arith.extsi` on the i32 side, and so is `v_i32.mul(x)` for a + runtime Zig `i64` `x`. Narrow with `@as(i32, @intCast(x))` or `.to(.i32)` + when the result must stay i32. Comptime literals take the other operand's + type instead. + +2. **Tensor-of-pointers via `.addPtr`.** Auto-splats either side: scalar + `!tt.ptr` + tensor offset (`x_ptr.addPtr(tensor_offs)`) and tensor + `tensor<...x!tt.ptr>` + scalar offset (`a_ptrs.addPtr(BLOCK_K)`) + both work without manual `.splatTo`. + +3. **Scope-based control flow — no closures needed.** `openFor` / `openIf` / + `openIfElse` / `openWhile` push a block onto the kernel's insertion stack; + body ops emitted in the bare `{}` that follows go into that block. Captures + are plain lexical scope, so you can reference outer Values, sizes, etc. + directly — no `ctx` struct, no function literal. + +4. **`Value.kernel == null` panics fluent calls.** Only happens if you build a + `Value` by hand via `.{ .inner = raw }`. Route through `b.emit(...)` to keep + the back-pointer populated — or just stick with the explicit `Builder.*` + helpers for that one op. + +5. **Don't worry about duplicate constants.** Every lifted literal (from + `.add(42)`, `.lt(0)`, `b.splat(1.0, shape)`, an `openFor(0, N, 1, …)` + bound, etc.) emits a fresh `arith.constant`. This matches Python Triton's + own frontend — the MLIR canonicalizer + CSE pass collapses duplicates on the + way to PTX, so the generated code is identical whether you emit one + constant or a hundred. Optimize for readability at the builder level. + +6. **`finish()` verifies before serializing.** A verifier + failure here almost always means an operand-type mismatch or a + missing block terminator — a common one is forgetting that `scf.yield` + operand types must match the loop's result types. + +7. **Yielding from loop / if / while bodies.** Inside a scope (`openFor`, + `openIf`, `openIfElse`, `openWhile`), you *don't* return values — you call + `scope.yield(...)` (or `yieldThen` / `yieldElse` / `yieldBefore` / + `yieldAfter`) with a tuple of Values matching the scope's declared arity. + Scopes with results (`openFor`, `openIfElse`, `openWhile`) store the scf + op's results in `scope.results` as a fixed-size `[N]Value` array. `openIf` + has no results — `close()` closes and builds scf.if with an empty + else block in one step. diff --git a/dsl/triton/builder.zig b/dsl/triton/builder.zig new file mode 100644 index 00000000..5ccec273 --- /dev/null +++ b/dsl/triton/builder.zig @@ -0,0 +1,2132 @@ +const std = @import("std"); + +const arith = @import("mlir/dialects/arith"); +const cf = @import("mlir/dialects/cf"); +const common = @import("dsl/common/builder"); +const func = @import("mlir/dialects/func"); +const llvm = @import("mlir/dialects/llvm"); +const math = @import("mlir/dialects/math"); +const scf = @import("mlir/dialects/scf"); +const ttir = @import("mlir/dialects/ttir"); +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +const dtypes = @import("dtype.zig"); +pub const DType = dtypes.DType; +const isFloatDtype = dtypes.isFloatDtype; +const dtypeBitwidth = dtypes.dtypeBitwidth; +const intBitwidth = dtypes.intBitwidth; + +test { + std.testing.refAllDecls(@This()); + std.testing.refAllDecls(Builder); + std.testing.refAllDecls(Value); +} + +pub const Value = struct { + inner: *const mlir.Value, + kernel: ?*Builder = null, + + pub const Shape = stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK); + + pub fn type_(self: Value) *const mlir.Type { + return self.inner.type_(); + } + + fn kern(self: Value) *Builder { + return self.kernel orelse @panic("Value has no owning kernel; use Builder.* helpers instead"); + } + + pub fn elemType(self: Value) *const mlir.Type { + if (self.type_().isA(mlir.ShapedType)) |shaped| return shaped.elementType(); + return self.type_(); + } + + pub fn isTensor(self: Value) bool { + return self.type_().isA(mlir.ShapedType) != null; + } + + pub fn rank(self: Value) usize { + if (self.type_().isA(mlir.ShapedType)) |shaped| return shaped.rank(); + return 0; + } + + pub fn dim(self: Value, i: usize) i64 { + return self.type_().isA(mlir.ShapedType).?.dimension(i); + } + + pub fn shape(self: Value) Shape { + var out: Shape = .empty; + if (self.type_().isA(mlir.ShapedType)) |shaped| { + const r = shaped.rank(); + for (0..r) |i| out.appendAssumeCapacity(shaped.dimension(i)); + } + return out; + } + + pub fn isFloatElem(self: Value) bool { + const et = self.elemType(); + inline for (comptime std.meta.fieldNames(mlir.FloatTypes)) |field_name| { + if (et.isA(mlir.FloatType(@field(mlir.FloatTypes, field_name))) != null) return true; + } + return false; + } + + pub fn isIntElem(self: Value) bool { + return self.elemType().isA(mlir.IntegerType) != null; + } + + pub fn add(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.broadcast(self, rhs); + return if (l.isFloatElem()) k.addf(l, r) else k.addi(l, r); + } + + pub fn sub(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.broadcast(self, rhs); + return if (l.isFloatElem()) k.subf(l, r) else k.subi(l, r); + } + + pub fn mul(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.broadcast(self, rhs); + return if (l.isFloatElem()) k.mulf(l, r) else k.muli(l, r); + } + + pub fn div(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.broadcast(self, rhs); + return if (l.isFloatElem()) k.divf(l, r) else k.divsi(l, r); + } + + pub fn rem(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.broadcast(self, rhs); + return if (l.isFloatElem()) k.remf(l, r) else k.remsi(l, r); + } + + /// Ceil-div: `(x + (div - 1)) // div`. Comptime rhs folds `rhs - 1`. + pub fn cdiv(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.broadcast(self, rhs); + return switch (@typeInfo(@TypeOf(rhs))) { + .comptime_int, .comptime_float => k.divsi(l.add(rhs - 1), r), + else => k.divsi(k.addi(l, r.sub(1)), r), + }; + } + + pub fn bitAnd(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.broadcast(self, rhs); + return k.andi(l, r); + } + + pub fn bitOr(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.broadcast(self, rhs); + return k.ori(l, r); + } + + pub fn minimum(self: Value, rhs: anytype) Value { + return self.kern().minimum(self, rhs); + } + pub fn minimumOpts(self: Value, rhs: anytype, opts: MinMaxOpts) Value { + return self.kern().minimumOpts(self, rhs, opts); + } + + pub fn maximum(self: Value, rhs: anytype) Value { + return self.kern().maximum(self, rhs); + } + pub fn maximumOpts(self: Value, rhs: anytype, opts: MinMaxOpts) Value { + return self.kern().maximumOpts(self, rhs, opts); + } + + pub fn lt(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.broadcast(self, rhs); + return if (l.isFloatElem()) k.cmpf(.olt, l, r) else k.cmpi(.slt, l, r); + } + pub fn le(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.broadcast(self, rhs); + return if (l.isFloatElem()) k.cmpf(.ole, l, r) else k.cmpi(.sle, l, r); + } + pub fn gt(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.broadcast(self, rhs); + return if (l.isFloatElem()) k.cmpf(.ogt, l, r) else k.cmpi(.sgt, l, r); + } + pub fn ge(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.broadcast(self, rhs); + return if (l.isFloatElem()) k.cmpf(.oge, l, r) else k.cmpi(.sge, l, r); + } + pub fn eq(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.broadcast(self, rhs); + return if (l.isFloatElem()) k.cmpf(.oeq, l, r) else k.cmpi(.eq, l, r); + } + pub fn ne(self: Value, rhs: anytype) Value { + const k = self.kern(); + const l, const r = k.broadcast(self, rhs); + return if (l.isFloatElem()) k.cmpf(.one, l, r) else k.cmpi(.ne, l, r); + } + + pub fn to(self: Value, dtype: DType) Value { + return self.kern().cast(self, dtype); + } + pub fn bitCast(self: Value, dtype: DType) Value { + return self.kern().bitCast(self, dtype); + } + pub fn toOpts(self: Value, dtype: DType, opts: CastOpts) Value { + return self.kern().castOpts(self, dtype, opts); + } + + pub fn expandDims(self: Value, axis: i32) Value { + return self.kern().expandDims(self, axis); + } + + pub fn splatTo(self: Value, shape_: []const i64) Value { + return self.kern().splat(self, shape_); + } + + pub fn sum(self: Value) Value { + return self.kern().sum(self); + } + pub fn sumOpts(self: Value, opts: ReduceOpts) Value { + return self.kern().sumOpts(self, opts); + } + + pub fn max(self: Value) Value { + return self.kern().max(self); + } + pub fn maxOpts(self: Value, opts: ReduceOpts) Value { + return self.kern().maxOpts(self, opts); + } + + pub fn min(self: Value) Value { + return self.kern().min(self); + } + pub fn minOpts(self: Value, opts: ReduceOpts) Value { + return self.kern().minOpts(self, opts); + } + + pub fn cumsum(self: Value) Value { + return self.kern().cumsum(self); + } + pub fn cumsumOpts(self: Value, opts: ScanOpts) Value { + return self.kern().cumsumOpts(self, opts); + } + + pub fn cumprod(self: Value) Value { + return self.kern().cumprod(self); + } + pub fn cumprodOpts(self: Value, opts: ScanOpts) Value { + return self.kern().cumprodOpts(self, opts); + } + + pub fn gather(self: Value, indices: Value, axis: i32) Value { + return self.kern().gather(self, indices, axis); + } + + pub fn abs(self: Value) Value { + return self.kern().abs(self); + } + + pub fn addPtr(self: Value, offset: anytype) Value { + const k = self.kern(); + const off: Value = if (@TypeOf(offset) == Value) offset else k.lift(offset); + const ptr = if (!self.isTensor() and off.isTensor()) k.splat(self, off.shape().constSlice()) else self; + const off2 = if (self.isTensor() and !off.isTensor()) k.splat(off, self.shape().constSlice()) else off; + return k.addptr(ptr, off2); + } +}; + +fn computeReducedType(src_shape: []const i64, axis: i32, elem: *const mlir.Type) *const mlir.Type { + if (src_shape.len <= 1) return elem; + var out: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; + for (src_shape, 0..) |d, i| { + if (@as(i32, @intCast(i)) != axis) out.appendAssumeCapacity(d); + } + return .rankedTensor(out.constSlice(), elem); +} + +pub const ArgSpec = struct { + name: []const u8, + kind: Kind, + + pub const Kind = union(enum) { + ptr: DType, + scalar: DType, + tensor: struct { []const i64, DType }, + ptr_opts: PtrOpts, + scalar_opts: ScalarOpts, + }; + + pub const PtrOpts = struct { + dtype: DType, + address_space: i32 = 1, + divisibility: ?u32 = 16, + }; + + pub const ScalarOpts = struct { + dtype: DType, + divisibility: ?u32 = null, + }; +}; + +pub const FinishError = error{InvalidMlir} || std.mem.Allocator.Error || std.Io.Writer.Error; + +pub const ClampOpts = struct { + propagate_nan: ttir.PropagateNan = .none, +}; + +pub const HistogramOpts = struct { + mask: ?Value = null, +}; + +pub const PrintOpts = struct { + hex: bool = false, +}; + +/// `axis=null` reduces all dims. +pub const ReduceOpts = struct { + axis: ?i32 = null, + keep_dims: bool = false, +}; + +pub const ScanOpts = struct { + axis: i32 = 0, + reverse: bool = false, +}; + +pub const LoadOpts = struct { + mask: ?Value = null, + other: ?Value = null, + cache_modifier: ttir.CacheModifier = .none, + eviction_policy: ttir.EvictionPolicy = .normal, + @"volatile": bool = false, +}; + +pub const StoreOpts = struct { + mask: ?Value = null, + cache_modifier: ttir.CacheModifier = .none, + eviction_policy: ttir.EvictionPolicy = .normal, +}; + +pub const ReshapeOpts = struct { + can_reorder: bool = false, + efficient_layout: bool = false, +}; + +pub const DotOpts = struct { + input_precision: ttir.InputPrecision = .tf32, + max_num_imprecise_acc: i32 = 0, +}; + +pub const FpToFpOpts = struct { + rounding: ?ttir.RoundingMode = null, +}; + +pub const AtomicRMWOpts = struct { + mask: ?Value = null, + sem: ttir.MemSemantic = .acq_rel, + scope: ttir.MemSyncScope = .gpu, +}; + +pub const AtomicCasOpts = struct { + sem: ttir.MemSemantic = .acq_rel, + scope: ttir.MemSyncScope = .gpu, +}; + +pub const DotScaledOpts = struct { + fast_math: bool = false, + lhs_k_pack: bool = true, + rhs_k_pack: bool = true, +}; + +pub const ExternElementwiseOpts = struct { + is_pure: bool = true, +}; + +pub const DescriptorLoadOpts = struct { + cache_modifier: ttir.CacheModifier = .none, + eviction_policy: ttir.EvictionPolicy = .normal, +}; + +pub const InlineAsmOpts = struct { + is_pure: bool = true, + pack: i32 = 1, +}; + +pub const CatOpts = struct { + can_reorder: bool = false, + dim: i32 = 0, +}; + +pub const CastOpts = struct { + fp_downcast_rounding: ?ttir.RoundingMode = null, + bitcast: bool = false, +}; + +pub const MinMaxOpts = struct { + propagate_nan: ttir.PropagateNan = .none, +}; + +pub const DeviceAssertOpts = struct { + mask: ?Value = null, +}; + +pub const ReturnIfOpts = struct { + inline_return: bool = false, +}; + +pub fn ForScope(comptime N: usize) type { + return common.ForScope(Builder, Value, N); +} + +/// `if (cond) { ...; return; }`: `close` returns at the end of the body, +/// then the kernel continues after it. +pub const ReturnIfScope = struct { + kernel: *Builder, + ret_block: *mlir.Block, + cont_block: *mlir.Block, + opts: ReturnIfOpts, + + pub fn close(self: *ReturnIfScope) void { + const k = self.kernel; + if (self.opts.inline_return) { + _ = ttir.return_(k.ctx, &.{}, k.loc()).appendTo(self.ret_block); + } else { + _ = cf.br(k.ctx, k.exitBlock(), &.{}, k.loc()).appendTo(self.ret_block); + } + k.popBlock(); + k.continueIn(self.cont_block); + } +}; + +pub const IfOnlyScope = common.IfOnlyScope(Builder, Value); + +pub fn IfScope(comptime N: usize) type { + return common.IfScope(Builder, Value, N); +} + +pub fn WhileScope(comptime N: usize, comptime M: usize) type { + return common.WhileScope(Builder, Value, N, M); +} + +pub fn ReduceArgs(comptime CtxT: type) type { + return struct { + src: Value, + axis: i32, + elem: *const mlir.Type, + result: *const mlir.Type, + combine: *const fn (*Builder, Value, Value, CtxT) Value, + }; +} + +pub fn ScanArgs(comptime CtxT: type) type { + return struct { + src: Value, + axis: i32, + reverse: bool = false, + elem: *const mlir.Type, + result: *const mlir.Type, + combine: *const fn (*Builder, Value, Value, CtxT) Value, + }; +} + +pub fn registerDialects(registry: *mlir.DialectRegistry) void { + inline for (.{ func, ttir, arith, scf, math, cf, llvm }) |dialect| dialect.registerDialect(registry); +} + +pub const Builder = struct { + allocator: std.mem.Allocator, + arena: std.heap.ArenaAllocator, + ctx: *mlir.Context, + module: *mlir.Module, + name: []const u8, + func_op: ?*mlir.Operation, + entry_block: ?*mlir.Block, + exit_block: ?*mlir.Block = null, + block_stack: common.BlockStack = .{}, + + /// `noinline` is a Zig keyword — write `.{ .@"noinline" = true }`. + pub const Opts = struct { + @"noinline": bool = false, + }; + + pub fn init( + allocator: std.mem.Allocator, + ctx: *mlir.Context, + name: []const u8, + args: []const ArgSpec, + ) !Builder { + return initOpts(allocator, ctx, name, args, .{}); + } + + pub fn initOpts( + allocator: std.mem.Allocator, + ctx: *mlir.Context, + name: []const u8, + args: []const ArgSpec, + opts: Opts, + ) !Builder { + var b = try open(allocator, ctx, name); + errdefer b.deinit(); + try b.declareArgsLowOpts(args, opts); + return b; + } + + pub fn open( + allocator: std.mem.Allocator, + ctx: *mlir.Context, + name: []const u8, + ) !Builder { + const unknown_loc: *const mlir.Location = .unknown(ctx); + const module: *mlir.Module = .init(unknown_loc); + errdefer module.deinit(); + + var arena: std.heap.ArenaAllocator = .init(allocator); + errdefer arena.deinit(); + + return .{ + .allocator = allocator, + .arena = arena, + .ctx = ctx, + .module = module, + .name = name, + .func_op = null, + .entry_block = null, + .block_stack = .{}, + }; + } + + pub fn declareArgs(self: *Builder, spec: anytype) !common.NamedArgs(@TypeOf(spec), Value) { + return self.declareArgsOpts(spec, .{}); + } + + pub fn declareArgsOpts( + self: *Builder, + spec: anytype, + opts: Opts, + ) !common.NamedArgs(@TypeOf(spec), Value) { + const Spec = @TypeOf(spec); + const field_names = @typeInfo(Spec).@"struct".field_names; + + var arg_specs: [field_names.len]ArgSpec = undefined; + inline for (field_names, 0..) |field_name, i| { + const raw = @field(spec, field_name); + const kind: ArgSpec.Kind = if (@TypeOf(raw) == ArgSpec.Kind) raw else blk: { + const variant = @typeInfo(@TypeOf(raw)).@"struct".field_names[0]; + const tag = @field(std.meta.Tag(ArgSpec.Kind), variant); + const inner = @field(raw, variant); + break :blk switch (tag) { + .ptr => .{ .ptr = inner }, + .scalar => .{ .scalar = inner }, + .tensor => .{ .tensor = inner }, + .ptr_opts => .{ .ptr_opts = .{ + .dtype = inner.dtype, + .address_space = if (@hasField(@TypeOf(inner), "address_space")) inner.address_space else 1, + .divisibility = if (@hasField(@TypeOf(inner), "divisibility")) inner.divisibility else 16, + } }, + .scalar_opts => .{ .scalar_opts = .{ + .dtype = inner.dtype, + .divisibility = if (@hasField(@TypeOf(inner), "divisibility")) inner.divisibility else null, + } }, + }; + }; + arg_specs[i] = .{ .name = field_name, .kind = kind }; + } + + try self.declareArgsLowOpts(&arg_specs, opts); + + var named: common.NamedArgs(Spec, Value) = undefined; + inline for (field_names, 0..) |field_name, i| { + @field(named, field_name) = self.arg(i); + } + return named; + } + + fn declareArgsLowOpts( + self: *Builder, + args: []const ArgSpec, + opts: Opts, + ) !void { + std.debug.assert(self.entry_block == null); + const ctx = self.ctx; + const unknown_loc: *const mlir.Location = .unknown(ctx); + const scratch = self.arena.allocator(); + + const arg_types = try scratch.alloc(*const mlir.Type, args.len); + const arg_locs = try scratch.alloc(*const mlir.Location, args.len); + for (arg_types, arg_locs, args) |*ty, *a_loc, a| { + ty.* = switch (a.kind) { + .ptr => |dt| ttir.pointerType(dt.toMlir(ctx), 1), + .ptr_opts => |p| ttir.pointerType(p.dtype.toMlir(ctx), p.address_space), + .scalar => |dt| dt.toMlir(ctx), + .scalar_opts => |s| s.dtype.toMlir(ctx), + .tensor => |t| .rankedTensor(t[0], t[1].toMlir(ctx)), + }; + a_loc.* = unknown_loc.named(ctx, a.name); + } + + const entry = mlir.Block.init(arg_types, arg_locs); + + const arg_attrs = try scratch.alloc(*const mlir.Attribute, args.len); + const empty_dict: *const mlir.Attribute = .dict(ctx, &.{}); + var any_arg_attr = false; + for (arg_attrs, args) |*attr, a| { + attr.* = switch (a.kind) { + .ptr => .dict(ctx, &.{ttir.divisibility(ctx, 16)}), + .ptr_opts => |p| if (p.divisibility) |v| + .dict(ctx, &.{ttir.divisibility(ctx, v)}) + else + empty_dict, + .scalar_opts => |s| if (s.divisibility) |v| + .dict(ctx, &.{ttir.divisibility(ctx, v)}) + else + empty_dict, + else => empty_dict, + }; + if (attr.* != empty_dict) any_arg_attr = true; + } + + const func_op = ttir.func(ctx, .{ + .name = self.name, + .args = arg_types, + .args_attributes = if (any_arg_attr) arg_attrs else null, + .results = &.{}, + .block = entry, + .location = unknown_loc, + .no_inline = opts.@"noinline", + }); + _ = func_op.appendTo(self.module.body()); + + self.func_op = func_op; + self.entry_block = entry; + self.block_stack.push(self.arena.allocator(), entry); + } + + pub fn deinit(self: *Builder) void { + self.module.deinit(); + self.arena.deinit(); + } + + pub fn arg(self: *Builder, i: usize) Value { + const eb = self.entry_block orelse @panic("Builder.arg called before declareArgs"); + return .{ .inner = eb.argument(i), .kernel = self }; + } + + pub fn pushBlock(self: *Builder, b: *mlir.Block) void { + self.block_stack.push(self.arena.allocator(), b); + } + + pub fn popBlock(self: *Builder) void { + self.block_stack.pop(); + } + + pub fn currentBlock(self: *Builder) *mlir.Block { + return self.block_stack.current(); + } + + fn innerSlice(self: *Builder, values: []const Value) []const *const mlir.Value { + const out = self.arena.allocator().alloc(*const mlir.Value, values.len) catch @panic("Builder.innerSlice OOM"); + for (values, 0..) |v, i| out[i] = v.inner; + return out; + } + + pub fn emit(self: *Builder, op: *mlir.Operation) Value { + _ = op.appendTo(self.currentBlock()); + return .{ .inner = op.result(0), .kernel = self }; + } + + pub fn emitMulti(self: *Builder, op: *mlir.Operation, n: usize) []Value { + _ = op.appendTo(self.currentBlock()); + const out = self.arena.allocator().alloc(Value, n) catch @panic("Builder.emitMulti OOM"); + for (0..n) |i| out[i] = .{ .inner = op.result(i), .kernel = self }; + return out; + } + + pub fn yield(self: *Builder, values: anytype) []const Value { + const T = @TypeOf(values); + const info = @typeInfo(T); + if (info != .@"struct" or !info.@"struct".is_tuple) + @compileError("Builder.yield expects a tuple literal like `.{ v1, v2 }`"); + const n = info.@"struct".field_names.len; + const out = self.arena.allocator().alloc(Value, n) catch @panic("Builder.yield OOM"); + inline for (info.@"struct".field_names, info.@"struct".field_types, 0..) |field_name, Field, i| { + if (Field != Value) + @compileError("Builder.yield: every tuple element must be a Value; got " ++ @typeName(Field)); + out[i] = @field(values, field_name); + } + return out; + } + + pub fn loc(self: *const Builder) *const mlir.Location { + return .unknown(self.ctx); + } + + pub fn scalarTy(self: *const Builder, dtype: DType) *const mlir.Type { + return dtype.toMlir(self.ctx); + } + + pub fn tensorTy(self: *const Builder, shape: []const i64, dtype: DType) *const mlir.Type { + return .rankedTensor(shape, dtype.toMlir(self.ctx)); + } + + fn loadResultType(self: *const Builder, ptr: Value) *const mlir.Type { + _ = self; + const pt = ptr.type_(); + if (pt.isA(mlir.ShapedType)) |shaped| { + const elem_ptr_ty = shaped.elementType(); + const pointee = elem_ptr_ty.isA(ttir.PointerType).?.pointee(); + var shape: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; + for (0..shaped.rank()) |i| shape.appendAssumeCapacity(shaped.dimension(i)); + return .rankedTensor(shape.constSlice(), pointee); + } + return pt.isA(ttir.PointerType).?.pointee(); + } + + fn swapElem(self: *const Builder, src: Value, out_dtype: DType) *const mlir.Type { + const out_elem = out_dtype.toMlir(self.ctx); + if (src.isTensor()) return .rankedTensor(src.shape().constSlice(), out_elem); + return out_elem; + } + + // ==================== SPMD ==================== + + pub fn programId(self: *Builder, dim: ttir.ProgramIDDim) Value { + return self.emit(ttir.get_program_id(self.ctx, dim, self.loc())); + } + + pub fn numPrograms(self: *Builder, dim: ttir.ProgramIDDim) Value { + return self.emit(ttir.get_num_programs(self.ctx, dim, self.loc())); + } + + pub fn assume(self: *Builder, cond: Value) void { + _ = llvm.intr_assume(self.ctx, cond.inner, self.loc()).appendTo(self.currentBlock()); + } + + // ==================== constants ==================== + + fn emitInt(self: *Builder, dtype: DType, value: i64) Value { + return self.emit(arith.constant_int(self.ctx, value, dtype.toMlir(self.ctx), self.loc())); + } + + fn emitFloat(self: *Builder, dtype: DType, value: f64) Value { + return switch (dtype) { + .f32 => self.emit(arith.constant_float(self.ctx, value, .f32, self.loc())), + .f64 => self.emit(arith.constant_float(self.ctx, value, .f64, self.loc())), + else => self.fpToFp(self.emitFloat(.f32, value), dtype), + }; + } + + // ==================== ranges and shape manipulation ==================== + + /// `[start, end)` as `tensor<(end-start)xi32>`. Both must fit in i32. + pub fn makeRange(self: *Builder, start: anytype, end: anytype) Value { + const s: i32 = @intCast(start); + const e: i32 = @intCast(end); + std.debug.assert(e >= s); + const len: i64 = @intCast(e - s); + const ty = mlir.Type.rankedTensor(&.{len}, .int(self.ctx, .i32)); + return self.emit(ttir.make_range(self.ctx, s, e, ty, self.loc())); + } + + pub fn arange(self: *Builder, start: anytype, end: anytype, dtype: DType) Value { + const r = self.makeRange(start, end); + return switch (dtype) { + .i32 => r, + .i64 => self.extsi(r, dtype), + .i16, .i8 => self.trunci(r, dtype), + .f16, .bf16, .f32, .f64, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2 => self.sitofp(r, dtype), + else => @panic("Builder.arange: unsupported dtype"), + }; + } + + /// Broadcast a scalar Value or literal across `shape`. Literals lift first via `lift`. + pub fn splat(self: *Builder, value: anytype, shape: []const i64) Value { + const v: Value = if (@TypeOf(value) == Value) value else self.lift(value); + const ty: *const mlir.Type = .rankedTensor(shape, v.type_()); + return self.emit(ttir.splat(self.ctx, v.inner, ty, self.loc())); + } + + /// Lift a Zig scalar to a DSL constant. Dtype follows the source type; + /// unsigned ints are bit-cast (signless MLIR). Use `constant` for an explicit dtype. + pub fn lift(self: *Builder, value: anytype) Value { + const T = @TypeOf(value); + if (T == Value) return value; + return switch (@typeInfo(T)) { + .comptime_int => blk: { + if (value >= std.math.minInt(i32) and value <= std.math.maxInt(i32)) + break :blk self.emitInt(.i32, value); + if (value >= std.math.minInt(i64) and value <= std.math.maxInt(i64)) + break :blk self.emitInt(.i64, value); + if (value >= 0 and value <= std.math.maxInt(u64)) + break :blk self.emitInt(.i64, @bitCast(@as(u64, value))); + @compileError("Builder.lift: integer literal out of 64-bit range"); + }, + .int => |info| blk: { + const signed = info.signedness == .signed; + if (info.bits > 32) { + const v: i64 = if (signed) @intCast(value) else @bitCast(@as(u64, @intCast(value))); + break :blk self.emitInt(.i64, v); + } + const v32: i32 = if (signed) @intCast(value) else @bitCast(@as(u32, @intCast(value))); + break :blk self.emitInt(.i32, v32); + }, + .comptime_float => self.emitFloat(.f32, @floatCast(value)), + .float => |info| switch (info.bits) { + 16 => self.emitFloat(.f16, @floatCast(value)), + 32 => self.emitFloat(.f32, @floatCast(value)), + 64 => self.emitFloat(.f64, value), + else => @compileError("Builder.lift: unsupported float bitwidth"), + }, + else => @compileError("Builder.lift: unsupported type " ++ @typeName(T)), + }; + } + + /// `lift` but comptime scalars match `ref_elem`. Runtime Zig scalars keep their source width. + fn liftMatching(self: *Builder, value: anytype, ref_elem: *const mlir.Type) Value { + const T = @TypeOf(value); + if (T == Value) return value; + return switch (@typeInfo(T)) { + .comptime_int, .comptime_float => self.constMatching(value, ref_elem), + else => self.lift(value), + }; + } + + /// Lift `value` and splat to `ref.shape` when `ref` is a tensor. + pub fn broadcastLike(self: *Builder, value: anytype, ref: Value) Value { + const v = self.liftMatching(value, ref.elemType()); + if (ref.isTensor() and !v.isTensor()) return self.splat(v, ref.shape().constSlice()); + return v; + } + + /// Binary-op coercion: lift, int-width promote, scalar↔tensor splat, rank-align, size-1 broadcast. + pub fn broadcast(self: *Builder, a: anytype, b: anytype) struct { Value, Value } { + const a_ref: ?*const mlir.Type = if (@TypeOf(b) == Value) b.elemType() else null; + const b_ref: ?*const mlir.Type = if (@TypeOf(a) == Value) a.elemType() else null; + var av = if (a_ref) |t| self.liftMatching(a, t) else self.lift(a); + var bv = if (b_ref) |t| self.liftMatching(b, t) else self.lift(b); + + if (av.isIntElem() and bv.isIntElem()) { + const a_dt = self.mlirElemToDType(av.elemType()); + const b_dt = self.mlirElemToDType(bv.elemType()); + const aw = intBitwidth(a_dt); + const bw = intBitwidth(b_dt); + if (aw < bw) av = av.to(b_dt); + if (bw < aw) bv = bv.to(a_dt); + } + + if (av.isTensor() and !bv.isTensor()) bv = self.splat(bv, av.shape().constSlice()); + if (!av.isTensor() and bv.isTensor()) av = self.splat(av, bv.shape().constSlice()); + + if (av.isTensor() and bv.isTensor()) { + const a_rank = av.rank(); + const b_rank = bv.rank(); + if (a_rank < b_rank) { + var n = b_rank - a_rank; + while (n > 0) : (n -= 1) av = self.expandDims(av, 0); + } else if (b_rank < a_rank) { + var n = a_rank - b_rank; + while (n > 0) : (n -= 1) bv = self.expandDims(bv, 0); + } + const a_sh = av.shape(); + const b_sh = bv.shape(); + var ret_shape: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; + var grow_a = false; + var grow_b = false; + for (0..a_sh.len) |i| { + const l = a_sh.get(i); + const r = b_sh.get(i); + if (l == r) { + ret_shape.appendAssumeCapacity(l); + } else if (l == 1) { + ret_shape.appendAssumeCapacity(r); + grow_a = true; + } else if (r == 1) { + ret_shape.appendAssumeCapacity(l); + grow_b = true; + } else { + std.debug.panic( + "Builder.broadcast: incompatible shapes {any} vs {any} at axis {d}", + .{ a_sh.constSlice(), b_sh.constSlice(), i }, + ); + } + } + if (grow_a) av = self.broadcastTo(av, ret_shape.constSlice()); + if (grow_b) bv = self.broadcastTo(bv, ret_shape.constSlice()); + } + + return .{ av, bv }; + } + + fn mlirElemToDType(self: *const Builder, elem: *const mlir.Type) DType { + inline for (comptime std.meta.fieldNames(DType)) |field_name| { + const dt = @field(DType, field_name); + if (elem.eql(dt.toMlir(self.ctx))) return dt; + } + @panic("element type not a recognized DSL DType"); + } + + /// `value` as a constant of `dtype`; a Value passes through. An integer is + /// signed or an unsigned bit pattern of `dtype`'s width; anything else + /// panics. + pub fn constant(self: *Builder, dtype: DType, value: anytype) Value { + const T = @TypeOf(value); + if (T == Value) return value; + if (isFloatDtype(dtype)) { + const v_f64: f64 = switch (@typeInfo(T)) { + .comptime_int, .int => @floatFromInt(value), + .comptime_float, .float => @floatCast(value), + else => @compileError("Builder.constant: unsupported scalar " ++ @typeName(T)), + }; + return self.emitFloat(dtype, v_f64); + } + const v: i128 = switch (@typeInfo(T)) { + .comptime_int, .int => value, + .comptime_float, .float => @intFromFloat(value), + else => @compileError("Builder.constant: unsupported scalar " ++ @typeName(T)), + }; + return self.emitInt(dtype, intBits(v, @intCast(intBitwidth(dtype)), @tagName(dtype))); + } + + /// `value` as the `i64` an integer constant of `bits` bits is built from. + /// It may be signed or an unsigned bit pattern, so it must lie in + /// [-2^(bits-1), 2^bits). + fn intBits(value: i128, bits: u7, dtype: []const u8) i64 { + if (value < -(@as(i128, 1) << (bits - 1)) or value >= @as(i128, 1) << bits) + std.debug.panic("Builder.constant: {d} does not fit {s}", .{ value, dtype }); + return @truncate(value); + } + + /// `value` as a constant of `like`'s element type; a Value passes through. + pub fn constantLike(self: *Builder, value: anytype, like: Value) Value { + if (@TypeOf(value) == Value) return value; + return self.constMatching(value, like.elemType()); + } + + fn constMatching(self: *Builder, value: anytype, elem: *const mlir.Type) Value { + return self.constant(self.mlirElemToDType(elem), value); + } + + pub fn zeros(self: *Builder, shape: []const i64, dtype: DType) Value { + return self.full(shape, 0, dtype); + } + + pub fn ones(self: *Builder, shape: []const i64, dtype: DType) Value { + return self.full(shape, 1, dtype); + } + + pub fn full(self: *Builder, shape: []const i64, value: anytype, dtype: DType) Value { + return self.splat(self.constant(dtype, value), shape); + } + + /// 2-D mask `(m, 1) & (1, n)` — auto-broadcasts to `(m, n)`. + pub fn mask2d(self: *Builder, cond_m: Value, cond_n: Value, _: i64, _: i64) Value { + const cm2 = self.expandDims(cond_m, 1); + const cn2 = self.expandDims(cond_n, 0); + return cm2.bitAnd(cn2); + } + + pub fn sum(self: *Builder, src: Value) Value { + return self.sumOpts(src, .{}); + } + + pub fn sumOpts(self: *Builder, src: Value, opts: ReduceOpts) Value { + return self.reduceDispatch(src, opts, .sum); + } + + pub fn max(self: *Builder, src: Value) Value { + return self.maxOpts(src, .{}); + } + + pub fn maxOpts(self: *Builder, src: Value, opts: ReduceOpts) Value { + return self.reduceDispatch(src, opts, .max); + } + + pub fn min(self: *Builder, src: Value) Value { + return self.minOpts(src, .{}); + } + + pub fn minOpts(self: *Builder, src: Value, opts: ReduceOpts) Value { + return self.reduceDispatch(src, opts, .min); + } + + fn reduceDispatch(self: *Builder, src: Value, opts: ReduceOpts, comptime kind: enum { sum, max, min }) Value { + const src_shape = src.shape(); + const input: Value, const axis: i32 = if (opts.axis) |a| .{ src, a } else blk: { + var total: i64 = 1; + for (0..src_shape.len) |i| total *= src_shape.get(i); + break :blk .{ self.reshapeOpts(src, &.{total}, .{ .can_reorder = true }), 0 }; + }; + + const elem = input.elemType(); + const in_shape = input.shape(); + const result_ty: *const mlir.Type = computeReducedType(in_shape.constSlice(), axis, elem); + const is_float = input.isFloatElem(); + const Ctx = struct { is_float: bool, kind: @TypeOf(kind) }; + const combine = struct { + fn c(kk: *Builder, lhs: Value, rhs: Value, cc: Ctx) Value { + return switch (cc.kind) { + .sum => if (cc.is_float) kk.addf(lhs, rhs) else kk.addi(lhs, rhs), + .max => if (cc.is_float) kk.maxnumf(lhs, rhs) else kk.maxsi(lhs, rhs), + .min => if (cc.is_float) kk.minnumf(lhs, rhs) else kk.minsi(lhs, rhs), + }; + } + }.c; + const out = self.reduce(Ctx{ .is_float = is_float, .kind = kind }, .{ + .src = input, + .axis = axis, + .elem = elem, + .result = result_ty, + .combine = combine, + }); + + if (!opts.keep_dims) return out; + if (opts.axis == null) { + var ones_shape: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; + for (0..src_shape.len) |_| ones_shape.appendAssumeCapacity(1); + return self.splat(out, ones_shape.constSlice()); + } + if (in_shape.len <= 1) return self.splat(out, &.{1}); + return self.expandDims(out, axis); + } + + pub fn cumsum(self: *Builder, src: Value) Value { + return self.cumsumOpts(src, .{}); + } + + pub fn cumsumOpts(self: *Builder, src: Value, opts: ScanOpts) Value { + return self.scanDispatch(src, opts, .sum); + } + + pub fn cumprod(self: *Builder, src: Value) Value { + return self.cumprodOpts(src, .{}); + } + + pub fn cumprodOpts(self: *Builder, src: Value, opts: ScanOpts) Value { + return self.scanDispatch(src, opts, .prod); + } + + fn scanDispatch(self: *Builder, src: Value, opts: ScanOpts, comptime kind: enum { sum, prod }) Value { + const elem = src.elemType(); + const is_float = src.isFloatElem(); + const Ctx = struct { is_float: bool, kind: @TypeOf(kind) }; + const combine = struct { + fn c(kk: *Builder, lhs: Value, rhs: Value, cc: Ctx) Value { + return switch (cc.kind) { + .sum => if (cc.is_float) kk.addf(lhs, rhs) else kk.addi(lhs, rhs), + .prod => if (cc.is_float) kk.mulf(lhs, rhs) else kk.muli(lhs, rhs), + }; + } + }.c; + return self.scan(Ctx{ .is_float = is_float, .kind = kind }, .{ + .src = src, + .axis = opts.axis, + .reverse = opts.reverse, + .elem = elem, + .result = src.type_(), + .combine = combine, + }); + } + + pub fn expandDims(self: *Builder, value: Value, axis: i32) Value { + const in_shape = value.shape(); + var out_shape: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; + const ax: usize = @intCast(axis); + for (0..in_shape.len + 1) |i| { + if (i == ax) out_shape.appendAssumeCapacity(1) else out_shape.appendAssumeCapacity(in_shape.get(if (i < ax) i else i - 1)); + } + const ty = mlir.Type.rankedTensor(out_shape.constSlice(), value.elemType()); + return self.emit(ttir.expand_dims(self.ctx, value.inner, axis, ty, self.loc())); + } + + pub fn broadcastTo(self: *Builder, value: Value, result_shape: []const i64) Value { + const elem = mlir.RankedTensorType.fromShaped(value.inner.type_().isA(mlir.ShapedType).?).elementType(); + const ty = mlir.Type.rankedTensor(result_shape, elem); + return self.emit(ttir.broadcast(self.ctx, value.inner, ty, self.loc())); + } + + // ==================== pointer arithmetic ==================== + + pub fn addptr(self: *Builder, ptr: Value, offset: Value) Value { + return self.emit(ttir.addptr(self.ctx, ptr.inner, offset.inner, self.loc())); + } + + pub fn load(self: *Builder, ptr: Value) Value { + return self.loadOpts(ptr, .{}); + } + + pub fn loadOpts(self: *Builder, ptr: Value, opts: LoadOpts) Value { + var mask_v = opts.mask; + var other_v = opts.other; + if (ptr.isTensor()) { + if (mask_v) |m| { + _, mask_v = self.broadcast(ptr, m); + } + if (other_v) |o| { + _, other_v = self.broadcast(ptr, o); + } + } + const mask_inner: ?*const mlir.Value = if (mask_v) |m| m.inner else null; + const other_inner: ?*const mlir.Value = if (other_v) |o| o.inner else null; + return self.emit(ttir.load( + self.ctx, + ptr.inner, + self.loadResultType(ptr), + opts.cache_modifier, + opts.eviction_policy, + opts.@"volatile", + .{ .mask = mask_inner, .other = other_inner }, + self.loc(), + )); + } + + pub fn store(self: *Builder, ptr: Value, value: Value) void { + self.storeOpts(ptr, value, .{}); + } + + pub fn storeOpts(self: *Builder, ptr: Value, value: Value, opts: StoreOpts) void { + var mask_v = opts.mask; + if (ptr.isTensor()) { + if (mask_v) |m| { + _, mask_v = self.broadcast(ptr, m); + } + } + const mask_inner: ?*const mlir.Value = if (mask_v) |m| m.inner else null; + _ = ttir.store(self.ctx, ptr.inner, value.inner, opts.cache_modifier, opts.eviction_policy, .{ .mask = mask_inner }, self.loc()).appendTo(self.currentBlock()); + } + + // ==================== arith: binary ==================== + + pub fn addi(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.addi(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn subi(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.subi(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn muli(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.muli(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn addf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.addf(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn subf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.subf(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn mulf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.mulf(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn divf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.divf(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + + pub fn cmpi(self: *Builder, predicate: arith.CmpIPredicate, lhs: Value, rhs: Value) Value { + return self.emit(arith.cmpi(self.ctx, predicate, lhs.inner, rhs.inner, self.loc())); + } + pub fn cmpf(self: *Builder, predicate: arith.CmpFPredicate, lhs: Value, rhs: Value) Value { + return self.emit(arith.cmpf(self.ctx, predicate, lhs.inner, rhs.inner, self.loc())); + } + + pub fn select(self: *Builder, cond: Value, t: Value, f: Value) Value { + return self.emit(arith.select(self.ctx, cond.inner, t.inner, f.inner, self.loc())); + } + + /// Select `x` where condition is true, `y` elsewhere. Auto-broadcasts. + pub fn where(self: *Builder, condition: Value, x: Value, y: Value) Value { + var c = condition; + var xv, var yv = .{ x, y }; + xv, yv = self.broadcast(xv, yv); + if (c.isTensor()) { + c, xv = self.broadcast(c, xv); + xv, yv = self.broadcast(xv, yv); + } else { + c, _ = self.broadcast(c, xv); + } + // Arithmetic broadcasting can widen i1 when the selected values are + // integers. The select predicate must remain boolean. + return self.select(c.to(.i1), xv, yv); + } + + pub fn divsi(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.divsi(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn divui(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.divui(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn remsi(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.remsi(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn remui(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.remui(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn ceildivsi(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.ceildivsi(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn floordivsi(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.floordivsi(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + + pub fn andi(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.andi(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn ori(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.ori(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn xori(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.xori(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn shli(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.shli(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn shrsi(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.shrsi(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn shrui(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.shrui(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + + pub fn maxsi(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.maxsi(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn minsi(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.minsi(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + + pub fn remf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.remf(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn maximumf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.maximumf(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn minimumf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.minimumf(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn maxnumf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.maxnumf(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn minnumf(self: *Builder, lhs: Value, rhs: Value) Value { + return self.emit(arith.minnumf(self.ctx, lhs.inner, rhs.inner, self.loc())); + } + pub fn negf(self: *Builder, src: Value) Value { + return self.emit(arith.negf(self.ctx, src.inner, self.loc())); + } + + pub fn maximum(self: *Builder, a: anytype, b: anytype) Value { + return self.maximumOpts(a, b, .{}); + } + + pub fn maximumOpts(self: *Builder, a: anytype, b: anytype, opts: MinMaxOpts) Value { + const l, const r = self.broadcast(a, b); + if (l.isFloatElem()) { + return switch (opts.propagate_nan) { + .none => self.maxnumf(l, r), + .all => self.maximumf(l, r), + }; + } + return self.maxsi(l, r); + } + + pub fn minimum(self: *Builder, a: anytype, b: anytype) Value { + return self.minimumOpts(a, b, .{}); + } + + pub fn minimumOpts(self: *Builder, a: anytype, b: anytype, opts: MinMaxOpts) Value { + const l, const r = self.broadcast(a, b); + if (l.isFloatElem()) { + return switch (opts.propagate_nan) { + .none => self.minnumf(l, r), + .all => self.minimumf(l, r), + }; + } + return self.minsi(l, r); + } + + // Casts — each preserves the src shape and swaps the element type to `out_dtype`. + pub fn extsi(self: *Builder, src: Value, out_dtype: DType) Value { + return self.emit(arith.extsi(self.ctx, src.inner, self.swapElem(src, out_dtype), self.loc())); + } + pub fn extui(self: *Builder, src: Value, out_dtype: DType) Value { + return self.emit(arith.extui(self.ctx, src.inner, self.swapElem(src, out_dtype), self.loc())); + } + pub fn extf(self: *Builder, src: Value, out_dtype: DType) Value { + return self.emit(arith.extf(self.ctx, src.inner, self.swapElem(src, out_dtype), self.loc())); + } + pub fn trunci(self: *Builder, src: Value, out_dtype: DType) Value { + return self.emit(arith.trunci(self.ctx, src.inner, self.swapElem(src, out_dtype), self.loc())); + } + pub fn truncf(self: *Builder, src: Value, out_dtype: DType) Value { + return self.emit(arith.truncf(self.ctx, src.inner, self.swapElem(src, out_dtype), self.loc())); + } + pub fn sitofp(self: *Builder, src: Value, out_dtype: DType) Value { + return self.emit(arith.sitofp(self.ctx, src.inner, self.swapElem(src, out_dtype), self.loc())); + } + pub fn uitofp(self: *Builder, src: Value, out_dtype: DType) Value { + return self.emit(arith.uitofp(self.ctx, src.inner, self.swapElem(src, out_dtype), self.loc())); + } + pub fn fptosi(self: *Builder, src: Value, out_dtype: DType) Value { + return self.emit(arith.fptosi(self.ctx, src.inner, self.swapElem(src, out_dtype), self.loc())); + } + pub fn fptoui(self: *Builder, src: Value, out_dtype: DType) Value { + return self.emit(arith.fptoui(self.ctx, src.inner, self.swapElem(src, out_dtype), self.loc())); + } + pub fn arithBitcast(self: *Builder, src: Value, out_dtype: DType) Value { + return self.emit(arith.bitcast(self.ctx, src.inner, self.swapElem(src, out_dtype), self.loc())); + } + + // ==================== Triton-specific ops ==================== + + /// tt.bitcast — same-bitwidth cast. Shape preserved from src. + pub fn bitCast(self: *Builder, src: Value, out_dtype: DType) Value { + return self.emit(ttir.bitcast(self.ctx, src.inner, self.swapElem(src, out_dtype), self.loc())); + } + + /// Numeric cast with auto-dispatch. Use `castOpts` for rounding / bitcast. + pub fn cast(self: *Builder, src: Value, dtype: DType) Value { + return self.castOpts(src, dtype, .{}); + } + + /// Numeric cast with options. Dispatch: + /// - fp↔fp with fp8: tt.fp_to_fp, with rounding only on downcasts. + /// - other fp↔fp narrow: truncf, except custom rounding → tt.fp_to_fp. + /// - other fp↔fp wide: extf. + /// - i1→fp: uitofp. int↔int: extsi/trunci/bitcast by width. + pub fn castOpts(self: *Builder, src: Value, dtype: DType, opts: CastOpts) Value { + if (opts.bitcast) return self.bitCast(src, dtype); + const cur_elem = src.elemType(); + const tgt_elem = dtype.toMlir(self.ctx); + if (cur_elem.eql(tgt_elem)) return src; + const cur_dtype = self.mlirElemToDType(cur_elem); + const cur_is_float = src.isFloatElem(); + const tgt_is_float = isFloatDtype(dtype); + if (cur_is_float and tgt_is_float) { + const cur_bw = dtypeBitwidth(cur_dtype); + const tgt_bw = dtypeBitwidth(dtype); + const fp8_involved = (cur_dtype == .f8e4m3fn or cur_dtype == .f8e4m3fnuz or cur_dtype == .f8e5m2 or + dtype == .f8e4m3fn or dtype == .f8e4m3fnuz or dtype == .f8e5m2); + if (fp8_involved) { + return self.fpToFpOpts(src, dtype, .{ + .rounding = if (tgt_bw < cur_bw) opts.fp_downcast_rounding orelse .rtne else null, + }); + } + if (tgt_bw > cur_bw) return self.extf(src, dtype); + const custom_rounding = opts.fp_downcast_rounding != null and opts.fp_downcast_rounding.? != .rtne; + if (custom_rounding) { + return self.fpToFpOpts(src, dtype, .{ .rounding = opts.fp_downcast_rounding orelse .rtne }); + } + return self.truncf(src, dtype); + } + if (cur_is_float and !tgt_is_float) return self.fptosi(src, dtype); + if (!cur_is_float and tgt_is_float) { + if (cur_dtype == .i1) return self.uitofp(src, dtype); + return self.sitofp(src, dtype); + } + const cur_bw = dtypeBitwidth(cur_dtype); + const tgt_bw = dtypeBitwidth(dtype); + if (tgt_bw > cur_bw) { + if (cur_dtype == .i1) return self.extui(src, dtype); + return self.extsi(src, dtype); + } + if (tgt_bw < cur_bw) return self.trunci(src, dtype); + return self.arithBitcast(src, dtype); + } + /// tt.int_to_ptr — cast int (scalar or tensor) to pointer, preserving shape. + pub fn intToPtr(self: *Builder, src: Value, pointee: DType, address_space: i32) Value { + const ptr_elem = ttir.pointerType(pointee.toMlir(self.ctx), address_space); + const result_type: *const mlir.Type = if (src.isTensor()) + .rankedTensor(src.shape().constSlice(), ptr_elem) + else + ptr_elem; + return self.emit(ttir.int_to_ptr(self.ctx, src.inner, result_type, self.loc())); + } + /// tt.ptr_to_int — cast pointer (scalar or tensor) to integer, preserving shape. + pub fn ptrToInt(self: *Builder, src: Value, out_dtype: DType) Value { + return self.emit(ttir.ptr_to_int(self.ctx, src.inner, self.swapElem(src, out_dtype), self.loc())); + } + /// tt.fp_to_fp with default rtne rounding. Use `fpToFpOpts` for other rounding modes. + pub fn fpToFp(self: *Builder, src: Value, out_dtype: DType) Value { + return self.fpToFpOpts(src, out_dtype, .{ .rounding = .rtne }); + } + /// tt.fp_to_fp with rounding option. + pub fn fpToFpOpts(self: *Builder, src: Value, out_dtype: DType, opts: FpToFpOpts) Value { + return self.emit(ttir.fp_to_fp(self.ctx, src.inner, self.swapElem(src, out_dtype), .{ .rounding = opts.rounding }, self.loc())); + } + /// tt.clampf with propagate_nan=.none. Use `clampfOpts` to override. + pub fn clampf(self: *Builder, x: Value, lo: Value, hi: Value) Value { + return self.clampfOpts(x, lo, hi, .{}); + } + /// tt.clampf with full options. + pub fn clampfOpts(self: *Builder, x: Value, lo: Value, hi: Value, opts: ClampOpts) Value { + return self.emit(ttir.clampf(self.ctx, x.inner, lo.inner, hi.inner, opts.propagate_nan, self.loc())); + } + /// `tl.sqrt_rn(x)` — precise square root (IEEE round-to-nearest). + pub fn sqrtRn(self: *Builder, x: Value) Value { + return self.emit(ttir.precise_sqrt(self.ctx, x.inner, self.loc())); + } + /// `tl.div_rn(x, y)` — precise division (IEEE round-to-nearest). + pub fn divRn(self: *Builder, x: Value, y: Value) Value { + return self.emit(ttir.precise_divf(self.ctx, x.inner, y.inner, self.loc())); + } + /// `tl.umulhi(x, y)` — most-significant N bits of the 2N-bit unsigned product. + pub fn umulhi(self: *Builder, x: Value, y: Value) Value { + return self.emit(ttir.mulhiui(self.ctx, x.inner, y.inner, self.loc())); + } + + /// tt.dot — matmul with accumulator. Result type = c_acc. Use `dotOpts` for precision. + pub fn dot(self: *Builder, a: Value, b: Value, c_acc: Value) Value { + return self.dotOpts(a, b, c_acc, .{}); + } + + /// tt.dot with full options. Result type = `c_acc.type_()`. + pub fn dotOpts(self: *Builder, a: Value, b: Value, c_acc: Value, opts: DotOpts) Value { + return self.emit(ttir.dot(self.ctx, a.inner, b.inner, c_acc.inner, c_acc.type_(), opts.input_precision, opts.max_num_imprecise_acc, self.loc())); + } + + /// tt.reshape, element type preserved. Use `reshapeOpts` for can_reorder / efficient_layout. + pub fn reshape(self: *Builder, src: Value, new_shape: []const i64) Value { + return self.reshapeOpts(src, new_shape, .{}); + } + + /// tt.reshape with full options. + pub fn reshapeOpts(self: *Builder, src: Value, new_shape: []const i64, opts: ReshapeOpts) Value { + const result_ty = mlir.Type.rankedTensor(new_shape, src.elemType()); + return self.emit(ttir.reshape(self.ctx, src.inner, result_ty, opts.can_reorder, opts.efficient_layout, self.loc())); + } + + /// tt.trans — permute dimensions. Result shape is src.shape permuted by order. + pub fn trans(self: *Builder, src: Value, order: []const i32) Value { + const src_shape = src.shape(); + std.debug.assert(order.len == src_shape.len); + var out_shape: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; + for (order) |i| out_shape.appendAssumeCapacity(src_shape.get(@intCast(i))); + const result_type = mlir.Type.rankedTensor(out_shape.constSlice(), src.elemType()); + return self.emit(ttir.trans(self.ctx, src.inner, order, result_type, self.loc())); + } + + /// `tl.permute(src, *dims)` — alias for `trans`. + pub fn permute(self: *Builder, src: Value, order: []const i32) Value { + return self.trans(src, order); + } + + /// `tl.cat` along dim 0. Use `catOpts` for can_reorder / dim. + pub fn cat(self: *Builder, lhs: Value, rhs: Value) Value { + return self.catOpts(lhs, rhs, .{}); + } + + /// `tl.cat` with full options: `join`, `permute` and `reshape`, as Triton lowers it. + pub fn catOpts(self: *Builder, lhs: Value, rhs: Value, opts: CatOpts) Value { + _ = opts.can_reorder; + const ls = lhs.shape(); + const rs = rhs.shape(); + std.debug.assert(ls.len == rs.len); + for (0..ls.len) |i| std.debug.assert(ls.get(i) == rs.get(i)); + const rank: i32 = @intCast(ls.len); + const dim: usize = @intCast(if (opts.dim < 0) opts.dim + rank else opts.dim); + var order: stdx.BoundedArray(i32, mlir.ShapedType.MAX_RANK) = .empty; + for (0..ls.len) |i| order.appendAssumeCapacity(@intCast(i)); + order.insert(dim, rank) catch unreachable; + var new_shape: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; + for (0..ls.len) |i| new_shape.appendAssumeCapacity(ls.get(i)); + new_shape.slice()[dim] = ls.get(dim) + rs.get(dim); + return self.reshape(self.permute(self.join(lhs, rhs), order.constSlice()), new_shape.constSlice()); + } + + /// `tt.join` — stack two equal-shape tensors along a new minor dim of size 2. + pub fn join(self: *Builder, lhs: Value, rhs: Value) Value { + const ls = lhs.shape(); + var out_shape: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; + for (0..ls.len) |i| out_shape.appendAssumeCapacity(ls.get(i)); + out_shape.appendAssumeCapacity(2); + const result_type = mlir.Type.rankedTensor(out_shape.constSlice(), lhs.elemType()); + return self.emit(ttir.join(self.ctx, lhs.inner, rhs.inner, result_type, self.loc())); + } + + /// `tt.split` — split a tensor whose last dim is 2 into two tensors with + /// that last dim dropped. Returns (lhs, rhs). + pub fn split(self: *Builder, src: Value) [2]Value { + const ss = src.shape(); + std.debug.assert(ss.len >= 1 and ss.get(ss.len - 1) == 2); + var out_shape: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; + for (0..ss.len - 1) |i| out_shape.appendAssumeCapacity(ss.get(i)); + const result_type = mlir.Type.rankedTensor(out_shape.constSlice(), src.elemType()); + const op = ttir.split(self.ctx, src.inner, result_type, self.loc()); + _ = op.appendTo(self.currentBlock()); + return .{ + .{ .inner = op.result(0), .kernel = self }, + .{ .inner = op.result(1), .kernel = self }, + }; + } + + /// `tl.item(src)` / `tt.unsplat` — 1-element tensor → scalar of the same element type. + pub fn item(self: *Builder, src: Value) Value { + return self.emit(ttir.unsplat(self.ctx, src.inner, src.elemType(), self.loc())); + } + + /// tt.gather — output shape matches indices, element type matches src. + pub fn gather(self: *Builder, src: Value, indices: Value, axis: i32) Value { + const result_ty = mlir.Type.rankedTensor(indices.shape().constSlice(), src.elemType()); + return self.emit(ttir.gather(self.ctx, src.inner, indices.inner, axis, result_ty, false, self.loc())); + } + + /// tt.histogram — output is `tensor`. Use `histogramOpts` for mask. + pub fn histogram(self: *Builder, src: Value, num_bins: i64) Value { + return self.histogramOpts(src, num_bins, .{}); + } + + /// tt.histogram with mask option. + pub fn histogramOpts(self: *Builder, src: Value, num_bins: i64, opts: HistogramOpts) Value { + const result_ty = mlir.Type.rankedTensor(&.{num_bins}, DType.i32.toMlir(self.ctx)); + const mask_inner: ?*const mlir.Value = if (opts.mask) |m| m.inner else null; + return self.emit(ttir.histogram(self.ctx, src.inner, result_ty, .{ .mask = mask_inner }, self.loc())); + } + + /// tt.assert — device-side assert (i1 or tensor<...xi1>). Use `deviceAssertOpts` for mask. + pub fn deviceAssert(self: *Builder, condition: Value, message: []const u8) void { + self.deviceAssertOpts(condition, message, .{}); + } + + /// tt.assert with mask option. + pub fn deviceAssertOpts(self: *Builder, condition: Value, message: []const u8, opts: DeviceAssertOpts) void { + // TODO: threading mask through ttir.assert_ when supported. + _ = opts.mask; + _ = ttir.assert_(self.ctx, condition.inner, message, self.loc()).appendTo(self.currentBlock()); + } + + /// tt.atomic_rmw — returns old value. Use `atomicRmwOpts` for mask / sem / scope. + pub fn atomicRmw(self: *Builder, rmw: ttir.RMWOp, ptr: Value, val: Value) Value { + return self.atomicRmwOpts(rmw, ptr, val, .{}); + } + + /// tt.atomic_rmw with full options. + pub fn atomicRmwOpts(self: *Builder, rmw: ttir.RMWOp, ptr: Value, val: Value, opts: AtomicRMWOpts) Value { + const mask_inner: ?*const mlir.Value = if (opts.mask) |m| m.inner else null; + return self.emit(ttir.atomic_rmw(self.ctx, rmw, ptr.inner, val.inner, opts.sem, opts.scope, .{ .mask = mask_inner }, self.loc())); + } + + /// tt.atomic_cas — returns old value. Use `atomicCasOpts` for sem / scope. + pub fn atomicCas(self: *Builder, ptr: Value, cmp: Value, val: Value) Value { + return self.atomicCasOpts(ptr, cmp, val, .{}); + } + + /// tt.atomic_cas with full options. + pub fn atomicCasOpts(self: *Builder, ptr: Value, cmp: Value, val: Value, opts: AtomicCasOpts) Value { + return self.emit(ttir.atomic_cas(self.ctx, ptr.inner, cmp.inner, val.inner, opts.sem, opts.scope, self.loc())); + } + + /// `tt.call` — call another tt.func in this module by symbol name. + pub fn call(self: *Builder, callee: []const u8, operands: []const Value, result_types: []const *const mlir.Type) []Value { + const op = ttir.call(self.ctx, callee, self.innerSlice(operands), result_types, .{}, self.loc()); + return self.emitMulti(op, result_types.len); + } + + /// tt.dot_scaled — dot with microscaling factors. Use `dotScaledOpts` for options. + pub fn dotScaled( + self: *Builder, + a: Value, + b: Value, + c_acc: Value, + a_scale: ?Value, + b_scale: ?Value, + a_elem_type: ttir.ScaleDotElemType, + b_elem_type: ttir.ScaleDotElemType, + ) Value { + return self.dotScaledOpts(a, b, c_acc, a_scale, b_scale, a_elem_type, b_elem_type, .{}); + } + + /// `tt.dot_scaled` with the full named-param set. Result type = `c_acc.type_()`. + pub fn dotScaledOpts( + self: *Builder, + a: Value, + b: Value, + c_acc: Value, + a_scale: ?Value, + b_scale: ?Value, + a_elem_type: ttir.ScaleDotElemType, + b_elem_type: ttir.ScaleDotElemType, + opts: DotScaledOpts, + ) Value { + const a_s: ?*const mlir.Value = if (a_scale) |s| s.inner else null; + const b_s: ?*const mlir.Value = if (b_scale) |s| s.inner else null; + return self.emit(ttir.dot_scaled( + self.ctx, + a.inner, + b.inner, + c_acc.inner, + a_elem_type, + b_elem_type, + c_acc.type_(), + opts.fast_math, + opts.lhs_k_pack, + opts.rhs_k_pack, + .{ .a_scale = a_s, .b_scale = b_s }, + self.loc(), + )); + } + + /// tt.extern_elementwise — call a library symbol pointwise. + /// Result is `tensor`; pass `&.{}` for scalar. + pub fn externElementwise( + self: *Builder, + srcs: []const Value, + result_shape: []const i64, + result_dtype: DType, + libname: []const u8, + libpath: []const u8, + symbol: []const u8, + ) Value { + return self.externElementwiseOpts(srcs, result_shape, result_dtype, libname, libpath, symbol, .{}); + } + + /// `tt.extern_elementwise` with the full named-param set. + pub fn externElementwiseOpts( + self: *Builder, + srcs: []const Value, + result_shape: []const i64, + result_dtype: DType, + libname: []const u8, + libpath: []const u8, + symbol: []const u8, + opts: ExternElementwiseOpts, + ) Value { + const result_type: *const mlir.Type = if (result_shape.len == 0) + result_dtype.toMlir(self.ctx) + else + .rankedTensor(result_shape, result_dtype.toMlir(self.ctx)); + return self.emit(ttir.extern_elementwise( + self.ctx, + self.innerSlice(srcs), + result_type, + libname, + libpath, + symbol, + opts.is_pure, + self.loc(), + )); + } + + /// tt.print — device-side print. `is_signed` has one entry per arg (1=signed, 0=unsigned). + /// Use `devicePrintOpts` for hex. + pub fn devicePrint( + self: *Builder, + prefix: []const u8, + args: []const Value, + is_signed: []const i32, + ) void { + self.devicePrintOpts(prefix, args, is_signed, .{}); + } + + /// tt.print with full options. + pub fn devicePrintOpts( + self: *Builder, + prefix: []const u8, + args: []const Value, + is_signed: []const i32, + opts: PrintOpts, + ) void { + _ = ttir.print(self.ctx, prefix, opts.hex, self.innerSlice(args), is_signed, self.loc()).appendTo(self.currentBlock()); + } + + /// tt.make_tensor_descriptor — build a TMA descriptor. + /// shape/strides are runtime; block_shape/dtype describe the compile-time tile. + pub fn makeTensorDescriptor( + self: *Builder, + base: Value, + shape: []const Value, + strides: []const Value, + padding: ttir.PaddingOption, + block_shape: []const i64, + dtype: DType, + ) Value { + const result_type = ttir.tensorDescType(block_shape, dtype.toMlir(self.ctx), .{}); + return self.emit(ttir.make_tensor_descriptor( + self.ctx, + base.inner, + self.innerSlice(shape), + self.innerSlice(strides), + padding, + result_type, + self.loc(), + )); + } + + /// tt.descriptor_load — TMA load. Use `descriptorLoadOpts` for cache / eviction. + pub fn descriptorLoad( + self: *Builder, + desc: Value, + indices: []const Value, + shape: []const i64, + dtype: DType, + ) Value { + return self.descriptorLoadOpts(desc, indices, shape, dtype, .{}); + } + + /// `tt.descriptor_load` with the full named-param set. + pub fn descriptorLoadOpts( + self: *Builder, + desc: Value, + indices: []const Value, + shape: []const i64, + dtype: DType, + opts: DescriptorLoadOpts, + ) Value { + const result_ty = mlir.Type.rankedTensor(shape, dtype.toMlir(self.ctx)); + return self.emit(ttir.descriptor_load(self.ctx, desc.inner, self.innerSlice(indices), result_ty, opts.cache_modifier, opts.eviction_policy, self.loc())); + } + + /// `tt.descriptor_store` — TMA store. + pub fn descriptorStore(self: *Builder, desc: Value, src: Value, indices: []const Value) void { + _ = ttir.descriptor_store(self.ctx, desc.inner, src.inner, self.innerSlice(indices), self.loc()).appendTo(self.currentBlock()); + } + + /// `tt.descriptor_reduce` — TMA reducing store. + pub fn descriptorReduce( + self: *Builder, + kind: ttir.DescriptorReduceKind, + desc: Value, + src: Value, + indices: []const Value, + ) void { + _ = ttir.descriptor_reduce(self.ctx, kind, desc.inner, src.inner, self.innerSlice(indices), self.loc()).appendTo(self.currentBlock()); + } + + /// tt.descriptor_gather — TMA gather by x_offsets + y_offset. + pub fn descriptorGather( + self: *Builder, + desc: Value, + x_offsets: Value, + y_offset: Value, + shape: []const i64, + dtype: DType, + ) Value { + const result_ty = mlir.Type.rankedTensor(shape, dtype.toMlir(self.ctx)); + return self.emit(ttir.descriptor_gather(self.ctx, desc.inner, x_offsets.inner, y_offset.inner, result_ty, self.loc())); + } + + /// `tt.descriptor_scatter` — TMA scatter. + pub fn descriptorScatter(self: *Builder, desc: Value, x_offsets: Value, y_offset: Value, src: Value) void { + _ = ttir.descriptor_scatter(self.ctx, desc.inner, x_offsets.inner, y_offset.inner, src.inner, self.loc()).appendTo(self.currentBlock()); + } + + // ==================== math dialect ==================== + + pub fn exp(self: *Builder, x: Value) Value { + return self.emit(math.exp(self.ctx, x.inner, self.loc())); + } + pub fn exp2(self: *Builder, x: Value) Value { + return self.emit(math.exp2(self.ctx, x.inner, self.loc())); + } + pub fn log(self: *Builder, x: Value) Value { + return self.emit(math.log(self.ctx, x.inner, self.loc())); + } + pub fn log2(self: *Builder, x: Value) Value { + return self.emit(math.log2(self.ctx, x.inner, self.loc())); + } + pub fn sqrt(self: *Builder, x: Value) Value { + return self.emit(math.sqrt(self.ctx, x.inner, self.loc())); + } + pub fn rsqrt(self: *Builder, x: Value) Value { + return self.emit(math.rsqrt(self.ctx, x.inner, self.loc())); + } + pub fn sin(self: *Builder, x: Value) Value { + return self.emit(math.sin(self.ctx, x.inner, self.loc())); + } + pub fn cos(self: *Builder, x: Value) Value { + return self.emit(math.cos(self.ctx, x.inner, self.loc())); + } + pub fn tan(self: *Builder, x: Value) Value { + return self.emit(math.tan(self.ctx, x.inner, self.loc())); + } + pub fn tanh(self: *Builder, x: Value) Value { + return self.emit(math.tanh(self.ctx, x.inner, self.loc())); + } + pub fn erf(self: *Builder, x: Value) Value { + return self.emit(math.erf(self.ctx, x.inner, self.loc())); + } + pub fn absf(self: *Builder, x: Value) Value { + return self.emit(math.absf(self.ctx, x.inner, self.loc())); + } + pub fn absi(self: *Builder, x: Value) Value { + return self.emit(math.absi(self.ctx, x.inner, self.loc())); + } + /// `tl.abs(x)` — absolute value, auto-dispatch by element type (float→`absf`, + /// signed int→`absi`, unsigned int→no-op). + pub fn abs(self: *Builder, x: Value) Value { + if (x.isFloatElem()) return self.absf(x); + return self.absi(x); + } + pub fn floor(self: *Builder, x: Value) Value { + return self.emit(math.floor(self.ctx, x.inner, self.loc())); + } + pub fn ceil(self: *Builder, x: Value) Value { + return self.emit(math.ceil(self.ctx, x.inner, self.loc())); + } + pub fn powf(self: *Builder, x: Value, y: Value) Value { + return self.emit(math.powf(self.ctx, x.inner, y.inner, self.loc())); + } + pub fn fma(self: *Builder, a: Value, b: Value, c_: Value) Value { + return self.emit(math.fma(self.ctx, a.inner, b.inner, c_.inner, self.loc())); + } + /// `math.clampf` — distinct from `tt.clampf` (no propagateNan attr). + pub fn mathClampf(self: *Builder, value: Value, lo: Value, hi: Value) Value { + return self.emit(math.clampf(self.ctx, value.inner, lo.inner, hi.inner, self.loc())); + } + /// `math.sincos` — returns (sin, cos). + pub fn sincos(self: *Builder, x: Value) [2]Value { + const op = math.sincos(self.ctx, x.inner, self.loc()); + _ = op.appendTo(self.currentBlock()); + return .{ .{ .inner = op.result(0), .kernel = self }, .{ .inner = op.result(1), .kernel = self } }; + } + /// `math.fpowi` — float base, integer exponent. + pub fn fpowi(self: *Builder, base: Value, power: Value) Value { + return self.emit(math.fpowi(self.ctx, base.inner, power.inner, self.loc())); + } + pub fn ipowi(self: *Builder, base: Value, power: Value) Value { + return self.emit(math.ipowi(self.ctx, base.inner, power.inner, self.loc())); + } + pub fn atan2(self: *Builder, y: Value, x: Value) Value { + return self.emit(math.atan2(self.ctx, y.inner, x.inner, self.loc())); + } + pub fn copysign(self: *Builder, mag: Value, sign: Value) Value { + return self.emit(math.copysign(self.ctx, mag.inner, sign.inner, self.loc())); + } + pub fn mathRound(self: *Builder, x: Value) Value { + return self.emit(math.round(self.ctx, x.inner, self.loc())); + } + pub fn mathRoundEven(self: *Builder, x: Value) Value { + return self.emit(math.roundeven(self.ctx, x.inner, self.loc())); + } + pub fn mathTrunc(self: *Builder, x: Value) Value { + return self.emit(math.trunc(self.ctx, x.inner, self.loc())); + } + + /// `arith.convertf` — same-bitwidth float cast (e.g. f16 ↔ bf16) with + /// default rounding. Shape preserved from `src`. Use `convertfOpts` for + /// `fastMath`. + pub fn convertf(self: *Builder, src: Value, out_dtype: DType) Value { + return self.convertfOpts(src, out_dtype, .{}); + } + /// `arith.convertf` with the full named-param set. + pub fn convertfOpts(self: *Builder, src: Value, out_dtype: DType, opts: arith.ConvertFOpts) Value { + return self.emit(arith.convertf(self.ctx, src.inner, self.swapElem(src, out_dtype), opts, self.loc())); + } + /// `arith.scaling_truncf` — MXFP downcast with per-block scales, default + /// options. Shape preserved from `src`. Use `scalingTruncfOpts` for + /// `fastMath`. + pub fn scalingTruncf(self: *Builder, src: Value, scale: Value, out_dtype: DType) Value { + return self.scalingTruncfOpts(src, scale, out_dtype, .{}); + } + /// `arith.scaling_truncf` with the full named-param set. + pub fn scalingTruncfOpts(self: *Builder, src: Value, scale: Value, out_dtype: DType, opts: arith.ConvertFOpts) Value { + return self.emit(arith.scaling_truncf(self.ctx, src.inner, scale.inner, self.swapElem(src, out_dtype), opts, self.loc())); + } + + // ==================== tt.map_elementwise / elementwise_inline_asm ==================== + + /// `tt.map_elementwise` — body builder receives one scalar Value per + /// `srcs` tensor (repeated `pack` times, flattened), and must return the + /// scalar results to feed back via `tt.map_elementwise.return`. + pub fn mapElementwise( + self: *Builder, + srcs: []const Value, + pack: i32, + scalar_arg_types: []const *const mlir.Type, + result_tensor_types: []const *const mlir.Type, + comptime body_fn: anytype, + body_ctx: anytype, + ) []Value { + const scratch = self.arena.allocator(); + const locs = scratch.alloc(*const mlir.Location, scalar_arg_types.len) catch @panic("Builder.mapElementwise OOM"); + for (locs) |*l| l.* = self.loc(); + const body_block = mlir.Block.init(scalar_arg_types, locs); + + self.pushBlock(body_block); + const block_args = scratch.alloc(Value, scalar_arg_types.len) catch @panic("Builder.mapElementwise OOM"); + for (block_args, 0..) |*v, i| v.* = .{ .inner = body_block.argument(i), .kernel = self }; + const out_vals: []const Value = body_fn(self, block_args, body_ctx); + _ = ttir.map_elementwise_return(self.ctx, self.innerSlice(out_vals), self.loc()).appendTo(body_block); + self.popBlock(); + + const op = ttir.map_elementwise( + self.ctx, + self.innerSlice(srcs), + pack, + body_block, + result_tensor_types, + self.loc(), + ); + return self.emitMulti(op, result_tensor_types.len); + } + + /// `tl.inline_asm_elementwise(asm, constraints, args, dtype)` — inline asm + /// pointwise op with default options. Use `inlineAsmElementwiseOpts` for + /// `is_pure` / `pack`. + pub fn inlineAsmElementwise( + self: *Builder, + asm_string: []const u8, + constraints: []const u8, + args: []const Value, + result_types: []const *const mlir.Type, + ) []Value { + return self.inlineAsmElementwiseOpts(asm_string, constraints, args, result_types, .{}); + } + + /// `tl.inline_asm_elementwise(asm, constraints, args, dtype, is_pure=..., pack=...)` + /// with the full named-param set. + pub fn inlineAsmElementwiseOpts( + self: *Builder, + asm_string: []const u8, + constraints: []const u8, + args: []const Value, + result_types: []const *const mlir.Type, + opts: InlineAsmOpts, + ) []Value { + const op = ttir.elementwise_inline_asm(self.ctx, asm_string, constraints, self.innerSlice(args), result_types, opts.is_pure, opts.pack, self.loc()); + return self.emitMulti(op, result_types.len); + } + + // ==================== scf.parallel / scf.reduce / scf.forall ==================== + + pub fn parallel( + self: *Builder, + lbs: []const Value, + ubs: []const Value, + steps: []const Value, + inits: []const Value, + reduction_result_types: []const *const mlir.Type, + comptime body_fn: anytype, + comptime combiners: anytype, + body_ctx: anytype, + ) []Value { + std.debug.assert(lbs.len == ubs.len); + std.debug.assert(inits.len == reduction_result_types.len); + + const scratch = self.arena.allocator(); + const idx_ty = .index(self.ctx); + const iv_types = scratch.alloc(*const mlir.Type, lbs.len) catch @panic("Builder.parallel OOM"); + const iv_locs = scratch.alloc(*const mlir.Location, lbs.len) catch @panic("Builder.parallel OOM"); + for (iv_types, iv_locs) |*t, *l| { + t.* = idx_ty; + l.* = self.loc(); + } + const body_block = mlir.Block.init(iv_types, iv_locs); + + self.pushBlock(body_block); + const iv_args = scratch.alloc(Value, lbs.len) catch @panic("Builder.parallel OOM"); + for (iv_args, 0..) |*v, i| v.* = .{ .inner = body_block.argument(i), .kernel = self }; + const reduce_operands: []const Value = body_fn(self, iv_args, body_ctx); + std.debug.assert(reduce_operands.len == reduction_result_types.len); + + // Build scf.reduce with one region per reduction. + const red_blocks = scratch.alloc(*mlir.Block, combiners.len) catch @panic("Builder.parallel OOM"); + inline for (combiners, 0..) |comb, i| { + const arg_ty = reduce_operands[i].type_(); + const arg_locs: [2]*const mlir.Location = .{ self.loc(), self.loc() }; + const red_block = mlir.Block.init(&.{ arg_ty, arg_ty }, &arg_locs); + self.pushBlock(red_block); + const lhs_v: Value = .{ .inner = red_block.argument(0), .kernel = self }; + const rhs_v: Value = .{ .inner = red_block.argument(1), .kernel = self }; + const combined: Value = comb(self, lhs_v, rhs_v, body_ctx); + _ = scf.reduce_return(self.ctx, combined.inner, self.loc()).appendTo(red_block); + self.popBlock(); + red_blocks[i] = red_block; + } + + _ = scf.reduce(self.ctx, self.innerSlice(reduce_operands), red_blocks, self.loc()).appendTo(body_block); + self.popBlock(); + + const op = scf.parallel( + self.ctx, + self.innerSlice(lbs), + self.innerSlice(ubs), + self.innerSlice(steps), + self.innerSlice(inits), + body_block, + reduction_result_types, + self.loc(), + ); + _ = op.appendTo(self.currentBlock()); + + const out = self.arena.allocator().alloc(Value, reduction_result_types.len) catch @panic("Builder.parallel OOM"); + for (0..reduction_result_types.len) |i| out[i] = .{ .inner = op.result(i), .kernel = self }; + return out; + } + + // ==================== reduce / scan ==================== + + /// `tt.reduce` along `axis`, single-input. See `ReduceArgs` for fields. + /// `ctx` is separate from `args` so `ReduceArgs(@TypeOf(ctx))` can be + /// inferred. + pub fn reduce(self: *Builder, ctx: anytype, args: ReduceArgs(@TypeOf(ctx))) Value { + const region = mlir.Block.init(&.{ args.elem, args.elem }, &.{ self.loc(), self.loc() }); + + self.pushBlock(region); + const lhs: Value = .{ .inner = region.argument(0), .kernel = self }; + const rhs: Value = .{ .inner = region.argument(1), .kernel = self }; + const combined: Value = args.combine(self, lhs, rhs, ctx); + _ = ttir.reduce_return(self.ctx, &.{combined.inner}, self.loc()).appendTo(region); + self.popBlock(); + + const op = ttir.reduce(self.ctx, &.{args.src.inner}, args.axis, region, &.{args.result}, self.loc()); + _ = op.appendTo(self.currentBlock()); + return .{ .inner = op.result(0), .kernel = self }; + } + + /// `tt.scan` along `axis`, single-input. See `ScanArgs` for fields. + pub fn scan(self: *Builder, ctx: anytype, args: ScanArgs(@TypeOf(ctx))) Value { + const region = mlir.Block.init(&.{ args.elem, args.elem }, &.{ self.loc(), self.loc() }); + + self.pushBlock(region); + const lhs: Value = .{ .inner = region.argument(0), .kernel = self }; + const rhs: Value = .{ .inner = region.argument(1), .kernel = self }; + const combined: Value = args.combine(self, lhs, rhs, ctx); + _ = ttir.scan_return(self.ctx, &.{combined.inner}, self.loc()).appendTo(region); + self.popBlock(); + + const op = ttir.scan(self.ctx, &.{args.src.inner}, args.axis, args.reverse, region, &.{args.result}, self.loc()); + _ = op.appendTo(self.currentBlock()); + return .{ .inner = op.result(0), .kernel = self }; + } + + // ==================== SCF regions ==================== + + /// `for iv in [lower, upper) step` carrying `inits`. Literal bounds take the + /// type of a Value bound, else `lower` is lifted (`i32` when it fits). + pub fn openFor( + self: *Builder, + lower: anytype, + upper: anytype, + step: anytype, + inits: anytype, + ) ForScope(common.arity(@TypeOf(inits), "openFor: inits")) { + return self.openForOpts(lower, upper, step, inits, .{}); + } + + /// `openFor` with `scf.for` options. + pub fn openForOpts( + self: *Builder, + lower: anytype, + upper: anytype, + step: anytype, + inits: anytype, + opts: common.ForOpts, + ) ForScope(common.arity(@TypeOf(inits), "openFor: inits")) { + return common.openFor(self, Value, common.forBounds(self, Value, lower, upper, step, Builder.lift), inits, opts); + } + + pub fn returnIf(self: *Builder, cond: Value) void { + self.returnIfOpts(cond, .{}); + } + + pub fn returnIfOpts(self: *Builder, cond: Value, opts: ReturnIfOpts) void { + const current = self.currentBlock(); + const region = current.parentRegion(); + const cont_block = mlir.Block.init(&.{}, &.{}); + + const ret_target: *mlir.Block = if (opts.inline_return) blk: { + const rb = mlir.Block.init(&.{}, &.{}); + _ = ttir.return_(self.ctx, &.{}, self.loc()).appendTo(rb); + region.appendOwnedBlock(rb); + break :blk rb; + } else self.exitBlock(); + + region.appendOwnedBlock(cont_block); + + _ = cf.cond_br(self.ctx, cond.inner, ret_target, &.{}, cont_block, &.{}, .{}, self.loc()).appendTo(current); + self.continueIn(cont_block); + } + + /// The block every non-inline return branches to, which `finish` appends. + fn exitBlock(self: *Builder) *mlir.Block { + if (self.exit_block == null) { + self.exit_block = mlir.Block.init(&.{}, &.{}); + _ = ttir.return_(self.ctx, &.{}, self.loc()).appendTo(self.exit_block.?); + } + return self.exit_block.?; + } + + /// Continues in `block`, the fall-through of an early return, in place of + /// the current block. + fn continueIn(self: *Builder, block: *mlir.Block) void { + self.block_stack.replaceTop(block); + } + + pub fn openReturnIf(self: *Builder, cond: Value) ReturnIfScope { + return self.openReturnIfOpts(cond, .{}); + } + + pub fn openReturnIfOpts(self: *Builder, cond: Value, opts: ReturnIfOpts) ReturnIfScope { + const current = self.currentBlock(); + const region = current.parentRegion(); + + const ret_block = mlir.Block.init(&.{}, &.{}); + const cont_block = mlir.Block.init(&.{}, &.{}); + region.appendOwnedBlock(ret_block); + region.appendOwnedBlock(cont_block); + + _ = cf.cond_br(self.ctx, cond.inner, ret_block, &.{}, cont_block, &.{}, .{}, self.loc()).appendTo(current); + + self.pushBlock(ret_block); + + return .{ + .kernel = self, + .ret_block = ret_block, + .cont_block = cont_block, + .opts = opts, + }; + } + + pub fn openIf(self: *Builder, cond: Value) IfOnlyScope { + return common.openIf(self, Value, cond); + } + + pub fn openIfElse(self: *Builder, cond: Value, result_types: anytype) IfScope(common.arity(@TypeOf(result_types), "openIfElse: result_types")) { + return common.openIfElse(self, Value, cond, result_types); + } + + pub fn openWhile( + self: *Builder, + inits: anytype, + after_types: anytype, + ) WhileScope(common.arity(@TypeOf(inits), "openWhile: inits"), common.arity(@TypeOf(after_types), "openWhile: after_types")) { + return common.openWhile(self, Value, inits, after_types); + } + + // ==================== finalization ==================== + + /// Append the final `tt.return` (kernels return nothing), verify the + /// module, and serialize to a NUL-terminated TTIR string owned by the + /// kernel's allocator (passed to `init`). Caller must + /// `defer allocator.free(ir)`. + pub fn finish(self: *Builder) FinishError![:0]const u8 { + // After any `returnIf` call, the entry block already has a + // `cf.cond_br` terminator; the fall-through block sits on the stack + // as the new insertion point. `currentBlock()` returns that block + // (the entry block if no cf was emitted), so the final terminator + // goes to the correct "tail" block in either case. + const current = self.currentBlock(); + if (self.exit_block) |exit| { + const region = self.entry_block.?.parentRegion(); + region.appendOwnedBlock(exit); + _ = cf.br(self.ctx, exit, &.{}, self.loc()).appendTo(current); + } else { + _ = ttir.return_(self.ctx, &.{}, self.loc()).appendTo(current); + } + + try common.verify(self.module, "dsl/triton"); + + var al: std.Io.Writer.Allocating = .init(self.allocator); + defer al.deinit(); + + try al.writer.print("{f}", .{self.module.operation().fmt(.{ + .debug_info = true, + .debug_info_pretty_form = false, + .print_name_loc_as_prefix = true, + })}); + + return try self.allocator.dupeSentinel(u8, al.written(), 0); + } +}; diff --git a/dsl/triton/dtype.zig b/dsl/triton/dtype.zig new file mode 100644 index 00000000..45018e67 --- /dev/null +++ b/dsl/triton/dtype.zig @@ -0,0 +1,74 @@ +const std = @import("std"); +const mlir = @import("mlir"); + +pub const DType = enum { + i1, + i8, + i16, + i32, + i64, + f16, + bf16, + f32, + f64, + f8e4m3fn, + f8e4m3fnuz, + f8e5m2, + + pub fn toMlir(self: DType, ctx: *mlir.Context) *const mlir.Type { + return switch (self) { + .i1 => .int(ctx, .i1), + .i8 => .int(ctx, .i8), + .i16 => .int(ctx, .i16), + .i32 => .int(ctx, .i32), + .i64 => .int(ctx, .i64), + .f16 => .float(ctx, .f16), + .bf16 => .float(ctx, .bf16), + .f32 => .float(ctx, .f32), + .f64 => .float(ctx, .f64), + .f8e4m3fn => .float(ctx, .f8e4m3fn), + .f8e4m3fnuz => .float(ctx, .f8e4m3fnuz), + .f8e5m2 => .float(ctx, .f8e5m2), + }; + } +}; + +pub fn isFloatDtype(dt: DType) bool { + return switch (dt) { + .f16, .bf16, .f32, .f64, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2 => true, + else => false, + }; +} + +pub fn dtypeBitwidth(dt: DType) usize { + return switch (dt) { + .i1 => 1, + .i8, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2 => 8, + .i16, .f16, .bf16 => 16, + .i32, .f32 => 32, + .i64, .f64 => 64, + }; +} + +pub fn intBitwidth(dt: DType) u32 { + return switch (dt) { + .i1 => 1, + .i8 => 8, + .i16 => 16, + .i32 => 32, + .i64 => 64, + else => std.debug.panic("intBitwidth: not an integer DType: {s}", .{@tagName(dt)}), + }; +} + +pub fn mlirElemToDType(ctx: *mlir.Context, elem: *const mlir.Type) DType { + inline for (comptime std.meta.fieldNames(DType)) |field_name| { + const dt = @field(DType, field_name); + if (elem.eql(dt.toMlir(ctx))) return dt; + } + @panic("element type not a recognized Triton DType"); +} + +test { + std.testing.refAllDecls(@This()); +} diff --git a/examples/.bazelrc b/examples/.bazelrc deleted file mode 100644 index 26768e9b..00000000 --- a/examples/.bazelrc +++ /dev/null @@ -1,82 +0,0 @@ -# If we are running from the ZML repo, try to load the .bazelrc-zml above -try-import %workspace%/../.bazelrc-examples -try-import %workspace%/.bazelrc-zml - -# Faster hash function -startup --digest_function=blake3 - -common --enable_workspace=false - -# Don't need for now -common --lockfile_mode=update - -# Needed by LLVM and/or XLA -common --experimental_repo_remote_exec - -# Self explanatory -common --enable_platform_specific_config - -# Ensure host tools are built optimized -build --host_compilation_mode=opt - -# Suppress all warnings, LLVM and friends are too noisy -build --copt='-w' -build --host_copt='-w' - -# Enable C++17 -build --cxxopt='-std=c++17' -build --host_cxxopt='-std=c++17' -build --enable_runfiles - -# Ensure the local env doesn't leak into the build -build --incompatible_strict_action_env - -# We don't to check the built files in the sandbox or repositories -build --noexperimental_check_output_files -fetch --noexperimental_check_output_files -query --noexperimental_check_output_files -build --noexperimental_check_external_repository_files -fetch --noexperimental_check_external_repository_files -query --noexperimental_check_external_repository_files - -# Performance -build --reuse_sandbox_directories -build --nolegacy_external_runfiles - -build --experimental_output_directory_naming_scheme=diff_against_dynamic_baseline -# Mount the zig cache into the sandboxes -build:macos --sandbox_add_mount_pair=/var/tmp -build:linux --sandbox_add_mount_pair=/tmp - -# Ensure workers are sandboxed -build --worker_sandboxing - -# Allows symlinks with spaces -build --experimental_inprocess_symlink_creation - -# Build in debug for now, by default -build --compilation_mode=dbg - -build --experimental_cc_shared_library -build --incompatible_use_cc_configure_from_rules_cc -build --experimental_starlark_cc_import - -build --compiler=clang - -build:macos --macos_sdk_version=13.0 - -# Print test errors in the console -test --test_output=errors - -build:debug --compilation_mode=opt -build:debug --strategy=ZigBuildLib=local -build:debug --strategy=ZigBuildObj=local -build:debug --strategy=ZigBuildTestLib=local - -common:silent --noshow_progress -common:silent --ui_event_filters=,+error,+fail -common:silent --show_result=0 -common:silent --logging=0 - -common --registry=https://bazel-registry.zml.ai -common --registry=https://bcr.bazel.build diff --git a/examples/.bazelversion b/examples/.bazelversion deleted file mode 100644 index 643916c0..00000000 --- a/examples/.bazelversion +++ /dev/null @@ -1 +0,0 @@ -7.3.1 diff --git a/examples/.gitignore b/examples/.gitignore deleted file mode 100644 index 6add214f..00000000 --- a/examples/.gitignore +++ /dev/null @@ -1,30 +0,0 @@ -# Bazel symlinks -bazel-* - -# macOS -.DS_Store - -# Local Checkouts -/stablehlo/ -/llvm-project/ -/xla/ - -# Just in case, Zig caches -.zig-cache/ -zig-out/ - -# Models -*.safetensors -*.pb -*.trace.json -*.pt -*.bin - -# Data files -*.wav -*.ecdc -.venv - -# Editor specific -## Neovim -.nvim.local.lua diff --git a/examples/.nvim.lua b/examples/.nvim.lua deleted file mode 100644 index 041d86c0..00000000 --- a/examples/.nvim.lua +++ /dev/null @@ -1,13 +0,0 @@ -local f, err = loadfile('.nvim.local.lua') -if not err then - f() -else - require('lspconfig')["zls"].setup { - cmd = { "tools/zls.sh" }, - settings = { - zls = { - enable_autofix = true, - }, - }, - } -end diff --git a/examples/MODULE.bazel b/examples/MODULE.bazel deleted file mode 100644 index 942b2f4b..00000000 --- a/examples/MODULE.bazel +++ /dev/null @@ -1,163 +0,0 @@ -module(name = "examples") - -bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "rules_zig", version = "20240912.0-41bfe84") -bazel_dep(name = "platforms", version = "0.0.10") -bazel_dep(name = "zml", version = "0.1.0") -bazel_dep(name = "aspect_bazel_lib", version = "2.8.1.1") - -bazel_dep(name = "rules_oci", version = "2.0.0") -oci = use_extension("@rules_oci//oci:extensions.bzl", "oci") -oci.pull( - name = "distroless_cc_debian12", - digest = "sha256:1850aee2ff72864350058d83d681c757d45c885986d15fcca7309b9e5c69f39a", - image = "gcr.io/distroless/cc-debian12", - platforms = [ - "linux/amd64", - ], -) -use_repo(oci, "distroless_cc_debian12", "distroless_cc_debian12_linux_amd64") - -# Mnist weights -http_file = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file") - -http_file( - name = "com_github_ggerganov_ggml_mnist", - downloaded_file_path = "mnist.pt", - sha256 = "d8a25252e28915e147720c19223721f0f53e3317493727ca754a2dd672450ba9", - url = "https://github.com/ggerganov/ggml/raw/18703ad600cc68dbdb04d57434c876989a841d12/examples/mnist/models/mnist/mnist_model.state_dict", -) - -http_file( - name = "com_github_ggerganov_ggml_mnist_data", - downloaded_file_path = "mnist.ylc", - sha256 = "0fa7898d509279e482958e8ce81c8e77db3f2f8254e26661ceb7762c4d494ce7", - url = "https://github.com/ggerganov/ggml/raw/18703ad600cc68dbdb04d57434c876989a841d12/examples/mnist/models/mnist/t10k-images.idx3-ubyte", -) - -# Llama weights -huggingface = use_extension("@zml//bazel:huggingface.bzl", "huggingface") - -huggingface.model( - name = "Karpathy-TinyLlama-Stories", - build_file_content = """\ -load("@bazel_skylib//rules:copy_file.bzl", "copy_file") - -# leverage copy_file to rename tokenizer extension -# which allow zml.aio.detectFormatAndLoadTokenizer -# to leverage the right tokenizer -copy_file( - name = "stories15M", - src = "stories15M.bin", - out = "stories15M.tinyllama", - allow_symlink = True, - visibility = ["//visibility:public"], -) - -copy_file( - name = "stories110M", - src = "stories110M.bin", - out = "stories110M.tinyllama", - allow_symlink = True, - visibility = ["//visibility:public"], -) -""", - commit = "0bd21da7698eaf29a0d7de3992de8a46ef624add", - includes = [ - "stories15M.bin", - "stories110M.bin", - ], - model = "karpathy/tinyllamas", -) -use_repo(huggingface, "Karpathy-TinyLlama-Stories") - -http_file( - name = "Karpathy-TinyLlama-Tokenizer", - downloaded_file_path = "stories260K.tinyllama", - sha256 = "50a52ef822ee9e83de5ce9d0be0a025a773d019437f58b5ff9dcafb063ece361", - url = "https://github.com/karpathy/llama2.c/raw/c02865df300f3bd9e567ce061000dc23bf785a17/tokenizer.bin", -) - -huggingface.model( - name = "Meta-Llama-3.1-8B-Instruct", - build_file_content = """\ -package(default_visibility = ["//visibility:public"]) -filegroup( - name = "model", - srcs = glob(["*.safetensors"]) + ["model.safetensors.index.json"], -) - -filegroup( - name = "tokenizer", - srcs = ["tokenizer.json"], -) -""", - commit = "5206a32e0bd3067aef1ce90f5528ade7d866253f", - includes = [ - "*.safetensors", - "model.safetensors.index.json", - "tokenizer.json", - ], - model = "meta-llama/Meta-Llama-3.1-8B-Instruct", -) -use_repo(huggingface, "Meta-Llama-3.1-8B-Instruct") - -huggingface.model( - name = "TinyLlama-1.1B-Chat-v1.0", - build_file_content = """\ -package(default_visibility = ["//visibility:public"]) - -filegroup( - name = "model", - srcs = ["model.safetensors"], -) - -filegroup( - name = "tokenizer", - srcs = ["tokenizer.model"], -) -""", - commit = "fe8a4ea1ffedaf415f4da2f062534de366a451e6", - includes = [ - "model.safetensors", - "tokenizer.model", - ], - model = "TinyLlama/TinyLlama-1.1B-Chat-v1.0", -) -use_repo(huggingface, "TinyLlama-1.1B-Chat-v1.0") - -huggingface.model( - name = "OpenLM-Research-OpenLLaMA-3B", - build_file_content = """\ -load("@bazel_skylib//rules:copy_file.bzl", "copy_file") - -package(default_visibility = ["//visibility:public"]) - -filegroup( - name = "model", - srcs = ["model.safetensors"], -) - -filegroup( - name = "tokenizer", - srcs = [":tokenizer_pb"], -) - -# leverage copy_file to rename tokenizer extension -# which allow zml.aio.detectFormatAndLoadTokenizer -# to leverage the right tokenizer -copy_file( - name = "tokenizer_pb", - src = "tokenizer.model", - out = "tokenizer.pb", - allow_symlink = True, -) -""", - commit = "fcc2e809eb8f14dabba84d76a0ddc17b8ea05356", - includes = [ - "model.safetensors", - "tokenizer.model", - ], - model = "openlm-research/open_llama_3b", -) -use_repo(huggingface, "OpenLM-Research-OpenLLaMA-3B") diff --git a/examples/MODULE.bazel.lock b/examples/MODULE.bazel.lock deleted file mode 100644 index a8d0c371..00000000 --- a/examples/MODULE.bazel.lock +++ /dev/null @@ -1,3205 +0,0 @@ -{ - "lockFileVersion": 11, - "registryFileHashes": { - "https://bazel-registry.zml.ai/modules/abseil-cpp/20210324.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/abseil-cpp/20211102.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/abseil-cpp/20230125.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/abseil-cpp/20230802.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/abseil-cpp/20240116.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/apple_support/1.11.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/apple_support/1.5.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/aspect_bazel_lib/2.7.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/aspect_bazel_lib/2.8.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_features/1.1.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_features/1.10.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_features/1.11.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_features/1.3.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_features/1.4.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_skylib/1.0.3/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_skylib/1.1.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_skylib/1.2.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_skylib/1.2.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_skylib/1.3.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_skylib/1.4.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_skylib/1.4.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_skylib/1.5.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_skylib/1.6.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/bazel_skylib/1.7.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/buildozer/7.1.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/google_benchmark/1.8.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/googletest/1.11.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/googletest/1.14.0.bcr.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/googletest/1.14.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/hermetic_cc_toolchain/3.1.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/libpfm/4.11.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/nlohmann_json/3.6.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/patchelf/0.18.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/platforms/0.0.10/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/platforms/0.0.4/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/platforms/0.0.5/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/platforms/0.0.6/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/platforms/0.0.7/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/platforms/0.0.8/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/platforms/0.0.9/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/protobuf/21.7/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/protobuf/3.19.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/protobuf/3.19.6/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/pybind11_bazel/2.11.1.bzl.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/pybind11_bazel/2.11.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/re2/2023-09-01/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/re2/2024-02-01/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_apple/3.2.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_cc/0.0.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_cc/0.0.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_cc/0.0.6/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_cc/0.0.8/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_cc/0.0.9/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_foreign_cc/0.9.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_java/4.0.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_java/5.3.5/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_java/6.3.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_java/7.3.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_java/7.6.5/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_jvm_external/4.4.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_jvm_external/5.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_license/0.0.3/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_license/0.0.4/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_license/0.0.7/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_license/0.0.8/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_oci/2.0.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_pkg/0.7.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_pkg/0.9.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_pkg/1.0.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_proto/4.0.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_proto/5.3.0-21.7/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_proto/6.0.0-rc1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_proto/6.0.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_python/0.10.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_python/0.22.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_python/0.25.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_python/0.29.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_python/0.31.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_python/0.4.0/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/rules_swift/1.15.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/stardoc/0.5.1/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/stardoc/0.5.3/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/stardoc/0.5.4/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/stardoc/0.6.2/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/zlib/1.2.11/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/zlib/1.2.12/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/zlib/1.2.13/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/zlib/1.3.1.bcr.3/MODULE.bazel": "not found", - "https://bazel-registry.zml.ai/modules/zstd/1.5.6/MODULE.bazel": "not found", - "https://bcr.bazel.build/bazel_registry.json": "8a28e4aff06ee60aed2a8c281907fb8bcbf3b753c91fb5a5c57da3215d5b3497", - "https://bcr.bazel.build/modules/abseil-cpp/20210324.2/MODULE.bazel": "7cd0312e064fde87c8d1cd79ba06c876bd23630c83466e9500321be55c96ace2", - "https://bcr.bazel.build/modules/abseil-cpp/20211102.0/MODULE.bazel": "70390338f7a5106231d20620712f7cccb659cd0e9d073d1991c038eb9fc57589", - "https://bcr.bazel.build/modules/abseil-cpp/20230125.1/MODULE.bazel": "89047429cb0207707b2dface14ba7f8df85273d484c2572755be4bab7ce9c3a0", - "https://bcr.bazel.build/modules/abseil-cpp/20230802.0/MODULE.bazel": "d253ae36a8bd9ee3c5955384096ccb6baf16a1b1e93e858370da0a3b94f77c16", - "https://bcr.bazel.build/modules/abseil-cpp/20240116.0/MODULE.bazel": "98dc378d64c12a4e4741ad3362f87fb737ee6a0886b2d90c3cdbb4d93ea3e0bf", - "https://bcr.bazel.build/modules/abseil-cpp/20240116.0/source.json": "251927429731bd37831facad38ddfc83b8f6f680f819e1d4d35e811b30479d5b", - "https://bcr.bazel.build/modules/apple_support/1.11.1/MODULE.bazel": "1843d7cd8a58369a444fc6000e7304425fba600ff641592161d9f15b179fb896", - "https://bcr.bazel.build/modules/apple_support/1.11.1/source.json": "c797537544f4b76d46d12f488456529d7372b6ccff249c82fca50ce0a665957f", - "https://bcr.bazel.build/modules/apple_support/1.5.0/MODULE.bazel": "50341a62efbc483e8a2a6aec30994a58749bd7b885e18dd96aa8c33031e558ef", - "https://bcr.bazel.build/modules/aspect_bazel_lib/2.7.2/MODULE.bazel": "780d1a6522b28f5edb7ea09630748720721dfe27690d65a2d33aa7509de77e07", - "https://bcr.bazel.build/modules/aspect_bazel_lib/2.8.1/MODULE.bazel": "812d2dd42f65dca362152101fbec418029cc8fd34cbad1a2fde905383d705838", - "https://bcr.bazel.build/modules/bazel_features/1.1.1/MODULE.bazel": "27b8c79ef57efe08efccbd9dd6ef70d61b4798320b8d3c134fd571f78963dbcd", - "https://bcr.bazel.build/modules/bazel_features/1.10.0/MODULE.bazel": "f75e8807570484a99be90abcd52b5e1f390362c258bcb73106f4544957a48101", - "https://bcr.bazel.build/modules/bazel_features/1.11.0/MODULE.bazel": "f9382337dd5a474c3b7d334c2f83e50b6eaedc284253334cf823044a26de03e8", - "https://bcr.bazel.build/modules/bazel_features/1.11.0/source.json": "c9320aa53cd1c441d24bd6b716da087ad7e4ff0d9742a9884587596edfe53015", - "https://bcr.bazel.build/modules/bazel_features/1.3.0/MODULE.bazel": "cdcafe83ec318cda34e02948e81d790aab8df7a929cec6f6969f13a489ccecd9", - "https://bcr.bazel.build/modules/bazel_features/1.4.1/MODULE.bazel": "e45b6bb2350aff3e442ae1111c555e27eac1d915e77775f6fdc4b351b758b5d7", - "https://bcr.bazel.build/modules/bazel_skylib/1.0.3/MODULE.bazel": "bcb0fd896384802d1ad283b4e4eb4d718eebd8cb820b0a2c3a347fb971afd9d8", - "https://bcr.bazel.build/modules/bazel_skylib/1.1.1/MODULE.bazel": "1add3e7d93ff2e6998f9e118022c84d163917d912f5afafb3058e3d2f1545b5e", - "https://bcr.bazel.build/modules/bazel_skylib/1.2.0/MODULE.bazel": "44fe84260e454ed94ad326352a698422dbe372b21a1ac9f3eab76eb531223686", - "https://bcr.bazel.build/modules/bazel_skylib/1.2.1/MODULE.bazel": "f35baf9da0efe45fa3da1696ae906eea3d615ad41e2e3def4aeb4e8bc0ef9a7a", - "https://bcr.bazel.build/modules/bazel_skylib/1.3.0/MODULE.bazel": "20228b92868bf5cfc41bda7afc8a8ba2a543201851de39d990ec957b513579c5", - "https://bcr.bazel.build/modules/bazel_skylib/1.4.1/MODULE.bazel": "a0dcb779424be33100dcae821e9e27e4f2901d9dfd5333efe5ac6a8d7ab75e1d", - "https://bcr.bazel.build/modules/bazel_skylib/1.4.2/MODULE.bazel": "3bd40978e7a1fac911d5989e6b09d8f64921865a45822d8b09e815eaa726a651", - "https://bcr.bazel.build/modules/bazel_skylib/1.5.0/MODULE.bazel": "32880f5e2945ce6a03d1fbd588e9198c0a959bb42297b2cfaf1685b7bc32e138", - "https://bcr.bazel.build/modules/bazel_skylib/1.6.1/MODULE.bazel": "8fdee2dbaace6c252131c00e1de4b165dc65af02ea278476187765e1a617b917", - "https://bcr.bazel.build/modules/bazel_skylib/1.7.1/MODULE.bazel": "3120d80c5861aa616222ec015332e5f8d3171e062e3e804a2a0253e1be26e59b", - "https://bcr.bazel.build/modules/bazel_skylib/1.7.1/source.json": "f121b43eeefc7c29efbd51b83d08631e2347297c95aac9764a701f2a6a2bb953", - "https://bcr.bazel.build/modules/buildozer/7.1.2/MODULE.bazel": "2e8dd40ede9c454042645fd8d8d0cd1527966aa5c919de86661e62953cd73d84", - "https://bcr.bazel.build/modules/buildozer/7.1.2/source.json": "c9028a501d2db85793a6996205c8de120944f50a0d570438fcae0457a5f9d1f8", - "https://bcr.bazel.build/modules/google_benchmark/1.8.2/MODULE.bazel": "a70cf1bba851000ba93b58ae2f6d76490a9feb74192e57ab8e8ff13c34ec50cb", - "https://bcr.bazel.build/modules/googletest/1.11.0/MODULE.bazel": "3a83f095183f66345ca86aa13c58b59f9f94a2f81999c093d4eeaa2d262d12f4", - "https://bcr.bazel.build/modules/googletest/1.14.0.bcr.1/MODULE.bazel": "22c31a561553727960057361aa33bf20fb2e98584bc4fec007906e27053f80c6", - "https://bcr.bazel.build/modules/googletest/1.14.0.bcr.1/source.json": "41e9e129f80d8c8bf103a7acc337b76e54fad1214ac0a7084bf24f4cd924b8b4", - "https://bcr.bazel.build/modules/googletest/1.14.0/MODULE.bazel": "cfbcbf3e6eac06ef9d85900f64424708cc08687d1b527f0ef65aa7517af8118f", - "https://bcr.bazel.build/modules/hermetic_cc_toolchain/3.1.0/MODULE.bazel": "ea4b3a25a9417a7db57a8a2f9ebdee91d679823c6274b482b817ed128d81c594", - "https://bcr.bazel.build/modules/hermetic_cc_toolchain/3.1.0/source.json": "9d1df0459caefdf41052d360469922a73e219f67c8ce4da0628cc604469822b9", - "https://bcr.bazel.build/modules/libpfm/4.11.0/MODULE.bazel": "45061ff025b301940f1e30d2c16bea596c25b176c8b6b3087e92615adbd52902", - "https://bcr.bazel.build/modules/nlohmann_json/3.6.1/MODULE.bazel": "6f7b417dcc794d9add9e556673ad25cb3ba835224290f4f848f8e2db1e1fca74", - "https://bcr.bazel.build/modules/nlohmann_json/3.6.1/source.json": "f448c6e8963fdfa7eb831457df83ad63d3d6355018f6574fb017e8169deb43a9", - "https://bcr.bazel.build/modules/patchelf/0.18.0/MODULE.bazel": "15a6beff7e828d585c5bd0f9f93589df117b5594e9d19e43096c77de58b9ae5f", - "https://bcr.bazel.build/modules/patchelf/0.18.0/source.json": "57caf6bcaa5ba515c6fb1c2eacee00735afbeb1ffacb34a57553fb139c8e4333", - "https://bcr.bazel.build/modules/platforms/0.0.10/MODULE.bazel": "8cb8efaf200bdeb2150d93e162c40f388529a25852b332cec879373771e48ed5", - "https://bcr.bazel.build/modules/platforms/0.0.10/source.json": "f22828ff4cf021a6b577f1bf6341cb9dcd7965092a439f64fc1bb3b7a5ae4bd5", - "https://bcr.bazel.build/modules/platforms/0.0.4/MODULE.bazel": "9b328e31ee156f53f3c416a64f8491f7eb731742655a47c9eec4703a71644aee", - "https://bcr.bazel.build/modules/platforms/0.0.5/MODULE.bazel": "5733b54ea419d5eaf7997054bb55f6a1d0b5ff8aedf0176fef9eea44f3acda37", - "https://bcr.bazel.build/modules/platforms/0.0.6/MODULE.bazel": "ad6eeef431dc52aefd2d77ed20a4b353f8ebf0f4ecdd26a807d2da5aa8cd0615", - "https://bcr.bazel.build/modules/platforms/0.0.7/MODULE.bazel": "72fd4a0ede9ee5c021f6a8dd92b503e089f46c227ba2813ff183b71616034814", - "https://bcr.bazel.build/modules/platforms/0.0.8/MODULE.bazel": "9f142c03e348f6d263719f5074b21ef3adf0b139ee4c5133e2aa35664da9eb2d", - "https://bcr.bazel.build/modules/platforms/0.0.9/MODULE.bazel": "4a87a60c927b56ddd67db50c89acaa62f4ce2a1d2149ccb63ffd871d5ce29ebc", - "https://bcr.bazel.build/modules/protobuf/21.7/MODULE.bazel": "a5a29bb89544f9b97edce05642fac225a808b5b7be74038ea3640fae2f8e66a7", - "https://bcr.bazel.build/modules/protobuf/21.7/source.json": "bbe500720421e582ff2d18b0802464205138c06056f443184de39fbb8187b09b", - "https://bcr.bazel.build/modules/protobuf/3.19.0/MODULE.bazel": "6b5fbb433f760a99a22b18b6850ed5784ef0e9928a72668b66e4d7ccd47db9b0", - "https://bcr.bazel.build/modules/protobuf/3.19.6/MODULE.bazel": "9233edc5e1f2ee276a60de3eaa47ac4132302ef9643238f23128fea53ea12858", - "https://bcr.bazel.build/modules/pybind11_bazel/2.11.1.bzl.1/MODULE.bazel": "1ef2994c097ee88f8f7ae8fbf991aaefb0603b2540fe575eca14943bc9f220a6", - "https://bcr.bazel.build/modules/pybind11_bazel/2.11.1.bzl.1/source.json": "d8f5b2ebc186b90c6112f92f43d86359dda2c3649c754d9680fd668f8dd28a79", - "https://bcr.bazel.build/modules/pybind11_bazel/2.11.1/MODULE.bazel": "88af1c246226d87e65be78ed49ecd1e6f5e98648558c14ce99176da041dc378e", - "https://bcr.bazel.build/modules/re2/2023-09-01/MODULE.bazel": "cb3d511531b16cfc78a225a9e2136007a48cf8a677e4264baeab57fe78a80206", - "https://bcr.bazel.build/modules/re2/2024-02-01/MODULE.bazel": "5ed922cb8b6c110e30969695e73bd0d3159576bf17ce8ee2443a7d07bf500551", - "https://bcr.bazel.build/modules/re2/2024-02-01/source.json": "0b108d5e9836cb66815fbbb70e4395eab52dc29dd2c39a9785f01d6661a8f37e", - "https://bcr.bazel.build/modules/rules_apple/3.2.1/MODULE.bazel": "55f19b572fdb4a4bd5a1c2231b60c663f8b4e3146769062b2bdeba72226c74b7", - "https://bcr.bazel.build/modules/rules_apple/3.2.1/source.json": "b7e8aa85c5fcd9dfec8ea62642b7e93ca161a03b2c403ae05872813f5dfeb7be", - "https://bcr.bazel.build/modules/rules_cc/0.0.1/MODULE.bazel": "cb2aa0747f84c6c3a78dad4e2049c154f08ab9d166b1273835a8174940365647", - "https://bcr.bazel.build/modules/rules_cc/0.0.2/MODULE.bazel": "6915987c90970493ab97393024c156ea8fb9f3bea953b2f3ec05c34f19b5695c", - "https://bcr.bazel.build/modules/rules_cc/0.0.6/MODULE.bazel": "abf360251023dfe3efcef65ab9d56beefa8394d4176dd29529750e1c57eaa33f", - "https://bcr.bazel.build/modules/rules_cc/0.0.8/MODULE.bazel": "964c85c82cfeb6f3855e6a07054fdb159aced38e99a5eecf7bce9d53990afa3e", - "https://bcr.bazel.build/modules/rules_cc/0.0.9/MODULE.bazel": "836e76439f354b89afe6a911a7adf59a6b2518fafb174483ad78a2a2fde7b1c5", - "https://bcr.bazel.build/modules/rules_cc/0.0.9/source.json": "1f1ba6fea244b616de4a554a0f4983c91a9301640c8fe0dd1d410254115c8430", - "https://bcr.bazel.build/modules/rules_foreign_cc/0.9.0/MODULE.bazel": "c9e8c682bf75b0e7c704166d79b599f93b72cfca5ad7477df596947891feeef6", - "https://bcr.bazel.build/modules/rules_java/4.0.0/MODULE.bazel": "5a78a7ae82cd1a33cef56dc578c7d2a46ed0dca12643ee45edbb8417899e6f74", - "https://bcr.bazel.build/modules/rules_java/5.3.5/MODULE.bazel": "a4ec4f2db570171e3e5eb753276ee4b389bae16b96207e9d3230895c99644b86", - "https://bcr.bazel.build/modules/rules_java/6.3.0/MODULE.bazel": "a97c7678c19f236a956ad260d59c86e10a463badb7eb2eda787490f4c969b963", - "https://bcr.bazel.build/modules/rules_java/7.3.2/MODULE.bazel": "50dece891cfdf1741ea230d001aa9c14398062f2b7c066470accace78e412bc2", - "https://bcr.bazel.build/modules/rules_java/7.6.5/MODULE.bazel": "481164be5e02e4cab6e77a36927683263be56b7e36fef918b458d7a8a1ebadb1", - "https://bcr.bazel.build/modules/rules_java/7.6.5/source.json": "a805b889531d1690e3c72a7a7e47a870d00323186a9904b36af83aa3d053ee8d", - "https://bcr.bazel.build/modules/rules_jvm_external/4.4.2/MODULE.bazel": "a56b85e418c83eb1839819f0b515c431010160383306d13ec21959ac412d2fe7", - "https://bcr.bazel.build/modules/rules_jvm_external/5.2/MODULE.bazel": "d9351ba35217ad0de03816ef3ed63f89d411349353077348a45348b096615036", - "https://bcr.bazel.build/modules/rules_jvm_external/5.2/source.json": "10572111995bc349ce31c78f74b3c147f6b3233975c7fa5eff9211f6db0d34d9", - "https://bcr.bazel.build/modules/rules_license/0.0.3/MODULE.bazel": "627e9ab0247f7d1e05736b59dbb1b6871373de5ad31c3011880b4133cafd4bd0", - "https://bcr.bazel.build/modules/rules_license/0.0.4/MODULE.bazel": "6a88dd22800cf1f9f79ba32cacad0d3a423ed28efa2c2ed5582eaa78dd3ac1e5", - "https://bcr.bazel.build/modules/rules_license/0.0.7/MODULE.bazel": "088fbeb0b6a419005b89cf93fe62d9517c0a2b8bb56af3244af65ecfe37e7d5d", - "https://bcr.bazel.build/modules/rules_license/0.0.8/MODULE.bazel": "5669c6fe49b5134dbf534db681ad3d67a2d49cfc197e4a95f1ca2fd7f3aebe96", - "https://bcr.bazel.build/modules/rules_license/0.0.8/source.json": "ccfd3964cd0cd1739202efb8dbf9a06baab490e61e174b2ad4790f9c4e610beb", - "https://bcr.bazel.build/modules/rules_oci/2.0.0/MODULE.bazel": "5d5cf1932238b009f874d5a9f214bbedf5d8cec8e5028e083f1147726876572f", - "https://bcr.bazel.build/modules/rules_oci/2.0.0/source.json": "ea89dd54d1f473a6fb1a6c2d0660b741ee832d5141ac0b9c90d5c8fc9b5e3bb3", - "https://bcr.bazel.build/modules/rules_pkg/0.7.0/MODULE.bazel": "df99f03fc7934a4737122518bb87e667e62d780b610910f0447665a7e2be62dc", - "https://bcr.bazel.build/modules/rules_pkg/0.9.1/MODULE.bazel": "af00144208c4be503bc920d043ba3284fb37385b3f6160b4a4daf4df80b4b823", - "https://bcr.bazel.build/modules/rules_pkg/1.0.1/MODULE.bazel": "5b1df97dbc29623bccdf2b0dcd0f5cb08e2f2c9050aab1092fd39a41e82686ff", - "https://bcr.bazel.build/modules/rules_pkg/1.0.1/source.json": "bd82e5d7b9ce2d31e380dd9f50c111d678c3bdaca190cb76b0e1c71b05e1ba8a", - "https://bcr.bazel.build/modules/rules_proto/4.0.0/MODULE.bazel": "a7a7b6ce9bee418c1a760b3d84f83a299ad6952f9903c67f19e4edd964894e06", - "https://bcr.bazel.build/modules/rules_proto/5.3.0-21.7/MODULE.bazel": "e8dff86b0971688790ae75528fe1813f71809b5afd57facb44dad9e8eca631b7", - "https://bcr.bazel.build/modules/rules_proto/6.0.0-rc1/MODULE.bazel": "1e5b502e2e1a9e825eef74476a5a1ee524a92297085015a052510b09a1a09483", - "https://bcr.bazel.build/modules/rules_proto/6.0.2/MODULE.bazel": "ce916b775a62b90b61888052a416ccdda405212b6aaeb39522f7dc53431a5e73", - "https://bcr.bazel.build/modules/rules_proto/6.0.2/source.json": "17a2e195f56cb28d6bbf763e49973d13890487c6945311ed141e196fb660426d", - "https://bcr.bazel.build/modules/rules_python/0.10.2/MODULE.bazel": "cc82bc96f2997baa545ab3ce73f196d040ffb8756fd2d66125a530031cd90e5f", - "https://bcr.bazel.build/modules/rules_python/0.22.1/MODULE.bazel": "26114f0c0b5e93018c0c066d6673f1a2c3737c7e90af95eff30cfee38d0bbac7", - "https://bcr.bazel.build/modules/rules_python/0.25.0/MODULE.bazel": "72f1506841c920a1afec76975b35312410eea3aa7b63267436bfb1dd91d2d382", - "https://bcr.bazel.build/modules/rules_python/0.29.0/MODULE.bazel": "2ac8cd70524b4b9ec49a0b8284c79e4cd86199296f82f6e0d5da3f783d660c82", - "https://bcr.bazel.build/modules/rules_python/0.31.0/MODULE.bazel": "93a43dc47ee570e6ec9f5779b2e64c1476a6ce921c48cc9a1678a91dd5f8fd58", - "https://bcr.bazel.build/modules/rules_python/0.31.0/source.json": "a41c836d4065888eef4377f2f27b6eea0fedb9b5adb1bab1970437373fe90dc7", - "https://bcr.bazel.build/modules/rules_python/0.4.0/MODULE.bazel": "9208ee05fd48bf09ac60ed269791cf17fb343db56c8226a720fbb1cdf467166c", - "https://bcr.bazel.build/modules/rules_swift/1.15.1/MODULE.bazel": "7031d3e5221b54dfee3901d221f10f54a8b0f2a64d4616de7879bcf83fe8d85d", - "https://bcr.bazel.build/modules/rules_swift/1.15.1/source.json": "6472a4607d0a2f39eeb7daae78054b995aba593caad99aa64561f631fcf93a63", - "https://bcr.bazel.build/modules/stardoc/0.5.1/MODULE.bazel": "1a05d92974d0c122f5ccf09291442580317cdd859f07a8655f1db9a60374f9f8", - "https://bcr.bazel.build/modules/stardoc/0.5.3/MODULE.bazel": "c7f6948dae6999bf0db32c1858ae345f112cacf98f174c7a8bb707e41b974f1c", - "https://bcr.bazel.build/modules/stardoc/0.5.4/MODULE.bazel": "6569966df04610b8520957cb8e97cf2e9faac2c0309657c537ab51c16c18a2a4", - "https://bcr.bazel.build/modules/stardoc/0.6.2/MODULE.bazel": "7060193196395f5dd668eda046ccbeacebfd98efc77fed418dbe2b82ffaa39fd", - "https://bcr.bazel.build/modules/stardoc/0.6.2/source.json": "d2ff8063b63b4a85e65fe595c4290f99717434fa9f95b4748a79a7d04dfed349", - "https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "7298990c00040a0e2f121f6c32544bab27d4452f80d9ce51349b1a28f3005c43", - "https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/source.json": "f1ef7d3f9e0e26d4b23d1c39b5f5de71f584dd7d1b4ef83d9bbba6ec7a6a6459", - "https://bcr.bazel.build/modules/zlib/1.2.11/MODULE.bazel": "07b389abc85fdbca459b69e2ec656ae5622873af3f845e1c9d80fe179f3effa0", - "https://bcr.bazel.build/modules/zlib/1.2.12/MODULE.bazel": "3b1a8834ada2a883674be8cbd36ede1b6ec481477ada359cd2d3ddc562340b27", - "https://bcr.bazel.build/modules/zlib/1.2.13/MODULE.bazel": "aa6deb1b83c18ffecd940c4119aff9567cd0a671d7bba756741cb2ef043a29d5", - "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.3/MODULE.bazel": "af322bc08976524477c79d1e45e241b6efbeb918c497e8840b8ab116802dda79", - "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.3/source.json": "2be409ac3c7601245958cd4fcdff4288be79ed23bd690b4b951f500d54ee6e7d", - "https://bcr.bazel.build/modules/zstd/1.5.6/MODULE.bazel": "471ebe7d3cdd8c6469390fcf623eb4779ff55fbee0a87f1dc57a1def468b96d4", - "https://bcr.bazel.build/modules/zstd/1.5.6/source.json": "02010c3333fc89b44fe861db049968decb6e688411f7f9d4f6791d74f9adfb51" - }, - "selectedYankedVersions": {}, - "moduleExtensions": { - "@@apple_support~//crosstool:setup.bzl%apple_cc_configure_extension": { - "general": { - "bzlTransitiveDigest": "hDHJiBbKme6a+N8oiSQcVnU1v5B7tHMjJaAzS6GFfPc=", - "usagesDigest": "A++qEP50bGWXQU02cK/pyNUUi4X4mKkI6z3MWeGO4VM=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "local_config_apple_cc": { - "bzlFile": "@@apple_support~//crosstool:setup.bzl", - "ruleClassName": "_apple_cc_autoconf", - "attributes": {} - }, - "local_config_apple_cc_toolchains": { - "bzlFile": "@@apple_support~//crosstool:setup.bzl", - "ruleClassName": "_apple_cc_autoconf_toolchains", - "attributes": {} - } - }, - "recordedRepoMappingEntries": [ - [ - "apple_support~", - "bazel_tools", - "bazel_tools" - ] - ] - } - }, - "@@aspect_bazel_lib~//lib:extensions.bzl%toolchains": { - "general": { - "bzlTransitiveDigest": "p8GR5EeSAP5RvoQ0821nQQ9nelAyTrNoUULFFBUdGlU=", - "usagesDigest": "53S1BIVDbvpXqyBqIHwVdm4b0MSXSxM+QbA3pnTYns8=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "expand_template_windows_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:expand_template_toolchain.bzl", - "ruleClassName": "expand_template_platform_repo", - "attributes": { - "platform": "windows_amd64" - } - }, - "copy_to_directory_windows_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_to_directory_toolchain.bzl", - "ruleClassName": "copy_to_directory_platform_repo", - "attributes": { - "platform": "windows_amd64" - } - }, - "jq_darwin_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:jq_toolchain.bzl", - "ruleClassName": "jq_platform_repo", - "attributes": { - "platform": "darwin_amd64", - "version": "1.7" - } - }, - "copy_to_directory_freebsd_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_to_directory_toolchain.bzl", - "ruleClassName": "copy_to_directory_platform_repo", - "attributes": { - "platform": "freebsd_amd64" - } - }, - "expand_template_linux_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:expand_template_toolchain.bzl", - "ruleClassName": "expand_template_platform_repo", - "attributes": { - "platform": "linux_amd64" - } - }, - "jq_linux_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:jq_toolchain.bzl", - "ruleClassName": "jq_platform_repo", - "attributes": { - "platform": "linux_arm64", - "version": "1.7" - } - }, - "coreutils_darwin_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:coreutils_toolchain.bzl", - "ruleClassName": "coreutils_platform_repo", - "attributes": { - "platform": "darwin_arm64", - "version": "0.0.26" - } - }, - "copy_to_directory_linux_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_to_directory_toolchain.bzl", - "ruleClassName": "copy_to_directory_platform_repo", - "attributes": { - "platform": "linux_arm64" - } - }, - "bsd_tar_linux_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:tar_toolchain.bzl", - "ruleClassName": "bsdtar_binary_repo", - "attributes": { - "platform": "linux_arm64" - } - }, - "copy_directory_darwin_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_directory_toolchain.bzl", - "ruleClassName": "copy_directory_platform_repo", - "attributes": { - "platform": "darwin_amd64" - } - }, - "coreutils_darwin_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:coreutils_toolchain.bzl", - "ruleClassName": "coreutils_platform_repo", - "attributes": { - "platform": "darwin_amd64", - "version": "0.0.26" - } - }, - "coreutils_linux_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:coreutils_toolchain.bzl", - "ruleClassName": "coreutils_platform_repo", - "attributes": { - "platform": "linux_arm64", - "version": "0.0.26" - } - }, - "zstd_linux_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:zstd_toolchain.bzl", - "ruleClassName": "zstd_binary_repo", - "attributes": { - "platform": "linux_arm64" - } - }, - "yq_linux_s390x": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:yq_toolchain.bzl", - "ruleClassName": "yq_platform_repo", - "attributes": { - "platform": "linux_s390x", - "version": "4.25.2" - } - }, - "yq": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:yq_toolchain.bzl", - "ruleClassName": "yq_host_alias_repo", - "attributes": {} - }, - "expand_template_darwin_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:expand_template_toolchain.bzl", - "ruleClassName": "expand_template_platform_repo", - "attributes": { - "platform": "darwin_amd64" - } - }, - "copy_directory_linux_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_directory_toolchain.bzl", - "ruleClassName": "copy_directory_platform_repo", - "attributes": { - "platform": "linux_amd64" - } - }, - "jq_darwin_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:jq_toolchain.bzl", - "ruleClassName": "jq_platform_repo", - "attributes": { - "platform": "darwin_arm64", - "version": "1.7" - } - }, - "yq_darwin_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:yq_toolchain.bzl", - "ruleClassName": "yq_platform_repo", - "attributes": { - "platform": "darwin_amd64", - "version": "4.25.2" - } - }, - "copy_directory_linux_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_directory_toolchain.bzl", - "ruleClassName": "copy_directory_platform_repo", - "attributes": { - "platform": "linux_arm64" - } - }, - "expand_template_toolchains": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:expand_template_toolchain.bzl", - "ruleClassName": "expand_template_toolchains_repo", - "attributes": { - "user_repository_name": "expand_template" - } - }, - "bats_assert": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "sha256": "98ca3b685f8b8993e48ec057565e6e2abcc541034ed5b0e81f191505682037fd", - "urls": [ - "https://github.com/bats-core/bats-assert/archive/v2.1.0.tar.gz" - ], - "strip_prefix": "bats-assert-2.1.0", - "build_file_content": "load(\"@aspect_bazel_lib//lib:copy_to_directory.bzl\", \"copy_to_directory\")\n\ncopy_to_directory(\n name = \"assert\",\n hardlink = \"on\",\n srcs = glob([\n \"src/**\",\n \"load.bash\",\n ]),\n out = \"bats-assert\",\n visibility = [\"//visibility:public\"]\n)\n" - } - }, - "copy_to_directory_darwin_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_to_directory_toolchain.bzl", - "ruleClassName": "copy_to_directory_platform_repo", - "attributes": { - "platform": "darwin_amd64" - } - }, - "zstd_darwin_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:zstd_toolchain.bzl", - "ruleClassName": "zstd_binary_repo", - "attributes": { - "platform": "darwin_arm64" - } - }, - "bsd_tar_linux_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:tar_toolchain.bzl", - "ruleClassName": "bsdtar_binary_repo", - "attributes": { - "platform": "linux_amd64" - } - }, - "yq_toolchains": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:yq_toolchain.bzl", - "ruleClassName": "yq_toolchains_repo", - "attributes": { - "user_repository_name": "yq" - } - }, - "zstd_linux_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:zstd_toolchain.bzl", - "ruleClassName": "zstd_binary_repo", - "attributes": { - "platform": "linux_amd64" - } - }, - "bats_support": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "sha256": "7815237aafeb42ddcc1b8c698fc5808026d33317d8701d5ec2396e9634e2918f", - "urls": [ - "https://github.com/bats-core/bats-support/archive/v0.3.0.tar.gz" - ], - "strip_prefix": "bats-support-0.3.0", - "build_file_content": "load(\"@aspect_bazel_lib//lib:copy_to_directory.bzl\", \"copy_to_directory\")\n\ncopy_to_directory(\n name = \"support\",\n hardlink = \"on\",\n srcs = glob([\n \"src/**\",\n \"load.bash\",\n ]),\n out = \"bats-support\",\n visibility = [\"//visibility:public\"]\n)\n" - } - }, - "bsd_tar_windows_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:tar_toolchain.bzl", - "ruleClassName": "bsdtar_binary_repo", - "attributes": { - "platform": "windows_amd64" - } - }, - "jq": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:jq_toolchain.bzl", - "ruleClassName": "jq_host_alias_repo", - "attributes": {} - }, - "expand_template_darwin_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:expand_template_toolchain.bzl", - "ruleClassName": "expand_template_platform_repo", - "attributes": { - "platform": "darwin_arm64" - } - }, - "bsd_tar_darwin_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:tar_toolchain.bzl", - "ruleClassName": "bsdtar_binary_repo", - "attributes": { - "platform": "darwin_arm64" - } - }, - "copy_to_directory_linux_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_to_directory_toolchain.bzl", - "ruleClassName": "copy_to_directory_platform_repo", - "attributes": { - "platform": "linux_amd64" - } - }, - "coreutils_linux_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:coreutils_toolchain.bzl", - "ruleClassName": "coreutils_platform_repo", - "attributes": { - "platform": "linux_amd64", - "version": "0.0.26" - } - }, - "copy_directory_toolchains": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_directory_toolchain.bzl", - "ruleClassName": "copy_directory_toolchains_repo", - "attributes": { - "user_repository_name": "copy_directory" - } - }, - "yq_linux_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:yq_toolchain.bzl", - "ruleClassName": "yq_platform_repo", - "attributes": { - "platform": "linux_amd64", - "version": "4.25.2" - } - }, - "copy_to_directory_darwin_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_to_directory_toolchain.bzl", - "ruleClassName": "copy_to_directory_platform_repo", - "attributes": { - "platform": "darwin_arm64" - } - }, - "coreutils_toolchains": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:coreutils_toolchain.bzl", - "ruleClassName": "coreutils_toolchains_repo", - "attributes": { - "user_repository_name": "coreutils" - } - }, - "copy_directory_freebsd_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_directory_toolchain.bzl", - "ruleClassName": "copy_directory_platform_repo", - "attributes": { - "platform": "freebsd_amd64" - } - }, - "zstd_darwin_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:zstd_toolchain.bzl", - "ruleClassName": "zstd_binary_repo", - "attributes": { - "platform": "darwin_amd64" - } - }, - "zstd_toolchains": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:zstd_toolchain.bzl", - "ruleClassName": "zstd_toolchains_repo", - "attributes": { - "user_repository_name": "zstd" - } - }, - "bats_file": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "sha256": "9b69043241f3af1c2d251f89b4fcafa5df3f05e97b89db18d7c9bdf5731bb27a", - "urls": [ - "https://github.com/bats-core/bats-file/archive/v0.4.0.tar.gz" - ], - "strip_prefix": "bats-file-0.4.0", - "build_file_content": "load(\"@aspect_bazel_lib//lib:copy_to_directory.bzl\", \"copy_to_directory\")\n\ncopy_to_directory(\n name = \"file\",\n hardlink = \"on\",\n srcs = glob([\n \"src/**\",\n \"load.bash\",\n ]),\n out = \"bats-file\",\n visibility = [\"//visibility:public\"]\n)\n" - } - }, - "expand_template_linux_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:expand_template_toolchain.bzl", - "ruleClassName": "expand_template_platform_repo", - "attributes": { - "platform": "linux_arm64" - } - }, - "jq_linux_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:jq_toolchain.bzl", - "ruleClassName": "jq_platform_repo", - "attributes": { - "platform": "linux_amd64", - "version": "1.7" - } - }, - "bsd_tar_darwin_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:tar_toolchain.bzl", - "ruleClassName": "bsdtar_binary_repo", - "attributes": { - "platform": "darwin_amd64" - } - }, - "bsd_tar_toolchains": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:tar_toolchain.bzl", - "ruleClassName": "tar_toolchains_repo", - "attributes": { - "user_repository_name": "bsd_tar" - } - }, - "bats_toolchains": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "sha256": "a1a9f7875aa4b6a9480ca384d5865f1ccf1b0b1faead6b47aa47d79709a5c5fd", - "urls": [ - "https://github.com/bats-core/bats-core/archive/v1.10.0.tar.gz" - ], - "strip_prefix": "bats-core-1.10.0", - "build_file_content": "load(\"@local_config_platform//:constraints.bzl\", \"HOST_CONSTRAINTS\")\nload(\"@aspect_bazel_lib//lib/private:bats_toolchain.bzl\", \"bats_toolchain\")\nload(\"@aspect_bazel_lib//lib:copy_to_directory.bzl\", \"copy_to_directory\")\n\ncopy_to_directory(\n name = \"core\",\n hardlink = \"on\",\n srcs = glob([\n \"lib/**\",\n \"libexec/**\"\n ]) + [\"bin/bats\"],\n out = \"bats-core\",\n)\n\nbats_toolchain(\n name = \"toolchain\",\n core = \":core\",\n libraries = [\"@bats_support//:support\", \"@bats_assert//:assert\", \"@bats_file//:file\"]\n)\n\ntoolchain(\n name = \"bats_toolchain\",\n exec_compatible_with = HOST_CONSTRAINTS,\n toolchain = \":toolchain\",\n toolchain_type = \"@aspect_bazel_lib//lib:bats_toolchain_type\",\n)\n" - } - }, - "yq_windows_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:yq_toolchain.bzl", - "ruleClassName": "yq_platform_repo", - "attributes": { - "platform": "windows_amd64", - "version": "4.25.2" - } - }, - "jq_windows_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:jq_toolchain.bzl", - "ruleClassName": "jq_platform_repo", - "attributes": { - "platform": "windows_amd64", - "version": "1.7" - } - }, - "expand_template_freebsd_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:expand_template_toolchain.bzl", - "ruleClassName": "expand_template_platform_repo", - "attributes": { - "platform": "freebsd_amd64" - } - }, - "yq_linux_ppc64le": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:yq_toolchain.bzl", - "ruleClassName": "yq_platform_repo", - "attributes": { - "platform": "linux_ppc64le", - "version": "4.25.2" - } - }, - "copy_to_directory_toolchains": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_to_directory_toolchain.bzl", - "ruleClassName": "copy_to_directory_toolchains_repo", - "attributes": { - "user_repository_name": "copy_to_directory" - } - }, - "jq_toolchains": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:jq_toolchain.bzl", - "ruleClassName": "jq_toolchains_repo", - "attributes": { - "user_repository_name": "jq" - } - }, - "copy_directory_darwin_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_directory_toolchain.bzl", - "ruleClassName": "copy_directory_platform_repo", - "attributes": { - "platform": "darwin_arm64" - } - }, - "copy_directory_windows_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_directory_toolchain.bzl", - "ruleClassName": "copy_directory_platform_repo", - "attributes": { - "platform": "windows_amd64" - } - }, - "yq_darwin_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:yq_toolchain.bzl", - "ruleClassName": "yq_platform_repo", - "attributes": { - "platform": "darwin_arm64", - "version": "4.25.2" - } - }, - "coreutils_windows_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:coreutils_toolchain.bzl", - "ruleClassName": "coreutils_platform_repo", - "attributes": { - "platform": "windows_amd64", - "version": "0.0.26" - } - }, - "yq_linux_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:yq_toolchain.bzl", - "ruleClassName": "yq_platform_repo", - "attributes": { - "platform": "linux_arm64", - "version": "4.25.2" - } - } - }, - "recordedRepoMappingEntries": [ - [ - "aspect_bazel_lib~", - "aspect_bazel_lib", - "aspect_bazel_lib~" - ], - [ - "aspect_bazel_lib~", - "bazel_skylib", - "bazel_skylib~" - ], - [ - "aspect_bazel_lib~", - "bazel_tools", - "bazel_tools" - ] - ] - } - }, - "@@hermetic_cc_toolchain~//toolchain:ext.bzl%toolchains": { - "general": { - "bzlTransitiveDigest": "YrUOAIuZsxXEx+Q8Bh4BooEu9qcrrOJRxaoC368wqAs=", - "usagesDigest": "gsOtNXDFGoYHNJh4CObQXvyzXRpiz00d4DzFTD62QQQ=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "zig_sdk": { - "bzlFile": "@@hermetic_cc_toolchain~//toolchain:defs.bzl", - "ruleClassName": "zig_repository", - "attributes": { - "version": "0.12.0", - "url_formats": [ - "https://mirror.bazel.build/ziglang.org/download/{version}/zig-{host_platform}-{version}.{_ext}", - "https://ziglang.org/download/{version}/zig-{host_platform}-{version}.{_ext}" - ], - "host_platform_sha256": { - "linux-aarch64": "754f1029484079b7e0ca3b913a0a2f2a6afd5a28990cb224fe8845e72f09de63", - "linux-x86_64": "c7ae866b8a76a568e2d5cfd31fe89cdb629bdd161fdd5018b29a4a0a17045cad", - "macos-aarch64": "294e224c14fd0822cfb15a35cf39aa14bd9967867999bf8bdfe3db7ddec2a27f", - "macos-x86_64": "4d411bf413e7667821324da248e8589278180dbc197f4f282b7dbb599a689311", - "windows-aarch64": "04c6b92689241ca7a8a59b5f12d2ca2820c09d5043c3c4808b7e93e41c7bf97b", - "windows-x86_64": "2199eb4c2000ddb1fba85ba78f1fcf9c1fb8b3e57658f6a627a8e513131893f5" - }, - "host_platform_ext": { - "linux-aarch64": "tar.xz", - "linux-x86_64": "tar.xz", - "macos-aarch64": "tar.xz", - "macos-x86_64": "tar.xz", - "windows-x86_64": "zip" - } - } - } - }, - "recordedRepoMappingEntries": [ - [ - "hermetic_cc_toolchain~", - "bazel_tools", - "bazel_tools" - ], - [ - "hermetic_cc_toolchain~", - "hermetic_cc_toolchain", - "hermetic_cc_toolchain~" - ] - ] - } - }, - "@@platforms//host:extension.bzl%host_platform": { - "general": { - "bzlTransitiveDigest": "WewbYICdNVp22bzUQafEVMzMIpBnNjJ3zqKlUOCDIGc=", - "usagesDigest": "V1R2Y2oMxKNfx2WCWpSCaUV1WefW1o8HZGm3v1vHgY4=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "host_platform": { - "bzlFile": "@@platforms//host:extension.bzl", - "ruleClassName": "host_platform_repo", - "attributes": {} - } - }, - "recordedRepoMappingEntries": [] - } - }, - "@@pybind11_bazel~//:python_configure.bzl%extension": { - "general": { - "bzlTransitiveDigest": "NEb6eiWbiR50fBHwAo2GRT9cn5ZeUwkl+u9SH8oW+QA=", - "usagesDigest": "zwO78x/IRE/HBnfEhB+5Qb89k2iqdXjrKmijtwGKKhU=", - "recordedFileInputs": { - "@@pybind11_bazel~//MODULE.bazel": "c77a018c0d59b0cb30ccb8c717cd1b68f24e99061bf3e6107b8edf9a51c289f9" - }, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "local_config_python": { - "bzlFile": "@@pybind11_bazel~//:python_configure.bzl", - "ruleClassName": "python_configure", - "attributes": {} - }, - "pybind11": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "build_file": "@@pybind11_bazel~//:pybind11.BUILD", - "strip_prefix": "pybind11-2.11.1", - "urls": [ - "https://github.com/pybind/pybind11/archive/v2.11.1.zip" - ] - } - } - }, - "recordedRepoMappingEntries": [ - [ - "pybind11_bazel~", - "bazel_tools", - "bazel_tools" - ] - ] - } - }, - "@@rules_apple~//apple:apple.bzl%provisioning_profile_repository_extension": { - "general": { - "bzlTransitiveDigest": "un6MvJaKPvWpNVTc815CMIyNlD+sEvl3KbRI6aT/TBE=", - "usagesDigest": "SsTSvXhS9/8B9kLnKgoSoy9BHXv1NcIMn6f36U5+XnM=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "local_provisioning_profiles": { - "bzlFile": "@@rules_apple~//apple/internal:local_provisioning_profiles.bzl", - "ruleClassName": "provisioning_profile_repository", - "attributes": {} - } - }, - "recordedRepoMappingEntries": [ - [ - "apple_support~", - "bazel_skylib", - "bazel_skylib~" - ], - [ - "bazel_features~", - "bazel_features_globals", - "bazel_features~~version_extension~bazel_features_globals" - ], - [ - "bazel_features~", - "bazel_features_version", - "bazel_features~~version_extension~bazel_features_version" - ], - [ - "rules_apple~", - "bazel_skylib", - "bazel_skylib~" - ], - [ - "rules_apple~", - "bazel_tools", - "bazel_tools" - ], - [ - "rules_apple~", - "build_bazel_apple_support", - "apple_support~" - ], - [ - "rules_apple~", - "build_bazel_rules_apple", - "rules_apple~" - ], - [ - "rules_apple~", - "build_bazel_rules_swift", - "rules_swift~" - ], - [ - "rules_proto~", - "bazel_features", - "bazel_features~" - ], - [ - "rules_swift~", - "bazel_skylib", - "bazel_skylib~" - ], - [ - "rules_swift~", - "bazel_tools", - "bazel_tools" - ], - [ - "rules_swift~", - "build_bazel_apple_support", - "apple_support~" - ], - [ - "rules_swift~", - "build_bazel_rules_swift", - "rules_swift~" - ], - [ - "rules_swift~", - "build_bazel_rules_swift_local_config", - "rules_swift~~non_module_deps~build_bazel_rules_swift_local_config" - ], - [ - "rules_swift~", - "com_github_apple_swift_protobuf", - "rules_swift~~non_module_deps~com_github_apple_swift_protobuf" - ], - [ - "rules_swift~", - "com_github_grpc_grpc_swift", - "rules_swift~~non_module_deps~com_github_grpc_grpc_swift" - ], - [ - "rules_swift~", - "rules_proto", - "rules_proto~" - ] - ] - } - }, - "@@rules_apple~//apple:extensions.bzl%non_module_deps": { - "general": { - "bzlTransitiveDigest": "9xAH1dOtuO3kWLuDyxIB9C4GRnrKxcUqWvr4pDgkvcw=", - "usagesDigest": "j4AxtqPSAvljAB996Wzf/CwQzg9iGP5epLeVmXfPdvY=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "xctestrunner": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "urls": [ - "https://github.com/google/xctestrunner/archive/b7698df3d435b6491b4b4c0f9fc7a63fbed5e3a6.tar.gz" - ], - "strip_prefix": "xctestrunner-b7698df3d435b6491b4b4c0f9fc7a63fbed5e3a6", - "sha256": "ae3a063c985a8633cb7eb566db21656f8db8eb9a0edb8c182312c7f0db53730d" - } - } - }, - "recordedRepoMappingEntries": [ - [ - "rules_apple~", - "bazel_tools", - "bazel_tools" - ] - ] - } - }, - "@@rules_jvm_external~//:extensions.bzl%maven": { - "general": { - "bzlTransitiveDigest": "XKliOxeWFV00BP283t3DmOxdKKMhSls2Tz8yLm8OesM=", - "usagesDigest": "oHb+Lj50+DYOm43UmeLXXrg+zk77nwDdEQsJiwXNisM=", - "recordedFileInputs": { - "@@stardoc~//maven_install.json": "554795024dda3ce71869a7d908f6a31ac7498ba55b720d1bdc3a69571a32ee05", - "@@rules_jvm_external~//rules_jvm_external_deps_install.json": "c735590fdd362bd83e8de34b8e1b4bd4e1f47d4e2b77cb0f1cd3d8a0f4f475bc" - }, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "org_slf4j_slf4j_api_1_7_30": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "cdba07964d1bb40a0761485c6b1e8c2f8fd9eb1d19c53928ac0d7f9510105c57", - "urls": [ - "https://repo1.maven.org/maven2/org/slf4j/slf4j-api/1.7.30/slf4j-api-1.7.30.jar", - "https://maven.google.com/org/slf4j/slf4j-api/1.7.30/slf4j-api-1.7.30.jar" - ], - "downloaded_file_path": "org/slf4j/slf4j-api/1.7.30/slf4j-api-1.7.30.jar" - } - }, - "com_google_api_grpc_proto_google_common_protos_2_0_1": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "5ce71656118618731e34a5d4c61aa3a031be23446dc7de8b5a5e77b66ebcd6ef", - "urls": [ - "https://repo1.maven.org/maven2/com/google/api/grpc/proto-google-common-protos/2.0.1/proto-google-common-protos-2.0.1.jar", - "https://maven.google.com/com/google/api/grpc/proto-google-common-protos/2.0.1/proto-google-common-protos-2.0.1.jar" - ], - "downloaded_file_path": "com/google/api/grpc/proto-google-common-protos/2.0.1/proto-google-common-protos-2.0.1.jar" - } - }, - "com_google_api_gax_1_60_0": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "02f37d4ff1a7b8d71dff8064cf9568aa4f4b61bcc4485085d16130f32afa5a79", - "urls": [ - "https://repo1.maven.org/maven2/com/google/api/gax/1.60.0/gax-1.60.0.jar", - "https://maven.google.com/com/google/api/gax/1.60.0/gax-1.60.0.jar" - ], - "downloaded_file_path": "com/google/api/gax/1.60.0/gax-1.60.0.jar" - } - }, - "com_google_guava_failureaccess_1_0_1": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "a171ee4c734dd2da837e4b16be9df4661afab72a41adaf31eb84dfdaf936ca26", - "urls": [ - "https://repo1.maven.org/maven2/com/google/guava/failureaccess/1.0.1/failureaccess-1.0.1.jar" - ], - "downloaded_file_path": "com/google/guava/failureaccess/1.0.1/failureaccess-1.0.1.jar" - } - }, - "commons_logging_commons_logging_1_2": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "daddea1ea0be0f56978ab3006b8ac92834afeefbd9b7e4e6316fca57df0fa636", - "urls": [ - "https://repo1.maven.org/maven2/commons-logging/commons-logging/1.2/commons-logging-1.2.jar", - "https://maven.google.com/commons-logging/commons-logging/1.2/commons-logging-1.2.jar" - ], - "downloaded_file_path": "commons-logging/commons-logging/1.2/commons-logging-1.2.jar" - } - }, - "com_google_http_client_google_http_client_appengine_1_38_0": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "f97b495fd97ac3a3d59099eb2b55025f4948230da15a076f189b9cff37c6b4d2", - "urls": [ - "https://repo1.maven.org/maven2/com/google/http-client/google-http-client-appengine/1.38.0/google-http-client-appengine-1.38.0.jar", - "https://maven.google.com/com/google/http-client/google-http-client-appengine/1.38.0/google-http-client-appengine-1.38.0.jar" - ], - "downloaded_file_path": "com/google/http-client/google-http-client-appengine/1.38.0/google-http-client-appengine-1.38.0.jar" - } - }, - "com_google_cloud_google_cloud_storage_1_113_4": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "796833e9bdab80c40bbc820e65087eb8f28c6bfbca194d2e3e00d98cb5bc55d6", - "urls": [ - "https://repo1.maven.org/maven2/com/google/cloud/google-cloud-storage/1.113.4/google-cloud-storage-1.113.4.jar", - "https://maven.google.com/com/google/cloud/google-cloud-storage/1.113.4/google-cloud-storage-1.113.4.jar" - ], - "downloaded_file_path": "com/google/cloud/google-cloud-storage/1.113.4/google-cloud-storage-1.113.4.jar" - } - }, - "unpinned_stardoc_maven": { - "bzlFile": "@@rules_jvm_external~//:coursier.bzl", - "ruleClassName": "coursier_fetch", - "attributes": { - "repositories": [ - "{ \"repo_url\": \"https://repo1.maven.org/maven2\" }" - ], - "artifacts": [ - "{ \"group\": \"com.beust\", \"artifact\": \"jcommander\", \"version\": \"1.82\" }", - "{ \"group\": \"com.google.escapevelocity\", \"artifact\": \"escapevelocity\", \"version\": \"1.1\" }", - "{ \"group\": \"com.google.guava\", \"artifact\": \"guava\", \"version\": \"31.1-jre\" }", - "{ \"group\": \"com.google.truth\", \"artifact\": \"truth\", \"version\": \"1.1.3\" }", - "{ \"group\": \"junit\", \"artifact\": \"junit\", \"version\": \"4.13.2\" }" - ], - "fail_on_missing_checksum": true, - "fetch_sources": true, - "fetch_javadoc": false, - "excluded_artifacts": [], - "generate_compat_repositories": false, - "version_conflict_policy": "default", - "override_targets": {}, - "strict_visibility": true, - "strict_visibility_value": [ - "@@//visibility:private" - ], - "maven_install_json": "@@stardoc~//:maven_install.json", - "resolve_timeout": 600, - "jetify": false, - "jetify_include_list": [ - "*" - ], - "use_starlark_android_rules": false, - "aar_import_bzl_label": "@build_bazel_rules_android//android:rules.bzl", - "duplicate_version_warning": "warn" - } - }, - "io_grpc_grpc_context_1_33_1": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "99b8aea2b614fe0e61c3676e681259dc43c2de7f64620998e1a8435eb2976496", - "urls": [ - "https://repo1.maven.org/maven2/io/grpc/grpc-context/1.33.1/grpc-context-1.33.1.jar", - "https://maven.google.com/io/grpc/grpc-context/1.33.1/grpc-context-1.33.1.jar" - ], - "downloaded_file_path": "io/grpc/grpc-context/1.33.1/grpc-context-1.33.1.jar" - } - }, - "com_google_api_grpc_proto_google_iam_v1_1_0_3": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "64cee7383a97e846da8d8e160e6c8fe30561e507260552c59e6ccfc81301fdc8", - "urls": [ - "https://repo1.maven.org/maven2/com/google/api/grpc/proto-google-iam-v1/1.0.3/proto-google-iam-v1-1.0.3.jar", - "https://maven.google.com/com/google/api/grpc/proto-google-iam-v1/1.0.3/proto-google-iam-v1-1.0.3.jar" - ], - "downloaded_file_path": "com/google/api/grpc/proto-google-iam-v1/1.0.3/proto-google-iam-v1-1.0.3.jar" - } - }, - "com_google_api_api_common_1_10_1": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "2a033f24bb620383eda440ad307cb8077cfec1c7eadc684d65216123a1b9613a", - "urls": [ - "https://repo1.maven.org/maven2/com/google/api/api-common/1.10.1/api-common-1.10.1.jar", - "https://maven.google.com/com/google/api/api-common/1.10.1/api-common-1.10.1.jar" - ], - "downloaded_file_path": "com/google/api/api-common/1.10.1/api-common-1.10.1.jar" - } - }, - "com_google_auth_google_auth_library_oauth2_http_0_22_0": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "1722d895c42dc42ea1d1f392ddbec1fbb28f7a979022c3a6c29acc39cc777ad1", - "urls": [ - "https://repo1.maven.org/maven2/com/google/auth/google-auth-library-oauth2-http/0.22.0/google-auth-library-oauth2-http-0.22.0.jar", - "https://maven.google.com/com/google/auth/google-auth-library-oauth2-http/0.22.0/google-auth-library-oauth2-http-0.22.0.jar" - ], - "downloaded_file_path": "com/google/auth/google-auth-library-oauth2-http/0.22.0/google-auth-library-oauth2-http-0.22.0.jar" - } - }, - "com_typesafe_netty_netty_reactive_streams_2_0_5": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "f949849fc8ee75fde468ba3a35df2e04577fa31a2940b83b2a7dc9d14dac13d6", - "urls": [ - "https://repo1.maven.org/maven2/com/typesafe/netty/netty-reactive-streams/2.0.5/netty-reactive-streams-2.0.5.jar", - "https://maven.google.com/com/typesafe/netty/netty-reactive-streams/2.0.5/netty-reactive-streams-2.0.5.jar" - ], - "downloaded_file_path": "com/typesafe/netty/netty-reactive-streams/2.0.5/netty-reactive-streams-2.0.5.jar" - } - }, - "com_typesafe_netty_netty_reactive_streams_http_2_0_5": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "b39224751ad936758176e9d994230380ade5e9079e7c8ad778e3995779bcf303", - "urls": [ - "https://repo1.maven.org/maven2/com/typesafe/netty/netty-reactive-streams-http/2.0.5/netty-reactive-streams-http-2.0.5.jar", - "https://maven.google.com/com/typesafe/netty/netty-reactive-streams-http/2.0.5/netty-reactive-streams-http-2.0.5.jar" - ], - "downloaded_file_path": "com/typesafe/netty/netty-reactive-streams-http/2.0.5/netty-reactive-streams-http-2.0.5.jar" - } - }, - "javax_annotation_javax_annotation_api_1_3_2": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "e04ba5195bcd555dc95650f7cc614d151e4bcd52d29a10b8aa2197f3ab89ab9b", - "urls": [ - "https://repo1.maven.org/maven2/javax/annotation/javax.annotation-api/1.3.2/javax.annotation-api-1.3.2.jar", - "https://maven.google.com/javax/annotation/javax.annotation-api/1.3.2/javax.annotation-api-1.3.2.jar" - ], - "downloaded_file_path": "javax/annotation/javax.annotation-api/1.3.2/javax.annotation-api-1.3.2.jar" - } - }, - "com_google_j2objc_j2objc_annotations_1_3": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "21af30c92267bd6122c0e0b4d20cccb6641a37eaf956c6540ec471d584e64a7b", - "urls": [ - "https://repo1.maven.org/maven2/com/google/j2objc/j2objc-annotations/1.3/j2objc-annotations-1.3.jar" - ], - "downloaded_file_path": "com/google/j2objc/j2objc-annotations/1.3/j2objc-annotations-1.3.jar" - } - }, - "software_amazon_awssdk_metrics_spi_2_17_183": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "08a11dc8c4ba464beafbcc7ac05b8c724c1ccb93da99482e82a68540ac704e4a", - "urls": [ - "https://repo1.maven.org/maven2/software/amazon/awssdk/metrics-spi/2.17.183/metrics-spi-2.17.183.jar", - "https://maven.google.com/software/amazon/awssdk/metrics-spi/2.17.183/metrics-spi-2.17.183.jar" - ], - "downloaded_file_path": "software/amazon/awssdk/metrics-spi/2.17.183/metrics-spi-2.17.183.jar" - } - }, - "com_google_escapevelocity_escapevelocity_1_1": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "37e76e4466836dedb864fb82355cd01c3bd21325ab642d89a0f759291b171231", - "urls": [ - "https://repo1.maven.org/maven2/com/google/escapevelocity/escapevelocity/1.1/escapevelocity-1.1.jar" - ], - "downloaded_file_path": "com/google/escapevelocity/escapevelocity/1.1/escapevelocity-1.1.jar" - } - }, - "org_reactivestreams_reactive_streams_1_0_3": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "1dee0481072d19c929b623e155e14d2f6085dc011529a0a0dbefc84cf571d865", - "urls": [ - "https://repo1.maven.org/maven2/org/reactivestreams/reactive-streams/1.0.3/reactive-streams-1.0.3.jar", - "https://maven.google.com/org/reactivestreams/reactive-streams/1.0.3/reactive-streams-1.0.3.jar" - ], - "downloaded_file_path": "org/reactivestreams/reactive-streams/1.0.3/reactive-streams-1.0.3.jar" - } - }, - "com_google_http_client_google_http_client_jackson2_1_38_0": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "e6504a82425fcc2168a4ca4175138ddcc085168daed8cdedb86d8f6fdc296e1e", - "urls": [ - "https://repo1.maven.org/maven2/com/google/http-client/google-http-client-jackson2/1.38.0/google-http-client-jackson2-1.38.0.jar", - "https://maven.google.com/com/google/http-client/google-http-client-jackson2/1.38.0/google-http-client-jackson2-1.38.0.jar" - ], - "downloaded_file_path": "com/google/http-client/google-http-client-jackson2/1.38.0/google-http-client-jackson2-1.38.0.jar" - } - }, - "io_netty_netty_transport_4_1_72_Final": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "c5fb68e9a65b6e8a516adfcb9fa323479ee7b4d9449d8a529d2ecab3d3711d5a", - "urls": [ - "https://repo1.maven.org/maven2/io/netty/netty-transport/4.1.72.Final/netty-transport-4.1.72.Final.jar", - "https://maven.google.com/io/netty/netty-transport/4.1.72.Final/netty-transport-4.1.72.Final.jar" - ], - "downloaded_file_path": "io/netty/netty-transport/4.1.72.Final/netty-transport-4.1.72.Final.jar" - } - }, - "com_beust_jcommander_1_82": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "deeac157c8de6822878d85d0c7bc8467a19cc8484d37788f7804f039dde280b1", - "urls": [ - "https://repo1.maven.org/maven2/com/beust/jcommander/1.82/jcommander-1.82.jar" - ], - "downloaded_file_path": "com/beust/jcommander/1.82/jcommander-1.82.jar" - } - }, - "io_netty_netty_codec_http2_4_1_72_Final": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "c89a70500f59e8563e720aaa808263a514bd9e2bd91ba84eab8c2ccb45f234b2", - "urls": [ - "https://repo1.maven.org/maven2/io/netty/netty-codec-http2/4.1.72.Final/netty-codec-http2-4.1.72.Final.jar", - "https://maven.google.com/io/netty/netty-codec-http2/4.1.72.Final/netty-codec-http2-4.1.72.Final.jar" - ], - "downloaded_file_path": "io/netty/netty-codec-http2/4.1.72.Final/netty-codec-http2-4.1.72.Final.jar" - } - }, - "io_opencensus_opencensus_api_0_24_0": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "f561b1cc2673844288e596ddf5bb6596868a8472fd2cb8993953fc5c034b2352", - "urls": [ - "https://repo1.maven.org/maven2/io/opencensus/opencensus-api/0.24.0/opencensus-api-0.24.0.jar", - "https://maven.google.com/io/opencensus/opencensus-api/0.24.0/opencensus-api-0.24.0.jar" - ], - "downloaded_file_path": "io/opencensus/opencensus-api/0.24.0/opencensus-api-0.24.0.jar" - } - }, - "rules_jvm_external_deps": { - "bzlFile": "@@rules_jvm_external~//:coursier.bzl", - "ruleClassName": "pinned_coursier_fetch", - "attributes": { - "repositories": [ - "{ \"repo_url\": \"https://repo1.maven.org/maven2\" }" - ], - "artifacts": [ - "{ \"group\": \"com.google.auth\", \"artifact\": \"google-auth-library-credentials\", \"version\": \"0.22.0\" }", - "{ \"group\": \"com.google.auth\", \"artifact\": \"google-auth-library-oauth2-http\", \"version\": \"0.22.0\" }", - "{ \"group\": \"com.google.cloud\", \"artifact\": \"google-cloud-core\", \"version\": \"1.93.10\" }", - "{ \"group\": \"com.google.cloud\", \"artifact\": \"google-cloud-storage\", \"version\": \"1.113.4\" }", - "{ \"group\": \"com.google.code.gson\", \"artifact\": \"gson\", \"version\": \"2.9.0\" }", - "{ \"group\": \"com.google.googlejavaformat\", \"artifact\": \"google-java-format\", \"version\": \"1.15.0\" }", - "{ \"group\": \"com.google.guava\", \"artifact\": \"guava\", \"version\": \"31.1-jre\" }", - "{ \"group\": \"org.apache.maven\", \"artifact\": \"maven-artifact\", \"version\": \"3.8.6\" }", - "{ \"group\": \"software.amazon.awssdk\", \"artifact\": \"s3\", \"version\": \"2.17.183\" }" - ], - "fetch_sources": true, - "fetch_javadoc": false, - "generate_compat_repositories": false, - "maven_install_json": "@@rules_jvm_external~//:rules_jvm_external_deps_install.json", - "override_targets": {}, - "strict_visibility": false, - "strict_visibility_value": [ - "@@//visibility:private" - ], - "jetify": false, - "jetify_include_list": [ - "*" - ], - "additional_netrc_lines": [], - "fail_if_repin_required": false, - "use_starlark_android_rules": false, - "aar_import_bzl_label": "@build_bazel_rules_android//android:rules.bzl", - "duplicate_version_warning": "warn" - } - }, - "org_threeten_threetenbp_1_5_0": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "dcf9c0f940739f2a825cd8626ff27113459a2f6eb18797c7152f93fff69c264f", - "urls": [ - "https://repo1.maven.org/maven2/org/threeten/threetenbp/1.5.0/threetenbp-1.5.0.jar", - "https://maven.google.com/org/threeten/threetenbp/1.5.0/threetenbp-1.5.0.jar" - ], - "downloaded_file_path": "org/threeten/threetenbp/1.5.0/threetenbp-1.5.0.jar" - } - }, - "software_amazon_awssdk_http_client_spi_2_17_183": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "fe7120f175df9e47ebcc5d946d7f40110faf2ba0a30364f3b935d5b8a5a6c3c6", - "urls": [ - "https://repo1.maven.org/maven2/software/amazon/awssdk/http-client-spi/2.17.183/http-client-spi-2.17.183.jar", - "https://maven.google.com/software/amazon/awssdk/http-client-spi/2.17.183/http-client-spi-2.17.183.jar" - ], - "downloaded_file_path": "software/amazon/awssdk/http-client-spi/2.17.183/http-client-spi-2.17.183.jar" - } - }, - "software_amazon_awssdk_third_party_jackson_core_2_17_183": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "1bc27c9960993c20e1ab058012dd1ae04c875eec9f0f08f2b2ca41e578dee9a4", - "urls": [ - "https://repo1.maven.org/maven2/software/amazon/awssdk/third-party-jackson-core/2.17.183/third-party-jackson-core-2.17.183.jar", - "https://maven.google.com/software/amazon/awssdk/third-party-jackson-core/2.17.183/third-party-jackson-core-2.17.183.jar" - ], - "downloaded_file_path": "software/amazon/awssdk/third-party-jackson-core/2.17.183/third-party-jackson-core-2.17.183.jar" - } - }, - "software_amazon_eventstream_eventstream_1_0_1": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "0c37d8e696117f02c302191b8110b0d0eb20fa412fce34c3a269ec73c16ce822", - "urls": [ - "https://repo1.maven.org/maven2/software/amazon/eventstream/eventstream/1.0.1/eventstream-1.0.1.jar", - "https://maven.google.com/software/amazon/eventstream/eventstream/1.0.1/eventstream-1.0.1.jar" - ], - "downloaded_file_path": "software/amazon/eventstream/eventstream/1.0.1/eventstream-1.0.1.jar" - } - }, - "com_google_oauth_client_google_oauth_client_1_31_1": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "4ed4e2948251dbda66ce251bd7f3b32cd8570055e5cdb165a3c7aea8f43da0ff", - "urls": [ - "https://repo1.maven.org/maven2/com/google/oauth-client/google-oauth-client/1.31.1/google-oauth-client-1.31.1.jar", - "https://maven.google.com/com/google/oauth-client/google-oauth-client/1.31.1/google-oauth-client-1.31.1.jar" - ], - "downloaded_file_path": "com/google/oauth-client/google-oauth-client/1.31.1/google-oauth-client-1.31.1.jar" - } - }, - "maven": { - "bzlFile": "@@rules_jvm_external~//:coursier.bzl", - "ruleClassName": "coursier_fetch", - "attributes": { - "repositories": [ - "{ \"repo_url\": \"https://repo1.maven.org/maven2\" }" - ], - "artifacts": [ - "{ \"group\": \"com.google.code.findbugs\", \"artifact\": \"jsr305\", \"version\": \"3.0.2\" }", - "{ \"group\": \"com.google.code.gson\", \"artifact\": \"gson\", \"version\": \"2.8.9\" }", - "{ \"group\": \"com.google.errorprone\", \"artifact\": \"error_prone_annotations\", \"version\": \"2.3.2\" }", - "{ \"group\": \"com.google.j2objc\", \"artifact\": \"j2objc-annotations\", \"version\": \"1.3\" }", - "{ \"group\": \"com.google.guava\", \"artifact\": \"guava\", \"version\": \"31.1-jre\" }", - "{ \"group\": \"com.google.guava\", \"artifact\": \"guava-testlib\", \"version\": \"31.1-jre\" }", - "{ \"group\": \"com.google.truth\", \"artifact\": \"truth\", \"version\": \"1.1.2\" }", - "{ \"group\": \"junit\", \"artifact\": \"junit\", \"version\": \"4.13.2\" }", - "{ \"group\": \"org.mockito\", \"artifact\": \"mockito-core\", \"version\": \"4.3.1\" }" - ], - "fail_on_missing_checksum": true, - "fetch_sources": true, - "fetch_javadoc": false, - "excluded_artifacts": [], - "generate_compat_repositories": false, - "version_conflict_policy": "default", - "override_targets": {}, - "strict_visibility": false, - "strict_visibility_value": [ - "@@//visibility:private" - ], - "resolve_timeout": 600, - "jetify": false, - "jetify_include_list": [ - "*" - ], - "use_starlark_android_rules": false, - "aar_import_bzl_label": "@build_bazel_rules_android//android:rules.bzl", - "duplicate_version_warning": "warn" - } - }, - "software_amazon_awssdk_aws_xml_protocol_2_17_183": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "566bba05d49256fa6994efd68fa625ae05a62ea45ee74bb9130d20ea20988363", - "urls": [ - "https://repo1.maven.org/maven2/software/amazon/awssdk/aws-xml-protocol/2.17.183/aws-xml-protocol-2.17.183.jar", - "https://maven.google.com/software/amazon/awssdk/aws-xml-protocol/2.17.183/aws-xml-protocol-2.17.183.jar" - ], - "downloaded_file_path": "software/amazon/awssdk/aws-xml-protocol/2.17.183/aws-xml-protocol-2.17.183.jar" - } - }, - "software_amazon_awssdk_annotations_2_17_183": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "8e4d72361ca805a0bd8bbd9017cd7ff77c8d170f2dd469c7d52d5653330bb3fd", - "urls": [ - "https://repo1.maven.org/maven2/software/amazon/awssdk/annotations/2.17.183/annotations-2.17.183.jar", - "https://maven.google.com/software/amazon/awssdk/annotations/2.17.183/annotations-2.17.183.jar" - ], - "downloaded_file_path": "software/amazon/awssdk/annotations/2.17.183/annotations-2.17.183.jar" - } - }, - "software_amazon_awssdk_netty_nio_client_2_17_183": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "a6d356f364c56d7b90006b0b7e503b8630010993a5587ce42e74b10b8dca2238", - "urls": [ - "https://repo1.maven.org/maven2/software/amazon/awssdk/netty-nio-client/2.17.183/netty-nio-client-2.17.183.jar", - "https://maven.google.com/software/amazon/awssdk/netty-nio-client/2.17.183/netty-nio-client-2.17.183.jar" - ], - "downloaded_file_path": "software/amazon/awssdk/netty-nio-client/2.17.183/netty-nio-client-2.17.183.jar" - } - }, - "com_google_guava_guava_31_1_jre": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "a42edc9cab792e39fe39bb94f3fca655ed157ff87a8af78e1d6ba5b07c4a00ab", - "urls": [ - "https://repo1.maven.org/maven2/com/google/guava/guava/31.1-jre/guava-31.1-jre.jar" - ], - "downloaded_file_path": "com/google/guava/guava/31.1-jre/guava-31.1-jre.jar" - } - }, - "com_google_auto_value_auto_value_annotations_1_7_4": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "fedd59b0b4986c342f6ab2d182f2a4ee9fceb2c7e2d5bdc4dc764c92394a23d3", - "urls": [ - "https://repo1.maven.org/maven2/com/google/auto/value/auto-value-annotations/1.7.4/auto-value-annotations-1.7.4.jar", - "https://maven.google.com/com/google/auto/value/auto-value-annotations/1.7.4/auto-value-annotations-1.7.4.jar" - ], - "downloaded_file_path": "com/google/auto/value/auto-value-annotations/1.7.4/auto-value-annotations-1.7.4.jar" - } - }, - "io_netty_netty_transport_native_unix_common_4_1_72_Final": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "6f8f1cc29b5a234eeee9439a63eb3f03a5994aa540ff555cb0b2c88cefaf6877", - "urls": [ - "https://repo1.maven.org/maven2/io/netty/netty-transport-native-unix-common/4.1.72.Final/netty-transport-native-unix-common-4.1.72.Final.jar", - "https://maven.google.com/io/netty/netty-transport-native-unix-common/4.1.72.Final/netty-transport-native-unix-common-4.1.72.Final.jar" - ], - "downloaded_file_path": "io/netty/netty-transport-native-unix-common/4.1.72.Final/netty-transport-native-unix-common-4.1.72.Final.jar" - } - }, - "junit_junit_4_13_2": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "8e495b634469d64fb8acfa3495a065cbacc8a0fff55ce1e31007be4c16dc57d3", - "urls": [ - "https://repo1.maven.org/maven2/junit/junit/4.13.2/junit-4.13.2.jar" - ], - "downloaded_file_path": "junit/junit/4.13.2/junit-4.13.2.jar" - } - }, - "io_opencensus_opencensus_contrib_http_util_0_24_0": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "7155273bbb1ed3d477ea33cf19d7bbc0b285ff395f43b29ae576722cf247000f", - "urls": [ - "https://repo1.maven.org/maven2/io/opencensus/opencensus-contrib-http-util/0.24.0/opencensus-contrib-http-util-0.24.0.jar", - "https://maven.google.com/io/opencensus/opencensus-contrib-http-util/0.24.0/opencensus-contrib-http-util-0.24.0.jar" - ], - "downloaded_file_path": "io/opencensus/opencensus-contrib-http-util/0.24.0/opencensus-contrib-http-util-0.24.0.jar" - } - }, - "com_fasterxml_jackson_core_jackson_core_2_11_3": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "78cd0a6b936232e06dd3e38da8a0345348a09cd1ff9c4d844c6ee72c75cfc402", - "urls": [ - "https://repo1.maven.org/maven2/com/fasterxml/jackson/core/jackson-core/2.11.3/jackson-core-2.11.3.jar", - "https://maven.google.com/com/fasterxml/jackson/core/jackson-core/2.11.3/jackson-core-2.11.3.jar" - ], - "downloaded_file_path": "com/fasterxml/jackson/core/jackson-core/2.11.3/jackson-core-2.11.3.jar" - } - }, - "com_google_cloud_google_cloud_core_1_93_10": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "832d74eca66f4601e162a8460d6f59f50d1d23f93c18b02654423b6b0d67c6ea", - "urls": [ - "https://repo1.maven.org/maven2/com/google/cloud/google-cloud-core/1.93.10/google-cloud-core-1.93.10.jar", - "https://maven.google.com/com/google/cloud/google-cloud-core/1.93.10/google-cloud-core-1.93.10.jar" - ], - "downloaded_file_path": "com/google/cloud/google-cloud-core/1.93.10/google-cloud-core-1.93.10.jar" - } - }, - "com_google_auth_google_auth_library_credentials_0_22_0": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "42c76031276de5b520909e9faf88c5b3c9a722d69ee9cfdafedb1c52c355dfc5", - "urls": [ - "https://repo1.maven.org/maven2/com/google/auth/google-auth-library-credentials/0.22.0/google-auth-library-credentials-0.22.0.jar", - "https://maven.google.com/com/google/auth/google-auth-library-credentials/0.22.0/google-auth-library-credentials-0.22.0.jar" - ], - "downloaded_file_path": "com/google/auth/google-auth-library-credentials/0.22.0/google-auth-library-credentials-0.22.0.jar" - } - }, - "software_amazon_awssdk_profiles_2_17_183": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "78833b32fde3f1c5320373b9ea955c1bbc28f2c904010791c4784e610193ee56", - "urls": [ - "https://repo1.maven.org/maven2/software/amazon/awssdk/profiles/2.17.183/profiles-2.17.183.jar", - "https://maven.google.com/software/amazon/awssdk/profiles/2.17.183/profiles-2.17.183.jar" - ], - "downloaded_file_path": "software/amazon/awssdk/profiles/2.17.183/profiles-2.17.183.jar" - } - }, - "org_apache_httpcomponents_httpcore_4_4_13": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "e06e89d40943245fcfa39ec537cdbfce3762aecde8f9c597780d2b00c2b43424", - "urls": [ - "https://repo1.maven.org/maven2/org/apache/httpcomponents/httpcore/4.4.13/httpcore-4.4.13.jar", - "https://maven.google.com/org/apache/httpcomponents/httpcore/4.4.13/httpcore-4.4.13.jar" - ], - "downloaded_file_path": "org/apache/httpcomponents/httpcore/4.4.13/httpcore-4.4.13.jar" - } - }, - "io_netty_netty_common_4_1_72_Final": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "8adb4c291260ceb2859a68c49f0adeed36bf49587608e2b81ecff6aaf06025e9", - "urls": [ - "https://repo1.maven.org/maven2/io/netty/netty-common/4.1.72.Final/netty-common-4.1.72.Final.jar", - "https://maven.google.com/io/netty/netty-common/4.1.72.Final/netty-common-4.1.72.Final.jar" - ], - "downloaded_file_path": "io/netty/netty-common/4.1.72.Final/netty-common-4.1.72.Final.jar" - } - }, - "io_netty_netty_transport_classes_epoll_4_1_72_Final": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "e1528a9751c1285aa7beaf3a1eb0597151716426ce38598ac9bc0891209b9e68", - "urls": [ - "https://repo1.maven.org/maven2/io/netty/netty-transport-classes-epoll/4.1.72.Final/netty-transport-classes-epoll-4.1.72.Final.jar", - "https://maven.google.com/io/netty/netty-transport-classes-epoll/4.1.72.Final/netty-transport-classes-epoll-4.1.72.Final.jar" - ], - "downloaded_file_path": "io/netty/netty-transport-classes-epoll/4.1.72.Final/netty-transport-classes-epoll-4.1.72.Final.jar" - } - }, - "org_checkerframework_checker_qual_3_12_0": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "ff10785ac2a357ec5de9c293cb982a2cbb605c0309ea4cc1cb9b9bc6dbe7f3cb", - "urls": [ - "https://repo1.maven.org/maven2/org/checkerframework/checker-qual/3.12.0/checker-qual-3.12.0.jar", - "https://maven.google.com/org/checkerframework/checker-qual/3.12.0/checker-qual-3.12.0.jar" - ], - "downloaded_file_path": "org/checkerframework/checker-qual/3.12.0/checker-qual-3.12.0.jar" - } - }, - "org_hamcrest_hamcrest_core_1_3": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "66fdef91e9739348df7a096aa384a5685f4e875584cce89386a7a47251c4d8e9", - "urls": [ - "https://repo1.maven.org/maven2/org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar" - ], - "downloaded_file_path": "org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar" - } - }, - "com_google_cloud_google_cloud_core_http_1_93_10": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "81ac67c14c7c4244d2b7db2607ad352416aca8d3bb2adf338964e8fea25b1b3c", - "urls": [ - "https://repo1.maven.org/maven2/com/google/cloud/google-cloud-core-http/1.93.10/google-cloud-core-http-1.93.10.jar", - "https://maven.google.com/com/google/cloud/google-cloud-core-http/1.93.10/google-cloud-core-http-1.93.10.jar" - ], - "downloaded_file_path": "com/google/cloud/google-cloud-core-http/1.93.10/google-cloud-core-http-1.93.10.jar" - } - }, - "software_amazon_awssdk_utils_2_17_183": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "7bd849bb5aa71bfdf6b849643736ecab3a7b3f204795804eefe5754104231ec6", - "urls": [ - "https://repo1.maven.org/maven2/software/amazon/awssdk/utils/2.17.183/utils-2.17.183.jar", - "https://maven.google.com/software/amazon/awssdk/utils/2.17.183/utils-2.17.183.jar" - ], - "downloaded_file_path": "software/amazon/awssdk/utils/2.17.183/utils-2.17.183.jar" - } - }, - "org_apache_commons_commons_lang3_3_8_1": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "dac807f65b07698ff39b1b07bfef3d87ae3fd46d91bbf8a2bc02b2a831616f68", - "urls": [ - "https://repo1.maven.org/maven2/org/apache/commons/commons-lang3/3.8.1/commons-lang3-3.8.1.jar", - "https://maven.google.com/org/apache/commons/commons-lang3/3.8.1/commons-lang3-3.8.1.jar" - ], - "downloaded_file_path": "org/apache/commons/commons-lang3/3.8.1/commons-lang3-3.8.1.jar" - } - }, - "software_amazon_awssdk_aws_core_2_17_183": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "bccbdbea689a665a702ff19828662d87fb7fe81529df13f02ef1e4c474ea9f93", - "urls": [ - "https://repo1.maven.org/maven2/software/amazon/awssdk/aws-core/2.17.183/aws-core-2.17.183.jar", - "https://maven.google.com/software/amazon/awssdk/aws-core/2.17.183/aws-core-2.17.183.jar" - ], - "downloaded_file_path": "software/amazon/awssdk/aws-core/2.17.183/aws-core-2.17.183.jar" - } - }, - "com_google_api_gax_httpjson_0_77_0": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "fd4dae47fa016d3b26e8d90b67ddc6c23c4c06e8bcdf085c70310ab7ef324bd6", - "urls": [ - "https://repo1.maven.org/maven2/com/google/api/gax-httpjson/0.77.0/gax-httpjson-0.77.0.jar", - "https://maven.google.com/com/google/api/gax-httpjson/0.77.0/gax-httpjson-0.77.0.jar" - ], - "downloaded_file_path": "com/google/api/gax-httpjson/0.77.0/gax-httpjson-0.77.0.jar" - } - }, - "unpinned_rules_jvm_external_deps": { - "bzlFile": "@@rules_jvm_external~//:coursier.bzl", - "ruleClassName": "coursier_fetch", - "attributes": { - "repositories": [ - "{ \"repo_url\": \"https://repo1.maven.org/maven2\" }" - ], - "artifacts": [ - "{ \"group\": \"com.google.auth\", \"artifact\": \"google-auth-library-credentials\", \"version\": \"0.22.0\" }", - "{ \"group\": \"com.google.auth\", \"artifact\": \"google-auth-library-oauth2-http\", \"version\": \"0.22.0\" }", - "{ \"group\": \"com.google.cloud\", \"artifact\": \"google-cloud-core\", \"version\": \"1.93.10\" }", - "{ \"group\": \"com.google.cloud\", \"artifact\": \"google-cloud-storage\", \"version\": \"1.113.4\" }", - "{ \"group\": \"com.google.code.gson\", \"artifact\": \"gson\", \"version\": \"2.9.0\" }", - "{ \"group\": \"com.google.googlejavaformat\", \"artifact\": \"google-java-format\", \"version\": \"1.15.0\" }", - "{ \"group\": \"com.google.guava\", \"artifact\": \"guava\", \"version\": \"31.1-jre\" }", - "{ \"group\": \"org.apache.maven\", \"artifact\": \"maven-artifact\", \"version\": \"3.8.6\" }", - "{ \"group\": \"software.amazon.awssdk\", \"artifact\": \"s3\", \"version\": \"2.17.183\" }" - ], - "fail_on_missing_checksum": true, - "fetch_sources": true, - "fetch_javadoc": false, - "excluded_artifacts": [], - "generate_compat_repositories": false, - "version_conflict_policy": "default", - "override_targets": {}, - "strict_visibility": false, - "strict_visibility_value": [ - "@@//visibility:private" - ], - "maven_install_json": "@@rules_jvm_external~//:rules_jvm_external_deps_install.json", - "resolve_timeout": 600, - "jetify": false, - "jetify_include_list": [ - "*" - ], - "use_starlark_android_rules": false, - "aar_import_bzl_label": "@build_bazel_rules_android//android:rules.bzl", - "duplicate_version_warning": "warn" - } - }, - "com_google_errorprone_error_prone_annotations_2_11_0": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "721cb91842b46fa056847d104d5225c8b8e1e8b62263b993051e1e5a0137b7ec", - "urls": [ - "https://repo1.maven.org/maven2/com/google/errorprone/error_prone_annotations/2.11.0/error_prone_annotations-2.11.0.jar" - ], - "downloaded_file_path": "com/google/errorprone/error_prone_annotations/2.11.0/error_prone_annotations-2.11.0.jar" - } - }, - "software_amazon_awssdk_regions_2_17_183": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "d3079395f3ffc07d04ffcce16fca29fb5968197f6e9ea3dbff6be297102b40a5", - "urls": [ - "https://repo1.maven.org/maven2/software/amazon/awssdk/regions/2.17.183/regions-2.17.183.jar", - "https://maven.google.com/software/amazon/awssdk/regions/2.17.183/regions-2.17.183.jar" - ], - "downloaded_file_path": "software/amazon/awssdk/regions/2.17.183/regions-2.17.183.jar" - } - }, - "io_netty_netty_handler_4_1_72_Final": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "9cb6012af7e06361d738ac4e3bdc49a158f8cf87d9dee0f2744056b7d99c28d5", - "urls": [ - "https://repo1.maven.org/maven2/io/netty/netty-handler/4.1.72.Final/netty-handler-4.1.72.Final.jar", - "https://maven.google.com/io/netty/netty-handler/4.1.72.Final/netty-handler-4.1.72.Final.jar" - ], - "downloaded_file_path": "io/netty/netty-handler/4.1.72.Final/netty-handler-4.1.72.Final.jar" - } - }, - "software_amazon_awssdk_aws_query_protocol_2_17_183": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "4dace03c76f80f3dec920cb3dedb2a95984c4366ef4fda728660cb90bed74848", - "urls": [ - "https://repo1.maven.org/maven2/software/amazon/awssdk/aws-query-protocol/2.17.183/aws-query-protocol-2.17.183.jar", - "https://maven.google.com/software/amazon/awssdk/aws-query-protocol/2.17.183/aws-query-protocol-2.17.183.jar" - ], - "downloaded_file_path": "software/amazon/awssdk/aws-query-protocol/2.17.183/aws-query-protocol-2.17.183.jar" - } - }, - "io_netty_netty_codec_http_4_1_72_Final": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "fa6fec88010bfaf6a7415b5364671b6b18ffb6b35a986ab97b423fd8c3a0174b", - "urls": [ - "https://repo1.maven.org/maven2/io/netty/netty-codec-http/4.1.72.Final/netty-codec-http-4.1.72.Final.jar", - "https://maven.google.com/io/netty/netty-codec-http/4.1.72.Final/netty-codec-http-4.1.72.Final.jar" - ], - "downloaded_file_path": "io/netty/netty-codec-http/4.1.72.Final/netty-codec-http-4.1.72.Final.jar" - } - }, - "io_netty_netty_resolver_4_1_72_Final": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "6474598aab7cc9d8d6cfa06c05bd1b19adbf7f8451dbdd73070b33a6c60b1b90", - "urls": [ - "https://repo1.maven.org/maven2/io/netty/netty-resolver/4.1.72.Final/netty-resolver-4.1.72.Final.jar", - "https://maven.google.com/io/netty/netty-resolver/4.1.72.Final/netty-resolver-4.1.72.Final.jar" - ], - "downloaded_file_path": "io/netty/netty-resolver/4.1.72.Final/netty-resolver-4.1.72.Final.jar" - } - }, - "software_amazon_awssdk_protocol_core_2_17_183": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "10e7c4faa1f05e2d73055d0390dbd0bb6450e2e6cb85beda051b1e4693c826ce", - "urls": [ - "https://repo1.maven.org/maven2/software/amazon/awssdk/protocol-core/2.17.183/protocol-core-2.17.183.jar", - "https://maven.google.com/software/amazon/awssdk/protocol-core/2.17.183/protocol-core-2.17.183.jar" - ], - "downloaded_file_path": "software/amazon/awssdk/protocol-core/2.17.183/protocol-core-2.17.183.jar" - } - }, - "stardoc_maven": { - "bzlFile": "@@rules_jvm_external~//:coursier.bzl", - "ruleClassName": "pinned_coursier_fetch", - "attributes": { - "repositories": [ - "{ \"repo_url\": \"https://repo1.maven.org/maven2\" }" - ], - "artifacts": [ - "{ \"group\": \"com.beust\", \"artifact\": \"jcommander\", \"version\": \"1.82\" }", - "{ \"group\": \"com.google.escapevelocity\", \"artifact\": \"escapevelocity\", \"version\": \"1.1\" }", - "{ \"group\": \"com.google.guava\", \"artifact\": \"guava\", \"version\": \"31.1-jre\" }", - "{ \"group\": \"com.google.truth\", \"artifact\": \"truth\", \"version\": \"1.1.3\" }", - "{ \"group\": \"junit\", \"artifact\": \"junit\", \"version\": \"4.13.2\" }" - ], - "fetch_sources": true, - "fetch_javadoc": false, - "generate_compat_repositories": false, - "maven_install_json": "@@stardoc~//:maven_install.json", - "override_targets": {}, - "strict_visibility": true, - "strict_visibility_value": [ - "@@//visibility:private" - ], - "jetify": false, - "jetify_include_list": [ - "*" - ], - "additional_netrc_lines": [], - "fail_if_repin_required": true, - "use_starlark_android_rules": false, - "aar_import_bzl_label": "@build_bazel_rules_android//android:rules.bzl", - "duplicate_version_warning": "warn" - } - }, - "com_google_truth_truth_1_1_3": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "fc0b67782289a2aabfddfdf99eff1dcd5edc890d49143fcd489214b107b8f4f3", - "urls": [ - "https://repo1.maven.org/maven2/com/google/truth/truth/1.1.3/truth-1.1.3.jar" - ], - "downloaded_file_path": "com/google/truth/truth/1.1.3/truth-1.1.3.jar" - } - }, - "org_checkerframework_checker_compat_qual_2_5_5": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "11d134b245e9cacc474514d2d66b5b8618f8039a1465cdc55bbc0b34e0008b7a", - "urls": [ - "https://repo1.maven.org/maven2/org/checkerframework/checker-compat-qual/2.5.5/checker-compat-qual-2.5.5.jar", - "https://maven.google.com/org/checkerframework/checker-compat-qual/2.5.5/checker-compat-qual-2.5.5.jar" - ], - "downloaded_file_path": "org/checkerframework/checker-compat-qual/2.5.5/checker-compat-qual-2.5.5.jar" - } - }, - "com_google_apis_google_api_services_storage_v1_rev20200927_1_30_10": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "52d26a9d105f8d8a0850807285f307a76cea8f3e0cdb2be4d3b15b1adfa77351", - "urls": [ - "https://repo1.maven.org/maven2/com/google/apis/google-api-services-storage/v1-rev20200927-1.30.10/google-api-services-storage-v1-rev20200927-1.30.10.jar", - "https://maven.google.com/com/google/apis/google-api-services-storage/v1-rev20200927-1.30.10/google-api-services-storage-v1-rev20200927-1.30.10.jar" - ], - "downloaded_file_path": "com/google/apis/google-api-services-storage/v1-rev20200927-1.30.10/google-api-services-storage-v1-rev20200927-1.30.10.jar" - } - }, - "org_ow2_asm_asm_9_1": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "cda4de455fab48ff0bcb7c48b4639447d4de859a7afc30a094a986f0936beba2", - "urls": [ - "https://repo1.maven.org/maven2/org/ow2/asm/asm/9.1/asm-9.1.jar" - ], - "downloaded_file_path": "org/ow2/asm/asm/9.1/asm-9.1.jar" - } - }, - "com_google_api_client_google_api_client_1_30_11": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "ee6f97865cc7de6c7c80955c3f37372cf3887bd75e4fc06f1058a6b4cd9bf4da", - "urls": [ - "https://repo1.maven.org/maven2/com/google/api-client/google-api-client/1.30.11/google-api-client-1.30.11.jar", - "https://maven.google.com/com/google/api-client/google-api-client/1.30.11/google-api-client-1.30.11.jar" - ], - "downloaded_file_path": "com/google/api-client/google-api-client/1.30.11/google-api-client-1.30.11.jar" - } - }, - "software_amazon_awssdk_s3_2_17_183": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "ab073b91107a9e4ed9f030314077d137fe627e055ad895fabb036980a050e360", - "urls": [ - "https://repo1.maven.org/maven2/software/amazon/awssdk/s3/2.17.183/s3-2.17.183.jar", - "https://maven.google.com/software/amazon/awssdk/s3/2.17.183/s3-2.17.183.jar" - ], - "downloaded_file_path": "software/amazon/awssdk/s3/2.17.183/s3-2.17.183.jar" - } - }, - "org_apache_maven_maven_artifact_3_8_6": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "de22a4c6f54fe31276a823b1bbd3adfd6823529e732f431b5eff0852c2b9252b", - "urls": [ - "https://repo1.maven.org/maven2/org/apache/maven/maven-artifact/3.8.6/maven-artifact-3.8.6.jar", - "https://maven.google.com/org/apache/maven/maven-artifact/3.8.6/maven-artifact-3.8.6.jar" - ], - "downloaded_file_path": "org/apache/maven/maven-artifact/3.8.6/maven-artifact-3.8.6.jar" - } - }, - "com_google_googlejavaformat_google_java_format_1_15_0": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "4f546cfe159547ac3b9547daa9649e728f6abc254979c975f1cb9971793692c3", - "urls": [ - "https://repo1.maven.org/maven2/com/google/googlejavaformat/google-java-format/1.15.0/google-java-format-1.15.0.jar", - "https://maven.google.com/com/google/googlejavaformat/google-java-format/1.15.0/google-java-format-1.15.0.jar" - ], - "downloaded_file_path": "com/google/googlejavaformat/google-java-format/1.15.0/google-java-format-1.15.0.jar" - } - }, - "org_apache_httpcomponents_httpclient_4_5_13": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "6fe9026a566c6a5001608cf3fc32196641f6c1e5e1986d1037ccdbd5f31ef743", - "urls": [ - "https://repo1.maven.org/maven2/org/apache/httpcomponents/httpclient/4.5.13/httpclient-4.5.13.jar", - "https://maven.google.com/org/apache/httpcomponents/httpclient/4.5.13/httpclient-4.5.13.jar" - ], - "downloaded_file_path": "org/apache/httpcomponents/httpclient/4.5.13/httpclient-4.5.13.jar" - } - }, - "com_google_guava_listenablefuture_9999_0_empty_to_avoid_conflict_with_guava": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "b372a037d4230aa57fbeffdef30fd6123f9c0c2db85d0aced00c91b974f33f99", - "urls": [ - "https://repo1.maven.org/maven2/com/google/guava/listenablefuture/9999.0-empty-to-avoid-conflict-with-guava/listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar" - ], - "downloaded_file_path": "com/google/guava/listenablefuture/9999.0-empty-to-avoid-conflict-with-guava/listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar" - } - }, - "com_google_http_client_google_http_client_1_38_0": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "411f4a42519b6b78bdc0fcfdf74c9edcef0ee97afa4a667abe04045a508d6302", - "urls": [ - "https://repo1.maven.org/maven2/com/google/http-client/google-http-client/1.38.0/google-http-client-1.38.0.jar", - "https://maven.google.com/com/google/http-client/google-http-client/1.38.0/google-http-client-1.38.0.jar" - ], - "downloaded_file_path": "com/google/http-client/google-http-client/1.38.0/google-http-client-1.38.0.jar" - } - }, - "software_amazon_awssdk_apache_client_2_17_183": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "78ceae502fce6a97bbe5ff8f6a010a52ab7ea3ae66cb1a4122e18185fce45022", - "urls": [ - "https://repo1.maven.org/maven2/software/amazon/awssdk/apache-client/2.17.183/apache-client-2.17.183.jar", - "https://maven.google.com/software/amazon/awssdk/apache-client/2.17.183/apache-client-2.17.183.jar" - ], - "downloaded_file_path": "software/amazon/awssdk/apache-client/2.17.183/apache-client-2.17.183.jar" - } - }, - "software_amazon_awssdk_arns_2_17_183": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "659a185e191d66c71de81209490e66abeaccae208ea7b2831a738670823447aa", - "urls": [ - "https://repo1.maven.org/maven2/software/amazon/awssdk/arns/2.17.183/arns-2.17.183.jar", - "https://maven.google.com/software/amazon/awssdk/arns/2.17.183/arns-2.17.183.jar" - ], - "downloaded_file_path": "software/amazon/awssdk/arns/2.17.183/arns-2.17.183.jar" - } - }, - "com_google_auto_value_auto_value_annotations_1_8_1": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "37ec09b47d7ed35a99d13927db5c86fc9071f620f943ead5d757144698310852", - "urls": [ - "https://repo1.maven.org/maven2/com/google/auto/value/auto-value-annotations/1.8.1/auto-value-annotations-1.8.1.jar" - ], - "downloaded_file_path": "com/google/auto/value/auto-value-annotations/1.8.1/auto-value-annotations-1.8.1.jar" - } - }, - "com_google_code_gson_gson_2_9_0": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "c96d60551331a196dac54b745aa642cd078ef89b6f267146b705f2c2cbef052d", - "urls": [ - "https://repo1.maven.org/maven2/com/google/code/gson/gson/2.9.0/gson-2.9.0.jar", - "https://maven.google.com/com/google/code/gson/gson/2.9.0/gson-2.9.0.jar" - ], - "downloaded_file_path": "com/google/code/gson/gson/2.9.0/gson-2.9.0.jar" - } - }, - "io_netty_netty_buffer_4_1_72_Final": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "568ff7cd9d8e2284ec980730c88924f686642929f8f219a74518b4e64755f3a1", - "urls": [ - "https://repo1.maven.org/maven2/io/netty/netty-buffer/4.1.72.Final/netty-buffer-4.1.72.Final.jar", - "https://maven.google.com/io/netty/netty-buffer/4.1.72.Final/netty-buffer-4.1.72.Final.jar" - ], - "downloaded_file_path": "io/netty/netty-buffer/4.1.72.Final/netty-buffer-4.1.72.Final.jar" - } - }, - "com_google_code_findbugs_jsr305_3_0_2": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "766ad2a0783f2687962c8ad74ceecc38a28b9f72a2d085ee438b7813e928d0c7", - "urls": [ - "https://repo1.maven.org/maven2/com/google/code/findbugs/jsr305/3.0.2/jsr305-3.0.2.jar" - ], - "downloaded_file_path": "com/google/code/findbugs/jsr305/3.0.2/jsr305-3.0.2.jar" - } - }, - "commons_codec_commons_codec_1_11": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "e599d5318e97aa48f42136a2927e6dfa4e8881dff0e6c8e3109ddbbff51d7b7d", - "urls": [ - "https://repo1.maven.org/maven2/commons-codec/commons-codec/1.11/commons-codec-1.11.jar", - "https://maven.google.com/commons-codec/commons-codec/1.11/commons-codec-1.11.jar" - ], - "downloaded_file_path": "commons-codec/commons-codec/1.11/commons-codec-1.11.jar" - } - }, - "software_amazon_awssdk_auth_2_17_183": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "8820c6636e5c14efc29399fb5565ce50212b0c1f4ed720a025a2c402d54e0978", - "urls": [ - "https://repo1.maven.org/maven2/software/amazon/awssdk/auth/2.17.183/auth-2.17.183.jar", - "https://maven.google.com/software/amazon/awssdk/auth/2.17.183/auth-2.17.183.jar" - ], - "downloaded_file_path": "software/amazon/awssdk/auth/2.17.183/auth-2.17.183.jar" - } - }, - "software_amazon_awssdk_json_utils_2_17_183": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "51ab7f550adc06afcb49f5270cdf690f1bfaaee243abaa5d978095e2a1e4e1a5", - "urls": [ - "https://repo1.maven.org/maven2/software/amazon/awssdk/json-utils/2.17.183/json-utils-2.17.183.jar", - "https://maven.google.com/software/amazon/awssdk/json-utils/2.17.183/json-utils-2.17.183.jar" - ], - "downloaded_file_path": "software/amazon/awssdk/json-utils/2.17.183/json-utils-2.17.183.jar" - } - }, - "org_codehaus_plexus_plexus_utils_3_3_1": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "4b570fcdbe5a894f249d2eb9b929358a9c88c3e548d227a80010461930222f2a", - "urls": [ - "https://repo1.maven.org/maven2/org/codehaus/plexus/plexus-utils/3.3.1/plexus-utils-3.3.1.jar", - "https://maven.google.com/org/codehaus/plexus/plexus-utils/3.3.1/plexus-utils-3.3.1.jar" - ], - "downloaded_file_path": "org/codehaus/plexus/plexus-utils/3.3.1/plexus-utils-3.3.1.jar" - } - }, - "org_checkerframework_checker_qual_3_13_0": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "3ea0dcd73b4d6cb2fb34bd7ed4dad6db327a01ebad7db05eb7894076b3d64491", - "urls": [ - "https://repo1.maven.org/maven2/org/checkerframework/checker-qual/3.13.0/checker-qual-3.13.0.jar" - ], - "downloaded_file_path": "org/checkerframework/checker-qual/3.13.0/checker-qual-3.13.0.jar" - } - }, - "com_google_protobuf_protobuf_java_util_3_13_0": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "d9de66b8c9445905dfa7064f6d5213d47ce88a20d34e21d83c4a94a229e14e62", - "urls": [ - "https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java-util/3.13.0/protobuf-java-util-3.13.0.jar", - "https://maven.google.com/com/google/protobuf/protobuf-java-util/3.13.0/protobuf-java-util-3.13.0.jar" - ], - "downloaded_file_path": "com/google/protobuf/protobuf-java-util/3.13.0/protobuf-java-util-3.13.0.jar" - } - }, - "io_netty_netty_codec_4_1_72_Final": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "5d8591ca271a1e9c224e8de3873aa9936acb581ee0db514e7dc18523df36d16c", - "urls": [ - "https://repo1.maven.org/maven2/io/netty/netty-codec/4.1.72.Final/netty-codec-4.1.72.Final.jar", - "https://maven.google.com/io/netty/netty-codec/4.1.72.Final/netty-codec-4.1.72.Final.jar" - ], - "downloaded_file_path": "io/netty/netty-codec/4.1.72.Final/netty-codec-4.1.72.Final.jar" - } - }, - "com_google_protobuf_protobuf_java_3_13_0": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "97d5b2758408690c0dc276238707492a0b6a4d71206311b6c442cdc26c5973ff", - "urls": [ - "https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java/3.13.0/protobuf-java-3.13.0.jar", - "https://maven.google.com/com/google/protobuf/protobuf-java/3.13.0/protobuf-java-3.13.0.jar" - ], - "downloaded_file_path": "com/google/protobuf/protobuf-java/3.13.0/protobuf-java-3.13.0.jar" - } - }, - "io_netty_netty_tcnative_classes_2_0_46_Final": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "d3ec888dcc4ac7915bf88b417c5e04fd354f4311032a748a6882df09347eed9a", - "urls": [ - "https://repo1.maven.org/maven2/io/netty/netty-tcnative-classes/2.0.46.Final/netty-tcnative-classes-2.0.46.Final.jar", - "https://maven.google.com/io/netty/netty-tcnative-classes/2.0.46.Final/netty-tcnative-classes-2.0.46.Final.jar" - ], - "downloaded_file_path": "io/netty/netty-tcnative-classes/2.0.46.Final/netty-tcnative-classes-2.0.46.Final.jar" - } - }, - "software_amazon_awssdk_sdk_core_2_17_183": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "677e9cc90fdd82c1f40f97b99cb115b13ad6c3f58beeeab1c061af6954d64c77", - "urls": [ - "https://repo1.maven.org/maven2/software/amazon/awssdk/sdk-core/2.17.183/sdk-core-2.17.183.jar", - "https://maven.google.com/software/amazon/awssdk/sdk-core/2.17.183/sdk-core-2.17.183.jar" - ], - "downloaded_file_path": "software/amazon/awssdk/sdk-core/2.17.183/sdk-core-2.17.183.jar" - } - } - }, - "recordedRepoMappingEntries": [ - [ - "rules_jvm_external~", - "bazel_tools", - "bazel_tools" - ], - [ - "rules_jvm_external~", - "rules_jvm_external", - "rules_jvm_external~" - ] - ] - } - }, - "@@rules_jvm_external~//:non-module-deps.bzl%non_module_deps": { - "general": { - "bzlTransitiveDigest": "iEno7cg5adLHR93u99plc5k2u8XdclqIZI8tyihJoqQ=", - "usagesDigest": "hiuzyio8ny4T3UoEFpHaxXzNFc6OGUFvx5DDVLBBUmU=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "io_bazel_rules_kotlin": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "sha256": "946747acdbeae799b085d12b240ec346f775ac65236dfcf18aa0cd7300f6de78", - "urls": [ - "https://github.com/bazelbuild/rules_kotlin/releases/download/v1.7.0-RC-2/rules_kotlin_release.tgz" - ] - } - } - }, - "recordedRepoMappingEntries": [ - [ - "rules_jvm_external~", - "bazel_tools", - "bazel_tools" - ] - ] - } - }, - "@@rules_oci~//oci:extensions.bzl%oci": { - "general": { - "bzlTransitiveDigest": "isxSu3lTE3wtlF3Tpyb8WdKFOF9wxYEls/7wynmIriw=", - "usagesDigest": "vyT6vhzSqo9Ht7Q/BAmiU5gmr2XdZxY5yEdLfo6junQ=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "bazel_features_version": { - "bzlFile": "@@bazel_features~//private:version_repo.bzl", - "ruleClassName": "version_repo", - "attributes": {} - }, - "copy_to_directory_windows_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_to_directory_toolchain.bzl", - "ruleClassName": "copy_to_directory_platform_repo", - "attributes": { - "platform": "windows_amd64" - } - }, - "jq_darwin_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:jq_toolchain.bzl", - "ruleClassName": "jq_platform_repo", - "attributes": { - "platform": "darwin_amd64", - "version": "1.7" - } - }, - "copy_to_directory_freebsd_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_to_directory_toolchain.bzl", - "ruleClassName": "copy_to_directory_platform_repo", - "attributes": { - "platform": "freebsd_amd64" - } - }, - "oci_crane_linux_arm64": { - "bzlFile": "@@rules_oci~//oci:repositories.bzl", - "ruleClassName": "crane_repositories", - "attributes": { - "platform": "linux_arm64", - "crane_version": "v0.18.0" - } - }, - "jq_linux_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:jq_toolchain.bzl", - "ruleClassName": "jq_platform_repo", - "attributes": { - "platform": "linux_arm64", - "version": "1.7" - } - }, - "coreutils_darwin_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:coreutils_toolchain.bzl", - "ruleClassName": "coreutils_platform_repo", - "attributes": { - "platform": "darwin_arm64", - "version": "0.0.26" - } - }, - "bsd_tar_linux_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:tar_toolchain.bzl", - "ruleClassName": "bsdtar_binary_repo", - "attributes": { - "platform": "linux_arm64" - } - }, - "copy_to_directory_linux_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_to_directory_toolchain.bzl", - "ruleClassName": "copy_to_directory_platform_repo", - "attributes": { - "platform": "linux_arm64" - } - }, - "oci_regctl_toolchains": { - "bzlFile": "@@rules_oci~//oci/private:toolchains_repo.bzl", - "ruleClassName": "toolchains_repo", - "attributes": { - "toolchain_type": "@rules_oci//oci:regctl_toolchain_type", - "toolchain": "@oci_regctl_{platform}//:regctl_toolchain" - } - }, - "oci_regctl_windows_armv6": { - "bzlFile": "@@rules_oci~//oci:repositories.bzl", - "ruleClassName": "regctl_repositories", - "attributes": { - "platform": "windows_armv6" - } - }, - "oci_crane_linux_amd64": { - "bzlFile": "@@rules_oci~//oci:repositories.bzl", - "ruleClassName": "crane_repositories", - "attributes": { - "platform": "linux_amd64", - "crane_version": "v0.18.0" - } - }, - "coreutils_darwin_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:coreutils_toolchain.bzl", - "ruleClassName": "coreutils_platform_repo", - "attributes": { - "platform": "darwin_amd64", - "version": "0.0.26" - } - }, - "coreutils_linux_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:coreutils_toolchain.bzl", - "ruleClassName": "coreutils_platform_repo", - "attributes": { - "platform": "linux_arm64", - "version": "0.0.26" - } - }, - "zstd_linux_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:zstd_toolchain.bzl", - "ruleClassName": "zstd_binary_repo", - "attributes": { - "platform": "linux_arm64" - } - }, - "oci_crane_darwin_arm64": { - "bzlFile": "@@rules_oci~//oci:repositories.bzl", - "ruleClassName": "crane_repositories", - "attributes": { - "platform": "darwin_arm64", - "crane_version": "v0.18.0" - } - }, - "jq_darwin_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:jq_toolchain.bzl", - "ruleClassName": "jq_platform_repo", - "attributes": { - "platform": "darwin_arm64", - "version": "1.7" - } - }, - "oci_regctl_linux_s390x": { - "bzlFile": "@@rules_oci~//oci:repositories.bzl", - "ruleClassName": "regctl_repositories", - "attributes": { - "platform": "linux_s390x" - } - }, - "oci_regctl_darwin_amd64": { - "bzlFile": "@@rules_oci~//oci:repositories.bzl", - "ruleClassName": "regctl_repositories", - "attributes": { - "platform": "darwin_amd64" - } - }, - "oci_crane_linux_i386": { - "bzlFile": "@@rules_oci~//oci:repositories.bzl", - "ruleClassName": "crane_repositories", - "attributes": { - "platform": "linux_i386", - "crane_version": "v0.18.0" - } - }, - "oci_regctl_windows_amd64": { - "bzlFile": "@@rules_oci~//oci:repositories.bzl", - "ruleClassName": "regctl_repositories", - "attributes": { - "platform": "windows_amd64" - } - }, - "oci_crane_windows_armv6": { - "bzlFile": "@@rules_oci~//oci:repositories.bzl", - "ruleClassName": "crane_repositories", - "attributes": { - "platform": "windows_armv6", - "crane_version": "v0.18.0" - } - }, - "oci_crane_toolchains": { - "bzlFile": "@@rules_oci~//oci/private:toolchains_repo.bzl", - "ruleClassName": "toolchains_repo", - "attributes": { - "toolchain_type": "@rules_oci//oci:crane_toolchain_type", - "toolchain": "@oci_crane_{platform}//:crane_toolchain" - } - }, - "copy_to_directory_darwin_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_to_directory_toolchain.bzl", - "ruleClassName": "copy_to_directory_platform_repo", - "attributes": { - "platform": "darwin_amd64" - } - }, - "zstd_darwin_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:zstd_toolchain.bzl", - "ruleClassName": "zstd_binary_repo", - "attributes": { - "platform": "darwin_arm64" - } - }, - "bsd_tar_linux_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:tar_toolchain.bzl", - "ruleClassName": "bsdtar_binary_repo", - "attributes": { - "platform": "linux_amd64" - } - }, - "oci_crane_windows_amd64": { - "bzlFile": "@@rules_oci~//oci:repositories.bzl", - "ruleClassName": "crane_repositories", - "attributes": { - "platform": "windows_amd64", - "crane_version": "v0.18.0" - } - }, - "oci_regctl_linux_arm64": { - "bzlFile": "@@rules_oci~//oci:repositories.bzl", - "ruleClassName": "regctl_repositories", - "attributes": { - "platform": "linux_arm64" - } - }, - "oci_crane_linux_s390x": { - "bzlFile": "@@rules_oci~//oci:repositories.bzl", - "ruleClassName": "crane_repositories", - "attributes": { - "platform": "linux_s390x", - "crane_version": "v0.18.0" - } - }, - "zstd_linux_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:zstd_toolchain.bzl", - "ruleClassName": "zstd_binary_repo", - "attributes": { - "platform": "linux_amd64" - } - }, - "distroless_cc_debian12_linux_amd64": { - "bzlFile": "@@rules_oci~//oci/private:pull.bzl", - "ruleClassName": "oci_pull", - "attributes": { - "scheme": "https", - "registry": "gcr.io", - "repository": "distroless/cc-debian12", - "identifier": "sha256:1850aee2ff72864350058d83d681c757d45c885986d15fcca7309b9e5c69f39a", - "platform": "linux/amd64", - "target_name": "distroless_cc_debian12_linux_amd64", - "bazel_tags": [] - } - }, - "oci_regctl_darwin_arm64": { - "bzlFile": "@@rules_oci~//oci:repositories.bzl", - "ruleClassName": "regctl_repositories", - "attributes": { - "platform": "darwin_arm64" - } - }, - "bsd_tar_windows_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:tar_toolchain.bzl", - "ruleClassName": "bsdtar_binary_repo", - "attributes": { - "platform": "windows_amd64" - } - }, - "jq": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:jq_toolchain.bzl", - "ruleClassName": "jq_host_alias_repo", - "attributes": {} - }, - "oci_crane_darwin_amd64": { - "bzlFile": "@@rules_oci~//oci:repositories.bzl", - "ruleClassName": "crane_repositories", - "attributes": { - "platform": "darwin_amd64", - "crane_version": "v0.18.0" - } - }, - "bsd_tar_darwin_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:tar_toolchain.bzl", - "ruleClassName": "bsdtar_binary_repo", - "attributes": { - "platform": "darwin_arm64" - } - }, - "copy_to_directory_linux_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_to_directory_toolchain.bzl", - "ruleClassName": "copy_to_directory_platform_repo", - "attributes": { - "platform": "linux_amd64" - } - }, - "coreutils_linux_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:coreutils_toolchain.bzl", - "ruleClassName": "coreutils_platform_repo", - "attributes": { - "platform": "linux_amd64", - "version": "0.0.26" - } - }, - "bazel_skylib": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "sha256": "9f38886a40548c6e96c106b752f242130ee11aaa068a56ba7e56f4511f33e4f2", - "urls": [ - "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.6.1/bazel-skylib-1.6.1.tar.gz", - "https://github.com/bazelbuild/bazel-skylib/releases/download/1.6.1/bazel-skylib-1.6.1.tar.gz" - ] - } - }, - "oci_crane_linux_armv6": { - "bzlFile": "@@rules_oci~//oci:repositories.bzl", - "ruleClassName": "crane_repositories", - "attributes": { - "platform": "linux_armv6", - "crane_version": "v0.18.0" - } - }, - "copy_to_directory_darwin_arm64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_to_directory_toolchain.bzl", - "ruleClassName": "copy_to_directory_platform_repo", - "attributes": { - "platform": "darwin_arm64" - } - }, - "coreutils_toolchains": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:coreutils_toolchain.bzl", - "ruleClassName": "coreutils_toolchains_repo", - "attributes": { - "user_repository_name": "coreutils" - } - }, - "zstd_darwin_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:zstd_toolchain.bzl", - "ruleClassName": "zstd_binary_repo", - "attributes": { - "platform": "darwin_amd64" - } - }, - "zstd_toolchains": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:zstd_toolchain.bzl", - "ruleClassName": "zstd_toolchains_repo", - "attributes": { - "user_repository_name": "zstd" - } - }, - "jq_linux_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:jq_toolchain.bzl", - "ruleClassName": "jq_platform_repo", - "attributes": { - "platform": "linux_amd64", - "version": "1.7" - } - }, - "bsd_tar_darwin_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:tar_toolchain.bzl", - "ruleClassName": "bsdtar_binary_repo", - "attributes": { - "platform": "darwin_amd64" - } - }, - "distroless_cc_debian12": { - "bzlFile": "@@rules_oci~//oci/private:pull.bzl", - "ruleClassName": "oci_alias", - "attributes": { - "target_name": "distroless_cc_debian12", - "scheme": "https", - "registry": "gcr.io", - "repository": "distroless/cc-debian12", - "identifier": "sha256:1850aee2ff72864350058d83d681c757d45c885986d15fcca7309b9e5c69f39a", - "platforms": { - "@@platforms//cpu:x86_64": "@distroless_cc_debian12_linux_amd64" - }, - "bzlmod_repository": "distroless_cc_debian12", - "reproducible": true - } - }, - "oci_regctl_linux_amd64": { - "bzlFile": "@@rules_oci~//oci:repositories.bzl", - "ruleClassName": "regctl_repositories", - "attributes": { - "platform": "linux_amd64" - } - }, - "bsd_tar_toolchains": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:tar_toolchain.bzl", - "ruleClassName": "tar_toolchains_repo", - "attributes": { - "user_repository_name": "bsd_tar" - } - }, - "jq_windows_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:jq_toolchain.bzl", - "ruleClassName": "jq_platform_repo", - "attributes": { - "platform": "windows_amd64", - "version": "1.7" - } - }, - "oci_regctl_linux_i386": { - "bzlFile": "@@rules_oci~//oci:repositories.bzl", - "ruleClassName": "regctl_repositories", - "attributes": { - "platform": "linux_i386" - } - }, - "jq_toolchains": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:jq_toolchain.bzl", - "ruleClassName": "jq_toolchains_repo", - "attributes": { - "user_repository_name": "jq" - } - }, - "copy_to_directory_toolchains": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:copy_to_directory_toolchain.bzl", - "ruleClassName": "copy_to_directory_toolchains_repo", - "attributes": { - "user_repository_name": "copy_to_directory" - } - }, - "oci_regctl_linux_armv6": { - "bzlFile": "@@rules_oci~//oci:repositories.bzl", - "ruleClassName": "regctl_repositories", - "attributes": { - "platform": "linux_armv6" - } - }, - "bazel_features_globals": { - "bzlFile": "@@bazel_features~//private:globals_repo.bzl", - "ruleClassName": "globals_repo", - "attributes": { - "globals": { - "RunEnvironmentInfo": "5.3.0", - "DefaultInfo": "0.0.1", - "__TestingOnly_NeverAvailable": "1000000000.0.0" - } - } - }, - "coreutils_windows_amd64": { - "bzlFile": "@@aspect_bazel_lib~//lib/private:coreutils_toolchain.bzl", - "ruleClassName": "coreutils_platform_repo", - "attributes": { - "platform": "windows_amd64", - "version": "0.0.26" - } - } - }, - "moduleExtensionMetadata": { - "explicitRootModuleDirectDeps": [ - "distroless_cc_debian12", - "distroless_cc_debian12_linux_amd64" - ], - "explicitRootModuleDirectDevDeps": [], - "useAllRepos": "NO", - "reproducible": false - }, - "recordedRepoMappingEntries": [ - [ - "aspect_bazel_lib~", - "bazel_tools", - "bazel_tools" - ], - [ - "bazel_features~", - "bazel_tools", - "bazel_tools" - ], - [ - "rules_oci~", - "aspect_bazel_lib", - "aspect_bazel_lib~" - ], - [ - "rules_oci~", - "bazel_features", - "bazel_features~" - ], - [ - "rules_oci~", - "bazel_skylib", - "bazel_skylib~" - ] - ] - } - }, - "@@rules_python~//python/extensions:python.bzl%python": { - "general": { - "bzlTransitiveDigest": "uFBdNKDewkX8DPVvKoPEcW0b3dXtksiE7SShRkuCA3E=", - "usagesDigest": "fQEsnAYwqRJT7/lTBAe+NllONXk6f/Tc57oiPxLG8SI=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": { - "RULES_PYTHON_BZLMOD_DEBUG": null - }, - "generatedRepoSpecs": { - "python_3_11_s390x-unknown-linux-gnu": { - "bzlFile": "@@rules_python~//python:repositories.bzl", - "ruleClassName": "python_repository", - "attributes": { - "sha256": "49520e3ff494708020f306e30b0964f079170be83e956be4504f850557378a22", - "patches": [], - "platform": "s390x-unknown-linux-gnu", - "python_version": "3.11.7", - "release_filename": "20240107/cpython-3.11.7+20240107-s390x-unknown-linux-gnu-install_only.tar.gz", - "urls": [ - "https://github.com/indygreg/python-build-standalone/releases/download/20240107/cpython-3.11.7+20240107-s390x-unknown-linux-gnu-install_only.tar.gz" - ], - "distutils_content": "", - "strip_prefix": "python", - "coverage_tool": "", - "ignore_root_user_error": false - } - }, - "python_3_11_host": { - "bzlFile": "@@rules_python~//python/private:toolchains_repo.bzl", - "ruleClassName": "host_toolchain", - "attributes": { - "python_version": "3.11.7", - "user_repository_name": "python_3_11", - "platforms": [ - "aarch64-apple-darwin", - "aarch64-unknown-linux-gnu", - "ppc64le-unknown-linux-gnu", - "s390x-unknown-linux-gnu", - "x86_64-apple-darwin", - "x86_64-pc-windows-msvc", - "x86_64-unknown-linux-gnu" - ] - } - }, - "python_3_11": { - "bzlFile": "@@rules_python~//python/private:toolchains_repo.bzl", - "ruleClassName": "toolchain_aliases", - "attributes": { - "python_version": "3.11.7", - "user_repository_name": "python_3_11", - "platforms": [ - "aarch64-apple-darwin", - "aarch64-unknown-linux-gnu", - "ppc64le-unknown-linux-gnu", - "s390x-unknown-linux-gnu", - "x86_64-apple-darwin", - "x86_64-pc-windows-msvc", - "x86_64-unknown-linux-gnu" - ] - } - }, - "python_3_11_aarch64-unknown-linux-gnu": { - "bzlFile": "@@rules_python~//python:repositories.bzl", - "ruleClassName": "python_repository", - "attributes": { - "sha256": "b102eaf865eb715aa98a8a2ef19037b6cc3ae7dfd4a632802650f29de635aa13", - "patches": [], - "platform": "aarch64-unknown-linux-gnu", - "python_version": "3.11.7", - "release_filename": "20240107/cpython-3.11.7+20240107-aarch64-unknown-linux-gnu-install_only.tar.gz", - "urls": [ - "https://github.com/indygreg/python-build-standalone/releases/download/20240107/cpython-3.11.7+20240107-aarch64-unknown-linux-gnu-install_only.tar.gz" - ], - "distutils_content": "", - "strip_prefix": "python", - "coverage_tool": "", - "ignore_root_user_error": false - } - }, - "python_3_11_aarch64-apple-darwin": { - "bzlFile": "@@rules_python~//python:repositories.bzl", - "ruleClassName": "python_repository", - "attributes": { - "sha256": "b042c966920cf8465385ca3522986b12d745151a72c060991088977ca36d3883", - "patches": [], - "platform": "aarch64-apple-darwin", - "python_version": "3.11.7", - "release_filename": "20240107/cpython-3.11.7+20240107-aarch64-apple-darwin-install_only.tar.gz", - "urls": [ - "https://github.com/indygreg/python-build-standalone/releases/download/20240107/cpython-3.11.7+20240107-aarch64-apple-darwin-install_only.tar.gz" - ], - "distutils_content": "", - "strip_prefix": "python", - "coverage_tool": "", - "ignore_root_user_error": false - } - }, - "python_3_11_ppc64le-unknown-linux-gnu": { - "bzlFile": "@@rules_python~//python:repositories.bzl", - "ruleClassName": "python_repository", - "attributes": { - "sha256": "b44e1b74afe75c7b19143413632c4386708ae229117f8f950c2094e9681d34c7", - "patches": [], - "platform": "ppc64le-unknown-linux-gnu", - "python_version": "3.11.7", - "release_filename": "20240107/cpython-3.11.7+20240107-ppc64le-unknown-linux-gnu-install_only.tar.gz", - "urls": [ - "https://github.com/indygreg/python-build-standalone/releases/download/20240107/cpython-3.11.7+20240107-ppc64le-unknown-linux-gnu-install_only.tar.gz" - ], - "distutils_content": "", - "strip_prefix": "python", - "coverage_tool": "", - "ignore_root_user_error": false - } - }, - "python_3_11_x86_64-apple-darwin": { - "bzlFile": "@@rules_python~//python:repositories.bzl", - "ruleClassName": "python_repository", - "attributes": { - "sha256": "a0e615eef1fafdc742da0008425a9030b7ea68a4ae4e73ac557ef27b112836d4", - "patches": [], - "platform": "x86_64-apple-darwin", - "python_version": "3.11.7", - "release_filename": "20240107/cpython-3.11.7+20240107-x86_64-apple-darwin-install_only.tar.gz", - "urls": [ - "https://github.com/indygreg/python-build-standalone/releases/download/20240107/cpython-3.11.7+20240107-x86_64-apple-darwin-install_only.tar.gz" - ], - "distutils_content": "", - "strip_prefix": "python", - "coverage_tool": "", - "ignore_root_user_error": false - } - }, - "pythons_hub": { - "bzlFile": "@@rules_python~//python/private/bzlmod:pythons_hub.bzl", - "ruleClassName": "hub_repo", - "attributes": { - "default_python_version": "3.11", - "toolchain_prefixes": [ - "_0000_python_3_11_" - ], - "toolchain_python_versions": [ - "3.11" - ], - "toolchain_set_python_version_constraints": [ - "False" - ], - "toolchain_user_repository_names": [ - "python_3_11" - ] - } - }, - "python_versions": { - "bzlFile": "@@rules_python~//python/private:toolchains_repo.bzl", - "ruleClassName": "multi_toolchain_aliases", - "attributes": { - "python_versions": { - "3.11": "python_3_11" - } - } - }, - "python_3_11_x86_64-pc-windows-msvc": { - "bzlFile": "@@rules_python~//python:repositories.bzl", - "ruleClassName": "python_repository", - "attributes": { - "sha256": "67077e6fa918e4f4fd60ba169820b00be7c390c497bf9bc9cab2c255ea8e6f3e", - "patches": [], - "platform": "x86_64-pc-windows-msvc", - "python_version": "3.11.7", - "release_filename": "20240107/cpython-3.11.7+20240107-x86_64-pc-windows-msvc-shared-install_only.tar.gz", - "urls": [ - "https://github.com/indygreg/python-build-standalone/releases/download/20240107/cpython-3.11.7+20240107-x86_64-pc-windows-msvc-shared-install_only.tar.gz" - ], - "distutils_content": "", - "strip_prefix": "python", - "coverage_tool": "", - "ignore_root_user_error": false - } - }, - "python_3_11_x86_64-unknown-linux-gnu": { - "bzlFile": "@@rules_python~//python:repositories.bzl", - "ruleClassName": "python_repository", - "attributes": { - "sha256": "4a51ce60007a6facf64e5495f4cf322e311ba9f39a8cd3f3e4c026eae488e140", - "patches": [], - "platform": "x86_64-unknown-linux-gnu", - "python_version": "3.11.7", - "release_filename": "20240107/cpython-3.11.7+20240107-x86_64-unknown-linux-gnu-install_only.tar.gz", - "urls": [ - "https://github.com/indygreg/python-build-standalone/releases/download/20240107/cpython-3.11.7+20240107-x86_64-unknown-linux-gnu-install_only.tar.gz" - ], - "distutils_content": "", - "strip_prefix": "python", - "coverage_tool": "", - "ignore_root_user_error": false - } - } - }, - "recordedRepoMappingEntries": [ - [ - "rules_python~", - "bazel_skylib", - "bazel_skylib~" - ], - [ - "rules_python~", - "bazel_tools", - "bazel_tools" - ] - ] - } - }, - "@@rules_python~//python/private/bzlmod:internal_deps.bzl%internal_deps": { - "general": { - "bzlTransitiveDigest": "ZKHEOaFX4G/WohTzMTV688TfCsSfLvbwBC2ma87N24w=", - "usagesDigest": "4Fj9JSpEDoJSLPRSbvSTol2bTL7baZjuA3k9U7kG/1k=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "pypi__wheel": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/b8/8b/31273bf66016be6ad22bb7345c37ff350276cfd46e389a0c2ac5da9d9073/wheel-0.41.2-py3-none-any.whl", - "sha256": "75909db2664838d015e3d9139004ee16711748a52c8f336b52882266540215d8", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:defs.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude in /python/pip_install/tools/bazel.py\n # to avoid non-determinism following pip install's behavior.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/* *\",\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__click": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/00/2e/d53fa4befbf2cfa713304affc7ca780ce4fc1fd8710527771b58311a3229/click-8.1.7-py3-none-any.whl", - "sha256": "ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:defs.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude in /python/pip_install/tools/bazel.py\n # to avoid non-determinism following pip install's behavior.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/* *\",\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__importlib_metadata": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/cc/37/db7ba97e676af155f5fcb1a35466f446eadc9104e25b83366e8088c9c926/importlib_metadata-6.8.0-py3-none-any.whl", - "sha256": "3ebb78df84a805d7698245025b975d9d67053cd94c79245ba4b3eb694abe68bb", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:defs.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude in /python/pip_install/tools/bazel.py\n # to avoid non-determinism following pip install's behavior.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/* *\",\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__pyproject_hooks": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/d5/ea/9ae603de7fbb3df820b23a70f6aff92bf8c7770043254ad8d2dc9d6bcba4/pyproject_hooks-1.0.0-py3-none-any.whl", - "sha256": "283c11acd6b928d2f6a7c73fa0d01cb2bdc5f07c57a2eeb6e83d5e56b97976f8", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:defs.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude in /python/pip_install/tools/bazel.py\n # to avoid non-determinism following pip install's behavior.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/* *\",\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__pep517": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/ee/2f/ef63e64e9429111e73d3d6cbee80591672d16f2725e648ebc52096f3d323/pep517-0.13.0-py3-none-any.whl", - "sha256": "4ba4446d80aed5b5eac6509ade100bff3e7943a8489de249654a5ae9b33ee35b", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:defs.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude in /python/pip_install/tools/bazel.py\n # to avoid non-determinism following pip install's behavior.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/* *\",\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__packaging": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/ab/c3/57f0601a2d4fe15de7a553c00adbc901425661bf048f2a22dfc500caf121/packaging-23.1-py3-none-any.whl", - "sha256": "994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:defs.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude in /python/pip_install/tools/bazel.py\n # to avoid non-determinism following pip install's behavior.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/* *\",\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__pip_tools": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/e8/df/47e6267c6b5cdae867adbdd84b437393e6202ce4322de0a5e0b92960e1d6/pip_tools-7.3.0-py3-none-any.whl", - "sha256": "8717693288720a8c6ebd07149c93ab0be1fced0b5191df9e9decd3263e20d85e", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:defs.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude in /python/pip_install/tools/bazel.py\n # to avoid non-determinism following pip install's behavior.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/* *\",\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__setuptools": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/4f/ab/0bcfebdfc3bfa8554b2b2c97a555569c4c1ebc74ea288741ea8326c51906/setuptools-68.1.2-py3-none-any.whl", - "sha256": "3d8083eed2d13afc9426f227b24fd1659489ec107c0e86cec2ffdde5c92e790b", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:defs.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude in /python/pip_install/tools/bazel.py\n # to avoid non-determinism following pip install's behavior.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/* *\",\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__zipp": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/8c/08/d3006317aefe25ea79d3b76c9650afabaf6d63d1c8443b236e7405447503/zipp-3.16.2-py3-none-any.whl", - "sha256": "679e51dd4403591b2d6838a48de3d283f3d188412a9782faadf845f298736ba0", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:defs.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude in /python/pip_install/tools/bazel.py\n # to avoid non-determinism following pip install's behavior.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/* *\",\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__colorama": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", - "sha256": "4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:defs.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude in /python/pip_install/tools/bazel.py\n # to avoid non-determinism following pip install's behavior.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/* *\",\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__build": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/58/91/17b00d5fac63d3dca605f1b8269ba3c65e98059e1fd99d00283e42a454f0/build-0.10.0-py3-none-any.whl", - "sha256": "af266720050a66c893a6096a2f410989eeac74ff9a68ba194b3f6473e8e26171", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:defs.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude in /python/pip_install/tools/bazel.py\n # to avoid non-determinism following pip install's behavior.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/* *\",\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "rules_python_internal": { - "bzlFile": "@@rules_python~//python/private:internal_config_repo.bzl", - "ruleClassName": "internal_config_repo", - "attributes": {} - }, - "pypi__pip": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/50/c2/e06851e8cc28dcad7c155f4753da8833ac06a5c704c109313b8d5a62968a/pip-23.2.1-py3-none-any.whl", - "sha256": "7ccf472345f20d35bdc9d1841ff5f313260c2c33fe417f48c30ac46cccabf5be", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:defs.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude in /python/pip_install/tools/bazel.py\n # to avoid non-determinism following pip install's behavior.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/* *\",\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__installer": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/e5/ca/1172b6638d52f2d6caa2dd262ec4c811ba59eee96d54a7701930726bce18/installer-0.7.0-py3-none-any.whl", - "sha256": "05d1933f0a5ba7d8d6296bb6d5018e7c94fa473ceb10cf198a92ccea19c27b53", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:defs.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude in /python/pip_install/tools/bazel.py\n # to avoid non-determinism following pip install's behavior.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/* *\",\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__more_itertools": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/5a/cb/6dce742ea14e47d6f565589e859ad225f2a5de576d7696e0623b784e226b/more_itertools-10.1.0-py3-none-any.whl", - "sha256": "64e0735fcfdc6f3464ea133afe8ea4483b1c5fe3a3d69852e6503b43a0b222e6", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:defs.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude in /python/pip_install/tools/bazel.py\n # to avoid non-determinism following pip install's behavior.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/* *\",\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - }, - "pypi__tomli": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "url": "https://files.pythonhosted.org/packages/97/75/10a9ebee3fd790d20926a90a2547f0bf78f371b2f13aa822c759680ca7b9/tomli-2.0.1-py3-none-any.whl", - "sha256": "939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc", - "type": "zip", - "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:defs.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude in /python/pip_install/tools/bazel.py\n # to avoid non-determinism following pip install's behavior.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/* *\",\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" - } - } - }, - "recordedRepoMappingEntries": [ - [ - "rules_python~", - "bazel_tools", - "bazel_tools" - ] - ] - } - }, - "@@rules_swift~//swift:extensions.bzl%non_module_deps": { - "general": { - "bzlTransitiveDigest": "/runDIrzIkL0JBXKKPK9sUS1rQUQZ6IhXHHYnThEwnA=", - "usagesDigest": "dic5Qhy52rBP0Rzbez+t0M7mJxiALIIz7QKxysCHnug=", - "recordedFileInputs": {}, - "recordedDirentsInputs": {}, - "envVariables": {}, - "generatedRepoSpecs": { - "com_github_grpc_grpc_swift": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "urls": [ - "https://github.com/grpc/grpc-swift/archive/1.16.0.tar.gz" - ], - "sha256": "58b60431d0064969f9679411264b82e40a217ae6bd34e17096d92cc4e47556a5", - "strip_prefix": "grpc-swift-1.16.0/", - "build_file": "@@rules_swift~//third_party:com_github_grpc_grpc_swift/BUILD.overlay" - } - }, - "com_github_apple_swift_nio_extras": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-nio-extras/archive/1.4.0.tar.gz" - ], - "sha256": "4684b52951d9d9937bb3e8ccd6b5daedd777021ef2519ea2f18c4c922843b52b", - "strip_prefix": "swift-nio-extras-1.4.0/", - "build_file": "@@rules_swift~//third_party:com_github_apple_swift_nio_extras/BUILD.overlay" - } - }, - "com_github_apple_swift_protobuf": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-protobuf/archive/1.20.2.tar.gz" - ], - "sha256": "3fb50bd4d293337f202d917b6ada22f9548a0a0aed9d9a4d791e6fbd8a246ebb", - "strip_prefix": "swift-protobuf-1.20.2/", - "build_file": "@@rules_swift~//third_party:com_github_apple_swift_protobuf/BUILD.overlay" - } - }, - "com_github_apple_swift_nio_ssl": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-nio-ssl/archive/2.23.0.tar.gz" - ], - "sha256": "4787c63f61dd04d99e498adc3d1a628193387e41efddf8de19b8db04544d016d", - "strip_prefix": "swift-nio-ssl-2.23.0/", - "build_file": "@@rules_swift~//third_party:com_github_apple_swift_nio_ssl/BUILD.overlay" - } - }, - "com_github_apple_swift_atomics": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-atomics/archive/1.1.0.tar.gz" - ], - "sha256": "1bee7f469f7e8dc49f11cfa4da07182fbc79eab000ec2c17bfdce468c5d276fb", - "strip_prefix": "swift-atomics-1.1.0/", - "build_file": "@@rules_swift~//third_party:com_github_apple_swift_atomics/BUILD.overlay" - } - }, - "com_github_apple_swift_nio_http2": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-nio-http2/archive/1.26.0.tar.gz" - ], - "sha256": "f0edfc9d6a7be1d587e5b403f2d04264bdfae59aac1d74f7d974a9022c6d2b25", - "strip_prefix": "swift-nio-http2-1.26.0/", - "build_file": "@@rules_swift~//third_party:com_github_apple_swift_nio_http2/BUILD.overlay" - } - }, - "com_github_apple_swift_nio_transport_services": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-nio-transport-services/archive/1.15.0.tar.gz" - ], - "sha256": "f3498dafa633751a52b9b7f741f7ac30c42bcbeb3b9edca6d447e0da8e693262", - "strip_prefix": "swift-nio-transport-services-1.15.0/", - "build_file": "@@rules_swift~//third_party:com_github_apple_swift_nio_transport_services/BUILD.overlay" - } - }, - "build_bazel_rules_swift_index_import": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "build_file": "@@rules_swift~//third_party:build_bazel_rules_swift_index_import/BUILD.overlay", - "canonical_id": "index-import-5.8", - "urls": [ - "https://github.com/MobileNativeFoundation/index-import/releases/download/5.8.0.1/index-import.tar.gz" - ], - "sha256": "28c1ffa39d99e74ed70623899b207b41f79214c498c603915aef55972a851a15" - } - }, - "com_github_apple_swift_nio": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-nio/archive/2.42.0.tar.gz" - ], - "sha256": "e3304bc3fb53aea74a3e54bd005ede11f6dc357117d9b1db642d03aea87194a0", - "strip_prefix": "swift-nio-2.42.0/", - "build_file": "@@rules_swift~//third_party:com_github_apple_swift_nio/BUILD.overlay" - } - }, - "com_github_apple_swift_log": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-log/archive/1.4.4.tar.gz" - ], - "sha256": "48fe66426c784c0c20031f15dc17faf9f4c9037c192bfac2f643f65cb2321ba0", - "strip_prefix": "swift-log-1.4.4/", - "build_file": "@@rules_swift~//third_party:com_github_apple_swift_log/BUILD.overlay" - } - }, - "com_github_apple_swift_collections": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "urls": [ - "https://github.com/apple/swift-collections/archive/1.0.4.tar.gz" - ], - "sha256": "d9e4c8a91c60fb9c92a04caccbb10ded42f4cb47b26a212bc6b39cc390a4b096", - "strip_prefix": "swift-collections-1.0.4/", - "build_file": "@@rules_swift~//third_party:com_github_apple_swift_collections/BUILD.overlay" - } - }, - "build_bazel_rules_swift_local_config": { - "bzlFile": "@@rules_swift~//swift/internal:swift_autoconfiguration.bzl", - "ruleClassName": "swift_autoconfiguration", - "attributes": {} - } - }, - "recordedRepoMappingEntries": [ - [ - "rules_swift~", - "bazel_tools", - "bazel_tools" - ], - [ - "rules_swift~", - "build_bazel_rules_swift", - "rules_swift~" - ] - ] - } - } - } -} diff --git a/examples/benchmark/BUILD.bazel b/examples/benchmark/BUILD.bazel index 7ca74ada..72a2dc6c 100644 --- a/examples/benchmark/BUILD.bazel +++ b/examples/benchmark/BUILD.bazel @@ -1,11 +1,13 @@ -load("@zml//bazel:zig.bzl", "zig_cc_binary") +load("@bazel_skylib//rules:build_test.bzl", "build_test") +load("@rules_zig//zig:defs.bzl", "zig_binary") -zig_cc_binary( +zig_binary( name = "benchmark", main = "main.zig", - deps = [ - "@zml//async", - "@zml//zml", - "//third_party/tigerbeetle:flags", - ], + deps = ["//zml"], +) + +build_test( + name = "test", + targets = [":benchmark"], ) diff --git a/examples/benchmark/main.zig b/examples/benchmark/main.zig index 7b17b476..7ff70707 100644 --- a/examples/benchmark/main.zig +++ b/examples/benchmark/main.zig @@ -1,187 +1,123 @@ const std = @import("std"); +const log = std.log; + const zml = @import("zml"); -const asynk = @import("async"); -const flags = @import("tigerbeetle/flags"); +const stdx = zml.stdx; -const async_ = asynk.async_; - -// set log level to debug to print the generated IR -pub const std_options = .{ - .log_level = .debug, -}; - -/// Model definition -const Benchmark = struct { - pub fn forward(self: Benchmark, a: zml.Tensor, b: zml.Tensor) zml.Tensor { - _ = self; - return a.dot(b, .{.k}).withSharding(.{.n}); - } -}; - -pub fn main() !void { - var gpa = std.heap.GeneralPurposeAllocator(.{}){}; - defer _ = gpa.deinit(); - try asynk.AsyncThread.main(gpa.allocator(), asyncMain, .{}); +pub fn benchmark(a: zml.Tensor, b: zml.Tensor) zml.Tensor { + return a.dot(b, .k).withPartitioning(.benchmark_mesh, .{ .m = .m, .n = .replicated }); } -pub fn asyncMain() !void { +pub fn main(init: std.process.Init) !void { const CliArgs = struct { pub const help = \\ benchmark --size=4096 --dtype=f16 ; size: usize = 4096, - batch_size: usize = 2, dtype: zml.DataType = .f16, - iterations: usize = 1, - sharded: bool = false, }; - // Short lived allocations - var gpa = std.heap.GeneralPurposeAllocator(.{}){}; - defer _ = gpa.deinit(); - const allocator = gpa.allocator(); - - // Arena allocator for BufferStore etc. - var arena_state = std.heap.ArenaAllocator.init(allocator); - defer arena_state.deinit(); - const arena = arena_state.allocator(); - - var args = std.process.args(); - const cli_args = flags.parse(&args, CliArgs); - - var context = try zml.Context.init(); - defer context.deinit(); + const allocator = init.gpa; + const io = init.io; // Auto-select platform - const platform = context.autoPlatform().withCompilationOptions(.{ - .sharding_enabled = cli_args.sharded, - }); + const platform: *zml.Platform = try .auto(allocator, io, .{}); + defer platform.deinit(allocator, io); - { - // List available targets - std.debug.print("Available Platforms:\n", .{}); - const selected_prefix = "✅"; - const not_selected_prefix = "• "; - const selected_postfix = "(AUTO-SELECTED)\n"; - const not_selected_postfix = "\n"; - for (zml.platform.available_targets) |target| { - std.debug.print(" {s} {s} {s}", .{ - if (target == platform.target) selected_prefix else not_selected_prefix, - @tagName(target), - if (target == platform.target) selected_postfix else not_selected_postfix, - }); + log.info("\n{f}", .{platform.fmtVerbose()}); - // now the platform's devices - if (context.platforms.get(target)) |pfm| { - for (pfm.getDevices(), 0..) |device, index| { - const deviceKind = device.getDescription(platform.pjrt_api).getKind(platform.pjrt_api); - std.debug.print(" ◦ #{d}: {s}\n", .{ - index, - deviceKind, - }); - // we only list 1 CPU device - if (target == .cpu and platform.sharding().num_partitions == 1) break; - } - } - } - std.debug.print("\n", .{}); - } + const mesh: *const zml.Mesh = try platform.registerMesh("mesh", .mesh( + .{ .m = .low_bandwidth, .n = .high_bandwidth }, + )); - const a_shape = zml.Shape.init(.{ cli_args.batch_size, cli_args.size, cli_args.size }, cli_args.dtype).withTags(.{ .b, .m, .k }); - const b_shape = a_shape.withTags(.{ .b, .k, .n }).withSharding(.{.n}); - var timer = try std.time.Timer.start(); + const cli_args: CliArgs = stdx.flags.parse(init.minimal.args, CliArgs); - std.debug.print("\nCompiling model to MLIR....\n", .{}); - std.debug.print("-" ** 160 ++ "\n", .{}); - // Start compiling. - // The shape of the input tensor, we have to pass in manually. - timer.reset(); - var compilation = try async_(zml.module.compileModel, .{ allocator, Benchmark{}, .forward, .{ a_shape, b_shape }, platform }); + const a = zml.Tensor.init(.{ .m = cli_args.size, .k = cli_args.size }, cli_args.dtype) + .withPartitioning(mesh, .{ .m = .m, .k = .replicated }); + const b = zml.Tensor.init(.{ .k = cli_args.size, .n = cli_args.size }, cli_args.dtype) + .withPartitioning(mesh, .{ .k = .replicated, .n = .n }); - // Wait for compilation to finish - const compiled = try compilation.await_(); - const compilation_elapsed = timer.lap() / std.time.ns_per_ms; - std.debug.print("-" ** 160 ++ "\n\n", .{}); - std.debug.print("✅ Compiled Benchmark model in {d} milliseconds! \n", .{compilation_elapsed}); - - // pass the model weights to the compiled module to create an executable module - var executable = try compiled.prepare(arena, .{}); - defer executable.deinit(); + var exe = blk: { + log.info("⏱️ Compiling benchmark...", .{}); + const now: std.Io.Timestamp = .now(io, .awake); + defer log.info("✅ Compiled benchmark [{f}]", .{now.untilNow(io, .awake)}); + break :blk try platform.compileFn(allocator, io, benchmark, .{ a, b }, .{ .meshes = &.{mesh} }); + }; + defer exe.deinit(); var rng = std.Random.DefaultPrng.init(0); const random = rng.random(); - var a_buffer = try createRandomBuffer(allocator, platform, a_shape, random); + var a_buffer = try createRandomBuffer(allocator, io, platform, a.shape(), random); defer a_buffer.deinit(); - var b_buffer = try createRandomBuffer(allocator, platform, b_shape, random); + var b_buffer = try createRandomBuffer(allocator, io, platform, b.shape(), random); defer b_buffer.deinit(); - std.debug.print("\nRunning benchmark....\n", .{}); + var exe_args = try exe.args(allocator); + defer exe_args.deinit(allocator); + + var exe_results = try exe.results(allocator); + defer exe_results.deinit(allocator); + + exe_args.set(.{ a_buffer, b_buffer }); + + log.info("⏱️ Running benchmark...", .{}); // Ignore first run { - var result: zml.Buffer = executable.call(.{ a_buffer, b_buffer }); + exe.call(io, exe_args, &exe_results, .{}); + var result = exe_results.get(zml.Buffer); defer result.deinit(); } // call our executable module - var elapsed_ns: u64 = 0; - for (0..cli_args.iterations) |_| { - timer.reset(); - var result: zml.Buffer = executable.call(.{ a_buffer, b_buffer }); - elapsed_ns += timer.lap(); - defer result.deinit(); - } - const elapsed_ms = @as(f64, @floatFromInt(elapsed_ns)) / std.time.ns_per_ms; - const avg_elapsed_ms = elapsed_ms / @as(f64, @floatFromInt(cli_args.iterations)); + const run_start: std.Io.Timestamp = .now(io, .awake); + exe.call(io, exe_args, &exe_results, .{}); + var result = exe_results.get(zml.Buffer); + _ = try result.await(io); + defer result.deinit(); + const elapsed = run_start.untilNow(io, .awake); + const elapsed_ns = elapsed.toNanoseconds(); const elapsed_s = @as(f64, @floatFromInt(elapsed_ns)) / std.time.ns_per_s; - std.debug.print("\n✅ Benchmark done!\n\n", .{}); + log.info("✅ Benchmark done!", .{}); - const floating_op_count = 2 * cli_args.batch_size * cli_args.size * cli_args.size * cli_args.size * cli_args.iterations; + const floating_op_count = 2 * cli_args.size * cli_args.size * cli_args.size; const flops = @as(f64, @floatFromInt(floating_op_count)) / elapsed_s; - std.debug.print( - \\ Dot product size: {d}x{d} - \\ Datatype: {s} - \\ Iterations: {d} - \\ Total elapsed: {d:.3}ms (avg: {d:.3}ms/it) - \\ FLOPs: {d:.3}GFLOP/s - \\ - \\ - , - .{ cli_args.size, cli_args.size, @tagName(cli_args.dtype), cli_args.iterations, elapsed_ms, avg_elapsed_ms, flops / 1_000_000_000} - ); + log.info("Dot product size: {d}x{d} - Datatype: {s} - Elapsed: {f} - {d:.3} GFLOP/s", .{ + cli_args.size, + cli_args.size, + @tagName(cli_args.dtype), + elapsed, + flops / 1_000_000_000, + }); } -fn createRandomBuffer(allocator: std.mem.Allocator, platform: zml.Platform, shape: zml.Shape, random: std.Random) !zml.Buffer { - const data = try allocator.alloc(u8, shape.byteSize()); - defer allocator.free(data); +fn createRandomBuffer(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, shape: zml.Shape, random: std.Random) !zml.Buffer { + const slice = try zml.Slice.alloc(allocator, shape); + defer slice.free(allocator); switch (shape.dtype()) { inline else => |v| { - const ZigType = v.toZigType(); + const ZigType = v.toPackedZigType(); switch (comptime v.class()) { .bool => unreachable, .integer => { - for (std.mem.bytesAsSlice(ZigType, data)) |*e| e.* = random.int(ZigType); + for (slice.items(ZigType)) |*e| e.* = @bitCast(random.int(@Int(.unsigned, @bitSizeOf(ZigType)))); }, .float => { - const value = random.float(f64); - for (std.mem.bytesAsSlice(ZigType, data)) |*e| e.* = if (ZigType == f64) - value - else if (ZigType == f32) - @floatCast(value) - else if (ZigType == f16) - @floatCast(value) - else - @bitCast(random.int(std.meta.Int(.unsigned, @bitSizeOf(ZigType)))); + const value = random.float(f32); + for (slice.items(ZigType)) |*e| e.* = switch (ZigType) { + f64, f32 => value, + f16 => @floatCast(value), + zml.floats.Float4E2M1.Packed => .fromF32(value, -value), + inline else => |T| if (@hasDecl(T, "fromF32")) .fromF32(value) else unreachable, + }; }, .complex => unreachable, } }, } - var host_buffer = zml.HostBuffer.fromBytes(shape, data); - errdefer host_buffer.deinit(allocator); - return zml.Buffer.from(platform, host_buffer); + return .fromSlice(io, platform, slice); } diff --git a/examples/io/BUILD.bazel b/examples/io/BUILD.bazel new file mode 100644 index 00000000..d471f74d --- /dev/null +++ b/examples/io/BUILD.bazel @@ -0,0 +1,11 @@ +load("@bazel_skylib//rules:build_test.bzl", "build_test") +load("@rules_zig//zig:defs.bzl", "zig_binary") + +zig_binary( + name = "playground", + args = [], + data = [], + main = "main.zig", + deps = ["//zml"], + visibility = ["//visibility:public"], +) diff --git a/examples/io/main.zig b/examples/io/main.zig new file mode 100644 index 00000000..c9dc05ea --- /dev/null +++ b/examples/io/main.zig @@ -0,0 +1,362 @@ +const std = @import("std"); + +const zml = @import("zml"); +const stdx = zml.stdx; + +const log = std.log.scoped(.vfs); + +pub const std_options: std.Options = .{ + .log_level = .info, +}; + +const Command = enum { cat, tree, ls, cp, stat, realpath, safetensors, load }; + +// -- ls hf://openai/gpt-oss-20b@6cee5e8 +// -- ls hf://Qwen/Qwen3-235B-A22B-Instruct-2507 +// -- ls hf://meta-llama/Llama-3.1-8B-Instruct@0e9e39f +// -- tree s3://noaa-goes19/ABI-Flood-Day-Shapefiles/2025/08 +// -- cat https://iprs.fly.dev +pub fn main(init: std.process.Init) !void { + const allocator = init.gpa; + + var it = init.minimal.args.iterate(); + _ = it.next(); // skip program name + const command: Command = std.meta.stringToEnum(Command, it.next() orelse return error.MissingCommand) orelse return error.CommandInvalid; + const path = it.next() orelse return error.MissingPath; + + var http_client: std.http.Client = .{ .allocator = allocator, .io = init.io }; + + try http_client.initDefaultProxies(allocator, init.environ_map); + defer http_client.deinit(); + + var vfs_file: zml.io.VFS.File = .init(allocator, init.io, .{}); + defer vfs_file.deinit(); + + var vfs_https: zml.io.VFS.HTTP = try .init(allocator, init.io, &http_client, .https); + defer vfs_https.deinit(); + + var hf_vfs: zml.io.VFS.HF = try .auto(allocator, init.io, &http_client, init.environ_map); + defer hf_vfs.deinit(); + + var s3_vfs: zml.io.VFS.S3 = try .auto(allocator, init.io, &http_client, init.environ_map); + defer s3_vfs.deinit(); + + var gcs_vfs: zml.io.VFS.GCS = try .auto(allocator, init.io, &http_client, init.environ_map); + defer gcs_vfs.deinit(); + + var vfs: zml.io.VFS = try .init(allocator, init.io); + defer vfs.deinit(); + + try vfs.register("file", vfs_file.io()); + try vfs.register("https", vfs_https.io()); + try vfs.register("hf", hf_vfs.io()); + try vfs.register("s3", s3_vfs.io()); + try vfs.register("gs", gcs_vfs.io()); + + const io = vfs.io(); + + const buffer = try allocator.alignedAlloc(u8, .fromByteUnits(4 * 1024), 16 * 1024 * 1024); + defer allocator.free(buffer); + + var stdout_writer = std.Io.File.stdout().writer(io, buffer); + defer stdout_writer.interface.flush() catch {}; + + switch (command) { + .cat => { + var file = try std.Io.Dir.openFile(.cwd(), io, path, .{ .mode = .read_only }); + defer file.close(io); + + var reader: std.Io.File.Reader = .initStreaming(file, io, &.{}); + + const read = try reader.interface.streamRemaining(&stdout_writer.interface); + _ = try stdout_writer.interface.write("\n"); + + try stdout_writer.interface.print("Wrote {B:.2} to stdout from {s}\n", .{ read, path }); + }, + .ls, .tree => { + var dir = try std.Io.Dir.openDir(.cwd(), io, path, .{ .iterate = true }); + defer dir.close(io); + + const dir_stat = try dir.stat(io); + try stdout_writer.interface.print("{s} - {B:.2}\n", .{ path, dir_stat.size }); + + var counts: TreeCounts = .{}; + try printTree(io, &stdout_writer.interface, dir, "", if (command == .tree) 10 else 1, &counts); + try stdout_writer.interface.print("\n{d} directories, {d} files\n", .{ counts.dirs, counts.files }); + }, + .cp => { + const destination_path = it.next() orelse { + try stdout_writer.interface.print("Usage: cp \n", .{}); + return error.InvalidArgument; + }; + + const source = try std.Io.Dir.openFile(.cwd(), io, path, .{}); + defer source.close(io); + + const destination = try std.Io.Dir.createFile(.cwd(), io, destination_path, .{}); + defer destination.close(io); + + var reader: std.Io.File.Reader = .initStreaming(source, io, &.{}); + var writer = destination.writer(io, buffer); + + const read = try reader.interface.streamRemaining(&writer.interface); + try writer.interface.flush(); + + try stdout_writer.interface.print("Copied {B:.2} from {s} to {s}\n", .{ read, path, destination_path }); + }, + .stat => { + const stat = std.Io.Dir.statFile(.cwd(), io, path, .{}) catch |err| blk: { + if (err == error.IsDir) { + var dir = try std.Io.Dir.openDir(.cwd(), io, path, .{}); + defer dir.close(io); + + break :blk try dir.stat(io); + } + return err; + }; + + try stdout_writer.interface.print("{s}: {B:.2} ({s})\n", .{ path, stat.size, @tagName(stat.kind) }); + }, + .realpath => { + var dir = try std.Io.Dir.openDir(.cwd(), io, path, .{}); + defer dir.close(io); + + var real_path_buf: [256]u8 = undefined; + const len = try dir.realPath(io, &real_path_buf); + + try stdout_writer.interface.print("{s}\n", .{real_path_buf[0..len]}); + }, + .safetensors => { + var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, path); + defer registry.deinit(); + + const root = try TensorNode.init(init.arena.allocator(), ""); + + for (registry.tensors.values()) |tensor| { + var current = root; + var parts = std.mem.tokenizeScalar(u8, tensor.name, '.'); + while (parts.next()) |part| { + const gop = try current.children.getOrPut(current.allocator, part); + if (!gop.found_existing) { + gop.value_ptr.* = try TensorNode.init(init.arena.allocator(), part); + } + current = gop.value_ptr.*; + } + + current.tensor = tensor; + } + + try stdout_writer.interface.print("{s}\n", .{path}); + try printTensorTree(&stdout_writer.interface, root, "", true, true); + try stdout_writer.interface.flush(); + }, + .load => { + const MeshType = enum { replicated, sharded }; + + const mesh_type: MeshType = std.meta.stringToEnum(MeshType, it.next() orelse "sharded") orelse return error.InvalidMesheKind; + + const platform: *zml.Platform = try .auto(allocator, io, .{}); + defer platform.deinit(allocator, io); + + var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, path); + defer registry.deinit(); + + const tensor_parallel: *const zml.Mesh = try platform.registerMesh( + "tp", + .mesh(.{ .model = .high_bandwidth }), + ); + const tp_partitioning: zml.Sharding.Partitioning = .init(&.{.sharded(@intCast(tensor_parallel.resolveLogicalAxis("model").?))}); + + var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &.{tensor_parallel}); + defer store.deinit(); + + const AllTensorsModel = struct { + tensors: []zml.Tensor, + }; + + const tensor_count = registry.tensors.count(); + + const tensors = try allocator.alloc(zml.Tensor, tensor_count); + defer allocator.free(tensors); + + var load_count: usize = 0; + for (registry.tensors.keys(), registry.tensors.values()) |name, tensor| { + tensors[load_count] = switch (mesh_type) { + .replicated => store.view().createReplicatedTensor(name, null), + .sharded => if (tensor.shape.rank() > 0) + store.view().createTensor(name, null, .tp, tp_partitioning) + else + store.view().createReplicatedTensor(name, null), + }; + load_count += 1; + } + + const model: AllTensorsModel = .{ .tensors = tensors }; + + var progress = std.Progress.start(io, .{ .root_name = "zml.examples.load" }); + progress.increaseEstimatedTotalItems(load_count); + defer progress.end(); + + const now: std.Io.Timestamp = .now(io, .awake); + + var buffers = try zml.mem.bufferize(allocator, AllTensorsModel, &model); + defer { + for (buffers.tensors) |*b| b.deinit(); + allocator.free(buffers.tensors); + } + + var loader: zml.io.Loader = try .init(allocator, platform, .{ + .parallelism = 8, + .dma_chunks = 16, + .dma_chunk_size = 256 * zml.MiB, + }); + defer loader.deinit(); + + try loader.load(io, AllTensorsModel, &model, &buffers, &store, .{ .progress = &progress }); + try loader.await(io); + + const took = now.untilNow(io, .awake); + const total_bytes: u64 = loader.bytes_loaded.raw; + const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); + log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); + }, + } +} + +const TreeCounts = struct { + dirs: usize = 0, + files: usize = 0, +}; + +fn printTree(io: std.Io, writer: *std.Io.Writer, dir: std.Io.Dir, prefix: []const u8, max_depth: usize, counts: *TreeCounts) !void { + if (max_depth == 0) return; + + var entries: stdx.BoundedArray(std.Io.Dir.Entry, 1024) = .empty; + var it = dir.iterate(); + while (try it.next(io)) |entry| { + entries.appendAssumeCapacity(entry); + } + + for (entries.constSlice(), 0..) |entry, idx| { + const is_last = (idx == entries.len - 1); + const connector = if (is_last) "└── " else "├── "; + const extension = if (is_last) " " else "│ "; + + const size: u64 = switch (entry.kind) { + .file, .sym_link => blk: { + const stat = try dir.statFile(io, entry.name, .{}); + break :blk stat.size; + }, + .directory => blk: { + var sub_dir = dir.openDir(io, entry.name, .{}) catch break :blk 0; + defer sub_dir.close(io); + + const stat = try sub_dir.stat(io); + break :blk stat.size; + }, + else => 0, + }; + + try writer.print("{s}{s}{s} - {B:.2}\n", .{ prefix, connector, entry.name, size }); + + if (entry.kind == .directory) { + counts.dirs += 1; + var sub_dir = dir.openDir(io, entry.name, .{ .iterate = true }) catch continue; + defer sub_dir.close(io); + + var new_prefix_buf: [4096]u8 = undefined; + const new_prefix = std.fmt.bufPrint(&new_prefix_buf, "{s}{s}", .{ prefix, extension }) catch continue; + try printTree(io, writer, sub_dir, new_prefix, max_depth - 1, counts); + } else { + counts.files += 1; + } + } +} + +const TensorNode = struct { + name: []const u8, + children: std.StringArrayHashMapUnmanaged(*TensorNode) = .empty, + allocator: std.mem.Allocator, + tensor: ?zml.safetensors.Tensor = null, + + fn init(allocator: std.mem.Allocator, name: []const u8) !*TensorNode { + const node = try allocator.create(TensorNode); + node.* = .{ + .name = name, + .allocator = allocator, + }; + return node; + } +}; + +fn getSortedChildren(allocator: std.mem.Allocator, node: *TensorNode) ![]*TensorNode { + const children_nodes = try allocator.alloc(*TensorNode, node.children.count()); + var it = node.children.iterator(); + var idx: usize = 0; + while (it.next()) |entry| : (idx += 1) { + children_nodes[idx] = entry.value_ptr.*; + } + + std.mem.sort(*TensorNode, children_nodes, {}, struct { + fn lessThan(_: void, a: *TensorNode, b: *TensorNode) bool { + const a_num = std.fmt.parseInt(usize, a.name, 10) catch null; + const b_num = std.fmt.parseInt(usize, b.name, 10) catch null; + if (a_num != null and b_num != null) return a_num.? < b_num.?; + return std.mem.lessThan(u8, a.name, b.name); + } + }.lessThan); + return children_nodes; +} + +fn printTensorTree( + writer: anytype, + node: *TensorNode, + prefix: []const u8, + is_last: bool, + is_root: bool, +) !void { + const allocator = node.allocator; + + if (is_root) { + const children_nodes = try getSortedChildren(allocator, node); + for (children_nodes, 0..) |child, i| { + try printTensorTree(writer, child, "", i == children_nodes.len - 1, false); + } + return; + } + + var compacted_name = node.name; + var walk = node; + while (walk.children.count() == 1 and walk.tensor == null) { + const next = walk.children.values()[0]; + compacted_name = try std.fmt.allocPrint(allocator, "{s}.{s}", .{ compacted_name, next.name }); + walk = next; + } + + const connector = if (is_last) "└── " else "├── "; + try writer.print("{s}{s}{s}", .{ prefix, connector, compacted_name }); + + if (walk.tensor) |t| { + try writer.print(" [shape={f} size={B:.2}]", .{ t.shape, t.byteSize() }); + } + try writer.print("\n", .{}); + + const extension = if (is_last) " " else "│ "; + const child_prefix = try std.fmt.allocPrint(allocator, "{s}{s}", .{ prefix, extension }); + const children_nodes = try getSortedChildren(allocator, walk); + + const show_count = 2; + const skip_threshold = 8; + + for (children_nodes, 0..) |child, i| { + if (children_nodes.len > skip_threshold) { + if (i >= show_count and i < children_nodes.len - show_count) { + if (i == show_count) { + try writer.print("{s}├── ...\n", .{child_prefix}); + } + continue; + } + } + try printTensorTree(writer, child, child_prefix, i == children_nodes.len - 1, false); + } +} diff --git a/examples/llama/BUILD.bazel b/examples/llama/BUILD.bazel deleted file mode 100644 index b3ad3fb3..00000000 --- a/examples/llama/BUILD.bazel +++ /dev/null @@ -1,170 +0,0 @@ -load("@aspect_bazel_lib//lib:tar.bzl", "mtree_spec", "tar") -load("@aspect_bazel_lib//lib:transitions.bzl", "platform_transition_filegroup") -load("@bazel_skylib//rules:native_binary.bzl", "native_binary") -load("@rules_oci//oci:defs.bzl", "oci_image", "oci_load", "oci_push") -load("@zml//bazel:zig.bzl", "zig_cc_binary") -load("@rules_zig//zig:defs.bzl", "zig_library") - -zig_cc_binary( - name = "llama", - srcs = [ - "llama.zig", - ], - main = "main.zig", - deps = [ - "//third_party/tigerbeetle:flags", - "@zml//async", - "@zml//zml", - ], -) - -zig_library( - name = "llamalib", - copts = ["-lc"], - main = "llama.zig", - visibility = ["//visibility:public"], - deps = [ - "@zml//async", - "@zml//zml", - ], -) - -native_binary( - name = "Llama-3.1-8B-Instruct", - src = ":llama", - args = [ - "--model=$(location @Meta-Llama-3.1-8B-Instruct//:model.safetensors.index.json)", - "--tokenizer=$(location @Meta-Llama-3.1-8B-Instruct//:tokenizer)", - "--num-heads=32", - "--num-kv-heads=8", - "--rope-freq-base=500000", - ], - data = [ - "@Meta-Llama-3.1-8B-Instruct//:model", - "@Meta-Llama-3.1-8B-Instruct//:model.safetensors.index.json", - "@Meta-Llama-3.1-8B-Instruct//:tokenizer", - ], -) - -native_binary( - name = "OpenLLaMA-3B", - src = ":llama", - args = [ - "--model=$(location @OpenLM-Research-OpenLLaMA-3B//:model)", - "--tokenizer=$(location @OpenLM-Research-OpenLLaMA-3B//:tokenizer)", - "--num-heads=32", - "--num-kv-heads=32", - "--rope-freq-base=10000", - ], - data = [ - "@OpenLM-Research-OpenLLaMA-3B//:model", - "@OpenLM-Research-OpenLLaMA-3B//:tokenizer", - ], -) - -native_binary( - name = "TinyLlama-1.1B-Chat", - src = ":llama", - args = [ - "--model=$(location @TinyLlama-1.1B-Chat-v1.0//:model.safetensors)", - "--tokenizer=$(location @TinyLlama-1.1B-Chat-v1.0//:tokenizer)", - "--num-heads=32", - "--num-kv-heads=4", - "--rope-freq-base=10000", - ], - data = [ - "@TinyLlama-1.1B-Chat-v1.0//:model.safetensors", - "@TinyLlama-1.1B-Chat-v1.0//:tokenizer", - ], -) - -native_binary( - name = "TinyLlama-Stories-110M", - src = ":llama", - args = [ - "--model=$(location @Karpathy-TinyLlama-Stories//:stories110M)", - "--tokenizer=$(location @Karpathy-TinyLlama-Tokenizer//file)", - ], - data = [ - "@Karpathy-TinyLlama-Stories//:stories110M", - "@Karpathy-TinyLlama-Tokenizer//file", - ], -) - -native_binary( - name = "TinyLlama-Stories-15M", - src = ":llama", - args = [ - "--model=$(location @Karpathy-TinyLlama-Stories//:stories15M)", - "--tokenizer=$(location @Karpathy-TinyLlama-Tokenizer//file)", - ], - data = [ - "@Karpathy-TinyLlama-Stories//:stories15M", - "@Karpathy-TinyLlama-Tokenizer//file", - ], -) - -zig_cc_binary( - name = "test-implementation", - srcs = ["llama.zig"], - args = [ - "--model=$(location @Meta-Llama-3.1-8B-Instruct//:model.safetensors.index.json)", - "--num-heads=32", - "--num-kv-heads=8", - "--rope-freq-base=500000", - ], - data = [ - "@Meta-Llama-3.1-8B-Instruct//:model", - "@Meta-Llama-3.1-8B-Instruct//:model.safetensors.index.json", - ], - main = "test.zig", - deps = [ - "//third_party/tigerbeetle:flags", - "@zml//async", - "@zml//zml", - ], -) - -mtree_spec( - name = "mtree", - srcs = [":llama"], -) - -tar( - name = "archive", - srcs = [":llama"], - args = [ - "--options", - "zstd:compression-level=9", - ], - compress = "zstd", - mtree = ":mtree", -) - -oci_image( - name = "image_", - base = "@distroless_cc_debian12", - entrypoint = ["./{}/llama".format(package_name())], - tars = [":archive"], -) - -platform_transition_filegroup( - name = "image", - srcs = [":image_"], - target_platform = "@zml//platforms:linux_amd64", -) - -oci_load( - name = "load", - image = ":image", - repo_tags = [ - "distroless/llama:latest", - ], -) - -oci_push( - name = "push", - image = ":image", - remote_tags = ["latest"], - repository = "index.docker.io/steeve/llama", -) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig deleted file mode 100644 index c7065716..00000000 --- a/examples/llama/llama.zig +++ /dev/null @@ -1,402 +0,0 @@ -const std = @import("std"); -const testing = std.testing; - -const zml = @import("zml"); -const meta = zml.meta; -const flags = @import("tigerbeetle/flags"); - -const log = std.log.scoped(.llama); -const gguf = zml.io.gguf; -const Buffer = zml.Buffer; -const Tensor = zml.Tensor; -const ShapeOf = zml.ShapeOf; -const expectClose = zml.testing.expectClose; - -pub const LlamaOptions = struct { - gen_opts: zml.nn.SamplingStrategy, - max_seq_len: u32, - num_heads: i64, - num_kv_heads: i64, - rms_norm_eps: f32, - rope_opts: zml.nn.RopeOpts, -}; - -/// Llama architecture, using huggingface transformers naming. -/// Dimensions of activations: {.b, .s, .d} -pub const LlamaLM = struct { - lm_head: zml.nn.Linear, - model: Llama, - - // Options controlling generation - gen_opts: zml.nn.SamplingStrategy = .{}, - - pub fn init(self: *LlamaLM, options: LlamaOptions) void { - self.gen_opts = options.gen_opts; - self.model.max_seq_len = options.max_seq_len; - self.model.num_heads = options.num_heads; - self.model.num_kv_heads = options.num_kv_heads; - self.model.rope_opts = options.rope_opts; - for (self.model.layers) |*layer| { - layer.self_attn.num_heads = options.num_heads; - layer.self_attn.num_kv_heads = options.num_kv_heads; - layer.self_attn.rope_opts = options.rope_opts; - layer.input_layernorm.eps = options.rms_norm_eps; - layer.post_attention_layernorm.eps = options.rms_norm_eps; - layer.mlp.up_proj.weight = layer.mlp.up_proj.weight.withSharding(.{0}); - layer.mlp.gate_proj.weight = layer.mlp.gate_proj.weight.withSharding(.{0}); - layer.mlp.down_proj.weight = layer.mlp.down_proj.weight.withSharding(.{1}); - - layer.self_attn.q_proj.weight = layer.self_attn.q_proj.weight.withSharding(.{0}); - layer.self_attn.k_proj.weight = layer.self_attn.k_proj.weight.withSharding(.{0}); - layer.self_attn.v_proj.weight = layer.self_attn.v_proj.weight.withSharding(.{0}); - layer.self_attn.o_proj.weight = layer.self_attn.o_proj.weight.withSharding(.{1}); - } - - //zml.meta.visit(struct { - // fn cb(_: void, layer: *zml.nn.Linear) void { - // layer.weight = layer.weight.withSharding(.{-1}); - // if (layer.bias) |*b| { - // b.* = b.withSharding(.{-1}); - // } - // } - //}.cb, {}, &self.model); - self.lm_head.weight = self.lm_head.weight.withSharding(.{0}); - } - - /// Predicts the token at `token_index` position. - /// Returns: - /// - updated `tokens`, - /// - `token_idx` + 1, - /// - updated KV cache - /// - a Rng state to allow for probabilistic generation - pub fn forward( - self: LlamaLM, - tokens_: Tensor, - token_index: Tensor, - kv_cache: ?KvCache, - rng: Tensor.Rng, - ) struct { Tensor, Tensor, KvCache, Tensor.Rng } { - meta.assert(tokens_.dtype() == .i32 and tokens_.rank() >= 1 and token_index.dtype() == .i32 and token_index.rank() == 0, "Can't run Llama ! Expected >=1d tokens and 0d token_index, got: {} and {}", .{ tokens_, token_index }); - - var tokens = tokens_.withPartialTags(.{.s}); - const out, const updated_kv_cache = zml.call(self.model, .forward, .{ tokens, if (kv_cache == null) null else token_index, kv_cache }); - tokens, const new_rng = updateTokens(self.lm_head, tokens, token_index, out, rng, self.gen_opts); - return .{ tokens, increment(0, token_index), updated_kv_cache, new_rng }; - } - - pub fn updateTokens( - lm_head: zml.nn.Linear, - tokens_: Tensor, - token_index: Tensor, - out_: Tensor, - rng: Tensor.Rng, - opts: zml.nn.SamplingStrategy, - ) struct { Tensor, Tensor.Rng } { - const tokens = tokens_.withPartialTags(.{.s}); - const out = out_.withPartialTags(.{ .s, .d }); - - const next_token_pred = out.gatherValues(.s, token_index, .{}); - var logits = zml.call(lm_head, .forward, .{next_token_pred}); - if (logits.shape().hasTag(.voc) == null) - logits = logits.rename(.{ .d = .voc }); - - const next_token, const new_rng = zml.nn.sampleTokens(logits, opts, rng); - const next_token_index = token_index.addConstant(1); - const new_tokens = tokens.dynamicUpdateSlice(.{ .s = next_token_index }, next_token); - - return .{ new_tokens.reuseBuffer(tokens_), new_rng }; - } - - pub fn increment(_: u8, token_index: Tensor) Tensor { - return token_index.addConstant(1).reuseBuffer(token_index); - } - - /// Run the generation entirely within pjrt. - pub fn generate(self: LlamaLM, tokens: Tensor, token_index: Tensor, rng: Tensor.Rng) Tensor { - // Generate the first token using the prompt and generate the KV-cache initial values. - const prefill = zml.call(self, .forward, .{ tokens, token_index, null, rng }); - - const Gen = struct { - /// Same as LlamaLM.forward but without optional in the signature - pub fn forward(lm: LlamaLM, t_ids: Tensor, t_idx: Tensor, kv_cache_: KvCache, inner_rng: Tensor.Rng) struct { Tensor, Tensor, KvCache, Tensor.Rng } { - var kv_cache = kv_cache_; - kv_cache.k = kv_cache.k.withPartialTags(.{ .layer, .h, .k, .hd }); - kv_cache.v = kv_cache.v.withPartialTags(.{ .layer, .h, .k, .hd }); - return zml.call(lm, .forward, .{ t_ids._ctx, t_ids, t_idx, kv_cache, inner_rng }); - } - // / Stops when we generated `max_seq_len` tokens. - pub fn shouldContinue(lm: LlamaLM, t_ids: Tensor, t_idx: Tensor, kv_cache: KvCache, inner_rng: Tensor.Rng) Tensor { - _ = kv_cache; - _ = inner_rng; - std.debug.assert(t_ids.dim(1) == lm.model.max_seq_len); - return t_idx.cmp(.LT, Tensor.scalar(t_ids._ctx, lm.model.max_seq_len, t_idx.dtype())); - } - }; - // Generate remaining tokens using the KV-cache, return tokens. - return zml.ops.while_(Gen.shouldContinue, Gen.forward, self, prefill)[0]; - } -}; - -pub const Llama = struct { - embed_tokens: zml.nn.TokenEmbedding, - norm: RmsNorm, - layers: []TransformerLayer, - - max_seq_len: u32 = 0, - num_heads: i64 = 32, - num_kv_heads: i64 = 32, - rope_opts: zml.nn.RopeOpts = .{ - .impl = .interleaved, - .freq_base = 10_000, - }, - - const Shape = struct { - s: u32, - layer: u16, - hd: u16, - nh: u16, - nkvh: u16, - dtype: zml.DataType, - }; - - pub fn shape(self: Llama) Shape { - const key_dim = self.layers[0].self_attn.k_proj.weight.dim(0); - const num_kv_heads = if (self.num_kv_heads > 0) self.num_kv_heads else self.num_heads; - - return .{ - .s = self.max_seq_len, - .layer = @intCast(self.layers.len), - .hd = @intCast(@divExact(key_dim, num_kv_heads)), - .nh = @intCast(self.num_heads), - .nkvh = @intCast(num_kv_heads), - .dtype = self.embed_tokens.weight.dtype(), - }; - } - - /// Forward one token, using KV cache for previous tokens. - /// Returns result and updated KV cache. - pub fn forward(self: Llama, tokens: Tensor, token_index: ?Tensor, kv_cache: ?KvCache) struct { Tensor, KvCache } { - const embeds = embed(self.embed_tokens, tokens, token_index); - - var hidden = embeds; - const kv_cache0 = kv_cache orelse self.initKvCache(embeds.shape()); - - var updated_kv_cache = kv_cache0; - for (self.layers, 0..) |layer, i| { - hidden, updated_kv_cache = zml.call(layer, .forward, .{ hidden, token_index, updated_kv_cache.atLayer(i) }); - hidden = hidden.withPartialTags(.{ .s, .d }); - } - // TODO: tags seem to be lost by `callFunc`. - const output = zml.call(self.norm, .forward, .{hidden.withPartialTags(.{ .s, .d })}); - - return .{ output, updated_kv_cache.reuseBuffer(kv_cache0) }; - } - - pub fn embed(embed_tokens_: zml.nn.TokenEmbedding, tokens_: Tensor, token_index: ?Tensor) Tensor { - const tokens = if (token_index) |idx| - tokens_.dynamicSlice1d(-1, 1, idx) - else - tokens_; - return zml.call(embed_tokens_, .forward, .{tokens}).withPartialTags(.{ .s, .d }); - } - - fn initKvCache(self: Llama, embed_shape: zml.Shape) KvCache { - const dims = self.shape(); - var kv_shape = embed_shape.insert(0, .{ .layer = dims.layer }).rename(.{ .s = .k }).splitAxes(.{ .d = .{ .h = dims.nkvh, .hd = dims.hd } }); - const perm = kv_shape.contiguousPerm(.{ .h, .k, .hd }); - kv_shape = kv_shape.transpose(perm.constSlice()).withSharding(.{.h}); - return KvCache.init(kv_shape); - } -}; - -pub const TransformerLayer = struct { - input_layernorm: RmsNorm, - self_attn: SelfAttn, - post_attention_layernorm: RmsNorm, - mlp: Mlp, - - pub fn forward( - self: TransformerLayer, - x0: Tensor, - token_index: ?Tensor, - kv_cache: ?KvCache, - ) struct { Tensor, KvCache } { - // Self Attention - //log.debug("TransformerLayer({}) -> {}", .{ x0, self.input_layernorm.forward(x0) }); - meta.assert(x0.rank() >= 2 and x0.shape().hasTags(.{ .s, .d }), "TransformerLayer expected input shape: {{..., .s, .d}}, received: {}", .{x0}); - - const x0_normalized = zml.call(self.input_layernorm, .forward, .{x0}); - const delta0, const updated_kv_cache = zml.call(self.self_attn, .forward, .{ x0_normalized, token_index, kv_cache }); - const x1 = x0.add(delta0); - - // Fully Connected - const x1_normalized = zml.call(self.post_attention_layernorm, .forward, .{x1}); - const x2 = zml.call(self.mlp, .forward, .{x1_normalized}).add(x1); - - return .{ x2.reuseBuffer(x0), updated_kv_cache }; - } -}; - -const RmsNorm = struct { - weight: Tensor, - eps: f32 = 1e-5, - - /// L2 normalization of input tensor along `.d` axis. - pub fn forward(self: RmsNorm, input: Tensor) Tensor { - const x = if (input.shape().isFullyTagged()) input else input.withPartialTags(.{.d}); - // upcast to improve precision - const xf32 = x.convert(.f32); - const mean = xf32.mul(xf32).mean(.d); - const rsqrt = Tensor.rsqrt(mean.addConstant(self.eps)).convert(x.dtype()); - const normalized = x.mul(rsqrt.broad(x.shape())); - - return normalized.mul(self.weight.convert(x.dtype()).withTags(.{.d}).broad(x.shape())); - } -}; - -pub const Mlp = struct { - up_proj: zml.nn.Linear, // (dim -> hidden_dim) - gate_proj: zml.nn.Linear, // (dim -> hidden_dim) - down_proj: zml.nn.Linear, // (hidden_dim -> dim) - - pub fn forward(self: Mlp, x: Tensor) Tensor { - const proj = zml.call(self.up_proj, .forward, .{x}); - var output = zml.call(self.gate_proj, .forward, .{x}); - output = output.silu().mul(proj); - return zml.call(self.down_proj, .forward, .{output}); - } -}; - -pub const SelfAttn = struct { - q_proj: zml.nn.Linear, - k_proj: zml.nn.Linear, - v_proj: zml.nn.Linear, - - o_proj: zml.nn.Linear, - num_heads: i64 = undefined, - num_kv_heads: i64 = 0, - rope_opts: zml.nn.RopeOpts = undefined, - - /// Self Attention. - /// - If token_index is set, x is assumed to be the representation of one new token, - /// and kv_cache will be read for the previous tokens. - /// - If token_index is not set, x is assumed to be the representation of all tokens - /// since the beginning of the sequence, and kv_cache won't be read. - /// In both case, kv_cache will be updated with the computed key and value. - /// x: {.b, .s, .d } -> .{.b, .s, .d} - pub fn forward( - self: SelfAttn, - x: Tensor, - token_index: ?Tensor, - kv_cache_: ?KvCache, - ) struct { Tensor, KvCache } { - // log.debug("x.shape: {}", .{x.shape()}); - const num_kv_heads = if (self.num_kv_heads > 0) self.num_kv_heads else self.num_heads; - var q = zml.call(self.q_proj, .forward, .{x}).splitAxis(-1, .{ .h = self.num_heads, .hd = .auto }).withSharding(.{.h}); - var k = zml.call(self.k_proj, .forward, .{x}).splitAxis(-1, .{ .h = num_kv_heads, .hd = .auto }).withSharding(.{.h}); - var v = zml.call(self.v_proj, .forward, .{x}).splitAxis(-1, .{ .h = num_kv_heads, .hd = .auto }).withSharding(.{.h}); - // Generate the attention mask. - const kv_cache = kv_cache_ orelse initKvCache(k.shape()); - const seq_len = kv_cache.k.dim(.k); - var attn_mask = zml.nn.causalAttnMask(.{ .q = seq_len, .k = seq_len }, x.dtype(), null); - var cos, var sin = zml.nn.ropeCosSin(.{ .s = seq_len, .hd = k.dim(.hd) }, x.dtype(), self.rope_opts); - if (token_index) |idx| { - // Note: in Pytorch it would be very inefficient to generate the full ropeCosSin and attn_mask matrices, then slice into it, - // but XLA is able to optimize this correctly. - attn_mask = attn_mask.dynamicSlice(.{ .q = .{ .start = idx, .len = 1 } }); - cos = cos.dynamicSlice(.{ .s = .{ .start = idx, .len = 1 } }); - sin = sin.dynamicSlice(.{ .s = .{ .start = idx, .len = 1 } }); - } - - // In self-attention, .s axis is used both for keys and queries. - q = zml.nn.rope(q, .{ cos, sin }, self.rope_opts); - k = zml.nn.rope(k, .{ cos, sin }, self.rope_opts); - q = q.rename(.{ .s = .q }); - k = k.rename(.{ .s = .k }); - v = v.rename(.{ .s = .k }); - - const new_kv_cache = kv_cache.update(k, v, token_index orelse Tensor.scalar(0, .i32)); - if (token_index) |_| { - meta.assert(q.dim(.q) == 1, "Expected dimension .q to be 1, got {}", .{q.dim(.q)}); - k = new_kv_cache.keys(); - v = new_kv_cache.values(); - } - - const attn_output = zml.nn.sdpa(q, k, v, .{ .attn_mask = attn_mask, .allow_cudnn = false }); - const attn = attn_output.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); - return .{ zml.call(self.o_proj, .forward, .{attn}), new_kv_cache }; - } - - fn initKvCache(key_shape: zml.Shape) KvCache { - // When we call initKvCache, we haven't renamed .s to .k yet. - var kv_shape = key_shape.insert(0, .{ .layer = 1 }).rename(.{ .s = .k }); - const perm = kv_shape.contiguousPerm(.{ .h, .k, .hd }); - kv_shape = kv_shape.transpose(perm.constSlice()); - var res = KvCache.init(kv_shape); - res.layer_index = Tensor.scalar(0, .i32); - return res; - } -}; - -pub const KvCache = struct { - k: Tensor, - v: Tensor, - layer_index: Tensor, - - pub fn init(kv_shape: zml.Shape) KvCache { - // The KV-cache is initialized with ones to detect reads of uninitialized memory. - return .{ - .k = Tensor.constant(kv_shape, kv_shape.dtype().one()).withSharding(.{.h}), - .v = Tensor.constant(kv_shape, kv_shape.dtype().one()).withSharding(.{.h}), - .layer_index = Tensor.scalar(-1, .i32), - }; - } - - pub fn initShape(kv_shape: zml.Shape) ShapeOf(KvCache) { - return .{ - .k = kv_shape, - .v = kv_shape, - .layer_index = zml.Shape.init(.{}, .i32), - }; - } - - pub fn keys(self: KvCache) Tensor { - return self.k.dynamicSlice(.{ .layer = .{ .start = self.layer_index, .len = 1 } }).squeeze(.layer); - } - - pub fn values(self: KvCache) Tensor { - return self.v.dynamicSlice(.{ .layer = .{ .start = self.layer_index, .len = 1 } }).squeeze(.layer); - } - - pub fn update(self: KvCache, new_k: Tensor, new_v: Tensor, token_index: Tensor) KvCache { - return .{ - .k = self.k.dynamicUpdateSlice( - .{ .layer = self.layer_index, .k = token_index }, - // transpose to match kv-cache layout - new_k.contiguous(.{ .h, .k, .hd }), - ).reuseBuffer(self.k), - .v = self.v.dynamicUpdateSlice( - .{ .layer = self.layer_index, .k = token_index }, - // transpose to match kv-cache layout - new_v.contiguous(.{ .h, .k, .hd }), - ).reuseBuffer(self.v), - .layer_index = self.layer_index, - }; - } - - pub fn atLayer(self: KvCache, layer_index: usize) KvCache { - return .{ - .k = self.k, - .v = self.v, - .layer_index = Tensor.scalar(layer_index, .i32), - }; - } - - pub fn reuseBuffer(self: KvCache, other: KvCache) KvCache { - return .{ - .k = self.k.reuseBuffer(other.k), - .v = self.v.reuseBuffer(other.v), - .layer_index = self.layer_index.reuseBuffer(other.layer_index), - }; - } -}; diff --git a/examples/llama/main.zig b/examples/llama/main.zig deleted file mode 100644 index 454ab06f..00000000 --- a/examples/llama/main.zig +++ /dev/null @@ -1,266 +0,0 @@ -const std = @import("std"); - -const zml = @import("zml"); -const meta = zml.meta; -const asynk = @import("async"); -const flags = @import("tigerbeetle/flags"); -const llama_mod = @import("llama.zig"); - -const async_ = asynk.async_; - -const LlamaLM = llama_mod.LlamaLM; -const Llama = llama_mod.Llama; -const KvCache = llama_mod.KvCache; -const TransformerLayer = llama_mod.TransformerLayer; -const SelfAttn = llama_mod.SelfAttn; -const Buffer = zml.Buffer; -const Tensor = zml.Tensor; -const ShapeOf = zml.ShapeOf; - -const log = std.log.scoped(.llama); - -// set this to false to disable the verbose logging -const show_mlir = true; - -pub const std_options = .{ - .log_level = .err, - .log_scope_levels = &[_]std.log.ScopeLevel{ - .{ .scope = .pjrt, .level = if (show_mlir) .debug else .err }, - .{ .scope = .zml_module, .level = if (show_mlir) .debug else .err }, - .{ .scope = .zml, .level = if (show_mlir) .debug else .err }, - .{ .scope = .llama, .level = if (show_mlir) .debug else .info }, - }, -}; - -pub fn generateText( - llama: LlamaLM, - mod_prefill: zml.module.ExeWithWeights(LlamaLM.forward), - mod: zml.module.ExeWithWeights(LlamaLM.forward), - tokenizer: zml.tokenizer.Tokenizer, - allocator: std.mem.Allocator, - seed: u128, - prompt: []const u8, -) ![]const u8 { - const prompt_tok = tokenizer.encode(allocator, prompt, .{}) catch unreachable; - log.debug("Tokenized Prompt {d}", .{prompt_tok}); - const dims = llama.model.shape(); - const max_seq_len = dims.s; - const token_buffer = try allocator.alloc(i32, @intCast(max_seq_len)); - @memset(token_buffer, 0); - for (0..prompt_tok.len) |i| { - token_buffer[i] = @intCast(prompt_tok[i]); - } - - const tracer_buffer = try allocator.alloc(u8, @intCast(max_seq_len)); - defer allocator.free(token_buffer); - defer allocator.free(tracer_buffer); - defer allocator.free(prompt_tok); - var output = std.ArrayList(u8).init(allocator); - defer output.deinit(); - - var tokens = try zml.Buffer.fromSlice(mod.platform(), .{max_seq_len}, token_buffer); - var token_index = try zml.Buffer.fromSlice(mod.platform(), .{}, &[_]i32{@intCast(prompt_tok.len - 1)}); - - var rng = try zml.Tensor.Rng.init(mod.platform(), seed); - tokens, token_index, var kv_cache, rng = mod_prefill.call(.{ tokens, token_index, null, rng }); - defer kv_cache.k.deinit(); - defer kv_cache.v.deinit(); - defer kv_cache.layer_index.deinit(); - - const tracer = zml.tools.Tracer.init("ai.zml.models.llama"); - var decode_progress = prompt_tok.len; - const output_tokens_len = max_seq_len - prompt_tok.len - 1; - - const start = std.time.microTimestamp(); - const output_freq: u8 = 1; - for (0..output_tokens_len) |i| { - //_ = i; - const frame_id = tracer.frameStart(try std.fmt.bufPrintZ(tracer_buffer, "Generate token {}/{}", .{ i + 1, output_tokens_len })); - tokens, token_index, kv_cache, rng = mod.call(.{ tokens, token_index, kv_cache, rng }); - if ((i + 1) % output_freq == 0) { - const n = output.items.len; - _ = try tokens.toHost(std.mem.sliceAsBytes(token_buffer)); - try tokenizer.decodeWithOpts(&output, @ptrCast(token_buffer[decode_progress..][0..output_freq]), .{}); - decode_progress += output_freq; - std.debug.print("{s}", .{output.items[n..]}); - tracer.frameEnd(frame_id, try std.fmt.bufPrintZ(tracer_buffer, "Decoded token {}/{} : {s}", .{ i + 1, output_tokens_len, output.items[n..] })); - } else { - tracer.frameEnd(frame_id, try std.fmt.bufPrintZ(tracer_buffer, "Generated token {}/{}", .{ i + 1, output_tokens_len })); - } - } - std.debug.print("\n", .{}); - - const n = output.items.len; - try tokenizer.decodeWithOpts(&output, @ptrCast(token_buffer[decode_progress..]), .{}); - std.debug.print("{s}\n", .{output.items[n..]}); - const end = std.time.microTimestamp(); - - const duration = zml.meta.divFloat(f64, end - start, std.time.us_per_s); - const speed = @as(f64, @floatFromInt(max_seq_len)) / duration; - log.info("✅ Generated {d} tokens in {:.3}s: {d:.3}tok/s", .{ max_seq_len, duration, speed }); - - _ = try tokens.toHost(std.mem.sliceAsBytes(token_buffer)); - const end_index = std.mem.indexOfScalar(i32, token_buffer, 128001) orelse max_seq_len; - output.clearRetainingCapacity(); - - try tokenizer.decodeWithOpts(&output, @ptrCast(token_buffer[0..end_index]), .{}); - return output.toOwnedSlice(); -} - -pub fn main() !void { - try asynk.AsyncThread.main(std.heap.c_allocator, asyncMain, .{}); -} - -pub fn asyncMain() !void { - const CliArgs = struct { - pub const help = - \\ llama --model=llama3.7B.safetensors --tokenizer=vocab.json --num_layers=2 - ; - model: []const u8, - tokenizer: ?[]const u8 = null, - layer_start: u8 = 0, - num_layers: ?u8 = null, - seq_len: u32 = 256, - topk: u32 = 2, - temperature: u32 = 1, - num_heads: ?i64 = null, - num_kv_heads: ?i64 = null, - rope_freq_base: ?i64 = null, - prompt: ?[]const u8 = null, - test_activations: ?[]const u8 = null, - seed: ?u128 = null, - }; - - log.info(" LLama was compiled with {}", .{@import("builtin").mode}); - - var gpa = std.heap.GeneralPurposeAllocator(.{ .thread_safe = true }){}; - defer _ = gpa.deinit(); - const allocator = gpa.allocator(); - - const tmp = try std.fs.openDirAbsolute("/tmp", .{}); - try tmp.makePath("zml/llama/cache"); - var context = try zml.Context.init(); - defer context.deinit(); - - const compilation_options = zml.CompilationOptions{ - //.cache_location = "/tmp/zml/llama/cache", - .xla_dump_to = "/tmp/zml/llama", - .sharding_enabled = true, - }; - - const platform = context.autoPlatform().withCompilationOptions(compilation_options); - { - // List available targets - std.debug.print("\nSupported Platforms:\n", .{}); - const selected_prefix = "✅"; - const not_selected_prefix = "• "; - const selected_postfix = "(AUTO-SELECTED)\n"; - const not_selected_postfix = "\n"; - for (zml.platform.available_targets) |target| { - std.debug.print(" {s} {s} {s}", .{ - if (target == platform.target) selected_prefix else not_selected_prefix, - @tagName(target), - if (target == platform.target) selected_postfix else not_selected_postfix, - }); - - // now the platform's devices - if (context.platforms.get(target)) |pfm| { - for (pfm.getDevices(), 0..) |device, index| { - const deviceKind = device.getDescription(platform.pjrt_api).getKind(platform.pjrt_api); - std.debug.print(" ◦ #{d}: {s}\n", .{ - index, - deviceKind, - }); - // we only list 1 CPU device - if (target == .cpu) break; - } - } - } - std.debug.print("\n", .{}); - } - - var args = std.process.args(); - const cli_args = flags.parse(&args, CliArgs); - const model_file = cli_args.model; - - var arena_state = std.heap.ArenaAllocator.init(allocator); - defer arena_state.deinit(); - const model_arena = arena_state.allocator(); - - log.info("Model file: {s}", .{model_file}); - - var ts = try zml.aio.detectFormatAndOpen(allocator, model_file); - defer ts.deinit(); - - var llama = try zml.aio.populateModel(LlamaLM, model_arena, ts); - const num_heads: i64 = cli_args.num_heads orelse ts.metadata("num_heads", .int64) orelse @panic("--num_heads is required for this model"); - const num_kv_heads: i64 = cli_args.num_kv_heads orelse ts.metadata("num_kv_heads", .int64) orelse num_heads; - - const rope_impl = if (ts.metadata("rope_impl", .string)) |val| - std.meta.stringToEnum(zml.nn.RopeOpts.Implementation, val).? - else - .sequential; - - const llama_options: llama_mod.LlamaOptions = .{ - .max_seq_len = cli_args.seq_len, - .num_kv_heads = num_kv_heads, - .num_heads = num_heads, - .gen_opts = .{ - .topk = cli_args.topk, - .temperature = @floatFromInt(cli_args.temperature), - }, - .rms_norm_eps = @floatCast(ts.metadata("rms_norm_eps", .float64) orelse 1e-5), - .rope_opts = .{ - .impl = rope_impl, - .freq_base = @floatCast(ts.metadata("rope_freq_base", .float64) orelse @as(f32, @floatFromInt(cli_args.rope_freq_base orelse 10_000))), - }, - }; - log.info("✅ Parsed llama config: {}", .{llama_options}); - llama.init(llama_options); - - if (cli_args.tokenizer == null and !std.mem.endsWith(u8, cli_args.model, ".gguf")) { - log.err("Model doesn't have an embbedded tokenizer, please provide a path to a tokenizer.", .{}); - @panic("No tokenizer provided"); - } - const tokenizer_path = cli_args.tokenizer orelse cli_args.model; - log.info(" Loading tokenizer from {s}", .{tokenizer_path}); - var tokenizer = try zml.aio.detectFormatAndLoadTokenizer(allocator, tokenizer_path); - log.info("✅ Loaded tokenizer from {s}", .{tokenizer_path}); - defer tokenizer.deinit(); - - const dims = llama.model.shape(); - const dtype = llama.lm_head.weight.dtype(); - - // Note: we compile the model without a batching dimension. - // To do so, we would just need to add `.b = batch_size` to `token_shape` and `kv_shape`. - const tokens_shape = zml.Shape.init(.{ .s = dims.s }, .i32); - const token_idx_shape = zml.Shape.init(.{}, .i32); - const kv_shape = zml.Shape.init(.{ .layer = llama.model.layers.len, .h = dims.nkvh, .k = dims.s, .hd = dims.hd }, dtype).withSharding(.{.h}); - // needs to be optional - const kv_cache_shape: ?ShapeOf(KvCache) = KvCache.initShape(kv_shape); - const rng_shape = Tensor.Rng.shape(); - - const compile_start = std.time.milliTimestamp(); - var fut_mod_prefill = try async_(zml.compile, .{ allocator, LlamaLM, .{llama_options}, .forward, .{ tokens_shape, token_idx_shape, null, rng_shape }, ts, platform }); - var fut_mod = try async_(zml.compile, .{ allocator, LlamaLM, .{llama_options}, .forward, .{ tokens_shape, token_idx_shape, kv_cache_shape, rng_shape }, ts, platform }); - - log.info("Starting loading weights", .{}); - var llama_weights = try zml.aio.loadBuffers(LlamaLM, .{llama_options}, ts, model_arena, platform); - defer zml.aio.unloadBuffers(&llama_weights); - log.info("✅ Done loading weights", .{}); - log.info("✅ Llama model loaded from {s}", .{cli_args.model}); - - var llama_module_prefill = try (try fut_mod_prefill.await_()).prepare(allocator, llama_weights); - defer llama_module_prefill.deinit(); - var llama_module = try (try fut_mod.await_()).prepare(allocator, llama_weights); - defer llama_module.deinit(); - const compile_end = std.time.milliTimestamp(); - log.info("✅ Compiled model in {d} milliseconds! \n", .{compile_end - compile_start}); - - const prompt = cli_args.prompt orelse "Once upon a time, there was a little girl named Lily."; - log.info("✅ Prompt: {s}\n", .{prompt}); - - const seed = cli_args.seed orelse @as(u128, @bitCast(std.time.nanoTimestamp())); - const story = try generateText(llama, llama_module_prefill, llama_module, tokenizer, allocator, seed, prompt); - defer allocator.free(story); -} diff --git a/examples/llama/test.zig b/examples/llama/test.zig deleted file mode 100644 index 0c9a35ea..00000000 --- a/examples/llama/test.zig +++ /dev/null @@ -1,126 +0,0 @@ -const std = @import("std"); - -const zml = @import("zml"); -const asynk = @import("async"); -const flags = @import("tigerbeetle/flags"); - -const llama_mod = @import("./llama.zig"); -const LlamaLM = llama_mod.LlamaLM; - -const Tensor = zml.Tensor; - -pub fn main() !void { - try asynk.AsyncThread.main(std.heap.c_allocator, asyncMain, .{}); -} - -pub fn asyncMain() !void { - const CliArgs = struct { - pub const help = - \\ test-implementation --model=llama3.8B.safetensors --reference=activation.safetensors - ; - model: []const u8, - reference: []const u8, - num_heads: ?i64 = null, - num_kv_heads: ?i64 = null, - rope_freq_base: ?i64 = null, - }; - var gpa = std.heap.GeneralPurposeAllocator(.{ .thread_safe = true }){}; - defer _ = gpa.deinit(); - const allocator = gpa.allocator(); - - // Create ZML context - var context = try zml.Context.init(); - defer context.deinit(); - - // Select platform - const platform = context.autoPlatform(); - - // Parse program args - var args = std.process.args(); - const cli_args = flags.parse(&args, CliArgs); - const model_file = cli_args.model; - - // Memory arena dedicated to model shapes and weights - var arena_state = std.heap.ArenaAllocator.init(allocator); - defer arena_state.deinit(); - const model_arena = arena_state.allocator(); - - std.log.info("Model file: {s}", .{model_file}); - - // Read model shapes. - var buffer_store = try zml.aio.detectFormatAndOpen(allocator, model_file); - defer buffer_store.deinit(); - - // Create the model and configure it. - var llama = try zml.aio.populateModel(LlamaLM, model_arena, buffer_store); - const num_heads: i64 = cli_args.num_heads orelse buffer_store.metadata("num_heads", .int64) orelse @panic("--num_heads is required for this model"); - const num_kv_heads: i64 = cli_args.num_kv_heads orelse buffer_store.metadata("num_kv_heads", .int64) orelse num_heads; - - const rope_impl = if (buffer_store.metadata("rope_impl", .string)) |val| - std.meta.stringToEnum(zml.nn.RopeOpts.Implementation, val).? - else - .sequential; - - const llama_options: llama_mod.LlamaOptions = .{ - .max_seq_len = 256, - .num_kv_heads = num_kv_heads, - .num_heads = num_heads, - .gen_opts = .{}, - .rms_norm_eps = @floatCast(buffer_store.metadata("rms_norm_eps", .float64) orelse 1e-5), - .rope_opts = .{ - .impl = rope_impl, - .freq_base = @floatCast(buffer_store.metadata("rope_freq_base", .float64) orelse @as(f32, @floatFromInt(cli_args.rope_freq_base orelse 10_000))), - }, - }; - std.log.info("Parsed llama config: {}", .{llama_options}); - llama.init(llama_options); - - // Load the weights. - var llama_weights = try zml.aio.loadBuffers(LlamaLM, .{llama_options}, buffer_store, model_arena, platform); - defer zml.aio.unloadBuffers(&llama_weights); - - // Load the activations. - var activation_buffer_store = try zml.aio.torch.open(allocator, cli_args.reference); - defer activation_buffer_store.deinit(); - - // Test implementation - try testImplementation(platform, llama, llama_weights, activation_buffer_store); -} - -fn testImplementation( - platform: zml.Platform, - llama: LlamaLM, - llama_weights: zml.Bufferized(LlamaLM), - buffer_store: zml.aio.BufferStore, -) !void { - try zml.testing.testLayer(platform, buffer_store, "embed_tokens", llama.model.embed_tokens, llama_weights.model.embed_tokens, 1e-3); - try zml.testing.testLayer(platform, buffer_store, "layers.0.self_attn.v_proj", llama.model.layers[0].self_attn.v_proj, llama_weights.model.layers[0].self_attn.v_proj, 1e-2); - try zml.testing.testLayer(platform, buffer_store, "layers.0.self_attn.q_proj", llama.model.layers[0].self_attn.q_proj, llama_weights.model.layers[0].self_attn.q_proj, 2e-2); - try zml.testing.testLayer(platform, buffer_store, "layers.0.self_attn.k_proj", llama.model.layers[0].self_attn.k_proj, llama_weights.model.layers[0].self_attn.k_proj, 2e-2); - try zml.testing.testLayer(platform, buffer_store, "layers.0.self_attn.o_proj", llama.model.layers[0].self_attn.o_proj, llama_weights.model.layers[0].self_attn.o_proj, 2e-2); - try zml.testing.testLayer(platform, buffer_store, "layers.0.mlp", llama.model.layers[0].mlp, llama_weights.model.layers[0].mlp, 1e-2); - try zml.testing.testLayer(platform, buffer_store, "layers.0.input_layernorm", llama.model.layers[0].input_layernorm, llama_weights.model.layers[0].input_layernorm, 1e-2); - try zml.testing.testLayer(platform, buffer_store, "layers.0.post_attention_layernorm", llama.model.layers[0].post_attention_layernorm, llama_weights.model.layers[0].post_attention_layernorm, 1e-2); - - { - const test_case = "layers.0.self_attn"; - std.log.info("Testing {s}", .{test_case}); - // Small wrapper to explicitly tag the input, and ignore the extra arguments used in HF implementation. - const SelfAttnPrefill = struct { - inner: llama_mod.SelfAttn, - - pub fn forward(self: @This(), x_: Tensor) struct { Tensor, llama_mod.KvCache } { - return self.inner.forward(x_.withTags(.{ .b, .s, .d }), null, null); - } - }; - - try zml.testing.testLayer( - platform, - buffer_store, - "layers.0.self_attn", - SelfAttnPrefill{ .inner = llama.model.layers[0].self_attn }, - .{ .inner = llama_weights.model.layers[0].self_attn }, - 1e-3, - ); - } -} diff --git a/examples/llm/BUILD.bazel b/examples/llm/BUILD.bazel new file mode 100644 index 00000000..6a501aea --- /dev/null +++ b/examples/llm/BUILD.bazel @@ -0,0 +1,117 @@ +load("@bazel_lib//lib:transitions.bzl", "platform_transition_filegroup") +load("@bazel_skylib//rules:build_test.bzl", "build_test") +load("@rules_oci//oci:defs.bzl", "oci_image", "oci_load", "oci_push") +load("@rules_zig//zig:defs.bzl", "zig_binary") +load("@tar.bzl//tar:mtree.bzl", "mtree_spec") +load("@tar.bzl//tar:tar.bzl", "tar") + +zig_binary( + name = "llm", + srcs = glob(["**/*.zig"]), + main = "main.zig", + visibility = ["//visibility:public"], + deps = ["//zml", "@linenoise"], +) + +zig_binary( + name = "lfm2_tests", + srcs = glob(["**/*.zig"]), + main = "models/lfm2_tests.zig", + visibility = ["//visibility:public"], + deps = ["//zml"], +) + +zig_binary( + name = "llama_tests", + srcs = glob(["**/*.zig"]), + main = "models/llama_tests.zig", + visibility = ["//visibility:public"], + deps = ["//zml"], +) + +build_test( + name = "test", + targets = [":llm"], +) + +mtree_spec( + name = "mtree", + srcs = [":llm"], + tags = ["manual"] +) + +tar( + name = "archive", + srcs = [":llm"], + args = [ + "--options", + ",".join([ + "zstd:compression-level=9", + "zstd:threads=16", + ]), + ], + compress = "zstd", + mtree = ":mtree", + tags = ["manual"] +) + +oci_image( + name = "image_", + base = "@distroless_cc_debian12", + entrypoint = ["./{}/llm".format(package_name())], + target_compatible_with = [ + "@platforms//os:linux", + "@platforms//cpu:x86_64", + ], + tars = [":archive"], + tags = ["manual"] +) + +platform_transition_filegroup( + name = "image", + srcs = [":image_"], + tags = ["manual"], + target_platform = "//platforms:linux_amd64", +) + +oci_load( + name = "load", + image = ":image", + repo_tags = ["zmlai/llm:latest"], + tags = ["manual"], +) + +oci_push( + name = "push", + image = ":image", + remote_tags = ["latest"], + repository = "index.docker.io/zmlai/llm", + tags = ["manual"], +) + + +oci_image( + name = "image_debug_", + base = "@debian_12", + entrypoint = ["./examples/llm/llm"], + target_compatible_with = [ + "@platforms//os:linux", + "@platforms//cpu:x86_64", + ], + tars = [":archive"], + tags = ["manual"], +) + +platform_transition_filegroup( + name = "image_debug", + srcs = [":image_debug_"], + tags = ["manual"], + target_platform = "//platforms:linux_amd64", +) + +oci_load( + name = "load_debug", + image = ":image_debug", + repo_tags = ["zmlai/llm:debug"], + tags = ["manual"], +) diff --git a/examples/llm/README.md b/examples/llm/README.md new file mode 100644 index 00000000..fcd061ba --- /dev/null +++ b/examples/llm/README.md @@ -0,0 +1,41 @@ +# LLM + +`//examples/llm` runs interactive or one-shot text generation from a model repository from HuggingFace. + +We support the following models, automatically detected from the `model_type` in the `config.json`: + +- Llama 3.1 +- Qwen 3.5 +- LFM 2.5 + +## Run + +To load a model from HuggingFace directly: + +```bash +# CPU +bazel run //examples/llm -- --model=hf://meta-llama/Llama-3.1-8B-Instruct +# CUDA +bazel run //examples/llm --@zml//platforms:cuda=true -- --model=hf://meta-llama/Llama-3.1-8B-Instruct +# ROCm +bazel run //examples/llm --@zml//platforms:rocm=true -- --model=hf://meta-llama/Llama-3.1-8B-Instruct +``` + +From a local directory: + +```bash +bazel run //examples/llm --@zml//platforms:cuda=true -- --model=/var/models/meta-llama/Llama-3.1-8B-Instruct/ +``` + +For a single non-interactive prompt: + +```bash +bazel run //examples/llm --@zml//platforms:cuda=true -- --model=hf://meta-llama/Llama-3.1-8B-Instruct --prompt="Write a haiku about Zig" +``` + +## Options + +- `--model=`: Required. Model repository to load. This can be a local path or a huggingface/S3 URI such as `hf://...` or `s3://...`. +- `--prompt=`: Optional. Runs a single prompt instead of opening the interactive chat loop. +- `--seqlen=`: Optional. Maximum sequence length. Defaults to `2048`. +- `--backend=`: Optional. Attention backend. `cute` uses CuTe flash attention on CUDA GPUs with compute capability 8.0 or newer. If omitted, the program auto-selects one for the current platform, including `cute` on supported CUDA GPUs. diff --git a/examples/llm/chat.zig b/examples/llm/chat.zig new file mode 100644 index 00000000..c02efd56 --- /dev/null +++ b/examples/llm/chat.zig @@ -0,0 +1,176 @@ +const std = @import("std"); + +const c = @import("c"); +const zml = @import("zml"); + +const models = @import("models.zig"); + +const log = std.log.scoped(.llm); + +pub const prompt_prefix: [:0]const u8 = "> "; +pub const history_file_path: [:0]const u8 = ".llm_chat_history"; + +pub fn initHistory() void { + _ = c.linenoiseHistorySetMaxLen(1_000); + _ = c.linenoiseHistoryLoad(history_file_path.ptr); +} + +pub fn rememberPrompt(line: [*:0]const u8) void { + if (line[0] == 0) return; + if (c.linenoiseHistoryAdd(line) == 1) { + _ = c.linenoiseHistorySave(history_file_path.ptr); + } +} + +pub const Chat = struct { + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + session: models.Session, + tokens: std.ArrayList(u32), + + pub fn init( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + tokenizer: zml.tokenizer.Tokenizer, + compiled_model: *models.CompiledModel, + model_buffers: *models.Buffers, + ) !Chat { + var tokens: std.ArrayList(u32) = try .initCapacity(allocator, compiled_model.seqlen); + errdefer tokens.deinit(allocator); + + const session = try compiled_model.newSession( + allocator, + io, + platform, + model_buffers, + tokenizer, + ); + + return .{ + .allocator = allocator, + .io = io, + .platform = platform, + .session = session, + .tokens = tokens, + }; + } + + pub fn deinit(self: *Chat) void { + self.tokens.deinit(self.allocator); + self.session.deinit(); + } + + pub const RunOnceOptions = struct { + profile: bool = false, + }; + + pub fn runOnce(self: *Chat, prompt: []const u8, opts: RunOnceOptions) !void { + const prompt_tokens = try self.session.tokenizePrompt(self.allocator, prompt); + defer self.allocator.free(prompt_tokens); + + var stdout = std.Io.File.stdout().writerStreaming(self.io, &.{}); + var profiler: ?zml.Platform.Profiler = null; + defer if (profiler) |*p| p.deinit(); + + if (opts.profile) { + var session_id_buf: [64]u8 = undefined; + const session_id = try std.fmt.bufPrint( + &session_id_buf, + "llm-runonce-{d}", + .{@as(u64, @intCast(std.Io.Clock.now(.real, self.io).toNanoseconds()))}, + ); + profiler = try self.platform.profiler(self.allocator, self.io, .{ + .session_id = session_id, + }); + try profiler.?.start(); + } + + try self.runPrefill(prompt_tokens, &stdout.interface); + try self.runDecodeTurn(&stdout.interface); + _ = if (profiler) |*p| try p.stop() else null; + + try stdout.interface.flush(); + } + + pub fn runInteractive(self: *Chat, initial_prompt: []const u8) !void { + var turn_tokens = try self.session.tokenizePrompt(self.allocator, initial_prompt); + defer self.allocator.free(turn_tokens); + + var stdout = std.Io.File.stdout().writerStreaming(self.io, &.{}); + + while (self.tokens.items.len <= self.session.maxTokens()) { + try self.runPrefill(turn_tokens, &stdout.interface); + try self.runDecodeTurn(&stdout.interface); + + turn_tokens = if (self.tokens.items.len >= self.session.maxTokens()) b: { + self.allocator.free(turn_tokens); + break :b try self.allocator.alloc(u32, 0); + } else try self.readNextTurn(turn_tokens, &stdout.interface) orelse return; + if (self.tokens.items.len + turn_tokens.len >= self.session.maxTokens()) { + log.warn("Not enough tokens remaining ({} available, {} needed). Consider using a higher seqlen, for example: `--seqlen={d}`.", .{ + self.session.maxTokens() -| self.tokens.items.len, + turn_tokens.len, + std.math.ceilPowerOfTwo(u32, self.session.maxTokens() + 1) catch std.math.maxInt(u32), + }); + return; + } + } + } + + fn runPrefill(self: *Chat, prompt_tokens: []const u32, stdout: *std.Io.Writer) !void { + if (prompt_tokens.len + self.tokens.items.len > self.session.maxTokens()) { + return error.PromptTooLong; + } + try stdout.writeAll("\x1b[2mprefill...\x1b[0m"); + try stdout.flush(); + + try self.tokens.appendSlice(self.allocator, prompt_tokens); + try self.session.runPrefill(self.tokens.items); + + try stdout.writeAll("\r \r"); + try stdout.flush(); + } + + fn runDecodeTurn(self: *Chat, stdout: *std.Io.Writer) !void { + const decode_start: std.Io.Timestamp = .now(self.io, .awake); + const decode_pos_before = self.tokens.items.len; + try self.session.runDecode(&self.tokens, stdout); + const decode_duration = decode_start.untilNow(self.io, .awake); + const decode_tokens: u64 = self.tokens.items.len - decode_pos_before; + + try stdout.writeAll("\n\n"); + try stdout.print("\x1b[36mdecode\x1b[0m \x1b[2m{f} · {:.1}tok/s\x1b[0m\n\n\n", .{ + decode_duration, + tokensPerSecond(decode_duration, decode_tokens), + }); + + try stdout.flush(); + } + + fn readNextTurn(self: *Chat, previous_turn_tokens: []const u32, stdout: *std.Io.Writer) !?[]const u32 { + while (true) { + const line = c.linenoise(prompt_prefix.ptr) orelse { + try stdout.print("\x1b[2m{}/{} tokens used\x1b[0m\n\n", .{ self.tokens.items.len, self.session.maxTokens() }); + try stdout.flush(); + return null; + }; + defer c.linenoiseFree(line); + + rememberPrompt(line); + const input = std.mem.sliceTo(line, 0); + if (input.len == 0) continue; + + self.allocator.free(previous_turn_tokens); + return try self.session.tokenizeTurn(self.allocator, input); + } + } +}; + +fn tokensPerSecond(duration: std.Io.Duration, tokens: u64) f64 { + if (tokens == 0) return 0; + const seconds = @as(f64, @floatFromInt(duration.toNanoseconds())) / 1e9; + if (seconds <= 0) return 0; + return @as(f64, @floatFromInt(tokens)) / seconds; +} diff --git a/examples/llm/main.zig b/examples/llm/main.zig new file mode 100644 index 00000000..b24c022a --- /dev/null +++ b/examples/llm/main.zig @@ -0,0 +1,205 @@ +const std = @import("std"); + +const c = @import("c"); +const zml = @import("zml"); +const stdx = zml.stdx; + +const chat = @import("chat.zig"); +const models = @import("models.zig"); + +pub const std_options: std.Options = .{ + .log_level = .info, +}; + +const log = std.log.scoped(.llm); + +const Args = struct { + model: []const u8, + prompt: ?[]const u8 = null, + seqlen: u32 = 2048, + topk: u32 = 4, + backend: ?enum { stablehlo, cute, nki, metal } = null, + profile: bool = false, + + pub const help = + \\ Use llm --model= [options] + \\ + \\ Run text generation with a model selected from `model_type` in the `config.json`. + \\ + \\ Options: + \\ --model= Path to the model repository (required) + \\ --prompt= Prompt to use for generation (default: none) + \\ --seqlen= Sequence length (default: 2048) + \\ --topk= Top-k sampling cutoff (default: 4) + \\ --backend= Attention backend to use ([stablehlo, cute, nki, metal], default: auto-selection) + \\ --profile Capture a PJRT profile for non-interactive runs and write a Perfetto trace + \\ + ; +}; + +pub fn main(init: std.process.Init) !void { + const allocator = init.gpa; + + // `bazel run` executes binaries from Bazel's runfiles tree by default. + // If available, switch back to the shell's original working directory. + if (init.environ_map.get("BUILD_WORKING_DIRECTORY")) |build_working_directory| { + var working_dir = try std.Io.Dir.openDirAbsolute(init.io, build_working_directory, .{}); + defer working_dir.close(init.io); + try std.process.setCurrentDir(init.io, working_dir); + } + + const args = stdx.flags.parse(init.minimal.args, Args); + + // + // Virtual File Systems + // + var vfs_file: zml.io.VFS.File = .init(allocator, init.io, .{}); + defer vfs_file.deinit(); + + var http_client: std.http.Client = .{ .allocator = allocator, .io = init.io }; + defer http_client.deinit(); + + var hf_vfs: zml.io.VFS.HF = try .auto(allocator, init.io, &http_client, init.environ_map); + defer hf_vfs.deinit(); + + var s3_vfs: zml.io.VFS.S3 = try .auto(allocator, init.io, &http_client, init.environ_map); + defer s3_vfs.deinit(); + + var gcs_vfs: zml.io.VFS.GCS = try .auto(allocator, init.io, &http_client, init.environ_map); + defer gcs_vfs.deinit(); + + var vfs: zml.io.VFS = try .init(allocator, init.io); + defer vfs.deinit(); + + try vfs.register("file", vfs_file.io()); + try vfs.register("gs", gcs_vfs.io()); + try vfs.register("hf", hf_vfs.io()); + try vfs.register("s3", s3_vfs.io()); + + const io = vfs.io(); + + // + // Platform and Backend Selection + // + const platform: *zml.Platform = try .auto(allocator, io, .{}); + defer platform.deinit(allocator, io); + + log.info("\n{f}", .{platform.fmtVerbose()}); + + const backend: zml.attention.Backend = if (args.backend) |selected| switch (selected) { + .stablehlo => .{ .stablehlo = {} }, + .cute => .{ .flash_attention = .cute }, + .nki => .{ .flash_attention = .nki }, + .metal => .{ .flash_attention = .metal }, + } else zml.attention.Backend.auto(platform); + log.info("Selected backend: {}", .{backend}); + + // + // Model initialization + // + log.info("Resolving model repository..", .{}); + const repo = try zml.safetensors.resolveModelRepo(io, args.model); + defer repo.close(io); + + log.info("Initializing model..", .{}); + var registry: zml.safetensors.TensorRegistry = try .fromRepo(allocator, io, repo); + defer registry.deinit(); + + // Defines how the model's tensors are sharded across the available devices. + const meshes: models.Meshes = try .init(platform); + var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &meshes.all()); + defer store.deinit(); + + const generation: models.GenerationOptions = .{ + .sampling_strategy = .{ + .topk = args.topk, + }, + }; + + var model = try models.LoadedModel.load(allocator, io, repo, store.view(), generation); + defer model.deinit(allocator); + + // + // Load the model and compile it + // + var progress = std.Progress.start(io, .{ .root_name = args.model }); + errdefer progress.end(); + + var tokenizer = try loadTokenizer(allocator, io, repo, &progress); + defer tokenizer.deinit(); + + var compiled_model = try allocator.create(models.CompiledModel); + defer allocator.destroy(compiled_model); + compiled_model.* = try models.LoadedModel.compile(&model, allocator, io, platform, backend, meshes, args.seqlen, &progress); + defer compiled_model.deinit(); + + // Load buffers after the model compilation to be sure to give enough room to the autotune. + var model_buffers = try models.LoadedModel.loadBuffers(&model, allocator, io, platform, &store, &progress); + defer model.unloadBuffers(&model_buffers, allocator); + + progress.end(); + + // + // We're ready to go! Let's start a chat... + // + try printZmlLogo(io, init.environ_map); + + const interactive = args.prompt == null; + const prompt = if (args.prompt) |prompt_arg| b: { + break :b try allocator.dupe(u8, prompt_arg); + } else b: { + chat.initHistory(); + const line = c.linenoise(chat.prompt_prefix) orelse return; + defer c.linenoiseFree(line); + chat.rememberPrompt(line); + break :b try allocator.dupe(u8, std.mem.sliceTo(line, 0)); + }; + defer allocator.free(prompt); + + var llm_chat = try chat.Chat.init( + allocator, + io, + platform, + tokenizer, + compiled_model, + &model_buffers, + ); + defer llm_chat.deinit(); + + if (interactive) { + try llm_chat.runInteractive(prompt); + } else { + try llm_chat.runOnce(prompt, .{ + .profile = args.profile, + }); + } +} + +fn loadTokenizer(allocator: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, progress: *std.Progress.Node) !zml.tokenizer.Tokenizer { + progress.increaseEstimatedTotalItems(1); + var node = progress.start("Loading tokenizer...", 1); + defer node.end(); + const now: std.Io.Timestamp = .now(io, .awake); + defer log.info("Loaded tokenizer [{f}]", .{now.untilNow(io, .awake)}); + const bytes = b: { + const file = try dir.openFile(io, "tokenizer.json", .{}); + defer file.close(io); + var reader = file.reader(io, &.{}); + break :b try reader.interface.readAlloc(allocator, try file.length(io)); + }; + defer allocator.free(bytes); + + return try .fromBytes(allocator, bytes); +} + +pub fn printZmlLogo(io: std.Io, environ_map: *const std.process.Environ.Map) !void { + const stdout = std.Io.File.stdout(); + const logo_options: zml.logo.Options = .{ + .color_support = zml.logo.detectColorSupport(io, stdout, environ_map), + }; + var writer = stdout.writerStreaming(io, &.{}); + try writer.interface.writeAll("\n\n"); + try zml.logo.writeZmlArt(&writer.interface, logo_options); + try writer.interface.writeAll("\n\n\n"); + try writer.interface.flush(); +} diff --git a/examples/llm/models.zig b/examples/llm/models.zig new file mode 100644 index 00000000..20aa60c7 --- /dev/null +++ b/examples/llm/models.zig @@ -0,0 +1,201 @@ +const std = @import("std"); + +const zml = @import("zml"); + +const common = @import("models/common.zig"); +pub const Meshes = common.Meshes; +pub const GenerationOptions = common.GenerationOptions; +pub const parseConfig = common.parseConfig; +pub const lfm2 = @import("models/lfm2.zig"); +pub const llama = @import("models/llama.zig"); +pub const qwen3_5 = @import("models/qwen3_5.zig"); +pub const qwen3_5_moe = @import("models/qwen3_5_moe.zig"); + +const log = std.log.scoped(.llm); + +pub const ModelType = enum { + lfm2, + llama, + qwen3_5, + qwen3_5_moe, +}; + +const RawConfig = struct { + model_type: []const u8, +}; + +pub const LoadedModel = union(ModelType) { + lfm2: lfm2.LoadedModel, + llama: llama.LoadedModel, + qwen3_5: qwen3_5.LoadedModel, + qwen3_5_moe: qwen3_5_moe.LoadedModel, + + pub fn load( + allocator: std.mem.Allocator, + io: std.Io, + repo: std.Io.Dir, + store: zml.io.TensorStore.View, + generation: GenerationOptions, + ) !LoadedModel { + const model_type = try detectModelType(allocator, io, repo); + log.info("Detected model type: {}", .{model_type}); + + return switch (model_type) { + inline else => |t| @unionInit( + LoadedModel, + @tagName(t), + try .init(allocator, io, repo, store, generation), + ), + }; + } + + pub fn deinit(self: *LoadedModel, allocator: std.mem.Allocator) void { + switch (self.*) { + inline else => |*m| m.deinit(allocator), + } + } + + pub fn loadBuffers( + self: *LoadedModel, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + store: *zml.io.TensorStore, + progress: *std.Progress.Node, + ) !Buffers { + return switch (self.*) { + inline else => |*m, t| @unionInit( + Buffers, + @tagName(t), + try m.loadBuffers(allocator, io, platform, store, progress), + ), + }; + } + + pub fn unloadBuffers(self: *const LoadedModel, buffers: *Buffers, allocator: std.mem.Allocator) void { + switch (self.*) { + inline else => |*loaded, t| loaded.unloadBuffers(&@field(buffers, @tagName(t)), allocator), + } + } + + pub fn compile( + self: *const LoadedModel, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + backend: zml.attention.Backend, + meshes: Meshes, + seqlen: usize, + progress: *std.Progress.Node, + ) !CompiledModel { + const inner: CompiledModel.Inner = switch (self.*) { + inline else => |*m, t| @unionInit( + CompiledModel.Inner, + @tagName(t), + try m.compile(allocator, io, platform, backend, meshes, seqlen, progress), + ), + }; + return .{ + .inner = inner, + .seqlen = @intCast(seqlen), + }; + } +}; + +pub const CompiledModel = struct { + const Inner = union(ModelType) { + lfm2: lfm2.inference.CompiledModel, + llama: llama.inference.CompiledModel, + qwen3_5: qwen3_5.inference.CompiledModel, + qwen3_5_moe: qwen3_5_moe.inference.CompiledModel, + }; + + inner: Inner, + seqlen: u32, + + pub fn deinit(self: *CompiledModel) void { + switch (self.inner) { + inline else => |*b| b.deinit(), + } + } + + pub fn newSession( + self: *CompiledModel, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + model_buffers: *Buffers, + tokenizer: zml.tokenizer.Tokenizer, + ) !Session { + return switch (self.inner) { + inline else => |*compiled, t| .{ + .inner = @unionInit( + Session.Inner, + @tagName(t), + try .init(allocator, io, platform, tokenizer, compiled, &@field(model_buffers, @tagName(t))), + ), + .seqlen = self.seqlen, + }, + }; + } +}; + +pub const Buffers = union(ModelType) { + lfm2: lfm2.Buffers, + llama: llama.Buffers, + qwen3_5: qwen3_5.Buffers, + qwen3_5_moe: qwen3_5_moe.Buffers, +}; + +pub const Session = struct { + const Inner = union(ModelType) { + lfm2: lfm2.Session, + llama: llama.Session, + qwen3_5: qwen3_5.Session, + qwen3_5_moe: qwen3_5_moe.Session, + }; + + inner: Inner, + seqlen: u32, + + pub fn deinit(self: *Session) void { + switch (self.inner) { + inline else => |*s| s.deinit(), + } + } + + pub fn runPrefill(self: *Session, all_tokens: []const u32) !void { + try switch (self.inner) { + inline else => |*s| s.runPrefill(all_tokens), + }; + } + + pub fn runDecode(self: *Session, all_tokens: *std.ArrayList(u32), writer: *std.Io.Writer) !void { + try switch (self.inner) { + inline else => |*s| s.runDecode(all_tokens, writer), + }; + } + + pub fn tokenizePrompt(self: *const Session, allocator: std.mem.Allocator, prompt: []const u8) ![]const u32 { + return switch (self.inner) { + inline else => |*s| s.tokenizePrompt(allocator, prompt), + }; + } + + pub fn tokenizeTurn(self: *const Session, allocator: std.mem.Allocator, prompt: []const u8) ![]const u32 { + return switch (self.inner) { + inline else => |*s| s.tokenizeTurn(allocator, prompt), + }; + } + + pub fn maxTokens(self: *const Session) u32 { + return self.seqlen; + } +}; + +pub fn detectModelType(allocator: std.mem.Allocator, io: std.Io, repo: std.Io.Dir) !ModelType { + const parsed = try parseConfig(RawConfig, allocator, io, repo); + defer parsed.deinit(); + if (std.meta.stringToEnum(ModelType, parsed.value.model_type)) |model_type| return model_type; + return error.UnknownModelType; +} diff --git a/examples/llm/models/common.zig b/examples/llm/models/common.zig new file mode 100644 index 00000000..8a5c785d --- /dev/null +++ b/examples/llm/models/common.zig @@ -0,0 +1,88 @@ +const std = @import("std"); + +const zml = @import("zml"); + +pub const SessionOptions = struct { + seqlen: u32, + backend: zml.attention.Backend, +}; + +pub const GenerationOptions = struct { + sampling_strategy: zml.nn.SamplingStrategy = .{}, +}; + +pub const Phase = enum { + prefill, + decode, + + pub fn isPrefill(self: Phase) bool { + return self == .prefill; + } + + pub fn label(self: Phase) []const u8 { + return @tagName(self); + } + + pub fn startMessage(self: Phase, comptime component: []const u8) []const u8 { + return switch (self) { + .prefill => "Compiling prefill " ++ component ++ "...", + .decode => "Compiling decode " ++ component ++ "...", + }; + } + + pub fn programName(self: Phase, comptime model_name: []const u8, comptime component: []const u8) []const u8 { + return switch (self) { + .prefill => "llm_" ++ model_name ++ "_prefill_" ++ component, + .decode => "llm_" ++ model_name ++ "_decode_" ++ component, + }; + } + + pub fn logCompileDone(self: Phase, logger: anytype, comptime component: []const u8, io: std.Io, from: std.Io.Timestamp) void { + logger.info("Compiled {s} " ++ component ++ " [{f}]", .{ self.label(), from.untilNow(io, .awake) }); + } +}; + +pub const Meshes = struct { + model: *const zml.Mesh, + experts: *const zml.Mesh, + + pub fn init(platform: *zml.Platform) !Meshes { + switch (platform.target) { + .tpu => { + const has_link_y = platform.physical_mesh.hasAxis(.link_y); + + var strategy_experts: zml.Sharding.Strategy = .parseBindings(.{ .experts = .link_x }); + var strategy_model: zml.Sharding.Strategy = .parseBindings(.{ .model = .link_x }); + if (has_link_y) { + strategy_experts.addFold(.link_x, &.{ .link_x, .link_y }); + strategy_model.addFold(.link_x, &.{ .link_x, .link_y }); + } + + return .{ + .model = try platform.registerMeshWithStrategy("model", .mesh(.{ .model = .high_bandwidth }), strategy_model), + .experts = try platform.registerMeshWithStrategy("experts", .mesh(.{ .experts = .high_bandwidth }), strategy_experts), + }; + }, + .cuda, .rocm, .oneapi, .neuron, .metal, .cpu, .furiosa => return .{ + .model = try platform.registerMesh("model", .mesh(.{ .model = .high_bandwidth })), + .experts = try platform.registerMesh("experts", .mesh(.{ .experts = .high_bandwidth })), + }, + } + } + + pub fn all(self: Meshes) [2]*const zml.Mesh { + return .{ self.model, self.experts }; + } +}; + +pub fn parseConfig(comptime T: type, allocator: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) !std.json.Parsed(T) { + const file = try dir.openFile(io, "config.json", .{}); + defer file.close(io); + + var buffer: [256]u8 = undefined; + var file_reader = file.reader(io, &buffer); + var reader: std.json.Reader = .init(allocator, &file_reader.interface); + defer reader.deinit(); + + return try std.json.parseFromTokenSource(T, allocator, &reader, .{ .ignore_unknown_fields = true }); +} diff --git a/examples/llm/models/lfm2.zig b/examples/llm/models/lfm2.zig new file mode 100644 index 00000000..4b58c18a --- /dev/null +++ b/examples/llm/models/lfm2.zig @@ -0,0 +1,11 @@ +pub const inference = @import("lfm2/inference.zig"); +pub const CompilationParameters = inference.CompilationParameters; +pub const CompilationOptions = inference.CompilationParameters; +pub const CompiledModel = inference.CompiledModel; +pub const model = @import("lfm2/model.zig"); +pub const Config = model.Config; +pub const Buffers = model.Buffers; +pub const Model = model.Model; +pub const LoadedModel = model.LoadedModel; +pub const session = @import("lfm2/session.zig"); +pub const Session = session.Session; diff --git a/examples/llm/models/lfm2/inference.zig b/examples/llm/models/lfm2/inference.zig new file mode 100644 index 00000000..0e37a999 --- /dev/null +++ b/examples/llm/models/lfm2/inference.zig @@ -0,0 +1,321 @@ +const std = @import("std"); + +const zml = @import("zml"); +const stdx = zml.stdx; + +const common = @import("../common.zig"); +const Phase = common.Phase; +const model = @import("model.zig"); + +const log = std.log.scoped(.lfm); + +pub const CompilationParameters = struct { + hidden_dim: usize, + batch_dim: usize, + rng: zml.Tensor.Rng, + cache: model.Cache, + attention_metadata: zml.attention.Metadata, + attention_parameters: zml.attention.Parameters, + seqlen: u32, + meshes: common.Meshes, + + pub fn init(mdl: model.Model, config: model.Config, seqlen: u32, backend: zml.attention.Backend, meshes: common.Meshes) CompilationParameters { + stdx.debug.assert(seqlen >= config.conv_L_cache, "seqlen ({}) must be at least conv_L_cache ({})", .{ seqlen, config.conv_L_cache }); + const cache: model.Cache = .{ + .kv = .init(.init(.{ + .layer = mdl.num_attention_layers, + .batch = 1, + .k = seqlen, + .h = config.num_key_value_heads, + .hd = config.hidden_size / config.num_attention_heads, + }, mdl.embed_tokens.weight.dtype()), meshes.model), + .conv = .init(.init(.{ + .layer = mdl.num_conv_layers, + .batch = 1, + .seq = config.conv_L_cache, + .d = config.hidden_size, + }, mdl.embed_tokens.weight.dtype())), + }; + + return .{ + .hidden_dim = config.hidden_size, + .batch_dim = 1, + .rng = .init(), + .cache = cache, + .attention_metadata = .init(.fromBackend(backend)), + .attention_parameters = .init(.fromBackend(backend)), + .seqlen = seqlen, + .meshes = meshes, + }; + } +}; + +pub const CompilationOptions = CompilationParameters; + +pub const Args = struct { + io: std.Io, + tokens_buf: *zml.Buffer, + next_token_buf: *zml.Buffer, + tokens_pos_buf: *zml.Buffer, + active_length_buf: *zml.Buffer, + rng_buf: *zml.Bufferized(zml.Tensor.Rng), + cache_buffers: *zml.Bufferized(model.Cache), + attention_metadata_buffers: zml.Bufferized(zml.attention.Metadata), +}; + +pub const CompiledModel = struct { + loaded_model: *const model.LoadedModel, + prefill: KernelExe, + decode: KernelExe, + params: CompilationParameters, + + pub fn init( + allocator: std.mem.Allocator, + io: std.Io, + platform: *zml.Platform, + loaded_model: *const model.LoadedModel, + mdl: model.Model, + opts: CompilationParameters, + progress: *std.Progress.Node, + ) !CompiledModel { + const prefill = try compileKernel(allocator, io, platform, mdl, opts, opts.seqlen, .prefill, progress); + errdefer prefill.deinit(); + const decode = try compileKernel(allocator, io, platform, mdl, opts, 1, .decode, progress); + return .{ + .loaded_model = loaded_model, + .prefill = prefill, + .decode = decode, + .params = opts, + }; + } + + pub fn deinit(self: *CompiledModel) void { + self.prefill.deinit(); + self.decode.deinit(); + } +}; + +pub const Inference = CompiledModel; + +pub const KernelExe = struct { + embed: zml.FnExe(model.TokenEmbedding.forward), + conv: zml.FnExe(model.DecoderLayer.forward), + self_attn: zml.FnExe(model.DecoderLayer.forward), + sample: zml.FnExe(model.LmHead.forward), + + pub fn deinit(self: *const KernelExe) void { + self.embed.deinit(); + self.conv.deinit(); + self.self_attn.deinit(); + self.sample.deinit(); + } +}; + +pub const KernelRunner = struct { + embed: zml.FnExe(model.TokenEmbedding.forward).Runner(.{.embedding}), + layers: []zml.FnExe(model.DecoderLayer.forward).Runner(.{.layer}), + sample: zml.FnExe(model.LmHead.forward).Runner(.{ .lm_head, .embed_tokens }), + + pub fn init(allocator: std.mem.Allocator, exe: *const KernelExe, buffers: *const model.Buffers) !KernelRunner { + var embed = try zml.FnExe(model.TokenEmbedding.forward).Runner(.{.embedding}).init(&exe.embed, allocator, .{ .embedding = buffers.embed_tokens }); + errdefer embed.deinit(allocator); + + const layers = try allocator.alloc(zml.FnExe(model.DecoderLayer.forward).Runner(.{.layer}), buffers.layers.len); + errdefer allocator.free(layers); + var initialized_layers: usize = 0; + errdefer for (layers[0..initialized_layers]) |*layer| layer.deinit(allocator); + for (layers, buffers.layers) |*layer, layer_buffers| { + const layer_exe: *const zml.FnExe(model.DecoderLayer.forward) = switch (layer_buffers.operator) { + .conv => &exe.conv, + .self_attn => &exe.self_attn, + }; + layer.* = try zml.FnExe(model.DecoderLayer.forward).Runner(.{.layer}).init(layer_exe, allocator, .{ .layer = layer_buffers }); + initialized_layers += 1; + } + + var sample = try zml.FnExe(model.LmHead.forward).Runner(.{ .lm_head, .embed_tokens }).init(&exe.sample, allocator, .{ + .lm_head = buffers.lm_head, + .embed_tokens = buffers.embed_tokens, + }); + errdefer sample.deinit(allocator); + + return .{ .embed = embed, .layers = layers, .sample = sample }; + } + + pub fn deinit(self: *KernelRunner, allocator: std.mem.Allocator) void { + self.embed.deinit(allocator); + for (self.layers) |*layer| layer.deinit(allocator); + allocator.free(self.layers); + self.sample.deinit(allocator); + } +}; + +pub fn run( + runner: *KernelRunner, + args: Args, + conv_cache_index_buffer: *zml.Buffer, + kv_cache_index_buffer: *zml.Buffer, +) void { + var hidden_buffer: zml.Buffer = undefined; + runner.embed.run(args.io, .{ + .inputs = .{ + .tokens = args.tokens_buf.*, + }, + .outputs = .{ .hidden = &hidden_buffer }, + }, .{}); + defer hidden_buffer.deinit(); + + for (runner.layers) |*layer| { + layer.run(args.io, .{ + .inputs = .{ + .hidden = hidden_buffer, + .tokens_position_offset = args.tokens_pos_buf.*, + .active_length = args.active_length_buf.*, + .cache = args.cache_buffers.*, + .conv_cache_index = conv_cache_index_buffer.*, + .kv_cache_index = kv_cache_index_buffer.*, + .attention_metadata = args.attention_metadata_buffers, + }, + .outputs = .{ + .hidden = &hidden_buffer, + .cache = args.cache_buffers, + .conv_cache_index = conv_cache_index_buffer, + .kv_cache_index = kv_cache_index_buffer, + }, + }, .{}); + } + + runner.sample.run(args.io, .{ + .inputs = .{ + .hidden = hidden_buffer, + .tokens = args.tokens_buf.*, + .active_length = args.active_length_buf.*, + .rng = args.rng_buf.*, + }, + .outputs = .{ + .tokens = args.next_token_buf, + .rng = args.rng_buf, + }, + }, .{}); +} + +fn compileKernel( + allocator: std.mem.Allocator, + io: std.Io, + platform: *zml.Platform, + mdl: model.Model, + opts: CompilationOptions, + seqlen: u32, + phase: Phase, + progress: *std.Progress.Node, +) !KernelExe { + const embed = try compileEmbed(allocator, io, platform, mdl.embed_tokens, opts, seqlen, phase, progress); + errdefer embed.deinit(); + const conv_layer = try compileLayer(allocator, io, platform, mdl, opts, seqlen, .conv, phase, progress); + errdefer conv_layer.deinit(); + const attn_layer = try compileLayer(allocator, io, platform, mdl, opts, seqlen, .full_attention, phase, progress); + errdefer attn_layer.deinit(); + const sample = try compileSample(allocator, io, platform, mdl, opts, seqlen, phase, progress); + errdefer sample.deinit(); + return .{ .embed = embed, .conv = conv_layer, .self_attn = attn_layer, .sample = sample }; +} + +fn compileEmbed( + allocator: std.mem.Allocator, + io: std.Io, + platform: *zml.Platform, + embed_tokens: model.TokenEmbedding, + opts: CompilationOptions, + seqlen: u32, + phase: Phase, + progress: *std.Progress.Node, +) !zml.FnExe(model.TokenEmbedding.forward) { + progress.increaseEstimatedTotalItems(1); + var node = progress.start(phase.startMessage("embed_tokens"), 1); + defer node.end(); + const from: std.Io.Timestamp = .now(io, .awake); + defer phase.logCompileDone(log, "embed_tokens", io, from); + + return zml.FnExe(model.TokenEmbedding.forward).compile(allocator, io, platform, .{ + .meshes = &opts.meshes.all(), + .program_name = phase.programName("lfm2", "embed_tokens"), + }, .{.{ + .embedding = embed_tokens, + .tokens = zml.Tensor.init(.{ .batch = opts.batch_dim, .seq = seqlen }, .u32), + }}); +} + +fn compileLayer( + allocator: std.mem.Allocator, + io: std.Io, + platform: *zml.Platform, + mdl: model.Model, + opts: CompilationOptions, + seqlen: u32, + comptime kind: model.OperatorKind, + phase: Phase, + progress: *std.Progress.Node, +) !zml.FnExe(model.DecoderLayer.forward) { + const label = switch (kind) { + .conv => "conv layer", + .full_attention => "attn layer", + }; + progress.increaseEstimatedTotalItems(1); + var node = progress.start(phase.startMessage(label), 1); + defer node.end(); + const from: std.Io.Timestamp = .now(io, .awake); + defer phase.logCompileDone(log, label, io, from); + + const layer = for (mdl.layers) |candidate| { + const candidate_kind: model.OperatorKind = switch (candidate.operator) { + .conv => .conv, + .self_attn => .full_attention, + }; + if (candidate_kind == kind) break candidate; + } else unreachable; + + return zml.FnExe(model.DecoderLayer.forward).compile(allocator, io, platform, .{ + .meshes = &opts.meshes.all(), + .program_name = phase.programName("lfm2", if (kind == .conv) "conv_layer" else "attn_layer"), + }, .{.{ + .layer = layer, + .hidden = zml.Tensor.init(.{ .batch = opts.batch_dim, .seq = seqlen, .d = opts.hidden_dim }, mdl.embed_tokens.weight.dtype()), + .tokens_position_offset = zml.Tensor.init(.{ .batch = opts.batch_dim }, .u32), + .active_length = .init(.{}, .u32), + .cache = opts.cache, + .conv_cache_index = zml.Tensor.init(.{}, .u32), + .kv_cache_index = zml.Tensor.init(.{}, .u32), + .attention_metadata = opts.attention_metadata, + .attention_parameters = opts.attention_parameters, + .conv_parameters = .{ .is_prefill = phase.isPrefill() }, + }}); +} + +fn compileSample( + allocator: std.mem.Allocator, + io: std.Io, + platform: *zml.Platform, + mdl: model.Model, + opts: CompilationOptions, + seqlen: u32, + phase: Phase, + progress: *std.Progress.Node, +) !zml.FnExe(model.LmHead.forward) { + progress.increaseEstimatedTotalItems(1); + var node = progress.start(phase.startMessage("lm_head"), 1); + defer node.end(); + const from: std.Io.Timestamp = .now(io, .awake); + defer phase.logCompileDone(log, "lm_head", io, from); + + return zml.FnExe(model.LmHead.forward).compile(allocator, io, platform, .{ + .meshes = &opts.meshes.all(), + .program_name = phase.programName("lfm2", "lm_head"), + }, .{.{ + .lm_head = mdl.lm_head, + .embed_tokens = mdl.embed_tokens, + .hidden = zml.Tensor.init(.{ .batch = opts.batch_dim, .seq = seqlen, .d = opts.hidden_dim }, mdl.embed_tokens.weight.dtype()), + .tokens = .init(.{ .batch = opts.batch_dim, .seq = seqlen }, .u32), + .active_length = .init(.{}, .u32), + .rng = opts.rng, + }}); +} diff --git a/examples/llm/models/lfm2/model.zig b/examples/llm/models/lfm2/model.zig new file mode 100644 index 00000000..9f7cb909 --- /dev/null +++ b/examples/llm/models/lfm2/model.zig @@ -0,0 +1,761 @@ +const std = @import("std"); + +const zml = @import("zml"); +const stdx = zml.stdx; + +const common = @import("../common.zig"); +const inference = @import("inference.zig"); + +const log = std.log.scoped(.lfm); + +pub const Config = struct { + architectures: []const []const u8, + block_auto_adjust_ff_dim: bool, + block_dim: u32, + block_ff_dim: u32, + block_ffn_dim_multiplier: f32, + block_mlp_init_scale: f32, + block_multiple_of: u32, + block_norm_eps: f32, + block_out_init_scale: f32, + block_use_swiglu: bool, + block_use_xavier_init: bool, + bos_token_id: u32, + conv_L_cache: u32, + conv_bias: bool, + conv_dim: u32, + conv_use_xavier_init: bool, + dtype: []const u8, + eos_token_id: u32, + hidden_size: u32, + initializer_range: f32, + intermediate_size: u32, + layer_types: []const []const u8, + max_position_embeddings: u32, + model_type: []const u8, + norm_eps: f32, + num_attention_heads: u32, + num_heads: u32, + num_hidden_layers: u32, + num_key_value_heads: u32, + pad_token_id: u32, + rope_theta: f32, + tie_embedding: bool, + transformers_version: []const u8, + use_cache: bool, + use_pos_enc: bool, + vocab_size: u32, +}; + +pub const LoadedModel = struct { + inner: Model, + parsed_config: std.json.Parsed(Config), + + pub fn init( + allocator: std.mem.Allocator, + io: std.Io, + repo: std.Io.Dir, + store: zml.io.TensorStore.View, + generation: common.GenerationOptions, + ) !LoadedModel { + const parsed_config = try common.parseConfig(Config, allocator, io, repo); + errdefer parsed_config.deinit(); + + return .{ + .inner = .init(allocator, store, parsed_config.value, generation), + .parsed_config = parsed_config, + }; + } + + pub fn deinit(self: *LoadedModel, allocator: std.mem.Allocator) void { + self.inner.deinit(allocator); + self.parsed_config.deinit(); + } + + pub fn loadBuffers( + self: *const LoadedModel, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + store: *zml.io.TensorStore, + progress: *std.Progress.Node, + ) !Buffers { + progress.increaseEstimatedTotalItems(store.view().count()); + const now: std.Io.Timestamp = .now(io, .awake); + + var buffers = try zml.mem.bufferize(allocator, Model, &self.inner); + errdefer self.unloadBuffers(&buffers, allocator); + + var loader: zml.io.Loader = try .init(allocator, platform, .{ + .dma_chunks = 32, + .dma_chunk_size = 256 * zml.MiB, + .parallelism = 16, + }); + defer loader.deinit(); + + try loader.load(io, Model, &self.inner, &buffers, store, .{ .progress = progress }); + try loader.await(io); + + const took = now.untilNow(io, .awake); + const total_bytes: u64 = loader.bytes_loaded.raw; + const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); + log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); + + return buffers; + } + + pub fn unloadBuffers(self: *const LoadedModel, buffers: *Buffers, allocator: std.mem.Allocator) void { + _ = self; + Model.unloadBuffers(buffers, allocator); + } + + pub fn compile( + self: *const LoadedModel, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + backend: zml.attention.Backend, + meshes: common.Meshes, + seqlen: usize, + progress: *std.Progress.Node, + ) !inference.CompiledModel { + const params = inference.CompilationParameters.init(self.inner, self.parsed_config.value, @intCast(seqlen), backend, meshes); + return inference.CompiledModel.init(allocator, io, @constCast(platform), self, self.inner, params, progress); + } +}; + +pub const Buffers = zml.Bufferized(Model); + +pub const Model = struct { + embed_tokens: TokenEmbedding, + lm_head: LmHead, + layers: []DecoderLayer, + num_attention_layers: usize, + num_conv_layers: usize, + + pub fn init( + allocator: std.mem.Allocator, + root_store: zml.io.TensorStore.View, + config: Config, + generation: common.GenerationOptions, + ) Model { + const store = root_store.withPrefix("model"); + stdx.debug.assert(config.layer_types.len == config.num_hidden_layers, "Expected layer_types len {} to match num_hidden_layers {}", .{ config.layer_types.len, config.num_hidden_layers }); + + const layers = allocator.alloc(DecoderLayer, config.num_hidden_layers) catch unreachable; + var num_attention_layers: usize = 0; + var num_conv_layers: usize = 0; + for (layers, 0..) |*layer, i| { + const layer_store = store.withPrefix("layers").withLayer(i); + const kind = DecoderLayer.parseOperatorKind(config.layer_types[i]); + switch (kind) { + .conv => num_conv_layers += 1, + .full_attention => num_attention_layers += 1, + } + layer.* = DecoderLayer.init(config, layer_store, kind); + } + + return .{ + .embed_tokens = .init(store.withPrefix("embed_tokens")), + .lm_head = LmHead.init(store, config, generation.sampling_strategy), + .layers = layers, + .num_attention_layers = num_attention_layers, + .num_conv_layers = num_conv_layers, + }; + } + + pub fn loadBuffers( + self: *const Model, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + store: *zml.io.TensorStore, + progress: *std.Progress.Node, + ) !zml.Bufferized(Model) { + progress.increaseEstimatedTotalItems(store.view().count()); + const now: std.Io.Timestamp = .now(io, .awake); + + var buffers = try zml.mem.bufferize(allocator, Model, self); + errdefer Model.unloadBuffers(&buffers, allocator); + + var loader: zml.io.Loader = try .init(allocator, platform, .{ + .dma_chunks = 8, + .dma_chunk_size = 256 * zml.MiB, + .parallelism = 16, + }); + defer loader.deinit(); + + loader.load(io, Model, self, &buffers, store); + try loader.await(io); + + const took = now.untilNow(io, .awake); + const total_bytes: u64 = loader.bytes_loaded.raw; + const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); + log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); + + return buffers; + } + + pub fn unloadBuffers(self: *zml.Bufferized(Model), allocator: std.mem.Allocator) void { + self.embed_tokens.weight.deinit(); + LmHead.unloadBuffers(&self.lm_head); + for (self.layers) |*layer| { + DecoderLayer.unloadBuffers(layer); + } + allocator.free(self.layers); + } + + pub fn forward( + self: Model, + tokens: zml.Tensor, + tokens_position_offset: zml.Tensor, + active_length: zml.Tensor, + rng: zml.Tensor.Rng, + cache_: Cache, + attention_metadata: zml.attention.Metadata, + attention_parameters: zml.attention.Parameters, + conv_parameters: ConvParameters, + ) struct { zml.Tensor, Cache, zml.Tensor.Rng } { + stdx.debug.assert(tokens.shape().hasTags(.{ .batch, .seq }), "Tokens should have tags {{.batch, .seq}}, got {f}", .{tokens.shape()}); + + const embeds = TokenEmbedding.forward(.{ + .embedding = self.embed_tokens, + .tokens = tokens, + }).hidden; + + var hidden = embeds; + var cache = cache_; + // Incremented by the individual layers. + var conv_cache_index = zml.Tensor.scalar(@as(u32, 0), .u32); + var kv_cache_index = zml.Tensor.scalar(@as(u32, 0), .u32); + for (self.layers) |layer| { + const result = DecoderLayer.forward(.{ + .layer = layer, + .hidden = hidden, + .tokens_position_offset = tokens_position_offset, + .active_length = active_length, + .cache = cache, + .conv_cache_index = conv_cache_index, + .kv_cache_index = kv_cache_index, + .attention_metadata = attention_metadata, + .attention_parameters = attention_parameters, + .conv_parameters = conv_parameters, + }); + hidden = result.hidden; + cache = result.cache; + conv_cache_index = result.conv_cache_index; + kv_cache_index = result.kv_cache_index; + } + const result = LmHead.forward(.{ + .lm_head = self.lm_head, + .embed_tokens = self.embed_tokens, + .hidden = hidden, + .tokens = tokens, + .active_length = active_length, + .rng = rng, + }); + return .{ result.tokens, cache.reuseBuffer(cache_), result.rng }; + } + + pub fn deinit(self: Model, allocator: std.mem.Allocator) void { + allocator.free(self.layers); + } +}; + +pub const TokenEmbedding = struct { + weight: zml.Tensor, + + pub const Input = struct { + embedding: TokenEmbedding, + tokens: zml.Tensor, + }; + + pub const Output = struct { + hidden: zml.Tensor, + }; + + pub fn init(store: zml.io.TensorStore.View) TokenEmbedding { + return .{ .weight = store.createTensor("weight", .{ .voc, .d }, .model, .{ .voc = .replicated, .d = .model }) }; + } + + pub fn forward(input: Input) Output { + stdx.debug.assert(input.tokens.dtype().isInteger(), "TokenEmbedding expects an integer input, received: {f}", .{input.tokens}); + stdx.debug.assert(input.embedding.weight.rank() == 2, "TokenEmbedding expects it's weight zml.Tensor to be a 2D matrix, got {f}", .{input.embedding.weight}); + return .{ .hidden = input.embedding.weight.gather(.{ .voc = input.tokens }, .{}) }; + } + + pub fn unembed(self: TokenEmbedding, embeds: zml.Tensor) zml.Tensor { + stdx.debug.assert(embeds.shape().hasTags(.{.d}), "TokenEmbedding expects the input embeds to have a .d tag, got {f}", .{embeds.shape()}); + return self.weight.dot(embeds, .d); + } +}; + +pub const LmHead = struct { + embedding_norm: RmsNorm, + sampling_strategy: zml.nn.SamplingStrategy, + + pub const Input = struct { + lm_head: LmHead, + embed_tokens: TokenEmbedding, + hidden: zml.Tensor, + tokens: zml.Tensor, + active_length: zml.Tensor, + rng: zml.Tensor.Rng, + }; + + pub const Output = struct { + tokens: zml.Tensor, + rng: zml.Tensor.Rng, + }; + + pub fn init(store: zml.io.TensorStore.View, config: Config, sampling_strategy: zml.nn.SamplingStrategy) LmHead { + return .{ + .embedding_norm = RmsNorm.init(store.withPrefix("embedding_norm"), config.norm_eps, .d), + .sampling_strategy = sampling_strategy, + }; + } + + pub fn forward(input: Input) Output { + const self = input.lm_head; + const last_index = input.active_length.convert(.i32).addConstant(-1); + const hidden = input.hidden.slice(.seq, .dyn(last_index, 1)); + const logits = input.embed_tokens.unembed(self.embedding_norm.forward(hidden)); + const new_tokens, const new_rng = zml.nn.sampleTokens(logits, self.sampling_strategy, input.rng); + const tokens = new_tokens.convert(.u32); + return .{ .tokens = if (input.tokens.dim(.seq) == 1) tokens.reuseBuffer(input.tokens) else tokens, .rng = new_rng }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(LmHead)) void { + RmsNorm.unloadBuffers(&self.embedding_norm); + } +}; + +pub const OperatorKind = enum { conv, full_attention }; +const Operator = union(enum) { conv: ShortConv, self_attn: Attention }; + +pub const DecoderLayer = struct { + operator: Operator, + operator_norm: RmsNorm, + ffn_norm: RmsNorm, + feed_forward: Mlp, + + pub const Input = struct { + layer: DecoderLayer, + hidden: zml.Tensor, + tokens_position_offset: zml.Tensor, + active_length: zml.Tensor, + cache: Cache, + conv_cache_index: zml.Tensor, + kv_cache_index: zml.Tensor, + attention_metadata: zml.attention.Metadata, + attention_parameters: zml.attention.Parameters, + conv_parameters: ConvParameters, + }; + + pub const Output = struct { + hidden: zml.Tensor, + cache: Cache, + conv_cache_index: zml.Tensor, + kv_cache_index: zml.Tensor, + }; + + pub fn parseOperatorKind(layer_type: []const u8) OperatorKind { + return std.meta.stringToEnum(OperatorKind, layer_type) orelse { + stdx.debug.assert(false, "Unsupported layer type {s}", .{layer_type}); + unreachable; + }; + } + + pub fn init(config: Config, store: zml.io.TensorStore.View, kind: OperatorKind) DecoderLayer { + const operator_norm = RmsNorm.init(store.withPrefix("operator_norm"), config.norm_eps, .d); + const ffn_norm = RmsNorm.init(store.withPrefix("ffn_norm"), config.norm_eps, .d); + const feed_forward = Mlp.init(store.withPrefix("feed_forward")); + const operator: Operator = switch (kind) { + .conv => .{ .conv = ShortConv.init(config, store.withPrefix("conv")) }, + .full_attention => .{ .self_attn = Attention.init(config, store.withPrefix("self_attn")) }, + }; + + return .{ + .operator = operator, + .operator_norm = operator_norm, + .ffn_norm = ffn_norm, + .feed_forward = feed_forward, + }; + } + + pub fn forward(input: Input) Output { + const self = input.layer; + var cache = input.cache; + var conv_cache_index = input.conv_cache_index; + var kv_cache_index = input.kv_cache_index; + const residual = switch (self.operator) { + .conv => |operator| b: { + const residual, const updated_conv_cache = operator.forward( + self.operator_norm.forward(input.hidden), + input.tokens_position_offset, + input.active_length, + cache.conv, + conv_cache_index, + input.conv_parameters, + ); + cache.conv = updated_conv_cache; + conv_cache_index = conv_cache_index.add(zml.Tensor.scalar(@as(u32, 1), .u32)); + break :b residual; + }, + .self_attn => |operator| b: { + const residual, const updated_kv_cache = operator.forward( + self.operator_norm.forward(input.hidden), + input.tokens_position_offset, + cache.kv, + kv_cache_index, + input.attention_metadata, + input.attention_parameters, + ); + cache.kv = updated_kv_cache; + kv_cache_index = kv_cache_index.add(zml.Tensor.scalar(@as(u32, 1), .u32)); + break :b residual; + }, + }; + + const x = input.hidden.add(residual); + + return .{ + .hidden = x.add(self.feed_forward.forward(self.ffn_norm.forward(x))).reuseBuffer(input.hidden), + .cache = cache.reuseBuffer(input.cache), + .conv_cache_index = conv_cache_index.reuseBuffer(input.conv_cache_index), + .kv_cache_index = kv_cache_index.reuseBuffer(input.kv_cache_index), + }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(DecoderLayer)) void { + switch (self.operator) { + .conv => |*operator| ShortConv.unloadBuffers(operator), + .self_attn => |*operator| Attention.unloadBuffers(operator), + } + RmsNorm.unloadBuffers(&self.operator_norm); + RmsNorm.unloadBuffers(&self.ffn_norm); + Mlp.unloadBuffers(&self.feed_forward); + } +}; + +pub const ConvParameters = struct { is_prefill: bool }; + +pub const ShortConv = struct { + in_proj: Linear, + out_proj: Linear, + kernel: zml.Tensor, + config: Config, + + pub fn init(config: Config, store: zml.io.TensorStore.View) ShortConv { + stdx.debug.assert(!config.conv_bias, "conv_bias is not supported.", .{}); + return .{ + .in_proj = .init(store.withPrefix("in_proj").createTensor("weight", .{ .out, .d }, .model, .{ .out = .model, .d = .replicated }), null, .d), + .out_proj = .init(store.withPrefix("out_proj").createTensor("weight", .{ .out, .d }, .model, .{ .out = .replicated, .d = .model }), null, .d), + .kernel = store.createReplicatedTensor("conv.weight", .{ .out, .in, .kernel_size }), + .config = config, + }; + } + + pub fn forward(self: ShortConv, input: zml.Tensor, tokens_position_offset: zml.Tensor, active_length: zml.Tensor, cache_: ConvCache, cache_index: zml.Tensor, parameters: ConvParameters) struct { zml.Tensor, ConvCache } { + var cache = cache_; + const BCx = self.in_proj.forward(input.withPartitioning(.model, .{ .d = .replicated })); + + const B, const C, const x = BCx.chunkExact(.d, 3); + const Bx = B.mul(x); + + const conv_out = if (parameters.is_prefill) b: { + const active_length_i32 = active_length.convert(.i32); + const start = active_length_i32.sub(.scalar(self.config.conv_L_cache, .i32)).maximum(.scalar(0, .i32)); + const cache_seq_indices = lkp: { + const n = active_length_i32.sub(start); + const left_pad = zml.Tensor.scalar(@as(i32, @intCast(self.config.conv_L_cache)), .i32).sub(n); + const sh = tokens_position_offset.shape().insert(.last, .{ .seq = self.config.conv_L_cache }); + break :lkp zml.Tensor.iota(sh, .seq).convert(.u32).add(left_pad.convert(.u32).broad(sh)).broad(sh); + }; + const scatter_data = Bx.slice(.seq, .dyn(start, self.config.conv_L_cache)); + cache.state = cache.state.scatterSlices(.{ .layer = cache_index, .seq = cache_seq_indices }, scatter_data, .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }).reuseBuffer(cache.state); + const padding = self.config.conv_L_cache - 1; + const conv_out_ = zml.Tensor.conv1d(Bx, self.kernel, .{ + .padding = &.{ padding, 0 }, + .input_batch_dimension = Bx.axis(.batch), + .input_feature_dimension = B.axis(.d), + .input_spatial_dimensions = Bx.axis(.seq), + .kernel_output_feature_dimension = self.kernel.axis(.out), + .kernel_input_feature_dimension = self.kernel.axis(.in), + .kernel_spatial_dimensions = self.kernel.axis(.kernel_size), + .output_batch_dimension = Bx.axis(.batch), + .output_feature_dimension = Bx.axis(.d), + .output_spatial_dimensions = Bx.axis(.seq), + .feature_group_count = Bx.dim(.d), + }); + break :b conv_out_.slice(.seq, .{ .end = Bx.dim(.seq) }); + } else b: { + cache.state = cache.state.rollRight1d(.seq, -1).scatterSlices(.{ .layer = cache_index, .seq = tokens_position_offset.convert(.u32).clamp(.scalar(@as(u32, 0), .u32), .scalar(self.config.conv_L_cache - 1, .u32)) }, Bx, .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }).reuseBuffer(cache.state); + const kernel = self.kernel.squeeze(.in).rename(.{ .out = .d, .kernel_size = .seq }); + const sh = kernel.shape().insert(.last, .{ .batch = tokens_position_offset.dim(.batch) }); + break :b cache.state.slice(.layer, .dynSingle(cache_index)).mul(kernel.broad(sh).transpose(.{ .batch, .seq, .d })).sum(.seq); + }; + + const y = C.mul(conv_out); + + const output = self.out_proj.forward(y); + return .{ output.reuseBuffer(input), cache.reuseBuffer(cache_) }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(ShortConv)) void { + Linear.unloadBuffers(&self.in_proj); + Linear.unloadBuffers(&self.out_proj); + self.kernel.deinit(); + } +}; + +pub const Attention = struct { + q_proj: Linear, + k_proj: Linear, + v_proj: Linear, + out_proj: Linear, + q_layernorm: RmsNorm, + k_layernorm: RmsNorm, + head_dim: usize, + num_key_value_groups: usize, + rope_opts: zml.nn.RopeOpts, + + pub fn init(config: Config, store: zml.io.TensorStore.View) Attention { + const head_dim = config.hidden_size / config.num_attention_heads; + const num_key_value_groups = config.num_attention_heads / config.num_key_value_heads; + return .{ + .q_proj = .init(store.withPrefix("q_proj").createTensor("weight", .{ .out, .d }, .model, .{ .out = .model, .d = .replicated }), null, .d), + .k_proj = .init(store.withPrefix("k_proj").createTensor("weight", .{ .out, .d }, .model, .{ .out = .model, .d = .replicated }), null, .d), + .v_proj = .init(store.withPrefix("v_proj").createTensor("weight", .{ .out, .d }, .model, .{ .out = .model, .d = .replicated }), null, .d), + .out_proj = .init(store.withPrefix("out_proj").createTensor("weight", .{ .out, .d }, .model, .{ .out = .replicated, .d = .model }), null, .d), + .q_layernorm = RmsNorm.init(store.withPrefix("q_layernorm"), config.norm_eps, .hd), + .k_layernorm = RmsNorm.init(store.withPrefix("k_layernorm"), config.norm_eps, .hd), + .head_dim = head_dim, + .num_key_value_groups = num_key_value_groups, + .rope_opts = .{ .layout = .real_im_pass, .scaling = .{ .default = .{ .rope_theta = config.rope_theta } } }, + }; + } + + pub fn forward( + self: Attention, + x: zml.Tensor, + tokens_position_offset: zml.Tensor, + kv_cache: KvCache, + cache_index: zml.Tensor, + attention_metadata: zml.attention.Metadata, + attention_parameters: zml.attention.Parameters, + ) struct { zml.Tensor, KvCache } { + const x_qkv = x.withPartitioning(.model, .{ .d = .replicated }); + + var q = self.q_proj.forward(x_qkv).splitAxis(-1, .{ .h = .auto, .hd = self.head_dim }); + var k = self.k_proj.forward(x_qkv).splitAxis(-1, .{ .h = .auto, .hd = self.head_dim }); + var v = self.v_proj.forward(x_qkv).splitAxis(-1, .{ .h = .auto, .hd = self.head_dim }); + + q = self.q_layernorm.forward(q); + k = self.k_layernorm.forward(k); + + const token_positions = b: { + const sh = tokens_position_offset.shape().insert(.last, .{ .seq = x.dim(.seq) }); + break :b zml.Tensor.iota(sh, .seq).convert(.u32).add(tokens_position_offset.broad(sh)); + }; + + q = zml.nn.rope(q, token_positions, self.rope_opts); + k = zml.nn.rope(k, token_positions, self.rope_opts); + + q = q.rename(.{ .seq = .q }); + k = k.rename(.{ .seq = .k }); + v = v.rename(.{ .seq = .k }); + + const new_kv_cache = kv_cache.update(k, v, tokens_position_offset, cache_index); + k = new_kv_cache.keys(cache_index); + v = new_kv_cache.values(cache_index); + + stdx.debug.assert(q.dim(.batch) == 1, "LFM attention currently expects batch size 1 for flash attention backend, got {}", .{q.dim(.batch)}); + const mesh = zml.Compiler.current().mesh(.model); + const partitioning: zml.attention.Partitioning = .head(.model); + const attn = switch (attention_parameters) { + .stablehlo => zml.attention.attention( + q, + k, + v, + tokens_position_offset, + attention_metadata, + attention_parameters, + mesh, + partitioning, + ).merge(.{ .d = .{ .h, .hd } }), + else => zml.attention.attention( + q.squeeze(.batch), + k.squeeze(.batch), + v.squeeze(.batch), + tokens_position_offset.squeeze(.batch), + attention_metadata, + attention_parameters, + mesh, + partitioning, + ).merge(.{ .d = .{ .h, .hd } }) + .rename(.{ .q = .seq }) + .insertAxes(.seq, .{.batch}), + }; + + return .{ self.out_proj.forward(attn).reuseBuffer(x), new_kv_cache.reuseBuffer(kv_cache) }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(Attention)) void { + Linear.unloadBuffers(&self.q_proj); + Linear.unloadBuffers(&self.k_proj); + Linear.unloadBuffers(&self.v_proj); + Linear.unloadBuffers(&self.out_proj); + RmsNorm.unloadBuffers(&self.q_layernorm); + RmsNorm.unloadBuffers(&self.k_layernorm); + } +}; + +pub const Cache = struct { + conv: ConvCache, + kv: KvCache, + + pub fn initBuffers(self: Cache, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(Cache) { + return .{ .conv = try self.conv.initBuffers(allocator, io, platform), .kv = try self.kv.initBuffers(io, platform) }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(Cache)) void { + ConvCache.unloadBuffers(&self.conv); + KvCache.unloadBuffers(&self.kv); + } + + pub fn reuseBuffer(self: Cache, other: Cache) Cache { + return .{ .conv = self.conv.reuseBuffer(other.conv), .kv = self.kv.reuseBuffer(other.kv) }; + } +}; + +pub const ConvCache = struct { + state: zml.Tensor, + + pub fn init(shape: zml.Shape) ConvCache { + return .{ .state = .fromShape(shape) }; + } + + pub fn initBuffers(self: ConvCache, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(ConvCache) { + const sh = self.state.shape(); + const host = try allocator.alloc(u8, sh.byteSize()); + defer allocator.free(host); + @memset(host, 0); + return .{ .state = try zml.Buffer.fromBytes(io, platform, sh, host) }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(ConvCache)) void { + self.state.deinit(); + } + + pub fn reuseBuffer(self: ConvCache, other: ConvCache) ConvCache { + return .{ .state = self.state.reuseBuffer(other.state) }; + } +}; + +pub const KvCache = struct { + k: zml.Tensor, + v: zml.Tensor, + + pub fn init(kv_shape: zml.Shape, mesh: *const zml.Mesh) KvCache { + const sharded_shape = kv_shape.withPartitioning(mesh, .{ .h = .model }); + return .{ .k = .fromShape(sharded_shape), .v = .fromShape(sharded_shape) }; + } + + pub fn initBuffers(self: KvCache, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(KvCache) { + return .{ .k = try zml.Buffer.uninitialized(io, platform, self.k.shape(), .{}), .v = try zml.Buffer.uninitialized(io, platform, self.v.shape(), .{}) }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(KvCache)) void { + self.k.deinit(); + self.v.deinit(); + } + + pub fn reuseBuffer(self: KvCache, other: KvCache) KvCache { + return .{ .k = self.k.reuseBuffer(other.k), .v = self.v.reuseBuffer(other.v) }; + } + + pub fn keys(self: KvCache, cache_index: zml.Tensor) zml.Tensor { + return self.k.slice(.layer, .dynSingle(cache_index)); + } + + pub fn values(self: KvCache, cache_index: zml.Tensor) zml.Tensor { + return self.v.slice(.layer, .dynSingle(cache_index)); + } + + pub fn update(self: KvCache, new_k: zml.Tensor, new_v: zml.Tensor, token_position: zml.Tensor, cache_index: zml.Tensor) KvCache { + const k_shape = self.k.shape().drop(.layer); + const layer = cache_index.broad(token_position.shape()); + return .{ + .k = self.k.scatterSlices(.{ .layer = layer, .k = token_position }, new_k.transpose(k_shape), .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }).reuseBuffer(self.k), + .v = self.v.scatterSlices(.{ .layer = layer, .k = token_position }, new_v.transpose(k_shape), .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }).reuseBuffer(self.v), + }; + } +}; + +pub const Linear = struct { + weight: zml.Tensor, + bias: ?zml.Tensor = null, + in_tag: zml.Shape.Tag, + out_tag: zml.Shape.Tag, + + pub fn init(weight: zml.Tensor, bias: ?zml.Tensor, tag: anytype) Linear { + stdx.debug.guard(weight.shape().hasTag(tag) != null, @src()); + const axis = weight.shape().axis(tag); + const out_tag = weight.shape().tag(1 - axis); + return .{ .weight = weight, .bias = bias, .in_tag = zml.Shape.toTag(tag), .out_tag = out_tag }; + } + + pub fn unloadBuffers(linear: *zml.Bufferized(Linear)) void { + linear.weight.deinit(); + if (linear.bias) |*bias| bias.deinit(); + } + + pub fn forward(self: Linear, x: zml.Tensor) zml.Tensor { + var y = x.dot(self.weight, self.in_tag).renameTag(self.out_tag, self.in_tag); + return if (self.bias) |bias| y.add(bias.broad(y.shape())).reuseBuffer(x) else y; + } +}; + +const Mlp = struct { + w1: Linear, + w2: Linear, + w3: Linear, + + pub fn init(store: zml.io.TensorStore.View) Mlp { + return .{ + .w1 = .init(store.withPrefix("w1").createTensor("weight", .{ .out, .d }, .model, .{ .out = .model, .d = .replicated }), null, .d), + .w2 = .init(store.withPrefix("w2").createTensor("weight", .{ .out, .d }, .model, .{ .out = .replicated, .d = .model }), null, .d), + .w3 = .init(store.withPrefix("w3").createTensor("weight", .{ .out, .d }, .model, .{ .out = .model, .d = .replicated }), null, .d), + }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(Mlp)) void { + Linear.unloadBuffers(&self.w1); + Linear.unloadBuffers(&self.w2); + Linear.unloadBuffers(&self.w3); + } + + pub fn forward(self: Mlp, x: zml.Tensor) zml.Tensor { + return self.w2.forward(self.w1.forward(x).silu().mul(self.w3.forward(x))).reuseBuffer(x); + } +}; + +const RmsNorm = struct { + weight: zml.Tensor, + eps: f32, + tag: zml.Shape.Tag, + + pub fn init(store: zml.io.TensorStore.View, eps: f32, tag: anytype) RmsNorm { + return .{ .weight = store.createReplicatedTensor("weight", .{tag}), .eps = eps, .tag = zml.Shape.toTag(tag) }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(RmsNorm)) void { + self.weight.deinit(); + } + + pub fn forward(self: RmsNorm, input: zml.Tensor) zml.Tensor { + const normalized = zml.nn.rmsNorm(input, self.tag, self.eps); + return normalized.mul(self.weight.broad(input.shape())).reuseBuffer(input); + } +}; diff --git a/examples/llm/models/lfm2/session.zig b/examples/llm/models/lfm2/session.zig new file mode 100644 index 00000000..ecc5abf8 --- /dev/null +++ b/examples/llm/models/lfm2/session.zig @@ -0,0 +1,227 @@ +const std = @import("std"); + +const zml = @import("zml"); + +const inference = @import("inference.zig"); +const model = @import("model.zig"); + +pub const Session = struct { + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + compiled_model: *inference.CompiledModel, + prefill: inference.KernelRunner, + decode: inference.KernelRunner, + config: *const model.Config, + seqlen: u32, + cache_buffers: zml.Bufferized(model.Cache), + rng_buf: zml.Bufferized(zml.Tensor.Rng), + generated_token_slice: zml.Slice, + think_start: ?u32, + think_end: ?u32, + tokenizer: zml.tokenizer.Tokenizer, + + pub fn init( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + tokenizer: zml.tokenizer.Tokenizer, + compiled_model: *inference.CompiledModel, + model_buffers: *model.Buffers, + ) !Session { + const seed: u128 = @intCast(std.Io.Clock.now(.real, io).toNanoseconds()); + var cache_buffers = try compiled_model.params.cache.initBuffers(allocator, io, platform); + errdefer model.Cache.unloadBuffers(&cache_buffers); + var rng_buf = try zml.Tensor.Rng.init().initBuffer(io, platform, seed); + errdefer zml.Tensor.Rng.deinitBuffer(&rng_buf); + const generated_token_slice = try zml.Slice.alloc(allocator, zml.Shape.init(.{ .batch = 1, .seq = 1 }, .u32)); + errdefer generated_token_slice.free(allocator); + + var prefill = try inference.KernelRunner.init(allocator, &compiled_model.prefill, model_buffers); + errdefer prefill.deinit(allocator); + const decode = try inference.KernelRunner.init(allocator, &compiled_model.decode, model_buffers); + + return .{ + .allocator = allocator, + .io = io, + .platform = platform, + .compiled_model = compiled_model, + .prefill = prefill, + .decode = decode, + .tokenizer = tokenizer, + .config = &compiled_model.loaded_model.parsed_config.value, + .seqlen = compiled_model.params.seqlen, + .cache_buffers = cache_buffers, + .rng_buf = rng_buf, + .generated_token_slice = generated_token_slice, + .think_start = tokenizer.tokenId("") orelse unreachable, + .think_end = tokenizer.tokenId("") orelse unreachable, + }; + } + + pub fn deinit(self: *Session) void { + self.prefill.deinit(self.allocator); + self.decode.deinit(self.allocator); + model.Cache.unloadBuffers(&self.cache_buffers); + zml.Tensor.Rng.deinitBuffer(&self.rng_buf); + self.generated_token_slice.free(self.allocator); + } + + pub fn tokenizePrompt(self: *const Session, allocator: std.mem.Allocator, prompt: []const u8) ![]const u32 { + var encoder = try self.tokenizer.encoder(); + defer encoder.deinit(); + + const im_start = self.tokenizer.tokenId("<|im_start|>") orelse return error.NoSuchToken; + const im_end = self.tokenizer.tokenId("<|im_end|>") orelse return error.NoSuchToken; + const newline = self.tokenizer.tokenId("\\n") orelse return error.NoSuchToken; + + var tokens: std.ArrayList(u32) = try .initCapacity(allocator, prompt.len); + try tokens.appendSlice(allocator, &.{ self.config.bos_token_id, im_start }); + const user_tokens = try encoder.encodeAlloc(allocator, "user\n"); + defer allocator.free(user_tokens); + try tokens.appendSlice(allocator, user_tokens); + const prompt_tokens = try encoder.encodeAlloc(allocator, prompt); + defer allocator.free(prompt_tokens); + try tokens.appendSlice(allocator, prompt_tokens); + try tokens.appendSlice(allocator, &.{ im_end, newline, im_start }); + const assistant_tokens = try encoder.encodeAlloc(allocator, "assistant\n"); + defer allocator.free(assistant_tokens); + try tokens.appendSlice(allocator, assistant_tokens); + return tokens.toOwnedSlice(allocator); + } + + pub fn tokenizeTurn(self: *const Session, allocator: std.mem.Allocator, prompt: []const u8) ![]const u32 { + var encoder = try self.tokenizer.encoder(); + defer encoder.deinit(); + + const im_start = self.tokenizer.tokenId("<|im_start|>") orelse return error.NoSuchToken; + const im_end = self.tokenizer.tokenId("<|im_end|>") orelse return error.NoSuchToken; + const newline = self.tokenizer.tokenId("\\n") orelse return error.NoSuchToken; + + var tokens: std.ArrayList(u32) = try .initCapacity(allocator, prompt.len); + try tokens.appendSlice(allocator, &.{ im_end, newline, im_start }); + const user_tokens = try encoder.encodeAlloc(allocator, "user\n"); + defer allocator.free(user_tokens); + try tokens.appendSlice(allocator, user_tokens); + const prompt_tokens = try encoder.encodeAlloc(allocator, prompt); + defer allocator.free(prompt_tokens); + try tokens.appendSlice(allocator, prompt_tokens); + try tokens.appendSlice(allocator, &.{ im_end, newline, im_start }); + const assistant_tokens = try encoder.encodeAlloc(allocator, "assistant\n"); + defer allocator.free(assistant_tokens); + try tokens.appendSlice(allocator, assistant_tokens); + return tokens.toOwnedSlice(allocator); + } + + pub fn runPrefill(self: *Session, all_tokens: []const u32) !void { + const tokens_slice: zml.Slice = try .alloc(self.allocator, .init(.{ .batch = 1, .seq = self.seqlen }, .u32)); + defer tokens_slice.free(self.allocator); + const tokens = tokens_slice.items(u32); + @memset(tokens, self.config.pad_token_id); + @memcpy(tokens[0..all_tokens.len], all_tokens); + + var tokens_buf: zml.Buffer = try .fromSlice(self.io, self.platform, tokens_slice); + defer tokens_buf.deinit(); + + const token_pos_slice: zml.Slice = .init(zml.Shape.init(.{ .batch = 1 }, .u32), std.mem.sliceAsBytes(&[_]u32{0})); + var tokens_pos_buf: zml.Buffer = try .fromSlice(self.io, self.platform, token_pos_slice); + defer tokens_pos_buf.deinit(); + + const active_length_slice: zml.Slice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{@intCast(all_tokens.len)})); + var active_length_buf: zml.Buffer = try .fromSlice(self.io, self.platform, active_length_slice); + defer active_length_buf.deinit(); + + const params = self.compiled_model.params; + var attention_metadata_buffers: zml.Bufferized(zml.attention.Metadata) = switch (params.attention_metadata) { + .stablehlo => try params.attention_metadata.initBuffer(self.io, self.platform), + .flash_attention => |metadata| .{ .flash_attention = switch (metadata) { + .metal => .{ .metal = .{ .num_tokens = try .scalar(self.io, self.platform, all_tokens.len, .u32) } }, + else => try metadata.initBuffer(self.io, self.platform), + } }, + }; + defer zml.attention.Metadata.deinitBuffer(&attention_metadata_buffers); + + var conv_cache_index_buffer: zml.Buffer = try .scalar(self.io, self.platform, 0, .u32); + defer conv_cache_index_buffer.deinit(); + var kv_cache_index_buffer: zml.Buffer = try .scalar(self.io, self.platform, 0, .u32); + defer kv_cache_index_buffer.deinit(); + + var next_token_buf: zml.Buffer = undefined; + inference.run(&self.prefill, .{ + .io = self.io, + .tokens_buf = &tokens_buf, + .next_token_buf = &next_token_buf, + .tokens_pos_buf = &tokens_pos_buf, + .active_length_buf = &active_length_buf, + .rng_buf = &self.rng_buf, + .cache_buffers = &self.cache_buffers, + .attention_metadata_buffers = attention_metadata_buffers, + }, &conv_cache_index_buffer, &kv_cache_index_buffer); + + defer next_token_buf.deinit(); + try next_token_buf.toSlice(self.io, self.generated_token_slice); + } + + pub fn runDecode(self: *Session, all_tokens: *std.ArrayList(u32), stdout: *std.Io.Writer) !void { + var decoder = try self.tokenizer.decoder(); + defer decoder.deinit(); + + var current_token_buffer: zml.Buffer = try .fromSlice(self.io, self.platform, self.generated_token_slice); + defer current_token_buffer.deinit(); + + const active_length_slice: zml.Slice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{1})); + var active_length_buf: zml.Buffer = try .fromSlice(self.io, self.platform, active_length_slice); + defer active_length_buf.deinit(); + + const out_tokens_buffer: []u8 = try self.allocator.alloc(u8, 1024); + defer self.allocator.free(out_tokens_buffer); + + const params = self.compiled_model.params; + var attention_metadata_buffers = try params.attention_metadata.initBuffer(self.io, self.platform); + defer zml.attention.Metadata.deinitBuffer(&attention_metadata_buffers); + + generation: while (true) { + const token_id = self.generated_token_slice.items(u32)[0]; + + if (token_id == self.config.eos_token_id) break :generation; + + const token = try decoder.feedOne(token_id, out_tokens_buffer); + if (self.think_start) |think_start| if (token_id == think_start) { + try stdout.writeAll("\x1b[2m"); + }; + try stdout.writeAll(token); + if (self.think_end) |think_end| if (token_id == think_end) { + try stdout.writeAll("\x1b[0m"); + }; + try stdout.flush(); + + try all_tokens.append(self.allocator, token_id); + if (all_tokens.items.len >= self.seqlen) break :generation; + + const token_pos_slice: zml.Slice = .init(zml.Shape.init(.{ .batch = 1 }, .u32), std.mem.sliceAsBytes(&[_]u32{@intCast(all_tokens.items.len)})); + var token_pos_buffer: zml.Buffer = try .fromSlice(self.io, self.platform, token_pos_slice); + defer token_pos_buffer.deinit(); + + var conv_cache_index_buffer: zml.Buffer = try .scalar(self.io, self.platform, 0, .u32); + defer conv_cache_index_buffer.deinit(); + var kv_cache_index_buffer: zml.Buffer = try .scalar(self.io, self.platform, 0, .u32); + defer kv_cache_index_buffer.deinit(); + + inference.run(&self.decode, .{ + .io = self.io, + .tokens_buf = ¤t_token_buffer, + .next_token_buf = ¤t_token_buffer, + .tokens_pos_buf = &token_pos_buffer, + .active_length_buf = &active_length_buf, + .rng_buf = &self.rng_buf, + .cache_buffers = &self.cache_buffers, + .attention_metadata_buffers = attention_metadata_buffers, + }, &conv_cache_index_buffer, &kv_cache_index_buffer); + + try current_token_buffer.toSlice(self.io, self.generated_token_slice); + } + + try stdout.writeAll(try decoder.finalize(out_tokens_buffer)); + try stdout.flush(); + } +}; diff --git a/examples/llm/models/lfm2_tests.zig b/examples/llm/models/lfm2_tests.zig new file mode 100644 index 00000000..19b28df6 --- /dev/null +++ b/examples/llm/models/lfm2_tests.zig @@ -0,0 +1,319 @@ +const std = @import("std"); + +const zml = @import("zml"); + +const common = @import("common.zig"); +const lfm2 = @import("lfm2.zig"); +const model = @import("lfm2/model.zig"); + +pub const std_options: std.Options = .{ + .log_level = .info, +}; + +const Args = struct { + model: []const u8, + activations: []const u8, + backend: ?enum { stablehlo, cute, nki, metal } = null, + + pub const help = + \\Use lfm2_tests --model= --activations= [options] + \\ + \\ Validate the LFM2 implementation against activation fixtures. + \\ + \\ Options: + \\ --model= Path to the model repository + \\ --activations= Path to activation safetensors + \\ --backend= Attention backend to use + \\ + ; +}; + +pub fn main(init: std.process.Init) !void { + const allocator = init.gpa; + const io = init.io; + const args = zml.stdx.flags.parse(init.minimal.args, Args); + + const platform: *zml.Platform = try .auto(allocator, io, .{}); + defer platform.deinit(allocator, io); + + const repo = try zml.safetensors.resolveModelRepo(io, args.model); + defer repo.close(io); + var registry: zml.safetensors.TensorRegistry = try .fromRepo(allocator, io, repo); + defer registry.deinit(); + const meshes: common.Meshes = try .init(platform); + var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &meshes.all()); + defer store.deinit(); + + var repo_model = try lfm2.LoadedModel.init(allocator, io, repo, store.view(), .{}); + defer repo_model.deinit(allocator); + + var progress = std.Progress.start(io, .{ .root_name = args.model }); + + var model_buffers = try repo_model.loadBuffers(allocator, io, platform, &store, &progress); + defer repo_model.unloadBuffers(&model_buffers, allocator); + + const backend: zml.attention.Backend = if (args.backend) |selected| switch (selected) { + .stablehlo => .{ .stablehlo = {} }, + .cute => .{ .flash_attention = .cute }, + .nki => .{ .flash_attention = .nki }, + .metal => .{ .flash_attention = .metal }, + } else zml.attention.Backend.auto(platform); + const params = lfm2.CompilationParameters.init( + repo_model.inner, + repo_model.parsed_config.value, + repo_model.parsed_config.value.max_position_embeddings, + backend, + meshes, + ); + progress.end(); + + try run(allocator, io, platform, args.activations, repo_model.parsed_config.value, repo_model.inner, &model_buffers, params.attention_metadata, params.attention_parameters, meshes); +} + +pub fn run( + allocator: std.mem.Allocator, + io: std.Io, + platform: *zml.Platform, + activations_path: []const u8, + config: lfm2.Config, + mdl: lfm2.Model, + model_buffers: *lfm2.Buffers, + attention_metadata: zml.attention.Metadata, + attention_parameters: zml.attention.Parameters, + meshes: common.Meshes, +) !void { + var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, activations_path); + defer registry.deinit(); + + var activation_store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &meshes.all()); + defer activation_store.deinit(); + + var ctx = TestContext{ + .allocator = allocator, + .io = io, + .platform = platform, + .activations_store = &activation_store, + .attention_metadata = attention_metadata, + .attention_parameters = attention_parameters, + .mesh = platform.replicated_mesh, + }; + + try ctx.testLayer("embed_tokens", .{ .batch, .seq }, mdl.embed_tokens, model_buffers.embed_tokens, .{}); + + var num_attn_layers: usize = 0; + var num_conv_layers: usize = 0; + for (0..config.num_hidden_layers) |i| { + const kind = std.meta.stringToEnum(model.OperatorKind, config.layer_types[i]) orelse unreachable; + const layer = mdl.layers[i]; + const layer_buffers = model_buffers.layers[i]; + switch (kind) { + .conv => { + try ctx.testLayerPrint("layers.{d}.conv.in_proj", .{i}, .{ .batch, .seq, .d }, layer.operator.conv.in_proj, layer_buffers.operator.conv.in_proj, .{}); + try ctx.testLayerPrint("layers.{d}.conv.out_proj", .{i}, .{ .batch, .seq, .d }, layer.operator.conv.out_proj, layer_buffers.operator.conv.out_proj, .{}); + try ctx.testLayerPrint("layers.{d}.feed_forward.w1", .{i}, .{ .batch, .seq, .d }, layer.feed_forward.w1, layer_buffers.feed_forward.w1, .{}); + try ctx.testLayerPrint("layers.{d}.feed_forward.w2", .{i}, .{ .batch, .seq, .d }, layer.feed_forward.w2, layer_buffers.feed_forward.w2, .{}); + try ctx.testLayerPrint("layers.{d}.feed_forward.w3", .{i}, .{ .batch, .seq, .d }, layer.feed_forward.w3, layer_buffers.feed_forward.w3, .{}); + try ctx.testLayerPrint("layers.{d}.feed_forward", .{i}, .{ .batch, .seq, .d }, layer.feed_forward, layer_buffers.feed_forward, .{}); + try ctx.testConvLayer(i, num_conv_layers, mdl.layers[i].operator.conv, model_buffers.layers[i].operator.conv, .{}); + num_conv_layers += 1; + }, + .full_attention => { + try ctx.testLayerPrint("layers.{d}.self_attn.k_proj", .{i}, .{ .batch, .seq, .d }, layer.operator.self_attn.k_proj, layer_buffers.operator.self_attn.k_proj, .{}); + try ctx.testLayerPrint("layers.{d}.self_attn.q_proj", .{i}, .{ .batch, .seq, .d }, layer.operator.self_attn.q_proj, layer_buffers.operator.self_attn.q_proj, .{}); + try ctx.testLayerPrint("layers.{d}.self_attn.out_proj", .{i}, .{ .batch, .seq, .d }, layer.operator.self_attn.out_proj, layer_buffers.operator.self_attn.out_proj, .{}); + try ctx.testLayerPrint("layers.{d}.self_attn.v_proj", .{i}, .{ .batch, .seq, .d }, layer.operator.self_attn.v_proj, layer_buffers.operator.self_attn.v_proj, .{}); + try ctx.testLayerPrint("layers.{d}.feed_forward.w1", .{i}, .{ .batch, .seq, .d }, layer.feed_forward.w1, layer_buffers.feed_forward.w1, .{}); + try ctx.testLayerPrint("layers.{d}.feed_forward.w2", .{i}, .{ .batch, .seq, .d }, layer.feed_forward.w2, layer_buffers.feed_forward.w2, .{}); + try ctx.testLayerPrint("layers.{d}.feed_forward.w3", .{i}, .{ .batch, .seq, .d }, layer.feed_forward.w3, layer_buffers.feed_forward.w3, .{}); + try ctx.testLayerPrint("layers.{d}.feed_forward", .{i}, .{ .batch, .seq, .d }, layer.feed_forward, layer_buffers.feed_forward, .{}); + try ctx.testAttnLayer(i, num_attn_layers, mdl.layers[i].operator.self_attn, model_buffers.layers[i].operator.self_attn, .{}); + num_attn_layers += 1; + }, + } + } +} + +const TestContext = struct { + allocator: std.mem.Allocator, + io: std.Io, + platform: *zml.Platform, + activations_store: *zml.io.TensorStore, + attention_metadata: zml.attention.Metadata, + attention_parameters: zml.attention.Parameters, + mesh: *const zml.Mesh, + + fn testLayerPrint(self: *TestContext, comptime name_fmt: []const u8, name_args: anytype, tagz: anytype, layer: anytype, layer_buffers: anytype, opts: zml.testing.CompareOpts) !void { + const name = try std.fmt.allocPrint(self.allocator, name_fmt, name_args); + defer self.allocator.free(name); + try self.testLayer(name, tagz, layer, layer_buffers, opts); + } + + fn testLayer(self: *TestContext, name: []const u8, tagz: anytype, layer: anytype, layer_buffers: anytype, opts: zml.testing.CompareOpts) !void { + std.log.info("Testing layer: {s}", .{name}); + + const in_key = try std.fmt.allocPrint(self.allocator, "{s}.in", .{name}); + defer self.allocator.free(in_key); + var in_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, in_key); + defer in_buffer.deinit(); + const in_tensor = zml.Tensor.fromShape(in_buffer.shape()).withTags(tagz); + + const out_key = try std.fmt.allocPrint(self.allocator, "{s}.out", .{name}); + defer self.allocator.free(out_key); + var out_buffer_expected = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, out_key); + defer out_buffer_expected.deinit(); + + const exe = if (comptime @TypeOf(layer) == model.TokenEmbedding) + try self.platform.compileFn(self.allocator, self.io, model.TokenEmbedding.forward, .{.{ .embedding = layer, .tokens = in_tensor }}, .{ .meshes = &.{self.mesh} }) + else + try self.platform.compileFn(self.allocator, self.io, @TypeOf(layer).forward, .{ layer, in_tensor }, .{ .meshes = &.{self.mesh} }); + defer exe.deinit(); + + var args = try exe.args(self.allocator); + defer args.deinit(self.allocator); + args.set(.{ layer_buffers, in_buffer }); + + var res = try exe.results(self.allocator); + defer res.deinit(self.allocator); + + exe.call(self.io, args, &res, .{}); + + var out_result = res.get(zml.Buffer); + defer out_result.deinit(); + + try zml.testing.expectClose(self.io, out_result, out_buffer_expected, opts); + std.log.info("Layer {s} passed!", .{name}); + } + + fn testConvLayer(self: *TestContext, ix: usize, cache_ix: usize, layer: model.ShortConv, layer_buffers: zml.Bufferized(model.ShortConv), opts: zml.testing.CompareOpts) !void { + const name = try std.fmt.allocPrint(self.allocator, "layers.{d}.conv", .{ix}); + defer self.allocator.free(name); + std.log.info("Testing layer: {s}", .{name}); + + const in_key = try std.fmt.allocPrint(self.allocator, "{s}.in", .{name}); + defer self.allocator.free(in_key); + var in_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, in_key); + defer in_buffer.deinit(); + const in_tensor = zml.Tensor.fromShape(in_buffer.shape()).withTags(.{ .batch, .seq, .d }); + + const cache_key = try std.fmt.allocPrint(self.allocator, "{s}.cache", .{name}); + defer self.allocator.free(cache_key); + var cache_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, cache_key); + defer cache_buffer.deinit(); + const cache_tensor = zml.Tensor.fromShape(cache_buffer.shape()).withTags(.{ .layer, .batch, .seq, .d }); + + const cache_pos_key = try std.fmt.allocPrint(self.allocator, "{s}.cache_position", .{name}); + defer self.allocator.free(cache_pos_key); + var cache_pos_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, cache_pos_key); + defer cache_pos_buffer.deinit(); + const cache_pos_tensor = zml.Tensor.fromShape(cache_pos_buffer.shape()).withTags(.{.batch}); + + const out_key = try std.fmt.allocPrint(self.allocator, "{s}.out", .{name}); + defer self.allocator.free(out_key); + var out_buffer_expected = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, out_key); + defer out_buffer_expected.deinit(); + + const actual_seq_len_tensor: zml.Tensor = .init(.{}, .u32); + const actual_seq_len: u32 = @intCast(in_tensor.dim(.seq)); + const cache_index_tensor: zml.Tensor = .init(.{}, .u32); + + const exe = try self.platform.compileFn(self.allocator, self.io, @TypeOf(layer).forward, .{ layer, in_tensor, cache_pos_tensor, actual_seq_len_tensor, model.ConvCache{ .state = cache_tensor }, cache_index_tensor, model.ConvParameters{ .is_prefill = false } }, .{ .meshes = &.{self.mesh} }); + defer exe.deinit(); + + const conv_cache: zml.Bufferized(model.ConvCache) = .{ .state = cache_buffer }; + + const actual_seq_len_slice: zml.Slice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{actual_seq_len})); + var actual_seq_len_buf: zml.Buffer = try .fromSlice(self.io, self.platform, actual_seq_len_slice); + defer actual_seq_len_buf.deinit(); + + const cache_index_slice: zml.Slice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{@intCast(cache_ix)})); + var cache_index_buf: zml.Buffer = try .fromSlice(self.io, self.platform, cache_index_slice); + defer cache_index_buf.deinit(); + + var runner = try exe.runner(self.allocator); + defer runner.deinit(self.allocator); + runner.run( + self.io, + .{ layer_buffers, in_buffer, cache_pos_buffer, actual_seq_len_buf, conv_cache, cache_index_buf }, + .{ &in_buffer, &cache_buffer }, + .{}, + ); + + try zml.testing.expectClose(self.io, in_buffer, out_buffer_expected, opts); + std.log.info("Layer {s} passed!", .{name}); + } + + fn testAttnLayer(self: *TestContext, ix: usize, cache_ix: usize, layer: model.Attention, layer_buffers: zml.Bufferized(model.Attention), opts: zml.testing.CompareOpts) !void { + const name = try std.fmt.allocPrint(self.allocator, "layers.{d}.self_attn", .{ix}); + defer self.allocator.free(name); + std.log.info("Testing layer: {s}", .{name}); + + const in_key = try std.fmt.allocPrint(self.allocator, "{s}.in", .{name}); + defer self.allocator.free(in_key); + var in_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, in_key); + defer in_buffer.deinit(); + const in_tensor = zml.Tensor.fromShape(in_buffer.shape()).withTags(.{ .batch, .seq, .d }); + + const key_cache_key = try std.fmt.allocPrint(self.allocator, "{s}.cache.key", .{name}); + defer self.allocator.free(key_cache_key); + var key_cache_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, key_cache_key); + defer key_cache_buffer.deinit(); + const key_cache_tensor = zml.Tensor.fromShape(key_cache_buffer.shape()).withTags(.{ .layer, .batch, .h, .k, .hd }); + + const value_cache_key = try std.fmt.allocPrint(self.allocator, "{s}.cache.value", .{name}); + defer self.allocator.free(value_cache_key); + var value_cache_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, value_cache_key); + defer value_cache_buffer.deinit(); + const value_cache_tensor = zml.Tensor.fromShape(value_cache_buffer.shape()).withTags(.{ .layer, .batch, .h, .k, .hd }); + + const cache_pos_key = try std.fmt.allocPrint(self.allocator, "{s}.cache_position", .{name}); + defer self.allocator.free(cache_pos_key); + var cache_pos_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, cache_pos_key); + defer cache_pos_buffer.deinit(); + const cache_pos_tensor = zml.Tensor.fromShape(cache_pos_buffer.shape()).withTags(.{.batch}); + + const out_key = try std.fmt.allocPrint(self.allocator, "{s}.out", .{name}); + defer self.allocator.free(out_key); + var out_buffer_expected = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, out_key); + defer out_buffer_expected.deinit(); + + const cache_index_tensor: zml.Tensor = .init(.{}, .u32); + + const exe = try self.platform.compileFn(self.allocator, self.io, @TypeOf(layer).forward, .{ layer, in_tensor, cache_pos_tensor, model.KvCache{ .k = key_cache_tensor, .v = value_cache_tensor }, cache_index_tensor, self.attention_metadata, self.attention_parameters }, .{ .meshes = &.{self.mesh} }); + defer exe.deinit(); + + const kv_cache: zml.Bufferized(model.KvCache) = .{ .k = key_cache_buffer, .v = value_cache_buffer }; + + var attention_metadata_buffers = try self.attention_metadata.initBuffer(self.io, self.platform); + defer zml.attention.Metadata.deinitBuffer(&attention_metadata_buffers); + + var cache_index_buf: zml.Buffer = try .scalar(self.io, self.platform, cache_ix, .u32); + defer cache_index_buf.deinit(); + + var runner = try exe.runner(self.allocator); + defer runner.deinit(self.allocator); + runner.run( + self.io, + .{ layer_buffers, in_buffer, cache_pos_buffer, kv_cache, cache_index_buf, attention_metadata_buffers }, + .{ &in_buffer, &key_cache_buffer, &value_cache_buffer }, + .{}, + ); + + try zml.testing.expectClose(self.io, in_buffer, out_buffer_expected, opts); + std.log.info("Layer {s} passed!", .{name}); + } +}; + +fn loadBufferFromStore(allocator: std.mem.Allocator, io: anytype, platform: *zml.Platform, store: *zml.io.TensorStore, key: []const u8) !zml.Buffer { + const shape = store.view().getShape(key) orelse return error.NotFound; + + const host_bytes = try allocator.alloc(u8, shape.byteSize()); + defer allocator.free(host_bytes); + + var io_buffer: [8 * 1024]u8 = undefined; + var reader = try store.view().getReader(key, io, &io_buffer); + defer reader.deinit(); + + _ = try reader.interface.readSliceAll(host_bytes); + + return zml.Buffer.fromBytes(io, platform, shape, host_bytes); +} diff --git a/examples/llm/models/llama.zig b/examples/llm/models/llama.zig new file mode 100644 index 00000000..79f69e36 --- /dev/null +++ b/examples/llm/models/llama.zig @@ -0,0 +1,9 @@ +pub const inference = @import("llama/inference.zig"); +pub const CompilationParameters = inference.CompilationParameters; +pub const model = @import("llama/model.zig"); +pub const Config = model.Config; +pub const Buffers = model.Buffers; +pub const Model = model.Model; +pub const LoadedModel = model.LoadedModel; +pub const session = @import("llama/session.zig"); +pub const Session = session.Session; diff --git a/examples/llm/models/llama/inference.zig b/examples/llm/models/llama/inference.zig new file mode 100644 index 00000000..51ea3f9c --- /dev/null +++ b/examples/llm/models/llama/inference.zig @@ -0,0 +1,209 @@ +const std = @import("std"); + +const zml = @import("zml"); + +const common = @import("../common.zig"); +const model = @import("model.zig"); + +const log = std.log.scoped(.llama); +const Phase = common.Phase; + +pub const CompilationParameters = struct { + prefill_tokens: zml.Tensor, + decode_tokens: zml.Tensor, + token_index: zml.Tensor, + kv_cache: model.KvCache, + rng: zml.Tensor.Rng, + attention_metadata: zml.attention.Metadata, + prefill_attention_parameters: zml.attention.Parameters, + decode_attention_parameters: zml.attention.Parameters, + seqlen: usize, + meshes: common.Meshes, + + pub fn init(mdl: model.Model, config: model.Config, seqlen: u32, backend: zml.attention.Backend, meshes: common.Meshes) CompilationParameters { + const head_dim = config.head_dim orelse @divExact(config.hidden_size, config.num_attention_heads); + + return .{ + .prefill_tokens = .init(.{ .s = seqlen }, .u32), + .decode_tokens = .init(.{ .s = 1 }, .u32), + .token_index = .init(.{}, .u32), + .kv_cache = .init(.init(.{ + .layer = mdl.model.layers.len, + .k = seqlen, + .h = config.num_key_value_heads, + .hd = head_dim, + }, mdl.model.embed_tokens.weight.dtype()), meshes.model), + .rng = .init(), + .attention_metadata = .init(.fromBackend(backend)), + .prefill_attention_parameters = .init(.fromBackend(backend)), + .decode_attention_parameters = .init(.fromBackend(backend)), + .seqlen = seqlen, + .meshes = meshes, + }; + } +}; + +pub const CompilationOptions = CompilationParameters; + +pub const Args = struct { + io: std.Io, + tokens_buf: *zml.Buffer, + next_token_buf: *zml.Buffer, + token_index_buf: *zml.Buffer, + active_length_buf: *zml.Buffer, + kv_cache_buffers: *zml.Bufferized(model.KvCache), + rng_buffers: *zml.Bufferized(zml.Tensor.Rng), + attention_metadata_buffers: *const zml.Bufferized(zml.attention.Metadata), +}; + +pub const CompiledModel = struct { + loaded_model: *const model.LoadedModel, + prefill: KernelExe, + decode: KernelExe, + params: CompilationParameters, + + pub fn init( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + loaded_model: *const model.LoadedModel, + llama_model: model.Model, + parameters: CompilationParameters, + progress: *std.Progress.Node, + ) !CompiledModel { + const prefill = try compileKernel(allocator, io, platform, llama_model, parameters, @intCast(parameters.prefill_tokens.dim(.s)), parameters.prefill_attention_parameters, .prefill, progress); + errdefer prefill.deinit(); + const decode = try compileKernel(allocator, io, platform, llama_model, parameters, @intCast(parameters.decode_tokens.dim(.s)), parameters.decode_attention_parameters, .decode, progress); + + return .{ + .loaded_model = loaded_model, + .prefill = prefill, + .decode = decode, + .params = parameters, + }; + } + + pub fn deinit(self: *CompiledModel) void { + self.prefill.deinit(); + self.decode.deinit(); + } +}; + +pub const Inference = CompiledModel; + +const Forward = struct { + pub const Input = struct { + weights: model.Model, + tokens: zml.Tensor, + token_index: zml.Tensor, + active_length: zml.Tensor, + kv_cache: model.KvCache, + rng: zml.Tensor.Rng, + attention_metadata: zml.attention.Metadata, + attention_parameters: zml.attention.Parameters, + }; + + pub const Output = struct { + tokens: zml.Tensor, + kv_cache: model.KvCache, + rng: zml.Tensor.Rng, + }; + + pub fn forward(input: Input) Output { + var hidden = model.EmbedTokens.forward(.{ + .embedding = .{ .embed_tokens = input.weights.model.embed_tokens }, + .tokens = input.tokens, + }).hidden; + var kv_cache = input.kv_cache; + for (input.weights.model.layers, 0..) |layer, index| { + const result = model.TransformerLayer.forward(.{ + .layer = layer, + .hidden = hidden, + .token_index = input.token_index, + .kv_cache = kv_cache, + .kv_cache_index = .scalar(@as(u32, @intCast(index)), .u32), + .attention_metadata = input.attention_metadata, + .attention_parameters = input.attention_parameters, + }); + hidden = result.hidden; + kv_cache = result.kv_cache; + } + // LmHead owns final normalization; Llama.forward would apply it twice. + const result = model.LmHead.forward(.{ + .lm_head = .init(input.weights), + .hidden = hidden, + .active_length = input.active_length, + .rng = input.rng, + }); + return .{ + .tokens = if (input.tokens.dim(.s) == 1) result.tokens.reuseBuffer(input.tokens) else result.tokens, + .kv_cache = kv_cache.reuseBuffer(input.kv_cache), + .rng = .{ ._state = result.rng._state.reuseBuffer(input.rng._state) }, + }; + } +}; + +pub const KernelExe = zml.FnExe(Forward.forward); + +pub const KernelRunner = struct { + forward: KernelExe.Runner(.{.weights}), + + pub fn init(allocator: std.mem.Allocator, exe: *const KernelExe, buffers: *const model.Buffers) !KernelRunner { + return .{ .forward = try .init(exe, allocator, .{ .weights = buffers.* }) }; + } + + pub fn deinit(self: *KernelRunner, allocator: std.mem.Allocator) void { + self.forward.deinit(allocator); + } +}; + +pub fn run(runner: *KernelRunner, args: Args) void { + runner.forward.run(args.io, .{ + .inputs = .{ + .tokens = args.tokens_buf.*, + .token_index = args.token_index_buf.*, + .active_length = args.active_length_buf.*, + .kv_cache = args.kv_cache_buffers.*, + .rng = args.rng_buffers.*, + .attention_metadata = args.attention_metadata_buffers.*, + }, + .outputs = .{ + .tokens = args.next_token_buf, + .kv_cache = args.kv_cache_buffers, + .rng = args.rng_buffers, + }, + }, .{}); +} + +fn compileKernel( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + llama_model: model.Model, + parameters: CompilationOptions, + seqlen: usize, + attention_parameters: zml.attention.Parameters, + phase: Phase, + progress: *std.Progress.Node, +) !KernelExe { + progress.increaseEstimatedTotalItems(1); + var node = progress.start(phase.startMessage("forward"), 1); + defer node.end(); + + const from: std.Io.Timestamp = .now(io, .awake); + defer phase.logCompileDone(log, "forward", io, from); + + return KernelExe.compile(allocator, io, platform, .{ + .meshes = ¶meters.meshes.all(), + .program_name = phase.programName("llama", "forward"), + }, .{.{ + .weights = llama_model, + .tokens = zml.Tensor.init(.{ .s = seqlen }, .u32), + .token_index = parameters.token_index, + .active_length = .init(.{}, .u32), + .kv_cache = parameters.kv_cache, + .rng = parameters.rng, + .attention_metadata = parameters.attention_metadata, + .attention_parameters = attention_parameters, + }}); +} diff --git a/examples/llm/models/llama/model.zig b/examples/llm/models/llama/model.zig new file mode 100644 index 00000000..8a6a09e9 --- /dev/null +++ b/examples/llm/models/llama/model.zig @@ -0,0 +1,651 @@ +const std = @import("std"); + +const zml = @import("zml"); +const stdx = zml.stdx; + +const common = @import("../common.zig"); +const inference = @import("inference.zig"); + +const log = std.log.scoped(.llama); + +pub const Config = struct { + bos_token_id: u32, + eos_token_id: EosTokens, + head_dim: ?u32 = null, + hidden_size: u32, + num_hidden_layers: u32, + num_attention_heads: u32, + num_key_value_heads: u32, + rope_theta: f32, + max_position_embeddings: u32, + rms_norm_eps: f32, + hf_rope_impl: bool = true, + tie_word_embeddings: bool = false, + rope_scaling: zml.nn.RopeOpts.Scaling = .{ .default = .{} }, + + pub const EosTokens = union(enum) { + int: u32, + ints: []u32, + + const Helpers = stdx.json.UnionHelpers(@This()); + pub const jsonParse = Helpers.jsonParse; + pub const jsonParseFromValue = Helpers.jsonParseFromValue; + pub const jsonStringify = Helpers.jsonStringify; + }; +}; + +const Options = struct { + sampling_strategy: ?zml.nn.SamplingStrategy, + max_seq_len: u32, +}; + +pub const LoadedModel = struct { + inner: Model, + parsed_config: std.json.Parsed(Config), + + pub fn init( + allocator: std.mem.Allocator, + io: std.Io, + repo: std.Io.Dir, + store: zml.io.TensorStore.View, + generation: common.GenerationOptions, + ) !LoadedModel { + const parsed_config = try common.parseConfig(Config, allocator, io, repo); + errdefer parsed_config.deinit(); + + const options: Options = .{ + .sampling_strategy = generation.sampling_strategy, + .max_seq_len = parsed_config.value.max_position_embeddings, + }; + + return .{ + .inner = try .init(allocator, store, parsed_config.value, options), + .parsed_config = parsed_config, + }; + } + + pub fn deinit(self: *LoadedModel, allocator: std.mem.Allocator) void { + self.inner.deinit(allocator); + self.parsed_config.deinit(); + } + + pub fn loadBuffers( + self: *const LoadedModel, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + store: *zml.io.TensorStore, + progress: *std.Progress.Node, + ) !Buffers { + progress.increaseEstimatedTotalItems(store.view().count()); + const now: std.Io.Timestamp = .now(io, .awake); + + var buffers = try zml.mem.bufferize(allocator, Model, &self.inner); + errdefer self.unloadBuffers(&buffers, allocator); + + var loader: zml.io.Loader = try .init(allocator, platform, .{ + .dma_chunks = 32, + .dma_chunk_size = 256 * zml.MiB, + .parallelism = 16, + }); + defer loader.deinit(); + + try loader.load(io, Model, &self.inner, &buffers, store, .{ .progress = progress }); + try loader.await(io); + + const took = now.untilNow(io, .awake); + const total_bytes: u64 = loader.bytes_loaded.raw; + const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); + log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); + + return buffers; + } + + pub fn unloadBuffers(_: *const LoadedModel, buffers: *Buffers, allocator: std.mem.Allocator) void { + if (buffers.lm_head) |*lm_head| zml.nn.Linear.unloadBuffers(lm_head); + Llama.unloadBuffers(&buffers.model, allocator); + } + + pub fn compile( + self: *const LoadedModel, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + backend: zml.attention.Backend, + meshes: common.Meshes, + seqlen: usize, + progress: *std.Progress.Node, + ) !inference.CompiledModel { + const params = inference.CompilationParameters.init(self.inner, self.parsed_config.value, @intCast(seqlen), backend, meshes); + + return inference.CompiledModel.init(allocator, io, platform, self, self.inner, params, progress); + } +}; + +pub const Buffers = zml.Bufferized(Model); + +/// Llama architecture, using huggingface transformers naming. +/// Dimensions of activations: {.b, .s, .d} +pub const Model = struct { + lm_head: ?zml.nn.Linear, + model: Llama, + + gen_opts: zml.nn.SamplingStrategy = .{}, + config: Config, + + pub fn init( + allocator: std.mem.Allocator, + store: zml.io.TensorStore.View, + config: Config, + options: Options, + ) !Model { + const lm_head: ?zml.nn.Linear = if (store.withPrefix("lm_head").maybeCreateTensor( + "weight", + .{ .dout, .d }, + .model, + .{ .dout = .model, .d = .replicated }, + )) |weight| + .init(weight, null, .d) + else + null; + + return .{ + .lm_head = lm_head, + .model = try .init(allocator, store.withPrefix("model"), config), + .gen_opts = options.sampling_strategy orelse .{}, + .config = config, + }; + } + + pub fn deinit(self: Model, allocator: std.mem.Allocator) void { + self.model.deinit(allocator); + } + + pub fn loadBuffers( + self: *const Model, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + store: *zml.io.TensorStore, + progress: *std.Progress.Node, + ) !zml.Bufferized(Model) { + progress.increaseEstimatedTotalItems(store.view().count()); + const now: std.Io.Timestamp = .now(io, .awake); + + var buffers = try zml.mem.bufferize(allocator, Model, self); + errdefer Model.unloadBuffers(&buffers, allocator); + + var loader: zml.io.Loader = try .init(allocator, platform, .{ + .dma_chunks = 32, + .dma_chunk_size = 256 * zml.MiB, + .parallelism = 16, + }); + defer loader.deinit(); + + loader.load(io, Model, self, &buffers, store); + try loader.await(io); + + const took = now.untilNow(io, .awake); + const total_bytes: u64 = loader.bytes_loaded.raw; + const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); + log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); + + return buffers; + } + + pub fn unloadBuffers(self: *zml.Bufferized(Model), allocator: std.mem.Allocator) void { + if (self.lm_head) |*lm_head| zml.nn.Linear.unloadBuffers(lm_head); + Llama.unloadBuffers(&self.model, allocator); + } + + fn lmHead(self: Model) LmHead { + return .init(self); + } + + /// Predicts the token at `token_index` position. + /// Returns: + /// - updated `tokens`, + /// - updated KV cache + /// - a Rng state to allow for probabilistic generation + pub fn forward( + self: Model, + tokens_: zml.Tensor, + token_index: zml.Tensor, + active_length: zml.Tensor, + kv_cache: KvCache, + rng: zml.Tensor.Rng, + attention_metadata: zml.attention.Metadata, + attention_parameters: zml.attention.Parameters, + ) struct { zml.Tensor, KvCache, zml.Tensor.Rng } { + const tokens = tokens_.withPartialTags(.{.s}); + + const out, const updated_kv_cache = self.model.forward( + tokens, + token_index, + kv_cache, + attention_metadata, + attention_parameters, + ); + + const sample = LmHead.forward(.{ + .lm_head = self.lmHead(), + .hidden = out, + .active_length = active_length, + .rng = rng, + }); + + return .{ sample.tokens, updated_kv_cache, sample.rng }; + } +}; + +const Llama = struct { + embed_tokens: zml.nn.TokenEmbedding, + norm: RmsNorm, + layers: []TransformerLayer, + + pub fn init(allocator: std.mem.Allocator, store: zml.io.TensorStore.View, config: Config) !Llama { + const layers = try allocator.alloc(TransformerLayer, config.num_hidden_layers); + errdefer allocator.free(layers); + + for (layers, 0..) |*layer, i| { + layer.* = try .init(store.withPrefix("layers").withLayer(i), config); + } + + return .{ + .embed_tokens = .{ .weight = store.createTensor( + "embed_tokens.weight", + .{ .voc, .d }, + .model, + .{ .voc = .replicated, .d = .model }, + ) }, + .norm = .{ + .weight = store.withPrefix("norm").createTensor("weight", .{.d}, .model, .{ .d = .replicated }), + .eps = config.rms_norm_eps, + }, + .layers = layers, + }; + } + + pub fn deinit(self: Llama, allocator: std.mem.Allocator) void { + allocator.free(self.layers); + } + + pub fn unloadBuffers(self: *zml.Bufferized(Llama), allocator: std.mem.Allocator) void { + self.embed_tokens.weight.deinit(); + RmsNorm.unloadBuffers(&self.norm); + for (self.layers) |*layer| { + TransformerLayer.unloadBuffers(layer); + } + allocator.free(self.layers); + } + + /// Forward one token, using KV cache for previous tokens. + /// Returns result and updated KV cache. + pub fn forward( + self: Llama, + tokens: zml.Tensor, + token_index: zml.Tensor, + kv_cache: KvCache, + attention_metadata: zml.attention.Metadata, + attention_parameters: zml.attention.Parameters, + ) struct { zml.Tensor, KvCache } { + const embeds = self.embed_tokens.forward(tokens).withPartialTags(.{.d}); + var hidden = embeds; + var updated_kv_cache = kv_cache; + var kv_cache_index = zml.Tensor.scalar(0, .u32); + + for (self.layers) |layer| { + const result = TransformerLayer.forward(.{ + .layer = layer, + .hidden = hidden, + .token_index = token_index, + .kv_cache = updated_kv_cache, + .kv_cache_index = kv_cache_index, + .attention_metadata = attention_metadata, + .attention_parameters = attention_parameters, + }); + hidden = result.hidden; + updated_kv_cache = result.kv_cache; + kv_cache_index = kv_cache_index.add(zml.Tensor.scalar(@as(u32, 1), .u32)); + } + + return .{ self.norm.forward(hidden), updated_kv_cache.reuseBuffer(kv_cache) }; + } +}; + +pub const EmbedTokens = struct { + embed_tokens: zml.nn.TokenEmbedding, + + pub const Input = struct { + embedding: EmbedTokens, + tokens: zml.Tensor, + }; + + pub const Output = struct { + hidden: zml.Tensor, + }; + + pub fn forward(input: Input) Output { + const tokens = input.tokens.withPartialTags(.{.s}); + return .{ .hidden = input.embedding.embed_tokens.forward(tokens) + .withPartialTags(.{.d}) + .replicate() }; + } +}; + +pub const LmHead = struct { + lm_head: ?zml.nn.Linear, + embed_tokens: zml.nn.TokenEmbedding, + norm: RmsNorm, + gen_opts: zml.nn.SamplingStrategy, + + pub fn init(mdl: Model) LmHead { + return .{ + .lm_head = mdl.lm_head, + .embed_tokens = mdl.model.embed_tokens, + .norm = mdl.model.norm, + .gen_opts = mdl.gen_opts, + }; + } + + pub const Input = struct { + lm_head: LmHead, + hidden: zml.Tensor, + active_length: zml.Tensor, + rng: zml.Tensor.Rng, + }; + + pub const Output = struct { + tokens: zml.Tensor, + rng: zml.Tensor.Rng, + }; + + pub fn forward(input: Input) Output { + const self = input.lm_head; + const last_index = input.active_length.convert(.i32).addConstant(-1); + const hidden = self.norm.forward(input.hidden.withPartialTags(.{ .s, .d }).slice(.s, .dyn(last_index, 1))); + + var logits = blk: { + if (self.lm_head) |lm_head| { + break :blk lm_head.forward(hidden, hidden.dtype()).rename(.{ .dout = .d }); + } else { + break :blk self.embed_tokens.weight.withTags(.{ .voc, .d }).dot(hidden, .d); + } + }; + + if (logits.shape().hasTag(.voc) == null) + logits = logits.rename(.{ .d = .voc }); + + const next_tokens, const new_rng = zml.nn.sampleTokens(logits, self.gen_opts, input.rng); + return .{ .tokens = next_tokens.convert(.u32), .rng = new_rng }; + } +}; + +pub const TransformerLayer = struct { + input_layernorm: RmsNorm, + self_attn: SelfAttn, + post_attention_layernorm: RmsNorm, + mlp: Mlp, + + pub fn init(store: zml.io.TensorStore.View, config: Config) !TransformerLayer { + return .{ + .input_layernorm = .init(store.withPrefix("input_layernorm"), config.rms_norm_eps), + .self_attn = try .init(store.withPrefix("self_attn"), config), + .post_attention_layernorm = .init(store.withPrefix("post_attention_layernorm"), config.rms_norm_eps), + .mlp = .init(store.withPrefix("mlp")), + }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(TransformerLayer)) void { + RmsNorm.unloadBuffers(&self.input_layernorm); + SelfAttn.unloadBuffers(&self.self_attn); + RmsNorm.unloadBuffers(&self.post_attention_layernorm); + Mlp.unloadBuffers(&self.mlp); + } + + pub const Input = struct { + layer: TransformerLayer, + hidden: zml.Tensor, + token_index: zml.Tensor, + kv_cache: KvCache, + kv_cache_index: zml.Tensor, + attention_metadata: zml.attention.Metadata, + attention_parameters: zml.attention.Parameters, + }; + + pub const Output = struct { + hidden: zml.Tensor, + kv_cache: KvCache, + }; + + pub fn forward(input: Input) Output { + const self = input.layer; + const x0 = input.hidden; + // Self Attention + //log.debug("TransformerLayer({f}) -> {f}", .{ x0, self.input_layernorm.forward(x0) }); + stdx.debug.assert(x0.rank() >= 2 and x0.shape().hasTags(.{ .s, .d }), "TransformerLayer expected input shape: {{..., .s, .d}}, received: {f}", .{x0}); + + // Keep the residual stream replicated to avoid repeated gathers before q/k/v. + const x0_replicated = x0.withPartitioning(.model, .{ .d = .replicated }); + const x0_normalized = self.input_layernorm.forward(x0_replicated); + const delta0, const updated_kv_cache = self.self_attn.forward( + x0_normalized, + input.token_index, + input.kv_cache, + input.kv_cache_index, + input.attention_metadata, + input.attention_parameters, + ); + + // Fully Connected + const x1 = x0_replicated.add(delta0).withPartitioning(.model, .{ .d = .replicated }); + const x1_normalized = self.post_attention_layernorm.forward(x1); + const x2 = self.mlp.forward(x1_normalized) + .rename(.{ .dout = .d }) + .withPartitioning(.model, .{ .d = .replicated }) + .add(x1) + .withPartitioning(.model, .{ .d = .replicated }); + + return .{ .hidden = x2.reuseBuffer(x0), .kv_cache = updated_kv_cache }; + } +}; + +const RmsNorm = struct { + weight: zml.Tensor, + eps: f32, + + pub fn init(store: zml.io.TensorStore.View, eps: f32) RmsNorm { + return .{ + .weight = store.createTensor("weight", .{.d}, .model, .{ .d = .replicated }), + .eps = eps, + }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(RmsNorm)) void { + self.weight.deinit(); + } + + /// L2 normalization of input tensor along `.d` axis. + pub fn forward(self: RmsNorm, input: zml.Tensor) zml.Tensor { + const x = if (input.shape().isFullyTagged()) input else input.withPartialTags(.{.d}); + const normalized = zml.nn.rmsNorm(x, .d, self.eps); + return normalized.mul(self.weight.convert(x.dtype()).withTags(.{.d}).broad(x.shape())); + } +}; + +const Mlp = struct { + up_proj: zml.nn.Linear, // (dim -> hidden_dim) + gate_proj: zml.nn.Linear, // (dim -> hidden_dim) + down_proj: zml.nn.Linear, // (hidden_dim -> dim) + + pub fn init(store: zml.io.TensorStore.View) Mlp { + return .{ + .up_proj = .init(store.createTensor("up_proj.weight", .{ .dout, .d }, .model, .{ .dout = .model }), null, .d), + .gate_proj = .init(store.createTensor("gate_proj.weight", .{ .dout, .d }, .model, .{ .dout = .model }), null, .d), + .down_proj = .init(store.createTensor("down_proj.weight", .{ .dout, .d }, .model, .{ .d = .model }), null, .d), + }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(Mlp)) void { + zml.Buffer.deinitAll(Mlp, self); + } + + pub fn forward(self: Mlp, x: zml.Tensor) zml.Tensor { + const proj = self.up_proj.forward(x, x.dtype()); + var output = self.gate_proj.forward(x, x.dtype()); + output = output.silu().mul(proj).rename(.{ .dout = .d }); + return self.down_proj.forward(output, output.dtype()); + } +}; + +const SelfAttn = struct { + q_proj: zml.nn.Linear, + k_proj: zml.nn.Linear, + v_proj: zml.nn.Linear, + + q_norm: ?RmsNorm, + k_norm: ?RmsNorm, + + o_proj: zml.nn.Linear, + num_heads: i64 = undefined, + num_kv_heads: i64 = 0, + rope_opts: zml.nn.RopeOpts = undefined, + + pub fn init(store: zml.io.TensorStore.View, config: Config) !SelfAttn { + var rope_scaling = config.rope_scaling; + rope_scaling.setRopeTheta(config.rope_theta); + return .{ + .q_proj = .init(store.createTensor("q_proj.weight", .{ .dout, .d }, .model, .{ .dout = .model }), null, .d), + .k_proj = .init(store.createTensor("k_proj.weight", .{ .dout, .d }, .model, .{ .dout = .model }), null, .d), + .v_proj = .init(store.createTensor("v_proj.weight", .{ .dout, .d }, .model, .{ .dout = .model }), null, .d), + .o_proj = .init(store.createTensor("o_proj.weight", .{ .dout, .d }, .model, .{ .d = .model }), null, .d), + // TODO(Corentin): fix that + .q_norm = null, + .k_norm = null, + .num_heads = @intCast(config.num_attention_heads), + .num_kv_heads = @intCast(config.num_key_value_heads), + .rope_opts = .{ + .layout = if (config.hf_rope_impl) .real_im_pass else .interleaved, + .scaling = rope_scaling, + }, + }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(SelfAttn)) void { + zml.Buffer.deinitAll(SelfAttn, self); + } + + /// Self zml.attention. + /// - If token_index is set, x is assumed to be the representation of one new token, + /// and kv_cache will be read for the previous tokens. + /// - If token_index is not set, x is assumed to be the representation of all tokens + /// since the beginning of the sequence, and kv_cache won't be read. + /// In both case, kv_cache will be updated with the computed key and value. + /// x: {.b, .s, .d } -> .{.b, .s, .d} + pub fn forward( + self: SelfAttn, + x: zml.Tensor, + token_index: zml.Tensor, + kv_cache: KvCache, + kv_cache_index: zml.Tensor, + attention_metadata: zml.attention.Metadata, + attention_parameters: zml.attention.Parameters, + ) struct { zml.Tensor, KvCache } { + const num_kv_heads = if (self.num_kv_heads > 0) self.num_kv_heads else self.num_heads; + + // Make hidden state replicated once and reuse it across q/k/v projections. + // This avoids paying gather-style collectives independently for each projection. + const x_qkv = x.withPartitioning(.model, .{ .d = .replicated }); + + var q = self.q_proj.forward(x_qkv, x_qkv.dtype()).splitAxis(-1, .{ .h = self.num_heads, .hd = .auto }); + var k = self.k_proj.forward(x_qkv, x_qkv.dtype()).splitAxis(-1, .{ .h = num_kv_heads, .hd = .auto }); + var v = self.v_proj.forward(x_qkv, x_qkv.dtype()).splitAxis(-1, .{ .h = num_kv_heads, .hd = .auto }); + + // In self-attention, .s axis is used both for keys and queries. + const pos_index = b: { + const temp = zml.Tensor.arange(.{ .end = x.dim(.s) }, token_index.dtype()).withTags(.{.s}).broad(zml.Shape.init(.{ .s = x.dim(.s) }, token_index.dtype())); + break :b temp.add(token_index.broad(temp.shape())); + }; + + if (self.q_norm) |norm| q = norm.forward(q.rename(.{ .hd = .d })).rename(.{ .d = .hd }); + if (self.k_norm) |norm| k = norm.forward(k.rename(.{ .hd = .d })).rename(.{ .d = .hd }); + q = zml.nn.rope(q, pos_index, self.rope_opts); + k = zml.nn.rope(k, pos_index, self.rope_opts); + q = q.rename(.{ .s = .q }); + k = k.rename(.{ .s = .k }); + v = v.rename(.{ .s = .k }); + + const dtype = q.dtype(); + const new_kv_cache = kv_cache.updateAt(k, v, token_index, kv_cache_index); + k = new_kv_cache.keysAt(kv_cache_index).convert(dtype); + v = new_kv_cache.valuesAt(kv_cache_index).convert(dtype); + + const attn_output = zml.attention.attention( + q, + k, + v, + token_index, + attention_metadata, + attention_parameters, + zml.Compiler.current().mesh(.model), + .head(.model), + ); + + const attn = attn_output.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); + const delta = self.o_proj.forward(attn, attn.dtype()) + .rename(.{ .dout = .d }) + .withPartitioning(.model, .{ .d = .replicated }); + return .{ delta, new_kv_cache }; + } +}; + +pub const KvCache = struct { + k: zml.Tensor, + v: zml.Tensor, + + pub const Buffer = zml.Bufferized(KvCache); + + pub fn init(kv_shape: zml.Shape, mesh: *const zml.Mesh) KvCache { + const sharded_shape = kv_shape.withPartitioning(mesh, .{ .h = .model }); + + return .{ + .k = .fromShape(sharded_shape), + .v = .fromShape(sharded_shape), + }; + } + + pub fn initBuffer(kv: KvCache, io: std.Io, platform: *const zml.Platform) !Buffer { + return .{ + .k = try zml.Buffer.uninitialized(io, platform, kv.k.shape(), .{}), + .v = try zml.Buffer.uninitialized(io, platform, kv.v.shape(), .{}), + }; + } + + pub fn deinitBuffer(kv: *Buffer) void { + kv.k.deinit(); + kv.v.deinit(); + } + + pub fn keysAt(kv: KvCache, layer_index: zml.Tensor) zml.Tensor { + return kv.k.slice(.layer, .dynSingle(layer_index)); + } + + pub fn valuesAt(kv: KvCache, layer_index: zml.Tensor) zml.Tensor { + return kv.v.slice(.layer, .dynSingle(layer_index)); + } + + pub fn updateAt(kv: KvCache, new_k: zml.Tensor, new_v: zml.Tensor, token_index: zml.Tensor, layer_index: zml.Tensor) KvCache { + const k_shape = kv.k.shape().drop(.layer); + return .{ + .k = kv.k.scatterSlices(.{ .layer = layer_index, .k = token_index }, new_k.convert(kv.k.dtype()).transpose(k_shape), .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }).reuseBuffer(kv.k), + .v = kv.v.scatterSlices(.{ .layer = layer_index, .k = token_index }, new_v.convert(kv.v.dtype()).transpose(kv.v.shape().drop(.layer)), .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }).reuseBuffer(kv.v), + }; + } + + pub fn reuseBuffer(kv: KvCache, other: KvCache) KvCache { + return .{ + .k = kv.k.reuseBuffer(other.k), + .v = kv.v.reuseBuffer(other.v), + }; + } +}; diff --git a/examples/llm/models/llama/session.zig b/examples/llm/models/llama/session.zig new file mode 100644 index 00000000..8a6532e6 --- /dev/null +++ b/examples/llm/models/llama/session.zig @@ -0,0 +1,222 @@ +const std = @import("std"); + +const zml = @import("zml"); + +const inference = @import("inference.zig"); +const model = @import("model.zig"); + +pub const Session = struct { + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + compiled_model: *inference.CompiledModel, + prefill: inference.KernelRunner, + decode: inference.KernelRunner, + kv_cache_buffers: zml.Bufferized(model.KvCache), + token_index_buffers: []zml.Buffer, + rng_buffers: zml.Bufferized(zml.Tensor.Rng), + tokenizer: zml.tokenizer.Tokenizer, + config: *const model.Config, + seqlen: u32, + last_generated_token: u32 = 0, + conversation_id: u64, + + pub fn init( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + tokenizer: zml.tokenizer.Tokenizer, + compiled_model: *inference.CompiledModel, + model_buffers: *model.Buffers, + ) !Session { + var kv_cache_buffers = try compiled_model.params.kv_cache.initBuffer(io, platform); + errdefer model.KvCache.deinitBuffer(&kv_cache_buffers); + + const token_index_buffers = try allocator.alloc(zml.Buffer, compiled_model.params.seqlen); + errdefer allocator.free(token_index_buffers); + var initialized_token_index_buffers: usize = 0; + errdefer { + for (token_index_buffers[0..initialized_token_index_buffers]) |*token_index_buffer| { + token_index_buffer.deinit(); + } + } + for (token_index_buffers, 0..) |*token_index_buffer, i| { + token_index_buffer.* = try zml.Buffer.scalar(io, platform, i, .u32); + initialized_token_index_buffers = i + 1; + } + + const conversation_id: u64 = @bitCast(std.Io.Clock.now(.real, io).toMicroseconds()); + + const seed: u128 = @intCast(std.Io.Clock.now(.real, io).toNanoseconds()); + var rng_buffers = try zml.Tensor.Rng.init().initBuffer(io, platform, seed); + errdefer zml.Tensor.Rng.deinitBuffer(&rng_buffers); + + var prefill = try inference.KernelRunner.init(allocator, &compiled_model.prefill, model_buffers); + errdefer prefill.deinit(allocator); + const decode = try inference.KernelRunner.init(allocator, &compiled_model.decode, model_buffers); + + return .{ + .allocator = allocator, + .io = io, + .platform = platform, + .compiled_model = compiled_model, + .prefill = prefill, + .decode = decode, + .kv_cache_buffers = kv_cache_buffers, + .token_index_buffers = token_index_buffers, + .rng_buffers = rng_buffers, + .tokenizer = tokenizer, + .config = &compiled_model.loaded_model.parsed_config.value, + .seqlen = @intCast(compiled_model.params.seqlen), + .conversation_id = conversation_id, + }; + } + + pub fn deinit(self: *Session) void { + self.prefill.deinit(self.allocator); + self.decode.deinit(self.allocator); + model.KvCache.deinitBuffer(&self.kv_cache_buffers); + for (self.token_index_buffers) |*token_index_buffer| { + token_index_buffer.deinit(); + } + self.allocator.free(self.token_index_buffers); + zml.Tensor.Rng.deinitBuffer(&self.rng_buffers); + } + + pub fn tokenizePrompt(self: *const Session, allocator: std.mem.Allocator, prompt: []const u8) ![]const u32 { + var encoder = try self.tokenizer.encoder(); + defer encoder.deinit(); + + const start_header = self.tokenizer.tokenId("<|start_header_id|>") orelse return error.NoSuchToken; + const end_header = self.tokenizer.tokenId("<|end_header_id|>") orelse return error.NoSuchToken; + const eot = self.tokenizer.tokenId("<|eot_id|>") orelse return error.NoSuchToken; + const newline = self.tokenizer.tokenId("\\n") orelse return error.NoSuchToken; + + var tokens = std.Io.Writer.Allocating.initAligned(allocator, .of(u32)); + try tokens.ensureUnusedCapacity(prompt.len); + + const w: *std.Io.Writer = &tokens.writer; + try encoder.appendTokens(w, &.{ self.config.bos_token_id, start_header }); + try encoder.encode(w, "user"); + try encoder.appendTokens(w, &.{ end_header, newline }); + try encoder.encode(w, prompt); + try encoder.appendTokens(w, &.{ eot, newline, start_header }); + try encoder.encode(w, "assistant"); + try encoder.appendTokens(w, &.{ end_header, newline }); + + return @ptrCast(@alignCast(try tokens.toOwnedSlice())); + } + + pub fn tokenizeTurn(self: *const Session, allocator: std.mem.Allocator, prompt: []const u8) ![]const u32 { + var encoder = try self.tokenizer.encoder(); + defer encoder.deinit(); + + const start_header = self.tokenizer.tokenId("<|start_header_id|>") orelse return error.NoSuchToken; + const end_header = self.tokenizer.tokenId("<|end_header_id|>") orelse return error.NoSuchToken; + const eot = self.tokenizer.tokenId("<|eot_id|>") orelse return error.NoSuchToken; + const newline = self.tokenizer.tokenId("\\n") orelse return error.NoSuchToken; + + var tokens = std.Io.Writer.Allocating.initAligned(allocator, .of(u32)); + try tokens.ensureUnusedCapacity(prompt.len); + + const w: *std.Io.Writer = &tokens.writer; + try encoder.appendTokens(w, &.{ eot, newline, start_header }); + try encoder.encode(w, "user"); + try encoder.appendTokens(w, &.{ end_header, newline }); + try encoder.encode(w, prompt); + try encoder.appendTokens(w, &.{ eot, newline, start_header }); + try encoder.encode(w, "assistant"); + try encoder.appendTokens(w, &.{ end_header, newline }); + + return @ptrCast(@alignCast(try tokens.toOwnedSlice())); + } + + pub fn runPrefill(self: *Session, all_tokens: []const u32) !void { + const prefill_tokens_slice: zml.Slice = try .alloc(self.allocator, .init(.{self.seqlen}, .u32)); + defer prefill_tokens_slice.free(self.allocator); + @memset(prefill_tokens_slice.items(u32), 0); + @memcpy(prefill_tokens_slice.items(u32)[0..all_tokens.len], all_tokens); + + var prefill_tokens_buffer: zml.Buffer = try .fromSlice(self.io, self.platform, prefill_tokens_slice); + defer prefill_tokens_buffer.deinit(); + var active_length_buffer: zml.Buffer = try .scalar(self.io, self.platform, all_tokens.len, .u32); + defer active_length_buffer.deinit(); + + const params = self.compiled_model.params; + var attention_metadata_buffers: zml.Bufferized(zml.attention.Metadata) = switch (params.attention_metadata) { + .stablehlo => try params.attention_metadata.initBuffer(self.io, self.platform), + .flash_attention => |metadata| .{ .flash_attention = switch (metadata) { + .metal => .{ .metal = .{ .num_tokens = try .scalar(self.io, self.platform, all_tokens.len, .u32) } }, + else => try metadata.initBuffer(self.io, self.platform), + } }, + }; + defer zml.attention.Metadata.deinitBuffer(&attention_metadata_buffers); + + var next_token_buffer: zml.Buffer = undefined; + inference.run(&self.prefill, .{ + .io = self.io, + .tokens_buf = &prefill_tokens_buffer, + .next_token_buf = &next_token_buffer, + .token_index_buf = &self.token_index_buffers[0], + .active_length_buf = &active_length_buffer, + .kv_cache_buffers = &self.kv_cache_buffers, + .rng_buffers = &self.rng_buffers, + .attention_metadata_buffers = &attention_metadata_buffers, + }); + + defer next_token_buffer.deinit(); + self.last_generated_token = try next_token_buffer.getValue(u32, self.io); + } + + pub fn runDecode(self: *Session, all_tokens: *std.ArrayList(u32), stdout: *std.Io.Writer) !void { + var decoder: zml.tokenizer.Tokenizer.Decoder = try self.tokenizer.decoder(); + defer decoder.deinit(); + + const decoder_out_buffer: []u8 = try self.allocator.alloc(u8, 256); + defer self.allocator.free(decoder_out_buffer); + + var last_token_id: u32 = self.last_generated_token; + var current_token_buffer: zml.Buffer = try .fromBytes(self.io, self.platform, .init(.{ .s = 1 }, .u32), @ptrCast(&last_token_id)); + defer current_token_buffer.deinit(); + var active_length_buffer: zml.Buffer = try .scalar(self.io, self.platform, 1, .u32); + defer active_length_buffer.deinit(); + + const params = self.compiled_model.params; + var attention_metadata_buffers = try params.attention_metadata.initBuffer(self.io, self.platform); + defer zml.attention.Metadata.deinitBuffer(&attention_metadata_buffers); + + generation: while (true) { + if (isEosToken(self.config, last_token_id)) break :generation; + + try stdout.writeAll(try decoder.feedOne(last_token_id, decoder_out_buffer)); + try stdout.flush(); + + try all_tokens.append(self.allocator, last_token_id); + if (all_tokens.items.len >= self.seqlen) break :generation; + + inference.run(&self.decode, .{ + .io = self.io, + .tokens_buf = ¤t_token_buffer, + .next_token_buf = ¤t_token_buffer, + .token_index_buf = &self.token_index_buffers[all_tokens.items.len], + .active_length_buf = &active_length_buffer, + .kv_cache_buffers = &self.kv_cache_buffers, + .rng_buffers = &self.rng_buffers, + .attention_metadata_buffers = &attention_metadata_buffers, + }); + last_token_id = try current_token_buffer.getValue(u32, self.io); + } + + try stdout.writeAll(try decoder.finalize(decoder_out_buffer)); + try stdout.flush(); + } +}; + +fn isEosToken(config: *const model.Config, token_id: u32) bool { + return switch (config.eos_token_id) { + .int => |eos| token_id == eos, + .ints => |eos_list| for (eos_list) |eos| { + if (token_id == eos) break true; + } else false, + }; +} diff --git a/examples/llm/models/llama_tests.zig b/examples/llm/models/llama_tests.zig new file mode 100644 index 00000000..1b5894f1 --- /dev/null +++ b/examples/llm/models/llama_tests.zig @@ -0,0 +1,149 @@ +const std = @import("std"); + +const zml = @import("zml"); + +const common = @import("common.zig"); +const llama = @import("llama.zig"); +const model = @import("llama/model.zig"); + +pub const std_options: std.Options = .{ + .log_level = .info, +}; + +const Args = struct { + model: []const u8, + activations: []const u8, + + pub const help = + \\Use llama_tests --model= --activations= + \\ + \\ Validate the LLaMA implementation against activation fixtures. + \\ + \\ Options: + \\ --model= Path to the model repository + \\ --activations= Path to activation safetensors + \\ + ; +}; + +pub fn main(init: std.process.Init) !void { + const allocator = init.gpa; + const io = init.io; + const args = zml.stdx.flags.parse(init.minimal.args, Args); + + const platform: *zml.Platform = try .auto(allocator, io, .{}); + defer platform.deinit(allocator, io); + + const repo = try zml.safetensors.resolveModelRepo(io, args.model); + defer repo.close(io); + var registry: zml.safetensors.TensorRegistry = try .fromRepo(allocator, io, repo); + defer registry.deinit(); + const meshes: common.Meshes = try .init(platform); + var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &meshes.all()); + defer store.deinit(); + + var repo_model = try llama.LoadedModel.init(allocator, io, repo, store.view(), .{}); + defer repo_model.deinit(allocator); + + var progress = std.Progress.start(io, .{ .root_name = args.model }); + + var model_buffers = try repo_model.loadBuffers(allocator, io, platform, &store, &progress); + defer repo_model.unloadBuffers(&model_buffers, allocator); + progress.end(); + + try run(allocator, io, platform, args.activations, repo_model.inner, &model_buffers, &meshes.all()); +} + +fn run( + allocator: std.mem.Allocator, + io: std.Io, + platform: *zml.Platform, + activations_path: []const u8, + mdl: model.Model, + model_buffers: *model.Buffers, + meshes: []const *const zml.Mesh, +) !void { + var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, activations_path); + defer registry.deinit(); + + var activation_store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, meshes); + defer activation_store.deinit(); + + try testLayer(allocator, io, platform, activation_store.view(), "embed_tokens", mdl.model.embed_tokens, model_buffers.model.embed_tokens, meshes, .{ .absolute_tolerance = 1e-3 }); + + if (mdl.model.layers.len == 0) return; + + const layer = mdl.model.layers[0]; + const layer_buffers = model_buffers.model.layers[0]; + + try testLayer(allocator, io, platform, activation_store.view(), "layers.0.self_attn.v_proj", layer.self_attn.v_proj, layer_buffers.self_attn.v_proj, meshes, .{ .absolute_tolerance = 1e-2 }); + try testLayer(allocator, io, platform, activation_store.view(), "layers.0.self_attn.q_proj", layer.self_attn.q_proj, layer_buffers.self_attn.q_proj, meshes, .{ .absolute_tolerance = 2e-2 }); + try testLayer(allocator, io, platform, activation_store.view(), "layers.0.self_attn.k_proj", layer.self_attn.k_proj, layer_buffers.self_attn.k_proj, meshes, .{ .absolute_tolerance = 2e-2 }); + try testLayer(allocator, io, platform, activation_store.view(), "layers.0.self_attn.o_proj", layer.self_attn.o_proj, layer_buffers.self_attn.o_proj, meshes, .{ .absolute_tolerance = 2e-2 }); + try testLayer(allocator, io, platform, activation_store.view(), "layers.0.mlp", layer.mlp, layer_buffers.mlp, meshes, .{ .absolute_tolerance = 1e-2 }); + try testLayer(allocator, io, platform, activation_store.view(), "layers.0.input_layernorm", layer.input_layernorm, layer_buffers.input_layernorm, meshes, .{ .absolute_tolerance = 1e-2 }); + try testLayer(allocator, io, platform, activation_store.view(), "layers.0.post_attention_layernorm", layer.post_attention_layernorm, layer_buffers.post_attention_layernorm, meshes, .{ .absolute_tolerance = 1e-2 }); +} + +fn testLayer( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + activation_store: zml.io.TensorStore.View, + name: []const u8, + layer: anytype, + layer_weights: zml.Bufferized(@TypeOf(layer)), + meshes: []const *const zml.Mesh, + opts: zml.testing.CompareOpts, +) !void { + const in_key = try std.fmt.allocPrint(allocator, "{s}.in", .{name}); + defer allocator.free(in_key); + const in_shape = activation_store.getShape(in_key) orelse return error.NotFound; + var in_buffer = try loadBufferFromStore(allocator, io, platform, activation_store, in_key); + defer in_buffer.deinit(); + const in_tensor = zml.Tensor.fromShape(in_shape); + + const out_key = try std.fmt.allocPrint(allocator, "{s}.out", .{name}); + defer allocator.free(out_key); + var out_buffer_expected = try loadBufferFromStore(allocator, io, platform, activation_store, out_key); + defer out_buffer_expected.deinit(); + + // `zml.nn.Linear.forward` takes an explicit output dtype; every other layer here + // is `fn (self, Tensor) Tensor`. + const Layer = @TypeOf(layer); + const Call = struct { + fn forward(l: Layer, x: zml.Tensor) zml.Tensor { + return if (Layer == zml.nn.Linear) l.forward(x, x.dtype()) else Layer.forward(l, x); + } + }; + const exe = try platform.compileFn(allocator, io, Call.forward, .{ layer, in_tensor }, .{ .meshes = meshes }); + defer exe.deinit(); + + var args = try exe.args(allocator); + defer args.deinit(allocator); + args.set(.{ layer_weights, in_buffer }); + + var res = try exe.results(allocator); + defer res.deinit(allocator); + + exe.call(io, args, &res, .{}); + + var out_result = res.get(zml.Buffer); + defer out_result.deinit(); + try zml.testing.expectClose(io, out_result, out_buffer_expected, opts); +} + +fn loadBufferFromStore(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, store: zml.io.TensorStore.View, key: []const u8) !zml.Buffer { + const shape = store.getShape(key) orelse return error.NotFound; + + const host_bytes = try allocator.alloc(u8, shape.byteSize()); + defer allocator.free(host_bytes); + + var io_buffer: [8 * 1024]u8 = undefined; + var reader = try store.getReader(key, io, &io_buffer); + defer reader.deinit(); + + _ = try reader.interface.readSliceAll(host_bytes); + + return zml.Buffer.fromBytes(io, platform, shape, host_bytes); +} diff --git a/examples/llm/models/qwen3_5.zig b/examples/llm/models/qwen3_5.zig new file mode 100644 index 00000000..4147f695 --- /dev/null +++ b/examples/llm/models/qwen3_5.zig @@ -0,0 +1,11 @@ +pub const inference = @import("qwen3_5/inference.zig"); +pub const CompilationParameters = inference.CompilationParameters; +pub const CompilationOptions = inference.CompilationParameters; +pub const CompiledModel = inference.CompiledModel; +pub const model = @import("qwen3_5/model.zig"); +pub const Config = model.Config; +pub const Buffers = model.Buffers; +pub const Model = model.Model; +pub const LoadedModel = model.LoadedModel; +pub const session = @import("qwen3_5/session.zig"); +pub const Session = session.Session; diff --git a/examples/llm/models/qwen3_5/inference.zig b/examples/llm/models/qwen3_5/inference.zig new file mode 100644 index 00000000..50d3da30 --- /dev/null +++ b/examples/llm/models/qwen3_5/inference.zig @@ -0,0 +1,320 @@ +const std = @import("std"); + +const zml = @import("zml"); + +const common = @import("../common.zig"); +const model = @import("model.zig"); + +const log = std.log.scoped(.qwen3_5); +const Phase = common.Phase; + +pub const CompilationParameters = struct { + prefill_tokens: zml.Tensor, + decode_tokens: zml.Tensor, + token_index: zml.Tensor, + kv_cache: model.KvCache, + rng: zml.Tensor.Rng, + seqlen: u32, + meshes: common.Meshes, + + pub fn init(mdl: model.Model, config: model.Config, seqlen: u32, meshes: common.Meshes) CompilationParameters { + const dtype = mdl.text_model.embed_tokens.weight.dtype(); + return .{ + .prefill_tokens = .init(.{ .b = 1, .s = seqlen }, .u32), + .decode_tokens = .init(.{ .b = 1, .s = 1 }, .u32), + .token_index = .init(.{}, .u32), + .kv_cache = .init(config, 1, seqlen, dtype, .f32, meshes.model), + .rng = .init(), + .seqlen = seqlen, + .meshes = meshes, + }; + } +}; + +pub const CompilationOptions = CompilationParameters; + +pub const Args = struct { + io: std.Io, + tokens_buf: *zml.Buffer, + next_token_buf: *zml.Buffer, + token_index_buf: *zml.Buffer, + active_length_buf: *zml.Buffer, + kv_cache_buffers: *zml.Bufferized(model.KvCache), + rng_buffers: *zml.Bufferized(zml.Tensor.Rng), +}; + +pub const CompiledModel = struct { + loaded_model: *const model.LoadedModel, + prefill: KernelExe, + decode: KernelExe, + params: CompilationParameters, + + pub fn init( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + loaded_model: *const model.LoadedModel, + qwen_model: model.Model, + parameters: CompilationParameters, + progress: *std.Progress.Node, + ) !CompiledModel { + const prefill = try compileKernel(allocator, io, platform, qwen_model, parameters, @intCast(parameters.prefill_tokens.dim(.s)), .prefill, progress); + errdefer prefill.deinit(); + const decode = try compileKernel(allocator, io, platform, qwen_model, parameters, @intCast(parameters.decode_tokens.dim(.s)), .decode, progress); + return .{ + .loaded_model = loaded_model, + .prefill = prefill, + .decode = decode, + .params = parameters, + }; + } + + pub fn deinit(self: *CompiledModel) void { + self.prefill.deinit(); + self.decode.deinit(); + } +}; + +pub const Inference = CompiledModel; + +pub const KernelExe = struct { + embed: zml.FnExe(model.EmbedTokens.forward), + full_attention: zml.FnExe(model.TransformerLayer.forwardSelfAttn), + linear_attention: zml.FnExe(model.TransformerLayer.forwardLinearAttn), + sample: zml.FnExe(model.Sampler.sampleTokens), + + pub fn deinit(self: *const KernelExe) void { + self.embed.deinit(); + self.full_attention.deinit(); + self.linear_attention.deinit(); + self.sample.deinit(); + } +}; + +pub const LayerRunner = union(enum) { + full_attention: zml.FnExe(model.TransformerLayer.forwardSelfAttn).Runner(.{.layer}), + linear_attention: zml.FnExe(model.TransformerLayer.forwardLinearAttn).Runner(.{.layer}), + + pub fn deinit(self: *LayerRunner, allocator: std.mem.Allocator) void { + switch (self.*) { + inline else => |*runner| runner.deinit(allocator), + } + } +}; + +pub const KernelRunner = struct { + embed: zml.FnExe(model.EmbedTokens.forward).Runner(.{.embedding}), + layers: []LayerRunner, + sample: zml.FnExe(model.Sampler.sampleTokens).Runner(.{.sampler}), + + pub fn init(allocator: std.mem.Allocator, exe: *const KernelExe, buffers: *const model.Buffers) !KernelRunner { + var embed = try zml.FnExe(model.EmbedTokens.forward).Runner(.{.embedding}).init(&exe.embed, allocator, .{ + .embedding = .{ .embed_tokens = buffers.text_model.embed_tokens }, + }); + errdefer embed.deinit(allocator); + + const layers = try allocator.alloc(LayerRunner, buffers.text_model.layers.len); + errdefer allocator.free(layers); + var initialized_layers: usize = 0; + errdefer for (layers[0..initialized_layers]) |*layer| layer.deinit(allocator); + for (layers, buffers.text_model.layers) |*layer, layer_buffers| { + layer.* = switch (layer_buffers.attn) { + .full_attention => .{ .full_attention = try zml.FnExe(model.TransformerLayer.forwardSelfAttn).Runner(.{.layer}).init( + &exe.full_attention, + allocator, + .{ .layer = layer_buffers }, + ) }, + .linear_attention => .{ .linear_attention = try zml.FnExe(model.TransformerLayer.forwardLinearAttn).Runner(.{.layer}).init( + &exe.linear_attention, + allocator, + .{ .layer = layer_buffers }, + ) }, + }; + initialized_layers += 1; + } + + var sample = try zml.FnExe(model.Sampler.sampleTokens).Runner(.{.sampler}).init(&exe.sample, allocator, .{ + .sampler = .{ + .norm = buffers.text_model.norm, + .lm_head = buffers.lm_head, + }, + }); + errdefer sample.deinit(allocator); + + return .{ .embed = embed, .layers = layers, .sample = sample }; + } + + pub fn deinit(self: *KernelRunner, allocator: std.mem.Allocator) void { + self.embed.deinit(allocator); + for (self.layers) |*layer| layer.deinit(allocator); + allocator.free(self.layers); + self.sample.deinit(allocator); + } +}; + +pub fn run(runner: *KernelRunner, args: Args, layer_index_buffers: []const zml.Buffer) void { + var hidden_buffer: zml.Buffer = undefined; + runner.embed.run(args.io, .{ + .inputs = .{ + .tokens = args.tokens_buf.*, + }, + .outputs = .{ .hidden = &hidden_buffer }, + }, .{}); + defer hidden_buffer.deinit(); + + for (runner.layers, layer_index_buffers) |*layer_runner, layer_index_buffer| { + switch (layer_runner.*) { + .full_attention => |*layer| { + var layer_cache: zml.Bufferized(model.KvCache.SelfAttnCache) = .{ + .k = args.kv_cache_buffers.self_attn.k, + .v = args.kv_cache_buffers.self_attn.v, + .layer_index = layer_index_buffer, + }; + layer.run(args.io, .{ + .inputs = .{ + .hidden = hidden_buffer, + .token_index = args.token_index_buf.*, + .cache = layer_cache, + }, + .outputs = .{ .hidden = &hidden_buffer, .cache = &layer_cache }, + }, .{}); + args.kv_cache_buffers.self_attn.k = layer_cache.k; + args.kv_cache_buffers.self_attn.v = layer_cache.v; + layer_cache.layer_index.deinit(); + }, + .linear_attention => |*layer| { + var layer_cache: zml.Bufferized(model.KvCache.GatedDeltaNetCache) = .{ + .conv_state = args.kv_cache_buffers.gated_delta_net.conv_state, + .recurrent_state = args.kv_cache_buffers.gated_delta_net.recurrent_state, + .layer_index = layer_index_buffer, + }; + layer.run(args.io, .{ + .inputs = .{ + .hidden = hidden_buffer, + .active_length = args.active_length_buf.*, + .cache = layer_cache, + }, + .outputs = .{ .hidden = &hidden_buffer, .cache = &layer_cache }, + }, .{}); + args.kv_cache_buffers.gated_delta_net.conv_state = layer_cache.conv_state; + args.kv_cache_buffers.gated_delta_net.recurrent_state = layer_cache.recurrent_state; + layer_cache.layer_index.deinit(); + }, + } + } + + runner.sample.run(args.io, .{ + .inputs = .{ + .hidden = hidden_buffer, + .tokens = args.tokens_buf.*, + .active_length = args.active_length_buf.*, + .rng = args.rng_buffers.*, + .token_index = args.token_index_buf.*, + }, + .outputs = .{ + .tokens = args.next_token_buf, + .rng = args.rng_buffers, + .token_index = args.token_index_buf, + }, + }, .{}); +} + +fn compileKernel( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + qwen_model: model.Model, + parameters: CompilationOptions, + seqlen: usize, + phase: Phase, + progress: *std.Progress.Node, +) !KernelExe { + const full_index = findFirstLayerIndex(qwen_model.config.text_config.layer_types, .full_attention) orelse return error.MissingFullAttentionLayer; + const linear_index = findFirstLayerIndex(qwen_model.config.text_config.layer_types, .linear_attention) orelse return error.MissingLinearAttentionLayer; + + const embed = try compileEmbed(allocator, io, platform, qwen_model, parameters, seqlen, phase, progress); + errdefer embed.deinit(); + const full_attention = try compileFullAttention(allocator, io, platform, qwen_model, parameters, seqlen, full_index, phase, progress); + errdefer full_attention.deinit(); + const linear_attention = try compileLinearAttention(allocator, io, platform, qwen_model, parameters, seqlen, linear_index, phase, progress); + errdefer linear_attention.deinit(); + const sample = try compileSample(allocator, io, platform, qwen_model, parameters, seqlen, phase, progress); + errdefer sample.deinit(); + return .{ .embed = embed, .full_attention = full_attention, .linear_attention = linear_attention, .sample = sample }; +} + +fn compileEmbed(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mdl: model.Model, parameters: CompilationOptions, seqlen: usize, phase: Phase, progress: *std.Progress.Node) !zml.FnExe(model.EmbedTokens.forward) { + progress.increaseEstimatedTotalItems(1); + var node = progress.start(phase.startMessage("embed tokens"), 1); + defer node.end(); + const from: std.Io.Timestamp = .now(io, .awake); + defer phase.logCompileDone(log, "embed tokens", io, from); + return zml.FnExe(model.EmbedTokens.forward).compile(allocator, io, platform, .{ .meshes = ¶meters.meshes.all(), .program_name = phase.programName("qwen3_5", "embed_tokens") }, .{.{ + .embedding = .{ .embed_tokens = mdl.text_model.embed_tokens }, + .tokens = zml.Tensor.init(.{ .b = 1, .s = seqlen }, .u32), + }}); +} + +fn compileFullAttention(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mdl: model.Model, parameters: CompilationOptions, seqlen: usize, layer_index: usize, phase: Phase, progress: *std.Progress.Node) !zml.FnExe(model.TransformerLayer.forwardSelfAttn) { + progress.increaseEstimatedTotalItems(1); + var node = progress.start(phase.startMessage("full attention layer"), 1); + defer node.end(); + const from: std.Io.Timestamp = .now(io, .awake); + defer phase.logCompileDone(log, "full attention layer", io, from); + return zml.FnExe(model.TransformerLayer.forwardSelfAttn).compile(allocator, io, platform, .{ .meshes = ¶meters.meshes.all(), .program_name = phase.programName("qwen3_5", "full_attention_layer") }, .{.{ + .layer = mdl.text_model.layers[layer_index], + .hidden = hiddenTensor(mdl, seqlen, parameters.meshes.model), + .token_index = parameters.token_index, + .cache = .{ + .k = parameters.kv_cache.self_attn.k, + .v = parameters.kv_cache.self_attn.v, + .layer_index = zml.Tensor.init(.{}, .u32), + }, + }}); +} + +fn compileLinearAttention(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mdl: model.Model, parameters: CompilationOptions, seqlen: usize, layer_index: usize, phase: Phase, progress: *std.Progress.Node) !zml.FnExe(model.TransformerLayer.forwardLinearAttn) { + progress.increaseEstimatedTotalItems(1); + var node = progress.start(phase.startMessage("linear attention layer"), 1); + defer node.end(); + const from: std.Io.Timestamp = .now(io, .awake); + defer phase.logCompileDone(log, "linear attention layer", io, from); + return zml.FnExe(model.TransformerLayer.forwardLinearAttn).compile(allocator, io, platform, .{ .meshes = ¶meters.meshes.all(), .program_name = phase.programName("qwen3_5", "linear_attention_layer") }, .{.{ + .layer = mdl.text_model.layers[layer_index], + .hidden = hiddenTensor(mdl, seqlen, parameters.meshes.model), + .active_length = zml.Tensor.init(.{}, .u32), + .cache = .{ + .conv_state = parameters.kv_cache.gated_delta_net.conv_state, + .recurrent_state = parameters.kv_cache.gated_delta_net.recurrent_state, + .layer_index = zml.Tensor.init(.{}, .u32), + }, + }}); +} + +fn compileSample(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mdl: model.Model, parameters: CompilationOptions, seqlen: usize, phase: Phase, progress: *std.Progress.Node) !zml.FnExe(model.Sampler.sampleTokens) { + progress.increaseEstimatedTotalItems(1); + var node = progress.start(phase.startMessage("sampler"), 1); + defer node.end(); + const from: std.Io.Timestamp = .now(io, .awake); + defer phase.logCompileDone(log, "sampler", io, from); + return zml.FnExe(model.Sampler.sampleTokens).compile(allocator, io, platform, .{ .meshes = ¶meters.meshes.all(), .program_name = phase.programName("qwen3_5", "sampler") }, .{.{ + .sampler = mdl.sampler(), + .hidden = hiddenTensor(mdl, seqlen, parameters.meshes.model), + .tokens = .init(.{ .b = 1, .s = seqlen }, .u32), + .active_length = .init(.{}, .u32), + .rng = parameters.rng, + .token_index = parameters.token_index, + }}); +} + +fn hiddenTensor(mdl: model.Model, seqlen: usize, mesh: *const zml.Mesh) zml.Tensor { + return .fromShape(zml.Shape.init( + .{ .b = 1, .s = seqlen, .d = mdl.config.text_config.hidden_size }, + mdl.text_model.embed_tokens.weight.dtype(), + ).withPartitioning(mesh, .{ .b = .replicated, .s = .replicated, .d = .replicated })); +} + +fn findFirstLayerIndex(layer_types: []const model.LayerType, target: model.LayerType) ?usize { + for (layer_types, 0..) |layer_type, index| if (layer_type == target) return index; + return null; +} diff --git a/examples/llm/models/qwen3_5/model.zig b/examples/llm/models/qwen3_5/model.zig new file mode 100644 index 00000000..40174f5c --- /dev/null +++ b/examples/llm/models/qwen3_5/model.zig @@ -0,0 +1,1312 @@ +const std = @import("std"); + +const zml = @import("zml"); +const stdx = zml.stdx; + +const common = @import("../common.zig"); +const inference = @import("inference.zig"); + +const log = std.log.scoped(.qwen3_5); + +pub const Config = struct { + text_config: TextConfig, +}; + +pub const TextConfig = struct { + // General + num_hidden_layers: i64, + layer_types: []const LayerType, + hidden_size: i64, + max_position_embeddings: i64, + rms_norm_eps: f32, + // Self attention + head_dim: i64, + num_attention_heads: i64, + num_key_value_heads: i64, + rope_parameters: RopeParameters, + // Linear attention + linear_conv_kernel_dim: i64, + linear_key_head_dim: i64, + linear_num_key_heads: i64, + linear_num_value_heads: i64, + linear_value_head_dim: i64, +}; + +// Each layer uses either: full attention (SelfAttn) or linear attention (GatedDeltaNet). +pub const LayerType = enum { + linear_attention, + full_attention, +}; + +pub const RopeParameters = struct { + mrope_section: [3]i64, + partial_rotary_factor: f32, + rope_theta: f32, +}; + +pub const LoadedModel = struct { + inner: Model, + parsed_config: std.json.Parsed(Config), + + pub fn init( + allocator: std.mem.Allocator, + io: std.Io, + repo: std.Io.Dir, + store: zml.io.TensorStore.View, + generation: common.GenerationOptions, + ) !LoadedModel { + const parsed_config = try common.parseConfig(Config, allocator, io, repo); + errdefer parsed_config.deinit(); + + const options: Model.GenOptions = .{ + .sampling_strategy = generation.sampling_strategy, + .max_seq_len = parsed_config.value.text_config.max_position_embeddings, + }; + + return .{ + .inner = try .init(allocator, store, parsed_config.value, options), + .parsed_config = parsed_config, + }; + } + + pub fn deinit(self: *LoadedModel, allocator: std.mem.Allocator) void { + self.inner.deinit(allocator); + self.parsed_config.deinit(); + } + + pub fn loadBuffers( + self: *const LoadedModel, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + store: *zml.io.TensorStore, + progress: *std.Progress.Node, + ) !Buffers { + progress.increaseEstimatedTotalItems(store.view().count()); + const now: std.Io.Timestamp = .now(io, .awake); + + var buffers = try zml.mem.bufferize(allocator, Model, &self.inner); + errdefer self.unloadBuffers(&buffers, allocator); + + var loader: zml.io.Loader = try .init(allocator, platform, .{ + .dma_chunks = 32, + .dma_chunk_size = 256 * zml.MiB, + .parallelism = 16, + }); + defer loader.deinit(); + + try loader.load(io, Model, &self.inner, &buffers, store, .{ .progress = progress }); + try loader.await(io); + + const took = now.untilNow(io, .awake); + const total_bytes: u64 = loader.bytes_loaded.raw; + const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); + log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); + + return buffers; + } + + pub fn unloadBuffers(self: *const LoadedModel, buffers: *Buffers, allocator: std.mem.Allocator) void { + _ = self; + TextModel.unloadBuffers(&buffers.text_model, allocator); + zml.nn.Linear.unloadBuffers(&buffers.lm_head); + } + + pub fn compile( + self: *const LoadedModel, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + backend: zml.attention.Backend, + meshes: common.Meshes, + seqlen: usize, + progress: *std.Progress.Node, + ) !inference.CompiledModel { + _ = backend; + const params = inference.CompilationParameters.init(self.inner, self.parsed_config.value, @intCast(seqlen), meshes); + return inference.CompiledModel.init(allocator, io, platform, self, self.inner, params, progress); + } +}; + +pub const Buffers = zml.Bufferized(Model); + +pub const Model = struct { + pub const GenOptions = struct { sampling_strategy: zml.nn.SamplingStrategy = .{}, max_seq_len: i64 }; + + pub const SpecialTokens = struct { + im_start_token_id: u32, + im_end_token_id: u32, + end_of_text_token_id: u32, + }; + + text_model: TextModel, + lm_head: zml.nn.Linear, + + config: Config, + gen_options: GenOptions, + special_tokens: SpecialTokens = .{ + .im_start_token_id = 248045, + .im_end_token_id = 248046, + .end_of_text_token_id = 248044, + }, + + pub fn init(allocator: std.mem.Allocator, store: zml.io.TensorStore.View, config: Config, gen_options: GenOptions) !Model { + // For some Qwen3.5 versions, the output projection lm_head has a standalone weight tensor, while for others it's the same as the input embedding layer + const lm_head_prefix = b: { + if (store.hasKey("lm_head.weight")) break :b "lm_head"; + break :b "model.language_model.embed_tokens"; + }; + return .{ + .text_model = try .init(allocator, store.withPrefix("model.language_model"), config), + .lm_head = .init( + store.withPrefix(lm_head_prefix).createTensor("weight", .{ .dout, .d }, .model, .{ .dout = .model, .d = .replicated }), + null, + .d, + ), + .config = config, + .gen_options = gen_options, + }; + } + + pub fn deinit(self: Model, allocator: std.mem.Allocator) void { + self.text_model.deinit(allocator); + } + + pub fn load( + self: *const Model, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + store: *zml.io.TensorStore, + meshes: []const *const zml.Mesh, + progress: *std.Progress.Node, + ) !zml.Bufferized(Model) { + progress.increaseEstimatedTotalItems(store.view().count()); + const now: std.Io.Timestamp = .now(io, .awake); + + var buffers = try zml.mem.bufferize(allocator, Model, self); + errdefer Model.unloadBuffers(&buffers, allocator); + + var loader: zml.io.Loader = try .init(allocator, platform, .{ + .dma_chunks = 32, + .dma_chunk_size = 256 * zml.MiB, + .parallelism = 16, + }); + defer loader.deinit(); + + loader.load(io, Model, self, &buffers, store, meshes); + try loader.await(io); + + const took = now.untilNow(io, .awake); + const total_bytes: u64 = loader.bytes_loaded.raw; + const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); + log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); + + return buffers; + } + + pub fn unloadBuffers(self: *zml.Bufferized(Model), allocator: std.mem.Allocator) void { + TextModel.unloadBuffers(&self.text_model, allocator); + zml.nn.Linear.unloadBuffers(&self.lm_head); + } + pub fn forward( + self: Model, + tokens_: zml.Tensor, + token_index: zml.Tensor, + active_length: zml.Tensor, + kv_cache: KvCache, + rng: zml.Tensor.Rng, + ) struct { zml.Tensor, KvCache, zml.Tensor.Rng } { + const tokens = tokens_.withPartialTags(.{.s}); + const text_model_output, const updated_kv_cache = self.text_model.forward(tokens, token_index, active_length, kv_cache); + const result = Sampler.sampleTokens(.{ + .sampler = self.sampler(), + .hidden = text_model_output, + .tokens = tokens, + .active_length = active_length, + .rng = rng, + .token_index = token_index, + }); + return .{ result.tokens, updated_kv_cache, result.rng }; + } + + pub fn sampler(self: Model) Sampler { + return .{ + .norm = self.text_model.norm, + .lm_head = self.lm_head, + .gen_options = self.gen_options, + }; + } +}; + +pub const Sampler = struct { + norm: RmsNorm, + lm_head: zml.nn.Linear, + gen_options: Model.GenOptions, + + pub const Input = struct { + sampler: Sampler, + hidden: zml.Tensor, + tokens: zml.Tensor, + active_length: zml.Tensor, + rng: zml.Tensor.Rng, + token_index: zml.Tensor, + }; + + pub const Output = struct { + tokens: zml.Tensor, + rng: zml.Tensor.Rng, + token_index: zml.Tensor, + }; + + pub fn sampleTokens(input: Input) Output { + const self = input.sampler; + const last_index = input.active_length.convert(.i32).addConstant(-1); + const x = self.norm.forward(input.hidden.slice(.s, .dyn(last_index, 1))); + const logits = self.lm_head.forward(x.withPartialTags(.{.d}), x.dtype()).rename(.{ .dout = .voc }); + const next_tokens, const new_rng = zml.nn.sampleTokens(logits, self.gen_options.sampling_strategy, input.rng); + const tokens = next_tokens.convert(.u32); + return .{ + .tokens = if (input.tokens.dim(.s) == 1) tokens.reuseBuffer(input.tokens) else tokens, + .rng = new_rng, + .token_index = input.token_index.addConstant(1).reuseBuffer(input.token_index), + }; + } +}; + +pub const EmbedTokens = struct { + embed_tokens: zml.nn.TokenEmbedding, + + pub const Input = struct { + embedding: EmbedTokens, + tokens: zml.Tensor, + }; + + pub const Output = struct { + hidden: zml.Tensor, + }; + + pub fn forward(input: Input) Output { + const tokens = input.tokens.withPartialTags(.{.s}); + return .{ .hidden = input.embedding.embed_tokens.forward(tokens) + .withPartialTags(.{.d}) + .replicate() }; + } +}; + +pub const TextModel = struct { + embed_tokens: zml.nn.TokenEmbedding, + layers: []TransformerLayer, + norm: RmsNorm, + + pub fn init( + allocator: std.mem.Allocator, + store: zml.io.TensorStore.View, + config: Config, + ) !TextModel { + const layers = try allocator.alloc(TransformerLayer, @intCast(config.text_config.num_hidden_layers)); + errdefer allocator.free(layers); + + for (layers, 0..) |*layer, i| { + layer.* = try .init(store.withPrefix("layers").withLayer(i), config, i); + } + + return .{ + .embed_tokens = .{ .weight = store.createTensor("embed_tokens.weight", .{ .voc, .d }, .model, .{ .voc = .replicated, .d = .model }) }, + .layers = layers, + .norm = RmsNorm.init(store.withPrefix("norm"), config.text_config.rms_norm_eps), + }; + } + + pub fn deinit(self: TextModel, allocator: std.mem.Allocator) void { + allocator.free(self.layers); + } + + pub fn unloadBuffers(self: *zml.Bufferized(TextModel), allocator: std.mem.Allocator) void { + self.embed_tokens.weight.deinit(); + for (self.layers) |*layer| { + TransformerLayer.unloadBuffers(layer); + } + allocator.free(self.layers); + RmsNorm.unloadBuffers(&self.norm); + } + + pub fn forward( + self: TextModel, + tokens: zml.Tensor, + token_index: zml.Tensor, + active_length: zml.Tensor, + kv_cache: KvCache, + ) struct { zml.Tensor, KvCache } { + var hidden_states = EmbedTokens.forward(.{ + .embedding = .{ .embed_tokens = self.embed_tokens }, + .tokens = tokens, + }).hidden; + + var updated_kv_cache = kv_cache; + for (self.layers, 0..) |layer, i| { + hidden_states, updated_kv_cache = layer.forward(hidden_states, token_index, active_length, updated_kv_cache.atLayer(i)); + } + + return .{ hidden_states, updated_kv_cache.reuseBuffer(kv_cache) }; + } +}; + +pub const TransformerLayer = struct { + const Attn = union(LayerType) { + linear_attention: GatedDeltaNet, + full_attention: SelfAttn, + }; + + input_layernorm: RmsNorm, + attn: Attn, + mlp: Mlp, + post_attention_layernorm: RmsNorm, + + pub const SelfAttnInput = struct { + layer: TransformerLayer, + hidden: zml.Tensor, + token_index: zml.Tensor, + cache: KvCache.SelfAttnCache, + }; + + pub const SelfAttnOutput = struct { + hidden: zml.Tensor, + cache: KvCache.SelfAttnCache, + }; + + pub const LinearAttnInput = struct { + layer: TransformerLayer, + hidden: zml.Tensor, + active_length: zml.Tensor, + cache: KvCache.GatedDeltaNetCache, + }; + + pub const LinearAttnOutput = struct { + hidden: zml.Tensor, + cache: KvCache.GatedDeltaNetCache, + }; + + pub fn init(store: zml.io.TensorStore.View, config: Config, layer_index: usize) !TransformerLayer { + const is_full_attention = config.text_config.layer_types[layer_index] == .full_attention; + const attn: Attn = switch (is_full_attention) { + true => .{ .full_attention = try .init(store.withPrefix("self_attn"), config) }, + false => .{ .linear_attention = .init(store.withPrefix("linear_attn"), config) }, + }; + return .{ + .input_layernorm = RmsNorm.init(store.withPrefix("input_layernorm"), config.text_config.rms_norm_eps), + .attn = attn, + .mlp = .init(store.withPrefix("mlp")), + .post_attention_layernorm = RmsNorm.init(store.withPrefix("post_attention_layernorm"), config.text_config.rms_norm_eps), + }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(TransformerLayer)) void { + RmsNorm.unloadBuffers(&self.input_layernorm); + switch (self.attn) { + .full_attention => |*self_attn| SelfAttn.unloadBuffers(self_attn), + .linear_attention => |*linear_attn| GatedDeltaNet.unloadBuffers(linear_attn), + } + Mlp.unloadBuffers(&self.mlp); + RmsNorm.unloadBuffers(&self.post_attention_layernorm); + } + + pub fn forward( + self: TransformerLayer, + x0: zml.Tensor, + token_index: zml.Tensor, + active_length: zml.Tensor, + kv_cache: KvCache.LayerView, + ) struct { zml.Tensor, KvCache } { + const x0_replicated = x0.withPartitioning(.model, .{ .d = .replicated }); + const normalized_x0 = self.input_layernorm.forward(x0_replicated); + + var attention_output: zml.Tensor = undefined; + var updated_kv_cache: KvCache = kv_cache.parent; + switch (self.attn) { + .full_attention => |*self_attn| { + const cache = switch (kv_cache.cache) { + .self_attn => |cache| cache, + .linear_attn => unreachable, + }; + const result = self_attn.forward(normalized_x0, token_index, cache); + attention_output = result[0]; + updated_kv_cache.self_attn = result[1].reuseBuffer(updated_kv_cache.self_attn); + }, + .linear_attention => |*linear_attn| { + const cache = switch (kv_cache.cache) { + .linear_attn => |cache| cache, + .self_attn => unreachable, + }; + const result = linear_attn.forward(normalized_x0, cache, active_length); + attention_output = result[0]; + updated_kv_cache.gated_delta_net = result[1].reuseBuffer(updated_kv_cache.gated_delta_net); + }, + } + + const x1 = attention_output.add(x0_replicated).withPartitioning(.model, .{ .d = .replicated }); + const normalized_hidden = self.post_attention_layernorm.forward(x1); + + const mlp_output = self.mlp.forward(normalized_hidden).withPartitioning(.model, .{ .d = .replicated }); + + return .{ mlp_output.add(x1).withPartitioning(.model, .{ .d = .replicated }).reuseBuffer(x0), updated_kv_cache }; + } + + pub fn forwardSelfAttn(input: SelfAttnInput) SelfAttnOutput { + const self = input.layer; + const x0 = input.hidden; + const x0_replicated = x0.withPartitioning(.model, .{ .d = .replicated }); + const normalized_x0 = self.input_layernorm.forward(x0_replicated); + + const self_attn = switch (self.attn) { + .full_attention => |self_attn| self_attn, + .linear_attention => unreachable, + }; + const attention_output, const updated_kv_cache = self_attn.forward(normalized_x0, input.token_index, input.cache); + + const x1 = attention_output.add(x0_replicated).withPartitioning(.model, .{ .d = .replicated }); + const normalized_hidden = self.post_attention_layernorm.forward(x1); + const mlp_output = self.mlp.forward(normalized_hidden).withPartitioning(.model, .{ .d = .replicated }); + + return .{ + .hidden = mlp_output.add(x1).withPartitioning(.model, .{ .d = .replicated }).reuseBuffer(x0), + .cache = updated_kv_cache, + }; + } + + pub fn forwardLinearAttn(input: LinearAttnInput) LinearAttnOutput { + const self = input.layer; + const x0 = input.hidden; + const x0_replicated = x0.withPartitioning(.model, .{ .d = .replicated }); + const normalized_x0 = self.input_layernorm.forward(x0_replicated); + + const linear_attn = switch (self.attn) { + .linear_attention => |linear_attn| linear_attn, + .full_attention => unreachable, + }; + const attention_output, const updated_kv_cache = linear_attn.forward(normalized_x0, input.cache, input.active_length); + + const x1 = attention_output.add(x0_replicated).withPartitioning(.model, .{ .d = .replicated }); + const normalized_hidden = self.post_attention_layernorm.forward(x1); + const mlp_output = self.mlp.forward(normalized_hidden).withPartitioning(.model, .{ .d = .replicated }); + + return .{ + .hidden = mlp_output.add(x1).withPartitioning(.model, .{ .d = .replicated }).reuseBuffer(x0), + .cache = updated_kv_cache, + }; + } +}; + +pub const Mlp = struct { + up_proj: zml.nn.Linear, + gate_proj: zml.nn.Linear, + down_proj: zml.nn.Linear, + + pub fn init(store: zml.io.TensorStore.View) Mlp { + return .{ + .up_proj = .init( + store.withPrefix("up_proj").createTensor("weight", .{ .dout, .d }, .model, .{ .dout = .model, .d = .replicated }), + store.withPrefix("up_proj").maybeCreateTensor("bias", .{.dout}, .model, .{ .dout = .model }), + .d, + ), + .gate_proj = .init( + store.withPrefix("gate_proj").createTensor("weight", .{ .dout, .d }, .model, .{ .dout = .model, .d = .replicated }), + store.withPrefix("gate_proj").maybeCreateTensor("bias", .{.dout}, .model, .{ .dout = .model }), + .d, + ), + .down_proj = .init( + store.withPrefix("down_proj").createTensor("weight", .{ .d, .dout }, .model, .{ .d = .replicated, .dout = .model }), + store.withPrefix("down_proj").maybeCreateTensor("bias", .{.d}, .model, .{ .d = .replicated }), + .dout, + ), + }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(Mlp)) void { + zml.Buffer.deinitAll(Mlp, self); + } + + pub fn forward(self: Mlp, x: zml.Tensor) zml.Tensor { + const up_projed = self.up_proj.forward(x, x.dtype()); + const gate = self.gate_proj.forward(x, x.dtype()); + const hidden = gate.silu().mul(up_projed); + + const output = self.down_proj.forward(hidden, hidden.dtype()); + return output; + } +}; + +pub const SelfAttn = struct { + q_proj: zml.nn.Linear, + k_proj: zml.nn.Linear, + v_proj: zml.nn.Linear, + + q_norm: RmsNorm, + k_norm: RmsNorm, + + num_heads: i64, + num_kv_heads: i64, + head_dim: i64, + rotary_dim: i64, + rotary_embed: TextRotaryEmbedding, + o_proj: zml.nn.Linear, + + fn initProj(store: zml.io.TensorStore.View, partitions: anytype, bias_partitions: anytype) zml.nn.Linear { + return .init( + store.createTensor("weight", .{ .dout, .d }, .model, partitions), + store.maybeCreateTensor("bias", .{.dout}, .model, bias_partitions), + .d, + ); + } + + pub fn init(store: zml.io.TensorStore.View, config: Config) !SelfAttn { + const rotary_dim: i64 = @intFromFloat(@as(f32, @floatFromInt(config.text_config.head_dim)) * + config.text_config.rope_parameters.partial_rotary_factor); + return .{ + .q_proj = initProj(store.withPrefix("q_proj"), .{ .dout = .model, .d = .replicated }, .{ .dout = .model }), + .k_proj = initProj(store.withPrefix("k_proj"), .{ .dout = .model, .d = .replicated }, .{ .dout = .model }), + .v_proj = initProj(store.withPrefix("v_proj"), .{ .dout = .model, .d = .replicated }, .{ .dout = .model }), + .o_proj = initProj(store.withPrefix("o_proj"), .{ .dout = .replicated, .d = .model }, .{ .dout = .replicated }), + .q_norm = RmsNorm.init(store.withPrefix("q_norm"), config.text_config.rms_norm_eps), + .k_norm = RmsNorm.init(store.withPrefix("k_norm"), config.text_config.rms_norm_eps), + .num_heads = config.text_config.num_attention_heads, + .num_kv_heads = config.text_config.num_key_value_heads, + .head_dim = config.text_config.head_dim, + .rotary_dim = rotary_dim, + .rotary_embed = .init(rotary_dim, config.text_config.rope_parameters.rope_theta, config.text_config.rope_parameters.mrope_section), + }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(SelfAttn)) void { + zml.Buffer.deinitAll(SelfAttn, self); + } + + fn projectQAndGate(self: SelfAttn, x: zml.Tensor) struct { zml.Tensor, zml.Tensor } { + const q_proj = self.q_proj.forward(x, x.dtype()).splitAxis(.dout, .{ .h = self.num_heads, .hd = 2 * self.head_dim }); + const q, var gate = q_proj.chunkExact(.hd, 2); + gate = gate.merge(.{ .d_out_proj = .{ .h, .hd } }); + return .{ q, gate }; + } + + fn projectKV(self: SelfAttn, x: zml.Tensor) struct { zml.Tensor, zml.Tensor } { + const num_kv_heads = b: { + if (self.num_kv_heads > 0) break :b self.num_kv_heads; + break :b self.num_heads; + }; + const k = self.k_proj.forward(x, x.dtype()).splitAxis(.dout, .{ .h = num_kv_heads, .hd = self.head_dim }); + const v = self.v_proj.forward(x, x.dtype()).splitAxis(.dout, .{ .h = num_kv_heads, .hd = self.head_dim }); + return .{ k, v }; + } + + fn partitionProjectedKv(kv: zml.Tensor, kv_head_sharding: zml.Sharding.DimSharding) zml.Tensor { + return switch (kv_head_sharding) { + .sharded => |heads| blk: { + const sharded_kv = if (heads.factor != 1) kv.stutter1d(kv.axis(.h), heads.factor) else kv; + break :blk sharded_kv.withPartitioning(.model, .{ .s = .replicated, .h = .model, .hd = .replicated }); + }, + .replicated => kv.withPartitioning(.model, .{ .s = .replicated, .h = .replicated, .hd = .replicated }), + }; + } + + fn partitionCachedKv(tensor: zml.Tensor, kv_head_sharding: zml.Sharding.DimSharding) zml.Tensor { + var cache_tensor = tensor.rename(.{ .s = .k }); + return switch (kv_head_sharding) { + .sharded => |heads| blk: { + if (heads.factor != 1) { + cache_tensor = cache_tensor.stutter1d(cache_tensor.axis(.h), heads.factor); + } + break :blk cache_tensor.withPartitioning(.model, .{ .k = .replicated, .h = .model, .hd = .replicated }); + }, + .replicated => cache_tensor.withPartitioning(.model, .{ .k = .replicated, .h = .replicated, .hd = .replicated }), + }; + } + + pub fn forward( + self: SelfAttn, + x: zml.Tensor, + token_index: zml.Tensor, + kv_cache: KvCache.SelfAttnCache, + ) struct { zml.Tensor, KvCache.SelfAttnCache } { + const tp = zml.Compiler.current().mesh(.model); + const x_qkv = x.withPartitioning(tp, .{ .d = .replicated }); + + var q, var gate = self.projectQAndGate(x_qkv); + var k, var v = self.projectKV(x_qkv); + const kv_head_sharding = tp.shardableDim(k.dim(.h), .model, q.dim(.h)); + + k = partitionProjectedKv(k, kv_head_sharding); + v = partitionProjectedKv(v, kv_head_sharding); + q = self.q_norm.forward(q.rename(.{ .hd = .d })).rename(.{ .d = .hd }); + k = self.k_norm.forward(k.rename(.{ .hd = .d })).rename(.{ .d = .hd }); + + const dtype = q.dtype(); + const position_ids = zml.Tensor.arange(.{ .end = x.dim(.s) }, .i64) + .withTags(.{.s}).insertAxes(.s, .{.b}).broad(zml.Shape.init(.{ .b = x.dim(.b), .s = x.dim(.s) }, .i64)) + .add(token_index.convert(.i64).broad(zml.Shape.init(.{ .b = x.dim(.b), .s = x.dim(.s) }, .i64))); + + const cos, const sin = self.rotary_embed.getCosAndSin(position_ids, dtype); + q = self.rotary_embed.applyRope(q, cos, sin); + k = self.rotary_embed.applyRope(k, cos, sin); + k = partitionProjectedKv(k, kv_head_sharding); + v = partitionProjectedKv(v, kv_head_sharding); + + const new_kv_cache = kv_cache.update(k, v, token_index.convert(.u32)); + k = new_kv_cache.keys().convert(dtype); + v = new_kv_cache.values().convert(dtype); + q = q.rename(.{ .s = .q }); + k = partitionCachedKv(k, kv_head_sharding); + v = partitionCachedKv(v, kv_head_sharding); + + const head_partitioning: zml.attention.Partitioning = switch (kv_head_sharding) { + .sharded => .head(.model), + .replicated => .head(.replicated), + }; + const attn_output = zml.attention.attention( + q, + k, + v, + token_index, + zml.attention.Metadata.init(.fromBackend(.stablehlo)), + zml.attention.Parameters.init(.fromBackend(.stablehlo)), + zml.Compiler.current().mesh(.model), + head_partitioning, + ).rename(.{ .q = .s }).merge(.{ .d_out_proj = .{ .h, .hd } }); + + const gated_output = attn_output.mul(gate.sigmoid()); + const projected_output = self.o_proj + .forward(gated_output.rename(.{ .d_out_proj = .d }), gated_output.dtype()) + .rename(.{ .dout = .d }) + .withPartitioning(tp, .{ .d = .replicated }); + + return .{ projected_output, new_kv_cache }; + } +}; + +pub const TextRotaryEmbedding = struct { + rope_opts: zml.nn.RopeOpts, + rotary_dim: i64, + mrope_section: [3]i64, + + pub fn init(rotary_dim: i64, theta: f32, mrope_section: [3]i64) TextRotaryEmbedding { + return .{ + .rope_opts = .{ + .layout = .real_im_pass, + .scaling = .{ .default = .{ .rope_theta = theta } }, + }, + .rotary_dim = rotary_dim, + .mrope_section = mrope_section, + }; + } + + pub fn getCosAndSin(self: TextRotaryEmbedding, position_ids: zml.Tensor, dtype: zml.DataType) struct { zml.Tensor, zml.Tensor } { + const inv_freq = zml.nn.invFreq(self.rotary_dim, self.rope_opts).withTags(.{.hd}); + + const freqs_t = position_ids.convert(.f32).outer(inv_freq); + + const emb = zml.Tensor.concatenate(&.{ freqs_t, freqs_t }, -1); + const cos = emb.cos().convert(dtype); + const sin = emb.sin().convert(dtype); + + return .{ cos, sin }; + } + + pub fn getCosAndSinInterleaved( + self: TextRotaryEmbedding, + position_ids: zml.Tensor, + dtype: zml.DataType, + ) struct { zml.Tensor, zml.Tensor } { + // To be used later in image extension. + const stacked_position_ids = zml.Tensor.stack(&.{ position_ids, position_ids, position_ids }, 0, .g).convert(.f32); + const inv_freq = zml.nn.InvFreq(self.rotary_dim, self.rope_opts).withTags(.{.hd}); + + var freqs = stacked_position_ids.outer(inv_freq); + var freqs_t, var freqs_h, var freqs_w = freqs.chunkExact(.g, 3); + freqs_t = freqs_t.squeeze(.g); + freqs_h = freqs_h.squeeze(.g); + freqs_w = freqs_w.squeeze(.g); + + const h_indices = zml.Tensor.iota(zml.Shape.init(.{ .h = self.mrope_section[1] }, .i32), .h).scale(3).addConstant(1); + const w_indices = zml.Tensor.iota(zml.Shape.init(.{ .h = self.mrope_section[2] }, .i32), .h).scale(3).addConstant(2); + + const h_input = freqs_h.gather(.{ .dh = h_indices }, .{ .indices_are_sorted = true }); + const w_input = freqs_w.gather(.{ .dh = w_indices }, .{ .indices_are_sorted = true }); + freqs_t = freqs_t.scatterSlices(.{ .dh = h_indices }, h_input, .{ .update_fn = zml.Tensor.ScatterOpts.override }); + freqs = freqs_t.scatterSlices(.{ .dh = w_indices }, w_input, .{ .update_fn = zml.Tensor.ScatterOpts.override }); + + const emb = zml.Tensor.concatenate(&.{ freqs, freqs }, -1); + const cos = emb.cos().convert(dtype); + const sin = emb.sin().convert(dtype); + + return .{ cos, sin }; + } + + fn rotateHalf(x: zml.Tensor) zml.Tensor { + const half_dim = @divExact(x.dim(-1), 2); + const x1 = x.slice(-1, .{ .start = 0, .end = half_dim }); + const x2 = x.slice(-1, .{ .start = half_dim, .end = x.dim(-1) }); + return zml.Tensor.concatenate(&.{ x2.negate(), x1 }, -1); + } + + pub fn applyRope(self: TextRotaryEmbedding, x: zml.Tensor, cos: zml.Tensor, sin: zml.Tensor) zml.Tensor { + const x_rot = x.slice(-1, .{ .start = 0, .end = self.rotary_dim }); + const x_pass = x.slice(-1, .{ .start = self.rotary_dim, .end = x.dim(-1) }); + + const cos_x = cos.insertAxes(.hd, .{.h}).broad(x_rot.shape()); + const sin_x = sin.insertAxes(.hd, .{.h}).broad(x_rot.shape()); + + const rotated = x_rot.mul(cos_x).add(rotateHalf(x_rot).mul(sin_x)); + + return zml.Tensor.concatenate(&.{ rotated, x_pass }, -1); + } +}; + +pub const GatedDeltaNet = struct { + in_proj_qkv: zml.nn.Linear, + in_proj_z: zml.nn.Linear, + in_proj_b: zml.nn.Linear, + in_proj_a: zml.nn.Linear, + out_proj: zml.nn.Linear, + conv1d_weight: zml.Tensor, + dt_bias: zml.Tensor, + aLog: zml.Tensor, + norm: RmsNormGated, + + num_k_heads: i64, + num_v_heads: i64, + qk_head_repetition: u32, + head_k_dim: i64, + head_v_dim: i64, + conv_kernel_size: i64, + + fn initProj(store: zml.io.TensorStore.View, partitions: anytype) zml.nn.Linear { + return .init(store.createTensor("weight", .{ .dout, .d }, .model, partitions), null, .d); + } + + pub fn init(store: zml.io.TensorStore.View, config: Config) GatedDeltaNet { + const qk_head_repetition: u32 = + @intCast(@divExact(config.text_config.linear_num_value_heads, config.text_config.linear_num_key_heads)); + return .{ + .in_proj_qkv = initProj(store.withPrefix("in_proj_qkv"), .{ .dout = .model, .d = .replicated }), + .in_proj_z = initProj(store.withPrefix("in_proj_z"), .{ .dout = .model, .d = .replicated }), + .in_proj_b = initProj(store.withPrefix("in_proj_b"), .{ .dout = .model, .d = .replicated }), + .in_proj_a = initProj(store.withPrefix("in_proj_a"), .{ .dout = .model, .d = .replicated }), + .out_proj = initProj(store.withPrefix("out_proj"), .{ .dout = .replicated, .d = .model }), + .conv1d_weight = store.withPrefix("conv1d").createTensor("weight", .{ .out, .in, .kernel_size }, .model, .{ .out = .model, .in = .replicated, .kernel_size = .replicated }), + .dt_bias = store.createTensor("dt_bias", .{.vh}, .model, .{ .vh = .model }), + .aLog = store.createTensor("A_log", .{.vh}, .model, .{ .vh = .model }), + .norm = RmsNormGated.init(store.withPrefix("norm"), config.text_config.rms_norm_eps), + .num_k_heads = config.text_config.linear_num_key_heads, + .num_v_heads = config.text_config.linear_num_value_heads, + .qk_head_repetition = qk_head_repetition, + .head_k_dim = config.text_config.linear_key_head_dim, + .head_v_dim = config.text_config.linear_value_head_dim, + .conv_kernel_size = config.text_config.linear_conv_kernel_dim, + }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(GatedDeltaNet)) void { + zml.Buffer.deinitAll(GatedDeltaNet, self); + } + + fn recurrentGatedDeltaRule( + query: zml.Tensor, + key: zml.Tensor, + value: zml.Tensor, + g: zml.Tensor, + beta: zml.Tensor, + initial_state: ?zml.Tensor, + active_length: zml.Tensor, + ) struct { zml.Tensor, zml.Tensor } { + const scale: f32 = 1.0 / @sqrt(@as(f32, @floatFromInt(query.dim(.khd)))); + const query_norm = zml.nn.normalizeL2(query.rename(.{ .kh = .vh }), 1e-6); + const key_norm = zml.nn.normalizeL2(key.rename(.{ .kh = .vh }), 1e-6); + + const query_f32, const key_f32, const value_f32, const alpha_f32, const beta_f32 = .{ + query_norm.convert(.f32).scale(scale).rename(.{ .vh = .h, .khd = .k }), + key_norm.convert(.f32).rename(.{ .vh = .h, .khd = .k }), + value.convert(.f32).rename(.{ .vh = .h, .vhd = .v }), + g.convert(.f32).exp().rename(.{ .vh = .h }), + beta.convert(.f32).rename(.{ .vh = .h }), + }; + + const initial_recurrent_state = b: { + if (initial_state) |state| { + break :b state.convert(.f32).transpose(.{ .b, .vh, .vhd, .khd }).rename(.{ .vh = .h, .vhd = .v, .khd = .k }); + } + + break :b zml.Tensor.constant(zml.DataType.zero(.f32)).broad(zml.Shape.init(.{ + .b = value.dim(.b), + .h = value.dim(.vh), + .v = value.dim(.vhd), + .k = query.dim(.khd), + }, .f32)); + }; + + const result = zml.nn.GatedDeltaNet.forward( + query_f32, + key_f32, + value_f32, + alpha_f32, + beta_f32, + .{ .s = initial_recurrent_state }, + active_length, + ); + + return .{ + result.outputs.rename(.{ .h = .vh, .v = .vhd }).convert(query.dtype()), + result.state.s.transpose(.{ .b, .h, .k, .v }).rename(.{ .h = .vh, .k = .khd, .v = .vhd }), + }; + } + + fn buildUpdatedConvState(input: zml.Tensor, left_pad: i64) zml.Tensor { + const copy_len = @min(input.dim(.s), left_pad); + const tail = input.slice(.s, .{ .start = input.dim(.s) - copy_len, .end = input.dim(.s) }); + if (copy_len == left_pad) return tail; + + const padding_shape = zml.Shape.init(.{ .b = input.dim(.b), .s = left_pad - copy_len, .mix = input.dim(.mix) }, input.dtype()); + const padding = zml.Tensor.constant(input.dtype().zero()).broad(padding_shape); + return zml.Tensor.concatenate(&.{ padding, tail }, .s); + } + + fn buildUpdatedConvStateFromPrefix(input: zml.Tensor, left_pad: i64, active_length: zml.Tensor) zml.Tensor { + const padding = zml.Tensor.zeroes(input.shape().setDim(.s, left_pad)); + const padded = zml.Tensor.concatenate(&.{ padding, input }, .s); + return padded.slice(.s, .dyn(active_length.convert(.i64), left_pad)); + } + + pub fn forward(self: GatedDeltaNet, x: zml.Tensor, cache: KvCache.GatedDeltaNetCache, active_length: zml.Tensor) struct { zml.Tensor, KvCache.GatedDeltaNetCache } { + const key_dim = self.num_k_heads * self.head_k_dim; + const value_dim = self.num_v_heads * self.head_v_dim; + const conv_dim = 2 * key_dim + value_dim; + const left_pad = self.conv_kernel_size - 1; + + const x_in = x.withPartitioning(.model, .{ .d = .replicated }); + const projected_qkv = self.in_proj_qkv.forward(x_in, x_in.dtype()) + .rename(.{ .dout = .mix }) + .withPartitioning(.model, .{ .s = .replicated, .mix = .model }); + const use_cached_state = x.dim(.s) == 1 and left_pad > 0; + const conv_input = b: { + if (use_cached_state) break :b zml.Tensor.concatenate(&.{ cache.convState(), projected_qkv }, .s); + break :b projected_qkv; + }; + + const kernel = self.conv1d_weight; + var mixed_qkv = zml.Tensor.conv1d( + conv_input, + kernel, + .{ + .padding = &.{ left_pad, 0 }, + .input_batch_dimension = 0, + .input_feature_dimension = 2, + .input_spatial_dimensions = 1, + .kernel_output_feature_dimension = 0, + .kernel_input_feature_dimension = 1, + .kernel_spatial_dimensions = 2, + .output_batch_dimension = 0, + .output_feature_dimension = 2, + .output_spatial_dimensions = 1, + .feature_group_count = conv_dim, + }, + ) + .silu(); + + if (use_cached_state) { + mixed_qkv = mixed_qkv.slice(.s, .{ .start = mixed_qkv.dim(.s) - 1, .end = mixed_qkv.dim(.s) }); + } + mixed_qkv = mixed_qkv.withPartitioning(.model, .{ .s = .replicated, .mix = .model }); + + const z = self.in_proj_z.forward(x_in, x_in.dtype()) + .splitAxis(.dout, .{ .vh = self.num_v_heads, .vhd = self.head_v_dim }); + const b = self.in_proj_b.forward(x_in, x_in.dtype()).rename(.{ .dout = .vh }); + const a = self.in_proj_a.forward(x_in, x_in.dtype()).rename(.{ .dout = .vh }); + + const query = mixed_qkv + .slice(.mix, .{ .start = 0, .end = key_dim }) + .splitAxis(.mix, .{ .kh = self.num_k_heads, .khd = self.head_k_dim }); + const key = mixed_qkv + .slice(.mix, .{ .start = key_dim, .end = 2 * key_dim }) + .splitAxis(.mix, .{ .kh = self.num_k_heads, .khd = self.head_k_dim }); + const value = mixed_qkv + .slice(.mix, .{ .start = 2 * key_dim, .end = 2 * key_dim + value_dim }) + .splitAxis(.mix, .{ .vh = self.num_v_heads, .vhd = self.head_v_dim }); + + const beta = b.sigmoid(); + const aLog_type = self.aLog.dtype(); + const g = self.aLog.broad(a.shape()).exp().mul(zml.nn.softplus(a.convert(aLog_type).add(self.dt_bias.convert(aLog_type).broad(a.shape())), .{})).negate(); + + const query_for_rule = b: { + if (self.qk_head_repetition == 1) break :b query; + break :b query.stutter1d(query.axis(.kh), self.qk_head_repetition); + }; + const key_for_rule = b: { + if (self.qk_head_repetition == 1) break :b key; + break :b key.stutter1d(key.axis(.kh), self.qk_head_repetition); + }; + + const core_attn_out, const last_recurrent_state = recurrentGatedDeltaRule( + query_for_rule, + key_for_rule, + value, + g, + beta, + b: { + if (use_cached_state) break :b cache.recurrentState(); + break :b null; + }, + active_length, + ); + + const core_attn_out_normed = self.norm + .forward( + core_attn_out.rename(.{ .vhd = .d }), + z.rename(.{ .vhd = .d }), + ) + .rename(.{ .d = .vhd }); + + const output = self.out_proj + .forward(core_attn_out_normed.merge(.{ .d = .{ .vh, .vhd } }), core_attn_out_normed.dtype()) + .rename(.{ .dout = .d }) + .withPartitioning(.model, .{ .d = .replicated }); + const updated_cache = cache.update( + if (use_cached_state) buildUpdatedConvState(conv_input, left_pad) else buildUpdatedConvStateFromPrefix(projected_qkv, left_pad, active_length), + last_recurrent_state, + ); + return .{ output, updated_cache }; + } +}; + +pub const RmsNorm = struct { + weight: zml.Tensor, + eps: f32 = 1e-6, + + pub fn init(store: zml.io.TensorStore.View, eps: f32) RmsNorm { + return .{ .weight = store.createTensor("weight", .{.d}, .model, .{ .d = .replicated }), .eps = eps }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(RmsNorm)) void { + self.weight.deinit(); + } + + pub fn forward(self: RmsNorm, x: zml.Tensor) zml.Tensor { + const x_f32 = x.convert(.f32); + const weight_f32 = self.weight.convert(.f32); + + const normalized = zml.nn.rmsNorm(x_f32, .d, self.eps); + return normalized.mul(weight_f32.broad(x.shape())).add(normalized).convert(x.dtype()); + } +}; + +pub const RmsNormGated = struct { + weight: zml.Tensor, + eps: f32 = 1e-6, + + pub fn init(store: zml.io.TensorStore.View, eps: f32) RmsNormGated { + return .{ .weight = store.createTensor("weight", .{.d}, .model, .{ .d = .replicated }), .eps = eps }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(RmsNormGated)) void { + self.weight.deinit(); + } + + pub fn forward(self: RmsNormGated, x: zml.Tensor, gate: zml.Tensor) zml.Tensor { + const x_f32 = x.convert(.f32); + const gate_f32 = gate.convert(.f32); + const weight_f32 = self.weight.convert(.f32); + + const normalized = zml.nn.rmsNorm(x_f32, .d, self.eps); + const output = normalized.mul(weight_f32.broad(x.shape())); + + const gated_output = output.mul(gate_f32.silu()); + return gated_output.convert(x.dtype()); + } +}; + +pub const KvCache = struct { + layer_types: []const LayerType, + self_attn: SelfAttnCache, + gated_delta_net: GatedDeltaNetCache, + + pub const SelfAttnCache = struct { + k: zml.Tensor, + v: zml.Tensor, + layer_index: zml.Tensor, + + pub const Buffers = zml.Bufferized(SelfAttnCache); + + pub fn init(config: Config, batch_dim: i64, max_seq_len: i64, dtype: zml.DataType, tp: *const zml.Mesh) SelfAttnCache { + const num_self_attn_layers = countLayers(config.text_config.layer_types, .full_attention); + const kv_shape = zml.Shape.init(.{ + .b = batch_dim, + .layer = num_self_attn_layers, + .s = max_seq_len, + .h = config.text_config.num_key_value_heads, + .hd = config.text_config.head_dim, + }, dtype); + const kv_head_sharding = tp.shardableDim(kv_shape.dim(.h), .model, config.text_config.num_attention_heads); + const sharded_kv_shape = switch (kv_head_sharding) { + .sharded => |heads| kv_shape.setDim(.h, heads.dim).withPartitioning(tp, .{ .h = .model }), + .replicated => kv_shape.withPartitioning(tp, .{ .h = .replicated }), + }; + return .{ + .k = .fromShape(sharded_kv_shape), + .v = .fromShape(sharded_kv_shape), + .layer_index = .init(.{}, .u32), + }; + } + + pub fn initBuffer(kv: SelfAttnCache, io: std.Io, platform: *const zml.Platform) !SelfAttnCache.Buffers { + return .{ + .k = try zml.Buffer.uninitialized(io, platform, kv.k.shape(), .{}), + .v = try zml.Buffer.uninitialized(io, platform, kv.v.shape(), .{}), + .layer_index = try zml.Buffer.scalar(io, platform, 0, .u32), + }; + } + + pub fn deinitBuffer(kv: *SelfAttnCache.Buffers) void { + kv.k.deinit(); + kv.v.deinit(); + kv.layer_index.deinit(); + } + + pub fn keys(kv: SelfAttnCache) zml.Tensor { + return kv.k.slice(.layer, .dynSingle(kv.layer_index)); + } + + pub fn values(kv: SelfAttnCache) zml.Tensor { + return kv.v.slice(.layer, .dynSingle(kv.layer_index)); + } + + pub fn update(kv: SelfAttnCache, new_k: zml.Tensor, new_v: zml.Tensor, token_index: ?zml.Tensor) SelfAttnCache { + const k_shape = kv.k.shape().drop(.layer); + var layer = kv.layer_index; + if (token_index) |idx| { + layer = layer.broad(idx.shape()); + return .{ + .k = kv.k.scatterSlices( + .{ .layer = layer, .s = idx }, + new_k.convert(kv.k.dtype()).transpose(k_shape), + .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ).reuseBuffer(kv.k), + .v = kv.v.scatterSlices( + .{ .layer = layer, .s = idx }, + new_v.convert(kv.v.dtype()).transpose(k_shape), + .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ).reuseBuffer(kv.v), + .layer_index = kv.layer_index, + }; + } + + return .{ + .k = kv.k.scatterSlices( + .{ .layer = layer }, + new_k.convert(kv.k.dtype()).transpose(k_shape), + .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ).reuseBuffer(kv.k), + .v = kv.v.scatterSlices( + .{ .layer = layer }, + new_v.convert(kv.v.dtype()).transpose(k_shape), + .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ).reuseBuffer(kv.v), + .layer_index = kv.layer_index, + }; + } + pub fn atLayer(kv: SelfAttnCache, layer_index: usize) SelfAttnCache { + return .{ + .k = kv.k, + .v = kv.v, + .layer_index = zml.Tensor.scalar(layer_index, .u32), + }; + } + + pub fn reuseBuffer(kv: SelfAttnCache, other: SelfAttnCache) SelfAttnCache { + return .{ + .k = kv.k.reuseBuffer(other.k), + .v = kv.v.reuseBuffer(other.v), + .layer_index = kv.layer_index.reuseBuffer(other.layer_index), + }; + } + }; + + pub const GatedDeltaNetCache = struct { + conv_state: zml.Tensor, + recurrent_state: zml.Tensor, + layer_index: zml.Tensor, + + pub const Buffers = zml.Bufferized(GatedDeltaNetCache); + + pub fn init(config: Config, batch_dim: i64, conv_dtype: zml.DataType, recurrent_dtype: zml.DataType, tp: *const zml.Mesh) GatedDeltaNetCache { + const num_linear_attn_layers = countLayers(config.text_config.layer_types, .linear_attention); + const conv_dim = + 2 * config.text_config.linear_num_key_heads * config.text_config.linear_key_head_dim + + config.text_config.linear_num_value_heads * config.text_config.linear_value_head_dim; + const conv_state_shape = zml.Shape.init(.{ + .b = batch_dim, + .layer = num_linear_attn_layers, + .s = config.text_config.linear_conv_kernel_dim - 1, + .mix = conv_dim, + }, conv_dtype); + const recurrent_state_shape = zml.Shape.init(.{ + .b = batch_dim, + .layer = num_linear_attn_layers, + .vh = config.text_config.linear_num_value_heads, + .khd = config.text_config.linear_key_head_dim, + .vhd = config.text_config.linear_value_head_dim, + }, recurrent_dtype); + const sharded_conv_state_shape = conv_state_shape.withPartitioning(tp, .{ .mix = .model }); + const sharded_recurrent_state_shape = recurrent_state_shape.withPartitioning(tp, .{ .vh = .model }); + return .{ + .conv_state = zml.Tensor.fromShape(sharded_conv_state_shape), + .recurrent_state = zml.Tensor.fromShape(sharded_recurrent_state_shape), + .layer_index = .init(.{}, .u32), + }; + } + + pub fn initBuffer(self: GatedDeltaNetCache, io: std.Io, platform: *const zml.Platform) !GatedDeltaNetCache.Buffers { + return .{ + .conv_state = try zml.Buffer.uninitialized(io, platform, self.conv_state.shape(), .{}), + .recurrent_state = try zml.Buffer.uninitialized(io, platform, self.recurrent_state.shape(), .{}), + .layer_index = try zml.Buffer.scalar(io, platform, 0, .u32), + }; + } + + pub fn deinitBuffer(self: *GatedDeltaNetCache.Buffers) void { + self.conv_state.deinit(); + self.recurrent_state.deinit(); + self.layer_index.deinit(); + } + + pub fn convState(self: GatedDeltaNetCache) zml.Tensor { + return self.conv_state.slice(.layer, .dynSingle(self.layer_index)); + } + + pub fn recurrentState(self: GatedDeltaNetCache) zml.Tensor { + return self.recurrent_state.slice(.layer, .dynSingle(self.layer_index)); + } + + pub fn update(self: GatedDeltaNetCache, new_conv_state: ?zml.Tensor, new_recurrent_state: ?zml.Tensor) GatedDeltaNetCache { + const conv_state = b: { + if (new_conv_state) |state| { + break :b self.conv_state.scatterSlices( + .{ .layer = self.layer_index }, + state.convert(self.conv_state.dtype()).transpose(self.conv_state.shape().drop(.layer)), + .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ).reuseBuffer(self.conv_state); + } + + break :b self.conv_state; + }; + + const recurrent_state = b: { + if (new_recurrent_state) |state| { + break :b self.recurrent_state.scatterSlices( + .{ .layer = self.layer_index }, + state.convert(self.recurrent_state.dtype()).transpose(self.recurrent_state.shape().drop(.layer)), + .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ).reuseBuffer(self.recurrent_state); + } + + break :b self.recurrent_state; + }; + + return .{ + .conv_state = conv_state, + .recurrent_state = recurrent_state, + .layer_index = self.layer_index, + }; + } + + pub fn atLayer(self: GatedDeltaNetCache, layer_index: usize) GatedDeltaNetCache { + return .{ + .conv_state = self.conv_state, + .recurrent_state = self.recurrent_state, + .layer_index = zml.Tensor.scalar(layer_index, .u32), + }; + } + + pub fn reuseBuffer(self: GatedDeltaNetCache, other: GatedDeltaNetCache) GatedDeltaNetCache { + return .{ + .conv_state = self.conv_state.reuseBuffer(other.conv_state), + .recurrent_state = self.recurrent_state.reuseBuffer(other.recurrent_state), + .layer_index = self.layer_index.reuseBuffer(other.layer_index), + }; + } + }; + + pub fn init( + config: Config, + batch_dim: i64, + max_seq_len: i64, + cache_dtype: zml.DataType, + recurrent_dtype: zml.DataType, + tp: *const zml.Mesh, + ) KvCache { + return .{ + .layer_types = config.text_config.layer_types, + .self_attn = .init(config, batch_dim, max_seq_len, cache_dtype, tp), + .gated_delta_net = .init(config, batch_dim, cache_dtype, recurrent_dtype, tp), + }; + } + + pub fn initBuffer(self: KvCache, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(KvCache) { + return .{ + .self_attn = try self.self_attn.initBuffer(io, platform), + .gated_delta_net = try self.gated_delta_net.initBuffer(io, platform), + }; + } + + pub fn deinitBuffer(self: *zml.Bufferized(KvCache)) void { + SelfAttnCache.deinitBuffer(&self.self_attn); + GatedDeltaNetCache.deinitBuffer(&self.gated_delta_net); + } + + pub fn reuseBuffer(self: KvCache, other: KvCache) KvCache { + return .{ + .layer_types = self.layer_types, + .self_attn = self.self_attn.reuseBuffer(other.self_attn), + .gated_delta_net = self.gated_delta_net.reuseBuffer(other.gated_delta_net), + }; + } + + pub const LayerView = struct { + parent: KvCache, + cache: union(enum) { + self_attn: SelfAttnCache, + linear_attn: GatedDeltaNetCache, + }, + }; + + pub fn atLayer(self: KvCache, layer_index: usize) LayerView { + return switch (denseIndex(self.layer_types, layer_index)) { + .full_attention => |dense_index| .{ + .parent = self, + .cache = .{ .self_attn = self.self_attn.atLayer(dense_index.layer_dense_index) }, + }, + .linear_attention => |dense_index| .{ + .parent = self, + .cache = .{ .linear_attn = self.gated_delta_net.atLayer(dense_index.layer_dense_index) }, + }, + }; + } + + fn countLayers(layer_types: []const LayerType, layer_type: LayerType) i64 { + return @intCast(std.mem.countScalar(LayerType, layer_types, layer_type)); + } + + fn denseIndex(layer_types: []const LayerType, layer_index: usize) union(enum) { + full_attention: struct { layer_dense_index: usize }, + linear_attention: struct { layer_dense_index: usize }, + } { + var self_attn_layer_index: usize = 0; + var linear_attn_layer_index: usize = 0; + for (layer_types[0..layer_index]) |layer_type| { + switch (layer_type) { + .full_attention => self_attn_layer_index += 1, + .linear_attention => linear_attn_layer_index += 1, + } + } + return switch (layer_types[layer_index]) { + .full_attention => .{ .full_attention = .{ .layer_dense_index = self_attn_layer_index } }, + .linear_attention => .{ .linear_attention = .{ .layer_dense_index = linear_attn_layer_index } }, + }; + } +}; diff --git a/examples/llm/models/qwen3_5/session.zig b/examples/llm/models/qwen3_5/session.zig new file mode 100644 index 00000000..5121fd30 --- /dev/null +++ b/examples/llm/models/qwen3_5/session.zig @@ -0,0 +1,219 @@ +const std = @import("std"); + +const zml = @import("zml"); + +const inference = @import("inference.zig"); +const model = @import("model.zig"); + +pub const Session = struct { + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + compiled_model: *inference.CompiledModel, + prefill: inference.KernelRunner, + decode: inference.KernelRunner, + layer_index_buffers: []zml.Buffer, + kv_cache_buffers: zml.Bufferized(model.KvCache), + rng_buffers: zml.Bufferized(zml.Tensor.Rng), + tokenizer: zml.tokenizer.Tokenizer, + generated_token_slice: zml.Slice, + seqlen: u32, + eos_token_id: u32, + special_tokens: model.Model.SpecialTokens, + think_start: ?u32, + think_end: ?u32, + + pub fn init( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + tokenizer: zml.tokenizer.Tokenizer, + compiled_model: *inference.CompiledModel, + model_buffers: *model.Buffers, + ) !Session { + var kv_cache_buffers = try compiled_model.params.kv_cache.initBuffer(io, platform); + errdefer model.KvCache.deinitBuffer(&kv_cache_buffers); + + const seed: u128 = @intCast(std.Io.Clock.now(.real, io).toNanoseconds()); + var rng_buffers = try zml.Tensor.Rng.init().initBuffer(io, platform, seed); + errdefer zml.Tensor.Rng.deinitBuffer(&rng_buffers); + + const layer_types = compiled_model.loaded_model.inner.config.text_config.layer_types; + const layer_index_buffers = try allocator.alloc(zml.Buffer, layer_types.len); + errdefer allocator.free(layer_index_buffers); + var initialized_layer_index_buffers: usize = 0; + errdefer for (layer_index_buffers[0..initialized_layer_index_buffers]) |*buffer| buffer.deinit(); + var full_attention_index: u32 = 0; + var linear_attention_index: u32 = 0; + for (layer_index_buffers, layer_types) |*buffer, layer_type| { + buffer.* = try switch (layer_type) { + .full_attention => b: { + defer full_attention_index += 1; + break :b zml.Buffer.scalar(io, platform, full_attention_index, .u32); + }, + .linear_attention => b: { + defer linear_attention_index += 1; + break :b zml.Buffer.scalar(io, platform, linear_attention_index, .u32); + }, + }; + initialized_layer_index_buffers += 1; + } + + const generated_token_slice = try zml.Slice.alloc(allocator, zml.Shape.init(.{ .b = 1, .s = 1 }, .u32)); + errdefer generated_token_slice.free(allocator); + + var prefill = try inference.KernelRunner.init(allocator, &compiled_model.prefill, model_buffers); + errdefer prefill.deinit(allocator); + const decode = try inference.KernelRunner.init(allocator, &compiled_model.decode, model_buffers); + + return .{ + .allocator = allocator, + .io = io, + .platform = platform, + .compiled_model = compiled_model, + .prefill = prefill, + .decode = decode, + .layer_index_buffers = layer_index_buffers, + .kv_cache_buffers = kv_cache_buffers, + .rng_buffers = rng_buffers, + .tokenizer = tokenizer, + .generated_token_slice = generated_token_slice, + .seqlen = compiled_model.params.seqlen, + .eos_token_id = compiled_model.loaded_model.inner.special_tokens.end_of_text_token_id, + .special_tokens = compiled_model.loaded_model.inner.special_tokens, + .think_start = tokenizer.tokenId(""), + .think_end = tokenizer.tokenId(""), + }; + } + + pub fn deinit(self: *Session) void { + self.prefill.deinit(self.allocator); + self.decode.deinit(self.allocator); + for (self.layer_index_buffers) |*buffer| buffer.deinit(); + self.allocator.free(self.layer_index_buffers); + model.KvCache.deinitBuffer(&self.kv_cache_buffers); + zml.Tensor.Rng.deinitBuffer(&self.rng_buffers); + self.generated_token_slice.free(self.allocator); + } + + pub fn tokenizePrompt(self: *const Session, allocator: std.mem.Allocator, prompt: []const u8) ![]const u32 { + return tokenizeChatPrompt(allocator, self.tokenizer, prompt, self.special_tokens, true); + } + + pub fn tokenizeTurn(self: *const Session, allocator: std.mem.Allocator, prompt: []const u8) ![]const u32 { + return tokenizeChatPrompt(allocator, self.tokenizer, prompt, self.special_tokens, false); + } + + pub fn runPrefill(self: *Session, all_tokens: []const u32) !void { + const prefill_tokens_shape = zml.Shape.init(.{ .b = 1, .s = self.seqlen }, .u32); + const prefill_tokens_slice = try zml.Slice.alloc(self.allocator, prefill_tokens_shape); + defer prefill_tokens_slice.free(self.allocator); + @memset(prefill_tokens_slice.items(u32), 0); + @memcpy(prefill_tokens_slice.items(u32)[0..all_tokens.len], all_tokens); + + var prefill_tokens_buffer = try zml.Buffer.fromSlice(self.io, self.platform, prefill_tokens_slice); + defer prefill_tokens_buffer.deinit(); + + var prefill_token_index_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, 0), .u32); + defer prefill_token_index_buffer.deinit(); + var active_length_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, @intCast(all_tokens.len)), .u32); + defer active_length_buffer.deinit(); + + var next_token_buffer: zml.Buffer = undefined; + inference.run(&self.prefill, .{ + .io = self.io, + .tokens_buf = &prefill_tokens_buffer, + .next_token_buf = &next_token_buffer, + .token_index_buf = &prefill_token_index_buffer, + .active_length_buf = &active_length_buffer, + .kv_cache_buffers = &self.kv_cache_buffers, + .rng_buffers = &self.rng_buffers, + }, self.layer_index_buffers); + + defer next_token_buffer.deinit(); + try next_token_buffer.toSlice(self.io, self.generated_token_slice); + } + + pub fn runDecode(self: *Session, all_tokens: *std.ArrayList(u32), stdout: *std.Io.Writer) !void { + var decoder = try self.tokenizer.decoder(); + defer decoder.deinit(); + + const out_tokens_buffer: []u8 = try self.allocator.alloc(u8, 1024); + defer self.allocator.free(out_tokens_buffer); + + var current_token_buffer = try zml.Buffer.fromSlice(self.io, self.platform, self.generated_token_slice); + defer current_token_buffer.deinit(); + + var token_index_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, @intCast(all_tokens.items.len)), .u32); + defer token_index_buffer.deinit(); + var active_length_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, 1), .u32); + defer active_length_buffer.deinit(); + + generation: while (true) { + const token_id = self.generated_token_slice.items(u32)[0]; + if (token_id == self.eos_token_id) break :generation; + + const token = try decoder.feedOne(token_id, out_tokens_buffer); + if (self.think_start) |think_start| if (token_id == think_start) { + try stdout.writeAll("\x1b[2m"); + }; + try stdout.writeAll(token); + if (self.think_end) |think_end| if (token_id == think_end) { + try stdout.writeAll("\x1b[0m"); + }; + try stdout.flush(); + + try all_tokens.append(self.allocator, token_id); + if (all_tokens.items.len >= self.seqlen) break :generation; + + inference.run(&self.decode, .{ + .io = self.io, + .tokens_buf = ¤t_token_buffer, + .next_token_buf = ¤t_token_buffer, + .token_index_buf = &token_index_buffer, + .active_length_buf = &active_length_buffer, + .kv_cache_buffers = &self.kv_cache_buffers, + .rng_buffers = &self.rng_buffers, + }, self.layer_index_buffers); + + try current_token_buffer.toSlice(self.io, self.generated_token_slice); + } + + try stdout.writeAll(try decoder.finalize(out_tokens_buffer)); + try stdout.flush(); + } +}; + +fn tokenizeChatPrompt( + allocator: std.mem.Allocator, + tokenizer: zml.tokenizer.Tokenizer, + prompt: []const u8, + special_tokens: model.Model.SpecialTokens, + is_first_turn: bool, +) ![]const u32 { + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + const im_start = tokenizer.tokenId("<|im_start|>") orelse special_tokens.im_start_token_id; + const im_end = tokenizer.tokenId("<|im_end|>") orelse special_tokens.im_end_token_id; + const newline = tokenizer.tokenId("\\n") orelse return error.NoSuchToken; + + var tokens: std.ArrayList(u32) = try .initCapacity(allocator, 32); + if (!is_first_turn) { + try tokens.appendSlice(allocator, &.{ im_end, newline }); + } + + try tokens.append(allocator, im_start); + const user_tokens = try encoder.encodeAlloc(allocator, "user\n"); + defer allocator.free(user_tokens); + try tokens.appendSlice(allocator, user_tokens); + const prompt_tokens = try encoder.encodeAlloc(allocator, prompt); + defer allocator.free(prompt_tokens); + try tokens.appendSlice(allocator, prompt_tokens); + try tokens.appendSlice(allocator, &.{ im_end, newline, im_start }); + const assistant_tokens = try encoder.encodeAlloc(allocator, "assistant\n"); + defer allocator.free(assistant_tokens); + try tokens.appendSlice(allocator, assistant_tokens); + + return tokens.toOwnedSlice(allocator); +} diff --git a/examples/llm/models/qwen3_5_moe.zig b/examples/llm/models/qwen3_5_moe.zig new file mode 100644 index 00000000..a97a3207 --- /dev/null +++ b/examples/llm/models/qwen3_5_moe.zig @@ -0,0 +1,11 @@ +pub const inference = @import("qwen3_5_moe/inference.zig"); +pub const CompilationParameters = inference.CompilationParameters; +pub const CompilationOptions = inference.CompilationParameters; +pub const CompiledModel = inference.CompiledModel; +pub const model = @import("qwen3_5_moe/model.zig"); +pub const Config = model.Config; +pub const Buffers = model.Buffers; +pub const Model = model.Model; +pub const LoadedModel = model.LoadedModel; +pub const session = @import("qwen3_5_moe/session.zig"); +pub const Session = session.Session; diff --git a/examples/llm/models/qwen3_5_moe/inference.zig b/examples/llm/models/qwen3_5_moe/inference.zig new file mode 100644 index 00000000..896744b3 --- /dev/null +++ b/examples/llm/models/qwen3_5_moe/inference.zig @@ -0,0 +1,354 @@ +const std = @import("std"); + +const zml = @import("zml"); + +const common = @import("../common.zig"); +const model = @import("model.zig"); + +const log = std.log.scoped(.qwen3_5_moe); + +pub const CompilationParameters = struct { + kv_cache: model.KvCache, + rng: zml.Tensor.Rng, + moe_backend: zml.moe.Backend, + seqlen: u32, + meshes: common.Meshes, + xla_dump_to: ?[]const u8, + + pub fn init(mdl: model.Model, config: model.Config, seqlen: u32, moe_backend: zml.moe.Backend, meshes: common.Meshes) CompilationParameters { + const dtype = mdl.text_model.embed_tokens.weight.dtype(); + return .{ + .kv_cache = .init(config, 1, seqlen, dtype, .f32, meshes.model), + .rng = .init(), + .moe_backend = moe_backend, + .seqlen = seqlen, + .meshes = meshes, + .xla_dump_to = "/home/ubuntu/xla_dump", + }; + } +}; + +pub const CompilationOptions = CompilationParameters; + +pub const LayerIndexBuffer = union(enum) { + self_attn: zml.Buffer, + linear_attn: zml.Buffer, +}; + +pub const RunArgs = struct { + io: std.Io, + tokens_buffer: *zml.Buffer, + next_token_buffer: *zml.Buffer, + token_index_buffer: *zml.Buffer, + active_length_buffer: *zml.Buffer, + kv_cache_buffers: *zml.Bufferized(model.KvCache), + rng_buffers: *zml.Bufferized(zml.Tensor.Rng), + layer_index_buffers: []const LayerIndexBuffer, +}; + +pub const CompiledModel = struct { + loaded_model: *const model.LoadedModel, + prefill: KernelExe, + decode: KernelExe, + params: CompilationParameters, + + pub fn init( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + loaded_model: *const model.LoadedModel, + qwen_model: model.Model, + parameters: CompilationParameters, + progress: *std.Progress.Node, + ) !CompiledModel { + const prefill = try compileKernel(allocator, io, platform, qwen_model, parameters, parameters.seqlen, "prefill", progress); + errdefer prefill.deinit(); + const decode = try compileKernel(allocator, io, platform, qwen_model, parameters, 1, "decode", progress); + return .{ + .loaded_model = loaded_model, + .prefill = prefill, + .decode = decode, + .params = parameters, + }; + } + + pub fn deinit(self: *CompiledModel) void { + self.prefill.deinit(); + self.decode.deinit(); + } +}; + +pub const Inference = CompiledModel; + +pub const KernelExe = struct { + embed: zml.FnExe(model.EmbedTokens.forward), + full_attention: zml.FnExe(model.TransformerLayer.forwardSelfAttn), + linear_attention: zml.FnExe(model.TransformerLayer.forwardLinearAttn), + sample: zml.FnExe(model.Sampler.sampleTokens), + + pub fn deinit(self: *const KernelExe) void { + self.embed.deinit(); + self.full_attention.deinit(); + self.linear_attention.deinit(); + self.sample.deinit(); + } +}; + +pub const LayerRunner = union(enum) { + full_attention: zml.FnExe(model.TransformerLayer.forwardSelfAttn).Runner(.{.layer}), + linear_attention: zml.FnExe(model.TransformerLayer.forwardLinearAttn).Runner(.{.layer}), + + pub fn deinit(self: *LayerRunner, allocator: std.mem.Allocator) void { + switch (self.*) { + inline else => |*runner| runner.deinit(allocator), + } + } +}; + +pub const KernelRunner = struct { + embed: zml.FnExe(model.EmbedTokens.forward).Runner(.{.embedding}), + layers: []LayerRunner, + sample: zml.FnExe(model.Sampler.sampleTokens).Runner(.{.sampler}), + + pub fn init(allocator: std.mem.Allocator, exe: *const KernelExe, buffers: *const model.Buffers) !KernelRunner { + var embed = try zml.FnExe(model.EmbedTokens.forward).Runner(.{.embedding}).init(&exe.embed, allocator, .{ + .embedding = buffers.text_model.embed_tokens, + }); + errdefer embed.deinit(allocator); + + const layers = try allocator.alloc(LayerRunner, buffers.text_model.layers.len); + errdefer allocator.free(layers); + var initialized_layers: usize = 0; + errdefer for (layers[0..initialized_layers]) |*layer| layer.deinit(allocator); + for (layers, buffers.text_model.layers) |*layer, layer_buffers| { + layer.* = switch (layer_buffers.attn) { + .self_attn => .{ .full_attention = try zml.FnExe(model.TransformerLayer.forwardSelfAttn).Runner(.{.layer}).init( + &exe.full_attention, + allocator, + .{ .layer = layer_buffers }, + ) }, + .linear_attn => .{ .linear_attention = try zml.FnExe(model.TransformerLayer.forwardLinearAttn).Runner(.{.layer}).init( + &exe.linear_attention, + allocator, + .{ .layer = layer_buffers }, + ) }, + }; + initialized_layers += 1; + } + + var sample = try zml.FnExe(model.Sampler.sampleTokens).Runner(.{.sampler}).init(&exe.sample, allocator, .{ + .sampler = .{ + .norm = buffers.text_model.norm, + .lm_head = buffers.text_model.lm_head, + }, + }); + errdefer sample.deinit(allocator); + + return .{ .embed = embed, .layers = layers, .sample = sample }; + } + + pub fn deinit(self: *KernelRunner, allocator: std.mem.Allocator) void { + self.embed.deinit(allocator); + for (self.layers) |*layer| layer.deinit(allocator); + allocator.free(self.layers); + self.sample.deinit(allocator); + } +}; + +pub fn run(runner: *KernelRunner, args: RunArgs) void { + var hidden_buffer: zml.Buffer = undefined; + runner.embed.run(args.io, .{ + .inputs = .{ + .tokens = args.tokens_buffer.*, + }, + .outputs = .{ .hidden = &hidden_buffer }, + }, .{}); + defer hidden_buffer.deinit(); + + for (runner.layers, args.layer_index_buffers) |*layer_runner, layer_index_buffer| { + switch (layer_runner.*) { + .full_attention => |*layer| { + const index_buffer = switch (layer_index_buffer) { + .self_attn => |buffer| buffer, + .linear_attn => unreachable, + }; + var layer_cache: zml.Bufferized(model.KvCache.SelfAttnCache) = .{ + .k = args.kv_cache_buffers.self_attn.k, + .v = args.kv_cache_buffers.self_attn.v, + .layer_index = index_buffer, + }; + layer.run(args.io, .{ + .inputs = .{ + .hidden = hidden_buffer, + .token_index = args.token_index_buffer.*, + .cache = layer_cache, + }, + .outputs = .{ .hidden = &hidden_buffer, .cache = &layer_cache }, + }, .{}); + args.kv_cache_buffers.self_attn.k = layer_cache.k; + args.kv_cache_buffers.self_attn.v = layer_cache.v; + layer_cache.layer_index.deinit(); + }, + .linear_attention => |*layer| { + const index_buffer = switch (layer_index_buffer) { + .linear_attn => |buffer| buffer, + .self_attn => unreachable, + }; + var layer_cache: zml.Bufferized(model.KvCache.GatedDeltaNetCache) = .{ + .conv_state = args.kv_cache_buffers.gated_delta_net.conv_state, + .recurrent_state = args.kv_cache_buffers.gated_delta_net.recurrent_state, + .layer_index = index_buffer, + }; + layer.run(args.io, .{ + .inputs = .{ + .hidden = hidden_buffer, + .active_length = args.active_length_buffer.*, + .cache = layer_cache, + }, + .outputs = .{ .hidden = &hidden_buffer, .cache = &layer_cache }, + }, .{}); + args.kv_cache_buffers.gated_delta_net.conv_state = layer_cache.conv_state; + args.kv_cache_buffers.gated_delta_net.recurrent_state = layer_cache.recurrent_state; + layer_cache.layer_index.deinit(); + }, + } + } + + runner.sample.run(args.io, .{ + .inputs = .{ + .hidden = hidden_buffer, + .tokens = args.tokens_buffer.*, + .active_length = args.active_length_buffer.*, + .rng = args.rng_buffers.*, + .token_index = args.token_index_buffer.*, + }, + .outputs = .{ + .tokens = args.next_token_buffer, + .rng = args.rng_buffers, + .token_index = args.token_index_buffer, + }, + }, .{}); +} + +fn compileKernel( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + mdl: model.Model, + parameters: CompilationParameters, + seqlen: usize, + phase: []const u8, + progress: *std.Progress.Node, +) !KernelExe { + const full_index = findFirstLayerIndex(mdl.config.text_config.layer_types, .full_attention) orelse return error.MissingFullAttentionLayer; + const linear_index = findFirstLayerIndex(mdl.config.text_config.layer_types, .linear_attention) orelse return error.MissingLinearAttentionLayer; + + var embed_future = try io.concurrent(compileEmbed, .{ allocator, io, platform, mdl, parameters, seqlen, phase, progress }); + errdefer if (embed_future.cancel(io)) |exe| exe.deinit() else |_| {}; + var full_attention_future = try io.concurrent(compileFullAttention, .{ allocator, io, platform, mdl, parameters, seqlen, full_index, phase, progress }); + errdefer if (full_attention_future.cancel(io)) |exe| exe.deinit() else |_| {}; + var linear_attention_future = try io.concurrent(compileLinearAttention, .{ allocator, io, platform, mdl, parameters, seqlen, linear_index, phase, progress }); + errdefer if (linear_attention_future.cancel(io)) |exe| exe.deinit() else |_| {}; + var sample_future = try io.concurrent(compileSample, .{ allocator, io, platform, mdl, parameters, seqlen, phase, progress }); + errdefer if (sample_future.cancel(io)) |exe| exe.deinit() else |_| {}; + + const embed = try embed_future.await(io); + errdefer embed.deinit(); + const full_attention = try full_attention_future.await(io); + errdefer full_attention.deinit(); + const linear_attention = try linear_attention_future.await(io); + errdefer linear_attention.deinit(); + const sample = try sample_future.await(io); + errdefer sample.deinit(); + + return .{ + .embed = embed, + .full_attention = full_attention, + .linear_attention = linear_attention, + .sample = sample, + }; +} + +fn compileEmbed(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mdl: model.Model, parameters: CompilationParameters, seqlen: usize, phase: []const u8, progress: *std.Progress.Node) !zml.FnExe(model.EmbedTokens.forward) { + return compileExe(allocator, io, platform, model.EmbedTokens.forward, .{.{ + .embedding = mdl.text_model.embed_tokens, + .tokens = zml.Tensor.init(.{ .b = 1, .s = seqlen }, .u32), + }}, parameters, progress, phase, "embedding"); +} + +fn compileFullAttention(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mdl: model.Model, parameters: CompilationParameters, seqlen: usize, layer_index: usize, phase: []const u8, progress: *std.Progress.Node) !zml.FnExe(model.TransformerLayer.forwardSelfAttn) { + return compileExe(allocator, io, platform, model.TransformerLayer.forwardSelfAttn, .{.{ + .layer = mdl.text_model.layers[layer_index], + .hidden = hiddenTensor(mdl, seqlen, parameters.meshes.model), + .token_index = zml.Tensor.init(.{}, .u32), + .cache = .{ + .k = parameters.kv_cache.self_attn.k, + .v = parameters.kv_cache.self_attn.v, + .layer_index = zml.Tensor.init(.{}, .u32), + }, + .config = mdl.config, + .moe_backend = parameters.moe_backend, + }}, parameters, progress, phase, "full-attention layer"); +} + +fn compileLinearAttention(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mdl: model.Model, parameters: CompilationParameters, seqlen: usize, layer_index: usize, phase: []const u8, progress: *std.Progress.Node) !zml.FnExe(model.TransformerLayer.forwardLinearAttn) { + return compileExe(allocator, io, platform, model.TransformerLayer.forwardLinearAttn, .{.{ + .layer = mdl.text_model.layers[layer_index], + .hidden = hiddenTensor(mdl, seqlen, parameters.meshes.model), + .active_length = zml.Tensor.init(.{}, .u32), + .cache = .{ + .conv_state = parameters.kv_cache.gated_delta_net.conv_state, + .recurrent_state = parameters.kv_cache.gated_delta_net.recurrent_state, + .layer_index = zml.Tensor.init(.{}, .u32), + }, + .config = mdl.config, + .moe_backend = parameters.moe_backend, + }}, parameters, progress, phase, "linear-attention layer"); +} + +fn compileSample(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mdl: model.Model, parameters: CompilationParameters, seqlen: usize, phase: []const u8, progress: *std.Progress.Node) !zml.FnExe(model.Sampler.sampleTokens) { + return compileExe(allocator, io, platform, model.Sampler.sampleTokens, .{.{ + .sampler = mdl.text_model.sampler(), + .hidden = hiddenTensor(mdl, seqlen, parameters.meshes.model), + .tokens = .init(.{ .b = 1, .s = seqlen }, .u32), + .active_length = .init(.{}, .u32), + .rng = parameters.rng, + .token_index = zml.Tensor.init(.{}, .u32), + }}, parameters, progress, phase, "sampling"); +} + +fn compileExe( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + comptime function: anytype, + args: std.meta.ArgsTuple(@TypeOf(function)), + parameters: CompilationParameters, + progress: *std.Progress.Node, + phase: []const u8, + component: []const u8, +) !zml.FnExe(function) { + progress.increaseEstimatedTotalItems(1); + const label = try std.fmt.allocPrint(allocator, "Compiling {s} {s}...", .{ phase, component }); + defer allocator.free(label); + var node = progress.start(label, 1); + defer node.end(); + const now: std.Io.Timestamp = .now(io, .awake); + defer log.info("Compiled {s} {s} [{f}]", .{ phase, component, now.untilNow(io, .awake) }); + return zml.FnExe(function).compile(allocator, io, platform, .{ + .meshes = ¶meters.meshes.all(), + .xla_dump_to = parameters.xla_dump_to, + }, args); +} + +fn hiddenTensor(mdl: model.Model, seqlen: usize, mesh: *const zml.Mesh) zml.Tensor { + return .fromShape(zml.Shape.init( + .{ .b = 1, .s = seqlen, .d = mdl.config.text_config.hidden_size }, + mdl.text_model.embed_tokens.weight.dtype(), + ).withPartitioning(mesh, .{ .b = .replicated, .s = .replicated, .d = .replicated })); +} + +fn findFirstLayerIndex(layer_types: []const model.LayerType, target: model.LayerType) ?usize { + for (layer_types, 0..) |layer_type, index| if (layer_type == target) return index; + return null; +} diff --git a/examples/llm/models/qwen3_5_moe/model.zig b/examples/llm/models/qwen3_5_moe/model.zig new file mode 100644 index 00000000..2a3b0ba8 --- /dev/null +++ b/examples/llm/models/qwen3_5_moe/model.zig @@ -0,0 +1,1423 @@ +const std = @import("std"); + +const zml = @import("zml"); +const stdx = zml.stdx; + +const common = @import("../common.zig"); +const inference = @import("inference.zig"); + +const log = std.log.scoped(.qwen3_5_moe); + +pub const Config = struct { + text_config: TextConfig, +}; + +pub const TextConfig = struct { + // General + num_hidden_layers: i64, + layer_types: []const LayerType, + hidden_size: i64, + max_position_embeddings: i64, + rms_norm_eps: f32, + // Self attention + head_dim: i64, + num_attention_heads: i64, + num_key_value_heads: i64, + rope_parameters: RopeParameters, + // Linear attention + linear_conv_kernel_dim: i64, + linear_key_head_dim: i64, + linear_num_key_heads: i64, + linear_num_value_heads: i64, + linear_value_head_dim: i64, + // MoE + num_experts: i64, + num_experts_per_tok: u32, +}; + +// Each layer uses either: full attention (SelfAttn) or linear attention (GatedDeltaNet). +pub const LayerType = enum { + linear_attention, + full_attention, +}; + +pub const RopeParameters = struct { + mrope_section: [3]i64, + partial_rotary_factor: f32, + rope_theta: f32, +}; + +fn partitionProjectedKv(kv: zml.Tensor, kv_head_sharding: zml.Sharding.DimSharding) zml.Tensor { + return switch (kv_head_sharding) { + .sharded => |heads| blk: { + const sharded_kv = if (heads.factor != 1) kv.stutter1d(kv.axis(.h), heads.factor) else kv; + break :blk sharded_kv.withPartitioning(.model, .{ .s = .replicated, .h = .model, .hd = .replicated }); + }, + .replicated => kv.withPartitioning(.model, .{ .s = .replicated, .h = .replicated, .hd = .replicated }), + }; +} + +fn partitionCachedKv(tensor: zml.Tensor, kv_head_sharding: zml.Sharding.DimSharding) zml.Tensor { + var kv = tensor.rename(.{ .s = .k }); + return switch (kv_head_sharding) { + .sharded => |heads| blk: { + if (heads.factor != 1) { + kv = kv.stutter1d(kv.axis(.h), heads.factor); + } + break :blk kv.withPartitioning(.model, .{ .k = .replicated, .h = .model, .hd = .replicated }); + }, + .replicated => kv.withPartitioning(.model, .{ .k = .replicated, .h = .replicated, .hd = .replicated }), + }; +} + +pub const LoadedModel = struct { + inner: Model, + parsed_config: std.json.Parsed(Config), + + pub fn init( + allocator: std.mem.Allocator, + io: std.Io, + repo: std.Io.Dir, + store: zml.io.TensorStore.View, + generation: common.GenerationOptions, + ) !LoadedModel { + const parsed_config = try common.parseConfig(Config, allocator, io, repo); + errdefer parsed_config.deinit(); + + const options: Model.GenOptions = .{ + .sampling_strategy = generation.sampling_strategy, + .max_seq_len = parsed_config.value.text_config.max_position_embeddings, + }; + + return .{ + .inner = try .init(allocator, store, parsed_config.value, options), + .parsed_config = parsed_config, + }; + } + + pub fn deinit(self: *LoadedModel, allocator: std.mem.Allocator) void { + self.inner.deinit(allocator); + self.parsed_config.deinit(); + } + + pub fn loadBuffers( + self: *LoadedModel, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + store: *zml.io.TensorStore, + progress: *std.Progress.Node, + ) !Buffers { + progress.increaseEstimatedTotalItems(store.view().count()); + const now: std.Io.Timestamp = .now(io, .awake); + + var buffers = try zml.mem.bufferize(allocator, Model, &self.inner); + errdefer self.unloadBuffers(&buffers, allocator); + + var loader: zml.io.Loader = try .init(allocator, platform, .{ + .dma_chunks = 32, + .dma_chunk_size = 256 * zml.MiB, + .parallelism = 16, + }); + defer loader.deinit(); + + try loader.load(io, Model, &self.inner, &buffers, store, .{ .progress = progress }); + try loader.await(io); + + const took = now.untilNow(io, .awake); + const total_bytes: u64 = loader.bytes_loaded.raw; + const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); + log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); + + return buffers; + } + + pub fn unloadBuffers(self: *const LoadedModel, buffers: *Buffers, allocator: std.mem.Allocator) void { + _ = self; + TextModel.unloadBuffers(&buffers.text_model, allocator); + } + + pub fn compile( + self: *const LoadedModel, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + backend: zml.attention.Backend, + meshes: common.Meshes, + seqlen: usize, + progress: *std.Progress.Node, + ) !inference.CompiledModel { + _ = backend; + const moe_gate_up = self.inner.text_model.layers[0].moe.gate_up_proj; + log.info("Moe dtype : {}", .{moe_gate_up.weight.dtype()}); + const moe_backend = try zml.moe.Backend.auto(platform, moe_gate_up.quantizationScheme(), moe_gate_up.weight.dtype()); + const params = inference.CompilationParameters.init(self.inner, self.parsed_config.value, @intCast(seqlen), moe_backend, meshes); + return inference.CompiledModel.init(allocator, io, platform, self, self.inner, params, progress); + } +}; + +pub const Buffers = zml.Bufferized(Model); + +pub const Model = struct { + pub const GenOptions = struct { sampling_strategy: zml.nn.SamplingStrategy = .{}, max_seq_len: i64 }; + + pub const SpecialTokens = struct { + im_start_token_id: u32, + im_end_token_id: u32, + end_of_text_token_id: u32, + }; + + text_model: TextModel, + + config: Config, + special_tokens: SpecialTokens = .{ + .im_start_token_id = 248045, + .im_end_token_id = 248046, + .end_of_text_token_id = 248044, + }, + + pub fn init(allocator: std.mem.Allocator, store: zml.io.TensorStore.View, config: Config, gen_options: GenOptions) !Model { + // For some Qwen3.5 versions, the output projection lm_head has a standalone weight tensor, while for others it's the same as the input embedding layer + return .{ + .text_model = try .init(allocator, store.withPrefix("model.language_model"), config, gen_options), + .config = config, + }; + } + + pub fn deinit(self: Model, allocator: std.mem.Allocator) void { + self.text_model.deinit(allocator); + } + + pub fn load( + self: *const Model, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + store: *zml.io.TensorStore, + meshes: []const *const zml.Mesh, + progress: *std.Progress.Node, + ) !zml.Bufferized(Model) { + progress.increaseEstimatedTotalItems(store.view().count()); + const now: std.Io.Timestamp = .now(io, .awake); + + var buffers = try zml.mem.bufferize(allocator, Model, self); + errdefer Model.unloadBuffers(&buffers, allocator); + + var loader: zml.io.Loader = try .init(allocator, platform, .{ + .dma_chunks = 32, + .dma_chunk_size = 256 * zml.MiB, + .parallelism = 16, + }); + defer loader.deinit(); + + loader.load(io, Model, self, &buffers, store, meshes); + try loader.await(io); + + const took = now.untilNow(io, .awake); + const total_bytes: u64 = loader.bytes_loaded.raw; + const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); + log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); + + return buffers; + } + + pub fn unloadBuffers(self: *zml.Bufferized(Model), allocator: std.mem.Allocator) void { + TextModel.unloadBuffers(&self.text_model, allocator); + } + pub fn forward( + self: Model, + tokens_: zml.Tensor, + token_index: zml.Tensor, + active_length: zml.Tensor, + kv_cache: KvCache, + rng: zml.Tensor.Rng, + moe_backend: zml.moe.Backend, + ) struct { zml.Tensor, KvCache, zml.Tensor.Rng } { + const tokens = tokens_.withPartialTags(.{.s}); + const new_tokens, const updated_kv_cache, const new_rng = self.text_model.forward(tokens, token_index, active_length, kv_cache, self.config, rng, moe_backend); + return .{ new_tokens, updated_kv_cache, new_rng }; + } +}; + +//========================Text model======================== + +// Use this intermediate struct to compile sampleTokens without taking the whole TextModel as input +pub const Sampler = struct { + norm: RmsNorm, + lm_head: zml.nn.Linear, + gen_options: Model.GenOptions, + + pub const Input = struct { + sampler: Sampler, + hidden: zml.Tensor, + tokens: zml.Tensor, + active_length: zml.Tensor, + rng: zml.Tensor.Rng, + token_index: zml.Tensor, + }; + + pub const Output = struct { + tokens: zml.Tensor, + rng: zml.Tensor.Rng, + token_index: zml.Tensor, + }; + + pub fn sampleTokens(input: Input) Output { + const self = input.sampler; + const last_index = input.active_length.convert(.i32).addConstant(-1); + const x = self.norm.forward(input.hidden.slice(.s, .dyn(last_index, 1))); + const logits = self.lm_head.forward(x.withPartialTags(.{.d}), x.dtype()).rename(.{ .dout = .voc }); + const next_tokens, const new_rng = zml.nn.sampleTokens(logits, self.gen_options.sampling_strategy, input.rng); + const tokens = next_tokens.convert(.u32); + return .{ + .tokens = if (input.tokens.dim(.s) == 1) tokens.reuseBuffer(input.tokens) else tokens, + .rng = new_rng, + .token_index = input.token_index.addConstant(1).reuseBuffer(input.token_index), + }; + } +}; + +pub const EmbedTokens = struct { + pub const Input = struct { + embedding: zml.nn.TokenEmbedding, + tokens: zml.Tensor, + }; + + pub const Output = struct { + hidden: zml.Tensor, + }; + + pub fn forward(input: Input) Output { + return .{ .hidden = input.embedding.forward(input.tokens) }; + } +}; + +pub const TextModel = struct { + embed_tokens: zml.nn.TokenEmbedding, + layers: []TransformerLayer, + norm: RmsNorm, + lm_head: zml.nn.Linear, + gen_options: Model.GenOptions, + + pub fn init( + allocator: std.mem.Allocator, + store: zml.io.TensorStore.View, + config: Config, + gen_options: Model.GenOptions, + ) !TextModel { + const lm_head_prefix = if (store.root().hasKey("lm_head.weight")) "lm_head" else "model.language_model.embed_tokens"; + + const layers = try allocator.alloc(TransformerLayer, @intCast(config.text_config.num_hidden_layers)); + errdefer allocator.free(layers); + + for (layers, 0..) |*layer, i| { + errdefer for (layers[0..i]) |previous_layer| previous_layer.deinit(allocator); + layer.* = try .init(allocator, store.withPrefix("layers").withLayer(i), config, i); + } + + return .{ + .embed_tokens = .{ .weight = store.createTensor("embed_tokens.weight", .{ .voc, .d }, .model, .{ .voc = .replicated, .d = .model }) }, + .layers = layers, + .norm = RmsNorm.init(store.withPrefix("norm"), config.text_config.rms_norm_eps), + .lm_head = .init(store.root().withPrefix(lm_head_prefix).createTensor("weight", .{ .dout, .d }, .model, .{ .dout = .model, .d = .replicated }), null, .d), + .gen_options = gen_options, + }; + } + + pub fn deinit(self: TextModel, allocator: std.mem.Allocator) void { + for (self.layers) |layer| { + layer.deinit(allocator); + } + allocator.free(self.layers); + } + + pub fn unloadBuffers(self: *zml.Bufferized(TextModel), allocator: std.mem.Allocator) void { + self.embed_tokens.weight.deinit(); + for (self.layers) |*layer| { + TransformerLayer.unloadBuffers(layer, allocator); + } + allocator.free(self.layers); + RmsNorm.unloadBuffers(&self.norm); + zml.nn.Linear.unloadBuffers(&self.lm_head); + } + + pub fn sampler(self: TextModel) Sampler { + return .{ + .norm = self.norm, + .lm_head = self.lm_head, + .gen_options = self.gen_options, + }; + } + + pub fn forward( + self: TextModel, + tokens: zml.Tensor, + token_index: zml.Tensor, + active_length: zml.Tensor, + kv_cache: KvCache, + config: Config, + rng: zml.Tensor.Rng, + moe_backend: zml.moe.Backend, + ) struct { zml.Tensor, KvCache, zml.Tensor.Rng } { + var hidden_states = EmbedTokens.forward(.{ + .embedding = self.embed_tokens, + .tokens = tokens, + }).hidden; + + var updated_kv_cache = kv_cache; + for (self.layers, 0..) |layer, i| { + hidden_states, updated_kv_cache = layer.forward(hidden_states, token_index, active_length, updated_kv_cache.atLayer(i), config, moe_backend); + } + + const result = Sampler.sampleTokens(.{ + .sampler = self.sampler(), + .hidden = hidden_states, + .tokens = tokens, + .active_length = active_length, + .rng = rng, + .token_index = token_index, + }); + return .{ result.tokens, updated_kv_cache.reuseBuffer(kv_cache), result.rng }; + } +}; + +pub const TransformerLayer = struct { + const Attn = union(enum) { + self_attn: SelfAttn, + linear_attn: GatedDeltaNet, + }; + + input_layernorm: RmsNorm, + attn: Attn, + moe: Moe, + post_attention_layernorm: RmsNorm, + + pub const SelfAttnInput = struct { + layer: TransformerLayer, + hidden: zml.Tensor, + token_index: zml.Tensor, + cache: KvCache.SelfAttnCache, + config: Config, + moe_backend: zml.moe.Backend, + }; + + pub const SelfAttnOutput = struct { + hidden: zml.Tensor, + cache: KvCache.SelfAttnCache, + }; + + pub const LinearAttnInput = struct { + layer: TransformerLayer, + hidden: zml.Tensor, + active_length: zml.Tensor, + cache: KvCache.GatedDeltaNetCache, + config: Config, + moe_backend: zml.moe.Backend, + }; + + pub const LinearAttnOutput = struct { + hidden: zml.Tensor, + cache: KvCache.GatedDeltaNetCache, + }; + + pub fn init(allocator: std.mem.Allocator, store: zml.io.TensorStore.View, config: Config, layer_index: usize) !TransformerLayer { + const is_full_attention = config.text_config.layer_types[layer_index] == .full_attention; + return .{ + .input_layernorm = RmsNorm.init(store.withPrefix("input_layernorm"), config.text_config.rms_norm_eps), + .attn = if (is_full_attention) + .{ .self_attn = try .init(store.withPrefix("self_attn"), config) } + else + .{ .linear_attn = .init(store.withPrefix("linear_attn"), config) }, + .moe = try .init(allocator, store.withPrefix("mlp"), config), + .post_attention_layernorm = RmsNorm.init(store.withPrefix("post_attention_layernorm"), config.text_config.rms_norm_eps), + }; + } + + pub fn deinit(self: TransformerLayer, allocator: std.mem.Allocator) void { + self.moe.deinit(allocator); + } + + pub fn unloadBuffers(self: *zml.Bufferized(TransformerLayer), allocator: std.mem.Allocator) void { + RmsNorm.unloadBuffers(&self.input_layernorm); + switch (self.attn) { + .self_attn => |*self_attn| SelfAttn.unloadBuffers(self_attn), + .linear_attn => |*linear_attn| GatedDeltaNet.unloadBuffers(linear_attn), + } + Moe.unloadBuffers(&self.moe, allocator); + RmsNorm.unloadBuffers(&self.post_attention_layernorm); + } + + pub fn forwardSelfAttn(input: SelfAttnInput) SelfAttnOutput { + const self = input.layer; + const x0 = input.hidden; + _ = input.config; + const x0_replicated = x0.withPartitioning(.model, .{ .d = .replicated }); + const normalized_x0 = self.input_layernorm.forward(x0_replicated); + + const self_attn = switch (self.attn) { + .self_attn => |self_attn| self_attn, + .linear_attn => unreachable, + }; + const attention_output, const updated_kv_cache = self_attn.forward(normalized_x0, input.token_index, input.cache); + + const x1 = attention_output.add(x0_replicated).withPartitioning(.model, .{ .d = .replicated }); + const normalized_hidden = self.post_attention_layernorm.forward(x1); + + const moe_output = self.moe.forward(normalized_hidden, input.moe_backend); + + return .{ + .hidden = moe_output.add(x1).withPartitioning(.model, .{ .d = .replicated }).reuseBuffer(x0), + .cache = updated_kv_cache, + }; + } + + pub fn forwardLinearAttn(input: LinearAttnInput) LinearAttnOutput { + const self = input.layer; + const x0 = input.hidden; + _ = input.config; + const x0_replicated = x0.withPartitioning(.model, .{ .d = .replicated }); + const normalized_x0 = self.input_layernorm.forward(x0_replicated); + + const linear_attn = switch (self.attn) { + .linear_attn => |linear_attn| linear_attn, + .self_attn => unreachable, + }; + const attention_output, const updated_kv_cache = linear_attn.forward(normalized_x0, input.cache, input.active_length); + + const x1 = attention_output.add(x0_replicated).withPartitioning(.model, .{ .d = .replicated }); + const normalized_hidden = self.post_attention_layernorm.forward(x1); + + const moe_output = self.moe.forward(normalized_hidden, input.moe_backend); + + return .{ + .hidden = moe_output.add(x1).withPartitioning(.model, .{ .d = .replicated }).reuseBuffer(x0), + .cache = updated_kv_cache, + }; + } + + pub fn forward( + self: TransformerLayer, + x0: zml.Tensor, + token_index: zml.Tensor, + active_length: zml.Tensor, + kv_cache: KvCache.LayerView, + config: Config, + moe_backend: zml.moe.Backend, + ) struct { zml.Tensor, KvCache } { + _ = config; + const x0_replicated = x0.withPartitioning(.model, .{ .d = .replicated }); + const normalized_x0 = self.input_layernorm.forward(x0_replicated); + + var attention_output: zml.Tensor = undefined; + var updated_kv_cache: KvCache = kv_cache.parent; + switch (self.attn) { + .self_attn => |*self_attn| { + const result = self_attn.forward(normalized_x0, token_index, kv_cache.cache.self_attn); + attention_output = result[0]; + updated_kv_cache.self_attn = result[1]; + }, + .linear_attn => |*linear_attn| { + const result = linear_attn.forward(normalized_x0, kv_cache.cache.linear_attn, active_length); + attention_output = result[0]; + updated_kv_cache.gated_delta_net = result[1]; + }, + } + + const x1 = attention_output.add(x0_replicated).withPartitioning(.model, .{ .d = .replicated }); + const normalized_hidden = self.post_attention_layernorm.forward(x1); + + const moe_output = self.moe.forward(normalized_hidden, moe_backend); + + return .{ moe_output.add(x1).withPartitioning(.model, .{ .d = .replicated }), updated_kv_cache }; + } +}; + +pub const SelfAttn = struct { + q_proj: zml.nn.Linear, + q_proj_scale: ?zml.Tensor, + k_proj: zml.nn.Linear, + k_proj_scale: ?zml.Tensor, + v_proj: zml.nn.Linear, + v_proj_scale: ?zml.Tensor, + + q_norm: RmsNorm, + k_norm: RmsNorm, + + num_heads: i64, + num_kv_heads: i64, + head_dim: i64, + rotary_dim: i64, + rotary_embed: TextRotaryEmbedding, + o_proj: zml.nn.Linear, + + o_proj_scale: ?zml.Tensor, + + fn initProj(store: zml.io.TensorStore.View, partitions: anytype, bias_partitions: anytype) zml.nn.Linear { + return .init( + store.createTensor("weight", .{ .dout, .d }, .model, partitions), + store.maybeCreateTensor("bias", .{.dout}, .model, bias_partitions), + .d, + ); + } + + pub fn init(store: zml.io.TensorStore.View, config: Config) !SelfAttn { + const rotary_dim: i64 = @intFromFloat(@as(f32, @floatFromInt(config.text_config.head_dim)) * + config.text_config.rope_parameters.partial_rotary_factor); + return .{ + .q_proj = initProj(store.withPrefix("q_proj"), .{ .dout = .model, .d = .replicated }, .{ .dout = .model }), + .q_proj_scale = null, + .k_proj = initProj(store.withPrefix("k_proj"), .{ .dout = .model, .d = .replicated }, .{ .dout = .model }), + .k_proj_scale = null, + .v_proj = initProj(store.withPrefix("v_proj"), .{ .dout = .model, .d = .replicated }, .{ .dout = .model }), + .v_proj_scale = null, + .o_proj = initProj(store.withPrefix("o_proj"), .{ .dout = .replicated, .d = .model }, .{ .dout = .replicated }), + .o_proj_scale = null, + .q_norm = RmsNorm.init(store.withPrefix("q_norm"), config.text_config.rms_norm_eps), + .k_norm = RmsNorm.init(store.withPrefix("k_norm"), config.text_config.rms_norm_eps), + .num_heads = config.text_config.num_attention_heads, + .num_kv_heads = config.text_config.num_key_value_heads, + .head_dim = config.text_config.head_dim, + .rotary_dim = rotary_dim, + .rotary_embed = .init(rotary_dim, config.text_config.rope_parameters.rope_theta, config.text_config.rope_parameters.mrope_section), + }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(SelfAttn)) void { + zml.Buffer.deinitAll(SelfAttn, self); + } + + fn projectQAndGate(self: SelfAttn, x: zml.Tensor) struct { zml.Tensor, zml.Tensor } { + const q_proj = self.q_proj.forward(x, x.dtype()) + .splitAxis(.dout, .{ .h = self.num_heads, .hd = 2 * self.head_dim }); + const q, var gate = q_proj.chunkExact(.hd, 2); + gate = gate.merge(.{ .d_out_proj = .{ .h, .hd } }); + return .{ q, gate }; + } + + fn projectKV(self: SelfAttn, x: zml.Tensor) struct { zml.Tensor, zml.Tensor } { + const num_kv_heads = if (self.num_kv_heads > 0) self.num_kv_heads else self.num_heads; + const k = self.k_proj.forward(x, x.dtype()) + .splitAxis(.dout, .{ .h = num_kv_heads, .hd = self.head_dim }); + const v = self.v_proj.forward(x, x.dtype()) + .splitAxis(.dout, .{ .h = num_kv_heads, .hd = self.head_dim }); + return .{ k, v }; + } + + pub fn forward( + self: SelfAttn, + x: zml.Tensor, + token_index: zml.Tensor, + kv_cache: KvCache.SelfAttnCache, + ) struct { zml.Tensor, KvCache.SelfAttnCache } { + const tp = zml.Compiler.current().mesh(.model); + const x_qkv = x.withPartitioning(tp, .{ .d = .replicated }); + var q, var gate = self.projectQAndGate(x_qkv); + var k, var v = self.projectKV(x_qkv); + const kv_head_sharding = tp.shardableDim(k.dim(.h), .model, q.dim(.h)); + + q = q.withPartitioning(tp, .{ .s = .replicated, .h = .model, .hd = .replicated }); + gate = gate.withPartitioning(tp, .{ .s = .replicated, .d_out_proj = .model }); + k = partitionProjectedKv(k, kv_head_sharding); + v = partitionProjectedKv(v, kv_head_sharding); + q = self.q_norm.forward(q.rename(.{ .hd = .d })).rename(.{ .d = .hd }); + k = self.k_norm.forward(k.rename(.{ .hd = .d })).rename(.{ .d = .hd }); + + const dtype = q.dtype(); + const position_ids: zml.Tensor = zml.Tensor.arange(.{ .end = x.dim(.s) }, .i64) + .withTags(.{.s}).insertAxes(.s, .{.b}).broad(zml.Shape.init(.{ .b = x.dim(.b), .s = x.dim(.s) }, .i64)) + .add(token_index.convert(.i64).broad(zml.Shape.init(.{ .b = x.dim(.b), .s = x.dim(.s) }, .i64))); + + const cos, const sin = self.rotary_embed.getCosAndSin(position_ids, dtype); + q = self.rotary_embed.applyRope(q, cos, sin); + k = self.rotary_embed.applyRope(k, cos, sin); + q = q.withPartitioning(tp, .{ .s = .replicated, .h = .model, .hd = .replicated }); + k = partitionProjectedKv(k, kv_head_sharding); + v = partitionProjectedKv(v, kv_head_sharding); + + const new_kv_cache = kv_cache.update(k, v, token_index.convert(.u32)); + k = new_kv_cache.keys().convert(dtype); + v = new_kv_cache.values().convert(dtype); + q = q.rename(.{ .s = .q }).withPartitioning(tp, .{ .q = .replicated, .h = .model, .hd = .replicated }); + k = partitionCachedKv(k, kv_head_sharding); + v = partitionCachedKv(v, kv_head_sharding); + + const head_partitioning: zml.attention.Partitioning = switch (kv_head_sharding) { + .sharded => .head(.model), + .replicated => .head(.replicated), + }; + const attn_output = zml.attention.attention( + q, + k, + v, + token_index, + zml.attention.Metadata.init(.fromBackend(.stablehlo)), + zml.attention.Parameters.init(.fromBackend(.stablehlo)), + zml.Compiler.current().mesh(.model), + head_partitioning, + ).rename(.{ .q = .s }).merge(.{ .d_out_proj = .{ .h, .hd } }); + + const gated_output = attn_output.mul(gate.sigmoid()); + const projected_output = self.o_proj.forward(gated_output.rename(.{ .d_out_proj = .d }), gated_output.dtype()).rename(.{ .dout = .d }).withPartitioning(tp, .{ .d = .replicated }); + + return .{ projected_output, new_kv_cache }; + } +}; + +pub const Mlp = struct { + up_proj: zml.nn.Linear, + up_proj_scale: ?zml.Tensor, + gate_proj: zml.nn.Linear, + gate_proj_scale: ?zml.Tensor, + down_proj: zml.nn.Linear, + down_proj_scale: ?zml.Tensor, + + pub fn init(store: zml.io.TensorStore.View) Mlp { + return .{ + .up_proj = .init( + store.withPrefix("up_proj").createTensor("weight", .{ .dout, .d }, .model, .{ .dout = .model, .d = .replicated }), + store.withPrefix("up_proj").maybeCreateTensor("bias", .{.dout}, .model, .{ .dout = .model }), + .d, + ), + .up_proj_scale = null, + .gate_proj = .init( + store.withPrefix("gate_proj").createTensor("weight", .{ .dout, .d }, .model, .{ .dout = .model, .d = .replicated }), + store.withPrefix("gate_proj").maybeCreateTensor("bias", .{.dout}, .model, .{ .dout = .model }), + .d, + ), + .gate_proj_scale = null, + .down_proj = .init( + store.withPrefix("down_proj").createTensor("weight", .{ .dout, .d }, .model, .{ .dout = .replicated, .d = .model }), + store.withPrefix("down_proj").maybeCreateTensor("bias", .{.d}, .model, .{ .d = .replicated }), + .d, + ), + .down_proj_scale = null, + }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(Mlp)) void { + zml.Buffer.deinitAll(Mlp, self); + } + + pub fn forward(self: Mlp, x: zml.Tensor) zml.Tensor { + const up_projed = self.up_proj.forward(x, x.dtype()); + const gate = self.gate_proj.forward(x, x.dtype()); + const hidden = gate.silu().mul(up_projed).rename(.{ .dout = .d }); + + const output = self.down_proj.forward(hidden, hidden.dtype()); + return output; + } +}; + +const Router = struct { + router: zml.nn.Linear, + num_experts_per_tok: u32, + + pub fn init(store: zml.io.TensorStore.View, num_experts_per_tok: u32) Router { + return .{ + .router = .init( + store.createTensor("weight", .{ .expert, .d }, .replicated, .{ .expert = .replicated, .d = .replicated }), + store.maybeCreateTensor("bias", .{.expert}, .replicated, .{ .expert = .replicated }), + .d, + ), + .num_experts_per_tok = num_experts_per_tok, + }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(Router)) void { + zml.Buffer.deinitAll(Router, self); + } + + pub fn forward(self: Router, x: zml.Tensor) struct { zml.Tensor, zml.Tensor } { + const router_logits = self.router.forward(x, .f32); + const routing = router_logits.topK(.{ .topk = .expert }, self.num_experts_per_tok, .{}); + const topk_ids = routing.indices.convert(.i32); + const router_scores = routing.values.softmax(.topk); + return .{ router_scores, topk_ids }; + } +}; + +pub const Moe = struct { + shared_expert: Mlp, + shared_expert_gate: zml.nn.Linear, + gate_up_proj: zml.nn.Linear, + down_proj: zml.nn.Linear, + router: Router, + + pub fn init(allocator: std.mem.Allocator, store: zml.io.TensorStore.View, config: Config) !Moe { + _ = allocator; + const experts_store = store.withPrefix("experts"); + const gate_up_proj_tensor = experts_store.createTensor( + "gate_up_proj", + .{ .expert, .dout, .d }, + .experts, + .{ .expert = .experts, .dout = .replicated, .d = .replicated }, + ); + const down_proj_tensor = experts_store.createTensor( + "down_proj", + .{ .expert, .d, .dout }, + .experts, + .{ .expert = .experts, .d = .replicated, .dout = .replicated }, + ); + + return .{ + .shared_expert = Mlp.init(store.withPrefix("shared_expert")), + .shared_expert_gate = .init( + store.withPrefix("shared_expert_gate").createTensor("weight", .{ .dout, .d }, .replicated, .{ .dout = .replicated, .d = .replicated }), + store.withPrefix("shared_expert_gate").maybeCreateTensor("bias", .{.dout}, .replicated, .{ .dout = .replicated }), + .d, + ), + .gate_up_proj = .init(gate_up_proj_tensor, null, .d), + .down_proj = .init(down_proj_tensor, null, .dout), + .router = Router.init(store.withPrefix("gate"), config.text_config.num_experts_per_tok), + }; + } + + pub fn deinit(self: Moe, allocator: std.mem.Allocator) void { + _ = self; + _ = allocator; + } + + pub fn forward(self: Moe, x: zml.Tensor, moe_backend: zml.moe.Backend) zml.Tensor { + const routing_scores, const topk_ids = self.router.forward(x); + + const moe_output = zml.moe.forwardMoe( + x.merge(.{ .t = .{ .b, .s } }), + topk_ids.merge(.{ .t = .{ .b, .s } }), + routing_scores.merge(.{ .t = .{ .b, .s } }), + self.gate_up_proj, + self.down_proj, + moe_backend, + zml.Compiler.current().mesh(.experts), + .expert(.experts), + .{ + .activation = .{ .swiglu = .{} }, + .quantize_input = false, + .routing_weight_placement = .after_down, + }, + ).reshape(x.shape()); + + const shared_gate = self.shared_expert_gate.forward(x, x.dtype()).sigmoid().broad(x.shape()); + const shared = self.shared_expert.forward(x).rename(.{ .dout = .d }).mul(shared_gate).withPartitioning(.model, .{ + .b = .replicated, + .s = .replicated, + .d = .replicated, + }); + + return moe_output.add(shared); + } + + pub fn unloadBuffers(self: *zml.Bufferized(Moe), allocator: std.mem.Allocator) void { + _ = allocator; + zml.Buffer.deinitAll(Moe, self); + } +}; + +pub const TextRotaryEmbedding = struct { + rope_opts: zml.nn.RopeOpts, + rotary_dim: i64, + mrope_section: [3]i64, + + pub fn init(rotary_dim: i64, theta: f32, mrope_section: [3]i64) TextRotaryEmbedding { + return .{ + .rope_opts = .{ + .layout = .real_im_pass, + .scaling = .{ .default = .{ .rope_theta = theta } }, + }, + .rotary_dim = rotary_dim, + .mrope_section = mrope_section, + }; + } + + pub fn getCosAndSin(self: TextRotaryEmbedding, position_ids: zml.Tensor, dtype: zml.DataType) struct { zml.Tensor, zml.Tensor } { + const inv_freq = zml.nn.invFreq(self.rotary_dim, self.rope_opts).withTags(.{.hd}); + + const freqs_t = position_ids.convert(.f32).outer(inv_freq); + + const emb = zml.Tensor.concatenate(&.{ freqs_t, freqs_t }, -1); + const cos = emb.cos().convert(dtype); + const sin = emb.sin().convert(dtype); + + return .{ cos, sin }; + } + + pub fn getCosAndSinInterleaved(self: TextRotaryEmbedding, position_ids: zml.Tensor, dtype: zml.DataType) struct { zml.Tensor, zml.Tensor } { // To be used later in image extension + const stacked_position_ids = zml.Tensor.stack(&.{ position_ids, position_ids, position_ids }, 0, .g).convert(.f32); + const inv_freq = zml.nn.InvFreq(self.rotary_dim, self.rope_opts).withTags(.{.hd}); + + var freqs = stacked_position_ids.outer(inv_freq); + var freqs_t, var freqs_h, var freqs_w = freqs.chunkExact(.g, 3); + freqs_t = freqs_t.squeeze(.g); + freqs_h = freqs_h.squeeze(.g); + freqs_w = freqs_w.squeeze(.g); + + const h_indices = zml.Tensor.iota(zml.Shape.init(.{ .h = self.mrope_section[1] }, .i32), .h).scale(3).addConstant(1); + const w_indices = zml.Tensor.iota(zml.Shape.init(.{ .h = self.mrope_section[2] }, .i32), .h).scale(3).addConstant(2); + + const h_input = freqs_h.gather(.{ .dh = h_indices }, .{ .indices_are_sorted = true }); + const w_input = freqs_w.gather(.{ .dh = w_indices }, .{ .indices_are_sorted = true }); + freqs_t = freqs_t.scatterSlices(.{ .dh = h_indices }, h_input, .{ .update_fn = zml.Tensor.ScatterOpts.override }); + freqs = freqs_t.scatterSlices(.{ .dh = w_indices }, w_input, .{ .update_fn = zml.Tensor.ScatterOpts.override }); + + const emb = zml.Tensor.concatenate(&.{ freqs, freqs }, -1); + const cos = emb.cos().convert(dtype); + const sin = emb.sin().convert(dtype); + + return .{ cos, sin }; + } + + fn rotateHalf(x: zml.Tensor) zml.Tensor { + const half_dim = @divExact(x.dim(-1), 2); + const x1 = x.slice(-1, .{ .start = 0, .end = half_dim }); + const x2 = x.slice(-1, .{ .start = half_dim, .end = x.dim(-1) }); + return zml.Tensor.concatenate(&.{ x2.negate(), x1 }, -1); + } + + pub fn applyRope(self: TextRotaryEmbedding, x: zml.Tensor, cos: zml.Tensor, sin: zml.Tensor) zml.Tensor { + const x_rot = x.slice(-1, .{ .start = 0, .end = self.rotary_dim }); + const x_pass = x.slice(-1, .{ .start = self.rotary_dim, .end = x.dim(-1) }); + + const cos_x = cos.insertAxes(.hd, .{.h}).broad(x_rot.shape()); + const sin_x = sin.insertAxes(.hd, .{.h}).broad(x_rot.shape()); + + const rotated = x_rot.mul(cos_x).add(rotateHalf(x_rot).mul(sin_x)); + + return zml.Tensor.concatenate(&.{ rotated, x_pass }, -1); + } +}; + +pub const GatedDeltaNet = struct { + in_proj_qkv: zml.nn.Linear, + in_proj_qkv_scale: ?zml.Tensor, + in_proj_z: zml.nn.Linear, + in_proj_z_scale: ?zml.Tensor, + in_proj_b: zml.nn.Linear, + in_proj_a: zml.nn.Linear, + out_proj: zml.nn.Linear, + out_proj_scale: ?zml.Tensor, + conv1d_weight: zml.Tensor, + dt_bias: zml.Tensor, + aLog: zml.Tensor, + norm: RmsNormGated, + + num_k_heads: i64, + num_v_heads: i64, + qk_head_repetition: u32, + head_k_dim: i64, + head_v_dim: i64, + conv_kernel_size: i64, + + fn initProj(store: zml.io.TensorStore.View, partitions: anytype) zml.nn.Linear { + return .init(store.createTensor("weight", .{ .dout, .d }, .model, partitions), null, .d); + } + + pub fn init(store: zml.io.TensorStore.View, config: Config) GatedDeltaNet { + const qk_head_repetition: u32 = + @intCast(@divExact(config.text_config.linear_num_value_heads, config.text_config.linear_num_key_heads)); + return .{ + .in_proj_qkv = initProj(store.withPrefix("in_proj_qkv"), .{ .dout = .model, .d = .replicated }), + .in_proj_qkv_scale = null, + .in_proj_z = initProj(store.withPrefix("in_proj_z"), .{ .dout = .model, .d = .replicated }), + .in_proj_z_scale = null, + .in_proj_b = initProj(store.withPrefix("in_proj_b"), .{ .dout = .model, .d = .replicated }), + .in_proj_a = initProj(store.withPrefix("in_proj_a"), .{ .dout = .model, .d = .replicated }), + .out_proj = initProj(store.withPrefix("out_proj"), .{ .dout = .replicated, .d = .model }), + .out_proj_scale = null, + .conv1d_weight = store.withPrefix("conv1d").createTensor("weight", .{ .out, .in, .kernel_size }, .model, .{ .out = .model, .in = .replicated, .kernel_size = .replicated }), + .dt_bias = store.createTensor("dt_bias", .{.vh}, .model, .{ .vh = .model }), + .aLog = store.createTensor("A_log", .{.vh}, .model, .{ .vh = .model }), + .norm = RmsNormGated.init(store.withPrefix("norm"), config.text_config.rms_norm_eps), + .num_k_heads = config.text_config.linear_num_key_heads, + .num_v_heads = config.text_config.linear_num_value_heads, + .qk_head_repetition = qk_head_repetition, + .head_k_dim = config.text_config.linear_key_head_dim, + .head_v_dim = config.text_config.linear_value_head_dim, + .conv_kernel_size = config.text_config.linear_conv_kernel_dim, + }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(GatedDeltaNet)) void { + zml.Buffer.deinitAll(GatedDeltaNet, self); + } + + fn recurrentGatedDeltaRule( + query: zml.Tensor, + key: zml.Tensor, + value: zml.Tensor, + g: zml.Tensor, + beta: zml.Tensor, + initial_state: ?zml.Tensor, + active_length: zml.Tensor, + ) struct { zml.Tensor, zml.Tensor } { + const scale: f32 = 1.0 / @sqrt(@as(f32, @floatFromInt(query.dim(.khd)))); + const query_norm = zml.nn.normalizeL2(query.rename(.{ .kh = .vh }), 1e-6); + const key_norm = zml.nn.normalizeL2(key.rename(.{ .kh = .vh }), 1e-6); + + const query_f32, const key_f32, const value_f32, const alpha_f32, const beta_f32 = .{ + query_norm.convert(.f32).scale(scale).rename(.{ .vh = .h, .khd = .k }), + key_norm.convert(.f32).rename(.{ .vh = .h, .khd = .k }), + value.convert(.f32).rename(.{ .vh = .h, .vhd = .v }), + g.convert(.f32).exp().rename(.{ .vh = .h }), + beta.convert(.f32).rename(.{ .vh = .h }), + }; + + const initial_recurrent_state = if (initial_state) |state| + state.convert(.f32).transpose(.{ .b, .vh, .vhd, .khd }).rename(.{ .vh = .h, .vhd = .v, .khd = .k }) + else + zml.Tensor.constant(zml.DataType.zero(.f32)).broad(zml.Shape.init(.{ + .b = value.dim(.b), + .h = value.dim(.vh), + .v = value.dim(.vhd), + .k = query.dim(.khd), + }, .f32)); + + const result = zml.nn.GatedDeltaNet.forward( + query_f32, + key_f32, + value_f32, + alpha_f32, + beta_f32, + .{ .s = initial_recurrent_state }, + active_length, + ); + + return .{ + result.outputs.rename(.{ .h = .vh, .v = .vhd }).convert(query.dtype()), + result.state.s.transpose(.{ .b, .h, .k, .v }).rename(.{ .h = .vh, .k = .khd, .v = .vhd }), + }; + } + + fn buildUpdatedConvState(input: zml.Tensor, left_pad: i64) zml.Tensor { + const copy_len = @min(input.dim(.s), left_pad); + const tail = input.slice(.s, .{ .start = input.dim(.s) - copy_len, .end = input.dim(.s) }); + if (copy_len == left_pad) return tail; + + const padding_shape = zml.Shape.init(.{ .b = input.dim(.b), .s = left_pad - copy_len, .mix = input.dim(.mix) }, input.dtype()); + const padding = zml.Tensor.constant(input.dtype().zero()).broad(padding_shape); + return zml.Tensor.concatenate(&.{ padding, tail }, .s); + } + + fn buildUpdatedConvStateFromPrefix(input: zml.Tensor, left_pad: i64, active_length: zml.Tensor) zml.Tensor { + const padding = zml.Tensor.zeroes(input.shape().setDim(.s, left_pad)); + const padded = zml.Tensor.concatenate(&.{ padding, input }, .s); + return padded.slice(.s, .dyn(active_length.convert(.i64), left_pad)); + } + + pub fn forward(self: GatedDeltaNet, x: zml.Tensor, cache: KvCache.GatedDeltaNetCache, active_length: zml.Tensor) struct { zml.Tensor, KvCache.GatedDeltaNetCache } { + const key_dim = self.num_k_heads * self.head_k_dim; + const value_dim = self.num_v_heads * self.head_v_dim; + const conv_dim = 2 * key_dim + value_dim; + const left_pad = self.conv_kernel_size - 1; + + const x_in = x.withPartitioning(.model, .{ .d = .replicated }); + const projected_qkv = self.in_proj_qkv.forward(x_in, x_in.dtype()) + .rename(.{ .dout = .mix }).withPartitioning(.model, .{ .s = .replicated, .mix = .model }); + const use_cached_state = x.dim(.s) == 1 and left_pad > 0; + const conv_input = if (use_cached_state) + zml.Tensor.concatenate(&.{ cache.convState(), projected_qkv }, .s) + else + projected_qkv; + + const kernel = self.conv1d_weight; + var mixed_qkv = zml.Tensor.conv1d( + conv_input, + kernel, + .{ + .padding = &.{ left_pad, 0 }, + .input_batch_dimension = 0, + .input_feature_dimension = 2, + .input_spatial_dimensions = 1, + .kernel_output_feature_dimension = 0, + .kernel_input_feature_dimension = 1, + .kernel_spatial_dimensions = 2, + .output_batch_dimension = 0, + .output_feature_dimension = 2, + .output_spatial_dimensions = 1, + .feature_group_count = conv_dim, + }, + ) + .silu(); + + if (use_cached_state) { + mixed_qkv = mixed_qkv.slice(.s, .{ .start = mixed_qkv.dim(.s) - 1, .end = mixed_qkv.dim(.s) }); + } + mixed_qkv = mixed_qkv.withPartitioning(.model, .{ .s = .replicated, .mix = .model }); + + const z = self.in_proj_z.forward(x_in, x_in.dtype()) + .splitAxis(.dout, .{ .vh = self.num_v_heads, .vhd = self.head_v_dim }) + .withPartitioning(.model, .{ .s = .replicated, .vh = .model, .vhd = .replicated }); + const b = self.in_proj_b.forward(x_in, x_in.dtype()).rename(.{ .dout = .vh }).withPartitioning(.model, .{ .s = .replicated, .vh = .model }); + const a = self.in_proj_a.forward(x_in, x_in.dtype()).rename(.{ .dout = .vh }).withPartitioning(.model, .{ .s = .replicated, .vh = .model }); + + const query = mixed_qkv + .slice(.mix, .{ .start = 0, .end = key_dim }) + .splitAxis(.mix, .{ .kh = self.num_k_heads, .khd = self.head_k_dim }) + .withPartitioning(.model, .{ .s = .replicated, .kh = .model, .khd = .replicated }); + const key = mixed_qkv + .slice(.mix, .{ .start = key_dim, .end = 2 * key_dim }) + .splitAxis(.mix, .{ .kh = self.num_k_heads, .khd = self.head_k_dim }) + .withPartitioning(.model, .{ .s = .replicated, .kh = .model, .khd = .replicated }); + const value = mixed_qkv + .slice(.mix, .{ .start = 2 * key_dim, .end = 2 * key_dim + value_dim }) + .splitAxis(.mix, .{ .vh = self.num_v_heads, .vhd = self.head_v_dim }) + .withPartitioning(.model, .{ .s = .replicated, .vh = .model, .vhd = .replicated }); + + const beta = b.sigmoid(); + const aLog_type = self.aLog.dtype(); + const g = self.aLog.broad(a.shape()).exp().mul(zml.nn.softplus(a.convert(aLog_type).add(self.dt_bias.convert(aLog_type).broad(a.shape())), .{})).negate(); + + const query_for_rule = if (self.qk_head_repetition == 1) query else query.stutter1d(query.axis(.kh), self.qk_head_repetition); + const key_for_rule = if (self.qk_head_repetition == 1) key else key.stutter1d(key.axis(.kh), self.qk_head_repetition); + + const core_attn_out, const last_recurrent_state = recurrentGatedDeltaRule( + query_for_rule, + key_for_rule, + value, + g, + beta, + if (use_cached_state) cache.recurrentState() else null, + active_length, + ); + + const core_attn_out_normed = self.norm + .forward( + core_attn_out.rename(.{ .vhd = .d }), + z.rename(.{ .vhd = .d }), + ) + .rename(.{ .d = .vhd }) + .withPartitioning(.model, .{ .s = .replicated, .vh = .model, .vhd = .replicated }); + + const output = self.out_proj.forward(core_attn_out_normed.merge(.{ .d = .{ .vh, .vhd } }), core_attn_out_normed.dtype()) + .rename(.{ .dout = .d }).withPartitioning(.model, .{ .d = .replicated }); + const updated_cache = cache.update( + if (use_cached_state) buildUpdatedConvState(conv_input, left_pad) else buildUpdatedConvStateFromPrefix(projected_qkv, left_pad, active_length), + last_recurrent_state, + ); + return .{ output, updated_cache }; + } +}; + +pub const RmsNorm = struct { + weight: zml.Tensor, + eps: f32 = 1e-6, + + pub fn init(store: zml.io.TensorStore.View, eps: f32) RmsNorm { + return .{ .weight = store.createTensor("weight", .{.d}, .model, .{ .d = .replicated }), .eps = eps }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(RmsNorm)) void { + self.weight.deinit(); + } + + pub fn forward(self: RmsNorm, x: zml.Tensor) zml.Tensor { + const x_f32 = x.convert(.f32); + const weight_f32 = self.weight.convert(.f32); + + const normalized = zml.nn.rmsNorm(x_f32, .d, self.eps); + return normalized.mul(weight_f32.broad(x.shape())).add(normalized).convert(x.dtype()); + } +}; + +pub const RmsNormGated = struct { + weight: zml.Tensor, + eps: f32 = 1e-6, + + pub fn init(store: zml.io.TensorStore.View, eps: f32) RmsNormGated { + return .{ .weight = store.createTensor("weight", .{.d}, .model, .{ .d = .replicated }), .eps = eps }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(RmsNormGated)) void { + self.weight.deinit(); + } + + pub fn forward(self: RmsNormGated, x: zml.Tensor, gate: zml.Tensor) zml.Tensor { + const x_f32 = x.convert(.f32); + const gate_f32 = gate.convert(.f32); + const weight_f32 = self.weight.convert(.f32); + + const normalized = zml.nn.rmsNorm(x_f32, .d, self.eps); + const output = normalized.mul(weight_f32.broad(x.shape())); + + const gated_output = output.mul(gate_f32.silu()); + return gated_output.convert(x.dtype()); + } +}; + +pub const KvCache = struct { + layer_types: []const LayerType, + self_attn: SelfAttnCache, + gated_delta_net: GatedDeltaNetCache, + + pub const SelfAttnCache = struct { + k: zml.Tensor, + v: zml.Tensor, + layer_index: zml.Tensor, + + pub fn init(config: Config, batch_dim: i64, max_seq_len: i64, dtype: zml.DataType, model_meshe: *const zml.Mesh) SelfAttnCache { + const num_self_attn_layers = countLayers(config.text_config.layer_types, .full_attention); + const kv_shape = zml.Shape.init(.{ + .b = batch_dim, + .layer = num_self_attn_layers, + .s = max_seq_len, + .h = config.text_config.num_key_value_heads, + .hd = config.text_config.head_dim, + }, dtype); + const kv_head_sharding = model_meshe.shardableDim(kv_shape.dim(.h), .model, config.text_config.num_attention_heads); + const sharded_kv_shape = switch (kv_head_sharding) { + .sharded => |heads| kv_shape.setDim(.h, heads.dim).withPartitioning(model_meshe, .{ .h = .model }), + .replicated => kv_shape.withPartitioning(model_meshe, .{ .h = .replicated }), + }; + return .{ + .k = zml.Tensor.fromShape(sharded_kv_shape), + .v = zml.Tensor.fromShape(sharded_kv_shape), + .layer_index = .init(.{}, .u32), + }; + } + + pub fn initBuffer(self: SelfAttnCache, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(SelfAttnCache) { + return .{ + .k = try zml.Buffer.uninitialized(io, platform, self.k.shape(), .{}), + .v = try zml.Buffer.uninitialized(io, platform, self.v.shape(), .{}), + .layer_index = try zml.Buffer.scalar(io, platform, 0, .u32), + }; + } + + pub fn deinitBuffer(self: *zml.Bufferized(SelfAttnCache)) void { + self.k.deinit(); + self.v.deinit(); + self.layer_index.deinit(); + } + + pub fn keys(self: SelfAttnCache) zml.Tensor { + return self.k.slice(.layer, .dynSingle(self.layer_index)); + } + + pub fn values(self: SelfAttnCache) zml.Tensor { + return self.v.slice(.layer, .dynSingle(self.layer_index)); + } + + pub fn update(self: SelfAttnCache, new_k: zml.Tensor, new_v: zml.Tensor, token_index: ?zml.Tensor) SelfAttnCache { + const k_shape = self.k.shape().drop(.layer); + var layer = self.layer_index; + layer = if (token_index) |idx| layer.broad(idx.shape()) else layer; + + return if (token_index) |idx| .{ + .k = self.k.scatterSlices( + .{ .layer = layer, .s = idx }, + new_k.convert(self.k.dtype()).transpose(k_shape), + .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ).reuseBuffer(self.k), + .v = self.v.scatterSlices( + .{ .layer = layer, .s = idx }, + new_v.convert(self.v.dtype()).transpose(k_shape), + .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ).reuseBuffer(self.v), + .layer_index = self.layer_index, + } else .{ + .k = self.k.scatterSlices( + .{ .layer = layer }, + new_k.convert(self.k.dtype()).transpose(k_shape), + .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ).reuseBuffer(self.k), + .v = self.v.scatterSlices( + .{ .layer = layer }, + new_v.convert(self.v.dtype()).transpose(k_shape), + .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ).reuseBuffer(self.v), + .layer_index = self.layer_index, + }; + } + + pub fn atLayer(self: SelfAttnCache, layer_index: usize) SelfAttnCache { + return .{ + .k = self.k, + .v = self.v, + .layer_index = zml.Tensor.scalar(layer_index, .u32), + }; + } + + pub fn reuseBuffer(self: SelfAttnCache, other: SelfAttnCache) SelfAttnCache { + return .{ + .k = self.k.reuseBuffer(other.k), + .v = self.v.reuseBuffer(other.v), + .layer_index = self.layer_index.reuseBuffer(other.layer_index), + }; + } + }; + + pub const GatedDeltaNetCache = struct { + conv_state: zml.Tensor, + recurrent_state: zml.Tensor, + layer_index: zml.Tensor, + + pub fn init(config: Config, batch_dim: i64, conv_dtype: zml.DataType, recurrent_dtype: zml.DataType, model_meshe: *const zml.Mesh) GatedDeltaNetCache { + const num_linear_attn_layers = countLayers(config.text_config.layer_types, .linear_attention); + const conv_dim = 2 * config.text_config.linear_num_key_heads * config.text_config.linear_key_head_dim + config.text_config.linear_num_value_heads * config.text_config.linear_value_head_dim; + const conv_state_shape = zml.Shape.init(.{ + .b = batch_dim, + .layer = num_linear_attn_layers, + .s = config.text_config.linear_conv_kernel_dim - 1, + .mix = conv_dim, + }, conv_dtype); + const recurrent_state_shape = zml.Shape.init(.{ + .b = batch_dim, + .layer = num_linear_attn_layers, + .vh = config.text_config.linear_num_value_heads, + .khd = config.text_config.linear_key_head_dim, + .vhd = config.text_config.linear_value_head_dim, + }, recurrent_dtype); + const sharded_conv_state_shape = conv_state_shape.withPartitioning(model_meshe, .{ .mix = .model }); + const sharded_recurrent_state_shape = recurrent_state_shape.withPartitioning(model_meshe, .{ .vh = .model }); + return .{ + .conv_state = zml.Tensor.fromShape(sharded_conv_state_shape), + .recurrent_state = zml.Tensor.fromShape(sharded_recurrent_state_shape), + .layer_index = .init(.{}, .u32), + }; + } + + pub fn initBuffer(self: GatedDeltaNetCache, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(GatedDeltaNetCache) { + return .{ + .conv_state = try zml.Buffer.uninitialized(io, platform, self.conv_state.shape(), .{}), + .recurrent_state = try zml.Buffer.uninitialized(io, platform, self.recurrent_state.shape(), .{}), + .layer_index = try zml.Buffer.scalar(io, platform, 0, .u32), + }; + } + + pub fn deinitBuffer(self: *zml.Bufferized(GatedDeltaNetCache)) void { + self.conv_state.deinit(); + self.recurrent_state.deinit(); + self.layer_index.deinit(); + } + + pub fn convState(self: GatedDeltaNetCache) zml.Tensor { + return self.conv_state.slice(.layer, .dynSingle(self.layer_index)); + } + + pub fn recurrentState(self: GatedDeltaNetCache) zml.Tensor { + return self.recurrent_state.slice(.layer, .dynSingle(self.layer_index)); + } + + pub fn update(self: GatedDeltaNetCache, new_conv_state: ?zml.Tensor, new_recurrent_state: ?zml.Tensor) GatedDeltaNetCache { + const conv_state = if (new_conv_state) |state| + self.conv_state.scatterSlices( + .{ .layer = self.layer_index }, + state.convert(self.conv_state.dtype()).transpose(self.conv_state.shape().drop(.layer)), + .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ).reuseBuffer(self.conv_state) + else + self.conv_state; + + const recurrent_state = if (new_recurrent_state) |state| + self.recurrent_state.scatterSlices( + .{ .layer = self.layer_index }, + state.convert(self.recurrent_state.dtype()).transpose(self.recurrent_state.shape().drop(.layer)), + .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ).reuseBuffer(self.recurrent_state) + else + self.recurrent_state; + + return .{ + .conv_state = conv_state, + .recurrent_state = recurrent_state, + .layer_index = self.layer_index, + }; + } + + pub fn atLayer(self: GatedDeltaNetCache, layer_index: usize) GatedDeltaNetCache { + return .{ + .conv_state = self.conv_state, + .recurrent_state = self.recurrent_state, + .layer_index = zml.Tensor.scalar(layer_index, .u32), + }; + } + + pub fn reuseBuffer(self: GatedDeltaNetCache, other: GatedDeltaNetCache) GatedDeltaNetCache { + return .{ + .conv_state = self.conv_state.reuseBuffer(other.conv_state), + .recurrent_state = self.recurrent_state.reuseBuffer(other.recurrent_state), + .layer_index = self.layer_index.reuseBuffer(other.layer_index), + }; + } + }; + + pub fn init( + config: Config, + batch_dim: i64, + max_seq_len: i64, + cache_dtype: zml.DataType, + recurrent_dtype: zml.DataType, + model_meshe: *const zml.Mesh, + ) KvCache { + return .{ + .layer_types = config.text_config.layer_types, + .self_attn = SelfAttnCache.init(config, batch_dim, max_seq_len, cache_dtype, model_meshe), + .gated_delta_net = GatedDeltaNetCache.init(config, batch_dim, cache_dtype, recurrent_dtype, model_meshe), + }; + } + + pub fn initBuffer(self: KvCache, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(KvCache) { + return .{ + .self_attn = try self.self_attn.initBuffer(io, platform), + .gated_delta_net = try self.gated_delta_net.initBuffer(io, platform), + }; + } + + pub fn deinitBuffer(self: *zml.Bufferized(KvCache)) void { + SelfAttnCache.deinitBuffer(&self.self_attn); + GatedDeltaNetCache.deinitBuffer(&self.gated_delta_net); + } + + pub const LayerView = struct { + parent: KvCache, + cache: union(enum) { + self_attn: SelfAttnCache, + linear_attn: GatedDeltaNetCache, + }, + }; + + pub fn atLayer(self: KvCache, layer_index: usize) LayerView { + return switch (getDenseIndex(self.layer_types, layer_index)) { + .full_attention => |dense_index| .{ + .parent = self, + .cache = .{ .self_attn = self.self_attn.atLayer(dense_index.layer_dense_index) }, + }, + .linear_attention => |dense_index| .{ + .parent = self, + .cache = .{ .linear_attn = self.gated_delta_net.atLayer(dense_index.layer_dense_index) }, + }, + }; + } + + pub fn reuseBuffer(self: KvCache, other: KvCache) KvCache { + return .{ + .layer_types = self.layer_types, + .self_attn = self.self_attn.reuseBuffer(other.self_attn), + .gated_delta_net = self.gated_delta_net.reuseBuffer(other.gated_delta_net), + }; + } + + fn countLayers(layer_types: []const LayerType, layer_type: LayerType) i64 { + var count: i64 = 0; + for (layer_types) |registered_layer_type| { + if (registered_layer_type == layer_type) count += 1; + } + return count; + } + + fn getDenseIndex(layer_types: []const LayerType, layer_index: usize) union(enum) { + full_attention: struct { layer_dense_index: usize }, + linear_attention: struct { layer_dense_index: usize }, + } { + var self_attn_layer_index: usize = 0; + var linear_attn_layer_index: usize = 0; + for (layer_types[0..layer_index]) |layer_type| { + switch (layer_type) { + .full_attention => self_attn_layer_index += 1, + .linear_attention => linear_attn_layer_index += 1, + } + } + return switch (layer_types[layer_index]) { + .full_attention => .{ .full_attention = .{ .layer_dense_index = self_attn_layer_index } }, + .linear_attention => .{ .linear_attention = .{ .layer_dense_index = linear_attn_layer_index } }, + }; + } +}; diff --git a/examples/llm/models/qwen3_5_moe/session.zig b/examples/llm/models/qwen3_5_moe/session.zig new file mode 100644 index 00000000..6a819f26 --- /dev/null +++ b/examples/llm/models/qwen3_5_moe/session.zig @@ -0,0 +1,223 @@ +const std = @import("std"); + +const zml = @import("zml"); + +const inference = @import("inference.zig"); +const model = @import("model.zig"); + +pub const Session = struct { + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + compiled_model: *inference.CompiledModel, + prefill: inference.KernelRunner, + decode: inference.KernelRunner, + kv_cache_buffers: zml.Bufferized(model.KvCache), + rng_buffers: zml.Bufferized(zml.Tensor.Rng), + layer_index_buffers: []inference.LayerIndexBuffer, + generated_token_slice: zml.Slice, + tokenizer: zml.tokenizer.Tokenizer, + seqlen: u32, + eos_token_id: u32, + special_tokens: model.Model.SpecialTokens, + think_start: ?u32, + think_end: ?u32, + + pub fn init( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + tokenizer: zml.tokenizer.Tokenizer, + compiled_model: *inference.CompiledModel, + model_buffers: *model.Buffers, + ) !Session { + var kv_cache_buffers = try compiled_model.params.kv_cache.initBuffer(io, platform); + errdefer model.KvCache.deinitBuffer(&kv_cache_buffers); + + const seed: u128 = @intCast(std.Io.Clock.now(.real, io).toNanoseconds()); + var rng_buffers = try zml.Tensor.Rng.init().initBuffer(io, platform, seed); + errdefer zml.Tensor.Rng.deinitBuffer(&rng_buffers); + + const layer_types = compiled_model.loaded_model.inner.config.text_config.layer_types; + const layer_index_buffers = try allocator.alloc(inference.LayerIndexBuffer, layer_types.len); + errdefer allocator.free(layer_index_buffers); + var initialized_layer_index_buffers: usize = 0; + errdefer for (layer_index_buffers[0..initialized_layer_index_buffers]) |*layer_index_buffer| { + switch (layer_index_buffer.*) { + .self_attn => |*buffer| buffer.deinit(), + .linear_attn => |*buffer| buffer.deinit(), + } + }; + + var self_attn_layer_index: u32 = 0; + var linear_attn_layer_index: u32 = 0; + for (layer_types, layer_index_buffers) |layer_type, *layer_index_buffer| { + layer_index_buffer.* = switch (layer_type) { + .full_attention => b: { + defer self_attn_layer_index += 1; + break :b .{ .self_attn = try .scalar(io, platform, self_attn_layer_index, .u32) }; + }, + .linear_attention => b: { + defer linear_attn_layer_index += 1; + break :b .{ .linear_attn = try .scalar(io, platform, linear_attn_layer_index, .u32) }; + }, + }; + initialized_layer_index_buffers += 1; + } + + const generated_token_slice = try zml.Slice.alloc(allocator, zml.Shape.init(.{ .b = 1, .s = 1 }, .u32)); + errdefer generated_token_slice.free(allocator); + + var prefill = try inference.KernelRunner.init(allocator, &compiled_model.prefill, model_buffers); + errdefer prefill.deinit(allocator); + const decode = try inference.KernelRunner.init(allocator, &compiled_model.decode, model_buffers); + + return .{ + .allocator = allocator, + .io = io, + .platform = platform, + .compiled_model = compiled_model, + .prefill = prefill, + .decode = decode, + .kv_cache_buffers = kv_cache_buffers, + .rng_buffers = rng_buffers, + .layer_index_buffers = layer_index_buffers, + .generated_token_slice = generated_token_slice, + .tokenizer = tokenizer, + .seqlen = compiled_model.params.seqlen, + .eos_token_id = compiled_model.loaded_model.inner.special_tokens.end_of_text_token_id, + .special_tokens = compiled_model.loaded_model.inner.special_tokens, + .think_start = tokenizer.tokenId(""), + .think_end = tokenizer.tokenId(""), + }; + } + + pub fn deinit(self: *Session) void { + self.prefill.deinit(self.allocator); + self.decode.deinit(self.allocator); + model.KvCache.deinitBuffer(&self.kv_cache_buffers); + zml.Tensor.Rng.deinitBuffer(&self.rng_buffers); + for (self.layer_index_buffers) |*layer_index_buffer| { + switch (layer_index_buffer.*) { + .self_attn => |*buffer| buffer.deinit(), + .linear_attn => |*buffer| buffer.deinit(), + } + } + self.allocator.free(self.layer_index_buffers); + self.generated_token_slice.free(self.allocator); + } + + pub fn tokenizePrompt(self: *const Session, allocator: std.mem.Allocator, prompt: []const u8) ![]const u32 { + return tokenizeChatPrompt(allocator, self.tokenizer, prompt, self.special_tokens, true); + } + + pub fn tokenizeTurn(self: *const Session, allocator: std.mem.Allocator, prompt: []const u8) ![]const u32 { + return tokenizeChatPrompt(allocator, self.tokenizer, prompt, self.special_tokens, false); + } + + pub fn runPrefill(self: *Session, all_tokens: []const u32) !void { + const tokens_slice = try zml.Slice.alloc(self.allocator, .init(.{ .b = 1, .s = self.seqlen }, .u32)); + defer tokens_slice.free(self.allocator); + @memset(tokens_slice.items(u32), 0); + @memcpy(tokens_slice.items(u32)[0..all_tokens.len], all_tokens); + + var tokens_buffer = try zml.Buffer.fromSlice(self.io, self.platform, tokens_slice); + defer tokens_buffer.deinit(); + var token_index_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, 0), .u32); + defer token_index_buffer.deinit(); + var active_length_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, @intCast(all_tokens.len)), .u32); + defer active_length_buffer.deinit(); + + var next_token_buffer: zml.Buffer = undefined; + inference.run(&self.prefill, .{ + .io = self.io, + .tokens_buffer = &tokens_buffer, + .next_token_buffer = &next_token_buffer, + .token_index_buffer = &token_index_buffer, + .active_length_buffer = &active_length_buffer, + .kv_cache_buffers = &self.kv_cache_buffers, + .rng_buffers = &self.rng_buffers, + .layer_index_buffers = self.layer_index_buffers, + }); + + defer next_token_buffer.deinit(); + try next_token_buffer.toSlice(self.io, self.generated_token_slice); + } + + pub fn runDecode(self: *Session, all_tokens: *std.ArrayList(u32), stdout: *std.Io.Writer) !void { + var decoder = try self.tokenizer.decoder(); + defer decoder.deinit(); + const out_tokens_buffer = try self.allocator.alloc(u8, 1024); + defer self.allocator.free(out_tokens_buffer); + + var current_token_buffer = try zml.Buffer.fromSlice(self.io, self.platform, self.generated_token_slice); + defer current_token_buffer.deinit(); + var token_index_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, @intCast(all_tokens.items.len)), .u32); + defer token_index_buffer.deinit(); + var active_length_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, 1), .u32); + defer active_length_buffer.deinit(); + + generation: while (true) { + const token_id = self.generated_token_slice.items(u32)[0]; + if (token_id == self.eos_token_id) break :generation; + + const token = try decoder.feedOne(token_id, out_tokens_buffer); + if (self.think_start) |think_start| if (token_id == think_start) try stdout.writeAll("\x1b[2m"); + try stdout.writeAll(token); + if (self.think_end) |think_end| if (token_id == think_end) try stdout.writeAll("\x1b[0m"); + try stdout.flush(); + + try all_tokens.append(self.allocator, token_id); + if (all_tokens.items.len >= self.seqlen) break :generation; + + inference.run(&self.decode, .{ + .io = self.io, + .tokens_buffer = ¤t_token_buffer, + .next_token_buffer = ¤t_token_buffer, + .token_index_buffer = &token_index_buffer, + .active_length_buffer = &active_length_buffer, + .kv_cache_buffers = &self.kv_cache_buffers, + .rng_buffers = &self.rng_buffers, + .layer_index_buffers = self.layer_index_buffers, + }); + try current_token_buffer.toSlice(self.io, self.generated_token_slice); + } + + try stdout.writeAll(try decoder.finalize(out_tokens_buffer)); + try stdout.flush(); + } +}; + +fn tokenizeChatPrompt(allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, prompt: []const u8, special_tokens: model.Model.SpecialTokens, is_first_turn: bool) ![]const u32 { + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + const im_start = tokenizer.tokenId("<|im_start|>") orelse special_tokens.im_start_token_id; + const im_end = tokenizer.tokenId("<|im_end|>") orelse special_tokens.im_end_token_id; + + var tokens: std.ArrayList(u32) = try .initCapacity(allocator, prompt.len + 32); + if (!is_first_turn) { + try tokens.append(allocator, im_end); + const newline = try encoder.encodeAlloc(allocator, "\n"); + defer allocator.free(newline); + try tokens.appendSlice(allocator, newline); + } + + try tokens.append(allocator, im_start); + const user = try encoder.encodeAlloc(allocator, "user\n"); + defer allocator.free(user); + try tokens.appendSlice(allocator, user); + const prompt_encoded = try encoder.encodeAlloc(allocator, prompt); + defer allocator.free(prompt_encoded); + try tokens.appendSlice(allocator, prompt_encoded); + try tokens.append(allocator, im_end); + const newline = try encoder.encodeAlloc(allocator, "\n"); + defer allocator.free(newline); + try tokens.appendSlice(allocator, newline); + try tokens.append(allocator, im_start); + const assistant = try encoder.encodeAlloc(allocator, "assistant\n"); + defer allocator.free(assistant); + try tokens.appendSlice(allocator, assistant); + + return tokens.toOwnedSlice(allocator); +} diff --git a/examples/loader/BUILD.bazel b/examples/loader/BUILD.bazel deleted file mode 100644 index 3310cc22..00000000 --- a/examples/loader/BUILD.bazel +++ /dev/null @@ -1,10 +0,0 @@ -load("@zml//bazel:zig.bzl", "zig_cc_binary") - -zig_cc_binary( - name = "safetensors", - main = "main.zig", - deps = [ - "@zml//async", - "@zml//zml", - ], -) diff --git a/examples/loader/main.zig b/examples/loader/main.zig deleted file mode 100644 index 6ac651bc..00000000 --- a/examples/loader/main.zig +++ /dev/null @@ -1,72 +0,0 @@ -const std = @import("std"); -const zml = @import("zml"); -const asynk = @import("async"); - -const async_ = asynk.async_; - -pub fn main() !void { - var gpa = std.heap.GeneralPurposeAllocator(.{}){}; - defer _ = gpa.deinit(); - try asynk.AsyncThread.main(gpa.allocator(), asyncMain, .{}); -} - -pub fn asyncMain() !void { - // Short lived allocations - var gpa = std.heap.GeneralPurposeAllocator(.{}){}; - defer _ = gpa.deinit(); - const allocator = gpa.allocator(); - - var args = std.process.args(); - // Skip executable path - _ = args.next().?; - - const file = if (args.next()) |path| blk: { - std.debug.print("File path: {s}\n", .{path}); - break :blk path; - } else { - std.debug.print("Missing file path argument\n", .{}); - std.debug.print("Try: bazel run -c opt //loader:safetensors -- /path/to/mymodel.safetensors or /path/to/model.safetensors.index.json \n", .{}); - std.process.exit(0); - }; - - var buffer_store = try zml.aio.safetensors.open(allocator, file); - defer buffer_store.deinit(); - - var context = try zml.Context.init(); - defer context.deinit(); - - const platform = context.autoPlatform(); - const devices = platform.getDevices(); - - for (devices) |device| { - std.debug.print("Device visible: {s}\n", .{device.getDescription(platform.pjrt_api).debugString(platform.pjrt_api)}); - } - - var buffers = try gpa.allocator().alloc(zml.Buffer, buffer_store.buffers.count()); - defer { - for (buffers) |*buf| { - buf.deinit(); - } - gpa.allocator().free(buffers); - } - - var total_bytes: usize = 0; - var timer = try std.time.Timer.start(); - - var it = buffer_store.buffers.iterator(); - var i: usize = 0; - std.debug.print("\nStart to read {d} buffers from store..\n", .{buffer_store.buffers.count()}); - - while (it.next()) |entry| : (i += 1) { - const host_buffer = entry.value_ptr.*; - total_bytes += host_buffer.data.len; - std.debug.print("Buffer: {s} ({any} / {any})\n", .{ entry.key_ptr.*, i + 1, buffer_store.buffers.count() }); - buffers[i] = try zml.Buffer.from(platform, host_buffer); - } - - const stop = timer.read(); - const time_in_s = zml.meta.divFloat(f64, stop, std.time.ns_per_s); - const mbs = zml.meta.divFloat(f64, total_bytes, 1024 * 1024); - - std.debug.print("\nLoading speed: {d:.2} MB/s\n\n", .{mbs / time_in_s}); -} diff --git a/examples/mnist/BUILD.bazel b/examples/mnist/BUILD.bazel index 2f77b40a..f0ddf811 100644 --- a/examples/mnist/BUILD.bazel +++ b/examples/mnist/BUILD.bazel @@ -1,30 +1,51 @@ -load("@aspect_bazel_lib//lib:expand_template.bzl", "expand_template") -load("@aspect_bazel_lib//lib:tar.bzl", "mtree_spec", "tar") -load("@aspect_bazel_lib//lib:transitions.bzl", "platform_transition_filegroup") +load("@bazel_lib//lib:transitions.bzl", "platform_transition_filegroup") +load("@bazel_lib//lib:expand_template.bzl", "expand_template") load("@rules_oci//oci:defs.bzl", "oci_image", "oci_load", "oci_push") -load("@zml//bazel:zig.bzl", "zig_cc_binary") +load("@rules_zig//zig:defs.bzl", "zig_binary") +load("@tar.bzl//tar:mtree.bzl", "mtree_spec") +load("@tar.bzl//tar:tar.bzl", "tar") -# Executable -zig_cc_binary( +zig_binary( name = "mnist", args = [ - "$(location @com_github_ggerganov_ggml_mnist//file)", - "$(location @com_github_ggerganov_ggml_mnist_data//file)", + "$(location @mnist//:mnist.safetensors)", + "$(location @mnist//:t10k-images.idx3-ubyte)", ], data = [ - "@com_github_ggerganov_ggml_mnist//file", - "@com_github_ggerganov_ggml_mnist_data//file", + "@mnist//:mnist.safetensors", + "@mnist//:t10k-images.idx3-ubyte", ], main = "mnist.zig", + visibility = ["//visibility:public"], deps = [ - "@zml//async", - "@zml//zml", + "//bazel", + "//zml", + ], +) + +zig_binary( + name = "mnist_dp", + args = [ + "$(location @mnist//:mnist.safetensors)", + "$(location @mnist//:t10k-images.idx3-ubyte)", + ], + data = [ + "@mnist//:mnist.safetensors", + "@mnist//:t10k-images.idx3-ubyte", + ], + main = "mnist_data_parallel.zig", + srcs = ["mnist.zig"], + visibility = ["//visibility:public"], + deps = [ + "//bazel", + "//zml", ], ) mtree_spec( name = "mtree", srcs = [":mnist"], + tags = ["manual"], ) tar( @@ -36,58 +57,65 @@ tar( ], compress = "zstd", mtree = ":mtree", + tags = ["manual"], ) expand_template( name = "entrypoint", data = [ ":mnist", - "@com_github_ggerganov_ggml_mnist//file", - "@com_github_ggerganov_ggml_mnist_data//file", + "@mnist//:mnist.safetensors", + "@mnist//:t10k-images.idx3-ubyte", ], substitutions = { - ":model": "$(rlocationpath @com_github_ggerganov_ggml_mnist//file)", - ":data": "$(rlocationpath @com_github_ggerganov_ggml_mnist_data//file)", + ":model": "$(rlocationpath @mnist//:mnist.safetensors)", + ":data": "$(rlocationpath @mnist//:t10k-images.idx3-ubyte)", }, template = [ "./{}/mnist".format(package_name()), - "./{}/mnist.runfiles/:model".format(package_name()), - "./{}/mnist.runfiles/:data".format(package_name()), + "/{}/mnist.runfiles/:model".format(package_name()), + "/{}/mnist.runfiles/:data".format(package_name()), ], + tags = ["manual"], ) +# The actual docker image, with entrypoint, created from tar archive oci_image( name = "image_", base = "@distroless_cc_debian12", entrypoint = ":entrypoint", + target_compatible_with = [ + "@platforms//os:linux", + "@platforms//cpu:x86_64", + ], tars = [":archive"], + tags = ["manual"], ) +# We always want to create the image for Linux platform_transition_filegroup( name = "image", srcs = [":image_"], target_platform = "@zml//platforms:linux_amd64", + tags = ["manual"], ) +# Load will immediately load the image (eg: docker load) oci_load( name = "load", image = ":image", repo_tags = [ "distroless/mnist:latest", ], + tags = ["manual"], ) +# Bazel target for pushing the Linux image to our docker registry oci_push( name = "push", image = ":image", remote_tags = ["latest"], + # override with -- --repository foo.bar/org/image repository = "index.docker.io/steeve/mnist", -) - -oci_load( - name = "debug_image", - image = ":image", - repo_tags = [ - "distroless/mnist:latest", - ], + tags = ["manual"], ) diff --git a/examples/mnist/mnist.zig b/examples/mnist/mnist.zig index 498e5b27..f5842af8 100644 --- a/examples/mnist/mnist.zig +++ b/examples/mnist/mnist.zig @@ -1,10 +1,13 @@ const std = @import("std"); +const log = std.log; + const zml = @import("zml"); -const asynk = @import("async"); -const async_ = asynk.async_; +pub const std_options: std.Options = .{ + .log_level = .info, +}; -const show_mlir = true; +pub const Img = [28 * 28]u8; /// Model definition const Mnist = struct { @@ -15,153 +18,156 @@ const Mnist = struct { weight: zml.Tensor, bias: zml.Tensor, + pub fn init(store: zml.io.TensorStore.View) Layer { + return .{ + .weight = store.createReplicatedTensor("weight", .{ .d_out, .d }), + .bias = store.createReplicatedTensor("bias", .{.d_out}), + }; + } + pub fn forward(self: Layer, input: zml.Tensor) zml.Tensor { - return self.weight.matmul(input).add(self.bias).relu(); + return self.weight.dot(input, .d).add(self.bias).relu().withTags(.{.d}); } }; + pub fn init(store: zml.io.TensorStore.View) Mnist { + return .{ + .fc1 = .init(store.withPrefix("fc1")), + .fc2 = .init(store.withPrefix("fc2")), + }; + } + + pub fn load( + self: *const Mnist, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + store: *const zml.io.TensorStore, + ) !zml.Bufferized(Mnist) { + var buffers = try zml.mem.bufferize(allocator, Mnist, self); + errdefer unloadBuffers(&buffers); + + var loader: zml.io.Loader = try .init(allocator, platform, .default); + errdefer loader.deinit(); + + try loader.load(io, Mnist, self, &buffers, store, .{}); + try loader.await(io); + + return buffers; + } + + pub fn unloadBuffers(self: *zml.Bufferized(Mnist)) void { + self.fc1.weight.deinit(); + self.fc1.bias.deinit(); + self.fc2.weight.deinit(); + self.fc2.bias.deinit(); + } + /// just two linear layers + relu activation pub fn forward(self: Mnist, input: zml.Tensor) zml.Tensor { - // std.log.info("Compiling for target: {s}", .{@tagName(input.getContext().target())}); - var x = input.flattenAll().convert(.f32); + var x = input.flatten().convert(.f32).withTags(.{.d}); const layers: []const Layer = &.{ self.fc1, self.fc2 }; - for (layers) |layer| { - x = zml.call(layer, .forward, .{x}); + const compiler = zml.Compiler.current(); + for (0.., layers) |i, layer| { + compiler.pushLocationFmt(@src(), "layer[{d}]", .{i}); + defer compiler.popLocation(); + x = layer.forward(x); } - return x.argMax(0, .u8).indices; + return x.argMax(0).indices.convert(.u8); } }; -pub fn main() !void { - var gpa = std.heap.GeneralPurposeAllocator(.{}){}; - defer _ = gpa.deinit(); - const allocator = gpa.allocator(); - - try asynk.AsyncThread.main(allocator, asyncMain, .{}); -} - -pub fn asyncMain() !void { - // Short lived allocations - var gpa = std.heap.GeneralPurposeAllocator(.{}){}; - defer _ = gpa.deinit(); - const allocator = gpa.allocator(); - - // Create ZML context - var context = try zml.Context.init(); - defer context.deinit(); - - std.debug.print("\n===========================\n== ZML MNIST Example ==\n===========================\n\n", .{}); - - // Auto-select platform - const platform = context.autoPlatform(); - { - // List available targets - std.debug.print("Available Platforms:\n", .{}); - const selected_prefix = "✅"; - const not_selected_prefix = "• "; - const selected_postfix = "(AUTO-SELECTED)\n"; - const not_selected_postfix = "\n"; - for (zml.platform.available_targets) |target| { - std.debug.print(" {s} {s} {s}", .{ - if (target == platform.target) selected_prefix else not_selected_prefix, - @tagName(target), - if (target == platform.target) selected_postfix else not_selected_postfix, - }); - - // now the platform's devices - if (context.platforms.get(target)) |pfm| { - for (pfm.getDevices(), 0..) |device, index| { - const deviceKind = device.getDescription(platform.pjrt_api).getKind(platform.pjrt_api); - std.debug.print(" ◦ #{d}: {s}\n", .{ - index, - deviceKind, - }); - // we only list 1 CPU device - if (target == .cpu) break; - } - } - } - std.debug.print("\n", .{}); - } +pub fn main(init: std.process.Init) !void { + const arena = init.arena; + const allocator = init.gpa; + const io = init.io; // Parse program args - const process_args = try std.process.argsAlloc(allocator); - defer std.process.argsFree(allocator, process_args); - const pt_model = process_args[1]; + const process_args = try init.minimal.args.toSlice(arena.allocator()); + const model_path = process_args[1]; const t10kfilename = process_args[2]; - // Memory arena dedicated to model shapes and weights - var arena_state = std.heap.ArenaAllocator.init(allocator); - defer arena_state.deinit(); - const arena = arena_state.allocator(); - // Read model shapes. - // Note this works because Mnist struct uses the same layer names as the pytorch model - var buffer_store = try zml.aio.torch.open(allocator, pt_model); - defer buffer_store.deinit(); + var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, model_path); + defer registry.deinit(); - const mnist_model = try zml.aio.populateModel(Mnist, allocator, buffer_store); - std.debug.print("✅ Read model shapes from PyTorch file {s}\n", .{pt_model}); + // Auto-select platform + const platform: *zml.Platform = try .auto(allocator, io, .{}); + defer platform.deinit(allocator, io); - // Start loading weights - var model_weights = try zml.aio.loadModelBuffers(Mnist, mnist_model, buffer_store, arena, platform); - defer zml.aio.unloadBuffers(&model_weights); + // Init model + var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &.{platform.replicated_mesh}); + defer store.deinit(); + const mnist_model: Mnist = .init(store.view()); - // Start compiling - const comp_start_time = std.time.milliTimestamp(); - if (show_mlir) { - std.debug.print("\nCompiling model to MLIR....\n", .{}); - std.debug.print("-" ** 160 ++ "\n", .{}); - } else { - std.debug.print("Compiling model to MLIR....\r", .{}); - } - var compilation = try async_(zml.compile, .{ allocator, Mnist, .{}, .forward, .{zml.Shape.init(.{ 28, 28 }, .u8)}, buffer_store, platform }); + // // Compile model + const input: zml.Tensor = .init(.{ 28, 28 }, .u8); + var exe = blk: { + log.info("Compiling model....", .{}); + const start: std.Io.Timestamp = .now(io, .awake); + defer log.info("✅ Compiled model [{f}]", .{start.untilNow(io, .awake)}); + break :blk try platform.compile(allocator, io, mnist_model, .forward, .{input}, .{}); + }; + defer exe.deinit(); - // Wait for end of compilation and end of weights loading. - const compiled_mnist = try compilation.await_(); - const comp_end_time = std.time.milliTimestamp(); - if (show_mlir) std.debug.print("-" ** 160 ++ "\n", .{}); - std.debug.print("✅ Compiled MNIST model in {d} milliseconds! \n", .{comp_end_time - comp_start_time}); + // Load buffers + var mnist_buffers = blk: { + log.info("Transfering weights....", .{}); + const start: std.Io.Timestamp = .now(io, .awake); + defer log.info("✅ Transferred weights [{f}]", .{ + start.untilNow(io, .awake), + }); + break :blk try mnist_model.load(init.arena.allocator(), io, platform, &store); + }; + defer Mnist.unloadBuffers(&mnist_buffers); - // send weights to accelerator / GPU - var mnist = try compiled_mnist.prepare(allocator, model_weights); - defer mnist.deinit(); - std.debug.print("✅ Weights transferred, starting inference...\n\n", .{}); + var args = try exe.args(allocator); + defer args.deinit(allocator); + + var results = try exe.results(allocator); + defer results.deinit(allocator); // Load a random digit image from the dataset. - const dataset = try asynk.File.open(t10kfilename, .{ .mode = .read_only }); - defer dataset.close() catch unreachable; - var rng = std.Random.Xoshiro256.init(@intCast(std.time.timestamp())); + const dataset = try std.Io.Dir.openFile(.cwd(), io, t10kfilename, .{ .mode = .read_only }); + defer dataset.close(io); + + var rng: std.Random.DefaultPrng = blk: { + const now: std.Io.Timestamp = .now(io, .awake); + break :blk .init(@intCast(now.toMilliseconds())); + }; // inference - can be looped - { - const idx = rng.random().intRangeAtMost(u64, 0, 10000 - 1); - var sample: [28 * 28]u8 align(16) = undefined; - _ = try dataset.pread(&sample, 16 + (idx * 28 * 28)); - var input = try zml.Buffer.from(platform, zml.HostBuffer.fromBytes(zml.Shape.init(.{ 28, 28 }, .u8), &sample)); - defer input.deinit(); + const idx = rng.random().uintLessThan(u64, 10000); + var sample: Img align(16) = undefined; + _ = try dataset.readPositionalAll(io, &sample, 16 + (idx * @sizeOf(Img))); - printDigit(sample); - var result: zml.Buffer = mnist.call(.{input}); - defer result.deinit(); + var input_buffer: zml.Buffer = try .fromSlice(io, platform, zml.Slice.init(input.shape(), &sample)); + defer input_buffer.deinit(); - std.debug.print("\n✅ RECOGNIZED DIGIT:\n", .{}); - std.debug.print(" +-------------+\n", .{}); - std.debug.print("{s}\n", .{digits[try result.getValue(u8)]}); - std.debug.print(" +-------------+\n\n", .{}); - } + printDigit(&sample); + + args.set(.{ mnist_buffers, input_buffer }); + exe.call(io, args, &results, .{}); + var result: zml.Buffer = results.get(zml.Buffer); + defer result.deinit(); + + log.info( + \\✅ RECOGNIZED DIGIT: + \\ +-------------+ + \\{s} + \\ +-------------+ + \\ + , .{digits[try result.getValue(u8, io)]}); } -fn printDigit(digit: [28 * 28]u8) void { +pub fn printDigit(digit_flat: *const Img) void { + const digit: *const [28][28]u8 = @ptrCast(digit_flat); var buffer: [28][30][2]u8 = undefined; - std.debug.print(" R E C O G N I Z I N G I N P U T I M A G E :\n", .{}); - std.debug.print("+---------------------------------------------------------+\n", .{}); for (0..28) |y| { buffer[y][0] = .{ '|', ' ' }; buffer[y][29] = .{ '|', '\n' }; for (1..29) |x| { - const idx = (y * 28) + (x - 1); - const val = digit[idx]; + const val = digit[y][x - 1]; buffer[y][x] = blk: { if (val > 240) break :blk .{ '*', '*' }; if (val > 225) break :blk .{ 'o', 'o' }; @@ -170,11 +176,17 @@ fn printDigit(digit: [28 * 28]u8) void { }; } } - std.fmt.format(asynk.StdOut().writer(), "{s}", .{std.mem.asBytes(&buffer)}) catch unreachable; - std.debug.print("+---------------------------------------------------------+\n", .{}); + + log.info( + \\ + \\ R E C O G N I Z I N G I N P U T I M A G E : + \\+---------------------------------------------------------+ + \\{s}+---------------------------------------------------------+ + \\ + , .{std.mem.asBytes(&buffer)}); } -const digits = [_][]const u8{ +pub const digits = [_][]const u8{ \\ | ### | \\ | # # | \\ | # # | @@ -256,12 +268,3 @@ const digits = [_][]const u8{ \\ | ##### | , }; - -pub const std_options = .{ - // Set the global log level to err - .log_level = .err, - .log_scope_levels = &[_]std.log.ScopeLevel{ - .{ .scope = .pjrt, .level = .err }, - .{ .scope = .zml_module, .level = if (show_mlir) .debug else .err }, - }, -}; diff --git a/examples/mnist/mnist_data_parallel.zig b/examples/mnist/mnist_data_parallel.zig new file mode 100644 index 00000000..b7805e49 --- /dev/null +++ b/examples/mnist/mnist_data_parallel.zig @@ -0,0 +1,206 @@ +//! Data-parallel version of mnist.zig, +//! Separated to kept `mnist.zig` minimal + +const std = @import("std"); + +const zml = @import("zml"); + +const base = @import("mnist.zig"); + +const log = std.log.scoped(.mnist); + +pub const std_options: std.Options = .{ + .log_level = .info, +}; + +/// Model definition +const Mnist = struct { + fc1: Layer, + fc2: Layer, + + pub fn init(store: zml.io.TensorStore.View) Mnist { + return .{ + // Layer 1 is sharded following it's output axis + .fc1 = .{ + .weight = store.createTensor("fc1.weight", .{ .d_out, .d }, .dp_mp, .{ .d_out = .model }), + .bias = store.createTensor("fc1.bias", .{.d_out}, .dp_mp, .{ .d_out = .model }), + }, + // Layer 2 is sharded following it's input axis (and bias is fully replicated) + .fc2 = .{ + .weight = store.createTensor("fc2.weight", .{ .d_out, .d }, .dp_mp, .{ .d = .model }), + .bias = store.createReplicatedTensor("fc2.bias", .{.d_out}), + }, + }; + } + + pub fn load( + self: *const Mnist, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + store: *const zml.io.TensorStore, + ) !zml.Bufferized(Mnist) { + var buffers = try zml.mem.bufferize(allocator, Mnist, self); + errdefer zml.Buffer.deinitAll(Mnist, &buffers); + + var loader: zml.io.Loader = try .init(allocator, platform, .default); + errdefer loader.deinit(); + + try loader.load(io, Mnist, self, &buffers, store, .{}); + try loader.await(io); + + return buffers; + } + + /// just two linear layers + relu activation + pub fn forward(self: Mnist, input: zml.Tensor) zml.Tensor { + var x = input.onMemory(.host_pinned).toMemory(.device).merge(.{ .d = .{ .x, .y } }).convert(.f32); + + const layers: []const Layer = &.{ self.fc1, self.fc2 }; + + const compiler = zml.Compiler.current(); + + for (0.., layers) |i, layer| { + compiler.pushLocationFmt(@src(), "layer[{d}]", .{i}); + defer compiler.popLocation(); + x = layer.forward(x); + } + + return x.argMax(.d).indices.convert(.u8) + .withPartitioning(.dp_mp, .{ .b = .data, .d = .replicated }) + .toMemory(.host_pinned); + } + + const Layer = struct { + weight: zml.Tensor, + bias: zml.Tensor, + + pub fn forward(self: Layer, input: zml.Tensor) zml.Tensor { + log.info("Layer(input={f}, w={f}, b={f}) -> {f}", .{ input, self.weight, self.bias, input.dot(self.weight, .d) }); + const x = input.dot(self.weight, .d); + return x.add(self.bias.broad(x.shape())).relu().rename(.{ .d_out = .d }); + } + }; +}; + +pub fn main(init: std.process.Init) !void { + const arena = init.arena; + const allocator = init.gpa; + const io = init.io; + + // Parse program args + const process_args = try init.minimal.args.toSlice(arena.allocator()); + const model_path = process_args[1]; + const t10kfilename = process_args[2]; + + // Read model shapes. + var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, model_path); + defer registry.deinit(); + + // Auto-select platform + const platform: *zml.Platform = try .auto(allocator, io, .{}); + defer platform.deinit(allocator, io); + + // Decide how to partition the devices of our hardware. + // Here we use a mixed Data parallel / Model parallel + const dp_mp = try platform.registerMesh("dp_mp", try .init( + .{ .data, .model }, + &.{ .low_bandwidth, .high_bandwidth }, + &.{ 2, -1 }, + )); + log.info("{f}", .{dp_mp}); + + var profiler = try platform.profiler(allocator, io, .{ .session_id = "mnist" }); + defer profiler.deinit(); + + var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &.{dp_mp}); + defer store.deinit(); + + const mnist_model: Mnist = .init(store.view()); + + // Compile model + const bs: u32 = 4; + const input: zml.Tensor = .withPartitioning(.init(.{ .b = bs, .x = 28, .y = 28 }, .u8), dp_mp, .{ .b = .data }); + + var exe = exe: { + log.info("Compiling model....", .{}); + const start: std.Io.Timestamp = .now(io, .awake); + defer log.info("✅ Compiled model [{f}]", .{start.untilNow(io, .awake)}); + break :exe try platform.compile( + allocator, + io, + mnist_model, + .forward, + .{input}, + .{ + .program_name = "mnist", + .meshes = &.{dp_mp}, + .xla_dump_to = "/tmp/zml/mnist_dp", + }, + ); + }; + defer exe.deinit(); + + // Load buffers + var mnist_buffers = blk: { + log.info("Transfering weights....", .{}); + const start: std.Io.Timestamp = .now(io, .awake); + defer log.info("✅ Transferred weights [{f}]", .{ + start.untilNow(io, .awake), + }); + break :blk try mnist_model.load(init.arena.allocator(), io, platform, &store); + }; + defer zml.Buffer.deinitAll(Mnist, &mnist_buffers); + + var args = try exe.args(allocator); + defer args.deinit(allocator); + + var results = try exe.results(allocator); + defer results.deinit(allocator); + + // Load a random digit image from the dataset. + const dataset = try std.Io.Dir.openFile(.cwd(), io, t10kfilename, .{ .mode = .read_only }); + defer dataset.close(io); + + var rng: std.Random.DefaultPrng = blk: { + const now: std.Io.Timestamp = .now(io, .awake); + break :blk .init(@intCast(now.toMilliseconds())); + }; + + // inference - can be looped + var batch: []align(16) base.Img = try allocator.alignedAlloc(base.Img, .@"16", bs); + defer allocator.free(batch); + + for (0..bs) |i| { + const rand_id = rng.random().uintLessThan(u64, 10000); + _ = try dataset.readPositionalAll(io, @ptrCast(&batch[i]), 16 + (rand_id * @sizeOf(base.Img))); + } + + // This performs a host to host pinned memcpy. + // TODO: show how to allocate a Host pinned buffer and write into it. + var input_buffer: zml.Buffer = try .fromBytesOpts(io, platform, input.shape(), std.mem.sliceAsBytes(batch), .{ .memory = .host_pinned }); + defer input_buffer.deinit(); + + for (batch[0..]) |*sample| base.printDigit(sample); + + args.set(.{ mnist_buffers, input_buffer }); + + try profiler.start(); + exe.call(io, args, &results, .{}); + if (try profiler.stop()) |report| { + log.info("Wrote profiler files to {s} and {s}", .{ report.protobuf_path, report.perfetto_path }); + } + var recognized_digits_d: zml.Buffer = results.get(zml.Buffer); + defer recognized_digits_d.deinit(); + + const recognized_digits = try recognized_digits_d.getValue([bs]u8, io); + for (0..bs) |i| { + log.info( + \\✅ Shard {d} RECOGNIZED DIGIT: + \\ +-------------+ + \\{s} + \\ +-------------+ + \\ + , .{ i, base.digits[recognized_digits[i]] }); + } +} diff --git a/examples/platform_mappings b/examples/platform_mappings deleted file mode 100644 index 6fd7ceb3..00000000 --- a/examples/platform_mappings +++ /dev/null @@ -1,21 +0,0 @@ -platforms: - @zml//platforms:linux_amd64 - --cpu=k8 - - @zml//platforms:linux_arm64 - --cpu=aarch64 - - @zml//platforms:macos_arm64 - --cpu=darwin_arm64 - --apple_platform_type=macos - -flags: - --cpu=darwin_arm64 - --apple_platform_type=macos - @zml//platforms:macos_arm64 - - --cpu=k8 - @zml//platforms:linux_amd64 - - --cpu=aarch64 - @zml//platforms:linux_arm64 diff --git a/examples/sharding/BUILD.bazel b/examples/sharding/BUILD.bazel index dfff1ab7..81f6de89 100644 --- a/examples/sharding/BUILD.bazel +++ b/examples/sharding/BUILD.bazel @@ -1,172 +1,16 @@ -load("@aspect_bazel_lib//lib:tar.bzl", "mtree_spec", "tar") -load("@aspect_bazel_lib//lib:transitions.bzl", "platform_transition_filegroup") -load("@bazel_skylib//rules:native_binary.bzl", "native_binary") -load("@rules_oci//oci:defs.bzl", "oci_image", "oci_load", "oci_push") -load("@zml//bazel:zig.bzl", "zig_cc_binary") +load("@bazel_skylib//rules:build_test.bzl", "build_test") +load("@rules_zig//zig:defs.bzl", "zig_binary") -zig_cc_binary( +zig_binary( name = "sharding", - srcs = [], + args = [], + data = [], main = "main.zig", - deps = [ - "//third_party/tigerbeetle:flags", - "@zml//async", - "@zml//zml", - "//llama:llamalib", - ], + deps = ["//zml"], + visibility = ["//visibility:public"], ) -native_binary( - name = "Llama-3.1-8B-Instruct", - src = ":sharding", - args = [ - "--model=$(location @Meta-Llama-3.1-8B-Instruct//:model.safetensors.index.json)", - "--num-heads=32", - "--num-kv-heads=8", - "--rope-freq-base=500000", - ], - data = [ - "@Meta-Llama-3.1-8B-Instruct//:model", - "@Meta-Llama-3.1-8B-Instruct//:model.safetensors.index.json", - ], +build_test( + name = "test", + targets = [":sharding"], ) - -#native_binary( -# name = "Llama-3.1-8B-Instruct", -# src = ":llama", -# args = [ -# "--model=$(location @Meta-Llama-3.1-8B-Instruct//:model.safetensors.index.json)", -# "--tokenizer=$(location @Meta-Llama-3.1-8B-Instruct//:tokenizer)", -# "--num-heads=32", -# "--num-kv-heads=8", -# "--rope-freq-base=500000", -# ], -# data = [ -# "@Meta-Llama-3.1-8B-Instruct//:model", -# "@Meta-Llama-3.1-8B-Instruct//:model.safetensors.index.json", -# "@Meta-Llama-3.1-8B-Instruct//:tokenizer", -# ], -#) -# -#native_binary( -# name = "OpenLLaMA-3B", -# src = ":llama", -# args = [ -# "--model=$(location @OpenLM-Research-OpenLLaMA-3B//:model)", -# "--tokenizer=$(location @OpenLM-Research-OpenLLaMA-3B//:tokenizer)", -# "--num-heads=32", -# "--num-kv-heads=32", -# "--rope-freq-base=10000", -# ], -# data = [ -# "@OpenLM-Research-OpenLLaMA-3B//:model", -# "@OpenLM-Research-OpenLLaMA-3B//:tokenizer", -# ], -#) -# -#native_binary( -# name = "TinyLlama-1.1B-Chat", -# src = ":llama", -# args = [ -# "--model=$(location @TinyLlama-1.1B-Chat-v1.0//:model.safetensors)", -# "--tokenizer=$(location @TinyLlama-1.1B-Chat-v1.0//:tokenizer)", -# "--num-heads=32", -# "--num-kv-heads=4", -# "--rope-freq-base=10000", -# ], -# data = [ -# "@TinyLlama-1.1B-Chat-v1.0//:model.safetensors", -# "@TinyLlama-1.1B-Chat-v1.0//:tokenizer", -# ], -#) -# -#native_binary( -# name = "TinyLlama-Stories-110M", -# src = ":llama", -# args = [ -# "--model=$(location @Karpathy-TinyLlama-Stories//:stories110M)", -# "--tokenizer=$(location @Karpathy-TinyLlama-Tokenizer//file)", -# ], -# data = [ -# "@Karpathy-TinyLlama-Stories//:stories110M", -# "@Karpathy-TinyLlama-Tokenizer//file", -# ], -#) -# -#native_binary( -# name = "TinyLlama-Stories-15M", -# src = ":llama", -# args = [ -# "--model=$(location @Karpathy-TinyLlama-Stories//:stories15M)", -# "--tokenizer=$(location @Karpathy-TinyLlama-Tokenizer//file)", -# ], -# data = [ -# "@Karpathy-TinyLlama-Stories//:stories15M", -# "@Karpathy-TinyLlama-Tokenizer//file", -# ], -#) -# -#zig_cc_binary( -# name = "test-implementation", -# srcs = ["llama.zig"], -# args = [ -# "--model=$(location @Meta-Llama-3.1-8B-Instruct//:model.safetensors.index.json)", -# "--num-heads=32", -# "--num-kv-heads=8", -# "--rope-freq-base=500000", -# ], -# data = [ -# "@Meta-Llama-3.1-8B-Instruct//:model", -# "@Meta-Llama-3.1-8B-Instruct//:model.safetensors.index.json", -# ], -# main = "test.zig", -# deps = [ -# "//third_party/tigerbeetle:flags", -# "@zml//async", -# "@zml//zml", -# ], -#) -# -#mtree_spec( -# name = "mtree", -# srcs = [":llama"], -#) -# -#tar( -# name = "archive", -# srcs = [":llama"], -# args = [ -# "--options", -# "zstd:compression-level=9", -# ], -# compress = "zstd", -# mtree = ":mtree", -#) -# -#oci_image( -# name = "image_", -# base = "@distroless_cc_debian12", -# entrypoint = ["./{}/llama".format(package_name())], -# tars = [":archive"], -#) -# -#platform_transition_filegroup( -# name = "image", -# srcs = [":image_"], -# target_platform = "@zml//platforms:linux_amd64", -#) -# -#oci_load( -# name = "load", -# image = ":image", -# repo_tags = [ -# "distroless/llama:latest", -# ], -#) -# -#oci_push( -# name = "push", -# image = ":image", -# remote_tags = ["latest"], -# repository = "index.docker.io/steeve/llama", -#) diff --git a/examples/sharding/main.zig b/examples/sharding/main.zig index 32e846ac..34a78bfe 100644 --- a/examples/sharding/main.zig +++ b/examples/sharding/main.zig @@ -1,332 +1,263 @@ const std = @import("std"); const zml = @import("zml"); -const meta = zml.meta; -const asynk = @import("async"); -const flags = @import("tigerbeetle/flags"); -const llama_mod = @import("llamalib"); -const async_ = asynk.async_; +const log = std.log.scoped(.mesh); -const LlamaLM = llama_mod.LlamaLM; -const Llama = llama_mod.Llama; -const KvCache = llama_mod.KvCache; -const TransformerLayer = llama_mod.TransformerLayer; -const SelfAttn = llama_mod.SelfAttn; -const Buffer = zml.Buffer; -const Tensor = zml.Tensor; -const ShapeOf = zml.ShapeOf; - -const log = std.log.scoped(.llama); - -// set this to false to disable the verbose logging -const show_mlir = true; - -pub const std_options = .{ - .log_level = .err, - .log_scope_levels = &[_]std.log.ScopeLevel{ - .{ .scope = .pjrt, .level = if (show_mlir) .debug else .err }, - .{ .scope = .zml_module, .level = if (show_mlir) .debug else .err }, - .{ .scope = .zml, .level = if (show_mlir) .debug else .err }, - .{ .scope = .llama, .level = if (show_mlir) .debug else .info }, +pub const std_options: std.Options = .{ + .log_level = .info, + .log_scope_levels = &.{ + .{ .scope = .@"zml/compiler", .level = .debug }, }, }; -pub const MlpModule = struct { - mlp: llama_mod.Mlp, - - pub fn forward(self: MlpModule, x: zml.Tensor) zml.Tensor { - return zml.call(self.mlp, .forward, .{x}); - } +const CliMesh = enum { + auto, + mock, }; -pub const SelfAttnModule = struct { - self_attn: llama_mod.SelfAttn, - - pub fn forward(self: SelfAttnModule, x: zml.Tensor, token_index: zml.Tensor, kv_cache: KvCache) struct { zml.Tensor, KvCache } { - const y, const updated_kv_cache = self.self_attn.forward(x, token_index, kv_cache); - return .{ y, updated_kv_cache.reuseBuffer(kv_cache) }; - } -}; - -pub fn main() !void { - try asynk.AsyncThread.main(std.heap.c_allocator, asyncMain, .{}); -} - const CliArgs = struct { - pub const help = - \\ llama --model=llama3.7B.safetensors --tokenizer=vocab.json --num_layers=2 - ; - model: []const u8, - layer_start: u8 = 0, - num_layers: ?u8 = null, - seq_len: u32 = 256, - topk: u32 = 2, - temperature: u32 = 1, - num_heads: ?i64 = null, - num_kv_heads: ?i64 = null, - rope_freq_base: ?i64 = null, - prompt: ?[]const u8 = null, - test_activations: ?[]const u8 = null, - seed: ?u128 = null, - iterations: usize = 1, + partitioner: ?zml.Sharding.Partitioner = null, + mesh: CliMesh = .auto, }; -pub fn asyncMain() !void { - log.info(" LLama was compiled with {}", .{@import("builtin").mode}); +const DemoModel = struct { + w: zml.Tensor, // {feature, hidden} + b: zml.Tensor, // {hidden} - var gpa = std.heap.GeneralPurposeAllocator(.{ .thread_safe = true }){}; - defer _ = gpa.deinit(); - const allocator = gpa.allocator(); + pub fn init(w: zml.Tensor, b: zml.Tensor) DemoModel { + return .{ .w = w, .b = b }; + } - const tmp = try std.fs.openDirAbsolute("/tmp", .{}); - try tmp.makePath("zml/sharding/cache"); - var context = try zml.Context.init(); - defer context.deinit(); + pub fn forward(self: DemoModel, input: zml.Tensor) zml.Tensor { + const compiler = zml.Compiler.current(); - const compilation_options = zml.CompilationOptions{ - .xla_dump_to = "/tmp/zml/sharding", - .sharding_enabled = true, - }; + const x = input.convert(.f32); + var y = x.dot(self.w, .feature); + y = y.add(self.b.broad(y.shape())); + y = y.withPartitioning(.dp_mp, .{ .batch = .data, .hidden = .model }); + y.print("dense_out"); - const platform = context.autoPlatform().withCompilationOptions(compilation_options); - { - // List available targets - std.debug.print("\nSupported Platforms:\n", .{}); - const selected_prefix = "✅"; - const not_selected_prefix = "• "; - const selected_postfix = "(AUTO-SELECTED)\n"; - const not_selected_postfix = "\n"; - for (zml.platform.available_targets) |target| { - std.debug.print(" {s} {s} {s}", .{ - if (target == platform.target) selected_prefix else not_selected_prefix, - @tagName(target), - if (target == platform.target) selected_postfix else not_selected_postfix, - }); + const gate = y.scale(0.01).sigmoid(); - // now the platform's devices - if (context.platforms.get(target)) |pfm| { - for (pfm.getDevices(), 0..) |device, index| { - const deviceKind = device.getDescription(platform.pjrt_api).getKind(platform.pjrt_api); - std.debug.print(" ◦ #{d}: {s}\n", .{ - index, - deviceKind, - }); - // we only list 1 CPU device - if (target == .cpu) break; + const spec: zml.Sharding.Partitioning.Logical = .parse(.{ .batch = .data, .hidden = .model }); + + return zml.ops.manualComputation( + (struct { + y: zml.Tensor, + gate: zml.Tensor, + + fn body(ctx: @This(), output_sharded: zml.Shape) zml.Tensor { + const local_result = ctx.y.relu().mul(ctx.gate); + return local_result.reshape(output_sharded); } + }).body, + .{ .y = y, .gate = gate }, + y.shape(), + compiler.mesh(.dp_mp), + .{ + .in_specs = .{ .y = spec, .gate = spec }, + .out_specs = spec, + }, + ); + } +}; + +fn usage() void { + std.debug.print( + \\Usage: sharding [--partitioner=shardy|gspmd] [--mesh=auto|mock] + \\ + \\Simple sharding demo: + \\ - explicit partitioning on data/model axes + \\ - tensor print + \\ - manualComputation with multiple inputs and one output + \\ + \\Mesh modes: + \\ - auto: physical_mesh=auto, inferred cpu device_count=2 + \\ - mock: physical_mesh=mock topology, inferred cpu device_count=9 + \\ + , .{}); +} + +fn parseArgs(init: std.process.Init) !CliArgs { + var args: CliArgs = .{}; + var it = init.minimal.args.iterate(); + _ = it.next(); // program name + + while (it.next()) |arg| { + if (std.mem.eql(u8, arg, "-h") or std.mem.eql(u8, arg, "--help")) { + usage(); + std.process.exit(0); + } else if (std.mem.startsWith(u8, arg, "--partitioner=")) { + const value = arg["--partitioner=".len..]; + if (std.mem.eql(u8, value, "shardy")) { + args.partitioner = .shardy; + } else if (std.mem.eql(u8, value, "gspmd")) { + args.partitioner = .gspmd; + } else { + std.debug.print("error: unknown partitioner '{s}'\n\n", .{value}); + usage(); + return error.InvalidPartitioner; } + } else if (std.mem.startsWith(u8, arg, "--mesh=")) { + const value = arg["--mesh=".len..]; + if (std.mem.eql(u8, value, "auto")) { + args.mesh = .auto; + } else if (std.mem.eql(u8, value, "mock")) { + args.mesh = .mock; + } else { + std.debug.print("error: unknown mesh mode '{s}'\n\n", .{value}); + usage(); + return error.InvalidMeshMode; + } + } else { + std.debug.print("error: unknown argument '{s}'\n\n", .{arg}); + usage(); + return error.InvalidArgument; } - std.debug.print("\n", .{}); + } + return args; +} + +fn buildMockMesh( + allocator: std.mem.Allocator, + target: zml.Target, + devices: []const zml.platform.Device, +) !zml.Sharding.PhysicalMesh { + if (devices.len < 8) return error.NotEnoughDevicesForMockMesh; + const topology: zml.Sharding.PhysicalMesh.Tree = .axis(.link_x, .{ .mesh = .torus }, &.{ + .axis(.link_y, .{ .mesh = .torus }, &.{ + .axis(.link_z, .{ .mesh = .torus }, &.{ + .device(devices[3]), .device(devices[1]), + }), + .axis(.link_z, .{ .mesh = .torus }, &.{ + .device(devices[2]), .device(devices[0]), + }), + }), + .axis(.link_y, .{ .mesh = .torus }, &.{ + .axis(.link_z, .{ .mesh = .torus }, &.{ + .device(devices[4]), .device(devices[5]), + }), + .axis(.link_z, .{ .mesh = .torus }, &.{ + .device(devices[6]), .device(devices[7]), + }), + }), + }); + return zml.Sharding.PhysicalMesh.fromTree(allocator, target, topology); +} + +fn createSequenceBuffer( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + shape: zml.Shape, + start: f32, +) !zml.Buffer { + const slice = try zml.Slice.alloc(allocator, shape); + defer slice.free(allocator); + + for (slice.items(f32), 0..) |*e, i| { + e.* = start + @as(f32, @floatFromInt(i)); } - var args = std.process.args(); - const cli_args = flags.parse(&args, CliArgs); - const model_file = cli_args.model; + return zml.Buffer.fromSlice(io, platform, slice); +} - var arena_state = std.heap.ArenaAllocator.init(allocator); - defer arena_state.deinit(); - const model_arena = arena_state.allocator(); +pub fn main(init: std.process.Init) !void { + const allocator = init.gpa; + const io = init.io; + var args = try parseArgs(init); - log.info("Model file: {s}", .{model_file}); - - var ts = try zml.aio.detectFormatAndOpen(allocator, model_file); - defer ts.deinit(); - - var llama: LlamaLM = try zml.aio.populateModel(LlamaLM, model_arena, ts); - const num_heads: i64 = cli_args.num_heads orelse ts.metadata("num_heads", .int64) orelse @panic("--num_heads is required for this model"); - const num_kv_heads: i64 = cli_args.num_kv_heads orelse ts.metadata("num_kv_heads", .int64) orelse num_heads; - - const rope_impl = if (ts.metadata("rope_impl", .string)) |val| - std.meta.stringToEnum(zml.nn.RopeOpts.Implementation, val).? - else - .sequential; - - const llama_options: llama_mod.LlamaOptions = .{ - .max_seq_len = cli_args.seq_len, - .num_kv_heads = num_kv_heads, - .num_heads = num_heads, - .gen_opts = .{ - .topk = cli_args.topk, - .temperature = @floatFromInt(cli_args.temperature), - }, - .rms_norm_eps = @floatCast(ts.metadata("rms_norm_eps", .float64) orelse 1e-5), - .rope_opts = .{ - .impl = rope_impl, - .freq_base = @floatCast(ts.metadata("rope_freq_base", .float64) orelse @as(f32, @floatFromInt(cli_args.rope_freq_base orelse 10_000))), + const create_options: zml.platform.CreateOptions = switch (args.mesh) { + .auto => .{}, + .mock => .{ + .cpu = .{ .device_count = 8 }, + .physical_mesh = .{ .custom = buildMockMesh }, }, }; - log.info("✅ Parsed llama config: {}", .{llama_options}); - llama.init(llama_options); - try runSelfAttnBenchmark(cli_args, &llama, llama_options, ts, model_arena, platform); -} + var platform: *zml.Platform = try .auto(allocator, io, create_options); + defer platform.deinit(allocator, io); -fn runMlpBenchmark(cli_args: CliArgs, llama: *LlamaLM, llama_options: llama_mod.LlamaOptions, ts: zml.aio.BufferStore, arena: std.mem.Allocator, platform: zml.Platform) !void { - const dims = llama.model.shape(); - const S: usize = @intCast(cli_args.seq_len); - const D: usize = @intCast(dims.nh * dims.hd); - const H: usize = @intCast(llama.model.layers[0].mlp.up_proj.weight.dim(0)); - const dtype = llama.lm_head.weight.dtype(); + log.info("\n{f}", .{platform.fmtVerbose()}); - log.info( - \\ - \\S: {} - \\D: {} - \\H: {} - \\ - , .{ S, D, H }); + var profiler_options: zml.Platform.ProfilerOptions = .defaults; + profiler_options.repository_path = "/tmp/xprof"; + profiler_options.session_id = "profiling"; - const x_shape = zml.Shape.init(.{ .s = cli_args.seq_len, .d = dims.nh * dims.hd }, dtype); + var profiler = try platform.profiler(allocator, io, profiler_options); + defer profiler.deinit(); - var compilation = try async_(zml.compileModel, .{ arena, MlpModule{ .mlp = llama.model.layers[0].mlp }, .forward, .{x_shape}, platform }); - - var llama_weights = try zml.aio.loadBuffers(LlamaLM, .{llama_options}, ts, arena, platform); - defer zml.aio.unloadBuffers(&llama_weights); - - const compiled = try compilation.await_(); - var executable = try compiled.prepare(arena, .{ .mlp = llama_weights.model.layers[0].mlp }); - defer executable.deinit(); - - var rng = std.Random.DefaultPrng.init(0); - const random = rng.random(); - - var x_buffer = try createRandomBuffer(arena, platform, x_shape, random); - defer x_buffer.deinit(); - std.debug.print("\nRunning benchmark....\n", .{}); - - // Ignore first run - { - var result: zml.Buffer = executable.call(.{x_buffer}); - defer result.deinit(); + try profiler.start(); + defer { + _ = profiler.stop() catch unreachable; } - // call our executable module - var elapsed_ns: u64 = 0; - var timer = try std.time.Timer.start(); - for (0..cli_args.iterations) |_| { - timer.reset(); - var result: zml.Buffer = executable.call(.{x_buffer}); - elapsed_ns += timer.lap(); - defer result.deinit(); + if (args.partitioner) |partitioner| { + log.info("Partitioner: {s}", .{@tagName(partitioner)}); + } else { + args.partitioner = .fromTarget(platform.target); + log.info("Partitioner: {s} (default)", .{@tagName(args.partitioner.?)}); } - const elapsed_ms = @as(f64, @floatFromInt(elapsed_ns)) / std.time.ns_per_ms; - const avg_elapsed_ms = elapsed_ms / @as(f64, @floatFromInt(cli_args.iterations)); + log.info("{f}", .{platform.physical_mesh}); - std.debug.print("\n✅ Benchmark done!\n\n", .{}); + const mesh: *const zml.Mesh = try platform.registerMesh( + "dp_mp", + .mesh(.{ .data = .low_bandwidth, .model = .high_bandwidth }), + ); - std.debug.print( - \\ x_shape: {} - \\ Datatype: {s} - \\ Iterations: {d} - \\ Total elapsed: {d:.3}ms (avg: {d:.3}ms/it) - \\ - \\ - , .{ x_shape, @tagName(x_shape.dtype()), cli_args.iterations, elapsed_ms, avg_elapsed_ms }); -} + log.info("{f}", .{mesh.logical}); + log.info("{f}", .{mesh}); -fn runSelfAttnBenchmark(cli_args: CliArgs, llama: *LlamaLM, llama_options: llama_mod.LlamaOptions, ts: zml.aio.BufferStore, arena: std.mem.Allocator, platform: zml.Platform) !void { - const dims = llama.model.shape(); - const dtype = llama.lm_head.weight.dtype(); - const x_shape = zml.Shape.init(.{ .s = 1, .d = dims.nh * dims.hd }, dtype); - const token_idx_shape = zml.Shape.init(.{}, .i32); - const kv_shape = zml.Shape.init(.{ .layer = llama.model.layers.len, .h = dims.nkvh, .k = dims.s, .hd = dims.hd }, dtype).withSharding(.{.h}); - // needs to be optional - const kv_cache_shape: ShapeOf(KvCache) = KvCache.initShape(kv_shape); + const input_shape = zml.Shape.init(.{ .batch = 16, .feature = 32 }, .f32) + .withPartitioning(mesh, .{ .batch = .data, .feature = .replicated }); + const w_shape = zml.Shape.init(.{ .feature = 32, .hidden = 64 }, .f32) + .withPartitioning(mesh, .{ .feature = .replicated, .hidden = .model }); + const b_shape = zml.Shape.init(.{ .hidden = 64 }, .f32) + .withPartitioning(mesh, .{ .hidden = .model }); - var compilation = try async_(zml.compileModel, .{ arena, SelfAttnModule{ .self_attn = llama.model.layers[0].self_attn }, .forward, .{ x_shape, token_idx_shape, kv_cache_shape }, platform }); + const input: zml.Tensor = zml.Tensor.fromShape(input_shape); + const w: zml.Tensor = zml.Tensor.fromShape(w_shape); + const b: zml.Tensor = zml.Tensor.fromShape(b_shape); + const model: DemoModel = .init(w, b); - var llama_weights = try zml.aio.loadBuffers(LlamaLM, .{llama_options}, ts, arena, platform); - defer zml.aio.unloadBuffers(&llama_weights); - - const compiled = try compilation.await_(); - var executable = try compiled.prepare(arena, .{ .self_attn = llama_weights.model.layers[0].self_attn }); - defer executable.deinit(); - - var rng = std.Random.DefaultPrng.init(0); - const random = rng.random(); - - var x_buffer = try createRandomBuffer(arena, platform, x_shape, random); - defer x_buffer.deinit(); - var token_index_buffer: zml.Buffer = try zml.Buffer.fromSlice(platform, .{}, &[1]i32{1}); - defer token_index_buffer.deinit(); - var k_cache_buffer = try createRandomBuffer(arena, platform, kv_shape, random); - defer k_cache_buffer.deinit(); - var v_cache_buffer = try createRandomBuffer(arena, platform, kv_shape, random); - defer v_cache_buffer.deinit(); - var layer_index_buffer = try zml.Buffer.fromSlice(platform, .{}, &[1]i32{0}); - defer layer_index_buffer.deinit(); - var updated_kv_cache: zml.Bufferized(KvCache) = .{ .k = k_cache_buffer, .v = v_cache_buffer, .layer_index = layer_index_buffer }; - std.debug.print("\nRunning benchmark....\n", .{}); - - // Ignore first run - { - var x: zml.Buffer, updated_kv_cache = executable.call(.{ x_buffer, token_index_buffer, updated_kv_cache }); - defer x.deinit(); - } - - // call our executable module - var elapsed_ns: u64 = 0; - var timer = try std.time.Timer.start(); - for (0..cli_args.iterations) |_| { - timer.reset(); - var x: zml.Buffer, updated_kv_cache = executable.call(.{ x_buffer, token_index_buffer, updated_kv_cache }); - elapsed_ns += timer.lap(); - defer x.deinit(); - } - const elapsed_ms = @as(f64, @floatFromInt(elapsed_ns)) / std.time.ns_per_ms; - const avg_elapsed_ms = elapsed_ms / @as(f64, @floatFromInt(cli_args.iterations)); - - std.debug.print("\n✅ Benchmark done!\n\n", .{}); - - std.debug.print( - \\ x_shape: {} - \\ Datatype: {s} - \\ Iterations: {d} - \\ Total elapsed: {d:.3}ms (avg: {d:.3}ms/it) - \\ - \\ - , .{ x_shape, @tagName(x_shape.dtype()), cli_args.iterations, elapsed_ms, avg_elapsed_ms }); -} - -fn computeFloatingOpCount(S: usize, D: usize, H: usize) usize { - const up_proj_flops = 2 * S * D * H; - const gate_proj_flops = 2 * S * D * H; - const silu_flops = 4 * S * H; - const up_gate_mul_flops = S * H; - const down_proj_flops = 2 * S * H * D; - return up_proj_flops + gate_proj_flops + silu_flops + up_gate_mul_flops + down_proj_flops; -} - -fn createRandomBuffer(allocator: std.mem.Allocator, platform: zml.Platform, shape: zml.Shape, random: std.Random) !zml.Buffer { - const data = try allocator.alloc(u8, shape.byteSize()); - defer allocator.free(data); - - switch (shape.dtype()) { - inline else => |v| { - const ZigType = v.toZigType(); - switch (comptime v.class()) { - .bool => unreachable, - .integer => { - for (std.mem.bytesAsSlice(ZigType, data)) |*e| e.* = random.int(ZigType); - }, - .float => { - const value = random.float(f64); - for (std.mem.bytesAsSlice(ZigType, data)) |*e| e.* = if (ZigType == f64) - value - else if (ZigType == f32) - @floatCast(value) - else if (ZigType == f16) - @floatCast(value) - else - @bitCast(random.int(std.meta.Int(.unsigned, @bitSizeOf(ZigType)))); - }, - .complex => unreachable, - } + var exe = try platform.compile( + allocator, + io, + model, + .forward, + .{input}, + .{ + .partitioner = args.partitioner, + .meshes = &.{mesh}, }, - } + ); + defer exe.deinit(); - var host_buffer = zml.HostBuffer.fromBytes(shape, data); - errdefer host_buffer.deinit(allocator); - return zml.Buffer.from(platform, host_buffer); + var w_buf = try createSequenceBuffer(allocator, io, platform, w_shape, 0.0); + defer w_buf.deinit(); + var b_buf = try createSequenceBuffer(allocator, io, platform, b_shape, 100.0); + defer b_buf.deinit(); + var input_buf = try createSequenceBuffer(allocator, io, platform, input_shape, 1000.0); + defer input_buf.deinit(); + + log.info("input placement: {f}", .{input_buf}); + log.info("weight placement: {f}", .{w_buf}); + log.info("bias placement: {f}", .{b_buf}); + + var exe_args = try exe.args(allocator); + defer exe_args.deinit(allocator); + var exe_results = try exe.results(allocator); + defer exe_results.deinit(allocator); + exe_args.set(.{ w_buf, b_buf, input_buf }); + + exe.call(io, exe_args, &exe_results, .{}); + var out = exe_results.get(zml.Buffer); + defer out.deinit(); + + const out_slice = try out.toSliceAlloc(allocator, io); + defer out_slice.free(allocator); + const out_items = out_slice.items(f32); + const preview_len = @min(out_items.len, @as(usize, 16)); + log.info("output preview: {any}", .{out_items[0..preview_len]}); } diff --git a/examples/simple_layer/BUILD.bazel b/examples/simple_layer/BUILD.bazel deleted file mode 100644 index 17ad673a..00000000 --- a/examples/simple_layer/BUILD.bazel +++ /dev/null @@ -1,64 +0,0 @@ -load("@aspect_bazel_lib//lib:tar.bzl", "mtree_spec", "tar") -load("@aspect_bazel_lib//lib:transitions.bzl", "platform_transition_filegroup") -load("@rules_oci//oci:defs.bzl", "oci_image", "oci_load", "oci_push") -load("@zml//bazel:zig.bzl", "zig_cc_binary") - -zig_cc_binary( - name = "simple_layer", - main = "main.zig", - deps = [ - "@zml//async", - "@zml//zml", - ], -) - -# Manifest created from the simple_layer binary and friends -mtree_spec( - name = "mtree", - srcs = [":simple_layer"], -) - -# Create a tar archive from the above manifest -tar( - name = "archive", - srcs = [":simple_layer"], - args = [ - "--options", - "zstd:compression-level=9", - ], - compress = "zstd", - mtree = ":mtree", -) - -# The actual docker image, with entrypoint, created from tar archive -oci_image( - name = "image_", - base = "@distroless_cc_debian12", - entrypoint = ["./{}/simple_layer".format(package_name())], - tars = [":archive"], -) - -# We always want to create the image for Linux -platform_transition_filegroup( - name = "image", - srcs = [":image_"], - target_platform = "@zml//platforms:linux_amd64", -) - -# Load will immediatly load the image (eg: docker load) -oci_load( - name = "load", - image = ":image", - repo_tags = [ - "distroless/simple_layer:latest", - ], -) - -# Bazel target for pushing the Linux image to the docker registry -oci_push( - name = "push", - image = ":image", - remote_tags = ["latest"], - # override with -- --repository foo.bar/org/image - repository = "index.docker.io/renerocksai/simple_layer", -) diff --git a/examples/simple_layer/main.zig b/examples/simple_layer/main.zig deleted file mode 100644 index dbe3242c..00000000 --- a/examples/simple_layer/main.zig +++ /dev/null @@ -1,101 +0,0 @@ -const std = @import("std"); -const zml = @import("zml"); -const asynk = @import("async"); - -const async_ = asynk.async_; - -/// Model definition -const Layer = struct { - bias: ?zml.Tensor = null, - weight: zml.Tensor, - - pub fn forward(self: Layer, x: zml.Tensor) zml.Tensor { - var y = self.weight.mul(x); - if (self.bias) |bias| { - y = y.add(bias); - } - return y; - } -}; - -pub fn main() !void { - var gpa = std.heap.GeneralPurposeAllocator(.{}){}; - defer _ = gpa.deinit(); - try asynk.AsyncThread.main(gpa.allocator(), asyncMain, .{}); -} - -pub fn asyncMain() !void { - // Short lived allocations - var gpa = std.heap.GeneralPurposeAllocator(.{}){}; - defer _ = gpa.deinit(); - const allocator = gpa.allocator(); - - // Arena allocator for BufferStore etc. - var arena_state = std.heap.ArenaAllocator.init(allocator); - defer arena_state.deinit(); - const arena = arena_state.allocator(); - - var context = try zml.Context.init(); - defer context.deinit(); - - const platform = context.autoPlatform(); - - // Our weights and bias to use - var weights = [4]f16{ 2.0, 2.0, 2.0, 2.0 }; - var bias = [4]f16{ 1.0, 2.0, 3.0, 4.0 }; - const input_shape = zml.Shape.init(.{4}, .f16); - - // We manually produce a BufferStore. You would not normally do that. - // A BufferStore is usually created by loading model data from a file. - var buffers: zml.aio.BufferStore.Buffers = .{}; - try buffers.put(arena, "weight", zml.HostBuffer.fromArray(&weights)); - try buffers.put(arena, "bias", zml.HostBuffer.fromArray(&bias)); - - // the actual BufferStore - const buffer_store: zml.aio.BufferStore = .{ - .arena = arena_state, - .buffers = buffers, - }; - - // A clone of our model, consisting of shapes. We only need shapes for compiling. - // We use the BufferStore to infer the shapes. - var model_shapes = try zml.aio.populateModel(Layer, allocator, buffer_store); - model_shapes.weight = model_shapes.weight.withSharding(.{-1}); - model_shapes.bias = model_shapes.bias.?.withSharding(.{-1}); - - // Start compiling. This uses the inferred shapes from the BufferStore. - // The shape of the input tensor, we have to pass in manually. - var compilation = try async_(zml.compileModel, .{ allocator, model_shapes, .forward, .{input_shape}, platform }); - - // Produce a bufferized weights struct from the fake BufferStore. - // This is like the inferred shapes, but with actual values. - // We will need to send those to the computation device later. - var model_weights = try zml.aio.loadModelBuffers(Layer, model_shapes, buffer_store, arena, platform); - defer zml.aio.unloadBuffers(&model_weights); // for good practice - - // Wait for compilation to finish - const compiled = try compilation.await_(); - - // pass the model weights to the compiled module to create an executable module - var executable = try compiled.prepare(arena, model_weights); - defer executable.deinit(); - - // prepare an input buffer - // Here, we use zml.HostBuffer.fromSlice to show how you would create a HostBuffer - // with a specific shape from an array. - // For situations where e.g. you have an [4]f16 array but need a .{2, 2} input shape. - var input = [4]f16{ 5.0, 5.0, 5.0, 5.0 }; - var input_buffer = try zml.Buffer.from(platform, zml.HostBuffer.fromSlice(input_shape, &input)); - defer input_buffer.deinit(); - - // call our executable module - var result: zml.Buffer = executable.call(.{input_buffer}); - defer result.deinit(); - - // fetch the result to CPU memory - const cpu_result = try result.toHostAlloc(arena); - std.debug.print( - "\nThe result of {d} * {d} + {d} = {d}\n", - .{ &weights, &input, &bias, cpu_result.items(f16) }, - ); -} diff --git a/examples/third_party/tigerbeetle/BUILD.bazel b/examples/third_party/tigerbeetle/BUILD.bazel deleted file mode 100644 index 23baa5b4..00000000 --- a/examples/third_party/tigerbeetle/BUILD.bazel +++ /dev/null @@ -1,11 +0,0 @@ -# Files in this folder come from the very fine folks of TigerBeetle -# and is licensed under Apache 2.0 attribution license. -# https://github.com/tigerbeetle/tigerbeetle -load("@rules_zig//zig:defs.bzl", "zig_library") - -zig_library( - name = "flags", - import_name = "tigerbeetle/flags", - main = "flags.zig", - visibility = ["//visibility:public"], -) diff --git a/examples/tools/buildifier.sh b/examples/tools/buildifier.sh deleted file mode 100755 index 3e1b0930..00000000 --- a/examples/tools/buildifier.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -cd "$(dirname "${BASH_SOURCE[0]}")" -exec bazel run -- @buildifier_prebuilt//:buildifier "$@" diff --git a/examples/tools/zig.sh b/examples/tools/zig.sh deleted file mode 100755 index 9108ec88..00000000 --- a/examples/tools/zig.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -cd "$(dirname "${BASH_SOURCE[0]}")" -exec bazel run --config=silent @zml//third_party/zls:zig -- "${@}" diff --git a/examples/tools/zls.sh b/examples/tools/zls.sh deleted file mode 100755 index fda54bdb..00000000 --- a/examples/tools/zls.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -cd "$(dirname "${BASH_SOURCE[0]}")" -exec bazel run --config=silent @zml//third_party/zls:zls -- "${@}" diff --git a/examples/zls.build.json b/examples/zls.build.json deleted file mode 100644 index a4268323..00000000 --- a/examples/zls.build.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "build_options": [ - { - "name": "cmd", - "value": "bazel run @zml//zml:completion" - } - ] -} diff --git a/ffi/BUILD.bazel b/ffi/BUILD.bazel new file mode 100644 index 00000000..f53c70ed --- /dev/null +++ b/ffi/BUILD.bazel @@ -0,0 +1,25 @@ +load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library") + +cc_library( + name = "cc", + tags = ["manual"], + hdrs = [ + "zig_allocator.h", + "zig_slice.h", + ], + visibility = ["//visibility:public"], +) + +zig_library( + name = "zig", + tags = ["manual"], + srcs = [ + "zig_allocator.zig", + "zig_slice.zig", + ], + import_name = "ffi", + main = "ffi.zig", + visibility = ["//visibility:public"], + deps = [":cc"], +) diff --git a/ffi/ffi.zig b/ffi/ffi.zig new file mode 100644 index 00000000..ad3c09a9 --- /dev/null +++ b/ffi/ffi.zig @@ -0,0 +1,12 @@ +const std = @import("std"); +const c = @import("c"); + +pub const ZigAllocator = @import("zig_allocator.zig").ZigAllocator; +pub const ZigSlice = @import("zig_slice.zig").ZigSlice; + +pub fn as_path(path: []const u8) [std.fs.max_path_bytes:0]u8 { + var result: [std.fs.max_path_bytes:0]u8 = undefined; + @memcpy(result[0..path.len], path); + result[path.len] = 0; + return result; +} diff --git a/ffi/zig_allocator.h b/ffi/zig_allocator.h new file mode 100644 index 00000000..04e6885c --- /dev/null +++ b/ffi/zig_allocator.h @@ -0,0 +1,22 @@ +#pragma once + +#include + +typedef struct +{ + const void *ctx; + void *(*alloc)(const void *ctx, size_t elem, size_t nelems, size_t alignment); + void (*free)(const void *ctx, void *ptr, size_t elem, size_t nelems, size_t alignment); + +#ifdef __cplusplus + template [[nodiscard]] T *allocate(size_t n) + { + return static_cast(this->alloc(this->ctx, sizeof(T), n, _Alignof(T))); + } + + template [[nodiscard]] void deallocate(T *p, size_t n) + { + this->free(this->ctx, static_cast(p), sizeof(T), n, _Alignof(T)); + } +#endif +} zig_allocator; diff --git a/ffi/zig_allocator.zig b/ffi/zig_allocator.zig new file mode 100644 index 00000000..52c000bd --- /dev/null +++ b/ffi/zig_allocator.zig @@ -0,0 +1,26 @@ +const std = @import("std"); + +const c = @import("c"); + +pub const ZigAllocator = struct { + pub inline fn from(allocator: std.mem.Allocator) c.zig_allocator { + return .{ + .ctx = @ptrCast(@alignCast(&allocator)), + .alloc = &alloc, + .free = &free, + }; + } + + pub fn alloc(ctx: ?*const anyopaque, elem: usize, nelems: usize, alignment: usize) callconv(.c) ?*anyopaque { + const self: *const std.mem.Allocator = @ptrCast(@alignCast(ctx)); + const ret = self.rawAlloc(elem * nelems, .fromByteUnits(alignment), @returnAddress()) orelse return null; + return @ptrCast(ret); + } + + pub fn free(ctx: ?*const anyopaque, ptr: ?*anyopaque, elem: usize, nelems: usize, alignment: usize) callconv(.c) void { + const self: *const std.mem.Allocator = @ptrCast(@alignCast(ctx)); + const memory: [*c]u8 = @ptrCast(ptr); + const size = elem * nelems; + self.rawFree(memory[0..size], .fromByteUnits(alignment), @returnAddress()); + } +}; diff --git a/ffi/zig_slice.h b/ffi/zig_slice.h new file mode 100644 index 00000000..e0a6ed30 --- /dev/null +++ b/ffi/zig_slice.h @@ -0,0 +1,9 @@ +#pragma once + +#include + +typedef struct +{ + void *ptr; + size_t len; +} zig_slice; diff --git a/ffi/zig_slice.zig b/ffi/zig_slice.zig new file mode 100644 index 00000000..c7360465 --- /dev/null +++ b/ffi/zig_slice.zig @@ -0,0 +1,15 @@ +const std = @import("std"); +const c = @import("c"); + +pub const ZigSlice = struct { + pub fn from(slice: anytype) c.zig_slice { + return .{ + .ptr = @ptrCast(@constCast(slice.ptr)), + .len = slice.len, + }; + } + + pub fn to(comptime T: type, slice: c.zig_slice) []T { + return @as([*c]T, @ptrCast(@alignCast(slice.ptr)))[0..slice.len]; + } +}; diff --git a/mlir/.clang-format b/mlir/.clang-format new file mode 100644 index 00000000..9b3aa8b7 --- /dev/null +++ b/mlir/.clang-format @@ -0,0 +1 @@ +BasedOnStyle: LLVM diff --git a/mlir/BUILD.bazel b/mlir/BUILD.bazel index 93968e83..8333c579 100644 --- a/mlir/BUILD.bazel +++ b/mlir/BUILD.bazel @@ -1,24 +1,12 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") -load("//bazel:zig.bzl", "zig_cc_test") - -cc_library( - name = "mlirx", - srcs = ["mlirx.cc"], - hdrs = ["mlirx.h"], - includes = ["."], - deps = [ - "@llvm-project//mlir:CAPIIR", - ], -) +load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") cc_library( name = "c", - hdrs = ["c.h"], - visibility = ["//mlir:__subpackages__"], + hdrs = ["mlir.h"], + tags = ["manual"], deps = [ - "@llvm-project//mlir:CAPIArith", "@llvm-project//mlir:CAPIIR", - "@llvm-project//mlir:CAPIMath", "@llvm-project//mlir:CAPITransforms", ], ) @@ -26,14 +14,15 @@ cc_library( zig_library( name = "mlir", main = "mlir.zig", + tags = ["manual"], visibility = ["//visibility:public"], deps = [ ":c", - ":mlirx", + "//stdx", ], ) -zig_cc_test( +zig_test( name = "test", deps = [":mlir"], ) diff --git a/mlir/c.h b/mlir/c.h deleted file mode 100644 index 014666a7..00000000 --- a/mlir/c.h +++ /dev/null @@ -1,8 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include diff --git a/mlir/dialects/BUILD.bazel b/mlir/dialects/BUILD.bazel deleted file mode 100644 index 847e60aa..00000000 --- a/mlir/dialects/BUILD.bazel +++ /dev/null @@ -1,41 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") -load("//bazel:zig.bzl", "zig_cc_test") - -zig_library( - name = "dialects", - srcs = [ - "arith.zig", - "func.zig", - "math.zig", - "tensor.zig", - ], - import_name = "mlir/dialects", - main = "dialects.zig", - visibility = ["//visibility:public"], - deps = [ - ":stablehlo", - "//mlir", - ], -) - -zig_cc_test( - name = "test", - deps = [":dialects"], -) - -zig_library( - name = "stablehlo", - import_name = "mlir/dialects/stablehlo", - main = "stablehlo.zig", - visibility = ["//mlir/dialects:__subpackages__"], - deps = [ - "//mlir", - "//mlir:c", - "@stablehlo//:stablehlo_capi", - ], -) - -zig_cc_test( - name = "stablehlo_test", - deps = [":stablehlo"], -) diff --git a/mlir/dialects/README.md b/mlir/dialects/README.md new file mode 100644 index 00000000..7df47c5c --- /dev/null +++ b/mlir/dialects/README.md @@ -0,0 +1,269 @@ +# MLIR dialects + +One Bazel package per MLIR dialect that ZML emits. The rules come in three +tiers: + +- **Guarantees** hold for every package. Tests and the conformance checks + below enforce them, and callers rely on them. +- **Conventions** are the defaults. Follow them in new code; existing code + that differs is fine unless it confuses callers. +- **Exceptions** list the known, justified departures from the conventions. + +References: `cuda/`, `nvvm/` and `cute_nvgpu/` (generated from ODS, with +their own TableGen and C API), `gpu/`, `arith/` and `func/` (upstream +dialects, with a small C API for what upstream lacks). + +## Guarantees + +- **Registration.** The package exposes `registerDialect(registry)`, which + calls `registry.registerDialect("")` with a string literal, where + `` is the second argument of + `MLIR_DECLARE_CAPI_DIALECT_REGISTRATION`. Tests and consumers register + through `.registerDialect`, never through raw strings or + `DialectHandle.fromString`. A DSL exposes `registerDialects(registry)`, + which calls the `registerDialect` of each package it emits. +- **Operation list.** A package that builds operations has, in its + `test.zig`, a sorted `const operation_names: []const []const u8` naming + every operation its builders create. `test "every bound operation is + registered"` checks each name against the registry, and checks that + unregistered dialects are not allowed. A builder for an operation the pinned dialect does not register + does not exist; a DSL method builds it from the operations the Python DSL + lowers it to instead. +- **Bindings, not text.** A dialect package builds attributes and types + through C API constructors (`get`/`getChecked`), never by printing and + parsing text. A type or attribute the C API lacks gets a binding (a C + constructor and a Zig wrapper) before anything uses it. `mlir.Type.parse` + and `mlir.Attribute.parse` are for tests. Consumers use the bindings too. +- **Everything compiles.** `test.zig` runs `std.testing.refAllDecls` on the + package and on each of its wrapper types, so no declaration hides behind + lazy analysis. +- **Argument order.** A builder takes `ctx`, the required operands, the + result types (omitted when inferred), the required attributes, then the + optional arguments (an `opts` struct), then `location`, which is always + last. +- **Verification.** A builder verifies the operation it creates. One that + cannot (its region is filled later, or it only verifies inside a parent) + passes `.verify = false` and says so in its doc comment. +- **C API safety.** Every declaration is `MLIR_CAPI_EXPORTED`. Constructors + load the dialect, return a null handle for a null context, and verifying + constructors (`getChecked`) return a null handle for invalid input. No + `alwayslink` and no static initializers: the Zig side calls the + registration function by name, so the linker keeps what it uses. +- **Hygiene.** LLVM file banners and include-path guards (below), clean under + `mlir/.clang-format` and the Bazel toolchain's `zig fmt` (0.17), tests only + in `test.zig`, and the BUILD essentials: `capi` and the `zig_library` + tagged `manual`, an explicit `srcs` list in `td_files`, no `glob(`. + +## Conventions + +### Package layout + +- `mlir/dialects//` is imported as `@import("mlir/dialects/")`, + main file `.zig`. Generated builders go in `ops.zig`, exposed as + `.ops`, with hand-written helpers in `.zig`. +- The C API is `capi.h` / `capi.cc`. Dialects written here use + `{Dialect,Ops,Types,Attributes,Enums}.{td,h,cpp}`; vendored + sources keep their upstream file names. +- A generated file starts with a header naming its source and saying it is + generated. The package that generates it has a `REGENERATING.md` of agent + instructions (sources, generated vs maintained files, steps, checks) that + points here for conventions. Generator scripts are not committed. + +### BUILD.bazel + +Targets in this order: `capi` (`cc_library`, deps `:dialect`, `:headers`, +`@llvm-project//mlir:CAPIIR`), the `zig_library` (public, deps `:capi`, +`//mlir`), `test` (`zig_test`, `main = "test.zig"`, plus `:capi` when +`test.zig` imports `c`), then `td_files` (`includes = ["/"]`, direct td +deps such as `@llvm-project//mlir:OpBaseTdFiles`), the `gentbl_cc_library` +targets the dialect needs (`dialect_inc_gen`, `enums_inc_gen`, +`attrs_inc_gen`, `types_inc_gen`, `ops_inc_gen`), `headers` and `dialect`. +Every dep a source includes or imports is listed directly. Format with +`./tools/buildifier.sh`. + +### C and C++ + +- The first line is an 80-column LLVM banner: + `//===- capi.h - C API for NVIDIA's cuda dialect -------------------*- C -*-===//`. + The tag is `*- C -*-` for C headers, `*- C++ -*-` for C++ headers and + `*- tablegen -*-` for `.td`; `.cc`/`.cpp`/`.inc` banners end in `---===//`. + An optional `//` description follows, then a blank line. +- The guard is the include path in upper case with `/` and `.` as `_`, no + `ZML_` prefix (`MLIR_DIALECTS_CUDA_CUDADIALECT_H`, `..._TD`), closed with + `#endif // `. +- Includes follow LLVM order: own header, package headers, `mlir/` and + `llvm/`, then system headers. `capi.h` includes `mlir-c/IR.h` (which + provides `stdbool` and `stdint`) and the upstream `mlir-c/Dialect/.h` it + extends, inside `extern "C"` under `#ifdef __cplusplus`. +- Hand-written C APIs use fixed-width integers (`uint32_t`, `int64_t`), + parameters `MlirContext context`, `MlirAttribute attr`, `MlirType type`, + and C++ locals `MLIRContext *ctx`. `capi.cc` uses `static` helpers, not + anonymous namespaces, and one static helper loads the dialect + (`loadGPU` in `gpu/capi.cc`). + +### Names and where they come from + +`` is the dialect's C prefix, the first argument of +`MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(, )`: `Cuda`, `CuteNVGPU`, +`Triton`, `ROCDL`. Everything else is named as ODS names it: + +| ODS | C API | Zig | +|---|---|---| +| `AttrDef` (class `NameAttr`) | `mlirAttributeIsAName`, `mlirNameAttrGet` | `NameAttr`, `NameAttr.get(ctx, .{ ... })` | +| `TypeDef` (class `NameType`) | `mlirTypeIsAName`, `mlirNameTypeGet` | `NameType` | +| parameter `$elementType` | `mlirNameAttrGetElementType` | field `.elementType`, accessor `elementType()` | +| list parameter `$archs` | `...GetNumArchs`, `...GetArchs(attr, pos)` | `numArchs()`, `arch(i)` | +| `EnumAttr` / `I32EnumAttr` | `...AttrGet(context, value)`, `...AttrGetValue` | Zig enum, `XAttr.get(ctx, .{ .value = ... })`, `value()` | +| op mnemonic `fast_sin.f32` | generic operation API | builder `fast_sin_f32` | + +- Predicates have no `Attr`/`Type` suffix (`mlirTypeIsACudaEvent`). +- C getters keep MLIR's `Get`. Zig accessors are nouns, as in the rest + of ZML (`value()`, `elementType()`, `numAxes()`, `axis(i)`), never + `get`-prefixed. Predicates keep `is`/`has`. +- Enum fields use the IR spelling (`@"and"`, `@"false"`). +- A builder is named after its mnemonic with dots turned into underscores, + case kept. Its parameters are the ODS operand and attribute names, and its + result types `_type` (`_types` when variadic). +- A name that is a Zig keyword, or that collides with a member, a declaration + of the file or a builder local, takes a trailing underscore (`for_`, + `return_`, `ptr_()`, `format_()`). It is quoted only when that name is taken + too (`TupleType.@"type"(i)`, next to the `type_()` upcast). +- Alternative constructors are `mlirNameAttr
Get` in C, as MLIR's + `mlirDenseElementsAttrSplatGet`, and `get(ctx, ...)` in Zig. + +### Zig API + +```zig +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("cuda"); +} + +/// `#cuda.executable`: Format of a compiled executable +pub const ExecutableAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACudaExecutable; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const InitArgs = struct { format: ExecutableFormat }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { ... } + pub fn attribute(self: *const Self) *const mlir.Attribute { ... } + pub fn format_(self: *const Self) ExecutableFormat { ... } +}; +``` + +- Wrappers use `mlir.Methods` with `c.MlirAttribute` or `c.MlirType`, upcast + with `attribute()` or `type_()`, and build with `get(ctx, InitArgs)`, a + struct even for one field. `get` returns `mlir.Error!*const X` when the C + getter can return null, a plain pointer otherwise. +- Free helpers are `Attr(ctx, ...)` and `Type(ctx, ...)`. +- Optional builder arguments go in `opts`: generated builders declare it + inline (`opts: struct { ... }`), hand-written ones may name it `Opts`. + Its fields are the optional operands, result types and optional, defaulted + or unit attributes, each defaulting to `null` (or the ODS default), so + callers pass `.{}`. +- Sorted imports (`std` first), no unused declarations. + +### Comments + +- Porting adds no explanatory comments; doc comments on public API stay + short. +- A generated wrapper's doc comment is the IR spelling, a colon and the ODS + summary (``/// `#cuda.executable`: Format of a compiled executable``); a + generated builder's is the IR name, a period, then the summary. + +### Tests + +Beyond the guaranteed `refAllDecls` and registration tests: a +`testContext()` that registers the package and its dependencies and calls +`mlir.Context.init(.{ .registry = registry, .threading = false })` and +`loadAllAvailableDialects()`; attributes and types round-tripped (build with +`get`, print with `{f}`, reparse, compare with `eql`); operations built and +verified. + +## Exceptions + +- Generated `.td` files start with a "generated" header instead of a banner. +- Generated C APIs keep ODS's integer types (`int`) and repeat the null check + and dialect load in every constructor. +- `cute_ir/` and `cute_nvgpu/` keep their generated builders in `.zig`; + `llvm/` keeps its builders in a private `ops.zig`. +- `cuda/`, `cute_nvgpu/` and `nvvm/` are regenerated through + `cute_ir/REGENERATING.md`, `fly_rocdl/` through `fly/REGENERATING.md`. +- `cute_ir:capi` and `cute_nvgpu:capi` each depend on the other's `:dialect`: + their C++ sources use each other's types. +- `cf`, `math`, `memref`, `scf`, `shardy` and `stablehlo` have no `capi` of + their own and bind upstream's C API target directly; `llvm`'s `capi` only + adds what upstream's leaves out (loop annotations). +- Hand-written builders with many parameters take an `Args` struct + (`func.func`, `gpu.func`, `ttir.func`, `cuda.kernel`, `cuda_tile.entry`, + `air.kernel`, `memref.subview`, `vector.transfer_read`, ...); + function-like ones carry `location` inside it. +- Some hand-written `Opts` hold required or non-null defaulted fields where + named arguments read better (`stablehlo.PadOpts`, + `mosaic_tpu.LoadOpts.sublane_mask`, `affine.ForOpts.step`). +- stablehlo, shardy, cuda_tile and ttir getters return plain pointers (their + C calls cannot fail); `ttir.pointerType` and `ttir.tensorDescType` take no + `ctx` (the C API derives it from the element type). +- Enum cases whose IR spelling is not an identifier use the C++ symbol + (rocdl's `volatile_op`), as do the generated cute_nvgpu and ttir enums. +- fly's attributes and cute_ir's algebra attributes have named constructors + (`getStatic`, `getDynamic`, `getTuple`, ...); fly's generated builders use + snake_case parameters. +- tcl mirrors the Furiosa plugin's file names and C++ style. One + `graph(op)` builder covers the 17 `tcl.graph.*` operations that share a + shape, and `loop`, `yield` and `copy` build `tcl.graph.for`, `yield` and + `copy`. Its `Tactic` and `ReduceMode` enums map to string attribute values, + its C API covers what ZML emits, and its operations are verified through + `dsl/tcl`'s tests. + +## Porting checklist + +1. Choose ``, the C prefix and the handle; create the package with the + BUILD targets above. +2. Vendor or write the `.td`/C++ sources and list every `.td` in `td_files`. +3. Write `capi.h`/`capi.cc` for registration and whatever upstream's C API + lacks: per attribute and type, a predicate, a `Get` and the parameter + getters. +4. Bind in `.zig`, or generate `ops.zig`: `registerDialect`, enums, + wrappers, helpers, builders. +5. Write `test.zig`, with the `operation_names` list. +6. If anything is generated, add a `REGENERATING.md`. +7. Make consumers depend on `//mlir/dialects/` and call + `.registerDialect(registry)`. +8. Run the verification below. + +## Verification + +Run from the repository root, with `ZIG` set to the Bazel toolchain's `zig` +(`AGENTS.md` shows how to find its standard library; `zig` is next to it). + +```bash +#!/usr/bin/env bash +d=mlir/dialects/ +bazel test "//$d:test" && bazel test //mlir/... && bazel build //... +./tools/buildifier.sh -mode=check -r "$PWD/$d" +"$ZIG" fmt --check "$d" +src=(); while IFS= read -r f; do src+=("$f"); done < <(find "$d" -maxdepth 1 \( -name '*.h' -o -name '*.cc' -o -name '*.cpp' -o -name '*.td' -o -name '*.inc' \)) +cc=(); for f in "${src[@]}"; do [[ $f == *.td || $f == *.inc ]] || cc+=("$f"); done +[ ${#cc[@]} -eq 0 ] || clang-format --dry-run --Werror --style=file:mlir/.clang-format "${cc[@]}" + +# Conformance: prints nothing on a conforming package. +for f in "${src[@]}"; do l=$(head -1 "$f"); [ ${#l} -eq 80 ] || grep -q generated <<<"$l" || echo "banner: $f"; done +grep -rln --include='*.h' --include='*.td' 'ZML_' "$d" +grep -nE 'alwayslink|glob\(' "$d/BUILD.bazel" +[ -f "$d/capi.cc" ] && grep -nE 'extern "C"|^namespace \{' "$d/capi.cc" +[ -f "$d/capi.h" ] && grep -nw unsigned "$d/capi.h" +grep -rnE --include='*.zig' 'pub fn get[A-Z][A-Za-z]*\(self' "$d" +grep -rnE --include='*.zig' --exclude=test.zig '(Type|Attribute)\.parse\(' "$d" +grep -rln --include='*.zig' --exclude=test.zig '^test ' "$d" +grep -rqF --include='*.zig' 'pub fn registerDialect' "$d" || echo "missing: registerDialect" +if grep -rqF --include='*.zig' '.make(' "$d"; then + grep -qF 'const operation_names' "$d/test.zig" || echo "missing: operation_names" + grep -qF 'test "every bound operation is registered"' "$d/test.zig" || echo "missing: registration test" +fi +git grep -n 'DialectHandle.fromString' -- dsl zml examples +exit 0 +``` diff --git a/mlir/dialects/affine/BUILD.bazel b/mlir/dialects/affine/BUILD.bazel new file mode 100644 index 00000000..06f31849 --- /dev/null +++ b/mlir/dialects/affine/BUILD.bazel @@ -0,0 +1,35 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +cc_library( + name = "capi", + srcs = ["capi.cc"], + hdrs = ["capi.h"], + tags = ["manual"], + deps = [ + "@llvm-project//mlir:AffineDialect", + "@llvm-project//mlir:CAPIIR", + ], +) + +zig_library( + name = "affine", + import_name = "mlir/dialects/affine", + main = "affine.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + ":capi", + "//mlir", + "//stdx", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":affine", + "//mlir", + ], +) diff --git a/mlir/dialects/affine/affine.zig b/mlir/dialects/affine/affine.zig new file mode 100644 index 00000000..c808619c --- /dev/null +++ b/mlir/dialects/affine/affine.zig @@ -0,0 +1,413 @@ +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("affine"); +} + +// ============================================================================= +// affine.apply / affine.min / affine.max +// ============================================================================= + +/// affine.apply — apply a 1-result affine map to a list of dim+symbol +/// operands. Result is `index`. +pub fn apply( + ctx: *mlir.Context, + map: *const mlir.AffineMap, + map_operands: []const *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "affine.apply", .{ + .operands = .{ .flat = map_operands }, + .results = .{ .flat = &.{.index(ctx)} }, + .attributes = &.{ + .named(ctx, "map", .affineMap(map)), + }, + .location = location, + }); +} + +/// affine.min — minimum of all results of `map` evaluated at `map_operands`. +pub fn min( + ctx: *mlir.Context, + map: *const mlir.AffineMap, + map_operands: []const *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "affine.min", .{ + .operands = .{ .flat = map_operands }, + .results = .{ .flat = &.{.index(ctx)} }, + .attributes = &.{ + .named(ctx, "map", .affineMap(map)), + }, + .location = location, + }); +} + +/// affine.max — maximum of all results of `map`. +pub fn max( + ctx: *mlir.Context, + map: *const mlir.AffineMap, + map_operands: []const *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "affine.max", .{ + .operands = .{ .flat = map_operands }, + .results = .{ .flat = &.{.index(ctx)} }, + .attributes = &.{ + .named(ctx, "map", .affineMap(map)), + }, + .location = location, + }); +} + +// ============================================================================= +// affine.for / affine.yield +// ============================================================================= + +pub const ForOpts = struct { + /// Constant lower bound. Pass `null` when supplying a `lower_bound_map`. + lower_bound: ?i64 = null, + /// Constant upper bound. Pass `null` when supplying an `upper_bound_map`. + upper_bound: ?i64 = null, + /// Optional map-based bounds. When non-null they take precedence over the + /// constant counterpart. + lower_bound_map: ?*const mlir.AffineMap = null, + upper_bound_map: ?*const mlir.AffineMap = null, + /// Operands feeding `lower_bound_map` (in dim+symbol order). + lower_bound_operands: []const *const mlir.Value = &.{}, + /// Operands feeding `upper_bound_map`. + upper_bound_operands: []const *const mlir.Value = &.{}, + /// Loop step (positive, defaults to 1). + step: i64 = 1, + /// Loop-carried initial values. + inits: []const *const mlir.Value = &.{}, +}; + +/// affine.for — `for %i = lower to upper step S iter_args(...)`. The body +/// block takes `index, *init_arg_types` and must terminate with `affine.yield`. +pub fn for_( + ctx: *mlir.Context, + body: *mlir.Block, + opts: ForOpts, + location: *const mlir.Location, +) *mlir.Operation { + // Resolve final maps. + const lb_map = opts.lower_bound_map orelse blk: { + const cst = mlir.AffineExpr.constant(ctx, opts.lower_bound orelse 0); + break :blk mlir.AffineMap.get(ctx, 0, 0, &.{cst}); + }; + const ub_map = opts.upper_bound_map orelse blk: { + const cst = mlir.AffineExpr.constant(ctx, opts.upper_bound orelse 0); + break :blk mlir.AffineMap.get(ctx, 0, 0, &.{cst}); + }; + + var operands: stdx.BoundedArray(*const mlir.Value, 64) = .empty; + operands.appendSliceAssumeCapacity(opts.lower_bound_operands); + operands.appendSliceAssumeCapacity(opts.upper_bound_operands); + operands.appendSliceAssumeCapacity(opts.inits); + + const seg_sizes = [3]i32{ + @intCast(opts.lower_bound_operands.len), + @intCast(opts.upper_bound_operands.len), + @intCast(opts.inits.len), + }; + + var result_types: stdx.BoundedArray(*const mlir.Type, 64) = .empty; + for (opts.inits) |v| result_types.appendAssumeCapacity(v.type_()); + + return mlir.Operation.make(ctx, "affine.for", .{ + .operands = .{ .flat = operands.constSlice() }, + .results = .{ .flat = result_types.constSlice() }, + .blocks = &.{body}, + .attributes = &.{ + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), + .named(ctx, "lowerBoundMap", .affineMap(lb_map)), + .named(ctx, "upperBoundMap", .affineMap(ub_map)), + .named(ctx, "step", .int(ctx, .i64, opts.step)), + }, + // The for body may reference outer SSA values that only become + // reachable once it's appended to its parent — defer to module-level + // verify (same trick used by scf.for). + .verify = false, + .location = location, + }); +} + +/// affine.yield — terminator for affine.for / affine.if / affine.parallel. +pub fn yield(ctx: *mlir.Context, values: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "affine.yield", .{ + .operands = .{ .flat = values }, + .verify = false, + .location = location, + }); +} + +// ============================================================================= +// affine.if +// ============================================================================= + +/// affine.if — `if condition[ ... ] { then } else { else }`. `condition` is an +/// IntegerSetAttribute (a conjunction of affine constraints). The result types +/// are inferred from the yielded values; pass `&.{}` for the no-result form. +pub fn if_( + ctx: *mlir.Context, + condition: *const mlir.IntegerSet, + operands: []const *const mlir.Value, + result_types: []const *const mlir.Type, + then_block: *mlir.Block, + opts: struct { + else_block: ?*mlir.Block = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var state: mlir.OperationState = .init("affine.if", location); + state.addOperands(operands); + state.addResults(result_types); + state.addAttributes(&.{ + .named(ctx, "condition", .integerSet(condition)), + }); + + const then_region = mlir.Region.init(); + then_region.appendOwnedBlock(then_block); + const else_region = mlir.Region.init(); + if (opts.else_block) |b| else_region.appendOwnedBlock(b); + state.addOwnedRegions(&.{ then_region, else_region }); + + return mlir.Operation.init(&state) catch @panic("Failed to create affine.if"); +} + +// ============================================================================= +// affine.load / affine.store / affine.vector_load / affine.vector_store +// ============================================================================= + +/// affine.load — load with affine indexing. +pub fn load( + ctx: *mlir.Context, + memref: *const mlir.Value, + map: *const mlir.AffineMap, + map_operands: []const *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; + operands_buf.appendAssumeCapacity(memref); + operands_buf.appendSliceAssumeCapacity(map_operands); + + return mlir.Operation.make(ctx, "affine.load", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "map", .affineMap(map)), + }, + .location = location, + }); +} + +pub fn store( + ctx: *mlir.Context, + value: *const mlir.Value, + memref: *const mlir.Value, + map: *const mlir.AffineMap, + map_operands: []const *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 17) = .empty; + operands_buf.appendSliceAssumeCapacity(&.{ value, memref }); + operands_buf.appendSliceAssumeCapacity(map_operands); + + return mlir.Operation.make(ctx, "affine.store", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .attributes = &.{ + .named(ctx, "map", .affineMap(map)), + }, + .location = location, + }); +} + +pub fn vector_load( + ctx: *mlir.Context, + memref: *const mlir.Value, + map: *const mlir.AffineMap, + map_operands: []const *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; + operands_buf.appendAssumeCapacity(memref); + operands_buf.appendSliceAssumeCapacity(map_operands); + + return mlir.Operation.make(ctx, "affine.vector_load", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "map", .affineMap(map)), + }, + .location = location, + }); +} + +pub fn vector_store( + ctx: *mlir.Context, + value: *const mlir.Value, + memref: *const mlir.Value, + map: *const mlir.AffineMap, + map_operands: []const *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 17) = .empty; + operands_buf.appendSliceAssumeCapacity(&.{ value, memref }); + operands_buf.appendSliceAssumeCapacity(map_operands); + + return mlir.Operation.make(ctx, "affine.vector_store", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .attributes = &.{ + .named(ctx, "map", .affineMap(map)), + }, + .location = location, + }); +} + +// ============================================================================= +// affine.parallel +// ============================================================================= + +pub const ParallelArgs = struct { + /// Per-loop-dim reductions, one entry per result. Use the same kinds as + /// `AtomicRMWKind` in `mlir/dialects/memref` (the dialect uses + /// `arith::AtomicRMWKind`). Pass `&.{}` for parallel loops with no + /// loop-carried reductions. + reductions: []const i64 = &.{}, + /// Map producing all lower-bound values; a single dim's bound is the + /// max of the results in its `lower_bounds_groups[i]` group. + lower_bounds_map: *const mlir.AffineMap, + /// One i32 per loop dim — number of map results that contribute to the + /// lower bound of that dim. Sum must equal `lower_bounds_map.numResults()`. + lower_bounds_groups: []const i32, + upper_bounds_map: *const mlir.AffineMap, + upper_bounds_groups: []const i32, + /// Per-dim integer steps (default 1). + steps: []const i64, + /// Operands feeding both bound maps. + map_operands: []const *const mlir.Value = &.{}, +}; + +/// affine.parallel — multi-dim parallel band. `body_block` takes one `index` +/// argument per loop dim and must terminate with `affine.yield`. +pub fn parallel( + ctx: *mlir.Context, + args: ParallelArgs, + body_block: *mlir.Block, + result_types: []const *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + // I32ElementsAttr — dense<[...]> : tensor. + const lb_groups_ty = mlir.RankedTensorType.get( + &.{@intCast(args.lower_bounds_groups.len)}, + .int(ctx, .i32), + null, + ).shaped(); + const ub_groups_ty = mlir.RankedTensorType.get( + &.{@intCast(args.upper_bounds_groups.len)}, + .int(ctx, .i32), + null, + ).shaped(); + + // I64SmallVectorArrayAttr — built as ArrayAttr>. + var steps_attrs: stdx.BoundedArray(*const mlir.Attribute, 16) = .empty; + for (args.steps) |s| { + steps_attrs.appendAssumeCapacity(.int(ctx, .i64, s)); + } + + // Reductions are AtomicRMWKindAttr (I64EnumAttr). We let callers pass + // raw int values to avoid coupling against the enum table. + var reductions_attrs: stdx.BoundedArray(*const mlir.Attribute, 16) = .empty; + for (args.reductions) |k| { + reductions_attrs.appendAssumeCapacity(.int(ctx, .i64, k)); + } + + return mlir.Operation.make(ctx, "affine.parallel", .{ + .operands = .{ .flat = args.map_operands }, + .results = .{ .flat = result_types }, + .blocks = &.{body_block}, + .attributes = &.{ + .named(ctx, "reductions", .array(ctx, reductions_attrs.constSlice())), + .named(ctx, "lowerBoundsMap", .affineMap(args.lower_bounds_map)), + .named(ctx, "lowerBoundsGroups", .denseElements(lb_groups_ty, args.lower_bounds_groups)), + .named(ctx, "upperBoundsMap", .affineMap(args.upper_bounds_map)), + .named(ctx, "upperBoundsGroups", .denseElements(ub_groups_ty, args.upper_bounds_groups)), + .named(ctx, "steps", .array(ctx, steps_attrs.constSlice())), + }, + .verify = false, + .location = location, + }); +} + +// ============================================================================= +// affine.delinearize_index / affine.linearize_index +// ============================================================================= + +pub const DelinearizeArgs = struct { + static_basis: []const i64, + dynamic_basis: []const *const mlir.Value = &.{}, +}; + +pub fn delinearize_index( + ctx: *mlir.Context, + linear_index: *const mlir.Value, + args: DelinearizeArgs, + result_types: []const *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; + operands_buf.appendAssumeCapacity(linear_index); + operands_buf.appendSliceAssumeCapacity(args.dynamic_basis); + + return mlir.Operation.make(ctx, "affine.delinearize_index", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = result_types }, + .attributes = &.{ + .named(ctx, "static_basis", .denseArray(ctx, .i64, args.static_basis)), + }, + .location = location, + }); +} + +pub const LinearizeArgs = struct { + multi_index: []const *const mlir.Value, + static_basis: []const i64, + dynamic_basis: []const *const mlir.Value = &.{}, + disjoint: bool = false, +}; + +pub fn linearize_index( + ctx: *mlir.Context, + args: LinearizeArgs, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 32) = .empty; + operands_buf.appendSliceAssumeCapacity(args.multi_index); + operands_buf.appendSliceAssumeCapacity(args.dynamic_basis); + + const seg_sizes = [2]i32{ + @intCast(args.multi_index.len), + @intCast(args.dynamic_basis.len), + }; + + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attrs.appendSliceAssumeCapacity(&.{ + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), + .named(ctx, "static_basis", .denseArray(ctx, .i64, args.static_basis)), + }); + if (args.disjoint) { + attrs.appendAssumeCapacity(.named(ctx, "disjoint", .unit(ctx))); + } + + return mlir.Operation.make(ctx, "affine.linearize_index", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} diff --git a/mlir/dialects/affine/capi.cc b/mlir/dialects/affine/capi.cc new file mode 100644 index 00000000..2110365f --- /dev/null +++ b/mlir/dialects/affine/capi.cc @@ -0,0 +1,9 @@ +//===- capi.cc - C API for the affine dialect -----------------------------===// + +#include "capi.h" + +#include "mlir/CAPI/Registration.h" +#include "mlir/Dialect/Affine/IR/AffineOps.h" + +MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(Affine, affine, + mlir::affine::AffineDialect) diff --git a/mlir/dialects/affine/capi.h b/mlir/dialects/affine/capi.h new file mode 100644 index 00000000..5ae66467 --- /dev/null +++ b/mlir/dialects/affine/capi.h @@ -0,0 +1,18 @@ +//===- capi.h - C API for the affine dialect ----------------------*- C -*-===// + +#ifndef MLIR_DIALECTS_AFFINE_CAPI_H +#define MLIR_DIALECTS_AFFINE_CAPI_H + +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(Affine, affine); + +#ifdef __cplusplus +} +#endif + +#endif // MLIR_DIALECTS_AFFINE_CAPI_H diff --git a/mlir/dialects/affine/test.zig b/mlir/dialects/affine/test.zig new file mode 100644 index 00000000..4248d4c3 --- /dev/null +++ b/mlir/dialects/affine/test.zig @@ -0,0 +1,40 @@ +const std = @import("std"); + +const affine = @import("mlir/dialects/affine"); +const mlir = @import("mlir"); + +const operation_names: []const []const u8 = &.{ + "affine.apply", + "affine.delinearize_index", + "affine.for", + "affine.if", + "affine.linearize_index", + "affine.load", + "affine.max", + "affine.min", + "affine.parallel", + "affine.store", + "affine.vector_load", + "affine.vector_store", + "affine.yield", +}; + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(affine); + inline for (comptime std.meta.declarations(affine)) |decl| { + const T = @field(affine, decl); + if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); + } +} + +test "every bound operation is registered" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + affine.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} diff --git a/mlir/dialects/air/AIRAttributes.cpp b/mlir/dialects/air/AIRAttributes.cpp new file mode 100644 index 00000000..287cb26b --- /dev/null +++ b/mlir/dialects/air/AIRAttributes.cpp @@ -0,0 +1,140 @@ +//===- AIRAttributes.cpp - Metal AIR attributes ---------------------------===// + +#include "mlir/dialects/air/AIRAttributes.h" + +#include "mlir/dialects/air/AIRDialect.h" + +#include "mlir/Dialect/LLVMIR/LLVMTypes.h" +#include "mlir/IR/Builders.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/DialectImplementation.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/TypeSwitch.h" +#include "llvm/Support/MathExtras.h" + +#include + +using namespace mlir; +using namespace mlir::metal::air; + +#include "mlir/dialects/air/AIREnums.cpp.inc" +#define GET_ATTRDEF_CLASSES +#include "mlir/dialects/air/AIRAttributes.cpp.inc" + +void AIRDialect::registerAttributes() { + addAttributes< +#define GET_ATTRDEF_LIST +#include "mlir/dialects/air/AIRAttributes.cpp.inc" + >(); +} + +static bool isValidName(StringRef name) { + return !name.empty() && !name.contains('\0'); +} + +// The size of a Metal scalar or vector, where 3-element vectors take the +// space of 4. A bool vector is an i8 vector in memory, as LLVM packs i1 +// vectors into bits. +static std::optional metalSize(Type type) { + uint64_t elements = 1; + if (auto vector = dyn_cast(type)) { + if (vector.isScalable() || vector.getRank() != 1 || + vector.getElementType().isInteger(1)) + return std::nullopt; + elements = vector.getNumElements(); + if (elements < 2 || elements > 4) + return std::nullopt; + if (elements == 3) + elements = 4; + type = vector.getElementType(); + } + // bool, char, short, int, long, half, bfloat and float. + if (!type.isInteger(1) && !type.isInteger(8) && !type.isInteger(16) && + !type.isInteger(32) && !type.isInteger(64) && !type.isF16() && + !type.isBF16() && !type.isF32()) + return std::nullopt; + return elements * llvm::divideCeil(type.getIntOrFloatBitWidth(), 8); +} + +LogicalResult +StructFieldAttr::verify(llvm::function_ref emitError, + uint64_t offset, uint64_t size, uint64_t count, + StringRef typeName, StringRef name) { + if (!size) + return emitError() << "expected a positive struct field size"; + if (!isValidName(typeName) || !isValidName(name)) + return emitError() << "expected nonempty field and type names"; + return success(); +} + +static LogicalResult +verifyStructFields(llvm::function_ref emitError, + LLVM::LLVMStructType type, uint64_t typeSize, + ArrayRef fields) { + if (type.isOpaque() || type.isPacked()) + return emitError() << "expected a sized, unpacked struct"; + ArrayRef body = type.getBody(); + if (body.size() != fields.size()) + return emitError() << "expected one field per struct element"; + uint64_t end = 0; + for (auto [member, field] : llvm::zip_equal(body, fields)) { + Type element = member; + uint64_t count = 0; + if (auto array = dyn_cast(element)) { + count = array.getNumElements(); + element = array.getElementType(); + } + std::optional size = metalSize(element); + if (!size || *size != field.getSize() || count != field.getCount()) + return emitError() << "field " << field.getName() + << " does not match the struct element " << element; + if (field.getOffset() < end || field.getOffset() % *size != 0) + return emitError() << "field " << field.getName() + << " is misplaced at offset " << field.getOffset(); + end = field.getOffset() + *size * std::max(count, 1); + } + if (end > typeSize) + return emitError() << "struct fields overflow the buffer type size"; + return success(); +} + +LogicalResult BufferAttr::verify( + llvm::function_ref emitError, unsigned locationIndex, + AddressSpace addressSpace, Access access, Type elementType, + StringRef typeName, uint64_t typeSize, uint64_t typeAlignment, + uint64_t bufferSize, bool noalias, ArrayRef fields) { + if (addressSpace == AddressSpace::thread) + return emitError() << "buffers are not in thread memory"; + if (addressSpace == AddressSpace::constant && access != Access::read) + return emitError() << "constant buffers are read-only"; + if (addressSpace == AddressSpace::threadgroup && access != Access::read_write) + return emitError() << "threadgroup buffers are read_write"; + if (!elementType) + return emitError() << "expected a buffer element type"; + if (!isValidName(typeName)) + return emitError() << "expected a nonempty Metal type name"; + if (!typeSize || !llvm::isPowerOf2_64(typeAlignment) || + typeSize % typeAlignment != 0) + return emitError() << "expected a positive ABI size divisible by a " + "power-of-two alignment"; + if (auto structType = dyn_cast(elementType)) { + if (addressSpace != AddressSpace::constant) + return emitError() << "struct buffers must be constant"; + if (failed(verifyStructFields(emitError, structType, typeSize, fields))) + return failure(); + } else { + std::optional size = metalSize(elementType); + if (!size) + return emitError() << "expected a scalar, vector or struct element type"; + if (*size != typeSize) + return emitError() << "ABI size " << typeSize << " does not match " + << elementType; + if (!fields.empty()) + return emitError() << "only struct buffers have fields"; + } + if (bufferSize && (addressSpace != AddressSpace::constant || + bufferSize < typeSize || bufferSize % typeAlignment)) + return emitError() << "only constant buffers have a buffer size, which " + "must cover the element and respect its alignment"; + return success(); +} diff --git a/mlir/dialects/air/AIRAttributes.h b/mlir/dialects/air/AIRAttributes.h new file mode 100644 index 00000000..4a31eb77 --- /dev/null +++ b/mlir/dialects/air/AIRAttributes.h @@ -0,0 +1,12 @@ +//===- AIRAttributes.h - Metal AIR attributes -------------------*- C++ -*-===// + +#ifndef MLIR_DIALECTS_AIR_AIRATTRIBUTES_H +#define MLIR_DIALECTS_AIR_AIRATTRIBUTES_H + +#include "mlir/dialects/air/AIREnums.h" + +#include "mlir/IR/Attributes.h" +#define GET_ATTRDEF_CLASSES +#include "mlir/dialects/air/AIRAttributes.h.inc" + +#endif // MLIR_DIALECTS_AIR_AIRATTRIBUTES_H diff --git a/mlir/dialects/air/AIRAttributes.td b/mlir/dialects/air/AIRAttributes.td new file mode 100644 index 00000000..3e4852d0 --- /dev/null +++ b/mlir/dialects/air/AIRAttributes.td @@ -0,0 +1,66 @@ +//===- AIRAttributes.td - Metal AIR attributes -------------*- tablegen -*-===// + +#ifndef MLIR_DIALECTS_AIR_AIRATTRIBUTES_TD +#define MLIR_DIALECTS_AIR_AIRATTRIBUTES_TD + +include "mlir/dialects/air/AIRDialect.td" +include "mlir/dialects/air/AIREnums.td" + +def AIR_BuiltinAttr : EnumAttr { + let summary = "Kernel argument builtin"; + let description = [{ + Attach as `air.builtin` to a kernel argument. The position and size + builtins are `uint` or `ushort` scalars or 2- and 3-element vectors + (`i32`, `i16`, `vector<3xi32>`, ...); the index builtins are `uint` or + `ushort` scalars. + }]; +} + +def AIR_MemFlagsAttr : EnumAttr { + let summary = "Memory a barrier orders"; +} + +def AIR_StructField : AttrDef { + let mnemonic = "struct_field"; + let summary = "Field of a constant buffer struct"; + let description = [{ + One `air.struct_type_info` entry: the byte offset, the element size, + the array length (zero for a non-array field), the Metal type name and + the field name. + }]; + let parameters = (ins "uint64_t":$offset, + "uint64_t":$size, + DefaultValuedParameter<"uint64_t", "0">:$count, + StringRefParameter<>:$typeName, + StringRefParameter<>:$name); + let assemblyFormat = "`<` struct(params) `>`"; + let genVerifyDecl = 1; +} + +def AIR_Buffer : AttrDef { + let mnemonic = "buffer"; + let summary = "Kernel buffer argument"; + let description = [{ + Attach as `air.buffer` to a kernel argument of type `!llvm.ptr` in + `addressSpace`. `elementType` is the pointee (a scalar, a vector, or an + `!llvm.struct` described by `fields`), `typeName` its Metal spelling, + and `typeSize` and `typeAlignment` its size and alignment in bytes. + `locationIndex` is the `[[buffer(n)]]` or `[[threadgroup(n)]]` index. + A constant reference records its size as `bufferSize`. `noalias` is a + `__restrict__` pointer. + }]; + let parameters = (ins "unsigned":$locationIndex, + EnumParameter:$addressSpace, + EnumParameter:$access, + "::mlir::Type":$elementType, + StringRefParameter<>:$typeName, + "uint64_t":$typeSize, + "uint64_t":$typeAlignment, + DefaultValuedParameter<"uint64_t", "0">:$bufferSize, + DefaultValuedParameter<"bool", "false">:$noalias, + OptionalArrayRefParameter<"StructFieldAttr">:$fields); + let assemblyFormat = "`<` struct(params) `>`"; + let genVerifyDecl = 1; +} + +#endif // MLIR_DIALECTS_AIR_AIRATTRIBUTES_TD diff --git a/mlir/dialects/air/AIRDialect.cpp b/mlir/dialects/air/AIRDialect.cpp new file mode 100644 index 00000000..7ad56427 --- /dev/null +++ b/mlir/dialects/air/AIRDialect.cpp @@ -0,0 +1,168 @@ +//===- AIRDialect.cpp - Metal AIR dialect ---------------------------------===// + +#include "mlir/dialects/air/AIRDialect.h" + +#include "mlir/dialects/air/AIRAttributes.h" +#include "mlir/dialects/air/AIROps.h" + +#include "mlir/Dialect/LLVMIR/LLVMDialect.h" +#include "mlir/IR/BuiltinOps.h" +#include "mlir/IR/DialectImplementation.h" +#include "mlir/Interfaces/FunctionInterfaces.h" +#include "llvm/ADT/DenseSet.h" + +#include + +using namespace mlir; +using namespace mlir::metal::air; + +#include "mlir/dialects/air/AIRDialect.cpp.inc" + +void AIRDialect::initialize() { + registerAttributes(); + addOperations< +#define GET_OP_LIST +#include "mlir/dialects/air/AIROps.cpp.inc" + >(); +} + +static bool isKernelArgumentAttribute(StringRef name) { + return name == AIRDialect::getBuiltinAttrName() || + name == AIRDialect::getBufferAttrName() || + name == AIRDialect::getArgNameAttrName(); +} + +// Position and size builtins may be 2- or 3-element vectors; index builtins +// are scalars. +static bool isVectorBuiltin(Builtin builtin) { + switch (builtin) { + case Builtin::thread_position_in_grid: + case Builtin::threadgroup_position_in_grid: + case Builtin::thread_position_in_threadgroup: + case Builtin::threads_per_threadgroup: + case Builtin::threadgroups_per_grid: + case Builtin::threads_per_grid: + return true; + default: + return false; + } +} + +static LogicalResult verifyKernelArgument(FunctionOpInterface function, + unsigned index) { + Operation *op = function.getOperation(); + Attribute builtin = + function.getArgAttr(index, AIRDialect::getBuiltinAttrName()); + Attribute buffer = + function.getArgAttr(index, AIRDialect::getBufferAttrName()); + if (bool(builtin) == bool(buffer)) + return op->emitOpError() + << "argument " << index + << " must have exactly one of air.builtin or air.buffer"; + if (Attribute name = + function.getArgAttr(index, AIRDialect::getArgNameAttrName())) + if (auto string = dyn_cast(name); + !string || string.getValue().empty()) + return op->emitOpError("air.arg_name must be a nonempty string"); + Type type = function.getArgumentTypes()[index]; + if (builtin) { + auto metadata = dyn_cast(builtin); + if (!metadata) + return op->emitOpError("air.builtin must be an #air.builtin"); + if (auto vector = dyn_cast(type)) { + if (!isVectorBuiltin(metadata.getValue()) || vector.isScalable() || + vector.getRank() != 1 || + (vector.getNumElements() != 2 && vector.getNumElements() != 3)) + return op->emitOpError() + << "builtin " << stringifyBuiltin(metadata.getValue()) + << " cannot be a " << type; + type = vector.getElementType(); + } + if (!type.isSignlessInteger(32) && !type.isSignlessInteger(16)) + return op->emitOpError("AIR builtins are uint or ushort scalars or " + "vectors"); + return success(); + } + auto metadata = dyn_cast(buffer); + if (!metadata) + return op->emitOpError("air.buffer must be an #air.buffer"); + auto pointer = dyn_cast(type); + if (!pointer || + pointer.getAddressSpace() != unsigned(metadata.getAddressSpace())) + return op->emitOpError() + << "argument " << index + << " must be an !llvm.ptr in the address space of its air.buffer"; + return success(); +} + +LogicalResult AIRDialect::verifyOperationAttribute(Operation *op, + NamedAttribute attr) { + StringRef name = attr.getName().getValue(); + if (isKernelArgumentAttribute(name)) + return op->emitOpError() << name << " belongs on a kernel argument"; + if (name == AIRDialect::getFastMathEnableAttrName()) { + if (!isa(op) || !isa(attr.getValue())) + return op->emitOpError() << name << " is a unit attribute of a module"; + return success(); + } + if (name != AIRDialect::getKernelAttrName()) + return op->emitOpError() << "unknown AIR attribute " << name; + auto function = dyn_cast(op); + if (!function || !isa(attr.getValue())) + return op->emitOpError( + "air.kernel requires a function and a unit attribute"); + if (failed(op->getName().verifyInvariants(op))) + return failure(); + if (function.isExternal()) + return op->emitOpError("AIR kernels need a body"); + if (function.getNumResults() != 0) + return op->emitOpError("AIR kernels cannot return values"); + if (auto llvmFunction = dyn_cast(op)) + if (llvmFunction.isVarArg()) + return op->emitOpError("AIR kernels cannot be variadic"); + llvm::DenseSet builtins; + llvm::DenseSet> bindings; + for (unsigned i = 0; i < function.getNumArguments(); ++i) { + if (failed(verifyKernelArgument(function, i))) + return failure(); + if (auto builtin = function.getArgAttrOfType( + i, AIRDialect::getBuiltinAttrName())) { + if (!builtins.insert(builtin.getValue()).second) + return op->emitOpError() << "duplicate AIR builtin " + << stringifyBuiltin(builtin.getValue()); + continue; + } + auto buffer = function.getArgAttrOfType( + i, AIRDialect::getBufferAttrName()); + // Device and constant buffers share bindings; threadgroup buffers do not. + auto key = std::make_pair( + unsigned(buffer.getAddressSpace() == AddressSpace::threadgroup), + buffer.getLocationIndex()); + if (!bindings.insert(key).second) + return op->emitOpError() + << "duplicate AIR buffer binding " << buffer.getLocationIndex(); + } + return success(); +} + +LogicalResult AIRDialect::verifyRegionArgAttribute(Operation *op, + unsigned regionIndex, + unsigned argIndex, + NamedAttribute attr) { + StringRef name = attr.getName().getValue(); + if (!isKernelArgumentAttribute(name)) + return op->emitOpError() << name << " is not an AIR argument attribute"; + auto function = dyn_cast(op); + if (!function || regionIndex != 0 || + !op->getAttrOfType(AIRDialect::getKernelAttrName())) + return op->emitOpError() + << name << " requires an air.kernel function argument"; + return verifyKernelArgument(function, argIndex); +} + +LogicalResult AIRDialect::verifyRegionResultAttribute(Operation *op, unsigned, + unsigned, + NamedAttribute attr) { + return op->emitOpError() << attr.getName() + << " cannot annotate a function result"; +} diff --git a/mlir/dialects/air/AIRDialect.h b/mlir/dialects/air/AIRDialect.h new file mode 100644 index 00000000..e28ec0bc --- /dev/null +++ b/mlir/dialects/air/AIRDialect.h @@ -0,0 +1,9 @@ +//===- AIRDialect.h - Metal AIR dialect -------------------------*- C++ -*-===// + +#ifndef MLIR_DIALECTS_AIR_AIRDIALECT_H +#define MLIR_DIALECTS_AIR_AIRDIALECT_H + +#include "mlir/IR/Dialect.h" +#include "mlir/dialects/air/AIRDialect.h.inc" + +#endif // MLIR_DIALECTS_AIR_AIRDIALECT_H diff --git a/mlir/dialects/air/AIRDialect.td b/mlir/dialects/air/AIRDialect.td new file mode 100644 index 00000000..96264d76 --- /dev/null +++ b/mlir/dialects/air/AIRDialect.td @@ -0,0 +1,53 @@ +//===- AIRDialect.td - Metal AIR dialect -------------------*- tablegen -*-===// + +#ifndef MLIR_DIALECTS_AIR_AIRDIALECT_TD +#define MLIR_DIALECTS_AIR_AIRDIALECT_TD + +include "mlir/IR/AttrTypeBase.td" +include "mlir/IR/OpBase.td" + +def AIR_Dialect : Dialect { + let name = "air"; + let cppNamespace = "::mlir::metal::air"; + let summary = "Apple Metal AIR intrinsics and kernel ABI metadata"; + let description = [{ + The Metal-specific part of the LLVM IR Apple's Metal compiler emits (AIR), + not AMD's asynchronous AIR dialect. Each operation is an `air.*` + intrinsic named as Apple names it, without the type suffix its operand + and result types determine: `air.fast_exp %x : f32` is + `@air.fast_exp.f32`, `air.max.s %a, %b : vector<2xi32>` is + `@air.max.s.v2i32`. Kernels are `func.func` with `air.kernel`, and their + arguments carry the `!air.kernel` descriptors as `air.builtin` or + `air.buffer`. Arithmetic, memory and control flow use the arith, llvm + and scf dialects; pointers are `!llvm.ptr` in AIR's address spaces. + }]; + let dependentDialects = ["::mlir::LLVM::LLVMDialect"]; + let useDefaultAttributePrinterParser = 1; + let hasOperationAttrVerify = 1; + let hasRegionArgAttrVerify = 1; + let hasRegionResultAttrVerify = 1; + let extraClassDeclaration = [{ + void registerAttributes(); + + static constexpr ::llvm::StringLiteral getKernelAttrName() { + return ::llvm::StringLiteral("air.kernel"); + } + static constexpr ::llvm::StringLiteral getBuiltinAttrName() { + return ::llvm::StringLiteral("air.builtin"); + } + static constexpr ::llvm::StringLiteral getBufferAttrName() { + return ::llvm::StringLiteral("air.buffer"); + } + static constexpr ::llvm::StringLiteral getArgNameAttrName() { + return ::llvm::StringLiteral("air.arg_name"); + } + static constexpr ::llvm::StringLiteral getFastMathEnableAttrName() { + return ::llvm::StringLiteral("air.fast_math_enable"); + } + }]; +} + +class AIR_Op traits = []> + : Op; + +#endif // MLIR_DIALECTS_AIR_AIRDIALECT_TD diff --git a/mlir/dialects/air/AIREnums.h b/mlir/dialects/air/AIREnums.h new file mode 100644 index 00000000..385f2041 --- /dev/null +++ b/mlir/dialects/air/AIREnums.h @@ -0,0 +1,10 @@ +//===- AIREnums.h - Metal AIR enums -----------------------------*- C++ -*-===// + +#ifndef MLIR_DIALECTS_AIR_AIRENUMS_H +#define MLIR_DIALECTS_AIR_AIRENUMS_H + +#include "mlir/IR/BuiltinAttributes.h" +#include "mlir/dialects/air/AIREnums.h.inc" +#include "llvm/ADT/StringExtras.h" + +#endif // MLIR_DIALECTS_AIR_AIRENUMS_H diff --git a/mlir/dialects/air/AIREnums.td b/mlir/dialects/air/AIREnums.td new file mode 100644 index 00000000..61e9c25a --- /dev/null +++ b/mlir/dialects/air/AIREnums.td @@ -0,0 +1,54 @@ +//===- AIREnums.td - Metal AIR enums -----------------------*- tablegen -*-===// + +#ifndef MLIR_DIALECTS_AIR_AIRENUMS_TD +#define MLIR_DIALECTS_AIR_AIRENUMS_TD + +include "mlir/IR/EnumAttr.td" + +def AIR_Builtin : I32EnumAttr<"Builtin", "Kernel argument builtin", [ + I32EnumAttrCase<"thread_position_in_grid", 0>, + I32EnumAttrCase<"threadgroup_position_in_grid", 1>, + I32EnumAttrCase<"thread_position_in_threadgroup", 2>, + I32EnumAttrCase<"threads_per_threadgroup", 3>, + I32EnumAttrCase<"threadgroups_per_grid", 4>, + I32EnumAttrCase<"threads_per_grid", 5>, + I32EnumAttrCase<"thread_index_in_simdgroup", 6>, + I32EnumAttrCase<"simdgroup_index_in_threadgroup", 7>, + I32EnumAttrCase<"threads_per_simdgroup", 8>, + I32EnumAttrCase<"simdgroups_per_threadgroup", 9>, + I32EnumAttrCase<"thread_index_in_threadgroup", 10>, +]> { + let cppNamespace = "::mlir::metal::air"; + let genSpecializedAttr = 0; +} + +def AIR_AddressSpace : I32EnumAttr<"AddressSpace", "Address space", [ + I32EnumAttrCase<"thread", 0>, + I32EnumAttrCase<"device", 1>, + I32EnumAttrCase<"constant", 2>, + I32EnumAttrCase<"threadgroup", 3>, +]> { + let cppNamespace = "::mlir::metal::air"; + let genSpecializedAttr = 0; +} + +def AIR_Access : I32EnumAttr<"Access", "Buffer access", [ + I32EnumAttrCase<"read", 0>, + I32EnumAttrCase<"read_write", 1>, +]> { + let cppNamespace = "::mlir::metal::air"; + let genSpecializedAttr = 0; +} + +def AIR_MemFlags : I32BitEnumAttr<"MemFlags", "Barrier memory flags", [ + I32BitEnumAttrCaseNone<"none">, + I32BitEnumAttrCaseBit<"device", 0>, + I32BitEnumAttrCaseBit<"threadgroup", 1>, + I32BitEnumAttrCaseBit<"texture", 2>, +]> { + let separator = "|"; + let cppNamespace = "::mlir::metal::air"; + let genSpecializedAttr = 0; +} + +#endif // MLIR_DIALECTS_AIR_AIRENUMS_TD diff --git a/mlir/dialects/air/AIROps.cpp b/mlir/dialects/air/AIROps.cpp new file mode 100644 index 00000000..85e48837 --- /dev/null +++ b/mlir/dialects/air/AIROps.cpp @@ -0,0 +1,64 @@ +//===- AIROps.cpp - Metal AIR operations ----------------------------------===// + +#include "mlir/dialects/air/AIROps.h" + +#include "mlir/dialects/air/AIRDialect.h" + +#include "mlir/IR/Builders.h" +#include "mlir/IR/OpImplementation.h" +#include "mlir/IR/TypeUtilities.h" + +using namespace mlir; +using namespace mlir::metal::air; + +#define GET_OP_CLASSES +#include "mlir/dialects/air/AIROps.cpp.inc" + +LogicalResult DotOp::verify() { + if (getElementTypeOrSelf(getX().getType()) != getResult().getType()) + return emitOpError("expects the operands' element type as result"); + return success(); +} + +static LogicalResult verifyConvert(Operation *op, bool toOrFromInteger) { + Type input = op->getOperand(0).getType(); + Type result = op->getResult(0).getType(); + if (failed(verifyCompatibleShape(input, result)) || + isa(input) != isa(result)) + return op->emitOpError("expects operands and results of the same shape"); + bool inputInt = isa(getElementTypeOrSelf(input)); + bool resultInt = isa(getElementTypeOrSelf(result)); + if (!toOrFromInteger && (inputInt || resultInt || input == result)) + return op->emitOpError("converts between distinct float types"); + if (!toOrFromInteger && !isa(input) && + (input.isF32() || result.isF32())) + return op->emitOpError("scalar f32 conversions are LLVM casts in Apple's " + "compiler"); + if (toOrFromInteger && inputInt == resultInt) + return op->emitOpError("converts between an integer and a float type"); + if (resultInt && cast(op) + .getFastMathFlagsAttr() + .getValue() != arith::FastMathFlags::none) + return op->emitOpError("integer results take no fastmath flags"); + return success(); +} + +LogicalResult ConvertOp::verify() { return verifyConvert(*this, false); } +LogicalResult ConvertSOp::verify() { return verifyConvert(*this, true); } +LogicalResult ConvertUOp::verify() { return verifyConvert(*this, true); } + +LogicalResult SimdgroupMatrixMultiplyAccumulateOp::verify() { + auto element = [](Value value) { + return cast(value.getType()).getElementType(); + }; + Type d = element(getResult()), a = element(getA()), b = element(getB()), + c = element(getC()); + // The combinations Apple's compiler emits for simdgroup_multiply_accumulate. + bool mixed = d.isF32() && c.isF32() && (a.isF32() || a == b) && + (b.isF16() || b.isBF16() || b.isF32()); + bool half = d.isF16() && a.isF16() && b.isF16() && c.isF16(); + if (!mixed && !half) + return emitOpError("unsupported element types ") + << a << " * " << b << " + " << c << " -> " << d; + return success(); +} diff --git a/mlir/dialects/air/AIROps.h b/mlir/dialects/air/AIROps.h new file mode 100644 index 00000000..baa5a517 --- /dev/null +++ b/mlir/dialects/air/AIROps.h @@ -0,0 +1,16 @@ +//===- AIROps.h - Metal AIR operations --------------------------*- C++ -*-===// + +#ifndef MLIR_DIALECTS_AIR_AIROPS_H +#define MLIR_DIALECTS_AIR_AIROPS_H + +#include "mlir/dialects/air/AIRAttributes.h" + +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/Dialect/LLVMIR/LLVMTypes.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/OpDefinition.h" +#include "mlir/Interfaces/SideEffectInterfaces.h" +#define GET_OP_CLASSES +#include "mlir/dialects/air/AIROps.h.inc" + +#endif // MLIR_DIALECTS_AIR_AIROPS_H diff --git a/mlir/dialects/air/AIROps.td b/mlir/dialects/air/AIROps.td new file mode 100644 index 00000000..6d072437 --- /dev/null +++ b/mlir/dialects/air/AIROps.td @@ -0,0 +1,352 @@ +//===- AIROps.td - Metal AIR operations --------------------*- tablegen -*-===// + +#ifndef MLIR_DIALECTS_AIR_AIROPS_TD +#define MLIR_DIALECTS_AIR_AIROPS_TD + +include "mlir/Dialect/Arith/IR/ArithBase.td" +include "mlir/Dialect/Arith/IR/ArithOpsInterfaces.td" +include "mlir/Dialect/LLVMIR/LLVMOpBase.td" +include "mlir/Interfaces/InferTypeOpInterface.td" +include "mlir/Interfaces/SideEffectInterfaces.td" +include "mlir/dialects/air/AIRAttributes.td" + +class AIR_VectorOf types> + : FixedVectorOfRankAndLengthAndType<[1], [2, 3, 4], types>; +class AIR_ScalarOrVectorOf types> + : AnyTypeOf])>; + +def AIR_FloatLike : AIR_ScalarOrVectorOf<[F16, F32]>; +def AIR_F32Like : AIR_ScalarOrVectorOf<[F32]>; +def AIR_IntLike : AIR_ScalarOrVectorOf<[I8, I16, I32, I64]>; +def AIR_SimdIntLike : AIR_ScalarOrVectorOf<[I8, I16, I32]>; +def AIR_ConvertLike : AIR_ScalarOrVectorOf<[F16, BF16, F32, I8, I16, I32, I64]>; +def AIR_Matrix : FixedVectorOfRankAndLengthAndType<[1], [64], [F16, BF16, F32]>; +def AIR_MatrixVector : FixedVectorOfShapeAndType<[2], I64>; +def AIR_MatrixPointer : AnyTypeOf<[LLVM_PointerInAddressSpace<1>, + LLVM_PointerInAddressSpace<3>]>; + +def AIR_FastMath : DefaultValuedAttr; + +//===----------------------------------------------------------------------===// +// Math +//===----------------------------------------------------------------------===// + +class AIR_ElementwiseOp traits = []> + : AIR_Op { + let summary = "`" # mnemonic # "` on each element"; + let description = "Calls `@air." # mnemonic # ".`."; +} + +class AIR_FloatUnaryOp + : AIR_ElementwiseOp]> { + let arguments = (ins type:$x, AIR_FastMath:$fastmath); + let results = (outs type:$result); + let assemblyFormat = + "$x (`fastmath` `` $fastmath^)? attr-dict `:` type($result)"; +} + +class AIR_FloatBinaryOp + : AIR_ElementwiseOp]> { + let arguments = (ins type:$x, type:$y, AIR_FastMath:$fastmath); + let results = (outs type:$result); + let assemblyFormat = + "$x `,` $y (`fastmath` `` $fastmath^)? attr-dict `:` type($result)"; +} + +class AIR_FloatTernaryOp + : AIR_ElementwiseOp]> { + let arguments = (ins type:$x, type:$y, type:$z, AIR_FastMath:$fastmath); + let results = (outs type:$result); + let assemblyFormat = [{ + $x `,` $y `,` $z (`fastmath` `` $fastmath^)? attr-dict `:` type($result) + }]; +} + +class AIR_IntUnaryOp : AIR_ElementwiseOp { + let arguments = (ins AIR_IntLike:$x); + let results = (outs AIR_IntLike:$result); + let assemblyFormat = "$x attr-dict `:` type($result)"; +} + +class AIR_IntBinaryOp : AIR_ElementwiseOp { + let arguments = (ins AIR_IntLike:$x, AIR_IntLike:$y); + let results = (outs AIR_IntLike:$result); + let assemblyFormat = "$x `,` $y attr-dict `:` type($result)"; +} + +class AIR_IntTernaryOp : AIR_ElementwiseOp { + let arguments = (ins AIR_IntLike:$x, AIR_IntLike:$y, AIR_IntLike:$z); + let results = (outs AIR_IntLike:$result); + let assemblyFormat = "$x `,` $y `,` $z attr-dict `:` type($result)"; +} + +// `exp` is `metal::precise::exp`; `fast_exp` is `metal::fast::exp`, which +// `exp` selects under fast math. Only f32 has fast variants. +def AIR_ExpOp : AIR_FloatUnaryOp<"exp">; +def AIR_FastExpOp : AIR_FloatUnaryOp<"fast_exp", AIR_F32Like>; +def AIR_Exp2Op : AIR_FloatUnaryOp<"exp2">; +def AIR_FastExp2Op : AIR_FloatUnaryOp<"fast_exp2", AIR_F32Like>; +def AIR_LogOp : AIR_FloatUnaryOp<"log">; +def AIR_FastLogOp : AIR_FloatUnaryOp<"fast_log", AIR_F32Like>; +def AIR_Log2Op : AIR_FloatUnaryOp<"log2">; +def AIR_FastLog2Op : AIR_FloatUnaryOp<"fast_log2", AIR_F32Like>; +def AIR_TanhOp : AIR_FloatUnaryOp<"tanh">; +def AIR_FastTanhOp : AIR_FloatUnaryOp<"fast_tanh", AIR_F32Like>; +def AIR_SqrtOp : AIR_FloatUnaryOp<"sqrt">; +def AIR_FastSqrtOp : AIR_FloatUnaryOp<"fast_sqrt", AIR_F32Like>; +def AIR_RsqrtOp : AIR_FloatUnaryOp<"rsqrt">; +def AIR_FastRsqrtOp : AIR_FloatUnaryOp<"fast_rsqrt", AIR_F32Like>; +def AIR_FabsOp : AIR_FloatUnaryOp<"fabs">; +def AIR_FastFabsOp : AIR_FloatUnaryOp<"fast_fabs", AIR_F32Like>; +def AIR_CeilOp : AIR_FloatUnaryOp<"ceil">; +def AIR_FastCeilOp : AIR_FloatUnaryOp<"fast_ceil", AIR_F32Like>; +def AIR_FloorOp : AIR_FloatUnaryOp<"floor">; +def AIR_FastFloorOp : AIR_FloatUnaryOp<"fast_floor", AIR_F32Like>; +def AIR_RoundOp : AIR_FloatUnaryOp<"round">; +def AIR_FastRoundOp : AIR_FloatUnaryOp<"fast_round", AIR_F32Like>; +def AIR_TruncOp : AIR_FloatUnaryOp<"trunc">; +def AIR_FastTruncOp : AIR_FloatUnaryOp<"fast_trunc", AIR_F32Like>; +def AIR_SinOp : AIR_FloatUnaryOp<"sin">; +def AIR_FastSinOp : AIR_FloatUnaryOp<"fast_sin", AIR_F32Like>; +def AIR_CosOp : AIR_FloatUnaryOp<"cos">; +def AIR_FastCosOp : AIR_FloatUnaryOp<"fast_cos", AIR_F32Like>; + +def AIR_FmaxOp : AIR_FloatBinaryOp<"fmax">; +def AIR_FastFmaxOp : AIR_FloatBinaryOp<"fast_fmax", AIR_F32Like>; +def AIR_FminOp : AIR_FloatBinaryOp<"fmin">; +def AIR_FastFminOp : AIR_FloatBinaryOp<"fast_fmin", AIR_F32Like>; +def AIR_PowOp : AIR_FloatBinaryOp<"pow">; +def AIR_FastPowOp : AIR_FloatBinaryOp<"fast_pow", AIR_F32Like>; +def AIR_FmodOp : AIR_FloatBinaryOp<"fmod">; +def AIR_FastFmodOp : AIR_FloatBinaryOp<"fast_fmod", AIR_F32Like>; + +def AIR_FmaOp : AIR_FloatTernaryOp<"fma">; +def AIR_ClampOp : AIR_FloatTernaryOp<"clamp">; +def AIR_FastClampOp : AIR_FloatTernaryOp<"fast_clamp", AIR_F32Like>; + +def AIR_DotOp : AIR_Op<"dot", [Pure, AllTypesMatch<["x", "y"]>, + DeclareOpInterfaceMethods]> { + let summary = "Dot product of two float vectors"; + let description = "Calls `@air.dot.`."; + let arguments = (ins AIR_VectorOf<[F16, F32]>:$x, AIR_VectorOf<[F16, F32]>:$y, + AIR_FastMath:$fastmath); + let results = (outs AnyTypeOf<[F16, F32]>:$result); + let assemblyFormat = [{ + $x `,` $y (`fastmath` `` $fastmath^)? attr-dict `:` type($x) `->` + type($result) + }]; + let hasVerifier = 1; +} + +// `.s` and `.u` are the signed and unsigned overloads of MSL's integer +// functions. +def AIR_MaxSOp : AIR_IntBinaryOp<"max.s">; +def AIR_MaxUOp : AIR_IntBinaryOp<"max.u">; +def AIR_MinSOp : AIR_IntBinaryOp<"min.s">; +def AIR_MinUOp : AIR_IntBinaryOp<"min.u">; +def AIR_AbsSOp : AIR_IntUnaryOp<"abs.s">; +def AIR_ClampSOp : AIR_IntTernaryOp<"clamp.s">; +def AIR_ClampUOp : AIR_IntTernaryOp<"clamp.u">; + +//===----------------------------------------------------------------------===// +// Conversions +//===----------------------------------------------------------------------===// + +class AIR_ConvertBaseOp + : AIR_Op]> { + let summary = summaryText; + let description = [{ + Calls `@air.convert....`, + where floats are `f` and the integer side is `s` or `u` as the mnemonic + says. Scalar f32 to and from f16 or bf16, and integer to integer, are + LLVM casts in Apple's compiler. + }]; + let arguments = (ins AIR_ConvertLike:$x, AIR_FastMath:$fastmath); + let results = (outs AIR_ConvertLike:$result); + let assemblyFormat = [{ + $x (`fastmath` `` $fastmath^)? attr-dict `:` type($x) `to` type($result) + }]; + let hasVerifier = 1; +} + +def AIR_ConvertOp : AIR_ConvertBaseOp<"convert", "Float to float conversion">; +def AIR_ConvertSOp : AIR_ConvertBaseOp<"convert.s", + "Conversion between floats and signed integers">; +def AIR_ConvertUOp : AIR_ConvertBaseOp<"convert.u", + "Conversion between floats and unsigned integers">; + +//===----------------------------------------------------------------------===// +// SIMD-group functions +//===----------------------------------------------------------------------===// + +// SIMD-group functions read the other lanes, so they are not Pure: unknown +// effects keep them from being speculated or moved across divergent control. +class AIR_SimdReduceOp traits = []> + : AIR_Op { + let summary = "`" # mnemonic # "` across the SIMD-group"; + let description = "Calls `@air." # mnemonic # ".`."; + let results = (outs type:$result); +} + +class AIR_SimdFloatReduceOp + : AIR_SimdReduceOp]> { + let arguments = (ins AIR_FloatLike:$x, AIR_FastMath:$fastmath); + let assemblyFormat = + "$x (`fastmath` `` $fastmath^)? attr-dict `:` type($result)"; +} + +class AIR_SimdIntReduceOp + : AIR_SimdReduceOp { + let arguments = (ins AIR_SimdIntLike:$x); + let assemblyFormat = "$x attr-dict `:` type($result)"; +} + +class AIR_SimdShuffleOp traits = []> + : AIR_Op]> { + let summary = "`" # mnemonic # "` across the SIMD-group"; + let description = [{ + Calls `@air.}] # mnemonic # [{.`. `lane` is the source lane, the + XOR mask or the lane delta, as for the MSL function. + }]; + let results = (outs type:$result); +} + +class AIR_SimdFloatShuffleOp + : AIR_SimdShuffleOp]> { + let arguments = (ins AIR_FloatLike:$data, I16:$lane, AIR_FastMath:$fastmath); + let assemblyFormat = [{ + $data `,` $lane (`fastmath` `` $fastmath^)? attr-dict `:` type($result) + }]; +} + +class AIR_SimdIntShuffleOp + : AIR_SimdShuffleOp { + let arguments = (ins AIR_SimdIntLike:$data, I16:$lane); + let assemblyFormat = "$data `,` $lane attr-dict `:` type($result)"; +} + +def AIR_SimdSumOp : AIR_SimdFloatReduceOp<"simd_sum">; +def AIR_SimdSumSOp : AIR_SimdIntReduceOp<"simd_sum.s">; +def AIR_SimdSumUOp : AIR_SimdIntReduceOp<"simd_sum.u">; +def AIR_SimdMaxOp : AIR_SimdFloatReduceOp<"simd_max">; +def AIR_SimdMaxSOp : AIR_SimdIntReduceOp<"simd_max.s">; +def AIR_SimdMaxUOp : AIR_SimdIntReduceOp<"simd_max.u">; +def AIR_SimdMinOp : AIR_SimdFloatReduceOp<"simd_min">; +def AIR_SimdMinSOp : AIR_SimdIntReduceOp<"simd_min.s">; +def AIR_SimdMinUOp : AIR_SimdIntReduceOp<"simd_min.u">; +def AIR_SimdPrefixExclusiveSumOp + : AIR_SimdFloatReduceOp<"simd_prefix_exclusive_sum">; + +def AIR_SimdShuffleOp : AIR_SimdFloatShuffleOp<"simd_shuffle">; +def AIR_SimdShuffleSOp : AIR_SimdIntShuffleOp<"simd_shuffle.s">; +def AIR_SimdShuffleUOp : AIR_SimdIntShuffleOp<"simd_shuffle.u">; +def AIR_SimdShuffleXorOp : AIR_SimdFloatShuffleOp<"simd_shuffle_xor">; +def AIR_SimdShuffleXorSOp : AIR_SimdIntShuffleOp<"simd_shuffle_xor.s">; +def AIR_SimdShuffleXorUOp : AIR_SimdIntShuffleOp<"simd_shuffle_xor.u">; +def AIR_SimdShuffleDownOp : AIR_SimdFloatShuffleOp<"simd_shuffle_down">; +def AIR_SimdShuffleDownSOp : AIR_SimdIntShuffleOp<"simd_shuffle_down.s">; +def AIR_SimdShuffleDownUOp : AIR_SimdIntShuffleOp<"simd_shuffle_down.u">; +def AIR_SimdBroadcastOp : AIR_SimdFloatShuffleOp<"simd_broadcast">; +def AIR_SimdBroadcastSOp : AIR_SimdIntShuffleOp<"simd_broadcast.s">; +def AIR_SimdBroadcastUOp : AIR_SimdIntShuffleOp<"simd_broadcast.u">; + +class AIR_SimdVoteOp : AIR_Op { + let summary = summaryText; + let description = "Calls `@air." # mnemonic # "`."; + let arguments = (ins I1:$x); + let results = (outs I1:$result); + let assemblyFormat = "$x attr-dict"; +} + +def AIR_SimdAnyOp : AIR_SimdVoteOp<"simd_any", + "Whether `x` is true in any active lane">; +def AIR_SimdAllOp : AIR_SimdVoteOp<"simd_all", + "Whether `x` is true in every active lane">; + +//===----------------------------------------------------------------------===// +// Barriers +//===----------------------------------------------------------------------===// + +def AIR_WgBarrierOp : AIR_Op<"wg.barrier"> { + let summary = "threadgroup_barrier"; + let description = "Calls `@air.wg.barrier(i32 mem_flags, i32 1)`."; + let arguments = (ins AIR_MemFlagsAttr:$mem_flags); + let assemblyFormat = "$mem_flags attr-dict"; +} + +def AIR_SimdgroupBarrierOp : AIR_Op<"simdgroup.barrier"> { + let summary = "simdgroup_barrier"; + let description = "Calls `@air.simdgroup.barrier(i32 mem_flags, i32 4)`."; + let arguments = (ins AIR_MemFlagsAttr:$mem_flags); + let assemblyFormat = "$mem_flags attr-dict"; +} + +//===----------------------------------------------------------------------===// +// SIMD-group matrices +//===----------------------------------------------------------------------===// + +// A simdgroup_matrix is a vector<64xT>; thread_elements() are its +// elements 0 and 1. +def AIR_SimdgroupMatrixLoadOp : AIR_Op<"simdgroup_matrix_8x8_load", [ + DeclareOpInterfaceMethods]> { + let summary = "simdgroup_load"; + let description = [{ + Calls `@air.simdgroup_matrix_8x8_load.v64.p
`. + `simdgroup_load(m, p, ld, origin, transpose)` passes `extents` (ld, 8) + and `strides` (1, ld), swapped when transposed. + }]; + let arguments = (ins AIR_MatrixPointer:$pointer, + AIR_MatrixVector:$extents, + AIR_MatrixVector:$strides, + AIR_MatrixVector:$origin, + AIR_FastMath:$fastmath); + let results = (outs AIR_Matrix:$result); + let assemblyFormat = [{ + $pointer `,` $extents `,` $strides `,` $origin (`fastmath` `` $fastmath^)? + attr-dict `:` qualified(type($pointer)) `->` type($result) + }]; +} + +def AIR_SimdgroupMatrixStoreOp : AIR_Op<"simdgroup_matrix_8x8_store"> { + let summary = "simdgroup_store"; + let description = [{ + Calls `@air.simdgroup_matrix_8x8_store.v64.p
`, with + the operands of `simdgroup_matrix_8x8_load`. + }]; + let arguments = (ins AIR_Matrix:$value, + AIR_MatrixPointer:$pointer, + AIR_MatrixVector:$extents, + AIR_MatrixVector:$strides, + AIR_MatrixVector:$origin); + let assemblyFormat = [{ + $value `,` $pointer `,` $extents `,` $strides `,` $origin attr-dict `:` + type($value) `,` qualified(type($pointer)) + }]; +} + +def AIR_SimdgroupMatrixMultiplyAccumulateOp + : AIR_Op<"simdgroup_matrix_8x8_multiply_accumulate", [ + DeclareOpInterfaceMethods]> { + let summary = "simdgroup_multiply_accumulate"; + let description = [{ + Calls `@air.simdgroup_matrix_8x8_multiply_accumulate.v64.v64.v64.v64`: + `result = a * b + c`. `simdgroup_multiply` passes a zero `c`. + }]; + let arguments = (ins AIR_Matrix:$a, AIR_Matrix:$b, AIR_Matrix:$c, + AIR_FastMath:$fastmath); + let results = (outs AIR_Matrix:$result); + let assemblyFormat = [{ + $a `,` $b `,` $c (`fastmath` `` $fastmath^)? attr-dict `:` type($a) `,` + type($b) `,` type($c) `->` type($result) + }]; + let hasVerifier = 1; +} + +#endif // MLIR_DIALECTS_AIR_AIROPS_TD diff --git a/mlir/dialects/air/BUILD.bazel b/mlir/dialects/air/BUILD.bazel new file mode 100644 index 00000000..9fb6ed6c --- /dev/null +++ b/mlir/dialects/air/BUILD.bazel @@ -0,0 +1,179 @@ +load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library", "td_library") +load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +package(default_visibility = ["//visibility:public"]) + +cc_library( + name = "capi", + srcs = ["capi.cc"], + hdrs = ["capi.h"], + tags = ["manual"], + deps = [ + ":dialect", + ":headers", + "@llvm-project//mlir:CAPIIR", + ], +) + +zig_library( + name = "air", + import_name = "mlir/dialects/air", + main = "air.zig", + tags = ["manual"], + deps = [ + ":capi", + "//mlir", + "//mlir/dialects/arith", + "//mlir/dialects/func", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":air", + "//mlir", + "//mlir/dialects/arith", + "//mlir/dialects/func", + "//mlir/dialects/llvm", + ], +) + +td_library( + name = "td_files", + srcs = [ + "AIRAttributes.td", + "AIRDialect.td", + "AIREnums.td", + "AIROps.td", + ], + includes = ["/"], + deps = [ + "@llvm-project//mlir:ArithOpsInterfacesTdFiles", + "@llvm-project//mlir:ArithOpsTdFiles", + "@llvm-project//mlir:InferTypeOpInterfaceTdFiles", + "@llvm-project//mlir:LLVMOpsTdFiles", + "@llvm-project//mlir:OpBaseTdFiles", + "@llvm-project//mlir:SideEffectInterfacesTdFiles", + ], +) + +gentbl_cc_library( + name = "dialect_inc_gen", + tbl_outs = [ + ( + [ + "-gen-dialect-decls", + "-dialect=air", + ], + "AIRDialect.h.inc", + ), + ( + [ + "-gen-dialect-defs", + "-dialect=air", + ], + "AIRDialect.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "AIRDialect.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "enums_inc_gen", + tbl_outs = [ + ( + ["-gen-enum-decls"], + "AIREnums.h.inc", + ), + ( + ["-gen-enum-defs"], + "AIREnums.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "AIREnums.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "attrs_inc_gen", + tbl_outs = [ + ( + [ + "-gen-attrdef-decls", + "-attrdefs-dialect=air", + ], + "AIRAttributes.h.inc", + ), + ( + [ + "-gen-attrdef-defs", + "-attrdefs-dialect=air", + ], + "AIRAttributes.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "AIRAttributes.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "ops_inc_gen", + tbl_outs = [ + ( + ["-gen-op-decls"], + "AIROps.h.inc", + ), + ( + ["-gen-op-defs"], + "AIROps.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "AIROps.td", + deps = [":td_files"], +) + +cc_library( + name = "headers", + hdrs = [ + "AIRAttributes.h", + "AIRDialect.h", + "AIREnums.h", + "AIROps.h", + ], + deps = [ + ":attrs_inc_gen", + ":dialect_inc_gen", + ":enums_inc_gen", + ":ops_inc_gen", + "@llvm-project//mlir:ArithDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:InferTypeOpInterface", + "@llvm-project//mlir:LLVMDialect", + "@llvm-project//mlir:SideEffectInterfaces", + ], +) + +cc_library( + name = "dialect", + srcs = [ + "AIRAttributes.cpp", + "AIRDialect.cpp", + "AIROps.cpp", + ], + deps = [ + ":headers", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:ArithDialect", + "@llvm-project//mlir:FunctionInterfaces", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:LLVMDialect", + ], +) diff --git a/mlir/dialects/air/README.md b/mlir/dialects/air/README.md new file mode 100644 index 00000000..cd8a0531 --- /dev/null +++ b/mlir/dialects/air/README.md @@ -0,0 +1,90 @@ +# Metal AIR dialect + +The Metal-specific part of AIR, the LLVM IR Apple's Metal compiler emits (not +AMD's asynchronous AIR dialect). Namespace `air`, C++ namespace +`mlir::metal::air`, C API prefix `mlirMetalAIR`. `dsl/metal` emits it, and +XLA's `__gpu$xla.gpu.air` custom call lowers it, from a vendored copy, to a +metallib. + +## Operations + +Each operation is an `air.*` intrinsic, named as Apple's compiler names it +minus the type suffix its operand and result types determine: + +| Operation | Intrinsic | +|---|---| +| `air.fast_exp %x : vector<2xf32>` | `@air.fast_exp.v2f32` | +| `air.max.u %a, %b : i32` | `@air.max.u.i32` | +| `air.simd_shuffle_xor %x, %lane : f16` | `@air.simd_shuffle_xor.f16` | +| `air.convert.s %x : i32 to f32` | `@air.convert.f.f32.s.i32` | +| `air.simdgroup_matrix_8x8_load %p, ... : !llvm.ptr<3> -> vector<64xbf16>` | `@air.simdgroup_matrix_8x8_load.v64bf16.p3bf16` | +| `air.wg.barrier ` | `@air.wg.barrier(i32 2, i32 1)` | + +- Float math: `exp`, `exp2`, `log`, `log2`, `tanh`, `sqrt`, `rsqrt`, `fabs`, + `ceil`, `floor`, `round`, `trunc`, `sin`, `cos`, `fmax`, `fmin`, `pow`, + `fmod`, `fma`, `clamp` and `dot` on f16 and f32. The `fast_` variants are + f32 only: under fast math (MSL's default) `exp(x)` is `fast_exp`, and + `metal::precise::exp` is `exp`. +- Integer math: `max`, `min`, `clamp` and `abs`, as `.s` and `.u` + overloads. +- SIMD-group functions: `simd_sum`, `simd_max`, `simd_min`, + `simd_prefix_exclusive_sum`, `simd_shuffle`, `simd_shuffle_xor`, + `simd_shuffle_down` and `simd_broadcast` on floats, the integer ones as + `.s` and `.u`, and `simd_any`, `simd_all`. There are no bfloat overloads. +- `convert` between float types, and `convert.s`, `convert.u` between floats + and integers. Apple's compiler uses LLVM casts for scalar f32 to and from + f16 or bf16, and between integers. +- `wg.barrier` and `simdgroup.barrier` (`threadgroup_barrier`, + `simdgroup_barrier`), with `#air.mem_flags`. +- `simdgroup_matrix_8x8_load`, `_store` and `_multiply_accumulate`. A + `simdgroup_matrix` is a `vector<64xT>`, and `thread_elements()` its + elements 0 and 1. + +Float operations carry `fastmath` flags, which become the call's flags (Apple's +compiler marks every float call `fast` under fast math). SIMD-group, barrier +and matrix operations have unknown effects, so they are not speculated or +moved across divergent control flow. Arithmetic, memory and control flow use +the arith, cf, scf, vector and llvm dialects, with pointers as `!llvm.ptr` in +AIR's address spaces: 0 thread, 1 device, 2 constant, 3 threadgroup (a buffer +is never in thread memory). + +## Kernels + +A kernel is a void `func.func` with `air.kernel`. Each argument carries one +`!air.kernel` descriptor: + +- `air.builtin = #air.builtin`: the position and + size builtins are `uint` or `ushort` scalars or 2- and 3-element vectors, + the index builtins (`thread_index_in_simdgroup`, ...) scalars. +- `air.buffer = #air.buffer<...>` on an `!llvm.ptr`: the `[[buffer(n)]]` or + `[[threadgroup(n)]]` index, the address space, the access (`const` pointers + are `read`), the pointee type with its Metal name, size and alignment, the + size of a constant reference, `noalias` for `__restrict__`, and the fields + of a constant struct (`air.struct_type_info`). Device and constant buffers + share indices; threadgroup buffers have their own. +- `air.arg_name`, optionally, the argument's name. + +`air.fast_math_enable` on the module records that it was compiled with fast +math (`air.compile.fast_math_enable`). + +## Evidence + +Names, signatures, argument descriptors and flags follow what Apple's Metal +compiler 32023.830 (the `@metal_toolchain` repository) emits for MSL probes and +for XLA's Metal kernels, with `metal -std=metal4.0 -S -emit-llvm` (fast math, +-O2, as XLA compiles them). Only the overloads it emits are accepted. + +## Build and test + +```sh +bazel build //mlir/dialects/air //mlir/dialects/air:capi +bazel test //mlir/dialects/air:test +``` + +The Zig library is `:air`, the C++ dialect `:dialect` and its C API `:capi`. +`registerDialect(registry)` registers AIR, which loads the LLVM dialect; +`kernel` builds a `func.func`, so callers also register `func`. Attributes are +built through the C API (`BuiltinAttr`, `MemFlagsAttr`, `StructFieldAttr`, +`BufferAttr`), kernel arguments through `builtinArgumentAttr` and +`bufferArgumentAttr`, and every operation has a builder named after it +(`fast_exp`, `max_u`, `wg_barrier`, `simdgroup_matrix_8x8_load`). diff --git a/mlir/dialects/air/air.zig b/mlir/dialects/air/air.zig new file mode 100644 index 00000000..2cdf2d25 --- /dev/null +++ b/mlir/dialects/air/air.zig @@ -0,0 +1,536 @@ +//! Metal AIR intrinsics and kernel ABI annotations. A builder emits the +//! `air.*` intrinsic it names, minus the type suffix its types select: +//! `fast_exp` on f32 is `@air.fast_exp.f32`, `max_u` on `vector<2xi32>` is +//! `@air.max.u.v2i32`. Arithmetic, memory and control flow use the arith, +//! llvm and scf dialects, with pointers as `!llvm.ptr` in AIR's address spaces. +const std = @import("std"); + +const arith = @import("mlir/dialects/arith"); +const c = @import("c"); +const func = @import("mlir/dialects/func"); +const mlir = @import("mlir"); + +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("air"); +} + +pub const Builtin = enum(u32) { + thread_position_in_grid, + threadgroup_position_in_grid, + thread_position_in_threadgroup, + threads_per_threadgroup, + threadgroups_per_grid, + threads_per_grid, + thread_index_in_simdgroup, + simdgroup_index_in_threadgroup, + threads_per_simdgroup, + simdgroups_per_threadgroup, + thread_index_in_threadgroup, +}; + +pub const AddressSpace = enum(u32) { thread = 0, device = 1, constant = 2, threadgroup = 3 }; + +pub const Access = enum(u32) { read, read_write }; + +pub const MemFlags = packed struct(u32) { + device: bool = false, + threadgroup: bool = false, + texture: bool = false, + _padding: u29 = 0, + + pub const none: MemFlags = .{}; +}; + +/// `#air.builtin`: Kernel argument builtin +pub const BuiltinAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsAMetalAIRBuiltin; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { value: Builtin }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirMetalAIRBuiltinAttrGet(ctx.ptr(), @backingInt(args.value)).ptr orelse return error.InvalidMlir); + } + + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + + pub fn value(self: *const Self) Builtin { + return @fromBackingInt(@intCast(c.mlirMetalAIRBuiltinAttrGetValue(self.ptr()))); + } +}; + +/// `#air.mem_flags`: Memory a barrier orders +pub const MemFlagsAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsAMetalAIRMemFlags; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { value: MemFlags }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirMetalAIRMemFlagsAttrGet(ctx.ptr(), @bitCast(args.value)).ptr orelse return error.InvalidMlir); + } + + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + + pub fn value(self: *const Self) MemFlags { + return @bitCast(c.mlirMetalAIRMemFlagsAttrGetValue(self.ptr())); + } +}; + +/// `#air.struct_field`: Field of a constant buffer struct +pub const StructFieldAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsAMetalAIRStructField; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + offset: u64, + size: u64, + /// The array length, zero for a non-array field. + count: u64 = 0, + typeName: []const u8, + name: []const u8, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirMetalAIRStructFieldAttrGet(ctx.ptr(), args.offset, args.size, args.count, mlir.stringRef(args.typeName), mlir.stringRef(args.name)).ptr orelse return error.InvalidMlir); + } + + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + + pub fn offset(self: *const Self) u64 { + return c.mlirMetalAIRStructFieldAttrGetOffset(self.ptr()); + } + + pub fn size(self: *const Self) u64 { + return c.mlirMetalAIRStructFieldAttrGetSize(self.ptr()); + } + + pub fn count(self: *const Self) u64 { + return c.mlirMetalAIRStructFieldAttrGetCount(self.ptr()); + } + + pub fn typeName(self: *const Self) []const u8 { + return mlir.string(c.mlirMetalAIRStructFieldAttrGetTypeName(self.ptr())); + } + + pub fn name(self: *const Self) []const u8 { + return mlir.string(c.mlirMetalAIRStructFieldAttrGetName(self.ptr())); + } +}; + +/// `#air.buffer`: Kernel buffer argument +pub const BufferAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsAMetalAIRBuffer; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + locationIndex: u32, + addressSpace: AddressSpace, + access: Access, + elementType: *const mlir.Type, + typeName: []const u8, + typeSize: u64, + typeAlignment: u64, + /// The size of a constant reference; zero omits it. + bufferSize: u64 = 0, + noalias_: bool = false, + fields: []const *const StructFieldAttr = &.{}, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirMetalAIRBufferAttrGet( + ctx.ptr(), + args.locationIndex, + @backingInt(args.addressSpace), + @backingInt(args.access), + args.elementType.ptr(), + mlir.stringRef(args.typeName), + args.typeSize, + args.typeAlignment, + args.bufferSize, + args.noalias_, + @intCast(args.fields.len), + @ptrCast(args.fields.ptr), + ).ptr orelse return error.InvalidMlir); + } + + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + + pub fn locationIndex(self: *const Self) u32 { + return c.mlirMetalAIRBufferAttrGetLocationIndex(self.ptr()); + } + + pub fn addressSpace(self: *const Self) AddressSpace { + return @fromBackingInt(@intCast(c.mlirMetalAIRBufferAttrGetAddressSpace(self.ptr()))); + } + + pub fn access(self: *const Self) Access { + return @fromBackingInt(@intCast(c.mlirMetalAIRBufferAttrGetAccess(self.ptr()))); + } + + pub fn elementType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirMetalAIRBufferAttrGetElementType(self.ptr()).ptr); + } + + pub fn typeName(self: *const Self) []const u8 { + return mlir.string(c.mlirMetalAIRBufferAttrGetTypeName(self.ptr())); + } + + pub fn typeSize(self: *const Self) u64 { + return c.mlirMetalAIRBufferAttrGetTypeSize(self.ptr()); + } + + pub fn typeAlignment(self: *const Self) u64 { + return c.mlirMetalAIRBufferAttrGetTypeAlignment(self.ptr()); + } + + pub fn bufferSize(self: *const Self) u64 { + return c.mlirMetalAIRBufferAttrGetBufferSize(self.ptr()); + } + + pub fn noalias_(self: *const Self) bool { + return c.mlirMetalAIRBufferAttrGetNoalias(self.ptr()); + } + + pub fn numFields(self: *const Self) usize { + return @intCast(c.mlirMetalAIRBufferAttrGetNumFields(self.ptr())); + } + + pub fn field(self: *const Self, i: usize) *const StructFieldAttr { + return @ptrCast(c.mlirMetalAIRBufferAttrGetField(self.ptr(), @intCast(i)).ptr); + } +}; + +pub const ArgumentOpts = struct { + /// `air.arg_name`, the argument's name in reflection. + name: ?[]const u8 = null, +}; + +/// The argument dictionary of a kernel argument bound to `builtin`. +pub fn builtinArgumentAttr(ctx: *mlir.Context, builtin: Builtin, opts: ArgumentOpts) mlir.Error!*const mlir.Attribute { + return argumentAttr(ctx, .named(ctx, "air.builtin", (try BuiltinAttr.get(ctx, .{ .value = builtin })).attribute()), opts); +} + +/// The argument dictionary of a kernel buffer argument. +pub fn bufferArgumentAttr(ctx: *mlir.Context, args: BufferAttr.InitArgs, opts: ArgumentOpts) mlir.Error!*const mlir.Attribute { + return argumentAttr(ctx, .named(ctx, "air.buffer", (try BufferAttr.get(ctx, args)).attribute()), opts); +} + +fn argumentAttr(ctx: *mlir.Context, annotation: mlir.NamedAttribute, opts: ArgumentOpts) *const mlir.Attribute { + if (opts.name) |name| return .dict(ctx, &.{ annotation, .named(ctx, "air.arg_name", .string(ctx, name)) }); + return .dict(ctx, &.{annotation}); +} + +/// Marks `module` as compiled with fast math (`air.compile.fast_math_enable`). +pub fn enableFastMath(ctx: *mlir.Context, module: *mlir.Operation) void { + module.setAttributeByName("air.fast_math_enable", .unit(ctx)); +} + +pub const KernelArgs = struct { + name: []const u8, + block: *mlir.Block, + args_attributes: []const *const mlir.Attribute, + location: *const mlir.Location, + verify: bool = true, +}; + +/// A `func.func` with `air.kernel`, one argument dictionary per block +/// argument. Takes ownership of `block`; set `verify = false` while the body +/// is incomplete. +pub fn kernel(ctx: *mlir.Context, args: KernelArgs) *mlir.Operation { + std.debug.assert(args.args_attributes.len == args.block.numArguments()); + return func.func(ctx, .{ + .name = args.name, + .block = args.block, + .args_attributes = args.args_attributes, + .results = &.{}, + .extra_attributes = &.{.named(ctx, "air.kernel", .unit(ctx))}, + .location = args.location, + .verify = args.verify, + }); +} + +pub const FloatOpts = struct { + fastmath: arith.FastMathFlags = .none, +}; + +/// The `fastmath` attribute of a float op, stored in the caller's `buf`. +fn fastmathAttrs(buf: *[1]mlir.NamedAttribute, ctx: *mlir.Context, fastmath: arith.FastMathFlags) []const mlir.NamedAttribute { + if (fastmath == arith.FastMathFlags.none) return &.{}; + const attr = arith.FastMathAttr.get(ctx, .{ .value = fastmath }) catch unreachable; + buf[0] = .named(ctx, "fastmath", attr.attribute()); + return buf; +} + +fn floatUnary(comptime name: []const u8) type { + return struct { + pub fn call(ctx: *mlir.Context, x: *const mlir.Value, opts: FloatOpts, location: *const mlir.Location) *mlir.Operation { + var attrs: [1]mlir.NamedAttribute = undefined; + return mlir.Operation.make(ctx, name, .{ + .operands = .{ .flat = &.{x} }, + .results = .{ .flat = &.{x.type_()} }, + .attributes = fastmathAttrs(&attrs, ctx, opts.fastmath), + .location = location, + }); + } + }; +} + +fn floatBinary(comptime name: []const u8) type { + return struct { + pub fn call(ctx: *mlir.Context, x: *const mlir.Value, y: *const mlir.Value, opts: FloatOpts, location: *const mlir.Location) *mlir.Operation { + var attrs: [1]mlir.NamedAttribute = undefined; + return mlir.Operation.make(ctx, name, .{ + .operands = .{ .flat = &.{ x, y } }, + .results = .{ .flat = &.{x.type_()} }, + .attributes = fastmathAttrs(&attrs, ctx, opts.fastmath), + .location = location, + }); + } + }; +} + +fn floatTernary(comptime name: []const u8) type { + return struct { + pub fn call(ctx: *mlir.Context, x: *const mlir.Value, y: *const mlir.Value, z: *const mlir.Value, opts: FloatOpts, location: *const mlir.Location) *mlir.Operation { + var attrs: [1]mlir.NamedAttribute = undefined; + return mlir.Operation.make(ctx, name, .{ + .operands = .{ .flat = &.{ x, y, z } }, + .results = .{ .flat = &.{x.type_()} }, + .attributes = fastmathAttrs(&attrs, ctx, opts.fastmath), + .location = location, + }); + } + }; +} + +fn intUnary(comptime name: []const u8) type { + return struct { + pub fn call(ctx: *mlir.Context, x: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, name, .{ + .operands = .{ .flat = &.{x} }, + .results = .{ .flat = &.{x.type_()} }, + .location = location, + }); + } + }; +} + +fn intBinary(comptime name: []const u8) type { + return struct { + pub fn call(ctx: *mlir.Context, x: *const mlir.Value, y: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, name, .{ + .operands = .{ .flat = &.{ x, y } }, + .results = .{ .flat = &.{x.type_()} }, + .location = location, + }); + } + }; +} + +fn intTernary(comptime name: []const u8) type { + return struct { + pub fn call(ctx: *mlir.Context, x: *const mlir.Value, y: *const mlir.Value, z: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, name, .{ + .operands = .{ .flat = &.{ x, y, z } }, + .results = .{ .flat = &.{x.type_()} }, + .location = location, + }); + } + }; +} + +fn floatShuffle(comptime name: []const u8) type { + return struct { + pub fn call(ctx: *mlir.Context, data: *const mlir.Value, lane: *const mlir.Value, opts: FloatOpts, location: *const mlir.Location) *mlir.Operation { + var attrs: [1]mlir.NamedAttribute = undefined; + return mlir.Operation.make(ctx, name, .{ + .operands = .{ .flat = &.{ data, lane } }, + .results = .{ .flat = &.{data.type_()} }, + .attributes = fastmathAttrs(&attrs, ctx, opts.fastmath), + .location = location, + }); + } + }; +} + +fn intShuffle(comptime name: []const u8) type { + return struct { + pub fn call(ctx: *mlir.Context, data: *const mlir.Value, lane: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, name, .{ + .operands = .{ .flat = &.{ data, lane } }, + .results = .{ .flat = &.{data.type_()} }, + .location = location, + }); + } + }; +} + +fn convertOp(comptime name: []const u8) type { + return struct { + pub fn call(ctx: *mlir.Context, x: *const mlir.Value, result_type: *const mlir.Type, opts: FloatOpts, location: *const mlir.Location) *mlir.Operation { + var attrs: [1]mlir.NamedAttribute = undefined; + return mlir.Operation.make(ctx, name, .{ + .operands = .{ .flat = &.{x} }, + .results = .{ .flat = &.{result_type} }, + .attributes = fastmathAttrs(&attrs, ctx, opts.fastmath), + .location = location, + }); + } + }; +} + +fn barrier(comptime name: []const u8) type { + return struct { + pub fn call(ctx: *mlir.Context, mem_flags: MemFlags, location: *const mlir.Location) *mlir.Operation { + const flags = MemFlagsAttr.get(ctx, .{ .value = mem_flags }) catch unreachable; + return mlir.Operation.make(ctx, name, .{ + .attributes = &.{.named(ctx, "mem_flags", flags.attribute())}, + .location = location, + }); + } + }; +} + +pub const exp = floatUnary("air.exp").call; +pub const fast_exp = floatUnary("air.fast_exp").call; +pub const exp2 = floatUnary("air.exp2").call; +pub const fast_exp2 = floatUnary("air.fast_exp2").call; +pub const log = floatUnary("air.log").call; +pub const fast_log = floatUnary("air.fast_log").call; +pub const log2 = floatUnary("air.log2").call; +pub const fast_log2 = floatUnary("air.fast_log2").call; +pub const tanh = floatUnary("air.tanh").call; +pub const fast_tanh = floatUnary("air.fast_tanh").call; +pub const sqrt = floatUnary("air.sqrt").call; +pub const fast_sqrt = floatUnary("air.fast_sqrt").call; +pub const rsqrt = floatUnary("air.rsqrt").call; +pub const fast_rsqrt = floatUnary("air.fast_rsqrt").call; +pub const fabs = floatUnary("air.fabs").call; +pub const fast_fabs = floatUnary("air.fast_fabs").call; +pub const ceil = floatUnary("air.ceil").call; +pub const fast_ceil = floatUnary("air.fast_ceil").call; +pub const floor = floatUnary("air.floor").call; +pub const fast_floor = floatUnary("air.fast_floor").call; +pub const round = floatUnary("air.round").call; +pub const fast_round = floatUnary("air.fast_round").call; +pub const trunc = floatUnary("air.trunc").call; +pub const fast_trunc = floatUnary("air.fast_trunc").call; +pub const sin = floatUnary("air.sin").call; +pub const fast_sin = floatUnary("air.fast_sin").call; +pub const cos = floatUnary("air.cos").call; +pub const fast_cos = floatUnary("air.fast_cos").call; +pub const fmax = floatBinary("air.fmax").call; +pub const fast_fmax = floatBinary("air.fast_fmax").call; +pub const fmin = floatBinary("air.fmin").call; +pub const fast_fmin = floatBinary("air.fast_fmin").call; +pub const pow = floatBinary("air.pow").call; +pub const fast_pow = floatBinary("air.fast_pow").call; +pub const fmod = floatBinary("air.fmod").call; +pub const fast_fmod = floatBinary("air.fast_fmod").call; +pub const fma = floatTernary("air.fma").call; +pub const clamp = floatTernary("air.clamp").call; +pub const fast_clamp = floatTernary("air.fast_clamp").call; +pub const abs_s = intUnary("air.abs.s").call; +pub const max_s = intBinary("air.max.s").call; +pub const max_u = intBinary("air.max.u").call; +pub const min_s = intBinary("air.min.s").call; +pub const min_u = intBinary("air.min.u").call; +pub const clamp_s = intTernary("air.clamp.s").call; +pub const clamp_u = intTernary("air.clamp.u").call; +pub const simd_sum = floatUnary("air.simd_sum").call; +pub const simd_sum_s = intUnary("air.simd_sum.s").call; +pub const simd_sum_u = intUnary("air.simd_sum.u").call; +pub const simd_max = floatUnary("air.simd_max").call; +pub const simd_max_s = intUnary("air.simd_max.s").call; +pub const simd_max_u = intUnary("air.simd_max.u").call; +pub const simd_min = floatUnary("air.simd_min").call; +pub const simd_min_s = intUnary("air.simd_min.s").call; +pub const simd_min_u = intUnary("air.simd_min.u").call; +pub const simd_prefix_exclusive_sum = floatUnary("air.simd_prefix_exclusive_sum").call; +pub const simd_shuffle = floatShuffle("air.simd_shuffle").call; +pub const simd_shuffle_s = intShuffle("air.simd_shuffle.s").call; +pub const simd_shuffle_u = intShuffle("air.simd_shuffle.u").call; +pub const simd_shuffle_xor = floatShuffle("air.simd_shuffle_xor").call; +pub const simd_shuffle_xor_s = intShuffle("air.simd_shuffle_xor.s").call; +pub const simd_shuffle_xor_u = intShuffle("air.simd_shuffle_xor.u").call; +pub const simd_shuffle_down = floatShuffle("air.simd_shuffle_down").call; +pub const simd_shuffle_down_s = intShuffle("air.simd_shuffle_down.s").call; +pub const simd_shuffle_down_u = intShuffle("air.simd_shuffle_down.u").call; +pub const simd_broadcast = floatShuffle("air.simd_broadcast").call; +pub const simd_broadcast_s = intShuffle("air.simd_broadcast.s").call; +pub const simd_broadcast_u = intShuffle("air.simd_broadcast.u").call; +pub const simd_any = intUnary("air.simd_any").call; +pub const simd_all = intUnary("air.simd_all").call; +pub const convert = convertOp("air.convert").call; +pub const convert_s = convertOp("air.convert.s").call; +pub const convert_u = convertOp("air.convert.u").call; +pub const wg_barrier = barrier("air.wg.barrier").call; +pub const simdgroup_barrier = barrier("air.simdgroup.barrier").call; + +pub fn dot(ctx: *mlir.Context, x: *const mlir.Value, y: *const mlir.Value, opts: FloatOpts, location: *const mlir.Location) *mlir.Operation { + var attrs: [1]mlir.NamedAttribute = undefined; + return mlir.Operation.make(ctx, "air.dot", .{ + .operands = .{ .flat = &.{ x, y } }, + .results = .{ .flat = &.{x.type_().isA(mlir.ShapedType).?.elementType()} }, + .attributes = fastmathAttrs(&attrs, ctx, opts.fastmath), + .location = location, + }); +} + +/// `simdgroup_load`: `extents`, `strides` and `origin` are `vector<2xi64>`. +pub fn simdgroup_matrix_8x8_load(ctx: *mlir.Context, pointer: *const mlir.Value, extents: *const mlir.Value, strides: *const mlir.Value, origin: *const mlir.Value, result_type: *const mlir.Type, opts: FloatOpts, location: *const mlir.Location) *mlir.Operation { + var attrs: [1]mlir.NamedAttribute = undefined; + return mlir.Operation.make(ctx, "air.simdgroup_matrix_8x8_load", .{ + .operands = .{ .flat = &.{ pointer, extents, strides, origin } }, + .results = .{ .flat = &.{result_type} }, + .attributes = fastmathAttrs(&attrs, ctx, opts.fastmath), + .location = location, + }); +} + +pub fn simdgroup_matrix_8x8_store(ctx: *mlir.Context, value: *const mlir.Value, pointer: *const mlir.Value, extents: *const mlir.Value, strides: *const mlir.Value, origin: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "air.simdgroup_matrix_8x8_store", .{ + .operands = .{ .flat = &.{ value, pointer, extents, strides, origin } }, + .location = location, + }); +} + +/// `simdgroup_multiply_accumulate`: `a * b + c`. +pub fn simdgroup_matrix_8x8_multiply_accumulate(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, c_: *const mlir.Value, result_type: *const mlir.Type, opts: FloatOpts, location: *const mlir.Location) *mlir.Operation { + var attrs: [1]mlir.NamedAttribute = undefined; + return mlir.Operation.make(ctx, "air.simdgroup_matrix_8x8_multiply_accumulate", .{ + .operands = .{ .flat = &.{ a, b, c_ } }, + .results = .{ .flat = &.{result_type} }, + .attributes = fastmathAttrs(&attrs, ctx, opts.fastmath), + .location = location, + }); +} diff --git a/mlir/dialects/air/capi.cc b/mlir/dialects/air/capi.cc new file mode 100644 index 00000000..64df5d20 --- /dev/null +++ b/mlir/dialects/air/capi.cc @@ -0,0 +1,189 @@ +//===- capi.cc - C API for the Metal AIR dialect --------------------------===// + +#include "capi.h" + +#include "mlir/dialects/air/AIRAttributes.h" +#include "mlir/dialects/air/AIRDialect.h" + +#include "mlir/CAPI/IR.h" +#include "mlir/CAPI/Registration.h" +#include "mlir/CAPI/Support.h" +#include "mlir/IR/Location.h" +#include "llvm/ADT/SmallVector.h" + +#include + +using namespace mlir; +using namespace mlir::metal::air; + +// Registration + +MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(MetalAIR, air, AIRDialect) + +static MLIRContext *loadAIR(MlirContext context) { + MLIRContext *ctx = unwrap(context); + if (ctx) + ctx->getOrLoadDialect(); + return ctx; +} + +static MlirAttribute invalid(MLIRContext *ctx, const Twine &message) { + emitError(UnknownLoc::get(ctx), message); + return {nullptr}; +} + +// Attributes + +bool mlirAttributeIsAMetalAIRBuiltin(MlirAttribute attr) { + return isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirMetalAIRBuiltinAttrGet(MlirContext context, uint32_t value) { + MLIRContext *ctx = loadAIR(context); + if (!ctx) + return {nullptr}; + std::optional builtin = symbolizeBuiltin(value); + if (!builtin) + return invalid(ctx, "unknown AIR builtin " + Twine(value)); + return wrap(BuiltinAttr::get(ctx, *builtin)); +} + +uint32_t mlirMetalAIRBuiltinAttrGetValue(MlirAttribute attr) { + return static_cast(cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsAMetalAIRMemFlags(MlirAttribute attr) { + return isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirMetalAIRMemFlagsAttrGet(MlirContext context, uint32_t value) { + MLIRContext *ctx = loadAIR(context); + if (!ctx) + return {nullptr}; + std::optional flags = symbolizeMemFlags(value); + if (!flags) + return invalid(ctx, "unknown AIR memory flags " + Twine(value)); + return wrap(MemFlagsAttr::get(ctx, *flags)); +} + +uint32_t mlirMetalAIRMemFlagsAttrGetValue(MlirAttribute attr) { + return static_cast(cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsAMetalAIRStructField(MlirAttribute attr) { + return isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirMetalAIRStructFieldAttrGet(MlirContext context, + uint64_t offset, uint64_t size, + uint64_t count, + MlirStringRef typeName, + MlirStringRef name) { + MLIRContext *ctx = loadAIR(context); + if (!ctx) + return {nullptr}; + return wrap(StructFieldAttr::getChecked(UnknownLoc::get(ctx), ctx, offset, + size, count, unwrap(typeName), + unwrap(name))); +} + +uint64_t mlirMetalAIRStructFieldAttrGetOffset(MlirAttribute attr) { + return cast(unwrap(attr)).getOffset(); +} + +uint64_t mlirMetalAIRStructFieldAttrGetSize(MlirAttribute attr) { + return cast(unwrap(attr)).getSize(); +} + +uint64_t mlirMetalAIRStructFieldAttrGetCount(MlirAttribute attr) { + return cast(unwrap(attr)).getCount(); +} + +MlirStringRef mlirMetalAIRStructFieldAttrGetTypeName(MlirAttribute attr) { + return wrap(cast(unwrap(attr)).getTypeName()); +} + +MlirStringRef mlirMetalAIRStructFieldAttrGetName(MlirAttribute attr) { + return wrap(cast(unwrap(attr)).getName()); +} + +bool mlirAttributeIsAMetalAIRBuffer(MlirAttribute attr) { + return isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirMetalAIRBufferAttrGet( + MlirContext context, uint32_t locationIndex, uint32_t addressSpace, + uint32_t access, MlirType elementType, MlirStringRef typeName, + uint64_t typeSize, uint64_t typeAlignment, uint64_t bufferSize, + bool noalias, intptr_t numFields, MlirAttribute const *fields) { + MLIRContext *ctx = loadAIR(context); + if (!ctx) + return {nullptr}; + std::optional space = symbolizeAddressSpace(addressSpace); + std::optional mode = symbolizeAccess(access); + if (!space || !mode) + return invalid(ctx, "unknown AIR buffer address space or access"); + Type element = unwrap(elementType); + if (!element || element.getContext() != ctx) + return invalid(ctx, + "buffer element type must belong to the attribute context"); + if (numFields < 0 || (numFields > 0 && !fields)) + return invalid(ctx, "invalid AIR struct fields"); + SmallVector structFields; + for (intptr_t i = 0; i < numFields; ++i) { + auto field = dyn_cast_or_null(unwrap(fields[i])); + if (!field || field.getContext() != ctx) + return invalid(ctx, "AIR struct fields must be #air.struct_field of " + "the attribute context"); + structFields.push_back(field); + } + return wrap(BufferAttr::getChecked( + UnknownLoc::get(ctx), ctx, locationIndex, *space, *mode, element, + unwrap(typeName), typeSize, typeAlignment, bufferSize, noalias, + ArrayRef(structFields))); +} + +uint32_t mlirMetalAIRBufferAttrGetLocationIndex(MlirAttribute attr) { + return cast(unwrap(attr)).getLocationIndex(); +} + +uint32_t mlirMetalAIRBufferAttrGetAddressSpace(MlirAttribute attr) { + return static_cast( + cast(unwrap(attr)).getAddressSpace()); +} + +uint32_t mlirMetalAIRBufferAttrGetAccess(MlirAttribute attr) { + return static_cast(cast(unwrap(attr)).getAccess()); +} + +MlirType mlirMetalAIRBufferAttrGetElementType(MlirAttribute attr) { + return wrap(cast(unwrap(attr)).getElementType()); +} + +MlirStringRef mlirMetalAIRBufferAttrGetTypeName(MlirAttribute attr) { + return wrap(cast(unwrap(attr)).getTypeName()); +} + +uint64_t mlirMetalAIRBufferAttrGetTypeSize(MlirAttribute attr) { + return cast(unwrap(attr)).getTypeSize(); +} + +uint64_t mlirMetalAIRBufferAttrGetTypeAlignment(MlirAttribute attr) { + return cast(unwrap(attr)).getTypeAlignment(); +} + +uint64_t mlirMetalAIRBufferAttrGetBufferSize(MlirAttribute attr) { + return cast(unwrap(attr)).getBufferSize(); +} + +bool mlirMetalAIRBufferAttrGetNoalias(MlirAttribute attr) { + return cast(unwrap(attr)).getNoalias(); +} + +intptr_t mlirMetalAIRBufferAttrGetNumFields(MlirAttribute attr) { + return cast(unwrap(attr)).getFields().size(); +} + +MlirAttribute mlirMetalAIRBufferAttrGetField(MlirAttribute attr, intptr_t pos) { + return wrap(cast(unwrap(attr)).getFields()[pos]); +} diff --git a/mlir/dialects/air/capi.h b/mlir/dialects/air/capi.h new file mode 100644 index 00000000..f91b9f64 --- /dev/null +++ b/mlir/dialects/air/capi.h @@ -0,0 +1,81 @@ +//===- capi.h - C API for the Metal AIR dialect -------------------*- C -*-===// + +#ifndef MLIR_DIALECTS_AIR_CAPI_H +#define MLIR_DIALECTS_AIR_CAPI_H + +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Registration + +MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(MetalAIR, air); + +// Attributes. Invalid parameters emit a diagnostic and return a null handle; +// typed getters require a handle of their attribute kind. + +MLIR_CAPI_EXPORTED bool mlirAttributeIsAMetalAIRBuiltin(MlirAttribute attr); +/// `value` is a `mlir::metal::air::Builtin`. +MLIR_CAPI_EXPORTED MlirAttribute mlirMetalAIRBuiltinAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirMetalAIRBuiltinAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED bool mlirAttributeIsAMetalAIRMemFlags(MlirAttribute attr); +/// `value` is a `mlir::metal::air::MemFlags` bit set. +MLIR_CAPI_EXPORTED MlirAttribute +mlirMetalAIRMemFlagsAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirMetalAIRMemFlagsAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED bool mlirAttributeIsAMetalAIRStructField(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirMetalAIRStructFieldAttrGet( + MlirContext context, uint64_t offset, uint64_t size, uint64_t count, + MlirStringRef typeName, MlirStringRef name); +MLIR_CAPI_EXPORTED uint64_t +mlirMetalAIRStructFieldAttrGetOffset(MlirAttribute attr); +MLIR_CAPI_EXPORTED uint64_t +mlirMetalAIRStructFieldAttrGetSize(MlirAttribute attr); +MLIR_CAPI_EXPORTED uint64_t +mlirMetalAIRStructFieldAttrGetCount(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef +mlirMetalAIRStructFieldAttrGetTypeName(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef +mlirMetalAIRStructFieldAttrGetName(MlirAttribute attr); + +MLIR_CAPI_EXPORTED bool mlirAttributeIsAMetalAIRBuffer(MlirAttribute attr); +/// `addressSpace` is a `mlir::metal::air::AddressSpace` and `access` a +/// `mlir::metal::air::Access`. Sizes are bytes; a zero `bufferSize` omits it. +/// `elementType` and `fields` must belong to `context`. +MLIR_CAPI_EXPORTED MlirAttribute mlirMetalAIRBufferAttrGet( + MlirContext context, uint32_t locationIndex, uint32_t addressSpace, + uint32_t access, MlirType elementType, MlirStringRef typeName, + uint64_t typeSize, uint64_t typeAlignment, uint64_t bufferSize, + bool noalias, intptr_t numFields, MlirAttribute const *fields); +MLIR_CAPI_EXPORTED uint32_t +mlirMetalAIRBufferAttrGetLocationIndex(MlirAttribute attr); +MLIR_CAPI_EXPORTED uint32_t +mlirMetalAIRBufferAttrGetAddressSpace(MlirAttribute attr); +MLIR_CAPI_EXPORTED uint32_t mlirMetalAIRBufferAttrGetAccess(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirType +mlirMetalAIRBufferAttrGetElementType(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef +mlirMetalAIRBufferAttrGetTypeName(MlirAttribute attr); +MLIR_CAPI_EXPORTED uint64_t +mlirMetalAIRBufferAttrGetTypeSize(MlirAttribute attr); +MLIR_CAPI_EXPORTED uint64_t +mlirMetalAIRBufferAttrGetTypeAlignment(MlirAttribute attr); +MLIR_CAPI_EXPORTED uint64_t +mlirMetalAIRBufferAttrGetBufferSize(MlirAttribute attr); +MLIR_CAPI_EXPORTED bool mlirMetalAIRBufferAttrGetNoalias(MlirAttribute attr); +MLIR_CAPI_EXPORTED intptr_t +mlirMetalAIRBufferAttrGetNumFields(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirMetalAIRBufferAttrGetField(MlirAttribute attr, intptr_t pos); + +#ifdef __cplusplus +} +#endif + +#endif // MLIR_DIALECTS_AIR_CAPI_H diff --git a/mlir/dialects/air/test.zig b/mlir/dialects/air/test.zig new file mode 100644 index 00000000..57f0427e --- /dev/null +++ b/mlir/dialects/air/test.zig @@ -0,0 +1,439 @@ +const std = @import("std"); + +const air = @import("mlir/dialects/air"); +const arith = @import("mlir/dialects/arith"); +const func = @import("mlir/dialects/func"); +const llvm = @import("mlir/dialects/llvm"); +const mlir = @import("mlir"); + +const operation_names: []const []const u8 = &.{ + "air.abs.s", + "air.ceil", + "air.clamp", + "air.clamp.s", + "air.clamp.u", + "air.convert", + "air.convert.s", + "air.convert.u", + "air.cos", + "air.dot", + "air.exp", + "air.exp2", + "air.fabs", + "air.fast_ceil", + "air.fast_clamp", + "air.fast_cos", + "air.fast_exp", + "air.fast_exp2", + "air.fast_fabs", + "air.fast_floor", + "air.fast_fmax", + "air.fast_fmin", + "air.fast_fmod", + "air.fast_log", + "air.fast_log2", + "air.fast_pow", + "air.fast_round", + "air.fast_rsqrt", + "air.fast_sin", + "air.fast_sqrt", + "air.fast_tanh", + "air.fast_trunc", + "air.floor", + "air.fma", + "air.fmax", + "air.fmin", + "air.fmod", + "air.log", + "air.log2", + "air.max.s", + "air.max.u", + "air.min.s", + "air.min.u", + "air.pow", + "air.round", + "air.rsqrt", + "air.simd_all", + "air.simd_any", + "air.simd_broadcast", + "air.simd_broadcast.s", + "air.simd_broadcast.u", + "air.simd_max", + "air.simd_max.s", + "air.simd_max.u", + "air.simd_min", + "air.simd_min.s", + "air.simd_min.u", + "air.simd_prefix_exclusive_sum", + "air.simd_shuffle", + "air.simd_shuffle.s", + "air.simd_shuffle.u", + "air.simd_shuffle_down", + "air.simd_shuffle_down.s", + "air.simd_shuffle_down.u", + "air.simd_shuffle_xor", + "air.simd_shuffle_xor.s", + "air.simd_shuffle_xor.u", + "air.simd_sum", + "air.simd_sum.s", + "air.simd_sum.u", + "air.simdgroup.barrier", + "air.simdgroup_matrix_8x8_load", + "air.simdgroup_matrix_8x8_multiply_accumulate", + "air.simdgroup_matrix_8x8_store", + "air.sin", + "air.sqrt", + "air.tanh", + "air.trunc", + "air.wg.barrier", +}; + +test { + std.testing.refAllDecls(air); + inline for (.{ air.BuiltinAttr, air.MemFlagsAttr, air.StructFieldAttr, air.BufferAttr }) |T| std.testing.refAllDecls(T); +} + +fn testContext() !*mlir.Context { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + inline for (.{ air, arith, func, llvm }) |dialect| dialect.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + return ctx; +} + +fn expectRoundTrip(ctx: *mlir.Context, value: anytype) !void { + var writer: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer writer.deinit(); + try writer.writer.print("{f}", .{value}); + try std.testing.expect((try mlir.Attribute.parse(ctx, writer.written())).eql(value.attribute())); +} + +test "every bound operation is registered" { + const ctx = try testContext(); + defer ctx.deinit(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} + +test "AIR attributes use the C API" { + const ctx = try testContext(); + defer ctx.deinit(); + inline for (std.meta.tags(air.Builtin)) |builtin| { + const attr = try air.BuiltinAttr.get(ctx, .{ .value = builtin }); + try std.testing.expectEqual(builtin, attr.value()); + try expectRoundTrip(ctx, attr); + } + for ([_]air.MemFlags{ .none, .{ .device = true }, .{ .threadgroup = true }, .{ .device = true, .threadgroup = true, .texture = true } }) |flags| { + const attr = try air.MemFlagsAttr.get(ctx, .{ .value = flags }); + try std.testing.expectEqual(flags, attr.value()); + try expectRoundTrip(ctx, attr); + } + try std.testing.expectError(error.InvalidMlir, air.MemFlagsAttr.get(ctx, .{ .value = .{ ._padding = 1 } })); + + const field = try air.StructFieldAttr.get(ctx, .{ .offset = 4, .size = 4, .count = 3, .typeName = "float", .name = "s" }); + try std.testing.expectEqual(4, field.offset()); + try std.testing.expectEqual(4, field.size()); + try std.testing.expectEqual(3, field.count()); + try std.testing.expectEqualStrings("float", field.typeName()); + try std.testing.expectEqualStrings("s", field.name()); + try expectRoundTrip(ctx, field); + try std.testing.expectError(error.InvalidMlir, air.StructFieldAttr.get(ctx, .{ .offset = 0, .size = 0, .typeName = "int", .name = "n" })); + + const i32_type: *const mlir.Type = .int(ctx, .i32); + const f32_type: *const mlir.Type = .float(ctx, .f32); + const device: air.BufferAttr.InitArgs = .{ + .locationIndex = 1, + .addressSpace = .device, + .access = .read, + .elementType = .vector(&.{4}, .float(ctx, .bf16)), + .typeName = "bfloat4", + .typeSize = 8, + .typeAlignment = 8, + .noalias_ = true, + }; + const buffer = try air.BufferAttr.get(ctx, device); + try std.testing.expectEqual(1, buffer.locationIndex()); + try std.testing.expectEqual(air.AddressSpace.device, buffer.addressSpace()); + try std.testing.expectEqual(air.Access.read, buffer.access()); + try std.testing.expect(buffer.elementType().eql(device.elementType)); + try std.testing.expectEqualStrings("bfloat4", buffer.typeName()); + try std.testing.expectEqual(8, buffer.typeSize()); + try std.testing.expectEqual(8, buffer.typeAlignment()); + try std.testing.expectEqual(0, buffer.bufferSize()); + try std.testing.expect(buffer.noalias_()); + try std.testing.expectEqual(0, buffer.numFields()); + try expectRoundTrip(ctx, buffer); + + // `struct A { int n; float s[3]; }`, as Apple's compiler describes it. + const fields = [_]*const air.StructFieldAttr{ + try air.StructFieldAttr.get(ctx, .{ .offset = 0, .size = 4, .typeName = "int", .name = "n" }), + field, + }; + const constant: air.BufferAttr.InitArgs = .{ + .locationIndex = 0, + .addressSpace = .constant, + .access = .read, + .elementType = llvm.structType(ctx, &.{ i32_type, llvm.arrayType(f32_type, 3) }), + .typeName = "A", + .typeSize = 16, + .typeAlignment = 4, + .bufferSize = 16, + .fields = &fields, + }; + const structured = try air.BufferAttr.get(ctx, constant); + try std.testing.expectEqual(2, structured.numFields()); + try std.testing.expect(structured.field(1).eql(field)); + try expectRoundTrip(ctx, structured); + + var invalid = device; + invalid.typeSize = 16; + try std.testing.expectError(error.InvalidMlir, air.BufferAttr.get(ctx, invalid)); + invalid = device; + invalid.addressSpace = .constant; + invalid.access = .read_write; + try std.testing.expectError(error.InvalidMlir, air.BufferAttr.get(ctx, invalid)); + invalid = device; + invalid.addressSpace = .thread; + try std.testing.expectError(error.InvalidMlir, air.BufferAttr.get(ctx, invalid)); + invalid = device; + invalid.addressSpace = .threadgroup; + try std.testing.expectError(error.InvalidMlir, air.BufferAttr.get(ctx, invalid)); + invalid = device; + invalid.bufferSize = 8; + try std.testing.expectError(error.InvalidMlir, air.BufferAttr.get(ctx, invalid)); + invalid = device; + invalid.fields = fields[0..1]; + try std.testing.expectError(error.InvalidMlir, air.BufferAttr.get(ctx, invalid)); + invalid = constant; + invalid.fields = fields[0..1]; + try std.testing.expectError(error.InvalidMlir, air.BufferAttr.get(ctx, invalid)); + invalid = constant; + invalid.addressSpace = .device; + try std.testing.expectError(error.InvalidMlir, air.BufferAttr.get(ctx, invalid)); + + const other = try testContext(); + defer other.deinit(); + try std.testing.expectError(error.InvalidMlir, air.BufferAttr.get(other, device)); +} + +test "all AIR operations construct and verify" { + const ctx = try testContext(); + defer ctx.deinit(); + const loc: *const mlir.Location = .unknown(ctx); + const f32x4: *const mlir.Type = .vector(&.{4}, .float(ctx, .f32)); + const i32x2: *const mlir.Type = .vector(&.{2}, .int(ctx, .i32)); + const i64x2: *const mlir.Type = .vector(&.{2}, .int(ctx, .i64)); + const block = mlir.Block.init( + &.{ .float(ctx, .f32), .float(ctx, .f16), f32x4, .int(ctx, .i32), i32x2, .int(ctx, .i16), .int(ctx, .i1), llvm.pointerType(ctx, air.AddressSpace.threadgroup), i64x2, .vector(&.{64}, .float(ctx, .f16)), .vector(&.{64}, .float(ctx, .f32)), .vector(&.{4}, .float(ctx, .bf16)) }, + &.{ loc, loc, loc, loc, loc, loc, loc, loc, loc, loc, loc, loc }, + ); + defer block.deinit(); + const f = block.argument(0); + const h = block.argument(1); + const v = block.argument(2); + const i = block.argument(3); + const iv = block.argument(4); + const lane = block.argument(5); + const b = block.argument(6); + const fast: air.FloatOpts = .{ .fastmath = .fast }; + var ops: std.ArrayList(*mlir.Operation) = .empty; + defer ops.deinit(std.testing.allocator); + inline for (.{ air.exp, air.exp2, air.log, air.log2, air.tanh, air.sqrt, air.rsqrt, air.fabs, air.ceil, air.floor, air.round, air.trunc, air.sin, air.cos, air.simd_sum, air.simd_max, air.simd_min, air.simd_prefix_exclusive_sum }) |op| { + try ops.append(std.testing.allocator, op(ctx, h, .{}, loc).appendTo(block)); + try ops.append(std.testing.allocator, op(ctx, v, fast, loc).appendTo(block)); + } + inline for (.{ air.fast_exp, air.fast_exp2, air.fast_log, air.fast_log2, air.fast_tanh, air.fast_sqrt, air.fast_rsqrt, air.fast_fabs, air.fast_ceil, air.fast_floor, air.fast_round, air.fast_trunc, air.fast_sin, air.fast_cos }) |op| { + try ops.append(std.testing.allocator, op(ctx, f, fast, loc).appendTo(block)); + try ops.append(std.testing.allocator, op(ctx, v, fast, loc).appendTo(block)); + } + inline for (.{ air.fmax, air.fmin, air.pow, air.fmod, air.fast_fmax, air.fast_fmin, air.fast_pow, air.fast_fmod }) |op| { + try ops.append(std.testing.allocator, op(ctx, f, f, fast, loc).appendTo(block)); + } + inline for (.{ air.fma, air.clamp, air.fast_clamp }) |op| { + try ops.append(std.testing.allocator, op(ctx, v, v, v, fast, loc).appendTo(block)); + } + try ops.append(std.testing.allocator, air.dot(ctx, v, v, fast, loc).appendTo(block)); + try ops.append(std.testing.allocator, air.abs_s(ctx, i, loc).appendTo(block)); + inline for (.{ air.max_s, air.max_u, air.min_s, air.min_u }) |op| { + try ops.append(std.testing.allocator, op(ctx, iv, iv, loc).appendTo(block)); + } + inline for (.{ air.clamp_s, air.clamp_u }) |op| { + try ops.append(std.testing.allocator, op(ctx, i, i, i, loc).appendTo(block)); + } + inline for (.{ air.simd_sum_s, air.simd_sum_u, air.simd_max_s, air.simd_max_u, air.simd_min_s, air.simd_min_u }) |op| { + try ops.append(std.testing.allocator, op(ctx, i, loc).appendTo(block)); + try ops.append(std.testing.allocator, op(ctx, lane, loc).appendTo(block)); + } + inline for (.{ air.simd_shuffle, air.simd_shuffle_xor, air.simd_shuffle_down, air.simd_broadcast }) |op| { + try ops.append(std.testing.allocator, op(ctx, v, lane, fast, loc).appendTo(block)); + } + inline for (.{ air.simd_shuffle_s, air.simd_shuffle_u, air.simd_shuffle_xor_s, air.simd_shuffle_xor_u, air.simd_shuffle_down_s, air.simd_shuffle_down_u, air.simd_broadcast_s, air.simd_broadcast_u }) |op| { + try ops.append(std.testing.allocator, op(ctx, iv, lane, loc).appendTo(block)); + } + inline for (.{ air.simd_any, air.simd_all }) |op| { + try ops.append(std.testing.allocator, op(ctx, b, loc).appendTo(block)); + } + try ops.append(std.testing.allocator, air.convert(ctx, block.argument(11), f32x4, fast, loc).appendTo(block)); + try ops.append(std.testing.allocator, air.convert(ctx, h, .float(ctx, .bf16), .{}, loc).appendTo(block)); + try ops.append(std.testing.allocator, air.convert_s(ctx, i, .float(ctx, .f32), fast, loc).appendTo(block)); + try ops.append(std.testing.allocator, air.convert_u(ctx, f, .int(ctx, .i32), .{}, loc).appendTo(block)); + inline for (.{ air.wg_barrier, air.simdgroup_barrier }) |op| { + try ops.append(std.testing.allocator, op(ctx, .{ .threadgroup = true }, loc).appendTo(block)); + } + const shape = block.argument(8); + const matrix = air.simdgroup_matrix_8x8_load(ctx, block.argument(7), shape, shape, shape, .vector(&.{64}, .float(ctx, .f16)), fast, loc).appendTo(block); + const product = air.simdgroup_matrix_8x8_multiply_accumulate(ctx, matrix.result(0), block.argument(9), block.argument(10), .vector(&.{64}, .float(ctx, .f32)), fast, loc).appendTo(block); + const stored = air.simdgroup_matrix_8x8_store(ctx, product.result(0), block.argument(7), shape, shape, shape, loc).appendTo(block); + try ops.appendSlice(std.testing.allocator, &.{ matrix, product, stored }); + for (ops.items) |op| try std.testing.expect(op.verify()); +} + +test "AIR operations round trip through their custom assembly" { + const ctx = try testContext(); + defer ctx.deinit(); + const source = + \\func.func @ops(%f: f32, %v: vector<4xbf16>, %i: i32, %s: i16, %b: i1, %p: !llvm.ptr<3>, %d: vector<2xi64>, %m: vector<64xf32>) { + \\ %0 = air.fast_exp %f fastmath : f32 + \\ %1 = air.tanh %f : f32 + \\ %2 = air.fma %f, %f, %f fastmath : f32 + \\ %3 = air.max.s %i, %i : i32 + \\ %4 = air.simd_shuffle_xor %f, %s fastmath : f32 + \\ %5 = air.simd_sum.u %i : i32 + \\ %6 = air.simd_any %b + \\ %7 = air.convert %v fastmath : vector<4xbf16> to vector<4xf32> + \\ %8 = air.convert.s %i fastmath : i32 to f32 + \\ %9 = air.dot %7, %7 fastmath : vector<4xf32> -> f32 + \\ air.wg.barrier + \\ air.simdgroup.barrier + \\ %10 = air.simdgroup_matrix_8x8_load %p, %d, %d, %d fastmath : !llvm.ptr<3> -> vector<64xf32> + \\ %11 = air.simdgroup_matrix_8x8_multiply_accumulate %10, %m, %m fastmath : vector<64xf32>, vector<64xf32>, vector<64xf32> -> vector<64xf32> + \\ air.simdgroup_matrix_8x8_store %11, %p, %d, %d, %d : vector<64xf32>, !llvm.ptr<3> + \\ return + \\} + ; + const module = try mlir.Module.parse(ctx, source); + defer module.deinit(); + var first: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer first.deinit(); + try first.writer.print("{f}", .{module.operation()}); + try std.testing.expect(std.mem.find(u8, first.written(), "\"air.") == null); + const reparsed = try mlir.Module.parse(ctx, first.written()); + defer reparsed.deinit(); + var second: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer second.deinit(); + try second.writer.print("{f}", .{reparsed.operation()}); + try std.testing.expectEqualStrings(first.written(), second.written()); +} + +test "AIR verifier rejects types Apple's compiler has no intrinsic for" { + const ctx = try testContext(); + defer ctx.deinit(); + const rejected = [_][]const u8{ + "func.func @f(%x: f16) { %0 = air.fast_exp %x : f16\n return }", + "func.func @f(%x: bf16) { %0 = air.exp %x : bf16\n return }", + "func.func @f(%x: f32) { %0 = air.max.s %x, %x : f32\n return }", + "func.func @f(%x: i64) { %0 = air.simd_sum.s %x : i64\n return }", + "func.func @f(%x: vector<8xf32>) { %0 = air.simd_sum %x : vector<8xf32>\n return }", + "func.func @f(%x: f32) { %0 = air.convert %x : f32 to f32\n return }", + "func.func @f(%x: i32) { %0 = air.convert %x : i32 to f32\n return }", + "func.func @f(%x: f32) { %0 = air.convert.s %x : f32 to f16\n return }", + "func.func @f(%x: f32) { %0 = air.convert.u %x fastmath : f32 to i32\n return }", + "func.func @f(%x: vector<4xf32>) { %0 = air.convert %x : vector<4xf32> to vector<2xf16>\n return }", + "func.func @f(%x: vector<4xf32>) { %0 = air.dot %x, %x : vector<4xf32> -> f16\n return }", + "func.func @f(%p: !llvm.ptr<2>, %d: vector<2xi64>) { %0 = air.simdgroup_matrix_8x8_load %p, %d, %d, %d : !llvm.ptr<2> -> vector<64xf32>\n return }", + "func.func @f(%a: vector<64xbf16>, %b: vector<64xf16>, %c: vector<64xf32>) { %0 = air.simdgroup_matrix_8x8_multiply_accumulate %a, %b, %c : vector<64xbf16>, vector<64xf16>, vector<64xf32> -> vector<64xf32>\n return }", + "func.func @f(%a: vector<64xf16>, %c: vector<64xf32>) { %0 = air.simdgroup_matrix_8x8_multiply_accumulate %a, %a, %c : vector<64xf16>, vector<64xf16>, vector<64xf32> -> vector<64xf16>\n return }", + }; + for (rejected) |source| try std.testing.expectError(error.InvalidMlir, mlir.Module.parse(ctx, source)); +} + +test "AIR verifier takes the 8-bit integers Apple's compiler emits for char" { + const ctx = try testContext(); + defer ctx.deinit(); + const module = try mlir.Module.parse(ctx, + \\func.func @f(%c: i8, %v: vector<2xi8>, %f: f32, %w: vector<2xf32>, %l: i16) { + \\ %0 = air.convert.s %c fastmath : i8 to f32 + \\ %1 = air.convert.u %f : f32 to i8 + \\ %2 = air.convert.s %v fastmath : vector<2xi8> to vector<2xf32> + \\ %3 = air.convert.s %w : vector<2xf32> to vector<2xi8> + \\ %4 = air.max.s %c, %c : i8 + \\ %5 = air.min.u %v, %v : vector<2xi8> + \\ %6 = air.abs.s %c : i8 + \\ %7 = air.clamp.u %c, %c, %c : i8 + \\ %8 = air.simd_shuffle.s %c, %l : i8 + \\ %9 = air.simd_sum.s %c : i8 + \\ %10 = air.simd_max.u %c : i8 + \\ return + \\} + ); + module.deinit(); +} + +test "AIR kernel ABI" { + const ctx = try testContext(); + defer ctx.deinit(); + const device = "#air.buffer"; + const threadgroup = "#air.buffer"; + const accepted = [_][]const u8{ + "func.func @k(%a: !llvm.ptr<1> {air.buffer = " ++ device ++ ", air.arg_name = \"a\"}, %b: !llvm.ptr<3> {air.buffer = " ++ threadgroup ++ "}, %t: vector<3xi32> {air.builtin = #air.builtin}, %l: i16 {air.builtin = #air.builtin}, %s: i32 {air.builtin = #air.builtin}) attributes {air.kernel} { return }", + "module attributes {air.fast_math_enable} {}", + }; + const rejected = [_][]const u8{ + "func.func @k(%a: i32) attributes {air.kernel} { return }", + "func.func @k(%a: !llvm.ptr<1> {air.buffer = " ++ device ++ ", air.builtin = #air.builtin}) attributes {air.kernel} { return }", + "func.func @k(%a: !llvm.ptr<3> {air.buffer = " ++ device ++ "}) attributes {air.kernel} { return }", + "func.func @k(%a: !llvm.ptr<1> {air.buffer = " ++ device ++ "}, %b: !llvm.ptr<1> {air.buffer = " ++ device ++ "}) attributes {air.kernel} { return }", + "func.func @k(%a: vector<3xi32> {air.builtin = #air.builtin}) attributes {air.kernel} { return }", + "func.func @k(%a: vector<4xi32> {air.builtin = #air.builtin}) attributes {air.kernel} { return }", + "func.func @k(%a: i64 {air.builtin = #air.builtin}) attributes {air.kernel} { return }", + "func.func @k(%a: i32 {air.builtin = #air.builtin}, %b: i32 {air.builtin = #air.builtin}) attributes {air.kernel} { return }", + "func.func @k(%a: i32 {air.builtin = #air.builtin, air.arg_name = \"\"}) attributes {air.kernel} { return }", + "func.func @k() -> i32 attributes {air.kernel} { %c = arith.constant 0 : i32\n return %c : i32 }", + "func.func @h(%a: i32 {air.builtin = #air.builtin}) { return }", + "func.func @h() attributes {air.fast_math_enable} { return }", + "func.func @h() attributes {air.unknown} { return }", + }; + for (accepted) |source| { + const module = try mlir.Module.parse(ctx, source); + module.deinit(); + } + for (rejected) |source| try std.testing.expectError(error.InvalidMlir, mlir.Module.parse(ctx, source)); +} + +test "AIR kernel builder" { + const ctx = try testContext(); + defer ctx.deinit(); + const loc: *const mlir.Location = .unknown(ctx); + const module: *mlir.Module = .init(loc); + defer module.deinit(); + const block = mlir.Block.init(&.{ llvm.pointerType(ctx, air.AddressSpace.device), .vector(&.{3}, .int(ctx, .i32)) }, &.{ loc, loc }); + _ = air.wg_barrier(ctx, .{ .threadgroup = true }, loc).appendTo(block); + _ = func.returns(ctx, &.{}, loc).appendTo(block); + _ = air.kernel(ctx, .{ + .name = "example", + .block = block, + .location = loc, + .args_attributes = &.{ + try air.bufferArgumentAttr(ctx, .{ + .locationIndex = 0, + .addressSpace = .device, + .access = .read_write, + .elementType = .float(ctx, .f32), + .typeName = "float", + .typeSize = 4, + .typeAlignment = 4, + }, .{ .name = "out" }), + try air.builtinArgumentAttr(ctx, .thread_position_in_grid, .{}), + }, + }).appendTo(module.body()); + air.enableFastMath(ctx, module.operation()); + try std.testing.expect(module.operation().verify()); + var writer: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer writer.deinit(); + try writer.writer.print("{f}", .{module.operation()}); + const parsed = try mlir.Module.parse(ctx, writer.written()); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); + try std.testing.expect(std.mem.find(u8, writer.written(), "air.arg_name = \"out\"") != null); +} diff --git a/mlir/dialects/arith.zig b/mlir/dialects/arith.zig deleted file mode 100644 index 2c9e9454..00000000 --- a/mlir/dialects/arith.zig +++ /dev/null @@ -1,109 +0,0 @@ -const std = @import("std"); -const mlir = @import("mlir"); - -pub fn constant(ctx: mlir.Context, value: mlir.Attribute, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "arith.constant", .{ - .attributes = &.{ - .{ "value", value }, - }, - .result_type_inference = true, - .location = location, - }); -} - -fn binary_fn(comptime op_name: [:0]const u8) fn (mlir.Context, mlir.Value, mlir.Value, mlir.Location) mlir.Operation { - return struct { - pub fn call(ctx: mlir.Context, lhs: mlir.Value, rhs: mlir.Value, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, op_name, .{ - .operands = &.{ lhs, rhs }, - .result_type_inference = true, - .location = location, - }); - } - }.call; -} - -fn cast_fn(comptime op_name: [:0]const u8) fn (mlir.Context, mlir.Value, mlir.Type, mlir.Location) mlir.Operation { - return struct { - pub fn call(ctx: mlir.Context, value: mlir.Value, new_type: mlir.Type, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, op_name, .{ - .operands = &.{value}, - .results = &.{new_type}, - .location = location, - }); - } - }.call; -} - -pub const addi = binary_fn("arith.addi"); -pub const addf = binary_fn("arith.addf"); -pub const subi = binary_fn("arith.subi"); -pub const subf = binary_fn("arith.subf"); -pub const muli = binary_fn("arith.muli"); -pub const mulf = binary_fn("arith.mulf"); -pub const divsi = binary_fn("arith.divsi"); -pub const divui = binary_fn("arith.divui"); -pub const divf = binary_fn("arith.divf"); -pub const extsi = cast_fn("arith.extsi"); -pub const extui = cast_fn("arith.extui"); -pub const extf = cast_fn("arith.extf"); -pub const trunci = cast_fn("arith.trunci"); -pub const truncf = cast_fn("arith.truncf"); -pub const fptosi = cast_fn("arith.fptosi"); -pub const fptoui = cast_fn("arith.fptoui"); -pub const sitofp = cast_fn("arith.sitofp"); -pub const uitofp = cast_fn("arith.uitofp"); - -pub const CmpIPredicate = enum { - eq, - ne, - slt, - sle, - sgt, - sge, - ult, - ule, - ugt, - uge, -}; - -pub fn cmpi(ctx: mlir.Context, predicate: CmpIPredicate, lhs: mlir.Value, rhs: mlir.Value, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "arith.cmpi", .{ - .operands = &.{ lhs, rhs }, - .result_type_inference = true, - .attributes = &.{ - .{ "predicate", mlir.IntegerAttribute(.i64).init(ctx, @intFromEnum(predicate)).as(mlir.Attribute).? }, - }, - .location = location, - }); -} - -pub const CmpFPredicate = enum { - false, - oeq, - ogt, - oge, - olt, - ole, - one, - ord, - ueq, - ugt, - uge, - ult, - ule, - une, - uno, - true, -}; - -pub fn cmpf(ctx: mlir.Context, predicate: CmpFPredicate, lhs: mlir.Value, rhs: mlir.Value, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "arith.cmpf", .{ - .operands = &.{ lhs, rhs }, - .result_type_inference = true, - .attributes = &.{ - .{ "predicate", mlir.IntegerAttribute(.i64).init(ctx, @intFromEnum(predicate)).as(mlir.Attribute).? }, - }, - .location = location, - }); -} diff --git a/mlir/dialects/arith/BUILD.bazel b/mlir/dialects/arith/BUILD.bazel new file mode 100644 index 00000000..90c84ff0 --- /dev/null +++ b/mlir/dialects/arith/BUILD.bazel @@ -0,0 +1,36 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +cc_library( + name = "capi", + srcs = ["capi.cc"], + hdrs = ["capi.h"], + tags = ["manual"], + deps = [ + "@llvm-project//mlir:ArithDialect", + "@llvm-project//mlir:CAPIIR", + ], +) + +zig_library( + name = "arith", + import_name = "mlir/dialects/arith", + main = "arith.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + ":capi", + "//mlir", + "//stdx", + "@llvm-project//mlir:CAPIArith", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":arith", + "//mlir", + ], +) diff --git a/mlir/dialects/arith/arith.zig b/mlir/dialects/arith/arith.zig new file mode 100644 index 00000000..c45ce5b5 --- /dev/null +++ b/mlir/dialects/arith/arith.zig @@ -0,0 +1,495 @@ +const c = @import("c"); +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("arith"); +} + +// ============================================================================= +// Predicate enums (match ArithBase.td exactly) +// ============================================================================= + +/// Arith cmpi predicates. Values match mlir::arith::CmpIPredicate (I64EnumAttr). +pub const CmpIPredicate = enum(i64) { + eq = 0, + ne = 1, + slt = 2, + sle = 3, + sgt = 4, + sge = 5, + ult = 6, + ule = 7, + ugt = 8, + uge = 9, +}; + +/// Arith cmpf predicates. Values match mlir::arith::CmpFPredicate. +pub const CmpFPredicate = enum(i64) { + false = 0, + oeq = 1, + ogt = 2, + oge = 3, + olt = 4, + ole = 5, + one = 6, + ord = 7, + ueq = 8, + ugt = 9, + uge = 10, + ult = 11, + ule = 12, + une = 13, + uno = 14, + true = 15, +}; + +// ============================================================================= +// Flag attributes +// ============================================================================= + +/// `mlir::arith::FastMathFlags`, one bit per flag. +pub const FastMathFlags = packed struct(u32) { + reassoc: bool = false, + nnan: bool = false, + ninf: bool = false, + nsz: bool = false, + arcp: bool = false, + contract: bool = false, + afn: bool = false, + _padding: u25 = 0, + + pub const none: FastMathFlags = .{}; + pub const fast: FastMathFlags = .{ .reassoc = true, .nnan = true, .ninf = true, .nsz = true, .arcp = true, .contract = true, .afn = true }; +}; + +/// `#arith.fastmath<...>`. +pub const FastMathAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsAArithFastMath; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "fastmath"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: FastMathFlags, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirArithFastMathAttrGet(ctx.ptr(), @bitCast(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) FastMathFlags { + return @bitCast(c.mlirArithFastMathAttrGetValue(self.ptr())); + } +}; + +/// `fastmath = #arith.fastmath`: the fast-math flags of a float +/// operation. +pub fn fastmath(ctx: *mlir.Context, flags: FastMathFlags) mlir.Error!mlir.NamedAttribute { + return .named(ctx, "fastmath", (try FastMathAttr.get(ctx, .{ .value = flags })).attribute()); +} + +/// `mlir::arith::IntegerOverflowFlags`, one bit per flag. +pub const IntegerOverflowFlags = packed struct(u32) { + nsw: bool = false, + nuw: bool = false, + _padding: u30 = 0, + + pub const none: IntegerOverflowFlags = .{}; +}; + +/// `#arith.overflow<...>`. +pub const IntegerOverflowAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsAArithIntegerOverflow; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "overflow"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: IntegerOverflowFlags, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirArithIntegerOverflowAttrGet(ctx.ptr(), @bitCast(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) IntegerOverflowFlags { + return @bitCast(c.mlirArithIntegerOverflowAttrGetValue(self.ptr())); + } +}; + +/// `overflowFlags = #arith.overflow`: the overflow flags of an +/// integer operation. +pub fn overflowFlags(ctx: *mlir.Context, flags: IntegerOverflowFlags) mlir.Error!mlir.NamedAttribute { + return .named(ctx, "overflowFlags", (try IntegerOverflowAttr.get(ctx, .{ .value = flags })).attribute()); +} + +// ============================================================================= +// Constants +// ============================================================================= + +/// arith.constant for an integer scalar. `value` is reinterpreted to `int_type`'s bit width. +pub fn constant_int( + ctx: *mlir.Context, + value: i64, + int_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "arith.constant", .{ + .results = .{ .flat = &.{int_type} }, + .attributes = &.{.named(ctx, "value", .intFromType(@ptrCast(int_type), value))}, + .location = location, + }); +} + +/// arith.constant for a float scalar of the given FloatTypes kind. +pub fn constant_float( + ctx: *mlir.Context, + value: f64, + comptime ft: mlir.FloatTypes, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "arith.constant", .{ + .results = .{ .flat = &.{.float(ctx, ft)} }, + .attributes = &.{.named(ctx, "value", .float(ctx, ft, value))}, + .location = location, + }); +} + +/// arith.constant for an `index`-typed integer. +pub fn constant_index(ctx: *mlir.Context, value: i64, location: *const mlir.Location) *mlir.Operation { + const idx_ty = mlir.Type.index(ctx); + const attr: *const mlir.Attribute = @ptrCast(c.mlirIntegerAttrGet(idx_ty.ptr(), value).ptr); + return mlir.Operation.make(ctx, "arith.constant", .{ + .results = .{ .flat = &.{idx_ty} }, + .attributes = &.{.named(ctx, "value", attr)}, + .location = location, + }); +} + +// ============================================================================= +// Binary ops (generated from shared template) +// ============================================================================= + +fn binaryOp(comptime op_name: []const u8) type { + return struct { + pub fn call( + ctx: *mlir.Context, + lhs: *const mlir.Value, + rhs: *const mlir.Value, + location: *const mlir.Location, + ) *mlir.Operation { + return mlir.Operation.make(ctx, op_name, .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .result_type_inference = true, + .location = location, + }); + } + }; +} + +// Integer arithmetic +pub const addi = binaryOp("arith.addi").call; +pub const subi = binaryOp("arith.subi").call; +pub const muli = binaryOp("arith.muli").call; +pub const divsi = binaryOp("arith.divsi").call; +pub const divui = binaryOp("arith.divui").call; +pub const remsi = binaryOp("arith.remsi").call; +pub const remui = binaryOp("arith.remui").call; + +// Integer bitwise / shifts +pub const andi = binaryOp("arith.andi").call; +pub const ori = binaryOp("arith.ori").call; +pub const xori = binaryOp("arith.xori").call; +pub const shli = binaryOp("arith.shli").call; +pub const shrsi = binaryOp("arith.shrsi").call; +pub const shrui = binaryOp("arith.shrui").call; + +// Integer min/max +pub const maxsi = binaryOp("arith.maxsi").call; +pub const maxui = binaryOp("arith.maxui").call; +pub const minsi = binaryOp("arith.minsi").call; +pub const minui = binaryOp("arith.minui").call; + +// Ceil/floor div +pub const ceildivsi = binaryOp("arith.ceildivsi").call; +pub const ceildivui = binaryOp("arith.ceildivui").call; +pub const floordivsi = binaryOp("arith.floordivsi").call; + +// Float arithmetic +pub const addf = binaryOp("arith.addf").call; +pub const subf = binaryOp("arith.subf").call; +pub const mulf = binaryOp("arith.mulf").call; +pub const divf = binaryOp("arith.divf").call; +pub const remf = binaryOp("arith.remf").call; + +// Float min/max +pub const maximumf = binaryOp("arith.maximumf").call; +pub const maxnumf = binaryOp("arith.maxnumf").call; +pub const minimumf = binaryOp("arith.minimumf").call; +pub const minnumf = binaryOp("arith.minnumf").call; + +// ============================================================================= +// Unary ops +// ============================================================================= + +pub fn negf(ctx: *mlir.Context, src: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "arith.negf", .{ + .operands = .{ .flat = &.{src} }, + .result_type_inference = true, + .location = location, + }); +} + +/// arith.flush_denormals — clamp subnormals to zero (SameOperandsAndResultType). +pub fn flush_denormals(ctx: *mlir.Context, src: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "arith.flush_denormals", .{ + .operands = .{ .flat = &.{src} }, + .result_type_inference = true, + .location = location, + }); +} + +/// arith.constant for a dense splat: builds `arith.constant dense : T` +/// where `T` is a ranked tensor type. Useful for tensor-typed zero/one constants. +/// For float splats, use `constant_dense_splat_f`; for ints, `constant_dense_splat_i`. +pub fn constant_dense_splat_f( + ctx: *mlir.Context, + value: f64, + tensor_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + const elem_c = c.mlirShapedTypeGetElementType(tensor_type.ptr()); + const elem_attr = c.mlirFloatAttrDoubleGet(ctx.ptr(), elem_c, value); + const dense_c = c.mlirDenseElementsAttrSplatGet(tensor_type.ptr(), elem_attr); + const dense_attr: *const mlir.Attribute = @ptrCast(dense_c.ptr); + return mlir.Operation.make(ctx, "arith.constant", .{ + .results = .{ .flat = &.{tensor_type} }, + .attributes = &.{.named(ctx, "value", dense_attr)}, + .location = location, + }); +} + +pub fn constant_dense_splat_i( + ctx: *mlir.Context, + value: i64, + tensor_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + const elem_c = c.mlirShapedTypeGetElementType(tensor_type.ptr()); + const elem_attr = c.mlirIntegerAttrGet(elem_c, value); + const dense_c = c.mlirDenseElementsAttrSplatGet(tensor_type.ptr(), elem_attr); + const dense_attr: *const mlir.Attribute = @ptrCast(dense_c.ptr); + return mlir.Operation.make(ctx, "arith.constant", .{ + .results = .{ .flat = &.{tensor_type} }, + .attributes = &.{.named(ctx, "value", dense_attr)}, + .location = location, + }); +} + +// ============================================================================= +// Extended-result binary ops — return two values +// ============================================================================= + +/// arith.addui_extended — unsigned add, returns (sum, overflow: i1 / i1-tensor). +/// The overflow type is typically i1 (scalar) or tensor<...xi1> matching lhs shape. +pub fn addui_extended( + ctx: *mlir.Context, + lhs: *const mlir.Value, + rhs: *const mlir.Value, + overflow_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "arith.addui_extended", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{ lhs.type_(), overflow_type } }, + .location = location, + }); +} + +/// arith.mulsi_extended — signed multiply, returns (low, high) both same type as lhs. +pub fn mulsi_extended( + ctx: *mlir.Context, + lhs: *const mlir.Value, + rhs: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "arith.mulsi_extended", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{ lhs.type_(), lhs.type_() } }, + .location = location, + }); +} + +/// arith.mului_extended — unsigned multiply, returns (low, high) both same type as lhs. +pub fn mului_extended( + ctx: *mlir.Context, + lhs: *const mlir.Value, + rhs: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "arith.mului_extended", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{ lhs.type_(), lhs.type_() } }, + .location = location, + }); +} + +// ============================================================================= +// Cast ops (1 operand, explicit result type) +// ============================================================================= + +fn castOp(comptime op_name: []const u8) type { + return struct { + pub fn call( + ctx: *mlir.Context, + value: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, + ) *mlir.Operation { + return mlir.Operation.make(ctx, op_name, .{ + .operands = .{ .flat = &.{value} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); + } + }; +} + +pub const extsi = castOp("arith.extsi").call; +pub const extui = castOp("arith.extui").call; +pub const extf = castOp("arith.extf").call; +pub const trunci = castOp("arith.trunci").call; +pub const truncf = castOp("arith.truncf").call; +pub const sitofp = castOp("arith.sitofp").call; +pub const uitofp = castOp("arith.uitofp").call; +pub const fptosi = castOp("arith.fptosi").call; +pub const fptoui = castOp("arith.fptoui").call; +pub const bitcast = castOp("arith.bitcast").call; +pub const index_cast = castOp("arith.index_cast").call; +pub const index_castui = castOp("arith.index_castui").call; + +// ============================================================================= +// convertf — same-bitwidth float cast (e.g. f16↔bf16). Not expressible via +// arith.extf or arith.truncf. +// ============================================================================= + +/// Matches `Arith_RoundingModeAttr` (I32EnumAttr; see arith_dialect.td). +pub const RoundingMode = enum(i32) { + to_nearest_even = 0, + downward = 1, + upward = 2, + toward_zero = 3, + to_nearest_away = 4, +}; + +pub const ConvertFOpts = struct { + rounding: ?RoundingMode = null, +}; + +pub fn convertf( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + opts: ConvertFOpts, + location: *const mlir.Location, +) *mlir.Operation { + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.rounding) |rm| { + attrs.appendAssumeCapacity(.named(ctx, "roundingmode", .int(ctx, .i32, @backingInt(rm)))); + } + return mlir.Operation.make(ctx, "arith.convertf", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +/// arith.scaling_truncf — OCP MXFP downcast with per-block scale factors. +pub fn scaling_truncf( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + result_type: *const mlir.Type, + opts: ConvertFOpts, + location: *const mlir.Location, +) *mlir.Operation { + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.rounding) |rm| { + attrs.appendAssumeCapacity(.named(ctx, "roundingmode", .int(ctx, .i32, @backingInt(rm)))); + } + return mlir.Operation.make(ctx, "arith.scaling_truncf", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{result_type} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +// ============================================================================= +// Compare +// ============================================================================= + +pub fn cmpi( + ctx: *mlir.Context, + predicate: CmpIPredicate, + lhs: *const mlir.Value, + rhs: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "arith.cmpi", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .result_type_inference = true, + .attributes = &.{ + .named(ctx, "predicate", .int(ctx, .i64, @backingInt(predicate))), + }, + .location = location, + }); +} + +pub fn cmpf( + ctx: *mlir.Context, + predicate: CmpFPredicate, + lhs: *const mlir.Value, + rhs: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "arith.cmpf", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .result_type_inference = true, + .attributes = &.{ + .named(ctx, "predicate", .int(ctx, .i64, @backingInt(predicate))), + }, + .location = location, + }); +} + +// ============================================================================= +// Select +// ============================================================================= + +/// arith.select — result type matches the true/false operand type. +pub fn select( + ctx: *mlir.Context, + cond: *const mlir.Value, + t: *const mlir.Value, + f: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "arith.select", .{ + .operands = .{ .flat = &.{ cond, t, f } }, + .results = .{ .flat = &.{t.type_()} }, + .location = location, + }); +} diff --git a/mlir/dialects/arith/capi.cc b/mlir/dialects/arith/capi.cc new file mode 100644 index 00000000..3d7359c2 --- /dev/null +++ b/mlir/dialects/arith/capi.cc @@ -0,0 +1,47 @@ +//===- capi.cc - C API for the arith dialect ------------------------------===// + +#include "capi.h" + +#include "mlir/CAPI/IR.h" +#include "mlir/Dialect/Arith/IR/Arith.h" + +bool mlirAttributeIsAArithFastMath(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirArithFastMathAttrGet(MlirContext context, uint32_t value) { + mlir::MLIRContext *ctx = unwrap(context); + std::optional flags = + mlir::arith::symbolizeFastMathFlags(value); + if (!ctx || !flags) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(mlir::arith::FastMathFlagsAttr::get(ctx, *flags)); +} + +uint32_t mlirArithFastMathAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsAArithIntegerOverflow(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} + +MlirAttribute mlirArithIntegerOverflowAttrGet(MlirContext context, + uint32_t value) { + mlir::MLIRContext *ctx = unwrap(context); + std::optional flags = + mlir::arith::symbolizeIntegerOverflowFlags(value); + if (!ctx || !flags) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(mlir::arith::IntegerOverflowFlagsAttr::get(ctx, *flags)); +} + +uint32_t mlirArithIntegerOverflowAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} diff --git a/mlir/dialects/arith/capi.h b/mlir/dialects/arith/capi.h new file mode 100644 index 00000000..ec8962af --- /dev/null +++ b/mlir/dialects/arith/capi.h @@ -0,0 +1,33 @@ +//===- capi.h - C API for the arith dialect -----------------------*- C -*-===// +// +// C entry points for the `arith` dialect's flag attributes, which upstream's +// C API leaves out. + +#ifndef MLIR_DIALECTS_ARITH_CAPI_H +#define MLIR_DIALECTS_ARITH_CAPI_H + +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// #arith.fastmath<...>: `value` is the `FastMathFlags` bit set; `Get` returns +// a null attribute when it has bits outside the enum. +MLIR_CAPI_EXPORTED bool mlirAttributeIsAArithFastMath(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirArithFastMathAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirArithFastMathAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED bool +mlirAttributeIsAArithIntegerOverflow(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirArithIntegerOverflowAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirArithIntegerOverflowAttrGetValue(MlirAttribute attr); + +#ifdef __cplusplus +} +#endif + +#endif // MLIR_DIALECTS_ARITH_CAPI_H diff --git a/mlir/dialects/arith/test.zig b/mlir/dialects/arith/test.zig new file mode 100644 index 00000000..844898ca --- /dev/null +++ b/mlir/dialects/arith/test.zig @@ -0,0 +1,142 @@ +const std = @import("std"); + +const arith = @import("mlir/dialects/arith"); +const mlir = @import("mlir"); + +const operation_names: []const []const u8 = &.{ + "arith.addf", + "arith.addi", + "arith.addui_extended", + "arith.andi", + "arith.bitcast", + "arith.ceildivsi", + "arith.ceildivui", + "arith.cmpf", + "arith.cmpi", + "arith.constant", + "arith.convertf", + "arith.divf", + "arith.divsi", + "arith.divui", + "arith.extf", + "arith.extsi", + "arith.extui", + "arith.floordivsi", + "arith.flush_denormals", + "arith.fptosi", + "arith.fptoui", + "arith.index_cast", + "arith.index_castui", + "arith.maximumf", + "arith.maxnumf", + "arith.maxsi", + "arith.maxui", + "arith.minimumf", + "arith.minnumf", + "arith.minsi", + "arith.minui", + "arith.mulf", + "arith.muli", + "arith.mulsi_extended", + "arith.mului_extended", + "arith.negf", + "arith.ori", + "arith.remf", + "arith.remsi", + "arith.remui", + "arith.scaling_truncf", + "arith.select", + "arith.shli", + "arith.shrsi", + "arith.shrui", + "arith.sitofp", + "arith.subf", + "arith.subi", + "arith.truncf", + "arith.trunci", + "arith.uitofp", + "arith.xori", +}; + +fn expectPrints(expected: []const u8, value: anytype) !void { + var buf: [128]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print("{f}", .{value}); + try std.testing.expectEqualStrings(expected, w.buffered()); +} + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(arith); + inline for (comptime std.meta.declarations(arith)) |decl| { + const T = @field(arith, decl); + if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); + } +} + +test "arith fastmath attribute is built through the C API" { + const ctx = try mlir.Context.init(.{}); + defer ctx.deinit(); + + const cases = [_]struct { arith.FastMathFlags, []const u8 }{ + .{ .none, "#arith.fastmath" }, + .{ .fast, "#arith.fastmath" }, + .{ .{ .reassoc = true }, "#arith.fastmath" }, + .{ .{ .nnan = true, .ninf = true, .afn = true }, "#arith.fastmath" }, + .{ .{ .nsz = true, .arcp = true, .contract = true }, "#arith.fastmath" }, + }; + for (cases) |case| { + const attr = try arith.FastMathAttr.get(ctx, .{ .value = case[0] }); + try std.testing.expectEqual(case[0], attr.value()); + try std.testing.expect(attr.attribute().isA(arith.FastMathAttr) != null); + try expectPrints(case[1], attr); + } + try std.testing.expectError(error.InvalidMlir, arith.FastMathAttr.get(ctx, .{ .value = .{ ._padding = 1 } })); +} + +test "fastmath names the attribute of a float operation" { + const ctx = try mlir.Context.init(.{}); + defer ctx.deinit(); + var buf: [64]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print("{f}", .{mlir.Attribute.dict(ctx, &.{try arith.fastmath(ctx, .fast)})}); + try std.testing.expectEqualStrings("{fastmath = #arith.fastmath}", w.buffered()); +} + +test "overflowFlags names the attribute of an integer operation" { + const ctx = try mlir.Context.init(.{}); + defer ctx.deinit(); + var buf: [64]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print("{f}", .{mlir.Attribute.dict(ctx, &.{try arith.overflowFlags(ctx, .{ .nsw = true })})}); + try std.testing.expectEqualStrings("{overflowFlags = #arith.overflow}", w.buffered()); +} + +test "arith overflow attribute is built through the C API" { + const ctx = try mlir.Context.init(.{}); + defer ctx.deinit(); + + const cases = [_]struct { arith.IntegerOverflowFlags, []const u8 }{ + .{ .none, "#arith.overflow" }, + .{ .{ .nsw = true }, "#arith.overflow" }, + .{ .{ .nsw = true, .nuw = true }, "#arith.overflow" }, + }; + for (cases) |case| { + const attr = try arith.IntegerOverflowAttr.get(ctx, .{ .value = case[0] }); + try std.testing.expectEqual(case[0], attr.value()); + try std.testing.expect(attr.attribute().isA(arith.IntegerOverflowAttr) != null); + try expectPrints(case[1], attr); + } + try std.testing.expectError(error.InvalidMlir, arith.IntegerOverflowAttr.get(ctx, .{ .value = .{ ._padding = 1 } })); +} + +test "every bound operation is registered" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + arith.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} diff --git a/mlir/dialects/cf/BUILD.bazel b/mlir/dialects/cf/BUILD.bazel new file mode 100644 index 00000000..3d308f63 --- /dev/null +++ b/mlir/dialects/cf/BUILD.bazel @@ -0,0 +1,23 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "cf", + import_name = "mlir/dialects/cf", + main = "cf.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//mlir", + "//stdx", + "@llvm-project//mlir:CAPICF", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":cf", + "//mlir", + ], +) diff --git a/mlir/dialects/cf/cf.zig b/mlir/dialects/cf/cf.zig new file mode 100644 index 00000000..9a63297c --- /dev/null +++ b/mlir/dialects/cf/cf.zig @@ -0,0 +1,168 @@ +const std = @import("std"); + +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("cf"); +} + +// ============================================================================= +// cf.assert +// ============================================================================= + +/// cf.assert %cond, "message" — aborts execution with `msg` when `cond` is +/// false. `cond` is i1. +pub fn assert( + ctx: *mlir.Context, + cond: *const mlir.Value, + msg: []const u8, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "cf.assert", .{ + .operands = .{ .flat = &.{cond} }, + .attributes = &.{ + .named(ctx, "msg", .string(ctx, msg)), + }, + .location = location, + }); +} + +// ============================================================================= +// cf.br +// ============================================================================= + +/// cf.br ^dest(operands...) — unconditional branch. Terminator of its block. +/// The types of `dest_operands` must match the block arguments of `dest`. +pub fn br( + ctx: *mlir.Context, + dest: *mlir.Block, + dest_operands: []const *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + _ = ctx; + var state: mlir.OperationState = .init("cf.br", location); + state.addOperands(dest_operands); + var successors: [1]*const mlir.Block = .{dest}; + state.addSuccessors(&successors); + return mlir.Operation.init(&state) catch @panic("Failed to create cf.br"); +} + +// ============================================================================= +// cf.cond_br +// ============================================================================= + +/// cf.cond_br %cond, ^trueDest(trueOps...), ^falseDest(falseOps...) — branch +/// on an i1. Terminator of its block. +/// +/// Operand layout (flat) is `[cond, *trueOperands, *falseOperands]` with an +/// `operandSegmentSizes = [1, |trueOperands|, |falseOperands|]` attribute per +/// the `AttrSizedOperandSegments` trait on the op. +/// +/// `branch_weights` (optional) lets profile-guided lowering bias the branch; +/// when provided, its length must be 2 (true weight, false weight). +pub fn cond_br( + ctx: *mlir.Context, + cond: *const mlir.Value, + true_dest: *mlir.Block, + true_operands: []const *const mlir.Value, + false_dest: *mlir.Block, + false_operands: []const *const mlir.Value, + opts: struct { + branch_weights: ?[]const i32 = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 128) = .empty; + operands_buf.appendAssumeCapacity(cond); + operands_buf.appendSliceAssumeCapacity(true_operands); + operands_buf.appendSliceAssumeCapacity(false_operands); + + const seg_sizes = [3]i32{ 1, @intCast(true_operands.len), @intCast(false_operands.len) }; + + var state: mlir.OperationState = .init("cf.cond_br", location); + state.addOperands(operands_buf.constSlice()); + var successors: [2]*const mlir.Block = .{ true_dest, false_dest }; + state.addSuccessors(&successors); + + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attrs.appendAssumeCapacity(.named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes))); + if (opts.branch_weights) |w| { + std.debug.assert(w.len == 2); + attrs.appendAssumeCapacity(.named(ctx, "branch_weights", .denseArray(ctx, .i32, w))); + } + state.addAttributes(attrs.constSlice()); + + return mlir.Operation.init(&state) catch @panic("Failed to create cf.cond_br"); +} + +// ============================================================================= +// cf.switch +// ============================================================================= + +/// cf.switch %flag : iN, [ default: ^def(defOps...), caseN: ^bbN(opsN...) ] +/// +/// `flag` is any integer type. `case_values` are matched signed; the matching +/// case's block is jumped to (with the corresponding operand segment +/// forwarded). If no case matches, `default_dest` is taken with +/// `default_operands`. +/// +/// Operand layout: `[flag, *defaultOperands, *caseOperands[0], ..., +/// *caseOperands[N-1]]`. `case_operand_segments[i]` carries the length of +/// each case's operand segment. The `AttrSizedOperandSegments` trait tracks +/// the top-level split between `defaultOperands` and the lumped +/// `caseOperands`. +pub fn switch_( + ctx: *mlir.Context, + flag: *const mlir.Value, + default_dest: *mlir.Block, + default_operands: []const *const mlir.Value, + case_values: []const i32, + case_dests: []const *mlir.Block, + case_operands: []const []const *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + std.debug.assert(case_values.len == case_dests.len); + std.debug.assert(case_values.len == case_operands.len); + + var operands_buf: stdx.BoundedArray(*const mlir.Value, 256) = .empty; + operands_buf.appendAssumeCapacity(flag); + operands_buf.appendSliceAssumeCapacity(default_operands); + + var case_seg_sizes: stdx.BoundedArray(i32, 64) = .empty; + var case_total: i32 = 0; + for (case_operands) |segment| { + operands_buf.appendSliceAssumeCapacity(segment); + case_seg_sizes.appendAssumeCapacity(@intCast(segment.len)); + case_total += @intCast(segment.len); + } + + // Top-level AttrSizedOperandSegments: [flag=1, defaultOperands, caseOperands lumped]. + const top_seg_sizes = [3]i32{ 1, @intCast(default_operands.len), case_total }; + + var state: mlir.OperationState = .init("cf.switch", location); + state.addOperands(operands_buf.constSlice()); + + var successors_buf: stdx.BoundedArray(*const mlir.Block, 64) = .empty; + successors_buf.appendAssumeCapacity(default_dest); + for (case_dests) |d| successors_buf.appendAssumeCapacity(d); + state.addSuccessors(@constCast(successors_buf.constSlice())); + + // case_values is stored as a DenseIntElementsAttr (tensor); + // the td uses `AnyIntElementsAttr`, and the parser prints decimal literals + // matching the flag's bit width. We use i32 for the common path (matches + // Python Triton's flag type for pid compares). + const shaped = mlir.RankedTensorType.get( + &.{@intCast(case_values.len)}, + .int(ctx, .i32), + null, + ).shaped(); + + state.addAttributes(&.{ + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &top_seg_sizes)), + .named(ctx, "case_values", .denseElements(shaped, case_values)), + .named(ctx, "case_operand_segments", .denseArray(ctx, .i32, case_seg_sizes.constSlice())), + }); + + return mlir.Operation.init(&state) catch @panic("Failed to create cf.switch"); +} diff --git a/mlir/dialects/cf/test.zig b/mlir/dialects/cf/test.zig new file mode 100644 index 00000000..5a91f8f4 --- /dev/null +++ b/mlir/dialects/cf/test.zig @@ -0,0 +1,31 @@ +const std = @import("std"); + +const cf = @import("mlir/dialects/cf"); +const mlir = @import("mlir"); + +const operation_names: []const []const u8 = &.{ + "cf.assert", + "cf.br", + "cf.cond_br", + "cf.switch", +}; + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(cf); + inline for (comptime std.meta.declarations(cf)) |decl| { + const T = @field(cf, decl); + if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); + } +} + +test "every bound operation is registered" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + cf.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} diff --git a/mlir/dialects/cuda/BUILD.bazel b/mlir/dialects/cuda/BUILD.bazel new file mode 100644 index 00000000..bcfabc4a --- /dev/null +++ b/mlir/dialects/cuda/BUILD.bazel @@ -0,0 +1,193 @@ +load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library", "td_library") +load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +package(default_visibility = ["//visibility:public"]) + +cc_library( + name = "capi", + srcs = ["capi.cc"], + hdrs = ["capi.h"], + tags = ["manual"], + deps = [ + ":dialect", + ":headers", + "@llvm-project//mlir:CAPIIR", + ], +) + +zig_library( + name = "cuda", + srcs = [ + "ops.zig", + ], + import_name = "mlir/dialects/cuda", + main = "cuda.zig", + tags = ["manual"], + deps = [ + ":capi", + "//mlir", + "//stdx", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":cuda", + "//mlir", + ], +) + +td_library( + name = "td_files", + srcs = [ + "CudaAttrs.td", + "CudaDialect.td", + "CudaEnums.td", + "CudaOps.td", + "CudaTypes.td", + ], + includes = ["/"], + deps = [ + "@llvm-project//mlir:InferTypeOpInterfaceTdFiles", + "@llvm-project//mlir:OpBaseTdFiles", + "@llvm-project//mlir:SideEffectInterfacesTdFiles", + ], +) + +gentbl_cc_library( + name = "dialect_inc_gen", + tbl_outs = [ + ( + [ + "-gen-dialect-decls", + "-dialect=cuda", + ], + "CudaDialect.h.inc", + ), + ( + [ + "-gen-dialect-defs", + "-dialect=cuda", + ], + "CudaDialect.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "CudaDialect.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "enums_inc_gen", + tbl_outs = [ + ( + ["-gen-enum-decls"], + "CudaEnums.h.inc", + ), + ( + ["-gen-enum-defs"], + "CudaEnums.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "CudaEnums.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "attrs_inc_gen", + tbl_outs = [ + ( + [ + "-gen-attrdef-decls", + "-attrdefs-dialect=cuda", + ], + "CudaAttrs.h.inc", + ), + ( + [ + "-gen-attrdef-defs", + "-attrdefs-dialect=cuda", + ], + "CudaAttrs.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "CudaAttrs.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "types_inc_gen", + tbl_outs = [ + ( + [ + "-gen-typedef-decls", + "-typedefs-dialect=cuda", + ], + "CudaTypes.h.inc", + ), + ( + [ + "-gen-typedef-defs", + "-typedefs-dialect=cuda", + ], + "CudaTypes.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "CudaTypes.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "ops_inc_gen", + tbl_outs = [ + ( + ["-gen-op-decls"], + "CudaOps.h.inc", + ), + ( + ["-gen-op-defs"], + "CudaOps.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "CudaOps.td", + deps = [":td_files"], +) + +cc_library( + name = "headers", + hdrs = ["CudaDialect.h"], + deps = [ + ":attrs_inc_gen", + ":dialect_inc_gen", + ":enums_inc_gen", + ":ops_inc_gen", + ":types_inc_gen", + "@llvm-project//mlir:BytecodeOpInterface", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:InferTypeOpInterface", + "@llvm-project//mlir:LLVMDialect", + "@llvm-project//mlir:SideEffectInterfaces", + ], +) + +cc_library( + name = "dialect", + srcs = [ + "CudaDialect.cpp", + "CudaInference.cpp", + ], + deps = [ + ":headers", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:InferTypeOpInterface", + "@llvm-project//mlir:LLVMDialect", + ], +) diff --git a/mlir/dialects/cuda/CudaAttrs.td b/mlir/dialects/cuda/CudaAttrs.td new file mode 100644 index 00000000..4d5e6f08 --- /dev/null +++ b/mlir/dialects/cuda/CudaAttrs.td @@ -0,0 +1,68 @@ +//===- CudaAttrs.td - CUDA host attributes of the CuTe DSL -*- tablegen -*-===// +// +// Generated for nvidia-cutlass-dsl 4.8.0 from the DSL compiler +// (`_cutlass_ir.cu13` parser messages, its mnemonics, and probing its +// attribute parser and printer); do not edit by hand. +// +// Every cuda attribute the compiler's parser knows. `device_attributes` and +// `func_attributes` print as `#cuda`: the compiler +// parses any dictionary there and prints back only the entries it keeps, +// whose keys were not recovered; these keep the dictionary as written, and +// print a space before it, which the compiler parses too. +// +//===----------------------------------------------------------------------===// + +#ifndef MLIR_DIALECTS_CUDA_CUDAATTRS_TD +#define MLIR_DIALECTS_CUDA_CUDAATTRS_TD + +include "mlir/dialects/cuda/CudaDialect.td" +include "mlir/dialects/cuda/CudaEnums.td" + +defvar Cuda_Ns = "::mlir::cutlass_compiler::cuda"; + +def Cuda_AssumeKernelAttr : Cuda_Attr<"AssumeKernel", "assume_kernel_attr"> { + let summary = "Whether `cuda.launch` may assume its callee is a kernel"; + let description = "`#cuda.assume_kernel_attr`"; + let parameters = (ins "bool":$value); + let assemblyFormat = "`<` $value `>`"; +} + +def Cuda_ComputeTargetAttr : Cuda_Attr<"ComputeTarget", "compute_target"> { + let summary = "Representation, portability and architectures of a compiled executable"; + let description = "`#cuda.compute_target`"; + let parameters = (ins + AttrParameter:$representation, + AttrParameter:$portability, + ArrayRefParameter:$archs + ); + let assemblyFormat = "`<` $representation `,` $portability `,` `[` $archs `]` `>`"; +} + +def Cuda_DevMaxSharedMemoryOptinAttr + : Cuda_Attr<"DevMaxSharedMemoryOptin", "dev_max_shared_memory_optin"> { + let summary = "The device's opt-in maximum of shared memory per block"; + let description = "`#cuda.dev_max_shared_memory_optin`"; +} + +def Cuda_DeviceAttributesAttr : Cuda_Attr<"DeviceAttributes", "device_attributes"> { + let summary = "CUDA device attributes"; + let description = "`#cuda`"; + let parameters = (ins "::mlir::DictionaryAttr":$values); + let assemblyFormat = "$values"; +} + +def Cuda_ExecutableAttr : Cuda_Attr<"Executable", "executable"> { + let summary = "Format of a compiled executable"; + let description = "`#cuda.executable`"; + let parameters = (ins AttrParameter:$format); + let assemblyFormat = "`<` $format `>`"; +} + +def Cuda_FuncAttributesAttr : Cuda_Attr<"FuncAttributes", "func_attributes"> { + let summary = "CUDA function attributes of a kernel"; + let description = "`#cuda`"; + let parameters = (ins "::mlir::DictionaryAttr":$values); + let assemblyFormat = "$values"; +} + +#endif // MLIR_DIALECTS_CUDA_CUDAATTRS_TD diff --git a/mlir/dialects/cuda/CudaDialect.cpp b/mlir/dialects/cuda/CudaDialect.cpp new file mode 100644 index 00000000..0ce2910d --- /dev/null +++ b/mlir/dialects/cuda/CudaDialect.cpp @@ -0,0 +1,38 @@ +//===- CudaDialect.cpp - NVIDIA's cuda dialect of the CuTe DSL ------------===// + +#include "mlir/dialects/cuda/CudaDialect.h" + +#include "mlir/IR/Builders.h" +#include "mlir/IR/DialectImplementation.h" +#include "llvm/ADT/TypeSwitch.h" + +using namespace mlir; +using namespace mlir::cutlass_compiler::cuda; + +#include "mlir/dialects/cuda/CudaDialect.cpp.inc" + +#include "mlir/dialects/cuda/CudaEnums.cpp.inc" + +#define GET_ATTRDEF_CLASSES +#include "mlir/dialects/cuda/CudaAttrs.cpp.inc" + +#define GET_TYPEDEF_CLASSES +#include "mlir/dialects/cuda/CudaTypes.cpp.inc" + +#define GET_OP_CLASSES +#include "mlir/dialects/cuda/CudaOps.cpp.inc" + +void CudaDialect::initialize() { + addAttributes< +#define GET_ATTRDEF_LIST +#include "mlir/dialects/cuda/CudaAttrs.cpp.inc" + >(); + addTypes< +#define GET_TYPEDEF_LIST +#include "mlir/dialects/cuda/CudaTypes.cpp.inc" + >(); + addOperations< +#define GET_OP_LIST +#include "mlir/dialects/cuda/CudaOps.cpp.inc" + >(); +} diff --git a/mlir/dialects/cuda/CudaDialect.h b/mlir/dialects/cuda/CudaDialect.h new file mode 100644 index 00000000..a96cdd17 --- /dev/null +++ b/mlir/dialects/cuda/CudaDialect.h @@ -0,0 +1,30 @@ +//===- CudaDialect.h - NVIDIA's cuda dialect of the CuTe DSL ----*- C++ -*-===// + +#ifndef MLIR_DIALECTS_CUDA_CUDADIALECT_H +#define MLIR_DIALECTS_CUDA_CUDADIALECT_H + +#include "mlir/Bytecode/BytecodeOpInterface.h" +#include "mlir/Dialect/LLVMIR/LLVMAttrs.h" +#include "mlir/Dialect/LLVMIR/LLVMDialect.h" +#include "mlir/Dialect/LLVMIR/LLVMTypes.h" +#include "mlir/IR/BuiltinAttributes.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/DialectImplementation.h" +#include "mlir/IR/OpImplementation.h" +#include "mlir/Interfaces/InferTypeOpInterface.h" +#include "mlir/Interfaces/SideEffectInterfaces.h" + +#include "mlir/dialects/cuda/CudaDialect.h.inc" + +#include "mlir/dialects/cuda/CudaEnums.h.inc" + +#define GET_ATTRDEF_CLASSES +#include "mlir/dialects/cuda/CudaAttrs.h.inc" + +#define GET_TYPEDEF_CLASSES +#include "mlir/dialects/cuda/CudaTypes.h.inc" + +#define GET_OP_CLASSES +#include "mlir/dialects/cuda/CudaOps.h.inc" + +#endif // MLIR_DIALECTS_CUDA_CUDADIALECT_H diff --git a/mlir/dialects/cuda/CudaDialect.td b/mlir/dialects/cuda/CudaDialect.td new file mode 100644 index 00000000..939e2d85 --- /dev/null +++ b/mlir/dialects/cuda/CudaDialect.td @@ -0,0 +1,38 @@ +//===- CudaDialect.td - CUDA host dialect of the CuTe DSL --*- tablegen -*-===// + +#ifndef MLIR_DIALECTS_CUDA_CUDADIALECT_TD +#define MLIR_DIALECTS_CUDA_CUDADIALECT_TD + +include "mlir/IR/AttrTypeBase.td" +include "mlir/IR/DialectBase.td" +include "mlir/IR/OpBase.td" + +def Cuda_Dialect : Dialect { + let name = "cuda"; + let cppNamespace = "::mlir::cutlass_compiler::cuda"; + let summary = "NVIDIA's CUDA host-program dialect used by CuTe-DSL"; + let description = [{ + This dialect mirrors the operation, type, attribute and enum registry of + the proprietary `cuda` dialect embedded in the nvidia-cutlass-dsl 4.8.0 + compiler: kernels, launch configurations and the CUDA runtime and driver + calls a host function makes. + }]; + let dependentDialects = ["::mlir::LLVM::LLVMDialect"]; + let useDefaultTypePrinterParser = 1; + let useDefaultAttributePrinterParser = 1; +} + +class Cuda_Op traits = []> + : Op; + +class Cuda_Type + : TypeDef { + let mnemonic = typeMnemonic; +} + +class Cuda_Attr + : AttrDef { + let mnemonic = attrMnemonic; +} + +#endif // MLIR_DIALECTS_CUDA_CUDADIALECT_TD diff --git a/mlir/dialects/cuda/CudaEnums.td b/mlir/dialects/cuda/CudaEnums.td new file mode 100644 index 00000000..f7980167 --- /dev/null +++ b/mlir/dialects/cuda/CudaEnums.td @@ -0,0 +1,336 @@ +// The cuda dialect's enums, generated from the nvidia-cutlass-dsl 4.8.0 enum +// bindings (`_cuda_enum_gen.py`); do not edit. +// +// Case values and spellings are the compiler's. An enum the bindings build +// as a dialect attribute also has its EnumAttr, printed `#` +// or, where the compiler prints it so, `#nvvm`. + +#ifndef MLIR_DIALECTS_CUDA_CUDAENUMS_TD +#define MLIR_DIALECTS_CUDA_CUDAENUMS_TD + +include "mlir/dialects/cuda/CudaDialect.td" +include "mlir/IR/EnumAttr.td" + +def Cuda_ArchPortabilityEnum : I32EnumAttr<"ArchPortability", "Enumerated type describing portability", [ + I32EnumAttrCase<"portable", 0, "portable">, + I32EnumAttrCase<"conditional", 1, "conditional">, + I32EnumAttrCase<"family", 2, "family"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cuda"; + let genSpecializedAttr = 0; +} + +def Cuda_CUDeviceAttrEnum : I32EnumAttr<"CUDeviceAttr", "Enumerated type describing CUDA device attributes can be set through Cuda Dialect", [ + I32EnumAttrCase<"maxThreadsPerBlock", 1, "maxThreadsPerBlock">, + I32EnumAttrCase<"maxBlockDimX", 2, "maxBlockDimX">, + I32EnumAttrCase<"maxBlockDimY", 3, "maxBlockDimY">, + I32EnumAttrCase<"maxBlockDimZ", 4, "maxBlockDimZ">, + I32EnumAttrCase<"maxGridDimX", 5, "maxGridDimX">, + I32EnumAttrCase<"maxGridDimY", 6, "maxGridDimY">, + I32EnumAttrCase<"maxGridDimZ", 7, "maxGridDimZ">, + I32EnumAttrCase<"maxSharedMemoryPerBlock", 8, "maxSharedMemoryPerBlock">, + I32EnumAttrCase<"totalConstantMemory", 9, "totalConstantMemory">, + I32EnumAttrCase<"warpSize", 10, "warpSize">, + I32EnumAttrCase<"maxPitch", 11, "maxPitch">, + I32EnumAttrCase<"maxRegistersPerBlock", 12, "maxRegistersPerBlock">, + I32EnumAttrCase<"clockRate", 13, "clockRate">, + I32EnumAttrCase<"textureAlignment", 14, "textureAlignment">, + I32EnumAttrCase<"gpuOverlap", 15, "gpuOverlap">, + I32EnumAttrCase<"multiProcessorCount", 16, "multiProcessorCount">, + I32EnumAttrCase<"kernelExecTimeout", 17, "kernelExecTimeout">, + I32EnumAttrCase<"integrated", 18, "integrated">, + I32EnumAttrCase<"canMapHostMemory", 19, "canMapHostMemory">, + I32EnumAttrCase<"computeMode", 20, "computeMode">, + I32EnumAttrCase<"maxTexture1DWidth", 21, "maxTexture1DWidth">, + I32EnumAttrCase<"maxTexture2DWidth", 22, "maxTexture2DWidth">, + I32EnumAttrCase<"maxTexture2DHeight", 23, "maxTexture2DHeight">, + I32EnumAttrCase<"maxTexture3DWidth", 24, "maxTexture3DWidth">, + I32EnumAttrCase<"maxTexture3DHeight", 25, "maxTexture3DHeight">, + I32EnumAttrCase<"maxTexture3DDepth", 26, "maxTexture3DDepth">, + I32EnumAttrCase<"maxTexture2DLayeredWidth", 27, "maxTexture2DLayeredWidth">, + I32EnumAttrCase<"maxTexture2DLayeredHeight", 28, "maxTexture2DLayeredHeight">, + I32EnumAttrCase<"maxTexture2DLayeredLayers", 29, "maxTexture2DLayeredLayers">, + I32EnumAttrCase<"surfaceAlignment", 30, "surfaceAlignment">, + I32EnumAttrCase<"concurrentKernels", 31, "concurrentKernels">, + I32EnumAttrCase<"eccEnabled", 32, "eccEnabled">, + I32EnumAttrCase<"pciBusId", 33, "pciBusId">, + I32EnumAttrCase<"pciDeviceId", 34, "pciDeviceId">, + I32EnumAttrCase<"tccDriver", 35, "tccDriver">, + I32EnumAttrCase<"memoryClockRate", 36, "memoryClockRate">, + I32EnumAttrCase<"globalMemoryBusWidth", 37, "globalMemoryBusWidth">, + I32EnumAttrCase<"l2CacheSize", 38, "l2CacheSize">, + I32EnumAttrCase<"maxThreadsPerMultiProcessor", 39, "maxThreadsPerMultiProcessor">, + I32EnumAttrCase<"asyncEngineCount", 40, "asyncEngineCount">, + I32EnumAttrCase<"unifiedAddressing", 41, "unifiedAddressing">, + I32EnumAttrCase<"maxTexture1DLayeredWidth", 42, "maxTexture1DLayeredWidth">, + I32EnumAttrCase<"maxTexture1DLayeredLayers", 43, "maxTexture1DLayeredLayers">, + I32EnumAttrCase<"maxTexture2DGatherWidth", 45, "maxTexture2DGatherWidth">, + I32EnumAttrCase<"maxTexture2DGatherHeight", 46, "maxTexture2DGatherHeight">, + I32EnumAttrCase<"maxTexture3DWidthAlt", 47, "maxTexture3DWidthAlt">, + I32EnumAttrCase<"maxTexture3DHeightAlt", 48, "maxTexture3DHeightAlt">, + I32EnumAttrCase<"maxTexture3DDepthAlt", 49, "maxTexture3DDepthAlt">, + I32EnumAttrCase<"pciDomainId", 50, "pciDomainId">, + I32EnumAttrCase<"texturePitchAlignment", 51, "texturePitchAlignment">, + I32EnumAttrCase<"maxTextureCubemapWidth", 52, "maxTextureCubemapWidth">, + I32EnumAttrCase<"maxTextureCubemapLayeredWidth", 53, "maxTextureCubemapLayeredWidth">, + I32EnumAttrCase<"maxTextureCubemapLayeredLayers", 54, "maxTextureCubemapLayeredLayers">, + I32EnumAttrCase<"maxSurface1DWidth", 55, "maxSurface1DWidth">, + I32EnumAttrCase<"maxSurface2DWidth", 56, "maxSurface2DWidth">, + I32EnumAttrCase<"maxSurface2DHeight", 57, "maxSurface2DHeight">, + I32EnumAttrCase<"maxSurface3DWidth", 58, "maxSurface3DWidth">, + I32EnumAttrCase<"maxSurface3DHeight", 59, "maxSurface3DHeight">, + I32EnumAttrCase<"maxSurface3DDepth", 60, "maxSurface3DDepth">, + I32EnumAttrCase<"maxSurface1DLayeredWidth", 61, "maxSurface1DLayeredWidth">, + I32EnumAttrCase<"maxSurface1DLayeredLayers", 62, "maxSurface1DLayeredLayers">, + I32EnumAttrCase<"maxSurface2DLayeredWidth", 63, "maxSurface2DLayeredWidth">, + I32EnumAttrCase<"maxSurface2DLayeredHeight", 64, "maxSurface2DLayeredHeight">, + I32EnumAttrCase<"maxSurface2DLayeredLayers", 65, "maxSurface2DLayeredLayers">, + I32EnumAttrCase<"maxSurfaceCubemapWidth", 66, "maxSurfaceCubemapWidth">, + I32EnumAttrCase<"maxSurfaceCubemapLayeredWidth", 67, "maxSurfaceCubemapLayeredWidth">, + I32EnumAttrCase<"maxSurfaceCubemapLayeredLayers", 68, "maxSurfaceCubemapLayeredLayers">, + I32EnumAttrCase<"maxTexture1DLinearWidth", 69, "maxTexture1DLinearWidth">, + I32EnumAttrCase<"maxTexture2DLinearWidth", 70, "maxTexture2DLinearWidth">, + I32EnumAttrCase<"maxTexture2DLinearHeight", 71, "maxTexture2DLinearHeight">, + I32EnumAttrCase<"maxTexture2DLinearPitch", 72, "maxTexture2DLinearPitch">, + I32EnumAttrCase<"maxTexture2DMipmappedWidth", 73, "maxTexture2DMipmappedWidth">, + I32EnumAttrCase<"maxTexture2DMipmappedHeight", 74, "maxTexture2DMipmappedHeight">, + I32EnumAttrCase<"computeCapabilityMajor", 75, "computeCapabilityMajor">, + I32EnumAttrCase<"computeCapabilityMinor", 76, "computeCapabilityMinor">, + I32EnumAttrCase<"maxTexture1DMipmappedWidth", 77, "maxTexture1DMipmappedWidth">, + I32EnumAttrCase<"streamPrioritiesSupported", 78, "streamPrioritiesSupported">, + I32EnumAttrCase<"globalL1CacheSupported", 79, "globalL1CacheSupported">, + I32EnumAttrCase<"localL1CacheSupported", 80, "localL1CacheSupported">, + I32EnumAttrCase<"maxSharedMemoryPerMultiprocessor", 81, "maxSharedMemoryPerMultiprocessor">, + I32EnumAttrCase<"maxRegistersPerMultiprocessor", 82, "maxRegistersPerMultiprocessor">, + I32EnumAttrCase<"managedMemory", 83, "managedMemory">, + I32EnumAttrCase<"isMultiGpuBoard", 84, "isMultiGpuBoard">, + I32EnumAttrCase<"multiGpuBoardGroupID", 85, "multiGpuBoardGroupID">, + I32EnumAttrCase<"hostNativeAtomicSupported", 86, "hostNativeAtomicSupported">, + I32EnumAttrCase<"singleToDoublePrecisionPerfRatio", 87, "singleToDoublePrecisionPerfRatio">, + I32EnumAttrCase<"pageableMemoryAccess", 88, "pageableMemoryAccess">, + I32EnumAttrCase<"concurrentManagedAccess", 89, "concurrentManagedAccess">, + I32EnumAttrCase<"computePreemptionSupported", 90, "computePreemptionSupported">, + I32EnumAttrCase<"canUseHostPointerForRegisteredMem", 91, "canUseHostPointerForRegisteredMem">, + I32EnumAttrCase<"reserved92", 92, "reserved92">, + I32EnumAttrCase<"reserved93", 93, "reserved93">, + I32EnumAttrCase<"reserved94", 94, "reserved94">, + I32EnumAttrCase<"cooperativeLaunch", 95, "cooperativeLaunch">, + I32EnumAttrCase<"reserved96", 96, "reserved96">, + I32EnumAttrCase<"maxSharedMemoryPerBlockOptin", 97, "maxSharedMemoryPerBlockOptin">, + I32EnumAttrCase<"canFlushRemoteWrites", 98, "canFlushRemoteWrites">, + I32EnumAttrCase<"hostRegisterSupported", 99, "hostRegisterSupported">, + I32EnumAttrCase<"pageableMemoryAccessUsesHostPageTables", 100, "pageableMemoryAccessUsesHostPageTables">, + I32EnumAttrCase<"directManagedMemAccessFromHost", 101, "directManagedMemAccessFromHost">, + I32EnumAttrCase<"maxBlocksPerMultiprocessor", 106, "maxBlocksPerMultiprocessor">, + I32EnumAttrCase<"maxPersistingL2CacheSize", 108, "maxPersistingL2CacheSize">, + I32EnumAttrCase<"maxAccessPolicyWindowSize", 109, "maxAccessPolicyWindowSize">, + I32EnumAttrCase<"reservedSharedMemoryPerBlock", 111, "reservedSharedMemoryPerBlock">, + I32EnumAttrCase<"sparseCudaArraySupported", 112, "sparseCudaArraySupported">, + I32EnumAttrCase<"hostRegisterReadOnlySupported", 113, "hostRegisterReadOnlySupported">, + I32EnumAttrCase<"timelineSemaphoreInteropSupported", 114, "timelineSemaphoreInteropSupported">, + I32EnumAttrCase<"memoryPoolsSupported", 115, "memoryPoolsSupported">, + I32EnumAttrCase<"gpuDirectRdmaSupported", 116, "gpuDirectRdmaSupported">, + I32EnumAttrCase<"gpuDirectRdmaFlushWritesOptions", 117, "gpuDirectRdmaFlushWritesOptions">, + I32EnumAttrCase<"gpuDirectRdmaWritesOrdering", 118, "gpuDirectRdmaWritesOrdering">, + I32EnumAttrCase<"memoryPoolSupportedHandleTypes", 119, "memoryPoolSupportedHandleTypes">, + I32EnumAttrCase<"clusterLaunch", 120, "clusterLaunch">, + I32EnumAttrCase<"deferredMappingCudaArraySupported", 121, "deferredMappingCudaArraySupported">, + I32EnumAttrCase<"reserved122", 122, "reserved122">, + I32EnumAttrCase<"reserved123", 123, "reserved123">, + I32EnumAttrCase<"reserved124", 124, "reserved124">, + I32EnumAttrCase<"ipcEventSupport", 125, "ipcEventSupport">, + I32EnumAttrCase<"memSyncDomainCount", 126, "memSyncDomainCount">, + I32EnumAttrCase<"reserved127", 127, "reserved127">, + I32EnumAttrCase<"reserved128", 128, "reserved128">, + I32EnumAttrCase<"reserved129", 129, "reserved129">, + I32EnumAttrCase<"numaConfig", 130, "numaConfig">, + I32EnumAttrCase<"numaId", 131, "numaId">, + I32EnumAttrCase<"reserved132", 132, "reserved132">, + I32EnumAttrCase<"mpsEnabled", 133, "mpsEnabled">, + I32EnumAttrCase<"hostNumaId", 134, "hostNumaId">, + I32EnumAttrCase<"d3d12CigSupported", 135, "d3d12CigSupported">, + I32EnumAttrCase<"vulkanCigSupported", 138, "vulkanCigSupported">, + I32EnumAttrCase<"gpuPciDeviceId", 139, "gpuPciDeviceId">, + I32EnumAttrCase<"gpuPciSubsystemId", 140, "gpuPciSubsystemId">, + I32EnumAttrCase<"reserved141", 141, "reserved141">, + I32EnumAttrCase<"hostNumaMemoryPoolsSupported", 142, "hostNumaMemoryPoolsSupported">, + I32EnumAttrCase<"hostNumaMultinodeIpcSupported", 143, "hostNumaMultinodeIpcSupported">, + I32EnumAttrCase<"hostMemoryPoolsSupported", 144, "hostMemoryPoolsSupported">, + I32EnumAttrCase<"reserved145", 145, "reserved145">, + I32EnumAttrCase<"onlyPartialHostNativeAtomicSupported", 147, "onlyPartialHostNativeAtomicSupported">, + I32EnumAttrCase<"cudaDevAttrOversizedSharedMemoryPerBlock", 150, "cudaDevAttrOversizedSharedMemoryPerBlock"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cuda"; + let genSpecializedAttr = 0; +} + +def Cuda_CUFunctionAttributeEnum : I32EnumAttr<"CUFunctionAttribute", "Enumerated type describing CUDA function attributes can be set through Cuda Dialect", [ + I32EnumAttrCase<"max_threads_per_block", 0, "max_threads_per_block">, + I32EnumAttrCase<"shared_size_bytes", 1, "shared_size_bytes">, + I32EnumAttrCase<"const_size_bytes", 2, "const_size_bytes">, + I32EnumAttrCase<"local_size_bytes", 3, "local_size_bytes">, + I32EnumAttrCase<"num_regs", 4, "num_regs">, + I32EnumAttrCase<"ptx_version", 5, "ptx_version">, + I32EnumAttrCase<"binary_version", 6, "binary_version">, + I32EnumAttrCase<"cache_mode_ca", 7, "cache_mode_ca">, + I32EnumAttrCase<"max_dynamic_shared_size_bytes", 8, "max_dynamic_shared_size_bytes">, + I32EnumAttrCase<"preferred_shared_memory_carveout", 9, "preferred_shared_memory_carveout">, + I32EnumAttrCase<"cluster_size_must_be_set", 10, "cluster_size_must_be_set">, + I32EnumAttrCase<"required_cluster_width", 11, "required_cluster_width">, + I32EnumAttrCase<"required_cluster_height", 12, "required_cluster_height">, + I32EnumAttrCase<"required_cluster_depth", 13, "required_cluster_depth">, + I32EnumAttrCase<"non_portable_cluster_size_allowed", 14, "non_portable_cluster_size_allowed">, + I32EnumAttrCase<"cluster_scheduling_policy_preference", 15, "cluster_scheduling_policy_preference"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cuda"; + let genSpecializedAttr = 0; +} + +def Cuda_CudaABIEnum : I32EnumAttr<"CudaABI", "Enumerated of the host library ABI", [ + I32EnumAttrCase<"LP64", 0, "LP64">, + I32EnumAttrCase<"ILP64", 1, "ILP64">, + I32EnumAttrCase<"LLP64", 2, "LLP64"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cuda"; + let genSpecializedAttr = 0; +} + +def Cuda_CudaAccessPropertyEnum : I32EnumAttr<"CudaAccessProperty", "Specifies performance hint with cudaAccessPolicyWindow for hitProp and missProp members", [ + I32EnumAttrCase<"cudaAccessPropertyNormal", 0, "cudaAccessPropertyNormal">, + I32EnumAttrCase<"cudaAccessPropertyStreaming", 1, "cudaAccessPropertyStreaming">, + I32EnumAttrCase<"cudaAccessPropertyPersisting", 2, "cudaAccessPropertyPersisting"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cuda"; + let genSpecializedAttr = 0; +} + +def Cuda_CudaClusterSchedulingPolicyEnum : I32EnumAttr<"CudaClusterSchedulingPolicy", "Enumerated type describing CUDA cluster scheduling policy", [ + I32EnumAttrCase<"cudaClusterSchedulingPolicyDefault", 0, "cudaClusterSchedulingPolicyDefault">, + I32EnumAttrCase<"cudaClusterSchedulingPolicySpread", 1, "cudaClusterSchedulingPolicySpread">, + I32EnumAttrCase<"cudaClusterSchedulingPolicyLoadBalancing", 2, "cudaClusterSchedulingPolicyLoadBalancing"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cuda"; + let genSpecializedAttr = 0; +} + +def Cuda_CudaFuncAttributeEnum : I32EnumAttr<"CudaFuncAttribute", "Enumerated type describing CUDA function attributes can be set through Cuda Dialect", [ + I32EnumAttrCase<"cudaFuncAttributeSharedMemoryMode", 16, "cudaFuncAttributeSharedMemoryMode"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cuda"; + let genSpecializedAttr = 0; +} + +def Cuda_CudaLaunchAttributeIDEnum : I32EnumAttr<"CudaLaunchAttributeID", "Enumerated type mirroring CudaLaunchAttributeID from CUDA Runtime API", [ + I32EnumAttrCase<"cudaLaunchAttributeIgnore", 0, "cudaLaunchAttributeIgnore">, + I32EnumAttrCase<"cudaLaunchAttributeAccessPolicyWindow", 1, "cudaLaunchAttributeAccessPolicyWindow">, + I32EnumAttrCase<"cudaLaunchAttributeCooperative", 2, "cudaLaunchAttributeCooperative">, + I32EnumAttrCase<"cudaLaunchAttributeSynchronizationPolicy", 3, "cudaLaunchAttributeSynchronizationPolicy">, + I32EnumAttrCase<"cudaLaunchAttributeClusterDimension", 4, "cudaLaunchAttributeClusterDimension">, + I32EnumAttrCase<"cudaLaunchAttributeClusterSchedulingPolicyPreference", 5, "cudaLaunchAttributeClusterSchedulingPolicyPreference">, + I32EnumAttrCase<"cudaLaunchAttributeProgrammaticStreamSerialization", 6, "cudaLaunchAttributeProgrammaticStreamSerialization">, + I32EnumAttrCase<"cudaLaunchAttributeProgrammaticEvent", 7, "cudaLaunchAttributeProgrammaticEvent">, + I32EnumAttrCase<"cudaLaunchAttributePriority", 8, "cudaLaunchAttributePriority">, + I32EnumAttrCase<"cudaLaunchAttributeMemSyncDomainMap", 9, "cudaLaunchAttributeMemSyncDomainMap">, + I32EnumAttrCase<"cudaLaunchAttributeMemSyncDomain", 10, "cudaLaunchAttributeMemSyncDomain">, + I32EnumAttrCase<"cudaLaunchAttributePreferredClusterDimension", 11, "cudaLaunchAttributePreferredClusterDimension">, + I32EnumAttrCase<"cudaLaunchAttributeLaunchCompletionEvent", 12, "cudaLaunchAttributeLaunchCompletionEvent">, + I32EnumAttrCase<"cudaLaunchAttributeDeviceUpdatableKernelNode", 13, "cudaLaunchAttributeDeviceUpdatableKernelNode">, + I32EnumAttrCase<"cudaLaunchAttributePreferredSharedMemoryCarveout", 14, "cudaLaunchAttributePreferredSharedMemoryCarveout">, + I32EnumAttrCase<"cudaLaunchAttributeNvlinkUtilCentricScheduling", 16, "cudaLaunchAttributeNvlinkUtilCentricScheduling"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cuda"; + let genSpecializedAttr = 0; +} + +def Cuda_CudaLaunchMemSyncDomainEnum : I32EnumAttr<"CudaLaunchMemSyncDomain", "Enumerated type describing CUDA memory sync domain", [ + I32EnumAttrCase<"cudaLaunchMemSyncDomainDefault", 0, "cudaLaunchMemSyncDomainDefault">, + I32EnumAttrCase<"cudaLaunchMemSyncDomainRemote", 1, "cudaLaunchMemSyncDomainRemote"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cuda"; + let genSpecializedAttr = 0; +} + +def Cuda_CudaSharedMemoryModeEnum : I32EnumAttr<"CudaSharedMemoryMode", "Enumerated type describing CUDA shared memory modes", [ + I32EnumAttrCase<"cudaSharedMemoryModeDefault", 0, "cudaSharedMemoryModeDefault">, + I32EnumAttrCase<"cudaSharedMemoryModeRequirePortable", 1, "cudaSharedMemoryModeRequirePortable">, + I32EnumAttrCase<"cudaSharedMemoryModeAllowNonPortable", 2, "cudaSharedMemoryModeAllowNonPortable">, + I32EnumAttrCase<"cudaSharedMemoryModeAllowOversizedSharedMemory", 3, "cudaSharedMemoryModeAllowOversizedSharedMemory">, + I32EnumAttrCase<"cudaSharedMemoryModePreferOversizedSharedMemory", 4, "cudaSharedMemoryModePreferOversizedSharedMemory"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cuda"; + let genSpecializedAttr = 0; +} + +def Cuda_CudaSynchronizationPolicyEnum : I32EnumAttr<"CudaSynchronizationPolicy", "Enumerated type describing CUDA synchronization policy", [ + I32EnumAttrCase<"cudaSyncPolicyAuto", 0, "cudaSyncPolicyAuto">, + I32EnumAttrCase<"cudaSyncPolicySpin", 1, "cudaSyncPolicySpin">, + I32EnumAttrCase<"cudaSyncPolicyYield", 2, "cudaSyncPolicyYield">, + I32EnumAttrCase<"cudaSyncPolicyBlockingSync", 3, "cudaSyncPolicyBlockingSync"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cuda"; + let genSpecializedAttr = 0; +} + +def Cuda_ExecutableFormatEnum : I32EnumAttr<"ExecutableFormat", "Enumerated type describing the format of an executable", [ + I32EnumAttrCase<"text", 0, "text">, + I32EnumAttrCase<"elf", 1, "elf">, + I32EnumAttrCase<"bitcode", 2, "bitcode">, + I32EnumAttrCase<"fatbin", 3, "fatbin"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cuda"; + let genSpecializedAttr = 0; +} + +def Cuda_ExecutableRepresentationEnum : I32EnumAttr<"ExecutableRepresentation", "Enumerated type describing the representation of an executable", [ + I32EnumAttrCase<"sass", 0, "sass">, + I32EnumAttrCase<"ptx", 2, "ptx">, + I32EnumAttrCase<"nvvm", 3, "nvvm">, + I32EnumAttrCase<"ltoir", 4, "ltoir">, + I32EnumAttrCase<"mlir", 5, "mlir">, + I32EnumAttrCase<"llvm", 6, "llvm">, + I32EnumAttrCase<"hostobj", 7, "hostobj"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cuda"; + let genSpecializedAttr = 0; +} + +def Cuda_GpuArchitectureEnum : I32EnumAttr<"GpuArchitecture", "Enumerated type denoting a GPU Architecture", [ + I32EnumAttrCase<"sm_80", 80, "sm_80">, + I32EnumAttrCase<"sm_86", 86, "sm_86">, + I32EnumAttrCase<"sm_87", 87, "sm_87">, + I32EnumAttrCase<"sm_88", 88, "sm_88">, + I32EnumAttrCase<"sm_89", 89, "sm_89">, + I32EnumAttrCase<"sm_90", 90, "sm_90">, + I32EnumAttrCase<"sm_100", 100, "sm_100">, + I32EnumAttrCase<"sm_101", 101, "sm_101">, + I32EnumAttrCase<"sm_103", 103, "sm_103">, + I32EnumAttrCase<"sm_107", 107, "sm_107">, + I32EnumAttrCase<"sm_110", 110, "sm_110">, + I32EnumAttrCase<"sm_120", 120, "sm_120">, + I32EnumAttrCase<"sm_121", 121, "sm_121"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cuda"; + let genSpecializedAttr = 0; +} + +def Cuda_MemcpyKindEnum : I32EnumAttr<"MemcpyKind", "Enumerated type describing memcpy directions", [ + I32EnumAttrCase<"HostToHost", 0, "HostToHost">, + I32EnumAttrCase<"HostToDevice", 1, "HostToDevice">, + I32EnumAttrCase<"DeviceToHost", 2, "DeviceToHost">, + I32EnumAttrCase<"DeviceToDevice", 3, "DeviceToDevice">, + I32EnumAttrCase<"Default", 4, "Default"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cuda"; + let genSpecializedAttr = 0; +} + +def Cuda_StreamCaptureModeEnumEnum : I32EnumAttr<"StreamCaptureModeEnum", "Enumerated type describing CUDA stream capture modes", [ + I32EnumAttrCase<"Global", 0, "Global">, + I32EnumAttrCase<"ThreadLocal", 1, "ThreadLocal">, + I32EnumAttrCase<"Relaxed", 2, "Relaxed"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cuda"; + let genSpecializedAttr = 0; +} + +#endif // MLIR_DIALECTS_CUDA_CUDAENUMS_TD diff --git a/mlir/dialects/cuda/CudaInference.cpp b/mlir/dialects/cuda/CudaInference.cpp new file mode 100644 index 00000000..1289679e --- /dev/null +++ b/mlir/dialects/cuda/CudaInference.cpp @@ -0,0 +1,17 @@ +//===- CudaInference.cpp - result types the compiler infers ---------------===// +// +// Generated from the nvidia-cutlass-dsl 4.8.0 compiler (its inference +// probed by varying operand types); do not edit. An operation without a +// recovered rule fails inference, so the result types written stand. + +#include "mlir/dialects/cuda/CudaDialect.h" + +using namespace mlir; +using namespace ::mlir::cutlass_compiler::cuda; + +LogicalResult LaunchCfgCreateFromStreamOp::inferReturnTypes( + MLIRContext *, std::optional, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} diff --git a/mlir/dialects/cuda/CudaOps.td b/mlir/dialects/cuda/CudaOps.td new file mode 100644 index 00000000..991814c1 --- /dev/null +++ b/mlir/dialects/cuda/CudaOps.td @@ -0,0 +1,3072 @@ +// The cuda dialect's operations, generated from the nvidia-cutlass-dsl 4.8.0 release; do not edit. +// +// Sources: the operation bindings (`_cuda_ops_gen.py`: names, classes, +// docstrings, operand/result/attribute structure, inference) and the live +// compiler (`_cutlass_ir` loaded from Python): each operand and result +// constraint carries the verifier's wording and accepts every type class the +// verifier let through when probed with one instance of every type; traits +// and interfaces are the registered operation's; memory effects come from +// loop-invariant code motion (hoisted: Pure) and CSE (merged: at most +// reads). Operations whose effects include writes or could not be told +// apart carry none (unknown effects). The compiler's own verifiers (operand +// combinations, shapes, instruction legality) are left to it. +// +// Operations keep the generic assembly form. +// +// Result types the compiler infers without a recovered rule (written types stand): +// cuda.launch_cfg.create_from_stream +// +// Constraints not recovered: + +#ifndef MLIR_DIALECTS_CUDA_CUDAOPS_TD +#define MLIR_DIALECTS_CUDA_CUDAOPS_TD + +include "mlir/dialects/cuda/CudaAttrs.td" +include "mlir/dialects/cuda/CudaEnums.td" +include "mlir/dialects/cuda/CudaTypes.td" + +include "mlir/Interfaces/InferTypeOpInterface.td" +include "mlir/Interfaces/SideEffectInterfaces.td" + +// Operand and result constraints, in the compiler's words. +def Cuda_AnyCUDAStructType : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeDim3Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeExtentType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeKernelNodeParamsType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeMemcpy3DParamsType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimePitchedPtrType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimePosType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeUuidType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValueCooperativeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValueClusterSchedulingPolicyPreferenceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValueNvlinkUtilCentricSchedulingType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowedType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValueMemSyncDomainType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValuePortableClusterSizeModeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValuePriorityType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValueSharedMemCarveoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValueSharedMemoryModeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValueSyncPolicyType>($_self)">, "any CUDA struct type">; +def Cuda_AnyType : Type, "any type">; +def Cuda_CUDACudaDevicePropStructTypeValueTyped : Type($_self)">, "CUDA cudaDeviceProp struct type (value-typed).">, BuildableType<"::mlir::cutlass_compiler::cuda::RuntimeDevicePropType::get($_builder.getContext())">; +def Cuda_CUDACudaDevicePropStructTypeValueTyped2 : Type($_self)">, "CUDA cudaDeviceProp struct type (value-typed).">; +def Cuda_CUDACudaExtentStructTypeValueTyped : Type($_self)">, "CUDA cudaExtent struct type (value-typed).">; +def Cuda_CUDACudaExtentStructTypeValueTyped2 : Type($_self)">, "CUDA cudaExtent struct type (value-typed).">, BuildableType<"::mlir::cutlass_compiler::cuda::RuntimeExtentType::get($_builder.getContext())">; +def Cuda_CUDACudaKernelNodeParamsStructTypeValueTyped : Type($_self)">, "CUDA cudaKernelNodeParams struct type (value-typed).">; +def Cuda_CUDACudaMemcpy3DParmsStructTypeValueTyped : Type($_self)">, "CUDA cudaMemcpy3DParms struct type (value-typed).">; +def Cuda_CUDACudaPitchedPtrStructTypeValueTyped : Type($_self)">, "CUDA cudaPitchedPtr struct type (value-typed).">, BuildableType<"::mlir::cutlass_compiler::cuda::RuntimePitchedPtrType::get($_builder.getContext())">; +def Cuda_CUDACudaPitchedPtrStructTypeValueTyped2 : Type($_self)">, "CUDA cudaPitchedPtr struct type (value-typed).">; +def Cuda_CUDACudaPosStructTypeValueTyped : Type($_self)">, "CUDA cudaPos struct type (value-typed).">, BuildableType<"::mlir::cutlass_compiler::cuda::RuntimePosType::get($_builder.getContext())">; +def Cuda_CUDACudaPosStructTypeValueTyped2 : Type($_self)">, "CUDA cudaPos struct type (value-typed).">; +def Cuda_CUDACudaUUIDTStructTypeValue : Type($_self)">, "CUDA cudaUUID_t struct type (value-typed).">, BuildableType<"::mlir::cutlass_compiler::cuda::RuntimeUuidType::get($_builder.getContext())">; +def Cuda_CUDACudaUUIDTStructTypeValue2 : Type($_self)">, "CUDA cudaUUID_t struct type (value-typed).">; +def Cuda_CUDADim3StructTypeValueTyped : Type($_self)">, "CUDA dim3 struct type (value-typed).">; +def Cuda_CUDADim3StructTypeValueTyped2 : Type($_self)">, "CUDA dim3 struct type (value-typed).">, BuildableType<"::mlir::cutlass_compiler::cuda::RuntimeDim3Type::get($_builder.getContext())">; +def Cuda_CUDAGraph : Type($_self)">, "CUDA Graph">; +def Cuda_CUDAGraph2 : Type($_self)">, "CUDA Graph">, BuildableType<"::mlir::cutlass_compiler::cuda::GraphType::get($_builder.getContext())">; +def Cuda_CUDAGraphExecution : Type($_self)">, "CUDA Graph Execution">, BuildableType<"::mlir::cutlass_compiler::cuda::GraphExecType::get($_builder.getContext())">; +def Cuda_CUDAGraphExecution2 : Type($_self)">, "CUDA Graph Execution">; +def Cuda_CUDAGraphNodeHandle : Type($_self)">, "CUDA Graph Node handle">, BuildableType<"::mlir::cutlass_compiler::cuda::GraphNodeType::get($_builder.getContext())">; +def Cuda_CUDAGraphNodeHandle2 : Type($_self)">, "CUDA Graph Node handle">; +def Cuda_CUDAKernelHandle : Type($_self)">, "CUDA Kernel handle">, BuildableType<"::mlir::cutlass_compiler::cuda::KernelType::get($_builder.getContext())">; +def Cuda_CUDAKernelHandle2 : Type($_self)">, "CUDA Kernel handle">; +def Cuda_CUDALibraryHandle : Type($_self)">, "CUDA Library handle">; +def Cuda_CUDALibraryHandle2 : Type($_self)">, "CUDA Library handle">, BuildableType<"::mlir::cutlass_compiler::cuda::LibraryType::get($_builder.getContext())">; +def Cuda_CUDAModuleHandle : Type($_self)">, "CUDA Module handle">, BuildableType<"::mlir::cutlass_compiler::cuda::ModuleType::get($_builder.getContext())">; +def Cuda_CUDAResultType : Type($_self)">, "CUDA Result type">, BuildableType<"::mlir::cutlass_compiler::cuda::ResultType::get($_builder.getContext())">; +def Cuda_CUDAResultType2 : Type($_self)">, "CUDA Result type">; +def Cuda_CUDAResultTypeOrCUDAStream : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::StreamType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::ResultType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::EventType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::GraphDeviceNodeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::TensorMapType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::IntType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::LaunchConfigType>($_self) || ($_self.getDialect().getNamespace() == \"ptr\" || (::llvm::isa<::mlir::OpaqueType>($_self) && ::llvm::cast<::mlir::OpaqueType>($_self).getDialectNamespace() == \"ptr\"))">, "CUDA Result type or CUDA Stream or `Event` represents CUDA event or CUDA Tensor Map type or Representation of CUDA's extensible launch configuration mirroring cuda runtime API or `GraphDeviceNode` represents CUDA device node handle for device-side node update or `cuda.int` is a integer type with a bitwidth matching the underlying ABI integer bitwidth or 32-bit signless integer or 64-bit signless integer or pointer type or LLVM pointer type">; +def Cuda_CUDAStream : Type($_self)">, "CUDA Stream">; +def Cuda_CUDAStream2 : Type($_self)">, "CUDA Stream">, BuildableType<"::mlir::cutlass_compiler::cuda::StreamType::get($_builder.getContext())">; +def Cuda_CUDATensorMapType : Type($_self)">, "CUDA Tensor Map type">, BuildableType<"::mlir::cutlass_compiler::cuda::TensorMapType::get($_builder.getContext())">; +def Cuda_CudaIntIsAIntegerType : Type($_self)">, "`cuda.int` is a integer type with a bitwidth matching the underlying ABI integer bitwidth">; +def Cuda_EventRepresentsCUDAEvent : Type($_self)">, "`Event` represents CUDA event">, BuildableType<"::mlir::cutlass_compiler::cuda::EventType::get($_builder.getContext())">; +def Cuda_EventRepresentsCUDAEvent2 : Type($_self)">, "`Event` represents CUDA event">; +def Cuda_FixedLengthVectorOf32Bit : Type($_self)">, "fixed-length vector of 32-bit signless integer values of length 3">; +def Cuda_FixedLengthVectorOf32Bit2 : Type($_self)">, "fixed-length vector of 32-bit signless integer values of length 2">; +def Cuda_FixedLengthVectorOf8Bit : Type($_self)">, "fixed-length vector of 8-bit signless integer values of length 8">; +def Cuda_FixedLengthVectorOf8Bit2 : Type($_self)">, "fixed-length vector of 8-bit signless integer values of length 256">; +def Cuda_FixedLengthVectorOf8Bit3 : Type($_self)">, "fixed-length vector of 8-bit signless integer values of length 16">; +def Cuda_GraphDeviceNodeRepresentsCUDADeviceNodeHandle : Type($_self)">, "`GraphDeviceNode` represents CUDA device node handle for device-side node update">; +def Cuda_LLVMPointerInAddressSpace0 : Type($_self)">, "LLVM pointer in address space 0">; +def Cuda_LLVMPointerType : Type($_self)">, "LLVM pointer type">; +def Cuda_PointerType : Type($_self) && ::llvm::cast<::mlir::OpaqueType>($_self).getDialectNamespace() == \"ptr\"))">, "pointer type">; +def Cuda_RepresentationOfCUDASExtensibleLaunch : Type($_self)">, "Representation of CUDA's extensible launch configuration mirroring cuda runtime API">; +def Cuda_T1BitSignlessInteger : Type, "1-bit signless integer">, BuildableType<"$_builder.getIntegerType(1)">; +def Cuda_T32BitFloat : Type($_self)">, "32-bit float">, BuildableType<"::mlir::Float32Type::get($_builder.getContext())">; +def Cuda_T32BitFloat2 : Type($_self)">, "32-bit float">; +def Cuda_T32BitSignlessInteger : Type, "32-bit signless integer">; +def Cuda_T32BitSignlessInteger2 : Type, "32-bit signless integer">, BuildableType<"$_builder.getIntegerType(32)">; +def Cuda_T64BitSignlessInteger : Type, "64-bit signless integer">; +def Cuda_T64BitSignlessInteger2 : Type, "64-bit signless integer">, BuildableType<"$_builder.getIntegerType(64)">; +def Cuda_VariadicOf32BitSignlessInteger : Type, "variadic of 32-bit signless integer">; +def Cuda_VariantCooperativeOfCUDACudaLaunchAttributeValueUnion : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValueClusterSchedulingPolicyPreferenceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValueNvlinkUtilCentricSchedulingType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowedType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValueMemSyncDomainType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValuePortableClusterSizeModeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValuePriorityType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValueSharedMemCarveoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValueSharedMemoryModeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValueSyncPolicyType>($_self)">, "Variant `cooperative` of CUDA `cudaLaunchAttributeValue` union (payload: int). or Variant `syncPolicy` of CUDA `cudaLaunchAttributeValue` union (payload: enum cudaSynchronizationPolicy). or Variant `clusterSchedulingPolicyPreference` of CUDA `cudaLaunchAttributeValue` union (payload: enum cudaClusterSchedulingPolicy). or Variant `programmaticStreamSerializationAllowed` of CUDA `cudaLaunchAttributeValue` union (payload: int). or Variant `priority` of CUDA `cudaLaunchAttributeValue` union (payload: int). or Variant `memSyncDomain` of CUDA `cudaLaunchAttributeValue` union (payload: cudaLaunchMemSyncDomain). or Variant `sharedMemCarveout` of CUDA `cudaLaunchAttributeValue` union (payload: unsigned int). or Variant `nvlinkUtilCentricScheduling` of CUDA `cudaLaunchAttributeValue` union (payload: unsigned int). or Variant `portableClusterSizeMode` of CUDA `cudaLaunchAttributeValue` union (payload: cudaLaunchAttributePortableClusterMode). or Variant `sharedMemoryMode` of CUDA `cudaLaunchAttributeValue` union (payload: cudaSharedMemoryMode).">; + +// Attribute constraints the bindings name but do not describe. +def Cuda_DenseResourceElementsAttr : Attr($_self)">, "dense resource elements attribute"> { + let storageType = "::mlir::DenseResourceElementsAttr"; + let returnType = "::mlir::DenseResourceElementsAttr"; + let convertFromStorage = "$_self"; +} + +def Cuda_FuncFunction_typeAttr : Attr($_self) && (::llvm::isa<::mlir::FunctionType>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()))">, "type attribute of function type"> { + let storageType = "::mlir::TypeAttr"; + let returnType = "::mlir::TypeAttr"; + let convertFromStorage = "$_self"; +} + +def Cuda_KernelFunction_typeAttr : Attr($_self) && (::llvm::isa<::mlir::FunctionType>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()))">, "type attribute of function type"> { + let storageType = "::mlir::TypeAttr"; + let returnType = "::mlir::TypeAttr"; + let convertFromStorage = "$_self"; +} + +def Cuda_IntegerAttr : Attr($_self)">, "integer attribute"> { + let storageType = "::mlir::IntegerAttr"; + let returnType = "::mlir::IntegerAttr"; + let convertFromStorage = "$_self"; +} + +// Operations whose result types are inferred in C++ (CudaInference.cpp). +// Verification compares them with the inferred ones where the rule is +// recovered; elsewhere the types written stand, as the compiler accepted them. +class Cuda_InferredOp traits = []> + : Cuda_Op]> { + let extraClassDefinition = [{ + ::llvm::LogicalResult $cppClass::refineReturnTypes( + ::mlir::MLIRContext *context, ::std::optional<::mlir::Location> location, + ::mlir::ValueRange operands, ::mlir::DictionaryAttr attributes, + ::mlir::PropertyRef properties, ::mlir::RegionRange regions, + ::llvm::SmallVectorImpl<::mlir::Type> &returnTypes) { + ::llvm::SmallVector<::mlir::Type, 4> inferred; + if (::mlir::failed(inferReturnTypes(context, ::std::nullopt, operands, + attributes, properties, regions, + inferred))) + return ::mlir::success(); + if (!isCompatibleReturnTypes(inferred, returnTypes)) + return ::mlir::emitOptionalError( + location, "'", getOperationName(), "' op inferred type(s) ", + inferred, " are incompatible with return type(s) of operation ", + returnTypes); + return ::mlir::success(); + } + }]; +} + +def Cuda_AddressOfOp : Cuda_Op<"addressof"> { + let arguments = (ins + SymbolRefAttr:$symname + ); + let results = (outs + Cuda_PointerType:$result + ); +} +def Cuda_BinaryOp : Cuda_Op<"binary", [NoTerminator]> { + let description = [{ + This operation represents a CUDA binary, which contains a number of functions and + a serialized representation of the binary. + + Examples: + + ```mlir + + cuda.binary @binary_name <...> { + cuda.kernel @function_name function_type + } + ``` + }]; + let arguments = (ins + SymbolNameAttr:$sym_name, + Cuda_DenseResourceElementsAttr:$binary + ); + let results = (outs); + let regions = (region AnyRegion:$body); +} +def Cuda_CastOp : Cuda_Op<"cast", [Pure]> { + let description = [{ + Casts between CUDA types and standard dialect or built-in types. + The source/destination types are statically checked and the IR + is invalid if the cast is not possible. + + Examples: + ```mlir + + %res_i32 = cuda.cast %res : cuda.result -> i32 + %res1 = cuda.cast %res_i32 : i32 -> cuda.result + + %s_ptr = cuda.cast %hstream : cuda.stream -> ptr.ptr + %s_i64 = cuda.cast %hstream : cuda.stream -> i64 + ``` + }]; + let arguments = (ins + Cuda_CUDAResultTypeOrCUDAStream:$src + ); + let results = (outs + Cuda_CUDAResultTypeOrCUDAStream:$dst + ); +} +def Cuda_ComposeOp : Cuda_Op<"compose"> { + let description = [{ + Generic constructor for any auto-generated CUDA struct type. The result + type identifies which struct is being built; operands are the struct's + non-padding fields in declaration order. The verifier checks operand + count and per-operand type against the result type's + `CudaStructTypeInterface::getOperandFieldTypes()`. + + Lowers to `llvm.mlir.undef` + a chain of `llvm.insertvalue` (one per + operand-mapped slot from `getOperandFieldIndices()`). Pure value SSA; + no memory. + + Example: + ```mlir + %pos = cuda.compose %x, %y, %z + : i64, i64, i64 -> !cuda.runtime.pos + ``` + }]; + let arguments = (ins + Variadic:$fields + ); + let results = (outs + Cuda_AnyCUDAStructType:$result + ); +} +def Cuda_DecomposeOp : Cuda_Op<"decompose"> { + let description = [{ + Generic accessor: returns each non-padding field of a CUDA struct value + as an SSA value, in declaration order. The verifier checks result count + and per-result type against the source type's + `CudaStructTypeInterface::getOperandFieldTypes()`. + + Lowers to a chain of `llvm.extractvalue` (one per operand-mapped slot + from `getOperandFieldIndices()`). Pure value SSA; no memory. + + Example: + ```mlir + %x, %y, %z = cuda.decompose %pos + : !cuda.runtime.pos -> i64, i64, i64 + ``` + }]; + let arguments = (ins + Cuda_AnyCUDAStructType:$source + ); + let results = (outs + Optional:$fields + ); +} +def Cuda_DeviceSyncOp : Cuda_Op<"device.sync"> { + let description = [{ + Operation to synchronize with the device. See cudaDeviceSynchronize() + }]; + let arguments = (ins); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_DriverEventCreateOp : Cuda_Op<"driver.event.create"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_T32BitSignlessInteger:$Flags + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result, + Cuda_EventRepresentsCUDAEvent:$phEvent + ); +} +def Cuda_DriverEventDestroyOp : Cuda_Op<"driver.event.destroy"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_EventRepresentsCUDAEvent2:$hEvent + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_DriverEventElapsedTimeOp : Cuda_Op<"driver.event.elapsed_time"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_EventRepresentsCUDAEvent2:$hStart, + Cuda_EventRepresentsCUDAEvent2:$hEnd + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result, + Cuda_T32BitFloat:$pMilliseconds + ); +} +def Cuda_DriverEventQueryOp : Cuda_Op<"driver.event.query", [MemoryEffects<[MemRead]>]> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_EventRepresentsCUDAEvent2:$hEvent + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_DriverEventRecordOp : Cuda_Op<"driver.event.record"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_EventRepresentsCUDAEvent2:$hEvent, + Cuda_CUDAStream:$hStream + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_DriverEventRecordWithFlagsOp : Cuda_Op<"driver.event.record_with_flags"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_EventRepresentsCUDAEvent2:$hEvent, + Cuda_CUDAStream:$hStream, + Cuda_T32BitSignlessInteger:$flags + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_DriverEventSynchronizeOp : Cuda_Op<"driver.event.synchronize"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_EventRepresentsCUDAEvent2:$hEvent + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_DriverLibraryGetGlobalOp : Cuda_Op<"driver.library.get_global", [MemoryEffects<[MemRead]>]> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_CUDALibraryHandle:$library, + Cuda_PointerType:$name_arg + ); + let results = (outs + Cuda_CUDAResultType2:$cuda_result, + Cuda_PointerType:$dptr, + Cuda_T64BitSignlessInteger:$bytes + ); +} +def Cuda_DriverLibraryGetKernelOp : Cuda_Op<"driver.library.get_kernel", [MemoryEffects<[MemRead]>]> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_CUDALibraryHandle:$library, + Cuda_PointerType:$name_arg + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result, + Cuda_CUDAKernelHandle:$pKernel + ); +} +def Cuda_DriverLibraryGetKernelCountOp : Cuda_Op<"driver.library.get_kernel_count", [MemoryEffects<[MemRead]>]> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_CUDALibraryHandle:$lib + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result, + Cuda_T32BitSignlessInteger2:$count + ); +} +def Cuda_DriverLibraryGetManagedOp : Cuda_Op<"driver.library.get_managed", [MemoryEffects<[MemRead]>]> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_CUDALibraryHandle:$library, + Cuda_PointerType:$name_arg + ); + let results = (outs + Cuda_CUDAResultType2:$cuda_result, + Cuda_PointerType:$dptr, + Cuda_T64BitSignlessInteger:$bytes + ); +} +def Cuda_DriverLibraryGetModuleOp : Cuda_Op<"driver.library.get_module", [MemoryEffects<[MemRead]>]> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_CUDALibraryHandle:$library + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result, + Cuda_CUDAModuleHandle:$pMod + ); +} +def Cuda_DriverLibraryGetUnifiedFunctionOp : Cuda_Op<"driver.library.get_unified_function", [MemoryEffects<[MemRead]>]> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_CUDALibraryHandle:$library, + Cuda_PointerType:$symbol + ); + let results = (outs + Cuda_CUDAResultType2:$cuda_result, + Cuda_PointerType:$fptr + ); +} +def Cuda_DriverLibraryLoadDataOp : Cuda_Op<"driver.library.load_data"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_PointerType:$code_arg, + Cuda_PointerType:$jitOptions, + Cuda_PointerType:$jitOptionsValues, + Cuda_T32BitSignlessInteger:$numJitOptions, + Cuda_PointerType:$libraryOptions, + Cuda_PointerType:$libraryOptionValues, + Cuda_T32BitSignlessInteger:$numLibraryOptions + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result, + Cuda_CUDALibraryHandle2:$library + ); +} +def Cuda_DriverLibraryLoadFromFileOp : Cuda_Op<"driver.library.load_from_file"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_PointerType:$fileName, + Cuda_PointerType:$jitOptions, + Cuda_PointerType:$jitOptionsValues, + Cuda_T32BitSignlessInteger:$numJitOptions, + Cuda_PointerType:$libraryOptions, + Cuda_PointerType:$libraryOptionValues, + Cuda_T32BitSignlessInteger:$numLibraryOptions + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result, + Cuda_CUDALibraryHandle2:$library + ); +} +def Cuda_DriverLibraryUnloadOp : Cuda_Op<"driver.library.unload"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_CUDALibraryHandle:$library + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_DriverTensorMapEncodeIm2colOp : Cuda_Op<"driver.tensor_map.encode_im2col"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_T32BitSignlessInteger:$tensorDataType, + Cuda_T32BitSignlessInteger:$tensorRank, + Cuda_PointerType:$globalAddress, + Cuda_PointerType:$globalDim, + Cuda_PointerType:$globalStrides, + Cuda_PointerType:$pixelBoxLowerCorner, + Cuda_PointerType:$pixelBoxUpperCorner, + Cuda_T32BitSignlessInteger:$channelsPerPixel, + Cuda_T32BitSignlessInteger:$pixelsPerColumn, + Cuda_PointerType:$elementStrides, + Cuda_T32BitSignlessInteger:$interleave, + Cuda_T32BitSignlessInteger:$swizzle, + Cuda_T32BitSignlessInteger:$l2Promotion, + Cuda_T32BitSignlessInteger:$oobFill + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result, + Cuda_CUDATensorMapType:$tensorMap + ); +} +def Cuda_DriverTensorMapEncodeIm2colWideOp : Cuda_Op<"driver.tensor_map.encode_im2col_wide"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_T32BitSignlessInteger:$tensorDataType, + Cuda_T32BitSignlessInteger:$tensorRank, + Cuda_PointerType:$globalAddress, + Cuda_PointerType:$globalDim, + Cuda_PointerType:$globalStrides, + Cuda_T32BitSignlessInteger:$pixelBoxLowerCornerWidth, + Cuda_T32BitSignlessInteger:$pixelBoxUpperCornerWidth, + Cuda_T32BitSignlessInteger:$channelsPerPixel, + Cuda_T32BitSignlessInteger:$pixelsPerColumn, + Cuda_PointerType:$elementStrides, + Cuda_T32BitSignlessInteger:$interleave, + Cuda_T32BitSignlessInteger:$mode, + Cuda_T32BitSignlessInteger:$swizzle, + Cuda_T32BitSignlessInteger:$l2Promotion, + Cuda_T32BitSignlessInteger:$oobFill + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result, + Cuda_CUDATensorMapType:$tensorMap + ); +} +def Cuda_DriverTensorMapReplaceAddressOp : Cuda_Op<"driver.tensor_map.replace_address"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_PointerType:$globalAddress + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result, + Cuda_CUDATensorMapType:$tensorMap + ); +} +def Cuda_EventCreateOp : Cuda_Op<"event.create"> { + let description = [{ + Operation to create a CUDA event. + }]; + let arguments = (ins); + let results = (outs + Cuda_CUDAResultType:$result, + Cuda_EventRepresentsCUDAEvent:$event + ); +} +def Cuda_EventDestroyOp : Cuda_Op<"event.destroy"> { + let description = [{ + Operation to destroy a CUDA event. + }]; + let arguments = (ins + Cuda_EventRepresentsCUDAEvent2:$event + ); + let results = (outs + Cuda_CUDAResultType:$result + ); +} +def Cuda_EventElapsedTimeOp : Cuda_Op<"event.elapsed_time"> { + let description = [{ + Operation to compute the elapsed time between events + }]; + let arguments = (ins + Cuda_EventRepresentsCUDAEvent2:$start_event, + Cuda_EventRepresentsCUDAEvent2:$end_event + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result, + Cuda_T32BitFloat:$elapsed_time + ); +} +def Cuda_EventRecordOp : Cuda_Op<"event.record"> { + let description = [{ + Operation to record an event in a CUDA stream. + }]; + let arguments = (ins + Cuda_EventRepresentsCUDAEvent2:$event, + Optional:$stream + ); + let results = (outs + Cuda_CUDAResultType:$result + ); +} +def Cuda_FuncOp : Cuda_Op<"func"> { + let description = [{ + This operation represents a CUDA device function which can be called from + another `cuda.func` or a `cuda.kernel`. It cannot be launched from the host. + + Examples: + + ```mlir + + cuda.module @module_name { + + cuda.func @device_func(%arg0 : i32) -> i32 { + %0 = arith.constant 0 : i32 + cuda.return %0 : i32 + } + + } + ``` + }]; + let arguments = (ins + SymbolNameAttr:$sym_name, + Cuda_FuncFunction_typeAttr:$function_type, + OptionalAttr:$arg_attrs, + OptionalAttr:$res_attrs + ); + let results = (outs); + let regions = (region AnyRegion:$body); +} +def Cuda_FuncGetNameOp : Cuda_Op<"func.get_name"> { + let arguments = (ins + Cuda_CUDAKernelHandle2:$kernel + ); + let results = (outs + Cuda_CUDAResultType2:$result, + Cuda_PointerType:$name + ); +} +def Cuda_GraphDestroyOp : Cuda_Op<"graph.destroy"> { + let description = [{ + Operation to destroy a CUDA graph. + }]; + let arguments = (ins + Cuda_CUDAGraph:$graph + ); + let results = (outs + Cuda_CUDAResultType:$result + ); +} +def Cuda_GraphInstantiateOp : Cuda_Op<"graph.instantiate"> { + let description = [{ + Operation to instantiate a CUDA graph. + }]; + let arguments = (ins + Cuda_CUDAGraph:$graph + ); + let results = (outs + Cuda_CUDAResultType:$result, + Cuda_CUDAGraphExecution:$graph_exec + ); +} +def Cuda_GraphLaunchOp : Cuda_Op<"graph.launch"> { + let description = [{ + Operation to launch a CUDA graph. + }]; + let arguments = (ins + Cuda_CUDAGraphExecution2:$graph_exec, + Cuda_CUDAStream:$stream + ); + let results = (outs + Cuda_CUDAResultType:$result + ); +} +def Cuda_GraphExecDestroyOp : Cuda_Op<"graph_exec.destroy"> { + let description = [{ + Operation to destroy a CUDA graph execution. + }]; + let arguments = (ins + Cuda_CUDAGraphExecution2:$graph_exec + ); + let results = (outs + Cuda_CUDAResultType:$result + ); +} +def Cuda_KernelOp : Cuda_Op<"kernel"> { + let description = [{ + This operation represents a CUDA kernel which may be launched as a grid. It + differs from a `cuda.func`. + + Examples: + + ```mlir + + cuda.module @module_name { + + cuda.func @kernel(%arg0 : i32) + attributes { + cu_attrs = { + num_regs = 128, + max_dynamic_shared_size_bytes = 2048 + // ... add more cuda function attributes here + } + } + } + + ... + + %res = cuda.launch @module_name::@device_func <<< %grid, %block, %cluster, %smem, %stream >>>(%0) + + %0 = cuda.result.succeeded(%res) + scf.if %0 { + + } + else { + + } + ``` + }]; + let arguments = (ins + SymbolNameAttr:$sym_name, + Cuda_KernelFunction_typeAttr:$function_type, + OptionalAttr:$arg_attrs, + OptionalAttr:$res_attrs, + OptionalAttr:$cu_func_attrs + ); + let results = (outs); + let regions = (region AnyRegion:$body); +} +def Cuda_LaunchOp : Cuda_Op<"launch", [AttrSizedOperandSegments]> { + let description = [{ + Operation to launch a `cuda.kernel` on the device. + + Examples: + + ```mlir + + cuda.module @module_name { + + cuda.func @kernel(%arg0 : i32) { + + } + + } + + ... + + %0 = ... // kernel arguments + + %res = cuda.launch @module_name::@device_func<<< %grid, %block, %smem, %stream >>>(%0) : !cuda.launch_cfg, ... -> !cuda.result + + // If calling with an Op that implements a FunctionOpInterface + %res = cuda.launch @module_name::@device_func<<< %grid, %block, %smem, %stream >>>(%0) : !cuda.launch_cfg, ... -> !cuda.result + + %res2 = cuda.result.succeeded(%res) + scf.if %res2 { + + } + else { + + } + ``` + }]; + let arguments = (ins + Variadic:$grid, + Variadic:$block, + Cuda_T64BitSignlessInteger:$dynamicSmem, + Cuda_CUDAStream:$cudaStream, + Variadic:$inputs, + SymbolRefAttr:$callee, + OptionalAttr:$arg_attrs, + OptionalAttr:$res_attrs, + OptionalAttr:$assume_kernel_attr + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_LaunchCfgAccessPolicyWindowOp : Cuda_Op<"launch_cfg.access_policy_window"> { + let description = [{ + Sets accessPolicyWindow for a launch_config. + + Example: + ```mlir + %cfg0 = cuda.launch_cfg.create(...) + cuda.launch_cfg.access_policy_window [%cfg0] ( + base_ptr=%basePtr, + hitProp=%hitProp, + hitRatio=%hitRatio, + missProp=%missProp, + numBytes=%numBytes + ) : !cuda.launch_cfg, !ptr.ptr<#ptr.generic_space>, f32, i64 + ``` + }]; + let arguments = (ins + Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, + Cuda_PointerType:$base_ptr, + Cuda_T32BitFloat2:$hitRatio, + Cuda_T64BitSignlessInteger:$num_bytes, + Cuda_CudaAccessPropertyEnum:$hitProp, + Cuda_CudaAccessPropertyEnum:$missProp + ); + let results = (outs); +} +def Cuda_LaunchCfgClusterDimOp : Cuda_Op<"launch_cfg.cluster_dim"> { + let description = [{ + Sets clusterDim for a launch_config + + Example: + ```mlir + %cfg0 = cuda.launch_cfg.create(...) + cuda.launch_cfg.cluster_dim [%cfg0] (1, 2, 8) : !cuda.launch_cfg, i32, i32, i32 + ``` + }]; + let arguments = (ins + Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, + Cuda_T32BitSignlessInteger:$clusterDimX, + Cuda_T32BitSignlessInteger:$clusterDimY, + Cuda_T32BitSignlessInteger:$clusterDimZ + ); + let results = (outs); +} +def Cuda_LaunchCfgClusterSchedulingPolicyOp : Cuda_Op<"launch_cfg.cluster_scheduling_policy"> { + let description = [{ + Sets clusterSchedulingPolicyPreference for a launch_config + + Example: + ```mlir + %cfg0 = cuda.launch_cfg.create... + cuda.launch_cfg.cluster_scheduling_policy [%cfg0] cudaClusterSchedulingPolicyLoadBalancing + : !cuda.launch_cfg + ``` + }]; + let arguments = (ins + Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, + Cuda_CudaClusterSchedulingPolicyEnum:$policy + ); + let results = (outs); +} +def Cuda_LaunchCfgCooperativeOp : Cuda_Op<"launch_cfg.cooperative"> { + let description = [{ + Sets cooperative attribute to a launch_config. Nonzero indicates a cooperative kernel. + + Example: + ```mlir + %cfg0 = cuda.launch_cfg.create... + cuda.launch_cfg.cooperative [%cfg0] 1 + : !cuda.launch_cfg, i32 + ``` + }]; + let arguments = (ins + Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, + Cuda_T32BitSignlessInteger:$cooperative + ); + let results = (outs); +} +def Cuda_LaunchCfgCreateOp : Cuda_Op<"launch_cfg.create"> { + let description = [{ + Creates a launch_config object with the specified runtime values. + The result is a Cuda_LaunchConfig type that contains all the runtime data. + + Example: + ```mlir + %cfg = cuda.launch_cfg.create( + blockDim=(%block_x, %block_y, %block_z), + dynamicSmemBytes=%dynamic_smem_bytes, + gridDim=(%grid_x, %grid_y, %grid_z), + stream=%hstream + ) : i32, i32, i32, i64, i32, i32, i32, !cuda.stream -> !cuda.launch_cfg + ``` + }]; + let arguments = (ins + Cuda_T32BitSignlessInteger:$blockDimX, + Cuda_T32BitSignlessInteger:$blockDimY, + Cuda_T32BitSignlessInteger:$blockDimZ, + Cuda_T64BitSignlessInteger:$dynamicSmemBytes, + Cuda_T32BitSignlessInteger:$gridDimX, + Cuda_T32BitSignlessInteger:$gridDimY, + Cuda_T32BitSignlessInteger:$gridDimZ, + Cuda_CUDAStream:$stream, + Cuda_IntegerAttr:$maxNumAttrs + ); + let results = (outs + Cuda_RepresentationOfCUDASExtensibleLaunch:$result + ); +} +def Cuda_LaunchCfgCreateFromStreamOp : Cuda_InferredOp<"launch_cfg.create_from_stream"> { + let description = [{ + Creates a launch_config object initialized with a CUDA stream and space for + the specified number of launch attributes. Grid dimensions, block + dimensions, and dynamic shared memory bytes can be supplied later with + `cuda.launch_cfg.grid_dimensions`. + + Example: + ```mlir + %cfg = cuda.launch_cfg.create_from_stream( + stream=%hstream + ) : !cuda.stream + ``` + }]; + let arguments = (ins + Cuda_CUDAStream:$stream, + Cuda_IntegerAttr:$maxNumAttrs + ); + let results = (outs + Cuda_RepresentationOfCUDASExtensibleLaunch:$result + ); +} +def Cuda_LaunchCfgDeviceUpdatableKernelNodeOp : Cuda_Op<"launch_cfg.device_updatable_kernel_node"> { + let description = [{ + Sets deviceUpdatableKernelNode attribute to a launch_config. + + Example: + ```mlir + %cfg0 = cuda.launch_cfg.create... + cuda.launch_cfg.device_updatable_kernel_node [%cfg0] (devNode=%devNode, deviceUpdatable=%deviceUpdatable) + ``` + }]; + let arguments = (ins + Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, + Cuda_GraphDeviceNodeRepresentsCUDADeviceNodeHandle:$devNode, + Cuda_T32BitSignlessInteger:$deviceUpdatable + ); + let results = (outs); +} +def Cuda_LaunchCfgGridDimensionsOp : Cuda_Op<"launch_cfg.grid_dimensions"> { + let description = [{ + Sets the fixed launch dimensions for a launch_config object. This updates + the direct `cudaLaunchConfig_t` fields, not the launch attribute array. + + Example: + ```mlir + cuda.launch_cfg.grid_dimensions [%cfg0] + grid(%gx, %gy, %gz), block(%bx, %by, %bz), dynamicSmem(%smem) + : !cuda.launch_cfg, i32, i32, i32, i32, i32, i32, i64 + ``` + }]; + let arguments = (ins + Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, + Cuda_T32BitSignlessInteger:$gridDimX, + Cuda_T32BitSignlessInteger:$gridDimY, + Cuda_T32BitSignlessInteger:$gridDimZ, + Cuda_T32BitSignlessInteger:$blockDimX, + Cuda_T32BitSignlessInteger:$blockDimY, + Cuda_T32BitSignlessInteger:$blockDimZ, + Cuda_T64BitSignlessInteger:$dynamicSmemBytes + ); + let results = (outs); +} +def Cuda_LaunchCfgLaunchCompletionEventOp : Cuda_Op<"launch_cfg.launch_completion_event"> { + let description = [{ + Sets launchCompletionEvent attribute to a launch_config. + + Example: + ```mlir + %cfg0 = cuda.launch_cfg.create... + cuda.launch_cfg.launch_completion_event [%cfg0] (event=%event, flags=%flags) + : !cuda.launch_cfg, !cuda.event, i32 + ``` + }]; + let arguments = (ins + Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, + Cuda_EventRepresentsCUDAEvent2:$event, + Cuda_T32BitSignlessInteger:$flags + ); + let results = (outs); +} +def Cuda_LaunchCfgMemSyncDomainOp : Cuda_Op<"launch_cfg.mem_sync_domain"> { + let description = [{ + Sets memSyncDomain attribute to a launch_config. + + Example: + ```mlir + %cfg0 = cuda.launch_cfg.create... + cuda.launch_cfg.mem_sync_domain [%cfg0] cudaMemSyncDomainRemote + : !cuda.launch_cfg + ``` + }]; + let arguments = (ins + Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, + Cuda_CudaLaunchMemSyncDomainEnum:$memSyncDomain + ); + let results = (outs); +} +def Cuda_LaunchCfgMemSyncDomainMapOp : Cuda_Op<"launch_cfg.mem_sync_domain_map"> { + let description = [{ + Sets memSyncDomainMap attribute to a launch_config. + + Example: + ```mlir + %cfg0 = cuda.launch_cfg.create... + cuda.launch_cfg.mem_sync_domain_map [%cfg0] (domain_default=0, domain_remote=1) + : !cuda.launch_cfg, i32, i32 + ``` + }]; + let arguments = (ins + Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, + Cuda_T32BitSignlessInteger:$domain_default, + Cuda_T32BitSignlessInteger:$domain_remote + ); + let results = (outs); +} +def Cuda_LaunchCfgNvlinkUtilCentricSchedulingOp : Cuda_Op<"launch_cfg.nvlink_util_centric_scheduling"> { + let description = [{ + Sets nvlinkUtilCentricScheduling attribute to a launch_config. + + Example: + ```mlir + %cfg0 = cuda.launch_cfg.create... + cuda.launch_cfg.nvlink_util_centric_scheduling [%cfg0] 1 : !cuda.launch_cfg, i32 + ``` + }]; + let arguments = (ins + Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, + Cuda_T32BitSignlessInteger:$nvlinkSchedulingEnabled + ); + let results = (outs); +} +def Cuda_LaunchCfgPreferredClusterDimOp : Cuda_Op<"launch_cfg.preferred_cluster_dim"> { + let description = [{ + Sets preferredClusterDim attribute to a launch_config. + + Example: + ```mlir + %cfg0 = cuda.launch_cfg.create... + cuda.launch_cfg.preferred_cluster_dim [%cfg0] (1, 1, 8) + : !cuda.launch_cfg, i32, i32, i32 + ``` + }]; + let arguments = (ins + Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, + Cuda_T32BitSignlessInteger:$preferredClusterDimX, + Cuda_T32BitSignlessInteger:$preferredClusterDimY, + Cuda_T32BitSignlessInteger:$preferredClusterDimZ + ); + let results = (outs); +} +def Cuda_LaunchCfgPrintOp : Cuda_Op<"launch_cfg.print"> { + let description = [{ + Prints a cuda launch config. + + Example: + ```mlir + %cfg0 = cuda.launch_cfg.create(...) + cuda.launch_cfg.print [%cfg0] : !cuda.launch_cfg + ``` + }]; + let arguments = (ins + Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg + ); + let results = (outs); +} +def Cuda_LaunchCfgPriorityOp : Cuda_Op<"launch_cfg.priority"> { + let description = [{ + Sets priority attribute to a launch_config. + + Example: + ```mlir + %cfg0 = cuda.launch_cfg.create... + cuda.launch_cfg.priority [%cfg0] 1 + : !cuda.launch_cfg, i32 + ``` + }]; + let arguments = (ins + Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, + Cuda_T32BitSignlessInteger:$priority + ); + let results = (outs); +} +def Cuda_LaunchCfgProgrammaticEventOp : Cuda_Op<"launch_cfg.programmatic_event"> { + let description = [{ + Sets programmaticEvent attribute to a launch_config. + + Example: + ```mlir + %cfg0 = cuda.launch_cfg.create... + cuda.launch_cfg.programmatic_event [%cfg0] ( + event=%event, + flags=%flags, + triggerAtBlockStart=%triggerAtBlockStart) + : !cuda.launch_cfg, !cuda.event, i32, i32 + ``` + }]; + let arguments = (ins + Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, + Cuda_EventRepresentsCUDAEvent2:$event, + Cuda_T32BitSignlessInteger:$flags, + Cuda_T32BitSignlessInteger:$triggerAtBlockStart + ); + let results = (outs); +} +def Cuda_LaunchCfgProgrammaticStreamSerializationAllowedOp : Cuda_Op<"launch_cfg.programmatic_stream_serialization_allowed"> { + let description = [{ + Sets programmaticStreamSerializationAllowed attribute to a launch_config. + + Example: + ```mlir + %cfg0 = cuda.launch_cfg.create... + cuda.launch_cfg.programmatic_stream_serialization_allowed [%cfg0] 1 + : !cuda.launch_cfg, i32 + ``` + }]; + let arguments = (ins + Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, + Cuda_T32BitSignlessInteger:$programmaticStreamSerializationAllowed + ); + let results = (outs); +} +def Cuda_LaunchCfgSharedMemCarveoutOp : Cuda_Op<"launch_cfg.shared_mem_carveout"> { + let description = [{ + Sets sharedMemCarveout attribute to a launch_config. + + Example: + ```mlir + %cfg0 = cuda.launch_cfg.create... + cuda.launch_cfg.shared_mem_carveout [%cfg0] 1024 + : !cuda.launch_cfg, i32 + ``` + }]; + let arguments = (ins + Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, + Cuda_T32BitSignlessInteger:$sharedMemCarveout + ); + let results = (outs); +} +def Cuda_LaunchCfgSyncPolicyOp : Cuda_Op<"launch_cfg.sync_policy"> { + let description = [{ + Sets syncPolicy for a launch_config + + Example: + ```mlir + %cfg0 = cuda.launch_cfg.create... + cuda.launch_cfg.sync_policy [%cfg0] cudaSynchronizationPolicyDefault + ``` + }]; + let arguments = (ins + Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, + Cuda_CudaSynchronizationPolicyEnum:$syncPolicy + ); + let results = (outs); +} +def Cuda_LaunchExOp : Cuda_Op<"launch_ex"> { + let description = [{ + Operation to launch a `cuda.kernel` on the device using extensible launch API. + + Examples: + + ```mlir + + cuda.module @module_name { + + cuda.func @kernel(%arg0 : i32) { + + } + + } + + ... + %0 = ... // kernel arguments + + %res = cuda.launch_ex @module_name::@device_func<%cfg>(%0) : !cuda.launch_cfg, ... -> !cuda.result + + // If calling with an Op that implements a FunctionOpInterface + %res = cuda.launch_ex @module_name::@device_func<%cfg>(%0) : !cuda.launch_cfg, ... -> !cuda.result + + %res2 = cuda.result.succeeded(%res) + scf.if %res2 { + + } + else { + + } + ``` + }]; + let arguments = (ins + Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, + Variadic:$inputs, + SymbolRefAttr:$callee, + OptionalAttr:$arg_attrs, + OptionalAttr:$res_attrs, + OptionalAttr:$assume_kernel_attr + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_LaunchKernelOp : Cuda_Op<"launch_kernel", [AttrSizedOperandSegments]> { + let description = [{ + Operation to launch a kernel from a kernel pointer on the device. This launch API + directly follows the CUDA C++ runtime API, passing all kernel arguments as a pointer + to a struct. + + Example: + + ```mlir + %res2 = cuda.launch_kernel %kernel, (%one32, %one32, %one32), (%one32, %one32, %one32), %zero64, %stream, %zeroPtr : !ptr.ptr<#ptr.generic_space> -> !cuda.result + + ``` + }]; + let arguments = (ins + Cuda_CUDAKernelHandle2:$kernel, + Variadic:$grid, + Variadic:$block, + Cuda_T64BitSignlessInteger:$dynamicSmem, + Cuda_CUDAStream:$cudaStream, + Cuda_PointerType:$kernel_args, + OptionalAttr:$arg_attrs, + OptionalAttr:$res_attrs + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_LibraryGetKernelOp : Cuda_Op<"library.get_kernel"> { + let description = [{ + Return a kernel handle object if found in the given library. + + ## Example + + ``` + %result_0, %kernel = cuda.library.get_kernel %l1, %name : !cuda.library, !ptr.ptr<#ptr.generic_space> -> !cuda.result, !cuda.kernel + ``` + + ## Behavior + + This op executes the CUDA API function cudaGetKernel. If the kernel is not found the cuda result will be `cudaErrorSymbolNotFound`. If the `kernel_name` is not a valid, null-terminated ASCII string, the behavior is UNDEFINED. Upon a result of `cudaSuccess`, `kernel` contains a valid kernel handle object. Is the result is not `cudaSuccess`, and use of the `kernel_name` handle is UNDEFINED. + }]; + let arguments = (ins + Cuda_CUDALibraryHandle:$library, + Cuda_PointerType:$kernel_name + ); + let results = (outs + Cuda_CUDAResultType:$result, + Cuda_CUDAKernelHandle:$kernel + ); +} +def Cuda_LibraryLoadDataOp : Cuda_Op<"library.load_data"> { + let description = [{ + Load a library from a pointer. + + ## Example + + ``` + %result,%l1 = cuda.library.load_data %modptr : !ptr.ptr<#ptr.generic_space> -> !cuda.result, !cuda.library + ``` + ## Behavior + + This op execute the CUDA API function cudaLibraryLoadData. Whether the library is loaded immediately depends on the module load environment. + }]; + let arguments = (ins + Cuda_PointerType:$data + ); + let results = (outs + Cuda_CUDAResultType:$result, + Cuda_CUDALibraryHandle2:$library + ); +} +def Cuda_LibraryUnloadOp : Cuda_Op<"library.unload"> { + let arguments = (ins + Cuda_CUDALibraryHandle:$library + ); + let results = (outs + Cuda_CUDAResultType:$result + ); +} +def Cuda_MemoryAllocOp : Cuda_Op<"memory.alloc"> { + let description = [{ + Operation to allocate device memory. Calls `cuMemAlloc()`. + + Examples: + ```mlir + + %bytes = arith.constant 0 : i64 + %res, %device_mem = cuda.memory.alloc(%bytes) : i64 -> cuda.result, cuda.ptr + + %res0 = cuda.launch @module::@kernel<<< %grid, %block, %cluster, %smem, %stream >>>(%device_mem) + + %res1 = cuda.memory.free(%device_mem) : i64 -> cuda.result + ``` + }]; + let arguments = (ins + Cuda_T64BitSignlessInteger:$bytes + ); + let results = (outs + Cuda_CUDAResultType2:$cuda_result, + Cuda_PointerType:$ptr + ); +} +def Cuda_MemoryAllocAsyncOp : Cuda_Op<"memory.alloc_async"> { + let description = [{ + Operation to allocate device memory asynchronously. Calls `cuMemAllocAsync()`. + + Examples: + ```mlir + + %res, %device_mem = cuda.memory.alloc_async(%bytes, %stream) : i64, cuda.ptr -> cuda.result + + cuda.launch @module::@kernel<<< %grid, %block, %cluster, %smem, %stream >>>(%device_mem) + + $res1 = cuda.memory.free_async(%device_mem, %stream) : i64, !cuda.stream -> cuda.result + ``` + }]; + let arguments = (ins + Cuda_T64BitSignlessInteger:$bytes, + Cuda_CUDAStream:$stream + ); + let results = (outs + Cuda_CUDAResultType2:$cuda_result, + Cuda_PointerType:$ptr + ); +} +def Cuda_MemoryAllocHostOp : Cuda_Op<"memory.alloc_host"> { + let description = [{ + Operation to allocate host memory. Calls `cuMemAllocHost()`. + + Examples: + ```mlir + + %bytes = arith.constant 0 : i64 + %host_mem, %res = cuda.memory.alloc_host(%bytes) : (i64) -> (cuda.ptr, cuda.result) + + %res0 = cuda.launch @module::@kernel<<< %grid, %block, %cluster, %smem, %stream >>>(%host_mem) + + %res1 = cuda.memory.free_host(%host_mem) + ``` + }]; + let arguments = (ins + Cuda_T64BitSignlessInteger:$bytes + ); + let results = (outs + Cuda_CUDAResultType2:$cuda_result, + Cuda_PointerType:$ptr + ); +} +def Cuda_MemoryAllocManagedOp : Cuda_Op<"memory.alloc_managed"> { + let description = [{ + Operation to allocate managed memory. + + Examples: + ```mlir + + %bytes = arith.constant 0 : i64 + %managed_mem, %res = cuda.memory.alloc_managed(%bytes) : (i64) -> (cuda.ptr, cuda.result) + + %res0 = cuda.launch @module::@kernel<<< %grid, %block, %cluster, %smem, %stream >>>(%managed_mem) + + %res1 = cuda.memory.free(%managed_mem) + ``` + }]; + let arguments = (ins + Cuda_T64BitSignlessInteger:$bytes + ); + let results = (outs + Cuda_CUDAResultType2:$cuda_result, + Cuda_PointerType:$ptr + ); +} +def Cuda_MemoryCopyOp : Cuda_Op<"memory.copy"> { + let description = [{ + Operation to copy memory between host and device. + + Examples: + ```mlir + + %bytes = arith.constant 1024 : i64 + + %device_mem, %res = cuda.memory.alloc %bytes : i64 -> (!cuda.result, !ptr.ptr) + + %res = cuda.memory.copy %device_mem, %host_mem, %bytes [HostToDevice] + + %res = cuda.launch @module::@kernel<<< (%gridX, %gridY, %gridZ), (%blockX, %blockY, %blockZ), %smem, %stream >>>(%device_mem) + + %res = cuda.memory.copy %host_mem, %device_mem, %bytes [DeviceToHost] + + %res = cuda.memory.free %device_mem : !ptr.ptr -> !cuda.result + ``` + }]; + let arguments = (ins + Cuda_PointerType:$src, + Cuda_PointerType:$dst, + Cuda_T64BitSignlessInteger:$bytes, + Cuda_MemcpyKindEnum:$kind + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_MemoryCopyAsyncOp : Cuda_Op<"memory.copy_async"> { + let description = [{ + Operation to copy memory between host and device asynchronously. + + Examples: + ```mlir + + %bytes = arith.constant 1024 : i64 + + %device_mem, %res = cuda.memory.alloc %bytes : i64 -> (!cuda.result, !ptr.ptr) + + %res = cuda.memory.copy_async %device_mem, %host_mem, %bytes [HostToDevice] + + %res = cuda.launch @module::@kernel<<< (%gridX, %gridY, %gridZ), (%blockX, %blockY, %blockZ), %smem, %stream >>>(%device_mem) + + %res = cuda.memory.copy_async %host_mem, %device_mem, %bytes [DeviceToHost] + + %res = cuda.memory.free %device_mem : !ptr.ptr -> !cuda.result + ``` + }]; + let arguments = (ins + Cuda_PointerType:$src, + Cuda_PointerType:$dst, + Cuda_T64BitSignlessInteger:$bytes, + Optional:$stream, + Cuda_MemcpyKindEnum:$kind + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_MemoryFreeOp : Cuda_Op<"memory.free"> { + let description = [{ + Operation to free allocated device memory. + + Examples: + ```mlir + + %bytes = arith.constant 0 : i64 + %res, %device_mem = cuda.memory.alloc(%bytes) : i64 -> cuda.result, cuda.ptr + + %res0 = cuda.launch @module::@kernel<<< %grid, %block, %cluster, %smem, %stream >>>(%device_mem) + + %res1 = cuda.memory.free(%device_mem) : i64 -> cuda.result + ``` + }]; + let arguments = (ins + Cuda_PointerType:$ptr + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_MemoryFreeAsyncOp : Cuda_Op<"memory.free_async"> { + let description = [{ + Operation to free allocated device memory asynchronously. + + Examples: + ```mlir + + %res, %device_mem = cuda.memory.alloc_async(%bytes, %stream) : i64, cuda.ptr -> cuda.result + + cuda.launch @module::@kernel<<< %grid, %block, %cluster, %smem, %stream >>>(%device_mem) + + $res1 = cuda.memory.free_async(%device_mem, %stream) : i64, !cuda.stream -> cuda.result + ``` + + Preserves semantics of [cuMemAllocAsync](https://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MALLOC__ASYNC.html#group__CUDA__MALLOC__ASYNC_1g13413273e84a641bce1929eae9e6501f) + }]; + let arguments = (ins + Cuda_PointerType:$ptr, + Cuda_CUDAStream:$stream + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_MemoryFreeHostOp : Cuda_Op<"memory.free_host"> { + let description = [{ + Operation to free allocated host memory. + + Examples: + ```mlir + + %bytes = arith.constant 0 : i64 + %host_mem, %res = cuda.memory.alloc_host(%bytes) : (i64) -> (cuda.ptr, cuda.result) + + %res0 = cuda.launch @module::@kernel<<< %grid, %block, %cluster, %smem, %stream >>>(%host_mem) + + %res1 = cuda.memory.free_host(%host_mem) + ``` + }]; + let arguments = (ins + Cuda_PointerType:$ptr + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_MemorySetOp : Cuda_Op<"memory.set"> { + let description = [{ + Operation to fill memory with a uniform value. + }]; + let arguments = (ins + Cuda_PointerType:$ptr, + Cuda_T32BitSignlessInteger:$value, + Cuda_T64BitSignlessInteger:$bytes + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_MemorySetAsyncOp : Cuda_Op<"memory.set_async"> { + let description = [{ + Operation to fill memory with a uniform value asynchronously. + }]; + let arguments = (ins + Cuda_PointerType:$ptr, + Cuda_T32BitSignlessInteger:$value, + Cuda_T64BitSignlessInteger:$bytes, + Optional:$stream + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_OccupancyMaxActiveBlocks : Cuda_Op<"occupancy_max_active_blocks_per_multiprocessor"> { + let description = [{ + Returns in `num_blocks` the maximum number of active blocks per streaming multiplrocessor for the device function. + + Example: + + ```mlir + %one64 = arith.constant 1 : i64 + %one32 = arith.constant 1 : i32 + %res, %num_blocks = cuda.occupancy_max_active_blocks_per_multiprocessor @funcname, %one32, %one64 : (i32, i64) -> (!cuda.result, i32) + ``` + }]; + let arguments = (ins + Cuda_CudaIntIsAIntegerType:$block_size, + Cuda_T64BitSignlessInteger:$dynamic_smem_size, + SymbolRefAttr:$func + ); + let results = (outs + Cuda_CUDAResultType:$res, + Cuda_T32BitSignlessInteger2:$num_blocks + ); +} +def Cuda_ProfilerStart : Cuda_Op<"profiler.start"> { + let description = [{ + Starts the CUDA profiler. + }]; + let arguments = (ins); + let results = (outs + Cuda_CUDAResultType:$result + ); +} +def Cuda_ProfilerStop : Cuda_Op<"profiler.stop"> { + let description = [{ + Stops the CUDA profiler. + }]; + let arguments = (ins); + let results = (outs + Cuda_CUDAResultType:$result + ); +} +def Cuda_ResultAssertSuccessOp : Cuda_Op<"result.assert_success"> { + let description = [{ + Asserts that a `cuda.result` indicates success. + }]; + let arguments = (ins + Cuda_CUDAResultType2:$cuda_result + ); + let results = (outs); +} +def Cuda_ResultFailedOp : Cuda_Op<"result.failed"> { + let description = [{ + Determines if a `cuda.result` indicates failure. + }]; + let arguments = (ins); + let results = (outs); +} +def Cuda_ResultPrintOnErrorOp : Cuda_Op<"result.print_on_error"> { + let description = [{ + Prints a message on error. + }]; + let arguments = (ins + Cuda_CUDAResultType2:$cuda_result + ); + let results = (outs); +} +def Cuda_ResultSucceededOp : Cuda_Op<"result.succeeded"> { + let description = [{ + Determines if a `cuda.result` indicates success. + }]; + let arguments = (ins + Cuda_CUDAResultType2:$cuda_result + ); + let results = (outs + Cuda_T1BitSignlessInteger:$success + ); +} +def Cuda_ReturnOp : Cuda_Op<"return", [Terminator]> { + let description = [{ + Returns a value from a `cuda.kernel` or `cuda.func` region. + + Note, the emitter seems to omit the dialect and simply prints `return` in the + custom assembly syntax. + + Examples: + + ```mlir + + cuda.module @module_name { + + cuda.func @device_func(%arg0 : i32) -> i32 { + %0 = arith.constant 0 : i32 + cuda.return %0 : i32 + } + + } + ``` + }]; + let arguments = (ins + Variadic:$inputs + ); + let results = (outs); +} +def Cuda_ReturnIfErrorOp : Cuda_Op<"return_if_error"> { + let description = [{ + The first operand must be the !cuda.result to check. + The subsequent operands are the values to be returned if an error exists. + This op is first lowered to an intermediate op `cuda.return_with_error`, + before final conversion to LLVM in a subsequent pass `cuda-finalize-error-handling`. + + Examples: + ```mlir + func.func @host_func() -> i32 { + %ten = arith.constant 10 : i32 + %cuda_result = cuda.launch @module::@kernel... + // Create an undefined/poison value for the error return path. + // Users can also input any arbitrary value for the error return path. + %poison = llvm.mlir.poison : i32 + // return poison if cuda_result is not cudaSuccess + %res_i32 = cuda.cast %cuda_result : !cuda.result -> i32 + cuda.return_if_error %res_i32, %poison : i32, i32 + + // return 10 if execution reaches here + return %ten : i32 + } + ``` + }]; + let arguments = (ins + Variadic:$inputs + ); + let results = (outs); +} +def Cuda_RuntimeGetDevicePropertiesOp : Cuda_Op<"runtime.device.properties"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_T32BitSignlessInteger:$device + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result, + Cuda_CUDACudaDevicePropStructTypeValueTyped:$prop + ); +} +def Cuda_RuntimeDevicePropGetAccessPolicyMaxWindowSizeOp : Cuda_Op<"runtime.device_prop.get_access_policy_max_window_size", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetAsyncEngineCountOp : Cuda_Op<"runtime.device_prop.get_async_engine_count", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetCanMapHostMemoryOp : Cuda_Op<"runtime.device_prop.get_can_map_host_memory", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetCanUseHostPointerForRegisteredMemOp : Cuda_Op<"runtime.device_prop.get_can_use_host_pointer_for_registered_mem", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetClusterLaunchOp : Cuda_Op<"runtime.device_prop.get_cluster_launch", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetComputePreemptionSupportedOp : Cuda_Op<"runtime.device_prop.get_compute_preemption_supported", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetConcurrentKernelsOp : Cuda_Op<"runtime.device_prop.get_concurrent_kernels", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetConcurrentManagedAccessOp : Cuda_Op<"runtime.device_prop.get_concurrent_managed_access", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetCooperativeLaunchOp : Cuda_Op<"runtime.device_prop.get_cooperative_launch", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetDeferredMappingCudaArraySupportedOp : Cuda_Op<"runtime.device_prop.get_deferred_mapping_cuda_array_supported", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetDeviceNumaConfigOp : Cuda_Op<"runtime.device_prop.get_device_numa_config", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetDeviceNumaIdOp : Cuda_Op<"runtime.device_prop.get_device_numa_id", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetDirectManagedMemAccessFromHostOp : Cuda_Op<"runtime.device_prop.get_direct_managed_mem_access_from_host", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetECCEnabledOp : Cuda_Op<"runtime.device_prop.get_ecc_enabled", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetGlobalL1CacheSupportedOp : Cuda_Op<"runtime.device_prop.get_global_l1cache_supported", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetGpuDirectRDMAFlushWritesOptionsOp : Cuda_Op<"runtime.device_prop.get_gpu_direct_rdma_flush_writes_options", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetGpuDirectRDMASupportedOp : Cuda_Op<"runtime.device_prop.get_gpu_direct_rdma_supported", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetGpuDirectRDMAWritesOrderingOp : Cuda_Op<"runtime.device_prop.get_gpu_direct_rdma_writes_ordering", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetGpuPciDeviceIDOp : Cuda_Op<"runtime.device_prop.get_gpu_pci_device_id", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetGpuPciSubsystemIDOp : Cuda_Op<"runtime.device_prop.get_gpu_pci_subsystem_id", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetHostNativeAtomicSupportedOp : Cuda_Op<"runtime.device_prop.get_host_native_atomic_supported", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetHostNumaIdOp : Cuda_Op<"runtime.device_prop.get_host_numa_id", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetHostNumaMultinodeIpcSupportedOp : Cuda_Op<"runtime.device_prop.get_host_numa_multinode_ipc_supported", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetHostRegisterReadOnlySupportedOp : Cuda_Op<"runtime.device_prop.get_host_register_read_only_supported", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetHostRegisterSupportedOp : Cuda_Op<"runtime.device_prop.get_host_register_supported", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetIntegratedOp : Cuda_Op<"runtime.device_prop.get_integrated", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetIpcEventSupportedOp : Cuda_Op<"runtime.device_prop.get_ipc_event_supported", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetIsMultiGpuBoardOp : Cuda_Op<"runtime.device_prop.get_is_multi_gpu_board", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetL2CacheSizeOp : Cuda_Op<"runtime.device_prop.get_l_2cache_size", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetLocalL1CacheSupportedOp : Cuda_Op<"runtime.device_prop.get_local_l1cache_supported", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetLuidOp : Cuda_Op<"runtime.device_prop.get_luid", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_FixedLengthVectorOf8Bit:$result + ); +} +def Cuda_RuntimeDevicePropGetLuidDeviceNodeMaskOp : Cuda_Op<"runtime.device_prop.get_luid_device_node_mask", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetMajorOp : Cuda_Op<"runtime.device_prop.get_major", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetManagedMemoryOp : Cuda_Op<"runtime.device_prop.get_managed_memory", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxBlocksPerMultiProcessorOp : Cuda_Op<"runtime.device_prop.get_max_blocks_per_multi_processor", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxGridSizeOp : Cuda_Op<"runtime.device_prop.get_max_grid_size", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_FixedLengthVectorOf32Bit:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxSurface1DOp : Cuda_Op<"runtime.device_prop.get_max_surface_1d", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxSurface1DLayeredOp : Cuda_Op<"runtime.device_prop.get_max_surface_1d_layered", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_FixedLengthVectorOf32Bit2:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxSurface2DOp : Cuda_Op<"runtime.device_prop.get_max_surface_2d", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_FixedLengthVectorOf32Bit2:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxSurface2DLayeredOp : Cuda_Op<"runtime.device_prop.get_max_surface_2d_layered", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_FixedLengthVectorOf32Bit:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxSurface3DOp : Cuda_Op<"runtime.device_prop.get_max_surface_3d", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_FixedLengthVectorOf32Bit:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxSurfaceCubemapOp : Cuda_Op<"runtime.device_prop.get_max_surface_cubemap", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxSurfaceCubemapLayeredOp : Cuda_Op<"runtime.device_prop.get_max_surface_cubemap_layered", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_FixedLengthVectorOf32Bit2:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxTexture1DOp : Cuda_Op<"runtime.device_prop.get_max_texture_1d", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxTexture1DLayeredOp : Cuda_Op<"runtime.device_prop.get_max_texture_1d_layered", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_FixedLengthVectorOf32Bit2:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxTexture1DMipmapOp : Cuda_Op<"runtime.device_prop.get_max_texture_1d_mipmap", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxTexture2DOp : Cuda_Op<"runtime.device_prop.get_max_texture_2d", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_FixedLengthVectorOf32Bit2:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxTexture2DGatherOp : Cuda_Op<"runtime.device_prop.get_max_texture_2d_gather", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_FixedLengthVectorOf32Bit2:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxTexture2DLayeredOp : Cuda_Op<"runtime.device_prop.get_max_texture_2d_layered", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_FixedLengthVectorOf32Bit:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxTexture2DLinearOp : Cuda_Op<"runtime.device_prop.get_max_texture_2d_linear", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_FixedLengthVectorOf32Bit:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxTexture2DMipmapOp : Cuda_Op<"runtime.device_prop.get_max_texture_2d_mipmap", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_FixedLengthVectorOf32Bit2:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxTexture3DOp : Cuda_Op<"runtime.device_prop.get_max_texture_3d", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_FixedLengthVectorOf32Bit:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxTexture3DAltOp : Cuda_Op<"runtime.device_prop.get_max_texture_3d_alt", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_FixedLengthVectorOf32Bit:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxTextureCubemapOp : Cuda_Op<"runtime.device_prop.get_max_texture_cubemap", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxTextureCubemapLayeredOp : Cuda_Op<"runtime.device_prop.get_max_texture_cubemap_layered", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_FixedLengthVectorOf32Bit2:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxThreadsDimOp : Cuda_Op<"runtime.device_prop.get_max_threads_dim", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_FixedLengthVectorOf32Bit:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxThreadsPerBlockOp : Cuda_Op<"runtime.device_prop.get_max_threads_per_block", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetMaxThreadsPerMultiProcessorOp : Cuda_Op<"runtime.device_prop.get_max_threads_per_multi_processor", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetMemPitchOp : Cuda_Op<"runtime.device_prop.get_mem_pitch", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T64BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetMemoryBusWidthOp : Cuda_Op<"runtime.device_prop.get_memory_bus_width", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetMemoryPoolSupportedHandleTypesOp : Cuda_Op<"runtime.device_prop.get_memory_pool_supported_handle_types", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetMemoryPoolsSupportedOp : Cuda_Op<"runtime.device_prop.get_memory_pools_supported", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetMinorOp : Cuda_Op<"runtime.device_prop.get_minor", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetMpsEnabledOp : Cuda_Op<"runtime.device_prop.get_mps_enabled", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetMultiGpuBoardGroupIDOp : Cuda_Op<"runtime.device_prop.get_multi_gpu_board_group_id", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetMultiProcessorCountOp : Cuda_Op<"runtime.device_prop.get_multi_processor_count", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetNameOp : Cuda_Op<"runtime.device_prop.get_name", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_FixedLengthVectorOf8Bit2:$result + ); +} +def Cuda_RuntimeDevicePropGetPageableMemoryAccessOp : Cuda_Op<"runtime.device_prop.get_pageable_memory_access", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetPageableMemoryAccessUsesHostPageTablesOp : Cuda_Op<"runtime.device_prop.get_pageable_memory_access_uses_host_page_tables", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetPciBusIDOp : Cuda_Op<"runtime.device_prop.get_pci_bus_id", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetPciDeviceIDOp : Cuda_Op<"runtime.device_prop.get_pci_device_id", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetPciDomainIDOp : Cuda_Op<"runtime.device_prop.get_pci_domain_id", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetPersistingL2CacheMaxSizeOp : Cuda_Op<"runtime.device_prop.get_persisting_l2cache_max_size", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetRegsPerBlockOp : Cuda_Op<"runtime.device_prop.get_regs_per_block", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetRegsPerMultiprocessorOp : Cuda_Op<"runtime.device_prop.get_regs_per_multiprocessor", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetReservedSharedMemPerBlockOp : Cuda_Op<"runtime.device_prop.get_reserved_shared_mem_per_block", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T64BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetSharedMemPerBlockOp : Cuda_Op<"runtime.device_prop.get_shared_mem_per_block", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T64BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetSharedMemPerBlockOptinOp : Cuda_Op<"runtime.device_prop.get_shared_mem_per_block_optin", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T64BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetSharedMemPerMultiprocessorOp : Cuda_Op<"runtime.device_prop.get_shared_mem_per_multiprocessor", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T64BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetSparseCudaArraySupportedOp : Cuda_Op<"runtime.device_prop.get_sparse_cuda_array_supported", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetStreamPrioritiesSupportedOp : Cuda_Op<"runtime.device_prop.get_stream_priorities_supported", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetSurfaceAlignmentOp : Cuda_Op<"runtime.device_prop.get_surface_alignment", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T64BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetTccDriverOp : Cuda_Op<"runtime.device_prop.get_tcc_driver", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetTextureAlignmentOp : Cuda_Op<"runtime.device_prop.get_texture_alignment", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T64BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetTexturePitchAlignmentOp : Cuda_Op<"runtime.device_prop.get_texture_pitch_alignment", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T64BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetTimelineSemaphoreInteropSupportedOp : Cuda_Op<"runtime.device_prop.get_timeline_semaphore_interop_supported", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetTotalConstMemOp : Cuda_Op<"runtime.device_prop.get_total_const_mem", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T64BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetTotalGlobalMemOp : Cuda_Op<"runtime.device_prop.get_total_global_mem", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T64BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetUnifiedAddressingOp : Cuda_Op<"runtime.device_prop.get_unified_addressing", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetUnifiedFunctionPointersOp : Cuda_Op<"runtime.device_prop.get_unified_function_pointers", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDevicePropGetUuidOp : Cuda_Op<"runtime.device_prop.get_uuid", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_CUDACudaUUIDTStructTypeValue:$result + ); +} +def Cuda_RuntimeDevicePropGetWarpSizeOp : Cuda_Op<"runtime.device_prop.get_warp_size", [Pure]> { + let arguments = (ins + Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDim3GetXOp : Cuda_Op<"runtime.dim_3.get_x", [Pure]> { + let arguments = (ins + Cuda_CUDADim3StructTypeValueTyped:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDim3GetYOp : Cuda_Op<"runtime.dim_3.get_y", [Pure]> { + let arguments = (ins + Cuda_CUDADim3StructTypeValueTyped:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeDim3GetZOp : Cuda_Op<"runtime.dim_3.get_z", [Pure]> { + let arguments = (ins + Cuda_CUDADim3StructTypeValueTyped:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeEventCreateWithFlagsOp : Cuda_Op<"runtime.event.create_with_flags"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_T32BitSignlessInteger:$flags + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result, + Cuda_EventRepresentsCUDAEvent:$event + ); +} +def Cuda_RuntimeEventQueryOp : Cuda_Op<"runtime.event.query", [MemoryEffects<[MemRead]>]> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_EventRepresentsCUDAEvent2:$event + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_RuntimeEventRecordWithFlagsOp : Cuda_Op<"runtime.event.record_with_flags"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_EventRepresentsCUDAEvent2:$event, + Cuda_CUDAStream:$stream, + Cuda_T32BitSignlessInteger:$flags + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_RuntimeEventSynchronizeOp : Cuda_Op<"runtime.event.synchronize"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_EventRepresentsCUDAEvent2:$event + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_RuntimeExtentGetDepthOp : Cuda_Op<"runtime.extent.get_depth", [Pure]> { + let arguments = (ins + Cuda_CUDACudaExtentStructTypeValueTyped:$source + ); + let results = (outs + Cuda_T64BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeExtentGetHeightOp : Cuda_Op<"runtime.extent.get_height", [Pure]> { + let arguments = (ins + Cuda_CUDACudaExtentStructTypeValueTyped:$source + ); + let results = (outs + Cuda_T64BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeExtentGetWidthOp : Cuda_Op<"runtime.extent.get_width", [Pure]> { + let arguments = (ins + Cuda_CUDACudaExtentStructTypeValueTyped:$source + ); + let results = (outs + Cuda_T64BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeGraphAddKernelNodeOp : Cuda_Op<"runtime.graph.add_kernel_node"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_CUDAGraph:$graph, + Cuda_PointerType:$pDependencies, + Cuda_T64BitSignlessInteger:$numDependencies, + Cuda_CUDACudaKernelNodeParamsStructTypeValueTyped:$pNodeParams + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result, + Cuda_CUDAGraphNodeHandle:$pGraphNode + ); +} +def Cuda_RuntimeGraphCreateOp : Cuda_Op<"runtime.graph.create"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_T32BitSignlessInteger:$flags + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result, + Cuda_CUDAGraph2:$pGraph + ); +} +def Cuda_RuntimeGraphKernelNodeGetAttributeOp : Cuda_Op<"runtime.graph.kernel_node_get_attribute", [MemoryEffects<[MemRead]>]> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_CUDAGraphNodeHandle2:$hNode, + Cuda_T32BitSignlessInteger:$attr + ); + let results = (outs + Cuda_CUDAResultType2:$cuda_result, + Cuda_VariantCooperativeOfCUDACudaLaunchAttributeValueUnion:$value_out + ); +} +def Cuda_RuntimeGraphKernelNodeSetAttributeOp : Cuda_Op<"runtime.graph.kernel_node_set_attribute"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_CUDAGraphNodeHandle2:$hNode, + Cuda_T32BitSignlessInteger:$attr, + Cuda_VariantCooperativeOfCUDACudaLaunchAttributeValueUnion:$value + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_RuntimeKernelNodeParamsGetBlockDimOp : Cuda_Op<"runtime.kernel_node_params.get_block_dim", [Pure]> { + let arguments = (ins + Cuda_CUDACudaKernelNodeParamsStructTypeValueTyped:$source + ); + let results = (outs + Cuda_CUDADim3StructTypeValueTyped2:$result + ); +} +def Cuda_RuntimeKernelNodeParamsGetExtraOp : Cuda_Op<"runtime.kernel_node_params.get_extra", [Pure]> { + let arguments = (ins + Cuda_CUDACudaKernelNodeParamsStructTypeValueTyped:$source + ); + let results = (outs + Cuda_LLVMPointerType:$result + ); +} +def Cuda_RuntimeKernelNodeParamsGetFuncOp : Cuda_Op<"runtime.kernel_node_params.get_func", [Pure]> { + let arguments = (ins + Cuda_CUDACudaKernelNodeParamsStructTypeValueTyped:$source + ); + let results = (outs + Cuda_LLVMPointerType:$result + ); +} +def Cuda_RuntimeKernelNodeParamsGetGridDimOp : Cuda_Op<"runtime.kernel_node_params.get_grid_dim", [Pure]> { + let arguments = (ins + Cuda_CUDACudaKernelNodeParamsStructTypeValueTyped:$source + ); + let results = (outs + Cuda_CUDADim3StructTypeValueTyped2:$result + ); +} +def Cuda_RuntimeKernelNodeParamsGetKernelParamsOp : Cuda_Op<"runtime.kernel_node_params.get_kernel_params", [Pure]> { + let arguments = (ins + Cuda_CUDACudaKernelNodeParamsStructTypeValueTyped:$source + ); + let results = (outs + Cuda_LLVMPointerType:$result + ); +} +def Cuda_RuntimeKernelNodeParamsGetSharedMemBytesOp : Cuda_Op<"runtime.kernel_node_params.get_shared_mem_bytes", [Pure]> { + let arguments = (ins + Cuda_CUDACudaKernelNodeParamsStructTypeValueTyped:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeLibraryGetGlobalOp : Cuda_Op<"runtime.library.get_global", [MemoryEffects<[MemRead]>]> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_CUDALibraryHandle:$library, + Cuda_PointerType:$name_arg + ); + let results = (outs + Cuda_CUDAResultType2:$cuda_result, + Cuda_PointerType:$dptr, + Cuda_T64BitSignlessInteger:$bytes + ); +} +def Cuda_RuntimeLibraryGetKernelOp : Cuda_Op<"runtime.library.get_kernel", [MemoryEffects<[MemRead]>]> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_CUDALibraryHandle:$library, + Cuda_PointerType:$name_arg + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result, + Cuda_CUDAKernelHandle:$pKernel + ); +} +def Cuda_RuntimeLibraryGetKernelCountOp : Cuda_Op<"runtime.library.get_kernel_count", [MemoryEffects<[MemRead]>]> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_CUDALibraryHandle:$lib + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result, + Cuda_T32BitSignlessInteger2:$count + ); +} +def Cuda_RuntimeLibraryGetManagedOp : Cuda_Op<"runtime.library.get_managed", [MemoryEffects<[MemRead]>]> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_CUDALibraryHandle:$library, + Cuda_PointerType:$name_arg + ); + let results = (outs + Cuda_CUDAResultType2:$cuda_result, + Cuda_PointerType:$dptr, + Cuda_T64BitSignlessInteger:$bytes + ); +} +def Cuda_RuntimeLibraryGetUnifiedFunctionOp : Cuda_Op<"runtime.library.get_unified_function", [MemoryEffects<[MemRead]>]> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_CUDALibraryHandle:$library, + Cuda_PointerType:$symbol + ); + let results = (outs + Cuda_CUDAResultType2:$cuda_result, + Cuda_PointerType:$fptr + ); +} +def Cuda_RuntimeLibraryLoadDataOp : Cuda_Op<"runtime.library.load_data"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_PointerType:$code_arg, + Cuda_PointerType:$jitOptions, + Cuda_PointerType:$jitOptionsValues, + Cuda_T32BitSignlessInteger:$numJitOptions, + Cuda_PointerType:$libraryOptions, + Cuda_PointerType:$libraryOptionValues, + Cuda_T32BitSignlessInteger:$numLibraryOptions + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result, + Cuda_CUDALibraryHandle2:$library + ); +} +def Cuda_RuntimeLibraryLoadFromFileOp : Cuda_Op<"runtime.library.load_from_file"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_PointerType:$fileName, + Cuda_PointerType:$jitOptions, + Cuda_PointerType:$jitOptionsValues, + Cuda_T32BitSignlessInteger:$numJitOptions, + Cuda_PointerType:$libraryOptions, + Cuda_PointerType:$libraryOptionValues, + Cuda_T32BitSignlessInteger:$numLibraryOptions + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result, + Cuda_CUDALibraryHandle2:$library + ); +} +def Cuda_RuntimeLibraryUnloadOp : Cuda_Op<"runtime.library.unload"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_CUDALibraryHandle:$library + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_RuntimeMemcpy3DParamsGetDstArrayOp : Cuda_Op<"runtime.memcpy_3d_params.get_dst_array", [Pure]> { + let arguments = (ins + Cuda_CUDACudaMemcpy3DParmsStructTypeValueTyped:$source + ); + let results = (outs + Cuda_PointerType:$result + ); +} +def Cuda_RuntimeMemcpy3DParamsGetDstPosOp : Cuda_Op<"runtime.memcpy_3d_params.get_dst_pos", [Pure]> { + let arguments = (ins + Cuda_CUDACudaMemcpy3DParmsStructTypeValueTyped:$source + ); + let results = (outs + Cuda_CUDACudaPosStructTypeValueTyped:$result + ); +} +def Cuda_RuntimeMemcpy3DParamsGetDstPtrOp : Cuda_Op<"runtime.memcpy_3d_params.get_dst_ptr", [Pure]> { + let arguments = (ins + Cuda_CUDACudaMemcpy3DParmsStructTypeValueTyped:$source + ); + let results = (outs + Cuda_CUDACudaPitchedPtrStructTypeValueTyped:$result + ); +} +def Cuda_RuntimeMemcpy3DParamsGetExtentOp : Cuda_Op<"runtime.memcpy_3d_params.get_extent", [Pure]> { + let arguments = (ins + Cuda_CUDACudaMemcpy3DParmsStructTypeValueTyped:$source + ); + let results = (outs + Cuda_CUDACudaExtentStructTypeValueTyped2:$result + ); +} +def Cuda_RuntimeMemcpy3DParamsGetKindOp : Cuda_Op<"runtime.memcpy_3d_params.get_kind", [Pure]> { + let arguments = (ins + Cuda_CUDACudaMemcpy3DParmsStructTypeValueTyped:$source + ); + let results = (outs + Cuda_T32BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeMemcpy3DParamsGetSrcArrayOp : Cuda_Op<"runtime.memcpy_3d_params.get_src_array", [Pure]> { + let arguments = (ins + Cuda_CUDACudaMemcpy3DParmsStructTypeValueTyped:$source + ); + let results = (outs + Cuda_PointerType:$result + ); +} +def Cuda_RuntimeMemcpy3DParamsGetSrcPosOp : Cuda_Op<"runtime.memcpy_3d_params.get_src_pos", [Pure]> { + let arguments = (ins + Cuda_CUDACudaMemcpy3DParmsStructTypeValueTyped:$source + ); + let results = (outs + Cuda_CUDACudaPosStructTypeValueTyped:$result + ); +} +def Cuda_RuntimeMemcpy3DParamsGetSrcPtrOp : Cuda_Op<"runtime.memcpy_3d_params.get_src_ptr", [Pure]> { + let arguments = (ins + Cuda_CUDACudaMemcpy3DParmsStructTypeValueTyped:$source + ); + let results = (outs + Cuda_CUDACudaPitchedPtrStructTypeValueTyped:$result + ); +} +def Cuda_RuntimeMemcpy3DOp : Cuda_Op<"runtime.memory.cpy3d"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_CUDACudaMemcpy3DParmsStructTypeValueTyped:$p + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_RuntimeMemcpy3DAsyncOp : Cuda_Op<"runtime.memory.cpy3d_async"> { + let description = [{ + Base class for all CUDA driver call builder operations. It provides a common + interface for building CUDA driver calls, including the ability to specify + the API to be called, the arguments, and the expected results. + This class is not intended to be used directly, but rather as a base class + for specific CUDA driver call builder operations. + }]; + let arguments = (ins + Cuda_CUDACudaMemcpy3DParmsStructTypeValueTyped:$p, + Cuda_CUDAStream:$stream + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result + ); +} +def Cuda_RuntimePitchedPtrGetPitchOp : Cuda_Op<"runtime.pitched_ptr.get_pitch", [Pure]> { + let arguments = (ins + Cuda_CUDACudaPitchedPtrStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T64BitSignlessInteger2:$result + ); +} +def Cuda_RuntimePitchedPtrGetPtrOp : Cuda_Op<"runtime.pitched_ptr.get_ptr", [Pure]> { + let arguments = (ins + Cuda_CUDACudaPitchedPtrStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_LLVMPointerType:$result + ); +} +def Cuda_RuntimePitchedPtrGetXsizeOp : Cuda_Op<"runtime.pitched_ptr.get_xsize", [Pure]> { + let arguments = (ins + Cuda_CUDACudaPitchedPtrStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T64BitSignlessInteger2:$result + ); +} +def Cuda_RuntimePitchedPtrGetYsizeOp : Cuda_Op<"runtime.pitched_ptr.get_ysize", [Pure]> { + let arguments = (ins + Cuda_CUDACudaPitchedPtrStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T64BitSignlessInteger2:$result + ); +} +def Cuda_RuntimePosGetXOp : Cuda_Op<"runtime.pos.get_x", [Pure]> { + let arguments = (ins + Cuda_CUDACudaPosStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T64BitSignlessInteger2:$result + ); +} +def Cuda_RuntimePosGetYOp : Cuda_Op<"runtime.pos.get_y", [Pure]> { + let arguments = (ins + Cuda_CUDACudaPosStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T64BitSignlessInteger2:$result + ); +} +def Cuda_RuntimePosGetZOp : Cuda_Op<"runtime.pos.get_z", [Pure]> { + let arguments = (ins + Cuda_CUDACudaPosStructTypeValueTyped2:$source + ); + let results = (outs + Cuda_T64BitSignlessInteger2:$result + ); +} +def Cuda_RuntimeUuidGetBytesOp : Cuda_Op<"runtime.uuid.get_bytes", [Pure]> { + let arguments = (ins + Cuda_CUDACudaUUIDTStructTypeValue2:$source + ); + let results = (outs + Cuda_FixedLengthVectorOf8Bit3:$result + ); +} +def Cuda_StreamAddCallbackOp : Cuda_Op<"stream.add_callback"> { + let description = [{ + Enqueues a host-side callback on the given CUDA stream. The callback is provided + as a symbol to a `func.func` with ABI: + `func @cb(!ptr.ptr<#ptr.generic_space>, i32, !ptr.ptr<#ptr.generic_space>) -> ()` + i.e., (stream, status, userData) -> void. + }]; + let arguments = (ins + Cuda_CUDAStream:$stream, + Cuda_PointerType:$user_data, + FlatSymbolRefAttr:$callback + ); + let results = (outs + Cuda_CUDAResultType:$result + ); +} +def Cuda_StreamBeginCaptureOp : Cuda_Op<"stream.begin_capture"> { + let description = [{ + Operation to begin capturing a CUDA stream. + }]; + let arguments = (ins + Cuda_CUDAStream:$stream, + Cuda_StreamCaptureModeEnumEnum:$mode + ); + let results = (outs + Cuda_CUDAResultType:$result + ); +} +def Cuda_StreamCreateOp : Cuda_Op<"stream.create"> { + let description = [{ + Operation to create a CUDA stream. + }]; + let arguments = (ins); + let results = (outs + Cuda_CUDAResultType:$cuda_result, + Cuda_CUDAStream2:$stream + ); +} +def Cuda_StreamDestroyOp : Cuda_Op<"stream.destroy"> { + let description = [{ + Operation to destroy a CUDA stream. + }]; + let arguments = (ins + Cuda_CUDAStream:$stream + ); + let results = (outs + Cuda_CUDAResultType:$result + ); +} +def Cuda_StreamEndCaptureOp : Cuda_Op<"stream.end_capture"> { + let description = [{ + Operation to end capturing a CUDA stream. + }]; + let arguments = (ins + Cuda_CUDAStream:$stream + ); + let results = (outs + Cuda_CUDAResultType:$result, + Cuda_CUDAGraph2:$graph + ); +} +def Cuda_StreamSynchronizeOp : Cuda_Op<"stream.synchronize", [MemoryEffects<[MemRead]>]> { + let description = [{ + Operation to synchronize with a CUDA Stream. + }]; + let arguments = (ins + Cuda_CUDAStream:$stream, + Optional:$event + ); + let results = (outs + Cuda_CUDAResultType:$result + ); +} +def Cuda_TensorMapEncodeTiledOp : Cuda_Op<"tensor_map_encode_tiled"> { + let description = [{ + Creates a descriptor for Tensor Memory Access (TMA) object specified by the parameters + describing a tiled region and returns it in tensorMap. + + This operation corresponds to the CUDA Driver API function: + `cuTensorMapEncodeTiled` + + Tensor map objects are only supported on devices of compute capability 9.0 or higher. + The tensorMap address must be aligned to 64 bytes. + + All parameters are provided as runtime SSA values. + }]; + let arguments = (ins + Cuda_T32BitSignlessInteger:$tensorDataType, + Cuda_T32BitSignlessInteger:$tensorRank, + Cuda_PointerType:$globalAddress, + Cuda_LLVMPointerInAddressSpace0:$globalDims, + Cuda_LLVMPointerInAddressSpace0:$globalStrides, + Cuda_LLVMPointerInAddressSpace0:$boxDims, + Cuda_LLVMPointerInAddressSpace0:$elementStrides, + Cuda_T32BitSignlessInteger:$interleave, + Cuda_T32BitSignlessInteger:$swizzle, + Cuda_T32BitSignlessInteger:$l2Promotion, + Cuda_T32BitSignlessInteger:$oobFill + ); + let results = (outs + Cuda_CUDAResultType:$cuda_result, + Cuda_CUDATensorMapType:$cuTensorMap + ); +} + +#endif // MLIR_DIALECTS_CUDA_CUDAOPS_TD diff --git a/mlir/dialects/cuda/CudaTypes.td b/mlir/dialects/cuda/CudaTypes.td new file mode 100644 index 00000000..d149651e --- /dev/null +++ b/mlir/dialects/cuda/CudaTypes.td @@ -0,0 +1,212 @@ +//===- CudaTypes.td - CUDA host types of the CuTe DSL ------*- tablegen -*-===// +// +// Generated for nvidia-cutlass-dsl 4.8.0 from the DSL compiler (its typed +// Python type classes, its mnemonics, `_cutlass_ir.cu13` parser messages and +// the verifier's wording for each type); do not edit by hand. +// +// Every cuda type the compiler's parser knows. Only `launch_cfg` has a +// parameter; the C++ class names not given by the typed Python classes +// (`!cuda.int`, the graph handles, `arch_set`, `kernel`, `module` and the +// launch attribute values) follow the mnemonic. +// +//===----------------------------------------------------------------------===// + +#ifndef MLIR_DIALECTS_CUDA_CUDATYPES_TD +#define MLIR_DIALECTS_CUDA_CUDATYPES_TD + +include "mlir/dialects/cuda/CudaDialect.td" + +def Cuda_ArchSetType : Cuda_Type<"ArchSet", "arch_set"> { + let summary = "`!cuda.arch_set`"; + let description = "`!cuda.arch_set`"; + let assemblyFormat = ""; +} + +def Cuda_EventType : Cuda_Type<"Event", "event"> { + let summary = "`Event` represents CUDA event"; + let description = "`!cuda.event`"; + let assemblyFormat = ""; +} + +def Cuda_GraphType : Cuda_Type<"Graph", "graph"> { + let summary = "CUDA Graph"; + let description = "`!cuda.graph`"; + let assemblyFormat = ""; +} + +def Cuda_GraphDeviceNodeType : Cuda_Type<"GraphDeviceNode", "graph_device_node"> { + let summary = "`GraphDeviceNode` represents CUDA device node handle for device-side node update"; + let description = "`!cuda.graph_device_node`"; + let assemblyFormat = ""; +} + +def Cuda_GraphExecType : Cuda_Type<"GraphExec", "graph_exec"> { + let summary = "CUDA Graph Execution"; + let description = "`!cuda.graph_exec`"; + let assemblyFormat = ""; +} + +def Cuda_GraphNodeType : Cuda_Type<"GraphNode", "graph_node"> { + let summary = "CUDA Graph Node handle"; + let description = "`!cuda.graph_node`"; + let assemblyFormat = ""; +} + +def Cuda_IntType : Cuda_Type<"Int", "int"> { + let summary = "`cuda.int` is a integer type with a bitwidth matching the underlying ABI integer bitwidth"; + let description = "`!cuda.int`"; + let assemblyFormat = ""; +} + +def Cuda_KernelType : Cuda_Type<"Kernel", "kernel"> { + let summary = "CUDA Kernel handle"; + let description = "`!cuda.kernel`"; + let assemblyFormat = ""; +} + +def Cuda_LaunchConfigType : Cuda_Type<"LaunchConfig", "launch_cfg"> { + let summary = "Representation of CUDA's extensible launch configuration mirroring cuda runtime API"; + let description = "`!cuda.launch_cfg`"; + let parameters = (ins "uint32_t":$maxNumAttrs); + let assemblyFormat = "`<` `max_attrs` `=` $maxNumAttrs `>`"; +} + +def Cuda_LibraryType : Cuda_Type<"Library", "library"> { + let summary = "CUDA Library handle"; + let description = "`!cuda.library`"; + let assemblyFormat = ""; +} + +def Cuda_ModuleType : Cuda_Type<"Module", "module"> { + let summary = "CUDA Module handle"; + let description = "`!cuda.module`"; + let assemblyFormat = ""; +} + +def Cuda_ResultType : Cuda_Type<"Result", "result"> { + let summary = "CUDA Result type"; + let description = "`!cuda.result`"; + let assemblyFormat = ""; +} + +def Cuda_RuntimeDevicePropType : Cuda_Type<"RuntimeDeviceProp", "runtime.device_prop"> { + let summary = "CUDA cudaDeviceProp struct type (value-typed)."; + let description = "`!cuda.runtime.device_prop`"; + let assemblyFormat = ""; +} + +def Cuda_RuntimeDim3Type : Cuda_Type<"RuntimeDim3", "runtime.dim_3"> { + let summary = "CUDA dim3 struct type (value-typed)."; + let description = "`!cuda.runtime.dim_3`"; + let assemblyFormat = ""; +} + +def Cuda_RuntimeExtentType : Cuda_Type<"RuntimeExtent", "runtime.extent"> { + let summary = "CUDA cudaExtent struct type (value-typed)."; + let description = "`!cuda.runtime.extent`"; + let assemblyFormat = ""; +} + +def Cuda_RuntimeKernelNodeParamsType : Cuda_Type<"RuntimeKernelNodeParams", "runtime.kernel_node_params"> { + let summary = "CUDA cudaKernelNodeParams struct type (value-typed)."; + let description = "`!cuda.runtime.kernel_node_params`"; + let assemblyFormat = ""; +} + +def Cuda_RuntimeLaunchAttributeValueClusterSchedulingPolicyPreferenceType : Cuda_Type<"RuntimeLaunchAttributeValueClusterSchedulingPolicyPreference", "runtime.launch_attribute_value.cluster_scheduling_policy_preference"> { + let summary = "`!cuda.runtime.launch_attribute_value.cluster_scheduling_policy_preference`"; + let description = "`!cuda.runtime.launch_attribute_value.cluster_scheduling_policy_preference`"; + let assemblyFormat = ""; +} + +def Cuda_RuntimeLaunchAttributeValueCooperativeType : Cuda_Type<"RuntimeLaunchAttributeValueCooperative", "runtime.launch_attribute_value.cooperative"> { + let summary = "`!cuda.runtime.launch_attribute_value.cooperative`"; + let description = "`!cuda.runtime.launch_attribute_value.cooperative`"; + let assemblyFormat = ""; +} + +def Cuda_RuntimeLaunchAttributeValueMemSyncDomainType : Cuda_Type<"RuntimeLaunchAttributeValueMemSyncDomain", "runtime.launch_attribute_value.mem_sync_domain"> { + let summary = "`!cuda.runtime.launch_attribute_value.mem_sync_domain`"; + let description = "`!cuda.runtime.launch_attribute_value.mem_sync_domain`"; + let assemblyFormat = ""; +} + +def Cuda_RuntimeLaunchAttributeValueNvlinkUtilCentricSchedulingType : Cuda_Type<"RuntimeLaunchAttributeValueNvlinkUtilCentricScheduling", "runtime.launch_attribute_value.nvlink_util_centric_scheduling"> { + let summary = "`!cuda.runtime.launch_attribute_value.nvlink_util_centric_scheduling`"; + let description = "`!cuda.runtime.launch_attribute_value.nvlink_util_centric_scheduling`"; + let assemblyFormat = ""; +} + +def Cuda_RuntimeLaunchAttributeValuePortableClusterSizeModeType : Cuda_Type<"RuntimeLaunchAttributeValuePortableClusterSizeMode", "runtime.launch_attribute_value.portable_cluster_size_mode"> { + let summary = "`!cuda.runtime.launch_attribute_value.portable_cluster_size_mode`"; + let description = "`!cuda.runtime.launch_attribute_value.portable_cluster_size_mode`"; + let assemblyFormat = ""; +} + +def Cuda_RuntimeLaunchAttributeValuePriorityType : Cuda_Type<"RuntimeLaunchAttributeValuePriority", "runtime.launch_attribute_value.priority"> { + let summary = "`!cuda.runtime.launch_attribute_value.priority`"; + let description = "`!cuda.runtime.launch_attribute_value.priority`"; + let assemblyFormat = ""; +} + +def Cuda_RuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowedType : Cuda_Type<"RuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowed", "runtime.launch_attribute_value.programmatic_stream_serialization_allowed"> { + let summary = "`!cuda.runtime.launch_attribute_value.programmatic_stream_serialization_allowed`"; + let description = "`!cuda.runtime.launch_attribute_value.programmatic_stream_serialization_allowed`"; + let assemblyFormat = ""; +} + +def Cuda_RuntimeLaunchAttributeValueSharedMemCarveoutType : Cuda_Type<"RuntimeLaunchAttributeValueSharedMemCarveout", "runtime.launch_attribute_value.shared_mem_carveout"> { + let summary = "`!cuda.runtime.launch_attribute_value.shared_mem_carveout`"; + let description = "`!cuda.runtime.launch_attribute_value.shared_mem_carveout`"; + let assemblyFormat = ""; +} + +def Cuda_RuntimeLaunchAttributeValueSharedMemoryModeType : Cuda_Type<"RuntimeLaunchAttributeValueSharedMemoryMode", "runtime.launch_attribute_value.shared_memory_mode"> { + let summary = "`!cuda.runtime.launch_attribute_value.shared_memory_mode`"; + let description = "`!cuda.runtime.launch_attribute_value.shared_memory_mode`"; + let assemblyFormat = ""; +} + +def Cuda_RuntimeLaunchAttributeValueSyncPolicyType : Cuda_Type<"RuntimeLaunchAttributeValueSyncPolicy", "runtime.launch_attribute_value.sync_policy"> { + let summary = "`!cuda.runtime.launch_attribute_value.sync_policy`"; + let description = "`!cuda.runtime.launch_attribute_value.sync_policy`"; + let assemblyFormat = ""; +} + +def Cuda_RuntimeMemcpy3DParamsType : Cuda_Type<"RuntimeMemcpy3DParams", "runtime.memcpy_3d_params"> { + let summary = "CUDA cudaMemcpy3DParms struct type (value-typed)."; + let description = "`!cuda.runtime.memcpy_3d_params`"; + let assemblyFormat = ""; +} + +def Cuda_RuntimePitchedPtrType : Cuda_Type<"RuntimePitchedPtr", "runtime.pitched_ptr"> { + let summary = "CUDA cudaPitchedPtr struct type (value-typed)."; + let description = "`!cuda.runtime.pitched_ptr`"; + let assemblyFormat = ""; +} + +def Cuda_RuntimePosType : Cuda_Type<"RuntimePos", "runtime.pos"> { + let summary = "CUDA cudaPos struct type (value-typed)."; + let description = "`!cuda.runtime.pos`"; + let assemblyFormat = ""; +} + +def Cuda_RuntimeUuidType : Cuda_Type<"RuntimeUuid", "runtime.uuid"> { + let summary = "CUDA cudaUUID_t struct type (value-typed)."; + let description = "`!cuda.runtime.uuid`"; + let assemblyFormat = ""; +} + +def Cuda_StreamType : Cuda_Type<"Stream", "stream"> { + let summary = "CUDA Stream"; + let description = "`!cuda.stream`"; + let assemblyFormat = ""; +} + +def Cuda_TensorMapType : Cuda_Type<"TensorMap", "tensor_map"> { + let summary = "CUDA Tensor Map type"; + let description = "`!cuda.tensor_map`"; + let assemblyFormat = ""; +} + +#endif // MLIR_DIALECTS_CUDA_CUDATYPES_TD diff --git a/mlir/dialects/cuda/capi.cc b/mlir/dialects/cuda/capi.cc new file mode 100644 index 00000000..4cdff5ee --- /dev/null +++ b/mlir/dialects/cuda/capi.cc @@ -0,0 +1,559 @@ +//===- capi.cc - C API for NVIDIA's cuda dialect --------------------------===// + +#include "capi.h" + +#include "mlir/dialects/cuda/CudaDialect.h" + +#include "mlir/CAPI/IR.h" +#include "mlir/CAPI/Registration.h" +#include "mlir/CAPI/Support.h" + +// Registration + +MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(Cuda, cuda, + mlir::cutlass_compiler::cuda::CudaDialect) + +// Attributes + +namespace cuda = mlir::cutlass_compiler::cuda; + +// Unwraps `handle` into `out`: false when it is of another context or not a +// T, or when it is null and not optional. +template +static bool unwrapAs(mlir::MLIRContext *ctx, H handle, bool optional, T &out) { + auto value = unwrap(handle); + if (!value) + return optional; + if (value.getContext() != ctx) + return false; + out = llvm::dyn_cast(value); + return static_cast(out); +} + +bool mlirAttributeIsACudaAssumeKernel(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCudaAssumeKernelAttrGet(MlirContext context, bool value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::AssumeKernelAttr::get(ctx, value)); +} +bool mlirCudaAssumeKernelAttrGetValue(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getValue(); +} + +bool mlirAttributeIsACudaComputeTarget(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCudaComputeTargetAttrGet(MlirContext context, + uint32_t representation, + uint32_t portability, + intptr_t numArchs, + const uint32_t *archs) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto representationValue = + ::mlir::cutlass_compiler::cuda::symbolizeExecutableRepresentation( + representation); + if (!representationValue) + return {nullptr}; + auto portabilityValue = + ::mlir::cutlass_compiler::cuda::symbolizeArchPortability(portability); + if (!portabilityValue) + return {nullptr}; + llvm::SmallVector<::mlir::cutlass_compiler::cuda::GpuArchitecture> archsValue; + for (intptr_t i = 0; i < numArchs; ++i) { + auto value = + ::mlir::cutlass_compiler::cuda::symbolizeGpuArchitecture(archs[i]); + if (!value) + return {nullptr}; + archsValue.push_back(*value); + } + return wrap(cuda::ComputeTargetAttr::get(ctx, *representationValue, + *portabilityValue, archsValue)); +} +uint32_t mlirCudaComputeTargetAttrGetRepresentation(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getRepresentation()); +} +uint32_t mlirCudaComputeTargetAttrGetPortability(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getPortability()); +} +intptr_t mlirCudaComputeTargetAttrGetNumArchs(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getArchs().size(); +} +uint32_t mlirCudaComputeTargetAttrGetArchs(MlirAttribute attr, intptr_t pos) { + return static_cast( + llvm::cast(unwrap(attr)).getArchs()[pos]); +} + +bool mlirAttributeIsACudaDevMaxSharedMemoryOptin(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCudaDevMaxSharedMemoryOptinAttrGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::DevMaxSharedMemoryOptinAttr::get(ctx)); +} + +bool mlirAttributeIsACudaDeviceAttributes(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCudaDeviceAttributesAttrGet(MlirContext context, + MlirAttribute values) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::DictionaryAttr valuesValue; + if (!unwrapAs(ctx, values, false, valuesValue)) + return {nullptr}; + return wrap(cuda::DeviceAttributesAttr::get(ctx, valuesValue)); +} +MlirAttribute mlirCudaDeviceAttributesAttrGetValues(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getValues())); +} + +bool mlirAttributeIsACudaExecutable(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCudaExecutableAttrGet(MlirContext context, uint32_t format) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto formatValue = + ::mlir::cutlass_compiler::cuda::symbolizeExecutableFormat(format); + if (!formatValue) + return {nullptr}; + return wrap(cuda::ExecutableAttr::get(ctx, *formatValue)); +} +uint32_t mlirCudaExecutableAttrGetFormat(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getFormat()); +} + +bool mlirAttributeIsACudaFuncAttributes(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCudaFuncAttributesAttrGet(MlirContext context, + MlirAttribute values) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::DictionaryAttr valuesValue; + if (!unwrapAs(ctx, values, false, valuesValue)) + return {nullptr}; + return wrap(cuda::FuncAttributesAttr::get(ctx, valuesValue)); +} +MlirAttribute mlirCudaFuncAttributesAttrGetValues(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getValues())); +} + +// Types + +bool mlirTypeIsACudaArchSet(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaArchSetTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::ArchSetType::get(ctx)); +} + +bool mlirTypeIsACudaEvent(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaEventTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::EventType::get(ctx)); +} + +bool mlirTypeIsACudaGraphDeviceNode(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaGraphDeviceNodeTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::GraphDeviceNodeType::get(ctx)); +} + +bool mlirTypeIsACudaGraphExec(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaGraphExecTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::GraphExecType::get(ctx)); +} + +bool mlirTypeIsACudaGraphNode(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaGraphNodeTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::GraphNodeType::get(ctx)); +} + +bool mlirTypeIsACudaGraph(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaGraphTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::GraphType::get(ctx)); +} + +bool mlirTypeIsACudaInt(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaIntTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::IntType::get(ctx)); +} + +bool mlirTypeIsACudaKernel(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaKernelTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::KernelType::get(ctx)); +} + +bool mlirTypeIsACudaLaunchConfig(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaLaunchConfigTypeGet(MlirContext context, + uint32_t maxNumAttrs) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::LaunchConfigType::get(ctx, maxNumAttrs)); +} +uint32_t mlirCudaLaunchConfigTypeGetMaxNumAttrs(MlirType type) { + return llvm::cast(unwrap(type)).getMaxNumAttrs(); +} + +bool mlirTypeIsACudaLibrary(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaLibraryTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::LibraryType::get(ctx)); +} + +bool mlirTypeIsACudaModule(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaModuleTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::ModuleType::get(ctx)); +} + +bool mlirTypeIsACudaResult(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaResultTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::ResultType::get(ctx)); +} + +bool mlirTypeIsACudaRuntimeDeviceProp(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaRuntimeDevicePropTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::RuntimeDevicePropType::get(ctx)); +} + +bool mlirTypeIsACudaRuntimeDim3(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaRuntimeDim3TypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::RuntimeDim3Type::get(ctx)); +} + +bool mlirTypeIsACudaRuntimeExtent(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaRuntimeExtentTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::RuntimeExtentType::get(ctx)); +} + +bool mlirTypeIsACudaRuntimeKernelNodeParams(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaRuntimeKernelNodeParamsTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::RuntimeKernelNodeParamsType::get(ctx)); +} + +bool mlirTypeIsACudaRuntimeLaunchAttributeValueClusterSchedulingPolicyPreference( + MlirType type) { + return llvm::isa_and_nonnull< + cuda::RuntimeLaunchAttributeValueClusterSchedulingPolicyPreferenceType>( + unwrap(type)); +} +MlirType +mlirCudaRuntimeLaunchAttributeValueClusterSchedulingPolicyPreferenceTypeGet( + MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap( + cuda::RuntimeLaunchAttributeValueClusterSchedulingPolicyPreferenceType:: + get(ctx)); +} + +bool mlirTypeIsACudaRuntimeLaunchAttributeValueCooperative(MlirType type) { + return llvm::isa_and_nonnull< + cuda::RuntimeLaunchAttributeValueCooperativeType>(unwrap(type)); +} +MlirType +mlirCudaRuntimeLaunchAttributeValueCooperativeTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::RuntimeLaunchAttributeValueCooperativeType::get(ctx)); +} + +bool mlirTypeIsACudaRuntimeLaunchAttributeValueMemSyncDomain(MlirType type) { + return llvm::isa_and_nonnull< + cuda::RuntimeLaunchAttributeValueMemSyncDomainType>(unwrap(type)); +} +MlirType +mlirCudaRuntimeLaunchAttributeValueMemSyncDomainTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::RuntimeLaunchAttributeValueMemSyncDomainType::get(ctx)); +} + +bool mlirTypeIsACudaRuntimeLaunchAttributeValueNvlinkUtilCentricScheduling( + MlirType type) { + return llvm::isa_and_nonnull< + cuda::RuntimeLaunchAttributeValueNvlinkUtilCentricSchedulingType>( + unwrap(type)); +} +MlirType mlirCudaRuntimeLaunchAttributeValueNvlinkUtilCentricSchedulingTypeGet( + MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap( + cuda::RuntimeLaunchAttributeValueNvlinkUtilCentricSchedulingType::get( + ctx)); +} + +bool mlirTypeIsACudaRuntimeLaunchAttributeValuePortableClusterSizeMode( + MlirType type) { + return llvm::isa_and_nonnull< + cuda::RuntimeLaunchAttributeValuePortableClusterSizeModeType>( + unwrap(type)); +} +MlirType mlirCudaRuntimeLaunchAttributeValuePortableClusterSizeModeTypeGet( + MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap( + cuda::RuntimeLaunchAttributeValuePortableClusterSizeModeType::get(ctx)); +} + +bool mlirTypeIsACudaRuntimeLaunchAttributeValuePriority(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType +mlirCudaRuntimeLaunchAttributeValuePriorityTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::RuntimeLaunchAttributeValuePriorityType::get(ctx)); +} + +bool mlirTypeIsACudaRuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowed( + MlirType type) { + return llvm::isa_and_nonnull< + cuda:: + RuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowedType>( + unwrap(type)); +} +MlirType +mlirCudaRuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowedTypeGet( + MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap( + cuda:: + RuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowedType:: + get(ctx)); +} + +bool mlirTypeIsACudaRuntimeLaunchAttributeValueSharedMemCarveout( + MlirType type) { + return llvm::isa_and_nonnull< + cuda::RuntimeLaunchAttributeValueSharedMemCarveoutType>(unwrap(type)); +} +MlirType mlirCudaRuntimeLaunchAttributeValueSharedMemCarveoutTypeGet( + MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::RuntimeLaunchAttributeValueSharedMemCarveoutType::get(ctx)); +} + +bool mlirTypeIsACudaRuntimeLaunchAttributeValueSharedMemoryMode(MlirType type) { + return llvm::isa_and_nonnull< + cuda::RuntimeLaunchAttributeValueSharedMemoryModeType>(unwrap(type)); +} +MlirType mlirCudaRuntimeLaunchAttributeValueSharedMemoryModeTypeGet( + MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::RuntimeLaunchAttributeValueSharedMemoryModeType::get(ctx)); +} + +bool mlirTypeIsACudaRuntimeLaunchAttributeValueSyncPolicy(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType +mlirCudaRuntimeLaunchAttributeValueSyncPolicyTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::RuntimeLaunchAttributeValueSyncPolicyType::get(ctx)); +} + +bool mlirTypeIsACudaRuntimeMemcpy3DParams(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaRuntimeMemcpy3DParamsTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::RuntimeMemcpy3DParamsType::get(ctx)); +} + +bool mlirTypeIsACudaRuntimePitchedPtr(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaRuntimePitchedPtrTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::RuntimePitchedPtrType::get(ctx)); +} + +bool mlirTypeIsACudaRuntimePos(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaRuntimePosTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::RuntimePosType::get(ctx)); +} + +bool mlirTypeIsACudaRuntimeUuid(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaRuntimeUuidTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::RuntimeUuidType::get(ctx)); +} + +bool mlirTypeIsACudaStream(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaStreamTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::StreamType::get(ctx)); +} + +bool mlirTypeIsACudaTensorMap(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCudaTensorMapTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cuda::TensorMapType::get(ctx)); +} diff --git a/mlir/dialects/cuda/capi.h b/mlir/dialects/cuda/capi.h new file mode 100644 index 00000000..126a87d8 --- /dev/null +++ b/mlir/dialects/cuda/capi.h @@ -0,0 +1,253 @@ +//===- capi.h - C API for NVIDIA's cuda dialect -------------------*- C -*-===// + +#ifndef MLIR_DIALECTS_CUDA_CAPI_H +#define MLIR_DIALECTS_CUDA_CAPI_H + +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Registration + +MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(Cuda, cuda); + +// Attributes + +// Constructors return null for invalid parameters or handles of another +// context; optional parameters take null handles. Enums are their integer +// values (the dialect's *Enums.td). Getters require a value accepted by the +// matching IsA. + +// `#cuda.assume_kernel_attr`: Whether `cuda.launch` may assume its callee is a +// kernel +MLIR_CAPI_EXPORTED bool mlirAttributeIsACudaAssumeKernel(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCudaAssumeKernelAttrGet(MlirContext context, bool value); +MLIR_CAPI_EXPORTED bool mlirCudaAssumeKernelAttrGetValue(MlirAttribute attr); + +// `#cuda.compute_target`: Representation, portability and architectures of a +// compiled executable +MLIR_CAPI_EXPORTED bool mlirAttributeIsACudaComputeTarget(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCudaComputeTargetAttrGet( + MlirContext context, uint32_t representation, uint32_t portability, + intptr_t numArchs, const uint32_t *archs); +MLIR_CAPI_EXPORTED uint32_t +mlirCudaComputeTargetAttrGetRepresentation(MlirAttribute attr); +MLIR_CAPI_EXPORTED uint32_t +mlirCudaComputeTargetAttrGetPortability(MlirAttribute attr); +MLIR_CAPI_EXPORTED intptr_t +mlirCudaComputeTargetAttrGetNumArchs(MlirAttribute attr); +MLIR_CAPI_EXPORTED uint32_t +mlirCudaComputeTargetAttrGetArchs(MlirAttribute attr, intptr_t pos); + +// `#cuda.dev_max_shared_memory_optin`: The device's opt-in maximum of shared +// memory per block +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACudaDevMaxSharedMemoryOptin(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCudaDevMaxSharedMemoryOptinAttrGet(MlirContext context); + +// `#cuda.device_attributes`: CUDA device attributes +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACudaDeviceAttributes(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCudaDeviceAttributesAttrGet(MlirContext context, MlirAttribute values); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCudaDeviceAttributesAttrGetValues(MlirAttribute attr); + +// `#cuda.executable`: Format of a compiled executable +MLIR_CAPI_EXPORTED bool mlirAttributeIsACudaExecutable(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCudaExecutableAttrGet(MlirContext context, + uint32_t format); +MLIR_CAPI_EXPORTED uint32_t mlirCudaExecutableAttrGetFormat(MlirAttribute attr); + +// `#cuda.func_attributes`: CUDA function attributes of a kernel +MLIR_CAPI_EXPORTED bool mlirAttributeIsACudaFuncAttributes(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCudaFuncAttributesAttrGet(MlirContext context, MlirAttribute values); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCudaFuncAttributesAttrGetValues(MlirAttribute attr); + +// Types + +// Constructors return null for invalid parameters or handles of another +// context; optional parameters take null handles. Enums are their integer +// values (the dialect's *Enums.td). Getters require a value accepted by the +// matching IsA. + +// `!cuda.arch_set`: `!cuda.arch_set` +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaArchSet(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaArchSetTypeGet(MlirContext context); + +// `!cuda.event`: `Event` represents CUDA event +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaEvent(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaEventTypeGet(MlirContext context); + +// `!cuda.graph_device_node`: `GraphDeviceNode` represents CUDA device node +// handle for device-side node update +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaGraphDeviceNode(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaGraphDeviceNodeTypeGet(MlirContext context); + +// `!cuda.graph_exec`: CUDA Graph Execution +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaGraphExec(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaGraphExecTypeGet(MlirContext context); + +// `!cuda.graph_node`: CUDA Graph Node handle +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaGraphNode(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaGraphNodeTypeGet(MlirContext context); + +// `!cuda.graph`: CUDA Graph +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaGraph(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaGraphTypeGet(MlirContext context); + +// `!cuda.int`: `cuda.int` is a integer type with a bitwidth matching the +// underlying ABI integer bitwidth +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaInt(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaIntTypeGet(MlirContext context); + +// `!cuda.kernel`: CUDA Kernel handle +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaKernel(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaKernelTypeGet(MlirContext context); + +// `!cuda.launch_cfg`: Representation of CUDA's extensible launch configuration +// mirroring cuda runtime API +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaLaunchConfig(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaLaunchConfigTypeGet(MlirContext context, + uint32_t maxNumAttrs); +MLIR_CAPI_EXPORTED uint32_t +mlirCudaLaunchConfigTypeGetMaxNumAttrs(MlirType type); + +// `!cuda.library`: CUDA Library handle +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaLibrary(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaLibraryTypeGet(MlirContext context); + +// `!cuda.module`: CUDA Module handle +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaModule(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaModuleTypeGet(MlirContext context); + +// `!cuda.result`: CUDA Result type +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaResult(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaResultTypeGet(MlirContext context); + +// `!cuda.runtime.device_prop`: CUDA cudaDeviceProp struct type (value-typed). +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimeDeviceProp(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCudaRuntimeDevicePropTypeGet(MlirContext context); + +// `!cuda.runtime.dim_3`: CUDA dim3 struct type (value-typed). +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimeDim3(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaRuntimeDim3TypeGet(MlirContext context); + +// `!cuda.runtime.extent`: CUDA cudaExtent struct type (value-typed). +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimeExtent(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaRuntimeExtentTypeGet(MlirContext context); + +// `!cuda.runtime.kernel_node_params`: CUDA cudaKernelNodeParams struct type +// (value-typed). +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimeKernelNodeParams(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCudaRuntimeKernelNodeParamsTypeGet(MlirContext context); + +// `!cuda.runtime.launch_attribute_value.cluster_scheduling_policy_preference` +MLIR_CAPI_EXPORTED bool +mlirTypeIsACudaRuntimeLaunchAttributeValueClusterSchedulingPolicyPreference( + MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCudaRuntimeLaunchAttributeValueClusterSchedulingPolicyPreferenceTypeGet( + MlirContext context); + +// `!cuda.runtime.launch_attribute_value.cooperative` +MLIR_CAPI_EXPORTED bool +mlirTypeIsACudaRuntimeLaunchAttributeValueCooperative(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCudaRuntimeLaunchAttributeValueCooperativeTypeGet(MlirContext context); + +// `!cuda.runtime.launch_attribute_value.mem_sync_domain` +MLIR_CAPI_EXPORTED bool +mlirTypeIsACudaRuntimeLaunchAttributeValueMemSyncDomain(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCudaRuntimeLaunchAttributeValueMemSyncDomainTypeGet(MlirContext context); + +// `!cuda.runtime.launch_attribute_value.nvlink_util_centric_scheduling` +MLIR_CAPI_EXPORTED bool +mlirTypeIsACudaRuntimeLaunchAttributeValueNvlinkUtilCentricScheduling( + MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCudaRuntimeLaunchAttributeValueNvlinkUtilCentricSchedulingTypeGet( + MlirContext context); + +// `!cuda.runtime.launch_attribute_value.portable_cluster_size_mode` +MLIR_CAPI_EXPORTED bool +mlirTypeIsACudaRuntimeLaunchAttributeValuePortableClusterSizeMode( + MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCudaRuntimeLaunchAttributeValuePortableClusterSizeModeTypeGet( + MlirContext context); + +// `!cuda.runtime.launch_attribute_value.priority` +MLIR_CAPI_EXPORTED bool +mlirTypeIsACudaRuntimeLaunchAttributeValuePriority(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCudaRuntimeLaunchAttributeValuePriorityTypeGet(MlirContext context); + +// `!cuda.runtime.launch_attribute_value.programmatic_stream_serialization_allowed` +MLIR_CAPI_EXPORTED bool +mlirTypeIsACudaRuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowed( + MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCudaRuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowedTypeGet( + MlirContext context); + +// `!cuda.runtime.launch_attribute_value.shared_mem_carveout` +MLIR_CAPI_EXPORTED bool +mlirTypeIsACudaRuntimeLaunchAttributeValueSharedMemCarveout(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCudaRuntimeLaunchAttributeValueSharedMemCarveoutTypeGet( + MlirContext context); + +// `!cuda.runtime.launch_attribute_value.shared_memory_mode` +MLIR_CAPI_EXPORTED bool +mlirTypeIsACudaRuntimeLaunchAttributeValueSharedMemoryMode(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCudaRuntimeLaunchAttributeValueSharedMemoryModeTypeGet(MlirContext context); + +// `!cuda.runtime.launch_attribute_value.sync_policy` +MLIR_CAPI_EXPORTED bool +mlirTypeIsACudaRuntimeLaunchAttributeValueSyncPolicy(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCudaRuntimeLaunchAttributeValueSyncPolicyTypeGet(MlirContext context); + +// `!cuda.runtime.memcpy_3d_params`: CUDA cudaMemcpy3DParms struct type +// (value-typed). +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimeMemcpy3DParams(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCudaRuntimeMemcpy3DParamsTypeGet(MlirContext context); + +// `!cuda.runtime.pitched_ptr`: CUDA cudaPitchedPtr struct type (value-typed). +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimePitchedPtr(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCudaRuntimePitchedPtrTypeGet(MlirContext context); + +// `!cuda.runtime.pos`: CUDA cudaPos struct type (value-typed). +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimePos(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaRuntimePosTypeGet(MlirContext context); + +// `!cuda.runtime.uuid`: CUDA cudaUUID_t struct type (value-typed). +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimeUuid(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaRuntimeUuidTypeGet(MlirContext context); + +// `!cuda.stream`: CUDA Stream +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaStream(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaStreamTypeGet(MlirContext context); + +// `!cuda.tensor_map`: CUDA Tensor Map type +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaTensorMap(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaTensorMapTypeGet(MlirContext context); + +#ifdef __cplusplus +} +#endif + +#endif // MLIR_DIALECTS_CUDA_CAPI_H diff --git a/mlir/dialects/cuda/cuda.zig b/mlir/dialects/cuda/cuda.zig new file mode 100644 index 00000000..2c74c920 --- /dev/null +++ b/mlir/dialects/cuda/cuda.zig @@ -0,0 +1,104 @@ +//! Builders for the CUDA host-program operations CuTe kernels emit, over the +//! bindings of NVIDIA's `cuda` dialect (`mlir/dialects/cuda`, `ops`), +//! which the CuTe-DSL compiler registers. Each builder fixes the attributes the +//! kernels do not choose. + +const mlir = @import("mlir"); + +pub const ops = @import("ops.zig"); + +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("cuda"); +} + +pub fn streamType(ctx: *mlir.Context) *const mlir.Type { + return (ops.StreamType.get(ctx, .{}) catch unreachable).type_(); +} + +pub fn resultType(ctx: *mlir.Context) *const mlir.Type { + return (ops.ResultType.get(ctx, .{}) catch unreachable).type_(); +} + +pub fn launchConfigType(ctx: *mlir.Context, max_attrs: u32) *const mlir.Type { + return (ops.LaunchConfigType.get(ctx, .{ .maxNumAttrs = max_attrs }) catch unreachable).type_(); +} + +/// `#cuda.assume_kernel_attr`, whether `cuda.launch_ex` may assume its +/// callee is a kernel. +pub fn assumeKernelAttr(ctx: *mlir.Context, value: bool) *const mlir.Attribute { + return (ops.AssumeKernelAttr.get(ctx, .{ .value = value }) catch unreachable).attribute(); +} + +pub const KernelArgs = struct { + name: []const u8, + function_type: *const mlir.Type, + body: *mlir.Block, + arg_attrs: ?*const mlir.Attribute = null, + extra_attributes: []const mlir.NamedAttribute = &.{}, + location: *const mlir.Location, +}; + +/// `cuda.kernel`; `body` is filled afterwards. +pub fn kernel(ctx: *mlir.Context, args: KernelArgs) *mlir.Operation { + const op = ops.kernel(ctx, .string(ctx, args.name), .typeAttr(args.function_type), .{args.body}, .{ .arg_attrs = args.arg_attrs }, args.location); + for (args.extra_attributes) |attr| op.setAttribute(attr); + return op; +} + +pub fn return_(ctx: *mlir.Context, values: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return ops.return_(ctx, values, location); +} + +pub fn cast(ctx: *mlir.Context, dst: *const mlir.Type, src: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return ops.cast(ctx, src, dst, location); +} + +pub const LaunchConfigCreateArgs = struct { + max_attrs: u32, + block: [3]*const mlir.Value, + dynamic_smem: *const mlir.Value, + grid: [3]*const mlir.Value, + stream: *const mlir.Value, + location: *const mlir.Location, +}; + +pub fn launch_cfg_create(ctx: *mlir.Context, args: LaunchConfigCreateArgs) *mlir.Operation { + return ops.launch_cfg_create( + ctx, + args.block[0], + args.block[1], + args.block[2], + args.dynamic_smem, + args.grid[0], + args.grid[1], + args.grid[2], + args.stream, + launchConfigType(ctx, args.max_attrs), + .int(ctx, .i32, args.max_attrs), + args.location, + ); +} + +pub fn launch_cfg_cluster_dim(ctx: *mlir.Context, config: *const mlir.Value, dims: [3]*const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return ops.launch_cfg_cluster_dim(ctx, config, dims[0], dims[1], dims[2], location); +} + +pub fn launch_cfg_cooperative(ctx: *mlir.Context, config: *const mlir.Value, enabled: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return ops.launch_cfg_cooperative(ctx, config, enabled, location); +} + +pub fn launch_cfg_programmatic_stream_serialization_allowed(ctx: *mlir.Context, config: *const mlir.Value, enabled: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return ops.launch_cfg_programmatic_stream_serialization_allowed(ctx, config, enabled, location); +} + +pub const LaunchExArgs = struct { + config: *const mlir.Value, + inputs: []const *const mlir.Value, + callee: *const mlir.Attribute, + assume_kernel_attr: ?*const mlir.Attribute = null, + location: *const mlir.Location, +}; + +pub fn launch_ex(ctx: *mlir.Context, args: LaunchExArgs) *mlir.Operation { + return ops.launch_ex(ctx, args.config, args.inputs, resultType(ctx), args.callee, .{ .assume_kernel_attr = args.assume_kernel_attr }, args.location); +} diff --git a/mlir/dialects/cuda/ops.zig b/mlir/dialects/cuda/ops.zig new file mode 100644 index 00000000..baeb0ded --- /dev/null +++ b/mlir/dialects/cuda/ops.zig @@ -0,0 +1,3346 @@ +//! Zig bindings of NVIDIA's CUDA host dialect of the CuTe DSL (nvidia-cutlass-dsl 4.8.0), generated +//! from the dialect's .td files; do not edit. + +const c = @import("c"); +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +/// The C dialect handle, `mlirGetDialectHandle__cuda__`. +pub const dialect_handle = "cuda"; + +// Enums, generated from the dialect's .td files; values are the compiler's. + +/// Enumerated type describing portability. +pub const ArchPortability = enum(u32) { + portable = 0, + conditional = 1, + family = 2, +}; + +/// Enumerated type describing CUDA device attributes can be set through Cuda Dialect. +pub const CUDeviceAttr = enum(u32) { + maxThreadsPerBlock = 1, + maxBlockDimX = 2, + maxBlockDimY = 3, + maxBlockDimZ = 4, + maxGridDimX = 5, + maxGridDimY = 6, + maxGridDimZ = 7, + maxSharedMemoryPerBlock = 8, + totalConstantMemory = 9, + warpSize = 10, + maxPitch = 11, + maxRegistersPerBlock = 12, + clockRate = 13, + textureAlignment = 14, + gpuOverlap = 15, + multiProcessorCount = 16, + kernelExecTimeout = 17, + integrated = 18, + canMapHostMemory = 19, + computeMode = 20, + maxTexture1DWidth = 21, + maxTexture2DWidth = 22, + maxTexture2DHeight = 23, + maxTexture3DWidth = 24, + maxTexture3DHeight = 25, + maxTexture3DDepth = 26, + maxTexture2DLayeredWidth = 27, + maxTexture2DLayeredHeight = 28, + maxTexture2DLayeredLayers = 29, + surfaceAlignment = 30, + concurrentKernels = 31, + eccEnabled = 32, + pciBusId = 33, + pciDeviceId = 34, + tccDriver = 35, + memoryClockRate = 36, + globalMemoryBusWidth = 37, + l2CacheSize = 38, + maxThreadsPerMultiProcessor = 39, + asyncEngineCount = 40, + unifiedAddressing = 41, + maxTexture1DLayeredWidth = 42, + maxTexture1DLayeredLayers = 43, + maxTexture2DGatherWidth = 45, + maxTexture2DGatherHeight = 46, + maxTexture3DWidthAlt = 47, + maxTexture3DHeightAlt = 48, + maxTexture3DDepthAlt = 49, + pciDomainId = 50, + texturePitchAlignment = 51, + maxTextureCubemapWidth = 52, + maxTextureCubemapLayeredWidth = 53, + maxTextureCubemapLayeredLayers = 54, + maxSurface1DWidth = 55, + maxSurface2DWidth = 56, + maxSurface2DHeight = 57, + maxSurface3DWidth = 58, + maxSurface3DHeight = 59, + maxSurface3DDepth = 60, + maxSurface1DLayeredWidth = 61, + maxSurface1DLayeredLayers = 62, + maxSurface2DLayeredWidth = 63, + maxSurface2DLayeredHeight = 64, + maxSurface2DLayeredLayers = 65, + maxSurfaceCubemapWidth = 66, + maxSurfaceCubemapLayeredWidth = 67, + maxSurfaceCubemapLayeredLayers = 68, + maxTexture1DLinearWidth = 69, + maxTexture2DLinearWidth = 70, + maxTexture2DLinearHeight = 71, + maxTexture2DLinearPitch = 72, + maxTexture2DMipmappedWidth = 73, + maxTexture2DMipmappedHeight = 74, + computeCapabilityMajor = 75, + computeCapabilityMinor = 76, + maxTexture1DMipmappedWidth = 77, + streamPrioritiesSupported = 78, + globalL1CacheSupported = 79, + localL1CacheSupported = 80, + maxSharedMemoryPerMultiprocessor = 81, + maxRegistersPerMultiprocessor = 82, + managedMemory = 83, + isMultiGpuBoard = 84, + multiGpuBoardGroupID = 85, + hostNativeAtomicSupported = 86, + singleToDoublePrecisionPerfRatio = 87, + pageableMemoryAccess = 88, + concurrentManagedAccess = 89, + computePreemptionSupported = 90, + canUseHostPointerForRegisteredMem = 91, + reserved92 = 92, + reserved93 = 93, + reserved94 = 94, + cooperativeLaunch = 95, + reserved96 = 96, + maxSharedMemoryPerBlockOptin = 97, + canFlushRemoteWrites = 98, + hostRegisterSupported = 99, + pageableMemoryAccessUsesHostPageTables = 100, + directManagedMemAccessFromHost = 101, + maxBlocksPerMultiprocessor = 106, + maxPersistingL2CacheSize = 108, + maxAccessPolicyWindowSize = 109, + reservedSharedMemoryPerBlock = 111, + sparseCudaArraySupported = 112, + hostRegisterReadOnlySupported = 113, + timelineSemaphoreInteropSupported = 114, + memoryPoolsSupported = 115, + gpuDirectRdmaSupported = 116, + gpuDirectRdmaFlushWritesOptions = 117, + gpuDirectRdmaWritesOrdering = 118, + memoryPoolSupportedHandleTypes = 119, + clusterLaunch = 120, + deferredMappingCudaArraySupported = 121, + reserved122 = 122, + reserved123 = 123, + reserved124 = 124, + ipcEventSupport = 125, + memSyncDomainCount = 126, + reserved127 = 127, + reserved128 = 128, + reserved129 = 129, + numaConfig = 130, + numaId = 131, + reserved132 = 132, + mpsEnabled = 133, + hostNumaId = 134, + d3d12CigSupported = 135, + vulkanCigSupported = 138, + gpuPciDeviceId = 139, + gpuPciSubsystemId = 140, + reserved141 = 141, + hostNumaMemoryPoolsSupported = 142, + hostNumaMultinodeIpcSupported = 143, + hostMemoryPoolsSupported = 144, + reserved145 = 145, + onlyPartialHostNativeAtomicSupported = 147, + cudaDevAttrOversizedSharedMemoryPerBlock = 150, +}; + +/// Enumerated type describing CUDA function attributes can be set through Cuda Dialect. +pub const CUFunctionAttribute = enum(u32) { + max_threads_per_block = 0, + shared_size_bytes = 1, + const_size_bytes = 2, + local_size_bytes = 3, + num_regs = 4, + ptx_version = 5, + binary_version = 6, + cache_mode_ca = 7, + max_dynamic_shared_size_bytes = 8, + preferred_shared_memory_carveout = 9, + cluster_size_must_be_set = 10, + required_cluster_width = 11, + required_cluster_height = 12, + required_cluster_depth = 13, + non_portable_cluster_size_allowed = 14, + cluster_scheduling_policy_preference = 15, +}; + +/// Enumerated of the host library ABI. +pub const CudaABI = enum(u32) { + LP64 = 0, + ILP64 = 1, + LLP64 = 2, +}; + +/// Specifies performance hint with cudaAccessPolicyWindow for hitProp and missProp members. +pub const CudaAccessProperty = enum(u32) { + cudaAccessPropertyNormal = 0, + cudaAccessPropertyStreaming = 1, + cudaAccessPropertyPersisting = 2, +}; + +/// Enumerated type describing CUDA cluster scheduling policy. +pub const CudaClusterSchedulingPolicy = enum(u32) { + cudaClusterSchedulingPolicyDefault = 0, + cudaClusterSchedulingPolicySpread = 1, + cudaClusterSchedulingPolicyLoadBalancing = 2, +}; + +/// Enumerated type describing CUDA function attributes can be set through Cuda Dialect. +pub const CudaFuncAttribute = enum(u32) { + cudaFuncAttributeSharedMemoryMode = 16, +}; + +/// Enumerated type mirroring CudaLaunchAttributeID from CUDA Runtime API. +pub const CudaLaunchAttributeID = enum(u32) { + cudaLaunchAttributeIgnore = 0, + cudaLaunchAttributeAccessPolicyWindow = 1, + cudaLaunchAttributeCooperative = 2, + cudaLaunchAttributeSynchronizationPolicy = 3, + cudaLaunchAttributeClusterDimension = 4, + cudaLaunchAttributeClusterSchedulingPolicyPreference = 5, + cudaLaunchAttributeProgrammaticStreamSerialization = 6, + cudaLaunchAttributeProgrammaticEvent = 7, + cudaLaunchAttributePriority = 8, + cudaLaunchAttributeMemSyncDomainMap = 9, + cudaLaunchAttributeMemSyncDomain = 10, + cudaLaunchAttributePreferredClusterDimension = 11, + cudaLaunchAttributeLaunchCompletionEvent = 12, + cudaLaunchAttributeDeviceUpdatableKernelNode = 13, + cudaLaunchAttributePreferredSharedMemoryCarveout = 14, + cudaLaunchAttributeNvlinkUtilCentricScheduling = 16, +}; + +/// Enumerated type describing CUDA memory sync domain. +pub const CudaLaunchMemSyncDomain = enum(u32) { + cudaLaunchMemSyncDomainDefault = 0, + cudaLaunchMemSyncDomainRemote = 1, +}; + +/// Enumerated type describing CUDA shared memory modes. +pub const CudaSharedMemoryMode = enum(u32) { + cudaSharedMemoryModeDefault = 0, + cudaSharedMemoryModeRequirePortable = 1, + cudaSharedMemoryModeAllowNonPortable = 2, + cudaSharedMemoryModeAllowOversizedSharedMemory = 3, + cudaSharedMemoryModePreferOversizedSharedMemory = 4, +}; + +/// Enumerated type describing CUDA synchronization policy. +pub const CudaSynchronizationPolicy = enum(u32) { + cudaSyncPolicyAuto = 0, + cudaSyncPolicySpin = 1, + cudaSyncPolicyYield = 2, + cudaSyncPolicyBlockingSync = 3, +}; + +/// Enumerated type describing the format of an executable. +pub const ExecutableFormat = enum(u32) { + text = 0, + elf = 1, + bitcode = 2, + fatbin = 3, +}; + +/// Enumerated type describing the representation of an executable. +pub const ExecutableRepresentation = enum(u32) { + sass = 0, + ptx = 2, + nvvm = 3, + ltoir = 4, + mlir = 5, + llvm = 6, + hostobj = 7, +}; + +/// Enumerated type denoting a GPU Architecture. +pub const GpuArchitecture = enum(u32) { + sm_80 = 80, + sm_86 = 86, + sm_87 = 87, + sm_88 = 88, + sm_89 = 89, + sm_90 = 90, + sm_100 = 100, + sm_101 = 101, + sm_103 = 103, + sm_107 = 107, + sm_110 = 110, + sm_120 = 120, + sm_121 = 121, +}; + +/// Enumerated type describing memcpy directions. +pub const MemcpyKind = enum(u32) { + HostToHost = 0, + HostToDevice = 1, + DeviceToHost = 2, + DeviceToDevice = 3, + Default = 4, +}; + +/// Enumerated type describing CUDA stream capture modes. +pub const StreamCaptureModeEnum = enum(u32) { + Global = 0, + ThreadLocal = 1, + Relaxed = 2, +}; + +// Types and attributes, generated from the dialect's .td files. + +/// `#cuda.assume_kernel_attr`: Whether `cuda.launch` may assume its callee is a kernel +pub const AssumeKernelAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACudaAssumeKernel; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "assume_kernel_attr"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: bool, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCudaAssumeKernelAttrGet(ctx.ptr(), args.value); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) bool { + return c.mlirCudaAssumeKernelAttrGetValue(self.ptr()); + } +}; + +/// `#cuda.compute_target`: Representation, portability and architectures of a compiled executable +pub const ComputeTargetAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACudaComputeTarget; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "compute_target"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + representation: ExecutableRepresentation, + portability: ArchPortability, + archs: []const GpuArchitecture, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCudaComputeTargetAttrGet(ctx.ptr(), @backingInt(args.representation), @backingInt(args.portability), @intCast(args.archs.len), @ptrCast(args.archs.ptr)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn representation(self: *const Self) ExecutableRepresentation { + return @fromBackingInt(@intCast(c.mlirCudaComputeTargetAttrGetRepresentation(self.ptr()))); + } + pub fn portability(self: *const Self) ArchPortability { + return @fromBackingInt(@intCast(c.mlirCudaComputeTargetAttrGetPortability(self.ptr()))); + } + pub fn numArchs(self: *const Self) usize { + return @intCast(c.mlirCudaComputeTargetAttrGetNumArchs(self.ptr())); + } + pub fn arch(self: *const Self, pos: usize) GpuArchitecture { + return @fromBackingInt(@intCast(c.mlirCudaComputeTargetAttrGetArchs(self.ptr(), @intCast(pos)))); + } +}; + +/// `#cuda.dev_max_shared_memory_optin`: The device's opt-in maximum of shared memory per block +pub const DevMaxSharedMemoryOptinAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACudaDevMaxSharedMemoryOptin; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "dev_max_shared_memory_optin"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaDevMaxSharedMemoryOptinAttrGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `#cuda.device_attributes`: CUDA device attributes +pub const DeviceAttributesAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACudaDeviceAttributes; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "device_attributes"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + values: *const mlir.Attribute, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCudaDeviceAttributesAttrGet(ctx.ptr(), args.values.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn values(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCudaDeviceAttributesAttrGetValues(self.ptr()).ptr.?); + } +}; + +/// `#cuda.executable`: Format of a compiled executable +pub const ExecutableAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACudaExecutable; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "executable"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + format: ExecutableFormat, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCudaExecutableAttrGet(ctx.ptr(), @backingInt(args.format)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn format_(self: *const Self) ExecutableFormat { + return @fromBackingInt(@intCast(c.mlirCudaExecutableAttrGetFormat(self.ptr()))); + } +}; + +/// `#cuda.func_attributes`: CUDA function attributes of a kernel +pub const FuncAttributesAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACudaFuncAttributes; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "func_attributes"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + values: *const mlir.Attribute, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCudaFuncAttributesAttrGet(ctx.ptr(), args.values.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn values(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCudaFuncAttributesAttrGetValues(self.ptr()).ptr.?); + } +}; + +/// `!cuda.arch_set`: `!cuda.arch_set` +pub const ArchSetType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaArchSet; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "arch_set"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaArchSetTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.event`: `Event` represents CUDA event +pub const EventType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaEvent; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "event"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaEventTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.graph_device_node`: `GraphDeviceNode` represents CUDA device node handle for device-side node update +pub const GraphDeviceNodeType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaGraphDeviceNode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "graph_device_node"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaGraphDeviceNodeTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.graph_exec`: CUDA Graph Execution +pub const GraphExecType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaGraphExec; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "graph_exec"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaGraphExecTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.graph_node`: CUDA Graph Node handle +pub const GraphNodeType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaGraphNode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "graph_node"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaGraphNodeTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.graph`: CUDA Graph +pub const GraphType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaGraph; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "graph"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaGraphTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.int`: `cuda.int` is a integer type with a bitwidth matching the underlying ABI integer bitwidth +pub const IntType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaInt; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "int"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaIntTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.kernel`: CUDA Kernel handle +pub const KernelType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaKernel; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "kernel"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaKernelTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.launch_cfg`: Representation of CUDA's extensible launch configuration mirroring cuda runtime API +pub const LaunchConfigType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaLaunchConfig; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "launch_cfg"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + maxNumAttrs: u32, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCudaLaunchConfigTypeGet(ctx.ptr(), args.maxNumAttrs); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn maxNumAttrs(self: *const Self) u32 { + return c.mlirCudaLaunchConfigTypeGetMaxNumAttrs(self.ptr()); + } +}; + +/// `!cuda.library`: CUDA Library handle +pub const LibraryType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaLibrary; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "library"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaLibraryTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.module`: CUDA Module handle +pub const ModuleType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaModule; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "module"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaModuleTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.result`: CUDA Result type +pub const ResultType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaResult; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "result"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaResultTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.runtime.device_prop`: CUDA cudaDeviceProp struct type (value-typed). +pub const RuntimeDevicePropType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaRuntimeDeviceProp; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "runtime.device_prop"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaRuntimeDevicePropTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.runtime.dim_3`: CUDA dim3 struct type (value-typed). +pub const RuntimeDim3Type = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaRuntimeDim3; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "runtime.dim_3"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaRuntimeDim3TypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.runtime.extent`: CUDA cudaExtent struct type (value-typed). +pub const RuntimeExtentType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaRuntimeExtent; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "runtime.extent"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaRuntimeExtentTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.runtime.kernel_node_params`: CUDA cudaKernelNodeParams struct type (value-typed). +pub const RuntimeKernelNodeParamsType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaRuntimeKernelNodeParams; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "runtime.kernel_node_params"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaRuntimeKernelNodeParamsTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.runtime.launch_attribute_value.cluster_scheduling_policy_preference`: `!cuda.runtime.launch_attribute_value.cluster_scheduling_policy_preference` +pub const RuntimeLaunchAttributeValueClusterSchedulingPolicyPreferenceType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaRuntimeLaunchAttributeValueClusterSchedulingPolicyPreference; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "runtime.launch_attribute_value.cluster_scheduling_policy_preference"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaRuntimeLaunchAttributeValueClusterSchedulingPolicyPreferenceTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.runtime.launch_attribute_value.cooperative`: `!cuda.runtime.launch_attribute_value.cooperative` +pub const RuntimeLaunchAttributeValueCooperativeType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaRuntimeLaunchAttributeValueCooperative; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "runtime.launch_attribute_value.cooperative"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaRuntimeLaunchAttributeValueCooperativeTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.runtime.launch_attribute_value.mem_sync_domain`: `!cuda.runtime.launch_attribute_value.mem_sync_domain` +pub const RuntimeLaunchAttributeValueMemSyncDomainType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaRuntimeLaunchAttributeValueMemSyncDomain; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "runtime.launch_attribute_value.mem_sync_domain"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaRuntimeLaunchAttributeValueMemSyncDomainTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.runtime.launch_attribute_value.nvlink_util_centric_scheduling`: `!cuda.runtime.launch_attribute_value.nvlink_util_centric_scheduling` +pub const RuntimeLaunchAttributeValueNvlinkUtilCentricSchedulingType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaRuntimeLaunchAttributeValueNvlinkUtilCentricScheduling; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "runtime.launch_attribute_value.nvlink_util_centric_scheduling"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaRuntimeLaunchAttributeValueNvlinkUtilCentricSchedulingTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.runtime.launch_attribute_value.portable_cluster_size_mode`: `!cuda.runtime.launch_attribute_value.portable_cluster_size_mode` +pub const RuntimeLaunchAttributeValuePortableClusterSizeModeType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaRuntimeLaunchAttributeValuePortableClusterSizeMode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "runtime.launch_attribute_value.portable_cluster_size_mode"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaRuntimeLaunchAttributeValuePortableClusterSizeModeTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.runtime.launch_attribute_value.priority`: `!cuda.runtime.launch_attribute_value.priority` +pub const RuntimeLaunchAttributeValuePriorityType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaRuntimeLaunchAttributeValuePriority; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "runtime.launch_attribute_value.priority"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaRuntimeLaunchAttributeValuePriorityTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.runtime.launch_attribute_value.programmatic_stream_serialization_allowed`: `!cuda.runtime.launch_attribute_value.programmatic_stream_serialization_allowed` +pub const RuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowedType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaRuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowed; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "runtime.launch_attribute_value.programmatic_stream_serialization_allowed"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaRuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowedTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.runtime.launch_attribute_value.shared_mem_carveout`: `!cuda.runtime.launch_attribute_value.shared_mem_carveout` +pub const RuntimeLaunchAttributeValueSharedMemCarveoutType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaRuntimeLaunchAttributeValueSharedMemCarveout; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "runtime.launch_attribute_value.shared_mem_carveout"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaRuntimeLaunchAttributeValueSharedMemCarveoutTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.runtime.launch_attribute_value.shared_memory_mode`: `!cuda.runtime.launch_attribute_value.shared_memory_mode` +pub const RuntimeLaunchAttributeValueSharedMemoryModeType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaRuntimeLaunchAttributeValueSharedMemoryMode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "runtime.launch_attribute_value.shared_memory_mode"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaRuntimeLaunchAttributeValueSharedMemoryModeTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.runtime.launch_attribute_value.sync_policy`: `!cuda.runtime.launch_attribute_value.sync_policy` +pub const RuntimeLaunchAttributeValueSyncPolicyType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaRuntimeLaunchAttributeValueSyncPolicy; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "runtime.launch_attribute_value.sync_policy"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaRuntimeLaunchAttributeValueSyncPolicyTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.runtime.memcpy_3d_params`: CUDA cudaMemcpy3DParms struct type (value-typed). +pub const RuntimeMemcpy3DParamsType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaRuntimeMemcpy3DParams; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "runtime.memcpy_3d_params"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaRuntimeMemcpy3DParamsTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.runtime.pitched_ptr`: CUDA cudaPitchedPtr struct type (value-typed). +pub const RuntimePitchedPtrType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaRuntimePitchedPtr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "runtime.pitched_ptr"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaRuntimePitchedPtrTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.runtime.pos`: CUDA cudaPos struct type (value-typed). +pub const RuntimePosType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaRuntimePos; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "runtime.pos"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaRuntimePosTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.runtime.uuid`: CUDA cudaUUID_t struct type (value-typed). +pub const RuntimeUuidType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaRuntimeUuid; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "runtime.uuid"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaRuntimeUuidTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.stream`: CUDA Stream +pub const StreamType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaStream; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "stream"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaStreamTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cuda.tensor_map`: CUDA Tensor Map type +pub const TensorMapType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaTensorMap; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "tensor_map"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCudaTensorMapTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +// Operation builders, generated from the operations' .td. +/// `cuda.addressof`. Result types are explicit; attributes use mlir.Attribute. +pub fn addressof(ctx: *mlir.Context, result_type: *const mlir.Type, symname: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.addressof", .{ + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "symname", symname), + }, + .location = location, + }); +} + +/// `cuda.binary`. Result types are explicit; attributes use mlir.Attribute. Not verified on creation: the blocks are usually filled afterwards. +pub fn binary(ctx: *mlir.Context, sym_name: *const mlir.Attribute, binary_: *const mlir.Attribute, blocks: [1]*mlir.Block, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.binary", .{ + .attributes = &.{ + .named(ctx, "sym_name", sym_name), + .named(ctx, "binary", binary_), + }, + .blocks = &blocks, + .verify = false, + .location = location, + }); +} + +/// `cuda.cast`. Result types are explicit; attributes use mlir.Attribute. +pub fn cast(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.cast", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .location = location, + }); +} + +/// `cuda.compose`. Result types are explicit; attributes use mlir.Attribute. +pub fn compose(ctx: *mlir.Context, fields: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.compose", .{ + .operands = .{ .flat = fields }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.decompose`. Result types are explicit; attributes use mlir.Attribute. +pub fn decompose( + ctx: *mlir.Context, + source: *const mlir.Value, + opts: struct { + fields_type: ?*const mlir.Type = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var results: stdx.BoundedArray(*const mlir.Type, 256) = .empty; + if (opts.fields_type) |value| results.appendAssumeCapacity(value); + return mlir.Operation.make(ctx, "cuda.decompose", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = results.constSlice() }, + .location = location, + }); +} + +/// `cuda.device.sync`. Result types are explicit; attributes use mlir.Attribute. +pub fn device_sync(ctx: *mlir.Context, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.device.sync", .{ + .results = .{ .flat = &.{cuda_result_type} }, + .location = location, + }); +} + +/// `cuda.driver.event.create`. Result types are explicit; attributes use mlir.Attribute. +pub fn driver_event_create(ctx: *mlir.Context, Flags: *const mlir.Value, cuda_result_type: *const mlir.Type, phEvent_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.driver.event.create", .{ + .operands = .{ .flat = &.{Flags} }, + .results = .{ .flat = &.{ cuda_result_type, phEvent_type } }, + .location = location, + }); +} + +/// `cuda.driver.event.destroy`. Result types are explicit; attributes use mlir.Attribute. +pub fn driver_event_destroy(ctx: *mlir.Context, hEvent: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.driver.event.destroy", .{ + .operands = .{ .flat = &.{hEvent} }, + .results = .{ .flat = &.{cuda_result_type} }, + .location = location, + }); +} + +/// `cuda.driver.event.elapsed_time`. Result types are explicit; attributes use mlir.Attribute. +pub fn driver_event_elapsed_time(ctx: *mlir.Context, hStart: *const mlir.Value, hEnd: *const mlir.Value, cuda_result_type: *const mlir.Type, pMilliseconds_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.driver.event.elapsed_time", .{ + .operands = .{ .flat = &.{ hStart, hEnd } }, + .results = .{ .flat = &.{ cuda_result_type, pMilliseconds_type } }, + .location = location, + }); +} + +/// `cuda.driver.event.query`. Result types are explicit; attributes use mlir.Attribute. +pub fn driver_event_query(ctx: *mlir.Context, hEvent: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.driver.event.query", .{ + .operands = .{ .flat = &.{hEvent} }, + .results = .{ .flat = &.{cuda_result_type} }, + .location = location, + }); +} + +/// `cuda.driver.event.record`. Result types are explicit; attributes use mlir.Attribute. +pub fn driver_event_record(ctx: *mlir.Context, hEvent: *const mlir.Value, hStream: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.driver.event.record", .{ + .operands = .{ .flat = &.{ hEvent, hStream } }, + .results = .{ .flat = &.{cuda_result_type} }, + .location = location, + }); +} + +/// `cuda.driver.event.record_with_flags`. Result types are explicit; attributes use mlir.Attribute. +pub fn driver_event_record_with_flags(ctx: *mlir.Context, hEvent: *const mlir.Value, hStream: *const mlir.Value, flags: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.driver.event.record_with_flags", .{ + .operands = .{ .flat = &.{ hEvent, hStream, flags } }, + .results = .{ .flat = &.{cuda_result_type} }, + .location = location, + }); +} + +/// `cuda.driver.event.synchronize`. Result types are explicit; attributes use mlir.Attribute. +pub fn driver_event_synchronize(ctx: *mlir.Context, hEvent: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.driver.event.synchronize", .{ + .operands = .{ .flat = &.{hEvent} }, + .results = .{ .flat = &.{cuda_result_type} }, + .location = location, + }); +} + +/// `cuda.driver.library.get_global`. Result types are explicit; attributes use mlir.Attribute. +pub fn driver_library_get_global(ctx: *mlir.Context, library: *const mlir.Value, name_arg: *const mlir.Value, cuda_result_type: *const mlir.Type, dptr_type: *const mlir.Type, bytes_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.driver.library.get_global", .{ + .operands = .{ .flat = &.{ library, name_arg } }, + .results = .{ .flat = &.{ cuda_result_type, dptr_type, bytes_type } }, + .location = location, + }); +} + +/// `cuda.driver.library.get_kernel`. Result types are explicit; attributes use mlir.Attribute. +pub fn driver_library_get_kernel(ctx: *mlir.Context, library: *const mlir.Value, name_arg: *const mlir.Value, cuda_result_type: *const mlir.Type, pKernel_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.driver.library.get_kernel", .{ + .operands = .{ .flat = &.{ library, name_arg } }, + .results = .{ .flat = &.{ cuda_result_type, pKernel_type } }, + .location = location, + }); +} + +/// `cuda.driver.library.get_kernel_count`. Result types are explicit; attributes use mlir.Attribute. +pub fn driver_library_get_kernel_count(ctx: *mlir.Context, lib: *const mlir.Value, cuda_result_type: *const mlir.Type, count_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.driver.library.get_kernel_count", .{ + .operands = .{ .flat = &.{lib} }, + .results = .{ .flat = &.{ cuda_result_type, count_type } }, + .location = location, + }); +} + +/// `cuda.driver.library.get_managed`. Result types are explicit; attributes use mlir.Attribute. +pub fn driver_library_get_managed(ctx: *mlir.Context, library: *const mlir.Value, name_arg: *const mlir.Value, cuda_result_type: *const mlir.Type, dptr_type: *const mlir.Type, bytes_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.driver.library.get_managed", .{ + .operands = .{ .flat = &.{ library, name_arg } }, + .results = .{ .flat = &.{ cuda_result_type, dptr_type, bytes_type } }, + .location = location, + }); +} + +/// `cuda.driver.library.get_module`. Result types are explicit; attributes use mlir.Attribute. +pub fn driver_library_get_module(ctx: *mlir.Context, library: *const mlir.Value, cuda_result_type: *const mlir.Type, pMod_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.driver.library.get_module", .{ + .operands = .{ .flat = &.{library} }, + .results = .{ .flat = &.{ cuda_result_type, pMod_type } }, + .location = location, + }); +} + +/// `cuda.driver.library.get_unified_function`. Result types are explicit; attributes use mlir.Attribute. +pub fn driver_library_get_unified_function(ctx: *mlir.Context, library: *const mlir.Value, symbol: *const mlir.Value, cuda_result_type: *const mlir.Type, fptr_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.driver.library.get_unified_function", .{ + .operands = .{ .flat = &.{ library, symbol } }, + .results = .{ .flat = &.{ cuda_result_type, fptr_type } }, + .location = location, + }); +} + +/// `cuda.driver.library.load_data`. Result types are explicit; attributes use mlir.Attribute. +pub fn driver_library_load_data(ctx: *mlir.Context, code_arg: *const mlir.Value, jitOptions: *const mlir.Value, jitOptionsValues: *const mlir.Value, numJitOptions: *const mlir.Value, libraryOptions: *const mlir.Value, libraryOptionValues: *const mlir.Value, numLibraryOptions: *const mlir.Value, cuda_result_type: *const mlir.Type, library_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.driver.library.load_data", .{ + .operands = .{ .flat = &.{ code_arg, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions } }, + .results = .{ .flat = &.{ cuda_result_type, library_type } }, + .location = location, + }); +} + +/// `cuda.driver.library.load_from_file`. Result types are explicit; attributes use mlir.Attribute. +pub fn driver_library_load_from_file(ctx: *mlir.Context, fileName: *const mlir.Value, jitOptions: *const mlir.Value, jitOptionsValues: *const mlir.Value, numJitOptions: *const mlir.Value, libraryOptions: *const mlir.Value, libraryOptionValues: *const mlir.Value, numLibraryOptions: *const mlir.Value, cuda_result_type: *const mlir.Type, library_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.driver.library.load_from_file", .{ + .operands = .{ .flat = &.{ fileName, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions } }, + .results = .{ .flat = &.{ cuda_result_type, library_type } }, + .location = location, + }); +} + +/// `cuda.driver.library.unload`. Result types are explicit; attributes use mlir.Attribute. +pub fn driver_library_unload(ctx: *mlir.Context, library: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.driver.library.unload", .{ + .operands = .{ .flat = &.{library} }, + .results = .{ .flat = &.{cuda_result_type} }, + .location = location, + }); +} + +/// `cuda.driver.tensor_map.encode_im2col`. Result types are explicit; attributes use mlir.Attribute. +pub fn driver_tensor_map_encode_im2col(ctx: *mlir.Context, tensorDataType: *const mlir.Value, tensorRank: *const mlir.Value, globalAddress: *const mlir.Value, globalDim: *const mlir.Value, globalStrides: *const mlir.Value, pixelBoxLowerCorner: *const mlir.Value, pixelBoxUpperCorner: *const mlir.Value, channelsPerPixel: *const mlir.Value, pixelsPerColumn: *const mlir.Value, elementStrides: *const mlir.Value, interleave: *const mlir.Value, swizzle: *const mlir.Value, l2Promotion: *const mlir.Value, oobFill: *const mlir.Value, cuda_result_type: *const mlir.Type, tensorMap_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.driver.tensor_map.encode_im2col", .{ + .operands = .{ .flat = &.{ tensorDataType, tensorRank, globalAddress, globalDim, globalStrides, pixelBoxLowerCorner, pixelBoxUpperCorner, channelsPerPixel, pixelsPerColumn, elementStrides, interleave, swizzle, l2Promotion, oobFill } }, + .results = .{ .flat = &.{ cuda_result_type, tensorMap_type } }, + .location = location, + }); +} + +/// `cuda.driver.tensor_map.encode_im2col_wide`. Result types are explicit; attributes use mlir.Attribute. +pub fn driver_tensor_map_encode_im2col_wide(ctx: *mlir.Context, tensorDataType: *const mlir.Value, tensorRank: *const mlir.Value, globalAddress: *const mlir.Value, globalDim: *const mlir.Value, globalStrides: *const mlir.Value, pixelBoxLowerCornerWidth: *const mlir.Value, pixelBoxUpperCornerWidth: *const mlir.Value, channelsPerPixel: *const mlir.Value, pixelsPerColumn: *const mlir.Value, elementStrides: *const mlir.Value, interleave: *const mlir.Value, mode: *const mlir.Value, swizzle: *const mlir.Value, l2Promotion: *const mlir.Value, oobFill: *const mlir.Value, cuda_result_type: *const mlir.Type, tensorMap_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.driver.tensor_map.encode_im2col_wide", .{ + .operands = .{ .flat = &.{ tensorDataType, tensorRank, globalAddress, globalDim, globalStrides, pixelBoxLowerCornerWidth, pixelBoxUpperCornerWidth, channelsPerPixel, pixelsPerColumn, elementStrides, interleave, mode, swizzle, l2Promotion, oobFill } }, + .results = .{ .flat = &.{ cuda_result_type, tensorMap_type } }, + .location = location, + }); +} + +/// `cuda.driver.tensor_map.replace_address`. Result types are explicit; attributes use mlir.Attribute. +pub fn driver_tensor_map_replace_address(ctx: *mlir.Context, globalAddress: *const mlir.Value, cuda_result_type: *const mlir.Type, tensorMap_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.driver.tensor_map.replace_address", .{ + .operands = .{ .flat = &.{globalAddress} }, + .results = .{ .flat = &.{ cuda_result_type, tensorMap_type } }, + .location = location, + }); +} + +/// `cuda.event.create`. Result types are explicit; attributes use mlir.Attribute. +pub fn event_create(ctx: *mlir.Context, result_type: *const mlir.Type, event_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.event.create", .{ + .results = .{ .flat = &.{ result_type, event_type } }, + .location = location, + }); +} + +/// `cuda.event.destroy`. Result types are explicit; attributes use mlir.Attribute. +pub fn event_destroy(ctx: *mlir.Context, event: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.event.destroy", .{ + .operands = .{ .flat = &.{event} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.event.elapsed_time`. Result types are explicit; attributes use mlir.Attribute. +pub fn event_elapsed_time(ctx: *mlir.Context, start_event: *const mlir.Value, end_event: *const mlir.Value, cuda_result_type: *const mlir.Type, elapsed_time_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.event.elapsed_time", .{ + .operands = .{ .flat = &.{ start_event, end_event } }, + .results = .{ .flat = &.{ cuda_result_type, elapsed_time_type } }, + .location = location, + }); +} + +/// `cuda.event.record`. Result types are explicit; attributes use mlir.Attribute. +pub fn event_record( + ctx: *mlir.Context, + event: *const mlir.Value, + result_type: *const mlir.Type, + opts: struct { + stream: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.event.record", .{ + .operands = .{ .flat = if (opts.stream) |value| &.{ event, value } else &.{event} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.func`. Result types are explicit; attributes use mlir.Attribute. Not verified on creation: the blocks are usually filled afterwards. +pub fn func( + ctx: *mlir.Context, + sym_name: *const mlir.Attribute, + function_type: *const mlir.Attribute, + blocks: [1]*mlir.Block, + opts: struct { + arg_attrs: ?*const mlir.Attribute = null, + res_attrs: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "sym_name", sym_name)); + attributes.appendAssumeCapacity(.named(ctx, "function_type", function_type)); + if (opts.arg_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "arg_attrs", value)); + if (opts.res_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "res_attrs", value)); + return mlir.Operation.make(ctx, "cuda.func", .{ + .attributes = attributes.constSlice(), + .blocks = &blocks, + .verify = false, + .location = location, + }); +} + +/// `cuda.func.get_name`. Result types are explicit; attributes use mlir.Attribute. +pub fn func_get_name(ctx: *mlir.Context, kernel_: *const mlir.Value, result_type: *const mlir.Type, name_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.func.get_name", .{ + .operands = .{ .flat = &.{kernel_} }, + .results = .{ .flat = &.{ result_type, name_type } }, + .location = location, + }); +} + +/// `cuda.graph.destroy`. Result types are explicit; attributes use mlir.Attribute. +pub fn graph_destroy(ctx: *mlir.Context, graph: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.graph.destroy", .{ + .operands = .{ .flat = &.{graph} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.graph.instantiate`. Result types are explicit; attributes use mlir.Attribute. +pub fn graph_instantiate(ctx: *mlir.Context, graph: *const mlir.Value, result_type: *const mlir.Type, graph_exec_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.graph.instantiate", .{ + .operands = .{ .flat = &.{graph} }, + .results = .{ .flat = &.{ result_type, graph_exec_type } }, + .location = location, + }); +} + +/// `cuda.graph.launch`. Result types are explicit; attributes use mlir.Attribute. +pub fn graph_launch(ctx: *mlir.Context, graph_exec: *const mlir.Value, stream: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.graph.launch", .{ + .operands = .{ .flat = &.{ graph_exec, stream } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.graph_exec.destroy`. Result types are explicit; attributes use mlir.Attribute. +pub fn graph_exec_destroy(ctx: *mlir.Context, graph_exec: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.graph_exec.destroy", .{ + .operands = .{ .flat = &.{graph_exec} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.kernel`. Result types are explicit; attributes use mlir.Attribute. Not verified on creation: the blocks are usually filled afterwards. +pub fn kernel( + ctx: *mlir.Context, + sym_name: *const mlir.Attribute, + function_type: *const mlir.Attribute, + blocks: [1]*mlir.Block, + opts: struct { + arg_attrs: ?*const mlir.Attribute = null, + res_attrs: ?*const mlir.Attribute = null, + cu_func_attrs: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "sym_name", sym_name)); + attributes.appendAssumeCapacity(.named(ctx, "function_type", function_type)); + if (opts.arg_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "arg_attrs", value)); + if (opts.res_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "res_attrs", value)); + if (opts.cu_func_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "cu_func_attrs", value)); + return mlir.Operation.make(ctx, "cuda.kernel", .{ + .attributes = attributes.constSlice(), + .blocks = &blocks, + .verify = false, + .location = location, + }); +} + +/// `cuda.launch`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch( + ctx: *mlir.Context, + grid: []const *const mlir.Value, + block: []const *const mlir.Value, + dynamicSmem: *const mlir.Value, + cudaStream: *const mlir.Value, + inputs: []const *const mlir.Value, + cuda_result_type: *const mlir.Type, + callee: *const mlir.Attribute, + opts: struct { + arg_attrs: ?*const mlir.Attribute = null, + res_attrs: ?*const mlir.Attribute = null, + assume_kernel_attr: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "callee", callee)); + if (opts.arg_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "arg_attrs", value)); + if (opts.res_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "res_attrs", value)); + if (opts.assume_kernel_attr) |value| attributes.appendAssumeCapacity(.named(ctx, "assume_kernel_attr", value)); + return mlir.Operation.make(ctx, "cuda.launch", .{ + .operands = .{ .variadic = &.{ + grid, + block, + &.{dynamicSmem}, + &.{cudaStream}, + inputs, + } }, + .results = .{ .flat = &.{cuda_result_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cuda.launch_cfg.access_policy_window`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch_cfg_access_policy_window(ctx: *mlir.Context, launch_cfg: *const mlir.Value, base_ptr: *const mlir.Value, hitRatio: *const mlir.Value, num_bytes: *const mlir.Value, hitProp: *const mlir.Attribute, missProp: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.launch_cfg.access_policy_window", .{ + .operands = .{ .flat = &.{ launch_cfg, base_ptr, hitRatio, num_bytes } }, + .attributes = &.{ + .named(ctx, "hitProp", hitProp), + .named(ctx, "missProp", missProp), + }, + .location = location, + }); +} + +/// `cuda.launch_cfg.cluster_dim`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch_cfg_cluster_dim(ctx: *mlir.Context, launch_cfg: *const mlir.Value, clusterDimX: *const mlir.Value, clusterDimY: *const mlir.Value, clusterDimZ: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.launch_cfg.cluster_dim", .{ + .operands = .{ .flat = &.{ launch_cfg, clusterDimX, clusterDimY, clusterDimZ } }, + .location = location, + }); +} + +/// `cuda.launch_cfg.cluster_scheduling_policy`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch_cfg_cluster_scheduling_policy(ctx: *mlir.Context, launch_cfg: *const mlir.Value, policy: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.launch_cfg.cluster_scheduling_policy", .{ + .operands = .{ .flat = &.{launch_cfg} }, + .attributes = &.{ + .named(ctx, "policy", policy), + }, + .location = location, + }); +} + +/// `cuda.launch_cfg.cooperative`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch_cfg_cooperative(ctx: *mlir.Context, launch_cfg: *const mlir.Value, cooperative: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.launch_cfg.cooperative", .{ + .operands = .{ .flat = &.{ launch_cfg, cooperative } }, + .location = location, + }); +} + +/// `cuda.launch_cfg.create`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch_cfg_create(ctx: *mlir.Context, blockDimX: *const mlir.Value, blockDimY: *const mlir.Value, blockDimZ: *const mlir.Value, dynamicSmemBytes: *const mlir.Value, gridDimX: *const mlir.Value, gridDimY: *const mlir.Value, gridDimZ: *const mlir.Value, stream: *const mlir.Value, result_type: *const mlir.Type, maxNumAttrs: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.launch_cfg.create", .{ + .operands = .{ .flat = &.{ blockDimX, blockDimY, blockDimZ, dynamicSmemBytes, gridDimX, gridDimY, gridDimZ, stream } }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "maxNumAttrs", maxNumAttrs), + }, + .location = location, + }); +} + +/// `cuda.launch_cfg.create_from_stream`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch_cfg_create_from_stream(ctx: *mlir.Context, stream: *const mlir.Value, result_type: *const mlir.Type, maxNumAttrs: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.launch_cfg.create_from_stream", .{ + .operands = .{ .flat = &.{stream} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "maxNumAttrs", maxNumAttrs), + }, + .location = location, + }); +} + +/// `cuda.launch_cfg.device_updatable_kernel_node`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch_cfg_device_updatable_kernel_node(ctx: *mlir.Context, launch_cfg: *const mlir.Value, devNode: *const mlir.Value, deviceUpdatable: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.launch_cfg.device_updatable_kernel_node", .{ + .operands = .{ .flat = &.{ launch_cfg, devNode, deviceUpdatable } }, + .location = location, + }); +} + +/// `cuda.launch_cfg.grid_dimensions`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch_cfg_grid_dimensions(ctx: *mlir.Context, launch_cfg: *const mlir.Value, gridDimX: *const mlir.Value, gridDimY: *const mlir.Value, gridDimZ: *const mlir.Value, blockDimX: *const mlir.Value, blockDimY: *const mlir.Value, blockDimZ: *const mlir.Value, dynamicSmemBytes: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.launch_cfg.grid_dimensions", .{ + .operands = .{ .flat = &.{ launch_cfg, gridDimX, gridDimY, gridDimZ, blockDimX, blockDimY, blockDimZ, dynamicSmemBytes } }, + .location = location, + }); +} + +/// `cuda.launch_cfg.launch_completion_event`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch_cfg_launch_completion_event(ctx: *mlir.Context, launch_cfg: *const mlir.Value, event: *const mlir.Value, flags: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.launch_cfg.launch_completion_event", .{ + .operands = .{ .flat = &.{ launch_cfg, event, flags } }, + .location = location, + }); +} + +/// `cuda.launch_cfg.mem_sync_domain`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch_cfg_mem_sync_domain(ctx: *mlir.Context, launch_cfg: *const mlir.Value, memSyncDomain: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.launch_cfg.mem_sync_domain", .{ + .operands = .{ .flat = &.{launch_cfg} }, + .attributes = &.{ + .named(ctx, "memSyncDomain", memSyncDomain), + }, + .location = location, + }); +} + +/// `cuda.launch_cfg.mem_sync_domain_map`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch_cfg_mem_sync_domain_map(ctx: *mlir.Context, launch_cfg: *const mlir.Value, domain_default: *const mlir.Value, domain_remote: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.launch_cfg.mem_sync_domain_map", .{ + .operands = .{ .flat = &.{ launch_cfg, domain_default, domain_remote } }, + .location = location, + }); +} + +/// `cuda.launch_cfg.nvlink_util_centric_scheduling`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch_cfg_nvlink_util_centric_scheduling(ctx: *mlir.Context, launch_cfg: *const mlir.Value, nvlinkSchedulingEnabled: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.launch_cfg.nvlink_util_centric_scheduling", .{ + .operands = .{ .flat = &.{ launch_cfg, nvlinkSchedulingEnabled } }, + .location = location, + }); +} + +/// `cuda.launch_cfg.preferred_cluster_dim`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch_cfg_preferred_cluster_dim(ctx: *mlir.Context, launch_cfg: *const mlir.Value, preferredClusterDimX: *const mlir.Value, preferredClusterDimY: *const mlir.Value, preferredClusterDimZ: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.launch_cfg.preferred_cluster_dim", .{ + .operands = .{ .flat = &.{ launch_cfg, preferredClusterDimX, preferredClusterDimY, preferredClusterDimZ } }, + .location = location, + }); +} + +/// `cuda.launch_cfg.print`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch_cfg_print(ctx: *mlir.Context, launch_cfg: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.launch_cfg.print", .{ + .operands = .{ .flat = &.{launch_cfg} }, + .location = location, + }); +} + +/// `cuda.launch_cfg.priority`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch_cfg_priority(ctx: *mlir.Context, launch_cfg: *const mlir.Value, priority: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.launch_cfg.priority", .{ + .operands = .{ .flat = &.{ launch_cfg, priority } }, + .location = location, + }); +} + +/// `cuda.launch_cfg.programmatic_event`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch_cfg_programmatic_event(ctx: *mlir.Context, launch_cfg: *const mlir.Value, event: *const mlir.Value, flags: *const mlir.Value, triggerAtBlockStart: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.launch_cfg.programmatic_event", .{ + .operands = .{ .flat = &.{ launch_cfg, event, flags, triggerAtBlockStart } }, + .location = location, + }); +} + +/// `cuda.launch_cfg.programmatic_stream_serialization_allowed`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch_cfg_programmatic_stream_serialization_allowed(ctx: *mlir.Context, launch_cfg: *const mlir.Value, programmaticStreamSerializationAllowed: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.launch_cfg.programmatic_stream_serialization_allowed", .{ + .operands = .{ .flat = &.{ launch_cfg, programmaticStreamSerializationAllowed } }, + .location = location, + }); +} + +/// `cuda.launch_cfg.shared_mem_carveout`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch_cfg_shared_mem_carveout(ctx: *mlir.Context, launch_cfg: *const mlir.Value, sharedMemCarveout: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.launch_cfg.shared_mem_carveout", .{ + .operands = .{ .flat = &.{ launch_cfg, sharedMemCarveout } }, + .location = location, + }); +} + +/// `cuda.launch_cfg.sync_policy`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch_cfg_sync_policy(ctx: *mlir.Context, launch_cfg: *const mlir.Value, syncPolicy: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.launch_cfg.sync_policy", .{ + .operands = .{ .flat = &.{launch_cfg} }, + .attributes = &.{ + .named(ctx, "syncPolicy", syncPolicy), + }, + .location = location, + }); +} + +/// `cuda.launch_ex`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch_ex( + ctx: *mlir.Context, + launch_cfg: *const mlir.Value, + inputs: []const *const mlir.Value, + cuda_result_type: *const mlir.Type, + callee: *const mlir.Attribute, + opts: struct { + arg_attrs: ?*const mlir.Attribute = null, + res_attrs: ?*const mlir.Attribute = null, + assume_kernel_attr: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var operands: stdx.BoundedArray(*const mlir.Value, 256) = .empty; + operands.appendAssumeCapacity(launch_cfg); + operands.appendSliceAssumeCapacity(inputs); + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "callee", callee)); + if (opts.arg_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "arg_attrs", value)); + if (opts.res_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "res_attrs", value)); + if (opts.assume_kernel_attr) |value| attributes.appendAssumeCapacity(.named(ctx, "assume_kernel_attr", value)); + return mlir.Operation.make(ctx, "cuda.launch_ex", .{ + .operands = .{ .flat = operands.constSlice() }, + .results = .{ .flat = &.{cuda_result_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cuda.launch_kernel`. Result types are explicit; attributes use mlir.Attribute. +pub fn launch_kernel( + ctx: *mlir.Context, + kernel_: *const mlir.Value, + grid: []const *const mlir.Value, + block: []const *const mlir.Value, + dynamicSmem: *const mlir.Value, + cudaStream: *const mlir.Value, + kernel_args: *const mlir.Value, + cuda_result_type: *const mlir.Type, + opts: struct { + arg_attrs: ?*const mlir.Attribute = null, + res_attrs: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.arg_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "arg_attrs", value)); + if (opts.res_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "res_attrs", value)); + return mlir.Operation.make(ctx, "cuda.launch_kernel", .{ + .operands = .{ .variadic = &.{ + &.{kernel_}, + grid, + block, + &.{dynamicSmem}, + &.{cudaStream}, + &.{kernel_args}, + } }, + .results = .{ .flat = &.{cuda_result_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cuda.library.get_kernel`. Result types are explicit; attributes use mlir.Attribute. +pub fn library_get_kernel(ctx: *mlir.Context, library: *const mlir.Value, kernel_name: *const mlir.Value, result_type: *const mlir.Type, kernel_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.library.get_kernel", .{ + .operands = .{ .flat = &.{ library, kernel_name } }, + .results = .{ .flat = &.{ result_type, kernel_type } }, + .location = location, + }); +} + +/// `cuda.library.load_data`. Result types are explicit; attributes use mlir.Attribute. +pub fn library_load_data(ctx: *mlir.Context, data: *const mlir.Value, result_type: *const mlir.Type, library_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.library.load_data", .{ + .operands = .{ .flat = &.{data} }, + .results = .{ .flat = &.{ result_type, library_type } }, + .location = location, + }); +} + +/// `cuda.library.unload`. Result types are explicit; attributes use mlir.Attribute. +pub fn library_unload(ctx: *mlir.Context, library: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.library.unload", .{ + .operands = .{ .flat = &.{library} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.memory.alloc`. Result types are explicit; attributes use mlir.Attribute. +pub fn memory_alloc(ctx: *mlir.Context, bytes: *const mlir.Value, cuda_result_type: *const mlir.Type, ptr_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.memory.alloc", .{ + .operands = .{ .flat = &.{bytes} }, + .results = .{ .flat = &.{ cuda_result_type, ptr_type } }, + .location = location, + }); +} + +/// `cuda.memory.alloc_async`. Result types are explicit; attributes use mlir.Attribute. +pub fn memory_alloc_async(ctx: *mlir.Context, bytes: *const mlir.Value, stream: *const mlir.Value, cuda_result_type: *const mlir.Type, ptr_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.memory.alloc_async", .{ + .operands = .{ .flat = &.{ bytes, stream } }, + .results = .{ .flat = &.{ cuda_result_type, ptr_type } }, + .location = location, + }); +} + +/// `cuda.memory.alloc_host`. Result types are explicit; attributes use mlir.Attribute. +pub fn memory_alloc_host(ctx: *mlir.Context, bytes: *const mlir.Value, cuda_result_type: *const mlir.Type, ptr_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.memory.alloc_host", .{ + .operands = .{ .flat = &.{bytes} }, + .results = .{ .flat = &.{ cuda_result_type, ptr_type } }, + .location = location, + }); +} + +/// `cuda.memory.alloc_managed`. Result types are explicit; attributes use mlir.Attribute. +pub fn memory_alloc_managed(ctx: *mlir.Context, bytes: *const mlir.Value, cuda_result_type: *const mlir.Type, ptr_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.memory.alloc_managed", .{ + .operands = .{ .flat = &.{bytes} }, + .results = .{ .flat = &.{ cuda_result_type, ptr_type } }, + .location = location, + }); +} + +/// `cuda.memory.copy`. Result types are explicit; attributes use mlir.Attribute. +pub fn memory_copy(ctx: *mlir.Context, src: *const mlir.Value, dst: *const mlir.Value, bytes: *const mlir.Value, cuda_result_type: *const mlir.Type, kind: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.memory.copy", .{ + .operands = .{ .flat = &.{ src, dst, bytes } }, + .results = .{ .flat = &.{cuda_result_type} }, + .attributes = &.{ + .named(ctx, "kind", kind), + }, + .location = location, + }); +} + +/// `cuda.memory.copy_async`. Result types are explicit; attributes use mlir.Attribute. +pub fn memory_copy_async( + ctx: *mlir.Context, + src: *const mlir.Value, + dst: *const mlir.Value, + bytes: *const mlir.Value, + cuda_result_type: *const mlir.Type, + kind: *const mlir.Attribute, + opts: struct { + stream: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.memory.copy_async", .{ + .operands = .{ .flat = if (opts.stream) |value| &.{ src, dst, bytes, value } else &.{ src, dst, bytes } }, + .results = .{ .flat = &.{cuda_result_type} }, + .attributes = &.{ + .named(ctx, "kind", kind), + }, + .location = location, + }); +} + +/// `cuda.memory.free`. Result types are explicit; attributes use mlir.Attribute. +pub fn memory_free(ctx: *mlir.Context, ptr: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.memory.free", .{ + .operands = .{ .flat = &.{ptr} }, + .results = .{ .flat = &.{cuda_result_type} }, + .location = location, + }); +} + +/// `cuda.memory.free_async`. Result types are explicit; attributes use mlir.Attribute. +pub fn memory_free_async(ctx: *mlir.Context, ptr: *const mlir.Value, stream: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.memory.free_async", .{ + .operands = .{ .flat = &.{ ptr, stream } }, + .results = .{ .flat = &.{cuda_result_type} }, + .location = location, + }); +} + +/// `cuda.memory.free_host`. Result types are explicit; attributes use mlir.Attribute. +pub fn memory_free_host(ctx: *mlir.Context, ptr: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.memory.free_host", .{ + .operands = .{ .flat = &.{ptr} }, + .results = .{ .flat = &.{cuda_result_type} }, + .location = location, + }); +} + +/// `cuda.memory.set`. Result types are explicit; attributes use mlir.Attribute. +pub fn memory_set(ctx: *mlir.Context, ptr: *const mlir.Value, value_: *const mlir.Value, bytes: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.memory.set", .{ + .operands = .{ .flat = &.{ ptr, value_, bytes } }, + .results = .{ .flat = &.{cuda_result_type} }, + .location = location, + }); +} + +/// `cuda.memory.set_async`. Result types are explicit; attributes use mlir.Attribute. +pub fn memory_set_async( + ctx: *mlir.Context, + ptr: *const mlir.Value, + value_: *const mlir.Value, + bytes: *const mlir.Value, + cuda_result_type: *const mlir.Type, + opts: struct { + stream: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.memory.set_async", .{ + .operands = .{ .flat = if (opts.stream) |value| &.{ ptr, value_, bytes, value } else &.{ ptr, value_, bytes } }, + .results = .{ .flat = &.{cuda_result_type} }, + .location = location, + }); +} + +/// `cuda.occupancy_max_active_blocks_per_multiprocessor`. Result types are explicit; attributes use mlir.Attribute. +pub fn occupancy_max_active_blocks_per_multiprocessor(ctx: *mlir.Context, block_size: *const mlir.Value, dynamic_smem_size: *const mlir.Value, res_type: *const mlir.Type, num_blocks_type: *const mlir.Type, func_: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.occupancy_max_active_blocks_per_multiprocessor", .{ + .operands = .{ .flat = &.{ block_size, dynamic_smem_size } }, + .results = .{ .flat = &.{ res_type, num_blocks_type } }, + .attributes = &.{ + .named(ctx, "func", func_), + }, + .location = location, + }); +} + +/// `cuda.profiler.start`. Result types are explicit; attributes use mlir.Attribute. +pub fn profiler_start(ctx: *mlir.Context, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.profiler.start", .{ + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.profiler.stop`. Result types are explicit; attributes use mlir.Attribute. +pub fn profiler_stop(ctx: *mlir.Context, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.profiler.stop", .{ + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.result.assert_success`. Result types are explicit; attributes use mlir.Attribute. +pub fn result_assert_success(ctx: *mlir.Context, cuda_result: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.result.assert_success", .{ + .operands = .{ .flat = &.{cuda_result} }, + .location = location, + }); +} + +/// `cuda.result.failed`. Result types are explicit; attributes use mlir.Attribute. +pub fn result_failed(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.result.failed", .{ + .location = location, + }); +} + +/// `cuda.result.print_on_error`. Result types are explicit; attributes use mlir.Attribute. +pub fn result_print_on_error(ctx: *mlir.Context, cuda_result: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.result.print_on_error", .{ + .operands = .{ .flat = &.{cuda_result} }, + .location = location, + }); +} + +/// `cuda.result.succeeded`. Result types are explicit; attributes use mlir.Attribute. +pub fn result_succeeded(ctx: *mlir.Context, cuda_result: *const mlir.Value, success_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.result.succeeded", .{ + .operands = .{ .flat = &.{cuda_result} }, + .results = .{ .flat = &.{success_type} }, + .location = location, + }); +} + +/// `cuda.return`. Result types are explicit; attributes use mlir.Attribute. Not verified on creation: a terminator is only one in its block. +pub fn return_(ctx: *mlir.Context, inputs: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.return", .{ + .operands = .{ .flat = inputs }, + .verify = false, + .location = location, + }); +} + +/// `cuda.return_if_error`. Result types are explicit; attributes use mlir.Attribute. +pub fn return_if_error(ctx: *mlir.Context, inputs: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.return_if_error", .{ + .operands = .{ .flat = inputs }, + .location = location, + }); +} + +/// `cuda.runtime.device.properties`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_properties(ctx: *mlir.Context, device: *const mlir.Value, cuda_result_type: *const mlir.Type, prop_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device.properties", .{ + .operands = .{ .flat = &.{device} }, + .results = .{ .flat = &.{ cuda_result_type, prop_type } }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_access_policy_max_window_size`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_access_policy_max_window_size(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_access_policy_max_window_size", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_async_engine_count`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_async_engine_count(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_async_engine_count", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_can_map_host_memory`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_can_map_host_memory(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_can_map_host_memory", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_can_use_host_pointer_for_registered_mem`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_can_use_host_pointer_for_registered_mem(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_can_use_host_pointer_for_registered_mem", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_cluster_launch`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_cluster_launch(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_cluster_launch", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_compute_preemption_supported`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_compute_preemption_supported(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_compute_preemption_supported", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_concurrent_kernels`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_concurrent_kernels(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_concurrent_kernels", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_concurrent_managed_access`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_concurrent_managed_access(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_concurrent_managed_access", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_cooperative_launch`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_cooperative_launch(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_cooperative_launch", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_deferred_mapping_cuda_array_supported`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_deferred_mapping_cuda_array_supported(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_deferred_mapping_cuda_array_supported", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_device_numa_config`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_device_numa_config(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_device_numa_config", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_device_numa_id`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_device_numa_id(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_device_numa_id", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_direct_managed_mem_access_from_host`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_direct_managed_mem_access_from_host(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_direct_managed_mem_access_from_host", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_ecc_enabled`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_ecc_enabled(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_ecc_enabled", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_global_l1cache_supported`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_global_l1cache_supported(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_global_l1cache_supported", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_gpu_direct_rdma_flush_writes_options`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_gpu_direct_rdma_flush_writes_options(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_gpu_direct_rdma_flush_writes_options", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_gpu_direct_rdma_supported`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_gpu_direct_rdma_supported(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_gpu_direct_rdma_supported", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_gpu_direct_rdma_writes_ordering`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_gpu_direct_rdma_writes_ordering(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_gpu_direct_rdma_writes_ordering", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_gpu_pci_device_id`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_gpu_pci_device_id(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_gpu_pci_device_id", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_gpu_pci_subsystem_id`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_gpu_pci_subsystem_id(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_gpu_pci_subsystem_id", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_host_native_atomic_supported`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_host_native_atomic_supported(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_host_native_atomic_supported", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_host_numa_id`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_host_numa_id(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_host_numa_id", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_host_numa_multinode_ipc_supported`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_host_numa_multinode_ipc_supported(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_host_numa_multinode_ipc_supported", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_host_register_read_only_supported`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_host_register_read_only_supported(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_host_register_read_only_supported", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_host_register_supported`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_host_register_supported(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_host_register_supported", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_integrated`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_integrated(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_integrated", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_ipc_event_supported`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_ipc_event_supported(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_ipc_event_supported", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_is_multi_gpu_board`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_is_multi_gpu_board(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_is_multi_gpu_board", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_l_2cache_size`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_l_2cache_size(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_l_2cache_size", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_local_l1cache_supported`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_local_l1cache_supported(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_local_l1cache_supported", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_luid`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_luid(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_luid", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_luid_device_node_mask`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_luid_device_node_mask(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_luid_device_node_mask", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_major`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_major(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_major", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_managed_memory`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_managed_memory(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_managed_memory", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_blocks_per_multi_processor`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_blocks_per_multi_processor(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_blocks_per_multi_processor", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_grid_size`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_grid_size(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_grid_size", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_surface_1d`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_surface_1d(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_surface_1d", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_surface_1d_layered`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_surface_1d_layered(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_surface_1d_layered", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_surface_2d`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_surface_2d(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_surface_2d", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_surface_2d_layered`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_surface_2d_layered(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_surface_2d_layered", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_surface_3d`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_surface_3d(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_surface_3d", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_surface_cubemap`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_surface_cubemap(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_surface_cubemap", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_surface_cubemap_layered`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_surface_cubemap_layered(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_surface_cubemap_layered", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_texture_1d`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_texture_1d(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_texture_1d", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_texture_1d_layered`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_texture_1d_layered(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_texture_1d_layered", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_texture_1d_mipmap`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_texture_1d_mipmap(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_texture_1d_mipmap", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_texture_2d`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_texture_2d(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_texture_2d", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_texture_2d_gather`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_texture_2d_gather(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_texture_2d_gather", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_texture_2d_layered`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_texture_2d_layered(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_texture_2d_layered", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_texture_2d_linear`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_texture_2d_linear(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_texture_2d_linear", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_texture_2d_mipmap`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_texture_2d_mipmap(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_texture_2d_mipmap", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_texture_3d`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_texture_3d(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_texture_3d", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_texture_3d_alt`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_texture_3d_alt(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_texture_3d_alt", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_texture_cubemap`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_texture_cubemap(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_texture_cubemap", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_texture_cubemap_layered`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_texture_cubemap_layered(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_texture_cubemap_layered", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_threads_dim`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_threads_dim(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_threads_dim", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_threads_per_block`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_threads_per_block(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_threads_per_block", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_max_threads_per_multi_processor`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_max_threads_per_multi_processor(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_threads_per_multi_processor", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_mem_pitch`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_mem_pitch(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_mem_pitch", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_memory_bus_width`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_memory_bus_width(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_memory_bus_width", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_memory_pool_supported_handle_types`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_memory_pool_supported_handle_types(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_memory_pool_supported_handle_types", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_memory_pools_supported`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_memory_pools_supported(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_memory_pools_supported", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_minor`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_minor(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_minor", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_mps_enabled`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_mps_enabled(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_mps_enabled", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_multi_gpu_board_group_id`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_multi_gpu_board_group_id(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_multi_gpu_board_group_id", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_multi_processor_count`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_multi_processor_count(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_multi_processor_count", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_name`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_name(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_name", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_pageable_memory_access`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_pageable_memory_access(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_pageable_memory_access", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_pageable_memory_access_uses_host_page_tables`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_pageable_memory_access_uses_host_page_tables(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_pageable_memory_access_uses_host_page_tables", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_pci_bus_id`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_pci_bus_id(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_pci_bus_id", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_pci_device_id`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_pci_device_id(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_pci_device_id", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_pci_domain_id`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_pci_domain_id(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_pci_domain_id", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_persisting_l2cache_max_size`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_persisting_l2cache_max_size(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_persisting_l2cache_max_size", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_regs_per_block`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_regs_per_block(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_regs_per_block", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_regs_per_multiprocessor`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_regs_per_multiprocessor(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_regs_per_multiprocessor", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_reserved_shared_mem_per_block`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_reserved_shared_mem_per_block(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_reserved_shared_mem_per_block", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_shared_mem_per_block`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_shared_mem_per_block(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_shared_mem_per_block", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_shared_mem_per_block_optin`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_shared_mem_per_block_optin(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_shared_mem_per_block_optin", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_shared_mem_per_multiprocessor`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_shared_mem_per_multiprocessor(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_shared_mem_per_multiprocessor", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_sparse_cuda_array_supported`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_sparse_cuda_array_supported(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_sparse_cuda_array_supported", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_stream_priorities_supported`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_stream_priorities_supported(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_stream_priorities_supported", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_surface_alignment`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_surface_alignment(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_surface_alignment", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_tcc_driver`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_tcc_driver(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_tcc_driver", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_texture_alignment`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_texture_alignment(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_texture_alignment", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_texture_pitch_alignment`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_texture_pitch_alignment(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_texture_pitch_alignment", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_timeline_semaphore_interop_supported`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_timeline_semaphore_interop_supported(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_timeline_semaphore_interop_supported", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_total_const_mem`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_total_const_mem(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_total_const_mem", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_total_global_mem`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_total_global_mem(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_total_global_mem", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_unified_addressing`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_unified_addressing(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_unified_addressing", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_unified_function_pointers`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_unified_function_pointers(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_unified_function_pointers", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_uuid`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_uuid(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_uuid", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.device_prop.get_warp_size`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_device_prop_get_warp_size(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_warp_size", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.dim_3.get_x`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_dim_3_get_x(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.dim_3.get_x", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.dim_3.get_y`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_dim_3_get_y(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.dim_3.get_y", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.dim_3.get_z`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_dim_3_get_z(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.dim_3.get_z", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.event.create_with_flags`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_event_create_with_flags(ctx: *mlir.Context, flags: *const mlir.Value, cuda_result_type: *const mlir.Type, event_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.event.create_with_flags", .{ + .operands = .{ .flat = &.{flags} }, + .results = .{ .flat = &.{ cuda_result_type, event_type } }, + .location = location, + }); +} + +/// `cuda.runtime.event.query`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_event_query(ctx: *mlir.Context, event: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.event.query", .{ + .operands = .{ .flat = &.{event} }, + .results = .{ .flat = &.{cuda_result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.event.record_with_flags`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_event_record_with_flags(ctx: *mlir.Context, event: *const mlir.Value, stream: *const mlir.Value, flags: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.event.record_with_flags", .{ + .operands = .{ .flat = &.{ event, stream, flags } }, + .results = .{ .flat = &.{cuda_result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.event.synchronize`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_event_synchronize(ctx: *mlir.Context, event: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.event.synchronize", .{ + .operands = .{ .flat = &.{event} }, + .results = .{ .flat = &.{cuda_result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.extent.get_depth`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_extent_get_depth(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.extent.get_depth", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.extent.get_height`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_extent_get_height(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.extent.get_height", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.extent.get_width`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_extent_get_width(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.extent.get_width", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.graph.add_kernel_node`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_graph_add_kernel_node(ctx: *mlir.Context, graph: *const mlir.Value, pDependencies: *const mlir.Value, numDependencies: *const mlir.Value, pNodeParams: *const mlir.Value, cuda_result_type: *const mlir.Type, pGraphNode_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.graph.add_kernel_node", .{ + .operands = .{ .flat = &.{ graph, pDependencies, numDependencies, pNodeParams } }, + .results = .{ .flat = &.{ cuda_result_type, pGraphNode_type } }, + .location = location, + }); +} + +/// `cuda.runtime.graph.create`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_graph_create(ctx: *mlir.Context, flags: *const mlir.Value, cuda_result_type: *const mlir.Type, pGraph_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.graph.create", .{ + .operands = .{ .flat = &.{flags} }, + .results = .{ .flat = &.{ cuda_result_type, pGraph_type } }, + .location = location, + }); +} + +/// `cuda.runtime.graph.kernel_node_get_attribute`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_graph_kernel_node_get_attribute(ctx: *mlir.Context, hNode: *const mlir.Value, attr: *const mlir.Value, cuda_result_type: *const mlir.Type, value_out_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.graph.kernel_node_get_attribute", .{ + .operands = .{ .flat = &.{ hNode, attr } }, + .results = .{ .flat = &.{ cuda_result_type, value_out_type } }, + .location = location, + }); +} + +/// `cuda.runtime.graph.kernel_node_set_attribute`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_graph_kernel_node_set_attribute(ctx: *mlir.Context, hNode: *const mlir.Value, attr: *const mlir.Value, value_: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.graph.kernel_node_set_attribute", .{ + .operands = .{ .flat = &.{ hNode, attr, value_ } }, + .results = .{ .flat = &.{cuda_result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.kernel_node_params.get_block_dim`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_kernel_node_params_get_block_dim(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.kernel_node_params.get_block_dim", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.kernel_node_params.get_extra`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_kernel_node_params_get_extra(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.kernel_node_params.get_extra", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.kernel_node_params.get_func`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_kernel_node_params_get_func(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.kernel_node_params.get_func", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.kernel_node_params.get_grid_dim`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_kernel_node_params_get_grid_dim(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.kernel_node_params.get_grid_dim", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.kernel_node_params.get_kernel_params`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_kernel_node_params_get_kernel_params(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.kernel_node_params.get_kernel_params", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.kernel_node_params.get_shared_mem_bytes`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_kernel_node_params_get_shared_mem_bytes(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.kernel_node_params.get_shared_mem_bytes", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.library.get_global`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_library_get_global(ctx: *mlir.Context, library: *const mlir.Value, name_arg: *const mlir.Value, cuda_result_type: *const mlir.Type, dptr_type: *const mlir.Type, bytes_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.library.get_global", .{ + .operands = .{ .flat = &.{ library, name_arg } }, + .results = .{ .flat = &.{ cuda_result_type, dptr_type, bytes_type } }, + .location = location, + }); +} + +/// `cuda.runtime.library.get_kernel`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_library_get_kernel(ctx: *mlir.Context, library: *const mlir.Value, name_arg: *const mlir.Value, cuda_result_type: *const mlir.Type, pKernel_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.library.get_kernel", .{ + .operands = .{ .flat = &.{ library, name_arg } }, + .results = .{ .flat = &.{ cuda_result_type, pKernel_type } }, + .location = location, + }); +} + +/// `cuda.runtime.library.get_kernel_count`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_library_get_kernel_count(ctx: *mlir.Context, lib: *const mlir.Value, cuda_result_type: *const mlir.Type, count_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.library.get_kernel_count", .{ + .operands = .{ .flat = &.{lib} }, + .results = .{ .flat = &.{ cuda_result_type, count_type } }, + .location = location, + }); +} + +/// `cuda.runtime.library.get_managed`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_library_get_managed(ctx: *mlir.Context, library: *const mlir.Value, name_arg: *const mlir.Value, cuda_result_type: *const mlir.Type, dptr_type: *const mlir.Type, bytes_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.library.get_managed", .{ + .operands = .{ .flat = &.{ library, name_arg } }, + .results = .{ .flat = &.{ cuda_result_type, dptr_type, bytes_type } }, + .location = location, + }); +} + +/// `cuda.runtime.library.get_unified_function`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_library_get_unified_function(ctx: *mlir.Context, library: *const mlir.Value, symbol: *const mlir.Value, cuda_result_type: *const mlir.Type, fptr_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.library.get_unified_function", .{ + .operands = .{ .flat = &.{ library, symbol } }, + .results = .{ .flat = &.{ cuda_result_type, fptr_type } }, + .location = location, + }); +} + +/// `cuda.runtime.library.load_data`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_library_load_data(ctx: *mlir.Context, code_arg: *const mlir.Value, jitOptions: *const mlir.Value, jitOptionsValues: *const mlir.Value, numJitOptions: *const mlir.Value, libraryOptions: *const mlir.Value, libraryOptionValues: *const mlir.Value, numLibraryOptions: *const mlir.Value, cuda_result_type: *const mlir.Type, library_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.library.load_data", .{ + .operands = .{ .flat = &.{ code_arg, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions } }, + .results = .{ .flat = &.{ cuda_result_type, library_type } }, + .location = location, + }); +} + +/// `cuda.runtime.library.load_from_file`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_library_load_from_file(ctx: *mlir.Context, fileName: *const mlir.Value, jitOptions: *const mlir.Value, jitOptionsValues: *const mlir.Value, numJitOptions: *const mlir.Value, libraryOptions: *const mlir.Value, libraryOptionValues: *const mlir.Value, numLibraryOptions: *const mlir.Value, cuda_result_type: *const mlir.Type, library_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.library.load_from_file", .{ + .operands = .{ .flat = &.{ fileName, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions } }, + .results = .{ .flat = &.{ cuda_result_type, library_type } }, + .location = location, + }); +} + +/// `cuda.runtime.library.unload`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_library_unload(ctx: *mlir.Context, library: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.library.unload", .{ + .operands = .{ .flat = &.{library} }, + .results = .{ .flat = &.{cuda_result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.memcpy_3d_params.get_dst_array`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_memcpy_3d_params_get_dst_array(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.memcpy_3d_params.get_dst_array", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.memcpy_3d_params.get_dst_pos`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_memcpy_3d_params_get_dst_pos(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.memcpy_3d_params.get_dst_pos", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.memcpy_3d_params.get_dst_ptr`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_memcpy_3d_params_get_dst_ptr(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.memcpy_3d_params.get_dst_ptr", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.memcpy_3d_params.get_extent`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_memcpy_3d_params_get_extent(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.memcpy_3d_params.get_extent", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.memcpy_3d_params.get_kind`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_memcpy_3d_params_get_kind(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.memcpy_3d_params.get_kind", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.memcpy_3d_params.get_src_array`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_memcpy_3d_params_get_src_array(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.memcpy_3d_params.get_src_array", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.memcpy_3d_params.get_src_pos`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_memcpy_3d_params_get_src_pos(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.memcpy_3d_params.get_src_pos", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.memcpy_3d_params.get_src_ptr`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_memcpy_3d_params_get_src_ptr(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.memcpy_3d_params.get_src_ptr", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.memory.cpy3d`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_memory_cpy3d(ctx: *mlir.Context, p: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.memory.cpy3d", .{ + .operands = .{ .flat = &.{p} }, + .results = .{ .flat = &.{cuda_result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.memory.cpy3d_async`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_memory_cpy3d_async(ctx: *mlir.Context, p: *const mlir.Value, stream: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.memory.cpy3d_async", .{ + .operands = .{ .flat = &.{ p, stream } }, + .results = .{ .flat = &.{cuda_result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.pitched_ptr.get_pitch`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_pitched_ptr_get_pitch(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.pitched_ptr.get_pitch", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.pitched_ptr.get_ptr`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_pitched_ptr_get_ptr(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.pitched_ptr.get_ptr", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.pitched_ptr.get_xsize`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_pitched_ptr_get_xsize(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.pitched_ptr.get_xsize", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.pitched_ptr.get_ysize`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_pitched_ptr_get_ysize(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.pitched_ptr.get_ysize", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.pos.get_x`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_pos_get_x(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.pos.get_x", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.pos.get_y`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_pos_get_y(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.pos.get_y", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.pos.get_z`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_pos_get_z(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.pos.get_z", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.runtime.uuid.get_bytes`. Result types are explicit; attributes use mlir.Attribute. +pub fn runtime_uuid_get_bytes(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.runtime.uuid.get_bytes", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.stream.add_callback`. Result types are explicit; attributes use mlir.Attribute. +pub fn stream_add_callback(ctx: *mlir.Context, stream: *const mlir.Value, user_data: *const mlir.Value, result_type: *const mlir.Type, callback: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.stream.add_callback", .{ + .operands = .{ .flat = &.{ stream, user_data } }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "callback", callback), + }, + .location = location, + }); +} + +/// `cuda.stream.begin_capture`. Result types are explicit; attributes use mlir.Attribute. +pub fn stream_begin_capture(ctx: *mlir.Context, stream: *const mlir.Value, result_type: *const mlir.Type, mode: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.stream.begin_capture", .{ + .operands = .{ .flat = &.{stream} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "mode", mode), + }, + .location = location, + }); +} + +/// `cuda.stream.create`. Result types are explicit; attributes use mlir.Attribute. +pub fn stream_create(ctx: *mlir.Context, cuda_result_type: *const mlir.Type, stream_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.stream.create", .{ + .results = .{ .flat = &.{ cuda_result_type, stream_type } }, + .location = location, + }); +} + +/// `cuda.stream.destroy`. Result types are explicit; attributes use mlir.Attribute. +pub fn stream_destroy(ctx: *mlir.Context, stream: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.stream.destroy", .{ + .operands = .{ .flat = &.{stream} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.stream.end_capture`. Result types are explicit; attributes use mlir.Attribute. +pub fn stream_end_capture(ctx: *mlir.Context, stream: *const mlir.Value, result_type: *const mlir.Type, graph_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.stream.end_capture", .{ + .operands = .{ .flat = &.{stream} }, + .results = .{ .flat = &.{ result_type, graph_type } }, + .location = location, + }); +} + +/// `cuda.stream.synchronize`. Result types are explicit; attributes use mlir.Attribute. +pub fn stream_synchronize( + ctx: *mlir.Context, + stream: *const mlir.Value, + result_type: *const mlir.Type, + opts: struct { + event: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.stream.synchronize", .{ + .operands = .{ .flat = if (opts.event) |value| &.{ stream, value } else &.{stream} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cuda.tensor_map_encode_tiled`. Result types are explicit; attributes use mlir.Attribute. +pub fn tensor_map_encode_tiled(ctx: *mlir.Context, tensorDataType: *const mlir.Value, tensorRank: *const mlir.Value, globalAddress: *const mlir.Value, globalDims: *const mlir.Value, globalStrides: *const mlir.Value, boxDims: *const mlir.Value, elementStrides: *const mlir.Value, interleave: *const mlir.Value, swizzle: *const mlir.Value, l2Promotion: *const mlir.Value, oobFill: *const mlir.Value, cuda_result_type: *const mlir.Type, cuTensorMap_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cuda.tensor_map_encode_tiled", .{ + .operands = .{ .flat = &.{ tensorDataType, tensorRank, globalAddress, globalDims, globalStrides, boxDims, elementStrides, interleave, swizzle, l2Promotion, oobFill } }, + .results = .{ .flat = &.{ cuda_result_type, cuTensorMap_type } }, + .location = location, + }); +} diff --git a/mlir/dialects/cuda/test.zig b/mlir/dialects/cuda/test.zig new file mode 100644 index 00000000..b17f50cd --- /dev/null +++ b/mlir/dialects/cuda/test.zig @@ -0,0 +1,324 @@ +const std = @import("std"); + +const cuda = @import("mlir/dialects/cuda"); +const mlir = @import("mlir"); + +const operation_names: []const []const u8 = &.{ + "cuda.addressof", + "cuda.binary", + "cuda.cast", + "cuda.compose", + "cuda.decompose", + "cuda.device.sync", + "cuda.driver.event.create", + "cuda.driver.event.destroy", + "cuda.driver.event.elapsed_time", + "cuda.driver.event.query", + "cuda.driver.event.record", + "cuda.driver.event.record_with_flags", + "cuda.driver.event.synchronize", + "cuda.driver.library.get_global", + "cuda.driver.library.get_kernel", + "cuda.driver.library.get_kernel_count", + "cuda.driver.library.get_managed", + "cuda.driver.library.get_module", + "cuda.driver.library.get_unified_function", + "cuda.driver.library.load_data", + "cuda.driver.library.load_from_file", + "cuda.driver.library.unload", + "cuda.driver.tensor_map.encode_im2col", + "cuda.driver.tensor_map.encode_im2col_wide", + "cuda.driver.tensor_map.replace_address", + "cuda.event.create", + "cuda.event.destroy", + "cuda.event.elapsed_time", + "cuda.event.record", + "cuda.func", + "cuda.func.get_name", + "cuda.graph.destroy", + "cuda.graph.instantiate", + "cuda.graph.launch", + "cuda.graph_exec.destroy", + "cuda.kernel", + "cuda.launch", + "cuda.launch_cfg.access_policy_window", + "cuda.launch_cfg.cluster_dim", + "cuda.launch_cfg.cluster_scheduling_policy", + "cuda.launch_cfg.cooperative", + "cuda.launch_cfg.create", + "cuda.launch_cfg.create_from_stream", + "cuda.launch_cfg.device_updatable_kernel_node", + "cuda.launch_cfg.grid_dimensions", + "cuda.launch_cfg.launch_completion_event", + "cuda.launch_cfg.mem_sync_domain", + "cuda.launch_cfg.mem_sync_domain_map", + "cuda.launch_cfg.nvlink_util_centric_scheduling", + "cuda.launch_cfg.preferred_cluster_dim", + "cuda.launch_cfg.print", + "cuda.launch_cfg.priority", + "cuda.launch_cfg.programmatic_event", + "cuda.launch_cfg.programmatic_stream_serialization_allowed", + "cuda.launch_cfg.shared_mem_carveout", + "cuda.launch_cfg.sync_policy", + "cuda.launch_ex", + "cuda.launch_kernel", + "cuda.library.get_kernel", + "cuda.library.load_data", + "cuda.library.unload", + "cuda.memory.alloc", + "cuda.memory.alloc_async", + "cuda.memory.alloc_host", + "cuda.memory.alloc_managed", + "cuda.memory.copy", + "cuda.memory.copy_async", + "cuda.memory.free", + "cuda.memory.free_async", + "cuda.memory.free_host", + "cuda.memory.set", + "cuda.memory.set_async", + "cuda.occupancy_max_active_blocks_per_multiprocessor", + "cuda.profiler.start", + "cuda.profiler.stop", + "cuda.result.assert_success", + "cuda.result.failed", + "cuda.result.print_on_error", + "cuda.result.succeeded", + "cuda.return", + "cuda.return_if_error", + "cuda.runtime.device.properties", + "cuda.runtime.device_prop.get_access_policy_max_window_size", + "cuda.runtime.device_prop.get_async_engine_count", + "cuda.runtime.device_prop.get_can_map_host_memory", + "cuda.runtime.device_prop.get_can_use_host_pointer_for_registered_mem", + "cuda.runtime.device_prop.get_cluster_launch", + "cuda.runtime.device_prop.get_compute_preemption_supported", + "cuda.runtime.device_prop.get_concurrent_kernels", + "cuda.runtime.device_prop.get_concurrent_managed_access", + "cuda.runtime.device_prop.get_cooperative_launch", + "cuda.runtime.device_prop.get_deferred_mapping_cuda_array_supported", + "cuda.runtime.device_prop.get_device_numa_config", + "cuda.runtime.device_prop.get_device_numa_id", + "cuda.runtime.device_prop.get_direct_managed_mem_access_from_host", + "cuda.runtime.device_prop.get_ecc_enabled", + "cuda.runtime.device_prop.get_global_l1cache_supported", + "cuda.runtime.device_prop.get_gpu_direct_rdma_flush_writes_options", + "cuda.runtime.device_prop.get_gpu_direct_rdma_supported", + "cuda.runtime.device_prop.get_gpu_direct_rdma_writes_ordering", + "cuda.runtime.device_prop.get_gpu_pci_device_id", + "cuda.runtime.device_prop.get_gpu_pci_subsystem_id", + "cuda.runtime.device_prop.get_host_native_atomic_supported", + "cuda.runtime.device_prop.get_host_numa_id", + "cuda.runtime.device_prop.get_host_numa_multinode_ipc_supported", + "cuda.runtime.device_prop.get_host_register_read_only_supported", + "cuda.runtime.device_prop.get_host_register_supported", + "cuda.runtime.device_prop.get_integrated", + "cuda.runtime.device_prop.get_ipc_event_supported", + "cuda.runtime.device_prop.get_is_multi_gpu_board", + "cuda.runtime.device_prop.get_l_2cache_size", + "cuda.runtime.device_prop.get_local_l1cache_supported", + "cuda.runtime.device_prop.get_luid", + "cuda.runtime.device_prop.get_luid_device_node_mask", + "cuda.runtime.device_prop.get_major", + "cuda.runtime.device_prop.get_managed_memory", + "cuda.runtime.device_prop.get_max_blocks_per_multi_processor", + "cuda.runtime.device_prop.get_max_grid_size", + "cuda.runtime.device_prop.get_max_surface_1d", + "cuda.runtime.device_prop.get_max_surface_1d_layered", + "cuda.runtime.device_prop.get_max_surface_2d", + "cuda.runtime.device_prop.get_max_surface_2d_layered", + "cuda.runtime.device_prop.get_max_surface_3d", + "cuda.runtime.device_prop.get_max_surface_cubemap", + "cuda.runtime.device_prop.get_max_surface_cubemap_layered", + "cuda.runtime.device_prop.get_max_texture_1d", + "cuda.runtime.device_prop.get_max_texture_1d_layered", + "cuda.runtime.device_prop.get_max_texture_1d_mipmap", + "cuda.runtime.device_prop.get_max_texture_2d", + "cuda.runtime.device_prop.get_max_texture_2d_gather", + "cuda.runtime.device_prop.get_max_texture_2d_layered", + "cuda.runtime.device_prop.get_max_texture_2d_linear", + "cuda.runtime.device_prop.get_max_texture_2d_mipmap", + "cuda.runtime.device_prop.get_max_texture_3d", + "cuda.runtime.device_prop.get_max_texture_3d_alt", + "cuda.runtime.device_prop.get_max_texture_cubemap", + "cuda.runtime.device_prop.get_max_texture_cubemap_layered", + "cuda.runtime.device_prop.get_max_threads_dim", + "cuda.runtime.device_prop.get_max_threads_per_block", + "cuda.runtime.device_prop.get_max_threads_per_multi_processor", + "cuda.runtime.device_prop.get_mem_pitch", + "cuda.runtime.device_prop.get_memory_bus_width", + "cuda.runtime.device_prop.get_memory_pool_supported_handle_types", + "cuda.runtime.device_prop.get_memory_pools_supported", + "cuda.runtime.device_prop.get_minor", + "cuda.runtime.device_prop.get_mps_enabled", + "cuda.runtime.device_prop.get_multi_gpu_board_group_id", + "cuda.runtime.device_prop.get_multi_processor_count", + "cuda.runtime.device_prop.get_name", + "cuda.runtime.device_prop.get_pageable_memory_access", + "cuda.runtime.device_prop.get_pageable_memory_access_uses_host_page_tables", + "cuda.runtime.device_prop.get_pci_bus_id", + "cuda.runtime.device_prop.get_pci_device_id", + "cuda.runtime.device_prop.get_pci_domain_id", + "cuda.runtime.device_prop.get_persisting_l2cache_max_size", + "cuda.runtime.device_prop.get_regs_per_block", + "cuda.runtime.device_prop.get_regs_per_multiprocessor", + "cuda.runtime.device_prop.get_reserved_shared_mem_per_block", + "cuda.runtime.device_prop.get_shared_mem_per_block", + "cuda.runtime.device_prop.get_shared_mem_per_block_optin", + "cuda.runtime.device_prop.get_shared_mem_per_multiprocessor", + "cuda.runtime.device_prop.get_sparse_cuda_array_supported", + "cuda.runtime.device_prop.get_stream_priorities_supported", + "cuda.runtime.device_prop.get_surface_alignment", + "cuda.runtime.device_prop.get_tcc_driver", + "cuda.runtime.device_prop.get_texture_alignment", + "cuda.runtime.device_prop.get_texture_pitch_alignment", + "cuda.runtime.device_prop.get_timeline_semaphore_interop_supported", + "cuda.runtime.device_prop.get_total_const_mem", + "cuda.runtime.device_prop.get_total_global_mem", + "cuda.runtime.device_prop.get_unified_addressing", + "cuda.runtime.device_prop.get_unified_function_pointers", + "cuda.runtime.device_prop.get_uuid", + "cuda.runtime.device_prop.get_warp_size", + "cuda.runtime.dim_3.get_x", + "cuda.runtime.dim_3.get_y", + "cuda.runtime.dim_3.get_z", + "cuda.runtime.event.create_with_flags", + "cuda.runtime.event.query", + "cuda.runtime.event.record_with_flags", + "cuda.runtime.event.synchronize", + "cuda.runtime.extent.get_depth", + "cuda.runtime.extent.get_height", + "cuda.runtime.extent.get_width", + "cuda.runtime.graph.add_kernel_node", + "cuda.runtime.graph.create", + "cuda.runtime.graph.kernel_node_get_attribute", + "cuda.runtime.graph.kernel_node_set_attribute", + "cuda.runtime.kernel_node_params.get_block_dim", + "cuda.runtime.kernel_node_params.get_extra", + "cuda.runtime.kernel_node_params.get_func", + "cuda.runtime.kernel_node_params.get_grid_dim", + "cuda.runtime.kernel_node_params.get_kernel_params", + "cuda.runtime.kernel_node_params.get_shared_mem_bytes", + "cuda.runtime.library.get_global", + "cuda.runtime.library.get_kernel", + "cuda.runtime.library.get_kernel_count", + "cuda.runtime.library.get_managed", + "cuda.runtime.library.get_unified_function", + "cuda.runtime.library.load_data", + "cuda.runtime.library.load_from_file", + "cuda.runtime.library.unload", + "cuda.runtime.memcpy_3d_params.get_dst_array", + "cuda.runtime.memcpy_3d_params.get_dst_pos", + "cuda.runtime.memcpy_3d_params.get_dst_ptr", + "cuda.runtime.memcpy_3d_params.get_extent", + "cuda.runtime.memcpy_3d_params.get_kind", + "cuda.runtime.memcpy_3d_params.get_src_array", + "cuda.runtime.memcpy_3d_params.get_src_pos", + "cuda.runtime.memcpy_3d_params.get_src_ptr", + "cuda.runtime.memory.cpy3d", + "cuda.runtime.memory.cpy3d_async", + "cuda.runtime.pitched_ptr.get_pitch", + "cuda.runtime.pitched_ptr.get_ptr", + "cuda.runtime.pitched_ptr.get_xsize", + "cuda.runtime.pitched_ptr.get_ysize", + "cuda.runtime.pos.get_x", + "cuda.runtime.pos.get_y", + "cuda.runtime.pos.get_z", + "cuda.runtime.uuid.get_bytes", + "cuda.stream.add_callback", + "cuda.stream.begin_capture", + "cuda.stream.create", + "cuda.stream.destroy", + "cuda.stream.end_capture", + "cuda.stream.synchronize", + "cuda.tensor_map_encode_tiled", +}; + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(cuda); + std.testing.refAllDecls(cuda.ops); + inline for (comptime std.meta.declarations(cuda.ops)) |decl| { + const T = @field(cuda.ops, decl); + if (@TypeOf(T) == type) std.testing.refAllDecls(T); + } +} + +fn testContext() !*mlir.Context { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + cuda.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + return ctx; +} + +test "every bound operation is registered" { + const ctx = try testContext(); + defer ctx.deinit(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} + +test "cuda types and attributes print as the compiler does" { + @setEvalBranchQuota(100_000); + const ctx = try testContext(); + defer ctx.deinit(); + // The compiler's printed forms, which these must print byte for byte. + inline for (.{ + .{ cuda.ops.AssumeKernelAttr, "#cuda.assume_kernel_attr" }, + .{ cuda.ops.ComputeTargetAttr, "#cuda.compute_target" }, + .{ cuda.ops.DevMaxSharedMemoryOptinAttr, "#cuda.dev_max_shared_memory_optin" }, + .{ cuda.ops.ExecutableAttr, "#cuda.executable" }, + .{ cuda.ops.LaunchConfigType, "!cuda.launch_cfg" }, + .{ cuda.ops.StreamType, "!cuda.stream" }, + .{ cuda.ops.ResultType, "!cuda.result" }, + .{ cuda.ops.IntType, "!cuda.int" }, + .{ cuda.ops.RuntimeDim3Type, "!cuda.runtime.dim_3" }, + .{ cuda.ops.RuntimeLaunchAttributeValueSyncPolicyType, "!cuda.runtime.launch_attribute_value.sync_policy" }, + .{ cuda.ops.RuntimeLaunchAttributeValueClusterSchedulingPolicyPreferenceType, "!cuda.runtime.launch_attribute_value.cluster_scheduling_policy_preference" }, + .{ cuda.ops.RuntimeLaunchAttributeValueNvlinkUtilCentricSchedulingType, "!cuda.runtime.launch_attribute_value.nvlink_util_centric_scheduling" }, + .{ cuda.ops.RuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowedType, "!cuda.runtime.launch_attribute_value.programmatic_stream_serialization_allowed" }, + }) |example| { + const T = example[0]; + const is_type = @hasDecl(T, "type_"); + const parsed = if (is_type) try mlir.Type.parse(ctx, example[1]) else try mlir.Attribute.parse(ctx, example[1]); + const value = parsed.isA(T) orelse return error.TestUnexpectedResult; + try std.testing.expect((try rebuildEnums(T, ctx, value)).eql(value)); + var buf: [512]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print("{f}", .{value}); + try std.testing.expectEqualStrings(example[1], w.buffered()); + } + // The compiler prints these without the space before the dictionary; it + // parses both. + inline for (.{ + .{ cuda.ops.DeviceAttributesAttr, "#cuda" }, + .{ cuda.ops.FuncAttributesAttr, "#cuda" }, + }) |example| { + const parsed = try mlir.Attribute.parse(ctx, example[1]); + const value = parsed.isA(example[0]) orelse return error.TestUnexpectedResult; + try std.testing.expect((try rebuildEnums(example[0], ctx, value)).eql(value)); + } +} + +/// `rebuild` for parameters that are slices of enums. +fn rebuildEnums(comptime T: type, ctx: *mlir.Context, value: *const T) !*const T { + var args: T.InitArgs = undefined; + var buffers: [4][16]u32 = undefined; + inline for (@typeInfo(T.InitArgs).@"struct".field_names, 0..) |field, i| { + const name = [_]u8{comptime std.ascii.toUpper(field[0])} ++ field[1..]; + const noun = [_]u8{comptime std.ascii.toLower(field[0])} ++ field[1..]; + const accessor = if (@hasDecl(T, noun ++ "_")) noun ++ "_" else noun; + const info = @typeInfo(@FieldType(T.InitArgs, field)); + if (info == .pointer and info.pointer.size == .slice and @typeInfo(info.pointer.child) == .@"enum") { + const E = info.pointer.child; + const count = @field(T, "num" ++ name)(value); + const elements: []E = @ptrCast(buffers[i][0..count]); + for (elements, 0..) |*element, pos| element.* = @field(T, noun[0 .. noun.len - 1])(value, pos); + @field(args, field) = elements; + } else { + @field(args, field) = @field(T, accessor)(value); + } + } + return T.get(ctx, args); +} diff --git a/mlir/dialects/cuda_tile/BUILD.bazel b/mlir/dialects/cuda_tile/BUILD.bazel new file mode 100644 index 00000000..92d622cf --- /dev/null +++ b/mlir/dialects/cuda_tile/BUILD.bazel @@ -0,0 +1,38 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +# The TileView type interface, which NVIDIA's C API does not expose. +cc_library( + name = "capi", + srcs = ["capi.cc"], + hdrs = ["capi.h"], + tags = ["manual"], + deps = [ + "@cuda_tile//:CudaTileDialect", + "@llvm-project//mlir:CAPIIR", + ], +) + +zig_library( + name = "cuda_tile", + import_name = "mlir/dialects/cuda_tile", + main = "cuda_tile.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + ":capi", + "//mlir", + "//stdx", + "@cuda_tile//:cuda_tile_capi", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":capi", + ":cuda_tile", + "//mlir", + ], +) diff --git a/mlir/dialects/cuda_tile/capi.cc b/mlir/dialects/cuda_tile/capi.cc new file mode 100644 index 00000000..879e3757 --- /dev/null +++ b/mlir/dialects/cuda_tile/capi.cc @@ -0,0 +1,204 @@ +//===- capi.cc - C API for the cuda_tile dialect --------------------------===// + +#include "capi.h" + +#include "cuda_tile/Dialect/CudaTile/IR/Attributes.h" +#include "cuda_tile/Dialect/CudaTile/IR/Dialect.h" +#include "cuda_tile/Dialect/CudaTile/IR/Types.h" + +#include "mlir/CAPI/IR.h" +#include "mlir/IR/BuiltinAttributes.h" +#include "mlir/IR/Diagnostics.h" + +#include + +using namespace mlir::cuda_tile; + +static mlir::MLIRContext *loadCudaTile(MlirContext context) { + mlir::MLIRContext *ctx = unwrap(context); + if (ctx) + ctx->getOrLoadDialect(); + return ctx; +} + +static std::optional optional(bool has, int64_t value) { + return has ? std::optional(value) : std::nullopt; +} + +//===----------------------------------------------------------------------===// +// TileView +//===----------------------------------------------------------------------===// + +bool mlirTypeIsACudaTileTileView(MlirType type) { + return llvm::isa(unwrap(type)); +} + +MlirType mlirCudaTileTileViewTypeGetViewTileType(MlirType type) { + return wrap(llvm::cast(unwrap(type)).getViewTileType()); +} + +intptr_t mlirCudaTileTileViewTypeGetViewIndexRank(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getViewIndexRank()); +} + +//===----------------------------------------------------------------------===// +// GatherScatterViewType +//===----------------------------------------------------------------------===// + +bool mlirTypeIsACudaTileGatherScatterView(MlirType type) { + return llvm::isa(unwrap(type)); +} + +MlirType mlirCudaTileGatherScatterViewTypeGet( + MlirContext context, intptr_t rank, const int32_t *tileShape, + MlirType tensorView, uint32_t sparseDim, MlirAttribute paddingValue) { + mlir::MLIRContext *ctx = loadCudaTile(context); + if (!ctx) + return {nullptr}; + auto view = llvm::dyn_cast_or_null(unwrap(tensorView)); + if (!view) + return {nullptr}; + PaddingValueAttr padding; + if (!mlirAttributeIsNull(paddingValue)) { + padding = llvm::dyn_cast(unwrap(paddingValue)); + if (!padding) + return {nullptr}; + } + auto shape = mlir::DenseI32ArrayAttr::get( + ctx, llvm::ArrayRef(tileShape, rank)); + return wrap(GatherScatterViewType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, shape, + view, sparseDim, padding)); +} + +MlirAttribute mlirCudaTileGatherScatterViewTypeGetTileShape(MlirType type) { + return wrap(llvm::cast(unwrap(type)).getTileShape()); +} + +MlirType mlirCudaTileGatherScatterViewTypeGetTensorView(MlirType type) { + return wrap(llvm::cast(unwrap(type)).getTensorView()); +} + +uint32_t mlirCudaTileGatherScatterViewTypeGetSparseDim(MlirType type) { + return llvm::cast(unwrap(type)).getSparseDim(); +} + +MlirAttribute mlirCudaTileGatherScatterViewTypeGetPaddingValue(MlirType type) { + return wrap( + llvm::cast(unwrap(type)).getPaddingValue()); +} + +//===----------------------------------------------------------------------===// +// OptimizationHintsAttr +//===----------------------------------------------------------------------===// + +MlirAttribute mlirCudaTileOptimizationHintsAttrGet(MlirContext context, + MlirAttribute value) { + mlir::MLIRContext *ctx = loadCudaTile(context); + if (!ctx) + return {nullptr}; + auto dict = llvm::dyn_cast_or_null(unwrap(value)); + if (!dict) + return {nullptr}; + return wrap(OptimizationHintsAttr::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, dict)); +} + +MlirAttribute mlirCudaTileOptimizationHintsAttrGetValue(MlirAttribute attr) { + return wrap(llvm::cast(unwrap(attr)).getValue()); +} + +//===----------------------------------------------------------------------===// +// DivByAttr +//===----------------------------------------------------------------------===// + +bool mlirAttributeIsACudaTileDivBy(MlirAttribute attr) { + return llvm::isa(unwrap(attr)); +} + +MlirAttribute mlirCudaTileDivByAttrGet(MlirContext context, uint64_t divisor, + bool hasEvery, int64_t every, + bool hasAlong, int64_t along) { + mlir::MLIRContext *ctx = loadCudaTile(context); + // The printer writes `every` and `along` together. + if (!ctx || hasEvery != hasAlong) + return {nullptr}; + return wrap(DivByAttr::get(ctx, divisor, optional(hasEvery, every), + optional(hasAlong, along))); +} + +uint64_t mlirCudaTileDivByAttrGetDivisor(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getDivisor(); +} + +bool mlirCudaTileDivByAttrHasEvery(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getEvery().has_value(); +} + +int64_t mlirCudaTileDivByAttrGetEvery(MlirAttribute attr) { + return *llvm::cast(unwrap(attr)).getEvery(); +} + +bool mlirCudaTileDivByAttrHasAlong(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getAlong().has_value(); +} + +int64_t mlirCudaTileDivByAttrGetAlong(MlirAttribute attr) { + return *llvm::cast(unwrap(attr)).getAlong(); +} + +//===----------------------------------------------------------------------===// +// SameElementsAttr +//===----------------------------------------------------------------------===// + +bool mlirAttributeIsACudaTileSameElements(MlirAttribute attr) { + return llvm::isa(unwrap(attr)); +} + +MlirAttribute mlirCudaTileSameElementsAttrGet(MlirContext context, + intptr_t rank, + const int64_t *values) { + mlir::MLIRContext *ctx = loadCudaTile(context); + if (!ctx) + return {nullptr}; + return wrap(SameElementsAttr::get( + ctx, mlir::DenseI64ArrayAttr::get( + ctx, llvm::ArrayRef(values, rank)))); +} + +MlirAttribute mlirCudaTileSameElementsAttrGetValues(MlirAttribute attr) { + return wrap(llvm::cast(unwrap(attr)).getValues()); +} + +//===----------------------------------------------------------------------===// +// BoundedAttr +//===----------------------------------------------------------------------===// + +bool mlirAttributeIsACudaTileBounded(MlirAttribute attr) { + return llvm::isa(unwrap(attr)); +} + +MlirAttribute mlirCudaTileBoundedAttrGet(MlirContext context, bool hasLb, + int64_t lb, bool hasUb, int64_t ub) { + mlir::MLIRContext *ctx = loadCudaTile(context); + if (!ctx) + return {nullptr}; + return wrap(BoundedAttr::get(ctx, optional(hasLb, lb), optional(hasUb, ub))); +} + +bool mlirCudaTileBoundedAttrHasLb(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getLb().has_value(); +} + +int64_t mlirCudaTileBoundedAttrGetLb(MlirAttribute attr) { + return *llvm::cast(unwrap(attr)).getLb(); +} + +bool mlirCudaTileBoundedAttrHasUb(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getUb().has_value(); +} + +int64_t mlirCudaTileBoundedAttrGetUb(MlirAttribute attr) { + return *llvm::cast(unwrap(attr)).getUb(); +} diff --git a/mlir/dialects/cuda_tile/capi.h b/mlir/dialects/cuda_tile/capi.h new file mode 100644 index 00000000..6f299c3f --- /dev/null +++ b/mlir/dialects/cuda_tile/capi.h @@ -0,0 +1,84 @@ +//===- capi.h - C API for the cuda_tile dialect -------------------*- C -*-===// +// +// C entry points for what NVIDIA's cuda_tile C API leaves out: the +// `TileView` type interface, shared by the partition, strided and +// gather/scatter views, the gather/scatter view type, the general +// `optimization_hints` constructor and the `assume` predicates. + +#ifndef MLIR_DIALECTS_CUDA_TILE_CAPI_H +#define MLIR_DIALECTS_CUDA_TILE_CAPI_H + +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Constructors return null for invalid parameters; optional attribute +// parameters take null handles, and std::optional integers a `has` flag. +// Getters require a value accepted by the matching IsA. + +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaTileTileView(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCudaTileTileViewTypeGetViewTileType(MlirType type); +MLIR_CAPI_EXPORTED intptr_t +mlirCudaTileTileViewTypeGetViewIndexRank(MlirType type); + +// `!cuda_tile.gather_scatter_view`. `paddingValue` is a +// `#cuda_tile.padding_value` attribute, or null for none. +MLIR_CAPI_EXPORTED bool mlirTypeIsACudaTileGatherScatterView(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCudaTileGatherScatterViewTypeGet( + MlirContext context, intptr_t rank, const int32_t *tileShape, + MlirType tensorView, uint32_t sparseDim, MlirAttribute paddingValue); +// A DenseI32ArrayAttr. +MLIR_CAPI_EXPORTED MlirAttribute +mlirCudaTileGatherScatterViewTypeGetTileShape(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCudaTileGatherScatterViewTypeGetTensorView(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCudaTileGatherScatterViewTypeGetSparseDim(MlirType type); +// Null when the view has no padding value. +MLIR_CAPI_EXPORTED MlirAttribute +mlirCudaTileGatherScatterViewTypeGetPaddingValue(MlirType type); + +// `#cuda_tile.optimization_hints` over its dictionary of per-architecture +// dictionaries (IsA: NVIDIA's mlirCudaTileAttributeIsAOptimizationHintsAttr). +MLIR_CAPI_EXPORTED MlirAttribute +mlirCudaTileOptimizationHintsAttrGet(MlirContext context, MlirAttribute value); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCudaTileOptimizationHintsAttrGetValue(MlirAttribute attr); + +// `#cuda_tile.div_by`. `every` and `along` are both present or both absent. +MLIR_CAPI_EXPORTED bool mlirAttributeIsACudaTileDivBy(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCudaTileDivByAttrGet(MlirContext context, uint64_t divisor, bool hasEvery, + int64_t every, bool hasAlong, int64_t along); +MLIR_CAPI_EXPORTED uint64_t mlirCudaTileDivByAttrGetDivisor(MlirAttribute attr); +MLIR_CAPI_EXPORTED bool mlirCudaTileDivByAttrHasEvery(MlirAttribute attr); +MLIR_CAPI_EXPORTED int64_t mlirCudaTileDivByAttrGetEvery(MlirAttribute attr); +MLIR_CAPI_EXPORTED bool mlirCudaTileDivByAttrHasAlong(MlirAttribute attr); +MLIR_CAPI_EXPORTED int64_t mlirCudaTileDivByAttrGetAlong(MlirAttribute attr); + +// `#cuda_tile.same_elements`. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACudaTileSameElements(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCudaTileSameElementsAttrGet( + MlirContext context, intptr_t rank, const int64_t *values); +// A DenseI64ArrayAttr. +MLIR_CAPI_EXPORTED MlirAttribute +mlirCudaTileSameElementsAttrGetValues(MlirAttribute attr); + +// `#cuda_tile.bounded`, each bound optional. +MLIR_CAPI_EXPORTED bool mlirAttributeIsACudaTileBounded(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCudaTileBoundedAttrGet( + MlirContext context, bool hasLb, int64_t lb, bool hasUb, int64_t ub); +MLIR_CAPI_EXPORTED bool mlirCudaTileBoundedAttrHasLb(MlirAttribute attr); +MLIR_CAPI_EXPORTED int64_t mlirCudaTileBoundedAttrGetLb(MlirAttribute attr); +MLIR_CAPI_EXPORTED bool mlirCudaTileBoundedAttrHasUb(MlirAttribute attr); +MLIR_CAPI_EXPORTED int64_t mlirCudaTileBoundedAttrGetUb(MlirAttribute attr); + +#ifdef __cplusplus +} +#endif + +#endif // MLIR_DIALECTS_CUDA_TILE_CAPI_H diff --git a/mlir/dialects/cuda_tile/cuda_tile.zig b/mlir/dialects/cuda_tile/cuda_tile.zig new file mode 100644 index 00000000..ddf0a0d5 --- /dev/null +++ b/mlir/dialects/cuda_tile/cuda_tile.zig @@ -0,0 +1,1747 @@ +//! Zig bindings for NVIDIA's `cuda_tile` MLIR dialect (CUDA Tile IR): one +//! function per op in `Ops.td` order. Types and attributes go through the +//! dialect's C API, completed by `capi.h` where it has no getter. + +const std = @import("std"); + +const c = @import("c"); +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +const dialect_namespace = "cuda_tile"; + +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("cuda_tile"); +} + +fn stringRef(s: []const u8) c.MlirStringRef { + return .{ .data = s.ptr, .length = s.len }; +} + +fn typeFromC(t: c.MlirType) *const mlir.Type { + return @ptrCast(t.ptr orelse @panic("cuda_tile: null MlirType")); +} + +fn attrFromC(a: c.MlirAttribute) *const mlir.Attribute { + return @ptrCast(a.ptr orelse @panic("cuda_tile: null MlirAttribute")); +} + +const null_attr: c.MlirAttribute = .{ .ptr = null }; + +/// `!cuda_tile.ptr` — a typed global-memory pointer. +pub const PointerType = opaque { + const M = mlir.Methods(PointerType, c.MlirType); + + pub const isAFn = c.mlirCudaTileTypeIsAPointerType; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + + pub fn get(ctx: *mlir.Context, pointee_type: *const mlir.Type) *const PointerType { + return @ptrCast(typeFromC(c.mlirCudaTilePointerTypeGet(ctx.ptr(), pointee_type.ptr()))); + } + + pub fn pointee(self: *const PointerType) *const mlir.Type { + return typeFromC(c.mlirCudaTilePointerTypeGetPointeeType(self.ptr())); + } + + pub fn type_(self: *const PointerType) *const mlir.Type { + return @ptrCast(self); + } +}; + +pub fn pointerType(ctx: *mlir.Context, pointee_type: *const mlir.Type) *const mlir.Type { + return @ptrCast(PointerType.get(ctx, pointee_type)); +} + +/// `!cuda_tile.tile` — every SSA value that carries data. Rank 0 is +/// the scalar. Dimensions must be powers of two. +pub const TileType = opaque { + const M = mlir.Methods(TileType, c.MlirType); + + pub const isAFn = c.mlirCudaTileTypeIsATileType; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + shape: []const i64, + elem: *const mlir.Type, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) *const TileType { + return @ptrCast(typeFromC(c.mlirCudaTileTileTypeGet(ctx.ptr(), @intCast(args.shape.len), args.shape.ptr, args.elem.ptr()))); + } + + pub fn elementType(self: *const TileType) *const mlir.Type { + return typeFromC(c.mlirCudaTileTileTypeGetElementType(self.ptr())); + } + + pub fn rank(self: *const TileType) usize { + return @intCast(c.mlirCudaTileTileTypeGetRank(self.ptr())); + } + + pub fn dimension(self: *const TileType, i: usize) i64 { + return c.mlirCudaTileTileTypeGetDimSize(self.ptr(), @intCast(i)); + } + + pub fn type_(self: *const TileType) *const mlir.Type { + return @ptrCast(self); + } +}; + +pub fn tileType(ctx: *mlir.Context, shape: []const i64, elem: *const mlir.Type) *const mlir.Type { + return @ptrCast(TileType.get(ctx, .{ .shape = shape, .elem = elem })); +} + +/// `!cuda_tile.token` — the ordering token every `*_tko` op produces. +pub const TokenType = opaque { + const M = mlir.Methods(TokenType, c.MlirType); + + pub const isAFn = c.mlirCudaTileTypeIsATokenType; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + + pub fn get(ctx: *mlir.Context) *const TokenType { + return @ptrCast(typeFromC(c.mlirCudaTileTokenTypeGet(ctx.ptr()))); + } + + pub fn type_(self: *const TokenType) *const mlir.Type { + return @ptrCast(self); + } +}; + +pub fn tokenType(ctx: *mlir.Context) *const mlir.Type { + return @ptrCast(TokenType.get(ctx)); +} + +/// `!cuda_tile.tensor_view`. `dynamic()` marks a +/// dimension or stride that arrives as an operand of `make_tensor_view`. +pub const TensorViewType = opaque { + const M = mlir.Methods(TensorViewType, c.MlirType); + + pub const isAFn = c.mlirCudaTileTypeIsATensorViewType; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + + pub fn dynamic() i64 { + return c.mlirCudaTileTensorViewTypeGetDynamicSize(); + } + + pub const InitArgs = struct { + elem: *const mlir.Type, + shape: []const i64, + strides: []const i64, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) *const TensorViewType { + const shape = args.shape; + const strides = args.strides; + return @ptrCast(typeFromC(c.mlirCudaTileTensorViewTypeGet( + ctx.ptr(), + args.elem.ptr(), + @intCast(shape.len), + shape.ptr, + @intCast(strides.len), + strides.ptr, + ))); + } + + pub fn elementType(self: *const TensorViewType) *const mlir.Type { + return typeFromC(c.mlirCudaTileTensorViewTypeGetElementType(self.ptr())); + } + + pub fn rank(self: *const TensorViewType) usize { + return @intCast(c.mlirCudaTileTensorViewTypeGetRank(self.ptr())); + } + + pub fn dimension(self: *const TensorViewType, i: usize) i64 { + return c.mlirCudaTileTensorViewTypeGetDimSize(self.ptr(), @intCast(i)); + } + + pub fn stride(self: *const TensorViewType, i: usize) i64 { + return c.mlirCudaTileTensorViewTypeGetStride(self.ptr(), @intCast(i)); + } + + pub fn type_(self: *const TensorViewType) *const mlir.Type { + return @ptrCast(self); + } +}; + +pub fn tensorViewType(ctx: *mlir.Context, elem: *const mlir.Type, shape: []const i64, strides: []const i64) *const mlir.Type { + return @ptrCast(TensorViewType.get(ctx, .{ .elem = elem, .shape = shape, .strides = strides })); +} + +const DimMap = stdx.BoundedArray(i32, mlir.ShapedType.MAX_RANK); + +fn identityDimMap(dim_map: []const i32, rank: usize) DimMap { + var out: DimMap = .empty; + if (dim_map.len > 0) { + out.appendSliceAssumeCapacity(dim_map); + } else { + for (0..rank) |i| out.appendAssumeCapacity(@intCast(i)); + } + return out; +} + +/// `!cuda_tile.partition_view, dim_map=[...]>`. +/// The tile shape, padding and dim_map all live in the type; the op that +/// builds one has no attributes of its own. +pub const PartitionViewType = opaque { + const M = mlir.Methods(PartitionViewType, c.MlirType); + + pub const isAFn = c.mlirCudaTileTypeIsAPartitionViewType; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + + /// An empty `dim_map` is the identity; the type requires one entry per + /// tile dimension. + pub const InitArgs = struct { + tile_shape: []const i32, + tensor_view: *const mlir.Type, + dim_map: []const i32 = &.{}, + padding: ?PaddingValue = null, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) *const PartitionViewType { + const tile_shape = args.tile_shape; + const tensor_view = args.tensor_view; + const dim_map = args.dim_map; + const tile_attr: *const mlir.Attribute = .denseArray(ctx, .i32, tile_shape); + const map = identityDimMap(dim_map, tile_shape.len); + return @ptrCast(typeFromC(c.mlirCudaTilePartitionViewTypeGet( + ctx.ptr(), + tile_attr.ptr(), + tensor_view.ptr(), + @intCast(map.len), + map.constSlice().ptr, + if (args.padding) |p| p.attribute(ctx).ptr() else null_attr, + ))); + } + + pub fn tensorView(self: *const PartitionViewType) *const mlir.Type { + return typeFromC(c.mlirCudaTilePartitionViewTypeGetTensorView(self.ptr())); + } + + pub fn viewTileType(self: *const PartitionViewType) *const mlir.Type { + return typeFromC(c.mlirCudaTilePartitionViewTypeGetViewTileType(self.ptr())); + } + + pub fn viewIndexRank(self: *const PartitionViewType) usize { + return @intCast(c.mlirCudaTilePartitionViewTypeGetViewIndexRank(self.ptr())); + } + + pub fn type_(self: *const PartitionViewType) *const mlir.Type { + return @ptrCast(self); + } +}; + +/// `!cuda_tile.strided_view` (13.3). +pub const StridedViewType = opaque { + const M = mlir.Methods(StridedViewType, c.MlirType); + + pub const isAFn = c.mlirCudaTileTypeIsAStridedViewType; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + tile_shape: []const i32, + traversal_strides: []const i32, + tensor_view: *const mlir.Type, + dim_map: []const i32 = &.{}, + padding: ?PaddingValue = null, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) *const StridedViewType { + const tile_shape = args.tile_shape; + const traversal_strides = args.traversal_strides; + const tensor_view = args.tensor_view; + const dim_map = args.dim_map; + const tile_attr: *const mlir.Attribute = .denseArray(ctx, .i32, tile_shape); + const strides_attr: *const mlir.Attribute = .denseArray(ctx, .i32, traversal_strides); + const map = identityDimMap(dim_map, tile_shape.len); + return @ptrCast(typeFromC(c.mlirCudaTileStridedViewTypeGet( + ctx.ptr(), + tile_attr.ptr(), + strides_attr.ptr(), + tensor_view.ptr(), + @intCast(map.len), + map.constSlice().ptr, + if (args.padding) |p| p.attribute(ctx).ptr() else null_attr, + ))); + } + + pub fn viewTileType(self: *const StridedViewType) *const mlir.Type { + return typeFromC(c.mlirCudaTileStridedViewTypeGetViewTileType(self.ptr())); + } + + pub fn viewIndexRank(self: *const StridedViewType) usize { + return @intCast(c.mlirCudaTileStridedViewTypeGetViewIndexRank(self.ptr())); + } + + pub fn type_(self: *const StridedViewType) *const mlir.Type { + return @ptrCast(self); + } +}; + +/// `!cuda_tile.gather_scatter_view, sparse_dim=N>` +/// (13.3): gathers or scatters along `sparseDim`, one tile per index. +pub const GatherScatterViewType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACudaTileGatherScatterView; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "gather_scatter_view"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + tileShape: []const i32, + tensorView: *const TensorViewType, + sparseDim: u32, + paddingValue: ?PaddingValue = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCudaTileGatherScatterViewTypeGet( + ctx.ptr(), + @intCast(args.tileShape.len), + args.tileShape.ptr, + args.tensorView.ptr(), + args.sparseDim, + if (args.paddingValue) |p| p.attribute(ctx).ptr() else null_attr, + ); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn tileShape(self: *const Self) *const mlir.DenseArrayAttribute(.i32) { + return @ptrCast(attrFromC(c.mlirCudaTileGatherScatterViewTypeGetTileShape(self.ptr()))); + } + pub fn tensorView(self: *const Self) *const TensorViewType { + return @ptrCast(typeFromC(c.mlirCudaTileGatherScatterViewTypeGetTensorView(self.ptr()))); + } + pub fn sparseDim(self: *const Self) u32 { + return c.mlirCudaTileGatherScatterViewTypeGetSparseDim(self.ptr()); + } + pub fn paddingValue(self: *const Self) ?PaddingValue { + const attr = c.mlirCudaTileGatherScatterViewTypeGetPaddingValue(self.ptr()); + if (attr.ptr == null) return null; + const name = c.mlirCudaTilePaddingValueAttrGetValue(attr); + return std.meta.stringToEnum(PaddingValue, name.data[0..name.length]).?; + } +}; + +pub fn gatherScatterViewType( + ctx: *mlir.Context, + tile_shape: []const i64, + tensor_view: *const mlir.Type, + sparse_dim: u32, + opts: struct { + padding: ?PaddingValue = null, + }, +) *const mlir.Type { + var shape: stdx.BoundedArray(i32, mlir.ShapedType.MAX_RANK) = .empty; + for (tile_shape) |d| shape.appendAssumeCapacity(@intCast(d)); + const view = GatherScatterViewType.get(ctx, .{ + .tileShape = shape.constSlice(), + .tensorView = tensor_view.isA(TensorViewType) orelse std.debug.panic("cuda_tile: {f} is not a tensor_view", .{tensor_view}), + .sparseDim = sparse_dim, + .paddingValue = opts.padding, + }) catch std.debug.panic("cuda_tile: invalid gather_scatter_view over {f}", .{tensor_view}); + return view.type_(); +} + +/// The `tile<...>` type a load through `view` yields, for any view kind — +/// through the dialect's `TileView` type interface (`capi.cc`), so +/// the gather/scatter view, which has no getters in NVIDIA's C API, works too. +pub fn tileTypeOfView(view: *const mlir.Type) *const mlir.Type { + if (!c.mlirTypeIsACudaTileTileView(view.ptr())) std.debug.panic("cuda_tile: {f} is not a tiled view", .{view}); + return typeFromC(c.mlirCudaTileTileViewTypeGetViewTileType(view.ptr())); +} + +pub fn indexRankOfView(view: *const mlir.Type) usize { + if (!c.mlirTypeIsACudaTileTileView(view.ptr())) std.debug.panic("cuda_tile: {f} is not a tiled view", .{view}); + return @intCast(c.mlirCudaTileTileViewTypeGetViewIndexRank(view.ptr())); +} + +// Enum attributes: built through the C getter from the tag name, so a typo is +// a null attribute rather than a wrong case. + +fn enumAttribute(comptime getter: anytype, ctx: *mlir.Context, name: []const u8) *const mlir.Attribute { + return attrFromC(getter(ctx.ptr(), stringRef(name))); +} + +pub const RoundingMode = enum { + nearest_even, + zero, + negative_inf, + positive_inf, + approx, + full, + nearest_int_to_zero, + + pub fn attribute(self: RoundingMode, ctx: *mlir.Context) *const mlir.Attribute { + return enumAttribute(c.mlirCudaTileRoundingModeAttrGet, ctx, @tagName(self)); + } +}; + +pub const Signedness = enum { + signed, + unsigned, + + pub fn attribute(self: Signedness, ctx: *mlir.Context) *const mlir.Attribute { + return enumAttribute(c.mlirCudaTileSignednessAttrGet, ctx, @tagName(self)); + } +}; + +pub const IntegerOverflow = enum { + none, + no_signed_wrap, + no_unsigned_wrap, + no_wrap, + + pub fn attribute(self: IntegerOverflow, ctx: *mlir.Context) *const mlir.Attribute { + return enumAttribute(c.mlirCudaTileIntegerOverflowAttrGet, ctx, @tagName(self)); + } +}; + +pub const ComparisonOrdering = enum { + unordered, + ordered, + + pub fn attribute(self: ComparisonOrdering, ctx: *mlir.Context) *const mlir.Attribute { + return enumAttribute(c.mlirCudaTileComparisonOrderingAttrGet, ctx, @tagName(self)); + } +}; + +pub const ComparisonPredicate = enum { + equal, + not_equal, + less_than, + less_than_or_equal, + greater_than, + greater_than_or_equal, + + pub fn attribute(self: ComparisonPredicate, ctx: *mlir.Context) *const mlir.Attribute { + return enumAttribute(c.mlirCudaTileComparisonPredicateAttrGet, ctx, @tagName(self)); + } +}; + +pub const AtomicRMWMode = enum { + @"and", + @"or", + xor, + add, + addf, + max, + min, + umax, + umin, + xchg, + + pub fn attribute(self: AtomicRMWMode, ctx: *mlir.Context) *const mlir.Attribute { + return enumAttribute(c.mlirCudaTileAtomicRMWModeAttrGet, ctx, @tagName(self)); + } +}; + +pub const MemoryScope = enum { + tl_blk, + device, + sys, + + pub fn attribute(self: MemoryScope, ctx: *mlir.Context) *const mlir.Attribute { + return enumAttribute(c.mlirCudaTileMemoryScopeAttrGet, ctx, @tagName(self)); + } +}; + +pub const MemoryOrdering = enum { + weak, + relaxed, + acquire, + release, + acq_rel, + + pub fn attribute(self: MemoryOrdering, ctx: *mlir.Context) *const mlir.Attribute { + return enumAttribute(c.mlirCudaTileMemoryOrderingSemanticsAttrGet, ctx, @tagName(self)); + } +}; + +pub const PaddingValue = enum { + zero, + neg_zero, + nan, + pos_inf, + neg_inf, + + pub fn attribute(self: PaddingValue, ctx: *mlir.Context) *const mlir.Attribute { + return enumAttribute(c.mlirCudaTilePaddingValueAttrGet, ctx, @tagName(self)); + } +}; + +pub const SymbolVisibility = enum { + public, + private, + + pub fn attribute(self: SymbolVisibility, ctx: *mlir.Context) *const mlir.Attribute { + return enumAttribute(c.mlirCudaTileSymbolVisibilityAttrGet, ctx, @tagName(self)); + } +}; + +// optimization_hints: one dictionary of hints per architecture. + +pub const Arch = enum { default, sm_80, sm_86, sm_87, sm_88, sm_89, sm_90, sm_100, sm_103, sm_110, sm_120, sm_121 }; + +/// Hints `entry` accepts: `num_cta_in_cga` (power of two <= 16), +/// `num_worker_warps_per_cta` (4 or 8), `occupancy`. +pub const EntryHint = struct { + arch: Arch = .default, + num_cta_in_cga: ?i32 = null, + num_worker_warps_per_cta: ?i32 = null, + occupancy: ?i32 = null, +}; + +/// Hints `load_view_tko` / `store_view_tko` accept. +pub const LoadStoreHint = struct { + arch: Arch = .default, + allow_tma: ?bool = null, + latency: ?i32 = null, +}; + +/// Hints `load_ptr_tko` / `store_ptr_tko` accept: `latency` only, the +/// pointer arm cannot use TMA. (The dialect drops unsupported keys silently.) +pub const PtrLoadStoreHint = struct { + arch: Arch = .default, + latency: ?i32 = null, +}; + +/// `#cuda_tile.optimization_hints`: a +/// dictionary from architecture name to a dictionary of hints. +pub const OptimizationHintsAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirCudaTileAttributeIsAOptimizationHintsAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "optimization_hints"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: *const mlir.DictionaryAttribute, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCudaTileOptimizationHintsAttrGet(ctx.ptr(), args.value.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) *const mlir.DictionaryAttribute { + return @ptrCast(attrFromC(c.mlirCudaTileOptimizationHintsAttrGetValue(self.ptr()))); + } +}; + +/// The hint dictionary of one architecture: the non-null fields of `hint` +/// but `arch`, integers as `i64` like the textual form. +fn hintDictionary(ctx: *mlir.Context, hint: anytype) *const mlir.Attribute { + const field_names = @typeInfo(@TypeOf(hint)).@"struct".field_names; + var entries: stdx.BoundedArray(mlir.NamedAttribute, field_names.len) = .empty; + inline for (field_names) |field_name| { + if (comptime std.mem.eql(u8, field_name, "arch")) continue; + if (@field(hint, field_name)) |v| { + const value: *const mlir.Attribute = switch (@TypeOf(v)) { + bool => .boolean(ctx, v), + else => .int(ctx, .i64, v), + }; + entries.appendAssumeCapacity(.named(ctx, field_name, value)); + } + } + return .dict(ctx, entries.constSlice()); +} + +/// `#cuda_tile.optimization_hints` from a slice of `EntryHint`, +/// `LoadStoreHint` or `PtrLoadStoreHint`. +pub fn optimizationHintsAttr(ctx: *mlir.Context, hints: anytype) *const mlir.Attribute { + var entries: stdx.BoundedArray(mlir.NamedAttribute, std.meta.fieldNames(Arch).len) = .empty; + for (hints) |h| entries.appendAssumeCapacity(.named(ctx, @tagName(h.arch), hintDictionary(ctx, h))); + const value: *const mlir.DictionaryAttribute = @ptrCast(mlir.Attribute.dict(ctx, entries.constSlice())); + const attr = OptimizationHintsAttr.get(ctx, .{ .value = value }) catch + std.debug.panic("cuda_tile: invalid optimization_hints {f}", .{value}); + return attr.attribute(); +} + +// assume predicates. + +/// `#cuda_tile.div_by`, or `` for a +/// per-group claim; `every` and `along` are both set or both null. +pub const DivByAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACudaTileDivBy; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "div_by"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + divisor: u64, + every: ?i64 = null, + along: ?i64 = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCudaTileDivByAttrGet( + ctx.ptr(), + args.divisor, + args.every != null, + args.every orelse 0, + args.along != null, + args.along orelse 0, + ); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn divisor(self: *const Self) u64 { + return c.mlirCudaTileDivByAttrGetDivisor(self.ptr()); + } + pub fn every(self: *const Self) ?i64 { + return if (c.mlirCudaTileDivByAttrHasEvery(self.ptr())) c.mlirCudaTileDivByAttrGetEvery(self.ptr()) else null; + } + pub fn along(self: *const Self) ?i64 { + return if (c.mlirCudaTileDivByAttrHasAlong(self.ptr())) c.mlirCudaTileDivByAttrGetAlong(self.ptr()) else null; + } +}; + +/// `along` defaults to 0 when only `every` is given. +pub fn divByAttr( + ctx: *mlir.Context, + divisor: u64, + opts: struct { + every: ?i64 = null, + along: ?i64 = null, + }, +) *const mlir.Attribute { + const attr = DivByAttr.get(ctx, .{ + .divisor = divisor, + .every = opts.every, + .along = if (opts.every != null) opts.along orelse 0 else null, + }) catch unreachable; + return attr.attribute(); +} + +/// `#cuda_tile.same_elements<[...]>`: per dimension, the size of the groups +/// of equal elements. +pub const SameElementsAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACudaTileSameElements; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "same_elements"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + values: []const i64, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCudaTileSameElementsAttrGet(ctx.ptr(), @intCast(args.values.len), args.values.ptr); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn values(self: *const Self) *const mlir.DenseArrayAttribute(.i64) { + return @ptrCast(attrFromC(c.mlirCudaTileSameElementsAttrGetValues(self.ptr()))); + } +}; + +pub fn sameElementsAttr(ctx: *mlir.Context, values: []const i64) *const mlir.Attribute { + const attr = SameElementsAttr.get(ctx, .{ .values = values }) catch unreachable; + return attr.attribute(); +} + +/// `#cuda_tile.bounded`, either side `?` when null. +pub const BoundedAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACudaTileBounded; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "bounded"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + lb: ?i64 = null, + ub: ?i64 = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCudaTileBoundedAttrGet(ctx.ptr(), args.lb != null, args.lb orelse 0, args.ub != null, args.ub orelse 0); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn lb(self: *const Self) ?i64 { + return if (c.mlirCudaTileBoundedAttrHasLb(self.ptr())) c.mlirCudaTileBoundedAttrGetLb(self.ptr()) else null; + } + pub fn ub(self: *const Self) ?i64 { + return if (c.mlirCudaTileBoundedAttrHasUb(self.ptr())) c.mlirCudaTileBoundedAttrGetUb(self.ptr()) else null; + } +}; + +pub fn boundedAttr( + ctx: *mlir.Context, + opts: struct { + lb: ?i64 = null, + ub: ?i64 = null, + }, +) *const mlir.Attribute { + const attr = BoundedAttr.get(ctx, .{ .lb = opts.lb, .ub = opts.ub }) catch unreachable; + return attr.attribute(); +} + +// Constant values: DenseElementsAttr over a tile type, which is a ShapedType. + +pub fn splatAttr(tile_ty: *const mlir.Type, elem_attr: *const mlir.Attribute) *const mlir.Attribute { + return attrFromC(c.mlirDenseElementsAttrSplatGet(tile_ty.ptr(), elem_attr.ptr())); +} + +pub fn intElemAttr(elem_ty: *const mlir.Type, value: i64) *const mlir.Attribute { + return attrFromC(c.mlirIntegerAttrGet(elem_ty.ptr(), value)); +} + +pub fn floatElemAttr(ctx: *mlir.Context, elem_ty: *const mlir.Type, value: f64) *const mlir.Attribute { + return attrFromC(c.mlirFloatAttrDoubleGet(ctx.ptr(), elem_ty.ptr(), value)); +} + +/// A dense `DenseElementsAttr` over `tile_ty` from a raw element buffer; the +/// element type's bit width must be a multiple of 8. +pub fn denseAttr(tile_ty: *const mlir.Type, values: anytype) *const mlir.Attribute { + return .denseElements(@ptrCast(tile_ty), values); +} + +const MakeArgs = mlir.Operation.MakeArgs; + +fn opName(comptime mnemonic: []const u8) []const u8 { + return dialect_namespace ++ "." ++ mnemonic; +} + +/// Attribute lists are passed by value so a helper's temporary outlives the +/// slice handed to `Operation.make`. +const Attrs = stdx.BoundedArray(mlir.NamedAttribute, 8); + +fn attrsOf(list: []const mlir.NamedAttribute) Attrs { + var attrs: Attrs = .empty; + attrs.appendSliceAssumeCapacity(list); + return attrs; +} + +fn unary(ctx: *mlir.Context, comptime mnemonic: []const u8, source: *const mlir.Value, attrs: Attrs, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName(mnemonic), .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{source.type_()} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +fn binary(ctx: *mlir.Context, comptime mnemonic: []const u8, lhs: *const mlir.Value, rhs: *const mlir.Value, attrs: Attrs, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName(mnemonic), .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{lhs.type_()} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +fn convert(ctx: *mlir.Context, comptime mnemonic: []const u8, source: *const mlir.Value, result_type: *const mlir.Type, attrs: Attrs, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName(mnemonic), .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +fn roundingAttrs(ctx: *mlir.Context, rounding: RoundingMode, flush_to_zero: bool) Attrs { + var attrs: Attrs = .empty; + attrs.appendAssumeCapacity(.named(ctx, "rounding_mode", rounding.attribute(ctx))); + if (flush_to_zero) attrs.appendAssumeCapacity(.named(ctx, "flush_to_zero", .unit(ctx))); + return attrs; +} + +fn overflowAttrs(ctx: *mlir.Context, overflow: IntegerOverflow) Attrs { + var attrs: Attrs = .empty; + if (overflow != .none) attrs.appendAssumeCapacity(.named(ctx, "overflow", overflow.attribute(ctx))); + return attrs; +} + +fn ftzAttrs(ctx: *mlir.Context, flush_to_zero: bool) Attrs { + var attrs: Attrs = .empty; + if (flush_to_zero) attrs.appendAssumeCapacity(.named(ctx, "flush_to_zero", .unit(ctx))); + return attrs; +} + +fn i1Like(ctx: *mlir.Context, v: *const mlir.Value) *const mlir.Type { + const t = v.type_().isA(TileType) orelse std.debug.panic("cuda_tile: {f} is not a tile", .{v.type_()}); + var shape: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; + for (0..t.rank()) |i| shape.appendAssumeCapacity(t.dimension(i)); + return tileType(ctx, shape.constSlice(), .int(ctx, .i1)); +} + +pub fn absf(ctx: *mlir.Context, source: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return unary(ctx, "absf", source, .empty, location); +} + +pub fn addf(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, rounding: RoundingMode, flush_to_zero: bool, location: *const mlir.Location) *mlir.Operation { + return binary(ctx, "addf", lhs, rhs, roundingAttrs(ctx, rounding, flush_to_zero), location); +} + +pub fn subf(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, rounding: RoundingMode, flush_to_zero: bool, location: *const mlir.Location) *mlir.Operation { + return binary(ctx, "subf", lhs, rhs, roundingAttrs(ctx, rounding, flush_to_zero), location); +} + +pub fn mulf(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, rounding: RoundingMode, flush_to_zero: bool, location: *const mlir.Location) *mlir.Operation { + return binary(ctx, "mulf", lhs, rhs, roundingAttrs(ctx, rounding, flush_to_zero), location); +} + +pub fn divf(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, rounding: RoundingMode, flush_to_zero: bool, location: *const mlir.Location) *mlir.Operation { + return binary(ctx, "divf", lhs, rhs, roundingAttrs(ctx, rounding, flush_to_zero), location); +} + +pub fn remf(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return binary(ctx, "remf", lhs, rhs, .empty, location); +} + +pub fn negf(ctx: *mlir.Context, source: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return unary(ctx, "negf", source, .empty, location); +} + +pub fn fma(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, acc: *const mlir.Value, rounding: RoundingMode, flush_to_zero: bool, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName("fma"), .{ + .operands = .{ .flat = &.{ lhs, rhs, acc } }, + .results = .{ .flat = &.{acc.type_()} }, + .attributes = roundingAttrs(ctx, rounding, flush_to_zero).constSlice(), + .location = location, + }); +} + +fn minMaxAttrs(ctx: *mlir.Context, propagate_nan: bool, flush_to_zero: bool) Attrs { + var attrs: Attrs = .empty; + if (propagate_nan) attrs.appendAssumeCapacity(.named(ctx, "propagate_nan", .unit(ctx))); + if (flush_to_zero) attrs.appendAssumeCapacity(.named(ctx, "flush_to_zero", .unit(ctx))); + return attrs; +} + +pub fn maxf(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, propagate_nan: bool, flush_to_zero: bool, location: *const mlir.Location) *mlir.Operation { + return binary(ctx, "maxf", lhs, rhs, minMaxAttrs(ctx, propagate_nan, flush_to_zero), location); +} + +pub fn minf(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, propagate_nan: bool, flush_to_zero: bool, location: *const mlir.Location) *mlir.Operation { + return binary(ctx, "minf", lhs, rhs, minMaxAttrs(ctx, propagate_nan, flush_to_zero), location); +} + +pub fn pow(ctx: *mlir.Context, source: *const mlir.Value, exponent: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return binary(ctx, "fpowf", source, exponent, .empty, location); +} + +pub fn atan2(ctx: *mlir.Context, x: *const mlir.Value, y: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return binary(ctx, "atan2", x, y, .empty, location); +} + +/// `rounding` null keeps the op's default (13.3 attribute). +pub fn exp( + ctx: *mlir.Context, + source: *const mlir.Value, + opts: struct { + rounding: ?RoundingMode = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attrs: Attrs = .empty; + if (opts.rounding) |r| attrs.appendAssumeCapacity(.named(ctx, "rounding_mode", r.attribute(ctx))); + return unary(ctx, "exp", source, attrs, location); +} + +pub fn exp2(ctx: *mlir.Context, source: *const mlir.Value, flush_to_zero: bool, location: *const mlir.Location) *mlir.Operation { + return unary(ctx, "exp2", source, ftzAttrs(ctx, flush_to_zero), location); +} + +pub fn log(ctx: *mlir.Context, source: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return unary(ctx, "log", source, .empty, location); +} + +pub fn log2(ctx: *mlir.Context, source: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return unary(ctx, "log2", source, .empty, location); +} + +pub fn sqrt(ctx: *mlir.Context, source: *const mlir.Value, rounding: RoundingMode, flush_to_zero: bool, location: *const mlir.Location) *mlir.Operation { + return unary(ctx, "sqrt", source, roundingAttrs(ctx, rounding, flush_to_zero), location); +} + +pub fn rsqrt(ctx: *mlir.Context, source: *const mlir.Value, flush_to_zero: bool, location: *const mlir.Location) *mlir.Operation { + return unary(ctx, "rsqrt", source, ftzAttrs(ctx, flush_to_zero), location); +} + +pub fn sin(ctx: *mlir.Context, source: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return unary(ctx, "sin", source, .empty, location); +} + +pub fn cos(ctx: *mlir.Context, source: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return unary(ctx, "cos", source, .empty, location); +} + +pub fn tan(ctx: *mlir.Context, source: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return unary(ctx, "tan", source, .empty, location); +} + +pub fn sinh(ctx: *mlir.Context, source: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return unary(ctx, "sinh", source, .empty, location); +} + +pub fn cosh(ctx: *mlir.Context, source: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return unary(ctx, "cosh", source, .empty, location); +} + +/// `rounding` null keeps the op's default (13.2 attribute). Only `approx` +/// and `full` are legal; the upstream verifier reports the others without +/// failing, so they are refused here. +pub fn tanh( + ctx: *mlir.Context, + source: *const mlir.Value, + opts: struct { + rounding: ?RoundingMode = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attrs: Attrs = .empty; + if (opts.rounding) |r| { + switch (r) { + .approx, .full => {}, + else => std.debug.panic("cuda_tile.tanh: rounding mode {s} is not allowed (only approx or full)", .{@tagName(r)}), + } + attrs.appendAssumeCapacity(.named(ctx, "rounding_mode", r.attribute(ctx))); + } + return unary(ctx, "tanh", source, attrs, location); +} + +pub fn ceil(ctx: *mlir.Context, source: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return unary(ctx, "ceil", source, .empty, location); +} + +pub fn floor(ctx: *mlir.Context, source: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return unary(ctx, "floor", source, .empty, location); +} + +pub fn cmpf(ctx: *mlir.Context, predicate: ComparisonPredicate, ordering: ComparisonOrdering, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName("cmpf"), .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{i1Like(ctx, lhs)} }, + .attributes = &.{ + .named(ctx, "comparison_predicate", predicate.attribute(ctx)), + .named(ctx, "comparison_ordering", ordering.attribute(ctx)), + }, + .location = location, + }); +} + +pub fn mmaf(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, acc: *const mlir.Value, fast_acc: bool, location: *const mlir.Location) *mlir.Operation { + var attrs: Attrs = .empty; + if (fast_acc) attrs.appendAssumeCapacity(.named(ctx, "fast_acc", .unit(ctx))); + return mlir.Operation.make(ctx, opName("mmaf"), .{ + .operands = .{ .flat = &.{ lhs, rhs, acc } }, + .results = .{ .flat = &.{acc.type_()} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +/// Block-scaled MMA (13.3): scales are plain tiles in logical M/N/K layout. +pub fn mmaf_scaled(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, acc: *const mlir.Value, lhs_scale: *const mlir.Value, rhs_scale: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName("mmaf_scaled"), .{ + .operands = .{ .flat = &.{ lhs, rhs, acc, lhs_scale, rhs_scale } }, + .results = .{ .flat = &.{acc.type_()} }, + .location = location, + }); +} + +pub fn absi(ctx: *mlir.Context, source: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return unary(ctx, "absi", source, .empty, location); +} + +pub fn addi(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, overflow: IntegerOverflow, location: *const mlir.Location) *mlir.Operation { + return binary(ctx, "addi", lhs, rhs, overflowAttrs(ctx, overflow), location); +} + +pub fn subi(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, overflow: IntegerOverflow, location: *const mlir.Location) *mlir.Operation { + return binary(ctx, "subi", lhs, rhs, overflowAttrs(ctx, overflow), location); +} + +pub fn muli(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, overflow: IntegerOverflow, location: *const mlir.Location) *mlir.Operation { + return binary(ctx, "muli", lhs, rhs, overflowAttrs(ctx, overflow), location); +} + +pub fn mulhii(ctx: *mlir.Context, x: *const mlir.Value, y: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return binary(ctx, "mulhii", x, y, .empty, location); +} + +/// `rounding` null keeps the op's default (truncation). +pub fn divi( + ctx: *mlir.Context, + lhs: *const mlir.Value, + rhs: *const mlir.Value, + signedness: Signedness, + opts: struct { + rounding: ?RoundingMode = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attrs: Attrs = .empty; + attrs.appendAssumeCapacity(.named(ctx, "signedness", signedness.attribute(ctx))); + if (opts.rounding) |r| attrs.appendAssumeCapacity(.named(ctx, "rounding", r.attribute(ctx))); + return binary(ctx, "divi", lhs, rhs, attrs, location); +} + +fn signedBinary(ctx: *mlir.Context, comptime mnemonic: []const u8, lhs: *const mlir.Value, rhs: *const mlir.Value, signedness: Signedness, location: *const mlir.Location) *mlir.Operation { + return binary(ctx, mnemonic, lhs, rhs, attrsOf(&.{.named(ctx, "signedness", signedness.attribute(ctx))}), location); +} + +pub fn remi(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, signedness: Signedness, location: *const mlir.Location) *mlir.Operation { + return signedBinary(ctx, "remi", lhs, rhs, signedness, location); +} + +pub fn maxi(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, signedness: Signedness, location: *const mlir.Location) *mlir.Operation { + return signedBinary(ctx, "maxi", lhs, rhs, signedness, location); +} + +pub fn mini(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, signedness: Signedness, location: *const mlir.Location) *mlir.Operation { + return signedBinary(ctx, "mini", lhs, rhs, signedness, location); +} + +pub fn negi(ctx: *mlir.Context, source: *const mlir.Value, overflow: IntegerOverflow, location: *const mlir.Location) *mlir.Operation { + return unary(ctx, "negi", source, overflowAttrs(ctx, overflow), location); +} + +pub fn shli(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, overflow: IntegerOverflow, location: *const mlir.Location) *mlir.Operation { + return binary(ctx, "shli", lhs, rhs, overflowAttrs(ctx, overflow), location); +} + +pub fn shri(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, signedness: Signedness, location: *const mlir.Location) *mlir.Operation { + return signedBinary(ctx, "shri", lhs, rhs, signedness, location); +} + +pub fn cmpi(ctx: *mlir.Context, predicate: ComparisonPredicate, signedness: Signedness, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName("cmpi"), .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{i1Like(ctx, lhs)} }, + .attributes = &.{ + .named(ctx, "comparison_predicate", predicate.attribute(ctx)), + .named(ctx, "signedness", signedness.attribute(ctx)), + }, + .location = location, + }); +} + +pub fn mmai(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, acc: *const mlir.Value, signedness_lhs: Signedness, signedness_rhs: Signedness, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName("mmai"), .{ + .operands = .{ .flat = &.{ lhs, rhs, acc } }, + .results = .{ .flat = &.{acc.type_()} }, + .attributes = &.{ + .named(ctx, "signedness_lhs", signedness_lhs.attribute(ctx)), + .named(ctx, "signedness_rhs", signedness_rhs.attribute(ctx)), + }, + .location = location, + }); +} + +pub fn andi(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return binary(ctx, "andi", lhs, rhs, .empty, location); +} + +pub fn ori(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return binary(ctx, "ori", lhs, rhs, .empty, location); +} + +pub fn xori(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return binary(ctx, "xori", lhs, rhs, .empty, location); +} + +pub fn bitcast(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return convert(ctx, "bitcast", source, result_type, .empty, location); +} + +pub fn exti(ctx: *mlir.Context, from: *const mlir.Value, signedness: Signedness, to: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return convert(ctx, "exti", from, to, attrsOf(&.{.named(ctx, "signedness", signedness.attribute(ctx))}), location); +} + +pub fn trunci(ctx: *mlir.Context, from: *const mlir.Value, to: *const mlir.Type, overflow: IntegerOverflow, location: *const mlir.Location) *mlir.Operation { + return convert(ctx, "trunci", from, to, overflowAttrs(ctx, overflow), location); +} + +/// `rounding` null keeps the op's default (nearest_even). +pub fn ftof( + ctx: *mlir.Context, + from: *const mlir.Value, + to: *const mlir.Type, + opts: struct { + rounding: ?RoundingMode = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attrs: Attrs = .empty; + if (opts.rounding) |r| attrs.appendAssumeCapacity(.named(ctx, "rounding_mode", r.attribute(ctx))); + return convert(ctx, "ftof", from, to, attrs, location); +} + +pub fn ftoi(ctx: *mlir.Context, from: *const mlir.Value, signedness: Signedness, rounding: RoundingMode, to: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return convert(ctx, "ftoi", from, to, attrsOf(&.{ + .named(ctx, "signedness", signedness.attribute(ctx)), + .named(ctx, "rounding_mode", rounding.attribute(ctx)), + }), location); +} + +pub fn itof(ctx: *mlir.Context, from: *const mlir.Value, signedness: Signedness, rounding: RoundingMode, to: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return convert(ctx, "itof", from, to, attrsOf(&.{ + .named(ctx, "signedness", signedness.attribute(ctx)), + .named(ctx, "rounding_mode", rounding.attribute(ctx)), + }), location); +} + +pub fn int_to_ptr(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return convert(ctx, "int_to_ptr", source, result_type, .empty, location); +} + +pub fn ptr_to_int(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return convert(ctx, "ptr_to_int", source, result_type, .empty, location); +} + +pub fn ptr_to_ptr(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return convert(ctx, "ptr_to_ptr", source, result_type, .empty, location); +} + +/// 13.3: sub-byte tile -> i8 tile. +pub fn pack(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return convert(ctx, "pack", source, result_type, .empty, location); +} + +/// 13.3: i8 tile -> sub-byte tile. +pub fn unpack(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return convert(ctx, "unpack", source, result_type, .empty, location); +} + +pub fn constant(ctx: *mlir.Context, value: *const mlir.Attribute, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName("constant"), .{ + .results = .{ .flat = &.{result_type} }, + .attributes = &.{.named(ctx, "value", value)}, + .location = location, + }); +} + +pub fn iota(ctx: *mlir.Context, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName("iota"), .{ + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +pub fn reshape(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return convert(ctx, "reshape", source, result_type, .empty, location); +} + +pub fn broadcast(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return convert(ctx, "broadcast", source, result_type, .empty, location); +} + +pub fn cat(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, dim: i64, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName("cat"), .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{.named(ctx, "dim", .int(ctx, .i64, dim))}, + .location = location, + }); +} + +pub fn extract(ctx: *mlir.Context, source: *const mlir.Value, indices: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + var operands: stdx.BoundedArray(*const mlir.Value, mlir.ShapedType.MAX_RANK + 1) = .empty; + operands.appendAssumeCapacity(source); + operands.appendSliceAssumeCapacity(indices); + return mlir.Operation.make(ctx, opName("extract"), .{ + .operands = .{ .flat = operands.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +pub fn permute(ctx: *mlir.Context, source: *const mlir.Value, permutation: []const i32, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return convert(ctx, "permute", source, result_type, attrsOf(&.{.named(ctx, "permutation", .denseArray(ctx, .i32, permutation))}), location); +} + +pub fn select(ctx: *mlir.Context, cond: *const mlir.Value, val_if_true: *const mlir.Value, val_if_false: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName("select"), .{ + .operands = .{ .flat = &.{ cond, val_if_true, val_if_false } }, + .results = .{ .flat = &.{val_if_true.type_()} }, + .location = location, + }); +} + +pub fn offset(ctx: *mlir.Context, ptr: *const mlir.Value, off: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return binary(ctx, "offset", ptr, off, .empty, location); +} + +fn threeI32(ctx: *mlir.Context, comptime mnemonic: []const u8, location: *const mlir.Location) *mlir.Operation { + const i32_tile = tileType(ctx, &.{}, .int(ctx, .i32)); + return mlir.Operation.make(ctx, opName(mnemonic), .{ + .results = .{ .flat = &.{ i32_tile, i32_tile, i32_tile } }, + .location = location, + }); +} + +pub fn get_tile_block_id(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return threeI32(ctx, "get_tile_block_id", location); +} + +pub fn get_num_tile_blocks(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return threeI32(ctx, "get_num_tile_blocks", location); +} + +pub const GlobalOpts = struct { + alignment: ?i64 = null, + constant: bool = false, + visibility: ?SymbolVisibility = null, +}; + +pub fn global(ctx: *mlir.Context, name: []const u8, value: *const mlir.Attribute, opts: GlobalOpts, location: *const mlir.Location) *mlir.Operation { + var attrs: Attrs = .empty; + attrs.appendAssumeCapacity(.named(ctx, "sym_name", .string(ctx, name))); + attrs.appendAssumeCapacity(.named(ctx, "value", value)); + if (opts.alignment) |a| attrs.appendAssumeCapacity(.named(ctx, "alignment", .int(ctx, .i64, a))); + if (opts.constant) attrs.appendAssumeCapacity(.named(ctx, "constant", .unit(ctx))); + if (opts.visibility) |v| attrs.appendAssumeCapacity(.named(ctx, "symbol_visibility", v.attribute(ctx))); + return mlir.Operation.make(ctx, opName("global"), .{ + .attributes = attrs.constSlice(), + .verify = false, + .location = location, + }); +} + +pub fn get_global(ctx: *mlir.Context, name: []const u8, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName("get_global"), .{ + .results = .{ .flat = &.{result_type} }, + .attributes = &.{.named(ctx, "name", .flatSymbolRef(ctx, name))}, + // Symbol lookup needs the enclosing module; verified with it. + .verify = false, + .location = location, + }); +} + +/// `cuda_tile.module @name { ... }`. `body` holds the entries and globals. +pub fn module(ctx: *mlir.Context, name: []const u8, body: *mlir.Block, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName("module"), .{ + .attributes = &.{.named(ctx, "sym_name", .string(ctx, name))}, + .blocks = &.{body}, + .verify = false, + .location = location, + }); +} + +pub const EntryArgs = struct { + name: []const u8, + /// The entry block; its arguments are the kernel parameters, every one a + /// rank-0 tile. + block: *mlir.Block, + arg_attrs: ?[]const *const mlir.Attribute = null, + optimization_hints: ?*const mlir.Attribute = null, + location: *const mlir.Location, +}; + +/// `entry @name(...) { ... }` — the kernel. Returns nothing; results reach +/// the host through pointer arguments. +pub fn entry(ctx: *mlir.Context, args: EntryArgs) *mlir.Operation { + var arg_types: stdx.BoundedArray(*const mlir.Type, 256) = .empty; + if (args.block.numArguments() > arg_types.capacity()) std.debug.panic("cuda_tile.entry: {d} parameters, at most {d} supported", .{ args.block.numArguments(), arg_types.capacity() }); + for (0..args.block.numArguments()) |i| arg_types.appendAssumeCapacity(args.block.argument(i).type_()); + + var attrs: Attrs = .empty; + attrs.appendAssumeCapacity(.named(ctx, "sym_name", .string(ctx, args.name))); + attrs.appendAssumeCapacity(.named(ctx, "function_type", .typeAttr(.function(ctx, arg_types.constSlice(), &.{})))); + if (args.arg_attrs) |a| attrs.appendAssumeCapacity(.named(ctx, "arg_attrs", .array(ctx, a))); + if (args.optimization_hints) |h| attrs.appendAssumeCapacity(.named(ctx, "optimization_hints", h)); + + return mlir.Operation.make(ctx, opName("entry"), .{ + .attributes = attrs.constSlice(), + .blocks = &.{args.block}, + .verify = false, + .location = args.location, + }); +} + +fn terminator(ctx: *mlir.Context, comptime mnemonic: []const u8, values: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName(mnemonic), .{ + .operands = .{ .flat = values }, + .verify = false, + .location = location, + }); +} + +pub fn return_(ctx: *mlir.Context, values: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return terminator(ctx, "return", values, location); +} + +pub fn yield(ctx: *mlir.Context, values: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return terminator(ctx, "yield", values, location); +} + +pub fn continue_(ctx: *mlir.Context, values: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return terminator(ctx, "continue", values, location); +} + +pub fn break_(ctx: *mlir.Context, values: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return terminator(ctx, "break", values, location); +} + +/// `for %iv in (%lb to %ub, step %step) iter_values(...)`. `body` takes +/// `[iv, inits...]` and ends with `continue`. +pub fn for_(ctx: *mlir.Context, lower: *const mlir.Value, upper: *const mlir.Value, step: *const mlir.Value, inits: []const *const mlir.Value, body: *mlir.Block, unsigned_cmp: bool, location: *const mlir.Location) *mlir.Operation { + // Built on the state directly: any number of carried values, and outer + // SSA uses only dominate once the op sits in its block, so no per-op + // verify. + var state: mlir.OperationState = .init(opName("for"), location); + state.addOperands(&.{ lower, upper, step }); + state.addOperands(inits); + for (inits) |v| state.addResults(&.{v.type_()}); + if (unsigned_cmp) state.addAttributes(&.{.named(ctx, "unsignedCmp", .unit(ctx))}); + const region = mlir.Region.init(); + region.appendOwnedBlock(body); + state.addOwnedRegions(&.{region}); + return mlir.Operation.init(&state) catch @panic("Failed to create cuda_tile.for"); +} + +/// `if %cond -> (types) { ... } else { ... }`. Branches end with `yield` +/// (or `continue`/`break`/`return` when they leave the enclosing scope). +pub fn if_( + ctx: *mlir.Context, + cond: *const mlir.Value, + result_types: []const *const mlir.Type, + then_block: *mlir.Block, + opts: struct { + else_block: ?*mlir.Block = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + _ = ctx; + var state: mlir.OperationState = .init(opName("if"), location); + state.addOperands(&.{cond}); + state.addResults(result_types); + const then_region = mlir.Region.init(); + then_region.appendOwnedBlock(then_block); + const else_region = mlir.Region.init(); + if (opts.else_block) |b| else_region.appendOwnedBlock(b); + state.addOwnedRegions(&.{ then_region, else_region }); + return mlir.Operation.init(&state) catch @panic("Failed to create cuda_tile.if"); +} + +/// An unstructured loop over `inits`; the body takes them as arguments and +/// leaves through `continue` (next iteration) or `break` (the results). +pub fn loop(ctx: *mlir.Context, inits: []const *const mlir.Value, result_types: []const *const mlir.Type, body: *mlir.Block, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName("loop"), .{ + .operands = .{ .flat = inits }, + .results = .{ .flat = result_types }, + .blocks = &.{body}, + .verify = false, + .location = location, + }); +} + +/// `reduce %operands dim=D identities=[...]` — `body` takes `2N` arguments +/// (element, accumulator per operand) and yields `N` accumulators. Results +/// drop dimension `dim`. +pub fn reduce(ctx: *mlir.Context, operands: []const *const mlir.Value, dim: i32, identities: []const *const mlir.Attribute, body: *mlir.Block, result_types: []const *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName("reduce"), .{ + .operands = .{ .flat = operands }, + .results = .{ .flat = result_types }, + .blocks = &.{body}, + .attributes = &.{ + .named(ctx, "dim", .int(ctx, .i32, dim)), + .named(ctx, "identities", .array(ctx, identities)), + }, + .verify = false, + .location = location, + }); +} + +/// `scan %operands dim=D reverse=R identities=[...]` — like `reduce`, but the +/// results keep the operand shape. +pub fn scan(ctx: *mlir.Context, operands: []const *const mlir.Value, dim: i32, reverse: bool, identities: []const *const mlir.Attribute, body: *mlir.Block, result_types: []const *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName("scan"), .{ + .operands = .{ .flat = operands }, + .results = .{ .flat = result_types }, + .blocks = &.{body}, + .attributes = &.{ + .named(ctx, "dim", .int(ctx, .i32, dim)), + .named(ctx, "reverse", .boolean(ctx, reverse)), + .named(ctx, "identities", .array(ctx, identities)), + }, + .verify = false, + .location = location, + }); +} + +pub fn assert_(ctx: *mlir.Context, condition: *const mlir.Value, message: []const u8, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName("assert"), .{ + .operands = .{ .flat = &.{condition} }, + .attributes = &.{.named(ctx, "message", .string(ctx, message))}, + .location = location, + }); +} + +/// `%r = assume #predicate, %value` — a refined copy of `value`. +pub fn assume(ctx: *mlir.Context, value: *const mlir.Value, predicate: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName("assume"), .{ + .operands = .{ .flat = &.{value} }, + .results = .{ .flat = &.{value.type_()} }, + .attributes = &.{.named(ctx, "predicate", predicate)}, + .location = location, + }); +} + +/// `make_tensor_view %base, shape = [...], strides = [...]`. Static extents +/// live in `result_type`; the operands fill its `?` slots in order. +pub fn make_tensor_view(ctx: *mlir.Context, base: *const mlir.Value, dynamic_shape: []const *const mlir.Value, dynamic_strides: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + var segs: Segments = .{}; + segs.add(&.{base}); + segs.add(dynamic_shape); + segs.add(dynamic_strides); + return mlir.Operation.make(ctx, opName("make_tensor_view"), .{ + .operands = .{ .flat = segs.values.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{segs.attr(ctx)}, + .location = location, + }); +} + +fn viewFromTensorView(ctx: *mlir.Context, comptime mnemonic: []const u8, tensor_view: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return convert(ctx, mnemonic, tensor_view, result_type, .empty, location); +} + +/// The tile shape, padding and dim_map are all in `result_type`. +pub fn make_partition_view(ctx: *mlir.Context, tensor_view: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return viewFromTensorView(ctx, "make_partition_view", tensor_view, result_type, location); +} + +pub fn make_strided_view(ctx: *mlir.Context, tensor_view: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return viewFromTensorView(ctx, "make_strided_view", tensor_view, result_type, location); +} + +pub fn make_gather_scatter_view(ctx: *mlir.Context, tensor_view: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return viewFromTensorView(ctx, "make_gather_scatter_view", tensor_view, result_type, location); +} + +/// One rank-0 integer tile per view index dimension. +pub fn get_index_space_shape(ctx: *mlir.Context, view: *const mlir.Value, result_types: []const *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName("get_index_space_shape"), .{ + .operands = .{ .flat = &.{view} }, + .results = .{ .flat = result_types }, + .location = location, + }); +} + +/// One rank-0 integer tile per tensor dimension. +pub fn get_tensor_shape(ctx: *mlir.Context, tensor_view: *const mlir.Value, result_types: []const *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName("get_tensor_shape"), .{ + .operands = .{ .flat = &.{tensor_view} }, + .results = .{ .flat = result_types }, + .location = location, + }); +} + +pub const MemOpts = struct { + ordering: MemoryOrdering = .weak, + scope: ?MemoryScope = null, + token: ?*const mlir.Value = null, + optimization_hints: ?*const mlir.Attribute = null, +}; + +fn memAttrs(ctx: *mlir.Context, opts: MemOpts) Attrs { + var attrs: Attrs = .empty; + attrs.appendAssumeCapacity(.named(ctx, "memory_ordering_semantics", opts.ordering.attribute(ctx))); + if (opts.scope) |s| attrs.appendAssumeCapacity(.named(ctx, "memory_scope", s.attribute(ctx))); + if (opts.optimization_hints) |h| attrs.appendAssumeCapacity(.named(ctx, "optimization_hints", h)); + return attrs; +} + +/// An optional operand as a 0- or 1-element segment. Held in a local so the +/// slice handed to `Operation.make` stays valid. +const Seg = struct { + buf: [1]*const mlir.Value = undefined, + len: usize = 0, + + fn slice(self: *const Seg) []const *const mlir.Value { + return self.buf[0..self.len]; + } +}; + +fn optional(v: ?*const mlir.Value) Seg { + var s: Seg = .{}; + if (v) |x| { + s.buf[0] = x; + s.len = 1; + } + return s; +} + +/// Operands of an `AttrSizedOperandSegments` op, flat plus the +/// `operandSegmentSizes` the op's properties expect (a DenseI32ArrayAttr). +const Segments = struct { + values: stdx.BoundedArray(*const mlir.Value, 128) = .empty, + sizes: stdx.BoundedArray(i32, 8) = .empty, + + fn add(self: *Segments, segment: []const *const mlir.Value) void { + self.values.appendSliceAssumeCapacity(segment); + self.sizes.appendAssumeCapacity(@intCast(segment.len)); + } + + fn attr(self: *const Segments, ctx: *mlir.Context) mlir.NamedAttribute { + return .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, self.sizes.constSlice())); + } +}; + +/// Two results: the tile (the view's tile type) and the token. +pub fn load_view_tko(ctx: *mlir.Context, view: *const mlir.Value, index: []const *const mlir.Value, opts: MemOpts, location: *const mlir.Location) *mlir.Operation { + const tok = optional(opts.token); + var segs: Segments = .{}; + segs.add(&.{view}); + segs.add(index); + segs.add(tok.slice()); + var attrs = memAttrs(ctx, opts); + // Preserve bounds checks unless the caller has proved a coordinate in bounds. + const inbounds: [mlir.ShapedType.MAX_RANK]i32 = @splat(0); + attrs.appendAssumeCapacity(.named(ctx, "inbounds", .denseArray(ctx, .bool, inbounds[0..index.len]))); + attrs.appendAssumeCapacity(segs.attr(ctx)); + return mlir.Operation.make(ctx, opName("load_view_tko"), .{ + .operands = .{ .flat = segs.values.constSlice() }, + .results = .{ .flat = &.{ tileTypeOfView(view.type_()), tokenType(ctx) } }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +/// One result: the token. +pub fn store_view_tko(ctx: *mlir.Context, tile: *const mlir.Value, view: *const mlir.Value, index: []const *const mlir.Value, opts: MemOpts, location: *const mlir.Location) *mlir.Operation { + const tok = optional(opts.token); + var segs: Segments = .{}; + segs.add(&.{tile}); + segs.add(&.{view}); + segs.add(index); + segs.add(tok.slice()); + var attrs = memAttrs(ctx, opts); + const inbounds: [mlir.ShapedType.MAX_RANK]i32 = @splat(0); + attrs.appendAssumeCapacity(.named(ctx, "inbounds", .denseArray(ctx, .bool, inbounds[0..index.len]))); + attrs.appendAssumeCapacity(segs.attr(ctx)); + return mlir.Operation.make(ctx, opName("store_view_tko"), .{ + .operands = .{ .flat = segs.values.constSlice() }, + .results = .{ .flat = &.{tokenType(ctx)} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +pub const LoadPtrOpts = struct { + mask: ?*const mlir.Value = null, + padding: ?*const mlir.Value = null, + mem: MemOpts = .{}, +}; + +/// Gather through a tile of pointers. Two results: the loaded tile and the +/// token. `result_type` is the pointee tile. +pub fn load_ptr_tko(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, opts: LoadPtrOpts, location: *const mlir.Location) *mlir.Operation { + const mask = optional(opts.mask); + const padding = optional(opts.padding); + const tok = optional(opts.mem.token); + var segs: Segments = .{}; + segs.add(&.{source}); + segs.add(mask.slice()); + segs.add(padding.slice()); + segs.add(tok.slice()); + var attrs = memAttrs(ctx, opts.mem); + attrs.appendAssumeCapacity(segs.attr(ctx)); + return mlir.Operation.make(ctx, opName("load_ptr_tko"), .{ + .operands = .{ .flat = segs.values.constSlice() }, + .results = .{ .flat = &.{ result_type, tokenType(ctx) } }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +pub const StorePtrOpts = struct { + mask: ?*const mlir.Value = null, + mem: MemOpts = .{}, +}; + +/// Scatter through a tile of pointers. One result: the token. +pub fn store_ptr_tko(ctx: *mlir.Context, destination: *const mlir.Value, value: *const mlir.Value, opts: StorePtrOpts, location: *const mlir.Location) *mlir.Operation { + const mask = optional(opts.mask); + const tok = optional(opts.mem.token); + var segs: Segments = .{}; + segs.add(&.{destination}); + segs.add(&.{value}); + segs.add(mask.slice()); + segs.add(tok.slice()); + var attrs = memAttrs(ctx, opts.mem); + attrs.appendAssumeCapacity(segs.attr(ctx)); + return mlir.Operation.make(ctx, opName("store_ptr_tko"), .{ + .operands = .{ .flat = segs.values.constSlice() }, + .results = .{ .flat = &.{tokenType(ctx)} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +pub const AtomicOpts = struct { + ordering: MemoryOrdering = .acq_rel, + scope: MemoryScope = .device, + mask: ?*const mlir.Value = null, + token: ?*const mlir.Value = null, +}; + +fn atomicAttrs(ctx: *mlir.Context, opts: AtomicOpts) Attrs { + var attrs: Attrs = .empty; + attrs.appendAssumeCapacity(.named(ctx, "memory_ordering_semantics", opts.ordering.attribute(ctx))); + attrs.appendAssumeCapacity(.named(ctx, "memory_scope", opts.scope.attribute(ctx))); + return attrs; +} + +/// Two results: the old values and the token. +pub fn atomic_cas_tko(ctx: *mlir.Context, pointers: *const mlir.Value, cmp: *const mlir.Value, val: *const mlir.Value, opts: AtomicOpts, location: *const mlir.Location) *mlir.Operation { + const mask = optional(opts.mask); + const tok = optional(opts.token); + var segs: Segments = .{}; + segs.add(&.{pointers}); + segs.add(&.{cmp}); + segs.add(&.{val}); + segs.add(mask.slice()); + segs.add(tok.slice()); + var attrs = atomicAttrs(ctx, opts); + attrs.appendAssumeCapacity(segs.attr(ctx)); + return mlir.Operation.make(ctx, opName("atomic_cas_tko"), .{ + .operands = .{ .flat = segs.values.constSlice() }, + .results = .{ .flat = &.{ val.type_(), tokenType(ctx) } }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +/// Two results: the old values and the token. +pub fn atomic_rmw_tko(ctx: *mlir.Context, pointers: *const mlir.Value, mode: AtomicRMWMode, arg: *const mlir.Value, opts: AtomicOpts, location: *const mlir.Location) *mlir.Operation { + const mask = optional(opts.mask); + const tok = optional(opts.token); + var segs: Segments = .{}; + segs.add(&.{pointers}); + segs.add(&.{arg}); + segs.add(mask.slice()); + segs.add(tok.slice()); + var attrs = atomicAttrs(ctx, opts); + attrs.appendAssumeCapacity(.named(ctx, "mode", mode.attribute(ctx))); + attrs.appendAssumeCapacity(segs.attr(ctx)); + return mlir.Operation.make(ctx, opName("atomic_rmw_tko"), .{ + .operands = .{ .flat = segs.values.constSlice() }, + .results = .{ .flat = &.{ arg.type_(), tokenType(ctx) } }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +/// `atomic_red_view_tko` is relaxed-only, block- or device-scoped, and has +/// no mask operand. +pub const RedViewOpts = struct { + scope: MemoryScope = .device, + token: ?*const mlir.Value = null, +}; + +/// 13.3: an atomic reduction through an unpadded tiled view. One result: +/// the token. `xchg` is not a reduction. +pub fn atomic_red_view_tko(ctx: *mlir.Context, view: *const mlir.Value, index: []const *const mlir.Value, mode: AtomicRMWMode, value: *const mlir.Value, opts: RedViewOpts, location: *const mlir.Location) *mlir.Operation { + if (mode == .xchg) @panic("cuda_tile.atomic_red_view_tko: xchg is not a reduction"); + if (opts.scope == .sys) @panic("cuda_tile.atomic_red_view_tko: only tl_blk and device scopes are allowed"); + const tok = optional(opts.token); + var segs: Segments = .{}; + segs.add(&.{view}); + segs.add(index); + segs.add(&.{value}); + segs.add(tok.slice()); + var attrs: Attrs = .empty; + attrs.appendAssumeCapacity(.named(ctx, "memory_ordering_semantics", MemoryOrdering.relaxed.attribute(ctx))); + attrs.appendAssumeCapacity(.named(ctx, "memory_scope", opts.scope.attribute(ctx))); + attrs.appendAssumeCapacity(.named(ctx, "mode", mode.attribute(ctx))); + attrs.appendAssumeCapacity(segs.attr(ctx)); + return mlir.Operation.make(ctx, opName("atomic_red_view_tko"), .{ + .operands = .{ .flat = segs.values.constSlice() }, + .results = .{ .flat = &.{tokenType(ctx)} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +/// 13.3: `num_elem` elements of the pointee, as a rank-0 pointer tile. +pub fn alloca(ctx: *mlir.Context, num_elem: i64, alignment: i64, is_global: bool, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + var attrs: Attrs = .empty; + attrs.appendAssumeCapacity(.named(ctx, "num_elem", .int(ctx, .i64, num_elem))); + attrs.appendAssumeCapacity(.named(ctx, "alignment", .int(ctx, .i64, alignment))); + if (is_global) attrs.appendAssumeCapacity(.named(ctx, "global", .unit(ctx))); + return mlir.Operation.make(ctx, opName("alloca"), .{ + .results = .{ .flat = &.{result_type} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +pub fn make_token(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName("make_token"), .{ + .results = .{ .flat = &.{tokenType(ctx)} }, + .location = location, + }); +} + +pub fn join_tokens(ctx: *mlir.Context, tokens: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, opName("join_tokens"), .{ + .operands = .{ .flat = tokens }, + .results = .{ .flat = &.{tokenType(ctx)} }, + .location = location, + }); +} + +/// Device-side printf. One result: the token. +pub fn print_tko( + ctx: *mlir.Context, + str: []const u8, + args: []const *const mlir.Value, + opts: struct { + token: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + const tok = optional(opts.token); + var segs: Segments = .{}; + segs.add(args); + segs.add(tok.slice()); + return mlir.Operation.make(ctx, opName("print_tko"), .{ + .operands = .{ .flat = segs.values.constSlice() }, + .results = .{ .flat = &.{tokenType(ctx)} }, + .attributes = &.{ .named(ctx, "str", .string(ctx, str)), segs.attr(ctx) }, + .location = location, + }); +} + +// ============================================================================= +// Tests +// ============================================================================= diff --git a/mlir/dialects/cuda_tile/test.zig b/mlir/dialects/cuda_tile/test.zig new file mode 100644 index 00000000..52ae9480 --- /dev/null +++ b/mlir/dialects/cuda_tile/test.zig @@ -0,0 +1,391 @@ +const std = @import("std"); + +const c = @import("c"); +const cuda_tile = @import("mlir/dialects/cuda_tile"); +const mlir = @import("mlir"); + +const operation_names: []const []const u8 = &.{ + "cuda_tile.absf", + "cuda_tile.absi", + "cuda_tile.addf", + "cuda_tile.addi", + "cuda_tile.alloca", + "cuda_tile.andi", + "cuda_tile.assert", + "cuda_tile.assume", + "cuda_tile.atan2", + "cuda_tile.atomic_cas_tko", + "cuda_tile.atomic_red_view_tko", + "cuda_tile.atomic_rmw_tko", + "cuda_tile.bitcast", + "cuda_tile.break", + "cuda_tile.broadcast", + "cuda_tile.cat", + "cuda_tile.ceil", + "cuda_tile.cmpf", + "cuda_tile.cmpi", + "cuda_tile.constant", + "cuda_tile.continue", + "cuda_tile.cos", + "cuda_tile.cosh", + "cuda_tile.divf", + "cuda_tile.divi", + "cuda_tile.entry", + "cuda_tile.exp", + "cuda_tile.exp2", + "cuda_tile.exti", + "cuda_tile.extract", + "cuda_tile.floor", + "cuda_tile.fma", + "cuda_tile.for", + "cuda_tile.fpowf", + "cuda_tile.ftof", + "cuda_tile.ftoi", + "cuda_tile.get_global", + "cuda_tile.get_index_space_shape", + "cuda_tile.get_num_tile_blocks", + "cuda_tile.get_tensor_shape", + "cuda_tile.get_tile_block_id", + "cuda_tile.global", + "cuda_tile.if", + "cuda_tile.int_to_ptr", + "cuda_tile.iota", + "cuda_tile.itof", + "cuda_tile.join_tokens", + "cuda_tile.load_ptr_tko", + "cuda_tile.load_view_tko", + "cuda_tile.log", + "cuda_tile.log2", + "cuda_tile.loop", + "cuda_tile.make_gather_scatter_view", + "cuda_tile.make_partition_view", + "cuda_tile.make_strided_view", + "cuda_tile.make_tensor_view", + "cuda_tile.make_token", + "cuda_tile.maxf", + "cuda_tile.maxi", + "cuda_tile.minf", + "cuda_tile.mini", + "cuda_tile.mmaf", + "cuda_tile.mmaf_scaled", + "cuda_tile.mmai", + "cuda_tile.module", + "cuda_tile.mulf", + "cuda_tile.mulhii", + "cuda_tile.muli", + "cuda_tile.negf", + "cuda_tile.negi", + "cuda_tile.offset", + "cuda_tile.ori", + "cuda_tile.pack", + "cuda_tile.permute", + "cuda_tile.print_tko", + "cuda_tile.ptr_to_int", + "cuda_tile.ptr_to_ptr", + "cuda_tile.reduce", + "cuda_tile.remf", + "cuda_tile.remi", + "cuda_tile.reshape", + "cuda_tile.return", + "cuda_tile.rsqrt", + "cuda_tile.scan", + "cuda_tile.select", + "cuda_tile.shli", + "cuda_tile.shri", + "cuda_tile.sin", + "cuda_tile.sinh", + "cuda_tile.sqrt", + "cuda_tile.store_ptr_tko", + "cuda_tile.store_view_tko", + "cuda_tile.subf", + "cuda_tile.subi", + "cuda_tile.tan", + "cuda_tile.tanh", + "cuda_tile.trunci", + "cuda_tile.unpack", + "cuda_tile.xori", + "cuda_tile.yield", +}; + +fn testContext() !*mlir.Context { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + cuda_tile.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + return ctx; +} + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(cuda_tile); + inline for (comptime std.meta.declarations(cuda_tile)) |decl| { + const T = @field(cuda_tile, decl); + if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); + } +} + +test "dialect registers and types round-trip" { + const ctx = try testContext(); + defer ctx.deinit(); + + try std.testing.expect(ctx.isRegisteredOperation("cuda_tile.entry")); + try std.testing.expect(ctx.isRegisteredOperation("cuda_tile.load_view_tko")); + try std.testing.expect(ctx.isRegisteredOperation("cuda_tile.mmaf")); + + const f32_ty = mlir.Type.float(ctx, .f32); + const tile = cuda_tile.tileType(ctx, &.{ 64, 64 }, f32_ty); + try std.testing.expect(tile.eql(try mlir.Type.parse(ctx, "!cuda_tile.tile<64x64xf32>"))); + try std.testing.expectEqual(@as(usize, 2), tile.isA(cuda_tile.TileType).?.rank()); + try std.testing.expectEqual(@as(i64, 64), tile.isA(cuda_tile.TileType).?.dimension(1)); + + const ptr = cuda_tile.pointerType(ctx, f32_ty); + try std.testing.expect(ptr.eql(try mlir.Type.parse(ctx, "!cuda_tile.ptr"))); + try std.testing.expect(ptr.isA(cuda_tile.PointerType).?.pointee().eql(f32_ty)); + + const dyn = cuda_tile.TensorViewType.dynamic(); + const tv = cuda_tile.tensorViewType(ctx, f32_ty, &.{ dyn, 32 }, &.{ dyn, 1 }); + try std.testing.expect(tv.eql(try mlir.Type.parse(ctx, "!cuda_tile.tensor_view"))); + + const static_tv = cuda_tile.tensorViewType(ctx, f32_ty, &.{16}, &.{1}); + const pv: *const mlir.Type = @ptrCast(cuda_tile.PartitionViewType.get(ctx, .{ .tile_shape = &.{2}, .tensor_view = static_tv, .padding = .zero })); + try std.testing.expect(pv.eql(try mlir.Type.parse(ctx, "!cuda_tile.partition_view>"))); + try std.testing.expect(cuda_tile.tileTypeOfView(pv).eql(cuda_tile.tileType(ctx, &.{2}, f32_ty))); + try std.testing.expectEqual(@as(usize, 1), cuda_tile.indexRankOfView(pv)); + + try std.testing.expect(cuda_tile.tokenType(ctx).eql(try mlir.Type.parse(ctx, "!cuda_tile.token"))); + + const gather_tv = cuda_tile.tensorViewType(ctx, f32_ty, &.{ 64, 32 }, &.{ 32, 1 }); + const gsv = cuda_tile.gatherScatterViewType(ctx, &.{ 8, 16 }, gather_tv, 0, .{ .padding = .zero }); + try std.testing.expect(gsv.eql(try mlir.Type.parse(ctx, "!cuda_tile.gather_scatter_view, sparse_dim=0>"))); + const typed_gsv = gsv.isA(cuda_tile.GatherScatterViewType).?; + const gsv_tile = typed_gsv.tileShape(); + var gsv_shape: [2]i32 = undefined; + try std.testing.expectEqual(gsv_shape.len, gsv_tile.numElements()); + for (&gsv_shape, 0..) |*d, i| d.* = gsv_tile.element(i); + const rebuilt_gsv = try cuda_tile.GatherScatterViewType.get(ctx, .{ + .tileShape = &gsv_shape, + .tensorView = typed_gsv.tensorView(), + .sparseDim = typed_gsv.sparseDim(), + .paddingValue = typed_gsv.paddingValue(), + }); + try std.testing.expect(rebuilt_gsv.eql(typed_gsv)); + try std.testing.expect(cuda_tile.tileTypeOfView(gsv).eql(cuda_tile.tileType(ctx, &.{ 8, 16 }, f32_ty))); + try std.testing.expect((try cuda_tile.GatherScatterViewType.get(ctx, .{ + .tileShape = &.{ 8, 16 }, + .tensorView = gather_tv.isA(cuda_tile.TensorViewType).?, + .sparseDim = 1, + })).paddingValue() == null); + try std.testing.expectError(error.InvalidMlir, cuda_tile.GatherScatterViewType.get(ctx, .{ + .tileShape = &.{8}, + .tensorView = gather_tv.isA(cuda_tile.TensorViewType).?, + .sparseDim = 0, + })); +} + +test "enum and hint attributes" { + const ctx = try testContext(); + defer ctx.deinit(); + + try std.testing.expect(cuda_tile.RoundingMode.nearest_even.attribute(ctx).eql(try mlir.Attribute.parse(ctx, "#cuda_tile.rounding"))); + const padding = cuda_tile.PaddingValue.neg_inf.attribute(ctx); + const padding_text = c.mlirCudaTilePaddingValueAttrGetValue(padding.ptr()); + try std.testing.expectEqualStrings("neg_inf", padding_text.data[0..padding_text.length]); + + const hints = cuda_tile.optimizationHintsAttr(ctx, &[_]cuda_tile.EntryHint{ + .{ .arch = .sm_100, .num_cta_in_cga = 2 }, + .{ .arch = .sm_120, .num_cta_in_cga = 2, .occupancy = 2 }, + }); + try std.testing.expect(hints.eql(try mlir.Attribute.parse(ctx, "#cuda_tile.optimization_hints"))); + const typed_hints = hints.isA(cuda_tile.OptimizationHintsAttr).?; + try std.testing.expect((try cuda_tile.OptimizationHintsAttr.get(ctx, .{ .value = typed_hints.value() })).eql(typed_hints)); + const ls_hints = cuda_tile.optimizationHintsAttr(ctx, &[_]cuda_tile.LoadStoreHint{.{ .arch = .sm_100, .allow_tma = false, .latency = 3 }}); + try std.testing.expect(ls_hints.eql(try mlir.Attribute.parse(ctx, "#cuda_tile.optimization_hints"))); + const not_a_dict: *const mlir.DictionaryAttribute = @ptrCast(mlir.Attribute.dict(ctx, &.{.named(ctx, "sm_100", .int(ctx, .i64, 1))})); + try std.testing.expectError(error.InvalidMlir, cuda_tile.OptimizationHintsAttr.get(ctx, .{ .value = not_a_dict })); + + const div = cuda_tile.divByAttr(ctx, 16, .{}); + try std.testing.expect(div.eql(try mlir.Attribute.parse(ctx, "#cuda_tile.div_by<16>"))); + const div_every = cuda_tile.divByAttr(ctx, 4, .{ .every = 4, .along = 1 }); + try std.testing.expect(div_every.eql(try mlir.Attribute.parse(ctx, "#cuda_tile.div_by<4, every 4 along 1>"))); + try std.testing.expect(cuda_tile.divByAttr(ctx, 32, .{ .every = 4 }).eql(try mlir.Attribute.parse(ctx, "#cuda_tile.div_by<32, every 4 along 0>"))); + const typed_div = div_every.isA(cuda_tile.DivByAttr).?; + try std.testing.expect((try cuda_tile.DivByAttr.get(ctx, .{ + .divisor = typed_div.divisor(), + .every = typed_div.every(), + .along = typed_div.along(), + })).eql(typed_div)); + try std.testing.expect(div.isA(cuda_tile.DivByAttr).?.every() == null); + try std.testing.expectError(error.InvalidMlir, cuda_tile.DivByAttr.get(ctx, .{ .divisor = 4, .every = 4 })); + + const same = cuda_tile.sameElementsAttr(ctx, &.{ 2, 4 }); + try std.testing.expect(same.eql(try mlir.Attribute.parse(ctx, "#cuda_tile.same_elements<[2, 4]>"))); + const same_values = same.isA(cuda_tile.SameElementsAttr).?.values(); + try std.testing.expectEqual(@as(usize, 2), same_values.numElements()); + try std.testing.expectEqual(@as(i64, 4), same_values.element(1)); + + const lower = cuda_tile.boundedAttr(ctx, .{ .lb = 0 }); + try std.testing.expect(lower.eql(try mlir.Attribute.parse(ctx, "#cuda_tile.bounded<0, ?>"))); + try std.testing.expect(cuda_tile.boundedAttr(ctx, .{ .ub = 7 }).eql(try mlir.Attribute.parse(ctx, "#cuda_tile.bounded"))); + try std.testing.expect(cuda_tile.boundedAttr(ctx, .{ .lb = -3, .ub = 7 }).eql(try mlir.Attribute.parse(ctx, "#cuda_tile.bounded<-3, 7>"))); + const typed_lower = lower.isA(cuda_tile.BoundedAttr).?; + try std.testing.expectEqual(@as(?i64, 0), typed_lower.lb()); + try std.testing.expect(typed_lower.ub() == null); +} + +test "entry round-trip through print and parse" { + const ctx = try testContext(); + defer ctx.deinit(); + + const loc: *const mlir.Location = .unknown(ctx); + const module_op: *mlir.Module = .init(loc); + defer module_op.deinit(); + + const f32_ty = mlir.Type.float(ctx, .f32); + const ptr_tile = cuda_tile.tileType(ctx, &.{}, cuda_tile.pointerType(ctx, f32_ty)); + const block = mlir.Block.init(&.{ ptr_tile, ptr_tile }, &.{ loc, loc }); + + // out[i] = in[i] + 1 over 128 elements, the shape of cuda-tile's README example. + const idx_ty = cuda_tile.tileType(ctx, &.{128}, .int(ctx, .i32)); + const offsets = cuda_tile.iota(ctx, idx_ty, loc).appendTo(block); + const ptr1 = cuda_tile.tileType(ctx, &.{1}, cuda_tile.pointerType(ctx, f32_ty)); + const ptr128 = cuda_tile.tileType(ctx, &.{128}, cuda_tile.pointerType(ctx, f32_ty)); + const in_r = cuda_tile.reshape(ctx, block.argument(0), ptr1, loc).appendTo(block); + const in_b = cuda_tile.broadcast(ctx, in_r.result(0), ptr128, loc).appendTo(block); + const in_p = cuda_tile.offset(ctx, in_b.result(0), offsets.result(0), loc).appendTo(block); + const f32_128 = cuda_tile.tileType(ctx, &.{128}, f32_ty); + const loaded = cuda_tile.load_ptr_tko(ctx, in_p.result(0), f32_128, .{}, loc).appendTo(block); + const one = cuda_tile.constant(ctx, cuda_tile.splatAttr(f32_128, cuda_tile.floatElemAttr(ctx, f32_ty, 1.0)), f32_128, loc).appendTo(block); + const sum = cuda_tile.addf(ctx, loaded.result(0), one.result(0), .nearest_even, false, loc).appendTo(block); + const out_r = cuda_tile.reshape(ctx, block.argument(1), ptr1, loc).appendTo(block); + const out_b = cuda_tile.broadcast(ctx, out_r.result(0), ptr128, loc).appendTo(block); + const out_p = cuda_tile.offset(ctx, out_b.result(0), offsets.result(0), loc).appendTo(block); + _ = cuda_tile.store_ptr_tko(ctx, out_p.result(0), sum.result(0), .{ .mem = .{ .token = loaded.result(1) } }, loc).appendTo(block); + _ = cuda_tile.return_(ctx, &.{}, loc).appendTo(block); + + const ct_body = mlir.Block.init(&.{}, &.{}); + _ = cuda_tile.entry(ctx, .{ .name = "add_one", .block = block, .location = loc }).appendTo(ct_body); + _ = cuda_tile.module(ctx, "m", ct_body, loc).appendTo(module_op.body()); + + try std.testing.expect(module_op.operation().verify()); + + var al: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer al.deinit(); + try al.writer.print("{f}", .{module_op.operation()}); + const text = al.written(); + try std.testing.expect(std.mem.indexOf(u8, text, "cuda_tile.module @m") != null); + try std.testing.expect(std.mem.indexOf(u8, text, "entry @add_one(") != null); + try std.testing.expect(std.mem.indexOf(u8, text, "load_ptr_tko weak") != null); + try std.testing.expect(std.mem.indexOf(u8, text, "addf") != null); + + const parsed = try mlir.Module.parse(ctx, text); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "views, control flow and mma verify" { + const ctx = try testContext(); + defer ctx.deinit(); + + const loc: *const mlir.Location = .unknown(ctx); + const module_op: *mlir.Module = .init(loc); + defer module_op.deinit(); + + const bf16_ty = mlir.Type.float(ctx, .bf16); + const f32_ty = mlir.Type.float(ctx, .f32); + const i32_ty = mlir.Type.int(ctx, .i32); + const bf16_ptr = cuda_tile.tileType(ctx, &.{}, cuda_tile.pointerType(ctx, bf16_ty)); + const f32_ptr = cuda_tile.tileType(ctx, &.{}, cuda_tile.pointerType(ctx, f32_ty)); + const block = mlir.Block.init(&.{ bf16_ptr, bf16_ptr, f32_ptr }, &.{ loc, loc, loc }); + + // C[128x128] = A[128x64] @ B[64x128] in 64-wide K tiles. + const tv_a = cuda_tile.make_tensor_view(ctx, block.argument(0), &.{}, &.{}, cuda_tile.tensorViewType(ctx, bf16_ty, &.{ 128, 64 }, &.{ 64, 1 }), loc).appendTo(block); + const tv_b = cuda_tile.make_tensor_view(ctx, block.argument(1), &.{}, &.{}, cuda_tile.tensorViewType(ctx, bf16_ty, &.{ 64, 128 }, &.{ 128, 1 }), loc).appendTo(block); + const tv_c = cuda_tile.make_tensor_view(ctx, block.argument(2), &.{}, &.{}, cuda_tile.tensorViewType(ctx, f32_ty, &.{ 128, 128 }, &.{ 128, 1 }), loc).appendTo(block); + const pv_a: *const mlir.Type = @ptrCast(cuda_tile.PartitionViewType.get(ctx, .{ .tile_shape = &.{ 64, 64 }, .tensor_view = tv_a.result(0).type_(), .padding = .zero })); + const pv_b: *const mlir.Type = @ptrCast(cuda_tile.PartitionViewType.get(ctx, .{ .tile_shape = &.{ 64, 64 }, .tensor_view = tv_b.result(0).type_(), .padding = .zero })); + const pv_c: *const mlir.Type = @ptrCast(cuda_tile.PartitionViewType.get(ctx, .{ .tile_shape = &.{ 64, 64 }, .tensor_view = tv_c.result(0).type_() })); + const va = cuda_tile.make_partition_view(ctx, tv_a.result(0), pv_a, loc).appendTo(block); + const vb = cuda_tile.make_partition_view(ctx, tv_b.result(0), pv_b, loc).appendTo(block); + const vc = cuda_tile.make_partition_view(ctx, tv_c.result(0), pv_c, loc).appendTo(block); + + const bid = cuda_tile.get_tile_block_id(ctx, loc).appendTo(block); + const i32_tile = cuda_tile.tileType(ctx, &.{}, i32_ty); + const zero = cuda_tile.constant(ctx, cuda_tile.splatAttr(i32_tile, cuda_tile.intElemAttr(i32_ty, 0)), i32_tile, loc).appendTo(block); + const one = cuda_tile.constant(ctx, cuda_tile.splatAttr(i32_tile, cuda_tile.intElemAttr(i32_ty, 1)), i32_tile, loc).appendTo(block); + const acc_ty = cuda_tile.tileType(ctx, &.{ 64, 64 }, f32_ty); + const acc0 = cuda_tile.constant(ctx, cuda_tile.splatAttr(acc_ty, cuda_tile.floatElemAttr(ctx, f32_ty, 0.0)), acc_ty, loc).appendTo(block); + + const body = mlir.Block.init(&.{ i32_tile, acc_ty }, &.{ loc, loc }); + const at = cuda_tile.load_view_tko(ctx, va.result(0), &.{ bid.result(0), body.argument(0) }, .{}, loc).appendTo(body); + const bt = cuda_tile.load_view_tko(ctx, vb.result(0), &.{ body.argument(0), bid.result(1) }, .{}, loc).appendTo(body); + try std.testing.expect(at.attributeByName("inbounds").?.eql(.denseArray(ctx, .bool, &.{ 0, 0 }))); + const acc = cuda_tile.mmaf(ctx, at.result(0), bt.result(0), body.argument(1), false, loc).appendTo(body); + _ = cuda_tile.continue_(ctx, &.{acc.result(0)}, loc).appendTo(body); + const loop_op = cuda_tile.for_(ctx, zero.result(0), one.result(0), one.result(0), &.{acc0.result(0)}, body, false, loc).appendTo(block); + + // if %cond -> (tile) { yield } else { yield } + const cond = cuda_tile.cmpi(ctx, .equal, .signed, bid.result(0), zero.result(0), loc).appendTo(block); + const then_block = mlir.Block.init(&.{}, &.{}); + _ = cuda_tile.yield(ctx, &.{loop_op.result(0)}, loc).appendTo(then_block); + const else_block = mlir.Block.init(&.{}, &.{}); + _ = cuda_tile.yield(ctx, &.{acc0.result(0)}, loc).appendTo(else_block); + const if_op = cuda_tile.if_(ctx, cond.result(0), &.{acc_ty}, then_block, .{ .else_block = else_block }, loc).appendTo(block); + + const store = cuda_tile.store_view_tko(ctx, if_op.result(0), vc.result(0), &.{ bid.result(0), bid.result(1) }, .{}, loc).appendTo(block); + try std.testing.expect(store.attributeByName("inbounds").?.eql(.denseArray(ctx, .bool, &.{ 0, 0 }))); + _ = cuda_tile.return_(ctx, &.{}, loc).appendTo(block); + + const ct_body = mlir.Block.init(&.{}, &.{}); + _ = cuda_tile.entry(ctx, .{ + .name = "gemm", + .block = block, + .optimization_hints = cuda_tile.optimizationHintsAttr(ctx, &[_]cuda_tile.EntryHint{.{ .arch = .sm_120, .num_worker_warps_per_cta = 4 }}), + .location = loc, + }).appendTo(ct_body); + _ = cuda_tile.module(ctx, "m", ct_body, loc).appendTo(module_op.body()); + + try std.testing.expect(module_op.operation().verify()); + + var al: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer al.deinit(); + try al.writer.print("{f}", .{module_op.operation()}); + const text = al.written(); + try std.testing.expect(std.mem.indexOf(u8, text, "optimization_hints=") != null); + try std.testing.expect(std.mem.indexOf(u8, text, "load_view_tko weak") != null); + try std.testing.expect(std.mem.indexOf(u8, text, "continue ") != null); + + const parsed = try mlir.Module.parse(ctx, text); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "the verifier rejects what the dialect forbids" { + const ctx = try testContext(); + defer ctx.deinit(); + const loc: *const mlir.Location = .unknown(ctx); + const f32_ty = mlir.Type.float(ctx, .f32); + + // An entry with a rank-1 argument. (Per-op negatives go through + // `Operation.try_make`, whose logged error the test runner counts as a + // failure, so the module-level verifier is the oracle here.) + const module_op: *mlir.Module = .init(loc); + defer module_op.deinit(); + const block = mlir.Block.init(&.{cuda_tile.tileType(ctx, &.{4}, f32_ty)}, &.{loc}); + _ = cuda_tile.return_(ctx, &.{}, loc).appendTo(block); + const ct_body = mlir.Block.init(&.{}, &.{}); + _ = cuda_tile.entry(ctx, .{ .name = "bad", .block = block, .location = loc }).appendTo(ct_body); + _ = cuda_tile.module(ctx, "m", ct_body, loc).appendTo(module_op.body()); + try std.testing.expect(!module_op.operation().verify()); +} + +test "every bound operation is registered" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + cuda_tile.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} diff --git a/mlir/dialects/cute_ir/BUILD.bazel b/mlir/dialects/cute_ir/BUILD.bazel new file mode 100644 index 00000000..c584509f --- /dev/null +++ b/mlir/dialects/cute_ir/BUILD.bazel @@ -0,0 +1,211 @@ +load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library", "td_library") +load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +package(default_visibility = ["//visibility:public"]) + +cc_library( + name = "capi", + srcs = ["capi.cc"], + hdrs = ["capi.h"], + tags = ["manual"], + deps = [ + ":dialect", + ":headers", + "//mlir/dialects/cute_nvgpu:dialect", + "@llvm-project//mlir:CAPIIR", + ], +) + +zig_library( + name = "cute_ir", + import_name = "mlir/dialects/cute_ir", + main = "cute_ir.zig", + tags = ["manual"], + deps = [ + ":capi", + "//mlir", + "//stdx", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":cute_ir", + "//mlir", + "//mlir/dialects/cute_nvgpu", + ], +) + +td_library( + name = "td_files", + srcs = [ + "CuteAttrsAll.td", + "CuteAttrsCompiler.td", + "CuteEnums.td", + "CuteOps.td", + "CuteTypesAll.td", + "CuteTypesCompiler.td", + ], + includes = ["/"], + deps = [ + "@cute_ir//:cute_ir_dialect_td", + "@llvm-project//mlir:InferTypeOpInterfaceTdFiles", + "@llvm-project//mlir:OpBaseTdFiles", + "@llvm-project//mlir:SideEffectInterfacesTdFiles", + ], +) + +gentbl_cc_library( + name = "enums_inc_gen", + tbl_outs = [ + ( + ["-gen-enum-decls"], + "CuteEnums.h.inc", + ), + ( + ["-gen-enum-defs"], + "CuteEnums.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "CuteEnums.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "attrs_inc_gen", + tbl_outs = [ + ( + [ + "-gen-attrdef-decls", + "-attrdefs-dialect=cute", + ], + "CuteAttrDecls.h.inc", + ), + ( + [ + "-gen-attrdef-defs", + "-attrdefs-dialect=cute", + ], + "CuteAttrs.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "CuteAttrsAll.td", + deps = [":td_files"], +) + +genrule( + name = "attrs_h_inc", + srcs = ["CuteAttrDecls.h.inc"], + outs = ["CuteAttrs.h.inc"], + cmd = "(echo '#ifdef GET_ATTRDEF_CLASSES'; " + + "echo '#include \"cute_ir/Dialect/Cute/IR/CuteEnums.h.inc\"'; " + + "echo '#endif'; cat $<) > $@", +) + +gentbl_cc_library( + name = "types_inc_gen", + tbl_outs = [ + ( + [ + "-gen-typedef-decls", + "-typedefs-dialect=cute", + ], + "CuteTypes.h.inc", + ), + ( + [ + "-gen-typedef-defs", + "-typedefs-dialect=cute", + ], + "CuteTypes.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "CuteTypesAll.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "ops_inc_gen", + tbl_outs = [ + ( + ["-gen-op-decls"], + "CuteOps.h.inc", + ), + ( + ["-gen-op-defs"], + "CuteOps.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "CuteOps.td", + deps = [":td_files"], +) + +cc_library( + name = "headers", + hdrs = [ + "CuteAttrs.h.inc", + "CuteInference.h", + ], + deps = [ + ":attrs_inc_gen", + ":enums_inc_gen", + ":ops_inc_gen", + ":types_inc_gen", + ":upstream_generated_headers", + "@cute_ir//:cute_ir_dialect_header", + "@cute_ir//:cutegen", + "@llvm-project//mlir:DataLayoutInterfaces", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:InferTypeOpInterface", + "@llvm-project//mlir:SideEffectInterfaces", + ], +) + +cc_library( + name = "dialect", + srcs = [ + "CuteInference.cpp", + "CuteOps.cpp", + "CuteTypes.cpp", + "@cute_ir//:cute_ir/lib/Dialect/IR/CuteAttributes.cpp", + "@cute_ir//:cute_ir/lib/Dialect/IR/CuteDialect.cpp", + ], + deps = [ + ":headers", + "//mlir/dialects/cute_nvgpu:headers", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:ArithDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:InferTypeOpInterface", + "@llvm-project//mlir:LLVMDialect", + "@llvm-project//mlir:MathDialect", + "@llvm-project//mlir:NVVMDialect", + "@llvm-project//mlir:SCFDialect", + "@llvm-project//mlir:UBDialect", + ], +) + +# Upstream CuTe sources retain their generated-header include paths. +cc_library( + name = "upstream_generated_headers", + hdrs = [ + "CuteAttrDecls.h.inc", + "CuteAttrs.cpp.inc", + "CuteAttrs.h.inc", + "CuteEnums.cpp.inc", + "CuteEnums.h.inc", + "CuteOps.cpp.inc", + "CuteOps.h.inc", + "CuteTypes.cpp.inc", + "CuteTypes.h.inc", + ], + include_prefix = "cute_ir/Dialect/Cute/IR", + strip_include_prefix = ".", +) diff --git a/mlir/dialects/cute_ir/CuteAttrsAll.td b/mlir/dialects/cute_ir/CuteAttrsAll.td new file mode 100644 index 00000000..3f42a1cb --- /dev/null +++ b/mlir/dialects/cute_ir/CuteAttrsAll.td @@ -0,0 +1,14 @@ +//===- CuteAttrsAll.td - all cute attributes ---------------*- tablegen -*-===// +// +// The dialect has one generated attribute parser and printer, so the +// release's attributes, the compiler's and its enum attributes are generated +// together from here. + +#ifndef MLIR_DIALECTS_CUTE_IR_CUTEATTRSALL_TD +#define MLIR_DIALECTS_CUTE_IR_CUTEATTRSALL_TD + +include "cute_ir/Dialect/Cute/IR/CuteAttrs.td" +include "mlir/dialects/cute_ir/CuteAttrsCompiler.td" +include "mlir/dialects/cute_ir/CuteEnums.td" + +#endif // MLIR_DIALECTS_CUTE_IR_CUTEATTRSALL_TD diff --git a/mlir/dialects/cute_ir/CuteAttrsCompiler.td b/mlir/dialects/cute_ir/CuteAttrsCompiler.td new file mode 100644 index 00000000..874763a5 --- /dev/null +++ b/mlir/dialects/cute_ir/CuteAttrsCompiler.td @@ -0,0 +1,220 @@ +// The cute dialect's attributes beyond the release, generated from the +// nvidia-cutlass-dsl 4.8.0 release; do not edit. +// +// Sources: the compiler library (`_cutlass_ir`): its generated parser +// messages give every class, its parameters in declaration order and their +// C++ types; the live compiler, loaded from Python, gives the syntax (what it +// prints for what it parses) and the verifier messages. +// +// The fragment parameters are `FragmentAttrInterface` in the compiler, an +// interface neither the release nor this tree defines: they are plain +// attributes here (the cute_nvgpu `*_frg` attributes). +// +// The reduction_op enum attribute is in CuteEnums.td. + +#ifndef MLIR_DIALECTS_CUTE_IR_CUTEATTRSCOMPILER_TD +#define MLIR_DIALECTS_CUTE_IR_CUTEATTRSCOMPILER_TD + +include "cute_ir/Dialect/Cute/IR/CuteAttrs.td" + +def Cute_BitLayoutAttr : Cute_Attr<"BitLayout", "bitlayout"> { + let summary = "Bit layout of a pointer's elements within a storage chunk"; + let description = [{ + A two-mode layout from (element bit, element index) to the bit offset in + a storage chunk. Used by `!cute.ptr` (`bitlayout<"...">`). + + ```mlir + #cute.bitlayout<<"(8,4):(1,8)">> + ``` + }]; + let parameters = (ins + Cute_Param<"LayoutAttr", "(element bit, element index) -> bit">:$layout + ); + let assemblyFormat = "`<` $layout `>`"; + let genVerifyDecl = 1; +} + +def Cute_CopyAtomAttr : Cute_Attr<"CopyAtom", "copy_atom"> { + let summary = "Thread and value layouts of a copy atom"; + let description = [{ + ```mlir + #cute.copy_atom, layout_tv = (<"(1,8):(0,1)">, <"(1,8):(0,1)">, <"(1,8):(0,1)">)> + ``` + }]; + let parameters = (ins + Cute_Param<"LayoutAttr", "thread id layout">:$thr_id, + Cute_Param<"LayoutAttr", "source thread-value layout">:$layout_src, + Cute_Param<"LayoutAttr", "destination thread-value layout">:$layout_dst, + Cute_Param<"LayoutAttr", "reference thread-value layout">:$layout_ref + ); + let assemblyFormat = [{ + `<` `thr_id` `=` $thr_id `,` `layout_tv` `=` + `(` $layout_src `,` $layout_dst `,` $layout_ref `)` `>` + }]; +} + +def Cute_CopyAtomV2Attr : Cute_Attr<"CopyAtomV2", "copy_atom_v2"> { + let summary = "Thread and value layouts and fragments of a V2 copy atom"; + let description = [{ + ```mlir + #cute.copy_atom_v2, layout_tv = (<"(1,8):(0,1)">, <"(1,8):(0,1)">), frg = (#cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>)> + ``` + }]; + let parameters = (ins + Cute_Param<"LayoutAttr", "thread id layout">:$thr_id, + Cute_Param<"LayoutAttr", "source thread-value layout">:$layoutSrc_TV, + Cute_Param<"LayoutAttr", "destination thread-value layout">:$layoutDst_TV, + Cute_Param<"mlir::Attribute", "source fragment">:$frgSrc, + Cute_Param<"mlir::Attribute", "destination fragment">:$frgDst + ); + let assemblyFormat = [{ + `<` `thr_id` `=` $thr_id `,` `layout_tv` `=` + `(` $layoutSrc_TV `,` $layoutDst_TV `)` `,` + `frg` `=` `(` $frgSrc `,` $frgDst `)` `>` + }]; +} + +def Cute_MmaAtomAttr : Cute_Attr<"MmaAtom", "mma_atom"> { + let summary = "Shapes, thread-value layouts and fragments of an MMA atom"; + let description = [{ + ```mlir + #cute.mma_atom<<"(16,8,16)">, <"32:1">, layout_TV = (<"1:0">, <"1:0">, <"1:0">), shapes = (<"(16,16)">, <"(8,16)">, <"(16,8)">) frgs = (#cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>)> + ``` + }]; + let parameters = (ins + Cute_Param<"ShapeAttr", "MNK shape">:$shape_MNK, + Cute_Param<"LayoutAttr", "thread id layout">:$thr_id, + Cute_Param<"LayoutAttr", "A thread-value layout">:$layoutA_TV, + Cute_Param<"LayoutAttr", "B thread-value layout">:$layoutB_TV, + Cute_Param<"LayoutAttr", "C thread-value layout">:$layoutC_TV, + Cute_Param<"ShapeAttr", "A shape (MK)">:$shapeA_MK, + Cute_Param<"ShapeAttr", "B shape (NK)">:$shapeB_NK, + Cute_Param<"ShapeAttr", "C shape (MN)">:$shapeC_MN, + Cute_Param<"mlir::Attribute", "A fragment">:$frgA, + Cute_Param<"mlir::Attribute", "B fragment">:$frgB, + Cute_Param<"mlir::Attribute", "C fragment">:$frgC + ); + let assemblyFormat = [{ + `<` $shape_MNK `,` $thr_id `,` + `layout_TV` `=` `(` $layoutA_TV `,` $layoutB_TV `,` $layoutC_TV `)` `,` + `shapes` `=` `(` $shapeA_MK `,` $shapeB_NK `,` $shapeC_MN `)` + `frgs` `=` `(` $frgA `,` $frgB `,` $frgC `)` `>` + }]; +} + +def Cute_MxMmaAtomAttr : Cute_Attr<"MxMmaAtom", "mx_mma_atom"> { + let summary = "MMA atom attribute with scale-factor operands"; + let description = [{ + Operands in the order A, SFA, B, SFB, C. + + ```mlir + #cute.mx_mma_atom<<"(16,8,16)">, <"32:1">, layout_TV = (<"1:0">, <"1:0">, <"1:0">, <"1:0">, <"1:0">), shapes = (<"(16,16)">, <"(8,16)">, <"(16,8)">, <"1">, <"2">) frgs = (#cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>)> + ``` + }]; + let parameters = (ins + Cute_Param<"ShapeAttr", "MNK shape">:$shape_MNK, + Cute_Param<"LayoutAttr", "thread id layout">:$thr_id, + Cute_Param<"LayoutAttr", "A thread-value layout">:$layoutA_TV, + Cute_Param<"LayoutAttr", "SFA thread-value layout">:$layoutSFA_TV, + Cute_Param<"LayoutAttr", "B thread-value layout">:$layoutB_TV, + Cute_Param<"LayoutAttr", "SFB thread-value layout">:$layoutSFB_TV, + Cute_Param<"LayoutAttr", "C thread-value layout">:$layoutC_TV, + Cute_Param<"ShapeAttr", "A shape (MK)">:$shapeA_MK, + Cute_Param<"ShapeAttr", "SFA shape (MK)">:$shapeSFA_MK, + Cute_Param<"ShapeAttr", "B shape (NK)">:$shapeB_NK, + Cute_Param<"ShapeAttr", "SFB shape (NK)">:$shapeSFB_NK, + Cute_Param<"ShapeAttr", "C shape (MN)">:$shapeC_MN, + Cute_Param<"mlir::Attribute", "A fragment">:$frgA, + Cute_Param<"mlir::Attribute", "SFA fragment">:$frgSFA, + Cute_Param<"mlir::Attribute", "B fragment">:$frgB, + Cute_Param<"mlir::Attribute", "SFB fragment">:$frgSFB, + Cute_Param<"mlir::Attribute", "C fragment">:$frgC + ); + let assemblyFormat = [{ + `<` $shape_MNK `,` $thr_id `,` + `layout_TV` `=` `(` $layoutA_TV `,` $layoutSFA_TV `,` $layoutB_TV `,` + $layoutSFB_TV `,` $layoutC_TV `)` `,` + `shapes` `=` `(` $shapeA_MK `,` $shapeSFA_MK `,` $shapeB_NK `,` + $shapeSFB_NK `,` $shapeC_MN `)` + `frgs` `=` `(` $frgA `,` $frgSFA `,` $frgB `,` $frgSFB `,` $frgC `)` `>` + }]; +} + +def Cute_SparseMmaAtomAttr : Cute_Attr<"SparseMmaAtom", "sparse_mma_atom"> { + let summary = "MMA atom attribute with a sparsity-metadata operand"; + let description = [{ + Operands in the order A, B, C, E. + + ```mlir + #cute.sparse_mma_atom<<"(16,8,16)">, <"32:1">, layout_TV = (<"1:0">, <"1:0">, <"1:0">, <"1:0">), shapes = (<"(16,16)">, <"(8,16)">, <"(16,8)">, <"1">) frgs = (#cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>)> + ``` + }]; + let parameters = (ins + Cute_Param<"ShapeAttr", "MNK shape">:$shape_MNK, + Cute_Param<"LayoutAttr", "thread id layout">:$thr_id, + Cute_Param<"LayoutAttr", "A thread-value layout">:$layoutA_TV, + Cute_Param<"LayoutAttr", "B thread-value layout">:$layoutB_TV, + Cute_Param<"LayoutAttr", "C thread-value layout">:$layoutC_TV, + Cute_Param<"LayoutAttr", "E thread-value layout">:$layoutE_TV, + Cute_Param<"ShapeAttr", "A shape (MK)">:$shapeA_MK, + Cute_Param<"ShapeAttr", "B shape (NK)">:$shapeB_NK, + Cute_Param<"ShapeAttr", "C shape (MN)">:$shapeC_MN, + Cute_Param<"ShapeAttr", "E shape (MK)">:$shapeE_MK, + Cute_Param<"mlir::Attribute", "A fragment">:$frgA, + Cute_Param<"mlir::Attribute", "B fragment">:$frgB, + Cute_Param<"mlir::Attribute", "C fragment">:$frgC, + Cute_Param<"mlir::Attribute", "E fragment">:$frgE + ); + let assemblyFormat = [{ + `<` $shape_MNK `,` $thr_id `,` + `layout_TV` `=` `(` $layoutA_TV `,` $layoutB_TV `,` $layoutC_TV `,` + $layoutE_TV `)` `,` + `shapes` `=` `(` $shapeA_MK `,` $shapeB_NK `,` $shapeC_MN `,` $shapeE_MK `)` + `frgs` `=` `(` $frgA `,` $frgB `,` $frgC `,` $frgE `)` `>` + }]; +} + +def Cute_SparseMxMmaAtomAttr + : Cute_Attr<"SparseMxMmaAtom", "sparse_mx_mma_atom"> { + let summary = "MMA atom attribute with scale-factor and metadata operands"; + let description = [{ + Operands in the order A, SFA, B, SFB, C, E. + + ```mlir + #cute.sparse_mx_mma_atom<<"(16,8,16)">, <"32:1">, layout_TV = (<"1:0">, <"2:0">, <"3:0">, <"4:0">, <"5:0">, <"6:0">), shapes = (<"1">, <"2">, <"3">, <"4">, <"5">, <"6">) frgs = (#cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>)> + ``` + }]; + let parameters = (ins + Cute_Param<"ShapeAttr", "MNK shape">:$shape_MNK, + Cute_Param<"LayoutAttr", "thread id layout">:$thr_id, + Cute_Param<"LayoutAttr", "A thread-value layout">:$layoutA_TV, + Cute_Param<"LayoutAttr", "SFA thread-value layout">:$layoutSFA_TV, + Cute_Param<"LayoutAttr", "B thread-value layout">:$layoutB_TV, + Cute_Param<"LayoutAttr", "SFB thread-value layout">:$layoutSFB_TV, + Cute_Param<"LayoutAttr", "C thread-value layout">:$layoutC_TV, + Cute_Param<"LayoutAttr", "E thread-value layout">:$layoutE_TV, + Cute_Param<"ShapeAttr", "A shape (MK)">:$shapeA_MK, + Cute_Param<"ShapeAttr", "SFA shape (MK)">:$shapeSFA_MK, + Cute_Param<"ShapeAttr", "B shape (NK)">:$shapeB_NK, + Cute_Param<"ShapeAttr", "SFB shape (NK)">:$shapeSFB_NK, + Cute_Param<"ShapeAttr", "C shape (MN)">:$shapeC_MN, + Cute_Param<"ShapeAttr", "E shape (MK)">:$shapeE_MK, + Cute_Param<"mlir::Attribute", "A fragment">:$frgA, + Cute_Param<"mlir::Attribute", "SFA fragment">:$frgSFA, + Cute_Param<"mlir::Attribute", "B fragment">:$frgB, + Cute_Param<"mlir::Attribute", "SFB fragment">:$frgSFB, + Cute_Param<"mlir::Attribute", "C fragment">:$frgC, + Cute_Param<"mlir::Attribute", "E fragment">:$frgE + ); + let assemblyFormat = [{ + `<` $shape_MNK `,` $thr_id `,` + `layout_TV` `=` `(` $layoutA_TV `,` $layoutSFA_TV `,` $layoutB_TV `,` + $layoutSFB_TV `,` $layoutC_TV `,` $layoutE_TV `)` `,` + `shapes` `=` `(` $shapeA_MK `,` $shapeSFA_MK `,` $shapeB_NK `,` + $shapeSFB_NK `,` $shapeC_MN `,` $shapeE_MK `)` + `frgs` `=` `(` $frgA `,` $frgSFA `,` $frgB `,` $frgSFB `,` $frgC `,` + $frgE `)` `>` + }]; +} + +#endif // MLIR_DIALECTS_CUTE_IR_CUTEATTRSCOMPILER_TD diff --git a/mlir/dialects/cute_ir/CuteEnums.td b/mlir/dialects/cute_ir/CuteEnums.td new file mode 100644 index 00000000..68018cc1 --- /dev/null +++ b/mlir/dialects/cute_ir/CuteEnums.td @@ -0,0 +1,70 @@ +// The cute dialect's enums, generated from the nvidia-cutlass-dsl 4.8.0 +// enum bindings (`_cute_enum_gen.py`); do not edit. +// +// Case values and spellings are the compiler's. An enum the bindings build +// as a dialect attribute also has its EnumAttr, printed `#`. + +#ifndef MLIR_DIALECTS_CUTE_IR_CUTEENUMS_TD +#define MLIR_DIALECTS_CUTE_IR_CUTEENUMS_TD + +include "cute_ir/Dialect/Cute/IR/CuteDialect.td" +include "mlir/IR/EnumAttr.td" + +def Cute_AddressSpaceEnum : I32EnumAttr<"AddressSpace", "Address spaces for CuTe memrefs and pointers", [ + I32EnumAttrCase<"generic", 0, "generic">, + I32EnumAttrCase<"gmem", 1, "gmem">, + I32EnumAttrCase<"smem", 3, "smem">, + I32EnumAttrCase<"rmem", 5, "rmem">, + I32EnumAttrCase<"tmem", 6, "tmem">, + I32EnumAttrCase<"dsmem", 7, "dsmem">, + I32EnumAttrCase<"cmem", 4, "cmem"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute"; + let genSpecializedAttr = 0; +} + +def Cute_CopyOperandEnum : I32EnumAttr<"CopyOperand", "Copy operand ID", [ + I32EnumAttrCase<"S", 0, "S">, + I32EnumAttrCase<"D", 1, "D"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute"; + let genSpecializedAttr = 0; +} + +def Cute_MajorModeEnum : I32EnumAttr<"MajorMode", "Major mode for MMA operations", [ + I32EnumAttrCase<"k", 0, "k">, + I32EnumAttrCase<"mn", 1, "mn"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute"; + let genSpecializedAttr = 0; +} + +def Cute_MmaOperandEnum : I32EnumAttr<"MmaOperand", "MMA operand ID", [ + I32EnumAttrCase<"A", 0, "A">, + I32EnumAttrCase<"B", 1, "B">, + I32EnumAttrCase<"C", 2, "C">, + I32EnumAttrCase<"D", 3, "D">, + I32EnumAttrCase<"SFA", 4, "SFA">, + I32EnumAttrCase<"SFB", 5, "SFB">, + I32EnumAttrCase<"E", 6, "E"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute"; + let genSpecializedAttr = 0; +} + +def Cute_ReductionOpEnum : I32EnumAttr<"ReductionOp", "Op for cute reduce operations", [ + I32EnumAttrCase<"ADD", 0, "ADD">, + I32EnumAttrCase<"MUL", 1, "MUL">, + I32EnumAttrCase<"MIN", 2, "MIN">, + I32EnumAttrCase<"MAX", 3, "MAX"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute"; + let genSpecializedAttr = 0; +} + +def Cute_ReductionOpAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute"; + let assemblyFormat = "`<` $value `>`"; +} + +#endif // MLIR_DIALECTS_CUTE_IR_CUTEENUMS_TD diff --git a/mlir/dialects/cute_ir/CuteInference.cpp b/mlir/dialects/cute_ir/CuteInference.cpp new file mode 100644 index 00000000..2f0415e4 --- /dev/null +++ b/mlir/dialects/cute_ir/CuteInference.cpp @@ -0,0 +1,3600 @@ +//===- CuteInference.cpp - result types the DSL compiler would infer ------===// +// +// Every operation the DSL compiler infers result types for, inferred here +// over cutegen, so a module built in this tree carries the types the compiler +// expects. Where the compiler's rule is not recovered for some inputs (other +// atoms, tilings or layouts than those covered), inference fails rather than +// guesses. test:dialect_inference_test holds these to the compiler's choice. + +#include "mlir/dialects/cute_ir/CuteInference.h" + +#include "mlir/dialects/cute_nvgpu/CuteNVGPUDialect.h" + +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/Matchers.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/TypeSwitch.h" + +#include +#include +#include +#include +#include +#include +#include + +using namespace mlir; +using namespace mlir::cutlass_compiler::cute; +namespace cg = cutegen; +namespace cute_nvgpu = mlir::cutlass_compiler::cute_nvgpu; +// The dialect's memref, not the builtin one. +using CuteMemRefType = ::mlir::cutlass_compiler::cute::MemRefType; + +namespace { + +// Composed layouts that turned out plain are typed as plain layouts, as the +// compiler types them. +Type layoutType(MLIRContext *ctx, cg::layout l) { + return LayoutType::get(ctx, std::move(l)); +} +Type layoutType(MLIRContext *ctx, cg::composed_layout l) { + if (l.is_normal_layout()) + return LayoutType::get(ctx, l.layout_b()); + return ComposedLayoutType::get(ctx, std::move(l)); +} + +// `f` runs over the algebra value of a layout or composed layout type; the +// result keeps the input's kind (see rebuild for views). +template Type mapLayout(MLIRContext *ctx, Type layout, F &&f) { + if (auto l = llvm::dyn_cast(layout)) { + auto r = f(l.getRef()); + if (!cg::is_valid(r)) + return {}; + return layoutType(ctx, std::move(r)); + } + if (auto l = llvm::dyn_cast(layout)) { + auto r = f(l.getRef()); + if (!cg::is_valid(r)) + return {}; + return ComposedLayoutType::get(ctx, std::move(r)); + } + return {}; +} + +// What carries a layout: the layout itself, a memref (pointer + layout) or a +// coordinate tensor (arithmetic tuple + layout). +struct Carrier { + Type layout; + PtrType ptr; // memref + IntTupleType iter; // coord_tensor + Type desc; // cute_nvgpu.smem_desc_view +}; + +std::optional carrierOf(Type t) { + if (llvm::isa(t)) + return Carrier{t, {}, {}, {}}; + if (auto m = llvm::dyn_cast(t)) + return Carrier{m.getLayout(), m.getPtr(), {}, {}}; + if (auto c = llvm::dyn_cast(t)) + return Carrier{c.getLayout(), {}, c.getArithTuple(), {}}; + if (auto v = llvm::dyn_cast(t)) + return Carrier{ + LayoutType::get(t.getContext(), v.getLayout()), {}, {}, v.getDesc()}; + return std::nullopt; +} + +Type rebuild(MLIRContext *ctx, const Carrier &c, Type layout) { + if (!layout) + return {}; + // A view's layout is plain when its composed layout turned out plain. + if (auto l = llvm::dyn_cast(layout); + l && (c.ptr || c.iter || c.desc)) + layout = layoutType(ctx, l.getRef()); + if (c.ptr) + return CuteMemRefType::get(ctx, c.ptr, layout); + if (c.iter) + return CoordTensorType::get(ctx, c.iter, layout); + if (c.desc) { + auto l = llvm::dyn_cast(layout); + return l ? cute_nvgpu::SmemDescViewType::get(ctx, c.desc, l.getAttr()) + : Type(); + } + return layout; +} + +// The bit width of an element type; tf32 counts as its 32-bit storage. +int64_t bitsOfType(Type t) { + if (t.isTF32()) + return 32; + return t.isIntOrFloat() ? t.getIntOrFloatBitWidth() : 0; +} + +// The alignment a pointer keeps after moving by `offset` elements: the gcd +// with what is known of the offset, a static value or a divisibility; the +// element's natural alignment when less than a byte is known. +uint64_t movedAlignment(PtrType ptr, const cg::int_tuple &offset) { + Type elem = ptr.getValueType(); + uint64_t bits = elem && elem.isIntOrFloat() ? bitsOfType(elem) : 8; + uint64_t natural = PtrType::getNaturalAlignment(elem); + uint64_t align = ptr.getAlignment() ? ptr.getAlignment() : natural; + uint64_t known = 0; + if (cg::holds_int(offset)) { + int64_t v = offset.as_int64(); + if (v == 0) + return ptr.getAlignment(); + known = static_cast(v < 0 ? -v : v); + } else if (cg::holds_dynamic_int(offset)) { + known = static_cast( + std::get(offset).get_properties().divisibility); + } else { + return natural; + } + uint64_t result = std::gcd(align * 8, known * bits) / 8; + return result ? result : natural; +} + +PtrType movedPtr(PtrType ptr, const cg::int_tuple &offset) { + return PtrType::get(ptr.getContext(), ptr.getValueType(), + ptr.getAddressSpace(), movedAlignment(ptr, offset), + ptr.getSwizzle(), ptr.getBitlayout()); +} + +// A coordinate with its underscores read as zero, for evaluating a layout. +std::optional zeroUnderscores(const cg::coord &c) { + std::string text = cg::to_string(c); + std::replace(text.begin(), text.end(), '_', '0'); + return cg::from_string(text); +} + +// Slicing a carrier: the layout is sliced and the iterator moves by the +// layout evaluated at the coordinate. +Type sliceCarrier(MLIRContext *ctx, const Carrier &c, const cg::coord &coord) { + cg::int_tuple offset; + Type layout; + if (auto l = llvm::dyn_cast(c.layout)) { + auto [sliced, off] = cg::slice_and_offset(coord, l.getRef()); + if (!cg::is_valid(sliced)) + return {}; + layout = layoutType(ctx, std::move(sliced)); + offset = off; + } else if (auto l = llvm::dyn_cast(c.layout)) { + // The composed layout's offset takes the move; the iterator stays. + auto sliced = cg::slice(coord, l.getRef()); + if (!cg::is_valid(sliced)) + return {}; + return rebuild(ctx, c, layoutType(ctx, std::move(sliced))); + } else { + return {}; + } + if (c.ptr) + return CuteMemRefType::get(ctx, movedPtr(c.ptr, offset), layout); + if (c.iter) { + auto sum = cg::arith_tuple_sum(c.iter.getRef(), offset); + if (!cg::is_valid(sum)) + return {}; + return CoordTensorType::get(ctx, IntTupleType::get(ctx, std::move(sum)), + layout); + } + // A descriptor iterator does not move. + return rebuild(ctx, c, layout); +} + +// The compiler writes a stride of 0 on every mode of extent 1 that a divide +// produces; cutegen keeps the stride the algebra gave it. +cg::layout zeroUnitModes(cg::layout l) { + if (cg::holds_leaf(l.shape())) { + if (cg::holds_int(l.shape()) && l.shape().as_int64() == 1) { + if (auto unit = cg::from_string("1:0")) + return *unit; + } + return l; + } + // Shapes and strides are congruent, so walk both spellings together. + auto shape = cg::to_string(l.shape()); + auto stride = cg::to_string(l.stride()); + auto fs = cg::flatten(l.shape()); + auto ft = cg::flatten(l.stride()); + if (cg::rank(fs) != cg::rank(ft)) + return l; + bool changed = false; + for (size_t i = 0; i < cg::rank(fs); ++i) { + if (cg::holds_int(fs[i]) && fs[i].as_int64() == 1 && + !(cg::holds_int(ft[i]) && ft[i].as_int64() == 0)) { + changed = true; + } + } + if (!changed) + return l; + // Rewrite the stride text leaf by leaf, in flattened order. + std::string out; + size_t leaf = 0; + auto stridesAreLeaves = [&](size_t i) { + return cg::holds_int(fs[i]) && fs[i].as_int64() == 1; + }; + size_t i = 0; + while (i < stride.size()) { + char ch = stride[i]; + if (ch == '(' || ch == ')' || ch == ',') { + out += ch; + ++i; + continue; + } + size_t j = i; + int depth = 0; + while (j < stride.size() && + !((stride[j] == ',' || stride[j] == ')') && depth == 0)) { + if (stride[j] == '<' || stride[j] == '{') + ++depth; + if (stride[j] == '>' || stride[j] == '}') + --depth; + ++j; + } + out += stridesAreLeaves(leaf) ? std::string("0") : stride.substr(i, j - i); + ++leaf; + i = j; + } + auto rebuilt = cg::from_string(shape + ":" + out); + return rebuilt ? *rebuilt : l; +} + +const cg::layout &outerOf(const cg::layout &l) { return l; } +cg::layout outerOf(const cg::composed_layout &l) { return l.layout_b(); } + +cg::composed_layout zeroUnitModes(cg::composed_layout l) { + if (l.is_a_swizzle()) + return cg::composed_layout(l.swizzle_a(), l.offset(), + zeroUnitModes(l.layout_b())); + return cg::composed_layout(l.layout_a(), l.offset(), + zeroUnitModes(l.layout_b())); +} + +template +LogicalResult infer(std::optional loc, SmallVectorImpl &out, + F &&f) { + Type t = f(); + if (!t) + return emitOptionalError(loc, "cannot infer the result type"); + out.push_back(t); + return success(); +} + +// `like`'s nesting with its leaves replaced, in order, by `leaves`. +template +std::string nestLike(const T &like, llvm::ArrayRef leaves, + size_t &i) { + if (!cg::holds_vector(like)) + return i < leaves.size() ? leaves[i++] : "x"; + std::string s = "("; + for (size_t k = 0; k < cg::rank(like); ++k) + s += (k ? "," : "") + nestLike(like[k], leaves, i); + return s + ")"; +} + +template void leavesOf(const T &t, std::vector &out) { + if (!cg::holds_vector(t)) + return out.push_back(cg::to_string(t)); + for (size_t k = 0; k < cg::rank(t); ++k) + leavesOf(t[k], out); +} + +bool isOne(const std::string &leaf) { return leaf == "1"; } + +// The plain layout of a layout, or the outer layout of a composed one. +std::optional outerLayout(Type t) { + if (auto l = llvm::dyn_cast(t)) + return l.getRef(); + if (auto c = llvm::dyn_cast(t)) + return c.getRef().layout_b(); + return std::nullopt; +} + +// The plain layout of a layout or swizzled composed layout. +std::optional swizzledOuter(Type t) { + if (auto l = llvm::dyn_cast(t)) + return l.getRef(); + auto c = llvm::dyn_cast(t); + if (c && c.getRef().is_a_swizzle()) + return c.getRef().layout_b(); + return std::nullopt; +} + +// A dynamic integer's spelling: `?`, with its width when 64 and its +// divisibility when above 1. +std::string dynamicInt(int64_t div, bool wide) { + std::string props = std::string(wide ? "i64" : "") + + (wide && div > 1 ? " " : "") + + (div > 1 ? "div=" + std::to_string(div) : ""); + return props.empty() ? "?" : "?{" + props + "}"; +} + +// idx2crd of a scalar `idx` over `shape` (column-major), keeping what a +// dynamic index's divisibility tells of each coordinate, as the compiler +// does (cutegen drops it): a divisible mode's coordinate is 0; a mode whose +// extent the divisibility divides keeps it; the rest know nothing. +std::optional splitIndex(const cg::int_tuple &idx, + const cg::shape &shape) { + std::vector leaves, crd; + leavesOf(shape, leaves); + bool known = cg::holds_int(idx); + if (!known && !cg::holds_dynamic_int(idx)) + return std::nullopt; + int64_t v = known ? idx.as_int64() : 0, div = 1; + bool wide = false; + if (!known) { + const auto &p = std::get(idx).get_properties(); + div = p.divisibility; + wide = p.width == 64; + } + for (size_t i = 0; i < leaves.size(); ++i) { + int64_t n = 0; + bool staticExtent = !StringRef(leaves[i]).getAsInteger(10, n) && n > 0; + bool last = i + 1 == leaves.size(); + if (known) { + if (!staticExtent) + return std::nullopt; + crd.push_back(std::to_string(last ? v : v % n)); + v /= n; + } else if (last) { + crd.push_back(dynamicInt(div, wide)); + } else if (staticExtent && div % n == 0) { + crd.push_back("0"); + div /= n; + } else { + crd.push_back(dynamicInt(staticExtent && n % div == 0 ? div : 1, wide)); + div = 1; + } + } + size_t i = 0; + return nestLike(shape, crd, i); +} +} // namespace + +// ---------------------------------------------------------------- cute + +LogicalResult AddOffsetOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto offset = llvm::dyn_cast(a.getOffset().getType()); + if (!offset) + return emitOptionalError(loc, "offset must be an int tuple"); + return infer(loc, out, [&]() -> Type { + Type src = a.getSrc().getType(); + if (auto p = llvm::dyn_cast(src)) + return movedPtr(p, offset.getRef()); + if (auto it = llvm::dyn_cast(src)) { + auto sum = cg::arith_tuple_sum(it.getArithTuple().getRef(), + offset.getRef()); + if (!cg::is_valid(sum)) + return {}; + return ArithTupleIteratorType::get(ctx, IntTupleType::get(ctx, sum)); + } + return {}; + }); +} + +namespace { +// Pads `input` to `rank` modes with `element`, after (append) or before. +template +LogicalResult inferToRank(MLIRContext *ctx, std::optional loc, + ValueRange operands, DictionaryAttr attrs, + PropertyRef props, SmallVectorImpl &out, + bool append) { + typename Op::Adaptor a(operands, attrs, props); + int32_t rank = a.getRank(); + Type in = a.getInput().getType(), e = a.getElement().getType(); + return infer(loc, out, [&]() -> Type { + if (rank <= 0) + return {}; + return llvm::TypeSwitch(in) + .Case([&](auto ty) -> Type { + using T = decltype(ty); + // A composed layout is padded with plain layouts. + using E = std::conditional_t, + LayoutType, T>; + auto el = llvm::dyn_cast(e); + if (!el) + return {}; + auto r = append + ? cg::append_to_rank_N(rank, ty.getRef(), el.getRef()) + : cg::prepend_to_rank_N(rank, ty.getRef(), el.getRef()); + if (!cg::is_valid(r)) + return {}; + return T::get(ctx, std::move(r)); + }) + .Default([](Type) -> Type { return {}; }); + }); +} +} // namespace + +LogicalResult AppendToRankOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + return inferToRank(ctx, loc, operands, attrs, props, out, + true); +} + +LogicalResult PrependToRankOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + return inferToRank(ctx, loc, operands, attrs, props, out, + false); +} + +Type mlir::cutlass_compiler::cute::assumeResultType(Type src) { + auto constrained = llvm::dyn_cast(src); + for (unsigned width : {64u, 32u}) + if (src.isInteger(width) || + (constrained && constrained.getWidth() == width)) + return ConstrainedIntType::get(src.getContext(), 1, width, false); + return {}; +} + +LogicalResult CeilDivOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + Type in = a.getInput().getType(), tiler = a.getTiler().getType(); + return infer(loc, out, [&]() -> Type { + // cutegen divides shapes; int tuples go through shape and back. + cg::shape shape; + bool tuple = false; + if (auto s = llvm::dyn_cast(in)) { + shape = s.getRef(); + } else if (auto t = llvm::dyn_cast(in)) { + shape = cg::rec_var_cast(t.getRef()); + tuple = true; + } else { + return {}; + } + cg::shape r; + if (auto t = llvm::dyn_cast(tiler)) { + r = cg::ceil_div(shape, t.getRef()); + } else if (auto t = llvm::dyn_cast(tiler)) { + r = cg::ceil_div(shape, t.getRef()); + } else if (auto t = llvm::dyn_cast(tiler)) { + r = cg::ceil_div(shape, cg::rec_var_cast(t.getRef())); + } else { + return {}; + } + if (!cg::is_valid(r)) + return {}; + if (tuple) + return IntTupleType::get(ctx, cg::rec_var_cast(r)); + return ShapeType::get(ctx, std::move(r)); + }); +} + +LogicalResult CoalesceOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto c = carrierOf(a.getInput().getType()); + if (!c) + return emitOptionalError(loc, "input carries no layout"); + CoordType profile; + if (a.getTargetProfile()) + profile = llvm::dyn_cast(a.getTargetProfile().getType()); + return infer(loc, out, [&]() -> Type { + return rebuild(ctx, *c, mapLayout(ctx, c->layout, [&](const auto &l) { + return profile ? cg::coalesce(l, profile.getRef()) + : cg::coalesce(l); + })); + }); +} + +LogicalResult ComposedGetOuterOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { + return infer(loc, out, [&]() -> Type { + auto c = llvm::dyn_cast(operands[0].getType()); + if (!c) + return {}; + return LayoutType::get(ctx, c.getRef().layout_b()); + }); +} + +LogicalResult CompositionOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto c = carrierOf(a.getLhs().getType()); + if (!c) + return emitOptionalError(loc, "lhs carries no layout"); + Type rhs = a.getRhs().getType(); + return infer(loc, out, [&]() -> Type { + return rebuild(ctx, *c, + llvm::TypeSwitch(rhs) + .Case([&](auto r) { + return mapLayout(ctx, c->layout, [&](const auto &l) { + return cg::composition(l, r.getRef()); + }); + }) + .Default([](Type) -> Type { return {}; })); + }); +} + +LogicalResult Crd2IdxOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto coord = llvm::dyn_cast(a.getCoord().getType()); + if (!coord) + return emitOptionalError(loc, "coord must be a coord"); + return infer(loc, out, [&]() -> Type { + auto c = zeroUnderscores(coord.getRef()); + if (!c) + return {}; + Type layout = a.getLayout().getType(); + cg::int_tuple idx; + if (auto l = llvm::dyn_cast(layout)) { + idx = cg::layout_eval(*c, l.getRef()); + } else if (auto s = llvm::dyn_cast(layout)) { + idx = cg::crd2idx(*c, s.getRef()); + } else { + return {}; + } + if (!cg::is_valid(idx)) + return {}; + return IntTupleType::get(ctx, std::move(idx)); + }); +} + +LogicalResult DereferenceArithTupleIteratorOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { + return infer(loc, out, [&]() -> Type { + auto it = llvm::dyn_cast(operands[0].getType()); + return it ? it.getArithTuple() : Type{}; + }); +} + +LogicalResult DiceOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, PropertyRef props, + RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + const cg::coord &coord = a.getCoordAttr().getRef(); + Type in = a.getInput().getType(); + return infer(loc, out, [&]() -> Type { + if (auto c = carrierOf(in)) { + return rebuild(ctx, *c, mapLayout(ctx, c->layout, [&](const auto &l) { + return cg::dice(coord, l); + })); + } + return llvm::TypeSwitch(in) + .Case( + [&](auto ty) -> Type { + auto r = cg::dice(coord, ty.getRef()); + if (!cg::is_valid(r)) + return {}; + return decltype(ty)::get(ctx, std::move(r)); + }) + .Default([](Type) -> Type { return {}; }); + }); +} + +Type mlir::cutlass_compiler::cute::fastDivmodComputeResultType(Type divisor) { + auto d = llvm::dyn_cast(divisor); + return d ? IntegerType::get(divisor.getContext(), d.getWidth()) : Type(); +} + +Type mlir::cutlass_compiler::cute::fastDivmodCreateDivisorResultType( + Type divisor) { + auto i = llvm::dyn_cast(divisor); + return i ? FastDivmodDivisorType::get(divisor.getContext(), i.getWidth(), + false) + : Type(); +} + +LogicalResult FilterZerosOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto c = carrierOf(a.getInput().getType()); + if (!c) + return emitOptionalError(loc, "input carries no layout"); + return infer(loc, out, [&]() -> Type { + return rebuild(ctx, *c, mapLayout(ctx, c->layout, [&](const auto &l) { + return cg::filter_zeros(l); + })); + }); +} + +namespace { +// zeroUnitModes on all but the last `keep` modes (of mode 1 when `zipped`): +// the modes a divide leaves untouched keep their strides. +cg::layout zeroUnitModesHead(const cg::layout &l, size_t keep, bool zipped) { + if (!keep || !cg::holds_vector(l.shape())) + return zeroUnitModes(l); + if (zipped) { + if (cg::rank(l.shape()) != 2) + return zeroUnitModes(l); + return cg::make_layout( + std::vector{zeroUnitModes(cg::get(l, 0)), + zeroUnitModesHead(cg::get(l, 1), keep, false)}); + } + size_t n = cg::rank(l.shape()); + if (keep > n) + return l; + std::vector modes; + for (size_t i = 0; i < n; ++i) + modes.push_back(i < n - keep ? zeroUnitModes(cg::get(l, i)) + : cg::get(l, i)); + return cg::make_layout(modes); +} +cg::composed_layout zeroUnitModesHead(const cg::composed_layout &l, size_t keep, + bool zipped) { + cg::layout b = zeroUnitModesHead(l.layout_b(), keep, zipped); + if (l.is_a_swizzle()) + return cg::composed_layout(l.swizzle_a(), l.offset(), b); + return cg::composed_layout(l.layout_a(), l.offset(), b); +} + +template +LogicalResult inferDivide(MLIRContext *ctx, std::optional loc, + ValueRange operands, DictionaryAttr attrs, + PropertyRef props, SmallVectorImpl &out, + F &÷, bool zipped = false) { + typename Op::Adaptor a(operands, attrs, props); + auto c = carrierOf(a.getInput().getType()); + if (!c) + return emitOptionalError(loc, "input carries no layout"); + Type tiler = a.getTiler().getType(); + return infer(loc, out, [&]() -> Type { + return rebuild( + ctx, *c, + llvm::TypeSwitch(tiler) + .template Case([&](auto t) { + // Unit modes the tiler passes through (`_` or extent 1) keep + // their nonzero strides, which zeroUnitModes does not model: not + // covered. + if constexpr (std::is_same_v) { + auto outer = outerLayout(c->layout); + for (size_t i = 0; outer && cg::holds_vector(t.getRef()) && + i < cg::rank(t.getRef()) && + cg::holds_vector(outer->shape()) && + i < cg::rank(outer->shape()); + ++i) { + std::string m = cg::to_string(t.getRef()[i]); + auto tl = cg::from_string(m); + bool pass = m == "_" || (tl && cg::holds_int(cg::size(*tl)) && + cg::size(*tl).as_int64() == 1 && + cg::to_string(tl->stride()) != "0"); + std::vector shapes, strides; + leavesOf(outer->shape()[i], shapes); + leavesOf(outer->stride()[i], strides); + for (size_t k = 0; + pass && k < shapes.size() && k < strides.size(); ++k) + if (shapes[k] == "1" && strides[k] != "0") + return Type(); + } + } + return mapLayout(ctx, c->layout, [&](const auto &l) { + // A tile or shape tiler leaves the input's trailing modes + // alone. + size_t keep = 0; + if constexpr (!std::is_same_v) { + size_t in = cg::rank(outerOf(l).shape()), + r = cg::rank(t.getRef()); + if (cg::holds_vector(outerOf(l).shape()) && r < in) + keep = in - r; + } + return zeroUnitModesHead(divide(l, t.getRef()), keep, zipped); + }); + }) + .Default([](Type) -> Type { return {}; })); + }); +} +} // namespace + +LogicalResult FlatDivideOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + return inferDivide( + ctx, loc, operands, attrs, props, out, + [](const auto &l, const auto &t) { return cg::flat_divide(l, t); }); +} + +LogicalResult LogicalDivideOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + return inferDivide( + ctx, loc, operands, attrs, props, out, + [](const auto &l, const auto &t) { return cg::logical_divide(l, t); }); +} + +LogicalResult TiledDivideOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + return inferDivide( + ctx, loc, operands, attrs, props, out, + [](const auto &l, const auto &t) { return cg::tiled_divide(l, t); }); +} + +LogicalResult ZippedDivideOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + return inferDivide( + ctx, loc, operands, attrs, props, out, + [](const auto &l, const auto &t) { return cg::zipped_divide(l, t); }, + true); +} + +Type mlir::cutlass_compiler::cute::getResultType(Type in, + ArrayRef mode) { + MLIRContext *ctx = in.getContext(); + return llvm::TypeSwitch(in) + .Case([&](auto ty) -> Type { + auto r = cg::get(ty.getRef(), mode); + if (!cg::is_valid(r)) + return {}; + if constexpr (std::is_same_v) + return layoutType(ctx, std::move(r)); + else + return decltype(ty)::get(ctx, std::move(r)); + }) + .Default([](Type) -> Type { return {}; }); +} + +LogicalResult GetIterOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, DictionaryAttr, + PropertyRef, RegionRange, + SmallVectorImpl &out) { + return infer(loc, out, [&]() -> Type { + Type src = operands[0].getType(); + if (auto m = llvm::dyn_cast(src)) + return m.getPtr(); + if (auto c = llvm::dyn_cast(src)) + return ArithTupleIteratorType::get(ctx, c.getArithTuple()); + if (auto v = llvm::dyn_cast(src)) + return v.getDesc(); + return {}; + }); +} + +LogicalResult GetLayoutOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, DictionaryAttr, + PropertyRef, RegionRange, + SmallVectorImpl &out) { + return infer(loc, out, [&]() -> Type { + auto c = carrierOf(operands[0].getType()); + return c ? c->layout : Type{}; + }); +} + +LogicalResult GetLeavesOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, DictionaryAttr, + PropertyRef, RegionRange, + SmallVectorImpl &out) { + return llvm::TypeSwitch(operands[0].getType()) + .Case( + [&](auto ty) -> LogicalResult { + auto flat = cg::flatten(ty.getRef()); + for (size_t i = 0; i < cg::rank(flat); ++i) + out.push_back(decltype(ty)::get(ctx, flat[i])); + return success(); + }) + .Case([&](TileType ty) -> LogicalResult { + // A tile's leaves: its layouts, and `_` as a tile. + auto flat = cg::flatten(ty.getRef()); + for (size_t i = 0; i < cg::rank(flat); ++i) { + flat[i].visit([&](const auto &e) { + using T = std::decay_t; + if constexpr (cg::is_layout::value) + out.push_back(LayoutType::get(ctx, e)); + else if constexpr (!std::is_same_v>) + out.push_back(TileType::get(ctx, flat[i])); + }); + } + return success(); + }) + .Default([&](Type t) -> LogicalResult { + return emitOptionalError(loc, "cannot take the leaves of ", t); + }); +} + +LogicalResult GetScalarsOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + bool onlyDynamic = a.getOnlyDynamic(); + return llvm::TypeSwitch(a.getCuteValue().getType()) + .Case([&](auto ty) -> LogicalResult { + SmallVector scalars; + cg::collect_scalar_types(ctx, ty.getRef(), scalars, onlyDynamic); + out.append(scalars.begin(), scalars.end()); + return success(); + }) + .Default([&](Type t) -> LogicalResult { + return emitOptionalError(loc, "cannot take the scalars of ", t); + }); +} + +LogicalResult GetShapeOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, DictionaryAttr, + PropertyRef, RegionRange, + SmallVectorImpl &out) { + return infer(loc, out, [&]() -> Type { + Type in = operands[0].getType(); + if (auto t = llvm::dyn_cast(in)) + return ShapeType::get(ctx, cg::extract_shape_from_tile(t.getRef())); + auto c = carrierOf(in); + if (!c) + return {}; + if (auto l = llvm::dyn_cast(c->layout)) + return ShapeType::get(ctx, l.getRef().shape()); + if (auto l = llvm::dyn_cast(c->layout)) + return ShapeType::get(ctx, l.getRef().shape()); + return {}; + }); +} + +LogicalResult GroupModesOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto c = carrierOf(a.getInput().getType()); + if (!c) + return emitOptionalError(loc, "input carries no layout"); + int begin = static_cast(a.getBegin()), + end = static_cast(a.getEnd()); + return infer(loc, out, [&]() -> Type { + return rebuild(ctx, *c, mapLayout(ctx, c->layout, [&](const auto &l) { + return cg::group(begin, end, l); + })); + }); +} + +LogicalResult LocalTileOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto c = carrierOf(a.getInput().getType()); + auto tile = llvm::dyn_cast(a.getTile().getType()); + auto coord = llvm::dyn_cast(a.getCoord().getType()); + if (!c || !tile || !coord) + return emitOptionalError(loc, + "expects a layout carrier, a tile and a coord"); + return infer(loc, out, [&]() -> Type { + // CuTe's inner_partition: divide by the tile, take the coordinate in the + // rest modes, and flatten the two modes into one rank. + cg::tile tiler = tile.getRef(); + cg::coord crd = coord.getRef(); + if (CoordAttr proj = a.getProjAttr()) { + tiler = cg::dice(proj.getRef(), tiler); + crd = cg::dice(proj.getRef(), crd); + } + auto layout = llvm::dyn_cast(c->layout); + if (!layout) + return {}; + cg::layout zipped = cg::zipped_divide(layout.getRef(), tiler); + if (!cg::is_valid(zipped)) + return {}; + cg::layout tiles = cg::get(zipped, 0), rest = cg::get(zipped, 1); + auto [sliced, offset] = cg::slice_and_offset(crd, rest); + if (!cg::is_valid(sliced)) + return {}; + // One rank: the tile modes, then what is left of the rest modes. + std::vector modes; + for (size_t i = 0; i < cg::rank(tiles.shape()); ++i) + modes.push_back(cg::get(tiles, i)); + if (cg::holds_leaf(sliced.shape())) { + modes.push_back(sliced); + } else { + for (size_t i = 0; i < cg::rank(sliced.shape()); ++i) + modes.push_back(cg::get(sliced, i)); + } + // The input's modes beyond the tiler that the coordinate keeps are + // untouched and keep their strides. + size_t keep = 0, in = cg::rank(layout.getRef().shape()), + r = cg::rank(tiler); + for (size_t i = r; cg::holds_vector(crd) && i < in && i < cg::rank(crd); + ++i) + keep += cg::to_string(crd[i]) == "_"; + cg::layout flat = zeroUnitModesHead(cg::make_layout(modes), keep, false); + Type result = layoutType(ctx, std::move(flat)); + if (c->ptr) + return CuteMemRefType::get(ctx, movedPtr(c->ptr, offset), result); + if (c->iter) { + auto sum = cg::arith_tuple_sum(c->iter.getRef(), offset); + if (!cg::is_valid(sum)) + return {}; + return CoordTensorType::get(ctx, IntTupleType::get(ctx, std::move(sum)), + result); + } + return result; + }); +} + +Type mlir::cutlass_compiler::cute::makeArithTupleIterResultType(Type tuple) { + auto t = llvm::dyn_cast(tuple); + return t ? ArithTupleIteratorType::get(tuple.getContext(), t) : Type(); +} + +LogicalResult MakeComposedLayoutOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto offset = llvm::dyn_cast(a.getOffset().getType()); + auto outer = llvm::dyn_cast(a.getOuter().getType()); + if (!offset || !outer) + return emitOptionalError(loc, "expects an int tuple offset and a layout"); + return infer(loc, out, [&]() -> Type { + Type inner = a.getInner().getType(); + if (auto l = llvm::dyn_cast(inner)) + return ComposedLayoutType::get( + ctx, + cg::composed_layout(l.getRef(), offset.getRef(), outer.getRef())); + if (auto s = llvm::dyn_cast(inner)) + return ComposedLayoutType::get( + ctx, + cg::composed_layout(s.getRef(), offset.getRef(), outer.getRef())); + return {}; + }); +} + +LogicalResult MakeIdentityLayoutOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { + return infer(loc, out, [&]() -> Type { + auto s = llvm::dyn_cast(operands[0].getType()); + if (!s) + return {}; + cg::layout l = cg::make_identity_layout(s.getRef()); + return cg::is_valid(l) ? LayoutType::get(ctx, std::move(l)) : Type{}; + }); +} + +LogicalResult MakeOrderedLayoutOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto shape = llvm::dyn_cast(a.getShape().getType()); + auto order = llvm::dyn_cast(a.getOrder().getType()); + if (!shape || !order) + return emitOptionalError(loc, "expects a shape and an order"); + return infer(loc, out, [&]() -> Type { + cg::layout l(shape.getRef(), + cg::compact_order(shape.getRef(), order.getRef())); + return cg::is_valid(l) ? LayoutType::get(ctx, std::move(l)) : Type{}; + }); +} + +Type mlir::cutlass_compiler::cute::makeViewResultType(Type iter, Type layout) { + MLIRContext *ctx = iter.getContext(); + if (!llvm::isa(layout)) + return {}; + if (auto p = llvm::dyn_cast(iter)) + return CuteMemRefType::get(ctx, p, layout); + if (auto it = llvm::dyn_cast(iter)) + return CoordTensorType::get(ctx, it.getArithTuple(), layout); + auto plain = llvm::dyn_cast(layout); + if (plain && llvm::isa(iter)) + return cute_nvgpu::SmemDescViewType::get(ctx, iter, plain.getAttr()); + return {}; +} + +LogicalResult +MemRefLoadOp::inferReturnTypes(MLIRContext *ctx, std::optional loc, + ValueRange operands, DictionaryAttr, PropertyRef, + RegionRange, SmallVectorImpl &out) { + return infer(loc, out, [&]() -> Type { + auto m = llvm::dyn_cast(operands[0].getType()); + return m ? m.getValueType() : Type{}; + }); +} + +LogicalResult MemRefLoadVecOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + return infer(loc, out, [&]() -> Type { + auto m = llvm::dyn_cast(a.getSrc().getType()); + if (!m) + return {}; + auto l = llvm::dyn_cast(m.getLayout()); + if (!l) + return {}; + auto n = cg::size(l.getRef()); + if (!cg::holds_int(n)) + return {}; + return VectorType::get({n.as_int64()}, m.getValueType()); + }); +} + +LogicalResult SizeOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, PropertyRef props, + RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + ArrayRef mode; + if (auto m = a.getModeAttr()) + mode = m.asArrayRef(); + Type in = a.getInput().getType(); + if (auto c = carrierOf(in)) + in = c->layout; + return infer(loc, out, [&]() -> Type { + return llvm::TypeSwitch(in) + .Case( + [&](auto ty) -> Type { + auto r = cg::rec_var_cast( + cg::size(cg::get(ty.getRef(), mode))); + if (!cg::is_valid(r)) + return {}; + return IntTupleType::get(ctx, std::move(r)); + }) + .Default([](Type) -> Type { return {}; }); + }); +} + +LogicalResult SliceOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, PropertyRef props, + RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto coord = llvm::dyn_cast(a.getCoord().getType()); + if (!coord) + return emitOptionalError(loc, "coord must be a coord"); + Type in = a.getInput().getType(); + return infer(loc, out, [&]() -> Type { + if (auto c = carrierOf(in)) + return sliceCarrier(ctx, *c, coord.getRef()); + return llvm::TypeSwitch(in) + .Case( + [&](auto ty) -> Type { + auto r = cg::slice(coord.getRef(), ty.getRef()); + if (!cg::is_valid(r)) + return {}; + return decltype(ty)::get(ctx, std::move(r)); + }) + .Default([](Type) -> Type { return {}; }); + }); +} + +LogicalResult TileToShapeOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto shape = llvm::dyn_cast(a.getTrgShape().getType()); + auto order = llvm::dyn_cast(a.getOrdShape().getType()); + if (!shape || !order) + return emitOptionalError(loc, "expects a shape and an order"); + return infer(loc, out, [&]() -> Type { + return mapLayout(ctx, a.getBlock().getType(), [&](const auto &l) { + return cg::tile_to_shape(l, shape.getRef(), order.getRef()); + }); + }); +} + +LogicalResult +ToIntTupleOp::inferReturnTypes(MLIRContext *ctx, std::optional loc, + ValueRange operands, DictionaryAttr, PropertyRef, + RegionRange, SmallVectorImpl &out) { + return infer(loc, out, [&]() -> Type { + return llvm::TypeSwitch(operands[0].getType()) + .Case( + [&](auto ty) -> Type { + return IntTupleType::get( + ctx, cg::rec_var_cast(ty.getRef())); + }) + .Default([](Type) -> Type { return {}; }); + }); +} + +namespace { +template +LogicalResult inferTupleArith(MLIRContext *ctx, std::optional loc, + ValueRange operands, SmallVectorImpl &out, + F &&f) { + return infer(loc, out, [&]() -> Type { + return llvm::TypeSwitch(operands[0].getType()) + .Case([&](auto lhs) -> Type { + using T = decltype(lhs); + auto rhs = llvm::dyn_cast(operands[1].getType()); + if (!rhs) + return {}; + auto r = f.template operator()(lhs.getRef(), + rhs.getRef()); + if (!cg::is_valid(r)) + return {}; + return T::get(ctx, std::move(r)); + }) + .Default([](Type) -> Type { return {}; }); + }); +} +} // namespace + +LogicalResult TupleAddOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, DictionaryAttr, + PropertyRef, RegionRange, + SmallVectorImpl &out) { + return inferTupleArith(ctx, loc, operands, out, + [](const auto &a, const auto &b) { + return cg::arith_tuple_sum(a, b); + }); +} + +LogicalResult TupleMulOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, DictionaryAttr, + PropertyRef, RegionRange, + SmallVectorImpl &out) { + return inferTupleArith(ctx, loc, operands, out, + [](const auto &a, const auto &b) { + return cg::arith_tuple_mul(a, b); + }); +} + +LogicalResult TupleDivOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, DictionaryAttr, + PropertyRef, RegionRange, + SmallVectorImpl &out) { + return inferTupleArith(ctx, loc, operands, out, + [](const auto &a, const auto &b) { + return cg::arith_tuple_div(a, b); + }); +} + +// Without a stride the layout is compact column-major, unit modes at 0. +Type mlir::cutlass_compiler::cute::makeLayoutResultType(Type shape, + Type stride) { + auto s = llvm::dyn_cast(shape); + if (!s) + return {}; + MLIRContext *ctx = shape.getContext(); + if (stride) { + auto st = llvm::dyn_cast(stride); + if (!st) + return {}; + return LayoutType::get(ctx, cg::layout(s.getRef(), st.getRef())); + } + cg::layout l(s.getRef(), + cg::compact_col_major(s.getRef())); + return LayoutType::get(ctx, zeroUnitModes(l)); +} + +namespace { +// `MxNxK` and `num_cta` of an SM100 block-scaled or plain mma atom. +struct MmaAtom { + int64_t m = 0, n = 0, k = 0, ctas = 1; + Type c; // accumulator element + bool aTmem = false; // frag_kind = ts +}; +std::optional mmaAtomOf(Type tiledMma) { + auto tiled = llvm::dyn_cast(tiledMma); + if (!tiled) + return std::nullopt; + // One atom, not permuted: other tilings are not covered here. + auto n = cg::size(tiled.getAtomLayout_MNK().getRef()); + TileAttr perm = tiled.getPermutation_MNK(); + if (!cg::holds_int(n) || n.as_int64() != 1 || + (perm && cg::to_string(perm.getRef()) != "[_;_;_]")) + return std::nullopt; + MmaAtom atom; + ShapeAttr shape; + if (auto bs = llvm::dyn_cast( + tiled.getMmaAtom())) { + shape = bs.getShapeMnk(); + atom.ctas = bs.getNumCta(); + atom.c = bs.getCType(); + atom.aTmem = bs.getAFragKind() == cute_nvgpu::MmaFragKind::tmem; + } else if (auto mma = llvm::dyn_cast( + tiled.getMmaAtom())) { + shape = mma.getShapeMnk(); + atom.ctas = mma.getNumCta(); + atom.c = mma.getCType(); + atom.aTmem = mma.getAFragKind() == cute_nvgpu::MmaFragKind::tmem; + } else { + return std::nullopt; + } + if (cg::rank(shape.getRef()) != 3) + return std::nullopt; + int64_t *mnk[] = {&atom.m, &atom.n, &atom.k}; + for (int i = 0; i < 3; ++i) { + auto e = cg::get(shape.getRef(), i); + if (!cg::holds_int(e)) + return std::nullopt; + *mnk[i] = e.as_int64(); + } + return atom; +} +} // namespace + +// partition_A/B/C of the trivial tiled mma: the leading two modes are divided +// by the per-CTA atom shape, the CTA's share of the first is taken at the +// thread coordinate, and what remains follows the atom modes. +LogicalResult TiledMmaPartitionOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto atom = mmaAtomOf(a.getTiledMma().getType()); + auto c = carrierOf(a.getInput().getType()); + auto coord = llvm::dyn_cast(a.getCoord().getType()); + auto which = llvm::dyn_cast_or_null(a.getOperandIdAttr()); + if (!atom || !c || !coord || !which) + return emitOptionalError( + loc, "expects a tiled mma, a tensor, a coord and an operand"); + return infer(loc, out, [&]() -> Type { + auto layout = llvm::dyn_cast(c->layout); + if (!layout) + return {}; + int64_t a0, a1; + switch (which.getInt()) { + case 0: + a0 = atom->m / atom->ctas; + a1 = atom->k; + break; // A: (M, K) + case 1: + a0 = atom->n / atom->ctas; + a1 = atom->k; + break; // B: (N, K) + case 2: + a0 = atom->m / atom->ctas; + a1 = atom->n; + break; // C: (M, N) + default: + return {}; + } + const cg::layout &l = layout.getRef(); + if (cg::rank(l.shape()) < 2) + return {}; + auto tile = cg::from_string("[" + std::to_string(a0) + ":1;" + + std::to_string(a1) + ":1]"); + if (!tile) + return {}; + cg::layout two = + cg::make_layout(std::vector{cg::get(l, 0), cg::get(l, 1)}); + cg::layout divided = cg::zipped_divide(two, *tile); + if (!cg::is_valid(divided)) + return {}; + cg::layout atoms = cg::get(divided, 0), rest = cg::get(divided, 1); + // The first rest mode holds the CTA index innermost. + auto ctaTile = + cg::from_string("[" + std::to_string(atom->ctas) + ":1]"); + if (!ctaTile) + return {}; + cg::layout rest0 = cg::zipped_divide(cg::get(rest, 0), *ctaTile); + if (!cg::is_valid(rest0) || cg::rank(rest0.shape()) != 2) + return {}; + // A lone CTA has nothing to select: its coordinate is 0 whatever the + // thread index says. + std::string which_cta = + atom->ctas == 1 ? "0" : cg::to_string(coord.getRef()); + auto [rest0Sliced, offset] = cg::slice_and_offset( + *cg::from_string("(" + which_cta + ",_)"), rest0); + if (!cg::is_valid(rest0Sliced)) + return {}; + // What the slice leaves is one mode, not a tuple around it. + while (cg::holds_vector(rest0Sliced.shape()) && + cg::rank(rest0Sliced.shape()) == 1) + rest0Sliced = cg::get(rest0Sliced, 0); + std::vector modes{atoms, rest0Sliced, cg::get(rest, 1)}; + for (size_t i = 2; i < cg::rank(l.shape()); ++i) + modes.push_back(cg::get(l, i)); + // Modes beyond M and K (or N) are untouched. + Type result = layoutType(ctx, zeroUnitModesHead(cg::make_layout(modes), + modes.size() - 3, false)); + if (c->ptr) + return CuteMemRefType::get(ctx, movedPtr(c->ptr, offset), result); + if (c->iter) { + auto sum = cg::arith_tuple_sum(c->iter.getRef(), offset); + if (!cg::is_valid(sum)) + return {}; + return CoordTensorType::get(ctx, IntTupleType::get(ctx, std::move(sum)), + result); + } + return result; + }); +} + +// The fragment an mma reads: for A and B a descriptor view over the staged +// smem tile, atom modes collapsed and strides in 16-byte units; for C the +// accumulator in tensor memory, one lane per row and one column per element. +LogicalResult MmaMakeFragmentOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto which = llvm::dyn_cast_or_null(a.getOperandIdAttr()); + auto atom = mmaAtomOf(a.getAtom().getType()); + // A in tensor memory (frag_kind = ts) is not covered. + if (!which || !atom || (which.getInt() == 0 && atom->aTmem)) + return emitOptionalError(loc, + "expects an SM100 tiled mma and an operand id"); + Type in = a.getInput().getType(); + return infer(loc, out, [&]() -> Type { + if (which.getInt() == 2) { + auto shape = llvm::dyn_cast(in); + if (!shape) + return {}; + const cg::shape &s = shape.getRef(); + if (cg::rank(s) < 1 || !cg::holds_vector(cg::get(s, 0))) + return {}; + auto m = cg::size(cg::get(s, 0), 0), n = cg::size(cg::get(s, 0), 1); + if (!cg::holds_int(m) || !cg::holds_int(n)) + return {}; + // ((M,N), rest...): rows on lanes, columns contiguous, stages by N + // columns; unit modes at 0. A 64-row share of a 2-CTA atom puts the + // second half of the columns on lanes 64 and up. + int64_t cols = n.as_int64(); + // Mode 0 is flattened to its (M,N) extents. + std::string text = "((" + cg::to_string(m) + "," + cg::to_string(n) + ")", + stride = "((65536,1)"; + if (atom->m / atom->ctas == 64) { + if (atom->ctas != 2 || m.as_int64() != 64 || cols % 2) + return {}; + cols /= 2; + text = "((64,(" + std::to_string(cols) + ",2))"; + stride = "((65536,(1,4194304))"; + } else if (atom->m / atom->ctas != 128) { + return {}; + } + for (size_t i = 1; i < cg::rank(s); ++i) { + auto e = cg::size(s, i); + text += "," + cg::to_string(cg::get(s, i)); + stride += "," + ((cg::holds_int(e) && e.as_int64() == 1) + ? std::string("0") + : std::to_string(cols)); + } + auto l = cg::from_string(text + "):" + stride + ")"); + if (!l) + return {}; + auto ptr = PtrType::get(ctx, atom->c, AddressSpace::tmem, 1, {}, {}); + return CuteMemRefType::get(ctx, ptr, LayoutType::get(ctx, *l)); + } + auto m = llvm::dyn_cast(in); + if (!m) + return {}; + cg::layout l; + if (auto plain = llvm::dyn_cast(m.getLayout())) + l = plain.getRef(); + else if (auto composed = llvm::dyn_cast(m.getLayout())) + l = composed.getRef().layout_b(); + else + return {}; + // The atom mode collapses to one; the rest modes follow. + int64_t bits = bitsOfType(m.getValueType()); + std::string shape = "(1", stride = "(0"; + for (size_t i = 1; i < cg::rank(l.shape()); ++i) { + auto e = cg::size(l.shape(), i); + auto st = cg::get(l.stride(), i); + if (!cg::holds_int(e) || !cg::holds_int(st)) + return {}; + shape += "," + std::to_string(e.as_int64()); + stride += "," + std::to_string(st.as_int64() * bits / 128); + } + auto view = cg::from_string(shape + "):" + stride + ")"); + if (!view) + return {}; + return cute_nvgpu::SmemDescViewType::get( + ctx, cute_nvgpu::SmemDescType::get(ctx), LayoutAttr::get(ctx, *view)); + }); +} + +LogicalResult TupleSubOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, DictionaryAttr, + PropertyRef, RegionRange, + SmallVectorImpl &out) { + return inferTupleArith(ctx, loc, operands, out, + [](const auto &a, const auto &b) { + return cg::arith_tuple_sub(a, b); + }); +} + +LogicalResult TupleModOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, DictionaryAttr, + PropertyRef, RegionRange, + SmallVectorImpl &out) { + if (failed(inferTupleArith(ctx, loc, operands, out, + [](const auto &a, const auto &b) { + return cg::arith_tuple_mod(a, b); + }))) + return failure(); + // A dynamic leaf modulo a static one: 0 when its divisibility is a + // multiple, itself when a divisor, else unknown (cutegen drops this). + std::vector lhs, rhs, res; + auto leaves = [](Type t, std::vector &l) { + if (auto i = llvm::dyn_cast(t)) + leavesOf(i.getRef(), l); + if (auto s = llvm::dyn_cast(t)) + leavesOf(s.getRef(), l); + }; + leaves(operands[0].getType(), lhs); + leaves(operands[1].getType(), rhs); + leaves(out[0], res); + if (lhs.size() != rhs.size() || lhs.size() != res.size()) + return success(); + for (size_t i = 0; i < res.size(); ++i) { + int64_t n, div = 1; + if (lhs[i].front() != '?' || StringRef(rhs[i]).getAsInteger(10, n) || + n <= 0) + continue; + size_t d = lhs[i].find("div="); + if (d != std::string::npos) + StringRef(lhs[i]) + .substr(d + 4) + .take_while(llvm::isDigit) + .getAsInteger(10, div); + res[i] = div % n == 0 ? "0" + : n % div == 0 + ? lhs[i] + : dynamicInt(1, lhs[i].find("i64") != std::string::npos); + } + size_t i = 0; + Type t = out[0]; + std::string text; + if (auto it = llvm::dyn_cast(t)) + text = nestLike(it.getRef(), res, i); + else + text = nestLike(llvm::cast(t).getRef(), res, i); + if (llvm::isa(t)) { + if (auto r = cg::from_string(text)) + out[0] = IntTupleType::get(ctx, *r); + } else if (auto r = cg::from_string(text)) { + out[0] = ShapeType::get(ctx, *r); + } + return success(); +} + +// ---------------------------------------------------------------- more cute + +namespace { + +template +LogicalResult inferProduct(MLIRContext *ctx, std::optional loc, + ValueRange operands, DictionaryAttr attrs, + PropertyRef props, SmallVectorImpl &out, + F &&f) { + typename Op::Adaptor a(operands, attrs, props); + auto tiler = llvm::dyn_cast(a.getTiler().getType()); + if (!tiler || !cg::is_int_or_dynamic_int_only(tiler.getRef().stride())) + return emitOptionalError(loc, "expects an integer-strided layout tiler"); + Type in = a.getInput().getType(); + return infer(loc, out, [&]() -> Type { + auto outer = outerLayout(in); + if (!outer || !cg::is_static(*outer) || + (!cg::is_int_or_dynamic_int_only(outer->stride()) && + !cg::is_static(tiler.getRef()))) + return {}; + return mapLayout(ctx, in, + [&](const auto &l) { return f(l, tiler.getRef()); }); + }); +} + +// A coordinate per leaf of `shape`: 0 where the extent is 1, else dynamic. +std::optional dynamicCoordLike(const cg::shape &shape) { + std::vector leaves; + leavesOf(shape, leaves); + for (auto &leaf : leaves) + leaf = isOne(leaf) ? "0" : "?"; + size_t i = 0; + return cg::from_string(nestLike(shape, leaves, i)); +} + +// Moves a carrier's iterator by `offset` and gives it `layout`. +Type moved(MLIRContext *ctx, const Carrier &c, const cg::int_tuple &offset, + Type layout) { + if (c.ptr) + return CuteMemRefType::get(ctx, movedPtr(c.ptr, offset), layout); + if (c.iter) { + auto sum = cg::arith_tuple_sum(c.iter.getRef(), offset); + if (!cg::is_valid(sum)) + return {}; + return CoordTensorType::get(ctx, IntTupleType::get(ctx, std::move(sum)), + layout); + } + return rebuild(ctx, c, layout); +} + +} // namespace + +#define CUTE_PRODUCT(Op, fn) \ + LogicalResult Op::inferReturnTypes( \ + MLIRContext *ctx, std::optional loc, ValueRange operands, \ + DictionaryAttr attrs, PropertyRef props, RegionRange, \ + SmallVectorImpl &out) { \ + return inferProduct( \ + ctx, loc, operands, attrs, props, out, \ + [](const auto &l, const auto &t) { return cg::fn(l, t); }); \ + } +CUTE_PRODUCT(LogicalProductOp, logical_product) +CUTE_PRODUCT(ZippedProductOp, zipped_product) +CUTE_PRODUCT(TiledProductOp, tiled_product) +CUTE_PRODUCT(FlatProductOp, flat_product) +CUTE_PRODUCT(BlockedProductOp, blocked_product) +#undef CUTE_PRODUCT + +// Unlike the other products, the raked one leaves unit modes at stride 0. +LogicalResult RakedProductOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + return inferProduct( + ctx, loc, operands, attrs, props, out, [](const auto &l, const auto &t) { + return zeroUnitModes(cg::raked_product(l, t)); + }); +} + +// The pointer's type is kept: the compiler records no swizzle on it. +LogicalResult ApplySwizzleOp::inferReturnTypes( + MLIRContext *, std::optional, ValueRange operands, DictionaryAttr, + PropertyRef, RegionRange, SmallVectorImpl &out) { + out.push_back(operands[0].getType()); + return success(); +} + +LogicalResult ComplementOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto in = llvm::dyn_cast(a.getInput().getType()); + Type co = a.getCotarget().getType(); + return infer(loc, out, [&]() -> Type { + if (!in) + return {}; + const cg::layout &l = in.getRef(); + if (!cg::is_int_or_dynamic_int_only(l.stride()) || + (!cg::is_static(l.stride()) && cg::rank(l.shape()) > 1)) + return {}; + cg::shape shape; + if (auto s = llvm::dyn_cast(co)) + shape = s.getRef(); + else if (auto c = llvm::dyn_cast(co)) + shape = c.getRef().shape(); + else + return {}; + cg::layout r = cg::complement(l, shape); + return cg::is_valid(r) ? LayoutType::get(ctx, std::move(r)) : Type(); + }); +} + +LogicalResult ComposedGetInnerOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { + return infer(loc, out, [&]() -> Type { + auto c = llvm::dyn_cast(operands[0].getType()); + if (!c) + return {}; + if (c.getRef().is_a_swizzle()) + return SwizzleType::get(ctx, c.getRef().swizzle_a()); + return LayoutType::get(ctx, c.getRef().layout_a()); + }); +} + +LogicalResult ComposedGetOffsetOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { + return infer(loc, out, [&]() -> Type { + auto c = llvm::dyn_cast(operands[0].getType()); + return c ? IntTupleType::get(ctx, c.getRef().offset()) : Type(); + }); +} + +LogicalResult CosizeOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + ArrayRef mode = a.getMode(); + auto c = carrierOf(a.getInput().getType()); + return infer(loc, out, [&]() -> Type { + if (!c) + return {}; + cg::int_tuple r; + if (auto l = llvm::dyn_cast(c->layout)) { + r = cg::rec_var_cast( + cg::cosize(cg::get(l.getRef(), mode))); + } else { + // An affine inner layout has no cosize. + auto cl = llvm::cast(c->layout); + if (!cl.getRef().is_a_swizzle()) + return {}; + r = cg::rec_var_cast( + cg::cosize(cg::get(cl.getRef(), mode))); + } + return cg::is_valid(r) ? IntTupleType::get(ctx, std::move(r)) : Type(); + }); +} + +// The multiplier and the two shifts of the divisor's magic numbers. +LogicalResult FastDivmodGetAuxOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { + Type i = fastDivmodComputeResultType(operands[0].getType()); + if (!i) + return emitOptionalError(loc, "expects a fast divmod divisor"); + out.append({i, IntegerType::get(ctx, 8), IntegerType::get(ctx, 8)}); + return success(); +} + +LogicalResult FastDivmodGetDivisorOp::inferReturnTypes( + MLIRContext *, std::optional loc, ValueRange operands, + DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { + return infer(loc, out, [&] { + return fastDivmodComputeResultType(operands[0].getType()); + }); +} + +LogicalResult FilterOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, DictionaryAttr, + PropertyRef, RegionRange, + SmallVectorImpl &out) { + auto c = carrierOf(operands[0].getType()); + if (!c) + return emitOptionalError(loc, "input carries no layout"); + return infer(loc, out, [&]() -> Type { + Type layout; + if (auto l = llvm::dyn_cast(c->layout)) { + layout = layoutType(ctx, cg::filter(l.getRef())); + } else { + // The outer layout is filtered; the inner function is kept. + const cg::composed_layout &cl = + llvm::cast(c->layout).getRef(); + cg::layout f = cg::filter(cl.layout_b()); + if (!cg::is_valid(f)) + return {}; + layout = layoutType( + ctx, cl.is_a_swizzle() + ? cg::composed_layout(cl.swizzle_a(), cl.offset(), f) + : cg::composed_layout(cl.layout_a(), cl.offset(), f)); + } + return rebuild(ctx, *c, layout); + }); +} + +namespace { +// Layout::get_hier_coord: each leaf's coordinate is (index / stride) % shape. +// A constant index gives static coordinates where the leaf is static; any +// other index gives dynamic ones, 0 on leaves of extent 1. +std::optional> hierCoord(Value index, + const cg::layout &l) { + std::vector shapes, strides; + leavesOf(l.shape(), shapes); + leavesOf(l.stride(), strides); + if (shapes.size() != strides.size()) + return std::nullopt; + APInt value; + bool constant = matchPattern(index, m_ConstantInt(&value)); + std::vector crd; + for (size_t i = 0; i < shapes.size(); ++i) { + int64_t n = 0, d = 0; + bool staticShape = !StringRef(shapes[i]).getAsInteger(10, n); + if (staticShape && n == 1) { + crd.push_back("0"); + } else if (!constant) { + crd.push_back("?"); + } else if (StringRef(strides[i]).getAsInteger(10, d) || d <= 0) { + return std::nullopt; // not recovered for dynamic or zero strides + } else { + int64_t q = value.getSExtValue() / d; + crd.push_back(staticShape ? std::to_string(q % n) : "?"); + } + } + return crd; +} + +// Column-major index of static coordinates in `shape`, or "?". +std::string colMajor(const std::vector &crd, + const std::vector &shapes, size_t begin, + size_t end) { + int64_t idx = 0, scale = 1; + for (size_t i = begin; i < end; ++i) { + int64_t c, n; + if (StringRef(crd[i]).getAsInteger(10, c)) + return "?"; + idx += c * scale; + if (i + 1 < end && StringRef(shapes[i]).getAsInteger(10, n)) + return "?"; + scale *= n; + } + return std::to_string(idx); +} +} // namespace + +LogicalResult GetHierCoordOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto layout = llvm::dyn_cast(a.getInput().getType()); + return infer(loc, out, [&]() -> Type { + auto crd = layout ? hierCoord(a.getIndex(), layout.getRef()) : std::nullopt; + if (!crd) + return {}; + size_t i = 0; + auto c = + cg::from_string(nestLike(layout.getRef().shape(), *crd, i)); + return c ? CoordType::get(ctx, std::move(*c)) : Type(); + }); +} + +// One coordinate per mode: the mode's hierarchical coordinate, column-major. +LogicalResult GetFlatCoordOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto layout = llvm::dyn_cast(a.getInput().getType()); + return infer(loc, out, [&]() -> Type { + auto crd = layout ? hierCoord(a.getIndex(), layout.getRef()) : std::nullopt; + if (!crd) + return {}; + const cg::shape &shape = layout.getRef().shape(); + std::vector shapes, modes; + leavesOf(shape, shapes); + if (!cg::holds_vector(shape)) { + modes.push_back(colMajor(*crd, shapes, 0, shapes.size())); + } else { + size_t begin = 0; + for (size_t m = 0; m < cg::rank(shape); ++m) { + std::vector sub; + leavesOf(shape[m], sub); + modes.push_back(colMajor(*crd, shapes, begin, begin + sub.size())); + begin += sub.size(); + } + } + auto c = cg::from_string(cg::holds_vector(shape) + ? "(" + llvm::join(modes, ",") + ")" + : modes[0]); + return c ? CoordType::get(ctx, std::move(*c)) : Type(); + }); +} + +LogicalResult GetIntegralCoordOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto layout = llvm::dyn_cast(a.getInput().getType()); + return infer(loc, out, [&]() -> Type { + auto crd = layout ? hierCoord(a.getIndex(), layout.getRef()) : std::nullopt; + if (!crd) + return {}; + std::vector shapes; + leavesOf(layout.getRef().shape(), shapes); + auto c = + cg::from_string(colMajor(*crd, shapes, 0, shapes.size())); + return c ? CoordType::get(ctx, std::move(*c)) : Type(); + }); +} + +LogicalResult GetLayoutsFromTileOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { + auto tile = llvm::dyn_cast(operands[0].getType()); + if (!tile) + return emitOptionalError(loc, "expects a tile"); + auto flat = cg::flatten(tile.getRef()); + for (size_t i = 0; i < cg::rank(flat); ++i) { + flat[i].visit([&](const auto &e) { + if constexpr (cg::is_layout>::value) + out.push_back(LayoutType::get(ctx, e)); + }); + } + if (out.empty()) + return emitOptionalError(loc, "the tile holds no layout"); + return success(); +} + +LogicalResult GetStrideOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, DictionaryAttr, + PropertyRef, RegionRange, + SmallVectorImpl &out) { + return infer(loc, out, [&]() -> Type { + auto l = llvm::dyn_cast(operands[0].getType()); + return l ? StrideType::get(ctx, l.getRef().stride()) : Type(); + }); +} + +LogicalResult Idx2CrdOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto index = llvm::dyn_cast(a.getIndex().getType()); + auto shape = llvm::dyn_cast(a.getShape().getType()); + return infer(loc, out, [&]() -> Type { + if (!index || !shape) + return {}; + size_t r = cg::rank(index.getRef()); + if (r > 1 && r != cg::rank(shape.getRef())) + return {}; + if (auto split = cg::holds_dynamic_int(index.getRef()) + ? splitIndex(index.getRef(), shape.getRef()) + : std::nullopt) { + auto c = cg::from_string(*split); + return c ? CoordType::get(ctx, std::move(*c)) : Type(); + } + auto c = cg::rec_var_cast( + cg::idx2crd(index.getRef(), shape.getRef())); + return cg::is_valid(c) ? CoordType::get(ctx, std::move(c)) : Type(); + }); +} + +LogicalResult IncrementCoordOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto coord = llvm::dyn_cast(a.getCoord().getType()); + auto shape = llvm::dyn_cast(a.getShape().getType()); + return infer(loc, out, [&]() -> Type { + if (!coord || !shape || !cg::is_congruent(coord.getRef(), shape.getRef()) || + cg::has_underscore(coord.getRef())) + return {}; + auto c = cg::increment_coord(coord.getRef(), shape.getRef()); + return cg::is_valid(c) ? CoordType::get(ctx, std::move(c)) : Type(); + }); +} + +namespace { +template +LogicalResult inferInverse(MLIRContext *ctx, std::optional loc, + Type input, bool staticStride, + SmallVectorImpl &out, F &&f) { + return infer(loc, out, [&]() -> Type { + auto l = llvm::dyn_cast(input); + if (!l || !cg::is_static(l.getRef().shape()) || + (staticStride && !cg::is_static(l.getRef().stride()))) + return {}; + cg::layout r = f(l.getRef()); + return cg::is_valid(r) ? LayoutType::get(ctx, std::move(r)) : Type(); + }); +} +} // namespace + +LogicalResult LeftInverseOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { + return inferInverse(ctx, loc, operands[0].getType(), true, out, + [](const cg::layout &l) { return cg::left_inverse(l); }); +} + +LogicalResult RightInverseOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { + return inferInverse(ctx, loc, operands[0].getType(), false, out, + [](const cg::layout &l) { return cg::right_inverse(l); }); +} + +LogicalResult LoadScaledIndexOp::inferReturnTypes( + MLIRContext *, std::optional, ValueRange operands, DictionaryAttr, + PropertyRef, RegionRange, SmallVectorImpl &out) { + out.push_back(operands[0].getType()); + return success(); +} + +// CuTe's local_partition: the tensor divided by the tiler's shape, sliced at +// the thread's (dynamic) coordinate in the tile. +LogicalResult LocalPartitionOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto c = carrierOf(a.getInput().getType()); + auto tiler = llvm::dyn_cast(a.getTiler().getType()); + return infer(loc, out, [&]() -> Type { + auto layout = c ? llvm::dyn_cast(c->layout) : LayoutType(); + if (!layout || !tiler || !cg::is_static(tiler.getRef())) + return {}; + cg::shape tile = cg::product_each(tiler.getRef().shape()); + cg::layout zipped = cg::zipped_divide(layout.getRef(), tile); + auto thread = dynamicCoordLike(tile); + if (!cg::is_valid(zipped) || !thread) + return {}; + auto crd = cg::from_string("(" + cg::to_string(*thread) + ",_)"); + if (!crd) + return {}; + auto [sliced, offset] = cg::slice_and_offset(*crd, zipped); + if (!cg::is_valid(sliced)) + return {}; + // The slice keeps the rest mode's own modes, not a tuple around them. + if (cg::holds_vector(sliced.shape()) && cg::rank(sliced.shape()) == 1) + sliced = cg::get(sliced, 0); + return moved(ctx, *c, offset, LayoutType::get(ctx, zeroUnitModes(sliced))); + }); +} + +// CuTe's make_fragment_like: the first mode compact column-major with its +// stride-0 modes kept, the other modes ordered by their strides (dynamic +// ones last) after it; column-major when there is one mode or the first +// mode's shape is dynamic. +LogicalResult MakeFragmentLikeOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { + auto src = swizzledOuter(operands[0].getType()); + return infer(loc, out, [&]() -> Type { + if (!src) + return {}; + const cg::shape &shape = src->shape(); + std::vector shapes, strides; + leavesOf(shape, shapes); + leavesOf(src->stride(), strides); + if (shapes.size() != strides.size()) + return {}; + size_t first = shapes.size(); // leaves of mode 0 + bool ordered = cg::holds_vector(shape) && cg::rank(shape) > 1 && + cg::is_static(shape[0]); + if (ordered) { + std::vector m0; + leavesOf(shape[0], m0); + first = m0.size(); + } + // The order the leaves are laid out in. + std::vector order(shapes.size()); + std::iota(order.begin(), order.end(), 0); + auto key = [&](size_t i) -> int64_t { + int64_t v; + return StringRef(strides[i]).getAsInteger(10, v) ? INT64_MAX : v; + }; + if (ordered) + std::stable_sort(order.begin() + first, order.end(), + [&](size_t x, size_t y) { return key(x) < key(y); }); + std::vector result(shapes.size()); + cg::int_tuple step = *cg::from_string("1"); + for (size_t i : order) { + if (isOne(shapes[i]) || (ordered && i < first && strides[i] == "0")) { + result[i] = "0"; + continue; + } + result[i] = cg::to_string(step); + auto n = cg::from_string(shapes[i]); + if (!n) + return {}; + step = cg::arith_tuple_mul(step, *n); + } + size_t i = 0; + auto l = cg::from_string(cg::to_string(shape) + ":" + + nestLike(shape, result, i)); + return l ? LayoutType::get(ctx, std::move(*l)) : Type(); + }); +} + +LogicalResult MakeIdentityTensorOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { + return infer(loc, out, [&]() -> Type { + auto s = llvm::dyn_cast(operands[0].getType()); + if (!s) + return {}; + std::vector leaves; + leavesOf(s.getRef(), leaves); + for (auto &leaf : leaves) + leaf = "0"; + size_t i = 0; + auto zero = cg::from_string(nestLike(s.getRef(), leaves, i)); + cg::layout l = cg::make_identity_layout(s.getRef()); + if (!zero || !cg::is_valid(l)) + return {}; + return CoordTensorType::get(ctx, IntTupleType::get(ctx, *zero), + LayoutType::get(ctx, l)); + }); +} + +LogicalResult MakeLayoutLikeOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { + return infer(loc, out, [&]() -> Type { + Type src = operands[0].getType(); + cg::layout r; + if (auto l = llvm::dyn_cast(src)) { + if (cg::has_scaled_basis(l.getRef().stride())) + return {}; + r = cg::make_layout_like(l.getRef()); + } else if (auto c = llvm::dyn_cast(src)) { + if (!c.getRef().is_a_swizzle() || + cg::has_scaled_basis(c.getRef().layout_b().stride())) + return {}; + r = cg::make_layout_like(c.getRef()); + } else { + return {}; + } + return cg::is_valid(r) ? LayoutType::get(ctx, std::move(r)) : Type(); + }); +} + +LogicalResult PtrLoadOp::inferReturnTypes(MLIRContext *, + std::optional loc, + ValueRange operands, DictionaryAttr, + PropertyRef, RegionRange, + SmallVectorImpl &out) { + return infer(loc, out, [&]() -> Type { + auto p = llvm::dyn_cast(operands[0].getType()); + return p ? p.getValueType() : Type(); + }); +} + +LogicalResult RecastLayoutOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + int newBits = a.getNewTypeBits(), oldBits = a.getOldTypeBits(); + return infer(loc, out, [&]() -> Type { + if (newBits <= 0 || oldBits <= 0) + return {}; + auto outer = outerLayout(a.getSrc().getType()); + if (!outer || !cg::is_int_or_dynamic_int_only(outer->stride())) + return {}; + return mapLayout(ctx, a.getSrc().getType(), [&](const auto &l) { + return cg::recast(newBits, oldBits, l); + }); + }); +} + +// Without a profile the whole vector reduces to a scalar; with one, the +// modes it marks reduce and the others (`_`) remain, as a vector and shape. +LogicalResult ReduceOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto vec = llvm::dyn_cast(a.getInput().getType()); + if (!vec) + return emitOptionalError(loc, "expects a vector input"); + if (!a.getReductionProfileAttr()) { + out.push_back(vec.getElementType()); + return success(); + } + auto shape = llvm::dyn_cast(a.getShape().getType()); + if (!shape) + return emitOptionalError(loc, "expects a shape"); + cg::shape rest = + cg::slice(a.getReductionProfileAttr().getRef(), shape.getRef()); + auto n = cg::size(rest); + if (!cg::is_valid(rest) || !cg::holds_int(n)) + return emitOptionalError(loc, "cannot reduce the shape"); + out.push_back(VectorType::get({n.as_int64()}, vec.getElementType())); + out.push_back(ShapeType::get(ctx, std::move(rest))); + return success(); +} + +LogicalResult SelectOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, + DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + ArrayRef mode = a.getMode(); + return infer(loc, out, [&]() -> Type { + return llvm::TypeSwitch(a.getInput().getType()) + .Case([&](auto ty) -> Type { + size_t r = cg::rank(ty.getRef()); + for (int32_t m : mode) + if (m < 0 || static_cast(m) >= r) + return {}; + auto res = cg::select(ty.getRef(), mode); + if (!cg::is_valid(res)) + return {}; + return decltype(ty)::get(ctx, std::move(res)); + }) + .Default([](Type) -> Type { return {}; }); + }); +} + +LogicalResult ShapeDivOp::inferReturnTypes(MLIRContext *ctx, + std::optional loc, + ValueRange operands, DictionaryAttr, + PropertyRef, RegionRange, + SmallVectorImpl &out) { + return infer(loc, out, [&]() -> Type { + auto x = llvm::dyn_cast(operands[0].getType()); + auto y = llvm::dyn_cast(operands[1].getType()); + if (!x || !y) + return {}; + cg::shape r = cg::shape_div(x.getRef(), y.getRef()); + return cg::is_valid(r) ? ShapeType::get(ctx, std::move(r)) : Type(); + }); +} + +// The compiler aborts on every input tried (bad_alloc, segfault): no rule. +LogicalResult StencilDivideOp::inferReturnTypes(MLIRContext *, + std::optional loc, + ValueRange, DictionaryAttr, + PropertyRef, RegionRange, + SmallVectorImpl &) { + return emitOptionalError(loc, + "cute.stencil_divide inference is not recovered"); +} + +// No type of the compiler's registry implements the descriptor iterator +// interface this needs, so the compiler rejects every input. +LogicalResult DereferenceDescriptorIteratorOp::inferReturnTypes( + MLIRContext *, std::optional loc, ValueRange, DictionaryAttr, + PropertyRef, RegionRange, SmallVectorImpl &) { + return emitOptionalError(loc, "expects a descriptor iterator"); +} + +LogicalResult TupleProductOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { + return infer(loc, out, [&]() -> Type { + return llvm::TypeSwitch(operands[0].getType()) + .Case([&](auto ty) -> Type { + auto r = cg::product(ty.getRef()); + return cg::is_valid(r) ? decltype(ty)::get(ctx, std::move(r)) + : Type(); + }) + .Default([](Type) -> Type { return {}; }); + }); +} + +LogicalResult TupleProductEachOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { + return infer(loc, out, [&]() -> Type { + return llvm::TypeSwitch(operands[0].getType()) + .Case([&](auto ty) -> Type { + auto r = cg::product_each(ty.getRef()); + return cg::is_valid(r) ? decltype(ty)::get(ctx, std::move(r)) + : Type(); + }) + .Default([](Type) -> Type { return {}; }); + }); +} + +// A SIMT copy's fragment is the tensor itself; the rule for other atoms is +// not recovered. +LogicalResult +CopyMakeFragmentOp::inferReturnTypes(MLIRContext *, std::optional loc, + ValueRange operands, DictionaryAttr attrs, + PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto tiled = llvm::dyn_cast(a.getAtom().getType()); + Type in = a.getInput().getType(); + if (!tiled || + !llvm::isa(tiled.getCopyAtom()) || + !llvm::isa(in)) + return emitOptionalError( + loc, "copy fragment inference covers tiled SIMT copies only"); + out.push_back(in); + return success(); +} + +// The atom's shape per CTA, then how often the atom repeats over the tile. +LogicalResult TiledMmaPartitionShapeOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto atom = mmaAtomOf(a.getTiledMma().getType()); + auto shape = llvm::dyn_cast(a.getInput().getType()); + auto which = llvm::dyn_cast_or_null(a.getOperandIdAttr()); + return infer(loc, out, [&]() -> Type { + if (!atom || !shape || !which || !cg::holds_vector(shape.getRef()) || + cg::rank(shape.getRef()) != 2) + return {}; + int64_t full[2], part[2]; + switch (which.getInt()) { + case 0: + full[0] = atom->m; + full[1] = atom->k; + part[0] = atom->m / atom->ctas; + break; + case 1: + full[0] = atom->n; + full[1] = atom->k; + part[0] = atom->n / atom->ctas; + break; + case 2: + full[0] = atom->m; + full[1] = atom->n; + part[0] = atom->m / atom->ctas; + break; + default: + return {}; + } + part[1] = full[1]; + std::string rest; + for (int i = 0; i < 2; ++i) { + auto n = cg::size(shape.getRef(), i); + if (!cg::holds_int(n) || n.as_int64() % full[i]) + return {}; + rest += "," + std::to_string(n.as_int64() / full[i]); + } + auto r = + cg::from_string("((" + std::to_string(part[0]) + "," + + std::to_string(part[1]) + ")" + rest + ")"); + return r ? ShapeType::get(ctx, std::move(*r)) : Type(); + }); +} + +// ---------------------------------------------------------------- copy atoms + +namespace { + +std::optional layoutOf(const std::string &text) { + return cg::from_string(text); +} + +template int64_t asInt(const V &v) { + return cg::holds_int(v) ? v.as_int64() : -1; +} + +// TMEM addresses count lanes in the upper half-word: 2^21 bits per lane. +constexpr int64_t kTmemLaneBits = int64_t(1) << 21; + +// A copy atom's traits as CuTe spells them (cute/atom/copy_traits_*.hpp): +// the thread count and the (thr,val) -> bit layouts, and for tensor memory +// atoms the bit -> address layout. +struct CopyTraits { + int64_t threads = 1; + int64_t bits = 8; + cg::layout src, dst, ref, valId; +}; + +std::optional copyTraitsOf(Type atom) { + CopyTraits t; + auto set = [&](std::string src, std::string dst, bool refIsSrc, + std::string valId = "") -> std::optional { + auto s = layoutOf(src), d = layoutOf(dst); + if (!s || !d) + return std::nullopt; + t.src = *s; + t.dst = *d; + t.ref = refIsSrc ? *s : *d; + if (!valId.empty()) { + auto v = layoutOf(valId); + if (!v) + return std::nullopt; + t.valId = *v; + } + return t; + }; + auto n = [](int64_t v) { return std::to_string(v); }; + if (auto u = llvm::dyn_cast(atom)) { + t.bits = bitsOfType(u.getValType()); + std::string l = + "(1," + n(u.getCopyBits() ? u.getCopyBits() : t.bits) + "):(0,1)"; + return set(l, l, true); + } + if (auto s = llvm::dyn_cast(atom)) { + // SM90_U32xN_STSM_N and SM90_U16x2N_STSM_T: the SM75 ldmatrix layouts + // with source and destination swapped. + t.bits = bitsOfType(s.getValType()); + t.threads = 32; + int64_t matrices = s.getNumMatrices(); + bool transposed = bool(s.getTranspose()); + std::string ldsmSrc, ldsmDst; + if (!transposed) { + switch (matrices) { + case 1: + ldsmSrc = "((8,4),128):((128,0),1)"; + ldsmDst = "(32,32):(32,1)"; + break; + case 2: + ldsmSrc = "((16,2),128):((128,0),1)"; + ldsmDst = "(32,(32,2)):(32,(1,1024))"; + break; + case 4: + ldsmSrc = "(32,128):(128,1)"; + ldsmDst = "(32,(32,4)):(32,(1,1024))"; + break; + default: + return std::nullopt; + } + } else { + switch (matrices) { + case 1: + ldsmSrc = "((8,4),128):((128,0),1)"; + ldsmDst = "((4,8),(16,2)):((256,16),(1,128))"; + break; + case 2: + ldsmSrc = "((16,2),128):((128,0),1)"; + ldsmDst = "((4,8),(16,2,2)):((256,16),(1,128,1024))"; + break; + case 4: + ldsmSrc = "(32,128):(128,1)"; + ldsmDst = "((4,8),(16,2,4)):((256,16),(1,128,1024))"; + break; + default: + return std::nullopt; + } + } + return set(ldsmDst, ldsmSrc, true); + } + // Tensor memory loads and stores: the tmem side is the reference; a store + // is a load with source and destination swapped. + auto tmem = [&](Type val, int64_t dp, int64_t bit, int64_t x, + bool load) -> std::optional { + t.bits = bitsOfType(val); + t.threads = 32; + std::string mem, reg, valId; + if (dp == 32 && bit == 32) { + mem = "(32," + n(1024 * x) + "):(0,1)"; + reg = "(32," + n(32 * x) + "):(" + n(32 * x) + ",1)"; + valId = "(" + n(32 * x) + ",32):(1," + n(kTmemLaneBits) + ")"; + } else if (dp == 16 && bit == 256) { + mem = "(32," + n(4096 * x) + "):(0,1)"; + reg = x == 1 ? "((4,8),(64,2)):((64,256),(1,2048))" + : "((4,8),(64,2," + n(x) + ")):((64," + n(256 * x) + + "),(1," + n(2048 * x) + ",256))"; + valId = "(" + n(256 * x) + ",16):(1," + n(kTmemLaneBits) + ")"; + } else { + return std::nullopt; + } + return load ? set(mem, reg, true, valId) : set(reg, mem, false, valId); + }; + // 16-bit packing and split reductions are not covered. + if (auto l = llvm::dyn_cast(atom)) + if (!l.getPack16b()) + return tmem(l.getValType(), l.getNumDp(), l.getNumBit(), l.getNumRep(), + true); + if (auto l = llvm::dyn_cast(atom)) + if (!l.getHalfSplitOff()) + return tmem(l.getValType(), l.getNumDp(), l.getNumBit(), l.getNumRep(), + true); + if (auto st = llvm::dyn_cast(atom)) + if (!st.getExpand16b()) + return tmem(st.getValType(), st.getNumDp(), st.getNumBit(), + st.getNumRep(), false); + if (auto s = llvm::dyn_cast(atom)) { + // : SM100_UTCCP_4x32dp128bit_{C}cta. + t.bits = bitsOfType(s.getValType()); + int64_t dp = s.getNumDp(), bit = s.getNumBit(); + t.threads = s.getNumCta(); + if (dp != 32 || bit != 128 || + s.getBroadcast() != cute_nvgpu::CopyS2TBroadcast::x4 || t.threads < 1) + return std::nullopt; + return set("(" + n(t.threads) + ",(32,128,4)):(0,(1,32,0))", + "(" + n(t.threads) + ",16384):(0,1)", false, + "(32,128,4):(" + n(kTmemLaneBits) + ",1," + + n(32 * kTmemLaneBits) + ")"); + } + return std::nullopt; +} + +// A tiled copy: its atom in element units, the (thr,val) -> tile layout and +// the tiler over MN. +struct TiledCopy { + Type atom; + CopyTraits traits; + cg::layout tv; + cg::tile tiler; + int64_t atomThreads = 0, atomVals = 0, tiledThreads = 0; +}; + +std::optional tiledCopyOf(Type t) { + auto tiled = llvm::dyn_cast(t); + if (!tiled) + return std::nullopt; + auto traits = copyTraitsOf(tiled.getCopyAtom()); + if (!traits) + return std::nullopt; + TiledCopy tc; + tc.atom = tiled.getCopyAtom(); + tc.traits = *traits; + // Copy_Atom recasts the bit layouts to the value type. + tc.traits.src = cg::upcast(traits->bits, traits->src); + tc.traits.dst = cg::upcast(traits->bits, traits->dst); + tc.traits.ref = cg::upcast(traits->bits, traits->ref); + if (!cg::is_valid(tc.traits.src) || !cg::is_valid(tc.traits.dst) || + !cg::is_valid(tc.traits.ref)) + return std::nullopt; + tc.tv = tiled.getLayoutCopyTv().getRef(); + tc.tiler = tiled.getTilerMn().getRef(); + tc.atomThreads = asInt(cg::size(tc.traits.ref, 0)); + tc.atomVals = asInt(cg::size(tc.traits.ref, 1)); + tc.tiledThreads = asInt(cg::size(tc.tv, 0)); + if (tc.atomThreads <= 0 || tc.atomVals <= 0 || tc.tiledThreads <= 0) + return std::nullopt; + return tc; +} + +// The modes of a layout, one leaf mode as itself. +std::vector modesOf(const cg::layout &l) { + std::vector out; + if (!cg::holds_vector(l.shape())) + return {l}; + for (size_t i = 0; i < cg::rank(l.shape()); ++i) + out.push_back(cg::get(l, i)); + return out; +} + +// `l` with its first mode composed with `f` and the others kept: CuTe's +// tensor.compose(f, _). +cg::layout composeFirst(const cg::layout &l, const cg::layout &f) { + std::vector modes = modesOf(l); + modes[0] = cg::composition(modes[0], f); + return cg::make_layout(modes); +} + +// ((a,b),(c,d)) -> ((a,c),(b,d)) +cg::layout zip2(const cg::layout &l) { + return cg::make_layout(std::vector{ + cg::make_layout( + std::vector{cg::get(l, 0, 0), cg::get(l, 1, 0)}), + cg::make_layout( + std::vector{cg::get(l, 0, 1), cg::get(l, 1, 1)})}); +} + +cg::coord profileOf(const char *text) { + return *cg::from_string(text); +} + +// TiledCopy::tile2thrfrg: ((Tile),(Rest...)) -> (Thr, (FrgV, FrgX), Rest...) +// through the atom's (src or dst) thread-value map `ref2trg`. +cg::layout tile2thrfrg(const TiledCopy &tc, const cg::layout &tiled, + const cg::layout &ref2trg, bool flatRest = true) { + auto atomShape = + cg::from_string("(" + std::to_string(tc.atomThreads) + "," + + std::to_string(tc.atomVals) + ")"); + if (!atomShape) + return cg::layout(cg::cg_error_t{}); + cg::layout atomTV = cg::zipped_divide(tc.tv, *atomShape); + if (!cg::is_valid(atomTV)) + return atomTV; + cg::layout trgTV = composeFirst(atomTV, ref2trg); + cg::layout thrval2mn = cg::coalesce(zip2(trgTV), profileOf("(1,(1,1))")); + if (!cg::is_valid(thrval2mn)) + return thrval2mn; + cg::layout tv = cg::composition(cg::get(tiled, 0), thrval2mn); + if (!cg::is_valid(tv)) + return tv; + std::vector modes{cg::get(tv, 0), cg::get(tv, 1)}; + // The rest modes follow; a rank-1 tensor keeps its one rest mode whole. + if (!flatRest) + modes.push_back(cg::get(tiled, 1)); + else + for (const cg::layout &m : modesOf(cg::get(tiled, 1))) + modes.push_back(m); + return cg::make_layout(modes); +} + +// TiledCopy::tidfrg_S/D: (M,N,...) -> (Thr, (FrgV, FrgX), (RestM, RestN, ...)). +cg::layout tidfrg(const TiledCopy &tc, const cg::layout &tensor, bool src) { + cg::layout tiled = cg::zipped_divide(tensor, tc.tiler); + if (!cg::is_valid(tiled)) + return tiled; + cg::layout ref2trg = cg::composition(cg::right_inverse(tc.traits.ref), + src ? tc.traits.src : tc.traits.dst); + bool flatRest = + cg::holds_vector(tensor.shape()) && cg::rank(tensor.shape()) > 1; + return tile2thrfrg(tc, tiled, ref2trg, flatRest); +} + +// ThrCopy::partition_S/D: the thread's slice of tidfrg. +Type partitionCopy(MLIRContext *ctx, const TiledCopy &tc, const Carrier &c, + const cg::coord &thread, bool src) { + auto layout = llvm::dyn_cast(c.layout); + if (!layout) + return {}; + cg::layout tv = tidfrg(tc, layout.getRef(), src); + if (!cg::is_valid(tv)) + return {}; + std::string coord = "(" + cg::to_string(thread); + for (size_t i = 1; i < cg::rank(tv.shape()); ++i) + coord += ",_"; + coord += ")"; + auto crd = cg::from_string(coord); + if (!crd) + return {}; + auto [sliced, offset] = cg::slice_and_offset(*crd, tv); + if (!cg::is_valid(sliced)) + return {}; + // The tensor's modes beyond the tiler are untouched and keep their strides. + size_t in = cg::rank(layout.getRef().shape()), r = cg::rank(tc.tiler); + size_t keep = + cg::holds_vector(layout.getRef().shape()) && in > r ? in - r : 0; + Type result = LayoutType::get(ctx, zeroUnitModesHead(sliced, keep, false)); + if (c.ptr) + return CuteMemRefType::get(ctx, movedPtr(c.ptr, offset), result); + if (c.iter) { + auto sum = cg::arith_tuple_sum(c.iter.getRef(), offset); + if (!cg::is_valid(sum)) + return {}; + return CoordTensorType::get(ctx, IntTupleType::get(ctx, std::move(sum)), + result); + } + return rebuild(ctx, c, result); +} + +// The shapes of a tiler's modes, as one shape. +std::optional tilerShape(const cg::tile &tiler) { + std::string text = cg::to_string(tiler); + if (text.size() < 2 || text.front() != '[') + return std::nullopt; + SmallVector parts; + StringRef(text).drop_front().drop_back().split(parts, ';'); + std::string shape = "("; + for (size_t i = 0; i < parts.size(); ++i) { + auto l = cg::from_string(parts[i].str()); + if (!l) + return std::nullopt; + shape += (i ? "," : "") + cg::to_string(l->shape()); + } + return cg::from_string(shape + ")"); +} + +// TiledCopy::retile: (V, RestM, RestN, ...) of one thread's values regrouped +// into the atom's values. +Type retileCopy(MLIRContext *ctx, const TiledCopy &tc, const Carrier &c) { + auto layout = llvm::dyn_cast(c.layout); + if (!layout) + return {}; + const cg::layout &l = layout.getRef(); + size_t R = cg::rank(l.shape()); + int64_t V = asInt(cg::size(l, 0)); + auto shape = tilerShape(tc.tiler); + if (V <= 0 || !shape) + return {}; + cg::layout inv = cg::right_inverse(tc.tv).with_shape(*shape); + cg::layout frgMN = cg::upcast(tc.tiledThreads * V, inv); + if (!cg::is_valid(frgMN)) + return {}; + auto vLayout = layoutOf(std::to_string(V) + ":1"); + auto atomVals = layoutOf(std::to_string(tc.atomVals) + ":1"); + if (!vLayout || !atomVals) + return {}; + cg::layout frgV = cg::zipped_divide( + cg::logical_product(*vLayout, cg::right_inverse(frgMN)), *atomVals); + if (!cg::is_valid(frgV)) + return {}; + std::string tiler = "(" + std::to_string(V); + auto each = cg::product_each(frgMN.shape()); + for (size_t i = 0; i < cg::rank(each); ++i) + tiler += "," + cg::to_string(each[i]); + auto tilerShape = cg::from_string(tiler + ")"); + if (!tilerShape) + return {}; + cg::layout tiled = cg::zipped_divide(l, *tilerShape); + if (!cg::is_valid(tiled)) + return {}; + cg::layout v = composeFirst(tiled, frgV); + if (!cg::is_valid(v)) + return {}; + // v(_, (0, _, _...)): the first rest mode is the one the tile added. + std::vector modes{cg::get(v, 0)}; + std::vector rest = modesOf(cg::get(v, 1)); + for (size_t i = 1; i < rest.size() && i < R; ++i) + modes.push_back(rest[i]); + return rebuild(ctx, c, + LayoutType::get(ctx, zeroUnitModes(cg::make_layout(modes)))); +} + +// make_cotiled_copy: the (thr,val) -> tile layout and the tiler of a copy +// whose atom addresses `data` directly. +std::optional> +cotiledCopy(const cg::layout &atomTV, const cg::layout &data) { + auto unit = layoutOf("1:0"); + cg::layout invData = + cg::make_layout(std::vector{cg::left_inverse(data), *unit}); + cg::layout tvData = cg::composition(invData, atomTV); + if (!cg::is_valid(tvData)) + return std::nullopt; + auto flat = cg::product_each(data.shape()); + size_t R = cg::rank(flat); + std::string flatText = cg::to_string(flat); + std::string tilerText = "["; + for (size_t i = 0; i < R; ++i) { + std::string stride = "("; + for (size_t j = 0; j < R; ++j) + stride += std::string(j ? "," : "") + (i == j ? "1" : "0"); + auto pick = layoutOf(flatText + ":" + stride + ")"); + if (!pick) + return std::nullopt; + cg::layout mode = cg::filter(cg::composition(*pick, tvData)); + if (!cg::is_valid(mode)) + return std::nullopt; + tilerText += (i ? ";" : "") + cg::to_string(mode); + } + auto tiler = cg::from_string(tilerText + "]"); + auto flatShape = cg::from_string(flatText); + if (!tiler || !flatShape) + return std::nullopt; + cg::layout tile2data = cg::composition(cg::layout(*flatShape), *tiler); + cg::layout tv = cg::composition(cg::left_inverse(tile2data), tvData); + if (!cg::is_valid(tv)) + return std::nullopt; + return std::make_pair(tv, *tiler); +} + +Type tiledCopyType(MLIRContext *ctx, Type atom, const cg::layout &tv, + const cg::tile &tiler) { + return TiledCopyType::get(ctx, atom, LayoutAttr::get(ctx, tv), + TileAttr::get(ctx, tiler)); +} + +// ---------------------------------------------------------------- TMA + +struct TmaFormat { + std::string name; + int64_t bits = 0; +}; + +// The descriptor's data format: the attribute when given, else the element's. +std::optional tmaFormatOf(Type elem, Attribute attr) { + TmaFormat f; + if (attr) { + auto format = llvm::dyn_cast(attr); + if (!format) + return std::nullopt; + f.name = cute_nvgpu::stringifyTmaDataFormat(format.getValue()).str(); + StringRef n = f.name; + if (n == "F16_RN" || n == "BF16_RN") + f.bits = 16; + else if (n == "F32_RN" || n == "F32_FTZ_RN" || n == "INT32" || n == "TF32") + f.bits = 32; + else if (n == "F64_RN" || n == "INT64") + f.bits = 64; + else if (!n.drop_until(llvm::isDigit) + .take_while(llvm::isDigit) + .getAsInteger(10, f.bits)) { + } else + return std::nullopt; + return f; + } + f.bits = bitsOfType(elem); + if (elem.isTF32()) + f.name = "TF32_RN"; + else if (elem.isF16()) + f.name = "F16_RN"; + else if (elem.isBF16()) + f.name = "BF16_RN"; + else if (elem.isF32()) + f.name = "F32_RN"; + else if (elem.isF64()) + f.name = "F64_RN"; + else + f.name = "U" + std::to_string(f.bits); + return f; +} + +// CuTe's coalesce_256: which adjacent modes of a flattened layout merge into +// one TMA box dimension, the merged extent staying within 256. +std::vector> coalesce256Groups(const cg::layout &l) { + auto fs = cg::flatten(l.shape()); + auto ft = cg::flatten(l.stride()); + if (!cg::holds_vector(fs)) + return {{0}}; + std::vector> groups; + std::vector cur{0}; + int64_t curS = asInt(fs[0]), curD = asInt(ft[0]); + for (size_t i = 1; i < cg::rank(fs); ++i) { + int64_t s = asInt(fs[i]), d = asInt(ft[i]); + if (curS == 1) { + cur = {i}; + curS = s; + curD = d; + } else if (s == 1) { + continue; + } else if (curS > 0 && curD >= 0 && s > 0 && d >= 0 && curS * curD == d && + s * curS <= 256) { + cur.push_back(i); + curS *= s; + } else { + groups.push_back(cur); + cur = {i}; + curS = s; + curD = d; + } + } + groups.push_back(cur); + return groups; +} + +// The layout with `shape`'s leaves regrouped by `groups`, over `stride`'s +// leaves: what composing with the box shape does, keeping every basis. +std::optional +regroup(const cg::layout &shape, const cg::layout &stride, + const std::vector> &groups) { + auto fs = cg::flatten(shape.shape()); + auto ft = cg::flatten(stride.stride()); + auto leaf = [&](const auto &flat, size_t i) { + return cg::holds_vector(flat) ? cg::to_string(flat[i]) + : cg::to_string(flat); + }; + std::string shapes, strides; + for (size_t g = 0; g < groups.size(); ++g) { + if (g) { + shapes += ","; + strides += ","; + } + if (groups[g].size() == 1) { + shapes += leaf(fs, groups[g][0]); + strides += leaf(ft, groups[g][0]); + continue; + } + shapes += "("; + strides += "("; + for (size_t k = 0; k < groups[g].size(); ++k) { + if (k) { + shapes += ","; + strides += ","; + } + shapes += leaf(fs, groups[g][k]); + strides += leaf(ft, groups[g][k]); + } + shapes += ")"; + strides += ")"; + } + return layoutOf("(" + shapes + "):(" + strides + ")"); +} + +// A layout composed with a coordinate layout: each mode of `rhs` selects, +// through its basis stride, the mode of `lhs` it indexes. +cg::layout composeCoord(const cg::layout &lhs, const cg::layout &rhs) { + std::vector modes; + for (const cg::layout &m : modesOf(rhs)) { + const auto &d = m.stride(); + if (cg::holds_int(d)) { + modes.push_back(m); + continue; + } + if (!cg::holds_scaled_basis(d)) + return cg::layout(cg::cg_error_t{}); + const auto &basis = std::get(d); + if (!basis.value_holds_int()) + return cg::layout(cg::cg_error_t{}); + auto scale = layoutOf(cg::to_string(m.shape()) + ":" + + std::to_string(basis.static_integral_value())); + if (!scale) + return cg::layout(cg::cg_error_t{}); + cg::layout sub = cg::get(lhs, basis.modes()); + cg::layout composed = cg::composition(sub, *scale); + if (!cg::is_valid(composed)) + return composed; + modes.push_back(composed); + } + return cg::make_layout(modes); +} + +// Leaves of congruent shape, stride and basis, with the path to each. +template +void walkLeaves(const cg::shape &s, const cg::stride &d, const cg::stride &e, + std::vector &path, F &&f) { + if (cg::holds_vector(s)) { + for (size_t i = 0; i < cg::rank(s); ++i) { + path.push_back(i); + walkLeaves(s[i], d[i], e[i], path, f); + path.pop_back(); + } + return; + } + f(s, d, e, path); +} + +bool basisAt(const cg::stride &leaf, const std::vector &path) { + return cg::holds_scaled_basis(leaf) && + std::get(leaf).modes() == path; +} + +// Which mode of a stride tuple carries the basis `path`, if any. +std::optional modeWithBasis(const cg::stride &strides, + const std::vector &path) { + for (size_t j = 0; j < cg::rank(strides); ++j) { + auto flat = cg::flatten(strides[j]); + if (!cg::holds_vector(flat)) { + if (basisAt(flat, path)) + return j; + continue; + } + for (size_t k = 0; k < cg::rank(flat); ++k) + if (basisAt(flat[k], path)) + return j; + } + return std::nullopt; +} + +// construct_tma_gbasis: the TMA box shape with, per mode, the gmem modes it +// runs over. +cg::layout tmaGbasis(const cg::layout &gmem, int64_t elemBits, int64_t tmaBits, + const cg::layout &smem, const cg::layout &ctaVMap) { + auto fail = [] { return cg::layout(cg::cg_error_t{}); }; + cg::layout inv = cg::right_inverse(smem); + cg::layout full = cg::coalesce(cg::composition(ctaVMap, inv)); + if (!cg::is_valid(full)) + return full; + std::vector fullModes = modesOf(full); + // Keep the leading modes that step through gmem one element at a time. + size_t smemRank = 0; + for (; smemRank < fullModes.size(); ++smemRank) { + const auto &d = fullModes[smemRank].stride(); + if (!cg::holds_scaled_basis(d)) + break; + const auto &b = std::get(d); + if (!b.value_holds_int() || b.static_integral_value() != 1) + break; + } + if (smemRank == 0) + return fail(); + fullModes.resize(smemRank); + cg::layout sidx2gmode = cg::make_layout(fullModes); + cg::layout tileGstride = + cg::recast(tmaBits, elemBits, composeCoord(gmem, sidx2gmode)); + if (!cg::is_valid(tileGstride)) + return tileGstride; + cg::layout gbasis = cg::make_identity_layout(gmem.shape()); + cg::layout tileGbasisTmp = composeCoord(gbasis, sidx2gmode); + if (!cg::is_valid(tileGbasisTmp)) + return tileGbasisTmp; + auto regrouped = + regroup(tileGstride, tileGbasisTmp, coalesce256Groups(tileGstride)); + if (!regrouped) + return fail(); + cg::layout tmaGbasisTile = *regrouped; + // Gmem modes the tile misses still shape the descriptor, as size-1 modes. + cg::layout gmemT = cg::recast(tmaBits, elemBits, gmem); + if (!cg::is_valid(gmemT)) + return gmemT; + std::vector modes = modesOf(tmaGbasisTile); + auto flatTile = cg::flatten(tmaGbasisTile.stride()); + std::vector path; + bool ok = true; + walkLeaves(gmemT.shape(), gmemT.stride(), gbasis.stride(), path, + [&](const auto &s, const auto &d, const auto &e, + const std::vector &p) { + if (asInt(s) == 1 || asInt(d) == 0) + return; + bool found = false; + if (!cg::holds_vector(flatTile)) { + found = basisAt(flatTile, p); + } else { + for (size_t k = 0; k < cg::rank(flatTile) && !found; ++k) + found = basisAt(flatTile[k], p); + } + if (found) + return; + auto extra = layoutOf("1:" + cg::to_string(e)); + if (!extra) + ok = false; + else + modes.push_back(*extra); + }); + if (!ok) + return fail(); + cg::layout result = cg::make_layout(modes); + if (modes.size() > 4) + result = cg::group(4, modes.size(), result); + return result; +} + +// The TMA tensor's layout: gmem shape over TMA coordinate bases, scaled to +// the descriptor's element. +std::optional tmaTensorLayout(const cg::layout &gmem, + int64_t elemBits, int64_t tmaBits, + const cg::layout &gbasis) { + cg::layout basis = cg::make_identity_layout(gmem.shape()); + const auto &tmaStrides = gbasis.stride(); + bool ok = true; + std::string text; + std::vector path; + // Written in the shape's own nesting. + std::function + write = [&](const cg::shape &s, const cg::stride &d, + const cg::stride &e) { + if (cg::holds_vector(s)) { + text += "("; + for (size_t i = 0; i < cg::rank(s); ++i) { + if (i) + text += ","; + path.push_back(i); + write(s[i], d[i], e[i]); + path.pop_back(); + } + text += ")"; + return; + } + if (asInt(s) == 1 || asInt(d) == 0) { + text += "0"; + return; + } + auto j = cg::holds_vector(tmaStrides) + ? modeWithBasis(tmaStrides, path) + : (basisAt(tmaStrides, path) ? std::optional(0) + : std::nullopt); + if (!j) { + text += "0"; + return; + } + if (*j == 0) { + int64_t stride = asInt(d); + if (stride <= 0) { + ok = false; + return; + } + int64_t num = stride * elemBits, den = tmaBits; + int64_t g = std::gcd(num, den); + num /= g; + den /= g; + text += std::to_string(num) + + (den == 1 ? "" : "/" + std::to_string(den)) + "@0"; + return; + } + auto mode = cg::holds_vector(tmaStrides) ? tmaStrides[*j] : tmaStrides; + auto flat = cg::flatten(mode); + if (cg::holds_vector(flat) && cg::rank(flat) != 1) { + ok = false; + return; + } + text += "1@" + std::to_string(*j); + }; + write(gmem.shape(), gmem.stride(), basis.stride()); + if (!ok) + return std::nullopt; + return layoutOf(cg::to_string(gmem.shape()) + ":" + text); +} + +// The smem tile layout without its swizzle. +std::optional plainLayoutOf(Type t) { + if (auto l = llvm::dyn_cast(t)) + return l.getRef(); + if (auto c = llvm::dyn_cast(t)) + return c.getRef().layout_b(); + return std::nullopt; +} + +struct TmaAtomInfo { + int64_t elemBits = 0, copyBits = 0; +}; + +std::optional tmaAtomInfoOf(Type atom) { + TmaAtomInfo info; + if (auto l = + llvm::dyn_cast(atom)) { + info.elemBits = bitsOfType(l.getValType()); + info.copyBits = l.getCopyBits(); + } else if (auto s = + llvm::dyn_cast( + atom)) { + info.elemBits = bitsOfType(s.getValType()); + info.copyBits = s.getCopyBits(); + } else if (auto r = + llvm::dyn_cast( + atom)) { + info.elemBits = bitsOfType(r.getValType()); + info.copyBits = r.getCopyBits(); + } else if (auto l = + llvm::dyn_cast( + atom)) { + info.elemBits = bitsOfType(l.getValType()); + info.copyBits = l.getCopyBits(); + } else if (auto s = + llvm::dyn_cast( + atom)) { + info.elemBits = bitsOfType(s.getValType()); + info.copyBits = s.getCopyBits(); + } else { + return std::nullopt; + } + if (info.elemBits <= 0 || info.copyBits <= 0) + return std::nullopt; + return info; +} + +// make_non_exec_tiled_tma_load/store: the atom and the TMA tensor. +LogicalResult inferTmaAtom(MLIRContext *ctx, std::optional loc, + Type gmemType, Type smemType, Type ctaVMapType, + Attribute kind, Attribute format, + SmallVectorImpl &out) { + auto gmem = llvm::dyn_cast(gmemType); + auto smem = plainLayoutOf(smemType); + auto ctaVMap = llvm::dyn_cast(ctaVMapType); + if (!gmem || !smem || !ctaVMap) + return emitOptionalError( + loc, "expects a gmem tensor, an smem layout and a CTA value map"); + auto glayout = llvm::dyn_cast(gmem.getLayout()); + if (!glayout) + return emitOptionalError(loc, "gmem tensor needs a plain layout"); + Type elem = gmem.getValueType(); + auto fmt = tmaFormatOf(elem, format); + if (!fmt || fmt->bits <= 0) + return emitOptionalError(loc, "unknown TMA data format"); + int64_t elemBits = bitsOfType(elem); + cg::layout gbasis = + tmaGbasis(glayout.getRef(), elemBits, fmt->bits, *smem, ctaVMap.getRef()); + if (!cg::is_valid(gbasis)) + return emitOptionalError(loc, "cannot build the TMA basis"); + auto tensor = tmaTensorLayout(glayout.getRef(), elemBits, fmt->bits, gbasis); + if (!tensor) + return emitOptionalError(loc, "cannot build the TMA tensor"); + int64_t copyBits = asInt(cg::size(gbasis)) * fmt->bits; + auto gbasisType = LayoutType::get(ctx, gbasis); + std::optional dataFormat = + cute_nvgpu::symbolizeTmaDataFormat(fmt->name); + Type atom; + if (auto k = llvm::dyn_cast_or_null(kind)) { + atom = cute_nvgpu::CopyAtomNonExecTiledTmaLoadType::get( + ctx, k.getValue(), elem, copyBits, gbasisType, dataFormat); + } else if (auto k = + llvm::dyn_cast_or_null(kind)) { + atom = cute_nvgpu::CopyAtomNonExecTiledTmaReduceType::get( + ctx, k.getValue(), elem, copyBits, gbasisType, dataFormat); + } else { + atom = cute_nvgpu::CopyAtomNonExecTiledTmaStoreType::get( + ctx, elem, copyBits, gbasisType, dataFormat); + } + std::string zeros = "("; + for (size_t i = 0; i < cg::rank(gbasis.shape()); ++i) + zeros += i ? ",0" : "0"; + auto iter = cg::from_string(zeros + ")"); + if (!iter) + return emitOptionalError(loc, "cannot build the TMA coordinate"); + out.push_back(atom); + out.push_back(CoordTensorType::get(ctx, IntTupleType::get(ctx, *iter), + LayoutType::get(ctx, *tensor))); + return success(); +} + +} // namespace + +// ---------------------------------------------------------------- cute copy +// ops + +// TiledCopy::get_layoutD_TV over the reference tile, then the tiled copy of +// another atom with that layout (make_tiled_copy_D). +Type mlir::cutlass_compiler::cute::tiledCopyD(Type atom, TiledCopyType tiled) { + auto tc = tiledCopyOf(tiled); + if (!tc) + return {}; + auto shape = tilerShape(tc->tiler); + if (!shape) + return {}; + auto refShape = + cg::from_string("(" + cg::to_string(*shape) + ",1)"); + if (!refShape) + return {}; + cg::layout refD(*refShape); + cg::layout ref2trg = + cg::composition(cg::right_inverse(tc->traits.ref), tc->traits.dst); + cg::layout tv = tile2thrfrg(*tc, refD, ref2trg); + if (!cg::is_valid(tv) || cg::rank(tv.shape()) < 2) + return {}; + cg::layout layoutTV = zeroUnitModes( + cg::make_layout(std::vector{cg::get(tv, 0), cg::get(tv, 1)})); + MLIRContext *ctx = atom.getContext(); + return TiledCopyType::get(ctx, atom, LayoutAttr::get(ctx, layoutTV), + tiled.getTilerMn()); +} + +LogicalResult TiledCopyPartitionSOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto tc = tiledCopyOf(a.getTiledCopy().getType()); + auto c = carrierOf(a.getInput().getType()); + auto coord = llvm::dyn_cast(a.getCoord().getType()); + if (!tc || !c || !coord) + return emitOptionalError( + loc, "expects a tiled copy, a tensor and a thread coord"); + return infer(loc, out, [&] { + return partitionCopy(ctx, *tc, *c, coord.getRef(), true); + }); +} + +LogicalResult TiledCopyPartitionDOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto tc = tiledCopyOf(a.getTiledCopy().getType()); + auto c = carrierOf(a.getInput().getType()); + auto coord = llvm::dyn_cast(a.getCoord().getType()); + if (!tc || !c || !coord) + return emitOptionalError( + loc, "expects a tiled copy, a tensor and a thread coord"); + return infer(loc, out, [&] { + return partitionCopy(ctx, *tc, *c, coord.getRef(), false); + }); +} + +LogicalResult TiledCopyRetileOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto tc = tiledCopyOf(a.getTiledCopy().getType()); + auto c = carrierOf(a.getInput().getType()); + if (!tc || !c) + return emitOptionalError(loc, "expects a tiled copy and a tensor"); + return infer(loc, out, [&] { return retileCopy(ctx, *tc, *c); }); +} + +// ---------------------------------------------------------------- cute_nvgpu + +namespace mlir::cutlass_compiler::cute_nvgpu { + +LogicalResult AtomSetValueOp::inferReturnTypes( + MLIRContext *, std::optional, ValueRange operands, DictionaryAttr, + PropertyRef, RegionRange, SmallVectorImpl &out) { + out.push_back(operands[0].getType()); + return success(); +} + +// The executable atom of a TMA descriptor: the element, copy size and basis +// carry over; the kind becomes the mode and, for loads, the CTA count. +LogicalResult AtomCopyMakeExecTmaOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + bool over = a.getOverride(), noOob = a.getNoFullyOobTile(); + Type in = a.getInput().getType(); + auto stride = cg::from_string("()"); + if (!stride) + return emitOptionalError(loc, "cannot build the TMA stride"); + auto gStride = StrideType::get(ctx, *stride); + if (auto load = llvm::dyn_cast(in)) { + TiledTmaLoad kind = load.getKind(); + bool twoSm = kind == TiledTmaLoad::sm_100_2sm || + kind == TiledTmaLoad::sm_100_2sm_multicast; + bool mcast = kind == TiledTmaLoad::sm_90_multicast || + kind == TiledTmaLoad::sm_100_2sm_multicast; + out.push_back(CopyAtomTmaLoadType::get( + ctx, load.getValType(), /*sparsity=*/1, load.getCopyBits(), + TmaLoadMode::tiled, twoSm ? 2 : 1, gStride, mcast, load.getTmaGbasis(), + over, noOob)); + return success(); + } + if (auto store = llvm::dyn_cast(in)) { + out.push_back(CopyAtomTmaStoreType::get( + ctx, store.getValType(), /*sparsity=*/1, store.getCopyBits(), + TmaStoreMode::tiled, gStride, store.getTmaGbasis(), over, noOob)); + return success(); + } + if (auto red = llvm::dyn_cast(in)) { + out.push_back(CopyAtomTmaReduceType::get( + ctx, red.getValType(), red.getCopyBits(), TmaStoreMode::tiled, + red.getKind(), gStride, red.getTmaGbasis())); + return success(); + } + if (auto load = llvm::dyn_cast(in)) { + Im2ColTmaLoad kind = load.getKind(); + bool twoSm = kind == Im2ColTmaLoad::sm_100_2sm || + kind == Im2ColTmaLoad::sm_100_2sm_multicast; + bool mcast = kind == Im2ColTmaLoad::sm_90_multicast || + kind == Im2ColTmaLoad::sm_100_2sm_multicast; + out.push_back(CopyAtomIm2ColTmaLoadType::get( + ctx, load.getValType(), load.getCopyBits(), twoSm ? 2 : 1, gStride, + mcast, load.getTmaGbasis())); + return success(); + } + if (auto store = llvm::dyn_cast(in)) { + out.push_back(CopyAtomIm2ColTmaStoreType::get(ctx, store.getValType(), + store.getCopyBits(), gStride, + store.getTmaGbasis())); + return success(); + } + // The gather/scatter atoms' executable forms are not recovered. + return emitOptionalError(loc, + "expects a non-executable tiled or im2col TMA atom"); +} + +LogicalResult AtomCopyMakeNonExecTiledTmaLoadOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + return inferTmaAtom(ctx, loc, a.getGmemTensor().getType(), + a.getSmemLayout().getType(), a.getCtaVMap().getType(), + a.getKindAttr(), a.getTmaFormatAttr(), out); +} + +LogicalResult AtomCopyMakeNonExecTiledTmaStoreOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + return inferTmaAtom(ctx, loc, a.getGmemTensor().getType(), + a.getSmemLayout().getType(), a.getCtaVMap().getType(), + /*kind=*/nullptr, a.getTmaFormatAttr(), out); +} + +LogicalResult AtomCopyMakeNonExecTiledTmaReduceOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + return inferTmaAtom(ctx, loc, a.getGmemTensor().getType(), + a.getSmemLayout().getType(), a.getCtaVMap().getType(), + a.getKindAttr(), a.getTmaFormatAttr(), out); +} + +// tma_partition: the smem tile's contiguous vectors, one instruction each, +// read through every tensor; a multicast CTA starts at its share. +LogicalResult AtomTmaPartitionOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto info = tmaAtomInfoOf(a.getTmaAtom().getType()); + auto coord = llvm::dyn_cast(a.getCtaCoord().getType()); + auto ctaLayout = llvm::dyn_cast(a.getCtaLayout().getType()); + auto smem = carrierOf(a.getSmemTensor().getType()); + if (!info || !coord || !ctaLayout || !smem) + return emitOptionalError( + loc, "expects a TMA atom, a CTA coord and layout, and tensors"); + auto smemLayout = llvm::dyn_cast_or_null(smem->layout); + if (!smemLayout || !cg::holds_vector(smemLayout.getRef().shape())) + return emitOptionalError(loc, + "smem tensor needs a plain layout of rank >= 1"); + int64_t numVal = info->copyBits / info->elemBits; + cg::layout inv = cg::right_inverse(cg::get(smemLayout.getRef(), 0)); + int64_t b = asInt(cg::size(inv)); + int64_t n = asInt(cg::size(smemLayout.getRef(), 0)); + if (b <= 0 || n <= 0 || n % b || numVal <= 0) + return emitOptionalError(loc, "cannot tile the smem vector"); + // tile_to_shape: the vector repeated over the tile, broadcast modes at 0. + cg::layout layoutV = inv; + if (n != b) { + auto rest = layoutOf(std::to_string(n / b) + ":1"); + layoutV = cg::logical_product(inv, *rest); + if (!cg::is_valid(layoutV)) + return emitOptionalError(loc, "cannot tile the smem vector"); + } + auto tmaV = layoutOf(std::to_string(numVal) + ":1"); + layoutV = cg::logical_divide(layoutV, *tmaV); + if (!cg::is_valid(layoutV)) + return emitOptionalError(loc, "cannot divide the smem vector"); + // The multicast offset inside the instruction's vector. + int64_t ctas = asInt(cg::cosize(ctaLayout.getRef())); + std::optional multicast; + if (ctas > 1) { + // The CTA layout scaled by the share: (cosize:share) o cta_layout. + auto share = + layoutOf(std::to_string(ctas) + ":" + std::to_string(numVal / ctas)); + cg::layout scaled = cg::composition(*share, ctaLayout.getRef()); + if (!cg::is_valid(scaled)) + return emitOptionalError(loc, "cannot scale the CTA layout"); + multicast = cg::layout_eval(coord.getRef(), scaled); + } + SmallVector results; + auto partition = [&](Type tensor) -> Type { + auto c = carrierOf(tensor); + if (!c) + return {}; + auto layout = llvm::dyn_cast(c->layout); + if (!layout) + return {}; + cg::layout v = cg::coalesce(composeFirst(layout.getRef(), layoutV), + profileOf("((1,1))")); + if (!cg::is_valid(v)) + return {}; + // Only the vector's unit modes get a zero stride; the rest keep theirs. + Type result = LayoutType::get( + ctx, zeroUnitModesHead(v, cg::rank(v.shape()) - 1, false)); + if (!multicast) + return rebuild(ctx, *c, result); + // The index split over the vector's modes, column-major, keeping what + // its divisibility says of each coordinate. + auto vec = cg::get(v.shape(), 0); + if (!cg::holds_vector(vec)) + return {}; + auto split = splitIndex(*multicast, vec[0]); + if (!split) + return {}; + std::string crd = "((" + *split + ",0)"; + for (size_t i = 1; i < cg::rank(v.shape()); ++i) + crd += ",0"; + auto at = cg::from_string(crd + ")"); + if (!at) + return {}; + cg::int_tuple offset = cg::layout_eval(*at, v); + if (c->ptr) + return CuteMemRefType::get(ctx, movedPtr(c->ptr, offset), result); + if (c->iter) { + auto sum = cg::arith_tuple_sum(c->iter.getRef(), offset); + if (!cg::is_valid(sum)) + return {}; + return CoordTensorType::get(ctx, IntTupleType::get(ctx, std::move(sum)), + result); + } + return rebuild(ctx, *c, result); + }; + results.push_back(partition(a.getSmemTensor().getType())); + for (Value target : a.getTargetTensors()) + results.push_back(partition(target.getType())); + for (Type t : results) + if (!t) + return emitOptionalError(loc, "cannot partition for TMA"); + out.append(results.begin(), results.end()); + return success(); +} + +namespace { +// make_cotiled_copy over a tensor memory tensor for an atom whose thread +// layout is `atomT`. +Type tmemTiledCopy(MLIRContext *ctx, std::optional loc, Type atom, + Type tmem, const std::string &atomT, + SmallVectorImpl &out) { + auto traits = copyTraitsOf(atom); + auto m = llvm::dyn_cast(tmem); + auto layout = m ? llvm::dyn_cast(m.getLayout()) : LayoutType(); + if (!traits || !layout || !cg::is_valid(traits->valId)) + return {}; + auto t = layoutOf(atomT); + if (!t) + return {}; + cg::layout v = cg::coalesce(cg::upcast(traits->bits, traits->valId)); + if (!cg::is_valid(v)) + return {}; + auto tiled = cotiledCopy(cg::make_layout(std::vector{*t, v}), + layout.getRef()); + if (!tiled) + return {}; + return tiledCopyType(ctx, atom, tiled->first, tiled->second); +} +} // namespace + +// make_tmem_copy: four warps, each at the same place in its own lane quarter. +LogicalResult AtomCopyMakeTmemCopyOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto traits = copyTraitsOf(a.getAtom().getType()); + if (!traits || traits->bits <= 0) + return emitOptionalError(loc, "expects a tmem atom"); + int64_t lane = kTmemLaneBits / traits->bits; + return infer(loc, out, [&] { + return tmemTiledCopy(ctx, loc, a.getAtom().getType(), + a.getTmemMemref().getType(), + "(32,4):(0," + std::to_string(32 * lane) + ")", out); + }); +} + +// make_s2t_copy (make_utccp_copy): one thread per CTA. +LogicalResult AtomCopyMakeS2TCopyOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + auto traits = copyTraitsOf(a.getCopyS2tAtom().getType()); + if (!traits) + return emitOptionalError(loc, "expects an s2t atom"); + return infer(loc, out, [&] { + return tmemTiledCopy(ctx, loc, a.getCopyS2tAtom().getType(), + a.getTmemMemref().getType(), + std::to_string(traits->threads) + ":0", out); + }); +} + +namespace { +// The smem side of an s2t copy as descriptor units: the tile recast to 128 +// bits, viewed through `desc`. +LogicalResult s2tSmemDescView(std::optional loc, Type atom, Type view, + Type desc, SmallVectorImpl &out) { + if (auto tiled = llvm::dyn_cast(atom)) + atom = tiled.getCopyAtom(); + auto m = llvm::dyn_cast(view); + auto layout = m ? llvm::dyn_cast(m.getLayout()) : LayoutType(); + if (!llvm::isa(atom) || !layout) + return emitOptionalError(loc, "expects an s2t atom and an smem tensor"); + // The compiler wants mode 0 as (values, 1). + const cg::layout &l = layout.getRef(); + auto m0 = cg::get(l.shape(), 0); + if (!cg::holds_vector(m0) || cg::rank(m0) != 2 || + !cg::holds_int(cg::size(m0, 1)) || cg::size(m0, 1).as_int64() != 1) + return emitOptionalError(loc, "expects mode 0 as (values, 1)"); + return infer(loc, out, [&]() -> Type { + cg::layout v = cg::recast(128, bitsOfType(m.getValueType()), l); + if (!cg::is_valid(v)) + return {}; + return SmemDescViewType::get(atom.getContext(), desc, + LayoutAttr::get(atom.getContext(), v)); + }); +} + +// An operation whose inference rule is not recovered: the compiler either +// aborts on the inputs tried or needs inputs no probe produced. +LogicalResult unrecovered(std::optional loc, StringRef op) { + return emitOptionalError(loc, op, " result inference is not recovered"); +} +} // namespace + +LogicalResult AtomGetCopyS2TSmemDescViewOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + return s2tSmemDescView(loc, a.getAtom().getType(), a.getView().getType(), + SmemDescType::get(ctx), out); +} + +LogicalResult AtomGetCopyS2TSmemDescViewSM107Op::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + return s2tSmemDescView(loc, a.getAtom().getType(), a.getView().getType(), + SmemDescSM107Type::get(ctx), out); +} + +// A generic-space pointer to the parameter, at its type's natural alignment. +LogicalResult GetGridConstantPointerOp::inferReturnTypes( + MLIRContext *ctx, std::optional, ValueRange operands, + DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { + Type t = operands[0].getType(); + bool desc = llvm::isa(t); + out.push_back(PtrType::get(ctx, t, AddressSpace::generic, + desc ? 64 : PtrType::getNaturalAlignment(t), {}, + {})); + return success(); +} + +// The residues of the TMA tensor's coordinates: one per box mode, starting +// at the extent of the gmem mode it runs over and counting down. +LogicalResult AtomMakeTmaResidueTensorOp::inferReturnTypes( + MLIRContext *ctx, std::optional loc, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange, + SmallVectorImpl &out) { + Adaptor a(operands, attrs, props); + Type atom = a.getTmaAtom().getType(), gbasis; + if (auto l = llvm::dyn_cast(atom)) + gbasis = l.getTmaGbasis(); + if (auto s = llvm::dyn_cast(atom)) + gbasis = s.getTmaGbasis(); + auto basis = llvm::dyn_cast_or_null(gbasis); + auto m = llvm::dyn_cast(a.getGmemTensor().getType()); + auto gmem = m ? llvm::dyn_cast(m.getLayout()) : LayoutType(); + return infer(loc, out, [&]() -> Type { + if (!basis || !gmem || !cg::holds_vector(basis.getRef().stride())) + return {}; + const cg::stride &modes = basis.getRef().stride(); + // Each box mode's one basis: the gmem leaf it runs over. + std::vector> paths; + for (size_t j = 0; j < cg::rank(modes); ++j) { + auto flat = cg::flatten(modes[j]); + const cg::stride &leaf = + cg::holds_vector(flat) && cg::rank(flat) == 1 ? flat[0] : flat; + if (!cg::holds_scaled_basis(leaf)) + return {}; + paths.push_back(std::get(leaf).modes()); + } + std::vector iter(paths.size()), strides; + std::vector path; + const cg::layout &g = gmem.getRef(); + walkLeaves(g.shape(), g.stride(), g.stride(), path, + [&](const auto &s, const auto &, const auto &, + const std::vector &p) { + auto j = + std::find(paths.begin(), paths.end(), p) - paths.begin(); + if (j == static_cast(paths.size())) + return strides.push_back("0"); + iter[j] = cg::to_string(s); + strides.push_back("-1@" + std::to_string(j)); + }); + for (const std::string &e : iter) + if (e.empty()) + return {}; + size_t i = 0; + auto layout = cg::from_string(cg::to_string(g.shape()) + ":" + + nestLike(g.shape(), strides, i)); + auto start = + cg::from_string("(" + llvm::join(iter, ",") + ")"); + if (!layout || !start) + return {}; + return CoordTensorType::get(ctx, IntTupleType::get(ctx, *start), + LayoutType::get(ctx, *layout)); + }); +} + +LogicalResult AtomGetCoordTensorOp::inferReturnTypes( + MLIRContext *, std::optional loc, ValueRange, DictionaryAttr, + PropertyRef, RegionRange, SmallVectorImpl &) { + return unrecovered(loc, "cute_nvgpu.atom.get_coord_tensor"); +} + +// The im2col and gather/scatter TMA atoms and the circular descriptors +// validate inputs (canonical UMMA layouts, im2col corners) whose rules are +// not recovered. +#define CUTE_UNRECOVERED(Op, name) \ + LogicalResult Op::inferReturnTypes( \ + MLIRContext *, std::optional loc, ValueRange, DictionaryAttr, \ + PropertyRef, RegionRange, SmallVectorImpl &) { \ + return unrecovered(loc, name); \ + } +CUTE_UNRECOVERED(AtomCopyMakeNonExec2DGather4TmaLoadOp, + "cute_nvgpu.atom.make_non_exec_2d_gather4_tma_load") +CUTE_UNRECOVERED(AtomCopyMakeNonExec2DScatter4TmaStoreOp, + "cute_nvgpu.atom.make_non_exec_2d_scatter4_tma_store") +CUTE_UNRECOVERED(AtomCopyMakeNonExecIm2ColTmaLoadOp, + "cute_nvgpu.atom.make_non_exec_im2col_tma_load") +CUTE_UNRECOVERED(AtomCopyMakeNonExecIm2ColTmaStoreOp, + "cute_nvgpu.atom.make_non_exec_im2col_tma_store") +CUTE_UNRECOVERED(MakeUMMASmemDescCircularSM103Op, + "cute_nvgpu.sm103.make_umma_smem_desc_circular") +CUTE_UNRECOVERED(MakeUMMASmemDescCircularSM107Op, + "cute_nvgpu.sm107.make_umma_smem_desc_circular") +#undef CUTE_UNRECOVERED + +} // namespace mlir::cutlass_compiler::cute_nvgpu diff --git a/mlir/dialects/cute_ir/CuteInference.h b/mlir/dialects/cute_ir/CuteInference.h new file mode 100644 index 00000000..c7837b20 --- /dev/null +++ b/mlir/dialects/cute_ir/CuteInference.h @@ -0,0 +1,24 @@ +//===- CuteInference.h - result types the compiler infers -------*- C++ -*-===// + +#ifndef MLIR_DIALECTS_CUTE_IR_CUTEINFERENCE_H +#define MLIR_DIALECTS_CUTE_IR_CUTEINFERENCE_H + +#include "cute_ir/Dialect/Cute/IR/CuteDialect.h" + +namespace mlir::cutlass_compiler::cute { +// make_tiled_copy_D: `atom` tiled over `tiled`'s destination thread-value +// layout and tiler, or null when the atom is not one this tree knows. +Type tiledCopyD(Type atom, TiledCopyType tiled); + +// Result types of operations the compiler does not infer (their builders +// take the type): what the DSL passes, or null when not computable here. +Type makeLayoutResultType(Type shape, Type stride); // stride may be null +Type makeViewResultType(Type iter, Type layout); +Type getResultType(Type input, ArrayRef mode); +Type assumeResultType(Type src); +Type makeArithTupleIterResultType(Type tuple); +Type fastDivmodCreateDivisorResultType(Type divisor); +Type fastDivmodComputeResultType(Type divisor); // both results +} // namespace mlir::cutlass_compiler::cute + +#endif // MLIR_DIALECTS_CUTE_IR_CUTEINFERENCE_H diff --git a/mlir/dialects/cute_ir/CuteOps.cpp b/mlir/dialects/cute_ir/CuteOps.cpp new file mode 100644 index 00000000..0c8ddaf3 --- /dev/null +++ b/mlir/dialects/cute_ir/CuteOps.cpp @@ -0,0 +1,9 @@ +//===- CuteOps.cpp - the compiler's CuTe operations -----------------------===// + +#include "cute_ir/Dialect/Cute/IR/CuteDialect.h" +// Operand constraints name cute_nvgpu and LLVM types. +#include "mlir/Dialect/LLVMIR/LLVMTypes.h" +#include "mlir/dialects/cute_nvgpu/CuteNVGPUDialect.h" + +#define GET_OP_CLASSES +#include "mlir/dialects/cute_ir/CuteOps.cpp.inc" diff --git a/mlir/dialects/cute_ir/CuteOps.td b/mlir/dialects/cute_ir/CuteOps.td new file mode 100644 index 00000000..d2e342df --- /dev/null +++ b/mlir/dialects/cute_ir/CuteOps.td @@ -0,0 +1,3795 @@ +// The cute dialect's operations, generated from the nvidia-cutlass-dsl 4.8.0 release; do not edit. +// +// Sources: the operation bindings (`_cute_ops_gen.py`: names, classes, +// docstrings, operand/result/attribute structure, inference) and the live +// compiler (`_cutlass_ir` loaded from Python): each operand and result +// constraint carries the verifier's wording and accepts every type class the +// verifier let through when probed with one instance of every type; traits +// and interfaces are the registered operation's; memory effects come from +// loop-invariant code motion (hoisted: Pure), canonicalize (dead: no write) +// and CSE (merged: at most reads). Operations whose effects include writes +// or could not be told apart carry none (unknown effects). +// +// Operations keep the generic assembly form; the compiler's custom syntax is +// shown in each description. +// +// Constraints not recovered: +// cute.deref_desc_iter operand #0: no probe type accepted; any type allowed +// cute.make_desc_iter operand #0: no probe type accepted; any type allowed + +#ifndef MLIR_DIALECTS_CUTE_IR_CUTEOPS_TD +#define MLIR_DIALECTS_CUTE_IR_CUTEOPS_TD + +include "cute_ir/Dialect/Cute/IR/CuteAttrs.td" +include "mlir/dialects/cute_ir/CuteEnums.td" + +include "mlir/Interfaces/InferTypeOpInterface.td" +include "mlir/Interfaces/SideEffectInterfaces.td" + +// Operand and result constraints, in the compiler's words. +def Cute_32BitSignlessIntegerOr64BitSignless : Type, "32-bit signless integer or 64-bit signless integer">; +def Cute_AMemoryReferenceTypeThatRepresentsATensor : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::MemRefType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ShapeType>($_self))">, "A memory reference type that represents a tensor view with an iterator and layout or CuTe Layout Type or Cute shape type">; +def Cute_AMemoryReferenceTypeThatRepresentsATensor2 : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::MemRefType>($_self))">, "A memory reference type that represents a tensor view with an iterator and layout or Cute coord_tensor type">; +def Cute_AMemoryReferenceTypeThatRepresentsATensor3 : Type($_self))">, "A memory reference type that represents a tensor view with an iterator and layout">; +def Cute_ATupleIsAFixedLengthOrderedList : Type($_self))">, "A tuple is a fixed-length, ordered list consisting of multiple elements.">; +def Cute_ATypeInterfaceForDescriptorIteratorTypes : Type, "A type interface for descriptor iterator types.">; // not recovered: no pool type passed +def Cute_ATypeInterfaceForMemoryDescriptorsSuchAs : Type, "A type interface for memory descriptors such as smem desc etc.">; // not recovered: no pool type passed +def Cute_ArithmeticTupleType : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ShapeType>($_self))">, "Arithmetic tuple type">; +def Cute_AtomTypeInterfaceInstance : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TiledCopyV2Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TiledMmaType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM120BlockScaledType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyG2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyS2GType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyS2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomDsmemStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomG2RType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomLdsmType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DGather4TmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DScatter4TmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomR2GType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomR2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomS2RType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100CopyS2TType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTAsyncCopyType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemLdReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemRedType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemStType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100S2TCopyV2Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomStsmType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100TmemLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM10xTmemLoadRedType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM107TmemLoadSPCompressType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100TmemStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTSyncCopyType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::UniversalFmaAtomType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMAType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMABlockScaledType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMABlockScaledSparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMASparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMAType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMABlockScaledType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMABlockScaledSparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMASparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM80Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM80SparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM89Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM90Type>($_self))">, "AtomTypeInterface instance">; +def Cute_CopyAtomTypeInterfaceInstance : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyG2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyS2GType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyS2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomDsmemStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomG2RType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomLdsmType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DGather4TmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DScatter4TmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomR2GType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomR2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomS2RType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100CopyS2TType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTAsyncCopyType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemLdReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemRedType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemStType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomStsmType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100TmemLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM10xTmemLoadRedType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM107TmemLoadSPCompressType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100TmemStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTSyncCopyType>($_self))">, "CopyAtomTypeInterface instance">; +def Cute_CopyAtomTypeInterfaceInstanceOrCopyAtomTypeInterfaceV2Instance : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TiledCopyV2Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyG2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyS2GType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyS2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomDsmemStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomG2RType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomLdsmType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DGather4TmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DScatter4TmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomR2GType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomR2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomS2RType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100CopyS2TType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTAsyncCopyType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemLdReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemRedType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemStType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100S2TCopyV2Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomStsmType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100TmemLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM10xTmemLoadRedType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM107TmemLoadSPCompressType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100TmemStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTSyncCopyType>($_self))">, "CopyAtomTypeInterface instance or CopyAtomTypeInterfaceV2 instance">; +def Cute_CopyAtomTypeInterfaceV2Instance : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100S2TCopyV2Type>($_self))">, "CopyAtomTypeInterfaceV2 instance">; +def Cute_CuTeComposedLayoutType : Type($_self))">, "CuTe composed layout type">; +def Cute_CuTeIteratorTypeWithIntTupleAsIts : Type($_self))">, "CuTe iterator type with int_tuple as its underlying implementation">; +def Cute_CuTeLayoutType : Type($_self))">, "CuTe Layout Type">; +def Cute_CuTeLayoutTypeOrCuteSwizzleType : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::SwizzleType>($_self))">, "CuTe Layout Type or Cute swizzle type">; +def Cute_CuTeLayoutTypeOrSwizzleWithValidParameters : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::SwizzleType>($_self))">, "CuTe Layout Type or swizzle with valid parameters that can be represented by 32-bit masks">; +def Cute_CuTePointerTo32BitSignlessInteger : Type($_self))">, "CuTe pointer to 32-bit signless integer">; +def Cute_CuTePointerType : Type($_self))">, " CuTe pointer type ">; +def Cute_CuTePointerTypeOrAMemoryReferenceType : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordTensorType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::FastDivmodDivisorType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ConstrainedIntType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::IntTupleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::LayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::MemRefType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::PtrType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ShapeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::StrideType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TileType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TiledCopyV2Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TiledMmaType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM120BlockScaledType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyG2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyS2GType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyS2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomDsmemStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomG2RType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DGather4TmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DScatter4TmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomR2GType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100S2TCopyV2Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMAType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMABlockScaledType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMABlockScaledSparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMASparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMAType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMABlockScaledType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMABlockScaledSparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMASparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM80SparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM90Type>($_self) || ::llvm::isa<::mlir::IntegerType>($_self) || ::llvm::isa<::mlir::FloatType>($_self) || ::llvm::isa<::mlir::LLVM::LLVMArrayType>($_self) || ::llvm::isa<::mlir::LLVM::LLVMPointerType>($_self) || ::llvm::isa<::mlir::LLVM::LLVMStructType>($_self) || ($_self.getDialect().getNamespace() == \"ptr\" || (::llvm::isa<::mlir::OpaqueType>($_self) && ::llvm::cast<::mlir::OpaqueType>($_self).getDialectNamespace() == \"ptr\")) || ::llvm::isa<::mlir::VectorType>($_self))">, " CuTe pointer type or A memory reference type that represents a tensor view with an iterator and layout or Integer type with constraints. or CuTe fast divmod divisor type or loadable maybe-static CuTe type with runtime storage or LLVM type with size">; +def Cute_CuTePointerTypeOrLLVMPointerType : Type($_self) || ::llvm::isa<::mlir::LLVM::LLVMPointerType>($_self))">, " CuTe pointer type or LLVM pointer type">; +def Cute_CuTeSparseElementType : Type($_self))">, " CuTe sparse element type">; +def Cute_CuteCoordTensorType : Type($_self))">, "Cute coord_tensor type">; +def Cute_CuteCoordType : Type($_self))">, "Cute coord type">; +def Cute_CuteShapeType : Type($_self))">, "Cute shape type">; +def Cute_CuteShapeTypeOrIntTupleAsIntegerOr : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::IntTupleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ShapeType>($_self))">, "Cute shape type or IntTuple as integer or tuple of IntTuple(s) or Cute coord type">; +def Cute_CuteShapeTypeOrIntTupleAsIntegerOr2 : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordTensorType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::IntTupleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::LayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::MemRefType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ShapeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::SmemDescViewType>($_self))">, "Cute shape type or IntTuple as integer or tuple of IntTuple(s) or Cute coord type or CuTe Layout Type or CuTe composed layout type or ViewTypeInterface instance">; +def Cute_CuteStrideType : Type($_self))">, "Cute stride type">; +def Cute_CuteTileType : Type($_self))">, "Cute tile type">; +def Cute_DivisorForFastUnsignedIntegerDivmodsByInvariant : Type($_self))">, "Divisor for fast unsigned integer divmods by invariant integers.">; +def Cute_GeneralizedIteratorTypeOfCuTe : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::PtrType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::SmemDescSM107Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::SmemDescType>($_self))">, " Generalized iterator type of CuTe ">; +def Cute_GeneralizedLayoutTypeOfCuTe : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::LayoutType>($_self))">, " Generalized layout type of CuTe ">; +def Cute_GeneralizedLayoutTypeOfCuTeOrViewTypeInterfaceInstance : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordTensorType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::LayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::MemRefType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::SmemDescViewType>($_self))">, " Generalized layout type of CuTe or ViewTypeInterface instance">; +def Cute_I32I64OrConstrainedInteger : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::IntTupleType>($_self) || $_self.isSignlessInteger(32) || $_self.isSignlessInteger(64))">, "i32, i64 or Constrained integer">; +def Cute_IntTupleAsIntegerOrTupleOfIntTupleS : Type($_self))">, "IntTuple as integer or tuple of IntTuple(s)">; +def Cute_IntTupleAsIntegerOrTupleOfIntTupleS2 : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ShapeType>($_self))">, "IntTuple as integer or tuple of IntTuple(s) or Cute shape type">; +def Cute_IntTupleTypeThatDepthIs0 : Type($_self))">, "Int tuple type that depth is 0">; +def Cute_Integer : Type($_self))">, "integer">; +def Cute_IntegerOrIntegerTypeWithConstraints : Type($_self) || ::llvm::isa<::mlir::IntegerType>($_self))">, "integer or Integer type with constraints. ">; +def Cute_IntegerTypeWithConstraints : Type($_self))">, " Integer type with constraints. ">; +def Cute_MaybeStaticTypeInterfaceInstance : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ComposedLayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordTensorType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::IntTupleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::LayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ShapeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::StrideType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::SwizzleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TileType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TiledCopyType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TiledCopyV2Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TiledMmaType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM120BlockScaledType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyG2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyS2GType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyS2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomDsmemStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomG2RType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomLdsmType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DGather4TmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DScatter4TmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomR2GType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomR2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomS2RType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100CopyS2TType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTAsyncCopyType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemLdReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemRedType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemStType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100S2TCopyV2Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomStsmType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100TmemLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM10xTmemLoadRedType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM107TmemLoadSPCompressType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100TmemStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTSyncCopyType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::UniversalFmaAtomType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMAType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMABlockScaledType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMABlockScaledSparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMASparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMAType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMABlockScaledType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMABlockScaledSparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMASparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM80Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM80SparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM89Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM90Type>($_self))">, "MaybeStaticTypeInterface instance">; +def Cute_MemrefInGenericOrRegisterMemOrSmem : Type($_self))">, "memref in generic or register mem or smem">; +def Cute_MemrefInSmemWithStaticAndNonComposed : Type($_self))">, "memref in smem with static and non-composed layout">; +def Cute_MmaAtomTypeInterfaceInstance : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM120BlockScaledType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::UniversalFmaAtomType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMAType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMABlockScaledType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMABlockScaledSparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMASparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMAType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMABlockScaledType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMABlockScaledSparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMASparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM80Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM80SparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM89Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM90Type>($_self))">, "MmaAtomTypeInterface instance">; +def Cute_OfRanks1 : Type($_self))">, " of ranks 1">; +def Cute_OfRanks1OrFloatingPointOrInteger : Type($_self) || ::llvm::isa<::mlir::FloatType>($_self) || ::llvm::isa<::mlir::VectorType>($_self))">, " of ranks 1 or floating-point or integer">; +def Cute_PointerTypeInterfaceInstance : Type($_self))">, "PointerTypeInterface instance">; +def Cute_PrefetchAtomTypeInterfaceOrTiledCopyTypeWithPrefetchAtomTypeInterfaceCopyAtom : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaLoadType>($_self))">, "PrefetchAtomTypeInterface or TiledCopyType with PrefetchAtomTypeInterface copy atom">; +def Cute_RefinableTypeInterfaceInstance : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ComposedLayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordTensorType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ConstrainedIntType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::IntTupleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::LayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::MemRefType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::PtrType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ShapeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::StrideType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::SwizzleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TileType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DGather4TmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DScatter4TmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::SmemDescSM107Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::SmemDescType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::SmemDescViewType>($_self))">, "RefinableTypeInterface instance">; +def Cute_RefinableTypeInterfaceInstanceOrInteger : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ComposedLayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordTensorType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ConstrainedIntType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::IntTupleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::LayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::MemRefType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::PtrType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ShapeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::StrideType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::SwizzleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TileType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DGather4TmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DScatter4TmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::SmemDescSM107Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::SmemDescType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::SmemDescViewType>($_self) || ::llvm::isa<::mlir::IntegerType>($_self))">, "RefinableTypeInterface instance or integer">; +def Cute_StaticLayout : Type($_self))">, "static layout">; +def Cute_StaticShape : Type($_self))">, "static shape">; +def Cute_TileLikeTypesEGLayoutTypeShapeTypeTileType : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ShapeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TileType>($_self))">, "Tile-like types, e.g. LayoutType, ShapeType, TileType">; +def Cute_TiledCopyAtomType : Type($_self))">, "Tiled copy atom type.">; +def Cute_TiledCopyAtomTypeOrTiledCopyV2 : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TiledCopyV2Type>($_self))">, "Tiled copy atom type. or Tiled copy V2 atom type.">; +def Cute_TiledCopyV2AtomType : Type($_self))">, "Tiled copy V2 atom type.">; +def Cute_TiledMMAAtomType : Type($_self))">, "Tiled MMA atom type.">; +def Cute_TypeInterfaceForCuteTypesCanBePrinted : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ComposedLayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordTensorType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::IntTupleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::LayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::MemRefType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::PtrType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ShapeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::StrideType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TileType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::SmemDescViewType>($_self) || ::llvm::isa<::mlir::IntegerType>($_self) || ::llvm::isa<::mlir::FloatType>($_self))">, "Type interface for cute types can be printed by `cute.print`">; +def Cute_ValueTypeInterfaceInstance : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::IntTupleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::LayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ShapeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::StrideType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::SwizzleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TileType>($_self))">, "ValueTypeInterface instance">; +def Cute_ValueTypeInterfaceInstanceOrIntegerTypeWithConstraints : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ConstrainedIntType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::IntTupleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::LayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ShapeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::StrideType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::SwizzleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TileType>($_self))">, "ValueTypeInterface instance or Integer type with constraints. ">; +def Cute_ValueTypeInterfaceInstanceOrIntegerTypeWithConstraintsOr : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ConstrainedIntType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::IntTupleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::LayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ShapeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::StrideType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::SwizzleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TileType>($_self) || $_self.isSignlessInteger(8) || $_self.isSignlessInteger(16) || $_self.isSignlessInteger(32) || $_self.isSignlessInteger(64) || ::llvm::isa<::mlir::IndexType>($_self))">, "ValueTypeInterface instance or Integer type with constraints. or 8/16/32/64-bit integer or index">; +def Cute_VectorOf1BitSignlessIntegerValuesOf : Type($_self))">, "vector of 1-bit signless integer values of ranks 1">; +def Cute_ViewTypeInterfaceInstance : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::MemRefType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::SmemDescViewType>($_self))">, "ViewTypeInterface instance">; +def Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordTensorType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::LayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::MemRefType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::SmemDescViewType>($_self))">, "ViewTypeInterface instance or Generalized layout type of CuTe ">; +def Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe2 : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordTensorType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::LayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::MemRefType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TileType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::SmemDescViewType>($_self))">, "ViewTypeInterface instance or Generalized layout type of CuTe or Cute tile type">; +def Cute_XTupleLikeType : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::IntTupleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ShapeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::StrideType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TileType>($_self))">, "XTuple-like type">; +def Cute_XTupleLikeTypeOrCuTeLayoutType : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::IntTupleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::LayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ShapeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::StrideType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TileType>($_self))">, "XTuple-like type or CuTe Layout Type">; + +// Operations whose result types are inferred in C++. Verification compares +// them with the inferred ones where the rule is recovered for the operands; +// elsewhere the types written stand, as the compiler accepted them. +class Cute_InferredOp traits = []> + : Cute_Op]> { + let extraClassDefinition = [{ + ::llvm::LogicalResult $cppClass::refineReturnTypes( + ::mlir::MLIRContext *context, ::std::optional<::mlir::Location> location, + ::mlir::ValueRange operands, ::mlir::DictionaryAttr attributes, + ::mlir::PropertyRef properties, ::mlir::RegionRange regions, + ::llvm::SmallVectorImpl<::mlir::Type> &returnTypes) { + ::llvm::SmallVector<::mlir::Type, 4> inferred; + if (::mlir::failed(inferReturnTypes(context, ::std::nullopt, operands, + attributes, properties, regions, + inferred))) + return ::mlir::success(); + if (!isCompatibleReturnTypes(inferred, returnTypes)) + return ::mlir::emitOptionalError( + location, "'", getOperationName(), "' op inferred type(s) ", + inferred, " are incompatible with return type(s) of operation ", + returnTypes); + return ::mlir::success(); + } + }]; +} + +def Cute_AddOffsetOp : Cute_InferredOp<"add_offset", [Pure]> { + let description = [{ + The `cute.add_offset` operation accepts an input IteratorTypeInterface and + a corresponding offset type which must be an + arithmetic tuple. It returns a new SSA value as type IteratorTypeInterface. + + When src operand is a sparse pointer, the logical offset must be a multiple + of the sparsity factor. The actual offset added to the src operand is + equal to the logical offset divided by the sparsity factor. + + `add_offset(%it, N)` returns a new iterator pointing to an object with an + offset of`N` elements. It is similar to C++'s iterator arithmetic. + + Example: + + ```mlir + // Add offset to 1-D linear pointer + %ptr = cute.add_offset(%0, %1) : (!cute.ptr>, !cute.int_tuple<"?">) -> !cute.ptr> + + // Add offset to 1-D linear pointer with static offset + %off = cute.static : !cute.int_tuple<"1012"> + %ptr = cute.add_offset(%0, %off) : (!cute.ptr>) -> !cute.ptr> + + // Add offset as arithmetic tuple to N-D nested arithmetic tuple + %arith_tuple = cute.add_offset(%0, %1) : (!cute.int_tuple<"(?,(3,?))">, !cute.int_tuple<"(3,(1,?))">) -> !cute.int_tuple<"(?,(4,?))"> + + // Add offset as arithmetic tuple to N-D nested arithmetic tuple with static offset + %arith_tuple = cute.add_offset(%0, "(3,(1,3))") : (!cute.int_tuple<"(?,(3,?))">) -> !cute.int_tuple<"(?,(4,?))"> + ``` + }]; + let arguments = (ins + Cute_GeneralizedIteratorTypeOfCuTe:$src, + Cute_IntTupleAsIntegerOrTupleOfIntTupleS:$offset + ); + let results = (outs + Cute_GeneralizedIteratorTypeOfCuTe:$dst + ); +} +def Cute_AlgorithmCopyOp : Cute_Op<"copy", [AttrSizedOperandSegments]> { + let description = [{ + CuTe's copy algorithm copies the elements of a source Tensor into the elements + of a destination Tensor. + + See the file cutlass/media/docs/cute/04_algorithms.md in the CUTLASS repository + for more information. + + A Tensor (cute.memref) encapsulates the data type, data location, and possibly + also the shape and stride of the tensor at compile time. As a result, copy can + and does dispatch, based on the types of its arguments, to use any of various + synchronous or asynchronous hardware copy instructions. + + An .copy accepts CuteCopyAtomTypeInterface as first parameter. To keep consistent + interface with CUTLASS C++, it could be either atom.tiled_copy or atom.universal_copy. + + Optional argument 'pred' is used to guard memory accesses performed in copy. + Predication works in granularity of atom_v. For src tensor whose shape is: + `((atom_v, rest_v), rest_m, rest_n, ...)`, the pred tensor shape should be: + `((rest_v), rest_m, rest_n, ...)`, we denoted this as the canonical format. + If the pred tensor's value equals 0, the corresponding copy will be skipped. + + When using atom.universal_copy instead of atom.tiled_copy, we allow the layout of + `pred` to deviate from the canonical format. Instead, the layout shape of `pred` + must match the layout shape of `src` or `dst`. Additionally, the `copy_bits` + parameter cannot be set for atom.universal_copy when predicate is present. + + The operands `src` and `dst` are variadic, each containing a variable number of tensors: + - For regular copy, `src` and `dst` each contain a single tensor. + - For copy with auxiliary operands, they contain the main tensor followed by + auxiliary tensors. For example: + - For load with row-wise reduction, `dst` = [data, stat]. + - For SPARSIFY (structured sparsity compression), `dst` = [data, metadata]. + - For TMA gather4, `src` = [coord0, coord1, coord2, coord3] (four 2D coordinate tensors). + - For TMA scatter4, `dst` = [coord0, coord1, coord2, coord3] (four 2D coordinate tensors). + + Example: + + ```mlir + cute.copy(%tiled_copy, %src, %dst) : ( + !cute.tiled_copy, + layout_copy_tv = <"((4,2),(64,128)):((0,64),(128,1))">, + tiler_mn = <"[2048:1;1:1;4:1;1:1]">>, + !cute.coord_tensor<"(0,0)", "(((64, 128), 1), 1, 1, 2):(((1@0, 1@1), 0), 0, 0, 64@0)">, + !cute.memref, "(((64,128),1),1,1,2):(((1,64),0),0,0,8192)">) + + // atom.universal_copy with canonicalize layout format + cute.copy(%copy_atom, %src, %dst, %pred) : ( + !cute_nvgpu.atom.universal_copy, + !cute.memref, "((1,1),2,4):((0,0),1,2)">, + !cute.memref, "((1,1),2,4):((0,0),1,2)">, + !cute.memref, "((1),2,4):((0),1,2)">) + + // atom.universal_copy with relaxed layout format + cute.copy(%atom, %arg0, %arg1, %arg2) : ( + !cute_nvgpu.atom.universal_copy, + !cute.memref, "(4,4):(1,4)">, + !cute.memref, + !cute.memref, "(4,4):(1,4)">) + + ``` + }]; + let arguments = (ins + Cute_CopyAtomTypeInterfaceInstanceOrCopyAtomTypeInterfaceV2Instance:$copy_atom, + Variadic:$src, + Variadic:$dst, + Optional:$pred + ); + let results = (outs + ); +} +def Cute_AlgorithmGemmOp : Cute_Op<"gemm", [AttrSizedOperandSegments]> { + let description = [{ + CuTe's gemm algorithm takes 1 mma atom and 4 inputs(cute.memref/cute_nvgpu.smem_desc_view) + as input. + It performs the general matrix multiplication: + `D += A * B + C`. + Layout carried by operads determines the exact form of the gemm computation. + Generally speaking, there are 5 valid forms: + 1. `(V) x (V) => (V)`. + The element-wise product of vectors. Dispatches to FMA or MMA. + In this case, `rank(A) = rank(B) = rank(C) = rank(D) = 1`. + 2. `(M) x (N) => (M,N)`. + The outer product of vectors. Dispatches to [3] with new mode K=(1). + In this case, `rank(A) = rank(B) = 1` and `rank(C) = rank(D) = 2`. + 3. `(M,K) x (N,K) => (M,N)`. + The product of matrices. Dispatches to [5] with MMA vector-mode V. + In this case, `rank(A) = rank(B) = rank(C) = rank(D) = 2`. + 4. `(V,M) x (V,N) => (V,M,N)`. + The batched outer product of vectors. Accounts for register reuse and dispatches to [1] for each (m,n). + In this case, `rank(A) = rank(B) = 2` and `rank(C) = rank(D) = 3`. + 5. `(V,M,K) x (V,N,K) => (V,M,N)`. + The batched product of matrices. Dispatches to [4] for each (k). + In this case, `rank(A) = rank(B) = rank(C) = rank(D) = 3`. + + Currently, only the form in item 5 is supported. + + The operands `a` and `b` are variadic, each containing a variable number of tensors: + - For regular GEMM, `a` and `b` contain the GEMM A and B tensors respectively. + - For GEMM with auxiliary operands, `a` and `b` contain the GEMM A and B tensors followed by their respective auxiliary tensors. For example: + - For BlockScaledGemm, `a` = [A, SFA] and `b` = [B, SFB]. + - For SparseGemm, `a` = [A, E] and `b` = [B]. + - For BlockScaledSparseGemm, `a` = [A, SFA, E] and `b` = [B, SFB]. + See the file cutlass/media/docs/cute/04_algorithms.md and include/cute/algorithm/gemm.hpp in the CUTLASS repository for more information. + + Example (regular GEMM): + ```mlir + cute.gemm(%mma, %d, %a, %b, %c) { + } : (!cute.tiled_mma, + atom_layout_MNK = <"(1,1,1)">>, + !cute.memref, "((128,128),1,1):((131072,1),0,0)">, + !cute_nvgpu.smem_desc_view o 0 o ((128,16),1,4):((64,1),0,16)">, + !cute_nvgpu.smem_desc_view o 0 o ((128,16),1,4):((64,1),0,16)">, + !cute.memref, "((128,128),1,1):((131072,1),0,0)">) + ``` + + Example (block-scaled GEMM): + ```mlir + cute.gemm(%mma, %d, [%a, %sfa], [%b, %sfb], %c) : ( + !cute.tiled_mma, + atom_layout_MNK = <"(1,2,1):(0,1,0)">>, + !cute.memref, "((128,128),1,1):((65536,1),0,0)">, + [!cute_nvgpu.smem_desc_view, + !cute.memref, "((((32,4),4),(16,4)),1):((((262144,4),8388608),(0,1)),0)">], + [!cute_nvgpu.smem_desc_view, + !cute.memref, "((((32,4),4),(16,4)),1):((((262144,4),8388608),(0,1)),0)">], + !cute.memref, "((128,128),1,1):((65536,1),0,0)">) + ``` + }]; + let arguments = (ins + Cute_MmaAtomTypeInterfaceInstance:$mma_atom, + Cute_ViewTypeInterfaceInstance:$d, + Variadic:$a, + Variadic:$b, + Cute_ViewTypeInterfaceInstance:$c + ); + let results = (outs + ); +} +def Cute_AlgorithmPrefetchOp : Cute_Op<"prefetch"> { + let summary = "CuTe's prefetch algorithm prefetches the elements of source Tensor(s) into cache."; + let description = [{ + A Tensor encapsulates the data type, data location, and possibly + also the shape and stride of the tensor at compile time. As a result, prefetch can + and does dispatch, based on the types of its arguments, to use any of various + synchronous or asynchronous hardware prefetch instructions. + + An .prefetch accepts CutePrefetchAtomTypeinterface as first parameter. + + The `src` operand is variadic: + - For regular TMA prefetch, `src` is a single tensor. + - For 2D `tile::gather4` mode, `src` is `[data_coord_tensor, gmem_index_tensor]`, + mirroring `cute.copy` for the same mode. + + Example: + + ```mlir + cute.prefetch(%tiled_copy, %src) : ( + !cute.tiled_copy, + layout_copy_tv = <"((4,2),(64,128)):((0,64),(128,1))">, + tiler_mn = <"[2048:1;1:1;4:1;1:1]">>, + !cute.coord_tensor<"(0,0)", "(((64, 128), 1), 1, 1, 2):(((1@0, 1@1), 0), 0, 0, 64@0)">) + + // gather4 mode: data tensor + gmem index tensor + cute.prefetch(%tiled_copy, %src_data, %src_idx) : ( + !cute.tiled_copy, ...>, + !cute.coord_tensor<...>, + !cute.coord_tensor<...>) + ``` + }]; + let arguments = (ins + Cute_PrefetchAtomTypeInterfaceOrTiledCopyTypeWithPrefetchAtomTypeInterfaceCopyAtom:$prefetch_atom, + Variadic:$src + ); + let results = (outs + ); +} +def Cute_AppendToRankOp : Cute_InferredOp<"append_to_rank", [Pure]> { + let summary = "Append element to input to rank N."; + let description = [{ + Example: + + ```mlir + %shp = cute.make_shape() : !cute.shape<"(5,4,3)"> + %elem = cute.make_shape() : !cute.shape<"(2,1)"> + %new_layout = cute.append_to_rank<5> (%shp, %elem) + : (..., ...) -> !cute.shape<"(5,4,3,(2,1),(2,1))"> + ``` + }]; + let arguments = (ins + Cute_ValueTypeInterfaceInstance:$input, + Cute_ValueTypeInterfaceInstance:$element, + SI32Attr:$rank + ); + let results = (outs + Cute_ValueTypeInterfaceInstance:$result + ); +} +def Cute_ApplySwizzleOp : Cute_InferredOp<"apply_swizzle", [Pure]> { + let description = [{ + The `apply_swizzle` operation applies a swizzle transformation to a pointer's + address, producing a new pointer with the swizzled address. This is necessary + for dereferencing pointers that use swizzled memory layouts, such as those used + for improved memory access patterns on GPUs. + + Two forms are supported: + + 1. **CutePtrType form**: The swizzle is encoded in the pointer type itself. + No explicit swizzle operand is required or allowed. + + ```mlir + %swizzled = cute.apply_swizzle(%ptr) : !cute.ptr, S<3,4,3>> + ``` + + 2. **llvm.ptr form**: The swizzle is passed as an optional attribute in the + attribute dictionary. If the swizzle attribute is absent, the operation + is a no-op (identity swizzle). + + ```mlir + // With explicit swizzle: + %swizzled = cute.apply_swizzle(%ptr) { S<3,4,3> } : !llvm.ptr<3> + // Identity (no-op): + %swizzled = cute.apply_swizzle(%ptr) : !llvm.ptr<3> + ``` + + If the input pointer type has a non-identity swizzle attribute, this operation + computes the swizzled address by applying the swizzle transformation to the + pointer's address bits. The transformation is typically an XOR-based permutation + of certain address bits, as defined by the swizzle parameters. + + Swizzle is parameterized by three values: + - MBase: The number of least-significant bits to keep constant (not affected by the swizzle) + - BBits: The number of bits in the mask (the width of the swizzle region) + - SShift: The distance to shift the mask (positive values shift right, negative values shift left) + + The swizzle transformation operates on the address as follows: + + 0bxxxxxxxxxxxxxxxYYYxxxxxxxZZZxxxx + ^--^ MBase: least-significant bits kept constant + ^-^ ^-^ BBits: width of the mask (YYY and ZZZ) + ^---------^ SShift: shift distance for the YYY mask + + The swizzled address is computed by XOR'ing the ZZZ and YYY bitfields: + + Example: + + ``` + 0bxxxxxxxxxxxxxxxxYYxxxxxxxxxZZxxx + + the result is + 0bxxxxxxxxxxxxxxxxYYxxxxxxxxxAAxxx where AA = ZZ `xor` YY + ``` + + This operation is typically used just before dereferencing a pointer, + ensuring that the correct physical memory location is accessed according + to the swizzle pattern. If the pointer type has an identity swizzle, + this operation is a no-op and returns the original pointer. + }]; + let arguments = (ins + Cute_CuTePointerTypeOrLLVMPointerType:$ptr, + OptionalAttr:$swizzle + ); + let results = (outs + Cute_CuTePointerTypeOrLLVMPointerType:$swizzled_ptr + ); +} +def Cute_AssumeOp : Cute_Op<"assume", [Pure]> { + let description = [{ + `cute.assume` represents assumptions such as divisibility for an SSA value. + unlike `llvm.assume`, `cute.assume` produces a strong typed value as output. + It is limited to support all constrains defined by `ConstrainedIntType`. + + Please see [constrained integer type](### ConstrainedIntType) for details. + + Example: + ```mlir + // Claim %1 is divisible by 8 + %1 = cute.assume(%0 : i32) : !cute.i32 + + ``` + + Note: + + // xxx: need to revisit + + ```mlir + %0 = ... -> !cute.i32 + if (%cond) { + %1 = cute.assume(%0 : !cute.i32) : !cute.i32 + ... // %1 has the assumption. + + // %0 is not assumed here + %data = load %ptr[%0] : ? + } + + // %0 here does not have any assumption. + ``` + }]; + let arguments = (ins + Cute_Integer:$src + ); + let results = (outs + Cute_IntegerTypeWithConstraints:$dst + ); +} +def Cute_BlockedProductOp : Cute_InferredOp<"blocked_product", [Pure]> { + let description = [{ + This operation performs "blocked_product" on the input and produce a blocked_product result, + and it automatically infer the "blocked_product" layout for the result type. + + Example: + + ```mlir + %input = cute.make_layout() : !cute.layout<"(3,4):(4,1)"> + %tiler = cute.make_layout() : !cute.layout<"(2,5):(1,2)"> + %bprod = cute.blocked_product(%input, %tiler) + : (!cute.layout<"(3,4):(4,1)">, !cute.layout<"(2,5):(1,2)">) + -> !cute.layout<"((3,2),(4,5)):((4,12),(1,24))"> + ``` + + Example with dynamic composed_layout: + + ``` + %bprod = cute.blocked_product(%input, %tiler) + : (!cute.composed_layout<"(?):(?) o ? o (5,4):(4,1)">, !cute.layout<"(?,?):(?,?)">) + -> !cute.composed_layout<"(?):(?) o ? o ((5,?),(4,?)):((4,?{div=20}),(1,?{div=20}))"> + ``` + }]; + let arguments = (ins + Cute_GeneralizedLayoutTypeOfCuTe:$input, + Cute_CuTeLayoutType:$tiler + ); + let results = (outs + Cute_GeneralizedLayoutTypeOfCuTe:$result + ); +} +def Cute_CeilDivOp : Cute_InferredOp<"ceil_div", [Pure]> { + let arguments = (ins + Cute_IntTupleAsIntegerOrTupleOfIntTupleS2:$input, + Cute_TileLikeTypesEGLayoutTypeShapeTypeTileType:$tiler + ); + let results = (outs + Cute_IntTupleAsIntegerOrTupleOfIntTupleS2:$result + ); +} +def Cute_CoalesceOp : Cute_InferredOp<"coalesce", [Pure]> { + let summary = "`coalesce` op simplifies layout by combining modes that are possible to combine."; + let description = [{ + It does not respect the shape of the layout but preserve total size. + + ```mlir + // coalesce modes with contiguous/packed strides from left to right + %0 = cute.coalesce(%arg0) + : (!cute.layout<"((4,5),3):((1,4),20)">) -> !cute.layout<"60:1"> + + // coalesce from left to right but not right to left + %0 = cute.coalesce(%arg0) + : (!cute.layout<"(3,(4,5)):(20,(1,4))">) -> !cute.layout<"(3,20):(20,1)"> + %0 = cute.coalesce(%arg0, %arg1) + : (!cute.layout<"(3,(4,5)):(20,(1,4))">, !cute.coord<"(1,1)">) -> !cute.layout<"(3,20):(20,1)"> + ``` + }]; + let arguments = (ins + Cute_GeneralizedLayoutTypeOfCuTeOrViewTypeInterfaceInstance:$input, + Optional:$target_profile + ); + let results = (outs + Cute_GeneralizedLayoutTypeOfCuTeOrViewTypeInterfaceInstance:$result + ); +} +def Cute_ComplementOp : Cute_InferredOp<"complement", [Pure]> { + let summary = "Produces a disjoint layout."; + let description = [{ + The complement of a layout L is a layout L* such that (L,L*) achieves greater surjectivity. + That is, L* maps to elements in the codomain of L that are not in the image of L. + + It's worth to note that (L,L*) isn't alway surjective. + + Examples: + ``` + // trivial complement of L under it's domain is 1:0 + %res = cute.complement(%arg0, %arg1) + : (!cute.layout<"3:1">, !cute.shape<"3">) -> !cute.layout<"1:0"> + + // 3:2 is strided layout. complement of 3:2 under 6 is 2:1 filling stride + // (L,L*) => (3,2):(2,1) is a surjective layout + %res = cute.complement(%arg0, %arg1) + : (!cute.layout<"3:2">, !cute.shape<"6">) -> !cute.layout<"2:1"> + + %res = cute.complement(%arg0, %arg1) + : (!cute.layout<"(3,(5,2)):(1,(3,15))">, !cute.shape<"(6,20)">) -> !cute.layout<"4:30"> + ``` + }]; + let arguments = (ins + Cute_CuTeLayoutType:$input, + Cute_TileLikeTypesEGLayoutTypeShapeTypeTileType:$cotarget + ); + let results = (outs + Cute_CuTeLayoutType:$res + ); +} +def Cute_ComposedGetInnerOp : Cute_InferredOp<"composed_get_inner", [Pure]> { + let summary = "Gets inner layout from a layout."; + let description = [{ + Example: + + ```mlir + %layout = cute.make_layout(.., ..): !cute.layout<"(32,64):(1,32)>"> + %layout_a = cute.composed_get_layout_a(%layout) : (!cute.composed_layout<"(64,32):(1,64) o 0 o (32,64):(1,32)>">) -> !cute.layout<"(64,32):(1,64)"> + %offset = cute.composed_get_offset(%layout) : (!cute.composed_layout<"S<3,4,3> o 7 o (32,64):(1,32)>">) -> !cute.shape<"7"> + %layout_b = cute.composed_get_layout_b(%layout) : (!cute.composed_layout<"(64,32):(1,64) o 0 o (32,64):(1,32)>">) -> !cute.layout<"(32,64):(1,32)"> + ``` + }]; + let arguments = (ins + Cute_CuTeComposedLayoutType:$input + ); + let results = (outs + Cute_CuTeLayoutTypeOrCuteSwizzleType:$result + ); +} +def Cute_ComposedGetOffsetOp : Cute_InferredOp<"composed_get_offset", [Pure]> { + let summary = "Gets offset from a layout."; + let description = [{ + Example: + + ```mlir + %layout = cute.make_layout(.., ..): !cute.layout<"(32,64):(1,32)>"> + %layout_a = cute.composed_get_layout_a(%layout) : (!cute.composed_layout<"(64,32):(1,64) o 0 o (32,64):(1,32)>">) -> !cute.layout<"(64,32):(1,64)"> + %offset = cute.composed_get_offset(%layout) : (!cute.composed_layout<"S<3,4,3> o 7 o (32,64):(1,32)>">) -> !cute.shape<"7"> + %layout_b = cute.composed_get_layout_b(%layout) : (!cute.composed_layout<"(64,32):(1,64) o 0 o (32,64):(1,32)>">) -> !cute.layout<"(32,64):(1,32)"> + ``` + }]; + let arguments = (ins + Cute_CuTeComposedLayoutType:$input + ); + let results = (outs + Cute_IntTupleAsIntegerOrTupleOfIntTupleS:$result + ); +} +def Cute_ComposedGetOuterOp : Cute_InferredOp<"composed_get_outer", [Pure]> { + let summary = "Gets layout outer from composed layout."; + let description = [{ + Example: + + ```mlir + %layout_outer = cute.composed_get_outer(%layout) + : (!cute.composed_layout<"(64,32):(1,64) o 0 o (32,64):(1,32)>">) -> !cute.layout<"(32,64):(1,32)"> + ``` + }]; + let arguments = (ins + Cute_CuTeComposedLayoutType:$input + ); + let results = (outs + Cute_CuTeLayoutType:$result + ); +} +def Cute_CompositionOp : Cute_InferredOp<"composition", [Pure]> { + let description = [{ + The `composition` operation computes the composition of two layouts + (`!cute.layout`), in the following called `lhs` and `rhs`. + + Examples: + ``` + // Composition of two static layouts. + %0 = cute.composition(%arg0, %arg1) + : (!cute.layout<"(20):(2)">, !cute.layout<"(5,4):(4,1)">) -> !cute.layout<"(5,4):(8,2)"> + + // Composition of a static and a dynamic layout. + %1 = cute.composition(%arg0, %arg1) + : (!cute.layout<"(20):(2)">, !cute.layout<"(?,?):(?,?)">) -> !cute.layout<"(?,?):(?{div=2},?{div=2})"> + ``` + }]; + let arguments = (ins + Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$lhs, + Cute_TileLikeTypesEGLayoutTypeShapeTypeTileType:$rhs + ); + let results = (outs + Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$res + ); +} +def Cute_CopyAtomCallOp : Cute_Op<"copy_atom_call", [AttrSizedOperandSegments]> { + let summary = "`cute.copy_atom_call` op executes a copy atom operation with the given operands."; + let description = [{ + Following src/dst layout of atom are valid: + * ((atom_v, rest_v)) + * ((atom_v)) + * (atom_v) + + Both `src` and `dst` operands are variadic, containing a variable number of tensors: + + - For regular copy, `src` and `dst` each contain a single tensor. + - For copy with auxiliary operands, they contain the main tensor followed by + auxiliary tensors. For example: + + - For load with row-wise reduction, `dst` = [data, stat]. + - For SPARSIFY (structured sparsity compression), `dst` = [data, metadata]. + - For TMA gather4, `src` = [coord0, coord1, coord2, coord3] (four 2D coordinate tensors). + - For TMA scatter4, `dst` = [coord0, coord1, coord2, coord3] (four 2D coordinate tensors). + + Examples: + + ```mlir + // Regular copy atom operation + cute.copy_atom_call(%copy_load, %src_v, %dst_v) + : (!cute_nvgpu.atom.tmem_load, + !cute.memref, "(((2,32),1)):(((1,131072),0))">, + !cute.memref, "((2,1)):((1,0))">) + -> () + ``` + + ```mlir + // TMA gather4: combine four 2D coordinate tensors into single destination + cute.copy_atom_call(%tma_gather4_atom, [%coord0, %coord1, %coord2, %coord3], %dst_v) + : (!cute_nvgpu.atom.tma_gather4<...>, + [!cute.coord_tensor<...>, !cute.coord_tensor<...>, + !cute.coord_tensor<...>, !cute.coord_tensor<...>], + !cute.memref) + -> () + ``` + }]; + let arguments = (ins + Cute_CopyAtomTypeInterfaceInstanceOrCopyAtomTypeInterfaceV2Instance:$atom, + Variadic:$src, + Variadic:$dst, + Optional:$pred + ); + let results = (outs + ); +} +def Cute_CopyMakeFragmentOp : Cute_InferredOp<"copy.make_fragment", [Pure]> { + let description = [{ + Construct a copy fragment for the S or D side from either a partitioned view or an abstract TV + description. Intended for use between `cute.tiled.copy.partition_{S,D}` and `cute.copy`, but + also supports layout/shape inputs for RMEM/TMEM fragment construction similar to MMA. + + The op accepts: + 1. an operand id `S` or `D` (printed as a keyword); + 2. a copy atom value; + 3. either a partitioned view or a layout/shape to build the fragment. + + Returns a new ViewTypeInterface derived value to be consumed by `cute.copy`. + + Examples: + ```mlir + // Load from global memory (gmem -> rmem): build S and D fragments from S-partition + %atom_load = cute.make_atom() : () -> !cute_nvgpu.atom.universal_copy + %tiled_copy_load = cute.make_tiled_copy(%atom_load) + : !cute.tiled_copy, + layout_copy_tv = <"(128,8):(1,128)">, tiler_mn = <"[128;8]">> + %tGSrc_load = cute.tiled.copy.partition_S(%tiled_copy_load, %gSrc, %tid_coord) + : (!cute.tiled_copy, + layout_copy_tv = <"(128,8):(1,128)">, tiler_mn = <"[128;8]">>, + !cute.memref, "(128,128):(?{div=8},1)">, !cute.coord<"?">) + -> !cute.memref, "((8,1),1,16):((1,0),0,8)"> + + %frgS_load = cute.copy.make_fragment S (%atom_load, %tGSrc_load) : ( + !cute_nvgpu.atom.universal_copy, + !cute.memref, "((8,1),1,16):((1,0),0,8)"> + ) -> !cute.memref, "((8,1),1,16):((1,0),0,8)"> + + %rLoad = cute.copy.make_fragment D (%atom_load, %tGSrc_load) : ( + !cute_nvgpu.atom.universal_copy, + !cute.memref, "((8,1),1,16):((1,0),0,8)" + ) -> !cute.memref, "((8,1),1,16):((1,0),0,8)" + ``` + }]; + let arguments = (ins + Cute_CopyAtomTypeInterfaceInstanceOrCopyAtomTypeInterfaceV2Instance:$atom, + Cute_AMemoryReferenceTypeThatRepresentsATensor:$input, + Cute_CopyOperandEnum:$operand_id + ); + let results = (outs + Cute_ViewTypeInterfaceInstance:$result + ); +} +def Cute_CosizeOp : Cute_InferredOp<"cosize", [Pure]> { + let description = [{ + This operation calculates the cosize of a layout. + * For a layout `l`, cosize(l) = l(size(l) - 1) + 1. + * For a layout `l` with mode `mode`, cosize(l) = cosize(get(l)). + * For a composed layout 'l = a o b', cosize(l) = cosize(b). + + The `mode` attribute specifies the nested mode in the type for cosize calculation. + It indexes a sub-scope within the input type and returns the cosize of that sub-scope. + The rank of `mode` must be less than or equal to the depth of the type. + If `mode` is empty, the operation calculates the cosize of the entire type. + + ### Examples + + ```mlir + %input: !cute.composed_layout<"(100):(1) o 0 o (3,4):(4,1)"> + %0 = cute.cosize(%input) + : !cute.composed_layout<"(100):(1) o 0 o (3,4):(4,1)"> + // cosize(%input) = cosize((3,4):(4,1)), ret is 12 + ``` + + Example with mode: + ```mlir + %input: !cute.layout<"((4,3),((3,(2,2)),2)):((1,4),((12,(36,72)),144))"> + %0 = cute.cosize(%input) <{mode=[1,0,1]}> + : (!cute.layout<"((4,3),((3,(2,2)),2)):((1,4),((12,(36,72)),144))">) -> !cute.int_tuple<"109"> + // cosize(%input) = cosize((2,2):(36,72)), ret is 109 + ``` + }]; + let arguments = (ins + Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$input, + DefaultValuedAttr:$mode + ); + let results = (outs + Cute_IntTupleAsIntegerOrTupleOfIntTupleS:$result + ); +} +def Cute_Crd2IdxOp : Cute_InferredOp<"crd2idx", [Pure]> { + let description = [{ + Compute the index of a coordinate with respect to the given layout. I.e., + compute the natural coordinate of `coord` wrt. to the shape + and then compute the dot product with the strides. + + Examples: + ```mlir + // %crd0 is already a natural coordinate for %shape. + %0 = cute.crd2idx(%crd, %layout): (!cute.coord<"(?,(?,?))">, !cute.layout<"(?,(?,?)):(?,(?,?))">) -> (i32) + // Compute the natural coordinate, then dot product. + %1 = cute.crd2idx(%crd, %layout): (!cute.coord<"(?,?)">, !cute.layout<"(?,(?,?)):(?,(?,?))">) -> (i32) + // Static and dynamic shape/strides are supported. + %3 = cute.crd2idx(%crd, %layout): (!cute.coord<"(?,?)">, !cute.layout<"(4,2):(1,2)">) -> (i32) + // Strides with scaled basis produces a tuple. + %5 = cute.crd2idx(%crd, %layout) : (!cute.coord<"(?,?,?)">, !cute.layout<"(10,20,30):(1@0,1@1,20@1)">) -> (!cute.int_tuple<"(?,?)">) + ``` + }]; + let arguments = (ins + Cute_CuteCoordType:$coord, + Cute_GeneralizedLayoutTypeOfCuTe:$layout + ); + let results = (outs + Cute_IntTupleAsIntegerOrTupleOfIntTupleS:$index + ); +} +def Cute_DereferenceArithTupleIteratorOp : Cute_InferredOp<"deref_arith_tuple_iter", [Pure]> { + let summary = "Dereferences an arithmetic tuple iterator to return the int_tuple value it holds."; + let description = [{ + Example: + + ```mlir + %0 = cute.deref_arith_tuple_iter(%arg0) : !cute.arith_tuple_iter<"(?,4)"> -> !cute.int_tuple<"(?,4)"> + ``` + }]; + let arguments = (ins + Cute_CuTeIteratorTypeWithIntTupleAsIts:$iter + ); + let results = (outs + Cute_IntTupleAsIntegerOrTupleOfIntTupleS:$arith_tuple + ); +} +def Cute_DereferenceDescriptorIteratorOp : Cute_InferredOp<"deref_desc_iter", [Pure]> { + let description = [{ + This Operation dereferences a descriptor iterator to return the descriptor + value it holds. + }]; + let arguments = (ins + Cute_ATypeInterfaceForDescriptorIteratorTypes:$iter + ); + let results = (outs + AnyType:$value + ); +} +def Cute_DerefineOp : Cute_Op<"derefine", [Pure]> { + let description = [{ + The `cute.derefine` operation discard static information from modes + of a given cute algebra type like shape, stride, layout, coord or tile into + modes with less static information. + + For each mode, it either cast it into dynamic mode or keep it unchanged. In future, + it can be extended to support removal of `divby N` from dynamic mode. + + Casting dynamic mode into static mode is illegal. + + E.g. + + ```mlir + %dyn_0 = cute.derefine(%0) : !cute.shape<"(3,5)"> to !cute.shape<"(?,5)"> + %dyn_1 = cute.derefine(%1) : !cute.stride<"(?,5)"> to !cute.stride<"(?,?)"> + %dyn_2 = cute.derefine(%2) : !cute.coord<"(8,_)"> to !cute.coord<"(?,_)"> + ``` + }]; + let arguments = (ins + Cute_RefinableTypeInterfaceInstance:$src + ); + let results = (outs + Cute_RefinableTypeInterfaceInstanceOrInteger:$dst + ); +} +def Cute_DiceOp : Cute_InferredOp<"dice", [Pure]> { + let description = [{ + This operation dices a type implementing the `CuteValueTypeInterface` (ShapeType, IntTupleType, + LayoutType,...) using a static coordinate attribute. The profile of said coordinate must be + weakly congruent with that of `input`. Modes associated with an underscore are discarded in the + result. + }]; + let arguments = (ins + Cute_ValueTypeInterfaceInstance:$input, + Cute_CoordAttr:$coord + ); + let results = (outs + Cute_ValueTypeInterfaceInstance:$result + ); +} +def Cute_ElemLessOp : Cute_Op<"elem_less", [Pure]> { + let description = [{ + The cute.elem_less operation compares every elements in Lhs to the + corresponding elements in Rhs. The result is 1 if every Lhs elements + are less than corresponding Rhs elements, otherwise result is 0. + + `ElemLessOp` is useful when we want to know if a coordinate is out of bound: + + ```mlir + %is_in_bound = cute.elem_less(%coord, %shape) : !cute.coord<"(?,?)">, !cute.shape<"(?,?)"> + ``` + }]; + let arguments = (ins + Cute_CuteShapeTypeOrIntTupleAsIntegerOr:$lhs, + Cute_CuteShapeTypeOrIntTupleAsIntegerOr:$rhs + ); + let results = (outs + I1:$result + ); +} +def Cute_EqualOp : Cute_Op<"equal", [Pure]> { + let description = [{ + Checks two dynamic cute values for equality. Full static values are considered + equal only when their types are the same. + + Example: + + ```mlir + %1 = cute.make_shape : !cute.shape<"(32, 64)"> + %2 = cute.make_shape : !cute.shape<"(32, 64)"> + %result = cute.equal %1, %2 : !cute.shape<"(32,64)">, !cute.shape<"(32,64)"> + ``` + + The generated code only checks profile positions in which at least one side has a dynamic value. + Static comparisons are performed at compile time by the operation folder. + + ```mlir + %1 = cute.make_shape : !cute.shape<"(?, 64)"> + %2 = cute.make_shape : !cute.shape<"(?, 64)"> + %result = cute.equal %1, %2 : !cute.shape<"(?,64)">, !cute.shape<"(?,64)"> + ``` + + This would result in the first element being checked for equality in the runtime representation of the shapes. + + Note that this is dynamic if either of the operands will force that operand to be checked. So, + + ```mlir + %1 = cute.make_shape : !cute.shape<"(?, 64)"> + %2 = cute.make_shape : !cute.shape<"(64, ?)"> + %result = cute.equal %1, %2 : !cute.shape<"(?,64)">, !cute.shape<"(64,?)"> + ``` + + will result in both dimensions being checked. + }]; + let arguments = (ins + Cute_ValueTypeInterfaceInstance:$lhs, + Cute_ValueTypeInterfaceInstance:$rhs + ); + let results = (outs + I1:$result + ); +} +def Cute_FastDivmodComputeOp : Cute_Op<"fast_divmod.compute", [Pure]> { + let description = [{ + This operation performs a fast divmod using a fast divmod divisor. It computes the quotient and + the remainder without emitting division instructions. The quotient and the remainder are of the + same width as the divisor. + }]; + let arguments = (ins + Cute_32BitSignlessIntegerOr64BitSignless:$dividend, + Cute_DivisorForFastUnsignedIntegerDivmodsByInvariant:$divisor + ); + let results = (outs + Cute_32BitSignlessIntegerOr64BitSignless:$quotient, + Cute_32BitSignlessIntegerOr64BitSignless:$remainder + ); +} +def Cute_FastDivmodCreateDivisorOp : Cute_Op<"fast_divmod.create_divisor", [Pure]> { + let description = [{ + This operation creates a fast divmod divisor from an integer. It is meant to be pre-computed, + ideally in host code, and then used throughout the lifetime of a device kernel for performing + divmods without emitting division instructions. + }]; + let arguments = (ins + Cute_32BitSignlessIntegerOr64BitSignless:$divisor + ); + let results = (outs + Cute_DivisorForFastUnsignedIntegerDivmodsByInvariant:$fast_divmod_divisor + ); +} +def Cute_FastDivmodDivideOp : Cute_Op<"fast_divmod.divide", [Pure]> { + let description = [{ + This operation performs a fast divide using a fast divmod divisor. It computes the quotient + without emitting division instructions. The quotient is of the same width as the divisor. + }]; + let arguments = (ins + Cute_32BitSignlessIntegerOr64BitSignless:$dividend, + Cute_DivisorForFastUnsignedIntegerDivmodsByInvariant:$divisor + ); + let results = (outs + Cute_32BitSignlessIntegerOr64BitSignless:$quotient + ); +} +def Cute_FastDivmodGetAuxOp : Cute_InferredOp<"fast_divmod.get_aux", [Pure]> { + let arguments = (ins + Cute_DivisorForFastUnsignedIntegerDivmodsByInvariant:$divisor + ); + let results = (outs + Cute_32BitSignlessIntegerOr64BitSignless:$multiplier, + I8:$shift1, + I8:$shift2 + ); +} +def Cute_FastDivmodGetDivisorOp : Cute_InferredOp<"fast_divmod.get_divisor", [Pure]> { + let arguments = (ins + Cute_DivisorForFastUnsignedIntegerDivmodsByInvariant:$fast_divmod_divisor + ); + let results = (outs + Cute_32BitSignlessIntegerOr64BitSignless:$divisor + ); +} +def Cute_FastDivmodMakeDivisorOp : Cute_Op<"fast_divmod.make_divisor", [Pure]> { + let arguments = (ins + Cute_32BitSignlessIntegerOr64BitSignless:$divisor, + Cute_32BitSignlessIntegerOr64BitSignless:$multiplier, + I8:$sh1, + I8:$sh2 + ); + let results = (outs + Cute_DivisorForFastUnsignedIntegerDivmodsByInvariant:$fast_divmod_divisor + ); +} +def Cute_FilterOp : Cute_InferredOp<"filter", [Pure]> { + let summary = "This operation filters out dimensions with 0-strides and 1-sizes from a layout or view."; + let description = [{ + When working with tensor layouts, some dimensions may have stride 0 (indicating + broadcasting) or size 1 (singleton dimensions). This operation creates a new layout + or view with these dimensions removed, which can simplify subsequent operations. + + OptionalOperand `target_profile` is the user indicated filter control, with this operand, + the original layout's stride is ignored in the zero-stride selection. + But the `target_profile` will not take effect currently. This is a placeholder for future use. + + This operation accepts both layout and view types as valid input. + }]; + let arguments = (ins + Cute_GeneralizedLayoutTypeOfCuTeOrViewTypeInterfaceInstance:$input + ); + let results = (outs + Cute_GeneralizedLayoutTypeOfCuTeOrViewTypeInterfaceInstance:$result + ); +} +def Cute_FilterZerosOp : Cute_InferredOp<"filter_zeros", [Pure]> { + let description = [{ + This operation produces a transformed layout or view by replacing the shape + of any mode with stride zero and non-unit extent with 1. This is commonly + applied prior to copy operations to prevent redundant or duplicated + data movement arising from broadcasted dimensions. + + The optional operand `target_profile` allows the user to specify a stride + filter profile; when provided, the selection of zero-stride dimensions is + determined by this stride profile rather than the original layout's stride + information. + + This operation accepts both layout and view types as input. + }]; + let arguments = (ins + Cute_GeneralizedLayoutTypeOfCuTeOrViewTypeInterfaceInstance:$input, + Optional:$target_profile + ); + let results = (outs + Cute_GeneralizedLayoutTypeOfCuTeOrViewTypeInterfaceInstance:$res + ); +} +def Cute_FlatDivideOp : Cute_InferredOp<"flat_divide", [Pure]> { + let summary = "This operation performs a flat_divide on a view or a layout by a tiler."; + let arguments = (ins + Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$input, + Cute_TileLikeTypesEGLayoutTypeShapeTypeTileType:$tiler + ); + let results = (outs + Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$result + ); +} +def Cute_FlatProductOp : Cute_InferredOp<"flat_product", [Pure]> { + let arguments = (ins + Cute_GeneralizedLayoutTypeOfCuTe:$input, + Cute_CuTeLayoutType:$tiler + ); + let results = (outs + Cute_GeneralizedLayoutTypeOfCuTe:$result + ); +} +def Cute_GetFlatCoordOp : Cute_InferredOp<"get_flat_coord", [Pure]> { + let description = [{ + This OP accept a scalar index and a normal layout, + returns a flat coord. + + + The "flat" here means the reversed coordinate will be depth=1 coordinate. + This is useful when we just care about the rank semantic of the layout and don't care about the detail profile. + + + Example: + + ```mlir + %layout = cute.make_layout() : !cute.layout<"(5,(4,3),2):(1,(5,20),60)"> + %index = arith.constant 117 : i32 + %flat_coord = cute.get_flat_coord(%index, %layout) : (..., ...) -> !cute.coord<"(2,11,1)"> + %hier_coord = cute.get_hire_coord(%index, %layout) : (..., ...) -> !cute.coord<"(2,(3,2),1)"> + %integral_coord = cute.get_integral_coord(%index, %layout) : (..., ...) -> !cute.coord<"117"> + ``` + }]; + let arguments = (ins + Cute_32BitSignlessIntegerOr64BitSignless:$index, + Cute_CuTeLayoutType:$input + ); + let results = (outs + Cute_CuteCoordType:$result + ); +} +def Cute_GetHierCoordOp : Cute_InferredOp<"get_hier_coord", [Pure]> { + let description = [{ + This OP accept a scalar index and a normal layout, + returns a hier coord. + + + The "hier" here means the reversed coordinate will have the same profile with the given layout. + This is useful when we care about both the rank semantic and the detail layout profile. + + + Example: + + ```mlir + %layout = cute.make_layout() : !cute.layout<"(5,(4,3),2):(1,(5,20),60)"> + %index = arith.constant 117 : i32 + %flat_coord = cute.get_flat_coord(%index, %layout) : (..., ...) -> !cute.coord<"(2,11,1)"> + %hier_coord = cute.get_hire_coord(%index, %layout) : (..., ...) -> !cute.coord<"(2,(3,2),1)"> + %integral_coord = cute.get_integral_coord(%index, %layout) : (..., ...) -> !cute.coord<"117"> + ``` + }]; + let arguments = (ins + Cute_32BitSignlessIntegerOr64BitSignless:$index, + Cute_CuTeLayoutType:$input + ); + let results = (outs + Cute_CuteCoordType:$result + ); +} +def Cute_GetIntegralCoordOp : Cute_InferredOp<"get_integral_coord", [Pure]> { + let description = [{ + This OP accept a scalar index and a normal layout, + returns a integral coord. + + + The "integral" here means the reversed coordinate will be a "scalar". + This is useful when we don't care about the layout's semantic and profile and we just want to know the offset in "domain". + + + Example: + + ```mlir + %layout = cute.make_layout() : !cute.layout<"(5,(4,3),2):(1,(5,20),60)"> + %index = arith.constant 117 : i32 + %flat_coord = cute.get_flat_coord(%index, %layout) : (..., ...) -> !cute.coord<"(2,11,1)"> + %hier_coord = cute.get_hire_coord(%index, %layout) : (..., ...) -> !cute.coord<"(2,(3,2),1)"> + %integral_coord = cute.get_integral_coord(%index, %layout) : (..., ...) -> !cute.coord<"117"> + ``` + }]; + let arguments = (ins + Cute_32BitSignlessIntegerOr64BitSignless:$index, + Cute_CuTeLayoutType:$input + ); + let results = (outs + Cute_CuteCoordType:$result + ); +} +def Cute_GetIterOp : Cute_InferredOp<"get_iter", [Pure]> { + let description = [{ + The "cute.get_iter" gets the underlying iterator from memref or coord_tensor. + It is solely as step during lowering, it has no side effects. + + Example: + + ```mlir + %ptr = cute.get_iter(%0) : !cute.memref, "((64,128),2):((1,1),64)"> + + // Or for TMA tensor + + %arith_tuple = cute.get_iter(%0) : !cute.coord_tensor<"(128, 64)", "((64,64),1):((1@0,1@0),64@1)"> + ``` + }]; + let arguments = (ins + Cute_ViewTypeInterfaceInstance:$source + ); + let results = (outs + Cute_GeneralizedIteratorTypeOfCuTe:$ptr + ); +} +def Cute_GetLayoutOp : Cute_InferredOp<"get_layout", [Pure]> { + let summary = "Gets the layout out of a `CuteViewTypeInterface`."; + let description = [{ + Example: + + ```mlir + %0 = cute.get_layout(%arg0) : !cute.memref + + %0 = cute.get_layout(%arg0) : !cute.memref o 0 o (256,?):(?,1)"> + ``` + }]; + let arguments = (ins + Cute_ViewTypeInterfaceInstance:$input + ); + let results = (outs + Cute_GeneralizedLayoutTypeOfCuTe:$layout + ); +} +def Cute_GetLayoutsFromTileOp : Cute_InferredOp<"get_layouts_from_tile", [Pure]> { + let description = [{ + This operation extracts the layout elements out of a tile. A tile is a + layout, an underscore, or a tuple of tiles. + + Examples: + + - the type of the tile is !cute.tile<"[1:0;2:1]"> + 2 layouts are extracted of type !cute.layout<"1:0"> and !cute.layout<"2:1">, respectively + - the type of the tile is !cute.tile<[_;[1:0;2:1];3:1]> + 3 layouts are extracted of type !cute.layout<"1:0">, !cute.layout<"2:1">, and + !cute.layout<"3:1">, respectively + }]; + let arguments = (ins + Cute_CuteTileType:$tile + ); + let results = (outs + Variadic:$layouts + ); +} +def Cute_GetLeavesOp : Cute_InferredOp<"get_leaves", [Pure]> { + let description = [{ + The `cute.get_leaves` operation extracts all leaf values from a hierarchical tuple into + a flat sequence of results. Each result corresponds to a leaf value found in the input tuple, + preserving the order of a depth-first traversal. + + The operation handles nested tuples of any depth, including tuples containing other tuples. + The type of each result matches the type of the corresponding leaf value in the input. + + Example: + ```mlir + // Extract leaves from a hierarchical shape tuple + %0:3 = cute.get_leaves(%hier) : !cute.shape<"(3,?{div=4},5)"> + -> (!cute.shape<"3">, !cute.shape<"?{div=4}">, !cute.shape<"5">) + + // Extract leaves from a nested int tuple + %1:4 = cute.get_leaves(%t) : !cute.int_tuple<"((1,2),(3,4))"> + -> (!cute.int_tuple<"1">, !cute.int_tuple<"2">, + !cute.int_tuple<"3">, !cute.int_tuple<"4">) + ``` + }]; + let arguments = (ins + Cute_XTupleLikeType:$input + ); + let results = (outs + Variadic:$results_ + ); +} +def Cute_GetOp : Cute_Op<"get", [Pure]> { + let description = [{ + This operation returns a potentially nested mode out of a type implementing + the `CuteValueTypeInterface`, e.g. a shape, a tuple, a layout,... + + The optional `mode` attribute indices are used to select the portion of the + input type. The number of indices in the `mode` attribute should be less + than or equal to the depth of type. If `mode` is empty, a copy of the input + value is returned. + + Example: + + ```mlir + %shape = cute.make_shape() : () -> !cute.shape<"(1, (2, 4), 8)"> + %mode0 = cute.get(%shape) <{mode=[1]}> + : !cute.shape<"(1,(2,4),8)"> -> !cute.shape<"(2,4)"> + %mode1 = cute.get(%shape) <{mode=[1,1]}> + : !cute.shape<"(1,(2,4),8)"> -> !cute.shape<"4"> + ``` + }]; + let arguments = (ins + Cute_ValueTypeInterfaceInstance:$input, + OptionalAttr:$mode + ); + let results = (outs + Cute_ValueTypeInterfaceInstanceOrIntegerTypeWithConstraintsOr:$result + ); +} +def Cute_GetScalarsOp : Cute_InferredOp<"get_scalars", [Pure]> { + let summary = "Gets the scalars out of a value with CuteValueTypeInterface."; + let description = [{ + This operation returns only the dynamic scalars when `only_dynamic` is set. + + A "scalar" in this context refers to a single value used as part of the + state of a type. In general, each state value held by a type will be + considered a scalar if it is printed in the text format. This includes: + - static integers + - dynamic integers + - swizzle parameters + - scaled basis value (static integer, ratio (num + denom), dynamic integer) + - scaled basis modes + - indexed pointer address (always a dynamic value) + - indexed pointer stride (dynamic integer or dynamic) + + Note that underscore values in coordinate types are NOT considered scalars. + + Example: + + ```mlir + %1 = cute.get_shape(%0) : (!cute.layout<"(256,?):(?,1)">) -> !cute.shape<"(256,?)"> + %2 = cute.get_scalars(%1) <{only_dynamic}> : (!cute.shape<"(256,?)">) -> i32 + ``` + }]; + let arguments = (ins + Cute_ValueTypeInterfaceInstanceOrIntegerTypeWithConstraints:$cute_value, + UnitAttr:$only_dynamic + ); + let results = (outs + Variadic:$scalars + ); +} +def Cute_GetShapeOp : Cute_InferredOp<"get_shape", [Pure]> { + let summary = "get shape from view, layout or tile"; + let description = [{ + Example: + + ```mlir + %shape = cute.get_shape(%layout) : (!cute.layout<"(32,64):(1,32)>">) -> !cute.shape<"(32,64)"> + %shape = cute.get_shape(%tile) + : (!cute.tile<"[(32,64):(1,32);3:1]>">) -> !cute.shape<"((32,64),3)"> + ``` + }]; + let arguments = (ins + Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe2:$input + ); + let results = (outs + Cute_CuteShapeType:$result + ); +} +def Cute_GetStrideOp : Cute_InferredOp<"get_stride", [Pure]> { + let summary = "get stride from layout."; + let description = [{ + Please be noted that stride is not generalized for view and layout because composed layout + doesn't have generalized layout concept. + + Example: + + ```mlir + %stride = cute.get_stride(%layout) : (!cute.layout<"(32,64):(1,32)>">) -> !cute.stride<"(1, 32)"> + ``` + }]; + let arguments = (ins + Cute_CuTeLayoutType:$input + ); + let results = (outs + Cute_CuteStrideType:$result + ); +} +def Cute_GroupModesOp : Cute_InferredOp<"group_modes", [Pure]> { + let description = [{ + This operation will make the input view/layout's [B,E) modes + into one new mode with one-depth deeper. + + Example: + + ```mlir + %res = cute.group_modes(%arg0) <0,1> + : (!cute.coord_tensor<"(0,0)", "(512,64,2):(1@1,1@0,64@0)">) + -> !cute.coord_tensor<"(0,0)", "((512),64,2):((1@1),1@0,64@0)"> + %res = cute.group_modes(%arg0) <0,2> + : (!cute.composed_layout<"(3,4,5):(6,7,8) o (0,1,2) o (9,9,9):(1@1,1@0,1@2)">) + -> !cute.composed_layout<"(3,4,5):(6,7,8) o (0,1,2) o ((9,9),9):((1@1,1@0),1@2)"> + ``` + }]; + let arguments = (ins + Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$input, + I32Attr:$begin, + I32Attr:$end + ); + let results = (outs + Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$result + ); +} +def Cute_Idx2CrdOp : Cute_InferredOp<"idx2crd", [Pure]> { + let description = [{ + compute the coordinate corresponding to an index with respect to the given shape. I.e., + compute the coordinate that maps to the given index using given shape and default stride. + Note: This only works for compact layout. + + Examples: + ```mlir + // Convert a linear index to coordinate using shape and default stride. + %0 = cute.idx2crd(%index, %shape): (!cute.int_tuple<"?">, !cute.shape<"(4,2)">) -> (!cute.coord<"(?,?)">) + // Static and dynamic shape are supported. + %1 = cute.idx2crd(%index, %shape): (!cute.int_tuple<"?">, !cute.shape<"(?,(?,?))">) -> (!cute.coord<"(?,(?,?))">) + ``` + }]; + let arguments = (ins + Cute_IntTupleAsIntegerOrTupleOfIntTupleS:$index, + Cute_CuteShapeType:$shape + ); + let results = (outs + Cute_CuteCoordType:$coord + ); +} +def Cute_IncrementCoordOp : Cute_InferredOp<"increment_coord", [Pure]> { + let summary = "This operation colexicographically increments a coord within a coord space defined by a shape."; + let description = [{ + Example: + + ```mlir + // With static coord and shape + %coord1 = cute.increment_coord(%coord0, %shape) : + (!cute.coord<"(0,0)">, !cute.shape<"(2,2)">) -> !cute.coord<"(1,0)"> + %coord2 = cute.increment_coord(%coord1, %shape) : + (!cute.coord<"(1,0)">, !cute.shape<"(2,2)">) -> !cute.coord<"(0,1)"> + + // With dynamic coord or shape + %coord1 = cute.increment_coord(%coord0, %shape) : + (!cute.coord<"(?,?)">, !cute.shape<"(?,8)">) -> !cute.coord<"(?,?)"> + ``` + }]; + let arguments = (ins + Cute_CuteCoordType:$coord, + Cute_CuteShapeType:$shape + ); + let results = (outs + Cute_CuteCoordType:$result + ); +} +def Cute_IntToPtrOp : Cute_Op<"inttoptr", [Pure]> { + let description = [{ + The `cute.inttoptr` operation accepts an integer value and cast to cute + pointer value with align=1 (byte) by default. + + To create ptr with align > 1, we must use constrained int type with divby > 1. + Alignment of result ptr is the same as `divby` of constrained int type + + Example: + + ```mlir + // Convert to cute.ptr with unrestricted integer type. Can only have align<1> + %ptr = cute.inttoptr(%0) : i64 to !cute.ptr> + + // Convert to cute.ptr with constrained integer type. + %aligned_ptr = cute.inttoptr(%0) : !cute.i64 to !cute.ptr> + ``` + }]; + let arguments = (ins + Cute_IntegerOrIntegerTypeWithConstraints:$src + ); + let results = (outs + Cute_PointerTypeInterfaceInstance:$dst + ); +} +def Cute_KernelSmemSizeOp : Cute_Op<"kernel_smem_size", [Pure]> { + let description = [{ + This host-side operation gets the static smem size from a kernel function + and returns the size as an i64 integer. + }]; + let arguments = (ins + SymbolRefAttr:$kernel_name + ); + let results = (outs + I64:$size + ); +} +def Cute_LeftInverseOp : Cute_InferredOp<"left_inverse", [Pure]> { + let arguments = (ins + Cute_CuTeLayoutType:$input + ); + let results = (outs + Cute_CuTeLayoutType:$result + ); +} +def Cute_LoadScaledIndexOp : Cute_InferredOp<"load_scaled_index", [MemoryEffects<[MemRead]>]> { + let description = [{ + The `cute.load_scaled_index` operation adds the offset (%ptr_b[%index] * %stride) to a + cute pointer %ptr_a. %ptr_a can be any cute pointer type, while %ptr_b must be an I32 cute pointer. + The behavior is undefined if %ptr_b[%index] results in an out-of-bound access. + The result is a cute pointer that has identical type as %ptr_a except that its alignment is derived + from ptr_a's alignment and the stride. + + Example: + + ```mlir + %ptr = cute.load_scaled_index(%src_ptr, %table_ptr, %index, %stride) + : (!cute.ptr, !cute.ptr, !cute.int_tuple<"?">, !cute.int_tuple<"?{div=16}">) + -> !cute.ptr + ``` + }]; + let arguments = (ins + Cute_CuTePointerType:$ptr_a, + Cute_CuTePointerTo32BitSignlessInteger:$ptr_b, + Cute_IntTupleTypeThatDepthIs0:$index, + Cute_IntTupleTypeThatDepthIs0:$stride + ); + let results = (outs + Cute_CuTePointerType:$result + ); +} +def Cute_LocalPartitionOp : Cute_InferredOp<"local_partition", [Pure]> { + let description = [{ + The `local_partition` operation tiles a tensor according to the flat shape of a layout + that provides the coordinate of the target index. + This is typical at the Thread level where data is partitioned across repeated patterns + of threads. + + Example: + + ```mlir + %local_partition = cute.local_partition(%input, %tile, %index) + : (!cute.memref, "(16,?):(1,16)">, !cute.layout<"(2,4)">, i32) + -> !cute.memref + ``` + }]; + let arguments = (ins + Cute_ViewTypeInterfaceInstance:$input, + Cute_StaticLayout:$tiler, + Cute_32BitSignlessIntegerOr64BitSignless:$index + ); + let results = (outs + Cute_ViewTypeInterfaceInstance:$result + ); +} +def Cute_LocalTileOp : Cute_InferredOp<"local_tile", [Pure]> { + let description = [{ + The `local_tile` operation tiles a tensor according to a tiler and use a coord to + index into the remainder, keeping the tile. + + This is typical at the CTA level where tiles of data are extracted: + + ```mlir + %data = // ( M, N) + %crd = cute.make_coord(blockIdx.x,blockIdx.y) : !cute.coord<"(?,?)"> + %cta_data = cute.local_tile(%data, %tile, %crd) : ... // (_32,_64) + ``` + + Example: + + ```mlir + %0 = cute.local_tile(%alloc, %tile, %coord) + : (!cute.memref, "(128,128):(65536,1)">, + !cute.tile<"[128:1;32:1]">, + !cute.coord<"(_,_)">) + -> !cute.memref, "(128,32,1,4):(65536,1,0,32)"> + + %1 = cute.local_tile(%arg0, %shape, %coord_1) + : (!cute.coord_tensor<"(0,0,0)", "(256,64,1):(1@1,1@0,1@2)">, + !cute.shape<"(256,64)">, + !cute.coord<"(0,_,_)">) + -> !cute.coord_tensor<"(0,0,0)", "(256,64,1,1):(1@1,1@0,0,1@2)"> + ``` + + Utilizing a flat coordinate '(_)' will flatten the resulting tile. + + ```mlir + %0 = cute.local_tile(%alloc, %tile, %coord) + : (!cute.memref, "(128,128):(65536,1)">, + !cute.tile<"[128:1;32:1]">, + !cute.coord<"(_)">) + -> !cute.memref, "(128,32,1,4):(65536,1,0,32)"> + + %1 = cute.local_tile(%arg0, %shape, %coord_1) + : (!cute.coord_tensor<"(0,0,0)", "(256,64,1):(1@1,1@0,1@2)">, + !cute.shape<"(256,64)">, + !cute.coord<"(_)">) + -> !cute.coord_tensor<"(0,0,0)", "(256,64,1,1,1):(1@1,1@0,0,0,1@2)"> + ``` + }]; + let arguments = (ins + Cute_ViewTypeInterfaceInstance:$input, + Cute_TileLikeTypesEGLayoutTypeShapeTypeTileType:$tile, + Cute_CuteCoordType:$coord, + OptionalAttr:$proj + ); + let results = (outs + Cute_ViewTypeInterfaceInstance:$result + ); +} +def Cute_LogicalDivideOp : Cute_InferredOp<"logical_divide", [Pure]> { + let description = [{ + This operation performs "logical_divide" on the input and produces a logical_divided result, + and it automatically infers the "logical_divided" layout for the result type. + + Example: + + ```mlir + %alloc = cute.memref.alloc() : !cute.memref + %shape = cute.make_shape() : () -> !cute.shape<"(4,8)"> + %zdiv = cute.logical_divide(%alloc, %shape) + : (!cute.memref, !cute.shape<"(4,8)">) + -> !cute.memref + ``` + }]; + let arguments = (ins + Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$input, + Cute_TileLikeTypesEGLayoutTypeShapeTypeTileType:$tiler + ); + let results = (outs + Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$result + ); +} +def Cute_LogicalProductOp : Cute_InferredOp<"logical_product", [Pure]> { + let description = [{ + This operation performs "logical_product" on the input and produce a logical_product result, + and it automatically infer the "logical_product" layout for the result type. + + Example: + + ```mlir + %input = cute.make_layout() : !cute.layout<"(3,4):(4,1)"> + %tiler = cute.make_layout() : !cute.layout<"(2,5):(1,2)"> + %prod = cute.logical_product(%input, %tiler) + : (!cute.layout<"(3,4):(4,1)">, !cute.layout<"(2,5):(1,2)">) + -> !cute.layout<"((3,4),(2,5)):((4,1),(12,24))"> + ``` + + Example with dynamic composed_layout: + + ``` + %prod = cute.logical_product(%input, %tiler) + : (!cute.composed_layout<"(?):(?) o ? o (5,4):(4,1)">, + !cute.layout<"(?,?):(?,?)">) + -> !cute.composed_layout<"(?):(?) o ? o ((5,4),(?,?)):((4,1),(?{div=20},?{div=20}))"> + ``` + }]; + let arguments = (ins + Cute_GeneralizedLayoutTypeOfCuTe:$input, + Cute_CuTeLayoutType:$tiler + ); + let results = (outs + Cute_GeneralizedLayoutTypeOfCuTe:$result + ); +} +def Cute_MakeArithTupleIteratorOp : Cute_Op<"make_arith_tuple_iter", [Pure]> { + let description = [{ + This Operation makes an arithmetic tuple iterator (`!cute.arith_tuple_iter<...>`) + from an int_tuple value (`!cute.int_tuple<...>`) if provided, or it is empty + when all the modes are static. + + Its verifier checks that the value type of the result iterator matches the + operand value type if provided. + }]; + let arguments = (ins + Optional:$value + ); + let results = (outs + Cute_CuTeIteratorTypeWithIntTupleAsIts:$iter + ); +} +def Cute_MakeAtomOp : Cute_Op<"make_atom", [Pure]> { + let summary = "`cute.make_atom` op is generic constructor of SSA value of copy or mma atom type."; + let description = [{ + It optionally takes variadic list of SSA values as well as named attribute to + initialize corresponding fields of copy or mma atom. + + Examples: + + ```mlir + // create copy atom which doesn't require initialization + %0 = cute.make_atom() : () + -> !cute_nvgpu.atom.tmem_load + + // create mma atom and initialize it with SSA values + %1 = cute.make_atom(%accum_c, %a_neg, %b_neg) : (i1, i1, i1) + -> !cute_nvgpu.sm100.mma<128x128x16, + num_cta = 1, + ab_major = (k,k), + elem_type = (f16,f16,f32), + frag_kind = ss, + c_scale_exp = 0> + ``` + }]; + let arguments = (ins + Variadic:$values + ); + let results = (outs + Cute_AtomTypeInterfaceInstance:$atom + ); +} +def Cute_MakeComposedLayoutOp : Cute_InferredOp<"make_composed_layout", [Pure]> { + let summary = "The `make_composed_layout` operation constructs composed layout with"; + let description = [{ + * (inner) affined layout or swizzle layout + * offset + * (outer) layout + + In CuTe algebra, composed layout represent composition of + + ```math + L_{inner} o offset o L_{outer} + ``` + + where outer layout defined the domain of the layout which is visible to user. For given input coordinate + in domain of outer layout, composed layout maps coordinate to codomain by + + ```math + L_{inner}(L_{outer}(coordinate) + offset) + ``` + + Composed Layout is useful in cases where composition() does not or cannot apply to LayoutA and LayoutB. + For example, when the "divisibility condition" in shape_div is violated in composition(LayoutA, LayoutB). + + This ComposedLayout provides similar functionality to Layout including tiling, partitioning, + coordinate-to-index mapping and layout manipulations, but is not considered a "normal" layout. + For example, this layout provides shape() and size() functions, but does not provide stride() functions. + Mostly, the similar functionality is accomplished by applying each operation to LayoutB only + as LayoutB defines the domain. + + ### Examples: + + ```mlir + // Swizzle composed layout + %inner : !cute.swizzle<"S<3,4,3>"> + %layout = cute.make_composed_layout(%inner, %offset, %outer) + : !cute.composed_layout<"S<3,4,3> o 5 o (?,?):(?,?)"> + + // Linearized composed layout + %inner : !cute.layout<"((?,?),(?,?)):((?,?),(?,?))"> + %layout = cute.make_composed_layout(%inner, %offset, %outer) + : !cute.composed_layout<"((?,?),(?,?)):((?,?),(?,?)) o ? o (1,2):(3,4)"> + ``` + }]; + let arguments = (ins + Cute_CuTeLayoutTypeOrSwizzleWithValidParameters:$inner, + Cute_IntTupleAsIntegerOrTupleOfIntTupleS:$offset, + Cute_CuTeLayoutType:$outer + ); + let results = (outs + Cute_CuTeComposedLayoutType:$result + ); +} +def Cute_MakeCoordOp : Cute_Op<"make_coord", [Pure]> { + let description = [{ + Constructs a coord. The operands correspond to the + coord's dynamic elements, ordered from left to right. + The number of operands should match the count of dynamic elements in the + resulting coord type. + + Example: + + ```mlir + // Definitions for %0, %1 come earlier + %shape = cute.make_coord(%0, %1) : (i32, i32) + -> !cute.coord<"((?,32),?)"> + ``` + }]; + let arguments = (ins + Variadic:$dynamicElements + ); + let results = (outs + Cute_CuteCoordType:$result + ); +} +def Cute_MakeDescriptorIteratorOp : Cute_Op<"make_desc_iter", [Pure]> { + let description = [{ + This Operation makes a descriptor iterator from a descriptor value. Its + verifier verifies that the value Type of the result Type implementing the + iterator Type Interface is the same as the Type as the input operand. + }]; + let arguments = (ins + Cute_ATypeInterfaceForMemoryDescriptorsSuchAs:$value + ); + let results = (outs + AnyType:$iter + ); +} +def Cute_MakeFragmentLikeOp : Cute_InferredOp<"make_fragment_like", [Pure]> { + let summary = "A deprecated operation but used by collective DSL (keep it here to avoid breaking changes)"; + let description = [{ + TODO: remove this operation in the future + }]; + let arguments = (ins + Cute_GeneralizedLayoutTypeOfCuTe:$src + ); + let results = (outs + Cute_CuTeLayoutType:$dst + ); +} +def Cute_MakeIdentityLayoutOp : Cute_InferredOp<"make_identity_layout", [Pure]> { + let summary = "This operation makes an identity layout with the given shape."; + let description = [{ + Example: + ```mlir + %sha = cute.make_shape() : () -> !cute.shape<"(2,(3,4),5)"> + %lay = cute.make_identity_layout(%sha) : !cute.layout<"(2,(3,4),5):(1@0,(1@0@1,1@1@1),1@2)"> + ``` + }]; + let arguments = (ins + Cute_CuteShapeType:$shape + ); + let results = (outs + Cute_CuTeLayoutType:$result + ); +} +def Cute_MakeIdentityTensorOp : Cute_InferredOp<"make_identity_tensor", [Pure]> { + let description = [{ + The cute.make_identity_tensor operation construct an identity_tensor + based on the given shape. + + Example to make an identity_tensor from a static shape: + + ```mlir + %identity_tensor = cute.make_identity_tensor(%shape) + : !cute.coord_tensor<"(0,(0,0))", "(2,(3,4)):(1@0,(1@0@1,1@1@1))"> + ``` + + Example to make an identity_tensor from a dynamic shape: + + ```mlir + %identity_tensor = cute.make_identity_tensor(%shape) + : !cute.coord_tensor<"((0,0),(0,0))", "((?,?),(?,?)):((1@0@0,1@1@0),(1@0@1,1@1@1))"> + ``` + }]; + let arguments = (ins + Cute_CuteShapeType:$shape + ); + let results = (outs + Cute_CuteCoordTensorType:$result + ); +} +def Cute_MakeIntTupleOp : Cute_Op<"make_int_tuple", [Pure]> { + let description = [{ + Constructs a int_tuple. The operands correspond to the + int_tuple's dynamic elements, ordered from left to right. + The number of operands should match the count of dynamic elements in the + resulting int_tuple type. + + Example: + + ```mlir + // Definitions for %0, %1 come earlier + %shape = cute.make_int_tuple(%0, %1) : (i32, i32) + -> !cute.int_tuple<"((?,32),?)"> + ``` + }]; + let arguments = (ins + Variadic:$dynamicElements + ); + let results = (outs + Cute_IntTupleAsIntegerOrTupleOfIntTupleS:$result + ); +} +def Cute_MakeLayoutLikeOp : Cute_InferredOp<"make_layout_like", [Pure]> { + let description = [{ + Creates a compact memory layout that: + 1. Has a congruent shape with the source layout + 2. Has strides in the same order of major-ness as the input layout + 3. Mode 0 strides are `LayoutLeft` (column major) with zero strides preserved from the input layout + + Mode-0 constraints are applied due to its common usage with MMA_Atoms and Copy_Atoms + that need to create RMEM fragments. + + Example: + ```mlir + %0 = cute.make_layout_like(%layout) + : !cute.layout<"(4,8):(1,1024)"> to !cute.layout<"(4,8):(1,4)"> + ``` + }]; + let arguments = (ins + Cute_GeneralizedLayoutTypeOfCuTe:$src + ); + let results = (outs + Cute_CuTeLayoutType:$dst + ); +} +def Cute_MakeLayoutOp : Cute_Op<"make_layout", [AttrSizedOperandSegments, Pure]> { + let description = [{ + The `make_layout` operation generates a layout based on provided shape and stride. + If the stride component is omitted in the operation's result type + (e.g., `!cute.layout<"(?,(?,?))">`), the operation defaults to a column-major + layout, and missing stride details are automatically inferred to complete the type + of the resulting layout. + + The shape or stride operand must be supplied if they contain dynamic elements. + However, if the shape or stride is entirely static, corresponding operands can be + neglected. + + When the result type does not specify stride information (like `!cute.layout<"(?,?)">` + ), it is not necessary to provide a stride operand, even if the stride to be inferred + includes dynamic components (as in the inferred type `!cute.layout<"(?,?):(1,?)">`). + + ### Examples: + + ```mlir + // Shape and stride are both dynamic + %layout = cute.make_layout(%shape, %stride) : !cute.layout<"(?,64):(1,?)"> + + // Shape and stride are both static + %layout = cute.make_layout() : !cute.layout<"(32,64):(1,32)"> + + // Stride is omitted in the result type, implying it should be inferred + %layout = cute.make_layout(%shape) : !cute.layout<"(?,(?,?))"> + ``` + }]; + let arguments = (ins + Optional:$shape, + Optional:$stride + ); + let results = (outs + Cute_CuTeLayoutType:$layout + ); +} +def Cute_MakeOrderedLayoutOp : Cute_InferredOp<"make_ordered_layout", [Pure]> { + let description = [{ + The `make_ordered_layout` operation generates a compact layout based on provided `shape` + and stride is calculated by `shape` and `order`. + + Different from `make_layout`, `shape` operand is required, `order` is optional. When `order` + isn't set, default stride growth order is from left to right. Supports dynamic values in `shape`, + while currently `order` is restricted to static. Rank of `order` should be equal to rank of + shape but no need for congruent. + + Note that numbers in `order` are just used for relative order relationship, e.g. (67,46,23,32) + is equivalent to (4,3,1,2), (3,2,4) is equivalent to (2,1,3). + + ### Examples: + + ```mlir + %shape = cute.make_shape() : !cute.shape<"(32,64)"> + %shape_dyn = cute.make_shape() : !cute.shape<"(32,?)"> + %order = cute.make_shape() : !cute.shape<"(1,2)"> + %layout = cute.make_ordered_layout(%shape, %order) + : (!cute.shape<"(32,64)">, !cute.shape<"(1,2)">) -> !cute.layout<"(32,64):(1,32)"> + %layout_dyn = cute.make_ordered_layout(%shape_dyn, %order) + : (!cute.shape<"(32,?)">, !cute.shape<"(1,2)">) -> !cute.layout<"(32,?):(1,32)"> + + %order = cute.make_shape() : !cute.shape<"(2,1)"> + %layout = cute.make_ordered_layout(%shape, %order) + : (!cute.shape<"(32,64)">, !cute.shape<"(2,1)">) -> !cute.layout<"(32,64):(64,1)"> + %layout_dyn = cute.make_ordered_layout(%shape_dyn, %order) + : (!cute.shape<"(32,?)">, !cute.shape<"(2,1)">) -> !cute.layout<"(32,?):(?,1)"> + + %shape = cute.make_shape() : !cute.shape<"(2,(2,2),2)"> + %order = cute.make_shape() : !cute.shape<"(0,(2,3),1)"> + %layout = cute.make_ordered_layout(%shape, %order) : !cute.layout<"(2,(2,2),2):(1,(4,8),2)"> + + %order = cute.make_shape() : !cute.shape<"(2,1,3)"> + %layout = cute.make_ordered_layout(%shape, %order) : !cute.layout<"(2,(2,2),2):(4,(1,2),8)"> + }]; + let arguments = (ins + Cute_CuteShapeType:$shape, + Cute_IntTupleAsIntegerOrTupleOfIntTupleS:$order + ); + let results = (outs + Cute_CuTeLayoutType:$layout + ); +} +def Cute_MakeShapeOp : Cute_Op<"make_shape", [Pure]> { + let description = [{ + Constructs a shape. The operands correspond to the + shape's dynamic elements, ordered from left to right. + The number of operands should match the count of dynamic elements in the + resulting shape type. + + Example: + + ```mlir + // Definitions for %0, %1 come earlier + %shape = cute.make_shape(%0, %1) : (i32, i32) + -> !cute.shape<"((?,32),?)"> + ``` + }]; + let arguments = (ins + Variadic:$dynamicElements + ); + let results = (outs + Cute_CuteShapeType:$result + ); +} +def Cute_MakeSparseElemOp : Cute_Op<"make_sparse_elem", [Pure]> { + let summary = "Creates a sparse element from a physical storage."; + let description = [{ + Example: + + ```mlir + %v0 = arith.constant 8 : i8 + %v1 = arith.constant 31 : i8 + %physical_storage = vector.from_elements(vector<2xf16>, %v0, %v1) + %sparse_elem = cute.make_sparse_elem(%physical_storage) : (!cute.sparse_elem<4, vector<2xf16>>) + ``` + }]; + let arguments = (ins + AnyType:$physical_storage + ); + let results = (outs + Cute_CuTeSparseElementType:$result + ); +} +def Cute_MakeStrideOp : Cute_Op<"make_stride", [Pure]> { + let description = [{ + Constructs a stride. The operands correspond to the + stride's dynamic elements, ordered from left to right. + The number of operands should match the count of dynamic elements in the + resulting stride type. + + Example: + + ```mlir + // Definitions for %0, %1 come earlier + %shape = cute.make_stride(%0, %1) : (i32, i32) + -> !cute.stride<"((?,32),?)"> + ``` + }]; + let arguments = (ins + Variadic:$dynamicElements + ); + let results = (outs + Cute_CuteStrideType:$result + ); +} +def Cute_MakeTileOp : Cute_Op<"make_tile", [Pure]> { + let description = [{ + Constructs a tile. The operands correspond to the + tile's dynamic elements, ordered from left to right. + The number of operands should match the count of dynamic elements in the + resulting tile type. + + Example: + + ```mlir + // Definitions for %0, %1 come earlier + %shape = cute.make_tile(%0, %1) : (i32, i32) + -> !cute.tile<"((?,32),?)"> + ``` + }]; + let arguments = (ins + Variadic:$dynamicElements + ); + let results = (outs + Cute_CuteTileType:$result + ); +} +def Cute_MakeTiledCopyOp : Cute_Op<"make_tiled_copy", [Pure]> { + let description = [{ + This operation creates a new copy atom instance by tiling a "base" copy atom + instance. Copy atoms are used by the CuTe algorithm layer during lowering to + generate the appropriate `cute_nvgpu.arch` operations. + + Example: + The below values are origin from cutlass SM100_f16f16_UMMA kernel, the original code: + auto tAgA_mkl = cta_tma_a.partition_S(cta_mma.partition_A(gA_mkl))(_,_0{},_0{},_0{},_,_,_); + All the values are from the "cta_tma_a". + ```mlir + %0 = cute.make_atom(%tma_desc_ptr, %tma_bar) : (!cute.ptr>!cute.ptr>) + -> !cute_nvgpu.atom.tma_load + %1 = cute.make_tiled_copy(%0) + : !cute.tiled_copy, + layout_copy_tv = <"((4,2),(64,128)):((0,64),(128,1))">, + tiler_mn = <"[2048:1;1:1;4:1;1:1]">> + ``` + }]; + let arguments = (ins + Cute_CopyAtomTypeInterfaceInstance:$atom + ); + let results = (outs + Cute_TiledCopyAtomType:$result + ); +} +def Cute_MakeTiledCopyV2Op : Cute_Op<"make_tiled_copy_v2", [Pure]> { + let description = [{ + This operation creates a V2 tiled copy instance by tiling a V2 copy atom. + V2 copy atoms use fragment attributes for operand-specific behavior. + + Example: + ```mlir + %0 = cute.make_atom() : () -> !cute_nvgpu.atom.sm100_s2t_copy_v2 + %1 = cute.make_tiled_copy_v2(%0) + : !cute.tiled_copy_v2, + layout_copy_tv = <"((32,4),(2,32)):((0,1),(128,4))">, + tiler_mn = <"[(4,32):(32,1);2:1]">> + ``` + }]; + let arguments = (ins + Cute_CopyAtomTypeInterfaceV2Instance:$atom + ); + let results = (outs + Cute_TiledCopyV2AtomType:$result + ); +} +def Cute_MakeTiledMmaOp : Cute_Op<"make_tiled_mma", [Pure]> { + let description = [{ + This operation creates a new MMA atom instance by tiling a "base" MMA atom + instance. MMA atoms are used by the CuTe algorithm layer during lowering to + generate the appropriate `cute.arch` operations. + + Example: + ```mlir + %0 = cute.make_atom(%accum, %a_neg, %b_neg) : (i1, i1, i1) + -> !cute_nvgpu.sm100.mma<64x64x16, num_cta = 1, ab_major = (k,mn), elem_type = (f16,f16,f16), frag_kind = ss, c_scale_exp = 0> + %1 = cute.make_tiled_mma(%0) + : !cute.tiled_mma, + atom_layout_MNK = <"(1,1,1)">> + ``` + }]; + let arguments = (ins + Cute_MmaAtomTypeInterfaceInstance:$atom + ); + let results = (outs + Cute_TiledMMAAtomType:$result + ); +} +def Cute_MakeTupleOp : Cute_Op<"make_tuple", [Pure]> { + let description = [{ + Creates a tuple from multiple values. The number and types of the + parameters must match the number and types of the result tuple. + }]; + let arguments = (ins + Variadic:$values + ); + let results = (outs + Cute_ATupleIsAFixedLengthOrderedList:$result + ); +} +def Cute_MakeViewOp : Cute_Op<"make_view", [Pure]> { + let description = [{ + The cute.make_view operation construct `CuteViewTypeInterface` + (i.e., cute.memref or cute.coord_tensor) with given `iter` and `layout`. + Internally, we infer the exact return type by checking `iter`'s type: + - if `iter` is cute.ptr, return type is `CuteMemRefType` + - if `iter` is cute.tuple, return type is `CuteCoordTensorType` + + `iter` and `layout` could contain both static and dynamic values. + + `MakeViewOp` usually is the result of a previous operation + (e.g., cute.slice, cute.zipped_divide), where creating a new + `CuteViewTypeInterface` is needed. + + `MakeViewOp` does not involve allocating buffers of any kind. + + Example to make cute.coord_tensor with an iter of cute.tuple type: + + ```mlir + %coord_tensor= cute.make_view(%iter, %layout) : !cute.coord_tensor<"(0,2)", "(30):(20@1)"> + ``` + + Example to make cute.memref with an iter of cute.ptr type: + + ```mlir + %memref= cute.make_view(%iter, %layout) : !cute.memref + ``` + + Static Layout + + ```mlir + %memref= cute.make_view(%iter) : !cute.memref + ``` + + Composed+Static Layout + + ```mlir + %memref= cute.make_view(%iter) : !cute.memref o 9 o (64):(16)"> + ``` + }]; + let arguments = (ins + Cute_GeneralizedIteratorTypeOfCuTe:$iter, + Optional:$layout + ); + let results = (outs + Cute_ViewTypeInterfaceInstance:$result + ); +} +def Cute_MemRefAllocSmemOp : Cute_Op<"memref.alloc_smem"> { + let description = [{ + The memref.alloc_smem operation is utilized for the allocation of static + shared memory. The cosize of the memref layout, which must be static, + determines the size of the memory allocated by this operation. + + ```mlir + %0 = cute.memref.alloc_smem() : !cute.memref + ``` + }]; + let arguments = (ins + ); + let results = (outs + Cute_MemrefInSmemWithStaticAndNonComposed:$memref + ); +} +def Cute_MemRefAllocaOp : Cute_Op<"memref.alloca"> { + let description = [{ + The memref.alloca operation is used to allocate shared/stack memory, which is automatically + released when control returns from the closest surrounding operation's region. This + operation only supports memref with the address space reg, and attempting to allocate + memory with other address spaces will result in an error. + + The size of memory allocated is determined by the layout operand's cosize, which can + be either static or dynamic. The layout operand can be omitted when the memref layout + is static. + + If the memref is sparse, the allocated memory size will be layout operand's cosize + divided by sparsity factor. + + ```mlir + // %dynamic_layout : !cute.layout<"(?,24)"> + %0 = cute.memref.alloca(%dynamic_layout) : !cute.memref + // static layout + %1 = cute.memref.alloca() : !cute.memref + ``` + }]; + let arguments = (ins + Optional:$layout + ); + let results = (outs + Cute_MemrefInGenericOrRegisterMemOrSmem:$result + ); +} +def Cute_MemRefLoadOp : Cute_InferredOp<"memref.load", [MemoryEffects<[MemRead]>]> { + let description = [{ + The `memref.load` operation loads one element at the specified logical coordinate from a memref + in gmem/rmem/smem/generic memory spaces to a register. This operation performs the following steps: + + 1. Converts the logical coordinate to a linear index using `cute.crd2idx` + 2. Retrieves the iterator from the memref using `cute.get_iter` + 3. Adds the offset to the iterator using `cute.add_offset` + 4. For CuteMemRefType, applies swizzling if needed, converts to LLVM pointer, and performs the load + 5. For CoordTensorType, returns the tuple directly + + The element type generally cannot be a sub-byte type (no f4, f6, i1, i2 etc.), with the exception + of i1 which has special handling. + + NOTE: For tf32 types, the operation treat it as f32 even bitwidth of MLIR type is 19. + + Example: + + ```mlir + // Load a fp32 + %coord = cute.make_coord(%c1_i32, %c2_i32) : (i32, i32) -> !cute.coord<"(?,?)"> + %0 = cute.memref.load(%memref, %coord) : (!cute.memref, !cute.coord<"(?,?)">) -> f32 + + // Load a tf32 value (will be handled with special bitwise casting) + %1 = cute.memref.load(%tf32_memref, %coord) : (!cute.memref, !cute.coord<"(?,?)">) -> tf32 + ``` + }]; + let arguments = (ins + Cute_AMemoryReferenceTypeThatRepresentsATensor2:$src, + Cute_CuteCoordType:$coord + ); + let results = (outs + AnyType:$result + ); +} +def Cute_MemRefLoadVecOp : Cute_InferredOp<"memref.load_vec", [AttrSizedOperandSegments, MemoryEffects<[MemRead]>]> { + let description = [{ + The `memref.load_vec` operation loads all elements in the logical coordinate + space from a memref generic or register memory to a 1D vector. Only memrefs with + static layouts are accepted. + + The resulting vector has the same element type as the memref, with a size + equal to the **size** (not **cosize**) of the memref layout. + + An optional mask vector (of the same size as the output vector) can be + provided to mask out elements. When the mask is present, an optional + pass_thru vector can also be provided to fill in values for elements that + are masked out, and thus the result is equivalent to a regular memref load + followed by a select between the loaded and the passthru values, predicated + on the same mask, except that the masked-off lanes are not accessed. + + Example: + + ```mlir + // Load vector + %0 = cute.memref.load_vec(%memref) : (!cute.memref) -> vector<32xf32> + + // Load vector from swizzled memory + %0 = cute.memref.load_vec %memref : !cute.memref, "((32,1),1,1):((1,0),0,0)"> + + // Load vector with mask and pass_thru + %0 = cute.memref.load_vec(%memref, %mask, %pass_thru) + : (!cute.memref, vector<32xi1>, vector<32xf32>) -> vector<32xf32> + ``` + }]; + let arguments = (ins + Cute_StaticShape:$src, + Optional:$mask, + Optional:$pass_thru + ); + let results = (outs + Cute_OfRanks1:$result + ); +} +def Cute_MemRefStoreOp : Cute_Op<"memref.store"> { + let description = [{ + The `memref.store` operation stores a register value to a memref in gmem/rmem/smem/generic at given logical coordinate. + This operation performs the following steps: + + 1. Converts the logical coordinate to a linear index using `cute.crd2idx` + 2. Retrieves the iterator from the memref using `cute.get_iter` + 3. Adds the offset to the iterator using `cute.add_offset` + 4. Converts the pointer to the appropriate LLVM pointer type and performs the store + + The element type generally cannot be a sub-byte type (no f4, f6, i1, i2 etc.), with the exception + of i1 which has special handling. + + NOTE: For tf32 types, the operation treats it as f32 even though the bitwidth of the MLIR type is 19. + This special handling allows for proper conversion between tf32 and f32 representations during store operations. + + Example: + + ```mlir + // Store a fp32 + %coord = cute.make_coord(%c1_i32, %c2_i32) : (i32, i32) -> !cute.coord<"(?,?)"> + cute.memref.store(%memref, %coord, %value) : (!cute.memref, !cute.coord<"(?,?)">, f32) + + // Store a tf32 value (will be handled with special bitwise casting) + cute.memref.store(%tf32_memref, %coord, %tf32_value) : (!cute.memref, !cute.coord<"(?,?)">, tf32) + ``` + }]; + let arguments = (ins + Cute_AMemoryReferenceTypeThatRepresentsATensor3:$dst, + Cute_CuteCoordType:$coord, + AnyType:$value + ); + let results = (outs + ); +} +def Cute_MemRefStoreVecOp : Cute_Op<"memref.store_vec"> { + let description = [{ + The `memref.store_vec` operation stores all elements of a 1D vector to a to the logical coordinate + space of memref in generic or register memory. Only memrefs with static layouts are accepted. + + The vector has the same element type as the memref, with a size equal to + the **size** (not **cosize**) of the memref layout. + + An optional mask vector (of the same size as the input vector) can be + provided to mask out elements. + + ```mlir + // Store vector + %0 = cute.memref.store_vec(%value, %memref) : (vector<32xf32>, !cute.memref) -> () + + // Store vector to swizzled memory + %0 = cute.memref.store_vec %value, %memref : !cute.memref, "((32,1),1,1):((1,0),0,0)"> + + // Store vector with mask + %0 = cute.memref.store_vec(%value, %memref, %mask) + : (vector<32xf32>, !cute.memref, vector<32xi1>) -> () + ``` + }]; + let arguments = (ins + Cute_OfRanks1:$value, + Cute_StaticShape:$dst, + Optional:$mask + ); + let results = (outs + ); +} +def Cute_MmaAtomCallOp : Cute_Op<"mma_atom_call", [AttrSizedOperandSegments]> { + let summary = "`cute.mma_atom_call` op executes a mma atom operation with the given operands."; + let description = [{ + The operands `a` and `b` are variadic, each containing a variable number of tensors: + - For regular MMA, `a` and `b` contain the MMA A and B tensors respectively. + - For MMA with auxiliary operands, `a` and `b` contain the MMA A and B tensors followed by their respective auxiliary tensors. For example: + - For BlockScaledMMA, `a` = [A, SFA] and `b` = [B, SFB]. + - For SparseMMA, `a` = [A, E] and `b` = [B]. + - For BlockScaledSparseMMA, `a` = [A, SFA, E] and `b` = [B, SFB]. + Example (regular MMA): + + ```mlir + // Call a mma atom operation + cute.mma_atom_call(%mma_atom, %d, %a, %b, %c) + : (!cute_nvgpu.sm100.mma<128x128x16, num_cta = 1, ab_major = (k,k), + elem_type = (f16,f16,f32), + frag_kind = ss, + c_scale_exp = 0>, + !cute.memref, "((128,128)):((131072,1))">, + !cute_nvgpu.smem_desc_view, + !cute_nvgpu.smem_desc_view, + !cute.memref, "((128,128)):((131072,1))">) + -> () + ``` + + Example (block-scaled MMA): + + ```mlir + cute.mma_atom_call(%mma_atom, %d, [%a, %sfa], [%b, %sfb], %c) + : (!cute_nvgpu.sm100.mma_bs<256x128x64, num_cta = 2, ab_major = (k,k), + elem_type = (f4E2M1FN, f4E2M1FN, f32), + sf_type = f8E4M3FN, frag_kind = ss, vec_size = 16>, + !cute.memref, "((128,128)):((65536,1))">, + [!cute_nvgpu.smem_desc_view, + !cute.memref, "((((32,4),4),(16,4))):((((262144,4),8388608),(0,1)))">], + [!cute_nvgpu.smem_desc_view, + !cute.memref, "((((32,4),4),(16,4))):((((262144,4),8388608),(0,1)))">], + !cute.memref, "((128,128)):((65536,1))">) + -> () + ``` + + Example (block-scaled MMA): + + ```mlir + cute.mma_atom_call(%mma_atom, %d, [%a, %sfa], [%b, %sfb], %c) + : (!cute_nvgpu.sm100.mma_bs<256x128x64, num_cta = 2, ab_major = (k,k), + elem_type = (f4E2M1FN, f4E2M1FN, f32), + sf_type = f8E4M3FN, frag_kind = ss, vec_size = 16>, + !cute.memref, "((128,128)):((65536,1))">, + [!cute_nvgpu.smem_desc_view, + !cute.memref, "((((32,4),4),(16,4))):((((262144,4),8388608),(0,1)))">], + [!cute_nvgpu.smem_desc_view, + !cute.memref, "((((32,4),4),(16,4))):((((262144,4),8388608),(0,1)))">], + !cute.memref, "((128,128)):((65536,1))">) + -> () + ``` + + Example (block-scaled MMA): + + ```mlir + cute.mma_atom_call(%mma_atom, %d, [%a, %sfa], [%b, %sfb], %c) + : (!cute_nvgpu.sm100.mma_bs<256x128x64, num_cta = 2, ab_major = (k,k), + elem_type = (f4E2M1FN, f4E2M1FN, f32), + sf_type = f8E4M3FN, frag_kind = ss, vec_size = 16>, + !cute.memref, "((128,128)):((65536,1))">, + [!cute_nvgpu.smem_desc_view, + !cute.memref, "((((32,4),4),(16,4))):((((262144,4),8388608),(0,1)))">], + [!cute_nvgpu.smem_desc_view, + !cute.memref, "((((32,4),4),(16,4))):((((262144,4),8388608),(0,1)))">], + !cute.memref, "((128,128)):((65536,1))">) + -> () + ``` + }]; + let arguments = (ins + Cute_MmaAtomTypeInterfaceInstance:$atom, + Cute_ViewTypeInterfaceInstance:$d, + Variadic:$a, + Variadic:$b, + Cute_ViewTypeInterfaceInstance:$c + ); + let results = (outs + ); +} +def Cute_MmaMakeFragmentOp : Cute_InferredOp<"mma.make_fragment", [Pure]> { + let description = [{ + Given a partitioned tensor, the `cute.mma.make_fragment` operation returns + a tensor with the correct memory space for the specified operand and dot + atom. This op is usually used between partition and gemm. It is the + "prepare" step before gemm. + + `cute.tiled.mma.partition` returns a view into the specified tensor, but it + does not copy the data to the correct memory space. This is done by + `cute.mma.make_fragment` and explicit memcpy operations. Whether a memcpy + is needed or not depend on the result memory space. + + * RMEM: The result tensor is a newly allocated tensor with unspecified + contents. A memcpy is needed. + * SMEM: The result tensor is a view into the input tensor in the form of a + CuteSmemDescViewType. A memcpy is not needed. + * TMEM: The result tensor uses a dummy TMEM engine (fake pointer 0x0) and + must be set correctly later, aligned with Cute C++. + + Instead of a tensor, this op also accepts cute.layout (for RMEM/TMEM) and + cute.shape (for RMEM/TMEM). + + Example: + + ```mlir + // Example 1: Full example for SM80. + // Partition a tensor for operand A, which is stored in SMEM. + %partitioned = cute.tiled.mma.partition A(%atom, %smem_A, %tid_coord) + : (!cute.tiled_mma, + atom_layout_MNK = <"(2,2,1)">>, + !cute.memref, S<2,4,3>, "(128,32):(32,1)">, + !cute.coord<"?">) + -> !cute.memref, S<2,4,3>, "((2,2,2),4,2):((1,256,8),1024,16)"> + + // The tensor is still in SMEM, but sm80.mma expects A in registers. + // Prepare a tensor with the correct shape and strides in RMEM. + %fragment = cute.mma.make_fragment A (%3, %partitioned) + : (!cute.tiled_mma, + atom_layout_MNK = <"(2,2,1)">>, + !cute.memref, S<2,4,3>, "((2,2,2),4,2):((1,256,8),1024,16)">) + -> !cute.memref, "((2,2,2),4,2):((1,2,4),16,8)"> + + // Copy the data from SMEM to RMEM. + cute.copy(..., %partitioned, %fragment) + + // Example 2: Make fragment for SM100 (SMEM). + %fragment = cute.mma.make_fragment A (%0, %1) + : (!cute.tiled_mma, + atom_layout_MNK = <"(1,1,1):(0,0,0)">>, + !cute.memref, "((128,16),2,32):((512,1),65536,16)">) + -> !cute_nvgpu.smem_desc_view + + // Example 3: Make fragment for SM100 (TMEM) from layout. + %fragment = cute.mma.make_fragment A (%0, %1) + : (!cute.tiled_mma, + atom_layout_MNK = <"(1,1,1):(0,0,0)">>, + !cute.layout<"((64,16),1,4):((1,64),0,1024)">) + -> !cute.memref, "((128,16),1,4):((131072,1),0,16)"> + + // Example 3: Make fragment for SM100 (TMEM) from shape. + %fragment = cute.mma.make_fragment A (%0, %1) + : (!cute.tiled_mma, + atom_layout_MNK = <"(1,1,1):(0,0,0)">>, + !cute.shape<"((64,16),1,4)">) + -> !cute.memref, "((128,16),1,4):((131072,1),0,16)"> + ``` + }]; + let arguments = (ins + Cute_MmaAtomTypeInterfaceInstance:$atom, + Cute_AMemoryReferenceTypeThatRepresentsATensor:$input, + Cute_MmaOperandEnum:$operand_id + ); + let results = (outs + Cute_ViewTypeInterfaceInstance:$result + ); +} +def Cute_PrefetchAtomCallOp : Cute_Op<"prefetch_atom_call", [MemoryEffects<[MemRead]>]> { + let summary = "`cute.exec_prefetch_atom` op executes a prefetch atom operation with the given operands."; + let description = [{ + Following src layout of atom are valid + * ((atom_v, rest_v)) + * ((atom_v)) + * (atom_v) + + The `src` operand is variadic: + - For regular TMA prefetch, `src` is a single tensor. + - For 2D `tile::gather4` mode, `src` is `[data_coord_tensor, gmem_index_tensor]`. + + Examples: + + ```mlir + // Call a prefetch atom operation + cute.prefetch_atom_call(%copy_tma_load, %src_v) + : (!cute_nvgpu.atom.tma_load, + %src: !cute.coord_tensor<"(0,0)", "(((64,128),1),1,1,2):(((1@0,1@1),0),0,0,64@0)">) + -> () + + // gather4 prefetch: data tensor + gmem index tensor + cute.prefetch_atom_call(%gather4_atom, %src_v, %idx_v) + : (!cute_nvgpu.atom.tma_load>, + !cute.coord_tensor<...>, + !cute.coord_tensor<...>) + -> () + ``` + }]; + let arguments = (ins + Cute_PrefetchAtomTypeInterfaceOrTiledCopyTypeWithPrefetchAtomTypeInterfaceCopyAtom:$atom, + Variadic:$src + ); + let results = (outs + ); +} +def Cute_PrependToRankOp : Cute_InferredOp<"prepend_to_rank", [Pure]> { + let summary = "Prepend element to input to rank N."; + let description = [{ + Example: + + ```mlir + %shp = cute.make_shape() : !cute.shape<"(3,4,5)"> + %elem = cute.make_shape() : !cute.shape<"(1,2)"> + %new_layout = cute.prepend_to_rank<5> (%shp, %elem) + : (!cute.shape<"(3,4,5)">, !cute.shape<"1">) -> !cute.shape<"((1,2),(1,2),3,4,5)"> + ``` + }]; + let arguments = (ins + Cute_ValueTypeInterfaceInstance:$input, + Cute_ValueTypeInterfaceInstance:$element, + SI32Attr:$rank + ); + let results = (outs + Cute_ValueTypeInterfaceInstance:$result + ); +} +def Cute_PrintOp : Cute_Op<"print"> { + let summary = "Debug operation to print out value of cute data type."; + let description = [{ + Supported following cute data types + * shape, stride, layout, tile + * View: memref, coord_tensor + * Iterator: ptr, tuple + + Examples: + + ```mlir + cute.print(%0) : !cute.shape<"(3,5)"> + + cute.print(%1) : !cute.layout<"(5,(3,?)):(?,(4,5))"> + + // variadic operands without format string + cute.print(%0, %1) : !cute.shape<"(3,5)">, !cute.layout<"(5,(3,?)):(?,(4,5))"> + + // with format string + cute.print("{} -> {}", %0, %1) : !cute.shape<"(3,5)">, !cute.layout<"(5,(3,?)):(?,(4,5))"> + ``` + + The optional `operand_signed` attribute carries the DSL-level signedness of + each operand, one entry per value. It exists because DSL integers lower to + signless MLIR integer types, so an unsigned value is indistinguishable from + a signed one by type alone. When an entry is false, a generic `{}` + placeholder for that integer operand is rendered with an unsigned specifier + (`%u`/`%llu`) instead of `%d`/`%lld`. When absent, all operands are treated + as signed, and an explicit `%u` in the format string still forces unsigned. + }]; + let arguments = (ins + Variadic:$values, + OptionalAttr:$fmt, + OptionalAttr:$operand_signed + ); + let results = (outs + ); +} +def Cute_PrintTMADescIm2ColOp : Cute_Op<"print_tma_desc_im2col"> { + let summary = "This operation interprets the input i64 as a pointer to a TmaDescriptorInternal."; + let description = [{ + Example: + ```mlir + %tma_desc_a_host = cute_nvgpu.make_tma_desc_im2col( + %tensor, + %slayout, + %cta_v_tile, + %lower_corner_whd, + %upper_corner_whd, + %lower_padding_whd, + %upper_padding_whd, + %stride_whd, + %lower_srt, + %stride_srt) + <{host = true}> : ( + !cute.memref, "((8,8,1,4),64,1):((64,512,4096,4096),1,0)">, + !cute.composed_layout<"S<2,4,3> o 0 o ((8,16),(32,1)):((32,256),(1,0))">, + !cute.layout<"(128,32):(1@0,1@1)">, + !cute.int_tuple<"(0,0,0)">, + !cute.int_tuple<"(0,0,0)">, + !cute.int_tuple<"(0,0,0)">, + !cute.int_tuple<"(0,0,0)">, + !cute.int_tuple<"(1,1,1)">, + !cute.int_tuple<"(0,0,0)">, + !cute.int_tuple<"(1,1,1)">) -> !cute.ptr> + %tma_desc_a_host_ptr_i64 = cute_nvgpu.cast_tma_desc_to_integer(%tma_desc_a_host: !cute.ptr) -> i64 + %fd = arith.constant 1 : i32 // stdout + cute.print_tma_desc_im2col(%tma_desc_a_host_ptr_i64, %fd) + }]; + let arguments = (ins + I64:$input, + I32:$fd + ); + let results = (outs + ); +} +def Cute_PrintTMADescTiledOp : Cute_Op<"print_tma_desc_tiled"> { + let summary = "This operation interprets the input i64 as a pointer to a TmaDescriptorInternal."; + let description = [{ + Example: + ```mlir + %tma_desc_a_host = cute_nvgpu.make_tma_desc_tiled(%a_tensor, %smem_ablayout_128_64_k_major, %cta_v_tile_128_64_k_major) + <{mode = host = true}> : + ( + !cute.memref, "(?,?,?) : (?,1,?)">, + !cute.composed_layout<"S<3, 4, 3> o 0 o ((128,64),1,1):((64,1),0,0)">, + !cute.layout<"((128,64),1,1):((1@0,1@1),0,0)"> + ) -> !cute.ptr + %tma_desc_a_host_ptr_i64 = cute_nvgpu.cast_tma_desc_to_integer(%tma_desc_a_host: !cute.ptr) -> i64 + %fd = arith.constant 1 : i32 // stdout + cute.print_tma_desc_tiled(%tma_desc_a_host_ptr_i64, %fd) + ``` + }]; + let arguments = (ins + I64:$input, + I32:$fd + ); + let results = (outs + ); +} +def Cute_PrintViewOp : Cute_Op<"print_view"> { + let summary = "Debug operation to print cute view types."; + let description = [{ + Currently supports cute.memref and cute.tensor. MemRefs with both static layout + and dynamic layout are accepted. An optional coord parameter can be used to + specify the element to be printed. If no coord is provided, all elements will + be printed. + + The `verbose` attribute controls the output format: + - When `verbose` is true (default), it prints the full details of the view + including memory address, shape, and strides. + - When `verbose` is false, it prints in a more compact torch-style matrix format. + + The `signed` attribute is used for integer data types to control whether values + should be interpreted as signed or unsigned. This attribute is ignored for + floating point types and coord tensors. + + Examples: + + ```mlir + cute.print_view(%0) : !cute.memref + output: + Memref raw_ptr(0x0000152397fffc80: f32, rmem, align<4>) o (2,2):(2,1) = ( + (0,0)=1.000000 + (0,1)=2.000000 + (1,0)=3.000000 + (1,1)=4.000000 + ) + + cute.print_view(%1) : !cute.memref + output: + Memref raw_ptr(0x0000148eabfffbf0: f32, rmem, align<4>) o (2,(2,2)):(4,(2,1)) = ( + (0,(0,0))=1.000000 + (0,(0,1))=2.000000 + (0,(1,0))=3.000000 + (0,(1,1))=4.000000 + (1,(0,0))=5.000000 + (1,(0,1))=6.000000 + (1,(1,0))=7.000000 + (1,(1,1))=8.000000 + ) + + cute.print_view(%3,%coord) : !cute.memref,!cute.coord<"(?,(?,?))"> + output: + Memref raw_ptr(0x0000150ce3fffc90: f32, rmem, align<4>) o (2,(2,2)):(4,(2,1)), coord (1,(1,0)) = (7.000000) + + // Using verbose=false to print in torch-style format + cute.print_view(%0) {verbose = false} : !cute.memref + + output: + tensor(raw_ptr(0x000000003746a2e0: f64, generic, align<8>) o (8,5,3):(15,3,1), data= + [[[ 1.426091, -0.279034, -1.635684, -0.026360, -0.732545, ], + [-0.733555, 0.886134, -1.418841, -0.203192, -1.836270, ], + ... + [-1.172012, 1.678573, -0.004324, 0.441605, 1.484385, ], + [-0.550848, 1.455505, -0.968547, 1.233722, 0.812084, ]], + + [[ 0.762814, -0.641657, 0.843695, -1.380960, 0.261200, ], + [ 1.500137, -0.921287, 1.001725, 0.106540, -1.326231, ], + ... + [-0.878001, -0.465694, -1.204548, -0.407944, -0.092967, ], + [ 1.035407, 0.043012, -1.959517, 0.075913, 0.893392, ]], + + [[-0.129989, 0.750885, 0.219408, 0.442481, -0.508588, ], + [ 0.791268, -0.547978, 0.194177, 0.652729, 2.642380, ], + ... + [-0.674993, -1.017359, -1.272791, 0.829529, 0.852879, ], + [-0.256070, -0.287006, 1.900199, -0.492318, -0.658134, ]]]) + + ``` + }]; + let arguments = (ins + Cute_ViewTypeInterfaceInstance:$src, + Optional:$coord, + DefaultValuedAttr:$verbose, + DefaultValuedAttr:$is_signed + ); + let results = (outs + ); +} +def Cute_PtrLoadOp : Cute_InferredOp<"ptr.load", [MemoryEffects<[MemRead]>]> { + let summary = "Loads an LLVM or CuTe value from the given `!cute.ptr`."; + let arguments = (ins + Cute_CuTePointerType:$ptr + ); + let results = (outs + Cute_CuTePointerTypeOrAMemoryReferenceType:$value + ); +} +def Cute_PtrStoreOp : Cute_Op<"ptr.store"> { + let description = [{ + Stores the given CuTe value at the provided pointer. Handles any + LLVM-storable type, including those that can be stored with + `llvm.store` *and* CuTe types. + }]; + let arguments = (ins + Cute_CuTePointerTypeOrAMemoryReferenceType:$value, + Cute_CuTePointerType:$ptr + ); + let results = (outs + ); +} +def Cute_PtrToIntOp : Cute_Op<"ptrtoint", [Pure]> { + let description = [{ + The `cute.ptrtoint` operation accepts a cute pointer value and casts it to an + integer value. The pointer must have byte alignment, meaning subbyte types + must have explicit alignment specified (e.g., align<1> or higher). + + Example: + + ```mlir + %int_aligned = cute.ptrtoint(%ptr) : !cute.ptr> to i64 + ``` + }]; + let arguments = (ins + Cute_PointerTypeInterfaceInstance:$src + ); + let results = (outs + Cute_Integer:$dst + ); +} +def Cute_RakedProductOp : Cute_InferredOp<"raked_product", [Pure]> { + let description = [{ + This operation performs "raked_product" on the input and produce a raked_product result, + and it automatically infer the "raked_product" layout for the result type. + + Example: + + ```mlir + %input = cute.make_layout() : !cute.layout<"(3,4):(4,1)"> + %tiler = cute.make_layout() : !cute.layout<"(2,5):(1,2)"> + %prod = cute.raked_product(%arg0, %arg1) + : (!cute.layout<"(3,4):(4,1)">, !cute.layout<"(2,5):(1,2)">) + -> !cute.layout<"((2,3),(5,4)):((12,4),(24,1))"> + ``` + + Example with dynamic composed_layout: + + ``` + %prod = cute.raked_product(%input, %tiler) + : (!cute.composed_layout<"(?):(?) o ? o (5,4):(4,1)">, + !cute.layout<"(?,?):(?,?)">) + -> !cute.composed_layout<"(?):(?) o ? o ((?,5),(?,4)):((?{div=20},4),(?{div=20},1))"> + ``` + }]; + let arguments = (ins + Cute_GeneralizedLayoutTypeOfCuTe:$input, + Cute_CuTeLayoutType:$tiler + ); + let results = (outs + Cute_GeneralizedLayoutTypeOfCuTe:$result + ); +} +def Cute_RandomDelayOp : Cute_Op<"random_delay"> { + let arguments = (ins + ); + let results = (outs + ); +} +def Cute_RecastIterOp : Cute_Op<"recast_iter", [Pure]> { + let description = [{ + This `cute.recast_iter` is similar to CuTe C++'s recast_ptr, convert two iterators + in certain aspects like type/swizzle for convenience. + But this operation has some usage limitation for correctness, + like it's not allowed for TMA tensor's iterator and it's not allowed to convert align attribute. + In general, it's not allowed to convert two different iterator kinds (PtrType <-> IntTupleType) + + Example: + + ```mlir + %ptr = cute.recast_iter(%0) : !cute.ptr, S<3,4,3>> + to !cute.ptr, S<2,4,3>> + ``` + }]; + let arguments = (ins + Cute_CuTePointerType:$src + ); + let results = (outs + Cute_CuTePointerType:$dst + ); +} +def Cute_RecastLayoutOp : Cute_InferredOp<"recast_layout", [Pure]> { + let summary = "Example:"; + let description = [{ + ```mlir + # upcast<8> operation + %0 = cute.recast_layout<32, 4>(%arg0) + : !cute.layout<"(128,64):(1,128)"> to !cute.layout<"(16,64):(1,16)"> + # upcast<2> for composed layout + %0 = cute.recast_layout<32, 16>(%arg0) + : !cute.composed_layout<"((6,7),8):((10,12),1) o 6 o (8,2):(1,4)"> + to !cute.composed_layout<"((6,7),4):((5,6),1) o 3 o (4,2):(1,2)"> + # downcast<8> operation + %0 = cute.recast_layout<8, 32>(%arg0) + : !!cute.layout<"(16,64):(1,16)"> to cute.layout<"(128,64):(1,128)"> + ``` + }]; + let arguments = (ins + Cute_GeneralizedLayoutTypeOfCuTe:$src, + I32Attr:$new_type_bits, + I32Attr:$old_type_bits + ); + let results = (outs + Cute_GeneralizedLayoutTypeOfCuTe:$dst + ); +} +def Cute_ReduceOp : Cute_InferredOp<"reduce", [Pure]> { + let description = [{ + The `cute.reduce` operation reduces elements of a vector according to a given + shape and reduction operator, optionally guided by a reduction profile and/or a + predication mask. + + The reduction is performed over the dimensions of the input vector as described + by the provided shape. By default, all dimensions are reduced to a scalar. If a + reduction profile is specified, it determines which dimensions are kept (marked + with '_') and which are reduced (marked with an integer or tuple structure). The + result is either a scalar or a lower-rank vector, along with the reduced shape + if applicable. + + An optional predication vector (of the same size as the input vector) can be + provided to mask out elements from the reduction, enabling conditional + reduction. + + The reduction operator (e.g., MIN, MAX, ADD, MUL) is specified via the `op` + attribute. + + Examples: + ```mlir + // Reduce all elements to a scalar (minimum) + %reduced = cute.reduce(%input, %shape, %init) + { op = #cute.reduction_op } + : (vector<20xf32>, !cute.shape<"(4,5)">, f32) -> f32 + + // Reduce all elements to a scalar (maximum) with predication + %reduced = cute.reduce(%input, %shape, %init, %pred) + { op = #cute.reduction_op } + : (vector<20xf32>, !cute.shape<"(4,5)">, f32, vector<20xi1>) -> f32 + + // Reduce all but the second dimension (profile keeps dimension 1) + %reduced, %red_shp = cute.reduce(%input, %shape, %init) + { op = #cute.reduction_op, profile = #cute.coord<"(1,_,1)"> } + : (vector<60xf32>, !cute.shape<"(5,4,3)">, f32) -> (vector<4xf32>, !cute.shape<"(4)>") + + // Reduce a nested dimension (profile keeps outer and first inner dimension) + %reduced, %red_shp = cute.reduce(%input, %shape, %init) + { op = #cute.reduction_op, profile = #cute.coord<"(_,(_,1))"> } + : (vector<30xf32>, !cute.shape<"(5,(3,2))">, f32) -> (vector<15xf32>, !cute.shape<"(5,3)>") + ``` + The operation supports both flat and nested shapes and profiles, enabling reductions over complex multidimensional data layouts. + }]; + let arguments = (ins + Cute_OfRanks1:$input, + AnyType:$shape, + AnyType:$init_val, + Optional:$predication, + Cute_ReductionOpAttr:$reduction_op, + OptionalAttr:$reduction_profile + ); + let results = (outs + Cute_OfRanks1OrFloatingPointOrInteger:$output, + Optional:$reduced_shape + ); +} +def Cute_RightInverseOp : Cute_InferredOp<"right_inverse", [Pure]> { + let description = [{ + This operation builds the right inverse of a layout. + The current implementation only supports static layout. + An error is returned when the layout contains dynamic values. + + Example: + + ```mlir + %layout = cute.make_layout(.., ..): !cute.layout<"(8, 4):(4, 1)"> + %inv_layout = cute.right_inverse %layout : (!cute.layout<"(8, 4):(4, 1)">) -> !cute.layout<"(4, 8):(8, 1)"> + ``` + }]; + let arguments = (ins + Cute_GeneralizedLayoutTypeOfCuTe:$input + ); + let results = (outs + Cute_GeneralizedLayoutTypeOfCuTe:$result + ); +} +def Cute_SelectOp : Cute_InferredOp<"select", [Pure]> { + let description = [{ + This operation extracts specific dimensions out of a type implementing + the `CuteValueTypeInterface`, e.g. a shape, a coordinate, a tuple, a layout, ... + + The `mode` attribute indices are used to select the portion of the input type. + The indices in the `mode` attribute can be permutated or repeated, but should + be less than the rank of type. + + Example: + + ```mlir + %shape = cute.make_shape() : () -> !cute.shape<"(1, (2, 4), 8)"> + %mode0 = cute.select<[1]>(%shape) + : (!cute.shape<"(1,(2,4),8)">) -> !cute.shape<"((2, 4))"> + %mode1 = cute.select<[0,1]>(%shape) + : (!cute.shape<"(1,(2,4),8)">) -> !cute.shape<"(1, (2,4))"> + %mode2 = cute.select<[2,0]>(%shape) + : (!cute.shape<"(1,(2,4),8)">) -> !cute.shape<"(8, 1)"> + %mode3 = cute.select<[2,2]>(%shape) + : (!cute.shape<"(1,(2,4),8)">) -> !cute.shape<"(8, 8)"> + ``` + }]; + let arguments = (ins + Cute_ValueTypeInterfaceInstance:$input, + DenseI32ArrayAttr:$mode + ); + let results = (outs + Cute_ValueTypeInterfaceInstance:$result + ); +} +def Cute_ShapeDivOp : Cute_InferredOp<"shape_div", [Pure]> { + let description = [{ + Division for Shapes. + Case Tuple Tuple: + Perform shape_div element-wise + Case Tuple Int: + Fold the division of b across each element of a + Example: shape_div((4,5,6),40) -> shape_div((1,5,6),10) -> shape_div((1,1,6),2) -> (1,1,3) + Case Int Tuple: + Return shape_div(a, product(b)) + Case Int Int: + Enforce the divisibility condition a % b == 0 || b % a == 0 when possible + Return a / b with rounding away from 0 (that is, 1 or -1 when a < b) + + ```mlir + %res = cute.shape_div(%a, %b) + : (!cute.int_tuple<"(4,5)">, !cute.int_tuple<"(40)">) -> !cute.int_tuple<"(1,1)"> + ``` + }]; + let arguments = (ins + Cute_CuteShapeType:$a, + Cute_CuteShapeType:$b + ); + let results = (outs + Cute_CuteShapeType:$result + ); +} +def Cute_SizeOp : Cute_InferredOp<"size", [Pure]> { + let summary = "This operation computes the size of a cute value type by multiplying its dimensions."; + let description = [{ + The `mode` attribute specifies the nested mode in the type for size calculation. + It indexes a sub-scope within the input type and returns the size of that sub-scope. + The rank of `mode` must be less than or equal to the depth of the type. + If `mode` is empty, the operation calculates the size of the entire type. + + ### Examples + + ```mlir + %shape = cute.make_shape() : () -> !cute.shape<"(1,(2,4),9)"> + %size = cute.size(%shape) <{mode=[1]}> : (!cute.shape<"(1,(2,4),9)">) -> !cute.int_tuple<"8"> + ``` + + In this example, the mode attribute is set to [1], which selects the second dimension of the + shape (1,(2,4),9). The size of this dimension is calculated as 2 * 4 = 8. + + ```mlir + %shape = cute.make_shape() : () -> !cute.shape<"(1,(2,4),8)"> + %size = cute.size(%shape) : (!cute.shape<"(1,(2,4),8)">) -> !cute.int_tuple<"64"> + ``` + + In this example, the mode attribute is empty, so the operation calculates the size of the entire + shape (1,(2,4),8). The size is calculated as 1 * 2 * 4 * 8 = 64. + }]; + let arguments = (ins + Cute_CuteShapeTypeOrIntTupleAsIntegerOr2:$input, + DefaultValuedAttr:$mode + ); + let results = (outs + Cute_IntTupleAsIntegerOrTupleOfIntTupleS:$result + ); +} +def Cute_SliceOp : Cute_InferredOp<"slice", [Pure]> { + let description = [{ + This operation takes slices of a input memref according to the `coord`. Note that + `!coord<"(_,_,k)">` means "create a slice that views (i, j, k) for all valid i, + all valid j, and a specific value of k. + + The `coord` elements can be static, dynamic (represented by "?"), or underscore + (similar to the ":" (the colon punctuation) in Python or Fortran array slices). + However, the mode of `coord` should be less than or equal to the mode of `input` + memref layout; otherwise, a runtime error will occur. + + Example: + + ```mlir + %slice = cute.slice(%alloc, %coord) + : !cute.memref, !cute.coord<"(0, (_, _), 0)"> + ``` + + Example with dynamic coord: + + ```mlir + %slice = cute.slice(%alloc, %coord) + : !cute.memref, !cute.coord<"(_,(64,?),0)"> + ``` + + Example with scaled basis in layout: + + ```mlir + %slice = cute.slice(%coord_tensor, %coord) + : !cute.coord_tensor<"(0,0)", "(10,20,30):(1@0,1@1,20@1)">, !cute.coord<"(0,2,_)"> + ``` + Example with swizzled layout: + + ```mlir + %slice = cute.slice(%swizzle_layout, %coord) + : (!cute.composed_layout<"S<1,5,1> o 0 o ((2,4),(8,4)):((1,16),(2,64))">, + !cute.coord<"((?,?),(_,_))">) -> !cute.composed_layout<"(8,(2,2)):(2,(?,128))"> + ``` + + Example with PDSL memref slice (swizzle in units of bytes): + + ```mlir + %res_memref = !cute.slice(%memref, %crd) : !cute.memref, "(64,32):(32,1)">, !cute.coord + ``` + + Example with PISL memref slice (swizzle in units of elements): + + ```mlir + %res_memref = !cute.slice(%memref, %crd) : !cute.memref o 0 o (64,32):(32,1)">, !cute.coord + ``` + }]; + let arguments = (ins + Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$input, + Cute_CuteCoordType:$coord + ); + let results = (outs + Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$result + ); +} +def Cute_SmemPartitionSizeOp : Cute_Op<"smem_partition_size", [Pure]> { + let description = [{ + This operation queries the static smem layout for one partition in the + current kernel and returns the partition size as an i64 integer. It is only + valid on the kernel side; host-side total smem queries should use + `cute.kernel_smem_size`. + }]; + let arguments = (ins + I32Attr:$partition_id, + UnitAttr:$cumulative + ); + let results = (outs + I64:$size + ); +} +def Cute_StaticOp : Cute_Op<"static", [Pure]> { + let description = [{ + Creates a compile-time constant value with a fully static CuTe type. This operation takes no + operands and produces a single result of one of the following CuTe types: + + - !cute.int_tuple: A tuple of static integers + - !cute.coord: A coordinate with static dimensions + - !cute.stride: A stride with static values + - !cute.shape: A shape with static dimensions + - !cute.tile: A tile with static dimensions and layout + - !cute.layout: A layout with static shape and strides + - !cute.composed_layout: A composed layout with static inner/outer layouts + + The type of the result must be fully static (contain no dynamic/unknown values). + + Some atom types can be fully static with no runtime states, e.g. `UniversalFma`. + + ### Examples: + + ```mlir + // Create a static shape with value 5 + %0 = cute.static : !cute.shape<"5"> + + // Create a static 2D layout with shape (32,64) and strides (64,1) + %1 = cute.static : !cute.layout<"(32,64):(64,1)"> + ``` + }]; + let arguments = (ins + ); + let results = (outs + Cute_MaybeStaticTypeInterfaceInstance:$result + ); +} +def Cute_StencilDivideOp : Cute_InferredOp<"stencil_divide", [AttrSizedOperandSegments, Pure]> { + let summary = "A convolution has the following input tensors:"; + let description = [{ + * Activation tensor (Act): `((N,(D,H,W,...)), (C,(1,1,1)))` + * Filter tensor (Flt): `(K,(C,(T,R,S, ...)))` + * Output tensor (Out): `((N,(Z,P,Q, ...)), K)` + + Where + + * `N` := number of samples in the signal + * `D`, `H`, `W`, ... := spatial dimensions of the activation tensor + * `C` := channel dimension of the activation tensor + * `K` := channel dimension of the filter and output tensor + * `T`,`R`,`S`,... := spoke dimensions of the filter tensor + * `Z`,`P`,`Q`,... := spatial dimensions of the output tensor + + The number of spatial dimensions must DHW... must match both TRS... and ZPQ... + + As is evident in the tensor shapes, these cannot be issued to a GEMM without transformation, since + there are no logical M, N, and K modes into which we can group the tensor modes. + + Every spoke of the filter tensor (TRS) will be applied to some (offset) + view of the activation tensor, thus expanding the logical size of the activation tensor. + Additionally, a similar logical transform of the spatial dimensions can be encoded as + a function of the padding, dilations, traversal strides, and filter spokes. + This gets us to our im2col transform: + + Im2col transform affects the component shapes/strides of the activation tensor in the following way: + + * `ZPQ` Shape : changes `DHW` domain with formula `(1 + (DHW + pad - (((TRS-1) * dilation) + 1)) / traversal_stride)` + * `TRS` Shape : `TRS` domain instead of `(1,1,1)` + * `ZPQ` Strides : Original `DHW` strides get scaled by traversal strides `DHW` + * `TRS` Strides : Original `DHW` strides get scaled by dilation `DHW`. + + With this transform applied, we end up with a set of input and output tensors that are logically + consistent in their `MNK` dimensions, thus allowing us to dispatch to a GEMM. + + * im2col activation layout: `((N,(Z,P,Q)), (C,(T,R,S)))` // logical (M,K) + * filter layout : `( K, (C,(T,R,S)))` // logical (N,K) + * output layout : `((N,(Z,P,Q)), K )` // logical (M,N) + + `stencil_divide` performs this operation given these parameters. + }]; + let arguments = (ins + Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$input, + Cute_CuteTileType:$stencil, + Optional:$padding_upper, + Optional:$padding_lower, + Optional:$traversal_stride + ); + let results = (outs + Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$result + ); +} +def Cute_SymbolicOp : Cute_Op<"symbolic", [Pure]> { + let description = [{ + A symbol in CuTe IR represent an abstracted value which could be materialized + later in a pass. If it's not materialized, the behavior is undefined. + + Example: + + ```mlir + %0 = cute.symbolic("tile_m") : i32 + %1 = cute.symbolic("cluster_size") : !cute.shape<"(?,2)"> + ``` + }]; + let arguments = (ins + StrAttr:$name + ); + let results = (outs + AnyType:$symbolic + ); +} +def Cute_TileToShapeOp : Cute_InferredOp<"tile_to_shape", [Pure]> { + let description = [{ + The `tile_to_shape` operation is similar to blocked_product, but specifies the + result shape instead of the product shape, which is more convenient in certain + circumstances. + + The op has three operands: + 1. `block`: The layout to repeat. + 2. `trg_shape`: The target shape of the result + 3. `ord_shape`: The order of the modes of `trg_shape` to tile layout `block` with. + E.g. (_2,_1,_3) will cause @a layout to repeat across the second mode first, + the first mode second, and the third mode last. Currently only supports static order. + + Example: + + ```mlir + %0 = cute.tile_to_shape(%block, %trg_shape, %ord_shape) + : (!cute.layout<"(128,128):(128,1)">, + !cute.shape<"(1024,512)">, + !cute.shape<"(2,1)">) + -> !cute.layout<"((128,8),(128,4)):((128,65536),(1,16384))"> + + %0 = cute.tile_to_shape(%block, %trg_shape, %ord_shape) + : (!cute.layout<"((32,4),(16,4)):((16,4),(0,1))">, + !cute.shape<"(256,512,2)">, + !cute.shape<"(1,2,3)">) + -> !cute.layout<"((32,4,2),(16,4,8),2):((16,4,512),(0,1,1024),8192)"> + + %0 = cute.tile_to_shape(%block, %trg_shape, %ord_shape) + : (!cute.layout<"((32,4),(16,4)):((16,4),(0,1))">, + !cute.shape<"(256,512,2)">, + !cute.shape<"(2,1,3)">) + -> !cute.layout<"((32,4,2),(16,4,8),2):((16,4,4096),(0,1,512),8192)"> + + %0 = cute.tile_to_shape(%block, %trg_shape, %ord_shape) + : (!cute.composed_layout<"S<3,4,3> o 0 o ((32,4),(?,4)):((16,4),(0,1))">, + !cute.shape<"(256,?,2)">, + !cute.shape<"(3,2,1)">) + -> !cute.composed_layout<"S<3,4,3> o 0 o ((32,4,2),(?,4,?),2):((16,4,?{div=1024}),(0,1,1024),512)"> + ``` + }]; + let arguments = (ins + Cute_GeneralizedLayoutTypeOfCuTe:$block, + Cute_CuteShapeType:$trg_shape, + Cute_IntTupleAsIntegerOrTupleOfIntTupleS:$ord_shape + ); + let results = (outs + Cute_GeneralizedLayoutTypeOfCuTe:$result + ); +} +def Cute_TiledCopyPartitionDOp : Cute_InferredOp<"tiled.copy.partition_D", [Pure]> { + let description = [{ + The `cute.tiled.copy.partition_X` operation accepts: + 1. cute.tiled_copy; + 2. A ViewTypeInterface; + 3. cute.coord. + returns a new ViewTypeInterface type (MemrefType/CoordTensorType) with partitioned layout + and sliced offset (sliced offset will be implemented in the lowering). + + The new partitioned layout has the following meaning: ((vector_mode), rest_m, rest_n). vector_mode + represents the number of instructions issued and the number of elements processed per + instruction; in our example below, we have (2, 1) : (1, 0), meaning that each + instruction is processing two data. Rest_m is the number of iterations + along the first mode, a single iteration in our example. Finally, rest_n is the + number of iterations along the second mode, 128. + + Example: + + ```mlir + %dst_partitioned = cute.tiled.copy.partition_D(...) + : (!cute.tiled_copy, + layout_copy_tv = <"((32,4),(2,32)):((0,1),(128,4))">, + tiler_mn = <"[(4,32):(32,1);2:1]">>, + !cute.memref, "(128,256):(256,1)">, + !cute.coord<"?">) + -> !cute.memref, "((2,1),1,128):((1,0),0,2)"> + ``` + }]; + let arguments = (ins + Cute_TiledCopyAtomTypeOrTiledCopyV2:$tiled_copy, + Cute_ViewTypeInterfaceInstance:$input, + Cute_CuteCoordType:$coord + ); + let results = (outs + Cute_ViewTypeInterfaceInstance:$result + ); +} +def Cute_TiledCopyPartitionSOp : Cute_InferredOp<"tiled.copy.partition_S", [Pure]> { + let description = [{ + The `cute.tiled.copy.partition_X` operation accepts: + 1. cute.tiled_copy; + 2. A ViewTypeInterface; + 3. cute.coord. + returns a new ViewTypeInterface type (MemrefType/CoordTensorType) with partitioned layout + and sliced offset (sliced offset will be implemented in the lowering). + + The new partitioned layout has the following meaning: ((vector_mode), rest_m, rest_n). vector_mode + represents the number of instructions issued and the number of elements processed per + instruction; in our example below, we have (2, 1) : (1, 0), meaning that each + instruction is processing two data. Rest_m is the number of iterations + along the first mode, a single iteration in our example. Finally, rest_n is the + number of iterations along the second mode, 128. + + Example: + + ```mlir + %dst_partitioned = cute.tiled.copy.partition_D(...) + : (!cute.tiled_copy, + layout_copy_tv = <"((32,4),(2,32)):((0,1),(128,4))">, + tiler_mn = <"[(4,32):(32,1);2:1]">>, + !cute.memref, "(128,256):(256,1)">, + !cute.coord<"?">) + -> !cute.memref, "((2,1),1,128):((1,0),0,2)"> + ``` + }]; + let arguments = (ins + Cute_TiledCopyAtomTypeOrTiledCopyV2:$tiled_copy, + Cute_ViewTypeInterfaceInstance:$input, + Cute_CuteCoordType:$coord + ); + let results = (outs + Cute_ViewTypeInterfaceInstance:$result + ); +} +def Cute_TiledCopyRetileOp : Cute_InferredOp<"tiled.copy.retile", [Pure]> { + let description = [{ + The cute.tiled.copy.retile operation is used to reorganize the layout of a tensor + that has already been tiled or partitioned. This is typically necessary when + two TiledMma or TiledCopy instances are connected, accessing the same memory, + but require different logical views of that memory. + + ### Use Case + + Consider a scenario where a TiledMma instance is the consumer, and a TiledCopy + instance is the producer. The TiledMma instance partitions the memory, creating + a logical view that suits its needs. However, this view may not be compatible with + the TiledCopy instance. To resolve this, the cute.tiled.copy.retile operation is used + to retile the TiledCopy instance, creating a new logical view that is compatible with + the TiledMma instance. + + ``` + // consumer + partition_A(%tiled_mma, ...) + %mma_view = partition_A(%tiled_mma, %sa_tensor, %idx) + + // producer + %tiled_copy = make_tiled_copy_A(%copy_atom, %tiled_mma) + // retile %mma_view to meet needs of tiled_copy + %lds_view = retile(%tiled_copy, %mma_view) + ``` + + ### Operation + + The cute.tiled.copy.retile operation takes two inputs: + * A cute.tiled_copy instance, which represents the tensor to be retiled. + * A CuTeTypeWithLayout derived type, which represents the tensor that has already been tiled or partitioned. + + + Example: + + ```mlir + %retiled = cute.tiled.copy.retile(%tiled_copy, %src_tensor) + : (!cute.tiled_copy, num_matrices=4>, + layout_copy_tv = <"((4,8,4),((2,2,4),1)):((128,4,1),((64,32,512),0))">, + tiler_mn = <"[(4,16):(32,1);32:1]">>, + !cute.memref) + -> !cute.memref + ``` + }]; + let arguments = (ins + Cute_TiledCopyAtomTypeOrTiledCopyV2:$tiled_copy, + Cute_ViewTypeInterfaceInstance:$input + ); + let results = (outs + Cute_ViewTypeInterfaceInstance:$result + ); +} +def Cute_TiledDivideOp : Cute_InferredOp<"tiled_divide", [Pure]> { + let summary = "This operation performs a tiled_divide on a view or a layout by a tiler."; + let arguments = (ins + Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$input, + Cute_TileLikeTypesEGLayoutTypeShapeTypeTileType:$tiler + ); + let results = (outs + Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$result + ); +} +def Cute_TiledMmaPartitionOp : Cute_InferredOp<"tiled.mma.partition", [Pure]> { + let description = [{ + `cute.tiled.mma.partition` reorganizes a tensor's layout to match what the + MMA atom expects. The operation takes 4 arguments: + + 1. The type of operand. E.g., `CUTE_MMA_OPERAND_A` or `CUTE_MMA_OPERAND_B`. + 2. A `!cute.tiled_mma` value that specifies the kind of MMA. + 3. The unpartitioned operand tensor (ViewTypeInterface-derived SSA value). + 4. The `!cute.coord` value of the execution unit for which to partition the + the input tensor. + + The coordinate type depends on the MMA atom and the operand memory space. + Some examples: + * SM80: thread ID + * SM90: It depends... + - When partitioning A with "frag_kind = ss": warp-group ID + - When partitioning A with "frag_kind = rs": thread ID + - When partitioning B: warp-group ID + - When partitioning C/D: thread ID + * SM100: CTA ID within the cluster (always 0 for 1-CTA mode) + + The operation returns a new ViewTypeInterface-derived value + (CuteMemrefType/CuteCoordTensorType) with partitioned layout and sliced + offset (sliced offset will be implemented in the lowering). + + Example: + + ```mlir + // Partition an operand A tensor of shape 128x32 (residing in shared + // memory) for a !tiled_mma atom that operates on shape 32x16x16. (I.e., + // four SM80 MMA atoms with shape 16x8x16.) The result tensor is a view + // into the A tensor that represents the part that thread %tid_coord is + // operating on. + // + // The last two modes of the result tensor are 4,2. This is because the + // tiled atom is applied 4 times in M direction and 2 times in K direction. + // + // K=0:15 K=16:31 + // ┌────────────────────┬────────────────────┐ + // M=0:31 │ tiled atom (0,0) │ tiled atom (0,1) │ + // ├────────────────────┼────────────────────┤ + // M=32:63 │ tiled atom (1,0) │ tiled atom (1,1) │ + // ├────────────────────┼────────────────────┤ + // M=64:95 │ tiled atom (2,0) │ tiled atom (2,1) │ + // ├────────────────────┼────────────────────┤ + // M=96:127 │ tiled atom (3,0) │ tiled atom (3,1) │ + // └────────────────────┴────────────────────┘ + // + // For each tiled atom application of size 32x16x16, thread %tid_coord gets + // a subview of size 2,2,2 of operand A. (128 threads are cooperatively + // computing the !tiled_mma of shape 32x16x16, but an individual thread + // does not need access to the full 32x16 matrix A part.) + // + %partitioned = cute.tiled.mma.partition A(%0, %1, %tid_coord) + : (!cute.tiled_mma, + atom_layout_MNK = <"(2,2,1)">>, + !cute.memref, S<2,4,3>, "(128,32):(32,1)">, + !cute.coord<"?">) + -> !cute.memref, S<2,4,3>, "((2,2,2),4,2):((1,256,8),1024,16)"> + + // Partition an operand A tensor of shape 256x512 for an SM100 !tiled_mma. + // There are 256/128=2 tiled atom applications in M direction and + // 512/16=32 tiled atom applications in K direction. The atom specifies a + // single CTA, so that CTA get the entire A tensor. The coordinate is not + // really need in this case, but we pass it anyway to keep the API uniform. + %partitioned = cute.tiled.mma.partition A (%0, %1, %cta_coord) + : (!cute.tiled_mma, + atom_layout_MNK = <"(1,1,1):(0,0,0)">>, + !cute.memref, "(256,512):(512,1)">, + !cute.coord<"0">) + -> !cute.memref, "((128,16),2,32):((512,1),65536,16)"> + ``` + }]; + let arguments = (ins + Cute_TiledMMAAtomType:$tiled_mma, + Cute_ViewTypeInterfaceInstance:$input, + Cute_CuteCoordType:$coord, + Cute_MmaOperandEnum:$operand_id + ); + let results = (outs + Cute_ViewTypeInterfaceInstance:$result + ); +} +def Cute_TiledMmaPartitionShapeOp : Cute_InferredOp<"tiled.mma.partition_shape", [Pure]> { + let description = [{ + This operation partitions a shape into an MMA shape. For example, it takes a logical (M,K,...) + shape and tiles it into a shape like (MMA,MMA_M,MMA_K,...) where + - MMA is the MMA mode, for example (128,16) for an MMA instruction of shape 128x128x16 + - MMA_M is the number of times the MMA repeats in the M-mode + - MMA_K is the number of times the MMA repeats in the K-mode + - ... denotes the "rest" + }]; + let arguments = (ins + Cute_TiledMMAAtomType:$tiled_mma, + Cute_CuteShapeType:$input, + Cute_MmaOperandEnum:$operand_id + ); + let results = (outs + Cute_CuteShapeType:$result + ); +} +def Cute_TiledProductOp : Cute_InferredOp<"tiled_product", [Pure]> { + let arguments = (ins + Cute_GeneralizedLayoutTypeOfCuTe:$input, + Cute_CuTeLayoutType:$tiler + ); + let results = (outs + Cute_GeneralizedLayoutTypeOfCuTe:$result + ); +} +def Cute_ToIntTupleOp : Cute_InferredOp<"to_int_tuple", [Pure]> { + let description = [{ + The `cute.cast_to_int_tuple` operation casts from any x-tuple type (shape, stride, coord) to an int-tuple type. + The operation preserves the values but changes their type to int-tuple. + + cast will fail if input types contains modes not supported by int-tuple, e.g. _, scaled based, etc. + + Example: + + ```mlir + // Cast from shape to int_tuple + %0 = cute.to_int_tuple(%shape) : !cute.shape<"(8,4)"> to !cute.int_tuple<"(8,4)"> + + // Cast from stride to int_tuple + %1 = cute.to_int_tuple(%stride) : !cute.stride<"(8,4)"> to !cute.int_tuple<"(8,4)"> + + // Cast from coord to int_tuple + %2 = cute.to_int_tuple(%coord) : !cute.coord<"(8,4)"> to !cute.int_tuple<"(8,4)"> + + // Cast from stride to int_tuple preserving divisibility + %3 = cute.to_int_tuple(%stride) : !cute.stride<"(8,?{div=4})"> to !cute.int_tuple<"(8,?{div=4})"> + ``` + }]; + let arguments = (ins + Cute_XTupleLikeType:$src + ); + let results = (outs + Cute_IntTupleAsIntegerOrTupleOfIntTupleS:$dst + ); +} +def Cute_TupleAddOp : Cute_InferredOp<"tuple_add", [Pure]> { + let description = [{ + The `cute.tuple_add` operation performs element-wise addition between two CuTe x-tuple types. + It accepts two input operands and returns a result of the same kind, where each element in the result + is the sum of the corresponding elements in the input tuples. + + This operation applies subtraction recursively for nested tuples, matching corresponding elements + at each level of the tuple hierarchy. + + Example: + + ```mlir + // Element-wise add two arithmetic tuples + %result = cute.tuple_add(%a, %b) + : (!cute.int_tuple<"(?,(3,?))">, !cute.int_tuple<"(3,(1,?))">) -> !cute.int_tuple<"(?,(4,?))"> + + // Element-wise add with static values + %b = cute.static : !cute.int_tuple<"(3,(1,3))"> + %result = cute.tuple_add(%a, %b) : (!cute.int_tuple<"(?,(3,?))">) -> !cute.int_tuple<"(?,(4,?))"> + ``` + }]; + let arguments = (ins + Cute_ArithmeticTupleType:$lhs, + Cute_ArithmeticTupleType:$rhs + ); + let results = (outs + Cute_ArithmeticTupleType:$result + ); +} +def Cute_TupleDivOp : Cute_InferredOp<"tuple_div", [Pure]> { + let description = [{ + The `cute.tuple_div` operation performs element-wise division between two CuTe x-tuple types. + It accepts two input operands and returns a result of the same kind, where each element in the result + is the quotient of the corresponding elements in the input tuples. + + This operation applies division recursively for nested tuples, matching corresponding elements + at each level of the tuple hierarchy. + + Example: + + ```mlir + // Element-wise divide two arithmetic tuples + %result = cute.tuple_div(%a, %b) + : (!cute.int_tuple<"(?,(3,?))">, !cute.int_tuple<"(3,(1,?))">) -> !cute.int_tuple<"(?,(3,?))"> + + // Element-wise divide with static values + %b = cute.static : !cute.int_tuple<"(3,(1,3))"> + %result = cute.tuple_div(%a, %b) : (!cute.int_tuple<"(?,(3,?))">) -> !cute.int_tuple<"(?,(0,?))"> + ``` + }]; + let arguments = (ins + Cute_ArithmeticTupleType:$lhs, + Cute_ArithmeticTupleType:$rhs + ); + let results = (outs + Cute_ArithmeticTupleType:$result + ); +} +def Cute_TupleModOp : Cute_InferredOp<"tuple_mod", [Pure]> { + let description = [{ + The `cute.tuple_mod` operation computes the element-wise modulo (remainder) + between two CuTe x-tuple types. Both operands must be of the same tuple kind + (e.g., `!cute.int_tuple`), and the result is a tuple of the same structure + and type. + + For each corresponding element in the input tuples, the operation computes + the remainder of the division of the left-hand side element by the right-hand + side element. If the tuples are nested, the operation is applied recursively, + matching elements at each level of the tuple hierarchy. + + This operation supports both static and dynamic tuple elements. When static + values are provided, the modulo is computed at compile time; otherwise, it is + performed at runtime. + + **Examples:** + + ```mlir + // Element-wise modulo of two arithmetic tuples + %result = cute.tuple_mod(%a, %b) + : (!cute.int_tuple<"(?,(3,?))">, !cute.int_tuple<"(3,(1,?))">) + -> !cute.int_tuple<"(?,(0,?))"> + + // Element-wise modulo with static values + %b = cute.static : !cute.int_tuple<"(3,(1,3))"> + %result = cute.tuple_mod(%a, %b) + : (!cute.int_tuple<"(?,(3,?))">) -> !cute.int_tuple<"(?,(1,?))"> + ``` + + **Semantics:** + - The operation traverses the tuple structure recursively. + - For each pair of corresponding elements `(x, y)`, the result is `x % y`. + - The structure and rank of the result tuple matches that of the operands. + - If the tuple contains nested tuples, the modulo is applied to each + corresponding sub-tuple. + + This operation is useful for broadcasting, tiling, and other tensor algebra + applications where modular arithmetic over structured tuples is required. + }]; + let arguments = (ins + Cute_ArithmeticTupleType:$lhs, + Cute_ArithmeticTupleType:$rhs + ); + let results = (outs + Cute_ArithmeticTupleType:$result + ); +} +def Cute_TupleMulOp : Cute_InferredOp<"tuple_mul", [Pure]> { + let description = [{ + The `cute.tuple_mul` operation performs element-wise multiplication between two CuTe x-tuple types. + It accepts two input operands and returns a result of the same kind, where each element in the result + is the product of the corresponding elements in the input tuples. + + This operation applies multiplication recursively for nested tuples, matching corresponding elements + at each level of the tuple hierarchy. + + Example: + + ```mlir + // Element-wise multiply two arithmetic tuples + %result = cute.tuple_mul(%a, %b) + : (!cute.int_tuple<"(?,(3,?))">, !cute.int_tuple<"(3,(1,?))">) -> !cute.int_tuple<"(?,(3,?))"> + + // Element-wise multiply with static values + %b = cute.static : !cute.int_tuple<"(3,(1,3))"> + %result = cute.tuple_mul(%a, %b) : (!cute.int_tuple<"(?,(3,?))">) -> !cute.int_tuple<"(?,(3,?))"> + ``` + }]; + let arguments = (ins + Cute_ArithmeticTupleType:$lhs, + Cute_ArithmeticTupleType:$rhs + ); + let results = (outs + Cute_ArithmeticTupleType:$result + ); +} +def Cute_TupleProductEachOp : Cute_InferredOp<"tuple.product_each", [Pure]> { + let summary = "This operation performs \"product_each\" on each element in a tuple"; + let description = [{ + Example: + + ```mlir + %tuple = cute.make_int_tuple() : !cute.int_tuple<"(2,(5,4))"> + %prod = cute.tuple.product_each(%tuple) : (!cute.int_tuple<"(2,(5,4))">) -> !cute.int_tuple<"(2, 20)"> + ``` + }]; + let arguments = (ins + Cute_IntTupleAsIntegerOrTupleOfIntTupleS2:$input + ); + let results = (outs + Cute_IntTupleAsIntegerOrTupleOfIntTupleS2:$result + ); +} +def Cute_TupleProductOp : Cute_InferredOp<"tuple.product", [Pure]> { + let summary = "This operation performs \"product\" on all elements in a tuple"; + let description = [{ + Example: + + ```mlir + %tuple = cute.make_int_tuple() : !cute.int_tuple<"(2,(5,4))"> + %prod = cute.product(%tuple) : (!cute.int_tuple<"(2,(5,4))">) -> !cute.int_tuple<"40"> + ``` + }]; + let arguments = (ins + Cute_IntTupleAsIntegerOrTupleOfIntTupleS2:$input + ); + let results = (outs + Cute_IntTupleAsIntegerOrTupleOfIntTupleS2:$result + ); +} +def Cute_TupleSubOp : Cute_InferredOp<"tuple_sub", [Pure]> { + let description = [{ + The `cute.tuple_sub` operation performs element-wise subtraction between two CuTe x-tuple types. + It accepts two input operands and returns a result of the same kind, where each element in the result + is the difference of the corresponding elements in the input tuples. + + This operation applies subtraction recursively for nested tuples, matching corresponding elements + at each level of the tuple hierarchy. + + Example: + + ```mlir + // Element-wise subtract two arithmetic tuples + %result = cute.tuple_sub(%a, %b) + : (!cute.int_tuple<"(?,(3,?))">, !cute.int_tuple<"(3,(1,?))">) -> !cute.int_tuple<"(?,(2,?))"> + + // Element-wise subtract with static values + %b = cute.static : !cute.int_tuple<"(3,(1,3))"> + %result = cute.tuple_sub(%a, %b) : (!cute.int_tuple<"(?,(3,?))">) -> !cute.int_tuple<"(?,(2,?))"> + ``` + }]; + let arguments = (ins + Cute_ArithmeticTupleType:$lhs, + Cute_ArithmeticTupleType:$rhs + ); + let results = (outs + Cute_ArithmeticTupleType:$result + ); +} +def Cute_UnpackTupleOp : Cute_Op<"unpack_tuple", [Pure]> { + let description = [{ + Unpacks a tuple into its individual components. The number and types + of the components must match the number and types of the results. + }]; + let arguments = (ins + Cute_ATupleIsAFixedLengthOrderedList:$tuple + ); + let results = (outs + Variadic:$result + ); +} +def Cute_ZippedDivideOp : Cute_InferredOp<"zipped_divide", [Pure]> { + let description = [{ + This operation performs "zipped_divide" on the input and produce a zipped_divided result, + and it automatically infer the "zipped_divided" layout for the result type. + + Example: + + ```mlir + %alloc = cute.memref.alloc() : !cute.memref + %shape = cute.make_shape() : () -> !cute.shape<"(4,8)"> + %zdiv = cute.zipped_divide(%alloc, %shape) + : !cute.memref, !cute.shape<"(4,8)"> + ``` + }]; + let arguments = (ins + Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$input, + Cute_TileLikeTypesEGLayoutTypeShapeTypeTileType:$tiler + ); + let results = (outs + Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$result + ); +} +def Cute_ZippedProductOp : Cute_InferredOp<"zipped_product", [Pure]> { + let arguments = (ins + Cute_GeneralizedLayoutTypeOfCuTe:$input, + Cute_CuTeLayoutType:$tiler + ); + let results = (outs + Cute_GeneralizedLayoutTypeOfCuTe:$result + ); +} + +#endif // MLIR_DIALECTS_CUTE_IR_CUTEOPS_TD diff --git a/mlir/dialects/cute_ir/CuteTypes.cpp b/mlir/dialects/cute_ir/CuteTypes.cpp new file mode 100644 index 00000000..604dfbf2 --- /dev/null +++ b/mlir/dialects/cute_ir/CuteTypes.cpp @@ -0,0 +1,1602 @@ +//===- CuteTypes.cpp - parsers, printers and verifiers of cute types ------===// + +// clang-format off +/*************************************************************************************************** + * Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +// clang-format on + +#include "cute_ir/Dialect/Cute/IR/CuteDialect.h" +#include "mlir/dialects/cute_nvgpu/CuteNVGPUDialect.h" + +#include "mlir/Dialect/LLVMIR/LLVMTypes.h" +#include "mlir/IR/Builders.h" +#include "mlir/IR/DialectImplementation.h" +#include "mlir/IR/OpImplementation.h" +#include "llvm/ADT/TypeSwitch.h" +#include "llvm/Support/FormatVariadic.h" +#include "llvm/Support/MathExtras.h" + +#include + +using namespace mlir; +using namespace mlir::cutlass_compiler::cute; + +namespace cg = cutegen; + +// The compiler's parser of the algebra types reads `<"text">` leniently: a +// missing or unreadable payload, including a bare mnemonic, gives the default +// value (`()`, `():()`, `[]`, `S<0,4,3>`) instead of an error. +template static T parseLenientPayload(mlir::AsmParser &parser) { + std::string str; + if (succeeded(parser.parseOptionalLess()) && + succeeded(parser.parseOptionalString(&str)) && + succeeded(parser.parseOptionalGreater())) + return cutegen::from_string(str).value_or(T{}); + return T{}; +} + +// The compiler's integer type is `!cute.i`, a mnemonic per width that +// the generated dispatch cannot match. The generated CuteDialect::parseType +// and printType pass the dialect parser and printer, so these overloads of +// the renamed dispatch functions catch them first (defined below). +static OptionalParseResult generatedCuteTypeParser(DialectAsmParser &parser, + StringRef *mnemonic, + Type &value); +static LogicalResult generatedCuteTypePrinter(Type type, + DialectAsmPrinter &printer); +#define generatedTypeParser generatedCuteTypeParser +#define generatedTypePrinter generatedCuteTypePrinter + +// Emit the tablegen-generated type storage / parsing machinery (also drives +// CuteDialect::parseType / printType via useDefaultTypePrinterParser). +#define GET_TYPEDEF_CLASSES +#include "mlir/dialects/cute_ir/CuteTypes.cpp.inc" +#undef generatedTypeParser +#undef generatedTypePrinter + +//===----------------------------------------------------------------------===// +// CuteDialect — type registration +//===----------------------------------------------------------------------===// + +namespace { +namespace cute = ::mlir::cutlass_compiler::cute; +namespace nv = ::mlir::cutlass_compiler::cute_nvgpu; + +std::string printed(Type type) { + std::string text; + llvm::raw_string_ostream os(text); + type.print(os); + return text; +} + +// The compiler's name for a tiled copy's atom; empty when it has none. +std::string copyAliasName(Type atom) { + return llvm::TypeSwitch(atom) + .Case([](auto) { return "copy_simt"; }) + .Case([](auto) { return "copy_ldgsts"; }) + .Case([](auto) { return "copy_dsmem_store"; }) + .Case([](auto) { return "copy_blkcp_g2s"; }) + .Case([](auto) { return "copy_blkcp_s2g"; }) + .Case([](auto) { return "copy_blkcp_s2s"; }) + .Case([](auto) { return "copy_g2r"; }) + .Case([](auto) { return "copy_r2g"; }) + .Case([](auto) { return "copy_r2s"; }) + .Case([](auto) { return "copy_s2r"; }) + .Case([](auto) { return "copy_utccp"; }) + .Case( + [](auto t) { return "copy_ldtm_" + std::to_string(t.getNumBit()); }) + .Case( + [](auto t) { return "copy_sttm_" + std::to_string(t.getNumBit()); }) + .Case([](auto t) { + return "copy_ldtm_red_" + std::to_string(t.getNumBit()); + }) + .Case([](auto t) { + return "copy_ldtm_spcompress_" + std::to_string(t.getNumBit()); + }) + .Case([](auto t) { + return "copy_ldsm_" + std::to_string(t.getNumMatrices()); + }) + .Case([](auto t) { + return "copy_stsm_" + std::to_string(t.getNumMatrices()); + }) + .Case([](auto t) { + return "copy_tmaldg_" + std::to_string(t.getCopyBits() / 8) + "b"; + }) + .Case([](auto t) { + return "copy_tmastg_" + std::to_string(t.getCopyBits() / 8) + "b"; + }) + .Case([](auto t) { + return "copy_tmaredg_" + std::to_string(t.getCopyBits() / 8) + "b"; + }) + .Default([](Type) { return "copy_unknown_type"; }); +} + +// `mma____xx`; empty for a type that is not an MMA atom. +std::string mmaAliasName(Type atom) { + auto name = [](auto t) { + std::string shape; + auto mnk = cutegen::flatten(t.getShapeMnk().getRef()); + for (size_t i = 0; i < cutegen::rank(mnk); ++i) + shape += (i ? "x" : "") + cutegen::to_string(cutegen::get(mnk, i)); + return "mma_" + printed(t.getAType()) + "_" + printed(t.getBType()) + "_" + + printed(t.getCType()) + "_" + shape; + }; + return llvm::TypeSwitch(atom) + .Case(name) + .Default([](Type) { return std::string(); }); +} + +// The compiler's type aliases: `!memref__` for a memref, +// `!copy_` for a tiled copy and `!mma____xx` for a +// tiled MMA (MLIR escapes and numbers them). Other types have none. +struct CuteOpAsmInterface : public OpAsmDialectInterface { + using OpAsmDialectInterface::OpAsmDialectInterface; + AliasResult getAlias(Type type, raw_ostream &os) const override { + std::string name; + if (auto memref = llvm::dyn_cast(type)) + name = "memref_" + + stringifyAddressSpace(memref.getPtr().getAddressSpace()).str() + + "_" + printed(memref.getPtr().getValueType()); + else if (auto copy = llvm::dyn_cast(type)) + name = copyAliasName(copy.getCopyAtom()); + else if (auto mma = llvm::dyn_cast(type)) + name = mmaAliasName(mma.getMmaAtom()); + if (name.empty()) + return AliasResult::NoAlias; + os << name; + return AliasResult::OverridableAlias; + } +}; +} // namespace + +void CuteDialect::registerCuteTypes(CuteDialect *dialect) { + dialect->addTypes< +#define GET_TYPEDEF_LIST +#include "mlir/dialects/cute_ir/CuteTypes.cpp.inc" + >(); + dialect->addInterfaces(); +} + +//===----------------------------------------------------------------------===// +// IntTupleType — custom assembly format +// + +// Grammar: < " cutegen-int-tuple-string " > (read leniently, see above) +//===----------------------------------------------------------------------===// + +mlir::Type IntTupleType::parse(mlir::AsmParser &parser) { + return get( + parser.getContext(), + IntTupleAttr::get(parser.getContext(), + parseLenientPayload(parser))); +} + +void IntTupleType::print(mlir::AsmPrinter &printer) const { + printer << '<'; + printer.printString(cutegen::to_string(getRef())); + printer << '>'; +} + +bool IntTupleType::isStatic() const { return cutegen::is_static(getRef()); } + +mlir::Attribute IntTupleType::getValueAttr() const { return getAttr(); } + +llvm::TypeSize +IntTupleType::getTypeSize(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeSize(cg::get_llvm_type(builder, getRef())); +} + +llvm::TypeSize +IntTupleType::getTypeSizeInBits(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeSizeInBits(cg::get_llvm_type(builder, getRef())); +} + +uint64_t +IntTupleType::getABIAlignment(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeABIAlignment(cg::get_llvm_type(builder, getRef())); +} + +uint64_t +IntTupleType::getPreferredAlignment(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypePreferredAlignment( + cg::get_llvm_type(builder, getRef())); +} + +std::optional +IntTupleType::getIndexBitwidth(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeIndexBitwidth(cg::get_llvm_type(builder, getRef())); +} + +bool IntTupleType::areCompatible( + mlir::DataLayoutEntryListRef oldLayout, + mlir::DataLayoutEntryListRef newLayout, + mlir::DataLayoutSpecInterface newSpec, + const mlir::DataLayoutIdentifiedEntryMap &identified) const { + OpBuilder builder(getContext()); + // The lowering can be a bare integer (scalar/depth-0 dynamic leaves), which + // does not implement DataLayoutTypeInterface; only delegate when it does and + // otherwise fall back to the interface default ("compatible"). + if (auto dlTy = llvm::dyn_cast( + cg::get_llvm_type(builder, getRef()))) { + return dlTy.areCompatible(oldLayout, newLayout, newSpec, identified); + } + return true; +} + +llvm::LogicalResult +IntTupleType::verifyEntries(mlir::DataLayoutEntryListRef entries, + mlir::Location loc) const { + OpBuilder builder(getContext()); + // See areCompatible: bare-integer lowerings cannot be queried, so only + // delegate when the lowered type implements the interface. + if (auto dlTy = llvm::dyn_cast( + cg::get_llvm_type(builder, getRef()))) { + return dlTy.verifyEntries(entries, loc); + } + return mlir::success(); +} + +//===----------------------------------------------------------------------===// +// CoordType — custom assembly format +//===----------------------------------------------------------------------===// + +mlir::Type CoordType::parse(mlir::AsmParser &parser) { + return get(parser.getContext(), + CoordAttr::get(parser.getContext(), + parseLenientPayload(parser))); +} + +void CoordType::print(mlir::AsmPrinter &printer) const { + printer << '<'; + printer.printString(cutegen::to_string(getRef())); + printer << '>'; +} + +bool CoordType::isStatic() const { return cutegen::is_static(getRef()); } + +mlir::Attribute CoordType::getValueAttr() const { return getAttr(); } + +llvm::TypeSize +CoordType::getTypeSize(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeSize(cg::get_llvm_type(builder, getRef())); +} + +llvm::TypeSize +CoordType::getTypeSizeInBits(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeSizeInBits(cg::get_llvm_type(builder, getRef())); +} + +uint64_t CoordType::getABIAlignment(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeABIAlignment(cg::get_llvm_type(builder, getRef())); +} + +uint64_t +CoordType::getPreferredAlignment(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypePreferredAlignment( + cg::get_llvm_type(builder, getRef())); +} + +std::optional +CoordType::getIndexBitwidth(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeIndexBitwidth(cg::get_llvm_type(builder, getRef())); +} + +bool CoordType::areCompatible( + mlir::DataLayoutEntryListRef oldLayout, + mlir::DataLayoutEntryListRef newLayout, + mlir::DataLayoutSpecInterface newSpec, + const mlir::DataLayoutIdentifiedEntryMap &identified) const { + OpBuilder builder(getContext()); + if (auto dlTy = llvm::dyn_cast( + cg::get_llvm_type(builder, getRef()))) { + return dlTy.areCompatible(oldLayout, newLayout, newSpec, identified); + } + return true; +} + +llvm::LogicalResult +CoordType::verifyEntries(mlir::DataLayoutEntryListRef entries, + mlir::Location loc) const { + OpBuilder builder(getContext()); + if (auto dlTy = llvm::dyn_cast( + cg::get_llvm_type(builder, getRef()))) { + return dlTy.verifyEntries(entries, loc); + } + return mlir::success(); +} + +//===----------------------------------------------------------------------===// +// ShapeType — custom assembly format +//===----------------------------------------------------------------------===// + +mlir::Type ShapeType::parse(mlir::AsmParser &parser) { + return get(parser.getContext(), + ShapeAttr::get(parser.getContext(), + parseLenientPayload(parser))); +} + +void ShapeType::print(mlir::AsmPrinter &printer) const { + printer << '<'; + printer.printString(cutegen::to_string(getRef())); + printer << '>'; +} + +bool ShapeType::isStatic() const { return cutegen::is_static(getRef()); } + +mlir::Attribute ShapeType::getValueAttr() const { return getAttr(); } + +llvm::TypeSize +ShapeType::getTypeSize(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeSize(cg::get_llvm_type(builder, getRef())); +} + +llvm::TypeSize +ShapeType::getTypeSizeInBits(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeSizeInBits(cg::get_llvm_type(builder, getRef())); +} + +uint64_t ShapeType::getABIAlignment(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeABIAlignment(cg::get_llvm_type(builder, getRef())); +} + +uint64_t +ShapeType::getPreferredAlignment(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypePreferredAlignment( + cg::get_llvm_type(builder, getRef())); +} + +std::optional +ShapeType::getIndexBitwidth(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeIndexBitwidth(cg::get_llvm_type(builder, getRef())); +} + +bool ShapeType::areCompatible( + mlir::DataLayoutEntryListRef oldLayout, + mlir::DataLayoutEntryListRef newLayout, + mlir::DataLayoutSpecInterface newSpec, + const mlir::DataLayoutIdentifiedEntryMap &identified) const { + OpBuilder builder(getContext()); + if (auto dlTy = llvm::dyn_cast( + cg::get_llvm_type(builder, getRef()))) { + return dlTy.areCompatible(oldLayout, newLayout, newSpec, identified); + } + return true; +} + +llvm::LogicalResult +ShapeType::verifyEntries(mlir::DataLayoutEntryListRef entries, + mlir::Location loc) const { + OpBuilder builder(getContext()); + if (auto dlTy = llvm::dyn_cast( + cg::get_llvm_type(builder, getRef()))) { + return dlTy.verifyEntries(entries, loc); + } + return mlir::success(); +} + +//===----------------------------------------------------------------------===// +// StrideType — custom assembly format +//===----------------------------------------------------------------------===// + +mlir::Type StrideType::parse(mlir::AsmParser &parser) { + auto loc = parser.getCurrentLocation(); + auto attr = StrideAttr::get(parser.getContext(), + parseLenientPayload(parser)); + return getChecked([&] { return parser.emitError(loc); }, parser.getContext(), + attr); +} + +void StrideType::print(mlir::AsmPrinter &printer) const { + printer << '<'; + printer.printString(cutegen::to_string(getRef())); + printer << '>'; +} + +bool StrideType::isStatic() const { return cutegen::is_static(getRef()); } + +mlir::Attribute StrideType::getValueAttr() const { return getAttr(); } + +// The compiler accepts every stride cutegen reads, mixed integer and +// scaled-basis leaves and mixed-depth basis paths included. +mlir::LogicalResult +StrideType::verify(llvm::function_ref emitError, + StrideAttr attr) { + return mlir::success(); +} + +llvm::TypeSize +StrideType::getTypeSize(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeSize(cg::get_llvm_type(builder, getRef())); +} + +llvm::TypeSize +StrideType::getTypeSizeInBits(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeSizeInBits(cg::get_llvm_type(builder, getRef())); +} + +uint64_t +StrideType::getABIAlignment(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeABIAlignment(cg::get_llvm_type(builder, getRef())); +} + +uint64_t +StrideType::getPreferredAlignment(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypePreferredAlignment( + cg::get_llvm_type(builder, getRef())); +} + +std::optional +StrideType::getIndexBitwidth(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeIndexBitwidth(cg::get_llvm_type(builder, getRef())); +} + +bool StrideType::areCompatible( + mlir::DataLayoutEntryListRef oldLayout, + mlir::DataLayoutEntryListRef newLayout, + mlir::DataLayoutSpecInterface newSpec, + const mlir::DataLayoutIdentifiedEntryMap &identified) const { + OpBuilder builder(getContext()); + if (auto dlTy = llvm::dyn_cast( + cg::get_llvm_type(builder, getRef()))) { + return dlTy.areCompatible(oldLayout, newLayout, newSpec, identified); + } + return true; +} + +llvm::LogicalResult +StrideType::verifyEntries(mlir::DataLayoutEntryListRef entries, + mlir::Location loc) const { + OpBuilder builder(getContext()); + if (auto dlTy = llvm::dyn_cast( + cg::get_llvm_type(builder, getRef()))) { + return dlTy.verifyEntries(entries, loc); + } + return mlir::success(); +} + +//===----------------------------------------------------------------------===// +// LayoutType — custom assembly format +//===----------------------------------------------------------------------===// + +mlir::Type LayoutType::parse(mlir::AsmParser &parser) { + auto loc = parser.getCurrentLocation(); + auto attr = LayoutAttr::get(parser.getContext(), + parseLenientPayload(parser)); + return getChecked([&] { return parser.emitError(loc); }, parser.getContext(), + attr); +} + +void LayoutType::print(mlir::AsmPrinter &printer) const { + printer << '<'; + printer.printString(cutegen::to_string(getRef())); + printer << '>'; +} + +bool LayoutType::isStatic() const { return cutegen::is_static(getRef()); } + +mlir::Attribute LayoutType::getValueAttr() const { return getAttr(); } + +/// Verify LayoutType storage invariants. cutegen leaves these +/// unchecked at construction time for performance; enforce them at the +/// IR boundary. +mlir::LogicalResult +LayoutType::verify(llvm::function_ref emitError, + LayoutAttr attr) { + auto const &layout = attr.getRef(); + if (cutegen::has_error(layout)) { + return emitError() << "expects no error(`x`) in layout, but got \"" + << cutegen::to_string(layout) << "\""; + } + if (cutegen::any_leaf_is(layout.shape(), [](auto const &e) { + return cutegen::holds_int(e) && cutegen::get_int(e) <= 0; + })) { + return emitError() << "expects positive shape mode, but got \"" + << cutegen::to_string(layout) << "\""; + } + if (!cutegen::is_congruent(layout.shape(), layout.stride())) { + return emitError() + << "expects shape and stride profile to match, but got \"" + << cutegen::to_string(layout) << "\""; + } + return mlir::success(); +} + +llvm::TypeSize +LayoutType::getTypeSize(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeSize(cg::get_llvm_type(builder, getRef())); +} + +llvm::TypeSize +LayoutType::getTypeSizeInBits(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeSizeInBits(cg::get_llvm_type(builder, getRef())); +} + +uint64_t +LayoutType::getABIAlignment(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeABIAlignment(cg::get_llvm_type(builder, getRef())); +} + +uint64_t +LayoutType::getPreferredAlignment(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypePreferredAlignment( + cg::get_llvm_type(builder, getRef())); +} + +std::optional +LayoutType::getIndexBitwidth(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeIndexBitwidth(cg::get_llvm_type(builder, getRef())); +} + +bool LayoutType::areCompatible( + mlir::DataLayoutEntryListRef oldLayout, + mlir::DataLayoutEntryListRef newLayout, + mlir::DataLayoutSpecInterface newSpec, + const mlir::DataLayoutIdentifiedEntryMap &identified) const { + OpBuilder builder(getContext()); + if (auto dlTy = llvm::dyn_cast( + cg::get_llvm_type(builder, getRef()))) { + return dlTy.areCompatible(oldLayout, newLayout, newSpec, identified); + } + return true; +} + +llvm::LogicalResult +LayoutType::verifyEntries(mlir::DataLayoutEntryListRef entries, + mlir::Location loc) const { + OpBuilder builder(getContext()); + if (auto dlTy = llvm::dyn_cast( + cg::get_llvm_type(builder, getRef()))) { + return dlTy.verifyEntries(entries, loc); + } + return mlir::success(); +} + +//===----------------------------------------------------------------------===// +// TileType — custom assembly format +//===----------------------------------------------------------------------===// + +mlir::Type TileType::parse(mlir::AsmParser &parser) { + auto loc = parser.getCurrentLocation(); + auto attr = TileAttr::get(parser.getContext(), + parseLenientPayload(parser)); + return getChecked([&] { return parser.emitError(loc); }, parser.getContext(), + attr); +} + +void TileType::print(mlir::AsmPrinter &printer) const { + printer << '<'; + printer.printString(cutegen::to_string(getRef())); + printer << '>'; +} + +bool TileType::isStatic() const { return cutegen::is_static(getRef()); } + +mlir::Attribute TileType::getValueAttr() const { return getAttr(); } + +// Like strides, the layouts of a tile are not restricted further. +mlir::LogicalResult +TileType::verify(llvm::function_ref emitError, + TileAttr attr) { + return mlir::success(); +} + +llvm::TypeSize +TileType::getTypeSize(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeSize(cg::get_llvm_type(builder, getRef())); +} + +llvm::TypeSize +TileType::getTypeSizeInBits(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeSizeInBits(cg::get_llvm_type(builder, getRef())); +} + +uint64_t TileType::getABIAlignment(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeABIAlignment(cg::get_llvm_type(builder, getRef())); +} + +uint64_t +TileType::getPreferredAlignment(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypePreferredAlignment( + cg::get_llvm_type(builder, getRef())); +} + +std::optional +TileType::getIndexBitwidth(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeIndexBitwidth(cg::get_llvm_type(builder, getRef())); +} + +bool TileType::areCompatible( + mlir::DataLayoutEntryListRef oldLayout, + mlir::DataLayoutEntryListRef newLayout, + mlir::DataLayoutSpecInterface newSpec, + const mlir::DataLayoutIdentifiedEntryMap &identified) const { + OpBuilder builder(getContext()); + if (auto dlTy = llvm::dyn_cast( + cg::get_llvm_type(builder, getRef()))) { + return dlTy.areCompatible(oldLayout, newLayout, newSpec, identified); + } + return true; +} + +llvm::LogicalResult +TileType::verifyEntries(mlir::DataLayoutEntryListRef entries, + mlir::Location loc) const { + OpBuilder builder(getContext()); + if (auto dlTy = llvm::dyn_cast( + cg::get_llvm_type(builder, getRef()))) { + return dlTy.verifyEntries(entries, loc); + } + return mlir::success(); +} + +//===----------------------------------------------------------------------===// +// ComposedLayoutType — custom assembly format +//===----------------------------------------------------------------------===// + +mlir::Type ComposedLayoutType::parse(mlir::AsmParser &parser) { + // Capture the start location so verify() failures (positivity, + // congruence) are attributed to the type literal. + auto loc = parser.getCurrentLocation(); + if (parser.parseLess()) { + return {}; + } + std::string str; + if (parser.parseString(&str)) { + return {}; + } + auto opt = cutegen::from_string(str); + if (!opt) { + parser.emitError(parser.getCurrentLocation(), + "failed to parse composed_layout from \"" + str + "\""); + return {}; + } + if (parser.parseGreater()) { + return {}; + } + auto attr = ComposedLayoutAttr::get(parser.getContext(), std::move(*opt)); + return ComposedLayoutType::getChecked([&] { return parser.emitError(loc); }, + parser.getContext(), attr); +} + +void ComposedLayoutType::print(mlir::AsmPrinter &printer) const { + printer << '<'; + printer.printString(cutegen::to_string(getRef())); + printer << '>'; +} + +bool ComposedLayoutType::isStatic() const { + // No free cutegen::is_static(composed_layout_t) exists — check each part. + const auto &cl = getRef(); + bool b_ok = cutegen::is_static(cl.layout_b()); + bool off_ok = cutegen::is_static(cl.offset()); + bool a_ok = cl.is_a_swizzle() || cutegen::is_static(cl.layout_a()); + return b_ok && off_ok && a_ok; +} + +mlir::Attribute ComposedLayoutType::getValueAttr() const { return getAttr(); } + +/// Verify ComposedLayoutType storage invariants: no error sentinel, +/// positive static shape leaves, addable offset, and a sum that's +/// composable into the inner layout (scalar for swizzle, scalar or +/// weakly congruent for affine). +mlir::LogicalResult ComposedLayoutType::verify( + llvm::function_ref emitError, + ComposedLayoutAttr attr) { + auto const &composed = attr.getRef(); + if (cutegen::has_error(composed)) { + return emitError() << "expects no error(`x`) in layout, but got \"" + << cutegen::to_string(composed) << "\""; + } + // A trivial wrap (identity swizzle, no offset) is accepted and prints as + // its plain layout, as the compiler does. + auto hasNonPositiveShape = [](cutegen::layout const &lyt) { + return cutegen::any_leaf_is(lyt.shape(), [](auto const &e) { + return cutegen::holds_int(e) && cutegen::get_int(e) <= 0; + }); + }; + if (hasNonPositiveShape(composed.layout_b()) || + (composed.is_a_affine() && hasNonPositiveShape(composed.layout_a()))) { + return emitError() << "expects positive shape mode, but got \"" + << cutegen::to_string(composed) << "\""; + } + + // Addable: the offset and layout_eval(0, outer) must be compatible + // under arith-tuple addition (no scaled-basis-vs-integer mismatch + // or rank disagreement). + auto offset_b = cutegen::layout_eval(0, composed.layout_b()); + auto sum = + cutegen::arith_tuple_sum(composed.offset(), offset_b); + if (cutegen::has_error(sum)) { + return emitError() << "expects offset and `layout_eval(0, outer)` to be " + "addable, but got " + << cutegen::to_string(composed.offset()) << " and " + << cutegen::to_string(offset_b); + } + + // Swizzle inner: the arith-tuple sum must be a scalar (int or + // dynamic int) — swizzles can only consume scalar indices, so any + // scaled-basis structure surviving from the offset or outer is a + // composability error. + if (composed.is_a_swizzle() && !cutegen::holds_int_or_dynamic_int(sum)) { + return emitError() << "swizzle layout expects scalar `offset` and " + "`outer` without scaled basis, but got " + << cutegen::to_string(composed.offset()) << " and " + << cutegen::to_string(composed.layout_b()); + } + + // Affine inner: the sum either is a scalar (and layout_a indexes + // it directly) or matches layout_a's shape profile so each leaf + // dispatches to the corresponding mode of A. + if (composed.is_a_affine() && !cutegen::holds_int_or_dynamic_int(sum) && + !cutegen::weakly_congruent(sum, composed.layout_a().shape())) { + return emitError() + << "expects arith sum of `layout_eval(0, outer)` and `offset` to be " + "scalar or weakly congruent to inner shape, but got sum " + << cutegen::to_string(sum) << " and inner shape " + << cutegen::to_string(composed.layout_a().shape()); + } + + return mlir::success(); +} + +llvm::TypeSize +ComposedLayoutType::getTypeSize(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeSize(cg::get_llvm_type(builder, getRef())); +} + +llvm::TypeSize ComposedLayoutType::getTypeSizeInBits( + const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeSizeInBits(cg::get_llvm_type(builder, getRef())); +} + +uint64_t +ComposedLayoutType::getABIAlignment(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeABIAlignment(cg::get_llvm_type(builder, getRef())); +} + +uint64_t ComposedLayoutType::getPreferredAlignment( + const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypePreferredAlignment( + cg::get_llvm_type(builder, getRef())); +} + +std::optional ComposedLayoutType::getIndexBitwidth( + const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + OpBuilder builder(getContext()); + return dataLayout.getTypeIndexBitwidth(cg::get_llvm_type(builder, getRef())); +} + +bool ComposedLayoutType::areCompatible( + mlir::DataLayoutEntryListRef oldLayout, + mlir::DataLayoutEntryListRef newLayout, + mlir::DataLayoutSpecInterface newSpec, + const mlir::DataLayoutIdentifiedEntryMap &identified) const { + OpBuilder builder(getContext()); + if (auto dlTy = llvm::dyn_cast( + cg::get_llvm_type(builder, getRef()))) { + return dlTy.areCompatible(oldLayout, newLayout, newSpec, identified); + } + return true; +} + +llvm::LogicalResult +ComposedLayoutType::verifyEntries(mlir::DataLayoutEntryListRef entries, + mlir::Location loc) const { + OpBuilder builder(getContext()); + if (auto dlTy = llvm::dyn_cast( + cg::get_llvm_type(builder, getRef()))) { + return dlTy.verifyEntries(entries, loc); + } + return mlir::success(); +} + +//===----------------------------------------------------------------------===// +// SwizzleType — custom assembly format +//===----------------------------------------------------------------------===// + +mlir::Type SwizzleType::parse(mlir::AsmParser &parser) { + auto loc = parser.getCurrentLocation(); + auto attr = SwizzleAttr::get(parser.getContext(), + parseLenientPayload(parser)); + return getChecked([&] { return parser.emitError(loc); }, parser.getContext(), + attr); +} + +void SwizzleType::print(mlir::AsmPrinter &printer) const { + printer << '<'; + printer.printString(cutegen::to_string(getRef())); + printer << '>'; +} + +bool SwizzleType::isStatic() const { return true; } + +mlir::Attribute SwizzleType::getValueAttr() const { return getAttr(); } + +/// Verify SwizzleType storage invariants. Defense-in-depth — the +/// string parser already rejects invalid swizzle parameters via +/// cutegen::from_string, but programmatic +/// construction goes around the parser, so re-check here. +/// +/// cutegen::swizzle::is_valid (swizzle.hpp) enforces two +/// invariants on S: +/// 1. |num_shift| + num_bits + num_base <= 32 — parameters +/// must fit within the swizzle's 32-bit working width. +/// 2. |num_shift| >= num_bits — the shift amount must cover +/// the mask width so the Y and Z masks do not overlap. +mlir::LogicalResult +SwizzleType::verify(llvm::function_ref emitError, + SwizzleAttr attr) { + auto const &sw = attr.getRef(); + uint32_t numBits = sw.num_bits(); + uint32_t numBase = sw.num_base(); + int32_t numShift = sw.num_shift(); + uint32_t absShift = static_cast(std::abs(numShift)); + constexpr uint32_t kBitWidth = 32; + + if (absShift + numBits + numBase > kBitWidth) { + return emitError() << "expects |num_shift| + num_bits + num_base <= " + << kBitWidth << ", but got " << absShift << " + " + << numBits << " + " << numBase << " = " + << (absShift + numBits + numBase) << " for swizzle " + << cutegen::to_string(sw); + } + if (absShift < numBits) { + return emitError() << "expects |num_shift| >= num_bits, but got " + << absShift << " < " << numBits << " for swizzle " + << cutegen::to_string(sw); + } + return mlir::success(); +} + +// A swizzle is a compile-time-only object whose runtime representation is an +// empty LLVM struct (mirroring convertSwizzleType in CuteTypeConverter.cpp), so +// the data-layout queries delegate to that struct rather than to +// cg::get_llvm_type. +static mlir::Type getSwizzleLLVMType(mlir::MLIRContext *ctx) { + return LLVM::LLVMStructType::getLiteral(ctx, SmallVector{}); +} + +llvm::TypeSize +SwizzleType::getTypeSize(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + return dataLayout.getTypeSize(getSwizzleLLVMType(getContext())); +} + +llvm::TypeSize +SwizzleType::getTypeSizeInBits(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + return dataLayout.getTypeSizeInBits(getSwizzleLLVMType(getContext())); +} + +uint64_t +SwizzleType::getABIAlignment(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + return dataLayout.getTypeABIAlignment(getSwizzleLLVMType(getContext())); +} + +uint64_t +SwizzleType::getPreferredAlignment(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + return dataLayout.getTypePreferredAlignment(getSwizzleLLVMType(getContext())); +} + +std::optional +SwizzleType::getIndexBitwidth(const mlir::DataLayout &dataLayout, + mlir::DataLayoutEntryListRef params) const { + return dataLayout.getTypeIndexBitwidth(getSwizzleLLVMType(getContext())); +} + +bool SwizzleType::areCompatible( + mlir::DataLayoutEntryListRef oldLayout, + mlir::DataLayoutEntryListRef newLayout, + mlir::DataLayoutSpecInterface newSpec, + const mlir::DataLayoutIdentifiedEntryMap &identified) const { + if (auto dlTy = llvm::dyn_cast( + getSwizzleLLVMType(getContext()))) { + return dlTy.areCompatible(oldLayout, newLayout, newSpec, identified); + } + return true; +} + +llvm::LogicalResult +SwizzleType::verifyEntries(mlir::DataLayoutEntryListRef entries, + mlir::Location loc) const { + if (auto dlTy = llvm::dyn_cast( + getSwizzleLLVMType(getContext()))) { + return dlTy.verifyEntries(entries, loc); + } + return mlir::success(); +} + +//===----------------------------------------------------------------------===// +// The DSL compiler's types and attributes (CuteTypesCompiler.td, +// CuteAttrsCompiler.td) +//===----------------------------------------------------------------------===// + +#include "mlir/dialects/cute_ir/CuteEnums.cpp.inc" + +static OptionalParseResult generatedCuteTypeParser(DialectAsmParser &parser, + StringRef *mnemonic, + Type &value) { + OptionalParseResult result = generatedCuteTypeParser( + static_cast(parser), mnemonic, value); + if (result.has_value()) + return result; + StringRef digits = *mnemonic; + unsigned width; + if (!digits.consume_front("i") || digits.empty() || + !llvm::all_of(digits, llvm::isDigit) || digits.getAsInteger(10, width)) + return std::nullopt; + value = ConstrainedIntType::parse(parser, width); + return success(!!value); +} + +static LogicalResult generatedCuteTypePrinter(Type type, + DialectAsmPrinter &printer) { + if (auto integer = llvm::dyn_cast(type)) { + integer.print(printer); + return success(); + } + return generatedCuteTypePrinter(type, static_cast(printer)); +} + +// Bits of an integer or float (tf32 is stored in 32), or of a vector of them. +static unsigned storageBits(Type type) { + if (auto integer = llvm::dyn_cast_or_null(type)) + return integer.getWidth(); + if (auto fp = llvm::dyn_cast_or_null(type)) + return fp.isTF32() ? 32 : fp.getWidth(); + if (auto vector = llvm::dyn_cast_or_null(type)) + return vector.getNumElements() * storageBits(vector.getElementType()); + return 0; +} + +static bool isIdentitySwizzle(SwizzleAttr swizzle) { + return !swizzle || cutegen::to_string(swizzle.getRef()) == "S<0,4,3>"; +} + +uint64_t PtrType::getNaturalAlignment(Type valueType) { + unsigned bits = 0; + if (auto sparse = llvm::dyn_cast_or_null(valueType)) + bits = storageBits(sparse.getPhysicalType()); + else if (llvm::isa_and_nonnull(valueType)) + bits = storageBits(valueType); + if (bits <= 1) + return 1; + return bits % 8 ? 0 : bits / 8; +} + +BitLayoutAttr PtrType::getDefaultBitLayout(Type valueType, + AddressSpace addressSpace) { + unsigned chunk = addressSpace == AddressSpace::tmem ? 32 : 8; + std::string text; + if (auto sparse = llvm::dyn_cast_or_null(valueType)) { + unsigned bits = storageBits(sparse.getPhysicalType()); + if (!bits) + return {}; + text = llvm::formatv("({0},({1},{2})):(1,(0,{0}))", bits, + sparse.getNumLogical(), chunk / std::gcd(bits, chunk)) + .str(); + } else if (llvm::isa_and_nonnull(valueType)) { + unsigned bits = storageBits(valueType); + unsigned stride = bits == 1 ? 8 : bits; + text = llvm::formatv("({0},{1}):(1,{2})", bits, + chunk / std::gcd(stride, chunk), stride) + .str(); + } else { + return {}; + } + auto layout = cutegen::from_string(text); + if (!layout) + return {}; + MLIRContext *ctx = valueType.getContext(); + return BitLayoutAttr::get(ctx, LayoutAttr::get(ctx, std::move(*layout))); +} + +// The bits of a chunk: the largest extent times stride of the element mode. +static std::optional chunkBits(const cutegen::layout &layout) { + auto elements = cutegen::get(layout, 1); + auto shape = cutegen::flatten(elements.shape()); + auto stride = cutegen::flatten(elements.stride()); + if (cutegen::holds_int(shape) && cutegen::holds_int(stride)) + return shape.as_int64() * stride.as_int64(); + int64_t bits = 0; + for (size_t i = 0; i < cutegen::rank(shape); ++i) { + auto extent = cutegen::get(shape, i); + auto step = cutegen::get(stride, i); + if (!cutegen::holds_int(extent) || !cutegen::holds_int(step)) + return std::nullopt; + bits = std::max(bits, extent.as_int64() * step.as_int64()); + } + return bits; +} + +LogicalResult +BitLayoutAttr::verify(function_ref emitError, + LayoutAttr layout) { + const cutegen::layout &l = layout.getRef(); + if (!cutegen::is_static(l)) + return emitError() << "bitlayout must be fully static but got " << layout; + if (cutegen::rank(l) != 2) + return emitError() << "bitlayout must have 2 modes corresponding to " + "(element bit, element index), but got " + << cutegen::rank(l); + auto elements = cutegen::size(l, 1); + if (cutegen::holds_int(elements) && elements.as_int64() == 0) + return emitError() + << "bitlayout must contain at least one logical element, but got 0"; + return success(); +} + +//===- ptr and memref -----------------------------------------------------===// + +namespace { +struct PtrFields { + Type valueType; + AddressSpace space = AddressSpace::generic; + std::optional alignment; + SwizzleAttr swizzle; + BitLayoutAttr bitlayout; +}; +} // namespace + +// `bitlayout<"layout">`, the keyword already read. +static BitLayoutAttr parseBitLayout(AsmParser &parser) { + std::string text; + if (failed(parser.parseLess()) || failed(parser.parseString(&text)) || + failed(parser.parseGreater())) + return {}; + auto layout = cutegen::from_string(text); + if (!layout) { + parser.emitError(parser.getCurrentLocation(), "expects cg::layout"); + return {}; + } + MLIRContext *ctx = parser.getContext(); + return BitLayoutAttr::getChecked( + [&] { return parser.emitError(parser.getCurrentLocation()); }, ctx, + LayoutAttr::get(ctx, std::move(*layout))); +} + +// `S`, the `S` already read. +static SwizzleAttr parseInlineSwizzle(AsmParser &parser) { + int64_t bits, base, shift; + if (failed(parser.parseLess()) || failed(parser.parseInteger(bits)) || + failed(parser.parseComma()) || failed(parser.parseInteger(base)) || + failed(parser.parseComma()) || failed(parser.parseInteger(shift)) || + failed(parser.parseGreater())) + return {}; + auto value = cutegen::from_string( + llvm::formatv("S<{0},{1},{2}>", bits, base, shift).str()); + if (!value) { + parser.emitError(parser.getCurrentLocation(), "invalid swizzle parameters"); + return {}; + } + return SwizzleAttr::get(parser.getContext(), std::move(*value)); +} + +// A layout or composed layout spelled as a string; a composed layout that is +// a plain one (identity swizzle, no offset) is read as that layout. +static Type parseLayoutPayload(AsmParser &parser) { + std::string text; + llvm::SMLoc loc = parser.getCurrentLocation(); + if (failed(parser.parseString(&text))) + return {}; + MLIRContext *ctx = parser.getContext(); + auto emitError = [&] { return parser.emitError(loc); }; + if (auto value = cutegen::from_string(text)) + return LayoutType::getChecked(emitError, ctx, + LayoutAttr::get(ctx, std::move(*value))); + if (auto value = cutegen::from_string(text)) { + if (value->is_normal_layout()) + return LayoutType::getChecked(emitError, ctx, + LayoutAttr::get(ctx, value->layout_b())); + return ComposedLayoutType::getChecked( + emitError, ctx, ComposedLayoutAttr::get(ctx, std::move(*value))); + } + parser.emitError(parser.getCurrentLocation(), "layout parse failure"); + return {}; +} + +static void printLayoutPayload(AsmPrinter &printer, Type layout) { + if (auto plain = llvm::dyn_cast(layout)) + printer.printString(cutegen::to_string(plain.getRef())); + else + printer.printString( + cutegen::to_string(llvm::cast(layout).getRef())); +} + +static LogicalResult memrefLayoutExpected(AsmParser &parser) { + return parser.emitError(parser.getCurrentLocation(), + "failed to parse CuteMemRefType parameter `,` " + "expected, layout cannot be omitted."); +} + +// `[type, ][space][, align][, S][, bitlayout<"L">]` in this order; +// a memref requires the type and ends with its layout string instead of the +// bit layout, which follows the layout. +static FailureOr parsePtrFields(AsmParser &parser, Type *layout) { + PtrFields fields; + OptionalParseResult typed = parser.parseOptionalType(fields.valueType); + if (typed.has_value() && failed(*typed)) + return failure(); + if (layout && !typed.has_value()) + return memrefLayoutExpected(parser); + enum { kSpace, kAlign, kSwizzle, kBitLayout, kDone } next = kSpace; + bool first = !typed.has_value(); + while (true) { + if (!first && failed(parser.parseOptionalComma())) + break; + StringRef keyword; + if (failed(parser.parseOptionalKeyword(&keyword))) { + if (layout) { + if (!(*layout = parseLayoutPayload(parser))) + return failure(); + return fields; + } + if (first) + break; + parser.emitError(parser.getCurrentLocation(), + "expected a pointer field after `,`"); + return failure(); + } + first = false; + if (auto space = symbolizeAddressSpace(keyword); space && next <= kSpace) { + fields.space = *space; + next = kAlign; + } else if (keyword == "align" && next <= kAlign) { + uint64_t alignment; + if (failed(parser.parseLess()) || + failed(parser.parseInteger(alignment)) || + failed(parser.parseGreater())) + return failure(); + fields.alignment = alignment; + next = kSwizzle; + } else if (keyword == "S" && next <= kSwizzle) { + if (!(fields.swizzle = parseInlineSwizzle(parser))) + return failure(); + next = kBitLayout; + } else if (keyword == "bitlayout" && !layout && next <= kBitLayout) { + if (!(fields.bitlayout = parseBitLayout(parser))) + return failure(); + next = kDone; + } else { + parser.emitError(parser.getCurrentLocation(), + "unexpected pointer field `") + << keyword << "`"; + return failure(); + } + } + if (layout) + return memrefLayoutExpected(parser); + return fields; +} + +// Defaults are left out: they are what the parser fills in. +static PtrType buildPtr(AsmParser &parser, const PtrFields &fields) { + SwizzleAttr swizzle = + isIdentitySwizzle(fields.swizzle) ? SwizzleAttr() : fields.swizzle; + BitLayoutAttr bitlayout = fields.bitlayout; + if (bitlayout && + bitlayout == PtrType::getDefaultBitLayout(fields.valueType, fields.space)) + bitlayout = {}; + return PtrType::getChecked( + [&] { return parser.emitError(parser.getCurrentLocation()); }, + parser.getContext(), fields.valueType, fields.space, + fields.alignment.value_or(PtrType::getNaturalAlignment(fields.valueType)), + swizzle, bitlayout); +} + +static void printPtrFields(AsmPrinter &printer, PtrType ptr) { + if (ptr.getValueType()) { + printer.printType(ptr.getValueType()); + printer << ", "; + } + printer << stringifyAddressSpace(ptr.getAddressSpace()); + if (ptr.getAlignment() != PtrType::getNaturalAlignment(ptr.getValueType())) + printer << ", align<" << ptr.getAlignment() << '>'; + if (!isIdentitySwizzle(ptr.getSwizzle())) + printer << ", " << cutegen::to_string(ptr.getSwizzle().getRef()); +} + +static void printBitLayout(AsmPrinter &printer, PtrType ptr) { + BitLayoutAttr bitlayout = ptr.getBitlayout(); + if (!bitlayout || bitlayout == PtrType::getDefaultBitLayout( + ptr.getValueType(), ptr.getAddressSpace())) + return; + printer << ", bitlayout<"; + printer.printString(cutegen::to_string(bitlayout.getLayout().getRef())); + printer << '>'; +} + +Type PtrType::parse(AsmParser &parser) { + if (failed(parser.parseLess())) + return {}; + FailureOr fields = parsePtrFields(parser, nullptr); + if (failed(fields) || failed(parser.parseGreater())) + return {}; + return buildPtr(parser, *fields); +} + +void PtrType::print(AsmPrinter &printer) const { + printer << '<'; + printPtrFields(printer, *this); + printBitLayout(printer, *this); + printer << '>'; +} + +LogicalResult PtrType::verify(function_ref emitError, + Type valueType, AddressSpace addressSpace, + uint64_t alignment, SwizzleAttr swizzle, + BitLayoutAttr bitlayout) { + if (alignment == 0 && getNaturalAlignment(valueType) != 0) + return emitError() << "expects alignment >= 1, got 0"; + bool tmem = addressSpace == AddressSpace::tmem; + if (tmem && swizzle && swizzle.getRef().num_bits() != 0) + return emitError() << "tmem pointer does not support non-identity swizzle"; + BitLayoutAttr defaultBitLayout = getDefaultBitLayout(valueType, addressSpace); + BitLayoutAttr effective = bitlayout ? bitlayout : defaultBitLayout; + if (!effective) + return success(); + if (auto bits = chunkBits(effective.getLayout().getRef())) { + if (*bits % 8 != 0) + return emitError() << "bitlayout chunk must be byte aligned, but size is " + << *bits << " bits"; + if (tmem && (*bits < 32 || !llvm::isPowerOf2_64(*bits))) + return emitError() + << "tmem pointer only supports 32 bit chunks that is a power of " + "2, got " + << *bits; + } + if (llvm::isa_and_nonnull(valueType) && + effective != defaultBitLayout) + return emitError() << "expects bitlayout to be " << defaultBitLayout + << " for valueType '" << valueType << "'but got " + << effective; + return success(); +} + +// Whether the offsets of `layout` are whole aligned blocks of `sparsity` +// elements: the contiguous run from offset 0 (stride-1 mode, then each mode +// whose stride is the run so far) is a multiple of it, and so is every other +// stride. +static bool isDenseForSparsity(const cutegen::layout &layout, + int64_t sparsity) { + if (!cutegen::is_static(layout)) + return false; + auto shape = cutegen::flatten(layout.shape()); + auto stride = cutegen::flatten(layout.stride()); + std::vector> modes; // (stride, extent) + auto add = [&](const auto &extent, const auto &step) { + if (extent.as_int64() != 1 && step.as_int64() != 0) + modes.emplace_back(step.as_int64(), extent.as_int64()); + }; + if (cutegen::holds_int(shape)) { + add(shape, stride); + } else { + for (size_t i = 0; i < cutegen::rank(shape); ++i) + add(cutegen::get(shape, i), cutegen::get(stride, i)); + } + llvm::sort(modes, [](auto a, auto b) { + return std::abs(a.first) < std::abs(b.first); + }); + int64_t run = 1; + bool contiguous = true; + for (auto [step, extent] : modes) { + if (contiguous && step == run) { + run *= extent; + continue; + } + contiguous = false; + if (step % sparsity != 0) + return false; + } + return run % sparsity == 0; +} + +Type mlir::cutlass_compiler::cute::MemRefType::parse(AsmParser &parser) { + if (failed(parser.parseLess())) + return {}; + Type layout; + FailureOr fields = parsePtrFields(parser, &layout); + if (failed(fields)) + return {}; + if (succeeded(parser.parseOptionalComma())) { + if (failed(parser.parseKeyword("bitlayout")) || + !(fields->bitlayout = parseBitLayout(parser))) + return {}; + } + if (failed(parser.parseGreater())) + return {}; + PtrType ptr = buildPtr(parser, *fields); + if (!ptr) + return {}; + return getChecked( + [&] { return parser.emitError(parser.getCurrentLocation()); }, + parser.getContext(), ptr, layout); +} + +void mlir::cutlass_compiler::cute::MemRefType::print( + AsmPrinter &printer) const { + printer << '<'; + printPtrFields(printer, getPtr()); + printer << ", "; + printLayoutPayload(printer, getLayout()); + printBitLayout(printer, getPtr()); + printer << '>'; +} + +LogicalResult mlir::cutlass_compiler::cute::MemRefType::verify( + function_ref emitError, PtrType ptr, Type layout) { + if (!ptr) + return emitError() << "expects a CuTe pointer"; + if (!llvm::isa(layout)) + return emitError() << "expects a CuTe layout or composed layout"; + Type valueType = ptr.getValueType(); + if (!valueType || !(valueType.isIntOrIndexOrFloat() || + llvm::isa(valueType))) + return emitError() << "expects value type to be int, float, ptr or " + "sparse_elem, but got '" + << valueType << "'"; + if (auto sparse = llvm::dyn_cast(valueType)) { + int64_t sparsity = sparse.getNumLogical(); + bool dense; + if (auto composed = llvm::dyn_cast(layout)) { + // A swizzle may not move elements within a block. + const auto &c = composed.getRef(); + dense = c.is_a_swizzle() && + (int64_t(1) << c.swizzle_a().num_base()) % sparsity == 0 && + cutegen::holds_int(c.offset()) && + c.offset().as_int64() % sparsity == 0 && + isDenseForSparsity(c.layout_b(), sparsity); + } else { + dense = + isDenseForSparsity(llvm::cast(layout).getRef(), sparsity); + } + if (!dense) + return emitError() + << "layout must be dense but cannot be divided by sparsity"; + } + if (auto composed = llvm::dyn_cast(layout)) { + if (!isIdentitySwizzle(ptr.getSwizzle()) && + composed.getRef().is_a_swizzle() && + !composed.getRef().swizzle_a().is_identity()) + return emitError() << "cannot have both a swizzle on the pointer and a " + "non-identity swizzle in the layout"; + } else if (cutegen::has_scaled_basis( + llvm::cast(layout).getRef().stride())) { + return emitError() << "expected iterator and layout codomain to have " + "congruent profiles, but got iterator '" + << ptr << "' and layout layout: " + << cutegen::to_string( + llvm::cast(layout).getRef()); + } + return success(); +} + +//===- arith_tuple_iter, coord_tensor, i, fast_divmod_divisor, ... -----===// + +Type ArithTupleIteratorType::parse(AsmParser &parser) { + auto tuple = IntTupleType::parse(parser); + return tuple ? get(parser.getContext(), llvm::cast(tuple)) + : Type(); +} + +void ArithTupleIteratorType::print(AsmPrinter &printer) const { + getArithTuple().print(printer); +} + +Type CoordTensorType::parse(AsmParser &parser) { + std::string text; + if (failed(parser.parseLess()) || failed(parser.parseString(&text))) + return {}; + auto tuple = cutegen::from_string(text); + if (!tuple) { + parser.emitError(parser.getCurrentLocation(), + "failed to parse CoordTensorType parameter 'arith_tuple' " + "which is to be a `cutegen::int_tuple`"); + return {}; + } + Type layout; + if (failed(parser.parseComma()) || !(layout = parseLayoutPayload(parser)) || + failed(parser.parseGreater())) + return {}; + return getChecked( + [&] { return parser.emitError(parser.getCurrentLocation()); }, + parser.getContext(), + IntTupleType::get(parser.getContext(), std::move(*tuple)), layout); +} + +void CoordTensorType::print(AsmPrinter &printer) const { + printer << '<'; + printer.printString(cutegen::to_string(getArithTuple().getRef())); + printer << ", "; + printLayoutPayload(printer, getLayout()); + printer << '>'; +} + +LogicalResult +CoordTensorType::verify(function_ref emitError, + IntTupleType arithTuple, Type layout) { + if (!arithTuple) + return emitError() << "expects an arithmetic tuple"; + if (!llvm::isa(layout)) + return emitError() << "expects a CuTe layout or composed layout"; + return success(); +} + +// `<[pow2, ]divby D>`, or nothing for a plain integer. +Type ConstrainedIntType::parse(AsmParser &parser, unsigned width) { + int64_t divisibility = 1; + bool isPow2 = false; + if (succeeded(parser.parseOptionalLess())) { + isPow2 = succeeded(parser.parseOptionalKeyword("pow2")); + bool divided = !isPow2 || succeeded(parser.parseOptionalComma()); + if (divided && (failed(parser.parseKeyword("divby")) || + failed(parser.parseInteger(divisibility)))) + return {}; + if (failed(parser.parseGreater())) + return {}; + } + return get(parser.getContext(), divisibility, width, isPow2); +} + +void ConstrainedIntType::print(AsmPrinter &printer) const { + printer << 'i' << getWidth(); + if (!getIsPow2() && getDivisibility() == 1) + return; + printer << '<'; + if (getIsPow2()) + printer << "pow2"; + if (getDivisibility() != 1) + printer << (getIsPow2() ? ", " : "") << "divby " << getDivisibility(); + printer << '>'; +} + +Type FastDivmodDivisorType::parse(AsmParser &parser) { + unsigned width; + if (failed(parser.parseLess()) || failed(parser.parseInteger(width))) + return {}; + bool isPow2 = false; + if (succeeded(parser.parseOptionalComma())) { + if (failed(parser.parseKeyword("is_pow_2"))) + return {}; + isPow2 = true; + } + if (failed(parser.parseGreater())) + return {}; + return get(parser.getContext(), width, isPow2); +} + +void FastDivmodDivisorType::print(AsmPrinter &printer) const { + printer << '<' << getWidth(); + if (getIsPow_2()) + printer << ", is_pow_2"; + printer << '>'; +} + +LogicalResult +SparseElemType::verify(function_ref emitError, + int numLogical, Type physicalType) { + if (numLogical <= 0) + return emitError() << "sparse element type logical count must be > 0"; + Type element = physicalType; + if (auto vector = llvm::dyn_cast_or_null(physicalType)) + element = vector.getElementType(); + if (!llvm::isa_and_nonnull(element)) + return emitError() << "unsupported physical element type '" << physicalType + << "'"; + return success(); +} + +LogicalResult TiledMmaType::verify(function_ref emitError, + Type mmaAtom, LayoutAttr atomLayoutMNK, + TileAttr permutationMNK) { + if (cutegen::rank(atomLayoutMNK.getRef()) != 3) + return emitError() << "TiledMMA requires rank-3 atomLayoutMNK"; + if (permutationMNK && cutegen::rank(permutationMNK.getRef()) != 3) + return emitError() << "TiledMMA requires rank-3 permutationMNK"; + return success(); +} diff --git a/mlir/dialects/cute_ir/CuteTypesAll.td b/mlir/dialects/cute_ir/CuteTypesAll.td new file mode 100644 index 00000000..99766dcf --- /dev/null +++ b/mlir/dialects/cute_ir/CuteTypesAll.td @@ -0,0 +1,12 @@ +//===- CuteTypesAll.td - all cute types --------------------*- tablegen -*-===// +// +// The dialect has one generated type parser and printer, so the release's +// types and the compiler's are generated together from here. + +#ifndef MLIR_DIALECTS_CUTE_IR_CUTETYPESALL_TD +#define MLIR_DIALECTS_CUTE_IR_CUTETYPESALL_TD + +include "cute_ir/Dialect/Cute/IR/CuteTypes.td" +include "mlir/dialects/cute_ir/CuteTypesCompiler.td" + +#endif // MLIR_DIALECTS_CUTE_IR_CUTETYPESALL_TD diff --git a/mlir/dialects/cute_ir/CuteTypesCompiler.td b/mlir/dialects/cute_ir/CuteTypesCompiler.td new file mode 100644 index 00000000..6ca5588a --- /dev/null +++ b/mlir/dialects/cute_ir/CuteTypesCompiler.td @@ -0,0 +1,269 @@ +// The cute dialect's types beyond the release, generated from the +// nvidia-cutlass-dsl 4.8.0 release; do not edit. +// +// Sources: the compiler library (`_cutlass_ir`): its generated parser +// messages give the classes with a declarative syntax, their parameters in +// declaration order and their C++ types; the typed Python bindings of the +// types (`get` signatures and getters) give the parameters of the others; the +// live compiler, loaded from Python, gives the syntax (what it prints for what +// it parses, including every default it leaves out) and the verifier +// messages. +// +// Differences from the compiler, where its C++ types are not defined here: +// - The atoms of tiled_copy / tiled_copy_v2 / tiled_mma are +// `CopyAtomTypeInterface` / `CopyAtomTypeInterfaceV2` / +// `MmaAtomTypeInterface` in the compiler; plain types here. +// - ptr, memref and i have bespoke parsers in the compiler, so their +// parameter names and order are not recorded; the order here follows the +// Python `get` of each. +// - coord_tensor and memref layouts are `cutegen::layout` in the compiler's +// messages but accept composed layouts; they are a layout or composed +// layout type here. +// Verifier rules not reproduced: the bitlayout's cosize and element-count +// rules and TiledMMA's "requires static permutations of MNK" (neither +// triggered by any probe). The memref's sparse density rule ("layout must be +// dense but cannot be divided by sparsity") is inferred from probes: every +// probe agrees with it. + +#ifndef MLIR_DIALECTS_CUTE_IR_CUTETYPESCOMPILER_TD +#define MLIR_DIALECTS_CUTE_IR_CUTETYPESCOMPILER_TD + +include "cute_ir/Dialect/Cute/IR/CuteTypes.td" +include "mlir/dialects/cute_ir/CuteAttrsCompiler.td" +include "mlir/dialects/cute_ir/CuteEnums.td" + +def Cute_PtrType : Cute_Type<"Ptr", "ptr"> { + let summary = "Pointer with address space, alignment, swizzle and bit layout"; + let description = [{ + `<[value_type, ][space][, align][, S][, bitlayout<"L">]>`, the + fields in this order. Defaults are left out when printing and filled in + when parsing: + + - the address space is `generic` (always printed); + - the alignment is the value type's natural one: its width in bytes for + an integer or float that is a whole number of bytes, 1 for `i1`, 0 + (none) for other sub-byte widths, the physical type's bytes for a + `!cute.sparse_elem`, and 1 for any other type or no type; + - the swizzle is the identity `S<0,4,3>`; + - the bit layout is `(b,n):(1,s)` for an integer or float of `b` bits + (`s` = 8 for `i1`, else `b`) with `n` the least count making a chunk of + whole bytes (32-bit chunks in `tmem`), and + `(pb,(L,n)):(1,(0,pb))` for a `!cute.sparse_elem` of `pb` bits. + + A sparse pointer (the Python `sparsity`) is a `!cute.sparse_elem` value + type. + + ```mlir + !cute.ptr, S<2,4,3>> + !cute.ptr + !cute.ptr> + ``` + }]; + let parameters = (ins + Cute_OptionalParam<"mlir::Type", "pointee type">:$value_type, + Cute_Param<"AddressSpace", "address space">:$address_space, + Cute_Param<"uint64_t", "alignment in bytes; 0 only as a sub-byte type's natural alignment">:$alignment, + Cute_OptionalParam<"SwizzleAttr", "swizzle; null for the identity">:$swizzle, + Cute_OptionalParam<"BitLayoutAttr", "bit layout; null for the default">:$bitlayout + ); + let hasCustomAssemblyFormat = 1; + let genVerifyDecl = 1; + let extraClassDeclaration = [{ + /// The alignment a pointer to `valueType` has when none is spelled. + static uint64_t getNaturalAlignment(mlir::Type valueType); + /// The bit layout a pointer has when none is spelled (null if none). + static BitLayoutAttr getDefaultBitLayout(mlir::Type valueType, + AddressSpace addressSpace); + }]; +} + +def Cute_MemRefType : Cute_Type<"MemRef", "memref"> { + let summary = "Tensor view: a pointer with a layout"; + let description = [{ + `][, S], "layout"[, bitlayout<"L">]>`: + the pointer fields as in `!cute.ptr` (the value type is required), then the + layout or composed layout, then a non-default bit layout. A composed + layout whose inner part is the identity swizzle is read as its outer + layout. + + ```mlir + !cute.memref, S<2,4,3>, "((8,16),(32,1),(1,3)):((32,256),(1,0),(0,4096))"> + !cute.memref> + ``` + }]; + let parameters = (ins + Cute_Param<"PtrType", "pointer">:$ptr, + Cute_Param<"mlir::Type", "layout or composed-layout type">:$layout + ); + let hasCustomAssemblyFormat = 1; + let genVerifyDecl = 1; + let extraClassDeclaration = [{ + mlir::Type getValueType() const { return getPtr().getValueType(); } + AddressSpace getAddressSpace() const { return getPtr().getAddressSpace(); } + uint64_t getAlignment() const { return getPtr().getAlignment(); } + SwizzleAttr getSwizzle() const { return getPtr().getSwizzle(); } + }]; +} + +def Cute_CoordTensorType : Cute_Type<"CoordTensor", "coord_tensor"> { + let summary = "Coordinate tensor: an arithmetic tuple with a layout"; + let description = [{ + ```mlir + !cute.coord_tensor<"(?,?,?)", "((512,1),1):((1/2@0,0),0)"> + ``` + }]; + let parameters = (ins + Cute_Param<"IntTupleType", "origin arithmetic tuple">:$arith_tuple, + Cute_Param<"mlir::Type", "layout or composed-layout type">:$layout + ); + let hasCustomAssemblyFormat = 1; + let genVerifyDecl = 1; +} + +def Cute_ArithTupleIteratorType + : Cute_Type<"ArithTupleIterator", "arith_tuple_iter"> { + let summary = "Iterator over an arithmetic tuple"; + let description = [{ + ```mlir + !cute.arith_tuple_iter<"(?,?{div=16})"> + ``` + }]; + let parameters = (ins + Cute_Param<"IntTupleType", "current arithmetic tuple">:$arith_tuple + ); + // `$arith_tuple` alone would print a space after the mnemonic. + let hasCustomAssemblyFormat = 1; +} + +// `!cute.i`: one mnemonic per width, which the generated type dispatch +// cannot match, so the class has no mnemonic and CuteTypes.cpp routes +// `i` keywords to it. +def Cute_ConstrainedIntType : TypeDef { + let summary = "Integer of any width with known divisibility and power of two"; + let description = [{ + `!cute.i[<[pow2, ]divby D>]`; `divby 1` is left out (the default), + `divby 0` and negative divisors are kept. The compiler also reads + `!cute_nvgpu.i` as this type. + + ```mlir + !cute.i32 + !cute.i16 + !cute.i64 + ``` + }]; + // The compiler's type name for it. + let typeName = "cute.ConstrainedInt"; + let parameters = (ins + Cute_Param<"int64_t", "known divisor">:$divisibility, + Cute_Param<"unsigned", "integer width">:$width, + Cute_Param<"bool", "known to be a power of two">:$is_pow2 + ); + let extraClassDeclaration = [{ + static mlir::Type parse(mlir::AsmParser &parser, unsigned width); + void print(mlir::AsmPrinter &printer) const; + }]; +} + +def Cute_FastDivmodDivisorType + : Cute_Type<"FastDivmodDivisor", "fast_divmod_divisor"> { + let summary = "Precomputed divisor of the fast divmod operations"; + let description = [{ + ```mlir + !cute.fast_divmod_divisor<32> + !cute.fast_divmod_divisor<64, is_pow_2> + ``` + }]; + let parameters = (ins + Cute_Param<"unsigned", "integer width">:$width, + Cute_Param<"bool", "whether the divisor is a power of two">:$is_pow_2 + ); + let hasCustomAssemblyFormat = 1; +} + +def Cute_SparseElemType : Cute_Type<"SparseElem", "sparse_elem"> { + let summary = "Logical sparse element stored in a physical type"; + let description = [{ + `num_logical` logical elements stored as `physical_type` (an integer, + float or vector of them). + + ```mlir + !cute.sparse_elem<2, vector<1xf16>> + ``` + }]; + let parameters = (ins + Cute_Param<"int", "number of logical elements">:$num_logical, + Cute_Param<"mlir::Type", "physical storage type">:$physical_type + ); + let assemblyFormat = "`<` $num_logical `,` $physical_type `>`"; + let genVerifyDecl = 1; +} + +def Cute_TiledCopyType : Cute_Type<"TiledCopy", "tiled_copy"> { + let summary = "Copy atom tiled over threads and values"; + let description = [{ + ```mlir + !cute.tiled_copy, layout_copy_tv = <"(32,4):(4,1)">, tiler_mn = <"[4:1;32:1]">> + ``` + }]; + let parameters = (ins + Cute_Param<"mlir::Type", "copy atom">:$copy_atom, + Cute_Param<"LayoutAttr", "thread-value layout">:$layout_copy_tv, + Cute_Param<"TileAttr", "MN tiler">:$tiler_mn + ); + let assemblyFormat = [{ + `<` $copy_atom `,` `layout_copy_tv` `=` $layout_copy_tv `,` + `tiler_mn` `=` $tiler_mn `>` + }]; +} + +def Cute_TiledCopyV2Type : Cute_Type<"TiledCopyV2", "tiled_copy_v2"> { + let summary = "V2 copy atom tiled over threads and values"; + let description = [{ + ```mlir + !cute.tiled_copy_v2, layout_copy_tv = <"(1,8):(0,1)">, tiler_mn = <"[8:1]">> + ``` + }]; + let parameters = (ins + Cute_Param<"mlir::Type", "V2 copy atom">:$copy_atom, + Cute_Param<"LayoutAttr", "thread-value layout">:$layout_copy_tv, + Cute_Param<"TileAttr", "MN tiler">:$tiler_mn + ); + let assemblyFormat = [{ + `<` $copy_atom `,` `layout_copy_tv` `=` $layout_copy_tv `,` + `tiler_mn` `=` $tiler_mn `>` + }]; +} + +def Cute_TiledMmaType : Cute_Type<"TiledMma", "tiled_mma"> { + let summary = "MMA atom tiled over MNK"; + let description = [{ + The permutation is optional (absent, not `[_;_;_]`, when not given). + + ```mlir + !cute.tiled_mma f32 >, atom_layout_MNK = <"(16,16,1):(16,1,0)">, permutation_MNK = <"[(16,4):(4,1);(16,4):(4,1);_]">> + ``` + }]; + let parameters = (ins + Cute_Param<"mlir::Type", "mma atom">:$mma_atom, + Cute_Param<"LayoutAttr", "rank-3 atom layout over MNK">:$atom_layout_MNK, + Cute_OptionalParam<"TileAttr", "rank-3 permutation over MNK">:$permutation_MNK + ); + let assemblyFormat = [{ + `<` $mma_atom `,` `atom_layout_MNK` `=` $atom_layout_MNK + (`,` `permutation_MNK` `=` $permutation_MNK^)? `>` + }]; + let genVerifyDecl = 1; +} + +def Cute_TupleType : Cute_Type<"Tuple", "tuple"> { + let summary = "Non-empty tuple of types"; + let description = [{ + ```mlir + !cute.tuple + ``` + }]; + let parameters = (ins ArrayRefParameter<"mlir::Type", "element types">:$types); + let assemblyFormat = "`<` $types `>`"; +} + +#endif // MLIR_DIALECTS_CUTE_IR_CUTETYPESCOMPILER_TD diff --git a/mlir/dialects/cute_ir/README.md b/mlir/dialects/cute_ir/README.md new file mode 100644 index 00000000..6a667ce5 --- /dev/null +++ b/mlir/dialects/cute_ir/README.md @@ -0,0 +1,106 @@ +# CuTe dialects and Zig bindings + +`mlir/dialects/cute_ir` is the CuTe dialect of nvidia-cutlass-dsl 4.8.0. The +dialect, attributes, public types and `cutegen` layout algebra come verbatim +from the CUTLASS release's `cutlass_compiler/` tree (`@cute_ir`, +pinned in `third_party/cute_ir`, one patch giving cutegen the +`?{div=N}` leaf); this tree carries what differs from it: `CuteTypes.cpp` with +the compiler's remaining types, attributes and enums and ratio strides +allowed, the compiler's attributes, types and enums beyond the release +(`CuteAttrsCompiler.td`, `CuteTypesCompiler.td`, `CuteEnums.td` and the +`../cute_nvgpu` definitions), and every operation of the compiler's registry in +`CuteOps.td` and the companion `CuteNVGPUOps.td`. + +The companion dialects live in `../cute_nvgpu`, `../nvvm`, and `../cuda`, +with separate Bazel targets and Zig modules. NVIDIA's `nvvm` differs from +upstream's. The CuTe builder registers these dialects together with upstream +`llvm`. + +Updating to a new DSL release means refreshing the compiler entry-point +offsets and regenerating the `.td` files from the release's Python bindings +and compiler. The C API (except the algebra attribute functions in `capi.cc`) +and the Zig types, attributes, enums and operation builders are generated +from those `.td` files and follow them. + +## Zig + +Depend on `//mlir/dialects/cute_ir` and import `mlir/dialects/cute_ir`. +`cute.registerDialect(registry)` registers `cute`. + +Import each companion dialect explicitly and add its Bazel dependency: +`mlir/dialects/cute_nvgpu`, `mlir/dialects/nvvm`, and `mlir/dialects/cuda`. +CUDA and NVVM expose generated bindings in `.ops` and convenience builders +at the module root. NVIDIA's NVVM handle is `cute_nvvm`; its dialect cannot +share a registry with upstream NVVM. + +```zig +const cute = @import("mlir/dialects/cute_ir"); + +const pointer = try cute.PtrType.get(ctx, .{ + .valueType = .float(ctx, .f32), + .addressSpace = .smem, + .alignment = 16, +}); +const layout_attr = try cute.LayoutAttr.get(ctx, .{ + .shape = try .getStatic(ctx, 4), + .stride = try .getStatic(ctx, 1), +}); +const layout = try cute.LayoutType.get(ctx, .{ .attr = layout_attr.attribute() }); +const memref = try cute.MemRefType.get(ctx, .{ + .ptr = pointer.type_(), + .layout = layout.type_(), +}); +const alloc = cute.memref_alloc_smem(ctx, memref.type_(), loc).appendTo(block); +``` + +The bindings follow `mlir/dialects/README.md`. Algebra types take an algebra +attribute, `cute.LayoutType.get(ctx, .{ .attr = layout_attr.attribute() })`. +Algebra attributes are built bottom up with named constructors: integer +tuples, coordinates, shapes and strides from `getStatic`, `getDynamic`, +`getBasis` (strides) or `getUnderscore` (coordinates) leaves and `getTuple`; +layouts from a shape and a stride; tiles from `getLayout`, `getUnderscore` and +`getTuple`; swizzles from their three parameters and composed layouts from an +inner swizzle or layout, an offset and an outer layout. `value()` returns the +canonical text. NVGPU atom types take their typed parameters, such +as `.valType` and `.copyBits`. For reading existing IR use MLIR's generic +`Type.parse`. + +Operation builders take the operands, then the result types, then the +attributes, then the location, and call `mlir.Operation.make`. Parameters +follow ODS names, with a trailing underscore where needed. Result parameters +append `_type` or `_types`; attributes take `*const mlir.Attribute`. Optional +operands, attributes and result types go in a trailing `opts` struct whose +fields default to null; pass `.{}` to omit them, and `&.{}` for empty variadic +operands. Dots in +mnemonics become underscores. Variable-length operand and result groups go +through `Operation.make`'s `.variadic` API, which writes the segment-size +attributes. Builders verify the created operations; to defer verification, use +`mlir.Operation.make` directly with `.verify = false`. + +Builders of operations with regions (`cuda.kernel`) and of terminators +(`cuda.return`) do not verify on creation, since their blocks are completed +afterwards; the module is verified instead. + +Each dialect exposes a `:capi` target and a package-local `capi.h`, independently +of Zig: `//mlir/dialects/cute_ir:capi`, `//mlir/dialects/cute_nvgpu:capi`, +`//mlir/dialects/nvvm:capi`, and `//mlir/dialects/cuda:capi`. C API declarations +and definitions are grouped by registration, attributes, and types in +`capi.h` and `capi.cc`. Constructors return a null handle for invalid parameters. + +C++ implementations, headers, and TableGen sources live directly in each +dialect package. Generated `.inc` files are emitted into the same package +under Bazel's output tree. Virtual headers preserve upstream CuTe include +paths required by the release sources. + +```sh +bazel test //mlir/dialects/cute_ir:test //mlir/dialects/cute_nvgpu:test \ + //mlir/dialects/nvvm:test //mlir/dialects/cuda:test +``` + +These tests only check the bindings: every bound operation is registered, +every type and attribute rebuilds from its getters, and a sample of the nvvm +and cuda attributes and types print the compiler's own text byte for byte +(`#cuda` and `#cuda` print with a +space the compiler omits; it parses both). The dialect itself is +tested where it is maintained. The dialects here do not implement the +compiler's semantic verification or lowering. diff --git a/mlir/dialects/cute_ir/REGENERATING.md b/mlir/dialects/cute_ir/REGENERATING.md new file mode 100644 index 00000000..52b49e69 --- /dev/null +++ b/mlir/dialects/cute_ir/REGENERATING.md @@ -0,0 +1,333 @@ +# Regenerating the dialect definitions + +These are instructions for an agent. They describe what this directory must +contain after a CuTe DSL release bump, what is available to derive it, and how +to check the result. How to get there is left to you: combine the sources as +you see fit, write throwaway scripts, probe the compiler. Scripts and +intermediate tables are not committed; only the definitions are. + +Bare file names are in this directory, `../` are the sibling packages, +and other paths are relative to the repository root. + +## Goal + +The `cute` dialect in this directory and its companions in `../cute_nvgpu`, +`../nvvm`, and `../cuda` must describe the DSL compiler's +registry as completely and as precisely as the release allows: every +operation, type, attribute and enum the compiler registers, with every fact +that can be established about each of them. Nothing is opaque unless it cannot +be recovered, and each remaining gap is stated in the file where it occurs. + +The DSL compiler stays the authority for semantics and lowering. These +definitions exist so that IR can be built, parsed, printed and type-inferred +without it, and so that anything built here is accepted by it unchanged. + +## What lives here + +- **Taken from the release archive.** Its `cutlass_compiler/` tree provides the + cute dialect declaration, the public types and attributes and their `.td` + files, the type interfaces, `CuteDialect.h`, `CuteAttributes.cpp`, + `CuteDialect.cpp` and the `cutegen` layout algebra. The build reads them + from the archive; they are never copied or edited here. One patch on the + archive (`cutegen_dynamic_divisibility.patch`, next to the archive's + repository rule) teaches `cutegen` the `?{div=N}` leaf. Refresh it if it no + longer applies. The archive's `CuteOps.td` is not used: its operations + differ from the compiler's. +- **Maintained here.** `CuteTypes.cpp` started as the release's copy and holds + the parsers, printers and verifiers of every cute type, fixed wherever the + release's parsing diverges from the compiler's. `CuteInference.{h,cpp}` holds + result type inference; the other dialect `.cpp` files and the `BUILD.bazel` + files complete them. Edit them as the new definitions require. For `nvvm` + and `cuda` these are `NVVMDialect.{h,cpp}` (with the `#nvvm.target` + verifier) and `CudaDialect.{h,cpp}` in their dialect package roots. +- **Generated: what you produce.** Every package-local `.td` file that is not + a dialect declaration: the operations (`CuteOps.td`, `CuteNVGPUOps.td`, + `NVVMOps.td`, `CudaOps.td`), the compiler's types, attributes and enums + beyond the release (`CuteTypesCompiler.td`, `CuteAttrsCompiler.td`, + `CuteEnums.td`, `CuteNVGPUTypes.td`, `CuteNVGPUAttrs.td`, `CuteNVGPUEnums.td`, + `NVVMAttrs.td`, `NVVMEnums.td`, `CudaTypes.td`, `CudaAttrs.td`, + `CudaEnums.td`), the inference of `nvvm` and `cuda` operations whose results + are not one buildable type (`NVVMInference.cpp`, `CudaInference.cpp`), and + any new `.td` you need to split them sensibly. The dialect declarations (`*Dialect.td`) change only as far as the + generated files need. +- **Language bindings, if present.** When the tree carries bindings derived + from these definitions (a `capi.cc`/`capi.h` C API in each dialect package, builder files in each + directory), regenerate them from the records of the new definitions + (`llvm-tblgen --dump-json` gives them all; do not regex the `.td` files). + When a class changes shape (opaque to parametric, merged, split, renamed + accessors), change the bindings to match, and keep the existing + convenience helpers working where it costs little. Names, builder + parameters and `opts` follow `mlir/dialects/README.md`. + +Consumers outside this directory use the dialect classes and must keep +compiling: search the repository for the dialect namespaces +(`cutlass_compiler::cute`, `cute_nvgpu`, `nvvm`, `cuda`), for the Zig modules +(`mlir/dialects/cute_nvgpu`, `mlir/dialects/nvvm`, and +`mlir/dialects/cuda`) and for the headers +in the dialect packages. When a definition changes shape, update those users in the +same change. Tighter operand constraints also break consumers that built IR +with loose types, and so does an operation that stops inferring its results: +fix the consumer, not the constraint. + +## NVIDIA's `nvvm` and `cuda` dialects + +The kernels the CuTe builder emits (`dsl/cute`) use two more dialects of +the compiler: `nvvm` (special registers, mbarriers, tcgen05, ...) and `cuda` +(kernels, launch configurations, host runtime calls). Both have sibling packages +and must be regenerated with every release, like `cute` and `cute_nvgpu`. + +- **Neither is upstream's.** The compiler is built on NVIDIA's own LLVM + revision (the `.so` names it: `LLVM version 23.0.0git ()`, a hash + upstream does not have), and its `nvvm` differs from upstream NVVM: it has + operations, operands and enums upstream lacks (`nvvm.mbarrier.wait.parity`, + `#nvvm.mbar_wait`) and lacks some upstream has. `cuda` exists only in the + compiler. No `.td` ships with the release for either: rebuild both from the + bindings and the live compiler, as for the cute dialects. Upstream's NVVM + `.td` is neither a source nor a substitute. +- **Where they live.** `../nvvm` and `../cuda` contain the declarations, + generated `.td` files, dialect and inference `.cpp` files, and consolidated + `capi.h`/`capi.cc` C APIs. Each package's `ops.zig` contains the generated + bindings; `nvvm.zig` and `cuda.zig` provide the convenience builders. + +- **Names.** The C++ namespaces are `mlir::cutlass_compiler::nvvm` and + `mlir::cutlass_compiler::cuda`. The dialects are named `nvvm` and `cuda` as + in the compiler, so a registry holds this `nvvm` or upstream's, never both; + upstream's NVVM C++ is linked into the tree and must stay unregistered where + this one is. The C dialect handles are `cute_nvvm` and `cuda`, and the C API + of `nvvm` is prefixed `CuteNVVM` so that neither collides with upstream's + `mlirGetDialectHandle__nvvm__` or a future upstream NVVM C API; `cuda`'s is + prefixed `Cuda`. +- **`llvm` is upstream's.** NVVM operands are `!llvm.ptr` and LLVM + structs, and the builder emits `llvm.*` operations. The CuTe context + registers the upstream LLVM dialect, bound by `mlir/dialects/llvm/llvm.zig` + (its builders, and `mlirLLVMPointerTypeGet` for `!llvm.ptr`); the nvvm + and cuda constraints name `::mlir::LLVM` types. NVIDIA's `llvm` is not + vendored: its types print and parse the same, and the compiler reads the + `llvm.*` operations upstream prints (custom and generic forms, with the + properties upstream fills in) into the same module. This holds only while + the compatibility check below passes; when it stops holding, vendor NVIDIA's + `llvm` the same way (its bindings are `_llvm_ops_gen.py` and + `_llvm_enum_gen.py`) rather than patching around it. + +Sources specific to them, besides those in the table below: + +- `_nvvm_ops_gen.py`, `_cuda_ops_gen.py`: the operations. Operand kinds come + from how `__init__` appends them (and `_ODS_OPERAND_SEGMENTS`), result names + and kinds from the result properties, inference from whether `__init__` + builds a `results` list or passes its `results=` argument through. +- `_nvvm_enum_gen.py`, `_cuda_enum_gen.py`: the enums. A builder registered + under the enum's own name builds the plain `I32EnumAttr` (it follows its + enum class in the file); one registered as `nvvm.Attr` builds the + dialect attribute and gives its mnemonic and form: `#nvvm.` + or `#nvvm`. Some enums only have the latter. Parsing an + invalid case names the C++ enum (`expected ::mlir::NVVM:: to be one + of: ...`) or its summary, which maps each attribute to its enum. +- `cutlass._mlir._mlir_libs._cutlass_ir._mlirDialectsCuda`: the typed cuda + type and attribute classes and their `get()` parameters. The other cuda + types and attributes, and the three non-enum nvvm attributes (`shape`, + `ld_st_matrix_shape`, `target`), are found by trying the `.so`'s + identifiers as mnemonics (`!cuda.`, `#nvvm.`); `nvvm` has no + types. +- Attribute constraints the bindings name but no builder covers + (`nvvm.FPArithRoundingMode`, the `nvvm.Multimem*` orders, anonymous + `TypeAttr` constraints) are recovered by probing values: the verifier's + `failed to satisfy constraint: ...` names the rule, and the values it lets + through give the predicate. +- The compiler aborts on some malformed probes (`nvvm.mma.sync`, + `nvvm.mma.sp.sync`): run those checks one process each. +- Results the compiler infers but that are not one buildable type get an + inference rule only where varying the operand types shows the result is one + operand's type; elsewhere inference fails and the types written stand + (`_InferredOp`, as in cute). Vary the attributes too: a rule must + hold for every attribute value (`nvvm.mul` with `mode = wide` doubles the + width, `nvvm.shfl.sync` with `return_value_and_is_valid` returns a struct), + or inference must fail. + +The bindings follow the cute ones: the C API and the Zig enums, typed +attribute and type wrappers and operation builders are generated from +`llvm-tblgen --dump-json` of the generated `.td` files, following +`mlir/dialects/README.md` (names, wrappers, comments, formatting). Builders of +operations with regions, and of terminators, do not verify on creation (their +blocks are filled, or their block is completed, afterwards). `mlir/dialects/nvvm/nvvm.zig` and `mlir/dialects/cuda/cuda.zig` are the +hand-written convenience layer the builder calls; they fix the attributes the +kernels do not choose and must keep building no text. + +## Sources + +Pin everything to one release version, read from the repository rule that +fetches the release archive. The same version names the Python wheels. + +The packaging below is the one of the release these instructions were last +run against. Releases move things around: when a source is missing or has +changed shape, find where the same fact now lives, and update this file. + +| Source | Where | What it gives | +|---|---|---| +| Release archive | `https://github.com/NVIDIA/cutlass/archive/refs/tags/v.tar.gz`, `cutlass_compiler/` | The release's own `.td` files: what is already defined, their style, the parameter C++ types and helpers (`Cute_Param`, interfaces). Anything defined there is not redefined here. | +| Operation bindings | wheel `nvidia-cutlass-dsl-libs-base`, `cutlass/_mlir/dialects/__ops_gen.py` | Per operation: name, class name, docstring (summary, description, examples), operands and results in order with single, optional or variadic kind, segment attributes, attribute names with their ODS constraint (`AttrBuilder.get('')`), whether the builder infers result types, region count, successors. | +| Enum bindings | same wheel, `__enum_gen.py` | Every enum: cases, integer values, spellings, and for the ones that are attributes, the mnemonic they print under. | +| Compiler library | wheel `nvidia-cutlass-dsl-libs-cu13` (or `-cu12`), `cutlass/_mlir/_mlir_libs/_cutlass_ir.cu13.*.so` | Its strings: type and attribute class names, parameter names and C++ types (from the generated parser messages `failed to parse parameter '

' which is to be a `), mnemonics, verifier messages and constraint summaries. The strings are not in declaration order, and parameters read by custom directives have no message at all. | +| Typed Python classes | the same `.so`, as `cutlass._mlir.dialects.cute`, `cute_nvgpu`, `cuda` | For every compiler type and attribute: a `get(...)` with its parameters in order, and getters. Building a value and printing it shows the canonical syntax of each parameter value. | +| The examples | the release archive's `examples/python/CuTeDSL` | Real compiler-accepted modules, the round-trip corpus (see below). | +| The live compiler | the same `.so`, loaded from Python | Ground truth for everything above. | + +Loading the compiler. The `.so` only runs on Linux, on the architecture it was +built for. Two ways: + +- **Full install** (needed for the typed classes): a venv with + `nvidia-cutlass-dsl==` installed, then `import cutlass` before + anything else. +- **Bare wheels** (generic API only): unpack the two wheels into one directory, + put its `nvidia_cutlass_dsl/dsl_packages` on `sys.path`, `import + cutlass._mlir._mlir_libs`. The typed classes fail to import this way. + +Then use `cutlass._mlir.ir` with a fresh `ir.Context()`. The compiler +segfaults on some malformed inputs (some bitlayouts, some bad inference +inputs): run probes in a forked process. + +What the live compiler tells you: + +- `ir.Type.parse` / `ir.Attribute.parse` accept or reject a spelling, and on + rejection say where and what they expected; `str()` gives the canonical + printed form. Together with the typed classes this establishes each syntax: + the generated parser messages name each missing parameter, and the bespoke + custom directives (`128x128x32`, `32 DP`, `x32`, trailing flags such as + `mcast` or `t`) show up as `expected ...` errors. Every mnemonic a parser + knows can be found by trying every identifier-like string of the `.so`. +- `ir.Module.parse` on generic-form operations with wrong types yields the + verifier's constraint for each operand and result, in the compiler's words + (`operand #0 must be ... , but got ...`). Probe each operand with one + instance of every type class to learn exactly which classes it accepts. +- `op.has_trait(ir.IsTerminatorTrait)`, `ir.NoTerminatorTrait`; constructing + `ir.InferTypeOpInterface(op)` or `ir.MemoryEffectsOpInterface(op)` fails when + the operation lacks the interface. +- `ir.InferTypeOpInterface().inferReturnTypes(operands=..., + attributes=..., context=...)` is the inference oracle, with operand values + taken as block arguments of a dummy region. It cannot take properties: + for operations with attributes, parse and verify a module instead. +- Loop-invariant code motion hoists `Pure` operations, `canonicalize` removes + unused ones that do not write, and `cse` merges ones that at most read. +- Printing a parsed generic operation shows its custom assembly. + +The round-trip corpus: run every example of the archive's +`examples/python/CuTeDSL` with `CUTE_DSL_KEEP=ir-debug`, which keeps each +compiled module's IR; set `CUTE_DSL_ARCH` for the architecture-restricted +ones. This gives hundreds of real modules covering far more operations than +the kernels' fixtures in the repository, which are added to it. + +When sources disagree, the live compiler wins; record the disagreement in a +comment. + +## What each definition must carry + +Operations: + +- The operation name, and the binding's class name as the def name. +- `summary` and `description` from the docstring, kept whole, including its + examples. Reflow nothing that is code. Every example must parse and verify in + the compiler; fix the ones that do not and say so. +- Every operand and result, in order, with its name, its kind (single, + `Optional`, `Variadic`), and a type constraint that accepts exactly the type + classes the compiler's verifier accepts, with the verifier's wording as its + summary. Where a class or interface is not defined here or in the release, + or no probe type satisfies it, the predicate accepts any type and the file + says so. Constraints on types of unregistered dialects (such as `ptr`) must + also match the opaque form those types take here. A constraint must never + reject what the compiler accepts. +- Every attribute with its constraint: the ODS constraint the bindings name, + or the generated enum or dialect attribute it corresponds to; `OptionalAttr` + and `DefaultValuedAttr` as the bindings show. +- `AttrSizedOperandSegments` / `AttrSizedResultSegments`, regions (count and + kind), successors, terminator traits. +- `Pure` (or the memory effects) when established; none when unknown. +- `InferTypeOpInterface` exactly on the operations whose result types the + compiler infers. ODS generates the inference when all results are buildable + types; the others are written in `CuteInference.cpp`. The compiler's + inference is not a function of types alone: it folds constant operands and + keeps divisibility that `cutegen` drops. Where a rule is not recovered, + inference must fail rather than guess, and those operations override + `refineReturnTypes` so written result types stand: compiler-accepted IR is + never rejected for want of an inference rule. +- Operations keep the generic assembly form unless an `assemblyFormat` has been + checked to round-trip against the compiler (see below); the custom syntax + stays in the description. + +Types and attributes: + +- Every class the compiler has that the release lacks, with its mnemonic, + summary, parameters in declaration order (from the typed `get()`s and the + order parsing asks for them) with their C++ types and optional or defaulted + status, and a printed example in the description. C++ types of parameters + read by custom directives are not observable: choose the natural one. +- The syntax exactly as the compiler prints and parses it: a declarative + `assemblyFormat` when it can express it, otherwise a custom parser and + printer. Parametric beats opaque: a verbatim string payload is acceptable + only when the syntax cannot be reproduced, and the file says why. A hidden + parameter that never prints is left out: keeping it would make a built value + differ from its own printed text. +- The compiler's type aliases (`!memref_smem_f16`, `!copy_simt`, ...) are part + of its printed form: reproduce them through the dialect's + `OpAsmDialectInterface`. +- Verifiers (`genVerifyDecl`) where the compiler rejects parameter values, when + the rule is known. Large legality tables (instruction shapes and forms) may + be left to the compiler; the file says which. +- Enums as `I32EnumAttr` with the compiler's values and spellings, and their + `EnumAttr` with the compiler's mnemonic and `` form. + +Every generated file starts with a short header naming the release version and +the sources it came from, saying it is generated, and listing what it leaves +out or leaves unconstrained, and why. + +ODS pitfalls met so far: + +- Naming `InferTypeOpInterface` in the traits of an operation with buildable + results suppresses the `inferReturnTypes` ODS would generate. +- `mlir-tblgen -gen-enum-decls` emits every enum it sees, so a dialect's enums + file must not include another dialect's. +- The release's `CuteDialect.h` includes the attributes before the types, so + enums the attributes use must be emitted ahead of the attributes' include. +- A mnemonic that varies with a value (`!cute.i32`, `!cute.i7`) needs a hook + in front of the generated type parser. +- An attribute whose body is not in `<...>` prints in the opaque form + (`#dialect<...>`); its pretty form does not parse. + +## Checking the result + +A change is done when all of these hold: + +1. **Coverage.** Every operation in the bindings, every enum, and every + mnemonic the compiler's parsers know is either defined here, defined by the + release, or listed in the file's header as left out with the reason. +2. **Round trip against the compiler,** on the corpus and on sweeps of + spellings for every type and attribute (combinations of parameter values): + whatever the compiler accepts, this dialect accepts, and prints text the + compiler reads back to its own canonical form. Where the compiler's printed + form is itself reparseable, this dialect prints it byte-identical. Where + the compiler prints text its own parser rejects, or its parser drops data, + the reparseable, lossless form wins; list those cases. +3. **Inference agrees** with the compiler on every result the corpus and + crafted oracle cases produce. The inference test holds recorded compiler + results, since the compiler cannot be asked at test time: add a case for + every inferred operation the emitted modules do not already cover. +4. **The repository's tests pass**, including the dialect's own tests and the + tests of every consumer found above (kernel emitters, host tools, language + bindings), built with the tree's usual Bazel configuration. +5. **NVIDIA's compiler reads what the kernels emit.** Dump the IR of every + kernel the repository's tests emit through `dsl/cute`: the builder's + own tests and the CuTe kernels under `zml/kernels/moe/cute`, + `zml/kernels/flash_attention/cute` and `zml/kernels/paged_attention/cute`. + Dump both with the previous definitions and with the new ones. In the + release's venv, `import cutlass`, then parse and + verify each module with `cutlass._mlir.ir.Module.parse`, and compare the + generic print (`get_asm(print_generic_op_form=True)`) of each module before + and after: it must be identical, and a module the compiler rejected before + must be rejected the same way after. This is also what keeps upstream + `llvm` acceptable (see above). +6. **Both copies agree.** When the same dialect is carried by more than one + repository, the shared files are identical in each, and only + repository-specific files (`BUILD.bazel`, bindings) differ. + +Report, at the end: what changed in each file, the coverage figures, the +round-trip and inference figures, what is still opaque or unconstrained and +why, the test results, and anything in these instructions that was wrong, +missing or unclear. diff --git a/mlir/dialects/cute_ir/capi.cc b/mlir/dialects/cute_ir/capi.cc new file mode 100644 index 00000000..6e7c296d --- /dev/null +++ b/mlir/dialects/cute_ir/capi.cc @@ -0,0 +1,1454 @@ +//===- capi.cc - C API for the cute dialect -------------------------------===// + +#include "capi.h" + +#include "cute_ir/Dialect/Cute/IR/CuteDialect.h" + +#include "mlir/CAPI/IR.h" +#include "mlir/CAPI/Registration.h" +#include "mlir/CAPI/Support.h" +#include "llvm/ADT/STLExtras.h" + +#include + +// Registration + +MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(Cute, cute, + mlir::cutlass_compiler::cute::CuteDialect) + +// Attributes + +namespace cute = mlir::cutlass_compiler::cute; + +// Unwraps `handle` into `out`: false when it is of another context or not a +// T, or when it is null and not optional. +template +static bool unwrapAs(mlir::MLIRContext *ctx, H handle, bool optional, T &out) { + auto value = unwrap(handle); + if (!value) + return optional; + if (value.getContext() != ctx) + return false; + out = llvm::dyn_cast(value); + return static_cast(out); +} + +template +static MlirAttribute getAlgebraAttr(MlirContext context, + typename Attr::algebra_t algebra) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(Attr::get(ctx, std::move(algebra))); +} + +template +static MlirAttribute getStaticAttr(MlirContext context, int64_t value) { + return getAlgebraAttr( + context, typename Attr::algebra_t(cutegen::static_int_t(value))); +} + +template +static MlirAttribute getDynamicAttr(MlirContext context, int32_t width, + int64_t divisibility) { + if (!llvm::is_contained({1, 8, 16, 32, 64}, width) || divisibility <= 0) + return {nullptr}; + return getAlgebraAttr( + context, typename Attr::algebra_t(cutegen::mlir_dynamic_t( + cutegen::dynamic_int_properties_t(width, divisibility)))); +} + +template +static MlirAttribute getUnderscoreAttr(MlirContext context) { + return getAlgebraAttr( + context, typename Attr::algebra_t(cutegen::underscore_t())); +} + +template +static MlirAttribute getTupleAttr(MlirContext context, intptr_t numElements, + MlirAttribute const *elements) { + auto *ctx = unwrap(context); + if (!ctx || numElements < 0 || (numElements > 0 && !elements)) + return {nullptr}; + typename Attr::algebra_t::vector_t children; + children.reserve(numElements); + for (intptr_t i = 0; i < numElements; ++i) { + Attr element; + if (!unwrapAs(ctx, elements[i], /*optional=*/false, element)) + return {nullptr}; + children.push_back(element.getRef()); + } + return getAlgebraAttr(context, + typename Attr::algebra_t(std::move(children))); +} + +template +static MlirStringRef getAlgebraValue(MlirAttribute attr) { + auto value = llvm::cast(unwrap(attr)); + // Intern the serialization so the returned view outlives this function. + auto text = mlir::StringAttr::get(value.getContext(), + cutegen::to_string(value.getRef())); + return wrap(text.getValue()); +} + +MlirAttribute mlirCuteIntTupleAttrStaticGet(MlirContext context, + int64_t value) { + return getStaticAttr(context, value); +} + +MlirAttribute mlirCuteIntTupleAttrDynamicGet(MlirContext context, int32_t width, + int64_t divisibility) { + return getDynamicAttr(context, width, divisibility); +} + +MlirAttribute mlirCuteIntTupleAttrTupleGet(MlirContext context, + intptr_t numElements, + MlirAttribute const *elements) { + return getTupleAttr(context, numElements, elements); +} + +bool mlirAttributeIsACuteIntTuple(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirStringRef mlirCuteIntTupleAttrGetValue(MlirAttribute attr) { + return getAlgebraValue(attr); +} + +MlirAttribute mlirCuteCoordAttrStaticGet(MlirContext context, int64_t value) { + return getStaticAttr(context, value); +} + +MlirAttribute mlirCuteCoordAttrDynamicGet(MlirContext context, int32_t width, + int64_t divisibility) { + return getDynamicAttr(context, width, divisibility); +} + +MlirAttribute mlirCuteCoordAttrUnderscoreGet(MlirContext context) { + return getUnderscoreAttr(context); +} + +MlirAttribute mlirCuteCoordAttrTupleGet(MlirContext context, + intptr_t numElements, + MlirAttribute const *elements) { + return getTupleAttr(context, numElements, elements); +} + +bool mlirAttributeIsACuteCoord(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirStringRef mlirCuteCoordAttrGetValue(MlirAttribute attr) { + return getAlgebraValue(attr); +} + +MlirAttribute mlirCuteShapeAttrStaticGet(MlirContext context, int64_t value) { + return getStaticAttr(context, value); +} + +MlirAttribute mlirCuteShapeAttrDynamicGet(MlirContext context, int32_t width, + int64_t divisibility) { + return getDynamicAttr(context, width, divisibility); +} + +MlirAttribute mlirCuteShapeAttrTupleGet(MlirContext context, + intptr_t numElements, + MlirAttribute const *elements) { + return getTupleAttr(context, numElements, elements); +} + +bool mlirAttributeIsACuteShape(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirStringRef mlirCuteShapeAttrGetValue(MlirAttribute attr) { + return getAlgebraValue(attr); +} + +intptr_t mlirCuteShapeAttrGetRank(MlirAttribute attr) { + return cutegen::rank(llvm::cast(unwrap(attr)).getRef()); +} + +MlirAttribute mlirCuteStrideAttrStaticGet(MlirContext context, int64_t value) { + return getStaticAttr(context, value); +} + +MlirAttribute mlirCuteStrideAttrDynamicGet(MlirContext context, int32_t width, + int64_t divisibility) { + return getDynamicAttr(context, width, divisibility); +} + +MlirAttribute mlirCuteStrideAttrBasisGet(MlirContext context, int64_t numerator, + int64_t denominator, intptr_t numModes, + int32_t const *modes) { + if (denominator <= 0 || numModes <= 0 || !modes) + return {nullptr}; + cutegen::scaled_basis::vector_t path(modes, modes + numModes); + auto basis = + denominator == 1 + ? cutegen::scaled_basis(std::move(path), numerator) + : cutegen::scaled_basis(std::move(path), + cutegen::ratio(numerator, denominator)); + return getAlgebraAttr(context, + cutegen::stride(std::move(basis))); +} + +MlirAttribute mlirCuteStrideAttrTupleGet(MlirContext context, + intptr_t numElements, + MlirAttribute const *elements) { + return getTupleAttr(context, numElements, elements); +} + +bool mlirAttributeIsACuteStride(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirStringRef mlirCuteStrideAttrGetValue(MlirAttribute attr) { + return getAlgebraValue(attr); +} + +MlirAttribute mlirCuteLayoutAttrGet(MlirContext context, MlirAttribute shape, + MlirAttribute stride) { + auto *ctx = unwrap(context); + cute::ShapeAttr shapeAttr; + cute::StrideAttr strideAttr; + if (!ctx || !unwrapAs(ctx, shape, /*optional=*/false, shapeAttr) || + !unwrapAs(ctx, stride, /*optional=*/false, strideAttr)) + return {nullptr}; + cutegen::layout layout(shapeAttr.getRef(), strideAttr.getRef()); + if (cutegen::has_error(layout)) + return {nullptr}; + return getAlgebraAttr(context, std::move(layout)); +} + +bool mlirAttributeIsACuteLayout(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirStringRef mlirCuteLayoutAttrGetValue(MlirAttribute attr) { + return getAlgebraValue(attr); +} + +MlirAttribute mlirCuteTileAttrLayoutGet(MlirContext context, + MlirAttribute layout) { + auto *ctx = unwrap(context); + cute::LayoutAttr layoutAttr; + if (!ctx || !unwrapAs(ctx, layout, /*optional=*/false, layoutAttr)) + return {nullptr}; + return getAlgebraAttr(context, + cutegen::tile(layoutAttr.getRef())); +} + +MlirAttribute mlirCuteTileAttrUnderscoreGet(MlirContext context) { + return getUnderscoreAttr(context); +} + +MlirAttribute mlirCuteTileAttrTupleGet(MlirContext context, + intptr_t numElements, + MlirAttribute const *elements) { + return getTupleAttr(context, numElements, elements); +} + +bool mlirAttributeIsACuteTile(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirStringRef mlirCuteTileAttrGetValue(MlirAttribute attr) { + return getAlgebraValue(attr); +} + +MlirAttribute mlirCuteComposedLayoutAttrGet(MlirContext context, + MlirAttribute inner, + MlirAttribute offset, + MlirAttribute outer) { + auto *ctx = unwrap(context); + cute::IntTupleAttr offsetAttr; + cute::LayoutAttr outerAttr; + if (!ctx || !unwrapAs(ctx, offset, /*optional=*/false, offsetAttr) || + !unwrapAs(ctx, outer, /*optional=*/false, outerAttr)) + return {nullptr}; + cute::SwizzleAttr swizzleAttr; + cute::LayoutAttr layoutAttr; + std::optional composed; + if (unwrapAs(ctx, inner, /*optional=*/false, swizzleAttr)) + composed.emplace(swizzleAttr.getRef(), offsetAttr.getRef(), + outerAttr.getRef()); + else if (unwrapAs(ctx, inner, /*optional=*/false, layoutAttr)) + composed.emplace(layoutAttr.getRef(), offsetAttr.getRef(), + outerAttr.getRef()); + if (!composed || cutegen::has_error(*composed)) + return {nullptr}; + return getAlgebraAttr(context, + std::move(*composed)); +} + +bool mlirAttributeIsACuteComposedLayout(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirStringRef mlirCuteComposedLayoutAttrGetValue(MlirAttribute attr) { + return getAlgebraValue(attr); +} + +MlirAttribute mlirCuteSwizzleAttrGet(MlirContext context, uint32_t bits, + uint32_t base, int32_t shift) { + auto swizzle = cutegen::swizzle::get(bits, base, shift); + if (!swizzle) + return {nullptr}; + return getAlgebraAttr(context, *swizzle); +} + +bool mlirAttributeIsACuteSwizzle(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirStringRef mlirCuteSwizzleAttrGetValue(MlirAttribute attr) { + return getAlgebraValue(attr); +} + +// Compiler attributes + +bool mlirAttributeIsACuteBitLayout(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteBitLayoutAttrGet(MlirContext context, + MlirAttribute layout) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::LayoutAttr layoutValue; + if (!unwrapAs(ctx, layout, false, layoutValue)) + return {nullptr}; + return wrap(cute::BitLayoutAttr::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + layoutValue)); +} +MlirAttribute mlirCuteBitLayoutAttrGetLayout(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayout())); +} + +bool mlirAttributeIsACuteCopyAtom(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteCopyAtomAttrGet(MlirContext context, MlirAttribute thrId, + MlirAttribute layoutSrc, + MlirAttribute layoutDst, + MlirAttribute layoutRef) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::LayoutAttr thrIdValue; + if (!unwrapAs(ctx, thrId, false, thrIdValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutSrcValue; + if (!unwrapAs(ctx, layoutSrc, false, layoutSrcValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutDstValue; + if (!unwrapAs(ctx, layoutDst, false, layoutDstValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutRefValue; + if (!unwrapAs(ctx, layoutRef, false, layoutRefValue)) + return {nullptr}; + return wrap(cute::CopyAtomAttr::get(ctx, thrIdValue, layoutSrcValue, + layoutDstValue, layoutRefValue)); +} +MlirAttribute mlirCuteCopyAtomAttrGetThrId(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getThrId())); +} +MlirAttribute mlirCuteCopyAtomAttrGetLayoutSrc(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutSrc())); +} +MlirAttribute mlirCuteCopyAtomAttrGetLayoutDst(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutDst())); +} +MlirAttribute mlirCuteCopyAtomAttrGetLayoutRef(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutRef())); +} + +bool mlirAttributeIsACuteCopyAtomV2(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute +mlirCuteCopyAtomV2AttrGet(MlirContext context, MlirAttribute thrId, + MlirAttribute layoutSrcTV, MlirAttribute layoutDstTV, + MlirAttribute frgSrc, MlirAttribute frgDst) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::LayoutAttr thrIdValue; + if (!unwrapAs(ctx, thrId, false, thrIdValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutSrcTVValue; + if (!unwrapAs(ctx, layoutSrcTV, false, layoutSrcTVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutDstTVValue; + if (!unwrapAs(ctx, layoutDstTV, false, layoutDstTVValue)) + return {nullptr}; + mlir::Attribute frgSrcValue; + if (!unwrapAs(ctx, frgSrc, false, frgSrcValue)) + return {nullptr}; + mlir::Attribute frgDstValue; + if (!unwrapAs(ctx, frgDst, false, frgDstValue)) + return {nullptr}; + return wrap(cute::CopyAtomV2Attr::get(ctx, thrIdValue, layoutSrcTVValue, + layoutDstTVValue, frgSrcValue, + frgDstValue)); +} +MlirAttribute mlirCuteCopyAtomV2AttrGetThrId(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getThrId())); +} +MlirAttribute mlirCuteCopyAtomV2AttrGetLayoutSrcTV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutSrc_TV())); +} +MlirAttribute mlirCuteCopyAtomV2AttrGetLayoutDstTV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutDst_TV())); +} +MlirAttribute mlirCuteCopyAtomV2AttrGetFrgSrc(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgSrc())); +} +MlirAttribute mlirCuteCopyAtomV2AttrGetFrgDst(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgDst())); +} + +bool mlirAttributeIsACuteMmaAtom(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteMmaAtomAttrGet( + MlirContext context, MlirAttribute shapeMNK, MlirAttribute thrId, + MlirAttribute layoutATV, MlirAttribute layoutBTV, MlirAttribute layoutCTV, + MlirAttribute shapeAMK, MlirAttribute shapeBNK, MlirAttribute shapeCMN, + MlirAttribute frgA, MlirAttribute frgB, MlirAttribute frgC) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::ShapeAttr shapeMNKValue; + if (!unwrapAs(ctx, shapeMNK, false, shapeMNKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr thrIdValue; + if (!unwrapAs(ctx, thrId, false, thrIdValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutATVValue; + if (!unwrapAs(ctx, layoutATV, false, layoutATVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutBTVValue; + if (!unwrapAs(ctx, layoutBTV, false, layoutBTVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutCTVValue; + if (!unwrapAs(ctx, layoutCTV, false, layoutCTVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeAMKValue; + if (!unwrapAs(ctx, shapeAMK, false, shapeAMKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeBNKValue; + if (!unwrapAs(ctx, shapeBNK, false, shapeBNKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeCMNValue; + if (!unwrapAs(ctx, shapeCMN, false, shapeCMNValue)) + return {nullptr}; + mlir::Attribute frgAValue; + if (!unwrapAs(ctx, frgA, false, frgAValue)) + return {nullptr}; + mlir::Attribute frgBValue; + if (!unwrapAs(ctx, frgB, false, frgBValue)) + return {nullptr}; + mlir::Attribute frgCValue; + if (!unwrapAs(ctx, frgC, false, frgCValue)) + return {nullptr}; + return wrap(cute::MmaAtomAttr::get( + ctx, shapeMNKValue, thrIdValue, layoutATVValue, layoutBTVValue, + layoutCTVValue, shapeAMKValue, shapeBNKValue, shapeCMNValue, frgAValue, + frgBValue, frgCValue)); +} +MlirAttribute mlirCuteMmaAtomAttrGetShapeMNK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShape_MNK())); +} +MlirAttribute mlirCuteMmaAtomAttrGetThrId(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getThrId())); +} +MlirAttribute mlirCuteMmaAtomAttrGetLayoutATV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutA_TV())); +} +MlirAttribute mlirCuteMmaAtomAttrGetLayoutBTV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutB_TV())); +} +MlirAttribute mlirCuteMmaAtomAttrGetLayoutCTV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutC_TV())); +} +MlirAttribute mlirCuteMmaAtomAttrGetShapeAMK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeA_MK())); +} +MlirAttribute mlirCuteMmaAtomAttrGetShapeBNK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeB_NK())); +} +MlirAttribute mlirCuteMmaAtomAttrGetShapeCMN(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeC_MN())); +} +MlirAttribute mlirCuteMmaAtomAttrGetFrgA(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgA())); +} +MlirAttribute mlirCuteMmaAtomAttrGetFrgB(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgB())); +} +MlirAttribute mlirCuteMmaAtomAttrGetFrgC(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgC())); +} + +bool mlirAttributeIsACuteMxMmaAtom(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteMxMmaAtomAttrGet( + MlirContext context, MlirAttribute shapeMNK, MlirAttribute thrId, + MlirAttribute layoutATV, MlirAttribute layoutSFATV, MlirAttribute layoutBTV, + MlirAttribute layoutSFBTV, MlirAttribute layoutCTV, MlirAttribute shapeAMK, + MlirAttribute shapeSFAMK, MlirAttribute shapeBNK, MlirAttribute shapeSFBNK, + MlirAttribute shapeCMN, MlirAttribute frgA, MlirAttribute frgSFA, + MlirAttribute frgB, MlirAttribute frgSFB, MlirAttribute frgC) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::ShapeAttr shapeMNKValue; + if (!unwrapAs(ctx, shapeMNK, false, shapeMNKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr thrIdValue; + if (!unwrapAs(ctx, thrId, false, thrIdValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutATVValue; + if (!unwrapAs(ctx, layoutATV, false, layoutATVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutSFATVValue; + if (!unwrapAs(ctx, layoutSFATV, false, layoutSFATVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutBTVValue; + if (!unwrapAs(ctx, layoutBTV, false, layoutBTVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutSFBTVValue; + if (!unwrapAs(ctx, layoutSFBTV, false, layoutSFBTVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutCTVValue; + if (!unwrapAs(ctx, layoutCTV, false, layoutCTVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeAMKValue; + if (!unwrapAs(ctx, shapeAMK, false, shapeAMKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeSFAMKValue; + if (!unwrapAs(ctx, shapeSFAMK, false, shapeSFAMKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeBNKValue; + if (!unwrapAs(ctx, shapeBNK, false, shapeBNKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeSFBNKValue; + if (!unwrapAs(ctx, shapeSFBNK, false, shapeSFBNKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeCMNValue; + if (!unwrapAs(ctx, shapeCMN, false, shapeCMNValue)) + return {nullptr}; + mlir::Attribute frgAValue; + if (!unwrapAs(ctx, frgA, false, frgAValue)) + return {nullptr}; + mlir::Attribute frgSFAValue; + if (!unwrapAs(ctx, frgSFA, false, frgSFAValue)) + return {nullptr}; + mlir::Attribute frgBValue; + if (!unwrapAs(ctx, frgB, false, frgBValue)) + return {nullptr}; + mlir::Attribute frgSFBValue; + if (!unwrapAs(ctx, frgSFB, false, frgSFBValue)) + return {nullptr}; + mlir::Attribute frgCValue; + if (!unwrapAs(ctx, frgC, false, frgCValue)) + return {nullptr}; + return wrap(cute::MxMmaAtomAttr::get( + ctx, shapeMNKValue, thrIdValue, layoutATVValue, layoutSFATVValue, + layoutBTVValue, layoutSFBTVValue, layoutCTVValue, shapeAMKValue, + shapeSFAMKValue, shapeBNKValue, shapeSFBNKValue, shapeCMNValue, frgAValue, + frgSFAValue, frgBValue, frgSFBValue, frgCValue)); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetShapeMNK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShape_MNK())); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetThrId(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getThrId())); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetLayoutATV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutA_TV())); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetLayoutSFATV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutSFA_TV())); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetLayoutBTV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutB_TV())); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetLayoutSFBTV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutSFB_TV())); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetLayoutCTV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutC_TV())); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetShapeAMK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeA_MK())); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetShapeSFAMK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeSFA_MK())); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetShapeBNK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeB_NK())); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetShapeSFBNK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeSFB_NK())); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetShapeCMN(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeC_MN())); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetFrgA(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgA())); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetFrgSFA(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgSFA())); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetFrgB(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgB())); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetFrgSFB(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgSFB())); +} +MlirAttribute mlirCuteMxMmaAtomAttrGetFrgC(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgC())); +} + +bool mlirAttributeIsACuteReductionOp(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteReductionOpAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::cute::symbolizeReductionOp(value); + if (!valueValue) + return {nullptr}; + return wrap(cute::ReductionOpAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteReductionOpAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteSparseMmaAtom(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteSparseMmaAtomAttrGet( + MlirContext context, MlirAttribute shapeMNK, MlirAttribute thrId, + MlirAttribute layoutATV, MlirAttribute layoutBTV, MlirAttribute layoutCTV, + MlirAttribute layoutETV, MlirAttribute shapeAMK, MlirAttribute shapeBNK, + MlirAttribute shapeCMN, MlirAttribute shapeEMK, MlirAttribute frgA, + MlirAttribute frgB, MlirAttribute frgC, MlirAttribute frgE) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::ShapeAttr shapeMNKValue; + if (!unwrapAs(ctx, shapeMNK, false, shapeMNKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr thrIdValue; + if (!unwrapAs(ctx, thrId, false, thrIdValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutATVValue; + if (!unwrapAs(ctx, layoutATV, false, layoutATVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutBTVValue; + if (!unwrapAs(ctx, layoutBTV, false, layoutBTVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutCTVValue; + if (!unwrapAs(ctx, layoutCTV, false, layoutCTVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutETVValue; + if (!unwrapAs(ctx, layoutETV, false, layoutETVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeAMKValue; + if (!unwrapAs(ctx, shapeAMK, false, shapeAMKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeBNKValue; + if (!unwrapAs(ctx, shapeBNK, false, shapeBNKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeCMNValue; + if (!unwrapAs(ctx, shapeCMN, false, shapeCMNValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeEMKValue; + if (!unwrapAs(ctx, shapeEMK, false, shapeEMKValue)) + return {nullptr}; + mlir::Attribute frgAValue; + if (!unwrapAs(ctx, frgA, false, frgAValue)) + return {nullptr}; + mlir::Attribute frgBValue; + if (!unwrapAs(ctx, frgB, false, frgBValue)) + return {nullptr}; + mlir::Attribute frgCValue; + if (!unwrapAs(ctx, frgC, false, frgCValue)) + return {nullptr}; + mlir::Attribute frgEValue; + if (!unwrapAs(ctx, frgE, false, frgEValue)) + return {nullptr}; + return wrap(cute::SparseMmaAtomAttr::get( + ctx, shapeMNKValue, thrIdValue, layoutATVValue, layoutBTVValue, + layoutCTVValue, layoutETVValue, shapeAMKValue, shapeBNKValue, + shapeCMNValue, shapeEMKValue, frgAValue, frgBValue, frgCValue, + frgEValue)); +} +MlirAttribute mlirCuteSparseMmaAtomAttrGetShapeMNK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShape_MNK())); +} +MlirAttribute mlirCuteSparseMmaAtomAttrGetThrId(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getThrId())); +} +MlirAttribute mlirCuteSparseMmaAtomAttrGetLayoutATV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutA_TV())); +} +MlirAttribute mlirCuteSparseMmaAtomAttrGetLayoutBTV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutB_TV())); +} +MlirAttribute mlirCuteSparseMmaAtomAttrGetLayoutCTV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutC_TV())); +} +MlirAttribute mlirCuteSparseMmaAtomAttrGetLayoutETV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutE_TV())); +} +MlirAttribute mlirCuteSparseMmaAtomAttrGetShapeAMK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeA_MK())); +} +MlirAttribute mlirCuteSparseMmaAtomAttrGetShapeBNK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeB_NK())); +} +MlirAttribute mlirCuteSparseMmaAtomAttrGetShapeCMN(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeC_MN())); +} +MlirAttribute mlirCuteSparseMmaAtomAttrGetShapeEMK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeE_MK())); +} +MlirAttribute mlirCuteSparseMmaAtomAttrGetFrgA(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgA())); +} +MlirAttribute mlirCuteSparseMmaAtomAttrGetFrgB(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgB())); +} +MlirAttribute mlirCuteSparseMmaAtomAttrGetFrgC(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgC())); +} +MlirAttribute mlirCuteSparseMmaAtomAttrGetFrgE(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgE())); +} + +bool mlirAttributeIsACuteSparseMxMmaAtom(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGet( + MlirContext context, MlirAttribute shapeMNK, MlirAttribute thrId, + MlirAttribute layoutATV, MlirAttribute layoutSFATV, MlirAttribute layoutBTV, + MlirAttribute layoutSFBTV, MlirAttribute layoutCTV, MlirAttribute layoutETV, + MlirAttribute shapeAMK, MlirAttribute shapeSFAMK, MlirAttribute shapeBNK, + MlirAttribute shapeSFBNK, MlirAttribute shapeCMN, MlirAttribute shapeEMK, + MlirAttribute frgA, MlirAttribute frgSFA, MlirAttribute frgB, + MlirAttribute frgSFB, MlirAttribute frgC, MlirAttribute frgE) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::ShapeAttr shapeMNKValue; + if (!unwrapAs(ctx, shapeMNK, false, shapeMNKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr thrIdValue; + if (!unwrapAs(ctx, thrId, false, thrIdValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutATVValue; + if (!unwrapAs(ctx, layoutATV, false, layoutATVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutSFATVValue; + if (!unwrapAs(ctx, layoutSFATV, false, layoutSFATVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutBTVValue; + if (!unwrapAs(ctx, layoutBTV, false, layoutBTVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutSFBTVValue; + if (!unwrapAs(ctx, layoutSFBTV, false, layoutSFBTVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutCTVValue; + if (!unwrapAs(ctx, layoutCTV, false, layoutCTVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutETVValue; + if (!unwrapAs(ctx, layoutETV, false, layoutETVValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeAMKValue; + if (!unwrapAs(ctx, shapeAMK, false, shapeAMKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeSFAMKValue; + if (!unwrapAs(ctx, shapeSFAMK, false, shapeSFAMKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeBNKValue; + if (!unwrapAs(ctx, shapeBNK, false, shapeBNKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeSFBNKValue; + if (!unwrapAs(ctx, shapeSFBNK, false, shapeSFBNKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeCMNValue; + if (!unwrapAs(ctx, shapeCMN, false, shapeCMNValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr shapeEMKValue; + if (!unwrapAs(ctx, shapeEMK, false, shapeEMKValue)) + return {nullptr}; + mlir::Attribute frgAValue; + if (!unwrapAs(ctx, frgA, false, frgAValue)) + return {nullptr}; + mlir::Attribute frgSFAValue; + if (!unwrapAs(ctx, frgSFA, false, frgSFAValue)) + return {nullptr}; + mlir::Attribute frgBValue; + if (!unwrapAs(ctx, frgB, false, frgBValue)) + return {nullptr}; + mlir::Attribute frgSFBValue; + if (!unwrapAs(ctx, frgSFB, false, frgSFBValue)) + return {nullptr}; + mlir::Attribute frgCValue; + if (!unwrapAs(ctx, frgC, false, frgCValue)) + return {nullptr}; + mlir::Attribute frgEValue; + if (!unwrapAs(ctx, frgE, false, frgEValue)) + return {nullptr}; + return wrap(cute::SparseMxMmaAtomAttr::get( + ctx, shapeMNKValue, thrIdValue, layoutATVValue, layoutSFATVValue, + layoutBTVValue, layoutSFBTVValue, layoutCTVValue, layoutETVValue, + shapeAMKValue, shapeSFAMKValue, shapeBNKValue, shapeSFBNKValue, + shapeCMNValue, shapeEMKValue, frgAValue, frgSFAValue, frgBValue, + frgSFBValue, frgCValue, frgEValue)); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetShapeMNK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShape_MNK())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetThrId(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getThrId())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetLayoutATV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutA_TV())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetLayoutSFATV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutSFA_TV())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetLayoutBTV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutB_TV())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetLayoutSFBTV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutSFB_TV())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetLayoutCTV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutC_TV())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetLayoutETV(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLayoutE_TV())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetShapeAMK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeA_MK())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetShapeSFAMK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeSFA_MK())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetShapeBNK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeB_NK())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetShapeSFBNK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeSFB_NK())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetShapeCMN(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeC_MN())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetShapeEMK(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getShapeE_MK())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetFrgA(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgA())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetFrgSFA(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgSFA())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetFrgB(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgB())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetFrgSFB(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgSFB())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetFrgC(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgC())); +} +MlirAttribute mlirCuteSparseMxMmaAtomAttrGetFrgE(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFrgE())); +} + +// Types + +bool mlirTypeIsACuteType(MlirType type) { + mlir::Type cppType = unwrap(type); + return cppType && + cppType.getDialect().getNamespace() == + mlir::cutlass_compiler::cute::CuteDialect::getDialectNamespace(); +} + +bool mlirTypeIsACuteArithTupleIterator(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteArithTupleIteratorTypeGet(MlirContext context, + MlirType arithTuple) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::IntTupleType arithTupleValue; + if (!unwrapAs(ctx, arithTuple, false, arithTupleValue)) + return {nullptr}; + return wrap(cute::ArithTupleIteratorType::get(ctx, arithTupleValue)); +} +MlirType mlirCuteArithTupleIteratorTypeGetArithTuple(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getArithTuple())); +} + +bool mlirTypeIsACuteComposedLayout(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteComposedLayoutTypeGet(MlirContext context, + MlirAttribute attr) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::ComposedLayoutAttr attrValue; + if (!unwrapAs(ctx, attr, false, attrValue)) + return {nullptr}; + return wrap(cute::ComposedLayoutType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + attrValue)); +} +MlirAttribute mlirCuteComposedLayoutTypeGetAttr(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getAttr())); +} + +bool mlirTypeIsACuteConstrainedInt(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteConstrainedIntTypeGet(MlirContext context, + int64_t divisibility, uint32_t width, + bool isPow2) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cute::ConstrainedIntType::get(ctx, divisibility, width, isPow2)); +} +int64_t mlirCuteConstrainedIntTypeGetDivisibility(MlirType type) { + return llvm::cast(unwrap(type)).getDivisibility(); +} +uint32_t mlirCuteConstrainedIntTypeGetWidth(MlirType type) { + return llvm::cast(unwrap(type)).getWidth(); +} +bool mlirCuteConstrainedIntTypeGetIsPow2(MlirType type) { + return llvm::cast(unwrap(type)).getIsPow2(); +} + +bool mlirTypeIsACuteCoordTensor(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteCoordTensorTypeGet(MlirContext context, MlirType arithTuple, + MlirType layout) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::IntTupleType arithTupleValue; + if (!unwrapAs(ctx, arithTuple, false, arithTupleValue)) + return {nullptr}; + mlir::Type layoutValue; + if (!unwrapAs(ctx, layout, false, layoutValue)) + return {nullptr}; + return wrap(cute::CoordTensorType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + arithTupleValue, layoutValue)); +} +MlirType mlirCuteCoordTensorTypeGetArithTuple(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getArithTuple())); +} +MlirType mlirCuteCoordTensorTypeGetLayout(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getLayout())); +} + +bool mlirTypeIsACuteCoord(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteCoordTypeGet(MlirContext context, MlirAttribute attr) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::CoordAttr attrValue; + if (!unwrapAs(ctx, attr, false, attrValue)) + return {nullptr}; + return wrap(cute::CoordType::get(ctx, attrValue)); +} +MlirAttribute mlirCuteCoordTypeGetAttr(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getAttr())); +} + +bool mlirTypeIsACuteFastDivmodDivisor(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteFastDivmodDivisorTypeGet(MlirContext context, uint32_t width, + bool isPow2) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cute::FastDivmodDivisorType::get(ctx, width, isPow2)); +} +uint32_t mlirCuteFastDivmodDivisorTypeGetWidth(MlirType type) { + return llvm::cast(unwrap(type)).getWidth(); +} +bool mlirCuteFastDivmodDivisorTypeGetIsPow2(MlirType type) { + return llvm::cast(unwrap(type)).getIsPow_2(); +} + +bool mlirTypeIsACuteIntTuple(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteIntTupleTypeGet(MlirContext context, MlirAttribute attr) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::IntTupleAttr attrValue; + if (!unwrapAs(ctx, attr, false, attrValue)) + return {nullptr}; + return wrap(cute::IntTupleType::get(ctx, attrValue)); +} +MlirAttribute mlirCuteIntTupleTypeGetAttr(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getAttr())); +} + +bool mlirTypeIsACuteLayout(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteLayoutTypeGet(MlirContext context, MlirAttribute attr) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::LayoutAttr attrValue; + if (!unwrapAs(ctx, attr, false, attrValue)) + return {nullptr}; + return wrap(cute::LayoutType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + attrValue)); +} +MlirAttribute mlirCuteLayoutTypeGetAttr(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getAttr())); +} + +bool mlirTypeIsACuteMemRef(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteMemRefTypeGet(MlirContext context, MlirType ptr, + MlirType layout) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::PtrType ptrValue; + if (!unwrapAs(ctx, ptr, false, ptrValue)) + return {nullptr}; + mlir::Type layoutValue; + if (!unwrapAs(ctx, layout, false, layoutValue)) + return {nullptr}; + return wrap(cute::MemRefType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + ptrValue, layoutValue)); +} +MlirType mlirCuteMemRefTypeGetPtr(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getPtr())); +} +MlirType mlirCuteMemRefTypeGetLayout(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getLayout())); +} + +bool mlirTypeIsACutePtr(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCutePtrTypeGet(MlirContext context, MlirType valueType, + uint32_t addressSpace, uint64_t alignment, + MlirAttribute swizzle, MlirAttribute bitlayout) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valueTypeValue; + if (!unwrapAs(ctx, valueType, true, valueTypeValue)) + return {nullptr}; + auto addressSpaceValue = + ::mlir::cutlass_compiler::cute::symbolizeAddressSpace(addressSpace); + if (!addressSpaceValue) + return {nullptr}; + ::mlir::cutlass_compiler::cute::SwizzleAttr swizzleValue; + if (!unwrapAs(ctx, swizzle, true, swizzleValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::BitLayoutAttr bitlayoutValue; + if (!unwrapAs(ctx, bitlayout, true, bitlayoutValue)) + return {nullptr}; + return wrap(cute::PtrType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + valueTypeValue, *addressSpaceValue, alignment, swizzleValue, + bitlayoutValue)); +} +MlirType mlirCutePtrTypeGetValueType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getValueType())); +} +uint32_t mlirCutePtrTypeGetAddressSpace(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getAddressSpace()); +} +uint64_t mlirCutePtrTypeGetAlignment(MlirType type) { + return llvm::cast(unwrap(type)).getAlignment(); +} +MlirAttribute mlirCutePtrTypeGetSwizzle(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getSwizzle())); +} +MlirAttribute mlirCutePtrTypeGetBitlayout(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getBitlayout())); +} + +bool mlirTypeIsACuteShape(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteShapeTypeGet(MlirContext context, MlirAttribute attr) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::ShapeAttr attrValue; + if (!unwrapAs(ctx, attr, false, attrValue)) + return {nullptr}; + return wrap(cute::ShapeType::get(ctx, attrValue)); +} +MlirAttribute mlirCuteShapeTypeGetAttr(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getAttr())); +} + +bool mlirTypeIsACuteSparseElem(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteSparseElemTypeGet(MlirContext context, int numLogical, + MlirType physicalType) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type physicalTypeValue; + if (!unwrapAs(ctx, physicalType, false, physicalTypeValue)) + return {nullptr}; + return wrap(cute::SparseElemType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + numLogical, physicalTypeValue)); +} +int mlirCuteSparseElemTypeGetNumLogical(MlirType type) { + return llvm::cast(unwrap(type)).getNumLogical(); +} +MlirType mlirCuteSparseElemTypeGetPhysicalType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getPhysicalType())); +} + +bool mlirTypeIsACuteStride(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteStrideTypeGet(MlirContext context, MlirAttribute attr) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::StrideAttr attrValue; + if (!unwrapAs(ctx, attr, false, attrValue)) + return {nullptr}; + return wrap(cute::StrideType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + attrValue)); +} +MlirAttribute mlirCuteStrideTypeGetAttr(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getAttr())); +} + +bool mlirTypeIsACuteSwizzle(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteSwizzleTypeGet(MlirContext context, MlirAttribute attr) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::SwizzleAttr attrValue; + if (!unwrapAs(ctx, attr, false, attrValue)) + return {nullptr}; + return wrap(cute::SwizzleType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + attrValue)); +} +MlirAttribute mlirCuteSwizzleTypeGetAttr(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getAttr())); +} + +bool mlirTypeIsACuteTile(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteTileTypeGet(MlirContext context, MlirAttribute attr) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::TileAttr attrValue; + if (!unwrapAs(ctx, attr, false, attrValue)) + return {nullptr}; + return wrap(cute::TileType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + attrValue)); +} +MlirAttribute mlirCuteTileTypeGetAttr(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getAttr())); +} + +bool mlirTypeIsACuteTiledCopy(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteTiledCopyTypeGet(MlirContext context, MlirType copyAtom, + MlirAttribute layoutCopyTv, + MlirAttribute tilerMn) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type copyAtomValue; + if (!unwrapAs(ctx, copyAtom, false, copyAtomValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutCopyTvValue; + if (!unwrapAs(ctx, layoutCopyTv, false, layoutCopyTvValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::TileAttr tilerMnValue; + if (!unwrapAs(ctx, tilerMn, false, tilerMnValue)) + return {nullptr}; + return wrap(cute::TiledCopyType::get(ctx, copyAtomValue, layoutCopyTvValue, + tilerMnValue)); +} +MlirType mlirCuteTiledCopyTypeGetCopyAtom(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getCopyAtom())); +} +MlirType mlirCuteTiledCopyTypeGetAtom(MlirType type) { + return mlirCuteTiledCopyTypeGetCopyAtom(type); +} +MlirAttribute mlirCuteTiledCopyTypeGetLayoutCopyTv(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getLayoutCopyTv())); +} +MlirAttribute mlirCuteTiledCopyTypeGetTilerMn(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getTilerMn())); +} + +bool mlirTypeIsACuteTiledCopyV2(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteTiledCopyV2TypeGet(MlirContext context, MlirType copyAtom, + MlirAttribute layoutCopyTv, + MlirAttribute tilerMn) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type copyAtomValue; + if (!unwrapAs(ctx, copyAtom, false, copyAtomValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutCopyTvValue; + if (!unwrapAs(ctx, layoutCopyTv, false, layoutCopyTvValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::TileAttr tilerMnValue; + if (!unwrapAs(ctx, tilerMn, false, tilerMnValue)) + return {nullptr}; + return wrap(cute::TiledCopyV2Type::get(ctx, copyAtomValue, layoutCopyTvValue, + tilerMnValue)); +} +MlirType mlirCuteTiledCopyV2TypeGetCopyAtom(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getCopyAtom())); +} +MlirAttribute mlirCuteTiledCopyV2TypeGetLayoutCopyTv(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getLayoutCopyTv())); +} +MlirAttribute mlirCuteTiledCopyV2TypeGetTilerMn(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getTilerMn())); +} + +bool mlirTypeIsACuteTiledMma(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteTiledMmaTypeGet(MlirContext context, MlirType mmaAtom, + MlirAttribute atomLayoutMNK, + MlirAttribute permutationMNK) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type mmaAtomValue; + if (!unwrapAs(ctx, mmaAtom, false, mmaAtomValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr atomLayoutMNKValue; + if (!unwrapAs(ctx, atomLayoutMNK, false, atomLayoutMNKValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::TileAttr permutationMNKValue; + if (!unwrapAs(ctx, permutationMNK, true, permutationMNKValue)) + return {nullptr}; + return wrap(cute::TiledMmaType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + mmaAtomValue, atomLayoutMNKValue, permutationMNKValue)); +} +MlirType mlirCuteTiledMmaTypeGetMmaAtom(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getMmaAtom())); +} +MlirType mlirCuteTiledMmaTypeGetAtom(MlirType type) { + return mlirCuteTiledMmaTypeGetMmaAtom(type); +} +MlirAttribute mlirCuteTiledMmaTypeGetAtomLayoutMNK(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getAtomLayout_MNK())); +} +MlirAttribute mlirCuteTiledMmaTypeGetAtomLayoutMnk(MlirType type) { + return mlirCuteTiledMmaTypeGetAtomLayoutMNK(type); +} +MlirAttribute mlirCuteTiledMmaTypeGetPermutationMNK(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getPermutation_MNK())); +} +MlirAttribute mlirCuteTiledMmaTypeGetPermutationMnk(MlirType type) { + return mlirCuteTiledMmaTypeGetPermutationMNK(type); +} + +bool mlirTypeIsACuteTuple(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteTupleTypeGet(MlirContext context, intptr_t numTypes, + MlirType const *types) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + llvm::SmallVector typesValue; + for (intptr_t i = 0; i < numTypes; ++i) { + mlir::Type element; + if (!unwrapAs(ctx, types[i], false, element)) + return {nullptr}; + typesValue.push_back(element); + } + return wrap(cute::TupleType::get(ctx, typesValue)); +} +intptr_t mlirCuteTupleTypeGetNumTypes(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getTypes().size()); +} +MlirType mlirCuteTupleTypeGetType(MlirType type, intptr_t pos) { + return wrap(llvm::cast(unwrap(type)).getTypes()[pos]); +} diff --git a/mlir/dialects/cute_ir/capi.h b/mlir/dialects/cute_ir/capi.h new file mode 100644 index 00000000..002f4885 --- /dev/null +++ b/mlir/dialects/cute_ir/capi.h @@ -0,0 +1,492 @@ +//===- capi.h - C API for the cute dialect ------------------------*- C -*-===// + +#ifndef MLIR_DIALECTS_CUTE_IR_CAPI_H +#define MLIR_DIALECTS_CUTE_IR_CAPI_H + +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Registration + +MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(Cute, cute); + +// Attributes + +// Algebra attributes are built bottom up. A leaf is a static integer, a +// dynamic integer of `width` bits (1, 8, 16, 32 or 64) known to be a multiple +// of `divisibility`, a scaled basis (strides) or an underscore (coordinates +// and tiles); a tuple groups attributes of the same kind. A basis `modes` is +// its mode path, outermost first, which CuTe prints innermost first: +// `1@1@0` is modes {0, 1}. Constructors return null for invalid parameters or +// handles of another context. GetValue returns canonical algebra text owned +// by the attribute's context; callers must not free it. Getters require an +// attribute accepted by the corresponding IsA predicate. + +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteIntTupleAttrStaticGet(MlirContext context, int64_t value); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteIntTupleAttrDynamicGet( + MlirContext context, int32_t width, int64_t divisibility); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteIntTupleAttrTupleGet( + MlirContext context, intptr_t numElements, MlirAttribute const *elements); +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteIntTuple(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef +mlirCuteIntTupleAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteCoordAttrStaticGet(MlirContext context, + int64_t value); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteCoordAttrDynamicGet( + MlirContext context, int32_t width, int64_t divisibility); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteCoordAttrUnderscoreGet(MlirContext context); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteCoordAttrTupleGet( + MlirContext context, intptr_t numElements, MlirAttribute const *elements); +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteCoord(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef mlirCuteCoordAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteShapeAttrStaticGet(MlirContext context, + int64_t value); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteShapeAttrDynamicGet( + MlirContext context, int32_t width, int64_t divisibility); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteShapeAttrTupleGet( + MlirContext context, intptr_t numElements, MlirAttribute const *elements); +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteShape(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef mlirCuteShapeAttrGetValue(MlirAttribute attr); +MLIR_CAPI_EXPORTED intptr_t mlirCuteShapeAttrGetRank(MlirAttribute attr); + +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteStrideAttrStaticGet(MlirContext context, int64_t value); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteStrideAttrDynamicGet( + MlirContext context, int32_t width, int64_t divisibility); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteStrideAttrBasisGet( + MlirContext context, int64_t numerator, int64_t denominator, + intptr_t numModes, int32_t const *modes); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteStrideAttrTupleGet( + MlirContext context, intptr_t numElements, MlirAttribute const *elements); +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteStride(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef mlirCuteStrideAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteLayoutAttrGet(MlirContext context, + MlirAttribute shape, + MlirAttribute stride); +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteLayout(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef mlirCuteLayoutAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteTileAttrLayoutGet(MlirContext context, MlirAttribute layout); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteTileAttrUnderscoreGet(MlirContext context); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteTileAttrTupleGet( + MlirContext context, intptr_t numElements, MlirAttribute const *elements); +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteTile(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef mlirCuteTileAttrGetValue(MlirAttribute attr); + +// `inner` is a swizzle or a layout attribute. +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteComposedLayoutAttrGet(MlirContext context, MlirAttribute inner, + MlirAttribute offset, MlirAttribute outer); +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteComposedLayout(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef +mlirCuteComposedLayoutAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteSwizzleAttrGet(MlirContext context, + uint32_t bits, + uint32_t base, + int32_t shift); +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteSwizzle(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef +mlirCuteSwizzleAttrGetValue(MlirAttribute attr); + +// Compiler attributes + +// Constructors return null for invalid parameters or handles of another +// context; optional parameters take null handles, and std::optional enums a +// negative value. Enums are their integer values (the dialect's *Enums.td). +// Getters require a value accepted by the matching IsA. + +// `#cute.bitlayout`: Bit layout of a pointer's elements within a storage chunk. +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteBitLayout(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteBitLayoutAttrGet(MlirContext context, + MlirAttribute layout); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteBitLayoutAttrGetLayout(MlirAttribute attr); + +// `#cute.copy_atom`: Thread and value layouts of a copy atom. +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteCopyAtom(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteCopyAtomAttrGet( + MlirContext context, MlirAttribute thrId, MlirAttribute layoutSrc, + MlirAttribute layoutDst, MlirAttribute layoutRef); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteCopyAtomAttrGetThrId(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteCopyAtomAttrGetLayoutSrc(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteCopyAtomAttrGetLayoutDst(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteCopyAtomAttrGetLayoutRef(MlirAttribute attr); + +// `#cute.copy_atom_v2`: Thread and value layouts and fragments of a V2 copy +// atom. +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteCopyAtomV2(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteCopyAtomV2AttrGet( + MlirContext context, MlirAttribute thrId, MlirAttribute layoutSrcTV, + MlirAttribute layoutDstTV, MlirAttribute frgSrc, MlirAttribute frgDst); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteCopyAtomV2AttrGetThrId(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteCopyAtomV2AttrGetLayoutSrcTV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteCopyAtomV2AttrGetLayoutDstTV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteCopyAtomV2AttrGetFrgSrc(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteCopyAtomV2AttrGetFrgDst(MlirAttribute attr); + +// `#cute.mma_atom`: Shapes, thread-value layouts and fragments of an MMA atom. +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteMmaAtom(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteMmaAtomAttrGet( + MlirContext context, MlirAttribute shapeMNK, MlirAttribute thrId, + MlirAttribute layoutATV, MlirAttribute layoutBTV, MlirAttribute layoutCTV, + MlirAttribute shapeAMK, MlirAttribute shapeBNK, MlirAttribute shapeCMN, + MlirAttribute frgA, MlirAttribute frgB, MlirAttribute frgC); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMmaAtomAttrGetShapeMNK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMmaAtomAttrGetThrId(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMmaAtomAttrGetLayoutATV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMmaAtomAttrGetLayoutBTV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMmaAtomAttrGetLayoutCTV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMmaAtomAttrGetShapeAMK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMmaAtomAttrGetShapeBNK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMmaAtomAttrGetShapeCMN(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteMmaAtomAttrGetFrgA(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteMmaAtomAttrGetFrgB(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteMmaAtomAttrGetFrgC(MlirAttribute attr); + +// `#cute.mx_mma_atom`: MMA atom attribute with scale-factor operands. +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteMxMmaAtom(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteMxMmaAtomAttrGet( + MlirContext context, MlirAttribute shapeMNK, MlirAttribute thrId, + MlirAttribute layoutATV, MlirAttribute layoutSFATV, MlirAttribute layoutBTV, + MlirAttribute layoutSFBTV, MlirAttribute layoutCTV, MlirAttribute shapeAMK, + MlirAttribute shapeSFAMK, MlirAttribute shapeBNK, MlirAttribute shapeSFBNK, + MlirAttribute shapeCMN, MlirAttribute frgA, MlirAttribute frgSFA, + MlirAttribute frgB, MlirAttribute frgSFB, MlirAttribute frgC); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetShapeMNK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetThrId(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetLayoutATV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetLayoutSFATV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetLayoutBTV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetLayoutSFBTV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetLayoutCTV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetShapeAMK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetShapeSFAMK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetShapeBNK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetShapeSFBNK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetShapeCMN(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetFrgA(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetFrgSFA(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetFrgB(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetFrgSFB(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteMxMmaAtomAttrGetFrgC(MlirAttribute attr); + +// `#cute.reduction_op`: Op for cute reduce operations. +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteReductionOp(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteReductionOpAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteReductionOpAttrGetValue(MlirAttribute attr); + +// `#cute.sparse_mma_atom`: MMA atom attribute with a sparsity-metadata operand. +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteSparseMmaAtom(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteSparseMmaAtomAttrGet( + MlirContext context, MlirAttribute shapeMNK, MlirAttribute thrId, + MlirAttribute layoutATV, MlirAttribute layoutBTV, MlirAttribute layoutCTV, + MlirAttribute layoutETV, MlirAttribute shapeAMK, MlirAttribute shapeBNK, + MlirAttribute shapeCMN, MlirAttribute shapeEMK, MlirAttribute frgA, + MlirAttribute frgB, MlirAttribute frgC, MlirAttribute frgE); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMmaAtomAttrGetShapeMNK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMmaAtomAttrGetThrId(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMmaAtomAttrGetLayoutATV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMmaAtomAttrGetLayoutBTV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMmaAtomAttrGetLayoutCTV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMmaAtomAttrGetLayoutETV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMmaAtomAttrGetShapeAMK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMmaAtomAttrGetShapeBNK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMmaAtomAttrGetShapeCMN(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMmaAtomAttrGetShapeEMK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMmaAtomAttrGetFrgA(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMmaAtomAttrGetFrgB(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMmaAtomAttrGetFrgC(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMmaAtomAttrGetFrgE(MlirAttribute attr); + +// `#cute.sparse_mx_mma_atom`: MMA atom attribute with scale-factor and metadata +// operands. +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteSparseMxMmaAtom(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteSparseMxMmaAtomAttrGet( + MlirContext context, MlirAttribute shapeMNK, MlirAttribute thrId, + MlirAttribute layoutATV, MlirAttribute layoutSFATV, MlirAttribute layoutBTV, + MlirAttribute layoutSFBTV, MlirAttribute layoutCTV, MlirAttribute layoutETV, + MlirAttribute shapeAMK, MlirAttribute shapeSFAMK, MlirAttribute shapeBNK, + MlirAttribute shapeSFBNK, MlirAttribute shapeCMN, MlirAttribute shapeEMK, + MlirAttribute frgA, MlirAttribute frgSFA, MlirAttribute frgB, + MlirAttribute frgSFB, MlirAttribute frgC, MlirAttribute frgE); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetShapeMNK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetThrId(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetLayoutATV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetLayoutSFATV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetLayoutBTV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetLayoutSFBTV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetLayoutCTV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetLayoutETV(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetShapeAMK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetShapeSFAMK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetShapeBNK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetShapeSFBNK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetShapeCMN(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetShapeEMK(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetFrgA(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetFrgSFA(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetFrgB(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetFrgSFB(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetFrgC(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteSparseMxMmaAtomAttrGetFrgE(MlirAttribute attr); + +// Types + +/// Returns true when `type` belongs to the CuTe dialect. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteType(MlirType type); + +// Constructors return null for invalid parameters or handles of another +// context; optional parameters take null handles, and std::optional enums a +// negative value. Enums are their integer values (the dialect's *Enums.td). +// Getters require a value accepted by the matching IsA. + +// `!cute.arith_tuple_iter`: Iterator over an arithmetic tuple. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteArithTupleIterator(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteArithTupleIteratorTypeGet(MlirContext context, MlirType arithTuple); +MLIR_CAPI_EXPORTED MlirType +mlirCuteArithTupleIteratorTypeGetArithTuple(MlirType type); + +// `!cute.composed_layout`: CuTe composed layout: A ∘ offset ∘ B. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteComposedLayout(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteComposedLayoutTypeGet(MlirContext context, + MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteComposedLayoutTypeGetAttr(MlirType type); + +// `!cute.i`: Integer of any width with known divisibility and power of two. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteConstrainedInt(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteConstrainedIntTypeGet(MlirContext context, + int64_t divisibility, + uint32_t width, + bool isPow2); +MLIR_CAPI_EXPORTED int64_t +mlirCuteConstrainedIntTypeGetDivisibility(MlirType type); +MLIR_CAPI_EXPORTED uint32_t mlirCuteConstrainedIntTypeGetWidth(MlirType type); +MLIR_CAPI_EXPORTED bool mlirCuteConstrainedIntTypeGetIsPow2(MlirType type); + +// `!cute.coord_tensor`: Coordinate tensor: an arithmetic tuple with a layout. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteCoordTensor(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteCoordTensorTypeGet(MlirContext context, + MlirType arithTuple, + MlirType layout); +MLIR_CAPI_EXPORTED MlirType mlirCuteCoordTensorTypeGetArithTuple(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteCoordTensorTypeGetLayout(MlirType type); + +// `!cute.coord`: Scalar integer, underscore wildcard, or recursive tuple of +// coordinates. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteCoord(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteCoordTypeGet(MlirContext context, + MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteCoordTypeGetAttr(MlirType type); + +// `!cute.fast_divmod_divisor`: Precomputed divisor of the fast divmod +// operations. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteFastDivmodDivisor(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteFastDivmodDivisorTypeGet( + MlirContext context, uint32_t width, bool isPow2); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteFastDivmodDivisorTypeGetWidth(MlirType type); +MLIR_CAPI_EXPORTED bool mlirCuteFastDivmodDivisorTypeGetIsPow2(MlirType type); + +// `!cute.int_tuple`: Scalar integer or recursive tuple of integers. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteIntTuple(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteIntTupleTypeGet(MlirContext context, + MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteIntTupleTypeGetAttr(MlirType type); + +// `!cute.layout`: CuTe layout: a shape/stride pair. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteLayout(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteLayoutTypeGet(MlirContext context, + MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteLayoutTypeGetAttr(MlirType type); + +// `!cute.memref`: Tensor view: a pointer with a layout. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteMemRef(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteMemRefTypeGet(MlirContext context, + MlirType ptr, + MlirType layout); +MLIR_CAPI_EXPORTED MlirType mlirCuteMemRefTypeGetPtr(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteMemRefTypeGetLayout(MlirType type); + +// `!cute.ptr`: Pointer with address space, alignment, swizzle and bit layout. +MLIR_CAPI_EXPORTED bool mlirTypeIsACutePtr(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCutePtrTypeGet( + MlirContext context, MlirType valueType, uint32_t addressSpace, + uint64_t alignment, MlirAttribute swizzle, MlirAttribute bitlayout); +MLIR_CAPI_EXPORTED MlirType mlirCutePtrTypeGetValueType(MlirType type); +MLIR_CAPI_EXPORTED uint32_t mlirCutePtrTypeGetAddressSpace(MlirType type); +MLIR_CAPI_EXPORTED uint64_t mlirCutePtrTypeGetAlignment(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute mlirCutePtrTypeGetSwizzle(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute mlirCutePtrTypeGetBitlayout(MlirType type); + +// `!cute.shape`: Scalar integer or recursive tuple of shape extents. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteShape(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteShapeTypeGet(MlirContext context, + MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteShapeTypeGetAttr(MlirType type); + +// `!cute.sparse_elem`: Logical sparse element stored in a physical type. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteSparseElem(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteSparseElemTypeGet(MlirContext context, + int numLogical, + MlirType physicalType); +MLIR_CAPI_EXPORTED int mlirCuteSparseElemTypeGetNumLogical(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteSparseElemTypeGetPhysicalType(MlirType type); + +// `!cute.stride`: Scalar or recursive tuple of stride elements. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteStride(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteStrideTypeGet(MlirContext context, + MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteStrideTypeGetAttr(MlirType type); + +// `!cute.swizzle`: CuTe swizzle: S. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteSwizzle(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteSwizzleTypeGet(MlirContext context, + MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteSwizzleTypeGetAttr(MlirType type); + +// `!cute.tile`: CuTe tile: a recursive sequence of layouts and underscores. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteTile(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteTileTypeGet(MlirContext context, + MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteTileTypeGetAttr(MlirType type); + +// `!cute.tiled_copy`: Copy atom tiled over threads and values. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteTiledCopy(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteTiledCopyTypeGet(MlirContext context, + MlirType copyAtom, + MlirAttribute layoutCopyTv, + MlirAttribute tilerMn); +MLIR_CAPI_EXPORTED MlirType mlirCuteTiledCopyTypeGetCopyAtom(MlirType type); +// Former name of mlirCuteTiledCopyTypeGetCopyAtom. +MLIR_CAPI_EXPORTED MlirType mlirCuteTiledCopyTypeGetAtom(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteTiledCopyTypeGetLayoutCopyTv(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteTiledCopyTypeGetTilerMn(MlirType type); + +// `!cute.tiled_copy_v2`: V2 copy atom tiled over threads and values. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteTiledCopyV2(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteTiledCopyV2TypeGet(MlirContext context, MlirType copyAtom, + MlirAttribute layoutCopyTv, MlirAttribute tilerMn); +MLIR_CAPI_EXPORTED MlirType mlirCuteTiledCopyV2TypeGetCopyAtom(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteTiledCopyV2TypeGetLayoutCopyTv(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteTiledCopyV2TypeGetTilerMn(MlirType type); + +// `!cute.tiled_mma`: MMA atom tiled over MNK. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteTiledMma(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteTiledMmaTypeGet( + MlirContext context, MlirType mmaAtom, MlirAttribute atomLayoutMNK, + MlirAttribute permutationMNK); +MLIR_CAPI_EXPORTED MlirType mlirCuteTiledMmaTypeGetMmaAtom(MlirType type); +// Former name of mlirCuteTiledMmaTypeGetMmaAtom. +MLIR_CAPI_EXPORTED MlirType mlirCuteTiledMmaTypeGetAtom(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteTiledMmaTypeGetAtomLayoutMNK(MlirType type); +// Former name of mlirCuteTiledMmaTypeGetAtomLayoutMNK. +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteTiledMmaTypeGetAtomLayoutMnk(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteTiledMmaTypeGetPermutationMNK(MlirType type); +// Former name of mlirCuteTiledMmaTypeGetPermutationMNK. +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteTiledMmaTypeGetPermutationMnk(MlirType type); + +// `!cute.tuple`: Non-empty tuple of types. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteTuple(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteTupleTypeGet(MlirContext context, + intptr_t numTypes, + MlirType const *types); +MLIR_CAPI_EXPORTED intptr_t mlirCuteTupleTypeGetNumTypes(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteTupleTypeGetType(MlirType type, + intptr_t pos); + +#ifdef __cplusplus +} +#endif + +#endif // MLIR_DIALECTS_CUTE_IR_CAPI_H diff --git a/mlir/dialects/cute_ir/cute_ir.zig b/mlir/dialects/cute_ir/cute_ir.zig new file mode 100644 index 00000000..b277d061 --- /dev/null +++ b/mlir/dialects/cute_ir/cute_ir.zig @@ -0,0 +1,2844 @@ +const std = @import("std"); + +const c = @import("c"); +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +pub const CuteType = opaque { + const M = mlir.Methods(CuteType, c.MlirType); + + pub const isAFn = c.mlirTypeIsACuteType; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; +}; + +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("cute"); +} + +/// `cute.kernel`: marks a function as a CuTe kernel. +pub fn kernel(ctx: *mlir.Context) mlir.NamedAttribute { + return .named(ctx, "cute.kernel", .unit(ctx)); +} + +/// The address space of a pointer or memref. +pub const MemorySpace = AddressSpace; + +pub fn pointerType( + ctx: *mlir.Context, + space: MemorySpace, + alignment: u64, + opts: struct { + element_type: ?*const mlir.Type = null, + }, +) !*const mlir.Type { + return (try PtrType.get(ctx, .{ + .valueType = opts.element_type, + .addressSpace = space, + .alignment = alignment, + })).type_(); +} + +pub fn constrainedIntType(ctx: *mlir.Context, width: enum { i32, i64 }, divisible_by: u64) !*const mlir.Type { + return (try ConstrainedIntType.get(ctx, .{ + .divisibility = std.math.cast(i64, divisible_by) orelse return error.InvalidMlir, + .width = switch (width) { + .i32 => 32, + .i64 => 64, + }, + .isPow2 = false, + })).type_(); +} + +/// Algebra attributes are built bottom up from leaves and tuples of the same +/// kind. A dynamic leaf has `width` bits (1, 8, 16, 32 or 64) and is known to +/// be a multiple of `divisibility`. Getter strings are owned by the MLIR +/// context and remain valid until that context is destroyed. +pub const IntTupleAttr = opaque { + const M = mlir.Methods(IntTupleAttr, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsACuteIntTuple; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub fn getStatic(ctx: *mlir.Context, value_: i64) mlir.Error!*const IntTupleAttr { + return @ptrCast(c.mlirCuteIntTupleAttrStaticGet(ctx.ptr(), value_).ptr orelse return error.InvalidMlir); + } + + pub fn getDynamic(ctx: *mlir.Context, width: i32, divisibility: i64) mlir.Error!*const IntTupleAttr { + return @ptrCast(c.mlirCuteIntTupleAttrDynamicGet(ctx.ptr(), width, divisibility).ptr orelse return error.InvalidMlir); + } + + pub fn getTuple(ctx: *mlir.Context, elements: []const *const IntTupleAttr) mlir.Error!*const IntTupleAttr { + return @ptrCast(c.mlirCuteIntTupleAttrTupleGet(ctx.ptr(), @intCast(elements.len), @ptrCast(elements)).ptr orelse return error.InvalidMlir); + } + + pub fn value(self: *const IntTupleAttr) []const u8 { + return mlir.string(c.mlirCuteIntTupleAttrGetValue(self.ptr())); + } + + pub fn attribute(self: *const IntTupleAttr) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +pub const CoordAttr = opaque { + const M = mlir.Methods(CoordAttr, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsACuteCoord; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub fn getStatic(ctx: *mlir.Context, value_: i64) mlir.Error!*const CoordAttr { + return @ptrCast(c.mlirCuteCoordAttrStaticGet(ctx.ptr(), value_).ptr orelse return error.InvalidMlir); + } + + pub fn getDynamic(ctx: *mlir.Context, width: i32, divisibility: i64) mlir.Error!*const CoordAttr { + return @ptrCast(c.mlirCuteCoordAttrDynamicGet(ctx.ptr(), width, divisibility).ptr orelse return error.InvalidMlir); + } + + pub fn getUnderscore(ctx: *mlir.Context) mlir.Error!*const CoordAttr { + return @ptrCast(c.mlirCuteCoordAttrUnderscoreGet(ctx.ptr()).ptr orelse return error.InvalidMlir); + } + + pub fn getTuple(ctx: *mlir.Context, elements: []const *const CoordAttr) mlir.Error!*const CoordAttr { + return @ptrCast(c.mlirCuteCoordAttrTupleGet(ctx.ptr(), @intCast(elements.len), @ptrCast(elements)).ptr orelse return error.InvalidMlir); + } + + pub fn value(self: *const CoordAttr) []const u8 { + return mlir.string(c.mlirCuteCoordAttrGetValue(self.ptr())); + } + + pub fn attribute(self: *const CoordAttr) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +pub const ShapeAttr = opaque { + const M = mlir.Methods(ShapeAttr, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsACuteShape; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub fn getStatic(ctx: *mlir.Context, value_: i64) mlir.Error!*const ShapeAttr { + return @ptrCast(c.mlirCuteShapeAttrStaticGet(ctx.ptr(), value_).ptr orelse return error.InvalidMlir); + } + + pub fn getDynamic(ctx: *mlir.Context, width: i32, divisibility: i64) mlir.Error!*const ShapeAttr { + return @ptrCast(c.mlirCuteShapeAttrDynamicGet(ctx.ptr(), width, divisibility).ptr orelse return error.InvalidMlir); + } + + pub fn getTuple(ctx: *mlir.Context, elements: []const *const ShapeAttr) mlir.Error!*const ShapeAttr { + return @ptrCast(c.mlirCuteShapeAttrTupleGet(ctx.ptr(), @intCast(elements.len), @ptrCast(elements)).ptr orelse return error.InvalidMlir); + } + + pub fn value(self: *const ShapeAttr) []const u8 { + return mlir.string(c.mlirCuteShapeAttrGetValue(self.ptr())); + } + + pub fn rank(self: *const ShapeAttr) usize { + return @intCast(c.mlirCuteShapeAttrGetRank(self.ptr())); + } + + pub fn attribute(self: *const ShapeAttr) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +pub const StrideAttr = opaque { + const M = mlir.Methods(StrideAttr, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsACuteStride; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub fn getStatic(ctx: *mlir.Context, value_: i64) mlir.Error!*const StrideAttr { + return @ptrCast(c.mlirCuteStrideAttrStaticGet(ctx.ptr(), value_).ptr orelse return error.InvalidMlir); + } + + pub fn getDynamic(ctx: *mlir.Context, width: i32, divisibility: i64) mlir.Error!*const StrideAttr { + return @ptrCast(c.mlirCuteStrideAttrDynamicGet(ctx.ptr(), width, divisibility).ptr orelse return error.InvalidMlir); + } + + /// `numerator/denominator` along `modes`, the mode path outermost first; + /// CuTe prints it innermost first, so `1@1@0` is modes `{ 0, 1 }`. + pub fn getBasis(ctx: *mlir.Context, numerator: i64, denominator: i64, modes: []const i32) mlir.Error!*const StrideAttr { + return @ptrCast(c.mlirCuteStrideAttrBasisGet(ctx.ptr(), numerator, denominator, @intCast(modes.len), modes.ptr).ptr orelse return error.InvalidMlir); + } + + pub fn getTuple(ctx: *mlir.Context, elements: []const *const StrideAttr) mlir.Error!*const StrideAttr { + return @ptrCast(c.mlirCuteStrideAttrTupleGet(ctx.ptr(), @intCast(elements.len), @ptrCast(elements)).ptr orelse return error.InvalidMlir); + } + + pub fn value(self: *const StrideAttr) []const u8 { + return mlir.string(c.mlirCuteStrideAttrGetValue(self.ptr())); + } + + pub fn attribute(self: *const StrideAttr) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +pub const LayoutAttr = opaque { + const M = mlir.Methods(LayoutAttr, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsACuteLayout; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + shape: *const ShapeAttr, + stride: *const StrideAttr, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const LayoutAttr { + const result = c.mlirCuteLayoutAttrGet(ctx.ptr(), args.shape.ptr(), args.stride.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + + pub fn value(self: *const LayoutAttr) []const u8 { + return mlir.string(c.mlirCuteLayoutAttrGetValue(self.ptr())); + } + + pub fn attribute(self: *const LayoutAttr) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +pub const TileAttr = opaque { + const M = mlir.Methods(TileAttr, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsACuteTile; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub fn getLayout(ctx: *mlir.Context, layout: *const LayoutAttr) mlir.Error!*const TileAttr { + return @ptrCast(c.mlirCuteTileAttrLayoutGet(ctx.ptr(), layout.ptr()).ptr orelse return error.InvalidMlir); + } + + pub fn getUnderscore(ctx: *mlir.Context) mlir.Error!*const TileAttr { + return @ptrCast(c.mlirCuteTileAttrUnderscoreGet(ctx.ptr()).ptr orelse return error.InvalidMlir); + } + + pub fn getTuple(ctx: *mlir.Context, elements: []const *const TileAttr) mlir.Error!*const TileAttr { + return @ptrCast(c.mlirCuteTileAttrTupleGet(ctx.ptr(), @intCast(elements.len), @ptrCast(elements)).ptr orelse return error.InvalidMlir); + } + + pub fn value(self: *const TileAttr) []const u8 { + return mlir.string(c.mlirCuteTileAttrGetValue(self.ptr())); + } + + pub fn attribute(self: *const TileAttr) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +pub const ComposedLayoutAttr = opaque { + const M = mlir.Methods(ComposedLayoutAttr, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsACuteComposedLayout; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + /// `inner ∘ offset ∘ outer`. + pub const InitArgs = struct { + inner: union(enum) { + swizzle: *const SwizzleAttr, + layout: *const LayoutAttr, + }, + offset: *const IntTupleAttr, + outer: *const LayoutAttr, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const ComposedLayoutAttr { + const inner = switch (args.inner) { + inline else => |attr| attr.ptr(), + }; + const result = c.mlirCuteComposedLayoutAttrGet(ctx.ptr(), inner, args.offset.ptr(), args.outer.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + + pub fn value(self: *const ComposedLayoutAttr) []const u8 { + return mlir.string(c.mlirCuteComposedLayoutAttrGetValue(self.ptr())); + } + + pub fn attribute(self: *const ComposedLayoutAttr) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +pub const SwizzleAttr = opaque { + const M = mlir.Methods(SwizzleAttr, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsACuteSwizzle; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + /// `S`. + pub const InitArgs = struct { + bits: u32, + base: u32, + shift: i32, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const SwizzleAttr { + const result = c.mlirCuteSwizzleAttrGet(ctx.ptr(), args.bits, args.base, args.shift); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + + pub fn value(self: *const SwizzleAttr) []const u8 { + return mlir.string(c.mlirCuteSwizzleAttrGetValue(self.ptr())); + } + + pub fn attribute(self: *const SwizzleAttr) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +// Enums, generated from the dialect's .td files; values are the compiler's. + +/// Address spaces for CuTe memrefs and pointers. +pub const AddressSpace = enum(u32) { + generic = 0, + gmem = 1, + smem = 3, + rmem = 5, + tmem = 6, + dsmem = 7, + cmem = 4, +}; + +/// Copy operand ID. +pub const CopyOperand = enum(u32) { + S = 0, + D = 1, +}; + +/// Major mode for MMA operations. +pub const MajorMode = enum(u32) { + k = 0, + mn = 1, +}; + +/// MMA operand ID. +pub const MmaOperand = enum(u32) { + A = 0, + B = 1, + C = 2, + D = 3, + SFA = 4, + SFB = 5, + E = 6, +}; + +/// Op for cute reduce operations. +pub const ReductionOp = enum(u32) { + ADD = 0, + MUL = 1, + MIN = 2, + MAX = 3, +}; + +// Types and attributes, generated from the dialect's .td files. + +/// `!cute.arith_tuple_iter`: Iterator over an arithmetic tuple. +pub const ArithTupleIteratorType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteArithTupleIterator; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "arith_tuple_iter"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + arithTuple: *const mlir.Type, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteArithTupleIteratorTypeGet(ctx.ptr(), args.arithTuple.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn arithTuple(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteArithTupleIteratorTypeGetArithTuple(self.ptr()).ptr.?); + } +}; + +/// `!cute.composed_layout`: CuTe composed layout: A ∘ offset ∘ B. +pub const ComposedLayoutType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteComposedLayout; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "composed_layout"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + attr: *const mlir.Attribute, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteComposedLayoutTypeGet(ctx.ptr(), args.attr.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn attr(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteComposedLayoutTypeGetAttr(self.ptr()).ptr.?); + } +}; + +/// `!cute.i`: Integer of any width with known divisibility and power of two. +pub const ConstrainedIntType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteConstrainedInt; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + divisibility: i64, + width: u32, + isPow2: bool, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteConstrainedIntTypeGet(ctx.ptr(), args.divisibility, args.width, args.isPow2); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn divisibility(self: *const Self) i64 { + return c.mlirCuteConstrainedIntTypeGetDivisibility(self.ptr()); + } + pub fn width(self: *const Self) u32 { + return c.mlirCuteConstrainedIntTypeGetWidth(self.ptr()); + } + pub fn isPow2(self: *const Self) bool { + return c.mlirCuteConstrainedIntTypeGetIsPow2(self.ptr()); + } +}; + +/// `!cute.coord_tensor`: Coordinate tensor: an arithmetic tuple with a layout. +pub const CoordTensorType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteCoordTensor; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "coord_tensor"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + arithTuple: *const mlir.Type, + layout: *const mlir.Type, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteCoordTensorTypeGet(ctx.ptr(), args.arithTuple.ptr(), args.layout.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn arithTuple(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteCoordTensorTypeGetArithTuple(self.ptr()).ptr.?); + } + pub fn layout(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteCoordTensorTypeGetLayout(self.ptr()).ptr.?); + } +}; + +/// `!cute.coord`: Scalar integer, underscore wildcard, or recursive tuple of coordinates. +pub const CoordType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteCoord; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "coord"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + attr: *const mlir.Attribute, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteCoordTypeGet(ctx.ptr(), args.attr.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn attr(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteCoordTypeGetAttr(self.ptr()).ptr.?); + } +}; + +/// `!cute.fast_divmod_divisor`: Precomputed divisor of the fast divmod operations. +pub const FastDivmodDivisorType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteFastDivmodDivisor; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "fast_divmod_divisor"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + width: u32, + isPow2: bool, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteFastDivmodDivisorTypeGet(ctx.ptr(), args.width, args.isPow2); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn width(self: *const Self) u32 { + return c.mlirCuteFastDivmodDivisorTypeGetWidth(self.ptr()); + } + pub fn isPow2(self: *const Self) bool { + return c.mlirCuteFastDivmodDivisorTypeGetIsPow2(self.ptr()); + } +}; + +/// `!cute.int_tuple`: Scalar integer or recursive tuple of integers. +pub const IntTupleType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteIntTuple; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "int_tuple"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + attr: *const mlir.Attribute, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteIntTupleTypeGet(ctx.ptr(), args.attr.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn attr(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteIntTupleTypeGetAttr(self.ptr()).ptr.?); + } +}; + +/// `!cute.layout`: CuTe layout: a shape/stride pair. +pub const LayoutType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteLayout; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "layout"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + attr: *const mlir.Attribute, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteLayoutTypeGet(ctx.ptr(), args.attr.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn attr(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteLayoutTypeGetAttr(self.ptr()).ptr.?); + } +}; + +/// `!cute.memref`: Tensor view: a pointer with a layout. +pub const MemRefType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteMemRef; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "memref"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + ptr: *const mlir.Type, + layout: *const mlir.Type, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteMemRefTypeGet(ctx.ptr(), args.ptr.ptr(), args.layout.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn ptr_(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteMemRefTypeGetPtr(self.ptr()).ptr.?); + } + pub fn layout(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteMemRefTypeGetLayout(self.ptr()).ptr.?); + } +}; + +/// `!cute.ptr`: Pointer with address space, alignment, swizzle and bit layout. +pub const PtrType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACutePtr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "ptr"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + valueType: ?*const mlir.Type = null, + addressSpace: AddressSpace, + alignment: u64, + swizzle: ?*const mlir.Attribute = null, + bitlayout: ?*const mlir.Attribute = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCutePtrTypeGet(ctx.ptr(), if (args.valueType) |v| v.ptr() else c.MlirType{ .ptr = null }, @backingInt(args.addressSpace), args.alignment, if (args.swizzle) |v| v.ptr() else c.MlirAttribute{ .ptr = null }, if (args.bitlayout) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn valueType(self: *const Self) ?*const mlir.Type { + return @ptrCast(c.mlirCutePtrTypeGetValueType(self.ptr()).ptr); + } + pub fn addressSpace(self: *const Self) AddressSpace { + return @fromBackingInt(@intCast(c.mlirCutePtrTypeGetAddressSpace(self.ptr()))); + } + pub fn alignment(self: *const Self) u64 { + return c.mlirCutePtrTypeGetAlignment(self.ptr()); + } + pub fn swizzle(self: *const Self) ?*const mlir.Attribute { + return @ptrCast(c.mlirCutePtrTypeGetSwizzle(self.ptr()).ptr); + } + pub fn bitlayout(self: *const Self) ?*const mlir.Attribute { + return @ptrCast(c.mlirCutePtrTypeGetBitlayout(self.ptr()).ptr); + } +}; + +/// `!cute.shape`: Scalar integer or recursive tuple of shape extents. +pub const ShapeType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteShape; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "shape"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + attr: *const mlir.Attribute, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteShapeTypeGet(ctx.ptr(), args.attr.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn attr(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteShapeTypeGetAttr(self.ptr()).ptr.?); + } +}; + +/// `!cute.sparse_elem`: Logical sparse element stored in a physical type. +pub const SparseElemType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteSparseElem; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "sparse_elem"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + numLogical: c_int, + physicalType: *const mlir.Type, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteSparseElemTypeGet(ctx.ptr(), args.numLogical, args.physicalType.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn numLogical(self: *const Self) c_int { + return c.mlirCuteSparseElemTypeGetNumLogical(self.ptr()); + } + pub fn physicalType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteSparseElemTypeGetPhysicalType(self.ptr()).ptr.?); + } +}; + +/// `!cute.stride`: Scalar or recursive tuple of stride elements. +pub const StrideType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteStride; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "stride"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + attr: *const mlir.Attribute, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteStrideTypeGet(ctx.ptr(), args.attr.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn attr(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteStrideTypeGetAttr(self.ptr()).ptr.?); + } +}; + +/// `!cute.swizzle`: CuTe swizzle: S. +pub const SwizzleType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteSwizzle; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "swizzle"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + attr: *const mlir.Attribute, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteSwizzleTypeGet(ctx.ptr(), args.attr.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn attr(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSwizzleTypeGetAttr(self.ptr()).ptr.?); + } +}; + +/// `!cute.tile`: CuTe tile: a recursive sequence of layouts and underscores. +pub const TileType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteTile; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "tile"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + attr: *const mlir.Attribute, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteTileTypeGet(ctx.ptr(), args.attr.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn attr(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteTileTypeGetAttr(self.ptr()).ptr.?); + } +}; + +/// `!cute.tiled_copy`: Copy atom tiled over threads and values. +pub const TiledCopyType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteTiledCopy; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "tiled_copy"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + copyAtom: *const mlir.Type, + layoutCopyTv: *const mlir.Attribute, + tilerMn: *const mlir.Attribute, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteTiledCopyTypeGet(ctx.ptr(), args.copyAtom.ptr(), args.layoutCopyTv.ptr(), args.tilerMn.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn copyAtom(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteTiledCopyTypeGetCopyAtom(self.ptr()).ptr.?); + } + pub fn layoutCopyTv(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteTiledCopyTypeGetLayoutCopyTv(self.ptr()).ptr.?); + } + pub fn tilerMn(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteTiledCopyTypeGetTilerMn(self.ptr()).ptr.?); + } + pub const atom = copyAtom; +}; + +/// `!cute.tiled_copy_v2`: V2 copy atom tiled over threads and values. +pub const TiledCopyV2Type = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteTiledCopyV2; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "tiled_copy_v2"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + copyAtom: *const mlir.Type, + layoutCopyTv: *const mlir.Attribute, + tilerMn: *const mlir.Attribute, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteTiledCopyV2TypeGet(ctx.ptr(), args.copyAtom.ptr(), args.layoutCopyTv.ptr(), args.tilerMn.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn copyAtom(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteTiledCopyV2TypeGetCopyAtom(self.ptr()).ptr.?); + } + pub fn layoutCopyTv(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteTiledCopyV2TypeGetLayoutCopyTv(self.ptr()).ptr.?); + } + pub fn tilerMn(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteTiledCopyV2TypeGetTilerMn(self.ptr()).ptr.?); + } +}; + +/// `!cute.tiled_mma`: MMA atom tiled over MNK. +pub const TiledMmaType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteTiledMma; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "tiled_mma"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + mmaAtom: *const mlir.Type, + atomLayoutMNK: *const mlir.Attribute, + permutationMNK: ?*const mlir.Attribute = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteTiledMmaTypeGet(ctx.ptr(), args.mmaAtom.ptr(), args.atomLayoutMNK.ptr(), if (args.permutationMNK) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn mmaAtom(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteTiledMmaTypeGetMmaAtom(self.ptr()).ptr.?); + } + pub fn atomLayoutMNK(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteTiledMmaTypeGetAtomLayoutMNK(self.ptr()).ptr.?); + } + pub fn permutationMNK(self: *const Self) ?*const mlir.Attribute { + return @ptrCast(c.mlirCuteTiledMmaTypeGetPermutationMNK(self.ptr()).ptr); + } + pub const atom = mmaAtom; + pub const getAtomLayoutMnk = atomLayoutMNK; + pub const getPermutationMnk = permutationMNK; +}; + +/// `!cute.tuple`: Non-empty tuple of types. +pub const TupleType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteTuple; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "tuple"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + types: []const *const mlir.Type, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteTupleTypeGet(ctx.ptr(), @intCast(args.types.len), @ptrCast(args.types.ptr)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn numTypes(self: *const Self) usize { + return @intCast(c.mlirCuteTupleTypeGetNumTypes(self.ptr())); + } + pub fn @"type"(self: *const Self, pos: usize) *const mlir.Type { + return @ptrCast(c.mlirCuteTupleTypeGetType(self.ptr(), @intCast(pos)).ptr.?); + } +}; + +/// `#cute.bitlayout`: Bit layout of a pointer's elements within a storage chunk. +pub const BitLayoutAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteBitLayout; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "bitlayout"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + layout: *const mlir.Attribute, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteBitLayoutAttrGet(ctx.ptr(), args.layout.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn layout(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteBitLayoutAttrGetLayout(self.ptr()).ptr.?); + } +}; + +/// `#cute.copy_atom`: Thread and value layouts of a copy atom. +pub const CopyAtomAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteCopyAtom; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "copy_atom"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + thrId: *const mlir.Attribute, + layoutSrc: *const mlir.Attribute, + layoutDst: *const mlir.Attribute, + layoutRef: *const mlir.Attribute, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteCopyAtomAttrGet(ctx.ptr(), args.thrId.ptr(), args.layoutSrc.ptr(), args.layoutDst.ptr(), args.layoutRef.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn thrId(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteCopyAtomAttrGetThrId(self.ptr()).ptr.?); + } + pub fn layoutSrc(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteCopyAtomAttrGetLayoutSrc(self.ptr()).ptr.?); + } + pub fn layoutDst(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteCopyAtomAttrGetLayoutDst(self.ptr()).ptr.?); + } + pub fn layoutRef(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteCopyAtomAttrGetLayoutRef(self.ptr()).ptr.?); + } +}; + +/// `#cute.copy_atom_v2`: Thread and value layouts and fragments of a V2 copy atom. +pub const CopyAtomV2Attr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteCopyAtomV2; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "copy_atom_v2"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + thrId: *const mlir.Attribute, + layoutSrcTV: *const mlir.Attribute, + layoutDstTV: *const mlir.Attribute, + frgSrc: *const mlir.Attribute, + frgDst: *const mlir.Attribute, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteCopyAtomV2AttrGet(ctx.ptr(), args.thrId.ptr(), args.layoutSrcTV.ptr(), args.layoutDstTV.ptr(), args.frgSrc.ptr(), args.frgDst.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn thrId(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteCopyAtomV2AttrGetThrId(self.ptr()).ptr.?); + } + pub fn layoutSrcTV(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteCopyAtomV2AttrGetLayoutSrcTV(self.ptr()).ptr.?); + } + pub fn layoutDstTV(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteCopyAtomV2AttrGetLayoutDstTV(self.ptr()).ptr.?); + } + pub fn frgSrc(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteCopyAtomV2AttrGetFrgSrc(self.ptr()).ptr.?); + } + pub fn frgDst(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteCopyAtomV2AttrGetFrgDst(self.ptr()).ptr.?); + } +}; + +/// `#cute.mma_atom`: Shapes, thread-value layouts and fragments of an MMA atom. +pub const MmaAtomAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteMmaAtom; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "mma_atom"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + shapeMNK: *const mlir.Attribute, + thrId: *const mlir.Attribute, + layoutATV: *const mlir.Attribute, + layoutBTV: *const mlir.Attribute, + layoutCTV: *const mlir.Attribute, + shapeAMK: *const mlir.Attribute, + shapeBNK: *const mlir.Attribute, + shapeCMN: *const mlir.Attribute, + frgA: *const mlir.Attribute, + frgB: *const mlir.Attribute, + frgC: *const mlir.Attribute, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteMmaAtomAttrGet(ctx.ptr(), args.shapeMNK.ptr(), args.thrId.ptr(), args.layoutATV.ptr(), args.layoutBTV.ptr(), args.layoutCTV.ptr(), args.shapeAMK.ptr(), args.shapeBNK.ptr(), args.shapeCMN.ptr(), args.frgA.ptr(), args.frgB.ptr(), args.frgC.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn shapeMNK(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMmaAtomAttrGetShapeMNK(self.ptr()).ptr.?); + } + pub fn thrId(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMmaAtomAttrGetThrId(self.ptr()).ptr.?); + } + pub fn layoutATV(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMmaAtomAttrGetLayoutATV(self.ptr()).ptr.?); + } + pub fn layoutBTV(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMmaAtomAttrGetLayoutBTV(self.ptr()).ptr.?); + } + pub fn layoutCTV(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMmaAtomAttrGetLayoutCTV(self.ptr()).ptr.?); + } + pub fn shapeAMK(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMmaAtomAttrGetShapeAMK(self.ptr()).ptr.?); + } + pub fn shapeBNK(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMmaAtomAttrGetShapeBNK(self.ptr()).ptr.?); + } + pub fn shapeCMN(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMmaAtomAttrGetShapeCMN(self.ptr()).ptr.?); + } + pub fn frgA(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMmaAtomAttrGetFrgA(self.ptr()).ptr.?); + } + pub fn frgB(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMmaAtomAttrGetFrgB(self.ptr()).ptr.?); + } + pub fn frgC(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMmaAtomAttrGetFrgC(self.ptr()).ptr.?); + } +}; + +/// `#cute.mx_mma_atom`: MMA atom attribute with scale-factor operands. +pub const MxMmaAtomAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteMxMmaAtom; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "mx_mma_atom"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + shapeMNK: *const mlir.Attribute, + thrId: *const mlir.Attribute, + layoutATV: *const mlir.Attribute, + layoutSFATV: *const mlir.Attribute, + layoutBTV: *const mlir.Attribute, + layoutSFBTV: *const mlir.Attribute, + layoutCTV: *const mlir.Attribute, + shapeAMK: *const mlir.Attribute, + shapeSFAMK: *const mlir.Attribute, + shapeBNK: *const mlir.Attribute, + shapeSFBNK: *const mlir.Attribute, + shapeCMN: *const mlir.Attribute, + frgA: *const mlir.Attribute, + frgSFA: *const mlir.Attribute, + frgB: *const mlir.Attribute, + frgSFB: *const mlir.Attribute, + frgC: *const mlir.Attribute, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteMxMmaAtomAttrGet(ctx.ptr(), args.shapeMNK.ptr(), args.thrId.ptr(), args.layoutATV.ptr(), args.layoutSFATV.ptr(), args.layoutBTV.ptr(), args.layoutSFBTV.ptr(), args.layoutCTV.ptr(), args.shapeAMK.ptr(), args.shapeSFAMK.ptr(), args.shapeBNK.ptr(), args.shapeSFBNK.ptr(), args.shapeCMN.ptr(), args.frgA.ptr(), args.frgSFA.ptr(), args.frgB.ptr(), args.frgSFB.ptr(), args.frgC.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn shapeMNK(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMxMmaAtomAttrGetShapeMNK(self.ptr()).ptr.?); + } + pub fn thrId(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMxMmaAtomAttrGetThrId(self.ptr()).ptr.?); + } + pub fn layoutATV(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMxMmaAtomAttrGetLayoutATV(self.ptr()).ptr.?); + } + pub fn layoutSFATV(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMxMmaAtomAttrGetLayoutSFATV(self.ptr()).ptr.?); + } + pub fn layoutBTV(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMxMmaAtomAttrGetLayoutBTV(self.ptr()).ptr.?); + } + pub fn layoutSFBTV(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMxMmaAtomAttrGetLayoutSFBTV(self.ptr()).ptr.?); + } + pub fn layoutCTV(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMxMmaAtomAttrGetLayoutCTV(self.ptr()).ptr.?); + } + pub fn shapeAMK(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMxMmaAtomAttrGetShapeAMK(self.ptr()).ptr.?); + } + pub fn shapeSFAMK(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMxMmaAtomAttrGetShapeSFAMK(self.ptr()).ptr.?); + } + pub fn shapeBNK(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMxMmaAtomAttrGetShapeBNK(self.ptr()).ptr.?); + } + pub fn shapeSFBNK(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMxMmaAtomAttrGetShapeSFBNK(self.ptr()).ptr.?); + } + pub fn shapeCMN(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMxMmaAtomAttrGetShapeCMN(self.ptr()).ptr.?); + } + pub fn frgA(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMxMmaAtomAttrGetFrgA(self.ptr()).ptr.?); + } + pub fn frgSFA(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMxMmaAtomAttrGetFrgSFA(self.ptr()).ptr.?); + } + pub fn frgB(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMxMmaAtomAttrGetFrgB(self.ptr()).ptr.?); + } + pub fn frgSFB(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMxMmaAtomAttrGetFrgSFB(self.ptr()).ptr.?); + } + pub fn frgC(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteMxMmaAtomAttrGetFrgC(self.ptr()).ptr.?); + } +}; + +/// `#cute.reduction_op`: Op for cute reduce operations. +pub const ReductionOpAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteReductionOp; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "reduction_op"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: ReductionOp, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteReductionOpAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) ReductionOp { + return @fromBackingInt(@intCast(c.mlirCuteReductionOpAttrGetValue(self.ptr()))); + } +}; + +/// `#cute.sparse_mma_atom`: MMA atom attribute with a sparsity-metadata operand. +pub const SparseMmaAtomAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteSparseMmaAtom; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "sparse_mma_atom"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + shapeMNK: *const mlir.Attribute, + thrId: *const mlir.Attribute, + layoutATV: *const mlir.Attribute, + layoutBTV: *const mlir.Attribute, + layoutCTV: *const mlir.Attribute, + layoutETV: *const mlir.Attribute, + shapeAMK: *const mlir.Attribute, + shapeBNK: *const mlir.Attribute, + shapeCMN: *const mlir.Attribute, + shapeEMK: *const mlir.Attribute, + frgA: *const mlir.Attribute, + frgB: *const mlir.Attribute, + frgC: *const mlir.Attribute, + frgE: *const mlir.Attribute, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteSparseMmaAtomAttrGet(ctx.ptr(), args.shapeMNK.ptr(), args.thrId.ptr(), args.layoutATV.ptr(), args.layoutBTV.ptr(), args.layoutCTV.ptr(), args.layoutETV.ptr(), args.shapeAMK.ptr(), args.shapeBNK.ptr(), args.shapeCMN.ptr(), args.shapeEMK.ptr(), args.frgA.ptr(), args.frgB.ptr(), args.frgC.ptr(), args.frgE.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn shapeMNK(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetShapeMNK(self.ptr()).ptr.?); + } + pub fn thrId(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetThrId(self.ptr()).ptr.?); + } + pub fn layoutATV(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetLayoutATV(self.ptr()).ptr.?); + } + pub fn layoutBTV(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetLayoutBTV(self.ptr()).ptr.?); + } + pub fn layoutCTV(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetLayoutCTV(self.ptr()).ptr.?); + } + pub fn layoutETV(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetLayoutETV(self.ptr()).ptr.?); + } + pub fn shapeAMK(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetShapeAMK(self.ptr()).ptr.?); + } + pub fn shapeBNK(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetShapeBNK(self.ptr()).ptr.?); + } + pub fn shapeCMN(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetShapeCMN(self.ptr()).ptr.?); + } + pub fn shapeEMK(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetShapeEMK(self.ptr()).ptr.?); + } + pub fn frgA(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetFrgA(self.ptr()).ptr.?); + } + pub fn frgB(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetFrgB(self.ptr()).ptr.?); + } + pub fn frgC(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetFrgC(self.ptr()).ptr.?); + } + pub fn frgE(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetFrgE(self.ptr()).ptr.?); + } +}; + +/// `#cute.sparse_mx_mma_atom`: MMA atom attribute with scale-factor and metadata operands. +pub const SparseMxMmaAtomAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteSparseMxMmaAtom; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "sparse_mx_mma_atom"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + shapeMNK: *const mlir.Attribute, + thrId: *const mlir.Attribute, + layoutATV: *const mlir.Attribute, + layoutSFATV: *const mlir.Attribute, + layoutBTV: *const mlir.Attribute, + layoutSFBTV: *const mlir.Attribute, + layoutCTV: *const mlir.Attribute, + layoutETV: *const mlir.Attribute, + shapeAMK: *const mlir.Attribute, + shapeSFAMK: *const mlir.Attribute, + shapeBNK: *const mlir.Attribute, + shapeSFBNK: *const mlir.Attribute, + shapeCMN: *const mlir.Attribute, + shapeEMK: *const mlir.Attribute, + frgA: *const mlir.Attribute, + frgSFA: *const mlir.Attribute, + frgB: *const mlir.Attribute, + frgSFB: *const mlir.Attribute, + frgC: *const mlir.Attribute, + frgE: *const mlir.Attribute, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteSparseMxMmaAtomAttrGet(ctx.ptr(), args.shapeMNK.ptr(), args.thrId.ptr(), args.layoutATV.ptr(), args.layoutSFATV.ptr(), args.layoutBTV.ptr(), args.layoutSFBTV.ptr(), args.layoutCTV.ptr(), args.layoutETV.ptr(), args.shapeAMK.ptr(), args.shapeSFAMK.ptr(), args.shapeBNK.ptr(), args.shapeSFBNK.ptr(), args.shapeCMN.ptr(), args.shapeEMK.ptr(), args.frgA.ptr(), args.frgSFA.ptr(), args.frgB.ptr(), args.frgSFB.ptr(), args.frgC.ptr(), args.frgE.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn shapeMNK(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetShapeMNK(self.ptr()).ptr.?); + } + pub fn thrId(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetThrId(self.ptr()).ptr.?); + } + pub fn layoutATV(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetLayoutATV(self.ptr()).ptr.?); + } + pub fn layoutSFATV(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetLayoutSFATV(self.ptr()).ptr.?); + } + pub fn layoutBTV(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetLayoutBTV(self.ptr()).ptr.?); + } + pub fn layoutSFBTV(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetLayoutSFBTV(self.ptr()).ptr.?); + } + pub fn layoutCTV(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetLayoutCTV(self.ptr()).ptr.?); + } + pub fn layoutETV(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetLayoutETV(self.ptr()).ptr.?); + } + pub fn shapeAMK(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetShapeAMK(self.ptr()).ptr.?); + } + pub fn shapeSFAMK(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetShapeSFAMK(self.ptr()).ptr.?); + } + pub fn shapeBNK(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetShapeBNK(self.ptr()).ptr.?); + } + pub fn shapeSFBNK(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetShapeSFBNK(self.ptr()).ptr.?); + } + pub fn shapeCMN(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetShapeCMN(self.ptr()).ptr.?); + } + pub fn shapeEMK(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetShapeEMK(self.ptr()).ptr.?); + } + pub fn frgA(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetFrgA(self.ptr()).ptr.?); + } + pub fn frgSFA(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetFrgSFA(self.ptr()).ptr.?); + } + pub fn frgB(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetFrgB(self.ptr()).ptr.?); + } + pub fn frgSFB(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetFrgSFB(self.ptr()).ptr.?); + } + pub fn frgC(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetFrgC(self.ptr()).ptr.?); + } + pub fn frgE(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetFrgE(self.ptr()).ptr.?); + } +}; + +// Operation builders, generated from the operations' .td. +/// `cute.add_offset`. Result types are explicit; attributes use mlir.Attribute. +pub fn add_offset(ctx: *mlir.Context, src: *const mlir.Value, offset: *const mlir.Value, dst_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.add_offset", .{ + .operands = .{ .flat = &.{ src, offset } }, + .results = .{ .flat = &.{dst_type} }, + .location = location, + }); +} + +/// `cute.append_to_rank`. Result types are explicit; attributes use mlir.Attribute. +pub fn append_to_rank(ctx: *mlir.Context, input: *const mlir.Value, element: *const mlir.Value, result_type: *const mlir.Type, rank: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.append_to_rank", .{ + .operands = .{ .flat = &.{ input, element } }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "rank", rank), + }, + .location = location, + }); +} + +/// `cute.apply_swizzle`. Result types are explicit; attributes use mlir.Attribute. +pub fn apply_swizzle( + ctx: *mlir.Context, + ptr: *const mlir.Value, + swizzled_ptr_type: *const mlir.Type, + opts: struct { + swizzle: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.swizzle) |value| attributes.appendAssumeCapacity(.named(ctx, "swizzle", value)); + return mlir.Operation.make(ctx, "cute.apply_swizzle", .{ + .operands = .{ .flat = &.{ptr} }, + .results = .{ .flat = &.{swizzled_ptr_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute.assume`. Result types are explicit; attributes use mlir.Attribute. +pub fn assume(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.assume", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .location = location, + }); +} + +/// `cute.blocked_product`. Result types are explicit; attributes use mlir.Attribute. +pub fn blocked_product(ctx: *mlir.Context, input: *const mlir.Value, tiler: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.blocked_product", .{ + .operands = .{ .flat = &.{ input, tiler } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.ceil_div`. Result types are explicit; attributes use mlir.Attribute. +pub fn ceil_div(ctx: *mlir.Context, input: *const mlir.Value, tiler: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.ceil_div", .{ + .operands = .{ .flat = &.{ input, tiler } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.coalesce`. Result types are explicit; attributes use mlir.Attribute. +pub fn coalesce( + ctx: *mlir.Context, + input: *const mlir.Value, + result_type: *const mlir.Type, + opts: struct { + target_profile: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.coalesce", .{ + .operands = .{ .variadic = &.{ + &.{input}, + if (opts.target_profile) |value| &.{value} else &.{}, + } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.complement`. Result types are explicit; attributes use mlir.Attribute. +pub fn complement(ctx: *mlir.Context, input: *const mlir.Value, cotarget: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.complement", .{ + .operands = .{ .flat = &.{ input, cotarget } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `cute.composed_get_inner`. Result types are explicit; attributes use mlir.Attribute. +pub fn composed_get_inner(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.composed_get_inner", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.composed_get_offset`. Result types are explicit; attributes use mlir.Attribute. +pub fn composed_get_offset(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.composed_get_offset", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.composed_get_outer`. Result types are explicit; attributes use mlir.Attribute. +pub fn composed_get_outer(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.composed_get_outer", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.composition`. Result types are explicit; attributes use mlir.Attribute. +pub fn composition(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.composition", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `cute.copy`. Result types are explicit; attributes use mlir.Attribute. +pub fn copy( + ctx: *mlir.Context, + copy_atom: *const mlir.Value, + src: []const *const mlir.Value, + dst: []const *const mlir.Value, + opts: struct { + pred: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.copy", .{ + .operands = .{ .variadic = &.{ + &.{copy_atom}, + src, + dst, + if (opts.pred) |value| &.{value} else &.{}, + } }, + .location = location, + }); +} + +/// `cute.copy.make_fragment`. Result types are explicit; attributes use mlir.Attribute. +pub fn copy_make_fragment(ctx: *mlir.Context, atom: *const mlir.Value, input: *const mlir.Value, result_type: *const mlir.Type, operand_id: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.copy.make_fragment", .{ + .operands = .{ .flat = &.{ atom, input } }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "operand_id", operand_id), + }, + .location = location, + }); +} + +/// `cute.copy_atom_call`. Result types are explicit; attributes use mlir.Attribute. +pub fn copy_atom_call( + ctx: *mlir.Context, + atom: *const mlir.Value, + src: []const *const mlir.Value, + dst: []const *const mlir.Value, + opts: struct { + pred: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.copy_atom_call", .{ + .operands = .{ .variadic = &.{ + &.{atom}, + src, + dst, + if (opts.pred) |value| &.{value} else &.{}, + } }, + .location = location, + }); +} + +/// `cute.cosize`. Result types are explicit; attributes use mlir.Attribute. +pub fn cosize( + ctx: *mlir.Context, + input: *const mlir.Value, + result_type: *const mlir.Type, + opts: struct { + mode: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); + return mlir.Operation.make(ctx, "cute.cosize", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute.crd2idx`. Result types are explicit; attributes use mlir.Attribute. +pub fn crd2idx(ctx: *mlir.Context, coord: *const mlir.Value, layout: *const mlir.Value, index_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.crd2idx", .{ + .operands = .{ .flat = &.{ coord, layout } }, + .results = .{ .flat = &.{index_type} }, + .location = location, + }); +} + +/// `cute.deref_arith_tuple_iter`. Result types are explicit; attributes use mlir.Attribute. +pub fn deref_arith_tuple_iter(ctx: *mlir.Context, iter: *const mlir.Value, arith_tuple_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.deref_arith_tuple_iter", .{ + .operands = .{ .flat = &.{iter} }, + .results = .{ .flat = &.{arith_tuple_type} }, + .location = location, + }); +} + +/// `cute.deref_desc_iter`. Result types are explicit; attributes use mlir.Attribute. +pub fn deref_desc_iter(ctx: *mlir.Context, iter: *const mlir.Value, value__type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.deref_desc_iter", .{ + .operands = .{ .flat = &.{iter} }, + .results = .{ .flat = &.{value__type} }, + .location = location, + }); +} + +/// `cute.derefine`. Result types are explicit; attributes use mlir.Attribute. +pub fn derefine(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.derefine", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .location = location, + }); +} + +/// `cute.dice`. Result types are explicit; attributes use mlir.Attribute. +pub fn dice(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, coord: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.dice", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "coord", coord), + }, + .location = location, + }); +} + +/// `cute.elem_less`. Result types are explicit; attributes use mlir.Attribute. +pub fn elem_less(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.elem_less", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.equal`. Result types are explicit; attributes use mlir.Attribute. +pub fn equal(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.equal", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.fast_divmod.compute`. Result types are explicit; attributes use mlir.Attribute. +pub fn fast_divmod_compute(ctx: *mlir.Context, dividend: *const mlir.Value, divisor: *const mlir.Value, quotient_type: *const mlir.Type, remainder_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.fast_divmod.compute", .{ + .operands = .{ .flat = &.{ dividend, divisor } }, + .results = .{ .flat = &.{ quotient_type, remainder_type } }, + .location = location, + }); +} + +/// `cute.fast_divmod.create_divisor`. Result types are explicit; attributes use mlir.Attribute. +pub fn fast_divmod_create_divisor(ctx: *mlir.Context, divisor: *const mlir.Value, fast_divmod_divisor_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.fast_divmod.create_divisor", .{ + .operands = .{ .flat = &.{divisor} }, + .results = .{ .flat = &.{fast_divmod_divisor_type} }, + .location = location, + }); +} + +/// `cute.fast_divmod.divide`. Result types are explicit; attributes use mlir.Attribute. +pub fn fast_divmod_divide(ctx: *mlir.Context, dividend: *const mlir.Value, divisor: *const mlir.Value, quotient_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.fast_divmod.divide", .{ + .operands = .{ .flat = &.{ dividend, divisor } }, + .results = .{ .flat = &.{quotient_type} }, + .location = location, + }); +} + +/// `cute.fast_divmod.get_aux`. Result types are explicit; attributes use mlir.Attribute. +pub fn fast_divmod_get_aux(ctx: *mlir.Context, divisor: *const mlir.Value, multiplier_type: *const mlir.Type, shift1_type: *const mlir.Type, shift2_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.fast_divmod.get_aux", .{ + .operands = .{ .flat = &.{divisor} }, + .results = .{ .flat = &.{ multiplier_type, shift1_type, shift2_type } }, + .location = location, + }); +} + +/// `cute.fast_divmod.get_divisor`. Result types are explicit; attributes use mlir.Attribute. +pub fn fast_divmod_get_divisor(ctx: *mlir.Context, fast_divmod_divisor: *const mlir.Value, divisor_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.fast_divmod.get_divisor", .{ + .operands = .{ .flat = &.{fast_divmod_divisor} }, + .results = .{ .flat = &.{divisor_type} }, + .location = location, + }); +} + +/// `cute.fast_divmod.make_divisor`. Result types are explicit; attributes use mlir.Attribute. +pub fn fast_divmod_make_divisor(ctx: *mlir.Context, divisor: *const mlir.Value, multiplier: *const mlir.Value, sh1: *const mlir.Value, sh2: *const mlir.Value, fast_divmod_divisor_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.fast_divmod.make_divisor", .{ + .operands = .{ .flat = &.{ divisor, multiplier, sh1, sh2 } }, + .results = .{ .flat = &.{fast_divmod_divisor_type} }, + .location = location, + }); +} + +/// `cute.filter`. Result types are explicit; attributes use mlir.Attribute. +pub fn filter(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.filter", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.filter_zeros`. Result types are explicit; attributes use mlir.Attribute. +pub fn filter_zeros( + ctx: *mlir.Context, + input: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + target_profile: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.filter_zeros", .{ + .operands = .{ .variadic = &.{ + &.{input}, + if (opts.target_profile) |value| &.{value} else &.{}, + } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `cute.flat_divide`. Result types are explicit; attributes use mlir.Attribute. +pub fn flat_divide(ctx: *mlir.Context, input: *const mlir.Value, tiler: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.flat_divide", .{ + .operands = .{ .flat = &.{ input, tiler } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.flat_product`. Result types are explicit; attributes use mlir.Attribute. +pub fn flat_product(ctx: *mlir.Context, input: *const mlir.Value, tiler: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.flat_product", .{ + .operands = .{ .flat = &.{ input, tiler } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.gemm`. Result types are explicit; attributes use mlir.Attribute. +pub fn gemm(ctx: *mlir.Context, mma_atom: *const mlir.Value, d: *const mlir.Value, a: []const *const mlir.Value, b: []const *const mlir.Value, c_: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.gemm", .{ + .operands = .{ .variadic = &.{ + &.{mma_atom}, + &.{d}, + a, + b, + &.{c_}, + } }, + .location = location, + }); +} + +/// `cute.get`. Result types are explicit; attributes use mlir.Attribute. +pub fn get( + ctx: *mlir.Context, + input: *const mlir.Value, + result_type: *const mlir.Type, + opts: struct { + mode: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); + return mlir.Operation.make(ctx, "cute.get", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute.get_flat_coord`. Result types are explicit; attributes use mlir.Attribute. +pub fn get_flat_coord(ctx: *mlir.Context, index: *const mlir.Value, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.get_flat_coord", .{ + .operands = .{ .flat = &.{ index, input } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.get_hier_coord`. Result types are explicit; attributes use mlir.Attribute. +pub fn get_hier_coord(ctx: *mlir.Context, index: *const mlir.Value, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.get_hier_coord", .{ + .operands = .{ .flat = &.{ index, input } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.get_integral_coord`. Result types are explicit; attributes use mlir.Attribute. +pub fn get_integral_coord(ctx: *mlir.Context, index: *const mlir.Value, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.get_integral_coord", .{ + .operands = .{ .flat = &.{ index, input } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.get_iter`. Result types are explicit; attributes use mlir.Attribute. +pub fn get_iter(ctx: *mlir.Context, source: *const mlir.Value, ptr_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.get_iter", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{ptr_type} }, + .location = location, + }); +} + +/// `cute.get_layout`. Result types are explicit; attributes use mlir.Attribute. +pub fn get_layout(ctx: *mlir.Context, input: *const mlir.Value, layout_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.get_layout", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{layout_type} }, + .location = location, + }); +} + +/// `cute.get_layouts_from_tile`. Result types are explicit; attributes use mlir.Attribute. +pub fn get_layouts_from_tile(ctx: *mlir.Context, tile: *const mlir.Value, layouts_types: []const *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.get_layouts_from_tile", .{ + .operands = .{ .flat = &.{tile} }, + .results = .{ .variadic = &.{ + layouts_types, + } }, + .location = location, + }); +} + +/// `cute.get_leaves`. Result types are explicit; attributes use mlir.Attribute. +pub fn get_leaves(ctx: *mlir.Context, input: *const mlir.Value, results__types: []const *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.get_leaves", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .variadic = &.{ + results__types, + } }, + .location = location, + }); +} + +/// `cute.get_scalars`. Result types are explicit; attributes use mlir.Attribute. +pub fn get_scalars( + ctx: *mlir.Context, + cute_value: *const mlir.Value, + scalars_types: []const *const mlir.Type, + opts: struct { + only_dynamic: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.only_dynamic) |value| attributes.appendAssumeCapacity(.named(ctx, "only_dynamic", value)); + return mlir.Operation.make(ctx, "cute.get_scalars", .{ + .operands = .{ .flat = &.{cute_value} }, + .results = .{ .variadic = &.{ + scalars_types, + } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute.get_shape`. Result types are explicit; attributes use mlir.Attribute. +pub fn get_shape(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.get_shape", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.get_stride`. Result types are explicit; attributes use mlir.Attribute. +pub fn get_stride(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.get_stride", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.group_modes`. Result types are explicit; attributes use mlir.Attribute. +pub fn group_modes(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, begin: *const mlir.Attribute, end: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.group_modes", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "begin", begin), + .named(ctx, "end", end), + }, + .location = location, + }); +} + +/// `cute.idx2crd`. Result types are explicit; attributes use mlir.Attribute. +pub fn idx2crd(ctx: *mlir.Context, index: *const mlir.Value, shape: *const mlir.Value, coord_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.idx2crd", .{ + .operands = .{ .flat = &.{ index, shape } }, + .results = .{ .flat = &.{coord_type} }, + .location = location, + }); +} + +/// `cute.increment_coord`. Result types are explicit; attributes use mlir.Attribute. +pub fn increment_coord(ctx: *mlir.Context, coord: *const mlir.Value, shape: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.increment_coord", .{ + .operands = .{ .flat = &.{ coord, shape } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.inttoptr`. Result types are explicit; attributes use mlir.Attribute. +pub fn inttoptr(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.inttoptr", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .location = location, + }); +} + +/// `cute.kernel_smem_size`. Result types are explicit; attributes use mlir.Attribute. +pub fn kernel_smem_size(ctx: *mlir.Context, size_type: *const mlir.Type, kernel_name: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.kernel_smem_size", .{ + .results = .{ .flat = &.{size_type} }, + .attributes = &.{ + .named(ctx, "kernel_name", kernel_name), + }, + .location = location, + }); +} + +/// `cute.left_inverse`. Result types are explicit; attributes use mlir.Attribute. +pub fn left_inverse(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.left_inverse", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.load_scaled_index`. Result types are explicit; attributes use mlir.Attribute. +pub fn load_scaled_index(ctx: *mlir.Context, ptr_a: *const mlir.Value, ptr_b: *const mlir.Value, index: *const mlir.Value, stride: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.load_scaled_index", .{ + .operands = .{ .flat = &.{ ptr_a, ptr_b, index, stride } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.local_partition`. Result types are explicit; attributes use mlir.Attribute. +pub fn local_partition(ctx: *mlir.Context, input: *const mlir.Value, tiler: *const mlir.Value, index: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.local_partition", .{ + .operands = .{ .flat = &.{ input, tiler, index } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.local_tile`. Result types are explicit; attributes use mlir.Attribute. +pub fn local_tile( + ctx: *mlir.Context, + input: *const mlir.Value, + tile: *const mlir.Value, + coord: *const mlir.Value, + result_type: *const mlir.Type, + opts: struct { + proj: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.proj) |value| attributes.appendAssumeCapacity(.named(ctx, "proj", value)); + return mlir.Operation.make(ctx, "cute.local_tile", .{ + .operands = .{ .flat = &.{ input, tile, coord } }, + .results = .{ .flat = &.{result_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute.logical_divide`. Result types are explicit; attributes use mlir.Attribute. +pub fn logical_divide(ctx: *mlir.Context, input: *const mlir.Value, tiler: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.logical_divide", .{ + .operands = .{ .flat = &.{ input, tiler } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.logical_product`. Result types are explicit; attributes use mlir.Attribute. +pub fn logical_product(ctx: *mlir.Context, input: *const mlir.Value, tiler: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.logical_product", .{ + .operands = .{ .flat = &.{ input, tiler } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.make_arith_tuple_iter`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_arith_tuple_iter( + ctx: *mlir.Context, + iter_type: *const mlir.Type, + opts: struct { + value_: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_arith_tuple_iter", .{ + .operands = .{ .variadic = &.{ + if (opts.value_) |value| &.{value} else &.{}, + } }, + .results = .{ .flat = &.{iter_type} }, + .location = location, + }); +} + +/// `cute.make_atom`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_atom(ctx: *mlir.Context, values: []const *const mlir.Value, atom_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_atom", .{ + .operands = .{ .variadic = &.{ + values, + } }, + .results = .{ .flat = &.{atom_type} }, + .location = location, + }); +} + +/// `cute.make_composed_layout`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_composed_layout(ctx: *mlir.Context, inner: *const mlir.Value, offset: *const mlir.Value, outer: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_composed_layout", .{ + .operands = .{ .flat = &.{ inner, offset, outer } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.make_coord`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_coord(ctx: *mlir.Context, dynamicElements: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_coord", .{ + .operands = .{ .variadic = &.{ + dynamicElements, + } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.make_desc_iter`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_desc_iter(ctx: *mlir.Context, value_: *const mlir.Value, iter_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_desc_iter", .{ + .operands = .{ .flat = &.{value_} }, + .results = .{ .flat = &.{iter_type} }, + .location = location, + }); +} + +/// `cute.make_fragment_like`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_fragment_like(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_fragment_like", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .location = location, + }); +} + +/// `cute.make_identity_layout`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_identity_layout(ctx: *mlir.Context, shape: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_identity_layout", .{ + .operands = .{ .flat = &.{shape} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.make_identity_tensor`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_identity_tensor(ctx: *mlir.Context, shape: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_identity_tensor", .{ + .operands = .{ .flat = &.{shape} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.make_int_tuple`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_int_tuple(ctx: *mlir.Context, dynamicElements: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_int_tuple", .{ + .operands = .{ .variadic = &.{ + dynamicElements, + } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.make_layout`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_layout( + ctx: *mlir.Context, + layout_type: *const mlir.Type, + opts: struct { + shape: ?*const mlir.Value = null, + stride: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_layout", .{ + .operands = .{ .variadic = &.{ + if (opts.shape) |value| &.{value} else &.{}, + if (opts.stride) |value| &.{value} else &.{}, + } }, + .results = .{ .flat = &.{layout_type} }, + .location = location, + }); +} + +/// `cute.make_layout_like`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_layout_like(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_layout_like", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .location = location, + }); +} + +/// `cute.make_ordered_layout`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_ordered_layout(ctx: *mlir.Context, shape: *const mlir.Value, order: *const mlir.Value, layout_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_ordered_layout", .{ + .operands = .{ .flat = &.{ shape, order } }, + .results = .{ .flat = &.{layout_type} }, + .location = location, + }); +} + +/// `cute.make_shape`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_shape(ctx: *mlir.Context, dynamicElements: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_shape", .{ + .operands = .{ .variadic = &.{ + dynamicElements, + } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.make_sparse_elem`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_sparse_elem(ctx: *mlir.Context, physical_storage: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_sparse_elem", .{ + .operands = .{ .flat = &.{physical_storage} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.make_stride`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_stride(ctx: *mlir.Context, dynamicElements: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_stride", .{ + .operands = .{ .variadic = &.{ + dynamicElements, + } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.make_tile`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_tile(ctx: *mlir.Context, dynamicElements: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_tile", .{ + .operands = .{ .variadic = &.{ + dynamicElements, + } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.make_tiled_copy`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_tiled_copy(ctx: *mlir.Context, atom: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_tiled_copy", .{ + .operands = .{ .flat = &.{atom} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.make_tiled_copy_v2`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_tiled_copy_v2(ctx: *mlir.Context, atom: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_tiled_copy_v2", .{ + .operands = .{ .flat = &.{atom} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.make_tiled_mma`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_tiled_mma(ctx: *mlir.Context, atom: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_tiled_mma", .{ + .operands = .{ .flat = &.{atom} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.make_tuple`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_tuple(ctx: *mlir.Context, values: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_tuple", .{ + .operands = .{ .variadic = &.{ + values, + } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.make_view`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_view( + ctx: *mlir.Context, + iter: *const mlir.Value, + result_type: *const mlir.Type, + opts: struct { + layout: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.make_view", .{ + .operands = .{ .variadic = &.{ + &.{iter}, + if (opts.layout) |value| &.{value} else &.{}, + } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.memref.alloc_smem`. Result types are explicit; attributes use mlir.Attribute. +pub fn memref_alloc_smem(ctx: *mlir.Context, memref_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.memref.alloc_smem", .{ + .results = .{ .flat = &.{memref_type} }, + .location = location, + }); +} + +/// `cute.memref.alloca`. Result types are explicit; attributes use mlir.Attribute. +pub fn memref_alloca( + ctx: *mlir.Context, + result_type: *const mlir.Type, + opts: struct { + layout: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.memref.alloca", .{ + .operands = .{ .variadic = &.{ + if (opts.layout) |value| &.{value} else &.{}, + } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.memref.load`. Result types are explicit; attributes use mlir.Attribute. +pub fn memref_load(ctx: *mlir.Context, src: *const mlir.Value, coord: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.memref.load", .{ + .operands = .{ .flat = &.{ src, coord } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.memref.load_vec`. Result types are explicit; attributes use mlir.Attribute. +pub fn memref_load_vec( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + opts: struct { + mask: ?*const mlir.Value = null, + pass_thru: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.memref.load_vec", .{ + .operands = .{ .variadic = &.{ + &.{src}, + if (opts.mask) |value| &.{value} else &.{}, + if (opts.pass_thru) |value| &.{value} else &.{}, + } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.memref.store`. Result types are explicit; attributes use mlir.Attribute. +pub fn memref_store(ctx: *mlir.Context, dst: *const mlir.Value, coord: *const mlir.Value, value_: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.memref.store", .{ + .operands = .{ .flat = &.{ dst, coord, value_ } }, + .location = location, + }); +} + +/// `cute.memref.store_vec`. Result types are explicit; attributes use mlir.Attribute. +pub fn memref_store_vec( + ctx: *mlir.Context, + value_: *const mlir.Value, + dst: *const mlir.Value, + opts: struct { + mask: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.memref.store_vec", .{ + .operands = .{ .variadic = &.{ + &.{value_}, + &.{dst}, + if (opts.mask) |value| &.{value} else &.{}, + } }, + .location = location, + }); +} + +/// `cute.mma.make_fragment`. Result types are explicit; attributes use mlir.Attribute. +pub fn mma_make_fragment(ctx: *mlir.Context, atom: *const mlir.Value, input: *const mlir.Value, result_type: *const mlir.Type, operand_id: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.mma.make_fragment", .{ + .operands = .{ .flat = &.{ atom, input } }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "operand_id", operand_id), + }, + .location = location, + }); +} + +/// `cute.mma_atom_call`. Result types are explicit; attributes use mlir.Attribute. +pub fn mma_atom_call(ctx: *mlir.Context, atom: *const mlir.Value, d: *const mlir.Value, a: []const *const mlir.Value, b: []const *const mlir.Value, c_: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.mma_atom_call", .{ + .operands = .{ .variadic = &.{ + &.{atom}, + &.{d}, + a, + b, + &.{c_}, + } }, + .location = location, + }); +} + +/// `cute.prefetch`. Result types are explicit; attributes use mlir.Attribute. +pub fn prefetch(ctx: *mlir.Context, prefetch_atom: *const mlir.Value, src: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.prefetch", .{ + .operands = .{ .variadic = &.{ + &.{prefetch_atom}, + src, + } }, + .location = location, + }); +} + +/// `cute.prefetch_atom_call`. Result types are explicit; attributes use mlir.Attribute. +pub fn prefetch_atom_call(ctx: *mlir.Context, atom: *const mlir.Value, src: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.prefetch_atom_call", .{ + .operands = .{ .variadic = &.{ + &.{atom}, + src, + } }, + .location = location, + }); +} + +/// `cute.prepend_to_rank`. Result types are explicit; attributes use mlir.Attribute. +pub fn prepend_to_rank(ctx: *mlir.Context, input: *const mlir.Value, element: *const mlir.Value, result_type: *const mlir.Type, rank: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.prepend_to_rank", .{ + .operands = .{ .flat = &.{ input, element } }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "rank", rank), + }, + .location = location, + }); +} + +/// `cute.print`. Result types are explicit; attributes use mlir.Attribute. +pub fn print( + ctx: *mlir.Context, + values: []const *const mlir.Value, + opts: struct { + fmt: ?*const mlir.Attribute = null, + operand_signed: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.fmt) |value| attributes.appendAssumeCapacity(.named(ctx, "fmt", value)); + if (opts.operand_signed) |value| attributes.appendAssumeCapacity(.named(ctx, "operand_signed", value)); + return mlir.Operation.make(ctx, "cute.print", .{ + .operands = .{ .variadic = &.{ + values, + } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute.print_tma_desc_im2col`. Result types are explicit; attributes use mlir.Attribute. +pub fn print_tma_desc_im2col(ctx: *mlir.Context, input: *const mlir.Value, fd: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.print_tma_desc_im2col", .{ + .operands = .{ .flat = &.{ input, fd } }, + .location = location, + }); +} + +/// `cute.print_tma_desc_tiled`. Result types are explicit; attributes use mlir.Attribute. +pub fn print_tma_desc_tiled(ctx: *mlir.Context, input: *const mlir.Value, fd: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.print_tma_desc_tiled", .{ + .operands = .{ .flat = &.{ input, fd } }, + .location = location, + }); +} + +/// `cute.print_view`. Result types are explicit; attributes use mlir.Attribute. +pub fn print_view( + ctx: *mlir.Context, + src: *const mlir.Value, + opts: struct { + coord: ?*const mlir.Value = null, + verbose: ?*const mlir.Attribute = null, + is_signed: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.verbose) |value| attributes.appendAssumeCapacity(.named(ctx, "verbose", value)); + if (opts.is_signed) |value| attributes.appendAssumeCapacity(.named(ctx, "is_signed", value)); + return mlir.Operation.make(ctx, "cute.print_view", .{ + .operands = .{ .variadic = &.{ + &.{src}, + if (opts.coord) |value| &.{value} else &.{}, + } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute.ptr.load`. Result types are explicit; attributes use mlir.Attribute. +pub fn ptr_load(ctx: *mlir.Context, ptr: *const mlir.Value, value__type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.ptr.load", .{ + .operands = .{ .flat = &.{ptr} }, + .results = .{ .flat = &.{value__type} }, + .location = location, + }); +} + +/// `cute.ptr.store`. Result types are explicit; attributes use mlir.Attribute. +pub fn ptr_store(ctx: *mlir.Context, value_: *const mlir.Value, ptr: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.ptr.store", .{ + .operands = .{ .flat = &.{ value_, ptr } }, + .location = location, + }); +} + +/// `cute.ptrtoint`. Result types are explicit; attributes use mlir.Attribute. +pub fn ptrtoint(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.ptrtoint", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .location = location, + }); +} + +/// `cute.raked_product`. Result types are explicit; attributes use mlir.Attribute. +pub fn raked_product(ctx: *mlir.Context, input: *const mlir.Value, tiler: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.raked_product", .{ + .operands = .{ .flat = &.{ input, tiler } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.random_delay`. Result types are explicit; attributes use mlir.Attribute. +pub fn random_delay(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.random_delay", .{ + .location = location, + }); +} + +/// `cute.recast_iter`. Result types are explicit; attributes use mlir.Attribute. +pub fn recast_iter(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.recast_iter", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .location = location, + }); +} + +/// `cute.recast_layout`. Result types are explicit; attributes use mlir.Attribute. +pub fn recast_layout(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const mlir.Type, new_type_bits: *const mlir.Attribute, old_type_bits: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.recast_layout", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .attributes = &.{ + .named(ctx, "new_type_bits", new_type_bits), + .named(ctx, "old_type_bits", old_type_bits), + }, + .location = location, + }); +} + +/// `cute.reduce`. Result types are explicit; attributes use mlir.Attribute. +pub fn reduce( + ctx: *mlir.Context, + input: *const mlir.Value, + shape: *const mlir.Value, + init_val: *const mlir.Value, + output_type: *const mlir.Type, + reduction_op: *const mlir.Attribute, + opts: struct { + predication: ?*const mlir.Value = null, + reduced_shape_type: ?*const mlir.Type = null, + reduction_profile: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "reduction_op", reduction_op)); + if (opts.reduction_profile) |value| attributes.appendAssumeCapacity(.named(ctx, "reduction_profile", value)); + return mlir.Operation.make(ctx, "cute.reduce", .{ + .operands = .{ .variadic = &.{ + &.{input}, + &.{shape}, + &.{init_val}, + if (opts.predication) |value| &.{value} else &.{}, + } }, + .results = .{ .variadic = &.{ + &.{output_type}, + if (opts.reduced_shape_type) |value| &.{value} else &.{}, + } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute.right_inverse`. Result types are explicit; attributes use mlir.Attribute. +pub fn right_inverse(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.right_inverse", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.select`. Result types are explicit; attributes use mlir.Attribute. +pub fn select(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, mode: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.select", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "mode", mode), + }, + .location = location, + }); +} + +/// `cute.shape_div`. Result types are explicit; attributes use mlir.Attribute. +pub fn shape_div(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.shape_div", .{ + .operands = .{ .flat = &.{ a, b } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.size`. Result types are explicit; attributes use mlir.Attribute. +pub fn size( + ctx: *mlir.Context, + input: *const mlir.Value, + result_type: *const mlir.Type, + opts: struct { + mode: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); + return mlir.Operation.make(ctx, "cute.size", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute.slice`. Result types are explicit; attributes use mlir.Attribute. +pub fn slice(ctx: *mlir.Context, input: *const mlir.Value, coord: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.slice", .{ + .operands = .{ .flat = &.{ input, coord } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.smem_partition_size`. Result types are explicit; attributes use mlir.Attribute. +pub fn smem_partition_size( + ctx: *mlir.Context, + size_type: *const mlir.Type, + partition_id: *const mlir.Attribute, + opts: struct { + cumulative: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "partition_id", partition_id)); + if (opts.cumulative) |value| attributes.appendAssumeCapacity(.named(ctx, "cumulative", value)); + return mlir.Operation.make(ctx, "cute.smem_partition_size", .{ + .results = .{ .flat = &.{size_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute.static`. Result types are explicit; attributes use mlir.Attribute. +pub fn static(ctx: *mlir.Context, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.static", .{ + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.stencil_divide`. Result types are explicit; attributes use mlir.Attribute. +pub fn stencil_divide( + ctx: *mlir.Context, + input: *const mlir.Value, + stencil: *const mlir.Value, + result_type: *const mlir.Type, + opts: struct { + padding_upper: ?*const mlir.Value = null, + padding_lower: ?*const mlir.Value = null, + traversal_stride: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.stencil_divide", .{ + .operands = .{ .variadic = &.{ + &.{input}, + &.{stencil}, + if (opts.padding_upper) |value| &.{value} else &.{}, + if (opts.padding_lower) |value| &.{value} else &.{}, + if (opts.traversal_stride) |value| &.{value} else &.{}, + } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.symbolic`. Result types are explicit; attributes use mlir.Attribute. +pub fn symbolic(ctx: *mlir.Context, symbolic_type: *const mlir.Type, name: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.symbolic", .{ + .results = .{ .flat = &.{symbolic_type} }, + .attributes = &.{ + .named(ctx, "name", name), + }, + .location = location, + }); +} + +/// `cute.tile_to_shape`. Result types are explicit; attributes use mlir.Attribute. +pub fn tile_to_shape(ctx: *mlir.Context, block: *const mlir.Value, trg_shape: *const mlir.Value, ord_shape: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.tile_to_shape", .{ + .operands = .{ .flat = &.{ block, trg_shape, ord_shape } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.tiled.copy.partition_D`. Result types are explicit; attributes use mlir.Attribute. +pub fn tiled_copy_partition_D(ctx: *mlir.Context, tiled_copy: *const mlir.Value, input: *const mlir.Value, coord: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.tiled.copy.partition_D", .{ + .operands = .{ .flat = &.{ tiled_copy, input, coord } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.tiled.copy.partition_S`. Result types are explicit; attributes use mlir.Attribute. +pub fn tiled_copy_partition_S(ctx: *mlir.Context, tiled_copy: *const mlir.Value, input: *const mlir.Value, coord: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.tiled.copy.partition_S", .{ + .operands = .{ .flat = &.{ tiled_copy, input, coord } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.tiled.copy.retile`. Result types are explicit; attributes use mlir.Attribute. +pub fn tiled_copy_retile(ctx: *mlir.Context, tiled_copy: *const mlir.Value, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.tiled.copy.retile", .{ + .operands = .{ .flat = &.{ tiled_copy, input } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.tiled.mma.partition`. Result types are explicit; attributes use mlir.Attribute. +pub fn tiled_mma_partition(ctx: *mlir.Context, tiled_mma: *const mlir.Value, input: *const mlir.Value, coord: *const mlir.Value, result_type: *const mlir.Type, operand_id: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.tiled.mma.partition", .{ + .operands = .{ .flat = &.{ tiled_mma, input, coord } }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "operand_id", operand_id), + }, + .location = location, + }); +} + +/// `cute.tiled.mma.partition_shape`. Result types are explicit; attributes use mlir.Attribute. +pub fn tiled_mma_partition_shape(ctx: *mlir.Context, tiled_mma: *const mlir.Value, input: *const mlir.Value, result_type: *const mlir.Type, operand_id: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.tiled.mma.partition_shape", .{ + .operands = .{ .flat = &.{ tiled_mma, input } }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "operand_id", operand_id), + }, + .location = location, + }); +} + +/// `cute.tiled_divide`. Result types are explicit; attributes use mlir.Attribute. +pub fn tiled_divide(ctx: *mlir.Context, input: *const mlir.Value, tiler: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.tiled_divide", .{ + .operands = .{ .flat = &.{ input, tiler } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.tiled_product`. Result types are explicit; attributes use mlir.Attribute. +pub fn tiled_product(ctx: *mlir.Context, input: *const mlir.Value, tiler: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.tiled_product", .{ + .operands = .{ .flat = &.{ input, tiler } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.to_int_tuple`. Result types are explicit; attributes use mlir.Attribute. +pub fn to_int_tuple(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.to_int_tuple", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .location = location, + }); +} + +/// `cute.tuple.product`. Result types are explicit; attributes use mlir.Attribute. +pub fn tuple_product(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.tuple.product", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.tuple.product_each`. Result types are explicit; attributes use mlir.Attribute. +pub fn tuple_product_each(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.tuple.product_each", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.tuple_add`. Result types are explicit; attributes use mlir.Attribute. +pub fn tuple_add(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.tuple_add", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.tuple_div`. Result types are explicit; attributes use mlir.Attribute. +pub fn tuple_div(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.tuple_div", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.tuple_mod`. Result types are explicit; attributes use mlir.Attribute. +pub fn tuple_mod(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.tuple_mod", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.tuple_mul`. Result types are explicit; attributes use mlir.Attribute. +pub fn tuple_mul(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.tuple_mul", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.tuple_sub`. Result types are explicit; attributes use mlir.Attribute. +pub fn tuple_sub(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.tuple_sub", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.unpack_tuple`. Result types are explicit; attributes use mlir.Attribute. +pub fn unpack_tuple(ctx: *mlir.Context, tuple: *const mlir.Value, result_types: []const *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.unpack_tuple", .{ + .operands = .{ .flat = &.{tuple} }, + .results = .{ .variadic = &.{ + result_types, + } }, + .location = location, + }); +} + +/// `cute.zipped_divide`. Result types are explicit; attributes use mlir.Attribute. +pub fn zipped_divide(ctx: *mlir.Context, input: *const mlir.Value, tiler: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.zipped_divide", .{ + .operands = .{ .flat = &.{ input, tiler } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute.zipped_product`. Result types are explicit; attributes use mlir.Attribute. +pub fn zipped_product(ctx: *mlir.Context, input: *const mlir.Value, tiler: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute.zipped_product", .{ + .operands = .{ .flat = &.{ input, tiler } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} diff --git a/mlir/dialects/cute_ir/test.zig b/mlir/dialects/cute_ir/test.zig new file mode 100644 index 00000000..c6f44763 --- /dev/null +++ b/mlir/dialects/cute_ir/test.zig @@ -0,0 +1,341 @@ +const std = @import("std"); + +const cute_ir = @import("mlir/dialects/cute_ir"); +const cute_nvgpu = @import("mlir/dialects/cute_nvgpu"); +const mlir = @import("mlir"); + +const ArithTupleIteratorType = cute_ir.ArithTupleIteratorType; +const ComposedLayoutType = cute_ir.ComposedLayoutType; +const ConstrainedIntType = cute_ir.ConstrainedIntType; +const CoordTensorType = cute_ir.CoordTensorType; +const CoordType = cute_ir.CoordType; +const FastDivmodDivisorType = cute_ir.FastDivmodDivisorType; +const IntTupleType = cute_ir.IntTupleType; +const LayoutType = cute_ir.LayoutType; +const MemRefType = cute_ir.MemRefType; +const PtrType = cute_ir.PtrType; +const ShapeType = cute_ir.ShapeType; +const SparseElemType = cute_ir.SparseElemType; +const StrideType = cute_ir.StrideType; +const SwizzleType = cute_ir.SwizzleType; +const TileType = cute_ir.TileType; +const TiledCopyType = cute_ir.TiledCopyType; +const TiledCopyV2Type = cute_ir.TiledCopyV2Type; +const TiledMmaType = cute_ir.TiledMmaType; +const TupleType = cute_ir.TupleType; +const BitLayoutAttr = cute_ir.BitLayoutAttr; +const CopyAtomAttr = cute_ir.CopyAtomAttr; +const CopyAtomV2Attr = cute_ir.CopyAtomV2Attr; +const MmaAtomAttr = cute_ir.MmaAtomAttr; +const MxMmaAtomAttr = cute_ir.MxMmaAtomAttr; +const SparseMmaAtomAttr = cute_ir.SparseMmaAtomAttr; +const SparseMxMmaAtomAttr = cute_ir.SparseMxMmaAtomAttr; + +const operation_names: []const []const u8 = &.{ + "cute.add_offset", + "cute.append_to_rank", + "cute.apply_swizzle", + "cute.assume", + "cute.blocked_product", + "cute.ceil_div", + "cute.coalesce", + "cute.complement", + "cute.composed_get_inner", + "cute.composed_get_offset", + "cute.composed_get_outer", + "cute.composition", + "cute.copy", + "cute.copy.make_fragment", + "cute.copy_atom_call", + "cute.cosize", + "cute.crd2idx", + "cute.deref_arith_tuple_iter", + "cute.deref_desc_iter", + "cute.derefine", + "cute.dice", + "cute.elem_less", + "cute.equal", + "cute.fast_divmod.compute", + "cute.fast_divmod.create_divisor", + "cute.fast_divmod.divide", + "cute.fast_divmod.get_aux", + "cute.fast_divmod.get_divisor", + "cute.fast_divmod.make_divisor", + "cute.filter", + "cute.filter_zeros", + "cute.flat_divide", + "cute.flat_product", + "cute.gemm", + "cute.get", + "cute.get_flat_coord", + "cute.get_hier_coord", + "cute.get_integral_coord", + "cute.get_iter", + "cute.get_layout", + "cute.get_layouts_from_tile", + "cute.get_leaves", + "cute.get_scalars", + "cute.get_shape", + "cute.get_stride", + "cute.group_modes", + "cute.idx2crd", + "cute.increment_coord", + "cute.inttoptr", + "cute.kernel_smem_size", + "cute.left_inverse", + "cute.load_scaled_index", + "cute.local_partition", + "cute.local_tile", + "cute.logical_divide", + "cute.logical_product", + "cute.make_arith_tuple_iter", + "cute.make_atom", + "cute.make_composed_layout", + "cute.make_coord", + "cute.make_desc_iter", + "cute.make_fragment_like", + "cute.make_identity_layout", + "cute.make_identity_tensor", + "cute.make_int_tuple", + "cute.make_layout", + "cute.make_layout_like", + "cute.make_ordered_layout", + "cute.make_shape", + "cute.make_sparse_elem", + "cute.make_stride", + "cute.make_tile", + "cute.make_tiled_copy", + "cute.make_tiled_copy_v2", + "cute.make_tiled_mma", + "cute.make_tuple", + "cute.make_view", + "cute.memref.alloc_smem", + "cute.memref.alloca", + "cute.memref.load", + "cute.memref.load_vec", + "cute.memref.store", + "cute.memref.store_vec", + "cute.mma.make_fragment", + "cute.mma_atom_call", + "cute.prefetch", + "cute.prefetch_atom_call", + "cute.prepend_to_rank", + "cute.print", + "cute.print_tma_desc_im2col", + "cute.print_tma_desc_tiled", + "cute.print_view", + "cute.ptr.load", + "cute.ptr.store", + "cute.ptrtoint", + "cute.raked_product", + "cute.random_delay", + "cute.recast_iter", + "cute.recast_layout", + "cute.reduce", + "cute.right_inverse", + "cute.select", + "cute.shape_div", + "cute.size", + "cute.slice", + "cute.smem_partition_size", + "cute.static", + "cute.stencil_divide", + "cute.symbolic", + "cute.tile_to_shape", + "cute.tiled.copy.partition_D", + "cute.tiled.copy.partition_S", + "cute.tiled.copy.retile", + "cute.tiled.mma.partition", + "cute.tiled.mma.partition_shape", + "cute.tiled_divide", + "cute.tiled_product", + "cute.to_int_tuple", + "cute.tuple.product", + "cute.tuple.product_each", + "cute.tuple_add", + "cute.tuple_div", + "cute.tuple_mod", + "cute.tuple_mul", + "cute.tuple_sub", + "cute.unpack_tuple", + "cute.zipped_divide", + "cute.zipped_product", +}; + +fn testContext() !*mlir.Context { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + cute_ir.registerDialect(registry); + cute_nvgpu.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + return ctx; +} + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(cute_ir); + inline for (comptime std.meta.declarations(cute_ir)) |decl| { + const T = @field(cute_ir, decl); + if (@TypeOf(T) == type) std.testing.refAllDecls(T); + } +} + +test "every bound operation is registered" { + const ctx = try testContext(); + defer ctx.deinit(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} + +test "kernel is a unit attribute of the cute dialect" { + const ctx = try testContext(); + defer ctx.deinit(); + var buf: [64]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print("{f}", .{mlir.Attribute.dict(ctx, &.{cute_ir.kernel(ctx)})}); + try std.testing.expectEqualStrings("{cute.kernel}", w.buffered()); +} + +test "generated types and attributes rebuild from their getters" { + @setEvalBranchQuota(100_000); + const ctx = try testContext(); + defer ctx.deinit(); + inline for (.{ + .{ ArithTupleIteratorType, "!cute.arith_tuple_iter<\"(?,?{div=16})\">" }, + .{ ComposedLayoutType, "!cute.composed_layout<\"(4,5):(1,4) o 2 o (2,3):(1,2)\">" }, + .{ ConstrainedIntType, "!cute.i16" }, + .{ CoordTensorType, "!cute.coord_tensor<\"(?,?,?)\", \"((512,1),1):((1/2@0,0),0)\">" }, + .{ CoordType, "!cute.coord<\"1\">" }, + .{ FastDivmodDivisorType, "!cute.fast_divmod_divisor<32>" }, + .{ IntTupleType, "!cute.int_tuple<\"1\">" }, + .{ LayoutType, "!cute.layout<\"(2,3):(1,2)\">" }, + .{ MemRefType, "!cute.memref, S<2,4,3>, \"((8,16),(32,1),(1,3)):((32,256),(1,0),(0,4096))\">" }, + .{ PtrType, "!cute.ptr, S<2,4,3>>" }, + .{ ShapeType, "!cute.shape<\"(2,3)\">" }, + .{ SparseElemType, "!cute.sparse_elem<2, vector<1xf16>>" }, + .{ StrideType, "!cute.stride<\"1\">" }, + .{ SwizzleType, "!cute.swizzle<\"S<0,4,3>\">" }, + .{ TileType, "!cute.tile<\"[(2,3):(1,2)]\">" }, + .{ TiledCopyType, "!cute.tiled_copy, layout_copy_tv = <\"(32,4):(4,1)\">, tiler_mn = <\"[4:1;32:1]\">>" }, + .{ TiledCopyV2Type, "!cute.tiled_copy_v2, layout_copy_tv = <\"(1,8):(0,1)\">, tiler_mn = <\"[8:1]\">>" }, + .{ TiledMmaType, "!cute.tiled_mma f32 >, atom_layout_MNK = <\"(16,16,1):(16,1,0)\">, permutation_MNK = <\"[(16,4):(4,1);(16,4):(4,1);_]\">>" }, + .{ TupleType, "!cute.tuple" }, + .{ BitLayoutAttr, "#cute.bitlayout<<\"(8,4):(1,8)\">>" }, + .{ CopyAtomAttr, "#cute.copy_atom, layout_tv = (<\"(1,8):(0,1)\">, <\"(1,8):(0,1)\">, <\"(1,8):(0,1)\">)>" }, + .{ CopyAtomV2Attr, "#cute.copy_atom_v2, layout_tv = (<\"(1,8):(0,1)\">, <\"(1,8):(0,1)\">), frg = (#cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>)>" }, + .{ MmaAtomAttr, "#cute.mma_atom<<\"(16,8,16)\">, <\"32:1\">, layout_TV = (<\"1:0\">, <\"1:0\">, <\"1:0\">), shapes = (<\"(16,16)\">, <\"(8,16)\">, <\"(16,8)\">) frgs = (#cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>)>" }, + .{ MxMmaAtomAttr, "#cute.mx_mma_atom<<\"(16,8,16)\">, <\"32:1\">, layout_TV = (<\"1:0\">, <\"1:0\">, <\"1:0\">, <\"1:0\">, <\"1:0\">), shapes = (<\"(16,16)\">, <\"(8,16)\">, <\"(16,8)\">, <\"1\">, <\"2\">) frgs = (#cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>)>" }, + .{ SparseMmaAtomAttr, "#cute.sparse_mma_atom<<\"(16,8,16)\">, <\"32:1\">, layout_TV = (<\"1:0\">, <\"1:0\">, <\"1:0\">, <\"1:0\">), shapes = (<\"(16,16)\">, <\"(8,16)\">, <\"(16,8)\">, <\"1\">) frgs = (#cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>)>" }, + .{ SparseMxMmaAtomAttr, "#cute.sparse_mx_mma_atom<<\"(16,8,16)\">, <\"32:1\">, layout_TV = (<\"1:0\">, <\"2:0\">, <\"3:0\">, <\"4:0\">, <\"5:0\">, <\"6:0\">), shapes = (<\"1\">, <\"2\">, <\"3\">, <\"4\">, <\"5\">, <\"6\">) frgs = (#cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>)>" }, + }) |example| { + const T = example[0]; + const is_type = @hasDecl(T, "type_"); + const parsed = if (is_type) try mlir.Type.parse(ctx, example[1]) else try mlir.Attribute.parse(ctx, example[1]); + const value = parsed.isA(T) orelse return error.TestUnexpectedResult; + try std.testing.expect((try rebuild(T, ctx, value)).eql(value)); + var text: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer text.deinit(); + try text.writer.print("{f}", .{value}); + try std.testing.expect(parsed.eql(if (is_type) try mlir.Type.parse(ctx, text.written()) else try mlir.Attribute.parse(ctx, text.written()))); + } + // Every case of every enum attribute. + inline for (.{cute_ir}) |namespace| { + inline for (comptime std.meta.declarations(namespace)) |decl| { + const T = @field(namespace, decl); + if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque" and @hasDecl(T, "InitArgs")) { + const fields = @typeInfo(T.InitArgs).@"struct".field_names; + if (fields.len == 1 and comptime std.mem.eql(u8, fields[0], "value")) { + inline for (@typeInfo(@FieldType(T.InitArgs, fields[0])).@"enum".field_names) |case| { + const case_value = @field(@FieldType(T.InitArgs, fields[0]), case); + const attr = try T.get(ctx, .{ .value = case_value }); + try std.testing.expectEqual(@backingInt(case_value), @backingInt(attr.value())); + try std.testing.expect((try rebuild(T, ctx, attr)).eql(attr)); + } + } + } + } + } +} + +test "algebra attributes build from their leaves and tuples" { + const ctx = try testContext(); + defer ctx.deinit(); + + const int_tuple = try cute_ir.IntTupleAttr.getTuple(ctx, &.{ + try .getStatic(ctx, 0), + try .getDynamic(ctx, 32, 1), + try .getDynamic(ctx, 32, 16), + try .getTuple(ctx, &.{ try .getStatic(ctx, 4), try .getDynamic(ctx, 64, 1) }), + }); + try expectType(ctx, "!cute.int_tuple<\"(0,?,?{div=16},(4,?{i64}))\">", (try IntTupleType.get(ctx, .{ .attr = int_tuple.attribute() })).type_()); + try expectType(ctx, "!cute.int_tuple<\"()\">", (try IntTupleType.get(ctx, .{ .attr = (try cute_ir.IntTupleAttr.getTuple(ctx, &.{})).attribute() })).type_()); + + const coord = try cute_ir.CoordAttr.getTuple(ctx, &.{ + try .getUnderscore(ctx), + try .getStatic(ctx, 1), + try .getTuple(ctx, &.{ try .getDynamic(ctx, 32, 1), try .getUnderscore(ctx) }), + }); + try expectType(ctx, "!cute.coord<\"(_,1,(?,_))\">", (try CoordType.get(ctx, .{ .attr = coord.attribute() })).type_()); + + const shape = try cute_ir.ShapeAttr.getTuple(ctx, &.{ + try .getTuple(ctx, &.{ try .getStatic(ctx, 512), try .getStatic(ctx, 1) }), + try .getStatic(ctx, 1), + }); + const stride = try cute_ir.StrideAttr.getTuple(ctx, &.{ + try .getTuple(ctx, &.{ try .getBasis(ctx, 1, 2, &.{0}), try .getStatic(ctx, 0) }), + try .getBasis(ctx, 3, 1, &.{ 0, 1 }), + }); + const layout = try cute_ir.LayoutAttr.get(ctx, .{ .shape = shape, .stride = stride }); + try expectType(ctx, "!cute.layout<\"((512,1),1):((1/2@0,0),3@1@0)\">", (try LayoutType.get(ctx, .{ .attr = layout.attribute() })).type_()); + try std.testing.expectError(error.InvalidMlir, cute_ir.LayoutAttr.get(ctx, .{ .shape = shape, .stride = try .getStatic(ctx, 1) })); + + const row = try cute_ir.LayoutAttr.get(ctx, .{ + .shape = try .getTuple(ctx, &.{ try .getStatic(ctx, 8), try .getStatic(ctx, 64) }), + .stride = try .getTuple(ctx, &.{ try .getStatic(ctx, 64), try .getDynamic(ctx, 32, 8) }), + }); + try expectType(ctx, "!cute.layout<\"(8,64):(64,?{div=8})\">", (try LayoutType.get(ctx, .{ .attr = row.attribute() })).type_()); + + const tile = try cute_ir.TileAttr.getTuple(ctx, &.{ try .getLayout(ctx, row), try .getUnderscore(ctx) }); + try expectType(ctx, "!cute.tile<\"[(8,64):(64,?{div=8});_]\">", (try TileType.get(ctx, .{ .attr = tile.attribute() })).type_()); + + const swizzle = try cute_ir.SwizzleAttr.get(ctx, .{ .bits = 3, .base = 4, .shift = 3 }); + try expectType(ctx, "!cute.swizzle<\"S<3,4,3>\">", (try SwizzleType.get(ctx, .{ .attr = swizzle.attribute() })).type_()); + + const outer = try cute_ir.LayoutAttr.get(ctx, .{ + .shape = try .getTuple(ctx, &.{ try .getStatic(ctx, 8), try .getStatic(ctx, 64) }), + .stride = try .getTuple(ctx, &.{ try .getStatic(ctx, 64), try .getStatic(ctx, 1) }), + }); + const composed = try cute_ir.ComposedLayoutAttr.get(ctx, .{ .inner = .{ .swizzle = swizzle }, .offset = try .getStatic(ctx, 0), .outer = outer }); + try expectType(ctx, "!cute.composed_layout<\"S<3,4,3> o 0 o (8,64):(64,1)\">", (try ComposedLayoutType.get(ctx, .{ .attr = composed.attribute() })).type_()); + + try std.testing.expectError(error.InvalidMlir, cute_ir.IntTupleAttr.getDynamic(ctx, 7, 1)); + try std.testing.expectError(error.InvalidMlir, cute_ir.StrideAttr.getBasis(ctx, 1, 1, &.{})); + try std.testing.expectError(error.InvalidMlir, cute_ir.SwizzleAttr.get(ctx, .{ .bits = 5, .base = 4, .shift = 3 })); +} + +fn expectType(ctx: *mlir.Context, expected: []const u8, actual: *const mlir.Type) !void { + const parsed = try mlir.Type.parse(ctx, expected); + if (!parsed.eql(actual)) { + std.debug.print("expected {s}, built {f}\n", .{ expected, actual }); + return error.TestUnexpectedResult; + } +} + +fn rebuild(comptime T: type, ctx: *mlir.Context, value: *const T) !*const T { + var args: T.InitArgs = undefined; + var elements: [8]*const mlir.Type = undefined; + inline for (@typeInfo(T.InitArgs).@"struct".field_names) |field| { + const name = [_]u8{comptime std.ascii.toUpper(field[0])} ++ field[1..]; + const noun = [_]u8{comptime std.ascii.toLower(field[0])} ++ field[1..]; + const accessor = if (@hasDecl(T, noun ++ "_")) noun ++ "_" else noun; + if (@FieldType(T.InitArgs, field) == []const *const mlir.Type) { + const count = @field(T, "num" ++ name)(value); + for (elements[0..count], 0..) |*element, i| element.* = @field(T, noun[0 .. noun.len - 1])(value, i); + @field(args, field) = elements[0..count]; + } else { + @field(args, field) = @field(T, accessor)(value); + } + } + return T.get(ctx, args); +} diff --git a/mlir/dialects/cute_nvgpu/BUILD.bazel b/mlir/dialects/cute_nvgpu/BUILD.bazel new file mode 100644 index 00000000..07c40ea9 --- /dev/null +++ b/mlir/dialects/cute_nvgpu/BUILD.bazel @@ -0,0 +1,192 @@ +load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library", "td_library") +load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +package(default_visibility = ["//visibility:public"]) + +cc_library( + name = "capi", + srcs = ["capi.cc"], + hdrs = ["capi.h"], + tags = ["manual"], + deps = [ + ":dialect", + ":headers", + "//mlir/dialects/cute_ir:dialect", + "@llvm-project//mlir:CAPIIR", + ], +) + +zig_library( + name = "cute_nvgpu", + import_name = "mlir/dialects/cute_nvgpu", + main = "cute_nvgpu.zig", + tags = ["manual"], + deps = [ + ":capi", + "//mlir", + "//mlir/dialects/cute_ir", + "//stdx", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":cute_nvgpu", + "//mlir", + "//mlir/dialects/cute_ir", + ], +) + +td_library( + name = "td_files", + srcs = [ + "CuteNVGPUAttrs.td", + "CuteNVGPUDialect.td", + "CuteNVGPUEnums.td", + "CuteNVGPUOps.td", + "CuteNVGPUTypes.td", + ], + includes = ["/"], + deps = [ + "//mlir/dialects/cute_ir:td_files", + "@cute_ir//:cute_ir_dialect_td", + "@llvm-project//mlir:InferTypeOpInterfaceTdFiles", + "@llvm-project//mlir:OpBaseTdFiles", + "@llvm-project//mlir:SideEffectInterfacesTdFiles", + ], +) + +gentbl_cc_library( + name = "dialect_inc_gen", + tbl_outs = [ + ( + [ + "-gen-dialect-decls", + "-dialect=cute_nvgpu", + ], + "CuteNVGPUDialect.h.inc", + ), + ( + [ + "-gen-dialect-defs", + "-dialect=cute_nvgpu", + ], + "CuteNVGPUDialect.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "CuteNVGPUDialect.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "types_inc_gen", + tbl_outs = [ + ( + [ + "-gen-typedef-decls", + "-typedefs-dialect=cute_nvgpu", + ], + "CuteNVGPUTypes.h.inc", + ), + ( + [ + "-gen-typedef-defs", + "-typedefs-dialect=cute_nvgpu", + ], + "CuteNVGPUTypes.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "CuteNVGPUTypes.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "enums_inc_gen", + tbl_outs = [ + ( + ["-gen-enum-decls"], + "CuteNVGPUEnums.h.inc", + ), + ( + ["-gen-enum-defs"], + "CuteNVGPUEnums.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "CuteNVGPUEnums.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "attrs_inc_gen", + tbl_outs = [ + ( + [ + "-gen-attrdef-decls", + "-attrdefs-dialect=cute_nvgpu", + ], + "CuteNVGPUAttrs.h.inc", + ), + ( + [ + "-gen-attrdef-defs", + "-attrdefs-dialect=cute_nvgpu", + ], + "CuteNVGPUAttrs.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "CuteNVGPUAttrs.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "ops_inc_gen", + tbl_outs = [ + ( + ["-gen-op-decls"], + "CuteNVGPUOps.h.inc", + ), + ( + ["-gen-op-defs"], + "CuteNVGPUOps.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "CuteNVGPUOps.td", + deps = [":td_files"], +) + +cc_library( + name = "headers", + hdrs = ["CuteNVGPUDialect.h"], + deps = [ + ":attrs_inc_gen", + ":dialect_inc_gen", + ":enums_inc_gen", + ":ops_inc_gen", + ":types_inc_gen", + "//mlir/dialects/cute_ir:headers", + "@llvm-project//mlir:BytecodeOpInterface", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:InferTypeOpInterface", + "@llvm-project//mlir:SideEffectInterfaces", + ], +) + +cc_library( + name = "dialect", + srcs = ["CuteNVGPUDialect.cpp"], + deps = [ + ":headers", + "//mlir/dialects/cute_ir:headers", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:AsmParser", + "@llvm-project//mlir:IR", + ], +) diff --git a/mlir/dialects/cute_nvgpu/CuteNVGPUAttrs.td b/mlir/dialects/cute_nvgpu/CuteNVGPUAttrs.td new file mode 100644 index 00000000..b4230c61 --- /dev/null +++ b/mlir/dialects/cute_nvgpu/CuteNVGPUAttrs.td @@ -0,0 +1,132 @@ +//===- CuteNVGPUAttrs.td - CuTe NVIDIA GPU attributes ------*- tablegen -*-===// +// +// Generated for nvidia-cutlass-dsl 4.8.0 from the DSL compiler +// (`_cutlass_ir.cu13` parser messages and probing its attribute parser and +// printer) and its enum bindings; do not edit by hand. +// +// The enum attributes are CuteNVGPUEnums.td; `#cute_nvgpu.major`, over the +// cute dialect's MajorMode, is here so that the enum generator run over +// CuteNVGPUEnums.td emits the nvgpu enums only. The fragment attributes are +// `key = value` lists in a fixed order; a defaulted field is printed only +// when set. +// +//===----------------------------------------------------------------------===// + +#ifndef MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUATTRS_TD +#define MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUATTRS_TD + +include "mlir/dialects/cute_nvgpu/CuteNVGPUDialect.td" +include "mlir/dialects/cute_nvgpu/CuteNVGPUEnums.td" +include "mlir/dialects/cute_ir/CuteEnums.td" + +def CuteNVGPU_MajorModeAttr + : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +class CuteNVGPU_Attr + : AttrDef { + let mnemonic = attrMnemonic; +} + +// `true` / `false`. +class CuteNVGPU_BoolParam : AttrOrTypeParameter<"bool", ""> { + let parser = "::mlir::cutlass_compiler::cute_nvgpu::parseBool($_parser)"; + let printer = "$_printer << ($_self ? \"true\" : \"false\")"; +} +def CuteNVGPU_Bool : CuteNVGPU_BoolParam; +def CuteNVGPU_FalseByDefault : CuteNVGPU_BoolParam { + let defaultValue = "false"; +} + +def CuteNVGPU_RmemFrgAttr : CuteNVGPU_Attr<"RmemFrg", "rmem_frg"> { + let summary = "Register-memory fragment of an MMA operand"; + let description = [{ + `#cute_nvgpu.rmem_frg` + }]; + let parameters = (ins + "::mlir::Type":$value_type, + "::mlir::cutlass_compiler::cute::MmaOperand":$operand, + CuteNVGPU_FalseByDefault:$derive_elem_type + ); + let assemblyFormat = [{ + `<` `value_type` `=` $value_type `,` `operand` `=` $operand + (`,` `derive_elem_type` `=` $derive_elem_type^)? `>` + }]; +} + +class CuteNVGPU_SmemFrg + : CuteNVGPU_Attr { + let summary = arch # " shared-memory fragment"; + let description = "`#cute_nvgpu." # attrMnemonic # "`"; + let parameters = (ins "::mlir::cutlass_compiler::cute::MajorMode":$major_mode); + let assemblyFormat = "`<` `major` `=` $major_mode `>`"; +} + +def CuteNVGPU_SM90SmemFrgAttr + : CuteNVGPU_SmemFrg<"SM90SmemFrg", "arch.sm90.smem_frg", "SM90">; +def CuteNVGPU_SM100SmemFrgAttr + : CuteNVGPU_SmemFrg<"SM100SmemFrg", "arch.sm100.smem_frg", "SM100">; +def CuteNVGPU_SM100CircularSmemFrgAttr + : CuteNVGPU_SmemFrg<"SM100CircularSmemFrg", "arch.sm100.circular_smem_frg", + "SM100 circular">; +def CuteNVGPU_SM107SmemFrgAttr + : CuteNVGPU_SmemFrg<"SM107SmemFrg", "arch.sm107.smem_frg", "SM107">; + +def CuteNVGPU_SM100TmemFrgAttr + : CuteNVGPU_Attr<"SM100TmemFrg", "arch.sm100.tmem_frg"> { + let summary = "SM100 tensor-memory fragment"; + let description = [{ + `#cute_nvgpu.arch.sm100.tmem_frg` + }]; + let parameters = (ins + "::mlir::Type":$data_type, + "::mlir::Type":$storage_type, + "int":$cta_group, + "TmemAllocMode":$tmem_alloc_mode + ); + let assemblyFormat = [{ + `<` `data_type` `=` $data_type `,` `storage_type` `=` $storage_type `,` + `cta_group` `=` $cta_group `,` `tmem_alloc_mode` `=` $tmem_alloc_mode `>` + }]; +} + +def CuteNVGPU_SM100TmemSfFrgAttr + : CuteNVGPU_Attr<"SM100TmemSfFrg", "arch.sm100.tmem_sf_frg"> { + let summary = "SM100 tensor-memory scale-factor fragment"; + let description = [{ + `#cute_nvgpu.arch.sm100.tmem_sf_frg` + }]; + let parameters = (ins + "::mlir::Type":$sf_type, + "int":$sf_vec_size, + "int":$cta_group, + CuteNVGPU_Bool:$is_sfa, + "TmemAllocMode":$tmem_alloc_mode + ); + let assemblyFormat = [{ + `<` `sf_type` `=` $sf_type `,` `sf_vec_size` `=` $sf_vec_size `,` + `cta_group` `=` $cta_group `,` `is_sfa` `=` $is_sfa `,` + `tmem_alloc_mode` `=` $tmem_alloc_mode `>` + }]; +} + +def CuteNVGPU_SM100TmemEFrgAttr + : CuteNVGPU_Attr<"SM100TmemEFrg", "arch.sm100.tmem_e_frg"> { + let summary = "SM100 tensor-memory sparse-metadata fragment"; + let description = [{ + `#cute_nvgpu.arch.sm100.tmem_e_frg` + }]; + let parameters = (ins "::mlir::Type":$a_type, "::mlir::Type":$e_type); + let assemblyFormat = "`<` `a_type` `=` $a_type `,` `e_type` `=` $e_type `>`"; +} + +def CuteNVGPU_NotImplementedFrgAttr + : CuteNVGPU_Attr<"NotImplementedFrg", "not_implemented_frg"> { + let summary = "Fragment of an atom that has none"; + let description = "`#cute_nvgpu.not_implemented_frg<>`"; + let assemblyFormat = "`<` `>`"; +} + +#endif // MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUATTRS_TD diff --git a/mlir/dialects/cute_nvgpu/CuteNVGPUDialect.cpp b/mlir/dialects/cute_nvgpu/CuteNVGPUDialect.cpp new file mode 100644 index 00000000..a912714e --- /dev/null +++ b/mlir/dialects/cute_nvgpu/CuteNVGPUDialect.cpp @@ -0,0 +1,1921 @@ +//===- CuteNVGPUDialect.cpp - CuTe NVIDIA GPU dialect ---------------------===// +// +// Parsers, printers and verifiers of the cute_nvgpu types that the declarative +// format cannot express. The syntax and the verifier messages are the DSL +// compiler's (see CuteNVGPUTypes.td). + +#include "mlir/dialects/cute_nvgpu/CuteNVGPUDialect.h" + +#include "mlir/AsmParser/AsmParser.h" +#include "mlir/IR/Builders.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/DialectImplementation.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/TypeSwitch.h" + +#include +#include + +using namespace mlir; +using namespace mlir::cutlass_compiler::cute_nvgpu; +namespace cute = mlir::cutlass_compiler::cute; + +#include "mlir/dialects/cute_nvgpu/CuteNVGPUDialect.cpp.inc" + +#include "mlir/dialects/cute_nvgpu/CuteNVGPUEnums.cpp.inc" + +#define GET_ATTRDEF_CLASSES +#include "mlir/dialects/cute_nvgpu/CuteNVGPUAttrs.cpp.inc" + +#define GET_TYPEDEF_CLASSES +#include "mlir/dialects/cute_nvgpu/CuteNVGPUTypes.cpp.inc" + +#define GET_OP_CLASSES +#include "mlir/dialects/cute_nvgpu/CuteNVGPUOps.cpp.inc" + +FailureOr mlir::cutlass_compiler::cute_nvgpu::parseBool(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + StringRef kw; + if (p.parseKeyword(&kw)) + return failure(); + if (kw == "true" || kw == "false") + return kw == "true"; + return p.emitError(loc, "expected `true` or `false`"); +} + +//===----------------------------------------------------------------------===// +// Helpers +//===----------------------------------------------------------------------===// + +namespace { + +using EmitFn = llvm::function_ref; + +template ParseResult field(AsmParser &p, T &out) { + FailureOr v = FieldParser::parse(p); + if (failed(v)) + return failure(); + out = *v; + return success(); +} +ParseResult field(AsmParser &p, Type &out) { return p.parseType(out); } +ParseResult field(AsmParser &p, int &out) { return p.parseInteger(out); } +ParseResult field(AsmParser &p, bool &out) { + FailureOr v = parseBool(p); + if (failed(v)) + return failure(); + out = *v; + return success(); +} + +// `key = value`. +template +ParseResult keyField(AsmParser &p, StringRef key, T &out) { + return failure(p.parseKeyword(key) || p.parseEqual() || field(p, out)); +} +// `, key = value`. +template +ParseResult nextKeyField(AsmParser &p, StringRef key, T &out) { + return failure(p.parseComma() || keyField(p, key, out)); +} +// An optional ` key = value` group. +template +ParseResult optKeyField(AsmParser &p, StringRef key, T &out) { + if (failed(p.parseOptionalKeyword(key))) + return success(); + return failure(p.parseEqual() || field(p, out)); +} +// `N word`, as in `128 b` or `32 DP`. +ParseResult countOf(AsmParser &p, StringRef word, int &out) { + return failure(p.parseInteger(out) || p.parseKeyword(word)); +} +// `(a, b)`. +template ParseResult pairOf(AsmParser &p, T &a, T &b) { + return failure(p.parseLParen() || field(p, a) || p.parseComma() || + field(p, b) || p.parseRParen()); +} +// `elem_type = (A, B, C)`. +ParseResult elemTypes(AsmParser &p, Type &a, Type &b, Type &c) { + return failure(p.parseKeyword("elem_type") || p.parseEqual() || + p.parseLParen() || p.parseType(a) || p.parseComma() || + p.parseType(b) || p.parseComma() || p.parseType(c) || + p.parseRParen()); +} +void printElemTypes(AsmPrinter &p, Type a, Type b, Type c) { + p << "elem_type = (" << a << ", " << b << ", " << c << ")"; +} +// `xR`. +ParseResult repeat(AsmParser &p, unsigned &out) { + SMLoc loc = p.getCurrentLocation(); + StringRef kw; + if (p.parseKeyword(&kw)) + return failure(); + if (!kw.consume_front("x")) + return p.emitError( + loc, "failed to parse parameter 'num_rep' which is to start with `x`"); + if (kw.getAsInteger(10, out)) + return p.emitError( + loc, "failed to parse parameter 'num_rep' which is to be a `unsigned`"); + return success(); +} + +template +void printIfNot(AsmPrinter &p, StringRef key, E value, E dflt) { + if (value != dflt) + p << ' ' << key << " = " << stringifyEnum(value); +} +template +void printAttrField(AsmPrinter &p, StringRef key, A attr) { + if (!attr) + return; + p << ", " << key << " = "; + p.printStrippedAttrOrType(attr); +} + +// The width of an element: integers and floats, 64 for index and pointers, +// a sparse element's share of its physical storage. +int elementBits(Type t) { + if (auto s = llvm::dyn_cast(t)) + return s.getNumLogical() + ? elementBits(s.getPhysicalType()) / s.getNumLogical() + : 0; + if (t.isTF32()) + return 32; + if (t.isIntOrFloat()) + return t.getIntOrFloatBitWidth(); + if (llvm::isa(t)) + return 64; + return 0; +} +int orElementBits(int bits, Type t) { return bits ? bits : elementBits(t); } + +// The integers of a flat tuple `(a,b,c)`. +std::optional> flatInts(StringRef text) { + if (!text.consume_front("(") || !text.consume_back(")")) + return std::nullopt; + SmallVector out; + for (StringRef part : llvm::split(text, ',')) { + int64_t v; + if (part.getAsInteger(10, v)) + return std::nullopt; + out.push_back(v); + } + return out; +} + +// `MxNxK`. +ParseResult parseShapeMNK(AsmParser &p, cute::ShapeAttr &out) { + SMLoc loc = p.getCurrentLocation(); + SmallVector dims; + if (p.parseDimensionList(dims, /*allowDynamic=*/false, + /*withTrailingX=*/false)) + return p.emitError(loc, "can not parse shape_MNK"); + auto shape = cutegen::from_string( + "(" + + llvm::join( + llvm::map_range(dims, [](int64_t d) { return std::to_string(d); }), + ",") + + ")"); + if (!shape) + return p.emitError(loc, "can not parse shape_MNK"); + out = cute::ShapeAttr::get(p.getContext(), *shape); + return success(); +} +void printShapeMNK(AsmPrinter &p, cute::ShapeAttr shape) { + std::string text = cutegen::to_string(shape.getRef()); + StringRef body(text); + body.consume_front("("); + body.consume_back(")"); + for (char c : body) + p << (c == ',' ? 'x' : c); +} +LogicalResult verifyShapeMNK(EmitFn emitError, cute::ShapeAttr shape) { + std::string text = cutegen::to_string(shape.getRef()); + auto dims = flatInts(text); + if (!dims || dims->size() != 3) + return emitError() << "expects flatten rank-3 tuple of shape, but got " + << text; + return success(); +} + +// The A fragment kind: `ss` (shared memory), `rs` (registers) or `ts` +// (tensor memory). +ParseResult parseFragKind(AsmParser &p, MmaFragKind other, MmaFragKind &out) { + SMLoc loc = p.getCurrentLocation(); + StringRef kw; + bool ts = other == MmaFragKind::tmem; + if (p.parseKeyword("frag_kind") || p.parseEqual() || p.parseKeyword(&kw)) + return failure(); + if (kw == "ss") + out = MmaFragKind::smem_desc; + else if (kw == (ts ? "ts" : "rs")) + out = other; + else + return p.emitError(loc, + ts ? "expects `ts` or `ss`" : "expects `rs` or `ss`"); + return success(); +} +StringRef fragKindText(MmaFragKind k) { + switch (k) { + case MmaFragKind::smem_desc: + return "ss"; + case MmaFragKind::rmem: + return "rs"; + case MmaFragKind::tmem: + return "ts"; + default: + return stringifyEnum(k); + } +} + +// `t` or `n`. +ParseResult parseTranspose(AsmParser &p, UnitAttr &out) { + SMLoc loc = p.getCurrentLocation(); + StringRef kw; + if (p.parseKeyword(&kw)) + return failure(); + if (kw != "t" && kw != "n") + return p.emitError(loc, "expects `t` or `n`"); + out = kw == "t" ? UnitAttr::get(p.getContext()) : UnitAttr(); + return success(); +} + +bool isPow2AtLeast8(int v) { return v >= 8 && llvm::isPowerOf2_32(v); } + +} // namespace + +//===----------------------------------------------------------------------===// +// Descriptors +//===----------------------------------------------------------------------===// + +// sm103/sm107.smem_desc_circular: static `(B,128,N):(128,1,128*B)`, N >= 2. +static LogicalResult verifyCircular(EmitFn emitError, cute::LayoutAttr layout) { + std::string text = cutegen::to_string(layout.getRef()); + StringRef shapeText, strideText; + std::tie(shapeText, strideText) = StringRef(text).split(':'); + if (StringRef(text).contains('?')) + return emitError() << "block layout must be fully static but got " << text; + auto shape = flatInts(shapeText), stride = flatInts(strideText); + if (!shape || shape->size() != 3 || !stride || stride->size() != 3) + return emitError() << "expects block layout to have 3 modes, but got " + << text; + if ((*shape)[1] != 128) + return emitError() << "expects block layout to have 128B 1st mode, but got " + << text; + if ((*stride)[1] != 1 || (*stride)[0] != 128 || + (*stride)[2] != 128 * (*shape)[0]) + return emitError() << "expects mode-1 major compact layout, but got " + << text; + if ((*shape)[2] < 2) + return emitError() + << "Circular SMEM descriptor requires at least 2 128B blocks since " + "MMA may span 2 buffers, but got " + << text; + return success(); +} +LogicalResult SmemDescCircularSM103Type::verify(EmitFn emitError, + cute::LayoutAttr layout) { + return verifyCircular(emitError, layout); +} +LogicalResult SmemDescCircularSM107Type::verify(EmitFn emitError, + cute::LayoutAttr layout) { + return verifyCircular(emitError, layout); +} + +Type SmemDescViewType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + Type desc; + std::string text; + if (p.parseLess() || p.parseType(desc) || p.parseComma()) + return {}; + SMLoc layoutLoc = p.getCurrentLocation(); + if (p.parseString(&text) || p.parseGreater()) + return {}; + auto layout = cutegen::from_string(text); + if (!layout) { + p.emitError(layoutLoc, "failed to parse SmemDescViewType parameter " + "'layout' as `cutegen::layout`"); + return {}; + } + return getChecked([&] { return p.emitError(loc); }, p.getContext(), desc, + cute::LayoutAttr::get(p.getContext(), *layout)); +} +void SmemDescViewType::print(AsmPrinter &p) const { + p << '<' << getDesc() << ", "; + p.printString(cutegen::to_string(getLayout().getRef())); + p << '>'; +} +LogicalResult SmemDescViewType::verify(EmitFn emitError, Type desc, + cute::LayoutAttr) { + if (!llvm::isa(desc)) + return emitError() << "expected SmemDescType, SmemDescCircularSM103Type, " + "SmemDescCircularSM107Type or SmemDescSM107Type"; + return success(); +} + +//===----------------------------------------------------------------------===// +// SIMT copy atoms +//===----------------------------------------------------------------------===// + +// ``; the compiler prints +// `` for copy_bits 0, which is accepted too. +Type CopyAtomSIMTSyncCopyType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + Type t; + int bits = 0; + auto src = cute::AddressSpace::generic, dst = src; + if (p.parseLess() || p.parseType(t)) + return {}; + // `src_space` / `dst_space` groups; one alone is what the compiler prints + // when the other is generic (or copy_bits is 0), so it is read too. + auto spaces = [&]() -> ParseResult { + if (succeeded(p.parseOptionalKeyword("dst_space"))) + return failure(p.parseEqual() || field(p, dst)); + if (p.parseKeyword("src_space") || p.parseEqual() || field(p, src)) + return failure(); + if (succeeded(p.parseOptionalComma())) + return keyField(p, "dst_space", dst); + return success(); + }; + if (succeeded(p.parseOptionalComma())) { + OptionalParseResult n = p.parseOptionalInteger(bits); + if (n.has_value() && (failed(*n) || p.parseKeyword("b"))) + return {}; + if ((!n.has_value() || succeeded(p.parseOptionalComma())) && spaces()) + return {}; + } + if (p.parseGreater()) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, bits, + src, dst); +} +void CopyAtomSIMTSyncCopyType::print(AsmPrinter &p) const { + auto generic = cute::AddressSpace::generic; + bool spaces = getSrcSpace() != generic || getDstSpace() != generic; + p << '<' << getValType(); + if (getCopyBits() || spaces) + p << ", " << getCopyBits() << " b"; + if (spaces) + p << ", src_space = " << stringifyEnum(getSrcSpace()); + if (getDstSpace() != generic) + p << ", dst_space = " << stringifyEnum(getDstSpace()); + p << '>'; +} +LogicalResult CopyAtomSIMTSyncCopyType::verify(EmitFn emitError, Type, int bits, + cute::AddressSpace, + cute::AddressSpace) { + if (bits != 0 && !isPow2AtLeast8(bits)) + return emitError() << "expected copy_bits to be 0 (auto) or a power of 2 " + "(>= 8), but got " + << bits; + return success(); +} + +// ``. +Type CopyAtomSIMTAsyncCopyType::parse(AsmParser &p) { + Type t; + int bits = 0; + LoadCacheMode cache; + if (p.parseLess() || p.parseType(t) || nextKeyField(p, "cache", cache)) + return {}; + if (succeeded(p.parseOptionalComma()) && countOf(p, "b", bits)) + return {}; + if (p.parseGreater()) + return {}; + return get(p.getContext(), t, cache, orElementBits(bits, t)); +} +void CopyAtomSIMTAsyncCopyType::print(AsmPrinter &p) const { + p << '<' << getValType() << ", cache = " << stringifyEnum(getCache()) << ", " + << getCopyBits() << " b>"; +} + +// ``. +Type CopyAtomDsmemStoreType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + Type t; + int bits = 0; + if (p.parseLess() || p.parseType(t)) + return {}; + if (succeeded(p.parseOptionalComma()) && countOf(p, "b", bits)) + return {}; + if (p.parseGreater()) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, + orElementBits(bits, t)); +} +void CopyAtomDsmemStoreType::print(AsmPrinter &p) const { + p << '<' << getValType() << ", " << getCopyBits() << " b>"; +} +LogicalResult CopyAtomDsmemStoreType::verify(EmitFn emitError, Type, int bits) { + if (bits != 0 && bits != 32 && bits != 64 && bits != 128) + return emitError() + << "expects copy_bits to be one of (0, 32, 64, 128), but got " + << bits; + return success(); +} + +// Bulk copies: ``. +Type CopyAtomBulkCopyG2SType::parse(AsmParser &p) { + Type t; + int bits = 0; + bool mcast = false; + if (p.parseLess() || p.parseType(t) || optKeyField(p, "copy_bits", bits) || + optKeyField(p, "mcast", mcast) || p.parseGreater()) + return {}; + return get(p.getContext(), t, bits, mcast); +} +void CopyAtomBulkCopyG2SType::print(AsmPrinter &p) const { + p << '<' << getValType(); + if (getCopyBits()) + p << " copy_bits = " << getCopyBits(); + if (getMcast()) + p << " mcast = true"; + p << '>'; +} +Type CopyAtomBulkCopyS2GType::parse(AsmParser &p) { + Type t; + int bits = 0; + bool mask = false; + if (p.parseLess() || p.parseType(t) || optKeyField(p, "copy_bits", bits) || + optKeyField(p, "mask", mask) || p.parseGreater()) + return {}; + return get(p.getContext(), t, bits, mask); +} +void CopyAtomBulkCopyS2GType::print(AsmPrinter &p) const { + p << '<' << getValType(); + if (getCopyBits()) + p << " copy_bits = " << getCopyBits(); + if (getMask()) + p << " mask = true"; + p << '>'; +} +Type CopyAtomBulkCopyS2SType::parse(AsmParser &p) { + Type t; + int bits = 0; + if (p.parseLess() || p.parseType(t) || optKeyField(p, "copy_bits", bits) || + p.parseGreater()) + return {}; + return get(p.getContext(), t, bits); +} +void CopyAtomBulkCopyS2SType::print(AsmPrinter &p) const { + p << '<' << getValType(); + if (getCopyBits()) + p << " copy_bits = " << getCopyBits(); + p << '>'; +} + +// g2r, r2g, r2s, s2r: ``. +static LogicalResult verifyCopyBits(EmitFn emitError, int bits) { + if (!isPow2AtLeast8(bits)) + return emitError() + << "expects copy_bits to be a power of 2 (>= 8), but got " << bits; + return success(); +} + +Type CopyAtomG2RType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + Type t; + int bits = 0; + auto order = MemOrderKind::WEAK; + auto scope = MemScopeKind::CTA; + auto prefetch = L2PrefetchSize::NONE; + auto evict = CacheEvictionPriority::EVICT_NORMAL; + auto cache = LoadCacheMode::always; + auto shared = SharedSpace::CTA; + bool invariant = false; + if (p.parseLess() || p.parseType(t) || optKeyField(p, "copy_bits", bits) || + optKeyField(p, "mem_order", order) || + optKeyField(p, "mem_scope", scope) || + optKeyField(p, "l2_prefetch_size", prefetch) || + optKeyField(p, "l1_cache_evict_priority", evict) || + optKeyField(p, "load_cache_mode", cache) || + optKeyField(p, "shared_space", shared) || + optKeyField(p, "invariant", invariant) || p.parseGreater()) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, + orElementBits(bits, t), order, scope, prefetch, evict, + cache, shared, invariant); +} +void CopyAtomG2RType::print(AsmPrinter &p) const { + p << '<' << getValType() << " copy_bits = " << getCopyBits(); + printIfNot(p, "mem_order", getMemOrder(), MemOrderKind::WEAK); + printIfNot(p, "mem_scope", getMemScope(), MemScopeKind::CTA); + printIfNot(p, "l2_prefetch_size", getL2PrefetchSize(), L2PrefetchSize::NONE); + printIfNot(p, "l1_cache_evict_priority", getL1CacheEvictPriority(), + CacheEvictionPriority::EVICT_NORMAL); + printIfNot(p, "load_cache_mode", getLoadCacheMode(), LoadCacheMode::always); + printIfNot(p, "shared_space", getSharedSpace(), SharedSpace::CTA); + if (getInvariant()) + p << " invariant = true"; + p << '>'; +} +LogicalResult CopyAtomG2RType::verify(EmitFn emitError, Type, int bits, + MemOrderKind order, MemScopeKind scope, + L2PrefetchSize prefetch, + CacheEvictionPriority evict, + LoadCacheMode cache, SharedSpace, + bool invariant) { + if (failed(verifyCopyBits(emitError, bits))) + return failure(); + if (invariant && (order != MemOrderKind::WEAK || scope != MemScopeKind::CTA || + prefetch != L2PrefetchSize::NONE || + evict != CacheEvictionPriority::EVICT_NORMAL || + cache != LoadCacheMode::always)) + return emitError() + << "invariant_load is not allowed when the other non-default memory " + "features (e.g., mem_order) are set to non-default values"; + if (order != MemOrderKind::WEAK && cache != LoadCacheMode::always) + return emitError() + << "expects load_cache_mode to be 'always' when mem_order is not " + "'weak', but got '" + << stringifyEnum(cache) << "' and '" << stringifyEnum(order) << "'"; + return success(); +} + +Type CopyAtomR2GType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + Type t; + int bits = 0; + auto order = MemOrderKind::WEAK; + auto scope = MemScopeKind::CTA; + auto evict = CacheEvictionPriority::EVICT_NORMAL; + auto cache = StoreCacheMode::write_back; + auto shared = SharedSpace::CTA; + if (p.parseLess() || p.parseType(t) || optKeyField(p, "copy_bits", bits) || + optKeyField(p, "mem_order", order) || + optKeyField(p, "mem_scope", scope) || + optKeyField(p, "l1_cache_evict_priority", evict) || + optKeyField(p, "store_cache_mode", cache) || + optKeyField(p, "shared_space", shared) || p.parseGreater()) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, + orElementBits(bits, t), order, scope, evict, cache, shared); +} +void CopyAtomR2GType::print(AsmPrinter &p) const { + p << '<' << getValType() << " copy_bits = " << getCopyBits(); + printIfNot(p, "mem_order", getMemOrder(), MemOrderKind::WEAK); + printIfNot(p, "mem_scope", getMemScope(), MemScopeKind::CTA); + printIfNot(p, "l1_cache_evict_priority", getL1CacheEvictPriority(), + CacheEvictionPriority::EVICT_NORMAL); + printIfNot(p, "store_cache_mode", getStoreCacheMode(), + StoreCacheMode::write_back); + printIfNot(p, "shared_space", getSharedSpace(), SharedSpace::CTA); + p << '>'; +} +LogicalResult CopyAtomR2GType::verify(EmitFn emitError, Type, int bits, + MemOrderKind order, MemScopeKind, + CacheEvictionPriority, + StoreCacheMode cache, SharedSpace) { + if (failed(verifyCopyBits(emitError, bits))) + return failure(); + if (order != MemOrderKind::WEAK && cache != StoreCacheMode::write_back) + return emitError() + << "expects store_cache_mode to be 'write_back' when mem_order is " + "not 'weak', but got '" + << stringifyEnum(cache) << "' and '" << stringifyEnum(order) << "'"; + return success(); +} + +template static Type parseSharedCopy(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + Type t; + int bits = 0; + auto order = MemOrderKind::WEAK; + auto scope = MemScopeKind::CTA; + auto shared = SharedSpace::CTA; + if (p.parseLess() || p.parseType(t) || optKeyField(p, "copy_bits", bits) || + optKeyField(p, "mem_order", order) || + optKeyField(p, "mem_scope", scope) || + optKeyField(p, "shared_space", shared) || p.parseGreater()) + return {}; + return T::getChecked([&] { return p.emitError(loc); }, p.getContext(), t, + orElementBits(bits, t), order, scope, shared); +} +template static void printSharedCopy(AsmPrinter &p, T type) { + p << '<' << type.getValType() << " copy_bits = " << type.getCopyBits(); + printIfNot(p, "mem_order", type.getMemOrder(), MemOrderKind::WEAK); + printIfNot(p, "mem_scope", type.getMemScope(), MemScopeKind::CTA); + printIfNot(p, "shared_space", type.getSharedSpace(), SharedSpace::CTA); + p << '>'; +} +Type CopyAtomR2SType::parse(AsmParser &p) { + return parseSharedCopy(p); +} +void CopyAtomR2SType::print(AsmPrinter &p) const { printSharedCopy(p, *this); } +LogicalResult CopyAtomR2SType::verify(EmitFn emitError, Type, int bits, + MemOrderKind, MemScopeKind, SharedSpace) { + return verifyCopyBits(emitError, bits); +} +Type CopyAtomS2RType::parse(AsmParser &p) { + return parseSharedCopy(p); +} +void CopyAtomS2RType::print(AsmPrinter &p) const { printSharedCopy(p, *this); } +LogicalResult CopyAtomS2RType::verify(EmitFn emitError, Type, int bits, + MemOrderKind, MemScopeKind, SharedSpace) { + return verifyCopyBits(emitError, bits); +} + +//===----------------------------------------------------------------------===// +// Multimem atoms: `, ORDER, SCOPE>`, positional. Parsing +// takes each field by its kind, which also reads what the compiler prints +// when it drops a default field before a set one. +//===----------------------------------------------------------------------===// + +namespace { +struct Multimem { + int bits = 0; + ReductionKind red = ReductionKind::ADD; + LdReduceAccPrecisionKindAttr acc; + MemOrderKind order = MemOrderKind::RELAXED; + MemScopeKind scope = MemScopeKind::SYS; +}; + +ParseResult parseMultimem(AsmParser &p, Type &t, Multimem &m, bool hasRed, + bool hasAcc) { + if (p.parseLess() || p.parseType(t)) + return failure(); + while (succeeded(p.parseOptionalComma())) { + SMLoc loc = p.getCurrentLocation(); + StringRef kw; + OptionalParseResult bits = p.parseOptionalInteger(m.bits); + if (bits.has_value()) { + if (failed(*bits) || p.parseKeyword("b")) + return failure(); + continue; + } + if (hasAcc && succeeded(p.parseOptionalLess())) { + LdReduceAccPrecisionKind kind; + if (field(p, kind) || p.parseGreater()) + return failure(); + m.acc = LdReduceAccPrecisionKindAttr::get(p.getContext(), kind); + continue; + } + if (p.parseKeyword(&kw)) + return failure(); + if (auto r = symbolizeReductionKind(kw); r && hasRed) + m.red = *r; + else if (auto o = symbolizeMemOrderKind(kw)) + m.order = *o; + else if (auto s = symbolizeMemScopeKind(kw)) + m.scope = *s; + else + return p.emitError(loc, "unexpected `") << kw << "`"; + } + return p.parseGreater(); +} + +// Prints the fields up to the last one that is not default. +void printMultimem(AsmPrinter &p, Type t, const Multimem &m, bool bitsSet, + bool hasRed, bool hasAcc) { + int last = m.scope != MemScopeKind::SYS ? 4 + : m.order != MemOrderKind::RELAXED ? 3 + : m.acc ? 2 + : hasRed && m.red != ReductionKind::ADD ? 1 + : bitsSet ? 0 + : -1; + p << '<' << t; + if (last >= 0) + p << ", " << m.bits << " b"; + if (hasRed && last >= 1) + p << ", " << stringifyEnum(m.red); + if (hasAcc && last >= 2 && m.acc) { + p << ", "; + p.printStrippedAttrOrType(m.acc); + } + if (last >= 3) + p << ", " << stringifyEnum(m.order); + if (last >= 4) + p << ", " << stringifyEnum(m.scope); + p << '>'; +} +} // namespace + +Type CopyAtomSIMTMultimemStType::parse(AsmParser &p) { + Type t; + Multimem m; + if (parseMultimem(p, t, m, /*hasRed=*/false, /*hasAcc=*/false)) + return {}; + return get(p.getContext(), t, m.bits, m.order, m.scope); +} +void CopyAtomSIMTMultimemStType::print(AsmPrinter &p) const { + Multimem m{ + getCopyBits(), ReductionKind::ADD, {}, getMemOrder(), getMemScope()}; + printMultimem(p, getValType(), m, getCopyBits() != 0, false, false); +} + +Type CopyAtomSIMTMultimemRedType::parse(AsmParser &p) { + Type t; + Multimem m; + if (parseMultimem(p, t, m, /*hasRed=*/true, /*hasAcc=*/false)) + return {}; + // An unset width is the element's, at least 32 bits (the f16x2 form). + int bits = m.bits ? m.bits : std::max(elementBits(t), 32); + return get(p.getContext(), t, bits, m.red, m.order, m.scope); +} +void CopyAtomSIMTMultimemRedType::print(AsmPrinter &p) const { + Multimem m{getCopyBits(), getReduction(), {}, getMemOrder(), getMemScope()}; + printMultimem(p, getValType(), m, /*bitsSet=*/true, true, false); +} + +Type CopyAtomSIMTMultimemLdReduceType::parse(AsmParser &p) { + Type t; + Multimem m; + if (parseMultimem(p, t, m, /*hasRed=*/true, /*hasAcc=*/true)) + return {}; + return get(p.getContext(), t, m.bits, m.red, m.acc, m.order, m.scope); +} +void CopyAtomSIMTMultimemLdReduceType::print(AsmPrinter &p) const { + Multimem m{getCopyBits(), getReduction(), getLdReduceAccPrecision(), + getMemOrder(), getMemScope()}; + printMultimem(p, getValType(), m, getCopyBits() != 0, true, true); +} + +//===----------------------------------------------------------------------===// +// Tensor-memory atoms: `` +//===----------------------------------------------------------------------===// + +static ParseResult parseTmemHead(AsmParser &p, Type &t, int &dp, int &bit, + unsigned &rep) { + return failure(p.parseLess() || p.parseType(t) || p.parseComma() || + countOf(p, "DP", dp) || p.parseComma() || + countOf(p, "bit", bit) || p.parseComma() || repeat(p, rep)); +} +static void printTmemHead(AsmPrinter &p, Type t, int dp, int bit, + unsigned rep) { + p << '<' << t << ", " << dp << " DP, " << bit << " bit, x" << rep; +} +// An optional `, word`. +static ParseResult parseFlag(AsmParser &p, StringRef word, UnitAttr &out) { + if (failed(p.parseOptionalComma())) + return success(); + if (p.parseKeyword(word)) + return failure(); + out = UnitAttr::get(p.getContext()); + return success(); +} + +static LogicalResult verifyTmemCopy(EmitFn emitError, int dp, int bit, + unsigned rep) { + if (dp != 16 && dp != 32) + return emitError() + << "expects the number of data paths to be one of [16,32], but got " + << dp; + if (bit != 32 && bit != 64 && bit != 128 && bit != 256) + return emitError() << "expects the number of bits to be one of " + "[32,64,128,256], but got " + << bit; + if (rep < 1 || rep > 128 || !llvm::isPowerOf2_32(rep)) + return emitError() << "expects the number of repetitions to be one of " + "[1,2,4,8,16,32,64,128], but got " + << rep; + if (dp == 32 && bit != 32) + return emitError() + << "only 32b patterns are supported when the number of data paths " + "is 32, but got " + << bit; + if (dp == 16 && bit == 256 && rep >= 64) + return emitError() + << "a repetition of 64 or 128 is not supported when the number of " + "data paths is 16 and the number of bits is 256"; + if (dp == 16 && bit == 128 && rep == 128) + return emitError() + << "a repetition of 128 is not supported when the number of data " + "paths is 16 and the number of bits is 128"; + return success(); +} + +Type CopyAtomSM100TmemLoadType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + Type t; + int dp, bit; + unsigned rep; + UnitAttr pack; + if (parseTmemHead(p, t, dp, bit, rep) || parseFlag(p, "pack16b", pack) || + p.parseGreater()) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, dp, + bit, rep, pack); +} +void CopyAtomSM100TmemLoadType::print(AsmPrinter &p) const { + printTmemHead(p, getValType(), getNumDp(), getNumBit(), getNumRep()); + p << (getPack16b() ? ", pack16b>" : ">"); +} +LogicalResult CopyAtomSM100TmemLoadType::verify(EmitFn emitError, Type, int dp, + int bit, unsigned rep, + UnitAttr) { + return verifyTmemCopy(emitError, dp, bit, rep); +} + +Type CopyAtomSM100TmemStoreType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + Type t; + int dp, bit; + unsigned rep; + UnitAttr expand; + if (parseTmemHead(p, t, dp, bit, rep) || parseFlag(p, "expand16b", expand) || + p.parseGreater()) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, dp, + bit, rep, expand); +} +void CopyAtomSM100TmemStoreType::print(AsmPrinter &p) const { + printTmemHead(p, getValType(), getNumDp(), getNumBit(), getNumRep()); + p << (getExpand16b() ? ", expand16b>" : ">"); +} +LogicalResult CopyAtomSM100TmemStoreType::verify(EmitFn emitError, Type, int dp, + int bit, unsigned rep, + UnitAttr) { + return verifyTmemCopy(emitError, dp, bit, rep); +} + +// `<..., OP[, nan][, half_split_off = N[ : T]]>`. +Type CopyAtomSM10xTmemLoadRedType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + Type t; + int dp, bit; + unsigned rep; + TmemLoadRedOp op; + UnitAttr nan; + IntegerAttr split; + if (parseTmemHead(p, t, dp, bit, rep) || p.parseComma() || field(p, op)) + return {}; + while (succeeded(p.parseOptionalComma())) { + if (!nan && !split && succeeded(p.parseOptionalKeyword("nan"))) { + nan = UnitAttr::get(p.getContext()); + continue; + } + if (p.parseKeyword("half_split_off") || p.parseEqual() || + p.parseAttribute(split)) + return {}; + } + if (p.parseGreater()) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, dp, + bit, rep, op, nan, split); +} +void CopyAtomSM10xTmemLoadRedType::print(AsmPrinter &p) const { + printTmemHead(p, getValType(), getNumDp(), getNumBit(), getNumRep()); + p << ", " << stringifyEnum(getRedOp()); + if (getNan()) + p << ", nan"; + if (getHalfSplitOff()) + p << ", half_split_off=" << getHalfSplitOff(); + p << '>'; +} +LogicalResult +CopyAtomSM10xTmemLoadRedType::verify(EmitFn emitError, Type t, int dp, int bit, + unsigned rep, TmemLoadRedOp op, + UnitAttr nan, IntegerAttr split) { + if (!(t.isF32() || t.isInteger(32))) + return emitError() << "expect val_type to be u32, s32, or f32, but got '" + << t << "'"; + if (nan && !t.isF32()) + return emitError() << "expect val_type to be f32 when nan is specified"; + if ((op == TmemLoadRedOp::minabs || op == TmemLoadRedOp::maxabs) && + !t.isF32()) + return emitError() + << "expect val_type to be f32 when red_op is minabs or maxabs"; + if (bit != 32) + return emitError() << "only 32b patterns are supported, but got " << bit; + if (rep < 2 || rep > 128 || !llvm::isPowerOf2_32(rep)) + return emitError() << "expects the number of repetitions to be one of " + "[2,4,8,16,32,64,128], but got " + << rep; + if (split && dp != 16) + return emitError() + << "expect half_split_off to be set only when mode is 16dp 32bit"; + return success(); +} + +// `<..., OP[, red[, nan]]>`. +Type CopyAtomSM107TmemLoadSPCompressType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + Type t; + int dp, bit; + unsigned rep; + TmemLoadRedOp op; + UnitAttr red, nan; + if (parseTmemHead(p, t, dp, bit, rep) || p.parseComma() || field(p, op) || + parseFlag(p, "red", red) || (red && parseFlag(p, "nan", nan)) || + p.parseGreater()) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, dp, + bit, rep, op, red, nan); +} +void CopyAtomSM107TmemLoadSPCompressType::print(AsmPrinter &p) const { + printTmemHead(p, getValType(), getNumDp(), getNumBit(), getNumRep()); + p << ", " << stringifyEnum(getRedOp()); + if (getRed()) + p << ", red"; + if (getNan()) + p << ", nan"; + p << '>'; +} +LogicalResult CopyAtomSM107TmemLoadSPCompressType::verify( + EmitFn emitError, Type t, int dp, int bit, unsigned rep, TmemLoadRedOp, + UnitAttr red, UnitAttr nan) { + if (dp != 32 || bit != 32) + return emitError() << "expect `num_dp` and `num_bit` to be 32, but got " + << dp << " and " << bit; + if (!t.isF32()) + return emitError() << "expect val_type to be f32, but got '" << t << "'"; + if (rep < 4 || rep > 128 || !llvm::isPowerOf2_32(rep)) + return emitError() << "expect the number of repetitions to be one of " + "[4,8,16,32,64,128], but got " + << rep; + if (nan && !red) + return emitError() << "expect `red` to be true when `nan` is true"; + return success(); +} + +// The UTCCP shapes of tcgen05.cp. +static bool isUtccp(int dp, int bit, CopyS2TBroadcast b) { + using B = CopyS2TBroadcast; + return (b == B::none && (dp == 4 || dp == 128) && + (bit == 128 || bit == 256)) || + (b == B::x4 && dp == 32 && bit == 128) || + ((b == B::lw_0213 || b == B::lw_0123) && dp == 64 && bit == 128); +} + +// ``. +Type CopyAtomSM100CopyS2TType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + Type t; + int dp, bit, cta; + auto bcast = CopyS2TBroadcast::none; + if (p.parseLess() || p.parseType(t) || p.parseComma() || + countOf(p, "DP", dp) || p.parseComma() || countOf(p, "bit", bit) || + p.parseComma() || countOf(p, "cta", cta)) + return {}; + if (succeeded(p.parseOptionalComma()) && field(p, bcast)) + return {}; + if (p.parseGreater()) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, dp, + bit, cta, bcast); +} +void CopyAtomSM100CopyS2TType::print(AsmPrinter &p) const { + p << '<' << getValType() << ", " << getNumDp() << " DP, " << getNumBit() + << " bit, " << getNumCta() << " cta"; + if (getBroadcast() != CopyS2TBroadcast::none) + p << ", " << stringifyEnum(getBroadcast()); + p << '>'; +} +LogicalResult CopyAtomSM100CopyS2TType::verify(EmitFn emitError, Type, int dp, + int bit, int cta, + CopyS2TBroadcast b) { + if (cta != 1 && cta != 2) + return emitError() << "Expect cta is 1/2 but got: " << cta; + if (!isUtccp(dp, bit, b)) + return emitError() << "The given UTCCP params: dp " << dp << ", bits " + << bit << ", broadcast " << stringifyEnum(b) + << " is unsupported or illegal."; + return success(); +} + +// ``. +Type CopyAtomSM100S2TCopyV2Type::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + Type t; + int dp, bit, cta; + cute::MajorMode major; + auto bcast = CopyS2TBroadcast::none; + if (p.parseLess() || p.parseType(t) || nextKeyField(p, "num_dp", dp) || + nextKeyField(p, "num_bit", bit) || nextKeyField(p, "num_cta", cta) || + nextKeyField(p, "smem_major", major)) + return {}; + if (succeeded(p.parseOptionalComma()) && keyField(p, "broadcast", bcast)) + return {}; + if (p.parseGreater()) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, dp, + bit, cta, major, bcast); +} +void CopyAtomSM100S2TCopyV2Type::print(AsmPrinter &p) const { + p << '<' << getValType() << ", num_dp = " << getNumDp() + << ", num_bit = " << getNumBit() << ", num_cta = " << getNumCta() + << ", smem_major = " << stringifyEnum(getSmemMajor()); + if (getBroadcast() != CopyS2TBroadcast::none) + p << ", broadcast = " << stringifyEnum(getBroadcast()); + p << '>'; +} +LogicalResult CopyAtomSM100S2TCopyV2Type::verify(EmitFn emitError, Type, int dp, + int bit, int cta, + cute::MajorMode, + CopyS2TBroadcast b) { + if (cta != 1 && cta != 2) + return emitError() << "expects cta to be 1 or 2, but got " << cta; + if (!isUtccp(dp, bit, b)) + return emitError() << "expects valid UTCCP params, but got dp " << dp + << ", bits " << bit << ", broadcast " + << stringifyEnum(b); + return success(); +} + +//===----------------------------------------------------------------------===// +// ldmatrix / stmatrix +//===----------------------------------------------------------------------===// + +Type CopyAtomLdsmType::parse(AsmParser &p) { + Type t; + cute::ShapeAttr mode; + LdsmSzPattern sz; + int n; + UnitAttr trans; + if (p.parseLess() || keyField(p, "val_type", t) || + nextKeyField(p, "mode", mode) || nextKeyField(p, "sz_pattern", sz) || + nextKeyField(p, "num_matrices", n) || p.parseComma() || + parseTranspose(p, trans) || p.parseGreater()) + return {}; + return get(p.getContext(), t, mode, sz, n, trans); +} +void CopyAtomLdsmType::print(AsmPrinter &p) const { + p << "" : ", n>"); +} + +Type CopyAtomStsmType::parse(AsmParser &p) { + Type t; + cute::ShapeAttr mode; + int n; + UnitAttr trans; + if (p.parseLess() || p.parseType(t) || nextKeyField(p, "mode", mode) || + nextKeyField(p, "num_matrices", n) || p.parseComma() || + parseTranspose(p, trans) || p.parseGreater()) + return {}; + return get(p.getContext(), t, mode, n, trans); +} +void CopyAtomStsmType::print(AsmPrinter &p) const { + p << '<' << getValType() << ", mode = "; + p.printStrippedAttrOrType(getMode()); + p << ", num_matrices = " << getNumMatrices() + << (getTranspose() ? ", t>" : ", n>"); +} + +//===----------------------------------------------------------------------===// +// TMA atoms +//===----------------------------------------------------------------------===// + +static LogicalResult verifySparsity(EmitFn emitError, int sparsity) { + if (sparsity != 1 && sparsity != 2 && sparsity != 4 && sparsity != 8 && + sparsity != 16) + return emitError() + << "expects sparsity to be one of (1, 2, 4, 8, 16), but got " + << sparsity; + return success(); +} +// `'; +} +LogicalResult CopyAtomTmaLoadType::verify(EmitFn emitError, Type, int sparsity, + int, TmaLoadMode, int, + cute::StrideType, bool, + cute::LayoutType, bool, bool) { + return verifySparsity(emitError, sparsity); +} + +Type CopyAtomTmaStoreType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + Type t; + int sparsity = 1, bits; + TmaStoreMode mode; + cute::StrideType stride; + cute::LayoutType gbasis; + bool override = false, noOob = false; + if (parseTmaHead(p, t, &sparsity, bits) || nextKeyField(p, "mode", mode) || + nextKeyField(p, "g_stride", stride) || + optKeyField(p, "tma_gbasis", gbasis) || + optKeyField(p, "override", override) || + optKeyField(p, "no_fully_oob_tile", noOob) || p.parseGreater()) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, + sparsity, bits, mode, stride, gbasis, override, noOob); +} +void CopyAtomTmaStoreType::print(AsmPrinter &p) const { + printTmaHead(p, getValType(), getSparsity(), getCopyBits()); + p << ", mode = " << stringifyEnum(getMode()); + printTmaTail(p, getGStride(), getTmaGbasis()); + printTrue(p, "override", getOverride()); + printTrue(p, "no_fully_oob_tile", getNoFullyOobTile()); + p << '>'; +} +LogicalResult CopyAtomTmaStoreType::verify(EmitFn emitError, Type, int sparsity, + int, TmaStoreMode, cute::StrideType, + cute::LayoutType, bool, bool) { + return verifySparsity(emitError, sparsity); +} + +Type CopyAtomTmaReduceType::parse(AsmParser &p) { + Type t; + int bits; + TmaStoreMode mode; + ReductionKind kind; + cute::StrideType stride; + cute::LayoutType gbasis; + if (parseTmaHead(p, t, nullptr, bits) || nextKeyField(p, "mode", mode) || + nextKeyField(p, "kind", kind) || nextKeyField(p, "g_stride", stride) || + optKeyField(p, "tma_gbasis", gbasis) || p.parseGreater()) + return {}; + return get(p.getContext(), t, bits, mode, kind, stride, gbasis); +} +void CopyAtomTmaReduceType::print(AsmPrinter &p) const { + printTmaHead(p, getValType(), 1, getCopyBits()); + p << ", mode = " << stringifyEnum(getMode()) + << ", kind = " << stringifyEnum(getKind()); + printTmaTail(p, getGStride(), getTmaGbasis()); + p << '>'; +} + +Type CopyAtomIm2ColTmaLoadType::parse(AsmParser &p) { + Type t; + int bits, ctas; + cute::StrideType stride; + cute::LayoutType gbasis; + bool mcast = false; + if (parseTmaHead(p, t, nullptr, bits) || nextKeyField(p, "num_cta", ctas) || + nextKeyField(p, "g_stride", stride) || optKeyField(p, "mcast", mcast) || + optKeyField(p, "tma_gbasis", gbasis) || p.parseGreater()) + return {}; + return get(p.getContext(), t, bits, ctas, stride, mcast, gbasis); +} +void CopyAtomIm2ColTmaLoadType::print(AsmPrinter &p) const { + printTmaHead(p, getValType(), 1, getCopyBits()); + p << ", num_cta = " << getNumCta() << ", g_stride = "; + p.printStrippedAttrOrType(getGStride()); + printTrue(p, "mcast", getMcast()); + if (getTmaGbasis()) { + p << " tma_gbasis = "; + p.printStrippedAttrOrType(getTmaGbasis()); + } + p << '>'; +} + +Type CopyAtomIm2ColTmaStoreType::parse(AsmParser &p) { + Type t; + int bits; + cute::StrideType stride; + cute::LayoutType gbasis; + if (parseTmaHead(p, t, nullptr, bits) || + nextKeyField(p, "g_stride", stride) || + optKeyField(p, "tma_gbasis", gbasis) || p.parseGreater()) + return {}; + return get(p.getContext(), t, bits, stride, gbasis); +} +void CopyAtomIm2ColTmaStoreType::print(AsmPrinter &p) const { + printTmaHead(p, getValType(), 1, getCopyBits()); + printTmaTail(p, getGStride(), getTmaGbasis()); + p << '>'; +} + +// The TMA data format of an element type when none is given. +std::optional +mlir::cutlass_compiler::cute_nvgpu::defaultTmaDataFormat(Type t) { + if (auto s = llvm::dyn_cast(t)) + t = s.getPhysicalType(); + if (t.isF16()) + return TmaDataFormat::F16_RN; + if (t.isBF16()) + return TmaDataFormat::BF16_RN; + if (t.isF32()) + return TmaDataFormat::F32_RN; + if (t.isTF32()) + return TmaDataFormat::TF32_RN; + if (t.isF64()) + return TmaDataFormat::F64_RN; + if (!t.isIntOrFloat()) + return std::nullopt; + unsigned bits = t.getIntOrFloatBitWidth(); + bool isSigned = t.isSignedInteger(); + if (bits == 4) + return TmaDataFormat::U4; + if (bits <= 8) + return TmaDataFormat::U8; + if (bits == 16) + return TmaDataFormat::U16; + if (bits == 32) + return isSigned ? TmaDataFormat::S32 : TmaDataFormat::U32; + if (bits == 64) + return isSigned ? TmaDataFormat::S64 : TmaDataFormat::U64; + return std::nullopt; +} + +namespace { +// `<[kind, ]T, copy_bits = N, tma_gbasis = L[, tma_format = F]>`. +template +ParseResult parseNonExecTma(AsmParser &p, Kind *kind, Type &t, int &bits, + cute::LayoutType &gbasis, + std::optional &format) { + if (p.parseLess()) + return failure(); + if (kind && (field(p, *kind) || p.parseComma())) + return failure(); + if (p.parseType(t) || nextKeyField(p, "copy_bits", bits) || + nextKeyField(p, "tma_gbasis", gbasis)) + return failure(); + format = defaultTmaDataFormat(t); + if (succeeded(p.parseOptionalComma())) { + // Empty for an element type without a format, as the compiler prints it. + if (p.parseKeyword("tma_format") || p.parseEqual()) + return failure(); + if (succeeded(p.parseOptionalGreater())) { + format = std::nullopt; + return success(); + } + TmaDataFormat f; + if (field(p, f)) + return failure(); + format = f; + } + return p.parseGreater(); +} +template void printNonExecTma(AsmPrinter &p, T type) { + p << type.getValType() << ", copy_bits = " << type.getCopyBits() + << ", tma_gbasis = "; + p.printStrippedAttrOrType(type.getTmaGbasis()); + p << ", tma_format = "; + if (type.getTmaFormat()) + p << stringifyEnum(*type.getTmaFormat()); + p << '>'; +} +LogicalResult verifyNonExecTma(EmitFn emitError, int bits) { + if (bits == 0) + return emitError() << "invalid copy_bits, got 0"; + return success(); +} +} // namespace + +#define NON_EXEC_TMA_WITH_KIND(TYPE, KIND) \ + Type TYPE::parse(AsmParser &p) { \ + SMLoc loc = p.getCurrentLocation(); \ + KIND kind; \ + Type t; \ + int bits; \ + cute::LayoutType gbasis; \ + std::optional format; \ + if (parseNonExecTma(p, &kind, t, bits, gbasis, format)) \ + return {}; \ + return getChecked([&] { return p.emitError(loc); }, p.getContext(), kind, \ + t, bits, gbasis, format); \ + } \ + void TYPE::print(AsmPrinter &p) const { \ + p << '<' << stringifyEnum(getKind()) << ", "; \ + printNonExecTma(p, *this); \ + } \ + LogicalResult TYPE::verify(EmitFn emitError, KIND, Type, int bits, \ + cute::LayoutType, std::optional) { \ + return verifyNonExecTma(emitError, bits); \ + } +#define NON_EXEC_TMA(TYPE) \ + Type TYPE::parse(AsmParser &p) { \ + SMLoc loc = p.getCurrentLocation(); \ + Type t; \ + int bits; \ + cute::LayoutType gbasis; \ + std::optional format; \ + if (parseNonExecTma(p, nullptr, t, bits, gbasis, format)) \ + return {}; \ + return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, \ + bits, gbasis, format); \ + } \ + void TYPE::print(AsmPrinter &p) const { \ + p << '<'; \ + printNonExecTma(p, *this); \ + } \ + LogicalResult TYPE::verify(EmitFn emitError, Type, int bits, \ + cute::LayoutType, std::optional) { \ + return verifyNonExecTma(emitError, bits); \ + } +NON_EXEC_TMA_WITH_KIND(CopyAtomNonExecTiledTmaLoadType, TiledTmaLoad) +NON_EXEC_TMA_WITH_KIND(CopyAtomNonExecTiledTmaReduceType, ReductionKind) +NON_EXEC_TMA_WITH_KIND(CopyAtomNonExecIm2ColTmaLoadType, Im2ColTmaLoad) +NON_EXEC_TMA_WITH_KIND(CopyAtomNonExec2DGather4TmaLoadType, + GatherScatterTmaLoad) +NON_EXEC_TMA(CopyAtomNonExecTiledTmaStoreType) +NON_EXEC_TMA(CopyAtomNonExecIm2ColTmaStoreType) +NON_EXEC_TMA(CopyAtomNonExec2DScatter4TmaStoreType) +#undef NON_EXEC_TMA_WITH_KIND +#undef NON_EXEC_TMA + +//===----------------------------------------------------------------------===// +// MMA atoms +//===----------------------------------------------------------------------===// + +Type UniversalFmaAtomType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + cute::ShapeAttr shape; + Type a, b, c; + if (p.parseLess() || parseShapeMNK(p, shape) || p.parseComma() || + p.parseLParen() || p.parseType(a) || p.parseComma() || p.parseType(b) || + p.parseRParen() || p.parseArrow() || p.parseType(c) || p.parseGreater()) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), shape, a, + b, c); +} +void UniversalFmaAtomType::print(AsmPrinter &p) const { + p << '<'; + printShapeMNK(p, getShapeMnk()); + p << ", (" << getAType() << ", " << getBType() << ") -> " << getCType() + << " >"; +} +LogicalResult UniversalFmaAtomType::verify(EmitFn emitError, + cute::ShapeAttr shape, Type, Type, + Type) { + return verifyShapeMNK(emitError, shape); +} + +// ``. +Type MmaAtomSM80Type::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + cute::ShapeAttr shape; + Type a, b, c; + MMAIntOverflowAttr overflow; + BinaryOpAttr op; + if (p.parseLess() || parseShapeMNK(p, shape) || p.parseComma() || + elemTypes(p, a, b, c)) + return {}; + if (succeeded(p.parseOptionalComma()) && + (keyField(p, "int_overflow", overflow) || + (succeeded(p.parseOptionalComma()) && keyField(p, "binary_op", op)))) + return {}; + if (p.parseGreater()) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), shape, a, + b, c, overflow, op); +} +void MmaAtomSM80Type::print(AsmPrinter &p) const { + p << '<'; + printShapeMNK(p, getShapeMnk()); + p << ", "; + printElemTypes(p, getAType(), getBType(), getCType()); + printAttrField(p, "int_overflow", getIntOverflow()); + printAttrField(p, "binary_op", getBinaryOp()); + p << " >"; +} +LogicalResult MmaAtomSM80Type::verify(EmitFn emitError, cute::ShapeAttr shape, + Type, Type, Type, MMAIntOverflowAttr, + BinaryOpAttr) { + return verifyShapeMNK(emitError, shape); +} + +// ``. +Type MmaAtomSM80SparseType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + cute::ShapeAttr shape; + Type a, b, c; + SparseMetadataFormat format; + MMAIntOverflowAttr overflow; + if (p.parseLess() || parseShapeMNK(p, shape) || p.parseComma() || + elemTypes(p, a, b, c) || nextKeyField(p, "md_format", format)) + return {}; + if (succeeded(p.parseOptionalComma()) && + keyField(p, "int_overflow", overflow)) + return {}; + if (p.parseGreater()) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), shape, a, + b, c, format, overflow); +} +void MmaAtomSM80SparseType::print(AsmPrinter &p) const { + p << '<'; + printShapeMNK(p, getShapeMnk()); + p << ", "; + printElemTypes(p, getAType(), getBType(), getCType()); + p << ", md_format = " << stringifyEnum(getSparseMetadataFormat()); + printAttrField(p, "int_overflow", getIntOverflow()); + p << " >"; +} +LogicalResult MmaAtomSM80SparseType::verify(EmitFn emitError, + cute::ShapeAttr shape, Type, Type, + Type, SparseMetadataFormat, + MMAIntOverflowAttr) { + return verifyShapeMNK(emitError, shape); +} + +Type MmaAtomSM89Type::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + cute::ShapeAttr shape; + Type a, b, c; + if (p.parseLess() || parseShapeMNK(p, shape) || p.parseComma() || + elemTypes(p, a, b, c) || p.parseGreater()) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), shape, a, + b, c); +} +void MmaAtomSM89Type::print(AsmPrinter &p) const { + p << '<'; + printShapeMNK(p, getShapeMnk()); + p << ", "; + printElemTypes(p, getAType(), getBType(), getCType()); + p << " >"; +} +LogicalResult MmaAtomSM89Type::verify(EmitFn emitError, cute::ShapeAttr shape, + Type, Type, Type) { + return verifyShapeMNK(emitError, shape); +} + +// ``. +Type MmaAtomSM90Type::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + cute::ShapeAttr shape; + cute::MajorMode aMajor, bMajor; + Type a, b, c; + MmaFragKind frag; + MMAIntOverflowAttr overflow; + UnitAttr aNeg, bNeg; + if (p.parseLess() || parseShapeMNK(p, shape) || p.parseComma() || + p.parseKeyword("ab_major") || p.parseEqual() || + pairOf(p, aMajor, bMajor) || p.parseComma() || elemTypes(p, a, b, c) || + p.parseComma() || parseFragKind(p, MmaFragKind::rmem, frag)) + return {}; + while (succeeded(p.parseOptionalComma())) { + SMLoc at = p.getCurrentLocation(); + if (succeeded(p.parseOptionalKeyword("a_neg"))) + aNeg = UnitAttr::get(p.getContext()); + else if (succeeded(p.parseOptionalKeyword("b_neg"))) + bNeg = UnitAttr::get(p.getContext()); + else if (succeeded(p.parseOptionalKeyword("int_overflow"))) { + if (p.parseEqual() || field(p, overflow)) + return {}; + } else { + p.emitError(at, "expected `int_overflow`, `a_neg`, or `b_neg`"); + return {}; + } + } + if (p.parseGreater()) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), shape, + aMajor, bMajor, a, b, c, frag, overflow, aNeg, bNeg); +} +void MmaAtomSM90Type::print(AsmPrinter &p) const { + p << '<'; + printShapeMNK(p, getShapeMnk()); + p << ", ab_major = (" << stringifyEnum(getAMajor()) << ", " + << stringifyEnum(getBMajor()) << "), "; + printElemTypes(p, getAType(), getBType(), getCType()); + p << ", frag_kind = " << fragKindText(getAFragKind()); + printAttrField(p, "int_overflow", getIntOverflow()); + if (getANeg()) + p << ", a_neg"; + if (getBNeg()) + p << ", b_neg"; + p << '>'; +} +LogicalResult MmaAtomSM90Type::verify(EmitFn emitError, cute::ShapeAttr shape, + cute::MajorMode, cute::MajorMode, Type, + Type, Type, MmaFragKind, + MMAIntOverflowAttr, UnitAttr, UnitAttr) { + return verifyShapeMNK(emitError, shape); +} + +// ``. +Type MmaAtomSM120BlockScaledType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + cute::ShapeAttr shape; + int vec; + Type a, b, c, sf; + bool useTV; + if (p.parseLess() || parseShapeMNK(p, shape) || + nextKeyField(p, "vec_size", vec) || p.parseComma() || + elemTypes(p, a, b, c) || nextKeyField(p, "sf_type", sf) || + nextKeyField(p, "use_sf_layout_TV", useTV) || p.parseGreater()) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), shape, + vec, a, b, c, sf, useTV); +} +void MmaAtomSM120BlockScaledType::print(AsmPrinter &p) const { + p << '<'; + printShapeMNK(p, getShapeMnk()); + p << ", vec_size = " << getVecSize() << ", "; + printElemTypes(p, getAType(), getBType(), getCType()); + p << ", sf_type = " << getSfType() + << ", use_sf_layout_TV = " << (getUseSfLayout_TV() ? "true" : "false") + << '>'; +} +LogicalResult MmaAtomSM120BlockScaledType::verify(EmitFn emitError, + cute::ShapeAttr shape, int, + Type, Type, Type, Type, + bool) { + return verifyShapeMNK(emitError, shape); +} + +// UMMA atoms: one grammar, fields present by kind (see CuteNVGPUTypes.td). +namespace { +struct Umma { + bool sf, sp, bs, sm107; + cute::ShapeAttr shape; + int numCta = 1; + cute::MajorMode aMajor, bMajor; + Type a, b, c, sfType, eType; + SparseMetadataFormat metadata = SparseMetadataFormat::tid; + MmaFragKind frag = MmaFragKind::smem_desc; + int scaleOrVec = 0; + Arch arch = Arch::sm_100; + MmaCollectorOp aCollector = MmaCollectorOp::discard, + bCollector = MmaCollectorOp::discard; + MMAIntOverflowAttr overflow; +}; + +ParseResult parseUmma(AsmParser &p, Umma &u) { + if (p.parseLess() || parseShapeMNK(p, u.shape) || + nextKeyField(p, "num_cta", u.numCta) || p.parseComma() || + p.parseKeyword("ab_major") || p.parseEqual() || + pairOf(p, u.aMajor, u.bMajor) || p.parseComma() || + elemTypes(p, u.a, u.b, u.c)) + return failure(); + if (u.sf && nextKeyField(p, "sf_type", u.sfType)) + return failure(); + if (u.sp && !u.sf && nextKeyField(p, "e_type", u.eType)) + return failure(); + if (u.sp && nextKeyField(p, "sparse_metadata_format", u.metadata)) + return failure(); + if (p.parseComma() || parseFragKind(p, MmaFragKind::tmem, u.frag) || + nextKeyField(p, u.bs ? "vec_size" : "c_scale_exp", u.scaleOrVec)) + return failure(); + bool comma = succeeded(p.parseOptionalComma()); + if (comma && u.bs && !u.sm107 && + succeeded(p.parseOptionalKeyword("arch_promote"))) { + if (p.parseEqual() || field(p, u.arch)) + return failure(); + comma = succeeded(p.parseOptionalComma()); + } + if (u.sm107) { + if (!comma) + return p.parseComma(); + if (p.parseKeyword("ab_collector_op") || p.parseEqual() || + pairOf(p, u.aCollector, u.bCollector)) + return failure(); + comma = succeeded(p.parseOptionalComma()); + } + if (comma && keyField(p, "int_overflow", u.overflow)) + return failure(); + return p.parseGreater(); +} + +void printUmma(AsmPrinter &p, const Umma &u) { + p << '<'; + printShapeMNK(p, u.shape); + p << ", num_cta = " << u.numCta << ", ab_major = (" << stringifyEnum(u.aMajor) + << ", " << stringifyEnum(u.bMajor) << "), "; + printElemTypes(p, u.a, u.b, u.c); + if (u.sf) + p << ", sf_type = " << u.sfType; + if (u.sp && !u.sf) + p << ", e_type = " << u.eType; + if (u.sp) + p << ", sparse_metadata_format = " << stringifyEnum(u.metadata); + p << ", frag_kind = " << fragKindText(u.frag) + << (u.bs ? ", vec_size = " : ", c_scale_exp = ") << u.scaleOrVec; + if (u.bs && !u.sm107 && (u.arch != Arch::sm_100 || u.overflow)) + p << ", arch_promote = " << stringifyEnum(u.arch); + if (u.sm107) + p << ", ab_collector_op = (" << stringifyEnum(u.aCollector) << ", " + << stringifyEnum(u.bCollector) << ")"; + printAttrField(p, "int_overflow", u.overflow); + p << '>'; +} + +// `sourced` picks the atom's wording of the tensor-memory A rule. +LogicalResult verifyUmma(EmitFn emitError, cute::ShapeAttr shape, int numCta, + MmaFragKind frag, cute::MajorMode aMajor, + int cScaleExp = 0, bool sourced = false) { + if (failed(verifyShapeMNK(emitError, shape))) + return failure(); + if (numCta != 1 && numCta != 2) + return emitError() << "expects `num_cta` equals to 1 or 2, but got " + << numCta; + if (cScaleExp > 15) + return emitError() << "expects `c_scale_exp` in range [0, 15], but got " + << cScaleExp; + if (frag == MmaFragKind::tmem && aMajor != cute::MajorMode::k) + return emitError() + << (sourced + ? "a operand must be k-major when sourced from tmem" + : "a operand must be of major-mode k when coming from tmem"); + return success(); +} +} // namespace + +Type MmaAtomSM100UMMAType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + Umma u{false, false, false, false}; + if (parseUmma(p, u)) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), u.shape, + u.numCta, u.aMajor, u.bMajor, u.a, u.b, u.c, u.frag, + u.scaleOrVec, u.overflow); +} +void MmaAtomSM100UMMAType::print(AsmPrinter &p) const { + Umma u{false, false, false, false}; + u.shape = getShapeMnk(), u.numCta = getNumCta(), u.aMajor = getAMajor(), + u.bMajor = getBMajor(), u.a = getAType(), u.b = getBType(), u.c = getCType(), + u.frag = getAFragKind(), u.scaleOrVec = getCScaleExp(), + u.overflow = getIntOverflow(); + printUmma(p, u); +} +LogicalResult MmaAtomSM100UMMAType::verify(EmitFn emitError, + cute::ShapeAttr shape, int numCta, + cute::MajorMode aMajor, + cute::MajorMode, Type, Type, Type, + MmaFragKind frag, int cScaleExp, + MMAIntOverflowAttr) { + return verifyUmma(emitError, shape, numCta, frag, aMajor, cScaleExp); +} + +Type MmaAtomSM100UMMASparseType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + Umma u{false, true, false, false}; + if (parseUmma(p, u)) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), u.shape, + u.numCta, u.aMajor, u.bMajor, u.a, u.b, u.c, u.eType, + u.metadata, u.frag, u.scaleOrVec, u.overflow); +} +void MmaAtomSM100UMMASparseType::print(AsmPrinter &p) const { + Umma u{false, true, false, false}; + u.shape = getShapeMnk(), u.numCta = getNumCta(), u.aMajor = getAMajor(), + u.bMajor = getBMajor(), u.a = getAType(), u.b = getBType(), u.c = getCType(), + u.eType = getEType(), u.metadata = getSparseMetadataFormat(), + u.frag = getAFragKind(), u.scaleOrVec = getCScaleExp(), + u.overflow = getIntOverflow(); + printUmma(p, u); +} +LogicalResult MmaAtomSM100UMMASparseType::verify( + EmitFn emitError, cute::ShapeAttr shape, int numCta, cute::MajorMode aMajor, + cute::MajorMode, Type, Type, Type, Type, SparseMetadataFormat, + MmaFragKind frag, int cScaleExp, MMAIntOverflowAttr) { + return verifyUmma(emitError, shape, numCta, frag, aMajor, cScaleExp); +} + +Type MmaAtomSM100UMMABlockScaledType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + Umma u{true, false, true, false}; + if (parseUmma(p, u)) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), u.shape, + u.numCta, u.aMajor, u.bMajor, u.a, u.b, u.c, u.sfType, + u.frag, u.scaleOrVec, u.arch, u.overflow); +} +void MmaAtomSM100UMMABlockScaledType::print(AsmPrinter &p) const { + Umma u{true, false, true, false}; + u.shape = getShapeMnk(), u.numCta = getNumCta(), u.aMajor = getAMajor(), + u.bMajor = getBMajor(), u.a = getAType(), u.b = getBType(), u.c = getCType(), + u.sfType = getSfType(), u.frag = getAFragKind(), u.scaleOrVec = getVecSize(), + u.arch = getArchPromote(), u.overflow = getIntOverflow(); + printUmma(p, u); +} +LogicalResult MmaAtomSM100UMMABlockScaledType::verify( + EmitFn emitError, cute::ShapeAttr shape, int numCta, cute::MajorMode aMajor, + cute::MajorMode, Type, Type, Type, Type, MmaFragKind frag, int, Arch, + MMAIntOverflowAttr) { + return verifyUmma(emitError, shape, numCta, frag, aMajor, 0, + /*sourced=*/true); +} + +Type MmaAtomSM100UMMABlockScaledSparseType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + Umma u{true, true, true, false}; + if (parseUmma(p, u)) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), u.shape, + u.numCta, u.aMajor, u.bMajor, u.a, u.b, u.c, u.sfType, + u.metadata, u.frag, u.scaleOrVec, u.arch, u.overflow); +} +void MmaAtomSM100UMMABlockScaledSparseType::print(AsmPrinter &p) const { + Umma u{true, true, true, false}; + u.shape = getShapeMnk(), u.numCta = getNumCta(), u.aMajor = getAMajor(), + u.bMajor = getBMajor(), u.a = getAType(), u.b = getBType(), u.c = getCType(), + u.sfType = getSfType(), u.metadata = getSparseMetadataFormat(), + u.frag = getAFragKind(), u.scaleOrVec = getVecSize(), + u.arch = getArchPromote(), u.overflow = getIntOverflow(); + printUmma(p, u); +} +LogicalResult MmaAtomSM100UMMABlockScaledSparseType::verify( + EmitFn emitError, cute::ShapeAttr shape, int numCta, cute::MajorMode aMajor, + cute::MajorMode, Type, Type, Type, Type, SparseMetadataFormat, + MmaFragKind frag, int, Arch, MMAIntOverflowAttr) { + return verifyUmma(emitError, shape, numCta, frag, aMajor); +} + +Type MmaAtomSM107UMMAType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + Umma u{false, false, false, true}; + if (parseUmma(p, u)) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), u.shape, + u.numCta, u.aMajor, u.bMajor, u.a, u.b, u.c, u.frag, + u.scaleOrVec, u.aCollector, u.bCollector, u.overflow); +} +void MmaAtomSM107UMMAType::print(AsmPrinter &p) const { + Umma u{false, false, false, true}; + u.shape = getShapeMnk(), u.numCta = getNumCta(), u.aMajor = getAMajor(), + u.bMajor = getBMajor(), u.a = getAType(), u.b = getBType(), u.c = getCType(), + u.frag = getAFragKind(), u.scaleOrVec = getCScaleExp(), + u.aCollector = getACollectorOp(), u.bCollector = getBCollectorOp(), + u.overflow = getIntOverflow(); + printUmma(p, u); +} +LogicalResult MmaAtomSM107UMMAType::verify( + EmitFn emitError, cute::ShapeAttr shape, int numCta, cute::MajorMode aMajor, + cute::MajorMode, Type, Type, Type, MmaFragKind frag, int cScaleExp, + MmaCollectorOp, MmaCollectorOp, MMAIntOverflowAttr) { + return verifyUmma(emitError, shape, numCta, frag, aMajor, cScaleExp); +} + +Type MmaAtomSM107UMMASparseType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + Umma u{false, true, false, true}; + if (parseUmma(p, u)) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), u.shape, + u.numCta, u.aMajor, u.bMajor, u.a, u.b, u.c, u.eType, + u.metadata, u.frag, u.scaleOrVec, u.aCollector, + u.bCollector, u.overflow); +} +void MmaAtomSM107UMMASparseType::print(AsmPrinter &p) const { + Umma u{false, true, false, true}; + u.shape = getShapeMnk(), u.numCta = getNumCta(), u.aMajor = getAMajor(), + u.bMajor = getBMajor(), u.a = getAType(), u.b = getBType(), u.c = getCType(), + u.eType = getEType(), u.metadata = getSparseMetadataFormat(), + u.frag = getAFragKind(), u.scaleOrVec = getCScaleExp(), + u.aCollector = getACollectorOp(), u.bCollector = getBCollectorOp(), + u.overflow = getIntOverflow(); + printUmma(p, u); +} +LogicalResult MmaAtomSM107UMMASparseType::verify( + EmitFn emitError, cute::ShapeAttr shape, int numCta, cute::MajorMode aMajor, + cute::MajorMode, Type, Type, Type, Type, SparseMetadataFormat, + MmaFragKind frag, int cScaleExp, MmaCollectorOp, MmaCollectorOp, + MMAIntOverflowAttr) { + return verifyUmma(emitError, shape, numCta, frag, aMajor, cScaleExp); +} + +Type MmaAtomSM107UMMABlockScaledType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + Umma u{true, false, true, true}; + if (parseUmma(p, u)) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), u.shape, + u.numCta, u.aMajor, u.bMajor, u.a, u.b, u.c, u.sfType, + u.frag, u.scaleOrVec, u.aCollector, u.bCollector, + u.overflow); +} +void MmaAtomSM107UMMABlockScaledType::print(AsmPrinter &p) const { + Umma u{true, false, true, true}; + u.shape = getShapeMnk(), u.numCta = getNumCta(), u.aMajor = getAMajor(), + u.bMajor = getBMajor(), u.a = getAType(), u.b = getBType(), u.c = getCType(), + u.sfType = getSfType(), u.frag = getAFragKind(), u.scaleOrVec = getVecSize(), + u.aCollector = getACollectorOp(), u.bCollector = getBCollectorOp(), + u.overflow = getIntOverflow(); + printUmma(p, u); +} +LogicalResult MmaAtomSM107UMMABlockScaledType::verify( + EmitFn emitError, cute::ShapeAttr shape, int numCta, cute::MajorMode aMajor, + cute::MajorMode, Type, Type, Type, Type, MmaFragKind frag, int, + MmaCollectorOp, MmaCollectorOp, MMAIntOverflowAttr) { + return verifyUmma(emitError, shape, numCta, frag, aMajor, 0, + /*sourced=*/true); +} + +Type MmaAtomSM107UMMABlockScaledSparseType::parse(AsmParser &p) { + SMLoc loc = p.getCurrentLocation(); + Umma u{true, true, true, true}; + if (parseUmma(p, u)) + return {}; + return getChecked([&] { return p.emitError(loc); }, p.getContext(), u.shape, + u.numCta, u.aMajor, u.bMajor, u.a, u.b, u.c, u.sfType, + u.metadata, u.frag, u.scaleOrVec, u.aCollector, + u.bCollector, u.overflow); +} +void MmaAtomSM107UMMABlockScaledSparseType::print(AsmPrinter &p) const { + Umma u{true, true, true, true}; + u.shape = getShapeMnk(), u.numCta = getNumCta(), u.aMajor = getAMajor(), + u.bMajor = getBMajor(), u.a = getAType(), u.b = getBType(), u.c = getCType(), + u.sfType = getSfType(), u.metadata = getSparseMetadataFormat(), + u.frag = getAFragKind(), u.scaleOrVec = getVecSize(), + u.aCollector = getACollectorOp(), u.bCollector = getBCollectorOp(), + u.overflow = getIntOverflow(); + printUmma(p, u); +} +LogicalResult MmaAtomSM107UMMABlockScaledSparseType::verify( + EmitFn emitError, cute::ShapeAttr shape, int numCta, cute::MajorMode aMajor, + cute::MajorMode, Type, Type, Type, Type, SparseMetadataFormat, + MmaFragKind frag, int, MmaCollectorOp, MmaCollectorOp, MMAIntOverflowAttr) { + return verifyUmma(emitError, shape, numCta, frag, aMajor, 0, + /*sourced=*/true); +} + +//===----------------------------------------------------------------------===// +// Dialect +//===----------------------------------------------------------------------===// + +// `!cute_nvgpu.iN[]` is the compiler's alias of `!cute.iN[]`. +Type CuteNVGPUDialect::parseType(DialectAsmParser &parser) const { + SMLoc loc = parser.getCurrentLocation(); + StringRef mnemonic; + Type type; + OptionalParseResult result = generatedTypeParser(parser, &mnemonic, type); + if (result.has_value()) + return succeeded(*result) ? type : Type(); + StringRef width = mnemonic; + if (width.consume_front("i") && !width.empty() && + llvm::all_of(width, llvm::isDigit)) { + std::string spec = ("!cute." + mnemonic).str(); + int64_t divisor; + if (succeeded(parser.parseOptionalLess())) { + if (parser.parseKeyword("divby") || parser.parseInteger(divisor) || + parser.parseGreater()) + return {}; + spec += ""; + } + return mlir::parseType(spec, getContext()); + } + parser.emitError(loc) << "unknown type `" << mnemonic << "` in dialect `" + << getNamespace() << "`"; + return {}; +} + +void CuteNVGPUDialect::printType(Type type, DialectAsmPrinter &printer) const { + if (succeeded(generatedTypePrinter(type, printer))) + return; + llvm_unreachable("unexpected non-CuteNVGPU type"); +} + +void CuteNVGPUDialect::initialize() { + addAttributes< +#define GET_ATTRDEF_LIST +#include "mlir/dialects/cute_nvgpu/CuteNVGPUAttrs.cpp.inc" + >(); + addTypes< +#define GET_TYPEDEF_LIST +#include "mlir/dialects/cute_nvgpu/CuteNVGPUTypes.cpp.inc" + >(); + addOperations< +#define GET_OP_LIST +#include "mlir/dialects/cute_nvgpu/CuteNVGPUOps.cpp.inc" + >(); +} diff --git a/mlir/dialects/cute_nvgpu/CuteNVGPUDialect.h b/mlir/dialects/cute_nvgpu/CuteNVGPUDialect.h new file mode 100644 index 00000000..19d8cb14 --- /dev/null +++ b/mlir/dialects/cute_nvgpu/CuteNVGPUDialect.h @@ -0,0 +1,38 @@ +//===- CuteNVGPUDialect.h - CuTe NVIDIA GPU dialect -------------*- C++ -*-===// + +#ifndef MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUDIALECT_H +#define MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUDIALECT_H + +#include "cute_ir/Dialect/Cute/IR/CuteDialect.h" + +#include "mlir/Bytecode/BytecodeOpInterface.h" +#include "mlir/IR/BuiltinAttributes.h" +#include "mlir/IR/DialectImplementation.h" +#include "mlir/IR/OpImplementation.h" +#include "mlir/Interfaces/InferTypeOpInterface.h" +#include "mlir/Interfaces/SideEffectInterfaces.h" + +#include + +#include "mlir/dialects/cute_nvgpu/CuteNVGPUDialect.h.inc" + +#include "mlir/dialects/cute_nvgpu/CuteNVGPUEnums.h.inc" + +namespace mlir::cutlass_compiler::cute_nvgpu { +/// Parses `true` or `false`. +FailureOr parseBool(AsmParser &parser); +/// The TMA data format a non-executable TMA atom takes for an element type +/// when none is given, if there is one. +std::optional defaultTmaDataFormat(Type elementType); +} // namespace mlir::cutlass_compiler::cute_nvgpu + +#define GET_ATTRDEF_CLASSES +#include "mlir/dialects/cute_nvgpu/CuteNVGPUAttrs.h.inc" + +#define GET_TYPEDEF_CLASSES +#include "mlir/dialects/cute_nvgpu/CuteNVGPUTypes.h.inc" + +#define GET_OP_CLASSES +#include "mlir/dialects/cute_nvgpu/CuteNVGPUOps.h.inc" + +#endif // MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUDIALECT_H diff --git a/mlir/dialects/cute_nvgpu/CuteNVGPUDialect.td b/mlir/dialects/cute_nvgpu/CuteNVGPUDialect.td new file mode 100644 index 00000000..5f6c6680 --- /dev/null +++ b/mlir/dialects/cute_nvgpu/CuteNVGPUDialect.td @@ -0,0 +1,36 @@ +//===- CuteNVGPUDialect.td - CuTe NVIDIA GPU dialect -------*- tablegen -*-===// + +#ifndef MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUDIALECT_TD +#define MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUDIALECT_TD + +include "mlir/IR/AttrTypeBase.td" +include "mlir/IR/DialectBase.td" +include "mlir/IR/OpBase.td" + +def CuteNVGPU_Dialect : Dialect { + let name = "cute_nvgpu"; + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let summary = "NVIDIA GPU operations and atom types used by CuTe-DSL"; + let description = [{ + This dialect mirrors the operation, type and attribute registry of + nvidia-cutlass-dsl 4.8.0, embedded in the proprietary CuTe-DSL compiler. + The type parser is hand-written only to accept `!cute_nvgpu.iN`, the + compiler's alias of `!cute.iN`. + }]; + let extraClassDeclaration = [{ + ::mlir::Type parseType(::mlir::DialectAsmParser &parser) const override; + void printType(::mlir::Type type, + ::mlir::DialectAsmPrinter &printer) const override; + }]; + let useDefaultAttributePrinterParser = 1; +} + +class CuteNVGPU_Op traits = []> + : Op; + +class CuteNVGPU_Type + : TypeDef { + let mnemonic = typeMnemonic; +} + +#endif // MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUDIALECT_TD diff --git a/mlir/dialects/cute_nvgpu/CuteNVGPUEnums.td b/mlir/dialects/cute_nvgpu/CuteNVGPUEnums.td new file mode 100644 index 00000000..a583a57e --- /dev/null +++ b/mlir/dialects/cute_nvgpu/CuteNVGPUEnums.td @@ -0,0 +1,683 @@ +// The cute_nvgpu dialect's enums, generated from the nvidia-cutlass-dsl 4.8.0 +// enum bindings (`_cute_nvgpu_enum_gen.py`); do not edit. +// +// Case values and spellings are the compiler's. An enum the bindings build +// as a dialect attribute also has its EnumAttr, printed `#`. +// The cute enums the nvgpu bindings repeat are the cute dialect's; their +// nvgpu attributes are in CuteNVGPUAttrs.td, so that -gen-enum-decls +// over this file emits the nvgpu enums only. + +#ifndef MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUENUMS_TD +#define MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUENUMS_TD + +include "mlir/dialects/cute_nvgpu/CuteNVGPUDialect.td" +include "mlir/IR/EnumAttr.td" + +def CuteNVGPU_BinaryOpEnum : I32EnumAttr<"BinaryOp", "Binary operation for single-bit MMA operations", [ + I32EnumAttrCase<"none", 0, "none">, + I32EnumAttrCase<"xor_popc", 1, "xor_popc">, + I32EnumAttrCase<"and_popc", 2, "and_popc"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomCopyFieldBulkCopyG2SEnum : I32EnumAttr<"AtomCopyFieldBulkCopyG2S", "Fields stored in Bulk Load Copy Atom type", [ + I32EnumAttrCase<"tma_bar", 1, "tma_bar">, + I32EnumAttrCase<"mcast_mask", 2, "mcast_mask">, + I32EnumAttrCase<"cache_policy", 7, "cache_policy"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomCopyFieldBulkCopyS2GEnum : I32EnumAttr<"AtomCopyFieldBulkCopyS2G", "Fields stored in Bulk Store Copy Atom type", [ + I32EnumAttrCase<"byte_mask", 4, "byte_mask">, + I32EnumAttrCase<"cache_policy", 7, "cache_policy"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomCopyFieldBulkCopyS2SEnum : I32EnumAttr<"AtomCopyFieldBulkCopyS2S", "Fields stored in Bulk CTA to Cluster Copy Atom type", [ + I32EnumAttrCase<"tma_bar", 1, "tma_bar">, + I32EnumAttrCase<"cta_rank", 5, "cta_rank"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomCopyFieldDsmemStoreEnum : I32EnumAttr<"AtomCopyFieldDsmemStore", "Fields stored in CopyAtomDsmemStoreType", [ + I32EnumAttrCase<"mbar_ptr", 8, "mbar_ptr"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomCopyFieldLoadGlobalEnum : I32EnumAttr<"AtomCopyFieldLoadGlobal", "Fields stored in CopyAtomG2RType", [ + I32EnumAttrCase<"cache_policy", 7, "cache_policy"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomCopyFieldNonExec2DGather4TmaLoadEnum : I32EnumAttr<"AtomCopyFieldNonExec2DGather4TmaLoad", "Fields stored in CopyAtomNonExec2DGather4TmaLoadType", [ + I32EnumAttrCase<"tma_desc", 3, "tma_desc"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomCopyFieldNonExec2DScatter4TmaStoreEnum : I32EnumAttr<"AtomCopyFieldNonExec2DScatter4TmaStore", "Fields stored in CopyAtomNonExec2DScatter4TmaStoreType", [ + I32EnumAttrCase<"tma_desc", 3, "tma_desc"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomCopyFieldNonExecIm2ColTmaLoadEnum : I32EnumAttr<"AtomCopyFieldNonExecIm2ColTmaLoad", "Fields stored in CopyAtomNonExecIm2ColTmaLoadType", [ + I32EnumAttrCase<"im2col_tma_desc", 6, "im2col_tma_desc"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomCopyFieldNonExecIm2ColTmaStoreEnum : I32EnumAttr<"AtomCopyFieldNonExecIm2ColTmaStore", "Fields stored in CopyAtomNonExecIm2ColTmaStoreType", [ + I32EnumAttrCase<"im2col_tma_desc", 6, "im2col_tma_desc"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomCopyFieldNonExecTiledTmaLoadEnum : I32EnumAttr<"AtomCopyFieldNonExecTiledTmaLoad", "Fields stored in CopyAtomNonExecTiledTmaLoadType", [ + I32EnumAttrCase<"tma_desc", 3, "tma_desc"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomCopyFieldNonExecTiledTmaReduceEnum : I32EnumAttr<"AtomCopyFieldNonExecTiledTmaReduce", "Fields stored in CopyAtomNonExecTiledTmaReduceType", [ + I32EnumAttrCase<"tma_desc", 3, "tma_desc"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomCopyFieldNonExecTiledTmaStoreEnum : I32EnumAttr<"AtomCopyFieldNonExecTiledTmaStore", "Fields stored in CopyAtomNonExecTiledTmaStoreType", [ + I32EnumAttrCase<"tma_desc", 3, "tma_desc"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomCopyFieldStoreGlobalEnum : I32EnumAttr<"AtomCopyFieldStoreGlobal", "Fields stored in CopyAtomR2GType", [ + I32EnumAttrCase<"cache_policy", 7, "cache_policy"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomCopyFieldTmaLoadEnum : I32EnumAttr<"AtomCopyFieldTmaLoad", "Fields stored in Tma Load Copy Atom type", [ + I32EnumAttrCase<"tma_descriptor_ptr", 0, "tma_descriptor_ptr">, + I32EnumAttrCase<"tma_bar", 1, "tma_bar">, + I32EnumAttrCase<"mcast_mask", 2, "mcast_mask">, + I32EnumAttrCase<"cache_policy", 7, "cache_policy">, + I32EnumAttrCase<"g_stride", 9, "g_stride"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomCopyFieldTmaReduceEnum : I32EnumAttr<"AtomCopyFieldTmaReduce", "Fields stored in Tma Reduce Copy Atom type", [ + I32EnumAttrCase<"tma_descriptor_ptr", 0, "tma_descriptor_ptr">, + I32EnumAttrCase<"cache_policy", 7, "cache_policy">, + I32EnumAttrCase<"g_stride", 9, "g_stride"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomCopyFieldTmaStoreEnum : I32EnumAttr<"AtomCopyFieldTmaStore", "Fields stored in Tma Store Copy Atom type", [ + I32EnumAttrCase<"tma_descriptor_ptr", 0, "tma_descriptor_ptr">, + I32EnumAttrCase<"cache_policy", 7, "cache_policy">, + I32EnumAttrCase<"g_stride", 9, "g_stride"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomMmaFieldSM80SparseEnum : I32EnumAttr<"AtomMmaFieldSM80Sparse", "Fields stored in MMA Atom types for SM80", [ + I32EnumAttrCase<"e", 0, "e"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomMmaFieldSM90Enum : I32EnumAttr<"AtomMmaFieldSM90", "Fields stored in MMA Atom types for SM90", [ + I32EnumAttrCase<"accum_c", 0, "accum_c"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomMmaFieldSM100BlockScaledEnum : I32EnumAttr<"AtomMmaFieldSM100BlockScaled", "Fields stored in MMA Atom types for SM100 (block scaled)", [ + I32EnumAttrCase<"accum_c", 0, "accum_c">, + I32EnumAttrCase<"neg_a", 1, "neg_a">, + I32EnumAttrCase<"neg_b", 2, "neg_b">, + I32EnumAttrCase<"sf_a", 3, "sf_a">, + I32EnumAttrCase<"sf_b", 4, "sf_b"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomMmaFieldSM100BlockScaledSparseEnum : I32EnumAttr<"AtomMmaFieldSM100BlockScaledSparse", "Fields stored in MMA Atom types for SM100 (block scaled)", [ + I32EnumAttrCase<"accum_c", 0, "accum_c">, + I32EnumAttrCase<"neg_a", 1, "neg_a">, + I32EnumAttrCase<"neg_b", 2, "neg_b">, + I32EnumAttrCase<"sf_a", 3, "sf_a">, + I32EnumAttrCase<"sf_b", 4, "sf_b">, + I32EnumAttrCase<"e", 5, "e"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomMmaFieldSM100Enum : I32EnumAttr<"AtomMmaFieldSM100", "Fields stored in MMA Atom types for SM100", [ + I32EnumAttrCase<"accum_c", 0, "accum_c">, + I32EnumAttrCase<"neg_a", 1, "neg_a">, + I32EnumAttrCase<"neg_b", 2, "neg_b">, + I32EnumAttrCase<"disable_output_lane", 6, "disable_output_lane"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomMmaFieldSM100SparseEnum : I32EnumAttr<"AtomMmaFieldSM100Sparse", "Fields stored in sparse MMA Atom types for SM100", [ + I32EnumAttrCase<"accum_c", 0, "accum_c">, + I32EnumAttrCase<"neg_a", 1, "neg_a">, + I32EnumAttrCase<"neg_b", 2, "neg_b">, + I32EnumAttrCase<"e", 5, "e"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_AtomMmaFieldSM120BlockScaledEnum : I32EnumAttr<"AtomMmaFieldSM120BlockScaled", "Fields stored in MMA Atom types for SM120 (block scaled)", [ + I32EnumAttrCase<"sf_a", 0, "sf_a">, + I32EnumAttrCase<"sf_b", 1, "sf_b"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_CacheEvictionPriorityEnum : I32EnumAttr<"CacheEvictionPriority", "Cute Cache Eviction Priority kind", [ + I32EnumAttrCase<"EVICT_NORMAL", 0, "evict_normal">, + I32EnumAttrCase<"EVICT_FIRST", 1, "evict_first">, + I32EnumAttrCase<"EVICT_LAST", 2, "evict_last">, + I32EnumAttrCase<"EVICT_UNCHANGED", 3, "evict_unchanged">, + I32EnumAttrCase<"NO_ALLOCATE", 4, "no_allocate"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_CopyS2TBroadcastEnum : I32EnumAttr<"CopyS2TBroadcast", "Broadcast modes for the different utccp instructions", [ + I32EnumAttrCase<"none", 0, "none">, + I32EnumAttrCase<"lw_0213", 1, "lw_0213">, + I32EnumAttrCase<"lw_0123", 2, "lw_0123">, + I32EnumAttrCase<"x4", 3, "x4"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_GatherScatterTmaLoadEnum : I32EnumAttr<"GatherScatterTmaLoad", "The various kinds of TMA loads in gather/scatter mode", [ + I32EnumAttrCase<"sm_100", 0, "sm_100">, + I32EnumAttrCase<"sm_100_multicast", 1, "sm_100_multicast">, + I32EnumAttrCase<"sm_100_2sm", 2, "sm_100_2sm">, + I32EnumAttrCase<"sm_100_2sm_multicast", 3, "sm_100_2sm_multicast"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_ArchEnum : I32EnumAttr<"Arch", "NVGpu arch number like sm_103", [ + I32EnumAttrCase<"sm_52", 520, "sm_52">, + I32EnumAttrCase<"sm_70", 700, "sm_70">, + I32EnumAttrCase<"sm_75", 750, "sm_75">, + I32EnumAttrCase<"sm_80", 800, "sm_80">, + I32EnumAttrCase<"sm_86", 860, "sm_86">, + I32EnumAttrCase<"sm_90", 900, "sm_90">, + I32EnumAttrCase<"sm_100", 1000, "sm_100">, + I32EnumAttrCase<"sm_103", 1030, "sm_103">, + I32EnumAttrCase<"sm_107", 1070, "sm_107"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_Im2ColTmaLoadEnum : I32EnumAttr<"Im2ColTmaLoad", "The various kinds of TMA loads in im2col mode", [ + I32EnumAttrCase<"sm_90", 0, "sm_90">, + I32EnumAttrCase<"sm_90_multicast", 1, "sm_90_multicast">, + I32EnumAttrCase<"sm_100_2sm", 2, "sm_100_2sm">, + I32EnumAttrCase<"sm_100_2sm_multicast", 3, "sm_100_2sm_multicast"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_L2PrefetchSizeEnum : I32EnumAttr<"L2PrefetchSize", "CuTe L2 prefetch size options", [ + I32EnumAttrCase<"NONE", 0, "none">, + I32EnumAttrCase<"RESERVED", 1, "reserved">, + I32EnumAttrCase<"SIZE_64B", 2, "size_64b">, + I32EnumAttrCase<"SIZE_128B", 3, "size_128b">, + I32EnumAttrCase<"SIZE_256B", 4, "size_256b"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_LdReduceAccPrecisionKindEnum : I32EnumAttr<"LdReduceAccPrecisionKind", "multimem ld_reduce accumulation precision kind", [ + I32EnumAttrCase<"F16", 0, "F16">, + I32EnumAttrCase<"F32", 1, "F32"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_LdsmSzPatternEnum : I32EnumAttr<"LdsmSzPattern", "LDSM's sz pattern, describing the bit size", [ + I32EnumAttrCase<"u16", 0, "u16">, + I32EnumAttrCase<"u4to8", 1, "u4to8">, + I32EnumAttrCase<"s4to8", 2, "s4to8">, + I32EnumAttrCase<"u2to4", 3, "u2to4">, + I32EnumAttrCase<"s2to4", 4, "s2to4">, + I32EnumAttrCase<"u4x16p64to8", 5, "u4x16p64to8">, + I32EnumAttrCase<"u6x16p32to8", 6, "u6x16p32to8">, + I32EnumAttrCase<"u8", 7, "u8"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_LoadCacheModeEnum : I32EnumAttr<"LoadCacheMode", "Cache modes for the load instructions", [ + I32EnumAttrCase<"always", 0, "always">, + I32EnumAttrCase<"global", 1, "global">, + I32EnumAttrCase<"streaming", 2, "streaming">, + I32EnumAttrCase<"last_use", 3, "last_use">, + I32EnumAttrCase<"none", 4, "none"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_MMAIntOverflowEnum : I32EnumAttr<"MMAIntOverflow", "MMA overflow options", [ + I32EnumAttrCase<"satfinite", 1, "satfinite">, + I32EnumAttrCase<"wrapped", 0, "wrapped"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_MemOrderKindEnum : I32EnumAttr<"MemOrderKind", "Memory Ordering kind", [ + I32EnumAttrCase<"WEAK", 0, "weak">, + I32EnumAttrCase<"RELAXED", 1, "relaxed">, + I32EnumAttrCase<"ACQUIRE", 2, "acquire">, + I32EnumAttrCase<"RELEASE", 3, "release">, + I32EnumAttrCase<"ACQ_REL", 4, "acq_rel">, + I32EnumAttrCase<"SC", 5, "sc">, + I32EnumAttrCase<"MMIO", 6, "mmio">, + I32EnumAttrCase<"CONSTANT", 7, "constant">, + I32EnumAttrCase<"VOLATILE", 8, "volatile"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_MemScopeKindEnum : I32EnumAttr<"MemScopeKind", "Cute Memory Scope kind", [ + I32EnumAttrCase<"CTA", 0, "cta">, + I32EnumAttrCase<"CLUSTER", 1, "cluster">, + I32EnumAttrCase<"GPU", 2, "gpu">, + I32EnumAttrCase<"SYS", 3, "sys"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_MmaCollectorOpEnum : I32EnumAttr<"MmaCollectorOp", "Enums for the mma collector op", [ + I32EnumAttrCase<"discard", 0, "discard">, + I32EnumAttrCase<"lastuse", 1, "lastuse">, + I32EnumAttrCase<"fill", 2, "fill">, + I32EnumAttrCase<"use", 3, "use"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_MmaFragKindEnum : I32EnumAttr<"MmaFragKind", "Enums for the mma frag type", [ + I32EnumAttrCase<"smem_desc", 0, "smem_desc">, + I32EnumAttrCase<"sparse_smem_desc", 1, "sparse_smem_desc">, + I32EnumAttrCase<"tmem", 2, "tmem">, + I32EnumAttrCase<"tmem_ws", 3, "tmem_ws">, + I32EnumAttrCase<"tmem_e", 4, "tmem_e">, + I32EnumAttrCase<"tmem_sf", 5, "tmem_sf">, + I32EnumAttrCase<"rmem", 6, "rmem"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_ReductionKindEnum : I32EnumAttr<"ReductionKind", "Op for the TMASTORE instruction", [ + I32EnumAttrCase<"ADD", 0, "ADD">, + I32EnumAttrCase<"MIN", 1, "MIN">, + I32EnumAttrCase<"MAX", 2, "MAX">, + I32EnumAttrCase<"INC", 3, "INC">, + I32EnumAttrCase<"DEC", 4, "DEC">, + I32EnumAttrCase<"AND", 5, "AND">, + I32EnumAttrCase<"OR", 6, "OR">, + I32EnumAttrCase<"XOR", 7, "XOR"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_SharedSpaceEnum : I32EnumAttr<"SharedSpace", "CuTe Shared Space kind", [ + I32EnumAttrCase<"CTA", 0, "cta">, + I32EnumAttrCase<"CLUSTER", 1, "cluster"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_SparseMetadataFormatEnum : I32EnumAttr<"SparseMetadataFormat", "Metadata format used in sparse MMA", [ + I32EnumAttrCase<"tid", 0, "tid"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_StoreCacheModeEnum : I32EnumAttr<"StoreCacheMode", "Cache modes for the store instructions", [ + I32EnumAttrCase<"write_back", 0, "write_back">, + I32EnumAttrCase<"global", 1, "global">, + I32EnumAttrCase<"streaming", 2, "streaming">, + I32EnumAttrCase<"write_through", 3, "write_through">, + I32EnumAttrCase<"none", 4, "none"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_TiledTmaLoadEnum : I32EnumAttr<"TiledTmaLoad", "The various kinds of TMA loads in tiled mode", [ + I32EnumAttrCase<"sm_90", 0, "sm_90">, + I32EnumAttrCase<"sm_90_multicast", 1, "sm_90_multicast">, + I32EnumAttrCase<"sm_100_2sm", 2, "sm_100_2sm">, + I32EnumAttrCase<"sm_100_2sm_multicast", 3, "sm_100_2sm_multicast"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_TmaDataFormatEnum : I32EnumAttr<"TmaDataFormat", "Bits 74-71 of the TMA descriptor", [ + I32EnumAttrCase<"U8", 0, "U8">, + I32EnumAttrCase<"U16", 1, "U16">, + I32EnumAttrCase<"U32", 2, "U32">, + I32EnumAttrCase<"S32", 3, "S32">, + I32EnumAttrCase<"U64", 4, "U64">, + I32EnumAttrCase<"S64", 5, "S64">, + I32EnumAttrCase<"F16_RN", 6, "F16_RN">, + I32EnumAttrCase<"F32_RN", 7, "F32_RN">, + I32EnumAttrCase<"F32_FTZ_RN", 8, "F32_FTZ_RN">, + I32EnumAttrCase<"F64_RN", 9, "F64_RN">, + I32EnumAttrCase<"BF16_RN", 10, "BF16_RN">, + I32EnumAttrCase<"U4", 11, "U4">, + I32EnumAttrCase<"U4_UNPACK_U8", 12, "U4_UNPACK_U8">, + I32EnumAttrCase<"U6_UNPACK_U8", 13, "U6_UNPACK_U8">, + I32EnumAttrCase<"TF32_RN", 14, "TF32_RN">, + I32EnumAttrCase<"TF32_FTZ_RN", 15, "TF32_FTZ_RN"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_TmaLoadModeEnum : I32EnumAttr<"TmaLoadMode", "Modes for the TMA load instruction", [ + I32EnumAttrCase<"tiled", 0, "tiled">, + I32EnumAttrCase<"im2col", 1, "im2col">, + I32EnumAttrCase<"w128", 2, "w128">, + I32EnumAttrCase<"w", 3, "w">, + I32EnumAttrCase<"gather4", 4, "gather4"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_TmaStoreModeEnum : I32EnumAttr<"TmaStoreMode", "Modes for the TMA store instruction", [ + I32EnumAttrCase<"tiled", 0, "tiled">, + I32EnumAttrCase<"im2col", 1, "im2col">, + I32EnumAttrCase<"scatter4", 2, "scatter4"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_TmemAllocModeEnum : I32EnumAttr<"TmemAllocMode", "Modes for the Tmem allocation", [ + I32EnumAttrCase<"Interleaved", 0, "Interleaved">, + I32EnumAttrCase<"NonInterleaved", 1, "NonInterleaved">, + I32EnumAttrCase<"Duplicated", 2, "Duplicated">, + I32EnumAttrCase<"SfDuplicated_4x1", 3, "SfDuplicated_4x1">, + I32EnumAttrCase<"SfDuplicated_2x2", 4, "SfDuplicated_2x2"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_TmemLoadRedOpEnum : I32EnumAttr<"TmemLoadRedOp", "Reduce operation for the Tmem load instruction", [ + I32EnumAttrCase<"max", 0, "max">, + I32EnumAttrCase<"maxabs", 1, "maxabs">, + I32EnumAttrCase<"min", 2, "min">, + I32EnumAttrCase<"minabs", 3, "minabs"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let genSpecializedAttr = 0; +} + +def CuteNVGPU_CopyS2TBroadcastAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomCopyFieldBulkCopyG2SAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomCopyFieldBulkCopyS2GAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomCopyFieldBulkCopyS2SAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomCopyFieldDsmemStoreAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomCopyFieldLoadGlobalAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomCopyFieldNonExec2DGather4TmaLoadAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomCopyFieldNonExec2DScatter4TmaStoreAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomCopyFieldNonExecIm2ColTmaLoadAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomCopyFieldNonExecIm2ColTmaStoreAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomCopyFieldNonExecTiledTmaLoadAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomCopyFieldNonExecTiledTmaReduceAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomCopyFieldNonExecTiledTmaStoreAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomCopyFieldStoreGlobalAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomCopyFieldTmaLoadAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomCopyFieldTmaReduceAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomCopyFieldTmaStoreAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomMmaFieldSM80SparseAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomMmaFieldSM90Attr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomMmaFieldSM100Attr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomMmaFieldSM100BlockScaledAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomMmaFieldSM100BlockScaledSparseAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomMmaFieldSM100SparseAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_AtomMmaFieldSM120BlockScaledAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_BinaryOpAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_GatherScatterTmaLoadAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_Im2ColTmaLoadAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_LdReduceAccPrecisionKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_LdsmSzPatternAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_LoadCacheModeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_MMAIntOverflowAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_MmaCollectorOpAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_MmaFragKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_ReductionKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_TiledTmaLoadAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_TmaDataFormatAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_TmaLoadModeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_TmaStoreModeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_TmemAllocModeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +def CuteNVGPU_TmemLoadRedOpAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; + let assemblyFormat = "`<` $value `>`"; +} + +#endif // MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUENUMS_TD diff --git a/mlir/dialects/cute_nvgpu/CuteNVGPUOps.td b/mlir/dialects/cute_nvgpu/CuteNVGPUOps.td new file mode 100644 index 00000000..f4c09d12 --- /dev/null +++ b/mlir/dialects/cute_nvgpu/CuteNVGPUOps.td @@ -0,0 +1,2012 @@ +// The cute_nvgpu dialect's operations, generated from the nvidia-cutlass-dsl 4.8.0 release; do not edit. +// +// Sources: the operation bindings (`_cute_nvgpu_ops_gen.py`: names, classes, +// docstrings, operand/result/attribute structure, inference) and the live +// compiler (`_cutlass_ir` loaded from Python): each operand and result +// constraint carries the verifier's wording and accepts every type class the +// verifier let through when probed with one instance of every type; traits +// and interfaces are the registered operation's; memory effects come from +// loop-invariant code motion (hoisted: Pure), canonicalize (dead: no write) +// and CSE (merged: at most reads). Operations whose effects include writes +// or could not be told apart carry none (unknown effects). +// +// Operations keep the generic assembly form; the compiler's custom syntax is +// shown in each description. +// +// Constraints not recovered: +// cute_nvgpu.arch.mma.SM107.umma_block_scaled operand #3: no probe type accepted; any type allowed +// cute_nvgpu.arch.mma.SM107.umma_block_scaled_sparse operand #3: no probe type accepted; any type allowed +// cute_nvgpu.arch.mma.SM107.umma_sparse operand #3: no probe type accepted; any type allowed + +#ifndef MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUOPS_TD +#define MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUOPS_TD + +include "mlir/dialects/cute_nvgpu/CuteNVGPUAttrs.td" +include "cute_ir/Dialect/Cute/IR/CuteAttrs.td" +include "mlir/dialects/cute_ir/CuteEnums.td" + +include "mlir/Interfaces/InferTypeOpInterface.td" +include "mlir/Interfaces/SideEffectInterfaces.td" + +// Operand and result constraints, in the compiler's words. +def CuteNVGPU_AMemoryReferenceTypeThatRepresentsATensor : Type($_self))">, "A memory reference type that represents a tensor view with an iterator and layout">; +def CuteNVGPU_AllowableDescriptorTypesForS2TSrcOperand : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::SmemDescType>($_self))">, "allowable descriptor types for S2T src operand">; +def CuteNVGPU_AllowableDescriptorTypesForSM107BlockscaledUMMAA : Type($_self))">, "allowable descriptor types for SM107 Blockscaled UMMA A operand">, BuildableType<"::mlir::cutlass_compiler::cute_nvgpu::SmemDescSM107Type::get($_builder.getContext())">; +def CuteNVGPU_AllowableDescriptorTypesForSM107UMMAAOperand : Type($_self))">, "allowable descriptor types for SM107 UMMA A operand">, BuildableType<"::mlir::cutlass_compiler::cute_nvgpu::SmemDescSM107Type::get($_builder.getContext())">; +def CuteNVGPU_AllowableRegisterTypesForSM120BlockScaledMMA : Type, "allowable register types for SM120 block scaled MMA operations">; +def CuteNVGPU_AllowableRegisterTypesForSM120BlockScaledMMA2 : Type($_self))">, "allowable register types for SM120 block scaled MMA operations">; +def CuteNVGPU_AllowableRegisterTypesForSM120BlockScaledMMA3 : Type, "allowable register types for SM120 block scaled MMA operations">; +def CuteNVGPU_AtomTypeInterfaceInstance : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TiledCopyV2Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TiledMmaType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM120BlockScaledType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyG2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyS2GType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyS2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomDsmemStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomG2RType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomLdsmType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DGather4TmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DScatter4TmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomR2GType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomR2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomS2RType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100CopyS2TType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTAsyncCopyType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemLdReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemRedType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemStType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100S2TCopyV2Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomStsmType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100TmemLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM10xTmemLoadRedType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM107TmemLoadSPCompressType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100TmemStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTSyncCopyType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::UniversalFmaAtomType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMAType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMABlockScaledType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMABlockScaledSparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMASparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMAType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMABlockScaledType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMABlockScaledSparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMASparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM80Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM80SparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM89Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM90Type>($_self))">, "AtomTypeInterface instance">; +def CuteNVGPU_CopyAtomTypeInterfaceInstance : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyG2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyS2GType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyS2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomDsmemStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomG2RType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomLdsmType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DGather4TmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DScatter4TmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomR2GType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomR2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomS2RType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100CopyS2TType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTAsyncCopyType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemLdReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemRedType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemStType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomStsmType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100TmemLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM10xTmemLoadRedType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM107TmemLoadSPCompressType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100TmemStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTSyncCopyType>($_self))">, "CopyAtomTypeInterface instance">; +def CuteNVGPU_CuTeComposedLayoutType : Type($_self))">, "CuTe composed layout type">; +def CuteNVGPU_CuTeLayoutType : Type($_self))">, "CuTe Layout Type">; +def CuteNVGPU_CuTeMemrefToMemorySpacesSmem : Type($_self))">, "CuTe memref to memory spaces (smem)">; +def CuteNVGPU_CuTePointerToAllowableTypesForSM100UMMA : Type($_self))">, "CuTe pointer to allowable types for SM100 UMMA C/D operands in spaces (tmem)">; +def CuteNVGPU_CuTePointerToAllowableTypesForSM100UMMA2 : Type, "CuTe pointer to allowable types for SM100 UMMA E operand in spaces (tmem)">; // not recovered: no pool type passed +def CuteNVGPU_CuTePointerToAllowableTypesForSM107UMMA : Type, "CuTe pointer to allowable types for SM107 UMMA SF A/B operands in spaces (tmem)">; // not recovered: no pool type passed +def CuteNVGPU_CuTePointerToAllowableTypesForSM90MMA : Type($_self))">, "CuTe pointer to allowable types for SM90 MMA A/B operands in spaces (smem) with byte alignment 1">; +def CuteNVGPU_CuTePointerToGlobalMemoryWith64Byte : Type($_self))">, "CuTe pointer to global memory with 64-byte alignment">; +def CuteNVGPU_CuTePointerToGlobalMemoryWithByteAlignment : Type($_self))">, "CuTe pointer to global memory with byte alignment 1">; +def CuteNVGPU_CuTePointerToMemorySpacesRmem : Type($_self))">, "CuTe pointer to memory spaces (rmem)">; +def CuteNVGPU_CuTePointerToMemorySpacesSmem : Type($_self))">, "CuTe pointer to memory spaces (smem)">; +def CuteNVGPU_CuTePointerToMemorySpacesSmemOrCuTe : Type($_self))">, "CuTe pointer to memory spaces (smem) or CuTe pointer to memory spaces (gmem)">; +def CuteNVGPU_CuTePointerToMemorySpacesSmemWithByte : Type($_self))">, "CuTe pointer to memory spaces (smem) with byte alignment 1">; +def CuteNVGPU_CuTePointerToMemorySpacesSmemWithByte2 : Type($_self))">, "CuTe pointer to memory spaces (smem) with byte alignment 128">; +def CuteNVGPU_CuTePointerToMemorySpacesTmem : Type($_self))">, "CuTe pointer to memory spaces (tmem)">; +def CuteNVGPU_CuTePointerToSharedMemoryWith8Byte : Type($_self))">, "CuTe pointer to shared memory with 8-byte alignment">; +def CuteNVGPU_CuTePointerToSharedMemoryWithSwizzleBase : Type($_self))">, "CuTe pointer to shared memory with swizzle base atom compatible alignment with byte alignment 1">; +def CuteNVGPU_CuTePointerToSharedMemoryWithSwizzleBase2 : Type($_self))">, "CuTe pointer to shared memory with swizzle base atom compatible alignment">; +def CuteNVGPU_CuTePointerToSharedOrDistributedSharedMemory : Type($_self))">, "CuTe pointer to shared or distributed shared memory with 8-byte alignment">; +def CuteNVGPU_CuTePointerType : Type($_self))">, " CuTe pointer type ">; +def CuteNVGPU_CuteCoordTensorType : Type($_self))">, "Cute coord_tensor type">; +def CuteNVGPU_CuteCoordType : Type($_self))">, "Cute coord type">; +def CuteNVGPU_CuteShapeType : Type($_self))">, "Cute shape type">; +def CuteNVGPU_CuteSm107SmemDescType : Type($_self))">, "Cute sm107.smem_desc type">, BuildableType<"::mlir::cutlass_compiler::cute_nvgpu::SmemDescSM107Type::get($_builder.getContext())">; +def CuteNVGPU_CuteSmemDescType : Type($_self))">, "Cute smem_desc type">, BuildableType<"::mlir::cutlass_compiler::cute_nvgpu::SmemDescType::get($_builder.getContext())">; +def CuteNVGPU_CuteSmemDescViewType : Type($_self))">, "Cute smem_desc_view type">; +def CuteNVGPU_GeneralizedLayoutTypeOfCuTe : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::LayoutType>($_self))">, " Generalized layout type of CuTe ">; +def CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS : Type($_self))">, "IntTuple as integer or tuple of IntTuple(s)">; +def CuteNVGPU_Integer : Type($_self))">, "integer">; +def CuteNVGPU_SM100BlackwellSharedToTensorMemoryCopyAtom : Type($_self))">, "SM100 (Blackwell) shared-to-tensor memory copy atom type via tcgen05.cp.">; +def CuteNVGPU_SM100BlackwellSharedToTensorMemoryCopyAtom2 : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100CopyS2TType>($_self))">, "SM100 (Blackwell) shared-to-tensor memory copy atom type via tcgen05.cp. or Tiled copy atom type.">; +def CuteNVGPU_SM90PointerToTMATensorDescriptor : Type($_self))">, "SM90+ pointer to TMA tensor descriptor">; +def CuteNVGPU_SM90TMATensorMemoryAcceleratorLoadAtomType : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DGather4TmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DScatter4TmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaStoreType>($_self))">, "SM90+ TMA (Tensor Memory Accelerator) load atom type via cp.async.bulk. or SM90+ TMA im2col mode load atom type via cp.async.bulk.tensor. or SM90+ TMA (Tensor Memory Accelerator) store atom type via cp.async.bulk. or SM90+ TMA im2col mode store atom type via cp.async.bulk.tensor. or SM90+ TMA store with atomic reduction atom type via cp.reduce.async.bulk. or Type interface for non-executable TMA atom types">; +def CuteNVGPU_SignlessIntegerOrIndexOrCuTeLayoutType : Type($_self) || ::llvm::isa<::mlir::IntegerType>($_self) || ::llvm::isa<::mlir::IndexType>($_self))">, "signless integer or index or CuTe Layout Type">; +def CuteNVGPU_TheNonExecutableCopyAtomTypeForTMA : Type($_self))">, "The non-executable copy atom type for TMA loads in 2D gather4 mode">; +def CuteNVGPU_TheNonExecutableCopyAtomTypeForTMA2 : Type($_self))">, "The non-executable copy atom type for TMA stores in 2D scatter4 mode">; +def CuteNVGPU_TheNonExecutableCopyAtomTypeForTMA3 : Type($_self))">, "The non-executable copy atom type for TMA loads in im2col mode">; +def CuteNVGPU_TheNonExecutableCopyAtomTypeForTMA4 : Type($_self))">, "The non-executable copy atom type for TMA stores in im2col mode">; +def CuteNVGPU_TheNonExecutableCopyAtomTypeForTMA5 : Type($_self))">, "The non-executable copy atom type for TMA loads in tiled mode">; +def CuteNVGPU_TheNonExecutableCopyAtomTypeForTMA6 : Type($_self))">, "The non-executable copy atom type for TMA reduce in tiled mode">; +def CuteNVGPU_TheNonExecutableCopyAtomTypeForTMA7 : Type($_self))">, "The non-executable copy atom type for TMA stores in tiled mode">; +def CuteNVGPU_TheNonExecutableCopyAtomTypeForTMA8 : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaStoreType>($_self))">, "The non-executable copy atom type for TMA loads in tiled mode or The non-executable copy atom type for TMA stores in tiled mode">; +def CuteNVGPU_TiledCopyAtomType : Type($_self))">, "Tiled copy atom type.">; +def CuteNVGPU_TypeInterfaceForNonExecutableTMAAtomTypes : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DScatter4TmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaStoreType>($_self))">, "Type interface for non-executable TMA atom types">; +def CuteNVGPU_VectorOf32BitSignlessIntegerValuesOf : Type($_self))">, "vector of 32-bit signless integer values of length 1/2/4/8/16/32/64/128">; +def CuteNVGPU_VectorOf32BitSignlessIntegerValuesOf2 : Type($_self))">, "vector of 32-bit signless integer values of length 4/8">; +def CuteNVGPU_ViewTypeInterfaceInstance : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::MemRefType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::SmemDescViewType>($_self))">, "ViewTypeInterface instance">; + +// Attribute constraints the bindings name but do not describe. +def CuteNVGPU_ArchCopySM100CopyS2TBitAttr : Attr($_self) && ::llvm::is_contained(::llvm::ArrayRef{128, 256}, ::llvm::cast<::mlir::IntegerAttr>($_self).getInt())">, "arbitrary integer attribute"> { + let storageType = "::mlir::IntegerAttr"; + let returnType = "::mlir::IntegerAttr"; + let convertFromStorage = "$_self"; +} +def CuteNVGPU_ArchCopySM100CopyS2TCtaAttr : Attr($_self) && ::llvm::is_contained(::llvm::ArrayRef{1, 2}, ::llvm::cast<::mlir::IntegerAttr>($_self).getInt())">, "arbitrary integer attribute"> { + let storageType = "::mlir::IntegerAttr"; + let returnType = "::mlir::IntegerAttr"; + let convertFromStorage = "$_self"; +} +def CuteNVGPU_ArchCopySM100CopyS2TDpAttr : Attr($_self) && ::llvm::is_contained(::llvm::ArrayRef{4, 32, 64, 128}, ::llvm::cast<::mlir::IntegerAttr>($_self).getInt())">, "arbitrary integer attribute"> { + let storageType = "::mlir::IntegerAttr"; + let returnType = "::mlir::IntegerAttr"; + let convertFromStorage = "$_self"; +} +def CuteNVGPU_ArchCopySM100TMEMNumBAttr : Attr($_self) && ::llvm::is_contained(::llvm::ArrayRef{32, 64, 128, 256}, ::llvm::cast<::mlir::IntegerAttr>($_self).getInt())">, "arbitrary integer attribute"> { + let storageType = "::mlir::IntegerAttr"; + let returnType = "::mlir::IntegerAttr"; + let convertFromStorage = "$_self"; +} +def CuteNVGPU_ArchCopySM100TMEMNumDpAttr : Attr($_self) && ::llvm::is_contained(::llvm::ArrayRef{16, 32}, ::llvm::cast<::mlir::IntegerAttr>($_self).getInt())">, "arbitrary integer attribute"> { + let storageType = "::mlir::IntegerAttr"; + let returnType = "::mlir::IntegerAttr"; + let convertFromStorage = "$_self"; +} +def CuteNVGPU_ArchCopySM100TMEMNumRepAttr : Attr($_self) && ::llvm::is_contained(::llvm::ArrayRef{1, 2, 4, 8, 16, 32, 64, 128}, ::llvm::cast<::mlir::IntegerAttr>($_self).getInt())">, "arbitrary integer attribute"> { + let storageType = "::mlir::IntegerAttr"; + let returnType = "::mlir::IntegerAttr"; + let convertFromStorage = "$_self"; +} +def CuteNVGPU_ArchCopySM100TmaLoadOpNumCTAAttr : Attr($_self) && ::llvm::is_contained(::llvm::ArrayRef{1, 2}, ::llvm::cast<::mlir::IntegerAttr>($_self).getInt())">, "arbitrary integer attribute"> { + let storageType = "::mlir::IntegerAttr"; + let returnType = "::mlir::IntegerAttr"; + let convertFromStorage = "$_self"; +} +def CuteNVGPU_SM120BlockScaledABTypeAttr : Attr($_self) && (::llvm::isa<::mlir::Float8E4M3FNType>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()) || ::llvm::isa<::mlir::Float8E5M2Type>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()) || ::llvm::isa<::mlir::Float4E2M1FNType>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()) || ::llvm::isa<::mlir::Float6E2M3FNType>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()) || ::llvm::isa<::mlir::Float6E3M2FNType>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()))">, "type attribute of allowable types for SM120 block scaled MMA A/B operands"> { + let storageType = "::mlir::TypeAttr"; + let returnType = "::mlir::TypeAttr"; + let convertFromStorage = "$_self"; +} +def CuteNVGPU_SM120BlockScaledSFTypeAttr : Attr($_self) && (::llvm::isa<::mlir::Float8E4M3FNType>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()) || ::llvm::isa<::mlir::Float8E8M0FNUType>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()))">, "type attribute of allowable types for SM120 block scaled MMA A/B scale factors"> { + let storageType = "::mlir::TypeAttr"; + let returnType = "::mlir::TypeAttr"; + let convertFromStorage = "$_self"; +} +def CuteNVGPU_TmaLoadOrStoreModeAttr : Attr($_self)">, "TMA load or store mode attribute"> { + let storageType = "::mlir::Attribute"; + let returnType = "::mlir::Attribute"; + let convertFromStorage = "$_self"; +} + +// Operations whose result types are inferred in C++. Verification compares +// them with the inferred ones where the rule is recovered for the operands; +// elsewhere the types written stand, as the compiler accepted them. +class CuteNVGPU_InferredOp traits = []> + : CuteNVGPU_Op]> { + let extraClassDefinition = [{ + ::llvm::LogicalResult $cppClass::refineReturnTypes( + ::mlir::MLIRContext *context, ::std::optional<::mlir::Location> location, + ::mlir::ValueRange operands, ::mlir::DictionaryAttr attributes, + ::mlir::PropertyRef properties, ::mlir::RegionRange regions, + ::llvm::SmallVectorImpl<::mlir::Type> &returnTypes) { + ::llvm::SmallVector<::mlir::Type, 4> inferred; + if (::mlir::failed(inferReturnTypes(context, ::std::nullopt, operands, + attributes, properties, regions, + inferred))) + return ::mlir::success(); + if (!isCompatibleReturnTypes(inferred, returnTypes)) + return ::mlir::emitOptionalError( + location, "'", getOperationName(), "' op inferred type(s) ", + inferred, " are incompatible with return type(s) of operation ", + returnTypes); + return ::mlir::success(); + } + }]; +} + +def CuteNVGPU_ArchAllocRmemOp : CuteNVGPU_Op<"arch.alloc_rmem", [MemoryEffects<[MemAlloc]>]> { + let description = [{ + This operation allocate a per-thread private buffer on the stack, with its size + determined by either the provided element count or the static layout. + + Examples: + + ```mlir + %c32_i32 = arith.constant 32 : i32 + %0 = cute_nvgpu.arch.alloc_rmem(%c32_i32) : i32 -> !cute.ptr + + // %static_layout : !cute.layout<"(8,4):(1,8)"> + %1 = cute_nvgpu.arch.alloc_rmem(%static_layout) : !cute.layout<"(8,4):(1,8)"> -> !cute.ptr + + // %dynamic_layout : !cute.layout<"(?,4):(1,?)"> + %2 = cute_nvgpu.arch.alloc_rmem(%dynamic_layout) : !cute.layout<"(?,4):(1,?)"> -> !cute.ptr + ``` + }]; + let arguments = (ins + CuteNVGPU_SignlessIntegerOrIndexOrCuTeLayoutType:$input + ); + let results = (outs + CuteNVGPU_CuTePointerToMemorySpacesRmem:$ptr + ); +} +def CuteNVGPU_ArchAllocSmemOp : CuteNVGPU_Op<"arch.alloc_smem", [MemoryEffects<[MemAlloc]>]> { + let description = [{ + This operation allocate static shared memory based on either a specified + element count or a static layout. + + Examples: + + ```mlir + // size provided by element count + %0 = cute_nvgpu.arch.alloc_smem(1024) : !cute.ptr> + + // size provided by static layout + %1 = cute_nvgpu.arch.alloc_smem("(8,4):(1,8)") : !cute.ptr> + ``` + }]; + let arguments = (ins + I32Attr:$input + ); + let results = (outs + CuteNVGPU_CuTePointerToMemorySpacesSmem:$ptr + ); +} +def CuteNVGPU_ArchCopySM90BulkCopyG2SOp : CuteNVGPU_Op<"arch.copy.SM90.bulk_copy_g2s", [AttrSizedOperandSegments]> { + let description = [{ + The `arch.copy.SM90.bulk_copy_g2s` operation performs a bulk load from global + to shared memory. + + The `gmem_data_addr` operand is a 16-byte aligned pointer to the source data + in global memory. + + The `dsmem_data_addr` operand is a 16-byte aligned pointer to the destination data + in shared cluster memory. + + The `dsmem_bar_addr` operand is a 8-byte aligned pointer to the barrier in shared + memory. + + The `size` operand specifies the bytes copied by BLKCP, aligned to 16 bytes. + + The `multicast_mask` operand is used when multicasting. + + The `cache_policy` operand is used to indicate the eviction priority of the L2 cache. + + Examples: + ```mlir + %gmem = ... : !cute.ptr> + %smem = ... : !cute.ptr> + %bar = ... : !cute.ptr> + + // Without multicast mask + %0 = cute_nvgpu.arch.copy.SM90.bulk_copy_g2s( + %gmem : !cute.ptr>, + %smem : !cute.ptr>, + %bar : !cute.ptr> + ) size = 128 + + // With multicast mask + %1 = cute_nvgpu.arch.copy.SM90.bulk_copy_g2s( + %gmem : !cute.ptr>, + %smem : !cute.ptr>, + %bar : !cute.ptr> + ) size = 128 mask = %mask + }]; + let arguments = (ins + CuteNVGPU_CuTePointerToGlobalMemoryWithByteAlignment:$gmem_data_addr, + CuteNVGPU_CuTePointerToSharedMemoryWithSwizzleBase:$dsmem_data_addr, + CuteNVGPU_CuTePointerToSharedMemoryWith8Byte:$dsmem_bar_addr, + Optional:$multicast_mask, + Optional:$cache_policy, + I32Attr:$size + ); + let results = (outs + ); +} +def CuteNVGPU_ArchCopySM90BulkCopyS2GOp : CuteNVGPU_Op<"arch.copy.SM90.bulk_copy_s2g", [AttrSizedOperandSegments]> { + let description = [{ + The `arch.copy.SM90.bulk_copy_s2g` operation performs a bulk load from shared + to global memory. + + The `smem_data_addr` operand is a 16-byte aligned pointer to the sourced data + in shared memory. + + The `gmem_data_addr` operand is a 16-byte aligned pointer to the estination data + in global memory. + + The `size` operand specifies the bytes copied by BLKCP, aligned to 16 bytes. + + The `byte_mask` operand is specifies whether the i-th byte of each 16-byte wide + chunk of source data is copied to the destination. + + The `cache_policy` operand is used to indicate the eviction priority of the L2 cache. + + Examples: + ```mlir + %smem = ... : !cute.ptr> + %gmem = ... : !cute.ptr> + + // Without byte mask + %0 = cute_nvgpu.arch.copy.SM90.bulk_copy_s2g( + %smem : !cute.ptr>, + %gmem : !cute.ptr>, + ) size = 128 + + // With byte mask + %1 = cute_nvgpu.arch.copy.SM90.bulk_copy_s2g( + %smem : !cute.ptr>, + %gmem : !cute.ptr>, + ) size = 128 byte_mask = %mask + }]; + let arguments = (ins + CuteNVGPU_CuTePointerToSharedMemoryWithSwizzleBase:$smem_data_addr, + CuteNVGPU_CuTePointerToGlobalMemoryWithByteAlignment:$gmem_data_addr, + Optional:$byte_mask, + Optional:$cache_policy, + I32Attr:$size + ); + let results = (outs + ); +} +def CuteNVGPU_ArchCopySM90BulkCopyS2SOp : CuteNVGPU_Op<"arch.copy.SM90.bulk_copy_s2s"> { + let description = [{ + The `arch.copy.SM90.bulk_copy_s2s` operation performs a bulk load from shared + cta to shared cluster. + + The `smem_data_addr` operand is a 16-byte aligned pointer to the source data + in shared memory. + + The `dsmem_data_addr` operand is a 16-byte aligned pointer to the destination data + in shared cluster memory. + + The `dsmem_bar_addr` operand is a 8-byte aligned pointer to the barrier in shared + memory. + + The `cta_rank` operand specifies the dst cta rank in cluster for BLKCP. + + The `size` attr specifies the bytes copied by BLKCP, aligned to 16 bytes. + + Examples: + ```mlir + %smem = ... : !cute.ptr> + %dsmem = ... : !cute.ptr> + %bar = ... : !cute.ptr> + %cta = ... : i32 + + %0 = cute_nvgpu.arch.copy.SM90.bulk_copy_s2s( + %smem : !cute.ptr>, + %dsmem : !cute.ptr>, + %bar : !cute.ptr>, + %cta : i32 + ) size = 128 + }]; + let arguments = (ins + CuteNVGPU_CuTePointerToSharedMemoryWithSwizzleBase:$smem_data_addr, + CuteNVGPU_CuTePointerToSharedMemoryWithSwizzleBase:$dsmem_data_addr, + CuteNVGPU_CuTePointerToSharedMemoryWith8Byte:$dsmem_bar_addr, + I32:$cta_rank, + I32Attr:$size + ); + let results = (outs + ); +} +def CuteNVGPU_ArchCopySM100CopyS2TOp : CuteNVGPU_Op<"arch.copy.SM100.copy_s2t"> { + let description = [{ + See the file cutlass/include/cute/arch/copy_sm100.hpp in the CUTLASS + repository for more information. + }]; + let arguments = (ins + CuteNVGPU_AllowableDescriptorTypesForS2TSrcOperand:$src, + CuteNVGPU_CuTePointerToMemorySpacesTmem:$dst, + CuteNVGPU_ArchCopySM100CopyS2TDpAttr:$dp, + CuteNVGPU_ArchCopySM100CopyS2TBitAttr:$bits, + CuteNVGPU_ArchCopySM100CopyS2TCtaAttr:$cta, + CuteNVGPU_CopyS2TBroadcastAttr:$broadcast + ); + let results = (outs + ); +} +def CuteNVGPU_ArchCopySM100TmaLoadOp : CuteNVGPU_Op<"arch.copy.SM100.tma_load", [AttrSizedOperandSegments]> { + let description = [{ + The `arch.copy.SM100.tma_load` operation performs a tensor load from global + to shared memory. + + The attribute `mode` indicates which mode is selected. It can be: + + - tiled + - im2col for image-to-column loads + - w128 for activation-stationary convolution MMA mode (requires multicast) + - w for weight-stationary convolution MMA mode (requires multicast) + - gather4 for gather mode + + The attribute `num_cta` specifies the scope of the instruction. + + The `src_desc` operand is a 64-byte aligned pointer to the source tensor in + global memory. + + The `dsmem_data_addr` is a 128-byte aligned pointer to the destination data + address in shared memory. + + The `dsmem_bar_addr` is an 8-byte aligned pointer to the barrier in shared + or distributed shared memory. + + The `coord` operand holds the tensor coordinates (up to 5 dimensions). + + The `multicast_mask` operand is used when multicasting. + + The `offsets` operand holds offsets for im2col, w128, or w modes. + + The `cache_policy` operand is used to indicate the eviction priority of the L2 cache. + }]; + let arguments = (ins + CuteNVGPU_CuTePointerToGlobalMemoryWith64Byte:$src_desc, + CuteNVGPU_CuTePointerToSharedMemoryWithSwizzleBase:$dsmem_data_addr, + CuteNVGPU_CuTePointerToSharedOrDistributedSharedMemory:$dsmem_bar_addr, + Variadic:$coord, + Optional:$multicast_mask, + Variadic:$offsets, + Optional:$cache_policy, + CuteNVGPU_TmaLoadModeAttr:$mode, + CuteNVGPU_ArchCopySM100TmaLoadOpNumCTAAttr:$num_cta + ); + let results = (outs + ); +} +def CuteNVGPU_ArchCopySM100TmaReduceOp : CuteNVGPU_Op<"arch.copy.SM100.tma_reduce", [AttrSizedOperandSegments]> { + let description = [{ + The `arch.copy.SM100.tma_reduce` operation performs a tensor store from shared memory + to global with reduction. + + The attribute `mode` indicates which mode is selected. It can be: + + - tiled + - im2col for image-to-column loads + + The attribute `Kind` indicates which kind of reduce op is selected. It can be: + + - ADD + - MIN + - MAX + - INC + - DEC + - AND + - OR + - XOR + + The `dst_desc` is a 64-byte aligned pointer to the destination data + address in global memory. + + The `src_smem_data_addr` operand is a 128-byte aligned pointer to the source address in + shared memory. + + The `coord` operand holds the tensor coordinates (up to 5 dimensions). + + The `cache_policy` operand is used to indicate the eviction priority of the L2 cache. + }]; + let arguments = (ins + CuteNVGPU_CuTePointerToGlobalMemoryWith64Byte:$dst_desc, + CuteNVGPU_CuTePointerToSharedMemoryWithSwizzleBase2:$src_smem_data_addr, + Variadic:$coord, + Optional:$cache_policy, + CuteNVGPU_TmaStoreModeAttr:$mode, + CuteNVGPU_ReductionKindAttr:$kind + ); + let results = (outs + ); +} +def CuteNVGPU_ArchCopySM100TmaStoreOp : CuteNVGPU_Op<"arch.copy.SM100.tma_store", [AttrSizedOperandSegments]> { + let description = [{ + The `arch.copy.SM100.tma_store` operation performs a tensor store from shared memory + to global. + + The attribute `mode` indicates which mode is selected. It can be: + + - tiled + - im2col for image-to-column loads + - scatter4 for scatter mode + + The `dst_desc` is a 64-byte aligned pointer to the destination data + address in global memory. + + The `src_smem_data_addr` operand is a 128-byte aligned pointer to the source address in + shared memory. + + The `coord` operand holds the tensor coordinates (up to 5 dimensions). + + The `cache_policy` operand is used to indicate the eviction priority of the L2 cache. + }]; + let arguments = (ins + CuteNVGPU_CuTePointerToGlobalMemoryWith64Byte:$dst_desc, + CuteNVGPU_CuTePointerToSharedMemoryWithSwizzleBase2:$src_smem_data_addr, + Variadic:$coord, + Optional:$cache_policy, + CuteNVGPU_TmaStoreModeAttr:$mode + ); + let results = (outs + ); +} +def CuteNVGPU_ArchCopySM100TmemLoadOp : CuteNVGPU_Op<"arch.copy.SM100.tmem_load", [MemoryEffects<[MemRead]>]> { + let description = [{ + See the file cutlass/include/cute/arch/copy_sm100.hpp in the CUTLASS + repository for more information. + }]; + let arguments = (ins + CuteNVGPU_CuTePointerToMemorySpacesTmem:$src, + CuteNVGPU_ArchCopySM100TMEMNumDpAttr:$num_dp, + CuteNVGPU_ArchCopySM100TMEMNumBAttr:$num_b, + CuteNVGPU_ArchCopySM100TMEMNumRepAttr:$num_rep, + UnitAttr:$pack_16 + ); + let results = (outs + CuteNVGPU_VectorOf32BitSignlessIntegerValuesOf:$dst + ); +} +def CuteNVGPU_ArchCopySM100TmemStoreOp : CuteNVGPU_Op<"arch.copy.SM100.tmem_store"> { + let description = [{ + See the file cutlass/include/cute/arch/copy_sm100.hpp in the CUTLASS + repository for more information. + }]; + let arguments = (ins + CuteNVGPU_CuTePointerToMemorySpacesTmem:$dst, + CuteNVGPU_VectorOf32BitSignlessIntegerValuesOf:$src, + CuteNVGPU_ArchCopySM100TMEMNumDpAttr:$num_dp, + CuteNVGPU_ArchCopySM100TMEMNumBAttr:$num_b, + CuteNVGPU_ArchCopySM100TMEMNumRepAttr:$num_rep, + UnitAttr:$expand_16 + ); + let results = (outs + ); +} +def CuteNVGPU_ArchCopySM107TmaLoadOverrideOp : CuteNVGPU_Op<"arch.copy.SM107.tma_load_override", [AttrSizedOperandSegments]> { + let description = [{ + The `arch.copy.SM107.tma_load_override` operation performs a TMA tensor load + from global to shared memory with runtime override of address, dimensions, + and strides. + + In addition to the normal TMA load operands, this op carries: + - `override_addr`: global pointer overriding the descriptor's base address + - `tensor_size`: per-mode sizes (encoded as size-1, i16) + - `lower_stride`: per-mode strides in 16B units (i32, modes 1..N-1) + - `upper_stride`: packed upper 4 bits of up to 4 strides (i16) + + Coordinates are passed as zeros (override form ignores them). + + Example: + + Suppose we want to load a 2D tile from a tensor with logical sizes + `{S0, S1}` and per-mode element strides `{1, ES1}` (mode 0 is contiguous, + so its stride is not passed). Let `EBITS` be the value type's bit-width + and `elemsPer16B = 128 / EBITS`. Mode 0's size is in `tensor_size[0]`, + mode 1's size is in `tensor_size[1]`, and mode 1's stride is split between + `lower_stride[0]` and the nibble at bits `[0:4]` of `upper_stride`. + + The encoding (matching `CopyAtomTmaLoadType::lowerOverrideToArch`) is: + + ```mlir + // ---- tensor_size: trunc(S_i - 1), one entry per mode ---- + %s0_m1 = arith.subi %S0_i32, %c1_i32 : i32 // S0 - 1 + %s1_m1 = arith.subi %S1_i32, %c1_i32 : i32 // S1 - 1 + %ts0 = arith.trunci %s0_m1 : i32 to i16 + %ts1 = arith.trunci %s1_m1 : i32 to i16 + + // ---- stride in 16B units: ES_i (element stride, i64) >> log2(elemsPer16B) + %sh = arith.constant log2(elemsPer16B) : i64 + %s1_16b = arith.shrui %ES1_i64, %sh : i64 // mode 1, stride in 16B + + // ---- lower_stride[idx]: low 32 bits of stride_in_16B (idx = mode - 1) ---- + %ls1 = arith.trunci %s1_16b : i64 to i32 // idx = 0 (mode 1) + + // ---- upper_stride: packed nibbles, OR-accumulated, truncated to i16 ---- + // nibble_i = (stride_in_16B[mode i] >> 32) & 0xF + // upper |= nibble_i << (4 * (i - 1)) + %c32 = arith.constant 32 : i64 + %c0xF = arith.constant 15 : i64 + %hi_1 = arith.shrui %s1_16b, %c32 : i64 // upper bits of mode 1 + %nib_1 = arith.andi %hi_1, %c0xF : i64 + %sh_1 = arith.constant 0 : i64 // 4 * (1 - 1) = 0 + %shf_1 = arith.shli %nib_1, %sh_1 : i64 + %acc = arith.ori %c0_i64, %shf_1 : i64 // accumulate over modes + %us = arith.trunci %acc : i64 to i16 + + // ---- the override TMA load itself ---- + cute_nvgpu.arch.copy.SM107.tma_load_override( + %tma_desc : !cute.ptr>, // src TMA descriptor + %dst_smem : !cute.ptr>, // dst data buffer + %dst_bar : !cute.ptr>, // mbarrier + %override_addr : !cute.ptr>, // runtime base addr override + [%c0_i32, %c0_i32] : i32, i32) // coords (ignored: zeros) + tensor_size = %ts0, %ts1 // (size - 1) per mode, i16 + lower_stride = %ls1 // strides for modes 1..N-1, in 16B units, low 32b (i32) + upper_stride = %us // packed upper 4 bits of up to 4 strides (i16) + cache_policy = %policy // L2 eviction policy (i64) + mode = num_cta = 1 + ``` + }]; + let arguments = (ins + CuteNVGPU_CuTePointerToGlobalMemoryWith64Byte:$src_desc, + CuteNVGPU_CuTePointerToSharedMemoryWithSwizzleBase:$dsmem_data_addr, + CuteNVGPU_CuTePointerToSharedOrDistributedSharedMemory:$dsmem_bar_addr, + CuteNVGPU_CuTePointerType:$override_addr, + Variadic:$tensor_size, + Variadic:$lower_stride, + Optional:$upper_stride, + Variadic:$coord, + Optional:$multicast_mask, + Optional:$cache_policy, + CuteNVGPU_TmaLoadModeAttr:$mode, + CuteNVGPU_ArchCopySM100TmaLoadOpNumCTAAttr:$num_cta + ); + let results = (outs + ); +} +def CuteNVGPU_ArchCopySM107TmaStoreOverrideOp : CuteNVGPU_Op<"arch.copy.SM107.tma_store_override", [AttrSizedOperandSegments]> { + let description = [{ + The `arch.copy.SM107.tma_store_override` operation performs a TMA tensor store + from shared memory to global with runtime override of address, dimensions, + and strides. + + In addition to the normal TMA store operands, this op carries: + - `override_addr`: global pointer overriding the descriptor's base address + - `tensor_size`: per-mode sizes (encoded as size-1, i16) + - `lower_stride`: per-mode strides in 16B units (i32, modes 1..N-1) + - `upper_stride`: packed upper 4 bits of up to 4 strides (i16) + + Example + The example of tma_load_override applies here except that tma_store_override + doesn't have the mbar argument. + + Coordinates are passed as zeros (override form ignores them). + }]; + let arguments = (ins + CuteNVGPU_CuTePointerToGlobalMemoryWith64Byte:$dst_desc, + CuteNVGPU_CuTePointerToSharedMemoryWithSwizzleBase2:$src_smem_data_addr, + CuteNVGPU_CuTePointerType:$override_addr, + Variadic:$tensor_size, + Variadic:$lower_stride, + Optional:$upper_stride, + Variadic:$coord, + Optional:$cache_policy, + CuteNVGPU_TmaStoreModeAttr:$mode + ); + let results = (outs + ); +} +def CuteNVGPU_ArchGetDynamicSmemOp : CuteNVGPU_Op<"arch.get_dyn_smem", [MemoryEffects<[MemAlloc]>]> { + let description = [{ + This operation acquires the pointer to the dynamic shared memory, whose size + is determined at the kernel launch time. + }]; + let arguments = (ins + ); + let results = (outs + CuteNVGPU_CuTePointerToMemorySpacesSmem:$ptr + ); +} +def CuteNVGPU_ArchGetDynamicSmemSizeOp : CuteNVGPU_Op<"arch.get_dyn_smem_size", [Pure]> { + let description = [{ + This operation gets the size in bytes of the dynamic shared memory that was + specified at kernel launch time. This can be used for bounds checking during + shared memory allocation. + }]; + let arguments = (ins + ); + let results = (outs + I32:$size + ); +} +def CuteNVGPU_ArchMakeWarpUniformOp : CuteNVGPU_Op<"arch.make_warp_uniform", [Pure]> { + let description = [{ + This operation is to reach a similar goal as canonical_warp_idx_sync() in cutlass C++, which is + typically used to hint to ptxas to use uniform register to store current value so as to avoid + redundant R2URs in mainloop. It's actually a shortcut usage of `nvvm.shfl.sync idx 31, val, 0, -1`. + + Example: + + ```mlir + %thread_id = nvvm.read.ptx.sreg.tid.x : i32 + %warp_size_log2 = arith.constant 5 : i32 + %warp_id = arith.shrsi %thread_id, %warp_size_log2 : i32 + %warp_id_uniform = cute_nvgpu.arch.make_warp_uniform %warp_id : i32 + ``` + And this IR will be lowered to: + ```mlir + %thread_id = nvvm.read.ptx.sreg.tid.x : i32 + %warp_size_log2 = arith.constant 5 : i32 + %warp_id = arith.shrsi %thread_id, %warp_size_log2 : i32 + %warp_full_mask = arith.constant -1 : i32 + %c0_i32 = arith.constant 0 : i32 + %warp_id_uniform = nvvm.shfl.sync idx %c0_i32, %warp_id, %c0_i32, %warp_full_mask : i32 -> i32 + ``` + }]; + let arguments = (ins + I32:$val + ); + let results = (outs + I32:$res + ); +} +def CuteNVGPU_ArchMmaSM107UMMABlockScaledOp : CuteNVGPU_Op<"arch.mma.SM107.umma_block_scaled"> { + let description = [{ + The `cute_nvgpu.arch.mma.SM107.umma_block_scaled` operation performs a matrix + multiply-accumulate operation. This arch op contains scaling factor for A/B. + Matrix elements for operand A can reside in tensor memory or shared memory. + Operand B always reside in shared memory. Elements for the accumulator input + and output (CD) always reside in tensor memory. The `accum` operand controls + whether the matrix multiplication result is added to data in `C` (`accum` = `true`) + or overwrites the data in `C` (`accum` = `false`). The `sf_a/b` operands indicate + the scaling factor of A/B and they always reside in tensor memory. The `vec_size` + attribute indicates the scaling vector size. + See the file cutlass/include/cute/arch/mma_sm100_umma.hpp in the CUTLASS + repository for more information. + Examples: + ```mlir + cute_nvgpu.arch.mma.SM107.umma_block_scaled(%A, %B, %C, %sf_a, %sf_b, %idesc, %accum) + { num_cta = 1, vec_size = 32, a_type = f8E5M2, shape_k = 32 } + : (!cute_nvgpu.smem_desc, !cute_nvgpu.smem_desc, !cute.ptr, !cute.ptr, + !cute.ptr, i32, i1) + ``` + }]; + let arguments = (ins + CuteNVGPU_AllowableDescriptorTypesForSM107BlockscaledUMMAA:$a, + CuteNVGPU_CuteSm107SmemDescType:$b, + CuteNVGPU_CuTePointerToAllowableTypesForSM100UMMA:$cd, + CuteNVGPU_CuTePointerToAllowableTypesForSM107UMMA:$sf_a, + AnyType:$sf_b, + AnyType:$idesc, + AnyType:$accum, + APIntAttr:$num_cta, + I32Attr:$vec_size, + TypeAttr:$a_type, + TypeAttr:$b_type, + I32Attr:$shape_k, + CuteNVGPU_MmaCollectorOpAttr:$a_collector_op, + CuteNVGPU_MmaCollectorOpAttr:$b_collector_op + ); + let results = (outs + ); +} +def CuteNVGPU_ArchMmaSM107UMMABlockScaledSparseOp : CuteNVGPU_Op<"arch.mma.SM107.umma_block_scaled_sparse"> { + let description = [{ + The `cute_nvgpu.arch.mma.SM107.umma_block_scaled_sparse` operation performs a matrix + multiply-accumulate operation. This arch op contains scaling factor for A/B and supports + structured sparsity. Matrix elements for operand A can reside in tensor memory or shared memory. + Operand B always reside in shared memory. Elements for the accumulator input + and output (CD) always reside in tensor memory. The `accum` operand controls + whether the matrix multiplication result is added to data in `C` (`accum` = `true`) + or overwrites the data in `C` (`accum` = `false`). The `sf_a/b` operands indicate + the scaling factor of A/B and they always reside in tensor memory. The `e_op` operand + provides the sparse metadata that describes the sparsity positional information of the matrix, + enabling efficient computation with structured sparse matrices. The `vec_size` + attribute indicates the scaling vector size. + See the file cutlass/include/cute/arch/mma_sm100_umma.hpp in the CUTLASS + repository for more information. + Examples: + ```mlir + cute_nvgpu.arch.mma.SM107.umma_block_scaled_sparse(%A, %B, %C, %sf_a, %sf_b, %E, %idesc, %accum) + { num_cta = 1, vec_size = 32, a_type = f8E5M2, shape_k = 32 } + : (!cute_nvgpu.sm107.smem_desc, !cute_nvgpu.sm107.smem_desc, !cute.ptr, !cute.ptr, + !cute.ptr, !cute.ptr>, i32, i1) + ``` + }]; + let arguments = (ins + CuteNVGPU_AllowableDescriptorTypesForSM107BlockscaledUMMAA:$a, + CuteNVGPU_CuteSm107SmemDescType:$b, + CuteNVGPU_CuTePointerToAllowableTypesForSM100UMMA:$cd, + CuteNVGPU_CuTePointerToAllowableTypesForSM107UMMA:$sf_a, + AnyType:$sf_b, + AnyType:$e_op, + AnyType:$idesc, + AnyType:$accum, + APIntAttr:$num_cta, + I32Attr:$vec_size, + TypeAttr:$a_type, + TypeAttr:$b_type, + I32Attr:$shape_k, + CuteNVGPU_MmaCollectorOpAttr:$a_collector_op, + CuteNVGPU_MmaCollectorOpAttr:$b_collector_op + ); + let results = (outs + ); +} +def CuteNVGPU_ArchMmaSM107UMMAOp : CuteNVGPU_Op<"arch.mma.SM107.umma"> { + let description = [{ + The `cute_nvgpu.arch.mma.SM107.umma` operation performs a matrix multiply- + accumulate operation. + Matrix elements for operand A can reside in either shared memory or tensor + memory (tmem). As such, the argument for operand A can be either a shared + memory descriptor or a pointer to tensor memory. + Elements for operand B always reside in shared memory. Elements for the + accumulator input and output (CD) always reside in tensor memory. The + `accum` operand controls whether the matrix multiplication result is added + to data in `C` (`accum` = `true`) or overwrites the data in `C` (`accum` = + `false`). Needs instruction desc according to nvvm backend and an additional + `a_type` for mmaKind inference. + See the file cutlass/include/cute/arch/mma_sm100_umma.hpp in the CUTLASS + repository for more information. + Examples: + ```mlir + cute_nvgpu.arch.mma.SM107.umma(%A, %B, %C, %idesc, %accum) + { num_cta = 1, ab_type = (f8E5M2, f8E5M2), shape_k = 64, ab_collector_op = (discard, discard) } + : (!cute_nvgpu.smem_desc, !cute_nvgpu.smem_desc, !cute.ptr, i32, i1) -> () + ``` + }]; + let arguments = (ins + CuteNVGPU_AllowableDescriptorTypesForSM107UMMAAOperand:$a, + CuteNVGPU_CuteSm107SmemDescType:$b, + CuteNVGPU_CuTePointerToAllowableTypesForSM100UMMA:$cd, + I32:$idesc, + I1:$accum, + CuteNVGPU_VectorOf32BitSignlessIntegerValuesOf2:$disable_output_lane, + APIntAttr:$num_cta, + APIntAttr:$scale, + TypeAttr:$a_type, + TypeAttr:$b_type, + I32Attr:$shape_k, + CuteNVGPU_MmaCollectorOpAttr:$a_collector_op, + CuteNVGPU_MmaCollectorOpAttr:$b_collector_op + ); + let results = (outs + ); +} +def CuteNVGPU_ArchMmaSM107UMMASparseOp : CuteNVGPU_Op<"arch.mma.SM107.umma_sparse"> { + let description = [{ + The `cute_nvgpu.arch.mma.SM107.umma_sparse` operation performs a matrix + multiply-accumulate operation where matrix A is structured sparse. + Matrix elements for operand A can reside in either shared memory or tensor + memory (tmem). As such, the argument for operand A can be either a shared + memory descriptor or a pointer to tensor memory. Operand E holds sparse + metadata that reside in tensor memory. + Elements for operand B always reside in shared memory. Elements for the + accumulator input and output (CD) always reside in tensor memory. The + `accum` operand controls whether the matrix multiplication result is added + to data in `C` (`accum` = `true`) or overwrites the data in `C` (`accum` = + `false`). Needs instruction desc according to nvvm backend and an additional + `a_type` for mmaKind inference. + See the file cutlass/include/cute/arch/mma_sm100_umma.hpp in the CUTLASS + repository for more information. + Examples: + ```mlir + cute_nvgpu.arch.mma.SM107.umma_sparse(%A, %B, %C, %E, %idesc, %accum) + {num_cta = 1 : i32, scale = 0 : i32, ab_type = (f16, f16), shape_k = 32 : i32, ab_collector_op = (discard, discard)} + : (!cute_nvgpu.sm107.smem_desc, !cute_nvgpu.sm107.smem_desc, !cute.ptr, + !cute.ptr, i32, i1) -> () + ``` + }]; + let arguments = (ins + CuteNVGPU_AllowableDescriptorTypesForSM107UMMAAOperand:$a, + CuteNVGPU_CuteSm107SmemDescType:$b, + CuteNVGPU_CuTePointerToAllowableTypesForSM100UMMA:$cd, + CuteNVGPU_CuTePointerToAllowableTypesForSM100UMMA2:$e, + AnyType:$idesc, + AnyType:$accum, + APIntAttr:$num_cta, + APIntAttr:$scale, + TypeAttr:$a_type, + TypeAttr:$b_type, + I32Attr:$shape_k, + CuteNVGPU_MmaCollectorOpAttr:$a_collector_op, + CuteNVGPU_MmaCollectorOpAttr:$b_collector_op + ); + let results = (outs + ); +} +def CuteNVGPU_ArchMmaSM120BlockScaledOp : CuteNVGPU_Op<"arch.mma.SM120.block_scaled", [AttrSizedOperandSegments]> { + let description = [{ + The `cute_nvgpu.arch.mma.SM120.block_scaled` operation performs a matrix + multiply-accumulate operation. This arch op contains scaling factor for A/B. + Matrix elements for A/B/C/D/SFA/SFB and accumulator are always in registers. + + C is always F32 under current SM120 arch, so there is no `c_type` here. + + See the file cutlass/include/cute/arch/mma_sm120.hpp in the CUTLASS repository + for more information. + + Examples: + ```mlir + cute_nvgpu.arch.mma.SM120.block_scaled + A[%a0, %a1, %a2, %a3] B[%b0, %b1] C[%c0, %c1, %c2, %c3] + sfA(%scaleAData) sfB(%scaleBData) + {shape_MNK=<"(16,8,32)">, vec_size = 32, a_type = f4E2M1FN, b_type = f4E2M1FN, sf_type = f8E8M0FNU} + : A[i32, i32, i32, i32] B[i32, i32] C[f32, f32, f32, f32] + sfA(i8) sfB(i8) -> (f32, f32, f32, f32) + ``` + }]; + let arguments = (ins + Variadic:$opA, + Variadic:$opB, + Variadic:$opC, + CuteNVGPU_AllowableRegisterTypesForSM120BlockScaledMMA3:$sfA, + CuteNVGPU_AllowableRegisterTypesForSM120BlockScaledMMA3:$sfB, + Optional:$byte_id_a, + Optional:$byte_id_b, + Cute_ShapeAttr:$shape_MNK, + I32Attr:$vec_size, + DefaultValuedAttr:$thread_id_a, + DefaultValuedAttr:$thread_id_b, + CuteNVGPU_SM120BlockScaledABTypeAttr:$a_type, + CuteNVGPU_SM120BlockScaledABTypeAttr:$b_type, + CuteNVGPU_SM120BlockScaledSFTypeAttr:$sf_type + ); + let results = (outs + Variadic:$res + ); +} +def CuteNVGPU_ArchPrefetchTmaDescOp : CuteNVGPU_Op<"arch.prefetch_tma_desc"> { + let description = [{ + The `arch.prefetch_tma_desc` operation does prefetch TMA descriptor using generic addressing + (i.e. no specific state space: const or param). + + Example: + + ```mlir + cute_nvgpu.arch.prefetch_tma_desc(%tma_a) : (!cute.ptr>) -> () + ``` + }]; + let arguments = (ins + CuteNVGPU_SM90PointerToTMATensorDescriptor:$tma_descriptor_ptr + ); + let results = (outs + ); +} +def CuteNVGPU_ArchSM100AllocTmemOp : CuteNVGPU_Op<"arch.sm100.alloc_tmem"> { + let summary = "The `arch.sm100.alloc_tmem` operation performs dynamic allocation of TMEM."; + let description = [{ + It dynamically allocates storage on TMEM with requested `n_cols` and stores + the allocated TMEM pointer into the shared memory buffer at the shared memory + location specified by `dst_ptr`. + + This operation is blocking, meaning it will wait until enough memory is available. + The action writting pointer value into shared memory is not synchronized. It's + user's responsibility to synchronize to make sure it's visible to all threads in the CTA. + + The allocation may fail if one of following conditions is true: + 1. The requested allocation size is not a multiple of 32 columns or greater + than 512 columns or not a power of 2, or + 2. If an attempt is made to allocate after executing the + `cute_nvgpu.arch.sm100.relinquish_tmem_alloc_permit` operation. + + This operation must be issued by a single fully active warp of the CTA and + must not be issued by more than one warp simultaneously. For repeated + allocations, the same warp must be used to issue all allocations. All + `alloc_tmem` operations must be issued by the same warp. + + For the allocation of 2 CTAs, one warp from each of the peer CTAs must + collectively perform the allocation, and the issuing warp must make sure that + peer CTA is launched and is still active. The participating warps from both + CTAs should have the same logical warp ID. There is no explicit need for + synchronization between these 2 CTAs before executing this operation. + + The behavior of this operation is undefined if all the threads in the warp do + not use the same values of `n_cols`, or if any thread in the warp has exited. + + The meaning of each argument is as follows: + - `n_cols`: The number of columns to be allocated. The unit of allocation is + 32 columns and the number of columns must be a power of 2. It must be within + the range [32, 512]. The number of columns allocated should not increase + between any two allocations in the execution order within the CTA. Failing to + meet these conditions may lead to undefined behavior. + - `dst_ptr`: Pointer to shared memory where the result TMEM pointer will be + written. It should have a natural alignment of 32-bits. + - `is_two_cta`: Specifies the number of CTAs involved in the allocation. + + Examples: + + ```mlir + cute_nvgpu.arch.sm100.alloc_tmem(%n_cols, %res_shm_ptr) [cta_1] + : i32, !cute.ptr> + + cute_nvgpu.arch.sm100.alloc_tmem(%n_cols, %res_shm_ptr) [cta_2] + : i32, !cute.ptr> + ``` + }]; + let arguments = (ins + I32:$n_cols, + CuteNVGPU_CuTePointerToMemorySpacesSmem:$dst_ptr, + UnitAttr:$is_two_cta, + UnitAttr:$exclusive + ); + let results = (outs + ); +} +def CuteNVGPU_ArchSM100DeallocTmemOp : CuteNVGPU_Op<"arch.sm100.dealloc_tmem"> { + let description = [{ + The `arch.sm100.dealloc_tmem` operation deallocates the Tensor Memory + specified by the TMEM address `tmem_ptr` and slice count. The address + `tmem_ptr` must point to a previously allocated TMEM. If the + deallocation fails, a runtime error occurs. + + This operation should only be performed by a single fully active warp within + the CTA. It should not be carried out by multiple warps simultaneously. The + warp ID that does alloc and dealloc can be different, but all `dealloc_tmem` + operations must be issued by the same warp. + + For the 2 CTAs deallocation, one warp from each of the peer CTAs must + collectively perform the de-allocation, and the issuing warp must make sure + that peer CTA is launched and is still active. The warps from both CTAs should + have the same logical warp ID. Prior explicit synchronization between the two + CTAs is not required before executing this operation. + + The behavior of this operation is undefined if all the threads in the warp do + not use the same values of `n_cols`, or if any thread in the warp has exited. + + The meaning of each argument is as follows: + - `tmem_ptr`: The base address of the TMEM buffer being deallocated. + - `n_cols`: The number of columns to be deallocated. The unit of deallocation + is 32 columns and the number of columns must be a power of 2. It must be within + the range [32, 512]. Failing to meet these conditions may lead to undefined + behavior. + - `is_two_cta`: Specifies the number of CTAs involved in the deallocation. + + Examples: + + ```mlir + cute_nvgpu.arch.sm100.dealloc_tmem(%tmem_ptr, %n_cols) [cta_1] + : !cute.ptr>, i32 + + cute_nvgpu.arch.sm100.dealloc_tmem(%tmem_ptr, %n_cols) [cta_2] + : !cute.ptr>, i32 + ``` + }]; + let arguments = (ins + CuteNVGPU_CuTePointerToMemorySpacesTmem:$tmem_ptr, + I32:$n_cols, + UnitAttr:$is_two_cta, + UnitAttr:$exclusive + ); + let results = (outs + ); +} +def CuteNVGPU_ArchSM100RelinquishTmemAllocPermitOp : CuteNVGPU_Op<"arch.sm100.relinquish_tmem_alloc_permit"> { + let description = [{ + This `arch.sm100.relinquish_tmem_alloc_permit` specifies that the CTA of the + executing thread is relinquishing the right to allocate Tensor Memory so that + other CTAs can allocate. + + Once this operation is executed, any subsequent attempts by the same CTA to + allocate TMEM are considered illegal. + + Example: + + ```mlir + cute_nvgpu.arch.sm100.relinquish_tmem_alloc_permit [cta_1] + ``` + }]; + let arguments = (ins + UnitAttr:$is_two_cta + ); + let results = (outs + ); +} +def CuteNVGPU_ArchSM100RetrieveTmemPtrOp : CuteNVGPU_Op<"arch.sm100.retrieve_tmem_ptr", [MemoryEffects<[MemRead]>]> { + let description = [{ + The `arch.sm100.retrieve_tmem_ptr` operation is a convenient shorthand for + loading the allocated TMEM pointer from shared memory and casting it into a + TMEM pointer. It includes a correctness check for the TMEM pointer and + returns a null pointer if the TMEM pointer is invalid. + + Example: + + ```mlir + %tmem_ptr = cute_nvgpu.arch.sm100.retrieve_tmem_ptr(%res_shm_ptr) + : !cute.ptr> -> !cute.ptr> + ``` + }]; + let arguments = (ins + CuteNVGPU_CuTePointerToMemorySpacesSmem:$smem_ptr + ); + let results = (outs + CuteNVGPU_CuTePointerToMemorySpacesTmem:$tmem_ptr + ); +} +def CuteNVGPU_AtomCopyMakeExecTmaOp : CuteNVGPU_InferredOp<"atom.make_exec_tma", [Pure]> { + let arguments = (ins + CuteNVGPU_TypeInterfaceForNonExecutableTMAAtomTypes:$input, + UnitAttr:$override, + UnitAttr:$no_fully_oob_tile + ); + let results = (outs + CuteNVGPU_CopyAtomTypeInterfaceInstance:$result + ); +} +def CuteNVGPU_AtomCopyMakeNonExec2DGather4TmaLoadOp : CuteNVGPU_InferredOp<"atom.make_non_exec_2d_gather4_tma_load", [Pure]> { + let summary = "This operation creates"; + let description = [{ + 1. a non-executable atom for TMA loads in tiled gather4 mode (.tile::gather4) + 2. the TMA tensor mapping the logical coordinates of gmem_tensor to TMA coordinate offsets + + The non-executable atom, which wraps a TMA descriptor and attaches type information to it for + safety, can then be used to create an executable one using + cute_nvgpu.atom.copy.make_exec_tma. It can also be used directly for TiledCopy-based + partitioning of source and destination tensors. + }]; + let arguments = (ins + CuteNVGPU_AMemoryReferenceTypeThatRepresentsATensor:$gmem_tensor, + CuteNVGPU_CuTeLayoutType:$gmem_coord_layout, + CuteNVGPU_GeneralizedLayoutTypeOfCuTe:$smem_layout, + CuteNVGPU_CuTeLayoutType:$cta_v_map, + CuteNVGPU_GatherScatterTmaLoadAttr:$kind, + OptionalAttr:$num_multicast, + OptionalAttr:$tma_format + ); + let results = (outs + CuteNVGPU_TheNonExecutableCopyAtomTypeForTMA:$non_exec_atom, + CuteNVGPU_CuteCoordTensorType:$tma_tensor + ); +} +def CuteNVGPU_AtomCopyMakeNonExec2DScatter4TmaStoreOp : CuteNVGPU_InferredOp<"atom.make_non_exec_2d_scatter4_tma_store", [Pure]> { + let summary = "This operation creates"; + let description = [{ + 1. a non-executable atom for TMA stores in tiled scatter4 mode (.tile::scatter4) + 2. the TMA tensor mapping the logical coordinates of the scatter index layout to TMA coordinate offsets + + The non-executable atom, which wraps a TMA descriptor and attaches type information to it for + safety, can then be used to create an executable one using + cute_nvgpu.atom.copy.make_exec_tma. It can also be used directly for TiledCopy-based + partitioning of source and destination tensors. + }]; + let arguments = (ins + CuteNVGPU_AMemoryReferenceTypeThatRepresentsATensor:$gmem_tensor, + CuteNVGPU_CuTeLayoutType:$gmem_index_layout, + CuteNVGPU_GeneralizedLayoutTypeOfCuTe:$smem_layout, + CuteNVGPU_CuTeLayoutType:$cta_v_map, + OptionalAttr:$tma_format + ); + let results = (outs + CuteNVGPU_TheNonExecutableCopyAtomTypeForTMA2:$non_exec_atom, + CuteNVGPU_CuteCoordTensorType:$tma_tensor + ); +} +def CuteNVGPU_AtomCopyMakeNonExecIm2ColTmaLoadOp : CuteNVGPU_InferredOp<"atom.make_non_exec_im2col_tma_load", [Pure]> { + let summary = "This operation creates"; + let description = [{ + 1. a non-executable atom for TMA loads in im2col mode + 2. the TMA tensor mapping the logical coordinates of gmem_tensor to TMA coordinate offsets + + The non-executable atom, which wraps a TMA descriptor and attaches type information to it for + safety, can then be used to create an executable one using + cute_nvgpu.atom.copy.make_exec_tma_load. It can also be used directly for TiledCopy-based + partitioning of source and destination tensors. + }]; + let arguments = (ins + CuteNVGPU_AMemoryReferenceTypeThatRepresentsATensor:$gmem_tensor, + CuteNVGPU_CuTeComposedLayoutType:$smem_layout, + CuteNVGPU_CuTeLayoutType:$cta_v_map, + CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$lower_corner_whd, + CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$upper_corner_whd, + CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$lower_padding_whd, + CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$upper_padding_whd, + CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$stride_whd, + CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$lower_srt, + CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$stride_srt, + CuteNVGPU_Im2ColTmaLoadAttr:$kind, + OptionalAttr:$num_multicast, + OptionalAttr:$tma_format + ); + let results = (outs + CuteNVGPU_TheNonExecutableCopyAtomTypeForTMA3:$non_exec_atom, + CuteNVGPU_CuteCoordTensorType:$tma_tensor + ); +} +def CuteNVGPU_AtomCopyMakeNonExecIm2ColTmaStoreOp : CuteNVGPU_InferredOp<"atom.make_non_exec_im2col_tma_store", [Pure]> { + let summary = "This operation creates"; + let description = [{ + 1. a non-executable atom for TMA stores in im2col mode + 2. the TMA tensor mapping the logical coordinates of gmem_tensor to TMA coordinate offsets + + The non-executable atom, which wraps a TMA descriptor and attaches type information to it for + safety, can then be used to create an executable one using + cute_nvgpu.atom.copy.make_exec_tma_store. It can also be used directly for TiledCopy-based + partitioning of source and destination tensors. + }]; + let arguments = (ins + CuteNVGPU_AMemoryReferenceTypeThatRepresentsATensor:$gmem_tensor, + CuteNVGPU_CuTeComposedLayoutType:$smem_layout, + CuteNVGPU_CuTeLayoutType:$cta_v_map, + OptionalAttr:$tma_format + ); + let results = (outs + CuteNVGPU_TheNonExecutableCopyAtomTypeForTMA4:$non_exec_atom, + CuteNVGPU_CuteCoordTensorType:$tma_tensor + ); +} +def CuteNVGPU_AtomCopyMakeNonExecTiledTmaLoadOp : CuteNVGPU_InferredOp<"atom.make_non_exec_tiled_tma_load", [Pure]> { + let summary = "This operation creates"; + let description = [{ + 1. a non-executable atom for TMA loads in tiled mode + 2. the TMA tensor mapping the logical coordinates of gmem_tensor to TMA coordinate offsets + + The non-executable atom, which wraps a TMA descriptor and attaches type information to it for + safety, can then be used to create an executable one using + cute_nvgpu.atom.copy.make_exec_tma. It can also be used directly for TiledCopy-based + partitioning of source and destination tensors. + }]; + let arguments = (ins + CuteNVGPU_AMemoryReferenceTypeThatRepresentsATensor:$gmem_tensor, + CuteNVGPU_GeneralizedLayoutTypeOfCuTe:$smem_layout, + CuteNVGPU_CuTeLayoutType:$cta_v_map, + CuteNVGPU_TiledTmaLoadAttr:$kind, + OptionalAttr:$num_multicast, + OptionalAttr:$tma_format + ); + let results = (outs + CuteNVGPU_TheNonExecutableCopyAtomTypeForTMA5:$non_exec_atom, + CuteNVGPU_CuteCoordTensorType:$tma_tensor + ); +} +def CuteNVGPU_AtomCopyMakeNonExecTiledTmaReduceOp : CuteNVGPU_InferredOp<"atom.make_non_exec_tiled_tma_reduce", [Pure]> { + let summary = "This operation creates"; + let description = [{ + 1. a non-executable atom for TMA reduce in tiled mode + 2. the TMA tensor mapping the logical coordinates of gmem_tensor to TMA coordinate offsets + + The non-executable atom, which wraps a TMA descriptor and attaches type information to it for safety, can then be used to create an executable one using + cute_nvgpu.atom.copy.make_exec_tma. It can also be used directly for TiledCopy-based partitioning of source and destination tensors. + }]; + let arguments = (ins + CuteNVGPU_AMemoryReferenceTypeThatRepresentsATensor:$gmem_tensor, + CuteNVGPU_GeneralizedLayoutTypeOfCuTe:$smem_layout, + CuteNVGPU_CuTeLayoutType:$cta_v_map, + CuteNVGPU_ReductionKindAttr:$kind, + OptionalAttr:$tma_format + ); + let results = (outs + CuteNVGPU_TheNonExecutableCopyAtomTypeForTMA6:$non_exec_atom, + CuteNVGPU_CuteCoordTensorType:$tma_tensor + ); +} +def CuteNVGPU_AtomCopyMakeNonExecTiledTmaStoreOp : CuteNVGPU_InferredOp<"atom.make_non_exec_tiled_tma_store", [Pure]> { + let summary = "This operation creates"; + let description = [{ + 1. a non-executable atom for TMA store in tiled mode + 2. the TMA tensor mapping the logical coordinates of gmem_tensor to TMA coordinate offsets + + The non-executable atom, which wraps a TMA descriptor and attaches type information to it for + safety, can then be used to create an executable one using + cute_nvgpu.atom.copy.make_exec_tma. It can also be used directly for TiledCopy-based + partitioning of source and destination tensors. + }]; + let arguments = (ins + CuteNVGPU_AMemoryReferenceTypeThatRepresentsATensor:$gmem_tensor, + CuteNVGPU_GeneralizedLayoutTypeOfCuTe:$smem_layout, + CuteNVGPU_CuTeLayoutType:$cta_v_map, + OptionalAttr:$tma_format + ); + let results = (outs + CuteNVGPU_TheNonExecutableCopyAtomTypeForTMA7:$non_exec_atom, + CuteNVGPU_CuteCoordTensorType:$tma_tensor + ); +} +def CuteNVGPU_AtomCopyMakeS2TCopyOp : CuteNVGPU_InferredOp<"atom.make_s2t_copy", [Pure]> { + let description = [{ + This operation creates a new copy atom instance by tiling an utccp copy atom + instance with a tmem memref. + + Example: + ```mlir + %copy_s2t_atom = cute_nvgpu.atom.make_s2t_copy() : !cute_nvgpu.atom.s2t_copy + %tmem_memref = cute.memref.alloca() : !cute.memref, "(128,32):(65536,1)"> + %0 = cute_nvgpu.atom.make_s2t_copy(%copy_s2t_atom, %tmem_memref) + : (!cute_nvgpu.atom.s2t_copy, + !cute.memref, "(128,32):(65536,1)">) + -> !cute.tiled_copy, + layout_copy_tv = <"(1,(128,8)):(0,(1,128))">, + tiler_mn = <"[128:1;8:1]">> + ``` + }]; + let arguments = (ins + CuteNVGPU_SM100BlackwellSharedToTensorMemoryCopyAtom:$copy_s2t_atom, + CuteNVGPU_AMemoryReferenceTypeThatRepresentsATensor:$tmem_memref + ); + let results = (outs + CuteNVGPU_TiledCopyAtomType:$result + ); +} +def CuteNVGPU_AtomCopyMakeTmemCopyOp : CuteNVGPU_InferredOp<"atom.make_tmem_copy", [Pure]> { + let description = [{ + This operation creates a new copy atom instance by tiling a load tmem copy atom + instance with a tmem memref. + + Example: + ```mlir + // %tmem_memref : !cute.memref, "(128,32):(65536,1)"> + %load_atom = cute.make_atom() : !cute_nvgpu.atom.tmem_load + %0 = cute_nvgpu.atom.make_tmem_copy(%load_atom, %tmem_memref) + : (!cute_nvgpu.atom.tmem_load, + !cute.memref, "(128,32):(65536,1)">) + -> !cute.tiled_copy, + layout_copy_tv = <"((32,4),(32,32)):((0,1),(128,4))">, + tiler_mn = <"[(4,32):(32,1);32:1]">> + ``` + }]; + let arguments = (ins + CuteNVGPU_CopyAtomTypeInterfaceInstance:$atom, + CuteNVGPU_AMemoryReferenceTypeThatRepresentsATensor:$tmem_memref + ); + let results = (outs + CuteNVGPU_TiledCopyAtomType:$result + ); +} +def CuteNVGPU_AtomGetCoordTensorOp : CuteNVGPU_InferredOp<"atom.get_coord_tensor", [SameVariadicOperandSize, Pure]> { + let description = [{ + This operation creates a get tma coord atom instance from copy atom and shapes. + The atom will be called before algo copy(tma copy) to provide tma coord for algo copy. + In Cute C++, copy_atom cache stride info during make_tiled_copy and the stride + info is associated with shapes. shapes in argument will make a congruent comparison + with cache first and then construct tma coord with stride. + + Example: + ```mlir + %5 = cute_nvgpu.atom.get_coord_tensor(%4, %5) + : (!cute_nvgpu.atom.tma_load>, + !cute.shape<128, 64>) + -> !cute.coord_tensor<"(0,0)", "(512,64,2):(1@1,1@0,64@0)"> + ``` + + ```mlir + %6 = cute_nvgpu.atom.get_coord_tensor(%4, %5, %2, %3) + : (!cute_nvgpu.atom.tma_load>, + !cute.shape<"(500, 128)">, + !cute.int_tuple<"(0, 0)">, + !cute.layout<"(128, 256): (1@0, 1@1) + -> !cute.coord_tensor<"(0,0)", "(500,128):(1@1,1@0) o (0,0) o (128,256):(1@0,1@1)"> + ``` + }]; + let arguments = (ins + CuteNVGPU_AtomTypeInterfaceInstance:$atom, + CuteNVGPU_CuteShapeType:$shape, + Optional:$offset_b, + Optional:$layout_b + ); + let results = (outs + CuteNVGPU_CuteCoordTensorType:$result + ); +} +def CuteNVGPU_AtomGetCopyS2TSmemDescViewOp : CuteNVGPU_InferredOp<"atom.get_copy_s2t_smem_desc_view", [Pure]> { + let description = [{ + This operation creates a SmemDescView instance from the given UtccpAtomType and + the SmemView. The given SmemView must match with the Utccp's TiledCopy src partitioned layout. + The result will be used as the src of algo.copy. + + Example: + ```mlir + %res_0 = cute_nvgpu.atom.get_copy_s2t_smem_desc_view(%atom_0, %src_0) + : (!cute_nvgpu.atom.s2t_copy, + !cute.memref, S<2,4,3>, "(((128,4),1),2,(4,2)):(((16,1),0),2048,(4,4096))">) + -> !cute_nvgpu.smem_desc_view + ``` + }]; + let arguments = (ins + CuteNVGPU_SM100BlackwellSharedToTensorMemoryCopyAtom2:$atom, + CuteNVGPU_CuTeMemrefToMemorySpacesSmem:$view + ); + let results = (outs + CuteNVGPU_CuteSmemDescViewType:$result + ); +} +def CuteNVGPU_AtomGetCopyS2TSmemDescViewSM107Op : CuteNVGPU_InferredOp<"atom.sm107.get_copy_s2t_smem_desc_view", [Pure]> { + let description = [{ + This operation creates a SmemDescView instance from the given UtccpAtomType and + the SmemView. The given SmemView must match with the Utccp's TiledCopy src partitioned layout. + The result will be used as the src of algo.copy. + + Example: + ```mlir + %res_0 = cute_nvgpu.atom.sm107.get_copy_s2t_smem_desc_view(%atom_0, %src_0) + : (!cute_nvgpu.atom.s2t_copy, + !cute.memref, S<2,4,3>, "(((128,4),1),2,(4,2)):(((16,1),0),2048,(4,4096))">) + -> !cute_nvgpu.smem_desc_view + ``` + }]; + let arguments = (ins + CuteNVGPU_SM100BlackwellSharedToTensorMemoryCopyAtom2:$atom, + CuteNVGPU_CuTeMemrefToMemorySpacesSmem:$view + ); + let results = (outs + CuteNVGPU_CuteSmemDescViewType:$result + ); +} +def CuteNVGPU_AtomGetValueOp : CuteNVGPU_Op<"atom.get_value", [Pure]> { + let description = [{ + CuTe atoms represent both compile-time and runtime information for an + architecture-specific `copy` or `MMA` operation. For runtime information, + an atom can be thought of as similar to a C/C++ `struct` with predefined + member variables. Instead of defining operations to get and set the member + variables of each architecture atom, the CuTe dialect uses `get_value` and + `set_value` operations to uniformly handle retrieving and updating atom + member variables (hereafter referred to as "fields"). + The fields of each architecture-specific atom are identified by an enum + attribute. See the documentation for the operations that create an atom + or the attribute associated with the atom type for the expected enum values + and associated operand types. + The `field` attribute must correspond to the type of the `atom` operand. + For example, if the `atom` type is `!cute_nvgpu.sm100.mma`, the enum + attribute must be `#cute_nvgpu.atom_mma_field_sm100`. The verifier for this + operation will also compare the value `Type` to that expected by the + specific field. + + Example: + ```mlir + %5 = cute_nvgpu.atom.get_value(%4) : (!cute_nvgpu.sm100.mma<64x64x16, + num_cta = 1, + ab_major = (k,mn), + elem_type = (f16,f16,f16), + frag_kind = ss, + c_scale_exp = 0>) -> i1 + ``` + }]; + let arguments = (ins + CuteNVGPU_AtomTypeInterfaceInstance:$atom, + AnyAttr:$field + ); + let results = (outs + AnyType:$result + ); +} +def CuteNVGPU_AtomMakeTmaResidueTensorOp : CuteNVGPU_InferredOp<"atom.make_tma_residue_tensor", [Pure]> { + let description = [{ + Builds the per-TMA-mode "remaining elements" coord tensor that + `cute.copy` consumes on the override path. The op takes a non-executable + TMA atom (whose type carries `tma_gbasis`) and the corresponding gmem + tensor, and produces a coord tensor whose + + - iter is gmem tensor's shape combined and ordered according to tma gbasis. + - layout has the same shape as the tma tensor while the stride is negated. + + The residue tensor tracks the remaining shape to copy during TMA override, + e.g, slicing the residue tensor with a cta coordinate gets the remaining + shape for it given the negated stride. + + Example: + ```mlir + %residue = cute_nvgpu.atom.make_tma_residue_tensor(%non_exec_atom, %gmem) + : (!cute_nvgpu.atom.non_exec_tiled_tma_load>, + !cute.memref, "(?,?):(?,1)">) + -> !cute.coord_tensor<"(?,?)", "(?,?):(-1@1,-1@0)"> + ``` + }]; + let arguments = (ins + CuteNVGPU_TheNonExecutableCopyAtomTypeForTMA8:$tma_atom, + CuteNVGPU_AMemoryReferenceTypeThatRepresentsATensor:$gmem_tensor + ); + let results = (outs + CuteNVGPU_CuteCoordTensorType:$residue + ); +} +def CuteNVGPU_AtomSetValueOp : CuteNVGPU_InferredOp<"atom.set_value", [Pure]> { + let description = [{ + CuTe atoms represent both compile-time and runtime information for an + architecture-specific `copy` or `MMA` operation. For runtime information, + an atom can be thought of as similar to a C/C++ `struct` with predefined + member variables. Instead of defining operations to get and set the member + variables of each architecture atom, the CuTe dialect uses `get_value` and + `set_value` operations to uniformly handle retrieving and updating atom + member variables (hereafter referred to as "fields"). + The fields of each architecture-specific atom are identified by an enum + attribute. See the documentation for the operations that create an atom + or the attribute associated with the atom type for the expected enum values + and associated operand types. + The `field` attribute must correspond to the type of the `atom` operand. + For example, if the `atom` type is `!cute_nvgpu.sm100.mma`, the enum + attribute must be `#cute_nvgpu.atom_mma_field_sm100`. The verifier for this + operation will also compare the value `Type` to that expected by the + specific field. + + Example: + ```mlir + %2 = cute_nvgpu.atom.set_value(%0, %no_accum) : + (!cute_nvgpu.sm100.mma<64x64x16, + num_cta = 1, + ab_major = (k,mn), + elem_type = (f16,f16,f16), + frag_kind = ss, + c_scale_exp = 0>, i1) + ``` + }]; + let arguments = (ins + CuteNVGPU_AtomTypeInterfaceInstance:$atom, + AnyType:$value, + AnyAttr:$field + ); + let results = (outs + CuteNVGPU_AtomTypeInterfaceInstance:$result + ); +} +def CuteNVGPU_AtomTmaPartitionOp : CuteNVGPU_InferredOp<"atom.tma_partition", [Pure]> { + let description = [{ + tma_partition is the new TMA routine for the dynamic and preferred cluster feature. + tma_partition op requires below arguments: + 1. tma_atom; + 2. cta_coord in the multicast dimension; + 3. cta_layout in the multicast dimension; + 4. smem_memref; + 5. variadic CuteViewTypeInterface tensors (one or more coord_tensor / memref), + covering the following cases: + 5.a: for regular TMA load/store, a single TMA tensor as input + 5.b: for tile::gather4 mode, a 2D TMA coord tensor plus the corresponding + gmem index tensor + and it produces: + 1. partitioned_smem_memref; + 2. one partitioned coord_tensor / memref per input, with the following cases: + 2.a: for regular TMA load/store, one partitioned TMA tensor + 2.b: for tile::gather4 mode, the partitioned TMA coord tensor plus the + partitioned gmem index tensor + + The return values are the src and dst arguments of algo.copy(atom, src, dst). + The second and the third input arguments can be runtime values, so the multicast + can be runtime determined. + + This operation is also compatible for the old static cluster TMA routines. + You only need to feed the cta_coord and cta_layout with static values, + then you can get compile-time multicast computation. + + The target_tensors operand is variadic and accepts any CuteViewTypeInterface + type (coord_tensor or memref). When a single tensor is provided, brackets + are omitted for backward compatibility. When multiple tensors are provided, + they are grouped in brackets. + + Example (single tensor): + ```mlir + %res_smem, %res_gmem = cute_nvgpu.atom.tma_partition( + %tma_atom, %cta_coord, %cta_layout, %stensor, %gtensor + ) : (!cute_nvgpu.atom.tma_load, + !cute.coord<"?">, + !cute.layout<"?:1">, + !cute.memref, S<3,4,3>, "(((128,16),1,4),8):(((64,1),0,16),8192)">, + !cute.coord_tensor<"(0,?,0)", "(((128,16),1,4),?,?,?):(((1@1,1@0),0,16@0),256@1,64@0,1@2)">) + ``` + + Example (multiple tensors): + ```mlir + %res_smem, %res_gmem0, %res_gmem1 = cute_nvgpu.atom.tma_partition( + %tma_atom, %cta_coord, %cta_layout, %stensor, [%gtensor0, %gtensor1] + ) : (!cute_nvgpu.atom.tma_load<...>, + !cute.coord<"?">, + !cute.layout<"?:1">, + !cute.memref, + [!cute.coord_tensor<...>, !cute.memref]) + -> (!cute.memref, + [!cute.coord_tensor<...>, !cute.memref]) + ``` + }]; + let arguments = (ins + CuteNVGPU_SM90TMATensorMemoryAcceleratorLoadAtomType:$tma_atom, + CuteNVGPU_CuteCoordType:$cta_coord, + CuteNVGPU_CuTeLayoutType:$cta_layout, + CuteNVGPU_CuTeMemrefToMemorySpacesSmem:$smem_tensor, + Variadic:$target_tensors + ); + let results = (outs + CuteNVGPU_CuTeMemrefToMemorySpacesSmem:$res_smem_tensor, + Variadic:$res_target_tensors + ); +} +def CuteNVGPU_CastTMADescToIntegerOp : CuteNVGPU_Op<"cast_tma_desc_to_integer", [Pure]> { + let summary = "cast_tma_desc_to_integer cast a tma_desc into an integer."; + let description = [{ + Example: + ```mlir + %23 = cute_nvgpu.cast_tma_desc_to_integer(%11 : !cute.ptr>) -> i64 + ``` + }]; + let arguments = (ins + CuteNVGPU_SM90PointerToTMATensorDescriptor:$src + ); + let results = (outs + CuteNVGPU_Integer:$dst + ); +} +def CuteNVGPU_CopyTMADescriptorOp : CuteNVGPU_Op<"copy_tma_desc"> { + let description = [{ + This operation copies the TMA descriptor from a non-executable TMA atom to the pointer. + The pointer could be a smem or gmem pointer and should be aligned to at least the size of TMA descriptor. + }]; + let arguments = (ins + CuteNVGPU_TypeInterfaceForNonExecutableTMAAtomTypes:$atom, + CuteNVGPU_CuTePointerToMemorySpacesSmemOrCuTe:$tma_desc_ptr + ); + let results = (outs + ); +} +def CuteNVGPU_GetGridConstantPointerOp : CuteNVGPU_InferredOp<"get_grid_constant_pointer", [Pure]> { + let description = [{ + 'cute_nvgpu.get_grid_constant_pointer' takes a grid constant value argument + from a kernel and returns a `cute.ptr` whose pointee type matches the + input. + + Before device ABI conversion, the input may be an intermediate result that + function finalization will promote to a grid-constant kernel argument. + Direct kernel arguments are verified immediately; intermediate provenance + is validated when the physical kernel signature is finalized. The result + type and alignment are fully inferred from the input type by + `InferTypeOpInterface`. + Example (TMA descriptor): + func.func @kernel( + %tma_desc_a : !cute_nvgpu.tma_descriptor_tiled {cute_nvgpu.grid_constant}) + attributes {cute.kernel} { + %tma_desc_a_ptr = cute_nvgpu.get_grid_constant_pointer (%tma_desc_a) : + !cute.ptr>; + ... + } + + Example (arbitrary value type): + func.func @kernel( + %params : !llvm.struct<(i64, i64)> {cute_nvgpu.grid_constant}) + attributes {cute.kernel} { + %params_ptr = cute_nvgpu.get_grid_constant_pointer (%params) : + !cute.ptr, generic>; + ... + } + }]; + let arguments = (ins + AnyType:$input + ); + let results = (outs + CuteNVGPU_CuTePointerType:$result + ); +} +def CuteNVGPU_GetTmaDescAddrOp : CuteNVGPU_Op<"get_tma_desc_addr", [Pure]> { + let description = [{ + The `cute_nvgpu.get_tma_desc_addr` operation accepts an input copy atom, + extracts the address within copy atom, then returns it as the address + of current copy atom's TMA desc in a single SSA value. The address within + input copy atom should have global/generic address space, and should be aligned to 64B. + Example: + + ```mlir + %1 = cute_nvgpu.get_tma_desc_addr(%cpy_atom : !cute_nvgpu.atom.tma_load) -> !cute.ptr> + ``` + }]; + let arguments = (ins + CuteNVGPU_CopyAtomTypeInterfaceInstance:$src + ); + let results = (outs + CuteNVGPU_CuTePointerToGlobalMemoryWith64Byte:$ptr + ); +} +def CuteNVGPU_MakeGMMASmemDescOp : CuteNVGPU_Op<"make_gmma_smem_desc", [Pure]> { + let description = [{ + The `cute_nvgpu.make_gmma_smem_desc` operation accepts an input cute.ptr alongside a cute.layout and a major attr, + then constructs the shared memory descriptor and returns it. The layout must be describing + the buffer pointed to by the input pointer and must carry valid swizzle information. + For GMMA, there are 4 supported swizzle variants: + + ``` + swizzle | cooresponding enum in gmma smem desc + S<0, 4, 3> | SWIZZLE_NONE + S<1, 4, 3> | SWIZZLE_32B + S<2, 4, 3> | SWIZZLE_64B + S<3, 4, 3> | SWIZZLE_128B + ``` + The cute.ptr must carry shared address space and must be aligned to 16B. + + Example: + + ```mlir + %1 = cute_nvgpu.make_gmma_smem_desc(%arg1 : !cute.ptr, S<3,4,3>>) + layout = <"((64,16),1):((64,1),0)">, major = #cute_nvgpu.major + -> !cute_nvgpu.smem_desc + ``` + }]; + let arguments = (ins + CuteNVGPU_CuTePointerToAllowableTypesForSM90MMA:$src, + Cute_LayoutAttr:$layout, + CuteNVGPU_MajorModeAttr:$major + ); + let results = (outs + CuteNVGPU_CuteSmemDescType:$res + ); +} +def CuteNVGPU_MakeTmaDescIm2ColAtOp : CuteNVGPU_Op<"make_tma_desc_im2col_at"> { + let description = [{ + In-place variant of Cute_MakeTmaDescIm2ColOp. The descriptor is stored + into the address passed as the last operand. + + Operands (identical semantics to the im2col op) + - All operands of MakeTmaDescIm2ColOp (see above). + - Value tma_desc_addr – destination address for the descriptor. + + Optional attributes (identical semantics to the im2col op) + - num_multicast, tma_format + + Results: none – descriptor is written to the supplied address. + + Example: + ```mlir + cute_nvgpu.make_tma_desc_im2col_at( + %tensor, + %slayout, + %cta_v_tile, + %lower_corner_whd, + %upper_corner_whd, + %lower_padding_whd, + %upper_padding_whd, + %stride_whd, + %lower_srt, + %stride_srt, + %tma_desc_addr) + <{num_multicast = 1 tma_format = #cute_nvgpu.tma_data_format}> : ( + !cute.memref, "((8,8,1,4),64,1):((64,512,4096,4096),1,0)">, + !cute.composed_layout<"S<2,4,3> o 0 o ((8,16),(32,1)):((32,256),(1,0))">, + !cute.layout<"(128,32):(1@0,1@1)">, + !cute.int_tuple<"(0,0,0)">, + !cute.int_tuple<"(0,0,0)">, + !cute.int_tuple<"(0,0,0)">, + !cute.int_tuple<"(0,0,0)">, + !cute.int_tuple<"(1,1,1)">, + !cute.int_tuple<"(0,0,0)">, + !cute.int_tuple<"(1,1,1)">, + !cute.ptr>) + ``` + }]; + let arguments = (ins + CuteNVGPU_ViewTypeInterfaceInstance:$gmem_view, + CuteNVGPU_CuTeComposedLayoutType:$smem_layout, + CuteNVGPU_CuTeLayoutType:$cta_value_tile, + CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$lower_corner_whd, + CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$upper_corner_whd, + CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$lower_padding_whd, + CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$upper_padding_whd, + CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$stride_whd, + CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$lower_srt, + CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$stride_srt, + CuteNVGPU_CuTePointerType:$tma_desc_addr, + OptionalAttr:$num_multicast, + OptionalAttr:$tma_format + ); + let results = (outs + ); +} +def CuteNVGPU_MakeTmaDescIm2ColOp : CuteNVGPU_Op<"make_tma_desc_im2col", [Pure]> { + let summary = "Create a TMA descriptor in im2col mode, suitable for convolutional forward passes."; + let description = [{ + Operands + - Value gmem_view, smem_layout, cta_value_tile – as in the tiled op. + - Value lower_corner_whd / upper_corner_whd – start & end (W,H,D) + positions of the + receptive-field window. + - Value lower_padding_whd / upper_padding_whd – padding applied + before/after each dim. + - Value stride_whd – traversal stride in WHD. + - Value lower_srt / stride_srt – split-K traversal + parameters (SRT). + + Optional attributes + - num_multicast : i32 + - tma_format : CuteTmaDataFormatAttr + + Result + - Cute pointer to the generated descriptor. + + Example: + ```mlir + %desc = cute_nvgpu.make_tma_desc_im2col( + %tensor, + %slayout, + %cta_v_tile, + %lower_corner_whd, + %upper_corner_whd, + %lower_padding_whd, + %upper_padding_whd, + %stride_whd, + %lower_srt, + %stride_srt) + <{num_multicast = 1 tma_format = #cute_nvgpu.tma_data_format}> : ( + !cute.memref, "((8,8,1,4),64,1):((64,512,4096,4096),1,0)">, + !cute.composed_layout<"S<2,4,3> o 0 o ((8,16),(32,1)):((32,256),(1,0))">, + !cute.layout<"(128,32):(1@0,1@1)">, + !cute.int_tuple<"(0,0,0)">, + !cute.int_tuple<"(0,0,0)">, + !cute.int_tuple<"(0,0,0)">, + !cute.int_tuple<"(0,0,0)">, + !cute.int_tuple<"(1,1,1)">, + !cute.int_tuple<"(0,0,0)">, + !cute.int_tuple<"(1,1,1)">) -> !cute.ptr> + ``` + }]; + let arguments = (ins + CuteNVGPU_ViewTypeInterfaceInstance:$gmem_view, + CuteNVGPU_CuTeComposedLayoutType:$smem_layout, + CuteNVGPU_CuTeLayoutType:$cta_value_tile, + CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$lower_corner_whd, + CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$upper_corner_whd, + CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$lower_padding_whd, + CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$upper_padding_whd, + CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$stride_whd, + CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$lower_srt, + CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$stride_srt, + OptionalAttr:$num_multicast, + OptionalAttr:$tma_format + ); + let results = (outs + CuteNVGPU_SM90PointerToTMATensorDescriptor:$result + ); +} +def CuteNVGPU_MakeTmaDescTiledAtOp : CuteNVGPU_Op<"make_tma_desc_tiled_at"> { + let summary = "In-place version of Cute_MakeTmaDescTiledOp."; + let description = [{ + Operands (identical semantics to the Tiled op) + - Value gmem_view – Global-memory tensor view. + - Value smem_layout – SMEM box layout. + - Value cta_value_tile – CTA-value-tile mapping. + - Value tma_desc_addr – Cute pointer that names the memory location + where the TMA descriptor should be written. + - Value traversal_stride – Cute int tuple that defines the traversal stride. + If omitted, the traversal stride is set to 1 for all dimensions. + + Optional attributes (identical semantics to the Tiled op) + - mode, num_multicast, tma_format + + Results: none – descriptor is written to the supplied address. + + Example: + ```mlir + cute_nvgpu.make_tma_desc_tiled_at(%a_tensor, %a_smem_clayout, %a_cta_v_tile, %tma_desc_addr) + <{mode = }> : (!cute.memref, "(?,?,?) : (1,?,?)">, + !cute.composed_layout<"S<3,4,3> o 0 o (((64,64),2)):(((1,64,4096))">, + !cute.layout<"(((64,64),2)):(((1@0,1@1),64@0))">, + !cute.ptr>) + + cute_nvgpu.make_tma_desc_tiled_at(%sfa_tensor, %sfa_smem_clayout, %sfa_cta_v_tile, %existing_tma_desc) + <{mode = num_multicast = 2 tma_format = #cute_nvgpu.tma_data_format}> : ( + !cute.memref, "((32,4,?),(32,4,?)):((16,4,?),(0,1,512))">, + !cute.composed_layout<"S<0,4,3> o 0 o ((((32,4),1),(32,1)),1,(4,1))">, + !cute.layout<"(((32,4),32),1,4):(((1@0@0,1@1@0),1@0@1),0,1@1@1)">, + !cute.ptr>) + + cute_nvgpu.make_tma_desc_tiled(%view, %slayout, %cta_v_tile, %tma_desc_addr %traversal_stride) + <{mode = }> : ( + !cute.memref, "((?,?,?,?),?):((?,?,?,?),?)">, + !cute.composed_layout<"S<2,4,3> o 0 o (128,32):(32,1)">, + !cute.layout<"((128),32):((1@0@0),1@1)">, + !cute.ptr>, + !cute.int_tuple<"(1,?,?,?)">) + ``` + }]; + let arguments = (ins + CuteNVGPU_ViewTypeInterfaceInstance:$gmem_view, + CuteNVGPU_GeneralizedLayoutTypeOfCuTe:$smem_layout, + CuteNVGPU_CuTeLayoutType:$cta_value_tile, + CuteNVGPU_CuTePointerType:$tma_desc_addr, + Optional:$traversal_stride, + CuteNVGPU_TmaLoadModeAttr:$mode, + OptionalAttr:$num_multicast, + OptionalAttr:$tma_format + ); + let results = (outs + ); +} +def CuteNVGPU_MakeTmaDescTiledOp : CuteNVGPU_Op<"make_tma_desc_tiled", [Pure]> { + let summary = "Create a TMA descriptor in tiled load mode."; + let description = [{ + Operands + - Value gmem_view – Cute view that describes the tensor in global memory. + - Value smem_layout – Composed/regular Cute layout that defines the + rectangular box in shared memory (SMEM tile). + - Value cta_value_tile – Cute layout that maps CTA threads to elements + within the SMEM tile. + - Value traversal_stride – Cute int tuple that defines the traversal stride. + If omitted, the traversal stride is set to 1 for all dimensions. + + Optional attributes + - mode : CuteTmaLoadModeAttr or CuteTmaStoreModeAttr (defaults to #cute_nvgpu.tma_load_mode) + - num_multicast : i32 (defaults to 1; 1 ⇒ no multicast) + - tma_format : CuteTmaDataFormatAttr (element format written to + bits 74–71 of the HW TMA + descriptor; if omitted an + appropriate default is deduced) + + Result + - Cute pointer to the freshly created TMA-descriptor object. + + Example: + ```mlir + %tma_desc_a = cute_nvgpu.make_tma_desc_tiled(%a_tensor, %a_smem_clayout, %a_cta_v_tile) + <{mode = #cute_nvgpu.tma_load_mode}> : (!cute.memref, "(?,?,?) : (1,?,?)">, + !cute.composed_layout<"S<3,4,3> o 0 o (((64,64),2)):(((1,64),4096))">, + !cute.layout<"(((64,64),2)):(((1@0,1@1),64@0))">) -> !cute.ptr> + + %tma_desc_sfa = cute_nvgpu.make_tma_desc_tiled(%sfa_tensor, %sfa_smem_clayout, %sfa_cta_v_tile) + <{mode = #cute_nvgpu.tma_load_mode num_multicast = 2 tma_format = #cute_nvgpu.tma_data_format}> : ( + !cute.memref, "((32,4,?),(32,4,?)):((16,4,?),(0,1,512))">, + !cute.composed_layout<"S<0,4,3> o 0 o ((((32,4),1),(32,1)),1,(4,1)):((((16,4),512),(0,1)),0,(1,512))">, + !cute.layout<"(((32,4),32),1,4):(((1@0@0,1@1@0),1@0@1),0,1@1@1)">) -> !cute.ptr> + + %tma_desc_strided_c = cute_nvgpu.make_tma_desc_tiled(%view, %slayout, %cta_v_tile, %traversal_stride) + <{mode = #cute_nvgpu.tma_load_mode}> : ( + !cute.memref, "((?,?,?,?),?):((?,?,?,?),?)">, + !cute.composed_layout<"S<2,4,3> o 0 o (128,32):(32,1)">, + !cute.layout<"((128),32):((1@0@0),1@1)">, + !cute.int_tuple<"(1,?,?,?)">) -> !cute.ptr> + ``` + }]; + let arguments = (ins + CuteNVGPU_ViewTypeInterfaceInstance:$gmem_view, + CuteNVGPU_GeneralizedLayoutTypeOfCuTe:$smem_layout, + CuteNVGPU_CuTeLayoutType:$cta_value_tile, + Optional:$traversal_stride, + CuteNVGPU_TmaLoadOrStoreModeAttr:$mode, + OptionalAttr:$num_multicast, + OptionalAttr:$tma_format + ); + let results = (outs + CuteNVGPU_SM90PointerToTMATensorDescriptor:$result + ); +} +def CuteNVGPU_MakeUMMASM107SmemDescOp : CuteNVGPU_Op<"sm107.make_umma_smem_desc", [Pure]> { + let summary = "Example:"; + let description = [{ + ```mlir + %1 = cute_nvgpu.sm107.make_umma_smem_desc(%arg1 : !cute.ptr) + layout = <"S<3, 4, 3> o 0 o ((64,16),1):((64,1),0)">, major = #cute_nvgpu.major + -> !cute_nvgpu.sm107.smem_desc + ``` + }]; + let arguments = (ins + CuteNVGPU_CuTePointerToMemorySpacesSmem:$src, + Optional:$nextSrc, + Cute_LayoutAttr:$layout, + CuteNVGPU_MajorModeAttr:$major + ); + let results = (outs + CuteNVGPU_CuteSm107SmemDescType:$res + ); +} +def CuteNVGPU_MakeUMMASmemDescCircularSM103Op : CuteNVGPU_InferredOp<"sm103.make_umma_smem_desc_circular", [Pure]> { + let summary = "Example:"; + let description = [{ + ```mlir + %1 = cute_nvgpu.sm103.make_umma_smem_desc_circular(%arg1 : !cute.ptr) + layout = <"S<3, 4, 3> o 0 o ((64,16),1):((64,1),0)">, major = #cute_nvgpu.major + -> !cute_nvgpu.smem_desc_circular + ``` + }]; + let arguments = (ins + CuteNVGPU_CuTePointerToMemorySpacesSmemWithByte:$src, + Optional:$nextSrc, + Cute_LayoutAttr:$layout, + CuteNVGPU_MajorModeAttr:$major + ); + let results = (outs + AnyType:$res + ); +} +def CuteNVGPU_MakeUMMASmemDescCircularSM107Op : CuteNVGPU_InferredOp<"sm107.make_umma_smem_desc_circular", [Pure]> { + let summary = "Example:"; + let description = [{ + ```mlir + %1 = cute_nvgpu.sm103.make_umma_smem_desc_circular(%arg1 : !cute.ptr) + layout = <"S<3, 4, 3> o 0 o ((64,16),1):((64,1),0)">, major = #cute_nvgpu.major + -> !cute_nvgpu.smem_desc_circular + ``` + }]; + let arguments = (ins + CuteNVGPU_CuTePointerToMemorySpacesSmemWithByte:$src, + Optional:$nextSrc, + Cute_LayoutAttr:$layout, + CuteNVGPU_MajorModeAttr:$major + ); + let results = (outs + AnyType:$res + ); +} +def CuteNVGPU_MakeUMMASmemDescOp : CuteNVGPU_Op<"make_umma_smem_desc", [Pure]> { + let summary = "Example:"; + let description = [{ + ```mlir + %1 = cute_nvgpu.make_umma_smem_desc(%arg1 : !cute.ptr) + layout = <"S<3, 4, 3> o 0 o ((64,16),1):((64,1),0)">, major = #cute_nvgpu.major + -> !cute_nvgpu.smem_desc + ``` + }]; + let arguments = (ins + CuteNVGPU_CuTePointerToMemorySpacesSmemWithByte:$src, + Optional:$nextSrc, + Cute_LayoutAttr:$layout, + CuteNVGPU_MajorModeAttr:$major + ); + let results = (outs + CuteNVGPU_CuteSmemDescType:$res + ); +} +def CuteNVGPU_PrefetchTmaDescriptorOp : CuteNVGPU_Op<"prefetch_tma_desc"> { + let arguments = (ins + CuteNVGPU_TypeInterfaceForNonExecutableTMAAtomTypes:$atom + ); + let results = (outs + ); +} +def CuteNVGPU_UpdateTMADescOp : CuteNVGPU_Op<"update_tma_desc"> { + let description = [{ + This operation updates the TMA descriptor in smem or gmem with the information from given gmem tensor. + Specifically, the following fields of the TMA descriptor will be updated: + 1. the global tensor base address will be updated with the base address of the global memory tensor. + 2. the tensor shapes will be updated with the shape of the global memory tensor. + 3. the tensor strides will be updated with the stride of the global memory tensor. + Other fields of the TMA descriptor will be left unchanged. + The TMA descriptor pointer specifies the address of the tma descriptor to be updated. + It could be a smem or gmem pointer. The pointer should be aligned to at least 64B. + Example: + ```mlir + cute_nvgpu.update_tma_desc(%tma_atom, %gmem_tensor, %smem_ptr) : ( + !cute_nvgpu.atom.non_exec_tiled_tma_load>, + !cute.memref, "(?,?,?) : (1,?,?)">, + !cute.ptr>) + -> () + + cute_nvgpu.update_tma_desc(%tma_atom, %gmem_tensor, %gmem_ptr) : ( + !cute_nvgpu.atom.non_exec_tiled_tma_load>, + !cute.memref, "(?,?,?) : (?,1,?)">, + !cute.ptr>) + -> () + ``` + }]; + let arguments = (ins + CuteNVGPU_TypeInterfaceForNonExecutableTMAAtomTypes:$tma_atom, + CuteNVGPU_AMemoryReferenceTypeThatRepresentsATensor:$gmem_tensor, + CuteNVGPU_CuTePointerToMemorySpacesSmemOrCuTe:$tma_desc_ptr + ); + let results = (outs + ); +} + +#endif // MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUOPS_TD diff --git a/mlir/dialects/cute_nvgpu/CuteNVGPUTypes.td b/mlir/dialects/cute_nvgpu/CuteNVGPUTypes.td new file mode 100644 index 00000000..817af564 --- /dev/null +++ b/mlir/dialects/cute_nvgpu/CuteNVGPUTypes.td @@ -0,0 +1,750 @@ +//===- CuteNVGPUTypes.td - CuTe NVIDIA GPU types -----------*- tablegen -*-===// +// +// Generated for nvidia-cutlass-dsl 4.8.0 from the DSL compiler +// (`_cutlass_ir.cu13` parser messages and verifier strings, its typed Python +// type classes, and probing its type parser and printer); do not edit by hand. +// +// Every cute_nvgpu type of the compiler is here, with its parameters in the +// compiler's order and C++ types. Parameters the compiler parses with custom +// directives (the `MxNxK` shape, `frag_kind`, `D DP`, `xR`, `t|n`, the sm90 +// negation flags) are named after the typed getters or the directive. +// +// Syntax is the compiler's, byte for byte, except where its printer emits text +// its own parser rejects; the parsers here also accept that text, and the +// printers here then emit the form the compiler parses back to the same type: +// - atom.universal_copy with copy_bits 0 and an address space prints `0 b`; +// - atom.simt_multimem_* print a default field when a later one is not; +// - sm100.mma_bs(_sp) print `arch_promote` when `int_overflow` is present. +// +// Verifiers carry the compiler's rules on parameter values where they are +// known, with its messages; instruction legality (the MMA shape and type +// tables, multimem PTX forms, ldmatrix/stmatrix configurations) is left to the +// compiler. +// +// Left out: the hidden `atom` attribute of the mma atoms' typed get()s. It +// overrides the atom's layouts, is never printed, and so cannot survive text; +// every mma atom parsed from text has it unset, which is what these types +// model. +// +//===----------------------------------------------------------------------===// + +#ifndef MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUTYPES_TD +#define MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUTYPES_TD + +include "mlir/dialects/cute_nvgpu/CuteNVGPUDialect.td" + +class CuteNVGPU_CustomType + : CuteNVGPU_Type { + let hasCustomAssemblyFormat = 1; +} + +defvar CuteNVGPU_Ty = "::mlir::Type"; +defvar CuteNVGPU_Shape = "::mlir::cutlass_compiler::cute::ShapeAttr"; +defvar CuteNVGPU_Layout = "::mlir::cutlass_compiler::cute::LayoutAttr"; +defvar CuteNVGPU_LayoutTy = "::mlir::cutlass_compiler::cute::LayoutType"; +defvar CuteNVGPU_StrideTy = "::mlir::cutlass_compiler::cute::StrideType"; +defvar CuteNVGPU_Major = "::mlir::cutlass_compiler::cute::MajorMode"; +defvar CuteNVGPU_Space = "::mlir::cutlass_compiler::cute::AddressSpace"; + +//===----------------------------------------------------------------------===// +// Descriptors +//===----------------------------------------------------------------------===// + +def CuteNVGPU_SmemDescType : CuteNVGPU_Type<"SmemDesc", "smem_desc"> { + let summary = "UMMA/GMMA shared-memory descriptor"; + let description = "`!cute_nvgpu.smem_desc`"; + let assemblyFormat = ""; +} + +def CuteNVGPU_SmemDescSM107Type + : CuteNVGPU_Type<"SmemDescSM107", "sm107.smem_desc"> { + let summary = "SM107 shared-memory descriptor"; + let description = "`!cute_nvgpu.sm107.smem_desc`"; + let assemblyFormat = ""; +} + +class CuteNVGPU_SmemDescCircular + : CuteNVGPU_Type { + let summary = "Circular shared-memory descriptor over a block layout in bytes"; + let description = [{ + The block layout is static, has three modes, a 128-byte mode 1 and is + mode-1-major compact, with at least two blocks in mode 2: + + `!cute_nvgpu.}] # typeMnemonic # [{<<"(8,128,2):(128,1,1024)">>` + }]; + let parameters = (ins CuteNVGPU_Layout:$block_layout_bytes); + let assemblyFormat = "`<` $block_layout_bytes `>`"; + let genVerifyDecl = 1; +} + +def CuteNVGPU_SmemDescCircularSM103Type + : CuteNVGPU_SmemDescCircular<"SmemDescCircularSM103", + "sm103.smem_desc_circular">; +def CuteNVGPU_SmemDescCircularSM107Type + : CuteNVGPU_SmemDescCircular<"SmemDescCircularSM107", + "sm107.smem_desc_circular">; + +def CuteNVGPU_SmemDescViewType + : CuteNVGPU_CustomType<"SmemDescView", "smem_desc_view"> { + let summary = "Shared-memory descriptor iterator with a layout"; + let description = [{ + The descriptor is a smem_desc, sm107.smem_desc or smem_desc_circular type; + the layout is a plain layout, as a string. + + `!cute_nvgpu.smem_desc_view` + }]; + let parameters = (ins CuteNVGPU_Ty:$desc, CuteNVGPU_Layout:$layout); + let genVerifyDecl = 1; +} + +def CuteNVGPU_TmaDescriptorTiledType + : CuteNVGPU_Type<"TmaDescriptorTiled", "tma_descriptor_tiled"> { + let summary = "Tiled TMA descriptor"; + let description = "`!cute_nvgpu.tma_descriptor_tiled`"; + let assemblyFormat = ""; +} + +def CuteNVGPU_TmaDescriptorIm2ColType + : CuteNVGPU_Type<"TmaDescriptorIm2Col", "tma_descriptor_im2col"> { + let summary = "Im2col TMA descriptor"; + let description = "`!cute_nvgpu.tma_descriptor_im2col`"; + let assemblyFormat = ""; +} + +def CuteNVGPU_WorkIdResponseType + : CuteNVGPU_Type<"WorkIdResponse", "workid_response"> { + let summary = "Cluster launch control work-id response"; + let description = "`!cute_nvgpu.workid_response`"; + let assemblyFormat = ""; +} + +//===----------------------------------------------------------------------===// +// SIMT copy atoms +//===----------------------------------------------------------------------===// + +def CuteNVGPU_CopyAtomSIMTSyncCopyType + : CuteNVGPU_CustomType<"CopyAtomSIMTSyncCopy", "atom.universal_copy"> { + let summary = "Universal (SIMT) copy atom"; + let description = [{ + `copy_bits` 0 means the element width and is kept as 0; the address + spaces default to generic. Each field needs the previous ones. + + `!cute_nvgpu.atom.universal_copy` + }]; + let parameters = (ins + CuteNVGPU_Ty:$val_type, + DefaultValuedParameter<"int", "0">:$copy_bits, + DefaultValuedParameter:$src_space, + DefaultValuedParameter:$dst_space + ); + let genVerifyDecl = 1; +} + +def CuteNVGPU_CopyAtomSIMTAsyncCopyType + : CuteNVGPU_CustomType<"CopyAtomSIMTAsyncCopy", "atom.simt_async_copy"> { + let summary = "cp.async copy atom"; + let description = [{ + `copy_bits` absent or 0 parses as the element width and is always printed. + + `!cute_nvgpu.atom.simt_async_copy` + }]; + let parameters = (ins + CuteNVGPU_Ty:$val_type, + "LoadCacheMode":$cache, + "int":$copy_bits + ); +} + +def CuteNVGPU_CopyAtomDsmemStoreType + : CuteNVGPU_CustomType<"CopyAtomDsmemStore", "atom.dsmem_store"> { + let summary = "Distributed shared-memory store atom"; + let description = [{ + `copy_bits` absent or 0 parses as the element width and is always printed. + + `!cute_nvgpu.atom.dsmem_store` + }]; + let parameters = (ins CuteNVGPU_Ty:$val_type, "int":$copy_bits); + let genVerifyDecl = 1; +} + +class CuteNVGPU_BulkCopy + : CuteNVGPU_CustomType { + let summary = "Bulk (TMA non-tensor) copy atom"; + let description = [{ + Space-separated groups, printed when not default (`copy_bits` 0). + + `!cute_nvgpu.}] # typeMnemonic # "<" # example # ">`"; + let parameters = !con( + (ins CuteNVGPU_Ty:$val_type, DefaultValuedParameter<"int", "0">:$copy_bits), + !if(!empty(flag), (ins), + !dag(ins, [DefaultValuedParameter<"bool", "false">], [flag]))); +} + +def CuteNVGPU_CopyAtomBulkCopyG2SType + : CuteNVGPU_BulkCopy<"CopyAtomBulkCopyG2S", "atom.bulk_copy_g2s", "mcast", + "f32 copy_bits = 128 mcast = true">; +def CuteNVGPU_CopyAtomBulkCopyS2GType + : CuteNVGPU_BulkCopy<"CopyAtomBulkCopyS2G", "atom.bulk_copy_s2g", "mask", + "f32 copy_bits = 128 mask = true">; +def CuteNVGPU_CopyAtomBulkCopyS2SType + : CuteNVGPU_BulkCopy<"CopyAtomBulkCopyS2S", "atom.bulk_copy_s2s", "", + "f32 copy_bits = 128">; + +// g2r, r2g, r2s, s2r: ``, space separated, +// fields in parameter order and printed when not default; copy_bits absent or +// 0 parses as the element width and is always printed. + +def CuteNVGPU_CopyAtomG2RType + : CuteNVGPU_CustomType<"CopyAtomG2R", "atom.g2r"> { + let summary = "Global-to-register load atom"; + let description = [{ + `!cute_nvgpu.atom.g2r`, + `!cute_nvgpu.atom.g2r` (invariant only with + the default memory features) + }]; + let parameters = (ins + CuteNVGPU_Ty:$val_type, + "int":$copy_bits, + DefaultValuedParameter<"MemOrderKind", "MemOrderKind::WEAK">:$mem_order, + DefaultValuedParameter<"MemScopeKind", "MemScopeKind::CTA">:$mem_scope, + DefaultValuedParameter<"L2PrefetchSize", "L2PrefetchSize::NONE">:$l2_prefetch_size, + DefaultValuedParameter<"CacheEvictionPriority", "CacheEvictionPriority::EVICT_NORMAL">:$l1_cache_evict_priority, + DefaultValuedParameter<"LoadCacheMode", "LoadCacheMode::always">:$load_cache_mode, + DefaultValuedParameter<"SharedSpace", "SharedSpace::CTA">:$shared_space, + DefaultValuedParameter<"bool", "false">:$invariant + ); + let genVerifyDecl = 1; +} + +def CuteNVGPU_CopyAtomR2GType + : CuteNVGPU_CustomType<"CopyAtomR2G", "atom.r2g"> { + let summary = "Register-to-global store atom"; + let description = [{ + `!cute_nvgpu.atom.r2g` + }]; + let parameters = (ins + CuteNVGPU_Ty:$val_type, + "int":$copy_bits, + DefaultValuedParameter<"MemOrderKind", "MemOrderKind::WEAK">:$mem_order, + DefaultValuedParameter<"MemScopeKind", "MemScopeKind::CTA">:$mem_scope, + DefaultValuedParameter<"CacheEvictionPriority", "CacheEvictionPriority::EVICT_NORMAL">:$l1_cache_evict_priority, + DefaultValuedParameter<"StoreCacheMode", "StoreCacheMode::write_back">:$store_cache_mode, + DefaultValuedParameter<"SharedSpace", "SharedSpace::CTA">:$shared_space + ); + let genVerifyDecl = 1; +} + +class CuteNVGPU_SharedCopy + : CuteNVGPU_CustomType { + let summary = what # " atom"; + let description = [{ + `!cute_nvgpu.}] # typeMnemonic # [{` + }]; + let parameters = (ins + CuteNVGPU_Ty:$val_type, + "int":$copy_bits, + DefaultValuedParameter<"MemOrderKind", "MemOrderKind::WEAK">:$mem_order, + DefaultValuedParameter<"MemScopeKind", "MemScopeKind::CTA">:$mem_scope, + DefaultValuedParameter<"SharedSpace", "SharedSpace::CTA">:$shared_space + ); + let genVerifyDecl = 1; +} + +def CuteNVGPU_CopyAtomR2SType + : CuteNVGPU_SharedCopy<"CopyAtomR2S", "atom.r2s", "Register-to-shared store">; +def CuteNVGPU_CopyAtomS2RType + : CuteNVGPU_SharedCopy<"CopyAtomS2R", "atom.s2r", "Shared-to-register load">; + +// Multimem atoms: positional `, ORDER, SCOPE>`; a trailing +// run of defaults is omitted (copy_bits 0, ADD, no accumulation precision, +// relaxed, sys). + +def CuteNVGPU_CopyAtomSIMTMultimemStType + : CuteNVGPU_CustomType<"CopyAtomSIMTMultimemSt", "atom.simt_multimem_st"> { + let summary = "multimem.st atom"; + let description = [{ + `copy_bits` 0 (the element width) is kept as 0. + + `!cute_nvgpu.atom.simt_multimem_st` + }]; + let parameters = (ins + CuteNVGPU_Ty:$val_type, + DefaultValuedParameter<"int", "0">:$copy_bits, + DefaultValuedParameter<"MemOrderKind", "MemOrderKind::RELAXED">:$mem_order, + DefaultValuedParameter<"MemScopeKind", "MemScopeKind::SYS">:$mem_scope + ); +} + +def CuteNVGPU_CopyAtomSIMTMultimemRedType + : CuteNVGPU_CustomType<"CopyAtomSIMTMultimemRed", "atom.simt_multimem_red"> { + let summary = "multimem.red atom"; + let description = [{ + `copy_bits` absent or 0 parses as the element width and is always printed. + + `!cute_nvgpu.atom.simt_multimem_red` + }]; + let parameters = (ins + CuteNVGPU_Ty:$val_type, + "int":$copy_bits, + DefaultValuedParameter<"ReductionKind", "ReductionKind::ADD">:$reduction, + DefaultValuedParameter<"MemOrderKind", "MemOrderKind::RELAXED">:$mem_order, + DefaultValuedParameter<"MemScopeKind", "MemScopeKind::SYS">:$mem_scope + ); +} + +def CuteNVGPU_CopyAtomSIMTMultimemLdReduceType + : CuteNVGPU_CustomType<"CopyAtomSIMTMultimemLdReduce", + "atom.simt_multimem_ld_reduce"> { + let summary = "multimem.ld_reduce atom"; + let description = [{ + `copy_bits` 0 (the element width) is kept as 0. The accumulation + precision must be given to give a memory order or scope. + + `!cute_nvgpu.atom.simt_multimem_ld_reduce, acquire, gpu>` + }]; + let parameters = (ins + CuteNVGPU_Ty:$val_type, + DefaultValuedParameter<"int", "0">:$copy_bits, + DefaultValuedParameter<"ReductionKind", "ReductionKind::ADD">:$reduction, + OptionalParameter<"LdReduceAccPrecisionKindAttr">:$ld_reduce_acc_precision, + DefaultValuedParameter<"MemOrderKind", "MemOrderKind::RELAXED">:$mem_order, + DefaultValuedParameter<"MemScopeKind", "MemScopeKind::SYS">:$mem_scope + ); +} + +//===----------------------------------------------------------------------===// +// Tensor-memory copy atoms: `` +//===----------------------------------------------------------------------===// + +class CuteNVGPU_TmemCopy + : CuteNVGPU_CustomType { + let summary = what; + let description = [{ + `!cute_nvgpu.}] # typeMnemonic # "`"; + let parameters = !con( + (ins CuteNVGPU_Ty:$val_type, "int":$num_dp, "int":$num_bit, + "unsigned":$num_rep), + !dag(ins, [OptionalParameter<"::mlir::UnitAttr">], [flag])); + let genVerifyDecl = 1; +} + +def CuteNVGPU_CopyAtomSM100TmemLoadType + : CuteNVGPU_TmemCopy<"CopyAtomSM100TmemLoad", "atom.tmem_load", + "tcgen05.ld atom", "pack16b">; +def CuteNVGPU_CopyAtomSM100TmemStoreType + : CuteNVGPU_TmemCopy<"CopyAtomSM100TmemStore", "atom.tmem_store", + "tcgen05.st atom", "expand16b">; + +def CuteNVGPU_CopyAtomSM10xTmemLoadRedType + : CuteNVGPU_CustomType<"CopyAtomSM10xTmemLoadRed", "atom.tmem_load_red"> { + let summary = "tcgen05.ld.red atom"; + let description = [{ + `half_split_off` prints as an integer attribute without spaces around `=`. + + `!cute_nvgpu.atom.tmem_load_red` + }]; + let parameters = (ins + CuteNVGPU_Ty:$val_type, + "int":$num_dp, + "int":$num_bit, + "unsigned":$num_rep, + "TmemLoadRedOp":$red_op, + OptionalParameter<"::mlir::UnitAttr">:$nan, + OptionalParameter<"::mlir::IntegerAttr">:$half_split_off + ); + let genVerifyDecl = 1; +} + +def CuteNVGPU_CopyAtomSM107TmemLoadSPCompressType + : CuteNVGPU_CustomType<"CopyAtomSM107TmemLoadSPCompress", + "atom.tmem_load_spcompress"> { + let summary = "SM107 tensor-memory load with sparse compression"; + let description = [{ + `!cute_nvgpu.atom.tmem_load_spcompress` + }]; + let parameters = (ins + CuteNVGPU_Ty:$val_type, + "int":$num_dp, + "int":$num_bit, + "unsigned":$num_rep, + "TmemLoadRedOp":$red_op, + OptionalParameter<"::mlir::UnitAttr">:$red, + OptionalParameter<"::mlir::UnitAttr">:$nan + ); + let genVerifyDecl = 1; +} + +def CuteNVGPU_CopyAtomSM100CopyS2TType + : CuteNVGPU_CustomType<"CopyAtomSM100CopyS2T", "atom.s2t_copy"> { + let summary = "tcgen05.cp (shared to tensor memory) atom"; + let description = [{ + The broadcast mode `none` is omitted. + + `!cute_nvgpu.atom.s2t_copy` + }]; + let parameters = (ins + CuteNVGPU_Ty:$val_type, + "int":$num_dp, + "int":$num_bit, + "int":$num_cta, + DefaultValuedParameter<"CopyS2TBroadcast", "CopyS2TBroadcast::none">:$broadcast + ); + let genVerifyDecl = 1; +} + +def CuteNVGPU_CopyAtomSM100S2TCopyV2Type + : CuteNVGPU_CustomType<"CopyAtomSM100S2TCopyV2", "atom.sm100_s2t_copy_v2"> { + let summary = "tcgen05.cp atom with the shared-memory major mode"; + let description = [{ + The broadcast mode `none` is omitted. + + `!cute_nvgpu.atom.sm100_s2t_copy_v2` + }]; + let parameters = (ins + CuteNVGPU_Ty:$val_type, + "int":$num_dp, + "int":$num_bit, + "int":$num_cta, + CuteNVGPU_Major:$smem_major, + DefaultValuedParameter<"CopyS2TBroadcast", "CopyS2TBroadcast::none">:$broadcast + ); + let genVerifyDecl = 1; +} + +//===----------------------------------------------------------------------===// +// ldmatrix / stmatrix +//===----------------------------------------------------------------------===// + +def CuteNVGPU_CopyAtomLdsmType + : CuteNVGPU_CustomType<"CopyAtomLdsm", "atom.ldsm"> { + let summary = "ldmatrix atom"; + let description = [{ + The last field is `t` (transposed) or `n`. + + `!cute_nvgpu.atom.ldsm, sz_pattern = u16, num_matrices = 4, t>` + }]; + let parameters = (ins + CuteNVGPU_Ty:$val_type, + CuteNVGPU_Shape:$mode, + "LdsmSzPattern":$sz_pattern, + "int":$num_matrices, + OptionalParameter<"::mlir::UnitAttr">:$transpose + ); +} + +def CuteNVGPU_CopyAtomStsmType + : CuteNVGPU_CustomType<"CopyAtomStsm", "atom.stsm"> { + let summary = "stmatrix atom"; + let description = [{ + The last field is `t` (transposed) or `n`. + + `!cute_nvgpu.atom.stsm, num_matrices = 4, n>` + }]; + let parameters = (ins + CuteNVGPU_Ty:$val_type, + CuteNVGPU_Shape:$mode, + "int":$num_matrices, + OptionalParameter<"::mlir::UnitAttr">:$transpose + ); +} + +//===----------------------------------------------------------------------===// +// TMA atoms +//===----------------------------------------------------------------------===// + +// Executable: comma-separated fields up to g_stride, then space-separated +// groups printed when set (booleans when true, sparsity when not 1). + +def CuteNVGPU_CopyAtomTmaLoadType + : CuteNVGPU_CustomType<"CopyAtomTmaLoad", "atom.tma_load"> { + let summary = "Executable TMA load atom"; + let description = [{ + `!cute_nvgpu.atom.tma_load mcast = true tma_gbasis = <"(64,1):(1@0,1@1)">>` + }]; + let parameters = (ins + CuteNVGPU_Ty:$val_type, + DefaultValuedParameter<"int", "1">:$sparsity, + "int":$copy_bits, + "TmaLoadMode":$mode, + "int":$num_cta, + CuteNVGPU_StrideTy:$g_stride, + DefaultValuedParameter<"bool", "false">:$mcast, + OptionalParameter:$tma_gbasis, + DefaultValuedParameter<"bool", "false">:$override, + DefaultValuedParameter<"bool", "false">:$no_fully_oob_tile + ); + let genVerifyDecl = 1; +} + +def CuteNVGPU_CopyAtomTmaStoreType + : CuteNVGPU_CustomType<"CopyAtomTmaStore", "atom.tma_store"> { + let summary = "Executable TMA store atom"; + let description = [{ + `!cute_nvgpu.atom.tma_store tma_gbasis = <"(64,16):(1@0,1@1)">>` + }]; + let parameters = (ins + CuteNVGPU_Ty:$val_type, + DefaultValuedParameter<"int", "1">:$sparsity, + "int":$copy_bits, + "TmaStoreMode":$mode, + CuteNVGPU_StrideTy:$g_stride, + OptionalParameter:$tma_gbasis, + DefaultValuedParameter<"bool", "false">:$override, + DefaultValuedParameter<"bool", "false">:$no_fully_oob_tile + ); + let genVerifyDecl = 1; +} + +def CuteNVGPU_CopyAtomTmaReduceType + : CuteNVGPU_CustomType<"CopyAtomTmaReduce", "atom.tma_reduce"> { + let summary = "Executable TMA reduce atom"; + let description = [{ + `!cute_nvgpu.atom.tma_reduce tma_gbasis = <"(32,128):(1@1,1@0)">>` + }]; + let parameters = (ins + CuteNVGPU_Ty:$val_type, + "int":$copy_bits, + "TmaStoreMode":$mode, + "ReductionKind":$kind, + CuteNVGPU_StrideTy:$g_stride, + OptionalParameter:$tma_gbasis + ); +} + +def CuteNVGPU_CopyAtomIm2ColTmaLoadType + : CuteNVGPU_CustomType<"CopyAtomIm2ColTmaLoad", "atom.im2col_tma_load"> { + let summary = "Executable im2col TMA load atom"; + let description = [{ + `!cute_nvgpu.atom.im2col_tma_load mcast = true tma_gbasis = <"(64,128):(1@1,1@0)">>` + }]; + let parameters = (ins + CuteNVGPU_Ty:$val_type, + "int":$copy_bits, + "int":$num_cta, + CuteNVGPU_StrideTy:$g_stride, + DefaultValuedParameter<"bool", "false">:$mcast, + OptionalParameter:$tma_gbasis + ); +} + +def CuteNVGPU_CopyAtomIm2ColTmaStoreType + : CuteNVGPU_CustomType<"CopyAtomIm2ColTmaStore", "atom.im2col_tma_store"> { + let summary = "Executable im2col TMA store atom"; + let description = [{ + `!cute_nvgpu.atom.im2col_tma_store tma_gbasis = <"(32,128):(1@1,1@0)">>` + }]; + let parameters = (ins + CuteNVGPU_Ty:$val_type, + "int":$copy_bits, + CuteNVGPU_StrideTy:$g_stride, + OptionalParameter:$tma_gbasis + ); +} + +// Non-executable: `<[kind, ]T, copy_bits = N, tma_gbasis = <"L">[, tma_format = F]>`. +// An omitted format is the element type's (F16_RN, BF16_RN, F32_RN, TF32_RN, +// F64_RN, U4 for 4-bit, U8 up to 8 bits, U16, U32/S32, U64/S64); it is always +// printed, and empty when the element type has none. + +class CuteNVGPU_NonExecTma + : CuteNVGPU_CustomType { + let summary = "Non-executable " # what # " atom"; + let description = "`!cute_nvgpu." # typeMnemonic # "<" # example # ">`"; + defvar common = (ins + CuteNVGPU_Ty:$val_type, + "int":$copy_bits, + CuteNVGPU_LayoutTy:$tma_gbasis, + "std::optional":$tma_format + ); + let parameters = !if(!empty(kindType), common, + !con((ins kindType:$kind), common)); + let genVerifyDecl = 1; +} + +def CuteNVGPU_CopyAtomNonExecTiledTmaLoadType + : CuteNVGPU_NonExecTma<"CopyAtomNonExecTiledTmaLoad", + "atom.non_exec_tiled_tma_load", "tiled TMA load", "TiledTmaLoad", + "sm_90, bf16, copy_bits = 1024, tma_gbasis = <\"(64,1):(1@0,1@1)\">, tma_format = BF16_RN">; +def CuteNVGPU_CopyAtomNonExecTiledTmaStoreType + : CuteNVGPU_NonExecTma<"CopyAtomNonExecTiledTmaStore", + "atom.non_exec_tiled_tma_store", "tiled TMA store", "", + "bf16, copy_bits = 1024, tma_gbasis = <\"(64,1):(1@0,1@1)\">, tma_format = BF16_RN">; +def CuteNVGPU_CopyAtomNonExecTiledTmaReduceType + : CuteNVGPU_NonExecTma<"CopyAtomNonExecTiledTmaReduce", + "atom.non_exec_tiled_tma_reduce", "tiled TMA reduce", "ReductionKind", + "ADD, f32, copy_bits = 1024, tma_gbasis = <\"(64,1):(1@0,1@1)\">, tma_format = F32_RN">; +def CuteNVGPU_CopyAtomNonExecIm2ColTmaLoadType + : CuteNVGPU_NonExecTma<"CopyAtomNonExecIm2ColTmaLoad", + "atom.non_exec_im2col_tma_load", "im2col TMA load", "Im2ColTmaLoad", + "sm_90, f16, copy_bits = 1024, tma_gbasis = <\"(64,1):(1@0,1@1)\">, tma_format = F16_RN">; +def CuteNVGPU_CopyAtomNonExecIm2ColTmaStoreType + : CuteNVGPU_NonExecTma<"CopyAtomNonExecIm2ColTmaStore", + "atom.non_exec_im2col_tma_store", "im2col TMA store", "", + "f16, copy_bits = 1024, tma_gbasis = <\"(64,1):(1@0,1@1)\">, tma_format = F16_RN">; +def CuteNVGPU_CopyAtomNonExec2DGather4TmaLoadType + : CuteNVGPU_NonExecTma<"CopyAtomNonExec2DGather4TmaLoad", + "atom.non_exec_2d_gather4_tma_load", "2-D gather4 TMA load", + "GatherScatterTmaLoad", + "sm_100, f16, copy_bits = 1024, tma_gbasis = <\"(64,1):(1@0,1@1)\">, tma_format = F16_RN">; +def CuteNVGPU_CopyAtomNonExec2DScatter4TmaStoreType + : CuteNVGPU_NonExecTma<"CopyAtomNonExec2DScatter4TmaStore", + "atom.non_exec_2d_scatter4_tma_store", "2-D scatter4 TMA store", "", + "f16, copy_bits = 1024, tma_gbasis = <\"(4,64):(1@0,1@1)\">, tma_format = F16_RN">; + +//===----------------------------------------------------------------------===// +// MMA atoms. The shape is `MxNxK` (a static rank-3 shape); `elem_type` lists +// A, B and C; `frag_kind` is the A fragment kind: `ss` (smem_desc), `rs` +// (rmem, sm90) or `ts` (tmem, sm100 and sm107). +//===----------------------------------------------------------------------===// + +class CuteNVGPU_MmaAtom + : CuteNVGPU_CustomType { + let summary = what # " MMA atom"; + let description = "`!cute_nvgpu." # typeMnemonic # "<" # example # ">`"; + let genVerifyDecl = 1; +} + +def CuteNVGPU_UniversalFmaAtomType + : CuteNVGPU_MmaAtom<"UniversalFmaAtom", "atom.universal_fma", "Universal FMA", + "1x1x1, (f32, f32) -> f32 "> { + let parameters = (ins CuteNVGPU_Shape:$shape_mnk, CuteNVGPU_Ty:$a_type, + CuteNVGPU_Ty:$b_type, CuteNVGPU_Ty:$c_type); +} + +def CuteNVGPU_MmaAtomSM80Type + : CuteNVGPU_MmaAtom<"MmaAtomSM80", "sm80.mma", "SM80 mma.sync", + "16x8x32, elem_type = (i8, i8, i32), int_overflow = "> { + let description = [{ + `binary_op` needs `int_overflow` before it. + + `!cute_nvgpu.sm80.mma<16x8x32, elem_type = (i8, i8, i32), int_overflow = , binary_op = >` + }]; + let parameters = (ins + CuteNVGPU_Shape:$shape_mnk, + CuteNVGPU_Ty:$a_type, CuteNVGPU_Ty:$b_type, CuteNVGPU_Ty:$c_type, + OptionalParameter<"MMAIntOverflowAttr">:$int_overflow, + OptionalParameter<"BinaryOpAttr">:$binary_op + ); +} + +def CuteNVGPU_MmaAtomSM80SparseType + : CuteNVGPU_MmaAtom<"MmaAtomSM80Sparse", "sm80.sparse_mma", + "SM80 sparse mma.sync", + "16x8x64, elem_type = (i8, i8, i32), md_format = tid, int_overflow = "> { + let parameters = (ins + CuteNVGPU_Shape:$shape_mnk, + CuteNVGPU_Ty:$a_type, CuteNVGPU_Ty:$b_type, CuteNVGPU_Ty:$c_type, + "SparseMetadataFormat":$sparse_metadata_format, + OptionalParameter<"MMAIntOverflowAttr">:$int_overflow + ); +} + +def CuteNVGPU_MmaAtomSM89Type + : CuteNVGPU_MmaAtom<"MmaAtomSM89", "sm89.mma", "SM89 fp8 mma.sync", + "16x8x32, elem_type = (f8E4M3FN, f8E4M3FN, f32) "> { + let parameters = (ins CuteNVGPU_Shape:$shape_mnk, CuteNVGPU_Ty:$a_type, + CuteNVGPU_Ty:$b_type, CuteNVGPU_Ty:$c_type); +} + +def CuteNVGPU_MmaAtomSM90Type + : CuteNVGPU_MmaAtom<"MmaAtomSM90", "sm90.mma", "SM90 wgmma", + "64x128x16, ab_major = (k, mn), elem_type = (f16, f16, f32), frag_kind = rs, a_neg, b_neg"> { + let description = [{ + The trailing `int_overflow`, `a_neg` and `b_neg` parse in any order and + print in this one. + + `!cute_nvgpu.sm90.mma<64x128x32, ab_major = (k, k), elem_type = (i8, i8, i32), frag_kind = ss, int_overflow = , a_neg, b_neg>` + }]; + let parameters = (ins + CuteNVGPU_Shape:$shape_mnk, + CuteNVGPU_Major:$a_major, CuteNVGPU_Major:$b_major, + CuteNVGPU_Ty:$a_type, CuteNVGPU_Ty:$b_type, CuteNVGPU_Ty:$c_type, + "MmaFragKind":$a_frag_kind, + OptionalParameter<"MMAIntOverflowAttr">:$int_overflow, + OptionalParameter<"::mlir::UnitAttr">:$a_neg, + OptionalParameter<"::mlir::UnitAttr">:$b_neg + ); +} + +def CuteNVGPU_MmaAtomSM120BlockScaledType + : CuteNVGPU_MmaAtom<"MmaAtomSM120BlockScaled", "SM120.mma_bs", + "SM120 block-scaled mma.sync", + "16x8x64, vec_size = 32, elem_type = (f4E2M1FN, f4E2M1FN, f32), sf_type = f8E8M0FNU, use_sf_layout_TV = true"> { + let parameters = (ins + CuteNVGPU_Shape:$shape_mnk, + "int":$vec_size, + CuteNVGPU_Ty:$a_type, CuteNVGPU_Ty:$b_type, CuteNVGPU_Ty:$c_type, + CuteNVGPU_Ty:$sf_type, + "bool":$use_sf_layout_TV + ); +} + +// UMMA (tcgen05.mma): `]>`. arch_promote (sm100 +// block-scaled only) defaults to sm_100 and is omitted then; the sm107 atoms +// require ab_collector_op. + +class CuteNVGPU_Umma + : CuteNVGPU_MmaAtom { + let parameters = !con( + (ins CuteNVGPU_Shape:$shape_mnk, "int":$num_cta, + CuteNVGPU_Major:$a_major, CuteNVGPU_Major:$b_major, + CuteNVGPU_Ty:$a_type, CuteNVGPU_Ty:$b_type, CuteNVGPU_Ty:$c_type), + !if(sf, (ins CuteNVGPU_Ty:$sf_type), (ins)), + !if(!and(sp, !not(sf)), (ins CuteNVGPU_Ty:$e_type), (ins)), + !if(sp, (ins "SparseMetadataFormat":$sparse_metadata_format), (ins)), + (ins "MmaFragKind":$a_frag_kind), + !if(bs, (ins "int":$vec_size), (ins "int":$c_scale_exp)), + !if(!and(bs, !not(sm107)), + (ins DefaultValuedParameter<"Arch", "Arch::sm_100">:$arch_promote), (ins)), + !if(sm107, (ins "MmaCollectorOp":$a_collector_op, + "MmaCollectorOp":$b_collector_op), (ins)), + (ins OptionalParameter<"MMAIntOverflowAttr">:$int_overflow)); +} + +def CuteNVGPU_MmaAtomSM100UMMAType + : CuteNVGPU_Umma<"MmaAtomSM100UMMA", "sm100.mma", "SM100 UMMA", + "128x128x16, num_cta = 1, ab_major = (k, mn), elem_type = (f16, f16, f32), frag_kind = ss, c_scale_exp = 0", + 0, 0, 0, 0>; +def CuteNVGPU_MmaAtomSM100UMMASparseType + : CuteNVGPU_Umma<"MmaAtomSM100UMMASparse", "sm100.mma_sp", "SM100 sparse UMMA", + "128x128x32, num_cta = 1, ab_major = (k, k), elem_type = (f16, f16, f32), e_type = i8, sparse_metadata_format = tid, frag_kind = ss, c_scale_exp = 0", + 0, 1, 0, 0>; +def CuteNVGPU_MmaAtomSM100UMMABlockScaledType + : CuteNVGPU_Umma<"MmaAtomSM100UMMABlockScaled", "sm100.mma_bs", + "SM100 block-scaled UMMA", + "128x128x96, num_cta = 1, ab_major = (k, k), elem_type = (f4E2M1FN, f4E2M1FN, f32), sf_type = f8E8M0FNU, frag_kind = ss, vec_size = 16, arch_promote = sm_103", + 1, 0, 1, 0>; +def CuteNVGPU_MmaAtomSM100UMMABlockScaledSparseType + : CuteNVGPU_Umma<"MmaAtomSM100UMMABlockScaledSparse", "sm100.mma_bs_sp", + "SM100 block-scaled sparse UMMA", + "128x128x64, num_cta = 1, ab_major = (k, k), elem_type = (f8E4M3FN, f8E4M3FN, f32), sf_type = f8E8M0FNU, sparse_metadata_format = tid, frag_kind = ss, vec_size = 32", + 1, 1, 1, 0>; +def CuteNVGPU_MmaAtomSM107UMMAType + : CuteNVGPU_Umma<"MmaAtomSM107UMMA", "sm107.mma", "SM107 UMMA", + "128x128x16, num_cta = 1, ab_major = (k, mn), elem_type = (f16, f16, f32), frag_kind = ss, c_scale_exp = 0, ab_collector_op = (lastuse, fill)", + 0, 0, 0, 1>; +def CuteNVGPU_MmaAtomSM107UMMASparseType + : CuteNVGPU_Umma<"MmaAtomSM107UMMASparse", "sm107.mma_sp", "SM107 sparse UMMA", + "128x128x32, num_cta = 1, ab_major = (k, k), elem_type = (f16, f16, f32), e_type = i8, sparse_metadata_format = tid, frag_kind = ss, c_scale_exp = 0, ab_collector_op = (discard, discard)", + 0, 1, 0, 1>; +def CuteNVGPU_MmaAtomSM107UMMABlockScaledType + : CuteNVGPU_Umma<"MmaAtomSM107UMMABlockScaled", "sm107.mma_bs", + "SM107 block-scaled UMMA", + "128x128x32, num_cta = 1, ab_major = (k, k), elem_type = (f8E4M3FN, f8E4M3FN, f32), sf_type = f8E8M0FNU, frag_kind = ss, vec_size = 32, ab_collector_op = (discard, use), int_overflow = ", + 1, 0, 1, 1>; +def CuteNVGPU_MmaAtomSM107UMMABlockScaledSparseType + : CuteNVGPU_Umma<"MmaAtomSM107UMMABlockScaledSparse", "sm107.mma_bs_sp", + "SM107 block-scaled sparse UMMA", + "128x128x64, num_cta = 1, ab_major = (k, k), elem_type = (f8E4M3FN, f8E4M3FN, f32), sf_type = f8E8M0FNU, sparse_metadata_format = tid, frag_kind = ss, vec_size = 32, ab_collector_op = (discard, discard)", + 1, 1, 1, 1>; + +#endif // MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUTYPES_TD diff --git a/mlir/dialects/cute_nvgpu/capi.cc b/mlir/dialects/cute_nvgpu/capi.cc new file mode 100644 index 00000000..73d55bc3 --- /dev/null +++ b/mlir/dialects/cute_nvgpu/capi.cc @@ -0,0 +1,4429 @@ +//===- capi.cc - C API for the cute_nvgpu dialect -------------------------===// + +#include "capi.h" + +#include "mlir/dialects/cute_nvgpu/CuteNVGPUDialect.h" + +#include "mlir/CAPI/IR.h" +#include "mlir/CAPI/Registration.h" + +// Registration + +MLIR_DEFINE_CAPI_DIALECT_REGISTRATION( + CuteNVGPU, cute_nvgpu, mlir::cutlass_compiler::cute_nvgpu::CuteNVGPUDialect) + +// Attributes + +namespace cute_nvgpu = mlir::cutlass_compiler::cute_nvgpu; + +// Unwraps `handle` into `out`: false when it is of another context or not a +// T, or when it is null and not optional. +template +static bool unwrapAs(mlir::MLIRContext *ctx, H handle, bool optional, T &out) { + auto value = unwrap(handle); + if (!value) + return optional; + if (value.getContext() != ctx) + return false; + out = llvm::dyn_cast(value); + return static_cast(out); +} + +bool mlirAttributeIsACuteNVGPUAtomCopyFieldBulkCopyG2S(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUAtomCopyFieldBulkCopyG2SAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomCopyFieldBulkCopyG2S( + value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomCopyFieldBulkCopyG2SAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomCopyFieldBulkCopyG2SAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomCopyFieldBulkCopyS2G(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUAtomCopyFieldBulkCopyS2GAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomCopyFieldBulkCopyS2G( + value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomCopyFieldBulkCopyS2GAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomCopyFieldBulkCopyS2GAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomCopyFieldBulkCopyS2S(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUAtomCopyFieldBulkCopyS2SAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomCopyFieldBulkCopyS2S( + value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomCopyFieldBulkCopyS2SAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomCopyFieldBulkCopyS2SAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomCopyFieldDsmemStore(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUAtomCopyFieldDsmemStoreAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomCopyFieldDsmemStore( + value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomCopyFieldDsmemStoreAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomCopyFieldDsmemStoreAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomCopyFieldLoadGlobal(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUAtomCopyFieldLoadGlobalAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomCopyFieldLoadGlobal( + value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomCopyFieldLoadGlobalAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomCopyFieldLoadGlobalAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomCopyFieldNonExec2DGather4TmaLoad( + MlirAttribute attr) { + return llvm::isa_and_nonnull< + cute_nvgpu::AtomCopyFieldNonExec2DGather4TmaLoadAttr>(unwrap(attr)); +} +MlirAttribute +mlirCuteNVGPUAtomCopyFieldNonExec2DGather4TmaLoadAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::cute_nvgpu:: + symbolizeAtomCopyFieldNonExec2DGather4TmaLoad(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomCopyFieldNonExec2DGather4TmaLoadAttr::get( + ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomCopyFieldNonExec2DGather4TmaLoadAttrGetValue( + MlirAttribute attr) { + return static_cast( + llvm::cast( + unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomCopyFieldNonExec2DScatter4TmaStore( + MlirAttribute attr) { + return llvm::isa_and_nonnull< + cute_nvgpu::AtomCopyFieldNonExec2DScatter4TmaStoreAttr>(unwrap(attr)); +} +MlirAttribute +mlirCuteNVGPUAtomCopyFieldNonExec2DScatter4TmaStoreAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::cute_nvgpu:: + symbolizeAtomCopyFieldNonExec2DScatter4TmaStore(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomCopyFieldNonExec2DScatter4TmaStoreAttr::get( + ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomCopyFieldNonExec2DScatter4TmaStoreAttrGetValue( + MlirAttribute attr) { + return static_cast( + llvm::cast( + unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecIm2ColTmaLoad( + MlirAttribute attr) { + return llvm::isa_and_nonnull< + cute_nvgpu::AtomCopyFieldNonExecIm2ColTmaLoadAttr>(unwrap(attr)); +} +MlirAttribute +mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaLoadAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::cute_nvgpu:: + symbolizeAtomCopyFieldNonExecIm2ColTmaLoad(value); + if (!valueValue) + return {nullptr}; + return wrap( + cute_nvgpu::AtomCopyFieldNonExecIm2ColTmaLoadAttr::get(ctx, *valueValue)); +} +uint32_t +mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaLoadAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast( + unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecIm2ColTmaStore( + MlirAttribute attr) { + return llvm::isa_and_nonnull< + cute_nvgpu::AtomCopyFieldNonExecIm2ColTmaStoreAttr>(unwrap(attr)); +} +MlirAttribute +mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaStoreAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::cute_nvgpu:: + symbolizeAtomCopyFieldNonExecIm2ColTmaStore(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomCopyFieldNonExecIm2ColTmaStoreAttr::get( + ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaStoreAttrGetValue( + MlirAttribute attr) { + return static_cast( + llvm::cast( + unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecTiledTmaLoad( + MlirAttribute attr) { + return llvm::isa_and_nonnull< + cute_nvgpu::AtomCopyFieldNonExecTiledTmaLoadAttr>(unwrap(attr)); +} +MlirAttribute +mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaLoadAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::cute_nvgpu:: + symbolizeAtomCopyFieldNonExecTiledTmaLoad(value); + if (!valueValue) + return {nullptr}; + return wrap( + cute_nvgpu::AtomCopyFieldNonExecTiledTmaLoadAttr::get(ctx, *valueValue)); +} +uint32_t +mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaLoadAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecTiledTmaReduce( + MlirAttribute attr) { + return llvm::isa_and_nonnull< + cute_nvgpu::AtomCopyFieldNonExecTiledTmaReduceAttr>(unwrap(attr)); +} +MlirAttribute +mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaReduceAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::cute_nvgpu:: + symbolizeAtomCopyFieldNonExecTiledTmaReduce(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomCopyFieldNonExecTiledTmaReduceAttr::get( + ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaReduceAttrGetValue( + MlirAttribute attr) { + return static_cast( + llvm::cast( + unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecTiledTmaStore( + MlirAttribute attr) { + return llvm::isa_and_nonnull< + cute_nvgpu::AtomCopyFieldNonExecTiledTmaStoreAttr>(unwrap(attr)); +} +MlirAttribute +mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaStoreAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::cute_nvgpu:: + symbolizeAtomCopyFieldNonExecTiledTmaStore(value); + if (!valueValue) + return {nullptr}; + return wrap( + cute_nvgpu::AtomCopyFieldNonExecTiledTmaStoreAttr::get(ctx, *valueValue)); +} +uint32_t +mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaStoreAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast( + unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomCopyFieldStoreGlobal(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUAtomCopyFieldStoreGlobalAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomCopyFieldStoreGlobal( + value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomCopyFieldStoreGlobalAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomCopyFieldStoreGlobalAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomCopyFieldTmaLoad(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUAtomCopyFieldTmaLoadAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomCopyFieldTmaLoad( + value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomCopyFieldTmaLoadAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomCopyFieldTmaLoadAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomCopyFieldTmaReduce(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUAtomCopyFieldTmaReduceAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomCopyFieldTmaReduce( + value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomCopyFieldTmaReduceAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomCopyFieldTmaReduceAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomCopyFieldTmaStore(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUAtomCopyFieldTmaStoreAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomCopyFieldTmaStore( + value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomCopyFieldTmaStoreAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomCopyFieldTmaStoreAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomMmaFieldSM100(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUAtomMmaFieldSM100AttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomMmaFieldSM100(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomMmaFieldSM100Attr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomMmaFieldSM100AttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomMmaFieldSM100BlockScaled(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute +mlirCuteNVGPUAtomMmaFieldSM100BlockScaledAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::cute_nvgpu:: + symbolizeAtomMmaFieldSM100BlockScaled(value); + if (!valueValue) + return {nullptr}; + return wrap( + cute_nvgpu::AtomMmaFieldSM100BlockScaledAttr::get(ctx, *valueValue)); +} +uint32_t +mlirCuteNVGPUAtomMmaFieldSM100BlockScaledAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomMmaFieldSM100BlockScaledSparse( + MlirAttribute attr) { + return llvm::isa_and_nonnull< + cute_nvgpu::AtomMmaFieldSM100BlockScaledSparseAttr>(unwrap(attr)); +} +MlirAttribute +mlirCuteNVGPUAtomMmaFieldSM100BlockScaledSparseAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::cute_nvgpu:: + symbolizeAtomMmaFieldSM100BlockScaledSparse(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomMmaFieldSM100BlockScaledSparseAttr::get( + ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomMmaFieldSM100BlockScaledSparseAttrGetValue( + MlirAttribute attr) { + return static_cast( + llvm::cast( + unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomMmaFieldSM100Sparse(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUAtomMmaFieldSM100SparseAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomMmaFieldSM100Sparse( + value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomMmaFieldSM100SparseAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomMmaFieldSM100SparseAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomMmaFieldSM120BlockScaled(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute +mlirCuteNVGPUAtomMmaFieldSM120BlockScaledAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::cute_nvgpu:: + symbolizeAtomMmaFieldSM120BlockScaled(value); + if (!valueValue) + return {nullptr}; + return wrap( + cute_nvgpu::AtomMmaFieldSM120BlockScaledAttr::get(ctx, *valueValue)); +} +uint32_t +mlirCuteNVGPUAtomMmaFieldSM120BlockScaledAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomMmaFieldSM80Sparse(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUAtomMmaFieldSM80SparseAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomMmaFieldSM80Sparse( + value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomMmaFieldSM80SparseAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomMmaFieldSM80SparseAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUAtomMmaFieldSM90(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUAtomMmaFieldSM90AttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomMmaFieldSM90(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::AtomMmaFieldSM90Attr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUAtomMmaFieldSM90AttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPUBinaryOp(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUBinaryOpAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeBinaryOp(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::BinaryOpAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUBinaryOpAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPUCopyS2TBroadcast(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUCopyS2TBroadcastAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeCopyS2TBroadcast(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::CopyS2TBroadcastAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUCopyS2TBroadcastAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPUGatherScatterTmaLoad(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUGatherScatterTmaLoadAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeGatherScatterTmaLoad( + value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::GatherScatterTmaLoadAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUGatherScatterTmaLoadAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPUIm2ColTmaLoad(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUIm2ColTmaLoadAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeIm2ColTmaLoad(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::Im2ColTmaLoadAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUIm2ColTmaLoadAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPULdReduceAccPrecisionKind(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute mlirCuteNVGPULdReduceAccPrecisionKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeLdReduceAccPrecisionKind( + value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::LdReduceAccPrecisionKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPULdReduceAccPrecisionKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVGPULdsmSzPattern(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPULdsmSzPatternAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeLdsmSzPattern(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::LdsmSzPatternAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPULdsmSzPatternAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPULoadCacheMode(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPULoadCacheModeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeLoadCacheMode(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::LoadCacheModeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPULoadCacheModeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPUMMAIntOverflow(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUMMAIntOverflowAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMMAIntOverflow(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::MMAIntOverflowAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUMMAIntOverflowAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPUMajorMode(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUMajorModeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::MajorModeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUMajorModeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPUMmaCollectorOp(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUMmaCollectorOpAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaCollectorOp(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::MmaCollectorOpAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUMmaCollectorOpAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPUMmaFragKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUMmaFragKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaFragKind(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::MmaFragKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUMmaFragKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPUNotImplementedFrg(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUNotImplementedFrgAttrGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cute_nvgpu::NotImplementedFrgAttr::get(ctx)); +} + +bool mlirAttributeIsACuteNVGPUReductionKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUReductionKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeReductionKind(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::ReductionKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUReductionKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPURmemFrg(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPURmemFrgAttrGet(MlirContext context, + MlirType valueType, uint32_t operand, + bool deriveElemType) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valueTypeValue; + if (!unwrapAs(ctx, valueType, false, valueTypeValue)) + return {nullptr}; + auto operandValue = + ::mlir::cutlass_compiler::cute::symbolizeMmaOperand(operand); + if (!operandValue) + return {nullptr}; + return wrap(cute_nvgpu::RmemFrgAttr::get(ctx, valueTypeValue, *operandValue, + deriveElemType)); +} +MlirType mlirCuteNVGPURmemFrgAttrGetValueType(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getValueType())); +} +uint32_t mlirCuteNVGPURmemFrgAttrGetOperand(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getOperand()); +} +bool mlirCuteNVGPURmemFrgAttrGetDeriveElemType(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getDeriveElemType(); +} + +bool mlirAttributeIsACuteNVGPUSM100CircularSmemFrg(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUSM100CircularSmemFrgAttrGet(MlirContext context, + uint32_t majorMode) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto majorModeValue = + ::mlir::cutlass_compiler::cute::symbolizeMajorMode(majorMode); + if (!majorModeValue) + return {nullptr}; + return wrap(cute_nvgpu::SM100CircularSmemFrgAttr::get(ctx, *majorModeValue)); +} +uint32_t mlirCuteNVGPUSM100CircularSmemFrgAttrGetMajorMode(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getMajorMode()); +} + +bool mlirAttributeIsACuteNVGPUSM100SmemFrg(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUSM100SmemFrgAttrGet(MlirContext context, + uint32_t majorMode) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto majorModeValue = + ::mlir::cutlass_compiler::cute::symbolizeMajorMode(majorMode); + if (!majorModeValue) + return {nullptr}; + return wrap(cute_nvgpu::SM100SmemFrgAttr::get(ctx, *majorModeValue)); +} +uint32_t mlirCuteNVGPUSM100SmemFrgAttrGetMajorMode(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getMajorMode()); +} + +bool mlirAttributeIsACuteNVGPUSM100TmemEFrg(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUSM100TmemEFrgAttrGet(MlirContext context, + MlirType aType, + MlirType eType) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type aTypeValue; + if (!unwrapAs(ctx, aType, false, aTypeValue)) + return {nullptr}; + mlir::Type eTypeValue; + if (!unwrapAs(ctx, eType, false, eTypeValue)) + return {nullptr}; + return wrap(cute_nvgpu::SM100TmemEFrgAttr::get(ctx, aTypeValue, eTypeValue)); +} +MlirType mlirCuteNVGPUSM100TmemEFrgAttrGetAType(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getAType())); +} +MlirType mlirCuteNVGPUSM100TmemEFrgAttrGetEType(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getEType())); +} + +bool mlirAttributeIsACuteNVGPUSM100TmemFrg(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUSM100TmemFrgAttrGet(MlirContext context, + MlirType dataType, + MlirType storageType, + int ctaGroup, + uint32_t tmemAllocMode) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type dataTypeValue; + if (!unwrapAs(ctx, dataType, false, dataTypeValue)) + return {nullptr}; + mlir::Type storageTypeValue; + if (!unwrapAs(ctx, storageType, false, storageTypeValue)) + return {nullptr}; + auto tmemAllocModeValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmemAllocMode( + tmemAllocMode); + if (!tmemAllocModeValue) + return {nullptr}; + return wrap(cute_nvgpu::SM100TmemFrgAttr::get( + ctx, dataTypeValue, storageTypeValue, ctaGroup, *tmemAllocModeValue)); +} +MlirType mlirCuteNVGPUSM100TmemFrgAttrGetDataType(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getDataType())); +} +MlirType mlirCuteNVGPUSM100TmemFrgAttrGetStorageType(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getStorageType())); +} +int mlirCuteNVGPUSM100TmemFrgAttrGetCtaGroup(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getCtaGroup(); +} +uint32_t mlirCuteNVGPUSM100TmemFrgAttrGetTmemAllocMode(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getTmemAllocMode()); +} + +bool mlirAttributeIsACuteNVGPUSM100TmemSfFrg(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUSM100TmemSfFrgAttrGet(MlirContext context, + MlirType sfType, int sfVecSize, + int ctaGroup, bool isSfa, + uint32_t tmemAllocMode) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type sfTypeValue; + if (!unwrapAs(ctx, sfType, false, sfTypeValue)) + return {nullptr}; + auto tmemAllocModeValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmemAllocMode( + tmemAllocMode); + if (!tmemAllocModeValue) + return {nullptr}; + return wrap(cute_nvgpu::SM100TmemSfFrgAttr::get( + ctx, sfTypeValue, sfVecSize, ctaGroup, isSfa, *tmemAllocModeValue)); +} +MlirType mlirCuteNVGPUSM100TmemSfFrgAttrGetSfType(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getSfType())); +} +int mlirCuteNVGPUSM100TmemSfFrgAttrGetSfVecSize(MlirAttribute attr) { + return llvm::cast(unwrap(attr)) + .getSfVecSize(); +} +int mlirCuteNVGPUSM100TmemSfFrgAttrGetCtaGroup(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getCtaGroup(); +} +bool mlirCuteNVGPUSM100TmemSfFrgAttrGetIsSfa(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getIsSfa(); +} +uint32_t mlirCuteNVGPUSM100TmemSfFrgAttrGetTmemAllocMode(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getTmemAllocMode()); +} + +bool mlirAttributeIsACuteNVGPUSM107SmemFrg(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUSM107SmemFrgAttrGet(MlirContext context, + uint32_t majorMode) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto majorModeValue = + ::mlir::cutlass_compiler::cute::symbolizeMajorMode(majorMode); + if (!majorModeValue) + return {nullptr}; + return wrap(cute_nvgpu::SM107SmemFrgAttr::get(ctx, *majorModeValue)); +} +uint32_t mlirCuteNVGPUSM107SmemFrgAttrGetMajorMode(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getMajorMode()); +} + +bool mlirAttributeIsACuteNVGPUSM90SmemFrg(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUSM90SmemFrgAttrGet(MlirContext context, + uint32_t majorMode) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto majorModeValue = + ::mlir::cutlass_compiler::cute::symbolizeMajorMode(majorMode); + if (!majorModeValue) + return {nullptr}; + return wrap(cute_nvgpu::SM90SmemFrgAttr::get(ctx, *majorModeValue)); +} +uint32_t mlirCuteNVGPUSM90SmemFrgAttrGetMajorMode(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getMajorMode()); +} + +bool mlirAttributeIsACuteNVGPUTiledTmaLoad(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUTiledTmaLoadAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTiledTmaLoad(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::TiledTmaLoadAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUTiledTmaLoadAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPUTmaDataFormat(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUTmaDataFormatAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmaDataFormat(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::TmaDataFormatAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUTmaDataFormatAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPUTmaLoadMode(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUTmaLoadModeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmaLoadMode(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::TmaLoadModeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUTmaLoadModeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPUTmaStoreMode(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUTmaStoreModeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmaStoreMode(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::TmaStoreModeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUTmaStoreModeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPUTmemAllocMode(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUTmemAllocModeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmemAllocMode(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::TmemAllocModeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUTmemAllocModeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVGPUTmemLoadRedOp(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVGPUTmemLoadRedOpAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmemLoadRedOp(value); + if (!valueValue) + return {nullptr}; + return wrap(cute_nvgpu::TmemLoadRedOpAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVGPUTmemLoadRedOpAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +// Types + +bool mlirTypeIsACuteNVGPUType(MlirType type) { + mlir::Type cppType = unwrap(type); + if (!cppType) + return false; + if (auto opaque = mlir::dyn_cast(cppType)) + return opaque.getDialectNamespace() == "cute_nvgpu"; + return cppType.getDialect().getNamespace() == "cute_nvgpu"; +} + +bool mlirTypeIsACuteNVGPUCopyAtomBulkCopyG2S(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomBulkCopyG2STypeGet(MlirContext context, + MlirType valType, int copyBits, + bool mcast) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomBulkCopyG2SType::get(ctx, valTypeValue, + copyBits, mcast)); +} +MlirType mlirCuteNVGPUCopyAtomBulkCopyG2STypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getValType())); +} +int mlirCuteNVGPUCopyAtomBulkCopyG2STypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)) + .getCopyBits(); +} +bool mlirCuteNVGPUCopyAtomBulkCopyG2STypeGetMcast(MlirType type) { + return llvm::cast(unwrap(type)) + .getMcast(); +} + +bool mlirTypeIsACuteNVGPUCopyAtomBulkCopyS2G(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomBulkCopyS2GTypeGet(MlirContext context, + MlirType valType, int copyBits, + bool mask) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomBulkCopyS2GType::get(ctx, valTypeValue, + copyBits, mask)); +} +MlirType mlirCuteNVGPUCopyAtomBulkCopyS2GTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getValType())); +} +int mlirCuteNVGPUCopyAtomBulkCopyS2GTypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)) + .getCopyBits(); +} +bool mlirCuteNVGPUCopyAtomBulkCopyS2GTypeGetMask(MlirType type) { + return llvm::cast(unwrap(type)) + .getMask(); +} + +bool mlirTypeIsACuteNVGPUCopyAtomBulkCopyS2S(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomBulkCopyS2STypeGet(MlirContext context, + MlirType valType, + int copyBits) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + return wrap( + cute_nvgpu::CopyAtomBulkCopyS2SType::get(ctx, valTypeValue, copyBits)); +} +MlirType mlirCuteNVGPUCopyAtomBulkCopyS2STypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getValType())); +} +int mlirCuteNVGPUCopyAtomBulkCopyS2STypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)) + .getCopyBits(); +} + +bool mlirTypeIsACuteNVGPUCopyAtomDsmemStore(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomDsmemStoreTypeGet(MlirContext context, + MlirType valType, + int copyBits) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomDsmemStoreType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + valTypeValue, copyBits)); +} +MlirType mlirCuteNVGPUCopyAtomDsmemStoreTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getValType())); +} +int mlirCuteNVGPUCopyAtomDsmemStoreTypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)) + .getCopyBits(); +} + +bool mlirTypeIsACuteNVGPUCopyAtomG2R(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomG2RTypeGet( + MlirContext context, MlirType valType, int copyBits, uint32_t memOrder, + uint32_t memScope, uint32_t l2PrefetchSize, uint32_t l1CacheEvictPriority, + uint32_t loadCacheMode, uint32_t sharedSpace, bool invariant) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + auto memOrderValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMemOrderKind(memOrder); + if (!memOrderValue) + return {nullptr}; + auto memScopeValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMemScopeKind(memScope); + if (!memScopeValue) + return {nullptr}; + auto l2PrefetchSizeValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeL2PrefetchSize( + l2PrefetchSize); + if (!l2PrefetchSizeValue) + return {nullptr}; + auto l1CacheEvictPriorityValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeCacheEvictionPriority( + l1CacheEvictPriority); + if (!l1CacheEvictPriorityValue) + return {nullptr}; + auto loadCacheModeValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeLoadCacheMode( + loadCacheMode); + if (!loadCacheModeValue) + return {nullptr}; + auto sharedSpaceValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeSharedSpace(sharedSpace); + if (!sharedSpaceValue) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomG2RType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + valTypeValue, copyBits, *memOrderValue, *memScopeValue, + *l2PrefetchSizeValue, *l1CacheEvictPriorityValue, *loadCacheModeValue, + *sharedSpaceValue, invariant)); +} +MlirType mlirCuteNVGPUCopyAtomG2RTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getValType())); +} +int mlirCuteNVGPUCopyAtomG2RTypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)).getCopyBits(); +} +uint32_t mlirCuteNVGPUCopyAtomG2RTypeGetMemOrder(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getMemOrder()); +} +uint32_t mlirCuteNVGPUCopyAtomG2RTypeGetMemScope(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getMemScope()); +} +uint32_t mlirCuteNVGPUCopyAtomG2RTypeGetL2PrefetchSize(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getL2PrefetchSize()); +} +uint32_t mlirCuteNVGPUCopyAtomG2RTypeGetL1CacheEvictPriority(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getL1CacheEvictPriority()); +} +uint32_t mlirCuteNVGPUCopyAtomG2RTypeGetLoadCacheMode(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getLoadCacheMode()); +} +uint32_t mlirCuteNVGPUCopyAtomG2RTypeGetSharedSpace(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getSharedSpace()); +} +bool mlirCuteNVGPUCopyAtomG2RTypeGetInvariant(MlirType type) { + return llvm::cast(unwrap(type)).getInvariant(); +} + +bool mlirTypeIsACuteNVGPUCopyAtomIm2ColTmaLoad(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGet(MlirContext context, + MlirType valType, + int copyBits, int numCta, + MlirType gStride, bool mcast, + MlirType tmaGbasis) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::StrideType gStrideValue; + if (!unwrapAs(ctx, gStride, false, gStrideValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutType tmaGbasisValue; + if (!unwrapAs(ctx, tmaGbasis, true, tmaGbasisValue)) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomIm2ColTmaLoadType::get( + ctx, valTypeValue, copyBits, numCta, gStrideValue, mcast, + tmaGbasisValue)); +} +MlirType mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getValType())); +} +int mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)) + .getCopyBits(); +} +int mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetNumCta(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumCta(); +} +MlirType mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetGStride(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getGStride())); +} +bool mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetMcast(MlirType type) { + return llvm::cast(unwrap(type)) + .getMcast(); +} +MlirType mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetTmaGbasis(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getTmaGbasis())); +} + +bool mlirTypeIsACuteNVGPUCopyAtomIm2ColTmaStore(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGet(MlirContext context, + MlirType valType, + int copyBits, + MlirType gStride, + MlirType tmaGbasis) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::StrideType gStrideValue; + if (!unwrapAs(ctx, gStride, false, gStrideValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutType tmaGbasisValue; + if (!unwrapAs(ctx, tmaGbasis, true, tmaGbasisValue)) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomIm2ColTmaStoreType::get( + ctx, valTypeValue, copyBits, gStrideValue, tmaGbasisValue)); +} +MlirType mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getValType())); +} +int mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)) + .getCopyBits(); +} +MlirType mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGetGStride(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getGStride())); +} +MlirType mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGetTmaGbasis(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getTmaGbasis())); +} + +bool mlirTypeIsACuteNVGPUCopyAtomLdsm(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomLdsmTypeGet(MlirContext context, MlirType valType, + MlirAttribute mode, + uint32_t szPattern, int numMatrices, + bool transpose) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr modeValue; + if (!unwrapAs(ctx, mode, false, modeValue)) + return {nullptr}; + auto szPatternValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeLdsmSzPattern(szPattern); + if (!szPatternValue) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomLdsmType::get( + ctx, valTypeValue, modeValue, *szPatternValue, numMatrices, + transpose ? mlir::UnitAttr::get(ctx) : mlir::UnitAttr())); +} +MlirType mlirCuteNVGPUCopyAtomLdsmTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getValType())); +} +MlirAttribute mlirCuteNVGPUCopyAtomLdsmTypeGetMode(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getMode())); +} +uint32_t mlirCuteNVGPUCopyAtomLdsmTypeGetSzPattern(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getSzPattern()); +} +int mlirCuteNVGPUCopyAtomLdsmTypeGetNumMatrices(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumMatrices(); +} +bool mlirCuteNVGPUCopyAtomLdsmTypeGetTranspose(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getTranspose()); +} + +bool mlirTypeIsACuteNVGPUCopyAtomNonExec2DGather4TmaLoad(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGet( + MlirContext context, uint32_t kind, MlirType valType, int copyBits, + MlirType tmaGbasis, int64_t tmaFormat) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto kindValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeGatherScatterTmaLoad(kind); + if (!kindValue) + return {nullptr}; + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutType tmaGbasisValue; + if (!unwrapAs(ctx, tmaGbasis, false, tmaGbasisValue)) + return {nullptr}; + std::optional<::mlir::cutlass_compiler::cute_nvgpu::TmaDataFormat> + tmaFormatValue; + if (tmaFormat >= 0) { + tmaFormatValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmaDataFormat( + static_cast(tmaFormat)); + if (!tmaFormatValue) + return {nullptr}; + } + return wrap(cute_nvgpu::CopyAtomNonExec2DGather4TmaLoadType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + *kindValue, valTypeValue, copyBits, tmaGbasisValue, tmaFormatValue)); +} +uint32_t +mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetKind(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getKind()); +} +MlirType +mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getValType())); +} +int mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetCopyBits(MlirType type) { + return llvm::cast( + unwrap(type)) + .getCopyBits(); +} +MlirType +mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetTmaGbasis(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getTmaGbasis())); +} +int64_t +mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetTmaFormat(MlirType type) { + auto value = + llvm::cast(unwrap(type)) + .getTmaFormat(); + return value ? static_cast(*value) : -1; +} + +bool mlirTypeIsACuteNVGPUCopyAtomNonExec2DScatter4TmaStore(MlirType type) { + return llvm::isa_and_nonnull< + cute_nvgpu::CopyAtomNonExec2DScatter4TmaStoreType>(unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGet( + MlirContext context, MlirType valType, int copyBits, MlirType tmaGbasis, + int64_t tmaFormat) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutType tmaGbasisValue; + if (!unwrapAs(ctx, tmaGbasis, false, tmaGbasisValue)) + return {nullptr}; + std::optional<::mlir::cutlass_compiler::cute_nvgpu::TmaDataFormat> + tmaFormatValue; + if (tmaFormat >= 0) { + tmaFormatValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmaDataFormat( + static_cast(tmaFormat)); + if (!tmaFormatValue) + return {nullptr}; + } + return wrap(cute_nvgpu::CopyAtomNonExec2DScatter4TmaStoreType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + valTypeValue, copyBits, tmaGbasisValue, tmaFormatValue)); +} +MlirType +mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast( + unwrap(type)) + .getValType())); +} +int mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGetCopyBits( + MlirType type) { + return llvm::cast( + unwrap(type)) + .getCopyBits(); +} +MlirType +mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGetTmaGbasis(MlirType type) { + return wrap(static_cast( + llvm::cast( + unwrap(type)) + .getTmaGbasis())); +} +int64_t +mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGetTmaFormat(MlirType type) { + auto value = llvm::cast( + unwrap(type)) + .getTmaFormat(); + return value ? static_cast(*value) : -1; +} + +bool mlirTypeIsACuteNVGPUCopyAtomNonExecIm2ColTmaLoad(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGet( + MlirContext context, uint32_t kind, MlirType valType, int copyBits, + MlirType tmaGbasis, int64_t tmaFormat) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto kindValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeIm2ColTmaLoad(kind); + if (!kindValue) + return {nullptr}; + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutType tmaGbasisValue; + if (!unwrapAs(ctx, tmaGbasis, false, tmaGbasisValue)) + return {nullptr}; + std::optional<::mlir::cutlass_compiler::cute_nvgpu::TmaDataFormat> + tmaFormatValue; + if (tmaFormat >= 0) { + tmaFormatValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmaDataFormat( + static_cast(tmaFormat)); + if (!tmaFormatValue) + return {nullptr}; + } + return wrap(cute_nvgpu::CopyAtomNonExecIm2ColTmaLoadType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + *kindValue, valTypeValue, copyBits, tmaGbasisValue, tmaFormatValue)); +} +uint32_t mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetKind(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getKind()); +} +MlirType +mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getValType())); +} +int mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)) + .getCopyBits(); +} +MlirType +mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetTmaGbasis(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getTmaGbasis())); +} +int64_t +mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetTmaFormat(MlirType type) { + auto value = + llvm::cast(unwrap(type)) + .getTmaFormat(); + return value ? static_cast(*value) : -1; +} + +bool mlirTypeIsACuteNVGPUCopyAtomNonExecIm2ColTmaStore(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGet(MlirContext context, + MlirType valType, + int copyBits, + MlirType tmaGbasis, + int64_t tmaFormat) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutType tmaGbasisValue; + if (!unwrapAs(ctx, tmaGbasis, false, tmaGbasisValue)) + return {nullptr}; + std::optional<::mlir::cutlass_compiler::cute_nvgpu::TmaDataFormat> + tmaFormatValue; + if (tmaFormat >= 0) { + tmaFormatValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmaDataFormat( + static_cast(tmaFormat)); + if (!tmaFormatValue) + return {nullptr}; + } + return wrap(cute_nvgpu::CopyAtomNonExecIm2ColTmaStoreType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + valTypeValue, copyBits, tmaGbasisValue, tmaFormatValue)); +} +MlirType +mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getValType())); +} +int mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)) + .getCopyBits(); +} +MlirType +mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGetTmaGbasis(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getTmaGbasis())); +} +int64_t +mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGetTmaFormat(MlirType type) { + auto value = + llvm::cast(unwrap(type)) + .getTmaFormat(); + return value ? static_cast(*value) : -1; +} + +bool mlirTypeIsACuteNVGPUCopyAtomNonExecTiledTmaLoad(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGet( + MlirContext context, uint32_t kind, MlirType valType, int copyBits, + MlirType tmaGbasis, int64_t tmaFormat) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto kindValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTiledTmaLoad(kind); + if (!kindValue) + return {nullptr}; + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutType tmaGbasisValue; + if (!unwrapAs(ctx, tmaGbasis, false, tmaGbasisValue)) + return {nullptr}; + std::optional<::mlir::cutlass_compiler::cute_nvgpu::TmaDataFormat> + tmaFormatValue; + if (tmaFormat >= 0) { + tmaFormatValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmaDataFormat( + static_cast(tmaFormat)); + if (!tmaFormatValue) + return {nullptr}; + } + return wrap(cute_nvgpu::CopyAtomNonExecTiledTmaLoadType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + *kindValue, valTypeValue, copyBits, tmaGbasisValue, tmaFormatValue)); +} +uint32_t mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetKind(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getKind()); +} +MlirType mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getValType())); +} +int mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)) + .getCopyBits(); +} +MlirType +mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetTmaGbasis(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getTmaGbasis())); +} +int64_t +mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetTmaFormat(MlirType type) { + auto value = + llvm::cast(unwrap(type)) + .getTmaFormat(); + return value ? static_cast(*value) : -1; +} + +bool mlirTypeIsACuteNVGPUCopyAtomNonExecTiledTmaReduce(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGet( + MlirContext context, uint32_t kind, MlirType valType, int copyBits, + MlirType tmaGbasis, int64_t tmaFormat) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto kindValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeReductionKind(kind); + if (!kindValue) + return {nullptr}; + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutType tmaGbasisValue; + if (!unwrapAs(ctx, tmaGbasis, false, tmaGbasisValue)) + return {nullptr}; + std::optional<::mlir::cutlass_compiler::cute_nvgpu::TmaDataFormat> + tmaFormatValue; + if (tmaFormat >= 0) { + tmaFormatValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmaDataFormat( + static_cast(tmaFormat)); + if (!tmaFormatValue) + return {nullptr}; + } + return wrap(cute_nvgpu::CopyAtomNonExecTiledTmaReduceType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + *kindValue, valTypeValue, copyBits, tmaGbasisValue, tmaFormatValue)); +} +uint32_t mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetKind(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getKind()); +} +MlirType +mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getValType())); +} +int mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)) + .getCopyBits(); +} +MlirType +mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetTmaGbasis(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getTmaGbasis())); +} +int64_t +mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetTmaFormat(MlirType type) { + auto value = + llvm::cast(unwrap(type)) + .getTmaFormat(); + return value ? static_cast(*value) : -1; +} + +bool mlirTypeIsACuteNVGPUCopyAtomNonExecTiledTmaStore(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGet(MlirContext context, + MlirType valType, + int copyBits, + MlirType tmaGbasis, + int64_t tmaFormat) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutType tmaGbasisValue; + if (!unwrapAs(ctx, tmaGbasis, false, tmaGbasisValue)) + return {nullptr}; + std::optional<::mlir::cutlass_compiler::cute_nvgpu::TmaDataFormat> + tmaFormatValue; + if (tmaFormat >= 0) { + tmaFormatValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmaDataFormat( + static_cast(tmaFormat)); + if (!tmaFormatValue) + return {nullptr}; + } + return wrap(cute_nvgpu::CopyAtomNonExecTiledTmaStoreType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + valTypeValue, copyBits, tmaGbasisValue, tmaFormatValue)); +} +MlirType +mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getValType())); +} +int mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)) + .getCopyBits(); +} +MlirType +mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGetTmaGbasis(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getTmaGbasis())); +} +int64_t +mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGetTmaFormat(MlirType type) { + auto value = + llvm::cast(unwrap(type)) + .getTmaFormat(); + return value ? static_cast(*value) : -1; +} + +bool mlirTypeIsACuteNVGPUCopyAtomR2G(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomR2GTypeGet(MlirContext context, MlirType valType, + int copyBits, uint32_t memOrder, + uint32_t memScope, + uint32_t l1CacheEvictPriority, + uint32_t storeCacheMode, + uint32_t sharedSpace) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + auto memOrderValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMemOrderKind(memOrder); + if (!memOrderValue) + return {nullptr}; + auto memScopeValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMemScopeKind(memScope); + if (!memScopeValue) + return {nullptr}; + auto l1CacheEvictPriorityValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeCacheEvictionPriority( + l1CacheEvictPriority); + if (!l1CacheEvictPriorityValue) + return {nullptr}; + auto storeCacheModeValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeStoreCacheMode( + storeCacheMode); + if (!storeCacheModeValue) + return {nullptr}; + auto sharedSpaceValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeSharedSpace(sharedSpace); + if (!sharedSpaceValue) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomR2GType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + valTypeValue, copyBits, *memOrderValue, *memScopeValue, + *l1CacheEvictPriorityValue, *storeCacheModeValue, *sharedSpaceValue)); +} +MlirType mlirCuteNVGPUCopyAtomR2GTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getValType())); +} +int mlirCuteNVGPUCopyAtomR2GTypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)).getCopyBits(); +} +uint32_t mlirCuteNVGPUCopyAtomR2GTypeGetMemOrder(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getMemOrder()); +} +uint32_t mlirCuteNVGPUCopyAtomR2GTypeGetMemScope(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getMemScope()); +} +uint32_t mlirCuteNVGPUCopyAtomR2GTypeGetL1CacheEvictPriority(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getL1CacheEvictPriority()); +} +uint32_t mlirCuteNVGPUCopyAtomR2GTypeGetStoreCacheMode(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getStoreCacheMode()); +} +uint32_t mlirCuteNVGPUCopyAtomR2GTypeGetSharedSpace(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getSharedSpace()); +} + +bool mlirTypeIsACuteNVGPUCopyAtomR2S(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomR2STypeGet(MlirContext context, MlirType valType, + int copyBits, uint32_t memOrder, + uint32_t memScope, + uint32_t sharedSpace) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + auto memOrderValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMemOrderKind(memOrder); + if (!memOrderValue) + return {nullptr}; + auto memScopeValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMemScopeKind(memScope); + if (!memScopeValue) + return {nullptr}; + auto sharedSpaceValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeSharedSpace(sharedSpace); + if (!sharedSpaceValue) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomR2SType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + valTypeValue, copyBits, *memOrderValue, *memScopeValue, + *sharedSpaceValue)); +} +MlirType mlirCuteNVGPUCopyAtomR2STypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getValType())); +} +int mlirCuteNVGPUCopyAtomR2STypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)).getCopyBits(); +} +uint32_t mlirCuteNVGPUCopyAtomR2STypeGetMemOrder(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getMemOrder()); +} +uint32_t mlirCuteNVGPUCopyAtomR2STypeGetMemScope(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getMemScope()); +} +uint32_t mlirCuteNVGPUCopyAtomR2STypeGetSharedSpace(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getSharedSpace()); +} + +bool mlirTypeIsACuteNVGPUCopyAtomS2R(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomS2RTypeGet(MlirContext context, MlirType valType, + int copyBits, uint32_t memOrder, + uint32_t memScope, + uint32_t sharedSpace) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + auto memOrderValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMemOrderKind(memOrder); + if (!memOrderValue) + return {nullptr}; + auto memScopeValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMemScopeKind(memScope); + if (!memScopeValue) + return {nullptr}; + auto sharedSpaceValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeSharedSpace(sharedSpace); + if (!sharedSpaceValue) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomS2RType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + valTypeValue, copyBits, *memOrderValue, *memScopeValue, + *sharedSpaceValue)); +} +MlirType mlirCuteNVGPUCopyAtomS2RTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getValType())); +} +int mlirCuteNVGPUCopyAtomS2RTypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)).getCopyBits(); +} +uint32_t mlirCuteNVGPUCopyAtomS2RTypeGetMemOrder(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getMemOrder()); +} +uint32_t mlirCuteNVGPUCopyAtomS2RTypeGetMemScope(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getMemScope()); +} +uint32_t mlirCuteNVGPUCopyAtomS2RTypeGetSharedSpace(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getSharedSpace()); +} + +bool mlirTypeIsACuteNVGPUCopyAtomSIMTAsyncCopy(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGet(MlirContext context, + MlirType valType, + uint32_t cache, + int copyBits) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + auto cacheValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeLoadCacheMode(cache); + if (!cacheValue) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomSIMTAsyncCopyType::get( + ctx, valTypeValue, *cacheValue, copyBits)); +} +MlirType mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getValType())); +} +uint32_t mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGetCache(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getCache()); +} +int mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)) + .getCopyBits(); +} + +bool mlirTypeIsACuteNVGPUCopyAtomSIMTMultimemLdReduce(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGet( + MlirContext context, MlirType valType, int copyBits, uint32_t reduction, + MlirAttribute ldReduceAccPrecision, uint32_t memOrder, uint32_t memScope) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + auto reductionValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeReductionKind(reduction); + if (!reductionValue) + return {nullptr}; + ::mlir::cutlass_compiler::cute_nvgpu::LdReduceAccPrecisionKindAttr + ldReduceAccPrecisionValue; + if (!unwrapAs(ctx, ldReduceAccPrecision, true, ldReduceAccPrecisionValue)) + return {nullptr}; + auto memOrderValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMemOrderKind(memOrder); + if (!memOrderValue) + return {nullptr}; + auto memScopeValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMemScopeKind(memScope); + if (!memScopeValue) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomSIMTMultimemLdReduceType::get( + ctx, valTypeValue, copyBits, *reductionValue, ldReduceAccPrecisionValue, + *memOrderValue, *memScopeValue)); +} +MlirType +mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getValType())); +} +int mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)) + .getCopyBits(); +} +uint32_t +mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetReduction(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getReduction()); +} +MlirAttribute +mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetLdReduceAccPrecision( + MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getLdReduceAccPrecision())); +} +uint32_t +mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetMemOrder(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getMemOrder()); +} +uint32_t +mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetMemScope(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getMemScope()); +} + +bool mlirTypeIsACuteNVGPUCopyAtomSIMTMultimemRed(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGet( + MlirContext context, MlirType valType, int copyBits, uint32_t reduction, + uint32_t memOrder, uint32_t memScope) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + auto reductionValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeReductionKind(reduction); + if (!reductionValue) + return {nullptr}; + auto memOrderValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMemOrderKind(memOrder); + if (!memOrderValue) + return {nullptr}; + auto memScopeValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMemScopeKind(memScope); + if (!memScopeValue) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomSIMTMultimemRedType::get( + ctx, valTypeValue, copyBits, *reductionValue, *memOrderValue, + *memScopeValue)); +} +MlirType mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getValType())); +} +int mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)) + .getCopyBits(); +} +uint32_t mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetReduction(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getReduction()); +} +uint32_t mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetMemOrder(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getMemOrder()); +} +uint32_t mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetMemScope(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getMemScope()); +} + +bool mlirTypeIsACuteNVGPUCopyAtomSIMTMultimemSt(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGet(MlirContext context, + MlirType valType, + int copyBits, + uint32_t memOrder, + uint32_t memScope) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + auto memOrderValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMemOrderKind(memOrder); + if (!memOrderValue) + return {nullptr}; + auto memScopeValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMemScopeKind(memScope); + if (!memScopeValue) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomSIMTMultimemStType::get( + ctx, valTypeValue, copyBits, *memOrderValue, *memScopeValue)); +} +MlirType mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getValType())); +} +int mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)) + .getCopyBits(); +} +uint32_t mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetMemOrder(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getMemOrder()); +} +uint32_t mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetMemScope(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getMemScope()); +} + +bool mlirTypeIsACuteNVGPUCopyAtomSIMTSyncCopy(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGet(MlirContext context, + MlirType valType, + int copyBits, + uint32_t srcSpace, + uint32_t dstSpace) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + auto srcSpaceValue = + ::mlir::cutlass_compiler::cute::symbolizeAddressSpace(srcSpace); + if (!srcSpaceValue) + return {nullptr}; + auto dstSpaceValue = + ::mlir::cutlass_compiler::cute::symbolizeAddressSpace(dstSpace); + if (!dstSpaceValue) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomSIMTSyncCopyType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + valTypeValue, copyBits, *srcSpaceValue, *dstSpaceValue)); +} +MlirType mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getValType())); +} +int mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)) + .getCopyBits(); +} +uint32_t mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetSrcSpace(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getSrcSpace()); +} +uint32_t mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetDstSpace(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getDstSpace()); +} + +bool mlirTypeIsACuteNVGPUCopyAtomSM100CopyS2T(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGet(MlirContext context, + MlirType valType, int numDp, + int numBit, int numCta, + uint32_t broadcast) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + auto broadcastValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeCopyS2TBroadcast( + broadcast); + if (!broadcastValue) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomSM100CopyS2TType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + valTypeValue, numDp, numBit, numCta, *broadcastValue)); +} +MlirType mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getValType())); +} +int mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetNumDp(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumDp(); +} +int mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetNumBit(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumBit(); +} +int mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetNumCta(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumCta(); +} +uint32_t mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetBroadcast(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getBroadcast()); +} + +bool mlirTypeIsACuteNVGPUCopyAtomSM100S2TCopyV2(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGet(MlirContext context, + MlirType valType, int numDp, + int numBit, int numCta, + uint32_t smemMajor, + uint32_t broadcast) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + auto smemMajorValue = + ::mlir::cutlass_compiler::cute::symbolizeMajorMode(smemMajor); + if (!smemMajorValue) + return {nullptr}; + auto broadcastValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeCopyS2TBroadcast( + broadcast); + if (!broadcastValue) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomSM100S2TCopyV2Type::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + valTypeValue, numDp, numBit, numCta, *smemMajorValue, *broadcastValue)); +} +MlirType mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getValType())); +} +int mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetNumDp(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumDp(); +} +int mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetNumBit(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumBit(); +} +int mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetNumCta(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumCta(); +} +uint32_t mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetSmemMajor(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getSmemMajor()); +} +uint32_t mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetBroadcast(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getBroadcast()); +} + +bool mlirTypeIsACuteNVGPUCopyAtomSM100TmemLoad(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGet(MlirContext context, + MlirType valType, int numDp, + int numBit, uint32_t numRep, + bool pack16b) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomSM100TmemLoadType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + valTypeValue, numDp, numBit, numRep, + pack16b ? mlir::UnitAttr::get(ctx) : mlir::UnitAttr())); +} +MlirType mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getValType())); +} +int mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetNumDp(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumDp(); +} +int mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetNumBit(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumBit(); +} +uint32_t mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetNumRep(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumRep(); +} +bool mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetPack16b(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getPack16b()); +} + +bool mlirTypeIsACuteNVGPUCopyAtomSM100TmemStore(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGet(MlirContext context, + MlirType valType, int numDp, + int numBit, uint32_t numRep, + bool expand16b) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomSM100TmemStoreType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + valTypeValue, numDp, numBit, numRep, + expand16b ? mlir::UnitAttr::get(ctx) : mlir::UnitAttr())); +} +MlirType mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getValType())); +} +int mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetNumDp(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumDp(); +} +int mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetNumBit(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumBit(); +} +uint32_t mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetNumRep(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumRep(); +} +bool mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetExpand16b(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getExpand16b()); +} + +bool mlirTypeIsACuteNVGPUCopyAtomSM107TmemLoadSPCompress(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGet( + MlirContext context, MlirType valType, int numDp, int numBit, + uint32_t numRep, uint32_t redOp, bool red, bool nan) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + auto redOpValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmemLoadRedOp(redOp); + if (!redOpValue) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomSM107TmemLoadSPCompressType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + valTypeValue, numDp, numBit, numRep, *redOpValue, + red ? mlir::UnitAttr::get(ctx) : mlir::UnitAttr(), + nan ? mlir::UnitAttr::get(ctx) : mlir::UnitAttr())); +} +MlirType +mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getValType())); +} +int mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetNumDp(MlirType type) { + return llvm::cast( + unwrap(type)) + .getNumDp(); +} +int mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetNumBit(MlirType type) { + return llvm::cast( + unwrap(type)) + .getNumBit(); +} +uint32_t +mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetNumRep(MlirType type) { + return llvm::cast( + unwrap(type)) + .getNumRep(); +} +uint32_t +mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetRedOp(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getRedOp()); +} +bool mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetRed(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getRed()); +} +bool mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetNan(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getNan()); +} + +bool mlirTypeIsACuteNVGPUCopyAtomSM10xTmemLoadRed(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGet( + MlirContext context, MlirType valType, int numDp, int numBit, + uint32_t numRep, uint32_t redOp, bool nan, MlirAttribute halfSplitOff) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + auto redOpValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmemLoadRedOp(redOp); + if (!redOpValue) + return {nullptr}; + ::mlir::IntegerAttr halfSplitOffValue; + if (!unwrapAs(ctx, halfSplitOff, true, halfSplitOffValue)) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomSM10xTmemLoadRedType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + valTypeValue, numDp, numBit, numRep, *redOpValue, + nan ? mlir::UnitAttr::get(ctx) : mlir::UnitAttr(), halfSplitOffValue)); +} +MlirType mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getValType())); +} +int mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetNumDp(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumDp(); +} +int mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetNumBit(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumBit(); +} +uint32_t mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetNumRep(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumRep(); +} +uint32_t mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetRedOp(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getRedOp()); +} +bool mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetNan(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getNan()); +} +MlirAttribute +mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetHalfSplitOff(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getHalfSplitOff())); +} + +bool mlirTypeIsACuteNVGPUCopyAtomStsm(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomStsmTypeGet(MlirContext context, MlirType valType, + MlirAttribute mode, int numMatrices, + bool transpose) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::ShapeAttr modeValue; + if (!unwrapAs(ctx, mode, false, modeValue)) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomStsmType::get( + ctx, valTypeValue, modeValue, numMatrices, + transpose ? mlir::UnitAttr::get(ctx) : mlir::UnitAttr())); +} +MlirType mlirCuteNVGPUCopyAtomStsmTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getValType())); +} +MlirAttribute mlirCuteNVGPUCopyAtomStsmTypeGetMode(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getMode())); +} +int mlirCuteNVGPUCopyAtomStsmTypeGetNumMatrices(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumMatrices(); +} +bool mlirCuteNVGPUCopyAtomStsmTypeGetTranspose(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getTranspose()); +} + +bool mlirTypeIsACuteNVGPUCopyAtomTmaLoad(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomTmaLoadTypeGet( + MlirContext context, MlirType valType, int sparsity, int copyBits, + uint32_t mode, int numCta, MlirType gStride, bool mcast, MlirType tmaGbasis, + bool override, bool noFullyOobTile) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + auto modeValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmaLoadMode(mode); + if (!modeValue) + return {nullptr}; + ::mlir::cutlass_compiler::cute::StrideType gStrideValue; + if (!unwrapAs(ctx, gStride, false, gStrideValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutType tmaGbasisValue; + if (!unwrapAs(ctx, tmaGbasis, true, tmaGbasisValue)) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomTmaLoadType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + valTypeValue, sparsity, copyBits, *modeValue, numCta, gStrideValue, mcast, + tmaGbasisValue, override, noFullyOobTile)); +} +MlirType mlirCuteNVGPUCopyAtomTmaLoadTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getValType())); +} +int mlirCuteNVGPUCopyAtomTmaLoadTypeGetSparsity(MlirType type) { + return llvm::cast(unwrap(type)) + .getSparsity(); +} +int mlirCuteNVGPUCopyAtomTmaLoadTypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)) + .getCopyBits(); +} +uint32_t mlirCuteNVGPUCopyAtomTmaLoadTypeGetMode(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getMode()); +} +int mlirCuteNVGPUCopyAtomTmaLoadTypeGetNumCta(MlirType type) { + return llvm::cast(unwrap(type)).getNumCta(); +} +MlirType mlirCuteNVGPUCopyAtomTmaLoadTypeGetGStride(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getGStride())); +} +bool mlirCuteNVGPUCopyAtomTmaLoadTypeGetMcast(MlirType type) { + return llvm::cast(unwrap(type)).getMcast(); +} +MlirType mlirCuteNVGPUCopyAtomTmaLoadTypeGetTmaGbasis(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getTmaGbasis())); +} +bool mlirCuteNVGPUCopyAtomTmaLoadTypeGetOverride(MlirType type) { + return llvm::cast(unwrap(type)) + .getOverride(); +} +bool mlirCuteNVGPUCopyAtomTmaLoadTypeGetNoFullyOobTile(MlirType type) { + return llvm::cast(unwrap(type)) + .getNoFullyOobTile(); +} + +bool mlirTypeIsACuteNVGPUCopyAtomTmaReduce(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomTmaReduceTypeGet(MlirContext context, + MlirType valType, int copyBits, + uint32_t mode, uint32_t kind, + MlirType gStride, + MlirType tmaGbasis) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + auto modeValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmaStoreMode(mode); + if (!modeValue) + return {nullptr}; + auto kindValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeReductionKind(kind); + if (!kindValue) + return {nullptr}; + ::mlir::cutlass_compiler::cute::StrideType gStrideValue; + if (!unwrapAs(ctx, gStride, false, gStrideValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutType tmaGbasisValue; + if (!unwrapAs(ctx, tmaGbasis, true, tmaGbasisValue)) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomTmaReduceType::get( + ctx, valTypeValue, copyBits, *modeValue, *kindValue, gStrideValue, + tmaGbasisValue)); +} +MlirType mlirCuteNVGPUCopyAtomTmaReduceTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getValType())); +} +int mlirCuteNVGPUCopyAtomTmaReduceTypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)) + .getCopyBits(); +} +uint32_t mlirCuteNVGPUCopyAtomTmaReduceTypeGetMode(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getMode()); +} +uint32_t mlirCuteNVGPUCopyAtomTmaReduceTypeGetKind(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getKind()); +} +MlirType mlirCuteNVGPUCopyAtomTmaReduceTypeGetGStride(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getGStride())); +} +MlirType mlirCuteNVGPUCopyAtomTmaReduceTypeGetTmaGbasis(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getTmaGbasis())); +} + +bool mlirTypeIsACuteNVGPUCopyAtomTmaStore(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUCopyAtomTmaStoreTypeGet(MlirContext context, + MlirType valType, int sparsity, + int copyBits, uint32_t mode, + MlirType gStride, + MlirType tmaGbasis, bool override, + bool noFullyOobTile) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type valTypeValue; + if (!unwrapAs(ctx, valType, false, valTypeValue)) + return {nullptr}; + auto modeValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmaStoreMode(mode); + if (!modeValue) + return {nullptr}; + ::mlir::cutlass_compiler::cute::StrideType gStrideValue; + if (!unwrapAs(ctx, gStride, false, gStrideValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutType tmaGbasisValue; + if (!unwrapAs(ctx, tmaGbasis, true, tmaGbasisValue)) + return {nullptr}; + return wrap(cute_nvgpu::CopyAtomTmaStoreType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + valTypeValue, sparsity, copyBits, *modeValue, gStrideValue, + tmaGbasisValue, override, noFullyOobTile)); +} +MlirType mlirCuteNVGPUCopyAtomTmaStoreTypeGetValType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getValType())); +} +int mlirCuteNVGPUCopyAtomTmaStoreTypeGetSparsity(MlirType type) { + return llvm::cast(unwrap(type)) + .getSparsity(); +} +int mlirCuteNVGPUCopyAtomTmaStoreTypeGetCopyBits(MlirType type) { + return llvm::cast(unwrap(type)) + .getCopyBits(); +} +uint32_t mlirCuteNVGPUCopyAtomTmaStoreTypeGetMode(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getMode()); +} +MlirType mlirCuteNVGPUCopyAtomTmaStoreTypeGetGStride(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getGStride())); +} +MlirType mlirCuteNVGPUCopyAtomTmaStoreTypeGetTmaGbasis(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getTmaGbasis())); +} +bool mlirCuteNVGPUCopyAtomTmaStoreTypeGetOverride(MlirType type) { + return llvm::cast(unwrap(type)) + .getOverride(); +} +bool mlirCuteNVGPUCopyAtomTmaStoreTypeGetNoFullyOobTile(MlirType type) { + return llvm::cast(unwrap(type)) + .getNoFullyOobTile(); +} + +bool mlirTypeIsACuteNVGPUMmaAtomSM100UMMABlockScaledSparse(MlirType type) { + return llvm::isa_and_nonnull< + cute_nvgpu::MmaAtomSM100UMMABlockScaledSparseType>(unwrap(type)); +} +MlirType mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGet( + MlirContext context, MlirAttribute shapeMnk, int numCta, uint32_t aMajor, + uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, + MlirType sfType, uint32_t sparseMetadataFormat, uint32_t aFragKind, + int vecSize, uint32_t archPromote, MlirAttribute intOverflow) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::ShapeAttr shapeMnkValue; + if (!unwrapAs(ctx, shapeMnk, false, shapeMnkValue)) + return {nullptr}; + auto aMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(aMajor); + if (!aMajorValue) + return {nullptr}; + auto bMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(bMajor); + if (!bMajorValue) + return {nullptr}; + mlir::Type aTypeValue; + if (!unwrapAs(ctx, aType, false, aTypeValue)) + return {nullptr}; + mlir::Type bTypeValue; + if (!unwrapAs(ctx, bType, false, bTypeValue)) + return {nullptr}; + mlir::Type cTypeValue; + if (!unwrapAs(ctx, cType, false, cTypeValue)) + return {nullptr}; + mlir::Type sfTypeValue; + if (!unwrapAs(ctx, sfType, false, sfTypeValue)) + return {nullptr}; + auto sparseMetadataFormatValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeSparseMetadataFormat( + sparseMetadataFormat); + if (!sparseMetadataFormatValue) + return {nullptr}; + auto aFragKindValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaFragKind(aFragKind); + if (!aFragKindValue) + return {nullptr}; + auto archPromoteValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeArch(archPromote); + if (!archPromoteValue) + return {nullptr}; + ::mlir::cutlass_compiler::cute_nvgpu::MMAIntOverflowAttr intOverflowValue; + if (!unwrapAs(ctx, intOverflow, true, intOverflowValue)) + return {nullptr}; + return wrap(cute_nvgpu::MmaAtomSM100UMMABlockScaledSparseType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + shapeMnkValue, numCta, *aMajorValue, *bMajorValue, aTypeValue, bTypeValue, + cTypeValue, sfTypeValue, *sparseMetadataFormatValue, *aFragKindValue, + vecSize, *archPromoteValue, intOverflowValue)); +} +MlirAttribute +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetShapeMnk(MlirType type) { + return wrap(static_cast( + llvm::cast( + unwrap(type)) + .getShapeMnk())); +} +int mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetNumCta(MlirType type) { + return llvm::cast( + unwrap(type)) + .getNumCta(); +} +uint32_t +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetAMajor(MlirType type) { + return static_cast( + llvm::cast( + unwrap(type)) + .getAMajor()); +} +uint32_t +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetBMajor(MlirType type) { + return static_cast( + llvm::cast( + unwrap(type)) + .getBMajor()); +} +MlirType +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetAType(MlirType type) { + return wrap(static_cast( + llvm::cast( + unwrap(type)) + .getAType())); +} +MlirType +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetBType(MlirType type) { + return wrap(static_cast( + llvm::cast( + unwrap(type)) + .getBType())); +} +MlirType +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetCType(MlirType type) { + return wrap(static_cast( + llvm::cast( + unwrap(type)) + .getCType())); +} +MlirType +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetSfType(MlirType type) { + return wrap(static_cast( + llvm::cast( + unwrap(type)) + .getSfType())); +} +uint32_t +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetSparseMetadataFormat( + MlirType type) { + return static_cast( + llvm::cast( + unwrap(type)) + .getSparseMetadataFormat()); +} +uint32_t +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetAFragKind(MlirType type) { + return static_cast( + llvm::cast( + unwrap(type)) + .getAFragKind()); +} +int mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetVecSize( + MlirType type) { + return llvm::cast( + unwrap(type)) + .getVecSize(); +} +uint32_t mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetArchPromote( + MlirType type) { + return static_cast( + llvm::cast( + unwrap(type)) + .getArchPromote()); +} +MlirAttribute mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetIntOverflow( + MlirType type) { + return wrap(static_cast( + llvm::cast( + unwrap(type)) + .getIntOverflow())); +} + +bool mlirTypeIsACuteNVGPUMmaAtomSM100UMMABlockScaled(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGet( + MlirContext context, MlirAttribute shapeMnk, int numCta, uint32_t aMajor, + uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, + MlirType sfType, uint32_t aFragKind, int vecSize, uint32_t archPromote, + MlirAttribute intOverflow) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::ShapeAttr shapeMnkValue; + if (!unwrapAs(ctx, shapeMnk, false, shapeMnkValue)) + return {nullptr}; + auto aMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(aMajor); + if (!aMajorValue) + return {nullptr}; + auto bMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(bMajor); + if (!bMajorValue) + return {nullptr}; + mlir::Type aTypeValue; + if (!unwrapAs(ctx, aType, false, aTypeValue)) + return {nullptr}; + mlir::Type bTypeValue; + if (!unwrapAs(ctx, bType, false, bTypeValue)) + return {nullptr}; + mlir::Type cTypeValue; + if (!unwrapAs(ctx, cType, false, cTypeValue)) + return {nullptr}; + mlir::Type sfTypeValue; + if (!unwrapAs(ctx, sfType, false, sfTypeValue)) + return {nullptr}; + auto aFragKindValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaFragKind(aFragKind); + if (!aFragKindValue) + return {nullptr}; + auto archPromoteValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeArch(archPromote); + if (!archPromoteValue) + return {nullptr}; + ::mlir::cutlass_compiler::cute_nvgpu::MMAIntOverflowAttr intOverflowValue; + if (!unwrapAs(ctx, intOverflow, true, intOverflowValue)) + return {nullptr}; + return wrap(cute_nvgpu::MmaAtomSM100UMMABlockScaledType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + shapeMnkValue, numCta, *aMajorValue, *bMajorValue, aTypeValue, bTypeValue, + cTypeValue, sfTypeValue, *aFragKindValue, vecSize, *archPromoteValue, + intOverflowValue)); +} +MlirAttribute +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetShapeMnk(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getShapeMnk())); +} +int mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetNumCta(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumCta(); +} +uint32_t mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetAMajor(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getAMajor()); +} +uint32_t mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetBMajor(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getBMajor()); +} +MlirType mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetAType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getAType())); +} +MlirType mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetBType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getBType())); +} +MlirType mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetCType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getCType())); +} +MlirType mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetSfType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getSfType())); +} +uint32_t +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetAFragKind(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getAFragKind()); +} +int mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetVecSize(MlirType type) { + return llvm::cast(unwrap(type)) + .getVecSize(); +} +uint32_t +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetArchPromote(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getArchPromote()); +} +MlirAttribute +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetIntOverflow(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getIntOverflow())); +} + +bool mlirTypeIsACuteNVGPUMmaAtomSM100UMMASparse(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGet( + MlirContext context, MlirAttribute shapeMnk, int numCta, uint32_t aMajor, + uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, + MlirType eType, uint32_t sparseMetadataFormat, uint32_t aFragKind, + int cScaleExp, MlirAttribute intOverflow) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::ShapeAttr shapeMnkValue; + if (!unwrapAs(ctx, shapeMnk, false, shapeMnkValue)) + return {nullptr}; + auto aMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(aMajor); + if (!aMajorValue) + return {nullptr}; + auto bMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(bMajor); + if (!bMajorValue) + return {nullptr}; + mlir::Type aTypeValue; + if (!unwrapAs(ctx, aType, false, aTypeValue)) + return {nullptr}; + mlir::Type bTypeValue; + if (!unwrapAs(ctx, bType, false, bTypeValue)) + return {nullptr}; + mlir::Type cTypeValue; + if (!unwrapAs(ctx, cType, false, cTypeValue)) + return {nullptr}; + mlir::Type eTypeValue; + if (!unwrapAs(ctx, eType, false, eTypeValue)) + return {nullptr}; + auto sparseMetadataFormatValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeSparseMetadataFormat( + sparseMetadataFormat); + if (!sparseMetadataFormatValue) + return {nullptr}; + auto aFragKindValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaFragKind(aFragKind); + if (!aFragKindValue) + return {nullptr}; + ::mlir::cutlass_compiler::cute_nvgpu::MMAIntOverflowAttr intOverflowValue; + if (!unwrapAs(ctx, intOverflow, true, intOverflowValue)) + return {nullptr}; + return wrap(cute_nvgpu::MmaAtomSM100UMMASparseType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + shapeMnkValue, numCta, *aMajorValue, *bMajorValue, aTypeValue, bTypeValue, + cTypeValue, eTypeValue, *sparseMetadataFormatValue, *aFragKindValue, + cScaleExp, intOverflowValue)); +} +MlirAttribute +mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetShapeMnk(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getShapeMnk())); +} +int mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetNumCta(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumCta(); +} +uint32_t mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetAMajor(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getAMajor()); +} +uint32_t mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetBMajor(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getBMajor()); +} +MlirType mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetAType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getAType())); +} +MlirType mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetBType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getBType())); +} +MlirType mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetCType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getCType())); +} +MlirType mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetEType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getEType())); +} +uint32_t +mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetSparseMetadataFormat(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getSparseMetadataFormat()); +} +uint32_t mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetAFragKind(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getAFragKind()); +} +int mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetCScaleExp(MlirType type) { + return llvm::cast(unwrap(type)) + .getCScaleExp(); +} +MlirAttribute +mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetIntOverflow(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getIntOverflow())); +} + +bool mlirTypeIsACuteNVGPUMmaAtomSM100UMMA(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUMmaAtomSM100UMMATypeGet( + MlirContext context, MlirAttribute shapeMnk, int numCta, uint32_t aMajor, + uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, + uint32_t aFragKind, int cScaleExp, MlirAttribute intOverflow) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::ShapeAttr shapeMnkValue; + if (!unwrapAs(ctx, shapeMnk, false, shapeMnkValue)) + return {nullptr}; + auto aMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(aMajor); + if (!aMajorValue) + return {nullptr}; + auto bMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(bMajor); + if (!bMajorValue) + return {nullptr}; + mlir::Type aTypeValue; + if (!unwrapAs(ctx, aType, false, aTypeValue)) + return {nullptr}; + mlir::Type bTypeValue; + if (!unwrapAs(ctx, bType, false, bTypeValue)) + return {nullptr}; + mlir::Type cTypeValue; + if (!unwrapAs(ctx, cType, false, cTypeValue)) + return {nullptr}; + auto aFragKindValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaFragKind(aFragKind); + if (!aFragKindValue) + return {nullptr}; + ::mlir::cutlass_compiler::cute_nvgpu::MMAIntOverflowAttr intOverflowValue; + if (!unwrapAs(ctx, intOverflow, true, intOverflowValue)) + return {nullptr}; + return wrap(cute_nvgpu::MmaAtomSM100UMMAType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + shapeMnkValue, numCta, *aMajorValue, *bMajorValue, aTypeValue, bTypeValue, + cTypeValue, *aFragKindValue, cScaleExp, intOverflowValue)); +} +MlirAttribute mlirCuteNVGPUMmaAtomSM100UMMATypeGetShapeMnk(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getShapeMnk())); +} +int mlirCuteNVGPUMmaAtomSM100UMMATypeGetNumCta(MlirType type) { + return llvm::cast(unwrap(type)).getNumCta(); +} +uint32_t mlirCuteNVGPUMmaAtomSM100UMMATypeGetAMajor(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getAMajor()); +} +uint32_t mlirCuteNVGPUMmaAtomSM100UMMATypeGetBMajor(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getBMajor()); +} +MlirType mlirCuteNVGPUMmaAtomSM100UMMATypeGetAType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getAType())); +} +MlirType mlirCuteNVGPUMmaAtomSM100UMMATypeGetBType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getBType())); +} +MlirType mlirCuteNVGPUMmaAtomSM100UMMATypeGetCType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getCType())); +} +uint32_t mlirCuteNVGPUMmaAtomSM100UMMATypeGetAFragKind(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getAFragKind()); +} +int mlirCuteNVGPUMmaAtomSM100UMMATypeGetCScaleExp(MlirType type) { + return llvm::cast(unwrap(type)) + .getCScaleExp(); +} +MlirAttribute mlirCuteNVGPUMmaAtomSM100UMMATypeGetIntOverflow(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getIntOverflow())); +} + +bool mlirTypeIsACuteNVGPUMmaAtomSM107UMMABlockScaledSparse(MlirType type) { + return llvm::isa_and_nonnull< + cute_nvgpu::MmaAtomSM107UMMABlockScaledSparseType>(unwrap(type)); +} +MlirType mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGet( + MlirContext context, MlirAttribute shapeMnk, int numCta, uint32_t aMajor, + uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, + MlirType sfType, uint32_t sparseMetadataFormat, uint32_t aFragKind, + int vecSize, uint32_t aCollectorOp, uint32_t bCollectorOp, + MlirAttribute intOverflow) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::ShapeAttr shapeMnkValue; + if (!unwrapAs(ctx, shapeMnk, false, shapeMnkValue)) + return {nullptr}; + auto aMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(aMajor); + if (!aMajorValue) + return {nullptr}; + auto bMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(bMajor); + if (!bMajorValue) + return {nullptr}; + mlir::Type aTypeValue; + if (!unwrapAs(ctx, aType, false, aTypeValue)) + return {nullptr}; + mlir::Type bTypeValue; + if (!unwrapAs(ctx, bType, false, bTypeValue)) + return {nullptr}; + mlir::Type cTypeValue; + if (!unwrapAs(ctx, cType, false, cTypeValue)) + return {nullptr}; + mlir::Type sfTypeValue; + if (!unwrapAs(ctx, sfType, false, sfTypeValue)) + return {nullptr}; + auto sparseMetadataFormatValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeSparseMetadataFormat( + sparseMetadataFormat); + if (!sparseMetadataFormatValue) + return {nullptr}; + auto aFragKindValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaFragKind(aFragKind); + if (!aFragKindValue) + return {nullptr}; + auto aCollectorOpValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaCollectorOp( + aCollectorOp); + if (!aCollectorOpValue) + return {nullptr}; + auto bCollectorOpValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaCollectorOp( + bCollectorOp); + if (!bCollectorOpValue) + return {nullptr}; + ::mlir::cutlass_compiler::cute_nvgpu::MMAIntOverflowAttr intOverflowValue; + if (!unwrapAs(ctx, intOverflow, true, intOverflowValue)) + return {nullptr}; + return wrap(cute_nvgpu::MmaAtomSM107UMMABlockScaledSparseType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + shapeMnkValue, numCta, *aMajorValue, *bMajorValue, aTypeValue, bTypeValue, + cTypeValue, sfTypeValue, *sparseMetadataFormatValue, *aFragKindValue, + vecSize, *aCollectorOpValue, *bCollectorOpValue, intOverflowValue)); +} +MlirAttribute +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetShapeMnk(MlirType type) { + return wrap(static_cast( + llvm::cast( + unwrap(type)) + .getShapeMnk())); +} +int mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetNumCta(MlirType type) { + return llvm::cast( + unwrap(type)) + .getNumCta(); +} +uint32_t +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetAMajor(MlirType type) { + return static_cast( + llvm::cast( + unwrap(type)) + .getAMajor()); +} +uint32_t +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetBMajor(MlirType type) { + return static_cast( + llvm::cast( + unwrap(type)) + .getBMajor()); +} +MlirType +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetAType(MlirType type) { + return wrap(static_cast( + llvm::cast( + unwrap(type)) + .getAType())); +} +MlirType +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetBType(MlirType type) { + return wrap(static_cast( + llvm::cast( + unwrap(type)) + .getBType())); +} +MlirType +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetCType(MlirType type) { + return wrap(static_cast( + llvm::cast( + unwrap(type)) + .getCType())); +} +MlirType +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetSfType(MlirType type) { + return wrap(static_cast( + llvm::cast( + unwrap(type)) + .getSfType())); +} +uint32_t +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetSparseMetadataFormat( + MlirType type) { + return static_cast( + llvm::cast( + unwrap(type)) + .getSparseMetadataFormat()); +} +uint32_t +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetAFragKind(MlirType type) { + return static_cast( + llvm::cast( + unwrap(type)) + .getAFragKind()); +} +int mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetVecSize( + MlirType type) { + return llvm::cast( + unwrap(type)) + .getVecSize(); +} +uint32_t mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetACollectorOp( + MlirType type) { + return static_cast( + llvm::cast( + unwrap(type)) + .getACollectorOp()); +} +uint32_t mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetBCollectorOp( + MlirType type) { + return static_cast( + llvm::cast( + unwrap(type)) + .getBCollectorOp()); +} +MlirAttribute mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetIntOverflow( + MlirType type) { + return wrap(static_cast( + llvm::cast( + unwrap(type)) + .getIntOverflow())); +} + +bool mlirTypeIsACuteNVGPUMmaAtomSM107UMMABlockScaled(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGet( + MlirContext context, MlirAttribute shapeMnk, int numCta, uint32_t aMajor, + uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, + MlirType sfType, uint32_t aFragKind, int vecSize, uint32_t aCollectorOp, + uint32_t bCollectorOp, MlirAttribute intOverflow) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::ShapeAttr shapeMnkValue; + if (!unwrapAs(ctx, shapeMnk, false, shapeMnkValue)) + return {nullptr}; + auto aMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(aMajor); + if (!aMajorValue) + return {nullptr}; + auto bMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(bMajor); + if (!bMajorValue) + return {nullptr}; + mlir::Type aTypeValue; + if (!unwrapAs(ctx, aType, false, aTypeValue)) + return {nullptr}; + mlir::Type bTypeValue; + if (!unwrapAs(ctx, bType, false, bTypeValue)) + return {nullptr}; + mlir::Type cTypeValue; + if (!unwrapAs(ctx, cType, false, cTypeValue)) + return {nullptr}; + mlir::Type sfTypeValue; + if (!unwrapAs(ctx, sfType, false, sfTypeValue)) + return {nullptr}; + auto aFragKindValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaFragKind(aFragKind); + if (!aFragKindValue) + return {nullptr}; + auto aCollectorOpValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaCollectorOp( + aCollectorOp); + if (!aCollectorOpValue) + return {nullptr}; + auto bCollectorOpValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaCollectorOp( + bCollectorOp); + if (!bCollectorOpValue) + return {nullptr}; + ::mlir::cutlass_compiler::cute_nvgpu::MMAIntOverflowAttr intOverflowValue; + if (!unwrapAs(ctx, intOverflow, true, intOverflowValue)) + return {nullptr}; + return wrap(cute_nvgpu::MmaAtomSM107UMMABlockScaledType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + shapeMnkValue, numCta, *aMajorValue, *bMajorValue, aTypeValue, bTypeValue, + cTypeValue, sfTypeValue, *aFragKindValue, vecSize, *aCollectorOpValue, + *bCollectorOpValue, intOverflowValue)); +} +MlirAttribute +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetShapeMnk(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getShapeMnk())); +} +int mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetNumCta(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumCta(); +} +uint32_t mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetAMajor(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getAMajor()); +} +uint32_t mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetBMajor(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getBMajor()); +} +MlirType mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetAType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getAType())); +} +MlirType mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetBType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getBType())); +} +MlirType mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetCType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getCType())); +} +MlirType mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetSfType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getSfType())); +} +uint32_t +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetAFragKind(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getAFragKind()); +} +int mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetVecSize(MlirType type) { + return llvm::cast(unwrap(type)) + .getVecSize(); +} +uint32_t +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetACollectorOp(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getACollectorOp()); +} +uint32_t +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetBCollectorOp(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getBCollectorOp()); +} +MlirAttribute +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetIntOverflow(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getIntOverflow())); +} + +bool mlirTypeIsACuteNVGPUMmaAtomSM107UMMASparse(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGet( + MlirContext context, MlirAttribute shapeMnk, int numCta, uint32_t aMajor, + uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, + MlirType eType, uint32_t sparseMetadataFormat, uint32_t aFragKind, + int cScaleExp, uint32_t aCollectorOp, uint32_t bCollectorOp, + MlirAttribute intOverflow) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::ShapeAttr shapeMnkValue; + if (!unwrapAs(ctx, shapeMnk, false, shapeMnkValue)) + return {nullptr}; + auto aMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(aMajor); + if (!aMajorValue) + return {nullptr}; + auto bMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(bMajor); + if (!bMajorValue) + return {nullptr}; + mlir::Type aTypeValue; + if (!unwrapAs(ctx, aType, false, aTypeValue)) + return {nullptr}; + mlir::Type bTypeValue; + if (!unwrapAs(ctx, bType, false, bTypeValue)) + return {nullptr}; + mlir::Type cTypeValue; + if (!unwrapAs(ctx, cType, false, cTypeValue)) + return {nullptr}; + mlir::Type eTypeValue; + if (!unwrapAs(ctx, eType, false, eTypeValue)) + return {nullptr}; + auto sparseMetadataFormatValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeSparseMetadataFormat( + sparseMetadataFormat); + if (!sparseMetadataFormatValue) + return {nullptr}; + auto aFragKindValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaFragKind(aFragKind); + if (!aFragKindValue) + return {nullptr}; + auto aCollectorOpValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaCollectorOp( + aCollectorOp); + if (!aCollectorOpValue) + return {nullptr}; + auto bCollectorOpValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaCollectorOp( + bCollectorOp); + if (!bCollectorOpValue) + return {nullptr}; + ::mlir::cutlass_compiler::cute_nvgpu::MMAIntOverflowAttr intOverflowValue; + if (!unwrapAs(ctx, intOverflow, true, intOverflowValue)) + return {nullptr}; + return wrap(cute_nvgpu::MmaAtomSM107UMMASparseType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + shapeMnkValue, numCta, *aMajorValue, *bMajorValue, aTypeValue, bTypeValue, + cTypeValue, eTypeValue, *sparseMetadataFormatValue, *aFragKindValue, + cScaleExp, *aCollectorOpValue, *bCollectorOpValue, intOverflowValue)); +} +MlirAttribute +mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetShapeMnk(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getShapeMnk())); +} +int mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetNumCta(MlirType type) { + return llvm::cast(unwrap(type)) + .getNumCta(); +} +uint32_t mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetAMajor(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getAMajor()); +} +uint32_t mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetBMajor(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getBMajor()); +} +MlirType mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetAType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getAType())); +} +MlirType mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetBType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getBType())); +} +MlirType mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetCType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getCType())); +} +MlirType mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetEType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getEType())); +} +uint32_t +mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetSparseMetadataFormat(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getSparseMetadataFormat()); +} +uint32_t mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetAFragKind(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getAFragKind()); +} +int mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetCScaleExp(MlirType type) { + return llvm::cast(unwrap(type)) + .getCScaleExp(); +} +uint32_t mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetACollectorOp(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getACollectorOp()); +} +uint32_t mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetBCollectorOp(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getBCollectorOp()); +} +MlirAttribute +mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetIntOverflow(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getIntOverflow())); +} + +bool mlirTypeIsACuteNVGPUMmaAtomSM107UMMA(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUMmaAtomSM107UMMATypeGet( + MlirContext context, MlirAttribute shapeMnk, int numCta, uint32_t aMajor, + uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, + uint32_t aFragKind, int cScaleExp, uint32_t aCollectorOp, + uint32_t bCollectorOp, MlirAttribute intOverflow) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::ShapeAttr shapeMnkValue; + if (!unwrapAs(ctx, shapeMnk, false, shapeMnkValue)) + return {nullptr}; + auto aMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(aMajor); + if (!aMajorValue) + return {nullptr}; + auto bMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(bMajor); + if (!bMajorValue) + return {nullptr}; + mlir::Type aTypeValue; + if (!unwrapAs(ctx, aType, false, aTypeValue)) + return {nullptr}; + mlir::Type bTypeValue; + if (!unwrapAs(ctx, bType, false, bTypeValue)) + return {nullptr}; + mlir::Type cTypeValue; + if (!unwrapAs(ctx, cType, false, cTypeValue)) + return {nullptr}; + auto aFragKindValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaFragKind(aFragKind); + if (!aFragKindValue) + return {nullptr}; + auto aCollectorOpValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaCollectorOp( + aCollectorOp); + if (!aCollectorOpValue) + return {nullptr}; + auto bCollectorOpValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaCollectorOp( + bCollectorOp); + if (!bCollectorOpValue) + return {nullptr}; + ::mlir::cutlass_compiler::cute_nvgpu::MMAIntOverflowAttr intOverflowValue; + if (!unwrapAs(ctx, intOverflow, true, intOverflowValue)) + return {nullptr}; + return wrap(cute_nvgpu::MmaAtomSM107UMMAType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + shapeMnkValue, numCta, *aMajorValue, *bMajorValue, aTypeValue, bTypeValue, + cTypeValue, *aFragKindValue, cScaleExp, *aCollectorOpValue, + *bCollectorOpValue, intOverflowValue)); +} +MlirAttribute mlirCuteNVGPUMmaAtomSM107UMMATypeGetShapeMnk(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getShapeMnk())); +} +int mlirCuteNVGPUMmaAtomSM107UMMATypeGetNumCta(MlirType type) { + return llvm::cast(unwrap(type)).getNumCta(); +} +uint32_t mlirCuteNVGPUMmaAtomSM107UMMATypeGetAMajor(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getAMajor()); +} +uint32_t mlirCuteNVGPUMmaAtomSM107UMMATypeGetBMajor(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getBMajor()); +} +MlirType mlirCuteNVGPUMmaAtomSM107UMMATypeGetAType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getAType())); +} +MlirType mlirCuteNVGPUMmaAtomSM107UMMATypeGetBType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getBType())); +} +MlirType mlirCuteNVGPUMmaAtomSM107UMMATypeGetCType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getCType())); +} +uint32_t mlirCuteNVGPUMmaAtomSM107UMMATypeGetAFragKind(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getAFragKind()); +} +int mlirCuteNVGPUMmaAtomSM107UMMATypeGetCScaleExp(MlirType type) { + return llvm::cast(unwrap(type)) + .getCScaleExp(); +} +uint32_t mlirCuteNVGPUMmaAtomSM107UMMATypeGetACollectorOp(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getACollectorOp()); +} +uint32_t mlirCuteNVGPUMmaAtomSM107UMMATypeGetBCollectorOp(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getBCollectorOp()); +} +MlirAttribute mlirCuteNVGPUMmaAtomSM107UMMATypeGetIntOverflow(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getIntOverflow())); +} + +bool mlirTypeIsACuteNVGPUMmaAtomSM120BlockScaled(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGet( + MlirContext context, MlirAttribute shapeMnk, int vecSize, MlirType aType, + MlirType bType, MlirType cType, MlirType sfType, bool useSfLayoutTV) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::ShapeAttr shapeMnkValue; + if (!unwrapAs(ctx, shapeMnk, false, shapeMnkValue)) + return {nullptr}; + mlir::Type aTypeValue; + if (!unwrapAs(ctx, aType, false, aTypeValue)) + return {nullptr}; + mlir::Type bTypeValue; + if (!unwrapAs(ctx, bType, false, bTypeValue)) + return {nullptr}; + mlir::Type cTypeValue; + if (!unwrapAs(ctx, cType, false, cTypeValue)) + return {nullptr}; + mlir::Type sfTypeValue; + if (!unwrapAs(ctx, sfType, false, sfTypeValue)) + return {nullptr}; + return wrap(cute_nvgpu::MmaAtomSM120BlockScaledType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + shapeMnkValue, vecSize, aTypeValue, bTypeValue, cTypeValue, sfTypeValue, + useSfLayoutTV)); +} +MlirAttribute +mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetShapeMnk(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getShapeMnk())); +} +int mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetVecSize(MlirType type) { + return llvm::cast(unwrap(type)) + .getVecSize(); +} +MlirType mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetAType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getAType())); +} +MlirType mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetBType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getBType())); +} +MlirType mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetCType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getCType())); +} +MlirType mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetSfType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getSfType())); +} +bool mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetUseSfLayoutTV(MlirType type) { + return llvm::cast(unwrap(type)) + .getUseSfLayout_TV(); +} + +bool mlirTypeIsACuteNVGPUMmaAtomSM80Sparse(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUMmaAtomSM80SparseTypeGet( + MlirContext context, MlirAttribute shapeMnk, MlirType aType, MlirType bType, + MlirType cType, uint32_t sparseMetadataFormat, MlirAttribute intOverflow) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::ShapeAttr shapeMnkValue; + if (!unwrapAs(ctx, shapeMnk, false, shapeMnkValue)) + return {nullptr}; + mlir::Type aTypeValue; + if (!unwrapAs(ctx, aType, false, aTypeValue)) + return {nullptr}; + mlir::Type bTypeValue; + if (!unwrapAs(ctx, bType, false, bTypeValue)) + return {nullptr}; + mlir::Type cTypeValue; + if (!unwrapAs(ctx, cType, false, cTypeValue)) + return {nullptr}; + auto sparseMetadataFormatValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeSparseMetadataFormat( + sparseMetadataFormat); + if (!sparseMetadataFormatValue) + return {nullptr}; + ::mlir::cutlass_compiler::cute_nvgpu::MMAIntOverflowAttr intOverflowValue; + if (!unwrapAs(ctx, intOverflow, true, intOverflowValue)) + return {nullptr}; + return wrap(cute_nvgpu::MmaAtomSM80SparseType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + shapeMnkValue, aTypeValue, bTypeValue, cTypeValue, + *sparseMetadataFormatValue, intOverflowValue)); +} +MlirAttribute mlirCuteNVGPUMmaAtomSM80SparseTypeGetShapeMnk(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getShapeMnk())); +} +MlirType mlirCuteNVGPUMmaAtomSM80SparseTypeGetAType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getAType())); +} +MlirType mlirCuteNVGPUMmaAtomSM80SparseTypeGetBType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getBType())); +} +MlirType mlirCuteNVGPUMmaAtomSM80SparseTypeGetCType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getCType())); +} +uint32_t +mlirCuteNVGPUMmaAtomSM80SparseTypeGetSparseMetadataFormat(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)) + .getSparseMetadataFormat()); +} +MlirAttribute mlirCuteNVGPUMmaAtomSM80SparseTypeGetIntOverflow(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getIntOverflow())); +} + +bool mlirTypeIsACuteNVGPUMmaAtomSM80(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUMmaAtomSM80TypeGet(MlirContext context, + MlirAttribute shapeMnk, MlirType aType, + MlirType bType, MlirType cType, + MlirAttribute intOverflow, + MlirAttribute binaryOp) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::ShapeAttr shapeMnkValue; + if (!unwrapAs(ctx, shapeMnk, false, shapeMnkValue)) + return {nullptr}; + mlir::Type aTypeValue; + if (!unwrapAs(ctx, aType, false, aTypeValue)) + return {nullptr}; + mlir::Type bTypeValue; + if (!unwrapAs(ctx, bType, false, bTypeValue)) + return {nullptr}; + mlir::Type cTypeValue; + if (!unwrapAs(ctx, cType, false, cTypeValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute_nvgpu::MMAIntOverflowAttr intOverflowValue; + if (!unwrapAs(ctx, intOverflow, true, intOverflowValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute_nvgpu::BinaryOpAttr binaryOpValue; + if (!unwrapAs(ctx, binaryOp, true, binaryOpValue)) + return {nullptr}; + return wrap(cute_nvgpu::MmaAtomSM80Type::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + shapeMnkValue, aTypeValue, bTypeValue, cTypeValue, intOverflowValue, + binaryOpValue)); +} +MlirAttribute mlirCuteNVGPUMmaAtomSM80TypeGetShapeMnk(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getShapeMnk())); +} +MlirType mlirCuteNVGPUMmaAtomSM80TypeGetAType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getAType())); +} +MlirType mlirCuteNVGPUMmaAtomSM80TypeGetBType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getBType())); +} +MlirType mlirCuteNVGPUMmaAtomSM80TypeGetCType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getCType())); +} +MlirAttribute mlirCuteNVGPUMmaAtomSM80TypeGetIntOverflow(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getIntOverflow())); +} +MlirAttribute mlirCuteNVGPUMmaAtomSM80TypeGetBinaryOp(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getBinaryOp())); +} + +bool mlirTypeIsACuteNVGPUMmaAtomSM89(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUMmaAtomSM89TypeGet(MlirContext context, + MlirAttribute shapeMnk, MlirType aType, + MlirType bType, MlirType cType) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::ShapeAttr shapeMnkValue; + if (!unwrapAs(ctx, shapeMnk, false, shapeMnkValue)) + return {nullptr}; + mlir::Type aTypeValue; + if (!unwrapAs(ctx, aType, false, aTypeValue)) + return {nullptr}; + mlir::Type bTypeValue; + if (!unwrapAs(ctx, bType, false, bTypeValue)) + return {nullptr}; + mlir::Type cTypeValue; + if (!unwrapAs(ctx, cType, false, cTypeValue)) + return {nullptr}; + return wrap(cute_nvgpu::MmaAtomSM89Type::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + shapeMnkValue, aTypeValue, bTypeValue, cTypeValue)); +} +MlirAttribute mlirCuteNVGPUMmaAtomSM89TypeGetShapeMnk(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getShapeMnk())); +} +MlirType mlirCuteNVGPUMmaAtomSM89TypeGetAType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getAType())); +} +MlirType mlirCuteNVGPUMmaAtomSM89TypeGetBType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getBType())); +} +MlirType mlirCuteNVGPUMmaAtomSM89TypeGetCType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getCType())); +} + +bool mlirTypeIsACuteNVGPUMmaAtomSM90(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUMmaAtomSM90TypeGet( + MlirContext context, MlirAttribute shapeMnk, uint32_t aMajor, + uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, + uint32_t aFragKind, MlirAttribute intOverflow, bool aNeg, bool bNeg) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::ShapeAttr shapeMnkValue; + if (!unwrapAs(ctx, shapeMnk, false, shapeMnkValue)) + return {nullptr}; + auto aMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(aMajor); + if (!aMajorValue) + return {nullptr}; + auto bMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(bMajor); + if (!bMajorValue) + return {nullptr}; + mlir::Type aTypeValue; + if (!unwrapAs(ctx, aType, false, aTypeValue)) + return {nullptr}; + mlir::Type bTypeValue; + if (!unwrapAs(ctx, bType, false, bTypeValue)) + return {nullptr}; + mlir::Type cTypeValue; + if (!unwrapAs(ctx, cType, false, cTypeValue)) + return {nullptr}; + auto aFragKindValue = + ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaFragKind(aFragKind); + if (!aFragKindValue) + return {nullptr}; + ::mlir::cutlass_compiler::cute_nvgpu::MMAIntOverflowAttr intOverflowValue; + if (!unwrapAs(ctx, intOverflow, true, intOverflowValue)) + return {nullptr}; + return wrap(cute_nvgpu::MmaAtomSM90Type::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + shapeMnkValue, *aMajorValue, *bMajorValue, aTypeValue, bTypeValue, + cTypeValue, *aFragKindValue, intOverflowValue, + aNeg ? mlir::UnitAttr::get(ctx) : mlir::UnitAttr(), + bNeg ? mlir::UnitAttr::get(ctx) : mlir::UnitAttr())); +} +MlirAttribute mlirCuteNVGPUMmaAtomSM90TypeGetShapeMnk(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getShapeMnk())); +} +uint32_t mlirCuteNVGPUMmaAtomSM90TypeGetAMajor(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getAMajor()); +} +uint32_t mlirCuteNVGPUMmaAtomSM90TypeGetBMajor(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getBMajor()); +} +MlirType mlirCuteNVGPUMmaAtomSM90TypeGetAType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getAType())); +} +MlirType mlirCuteNVGPUMmaAtomSM90TypeGetBType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getBType())); +} +MlirType mlirCuteNVGPUMmaAtomSM90TypeGetCType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getCType())); +} +uint32_t mlirCuteNVGPUMmaAtomSM90TypeGetAFragKind(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getAFragKind()); +} +MlirAttribute mlirCuteNVGPUMmaAtomSM90TypeGetIntOverflow(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getIntOverflow())); +} +bool mlirCuteNVGPUMmaAtomSM90TypeGetANeg(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getANeg()); +} +bool mlirCuteNVGPUMmaAtomSM90TypeGetBNeg(MlirType type) { + return static_cast( + llvm::cast(unwrap(type)).getBNeg()); +} + +bool mlirTypeIsACuteNVGPUSmemDescCircularSM103(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType +mlirCuteNVGPUSmemDescCircularSM103TypeGet(MlirContext context, + MlirAttribute blockLayoutBytes) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::LayoutAttr blockLayoutBytesValue; + if (!unwrapAs(ctx, blockLayoutBytes, false, blockLayoutBytesValue)) + return {nullptr}; + return wrap(cute_nvgpu::SmemDescCircularSM103Type::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + blockLayoutBytesValue)); +} +MlirAttribute +mlirCuteNVGPUSmemDescCircularSM103TypeGetBlockLayoutBytes(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getBlockLayoutBytes())); +} + +bool mlirTypeIsACuteNVGPUSmemDescCircularSM107(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType +mlirCuteNVGPUSmemDescCircularSM107TypeGet(MlirContext context, + MlirAttribute blockLayoutBytes) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::LayoutAttr blockLayoutBytesValue; + if (!unwrapAs(ctx, blockLayoutBytes, false, blockLayoutBytesValue)) + return {nullptr}; + return wrap(cute_nvgpu::SmemDescCircularSM107Type::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + blockLayoutBytesValue)); +} +MlirAttribute +mlirCuteNVGPUSmemDescCircularSM107TypeGetBlockLayoutBytes(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getBlockLayoutBytes())); +} + +bool mlirTypeIsACuteNVGPUSmemDescSM107(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUSmemDescSM107TypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cute_nvgpu::SmemDescSM107Type::get(ctx)); +} + +bool mlirTypeIsACuteNVGPUSmemDesc(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUSmemDescTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cute_nvgpu::SmemDescType::get(ctx)); +} + +bool mlirTypeIsACuteNVGPUSmemDescView(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUSmemDescViewTypeGet(MlirContext context, MlirType desc, + MlirAttribute layout) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + mlir::Type descValue; + if (!unwrapAs(ctx, desc, false, descValue)) + return {nullptr}; + ::mlir::cutlass_compiler::cute::LayoutAttr layoutValue; + if (!unwrapAs(ctx, layout, false, layoutValue)) + return {nullptr}; + return wrap(cute_nvgpu::SmemDescViewType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + descValue, layoutValue)); +} +MlirType mlirCuteNVGPUSmemDescViewTypeGetDesc(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getDesc())); +} +MlirAttribute mlirCuteNVGPUSmemDescViewTypeGetLayout(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getLayout())); +} + +bool mlirTypeIsACuteNVGPUTmaDescriptorIm2Col(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUTmaDescriptorIm2ColTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cute_nvgpu::TmaDescriptorIm2ColType::get(ctx)); +} + +bool mlirTypeIsACuteNVGPUTmaDescriptorTiled(MlirType type) { + return llvm::isa_and_nonnull( + unwrap(type)); +} +MlirType mlirCuteNVGPUTmaDescriptorTiledTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cute_nvgpu::TmaDescriptorTiledType::get(ctx)); +} + +bool mlirTypeIsACuteNVGPUUniversalFmaAtom(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUUniversalFmaAtomTypeGet(MlirContext context, + MlirAttribute shapeMnk, + MlirType aType, MlirType bType, + MlirType cType) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::cutlass_compiler::cute::ShapeAttr shapeMnkValue; + if (!unwrapAs(ctx, shapeMnk, false, shapeMnkValue)) + return {nullptr}; + mlir::Type aTypeValue; + if (!unwrapAs(ctx, aType, false, aTypeValue)) + return {nullptr}; + mlir::Type bTypeValue; + if (!unwrapAs(ctx, bType, false, bTypeValue)) + return {nullptr}; + mlir::Type cTypeValue; + if (!unwrapAs(ctx, cType, false, cTypeValue)) + return {nullptr}; + return wrap(cute_nvgpu::UniversalFmaAtomType::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + shapeMnkValue, aTypeValue, bTypeValue, cTypeValue)); +} +MlirAttribute mlirCuteNVGPUUniversalFmaAtomTypeGetShapeMnk(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)) + .getShapeMnk())); +} +MlirType mlirCuteNVGPUUniversalFmaAtomTypeGetAType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getAType())); +} +MlirType mlirCuteNVGPUUniversalFmaAtomTypeGetBType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getBType())); +} +MlirType mlirCuteNVGPUUniversalFmaAtomTypeGetCType(MlirType type) { + return wrap(static_cast( + llvm::cast(unwrap(type)).getCType())); +} + +bool mlirTypeIsACuteNVGPUWorkIdResponse(MlirType type) { + return llvm::isa_and_nonnull(unwrap(type)); +} +MlirType mlirCuteNVGPUWorkIdResponseTypeGet(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(cute_nvgpu::WorkIdResponseType::get(ctx)); +} diff --git a/mlir/dialects/cute_nvgpu/capi.h b/mlir/dialects/cute_nvgpu/capi.h new file mode 100644 index 00000000..f1e4e999 --- /dev/null +++ b/mlir/dialects/cute_nvgpu/capi.h @@ -0,0 +1,1481 @@ +//===- capi.h - C API for the cute_nvgpu dialect ------------------*- C -*-===// + +#ifndef MLIR_DIALECTS_CUTE_NVGPU_CAPI_H +#define MLIR_DIALECTS_CUTE_NVGPU_CAPI_H + +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Registration + +MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(CuteNVGPU, cute_nvgpu); + +// Attributes + +// Constructors return null for invalid parameters or handles of another +// context; optional parameters take null handles, and std::optional enums a +// negative value. Enums are their integer values (the dialect's *Enums.td). +// Getters require a value accepted by the matching IsA. + +// `#cute_nvgpu.atom_copy_field_bulkg2s`: Fields stored in Bulk Load Copy Atom +// type. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomCopyFieldBulkCopyG2S(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUAtomCopyFieldBulkCopyG2SAttrGet( + MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomCopyFieldBulkCopyG2SAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_copy_field_bulks2g`: Fields stored in Bulk Store Copy Atom +// type. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomCopyFieldBulkCopyS2G(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUAtomCopyFieldBulkCopyS2GAttrGet( + MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomCopyFieldBulkCopyS2GAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_copy_field_bulks2s`: Fields stored in Bulk CTA to Cluster +// Copy Atom type. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomCopyFieldBulkCopyS2S(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUAtomCopyFieldBulkCopyS2SAttrGet( + MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomCopyFieldBulkCopyS2SAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_copy_field_dsmem_store`: Fields stored in +// CopyAtomDsmemStoreType. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomCopyFieldDsmemStore(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUAtomCopyFieldDsmemStoreAttrGet( + MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomCopyFieldDsmemStoreAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_copy_field_g2r`: Fields stored in CopyAtomG2RType. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomCopyFieldLoadGlobal(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUAtomCopyFieldLoadGlobalAttrGet( + MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomCopyFieldLoadGlobalAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_copy_field_non_exec_2d_gather4_tma_load`: Fields stored in +// CopyAtomNonExec2DGather4TmaLoadType. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomCopyFieldNonExec2DGather4TmaLoad( + MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUAtomCopyFieldNonExec2DGather4TmaLoadAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomCopyFieldNonExec2DGather4TmaLoadAttrGetValue( + MlirAttribute attr); + +// `#cute_nvgpu.atom_copy_field_non_exec_2d_scatter4_tma_store`: Fields stored +// in CopyAtomNonExec2DScatter4TmaStoreType. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomCopyFieldNonExec2DScatter4TmaStore( + MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUAtomCopyFieldNonExec2DScatter4TmaStoreAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomCopyFieldNonExec2DScatter4TmaStoreAttrGetValue( + MlirAttribute attr); + +// `#cute_nvgpu.atom_copy_field_non_exec_im2col_tma_load`: Fields stored in +// CopyAtomNonExecIm2ColTmaLoadType. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecIm2ColTmaLoad(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaLoadAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaLoadAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_copy_field_non_exec_im2col_tma_store`: Fields stored in +// CopyAtomNonExecIm2ColTmaStoreType. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecIm2ColTmaStore(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaStoreAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaStoreAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_copy_field_non_exec_tma_load`: Fields stored in +// CopyAtomNonExecTiledTmaLoadType. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecTiledTmaLoad(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaLoadAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaLoadAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_copy_field_non_exec_tma_reduce`: Fields stored in +// CopyAtomNonExecTiledTmaReduceType. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecTiledTmaReduce(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaReduceAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaReduceAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_copy_field_non_exec_tma_store`: Fields stored in +// CopyAtomNonExecTiledTmaStoreType. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecTiledTmaStore(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaStoreAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaStoreAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_copy_field_r2g`: Fields stored in CopyAtomR2GType. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomCopyFieldStoreGlobal(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUAtomCopyFieldStoreGlobalAttrGet( + MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomCopyFieldStoreGlobalAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_copy_field_tmaload`: Fields stored in Tma Load Copy Atom +// type. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomCopyFieldTmaLoad(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUAtomCopyFieldTmaLoadAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomCopyFieldTmaLoadAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_copy_field_tmareduce`: Fields stored in Tma Reduce Copy +// Atom type. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomCopyFieldTmaReduce(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUAtomCopyFieldTmaReduceAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomCopyFieldTmaReduceAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_copy_field_tmastore`: Fields stored in Tma Store Copy Atom +// type. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomCopyFieldTmaStore(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUAtomCopyFieldTmaStoreAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomCopyFieldTmaStoreAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_mma_field_sm100`: Fields stored in MMA Atom types for +// SM100. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomMmaFieldSM100(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUAtomMmaFieldSM100AttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomMmaFieldSM100AttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_mma_field_sm100_block_scaled`: Fields stored in MMA Atom +// types for SM100 (block scaled). +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomMmaFieldSM100BlockScaled(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUAtomMmaFieldSM100BlockScaledAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomMmaFieldSM100BlockScaledAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_mma_field_sm100_block_scaled_sparse`: Fields stored in MMA +// Atom types for SM100 (block scaled). +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomMmaFieldSM100BlockScaledSparse(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUAtomMmaFieldSM100BlockScaledSparseAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomMmaFieldSM100BlockScaledSparseAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_mma_field_sm100_sparse`: Fields stored in sparse MMA Atom +// types for SM100. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomMmaFieldSM100Sparse(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUAtomMmaFieldSM100SparseAttrGet( + MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomMmaFieldSM100SparseAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_mma_field_sm120_block_scaled`: Fields stored in MMA Atom +// types for SM120 (block scaled). +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomMmaFieldSM120BlockScaled(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUAtomMmaFieldSM120BlockScaledAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomMmaFieldSM120BlockScaledAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_mma_field_sm80_sparse`: Fields stored in MMA Atom types for +// SM80. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomMmaFieldSM80Sparse(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUAtomMmaFieldSM80SparseAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomMmaFieldSM80SparseAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.atom_mma_field_sm90`: Fields stored in MMA Atom types for SM90. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUAtomMmaFieldSM90(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUAtomMmaFieldSM90AttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUAtomMmaFieldSM90AttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.bin_op`: Binary operation for single-bit MMA operations. +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVGPUBinaryOp(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUBinaryOpAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUBinaryOpAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.copy_s2t_broadcast_mode`: Broadcast modes for the different +// utccp instructions. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUCopyS2TBroadcast(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUCopyS2TBroadcastAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyS2TBroadcastAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.gather_scatter_tma_load`: The various kinds of TMA loads in +// gather/scatter mode. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUGatherScatterTmaLoad(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUGatherScatterTmaLoadAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUGatherScatterTmaLoadAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.im2col_tma_load`: The various kinds of TMA loads in im2col mode. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUIm2ColTmaLoad(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUIm2ColTmaLoadAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUIm2ColTmaLoadAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.ld_reduce_acc_precision_kind`: multimem ld_reduce accumulation +// precision kind. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPULdReduceAccPrecisionKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPULdReduceAccPrecisionKindAttrGet( + MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPULdReduceAccPrecisionKindAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.ldsm_sz_pattern`: LDSM's sz pattern, describing the bit size. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPULdsmSzPattern(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPULdsmSzPatternAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPULdsmSzPatternAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.load_cache_mode`: Cache modes for the load instructions. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPULoadCacheMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPULoadCacheModeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPULoadCacheModeAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.mma_int_overflow`: MMA overflow options. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUMMAIntOverflow(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMMAIntOverflowAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMMAIntOverflowAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.major`: Major mode for MMA operations. +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVGPUMajorMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMajorModeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMajorModeAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.mma_collector_op`: Enums for the mma collector op. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUMmaCollectorOp(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaCollectorOpAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaCollectorOpAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.mma_frag_kind`: Enums for the mma frag type. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUMmaFragKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaFragKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaFragKindAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.not_implemented_frg`: Fragment of an atom that has none. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUNotImplementedFrg(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUNotImplementedFrgAttrGet(MlirContext context); + +// `#cute_nvgpu.tma_reduce_kind`: Op for the TMASTORE instruction. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUReductionKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUReductionKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUReductionKindAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.rmem_frg`: Register-memory fragment of an MMA operand. +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVGPURmemFrg(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPURmemFrgAttrGet(MlirContext context, MlirType valueType, + uint32_t operand, bool deriveElemType); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPURmemFrgAttrGetValueType(MlirAttribute attr); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPURmemFrgAttrGetOperand(MlirAttribute attr); +MLIR_CAPI_EXPORTED bool +mlirCuteNVGPURmemFrgAttrGetDeriveElemType(MlirAttribute attr); + +// `#cute_nvgpu.arch.sm100.circular_smem_frg`: SM100 circular shared-memory +// fragment. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUSM100CircularSmemFrg(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUSM100CircularSmemFrgAttrGet( + MlirContext context, uint32_t majorMode); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUSM100CircularSmemFrgAttrGetMajorMode(MlirAttribute attr); + +// `#cute_nvgpu.arch.sm100.smem_frg`: SM100 shared-memory fragment. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUSM100SmemFrg(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUSM100SmemFrgAttrGet(MlirContext context, uint32_t majorMode); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUSM100SmemFrgAttrGetMajorMode(MlirAttribute attr); + +// `#cute_nvgpu.arch.sm100.tmem_e_frg`: SM100 tensor-memory sparse-metadata +// fragment. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUSM100TmemEFrg(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUSM100TmemEFrgAttrGet( + MlirContext context, MlirType aType, MlirType eType); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUSM100TmemEFrgAttrGetAType(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUSM100TmemEFrgAttrGetEType(MlirAttribute attr); + +// `#cute_nvgpu.arch.sm100.tmem_frg`: SM100 tensor-memory fragment. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUSM100TmemFrg(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUSM100TmemFrgAttrGet( + MlirContext context, MlirType dataType, MlirType storageType, int ctaGroup, + uint32_t tmemAllocMode); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUSM100TmemFrgAttrGetDataType(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUSM100TmemFrgAttrGetStorageType(MlirAttribute attr); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUSM100TmemFrgAttrGetCtaGroup(MlirAttribute attr); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUSM100TmemFrgAttrGetTmemAllocMode(MlirAttribute attr); + +// `#cute_nvgpu.arch.sm100.tmem_sf_frg`: SM100 tensor-memory scale-factor +// fragment. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUSM100TmemSfFrg(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUSM100TmemSfFrgAttrGet( + MlirContext context, MlirType sfType, int sfVecSize, int ctaGroup, + bool isSfa, uint32_t tmemAllocMode); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUSM100TmemSfFrgAttrGetSfType(MlirAttribute attr); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUSM100TmemSfFrgAttrGetSfVecSize(MlirAttribute attr); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUSM100TmemSfFrgAttrGetCtaGroup(MlirAttribute attr); +MLIR_CAPI_EXPORTED bool +mlirCuteNVGPUSM100TmemSfFrgAttrGetIsSfa(MlirAttribute attr); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUSM100TmemSfFrgAttrGetTmemAllocMode(MlirAttribute attr); + +// `#cute_nvgpu.arch.sm107.smem_frg`: SM107 shared-memory fragment. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUSM107SmemFrg(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUSM107SmemFrgAttrGet(MlirContext context, uint32_t majorMode); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUSM107SmemFrgAttrGetMajorMode(MlirAttribute attr); + +// `#cute_nvgpu.arch.sm90.smem_frg`: SM90 shared-memory fragment. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUSM90SmemFrg(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUSM90SmemFrgAttrGet(MlirContext context, uint32_t majorMode); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUSM90SmemFrgAttrGetMajorMode(MlirAttribute attr); + +// `#cute_nvgpu.tiled_tma_load`: The various kinds of TMA loads in tiled mode. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUTiledTmaLoad(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUTiledTmaLoadAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUTiledTmaLoadAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.tma_data_format`: Bits 74-71 of the TMA descriptor. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUTmaDataFormat(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUTmaDataFormatAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUTmaDataFormatAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.tma_load_mode`: Modes for the TMA load instruction. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUTmaLoadMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUTmaLoadModeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUTmaLoadModeAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.tma_store_mode`: Modes for the TMA store instruction. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUTmaStoreMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUTmaStoreModeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUTmaStoreModeAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.tmem_alloc_mode`: Modes for the Tmem allocation. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUTmemAllocMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUTmemAllocModeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUTmemAllocModeAttrGetValue(MlirAttribute attr); + +// `#cute_nvgpu.tmem_load_red_op`: Reduce operation for the Tmem load +// instruction. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVGPUTmemLoadRedOp(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUTmemLoadRedOpAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUTmemLoadRedOpAttrGetValue(MlirAttribute attr); + +// Types + +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUType(MlirType type); + +// Constructors return null for invalid parameters or handles of another +// context; optional parameters take null handles, and std::optional enums a +// negative value. Enums are their integer values (the dialect's *Enums.td). +// Getters require a value accepted by the matching IsA. + +// `!cute_nvgpu.atom.bulk_copy_g2s`: Bulk (TMA non-tensor) copy atom. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomBulkCopyG2S(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomBulkCopyG2STypeGet( + MlirContext context, MlirType valType, int copyBits, bool mcast); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomBulkCopyG2STypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomBulkCopyG2STypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED bool +mlirCuteNVGPUCopyAtomBulkCopyG2STypeGetMcast(MlirType type); + +// `!cute_nvgpu.atom.bulk_copy_s2g`: Bulk (TMA non-tensor) copy atom. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomBulkCopyS2G(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomBulkCopyS2GTypeGet( + MlirContext context, MlirType valType, int copyBits, bool mask); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomBulkCopyS2GTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomBulkCopyS2GTypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED bool +mlirCuteNVGPUCopyAtomBulkCopyS2GTypeGetMask(MlirType type); + +// `!cute_nvgpu.atom.bulk_copy_s2s`: Bulk (TMA non-tensor) copy atom. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomBulkCopyS2S(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomBulkCopyS2STypeGet( + MlirContext context, MlirType valType, int copyBits); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomBulkCopyS2STypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomBulkCopyS2STypeGetCopyBits(MlirType type); + +// `!cute_nvgpu.atom.dsmem_store`: Distributed shared-memory store atom. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomDsmemStore(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomDsmemStoreTypeGet( + MlirContext context, MlirType valType, int copyBits); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomDsmemStoreTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomDsmemStoreTypeGetCopyBits(MlirType type); + +// `!cute_nvgpu.atom.g2r`: Global-to-register load atom. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomG2R(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomG2RTypeGet( + MlirContext context, MlirType valType, int copyBits, uint32_t memOrder, + uint32_t memScope, uint32_t l2PrefetchSize, uint32_t l1CacheEvictPriority, + uint32_t loadCacheMode, uint32_t sharedSpace, bool invariant); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomG2RTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int mlirCuteNVGPUCopyAtomG2RTypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomG2RTypeGetMemOrder(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomG2RTypeGetMemScope(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomG2RTypeGetL2PrefetchSize(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomG2RTypeGetL1CacheEvictPriority(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomG2RTypeGetLoadCacheMode(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomG2RTypeGetSharedSpace(MlirType type); +MLIR_CAPI_EXPORTED bool mlirCuteNVGPUCopyAtomG2RTypeGetInvariant(MlirType type); + +// `!cute_nvgpu.atom.im2col_tma_load`: Executable im2col TMA load atom. +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUCopyAtomIm2ColTmaLoad(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGet( + MlirContext context, MlirType valType, int copyBits, int numCta, + MlirType gStride, bool mcast, MlirType tmaGbasis); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetNumCta(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetGStride(MlirType type); +MLIR_CAPI_EXPORTED bool +mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetMcast(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetTmaGbasis(MlirType type); + +// `!cute_nvgpu.atom.im2col_tma_store`: Executable im2col TMA store atom. +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUCopyAtomIm2ColTmaStore(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGet( + MlirContext context, MlirType valType, int copyBits, MlirType gStride, + MlirType tmaGbasis); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGetGStride(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGetTmaGbasis(MlirType type); + +// `!cute_nvgpu.atom.ldsm`: ldmatrix atom. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomLdsm(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomLdsmTypeGet( + MlirContext context, MlirType valType, MlirAttribute mode, + uint32_t szPattern, int numMatrices, bool transpose); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomLdsmTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUCopyAtomLdsmTypeGetMode(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomLdsmTypeGetSzPattern(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomLdsmTypeGetNumMatrices(MlirType type); +MLIR_CAPI_EXPORTED bool +mlirCuteNVGPUCopyAtomLdsmTypeGetTranspose(MlirType type); + +// `!cute_nvgpu.atom.non_exec_2d_gather4_tma_load`: Non-executable 2-D gather4 +// TMA load atom. +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUCopyAtomNonExec2DGather4TmaLoad(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGet( + MlirContext context, uint32_t kind, MlirType valType, int copyBits, + MlirType tmaGbasis, int64_t tmaFormat); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetKind(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetTmaGbasis(MlirType type); +MLIR_CAPI_EXPORTED int64_t +mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetTmaFormat(MlirType type); + +// `!cute_nvgpu.atom.non_exec_2d_scatter4_tma_store`: Non-executable 2-D +// scatter4 TMA store atom. +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUCopyAtomNonExec2DScatter4TmaStore(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGet(MlirContext context, + MlirType valType, + int copyBits, + MlirType tmaGbasis, + int64_t tmaFormat); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGetTmaGbasis(MlirType type); +MLIR_CAPI_EXPORTED int64_t +mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGetTmaFormat(MlirType type); + +// `!cute_nvgpu.atom.non_exec_im2col_tma_load`: Non-executable im2col TMA load +// atom. +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUCopyAtomNonExecIm2ColTmaLoad(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGet( + MlirContext context, uint32_t kind, MlirType valType, int copyBits, + MlirType tmaGbasis, int64_t tmaFormat); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetKind(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetTmaGbasis(MlirType type); +MLIR_CAPI_EXPORTED int64_t +mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetTmaFormat(MlirType type); + +// `!cute_nvgpu.atom.non_exec_im2col_tma_store`: Non-executable im2col TMA store +// atom. +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUCopyAtomNonExecIm2ColTmaStore(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGet( + MlirContext context, MlirType valType, int copyBits, MlirType tmaGbasis, + int64_t tmaFormat); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGetTmaGbasis(MlirType type); +MLIR_CAPI_EXPORTED int64_t +mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGetTmaFormat(MlirType type); + +// `!cute_nvgpu.atom.non_exec_tiled_tma_load`: Non-executable tiled TMA load +// atom. +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUCopyAtomNonExecTiledTmaLoad(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGet( + MlirContext context, uint32_t kind, MlirType valType, int copyBits, + MlirType tmaGbasis, int64_t tmaFormat); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetKind(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetTmaGbasis(MlirType type); +MLIR_CAPI_EXPORTED int64_t +mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetTmaFormat(MlirType type); + +// `!cute_nvgpu.atom.non_exec_tiled_tma_reduce`: Non-executable tiled TMA reduce +// atom. +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUCopyAtomNonExecTiledTmaReduce(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGet( + MlirContext context, uint32_t kind, MlirType valType, int copyBits, + MlirType tmaGbasis, int64_t tmaFormat); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetKind(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetTmaGbasis(MlirType type); +MLIR_CAPI_EXPORTED int64_t +mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetTmaFormat(MlirType type); + +// `!cute_nvgpu.atom.non_exec_tiled_tma_store`: Non-executable tiled TMA store +// atom. +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUCopyAtomNonExecTiledTmaStore(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGet( + MlirContext context, MlirType valType, int copyBits, MlirType tmaGbasis, + int64_t tmaFormat); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGetTmaGbasis(MlirType type); +MLIR_CAPI_EXPORTED int64_t +mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGetTmaFormat(MlirType type); + +// `!cute_nvgpu.atom.r2g`: Register-to-global store atom. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomR2G(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomR2GTypeGet( + MlirContext context, MlirType valType, int copyBits, uint32_t memOrder, + uint32_t memScope, uint32_t l1CacheEvictPriority, uint32_t storeCacheMode, + uint32_t sharedSpace); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomR2GTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int mlirCuteNVGPUCopyAtomR2GTypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomR2GTypeGetMemOrder(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomR2GTypeGetMemScope(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomR2GTypeGetL1CacheEvictPriority(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomR2GTypeGetStoreCacheMode(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomR2GTypeGetSharedSpace(MlirType type); + +// `!cute_nvgpu.atom.r2s`: Register-to-shared store atom. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomR2S(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomR2STypeGet( + MlirContext context, MlirType valType, int copyBits, uint32_t memOrder, + uint32_t memScope, uint32_t sharedSpace); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomR2STypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int mlirCuteNVGPUCopyAtomR2STypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomR2STypeGetMemOrder(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomR2STypeGetMemScope(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomR2STypeGetSharedSpace(MlirType type); + +// `!cute_nvgpu.atom.s2r`: Shared-to-register load atom. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomS2R(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomS2RTypeGet( + MlirContext context, MlirType valType, int copyBits, uint32_t memOrder, + uint32_t memScope, uint32_t sharedSpace); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomS2RTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int mlirCuteNVGPUCopyAtomS2RTypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomS2RTypeGetMemOrder(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomS2RTypeGetMemScope(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomS2RTypeGetSharedSpace(MlirType type); + +// `!cute_nvgpu.atom.simt_async_copy`: cp.async copy atom. +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUCopyAtomSIMTAsyncCopy(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGet( + MlirContext context, MlirType valType, uint32_t cache, int copyBits); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGetCache(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGetCopyBits(MlirType type); + +// `!cute_nvgpu.atom.simt_multimem_ld_reduce`: multimem.ld_reduce atom. +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUCopyAtomSIMTMultimemLdReduce(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGet( + MlirContext context, MlirType valType, int copyBits, uint32_t reduction, + MlirAttribute ldReduceAccPrecision, uint32_t memOrder, uint32_t memScope); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetReduction(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetLdReduceAccPrecision( + MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetMemOrder(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetMemScope(MlirType type); + +// `!cute_nvgpu.atom.simt_multimem_red`: multimem.red atom. +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUCopyAtomSIMTMultimemRed(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGet( + MlirContext context, MlirType valType, int copyBits, uint32_t reduction, + uint32_t memOrder, uint32_t memScope); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetReduction(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetMemOrder(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetMemScope(MlirType type); + +// `!cute_nvgpu.atom.simt_multimem_st`: multimem.st atom. +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUCopyAtomSIMTMultimemSt(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGet( + MlirContext context, MlirType valType, int copyBits, uint32_t memOrder, + uint32_t memScope); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetMemOrder(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetMemScope(MlirType type); + +// `!cute_nvgpu.atom.universal_copy`: Universal (SIMT) copy atom. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomSIMTSyncCopy(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGet( + MlirContext context, MlirType valType, int copyBits, uint32_t srcSpace, + uint32_t dstSpace); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetSrcSpace(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetDstSpace(MlirType type); + +// `!cute_nvgpu.atom.s2t_copy`: tcgen05.cp (shared to tensor memory) atom. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomSM100CopyS2T(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGet( + MlirContext context, MlirType valType, int numDp, int numBit, int numCta, + uint32_t broadcast); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetNumDp(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetNumBit(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetNumCta(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetBroadcast(MlirType type); + +// `!cute_nvgpu.atom.sm100_s2t_copy_v2`: tcgen05.cp atom with the shared-memory +// major mode. +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUCopyAtomSM100S2TCopyV2(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGet( + MlirContext context, MlirType valType, int numDp, int numBit, int numCta, + uint32_t smemMajor, uint32_t broadcast); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetNumDp(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetNumBit(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetNumCta(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetSmemMajor(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetBroadcast(MlirType type); + +// `!cute_nvgpu.atom.tmem_load`: tcgen05.ld atom. +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUCopyAtomSM100TmemLoad(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGet( + MlirContext context, MlirType valType, int numDp, int numBit, + uint32_t numRep, bool pack16b); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetNumDp(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetNumBit(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetNumRep(MlirType type); +MLIR_CAPI_EXPORTED bool +mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetPack16b(MlirType type); + +// `!cute_nvgpu.atom.tmem_store`: tcgen05.st atom. +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUCopyAtomSM100TmemStore(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGet( + MlirContext context, MlirType valType, int numDp, int numBit, + uint32_t numRep, bool expand16b); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetNumDp(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetNumBit(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetNumRep(MlirType type); +MLIR_CAPI_EXPORTED bool +mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetExpand16b(MlirType type); + +// `!cute_nvgpu.atom.tmem_load_spcompress`: SM107 tensor-memory load with sparse +// compression. +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUCopyAtomSM107TmemLoadSPCompress(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGet( + MlirContext context, MlirType valType, int numDp, int numBit, + uint32_t numRep, uint32_t redOp, bool red, bool nan); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetNumDp(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetNumBit(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetNumRep(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetRedOp(MlirType type); +MLIR_CAPI_EXPORTED bool +mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetRed(MlirType type); +MLIR_CAPI_EXPORTED bool +mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetNan(MlirType type); + +// `!cute_nvgpu.atom.tmem_load_red`: tcgen05.ld.red atom. +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUCopyAtomSM10xTmemLoadRed(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGet( + MlirContext context, MlirType valType, int numDp, int numBit, + uint32_t numRep, uint32_t redOp, bool nan, MlirAttribute halfSplitOff); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetNumDp(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetNumBit(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetNumRep(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetRedOp(MlirType type); +MLIR_CAPI_EXPORTED bool +mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetNan(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetHalfSplitOff(MlirType type); + +// `!cute_nvgpu.atom.stsm`: stmatrix atom. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomStsm(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomStsmTypeGet( + MlirContext context, MlirType valType, MlirAttribute mode, int numMatrices, + bool transpose); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomStsmTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUCopyAtomStsmTypeGetMode(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomStsmTypeGetNumMatrices(MlirType type); +MLIR_CAPI_EXPORTED bool +mlirCuteNVGPUCopyAtomStsmTypeGetTranspose(MlirType type); + +// `!cute_nvgpu.atom.tma_load`: Executable TMA load atom. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomTmaLoad(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomTmaLoadTypeGet( + MlirContext context, MlirType valType, int sparsity, int copyBits, + uint32_t mode, int numCta, MlirType gStride, bool mcast, MlirType tmaGbasis, + bool override, bool noFullyOobTile); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomTmaLoadTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomTmaLoadTypeGetSparsity(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomTmaLoadTypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomTmaLoadTypeGetMode(MlirType type); +MLIR_CAPI_EXPORTED int mlirCuteNVGPUCopyAtomTmaLoadTypeGetNumCta(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomTmaLoadTypeGetGStride(MlirType type); +MLIR_CAPI_EXPORTED bool mlirCuteNVGPUCopyAtomTmaLoadTypeGetMcast(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomTmaLoadTypeGetTmaGbasis(MlirType type); +MLIR_CAPI_EXPORTED bool +mlirCuteNVGPUCopyAtomTmaLoadTypeGetOverride(MlirType type); +MLIR_CAPI_EXPORTED bool +mlirCuteNVGPUCopyAtomTmaLoadTypeGetNoFullyOobTile(MlirType type); + +// `!cute_nvgpu.atom.tma_reduce`: Executable TMA reduce atom. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomTmaReduce(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomTmaReduceTypeGet( + MlirContext context, MlirType valType, int copyBits, uint32_t mode, + uint32_t kind, MlirType gStride, MlirType tmaGbasis); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomTmaReduceTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomTmaReduceTypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomTmaReduceTypeGetMode(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomTmaReduceTypeGetKind(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomTmaReduceTypeGetGStride(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomTmaReduceTypeGetTmaGbasis(MlirType type); + +// `!cute_nvgpu.atom.tma_store`: Executable TMA store atom. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomTmaStore(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomTmaStoreTypeGet( + MlirContext context, MlirType valType, int sparsity, int copyBits, + uint32_t mode, MlirType gStride, MlirType tmaGbasis, bool override, + bool noFullyOobTile); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomTmaStoreTypeGetValType(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomTmaStoreTypeGetSparsity(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUCopyAtomTmaStoreTypeGetCopyBits(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUCopyAtomTmaStoreTypeGetMode(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomTmaStoreTypeGetGStride(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUCopyAtomTmaStoreTypeGetTmaGbasis(MlirType type); +MLIR_CAPI_EXPORTED bool +mlirCuteNVGPUCopyAtomTmaStoreTypeGetOverride(MlirType type); +MLIR_CAPI_EXPORTED bool +mlirCuteNVGPUCopyAtomTmaStoreTypeGetNoFullyOobTile(MlirType type); + +// `!cute_nvgpu.sm100.mma_bs_sp`: SM100 block-scaled sparse UMMA MMA atom. +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUMmaAtomSM100UMMABlockScaledSparse(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGet( + MlirContext context, MlirAttribute shapeMnk, int numCta, uint32_t aMajor, + uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, + MlirType sfType, uint32_t sparseMetadataFormat, uint32_t aFragKind, + int vecSize, uint32_t archPromote, MlirAttribute intOverflow); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetShapeMnk(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetNumCta(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetAMajor(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetBMajor(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetAType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetBType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetCType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetSfType(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetSparseMetadataFormat( + MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetAFragKind(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetVecSize(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetArchPromote(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetIntOverflow(MlirType type); + +// `!cute_nvgpu.sm100.mma_bs`: SM100 block-scaled UMMA MMA atom. +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUMmaAtomSM100UMMABlockScaled(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGet( + MlirContext context, MlirAttribute shapeMnk, int numCta, uint32_t aMajor, + uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, + MlirType sfType, uint32_t aFragKind, int vecSize, uint32_t archPromote, + MlirAttribute intOverflow); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetShapeMnk(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetNumCta(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetAMajor(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetBMajor(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetAType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetBType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetCType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetSfType(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetAFragKind(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetVecSize(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetArchPromote(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetIntOverflow(MlirType type); + +// `!cute_nvgpu.sm100.mma_sp`: SM100 sparse UMMA MMA atom. +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUMmaAtomSM100UMMASparse(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGet( + MlirContext context, MlirAttribute shapeMnk, int numCta, uint32_t aMajor, + uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, + MlirType eType, uint32_t sparseMetadataFormat, uint32_t aFragKind, + int cScaleExp, MlirAttribute intOverflow); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetShapeMnk(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetNumCta(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetAMajor(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetBMajor(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetAType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetBType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetCType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetEType(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetSparseMetadataFormat(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetAFragKind(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetCScaleExp(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetIntOverflow(MlirType type); + +// `!cute_nvgpu.sm100.mma`: SM100 UMMA MMA atom. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUMmaAtomSM100UMMA(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM100UMMATypeGet( + MlirContext context, MlirAttribute shapeMnk, int numCta, uint32_t aMajor, + uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, + uint32_t aFragKind, int cScaleExp, MlirAttribute intOverflow); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM100UMMATypeGetShapeMnk(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUMmaAtomSM100UMMATypeGetNumCta(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM100UMMATypeGetAMajor(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM100UMMATypeGetBMajor(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM100UMMATypeGetAType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM100UMMATypeGetBType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM100UMMATypeGetCType(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM100UMMATypeGetAFragKind(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUMmaAtomSM100UMMATypeGetCScaleExp(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM100UMMATypeGetIntOverflow(MlirType type); + +// `!cute_nvgpu.sm107.mma_bs_sp`: SM107 block-scaled sparse UMMA MMA atom. +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUMmaAtomSM107UMMABlockScaledSparse(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGet( + MlirContext context, MlirAttribute shapeMnk, int numCta, uint32_t aMajor, + uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, + MlirType sfType, uint32_t sparseMetadataFormat, uint32_t aFragKind, + int vecSize, uint32_t aCollectorOp, uint32_t bCollectorOp, + MlirAttribute intOverflow); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetShapeMnk(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetNumCta(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetAMajor(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetBMajor(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetAType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetBType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetCType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetSfType(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetSparseMetadataFormat( + MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetAFragKind(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetVecSize(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetACollectorOp( + MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetBCollectorOp( + MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetIntOverflow(MlirType type); + +// `!cute_nvgpu.sm107.mma_bs`: SM107 block-scaled UMMA MMA atom. +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUMmaAtomSM107UMMABlockScaled(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGet( + MlirContext context, MlirAttribute shapeMnk, int numCta, uint32_t aMajor, + uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, + MlirType sfType, uint32_t aFragKind, int vecSize, uint32_t aCollectorOp, + uint32_t bCollectorOp, MlirAttribute intOverflow); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetShapeMnk(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetNumCta(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetAMajor(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetBMajor(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetAType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetBType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetCType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetSfType(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetAFragKind(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetVecSize(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetACollectorOp(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetBCollectorOp(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetIntOverflow(MlirType type); + +// `!cute_nvgpu.sm107.mma_sp`: SM107 sparse UMMA MMA atom. +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUMmaAtomSM107UMMASparse(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGet( + MlirContext context, MlirAttribute shapeMnk, int numCta, uint32_t aMajor, + uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, + MlirType eType, uint32_t sparseMetadataFormat, uint32_t aFragKind, + int cScaleExp, uint32_t aCollectorOp, uint32_t bCollectorOp, + MlirAttribute intOverflow); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetShapeMnk(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetNumCta(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetAMajor(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetBMajor(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetAType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetBType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetCType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetEType(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetSparseMetadataFormat(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetAFragKind(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetCScaleExp(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetACollectorOp(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetBCollectorOp(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetIntOverflow(MlirType type); + +// `!cute_nvgpu.sm107.mma`: SM107 UMMA MMA atom. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUMmaAtomSM107UMMA(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM107UMMATypeGet( + MlirContext context, MlirAttribute shapeMnk, int numCta, uint32_t aMajor, + uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, + uint32_t aFragKind, int cScaleExp, uint32_t aCollectorOp, + uint32_t bCollectorOp, MlirAttribute intOverflow); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM107UMMATypeGetShapeMnk(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUMmaAtomSM107UMMATypeGetNumCta(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMATypeGetAMajor(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMATypeGetBMajor(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM107UMMATypeGetAType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM107UMMATypeGetBType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM107UMMATypeGetCType(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMATypeGetAFragKind(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUMmaAtomSM107UMMATypeGetCScaleExp(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMATypeGetACollectorOp(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM107UMMATypeGetBCollectorOp(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM107UMMATypeGetIntOverflow(MlirType type); + +// `!cute_nvgpu.SM120.mma_bs`: SM120 block-scaled mma.sync MMA atom. +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUMmaAtomSM120BlockScaled(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGet( + MlirContext context, MlirAttribute shapeMnk, int vecSize, MlirType aType, + MlirType bType, MlirType cType, MlirType sfType, bool useSfLayoutTV); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetShapeMnk(MlirType type); +MLIR_CAPI_EXPORTED int +mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetVecSize(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetAType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetBType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetCType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetSfType(MlirType type); +MLIR_CAPI_EXPORTED bool +mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetUseSfLayoutTV(MlirType type); + +// `!cute_nvgpu.sm80.sparse_mma`: SM80 sparse mma.sync MMA atom. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUMmaAtomSM80Sparse(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM80SparseTypeGet( + MlirContext context, MlirAttribute shapeMnk, MlirType aType, MlirType bType, + MlirType cType, uint32_t sparseMetadataFormat, MlirAttribute intOverflow); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM80SparseTypeGetShapeMnk(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM80SparseTypeGetAType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM80SparseTypeGetBType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM80SparseTypeGetCType(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM80SparseTypeGetSparseMetadataFormat(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM80SparseTypeGetIntOverflow(MlirType type); + +// `!cute_nvgpu.sm80.mma`: SM80 mma.sync MMA atom. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUMmaAtomSM80(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM80TypeGet( + MlirContext context, MlirAttribute shapeMnk, MlirType aType, MlirType bType, + MlirType cType, MlirAttribute intOverflow, MlirAttribute binaryOp); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM80TypeGetShapeMnk(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM80TypeGetAType(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM80TypeGetBType(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM80TypeGetCType(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM80TypeGetIntOverflow(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM80TypeGetBinaryOp(MlirType type); + +// `!cute_nvgpu.sm89.mma`: SM89 fp8 mma.sync MMA atom. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUMmaAtomSM89(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUMmaAtomSM89TypeGet(MlirContext context, MlirAttribute shapeMnk, + MlirType aType, MlirType bType, MlirType cType); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM89TypeGetShapeMnk(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM89TypeGetAType(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM89TypeGetBType(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM89TypeGetCType(MlirType type); + +// `!cute_nvgpu.sm90.mma`: SM90 wgmma MMA atom. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUMmaAtomSM90(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM90TypeGet( + MlirContext context, MlirAttribute shapeMnk, uint32_t aMajor, + uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, + uint32_t aFragKind, MlirAttribute intOverflow, bool aNeg, bool bNeg); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM90TypeGetShapeMnk(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM90TypeGetAMajor(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM90TypeGetBMajor(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM90TypeGetAType(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM90TypeGetBType(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM90TypeGetCType(MlirType type); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVGPUMmaAtomSM90TypeGetAFragKind(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUMmaAtomSM90TypeGetIntOverflow(MlirType type); +MLIR_CAPI_EXPORTED bool mlirCuteNVGPUMmaAtomSM90TypeGetANeg(MlirType type); +MLIR_CAPI_EXPORTED bool mlirCuteNVGPUMmaAtomSM90TypeGetBNeg(MlirType type); + +// `!cute_nvgpu.sm103.smem_desc_circular`: Circular shared-memory descriptor +// over a block layout in bytes. +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUSmemDescCircularSM103(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUSmemDescCircularSM103TypeGet( + MlirContext context, MlirAttribute blockLayoutBytes); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUSmemDescCircularSM103TypeGetBlockLayoutBytes(MlirType type); + +// `!cute_nvgpu.sm107.smem_desc_circular`: Circular shared-memory descriptor +// over a block layout in bytes. +MLIR_CAPI_EXPORTED bool +mlirTypeIsACuteNVGPUSmemDescCircularSM107(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUSmemDescCircularSM107TypeGet( + MlirContext context, MlirAttribute blockLayoutBytes); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUSmemDescCircularSM107TypeGetBlockLayoutBytes(MlirType type); + +// `!cute_nvgpu.sm107.smem_desc`: SM107 shared-memory descriptor. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUSmemDescSM107(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUSmemDescSM107TypeGet(MlirContext context); + +// `!cute_nvgpu.smem_desc`: UMMA/GMMA shared-memory descriptor. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUSmemDesc(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUSmemDescTypeGet(MlirContext context); + +// `!cute_nvgpu.smem_desc_view`: Shared-memory descriptor iterator with a +// layout. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUSmemDescView(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUSmemDescViewTypeGet( + MlirContext context, MlirType desc, MlirAttribute layout); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUSmemDescViewTypeGetDesc(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUSmemDescViewTypeGetLayout(MlirType type); + +// `!cute_nvgpu.tma_descriptor_im2col`: Im2col TMA descriptor. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUTmaDescriptorIm2Col(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUTmaDescriptorIm2ColTypeGet(MlirContext context); + +// `!cute_nvgpu.tma_descriptor_tiled`: Tiled TMA descriptor. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUTmaDescriptorTiled(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUTmaDescriptorTiledTypeGet(MlirContext context); + +// `!cute_nvgpu.atom.universal_fma`: Universal FMA MMA atom. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUUniversalFmaAtom(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUUniversalFmaAtomTypeGet( + MlirContext context, MlirAttribute shapeMnk, MlirType aType, MlirType bType, + MlirType cType); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVGPUUniversalFmaAtomTypeGetShapeMnk(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUUniversalFmaAtomTypeGetAType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUUniversalFmaAtomTypeGetBType(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUUniversalFmaAtomTypeGetCType(MlirType type); + +// `!cute_nvgpu.workid_response`: Cluster launch control work-id response. +MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUWorkIdResponse(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirCuteNVGPUWorkIdResponseTypeGet(MlirContext context); + +#ifdef __cplusplus +} +#endif + +#endif // MLIR_DIALECTS_CUTE_NVGPU_CAPI_H diff --git a/mlir/dialects/cute_nvgpu/cute_nvgpu.zig b/mlir/dialects/cute_nvgpu/cute_nvgpu.zig new file mode 100644 index 00000000..ac92ab5a --- /dev/null +++ b/mlir/dialects/cute_nvgpu/cute_nvgpu.zig @@ -0,0 +1,5225 @@ +const std = @import("std"); + +const c = @import("c"); +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +const cute_ir = @import("mlir/dialects/cute_ir"); + +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("cute_nvgpu"); +} + +/// `cute_nvgpu.grid_constant`: marks a kernel argument passed as a +/// `__grid_constant__` parameter. +pub fn gridConstant(ctx: *mlir.Context) mlir.NamedAttribute { + return .named(ctx, "cute_nvgpu.grid_constant", .unit(ctx)); +} + +pub const NVGPUType = opaque { + const M = mlir.Methods(NVGPUType, c.MlirType); + + pub const isAFn = c.mlirTypeIsACuteNVGPUType; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; +}; + +// Enums, generated from the dialect's .td files; values are the compiler's. + +/// NVGpu arch number like sm_103. +pub const Arch = enum(u32) { + sm_52 = 520, + sm_70 = 700, + sm_75 = 750, + sm_80 = 800, + sm_86 = 860, + sm_90 = 900, + sm_100 = 1000, + sm_103 = 1030, + sm_107 = 1070, +}; + +/// Fields stored in Bulk Load Copy Atom type. +pub const AtomCopyFieldBulkCopyG2S = enum(u32) { + tma_bar = 1, + mcast_mask = 2, + cache_policy = 7, +}; + +/// Fields stored in Bulk Store Copy Atom type. +pub const AtomCopyFieldBulkCopyS2G = enum(u32) { + byte_mask = 4, + cache_policy = 7, +}; + +/// Fields stored in Bulk CTA to Cluster Copy Atom type. +pub const AtomCopyFieldBulkCopyS2S = enum(u32) { + tma_bar = 1, + cta_rank = 5, +}; + +/// Fields stored in CopyAtomDsmemStoreType. +pub const AtomCopyFieldDsmemStore = enum(u32) { + mbar_ptr = 8, +}; + +/// Fields stored in CopyAtomG2RType. +pub const AtomCopyFieldLoadGlobal = enum(u32) { + cache_policy = 7, +}; + +/// Fields stored in CopyAtomNonExec2DGather4TmaLoadType. +pub const AtomCopyFieldNonExec2DGather4TmaLoad = enum(u32) { + tma_desc = 3, +}; + +/// Fields stored in CopyAtomNonExec2DScatter4TmaStoreType. +pub const AtomCopyFieldNonExec2DScatter4TmaStore = enum(u32) { + tma_desc = 3, +}; + +/// Fields stored in CopyAtomNonExecIm2ColTmaLoadType. +pub const AtomCopyFieldNonExecIm2ColTmaLoad = enum(u32) { + im2col_tma_desc = 6, +}; + +/// Fields stored in CopyAtomNonExecIm2ColTmaStoreType. +pub const AtomCopyFieldNonExecIm2ColTmaStore = enum(u32) { + im2col_tma_desc = 6, +}; + +/// Fields stored in CopyAtomNonExecTiledTmaLoadType. +pub const AtomCopyFieldNonExecTiledTmaLoad = enum(u32) { + tma_desc = 3, +}; + +/// Fields stored in CopyAtomNonExecTiledTmaReduceType. +pub const AtomCopyFieldNonExecTiledTmaReduce = enum(u32) { + tma_desc = 3, +}; + +/// Fields stored in CopyAtomNonExecTiledTmaStoreType. +pub const AtomCopyFieldNonExecTiledTmaStore = enum(u32) { + tma_desc = 3, +}; + +/// Fields stored in CopyAtomR2GType. +pub const AtomCopyFieldStoreGlobal = enum(u32) { + cache_policy = 7, +}; + +/// Fields stored in Tma Load Copy Atom type. +pub const AtomCopyFieldTmaLoad = enum(u32) { + tma_descriptor_ptr = 0, + tma_bar = 1, + mcast_mask = 2, + cache_policy = 7, + g_stride = 9, +}; + +/// Fields stored in Tma Reduce Copy Atom type. +pub const AtomCopyFieldTmaReduce = enum(u32) { + tma_descriptor_ptr = 0, + cache_policy = 7, + g_stride = 9, +}; + +/// Fields stored in Tma Store Copy Atom type. +pub const AtomCopyFieldTmaStore = enum(u32) { + tma_descriptor_ptr = 0, + cache_policy = 7, + g_stride = 9, +}; + +/// Fields stored in MMA Atom types for SM100. +pub const AtomMmaFieldSM100 = enum(u32) { + accum_c = 0, + neg_a = 1, + neg_b = 2, + disable_output_lane = 6, +}; + +/// Fields stored in MMA Atom types for SM100 (block scaled). +pub const AtomMmaFieldSM100BlockScaled = enum(u32) { + accum_c = 0, + neg_a = 1, + neg_b = 2, + sf_a = 3, + sf_b = 4, +}; + +/// Fields stored in MMA Atom types for SM100 (block scaled). +pub const AtomMmaFieldSM100BlockScaledSparse = enum(u32) { + accum_c = 0, + neg_a = 1, + neg_b = 2, + sf_a = 3, + sf_b = 4, + e = 5, +}; + +/// Fields stored in sparse MMA Atom types for SM100. +pub const AtomMmaFieldSM100Sparse = enum(u32) { + accum_c = 0, + neg_a = 1, + neg_b = 2, + e = 5, +}; + +/// Fields stored in MMA Atom types for SM120 (block scaled). +pub const AtomMmaFieldSM120BlockScaled = enum(u32) { + sf_a = 0, + sf_b = 1, +}; + +/// Fields stored in MMA Atom types for SM80. +pub const AtomMmaFieldSM80Sparse = enum(u32) { + e = 0, +}; + +/// Fields stored in MMA Atom types for SM90. +pub const AtomMmaFieldSM90 = enum(u32) { + accum_c = 0, +}; + +/// Binary operation for single-bit MMA operations. +pub const BinaryOp = enum(u32) { + none = 0, + xor_popc = 1, + and_popc = 2, +}; + +/// Cute Cache Eviction Priority kind. +pub const CacheEvictionPriority = enum(u32) { + EVICT_NORMAL = 0, + EVICT_FIRST = 1, + EVICT_LAST = 2, + EVICT_UNCHANGED = 3, + NO_ALLOCATE = 4, +}; + +/// Broadcast modes for the different utccp instructions. +pub const CopyS2TBroadcast = enum(u32) { + none = 0, + lw_0213 = 1, + lw_0123 = 2, + x4 = 3, +}; + +/// The various kinds of TMA loads in gather/scatter mode. +pub const GatherScatterTmaLoad = enum(u32) { + sm_100 = 0, + sm_100_multicast = 1, + sm_100_2sm = 2, + sm_100_2sm_multicast = 3, +}; + +/// The various kinds of TMA loads in im2col mode. +pub const Im2ColTmaLoad = enum(u32) { + sm_90 = 0, + sm_90_multicast = 1, + sm_100_2sm = 2, + sm_100_2sm_multicast = 3, +}; + +/// CuTe L2 prefetch size options. +pub const L2PrefetchSize = enum(u32) { + NONE = 0, + RESERVED = 1, + SIZE_64B = 2, + SIZE_128B = 3, + SIZE_256B = 4, +}; + +/// multimem ld_reduce accumulation precision kind. +pub const LdReduceAccPrecisionKind = enum(u32) { + F16 = 0, + F32 = 1, +}; + +/// LDSM's sz pattern, describing the bit size. +pub const LdsmSzPattern = enum(u32) { + u16 = 0, + u4to8 = 1, + s4to8 = 2, + u2to4 = 3, + s2to4 = 4, + u4x16p64to8 = 5, + u6x16p32to8 = 6, + u8 = 7, +}; + +/// Cache modes for the load instructions. +pub const LoadCacheMode = enum(u32) { + always = 0, + global = 1, + streaming = 2, + last_use = 3, + none = 4, +}; + +/// MMA overflow options. +pub const MMAIntOverflow = enum(u32) { + satfinite = 1, + wrapped = 0, +}; + +/// Memory Ordering kind. +pub const MemOrderKind = enum(u32) { + WEAK = 0, + RELAXED = 1, + ACQUIRE = 2, + RELEASE = 3, + ACQ_REL = 4, + SC = 5, + MMIO = 6, + CONSTANT = 7, + VOLATILE = 8, +}; + +/// Cute Memory Scope kind. +pub const MemScopeKind = enum(u32) { + CTA = 0, + CLUSTER = 1, + GPU = 2, + SYS = 3, +}; + +/// Enums for the mma collector op. +pub const MmaCollectorOp = enum(u32) { + discard = 0, + lastuse = 1, + fill = 2, + use = 3, +}; + +/// Enums for the mma frag type. +pub const MmaFragKind = enum(u32) { + smem_desc = 0, + sparse_smem_desc = 1, + tmem = 2, + tmem_ws = 3, + tmem_e = 4, + tmem_sf = 5, + rmem = 6, +}; + +/// Op for the TMASTORE instruction. +pub const ReductionKind = enum(u32) { + ADD = 0, + MIN = 1, + MAX = 2, + INC = 3, + DEC = 4, + AND = 5, + OR = 6, + XOR = 7, +}; + +/// CuTe Shared Space kind. +pub const SharedSpace = enum(u32) { + CTA = 0, + CLUSTER = 1, +}; + +/// Metadata format used in sparse MMA. +pub const SparseMetadataFormat = enum(u32) { + tid = 0, +}; + +/// Cache modes for the store instructions. +pub const StoreCacheMode = enum(u32) { + write_back = 0, + global = 1, + streaming = 2, + write_through = 3, + none = 4, +}; + +/// The various kinds of TMA loads in tiled mode. +pub const TiledTmaLoad = enum(u32) { + sm_90 = 0, + sm_90_multicast = 1, + sm_100_2sm = 2, + sm_100_2sm_multicast = 3, +}; + +/// Bits 74-71 of the TMA descriptor. +pub const TmaDataFormat = enum(u32) { + U8 = 0, + U16 = 1, + U32 = 2, + S32 = 3, + U64 = 4, + S64 = 5, + F16_RN = 6, + F32_RN = 7, + F32_FTZ_RN = 8, + F64_RN = 9, + BF16_RN = 10, + U4 = 11, + U4_UNPACK_U8 = 12, + U6_UNPACK_U8 = 13, + TF32_RN = 14, + TF32_FTZ_RN = 15, +}; + +/// Modes for the TMA load instruction. +pub const TmaLoadMode = enum(u32) { + tiled = 0, + im2col = 1, + w128 = 2, + w = 3, + gather4 = 4, +}; + +/// Modes for the TMA store instruction. +pub const TmaStoreMode = enum(u32) { + tiled = 0, + im2col = 1, + scatter4 = 2, +}; + +/// Modes for the Tmem allocation. +pub const TmemAllocMode = enum(u32) { + Interleaved = 0, + NonInterleaved = 1, + Duplicated = 2, + SfDuplicated_4x1 = 3, + SfDuplicated_2x2 = 4, +}; + +/// Reduce operation for the Tmem load instruction. +pub const TmemLoadRedOp = enum(u32) { + max = 0, + maxabs = 1, + min = 2, + minabs = 3, +}; + +// Types and attributes, generated from the dialect's .td files. + +/// `!cute_nvgpu.atom.bulk_copy_g2s`: Bulk (TMA non-tensor) copy atom. +pub const CopyAtomBulkCopyG2SType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomBulkCopyG2S; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.bulk_copy_g2s"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + valType: *const mlir.Type, + copyBits: c_int = 0, + mcast: bool = false, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomBulkCopyG2STypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, args.mcast); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomBulkCopyG2STypeGetValType(self.ptr()).ptr.?); + } + pub fn copyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomBulkCopyG2STypeGetCopyBits(self.ptr()); + } + pub fn mcast(self: *const Self) bool { + return c.mlirCuteNVGPUCopyAtomBulkCopyG2STypeGetMcast(self.ptr()); + } +}; + +/// `!cute_nvgpu.atom.bulk_copy_s2g`: Bulk (TMA non-tensor) copy atom. +pub const CopyAtomBulkCopyS2GType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomBulkCopyS2G; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.bulk_copy_s2g"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + valType: *const mlir.Type, + copyBits: c_int = 0, + mask: bool = false, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomBulkCopyS2GTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, args.mask); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomBulkCopyS2GTypeGetValType(self.ptr()).ptr.?); + } + pub fn copyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomBulkCopyS2GTypeGetCopyBits(self.ptr()); + } + pub fn mask(self: *const Self) bool { + return c.mlirCuteNVGPUCopyAtomBulkCopyS2GTypeGetMask(self.ptr()); + } +}; + +/// `!cute_nvgpu.atom.bulk_copy_s2s`: Bulk (TMA non-tensor) copy atom. +pub const CopyAtomBulkCopyS2SType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomBulkCopyS2S; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.bulk_copy_s2s"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + valType: *const mlir.Type, + copyBits: c_int = 0, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomBulkCopyS2STypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomBulkCopyS2STypeGetValType(self.ptr()).ptr.?); + } + pub fn copyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomBulkCopyS2STypeGetCopyBits(self.ptr()); + } +}; + +/// `!cute_nvgpu.atom.dsmem_store`: Distributed shared-memory store atom. +pub const CopyAtomDsmemStoreType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomDsmemStore; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.dsmem_store"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + valType: *const mlir.Type, + copyBits: c_int, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomDsmemStoreTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomDsmemStoreTypeGetValType(self.ptr()).ptr.?); + } + pub fn copyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomDsmemStoreTypeGetCopyBits(self.ptr()); + } +}; + +/// `!cute_nvgpu.atom.g2r`: Global-to-register load atom. +pub const CopyAtomG2RType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomG2R; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.g2r"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + valType: *const mlir.Type, + copyBits: c_int, + memOrder: MemOrderKind = .WEAK, + memScope: MemScopeKind = .CTA, + l2PrefetchSize: L2PrefetchSize = .NONE, + l1CacheEvictPriority: CacheEvictionPriority = .EVICT_NORMAL, + loadCacheMode: LoadCacheMode = .always, + sharedSpace: SharedSpace = .CTA, + invariant: bool = false, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomG2RTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @backingInt(args.memOrder), @backingInt(args.memScope), @backingInt(args.l2PrefetchSize), @backingInt(args.l1CacheEvictPriority), @backingInt(args.loadCacheMode), @backingInt(args.sharedSpace), args.invariant); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomG2RTypeGetValType(self.ptr()).ptr.?); + } + pub fn copyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomG2RTypeGetCopyBits(self.ptr()); + } + pub fn memOrder(self: *const Self) MemOrderKind { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomG2RTypeGetMemOrder(self.ptr()))); + } + pub fn memScope(self: *const Self) MemScopeKind { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomG2RTypeGetMemScope(self.ptr()))); + } + pub fn l2PrefetchSize(self: *const Self) L2PrefetchSize { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomG2RTypeGetL2PrefetchSize(self.ptr()))); + } + pub fn l1CacheEvictPriority(self: *const Self) CacheEvictionPriority { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomG2RTypeGetL1CacheEvictPriority(self.ptr()))); + } + pub fn loadCacheMode(self: *const Self) LoadCacheMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomG2RTypeGetLoadCacheMode(self.ptr()))); + } + pub fn sharedSpace(self: *const Self) SharedSpace { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomG2RTypeGetSharedSpace(self.ptr()))); + } + pub fn invariant(self: *const Self) bool { + return c.mlirCuteNVGPUCopyAtomG2RTypeGetInvariant(self.ptr()); + } +}; + +/// `!cute_nvgpu.atom.im2col_tma_load`: Executable im2col TMA load atom. +pub const CopyAtomIm2ColTmaLoadType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomIm2ColTmaLoad; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.im2col_tma_load"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + valType: *const mlir.Type, + copyBits: c_int, + numCta: c_int, + gStride: *const mlir.Type, + mcast: bool = false, + tmaGbasis: ?*const mlir.Type = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, args.numCta, args.gStride.ptr(), args.mcast, if (args.tmaGbasis) |v| v.ptr() else c.MlirType{ .ptr = null }); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetValType(self.ptr()).ptr.?); + } + pub fn copyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetCopyBits(self.ptr()); + } + pub fn numCta(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetNumCta(self.ptr()); + } + pub fn gStride(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetGStride(self.ptr()).ptr.?); + } + pub fn mcast(self: *const Self) bool { + return c.mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetMcast(self.ptr()); + } + pub fn tmaGbasis(self: *const Self) ?*const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetTmaGbasis(self.ptr()).ptr); + } +}; + +/// `!cute_nvgpu.atom.im2col_tma_store`: Executable im2col TMA store atom. +pub const CopyAtomIm2ColTmaStoreType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomIm2ColTmaStore; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.im2col_tma_store"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + valType: *const mlir.Type, + copyBits: c_int, + gStride: *const mlir.Type, + tmaGbasis: ?*const mlir.Type = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, args.gStride.ptr(), if (args.tmaGbasis) |v| v.ptr() else c.MlirType{ .ptr = null }); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGetValType(self.ptr()).ptr.?); + } + pub fn copyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGetCopyBits(self.ptr()); + } + pub fn gStride(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGetGStride(self.ptr()).ptr.?); + } + pub fn tmaGbasis(self: *const Self) ?*const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGetTmaGbasis(self.ptr()).ptr); + } +}; + +/// `!cute_nvgpu.atom.ldsm`: ldmatrix atom. +pub const CopyAtomLdsmType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomLdsm; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.ldsm"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + valType: *const mlir.Type, + mode: *const mlir.Attribute, + szPattern: LdsmSzPattern, + numMatrices: c_int, + transpose: bool = false, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomLdsmTypeGet(ctx.ptr(), args.valType.ptr(), args.mode.ptr(), @backingInt(args.szPattern), args.numMatrices, args.transpose); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomLdsmTypeGetValType(self.ptr()).ptr.?); + } + pub fn mode(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUCopyAtomLdsmTypeGetMode(self.ptr()).ptr.?); + } + pub fn szPattern(self: *const Self) LdsmSzPattern { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomLdsmTypeGetSzPattern(self.ptr()))); + } + pub fn numMatrices(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomLdsmTypeGetNumMatrices(self.ptr()); + } + pub fn transpose(self: *const Self) bool { + return c.mlirCuteNVGPUCopyAtomLdsmTypeGetTranspose(self.ptr()); + } +}; + +/// `!cute_nvgpu.atom.non_exec_2d_gather4_tma_load`: Non-executable 2-D gather4 TMA load atom. +pub const CopyAtomNonExec2DGather4TmaLoadType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomNonExec2DGather4TmaLoad; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.non_exec_2d_gather4_tma_load"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + kind: GatherScatterTmaLoad, + valType: *const mlir.Type, + copyBits: c_int, + tmaGbasis: *const mlir.Type, + tmaFormat: ?TmaDataFormat = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGet(ctx.ptr(), @backingInt(args.kind), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @backingInt(v) else -1); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn kind(self: *const Self) GatherScatterTmaLoad { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetKind(self.ptr()))); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetValType(self.ptr()).ptr.?); + } + pub fn copyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetCopyBits(self.ptr()); + } + pub fn tmaGbasis(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetTmaGbasis(self.ptr()).ptr.?); + } + pub fn tmaFormat(self: *const Self) ?TmaDataFormat { + const value = c.mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetTmaFormat(self.ptr()); + return if (value < 0) null else @fromBackingInt(@intCast(value)); + } +}; + +/// `!cute_nvgpu.atom.non_exec_2d_scatter4_tma_store`: Non-executable 2-D scatter4 TMA store atom. +pub const CopyAtomNonExec2DScatter4TmaStoreType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomNonExec2DScatter4TmaStore; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.non_exec_2d_scatter4_tma_store"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + valType: *const mlir.Type, + copyBits: c_int, + tmaGbasis: *const mlir.Type, + tmaFormat: ?TmaDataFormat = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @backingInt(v) else -1); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGetValType(self.ptr()).ptr.?); + } + pub fn copyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGetCopyBits(self.ptr()); + } + pub fn tmaGbasis(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGetTmaGbasis(self.ptr()).ptr.?); + } + pub fn tmaFormat(self: *const Self) ?TmaDataFormat { + const value = c.mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGetTmaFormat(self.ptr()); + return if (value < 0) null else @fromBackingInt(@intCast(value)); + } +}; + +/// `!cute_nvgpu.atom.non_exec_im2col_tma_load`: Non-executable im2col TMA load atom. +pub const CopyAtomNonExecIm2ColTmaLoadType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomNonExecIm2ColTmaLoad; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.non_exec_im2col_tma_load"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + kind: Im2ColTmaLoad, + valType: *const mlir.Type, + copyBits: c_int, + tmaGbasis: *const mlir.Type, + tmaFormat: ?TmaDataFormat = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGet(ctx.ptr(), @backingInt(args.kind), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @backingInt(v) else -1); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn kind(self: *const Self) Im2ColTmaLoad { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetKind(self.ptr()))); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetValType(self.ptr()).ptr.?); + } + pub fn copyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetCopyBits(self.ptr()); + } + pub fn tmaGbasis(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetTmaGbasis(self.ptr()).ptr.?); + } + pub fn tmaFormat(self: *const Self) ?TmaDataFormat { + const value = c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetTmaFormat(self.ptr()); + return if (value < 0) null else @fromBackingInt(@intCast(value)); + } +}; + +/// `!cute_nvgpu.atom.non_exec_im2col_tma_store`: Non-executable im2col TMA store atom. +pub const CopyAtomNonExecIm2ColTmaStoreType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomNonExecIm2ColTmaStore; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.non_exec_im2col_tma_store"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + valType: *const mlir.Type, + copyBits: c_int, + tmaGbasis: *const mlir.Type, + tmaFormat: ?TmaDataFormat = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @backingInt(v) else -1); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGetValType(self.ptr()).ptr.?); + } + pub fn copyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGetCopyBits(self.ptr()); + } + pub fn tmaGbasis(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGetTmaGbasis(self.ptr()).ptr.?); + } + pub fn tmaFormat(self: *const Self) ?TmaDataFormat { + const value = c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGetTmaFormat(self.ptr()); + return if (value < 0) null else @fromBackingInt(@intCast(value)); + } +}; + +/// `!cute_nvgpu.atom.non_exec_tiled_tma_load`: Non-executable tiled TMA load atom. +pub const CopyAtomNonExecTiledTmaLoadType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomNonExecTiledTmaLoad; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.non_exec_tiled_tma_load"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + kind: TiledTmaLoad, + valType: *const mlir.Type, + copyBits: c_int, + tmaGbasis: *const mlir.Type, + tmaFormat: ?TmaDataFormat = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGet(ctx.ptr(), @backingInt(args.kind), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @backingInt(v) else -1); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn kind(self: *const Self) TiledTmaLoad { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetKind(self.ptr()))); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetValType(self.ptr()).ptr.?); + } + pub fn copyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetCopyBits(self.ptr()); + } + pub fn tmaGbasis(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetTmaGbasis(self.ptr()).ptr.?); + } + pub fn tmaFormat(self: *const Self) ?TmaDataFormat { + const value = c.mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetTmaFormat(self.ptr()); + return if (value < 0) null else @fromBackingInt(@intCast(value)); + } +}; + +/// `!cute_nvgpu.atom.non_exec_tiled_tma_reduce`: Non-executable tiled TMA reduce atom. +pub const CopyAtomNonExecTiledTmaReduceType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomNonExecTiledTmaReduce; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.non_exec_tiled_tma_reduce"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + kind: ReductionKind, + valType: *const mlir.Type, + copyBits: c_int, + tmaGbasis: *const mlir.Type, + tmaFormat: ?TmaDataFormat = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGet(ctx.ptr(), @backingInt(args.kind), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @backingInt(v) else -1); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn kind(self: *const Self) ReductionKind { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetKind(self.ptr()))); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetValType(self.ptr()).ptr.?); + } + pub fn copyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetCopyBits(self.ptr()); + } + pub fn tmaGbasis(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetTmaGbasis(self.ptr()).ptr.?); + } + pub fn tmaFormat(self: *const Self) ?TmaDataFormat { + const value = c.mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetTmaFormat(self.ptr()); + return if (value < 0) null else @fromBackingInt(@intCast(value)); + } +}; + +/// `!cute_nvgpu.atom.non_exec_tiled_tma_store`: Non-executable tiled TMA store atom. +pub const CopyAtomNonExecTiledTmaStoreType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomNonExecTiledTmaStore; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.non_exec_tiled_tma_store"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + valType: *const mlir.Type, + copyBits: c_int, + tmaGbasis: *const mlir.Type, + tmaFormat: ?TmaDataFormat = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @backingInt(v) else -1); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGetValType(self.ptr()).ptr.?); + } + pub fn copyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGetCopyBits(self.ptr()); + } + pub fn tmaGbasis(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGetTmaGbasis(self.ptr()).ptr.?); + } + pub fn tmaFormat(self: *const Self) ?TmaDataFormat { + const value = c.mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGetTmaFormat(self.ptr()); + return if (value < 0) null else @fromBackingInt(@intCast(value)); + } +}; + +/// `!cute_nvgpu.atom.r2g`: Register-to-global store atom. +pub const CopyAtomR2GType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomR2G; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.r2g"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + valType: *const mlir.Type, + copyBits: c_int, + memOrder: MemOrderKind = .WEAK, + memScope: MemScopeKind = .CTA, + l1CacheEvictPriority: CacheEvictionPriority = .EVICT_NORMAL, + storeCacheMode: StoreCacheMode = .write_back, + sharedSpace: SharedSpace = .CTA, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomR2GTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @backingInt(args.memOrder), @backingInt(args.memScope), @backingInt(args.l1CacheEvictPriority), @backingInt(args.storeCacheMode), @backingInt(args.sharedSpace)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomR2GTypeGetValType(self.ptr()).ptr.?); + } + pub fn copyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomR2GTypeGetCopyBits(self.ptr()); + } + pub fn memOrder(self: *const Self) MemOrderKind { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomR2GTypeGetMemOrder(self.ptr()))); + } + pub fn memScope(self: *const Self) MemScopeKind { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomR2GTypeGetMemScope(self.ptr()))); + } + pub fn l1CacheEvictPriority(self: *const Self) CacheEvictionPriority { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomR2GTypeGetL1CacheEvictPriority(self.ptr()))); + } + pub fn storeCacheMode(self: *const Self) StoreCacheMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomR2GTypeGetStoreCacheMode(self.ptr()))); + } + pub fn sharedSpace(self: *const Self) SharedSpace { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomR2GTypeGetSharedSpace(self.ptr()))); + } +}; + +/// `!cute_nvgpu.atom.r2s`: Register-to-shared store atom. +pub const CopyAtomR2SType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomR2S; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.r2s"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + valType: *const mlir.Type, + copyBits: c_int, + memOrder: MemOrderKind = .WEAK, + memScope: MemScopeKind = .CTA, + sharedSpace: SharedSpace = .CTA, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomR2STypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @backingInt(args.memOrder), @backingInt(args.memScope), @backingInt(args.sharedSpace)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomR2STypeGetValType(self.ptr()).ptr.?); + } + pub fn copyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomR2STypeGetCopyBits(self.ptr()); + } + pub fn memOrder(self: *const Self) MemOrderKind { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomR2STypeGetMemOrder(self.ptr()))); + } + pub fn memScope(self: *const Self) MemScopeKind { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomR2STypeGetMemScope(self.ptr()))); + } + pub fn sharedSpace(self: *const Self) SharedSpace { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomR2STypeGetSharedSpace(self.ptr()))); + } +}; + +/// `!cute_nvgpu.atom.s2r`: Shared-to-register load atom. +pub const CopyAtomS2RType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomS2R; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.s2r"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + valType: *const mlir.Type, + copyBits: c_int, + memOrder: MemOrderKind = .WEAK, + memScope: MemScopeKind = .CTA, + sharedSpace: SharedSpace = .CTA, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomS2RTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @backingInt(args.memOrder), @backingInt(args.memScope), @backingInt(args.sharedSpace)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomS2RTypeGetValType(self.ptr()).ptr.?); + } + pub fn copyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomS2RTypeGetCopyBits(self.ptr()); + } + pub fn memOrder(self: *const Self) MemOrderKind { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomS2RTypeGetMemOrder(self.ptr()))); + } + pub fn memScope(self: *const Self) MemScopeKind { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomS2RTypeGetMemScope(self.ptr()))); + } + pub fn sharedSpace(self: *const Self) SharedSpace { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomS2RTypeGetSharedSpace(self.ptr()))); + } +}; + +/// `!cute_nvgpu.atom.simt_async_copy`: cp.async copy atom. +pub const CopyAtomSIMTAsyncCopyType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSIMTAsyncCopy; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.simt_async_copy"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + valType: *const mlir.Type, + cache: LoadCacheMode, + copyBits: c_int, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGet(ctx.ptr(), args.valType.ptr(), @backingInt(args.cache), args.copyBits); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGetValType(self.ptr()).ptr.?); + } + pub fn cache(self: *const Self) LoadCacheMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGetCache(self.ptr()))); + } + pub fn copyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGetCopyBits(self.ptr()); + } +}; + +/// `!cute_nvgpu.atom.simt_multimem_ld_reduce`: multimem.ld_reduce atom. +pub const CopyAtomSIMTMultimemLdReduceType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSIMTMultimemLdReduce; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.simt_multimem_ld_reduce"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + valType: *const mlir.Type, + copyBits: c_int = 0, + reduction: ReductionKind = .ADD, + ldReduceAccPrecision: ?*const mlir.Attribute = null, + memOrder: MemOrderKind = .RELAXED, + memScope: MemScopeKind = .SYS, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @backingInt(args.reduction), if (args.ldReduceAccPrecision) |v| v.ptr() else c.MlirAttribute{ .ptr = null }, @backingInt(args.memOrder), @backingInt(args.memScope)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetValType(self.ptr()).ptr.?); + } + pub fn copyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetCopyBits(self.ptr()); + } + pub fn reduction(self: *const Self) ReductionKind { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetReduction(self.ptr()))); + } + pub fn ldReduceAccPrecision(self: *const Self) ?*const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetLdReduceAccPrecision(self.ptr()).ptr); + } + pub fn memOrder(self: *const Self) MemOrderKind { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetMemOrder(self.ptr()))); + } + pub fn memScope(self: *const Self) MemScopeKind { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetMemScope(self.ptr()))); + } +}; + +/// `!cute_nvgpu.atom.simt_multimem_red`: multimem.red atom. +pub const CopyAtomSIMTMultimemRedType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSIMTMultimemRed; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.simt_multimem_red"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + valType: *const mlir.Type, + copyBits: c_int, + reduction: ReductionKind = .ADD, + memOrder: MemOrderKind = .RELAXED, + memScope: MemScopeKind = .SYS, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @backingInt(args.reduction), @backingInt(args.memOrder), @backingInt(args.memScope)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetValType(self.ptr()).ptr.?); + } + pub fn copyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetCopyBits(self.ptr()); + } + pub fn reduction(self: *const Self) ReductionKind { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetReduction(self.ptr()))); + } + pub fn memOrder(self: *const Self) MemOrderKind { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetMemOrder(self.ptr()))); + } + pub fn memScope(self: *const Self) MemScopeKind { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetMemScope(self.ptr()))); + } +}; + +/// `!cute_nvgpu.atom.simt_multimem_st`: multimem.st atom. +pub const CopyAtomSIMTMultimemStType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSIMTMultimemSt; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.simt_multimem_st"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + valType: *const mlir.Type, + copyBits: c_int = 0, + memOrder: MemOrderKind = .RELAXED, + memScope: MemScopeKind = .SYS, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @backingInt(args.memOrder), @backingInt(args.memScope)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetValType(self.ptr()).ptr.?); + } + pub fn copyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetCopyBits(self.ptr()); + } + pub fn memOrder(self: *const Self) MemOrderKind { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetMemOrder(self.ptr()))); + } + pub fn memScope(self: *const Self) MemScopeKind { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetMemScope(self.ptr()))); + } +}; + +/// `!cute_nvgpu.atom.universal_copy`: Universal (SIMT) copy atom. +pub const CopyAtomSIMTSyncCopyType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSIMTSyncCopy; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.universal_copy"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + valType: *const mlir.Type, + copyBits: c_int = 0, + srcSpace: cute_ir.AddressSpace = .generic, + dstSpace: cute_ir.AddressSpace = .generic, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @backingInt(args.srcSpace), @backingInt(args.dstSpace)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetValType(self.ptr()).ptr.?); + } + pub fn copyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetCopyBits(self.ptr()); + } + pub fn srcSpace(self: *const Self) cute_ir.AddressSpace { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetSrcSpace(self.ptr()))); + } + pub fn dstSpace(self: *const Self) cute_ir.AddressSpace { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetDstSpace(self.ptr()))); + } +}; + +/// `!cute_nvgpu.atom.s2t_copy`: tcgen05.cp (shared to tensor memory) atom. +pub const CopyAtomSM100CopyS2TType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSM100CopyS2T; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.s2t_copy"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + valType: *const mlir.Type, + numDp: c_int, + numBit: c_int, + numCta: c_int, + broadcast: CopyS2TBroadcast = .none, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGet(ctx.ptr(), args.valType.ptr(), args.numDp, args.numBit, args.numCta, @backingInt(args.broadcast)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetValType(self.ptr()).ptr.?); + } + pub fn numDp(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetNumDp(self.ptr()); + } + pub fn numBit(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetNumBit(self.ptr()); + } + pub fn numCta(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetNumCta(self.ptr()); + } + pub fn broadcast(self: *const Self) CopyS2TBroadcast { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetBroadcast(self.ptr()))); + } +}; + +/// `!cute_nvgpu.atom.sm100_s2t_copy_v2`: tcgen05.cp atom with the shared-memory major mode. +pub const CopyAtomSM100S2TCopyV2Type = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSM100S2TCopyV2; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.sm100_s2t_copy_v2"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + valType: *const mlir.Type, + numDp: c_int, + numBit: c_int, + numCta: c_int, + smemMajor: cute_ir.MajorMode, + broadcast: CopyS2TBroadcast = .none, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGet(ctx.ptr(), args.valType.ptr(), args.numDp, args.numBit, args.numCta, @backingInt(args.smemMajor), @backingInt(args.broadcast)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetValType(self.ptr()).ptr.?); + } + pub fn numDp(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetNumDp(self.ptr()); + } + pub fn numBit(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetNumBit(self.ptr()); + } + pub fn numCta(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetNumCta(self.ptr()); + } + pub fn smemMajor(self: *const Self) cute_ir.MajorMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetSmemMajor(self.ptr()))); + } + pub fn broadcast(self: *const Self) CopyS2TBroadcast { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetBroadcast(self.ptr()))); + } +}; + +/// `!cute_nvgpu.atom.tmem_load`: tcgen05.ld atom. +pub const CopyAtomSM100TmemLoadType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSM100TmemLoad; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.tmem_load"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + valType: *const mlir.Type, + numDp: c_int, + numBit: c_int, + numRep: u32, + pack16b: bool = false, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGet(ctx.ptr(), args.valType.ptr(), args.numDp, args.numBit, args.numRep, args.pack16b); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetValType(self.ptr()).ptr.?); + } + pub fn numDp(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetNumDp(self.ptr()); + } + pub fn numBit(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetNumBit(self.ptr()); + } + pub fn numRep(self: *const Self) u32 { + return c.mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetNumRep(self.ptr()); + } + pub fn pack16b(self: *const Self) bool { + return c.mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetPack16b(self.ptr()); + } +}; + +/// `!cute_nvgpu.atom.tmem_store`: tcgen05.st atom. +pub const CopyAtomSM100TmemStoreType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSM100TmemStore; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.tmem_store"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + valType: *const mlir.Type, + numDp: c_int, + numBit: c_int, + numRep: u32, + expand16b: bool = false, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGet(ctx.ptr(), args.valType.ptr(), args.numDp, args.numBit, args.numRep, args.expand16b); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetValType(self.ptr()).ptr.?); + } + pub fn numDp(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetNumDp(self.ptr()); + } + pub fn numBit(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetNumBit(self.ptr()); + } + pub fn numRep(self: *const Self) u32 { + return c.mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetNumRep(self.ptr()); + } + pub fn expand16b(self: *const Self) bool { + return c.mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetExpand16b(self.ptr()); + } +}; + +/// `!cute_nvgpu.atom.tmem_load_spcompress`: SM107 tensor-memory load with sparse compression. +pub const CopyAtomSM107TmemLoadSPCompressType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSM107TmemLoadSPCompress; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.tmem_load_spcompress"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + valType: *const mlir.Type, + numDp: c_int, + numBit: c_int, + numRep: u32, + redOp: TmemLoadRedOp, + red: bool = false, + nan: bool = false, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGet(ctx.ptr(), args.valType.ptr(), args.numDp, args.numBit, args.numRep, @backingInt(args.redOp), args.red, args.nan); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetValType(self.ptr()).ptr.?); + } + pub fn numDp(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetNumDp(self.ptr()); + } + pub fn numBit(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetNumBit(self.ptr()); + } + pub fn numRep(self: *const Self) u32 { + return c.mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetNumRep(self.ptr()); + } + pub fn redOp(self: *const Self) TmemLoadRedOp { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetRedOp(self.ptr()))); + } + pub fn red(self: *const Self) bool { + return c.mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetRed(self.ptr()); + } + pub fn nan(self: *const Self) bool { + return c.mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetNan(self.ptr()); + } +}; + +/// `!cute_nvgpu.atom.tmem_load_red`: tcgen05.ld.red atom. +pub const CopyAtomSM10xTmemLoadRedType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSM10xTmemLoadRed; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.tmem_load_red"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + valType: *const mlir.Type, + numDp: c_int, + numBit: c_int, + numRep: u32, + redOp: TmemLoadRedOp, + nan: bool = false, + halfSplitOff: ?*const mlir.Attribute = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGet(ctx.ptr(), args.valType.ptr(), args.numDp, args.numBit, args.numRep, @backingInt(args.redOp), args.nan, if (args.halfSplitOff) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetValType(self.ptr()).ptr.?); + } + pub fn numDp(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetNumDp(self.ptr()); + } + pub fn numBit(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetNumBit(self.ptr()); + } + pub fn numRep(self: *const Self) u32 { + return c.mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetNumRep(self.ptr()); + } + pub fn redOp(self: *const Self) TmemLoadRedOp { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetRedOp(self.ptr()))); + } + pub fn nan(self: *const Self) bool { + return c.mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetNan(self.ptr()); + } + pub fn halfSplitOff(self: *const Self) ?*const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetHalfSplitOff(self.ptr()).ptr); + } +}; + +/// `!cute_nvgpu.atom.stsm`: stmatrix atom. +pub const CopyAtomStsmType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomStsm; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.stsm"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + valType: *const mlir.Type, + mode: *const mlir.Attribute, + numMatrices: c_int, + transpose: bool = false, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomStsmTypeGet(ctx.ptr(), args.valType.ptr(), args.mode.ptr(), args.numMatrices, args.transpose); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomStsmTypeGetValType(self.ptr()).ptr.?); + } + pub fn mode(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUCopyAtomStsmTypeGetMode(self.ptr()).ptr.?); + } + pub fn numMatrices(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomStsmTypeGetNumMatrices(self.ptr()); + } + pub fn transpose(self: *const Self) bool { + return c.mlirCuteNVGPUCopyAtomStsmTypeGetTranspose(self.ptr()); + } +}; + +/// `!cute_nvgpu.atom.tma_load`: Executable TMA load atom. +pub const CopyAtomTmaLoadType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomTmaLoad; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.tma_load"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + valType: *const mlir.Type, + sparsity: c_int = 1, + copyBits: c_int, + mode: TmaLoadMode, + numCta: c_int, + gStride: *const mlir.Type, + mcast: bool = false, + tmaGbasis: ?*const mlir.Type = null, + override: bool = false, + noFullyOobTile: bool = false, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomTmaLoadTypeGet(ctx.ptr(), args.valType.ptr(), args.sparsity, args.copyBits, @backingInt(args.mode), args.numCta, args.gStride.ptr(), args.mcast, if (args.tmaGbasis) |v| v.ptr() else c.MlirType{ .ptr = null }, args.override, args.noFullyOobTile); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomTmaLoadTypeGetValType(self.ptr()).ptr.?); + } + pub fn sparsity(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomTmaLoadTypeGetSparsity(self.ptr()); + } + pub fn copyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomTmaLoadTypeGetCopyBits(self.ptr()); + } + pub fn mode(self: *const Self) TmaLoadMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomTmaLoadTypeGetMode(self.ptr()))); + } + pub fn numCta(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomTmaLoadTypeGetNumCta(self.ptr()); + } + pub fn gStride(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomTmaLoadTypeGetGStride(self.ptr()).ptr.?); + } + pub fn mcast(self: *const Self) bool { + return c.mlirCuteNVGPUCopyAtomTmaLoadTypeGetMcast(self.ptr()); + } + pub fn tmaGbasis(self: *const Self) ?*const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomTmaLoadTypeGetTmaGbasis(self.ptr()).ptr); + } + pub fn override(self: *const Self) bool { + return c.mlirCuteNVGPUCopyAtomTmaLoadTypeGetOverride(self.ptr()); + } + pub fn noFullyOobTile(self: *const Self) bool { + return c.mlirCuteNVGPUCopyAtomTmaLoadTypeGetNoFullyOobTile(self.ptr()); + } +}; + +/// `!cute_nvgpu.atom.tma_reduce`: Executable TMA reduce atom. +pub const CopyAtomTmaReduceType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomTmaReduce; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.tma_reduce"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + valType: *const mlir.Type, + copyBits: c_int, + mode: TmaStoreMode, + kind: ReductionKind, + gStride: *const mlir.Type, + tmaGbasis: ?*const mlir.Type = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomTmaReduceTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @backingInt(args.mode), @backingInt(args.kind), args.gStride.ptr(), if (args.tmaGbasis) |v| v.ptr() else c.MlirType{ .ptr = null }); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomTmaReduceTypeGetValType(self.ptr()).ptr.?); + } + pub fn copyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomTmaReduceTypeGetCopyBits(self.ptr()); + } + pub fn mode(self: *const Self) TmaStoreMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomTmaReduceTypeGetMode(self.ptr()))); + } + pub fn kind(self: *const Self) ReductionKind { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomTmaReduceTypeGetKind(self.ptr()))); + } + pub fn gStride(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomTmaReduceTypeGetGStride(self.ptr()).ptr.?); + } + pub fn tmaGbasis(self: *const Self) ?*const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomTmaReduceTypeGetTmaGbasis(self.ptr()).ptr); + } +}; + +/// `!cute_nvgpu.atom.tma_store`: Executable TMA store atom. +pub const CopyAtomTmaStoreType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomTmaStore; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.tma_store"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + valType: *const mlir.Type, + sparsity: c_int = 1, + copyBits: c_int, + mode: TmaStoreMode, + gStride: *const mlir.Type, + tmaGbasis: ?*const mlir.Type = null, + override: bool = false, + noFullyOobTile: bool = false, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyAtomTmaStoreTypeGet(ctx.ptr(), args.valType.ptr(), args.sparsity, args.copyBits, @backingInt(args.mode), args.gStride.ptr(), if (args.tmaGbasis) |v| v.ptr() else c.MlirType{ .ptr = null }, args.override, args.noFullyOobTile); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn valType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomTmaStoreTypeGetValType(self.ptr()).ptr.?); + } + pub fn sparsity(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomTmaStoreTypeGetSparsity(self.ptr()); + } + pub fn copyBits(self: *const Self) c_int { + return c.mlirCuteNVGPUCopyAtomTmaStoreTypeGetCopyBits(self.ptr()); + } + pub fn mode(self: *const Self) TmaStoreMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomTmaStoreTypeGetMode(self.ptr()))); + } + pub fn gStride(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomTmaStoreTypeGetGStride(self.ptr()).ptr.?); + } + pub fn tmaGbasis(self: *const Self) ?*const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUCopyAtomTmaStoreTypeGetTmaGbasis(self.ptr()).ptr); + } + pub fn override(self: *const Self) bool { + return c.mlirCuteNVGPUCopyAtomTmaStoreTypeGetOverride(self.ptr()); + } + pub fn noFullyOobTile(self: *const Self) bool { + return c.mlirCuteNVGPUCopyAtomTmaStoreTypeGetNoFullyOobTile(self.ptr()); + } +}; + +/// `!cute_nvgpu.sm100.mma_bs_sp`: SM100 block-scaled sparse UMMA MMA atom. +pub const MmaAtomSM100UMMABlockScaledSparseType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM100UMMABlockScaledSparse; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "sm100.mma_bs_sp"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + shapeMnk: *const mlir.Attribute, + numCta: c_int, + aMajor: cute_ir.MajorMode, + bMajor: cute_ir.MajorMode, + aType: *const mlir.Type, + bType: *const mlir.Type, + cType: *const mlir.Type, + sfType: *const mlir.Type, + sparseMetadataFormat: SparseMetadataFormat, + aFragKind: MmaFragKind, + vecSize: c_int, + archPromote: Arch = .sm_100, + intOverflow: ?*const mlir.Attribute = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @backingInt(args.aMajor), @backingInt(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.sfType.ptr(), @backingInt(args.sparseMetadataFormat), @backingInt(args.aFragKind), args.vecSize, @backingInt(args.archPromote), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn shapeMnk(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetShapeMnk(self.ptr()).ptr.?); + } + pub fn numCta(self: *const Self) c_int { + return c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetNumCta(self.ptr()); + } + pub fn aMajor(self: *const Self) cute_ir.MajorMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetAMajor(self.ptr()))); + } + pub fn bMajor(self: *const Self) cute_ir.MajorMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetBMajor(self.ptr()))); + } + pub fn aType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetAType(self.ptr()).ptr.?); + } + pub fn bType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetBType(self.ptr()).ptr.?); + } + pub fn cType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetCType(self.ptr()).ptr.?); + } + pub fn sfType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetSfType(self.ptr()).ptr.?); + } + pub fn sparseMetadataFormat(self: *const Self) SparseMetadataFormat { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetSparseMetadataFormat(self.ptr()))); + } + pub fn aFragKind(self: *const Self) MmaFragKind { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetAFragKind(self.ptr()))); + } + pub fn vecSize(self: *const Self) c_int { + return c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetVecSize(self.ptr()); + } + pub fn archPromote(self: *const Self) Arch { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetArchPromote(self.ptr()))); + } + pub fn intOverflow(self: *const Self) ?*const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetIntOverflow(self.ptr()).ptr); + } +}; + +/// `!cute_nvgpu.sm100.mma_bs`: SM100 block-scaled UMMA MMA atom. +pub const MmaAtomSM100UMMABlockScaledType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM100UMMABlockScaled; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "sm100.mma_bs"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + shapeMnk: *const mlir.Attribute, + numCta: c_int, + aMajor: cute_ir.MajorMode, + bMajor: cute_ir.MajorMode, + aType: *const mlir.Type, + bType: *const mlir.Type, + cType: *const mlir.Type, + sfType: *const mlir.Type, + aFragKind: MmaFragKind, + vecSize: c_int, + archPromote: Arch = .sm_100, + intOverflow: ?*const mlir.Attribute = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @backingInt(args.aMajor), @backingInt(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.sfType.ptr(), @backingInt(args.aFragKind), args.vecSize, @backingInt(args.archPromote), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn shapeMnk(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetShapeMnk(self.ptr()).ptr.?); + } + pub fn numCta(self: *const Self) c_int { + return c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetNumCta(self.ptr()); + } + pub fn aMajor(self: *const Self) cute_ir.MajorMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetAMajor(self.ptr()))); + } + pub fn bMajor(self: *const Self) cute_ir.MajorMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetBMajor(self.ptr()))); + } + pub fn aType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetAType(self.ptr()).ptr.?); + } + pub fn bType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetBType(self.ptr()).ptr.?); + } + pub fn cType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetCType(self.ptr()).ptr.?); + } + pub fn sfType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetSfType(self.ptr()).ptr.?); + } + pub fn aFragKind(self: *const Self) MmaFragKind { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetAFragKind(self.ptr()))); + } + pub fn vecSize(self: *const Self) c_int { + return c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetVecSize(self.ptr()); + } + pub fn archPromote(self: *const Self) Arch { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetArchPromote(self.ptr()))); + } + pub fn intOverflow(self: *const Self) ?*const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetIntOverflow(self.ptr()).ptr); + } +}; + +/// `!cute_nvgpu.sm100.mma_sp`: SM100 sparse UMMA MMA atom. +pub const MmaAtomSM100UMMASparseType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM100UMMASparse; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "sm100.mma_sp"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + shapeMnk: *const mlir.Attribute, + numCta: c_int, + aMajor: cute_ir.MajorMode, + bMajor: cute_ir.MajorMode, + aType: *const mlir.Type, + bType: *const mlir.Type, + cType: *const mlir.Type, + eType: *const mlir.Type, + sparseMetadataFormat: SparseMetadataFormat, + aFragKind: MmaFragKind, + cScaleExp: c_int, + intOverflow: ?*const mlir.Attribute = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @backingInt(args.aMajor), @backingInt(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.eType.ptr(), @backingInt(args.sparseMetadataFormat), @backingInt(args.aFragKind), args.cScaleExp, if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn shapeMnk(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetShapeMnk(self.ptr()).ptr.?); + } + pub fn numCta(self: *const Self) c_int { + return c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetNumCta(self.ptr()); + } + pub fn aMajor(self: *const Self) cute_ir.MajorMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetAMajor(self.ptr()))); + } + pub fn bMajor(self: *const Self) cute_ir.MajorMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetBMajor(self.ptr()))); + } + pub fn aType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetAType(self.ptr()).ptr.?); + } + pub fn bType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetBType(self.ptr()).ptr.?); + } + pub fn cType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetCType(self.ptr()).ptr.?); + } + pub fn eType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetEType(self.ptr()).ptr.?); + } + pub fn sparseMetadataFormat(self: *const Self) SparseMetadataFormat { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetSparseMetadataFormat(self.ptr()))); + } + pub fn aFragKind(self: *const Self) MmaFragKind { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetAFragKind(self.ptr()))); + } + pub fn cScaleExp(self: *const Self) c_int { + return c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetCScaleExp(self.ptr()); + } + pub fn intOverflow(self: *const Self) ?*const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetIntOverflow(self.ptr()).ptr); + } +}; + +/// `!cute_nvgpu.sm100.mma`: SM100 UMMA MMA atom. +pub const MmaAtomSM100UMMAType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM100UMMA; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "sm100.mma"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + shapeMnk: *const mlir.Attribute, + numCta: c_int, + aMajor: cute_ir.MajorMode, + bMajor: cute_ir.MajorMode, + aType: *const mlir.Type, + bType: *const mlir.Type, + cType: *const mlir.Type, + aFragKind: MmaFragKind, + cScaleExp: c_int, + intOverflow: ?*const mlir.Attribute = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUMmaAtomSM100UMMATypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @backingInt(args.aMajor), @backingInt(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), @backingInt(args.aFragKind), args.cScaleExp, if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn shapeMnk(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetShapeMnk(self.ptr()).ptr.?); + } + pub fn numCta(self: *const Self) c_int { + return c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetNumCta(self.ptr()); + } + pub fn aMajor(self: *const Self) cute_ir.MajorMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetAMajor(self.ptr()))); + } + pub fn bMajor(self: *const Self) cute_ir.MajorMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetBMajor(self.ptr()))); + } + pub fn aType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetAType(self.ptr()).ptr.?); + } + pub fn bType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetBType(self.ptr()).ptr.?); + } + pub fn cType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetCType(self.ptr()).ptr.?); + } + pub fn aFragKind(self: *const Self) MmaFragKind { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetAFragKind(self.ptr()))); + } + pub fn cScaleExp(self: *const Self) c_int { + return c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetCScaleExp(self.ptr()); + } + pub fn intOverflow(self: *const Self) ?*const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetIntOverflow(self.ptr()).ptr); + } +}; + +/// `!cute_nvgpu.sm107.mma_bs_sp`: SM107 block-scaled sparse UMMA MMA atom. +pub const MmaAtomSM107UMMABlockScaledSparseType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM107UMMABlockScaledSparse; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "sm107.mma_bs_sp"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + shapeMnk: *const mlir.Attribute, + numCta: c_int, + aMajor: cute_ir.MajorMode, + bMajor: cute_ir.MajorMode, + aType: *const mlir.Type, + bType: *const mlir.Type, + cType: *const mlir.Type, + sfType: *const mlir.Type, + sparseMetadataFormat: SparseMetadataFormat, + aFragKind: MmaFragKind, + vecSize: c_int, + aCollectorOp: MmaCollectorOp, + bCollectorOp: MmaCollectorOp, + intOverflow: ?*const mlir.Attribute = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @backingInt(args.aMajor), @backingInt(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.sfType.ptr(), @backingInt(args.sparseMetadataFormat), @backingInt(args.aFragKind), args.vecSize, @backingInt(args.aCollectorOp), @backingInt(args.bCollectorOp), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn shapeMnk(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetShapeMnk(self.ptr()).ptr.?); + } + pub fn numCta(self: *const Self) c_int { + return c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetNumCta(self.ptr()); + } + pub fn aMajor(self: *const Self) cute_ir.MajorMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetAMajor(self.ptr()))); + } + pub fn bMajor(self: *const Self) cute_ir.MajorMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetBMajor(self.ptr()))); + } + pub fn aType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetAType(self.ptr()).ptr.?); + } + pub fn bType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetBType(self.ptr()).ptr.?); + } + pub fn cType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetCType(self.ptr()).ptr.?); + } + pub fn sfType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetSfType(self.ptr()).ptr.?); + } + pub fn sparseMetadataFormat(self: *const Self) SparseMetadataFormat { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetSparseMetadataFormat(self.ptr()))); + } + pub fn aFragKind(self: *const Self) MmaFragKind { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetAFragKind(self.ptr()))); + } + pub fn vecSize(self: *const Self) c_int { + return c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetVecSize(self.ptr()); + } + pub fn aCollectorOp(self: *const Self) MmaCollectorOp { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetACollectorOp(self.ptr()))); + } + pub fn bCollectorOp(self: *const Self) MmaCollectorOp { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetBCollectorOp(self.ptr()))); + } + pub fn intOverflow(self: *const Self) ?*const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetIntOverflow(self.ptr()).ptr); + } +}; + +/// `!cute_nvgpu.sm107.mma_bs`: SM107 block-scaled UMMA MMA atom. +pub const MmaAtomSM107UMMABlockScaledType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM107UMMABlockScaled; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "sm107.mma_bs"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + shapeMnk: *const mlir.Attribute, + numCta: c_int, + aMajor: cute_ir.MajorMode, + bMajor: cute_ir.MajorMode, + aType: *const mlir.Type, + bType: *const mlir.Type, + cType: *const mlir.Type, + sfType: *const mlir.Type, + aFragKind: MmaFragKind, + vecSize: c_int, + aCollectorOp: MmaCollectorOp, + bCollectorOp: MmaCollectorOp, + intOverflow: ?*const mlir.Attribute = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @backingInt(args.aMajor), @backingInt(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.sfType.ptr(), @backingInt(args.aFragKind), args.vecSize, @backingInt(args.aCollectorOp), @backingInt(args.bCollectorOp), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn shapeMnk(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetShapeMnk(self.ptr()).ptr.?); + } + pub fn numCta(self: *const Self) c_int { + return c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetNumCta(self.ptr()); + } + pub fn aMajor(self: *const Self) cute_ir.MajorMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetAMajor(self.ptr()))); + } + pub fn bMajor(self: *const Self) cute_ir.MajorMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetBMajor(self.ptr()))); + } + pub fn aType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetAType(self.ptr()).ptr.?); + } + pub fn bType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetBType(self.ptr()).ptr.?); + } + pub fn cType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetCType(self.ptr()).ptr.?); + } + pub fn sfType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetSfType(self.ptr()).ptr.?); + } + pub fn aFragKind(self: *const Self) MmaFragKind { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetAFragKind(self.ptr()))); + } + pub fn vecSize(self: *const Self) c_int { + return c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetVecSize(self.ptr()); + } + pub fn aCollectorOp(self: *const Self) MmaCollectorOp { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetACollectorOp(self.ptr()))); + } + pub fn bCollectorOp(self: *const Self) MmaCollectorOp { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetBCollectorOp(self.ptr()))); + } + pub fn intOverflow(self: *const Self) ?*const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetIntOverflow(self.ptr()).ptr); + } +}; + +/// `!cute_nvgpu.sm107.mma_sp`: SM107 sparse UMMA MMA atom. +pub const MmaAtomSM107UMMASparseType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM107UMMASparse; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "sm107.mma_sp"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + shapeMnk: *const mlir.Attribute, + numCta: c_int, + aMajor: cute_ir.MajorMode, + bMajor: cute_ir.MajorMode, + aType: *const mlir.Type, + bType: *const mlir.Type, + cType: *const mlir.Type, + eType: *const mlir.Type, + sparseMetadataFormat: SparseMetadataFormat, + aFragKind: MmaFragKind, + cScaleExp: c_int, + aCollectorOp: MmaCollectorOp, + bCollectorOp: MmaCollectorOp, + intOverflow: ?*const mlir.Attribute = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @backingInt(args.aMajor), @backingInt(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.eType.ptr(), @backingInt(args.sparseMetadataFormat), @backingInt(args.aFragKind), args.cScaleExp, @backingInt(args.aCollectorOp), @backingInt(args.bCollectorOp), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn shapeMnk(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetShapeMnk(self.ptr()).ptr.?); + } + pub fn numCta(self: *const Self) c_int { + return c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetNumCta(self.ptr()); + } + pub fn aMajor(self: *const Self) cute_ir.MajorMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetAMajor(self.ptr()))); + } + pub fn bMajor(self: *const Self) cute_ir.MajorMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetBMajor(self.ptr()))); + } + pub fn aType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetAType(self.ptr()).ptr.?); + } + pub fn bType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetBType(self.ptr()).ptr.?); + } + pub fn cType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetCType(self.ptr()).ptr.?); + } + pub fn eType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetEType(self.ptr()).ptr.?); + } + pub fn sparseMetadataFormat(self: *const Self) SparseMetadataFormat { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetSparseMetadataFormat(self.ptr()))); + } + pub fn aFragKind(self: *const Self) MmaFragKind { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetAFragKind(self.ptr()))); + } + pub fn cScaleExp(self: *const Self) c_int { + return c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetCScaleExp(self.ptr()); + } + pub fn aCollectorOp(self: *const Self) MmaCollectorOp { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetACollectorOp(self.ptr()))); + } + pub fn bCollectorOp(self: *const Self) MmaCollectorOp { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetBCollectorOp(self.ptr()))); + } + pub fn intOverflow(self: *const Self) ?*const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetIntOverflow(self.ptr()).ptr); + } +}; + +/// `!cute_nvgpu.sm107.mma`: SM107 UMMA MMA atom. +pub const MmaAtomSM107UMMAType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM107UMMA; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "sm107.mma"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + shapeMnk: *const mlir.Attribute, + numCta: c_int, + aMajor: cute_ir.MajorMode, + bMajor: cute_ir.MajorMode, + aType: *const mlir.Type, + bType: *const mlir.Type, + cType: *const mlir.Type, + aFragKind: MmaFragKind, + cScaleExp: c_int, + aCollectorOp: MmaCollectorOp, + bCollectorOp: MmaCollectorOp, + intOverflow: ?*const mlir.Attribute = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUMmaAtomSM107UMMATypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @backingInt(args.aMajor), @backingInt(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), @backingInt(args.aFragKind), args.cScaleExp, @backingInt(args.aCollectorOp), @backingInt(args.bCollectorOp), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn shapeMnk(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetShapeMnk(self.ptr()).ptr.?); + } + pub fn numCta(self: *const Self) c_int { + return c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetNumCta(self.ptr()); + } + pub fn aMajor(self: *const Self) cute_ir.MajorMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetAMajor(self.ptr()))); + } + pub fn bMajor(self: *const Self) cute_ir.MajorMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetBMajor(self.ptr()))); + } + pub fn aType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetAType(self.ptr()).ptr.?); + } + pub fn bType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetBType(self.ptr()).ptr.?); + } + pub fn cType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetCType(self.ptr()).ptr.?); + } + pub fn aFragKind(self: *const Self) MmaFragKind { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetAFragKind(self.ptr()))); + } + pub fn cScaleExp(self: *const Self) c_int { + return c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetCScaleExp(self.ptr()); + } + pub fn aCollectorOp(self: *const Self) MmaCollectorOp { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetACollectorOp(self.ptr()))); + } + pub fn bCollectorOp(self: *const Self) MmaCollectorOp { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetBCollectorOp(self.ptr()))); + } + pub fn intOverflow(self: *const Self) ?*const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetIntOverflow(self.ptr()).ptr); + } +}; + +/// `!cute_nvgpu.SM120.mma_bs`: SM120 block-scaled mma.sync MMA atom. +pub const MmaAtomSM120BlockScaledType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM120BlockScaled; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "SM120.mma_bs"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + shapeMnk: *const mlir.Attribute, + vecSize: c_int, + aType: *const mlir.Type, + bType: *const mlir.Type, + cType: *const mlir.Type, + sfType: *const mlir.Type, + useSfLayoutTV: bool, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.vecSize, args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.sfType.ptr(), args.useSfLayoutTV); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn shapeMnk(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetShapeMnk(self.ptr()).ptr.?); + } + pub fn vecSize(self: *const Self) c_int { + return c.mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetVecSize(self.ptr()); + } + pub fn aType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetAType(self.ptr()).ptr.?); + } + pub fn bType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetBType(self.ptr()).ptr.?); + } + pub fn cType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetCType(self.ptr()).ptr.?); + } + pub fn sfType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetSfType(self.ptr()).ptr.?); + } + pub fn useSfLayoutTV(self: *const Self) bool { + return c.mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetUseSfLayoutTV(self.ptr()); + } +}; + +/// `!cute_nvgpu.sm80.sparse_mma`: SM80 sparse mma.sync MMA atom. +pub const MmaAtomSM80SparseType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM80Sparse; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "sm80.sparse_mma"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + shapeMnk: *const mlir.Attribute, + aType: *const mlir.Type, + bType: *const mlir.Type, + cType: *const mlir.Type, + sparseMetadataFormat: SparseMetadataFormat, + intOverflow: ?*const mlir.Attribute = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUMmaAtomSM80SparseTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), @backingInt(args.sparseMetadataFormat), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn shapeMnk(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80SparseTypeGetShapeMnk(self.ptr()).ptr.?); + } + pub fn aType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80SparseTypeGetAType(self.ptr()).ptr.?); + } + pub fn bType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80SparseTypeGetBType(self.ptr()).ptr.?); + } + pub fn cType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80SparseTypeGetCType(self.ptr()).ptr.?); + } + pub fn sparseMetadataFormat(self: *const Self) SparseMetadataFormat { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM80SparseTypeGetSparseMetadataFormat(self.ptr()))); + } + pub fn intOverflow(self: *const Self) ?*const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80SparseTypeGetIntOverflow(self.ptr()).ptr); + } +}; + +/// `!cute_nvgpu.sm80.mma`: SM80 mma.sync MMA atom. +pub const MmaAtomSM80Type = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM80; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "sm80.mma"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + shapeMnk: *const mlir.Attribute, + aType: *const mlir.Type, + bType: *const mlir.Type, + cType: *const mlir.Type, + intOverflow: ?*const mlir.Attribute = null, + binaryOp: ?*const mlir.Attribute = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUMmaAtomSM80TypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }, if (args.binaryOp) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn shapeMnk(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80TypeGetShapeMnk(self.ptr()).ptr.?); + } + pub fn aType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80TypeGetAType(self.ptr()).ptr.?); + } + pub fn bType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80TypeGetBType(self.ptr()).ptr.?); + } + pub fn cType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80TypeGetCType(self.ptr()).ptr.?); + } + pub fn intOverflow(self: *const Self) ?*const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80TypeGetIntOverflow(self.ptr()).ptr); + } + pub fn binaryOp(self: *const Self) ?*const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80TypeGetBinaryOp(self.ptr()).ptr); + } +}; + +/// `!cute_nvgpu.sm89.mma`: SM89 fp8 mma.sync MMA atom. +pub const MmaAtomSM89Type = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM89; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "sm89.mma"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + shapeMnk: *const mlir.Attribute, + aType: *const mlir.Type, + bType: *const mlir.Type, + cType: *const mlir.Type, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUMmaAtomSM89TypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.aType.ptr(), args.bType.ptr(), args.cType.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn shapeMnk(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM89TypeGetShapeMnk(self.ptr()).ptr.?); + } + pub fn aType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM89TypeGetAType(self.ptr()).ptr.?); + } + pub fn bType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM89TypeGetBType(self.ptr()).ptr.?); + } + pub fn cType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM89TypeGetCType(self.ptr()).ptr.?); + } +}; + +/// `!cute_nvgpu.sm90.mma`: SM90 wgmma MMA atom. +pub const MmaAtomSM90Type = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM90; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "sm90.mma"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + shapeMnk: *const mlir.Attribute, + aMajor: cute_ir.MajorMode, + bMajor: cute_ir.MajorMode, + aType: *const mlir.Type, + bType: *const mlir.Type, + cType: *const mlir.Type, + aFragKind: MmaFragKind, + intOverflow: ?*const mlir.Attribute = null, + aNeg: bool = false, + bNeg: bool = false, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUMmaAtomSM90TypeGet(ctx.ptr(), args.shapeMnk.ptr(), @backingInt(args.aMajor), @backingInt(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), @backingInt(args.aFragKind), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }, args.aNeg, args.bNeg); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn shapeMnk(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM90TypeGetShapeMnk(self.ptr()).ptr.?); + } + pub fn aMajor(self: *const Self) cute_ir.MajorMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM90TypeGetAMajor(self.ptr()))); + } + pub fn bMajor(self: *const Self) cute_ir.MajorMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM90TypeGetBMajor(self.ptr()))); + } + pub fn aType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM90TypeGetAType(self.ptr()).ptr.?); + } + pub fn bType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM90TypeGetBType(self.ptr()).ptr.?); + } + pub fn cType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM90TypeGetCType(self.ptr()).ptr.?); + } + pub fn aFragKind(self: *const Self) MmaFragKind { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM90TypeGetAFragKind(self.ptr()))); + } + pub fn intOverflow(self: *const Self) ?*const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUMmaAtomSM90TypeGetIntOverflow(self.ptr()).ptr); + } + pub fn aNeg(self: *const Self) bool { + return c.mlirCuteNVGPUMmaAtomSM90TypeGetANeg(self.ptr()); + } + pub fn bNeg(self: *const Self) bool { + return c.mlirCuteNVGPUMmaAtomSM90TypeGetBNeg(self.ptr()); + } +}; + +/// `!cute_nvgpu.sm103.smem_desc_circular`: Circular shared-memory descriptor over a block layout in bytes. +pub const SmemDescCircularSM103Type = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUSmemDescCircularSM103; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "sm103.smem_desc_circular"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + blockLayoutBytes: *const mlir.Attribute, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUSmemDescCircularSM103TypeGet(ctx.ptr(), args.blockLayoutBytes.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn blockLayoutBytes(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUSmemDescCircularSM103TypeGetBlockLayoutBytes(self.ptr()).ptr.?); + } +}; + +/// `!cute_nvgpu.sm107.smem_desc_circular`: Circular shared-memory descriptor over a block layout in bytes. +pub const SmemDescCircularSM107Type = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUSmemDescCircularSM107; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "sm107.smem_desc_circular"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + blockLayoutBytes: *const mlir.Attribute, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUSmemDescCircularSM107TypeGet(ctx.ptr(), args.blockLayoutBytes.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn blockLayoutBytes(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUSmemDescCircularSM107TypeGetBlockLayoutBytes(self.ptr()).ptr.?); + } +}; + +/// `!cute_nvgpu.sm107.smem_desc`: SM107 shared-memory descriptor. +pub const SmemDescSM107Type = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUSmemDescSM107; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "sm107.smem_desc"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCuteNVGPUSmemDescSM107TypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cute_nvgpu.smem_desc`: UMMA/GMMA shared-memory descriptor. +pub const SmemDescType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUSmemDesc; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "smem_desc"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCuteNVGPUSmemDescTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cute_nvgpu.smem_desc_view`: Shared-memory descriptor iterator with a layout. +pub const SmemDescViewType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUSmemDescView; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "smem_desc_view"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + desc: *const mlir.Type, + layout: *const mlir.Attribute, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUSmemDescViewTypeGet(ctx.ptr(), args.desc.ptr(), args.layout.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn desc(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUSmemDescViewTypeGetDesc(self.ptr()).ptr.?); + } + pub fn layout(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUSmemDescViewTypeGetLayout(self.ptr()).ptr.?); + } +}; + +/// `!cute_nvgpu.tma_descriptor_im2col`: Im2col TMA descriptor. +pub const TmaDescriptorIm2ColType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUTmaDescriptorIm2Col; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "tma_descriptor_im2col"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCuteNVGPUTmaDescriptorIm2ColTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cute_nvgpu.tma_descriptor_tiled`: Tiled TMA descriptor. +pub const TmaDescriptorTiledType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUTmaDescriptorTiled; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "tma_descriptor_tiled"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCuteNVGPUTmaDescriptorTiledTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!cute_nvgpu.atom.universal_fma`: Universal FMA MMA atom. +pub const UniversalFmaAtomType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUUniversalFmaAtom; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "atom.universal_fma"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct { + shapeMnk: *const mlir.Attribute, + aType: *const mlir.Type, + bType: *const mlir.Type, + cType: *const mlir.Type, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUUniversalFmaAtomTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.aType.ptr(), args.bType.ptr(), args.cType.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn shapeMnk(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirCuteNVGPUUniversalFmaAtomTypeGetShapeMnk(self.ptr()).ptr.?); + } + pub fn aType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUUniversalFmaAtomTypeGetAType(self.ptr()).ptr.?); + } + pub fn bType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUUniversalFmaAtomTypeGetBType(self.ptr()).ptr.?); + } + pub fn cType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUUniversalFmaAtomTypeGetCType(self.ptr()).ptr.?); + } +}; + +/// `!cute_nvgpu.workid_response`: Cluster launch control work-id response. +pub const WorkIdResponseType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsACuteNVGPUWorkIdResponse; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "workid_response"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCuteNVGPUWorkIdResponseTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `#cute_nvgpu.atom_copy_field_bulkg2s`: Fields stored in Bulk Load Copy Atom type. +pub const AtomCopyFieldBulkCopyG2SAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldBulkCopyG2S; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "atom_copy_field_bulkg2s"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomCopyFieldBulkCopyG2S, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomCopyFieldBulkCopyG2SAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) AtomCopyFieldBulkCopyG2S { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldBulkCopyG2SAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.atom_copy_field_bulks2g`: Fields stored in Bulk Store Copy Atom type. +pub const AtomCopyFieldBulkCopyS2GAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldBulkCopyS2G; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "atom_copy_field_bulks2g"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomCopyFieldBulkCopyS2G, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomCopyFieldBulkCopyS2GAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) AtomCopyFieldBulkCopyS2G { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldBulkCopyS2GAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.atom_copy_field_bulks2s`: Fields stored in Bulk CTA to Cluster Copy Atom type. +pub const AtomCopyFieldBulkCopyS2SAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldBulkCopyS2S; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "atom_copy_field_bulks2s"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomCopyFieldBulkCopyS2S, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomCopyFieldBulkCopyS2SAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) AtomCopyFieldBulkCopyS2S { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldBulkCopyS2SAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.atom_copy_field_dsmem_store`: Fields stored in CopyAtomDsmemStoreType. +pub const AtomCopyFieldDsmemStoreAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldDsmemStore; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "atom_copy_field_dsmem_store"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomCopyFieldDsmemStore, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomCopyFieldDsmemStoreAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) AtomCopyFieldDsmemStore { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldDsmemStoreAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.atom_copy_field_g2r`: Fields stored in CopyAtomG2RType. +pub const AtomCopyFieldLoadGlobalAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldLoadGlobal; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "atom_copy_field_g2r"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomCopyFieldLoadGlobal, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomCopyFieldLoadGlobalAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) AtomCopyFieldLoadGlobal { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldLoadGlobalAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.atom_copy_field_non_exec_2d_gather4_tma_load`: Fields stored in CopyAtomNonExec2DGather4TmaLoadType. +pub const AtomCopyFieldNonExec2DGather4TmaLoadAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldNonExec2DGather4TmaLoad; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "atom_copy_field_non_exec_2d_gather4_tma_load"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomCopyFieldNonExec2DGather4TmaLoad, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomCopyFieldNonExec2DGather4TmaLoadAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) AtomCopyFieldNonExec2DGather4TmaLoad { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldNonExec2DGather4TmaLoadAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.atom_copy_field_non_exec_2d_scatter4_tma_store`: Fields stored in CopyAtomNonExec2DScatter4TmaStoreType. +pub const AtomCopyFieldNonExec2DScatter4TmaStoreAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldNonExec2DScatter4TmaStore; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "atom_copy_field_non_exec_2d_scatter4_tma_store"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomCopyFieldNonExec2DScatter4TmaStore, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomCopyFieldNonExec2DScatter4TmaStoreAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) AtomCopyFieldNonExec2DScatter4TmaStore { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldNonExec2DScatter4TmaStoreAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.atom_copy_field_non_exec_im2col_tma_load`: Fields stored in CopyAtomNonExecIm2ColTmaLoadType. +pub const AtomCopyFieldNonExecIm2ColTmaLoadAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecIm2ColTmaLoad; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "atom_copy_field_non_exec_im2col_tma_load"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomCopyFieldNonExecIm2ColTmaLoad, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaLoadAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) AtomCopyFieldNonExecIm2ColTmaLoad { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaLoadAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.atom_copy_field_non_exec_im2col_tma_store`: Fields stored in CopyAtomNonExecIm2ColTmaStoreType. +pub const AtomCopyFieldNonExecIm2ColTmaStoreAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecIm2ColTmaStore; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "atom_copy_field_non_exec_im2col_tma_store"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomCopyFieldNonExecIm2ColTmaStore, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaStoreAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) AtomCopyFieldNonExecIm2ColTmaStore { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaStoreAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.atom_copy_field_non_exec_tma_load`: Fields stored in CopyAtomNonExecTiledTmaLoadType. +pub const AtomCopyFieldNonExecTiledTmaLoadAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecTiledTmaLoad; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "atom_copy_field_non_exec_tma_load"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomCopyFieldNonExecTiledTmaLoad, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaLoadAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) AtomCopyFieldNonExecTiledTmaLoad { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaLoadAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.atom_copy_field_non_exec_tma_reduce`: Fields stored in CopyAtomNonExecTiledTmaReduceType. +pub const AtomCopyFieldNonExecTiledTmaReduceAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecTiledTmaReduce; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "atom_copy_field_non_exec_tma_reduce"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomCopyFieldNonExecTiledTmaReduce, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaReduceAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) AtomCopyFieldNonExecTiledTmaReduce { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaReduceAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.atom_copy_field_non_exec_tma_store`: Fields stored in CopyAtomNonExecTiledTmaStoreType. +pub const AtomCopyFieldNonExecTiledTmaStoreAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecTiledTmaStore; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "atom_copy_field_non_exec_tma_store"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomCopyFieldNonExecTiledTmaStore, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaStoreAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) AtomCopyFieldNonExecTiledTmaStore { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaStoreAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.atom_copy_field_r2g`: Fields stored in CopyAtomR2GType. +pub const AtomCopyFieldStoreGlobalAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldStoreGlobal; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "atom_copy_field_r2g"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomCopyFieldStoreGlobal, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomCopyFieldStoreGlobalAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) AtomCopyFieldStoreGlobal { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldStoreGlobalAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.atom_copy_field_tmaload`: Fields stored in Tma Load Copy Atom type. +pub const AtomCopyFieldTmaLoadAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldTmaLoad; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "atom_copy_field_tmaload"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomCopyFieldTmaLoad, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomCopyFieldTmaLoadAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) AtomCopyFieldTmaLoad { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldTmaLoadAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.atom_copy_field_tmareduce`: Fields stored in Tma Reduce Copy Atom type. +pub const AtomCopyFieldTmaReduceAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldTmaReduce; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "atom_copy_field_tmareduce"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomCopyFieldTmaReduce, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomCopyFieldTmaReduceAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) AtomCopyFieldTmaReduce { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldTmaReduceAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.atom_copy_field_tmastore`: Fields stored in Tma Store Copy Atom type. +pub const AtomCopyFieldTmaStoreAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldTmaStore; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "atom_copy_field_tmastore"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomCopyFieldTmaStore, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomCopyFieldTmaStoreAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) AtomCopyFieldTmaStore { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldTmaStoreAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.atom_mma_field_sm100`: Fields stored in MMA Atom types for SM100. +pub const AtomMmaFieldSM100Attr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomMmaFieldSM100; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "atom_mma_field_sm100"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomMmaFieldSM100, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomMmaFieldSM100AttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) AtomMmaFieldSM100 { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomMmaFieldSM100AttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.atom_mma_field_sm100_block_scaled`: Fields stored in MMA Atom types for SM100 (block scaled). +pub const AtomMmaFieldSM100BlockScaledAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomMmaFieldSM100BlockScaled; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "atom_mma_field_sm100_block_scaled"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomMmaFieldSM100BlockScaled, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomMmaFieldSM100BlockScaledAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) AtomMmaFieldSM100BlockScaled { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomMmaFieldSM100BlockScaledAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.atom_mma_field_sm100_block_scaled_sparse`: Fields stored in MMA Atom types for SM100 (block scaled). +pub const AtomMmaFieldSM100BlockScaledSparseAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomMmaFieldSM100BlockScaledSparse; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "atom_mma_field_sm100_block_scaled_sparse"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomMmaFieldSM100BlockScaledSparse, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomMmaFieldSM100BlockScaledSparseAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) AtomMmaFieldSM100BlockScaledSparse { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomMmaFieldSM100BlockScaledSparseAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.atom_mma_field_sm100_sparse`: Fields stored in sparse MMA Atom types for SM100. +pub const AtomMmaFieldSM100SparseAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomMmaFieldSM100Sparse; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "atom_mma_field_sm100_sparse"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomMmaFieldSM100Sparse, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomMmaFieldSM100SparseAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) AtomMmaFieldSM100Sparse { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomMmaFieldSM100SparseAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.atom_mma_field_sm120_block_scaled`: Fields stored in MMA Atom types for SM120 (block scaled). +pub const AtomMmaFieldSM120BlockScaledAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomMmaFieldSM120BlockScaled; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "atom_mma_field_sm120_block_scaled"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomMmaFieldSM120BlockScaled, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomMmaFieldSM120BlockScaledAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) AtomMmaFieldSM120BlockScaled { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomMmaFieldSM120BlockScaledAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.atom_mma_field_sm80_sparse`: Fields stored in MMA Atom types for SM80. +pub const AtomMmaFieldSM80SparseAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomMmaFieldSM80Sparse; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "atom_mma_field_sm80_sparse"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomMmaFieldSM80Sparse, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomMmaFieldSM80SparseAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) AtomMmaFieldSM80Sparse { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomMmaFieldSM80SparseAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.atom_mma_field_sm90`: Fields stored in MMA Atom types for SM90. +pub const AtomMmaFieldSM90Attr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomMmaFieldSM90; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "atom_mma_field_sm90"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomMmaFieldSM90, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUAtomMmaFieldSM90AttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) AtomMmaFieldSM90 { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomMmaFieldSM90AttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.bin_op`: Binary operation for single-bit MMA operations. +pub const BinaryOpAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUBinaryOp; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "bin_op"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: BinaryOp, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUBinaryOpAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) BinaryOp { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUBinaryOpAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.copy_s2t_broadcast_mode`: Broadcast modes for the different utccp instructions. +pub const CopyS2TBroadcastAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUCopyS2TBroadcast; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "copy_s2t_broadcast_mode"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: CopyS2TBroadcast, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUCopyS2TBroadcastAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) CopyS2TBroadcast { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyS2TBroadcastAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.gather_scatter_tma_load`: The various kinds of TMA loads in gather/scatter mode. +pub const GatherScatterTmaLoadAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUGatherScatterTmaLoad; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "gather_scatter_tma_load"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: GatherScatterTmaLoad, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUGatherScatterTmaLoadAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) GatherScatterTmaLoad { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUGatherScatterTmaLoadAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.im2col_tma_load`: The various kinds of TMA loads in im2col mode. +pub const Im2ColTmaLoadAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUIm2ColTmaLoad; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "im2col_tma_load"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: Im2ColTmaLoad, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUIm2ColTmaLoadAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) Im2ColTmaLoad { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUIm2ColTmaLoadAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.ld_reduce_acc_precision_kind`: multimem ld_reduce accumulation precision kind. +pub const LdReduceAccPrecisionKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPULdReduceAccPrecisionKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "ld_reduce_acc_precision_kind"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: LdReduceAccPrecisionKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPULdReduceAccPrecisionKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) LdReduceAccPrecisionKind { + return @fromBackingInt(@intCast(c.mlirCuteNVGPULdReduceAccPrecisionKindAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.ldsm_sz_pattern`: LDSM's sz pattern, describing the bit size. +pub const LdsmSzPatternAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPULdsmSzPattern; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "ldsm_sz_pattern"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: LdsmSzPattern, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPULdsmSzPatternAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) LdsmSzPattern { + return @fromBackingInt(@intCast(c.mlirCuteNVGPULdsmSzPatternAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.load_cache_mode`: Cache modes for the load instructions. +pub const LoadCacheModeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPULoadCacheMode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "load_cache_mode"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: LoadCacheMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPULoadCacheModeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) LoadCacheMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPULoadCacheModeAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.mma_int_overflow`: MMA overflow options. +pub const MMAIntOverflowAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUMMAIntOverflow; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "mma_int_overflow"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MMAIntOverflow, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUMMAIntOverflowAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) MMAIntOverflow { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMMAIntOverflowAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.major`: Major mode for MMA operations. +pub const MajorModeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUMajorMode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "major"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: cute_ir.MajorMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUMajorModeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) cute_ir.MajorMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMajorModeAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.mma_collector_op`: Enums for the mma collector op. +pub const MmaCollectorOpAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUMmaCollectorOp; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "mma_collector_op"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MmaCollectorOp, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUMmaCollectorOpAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) MmaCollectorOp { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaCollectorOpAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.mma_frag_kind`: Enums for the mma frag type. +pub const MmaFragKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUMmaFragKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "mma_frag_kind"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MmaFragKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUMmaFragKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) MmaFragKind { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaFragKindAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.not_implemented_frg`: Fragment of an atom that has none. +pub const NotImplementedFrgAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUNotImplementedFrg; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "not_implemented_frg"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirCuteNVGPUNotImplementedFrgAttrGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `#cute_nvgpu.tma_reduce_kind`: Op for the TMASTORE instruction. +pub const ReductionKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUReductionKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "tma_reduce_kind"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: ReductionKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUReductionKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) ReductionKind { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUReductionKindAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.rmem_frg`: Register-memory fragment of an MMA operand. +pub const RmemFrgAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPURmemFrg; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "rmem_frg"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + valueType: *const mlir.Type, + operand: cute_ir.MmaOperand, + deriveElemType: bool = false, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPURmemFrgAttrGet(ctx.ptr(), args.valueType.ptr(), @backingInt(args.operand), args.deriveElemType); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn valueType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPURmemFrgAttrGetValueType(self.ptr()).ptr.?); + } + pub fn operand(self: *const Self) cute_ir.MmaOperand { + return @fromBackingInt(@intCast(c.mlirCuteNVGPURmemFrgAttrGetOperand(self.ptr()))); + } + pub fn deriveElemType(self: *const Self) bool { + return c.mlirCuteNVGPURmemFrgAttrGetDeriveElemType(self.ptr()); + } +}; + +/// `#cute_nvgpu.arch.sm100.circular_smem_frg`: SM100 circular shared-memory fragment. +pub const SM100CircularSmemFrgAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUSM100CircularSmemFrg; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "arch.sm100.circular_smem_frg"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + majorMode: cute_ir.MajorMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUSM100CircularSmemFrgAttrGet(ctx.ptr(), @backingInt(args.majorMode)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn majorMode(self: *const Self) cute_ir.MajorMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUSM100CircularSmemFrgAttrGetMajorMode(self.ptr()))); + } +}; + +/// `#cute_nvgpu.arch.sm100.smem_frg`: SM100 shared-memory fragment. +pub const SM100SmemFrgAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUSM100SmemFrg; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "arch.sm100.smem_frg"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + majorMode: cute_ir.MajorMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUSM100SmemFrgAttrGet(ctx.ptr(), @backingInt(args.majorMode)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn majorMode(self: *const Self) cute_ir.MajorMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUSM100SmemFrgAttrGetMajorMode(self.ptr()))); + } +}; + +/// `#cute_nvgpu.arch.sm100.tmem_e_frg`: SM100 tensor-memory sparse-metadata fragment. +pub const SM100TmemEFrgAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUSM100TmemEFrg; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "arch.sm100.tmem_e_frg"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + aType: *const mlir.Type, + eType: *const mlir.Type, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUSM100TmemEFrgAttrGet(ctx.ptr(), args.aType.ptr(), args.eType.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn aType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUSM100TmemEFrgAttrGetAType(self.ptr()).ptr.?); + } + pub fn eType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUSM100TmemEFrgAttrGetEType(self.ptr()).ptr.?); + } +}; + +/// `#cute_nvgpu.arch.sm100.tmem_frg`: SM100 tensor-memory fragment. +pub const SM100TmemFrgAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUSM100TmemFrg; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "arch.sm100.tmem_frg"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + dataType: *const mlir.Type, + storageType: *const mlir.Type, + ctaGroup: c_int, + tmemAllocMode: TmemAllocMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUSM100TmemFrgAttrGet(ctx.ptr(), args.dataType.ptr(), args.storageType.ptr(), args.ctaGroup, @backingInt(args.tmemAllocMode)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn dataType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUSM100TmemFrgAttrGetDataType(self.ptr()).ptr.?); + } + pub fn storageType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUSM100TmemFrgAttrGetStorageType(self.ptr()).ptr.?); + } + pub fn ctaGroup(self: *const Self) c_int { + return c.mlirCuteNVGPUSM100TmemFrgAttrGetCtaGroup(self.ptr()); + } + pub fn tmemAllocMode(self: *const Self) TmemAllocMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUSM100TmemFrgAttrGetTmemAllocMode(self.ptr()))); + } +}; + +/// `#cute_nvgpu.arch.sm100.tmem_sf_frg`: SM100 tensor-memory scale-factor fragment. +pub const SM100TmemSfFrgAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUSM100TmemSfFrg; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "arch.sm100.tmem_sf_frg"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + sfType: *const mlir.Type, + sfVecSize: c_int, + ctaGroup: c_int, + isSfa: bool, + tmemAllocMode: TmemAllocMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUSM100TmemSfFrgAttrGet(ctx.ptr(), args.sfType.ptr(), args.sfVecSize, args.ctaGroup, args.isSfa, @backingInt(args.tmemAllocMode)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn sfType(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirCuteNVGPUSM100TmemSfFrgAttrGetSfType(self.ptr()).ptr.?); + } + pub fn sfVecSize(self: *const Self) c_int { + return c.mlirCuteNVGPUSM100TmemSfFrgAttrGetSfVecSize(self.ptr()); + } + pub fn ctaGroup(self: *const Self) c_int { + return c.mlirCuteNVGPUSM100TmemSfFrgAttrGetCtaGroup(self.ptr()); + } + pub fn isSfa(self: *const Self) bool { + return c.mlirCuteNVGPUSM100TmemSfFrgAttrGetIsSfa(self.ptr()); + } + pub fn tmemAllocMode(self: *const Self) TmemAllocMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUSM100TmemSfFrgAttrGetTmemAllocMode(self.ptr()))); + } +}; + +/// `#cute_nvgpu.arch.sm107.smem_frg`: SM107 shared-memory fragment. +pub const SM107SmemFrgAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUSM107SmemFrg; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "arch.sm107.smem_frg"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + majorMode: cute_ir.MajorMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUSM107SmemFrgAttrGet(ctx.ptr(), @backingInt(args.majorMode)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn majorMode(self: *const Self) cute_ir.MajorMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUSM107SmemFrgAttrGetMajorMode(self.ptr()))); + } +}; + +/// `#cute_nvgpu.arch.sm90.smem_frg`: SM90 shared-memory fragment. +pub const SM90SmemFrgAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUSM90SmemFrg; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "arch.sm90.smem_frg"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + majorMode: cute_ir.MajorMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUSM90SmemFrgAttrGet(ctx.ptr(), @backingInt(args.majorMode)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn majorMode(self: *const Self) cute_ir.MajorMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUSM90SmemFrgAttrGetMajorMode(self.ptr()))); + } +}; + +/// `#cute_nvgpu.tiled_tma_load`: The various kinds of TMA loads in tiled mode. +pub const TiledTmaLoadAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUTiledTmaLoad; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "tiled_tma_load"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: TiledTmaLoad, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUTiledTmaLoadAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) TiledTmaLoad { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUTiledTmaLoadAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.tma_data_format`: Bits 74-71 of the TMA descriptor. +pub const TmaDataFormatAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUTmaDataFormat; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "tma_data_format"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: TmaDataFormat, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUTmaDataFormatAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) TmaDataFormat { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUTmaDataFormatAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.tma_load_mode`: Modes for the TMA load instruction. +pub const TmaLoadModeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUTmaLoadMode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "tma_load_mode"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: TmaLoadMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUTmaLoadModeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) TmaLoadMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUTmaLoadModeAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.tma_store_mode`: Modes for the TMA store instruction. +pub const TmaStoreModeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUTmaStoreMode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "tma_store_mode"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: TmaStoreMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUTmaStoreModeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) TmaStoreMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUTmaStoreModeAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.tmem_alloc_mode`: Modes for the Tmem allocation. +pub const TmemAllocModeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUTmemAllocMode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "tmem_alloc_mode"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: TmemAllocMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUTmemAllocModeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) TmemAllocMode { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUTmemAllocModeAttrGetValue(self.ptr()))); + } +}; + +/// `#cute_nvgpu.tmem_load_red_op`: Reduce operation for the Tmem load instruction. +pub const TmemLoadRedOpAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVGPUTmemLoadRedOp; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "tmem_load_red_op"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: TmemLoadRedOp, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVGPUTmemLoadRedOpAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) TmemLoadRedOp { + return @fromBackingInt(@intCast(c.mlirCuteNVGPUTmemLoadRedOpAttrGetValue(self.ptr()))); + } +}; + +// Operation builders, generated from the operations' .td. +/// `cute_nvgpu.arch.alloc_rmem`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_alloc_rmem(ctx: *mlir.Context, input: *const mlir.Value, ptr_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.arch.alloc_rmem", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{ptr_type} }, + .location = location, + }); +} + +/// `cute_nvgpu.arch.alloc_smem`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_alloc_smem(ctx: *mlir.Context, ptr_type: *const mlir.Type, input: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.arch.alloc_smem", .{ + .results = .{ .flat = &.{ptr_type} }, + .attributes = &.{ + .named(ctx, "input", input), + }, + .location = location, + }); +} + +/// `cute_nvgpu.arch.copy.SM100.copy_s2t`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_copy_SM100_copy_s2t(ctx: *mlir.Context, src: *const mlir.Value, dst: *const mlir.Value, dp: *const mlir.Attribute, bits: *const mlir.Attribute, cta: *const mlir.Attribute, broadcast: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.arch.copy.SM100.copy_s2t", .{ + .operands = .{ .flat = &.{ src, dst } }, + .attributes = &.{ + .named(ctx, "dp", dp), + .named(ctx, "bits", bits), + .named(ctx, "cta", cta), + .named(ctx, "broadcast", broadcast), + }, + .location = location, + }); +} + +/// `cute_nvgpu.arch.copy.SM100.tma_load`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_copy_SM100_tma_load( + ctx: *mlir.Context, + src_desc: *const mlir.Value, + dsmem_data_addr: *const mlir.Value, + dsmem_bar_addr: *const mlir.Value, + coord: []const *const mlir.Value, + offsets: []const *const mlir.Value, + mode: *const mlir.Attribute, + num_cta: *const mlir.Attribute, + opts: struct { + multicast_mask: ?*const mlir.Value = null, + cache_policy: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.arch.copy.SM100.tma_load", .{ + .operands = .{ .variadic = &.{ + &.{src_desc}, + &.{dsmem_data_addr}, + &.{dsmem_bar_addr}, + coord, + if (opts.multicast_mask) |value| &.{value} else &.{}, + offsets, + if (opts.cache_policy) |value| &.{value} else &.{}, + } }, + .attributes = &.{ + .named(ctx, "mode", mode), + .named(ctx, "num_cta", num_cta), + }, + .location = location, + }); +} + +/// `cute_nvgpu.arch.copy.SM100.tma_reduce`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_copy_SM100_tma_reduce( + ctx: *mlir.Context, + dst_desc: *const mlir.Value, + src_smem_data_addr: *const mlir.Value, + coord: []const *const mlir.Value, + mode: *const mlir.Attribute, + kind: *const mlir.Attribute, + opts: struct { + cache_policy: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.arch.copy.SM100.tma_reduce", .{ + .operands = .{ .variadic = &.{ + &.{dst_desc}, + &.{src_smem_data_addr}, + coord, + if (opts.cache_policy) |value| &.{value} else &.{}, + } }, + .attributes = &.{ + .named(ctx, "mode", mode), + .named(ctx, "kind", kind), + }, + .location = location, + }); +} + +/// `cute_nvgpu.arch.copy.SM100.tma_store`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_copy_SM100_tma_store( + ctx: *mlir.Context, + dst_desc: *const mlir.Value, + src_smem_data_addr: *const mlir.Value, + coord: []const *const mlir.Value, + mode: *const mlir.Attribute, + opts: struct { + cache_policy: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.arch.copy.SM100.tma_store", .{ + .operands = .{ .variadic = &.{ + &.{dst_desc}, + &.{src_smem_data_addr}, + coord, + if (opts.cache_policy) |value| &.{value} else &.{}, + } }, + .attributes = &.{ + .named(ctx, "mode", mode), + }, + .location = location, + }); +} + +/// `cute_nvgpu.arch.copy.SM100.tmem_load`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_copy_SM100_tmem_load( + ctx: *mlir.Context, + src: *const mlir.Value, + dst_type: *const mlir.Type, + num_dp: *const mlir.Attribute, + num_b: *const mlir.Attribute, + num_rep: *const mlir.Attribute, + opts: struct { + pack_16: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "num_dp", num_dp)); + attributes.appendAssumeCapacity(.named(ctx, "num_b", num_b)); + attributes.appendAssumeCapacity(.named(ctx, "num_rep", num_rep)); + if (opts.pack_16) |value| attributes.appendAssumeCapacity(.named(ctx, "pack_16", value)); + return mlir.Operation.make(ctx, "cute_nvgpu.arch.copy.SM100.tmem_load", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute_nvgpu.arch.copy.SM100.tmem_store`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_copy_SM100_tmem_store( + ctx: *mlir.Context, + dst: *const mlir.Value, + src: *const mlir.Value, + num_dp: *const mlir.Attribute, + num_b: *const mlir.Attribute, + num_rep: *const mlir.Attribute, + opts: struct { + expand_16: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "num_dp", num_dp)); + attributes.appendAssumeCapacity(.named(ctx, "num_b", num_b)); + attributes.appendAssumeCapacity(.named(ctx, "num_rep", num_rep)); + if (opts.expand_16) |value| attributes.appendAssumeCapacity(.named(ctx, "expand_16", value)); + return mlir.Operation.make(ctx, "cute_nvgpu.arch.copy.SM100.tmem_store", .{ + .operands = .{ .flat = &.{ dst, src } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute_nvgpu.arch.copy.SM107.tma_load_override`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_copy_SM107_tma_load_override( + ctx: *mlir.Context, + src_desc: *const mlir.Value, + dsmem_data_addr: *const mlir.Value, + dsmem_bar_addr: *const mlir.Value, + override_addr: *const mlir.Value, + tensor_size: []const *const mlir.Value, + lower_stride: []const *const mlir.Value, + coord: []const *const mlir.Value, + mode: *const mlir.Attribute, + num_cta: *const mlir.Attribute, + opts: struct { + upper_stride: ?*const mlir.Value = null, + multicast_mask: ?*const mlir.Value = null, + cache_policy: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.arch.copy.SM107.tma_load_override", .{ + .operands = .{ .variadic = &.{ + &.{src_desc}, + &.{dsmem_data_addr}, + &.{dsmem_bar_addr}, + &.{override_addr}, + tensor_size, + lower_stride, + if (opts.upper_stride) |value| &.{value} else &.{}, + coord, + if (opts.multicast_mask) |value| &.{value} else &.{}, + if (opts.cache_policy) |value| &.{value} else &.{}, + } }, + .attributes = &.{ + .named(ctx, "mode", mode), + .named(ctx, "num_cta", num_cta), + }, + .location = location, + }); +} + +/// `cute_nvgpu.arch.copy.SM107.tma_store_override`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_copy_SM107_tma_store_override( + ctx: *mlir.Context, + dst_desc: *const mlir.Value, + src_smem_data_addr: *const mlir.Value, + override_addr: *const mlir.Value, + tensor_size: []const *const mlir.Value, + lower_stride: []const *const mlir.Value, + coord: []const *const mlir.Value, + mode: *const mlir.Attribute, + opts: struct { + upper_stride: ?*const mlir.Value = null, + cache_policy: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.arch.copy.SM107.tma_store_override", .{ + .operands = .{ .variadic = &.{ + &.{dst_desc}, + &.{src_smem_data_addr}, + &.{override_addr}, + tensor_size, + lower_stride, + if (opts.upper_stride) |value| &.{value} else &.{}, + coord, + if (opts.cache_policy) |value| &.{value} else &.{}, + } }, + .attributes = &.{ + .named(ctx, "mode", mode), + }, + .location = location, + }); +} + +/// `cute_nvgpu.arch.copy.SM90.bulk_copy_g2s`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_copy_SM90_bulk_copy_g2s( + ctx: *mlir.Context, + gmem_data_addr: *const mlir.Value, + dsmem_data_addr: *const mlir.Value, + dsmem_bar_addr: *const mlir.Value, + size: *const mlir.Attribute, + opts: struct { + multicast_mask: ?*const mlir.Value = null, + cache_policy: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.arch.copy.SM90.bulk_copy_g2s", .{ + .operands = .{ .variadic = &.{ + &.{gmem_data_addr}, + &.{dsmem_data_addr}, + &.{dsmem_bar_addr}, + if (opts.multicast_mask) |value| &.{value} else &.{}, + if (opts.cache_policy) |value| &.{value} else &.{}, + } }, + .attributes = &.{ + .named(ctx, "size", size), + }, + .location = location, + }); +} + +/// `cute_nvgpu.arch.copy.SM90.bulk_copy_s2g`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_copy_SM90_bulk_copy_s2g( + ctx: *mlir.Context, + smem_data_addr: *const mlir.Value, + gmem_data_addr: *const mlir.Value, + size: *const mlir.Attribute, + opts: struct { + byte_mask: ?*const mlir.Value = null, + cache_policy: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.arch.copy.SM90.bulk_copy_s2g", .{ + .operands = .{ .variadic = &.{ + &.{smem_data_addr}, + &.{gmem_data_addr}, + if (opts.byte_mask) |value| &.{value} else &.{}, + if (opts.cache_policy) |value| &.{value} else &.{}, + } }, + .attributes = &.{ + .named(ctx, "size", size), + }, + .location = location, + }); +} + +/// `cute_nvgpu.arch.copy.SM90.bulk_copy_s2s`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_copy_SM90_bulk_copy_s2s(ctx: *mlir.Context, smem_data_addr: *const mlir.Value, dsmem_data_addr: *const mlir.Value, dsmem_bar_addr: *const mlir.Value, cta_rank: *const mlir.Value, size: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.arch.copy.SM90.bulk_copy_s2s", .{ + .operands = .{ .flat = &.{ smem_data_addr, dsmem_data_addr, dsmem_bar_addr, cta_rank } }, + .attributes = &.{ + .named(ctx, "size", size), + }, + .location = location, + }); +} + +/// `cute_nvgpu.arch.get_dyn_smem`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_get_dyn_smem(ctx: *mlir.Context, ptr_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.arch.get_dyn_smem", .{ + .results = .{ .flat = &.{ptr_type} }, + .location = location, + }); +} + +/// `cute_nvgpu.arch.get_dyn_smem_size`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_get_dyn_smem_size(ctx: *mlir.Context, size_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.arch.get_dyn_smem_size", .{ + .results = .{ .flat = &.{size_type} }, + .location = location, + }); +} + +/// `cute_nvgpu.arch.make_warp_uniform`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_make_warp_uniform(ctx: *mlir.Context, val: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.arch.make_warp_uniform", .{ + .operands = .{ .flat = &.{val} }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `cute_nvgpu.arch.mma.SM107.umma`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_mma_SM107_umma(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, cd: *const mlir.Value, idesc: *const mlir.Value, accum: *const mlir.Value, disable_output_lane: *const mlir.Value, num_cta: *const mlir.Attribute, scale: *const mlir.Attribute, a_type: *const mlir.Attribute, b_type: *const mlir.Attribute, shape_k: *const mlir.Attribute, a_collector_op: *const mlir.Attribute, b_collector_op: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.arch.mma.SM107.umma", .{ + .operands = .{ .flat = &.{ a, b, cd, idesc, accum, disable_output_lane } }, + .attributes = &.{ + .named(ctx, "num_cta", num_cta), + .named(ctx, "scale", scale), + .named(ctx, "a_type", a_type), + .named(ctx, "b_type", b_type), + .named(ctx, "shape_k", shape_k), + .named(ctx, "a_collector_op", a_collector_op), + .named(ctx, "b_collector_op", b_collector_op), + }, + .location = location, + }); +} + +/// `cute_nvgpu.arch.mma.SM107.umma_block_scaled`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_mma_SM107_umma_block_scaled(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, cd: *const mlir.Value, sf_a: *const mlir.Value, sf_b: *const mlir.Value, idesc: *const mlir.Value, accum: *const mlir.Value, num_cta: *const mlir.Attribute, vec_size: *const mlir.Attribute, a_type: *const mlir.Attribute, b_type: *const mlir.Attribute, shape_k: *const mlir.Attribute, a_collector_op: *const mlir.Attribute, b_collector_op: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.arch.mma.SM107.umma_block_scaled", .{ + .operands = .{ .flat = &.{ a, b, cd, sf_a, sf_b, idesc, accum } }, + .attributes = &.{ + .named(ctx, "num_cta", num_cta), + .named(ctx, "vec_size", vec_size), + .named(ctx, "a_type", a_type), + .named(ctx, "b_type", b_type), + .named(ctx, "shape_k", shape_k), + .named(ctx, "a_collector_op", a_collector_op), + .named(ctx, "b_collector_op", b_collector_op), + }, + .location = location, + }); +} + +/// `cute_nvgpu.arch.mma.SM107.umma_block_scaled_sparse`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_mma_SM107_umma_block_scaled_sparse(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, cd: *const mlir.Value, sf_a: *const mlir.Value, sf_b: *const mlir.Value, e_op: *const mlir.Value, idesc: *const mlir.Value, accum: *const mlir.Value, num_cta: *const mlir.Attribute, vec_size: *const mlir.Attribute, a_type: *const mlir.Attribute, b_type: *const mlir.Attribute, shape_k: *const mlir.Attribute, a_collector_op: *const mlir.Attribute, b_collector_op: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.arch.mma.SM107.umma_block_scaled_sparse", .{ + .operands = .{ .flat = &.{ a, b, cd, sf_a, sf_b, e_op, idesc, accum } }, + .attributes = &.{ + .named(ctx, "num_cta", num_cta), + .named(ctx, "vec_size", vec_size), + .named(ctx, "a_type", a_type), + .named(ctx, "b_type", b_type), + .named(ctx, "shape_k", shape_k), + .named(ctx, "a_collector_op", a_collector_op), + .named(ctx, "b_collector_op", b_collector_op), + }, + .location = location, + }); +} + +/// `cute_nvgpu.arch.mma.SM107.umma_sparse`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_mma_SM107_umma_sparse(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, cd: *const mlir.Value, e: *const mlir.Value, idesc: *const mlir.Value, accum: *const mlir.Value, num_cta: *const mlir.Attribute, scale: *const mlir.Attribute, a_type: *const mlir.Attribute, b_type: *const mlir.Attribute, shape_k: *const mlir.Attribute, a_collector_op: *const mlir.Attribute, b_collector_op: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.arch.mma.SM107.umma_sparse", .{ + .operands = .{ .flat = &.{ a, b, cd, e, idesc, accum } }, + .attributes = &.{ + .named(ctx, "num_cta", num_cta), + .named(ctx, "scale", scale), + .named(ctx, "a_type", a_type), + .named(ctx, "b_type", b_type), + .named(ctx, "shape_k", shape_k), + .named(ctx, "a_collector_op", a_collector_op), + .named(ctx, "b_collector_op", b_collector_op), + }, + .location = location, + }); +} + +/// `cute_nvgpu.arch.mma.SM120.block_scaled`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_mma_SM120_block_scaled( + ctx: *mlir.Context, + opA: []const *const mlir.Value, + opB: []const *const mlir.Value, + opC: []const *const mlir.Value, + sfA: *const mlir.Value, + sfB: *const mlir.Value, + res_types: []const *const mlir.Type, + shape_MNK: *const mlir.Attribute, + vec_size: *const mlir.Attribute, + a_type: *const mlir.Attribute, + b_type: *const mlir.Attribute, + sf_type: *const mlir.Attribute, + opts: struct { + byte_id_a: ?*const mlir.Value = null, + byte_id_b: ?*const mlir.Value = null, + thread_id_a: ?*const mlir.Attribute = null, + thread_id_b: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 7) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "shape_MNK", shape_MNK)); + attributes.appendAssumeCapacity(.named(ctx, "vec_size", vec_size)); + if (opts.thread_id_a) |value| attributes.appendAssumeCapacity(.named(ctx, "thread_id_a", value)); + if (opts.thread_id_b) |value| attributes.appendAssumeCapacity(.named(ctx, "thread_id_b", value)); + attributes.appendAssumeCapacity(.named(ctx, "a_type", a_type)); + attributes.appendAssumeCapacity(.named(ctx, "b_type", b_type)); + attributes.appendAssumeCapacity(.named(ctx, "sf_type", sf_type)); + return mlir.Operation.make(ctx, "cute_nvgpu.arch.mma.SM120.block_scaled", .{ + .operands = .{ .variadic = &.{ + opA, + opB, + opC, + &.{sfA}, + &.{sfB}, + if (opts.byte_id_a) |value| &.{value} else &.{}, + if (opts.byte_id_b) |value| &.{value} else &.{}, + } }, + .results = .{ .variadic = &.{ + res_types, + } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute_nvgpu.arch.prefetch_tma_desc`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_prefetch_tma_desc(ctx: *mlir.Context, tma_descriptor_ptr: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.arch.prefetch_tma_desc", .{ + .operands = .{ .flat = &.{tma_descriptor_ptr} }, + .location = location, + }); +} + +/// `cute_nvgpu.arch.sm100.alloc_tmem`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_sm100_alloc_tmem( + ctx: *mlir.Context, + n_cols: *const mlir.Value, + dst_ptr: *const mlir.Value, + opts: struct { + is_two_cta: ?*const mlir.Attribute = null, + exclusive: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.is_two_cta) |value| attributes.appendAssumeCapacity(.named(ctx, "is_two_cta", value)); + if (opts.exclusive) |value| attributes.appendAssumeCapacity(.named(ctx, "exclusive", value)); + return mlir.Operation.make(ctx, "cute_nvgpu.arch.sm100.alloc_tmem", .{ + .operands = .{ .flat = &.{ n_cols, dst_ptr } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute_nvgpu.arch.sm100.dealloc_tmem`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_sm100_dealloc_tmem( + ctx: *mlir.Context, + tmem_ptr: *const mlir.Value, + n_cols: *const mlir.Value, + opts: struct { + is_two_cta: ?*const mlir.Attribute = null, + exclusive: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.is_two_cta) |value| attributes.appendAssumeCapacity(.named(ctx, "is_two_cta", value)); + if (opts.exclusive) |value| attributes.appendAssumeCapacity(.named(ctx, "exclusive", value)); + return mlir.Operation.make(ctx, "cute_nvgpu.arch.sm100.dealloc_tmem", .{ + .operands = .{ .flat = &.{ tmem_ptr, n_cols } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute_nvgpu.arch.sm100.relinquish_tmem_alloc_permit`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_sm100_relinquish_tmem_alloc_permit( + ctx: *mlir.Context, + opts: struct { + is_two_cta: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.is_two_cta) |value| attributes.appendAssumeCapacity(.named(ctx, "is_two_cta", value)); + return mlir.Operation.make(ctx, "cute_nvgpu.arch.sm100.relinquish_tmem_alloc_permit", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute_nvgpu.arch.sm100.retrieve_tmem_ptr`. Result types are explicit; attributes use mlir.Attribute. +pub fn arch_sm100_retrieve_tmem_ptr(ctx: *mlir.Context, smem_ptr: *const mlir.Value, tmem_ptr_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.arch.sm100.retrieve_tmem_ptr", .{ + .operands = .{ .flat = &.{smem_ptr} }, + .results = .{ .flat = &.{tmem_ptr_type} }, + .location = location, + }); +} + +/// `cute_nvgpu.atom.get_coord_tensor`. Result types are explicit; attributes use mlir.Attribute. +pub fn atom_get_coord_tensor( + ctx: *mlir.Context, + atom: *const mlir.Value, + shape: *const mlir.Value, + result_type: *const mlir.Type, + opts: struct { + offset_b: ?*const mlir.Value = null, + layout_b: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.atom.get_coord_tensor", .{ + .operands = .{ .variadic = &.{ + &.{atom}, + &.{shape}, + if (opts.offset_b) |value| &.{value} else &.{}, + if (opts.layout_b) |value| &.{value} else &.{}, + } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute_nvgpu.atom.get_copy_s2t_smem_desc_view`. Result types are explicit; attributes use mlir.Attribute. +pub fn atom_get_copy_s2t_smem_desc_view(ctx: *mlir.Context, atom: *const mlir.Value, view: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.atom.get_copy_s2t_smem_desc_view", .{ + .operands = .{ .flat = &.{ atom, view } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute_nvgpu.atom.get_value`. Result types are explicit; attributes use mlir.Attribute. +pub fn atom_get_value(ctx: *mlir.Context, atom: *const mlir.Value, result_type: *const mlir.Type, field: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.atom.get_value", .{ + .operands = .{ .flat = &.{atom} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "field", field), + }, + .location = location, + }); +} + +/// `cute_nvgpu.atom.make_exec_tma`. Result types are explicit; attributes use mlir.Attribute. +pub fn atom_make_exec_tma( + ctx: *mlir.Context, + input: *const mlir.Value, + result_type: *const mlir.Type, + opts: struct { + override: ?*const mlir.Attribute = null, + no_fully_oob_tile: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.override) |value| attributes.appendAssumeCapacity(.named(ctx, "override", value)); + if (opts.no_fully_oob_tile) |value| attributes.appendAssumeCapacity(.named(ctx, "no_fully_oob_tile", value)); + return mlir.Operation.make(ctx, "cute_nvgpu.atom.make_exec_tma", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute_nvgpu.atom.make_non_exec_2d_gather4_tma_load`. Result types are explicit; attributes use mlir.Attribute. +pub fn atom_make_non_exec_2d_gather4_tma_load( + ctx: *mlir.Context, + gmem_tensor: *const mlir.Value, + gmem_coord_layout: *const mlir.Value, + smem_layout: *const mlir.Value, + cta_v_map: *const mlir.Value, + non_exec_atom_type: *const mlir.Type, + tma_tensor_type: *const mlir.Type, + kind: *const mlir.Attribute, + opts: struct { + num_multicast: ?*const mlir.Attribute = null, + tma_format: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + if (opts.num_multicast) |value| attributes.appendAssumeCapacity(.named(ctx, "num_multicast", value)); + if (opts.tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); + return mlir.Operation.make(ctx, "cute_nvgpu.atom.make_non_exec_2d_gather4_tma_load", .{ + .operands = .{ .flat = &.{ gmem_tensor, gmem_coord_layout, smem_layout, cta_v_map } }, + .results = .{ .flat = &.{ non_exec_atom_type, tma_tensor_type } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute_nvgpu.atom.make_non_exec_2d_scatter4_tma_store`. Result types are explicit; attributes use mlir.Attribute. +pub fn atom_make_non_exec_2d_scatter4_tma_store( + ctx: *mlir.Context, + gmem_tensor: *const mlir.Value, + gmem_index_layout: *const mlir.Value, + smem_layout: *const mlir.Value, + cta_v_map: *const mlir.Value, + non_exec_atom_type: *const mlir.Type, + tma_tensor_type: *const mlir.Type, + opts: struct { + tma_format: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); + return mlir.Operation.make(ctx, "cute_nvgpu.atom.make_non_exec_2d_scatter4_tma_store", .{ + .operands = .{ .flat = &.{ gmem_tensor, gmem_index_layout, smem_layout, cta_v_map } }, + .results = .{ .flat = &.{ non_exec_atom_type, tma_tensor_type } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute_nvgpu.atom.make_non_exec_im2col_tma_load`. Result types are explicit; attributes use mlir.Attribute. +pub fn atom_make_non_exec_im2col_tma_load( + ctx: *mlir.Context, + gmem_tensor: *const mlir.Value, + smem_layout: *const mlir.Value, + cta_v_map: *const mlir.Value, + lower_corner_whd: *const mlir.Value, + upper_corner_whd: *const mlir.Value, + lower_padding_whd: *const mlir.Value, + upper_padding_whd: *const mlir.Value, + stride_whd: *const mlir.Value, + lower_srt: *const mlir.Value, + stride_srt: *const mlir.Value, + non_exec_atom_type: *const mlir.Type, + tma_tensor_type: *const mlir.Type, + kind: *const mlir.Attribute, + opts: struct { + num_multicast: ?*const mlir.Attribute = null, + tma_format: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + if (opts.num_multicast) |value| attributes.appendAssumeCapacity(.named(ctx, "num_multicast", value)); + if (opts.tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); + return mlir.Operation.make(ctx, "cute_nvgpu.atom.make_non_exec_im2col_tma_load", .{ + .operands = .{ .flat = &.{ gmem_tensor, smem_layout, cta_v_map, lower_corner_whd, upper_corner_whd, lower_padding_whd, upper_padding_whd, stride_whd, lower_srt, stride_srt } }, + .results = .{ .flat = &.{ non_exec_atom_type, tma_tensor_type } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute_nvgpu.atom.make_non_exec_im2col_tma_store`. Result types are explicit; attributes use mlir.Attribute. +pub fn atom_make_non_exec_im2col_tma_store( + ctx: *mlir.Context, + gmem_tensor: *const mlir.Value, + smem_layout: *const mlir.Value, + cta_v_map: *const mlir.Value, + non_exec_atom_type: *const mlir.Type, + tma_tensor_type: *const mlir.Type, + opts: struct { + tma_format: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); + return mlir.Operation.make(ctx, "cute_nvgpu.atom.make_non_exec_im2col_tma_store", .{ + .operands = .{ .flat = &.{ gmem_tensor, smem_layout, cta_v_map } }, + .results = .{ .flat = &.{ non_exec_atom_type, tma_tensor_type } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute_nvgpu.atom.make_non_exec_tiled_tma_load`. Result types are explicit; attributes use mlir.Attribute. +pub fn atom_make_non_exec_tiled_tma_load( + ctx: *mlir.Context, + gmem_tensor: *const mlir.Value, + smem_layout: *const mlir.Value, + cta_v_map: *const mlir.Value, + non_exec_atom_type: *const mlir.Type, + tma_tensor_type: *const mlir.Type, + kind: *const mlir.Attribute, + opts: struct { + num_multicast: ?*const mlir.Attribute = null, + tma_format: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + if (opts.num_multicast) |value| attributes.appendAssumeCapacity(.named(ctx, "num_multicast", value)); + if (opts.tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); + return mlir.Operation.make(ctx, "cute_nvgpu.atom.make_non_exec_tiled_tma_load", .{ + .operands = .{ .flat = &.{ gmem_tensor, smem_layout, cta_v_map } }, + .results = .{ .flat = &.{ non_exec_atom_type, tma_tensor_type } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute_nvgpu.atom.make_non_exec_tiled_tma_reduce`. Result types are explicit; attributes use mlir.Attribute. +pub fn atom_make_non_exec_tiled_tma_reduce( + ctx: *mlir.Context, + gmem_tensor: *const mlir.Value, + smem_layout: *const mlir.Value, + cta_v_map: *const mlir.Value, + non_exec_atom_type: *const mlir.Type, + tma_tensor_type: *const mlir.Type, + kind: *const mlir.Attribute, + opts: struct { + tma_format: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + if (opts.tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); + return mlir.Operation.make(ctx, "cute_nvgpu.atom.make_non_exec_tiled_tma_reduce", .{ + .operands = .{ .flat = &.{ gmem_tensor, smem_layout, cta_v_map } }, + .results = .{ .flat = &.{ non_exec_atom_type, tma_tensor_type } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute_nvgpu.atom.make_non_exec_tiled_tma_store`. Result types are explicit; attributes use mlir.Attribute. +pub fn atom_make_non_exec_tiled_tma_store( + ctx: *mlir.Context, + gmem_tensor: *const mlir.Value, + smem_layout: *const mlir.Value, + cta_v_map: *const mlir.Value, + non_exec_atom_type: *const mlir.Type, + tma_tensor_type: *const mlir.Type, + opts: struct { + tma_format: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); + return mlir.Operation.make(ctx, "cute_nvgpu.atom.make_non_exec_tiled_tma_store", .{ + .operands = .{ .flat = &.{ gmem_tensor, smem_layout, cta_v_map } }, + .results = .{ .flat = &.{ non_exec_atom_type, tma_tensor_type } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute_nvgpu.atom.make_s2t_copy`. Result types are explicit; attributes use mlir.Attribute. +pub fn atom_make_s2t_copy(ctx: *mlir.Context, copy_s2t_atom: *const mlir.Value, tmem_memref: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.atom.make_s2t_copy", .{ + .operands = .{ .flat = &.{ copy_s2t_atom, tmem_memref } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute_nvgpu.atom.make_tma_residue_tensor`. Result types are explicit; attributes use mlir.Attribute. +pub fn atom_make_tma_residue_tensor(ctx: *mlir.Context, tma_atom: *const mlir.Value, gmem_tensor: *const mlir.Value, residue_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.atom.make_tma_residue_tensor", .{ + .operands = .{ .flat = &.{ tma_atom, gmem_tensor } }, + .results = .{ .flat = &.{residue_type} }, + .location = location, + }); +} + +/// `cute_nvgpu.atom.make_tmem_copy`. Result types are explicit; attributes use mlir.Attribute. +pub fn atom_make_tmem_copy(ctx: *mlir.Context, atom: *const mlir.Value, tmem_memref: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.atom.make_tmem_copy", .{ + .operands = .{ .flat = &.{ atom, tmem_memref } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute_nvgpu.atom.set_value`. Result types are explicit; attributes use mlir.Attribute. +pub fn atom_set_value(ctx: *mlir.Context, atom: *const mlir.Value, value_: *const mlir.Value, result_type: *const mlir.Type, field: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.atom.set_value", .{ + .operands = .{ .flat = &.{ atom, value_ } }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "field", field), + }, + .location = location, + }); +} + +/// `cute_nvgpu.atom.sm107.get_copy_s2t_smem_desc_view`. Result types are explicit; attributes use mlir.Attribute. +pub fn atom_sm107_get_copy_s2t_smem_desc_view(ctx: *mlir.Context, atom: *const mlir.Value, view: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.atom.sm107.get_copy_s2t_smem_desc_view", .{ + .operands = .{ .flat = &.{ atom, view } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute_nvgpu.atom.tma_partition`. Result types are explicit; attributes use mlir.Attribute. +pub fn atom_tma_partition(ctx: *mlir.Context, tma_atom: *const mlir.Value, cta_coord: *const mlir.Value, cta_layout: *const mlir.Value, smem_tensor: *const mlir.Value, target_tensors: []const *const mlir.Value, res_smem_tensor_type: *const mlir.Type, res_target_tensors_types: []const *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.atom.tma_partition", .{ + .operands = .{ .variadic = &.{ + &.{tma_atom}, + &.{cta_coord}, + &.{cta_layout}, + &.{smem_tensor}, + target_tensors, + } }, + .results = .{ .variadic = &.{ + &.{res_smem_tensor_type}, + res_target_tensors_types, + } }, + .location = location, + }); +} + +/// `cute_nvgpu.cast_tma_desc_to_integer`. Result types are explicit; attributes use mlir.Attribute. +pub fn cast_tma_desc_to_integer(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.cast_tma_desc_to_integer", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .location = location, + }); +} + +/// `cute_nvgpu.copy_tma_desc`. Result types are explicit; attributes use mlir.Attribute. +pub fn copy_tma_desc(ctx: *mlir.Context, atom: *const mlir.Value, tma_desc_ptr: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.copy_tma_desc", .{ + .operands = .{ .flat = &.{ atom, tma_desc_ptr } }, + .location = location, + }); +} + +/// `cute_nvgpu.get_grid_constant_pointer`. Result types are explicit; attributes use mlir.Attribute. +pub fn get_grid_constant_pointer(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.get_grid_constant_pointer", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `cute_nvgpu.get_tma_desc_addr`. Result types are explicit; attributes use mlir.Attribute. +pub fn get_tma_desc_addr(ctx: *mlir.Context, src: *const mlir.Value, ptr_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.get_tma_desc_addr", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{ptr_type} }, + .location = location, + }); +} + +/// `cute_nvgpu.make_gmma_smem_desc`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_gmma_smem_desc(ctx: *mlir.Context, src: *const mlir.Value, res_type: *const mlir.Type, layout: *const mlir.Attribute, major: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.make_gmma_smem_desc", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{res_type} }, + .attributes = &.{ + .named(ctx, "layout", layout), + .named(ctx, "major", major), + }, + .location = location, + }); +} + +/// `cute_nvgpu.make_tma_desc_im2col`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_tma_desc_im2col( + ctx: *mlir.Context, + gmem_view: *const mlir.Value, + smem_layout: *const mlir.Value, + cta_value_tile: *const mlir.Value, + lower_corner_whd: *const mlir.Value, + upper_corner_whd: *const mlir.Value, + lower_padding_whd: *const mlir.Value, + upper_padding_whd: *const mlir.Value, + stride_whd: *const mlir.Value, + lower_srt: *const mlir.Value, + stride_srt: *const mlir.Value, + result_type: *const mlir.Type, + opts: struct { + num_multicast: ?*const mlir.Attribute = null, + tma_format: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.num_multicast) |value| attributes.appendAssumeCapacity(.named(ctx, "num_multicast", value)); + if (opts.tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); + return mlir.Operation.make(ctx, "cute_nvgpu.make_tma_desc_im2col", .{ + .operands = .{ .flat = &.{ gmem_view, smem_layout, cta_value_tile, lower_corner_whd, upper_corner_whd, lower_padding_whd, upper_padding_whd, stride_whd, lower_srt, stride_srt } }, + .results = .{ .flat = &.{result_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute_nvgpu.make_tma_desc_im2col_at`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_tma_desc_im2col_at( + ctx: *mlir.Context, + gmem_view: *const mlir.Value, + smem_layout: *const mlir.Value, + cta_value_tile: *const mlir.Value, + lower_corner_whd: *const mlir.Value, + upper_corner_whd: *const mlir.Value, + lower_padding_whd: *const mlir.Value, + upper_padding_whd: *const mlir.Value, + stride_whd: *const mlir.Value, + lower_srt: *const mlir.Value, + stride_srt: *const mlir.Value, + tma_desc_addr: *const mlir.Value, + opts: struct { + num_multicast: ?*const mlir.Attribute = null, + tma_format: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.num_multicast) |value| attributes.appendAssumeCapacity(.named(ctx, "num_multicast", value)); + if (opts.tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); + return mlir.Operation.make(ctx, "cute_nvgpu.make_tma_desc_im2col_at", .{ + .operands = .{ .flat = &.{ gmem_view, smem_layout, cta_value_tile, lower_corner_whd, upper_corner_whd, lower_padding_whd, upper_padding_whd, stride_whd, lower_srt, stride_srt, tma_desc_addr } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute_nvgpu.make_tma_desc_tiled`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_tma_desc_tiled( + ctx: *mlir.Context, + gmem_view: *const mlir.Value, + smem_layout: *const mlir.Value, + cta_value_tile: *const mlir.Value, + result_type: *const mlir.Type, + mode: *const mlir.Attribute, + opts: struct { + traversal_stride: ?*const mlir.Value = null, + num_multicast: ?*const mlir.Attribute = null, + tma_format: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "mode", mode)); + if (opts.num_multicast) |value| attributes.appendAssumeCapacity(.named(ctx, "num_multicast", value)); + if (opts.tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); + return mlir.Operation.make(ctx, "cute_nvgpu.make_tma_desc_tiled", .{ + .operands = .{ .variadic = &.{ + &.{gmem_view}, + &.{smem_layout}, + &.{cta_value_tile}, + if (opts.traversal_stride) |value| &.{value} else &.{}, + } }, + .results = .{ .flat = &.{result_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute_nvgpu.make_tma_desc_tiled_at`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_tma_desc_tiled_at( + ctx: *mlir.Context, + gmem_view: *const mlir.Value, + smem_layout: *const mlir.Value, + cta_value_tile: *const mlir.Value, + tma_desc_addr: *const mlir.Value, + mode: *const mlir.Attribute, + opts: struct { + traversal_stride: ?*const mlir.Value = null, + num_multicast: ?*const mlir.Attribute = null, + tma_format: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "mode", mode)); + if (opts.num_multicast) |value| attributes.appendAssumeCapacity(.named(ctx, "num_multicast", value)); + if (opts.tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); + return mlir.Operation.make(ctx, "cute_nvgpu.make_tma_desc_tiled_at", .{ + .operands = .{ .variadic = &.{ + &.{gmem_view}, + &.{smem_layout}, + &.{cta_value_tile}, + &.{tma_desc_addr}, + if (opts.traversal_stride) |value| &.{value} else &.{}, + } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `cute_nvgpu.make_umma_smem_desc`. Result types are explicit; attributes use mlir.Attribute. +pub fn make_umma_smem_desc( + ctx: *mlir.Context, + src: *const mlir.Value, + res_type: *const mlir.Type, + layout: *const mlir.Attribute, + major: *const mlir.Attribute, + opts: struct { + nextSrc: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.make_umma_smem_desc", .{ + .operands = .{ .variadic = &.{ + &.{src}, + if (opts.nextSrc) |value| &.{value} else &.{}, + } }, + .results = .{ .flat = &.{res_type} }, + .attributes = &.{ + .named(ctx, "layout", layout), + .named(ctx, "major", major), + }, + .location = location, + }); +} + +/// `cute_nvgpu.prefetch_tma_desc`. Result types are explicit; attributes use mlir.Attribute. +pub fn prefetch_tma_desc(ctx: *mlir.Context, atom: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.prefetch_tma_desc", .{ + .operands = .{ .flat = &.{atom} }, + .location = location, + }); +} + +/// `cute_nvgpu.sm103.make_umma_smem_desc_circular`. Result types are explicit; attributes use mlir.Attribute. +pub fn sm103_make_umma_smem_desc_circular( + ctx: *mlir.Context, + src: *const mlir.Value, + res_type: *const mlir.Type, + layout: *const mlir.Attribute, + major: *const mlir.Attribute, + opts: struct { + nextSrc: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.sm103.make_umma_smem_desc_circular", .{ + .operands = .{ .variadic = &.{ + &.{src}, + if (opts.nextSrc) |value| &.{value} else &.{}, + } }, + .results = .{ .flat = &.{res_type} }, + .attributes = &.{ + .named(ctx, "layout", layout), + .named(ctx, "major", major), + }, + .location = location, + }); +} + +/// `cute_nvgpu.sm107.make_umma_smem_desc`. Result types are explicit; attributes use mlir.Attribute. +pub fn sm107_make_umma_smem_desc( + ctx: *mlir.Context, + src: *const mlir.Value, + res_type: *const mlir.Type, + layout: *const mlir.Attribute, + major: *const mlir.Attribute, + opts: struct { + nextSrc: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.sm107.make_umma_smem_desc", .{ + .operands = .{ .variadic = &.{ + &.{src}, + if (opts.nextSrc) |value| &.{value} else &.{}, + } }, + .results = .{ .flat = &.{res_type} }, + .attributes = &.{ + .named(ctx, "layout", layout), + .named(ctx, "major", major), + }, + .location = location, + }); +} + +/// `cute_nvgpu.sm107.make_umma_smem_desc_circular`. Result types are explicit; attributes use mlir.Attribute. +pub fn sm107_make_umma_smem_desc_circular( + ctx: *mlir.Context, + src: *const mlir.Value, + res_type: *const mlir.Type, + layout: *const mlir.Attribute, + major: *const mlir.Attribute, + opts: struct { + nextSrc: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.sm107.make_umma_smem_desc_circular", .{ + .operands = .{ .variadic = &.{ + &.{src}, + if (opts.nextSrc) |value| &.{value} else &.{}, + } }, + .results = .{ .flat = &.{res_type} }, + .attributes = &.{ + .named(ctx, "layout", layout), + .named(ctx, "major", major), + }, + .location = location, + }); +} + +/// `cute_nvgpu.update_tma_desc`. Result types are explicit; attributes use mlir.Attribute. +pub fn update_tma_desc(ctx: *mlir.Context, tma_atom: *const mlir.Value, gmem_tensor: *const mlir.Value, tma_desc_ptr: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "cute_nvgpu.update_tma_desc", .{ + .operands = .{ .flat = &.{ tma_atom, gmem_tensor, tma_desc_ptr } }, + .location = location, + }); +} diff --git a/mlir/dialects/cute_nvgpu/test.zig b/mlir/dialects/cute_nvgpu/test.zig new file mode 100644 index 00000000..3c7de6ce --- /dev/null +++ b/mlir/dialects/cute_nvgpu/test.zig @@ -0,0 +1,219 @@ +const std = @import("std"); + +const cute_ir = @import("mlir/dialects/cute_ir"); +const cute_nvgpu = @import("mlir/dialects/cute_nvgpu"); +const mlir = @import("mlir"); + +const operation_names: []const []const u8 = &.{ + "cute_nvgpu.arch.alloc_rmem", + "cute_nvgpu.arch.alloc_smem", + "cute_nvgpu.arch.copy.SM100.copy_s2t", + "cute_nvgpu.arch.copy.SM100.tma_load", + "cute_nvgpu.arch.copy.SM100.tma_reduce", + "cute_nvgpu.arch.copy.SM100.tma_store", + "cute_nvgpu.arch.copy.SM100.tmem_load", + "cute_nvgpu.arch.copy.SM100.tmem_store", + "cute_nvgpu.arch.copy.SM107.tma_load_override", + "cute_nvgpu.arch.copy.SM107.tma_store_override", + "cute_nvgpu.arch.copy.SM90.bulk_copy_g2s", + "cute_nvgpu.arch.copy.SM90.bulk_copy_s2g", + "cute_nvgpu.arch.copy.SM90.bulk_copy_s2s", + "cute_nvgpu.arch.get_dyn_smem", + "cute_nvgpu.arch.get_dyn_smem_size", + "cute_nvgpu.arch.make_warp_uniform", + "cute_nvgpu.arch.mma.SM107.umma", + "cute_nvgpu.arch.mma.SM107.umma_block_scaled", + "cute_nvgpu.arch.mma.SM107.umma_block_scaled_sparse", + "cute_nvgpu.arch.mma.SM107.umma_sparse", + "cute_nvgpu.arch.mma.SM120.block_scaled", + "cute_nvgpu.arch.prefetch_tma_desc", + "cute_nvgpu.arch.sm100.alloc_tmem", + "cute_nvgpu.arch.sm100.dealloc_tmem", + "cute_nvgpu.arch.sm100.relinquish_tmem_alloc_permit", + "cute_nvgpu.arch.sm100.retrieve_tmem_ptr", + "cute_nvgpu.atom.get_coord_tensor", + "cute_nvgpu.atom.get_copy_s2t_smem_desc_view", + "cute_nvgpu.atom.get_value", + "cute_nvgpu.atom.make_exec_tma", + "cute_nvgpu.atom.make_non_exec_2d_gather4_tma_load", + "cute_nvgpu.atom.make_non_exec_2d_scatter4_tma_store", + "cute_nvgpu.atom.make_non_exec_im2col_tma_load", + "cute_nvgpu.atom.make_non_exec_im2col_tma_store", + "cute_nvgpu.atom.make_non_exec_tiled_tma_load", + "cute_nvgpu.atom.make_non_exec_tiled_tma_reduce", + "cute_nvgpu.atom.make_non_exec_tiled_tma_store", + "cute_nvgpu.atom.make_s2t_copy", + "cute_nvgpu.atom.make_tma_residue_tensor", + "cute_nvgpu.atom.make_tmem_copy", + "cute_nvgpu.atom.set_value", + "cute_nvgpu.atom.sm107.get_copy_s2t_smem_desc_view", + "cute_nvgpu.atom.tma_partition", + "cute_nvgpu.cast_tma_desc_to_integer", + "cute_nvgpu.copy_tma_desc", + "cute_nvgpu.get_grid_constant_pointer", + "cute_nvgpu.get_tma_desc_addr", + "cute_nvgpu.make_gmma_smem_desc", + "cute_nvgpu.make_tma_desc_im2col", + "cute_nvgpu.make_tma_desc_im2col_at", + "cute_nvgpu.make_tma_desc_tiled", + "cute_nvgpu.make_tma_desc_tiled_at", + "cute_nvgpu.make_umma_smem_desc", + "cute_nvgpu.prefetch_tma_desc", + "cute_nvgpu.sm103.make_umma_smem_desc_circular", + "cute_nvgpu.sm107.make_umma_smem_desc", + "cute_nvgpu.sm107.make_umma_smem_desc_circular", + "cute_nvgpu.update_tma_desc", +}; + +fn testContext() !*mlir.Context { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + cute_ir.registerDialect(registry); + cute_nvgpu.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + return ctx; +} + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(cute_nvgpu); + inline for (comptime std.meta.declarations(cute_nvgpu)) |decl| { + const T = @field(cute_nvgpu, decl); + if (@TypeOf(T) == type) std.testing.refAllDecls(T); + } +} + +test "every bound operation is registered" { + const ctx = try testContext(); + defer ctx.deinit(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} + +test "grid_constant is a unit attribute of the cute_nvgpu dialect" { + const ctx = try testContext(); + defer ctx.deinit(); + var buf: [64]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print("{f}", .{mlir.Attribute.dict(ctx, &.{cute_nvgpu.gridConstant(ctx)})}); + try std.testing.expectEqualStrings("{cute_nvgpu.grid_constant}", w.buffered()); +} + +test "generated types and attributes rebuild from their getters" { + @setEvalBranchQuota(100_000); + const ctx = try testContext(); + defer ctx.deinit(); + inline for (.{ + .{ cute_nvgpu.CopyAtomBulkCopyG2SType, "!cute_nvgpu.atom.bulk_copy_g2s" }, + .{ cute_nvgpu.CopyAtomBulkCopyS2GType, "!cute_nvgpu.atom.bulk_copy_s2g" }, + .{ cute_nvgpu.CopyAtomBulkCopyS2SType, "!cute_nvgpu.atom.bulk_copy_s2s" }, + .{ cute_nvgpu.CopyAtomDsmemStoreType, "!cute_nvgpu.atom.dsmem_store" }, + .{ cute_nvgpu.CopyAtomG2RType, "!cute_nvgpu.atom.g2r" }, + .{ cute_nvgpu.CopyAtomIm2ColTmaLoadType, "!cute_nvgpu.atom.im2col_tma_load mcast = true tma_gbasis = <\"(64,128):(1@1,1@0)\">>" }, + .{ cute_nvgpu.CopyAtomIm2ColTmaStoreType, "!cute_nvgpu.atom.im2col_tma_store tma_gbasis = <\"(32,128):(1@1,1@0)\">>" }, + .{ cute_nvgpu.CopyAtomLdsmType, "!cute_nvgpu.atom.ldsm, sz_pattern = u16, num_matrices = 4, t>" }, + .{ cute_nvgpu.CopyAtomNonExec2DGather4TmaLoadType, "!cute_nvgpu.atom.non_exec_2d_gather4_tma_load, tma_format = F16_RN>" }, + .{ cute_nvgpu.CopyAtomNonExec2DScatter4TmaStoreType, "!cute_nvgpu.atom.non_exec_2d_scatter4_tma_store, tma_format = F16_RN>" }, + .{ cute_nvgpu.CopyAtomNonExecIm2ColTmaLoadType, "!cute_nvgpu.atom.non_exec_im2col_tma_load, tma_format = F16_RN>" }, + .{ cute_nvgpu.CopyAtomNonExecIm2ColTmaStoreType, "!cute_nvgpu.atom.non_exec_im2col_tma_store, tma_format = F16_RN>" }, + .{ cute_nvgpu.CopyAtomNonExecTiledTmaLoadType, "!cute_nvgpu.atom.non_exec_tiled_tma_load, tma_format = BF16_RN>" }, + .{ cute_nvgpu.CopyAtomNonExecTiledTmaReduceType, "!cute_nvgpu.atom.non_exec_tiled_tma_reduce, tma_format = F32_RN>" }, + .{ cute_nvgpu.CopyAtomNonExecTiledTmaStoreType, "!cute_nvgpu.atom.non_exec_tiled_tma_store, tma_format = BF16_RN>" }, + .{ cute_nvgpu.CopyAtomR2GType, "!cute_nvgpu.atom.r2g" }, + .{ cute_nvgpu.CopyAtomR2SType, "!cute_nvgpu.atom.r2s" }, + .{ cute_nvgpu.CopyAtomS2RType, "!cute_nvgpu.atom.s2r" }, + .{ cute_nvgpu.CopyAtomSIMTAsyncCopyType, "!cute_nvgpu.atom.simt_async_copy" }, + .{ cute_nvgpu.CopyAtomSIMTMultimemLdReduceType, "!cute_nvgpu.atom.simt_multimem_ld_reduce, acquire, gpu>" }, + .{ cute_nvgpu.CopyAtomSIMTMultimemRedType, "!cute_nvgpu.atom.simt_multimem_red" }, + .{ cute_nvgpu.CopyAtomSIMTMultimemStType, "!cute_nvgpu.atom.simt_multimem_st" }, + .{ cute_nvgpu.CopyAtomSIMTSyncCopyType, "!cute_nvgpu.atom.universal_copy" }, + .{ cute_nvgpu.CopyAtomSM100CopyS2TType, "!cute_nvgpu.atom.s2t_copy" }, + .{ cute_nvgpu.CopyAtomSM100S2TCopyV2Type, "!cute_nvgpu.atom.sm100_s2t_copy_v2" }, + .{ cute_nvgpu.CopyAtomSM100TmemLoadType, "!cute_nvgpu.atom.tmem_load" }, + .{ cute_nvgpu.CopyAtomSM100TmemStoreType, "!cute_nvgpu.atom.tmem_store" }, + .{ cute_nvgpu.CopyAtomSM107TmemLoadSPCompressType, "!cute_nvgpu.atom.tmem_load_spcompress" }, + .{ cute_nvgpu.CopyAtomSM10xTmemLoadRedType, "!cute_nvgpu.atom.tmem_load_red" }, + .{ cute_nvgpu.CopyAtomStsmType, "!cute_nvgpu.atom.stsm, num_matrices = 4, n>" }, + .{ cute_nvgpu.CopyAtomTmaLoadType, "!cute_nvgpu.atom.tma_load mcast = true tma_gbasis = <\"(64,1):(1@0,1@1)\">>" }, + .{ cute_nvgpu.CopyAtomTmaReduceType, "!cute_nvgpu.atom.tma_reduce tma_gbasis = <\"(32,128):(1@1,1@0)\">>" }, + .{ cute_nvgpu.CopyAtomTmaStoreType, "!cute_nvgpu.atom.tma_store tma_gbasis = <\"(64,16):(1@0,1@1)\">>" }, + .{ cute_nvgpu.MmaAtomSM100UMMABlockScaledSparseType, "!cute_nvgpu.sm100.mma_bs_sp<128x128x64, num_cta = 1, ab_major = (k, k), elem_type = (f8E4M3FN, f8E4M3FN, f32), sf_type = f8E8M0FNU, sparse_metadata_format = tid, frag_kind = ss, vec_size = 32>" }, + .{ cute_nvgpu.MmaAtomSM100UMMABlockScaledType, "!cute_nvgpu.sm100.mma_bs<128x128x96, num_cta = 1, ab_major = (k, k), elem_type = (f4E2M1FN, f4E2M1FN, f32), sf_type = f8E8M0FNU, frag_kind = ss, vec_size = 16, arch_promote = sm_103>" }, + .{ cute_nvgpu.MmaAtomSM100UMMASparseType, "!cute_nvgpu.sm100.mma_sp<128x128x32, num_cta = 1, ab_major = (k, k), elem_type = (f16, f16, f32), e_type = i8, sparse_metadata_format = tid, frag_kind = ss, c_scale_exp = 0>" }, + .{ cute_nvgpu.MmaAtomSM100UMMAType, "!cute_nvgpu.sm100.mma<128x128x16, num_cta = 1, ab_major = (k, mn), elem_type = (f16, f16, f32), frag_kind = ss, c_scale_exp = 0>" }, + .{ cute_nvgpu.MmaAtomSM107UMMABlockScaledSparseType, "!cute_nvgpu.sm107.mma_bs_sp<128x128x64, num_cta = 1, ab_major = (k, k), elem_type = (f8E4M3FN, f8E4M3FN, f32), sf_type = f8E8M0FNU, sparse_metadata_format = tid, frag_kind = ss, vec_size = 32, ab_collector_op = (discard, discard)>" }, + .{ cute_nvgpu.MmaAtomSM107UMMABlockScaledType, "!cute_nvgpu.sm107.mma_bs<128x128x32, num_cta = 1, ab_major = (k, k), elem_type = (f8E4M3FN, f8E4M3FN, f32), sf_type = f8E8M0FNU, frag_kind = ss, vec_size = 32, ab_collector_op = (discard, use), int_overflow = >" }, + .{ cute_nvgpu.MmaAtomSM107UMMASparseType, "!cute_nvgpu.sm107.mma_sp<128x128x32, num_cta = 1, ab_major = (k, k), elem_type = (f16, f16, f32), e_type = i8, sparse_metadata_format = tid, frag_kind = ss, c_scale_exp = 0, ab_collector_op = (discard, discard)>" }, + .{ cute_nvgpu.MmaAtomSM107UMMAType, "!cute_nvgpu.sm107.mma<128x128x16, num_cta = 1, ab_major = (k, mn), elem_type = (f16, f16, f32), frag_kind = ss, c_scale_exp = 0, ab_collector_op = (lastuse, fill)>" }, + .{ cute_nvgpu.MmaAtomSM120BlockScaledType, "!cute_nvgpu.SM120.mma_bs<16x8x64, vec_size = 32, elem_type = (f4E2M1FN, f4E2M1FN, f32), sf_type = f8E8M0FNU, use_sf_layout_TV = true>" }, + .{ cute_nvgpu.MmaAtomSM80SparseType, "!cute_nvgpu.sm80.sparse_mma<16x8x64, elem_type = (i8, i8, i32), md_format = tid, int_overflow = >" }, + .{ cute_nvgpu.MmaAtomSM80Type, "!cute_nvgpu.sm80.mma<16x8x32, elem_type = (i8, i8, i32), int_overflow = , binary_op = >" }, + .{ cute_nvgpu.MmaAtomSM89Type, "!cute_nvgpu.sm89.mma<16x8x32, elem_type = (f8E4M3FN, f8E4M3FN, f32) >" }, + .{ cute_nvgpu.MmaAtomSM90Type, "!cute_nvgpu.sm90.mma<64x128x32, ab_major = (k, k), elem_type = (i8, i8, i32), frag_kind = ss, int_overflow = , a_neg, b_neg>" }, + .{ cute_nvgpu.SmemDescCircularSM103Type, "!cute_nvgpu.sm103.smem_desc_circular<<\"(8,128,2):(128,1,1024)\">>" }, + .{ cute_nvgpu.SmemDescCircularSM107Type, "!cute_nvgpu.sm107.smem_desc_circular<<\"(8,128,2):(128,1,1024)\">>" }, + .{ cute_nvgpu.SmemDescSM107Type, "!cute_nvgpu.sm107.smem_desc" }, + .{ cute_nvgpu.SmemDescType, "!cute_nvgpu.smem_desc" }, + .{ cute_nvgpu.SmemDescViewType, "!cute_nvgpu.smem_desc_view" }, + .{ cute_nvgpu.TmaDescriptorIm2ColType, "!cute_nvgpu.tma_descriptor_im2col" }, + .{ cute_nvgpu.TmaDescriptorTiledType, "!cute_nvgpu.tma_descriptor_tiled" }, + .{ cute_nvgpu.UniversalFmaAtomType, "!cute_nvgpu.atom.universal_fma<1x1x1, (f32, f32) -> f32 >" }, + .{ cute_nvgpu.WorkIdResponseType, "!cute_nvgpu.workid_response" }, + .{ cute_nvgpu.NotImplementedFrgAttr, "#cute_nvgpu.not_implemented_frg<>" }, + .{ cute_nvgpu.RmemFrgAttr, "#cute_nvgpu.rmem_frg" }, + .{ cute_nvgpu.SM100CircularSmemFrgAttr, "#cute_nvgpu.arch.sm100.circular_smem_frg" }, + .{ cute_nvgpu.SM100SmemFrgAttr, "#cute_nvgpu.arch.sm100.smem_frg" }, + .{ cute_nvgpu.SM100TmemEFrgAttr, "#cute_nvgpu.arch.sm100.tmem_e_frg" }, + .{ cute_nvgpu.SM100TmemFrgAttr, "#cute_nvgpu.arch.sm100.tmem_frg" }, + .{ cute_nvgpu.SM100TmemSfFrgAttr, "#cute_nvgpu.arch.sm100.tmem_sf_frg" }, + .{ cute_nvgpu.SM107SmemFrgAttr, "#cute_nvgpu.arch.sm107.smem_frg" }, + .{ cute_nvgpu.SM90SmemFrgAttr, "#cute_nvgpu.arch.sm90.smem_frg" }, + }) |example| { + const T = example[0]; + const is_type = @hasDecl(T, "type_"); + const parsed = if (is_type) try mlir.Type.parse(ctx, example[1]) else try mlir.Attribute.parse(ctx, example[1]); + const value = parsed.isA(T) orelse return error.TestUnexpectedResult; + try std.testing.expect((try rebuild(T, ctx, value)).eql(value)); + var text: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer text.deinit(); + try text.writer.print("{f}", .{value}); + try std.testing.expect(parsed.eql(if (is_type) try mlir.Type.parse(ctx, text.written()) else try mlir.Attribute.parse(ctx, text.written()))); + } + // Every case of every enum attribute. + inline for (.{cute_nvgpu}) |namespace| { + inline for (comptime std.meta.declarations(namespace)) |decl| { + const T = @field(namespace, decl); + if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque" and @hasDecl(T, "InitArgs")) { + const fields = @typeInfo(T.InitArgs).@"struct".field_names; + if (fields.len == 1 and comptime std.mem.eql(u8, fields[0], "value")) { + inline for (@typeInfo(@FieldType(T.InitArgs, fields[0])).@"enum".field_names) |case| { + const case_value = @field(@FieldType(T.InitArgs, fields[0]), case); + const attr = try T.get(ctx, .{ .value = case_value }); + try std.testing.expectEqual(@backingInt(case_value), @backingInt(attr.value())); + try std.testing.expect((try rebuild(T, ctx, attr)).eql(attr)); + } + } + } + } + } + try std.testing.expect((try cute_nvgpu.SmemDescType.get(ctx, .{})).type_().isA(cute_nvgpu.NVGPUType) != null); +} + +fn rebuild(comptime T: type, ctx: *mlir.Context, value: *const T) !*const T { + var args: T.InitArgs = undefined; + var elements: [8]*const mlir.Type = undefined; + inline for (@typeInfo(T.InitArgs).@"struct".field_names) |field| { + const name = [_]u8{comptime std.ascii.toUpper(field[0])} ++ field[1..]; + const noun = [_]u8{comptime std.ascii.toLower(field[0])} ++ field[1..]; + const accessor = if (@hasDecl(T, noun ++ "_")) noun ++ "_" else noun; + if (@FieldType(T.InitArgs, field) == []const *const mlir.Type) { + const count = @field(T, "num" ++ name)(value); + for (elements[0..count], 0..) |*element, i| element.* = @field(T, noun[0 .. noun.len - 1])(value, i); + @field(args, field) = elements[0..count]; + } else { + @field(args, field) = @field(T, accessor)(value); + } + } + return T.get(ctx, args); +} diff --git a/mlir/dialects/dialects.zig b/mlir/dialects/dialects.zig deleted file mode 100644 index 1da34b7c..00000000 --- a/mlir/dialects/dialects.zig +++ /dev/null @@ -1,11 +0,0 @@ -const std = @import("std"); - -pub const arith = @import("arith.zig"); -pub const func = @import("func.zig"); -pub const math = @import("math.zig"); -pub const tensor = @import("tensor.zig"); -pub const stablehlo = @import("mlir/dialects/stablehlo"); - -test { - std.testing.refAllDecls(@This()); -} diff --git a/mlir/dialects/fly/BUILD.bazel b/mlir/dialects/fly/BUILD.bazel new file mode 100644 index 00000000..87fe2405 --- /dev/null +++ b/mlir/dialects/fly/BUILD.bazel @@ -0,0 +1,56 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +cc_library( + name = "capi", + srcs = [ + "capi.cc", + ], + hdrs = [ + "capi.h", + ], + tags = ["manual"], + deps = [ + "@flydsl//:FlyDialect", + "@flydsl//:TiledOpTraits", + "@llvm-project//mlir:CAPIIR", + ], +) + +zig_library( + name = "fly", + srcs = [ + "attributes.zig", + "ops.zig", + "types.zig", + ], + import_name = "mlir/dialects/fly", + main = "fly.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + ":capi", + "//mlir", + "//stdx", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":fly", + "//mlir", + "//mlir/dialects/arith", + "//mlir/dialects/fly_rocdl", + "//mlir/dialects/func", + "//mlir/dialects/gpu", + "//mlir/dialects/llvm", + "//mlir/dialects/math", + "//mlir/dialects/memref", + "//mlir/dialects/rocdl", + "//mlir/dialects/scf", + "//mlir/dialects/ub", + "//mlir/dialects/vector", + ], +) diff --git a/mlir/dialects/fly/REGENERATING.md b/mlir/dialects/fly/REGENERATING.md new file mode 100644 index 00000000..9673870f --- /dev/null +++ b/mlir/dialects/fly/REGENERATING.md @@ -0,0 +1,79 @@ +# Regenerating the operation builders + +These are instructions for an agent. They describe what `ops.zig` and +`../fly_rocdl/ops.zig` must contain after a FlyDSL bump, where to derive them from, +and how to check the result. Scripts and intermediate files are not +committed; only the generated builders are. + +Paths below are relative to this directory. + +## Goal + +`ops.zig` binds every operation of the `fly` dialect and `../fly_rocdl/ops.zig` every +operation of the `fly_rocdl` dialect, one builder per operation, so that no +code builds a FlyDSL operation from its name. The attributes and types +(`attributes.zig`, `types.zig`, `../fly_rocdl/fly_rocdl.zig`, `capi.*`, `../fly_rocdl/capi.*`) are maintained by +hand and are not covered here. + +## Sources + +The FlyDSL commit is the one `third_party/flydsl/repo.bzl` pins, with its +patches applied: read the `.td` files from the fetched repository +(`$(bazel info output_base)/external/+non_module_deps+flydsl`), never from an +unpatched checkout. + +- `include/flydsl/Dialect/Fly/IR/FlyOps.td` for `ops.zig`. +- `include/flydsl/Dialect/FlyROCDL/IR/Ops.td` for `../fly_rocdl/ops.zig`. + +Dump their records with the LLVM the workspace builds: + +```sh +bazel build @llvm-project//llvm:llvm-tblgen +llvm-tblgen --dump-json \ + -I /include -I /external/+llvm+llvm-project/mlir/include \ + /include/flydsl/Dialect/Fly/IR/FlyOps.td -o FlyOps.json +``` + +Every record in `!instanceof.Op` whose `opDialect` names the dialect is an +operation. Do not regex the `.td` files. + +## What each builder must be + +Builder names, parameter order, `opts`, operand and result types, doc comments +and `operation_names` follow `mlir/dialects/README.md`. Specific to FlyDSL: + +- Builders are sorted by mnemonic. The doc comment gives the `summary` and + whether result types are inferred, explicit or absent. +- Unlike the README rule, parameter names are the ODS names in snake case. The + reserved locals `ctx`, `opts`, `location`, `make`, `operands`, `attributes` + and `present` also take a trailing underscore. +- Operations with `AttrSizedOperandSegments` pass every operand group through + `Operation.make`'s `.variadic` operands, which writes `operandSegmentSizes`; + others pass a flat list (an optional operand may only be last). +- Inferred operations set `.result_type_inference = true`. Inference comes + from the flattened trait list (trait lists such as `Pure` expand to their + members); an `OpInterface` record whose `cppInterfaceName` is + `InferTypeOpInterface` marks it. +- Attributes are `*const mlir.Attribute`, set under their ODS name. +- Builders call `make` from `fly.zig`, which verifies the operation and names + the operand types when creation fails. +- Each file starts with a header naming the FlyDSL commit and the `.td` it + was generated from. + +Regions, successors and `VariadicOfVariadic` operands have no use in either +dialect today; when one appears, extend the generator and this file. + +## Checks + +- Format and run the conformance checks of `mlir/dialects/README.md`. +- `bazel test //mlir/dialects/fly:test //dsl/fly:test`, which includes the + registration tests of the `operation_names` lists in both packages' + `test.zig`. +- Callers that no longer compile are the point of generating: an operation + whose signature changed in FlyDSL now fails at the call site. Fix them in + the same change (`dsl/fly/builder.zig`, `zml/kernel/fly/fly.zig`, `zml/kernels/*/fly/`, + and downstream llmd's `llmd/kernels/*/fly/`). +- Emit the IR of the fly kernels before and after (for instance + llmd's `sparse_mla.Main2D.emit` and the `zml/kernels/moe/fly/` kernels with + representative configs) and + diff it: a regeneration without an upstream change must be byte-identical. diff --git a/mlir/dialects/fly/attributes.zig b/mlir/dialects/fly/attributes.zig new file mode 100644 index 00000000..3fc86ad3 --- /dev/null +++ b/mlir/dialects/fly/attributes.zig @@ -0,0 +1,223 @@ +const c = @import("c"); +const mlir = @import("mlir"); + +/// `#fly.int`, the unified static/dynamic integer leaf. +pub const IntAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsAFlyInt; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + + pub fn getStatic(ctx: *mlir.Context, value: i32) mlir.Error!*const Self { + return @ptrCast(c.mlirFlyIntAttrStaticGet(ctx.ptr(), value).ptr orelse return error.InvalidMlir); + } + + pub fn getDynamic(ctx: *mlir.Context, width: i32, divisibility: i32) mlir.Error!*const Self { + return @ptrCast(c.mlirFlyIntAttrDynamicGet(ctx.ptr(), width, divisibility).ptr orelse return error.InvalidMlir); + } + + pub fn getNone(ctx: *mlir.Context) mlir.Error!*const Self { + return @ptrCast(c.mlirFlyIntAttrNoneGet(ctx.ptr()).ptr orelse return error.InvalidMlir); + } +}; + +/// `#fly.int_tuple`: a leaf, a basis element, or a nest of those. +pub const IntTupleAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsAFlyIntTuple; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + + pub const InitArgs = struct { + value: *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirFlyIntTupleAttrGet(ctx.ptr(), args.value.ptr()).ptr orelse return error.InvalidMlir); + } + + pub fn getStatic(ctx: *mlir.Context, value: i32) mlir.Error!*const Self { + return get(ctx, .{ .value = (try IntAttr.getStatic(ctx, value)).attribute() }); + } + + pub fn getDynamic(ctx: *mlir.Context, width: i32, divisibility: i32) mlir.Error!*const Self { + return get(ctx, .{ .value = (try IntAttr.getDynamic(ctx, width, divisibility)).attribute() }); + } + + pub fn getNone(ctx: *mlir.Context) mlir.Error!*const Self { + return get(ctx, .{ .value = (try IntAttr.getNone(ctx)).attribute() }); + } + + pub fn getBasis(ctx: *mlir.Context, value: *const IntAttr, modes: []const i32) mlir.Error!*const Self { + const result = c.mlirFlyIntTupleAttrBasisGet(ctx.ptr(), value.ptr(), @intCast(modes.len), modes.ptr); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + + pub fn getTuple(ctx: *mlir.Context, elements: []const *const Self) mlir.Error!*const Self { + const result = c.mlirFlyIntTupleAttrTupleGet(ctx.ptr(), @intCast(elements.len), @ptrCast(elements.ptr)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `#fly.layout`, a `shape:stride` pair. +pub const LayoutAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsAFlyLayout; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + + pub const InitArgs = struct { + shape: *const IntTupleAttr, + stride: *const IntTupleAttr, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirFlyLayoutAttrGet(ctx.ptr(), args.shape.ptr(), args.stride.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `#fly.tile`: a mode, or a `[a|b]` nest of modes. +pub const TileAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsAFlyTile; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + + pub const InitArgs = struct { + value: *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirFlyTileAttrGet(ctx.ptr(), args.value.ptr()).ptr orelse return error.InvalidMlir); + } + + pub fn getModes(ctx: *mlir.Context, modes: []const *const mlir.Attribute) mlir.Error!*const Self { + const result = c.mlirFlyTileAttrModesGet(ctx.ptr(), @intCast(modes.len), @ptrCast(modes.ptr)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `S`. +pub const SwizzleAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsAFlySwizzle; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + + pub const InitArgs = struct { + mask: i32, + base: i32, + shift: i32, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirFlySwizzleAttrGet(ctx.ptr(), args.mask, args.base, args.shift); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `align`, in bytes. +pub const AlignAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsAFlyAlign; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + + pub const InitArgs = struct { + alignment: i32, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirFlyAlignAttrGet(ctx.ptr(), args.alignment).ptr orelse return error.InvalidMlir); + } +}; + +pub const AddressSpace = enum(i32) { generic = 0, global = 1, shared = 2, register = 3 }; + +/// `global`, `shared`, ... A pointer or memref may also carry a foreign +/// address-space attribute, such as `#fly_rocdl.buffer_desc`. +pub const AddressSpaceAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsAFlyAddressSpace; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + + pub const InitArgs = struct { + addressSpace: AddressSpace, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirFlyAddressSpaceAttrGet(ctx.ptr(), @backingInt(args.addressSpace)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +pub const MmaOperand = enum(i32) { a = 0, b = 1, c = 2, d = 3 }; + +pub fn mmaOperandAttr(ctx: *mlir.Context, operand: MmaOperand) mlir.Error!*const mlir.Attribute { + const result = c.mlirFlyMmaOperandAttrGet(ctx.ptr(), @backingInt(operand)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); +} + +pub const GemmTraversalOrder = enum(i32) { + kmn = 0, + knm = 1, + mkn = 2, + mnk = 3, + nkm = 4, + nmk = 5, + kmn_serpentine = 6, + knm_serpentine = 7, + mkn_serpentine = 8, + mnk_serpentine = 9, + nkm_serpentine = 10, + nmk_serpentine = 11, +}; + +pub fn gemmTraversalOrderAttr(ctx: *mlir.Context, order: GemmTraversalOrder) mlir.Error!*const mlir.Attribute { + const result = c.mlirFlyGemmTraversalOrderAttrGet(ctx.ptr(), @backingInt(order)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); +} diff --git a/mlir/dialects/fly/capi.cc b/mlir/dialects/fly/capi.cc new file mode 100644 index 00000000..1f4913bd --- /dev/null +++ b/mlir/dialects/fly/capi.cc @@ -0,0 +1,529 @@ +//===- capi.cc - C API for the fly dialect --------------------------------===// + +#include "capi.h" + +#include "TiledOpTraits.h" +#include "flydsl/Dialect/Fly/IR/FlyDialect.h" + +#include "mlir/CAPI/IR.h" +#include "mlir/CAPI/Registration.h" + +MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(Fly, fly, mlir::fly::FlyDialect) + +using mlir::cast; +using mlir::isa; +using namespace mlir::fly; + +static mlir::MLIRContext *loadFly(MlirContext context) { + mlir::MLIRContext *ctx = unwrap(context); + if (ctx) + ctx->getOrLoadDialect(); + return ctx; +} + +/// `!attr` and `attr == other` are value arithmetic on a fly IntAttr, not a +/// null test: every handle check here goes through these two helpers. +template static bool isNull(T value) { + return !static_cast(value); +} + +template +static T checked(Handle handle, mlir::MLIRContext *ctx) { + auto value = mlir::dyn_cast_if_present(unwrap(handle)); + if (isNull(value) || value.getContext() != ctx) + return T{}; + return value; +} + +#define FLY_ATTR_ISA(Name, Attr) \ + bool mlirAttributeIsAFly##Name(MlirAttribute attr) { \ + return mlir::isa_and_nonnull(unwrap(attr)); \ + } + +FLY_ATTR_ISA(Int, IntAttr) +FLY_ATTR_ISA(IntTuple, IntTupleAttr) +FLY_ATTR_ISA(Layout, LayoutAttr) +FLY_ATTR_ISA(Tile, TileAttr) +FLY_ATTR_ISA(Swizzle, SwizzleAttr) +FLY_ATTR_ISA(Align, AlignAttr) +FLY_ATTR_ISA(AddressSpace, AddressSpaceAttr) + +#undef FLY_ATTR_ISA + +#define FLY_TYPE_ISA(Name, Type) \ + bool mlirTypeIsAFly##Name(MlirType type) { \ + return mlir::isa_and_nonnull(unwrap(type)); \ + } + +FLY_TYPE_ISA(IntTuple, IntTupleType) +FLY_TYPE_ISA(Layout, LayoutType) +FLY_TYPE_ISA(ComposedLayout, ComposedLayoutType) +FLY_TYPE_ISA(Tile, TileType) +FLY_TYPE_ISA(Swizzle, SwizzleType) +FLY_TYPE_ISA(Pointer, PointerType) +FLY_TYPE_ISA(MemRef, mlir::fly::MemRefType) +FLY_TYPE_ISA(CoordTensor, CoordTensorType) +FLY_TYPE_ISA(CopyAtom, CopyAtomType) +FLY_TYPE_ISA(MmaAtom, MmaAtomType) +FLY_TYPE_ISA(TiledCopy, TiledCopyType) +FLY_TYPE_ISA(TiledMma, TiledMmaType) +FLY_TYPE_ISA(CopyOpUniversalCopy, CopyOpUniversalCopyType) + +#undef FLY_TYPE_ISA + +MlirAttribute mlirFlyIntAttrStaticGet(MlirContext context, int32_t value) { + mlir::MLIRContext *ctx = loadFly(context); + if (!ctx) + return {nullptr}; + return wrap(static_cast(IntAttr::getStatic(ctx, value))); +} + +MlirAttribute mlirFlyIntAttrDynamicGet(MlirContext context, int32_t width, + int32_t divisibility) { + mlir::MLIRContext *ctx = loadFly(context); + if (!ctx) + return {nullptr}; + return wrap(static_cast( + IntAttr::getDynamic(ctx, width, divisibility))); +} + +MlirAttribute mlirFlyIntAttrNoneGet(MlirContext context) { + mlir::MLIRContext *ctx = loadFly(context); + if (!ctx) + return {nullptr}; + return wrap(static_cast(IntAttr::getNone(ctx))); +} + +MlirAttribute mlirFlyIntTupleAttrGet(MlirContext context, MlirAttribute value) { + mlir::MLIRContext *ctx = loadFly(context); + mlir::Attribute inner = unwrap(value); + if (!ctx || isNull(inner) || inner.getContext() != ctx) + return {nullptr}; + return wrap(static_cast(IntTupleAttr::get(ctx, inner))); +} + +MlirAttribute mlirFlyIntTupleAttrBasisGet(MlirContext context, + MlirAttribute value, intptr_t nModes, + const int32_t *modes) { + mlir::MLIRContext *ctx = loadFly(context); + if (!ctx) + return {nullptr}; + IntAttr intValue = checked(value, ctx); + if (isNull(intValue) || (nModes > 0 && !modes)) + return {nullptr}; + return wrap(static_cast( + IntTupleAttr::get(intValue, llvm::ArrayRef(modes, nModes)))); +} + +MlirAttribute mlirFlyIntTupleAttrTupleGet(MlirContext context, + intptr_t nElements, + MlirAttribute const *elements) { + mlir::MLIRContext *ctx = loadFly(context); + if (!ctx || (nElements > 0 && !elements)) + return {nullptr}; + llvm::SmallVector values; + values.reserve(nElements); + for (intptr_t i = 0; i < nElements; ++i) { + IntTupleAttr element = checked(elements[i], ctx); + if (isNull(element)) + return {nullptr}; + values.push_back(element); + } + return wrap(static_cast( + IntTupleAttr::get(ctx, mlir::ArrayAttr::get(ctx, values)))); +} + +MlirAttribute mlirFlyLayoutAttrGet(MlirContext context, MlirAttribute shape, + MlirAttribute stride) { + mlir::MLIRContext *ctx = loadFly(context); + if (!ctx) + return {nullptr}; + IntTupleAttr shapeAttr = checked(shape, ctx); + IntTupleAttr strideAttr = checked(stride, ctx); + if (isNull(shapeAttr) || isNull(strideAttr)) + return {nullptr}; + return wrap(static_cast( + LayoutAttr::get(ctx, shapeAttr, strideAttr))); +} + +MlirAttribute mlirFlyTileAttrGet(MlirContext context, MlirAttribute value) { + mlir::MLIRContext *ctx = loadFly(context); + mlir::Attribute inner = unwrap(value); + if (!ctx || isNull(inner) || inner.getContext() != ctx) + return {nullptr}; + return wrap(static_cast(TileAttr::get(ctx, inner))); +} + +MlirAttribute mlirFlyTileAttrModesGet(MlirContext context, intptr_t nModes, + MlirAttribute const *modes) { + mlir::MLIRContext *ctx = loadFly(context); + if (!ctx || (nModes > 0 && !modes)) + return {nullptr}; + llvm::SmallVector values; + values.reserve(nModes); + for (intptr_t i = 0; i < nModes; ++i) { + mlir::Attribute mode = unwrap(modes[i]); + if (isNull(mode) || mode.getContext() != ctx) + return {nullptr}; + values.push_back(mode); + } + return wrap(static_cast( + TileAttr::get(ctx, mlir::ArrayAttr::get(ctx, values)))); +} + +MlirAttribute mlirFlySwizzleAttrGet(MlirContext context, int32_t mask, + int32_t base, int32_t shift) { + mlir::MLIRContext *ctx = loadFly(context); + if (!ctx) + return {nullptr}; + return wrap( + static_cast(SwizzleAttr::get(ctx, mask, base, shift))); +} + +MlirAttribute mlirFlyAlignAttrGet(MlirContext context, int32_t alignment) { + mlir::MLIRContext *ctx = loadFly(context); + if (!ctx) + return {nullptr}; + return wrap(static_cast(AlignAttr::get(ctx, alignment))); +} + +MlirAttribute mlirFlyAddressSpaceAttrGet(MlirContext context, + int32_t addressSpace) { + mlir::MLIRContext *ctx = loadFly(context); + if (!ctx) + return {nullptr}; + return wrap(static_cast( + AddressSpaceAttr::get(ctx, static_cast(addressSpace)))); +} + +MlirAttribute mlirFlyMmaOperandAttrGet(MlirContext context, int32_t operand) { + mlir::MLIRContext *ctx = loadFly(context); + if (!ctx) + return {nullptr}; + return wrap(static_cast( + MmaOperandAttr::get(ctx, static_cast(operand)))); +} + +MlirAttribute mlirFlyGemmTraversalOrderAttrGet(MlirContext context, + int32_t order) { + mlir::MLIRContext *ctx = loadFly(context); + if (!ctx) + return {nullptr}; + return wrap(static_cast(GemmTraversalOrderAttr::get( + ctx, static_cast(order)))); +} + +MlirType mlirFlyIntTupleTypeGet(MlirContext context, MlirAttribute attr) { + mlir::MLIRContext *ctx = loadFly(context); + if (!ctx) + return {nullptr}; + IntTupleAttr value = checked(attr, ctx); + if (isNull(value)) + return {nullptr}; + return wrap(static_cast(IntTupleType::get(value))); +} + +MlirAttribute mlirFlyIntTupleTypeGetAttr(MlirType type) { + return wrap( + static_cast(cast(unwrap(type)).getAttr())); +} + +intptr_t mlirFlyIntTupleTypeGetNumElements(MlirType type) { + return cast(unwrap(type)).rank(); +} + +MlirType mlirFlyIntTupleTypeGetElement(MlirType type, intptr_t pos) { + return wrap( + static_cast(cast(unwrap(type)).at(pos))); +} + +bool mlirFlyIntTupleTypeIsLeaf(MlirType type) { + return cast(unwrap(type)).isLeaf(); +} + +bool mlirFlyIntTupleTypeIsStatic(MlirType type) { + return cast(unwrap(type)).isStatic(); +} + +bool mlirFlyIntTupleTypeIsStaticLeaf(MlirType type) { + IntTupleAttr attr = cast(unwrap(type)).getAttr(); + return attr.isLeafInt() && attr.getLeafAsInt().isStatic(); +} + +bool mlirFlyIntTupleTypeIsNoneLeaf(MlirType type) { + return cast(unwrap(type)).getAttr().isLeafNone(); +} + +bool mlirFlyIntTupleTypeIsBasisLeaf(MlirType type) { + return cast(unwrap(type)).getAttr().isLeafBasis(); +} + +int64_t mlirFlyIntTupleTypeGetStaticValue(MlirType type) { + return cast(unwrap(type)).getAttr().getLeafAsInt().getValue(); +} + +MlirType mlirFlyLayoutTypeGet(MlirContext context, MlirAttribute attr) { + mlir::MLIRContext *ctx = loadFly(context); + if (!ctx) + return {nullptr}; + LayoutAttr value = checked(attr, ctx); + if (isNull(value)) + return {nullptr}; + return wrap(static_cast(LayoutType::get(value))); +} + +MlirAttribute mlirFlyLayoutTypeGetAttr(MlirType type) { + return wrap( + static_cast(cast(unwrap(type)).getAttr())); +} + +MlirType mlirFlyLayoutTypeGetShape(MlirType type) { + return wrap(static_cast( + IntTupleType::get(cast(unwrap(type)).getAttr().getShape()))); +} + +MlirType mlirFlyLayoutTypeGetStride(MlirType type) { + return wrap(static_cast( + IntTupleType::get(cast(unwrap(type)).getAttr().getStride()))); +} + +MlirAttribute mlirFlyComposedLayoutTypeGetAttr(MlirType type) { + return wrap(static_cast( + cast(unwrap(type)).getAttr())); +} + +MlirType mlirFlyTileTypeGet(MlirContext context, MlirAttribute attr) { + mlir::MLIRContext *ctx = loadFly(context); + if (!ctx) + return {nullptr}; + TileAttr value = checked(attr, ctx); + if (isNull(value)) + return {nullptr}; + return wrap(static_cast(TileType::get(value))); +} + +MlirAttribute mlirFlyTileTypeGetAttr(MlirType type) { + return wrap( + static_cast(cast(unwrap(type)).getAttr())); +} + +MlirType mlirFlySwizzleTypeGet(MlirContext context, MlirAttribute attr) { + mlir::MLIRContext *ctx = loadFly(context); + if (!ctx) + return {nullptr}; + SwizzleAttr value = checked(attr, ctx); + if (isNull(value)) + return {nullptr}; + return wrap(static_cast(SwizzleType::get(value))); +} + +MlirType mlirFlyPointerTypeGet(MlirContext context, MlirType elemTy, + MlirAttribute addressSpace, + MlirAttribute alignment, MlirAttribute swizzle) { + mlir::MLIRContext *ctx = loadFly(context); + mlir::Type element = unwrap(elemTy); + mlir::Attribute space = unwrap(addressSpace); + if (!ctx || isNull(element) || element.getContext() != ctx) + return {nullptr}; + if (isNull(space) || space.getContext() != ctx) + return {nullptr}; + AlignAttr align = alignment.ptr ? checked(alignment, ctx) + : AlignAttr::getTrivialAlignment(element); + SwizzleAttr sw = swizzle.ptr ? checked(swizzle, ctx) + : SwizzleAttr::getTrivialSwizzle(ctx); + if (isNull(align) || isNull(sw)) + return {nullptr}; + return wrap(static_cast( + PointerType::get(ctx, element, space, align, sw))); +} + +MlirType mlirFlyPointerTypeGetElemTy(MlirType type) { + return wrap(cast(unwrap(type)).getElemTy()); +} + +MlirAttribute mlirFlyPointerTypeGetAddressSpace(MlirType type) { + return wrap(cast(unwrap(type)).getAddressSpace()); +} + +MlirAttribute mlirFlyPointerTypeGetAlignment(MlirType type) { + return wrap(static_cast( + cast(unwrap(type)).getAlignment())); +} + +MlirAttribute mlirFlyPointerTypeGetSwizzle(MlirType type) { + return wrap(static_cast( + cast(unwrap(type)).getSwizzle())); +} + +MlirType mlirFlyMemRefTypeGet(MlirContext context, MlirType elemTy, + MlirAttribute addressSpace, MlirAttribute layout, + MlirAttribute alignment, MlirAttribute swizzle) { + mlir::MLIRContext *ctx = loadFly(context); + mlir::Type element = unwrap(elemTy); + mlir::Attribute space = unwrap(addressSpace); + mlir::Attribute lay = unwrap(layout); + if (!ctx || isNull(element) || element.getContext() != ctx) + return {nullptr}; + if (isNull(space) || space.getContext() != ctx) + return {nullptr}; + if (isNull(lay) || lay.getContext() != ctx) + return {nullptr}; + AlignAttr align = alignment.ptr ? checked(alignment, ctx) + : AlignAttr::getTrivialAlignment(element); + SwizzleAttr sw = swizzle.ptr ? checked(swizzle, ctx) + : SwizzleAttr::getTrivialSwizzle(ctx); + if (isNull(align) || isNull(sw)) + return {nullptr}; + return wrap(static_cast( + mlir::fly::MemRefType::get(ctx, element, space, lay, align, sw))); +} + +MlirType mlirFlyMemRefTypeGetElemTy(MlirType type) { + return wrap(cast(unwrap(type)).getElemTy()); +} + +MlirAttribute mlirFlyMemRefTypeGetAddressSpace(MlirType type) { + return wrap(cast(unwrap(type)).getAddressSpace()); +} + +MlirAttribute mlirFlyMemRefTypeGetLayout(MlirType type) { + return wrap(cast(unwrap(type)).getLayout()); +} + +MlirType mlirFlyCoordTensorTypeGet(MlirContext context, MlirAttribute base, + MlirAttribute layout) { + mlir::MLIRContext *ctx = loadFly(context); + if (!ctx) + return {nullptr}; + IntTupleAttr baseAttr = checked(base, ctx); + mlir::Attribute lay = unwrap(layout); + if (isNull(baseAttr) || isNull(lay) || lay.getContext() != ctx) + return {nullptr}; + return wrap( + static_cast(CoordTensorType::get(ctx, baseAttr, lay))); +} + +static mlir::Attribute outerLayoutAttr(mlir::Attribute layout) { + while (auto composed = mlir::dyn_cast(layout)) + layout = composed.getOuter(); + return layout; +} + +MlirType mlirFlyLayoutLikeTypeGetShape(MlirType type) { + mlir::Type ty = unwrap(type); + mlir::Attribute layout; + if (auto l = mlir::dyn_cast(ty)) { + layout = l.getAttr(); + } else if (auto c = mlir::dyn_cast(ty)) { + layout = c.getAttr(); + } else if (auto m = mlir::dyn_cast(ty)) { + layout = m.getLayout(); + } else { + layout = cast(ty).getLayout(); + } + return wrap(static_cast( + IntTupleType::get(cast(outerLayoutAttr(layout)).getShape()))); +} + +MlirType mlirFlyCopyAtomTypeGet(MlirContext context, MlirType copyOp, + int32_t valBits) { + mlir::MLIRContext *ctx = loadFly(context); + mlir::Type op = unwrap(copyOp); + if (!ctx || isNull(op) || op.getContext() != ctx) + return {nullptr}; + return wrap(static_cast(CopyAtomType::get(ctx, op, valBits))); +} + +MlirType mlirFlyMmaAtomTypeGet(MlirContext context, MlirType mmaOp) { + mlir::MLIRContext *ctx = loadFly(context); + mlir::Type op = unwrap(mmaOp); + if (!ctx || isNull(op) || op.getContext() != ctx) + return {nullptr}; + return wrap(static_cast(MmaAtomType::get(ctx, op))); +} + +#define FLY_ATOM_LAYOUT(Name, AtomType, getter) \ + MlirType Name(MlirType type) { \ + return wrap(static_cast(LayoutType::get( \ + cast(cast(unwrap(type)).getter())))); \ + } + +FLY_ATOM_LAYOUT(mlirFlyCopyAtomTypeGetThrLayout, CopyAtomType, getThrLayout) +FLY_ATOM_LAYOUT(mlirFlyCopyAtomTypeGetThrValLayoutSrc, CopyAtomType, + getThrValLayoutSrc) +FLY_ATOM_LAYOUT(mlirFlyCopyAtomTypeGetThrValLayoutDst, CopyAtomType, + getThrValLayoutDst) +FLY_ATOM_LAYOUT(mlirFlyCopyAtomTypeGetThrValLayoutRef, CopyAtomType, + getThrValLayoutRef) +FLY_ATOM_LAYOUT(mlirFlyMmaAtomTypeGetThrLayout, MmaAtomType, getThrLayout) +FLY_ATOM_LAYOUT(mlirFlyMmaAtomTypeGetThrValLayoutA, MmaAtomType, + getThrValLayoutA) +FLY_ATOM_LAYOUT(mlirFlyMmaAtomTypeGetThrValLayoutB, MmaAtomType, + getThrValLayoutB) +FLY_ATOM_LAYOUT(mlirFlyMmaAtomTypeGetThrValLayoutC, MmaAtomType, + getThrValLayoutC) + +#undef FLY_ATOM_LAYOUT + +MlirType mlirFlyMmaAtomTypeGetShapeMNK(MlirType type) { + return wrap(static_cast(IntTupleType::get( + cast(cast(unwrap(type)).getShapeMNK())))); +} + +MlirType mlirFlyTiledCopyTypeGetTiledThrValLayoutSrc(MlirType type) { + auto ty = cast(unwrap(type)); + return wrap( + static_cast(LayoutType::get(tiledCopyGetTiledThrValLayoutSrc( + cast(ty.getCopyAtom()), ty.getLayoutThrVal().getAttr(), + ty.getTileMN().getAttr())))); +} + +MlirType mlirFlyTiledCopyTypeGetTiledThrValLayoutDst(MlirType type) { + auto ty = cast(unwrap(type)); + return wrap( + static_cast(LayoutType::get(tiledCopyGetTiledThrValLayoutDst( + cast(ty.getCopyAtom()), ty.getLayoutThrVal().getAttr(), + ty.getTileMN().getAttr())))); +} + +MlirType mlirFlyTiledMmaTypeGetTileSizeMNK(MlirType type) { + auto ty = cast(unwrap(type)); + return wrap(static_cast(IntTupleType::get(tiledMmaGetTileSizeMNK( + cast(ty.getMmaAtom()), ty.getAtomLayout().getAttr(), + ty.getPermutation().getAttr())))); +} + +MlirType mlirFlyTiledMmaTypeGetThrLayoutVMNK(MlirType type) { + auto ty = cast(unwrap(type)); + return wrap(static_cast(LayoutType::get(tiledMmaGetThrLayoutVMNK( + cast(ty.getMmaAtom()), ty.getAtomLayout().getAttr())))); +} + +static MlirType tiledMmaTiledThrVal(MlirType tiledMma, MmaOperand operand) { + auto ty = cast(unwrap(tiledMma)); + return wrap( + static_cast(LayoutType::get(tiledMmaGetTiledThrValLayout( + cast(ty.getMmaAtom()), ty.getAtomLayout().getAttr(), + ty.getPermutation().getAttr(), operand)))); +} + +MlirType mlirFlyTiledMmaTypeGetTiledThrValLayoutA(MlirType type) { + return tiledMmaTiledThrVal(type, MmaOperand::A); +} + +MlirType mlirFlyTiledMmaTypeGetTiledThrValLayoutB(MlirType type) { + return tiledMmaTiledThrVal(type, MmaOperand::B); +} + +MlirType mlirFlyTiledMmaTypeGetTiledThrValLayoutC(MlirType type) { + return tiledMmaTiledThrVal(type, MmaOperand::C); +} + +MlirType mlirFlyCopyOpUniversalCopyTypeGet(MlirContext context, + int32_t bitSize) { + mlir::MLIRContext *ctx = loadFly(context); + if (!ctx) + return {nullptr}; + return wrap( + static_cast(CopyOpUniversalCopyType::get(ctx, bitSize))); +} diff --git a/mlir/dialects/fly/capi.h b/mlir/dialects/fly/capi.h new file mode 100644 index 00000000..50fb0f49 --- /dev/null +++ b/mlir/dialects/fly/capi.h @@ -0,0 +1,168 @@ +//===- capi.h - C API for the fly dialect -------------------------*- C -*-===// + +#ifndef MLIR_DIALECTS_FLY_CAPI_H +#define MLIR_DIALECTS_FLY_CAPI_H + +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(Fly, fly); + +MLIR_CAPI_EXPORTED bool mlirAttributeIsAFlyInt(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyIntAttrStaticGet(MlirContext context, + int32_t value); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyIntAttrDynamicGet(MlirContext context, + int32_t width, + int32_t divisibility); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyIntAttrNoneGet(MlirContext context); + +MLIR_CAPI_EXPORTED bool mlirAttributeIsAFlyIntTuple(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyIntTupleAttrGet(MlirContext context, + MlirAttribute value); +MLIR_CAPI_EXPORTED MlirAttribute +mlirFlyIntTupleAttrBasisGet(MlirContext context, MlirAttribute value, + intptr_t nModes, const int32_t *modes); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyIntTupleAttrTupleGet( + MlirContext context, intptr_t nElements, MlirAttribute const *elements); + +MLIR_CAPI_EXPORTED bool mlirAttributeIsAFlyLayout(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyLayoutAttrGet(MlirContext context, + MlirAttribute shape, + MlirAttribute stride); + +MLIR_CAPI_EXPORTED bool mlirAttributeIsAFlyTile(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyTileAttrGet(MlirContext context, + MlirAttribute value); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyTileAttrModesGet( + MlirContext context, intptr_t nModes, MlirAttribute const *modes); + +MLIR_CAPI_EXPORTED bool mlirAttributeIsAFlySwizzle(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlySwizzleAttrGet(MlirContext context, + int32_t mask, + int32_t base, + int32_t shift); + +MLIR_CAPI_EXPORTED bool mlirAttributeIsAFlyAlign(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyAlignAttrGet(MlirContext context, + int32_t alignment); + +MLIR_CAPI_EXPORTED bool mlirAttributeIsAFlyAddressSpace(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirFlyAddressSpaceAttrGet(MlirContext context, int32_t addressSpace); + +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyMmaOperandAttrGet(MlirContext context, + int32_t operand); +MLIR_CAPI_EXPORTED MlirAttribute +mlirFlyGemmTraversalOrderAttrGet(MlirContext context, int32_t order); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyIntTuple(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyIntTupleTypeGet(MlirContext context, + MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyIntTupleTypeGetAttr(MlirType type); +MLIR_CAPI_EXPORTED intptr_t mlirFlyIntTupleTypeGetNumElements(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyIntTupleTypeGetElement(MlirType type, + intptr_t pos); +MLIR_CAPI_EXPORTED bool mlirFlyIntTupleTypeIsLeaf(MlirType type); +MLIR_CAPI_EXPORTED bool mlirFlyIntTupleTypeIsStatic(MlirType type); +MLIR_CAPI_EXPORTED bool mlirFlyIntTupleTypeIsStaticLeaf(MlirType type); +MLIR_CAPI_EXPORTED bool mlirFlyIntTupleTypeIsNoneLeaf(MlirType type); +MLIR_CAPI_EXPORTED bool mlirFlyIntTupleTypeIsBasisLeaf(MlirType type); +MLIR_CAPI_EXPORTED int64_t mlirFlyIntTupleTypeGetStaticValue(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyLayout(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyLayoutTypeGet(MlirContext context, + MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyLayoutTypeGetAttr(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyLayoutTypeGetShape(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyLayoutTypeGetStride(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyComposedLayout(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirFlyComposedLayoutTypeGetAttr(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyTile(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyTileTypeGet(MlirContext context, + MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyTileTypeGetAttr(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlySwizzle(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlySwizzleTypeGet(MlirContext context, + MlirAttribute attr); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyPointer(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyPointerTypeGet(MlirContext context, + MlirType elemTy, + MlirAttribute addressSpace, + MlirAttribute alignment, + MlirAttribute swizzle); +MLIR_CAPI_EXPORTED MlirType mlirFlyPointerTypeGetElemTy(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirFlyPointerTypeGetAddressSpace(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyPointerTypeGetAlignment(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyPointerTypeGetSwizzle(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyMemRef(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyMemRefTypeGet( + MlirContext context, MlirType elemTy, MlirAttribute addressSpace, + MlirAttribute layout, MlirAttribute alignment, MlirAttribute swizzle); +MLIR_CAPI_EXPORTED MlirType mlirFlyMemRefTypeGetElemTy(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute +mlirFlyMemRefTypeGetAddressSpace(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute mlirFlyMemRefTypeGetLayout(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyCoordTensor(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyCoordTensorTypeGet(MlirContext context, + MlirAttribute base, + MlirAttribute layout); + +MLIR_CAPI_EXPORTED MlirType mlirFlyLayoutLikeTypeGetShape(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyCopyAtom(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyCopyAtomTypeGet(MlirContext context, + MlirType copyOp, + int32_t valBits); +MLIR_CAPI_EXPORTED MlirType mlirFlyCopyAtomTypeGetThrLayout(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirFlyCopyAtomTypeGetThrValLayoutSrc(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirFlyCopyAtomTypeGetThrValLayoutDst(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirFlyCopyAtomTypeGetThrValLayoutRef(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyMmaAtom(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyMmaAtomTypeGet(MlirContext context, + MlirType mmaOp); +MLIR_CAPI_EXPORTED MlirType mlirFlyMmaAtomTypeGetThrLayout(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyMmaAtomTypeGetShapeMNK(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyMmaAtomTypeGetThrValLayoutA(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyMmaAtomTypeGetThrValLayoutB(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyMmaAtomTypeGetThrValLayoutC(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyTiledCopy(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirFlyTiledCopyTypeGetTiledThrValLayoutSrc(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirFlyTiledCopyTypeGetTiledThrValLayoutDst(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyTiledMma(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyTiledMmaTypeGetTileSizeMNK(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyTiledMmaTypeGetThrLayoutVMNK(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirFlyTiledMmaTypeGetTiledThrValLayoutA(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirFlyTiledMmaTypeGetTiledThrValLayoutB(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirFlyTiledMmaTypeGetTiledThrValLayoutC(MlirType type); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyCopyOpUniversalCopy(MlirType type); +MLIR_CAPI_EXPORTED MlirType +mlirFlyCopyOpUniversalCopyTypeGet(MlirContext context, int32_t bitSize); + +#ifdef __cplusplus +} +#endif + +#endif // MLIR_DIALECTS_FLY_CAPI_H diff --git a/mlir/dialects/fly/fly.zig b/mlir/dialects/fly/fly.zig new file mode 100644 index 00000000..15530388 --- /dev/null +++ b/mlir/dialects/fly/fly.zig @@ -0,0 +1,120 @@ +const std = @import("std"); + +const c = @import("c"); +const mlir = @import("mlir"); + +pub const attributes = @import("attributes.zig"); +pub const ops = @import("ops.zig"); +pub const types = @import("types.zig"); + +pub const AddressSpace = attributes.AddressSpace; +pub const GemmTraversalOrder = attributes.GemmTraversalOrder; +pub const MmaOperand = attributes.MmaOperand; + +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("fly"); +} + +/// `Operation.make` that names the op and its operand types on failure: a +/// rejected `inferReturnTypes` otherwise panics with nothing to go on. +pub fn make(ctx: *mlir.Context, name: []const u8, args: mlir.Operation.MakeArgs) *mlir.Operation { + return mlir.Operation.try_make(ctx, name, args) catch |err| { + var buf: [4096]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + if (args.operands) |operands| switch (operands) { + .flat => |vs| for (vs, 0..) |v, i| { + w.print("\n operand {d}: {f}", .{ i, v.type_() }) catch break; + }, + .variadic => |segments| for (segments) |seg| for (seg) |v| { + w.print("\n operand: {f}", .{v.type_()}) catch break; + }, + }; + if (args.results) |results| switch (results) { + .flat => |ts| for (ts, 0..) |t, i| { + w.print("\n result {d}: {f}", .{ i, t }) catch break; + }, + .variadic => {}, + }; + std.debug.panic("{s}: cannot create operation ({}){s}", .{ name, err, w.buffered() }); + }; +} + +pub const TypeKind = enum { + memref, + coord_tensor, + ptr, + int_tuple, + layout, + composed_layout, + tile, + swizzle, + copy_atom, + mma_atom, + tiled_copy, + tiled_mma, + other, +}; + +pub fn KindType(comptime kind: TypeKind) type { + return switch (kind) { + .memref => types.MemRefType, + .coord_tensor => types.CoordTensorType, + .ptr => types.PointerType, + .int_tuple => types.IntTupleType, + .layout => types.LayoutType, + .composed_layout => types.ComposedLayoutType, + .tile => types.TileType, + .swizzle => types.SwizzleType, + .copy_atom => types.CopyAtomType, + .mma_atom => types.MmaAtomType, + .tiled_copy => types.TiledCopyType, + .tiled_mma => types.TiledMmaType, + .other => @compileError("fly.KindType: `.other` is not a fly type"), + }; +} + +pub fn typeKind(ty: *const mlir.Type) TypeKind { + inline for (comptime std.meta.tags(TypeKind)) |kind| { + if (kind != .other and ty.isA(KindType(kind)) != null) return kind; + } + return .other; +} + +/// Panics if `ty` is not a `!fly.`. +pub fn expect(ty: *const mlir.Type, comptime kind: TypeKind) *const KindType(kind) { + return ty.isA(KindType(kind)) orelse + std.debug.panic("fly: `{f}` is not a !fly.{s}", .{ ty, @tagName(kind) }); +} + +/// The shape of the outermost plain layout of any layout-carrying type. +pub fn layoutLikeShape(ty: *const mlir.Type) *const types.IntTupleType { + if (ty.isA(types.LayoutType) == null and ty.isA(types.ComposedLayoutType) == null and + ty.isA(types.MemRefType) == null and ty.isA(types.CoordTensorType) == null) + { + std.debug.panic("fly: `{f}` carries no layout", .{ty}); + } + return @ptrCast(c.mlirFlyLayoutLikeTypeGetShape(ty.ptr()).ptr.?); +} + +pub fn elemType(ty: *const mlir.Type) *const mlir.Type { + if (ty.isA(types.MemRefType)) |m| return m.elemTy(); + if (ty.isA(types.PointerType)) |p| return p.elemTy(); + std.debug.panic("fly: `{f}` has no element type", .{ty}); +} + +pub fn addressSpace(ty: *const mlir.Type) *const mlir.Attribute { + if (ty.isA(types.MemRefType)) |m| return m.addressSpace(); + if (ty.isA(types.PointerType)) |p| return p.addressSpace(); + std.debug.panic("fly: `{f}` has no address space", .{ty}); +} + +/// Keeps the address space, alignment and swizzle. +pub fn ptrWithElem(ctx: *mlir.Context, ptr: *const mlir.Type, elem: *const mlir.Type) mlir.Error!*const types.PointerType { + const p = expect(ptr, .ptr); + return types.PointerType.get(ctx, .{ + .elemTy = elem, + .addressSpace = p.addressSpace(), + .alignment = p.alignment(), + .swizzle = p.swizzle(), + }); +} diff --git a/mlir/dialects/fly/ops.zig b/mlir/dialects/fly/ops.zig new file mode 100644 index 00000000..b9753136 --- /dev/null +++ b/mlir/dialects/fly/ops.zig @@ -0,0 +1,1152 @@ +//! `fly` operation builders, generated from FlyDSL's `include/flydsl/Dialect/Fly/IR/FlyOps.td` at +//! 1941889400621f1fc3d8f03bada1a4e7380cdf7e; do not edit, see REGENERATING.md. +//! +//! Builders take the operands, then the result types (only for operations +//! that do not infer them), then the required attributes, then an `opts` +//! struct of optional operands and attributes (pass `.{}` to omit them), then +//! the location. Parameters are the ODS names in snake case, with a trailing +//! underscore where needed. Pass `&.{}` for empty variadic operands; +//! variable-length operand groups of +//! `AttrSizedOperandSegments` operations get their segment sizes from +//! `Operation.make`. Attributes use `mlir.Attribute`. + +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +const make = @import("fly.zig").make; + +/// `fly.add_offset`. Result types are inferred. +pub fn add_offset(ctx: *mlir.Context, ptr: *const mlir.Value, offset: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.add_offset", .{ + .operands = .{ .flat = &.{ ptr, offset } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.append`. Result types are inferred. +pub fn append( + ctx: *mlir.Context, + tuple: *const mlir.Value, + elem: *const mlir.Value, + opts: struct { + n: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.n) |present| attributes.appendAssumeCapacity(.named(ctx, "n", present)); + return make(ctx, "fly.append", .{ + .operands = .{ .flat = &.{ tuple, elem } }, + .attributes = attributes.constSlice(), + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.apply_swizzle`. Result types are inferred. +pub fn apply_swizzle(ctx: *mlir.Context, ptr: *const mlir.Value, swizzle: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.apply_swizzle", .{ + .operands = .{ .flat = &.{ ptr, swizzle } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.assume`. Result types are explicit. +pub fn assume(ctx: *mlir.Context, dst: *const mlir.Value, src: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.assume", .{ + .operands = .{ .flat = &.{ dst, src } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `fly.atom.set_value`. Result types are inferred. +pub fn atom_set_value(ctx: *mlir.Context, atom: *const mlir.Value, value: *const mlir.Value, field: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.atom.set_value", .{ + .operands = .{ .flat = &.{ atom, value } }, + .attributes = &.{ + .named(ctx, "field", field), + }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.blocked_product`. Result types are inferred. +pub fn blocked_product(ctx: *mlir.Context, layout: *const mlir.Value, tile: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.blocked_product", .{ + .operands = .{ .flat = &.{ layout, tile } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.ceil_div`. Result types are inferred. +pub fn ceil_div(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.ceil_div", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.coalesce`. Result types are inferred. +pub fn coalesce( + ctx: *mlir.Context, + layout: *const mlir.Value, + opts: struct { + pattern: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return make(ctx, "fly.coalesce", .{ + .operands = .{ .flat = if (opts.pattern) |present| &.{ layout, present } else &.{layout} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.complement`. Result types are inferred. +pub fn complement( + ctx: *mlir.Context, + layout: *const mlir.Value, + opts: struct { + codomain_size: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return make(ctx, "fly.complement", .{ + .operands = .{ .flat = if (opts.codomain_size) |present| &.{ layout, present } else &.{layout} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.composed_get_inner`. Result types are inferred. +pub fn composed_get_inner(ctx: *mlir.Context, input: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.composed_get_inner", .{ + .operands = .{ .flat = &.{input} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.composed_get_offset`. Result types are inferred. +pub fn composed_get_offset(ctx: *mlir.Context, input: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.composed_get_offset", .{ + .operands = .{ .flat = &.{input} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.composed_get_outer`. Result types are inferred. +pub fn composed_get_outer(ctx: *mlir.Context, input: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.composed_get_outer", .{ + .operands = .{ .flat = &.{input} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.composition`. Result types are inferred. +pub fn composition(ctx: *mlir.Context, outer: *const mlir.Value, inner: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.composition", .{ + .operands = .{ .flat = &.{ outer, inner } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.coprofile`. Result types are inferred. +pub fn coprofile(ctx: *mlir.Context, layout: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.coprofile", .{ + .operands = .{ .flat = &.{layout} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.copy`. No results. +pub fn copy( + ctx: *mlir.Context, + copy_atom: *const mlir.Value, + src: *const mlir.Value, + dst: *const mlir.Value, + opts: struct { + pred: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return make(ctx, "fly.copy", .{ + .operands = .{ .flat = if (opts.pred) |present| &.{ copy_atom, src, dst, present } else &.{ copy_atom, src, dst } }, + .location = location, + }); +} + +/// `fly.copy_atom_call`. No results. +pub fn copy_atom_call( + ctx: *mlir.Context, + copy_atom: *const mlir.Value, + src: *const mlir.Value, + dst: *const mlir.Value, + opts: struct { + pred: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return make(ctx, "fly.copy_atom_call", .{ + .operands = .{ .flat = if (opts.pred) |present| &.{ copy_atom, src, dst, present } else &.{ copy_atom, src, dst } }, + .location = location, + }); +} + +/// `fly.copy_atom_call_ssa`. Result types are explicit. +pub fn copy_atom_call_ssa( + ctx: *mlir.Context, + copy_atom: *const mlir.Value, + src: *const mlir.Value, + results_types: []const *const mlir.Type, + opts: struct { + dst: ?*const mlir.Value = null, + pred: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return make(ctx, "fly.copy_atom_call_ssa", .{ + .operands = .{ .variadic = &.{ + &.{copy_atom}, + &.{src}, + if (opts.dst) |present| &.{present} else &.{}, + if (opts.pred) |present| &.{present} else &.{}, + } }, + .results = .{ .flat = results_types }, + .location = location, + }); +} + +/// `fly.coshape`. Result types are inferred. +pub fn coshape(ctx: *mlir.Context, layout: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.coshape", .{ + .operands = .{ .flat = &.{layout} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.cosize`. Result types are inferred. +pub fn cosize(ctx: *mlir.Context, layout: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.cosize", .{ + .operands = .{ .flat = &.{layout} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.crd2idx`. Result types are inferred. +pub fn crd2idx(ctx: *mlir.Context, coord: *const mlir.Value, layout: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.crd2idx", .{ + .operands = .{ .flat = &.{ coord, layout } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.decomposition`. Result types are inferred. +pub fn decomposition(ctx: *mlir.Context, tensor: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.decomposition", .{ + .operands = .{ .flat = &.{tensor} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.dice`. Result types are inferred. +pub fn dice(ctx: *mlir.Context, src: *const mlir.Value, coord: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.dice", .{ + .operands = .{ .flat = &.{ src, coord } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.elem_less`. Result types are explicit. +pub fn elem_less(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.elem_less", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `fly.equal`. Result types are explicit. +pub fn equal(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.equal", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `fly.extract_aligned_pointer_as_index`. Extract the raw pointer from a fly.memref. Result types are explicit. +pub fn extract_aligned_pointer_as_index(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.extract_aligned_pointer_as_index", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `fly.flat_divide`. Result types are inferred. +pub fn flat_divide(ctx: *mlir.Context, layout: *const mlir.Value, divisor: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.flat_divide", .{ + .operands = .{ .flat = &.{ layout, divisor } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.flat_product`. Result types are inferred. +pub fn flat_product(ctx: *mlir.Context, layout: *const mlir.Value, tile: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.flat_product", .{ + .operands = .{ .flat = &.{ layout, tile } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.gemm`. Fully expand tiled GEMM with variadic A and B operand groups. No results. +pub fn gemm( + ctx: *mlir.Context, + mma_atom: *const mlir.Value, + d: *const mlir.Value, + a: []const *const mlir.Value, + b: []const *const mlir.Value, + c: *const mlir.Value, + opts: struct { + traversal_layout: ?*const mlir.Value = null, + traversal_order: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.traversal_order) |present| attributes.appendAssumeCapacity(.named(ctx, "traversalOrder", present)); + return make(ctx, "fly.gemm", .{ + .operands = .{ .variadic = &.{ + &.{mma_atom}, + &.{d}, + a, + b, + &.{c}, + if (opts.traversal_layout) |present| &.{present} else &.{}, + } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `fly.get`. Result types are inferred. +pub fn get(ctx: *mlir.Context, input: *const mlir.Value, mode: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.get", .{ + .operands = .{ .flat = &.{input} }, + .attributes = &.{ + .named(ctx, "mode", mode), + }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.get_1d_coord`. Result types are inferred. +pub fn get_1d_coord(ctx: *mlir.Context, index: *const mlir.Value, layout: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.get_1d_coord", .{ + .operands = .{ .flat = &.{ index, layout } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.get_copy_atom`. Extract the copy atom and its runtime state from a tiled copy. Result types are inferred. +pub fn get_copy_atom(ctx: *mlir.Context, tiled_copy: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.get_copy_atom", .{ + .operands = .{ .flat = &.{tiled_copy} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.get_dyn_shared`. Result types are inferred. +pub fn get_dyn_shared(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.get_dyn_shared", .{ + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.get_flat_coord`. Result types are inferred. +pub fn get_flat_coord(ctx: *mlir.Context, index: *const mlir.Value, layout: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.get_flat_coord", .{ + .operands = .{ .flat = &.{ index, layout } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.get_iter`. Result types are inferred. +pub fn get_iter(ctx: *mlir.Context, memref: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.get_iter", .{ + .operands = .{ .flat = &.{memref} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.get_layout`. Result types are inferred. +pub fn get_layout(ctx: *mlir.Context, memref: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.get_layout", .{ + .operands = .{ .flat = &.{memref} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.get_leaves`. Result types are inferred. +pub fn get_leaves( + ctx: *mlir.Context, + input: *const mlir.Value, + opts: struct { + dynamic_only: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.dynamic_only) |present| attributes.appendAssumeCapacity(.named(ctx, "dynamicOnly", present)); + return make(ctx, "fly.get_leaves", .{ + .operands = .{ .flat = &.{input} }, + .attributes = attributes.constSlice(), + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.get_mma_atom`. Extract the MMA atom and its runtime state from a tiled MMA. Result types are inferred. +pub fn get_mma_atom(ctx: *mlir.Context, tiled_mma: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.get_mma_atom", .{ + .operands = .{ .flat = &.{tiled_mma} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.get_scalar`. Result types are inferred. +pub fn get_scalar(ctx: *mlir.Context, int_tuple: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.get_scalar", .{ + .operands = .{ .flat = &.{int_tuple} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.get_shape`. Result types are inferred. +pub fn get_shape(ctx: *mlir.Context, layout: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.get_shape", .{ + .operands = .{ .flat = &.{layout} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.get_stride`. Result types are inferred. +pub fn get_stride(ctx: *mlir.Context, layout: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.get_stride", .{ + .operands = .{ .flat = &.{layout} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.group`. Result types are inferred. +pub fn group(ctx: *mlir.Context, tuple: *const mlir.Value, begin: *const mlir.Attribute, end: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.group", .{ + .operands = .{ .flat = &.{tuple} }, + .attributes = &.{ + .named(ctx, "begin", begin), + .named(ctx, "end", end), + }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.idx2crd`. Result types are inferred. +pub fn idx2crd(ctx: *mlir.Context, coord: *const mlir.Value, layout: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.idx2crd", .{ + .operands = .{ .flat = &.{ coord, layout } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.int_tuple_add`. Result types are inferred. +pub fn int_tuple_add(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.int_tuple_add", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.int_tuple_div`. Result types are inferred. +pub fn int_tuple_div(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.int_tuple_div", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.int_tuple_mod`. Result types are inferred. +pub fn int_tuple_mod(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.int_tuple_mod", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.int_tuple_mul`. Result types are inferred. +pub fn int_tuple_mul(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.int_tuple_mul", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.int_tuple_product`. Result types are inferred. +pub fn int_tuple_product(ctx: *mlir.Context, input: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.int_tuple_product", .{ + .operands = .{ .flat = &.{input} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.int_tuple_product_each`. Result types are inferred. +pub fn int_tuple_product_each(ctx: *mlir.Context, input: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.int_tuple_product_each", .{ + .operands = .{ .flat = &.{input} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.int_tuple_product_like`. Result types are inferred. +pub fn int_tuple_product_like(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.int_tuple_product_like", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.int_tuple_sub`. Result types are inferred. +pub fn int_tuple_sub(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.int_tuple_sub", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.inttoptr`. Result types are explicit. +pub fn inttoptr(ctx: *mlir.Context, src: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.inttoptr", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `fly.left_inverse`. Result types are inferred. +pub fn left_inverse(ctx: *mlir.Context, layout: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.left_inverse", .{ + .operands = .{ .flat = &.{layout} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.logical_divide`. Result types are inferred. +pub fn logical_divide(ctx: *mlir.Context, layout: *const mlir.Value, divisor: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.logical_divide", .{ + .operands = .{ .flat = &.{ layout, divisor } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.logical_product`. Result types are inferred. +pub fn logical_product(ctx: *mlir.Context, layout: *const mlir.Value, tile: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.logical_product", .{ + .operands = .{ .flat = &.{ layout, tile } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.make_composed_layout`. Result types are inferred. +pub fn make_composed_layout(ctx: *mlir.Context, inner: *const mlir.Value, offset: *const mlir.Value, outer: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.make_composed_layout", .{ + .operands = .{ .flat = &.{ inner, offset, outer } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.make_coord`. Result types are explicit. +pub fn make_coord(ctx: *mlir.Context, dync_elems: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.make_coord", .{ + .operands = .{ .flat = dync_elems }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `fly.make_copy_atom`. Result types are explicit. +pub fn make_copy_atom(ctx: *mlir.Context, args: []const *const mlir.Value, result_type: *const mlir.Type, val_bits: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.make_copy_atom", .{ + .operands = .{ .flat = args }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "valBits", val_bits), + }, + .location = location, + }); +} + +/// `fly.make_fragment_layout_like`. Result types are inferred. +pub fn make_fragment_layout_like(ctx: *mlir.Context, src: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.make_fragment_layout_like", .{ + .operands = .{ .flat = &.{src} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.make_fragment_like`. Result types are inferred. +pub fn make_fragment_like( + ctx: *mlir.Context, + src: *const mlir.Value, + opts: struct { + dtype: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.dtype) |present| attributes.appendAssumeCapacity(.named(ctx, "dtype", present)); + return make(ctx, "fly.make_fragment_like", .{ + .operands = .{ .flat = &.{src} }, + .attributes = attributes.constSlice(), + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.make_identity_layout`. Result types are inferred. +pub fn make_identity_layout(ctx: *mlir.Context, shape: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.make_identity_layout", .{ + .operands = .{ .flat = &.{shape} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.make_int_tuple`. Result types are explicit. +pub fn make_int_tuple(ctx: *mlir.Context, dync_elems: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.make_int_tuple", .{ + .operands = .{ .flat = dync_elems }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `fly.make_layout`. Result types are inferred. +pub fn make_layout(ctx: *mlir.Context, shape: *const mlir.Value, stride: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.make_layout", .{ + .operands = .{ .flat = &.{ shape, stride } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.make_layout_like`. Result types are inferred. +pub fn make_layout_like(ctx: *mlir.Context, ref: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.make_layout_like", .{ + .operands = .{ .flat = &.{ref} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.make_mma_atom`. Result types are explicit. +pub fn make_mma_atom(ctx: *mlir.Context, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.make_mma_atom", .{ + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `fly.make_ordered_layout`. Result types are inferred. +pub fn make_ordered_layout(ctx: *mlir.Context, shape: *const mlir.Value, order: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.make_ordered_layout", .{ + .operands = .{ .flat = &.{ shape, order } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.make_ptr`. Result types are explicit. +pub fn make_ptr( + ctx: *mlir.Context, + args: []const *const mlir.Value, + result_type: *const mlir.Type, + opts: struct { + dict_attrs: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.dict_attrs) |present| attributes.appendAssumeCapacity(.named(ctx, "dictAttrs", present)); + return make(ctx, "fly.make_ptr", .{ + .operands = .{ .flat = args }, + .results = .{ .flat = &.{result_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `fly.make_shape`. Result types are explicit. +pub fn make_shape(ctx: *mlir.Context, dync_elems: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.make_shape", .{ + .operands = .{ .flat = dync_elems }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `fly.make_stride`. Result types are explicit. +pub fn make_stride(ctx: *mlir.Context, dync_elems: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.make_stride", .{ + .operands = .{ .flat = dync_elems }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `fly.make_tiled_copy`. Result types are inferred. +pub fn make_tiled_copy(ctx: *mlir.Context, copy_atom: *const mlir.Value, layout_thr_val: *const mlir.Value, tile_mn: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.make_tiled_copy", .{ + .operands = .{ .flat = &.{ copy_atom, layout_thr_val, tile_mn } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.make_tiled_mma`. Result types are inferred. +pub fn make_tiled_mma( + ctx: *mlir.Context, + mma_atom: *const mlir.Value, + atom_layout: *const mlir.Value, + opts: struct { + permutation: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return make(ctx, "fly.make_tiled_mma", .{ + .operands = .{ .flat = if (opts.permutation) |present| &.{ mma_atom, atom_layout, present } else &.{ mma_atom, atom_layout } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.make_view`. Result types are inferred. +pub fn make_view(ctx: *mlir.Context, iter: *const mlir.Value, layout: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.make_view", .{ + .operands = .{ .flat = &.{ iter, layout } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.memref.alloca`. Result types are explicit. +pub fn memref_alloca(ctx: *mlir.Context, layout: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.memref.alloca", .{ + .operands = .{ .flat = &.{layout} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `fly.memref.load`. Result types are inferred. +pub fn memref_load(ctx: *mlir.Context, memref: *const mlir.Value, indices: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.memref.load", .{ + .operands = .{ .flat = &.{ memref, indices } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.memref.load_vec`. Result types are inferred. +pub fn memref_load_vec(ctx: *mlir.Context, memref: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.memref.load_vec", .{ + .operands = .{ .flat = &.{memref} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.memref.store`. No results. +pub fn memref_store(ctx: *mlir.Context, value: *const mlir.Value, memref: *const mlir.Value, indices: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.memref.store", .{ + .operands = .{ .flat = &.{ value, memref, indices } }, + .location = location, + }); +} + +/// `fly.memref.store_vec`. No results. +pub fn memref_store_vec(ctx: *mlir.Context, vector: *const mlir.Value, memref: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.memref.store_vec", .{ + .operands = .{ .flat = &.{ vector, memref } }, + .location = location, + }); +} + +/// `fly.mma.make_fragment`. Result types are inferred. +pub fn mma_make_fragment( + ctx: *mlir.Context, + tiled_mma: *const mlir.Value, + input: *const mlir.Value, + operand_id: *const mlir.Attribute, + opts: struct { + stages: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "operand_id", operand_id)); + if (opts.stages) |present| attributes.appendAssumeCapacity(.named(ctx, "stages", present)); + return make(ctx, "fly.mma.make_fragment", .{ + .operands = .{ .flat = &.{ tiled_mma, input } }, + .attributes = attributes.constSlice(), + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.mma_atom_call`. No results. +pub fn mma_atom_call(ctx: *mlir.Context, mma_atom: *const mlir.Value, d: *const mlir.Value, a: []const *const mlir.Value, b: []const *const mlir.Value, c: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.mma_atom_call", .{ + .operands = .{ .variadic = &.{ + &.{mma_atom}, + &.{d}, + a, + b, + &.{c}, + } }, + .location = location, + }); +} + +/// `fly.mma_atom_call_ssa`. Result types are explicit. +pub fn mma_atom_call_ssa( + ctx: *mlir.Context, + mma_atom: *const mlir.Value, + a: []const *const mlir.Value, + b: []const *const mlir.Value, + c: *const mlir.Value, + results_types: []const *const mlir.Type, + opts: struct { + d: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return make(ctx, "fly.mma_atom_call_ssa", .{ + .operands = .{ .variadic = &.{ + &.{mma_atom}, + if (opts.d) |present| &.{present} else &.{}, + a, + b, + &.{c}, + } }, + .results = .{ .flat = results_types }, + .location = location, + }); +} + +/// `fly.prepend`. Result types are inferred. +pub fn prepend( + ctx: *mlir.Context, + tuple: *const mlir.Value, + elem: *const mlir.Value, + opts: struct { + n: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.n) |present| attributes.appendAssumeCapacity(.named(ctx, "n", present)); + return make(ctx, "fly.prepend", .{ + .operands = .{ .flat = &.{ tuple, elem } }, + .attributes = attributes.constSlice(), + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.print`. No results. +pub fn print(ctx: *mlir.Context, values: []const *const mlir.Value, format: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.print", .{ + .operands = .{ .flat = values }, + .attributes = &.{ + .named(ctx, "format", format), + }, + .location = location, + }); +} + +/// `fly.ptr.load`. Result types are explicit. +pub fn ptr_load(ctx: *mlir.Context, ptr: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.ptr.load", .{ + .operands = .{ .flat = &.{ptr} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `fly.ptr.store`. No results. +pub fn ptr_store(ctx: *mlir.Context, value: *const mlir.Value, ptr: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.ptr.store", .{ + .operands = .{ .flat = &.{ value, ptr } }, + .location = location, + }); +} + +/// `fly.ptrtoint`. Result types are inferred. +pub fn ptrtoint(ctx: *mlir.Context, ptr: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.ptrtoint", .{ + .operands = .{ .flat = &.{ptr} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.raked_product`. Result types are inferred. +pub fn raked_product(ctx: *mlir.Context, layout: *const mlir.Value, tile: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.raked_product", .{ + .operands = .{ .flat = &.{ layout, tile } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.recast_iter`. Result types are explicit. +pub fn recast_iter(ctx: *mlir.Context, src: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.recast_iter", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `fly.recast_layout`. Result types are inferred. +pub fn recast_layout(ctx: *mlir.Context, src: *const mlir.Value, new_type_bits: *const mlir.Attribute, old_type_bits: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.recast_layout", .{ + .operands = .{ .flat = &.{src} }, + .attributes = &.{ + .named(ctx, "new_type_bits", new_type_bits), + .named(ctx, "old_type_bits", old_type_bits), + }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.right_inverse`. Result types are inferred. +pub fn right_inverse(ctx: *mlir.Context, layout: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.right_inverse", .{ + .operands = .{ .flat = &.{layout} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.select`. Result types are inferred. +pub fn select(ctx: *mlir.Context, tuple: *const mlir.Value, indices: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.select", .{ + .operands = .{ .flat = &.{tuple} }, + .attributes = &.{ + .named(ctx, "indices", indices), + }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.shape_div`. Result types are inferred. +pub fn shape_div(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.shape_div", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.size`. Result types are inferred. +pub fn size(ctx: *mlir.Context, int_tuple: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.size", .{ + .operands = .{ .flat = &.{int_tuple} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.slice`. Result types are inferred. +pub fn slice(ctx: *mlir.Context, src: *const mlir.Value, coord: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.slice", .{ + .operands = .{ .flat = &.{ src, coord } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.static`. Result types are explicit. +pub fn static(ctx: *mlir.Context, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.static", .{ + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `fly.take`. Result types are inferred. +pub fn take(ctx: *mlir.Context, tuple: *const mlir.Value, begin: *const mlir.Attribute, end: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.take", .{ + .operands = .{ .flat = &.{tuple} }, + .attributes = &.{ + .named(ctx, "begin", begin), + .named(ctx, "end", end), + }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.tile_to_shape`. Result types are inferred. +pub fn tile_to_shape(ctx: *mlir.Context, block: *const mlir.Value, trg_shape: *const mlir.Value, ord_shape: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.tile_to_shape", .{ + .operands = .{ .flat = &.{ block, trg_shape, ord_shape } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.tiled_copy.partition_dst`. Result types are inferred. +pub fn tiled_copy_partition_dst(ctx: *mlir.Context, tiled_copy: *const mlir.Value, dst: *const mlir.Value, coord: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.tiled_copy.partition_dst", .{ + .operands = .{ .flat = &.{ tiled_copy, dst, coord } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.tiled_copy.partition_src`. Result types are inferred. +pub fn tiled_copy_partition_src(ctx: *mlir.Context, tiled_copy: *const mlir.Value, src: *const mlir.Value, coord: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.tiled_copy.partition_src", .{ + .operands = .{ .flat = &.{ tiled_copy, src, coord } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.tiled_copy.retile`. Result types are inferred. +pub fn tiled_copy_retile(ctx: *mlir.Context, tiled_copy: *const mlir.Value, input: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.tiled_copy.retile", .{ + .operands = .{ .flat = &.{ tiled_copy, input } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.tiled_divide`. Result types are inferred. +pub fn tiled_divide(ctx: *mlir.Context, layout: *const mlir.Value, divisor: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.tiled_divide", .{ + .operands = .{ .flat = &.{ layout, divisor } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.tiled_mma.partition`. Result types are inferred. +pub fn tiled_mma_partition(ctx: *mlir.Context, tiled_mma: *const mlir.Value, input: *const mlir.Value, coord: *const mlir.Value, operand_id: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.tiled_mma.partition", .{ + .operands = .{ .flat = &.{ tiled_mma, input, coord } }, + .attributes = &.{ + .named(ctx, "operand_id", operand_id), + }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.tiled_mma.partition_shape`. Result types are inferred. +pub fn tiled_mma_partition_shape(ctx: *mlir.Context, tiled_mma: *const mlir.Value, shape: *const mlir.Value, operand_id: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.tiled_mma.partition_shape", .{ + .operands = .{ .flat = &.{ tiled_mma, shape } }, + .attributes = &.{ + .named(ctx, "operand_id", operand_id), + }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.tiled_product`. Result types are inferred. +pub fn tiled_product(ctx: *mlir.Context, layout: *const mlir.Value, tile: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.tiled_product", .{ + .operands = .{ .flat = &.{ layout, tile } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.to_llvm_ptr`. Result types are inferred. +pub fn to_llvm_ptr(ctx: *mlir.Context, ptr: *const mlir.Value, llvm_address_space: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.to_llvm_ptr", .{ + .operands = .{ .flat = &.{ptr} }, + .attributes = &.{ + .named(ctx, "llvm_address_space", llvm_address_space), + }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.zipped_divide`. Result types are inferred. +pub fn zipped_divide(ctx: *mlir.Context, layout: *const mlir.Value, divisor: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.zipped_divide", .{ + .operands = .{ .flat = &.{ layout, divisor } }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly.zipped_product`. Result types are inferred. +pub fn zipped_product(ctx: *mlir.Context, layout: *const mlir.Value, tile: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly.zipped_product", .{ + .operands = .{ .flat = &.{ layout, tile } }, + .result_type_inference = true, + .location = location, + }); +} diff --git a/mlir/dialects/fly/test.zig b/mlir/dialects/fly/test.zig new file mode 100644 index 00000000..a30172ff --- /dev/null +++ b/mlir/dialects/fly/test.zig @@ -0,0 +1,378 @@ +const std = @import("std"); + +const arith = @import("mlir/dialects/arith"); +const fly = @import("mlir/dialects/fly"); +const fly_rocdl = @import("mlir/dialects/fly_rocdl"); +const func = @import("mlir/dialects/func"); +const gpu = @import("mlir/dialects/gpu"); +const llvm = @import("mlir/dialects/llvm"); +const math = @import("mlir/dialects/math"); +const memref = @import("mlir/dialects/memref"); +const mlir = @import("mlir"); +const rocdl = @import("mlir/dialects/rocdl"); +const scf = @import("mlir/dialects/scf"); +const ub = @import("mlir/dialects/ub"); +const vector = @import("mlir/dialects/vector"); + +const attributes = fly.attributes; +const ops = fly.ops; +const types = fly.types; +const TypeKind = fly.TypeKind; +const typeKind = fly.typeKind; +const expect = fly.expect; +const layoutLikeShape = fly.layoutLikeShape; +const elemType = fly.elemType; +const ptrWithElem = fly.ptrWithElem; + +const operation_names: []const []const u8 = &.{ + "fly.add_offset", + "fly.append", + "fly.apply_swizzle", + "fly.assume", + "fly.atom.set_value", + "fly.blocked_product", + "fly.ceil_div", + "fly.coalesce", + "fly.complement", + "fly.composed_get_inner", + "fly.composed_get_offset", + "fly.composed_get_outer", + "fly.composition", + "fly.coprofile", + "fly.copy", + "fly.copy_atom_call", + "fly.copy_atom_call_ssa", + "fly.coshape", + "fly.cosize", + "fly.crd2idx", + "fly.decomposition", + "fly.dice", + "fly.elem_less", + "fly.equal", + "fly.extract_aligned_pointer_as_index", + "fly.flat_divide", + "fly.flat_product", + "fly.gemm", + "fly.get", + "fly.get_1d_coord", + "fly.get_copy_atom", + "fly.get_dyn_shared", + "fly.get_flat_coord", + "fly.get_iter", + "fly.get_layout", + "fly.get_leaves", + "fly.get_mma_atom", + "fly.get_scalar", + "fly.get_shape", + "fly.get_stride", + "fly.group", + "fly.idx2crd", + "fly.int_tuple_add", + "fly.int_tuple_div", + "fly.int_tuple_mod", + "fly.int_tuple_mul", + "fly.int_tuple_product", + "fly.int_tuple_product_each", + "fly.int_tuple_product_like", + "fly.int_tuple_sub", + "fly.inttoptr", + "fly.left_inverse", + "fly.logical_divide", + "fly.logical_product", + "fly.make_composed_layout", + "fly.make_coord", + "fly.make_copy_atom", + "fly.make_fragment_layout_like", + "fly.make_fragment_like", + "fly.make_identity_layout", + "fly.make_int_tuple", + "fly.make_layout", + "fly.make_layout_like", + "fly.make_mma_atom", + "fly.make_ordered_layout", + "fly.make_ptr", + "fly.make_shape", + "fly.make_stride", + "fly.make_tiled_copy", + "fly.make_tiled_mma", + "fly.make_view", + "fly.memref.alloca", + "fly.memref.load", + "fly.memref.load_vec", + "fly.memref.store", + "fly.memref.store_vec", + "fly.mma.make_fragment", + "fly.mma_atom_call", + "fly.mma_atom_call_ssa", + "fly.prepend", + "fly.print", + "fly.ptr.load", + "fly.ptr.store", + "fly.ptrtoint", + "fly.raked_product", + "fly.recast_iter", + "fly.recast_layout", + "fly.right_inverse", + "fly.select", + "fly.shape_div", + "fly.size", + "fly.slice", + "fly.static", + "fly.take", + "fly.tile_to_shape", + "fly.tiled_copy.partition_dst", + "fly.tiled_copy.partition_src", + "fly.tiled_copy.retile", + "fly.tiled_divide", + "fly.tiled_mma.partition", + "fly.tiled_mma.partition_shape", + "fly.tiled_product", + "fly.to_llvm_ptr", + "fly.zipped_divide", + "fly.zipped_product", +}; + +test { + std.testing.refAllDecls(fly); + inline for (.{ types, attributes, ops, fly_rocdl, fly_rocdl.ops }) |module| { + std.testing.refAllDecls(module); + inline for (comptime std.meta.declarations(module)) |decl| { + const value = @field(module, decl); + if (@TypeOf(value) == type and @typeInfo(value) == .@"opaque") std.testing.refAllDecls(value); + } + } +} + +fn testContext() !*mlir.Context { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + func.registerDialect(registry); + arith.registerDialect(registry); + scf.registerDialect(registry); + math.registerDialect(registry); + vector.registerDialect(registry); + memref.registerDialect(registry); + llvm.registerDialect(registry); + gpu.registerDialect(registry); + rocdl.registerDialect(registry); + ub.registerDialect(registry); + fly.registerDialect(registry); + fly_rocdl.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + return ctx; +} + +fn expectPrints(expected: []const u8, value: anytype) !void { + var buf: [512]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print("{f}", .{value}); + try std.testing.expectEqualStrings(expected, w.buffered()); +} + +test "fly dialects register" { + const ctx = try testContext(); + defer ctx.deinit(); + + try std.testing.expect(ctx.isRegisteredOperation("fly.make_int_tuple")); + try std.testing.expect(ctx.isRegisteredOperation("fly.make_layout")); + try std.testing.expect(ctx.isRegisteredOperation("fly.ptr.load")); + try std.testing.expect(ctx.isRegisteredOperation("fly.copy_atom_call")); + try std.testing.expect(ctx.isRegisteredOperation("gpu.func")); + try std.testing.expect(ctx.isRegisteredOperation("rocdl.sched.barrier")); + try std.testing.expect(ctx.isRegisteredOperation("ub.poison")); +} + +test "fly types parse and print round-trip" { + const ctx = try testContext(); + defer ctx.deinit(); + + const cases = [_][]const u8{ + "!fly.int_tuple<((2, 4), 8)>", + "!fly.int_tuple<((1, 4), (32, 64))>", + "!fly.layout<((2, 3), 3) : ((57, 69), 19)>", + "!fly.memref", + "!fly.memref", + "!fly.ptr", + "!fly.ptr", + "!fly.ptr", + "!fly.memref", + "!fly_rocdl.cdna3.buffer_copy<128>", + "!fly_rocdl.cdna3.buffer_copy_lds<32>", + "!fly_rocdl.cdna3.mfma<16x16x16, (f16, f16) -> f32>", + "!fly.copy_atom, 32>", + "!fly.tile<[8|64]>", + "!fly.coord_tensor<(0,0), (100,1000):(1E0,1E1)>", + }; + for (cases) |src| { + const ty = mlir.Type.parse(ctx, src) catch |err| { + std.debug.print("failed to parse `{s}`: {}\n", .{ src, err }); + return err; + }; + var buf: [512]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print("{f}", .{ty}); + const ty2 = try mlir.Type.parse(ctx, w.buffered()); + try std.testing.expect(ty.eql(ty2)); + } +} + +test "types and attributes are built through the C API" { + const ctx = try testContext(); + defer ctx.deinit(); + + const two = try attributes.IntTupleAttr.getStatic(ctx, 2); + const four = try attributes.IntTupleAttr.getStatic(ctx, 4); + const one = try attributes.IntTupleAttr.getStatic(ctx, 1); + const shape = try attributes.IntTupleAttr.getTuple(ctx, &.{ two, four }); + const stride = try attributes.IntTupleAttr.getTuple(ctx, &.{ four, one }); + const layout = try types.LayoutType.get(ctx, .{ .attr = try .get(ctx, .{ .shape = shape, .stride = stride }) }); + try expectPrints("!fly.layout<(2,4):(4,1)>", layout); + try expectPrints("!fly.int_tuple<(2,4)>", layout.shape()); + try std.testing.expectEqual(@as(usize, 2), layout.shape().numElements()); + try std.testing.expectEqual(types.IntTupleType.Leaf{ .static = 4 }, layout.shape().element(1).leaf()); + + const global = try attributes.AddressSpaceAttr.get(ctx, .{ .addressSpace = .global }); + const memref_type = try types.MemRefType.get(ctx, .{ + .elemTy = .float(ctx, .f16), + .addressSpace = global.attribute(), + .layout = layout.attr().attribute(), + }); + try expectPrints("!fly.memref", memref_type); + try expectPrints("!fly.int_tuple<(2,4)>", layoutLikeShape(memref_type.type_())); + + const shared = try attributes.AddressSpaceAttr.get(ctx, .{ .addressSpace = .shared }); + const ptr = try types.PointerType.get(ctx, .{ + .elemTy = .int(ctx, .i8), + .addressSpace = shared.attribute(), + .alignment = try .get(ctx, .{ .alignment = 16 }), + }); + try expectPrints("!fly.ptr>", ptr); + try expectPrints("!fly.ptr>", try ptrWithElem(ctx, ptr.type_(), .float(ctx, .f32))); + + const tile = try types.TileType.get(ctx, .{ .attr = try .getModes(ctx, &.{ + (try attributes.IntAttr.getStatic(ctx, 8)).attribute(), + (try attributes.IntAttr.getStatic(ctx, 64)).attribute(), + }) }); + try expectPrints("!fly.tile<[8|64]>", tile); + + const copy_atom = try types.CopyAtomType.get(ctx, .{ + .copyOp = (try types.CopyOpUniversalCopyType.get(ctx, .{ .bitSize = 128 })).type_(), + .valBits = 32, + }); + try expectPrints("!fly.copy_atom, 32>", copy_atom); + + const buffer_desc = try fly_rocdl.BufferDescAddressAttr.get(ctx); + try expectPrints("!fly.ptr", try types.PointerType.get(ctx, .{ + .elemTy = .float(ctx, .f16), + .addressSpace = buffer_desc.attribute(), + })); +} + +test "fly enum attributes" { + const ctx = try testContext(); + defer ctx.deinit(); + _ = try attributes.mmaOperandAttr(ctx, .a); + _ = try attributes.gemmTraversalOrderAttr(ctx, .kmn); +} + +test "inferred fly ops compute layout algebra" { + const ctx = try testContext(); + defer ctx.deinit(); + const loc: *const mlir.Location = .unknown(ctx); + const module: *mlir.Module = .init(loc); + defer module.deinit(); + const block = module.body(); + + // raked_product((8,16):(16,1), (1,4):(1,1)) — the vectorAdd TV layout. + const thr = ops.static(ctx, try mlir.Type.parse(ctx, "!fly.layout<(8,16):(16,1)>"), loc).appendTo(block); + const val = ops.static(ctx, try mlir.Type.parse(ctx, "!fly.layout<(1,4):(1,1)>"), loc).appendTo(block); + const mn = ops.raked_product(ctx, thr.result(0), val.result(0), loc).appendTo(block); + const shape = ops.get_shape(ctx, mn.result(0), loc).appendTo(block); + const tiler = ops.int_tuple_product_each(ctx, shape.result(0), loc).appendTo(block); + + try expectPrints("!fly.int_tuple<(8,64)>", tiler.result(0).type_()); + try std.testing.expect(module.operation().verify()); +} + +test "atom traits" { + const ctx = try testContext(); + defer ctx.deinit(); + + // MFMA 16x16x4 f32 tiled over a (2,2,1) atom layout. + const tm = expect(try mlir.Type.parse(ctx, "!fly.tiled_mma f32>>, <(2,2,1):(1,2,0)>>"), .tiled_mma); + try expectPrints("!fly.int_tuple<(32,32,4)>", tm.tileSizeMNK()); + _ = tm.tiledThrValLayoutA(); + _ = tm.thrLayoutVMNK(); + + const atom = try types.MmaAtomType.get(ctx, .{ .mmaOp = (try fly_rocdl.MmaOpCDNA3MFMAType.get(ctx, .{ + .m = 16, + .n = 16, + .k = 4, + .elemTyA = .float(ctx, .f32), + .elemTyB = .float(ctx, .f32), + .elemTyAcc = .float(ctx, .f32), + })).type_() }); + try expectPrints("!fly.mma_atom f32>>", atom); + try expectPrints("!fly.int_tuple<(16,16,4)>", atom.shapeMNK()); + try expectPrints("!fly.layout<64:1>", atom.thrLayout()); + + const wmma = try types.MmaAtomType.get(ctx, .{ .mmaOp = (try fly_rocdl.MmaOpGFX11WMMAType.get(ctx, .{ + .m = 16, + .n = 16, + .k = 16, + .elemTyA = .float(ctx, .f16), + .elemTyB = .float(ctx, .f16), + .elemTyAcc = .float(ctx, .f32), + })).type_() }); + try expectPrints("!fly.layout<32:1>", wmma.thrLayout()); + + const ca = expect(try mlir.Type.parse(ctx, "!fly.copy_atom, 32>"), .copy_atom); + _ = ca.thrValLayoutSrc(); +} + +test "type kinds" { + const ctx = try testContext(); + defer ctx.deinit(); + const cases = .{ + .{ "!fly.memref", TypeKind.memref }, + .{ "!fly.ptr", TypeKind.ptr }, + .{ "!fly.int_tuple<(2,4)>", TypeKind.int_tuple }, + .{ "!fly.layout<(8,4):(4,1)>", TypeKind.layout }, + .{ "!fly.tile<[8|4]>", TypeKind.tile }, + .{ "!fly.copy_atom, 32>", TypeKind.copy_atom }, + .{ "!fly.mma_atom f32>>", TypeKind.mma_atom }, + }; + inline for (cases) |case| { + try std.testing.expectEqual(case[1], typeKind(try mlir.Type.parse(ctx, case[0]))); + } + try std.testing.expectEqual(TypeKind.other, typeKind(.float(ctx, .f32))); +} + +test "structural readers" { + const ctx = try testContext(); + defer ctx.deinit(); + const it = expect(try mlir.Type.parse(ctx, "!fly.int_tuple<((2,4),?)>"), .int_tuple); + try std.testing.expectEqual(@as(usize, 2), it.numElements()); + try std.testing.expect(!it.isLeaf()); + try std.testing.expect(!it.isStatic()); + const inner = it.element(0); + try std.testing.expectEqual(@as(usize, 2), inner.numElements()); + try std.testing.expectEqual(types.IntTupleType.Leaf{ .static = 4 }, inner.element(1).leaf()); + try std.testing.expectEqual(types.IntTupleType.Leaf.dynamic, it.element(1).leaf()); + + const m = try mlir.Type.parse(ctx, "!fly.memref o 0 o (64,32):(32,1), align<16>>"); + try expectPrints("!fly.int_tuple<(64,32)>", layoutLikeShape(m)); + try expectPrints("bf16", elemType(m)); +} + +test "every bound operation is registered" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + fly.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} diff --git a/mlir/dialects/fly/types.zig b/mlir/dialects/fly/types.zig new file mode 100644 index 00000000..27966997 --- /dev/null +++ b/mlir/dialects/fly/types.zig @@ -0,0 +1,429 @@ +const c = @import("c"); +const mlir = @import("mlir"); + +const attributes = @import("attributes.zig"); + +fn optAttr(value: anytype) c.MlirAttribute { + return if (value) |v| v.ptr() else .{ .ptr = null }; +} + +/// `!fly.int_tuple`. +pub const IntTupleType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsAFlyIntTuple; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + + pub const InitArgs = struct { + attr: *const attributes.IntTupleAttr, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirFlyIntTupleTypeGet(ctx.ptr(), args.attr.ptr()).ptr orelse return error.InvalidMlir); + } + + pub fn attr(self: *const Self) *const attributes.IntTupleAttr { + return @ptrCast(c.mlirFlyIntTupleTypeGetAttr(self.ptr()).ptr.?); + } + + /// 1 for a leaf. + pub fn numElements(self: *const Self) usize { + return @intCast(c.mlirFlyIntTupleTypeGetNumElements(self.ptr())); + } + + pub fn element(self: *const Self, pos: usize) *const Self { + return @ptrCast(c.mlirFlyIntTupleTypeGetElement(self.ptr(), @intCast(pos)).ptr.?); + } + + pub fn isLeaf(self: *const Self) bool { + return c.mlirFlyIntTupleTypeIsLeaf(self.ptr()); + } + + pub fn isStatic(self: *const Self) bool { + return c.mlirFlyIntTupleTypeIsStatic(self.ptr()); + } + + pub const Leaf = union(enum) { static: i64, dynamic, none, basis }; + + /// Asserts the tuple is a leaf. + pub fn leaf(self: *const Self) Leaf { + if (c.mlirFlyIntTupleTypeIsStaticLeaf(self.ptr())) { + return .{ .static = c.mlirFlyIntTupleTypeGetStaticValue(self.ptr()) }; + } + if (c.mlirFlyIntTupleTypeIsNoneLeaf(self.ptr())) return .none; + if (c.mlirFlyIntTupleTypeIsBasisLeaf(self.ptr())) return .basis; + return .dynamic; + } +}; + +/// `!fly.layout`. +pub const LayoutType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsAFlyLayout; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + + pub const InitArgs = struct { + attr: *const attributes.LayoutAttr, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirFlyLayoutTypeGet(ctx.ptr(), args.attr.ptr()).ptr orelse return error.InvalidMlir); + } + + pub fn attr(self: *const Self) *const attributes.LayoutAttr { + return @ptrCast(c.mlirFlyLayoutTypeGetAttr(self.ptr()).ptr.?); + } + + pub fn shape(self: *const Self) *const IntTupleType { + return @ptrCast(c.mlirFlyLayoutTypeGetShape(self.ptr()).ptr.?); + } + + pub fn stride(self: *const Self) *const IntTupleType { + return @ptrCast(c.mlirFlyLayoutTypeGetStride(self.ptr()).ptr.?); + } +}; + +/// `!fly.composed_layout`. +pub const ComposedLayoutType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsAFlyComposedLayout; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + + pub fn attr(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirFlyComposedLayoutTypeGetAttr(self.ptr()).ptr.?); + } +}; + +/// `!fly.tile`. +pub const TileType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsAFlyTile; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + + pub const InitArgs = struct { + attr: *const attributes.TileAttr, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirFlyTileTypeGet(ctx.ptr(), args.attr.ptr()).ptr orelse return error.InvalidMlir); + } + + pub fn attr(self: *const Self) *const attributes.TileAttr { + return @ptrCast(c.mlirFlyTileTypeGetAttr(self.ptr()).ptr.?); + } +}; + +/// `!fly.swizzle`. +pub const SwizzleType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsAFlySwizzle; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + + pub const InitArgs = struct { + attr: *const attributes.SwizzleAttr, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirFlySwizzleTypeGet(ctx.ptr(), args.attr.ptr()).ptr orelse return error.InvalidMlir); + } +}; + +/// `!fly.ptr`. +pub const PointerType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsAFlyPointer; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + + pub const InitArgs = struct { + elemTy: *const mlir.Type, + addressSpace: *const mlir.Attribute, + alignment: ?*const attributes.AlignAttr = null, + swizzle: ?*const attributes.SwizzleAttr = null, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirFlyPointerTypeGet(ctx.ptr(), args.elemTy.ptr(), args.addressSpace.ptr(), optAttr(args.alignment), optAttr(args.swizzle)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + + pub fn elemTy(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirFlyPointerTypeGetElemTy(self.ptr()).ptr.?); + } + + pub fn addressSpace(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirFlyPointerTypeGetAddressSpace(self.ptr()).ptr.?); + } + + pub fn alignment(self: *const Self) *const attributes.AlignAttr { + return @ptrCast(c.mlirFlyPointerTypeGetAlignment(self.ptr()).ptr.?); + } + + pub fn swizzle(self: *const Self) *const attributes.SwizzleAttr { + return @ptrCast(c.mlirFlyPointerTypeGetSwizzle(self.ptr()).ptr.?); + } +}; + +/// `!fly.memref`. +pub const MemRefType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsAFlyMemRef; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + + pub const InitArgs = struct { + elemTy: *const mlir.Type, + addressSpace: *const mlir.Attribute, + layout: *const mlir.Attribute, + alignment: ?*const attributes.AlignAttr = null, + swizzle: ?*const attributes.SwizzleAttr = null, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirFlyMemRefTypeGet(ctx.ptr(), args.elemTy.ptr(), args.addressSpace.ptr(), args.layout.ptr(), optAttr(args.alignment), optAttr(args.swizzle)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + + pub fn elemTy(self: *const Self) *const mlir.Type { + return @ptrCast(c.mlirFlyMemRefTypeGetElemTy(self.ptr()).ptr.?); + } + + pub fn addressSpace(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirFlyMemRefTypeGetAddressSpace(self.ptr()).ptr.?); + } + + pub fn layout(self: *const Self) *const mlir.Attribute { + return @ptrCast(c.mlirFlyMemRefTypeGetLayout(self.ptr()).ptr.?); + } +}; + +/// `!fly.coord_tensor`. +pub const CoordTensorType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsAFlyCoordTensor; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + + pub const InitArgs = struct { + base: *const attributes.IntTupleAttr, + layout: *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirFlyCoordTensorTypeGet(ctx.ptr(), args.base.ptr(), args.layout.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!fly.copy_atom`. +pub const CopyAtomType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsAFlyCopyAtom; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + + pub const InitArgs = struct { + copyOp: *const mlir.Type, + valBits: i32, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirFlyCopyAtomTypeGet(ctx.ptr(), args.copyOp.ptr(), args.valBits); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + + pub fn thrLayout(self: *const Self) *const LayoutType { + return @ptrCast(c.mlirFlyCopyAtomTypeGetThrLayout(self.ptr()).ptr.?); + } + + pub fn thrValLayoutSrc(self: *const Self) *const LayoutType { + return @ptrCast(c.mlirFlyCopyAtomTypeGetThrValLayoutSrc(self.ptr()).ptr.?); + } + + pub fn thrValLayoutDst(self: *const Self) *const LayoutType { + return @ptrCast(c.mlirFlyCopyAtomTypeGetThrValLayoutDst(self.ptr()).ptr.?); + } + + pub fn thrValLayoutRef(self: *const Self) *const LayoutType { + return @ptrCast(c.mlirFlyCopyAtomTypeGetThrValLayoutRef(self.ptr()).ptr.?); + } +}; + +/// `!fly.mma_atom`. +pub const MmaAtomType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsAFlyMmaAtom; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + + pub const InitArgs = struct { + mmaOp: *const mlir.Type, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirFlyMmaAtomTypeGet(ctx.ptr(), args.mmaOp.ptr()).ptr orelse return error.InvalidMlir); + } + + pub fn thrLayout(self: *const Self) *const LayoutType { + return @ptrCast(c.mlirFlyMmaAtomTypeGetThrLayout(self.ptr()).ptr.?); + } + + pub fn shapeMNK(self: *const Self) *const IntTupleType { + return @ptrCast(c.mlirFlyMmaAtomTypeGetShapeMNK(self.ptr()).ptr.?); + } + + pub fn thrValLayoutA(self: *const Self) *const LayoutType { + return @ptrCast(c.mlirFlyMmaAtomTypeGetThrValLayoutA(self.ptr()).ptr.?); + } + + pub fn thrValLayoutB(self: *const Self) *const LayoutType { + return @ptrCast(c.mlirFlyMmaAtomTypeGetThrValLayoutB(self.ptr()).ptr.?); + } + + pub fn thrValLayoutC(self: *const Self) *const LayoutType { + return @ptrCast(c.mlirFlyMmaAtomTypeGetThrValLayoutC(self.ptr()).ptr.?); + } +}; + +/// `!fly.tiled_copy`. +pub const TiledCopyType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsAFlyTiledCopy; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + + pub fn tiledThrValLayoutSrc(self: *const Self) *const LayoutType { + return @ptrCast(c.mlirFlyTiledCopyTypeGetTiledThrValLayoutSrc(self.ptr()).ptr.?); + } + + pub fn tiledThrValLayoutDst(self: *const Self) *const LayoutType { + return @ptrCast(c.mlirFlyTiledCopyTypeGetTiledThrValLayoutDst(self.ptr()).ptr.?); + } +}; + +/// `!fly.tiled_mma`. +pub const TiledMmaType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsAFlyTiledMma; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + + pub fn tileSizeMNK(self: *const Self) *const IntTupleType { + return @ptrCast(c.mlirFlyTiledMmaTypeGetTileSizeMNK(self.ptr()).ptr.?); + } + + pub fn thrLayoutVMNK(self: *const Self) *const LayoutType { + return @ptrCast(c.mlirFlyTiledMmaTypeGetThrLayoutVMNK(self.ptr()).ptr.?); + } + + pub fn tiledThrValLayoutA(self: *const Self) *const LayoutType { + return @ptrCast(c.mlirFlyTiledMmaTypeGetTiledThrValLayoutA(self.ptr()).ptr.?); + } + + pub fn tiledThrValLayoutB(self: *const Self) *const LayoutType { + return @ptrCast(c.mlirFlyTiledMmaTypeGetTiledThrValLayoutB(self.ptr()).ptr.?); + } + + pub fn tiledThrValLayoutC(self: *const Self) *const LayoutType { + return @ptrCast(c.mlirFlyTiledMmaTypeGetTiledThrValLayoutC(self.ptr()).ptr.?); + } +}; + +/// `!fly.universal_copy`. +pub const CopyOpUniversalCopyType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsAFlyCopyOpUniversalCopy; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + + pub const InitArgs = struct { + bitSize: i32, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirFlyCopyOpUniversalCopyTypeGet(ctx.ptr(), args.bitSize); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; diff --git a/mlir/dialects/fly_rocdl/BUILD.bazel b/mlir/dialects/fly_rocdl/BUILD.bazel new file mode 100644 index 00000000..334e520b --- /dev/null +++ b/mlir/dialects/fly_rocdl/BUILD.bazel @@ -0,0 +1,43 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +cc_library( + name = "capi", + srcs = [ + "capi.cc", + ], + hdrs = [ + "capi.h", + ], + tags = ["manual"], + deps = [ + "@flydsl//:FlyROCDLDialect", + "@llvm-project//mlir:CAPIIR", + ], +) + +zig_library( + name = "fly_rocdl", + srcs = [ + "ops.zig", + ], + import_name = "mlir/dialects/fly_rocdl", + main = "fly_rocdl.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + ":capi", + "//mlir", + "//mlir/dialects/fly", + "//stdx", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":fly_rocdl", + "//mlir", + ], +) diff --git a/mlir/dialects/fly_rocdl/capi.cc b/mlir/dialects/fly_rocdl/capi.cc new file mode 100644 index 00000000..b055e993 --- /dev/null +++ b/mlir/dialects/fly_rocdl/capi.cc @@ -0,0 +1,107 @@ +//===- capi.cc - C API for the fly_rocdl dialect --------------------------===// + +#include "capi.h" + +#include "flydsl/Dialect/FlyROCDL/IR/Dialect.h" + +#include "mlir/CAPI/IR.h" +#include "mlir/CAPI/Registration.h" + +MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(FlyROCDL, fly_rocdl, + mlir::fly_rocdl::FlyROCDLDialect) + +static mlir::MLIRContext *loadFlyROCDL(MlirContext context) { + mlir::MLIRContext *ctx = unwrap(context); + if (ctx) + ctx->getOrLoadDialect(); + return ctx; +} + +/// Null test for unwrapped handles, kept in one place as in fly's C API, where +/// `!attr` is value arithmetic on a fly IntAttr rather than a null test. +template static bool isNull(T value) { + return !static_cast(value); +} + +template +static MlirType mmaOpGet(MlirContext context, int32_t m, int32_t n, int32_t k, + MlirType elemTyA, MlirType elemTyB, MlirType elemTyAcc, + Flags... flags) { + mlir::MLIRContext *ctx = loadFlyROCDL(context); + mlir::Type a = unwrap(elemTyA), b = unwrap(elemTyB), acc = unwrap(elemTyAcc); + if (!ctx || isNull(a) || isNull(b) || isNull(acc)) + return {nullptr}; + if (a.getContext() != ctx || b.getContext() != ctx || acc.getContext() != ctx) + return {nullptr}; + return wrap( + static_cast(Type::get(ctx, m, n, k, a, b, acc, flags...))); +} + +#define FLY_ROCDL_TYPE_ISA(Name, Type) \ + bool mlirTypeIsAFlyROCDL##Name(MlirType type) { \ + return mlir::isa_and_nonnull(unwrap(type)); \ + } + +FLY_ROCDL_TYPE_ISA(CopyOpCDNA3BufferCopy, + mlir::fly_rocdl::CopyOpCDNA3BufferCopyType) +FLY_ROCDL_TYPE_ISA(CopyOpCDNA3BufferCopyLDS, + mlir::fly_rocdl::CopyOpCDNA3BufferCopyLDSType) +FLY_ROCDL_TYPE_ISA(MmaOpCDNA3MFMA, mlir::fly_rocdl::MmaOpCDNA3_MFMAType) +FLY_ROCDL_TYPE_ISA(MmaOpGFX11WMMA, mlir::fly_rocdl::MmaOpGFX11_WMMAType) +FLY_ROCDL_TYPE_ISA(MmaOpGFX120XWMMA, mlir::fly_rocdl::MmaOpGFX120X_WMMAType) + +#undef FLY_ROCDL_TYPE_ISA + +MlirAttribute mlirFlyROCDLBufferDescAddressAttrGet(MlirContext context) { + mlir::MLIRContext *ctx = loadFlyROCDL(context); + if (!ctx) + return {nullptr}; + return wrap(static_cast( + mlir::fly_rocdl::BufferDescAddressAttr::get(ctx))); +} + +MlirType mlirFlyROCDLCopyOpCDNA3BufferCopyTypeGet(MlirContext context, + int32_t bitSize, + int32_t cacheModifier) { + mlir::MLIRContext *ctx = loadFlyROCDL(context); + if (!ctx) + return {nullptr}; + return wrap( + static_cast(mlir::fly_rocdl::CopyOpCDNA3BufferCopyType::get( + ctx, bitSize, cacheModifier))); +} + +MlirType mlirFlyROCDLCopyOpCDNA3BufferCopyLDSTypeGet(MlirContext context, + int32_t bitSize) { + mlir::MLIRContext *ctx = loadFlyROCDL(context); + if (!ctx) + return {nullptr}; + return wrap(static_cast( + mlir::fly_rocdl::CopyOpCDNA3BufferCopyLDSType::get(ctx, bitSize))); +} + +MlirType mlirFlyROCDLMmaOpCDNA3MFMATypeGet(MlirContext context, int32_t m, + int32_t n, int32_t k, + MlirType elemTyA, MlirType elemTyB, + MlirType elemTyAcc) { + return mmaOpGet( + context, m, n, k, elemTyA, elemTyB, elemTyAcc); +} + +MlirType mlirFlyROCDLMmaOpGFX11WMMATypeGet(MlirContext context, int32_t m, + int32_t n, int32_t k, + MlirType elemTyA, MlirType elemTyB, + MlirType elemTyAcc, bool signA, + bool signB, bool clamp) { + return mmaOpGet( + context, m, n, k, elemTyA, elemTyB, elemTyAcc, signA, signB, clamp); +} + +MlirType mlirFlyROCDLMmaOpGFX120XWMMATypeGet(MlirContext context, int32_t m, + int32_t n, int32_t k, + MlirType elemTyA, MlirType elemTyB, + MlirType elemTyAcc, bool signA, + bool signB, bool clamp) { + return mmaOpGet( + context, m, n, k, elemTyA, elemTyB, elemTyAcc, signA, signB, clamp); +} diff --git a/mlir/dialects/fly_rocdl/capi.h b/mlir/dialects/fly_rocdl/capi.h new file mode 100644 index 00000000..0ee248bb --- /dev/null +++ b/mlir/dialects/fly_rocdl/capi.h @@ -0,0 +1,45 @@ +//===- capi.h - C API for the fly_rocdl dialect -------------------*- C -*-===// + +#ifndef MLIR_DIALECTS_FLY_ROCDL_CAPI_H +#define MLIR_DIALECTS_FLY_ROCDL_CAPI_H + +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(FlyROCDL, fly_rocdl); + +MLIR_CAPI_EXPORTED MlirAttribute +mlirFlyROCDLBufferDescAddressAttrGet(MlirContext context); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyROCDLCopyOpCDNA3BufferCopy(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyROCDLCopyOpCDNA3BufferCopyTypeGet( + MlirContext context, int32_t bitSize, int32_t cacheModifier); + +MLIR_CAPI_EXPORTED bool +mlirTypeIsAFlyROCDLCopyOpCDNA3BufferCopyLDS(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyROCDLCopyOpCDNA3BufferCopyLDSTypeGet( + MlirContext context, int32_t bitSize); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyROCDLMmaOpCDNA3MFMA(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyROCDLMmaOpCDNA3MFMATypeGet( + MlirContext context, int32_t m, int32_t n, int32_t k, MlirType elemTyA, + MlirType elemTyB, MlirType elemTyAcc); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyROCDLMmaOpGFX11WMMA(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyROCDLMmaOpGFX11WMMATypeGet( + MlirContext context, int32_t m, int32_t n, int32_t k, MlirType elemTyA, + MlirType elemTyB, MlirType elemTyAcc, bool signA, bool signB, bool clamp); + +MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyROCDLMmaOpGFX120XWMMA(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirFlyROCDLMmaOpGFX120XWMMATypeGet( + MlirContext context, int32_t m, int32_t n, int32_t k, MlirType elemTyA, + MlirType elemTyB, MlirType elemTyAcc, bool signA, bool signB, bool clamp); + +#ifdef __cplusplus +} +#endif + +#endif // MLIR_DIALECTS_FLY_ROCDL_CAPI_H diff --git a/mlir/dialects/fly_rocdl/fly_rocdl.zig b/mlir/dialects/fly_rocdl/fly_rocdl.zig new file mode 100644 index 00000000..7a91a925 --- /dev/null +++ b/mlir/dialects/fly_rocdl/fly_rocdl.zig @@ -0,0 +1,156 @@ +const c = @import("c"); +const mlir = @import("mlir"); + +pub const ops = @import("ops.zig"); + +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("fly_rocdl"); +} + +/// `#fly_rocdl.buffer_desc`, an address space for buffer-descriptor pointers. +pub const BufferDescAddressAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + + pub fn get(ctx: *mlir.Context) mlir.Error!*const Self { + return @ptrCast(c.mlirFlyROCDLBufferDescAddressAttrGet(ctx.ptr()).ptr orelse return error.InvalidMlir); + } +}; + +/// `!fly_rocdl.cdna3.buffer_copy`. +pub const CopyOpCDNA3BufferCopyType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsAFlyROCDLCopyOpCDNA3BufferCopy; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + + pub const InitArgs = struct { + bitSize: i32, + /// 0 = cached, 2 = non-temporal. + cacheModifier: i32 = 0, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirFlyROCDLCopyOpCDNA3BufferCopyTypeGet(ctx.ptr(), args.bitSize, args.cacheModifier); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!fly_rocdl.cdna3.buffer_copy_lds`. +pub const CopyOpCDNA3BufferCopyLDSType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsAFlyROCDLCopyOpCDNA3BufferCopyLDS; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + + pub const InitArgs = struct { + bitSize: i32, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirFlyROCDLCopyOpCDNA3BufferCopyLDSTypeGet(ctx.ptr(), args.bitSize); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!fly_rocdl.cdna3.mfma`. +pub const MmaOpCDNA3MFMAType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsAFlyROCDLMmaOpCDNA3MFMA; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + + pub const InitArgs = struct { + m: i32, + n: i32, + k: i32, + elemTyA: *const mlir.Type, + elemTyB: *const mlir.Type, + elemTyAcc: *const mlir.Type, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirFlyROCDLMmaOpCDNA3MFMATypeGet(ctx.ptr(), args.m, args.n, args.k, args.elemTyA.ptr(), args.elemTyB.ptr(), args.elemTyAcc.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!fly_rocdl.gfx11.wmma`. +pub const MmaOpGFX11WMMAType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsAFlyROCDLMmaOpGFX11WMMA; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + + pub const InitArgs = WmmaInitArgs; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirFlyROCDLMmaOpGFX11WMMATypeGet(ctx.ptr(), args.m, args.n, args.k, args.elemTyA.ptr(), args.elemTyB.ptr(), args.elemTyAcc.ptr(), args.signA, args.signB, args.clamp); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!fly_rocdl.gfx120x.wmma`. +pub const MmaOpGFX120XWMMAType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsAFlyROCDLMmaOpGFX120XWMMA; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + + pub const InitArgs = WmmaInitArgs; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirFlyROCDLMmaOpGFX120XWMMATypeGet(ctx.ptr(), args.m, args.n, args.k, args.elemTyA.ptr(), args.elemTyB.ptr(), args.elemTyAcc.ptr(), args.signA, args.signB, args.clamp); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `signA`/`signB`/`clamp` drive the integer WMMA intrinsics and must stay +/// false on the float paths. +pub const WmmaInitArgs = struct { + m: i32, + n: i32, + k: i32, + elemTyA: *const mlir.Type, + elemTyB: *const mlir.Type, + elemTyAcc: *const mlir.Type, + signA: bool = false, + signB: bool = false, + clamp: bool = false, +}; diff --git a/mlir/dialects/fly_rocdl/ops.zig b/mlir/dialects/fly_rocdl/ops.zig new file mode 100644 index 00000000..6cff2bf6 --- /dev/null +++ b/mlir/dialects/fly_rocdl/ops.zig @@ -0,0 +1,83 @@ +//! `fly_rocdl` operation builders, generated from FlyDSL's `include/flydsl/Dialect/FlyROCDL/IR/Ops.td` at +//! 1941889400621f1fc3d8f03bada1a4e7380cdf7e; do not edit, see ../fly/REGENERATING.md. +//! +//! Builders take the operands, then the result types (only for operations +//! that do not infer them), then the required attributes, then an `opts` +//! struct of optional operands and attributes (pass `.{}` to omit them), then +//! the location. Parameters are the ODS names in snake case, with a trailing +//! underscore where needed. Pass `&.{}` for empty variadic operands; +//! variable-length operand groups of +//! `AttrSizedOperandSegments` operations get their segment sizes from +//! `Operation.make`. Attributes use `mlir.Attribute`. + +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +const make = @import("mlir/dialects/fly").make; + +/// `fly_rocdl.get_buffer_rsrc`. Extract the raw ROCDL buffer resource from a buffer-descriptor pointer. Result types are inferred. +pub fn get_buffer_rsrc(ctx: *mlir.Context, ptr: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return make(ctx, "fly_rocdl.get_buffer_rsrc", .{ + .operands = .{ .flat = &.{ptr} }, + .result_type_inference = true, + .location = location, + }); +} + +/// `fly_rocdl.make_tiled_tdm_load_atom`. Build a CDNA5 TDM load atom and the coordinate tensor that addresses it. Result types are inferred. +pub fn make_tiled_tdm_load_atom( + ctx: *mlir.Context, + tensor: *const mlir.Value, + smem_layout: *const mlir.Value, + tiler: *const mlir.Value, + opts: struct { + num_warps: ?*const mlir.Attribute = null, + init_boundary_check: ?*const mlir.Attribute = null, + cache_modifier: ?*const mlir.Attribute = null, + atomic_barrier: ?*const mlir.Attribute = null, + internal_type: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + if (opts.num_warps) |present| attributes.appendAssumeCapacity(.named(ctx, "numWarps", present)); + if (opts.init_boundary_check) |present| attributes.appendAssumeCapacity(.named(ctx, "initBoundaryCheck", present)); + if (opts.cache_modifier) |present| attributes.appendAssumeCapacity(.named(ctx, "cacheModifier", present)); + if (opts.atomic_barrier) |present| attributes.appendAssumeCapacity(.named(ctx, "atomicBarrier", present)); + if (opts.internal_type) |present| attributes.appendAssumeCapacity(.named(ctx, "internalType", present)); + return make(ctx, "fly_rocdl.make_tiled_tdm_load_atom", .{ + .operands = .{ .flat = &.{ tensor, smem_layout, tiler } }, + .attributes = attributes.constSlice(), + .result_type_inference = true, + .location = location, + }); +} + +/// `fly_rocdl.make_tiled_tdm_store_atom`. Build a CDNA5 TDM store atom and the coordinate tensor that addresses it. Result types are inferred. +pub fn make_tiled_tdm_store_atom( + ctx: *mlir.Context, + tensor: *const mlir.Value, + smem_layout: *const mlir.Value, + tiler: *const mlir.Value, + opts: struct { + num_warps: ?*const mlir.Attribute = null, + init_boundary_check: ?*const mlir.Attribute = null, + cache_modifier: ?*const mlir.Attribute = null, + atomic_barrier: ?*const mlir.Attribute = null, + internal_type: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + if (opts.num_warps) |present| attributes.appendAssumeCapacity(.named(ctx, "numWarps", present)); + if (opts.init_boundary_check) |present| attributes.appendAssumeCapacity(.named(ctx, "initBoundaryCheck", present)); + if (opts.cache_modifier) |present| attributes.appendAssumeCapacity(.named(ctx, "cacheModifier", present)); + if (opts.atomic_barrier) |present| attributes.appendAssumeCapacity(.named(ctx, "atomicBarrier", present)); + if (opts.internal_type) |present| attributes.appendAssumeCapacity(.named(ctx, "internalType", present)); + return make(ctx, "fly_rocdl.make_tiled_tdm_store_atom", .{ + .operands = .{ .flat = &.{ tensor, smem_layout, tiler } }, + .attributes = attributes.constSlice(), + .result_type_inference = true, + .location = location, + }); +} diff --git a/mlir/dialects/fly_rocdl/test.zig b/mlir/dialects/fly_rocdl/test.zig new file mode 100644 index 00000000..bb85d6e5 --- /dev/null +++ b/mlir/dialects/fly_rocdl/test.zig @@ -0,0 +1,30 @@ +const std = @import("std"); + +const fly_rocdl = @import("mlir/dialects/fly_rocdl"); +const mlir = @import("mlir"); + +const operation_names: []const []const u8 = &.{ + "fly_rocdl.get_buffer_rsrc", + "fly_rocdl.make_tiled_tdm_load_atom", + "fly_rocdl.make_tiled_tdm_store_atom", +}; + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(fly_rocdl); + inline for (comptime std.meta.declarations(fly_rocdl)) |decl| { + const T = @field(fly_rocdl, decl); + if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); + } +} + +test "every bound operation is registered" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + fly_rocdl.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} diff --git a/mlir/dialects/func.zig b/mlir/dialects/func.zig deleted file mode 100644 index 96ba1d49..00000000 --- a/mlir/dialects/func.zig +++ /dev/null @@ -1,51 +0,0 @@ -const std = @import("std"); -const mlir = @import("mlir"); - -pub fn func( - ctx: mlir.Context, - args: struct { - sym_name: [:0]const u8, - args: []const mlir.Type, - arg_attrs: []const mlir.Attribute = &.{}, - results: []const mlir.Type, - res_attrs: []const mlir.Attribute = &.{}, - block: mlir.Block, - location: mlir.Location, - }, -) mlir.Operation { - const AttrTuple = struct { [:0]const u8, mlir.Attribute }; - var attrs_tuple_buffer = std.BoundedArray(AttrTuple, 4){}; - attrs_tuple_buffer.appendAssumeCapacity(.{ "sym_name", mlir.StringAttribute.init(ctx, args.sym_name).as(mlir.Attribute).? }); - attrs_tuple_buffer.appendAssumeCapacity(.{ "function_type", mlir.TypeAttribute.init((mlir.FunctionType.init(ctx, args.args, args.results) catch unreachable).as(mlir.Type).?).as(mlir.Attribute).? }); - if (args.arg_attrs.len > 0) { - attrs_tuple_buffer.appendAssumeCapacity(.{ "arg_attrs", mlir.ArrayAttribute.init(ctx, args.arg_attrs).as(mlir.Attribute).? }); - } - - if (args.res_attrs.len > 0) { - attrs_tuple_buffer.appendAssumeCapacity(.{ "res_attrs", mlir.ArrayAttribute.init(ctx, args.res_attrs).as(mlir.Attribute).? }); - } - - return mlir.Operation.make(ctx, "func.func", .{ - .blocks = &.{args.block}, - .attributes = attrs_tuple_buffer.constSlice(), - .location = args.location, - }); -} - -pub fn call(ctx: mlir.Context, name: [:0]const u8, values: []const mlir.Value, results: []const mlir.Type, loc: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "func.call", .{ - .variadic_operands = &.{values}, - .results = results, - .verify = true, - .attributes = &.{.{ "callee", mlir.FlatSymbolRefAttribute.init(ctx, name).as(mlir.Attribute).? }}, - .location = loc, - }); -} - -pub fn return_(ctx: mlir.Context, values: []const mlir.Value, loc: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "func.return", .{ - .operands = values, - .verify = false, - .location = loc, - }); -} diff --git a/mlir/dialects/func/BUILD.bazel b/mlir/dialects/func/BUILD.bazel new file mode 100644 index 00000000..4524f2dd --- /dev/null +++ b/mlir/dialects/func/BUILD.bazel @@ -0,0 +1,35 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +cc_library( + name = "capi", + srcs = ["capi.cc"], + hdrs = ["capi.h"], + tags = ["manual"], + deps = [ + "@llvm-project//mlir:CAPIIR", + "@llvm-project//mlir:FuncExtensions", + ], +) + +zig_library( + name = "func", + import_name = "mlir/dialects/func", + main = "func.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + ":capi", + "//mlir", + "//stdx", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":func", + "//mlir", + ], +) diff --git a/mlir/dialects/func/capi.cc b/mlir/dialects/func/capi.cc new file mode 100644 index 00000000..11a5e006 --- /dev/null +++ b/mlir/dialects/func/capi.cc @@ -0,0 +1,10 @@ +//===- capi.cc - C API for the func dialect -------------------------------===// + +#include "capi.h" + +#include "mlir/CAPI/IR.h" +#include "mlir/Dialect/Func/Extensions/AllExtensions.h" + +void mlirRegisterFuncExtensions(MlirDialectRegistry registry) { + mlir::func::registerAllExtensions(*unwrap(registry)); +} diff --git a/mlir/dialects/func/capi.h b/mlir/dialects/func/capi.h new file mode 100644 index 00000000..cd522b2d --- /dev/null +++ b/mlir/dialects/func/capi.h @@ -0,0 +1,20 @@ +//===- capi.h - C API for the func dialect ------------------------*- C -*-===// + +#ifndef MLIR_DIALECTS_FUNC_CAPI_H +#define MLIR_DIALECTS_FUNC_CAPI_H + +#include "mlir-c/Dialect/Func.h" +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +MLIR_CAPI_EXPORTED void +mlirRegisterFuncExtensions(MlirDialectRegistry registry); + +#ifdef __cplusplus +} +#endif + +#endif // MLIR_DIALECTS_FUNC_CAPI_H diff --git a/mlir/dialects/func/func.zig b/mlir/dialects/func/func.zig new file mode 100644 index 00000000..45c74674 --- /dev/null +++ b/mlir/dialects/func/func.zig @@ -0,0 +1,109 @@ +const c = @import("c"); + +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("func"); +} + +const FuncOpArgs = struct { + pub const Visibility = enum { + public, + private, + }; + + name: []const u8, + args: ?[]const *const mlir.Type = null, + args_attributes: ?[]const *const mlir.Attribute = null, + results: ?[]const *const mlir.Type = null, + results_attributes: ?[]const *const mlir.Attribute = null, + block: *mlir.Block, + location: *const mlir.Location, + no_inline: bool = false, + visibility: ?Visibility = null, + verify: bool = true, + extra_attributes: []const mlir.NamedAttribute = &.{}, +}; + +pub fn func(ctx: *mlir.Context, args: FuncOpArgs) *mlir.Operation { + var args_buffer: stdx.BoundedArray(*const mlir.Type, 1024) = .empty; + var results_buffer: stdx.BoundedArray(*const mlir.Type, 64) = .empty; + + var attr_tuples_buffer: stdx.BoundedArray(mlir.NamedAttribute, 16) = .empty; + attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "sym_name", .string(ctx, args.name))); + if (args.visibility) |v| { + attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "sym_visibility", .string(ctx, @tagName(v)))); + } + attr_tuples_buffer.appendSliceAssumeCapacity(&.{ + .named(ctx, "function_type", .typeAttr(.function( + ctx, + args.args orelse args: { + for (0..args.block.numArguments()) |i| { + args_buffer.appendAssumeCapacity(args.block.argument(i).type_()); + } + break :args args_buffer.constSlice(); + }, + args.results orelse results: { + const terminator = args.block.terminator() orelse { + @panic("block has no terminator"); + }; + for (0..terminator.numOperands()) |i| { + results_buffer.appendAssumeCapacity(terminator.operand(i).type_()); + } + break :results results_buffer.constSlice(); + }, + ))), + }); + + if (args.args_attributes) |args_attributes| { + attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "arg_attrs", .array(ctx, args_attributes))); + } + if (args.results_attributes) |results_attributes| { + attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "res_attrs", .array(ctx, results_attributes))); + } + if (args.no_inline) { + attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "no_inline", .unit(ctx))); + } + for (args.extra_attributes) |a| { + attr_tuples_buffer.appendAssumeCapacity(a); + } + + return mlir.Operation.make(ctx, "func.func", .{ + .blocks = &.{args.block}, + .attributes = attr_tuples_buffer.constSlice(), + .location = args.location, + .verify = args.verify, + }); +} + +pub fn call(ctx: *mlir.Context, name: []const u8, values: []const *const mlir.Value, results: []const *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "func.call", .{ + .operands = .{ .variadic = &.{values} }, + .results = .{ .flat = results }, + .attributes = &.{ + .named(ctx, "callee", .flatSymbolRef(ctx, name)), + }, + .location = location, + }); +} + +pub fn return_(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "func.return", .{ + .operands = .{ .flat = &.{value} }, + .location = location, + .verify = false, + }); +} + +pub fn returns(ctx: *mlir.Context, values: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "func.return", .{ + .operands = .{ .flat = values }, + .verify = false, + .location = location, + }); +} + +pub fn registerExtensions(registry: *mlir.DialectRegistry) void { + c.mlirRegisterFuncExtensions(registry.ptr()); +} diff --git a/mlir/dialects/func/test.zig b/mlir/dialects/func/test.zig new file mode 100644 index 00000000..7a3b17f9 --- /dev/null +++ b/mlir/dialects/func/test.zig @@ -0,0 +1,41 @@ +const std = @import("std"); + +const func = @import("mlir/dialects/func"); +const mlir = @import("mlir"); + +const operation_names: []const []const u8 = &.{ + "func.call", + "func.func", + "func.return", +}; + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(func); + inline for (comptime std.meta.declarations(func)) |decl| { + const T = @field(func, decl); + if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); + } +} + +test "func dialect and extensions register" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + func.registerDialect(registry); + func.registerExtensions(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + try std.testing.expect(ctx.isRegisteredOperation("func.func")); +} + +test "every bound operation is registered" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + func.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} diff --git a/mlir/dialects/gpu/BUILD.bazel b/mlir/dialects/gpu/BUILD.bazel new file mode 100644 index 00000000..7264487a --- /dev/null +++ b/mlir/dialects/gpu/BUILD.bazel @@ -0,0 +1,35 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +cc_library( + name = "capi", + srcs = ["capi.cc"], + hdrs = ["capi.h"], + tags = ["manual"], + deps = [ + "@llvm-project//mlir:CAPIIR", + "@llvm-project//mlir:GPUDialect", + ], +) + +zig_library( + name = "gpu", + import_name = "mlir/dialects/gpu", + main = "gpu.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + ":capi", + "//mlir", + "//stdx", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":gpu", + "//mlir", + ], +) diff --git a/mlir/dialects/gpu/capi.cc b/mlir/dialects/gpu/capi.cc new file mode 100644 index 00000000..2e215d58 --- /dev/null +++ b/mlir/dialects/gpu/capi.cc @@ -0,0 +1,52 @@ +//===- capi.cc - C API for the gpu dialect --------------------------------===// + +#include "capi.h" + +#include "mlir/CAPI/IR.h" +#include "mlir/CAPI/Registration.h" +#include "mlir/Dialect/GPU/IR/GPUDialect.h" + +MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(GPU, gpu, mlir::gpu::GPUDialect) + +static mlir::MLIRContext *loadGPU(MlirContext context) { + mlir::MLIRContext *ctx = unwrap(context); + if (ctx) + ctx->getOrLoadDialect(); + return ctx; +} + +bool mlirAttributeIsAGPUDimension(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirGPUDimensionAttrGet(MlirContext context, uint32_t value) { + mlir::MLIRContext *ctx = loadGPU(context); + std::optional dimension = + mlir::gpu::symbolizeDimension(value); + if (!ctx || !dimension) + return {nullptr}; + return wrap(mlir::gpu::DimensionAttr::get(ctx, *dimension)); +} + +uint32_t mlirGPUDimensionAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsAGPUShuffleMode(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirGPUShuffleModeAttrGet(MlirContext context, uint32_t value) { + mlir::MLIRContext *ctx = loadGPU(context); + std::optional mode = + mlir::gpu::symbolizeShuffleMode(value); + if (!ctx || !mode) + return {nullptr}; + return wrap(mlir::gpu::ShuffleModeAttr::get(ctx, *mode)); +} + +uint32_t mlirGPUShuffleModeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} diff --git a/mlir/dialects/gpu/capi.h b/mlir/dialects/gpu/capi.h new file mode 100644 index 00000000..670e23a9 --- /dev/null +++ b/mlir/dialects/gpu/capi.h @@ -0,0 +1,37 @@ +//===- capi.h - C API for the gpu dialect -------------------------*- C -*-===// +// +// C entry points for the `gpu` dialect that upstream's C API leaves out: its +// registration (outside `CAPIGPU`, which also brings the GPU passes) and the +// enum attributes kernels build. + +#ifndef MLIR_DIALECTS_GPU_CAPI_H +#define MLIR_DIALECTS_GPU_CAPI_H + +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(GPU, gpu); + +// Enum attributes take and return the C++ enum's integer value; `Get` returns +// a null attribute for a value outside the enum. + +// #gpu +MLIR_CAPI_EXPORTED bool mlirAttributeIsAGPUDimension(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirGPUDimensionAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirGPUDimensionAttrGetValue(MlirAttribute attr); + +// #gpu +MLIR_CAPI_EXPORTED bool mlirAttributeIsAGPUShuffleMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirGPUShuffleModeAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirGPUShuffleModeAttrGetValue(MlirAttribute attr); + +#ifdef __cplusplus +} +#endif + +#endif // MLIR_DIALECTS_GPU_CAPI_H diff --git a/mlir/dialects/gpu/gpu.zig b/mlir/dialects/gpu/gpu.zig new file mode 100644 index 00000000..4b9df47f --- /dev/null +++ b/mlir/dialects/gpu/gpu.zig @@ -0,0 +1,169 @@ +const c = @import("c"); +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("gpu"); +} + +/// A dimension, either `x`, `y` or `z`. +pub const Dim = enum(u32) { + x = 0, + y = 1, + z = 2, +}; + +/// Indexing modes supported by `gpu.shuffle`. +pub const ShuffleMode = enum(u32) { + xor = 0, + down = 1, + up = 2, + idx = 3, +}; + +/// `#gpu`. +pub const DimensionAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsAGPUDimension; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "dim"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: Dim, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirGPUDimensionAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) Dim { + return @fromBackingInt(@intCast(c.mlirGPUDimensionAttrGetValue(self.ptr()))); + } +}; + +/// `#gpu`. +pub const ShuffleModeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsAGPUShuffleMode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "shuffle_mode"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: ShuffleMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirGPUShuffleModeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) ShuffleMode { + return @fromBackingInt(@intCast(c.mlirGPUShuffleModeAttrGetValue(self.ptr()))); + } +}; + +/// `gpu.container_module`: marks a module whose `gpu.module`s hold the kernels +/// it launches. +pub fn containerModule(ctx: *mlir.Context) mlir.NamedAttribute { + return .named(ctx, "gpu.container_module", .unit(ctx)); +} + +/// `gpu.kernel`: marks a function as a kernel entry point. +pub fn kernel(ctx: *mlir.Context) mlir.NamedAttribute { + return .named(ctx, "gpu.kernel", .unit(ctx)); +} + +pub fn module(ctx: *mlir.Context, name: []const u8, body: *mlir.Block, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "gpu.module", .{ + .blocks = &.{body}, + .attributes = &.{ + .named(ctx, "sym_name", .string(ctx, name)), + }, + .location = location, + .verify = false, + }); +} + +pub const FuncArgs = struct { + name: []const u8, + args: []const *const mlir.Type, + block: *mlir.Block, + location: *const mlir.Location, + kernel: bool = true, + extra_attributes: []const mlir.NamedAttribute = &.{}, +}; + +/// `gpu.func @name(args) kernel { ... gpu.return }`. +pub fn func(ctx: *mlir.Context, args: FuncArgs) *mlir.Operation { + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 16) = .empty; + attrs.appendSliceAssumeCapacity(&.{ + .named(ctx, "sym_name", .string(ctx, args.name)), + .named(ctx, "function_type", .typeAttr(.function(ctx, args.args, &.{}))), + }); + if (args.kernel) attrs.appendAssumeCapacity(kernel(ctx)); + attrs.appendSliceAssumeCapacity(args.extra_attributes); + return mlir.Operation.make(ctx, "gpu.func", .{ + .blocks = &.{args.block}, + .attributes = attrs.constSlice(), + .location = args.location, + .verify = false, + }); +} + +pub fn return_(ctx: *mlir.Context, values: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "gpu.return", .{ + .operands = .{ .flat = values }, + .location = location, + .verify = false, + }); +} + +fn idOp(ctx: *mlir.Context, comptime name: []const u8, dim: Dim, location: *const mlir.Location) *mlir.Operation { + const dimension = DimensionAttr.get(ctx, .{ .value = dim }) catch unreachable; + return mlir.Operation.make(ctx, name, .{ + .results = .{ .flat = &.{.index(ctx)} }, + .attributes = &.{.named(ctx, "dimension", dimension.attribute())}, + .location = location, + }); +} + +pub fn thread_id(ctx: *mlir.Context, dim: Dim, location: *const mlir.Location) *mlir.Operation { + return idOp(ctx, "gpu.thread_id", dim, location); +} + +pub fn block_id(ctx: *mlir.Context, dim: Dim, location: *const mlir.Location) *mlir.Operation { + return idOp(ctx, "gpu.block_id", dim, location); +} + +pub fn block_dim(ctx: *mlir.Context, dim: Dim, location: *const mlir.Location) *mlir.Operation { + return idOp(ctx, "gpu.block_dim", dim, location); +} + +pub fn grid_dim(ctx: *mlir.Context, dim: Dim, location: *const mlir.Location) *mlir.Operation { + return idOp(ctx, "gpu.grid_dim", dim, location); +} + +/// `gpu.shuffle`: `value` from lane `offset` (per `mode`) within `width` lanes, +/// and whether that lane was valid (i1). +pub fn shuffle(ctx: *mlir.Context, value: *const mlir.Value, offset: *const mlir.Value, width: *const mlir.Value, mode: ShuffleMode, location: *const mlir.Location) *mlir.Operation { + const mode_attr = ShuffleModeAttr.get(ctx, .{ .value = mode }) catch unreachable; + return mlir.Operation.make(ctx, "gpu.shuffle", .{ + .operands = .{ .flat = &.{ value, offset, width } }, + .results = .{ .flat = &.{ value.type_(), .int(ctx, .i1) } }, + .attributes = &.{.named(ctx, "mode", mode_attr.attribute())}, + .location = location, + }); +} + +pub fn barrier(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "gpu.barrier", .{ .location = location }); +} diff --git a/mlir/dialects/gpu/test.zig b/mlir/dialects/gpu/test.zig new file mode 100644 index 00000000..d0f4a80d --- /dev/null +++ b/mlir/dialects/gpu/test.zig @@ -0,0 +1,71 @@ +const std = @import("std"); + +const gpu = @import("mlir/dialects/gpu"); +const mlir = @import("mlir"); + +const operation_names: []const []const u8 = &.{ + "gpu.barrier", + "gpu.block_dim", + "gpu.block_id", + "gpu.func", + "gpu.grid_dim", + "gpu.module", + "gpu.return", + "gpu.shuffle", + "gpu.thread_id", +}; + +fn expectPrints(expected: []const u8, value: anytype) !void { + var buf: [128]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print("{f}", .{value}); + try std.testing.expectEqualStrings(expected, w.buffered()); +} + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(gpu); + inline for (comptime std.meta.declarations(gpu)) |decl| { + const T = @field(gpu, decl); + if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); + } +} + +test "gpu enum attributes are built through the C API" { + const ctx = try mlir.Context.init(.{}); + defer ctx.deinit(); + + inline for (@typeInfo(gpu.Dim).@"enum".field_names) |name| { + const value: gpu.Dim = @field(gpu.Dim, name); + const attr = try gpu.DimensionAttr.get(ctx, .{ .value = value }); + try std.testing.expectEqual(value, attr.value()); + try std.testing.expect(attr.attribute().isA(gpu.DimensionAttr) != null); + try expectPrints("#gpu.dim<" ++ name ++ ">", attr); + } + inline for (@typeInfo(gpu.ShuffleMode).@"enum".field_names) |name| { + const value: gpu.ShuffleMode = @field(gpu.ShuffleMode, name); + const attr = try gpu.ShuffleModeAttr.get(ctx, .{ .value = value }); + try std.testing.expectEqual(value, attr.value()); + try expectPrints("#gpu.shuffle_mode<" ++ name ++ ">", attr); + } +} + +test "container_module and kernel are unit attributes of the gpu dialect" { + const ctx = try mlir.Context.init(.{}); + defer ctx.deinit(); + var buf: [128]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print("{f}", .{mlir.Attribute.dict(ctx, &.{ gpu.containerModule(ctx), gpu.kernel(ctx) })}); + try std.testing.expectEqualStrings("{gpu.container_module, gpu.kernel}", w.buffered()); +} + +test "every bound operation is registered" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + gpu.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} diff --git a/mlir/dialects/llvm/BUILD.bazel b/mlir/dialects/llvm/BUILD.bazel new file mode 100644 index 00000000..a03d215e --- /dev/null +++ b/mlir/dialects/llvm/BUILD.bazel @@ -0,0 +1,38 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +cc_library( + name = "capi", + srcs = ["capi.cc"], + hdrs = ["capi.h"], + tags = ["manual"], + deps = [ + "@llvm-project//mlir:CAPIIR", + "@llvm-project//mlir:CAPILLVM", + "@llvm-project//mlir:LLVMDialect", + ], +) + +zig_library( + name = "llvm", + srcs = ["ops.zig"], + import_name = "mlir/dialects/llvm", + main = "llvm.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + ":capi", + "//mlir", + "//stdx", + "@llvm-project//mlir:CAPILLVM", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":llvm", + "//mlir", + ], +) diff --git a/mlir/dialects/llvm/capi.cc b/mlir/dialects/llvm/capi.cc new file mode 100644 index 00000000..85b480e3 --- /dev/null +++ b/mlir/dialects/llvm/capi.cc @@ -0,0 +1,55 @@ +//===- capi.cc - C API for the llvm dialect -------------------------------===// + +#include "capi.h" + +#include "mlir/CAPI/IR.h" +#include "mlir/Dialect/LLVMIR/LLVMAttrs.h" +#include "mlir/Dialect/LLVMIR/LLVMDialect.h" + +using namespace mlir; + +static MLIRContext *loadLLVM(MlirContext context) { + MLIRContext *ctx = unwrap(context); + if (ctx) + ctx->getOrLoadDialect(); + return ctx; +} + +template static T optional(MlirAttribute attr) { + return llvm::dyn_cast_or_null(unwrap(attr)); +} + +bool mlirAttributeIsALLVMLoopUnroll(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirLLVMLoopUnrollAttrGet(MlirContext context, + MlirAttribute disable, + MlirAttribute count, + MlirAttribute runtimeDisable, + MlirAttribute full) { + MLIRContext *ctx = loadLLVM(context); + if (!ctx) + return {nullptr}; + return wrap(LLVM::LoopUnrollAttr::get(ctx, optional(disable), + optional(count), + optional(runtimeDisable), + optional(full), {}, {}, {})); +} + +bool mlirAttributeIsALLVMLoopAnnotation(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirLLVMLoopAnnotationAttrGet(MlirContext context, + MlirAttribute disableNonforced, + MlirAttribute unroll, + MlirAttribute mustProgress) { + MLIRContext *ctx = loadLLVM(context); + if (!ctx) + return {nullptr}; + return wrap(LLVM::LoopAnnotationAttr::get( + ctx, optional(disableNonforced), {}, {}, + optional(unroll), {}, {}, {}, {}, {}, {}, + optional(mustProgress), {}, {}, {}, {})); +} diff --git a/mlir/dialects/llvm/capi.h b/mlir/dialects/llvm/capi.h new file mode 100644 index 00000000..04528ddc --- /dev/null +++ b/mlir/dialects/llvm/capi.h @@ -0,0 +1,36 @@ +//===- capi.h - C API for the llvm dialect ------------------------*- C -*-===// +// +// C entry points for the `llvm` dialect's loop annotations, which upstream's +// C API leaves out. + +#ifndef MLIR_DIALECTS_LLVM_CAPI_H +#define MLIR_DIALECTS_LLVM_CAPI_H + +#include "mlir-c/Dialect/LLVM.h" +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Null parameters are unset. + +MLIR_CAPI_EXPORTED bool mlirAttributeIsALLVMLoopUnroll(MlirAttribute attr); +/// `disable`, `runtimeDisable` and `full` are bool attributes, `count` an +/// integer attribute. +MLIR_CAPI_EXPORTED MlirAttribute mlirLLVMLoopUnrollAttrGet( + MlirContext context, MlirAttribute disable, MlirAttribute count, + MlirAttribute runtimeDisable, MlirAttribute full); + +MLIR_CAPI_EXPORTED bool mlirAttributeIsALLVMLoopAnnotation(MlirAttribute attr); +/// `disableNonforced` and `mustProgress` are bool attributes, `unroll` a +/// loop unroll attribute; the other parameters are unset. +MLIR_CAPI_EXPORTED MlirAttribute mlirLLVMLoopAnnotationAttrGet( + MlirContext context, MlirAttribute disableNonforced, MlirAttribute unroll, + MlirAttribute mustProgress); + +#ifdef __cplusplus +} +#endif + +#endif // MLIR_DIALECTS_LLVM_CAPI_H diff --git a/mlir/dialects/llvm/llvm.zig b/mlir/dialects/llvm/llvm.zig new file mode 100644 index 00000000..d1197317 --- /dev/null +++ b/mlir/dialects/llvm/llvm.zig @@ -0,0 +1,52 @@ +// Keep function definitions in ops.zig: LLVM reserves symbols starting with "llvm.". +const ops = @import("ops.zig"); + +pub const NVPTXAddressSpace = ops.NVPTXAddressSpace; +pub const pointerType = ops.pointerType; +pub const structType = ops.structType; +pub const identifiedStructType = ops.identifiedStructType; +pub const arrayType = ops.arrayType; +pub const LoopUnrollAttr = ops.LoopUnrollAttr; +pub const LoopAnnotationAttr = ops.LoopAnnotationAttr; +pub const Linkage = ops.Linkage; +pub const linkageAttr = ops.linkageAttr; +pub const emitCInterface = ops.emitCInterface; +pub const linkage = ops.linkage; +pub const inlineHint = ops.inlineHint; +pub const noundef = ops.noundef; +pub const zeroext = ops.zeroext; +pub const signext = ops.signext; +pub const alignment = ops.alignment; +pub const dereferenceable = ops.dereferenceable; +pub const elementtype = ops.elementtype; +pub const AtomicBinOp = ops.AtomicBinOp; +pub const AtomicOrdering = ops.AtomicOrdering; +pub const inttoptr = ops.inttoptr; +pub const ptrtoint = ops.ptrtoint; +pub const bitcast = ops.bitcast; +pub const fmul = ops.fmul; +pub const extractvalue = ops.extractvalue; +pub const LoadOpts = ops.LoadOpts; +pub const load = ops.load; +pub const StoreOpts = ops.StoreOpts; +pub const store = ops.store; +pub const GEPIndex = ops.GEPIndex; +pub const getelementptr = ops.getelementptr; +pub const alloca = ops.alloca; +pub const atomicrmw = ops.atomicrmw; +pub const intr_assume = ops.intr_assume; +pub const intr_lifetime_start = ops.intr_lifetime_start; +pub const intr_lifetime_end = ops.intr_lifetime_end; +pub const extractelement = ops.extractelement; +pub const insertelement = ops.insertelement; +pub const shufflevector = ops.shufflevector; +pub const mlir_undef = ops.mlir_undef; +pub const mlir_poison = ops.mlir_poison; +pub const freeze = ops.freeze; +pub const BrOpts = ops.BrOpts; +pub const br = ops.br; +pub const unreachable_ = ops.unreachable_; +pub const call_intrinsic = ops.call_intrinsic; +pub const InlineAsmOpts = ops.InlineAsmOpts; +pub const inline_asm = ops.inline_asm; +pub const registerDialect = ops.registerDialect; diff --git a/mlir/dialects/llvm/ops.zig b/mlir/dialects/llvm/ops.zig new file mode 100644 index 00000000..d1fb5771 --- /dev/null +++ b/mlir/dialects/llvm/ops.zig @@ -0,0 +1,567 @@ +const std = @import("std"); + +const c = @import("c"); +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("llvm"); +} + +// ============================================================================= +// Types +// ============================================================================= + +/// Address spaces of `!llvm.ptr` on NVPTX. +pub const NVPTXAddressSpace = enum(u32) { + generic = 0, + global = 1, + shared = 3, + constant = 4, + local = 5, + /// Tensor memory (SM100 TMEM). + tensor = 6, + /// Shared memory of any CTA of the cluster (`shared::cluster`). + shared_cluster = 7, +}; + +/// `!llvm.ptr`, an opaque pointer. Loads the LLVM dialect into `ctx`. +pub fn pointerType(ctx: *mlir.Context, space: anytype) *const mlir.Type { + _ = c.mlirContextGetOrLoadDialect(ctx.ptr(), mlir.stringRef("llvm")); + return @ptrCast(c.mlirLLVMPointerTypeGet(ctx.ptr(), @intCast(@backingInt(space))).ptr); +} + +/// `!llvm.struct<(fields...)>`, a literal struct. +pub fn structType(ctx: *mlir.Context, fields: []const *const mlir.Type) *const mlir.Type { + _ = c.mlirContextGetOrLoadDialect(ctx.ptr(), mlir.stringRef("llvm")); + return @ptrCast(c.mlirLLVMStructTypeLiteralGet(ctx.ptr(), @intCast(fields.len), @ptrCast(fields), false).ptr); +} + +/// `!llvm.struct<"name", (fields...)>`, an identified struct; null if `name` +/// already names a struct of other fields. +pub fn identifiedStructType(ctx: *mlir.Context, name: []const u8, fields: []const *const mlir.Type) ?*const mlir.Type { + _ = c.mlirContextGetOrLoadDialect(ctx.ptr(), mlir.stringRef("llvm")); + const t = c.mlirLLVMStructTypeIdentifiedGet(ctx.ptr(), mlir.stringRef(name)); + if (c.mlirLLVMStructTypeSetBody(t, @intCast(fields.len), @ptrCast(fields), false).value == 0) return null; + return @ptrCast(t.ptr); +} + +/// `!llvm.array`. +pub fn arrayType(element_type: *const mlir.Type, count: u32) *const mlir.Type { + return @ptrCast(c.mlirLLVMArrayTypeGet(element_type.ptr(), count).ptr); +} + +// ============================================================================= +// Attributes +// ============================================================================= + +fn optionalBool(ctx: *mlir.Context, value: ?bool) c.MlirAttribute { + return if (value) |v| mlir.Attribute.boolean(ctx, v).ptr() else .{ .ptr = null }; +} + +/// `#llvm.loop_unroll`: the `llvm.loop.unroll.*` loop metadata. +pub const LoopUnrollAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsALLVMLoopUnroll; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + disable: ?bool = null, + count: ?u32 = null, + runtimeDisable: ?bool = null, + full: ?bool = null, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) *const Self { + const count: c.MlirAttribute = if (args.count) |n| mlir.Attribute.int(ctx, .i32, n).ptr() else .{ .ptr = null }; + return @ptrCast(c.mlirLLVMLoopUnrollAttrGet(ctx.ptr(), optionalBool(ctx, args.disable), count, optionalBool(ctx, args.runtimeDisable), optionalBool(ctx, args.full)).ptr); + } + + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +/// A global's linkage, as `#llvm.linkage` spells it. +pub const Linkage = enum(c_uint) { + external = c.MlirLLVMLinkageExternal, + available_externally = c.MlirLLVMLinkageAvailableExternally, + linkonce = c.MlirLLVMLinkageLinkonce, + linkonce_odr = c.MlirLLVMLinkageLinkonceODR, + weak = c.MlirLLVMLinkageWeak, + weak_odr = c.MlirLLVMLinkageWeakODR, + appending = c.MlirLLVMLinkageAppending, + internal = c.MlirLLVMLinkageInternal, + private = c.MlirLLVMLinkagePrivate, + extern_weak = c.MlirLLVMLinkageExternWeak, + common = c.MlirLLVMLinkageCommon, +}; + +/// `#llvm.linkage`; `llvm.linkage` on a `func.func` sets its +/// linkage once lowered to LLVM. Loads the LLVM dialect into `ctx`, which +/// must have it registered. +pub fn linkageAttr(ctx: *mlir.Context, value: Linkage) *const mlir.Attribute { + _ = c.mlirContextGetOrLoadDialect(ctx.ptr(), mlir.stringRef("llvm")); + return @ptrCast(c.mlirLLVMLinkageAttrGet(ctx.ptr(), @backingInt(value)).ptr); +} + +/// `llvm.emit_c_interface`: has the lowering of a `func.func` emit a +/// `_mlir_ciface_` wrapper that takes its memrefs by pointer. +pub fn emitCInterface(ctx: *mlir.Context) mlir.NamedAttribute { + return .named(ctx, "llvm.emit_c_interface", .unit(ctx)); +} + +/// `llvm.linkage = #llvm.linkage` on a `func.func`. +pub fn linkage(ctx: *mlir.Context, value: Linkage) mlir.NamedAttribute { + return .named(ctx, "llvm.linkage", linkageAttr(ctx, value)); +} + +/// `llvm.inline_hint` on a `func.func`: `inlinehint` once lowered. +pub fn inlineHint(ctx: *mlir.Context) mlir.NamedAttribute { + return .named(ctx, "llvm.inline_hint", .unit(ctx)); +} + +/// `llvm.noundef` on an argument: never undef or poison. +pub fn noundef(ctx: *mlir.Context) mlir.NamedAttribute { + return .named(ctx, "llvm.noundef", .unit(ctx)); +} + +/// `llvm.zeroext` on an argument: a narrow integer passed zero-extended. +pub fn zeroext(ctx: *mlir.Context) mlir.NamedAttribute { + return .named(ctx, "llvm.zeroext", .unit(ctx)); +} + +/// `llvm.signext` on an argument: a narrow integer passed sign-extended. +pub fn signext(ctx: *mlir.Context) mlir.NamedAttribute { + return .named(ctx, "llvm.signext", .unit(ctx)); +} + +/// `llvm.align = bytes` on a pointer argument. +pub fn alignment(ctx: *mlir.Context, bytes: u64) mlir.NamedAttribute { + return .named(ctx, "llvm.align", .int(ctx, .i64, @as(i64, @intCast(bytes)))); +} + +/// `llvm.dereferenceable = bytes` on a pointer argument: that many bytes are +/// readable. +pub fn dereferenceable(ctx: *mlir.Context, bytes: u64) mlir.NamedAttribute { + return .named(ctx, "llvm.dereferenceable", .int(ctx, .i64, @as(i64, @intCast(bytes)))); +} + +/// `llvm.elementtype = type` on a pointer argument: its pointee, which +/// typed-pointer IR prints. +pub fn elementtype(ctx: *mlir.Context, pointee: *const mlir.Type) mlir.NamedAttribute { + return .named(ctx, "llvm.elementtype", .typeAttr(pointee)); +} + +/// The operation of `llvm.atomicrmw`, as the LLVM dialect numbers it. +pub const AtomicBinOp = enum(i64) { + xchg = 0, + add = 1, + sub = 2, + _and = 3, + nand = 4, + _or = 5, + _xor = 6, + max = 7, + min = 8, + umax = 9, + umin = 10, + fadd = 11, + fsub = 12, + fmax = 13, + fmin = 14, + uinc_wrap = 15, + udec_wrap = 16, + usub_cond = 17, + usub_sat = 18, + fmaximum = 19, + fminimum = 20, + fmaximumnum = 21, + fminimumnum = 22, +}; + +/// The memory ordering of an atomic operation, as the LLVM dialect numbers it. +pub const AtomicOrdering = enum(i64) { + not_atomic = 0, + unordered = 1, + monotonic = 2, + acquire = 4, + release = 5, + acq_rel = 6, + seq_cst = 7, +}; + +/// `#llvm.loop_annotation`: the `llvm.loop` metadata of a loop. +pub const LoopAnnotationAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsALLVMLoopAnnotation; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + disableNonforced: ?bool = null, + unroll: ?*const LoopUnrollAttr = null, + mustProgress: ?bool = null, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) *const Self { + const unroll: c.MlirAttribute = if (args.unroll) |u| u.ptr() else .{ .ptr = null }; + return @ptrCast(c.mlirLLVMLoopAnnotationAttrGet(ctx.ptr(), optionalBool(ctx, args.disableNonforced), unroll, optionalBool(ctx, args.mustProgress)).ptr); + } + + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +// ============================================================================= +// Casts and arithmetic +// ============================================================================= + +/// llvm.inttoptr — an integer address as a pointer of type `result_type`. +pub fn inttoptr(ctx: *mlir.Context, address: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "llvm.inttoptr", .{ + .operands = .{ .flat = &.{address} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// llvm.ptrtoint — the integer address of an LLVM pointer. +pub fn ptrtoint(ctx: *mlir.Context, pointer: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "llvm.ptrtoint", .{ + .operands = .{ .flat = &.{pointer} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// llvm.bitcast — reinterpret `value` as `result_type` (same bit width). +pub fn bitcast(ctx: *mlir.Context, value: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "llvm.bitcast", .{ + .operands = .{ .flat = &.{value} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// llvm.fmul — floating-point (or floating-point vector) product of type `result_type`. +pub fn fmul(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "llvm.fmul", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +// ============================================================================= +// Aggregates +// ============================================================================= + +/// llvm.extractvalue — the element of type `result_type` at `position` in the +/// struct or array `container`. +pub fn extractvalue(ctx: *mlir.Context, container: *const mlir.Value, result_type: *const mlir.Type, position: []const i64, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "llvm.extractvalue", .{ + .operands = .{ .flat = &.{container} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{.named(ctx, "position", .denseArray(ctx, .i64, position))}, + .location = location, + }); +} + +// ============================================================================= +// Memory +// ============================================================================= + +pub const LoadOpts = struct { + /// Alignment in bytes (none: the natural alignment of the type). + alignment: ?u64 = null, + /// The memory is not written while the kernel runs (`!invariant.load`; on NVPTX + /// global memory this selects the non-coherent `ld.global.nc`). + invariant: bool = false, +}; + +/// llvm.load — a value of type `result_type` through `pointer`. +pub fn load(ctx: *mlir.Context, pointer: *const mlir.Value, result_type: *const mlir.Type, opts: LoadOpts, location: *const mlir.Location) *mlir.Operation { + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.alignment) |bytes| { + attrs.appendAssumeCapacity(.named(ctx, "alignment", .int(ctx, .i64, bytes))); + } + if (opts.invariant) { + attrs.appendAssumeCapacity(.named(ctx, "invariant", .unit(ctx))); + } + return mlir.Operation.make(ctx, "llvm.load", .{ + .operands = .{ .flat = &.{pointer} }, + .results = .{ .flat = &.{result_type} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +pub const StoreOpts = struct { + /// Alignment in bytes (none: the natural alignment of the type). + alignment: ?u64 = null, +}; + +/// llvm.store — `value` through `pointer`. +pub fn store(ctx: *mlir.Context, value: *const mlir.Value, pointer: *const mlir.Value, opts: StoreOpts, location: *const mlir.Location) *mlir.Operation { + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.alignment) |bytes| { + attrs.appendAssumeCapacity(.named(ctx, "alignment", .int(ctx, .i64, bytes))); + } + return mlir.Operation.make(ctx, "llvm.store", .{ + .operands = .{ .flat = &.{ value, pointer } }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +/// An index of `getelementptr`: a constant, or an SSA integer. +pub const GEPIndex = union(enum) { + constant: i32, + value: *const mlir.Value, +}; + +/// llvm.getelementptr — `base` offset by `indices` into `elem_type`, with +/// `base`'s pointer type. +pub fn getelementptr( + ctx: *mlir.Context, + base: *const mlir.Value, + indices: []const GEPIndex, + elem_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + const dynamic_index = std.math.minInt(i32); + var operands: stdx.BoundedArray(*const mlir.Value, 16) = .empty; + var raw: stdx.BoundedArray(i32, 16) = .empty; + operands.appendAssumeCapacity(base); + for (indices) |index| switch (index) { + .constant => |c_| raw.appendAssumeCapacity(c_), + .value => |v| { + raw.appendAssumeCapacity(dynamic_index); + operands.appendAssumeCapacity(v); + }, + }; + return mlir.Operation.make(ctx, "llvm.getelementptr", .{ + .operands = .{ .flat = operands.constSlice() }, + .results = .{ .flat = &.{base.type_()} }, + .attributes = &.{ + .named(ctx, "rawConstantIndices", .denseArray(ctx, .i32, raw.constSlice())), + .named(ctx, "elem_type", .typeAttr(elem_type)), + }, + .location = location, + }); +} + +/// llvm.alloca — stack storage for `array_size` elements of `elem_type`, as a +/// pointer of type `result_type`. +pub fn alloca( + ctx: *mlir.Context, + array_size: *const mlir.Value, + result_type: *const mlir.Type, + elem_type: *const mlir.Type, + opts: struct { + alignment: ?u64 = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attrs.appendAssumeCapacity(.named(ctx, "elem_type", .typeAttr(elem_type))); + if (opts.alignment) |bytes| attrs.appendAssumeCapacity(.named(ctx, "alignment", .int(ctx, .i64, bytes))); + return mlir.Operation.make(ctx, "llvm.alloca", .{ + .operands = .{ .flat = &.{array_size} }, + .results = .{ .flat = &.{result_type} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +/// llvm.atomicrmw — combine `val` into the value at `ptr` with `bin_op`, +/// atomically; the result is the previous value, of `val`'s type. +pub fn atomicrmw( + ctx: *mlir.Context, + ptr: *const mlir.Value, + val: *const mlir.Value, + bin_op: AtomicBinOp, + ordering: AtomicOrdering, + opts: struct { + /// Synchronization scope, e.g. `"agent"` (none: the whole system). + syncscope: ?[]const u8 = null, + /// Alignment in bytes (none: the natural alignment of the type). + alignment: ?u64 = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + attrs.appendSliceAssumeCapacity(&.{ + .named(ctx, "bin_op", .int(ctx, .i64, @backingInt(bin_op))), + .named(ctx, "ordering", .int(ctx, .i64, @backingInt(ordering))), + }); + if (opts.syncscope) |syncscope| attrs.appendAssumeCapacity(.named(ctx, "syncscope", .string(ctx, syncscope))); + if (opts.alignment) |bytes| attrs.appendAssumeCapacity(.named(ctx, "alignment", .int(ctx, .i64, bytes))); + return mlir.Operation.make(ctx, "llvm.atomicrmw", .{ + .operands = .{ .flat = &.{ ptr, val } }, + .results = .{ .flat = &.{val.type_()} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +// ============================================================================= +// Intrinsics and inline assembly +// ============================================================================= + +/// llvm.intr.assume — let the optimizer assume that `cond` (i1) holds. +pub fn intr_assume(ctx: *mlir.Context, cond: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "llvm.intr.assume", .{ + .operands = .{ .flat = &.{cond} }, + .attributes = &.{ + .named(ctx, "op_bundle_sizes", .denseArray(ctx, .i32, &.{})), + .named(ctx, "op_bundle_tags", .array(ctx, &.{})), + }, + .location = location, + }); +} + +pub fn intr_lifetime_start(ctx: *mlir.Context, ptr: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "llvm.intr.lifetime.start", .{ + .operands = .{ .flat = &.{ptr} }, + .location = location, + }); +} + +pub fn intr_lifetime_end(ctx: *mlir.Context, ptr: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "llvm.intr.lifetime.end", .{ + .operands = .{ .flat = &.{ptr} }, + .location = location, + }); +} + +pub fn extractelement(ctx: *mlir.Context, vector: *const mlir.Value, position: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "llvm.extractelement", .{ + .operands = .{ .flat = &.{ vector, position } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +pub fn insertelement(ctx: *mlir.Context, vector: *const mlir.Value, value: *const mlir.Value, position: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "llvm.insertelement", .{ + .operands = .{ .flat = &.{ vector, value, position } }, + .results = .{ .flat = &.{vector.type_()} }, + .location = location, + }); +} + +pub fn shufflevector(ctx: *mlir.Context, v1: *const mlir.Value, v2: *const mlir.Value, result_type: *const mlir.Type, mask: []const i32, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "llvm.shufflevector", .{ + .operands = .{ .flat = &.{ v1, v2 } }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{.named(ctx, "mask", .denseArray(ctx, .i32, mask))}, + .location = location, + }); +} + +pub fn mlir_undef(ctx: *mlir.Context, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "llvm.mlir.undef", .{ + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +pub fn mlir_poison(ctx: *mlir.Context, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "llvm.mlir.poison", .{ + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +pub fn freeze(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "llvm.freeze", .{ + .operands = .{ .flat = &.{value} }, + .results = .{ .flat = &.{value.type_()} }, + .location = location, + }); +} + +pub const BrOpts = struct { + loop_annotation: ?*const LoopAnnotationAttr = null, +}; + +/// `llvm.br`. Not verified: a terminator only verifies inside its block. +pub fn br(ctx: *mlir.Context, dest: *mlir.Block, dest_operands: []const *const mlir.Value, opts: BrOpts, location: *const mlir.Location) *mlir.Operation { + var state: mlir.OperationState = .init("llvm.br", location); + state.addOperands(dest_operands); + var successors: [1]*const mlir.Block = .{dest}; + state.addSuccessors(&successors); + if (opts.loop_annotation) |annotation| state.addAttributes(&.{.named(ctx, "loop_annotation", annotation.attribute())}); + return mlir.Operation.init(&state) catch @panic("llvm.br: failed to create the operation"); +} + +/// `llvm.unreachable`. Not verified: a terminator only verifies inside its +/// block. +pub fn unreachable_(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "llvm.unreachable", .{ .location = location, .verify = false }); +} + +/// llvm.call_intrinsic — call the LLVM intrinsic `name` (e.g. `llvm.nvvm.*`), with an +/// optional result of type `result_type`. +pub fn call_intrinsic( + ctx: *mlir.Context, + name: []const u8, + operands: []const *const mlir.Value, + opts: struct { + result_type: ?*const mlir.Type = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "llvm.call_intrinsic", .{ + .operands = .{ .flat = operands }, + .results = .{ .flat = if (opts.result_type) |r| &.{r} else &.{} }, + .attributes = &.{ + .named(ctx, "intrin", .string(ctx, name)), + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &.{ @intCast(operands.len), 0 })), + .named(ctx, "op_bundle_sizes", .denseArray(ctx, .i32, &.{})), + }, + .location = location, + }); +} + +pub const InlineAsmOpts = struct { + result_type: ?*const mlir.Type = null, + + /// LLVM constraint string, e.g. `"=r,r"` (outputs first). + constraints: []const u8 = "", + /// The assembly has effects beyond its outputs (it is never removed or merged). + has_side_effects: bool = false, +}; + +/// llvm.inline_asm — the PTX (or other target) assembly `asm_string`, with `$i` +/// placeholders bound to the result then `operands`. +pub fn inline_asm( + ctx: *mlir.Context, + asm_string: []const u8, + operands: []const *const mlir.Value, + opts: InlineAsmOpts, + location: *const mlir.Location, +) *mlir.Operation { + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attrs.appendAssumeCapacity(.named(ctx, "asm_string", .string(ctx, asm_string))); + attrs.appendAssumeCapacity(.named(ctx, "constraints", .string(ctx, opts.constraints))); + if (opts.has_side_effects) { + attrs.appendAssumeCapacity(.named(ctx, "has_side_effects", .unit(ctx))); + } + return mlir.Operation.make(ctx, "llvm.inline_asm", .{ + .operands = .{ .flat = operands }, + .results = .{ .flat = if (opts.result_type) |r| &.{r} else &.{} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} diff --git a/mlir/dialects/llvm/test.zig b/mlir/dialects/llvm/test.zig new file mode 100644 index 00000000..6ba84b4d --- /dev/null +++ b/mlir/dialects/llvm/test.zig @@ -0,0 +1,210 @@ +const std = @import("std"); + +const llvm = @import("mlir/dialects/llvm"); +const mlir = @import("mlir"); + +const operation_names: []const []const u8 = &.{ + "llvm.alloca", + "llvm.atomicrmw", + "llvm.bitcast", + "llvm.br", + "llvm.call_intrinsic", + "llvm.extractelement", + "llvm.extractvalue", + "llvm.fmul", + "llvm.freeze", + "llvm.getelementptr", + "llvm.inline_asm", + "llvm.insertelement", + "llvm.intr.assume", + "llvm.intr.lifetime.end", + "llvm.intr.lifetime.start", + "llvm.inttoptr", + "llvm.ptrtoint", + "llvm.load", + "llvm.mlir.poison", + "llvm.mlir.undef", + "llvm.shufflevector", + "llvm.store", + "llvm.unreachable", +}; + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(llvm); + inline for (comptime std.meta.declarations(llvm)) |decl| { + const T = @field(llvm, decl); + if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); + } +} + +test "every bound operation is registered" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + llvm.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} + +test "literal struct types build from their fields" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + llvm.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + const struct_type = llvm.structType(ctx, &.{ .int(ctx, .i32), .float(ctx, .f32) }); + try std.testing.expect(struct_type.eql(try mlir.Type.parse(ctx, "!llvm.struct<(i32, f32)>"))); +} + +test "identified struct types build from their name and fields" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + llvm.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + const fields: []const *const mlir.Type = &.{ .int(ctx, .i32), .float(ctx, .f32) }; + const struct_type = llvm.identifiedStructType(ctx, "struct.args", fields).?; + try std.testing.expect(struct_type.eql(try mlir.Type.parse(ctx, "!llvm.struct<\"struct.args\", (i32, f32)>"))); + try std.testing.expect(llvm.identifiedStructType(ctx, "struct.args", fields).?.eql(struct_type)); + try std.testing.expect(llvm.identifiedStructType(ctx, "struct.args", fields[0..1]) == null); +} + +test "getelementptr and alloca build verified operations" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + llvm.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + const loc: *const mlir.Location = .unknown(ctx); + const ptr = llvm.pointerType(ctx, llvm.NVPTXAddressSpace.global); + const block = mlir.Block.init(&.{ ptr, .int(ctx, .i64) }, &.{ loc, loc }); + defer block.deinit(); + const array = llvm.arrayType(.float(ctx, .f32), 8); + const gep = llvm.getelementptr(ctx, block.argument(0), &.{ .{ .constant = 0 }, .{ .value = block.argument(1) } }, array, loc).appendTo(block); + try std.testing.expect(gep.verify()); + try std.testing.expect(gep.result(0).type_().eql(ptr)); + const alloca = llvm.alloca(ctx, block.argument(1), llvm.pointerType(ctx, llvm.NVPTXAddressSpace.generic), array, .{ .alignment = 4 }, loc).appendTo(block); + try std.testing.expect(alloca.verify()); +} + +test "extractvalue, atomicrmw and intr.assume build verified operations" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + llvm.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + const loc: *const mlir.Location = .unknown(ctx); + const pair = llvm.structType(ctx, &.{ .int(ctx, .i32), .float(ctx, .f32) }); + const block = mlir.Block.init(&.{ pair, llvm.pointerType(ctx, llvm.NVPTXAddressSpace.global), .int(ctx, .i1) }, &.{ loc, loc, loc }); + defer block.deinit(); + const first = llvm.extractvalue(ctx, block.argument(0), .int(ctx, .i32), &.{0}, loc).appendTo(block); + try std.testing.expect(first.verify()); + const previous = llvm.atomicrmw(ctx, block.argument(1), first.result(0), .add, .monotonic, .{ .syncscope = "agent", .alignment = 4 }, loc).appendTo(block); + try std.testing.expect(previous.verify()); + try std.testing.expect(previous.result(0).type_().eql(.int(ctx, .i32))); + var writer: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer writer.deinit(); + try writer.writer.print("{f}", .{previous}); + try std.testing.expect(std.mem.indexOf(u8, writer.written(), "llvm.atomicrmw add ") != null); + try std.testing.expect(std.mem.indexOf(u8, writer.written(), " monotonic ") != null); + try std.testing.expect(llvm.intr_assume(ctx, block.argument(2), loc).appendTo(block).verify()); +} + +test "vector, freeze and lifetime operations build verified operations" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + llvm.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + const loc: *const mlir.Location = .unknown(ctx); + const i64x2: *const mlir.Type = .vector(&.{2}, .int(ctx, .i64)); + const block = mlir.Block.init(&.{ i64x2, .int(ctx, .i64), .int(ctx, .i32), llvm.pointerType(ctx, llvm.NVPTXAddressSpace.generic) }, &.{ loc, loc, loc, loc }); + defer block.deinit(); + const v = block.argument(0); + const inserted = llvm.insertelement(ctx, v, block.argument(1), block.argument(2), loc).appendTo(block); + try std.testing.expect(inserted.verify()); + try std.testing.expect(inserted.result(0).type_().eql(i64x2)); + try std.testing.expect(llvm.extractelement(ctx, v, block.argument(2), .int(ctx, .i64), loc).appendTo(block).verify()); + const poison = llvm.mlir_poison(ctx, i64x2, loc).appendTo(block); + try std.testing.expect(poison.verify()); + try std.testing.expect(llvm.mlir_undef(ctx, i64x2, loc).appendTo(block).verify()); + try std.testing.expect(llvm.shufflevector(ctx, v, poison.result(0), i64x2, &.{ 1, 0 }, loc).appendTo(block).verify()); + try std.testing.expect(llvm.freeze(ctx, v, loc).appendTo(block).verify()); + try std.testing.expect(llvm.intr_lifetime_start(ctx, block.argument(3), loc).appendTo(block).verify()); + try std.testing.expect(llvm.intr_lifetime_end(ctx, block.argument(3), loc).appendTo(block).verify()); +} + +test "br carries its loop annotation" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + llvm.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + const loc: *const mlir.Location = .unknown(ctx); + const dest = mlir.Block.init(&.{}, &.{}); + defer dest.deinit(); + const annotation = llvm.LoopAnnotationAttr.get(ctx, .{ .mustProgress = true }); + const op = llvm.br(ctx, dest, &.{}, .{ .loop_annotation = annotation }, loc); + defer op.deinit(); + try std.testing.expect(op.attributeByName("loop_annotation").?.eql(annotation.attribute())); + const end = llvm.unreachable_(ctx, loc); + defer end.deinit(); +} + +test "argument and function attributes are named by the LLVM dialect" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + llvm.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + var buf: [512]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print("{f}", .{mlir.Attribute.dict(ctx, &.{ + llvm.noundef(ctx), llvm.zeroext(ctx), llvm.signext(ctx), + llvm.alignment(ctx, 16), llvm.dereferenceable(ctx, 64), llvm.elementtype(ctx, .float(ctx, .f32)), + llvm.linkage(ctx, .internal), llvm.inlineHint(ctx), + })}); + try std.testing.expectEqualStrings("{llvm.align = 16 : i64, llvm.dereferenceable = 64 : i64, llvm.elementtype = f32, llvm.inline_hint, llvm.linkage = #llvm.linkage, llvm.noundef, llvm.signext, llvm.zeroext}", w.buffered()); +} + +test "emit_c_interface is a unit attribute of the LLVM dialect" { + const ctx = try mlir.Context.init(.{}); + defer ctx.deinit(); + var writer: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer writer.deinit(); + try writer.writer.print("{f}", .{mlir.Attribute.dict(ctx, &.{llvm.emitCInterface(ctx)})}); + try std.testing.expectEqualStrings("{llvm.emit_c_interface}", writer.written()); +} + +test "loop annotations are built through the C API" { + const ctx = try mlir.Context.init(.{}); + defer ctx.deinit(); + const unroll = llvm.LoopUnrollAttr.get(ctx, .{ .full = true }); + try std.testing.expect(unroll.attribute().isA(llvm.LoopUnrollAttr) != null); + const annotation = llvm.LoopAnnotationAttr.get(ctx, .{ .unroll = unroll, .mustProgress = true }); + var writer: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer writer.deinit(); + try writer.writer.print("{f}", .{annotation}); + try std.testing.expectEqualStrings("#llvm.loop_annotation, mustProgress = true>", writer.written()); + try std.testing.expect((try mlir.Attribute.parse(ctx, writer.written())).eql(annotation.attribute())); +} + +test "linkage attributes are built through the C API" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + llvm.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + var writer: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer writer.deinit(); + try writer.writer.print("{f}", .{llvm.linkageAttr(ctx, .weak_odr)}); + try std.testing.expectEqualStrings("#llvm.linkage", writer.written()); +} diff --git a/mlir/dialects/math.zig b/mlir/dialects/math.zig deleted file mode 100644 index 766f40c2..00000000 --- a/mlir/dialects/math.zig +++ /dev/null @@ -1,35 +0,0 @@ -const std = @import("std"); -const mlir = @import("mlir"); - -const namespace = "math"; - -fn unary_fn(comptime op_name: [:0]const u8) type { - return struct { - pub fn call(ctx: mlir.Context, value: mlir.Value, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, namespace ++ "." ++ op_name, .{ - .operands = &.{value}, - .results = &.{}, - .location = location, - }); - } - }; -} - -fn binary_fn(comptime op_name: [:0]const u8) type { - return struct { - pub fn call(ctx: mlir.Context, lhs: mlir.Value, rhs: mlir.Value, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, namespace ++ "." ++ op_name, .{ - .operands = &.{ lhs, rhs }, - .results = &.{}, - .location = location, - }); - } - }; -} - -pub const ipowi = binary_fn("ipowi").call; -pub const fpowi = binary_fn("fpowi").call; -pub const tanh = unary_fn("tanh").call; -pub const sqrt = unary_fn("sqrt").call; -pub const exp = unary_fn("exp").call; -pub const log = unary_fn("log").call; diff --git a/mlir/dialects/math/BUILD.bazel b/mlir/dialects/math/BUILD.bazel new file mode 100644 index 00000000..3845c969 --- /dev/null +++ b/mlir/dialects/math/BUILD.bazel @@ -0,0 +1,22 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "math", + import_name = "mlir/dialects/math", + main = "math.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//mlir", + "@llvm-project//mlir:CAPIMath", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":math", + "//mlir", + ], +) diff --git a/mlir/dialects/math/math.zig b/mlir/dialects/math/math.zig new file mode 100644 index 00000000..4ca6ee6d --- /dev/null +++ b/mlir/dialects/math/math.zig @@ -0,0 +1,210 @@ +const mlir = @import("mlir"); + +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("math"); +} + +// ============================================================================= +// Float unary — SameOperandsAndResultType. Result type inferred from operand. +// ============================================================================= + +fn floatUnaryOp(comptime op_name: []const u8) type { + return struct { + pub fn call( + ctx: *mlir.Context, + src: *const mlir.Value, + location: *const mlir.Location, + ) *mlir.Operation { + return mlir.Operation.make(ctx, op_name, .{ + .operands = .{ .flat = &.{src} }, + .result_type_inference = true, + .location = location, + }); + } + }; +} + +// Exponentials / logs +pub const exp = floatUnaryOp("math.exp").call; +pub const exp2 = floatUnaryOp("math.exp2").call; +pub const expm1 = floatUnaryOp("math.expm1").call; +pub const log = floatUnaryOp("math.log").call; +pub const log2 = floatUnaryOp("math.log2").call; +pub const log10 = floatUnaryOp("math.log10").call; +pub const log1p = floatUnaryOp("math.log1p").call; + +// Trig / hyperbolic +pub const sin = floatUnaryOp("math.sin").call; +pub const cos = floatUnaryOp("math.cos").call; +pub const tan = floatUnaryOp("math.tan").call; +pub const asin = floatUnaryOp("math.asin").call; +pub const acos = floatUnaryOp("math.acos").call; +pub const atan = floatUnaryOp("math.atan").call; +pub const sinh = floatUnaryOp("math.sinh").call; +pub const cosh = floatUnaryOp("math.cosh").call; +pub const tanh = floatUnaryOp("math.tanh").call; +pub const asinh = floatUnaryOp("math.asinh").call; +pub const acosh = floatUnaryOp("math.acosh").call; +pub const atanh = floatUnaryOp("math.atanh").call; + +// Roots / errors / abs +pub const sqrt = floatUnaryOp("math.sqrt").call; +pub const rsqrt = floatUnaryOp("math.rsqrt").call; +pub const cbrt = floatUnaryOp("math.cbrt").call; +pub const erf = floatUnaryOp("math.erf").call; +pub const erfc = floatUnaryOp("math.erfc").call; +pub const absf = floatUnaryOp("math.absf").call; + +// Rounding +pub const floor = floatUnaryOp("math.floor").call; +pub const ceil = floatUnaryOp("math.ceil").call; +pub const round = floatUnaryOp("math.round").call; +pub const roundeven = floatUnaryOp("math.roundeven").call; +pub const trunc = floatUnaryOp("math.trunc").call; + +// ============================================================================= +// Float binary — SameOperandsAndResultType. +// ============================================================================= + +fn floatBinaryOp(comptime op_name: []const u8) type { + return struct { + pub fn call( + ctx: *mlir.Context, + lhs: *const mlir.Value, + rhs: *const mlir.Value, + location: *const mlir.Location, + ) *mlir.Operation { + return mlir.Operation.make(ctx, op_name, .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .result_type_inference = true, + .location = location, + }); + } + }; +} + +pub const powf = floatBinaryOp("math.powf").call; +pub const atan2 = floatBinaryOp("math.atan2").call; +pub const copysign = floatBinaryOp("math.copysign").call; + +// ============================================================================= +// Float ternary — SameOperandsAndResultType. +// ============================================================================= + +pub fn fma( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "math.fma", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .result_type_inference = true, + .location = location, + }); +} + +/// math.clampf value, min, max — clamp to [min, max], same type throughout. +/// Distinct from `tt.clampf` (which has an explicit `propagateNan` attr); the +/// math version uses fastmath for NaN behaviour. +pub fn clampf( + ctx: *mlir.Context, + value: *const mlir.Value, + min: *const mlir.Value, + max: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "math.clampf", .{ + .operands = .{ .flat = &.{ value, min, max } }, + .result_type_inference = true, + .location = location, + }); +} + +// ============================================================================= +// Sincos — operand → (sin, cos), both same type as operand. +// ============================================================================= + +pub fn sincos( + ctx: *mlir.Context, + operand: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "math.sincos", .{ + .operands = .{ .flat = &.{operand} }, + .results = .{ .flat = &.{ operand.type_(), operand.type_() } }, + .location = location, + }); +} + +// ============================================================================= +// FPowI — float base, signless integer exponent → float (same type as base). +// ============================================================================= + +pub fn fpowi( + ctx: *mlir.Context, + base: *const mlir.Value, + power: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "math.fpowi", .{ + .operands = .{ .flat = &.{ base, power } }, + .results = .{ .flat = &.{base.type_()} }, + .location = location, + }); +} + +// ============================================================================= +// Integer unary +// ============================================================================= + +fn intUnaryOp(comptime op_name: []const u8) type { + return struct { + pub fn call( + ctx: *mlir.Context, + src: *const mlir.Value, + location: *const mlir.Location, + ) *mlir.Operation { + return mlir.Operation.make(ctx, op_name, .{ + .operands = .{ .flat = &.{src} }, + .result_type_inference = true, + .location = location, + }); + } + }; +} + +pub const absi = intUnaryOp("math.absi").call; +pub const ctlz = intUnaryOp("math.ctlz").call; +pub const cttz = intUnaryOp("math.cttz").call; +pub const ctpop = intUnaryOp("math.ctpop").call; + +/// math.ipowi(base, exp) — integer power. Result type matches base. +pub const ipowi = floatBinaryOp("math.ipowi").call; + +// ============================================================================= +// Float classification — operand is float-like, result is i1-like. +// ============================================================================= + +fn classifyOp(comptime op_name: []const u8) type { + return struct { + pub fn call( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, + ) *mlir.Operation { + return mlir.Operation.make(ctx, op_name, .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); + } + }; +} + +pub const isfinite = classifyOp("math.isfinite").call; +pub const isinf = classifyOp("math.isinf").call; +pub const isnan = classifyOp("math.isnan").call; +pub const isnormal = classifyOp("math.isnormal").call; diff --git a/mlir/dialects/math/test.zig b/mlir/dialects/math/test.zig new file mode 100644 index 00000000..fe2b7bc4 --- /dev/null +++ b/mlir/dialects/math/test.zig @@ -0,0 +1,73 @@ +const std = @import("std"); + +const math = @import("mlir/dialects/math"); +const mlir = @import("mlir"); + +const operation_names: []const []const u8 = &.{ + "math.absf", + "math.absi", + "math.acos", + "math.acosh", + "math.asin", + "math.asinh", + "math.atan", + "math.atan2", + "math.atanh", + "math.cbrt", + "math.ceil", + "math.clampf", + "math.copysign", + "math.cos", + "math.cosh", + "math.ctlz", + "math.ctpop", + "math.cttz", + "math.erf", + "math.erfc", + "math.exp", + "math.exp2", + "math.expm1", + "math.floor", + "math.fma", + "math.fpowi", + "math.ipowi", + "math.isfinite", + "math.isinf", + "math.isnan", + "math.isnormal", + "math.log", + "math.log10", + "math.log1p", + "math.log2", + "math.powf", + "math.round", + "math.roundeven", + "math.rsqrt", + "math.sin", + "math.sincos", + "math.sinh", + "math.sqrt", + "math.tan", + "math.tanh", + "math.trunc", +}; + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(math); + inline for (comptime std.meta.declarations(math)) |decl| { + const T = @field(math, decl); + if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); + } +} + +test "every bound operation is registered" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + math.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} diff --git a/mlir/dialects/memref/BUILD.bazel b/mlir/dialects/memref/BUILD.bazel new file mode 100644 index 00000000..be1a16bb --- /dev/null +++ b/mlir/dialects/memref/BUILD.bazel @@ -0,0 +1,23 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "memref", + import_name = "mlir/dialects/memref", + main = "memref.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//mlir", + "//stdx", + "@llvm-project//mlir:CAPIMemRef", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":memref", + "//mlir", + ], +) diff --git a/mlir/dialects/memref/memref.zig b/mlir/dialects/memref/memref.zig new file mode 100644 index 00000000..6c45eb33 --- /dev/null +++ b/mlir/dialects/memref/memref.zig @@ -0,0 +1,670 @@ +const std = @import("std"); + +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("memref"); +} + +/// Sentinel matching `mlir::ShapedType::kDynamic` (== INT64_MIN). +/// Use this in static-size/offset/stride arrays to mark a slot as dynamic. +pub const kDynamic: i64 = std.math.minInt(i64); + +// ============================================================================= +// Allocation — alloc / alloca / dealloc / realloc +// ============================================================================= + +pub const AllocOpts = struct { + alignment: ?i64 = null, +}; + +pub const ReallocOpts = struct { + alignment: ?i64 = null, + dynamic_size: ?*const mlir.Value = null, +}; + +/// memref.alloc — heap allocation. `dynamic_sizes` provides one Index value per +/// dynamic dim of `result_type` (left-to-right). `symbol_operands` binds to +/// the symbols of the layout map (rare; pass &.{} when none). +pub fn alloc( + ctx: *mlir.Context, + result_type: *const mlir.Type, + dynamic_sizes: []const *const mlir.Value, + symbol_operands: []const *const mlir.Value, + opts: AllocOpts, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 64) = .empty; + operands_buf.appendSliceAssumeCapacity(dynamic_sizes); + operands_buf.appendSliceAssumeCapacity(symbol_operands); + + const seg_sizes = [2]i32{ @intCast(dynamic_sizes.len), @intCast(symbol_operands.len) }; + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attrs.appendAssumeCapacity(.named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes))); + if (opts.alignment) |a| { + attrs.appendAssumeCapacity(.named(ctx, "alignment", .int(ctx, .i64, a))); + } + + return mlir.Operation.make(ctx, "memref.alloc", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +/// memref.alloca — stack allocation; same operand/attribute layout as alloc. +/// Unverified: it only verifies inside an `AutomaticAllocationScope`. +pub fn alloca( + ctx: *mlir.Context, + result_type: *const mlir.Type, + dynamic_sizes: []const *const mlir.Value, + symbol_operands: []const *const mlir.Value, + opts: AllocOpts, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 64) = .empty; + operands_buf.appendSliceAssumeCapacity(dynamic_sizes); + operands_buf.appendSliceAssumeCapacity(symbol_operands); + + const seg_sizes = [2]i32{ @intCast(dynamic_sizes.len), @intCast(symbol_operands.len) }; + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attrs.appendAssumeCapacity(.named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes))); + if (opts.alignment) |a| { + attrs.appendAssumeCapacity(.named(ctx, "alignment", .int(ctx, .i64, a))); + } + + return mlir.Operation.make(ctx, "memref.alloca", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = attrs.constSlice(), + .verify = false, + .location = location, + }); +} + +pub fn dealloc( + ctx: *mlir.Context, + memref: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "memref.dealloc", .{ + .operands = .{ .flat = &.{memref} }, + .location = location, + }); +} + +/// memref.realloc — grow/shrink an existing allocation. `dynamic_size` is +/// optional; omit `opts.dynamic_size` to keep the size encoded in `result_type`. +pub fn realloc( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + opts: ReallocOpts, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 2) = .empty; + operands_buf.appendAssumeCapacity(src); + if (opts.dynamic_size) |s| operands_buf.appendAssumeCapacity(s); + + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.alignment) |a| { + attrs.appendAssumeCapacity(.named(ctx, "alignment", .int(ctx, .i64, a))); + } + + return mlir.Operation.make(ctx, "memref.realloc", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +// ============================================================================= +// Load / Store — plain (non-affine) memref load/store +// ============================================================================= + +pub const LoadOpts = struct { + /// Hint that the lowering may use a non-temporal load. + nontemporal: bool = false, +}; + +/// memref.load — load a single element from a memref. +pub fn load( + ctx: *mlir.Context, + memref: *const mlir.Value, + indices: []const *const mlir.Value, + opts: LoadOpts, + location: *const mlir.Location, +) *mlir.Operation { + var buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; + buf.appendAssumeCapacity(memref); + buf.appendSliceAssumeCapacity(indices); + + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.nontemporal) { + attrs.appendAssumeCapacity(.named(ctx, "nontemporal", .boolean(ctx, true))); + } + + return mlir.Operation.make(ctx, "memref.load", .{ + .operands = .{ .flat = buf.constSlice() }, + .result_type_inference = true, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +pub const StoreOpts = struct { + nontemporal: bool = false, +}; + +/// memref.store — store a single element into a memref. +pub fn store( + ctx: *mlir.Context, + value: *const mlir.Value, + memref: *const mlir.Value, + indices: []const *const mlir.Value, + opts: StoreOpts, + location: *const mlir.Location, +) *mlir.Operation { + var buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; + buf.appendSliceAssumeCapacity(&.{ value, memref }); + buf.appendSliceAssumeCapacity(indices); + + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.nontemporal) { + attrs.appendAssumeCapacity(.named(ctx, "nontemporal", .boolean(ctx, true))); + } + + return mlir.Operation.make(ctx, "memref.store", .{ + .operands = .{ .flat = buf.constSlice() }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +// ============================================================================= +// Atomic RMW +// ============================================================================= + +/// Matches `mlir::arith::AtomicRMWKind` (I64EnumAttr) in upstream `Arith` td. +/// memref.atomic_rmw and affine.parallel both encode reductions through this. +pub const AtomicRMWKind = enum(i64) { + addf = 0, + addi = 1, + andi = 2, + assign = 3, + maximumf = 4, + maxnumf = 5, + maxs = 6, + maxu = 7, + minimumf = 8, + minnumf = 9, + mins = 10, + minu = 11, + mulf = 12, + muli = 13, + ori = 14, + xori = 15, +}; + +/// memref.atomic_rmw — atomic read-modify-write. +pub fn atomic_rmw( + ctx: *mlir.Context, + kind: AtomicRMWKind, + value: *const mlir.Value, + memref: *const mlir.Value, + indices: []const *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + var buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; + buf.appendSliceAssumeCapacity(&.{ value, memref }); + buf.appendSliceAssumeCapacity(indices); + + return mlir.Operation.make(ctx, "memref.atomic_rmw", .{ + .operands = .{ .flat = buf.constSlice() }, + .results = .{ .flat = &.{value.type_()} }, + .attributes = &.{ + .named(ctx, "kind", .int(ctx, .i64, @backingInt(kind))), + }, + .location = location, + }); +} + +// ============================================================================= +// Casts / metadata-only ops +// ============================================================================= + +/// memref.cast — change layout/element-shape compatibility (e.g. unranked ↔ +/// ranked, dynamic ↔ static). Element type and total memory must match. +pub fn cast( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "memref.cast", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// memref.memory_space_cast — change the memory-space attribute on a memref. +pub fn memory_space_cast( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "memref.memory_space_cast", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// memref.assume_alignment — pure, returns the same memref with an alignment +/// hint attached. +pub fn assume_alignment( + ctx: *mlir.Context, + src: *const mlir.Value, + alignment: i32, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "memref.assume_alignment", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{src.type_()} }, + .attributes = &.{ + .named(ctx, "alignment", .int(ctx, .i32, alignment)), + }, + .location = location, + }); +} + +// ============================================================================= +// Shape introspection +// ============================================================================= + +/// memref.dim — returns the size of dim `index` of `memref` as `index`. +pub fn dim( + ctx: *mlir.Context, + memref: *const mlir.Value, + index: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "memref.dim", .{ + .operands = .{ .flat = &.{ memref, index } }, + .results = .{ .flat = &.{.index(ctx)} }, + .location = location, + }); +} + +/// memref.rank — returns rank of `memref` as `index`. +pub fn rank( + ctx: *mlir.Context, + memref: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "memref.rank", .{ + .operands = .{ .flat = &.{memref} }, + .results = .{ .flat = &.{.index(ctx)} }, + .location = location, + }); +} + +/// memref.extract_aligned_pointer_as_index — get the aligned base pointer of +/// a memref as an `index`. +pub fn extract_aligned_pointer_as_index( + ctx: *mlir.Context, + src: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "memref.extract_aligned_pointer_as_index", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{.index(ctx)} }, + .location = location, + }); +} + +// ============================================================================= +// View / SubView / Reinterpret +// ============================================================================= + +/// memref.view — type-changing view of a flat 1-D `memref` as a higher- +/// rank memref. `byte_shift` is an `index` byte offset; `dynamic_sizes` +/// supplies one operand per dynamic dim of `result_type`. +pub fn view( + ctx: *mlir.Context, + source: *const mlir.Value, + byte_shift: *const mlir.Value, + dynamic_sizes: []const *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + var buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; + buf.appendSliceAssumeCapacity(&.{ source, byte_shift }); + buf.appendSliceAssumeCapacity(dynamic_sizes); + + return mlir.Operation.make(ctx, "memref.view", .{ + .operands = .{ .flat = buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// Mixed static/dynamic offsets-sizes-strides — `static_*` slots holding +/// `kDynamic` consume one entry from the corresponding `dynamic_*` slice. +pub const SubViewArgs = struct { + static_offsets: []const i64, + static_sizes: []const i64, + static_strides: []const i64, + dynamic_offsets: []const *const mlir.Value = &.{}, + dynamic_sizes: []const *const mlir.Value = &.{}, + dynamic_strides: []const *const mlir.Value = &.{}, +}; + +/// memref.subview — strided view into a source memref. The result type must +/// be precomputed by the caller (the dialect provides `inferResultType`, +/// but we don't expose that yet — pass an explicit `result_type`). +pub fn subview( + ctx: *mlir.Context, + source: *const mlir.Value, + args: SubViewArgs, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 64) = .empty; + operands_buf.appendAssumeCapacity(source); + operands_buf.appendSliceAssumeCapacity(args.dynamic_offsets); + operands_buf.appendSliceAssumeCapacity(args.dynamic_sizes); + operands_buf.appendSliceAssumeCapacity(args.dynamic_strides); + + const seg_sizes = [4]i32{ + 1, + @intCast(args.dynamic_offsets.len), + @intCast(args.dynamic_sizes.len), + @intCast(args.dynamic_strides.len), + }; + + return mlir.Operation.make(ctx, "memref.subview", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), + .named(ctx, "static_offsets", .denseArray(ctx, .i64, args.static_offsets)), + .named(ctx, "static_sizes", .denseArray(ctx, .i64, args.static_sizes)), + .named(ctx, "static_strides", .denseArray(ctx, .i64, args.static_strides)), + }, + .location = location, + }); +} + +pub const ReinterpretCastArgs = struct { + static_offsets: []const i64, + static_sizes: []const i64, + static_strides: []const i64, + dynamic_offsets: []const *const mlir.Value = &.{}, + dynamic_sizes: []const *const mlir.Value = &.{}, + dynamic_strides: []const *const mlir.Value = &.{}, +}; + +/// memref.reinterpret_cast — changes offset/sizes/strides of a memref without +/// touching the underlying memory. +pub fn reinterpret_cast( + ctx: *mlir.Context, + source: *const mlir.Value, + args: ReinterpretCastArgs, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 64) = .empty; + operands_buf.appendAssumeCapacity(source); + operands_buf.appendSliceAssumeCapacity(args.dynamic_offsets); + operands_buf.appendSliceAssumeCapacity(args.dynamic_sizes); + operands_buf.appendSliceAssumeCapacity(args.dynamic_strides); + + const seg_sizes = [4]i32{ + 1, + @intCast(args.dynamic_offsets.len), + @intCast(args.dynamic_sizes.len), + @intCast(args.dynamic_strides.len), + }; + + return mlir.Operation.make(ctx, "memref.reinterpret_cast", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), + .named(ctx, "static_offsets", .denseArray(ctx, .i64, args.static_offsets)), + .named(ctx, "static_sizes", .denseArray(ctx, .i64, args.static_sizes)), + .named(ctx, "static_strides", .denseArray(ctx, .i64, args.static_strides)), + }, + .location = location, + }); +} + +// ============================================================================= +// Reshape — expand_shape / collapse_shape / reshape +// ============================================================================= + +/// memref.reshape — reshape a memref using a separate `shape` memref of `index` +/// elements. `result_type` carries the new shape. +pub fn reshape( + ctx: *mlir.Context, + source: *const mlir.Value, + shape: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "memref.reshape", .{ + .operands = .{ .flat = &.{ source, shape } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// `reassociation[i]` lists the source dimensions that map into result dim i +/// (for collapse_shape) or the result dimensions that come from source dim i +/// (for expand_shape). Each inner slice is stored as an `IntegerAttr` array. +pub const ReassociationArgs = struct { + /// Index groupings. + reassociation: []const []const i64, + /// For expand_shape only: per-result-dim sizes, with `kDynamic` for slots + /// resolved from `dynamic_output_shape`. Pass &.{} for collapse_shape. + static_output_shape: []const i64 = &.{}, + /// For expand_shape only: the dynamic dim values referenced by `kDynamic` + /// entries in `static_output_shape`. + dynamic_output_shape: []const *const mlir.Value = &.{}, +}; + +fn buildReassociationAttr(ctx: *mlir.Context, reassociation: []const []const i64) *const mlir.Attribute { + var outer: stdx.BoundedArray(*const mlir.Attribute, 32) = .empty; + var inner_buf: stdx.BoundedArray(*const mlir.Attribute, 32) = .empty; + for (reassociation) |group| { + inner_buf = .empty; + for (group) |idx| { + inner_buf.appendAssumeCapacity(.int(ctx, .i64, idx)); + } + outer.appendAssumeCapacity(.array(ctx, inner_buf.constSlice())); + } + return .array(ctx, outer.constSlice()); +} + +/// memref.expand_shape — split source dimensions into multiple result +/// dimensions. `static_output_shape` must have one entry per result dim; +/// dynamic entries are filled from `dynamic_output_shape` in order. +pub fn expand_shape( + ctx: *mlir.Context, + source: *const mlir.Value, + args: ReassociationArgs, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 32) = .empty; + operands_buf.appendAssumeCapacity(source); + operands_buf.appendSliceAssumeCapacity(args.dynamic_output_shape); + + return mlir.Operation.make(ctx, "memref.expand_shape", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "reassociation", buildReassociationAttr(ctx, args.reassociation)), + .named(ctx, "static_output_shape", .denseArray(ctx, .i64, args.static_output_shape)), + }, + .location = location, + }); +} + +/// memref.collapse_shape — merge consecutive source dimensions into single +/// result dimensions. Reassociation map is canonical. +pub fn collapse_shape( + ctx: *mlir.Context, + source: *const mlir.Value, + reassociation: []const []const i64, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "memref.collapse_shape", .{ + .operands = .{ .flat = &.{source} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "reassociation", buildReassociationAttr(ctx, reassociation)), + }, + .location = location, + }); +} + +// ============================================================================= +// Transpose — metadata-only permutation. Permutation is an AffineMapAttr. +// ============================================================================= + +/// memref.transpose — permute the dimensions of `src` according to +/// `permutation` (which must be a permutation of `(d0, ..., dN-1)`). Build +/// the map with `mlir.AffineMap.get` or parse via +/// `mlir.parseAffineMapAttribute`. +pub fn transpose( + ctx: *mlir.Context, + src: *const mlir.Value, + permutation: *const mlir.Attribute, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "memref.transpose", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "permutation", permutation), + }, + .location = location, + }); +} + +// ============================================================================= +// Copy / Prefetch +// ============================================================================= + +/// memref.copy — copy source memref into target memref (same element type, +/// same total element count). +pub fn copy( + ctx: *mlir.Context, + source: *const mlir.Value, + target: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "memref.copy", .{ + .operands = .{ .flat = &.{ source, target } }, + .location = location, + }); +} + +pub const PrefetchOpts = struct { + /// True for a write prefetch, false for a read prefetch. + is_write: bool = false, + /// 0 = no locality, 3 = extreme locality (keep in cache). + locality_hint: i32 = 3, + /// True for data cache, false for instruction cache. + is_data_cache: bool = true, +}; + +/// memref.prefetch — non-binding hint to bring `memref[indices]` into cache. +pub fn prefetch( + ctx: *mlir.Context, + memref: *const mlir.Value, + indices: []const *const mlir.Value, + opts: PrefetchOpts, + location: *const mlir.Location, +) *mlir.Operation { + var buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; + buf.appendAssumeCapacity(memref); + buf.appendSliceAssumeCapacity(indices); + + return mlir.Operation.make(ctx, "memref.prefetch", .{ + .operands = .{ .flat = buf.constSlice() }, + .attributes = &.{ + .named(ctx, "isWrite", .boolean(ctx, opts.is_write)), + .named(ctx, "localityHint", .int(ctx, .i32, opts.locality_hint)), + .named(ctx, "isDataCache", .boolean(ctx, opts.is_data_cache)), + }, + .location = location, + }); +} + +// ============================================================================= +// Globals +// ============================================================================= + +pub const GlobalArgs = struct { + pub const Visibility = enum { public, private, nested }; + + name: []const u8, + type_: *const mlir.Type, + /// Optional dense initial value. `null` ⇒ uninitialised global. + initial_value: ?*const mlir.Attribute = null, + visibility: Visibility = .public, + constant: bool = false, + alignment: ?i64 = null, +}; + +/// memref.global — declare a module-scope memref global (matches the symbol +/// referenced by `memref.get_global`). +pub fn global(ctx: *mlir.Context, args: GlobalArgs, location: *const mlir.Location) *mlir.Operation { + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 6) = .empty; + attrs.appendSliceAssumeCapacity(&.{ + .named(ctx, "sym_name", .string(ctx, args.name)), + .named(ctx, "sym_visibility", .string(ctx, @tagName(args.visibility))), + .named(ctx, "type", .typeAttr(args.type_)), + }); + if (args.initial_value) |iv| { + attrs.appendAssumeCapacity(.named(ctx, "initial_value", iv)); + } + if (args.constant) { + attrs.appendAssumeCapacity(.named(ctx, "constant", .unit(ctx))); + } + if (args.alignment) |a| { + attrs.appendAssumeCapacity(.named(ctx, "alignment", .int(ctx, .i64, a))); + } + + return mlir.Operation.make(ctx, "memref.global", .{ + .attributes = attrs.constSlice(), + .verify = false, + .location = location, + }); +} + +/// memref.get_global — fetch the value of a `memref.global` by symbol name. +pub fn get_global( + ctx: *mlir.Context, + name: []const u8, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "memref.get_global", .{ + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "name", .flatSymbolRef(ctx, name)), + }, + .location = location, + }); +} diff --git a/mlir/dialects/memref/test.zig b/mlir/dialects/memref/test.zig new file mode 100644 index 00000000..cddb74c5 --- /dev/null +++ b/mlir/dialects/memref/test.zig @@ -0,0 +1,77 @@ +const std = @import("std"); + +const memref = @import("mlir/dialects/memref"); +const mlir = @import("mlir"); + +const operation_names: []const []const u8 = &.{ + "memref.alloc", + "memref.alloca", + "memref.assume_alignment", + "memref.atomic_rmw", + "memref.cast", + "memref.collapse_shape", + "memref.copy", + "memref.dealloc", + "memref.dim", + "memref.expand_shape", + "memref.extract_aligned_pointer_as_index", + "memref.get_global", + "memref.global", + "memref.load", + "memref.memory_space_cast", + "memref.prefetch", + "memref.rank", + "memref.realloc", + "memref.reinterpret_cast", + "memref.reshape", + "memref.store", + "memref.subview", + "memref.transpose", + "memref.view", +}; + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(memref); + inline for (comptime std.meta.declarations(memref)) |decl| { + const T = @field(memref, decl); + if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); + } +} + +test "every bound operation is registered" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + memref.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} + +test "atomic_rmw kinds print as their IR spelling" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + memref.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + const loc = mlir.Location.unknown(ctx); + const block = mlir.Block.init(&.{ .float(ctx, .f32), try mlir.Type.parse(ctx, "memref<4xf32>"), .int(ctx, .i32), try mlir.Type.parse(ctx, "memref<4xi32>"), .index(ctx) }, &.{ loc, loc, loc, loc, loc }); + defer block.deinit(); + inline for (comptime std.meta.tags(memref.AtomicRMWKind)) |kind| { + const float = switch (kind) { + .addf, .assign, .maximumf, .maxnumf, .minimumf, .minnumf, .mulf => true, + else => false, + }; + const value = block.argument(if (float) 0 else 2); + const ref = block.argument(if (float) 1 else 3); + const op = memref.atomic_rmw(ctx, kind, value, ref, &.{block.argument(4)}, loc); + defer op.deinit(); + var text: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer text.deinit(); + try text.writer.print("{f}", .{op}); + try std.testing.expect(std.mem.indexOf(u8, text.written(), "memref.atomic_rmw " ++ @tagName(kind) ++ " ") != null); + } +} diff --git a/mlir/dialects/mosaic_tpu/BUILD.bazel b/mlir/dialects/mosaic_tpu/BUILD.bazel new file mode 100644 index 00000000..88e67265 --- /dev/null +++ b/mlir/dialects/mosaic_tpu/BUILD.bazel @@ -0,0 +1,38 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +# The types and attributes Mosaic's C API does not expose. +cc_library( + name = "capi", + srcs = ["capi.cc"], + hdrs = ["capi.h"], + tags = ["manual"], + deps = [ + "@llvm-project//llvm:Support", + "@llvm-project//mlir:CAPIIR", + "@xla//xla/mosaic:tpu_dialect", + ], +) + +zig_library( + name = "mosaic_tpu", + import_name = "mlir/dialects/mosaic_tpu", + main = "mosaic_tpu.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + ":capi", + "//mlir", + "//stdx", + "@xla//xla/mosaic:tpu_dialect_capi", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":mosaic_tpu", + "//mlir", + ], +) diff --git a/mlir/dialects/mosaic_tpu/capi.cc b/mlir/dialects/mosaic_tpu/capi.cc new file mode 100644 index 00000000..22662082 --- /dev/null +++ b/mlir/dialects/mosaic_tpu/capi.cc @@ -0,0 +1,273 @@ +//===- capi.cc - C API for the Mosaic tpu dialect -------------------------===// + +#include "capi.h" + +#include "xla/mosaic/dialect/tpu/tpu_dialect.h" + +#include "mlir/CAPI/IR.h" +#include "mlir/IR/Diagnostics.h" +#include "mlir/IR/Location.h" +#include "llvm/ADT/ArrayRef.h" + +#include +#include + +static mlir::MLIRContext *loadTPU(MlirContext context) { + mlir::MLIRContext *ctx = unwrap(context); + if (ctx) + ctx->getOrLoadDialect(); + return ctx; +} + +template +static MlirAttribute enumAttrGet(MlirContext context, + std::optional value) { + mlir::MLIRContext *ctx = loadTPU(context); + if (!ctx || !value) + return {nullptr}; + return wrap(AttrT::get(ctx, *value)); +} + +template static uint32_t enumAttrGetValue(MlirAttribute attr) { + return static_cast(llvm::cast(unwrap(attr)).getValue()); +} + +static llvm::ArrayRef arrayRef(intptr_t size, const int64_t *data) { + return llvm::ArrayRef(data, static_cast(size)); +} + +bool mlirTypeIsATpuSemaphore(MlirType type) { + return llvm::isa(unwrap(type)); +} + +MlirType mlirTpuSemaphoreTypeGet(MlirContext context) { + mlir::MLIRContext *ctx = loadTPU(context); + if (!ctx) + return {nullptr}; + return wrap(mlir::tpu::SemaphoreType::get(ctx)); +} + +bool mlirTypeIsATpuDMASemaphore(MlirType type) { + return llvm::isa(unwrap(type)); +} + +MlirType mlirTpuDMASemaphoreTypeGet(MlirContext context) { + mlir::MLIRContext *ctx = loadTPU(context); + if (!ctx) + return {nullptr}; + return wrap(mlir::tpu::DMASemaphoreType::get(ctx)); +} + +bool mlirAttributeIsATpuReductionKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirTpuReductionKindAttrGet(MlirContext context, uint32_t value) { + return enumAttrGet( + context, mlir::tpu::symbolizeReductionKind(value)); +} + +uint32_t mlirTpuReductionKindAttrGetValue(MlirAttribute attr) { + return enumAttrGetValue(attr); +} + +bool mlirAttributeIsATpuContractPrecision(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirTpuContractPrecisionAttrGet(MlirContext context, + uint32_t value) { + return enumAttrGet( + context, mlir::tpu::symbolizeContractPrecision(value)); +} + +uint32_t mlirTpuContractPrecisionAttrGetValue(MlirAttribute attr) { + return enumAttrGetValue(attr); +} + +bool mlirAttributeIsATpuRoundingMode(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirTpuRoundingModeAttrGet(MlirContext context, uint32_t value) { + return enumAttrGet( + context, mlir::tpu::symbolizeRoundingMode(value)); +} + +uint32_t mlirTpuRoundingModeAttrGetValue(MlirAttribute attr) { + return enumAttrGetValue(attr); +} + +bool mlirAttributeIsATpuCoreType(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirTpuCoreTypeAttrGet(MlirContext context, uint32_t value) { + return enumAttrGet( + context, mlir::tpu::symbolizeCoreType(value)); +} + +uint32_t mlirTpuCoreTypeAttrGetValue(MlirAttribute attr) { + return enumAttrGetValue(attr); +} + +bool mlirAttributeIsATpuDimensionSemantics(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirTpuDimensionSemanticsAttrGet(MlirContext context, + uint32_t value) { + return enumAttrGet( + context, mlir::tpu::symbolizeDimensionSemantics(value)); +} + +uint32_t mlirTpuDimensionSemanticsAttrGetValue(MlirAttribute attr) { + return enumAttrGetValue(attr); +} + +bool mlirAttributeIsATpuPipelineMode(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirTpuPipelineModeAttrGet(MlirContext context, uint32_t value) { + return enumAttrGet( + context, mlir::tpu::symbolizePipelineMode(value)); +} + +uint32_t mlirTpuPipelineModeAttrGetValue(MlirAttribute attr) { + return enumAttrGetValue(attr); +} + +bool mlirAttributeIsATpuRevisitMode(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirTpuRevisitModeAttrGet(MlirContext context, uint32_t value) { + return enumAttrGet( + context, mlir::tpu::symbolizeRevisitMode(value)); +} + +uint32_t mlirTpuRevisitModeAttrGetValue(MlirAttribute attr) { + return enumAttrGetValue(attr); +} + +bool mlirAttributeIsATpuMemorySpace(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirTpuMemorySpaceAttrGet(MlirContext context, uint32_t value, + int64_t coreType) { + mlir::MLIRContext *ctx = loadTPU(context); + std::optional memorySpace = + mlir::tpu::symbolizeMemorySpace(value); + if (!ctx || !memorySpace) + return {nullptr}; + std::optional core; + if (coreType >= 0) { + if (coreType > UINT32_MAX) + return {nullptr}; + core = mlir::tpu::symbolizeCoreType(static_cast(coreType)); + if (!core) + return {nullptr}; + } + return wrap(mlir::tpu::MemorySpaceAttr::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, + *memorySpace, core)); +} + +uint32_t mlirTpuMemorySpaceAttrGetValue(MlirAttribute attr) { + return enumAttrGetValue(attr); +} + +int64_t mlirTpuMemorySpaceAttrGetCoreType(MlirAttribute attr) { + std::optional core = + llvm::cast(unwrap(attr)).getCoreType(); + return core ? static_cast(*core) : -1; +} + +bool mlirAttributeIsATpuElementWindow(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirTpuElementWindowAttrGet(MlirContext context, + intptr_t numPadLow, + const int64_t *padLow, + intptr_t numPadHigh, + const int64_t *padHigh) { + mlir::MLIRContext *ctx = loadTPU(context); + if (!ctx) + return {nullptr}; + return wrap(mlir::tpu::ElementWindowAttr::get( + ctx, arrayRef(numPadLow, padLow), arrayRef(numPadHigh, padHigh))); +} + +intptr_t mlirTpuElementWindowAttrGetNumPadLow(MlirAttribute attr) { + return llvm::cast(unwrap(attr)) + .getPadLow() + .size(); +} + +int64_t mlirTpuElementWindowAttrGetPadLow(MlirAttribute attr, intptr_t pos) { + return llvm::cast(unwrap(attr)) + .getPadLow()[pos]; +} + +intptr_t mlirTpuElementWindowAttrGetNumPadHigh(MlirAttribute attr) { + return llvm::cast(unwrap(attr)) + .getPadHigh() + .size(); +} + +int64_t mlirTpuElementWindowAttrGetPadHigh(MlirAttribute attr, intptr_t pos) { + return llvm::cast(unwrap(attr)) + .getPadHigh()[pos]; +} + +bool mlirAttributeIsATpuDotDimensionNumbers(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} + +MlirAttribute mlirTpuDotDimensionNumbersAttrGet( + MlirContext context, intptr_t numLhsContractingDims, + const int64_t *lhsContractingDims, intptr_t numRhsContractingDims, + const int64_t *rhsContractingDims, intptr_t numLhsNonContractingDims, + const int64_t *lhsNonContractingDims, intptr_t numRhsNonContractingDims, + const int64_t *rhsNonContractingDims, intptr_t numOutputDimOrder, + const int64_t *outputDimOrder, intptr_t numLhsBatchDims, + const int64_t *lhsBatchDims, intptr_t numRhsBatchDims, + const int64_t *rhsBatchDims) { + mlir::MLIRContext *ctx = loadTPU(context); + if (!ctx) + return {nullptr}; + return wrap(mlir::tpu::DotDimensionNumbersAttr::get( + ctx, arrayRef(numLhsContractingDims, lhsContractingDims), + arrayRef(numRhsContractingDims, rhsContractingDims), + arrayRef(numLhsNonContractingDims, lhsNonContractingDims), + arrayRef(numRhsNonContractingDims, rhsNonContractingDims), + arrayRef(numOutputDimOrder, outputDimOrder), + arrayRef(numLhsBatchDims, lhsBatchDims), + arrayRef(numRhsBatchDims, rhsBatchDims))); +} + +#define ZML_TPU_DOT_DIMS_GETTERS(Name) \ + intptr_t mlirTpuDotDimensionNumbersAttrGetNum##Name(MlirAttribute attr) { \ + return llvm::cast(unwrap(attr)) \ + .get##Name() \ + .size(); \ + } \ + int64_t mlirTpuDotDimensionNumbersAttrGet##Name(MlirAttribute attr, \ + intptr_t pos) { \ + return llvm::cast(unwrap(attr)) \ + .get##Name()[pos]; \ + } + +ZML_TPU_DOT_DIMS_GETTERS(LhsContractingDims) +ZML_TPU_DOT_DIMS_GETTERS(RhsContractingDims) +ZML_TPU_DOT_DIMS_GETTERS(LhsNonContractingDims) +ZML_TPU_DOT_DIMS_GETTERS(RhsNonContractingDims) +ZML_TPU_DOT_DIMS_GETTERS(OutputDimOrder) +ZML_TPU_DOT_DIMS_GETTERS(LhsBatchDims) +ZML_TPU_DOT_DIMS_GETTERS(RhsBatchDims) + +#undef ZML_TPU_DOT_DIMS_GETTERS diff --git a/mlir/dialects/mosaic_tpu/capi.h b/mlir/dialects/mosaic_tpu/capi.h new file mode 100644 index 00000000..2e1cc0bb --- /dev/null +++ b/mlir/dialects/mosaic_tpu/capi.h @@ -0,0 +1,145 @@ +//===- capi.h - C API for the Mosaic tpu dialect ------------------*- C -*-===// +// +// C entry points for the `tpu` dialect that Mosaic's C API +// (`tpu_dialect.h`) leaves out: the semaphore types and the attributes +// kernels build. + +#ifndef MLIR_DIALECTS_MOSAIC_TPU_CAPI_H +#define MLIR_DIALECTS_MOSAIC_TPU_CAPI_H + +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// !tpu.semaphore +MLIR_CAPI_EXPORTED bool mlirTypeIsATpuSemaphore(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirTpuSemaphoreTypeGet(MlirContext context); + +// !tpu.dma_semaphore +MLIR_CAPI_EXPORTED bool mlirTypeIsATpuDMASemaphore(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirTpuDMASemaphoreTypeGet(MlirContext context); + +// Enum attributes take and return the C++ enum's integer value (tpu_ops.td); +// `Get` returns a null attribute for a value outside the enum. + +// #tpu.reduction_kind<...> +MLIR_CAPI_EXPORTED bool mlirAttributeIsATpuReductionKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirTpuReductionKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirTpuReductionKindAttrGetValue(MlirAttribute attr); + +// #tpu.contract_precision<...> +MLIR_CAPI_EXPORTED bool +mlirAttributeIsATpuContractPrecision(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirTpuContractPrecisionAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirTpuContractPrecisionAttrGetValue(MlirAttribute attr); + +// #tpu.rounding_mode<...> +MLIR_CAPI_EXPORTED bool mlirAttributeIsATpuRoundingMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirTpuRoundingModeAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirTpuRoundingModeAttrGetValue(MlirAttribute attr); + +// #tpu.core_type<...> +MLIR_CAPI_EXPORTED bool mlirAttributeIsATpuCoreType(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirTpuCoreTypeAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirTpuCoreTypeAttrGetValue(MlirAttribute attr); + +// #tpu.dimension_semantics<...> +MLIR_CAPI_EXPORTED bool +mlirAttributeIsATpuDimensionSemantics(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirTpuDimensionSemanticsAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirTpuDimensionSemanticsAttrGetValue(MlirAttribute attr); + +// #tpu.pipeline_mode<...> +MLIR_CAPI_EXPORTED bool mlirAttributeIsATpuPipelineMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirTpuPipelineModeAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirTpuPipelineModeAttrGetValue(MlirAttribute attr); + +// #tpu.revisit_mode<...> +MLIR_CAPI_EXPORTED bool mlirAttributeIsATpuRevisitMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirTpuRevisitModeAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirTpuRevisitModeAttrGetValue(MlirAttribute attr); + +// #tpu.memory_space: a negative `coreType` leaves the +// optional core type out, and `GetCoreType` returns -1 when it is absent. +MLIR_CAPI_EXPORTED bool mlirAttributeIsATpuMemorySpace(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirTpuMemorySpaceAttrGet(MlirContext context, + uint32_t value, + int64_t coreType); +MLIR_CAPI_EXPORTED uint32_t mlirTpuMemorySpaceAttrGetValue(MlirAttribute attr); +MLIR_CAPI_EXPORTED int64_t +mlirTpuMemorySpaceAttrGetCoreType(MlirAttribute attr); + +// #tpu.element_window<[pad_low], [pad_high]> +MLIR_CAPI_EXPORTED bool mlirAttributeIsATpuElementWindow(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirTpuElementWindowAttrGet( + MlirContext context, intptr_t numPadLow, const int64_t *padLow, + intptr_t numPadHigh, const int64_t *padHigh); +MLIR_CAPI_EXPORTED intptr_t +mlirTpuElementWindowAttrGetNumPadLow(MlirAttribute attr); +MLIR_CAPI_EXPORTED int64_t mlirTpuElementWindowAttrGetPadLow(MlirAttribute attr, + intptr_t pos); +MLIR_CAPI_EXPORTED intptr_t +mlirTpuElementWindowAttrGetNumPadHigh(MlirAttribute attr); +MLIR_CAPI_EXPORTED int64_t +mlirTpuElementWindowAttrGetPadHigh(MlirAttribute attr, intptr_t pos); + +// #tpu.dot_dimension_numbers<...>: the dimension lists in ODS order. +MLIR_CAPI_EXPORTED bool +mlirAttributeIsATpuDotDimensionNumbers(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirTpuDotDimensionNumbersAttrGet( + MlirContext context, intptr_t numLhsContractingDims, + const int64_t *lhsContractingDims, intptr_t numRhsContractingDims, + const int64_t *rhsContractingDims, intptr_t numLhsNonContractingDims, + const int64_t *lhsNonContractingDims, intptr_t numRhsNonContractingDims, + const int64_t *rhsNonContractingDims, intptr_t numOutputDimOrder, + const int64_t *outputDimOrder, intptr_t numLhsBatchDims, + const int64_t *lhsBatchDims, intptr_t numRhsBatchDims, + const int64_t *rhsBatchDims); +MLIR_CAPI_EXPORTED intptr_t +mlirTpuDotDimensionNumbersAttrGetNumLhsContractingDims(MlirAttribute attr); +MLIR_CAPI_EXPORTED int64_t mlirTpuDotDimensionNumbersAttrGetLhsContractingDims( + MlirAttribute attr, intptr_t pos); +MLIR_CAPI_EXPORTED intptr_t +mlirTpuDotDimensionNumbersAttrGetNumRhsContractingDims(MlirAttribute attr); +MLIR_CAPI_EXPORTED int64_t mlirTpuDotDimensionNumbersAttrGetRhsContractingDims( + MlirAttribute attr, intptr_t pos); +MLIR_CAPI_EXPORTED intptr_t +mlirTpuDotDimensionNumbersAttrGetNumLhsNonContractingDims(MlirAttribute attr); +MLIR_CAPI_EXPORTED int64_t +mlirTpuDotDimensionNumbersAttrGetLhsNonContractingDims(MlirAttribute attr, + intptr_t pos); +MLIR_CAPI_EXPORTED intptr_t +mlirTpuDotDimensionNumbersAttrGetNumRhsNonContractingDims(MlirAttribute attr); +MLIR_CAPI_EXPORTED int64_t +mlirTpuDotDimensionNumbersAttrGetRhsNonContractingDims(MlirAttribute attr, + intptr_t pos); +MLIR_CAPI_EXPORTED intptr_t +mlirTpuDotDimensionNumbersAttrGetNumOutputDimOrder(MlirAttribute attr); +MLIR_CAPI_EXPORTED int64_t mlirTpuDotDimensionNumbersAttrGetOutputDimOrder( + MlirAttribute attr, intptr_t pos); +MLIR_CAPI_EXPORTED intptr_t +mlirTpuDotDimensionNumbersAttrGetNumLhsBatchDims(MlirAttribute attr); +MLIR_CAPI_EXPORTED int64_t +mlirTpuDotDimensionNumbersAttrGetLhsBatchDims(MlirAttribute attr, intptr_t pos); +MLIR_CAPI_EXPORTED intptr_t +mlirTpuDotDimensionNumbersAttrGetNumRhsBatchDims(MlirAttribute attr); +MLIR_CAPI_EXPORTED int64_t +mlirTpuDotDimensionNumbersAttrGetRhsBatchDims(MlirAttribute attr, intptr_t pos); + +#ifdef __cplusplus +} +#endif + +#endif // MLIR_DIALECTS_MOSAIC_TPU_CAPI_H diff --git a/mlir/dialects/mosaic_tpu/mosaic_tpu.zig b/mlir/dialects/mosaic_tpu/mosaic_tpu.zig new file mode 100644 index 00000000..8425e7b4 --- /dev/null +++ b/mlir/dialects/mosaic_tpu/mosaic_tpu.zig @@ -0,0 +1,1556 @@ +const std = @import("std"); + +const c = @import("c"); +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +/// Mosaic IR version stored as `stable_mosaic.version`, matching +/// `xla/mosaic/dialect/tpu/transforms/serde.h:kVersion`. This is distinct from +/// the custom-call JSON `serialization_format`, which remains 1. +pub const SERDE_VERSION: i32 = 18; + +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("tpu"); +} + +/// Register the `mosaic-serde` pass so `PassManager.parse` can pick it up by +/// name. Idempotent — safe to call repeatedly. Required before running the +/// `mosaic-serde{serialize=true}` pipeline that produces the bytecode embedded +/// in `tpu_custom_call`'s backend_config. +pub fn registerMosaicSerdePass() void { + c.mlirTpuRegisterMosaicSerdePass(); +} + +// ============================================================================= +// Types — !tpu.semaphore, !tpu.dma_semaphore, !tpu.float8_exmy<...> +// ============================================================================= + +/// `!tpu.semaphore`. +pub const SemaphoreType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsATpuSemaphore; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "semaphore"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirTpuSemaphoreTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// `!tpu.semaphore` element type. +pub fn semaphoreType(ctx: *mlir.Context) *const mlir.Type { + const ty = SemaphoreType.get(ctx, .{}) catch unreachable; + return ty.type_(); +} + +/// `!tpu.dma_semaphore`. +pub const DMASemaphoreType = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirType); + pub const isAFn = c.mlirTypeIsATpuDMASemaphore; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const mnemonic = "dma_semaphore"; + pub fn type_(self: *const Self) *const mlir.Type { + return @ptrCast(self); + } + pub const InitArgs = struct {}; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + _ = args; + const result = c.mlirTpuDMASemaphoreTypeGet(ctx.ptr()); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } +}; + +/// Build `!tpu.dma_semaphore`. +pub fn dmaSemaphoreType(ctx: *mlir.Context) *const mlir.Type { + const ty = DMASemaphoreType.get(ctx, .{}) catch unreachable; + return ty.type_(); +} + +/// `!tpu.float8_exmy` — Mosaic Float8 type. +pub const Float8EXMYType = opaque { + const M = mlir.Methods(Float8EXMYType, c.MlirType); + + pub const isAFn = c.mlirTpuIsAFloat8EXMYType; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + + pub fn get(ctx: *mlir.Context, underlying: *const mlir.Type) *const Float8EXMYType { + return @ptrCast(c.mlirTpuFloat8EXMYTypeGet(ctx.ptr(), underlying.ptr()).ptr); + } + + pub fn underlyingType(self: *const Float8EXMYType) *const mlir.Type { + return @ptrCast(c.mlirTpuFloat8EXMYTypeGetUnderlyingType(self.ptr()).ptr); + } + + pub fn type_(self: *const Float8EXMYType) *const mlir.Type { + return @ptrCast(self); + } +}; + +pub fn float8ExmyType(ctx: *mlir.Context, underlying: *const mlir.Type) *const mlir.Type { + return @ptrCast(Float8EXMYType.get(ctx, underlying)); +} + +// ============================================================================= +// Attributes +// ============================================================================= + +/// Reduction kind for `tpu.all_reduce`, `tpu.reduce_index`, `tpu.scan`. +pub const ReductionKind = enum(u32) { + sum = 0, + max = 1, + min = 2, + arg_max = 3, + arg_min = 4, + find_first_set = 5, + maxf = 6, + minf = 7, + maxsi = 8, + minsi = 9, + maxui = 10, + minui = 11, + + pub fn attribute(self: ReductionKind, ctx: *mlir.Context) *const mlir.Attribute { + const attr = ReductionKindAttr.get(ctx, .{ .value = self }) catch unreachable; + return attr.attribute(); + } + + fn forInput(self: ReductionKind, input: *const mlir.Value, unsigned_integer: bool) ReductionKind { + if (self != .max and self != .min) return self; + const element_type = input.type_().isA(mlir.ShapedType).?.elementType(); + if (element_type.isFloat()) return if (self == .max) .maxf else .minf; + // Match Mosaic's serde upgrade: scans used unsigned integer extrema, + // while all-reduce used signed integer extrema. + return if (unsigned_integer) + (if (self == .max) .maxui else .minui) + else + (if (self == .max) .maxsi else .minsi); + } +}; + +/// `#tpu.reduction_kind<...>`. +pub const ReductionKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsATpuReductionKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "reduction_kind"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: ReductionKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirTpuReductionKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) ReductionKind { + return @fromBackingInt(@intCast(c.mlirTpuReductionKindAttrGetValue(self.ptr()))); + } +}; + +/// Matmul precision setting of `tpu.matmul`. +pub const ContractPrecision = enum(u32) { + bf16 = 0, + fp32 = 1, + bf16x3 = 2, + + pub fn attribute(self: ContractPrecision, ctx: *mlir.Context) *const mlir.Attribute { + const attr = ContractPrecisionAttr.get(ctx, .{ .value = self }) catch unreachable; + return attr.attribute(); + } +}; + +/// `#tpu.contract_precision<...>`. +pub const ContractPrecisionAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsATpuContractPrecision; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "contract_precision"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: ContractPrecision, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirTpuContractPrecisionAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) ContractPrecision { + return @fromBackingInt(@intCast(c.mlirTpuContractPrecisionAttrGetValue(self.ptr()))); + } +}; + +/// Rounding mode of `tpu.fptosi` / `tpu.fptoui` / etc. +pub const RoundingMode = enum(u32) { + towards_zero = 0, + to_nearest_even = 1, + + pub fn attribute(self: RoundingMode, ctx: *mlir.Context) *const mlir.Attribute { + const attr = RoundingModeAttr.get(ctx, .{ .value = self }) catch unreachable; + return attr.attribute(); + } +}; + +/// `#tpu.rounding_mode<...>`. +pub const RoundingModeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsATpuRoundingMode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "rounding_mode"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: RoundingMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirTpuRoundingModeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) RoundingMode { + return @fromBackingInt(@intCast(c.mlirTpuRoundingModeAttrGetValue(self.ptr()))); + } +}; + +/// Target core of a kernel, remote DMAs and `tpu.sem_signal`. +pub const CoreType = enum(u32) { + tc = 0, + sc_scalar_subcore = 1, + sc_vector_subcore = 2, + + pub fn attribute(self: CoreType, ctx: *mlir.Context) *const mlir.Attribute { + const attr = CoreTypeAttr.get(ctx, .{ .value = self }) catch unreachable; + return attr.attribute(); + } +}; + +/// `#tpu.core_type<...>`. +pub const CoreTypeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsATpuCoreType; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "core_type"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: CoreType, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirTpuCoreTypeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) CoreType { + return @fromBackingInt(@intCast(c.mlirTpuCoreTypeAttrGetValue(self.ptr()))); + } +}; + +/// Grid dimension semantics annotation. +pub const DimensionSemantics = enum(u32) { + parallel = 0, + arbitrary = 1, + core_parallel = 2, + subcore_parallel = 3, + + pub fn attribute(self: DimensionSemantics, ctx: *mlir.Context) *const mlir.Attribute { + const attr = DimensionSemanticsAttr.get(ctx, .{ .value = self }) catch unreachable; + return attr.attribute(); + } +}; + +/// `#tpu.dimension_semantics<...>`. +pub const DimensionSemanticsAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsATpuDimensionSemantics; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "dimension_semantics"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: DimensionSemantics, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirTpuDimensionSemanticsAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) DimensionSemantics { + return @fromBackingInt(@intCast(c.mlirTpuDimensionSemanticsAttrGetValue(self.ptr()))); + } +}; + +/// Buffering mode of a `window_params` entry. +pub const PipelineMode = enum(u32) { + synchronous = 1, + double_buffered = 2, + + pub fn attribute(self: PipelineMode, ctx: *mlir.Context) *const mlir.Attribute { + const attr = PipelineModeAttr.get(ctx, .{ .value = self }) catch unreachable; + return attr.attribute(); + } +}; + +/// `#tpu.pipeline_mode<...>`. +pub const PipelineModeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsATpuPipelineMode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "pipeline_mode"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: PipelineMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirTpuPipelineModeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) PipelineMode { + return @fromBackingInt(@intCast(c.mlirTpuPipelineModeAttrGetValue(self.ptr()))); + } +}; + +/// Block revisit mode of a `window_params` entry. +pub const RevisitMode = enum(u32) { + immediate = 0, + any = 1, + + pub fn attribute(self: RevisitMode, ctx: *mlir.Context) *const mlir.Attribute { + const attr = RevisitModeAttr.get(ctx, .{ .value = self }) catch unreachable; + return attr.attribute(); + } +}; + +/// `#tpu.revisit_mode<...>`. +pub const RevisitModeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsATpuRevisitMode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "revisit_mode"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: RevisitMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirTpuRevisitModeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) RevisitMode { + return @fromBackingInt(@intCast(c.mlirTpuRevisitModeAttrGetValue(self.ptr()))); + } +}; + +/// Memory space of a memref, `#tpu.memory_space<...>`. +pub const MemorySpace = enum(u32) { + any = 0xffff_ffff, + vmem = 0, + smem = 1, + hbm = 2, + cmem = 3, + semaphore_mem = 4, + vmem_shared = 5, + host = 6, + + pub fn attribute(self: MemorySpace, ctx: *mlir.Context) *const mlir.Attribute { + const attr = MemorySpaceAttr.get(ctx, .{ .value = self }) catch unreachable; + return attr.attribute(); + } +}; + +/// `#tpu.memory_space` — passed as the `memory_space` of a memref's type. +pub const MemorySpaceAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsATpuMemorySpace; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "memory_space"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MemorySpace, + coreType: ?CoreType = null, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const core_type: i64 = if (args.coreType) |ct| @backingInt(ct) else -1; + const result = c.mlirTpuMemorySpaceAttrGet(ctx.ptr(), @backingInt(args.value), core_type); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) MemorySpace { + return @fromBackingInt(@intCast(c.mlirTpuMemorySpaceAttrGetValue(self.ptr()))); + } + pub fn coreType(self: *const Self) ?CoreType { + const core_type = c.mlirTpuMemorySpaceAttrGetCoreType(self.ptr()); + return if (core_type < 0) null else @fromBackingInt(@intCast(core_type)); + } +}; + +/// `#tpu.element_window<[pad_low], [pad_high]>` — per-arg window padding. +pub const ElementWindowAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsATpuElementWindow; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "element_window"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + padLow: []const i64, + padHigh: []const i64, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirTpuElementWindowAttrGet( + ctx.ptr(), + @intCast(args.padLow.len), + args.padLow.ptr, + @intCast(args.padHigh.len), + args.padHigh.ptr, + ); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn numPadLow(self: *const Self) usize { + return @intCast(c.mlirTpuElementWindowAttrGetNumPadLow(self.ptr())); + } + pub fn padLow(self: *const Self, pos: usize) i64 { + return c.mlirTpuElementWindowAttrGetPadLow(self.ptr(), @intCast(pos)); + } + pub fn numPadHigh(self: *const Self) usize { + return @intCast(c.mlirTpuElementWindowAttrGetNumPadHigh(self.ptr())); + } + pub fn padHigh(self: *const Self, pos: usize) i64 { + return c.mlirTpuElementWindowAttrGetPadHigh(self.ptr(), @intCast(pos)); + } +}; + +/// `#tpu.element_window<[pad_low], [pad_high]>` — per-arg window padding. +pub fn elementWindowAttr( + ctx: *mlir.Context, + pad_low: []const i64, + pad_high: []const i64, +) *const mlir.Attribute { + const attr = ElementWindowAttr.get(ctx, .{ .padLow = pad_low, .padHigh = pad_high }) catch unreachable; + return attr.attribute(); +} + +/// `#tpu.dot_dimension_numbers<...>` — the dimension mapping of `tpu.matmul`. +pub const DotDimensionNumbersAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsATpuDotDimensionNumbers; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "dot_dimension_numbers"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + lhsContractingDims: []const i64, + rhsContractingDims: []const i64, + lhsNonContractingDims: []const i64, + /// Empty when rhs is a 1-D vector. + rhsNonContractingDims: []const i64 = &.{}, + /// Flattened (operand, dim) pairs: operand 0 is lhs, 1 is rhs. + outputDimOrder: []const i64, + lhsBatchDims: []const i64 = &.{}, + rhsBatchDims: []const i64 = &.{}, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirTpuDotDimensionNumbersAttrGet( + ctx.ptr(), + @intCast(args.lhsContractingDims.len), + args.lhsContractingDims.ptr, + @intCast(args.rhsContractingDims.len), + args.rhsContractingDims.ptr, + @intCast(args.lhsNonContractingDims.len), + args.lhsNonContractingDims.ptr, + @intCast(args.rhsNonContractingDims.len), + args.rhsNonContractingDims.ptr, + @intCast(args.outputDimOrder.len), + args.outputDimOrder.ptr, + @intCast(args.lhsBatchDims.len), + args.lhsBatchDims.ptr, + @intCast(args.rhsBatchDims.len), + args.rhsBatchDims.ptr, + ); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn numLhsContractingDims(self: *const Self) usize { + return @intCast(c.mlirTpuDotDimensionNumbersAttrGetNumLhsContractingDims(self.ptr())); + } + pub fn lhsContractingDims(self: *const Self, pos: usize) i64 { + return c.mlirTpuDotDimensionNumbersAttrGetLhsContractingDims(self.ptr(), @intCast(pos)); + } + pub fn numRhsContractingDims(self: *const Self) usize { + return @intCast(c.mlirTpuDotDimensionNumbersAttrGetNumRhsContractingDims(self.ptr())); + } + pub fn rhsContractingDims(self: *const Self, pos: usize) i64 { + return c.mlirTpuDotDimensionNumbersAttrGetRhsContractingDims(self.ptr(), @intCast(pos)); + } + pub fn numLhsNonContractingDims(self: *const Self) usize { + return @intCast(c.mlirTpuDotDimensionNumbersAttrGetNumLhsNonContractingDims(self.ptr())); + } + pub fn lhsNonContractingDims(self: *const Self, pos: usize) i64 { + return c.mlirTpuDotDimensionNumbersAttrGetLhsNonContractingDims(self.ptr(), @intCast(pos)); + } + pub fn numRhsNonContractingDims(self: *const Self) usize { + return @intCast(c.mlirTpuDotDimensionNumbersAttrGetNumRhsNonContractingDims(self.ptr())); + } + pub fn rhsNonContractingDims(self: *const Self, pos: usize) i64 { + return c.mlirTpuDotDimensionNumbersAttrGetRhsNonContractingDims(self.ptr(), @intCast(pos)); + } + pub fn numOutputDimOrder(self: *const Self) usize { + return @intCast(c.mlirTpuDotDimensionNumbersAttrGetNumOutputDimOrder(self.ptr())); + } + pub fn outputDimOrder(self: *const Self, pos: usize) i64 { + return c.mlirTpuDotDimensionNumbersAttrGetOutputDimOrder(self.ptr(), @intCast(pos)); + } + pub fn numLhsBatchDims(self: *const Self) usize { + return @intCast(c.mlirTpuDotDimensionNumbersAttrGetNumLhsBatchDims(self.ptr())); + } + pub fn lhsBatchDims(self: *const Self, pos: usize) i64 { + return c.mlirTpuDotDimensionNumbersAttrGetLhsBatchDims(self.ptr(), @intCast(pos)); + } + pub fn numRhsBatchDims(self: *const Self) usize { + return @intCast(c.mlirTpuDotDimensionNumbersAttrGetNumRhsBatchDims(self.ptr())); + } + pub fn rhsBatchDims(self: *const Self, pos: usize) i64 { + return c.mlirTpuDotDimensionNumbersAttrGetRhsBatchDims(self.ptr(), @intCast(pos)); + } +}; + +// ============================================================================= +// Discardable attributes — tpu.core_type, tpu.dimension_semantics +// ============================================================================= + +/// `tpu.core_type`, on a kernel's `func.func`: the core it runs on. +pub fn coreTypeNamedAttr(ctx: *mlir.Context, value: CoreType) mlir.NamedAttribute { + return .named(ctx, "tpu.core_type", value.attribute(ctx)); +} + +/// `tpu.dimension_semantics`, on a kernel's iteration index argument. +pub fn dimensionSemanticsNamedAttr(ctx: *mlir.Context, value: DimensionSemantics) mlir.NamedAttribute { + return .named(ctx, "tpu.dimension_semantics", value.attribute(ctx)); +} + +// ============================================================================= +// Reductions / scan / sort +// ============================================================================= + +pub fn all_reduce( + ctx: *mlir.Context, + input: *const mlir.Value, + dim: i64, + kind: ReductionKind, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.all_reduce", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "dim", .int(ctx, .i64, dim)), + .named(ctx, "kind", kind.forInput(input, false).attribute(ctx)), + }, + .location = location, + }); +} + +pub fn reduce_index( + ctx: *mlir.Context, + input: *const mlir.Value, + axis: i32, + kind: ReductionKind, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.reduce_index", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "axis", .int(ctx, .i32, axis)), + .named(ctx, "kind", kind.attribute(ctx)), + }, + .location = location, + }); +} + +pub const ScanOpts = struct { + mask: ?*const mlir.Value = null, +}; + +/// Scan the last dimension; an optional rank-one mask spans that dimension. +pub fn scan( + ctx: *mlir.Context, + input: *const mlir.Value, + kind: ReductionKind, + result_type: *const mlir.Type, + opts: ScanOpts, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 2) = .empty; + operands_buf.appendAssumeCapacity(input); + if (opts.mask) |m| operands_buf.appendAssumeCapacity(m); + + return mlir.Operation.make(ctx, "tpu.scan", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "kind", kind.forInput(input, true).attribute(ctx)), + .named(ctx, "dimension", .int(ctx, .i64, @as(i64, @intCast(input.type_().isA(mlir.ShapedType).?.rank())) - 1)), + }, + .location = location, + }); +} + +pub const SortOpts = struct { + mask: ?*const mlir.Value = null, + + descending: bool = false, +}; + +pub fn sort( + ctx: *mlir.Context, + keys: *const mlir.Value, + values: *const mlir.Value, + output_mask_type: *const mlir.Type, + sorted_keys_type: *const mlir.Type, + sorted_values_type: *const mlir.Type, + opts: SortOpts, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 3) = .empty; + operands_buf.appendSliceAssumeCapacity(&.{ keys, values }); + if (opts.mask) |m| operands_buf.appendAssumeCapacity(m); + + return mlir.Operation.make(ctx, "tpu.sort", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{ output_mask_type, sorted_keys_type, sorted_values_type } }, + .attributes = &.{ + .named(ctx, "descending", .boolean(ctx, opts.descending)), + }, + .location = location, + }); +} + +// ============================================================================= +// Memory — load / store / vector_load / vector_store / strided_* +// ============================================================================= + +pub const LoadOpts = struct { + sublane_mask: []const bool, + sublane_stride: i32 = 1, +}; + +/// tpu.load — sublane-aware load into a vreg. +pub fn load( + ctx: *mlir.Context, + base: *const mlir.Value, + indices: []const *const mlir.Value, + result_type: *const mlir.Type, + opts: LoadOpts, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; + operands_buf.appendAssumeCapacity(base); + operands_buf.appendSliceAssumeCapacity(indices); + + return mlir.Operation.make(ctx, "tpu.load", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "sublane_mask", denseBoolArrayAttribute(ctx, opts.sublane_mask)), + .named(ctx, "sublane_stride", .int(ctx, .i32, opts.sublane_stride)), + }, + .location = location, + }); +} + +pub const StoreOpts = struct { + mask: ?*const mlir.Value = null, + + sublane_mask: []const bool, + sublane_stride: i32 = 1, + add: bool = false, +}; + +/// tpu.store — sublane-aware store from a vreg. Optional elementwise mask. +pub fn store( + ctx: *mlir.Context, + value: *const mlir.Value, + base: *const mlir.Value, + indices: []const *const mlir.Value, + opts: StoreOpts, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 18) = .empty; + operands_buf.appendSliceAssumeCapacity(&.{ value, base }); + operands_buf.appendSliceAssumeCapacity(indices); + const mask_len: i32 = if (opts.mask) |m| blk: { + operands_buf.appendAssumeCapacity(m); + break :blk 1; + } else 0; + + const seg_sizes = [4]i32{ 1, 1, @intCast(indices.len), mask_len }; + return mlir.Operation.make(ctx, "tpu.store", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .attributes = &.{ + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), + .named(ctx, "sublane_mask", denseBoolArrayAttribute(ctx, opts.sublane_mask)), + .named(ctx, "sublane_stride", .int(ctx, .i32, opts.sublane_stride)), + .named(ctx, "add", .boolean(ctx, opts.add)), + }, + .location = location, + }); +} + +pub const VectorLoadOpts = struct { + mask: ?*const mlir.Value = null, + + /// Per-dim load stride. Pass `&.{}` for unit-strided. + strides: []const i32 = &.{}, +}; + +/// tpu.vector_load — multi-dim strided load with optional elementwise mask. +pub fn vector_load( + ctx: *mlir.Context, + base: *const mlir.Value, + indices: []const *const mlir.Value, + result_type: *const mlir.Type, + opts: VectorLoadOpts, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 17) = .empty; + operands_buf.appendAssumeCapacity(base); + operands_buf.appendSliceAssumeCapacity(indices); + const mask_len: i32 = if (opts.mask) |m| blk: { + operands_buf.appendAssumeCapacity(m); + break :blk 1; + } else 0; + + const seg_sizes = [3]i32{ 1, @intCast(indices.len), mask_len }; + return mlir.Operation.make(ctx, "tpu.vector_load", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), + .named(ctx, "strides", .denseArray(ctx, .i32, opts.strides)), + }, + .location = location, + }); +} + +pub const VectorStoreOpts = struct { + mask: ?*const mlir.Value = null, + + strides: []const i32 = &.{}, + add: bool = false, +}; + +pub fn vector_store( + ctx: *mlir.Context, + value: *const mlir.Value, + base: *const mlir.Value, + indices: []const *const mlir.Value, + opts: VectorStoreOpts, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 18) = .empty; + operands_buf.appendSliceAssumeCapacity(&.{ value, base }); + operands_buf.appendSliceAssumeCapacity(indices); + const mask_len: i32 = if (opts.mask) |m| blk: { + operands_buf.appendAssumeCapacity(m); + break :blk 1; + } else 0; + + const seg_sizes = [4]i32{ 1, 1, @intCast(indices.len), mask_len }; + return mlir.Operation.make(ctx, "tpu.vector_store", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .attributes = &.{ + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), + .named(ctx, "strides", .denseArray(ctx, .i32, opts.strides)), + .named(ctx, "add", .boolean(ctx, opts.add)), + }, + .location = location, + }); +} + +pub fn strided_load( + ctx: *mlir.Context, + base: *const mlir.Value, + indices: []const *const mlir.Value, + strides: []const i32, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; + operands_buf.appendAssumeCapacity(base); + operands_buf.appendSliceAssumeCapacity(indices); + + return mlir.Operation.make(ctx, "tpu.strided_load", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "strides", .denseArray(ctx, .i32, strides)), + }, + .location = location, + }); +} + +pub fn strided_store( + ctx: *mlir.Context, + value: *const mlir.Value, + base: *const mlir.Value, + indices: []const *const mlir.Value, + strides: []const i32, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 17) = .empty; + operands_buf.appendSliceAssumeCapacity(&.{ value, base }); + operands_buf.appendSliceAssumeCapacity(indices); + + return mlir.Operation.make(ctx, "tpu.strided_store", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .attributes = &.{ + .named(ctx, "strides", .denseArray(ctx, .i32, strides)), + }, + .location = location, + }); +} + +// ============================================================================= +// Compute — matmul, iota, reciprocal, casts +// ============================================================================= + +pub const MatmulOpts = struct { + /// Deprecated when `dimension_numbers` is provided. + transpose_lhs: bool = false, + transpose_rhs: bool = false, + transpose_lhs_hint: bool = false, + /// Optional precision; pass null to omit. + precision: ?ContractPrecision = null, + /// Optional `#tpu.dot_dimension_numbers<...>` attribute. When omitted the + /// canonicalizer derives one. Build with `DotDimensionNumbersAttr` or + /// `dotDimensionNumbersAttr`. + dimension_numbers: ?*const mlir.Attribute = null, +}; + +pub fn matmul( + ctx: *mlir.Context, + lhs: *const mlir.Value, + rhs: *const mlir.Value, + acc: *const mlir.Value, + result_type: *const mlir.Type, + opts: MatmulOpts, + location: *const mlir.Location, +) *mlir.Operation { + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attrs.appendSliceAssumeCapacity(&.{ + .named(ctx, "transpose_lhs", .boolean(ctx, opts.transpose_lhs)), + .named(ctx, "transpose_rhs", .boolean(ctx, opts.transpose_rhs)), + .named(ctx, "transpose_lhs_hint", .boolean(ctx, opts.transpose_lhs_hint)), + }); + if (opts.precision) |p| { + attrs.appendAssumeCapacity(.named(ctx, "precision", p.attribute(ctx))); + } + if (opts.dimension_numbers) |dn| { + attrs.appendAssumeCapacity(.named(ctx, "dimension_numbers", dn)); + } + + return mlir.Operation.make(ctx, "tpu.matmul", .{ + .operands = .{ .flat = &.{ lhs, rhs, acc } }, + .results = .{ .flat = &.{result_type} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +pub const IotaOpts = struct { + dimensions: ?[]const i32 = null, +}; + +pub fn iota( + ctx: *mlir.Context, + result_type: *const mlir.Type, + opts: IotaOpts, + location: *const mlir.Location, +) *mlir.Operation { + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.dimensions) |d| { + attrs.appendAssumeCapacity(.named(ctx, "dimensions", .denseArray(ctx, .i32, d))); + } + return mlir.Operation.make(ctx, "tpu.iota", .{ + .results = .{ .flat = &.{result_type} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +pub const ReciprocalOpts = struct { + approx: bool = false, + full_range: bool = false, +}; + +pub fn reciprocal( + ctx: *mlir.Context, + input: *const mlir.Value, + opts: ReciprocalOpts, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.reciprocal", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{input.type_()} }, + .attributes = &.{ + .named(ctx, "approx", .boolean(ctx, opts.approx)), + .named(ctx, "full_range", .boolean(ctx, opts.full_range)), + }, + .location = location, + }); +} + +/// Generic 1-operand → 1-result builder for the various TPU casts. +fn castOp(comptime op_name: []const u8) type { + return struct { + pub fn call( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, + ) *mlir.Operation { + return mlir.Operation.make(ctx, op_name, .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); + } + }; +} + +pub const fptosi = castOp("tpu.fptosi").call; +pub const fptoui = castOp("tpu.fptoui").call; +pub const sitofp = castOp("tpu.sitofp").call; +pub const uitofp = castOp("tpu.uitofp").call; +pub const extf = castOp("tpu.extf").call; +pub const truncf = castOp("tpu.truncf").call; +pub const bitcast = castOp("tpu.bitcast").call; +pub const bitcast_vreg = castOp("tpu.bitcast_vreg").call; +pub const mask_cast = castOp("tpu.mask_cast").call; + +// ============================================================================= +// Shape — reshape / repeat / concatenate / transpose / broadcast_in_sublanes +// ============================================================================= + +pub fn reshape( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.reshape", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +pub fn concatenate( + ctx: *mlir.Context, + sources: []const *const mlir.Value, + dimension: i32, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.concatenate", .{ + .operands = .{ .flat = sources }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "dimension", .int(ctx, .i32, dimension)), + }, + .location = location, + }); +} + +pub fn transpose( + ctx: *mlir.Context, + src: *const mlir.Value, + permutation: []const i64, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.transpose", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "permutation", .denseArray(ctx, .i64, permutation)), + }, + .location = location, + }); +} + +pub fn broadcast_in_sublanes( + ctx: *mlir.Context, + src: *const mlir.Value, + lane: i32, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.broadcast_in_sublanes", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "lane", .int(ctx, .i32, lane)), + }, + .location = location, + }); +} + +pub const DynamicRotateOpts = struct { + dimension: i32, + stride: ?i32 = null, + stride_dimension: ?i32 = null, +}; + +pub fn dynamic_rotate( + ctx: *mlir.Context, + value: *const mlir.Value, + amount: *const mlir.Value, + opts: DynamicRotateOpts, + location: *const mlir.Location, +) *mlir.Operation { + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attrs.appendSliceAssumeCapacity(&.{ + .named(ctx, "dimension", .int(ctx, .si32, opts.dimension)), + }); + if (opts.stride) |s| attrs.appendAssumeCapacity(.named(ctx, "stride", .int(ctx, .si32, s))); + if (opts.stride_dimension) |sd| attrs.appendAssumeCapacity(.named(ctx, "stride_dimension", .int(ctx, .si32, sd))); + + return mlir.Operation.make(ctx, "tpu.dynamic_rotate", .{ + .operands = .{ .flat = &.{ value, amount } }, + .results = .{ .flat = &.{value.type_()} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +// ============================================================================= +// Memref-shaped ops — slice / squeeze / reshape / bitcast / reinterpret_cast +// ============================================================================= + +pub const MemRefSliceArgs = struct { + /// `base_idx` count must equal source memref rank. + base_idx: []const *const mlir.Value, + /// Optional dynamic sizes; left empty when target shape is fully static. + dynamic_sizes: []const *const mlir.Value = &.{}, +}; + +pub fn memref_slice( + ctx: *mlir.Context, + mem_ref: *const mlir.Value, + args: MemRefSliceArgs, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 32) = .empty; + operands_buf.appendAssumeCapacity(mem_ref); + operands_buf.appendSliceAssumeCapacity(args.base_idx); + operands_buf.appendSliceAssumeCapacity(args.dynamic_sizes); + + const seg_sizes = [3]i32{ 1, @intCast(args.base_idx.len), @intCast(args.dynamic_sizes.len) }; + return mlir.Operation.make(ctx, "tpu.memref_slice", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), + }, + .location = location, + }); +} + +pub fn memref_squeeze( + ctx: *mlir.Context, + mem_ref: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.memref_squeeze", .{ + .operands = .{ .flat = &.{mem_ref} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +pub fn memref_reshape( + ctx: *mlir.Context, + mem_ref: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.memref_reshape", .{ + .operands = .{ .flat = &.{mem_ref} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +pub fn memref_bitcast( + ctx: *mlir.Context, + mem_ref: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.memref_bitcast", .{ + .operands = .{ .flat = &.{mem_ref} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +pub fn reinterpret_cast( + ctx: *mlir.Context, + mem_ref: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.reinterpret_cast", .{ + .operands = .{ .flat = &.{mem_ref} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{.named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &.{ 1, 0, 0, 0 }))}, + .location = location, + }); +} + +pub fn assume_layout( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.assume_layout", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +pub fn erase_memref_layout( + ctx: *mlir.Context, + mem_ref: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.erase_memref_layout", .{ + .operands = .{ .flat = &.{mem_ref} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +// ============================================================================= +// Mask building — create_mask / create_subelement_mask / mask_cast +// ============================================================================= + +/// `tpu.create_mask` — two equal-length operand groups (low, high) per dim. +pub fn create_mask( + ctx: *mlir.Context, + low_bounds: []const *const mlir.Value, + high_bounds: []const *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + std.debug.assert(low_bounds.len == high_bounds.len); + var operands_buf: stdx.BoundedArray(*const mlir.Value, 32) = .empty; + operands_buf.appendSliceAssumeCapacity(low_bounds); + operands_buf.appendSliceAssumeCapacity(high_bounds); + + return mlir.Operation.make(ctx, "tpu.create_mask", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +// ============================================================================= +// Semaphores / DMA / barriers +// ============================================================================= + +pub fn sem_alloc( + ctx: *mlir.Context, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.sem_alloc", .{ + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +pub fn sem_barrier( + ctx: *mlir.Context, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.sem_barrier", .{ + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +pub fn sem_read( + ctx: *mlir.Context, + semaphore: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.sem_read", .{ + .operands = .{ .flat = &.{semaphore} }, + .results = .{ .flat = &.{.int(ctx, .i32)} }, + .location = location, + }); +} + +pub fn sem_wait( + ctx: *mlir.Context, + semaphore: *const mlir.Value, + amount: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.sem_wait", .{ + .operands = .{ .flat = &.{ semaphore, amount } }, + .location = location, + }); +} + +pub const SemSignalOpts = struct { + device_id: ?*const mlir.Value = null, + core_id: ?*const mlir.Value = null, + subcore_id: ?*const mlir.Value = null, +}; + +pub fn sem_signal( + ctx: *mlir.Context, + semaphore: *const mlir.Value, + amount: *const mlir.Value, + opts: SemSignalOpts, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 5) = .empty; + operands_buf.appendSliceAssumeCapacity(&.{ semaphore, amount }); + const dev_len: i32 = if (opts.device_id) |d| blk: { + operands_buf.appendAssumeCapacity(d); + break :blk 1; + } else 0; + const core_len: i32 = if (opts.core_id) |c_| blk: { + operands_buf.appendAssumeCapacity(c_); + break :blk 1; + } else 0; + + const subcore_len: i32 = if (opts.subcore_id) |subcore| blk: { + operands_buf.appendAssumeCapacity(subcore); + break :blk 1; + } else 0; + + const seg_sizes = [5]i32{ 1, 1, dev_len, core_len, subcore_len }; + return mlir.Operation.make(ctx, "tpu.sem_signal", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .attributes = &.{ + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), + }, + .location = location, + }); +} + +pub fn barrier( + ctx: *mlir.Context, + barrier_id: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.barrier", .{ + .operands = .{ .flat = &.{barrier_id} }, + .location = location, + }); +} + +pub const EnqueueDmaOpts = struct { + source_semaphore: ?*const mlir.Value = null, + device_id: ?*const mlir.Value = null, + core_id: ?*const mlir.Value = null, + subcore_id: ?*const mlir.Value = null, + priority: i32 = 0, + strict_ordering: bool = false, +}; + +pub fn enqueue_dma( + ctx: *mlir.Context, + source: *const mlir.Value, + target: *const mlir.Value, + target_semaphore: *const mlir.Value, + opts: EnqueueDmaOpts, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 7) = .empty; + operands_buf.appendAssumeCapacity(source); + const src_sem_len: i32 = if (opts.source_semaphore) |s| blk: { + operands_buf.appendAssumeCapacity(s); + break :blk 1; + } else 0; + operands_buf.appendAssumeCapacity(target); + operands_buf.appendAssumeCapacity(target_semaphore); + const dev_len: i32 = if (opts.device_id) |d| blk: { + operands_buf.appendAssumeCapacity(d); + break :blk 1; + } else 0; + const core_len: i32 = if (opts.core_id) |c_| blk: { + operands_buf.appendAssumeCapacity(c_); + break :blk 1; + } else 0; + + // Source / source_semaphore? / target / target_semaphore / device_id? / core_id? / subcore_id? + const subcore_len: i32 = if (opts.subcore_id) |subcore| blk: { + operands_buf.appendAssumeCapacity(subcore); + break :blk 1; + } else 0; + + const seg_sizes = [7]i32{ 1, src_sem_len, 1, 1, dev_len, core_len, subcore_len }; + return mlir.Operation.make(ctx, "tpu.enqueue_dma", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .attributes = &.{ + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), + .named(ctx, "priority", .int(ctx, .i32, opts.priority)), + .named(ctx, "strict_ordering", .boolean(ctx, opts.strict_ordering)), + }, + .location = location, + }); +} + +pub const WaitDma2Opts = struct { + device_id: ?*const mlir.Value = null, + core_id: ?*const mlir.Value = null, + strict_ordering: bool = false, +}; + +pub fn wait_dma2( + ctx: *mlir.Context, + semaphore: *const mlir.Value, + src: *const mlir.Value, + dst: *const mlir.Value, + opts: WaitDma2Opts, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 5) = .empty; + operands_buf.appendSliceAssumeCapacity(&.{ semaphore, src, dst }); + const dev_len: i32 = if (opts.device_id) |d| blk: { + operands_buf.appendAssumeCapacity(d); + break :blk 1; + } else 0; + const core_len: i32 = if (opts.core_id) |c_| blk: { + operands_buf.appendAssumeCapacity(c_); + break :blk 1; + } else 0; + + const seg_sizes = [5]i32{ 1, 1, 1, dev_len, core_len }; + return mlir.Operation.make(ctx, "tpu.wait_dma2", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .attributes = &.{ + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), + .named(ctx, "strict_ordering", .boolean(ctx, opts.strict_ordering)), + }, + .location = location, + }); +} + +// ============================================================================= +// Region-bearing — region / trace / yield +// ============================================================================= + +/// tpu.region — single-block region with an implicit `tpu.yield` terminator. +pub fn region( + ctx: *mlir.Context, + body: *mlir.Block, + result_types: []const *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.region", .{ + .results = .{ .flat = result_types }, + .blocks = &.{body}, + .verify = false, + .location = location, + }); +} + +pub fn trace( + ctx: *mlir.Context, + body: *mlir.Block, + result_types: []const *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.trace", .{ + .results = .{ .flat = result_types }, + .blocks = &.{body}, + .verify = false, + .location = location, + }); +} + +pub fn yield( + ctx: *mlir.Context, + values: []const *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.yield", .{ + .operands = .{ .flat = values }, + .verify = false, + .location = location, + }); +} + +pub fn trace_start( + ctx: *mlir.Context, + level: i32, + message: []const u8, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.trace_start", .{ + .attributes = &.{ + .named(ctx, "level", .int(ctx, .i32, level)), + .named(ctx, "message", .string(ctx, message)), + }, + .verify = false, + .location = location, + }); +} + +pub fn trace_stop(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.trace_stop", .{ + .verify = false, + .location = location, + }); +} + +/// Build a `#tpu.dot_dimension_numbers<...>` attribute. Mirrors +/// `tpu_ops.td:TPU_DotDimensionNumbersAttr`. Pass empty slices for the +/// fields you don't use (most dot operations leave batch dims empty). +pub fn dotDimensionNumbersAttr( + ctx: *mlir.Context, + lhs_contracting: []const i64, + rhs_contracting: []const i64, + lhs_non_contracting: []const i64, + rhs_non_contracting: []const i64, + output_dim_order: []const i64, + lhs_batch: []const i64, + rhs_batch: []const i64, +) *const mlir.Attribute { + const attr = DotDimensionNumbersAttr.get(ctx, .{ + .lhsContractingDims = lhs_contracting, + .rhsContractingDims = rhs_contracting, + .lhsNonContractingDims = lhs_non_contracting, + .rhsNonContractingDims = rhs_non_contracting, + .outputDimOrder = output_dim_order, + .lhsBatchDims = lhs_batch, + .rhsBatchDims = rhs_batch, + }) catch unreachable; + return attr.attribute(); +} + +// ============================================================================= +// Misc — device_id / delay / assume_multiple +// ============================================================================= + +pub fn device_id(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.device_id", .{ + .results = .{ .flat = &.{.int(ctx, .i32)} }, + .location = location, + }); +} + +pub fn delay( + ctx: *mlir.Context, + cycles: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.delay", .{ + .operands = .{ .flat = &.{cycles} }, + .location = location, + }); +} + +pub fn assume_multiple( + ctx: *mlir.Context, + src: *const mlir.Value, + multiple: i32, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "tpu.assume_multiple", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{src.type_()} }, + .attributes = &.{ + .named(ctx, "multiple", .int(ctx, .i32, multiple)), + }, + .location = location, + }); +} + +// ============================================================================= +// Internal helpers +// ============================================================================= + +/// `tpu.DenseBoolArrayAttr` stores one C `int` (i32) per bool, see `mlirDenseBoolArrayGet` +fn denseBoolArrayAttribute(ctx: *mlir.Context, values: []const bool) *const mlir.Attribute { + var ints: stdx.BoundedArray(i32, 64) = .empty; + for (values) |b| ints.appendAssumeCapacity(@intFromBool(b)); + return .denseArray(ctx, .bool, ints.constSlice()); +} diff --git a/mlir/dialects/mosaic_tpu/test.zig b/mlir/dialects/mosaic_tpu/test.zig new file mode 100644 index 00000000..20a20f42 --- /dev/null +++ b/mlir/dialects/mosaic_tpu/test.zig @@ -0,0 +1,281 @@ +const std = @import("std"); + +const mlir = @import("mlir"); +const mosaic_tpu = @import("mlir/dialects/mosaic_tpu"); + +const operation_names: []const []const u8 = &.{ + "tpu.all_reduce", + "tpu.assume_layout", + "tpu.assume_multiple", + "tpu.barrier", + "tpu.bitcast", + "tpu.bitcast_vreg", + "tpu.broadcast_in_sublanes", + "tpu.concatenate", + "tpu.create_mask", + "tpu.delay", + "tpu.device_id", + "tpu.dynamic_rotate", + "tpu.enqueue_dma", + "tpu.erase_memref_layout", + "tpu.extf", + "tpu.fptosi", + "tpu.fptoui", + "tpu.iota", + "tpu.load", + "tpu.mask_cast", + "tpu.matmul", + "tpu.memref_bitcast", + "tpu.memref_reshape", + "tpu.memref_slice", + "tpu.memref_squeeze", + "tpu.reciprocal", + "tpu.reduce_index", + "tpu.region", + "tpu.reinterpret_cast", + "tpu.reshape", + "tpu.scan", + "tpu.sem_alloc", + "tpu.sem_barrier", + "tpu.sem_read", + "tpu.sem_signal", + "tpu.sem_wait", + "tpu.sitofp", + "tpu.sort", + "tpu.store", + "tpu.strided_load", + "tpu.strided_store", + "tpu.trace", + "tpu.trace_start", + "tpu.trace_stop", + "tpu.transpose", + "tpu.truncf", + "tpu.uitofp", + "tpu.vector_load", + "tpu.vector_store", + "tpu.wait_dma2", + "tpu.yield", +}; + +fn testContext() !*mlir.Context { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + mosaic_tpu.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + return ctx; +} + +fn expectSegments(op: *const mlir.Operation, expected: []const i32) !void { + const segments = op.attributeByName("operandSegmentSizes").?.isA(mlir.DenseArrayAttribute(.i32)).?; + try std.testing.expectEqual(expected.len, segments.numElements()); + for (expected, 0..) |size, i| try std.testing.expectEqual(size, segments.element(i)); +} + +fn expectPrints(expected: []const u8, value: anytype) !void { + var buf: [256]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print("{f}", .{value}); + try std.testing.expectEqualStrings(expected, w.buffered()); +} + +fn expectEnumAttr(comptime Attr: type, ctx: *mlir.Context) !void { + const Enum = @FieldType(Attr.InitArgs, "value"); + inline for (@typeInfo(Enum).@"enum".field_names) |name| { + const value: Enum = @field(Enum, name); + const attr = try Attr.get(ctx, .{ .value = value }); + try std.testing.expectEqual(value, attr.value()); + try std.testing.expect(attr.attribute().isA(Attr) != null); + try std.testing.expect(value.attribute(ctx).eql(attr.attribute())); + try expectPrints("#tpu." ++ Attr.mnemonic ++ "<" ++ name ++ ">", attr); + } +} + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(mosaic_tpu); + inline for (comptime std.meta.declarations(mosaic_tpu)) |decl| { + const T = @field(mosaic_tpu, decl); + if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); + } +} + +test "DMA and semaphore wrappers satisfy current operand groups" { + const ctx = try testContext(); + defer ctx.deinit(); + const loc = mlir.Location.unknown(ctx); + const args = mlir.Block.init(&.{}, &.{}); + defer args.deinit(); + const source = args.addArgument(try mlir.Type.parse(ctx, "memref<8x128xbf16, #tpu.memory_space>"), loc); + const target = args.addArgument(try mlir.Type.parse(ctx, "memref<8x128xbf16, #tpu.memory_space>"), loc); + const dma_sem = args.addArgument(try mlir.Type.parse(ctx, "memref>"), loc); + const sem = args.addArgument(try mlir.Type.parse(ctx, "memref>"), loc); + const amount = args.addArgument(.int(ctx, .i32), loc); + + const dma = mosaic_tpu.enqueue_dma(ctx, source, target, dma_sem, .{}, loc); + defer dma.deinit(); + try expectSegments(dma, &.{ 1, 0, 1, 1, 0, 0, 0 }); + try std.testing.expect(dma.verify()); + const signal = mosaic_tpu.sem_signal(ctx, sem, amount, .{}, loc); + defer signal.deinit(); + try expectSegments(signal, &.{ 1, 1, 0, 0, 0 }); + try std.testing.expect(signal.verify()); + + const sc_sem = args.addArgument(try mlir.Type.parse(ctx, "memref>"), loc); + const sc_dma_sem = args.addArgument(try mlir.Type.parse(ctx, "memref>"), loc); + const core = args.addArgument(.int(ctx, .i32), loc); + const subcore = args.addArgument(.int(ctx, .i32), loc); + const remote_signal = mosaic_tpu.sem_signal(ctx, sc_sem, amount, .{ .core_id = core, .subcore_id = subcore }, loc); + defer remote_signal.deinit(); + try expectSegments(remote_signal, &.{ 1, 1, 0, 1, 1 }); + try std.testing.expect(remote_signal.operand(3).eql(subcore)); + const remote_dma = mosaic_tpu.enqueue_dma(ctx, source, target, sc_dma_sem, .{ + .source_semaphore = dma_sem, + .core_id = core, + .subcore_id = subcore, + }, loc); + defer remote_dma.deinit(); + try expectSegments(remote_dma, &.{ 1, 1, 1, 1, 0, 1, 1 }); + try std.testing.expect(remote_dma.operand(5).eql(subcore)); +} + +test "matmul and reinterpret cast satisfy current required properties" { + const ctx = try testContext(); + defer ctx.deinit(); + const loc = mlir.Location.unknown(ctx); + const args = mlir.Block.init(&.{}, &.{}); + defer args.deinit(); + const vector_type = mlir.Type.vector(&.{ 8, 8 }, .float(ctx, .f32)); + const lhs = args.addArgument(vector_type, loc); + const rhs = args.addArgument(vector_type, loc); + const acc = args.addArgument(vector_type, loc); + const product = mosaic_tpu.matmul(ctx, lhs, rhs, acc, vector_type, .{}, loc); + defer product.deinit(); + try std.testing.expect(product.verify()); + try std.testing.expect(!product.attributeByName("transpose_lhs_hint").?.isA(mlir.BoolAttribute).?.value()); + + const ref_type = try mlir.Type.parse(ctx, "memref<8x8xf32, #tpu.memory_space>"); + const ref = args.addArgument(ref_type, loc); + const cast = mosaic_tpu.reinterpret_cast(ctx, ref, ref_type, loc); + defer cast.deinit(); + try expectSegments(cast, &.{ 1, 0, 0, 0 }); + try std.testing.expect(cast.verify()); +} + +test "legacy reduction extrema and scan axis preserve their semantics" { + const ctx = try testContext(); + defer ctx.deinit(); + const loc = mlir.Location.unknown(ctx); + const args = mlir.Block.init(&.{}, &.{}); + defer args.deinit(); + const float_type = mlir.Type.vector(&.{ 2, 8 }, .float(ctx, .f32)); + const int_type = mlir.Type.vector(&.{ 2, 8 }, .int(ctx, .i32)); + const floats = args.addArgument(float_type, loc); + const integers = args.addArgument(int_type, loc); + const mask = args.addArgument(mlir.Type.vector(&.{8}, .int(ctx, .i1)), loc); + const float_min = mosaic_tpu.all_reduce(ctx, floats, 1, .min, float_type, loc); + defer float_min.deinit(); + const int_max = mosaic_tpu.all_reduce(ctx, integers, 1, .max, int_type, loc); + defer int_max.deinit(); + try std.testing.expect(float_min.attributeByName("kind").?.eql(try mlir.Attribute.parse(ctx, "#tpu.reduction_kind"))); + try std.testing.expect(int_max.attributeByName("kind").?.eql(try mlir.Attribute.parse(ctx, "#tpu.reduction_kind"))); + + const int_scan = mosaic_tpu.scan(ctx, integers, .max, int_type, .{ .mask = mask }, loc); + defer int_scan.deinit(); + try std.testing.expect(int_scan.verify()); + try std.testing.expect(int_scan.attributeByName("kind").?.eql(try mlir.Attribute.parse(ctx, "#tpu.reduction_kind"))); + try std.testing.expectEqual(@as(i64, 1), int_scan.attributeByName("dimension").?.isA(mlir.IntegerAttribute).?.value(i64)); + const float_scan = mosaic_tpu.scan(ctx, floats, .min, float_type, .{}, loc); + defer float_scan.deinit(); + try std.testing.expect(float_scan.attributeByName("kind").?.eql(try mlir.Attribute.parse(ctx, "#tpu.reduction_kind"))); +} + +test "tpu types are built through the C API" { + const ctx = try mlir.Context.init(.{}); + defer ctx.deinit(); + + const semaphore = try mosaic_tpu.SemaphoreType.get(ctx, .{}); + try std.testing.expect(mosaic_tpu.semaphoreType(ctx).isA(mosaic_tpu.SemaphoreType) != null); + try std.testing.expect(mosaic_tpu.semaphoreType(ctx).isA(mosaic_tpu.DMASemaphoreType) == null); + try expectPrints("!tpu.semaphore", semaphore); + + const dma_semaphore = try mosaic_tpu.DMASemaphoreType.get(ctx, .{}); + try std.testing.expect(mosaic_tpu.dmaSemaphoreType(ctx).isA(mosaic_tpu.DMASemaphoreType) != null); + try expectPrints("!tpu.dma_semaphore", dma_semaphore); +} + +test "tpu enum attributes are built through the C API" { + const ctx = try mlir.Context.init(.{}); + defer ctx.deinit(); + + try expectEnumAttr(mosaic_tpu.ReductionKindAttr, ctx); + try expectEnumAttr(mosaic_tpu.ContractPrecisionAttr, ctx); + try expectEnumAttr(mosaic_tpu.RoundingModeAttr, ctx); + try expectEnumAttr(mosaic_tpu.CoreTypeAttr, ctx); + try expectEnumAttr(mosaic_tpu.DimensionSemanticsAttr, ctx); + try expectEnumAttr(mosaic_tpu.PipelineModeAttr, ctx); + try expectEnumAttr(mosaic_tpu.RevisitModeAttr, ctx); + try expectEnumAttr(mosaic_tpu.MemorySpaceAttr, ctx); + + const smem_tc = try mosaic_tpu.MemorySpaceAttr.get(ctx, .{ .value = .smem, .coreType = .sc_scalar_subcore }); + try std.testing.expectEqual(mosaic_tpu.MemorySpace.smem, smem_tc.value()); + try std.testing.expectEqual(mosaic_tpu.CoreType.sc_scalar_subcore, smem_tc.coreType().?); + try expectPrints("#tpu.memory_space", smem_tc); + try std.testing.expectEqual(null, (try mosaic_tpu.MemorySpaceAttr.get(ctx, .{ .value = .hbm })).coreType()); + // The dialect verifier's pairing rules: no core type on hbm, no scalar subcore on vmem. + try std.testing.expectError(error.InvalidMlir, mosaic_tpu.MemorySpaceAttr.get(ctx, .{ .value = .hbm, .coreType = .tc })); + try std.testing.expectError(error.InvalidMlir, mosaic_tpu.MemorySpaceAttr.get(ctx, .{ .value = .vmem, .coreType = .sc_scalar_subcore })); +} + +test "tpu array attributes are built through the C API" { + const ctx = try mlir.Context.init(.{}); + defer ctx.deinit(); + + const window = try mosaic_tpu.ElementWindowAttr.get(ctx, .{ .padLow = &.{ 0, 2 }, .padHigh = &.{ 1, 3 } }); + try std.testing.expectEqual(2, window.numPadLow()); + try std.testing.expectEqual(2, window.padLow(1)); + try std.testing.expectEqual(2, window.numPadHigh()); + try std.testing.expectEqual(3, window.padHigh(1)); + try expectPrints("#tpu.element_window<[0, 2], [1, 3]>", window); + try std.testing.expect(mosaic_tpu.elementWindowAttr(ctx, &.{ 0, 2 }, &.{ 1, 3 }).eql(window.attribute())); + + const dims = mosaic_tpu.dotDimensionNumbersAttr(ctx, &.{1}, &.{0}, &.{0}, &.{1}, &.{ 0, 0, 1, 1 }, &.{}, &.{}); + try expectPrints("#tpu.dot_dimension_numbers<[1], [0], [0], [1], [0, 0, 1, 1], [], []>", dims); + + const batched = try mosaic_tpu.DotDimensionNumbersAttr.get(ctx, .{ + .lhsContractingDims = &.{2}, + .rhsContractingDims = &.{1}, + .lhsNonContractingDims = &.{1}, + .outputDimOrder = &.{ 0, 0, 0, 1 }, + .lhsBatchDims = &.{0}, + .rhsBatchDims = &.{0}, + }); + try std.testing.expectEqual(0, batched.numRhsNonContractingDims()); + try std.testing.expectEqual(4, batched.numOutputDimOrder()); + try std.testing.expectEqual(1, batched.outputDimOrder(3)); + try std.testing.expectEqual(0, batched.rhsBatchDims(0)); + try expectPrints("#tpu.dot_dimension_numbers<[2], [1], [1], [], [0, 0, 0, 1], [0], [0]>", batched); +} + +test "kernel attributes carry their tpu names" { + const ctx = try mlir.Context.init(.{}); + defer ctx.deinit(); + + const core_type = mosaic_tpu.coreTypeNamedAttr(ctx, .sc_vector_subcore); + try std.testing.expectEqualStrings("tpu.core_type", core_type.name().str()); + try std.testing.expect(core_type.attribute().eql(mosaic_tpu.CoreType.sc_vector_subcore.attribute(ctx))); + const semantics = mosaic_tpu.dimensionSemanticsNamedAttr(ctx, .arbitrary); + try std.testing.expectEqualStrings("tpu.dimension_semantics", semantics.name().str()); + try std.testing.expect(semantics.attribute().eql(mosaic_tpu.DimensionSemantics.arbitrary.attribute(ctx))); +} + +test "every bound operation is registered" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + mosaic_tpu.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} diff --git a/mlir/dialects/nvvm/BUILD.bazel b/mlir/dialects/nvvm/BUILD.bazel new file mode 100644 index 00000000..f6529062 --- /dev/null +++ b/mlir/dialects/nvvm/BUILD.bazel @@ -0,0 +1,168 @@ +load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library", "td_library") +load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +package(default_visibility = ["//visibility:public"]) + +cc_library( + name = "capi", + srcs = ["capi.cc"], + hdrs = ["capi.h"], + tags = ["manual"], + deps = [ + ":dialect", + ":headers", + "@llvm-project//mlir:CAPIIR", + ], +) + +zig_library( + name = "nvvm", + srcs = [ + "ops.zig", + ], + import_name = "mlir/dialects/nvvm", + main = "nvvm.zig", + tags = ["manual"], + deps = [ + ":capi", + "//mlir", + "//stdx", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":nvvm", + "//mlir", + ], +) + +td_library( + name = "td_files", + srcs = [ + "NVVMAttrs.td", + "NVVMDialect.td", + "NVVMEnums.td", + "NVVMOps.td", + ], + includes = ["/"], + deps = [ + "@llvm-project//mlir:InferTypeOpInterfaceTdFiles", + "@llvm-project//mlir:OpBaseTdFiles", + "@llvm-project//mlir:SideEffectInterfacesTdFiles", + ], +) + +gentbl_cc_library( + name = "dialect_inc_gen", + tbl_outs = [ + ( + [ + "-gen-dialect-decls", + "-dialect=nvvm", + ], + "NVVMDialect.h.inc", + ), + ( + [ + "-gen-dialect-defs", + "-dialect=nvvm", + ], + "NVVMDialect.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "NVVMDialect.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "enums_inc_gen", + tbl_outs = [ + ( + ["-gen-enum-decls"], + "NVVMEnums.h.inc", + ), + ( + ["-gen-enum-defs"], + "NVVMEnums.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "NVVMEnums.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "attrs_inc_gen", + tbl_outs = [ + ( + [ + "-gen-attrdef-decls", + "-attrdefs-dialect=nvvm", + ], + "NVVMAttrs.h.inc", + ), + ( + [ + "-gen-attrdef-defs", + "-attrdefs-dialect=nvvm", + ], + "NVVMAttrs.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "NVVMAttrs.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "ops_inc_gen", + tbl_outs = [ + ( + ["-gen-op-decls"], + "NVVMOps.h.inc", + ), + ( + ["-gen-op-defs"], + "NVVMOps.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "NVVMOps.td", + deps = [":td_files"], +) + +cc_library( + name = "headers", + hdrs = ["NVVMDialect.h"], + deps = [ + ":attrs_inc_gen", + ":dialect_inc_gen", + ":enums_inc_gen", + ":ops_inc_gen", + "@llvm-project//mlir:BytecodeOpInterface", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:InferTypeOpInterface", + "@llvm-project//mlir:LLVMDialect", + "@llvm-project//mlir:SideEffectInterfaces", + ], +) + +cc_library( + name = "dialect", + srcs = [ + "NVVMDialect.cpp", + "NVVMInference.cpp", + ], + deps = [ + ":headers", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:InferTypeOpInterface", + "@llvm-project//mlir:LLVMDialect", + ], +) diff --git a/mlir/dialects/nvvm/NVVMAttrs.td b/mlir/dialects/nvvm/NVVMAttrs.td new file mode 100644 index 00000000..20c3fa51 --- /dev/null +++ b/mlir/dialects/nvvm/NVVMAttrs.td @@ -0,0 +1,52 @@ +//===- NVVMAttrs.td - NVVM attributes of the CuTe DSL ------*- tablegen -*-===// +// +// Generated for nvidia-cutlass-dsl 4.8.0 from the DSL compiler +// (`_cutlass_ir.cu13` parser messages, its mnemonics, and probing its +// attribute parser and printer); do not edit by hand. +// +// The enum attributes are NVVMEnums.td. These are the compiler's other nvvm +// attributes: every mnemonic its parser knows is one of the two. +// +//===----------------------------------------------------------------------===// + +#ifndef MLIR_DIALECTS_NVVM_NVVMATTRS_TD +#define MLIR_DIALECTS_NVVM_NVVMATTRS_TD + +include "mlir/dialects/nvvm/NVVMDialect.td" +include "mlir/dialects/nvvm/NVVMEnums.td" + +def NVVM_MMAShapeAttr : NVVM_Attr<"MMAShape", "shape"> { + let summary = "Shape of an MMA operation"; + let description = "`#nvvm.shape`"; + let parameters = (ins "int":$m, "int":$n, "int":$k); + let assemblyFormat = "`<` struct(params) `>`"; +} + +def NVVM_LdStMatrixShapeAttr : NVVM_Attr<"LdStMatrixShape", "ld_st_matrix_shape"> { + let summary = "Matrix shape of ldmatrix, stmatrix and movmatrix"; + let description = "`#nvvm.ld_st_matrix_shape`"; + let parameters = (ins "int":$m, "int":$n); + let assemblyFormat = "`<` struct(params) `>`"; +} + +// Defaulted fields print only when set; the defaults and the verifier's rules +// and messages are the compiler's. +def NVVM_TargetAttr : NVVM_Attr<"Target", "target"> { + let summary = "GPU target of an NVVM module"; + let description = [{ + `#nvvm.target` + }]; + let parameters = (ins + DefaultValuedParameter<"int", "2">:$O, + StringRefParameter<"target triple", "\"nvptx64-nvidia-cuda\"">:$triple, + StringRefParameter<"target chip", "\"sm_75\"">:$chip, + StringRefParameter<"target features", "\"\"">:$features, + OptionalParameter<"::mlir::DictionaryAttr">:$flags, + OptionalParameter<"::mlir::ArrayAttr">:$link, + DefaultValuedParameter<"bool", "false">:$verifyTarget + ); + let assemblyFormat = "(`<` struct(params)^ `>`)?"; + let genVerifyDecl = 1; +} + +#endif // MLIR_DIALECTS_NVVM_NVVMATTRS_TD diff --git a/mlir/dialects/nvvm/NVVMDialect.cpp b/mlir/dialects/nvvm/NVVMDialect.cpp new file mode 100644 index 00000000..518e825d --- /dev/null +++ b/mlir/dialects/nvvm/NVVMDialect.cpp @@ -0,0 +1,51 @@ +//===- NVVMDialect.cpp - NVIDIA's nvvm dialect of the CuTe DSL ------------===// + +#include "mlir/dialects/nvvm/NVVMDialect.h" + +#include "mlir/IR/Builders.h" +#include "mlir/IR/DialectImplementation.h" +#include "llvm/ADT/TypeSwitch.h" + +using namespace mlir; +using namespace mlir::cutlass_compiler::nvvm; + +#include "mlir/dialects/nvvm/NVVMDialect.cpp.inc" + +#include "mlir/dialects/nvvm/NVVMEnums.cpp.inc" + +#define GET_ATTRDEF_CLASSES +#include "mlir/dialects/nvvm/NVVMAttrs.cpp.inc" + +#define GET_OP_CLASSES +#include "mlir/dialects/nvvm/NVVMOps.cpp.inc" + +void NVVMDialect::initialize() { + addAttributes< +#define GET_ATTRDEF_LIST +#include "mlir/dialects/nvvm/NVVMAttrs.cpp.inc" + >(); + addOperations< +#define GET_OP_LIST +#include "mlir/dialects/nvvm/NVVMOps.cpp.inc" + >(); +} + +// The compiler's rules and messages. +LogicalResult TargetAttr::verify(function_ref emitError, + int O, StringRef triple, StringRef chip, + StringRef, DictionaryAttr, ArrayAttr link, + bool) { + if (O < 0 || O > 3) + return emitError() + << "The optimization level must be a number between 0 and 3."; + if (triple.empty()) + return emitError() << "The target triple cannot be empty."; + if (chip.empty()) + return emitError() << "The target chip cannot be empty."; + if (link && !llvm::all_of(link, [](Attribute attr) { + return attr && isa(attr); + })) + return emitError() + << "All the elements in the `link` array must be strings."; + return success(); +} diff --git a/mlir/dialects/nvvm/NVVMDialect.h b/mlir/dialects/nvvm/NVVMDialect.h new file mode 100644 index 00000000..daa1c67e --- /dev/null +++ b/mlir/dialects/nvvm/NVVMDialect.h @@ -0,0 +1,27 @@ +//===- NVVMDialect.h - NVIDIA's nvvm dialect of the CuTe DSL ----*- C++ -*-===// + +#ifndef MLIR_DIALECTS_NVVM_NVVMDIALECT_H +#define MLIR_DIALECTS_NVVM_NVVMDIALECT_H + +#include "mlir/Bytecode/BytecodeOpInterface.h" +#include "mlir/Dialect/LLVMIR/LLVMAttrs.h" +#include "mlir/Dialect/LLVMIR/LLVMDialect.h" +#include "mlir/Dialect/LLVMIR/LLVMTypes.h" +#include "mlir/IR/BuiltinAttributes.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/DialectImplementation.h" +#include "mlir/IR/OpImplementation.h" +#include "mlir/Interfaces/InferTypeOpInterface.h" +#include "mlir/Interfaces/SideEffectInterfaces.h" + +#include "mlir/dialects/nvvm/NVVMDialect.h.inc" + +#include "mlir/dialects/nvvm/NVVMEnums.h.inc" + +#define GET_ATTRDEF_CLASSES +#include "mlir/dialects/nvvm/NVVMAttrs.h.inc" + +#define GET_OP_CLASSES +#include "mlir/dialects/nvvm/NVVMOps.h.inc" + +#endif // MLIR_DIALECTS_NVVM_NVVMDIALECT_H diff --git a/mlir/dialects/nvvm/NVVMDialect.td b/mlir/dialects/nvvm/NVVMDialect.td new file mode 100644 index 00000000..03ce7682 --- /dev/null +++ b/mlir/dialects/nvvm/NVVMDialect.td @@ -0,0 +1,33 @@ +//===- NVVMDialect.td - NVVM dialect of the CuTe DSL -------*- tablegen -*-===// + +#ifndef MLIR_DIALECTS_NVVM_NVVMDIALECT_TD +#define MLIR_DIALECTS_NVVM_NVVMDIALECT_TD + +include "mlir/IR/AttrTypeBase.td" +include "mlir/IR/DialectBase.td" +include "mlir/IR/OpBase.td" + +def NVVM_Dialect : Dialect { + let name = "nvvm"; + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let summary = "NVIDIA's NVVM dialect as the CuTe-DSL compiler registers it"; + let description = [{ + This dialect mirrors the operation, attribute and enum registry of the + `nvvm` dialect embedded in the nvidia-cutlass-dsl 4.8.0 compiler, built on + NVIDIA's own LLVM revision (23.0.0git 271b151610907). It differs from the + upstream NVVM dialect (`mlir::NVVM`) and is not meant to be registered + next to it: both are named `nvvm`. + }]; + let dependentDialects = ["::mlir::LLVM::LLVMDialect"]; + let useDefaultAttributePrinterParser = 1; +} + +class NVVM_Op traits = []> + : Op; + +class NVVM_Attr + : AttrDef { + let mnemonic = attrMnemonic; +} + +#endif // MLIR_DIALECTS_NVVM_NVVMDIALECT_TD diff --git a/mlir/dialects/nvvm/NVVMEnums.td b/mlir/dialects/nvvm/NVVMEnums.td new file mode 100644 index 00000000..fbff0bdc --- /dev/null +++ b/mlir/dialects/nvvm/NVVMEnums.td @@ -0,0 +1,1386 @@ +// The nvvm dialect's enums, generated from the nvidia-cutlass-dsl 4.8.0 enum +// bindings (`_nvvm_enum_gen.py`); do not edit. +// +// Case values and spellings are the compiler's. An enum the bindings build +// as a dialect attribute also has its EnumAttr, printed `#` +// or, where the compiler prints it so, `#nvvm`. + +#ifndef MLIR_DIALECTS_NVVM_NVVMENUMS_TD +#define MLIR_DIALECTS_NVVM_NVVMENUMS_TD + +include "mlir/dialects/nvvm/NVVMDialect.td" +include "mlir/IR/EnumAttr.td" + +def NVVM_AtomicOpKindEnum : I32EnumAttr<"AtomicOpKind", "operations supported by atom instruction", [ + I32EnumAttrCase<"AND", 0, "and">, + I32EnumAttrCase<"OR", 1, "or">, + I32EnumAttrCase<"XOR", 2, "xor">, + I32EnumAttrCase<"CAS", 3, "cas">, + I32EnumAttrCase<"EXCH", 4, "exch">, + I32EnumAttrCase<"ADD", 5, "add">, + I32EnumAttrCase<"INC", 6, "inc">, + I32EnumAttrCase<"DEC", 7, "dec">, + I32EnumAttrCase<"MIN", 8, "min">, + I32EnumAttrCase<"MAX", 9, "max">, + I32EnumAttrCase<"UMIN", 10, "umin">, + I32EnumAttrCase<"UMAX", 11, "umax">, + I32EnumAttrCase<"FADD", 12, "fadd">, + I32EnumAttrCase<"SUB", 13, "sub"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_BarrierReductionEnum : I32EnumAttr<"BarrierReduction", "NVVM barrier reduction operation", [ + I32EnumAttrCase<"POPC", 0, "popc">, + I32EnumAttrCase<"AND", 1, "and">, + I32EnumAttrCase<"OR", 2, "or"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_BarrierReduxKindEnum : I32EnumAttr<"BarrierReduxKind", "NVVM barrier redux kind", [ + I32EnumAttrCase<"AND", 0, "and">, + I32EnumAttrCase<"OR", 1, "or">, + I32EnumAttrCase<"POPC", 2, "popc"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_BlockScaleFormatEnum : I32EnumAttr<"BlockScaleFormat", "MMA Block Scale Format", [ + I32EnumAttrCase<"UE8M0", 0, "ue8m0">, + I32EnumAttrCase<"UE4M3", 1, "ue4m3"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_CTAGroupKindEnum : I32EnumAttr<"CTAGroupKind", "NVVM CTA group kind", [ + I32EnumAttrCase<"CTA_1", 0, "cta_1">, + I32EnumAttrCase<"CTA_2", 1, "cta_2"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_CVTPackFloatKindEnum : I32EnumAttr<"CVTPackFloatKind", "NVVM CVT Pack Float kind", [ + I32EnumAttrCase<"F32", 0, "f32">, + I32EnumAttrCase<"F16x2", 1, "f16x2">, + I32EnumAttrCase<"E4M3x2", 2, "e4m3x2">, + I32EnumAttrCase<"E5M2x2", 3, "e5m2x2">, + I32EnumAttrCase<"BF16x2", 4, "bf16x2">, + I32EnumAttrCase<"E2M1x2", 5, "e2m1x2">, + I32EnumAttrCase<"E2M3x2", 6, "e2m3x2">, + I32EnumAttrCase<"E3M2x2", 7, "e3m2x2">, + I32EnumAttrCase<"UE8M0x2", 8, "ue8m0x2"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_CacheEvictionPriorityEnum : I32EnumAttr<"CacheEvictionPriority", "NVVM Cache Eviction Priority", [ + I32EnumAttrCase<"EvictNormal", 0, "evict_normal">, + I32EnumAttrCase<"EvictFirst", 1, "evict_first">, + I32EnumAttrCase<"EvictLast", 2, "evict_last">, + I32EnumAttrCase<"EvictUnchanged", 3, "evict_unchanged">, + I32EnumAttrCase<"NoAllocate", 4, "no_allocate"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_ClusterLaunchControlQueryTypeEnum : I32EnumAttr<"ClusterLaunchControlQueryType", "NVVM ClusterLaunchControlQueryType", [ + I32EnumAttrCase<"IS_CANCELED", 0, "is_canceled">, + I32EnumAttrCase<"GET_FIRST_CTA_ID_X", 1, "get_first_cta_id_x">, + I32EnumAttrCase<"GET_FIRST_CTA_ID_Y", 2, "get_first_cta_id_y">, + I32EnumAttrCase<"GET_FIRST_CTA_ID_Z", 3, "get_first_cta_id_z"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_CompareOpKindEnum : I32EnumAttr<"CompareOpKind", "Comparison operator encoding", [ + I32EnumAttrCase<"EQ", 0, "eq">, + I32EnumAttrCase<"NE", 1, "ne">, + I32EnumAttrCase<"UGT", 2, "ugt">, + I32EnumAttrCase<"UGE", 3, "uge">, + I32EnumAttrCase<"ULT", 4, "ult">, + I32EnumAttrCase<"ULE", 5, "ule">, + I32EnumAttrCase<"SGT", 6, "sgt">, + I32EnumAttrCase<"SGE", 7, "sge">, + I32EnumAttrCase<"SLT", 8, "slt">, + I32EnumAttrCase<"SLE", 9, "sle"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_ConvertFP4TypeEnum : I32EnumAttr<"ConvertFP4Type", "NVVM ConvertFP4Type kind", [ + I32EnumAttrCase<"E2M1", 0, "e2m1"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_ConvertFP8TypeEnum : I32EnumAttr<"ConvertFP8Type", "NVVM ConvertFP8Type kind", [ + I32EnumAttrCase<"E4M3", 0, "e4m3">, + I32EnumAttrCase<"E5M2", 1, "e5m2">, + I32EnumAttrCase<"UE8M0", 2, "ue8m0">, + I32EnumAttrCase<"UE5M3", 3, "ue5m3"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_ConvertScaleKindEnum : I32EnumAttr<"ConvertScaleKind", "NVVM ConvertScale kind", [ + I32EnumAttrCase<"NONE", 0, "none">, + I32EnumAttrCase<"UE8M0", 1, "ue8m0"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_DotAccumulateTypeEnum : I32EnumAttr<"DotAccumulateType", "NVVM DotAccumulateType", [ + I32EnumAttrCase<"SIGNED", 1, "signed">, + I32EnumAttrCase<"UNSIGNED", 0, "unsigned"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_EvictKindEnum : I32EnumAttr<"EvictKind", "NVVM L2 Prefetch Size", [ + I32EnumAttrCase<"NORMAL", 0, "evict_normal">, + I32EnumAttrCase<"FIRST", 1, "evict_first">, + I32EnumAttrCase<"LAST", 2, "evict_last">, + I32EnumAttrCase<"UNCHANGED", 3, "evict_unchanged">, + I32EnumAttrCase<"NOALLOCATE", 4, "evict_no_allocate">, + I32EnumAttrCase<"NONE", 5, "none"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_FPRoundingModeEnum : I32EnumAttr<"FPRoundingMode", "NVVM FPRoundingMode kind", [ + I32EnumAttrCase<"NONE", 0, "none">, + I32EnumAttrCase<"RN", 1, "rn">, + I32EnumAttrCase<"RM", 2, "rm">, + I32EnumAttrCase<"RP", 3, "rp">, + I32EnumAttrCase<"RZ", 4, "rz">, + I32EnumAttrCase<"RNA", 5, "rna">, + I32EnumAttrCase<"RS", 6, "rs"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_GridDepActionKindEnum : I32EnumAttr<"GridDepActionKind", "Action kind for grid dependency control", [ + I32EnumAttrCase<"wait", 0, "wait">, + I32EnumAttrCase<"launch_dependents", 1, "launch_dependents"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_IntegerRoundingModeEnum : I32EnumAttr<"IntegerRoundingMode", "NVVM IntegerRoundingMode kind", [ + I32EnumAttrCase<"NONE", 0, "none">, + I32EnumAttrCase<"RNI", 1, "rni">, + I32EnumAttrCase<"RZI", 2, "rzi">, + I32EnumAttrCase<"RMI", 3, "rmi">, + I32EnumAttrCase<"RPI", 4, "rpi"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_L2PrefetchSizeEnum : I32EnumAttr<"L2PrefetchSize", "NVVM L2 Prefetch Size", [ + I32EnumAttrCase<"NONE", 0, "none">, + I32EnumAttrCase<"RESERVED", 1, "reserved">, + I32EnumAttrCase<"SIZE_64B", 2, "size_64b">, + I32EnumAttrCase<"SIZE_128B", 3, "size_128b">, + I32EnumAttrCase<"SIZE_256B", 4, "size_256b"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_LdStMatrixEltTypeEnum : I32EnumAttr<"LdStMatrixEltType", "Element type for ldmatrix and stmatrix", [ + I32EnumAttrCase<"B16", 0, "b16">, + I32EnumAttrCase<"B8", 1, "b8">, + I32EnumAttrCase<"B8X16_B6X16_P32", 2, "b8x16.b6x16_p32">, + I32EnumAttrCase<"B8X16_B4X16_P64", 3, "b8x16.b4x16_p64"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_LoadCacheModifierExtKindEnum : I32EnumAttr<"LoadCacheModifierExtKind", "NVVM load cache modifier kind(Ext)", [ + I32EnumAttrCase<"CA", 0, "ca">, + I32EnumAttrCase<"CG", 1, "cg">, + I32EnumAttrCase<"CS", 2, "cs">, + I32EnumAttrCase<"LU", 3, "lu">, + I32EnumAttrCase<"CV", 4, "cv">, + I32EnumAttrCase<"NONE", 5, "none"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_LoadCacheModifierKindEnum : I32EnumAttr<"LoadCacheModifierKind", "NVVM load cache modifier kind", [ + I32EnumAttrCase<"CA", 0, "ca">, + I32EnumAttrCase<"CG", 1, "cg">, + I32EnumAttrCase<"CS", 2, "cs">, + I32EnumAttrCase<"LU", 3, "lu">, + I32EnumAttrCase<"CV", 4, "cv"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_LoadShapeEnum : I32EnumAttr<"LoadShape", "shape attribute for ldmatrix", [ + I32EnumAttrCase<"M8N8", 0, "m8n8">, + I32EnumAttrCase<"M8N16", 1, "m8n16">, + I32EnumAttrCase<"M16N16", 3, "m16n16"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_LoadSrcFormatEnum : I32EnumAttr<"LoadSrcFormat", "source format for ldmatrix", [ + I32EnumAttrCase<"B6x16_P32", 0, "b6x16_p32">, + I32EnumAttrCase<"B4x16_P64", 1, "b4x16_p64">, + I32EnumAttrCase<"B8", 2, "b8"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_MBarrierLayoutEnum : I32EnumAttr<"MBarrierLayout", "NVVM MBarrier Layout", [ + I32EnumAttrCase<"V0", 0, "v0">, + I32EnumAttrCase<"V1", 1, "v1"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_MBarrierPhaseEnum : I32EnumAttr<"MBarrierPhase", "NVVM mbarrier phase type", [ + I32EnumAttrCase<"NONE", 0, "none">, + I32EnumAttrCase<"PRIMARY", 1, "primary">, + I32EnumAttrCase<"CONDITIONAL", 2, "conditional"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_MBarrierScopeKindEnum : I32EnumAttr<"MBarrierScopeKind", "NVVM MBarrier scope kind", [ + I32EnumAttrCase<"CTA", 0, "cta">, + I32EnumAttrCase<"CLUSTER", 1, "cluster"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_MBarrierSpaceKindEnum : I32EnumAttr<"MBarrierSpaceKind", "NVVM MBarrier space kind", [ + I32EnumAttrCase<"CTA", 0, "cta">, + I32EnumAttrCase<"CLUSTER", 1, "cluster"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_MBarrierTxnKindEnum : I32EnumAttr<"MBarrierTxnKind", "NVVM MBarrier Transaction kind", [ + I32EnumAttrCase<"ARRIVE", 0, "arrive">, + I32EnumAttrCase<"ARRIVE_DROP", 1, "arrive_drop">, + I32EnumAttrCase<"ARRIVE_EXPECT_TX", 2, "arrive_expect_tx">, + I32EnumAttrCase<"ARRIVE_DROP_EXPECT_TX", 3, "arrive_drop_expect_tx">, + I32EnumAttrCase<"EXPECT_TX", 4, "expect_tx">, + I32EnumAttrCase<"COMPLETE_TX", 5, "complete_tx"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_MBarrierWaitKindEnum : I32EnumAttr<"MBarrierWaitKind", "NVVM MBarrier wait kind", [ + I32EnumAttrCase<"TEST", 0, "test">, + I32EnumAttrCase<"TRY", 1, "try"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_MMAB1OpEnum : I32EnumAttr<"MMAB1Op", "MMA binary operations", [ + I32EnumAttrCase<"none", 0, "none">, + I32EnumAttrCase<"xor_popc", 1, "xor_popc">, + I32EnumAttrCase<"and_popc", 2, "and_popc"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_MMABlockScaleKindEnum : I32EnumAttr<"MMABlockScaleKind", "Block Scale Kind", [ + I32EnumAttrCase<"MXF8F6F4", 0, "mxf8f6f4">, + I32EnumAttrCase<"MXF4", 1, "mxf4">, + I32EnumAttrCase<"MXF4NVF4", 2, "mxf4nvf4"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_MMACtaCountEnum : I32EnumAttr<"MMACtaCount", "MMA CTA count", [ + I32EnumAttrCase<"cta1", 1, "cta1">, + I32EnumAttrCase<"cta2", 2, "cta2"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_MMAFragEnum : I32EnumAttr<"MMAFrag", "NVVM MMA frag type", [ + I32EnumAttrCase<"a", 0, "a">, + I32EnumAttrCase<"b", 1, "b">, + I32EnumAttrCase<"c", 2, "c"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_MMAIntOverflowEnum : I32EnumAttr<"MMAIntOverflow", "MMA overflow options", [ + I32EnumAttrCase<"satfinite", 1, "satfinite">, + I32EnumAttrCase<"wrapped", 0, "wrapped"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_MMAKindEnum : I32EnumAttr<"MMAKind", "MMA operation kind", [ + I32EnumAttrCase<"f8f6f4", 0, "f8f6f4"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_MMALayoutEnum : I32EnumAttr<"MMALayout", "NVVM MMA layout", [ + I32EnumAttrCase<"row", 0, "row">, + I32EnumAttrCase<"col", 1, "col"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_MMATypesEnum : I32EnumAttr<"MMATypes", "NVVM MMA types", [ + I32EnumAttrCase<"f16", 0, "f16">, + I32EnumAttrCase<"f32", 1, "f32">, + I32EnumAttrCase<"tf32", 2, "tf32">, + I32EnumAttrCase<"bf16", 9, "bf16">, + I32EnumAttrCase<"s8", 4, "s8">, + I32EnumAttrCase<"u8", 3, "u8">, + I32EnumAttrCase<"s32", 5, "s32">, + I32EnumAttrCase<"s4", 8, "s4">, + I32EnumAttrCase<"u4", 7, "u4">, + I32EnumAttrCase<"b1", 6, "b1">, + I32EnumAttrCase<"f64", 10, "f64">, + I32EnumAttrCase<"e4m3", 11, "e4m3">, + I32EnumAttrCase<"e5m2", 12, "e5m2">, + I32EnumAttrCase<"e3m2", 13, "e3m2">, + I32EnumAttrCase<"e2m3", 14, "e2m3">, + I32EnumAttrCase<"e2m1", 15, "e2m1"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_MatchSyncKindEnum : I32EnumAttr<"MatchSyncKind", "NVVM match sync kind", [ + I32EnumAttrCase<"any", 0, "any">, + I32EnumAttrCase<"all", 1, "all"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_MemOrderKindEnum : I32EnumAttr<"MemOrderKind", "NVVM Memory Ordering kind", [ + I32EnumAttrCase<"WEAK", 0, "weak">, + I32EnumAttrCase<"RELAXED", 1, "relaxed">, + I32EnumAttrCase<"ACQUIRE", 2, "acquire">, + I32EnumAttrCase<"RELEASE", 3, "release">, + I32EnumAttrCase<"ACQ_REL", 4, "acq_rel">, + I32EnumAttrCase<"SC", 5, "sc">, + I32EnumAttrCase<"MMIO", 6, "mmio">, + I32EnumAttrCase<"VOLATILE", 8, "volatile">, + I32EnumAttrCase<"CONSTANT", 7, "constant"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_MemScopeKindEnum : I32EnumAttr<"MemScopeKind", "NVVM Memory Scope kind", [ + I32EnumAttrCase<"CTA", 0, "cta">, + I32EnumAttrCase<"CLUSTER", 1, "cluster">, + I32EnumAttrCase<"GPU", 2, "gpu">, + I32EnumAttrCase<"SYS", 3, "sys"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_MulModeEnum : I32EnumAttr<"MulMode", "multiply mode attribute", [ + I32EnumAttrCase<"HI", 0, "hi">, + I32EnumAttrCase<"LO", 1, "lo">, + I32EnumAttrCase<"WIDE", 2, "wide"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_NVVMMemorySpaceEnum : I32EnumAttr<"NVVMMemorySpace", "NVVM Memory Space", [ + I32EnumAttrCase<"Generic", 0, "generic">, + I32EnumAttrCase<"Global", 1, "global">, + I32EnumAttrCase<"Shared", 3, "shared">, + I32EnumAttrCase<"Constant", 4, "constant">, + I32EnumAttrCase<"Local", 5, "local">, + I32EnumAttrCase<"Tensor", 6, "tensor">, + I32EnumAttrCase<"SharedCluster", 7, "shared_cluster"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_PermuteModeEnum : I32EnumAttr<"PermuteMode", "NVVM permute mode", [ + I32EnumAttrCase<"DEFAULT", 0, "default">, + I32EnumAttrCase<"F4E", 1, "f4e">, + I32EnumAttrCase<"B4E", 2, "b4e">, + I32EnumAttrCase<"RC8", 3, "rc8">, + I32EnumAttrCase<"ECL", 4, "ecl">, + I32EnumAttrCase<"ECR", 5, "ecr">, + I32EnumAttrCase<"RC16", 6, "rc16"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_PrefetchCacheLevelEnum : I32EnumAttr<"PrefetchCacheLevel", "NVVM Prefetch Cache Level", [ + I32EnumAttrCase<"L1", 0, "L1">, + I32EnumAttrCase<"L2", 1, "L2"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_ProxyKindEnum : I32EnumAttr<"ProxyKind", "Proxy kind", [ + I32EnumAttrCase<"alias", 0, "alias">, + I32EnumAttrCase<"async_", 1, "async">, + I32EnumAttrCase<"async_global", 2, "async.global">, + I32EnumAttrCase<"async_shared", 3, "async.shared">, + I32EnumAttrCase<"TENSORMAP", 4, "tensormap">, + I32EnumAttrCase<"GENERIC", 5, "generic"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_ReductionKindEnum : I32EnumAttr<"ReductionKind", "NVVM Reduction Kind attribute", [ + I32EnumAttrCase<"ADD", 1, "add">, + I32EnumAttrCase<"AND", 2, "and">, + I32EnumAttrCase<"MAX", 3, "max">, + I32EnumAttrCase<"MIN", 4, "min">, + I32EnumAttrCase<"OR", 5, "or">, + I32EnumAttrCase<"UMAX", 6, "umax">, + I32EnumAttrCase<"UMIN", 7, "umin">, + I32EnumAttrCase<"XOR", 8, "xor">, + I32EnumAttrCase<"FMIN", 9, "fmin">, + I32EnumAttrCase<"FMAX", 10, "fmax"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_ReductionOpEnum : I32EnumAttr<"ReductionOp", "Ops supported by red instruction", [ + I32EnumAttrCase<"AND", 0, "and">, + I32EnumAttrCase<"OR", 1, "or">, + I32EnumAttrCase<"XOR", 2, "xor">, + I32EnumAttrCase<"ADD", 3, "add">, + I32EnumAttrCase<"INC", 4, "inc">, + I32EnumAttrCase<"DEC", 5, "dec">, + I32EnumAttrCase<"MIN", 6, "min">, + I32EnumAttrCase<"MAX", 7, "max"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_ReductionTypeEnum : I32EnumAttr<"ReductionType", "types supported by red instruction", [ + I32EnumAttrCase<"B32", 0, "b32">, + I32EnumAttrCase<"B64", 1, "b64">, + I32EnumAttrCase<"U32", 2, "u32">, + I32EnumAttrCase<"U64", 3, "u64">, + I32EnumAttrCase<"S32", 4, "s32">, + I32EnumAttrCase<"S64", 5, "s64">, + I32EnumAttrCase<"F32", 6, "f32">, + I32EnumAttrCase<"F64", 7, "f64">, + I32EnumAttrCase<"F16", 8, "f16">, + I32EnumAttrCase<"F16X2", 9, "f16x2">, + I32EnumAttrCase<"BF16", 10, "bf16">, + I32EnumAttrCase<"BF16X2", 11, "bf16x2"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_SPCompressElemSizeEnum : I32EnumAttr<"SPCompressElemSize", "Sparse tensor compression element size", [ + I32EnumAttrCase<"B4", 4, "b4">, + I32EnumAttrCase<"B8", 8, "b8">, + I32EnumAttrCase<"B16", 16, "b16"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_SPCompressFactorTypeEnum : I32EnumAttr<"SPCompressFactorType", "Sparse tensor compression factor type", [ + I32EnumAttrCase<"SP2TO4", 0, "sp2to4"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_SPCompressIndexSizeEnum : I32EnumAttr<"SPCompressIndexSize", "Sparse tensor compression index size", [ + I32EnumAttrCase<"B2", 2, "b2">, + I32EnumAttrCase<"B4", 4, "b4"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_SPCompressOpKindEnum : I32EnumAttr<"SPCompressOpKind", "spcompress operation kind", [ + I32EnumAttrCase<"MIN", 0, "min">, + I32EnumAttrCase<"MAX", 1, "max"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_SPCompressRepFactorEnum : I32EnumAttr<"SPCompressRepFactor", "Sparse tensor compression repetition factor", [ + I32EnumAttrCase<"X1", 1, "x1">, + I32EnumAttrCase<"X2", 2, "x2">, + I32EnumAttrCase<"X4", 4, "x4">, + I32EnumAttrCase<"X8", 8, "x8">, + I32EnumAttrCase<"X16", 16, "x16">, + I32EnumAttrCase<"X32", 32, "x32">, + I32EnumAttrCase<"X64", 64, "x64">, + I32EnumAttrCase<"X128", 128, "x128"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_SPDecompressElemSizeEnum : I32EnumAttr<"SPDecompressElemSize", "Sparse tensor decompression element size", [ + I32EnumAttrCase<"B4", 4, "b4">, + I32EnumAttrCase<"B8", 8, "b8">, + I32EnumAttrCase<"B16", 16, "b16"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_SPDecompressFactorTypeEnum : I32EnumAttr<"SPDecompressFactorType", "Sparse tensor decompression factor type", [ + I32EnumAttrCase<"SP1TO2", 0, "sp1to2">, + I32EnumAttrCase<"SP1TO4", 1, "sp1to4">, + I32EnumAttrCase<"SP1TO8", 2, "sp1to8">, + I32EnumAttrCase<"SP1TO16", 3, "sp1to16">, + I32EnumAttrCase<"SP2TO4", 4, "sp2to4">, + I32EnumAttrCase<"SP2TO8", 5, "sp2to8">, + I32EnumAttrCase<"SP2TO16", 6, "sp2to16">, + I32EnumAttrCase<"SP4TO8", 7, "sp4to8">, + I32EnumAttrCase<"SP4TO16", 8, "sp4to16">, + I32EnumAttrCase<"SP8TO16", 9, "sp8to16"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_SPDecompressIndexSizeEnum : I32EnumAttr<"SPDecompressIndexSize", "Sparse tensor decompression index size", [ + I32EnumAttrCase<"B2", 2, "b2">, + I32EnumAttrCase<"B4", 4, "b4"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_SPDecompressRepFactorEnum : I32EnumAttr<"SPDecompressRepFactor", "Sparse tensor decompression repetition factor", [ + I32EnumAttrCase<"X1", 1, "x1">, + I32EnumAttrCase<"X2", 2, "x2">, + I32EnumAttrCase<"X4", 4, "x4">, + I32EnumAttrCase<"X8", 8, "x8">, + I32EnumAttrCase<"X16", 16, "x16">, + I32EnumAttrCase<"X32", 32, "x32">, + I32EnumAttrCase<"X64", 64, "x64"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_SaturationModeEnum : I32EnumAttr<"SaturationMode", "NVVM SaturationMode kind", [ + I32EnumAttrCase<"NONE", 0, "none">, + I32EnumAttrCase<"SATFINITE", 1, "satfinite">, + I32EnumAttrCase<"SAT", 2, "sat"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_SaturationModeKindEnum : I32EnumAttr<"SaturationModeKind", "NVVM SaturationMode kind", [ + I32EnumAttrCase<"NONE", 0, "none">, + I32EnumAttrCase<"SATFINITE", 1, "satfinite"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_ScaleVecSizeEnum : I32EnumAttr<"ScaleVecSize", "MMA Scale Vector Sizes", [ + I32EnumAttrCase<"X1", 0, "x1">, + I32EnumAttrCase<"X2", 1, "x2">, + I32EnumAttrCase<"X4", 2, "x4"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_SetMaxRegisterActionEnum : I32EnumAttr<"SetMaxRegisterAction", "NVVM set max register action", [ + I32EnumAttrCase<"decrease", 1, "decrease">, + I32EnumAttrCase<"increase", 0, "increase"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_SharedSpaceEnum : I32EnumAttr<"SharedSpace", "Shared memory space", [ + I32EnumAttrCase<"shared_cta", 0, "cta">, + I32EnumAttrCase<"shared_cluster", 1, "cluster"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_ShflKindEnum : I32EnumAttr<"ShflKind", "NVVM shuffle kind", [ + I32EnumAttrCase<"bfly", 0, "bfly">, + I32EnumAttrCase<"up", 1, "up">, + I32EnumAttrCase<"down", 2, "down">, + I32EnumAttrCase<"idx", 3, "idx"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_SparsityFormatEnum : I32EnumAttr<"SparsityFormat", "MMA Sparsity Format", [ + I32EnumAttrCase<"THREAD", 0, "thread"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_StateSpaceEnum : I32EnumAttr<"StateSpace", "NVVM State Space", [ + I32EnumAttrCase<"GENERIC", 0, "generic">, + I32EnumAttrCase<"GLOBAL", 1, "global">, + I32EnumAttrCase<"SHARED_CTA", 2, "shared_cta">, + I32EnumAttrCase<"SHARED_CLUSTER", 3, "shared_cluster">, + I32EnumAttrCase<"CONSTANT", 4, "constant">, + I32EnumAttrCase<"LOCAL", 5, "local">, + I32EnumAttrCase<"TENSOR", 6, "tensor"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_StoreCacheModifierKindEnum : I32EnumAttr<"StoreCacheModifierKind", "NVVM store cache modifier kind", [ + I32EnumAttrCase<"WB", 0, "wb">, + I32EnumAttrCase<"CG", 1, "cg">, + I32EnumAttrCase<"CS", 2, "cs">, + I32EnumAttrCase<"WT", 3, "wt">, + I32EnumAttrCase<"NONE", 4, "none"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_StoreShapeEnum : I32EnumAttr<"StoreShape", "shape attribute for stmatrix", [ + I32EnumAttrCase<"M8N8", 0, "m8n8">, + I32EnumAttrCase<"M16N8", 2, "m16n8"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_TCBarParamEnum : I32EnumAttr<"TCBarParam", "Cluster MMA Barrier Parameter Type", [ + I32EnumAttrCase<"a1t0", 0, "a1t0">, + I32EnumAttrCase<"a0tx", 1, "a0tx"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_TMALoadModeEnum : I32EnumAttr<"TMALoadMode", "NVVM TMA Load Mode", [ + I32EnumAttrCase<"TILE", 0, "tile">, + I32EnumAttrCase<"IM2COL", 1, "im2col">, + I32EnumAttrCase<"IM2COL_W", 2, "im2col_w">, + I32EnumAttrCase<"IM2COL_W_128", 3, "im2col_w_128">, + I32EnumAttrCase<"TILE_GATHER4", 4, "tile_gather4"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_TMAReduxKindEnum : I32EnumAttr<"TMAReduxKind", "NVVM TMA redux kind", [ + I32EnumAttrCase<"ADD", 0, "add">, + I32EnumAttrCase<"MAX", 2, "max">, + I32EnumAttrCase<"MIN", 1, "min">, + I32EnumAttrCase<"INC", 3, "inc">, + I32EnumAttrCase<"DEC", 4, "dec">, + I32EnumAttrCase<"AND", 5, "and">, + I32EnumAttrCase<"OR", 6, "or">, + I32EnumAttrCase<"XOR", 7, "xor"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_TMAStoreModeEnum : I32EnumAttr<"TMAStoreMode", "NVVM TMA Store Mode", [ + I32EnumAttrCase<"TILE", 0, "tile">, + I32EnumAttrCase<"IM2COL", 1, "im2col">, + I32EnumAttrCase<"TILE_SCATTER4", 2, "tile_scatter4">, + I32EnumAttrCase<"IM2COL_W", 3, "im2col_w"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_Tcgen05CpMulticastEnum : I32EnumAttr<"Tcgen05CpMulticast", "tcgen05 cp multicast", [ + I32EnumAttrCase<"NONE", 0, "none">, + I32EnumAttrCase<"WARPX2_02_13", 1, "warpx2_02_13">, + I32EnumAttrCase<"WARPX2_01_23", 2, "warpx2_01_23">, + I32EnumAttrCase<"WARPX4", 3, "warpx4"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_Tcgen05CpShapeEnum : I32EnumAttr<"Tcgen05CpShape", "tcgen05 cp shapes", [ + I32EnumAttrCase<"SHAPE_128x256b", 0, "shape_128x256b">, + I32EnumAttrCase<"SHAPE_4x256b", 1, "shape_4x256b">, + I32EnumAttrCase<"SHAPE_128x128b", 2, "shape_128x128b">, + I32EnumAttrCase<"SHAPE_64x128b", 3, "shape_64x128b">, + I32EnumAttrCase<"SHAPE_32x128b", 4, "shape_32x128b"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_Tcgen05CpSrcFormatEnum : I32EnumAttr<"Tcgen05CpSrcFormat", "tcgen05 cp source format", [ + I32EnumAttrCase<"B6x16_P32", 0, "b6x16_p32">, + I32EnumAttrCase<"B4x16_P64", 1, "b4x16_p64"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_Tcgen05FenceKindEnum : I32EnumAttr<"Tcgen05FenceKind", "NVVM Tcgen05 fence kind", [ + I32EnumAttrCase<"BEFORE_THREAD_SYNC", 0, "before">, + I32EnumAttrCase<"AFTER_THREAD_SYNC", 1, "after"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_Tcgen05LdStShapeEnum : I32EnumAttr<"Tcgen05LdStShape", "allowed 32-bit signless integer cases: 0, 1, 2, 3, 4", [ + I32EnumAttrCase<"SHAPE_16X64B", 0, "shape_16x64b">, + I32EnumAttrCase<"SHAPE_16X128B", 1, "shape_16x128b">, + I32EnumAttrCase<"SHAPE_16X256B", 2, "shape_16x256b">, + I32EnumAttrCase<"SHAPE_32X32B", 3, "shape_32x32b">, + I32EnumAttrCase<"SHAPE_16X32BX2", 4, "shape_16x32bx2"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_Tcgen05MMABlockScaleEnum : I32EnumAttr<"Tcgen05MMABlockScale", "tcgen05.mma block scale attribute", [ + I32EnumAttrCase<"DEFAULT", 0, "default">, + I32EnumAttrCase<"BLOCK16", 1, "block16">, + I32EnumAttrCase<"BLOCK32", 2, "block32"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_Tcgen05MMACollectorBBufferEnum : I32EnumAttr<"Tcgen05MMACollectorBBuffer", "tcgen05 MMA Collector Buffer B Attribute", [ + I32EnumAttrCase<"B0", 0, "b0">, + I32EnumAttrCase<"B1", 1, "b1">, + I32EnumAttrCase<"B2", 2, "b2">, + I32EnumAttrCase<"B3", 3, "b3"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_Tcgen05MMACollectorOpEnum : I32EnumAttr<"Tcgen05MMACollectorOp", "tcgen05.mma Collector Buffer Operation", [ + I32EnumAttrCase<"DISCARD", 0, "discard">, + I32EnumAttrCase<"LASTUSE", 1, "lastuse">, + I32EnumAttrCase<"FILL", 2, "fill">, + I32EnumAttrCase<"USE", 3, "use"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_Tcgen05MMAKindEnum : I32EnumAttr<"Tcgen05MMAKind", "tcgen05 MMA Supported Types", [ + I32EnumAttrCase<"F16", 0, "f16">, + I32EnumAttrCase<"TF32", 1, "tf32">, + I32EnumAttrCase<"F8F6F4", 2, "f8f6f4">, + I32EnumAttrCase<"I8", 3, "i8">, + I32EnumAttrCase<"MXF8F6F4", 4, "mxf8f6f4">, + I32EnumAttrCase<"MXF4", 5, "mxf4">, + I32EnumAttrCase<"MXF4NVF4", 6, "mxf4nvf4">, + I32EnumAttrCase<"TI16", 7, "ti16"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_Tcgen05WaitKindEnum : I32EnumAttr<"Tcgen05WaitKind", "NVVM Tcgen05 wait kind", [ + I32EnumAttrCase<"LOAD", 0, "load">, + I32EnumAttrCase<"STORE", 1, "store"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_TensormapElemtypeEnum : I32EnumAttr<"TensormapElemtype", "NVVM Tensormap Elemtype", [ + I32EnumAttrCase<"U8", 0, "u8">, + I32EnumAttrCase<"U16", 1, "u16">, + I32EnumAttrCase<"U32", 2, "u32">, + I32EnumAttrCase<"S32", 3, "s32">, + I32EnumAttrCase<"U64", 4, "u64">, + I32EnumAttrCase<"S64", 5, "s64">, + I32EnumAttrCase<"F16", 6, "f16">, + I32EnumAttrCase<"F32", 7, "f32">, + I32EnumAttrCase<"F32_FTZ", 8, "f32.ftz">, + I32EnumAttrCase<"F64", 9, "f64">, + I32EnumAttrCase<"BF16", 10, "bf16">, + I32EnumAttrCase<"TF32", 11, "tf32">, + I32EnumAttrCase<"TF32_FTZ", 12, "tf32.ftz">, + I32EnumAttrCase<"B4x16", 13, "b4x16">, + I32EnumAttrCase<"B4x16P64", 14, "b4x16_p64">, + I32EnumAttrCase<"B6x16P32", 15, "b6x16_p32"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_TensormapFieldEnum : I32EnumAttr<"TensormapField", "NVVM Tensormap Field Kind", [ + I32EnumAttrCase<"GLOBAL_ADDRESS", 0, "global_address">, + I32EnumAttrCase<"RANK", 1, "rank">, + I32EnumAttrCase<"BOX_DIM", 2, "box_dim">, + I32EnumAttrCase<"GLOBAL_DIM", 3, "global_dim">, + I32EnumAttrCase<"GLOBAL_STRIDE", 4, "global_stride">, + I32EnumAttrCase<"ELEMENT_STRIDE", 5, "element_stride">, + I32EnumAttrCase<"ELEMTYPE", 6, "elemtype">, + I32EnumAttrCase<"INTERLEAVE_LAYOUT", 7, "interleave_layout">, + I32EnumAttrCase<"SWIZZLE_MODE", 8, "swizzle_mode">, + I32EnumAttrCase<"SWIZZLE_ATOMICITY", 9, "swizzle_atomicity">, + I32EnumAttrCase<"FILL_MODE", 10, "fill_mode"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_TensormapFillModeEnum : I32EnumAttr<"TensormapFillMode", "NVVM Tensormap Fill Mode", [ + I32EnumAttrCase<"ZERO", 0, "zero">, + I32EnumAttrCase<"OOB_NAN", 1, "oob_nan"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_TensormapInterleaveLayoutEnum : I32EnumAttr<"TensormapInterleaveLayout", "NVVM Tensormap Interleave Layout", [ + I32EnumAttrCase<"NO_INTERLEAVE", 0, "no_interleave">, + I32EnumAttrCase<"B16", 1, "b16">, + I32EnumAttrCase<"B32", 2, "b32"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_TensormapSwizzleAtomicityEnum : I32EnumAttr<"TensormapSwizzleAtomicity", "NVVM Tensormap Swizzle Atomicity", [ + I32EnumAttrCase<"B16", 0, "b16">, + I32EnumAttrCase<"B32", 1, "b32">, + I32EnumAttrCase<"B32_FLIP_8B", 2, "b32_flip_b8">, + I32EnumAttrCase<"B64", 3, "b64"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_TensormapSwizzleModeEnum : I32EnumAttr<"TensormapSwizzleMode", "NVVM Tensormap Swizzle Mode", [ + I32EnumAttrCase<"NO_SWIZZLING", 0, "no_swizzling">, + I32EnumAttrCase<"B32", 1, "b32">, + I32EnumAttrCase<"B64", 2, "b64">, + I32EnumAttrCase<"B128", 3, "b128">, + I32EnumAttrCase<"B96", 4, "b96"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_TmemLayoutEnum : I32EnumAttr<"TmemLayout", "Tensor Memory Layout Enumerated Type", [ + I32EnumAttrCase<"tmem_16dp_128bit", 0, "tmem_16dp_128bit">, + I32EnumAttrCase<"tmem_16dp_256bit", 1, "tmem_16dp_256bit">, + I32EnumAttrCase<"tmem_32dp_32bit", 2, "tmem_32dp_32bit">, + I32EnumAttrCase<"tmem_16dp_64bit", 3, "tmem_16dp_64bit">, + I32EnumAttrCase<"tmem_16dp_32bit_t0_t15", 4, "tmem_16dp_32bit_t0_t15">, + I32EnumAttrCase<"tmem_16dp_32bit_t16_t31", 5, "tmem_16dp_32bit_t16_t31"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_ValidatePatternEnum : I32EnumAttr<"ValidatePattern", "NVVM validate data pattern", [ + I32EnumAttrCase<"NONE", 0, "none">, + I32EnumAttrCase<"PER_16BYTES_80000000", 1, "per_16bytes_80000000">, + I32EnumAttrCase<"PER_16BYTES_8000", 2, "per_16bytes_8000">, + I32EnumAttrCase<"PER_16BYTES_80", 3, "per_16bytes_80">, + I32EnumAttrCase<"PER_16BYTES_8", 4, "per_16bytes_8">, + I32EnumAttrCase<"PER_ELEMENT_FF", 5, "per_element_ff">, + I32EnumAttrCase<"PER_ELEMENT_80000000", 6, "per_element_80000000">, + I32EnumAttrCase<"PER_ELEMENT_8000", 7, "per_element_8000">, + I32EnumAttrCase<"PER_ELEMENT_80", 8, "per_element_80">, + I32EnumAttrCase<"PER_ELEMENT_8", 9, "per_element_8">, + I32EnumAttrCase<"PER_16BYTES_80000000_PAIRED", 10, "per_16bytes_80000000_paired"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_VoteSyncKindEnum : I32EnumAttr<"VoteSyncKind", "NVVM vote sync kind", [ + I32EnumAttrCase<"any", 0, "any">, + I32EnumAttrCase<"all", 1, "all">, + I32EnumAttrCase<"ballot", 2, "ballot">, + I32EnumAttrCase<"uni", 3, "uni"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_WGMMAScaleInEnum : I32EnumAttr<"WGMMAScaleIn", "WGMMA overflow options", [ + I32EnumAttrCase<"one", 1, "one">, + I32EnumAttrCase<"neg", 2, "neg"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_WGMMAScaleOutEnum : I32EnumAttr<"WGMMAScaleOut", "WGMMA input predicate", [ + I32EnumAttrCase<"zero", 0, "zero">, + I32EnumAttrCase<"one", 1, "one"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_WGMMATypesEnum : I32EnumAttr<"WGMMATypes", "NVVM WGMMA types", [ + I32EnumAttrCase<"f16", 0, "f16">, + I32EnumAttrCase<"tf32", 1, "tf32">, + I32EnumAttrCase<"u8", 2, "u8">, + I32EnumAttrCase<"s8", 3, "s8">, + I32EnumAttrCase<"b1", 4, "b1">, + I32EnumAttrCase<"bf16", 5, "bf16">, + I32EnumAttrCase<"e4m3", 6, "e4m3">, + I32EnumAttrCase<"e5m2", 7, "e5m2">, + I32EnumAttrCase<"f32", 8, "f32">, + I32EnumAttrCase<"s32", 9, "s32"> + ]> { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let genSpecializedAttr = 0; +} + +def NVVM_AtomicOpKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let assemblyFormat = "$value"; +} + +def NVVM_BarrierReductionAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_BarrierReduxKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_BlockScaleFormatAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_CTAGroupKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_CVTPackFloatKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_CacheEvictionPriorityAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let assemblyFormat = "$value"; +} + +def NVVM_ClusterLaunchControlQueryTypeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let assemblyFormat = "$value"; +} + +def NVVM_CompareOpKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_ConvertFP4TypeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_ConvertFP8TypeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_ConvertScaleKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_DotAccumulateTypeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_EvictKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_FPRoundingModeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_GridDepActionAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let assemblyFormat = "$value"; +} + +def NVVM_IntegerRoundingModeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_L2PrefetchSizeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_LdStMatrixEltTypeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_LoadCacheModifierAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let assemblyFormat = "$value"; +} + +def NVVM_LoadCacheModifierAttrExt : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_LoadShapeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_LoadSrcFormatAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_MBarrierLayoutAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_MBarrierScopeKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_MBarrierSpaceKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_MBarrierTxnKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_MBarrierWaitKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_MMAB1OpAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_MMABlockScaleKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_MMACtaCountAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_MMAFragAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_MMAIntOverflowAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_MMAKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_MMALayoutAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_MMATypesAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_MatchSyncKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let assemblyFormat = "$value"; +} + +def NVVM_MemOrderKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_MemScopeKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_MulModeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let assemblyFormat = "$value"; +} + +def NVVM_NVVMMemorySpaceAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_NVVM_MBarrierPhaseAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_PermuteModeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_PrefetchCacheLevelAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let assemblyFormat = "$value"; +} + +def NVVM_ProxyKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_ReductionKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let assemblyFormat = "$value"; +} + +def NVVM_ReductionOpAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_ReductionTypeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_SPCompressElemSizeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_SPCompressFactorTypeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_SPCompressIndexSizeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_SPCompressOpKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_SPCompressRepFactorAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_SPDecompressElemSizeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_SPDecompressFactorTypeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_SPDecompressIndexSizeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_SPDecompressRepFactorAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_SaturationModeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_SaturationModeKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_ScaleVecSizeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_SetMaxRegisterActionAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let assemblyFormat = "$value"; +} + +def NVVM_SharedSpaceAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_ShflKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let assemblyFormat = "$value"; +} + +def NVVM_SparsityFormatAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_StateSpaceAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_StoreCacheModifierAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_StoreShapeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_TCBarParamEnumAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_TMALoadModeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_TMAReduxKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_TMAStoreModeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_Tcgen05CpMulticastAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_Tcgen05CpShapeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_Tcgen05CpSrcFormatAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_Tcgen05FenceKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_Tcgen05LdStShapeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_Tcgen05MMABlockScaleAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_Tcgen05MMACollectorBBufferAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_Tcgen05MMACollectorOpAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_Tcgen05MMAKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_Tcgen05WaitKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_TensormapElemtypeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_TensormapFieldAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let assemblyFormat = "$value"; +} + +def NVVM_TensormapFillModeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_TensormapInterleaveLayoutAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_TensormapSwizzleAtomicityAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_TensormapSwizzleModeAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_TmemLayoutEnumAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_ValidatePatternAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_VoteSyncKindAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; + let assemblyFormat = "$value"; +} + +def NVVM_WGMMAScaleInAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_WGMMAScaleOutAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +def NVVM_WGMMATypesAttr : EnumAttr { + let cppNamespace = "::mlir::cutlass_compiler::nvvm"; +} + +#endif // MLIR_DIALECTS_NVVM_NVVMENUMS_TD diff --git a/mlir/dialects/nvvm/NVVMInference.cpp b/mlir/dialects/nvvm/NVVMInference.cpp new file mode 100644 index 00000000..78a9e56f --- /dev/null +++ b/mlir/dialects/nvvm/NVVMInference.cpp @@ -0,0 +1,253 @@ +//===- NVVMInference.cpp - result types the compiler infers ---------------===// +// +// Generated from the nvidia-cutlass-dsl 4.8.0 compiler (its inference +// probed by varying operand types); do not edit. An operation without a +// recovered rule fails inference, so the result types written stand. + +#include "mlir/dialects/nvvm/NVVMDialect.h" + +using namespace mlir; +using namespace ::mlir::cutlass_compiler::nvvm; + +LogicalResult AddFOp::inferReturnTypes(MLIRContext *, std::optional, + ValueRange operands, + DictionaryAttr attrs, PropertyRef props, + RegionRange regions, + SmallVectorImpl &out) { + Adaptor adaptor(operands, attrs, props, regions); + auto group = adaptor.getODSOperands(0); + if (group.empty()) + return failure(); + out.push_back(group.front().getType()); + return success(); +} + +LogicalResult +AddPackedF32x2Op::inferReturnTypes(MLIRContext *, std::optional, + ValueRange operands, DictionaryAttr attrs, + PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult +AtomicRMWOp::inferReturnTypes(MLIRContext *, std::optional, + ValueRange operands, DictionaryAttr attrs, + PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult +BarrierOp::inferReturnTypes(MLIRContext *, std::optional, + ValueRange operands, DictionaryAttr attrs, + PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult ClusterLaunchControlQueryCancelOp::inferReturnTypes( + MLIRContext *, std::optional, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult +CompareAndSetOp::inferReturnTypes(MLIRContext *, std::optional, + ValueRange operands, DictionaryAttr attrs, + PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + Adaptor adaptor(operands, attrs, props, regions); + auto group = adaptor.getODSOperands(0); + if (group.empty()) + return failure(); + out.push_back(group.front().getType()); + return success(); +} + +LogicalResult FAbsOp::inferReturnTypes(MLIRContext *, std::optional, + ValueRange operands, + DictionaryAttr attrs, PropertyRef props, + RegionRange regions, + SmallVectorImpl &out) { + Adaptor adaptor(operands, attrs, props, regions); + auto group = adaptor.getODSOperands(0); + if (group.empty()) + return failure(); + out.push_back(group.front().getType()); + return success(); +} + +LogicalResult +FMAPackedF32x2Op::inferReturnTypes(MLIRContext *, std::optional, + ValueRange operands, DictionaryAttr attrs, + PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult FmaOp::inferReturnTypes(MLIRContext *, std::optional, + ValueRange operands, DictionaryAttr attrs, + PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult InternalMmaBlockScaleOp::inferReturnTypes( + MLIRContext *, std::optional, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + Adaptor adaptor(operands, attrs, props, regions); + auto group = adaptor.getODSOperands(8); + if (group.empty()) + return failure(); + out.push_back(group.front().getType()); + return success(); +} + +LogicalResult InternalMmaSparseBlockScaleOp::inferReturnTypes( + MLIRContext *, std::optional, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + Adaptor adaptor(operands, attrs, props, regions); + auto group = adaptor.getODSOperands(8); + if (group.empty()) + return failure(); + out.push_back(group.front().getType()); + return success(); +} + +LogicalResult +LdMatrixOp::inferReturnTypes(MLIRContext *, std::optional, + ValueRange operands, DictionaryAttr attrs, + PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult MBarrierArriveDropExpectTxOp::inferReturnTypes( + MLIRContext *, std::optional, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult MBarrierArriveDropOp::inferReturnTypes( + MLIRContext *, std::optional, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult MBarrierArriveExpectTxOp::inferReturnTypes( + MLIRContext *, std::optional, ValueRange operands, + DictionaryAttr attrs, PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult +MBarrierArriveOp::inferReturnTypes(MLIRContext *, std::optional, + ValueRange operands, DictionaryAttr attrs, + PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult +MBarrierTestWaitOp::inferReturnTypes(MLIRContext *, std::optional, + ValueRange operands, DictionaryAttr attrs, + PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult +MBarrierTryWaitOp::inferReturnTypes(MLIRContext *, std::optional, + ValueRange operands, DictionaryAttr attrs, + PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult +MatchSyncOp::inferReturnTypes(MLIRContext *, std::optional, + ValueRange operands, DictionaryAttr attrs, + PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult MulOp::inferReturnTypes(MLIRContext *, std::optional, + ValueRange operands, DictionaryAttr attrs, + PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult +MulPackedF32x2Op::inferReturnTypes(MLIRContext *, std::optional, + ValueRange operands, DictionaryAttr attrs, + PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult ReduxOp::inferReturnTypes(MLIRContext *, std::optional, + ValueRange operands, + DictionaryAttr attrs, PropertyRef props, + RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult RsqrtOp::inferReturnTypes(MLIRContext *, std::optional, + ValueRange operands, + DictionaryAttr attrs, PropertyRef props, + RegionRange regions, + SmallVectorImpl &out) { + Adaptor adaptor(operands, attrs, props, regions); + auto group = adaptor.getODSOperands(0); + if (group.empty()) + return failure(); + out.push_back(group.front().getType()); + return success(); +} + +LogicalResult ShflOp::inferReturnTypes(MLIRContext *, std::optional, + ValueRange operands, + DictionaryAttr attrs, PropertyRef props, + RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult SubFOp::inferReturnTypes(MLIRContext *, std::optional, + ValueRange operands, + DictionaryAttr attrs, PropertyRef props, + RegionRange regions, + SmallVectorImpl &out) { + Adaptor adaptor(operands, attrs, props, regions); + auto group = adaptor.getODSOperands(0); + if (group.empty()) + return failure(); + out.push_back(group.front().getType()); + return success(); +} + +LogicalResult +SubPackedF32x2Op::inferReturnTypes(MLIRContext *, std::optional, + ValueRange operands, DictionaryAttr attrs, + PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} + +LogicalResult +VoteSyncOp::inferReturnTypes(MLIRContext *, std::optional, + ValueRange operands, DictionaryAttr attrs, + PropertyRef props, RegionRange regions, + SmallVectorImpl &out) { + return failure(); +} diff --git a/mlir/dialects/nvvm/NVVMOps.td b/mlir/dialects/nvvm/NVVMOps.td new file mode 100644 index 00000000..d77563d2 --- /dev/null +++ b/mlir/dialects/nvvm/NVVMOps.td @@ -0,0 +1,6445 @@ +// The nvvm dialect's operations, generated from the nvidia-cutlass-dsl 4.8.0 release; do not edit. +// +// Sources: the operation bindings (`_nvvm_ops_gen.py`: names, classes, +// docstrings, operand/result/attribute structure, inference) and the live +// compiler (`_cutlass_ir` loaded from Python): each operand and result +// constraint carries the verifier's wording and accepts every type class the +// verifier let through when probed with one instance of every type; traits +// and interfaces are the registered operation's; memory effects come from +// loop-invariant code motion (hoisted: Pure) and CSE (merged: at most +// reads). Operations whose effects include writes or could not be told +// apart carry none (unknown effects). The compiler's own verifiers (operand +// combinations, shapes, instruction legality) are left to it. +// +// Operations keep the generic assembly form. +// +// Result types the compiler infers without a recovered rule (written types stand): +// nvvm.add.packed.f32x2 +// nvvm.atomicrmw +// nvvm.barrier +// nvvm.clusterlaunchcontrol.query.cancel +// nvvm.fma +// nvvm.fma.packed.f32x2 +// nvvm.ldmatrix +// nvvm.match.sync +// nvvm.mbarrier.arrive +// nvvm.mbarrier.arrive.expect_tx +// nvvm.mbarrier.arrive_drop +// nvvm.mbarrier.arrive_drop.expect_tx +// nvvm.mbarrier.test.wait +// nvvm.mbarrier.try_wait +// nvvm.mul (`mode = wide` doubles the operand width) +// nvvm.mul.packed.f32x2 +// nvvm.redux.sync +// nvvm.shfl.sync (`return_value_and_is_valid` returns `!llvm.struct<(T, i1)>`) +// nvvm.sub.packed.f32x2 +// nvvm.vote.sync +// +// Constraints not recovered: +// nvvm.convert.bf16x2.to.f8x2 `dstTy`: `fnv8E5M3FNU` has no type here and is not accepted + +#ifndef MLIR_DIALECTS_NVVM_NVVMOPS_TD +#define MLIR_DIALECTS_NVVM_NVVMOPS_TD + +include "mlir/dialects/nvvm/NVVMAttrs.td" +include "mlir/dialects/nvvm/NVVMEnums.td" + +include "mlir/Interfaces/InferTypeOpInterface.td" +include "mlir/Interfaces/SideEffectInterfaces.td" + +// Operand and result constraints, in the compiler's words. +def NVVM_AnyType : Type, "any type">; +def NVVM_FixedLengthVectorOf32Bit : Type($_self)">, "fixed-length vector of 32-bit signless integer values of length 4/8">; +def NVVM_FixedLengthVectorOf8Bit : Type($_self)">, "fixed-length vector of 8-bit signless integer values of length 4 or fixed-length vector of 16-bit signless integer values of length 2">; +def NVVM_FloatingPointOrFixedLengthVector : Type($_self) || ::llvm::isa<::mlir::VectorType>($_self)">, "floating-point or fixed-length vector of floating-point values of length 2">; +def NVVM_LLVMDialectCompatibleType : Type($_self) || ::llvm::isa<::mlir::Float16Type>($_self) || ::llvm::isa<::mlir::BFloat16Type>($_self) || ::llvm::isa<::mlir::Float32Type>($_self) || ::llvm::isa<::mlir::Float64Type>($_self) || ::llvm::isa<::mlir::VectorType>($_self) || ::llvm::isa<::mlir::LLVM::LLVMPointerType>($_self) || ::llvm::isa<::mlir::LLVM::LLVMStructType>($_self) || ::llvm::isa<::mlir::LLVM::LLVMArrayType>($_self) || ::llvm::isa<::mlir::LLVM::LLVMFunctionType>($_self)">, "LLVM dialect-compatible type">; +def NVVM_LLVMPointerInAddressSpace0 : Type($_self)">, "LLVM pointer in address space 0">; +def NVVM_LLVMPointerInAddressSpace02 : Type($_self)">, "LLVM pointer in address space 0 or LLVM pointer in address space 3">; +def NVVM_LLVMPointerInAddressSpace03 : Type($_self)">, "LLVM pointer in address space 0 or LLVM pointer in address space 7">; +def NVVM_LLVMPointerInAddressSpace04 : Type($_self)">, "LLVM pointer in address space 0 or LLVM pointer in address space 3 or LLVM pointer in address space 7">; +def NVVM_LLVMPointerInAddressSpace05 : Type($_self)">, "LLVM pointer in address space 0 or LLVM pointer in address space 3 or LLVM pointer in address space 1 or LLVM pointer in address space 7">; +def NVVM_LLVMPointerInAddressSpace1 : Type($_self)">, "LLVM pointer in address space 1">; +def NVVM_LLVMPointerInAddressSpace12 : Type($_self)">, "LLVM pointer in address space 1 or LLVM pointer in address space 0 or LLVM pointer in address space 3 or LLVM pointer in address space 7">; +def NVVM_LLVMPointerInAddressSpace13 : Type($_self)">, "LLVM pointer in address space 1 or LLVM pointer in address space 5 or LLVM pointer in address space 0 or LLVM pointer in address space 4">; +def NVVM_LLVMPointerInAddressSpace14 : Type($_self)">, "LLVM pointer in address space 1 or LLVM pointer in address space 3">; +def NVVM_LLVMPointerInAddressSpace3 : Type($_self)">, "LLVM pointer in address space 3">; +def NVVM_LLVMPointerInAddressSpace32 : Type($_self)">, "LLVM pointer in address space 3 or LLVM pointer in address space 7">; +def NVVM_LLVMPointerInAddressSpace6 : Type($_self)">, "LLVM pointer in address space 6">; +def NVVM_LLVMPointerInAddressSpace62 : Type($_self)">, "LLVM pointer in address space 6 or 64-bit signless integer">; +def NVVM_LLVMPointerInAddressSpace7 : Type($_self)">, "LLVM pointer in address space 7">; +def NVVM_LLVMPointerInAddressSpace72 : Type($_self)">, "LLVM pointer in address space 7 or LLVM pointer in address space 3">; +def NVVM_LLVMPointerType : Type($_self)">, "LLVM pointer type">; +def NVVM_LLVMPointerTypeOrLLVMPointer : Type($_self)">, "LLVM pointer type or LLVM pointer in address space 3">; +def NVVM_LLVMStructureType : Type($_self)">, "LLVM structure type">; +def NVVM_LLVMStructureTypeOr64Bit : Type($_self) || ::llvm::isa<::mlir::LLVM::LLVMStructType>($_self)">, "LLVM structure type or 64-bit float">; +def NVVM_LLVMTypeWithSize : Type($_self) || ::llvm::isa<::mlir::Float16Type>($_self) || ::llvm::isa<::mlir::BFloat16Type>($_self) || ::llvm::isa<::mlir::Float32Type>($_self) || ::llvm::isa<::mlir::Float64Type>($_self) || ::llvm::isa<::mlir::VectorType>($_self) || ::llvm::isa<::mlir::LLVM::LLVMPointerType>($_self) || ::llvm::isa<::mlir::LLVM::LLVMStructType>($_self) || ::llvm::isa<::mlir::LLVM::LLVMArrayType>($_self)">, "LLVM type with size">; +def NVVM_T128BitSignlessInteger : Type, "128-bit signless integer">; +def NVVM_T16BitFloatOrBfloat16Type : Type($_self) || ::llvm::isa<::mlir::BFloat16Type>($_self) || ::llvm::isa<::mlir::Float32Type>($_self) || ::llvm::isa<::mlir::Float64Type>($_self) || ::llvm::isa<::mlir::VectorType>($_self)">, "16-bit float or bfloat16 type or 32-bit float or 64-bit float or vector of 16-bit float or bfloat16 type or 32-bit float or 64-bit float values of length 2">; +def NVVM_T16BitSignlessInteger : Type, "16-bit signless integer">; +def NVVM_T16BitSignlessInteger2 : Type, "16-bit signless integer">, BuildableType<"$_builder.getIntegerType(16)">; +def NVVM_T16BitSignlessIntegerOr32 : Type($_self) || ::llvm::isa<::mlir::BFloat16Type>($_self) || ::llvm::isa<::mlir::Float32Type>($_self) || ::llvm::isa<::mlir::Float64Type>($_self) || ::llvm::isa<::mlir::VectorType>($_self)">, "16-bit signless integer or 32-bit signless integer or 64-bit signless integer or 128-bit signless integer or 16-bit float or bfloat16 type or 32-bit float or 64-bit float or 128-bit float or vector of 32-bit float values of length 2/4 or vector of 16-bit float or bfloat16 type values of length 2/4/8">; +def NVVM_T16BitSignlessIntegerOr322 : Type, "16-bit signless integer or 32-bit signless integer">; +def NVVM_T16BitSignlessIntegerOr323 : Type, "16-bit signless integer or 32-bit signless integer or 64-bit signless integer">; +def NVVM_T16BitSignlessIntegerOrVector : Type($_self)">, "16-bit signless integer or vector of 8-bit signless integer values of length 2">; +def NVVM_T1BitSignlessInteger : Type, "1-bit signless integer">; +def NVVM_T1BitSignlessInteger2 : Type, "1-bit signless integer">, BuildableType<"$_builder.getIntegerType(1)">; +def NVVM_T1BitSignlessIntegerOr32 : Type, "1-bit signless integer or 32-bit signless integer">; +def NVVM_T1BitSignlessIntegerOrLLVM : Type($_self)">, "1-bit signless integer or LLVM structure type">; +def NVVM_T32BitFloat : Type($_self)">, "32-bit float">; +def NVVM_T32BitFloat2 : Type($_self)">, "32-bit float">, BuildableType<"::mlir::Float32Type::get($_builder.getContext())">; +def NVVM_T32BitFloatOr64Bit : Type($_self) || ::llvm::isa<::mlir::Float64Type>($_self)">, "32-bit float or 64-bit float">; +def NVVM_T32BitSignlessInteger : Type, "32-bit signless integer">; +def NVVM_T32BitSignlessInteger2 : Type, "32-bit signless integer">, BuildableType<"$_builder.getIntegerType(32)">; +def NVVM_T32BitSignlessIntegerOr1 : Type, "32-bit signless integer or 1-bit signless integer">; +def NVVM_T32BitSignlessIntegerOr32 : Type($_self)">, "32-bit signless integer or 32-bit float">; +def NVVM_T32BitSignlessIntegerOr322 : Type($_self) || ::llvm::isa<::mlir::LLVM::LLVMStructType>($_self)">, "32-bit signless integer or 32-bit float or LLVM struct type">; +def NVVM_T32BitSignlessIntegerOr64 : Type, "32-bit signless integer or 64-bit signless integer">; +def NVVM_T32BitSignlessIntegerOr642 : Type($_self) || ::llvm::isa<::mlir::BFloat16Type>($_self) || ::llvm::isa<::mlir::Float32Type>($_self) || ::llvm::isa<::mlir::Float64Type>($_self) || ::llvm::isa<::mlir::VectorType>($_self)">, "32-bit signless integer or 64-bit signless integer or 64-bit float or bfloat16 type or 16-bit float or 32-bit float or vector of 32-bit float or 32-bit signless integer values of length 2/4 or vector of 16-bit float or bfloat16 type values of length 2/4/8">; +def NVVM_T32BitSignlessIntegerOr643 : Type($_self)">, "32-bit signless integer or 64-bit signless integer or vector of 32-bit signless integer or 64-bit signless integer values of length 2/4">; +def NVVM_T32BitSignlessIntegerOrLLVM : Type($_self)">, "32-bit signless integer or LLVM struct type">; +def NVVM_T32BitSignlessIntegerOrVector : Type($_self)">, "32-bit signless integer or vector of 32-bit signless integer values of length 2/4/8/16/32/64">; +def NVVM_T32BitSignlessIntegerOrVector2 : Type($_self)">, "32-bit signless integer or vector of 32-bit signless integer values of length 2/4/8/16/32/64/128">; +def NVVM_T32BitSignlessIntegerOrVector3 : Type($_self)">, "32-bit signless integer or vector of 32-bit signless integer values of length 2/4">; +def NVVM_T64BitSignlessInteger : Type, "64-bit signless integer">; +def NVVM_T64BitSignlessInteger2 : Type, "64-bit signless integer">, BuildableType<"$_builder.getIntegerType(64)">; +def NVVM_T64BitSignlessIntegerOr32 : Type, "64-bit signless integer or 32-bit signless integer">; +def NVVM_T8BitSignlessInteger : Type, "8-bit signless integer">, BuildableType<"$_builder.getIntegerType(8)">; +def NVVM_T8BitSignlessInteger2 : Type, "8-bit signless integer">; +def NVVM_T8BitSignlessIntegerOr32 : Type, "8-bit signless integer or 32-bit signless integer">; +def NVVM_VariadicOf16BitSignlessInteger : Type, "variadic of 16-bit signless integer">; +def NVVM_VariadicOf32BitSignlessInteger : Type, "variadic of 32-bit signless integer">; +def NVVM_VariadicOfLLVMDialectCompatibleType : Type($_self) || ::llvm::isa<::mlir::Float16Type>($_self) || ::llvm::isa<::mlir::BFloat16Type>($_self) || ::llvm::isa<::mlir::Float32Type>($_self) || ::llvm::isa<::mlir::Float64Type>($_self) || ::llvm::isa<::mlir::VectorType>($_self) || ::llvm::isa<::mlir::LLVM::LLVMPointerType>($_self) || ::llvm::isa<::mlir::LLVM::LLVMStructType>($_self) || ::llvm::isa<::mlir::LLVM::LLVMArrayType>($_self) || ::llvm::isa<::mlir::LLVM::LLVMFunctionType>($_self)">, "variadic of LLVM dialect-compatible type">; +def NVVM_VectorOf16BitFloatValues : Type($_self)">, "vector of 16-bit float values of length 2">; +def NVVM_VectorOf16BitSignlessInteger : Type($_self)">, "vector of 16-bit signless integer values of length 2">; +def NVVM_VectorOf32BitFloatValues : Type($_self)">, "vector of 32-bit float values of length 2">; +def NVVM_VectorOf32BitFloatValues2 : Type($_self)">, "vector of 32-bit float values">; +def NVVM_VectorOf32BitFloatValues3 : Type($_self)">, "vector of 32-bit float values of length 4">; +def NVVM_VectorOf32BitSignlessInteger : Type($_self)">, "vector of 32-bit signless integer values of length 2/4/8/16/32/64/128">; +def NVVM_VectorOf32BitSignlessInteger2 : Type($_self)">, "vector of 32-bit signless integer values of length 2/4/8/16/32/64/128 or 32-bit signless integer">; +def NVVM_VectorOf32BitSignlessInteger3 : Type($_self)">, "vector of 32-bit signless integer or 32-bit float values of length 2/4/8/16/32/64/128">; +def NVVM_VectorOf32BitSignlessInteger4 : Type($_self)">, "vector of 32-bit signless integer values of length 2/4/8/16/32/64">; +def NVVM_VectorOf8BitSignlessInteger : Type($_self)">, "vector of 8-bit signless integer values of length 4">; +def NVVM_VectorOf8BitSignlessInteger2 : Type($_self)">, "vector of 8-bit signless integer values of length 2">; +def NVVM_VectorOfAnyTypeValues : Type($_self)">, "vector of any type values">; +def NVVM_VectorOfBfloat16TypeValuesOf : Type($_self)">, "vector of bfloat16 type values of length 2">; + +// Attribute constraints the bindings name but do not describe. +def NVVM_ConvertBF16x2ToF4x2DstTyAttr : Attr($_self) && (::llvm::isa<::mlir::Float4E2M1FNType>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()))">, "type attribute of f4E2M1FN type"> { + let storageType = "::mlir::TypeAttr"; + let returnType = "::mlir::TypeAttr"; + let convertFromStorage = "$_self"; +} + +def NVVM_ConvertBF16x2ToF6x2DstTyAttr : Attr($_self) && (::llvm::isa<::mlir::Float6E2M3FNType>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()) || ::llvm::isa<::mlir::Float6E3M2FNType>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()))">, "type attribute of f6E2M3FN type or f6E3M2FN type"> { + let storageType = "::mlir::TypeAttr"; + let returnType = "::mlir::TypeAttr"; + let convertFromStorage = "$_self"; +} + +def NVVM_ConvertBF16x2ToF8x2DstTyAttr : Attr($_self) && (::llvm::isa<::mlir::Float8E8M0FNUType>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()) || ::llvm::isa<::mlir::Float8E4M3FNType>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()) || ::llvm::isa<::mlir::Float8E5M2Type>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()))">, "type attribute of f8E8M0FNU type or f8E4M3FN type or f8E5M2 type or fnv8E5M3FNU type"> { + let storageType = "::mlir::TypeAttr"; + let returnType = "::mlir::TypeAttr"; + let convertFromStorage = "$_self"; +} + +def NVVM_ConvertF16x2ToF4x2DstTyAttr : Attr($_self) && (::llvm::isa<::mlir::Float4E2M1FNType>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()))">, "type attribute of f4E2M1FN type"> { + let storageType = "::mlir::TypeAttr"; + let returnType = "::mlir::TypeAttr"; + let convertFromStorage = "$_self"; +} + +def NVVM_ConvertF16x2ToF6x2DstTyAttr : Attr($_self) && (::llvm::isa<::mlir::Float6E2M3FNType>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()) || ::llvm::isa<::mlir::Float6E3M2FNType>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()))">, "type attribute of f6E2M3FN type or f6E3M2FN type"> { + let storageType = "::mlir::TypeAttr"; + let returnType = "::mlir::TypeAttr"; + let convertFromStorage = "$_self"; +} + +def NVVM_FmaRndAttr : Attr($_self) && ::llvm::is_contained(::llvm::ArrayRef<::mlir::cutlass_compiler::nvvm::FPRoundingMode>{::mlir::cutlass_compiler::nvvm::FPRoundingMode::NONE, ::mlir::cutlass_compiler::nvvm::FPRoundingMode::RN, ::mlir::cutlass_compiler::nvvm::FPRoundingMode::RM, ::mlir::cutlass_compiler::nvvm::FPRoundingMode::RP, ::mlir::cutlass_compiler::nvvm::FPRoundingMode::RZ}, ::llvm::cast<::mlir::cutlass_compiler::nvvm::FPRoundingModeAttr>($_self).getValue())">, "NVVM FPRoundingMode kind whose value is one of {none, rm, rn, rp, rz}"> { + let storageType = "::mlir::cutlass_compiler::nvvm::FPRoundingModeAttr"; + let returnType = "::mlir::cutlass_compiler::nvvm::FPRoundingModeAttr"; + let convertFromStorage = "$_self"; +} + +def NVVM_MultimemLdReduceOrderingAttr : Attr($_self) && ::llvm::is_contained(::llvm::ArrayRef<::mlir::cutlass_compiler::nvvm::MemOrderKind>{::mlir::cutlass_compiler::nvvm::MemOrderKind::WEAK, ::mlir::cutlass_compiler::nvvm::MemOrderKind::RELAXED, ::mlir::cutlass_compiler::nvvm::MemOrderKind::ACQUIRE}, ::llvm::cast<::mlir::cutlass_compiler::nvvm::MemOrderKindAttr>($_self).getValue())">, "NVVM Memory Ordering kind whose value is one of {weak, relaxed, acquire}"> { + let storageType = "::mlir::cutlass_compiler::nvvm::MemOrderKindAttr"; + let returnType = "::mlir::cutlass_compiler::nvvm::MemOrderKindAttr"; + let convertFromStorage = "$_self"; +} + +def NVVM_MultimemLdReduceOpAttr : Attr($_self) && ::llvm::is_contained(::llvm::ArrayRef<::mlir::cutlass_compiler::nvvm::ReductionOp>{::mlir::cutlass_compiler::nvvm::ReductionOp::AND, ::mlir::cutlass_compiler::nvvm::ReductionOp::OR, ::mlir::cutlass_compiler::nvvm::ReductionOp::XOR, ::mlir::cutlass_compiler::nvvm::ReductionOp::ADD, ::mlir::cutlass_compiler::nvvm::ReductionOp::MIN, ::mlir::cutlass_compiler::nvvm::ReductionOp::MAX}, ::llvm::cast<::mlir::cutlass_compiler::nvvm::ReductionOpAttr>($_self).getValue())">, "Ops supported by red instruction whose value is none of {inc, dec}"> { + let storageType = "::mlir::cutlass_compiler::nvvm::ReductionOpAttr"; + let returnType = "::mlir::cutlass_compiler::nvvm::ReductionOpAttr"; + let convertFromStorage = "$_self"; +} + +def NVVM_MultimemRedOrderingAttr : Attr($_self) && ::llvm::is_contained(::llvm::ArrayRef<::mlir::cutlass_compiler::nvvm::MemOrderKind>{::mlir::cutlass_compiler::nvvm::MemOrderKind::RELAXED, ::mlir::cutlass_compiler::nvvm::MemOrderKind::RELEASE}, ::llvm::cast<::mlir::cutlass_compiler::nvvm::MemOrderKindAttr>($_self).getValue())">, "NVVM Memory Ordering kind whose value is one of {relaxed, release}"> { + let storageType = "::mlir::cutlass_compiler::nvvm::MemOrderKindAttr"; + let returnType = "::mlir::cutlass_compiler::nvvm::MemOrderKindAttr"; + let convertFromStorage = "$_self"; +} + +def NVVM_MultimemRedOpAttr : Attr($_self) && ::llvm::is_contained(::llvm::ArrayRef<::mlir::cutlass_compiler::nvvm::ReductionOp>{::mlir::cutlass_compiler::nvvm::ReductionOp::AND, ::mlir::cutlass_compiler::nvvm::ReductionOp::OR, ::mlir::cutlass_compiler::nvvm::ReductionOp::XOR, ::mlir::cutlass_compiler::nvvm::ReductionOp::ADD, ::mlir::cutlass_compiler::nvvm::ReductionOp::INC, ::mlir::cutlass_compiler::nvvm::ReductionOp::DEC, ::mlir::cutlass_compiler::nvvm::ReductionOp::MIN, ::mlir::cutlass_compiler::nvvm::ReductionOp::MAX}, ::llvm::cast<::mlir::cutlass_compiler::nvvm::ReductionOpAttr>($_self).getValue())">, "ReductionOp attribute"> { + let storageType = "::mlir::cutlass_compiler::nvvm::ReductionOpAttr"; + let returnType = "::mlir::cutlass_compiler::nvvm::ReductionOpAttr"; + let convertFromStorage = "$_self"; +} + +def NVVM_MultimemStOrderingAttr : Attr($_self) && ::llvm::is_contained(::llvm::ArrayRef<::mlir::cutlass_compiler::nvvm::MemOrderKind>{::mlir::cutlass_compiler::nvvm::MemOrderKind::WEAK, ::mlir::cutlass_compiler::nvvm::MemOrderKind::RELAXED, ::mlir::cutlass_compiler::nvvm::MemOrderKind::RELEASE}, ::llvm::cast<::mlir::cutlass_compiler::nvvm::MemOrderKindAttr>($_self).getValue())">, "NVVM Memory Ordering kind whose value is one of {weak, relaxed, release}"> { + let storageType = "::mlir::cutlass_compiler::nvvm::MemOrderKindAttr"; + let returnType = "::mlir::cutlass_compiler::nvvm::MemOrderKindAttr"; + let convertFromStorage = "$_self"; +} + +def NVVM_LLVMConstantRangeAttr : Attr($_self)">, "A range of two integers, corresponding to LLVM's ConstantRange"> { + let storageType = "::mlir::LLVM::ConstantRangeAttr"; + let returnType = "::mlir::LLVM::ConstantRangeAttr"; + let convertFromStorage = "$_self"; +} + +def NVVM_TensormapFieldValueAttr : Attr($_self)">, "NVVM Tensormap Elemtype or NVVM Tensormap Interleave Layout or NVVM Tensormap Swizzle Mode or NVVM Tensormap Swizzle Atomicity or NVVM Tensormap Fill Mode"> { + let storageType = "::mlir::Attribute"; + let returnType = "::mlir::Attribute"; + let convertFromStorage = "$_self"; +} + +// Operations whose result types are inferred in C++ (NVVMInference.cpp). +// Verification compares them with the inferred ones where the rule is +// recovered; elsewhere the types written stand, as the compiler accepted them. +class NVVM_InferredOp traits = []> + : NVVM_Op]> { + let extraClassDefinition = [{ + ::llvm::LogicalResult $cppClass::refineReturnTypes( + ::mlir::MLIRContext *context, ::std::optional<::mlir::Location> location, + ::mlir::ValueRange operands, ::mlir::DictionaryAttr attributes, + ::mlir::PropertyRef properties, ::mlir::RegionRange regions, + ::llvm::SmallVectorImpl<::mlir::Type> &returnTypes) { + ::llvm::SmallVector<::mlir::Type, 4> inferred; + if (::mlir::failed(inferReturnTypes(context, ::std::nullopt, operands, + attributes, properties, regions, + inferred))) + return ::mlir::success(); + if (!isCompatibleReturnTypes(inferred, returnTypes)) + return ::mlir::emitOptionalError( + location, "'", getOperationName(), "' op inferred type(s) ", + inferred, " are incompatible with return type(s) of operation ", + returnTypes); + return ::mlir::success(); + } + }]; +} + +def NVVM_AddPacked_BF16x2_F32x2_F32x2Op : NVVM_Op<"add.packed.bf16x2.f32x2.f32x2"> { + let description = [{ + This Op performs the add operation on a pair of f32x2 vectors. + The inputs and output are of the form v2f32 and v2bf16 respectively. + The inputs are converted to a scalar of packed i64 where the 0th element of the + vector forms the low-32 bits and the 1st element occupies the high-32 bits. The round to zero + is the only rounding mode supported for this operation. + + Examples: + ```mlir + // Round to zero (only rounding mode supported) + %0 = nvvm.add .packed.bf16x2.f32x2.f32x2 %srcA, %srcB : vector<2xf32>, vector<2xf32> -> vector<2xbf16> + ``` + }]; + let arguments = (ins + NVVM_VectorOf32BitFloatValues:$srcA, + NVVM_VectorOf32BitFloatValues:$srcB + ); + let results = (outs + NVVM_VectorOfBfloat16TypeValuesOf:$res + ); +} +def NVVM_AddPacked_F16x2_F32x2_F32x2Op : NVVM_Op<"add.packed.f16x2.f32x2.f32x2"> { + let description = [{ + This Op performs the add operation on a pair of f32x2 vectors. + The inputs and output are of the form v2f32 and v2f16 respectively. + The inputs are converted to a scalar of packed i64 where the 0th element of the + vector forms the low-32 bits and the 1st element occupies the high-32 bits. The round to zero + is the only rounding mode supported for this operation. + + Examples: + ```mlir + // Round to zero (only rounding mode supported) + %0 = nvvm.add .packed.f16x2.f32x2.f32x2 %srcA, %srcB : vector<2xf32>, vector<2xf32> -> vector<2xf16> + ``` + }]; + let arguments = (ins + NVVM_VectorOf32BitFloatValues:$srcA, + NVVM_VectorOf32BitFloatValues:$srcB + ); + let results = (outs + NVVM_VectorOf16BitFloatValues:$res + ); +} +def NVVM_AddPackedF32x2Op : NVVM_InferredOp<"add.packed.f32x2"> { + let description = [{ + This Op performs addition of floating-point vectors. + The inputs and outputs are of the form . The `rnd` attribute + specifies the rounding mode. The `ftz` attribute, when set, flushes the + subnormal inputs and results to sign-preserving zero. + This Op is supported from SM100 and onwards. + [For more information, see PTX ISA] + (TODO: Add public PTX link when available) + }]; + let arguments = (ins + NVVM_VectorOf32BitFloatValues2:$srcA, + NVVM_VectorOf32BitFloatValues2:$srcB, + OptionalAttr:$rnd, + OptionalAttr:$ftz + ); + let results = (outs + NVVM_VectorOf32BitFloatValues2:$res + ); +} +def NVVM_AddPacked_F32x2_BF16x2_F32x2Op : NVVM_Op<"add.packed.f32x2.bf16x2.f32x2"> { + let description = [{ + This Op performs the add operation on a pair of floating-point values. + The inputs are of the form v2bf16 and v2f32. The first input `srcA` is + converted to a scalar of packed i32 where the 0th element of the vector forms + the low-16 bits and the 1st element occupies the high-16 bits. Similarly, + the second input `srcB` is converted to a scalar of packed i64 where the 0th + element of the vector forms the low-32 bits and the 1st element occupies the + high-32 bits. The `rnd` attribute specifies the rounding mode. + + The rounding mode can be one of the following: + - rn: round to nearest even + - rz: round toward zero + - rm: round toward negative infinity + - rp: round toward positive infinity + + Examples: + ```mlir + // Round to nearest even (default) + %0 = nvvm.add .packed.f32x2.bf16x2.f32x2 %srcA, %srcB : vector<2xbf16>, vector<2xf32> -> vector<2xf32> + + // Round toward negative infinity + %1 = nvvm.add .packed.f32x2.bf16x2.f32x2 %srcA, %srcB {rnd = #nvvm.fp_rnd_mode} : vector<2xbf16>, vector<2xf32> -> vector<2xf32> + + // Round toward zero with flush-to-zero + %2 = nvvm.add .packed.f32x2.bf16x2.f32x2 %srcA, %srcB {rnd = #nvvm.fp_rnd_mode} : vector<2xbf16>, vector<2xf32> -> vector<2xf32> + ``` + }]; + let arguments = (ins + NVVM_VectorOfBfloat16TypeValuesOf:$srcA, + NVVM_VectorOf32BitFloatValues:$srcB, + OptionalAttr:$rnd + ); + let results = (outs + NVVM_VectorOf32BitFloatValues:$res + ); +} +def NVVM_AddPacked_F32x2_F16x2_F32x2Op : NVVM_Op<"add.packed.f32x2.f16x2.f32x2"> { + let description = [{ + This Op performs the add operation on a pair of floating-point values. + The inputs are of the form v2f16 and v2f32. The first input `srcA` is + converted to a scalar of packed i32 where the 0th element of the vector forms + the low-16 bits and the 1st element occupies the high-16 bits. Similarly, + the second input `srcB` is converted to a scalar of packed i64 where the 0th + element of the vector forms the low-32 bits and the 1st element occupies the + high-32 bits. The `rnd` attribute specifies the rounding mode. + + The rounding mode can be one of the following: + - rn: round to nearest even + - rz: round toward zero + - rm: round toward negative infinity + - rp: round toward positive infinity + + Examples: + ```mlir + // Round to nearest even (default) + %0 = nvvm.add .packed.f32x2.f16x2.f32x2 %srcA, %srcB : vector<2xf16>, vector<2xf32> -> vector<2xf32> + + // Round toward negative infinity + %1 = nvvm.add .packed.f32x2.f16x2.f32x2 %srcA, %srcB {rnd = #nvvm.fp_rnd_mode} : vector<2xf16>, vector<2xf32> -> vector<2xf32> + + // Round toward zero with flush-to-zero + %2 = nvvm.add .packed.f32x2.f16x2.f32x2 %srcA, %srcB {rnd = #nvvm.fp_rnd_mode} : vector<2xf16>, vector<2xf32> -> vector<2xf32> + ``` + }]; + let arguments = (ins + NVVM_VectorOf16BitFloatValues:$srcA, + NVVM_VectorOf32BitFloatValues:$srcB, + OptionalAttr:$rnd + ); + let results = (outs + NVVM_VectorOf32BitFloatValues:$res + ); +} +def NVVM_AddFOp : NVVM_InferredOp<"addf", [Pure]> { + let description = [{ + The `nvvm.addf` operation performs floating point addition of two floating + point operands of the same type. + + The rounding mode is specified by the `rnd` attribute, saturation mode by + the `sat` attribute, and flush-to-zero by the `ftz` attribute. + + For more information, see PTX ISA: + - [floating point addition](https://docs.nvidia.com/cuda/parallel-thread-execution/#floating-point-instructions-add) + - [half-precision floating point addition](https://docs.nvidia.com/cuda/parallel-thread-execution/#half-precision-floating-point-instructions-add) + }]; + let arguments = (ins + NVVM_T16BitFloatOrBfloat16Type:$lhs, + NVVM_T16BitFloatOrBfloat16Type:$rhs, + OptionalAttr:$rnd, + OptionalAttr:$sat, + OptionalAttr:$ftz + ); + let results = (outs + NVVM_T16BitFloatOrBfloat16Type:$res + ); +} +def NVVM_ApplyPriorityAsyncBulkOp : NVVM_Op<"applypriority.async.bulk"> { + let description = [{ + Sets the eviction priority to evict normal for cache lines within the specified + region of L2 cache that are previously present and marked as evict last. + Other L2 cache lines in the specified region are not affected. + + Arguments: + - `ptr`: Global memory pointer. + - `size`: Address range of length. + - `evict`: Eviction priority. + + Example: + ```mlir + nvvm.applypriority.async.bulk %ptr, %size evict_priority = evict_normal + ``` + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace1:$ptr, + NVVM_T32BitSignlessInteger:$size, + OptionalAttr:$evict + ); + let results = (outs); +} +def NVVM_ApplyPriorityAsyncBulkTensorOp : NVVM_Op<"applypriority.async.bulk.tensor", [AttrSizedOperandSegments]> { + let description = [{ + Sets the eviction priority to evict normal for cache lines within the + specified region of L2 cache that are previously present and marked as + evict last. Other L2 cache lines in the specified region are not affected. + + Arguments: + - `tmaDesc`: The TMA descriptor. + - `box/coordinates`: The coordinates of the tensor. + - `im2col`: The im2col arguments. + - `mode`: Tensor load mode. + - `evict`: Eviction priority. + + Example: + ```mlir + nvvm.applypriority.async.bulk.tensor %tma_desc, box [ %d0] evict_priority = evict_normal {mode = #nvvm.tma_load_mode} : !llvm.ptr + nvvm.applypriority.async.bulk.tensor %tma_desc, box [ %d0, %d1, %d2] im2col [ %im2col0] evict_priority = evict_normal {mode = #nvvm.tma_load_mode} : !llvm.ptr + nvvm.applypriority.async.bulk.tensor %tma_desc, box [ %d0, %d1, %d2] im2col [ %im2w_halo, %im2w_offset] evict_priority = evict_normal {mode = #nvvm.tma_load_mode} : !llvm.ptr + ``` + }]; + let arguments = (ins + NVVM_LLVMPointerType:$tmaDesc, + Variadic:$coordinates, + Variadic:$im2colArgs, + OptionalAttr:$mode, + OptionalAttr:$evict + ); + let results = (outs); +} +def NVVM_ApplyPriorityAsyncBulkTensorOverrideOp : NVVM_Op<"applypriority.async.bulk.tensor.override", [AttrSizedOperandSegments]> { + let description = [{ + Sets the eviction priority to evict normal for cache lines within the + specified region of L2 cache that are previously present and marked as + evict last. Other L2 cache lines in the specified region are not affected. + It supports a way to override the global address of the tensor. + + Arguments: + - `tmaDesc`: The TMA descriptor. + - `overrideAdrr`: The global address to override. + - `box/coordinates`: The coordinates of the tensor. + - `tensorSize`: The size of the tensor. + - `lowerStride`: The lower stride of the tensor. + - `upperStride`: The upper stride of the tensor. + - `im2col`: The im2col arguments. + - `mode`: Tensor load mode. + - `evict`: Eviction priority. + + Example: + ```mlir + nvvm.applypriority.async.bulk.tensor.override %tma_desc, %gbl_addr, box [ %d0] evict_priority = evict_normal {mode = #nvvm.tma_load_mode} : !llvm.ptr + nvvm.applypriority.async.bulk.tensor.override %tma_desc, %gbl_addr, box [ %d0, %d1, %d2] im2col [ %im2col0] evict_priority = evict_normal {mode = #nvvm.tma_load_mode} : !llvm.ptr + nvvm.applypriority.async.bulk.tensor.override %tma_desc, %gbl_addr, box [ %d0] tensor_size [ %ts0] evict_priority = evict_normal {mode = #nvvm.tma_load_mode} : !llvm.ptr + nvvm.applypriority.async.bulk.tensor.override %tma_desc, %gbl_addr, box [ %d0, %d1] tensor_size [ %ts0, %ts1] lower_stride [ %lstrd0] upper_stride [ %ustrd] evict_priority = evict_normal {mode = #nvvm.tma_load_mode} : !llvm.ptr + ``` + }]; + let arguments = (ins + NVVM_LLVMPointerType:$tmaDesc, + NVVM_LLVMPointerInAddressSpace1:$overrideAdrr, + Variadic:$coordinates, + Variadic:$tensorSize, + Variadic:$lowerStride, + Optional:$upperStride, + Variadic:$im2colArgs, + OptionalAttr:$mode, + OptionalAttr:$evict + ); + let results = (outs); +} +def NVVM_AtomicRMWOp : NVVM_InferredOp<"atomicrmw"> { + let description = [{ + Atomically loads the original value at location `a` into destination register `res`, + performs a reduction operation with operand `b` and the value in location `a`, and + stores the result of the specified operation at location `a`, overwriting the + original value. Operand `a` specifies a location in the specified state space. + If no state space is given, perform the memory accesses using `Generic Addressing`. + For more information, refer [PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-atom) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace12:$ptr, + NVVM_T16BitSignlessIntegerOr32:$a, + Optional:$b, + NVVM_AtomicOpKindAttr:$op, + OptionalAttr:$memOrder, + OptionalAttr:$syncscope + ); + let results = (outs + NVVM_T16BitSignlessIntegerOr32:$res + ); +} +def NVVM_SyncWarpOp : NVVM_Op<"bar.warp.sync"> { + let description = [{ + The `nvvm.bar.warp.sync` operation performs barrier synchronization for threads + within a warp. + + This operation causes the executing thread to wait until all threads corresponding + to the `mask` operand have executed a `bar.warp.sync` with the same mask value + before resuming execution. + + The `mask` operand specifies the threads participating in the barrier, where each + bit position corresponds to the thread's lane ID within the warp. Only threads with + their corresponding bit set in the mask participate in the barrier synchronization. + + **Important constraints**: + - The behavior is undefined if the executing thread is not included in the mask + (i.e., the bit corresponding to the thread's lane ID is not set) + - For compute capability sm_6x or below, all threads in the mask must execute + the same `bar.warp.sync` instruction in convergence + + This operation also guarantees memory ordering among participating threads. + Threads within the warp that wish to communicate via memory can store to memory, + execute `bar.warp.sync`, and then safely read values stored by other threads + in the warp. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-bar-warp-sync) + }]; + let arguments = (ins + NVVM_T32BitSignlessInteger:$mask + ); + let results = (outs); +} +def NVVM_BarrierOp : NVVM_InferredOp<"barrier", [AttrSizedOperandSegments]> { + let description = [{ + The `nvvm.barrier` operation performs barrier synchronization and communication + within a CTA (Cooperative Thread Array). It causes executing threads to wait for + all non-exited threads participating in the barrier to arrive. + + The operation takes two optional operands: + + - `barrierId`: Specifies a logical barrier resource with value 0 through 15. + Each CTA instance has sixteen barriers numbered 0..15. Defaults to 0 if not specified. + - `numberOfThreads`: Specifies the number of threads participating in the barrier. + When specified, the value must be a multiple of the warp size. If not specified, + all threads in the CTA participate in the barrier. + - `reductionOp`: specifies the reduction operation (`popc`, `and`, `or`). + - `reductionPredicate`: specifies the predicate to be used with the + `reductionOp`. + + The barrier operation guarantees that when the barrier completes, prior memory + accesses requested by participating threads are performed relative to all threads + participating in the barrier. It also ensures that no new memory access is + requested by participating threads before the barrier completes. + + When a barrier completes, the waiting threads are restarted without delay, and + the barrier is reinitialized so that it can be immediately reused. + + This operation generates an aligned barrier, indicating that all threads in the CTA + will execute the same barrier instruction. Behavior is undefined if all threads in the + CTA do not reach this instruction. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-bar) + }]; + let arguments = (ins + Optional:$barrierId, + Optional:$numberOfThreads, + Optional:$reductionPredicate, + OptionalAttr:$reductionOp + ); + let results = (outs + Optional:$res + ); +} +def NVVM_BarrierArriveOp : NVVM_Op<"barrier.arrive"> { + let description = [{ + Thread that executes this op announces their arrival at the barrier with + given id and continue their execution. + + The default barrier id is 0 that is similar to `nvvm.barrier` Op. When + `barrierId` is not present, the default barrier id is used. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-bar) + }]; + let arguments = (ins + Optional:$barrierId, + NVVM_T32BitSignlessInteger:$numberOfThreads + ); + let results = (outs); +} +def NVVM_BarrierCTAArriveOp : NVVM_Op<"barrier.cta.arrive"> { + let description = [{ + The `nvvm.barrier.cta.arrive` is used by the threads within the CTA for synchronization and communication. + The `barrierId` specifies a logical barrier resource and the `threadCount` specifies the number of threads + participating in the barrier. The `barrier.cta.arrive` does not cause the executing thread to wait for + threads of other participating warps. The `aligned` attribute, when set to 1, generates the .aligned + version of the PTX instruction. + [For more information, see PTX ISA] + (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-bar-barrier) + }]; + let arguments = (ins + NVVM_T32BitSignlessInteger:$barrierId, + NVVM_T32BitSignlessInteger:$threadCount, + OptionalAttr:$aligned + ); + let results = (outs); +} +def NVVM_BarrierCTAReduxOp : NVVM_Op<"barrier.cta.red"> { + let description = [{ + The `nvvm.barrier.cta.red` is used to perform a reduction operation across threads in the CTA. + The `pred` from all the participating threads are combined using the specified reduction operator. + AND, OR, POPC (population-count) are available as reduction operators. The `barrierId` specifies + a logical barrier resource and the `threadCount` specifies the number of threads participating + in the barrier. When the `threadCount` is not specified, all threads in the CTA participate in + the reduction operation. The `aligned` attribute, when set to 1, generates the .aligned version + of the PTX instruction. + [For more information, see PTX ISA] + (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-bar-barrier) + }]; + let arguments = (ins + NVVM_T1BitSignlessInteger:$pred, + NVVM_T32BitSignlessInteger:$barrierId, + Optional:$threadCount, + NVVM_BarrierReduxKindAttr:$kind, + OptionalAttr:$aligned + ); + let results = (outs + NVVM_T32BitSignlessInteger2:$res + ); +} +def NVVM_BarrierCTASyncOp : NVVM_Op<"barrier.cta.sync"> { + let description = [{ + The `nvvm.barrier.cta.sync` is used by the threads within the CTA for synchronization and communication. + The `barrierId` specifies a logical barrier resource and the `threadCount` specifies the number of + threads participating in the barrier. When the `threadCount` is not specified, all threads in the CTA + participate in the barrier. The `aligned` attribute, when set to 1, generates the .aligned version of + the PTX instruction. + [For more information, see PTX ISA] + (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-bar-barrier) + }]; + let arguments = (ins + NVVM_T32BitSignlessInteger:$barrierId, + Optional:$threadCount, + OptionalAttr:$aligned + ); + let results = (outs); +} +def NVVM_Barrier0Op : NVVM_Op<"barrier0"> { + let description = [{ + The `nvvm.barrier0` operation is a convenience operation that performs barrier + synchronization and communication within a CTA (Cooperative Thread Array) using + barrier ID 0. It is functionally equivalent to `nvvm.barrier` or `nvvm.barrier id=0`. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-bar) + }]; + let arguments = (ins); + let results = (outs); +} +def NVVM_Breakpoint : NVVM_Op<"breakpoint"> { + let description = [{ + Breakpoint suspends execution of the program for debugging. + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#miscellaneous-instructions-brkpt) + }]; + let arguments = (ins); + let results = (outs); +} +def NVVM_ClmadOp : NVVM_Op<"clmad", [Pure]> { + let description = [{ + Performs a carryless multiplication of `a` and `b`, followed by a carryless + addition of `c`. + When `hi` is true, the result is the upper 64 bits of the product, with the + addition of `c`. Otherwise, the result is the lower 64 bits of the product, + with the addition of `c`. + }]; + let arguments = (ins + NVVM_T64BitSignlessInteger:$a, + NVVM_T64BitSignlessInteger:$b, + NVVM_T64BitSignlessInteger:$c, + OptionalAttr:$high + ); + let results = (outs + NVVM_T64BitSignlessInteger2:$res + ); +} +def NVVM_ClusterArriveOp : NVVM_Op<"cluster.arrive"> { + let description = [{ + The `cluster.arrive` can be used by the threads within the cluster for synchronization and + communication. The `cluster.arrive` instruction marks the warps' arrival at the barrier + without causing the executing thread to wait for other participating threads. + + The `aligned` attribute, when provided, generates the .aligned version of the PTX instruction. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-barrier-cluster) + }]; + let arguments = (ins + UnitAttr:$aligned + ); + let results = (outs); +} +def NVVM_ClusterArriveRelaxedOp : NVVM_Op<"cluster.arrive.relaxed"> { + let description = [{ + The `cluster.arrive` can be used by the threads within the cluster for synchronization and + communication. The `cluster.arrive` instruction marks the warps' arrival at the barrier + without causing the executing thread to wait for other participating threads. + + The `aligned` attribute, when provided, generates the .aligned version of the PTX instruction. + The .relaxed qualifier on `cluster.arrive` specifies that there are no memory + ordering and visibility guarantees provided for the memory accesses performed prior to + `cluster.arrive`. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-barrier-cluster) + }]; + let arguments = (ins + UnitAttr:$aligned + ); + let results = (outs); +} +def NVVM_ClusterWaitOp : NVVM_Op<"cluster.wait"> { + let description = [{ + The `cluster.wait` causes the executing thread to wait for all non-exited threads + of the cluster to perform `cluster.arrive`. The `aligned` attribute, when provided, + generates the .aligned version of the PTX instruction. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-barrier-cluster) + }]; + let arguments = (ins + UnitAttr:$aligned + ); + let results = (outs); +} +def NVVM_ClusterLaunchControlQueryCancelOp : NVVM_InferredOp<"clusterlaunchcontrol.query.cancel"> { + let description = [{ + `clusterlaunchcontrol.query.cancel` queries the response of a + `clusterlaunchcontrol.try.cancel` operation specified by operand + `try_cancel_response`. + + Operand `query_type` specifies the type of query to perform and can be one + of the following: + - `is_canceled` : Returns true if the try cancel request succeeded, + and false otherwise. + - `get_first_cta_id_{x/y/z}` : Returns the x, y, or z coordinate of the + first CTA in the canceled cluster. Behaviour is defined only if the try + cancel request succeeded. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-query-cancel) + }]; + let arguments = (ins + NVVM_T128BitSignlessInteger:$try_cancel_response, + NVVM_ClusterLaunchControlQueryTypeAttr:$query_type + ); + let results = (outs + NVVM_T1BitSignlessIntegerOr32:$res + ); +} +def NVVM_ClusterLaunchControlTryCancelOp : NVVM_Op<"clusterlaunchcontrol.try.cancel"> { + let description = [{ + `clusterlaunchcontrol.try.cancel` requests atomically canceling the launch + of a cluster that has not started running yet. It asynchronously writes an + opaque response to shared memory indicating whether the operation succeeded + or failed. + + Operand `smemAddress` specifies the naturally aligned address of the + 16-byte wide shared memory location where the request's response is written. + + Operand `mbarrier` specifies the mbarrier object used to track the + completion of the asynchronous operation. + + If `multicast` is specified, the response is asynchronously written to the + corresponding local shared memory location (specifed by `addr`) of each CTA + in the requesting cluster. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace3:$smemAddress, + NVVM_LLVMPointerInAddressSpace3:$mbarrier, + UnitAttr:$multicast + ); + let results = (outs); +} +def NVVM_CompareAndSetOp : NVVM_InferredOp<"compare.and.set"> { + let description = [{ + Compare two values and return boolean mask of 1's or 0's. Return value + will be of the same type as the operands (vector<4xi8> or vector<2xi16>). + + Arguments: + - `a`: The first value. + - `b`: The second value. + - `op`: The comparison operator. (eq, ne, ugt, uge, ult, ule, sgt, sge, slt, sle) + + Example: + ```mlir + %res = nvvm.compare.and.set %a, %b : vector<4xi8> + %res = nvvm.compare.and.set %a, %b : vector<2xi16> + ``` + }]; + let arguments = (ins + NVVM_FixedLengthVectorOf8Bit:$a, + NVVM_FixedLengthVectorOf8Bit:$b, + NVVM_CompareOpKindAttr:$op + ); + let results = (outs + NVVM_FixedLengthVectorOf8Bit:$result + ); +} +def NVVM_ConvertPackIntegerOp : NVVM_Op<"convert.and.pack.integer"> { + let description = [{ + This Op converts the given integer inputs to the specified integer type + with saturation and packs the result into the lower bits of the result. + + The result `dst` is represented as an i32 type. + + The optional operand `srcC` must be specified when the converted types add + up to less than 32 bits. The remaining bits of the result are filled with + the lower bits of `srcC`. + + The `convert_type` attribute specifies the integer type to convert to + before packing. The `signed` attribute can be specified to indicate that + the `convert_type` is signed. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-cvt-pack) + }]; + let arguments = (ins + NVVM_T32BitSignlessInteger:$srcA, + NVVM_T32BitSignlessInteger:$srcB, + Optional:$srcC, + UnitAttr:$is_signed, + TypeAttr:$convert_type + ); + let results = (outs + NVVM_T32BitSignlessInteger2:$dst + ); +} +def NVVM_ConvertBF16x2ToF4x2Op : NVVM_Op<"convert.bf16x2.to.f4x2"> { + let description = [{ + This Op converts each of the given BF16 inputs in an bf16x2 vector to the specified fp4 type. + The result `dst` is returned as an i8 type where the converted values are + packed such that the value converted from the first element of `a` is + stored in the lower 4 bits of `dst` and the value converted from the second + element of `a` is stored in the upper 4 bits of `dst`. + The `relu` attribute, when set, lowers to the '.relu' variant of + the cvt instruction. + The `pzo` (positive zero only), when provided, forces negative zero to be positive zero. + }]; + let arguments = (ins + NVVM_VectorOfBfloat16TypeValuesOf:$src, + Optional:$scale_factor, + UnitAttr:$isPZO, + OptionalAttr:$scale_factor_kind, + OptionalAttr:$rnd, + OptionalAttr:$relu, + NVVM_ConvertBF16x2ToF4x2DstTyAttr:$dstTy + ); + let results = (outs + NVVM_T8BitSignlessInteger:$dst + ); +} +def NVVM_ConvertBF16x2ToF6x2Op : NVVM_Op<"convert.bf16x2.to.f6x2"> { + let description = [{ + This Op converts each of the given BF16 inputs in an bf16x2 vector to the specified fp6 type. The result `dst` is + represented either as an i16 type or as a vector of two i8 types. + If `dst` is returned as an i16 type, the converted values are packed such + that the value converted from the first element of `a` is stored in the + lower 8 bits of `dst` with 2 MSB bits padded with zeros and the value + converted from the second element of `a` is stored in the upper 8 bits of + `dst` with 2 MSB bits padded with zeros. + If `dst` is returned as a vector type, each converted value is stored as an + i8 element in the vector with 2 MSB bits padded with zeros. + The `relu` attribute, when set, lowers to the '.relu' variant of + the cvt instruction. The `pzo` (positive zero only), when provided, forces negative zero to be positive zero. + }]; + let arguments = (ins + NVVM_VectorOfBfloat16TypeValuesOf:$src, + Optional:$scale_factor, + UnitAttr:$isPZO, + OptionalAttr:$scale_factor_kind, + OptionalAttr:$rnd, + OptionalAttr:$relu, + NVVM_ConvertBF16x2ToF6x2DstTyAttr:$dstTy + ); + let results = (outs + NVVM_T16BitSignlessIntegerOrVector:$dst + ); +} +def NVVM_ConvertBF16x2ToF8x2Op : NVVM_Op<"convert.bf16x2.to.f8x2"> { + let description = [{ + This Op converts the given bf16 inputs in a bf16x2 vector to the specified + f8 type. The result `dst` is represented either as a packed i16 type or as + a vector of two i8 types. + If `dst` is returned as an i16 type, the converted values are packed such + that the value converted from the first element of `a` is stored in the + lower 8 bits of `dst` and the value converted from the second element of + `a` is stored in the upper 8 bits of `dst`. + If `dst` is returned as a vector type, each converted value is stored as an + i8 element in the vector. + The `rnd` and `sat` attributes specify the rounding and saturation modes + respectively. + + The `pzo` (positive zero only), when provided, forces negative zero to be positive zero. + + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + NVVM_VectorOfBfloat16TypeValuesOf:$src, + Optional:$scale_factor, + UnitAttr:$isPZO, + OptionalAttr:$scale_factor_kind, + OptionalAttr:$rnd, + OptionalAttr:$sat, + OptionalAttr:$relu, + NVVM_ConvertBF16x2ToF8x2DstTyAttr:$dstTy + ); + let results = (outs + NVVM_T16BitSignlessIntegerOrVector:$dst + ); +} +def NVVM_ConvertBF16x2ToS2F6x2Op : NVVM_Op<"convert.bf16x2.to.s2f6x2"> { + let description = [{ + This Op converts each of the given BF16 inputs in a bf16x2 vector to the + S2F6x2 type. The result `dst` can be either a packed i16 type or a vector + of two i8 types. + If `dst` is returned as an i16 type, the converted values are packed such + that the value converted from the first element of `a` is stored in the + lower 8 bits of `dst` and the value converted from the second element of + `a` is stored in the upper 8 bits of `dst`. + If `dst` is returned as a vector type, each converted value is stored as an + i8 element in the vector. + The `relu` attribute, when set, lowers to the '.relu' variant + of the cvt instruction. + The optional scaling-factors for each of the inputs are provided through + the operand `scaleFactor` as a packed i16 type. Only `ue8m0` is supported + as the type of the scale-factor currently. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + NVVM_VectorOfBfloat16TypeValuesOf:$src, + Optional:$scaleFactor, + OptionalAttr:$relu + ); + let results = (outs + NVVM_T16BitSignlessIntegerOrVector:$dst + ); +} +def NVVM_ConvertF16x2ToF4x2Op : NVVM_Op<"convert.f16x2.to.f4x2"> { + let description = [{ + This Op converts each of the given F16 inputs in an f16x2 vector to the specified fp4 type. + The result `dst` is returned as an i8 type where the converted values are + packed such that the value converted from the first element of `a` is + stored in the lower 4 bits of `dst` and the value converted from the second + element of `a` is stored in the upper 4 bits of `dst`. + The `relu` attribute, when set, lowers to the '.relu' variant of + the cvt instruction. + The `pzo` (positive zero only), when provided, forces negative zero to be positive zero. + }]; + let arguments = (ins + NVVM_VectorOf16BitFloatValues:$src, + Optional:$scale_factor, + UnitAttr:$isPZO, + OptionalAttr:$scale_factor_kind, + OptionalAttr:$rnd, + OptionalAttr:$relu, + NVVM_ConvertF16x2ToF4x2DstTyAttr:$dstTy + ); + let results = (outs + NVVM_T8BitSignlessInteger:$dst + ); +} +def NVVM_ConvertF16x2ToF6x2Op : NVVM_Op<"convert.f16x2.to.f6x2"> { + let description = [{ + This Op converts each of the given F16 inputs in an f16x2 vector to the specified fp6 type. The result `dst` is + represented either as an i16 type or as a vector of two i8 types. + If `dst` is returned as an i16 type, the converted values are packed such + that the value converted from the first element of `a` is stored in the + lower 8 bits of `dst` with 2 MSB bits padded with zeros and the value + converted from the second element of `a` is stored in the upper 8 bits of + `dst` with 2 MSB bits padded with zeros. + If `dst` is returned as a vector type, each converted value is stored as an + i8 element in the vector with 2 MSB bits padded with zeros. + The `relu` attribute, when set, lowers to the '.relu' variant of + the cvt instruction. The `pzo` (positive zero only), when provided, forces negative zero to be positive zero. + }]; + let arguments = (ins + NVVM_VectorOf16BitFloatValues:$src, + Optional:$scale_factor, + UnitAttr:$isPZO, + OptionalAttr:$scale_factor_kind, + OptionalAttr:$rnd, + OptionalAttr:$relu, + NVVM_ConvertF16x2ToF6x2DstTyAttr:$dstTy + ); + let results = (outs + NVVM_T16BitSignlessIntegerOrVector:$dst + ); +} +def NVVM_ConvertF16x2ToF8x2Op : NVVM_Op<"convert.f16x2.to.f8x2"> { + let description = [{ + This Op converts the given f16 inputs in an f16x2 vector to the specified + f8 type. + The result `dst` is represented as an i16 type or as a vector + of two i8 types. + If `dst` is returned as an i16 type, the converted values from `a` + are packed such that the value converted from the first element of `a` + is stored in the upper 8 bits of `dst` and the value converted from the + second element of `a` is stored in the lower 8 bits of `dst`. + If `dst` is returned as a vector type, each converted value is stored as an + i8 element in the vector. + The `relu` attribute, when set, lowers to the '.relu' variant of + the cvt instruction. + The `rnd` and `sat` attributes specify the rounding and saturation modes + respectively. + The `pzo` (positive zero only), when provided, forces negative zero to be positive zero. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + NVVM_VectorOf16BitFloatValues:$a, + Optional:$scale_factor, + UnitAttr:$isPZO, + OptionalAttr:$scale_factor_kind, + OptionalAttr:$rnd, + OptionalAttr:$sat, + OptionalAttr:$relu, + TypeAttr:$dstTy + ); + let results = (outs + NVVM_T16BitSignlessIntegerOrVector:$dst + ); +} +def NVVM_ConvertF32x2ToBF16x2Op : NVVM_Op<"convert.f32x2.to.bf16x2"> { + let description = [{ + Converts two F32 values to packed bf16x2 format with + the specified rounding mode. The `src_hi` and `src_lo` parameters + correspond to operands `a` and `b` in the PTX ISA, respectively. + + The `random_bits` parameter is required for stochastic rounding and + provides the [random bits](https://docs.nvidia.com/cuda/parallel-thread-execution/#cvt-rs-rbits-layout-bf16) to be used for the conversion. + + The `relu` attribute clamps negative results to 0. + + The `sat` attribute determines saturation behavior. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + NVVM_T32BitFloat:$src_hi, + NVVM_T32BitFloat:$src_lo, + Optional:$random_bits, + OptionalAttr:$rnd, + OptionalAttr:$sat, + OptionalAttr:$relu + ); + let results = (outs + NVVM_VectorOfBfloat16TypeValuesOf:$dst + ); +} +def NVVM_ConvertF32x2ToF16x2Op : NVVM_Op<"convert.f32x2.to.f16x2"> { + let description = [{ + Converts two F32 values to packed f16x2 format with + the specified rounding mode. The `src_hi` and `src_lo` parameters + correspond to operands `a` and `b` in the PTX ISA, respectively. + + The `random_bits` parameter is required for stochastic rounding and + provides the [random bits](https://docs.nvidia.com/cuda/parallel-thread-execution/#cvt-rs-rbits-layout-f16) to be used for the conversion. + + The `relu` attribute clamps negative results to 0. + + The `sat` attribute determines saturation behavior. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + NVVM_T32BitFloat:$src_hi, + NVVM_T32BitFloat:$src_lo, + Optional:$random_bits, + OptionalAttr:$rnd, + OptionalAttr:$sat, + OptionalAttr:$relu + ); + let results = (outs + NVVM_VectorOf16BitFloatValues:$dst + ); +} +def NVVM_ConvertF32x2ToF4x2Op : NVVM_Op<"convert.f32x2.to.f4x2"> { + let description = [{ + This Op converts each of the given float inputs to the specified fp4 type. + + The result `dst` is returned as an i8 type where the converted values are + packed such that the value converted from `a` is stored in the upper 4 bits + of `dst` and the value converted from `b` is stored in the lower 4 bits of + `dst`. + + The `relu` attribute, when set, lowers to the '.relu' variant of + the cvt instruction. + + The `pzo` (positive zero only), when provided, forces negative zero to be positive zero. + + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + NVVM_T32BitFloat:$a, + NVVM_T32BitFloat:$b, + Optional:$scale_factor, + UnitAttr:$isPZO, + OptionalAttr:$scale_factor_kind, + OptionalAttr:$rnd, + OptionalAttr:$relu, + TypeAttr:$dstType + ); + let results = (outs + NVVM_T8BitSignlessInteger:$dst + ); +} +def NVVM_ConvertF32x2ToF6x2Op : NVVM_Op<"convert.f32x2.to.f6x2"> { + let description = [{ + This Op converts each of the given float inputs to the specified fp6 type. + The result `dst` is represented either as an i16 type or as a vector + of two i8 types. + If `dst` is returned as an i16 type, the converted values are packed such + that the value converted from `a` is stored in the upper 8 bits of `dst` + with 2 MSB bits padded with zeros and the value converted from `b` is + stored in the lower 8 bits of `dst` with 2 MSB bits padded with zeros. + If `dst` is returned as a vector type, each converted value is stored as an + i8 element in the vector. + The `relu` attribute, when set, lowers to the '.relu' variant of + the cvt instruction. + + The `pzo` (positive zero only), when provided, forces negative zero to be positive zero. + + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + NVVM_T32BitFloat:$a, + NVVM_T32BitFloat:$b, + Optional:$scale_factor, + UnitAttr:$isPZO, + OptionalAttr:$scale_factor_kind, + OptionalAttr:$rnd, + OptionalAttr:$relu, + TypeAttr:$dstTy + ); + let results = (outs + NVVM_T16BitSignlessIntegerOrVector:$dst + ); +} +def NVVM_ConvertF32x2ToF8x2Op : NVVM_Op<"convert.f32x2.to.f8x2"> { + let description = [{ + This Op converts each of the given float inputs to the specified fp8 type. + The result `dst` is represented as an i16 type or as a vector + of two i8 types. + If `dst` is returned as an i16 type, the converted values are packed such + that the value converted from `a` is stored in the upper 8 bits of `dst` + and the value converted from `b` is stored in the lower 8 bits of `dst`. + If `dst` is returned as a vector type, each converted value is stored as an + i8 element in the vector. + The `rnd` and `sat` attributes specify the rounding and saturation modes respectively. + The `relu` attribute, when set, lowers to the '.relu' variant of + the cvt instruction. + + The `pzo` (positive zero only), when provided, forces negative zero to be positive zero. + + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + NVVM_T32BitFloat:$a, + NVVM_T32BitFloat:$b, + Optional:$scale_factor, + UnitAttr:$isPZO, + OptionalAttr:$scale_factor_kind, + OptionalAttr:$rnd, + OptionalAttr:$sat, + OptionalAttr:$relu, + TypeAttr:$dstTy + ); + let results = (outs + NVVM_T16BitSignlessIntegerOrVector:$dst + ); +} +def NVVM_ConvertF32x2ToS2F6x2Op : NVVM_Op<"convert.f32x2.to.s2f6x2"> { + let description = [{ + This Op converts each of the given f32 inputs to the + S2F6x2 type. The result `dst` can be either a packed i16 type or a vector + of two i8 types. + If `dst` is returned as an i16 type, the converted values are packed such + that the value converted from `a` is stored in the upper 8 bits of `dst` + and the value converted from `b` is stored in the lower 8 bits of `dst`. + If `dst` is returned as a vector type, each converted value is stored as an + i8 element in the vector. + The `relu` attribute, when set, lowers to the '.relu' variant + of the cvt instruction. + The optional scaling-factors for each of the inputs are provided through + the operand `scaleFactor` as a packed i16 type. Only `ue8m0` is supported + as the type of the scale-factor currently. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + NVVM_T32BitFloat:$a, + NVVM_T32BitFloat:$b, + Optional:$scaleFactor, + OptionalAttr:$relu + ); + let results = (outs + NVVM_T16BitSignlessIntegerOrVector:$dst + ); +} +def NVVM_ConvertF32x4ToF4x4Op : NVVM_Op<"convert.f32x4.to.f4x4"> { + let description = [{ + Converts a vector<4xf32> to packed f4x4 format using + stochastic rounding (.rs) mode with SATFINITE saturation. Randomness is + provided by the `rbits` parameter. The `dstTy` attribute specifies the + target floating-point format. The `relu` attribute clamps negative results to 0. + + Note: These operations always use RS rounding mode and SATFINITE saturation mode. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + NVVM_VectorOf32BitFloatValues3:$src, + NVVM_T32BitSignlessInteger:$rbits, + OptionalAttr:$relu, + TypeAttr:$dstTy + ); + let results = (outs + NVVM_T16BitSignlessInteger2:$dst + ); +} +def NVVM_ConvertF32x4ToF6x4Op : NVVM_Op<"convert.f32x4.to.f6x4"> { + let description = [{ + Converts a vector<4xf32> to packed f6x4 format using + stochastic rounding (.rs) mode with SATFINITE saturation. Randomness is + provided by the `rbits` parameter. The `dstTy` attribute specifies the + target floating-point format. The `relu` attribute clamps negative results to 0. + + Note: These operations always use RS rounding mode and SATFINITE saturation mode. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + NVVM_VectorOf32BitFloatValues3:$src, + NVVM_T32BitSignlessInteger:$rbits, + OptionalAttr:$relu, + TypeAttr:$dstTy + ); + let results = (outs + NVVM_VectorOf8BitSignlessInteger:$dst + ); +} +def NVVM_ConvertF32x4ToF8x4Op : NVVM_Op<"convert.f32x4.to.f8x4"> { + let description = [{ + Converts a vector<4xf32> to packed f8x4 format using + stochastic rounding (.rs) mode with SATFINITE saturation. Randomness is + provided by the `rbits` parameter. The `dstTy` attribute specifies the + target floating-point format. The `relu` attribute clamps negative results to 0. + + Note: These operations always use RS rounding mode and SATFINITE saturation mode. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + NVVM_VectorOf32BitFloatValues3:$src, + NVVM_T32BitSignlessInteger:$rbits, + OptionalAttr:$relu, + TypeAttr:$dstTy + ); + let results = (outs + NVVM_VectorOf8BitSignlessInteger:$dst + ); +} +def NVVM_ConvertF4x2ToF16x2Op : NVVM_Op<"convert.f4x2.to.f16x2"> { + let description = [{ + This Op converts the given f4 inputs in a packed i8 to f16. + + The result `dst` is represented as a vector of f16 elements. + The `relu` attribute, when set, lowers to the '.relu' variant of + the cvt instruction." + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + NVVM_T8BitSignlessInteger2:$src, + OptionalAttr:$relu, + TypeAttr:$srcType + ); + let results = (outs + NVVM_VectorOf16BitFloatValues:$dst + ); +} +def NVVM_ConvertF6x2ToF16x2Op : NVVM_Op<"convert.f6x2.to.f16x2"> { + let description = [{ + This Op converts the given f6 inputs in a i8x2 vector to f16. + + The result `dst` is represented as a vector of f16 elements. + The `relu` attribute, when set, lowers to the '.relu' variant of + the cvt instruction." + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + NVVM_VectorOf8BitSignlessInteger2:$src, + OptionalAttr:$relu, + TypeAttr:$srcType + ); + let results = (outs + NVVM_VectorOf16BitFloatValues:$dst + ); +} +def NVVM_ConvertF8x2ToBF16x2Op : NVVM_Op<"convert.f8x2.to.bf16x2"> { + let description = [{ + This Op converts the given f8 inputs in a i8x2 vector to bf16. + + The result `dst` is represented as a vector of bf16 elements. + The `sat` attribute specifies the saturation mode. + + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + NVVM_VectorOf8BitSignlessInteger2:$src, + Optional:$scaleFactor, + OptionalAttr:$sat, + TypeAttr:$srcType + ); + let results = (outs + NVVM_VectorOfBfloat16TypeValuesOf:$dst + ); +} +def NVVM_ConvertF8x2ToF16x2Op : NVVM_Op<"convert.f8x2.to.f16x2"> { + let description = [{ + This Op converts the given f8 inputs in a i8x2 vector to f16. + + The result `dst` is represented as a vector of f16 elements. + The `relu` attribute, when set, lowers to the '.relu' variant of + the cvt instruction." + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + NVVM_VectorOf8BitSignlessInteger2:$src, + OptionalAttr:$relu, + TypeAttr:$srcType + ); + let results = (outs + NVVM_VectorOf16BitFloatValues:$dst + ); +} +def NVVM_ConvertFloatToIntegerOp : NVVM_Op<"convert.float.to.integer"> { + let description = [{ + This Op converts the given float input to the specified integer type. + + The `rnd` attribute specifies the rounding mode. + The `sat` atttribute, when set, limits the result to the range of the + integer type. + The `ftz` attribute, when set, converts denormal inputs to zero. + The `signed` attribute, when set, converts the input to a signed integer. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + NVVM_T32BitFloat:$src, + OptionalAttr:$rnd, + UnitAttr:$sat, + UnitAttr:$ftz, + UnitAttr:$is_signed + ); + let results = (outs + NVVM_T8BitSignlessIntegerOr32:$dst + ); +} +def NVVM_ConvertFloatToTF32Op : NVVM_Op<"convert.float.to.tf32"> { + let description = [{ + This Op converts the given f32 input to tf32. + The result `res` is represented as an i32 type. + The `relu` attribute, when set, lowers to the '.relu' variant of + the cvt instruction. The `rnd` and `sat` attributes specify the + the rounding and saturation modes respectively. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + NVVM_T32BitFloat:$src, + OptionalAttr:$rnd, + OptionalAttr:$sat, + OptionalAttr:$relu + ); + let results = (outs + NVVM_T32BitSignlessInteger2:$res + ); +} +def NVVM_ConvertS2F6x2ToBF16x2Op : NVVM_Op<"convert.s2f6x2.to.bf16x2"> { + let description = [{ + This Op converts a pair of s2f6x2 inputs to bf16x2 type. The result `dst` + is represented as a vector of two bf16 elements. + + The `relu` attribute, when set, lowers to the '.relu' variant + of the cvt instruction. + + The optional scaling-factors for each of the inputs are provided through + the operand `scaleFactor` as a packed i16 type. Only `ue8m0` is supported + as the type of the scale-factor currently. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + NVVM_VectorOf8BitSignlessInteger2:$src, + Optional:$scaleFactor, + OptionalAttr:$sat, + OptionalAttr:$relu + ); + let results = (outs + NVVM_VectorOfBfloat16TypeValuesOf:$dst + ); +} +def NVVM_CosOp : NVVM_Op<"cos", [Pure]> { + let description = [{ + Computes a fast approximation of the cosine of the input value (in + radians). The `ftz` attribute, when set, flushes subnormal inputs + and results to sign-preserving zero. + }]; + let arguments = (ins + NVVM_T32BitFloat:$src, + OptionalAttr:$ftz + ); + let results = (outs + NVVM_T32BitFloat2:$res + ); +} +def NVVM_CpAsyncBulkCommitGroupOp : NVVM_Op<"cp.async.bulk.commit.group"> { + let description = [{ + This Op commits all prior initiated but uncommitted cp.async.bulk + instructions into a cp.async.bulk-group. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-commit-group) + }]; + let arguments = (ins); + let results = (outs); +} +def NVVM_CpAsyncBulkSharedCTAToGlobalOp : NVVM_Op<"cp.async.bulk.global.shared.cta", [AttrSizedOperandSegments]> { + let description = [{ + Initiates an asynchronous copy operation from Shared CTA memory to + global memory. The 32-bit operand `size` specifies the amount of + memory to be copied, in terms of number of bytes. `size` must be a + multiple of 16. The `l2CacheHint` operand is optional, and it is used + to specify cache eviction policy that may be used during the memory + access. The `byteMask` operand is optional. The i-th bit in the 16-bit + wide `byteMask` specifies whether the i-th byte of each 16-byte wide + chunk of source data is copied to the destination. If the bit is set, + the byte is copied. + + Example: + ```mlir + nvvm.cp.async.bulk.global.shared.cta %dst, %src, %size + : !llvm.ptr<1>, !llvm.ptr<3> + + // with l2_cache_hint + nvvm.cp.async.bulk.global.shared.cta %dst, %src, %size l2_cache_hint = %ch + : !llvm.ptr<1>, !llvm.ptr<3> + + // with byte_mask + nvvm.cp.async.bulk.global.shared.cta %dst, %src, %size byte_mask = %mask + : !llvm.ptr<1>, !llvm.ptr<3> + + // with both l2_cache_hint and byte_mask + nvvm.cp.async.bulk.global.shared.cta %dst, %src, %size l2_cache_hint = %ch byte_mask = %mask + : !llvm.ptr<1>, !llvm.ptr<3> + ``` + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace1:$dstMem, + NVVM_LLVMPointerInAddressSpace3:$srcMem, + NVVM_T32BitSignlessInteger:$size, + Optional:$l2CacheHint, + Optional:$byteMask + ); + let results = (outs); +} +def NVVM_CpAsyncBulkPrefetchOp : NVVM_Op<"cp.async.bulk.prefetch"> { + let description = [{ + Initiates an asynchronous prefetch of data from the location + specified by `srcMem` to the L2 cache. + + The `l2CacheHint` operand is optional, and it is used to specify cache + eviction policy that may be used during the memory access. + The `evict` operand is optional, and it is used to specify cache eviction + priority. The `evict` operand is mutually exclusive with the `l2CacheHint` + operand. + + Example: + ```mlir + nvvm.cp.async.bulk.prefetch %src, %size : !llvm.ptr<1> + + // with l2_cache_hint + nvvm.cp.async.bulk.prefetch %src, %size l2_cache_hint = %ch : !llvm.ptr<1> + ``` + Example: + ```mlir + // with evict_priority + nvvm.cp.async.bulk.prefetch %src, %size evict_priority = evict_last : !llvm.ptr<1> + ``` + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-prefetch) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace1:$srcMem, + NVVM_T32BitSignlessInteger:$size, + Optional:$l2CacheHint, + OptionalAttr:$evict + ); + let results = (outs); +} +def NVVM_CpAsyncBulkGlobalToSharedClusterOp : NVVM_Op<"cp.async.bulk.shared.cluster.global", [AttrSizedOperandSegments]> { + let description = [{ + Initiates an asynchronous copy operation from global memory to shared + memory or shared_cluster memory. + + The `multicastMask` operand is optional and can be used only when the + destination is shared::cluster memory. When it is present, this Op copies + data from global memory to shared memory of multiple CTAs in the cluster. + Operand `multicastMask` specifies the destination CTAs in the cluster such + that each bit position in the 16-bit `multicastMask` operand corresponds to + the `nvvm.read.ptx.sreg.ctaid` of the destination CTA. + + The `l2CacheHint` operand is optional, and it is used to specify cache + eviction policy that may be used during the memory access. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace32:$dstMem, + NVVM_LLVMPointerInAddressSpace1:$srcMem, + NVVM_LLVMPointerInAddressSpace3:$mbar, + NVVM_T32BitSignlessInteger:$size, + Optional:$multicastMask, + Optional:$l2CacheHint, + OptionalAttr:$validatePattern + ); + let results = (outs); +} +def NVVM_CpAsyncBulkSharedCTAToSharedClusterOp : NVVM_Op<"cp.async.bulk.shared.cluster.shared.cta"> { + let description = [{ + Initiates an asynchronous copy operation from Shared CTA memory to Shared + cluster memory. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace7:$dstMem, + NVVM_LLVMPointerInAddressSpace3:$srcMem, + NVVM_LLVMPointerInAddressSpace3:$mbar, + NVVM_T32BitSignlessInteger:$size + ); + let results = (outs); +} +def NVVM_CpAsyncBulkTensorSharedCTAToGlobalOp : NVVM_Op<"cp.async.bulk.tensor.global.shared.cta", [AttrSizedOperandSegments]> { + let description = [{ + Initiates an asynchronous copy of the tensor data from shared::cta + memory to global memory. This Op supports all the store modes specified in + `TMAStoreMode`. + + The `l2CacheHint` operand is optional, and it is used to specify cache + eviction policy that may be used during the memory access. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-cp-async-bulk-tensor) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace0:$tmaDescriptor, + NVVM_LLVMPointerInAddressSpace3:$srcMem, + Variadic:$coordinates, + Optional:$l2CacheHint, + Optional:$predicate, + OptionalAttr:$mode + ); + let results = (outs); +} +def NVVM_CpAsyncBulkTensorSharedCTAToGlobalOverrideAddrOp : NVVM_Op<"cp.async.bulk.tensor.global.shared.cta.override", [AttrSizedOperandSegments]> { + let description = [{ + Initiates an asynchronous bulk tensor operation that transfers data from shared + memory to global memory overriding one or many of the fields: addr, size, and stride. + + The operation: + - Reads data from shared memory location specified by `srcMem` + - Uses a TMA descriptor `tmaDesc` to control the data access pattern + - Overrides the default destination address with `overrideAdrr` + - Specifies box coordinates via `coordinates` array + - Can specify tensor size via `tensorSize` (optional) + - Controls memory access patterns with `lowerStride` and `upperStride` (optional) + - Supports an optional L2 cache hint with `l2CacheHint` to support cache eviction policy (optional) + - Supports different store modes (`TILE`, `IM2COL`, `IM2COL_W`, and `TILE_SCATTER4`) via the `mode` attribute + + Examples: + + // Basic usage with 1D box coordinates (TILE mode) + ```mlir + nvvm.cp.async.bulk.tensor.global.shared.cta.override %tma_desc, %src, %override_addr, box[%d0] { + mode = #nvvm.tma_store_mode + } : !llvm.ptr, !llvm.ptr<3>, !llvm.ptr<1> + ``` + + // With L2 cache hint + ```mlir + nvvm.cp.async.bulk.tensor.global.shared.cta.override %tma_desc, %src, %override_addr, box[%d0, %d1, %d2] + l2_cache_hint = %ch { + mode = #nvvm.tma_store_mode + } : !llvm.ptr, !llvm.ptr<3>, !llvm.ptr<1> + ``` + + // IM2COL mode + ```mlir + nvvm.cp.async.bulk.tensor.global.shared.cta.override %tma_desc, %src, %override_addr, box[%d0, %d1, %d2] { + mode = #nvvm.tma_store_mode + } : !llvm.ptr, !llvm.ptr<3>, !llvm.ptr<1> + ``` + + // With tensor size + ```mlir + nvvm.cp.async.bulk.tensor.global.shared.cta.override %tma_desc, %src, %override_addr, + box[%d0] tensor_size[%ts0] { + mode = #nvvm.tma_store_mode + } : !llvm.ptr, !llvm.ptr<3>, !llvm.ptr<1> + ``` + + // With stride + ```mlir + nvvm.cp.async.bulk.tensor.global.shared.cta.override %tma_desc, %src, %override_addr, + box[%d0, %d1] tensor_size[%ts0, %ts1] lower_stride[%lstrd0] upper_stride[%ustrd] { + mode = #nvvm.tma_store_mode + } : !llvm.ptr, !llvm.ptr<3>, !llvm.ptr<1> + ``` + }]; + let arguments = (ins + NVVM_LLVMPointerType:$tmaDesc, + NVVM_LLVMPointerInAddressSpace3:$srcMem, + NVVM_LLVMPointerInAddressSpace1:$overrideAdrr, + Variadic:$coordinates, + Variadic:$tensorSize, + Variadic:$lowerStride, + Optional:$upperStride, + Optional:$l2CacheHint, + OptionalAttr:$mode + ); + let results = (outs); +} +def NVVM_CpAsyncBulkTensorPrefetchOp : NVVM_Op<"cp.async.bulk.tensor.prefetch", [AttrSizedOperandSegments]> { + let description = [{ + Initiates an asynchronous prefetch operation on the tensor data from global + memory to L2 cache. This Op supports all the load modes specified in + `TMALoadMode`. + + The `l2CacheHint` operand is optional, and it is used to specify cache + eviction policy that may be used during the memory access. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-prefetch-tensor) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace0:$tmaDescriptor, + Variadic:$coordinates, + Variadic:$im2colOffsets, + Optional:$l2CacheHint, + OptionalAttr:$mode, + OptionalAttr:$evict + ); + let results = (outs); +} +def NVVM_CpAsyncBulkTensorPrefetchOverrideAddrOp : NVVM_Op<"cp.async.bulk.tensor.prefetch.override", [AttrSizedOperandSegments]> { + let description = [{ + Initiates an asynchronous bulk tensor operation that prefetches data from + global memory to L2 cache overriding one or many of the fields: addr, size, and stride. + + The operation: + - Reads data from global memory location specified by `overrideAdrr` + - Uses a TMA descriptor `tmaDesc` to control the data access pattern + - Specifies box coordinates via `coordinates` array + - Can specify tensor size via `tensorSize` (optional) + - Controls memory access patterns with `lowerStride` and `upperStride` (optional) + - Can specify im2col offsets via `im2colOffsets` (optional) + - Supports an optional L2 cache hint with `l2CacheHint` to support cache eviction policy (optional) + - Supports an optional cache eviction priority with `evict` (optional) + - Supports different load modes (`TILE`/`IM2COL`/`IM2COL_W`/`IM2COL_W_128`/`TILE_GATHER4`) via the `mode` attribute (default = `TILE`) + + Examples: + + // Basic prefetch operations + ```mlir + nvvm.cp.async.bulk.tensor.prefetch.override %tma_desc, %override_addr, box[%d0] : !llvm.ptr, !llvm.ptr<1> + nvvm.cp.async.bulk.tensor.prefetch.override %tma_desc, %override_addr, box[%d0, %d1, %d2] : !llvm.ptr, !llvm.ptr<1> + ``` + + // With L2 cache hint + ```mlir + nvvm.cp.async.bulk.tensor.prefetch.override %tma_desc, %override_addr, box[%d0] l2_cache_hint = %ch : !llvm.ptr, !llvm.ptr<1> + nvvm.cp.async.bulk.tensor.prefetch.override %tma_desc, %override_addr, box[%d0, %d1, %d2] l2_cache_hint = %ch : !llvm.ptr, !llvm.ptr<1> + ``` + + // With evict_priority + ```mlir + nvvm.cp.async.bulk.tensor.prefetch.override %tma_desc, %override_addr, box[%d0] evict_priority = evict_last : !llvm.ptr, !llvm.ptr<1> + nvvm.cp.async.bulk.tensor.prefetch.override %tma_desc, %override_addr, box[%d0, %d1, %d2] evict_priority = evict_last : !llvm.ptr, !llvm.ptr<1> + ``` + + // With tensor size specification + ```mlir + nvvm.cp.async.bulk.tensor.prefetch.override %tma_desc, %override_addr, box[%d0] tensor_size[%ts0] : !llvm.ptr, !llvm.ptr<1> + ``` + + // With tensor size and stride specifications + ```mlir + nvvm.cp.async.bulk.tensor.prefetch.override %tma_desc, %override_addr, box[%d0, %d1] + tensor_size[%ts0, %ts1] lower_stride[%ls0] upper_stride[%us0] : !llvm.ptr, !llvm.ptr<1> + ``` + + // With im2col offsets + ```mlir + nvvm.cp.async.bulk.tensor.prefetch.override %tma_desc, %override_addr, box[%d0, %d1, %d2] + im2col[%off0] : !llvm.ptr, !llvm.ptr<1> + nvvm.cp.async.bulk.tensor.prefetch.override %tma_desc, %override_addr, box[%d0, %d1, %d2, %d3] + im2col[%off0, %off1] : !llvm.ptr, !llvm.ptr<1> + ``` + + // Dim stride with cache hint + ```mlir + nvvm.cp.async.bulk.tensor.prefetch.override %tma_desc, %override_addr, box[%d0, %d1, %d2, %d3, %d4] + tensor_size[%ts0, %ts1, %ts2, %ts3, %ts4] + lower_stride[%ls0, %ls1, %ls2, %ls3] + upper_stride[%us0] + l2_cache_hint = %ch : !llvm.ptr, !llvm.ptr<1> + ``` + }]; + let arguments = (ins + NVVM_LLVMPointerType:$tmaDesc, + NVVM_LLVMPointerInAddressSpace1:$overrideAdrr, + Variadic:$coordinates, + Variadic:$tensorSize, + Variadic:$lowerStride, + Optional:$upperStride, + Variadic:$im2colOffsets, + Optional:$l2CacheHint, + OptionalAttr:$evict, + OptionalAttr:$mode + ); + let results = (outs); +} +def NVVM_CpAsyncBulkTensorReduceOp : NVVM_Op<"cp.async.bulk.tensor.reduce", [AttrSizedOperandSegments]> { + let description = [{ + Initiates an asynchronous reduction operation of tensor data in + global memory with tensor data in shared memory. + + The `mode` attribute indicates whether the copy mode is tile or im2col. + The `redOp` attribute specifies the reduction operations applied. + The supported reduction operations are: + {add, min, max, inc, dec, and, or, xor} + + The `l2CacheHint` operand is optional, and it is used to specify cache + eviction policy that may be used during the memory access. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-reduce-async-bulk-tensor) + }]; + let arguments = (ins + NVVM_LLVMPointerType:$tmaDescriptor, + NVVM_LLVMPointerInAddressSpace3:$srcMem, + Variadic:$coordinates, + Optional:$l2CacheHint, + NVVM_TMAReduxKindAttr:$redKind, + OptionalAttr:$mode + ); + let results = (outs); +} +def NVVM_CpAsyncBulkTensorReduceOverrideAddrOp : NVVM_Op<"cp.async.bulk.tensor.reduce.override", [AttrSizedOperandSegments]> { + let description = [{ + Initiates an asynchronous bulk tensor operation that performs a reduction operation + overriding one or many of the fields: addr, size, and stride. + + The operation: + - Reads data from shared memory location specified by `srcMem` + - Uses a TMA descriptor `tmaDesc` to control the data access pattern + - Overrides the default destination address with `overrideAdrr` + - Specifies box coordinates via `coordinates` array + - Can specify tensor size via `tensorSize` (optional) + - Controls memory access patterns with `lowerStride` and `upperStride` (optional) + - Supports an optional L2 cache hint with `l2CacheHint` to support cache eviction policy (optional) + - Supports different store modes (`TILE` or `IM2COL`) via the `mode` attribute + - Performs a reduction operation specified by the `redKind` attribute (supports ADD, MIN, MAX, + INC, DEC, AND, OR, XOR) that combines the source data stored at the destination location + + Examples: + + // Override address with ADD reduction (TILE mode) + ```mlir + nvvm.cp.async.bulk.tensor.reduce.override %tma_desc, %src, %override_addr, box[%d0] { + redKind = #nvvm.tma_redux_kind, + mode = #nvvm.tma_store_mode + } : !llvm.ptr, !llvm.ptr<3>, !llvm.ptr<1> + ``` + + // With L2 cache hint + ```mlir + nvvm.cp.async.bulk.tensor.reduce.override %tma_desc, %src, %override_addr, box[%d0, %d1] + l2_cache_hint = %ch { + redKind = #nvvm.tma_redux_kind, + mode = #nvvm.tma_store_mode + } : !llvm.ptr, !llvm.ptr<3>, !llvm.ptr<1> + ``` + + // IM2COL mode with XOR reduction + ```mlir + nvvm.cp.async.bulk.tensor.reduce.override %tma_desc, %src, %override_addr, box[%d0, %d1, %d2] { + redKind = #nvvm.tma_redux_kind, + mode = #nvvm.tma_store_mode + } : !llvm.ptr, !llvm.ptr<3>, !llvm.ptr<1> + ``` + + // With tensor size (dim variant) + ```mlir + nvvm.cp.async.bulk.tensor.reduce.override %tma_desc, %src, %override_addr, + box[%d0] tensor_size[%ts0] { + redKind = #nvvm.tma_redux_kind, + mode = #nvvm.tma_store_mode + } : !llvm.ptr, !llvm.ptr<3>, !llvm.ptr<1> + ``` + + // With stride information (dim_stride variant) + ```mlir + nvvm.cp.async.bulk.tensor.reduce.override %tma_desc, %src, %override_addr, + box[%d0, %d1] tensor_size[%ts0, %ts1] lower_stride[%lstrd0] upper_stride[%ustrd] { + redKind = #nvvm.tma_redux_kind, + mode = #nvvm.tma_store_mode + } : !llvm.ptr, !llvm.ptr<3>, !llvm.ptr<1> + ``` + }]; + let arguments = (ins + NVVM_LLVMPointerType:$tmaDesc, + NVVM_LLVMPointerInAddressSpace3:$srcMem, + NVVM_LLVMPointerInAddressSpace1:$overrideAdrr, + Variadic:$coordinates, + Variadic:$tensorSize, + Variadic:$lowerStride, + Optional:$upperStride, + Optional:$l2CacheHint, + NVVM_TMAReduxKindAttr:$redKind, + OptionalAttr:$mode + ); + let results = (outs); +} +def NVVM_CpAsyncBulkTensorGlobalToSharedClusterOp : NVVM_Op<"cp.async.bulk.tensor.shared.cluster.global", [AttrSizedOperandSegments]> { + let description = [{ + Initiates an asynchronous copy operation on the tensor data from global + memory to shared::cluster (or) shared::cta memory. This Op supports all + the load modes specified in `TMALoadMode`. + + The `multicastMask` operand is optional. When it is present, the Op copies + data from global memory to shared memory of multiple CTAs in the cluster. + Operand `multicastMask` specifies the destination CTAs in the cluster such + that each bit position in the 16-bit `multicastMask` operand corresponds to + the `nvvm.read.ptx.sreg.ctaid` of the destination CTA. + + The `l2CacheHint` operand is optional, and it is used to specify cache + eviction policy that may be used during the memory access. + + When the `isCTAOnly` attribute is set to true, the destination is + shared::cta only. Hence, `multicastMask` and `CTAGroup` are not applicable + when `isCTAOnly` is true. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace32:$dstMem, + NVVM_LLVMPointerInAddressSpace0:$tmaDescriptor, + Variadic:$coordinates, + NVVM_LLVMPointerInAddressSpace3:$mbar, + Variadic:$im2colOffsets, + Optional:$multicastMask, + Optional:$l2CacheHint, + Optional:$predicate, + OptionalAttr:$validatePattern, + OptionalAttr:$mode, + OptionalAttr:$isCTAOnly, + OptionalAttr:$group + ); + let results = (outs); +} +def NVVM_CpAsyncBulkTensorGlobalToSharedClusterOverrideAddrOp : NVVM_Op<"cp.async.bulk.tensor.shared.cluster.global.override", [AttrSizedOperandSegments]> { + let description = [{ + Initiates an asynchronous bulk tensor operation that transfers data from global + memory to shared::cluster or shared::cta memory overriding one or many of the fields: addr, size, and stride. + + The operation: + - Writes data to shared::cluster or shared::cta memory location specified by `dstMem` + - Uses a TMA descriptor `tmaDesc` to specify the properties of the tensor copy operation + - Uses `mbar` to specify the memory barrier for the operation + - Overrides the default source address with `overrideAddr` + - Specifies box coordinates via `coordinates` array + - Specifies im2col arguments via `im2colArgs` (optional) + - Can specify tensor size via `tensorSize` (optional) + - Controls memory access patterns with `lowerStride` and `upperStride` (optional) + - Supports multicast mask via `multicastMask` (optional) + - Supports an optional L2 cache hint with `l2CacheHint` to support cache eviction policy (optional) + - Supports different load modes (`TILE`/`IM2COL`/`IM2COL_W`/`IM2COL_W_128`/`TILE_GATHER4`) via the `mode` attribute (default = `TILE`) + - Supports different group kinds (`CTA_1` or `CTA_2`) via the `group` attribute (optional) + - Supports different validate load data patterns via the `validatePattern` attribute (optional) + + Examples: + // Basic usage with box coordinates + ```mlir + nvvm.cp.async.bulk.tensor.shared.cluster.global.override %dst, %tma_desc, %mbar, %override_addr, box[%d0] { + mode = #nvvm.tma_load_mode + } : !llvm.ptr<3>, !llvm.ptr, !llvm.ptr<1> + ``` + + ```mlir + nvvm.cp.async.bulk.tensor.shared.cluster.global.override %dst, %tma_desc, %mbar, %override_addr, box[%d0, %d1] { + mode = #nvvm.tma_load_mode + } : !llvm.ptr<3>, !llvm.ptr, !llvm.ptr<1> + ``` + + // With L2 cache hint + ```mlir + nvvm.cp.async.bulk.tensor.shared.cluster.global.override %dst, %tma_desc, %mbar, %override_addr, box[%d0] + l2_cache_hint = %ch { + mode = #nvvm.tma_load_mode + } : !llvm.ptr<3>, !llvm.ptr, !llvm.ptr<1> + ``` + + // With multicast mask (i16) + ```mlir + nvvm.cp.async.bulk.tensor.shared.cluster.global.override %dst, %tma_desc, %mbar, %override_addr, box[%d0] + multicast_mask = %mc { + mode = #nvvm.tma_load_mode + } : !llvm.ptr<3>, !llvm.ptr, !llvm.ptr<1>, i16 + ``` + + // With multicast mask (i32) + ```mlir + nvvm.cp.async.bulk.tensor.shared.cluster.global.override %dst, %tma_desc, %mbar, %override_addr, box[%d0] + multicast_mask = %mc32 { + mode = #nvvm.tma_load_mode + } : !llvm.ptr<3>, !llvm.ptr, !llvm.ptr<1>, i32 + ``` + + // With both multicast mask and L2 cache hint + ```mlir + nvvm.cp.async.bulk.tensor.shared.cluster.global.override %dst, %tma_desc, %mbar, %override_addr, box[%d0] + multicast_mask = %mc + l2_cache_hint = %ch { + mode = #nvvm.tma_load_mode + } : !llvm.ptr<3>, !llvm.ptr, !llvm.ptr<1>, i16 + ``` + + // With CTA_2 group + ```mlir + nvvm.cp.async.bulk.tensor.shared.cluster.global.override %dst, %tma_desc, %mbar, %override_addr, box[%d0] + l2_cache_hint = %ch { + group = #nvvm.cta_group, + mode = #nvvm.tma_load_mode + } : !llvm.ptr<3>, !llvm.ptr, !llvm.ptr<1> + ``` + + // With tensor size and strides + ```mlir + nvvm.cp.async.bulk.tensor.shared.cluster.global.override %dst, %tma_desc, %mbar, %override_addr, box[%d0] + tensor_size[%ts0] { + mode = #nvvm.tma_load_mode + } : !llvm.ptr<3>, !llvm.ptr, !llvm.ptr<1> + ``` + + ```mlir + nvvm.cp.async.bulk.tensor.shared.cluster.global.override %dst, %tma_desc, %mbar, %override_addr, box[%d0, %d1] + tensor_size[%ts0, %ts1] + lower_stride[%lstrd0] + upper_stride[%ustrd] { + mode = #nvvm.tma_load_mode + } : !llvm.ptr<3>, !llvm.ptr, !llvm.ptr<1> + ``` + + // Using IM2COL mode with im2col arguments + ```mlir + nvvm.cp.async.bulk.tensor.shared.cluster.global.override %dst, %tma_desc, %mbar, %override_addr, box[%d0, %d1, %d2] + im2col[%im2col0] { + mode = #nvvm.tma_load_mode + } : !llvm.ptr<3>, !llvm.ptr, !llvm.ptr<1> + ``` + + // Using IM2COL_W mode + ```mlir + nvvm.cp.async.bulk.tensor.shared.cluster.global.override %dst, %tma_desc, %mbar, %override_addr, box[%d0, %d1, %d2] + im2col[%wHalo, %wOffset] { + mode = #nvvm.tma_load_mode + } : !llvm.ptr<3>, !llvm.ptr, !llvm.ptr<1> + ``` + + // Using IM2COL_W_128 mode + ```mlir + nvvm.cp.async.bulk.tensor.shared.cluster.global.override %dst, %tma_desc, %mbar, %override_addr, box[%d0, %d1, %d2] + im2col[%wHalo, %wOffset] { + mode = #nvvm.tma_load_mode + } : !llvm.ptr<3>, !llvm.ptr, !llvm.ptr<1> + ``` + + // Comprehensive example with all optional parameters + ```mlir + nvvm.cp.async.bulk.tensor.shared.cluster.global.override %dst, %tma_desc, %mbar, %override_addr, box[%d0, %d1] + tensor_size[%ts0, %ts1] + lower_stride[%lstrd0] + upper_stride[%ustrd] + multicast_mask = %mc32 + l2_cache_hint = %ch { + group = #nvvm.cta_group, + mode = #nvvm.tma_load_mode + } : !llvm.ptr<3>, !llvm.ptr, !llvm.ptr<1>, i32 + ``` + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace72:$dstMem, + NVVM_LLVMPointerType:$tmaDesc, + NVVM_LLVMPointerInAddressSpace3:$mbar, + NVVM_LLVMPointerInAddressSpace1:$overrideAddr, + Variadic:$coordinates, + Variadic:$im2colArgs, + Variadic:$tensorSize, + Variadic:$lowerStride, + Optional:$upperStride, + Optional:$multicastMask, + Optional:$l2CacheHint, + OptionalAttr:$group, + OptionalAttr:$validatePattern, + OptionalAttr:$mode + ); + let results = (outs); +} +def NVVM_CpAsyncBulkWaitGroupOp : NVVM_Op<"cp.async.bulk.wait_group"> { + let description = [{ + Op waits for completion of the most recent bulk async-groups. + + The `$group` operand tells waiting has to be done until for $group or fewer + of the most recent bulk async-groups. If `$group` is 0, the op wait until + all the most recent bulk async-groups have completed. + + The `$read` indicates that the waiting has to be done until all the bulk + async operations in the specified bulk async-group have completed reading + from their source locations. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-wait-group) + }]; + let arguments = (ins + I32Attr:$group, + UnitAttr:$read + ); + let results = (outs); +} +def NVVM_CpAsyncCommitGroupOp : NVVM_Op<"cp.async.commit.group"> { + let arguments = (ins); + let results = (outs); +} +def NVVM_CpAsyncMBarrierArriveOp : NVVM_Op<"cp.async.mbarrier.arrive"> { + let description = [{ + The `cp.async.mbarrier.arrive` Op makes the *mbarrier object* track + all prior cp.async operations initiated by the executing thread. + The `addr` operand specifies the address of the *mbarrier object* + in generic or shared::cta address space. When it is generic, the + underlying memory should fall within the shared::cta space; + otherwise the behavior is undefined. The `noinc` attr impacts + how the mbarrier's state is updated. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-cp-async-mbarrier-arrive) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace02:$addr, + OptionalAttr:$noinc + ); + let results = (outs); +} +def NVVM_CpAsyncOp : NVVM_Op<"cp.async.shared.global"> { + let arguments = (ins + NVVM_LLVMPointerInAddressSpace3:$dst, + NVVM_LLVMPointerInAddressSpace1:$src, + Optional:$cpSize, + I32Attr:$size, + NVVM_LoadCacheModifierAttr:$modifier + ); + let results = (outs); +} +def NVVM_CpAsyncWaitGroupOp : NVVM_Op<"cp.async.wait.group"> { + let arguments = (ins + I32Attr:$n + ); + let results = (outs); +} +def NVVM_CvtPackFloatOp : NVVM_Op<"cvt.packfloat"> { + let description = [{ + This Op converts the given floating point input (`srcA`) to another + packed floating point type (`res`). The operand `from` represents the + source type and `to` represents the destination type. + The result is merged with the given `srcC`. Both the input and result + are represented as 'i32' type. + + Below table describes the format of the result: + ``` + |--------------|-------------------------------------------------------| + | Dst Types | Description | + |--------------|-------------------------------------------------------| + | f16x2 | The result contains two fp16 values, in the low | + | bf16x2 | and high bits. | + |--------------|-------------------------------------------------------| + | e4m3x2 | The result contains two fp8 values in the low or | + | e5m2x2 | high 16-bits of `srcC` depending on `ExtractHi`. | + | ue8m0x2 | When `ExtractHi` is 0, the result is packed in | + | | bits `[7:0]` and `[15:8]` of `srcC` and the high | + | | 16-bits of srcC remain unchanged. | + | | When `ExtractHi` is 1, the result is packed in | + | | bits `[23:16]` and `[31:24]` of `srcC` and the low | + | | 16-bits of srcC remain unchanged. | + |--------------|-------------------------------------------------------| + | e2m3x2 | The result contains two 6-bit floating point values | + | e3m2x2 | with the two MSB's padded with zeroes. The 16-bits | + | | of the result are then packed in the same format | + | | described above. | + |--------------|-------------------------------------------------------| + | e2m1x2 | The result contains two 4-bit floating point values | + | | packed in bits `[7:0]` or `[23:16]` of `srcC` | + | | depending on `ExtractHi`. The other bits of `srcC` | + | | remain unchanged. | + |--------------|-------------------------------------------------------| + ``` + + Below table describes the format of the source operand: + ``` + |--------------|-------------------------------------------------------| + | Src Types | Description | + |--------------|-------------------------------------------------------| + | f16x2 | The two fp16 values are taken from the low and high | + | bf16x2 | high bits of operand `srcA`. | + |--------------|-------------------------------------------------------| + | e4m3x2 | The two fp8 values are taken from the low or high | + | e5m2x2 | 16-bits of `srcA` depending on `ExtractHi`. | + | ue8m0x2 | When `ExtractHi` is 0, the inputs are taken from | + | | bits `[7:0]` and `[15:8]` of `srcA`. | + | | When `ExtractHi` is 1, the inputs are sourced from | + | | bits `[23:16]` and `[31:24]` of `srcA`. | + |--------------|-------------------------------------------------------| + | e2m3x2 | Similar to the row above. | + | e3m2x2 | | + |--------------|-------------------------------------------------------| + | e2m1x2 | The two 4-bit floating point input values are taken | + | | from bits `[7:0]` or `[23:16]` of `srcA` depending | + | | on `ExtractHi`. | + |--------------|-------------------------------------------------------| + ``` + The `relu` attribute, when set, lowers to the '.relu' variant of the + cvt instruction. The `rnd` and `sat` attributes specify the rounding + and saturation respectively. The ue8m0x2, e2m3x2, e3m2x2 and e2m1x2 + types, either as source or destination types, are supported only in SM100a. + [For more information, see PTX ISA] + (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + NVVM_T32BitSignlessInteger:$srcA, + NVVM_T32BitSignlessInteger:$srcC, + NVVM_CVTPackFloatKindAttr:$from, + NVVM_CVTPackFloatKindAttr:$to, + OptionalAttr:$rnd, + OptionalAttr:$sat, + OptionalAttr:$relu, + OptionalAttr:$extractHi + ); + let results = (outs + NVVM_T32BitSignlessInteger2:$res + ); +} +def NVVM_CvtPackFloatF32Op : NVVM_Op<"cvt.packfloat.f32"> { + let description = [{ + This Op converts two given F32 inputs (in `srcA` and `srcB`) to a + packed floating point type requested in `to`, and the + result is merged with the given `srcC`. + `extractHi`= 0: the result is packed into the low 16-bits of `srcC`. + (and the high-16 bits of `srcC` remain as is) + `extractHi`= 1: the result is packed into the high 16-bits of `srcC`. + (and the low-16 bits of `srcC` remain as is) + The result `res` is represented as an i32 type. + The `relu` attribute, when set, lowers to the '.relu' variant of + the cvt instruction. The `rnd` and `sat` attributes specify the + the rounding and saturation modes respectively. The ue8m0x2, + e2m3x2, e3m2x2 and e2m1x2 types are supported only in SM100a. + [For more information, see PTX ISA] + (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + NVVM_T32BitFloat:$srcA, + NVVM_T32BitFloat:$srcB, + NVVM_T32BitSignlessInteger:$srcC, + NVVM_CVTPackFloatKindAttr:$to, + OptionalAttr:$rnd, + OptionalAttr:$sat, + OptionalAttr:$relu, + OptionalAttr:$extractHi + ); + let results = (outs + NVVM_T32BitSignlessInteger2:$res + ); +} +def NVVM_CvtToF4x8PackedOp : NVVM_Op<"cvt.to.f4x8.packed"> { + let description = [{ + Fused conversion + packing: performs 4 pairwise conversions to f4 and + packs the 4 byte-sized results into a single i32 using inline PTX + `{ .reg .b8; 4x cvt; mov.b32 }`. + + This avoids the NVPTX backend's lack of a B8 register class, which + causes 3x `prmt.b32` instructions instead of a single `mov.b32` when + packing 4x i8 values via `BUILD_VECTOR v4i8`. + + For f32 source: takes 8 f32 values (4 high-low pairs). + For f16 source: takes 4 i32 values (packed f16x2). + For bf16 source: takes 4 i32 values (packed bf16x2). + + The `srcType` attribute specifies the source element type (f32, f16, + or bf16) and is used to select the correct PTX cvt instruction suffix. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) + }]; + let arguments = (ins + Variadic:$srcs, + OptionalAttr:$rnd, + TypeAttr:$srcType, + TypeAttr:$dstType + ); + let results = (outs + NVVM_T32BitSignlessInteger2:$dst + ); +} +def NVVM_DotAccumulate2WayOp : NVVM_Op<"dot.accumulate.2way"> { + let description = [{ + Performs a two-way 16-bit to 8-bit dot-product which is accumulated in a + 32-bit result. + Operand `a` is a vector of two 16-bit elements and operand `b` a vector + of four 8-bit elements between which the dot product is computed. + + The `a_type` and `b_type` attributes specify the type of the elements in `a` + and `b` respectively. + If `a_type` or `b_type` is `s`, then the elements in the corresponding + vector are sign-extended to 32-bit before the dot product is computed. + If `a_type` or `b_type` is `u`, then the elements in the corresponding + vector are zero-extended to 32-bit instead. + + The `b_hi` boolean attribute specifies which two bytes of `b` are used for + the dot product. If `b_hi` is true, then the dot product is computed + between `a` and elements at indices 2 and 3 of `b`. If `b_hi` is false, + then the dot product is computed between `a` and elements at indices 0 and + 1 of `b`. + + Operand `c` is a 32-bit integer to which the result is accumulated. It is + treated as holding a signed integer if any of `a_type` or `b_type` is + signed. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#integer-arithmetic-instructions-dp2a) + }]; + let arguments = (ins + NVVM_VectorOf16BitSignlessInteger:$a, + NVVM_VectorOf8BitSignlessInteger:$b, + NVVM_T32BitSignlessInteger:$c, + NVVM_DotAccumulateTypeAttr:$a_type, + NVVM_DotAccumulateTypeAttr:$b_type, + BoolAttr:$b_hi + ); + let results = (outs + NVVM_T32BitSignlessInteger2:$res + ); +} +def NVVM_DotAccumulate4WayOp : NVVM_Op<"dot.accumulate.4way"> { + let description = [{ + Performs a four-way byte dot-product which is accumulated in a 32-bit + result. + Operand `a` and `b` are vectors of 4 bytes between which the dot product is + computed. + + The `a_type` and `b_type` attributes specify the type of the elements in `a` + and `b` respectively. + If `a_type` or `b_type` is `signed`, then the elements in the corresponding + vector are sign-extended to 32-bit before the dot product is computed. + If `a_type` or `b_type` is `unsigned`, then the elements in the + corresponding vector are zero-extended to 32-bit instead. + + Operand `c` is a 32-bit integer to which the result is accumulated. It is + treated as holding a signed integer if any of `a_type` or `b_type` is `s8`. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#integer-arithmetic-instructions-dp4a) + }]; + let arguments = (ins + NVVM_VectorOf8BitSignlessInteger:$a, + NVVM_VectorOf8BitSignlessInteger:$b, + NVVM_T32BitSignlessInteger:$c, + NVVM_DotAccumulateTypeAttr:$a_type, + NVVM_DotAccumulateTypeAttr:$b_type + ); + let results = (outs + NVVM_T32BitSignlessInteger2:$res + ); +} +def NVVM_ElectSyncOp : NVVM_Op<"elect.sync"> { + let description = [{ + The `elect.sync` instruction elects one predicated active leader + thread from among a set of threads specified in the `membermask`. + When the `membermask` is not provided explicitly, a default value + of `0xFFFFFFFF` is used. The predicate result is set to `True` for + the leader thread, and `False` for all other threads. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-elect-sync) + }]; + let arguments = (ins + Optional:$membermask + ); + let results = (outs + NVVM_T1BitSignlessInteger2:$pred + ); +} +def NVVM_Ex2Op : NVVM_Op<"ex2", [Pure]> { + let description = [{ + Computes a fast approximation of 2 raised to the power of the input + value. The `ftz` attribute, when set, flushes subnormal inputs and + results to sign-preserving zero. + }]; + let arguments = (ins + NVVM_T32BitFloat:$src, + OptionalAttr:$ftz + ); + let results = (outs + NVVM_T32BitFloat2:$res + ); +} +def NVVM_Exit : NVVM_Op<"exit"> { + let description = [{ + Ends execution of a thread. + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#control-flow-instructions-exit) + }]; + let arguments = (ins); + let results = (outs); +} +def NVVM_FAbsOp : NVVM_InferredOp<"fabs"> { + let description = [{ + Returns the absolute value of the floating point operand. The Op also has an + optional `ftz` attribute which flushes subnormal inputs and returns to + sign-preserving zero + + [For more information, see PTX ISA] + (http://sw-mobile-docs/CUDA/GPGPU/parallel-thread-execution/index.html#floating-point-instructions-abs) + }]; + let arguments = (ins + NVVM_FloatingPointOrFixedLengthVector:$value, + OptionalAttr:$ftz + ); + let results = (outs + NVVM_FloatingPointOrFixedLengthVector:$res + ); +} +def NVVM_FenceMbarrierInitOp : NVVM_Op<"fence.mbarrier.init"> { + let description = [{ + Fence operation that applies on the prior nvvm.mbarrier.init + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-membar) + }]; + let arguments = (ins); + let results = (outs); +} +def NVVM_FenceProxyOp : NVVM_Op<"fence.proxy"> { + let description = [{ + Fence operation with proxy to establish an ordering between memory accesses + that may happen through different proxies. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-membar) + }]; + let arguments = (ins + NVVM_ProxyKindAttr:$kind, + OptionalAttr:$space + ); + let results = (outs); +} +def NVVM_FenceProxyAcquireOp : NVVM_Op<"fence.proxy.acquire"> { + let description = [{ + `fence.proxy.acquire` is a uni-directional fence used to establish ordering + between a prior memory access performed via the generic proxy and a + subsequent memory access performed via the tensormap proxy + + The address operand `addr` and the operand `size` together specify the + memory range `[addr, addr+size)` on which the ordering guarantees on the + memory accesses across the proxies is to be provided. The only supported + value for the `size` operand is 128 and must be an immediate. Generic Addressing + is used unconditionally, and the address specified by the operand `addr` must + fall within the `.global` state space. Otherwise, the behavior is undefined + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-membar) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace0:$addr, + NVVM_T32BitSignlessInteger:$size, + NVVM_MemScopeKindAttr:$scope, + OptionalAttr:$fromProxy, + OptionalAttr:$toProxy + ); + let results = (outs); +} +def NVVM_FenceProxyReleaseOp : NVVM_Op<"fence.proxy.release"> { + let description = [{ + `fence.proxy.release` is a uni-directional fence used to establish ordering + between a prior memory access performed via the generic proxy and a + subsequent memory access performed via the tensormap proxy. `fence.proxy.release` + operation can form a release sequence that synchronizes with an acquire + sequence that contains the fence.proxy.acquire proxy fence operation + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-membar) + }]; + let arguments = (ins + NVVM_MemScopeKindAttr:$scope, + OptionalAttr:$fromProxy, + OptionalAttr:$toProxy + ); + let results = (outs); +} +def NVVM_FenceProxySyncRestrictOp : NVVM_Op<"fence.proxy.sync_restrict"> { + let description = [{ + The `nvvm.fence.proxy.sync_restrict` Op used to establish + ordering between a prior memory access performed between proxies. Currently, + the ordering is only supported between async and generic proxies. `sync_restrict` + restricts `acquire` memory semantics to `shared_cluster` and `release` memory + semantics to `shared_cta` with cluster scope. + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-membar) + }]; + let arguments = (ins + NVVM_MemOrderKindAttr:$order, + OptionalAttr:$fromProxy, + OptionalAttr:$toProxy + ); + let results = (outs); +} +def NVVM_FenceScClusterOp : NVVM_Op<"fence.sc.cluster"> { + let arguments = (ins); + let results = (outs); +} +def NVVM_FenceSyncRestrictOp : NVVM_Op<"fence.sync_restrict"> { + let description = [{ + The `nvvm.fence.sync_restrict` Op restricts the class of memory + operations for which the fence instruction provides the memory ordering guarantees. + `sync_restrict` restricts `acquire` memory semantics to `shared_cluster` and + `release` memory semantics to `shared_cta` with cluster scope. + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-membar) + }]; + let arguments = (ins + NVVM_MemOrderKindAttr:$order + ); + let results = (outs); +} +def NVVM_FmaOp : NVVM_InferredOp<"fma"> { + let description = [{ + The `nvvm.fma` operation performs floating point fused multiply-add of + three operands of the same type. + + The rounding mode is specified by the `rnd` attribute, saturation mode by + the `sat` attribute, flush-to-zero by the `ftz` attribute, and ReLU by the + `relu` attribute. + + Out-of-bounds (OOB) behavior is controlled by the `oob` attribute. `oob` + clamps the result to 0 if either of the operands is `OOB NaN` (see [Tensors](https://docs.nvidia.com/cuda/parallel-thread-execution/#tensors)). + + For more information, see PTX ISA: + - [floating point fused multiply-add](https://docs.nvidia.com/cuda/parallel-thread-execution/#floating-point-instructions-fma) + - [half-precision floating point fused multiply-add](https://docs.nvidia.com/cuda/parallel-thread-execution/#half-precision-floating-point-instructions-fma) + }]; + let arguments = (ins + NVVM_T16BitFloatOrBfloat16Type:$a, + NVVM_T16BitFloatOrBfloat16Type:$b, + NVVM_T16BitFloatOrBfloat16Type:$c, + NVVM_FmaRndAttr:$rnd, + OptionalAttr:$sat, + OptionalAttr:$ftz, + OptionalAttr:$relu, + OptionalAttr:$oob + ); + let results = (outs + NVVM_T16BitFloatOrBfloat16Type:$res + ); +} +def NVVM_FMAPackedF32x2Op : NVVM_InferredOp<"fma.packed.f32x2"> { + let description = [{ + This Op performs FMA of floating-point vectors. + The inputs and outputs are of the form . The `rnd` attribute + specifies the rounding mode. The `ftz` attribute, when set, flushes the + subnormal inputs and results to sign-preserving zero. + This Op is supported from SM100 and onwards. + [For more information, see PTX ISA] + (TODO: Add public PTX link when available) + }]; + let arguments = (ins + NVVM_VectorOf32BitFloatValues2:$srcA, + NVVM_VectorOf32BitFloatValues2:$srcB, + NVVM_VectorOf32BitFloatValues2:$srcC, + OptionalAttr:$rnd, + OptionalAttr:$ftz + ); + let results = (outs + NVVM_VectorOf32BitFloatValues2:$res + ); +} +def NVVM_FMAPacked_F32x2_BF16x2_F32x2_F32x2Op : NVVM_Op<"fma.packed.f32x2.bf16x2.f32x2.f32x2"> { + let description = [{ + This Op performs FMA of a pair of floating-point values. + The first operand is a vector<2xbf16>, and the second and third operands + are vector<2xf32>. The result is a vector<2xf32>. The first input is converted to + a scalar of packed i32 where the 0th element of the vector forms the low-16 bits + and the 1st element occupies the high-16 bits. The second and third inputs are + converted to a scalar of packed i64 where the 0th element of the vector forms the + low-32 bits and the 1st element occupies the high-32 bits. + + The rounding mode can be one of the following: + - rn: round to nearest even + - rz: round toward zero + - rm: round toward negative infinity + - rp: round toward positive infinity + + Examples: + ```mlir + // Round to nearest even (default) + %0 = nvvm.fma.packed.f32x2.bf16x2.f32x2.f32x2 %srcA, %srcB, %srcC : vector<2xbf16>, vector<2xf32>, vector<2xf32> -> vector<2xf32> + + // Round toward negative infinity + %1 = nvvm.fma.packed.f32x2.bf16x2.f32x2.f32x2 %srcA, %srcB, %srcC {rnd = #nvvm.fp_rnd_mode} : vector<2xbf16>, vector<2xf32>, vector<2xf32> -> vector<2xf32> + ``` + }]; + let arguments = (ins + NVVM_VectorOfBfloat16TypeValuesOf:$srcA, + NVVM_VectorOf32BitFloatValues:$srcB, + NVVM_VectorOf32BitFloatValues:$srcC, + OptionalAttr:$rnd + ); + let results = (outs + NVVM_VectorOf32BitFloatValues:$res + ); +} +def NVVM_FMAPacked_F32x2_F16x2_F32x2_F32x2Op : NVVM_Op<"fma.packed.f32x2.f16x2.f32x2.f32x2"> { + let description = [{ + This Op performs FMA of a pair of floating-point values. + The first operand is a vector<2xf16>, and the second and third operands + are vector<2xf32>. The result is a vector<2xf32>. The first input is converted to + a scalar of packed i32 where the 0th element of the vector forms the low-16 bits + and the 1st element occupies the high-16 bits. The second and third inputs are + converted to a scalar of packed i64 where the 0th element of the vector forms the + low-32 bits and the 1st element occupies the high-32 bits. + + The rounding mode can be one of the following: + - rn: round to nearest even + - rz: round toward zero + - rm: round toward negative infinity + - rp: round toward positive infinity + + Examples: + ```mlir + // Round to nearest even (default) + %0 = nvvm.fma.packed.f32x2.f16x2.f32x2.f32x2 %srcA, %srcB, %srcC : vector<2xf16>, vector<2xf32>, vector<2xf32> -> vector<2xf32> + + // Round toward negative infinity + %1 = nvvm.fma.packed.f32x2.f16x2.f32x2.f32x2 %srcA, %srcB, %srcC {rnd = #nvvm.fp_rnd_mode} : vector<2xf16>, vector<2xf32>, vector<2xf32> -> vector<2xf32> + ``` + }]; + let arguments = (ins + NVVM_VectorOf16BitFloatValues:$srcA, + NVVM_VectorOf32BitFloatValues:$srcB, + NVVM_VectorOf32BitFloatValues:$srcC, + OptionalAttr:$rnd + ); + let results = (outs + NVVM_VectorOf32BitFloatValues:$res + ); +} +def NVVM_MaxOp : NVVM_Op<"fmax"> { + let description = [{ + `nvvm.fmax` Op calculates maximum of given values. if `nan` attributes is + used, then the result is a canonical nan if any of the input is nan. + if `abs` attribute is used, the magnitude of destination operation d is + the maximum of absolute values of input arguments and for two operand + version, `xorsign` is enabled by default which sets the sign bit of the + result by XOR'ing the sign bits of the inputs. `ftz` modifiers flushes + the subnormal inputs and the results to sign preserving zero + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#floating-point-instructions-max) + }]; + let arguments = (ins + NVVM_T32BitFloat:$a, + NVVM_T32BitFloat:$b, + Optional:$c, + UnitAttr:$ftz, + UnitAttr:$nan, + UnitAttr:$abs + ); + let results = (outs + NVVM_T32BitFloat2:$res + ); +} +def NVVM_MinOp : NVVM_Op<"fmin"> { + let description = [{ + `nvvm.fmin` Op calculates minimum of given values. if `nan` attributes is + used, then the result is a canonical nan if any of the input is nan. + if `abs` attribute is used, the magnitude of destination operation d is + the minimum of absolute values of input arguments and for two operand + version, `xorsign` is enabled by default which sets the sign bit of the + result by XOR'ing the sign bits of the inputs. `ftz` modifiers flushes + the subnormal inputs and the results to sign preserving zero. + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#floating-point-instructions-min) + }]; + let arguments = (ins + NVVM_T32BitFloat:$a, + NVVM_T32BitFloat:$b, + Optional:$c, + UnitAttr:$ftz, + UnitAttr:$nan, + UnitAttr:$abs + ); + let results = (outs + NVVM_T32BitFloat2:$res + ); +} +def NVVM_GriddepcontrolOp : NVVM_Op<"griddepcontrol"> { + let description = [{ + If the $kind attribute is set to `wait`, it causes the + executing thread to wait until all prerequisite grids in flight + have completed and all the memory operations from the prerequisite grids + are performed and made visible to the current grid. + + When the $kind is launch_dependents, it signals that specific dependents + the runtime system designated to react to this instruction can be scheduled + as soon as all other CTAs in the grid issue the same instruction or have + completed. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-griddepcontrol) + }]; + let arguments = (ins + NVVM_GridDepActionAttr:$kind + ); + let results = (outs); +} +def NVVM_InlinePtxOp : NVVM_Op<"inline_ptx", [AttrSizedOperandSegments]> { + let description = [{ + This op allows using PTX directly within the NVVM + dialect, while greatly simplifying llvm.inline_asm generation. It + automatically handles register size selection and sets the correct + read/write access for each operand. The operation leverages the + `BasicPtxBuilderInterface` to abstract away low-level details of + PTX assembly formatting. + + The `predicate` attribute is used to specify a predicate for the + PTX instruction. + + Example 1: Read-only Parameters + ```mlir + nvvm.inline_ptx "mbarrier.init.b64 [$0], $1;" (%barrier_gen, %count) : !llvm.ptr, i32 + + // Lowers to: + llvm.inline_asm has_side_effects asm_dialect = att + "mbarrier.init.b64 [$0], $1;", "l,r" %arg0, %arg2 : (!llvm.ptr, i32) -> () + ``` + + Example 2: Read-only and Write-only Parameters + ```mlir + %0 = nvvm.inline_ptx "ex2.approx.ftz.f32 $0, $1;" (%input) : f32 -> f32 + + // Lowers to: + %0 = llvm.inline_asm has_side_effects asm_dialect = att + "ex2.approx.ftz.f32 $0, $1;", "=f,f" %arg0 : (f32) -> f32 + ``` + + Example 3: Predicate Usage + ```mlir + nvvm.inline_ptx "mbarrier.init.b64 [$0], $1;" (%barrier_gen, %count), + predicate = %pred : !llvm.ptr, i32, i1 + + // Lowers to: + llvm.inline_asm has_side_effects asm_dialect = att + "@$2 mbarrier.init.b64 [$0], $1;", "l,r,b" %arg0, %arg2, %arg3 + : (!llvm.ptr, i32, i1) -> () + ``` + }]; + let arguments = (ins + Variadic:$readOnlyArgs, + Variadic:$readWriteArgs, + Optional:$predicate, + StrAttr:$ptxCode + ); + let results = (outs + Optional:$writeOnlyArgs + ); +} +def NVVM_LdMatrixOp : NVVM_InferredOp<"ldmatrix"> { + let arguments = (ins + NVVM_LLVMPointerType:$ptr, + I32Attr:$num, + NVVM_MMALayoutAttr:$layout, + OptionalAttr:$shape, + OptionalAttr:$srcFormat + ); + let results = (outs + NVVM_AnyType:$res + ); +} +def NVVM_ExtLoadOp : NVVM_Op<"load.ext"> { + let description = [{ + The `load.ext` operation is used to read from memory in generic, + global or shared address space. This Op explicitly supports + various cache properties and memory orderings associated with + the load operation. + + The `l2CacheHint` operand is optional, and it is used to specify cache + eviction policy that may be used during the memory access. + [For more information, see PTX ISA] + (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-ld) + }]; + let arguments = (ins + NVVM_LLVMPointerType:$addr, + Optional:$l2CacheHint, + OptionalAttr:$order, + OptionalAttr:$scope, + OptionalAttr:$prefetch, + OptionalAttr:$evict, + OptionalAttr:$cacheModifier, + OptionalAttr:$sharedSpace, + OptionalAttr:$unified + ); + let results = (outs + NVVM_LLVMTypeWithSize:$res + ); +} +def NVVM_Log2Op : NVVM_Op<"log2", [Pure]> { + let description = [{ + Computes a fast approximation of the base-2 logarithm of the input + value. The `ftz` attribute, when set, flushes subnormal inputs and + results to sign-preserving zero. + }]; + let arguments = (ins + NVVM_T32BitFloat:$src, + OptionalAttr:$ftz + ); + let results = (outs + NVVM_T32BitFloat2:$res + ); +} +def NVVM_MapaOp : NVVM_Op<"mapa"> { + let arguments = (ins + NVVM_LLVMPointerInAddressSpace02:$a, + NVVM_T32BitSignlessInteger:$b + ); + let results = (outs + NVVM_LLVMPointerInAddressSpace03:$res + ); +} +def NVVM_MatchSyncOp : NVVM_InferredOp<"match.sync"> { + let description = [{ + The `match.sync` op performs broadcast and compare of operand `val` across + all non-exited threads in `thread_mask` and returns a mask depending on the + kind and an optional predicate. + + The matching operation kinds are: + - `any`: Returns a mask corresponding to the non-exited threads in the + `thread_mask` that have the same value of operand `val`. + - `all`: Returns a mask and a predicate. If all non-exited threads in the + `thread_mask` have the same value of operand `val`, the predicate is set to + true and the mask corresponds to the non-exited threads in the + `thread_mask`. Otherwise, the predicate is set to false and the mask is 0. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-match-sync) + }]; + let arguments = (ins + NVVM_T32BitSignlessInteger:$thread_mask, + NVVM_T32BitSignlessIntegerOr64:$val, + NVVM_MatchSyncKindAttr:$kind + ); + let results = (outs + NVVM_T32BitSignlessIntegerOrLLVM:$res + ); +} +def NVVM_MBarrierArriveOp : NVVM_InferredOp<"mbarrier.arrive", [AttrSizedOperandSegments]> { + let description = [{ + The `nvvm.mbarrier.arrive` operation performs an arrive-on operation on the + *mbarrier object* at the specified address. Uses the default `.release.cta` semantics. + This release pattern establishes memory ordering for operations occurring in program + order before this arrive instruction by making operations from the current thread + visible to subsequent operations in other threads within the CTA. When other threads + perform corresponding acquire operations (like 'mbarrier.test.wait'), they synchronize + with this release pattern. + + This operation causes the executing thread to signal its arrival at the barrier. + + - `res`: When the `space` is not shared_cluster, this operation returns an + opaque 64-bit value capturing the phase of the *mbarrier object* prior to + the arrive-on operation. The contents of this return value are + implementation-specific. An *mbarrier object* located in the shared_cluster + space cannot return a value. + + The operation takes the following operands: + - `addr`: A pointer to the memory location of the *mbarrier object*. The `addr` + must be a pointer to generic or shared_cta or shared_cluster memory. When it + is generic, the underlying address must be within the shared_cta memory space; + otherwise the behavior is undefined. + - `count`: This specifies the amount by which the pending arrival count is + decremented. If the `count` argument is not specified, the pending arrival + count is decremented by 1. + - `scope`: This specifies the set of threads that directly observe the memory + synchronizing effect of the `mbarrier.arrive` operation. + - `space`: This indicates the memory space where the mbarrier object resides. + - `relaxed`: When set to true, the `arrive` operation has relaxed memory semantics + and does not provide any ordering or visibility guarantees. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-arrive) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace04:$addr, + Optional:$count, + Optional:$multicastMask, + OptionalAttr:$scope, + OptionalAttr:$relaxed + ); + let results = (outs + Optional:$res + ); +} +def NVVM_MBarrierArriveExpectTxOp : NVVM_InferredOp<"mbarrier.arrive.expect_tx", [AttrSizedOperandSegments]> { + let description = [{ + The `nvvm.mbarrier.arrive.expect_tx` operation performs an expect-tx operation + followed by an arrive-on operation on the *mbarrier object*. Uses the default + `.release.cta` semantics. This release pattern establishes memory ordering for + operations occurring in program order before this arrive instruction by making + operations from the current thread visible to subsequent operations in other + threads within the CTA. When other threads perform corresponding acquire operations + (like 'mbarrier.test.wait'), they synchronize with this release pattern. + + This operation first performs an expect-tx operation with the specified transaction + count, then performs an arrive-on operation with an implicit count of 1. The + expect-tx operation increases the expect-count of the *mbarrier object* by the + specified value (i.e. `txcount`), setting the current phase to expect and track + the completion of additional asynchronous transactions. + + The operation takes the following operands: + - `addr`: A pointer to the memory location of the *mbarrier object*. Uses generic + addressing, but the address must still be in the shared memory space. + - `txcount`: An unsigned integer specifying the expected transaction count + for the expect-tx operation. This represents the number of asynchronous transactions + expected to complete before the barrier phase completes. + - `scope`: This specifies the set of threads that directly observe the memory + synchronizing effect of the `mbarrier.test.wait` operation. + - `relaxed`: When set to true, the `arrive` operation has relaxed memory semantics + and does not provide any ordering or visibility guarantees. + - `predicate`: Optional predicate for conditional execution used only when lowering to + inline-ptx. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-arrive-drop) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace04:$addr, + NVVM_T32BitSignlessInteger:$txcount, + Optional:$multicastMask, + Optional:$predicate, + OptionalAttr:$scope, + OptionalAttr:$relaxed + ); + let results = (outs + Optional:$res + ); +} +def NVVM_MBarrierArriveNocompleteOp : NVVM_Op<"mbarrier.arrive.nocomplete"> { + let description = [{ + The `nvvm.mbarrier.arrive.nocomplete` operation performs an arrive-on operation + on the *mbarrier object* with the guarantee that it will not cause the barrier to + complete its current phase. Uses the default `.release.cta` semantics. This release + pattern establishes memory ordering for operations occurring in program order before + this arrive instruction by making operations from the current thread visible to + subsequent operations in other threads within the CTA. When other threads perform + corresponding acquire operations (like 'mbarrier.test.wait'), they synchronize with + this release pattern. + + This operation causes the executing thread to signal its arrival at the barrier + with a specified count, but ensures that the barrier phase will not complete as + a result of this operation. The operation returns an opaque value that + captures the phase of the *mbarrier object* prior to the arrive-on operation. + + The operation takes the following operands: + - `addr`: A pointer to the memory location of the *mbarrier object*. The `addr` + must be a pointer to generic or shared::cta memory. When it is generic, the + underlying address must be within the shared::cta memory space; otherwise + the behavior is undefined. + - `count`: Integer specifying the count argument to the arrive-on operation. + Must be in the valid range as specified in the *mbarrier object* contents. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-arrive) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace02:$addr, + NVVM_T32BitSignlessInteger:$count + ); + let results = (outs + NVVM_T64BitSignlessInteger2:$res + ); +} +def NVVM_MBarrierArriveDropOp : NVVM_InferredOp<"mbarrier.arrive_drop", [AttrSizedOperandSegments]> { + let description = [{ + The `nvvm.mbarrier.arrive_drop` operation decrements the expected arrival + count of the *mbarrier object* by `count` and then performs an arrive-on + operation. When `count` is not specified, it defaults to 1. The decrement + of the expected arrival count applies to all the subsequent phases of the + *mbarrier object*. The remaining semantics are identical to those of the + `nvvm.mbarrier.arrive` operation. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-arrive-drop) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace04:$addr, + Optional:$count, + Optional:$multicastMask, + OptionalAttr:$scope, + OptionalAttr:$relaxed + ); + let results = (outs + Optional:$res + ); +} +def NVVM_MBarrierArriveDropExpectTxOp : NVVM_InferredOp<"mbarrier.arrive_drop.expect_tx"> { + let description = [{ + The `nvvm.mbarrier.arrive_drop.expect_tx` operation is similar to the + `nvvm.mbarrier.arrive.expect_tx` operation except that it performs an + `arrive_drop` operation instead of only an `arrive` operation. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-arrive-drop) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace04:$addr, + NVVM_T32BitSignlessInteger:$txcount, + Optional:$multicastMask, + OptionalAttr:$scope, + OptionalAttr:$relaxed + ); + let results = (outs + Optional:$res + ); +} +def NVVM_MBarrierArriveDropNocompleteOp : NVVM_Op<"mbarrier.arrive_drop.nocomplete"> { + let description = [{ + The `nvvm.mbarrier.arrive_drop.nocomplete` operation decrements the expected + arrival count of the *mbarrier object* by the amount `count` and then performs + an arrive-on operation on the *mbarrier object* with the guarantee that it + will not cause the barrier to complete its current phase. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-arrive-drop) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace02:$addr, + NVVM_T32BitSignlessInteger:$count + ); + let results = (outs + NVVM_T64BitSignlessInteger2:$res + ); +} +def NVVM_MBarrierCheckLayoutOp : NVVM_Op<"mbarrier.check.layout"> { + let description = [{ + The `nvvm.mbarrier.check.layout` operation returns `true` if the *mbarrier object* at `addr` + uses the given `layout`; otherwise it returns `false`. + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace3:$addr, + NVVM_MBarrierLayoutAttr:$layout + ); + let results = (outs + NVVM_T1BitSignlessInteger2:$res + ); +} +def NVVM_MBarrierCompleteTxOp : NVVM_Op<"mbarrier.complete_tx", [AttrSizedOperandSegments]> { + let description = [{ + The `nvvm.mbarrier.complete_tx` operation decrements the transaction + count of the *mbarrier object* at `addr` by `txcount`. It also signals + the completion of asynchronous transactions that were tracked by the + current phase. The `scope` specifies the set of threads that can directly + observe the memory synchronizing effect of the `mbarrier.complete_tx` + operation. `CTA` and `CLUSTER` are the only allowed values for `scope`. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-complete-tx) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace32:$addr, + NVVM_T32BitSignlessInteger:$txcount, + Optional:$report, + Optional:$multicastMask, + OptionalAttr:$scope + ); + let results = (outs); +} +def NVVM_MBarrierExpectTxOp : NVVM_Op<"mbarrier.expect_tx"> { + let description = [{ + The `nvvm.mbarrier.expect_tx` operation increases the transaction count + of the mbarrier located at `addr` by `txcount` amount. The `scope` + specifies the set of threads that can directly observe the memory + synchronizing effect of the `mbarrier.expect_tx` operation. `CTA` + and `CLUSTER` are the only allowed values for `scope`. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-expect-tx) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace32:$addr, + NVVM_T32BitSignlessInteger:$txcount, + Optional:$multicastMask, + OptionalAttr:$scope + ); + let results = (outs); +} +def NVVM_MBarrierInitOp : NVVM_Op<"mbarrier.init"> { + let description = [{ + The `nvvm.mbarrier.init` operation initializes an *mbarrier object* at the specified + memory location. + + This operation initializes the *mbarrier object* with the following state: + - Current phase: 0 + - Expected arrival count: `count` + - Pending arrival count: `count` + - Transaction count (tx-count): 0 + + The operation takes the following operands: + - `addr`: A pointer to the memory location of the *mbarrier object*. The `addr` + must be a pointer to generic or shared::cta memory. When it is generic, the + underlying address must be within the shared::cta memory space; otherwise + the behavior is undefined. + - `count`: Integer specifying the number of threads that will participate in barrier + synchronization. Must be in the range [1, 2²⁰ - 1]. + - `predicate`: Optional predicate for conditional execution. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-init) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace02:$addr, + NVVM_T32BitSignlessInteger:$count, + Optional:$predicate, + OptionalAttr:$layout + ); + let results = (outs); +} +def NVVM_MBarrierInvalOp : NVVM_Op<"mbarrier.inval"> { + let description = [{ + The `nvvm.mbarrier.inval` operation invalidates an *mbarrier object* at the + specified memory location. + + This operation marks the *mbarrier object* as invalid, making it safe to repurpose + the memory location for other uses or to reinitialize it as a new *mbarrier object*. + It is undefined behavior if the *mbarrier object* is already invalid. + + The operation takes the following operand: + - `addr`: A pointer to the memory location of the *mbarrier object*. The `addr` + must be a pointer to generic or shared::cta memory. When it is generic, the + underlying address must be within the shared::cta memory space; otherwise + the behavior is undefined. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-inval) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace02:$addr + ); + let results = (outs); +} +def NVVM_MBarrierTestWaitOp : NVVM_InferredOp<"mbarrier.test.wait"> { + let description = [{ + The `nvvm.mbarrier.test.wait` operation performs a non-blocking test for the + completion of a specific phase of an *mbarrier object*. It uses the default + `.acquire.cta` semantics. This acquire pattern establishes memory ordering for + operations occurring in program order after this wait instruction by making + operations from other threads in the CTA visible to subsequent operations in the current + thread. When this wait completes, it synchronizes with the corresponding release + pattern from the `mbarrier.arrive` operation, establishing memory ordering within + the CTA. + + This operation tests whether the mbarrier phase specified by the state operand + has completed. It is a non-blocking instruction that immediately returns the + completion status without suspending the executing thread. + + The operation takes the following operands: + - `addr`: A pointer to the memory location of the *mbarrier object*. Uses generic + addressing, but the address must still be in the shared memory space. + - `stateOrPhase`: This argument represents a `state` when it is a 64-bit value + and represents a `phase` when it is a 32-bit value. The `state` is an opaque + value returned by a previous `mbarrier.arrive` operation on the same + *mbarrier object* during the current or immediately preceding phase. + The `phase` is an integer specifying the phase parity (0 or 1). + Even phases have parity 0, odd phases have parity 1. + - `scope`: This specifies the set of threads that directly observe the memory + synchronizing effect of the `mbarrier.test.wait` operation. + - `relaxed`: When set to true, the `arrive` operation has relaxed memory semantics + and does not provide any ordering or visibility guarantees. + + The operation returns a boolean value indicating whether the specified phase + has completed: + - `true`: The immediately preceding phase has completed + - `false`: The phase is still incomplete (current phase) + + The primary-phase variants the result is an LLVM struct whose first + field (`res`) is the `i1` completion flag. An optional second field + (`isReportSeen`, `i1`) indicates whether a report was observed, and an + optional third field (`report`, `i32`) carries the report payload. + + **Memory ordering guarantees**: When this wait returns true, the following + ordering guarantees hold: + + 1. All memory accesses (except async operations) requested prior to + `mbarrier.arrive` having release semantics by participating CTA threads + are visible to the executing thread. + 2. All `cp.async` operations requested prior to `cp.async.mbarrier.arrive` + by participating CTA threads are visible to the executing thread. + 3. All `cp.async.bulk` operations using the same *mbarrier object* requested + prior to `mbarrier.arrive` having release semantics by participating CTA + threads are visible to the executing thread. + 4. Memory accesses requested after this wait are not visible to memory + accesses performed prior to `mbarrier.arrive` by other participating + threads. + 5. No ordering guarantee exists for memory accesses by the same thread + between `mbarrier.arrive` and this wait. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-try-wait) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace02:$addr, + NVVM_T64BitSignlessIntegerOr32:$stateOrPhase, + OptionalAttr:$scope, + OptionalAttr:$relaxed, + OptionalAttr:$mbarrierPhase + ); + let results = (outs + NVVM_T1BitSignlessIntegerOrLLVM:$res + ); +} +def NVVM_MBarrierTryWaitOp : NVVM_InferredOp<"mbarrier.try_wait"> { + let description = [{ + The `nvvm.mbarrier.try_wait` operation checks whether the specified + *mbarrier object* at `addr` has completed the given phase. Note that + unlike the `nvvm.mbarrier.test.wait` operation, the try_wait operation + is a potentially-blocking one. If the phase is not yet complete, the + calling thread may be suspended. A suspended thread resumes execution + once the phase completes or when a system-defined timeout occurs. + Optionally, the `ticks` operand can be used to provide a custom timeout + (in nanoseconds), overriding the system-defined one. The semantics of + this operation and its operands are otherwise similar to those of the + `nvvm.mbarrier.test.wait` Op. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-try-wait) + + The primary-phase variants the result is an LLVM struct whose first + field (`res`) is the `i1` completion flag. An optional second field + (`isReportSeen`, `i1`) indicates whether a report was observed, and an + optional third field (`report`, `i32`) carries the report payload. + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace02:$addr, + NVVM_T64BitSignlessIntegerOr32:$stateOrPhase, + Optional:$ticks, + OptionalAttr:$scope, + OptionalAttr:$relaxed, + OptionalAttr:$mbarrierPhase + ); + let results = (outs + NVVM_T1BitSignlessIntegerOrLLVM:$res + ); +} +def NVVM_MBarrierTryWaitParityOp : NVVM_Op<"mbarrier.try_wait.parity"> { + let description = [{ + The `nvvm.mbarrier.try_wait.parity` operation performs a potentially-blocking + test for the completion of a specific phase of an *mbarrier object* using phase + parity. It uses the default `.acquire.cta` semantics. This acquire pattern + establishes memory ordering for operations occurring in program order after this + wait instruction by making operations from other threads in the CTA visible to subsequent + operations in the current thread. When this wait completes, it synchronizes with + the corresponding release pattern from the `mbarrier.arrive` operation, establishing + memory ordering within the CTA. + + This operation waits for the completion of the mbarrier phase indicated by the + phase parity. While it uses the underlying PTX `mbarrier.try_wait.parity` + instruction, this MLIR operation generates a loop that enforces the test to + complete before continuing execution, ensuring the barrier phase is actually + completed rather than potentially timing out. + + The operation takes the following operands: + - `addr`: A pointer to the memory location of the *mbarrier object*. Uses generic + addressing, but the address must still be in the shared memory space. + - `phase`: An integer specifying the phase parity (0 or 1). Even phases + have parity 0, odd phases have parity 1. + - `ticks`: An unsigned integer specifying the suspend time hint in + nanoseconds. This may be used instead of the system-dependent time limit. + + **Memory ordering guarantees**: When this wait returns true, the following + ordering guarantees hold: + + 1. All memory accesses (except async operations) requested prior to + `mbarrier.arrive` having release semantics by participating CTA threads + are visible to the executing thread. + 2. All `cp.async` operations requested prior to `cp.async.mbarrier.arrive` + by participating CTA threads are visible to the executing thread. + 3. All `cp.async.bulk` operations using the same *mbarrier object* requested + prior to `mbarrier.arrive` having release semantics by participating CTA + threads are visible to the executing thread. + 4. Memory accesses requested after this wait are not visible to memory + accesses performed prior to `mbarrier.arrive` by other participating + threads. + 5. No ordering guarantee exists for memory accesses by the same thread + between `mbarrier.arrive` and this wait. + + **Implementation behavior**: + This operation generates a PTX loop that repeatedly calls the underlying + `mbarrier.try_wait.parity` instruction until the barrier phase completes. + Unlike the raw PTX instruction which may return without completion after a + timeout, this MLIR operation guarantees completion by continuing to loop until + the specified phase is reached. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-try-wait) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace02:$addr, + NVVM_T32BitSignlessInteger:$phase, + NVVM_T32BitSignlessInteger:$ticks, + OptionalAttr:$useIntrinsic + ); + let results = (outs); +} +def NVVM_MBarrierTryWaitParityTimeLimitOp : NVVM_Op<"mbarrier.try_wait.parity.timelimit"> { + let description = [{ + The `mbarrier.try_wait.parity.timelimit` is a potentially blocking + instruction which tests for the completion of the phase. If + the phase is not complete, the executing thread may be suspended. + Suspended thread resumes execution when the specified phase completes + or a system-dependent time limit elapses. The 32-bit `timeLimit` operand + specifies the time limit, in nanoseconds, that may be used instead of + the system-dependent limit. The `addr` operand specifies the address + of the mbarrier in generic or shared memory space. The `phase` operand + is the integer parity of either the current phase or the immediately + preceding phase of the mbarrier object. The `order` denotes the + ordering semantics for the memory consistency model. This Op only + supports 'acquire' or 'relaxed' ordering. + [For more information refer PTX ISA] + (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-mbarrier-try-wait) + }]; + let arguments = (ins + NVVM_LLVMPointerTypeOrLLVMPointer:$addr, + NVVM_T32BitSignlessInteger:$phase, + NVVM_T32BitSignlessInteger:$timeLimit, + OptionalAttr:$scope, + OptionalAttr:$order + ); + let results = (outs + NVVM_T1BitSignlessInteger2:$res + ); +} +def NVVM_MBarrierTryWaitTimeLimitOp : NVVM_Op<"mbarrier.try_wait.timelimit"> { + let description = [{ + The `mbarrier.try_wait.timelimit` is a potentially blocking + instruction which tests for the completion of the phase. If + the phase is not complete, the executing thread may be suspended. + Suspended thread resumes execution when the specified phase completes + or a system-dependent time limit elapses. The 32-bit `timeLimit` operand + specifies the time limit, in nanoseconds, that may be used instead of + the system-dependent limit. The `addr` operand specifies the address + of the mbarrier in generic or shared memory space. The `state` operand + is the captured state of the mbarrier object from prior operations. + The `order` denotes the ordering semantics for the memory consistency + model. This Op only supports 'acquire' or 'relaxed' ordering. + [For more information refer PTX ISA] + (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-mbarrier-try-wait) + }]; + let arguments = (ins + NVVM_LLVMPointerTypeOrLLVMPointer:$addr, + NVVM_T64BitSignlessInteger:$state, + NVVM_T32BitSignlessInteger:$timeLimit, + OptionalAttr:$scope, + OptionalAttr:$order + ); + let results = (outs + NVVM_T1BitSignlessInteger2:$res + ); +} +def NVVM_MBarrierTransactionOp : NVVM_Op<"mbarrier.txn"> { + let description = [{ + The `mbarrier.txn` Op exposes the transactions that can be done on an MBarrier object. + + Operands: + - `addr`: Address of the mbarrier in generic or shared memory space + - `count`: Transaction count for the given kind + + - `multicast`: (Optional) Multicast mask for cluster space operations + + + Attributes: + - `kind`: One of the transaction kinds from MBarrierTxnKind + - `space`: Memory space (CTA or CLUSTER) where MBarrier object is located + - `scope`: Scope of synchronization (CTA or CLUSTER) + - `order`: Memory ordering semantics (release or relaxed only) + + This Op supports MBarrier objects in both shared::CTA and shared::CLUSTER spaces. + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-mbarrier) + }]; + let arguments = (ins + NVVM_LLVMPointerTypeOrLLVMPointer:$addr, + NVVM_T32BitSignlessInteger:$count, + Optional:$multicast, + NVVM_MBarrierTxnKindAttr:$kind, + OptionalAttr:$space, + OptionalAttr:$scope, + OptionalAttr:$order + ); + let results = (outs); +} +def NVVM_MBarrierTransactionCTASpaceOp : NVVM_Op<"mbarrier.txn.cta"> { + let description = [{ + The `mbarrier.txn.cta` Op exposes the transactions that can be done + using an MBarrier object. The `addr` operand specifies the address of + the mbarrier in generic or shared memory space. The `kind` is one of + the various transaction kinds supported in `MBarrierTxnKind`. The + `count` argument is the transaction count for a given `kind`. The + `order` denotes the ordering semantics for the memory consistency + model. This Op only supports 'release' or 'relaxed' ordering. This + Op supports only MBarrier objects located in shared::CTA space (but + not in shared::CLUSTER space) and hence the 'CTASpace' suffix in the + naming. This Op returns an opaque 64-bit value capturing the phase of + the mbarrier object prior to the transaction. + [For more information on 'order/space/scope/kind-of-transactions' refer PTX ISA] + (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier) + }]; + let arguments = (ins + NVVM_LLVMPointerTypeOrLLVMPointer:$addr, + NVVM_T32BitSignlessInteger:$count, + NVVM_MBarrierTxnKindAttr:$kind, + OptionalAttr:$scope, + OptionalAttr:$order + ); + let results = (outs + NVVM_T64BitSignlessInteger2:$res + ); +} +def NVVM_MBarrierWaitOp : NVVM_Op<"mbarrier.wait"> { + let description = [{ + The `mbarrier.wait` Op exposes the try/test wait operations that + can be performed on MBarrier object. The `addr` operand specifies + the address of the mbarrier in generic or shared memory space. + The `state` operand is the captured state of the mbarrier object + from prior operations. The `kind` is one of TEST_WAIT or TRY_WAIT. + The `order` denotes the ordering semantics for the memory consistency + model. This Op only supports 'acquire' or 'relaxed' ordering. + [For more information refer PTX ISA] + (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-mbarrier-try-wait) + }]; + let arguments = (ins + NVVM_LLVMPointerTypeOrLLVMPointer:$addr, + NVVM_T64BitSignlessInteger:$state, + NVVM_MBarrierWaitKindAttr:$kind, + OptionalAttr:$scope, + OptionalAttr:$order + ); + let results = (outs + NVVM_T1BitSignlessInteger2:$res + ); +} +def NVVM_MBarrierWaitParityOp : NVVM_Op<"mbarrier.wait.parity"> { + let description = [{ + The `mbarrier.wait` Op exposes the try/test wait operations that + can be performed on MBarrier object. The `addr` operand specifies + the address of the mbarrier in generic or shared memory space. + The `phase` operand is the integer parity of either the current + phase or the immediately preceding phase of the mbarrier object. + The `kind` is one of TEST_WAIT or TRY_WAIT. + The `order` denotes the ordering semantics for the memory consistency + model. This Op only supports 'acquire' or 'relaxed' ordering. + [For more information refer PTX ISA] + (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-mbarrier-try-wait) + }]; + let arguments = (ins + NVVM_LLVMPointerTypeOrLLVMPointer:$addr, + NVVM_T32BitSignlessInteger:$phase, + NVVM_MBarrierWaitKindAttr:$kind, + OptionalAttr:$scope, + OptionalAttr:$order + ); + let results = (outs + NVVM_T1BitSignlessInteger2:$res + ); +} +def NVVM_MembarOp : NVVM_Op<"memory.barrier"> { + let description = [{ + `membar` operation guarantees that prior memory accesses requested by this + thread are performed at the specified `scope`, before later memory + operations requested by this thread following the membar instruction. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-membar) + }]; + let arguments = (ins + NVVM_MemScopeKindAttr:$scope + ); + let results = (outs); +} +def NVVM_MmaBlockScaleOp : NVVM_Op<"mma.block_scale", [AttrSizedOperandSegments]> { + let description = [{ + The `nvvm.mma.block_scale` operation collectively performs the operation + `D = matmul(A * SF_A, B * SF_B) + C` using all threads in a warp. + + A, B, C and D are dense matrices and SF_A and SF_B are scaling factors. + Dimensions of SF_A and SF_B are based on scale vector sizes (x1, x2, x4), + and the data type must be either ue8m0 or ue4m3. + + All the threads in the warp must execute the same `mma.block_scale` operation. + + This operation follows the same design pattern as `nvvm.mma.sync`, with additional + scaling operands for both A and B matrices. + + Example: + ```mlir + %d = nvvm.mma.block_scale A[%a0, %a1] B[%b0, %b1] C[%c0, %c1] + scaleA[%scaleAData, %byteIdA, %threadIdA] + scaleB[%scaleBData, %byteIdB, %threadIdB] + {shape = #nvvm.shape, + multiplicandAPtxType = #nvvm.mma_type, + multiplicandBPtxType = #nvvm.mma_type, + scaleVecSize = #nvvm.scale_vec_size, + blockScaleFormat = #nvvm.block_scale_format, + kind = #nvvm.block_scale_kind} + : (vector<4xf16>, vector<2xf16>, vector<2xf32>) -> !llvm.struct<(f32, f32)> + ``` + }]; + let arguments = (ins + Variadic:$operandA, + Variadic:$operandB, + Variadic:$operandC, + NVVM_T32BitSignlessInteger:$scaleAData, + NVVM_T16BitSignlessInteger:$byteIdA, + NVVM_T16BitSignlessInteger:$threadIdA, + NVVM_T32BitSignlessInteger:$scaleBData, + NVVM_T16BitSignlessInteger:$byteIdB, + NVVM_T16BitSignlessInteger:$threadIdB, + NVVM_MMAShapeAttr:$shape, + OptionalAttr:$multiplicandAPtxType, + OptionalAttr:$multiplicandBPtxType, + NVVM_ScaleVecSizeAttr:$scaleVecSize, + NVVM_BlockScaleFormatAttr:$blockScaleFormat, + NVVM_MMABlockScaleKindAttr:$kind + ); + let results = (outs + NVVM_LLVMStructureType:$res + ); +} +def NVVM_InternalMmaBlockScaleOp : NVVM_InferredOp<"mma.block_scale.internal"> { + let description = [{ + The `nvvm.mma.blockscale.internal` operation collectively performs the operation + `D = matmul(A * SF_A, B * SF_B) + C` using all threads in a warp. + + A, B, C and D are dense matrices and SF_A and SF_B are scaling factor. + Dimensions of SF_A and SF_B are based on scale vector sizes (X1, X2, X4), + the data type of must be either ue8m0 or ue4m3 + + Example: + ```mlir + + %0 = nvvm.mma.block_scale.internal %operandA, %scaleAData, %byteIdA, %threadIdA, + %operandB, %scaleBData, %byteIdB, %threadIdB, + %operandC + { + layoutA = #nvvm.mma_layout, + layoutB = #nvvm.mma_layout, + shape = #nvvm.shape, + aType = #nvvm.mma_type, + bType = #nvvm.mma_type, + cType = #nvvm.mma_type, + scaleVecSize = #nvvm.scale_vec_size, + blockScaleFormat = #nvvm.block_scale_format + kind = #nvvm.block_scale_kind + } + ``` + }]; + let arguments = (ins + NVVM_VectorOfAnyTypeValues:$operandA, + NVVM_T32BitSignlessInteger:$scaleAData, + NVVM_T16BitSignlessInteger:$byteIdA, + NVVM_T16BitSignlessInteger:$threadIdA, + NVVM_VectorOfAnyTypeValues:$operandB, + NVVM_T32BitSignlessInteger:$scaleBData, + NVVM_T16BitSignlessInteger:$byteIdB, + NVVM_T16BitSignlessInteger:$threadIdB, + NVVM_VectorOfAnyTypeValues:$operandC, + NVVM_MMAShapeAttr:$shape, + NVVM_MMALayoutAttr:$layoutA, + NVVM_MMALayoutAttr:$layoutB, + NVVM_MMATypesAttr:$aType, + NVVM_MMATypesAttr:$bType, + NVVM_MMATypesAttr:$cType, + NVVM_ScaleVecSizeAttr:$scaleVecSize, + NVVM_BlockScaleFormatAttr:$blockScaleFormat, + OptionalAttr:$kind + ); + let results = (outs + NVVM_VectorOfAnyTypeValues:$res + ); +} +def NVVM_MmaSpBlockScaleOp : NVVM_Op<"mma.sp.block_scale", [AttrSizedOperandSegments]> { + let description = [{ + The `nvvm.mma.sp.block_scale` operation collectively performs the operation + `D = matmul(A_sparse * SF_A, B * SF_B) + C` using all threads in a warp. + + A is a sparse matrix, and B, C and D are dense matrices. + SF_A and SF_B are scaling factors. + Dimensions of SF_A and SF_B are based on scale vector sizes (x1, x2, x4), + and the data type must be either ue8m0 or ue4m3. + + This operation is similar to `nvvm.mma.block_scale` but with structured sparsity + in the A operand. The sparsity follows the 2:4 structured sparse pattern + where 2 out of every 4 elements are non-zero. + + All the threads in the warp must execute the same `mma.sp.block_scale` operation. + + The `sparseMetadata` operand provides the sparsity indices that indicate + which elements in the A operand are non-zero. The `sparsitySelector` + controls how the indices are distributed among threads in the warp and + should typically be 0 or 1. + + This operation follows the same design pattern as `nvvm.mma.sp.sync`, with additional + scaling operands for both A and B matrices. Note that sparse block scale operations + always use ordered metadata (sm_90+). + + Example: + ```mlir + %d = nvvm.mma.sp.block_scale A[%a0, %a1] B[%b0, %b1] C[%c0, %c1] + sparseMetadata[%meta] selector[%sel] + scaleA[%scaleAData, %byteIdA, %threadIdA] + scaleB[%scaleBData, %byteIdB, %threadIdB] + {shape = #nvvm.shape, + multiplicandAPtxType = #nvvm.mma_type, + multiplicandBPtxType = #nvvm.mma_type, + scaleVecSize = #nvvm.scale_vec_size, + blockScaleFormat = #nvvm.block_scale_format, + kind = #nvvm.block_scale_kind} + : (vector<2xf16>, vector<2xf16>, vector<2xf32>) -> !llvm.struct<(f32, f32)> + ``` + }]; + let arguments = (ins + Variadic:$operandA, + Variadic:$operandB, + Variadic:$operandC, + NVVM_T32BitSignlessInteger:$sparseMetadata, + NVVM_T32BitSignlessInteger:$sparsitySelector, + NVVM_T32BitSignlessInteger:$scaleAData, + NVVM_T16BitSignlessInteger:$byteIdA, + NVVM_T16BitSignlessInteger:$threadIdA, + NVVM_T32BitSignlessInteger:$scaleBData, + NVVM_T16BitSignlessInteger:$byteIdB, + NVVM_T16BitSignlessInteger:$threadIdB, + NVVM_MMAShapeAttr:$shape, + OptionalAttr:$multiplicandAPtxType, + OptionalAttr:$multiplicandBPtxType, + NVVM_ScaleVecSizeAttr:$scaleVecSize, + NVVM_BlockScaleFormatAttr:$blockScaleFormat, + NVVM_MMABlockScaleKindAttr:$kind, + UnitAttr:$orderedMetadata + ); + let results = (outs + NVVM_LLVMStructureType:$res + ); +} +def NVVM_InternalMmaSparseBlockScaleOp : NVVM_InferredOp<"mma.sp.block_scale.internal"> { + let description = [{ + The `nvvm.mma.sparse.blockscale` operation collectively performs the + operation `D = matmul(A * SF_A, B * SF_B) + C` using all threads in a warp. + + Where A is structued sparse with atleast 50% zeros and B, C and D are + dense and SF_A and SF_B are scaling factor. Dimensions of SF_A and SF_B + are based on scale vector sizes (X1, X2, X4), the data type of + must be either ue8m0 or ue4m3 + + Example: + ```mlir + + %selector = llvm.mlir.constant(0) + %0 = nvvm.mma.sparse.block_scale + %operandA, %scaleAData, %byteIdA, %threadIdA, + %operandB, %scaleBData, %byteIdB, %threadIdB, + %operandC, %spMetadata, %selector + { + layoutA = #nvvm.mma_layout, + layoutB = #nvvm.mma_layout, + shape = #nvvm.shape, + aType = #nvvm.mma_type, + bType = #nvvm.mma_type, + cType = #nvvm.mma_type, + scaleVecSize = #nvvm.scale_vec_size, + blockScaleFormat = #nvvm.block_scale_format, + sparityFormat = #nvvm.sparsity_format + } + ``` + }]; + let arguments = (ins + NVVM_VectorOfAnyTypeValues:$operandA, + NVVM_T32BitSignlessInteger:$scaleAData, + NVVM_T16BitSignlessInteger:$byteIdA, + NVVM_T16BitSignlessInteger:$threadIdA, + NVVM_VectorOfAnyTypeValues:$operandB, + NVVM_T32BitSignlessInteger:$scaleBData, + NVVM_T16BitSignlessInteger:$byteIdB, + NVVM_T16BitSignlessInteger:$threadIdB, + NVVM_VectorOfAnyTypeValues:$operandC, + NVVM_T32BitSignlessInteger:$sp_metadata, + NVVM_T32BitSignlessInteger:$selector, + NVVM_MMAShapeAttr:$shape, + OptionalAttr:$b1Op, + OptionalAttr:$intOverflowBehavior, + NVVM_MMALayoutAttr:$layoutA, + NVVM_MMALayoutAttr:$layoutB, + NVVM_MMATypesAttr:$aType, + NVVM_MMATypesAttr:$bType, + NVVM_MMATypesAttr:$cType, + NVVM_ScaleVecSizeAttr:$scaleVecSize, + NVVM_BlockScaleFormatAttr:$blockScaleFormat, + NVVM_SparsityFormatAttr:$sparsityFormat + ); + let results = (outs + NVVM_VectorOfAnyTypeValues:$res + ); +} +def NVVM_MmaSpOp : NVVM_Op<"mma.sp.sync", [AttrSizedOperandSegments]> { + let description = [{ + The `nvvm.mma.sp.sync` operation collectively performs the sparse operation + `D = matmul(A_sparse, B) + C` using all threads in a warp. + + This operation is similar to `nvvm.mma.sync` but with structured sparsity + in the A operand. The sparsity follows the 2:4 structured sparse pattern + where 2 out of every 4 elements are non-zero. + + All the threads in the warp must execute the same `mma.sp.sync` operation. + + The `sparseMetadata` operand provides the sparsity indices that indicate + which elements in the A operand are non-zero. The `sparsitySelector` + controls how the indices are distributed among threads in the warp and + should typically be 0 or 1. + + The optional `orderedMetadata` attribute specifies the metadata ordering: + - Absence (default): Uses standard sparse metadata ordering + - Presence: Uses ordered metadata (PTX ISA 8.5+, sm_90+) + + The optional `kind` attribute specifies mixed-precision modes for FP8 operations: + - `f8f6f4`: Enables e3m2, e2m3, e2m1 FP8 types and f16 accumulator (PTX ISA 8.7+, sm_90+) + - Only valid with ordered metadata and m16n8k64 shape + + The shapes, layouts, and data types follow the same constraints as the + regular `nvvm.mma.sync` operation, but the A operand contains only the + non-zero elements in compressed format. + + Example: + ```mlir + %d = nvvm.mma.sp.sync A[%a0, %a1] B[%b0, %b1] C[%c0, %c1] + sparseMetadata[%meta] selector[%sel] + {shape = {k = 32 : i32, m = 16 : i32, n = 8 : i32}} + : (vector<2xf16>, vector<2xf16>, vector<2xf16>) -> !llvm.struct<(vector<2xf16>, vector<2xf16>)> + + // With ordered metadata: + %d = nvvm.mma.sp.sync A[%a0, %a1] B[%b0, %b1] C[%c0, %c1] + sparseMetadata[%meta] selector[%sel] + {orderedMetadata, shape = {k = 32 : i32, m = 16 : i32, n = 8 : i32}} + : (vector<2xf16>, vector<2xf16>, vector<2xf16>) -> !llvm.struct<(vector<2xf16>, vector<2xf16>)> + ``` + }]; + let arguments = (ins + Variadic:$operandA, + Variadic:$operandB, + Variadic:$operandC, + NVVM_T32BitSignlessInteger:$sparseMetadata, + NVVM_T32BitSignlessInteger:$sparsitySelector, + NVVM_MMAShapeAttr:$shape, + OptionalAttr:$intOverflowBehavior, + OptionalAttr:$multiplicandAPtxType, + OptionalAttr:$multiplicandBPtxType, + UnitAttr:$orderedMetadata, + OptionalAttr:$kind + ); + let results = (outs + NVVM_LLVMStructureType:$res + ); +} +def NVVM_MmaOp : NVVM_Op<"mma.sync", [AttrSizedOperandSegments]> { + let description = [{ + The `nvvm.mma.sync` operation collectively performs the operation + `D = matmul(A, B) + C` using all threads in a warp. + + All the threads in the warp must execute the same `mma.sync` operation. + + For each possible multiplicand PTX data type, there are one or more possible + instruction shapes given as "mMnNkK". The below table describes the posssibilities + as well as the types required for the operands. Note that the data type for + C (the accumulator) and D (the result) can vary independently when there are + multiple possibilities in the "C/D Type" column. + + When an optional attribute cannot be immediately inferred from the types of + the operands and the result during parsing or validation, an error will be + raised. + + `b1Op` is only relevant when the binary (b1) type is given to + `multiplicandDataType`. It specifies how the multiply-and-acumulate is + performed and is either `xor_popc` or `and_poc`. The default is `xor_popc`. + + `intOverflowBehavior` is only relevant when the `multiplicandType` attribute + is one of `u8, s8, u4, s4`, this attribute describes how overflow is handled + in the accumulator. When the attribute is `satfinite`, the accumulator values + are clamped in the int32 range on overflow. This is the default behavior. + Alternatively, accumulator behavior `wrapped` can also be specified, in + which case overflow wraps from one end of the range to the other. + + `layoutA` and `layoutB` are required and should generally be set to + `#nvvm.mma_layout` and `#nvvm.mma_layout` respectively, but other + combinations are possible for certain layouts according to the table below. + + ``` + | A/B Type | Shape | ALayout | BLayout | A Type | B Type | C/D Type | + |----------|-----------|---------|---------|----------|----------|-------------------| + | f64 | .m8n8k4 | row | col | 1x f64 | 1x f64 | 2x f64 | + | f16 | .m8n8k4 | row/col | row/col | 2x f16x2 | 2x f16x2 | 4x f16x2 or 8xf32 | + | | .m16n8k8 | row | col | 2x f16x2 | 1x f16x2 | 2x f16x2 or 4 f32 | + | | .m16n8k16 | row | col | 4x f16x2 | 2x f16x2 | 2x f16x2 or 4 f32 | + | bf16 | .m16n8k8 | row | col | 2x i32 | 1x i32 | 4x f32 | + | | .m16n8k16 | row | col | 4x i32 | 2x i32 | 4x f32 | + | tf32 | .m16n8k4 | row | col | 2x i32 | 1x i32 | 4x f32 | + | | .m16n8k8 | row | col | 4x i32 | 2x i32 | 2x f16x2 or 4 f32 | + | u8/s8 | .m8n8k16 | row | col | 1x i32 | 1x i32 | 2x i32 | + | | .m16n8k16 | row | col | 2x i32 | 1x i32 | 4x i32 | + | | .m16n8k32 | row | col | 4x i32 | 2x i32 | 4x i32 | + | u4/s4 | .m8n8k32 | row | col | 1x i32 | 1x i32 | 2x i32 | + | | m16n8k32 | row | col | 2x i32 | 1x i32 | 4x i32 | + | | m16n8k64 | row | col | 4x i32 | 2x i32 | 4x i32 | + | b1 | m8n8k128 | row | col | 1x i32 | 1x i32 | 2x i32 | + | | m16n8k128 | row | col | 2x i32 | 1x i32 | 4x i32 | + ``` + + Example: + ```mlir + + %128 = nvvm.mma.sync A[%120, %121, %122, %123] + B[%124, %125] + C[%126, %127] + {layoutA = #nvvm.mma_layout, + layoutB = #nvvm.mma_layout, + shape = {k = 16 : i32, m = 16 : i32, n = 8 : i32}} + : (vector<2xf16>, vector<2xf16>, vector<2xf16>) + -> !llvm.struct<(vector<2xf16>, vector<2xf16>)> + ``` + }]; + let arguments = (ins + Variadic:$operandA, + Variadic:$operandB, + Variadic:$operandC, + NVVM_MMAShapeAttr:$shape, + OptionalAttr:$b1Op, + OptionalAttr:$intOverflowBehavior, + NVVM_MMALayoutAttr:$layoutA, + NVVM_MMALayoutAttr:$layoutB, + OptionalAttr:$multiplicandAPtxType, + OptionalAttr:$multiplicandBPtxType + ); + let results = (outs + NVVM_LLVMStructureType:$res + ); +} +def NVVM_MmaSmemDescOp : NVVM_Op<"mma_smem_desc", [MemoryEffects<[MemRead]>]> { + let description = [{ + The `nvvm.mma_smem_desc` constructs a Shared Memory descriptor for use as the MMA Shared Memory operand. + }]; + let arguments = (ins + NVVM_T32BitSignlessInteger:$pointer, + NVVM_T32BitSignlessInteger:$ldm, + NVVM_T32BitSignlessInteger:$stride, + NVVM_T8BitSignlessInteger2:$baseOffset, + NVVM_T8BitSignlessInteger2:$swizzle, + OptionalAttr:$mmaDescVersion + ); + let results = (outs + NVVM_T64BitSignlessInteger2:$res + ); +} +def NVVM_MovMatrixOp : NVVM_Op<"movmatrix"> { + let description = [{ + Moves a row-major matrix across all threads in a warp, reading elements + from source `$src`, and writing the transposed elements to destination + `$dst`. + + The `shape` attribute indicates the dimensions of the matrix being + transposed. Each matrix element holds 16-bit data as indicated by the + `eltType` attribute. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#warp-level-matrix-movmatrix-instruction) + + Example: + ```mlir + %dst = nvvm.movmatrix %src {shape = #nvvm.ld_st_matrix_shape, + eltType = #nvvm.ld_st_matrix_elt_type} : i32 + ``` + }]; + let arguments = (ins + NVVM_T32BitSignlessInteger:$src, + NVVM_LdStMatrixShapeAttr:$shape, + OptionalAttr:$layout, + NVVM_LdStMatrixEltTypeAttr:$eltType + ); + let results = (outs + NVVM_T32BitSignlessInteger2:$dst + ); +} +def NVVM_MulOp : NVVM_InferredOp<"mul"> { + let description = [{ + nvvm.mul Op multiplies two values with a given `type` and `mode`. + The type of the operation represents the types of the `a` and `b` operands. + If `.hi` or `.lo` is specified, then `res` is the same size as `a` and `b`, + and either the upper or lower half of the result is written to the + destination register. If `.wide` is specified, then `res` is twice as + wide as `a` and `b` to receive the full result of the multiplication. The + `.wide` suffix is supported only for 16- and 32-bit integer types. + For more information, refer [PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#integer-arithmetic-instructions-mul) + }]; + let arguments = (ins + NVVM_T16BitSignlessIntegerOr323:$a, + NVVM_T16BitSignlessIntegerOr323:$b, + NVVM_MulModeAttr:$mode, + OptionalAttr:$isSigned + ); + let results = (outs + NVVM_T16BitSignlessIntegerOr323:$res + ); +} +def NVVM_MulPacked_BF16x2_BF16x2_F16x2Op : NVVM_Op<"mul.packed.bf16x2.bf16x2.f16x2"> { + let description = [{ + This Op performs the multiplication operation on a pair of "bf16"x2 and "f16"x2 vectors. + The inputs are of the form v2bf16 and v2f16, and the output is of the form v2bf16. + The inputs are converted to a scalar of packed i32 where the 0th element of the + vector forms the low-16 bits and the 1st element occupies the high-16 bits. + + Examples: + ```mlir + %0 = nvvm.mul.packed.bf16x2.bf16x2.f16x2 %srcA, %srcB : vector<2xbf16>, vector<2xf16> -> vector<2xbf16> + ``` + }]; + let arguments = (ins + NVVM_VectorOfBfloat16TypeValuesOf:$srcA, + NVVM_VectorOf16BitFloatValues:$srcB + ); + let results = (outs + NVVM_VectorOfBfloat16TypeValuesOf:$res + ); +} +def NVVM_MulPacked_BF16x2_F32x2_F32x2Op : NVVM_Op<"mul.packed.bf16x2.f32x2.f32x2"> { + let description = [{ + This Op performs the multiplication operation on a pair of f32x2 vectors. + The inputs are of the form v2f32 and the output is of the form v2bf16. + The inputs are converted to a scalar of packed i64 where the 0th element of the + vector forms the low-32 bits and the 1st element occupies the high-32 bits. The round to zero + is the only rounding mode supported for this operation. + + Examples: + ```mlir + // Round to zero (only rounding mode supported) + %0 = nvvm.mul.packed.bf16x2.f32x2.f32x2 %srcA, %srcB : vector<2xf32>, vector<2xf32> -> vector<2xbf16> + ``` + }]; + let arguments = (ins + NVVM_VectorOf32BitFloatValues:$srcA, + NVVM_VectorOf32BitFloatValues:$srcB + ); + let results = (outs + NVVM_VectorOfBfloat16TypeValuesOf:$res + ); +} +def NVVM_MulPacked_F16x2_F16x2_BF16x2Op : NVVM_Op<"mul.packed.f16x2.f16x2.bf16x2"> { + let description = [{ + This Op performs the multiplication operation on a pair of "f16"x2 and "bf16"x2 vectors. + The inputs are of the form v2f16 and v2bf16, and the output is of the form v2f16. + The inputs are converted to a scalar of packed i32 where the 0th element of the + vector forms the low-16 bits and the 1st element occupies the high-16 bits. + + Examples: + ```mlir + %0 = nvvm.mul.packed.f16x2.f16x2.bf16x2 %srcA, %srcB : vector<2xf16>, vector<2xbf16> -> vector<2xf16> + ``` + }]; + let arguments = (ins + NVVM_VectorOf16BitFloatValues:$srcA, + NVVM_VectorOfBfloat16TypeValuesOf:$srcB + ); + let results = (outs + NVVM_VectorOf16BitFloatValues:$res + ); +} +def NVVM_MulPacked_F16x2_F32x2_F32x2Op : NVVM_Op<"mul.packed.f16x2.f32x2.f32x2"> { + let description = [{ + This Op performs the multiplication operation on a pair of f32x2 vectors. + The inputs are of the form v2f32 and the output is of the form v2f16. + The inputs are converted to a scalar of packed i64 where the 0th element of the + vector forms the low-32 bits and the 1st element occupies the high-32 bits. The round to zero + is the only rounding mode supported for this operation. + + Examples: + ```mlir + // Round to zero (only rounding mode supported) + %0 = nvvm.mul.packed.f16x2.f32x2.f32x2 %srcA, %srcB : vector<2xf32>, vector<2xf32> -> vector<2xf16> + ``` + }]; + let arguments = (ins + NVVM_VectorOf32BitFloatValues:$srcA, + NVVM_VectorOf32BitFloatValues:$srcB + ); + let results = (outs + NVVM_VectorOf16BitFloatValues:$res + ); +} +def NVVM_MulPackedF32x2Op : NVVM_InferredOp<"mul.packed.f32x2"> { + let description = [{ + This Op multiplies floating-point vectors. + The inputs and outputs are of the form . The `rnd` attribute + specifies the rounding mode. The `ftz` attribute, when set, flushes the + subnormal inputs and results to sign-preserving zero. + This Op is supported from SM100 and onwards. + [For more information, see PTX ISA] + (TODO: Add public PTX link when available) + }]; + let arguments = (ins + NVVM_VectorOf32BitFloatValues2:$srcA, + NVVM_VectorOf32BitFloatValues2:$srcB, + OptionalAttr:$rnd, + OptionalAttr:$ftz + ); + let results = (outs + NVVM_VectorOf32BitFloatValues2:$res + ); +} +def NVVM_MultimemLdReduceOp : NVVM_Op<"multimem.ld.reduce"> { + let description = [{ + The `nvvm.multimem.ld.reduce` operation performs a multimem load-reduce + instruction. It reads from a multimem address and reduces partial + results across participating threads. + + The `ordering` attribute specifies the memory ordering semantics (weak, + relaxed, acquire). + + The `op` attribute specifies the reduction operation (and, or, + xor, add, min, max). + + The `dataType` attribute specifies the PTX data type: + - Integer: b32, s32, u32, b64, s64, u64 + - Floating-point: f16, bf16, f32, f64 + - FP8: e5m2, e4m3 + + The optional `accPrec` attribute specifies the accumulator precision type + for sub-single-precision types (f16/bf16/f8E5M2/f8E4M3FN): f16 or f32. + + Example: + ```mlir + %res = nvvm.multimem.ld.reduce %addr + {ordering = #nvvm.mem_order, + op = #nvvm.red_op, + dataType = f32} + : !llvm.ptr<1> -> f32 + ``` + + See PTX ISA: + https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-multimem + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace1:$addr, + NVVM_MultimemLdReduceOrderingAttr:$ordering, + NVVM_MultimemLdReduceOpAttr:$op, + TypeAttr:$dataType, + OptionalAttr:$accPrec + ); + let results = (outs + NVVM_AnyType:$res + ); +} +def NVVM_MultimemRedOp : NVVM_Op<"multimem.red"> { + let description = [{ + The `nvvm.multimem.red` operation performs a multimem atomic reduction, + applying a reduction operation between a source value and the value at a + multimem address. + + The `ordering` attribute specifies the memory ordering semantics. + The `scope` attribute specifies the scope of the operation. + The `op` attribute specifies the reduction operation. + The `dataType` attribute specifies the PTX data type (b32/s32/u32/b64/s64/ + u64 for integer types, f16/bf16/f32/f64 for floating-point types). + + Example: + ```mlir + nvvm.multimem.red %val, %addr + {ordering = #nvvm.mem_order, + scope = #nvvm.mem_scope, + op = #nvvm.red_op, + dataType = f32} + : f32, !llvm.ptr<1> + ``` + + See PTX ISA: + https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-multimem + }]; + let arguments = (ins + NVVM_AnyType:$val, + NVVM_AnyType:$addr, + NVVM_MultimemRedOrderingAttr:$ordering, + NVVM_MemScopeKindAttr:$scope, + NVVM_MultimemRedOpAttr:$op, + TypeAttr:$dataType + ); + let results = (outs); +} +def NVVM_MultimemStOp : NVVM_Op<"multimem.st"> { + let description = [{ + The `nvvm.multimem.st` operation performs a multimem store instruction, + broadcasting a value to a multimem address. + + The `ordering` attribute specifies the memory ordering semantics. + The `scope` attribute specifies the scope of the operation. + The `dataType` attribute specifies the PTX data type (b32/s32/u32/b64/s64/ + u64 for integer types, f16/bf16/f32/f64 for floating-point, e5m2/e4m3 for + FP8 types). + + Example: + ```mlir + nvvm.multimem.st %val, %addr + {ordering = #nvvm.mem_order, + scope = #nvvm.mem_scope, + dataType = f32} + : f32, !llvm.ptr<1> + ``` + + See PTX ISA: + https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-multimem + }]; + let arguments = (ins + NVVM_AnyType:$val, + NVVM_LLVMPointerInAddressSpace1:$addr, + NVVM_MultimemStOrderingAttr:$ordering, + NVVM_MemScopeKindAttr:$scope, + TypeAttr:$dataType + ); + let results = (outs); +} +def NVVM_NanosleepOp : NVVM_Op<"nanosleep"> { + let description = [{ + The op suspends the thread for a sleep duration approximately close to the + delay `$duration`, specified in nanoseconds. + + The sleep duration is approximated, but guaranteed to be in the + interval [0, 2*t]. The maximum sleep duration is 1 millisecond. + The implementation may reduce the sleep duration for individual threads + within a warp such that all sleeping threads in the warp wake up together. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#miscellaneous-instructions-nanosleep) + }]; + let arguments = (ins + NVVM_T32BitSignlessInteger:$duration + ); + let results = (outs); +} +def NVVM_PMEventOp : NVVM_Op<"pmevent"> { + let description = [{ + Triggers one or more of a fixed number of performance monitor events, with + event index or mask specified by immediate operand. + + Without `mask` it triggers a single performance monitor event indexed by + immediate operand a, in the range 0..15. + + With `mask` it triggers one or more of the performance monitor events. Each + bit in the 16-bit immediate operand controls an event. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#miscellaneous-instructions-pmevent) + }]; + let arguments = (ins + OptionalAttr:$maskedEventId, + OptionalAttr:$eventId + ); + let results = (outs); +} +def NVVM_PrefetchOp : NVVM_Op<"prefetch"> { + let description = [{ + Prefetches the cache line containing the address given by `addr`. The + operand may be a global, local, or generic pointer. When `tensormap` is + specified, the operand may instead be a constant or generic pointer. If the + address maps to shared memory, the operation has no effect. + + At most one of `cacheLevel` or `tensormap` may be present. The `cacheLevel` + attribute selects the target cache level. When combined with `uniform`, the + prefetch is performed to the uniform cache, in which case `addr` must be a + generic pointer. + + When `tensormap` is used, the line containing `addr` is brought from the + constant or parameter state space for later use by `cp.async.bulk.tensor`. + If `in_param_space` is specified, the generic pointer is interpreted as + referring to the parameter state space. + + `uniform` can be specified after the `cacheLevel` to indicate that the + prefetch is performed to the specified uniform cache level. If `uniform` is + specified, `addr` must be a generic address pointer and no operation is + performed if `addr` maps to a `const`, `local`, or `shared` memory location. + + The `evictPriority` attribute is optional and specifies the cache eviction + priority when `cacheLevel` is L2. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-prefetch-prefetchu) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace13:$addr, + Optional:$predicate, + OptionalAttr:$cacheLevel, + OptionalAttr:$evictPriority, + UnitAttr:$tensormap, + UnitAttr:$uniform, + UnitAttr:$in_param_space + ); + let results = (outs); +} +def NVVM_PermuteOp : NVVM_Op<"prmt", [Pure]> { + let description = [{ + The `nvvm.prmt` operation constructs a permutation of the + bytes of the first one or two operands, selecting based on + the 2 least significant bits of the final operand. + + The bytes in the first one or two source operands are numbered. + The first source operand (%lo) is numbered {b3, b2, b1, b0}, + in the case of the '``default``', '``f4e``' and '``b4e``' variants, + the second source operand (%hi) is numbered {b7, b6, b5, b4}. + + Modes: + - `default`: Index mode - each nibble in `selector` selects a byte from the 8-byte pool + - `f4e` : Forward 4 extract - extracts 4 contiguous bytes starting from position in `selector` + - `b4e` : Backward 4 extract - extracts 4 contiguous bytes in reverse order + - `rc8` : Replicate 8 - replicates the lower 8 bits across the 32-bit result + - `ecl` : Edge clamp left - clamps out-of-range indices to the leftmost valid byte + - `ecr` : Edge clamp right - clamps out-of-range indices to the rightmost valid byte + - `rc16` : Replicate 16 - replicates the lower 16 bits across the 32-bit result + + Depending on the 2 least significant bits of the %selector operand, the result + of the permutation is defined as follows: + + +------------+----------------+--------------+ + | Mode | %selector[1:0] | Output | + +------------+----------------+--------------+ + | '``f4e``' | 0 | {3, 2, 1, 0} | + | +----------------+--------------+ + | | 1 | {4, 3, 2, 1} | + | +----------------+--------------+ + | | 2 | {5, 4, 3, 2} | + | +----------------+--------------+ + | | 3 | {6, 5, 4, 3} | + +------------+----------------+--------------+ + | '``b4e``' | 0 | {5, 6, 7, 0} | + | +----------------+--------------+ + | | 1 | {6, 7, 0, 1} | + | +----------------+--------------+ + | | 2 | {7, 0, 1, 2} | + | +----------------+--------------+ + | | 3 | {0, 1, 2, 3} | + +------------+----------------+--------------+ + | '``rc8``' | 0 | {0, 0, 0, 0} | + | +----------------+--------------+ + | | 1 | {1, 1, 1, 1} | + | +----------------+--------------+ + | | 2 | {2, 2, 2, 2} | + | +----------------+--------------+ + | | 3 | {3, 3, 3, 3} | + +------------+----------------+--------------+ + | '``ecl``' | 0 | {3, 2, 1, 0} | + | +----------------+--------------+ + | | 1 | {3, 2, 1, 1} | + | +----------------+--------------+ + | | 2 | {3, 2, 2, 2} | + | +----------------+--------------+ + | | 3 | {3, 3, 3, 3} | + +------------+----------------+--------------+ + | '``ecr``' | 0 | {0, 0, 0, 0} | + | +----------------+--------------+ + | | 1 | {1, 1, 1, 0} | + | +----------------+--------------+ + | | 2 | {2, 2, 1, 0} | + | +----------------+--------------+ + | | 3 | {3, 2, 1, 0} | + +------------+----------------+--------------+ + | '``rc16``' | 0 | {1, 0, 1, 0} | + | +----------------+--------------+ + | | 1 | {3, 2, 3, 2} | + | +----------------+--------------+ + | | 2 | {1, 0, 1, 0} | + | +----------------+--------------+ + | | 3 | {3, 2, 3, 2} | + +------------+----------------+--------------+ + + [For more information, see PTX ISA] + (https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-prmt) + }]; + let arguments = (ins + NVVM_T32BitSignlessInteger:$lo, + Optional:$hi, + NVVM_T32BitSignlessInteger:$selector, + NVVM_PermuteModeAttr:$mode + ); + let results = (outs + NVVM_T32BitSignlessInteger2:$res + ); +} +def NVVM_RcpApproxFtzF32Op : NVVM_Op<"rcp.approx.ftz.f", [Pure]> { + let arguments = (ins + NVVM_T32BitFloat:$arg + ); + let results = (outs + NVVM_T32BitFloat2:$res + ); +} +def NVVM_AggrSmemSize : NVVM_Op<"read.ptx.sreg.aggr.smem.size", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_ClockOp : NVVM_Op<"read.ptx.sreg.clock"> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_Clock64Op : NVVM_Op<"read.ptx.sreg.clock64"> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_BlockInClusterIdXOp : NVVM_Op<"read.ptx.sreg.cluster.ctaid.x", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_BlockInClusterIdYOp : NVVM_Op<"read.ptx.sreg.cluster.ctaid.y", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_BlockInClusterIdZOp : NVVM_Op<"read.ptx.sreg.cluster.ctaid.z", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_ClusterId : NVVM_Op<"read.ptx.sreg.cluster.ctarank", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_ClusterDimBlocksXOp : NVVM_Op<"read.ptx.sreg.cluster.nctaid.x", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_ClusterDimBlocksYOp : NVVM_Op<"read.ptx.sreg.cluster.nctaid.y", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_ClusterDimBlocksZOp : NVVM_Op<"read.ptx.sreg.cluster.nctaid.z", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_ClusterDim : NVVM_Op<"read.ptx.sreg.cluster.nctarank", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_ClusterIdXOp : NVVM_Op<"read.ptx.sreg.clusterid.x", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_ClusterIdYOp : NVVM_Op<"read.ptx.sreg.clusterid.y", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_ClusterIdZOp : NVVM_Op<"read.ptx.sreg.clusterid.z", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_BlockIdXOp : NVVM_Op<"read.ptx.sreg.ctaid.x", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_BlockIdYOp : NVVM_Op<"read.ptx.sreg.ctaid.y", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_BlockIdZOp : NVVM_Op<"read.ptx.sreg.ctaid.z", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_DynamicSmemSize : NVVM_Op<"read.ptx.sreg.dynamic.smem.size", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg0Op : NVVM_Op<"read.ptx.sreg.envreg0", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg1Op : NVVM_Op<"read.ptx.sreg.envreg1", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg10Op : NVVM_Op<"read.ptx.sreg.envreg10", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg11Op : NVVM_Op<"read.ptx.sreg.envreg11", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg12Op : NVVM_Op<"read.ptx.sreg.envreg12", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg13Op : NVVM_Op<"read.ptx.sreg.envreg13", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg14Op : NVVM_Op<"read.ptx.sreg.envreg14", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg15Op : NVVM_Op<"read.ptx.sreg.envreg15", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg16Op : NVVM_Op<"read.ptx.sreg.envreg16", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg17Op : NVVM_Op<"read.ptx.sreg.envreg17", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg18Op : NVVM_Op<"read.ptx.sreg.envreg18", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg19Op : NVVM_Op<"read.ptx.sreg.envreg19", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg2Op : NVVM_Op<"read.ptx.sreg.envreg2", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg20Op : NVVM_Op<"read.ptx.sreg.envreg20", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg21Op : NVVM_Op<"read.ptx.sreg.envreg21", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg22Op : NVVM_Op<"read.ptx.sreg.envreg22", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg23Op : NVVM_Op<"read.ptx.sreg.envreg23", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg24Op : NVVM_Op<"read.ptx.sreg.envreg24", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg25Op : NVVM_Op<"read.ptx.sreg.envreg25", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg26Op : NVVM_Op<"read.ptx.sreg.envreg26", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg27Op : NVVM_Op<"read.ptx.sreg.envreg27", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg28Op : NVVM_Op<"read.ptx.sreg.envreg28", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg29Op : NVVM_Op<"read.ptx.sreg.envreg29", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg3Op : NVVM_Op<"read.ptx.sreg.envreg3", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg30Op : NVVM_Op<"read.ptx.sreg.envreg30", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg31Op : NVVM_Op<"read.ptx.sreg.envreg31", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg4Op : NVVM_Op<"read.ptx.sreg.envreg4", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg5Op : NVVM_Op<"read.ptx.sreg.envreg5", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg6Op : NVVM_Op<"read.ptx.sreg.envreg6", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg7Op : NVVM_Op<"read.ptx.sreg.envreg7", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg8Op : NVVM_Op<"read.ptx.sreg.envreg8", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_EnvReg9Op : NVVM_Op<"read.ptx.sreg.envreg9", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_GlobalTimerOp : NVVM_Op<"read.ptx.sreg.globaltimer"> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_GlobalTimerLoOp : NVVM_Op<"read.ptx.sreg.globaltimer.lo"> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_GridIdOp : NVVM_Op<"read.ptx.sreg.gridid", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_LaneIdOp : NVVM_Op<"read.ptx.sreg.laneid", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_LaneMaskEqOp : NVVM_Op<"read.ptx.sreg.lanemask.eq", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_LaneMaskGeOp : NVVM_Op<"read.ptx.sreg.lanemask.ge", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_LaneMaskGtOp : NVVM_Op<"read.ptx.sreg.lanemask.gt", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_LaneMaskLeOp : NVVM_Op<"read.ptx.sreg.lanemask.le", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_LaneMaskLtOp : NVVM_Op<"read.ptx.sreg.lanemask.lt", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_ClusterDimXOp : NVVM_Op<"read.ptx.sreg.nclusterid.x", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_ClusterDimYOp : NVVM_Op<"read.ptx.sreg.nclusterid.y", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_ClusterDimZOp : NVVM_Op<"read.ptx.sreg.nclusterid.z", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_GridDimXOp : NVVM_Op<"read.ptx.sreg.nctaid.x", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_GridDimYOp : NVVM_Op<"read.ptx.sreg.nctaid.y", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_GridDimZOp : NVVM_Op<"read.ptx.sreg.nctaid.z", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_SmDimOp : NVVM_Op<"read.ptx.sreg.nsmid", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_BlockDimXOp : NVVM_Op<"read.ptx.sreg.ntid.x", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_BlockDimYOp : NVVM_Op<"read.ptx.sreg.ntid.y", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_BlockDimZOp : NVVM_Op<"read.ptx.sreg.ntid.z", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_WarpDimOp : NVVM_Op<"read.ptx.sreg.nwarpid", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_SmIdOp : NVVM_Op<"read.ptx.sreg.smid", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_ThreadIdXOp : NVVM_Op<"read.ptx.sreg.tid.x", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_ThreadIdYOp : NVVM_Op<"read.ptx.sreg.tid.y", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_ThreadIdZOp : NVVM_Op<"read.ptx.sreg.tid.z", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_TotalSmemSize : NVVM_Op<"read.ptx.sreg.total.smem.size", [Pure]> { + let arguments = (ins); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_WarpIdOp : NVVM_Op<"read.ptx.sreg.warpid", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_WarpSizeOp : NVVM_Op<"read.ptx.sreg.warpsize", [Pure]> { + let arguments = (ins + OptionalAttr:$range + ); + let results = (outs + NVVM_LLVMDialectCompatibleType:$res + ); +} +def NVVM_RedOp : NVVM_Op<"red"> { + let description = [{ + Performs a reduction operation with operand `b` and the value in + location `a`, and stores the result of the specified operation at + location `a`. Operand `a` specifies a location in the specified state space. + If no state space is given, perform the memory accesses + using Generic Addressing + + [For more information, see PTX ISA] + (http://sw-mobile-docs/CUDA/GPGPU/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-red) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace05:$a, + NVVM_T32BitSignlessIntegerOr642:$b, + Optional:$cacheHint, + OptionalAttr:$memOrder, + OptionalAttr:$memScope, + NVVM_ReductionOpAttr:$op, + NVVM_ReductionTypeAttr:$type + ); + let results = (outs); +} +def NVVM_ReduxOp : NVVM_InferredOp<"redux.sync"> { + let description = [{ + `redux.sync` performs a reduction operation `kind` of the 32 bit source + register across all non-exited threads in the membermask. + + The `abs` and `nan` attributes can be used in the case of f32 input type, + where the `abs` attribute causes the absolute value of the input to be used + in the reduction operation, and the `nan` attribute causes the reduction + operation to return NaN if any of the inputs to participating threads are + NaN. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-redux-sync) + }]; + let arguments = (ins + NVVM_T32BitSignlessIntegerOr32:$val, + NVVM_T32BitSignlessInteger:$mask_and_clamp, + NVVM_ReductionKindAttr:$kind, + OptionalAttr:$abs, + OptionalAttr:$nan + ); + let results = (outs + NVVM_T32BitSignlessIntegerOr32:$res + ); +} +def NVVM_RsqrtOp : NVVM_InferredOp<"rsqrt", [Pure]> { + let description = [{ + Computes an approximation of the reciprocal of the square root of the + input value: `d = 1 / sqrt(a)`. Supports both f32 and f64. The maximum + relative error for the f32 form over the entire positive finite range + is 2^-22.9. + + The `ftz` attribute, when set, flushes subnormal inputs and results to + sign-preserving zero. For f64 inputs, `ftz=true` selects a coarser + approximation that uses only the upper 32 bits of the input (the lower + 32 bits of the result are zeroed). + + For more information, see PTX ISA: + [rsqrt](https://docs.nvidia.com/cuda/parallel-thread-execution/#floating-point-instructions-rsqrt) + }]; + let arguments = (ins + NVVM_T32BitFloatOr64Bit:$src, + OptionalAttr:$ftz + ); + let results = (outs + NVVM_T32BitFloatOr64Bit:$res + ); +} +def NVVM_SetMaxRegisterOp : NVVM_Op<"setmaxregister"> { + let arguments = (ins + I32Attr:$regCount, + NVVM_SetMaxRegisterActionAttr:$action + ); + let results = (outs); +} +def NVVM_ShflOp : NVVM_InferredOp<"shfl.sync"> { + let description = [{ + The `shfl.sync` Op implements data shuffle within threads of a warp. + The `thread_mask` denotes the threads participating in the Op where + the bit position corresponds to a particular thread's laneid. + The `offset` specifies a source lane or source lane offset + (depending on `kind`). The `val` is the input value to be copied from + the source. The `mask_and_clamp` contains two packed values specifying + a mask for logically splitting warps into sub-segments and an upper bound + for clamping the source lane index. + + The `return_value_and_is_valid` unit attribute can be specified to indicate + that the return value is a two-element struct, where the first element is + the result value and the second element is a predicate indicating if the + computed source lane index is valid. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-shfl-sync) + }]; + let arguments = (ins + NVVM_T32BitSignlessInteger:$thread_mask, + NVVM_T32BitSignlessIntegerOr32:$val, + NVVM_T32BitSignlessInteger:$offset, + NVVM_T32BitSignlessInteger:$mask_and_clamp, + NVVM_ShflKindAttr:$kind, + UnitAttr:$return_value_and_is_valid + ); + let results = (outs + NVVM_T32BitSignlessIntegerOr322:$res + ); +} +def NVVM_SinOp : NVVM_Op<"sin", [Pure]> { + let description = [{ + Computes a fast approximation of the sine of the input value (in radians). + The `ftz` attribute, when set, flushes subnormal inputs and results to + sign-preserving zero. + + For more information, see PTX ISA: + [sin](https://docs.nvidia.com/cuda/parallel-thread-execution/#floating-point-instructions-sin) + }]; + let arguments = (ins + NVVM_T32BitFloat:$src, + OptionalAttr:$ftz + ); + let results = (outs + NVVM_T32BitFloat2:$res + ); +} +def NVVM_SPCompressOp : NVVM_Op<"spcompress"> { + let description = [{ + Compresses elements from a dense vector `data` into a structured sparse vector `compressed_data`. + The metadata information is stored in `metadata`. + + Arguments: + - `spDesc`: Sparsity descriptor specifying the selection criteria for elements + - `data`: Input dense vector containing elements to be compressed + - `compFactor`: Compression factor attribute (only `sp2to4` is allowed, resulting in 2:4 compression) + - `indexSize`: Size of metadata index for each selected element (can only be `b2` or `b4` given 2:4 compression) + - `elemSize`: Size in bits for each element from input vector `data` + - `repFactor`: Repeat factor for the operation + + Results: + - `metadata`: Vector storing indices for chosen elements + - `compressed_data`: Output compressed vector containing selected elements + + Examples: + // Example 1: Basic compression with 4-bit elements, 2-bit indices, repeat factor of 1 + ```mlir + %metadata, %compressed_data = nvvm.spcompress %spdesc, %data { + compFactor = #nvvm.spcomp_factor, + indexSize = #nvvm.spcomp_index_size, + elemSize = #nvvm.spcomp_elem_size, + repFactor = #nvvm.spcomp_rep_factor + } : i32, vector<2xi32> -> i32, i32 + ``` + + // Example 2: Compression with 8-bit elements, 4-bit indices, repeat factor of 2 + ```mlir + %metadata, %compressed_data = nvvm.spcompress %spdesc, %data { + compFactor = #nvvm.spcomp_factor, + indexSize = #nvvm.spcomp_index_size, + elemSize = #nvvm.spcomp_elem_size, + repFactor = #nvvm.spcomp_rep_factor + } : i32, vector<4xi32> -> i32, vector<2xi32> + ``` + + // Example 3: Compression with 16-bit elements, 4-bit indices, larger repeat factor + ```mlir + %metadata, %compressed_data = nvvm.spcompress %spdesc, %data { + compFactor = #nvvm.spcomp_factor, + indexSize = #nvvm.spcomp_index_size, + elemSize = #nvvm.spcomp_elem_size, + repFactor = #nvvm.spcomp_rep_factor + } : i32, vector<16xi32> -> vector<2xi32>, vector<8xi32> + ``` + }]; + let arguments = (ins + NVVM_T32BitSignlessInteger:$spDesc, + NVVM_VectorOf32BitSignlessInteger:$data, + NVVM_SPCompressFactorTypeAttr:$compFactor, + NVVM_SPCompressIndexSizeAttr:$indexSize, + NVVM_SPCompressElemSizeAttr:$elemSize, + NVVM_SPCompressRepFactorAttr:$repFactor + ); + let results = (outs + NVVM_T32BitSignlessIntegerOrVector:$metadata, + NVVM_T32BitSignlessIntegerOrVector:$compressed_data + ); +} +def NVVM_SPDecompressOp : NVVM_Op<"spdecompress"> { + let description = [{ + Decompresses elements from a structured sparse vector `compressed_data` into a dense vector `data`. + The metadata information is stored in `metadata`. + + Arguments: + - `metadata`: Vector storing indices for chosen elements + - `compressed_data`: Compressed vector containing selected elements + - `compFactor`: Decompression factor attribute (sp1to2, sp1to4, sp1to8, sp1to16, sp2to4, sp2to8, sp2to16, sp4to8, sp4to16, sp8to16) + - `indexSize`: Size of metadata index for each selected element (b2 or b4) + - `elemSize`: Size in bits for each element from output vector `data` (b4, b8, b16) + - `repFactor`: Repeat factor for the operation (x1, x2, x4, x8, x16, x32, x64) + + Results: + - `data`: Output dense vector containing decompressed elements + + Examples: + // Example 1: Decompression with 4-bit elements, 2-bit indices, repeat factor of 2 + ```mlir + %res = nvvm.spdecompress %metadata, %compdata { + compFactor = #nvvm.spdecomp_factor, + indexSize = #nvvm.spdecomp_index_size, + elemSize = #nvvm.spdecomp_elem_size, + repFactor = #nvvm.spdecomp_rep_factor + } : i32, i32 -> i32 + ``` + + // Example 2: Decompression with 8-bit elements, 4-bit indices, repeat factor of 2 + ```mlir + %res = nvvm.spdecompress %metadata, %compdata { + compFactor = #nvvm.spdecomp_factor, + indexSize = #nvvm.spdecomp_index_size, + elemSize = #nvvm.spdecomp_elem_size, + repFactor = #nvvm.spdecomp_rep_factor + } : i32, i32 -> vector<2 x i32> + ``` + + // Example 3: Decompression with 16-bit elements, 4-bit indices, repeat factor of 64 + ```mlir + %res = nvvm.spdecompress %metadata, %compdata { + compFactor = #nvvm.spdecomp_factor, + indexSize = #nvvm.spdecomp_index_size, + elemSize = #nvvm.spdecomp_elem_size, + repFactor = #nvvm.spdecomp_rep_factor + } : vector<16 x i32>, vector<64 x i32> -> vector<128 x i32> + ``` + }]; + let arguments = (ins + NVVM_T32BitSignlessIntegerOrVector:$metadata, + NVVM_T32BitSignlessIntegerOrVector:$compressed_data, + NVVM_SPDecompressFactorTypeAttr:$compFactor, + NVVM_SPDecompressIndexSizeAttr:$indexSize, + NVVM_SPDecompressElemSizeAttr:$elemSize, + NVVM_SPDecompressRepFactorAttr:$repFactor + ); + let results = (outs + NVVM_VectorOf32BitSignlessInteger2:$data + ); +} +def NVVM_BulkStoreOp : NVVM_Op<"st.bulk"> { + let description = [{ + Initializes a region of shared memory at the address given by `addr`. + The `size` operand specifies the number of bytes to initialize and must be + a multiple of 8. + The `initVal` operand specifies the value to initialize the memory to. The + only supported value is 0. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-st-bulk) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace02:$addr, + NVVM_T64BitSignlessInteger:$size, + OptionalAttr:$initVal + ); + let results = (outs); +} +def NVVM_StMatrixOp : NVVM_Op<"stmatrix"> { + let description = [{ + Collectively store one or more matrices across all threads in a warp to the + location indicated by the address operand $ptr in shared memory. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#warp-level-matrix-store-instruction-stmatrix) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace3:$ptr, + Variadic:$sources, + NVVM_MMALayoutAttr:$layout, + OptionalAttr:$shape + ); + let results = (outs); +} +def NVVM_AsyncStoreOp : NVVM_Op<"store.async"> { + let arguments = (ins + NVVM_LLVMPointerInAddressSpace7:$addr, + NVVM_T32BitSignlessIntegerOr643:$value, + NVVM_LLVMPointerInAddressSpace7:$mbarrier + ); + let results = (outs); +} +def NVVM_ExtStoreOp : NVVM_Op<"store.ext"> { + let description = [{ + The `store.ext` operation is used to write to memory in generic, + global or shared address space. This Op explicitly supports various + cache properties and memory orderings associated with the store operation. + + The `l2CacheHint` operand is optional, and it is used to specify cache + eviction policy that may be used during the memory access. + [For more information, see PTX ISA] + (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-st) + }]; + let arguments = (ins + NVVM_LLVMTypeWithSize:$value, + NVVM_LLVMPointerType:$addr, + Optional:$l2CacheHint, + OptionalAttr:$order, + OptionalAttr:$scope, + OptionalAttr:$evict, + OptionalAttr:$cacheModifier, + OptionalAttr:$sharedSpace + ); + let results = (outs); +} +def NVVM_SubPacked_BF16x2_F32x2_F32x2Op : NVVM_Op<"sub.packed.bf16x2.f32x2.f32x2"> { + let description = [{ + This Op performs the sub operation on a pair of f32x2 vectors. + The inputs and output are of the form v2f32 and v2bf16 respectively. + The inputs are converted to a scalar of packed i64 where the 0th element of the + vector forms the low-32 bits and the 1st element occupies the high-32 bits. The round to zero + is the only rounding mode supported for this operation. + + Examples: + ```mlir + // Round to zero (only rounding mode supported) + %0 = nvvm.sub .packed.bf16x2.f32x2.f32x2 %srcA, %srcB : vector<2xf32>, vector<2xf32> -> vector<2xbf16> + ``` + }]; + let arguments = (ins + NVVM_VectorOf32BitFloatValues:$srcA, + NVVM_VectorOf32BitFloatValues:$srcB + ); + let results = (outs + NVVM_VectorOfBfloat16TypeValuesOf:$res + ); +} +def NVVM_SubPacked_F16x2_F32x2_F32x2Op : NVVM_Op<"sub.packed.f16x2.f32x2.f32x2"> { + let description = [{ + This Op performs the sub operation on a pair of f32x2 vectors. + The inputs and output are of the form v2f32 and v2f16 respectively. + The inputs are converted to a scalar of packed i64 where the 0th element of the + vector forms the low-32 bits and the 1st element occupies the high-32 bits. The round to zero + is the only rounding mode supported for this operation. + + Examples: + ```mlir + // Round to zero (only rounding mode supported) + %0 = nvvm.sub .packed.f16x2.f32x2.f32x2 %srcA, %srcB : vector<2xf32>, vector<2xf32> -> vector<2xf16> + ``` + }]; + let arguments = (ins + NVVM_VectorOf32BitFloatValues:$srcA, + NVVM_VectorOf32BitFloatValues:$srcB + ); + let results = (outs + NVVM_VectorOf16BitFloatValues:$res + ); +} +def NVVM_SubPackedF32x2Op : NVVM_InferredOp<"sub.packed.f32x2"> { + let description = [{ + This Op performs subtraction of floating-point vectors. + The inputs and outputs are of the form . The `rnd` attribute + specifies the rounding mode. The `ftz` attribute, when set, flushes the + subnormal inputs and results to sign-preserving zero. + This Op is supported from SM100 and onwards. + [For more information, see PTX ISA] + (TODO: Add public PTX link when available) + }]; + let arguments = (ins + NVVM_VectorOf32BitFloatValues2:$srcA, + NVVM_VectorOf32BitFloatValues2:$srcB, + OptionalAttr:$rnd, + OptionalAttr:$ftz + ); + let results = (outs + NVVM_VectorOf32BitFloatValues2:$res + ); +} +def NVVM_SubPacked_F32x2_BF16x2_F32x2Op : NVVM_Op<"sub.packed.f32x2.bf16x2.f32x2"> { + let description = [{ + This Op performs the sub operation on a pair of floating-point values. + The inputs are of the form v2bf16 and v2f32. The first input `srcA` is + converted to a scalar of packed i32 where the 0th element of the vector forms + the low-16 bits and the 1st element occupies the high-16 bits. Similarly, + the second input `srcB` is converted to a scalar of packed i64 where the 0th + element of the vector forms the low-32 bits and the 1st element occupies the + high-32 bits. The `rnd` attribute specifies the rounding mode. + + The rounding mode can be one of the following: + - rn: round to nearest even + - rz: round toward zero + - rm: round toward negative infinity + - rp: round toward positive infinity + + Examples: + ```mlir + // Round to nearest even (default) + %0 = nvvm.sub .packed.f32x2.bf16x2.f32x2 %srcA, %srcB : vector<2xbf16>, vector<2xf32> -> vector<2xf32> + + // Round toward negative infinity + %1 = nvvm.sub .packed.f32x2.bf16x2.f32x2 %srcA, %srcB {rnd = #nvvm.fp_rnd_mode} : vector<2xbf16>, vector<2xf32> -> vector<2xf32> + + // Round toward zero with flush-to-zero + %2 = nvvm.sub .packed.f32x2.bf16x2.f32x2 %srcA, %srcB {rnd = #nvvm.fp_rnd_mode} : vector<2xbf16>, vector<2xf32> -> vector<2xf32> + ``` + }]; + let arguments = (ins + NVVM_VectorOfBfloat16TypeValuesOf:$srcA, + NVVM_VectorOf32BitFloatValues:$srcB, + OptionalAttr:$rnd + ); + let results = (outs + NVVM_VectorOf32BitFloatValues:$res + ); +} +def NVVM_SubPacked_F32x2_F16x2_F32x2Op : NVVM_Op<"sub.packed.f32x2.f16x2.f32x2"> { + let description = [{ + This Op performs the sub operation on a pair of floating-point values. + The inputs are of the form v2f16 and v2f32. The first input `srcA` is + converted to a scalar of packed i32 where the 0th element of the vector forms + the low-16 bits and the 1st element occupies the high-16 bits. Similarly, + the second input `srcB` is converted to a scalar of packed i64 where the 0th + element of the vector forms the low-32 bits and the 1st element occupies the + high-32 bits. The `rnd` attribute specifies the rounding mode. + + The rounding mode can be one of the following: + - rn: round to nearest even + - rz: round toward zero + - rm: round toward negative infinity + - rp: round toward positive infinity + + Examples: + ```mlir + // Round to nearest even (default) + %0 = nvvm.sub .packed.f32x2.f16x2.f32x2 %srcA, %srcB : vector<2xf16>, vector<2xf32> -> vector<2xf32> + + // Round toward negative infinity + %1 = nvvm.sub .packed.f32x2.f16x2.f32x2 %srcA, %srcB {rnd = #nvvm.fp_rnd_mode} : vector<2xf16>, vector<2xf32> -> vector<2xf32> + + // Round toward zero with flush-to-zero + %2 = nvvm.sub .packed.f32x2.f16x2.f32x2 %srcA, %srcB {rnd = #nvvm.fp_rnd_mode} : vector<2xf16>, vector<2xf32> -> vector<2xf32> + ``` + }]; + let arguments = (ins + NVVM_VectorOf16BitFloatValues:$srcA, + NVVM_VectorOf32BitFloatValues:$srcB, + OptionalAttr:$rnd + ); + let results = (outs + NVVM_VectorOf32BitFloatValues:$res + ); +} +def NVVM_SubFOp : NVVM_InferredOp<"subf", [Pure]> { + let description = [{ + The `nvvm.subf` operation performs floating point subtraction of two + operands. + + It supports the same type combinations and modifiers as `nvvm.addf`. + This is equivalent to `nvvm.addf(lhs, -rhs)`. + + For more information, see PTX ISA: + - [floating point subtraction](https://docs.nvidia.com/cuda/parallel-thread-execution/#floating-point-instructions-sub) + - [half-precision floating point subtraction](https://docs.nvidia.com/cuda/parallel-thread-execution/#half-precision-floating-point-instructions-sub) + }]; + let arguments = (ins + NVVM_T16BitFloatOrBfloat16Type:$lhs, + NVVM_T16BitFloatOrBfloat16Type:$rhs, + OptionalAttr:$rnd, + OptionalAttr:$sat, + OptionalAttr:$ftz + ); + let results = (outs + NVVM_T16BitFloatOrBfloat16Type:$res + ); +} +def NVVM_Tcgen05AllocOp : NVVM_Op<"tcgen05.alloc"> { + let description = [{ + The `tcgen05.alloc` Op allocates tensor core memory for + the amount specified by `nCols` and writes the destination + address to the `addr` argument. The `nCols` operand specifies the + number of columns to be allocated and it must be a power-of-two. + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-memory-alloc-manage-instructions) + }]; + let arguments = (ins + NVVM_LLVMPointerTypeOrLLVMPointer:$addr, + NVVM_T32BitSignlessInteger:$nCols, + UnitAttr:$isExclusive, + OptionalAttr:$group + ); + let results = (outs); +} +def NVVM_Tcgen05CommitOp : NVVM_Op<"tcgen05.commit"> { + let description = [{ + The `tcgen05.commit` makes the *mbarrier object*, specified by + the operand `addr`, track the completion of all the prior + async-tcgen05 operations initiated by the executing thread. + The multicast variants allow signaling on the *mbarrier objects* + of multiple CTAs within the cluster. Operand `multicastMask`, + when present, specifies the destination CTAs in the cluster such + that each bit position in the 16-bit or 32-bit `multicastMask` operand + corresponds to the `nvvm.read.ptx.sreg.ctaid` of the destination CTA. + Operand `smem_a_read` when present, specifies track only shared memory + read of "Matrix A for the prior tcgen05.mma Ops" + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen-async-sync-operations-commit) + }]; + let arguments = (ins + NVVM_LLVMPointerTypeOrLLVMPointer:$addr, + Optional:$multicastMask, + UnitAttr:$smem_a_read, + OptionalAttr:$group + ); + let results = (outs); +} +def NVVM_Tcgen05CpOp : NVVM_Op<"tcgen05.cp"> { + let description = [{ + Instruction tcgen05.cp initiates an asynchronous copy operation from + shared memory to the location specified by the address operand `taddr` + in the Tensor Memory. The 64-bit register operand `smem_desc` specifies + the matrix descriptor representing the source matrix in the shared memory + that needs to be copied. + + Example: + ```mlir + nvvm.tcgen05.cp %taddr, %smem_desc { + group = #nvvm.cta_group, + shape = #nvvm.tcgen05_cp_shape, + multicast = #nvvm.tcgen05_cp_multicast, + srcFormat = #nvvm.tcgen05_cp_src_fmt + } + ``` + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tensorcore-5th-generation-instructions-tcgen05-cp) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace6:$taddr, + NVVM_T64BitSignlessInteger:$smem_desc, + NVVM_Tcgen05CpShapeAttr:$shape, + OptionalAttr:$group, + OptionalAttr:$multicast, + OptionalAttr:$srcFormat + ); + let results = (outs); +} +def NVVM_Tcgen05DeallocOp : NVVM_Op<"tcgen05.dealloc"> { + let description = [{ + The `tcgen05.dealloc` Op de-allocates the tensor core memory + specified by `tmemAddr`, which must be from a previous tensor + memory allocation. The `nCols` operand specifies the number + of columns to be de-allocated, and it must be a power-of-two. + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-memory-alloc-manage-instructions) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace6:$taddr, + NVVM_T32BitSignlessInteger:$nCols, + UnitAttr:$isExclusive, + OptionalAttr:$group + ); + let results = (outs); +} +def NVVM_Tcgen05FenceOp : NVVM_Op<"tcgen05.fence"> { + let description = [{ + The `tcgen05.fence` orders all prior async tcgen05 operations + with respect to the subsequent tcgen05 and execution ordering operations. + The `tcgen05.fence` orders all subsequent async tcgen05 operations + with respect to the prior tcgen05 and execution ordering operations. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tensorcore-5th-generation-instructions-tcgen05-fence) + }]; + let arguments = (ins + NVVM_Tcgen05FenceKindAttr:$kind + ); + let results = (outs); +} +def NVVM_Tcgen05LdOp : NVVM_Op<"tcgen05.ld"> { + let description = [{ + Instruction `tcgen05.ld` asynchronously loads data from the Tensor Memory at + the location specified by the 32-bit address operand `tmemAddr` into the + destination register `res`, collectively across all threads of the warps. + + The `shape` and the `num` attribute together determines the total + dimension of the data which is loaded from the Tensor Memory. The `shape` + attribute indicates the base dimension of data to be accessed as described + in the Data Movement Shape. The `num` attribute indicates the repeat + factor on the base dimension resulting in the total dimension of the data + that is accessed. + + The shape `16x32bx2` performs two accesses into Tensor Memory of the shape + `16x32b`. The base address of the first access is specified by `tmemAddr` + and the base address of the second access is specified by + `tmemAddr + offset`, where `offset` is an immediate argument. + + The unit attribute `pack` can be used to pack two 16-bit + elements from adjacent columns into a single 32-bit element during the load. + + The following table describes the size of the vector for various combinations + of `num` and `shape` attributes: + ``` + |=====================================================================| + | num/shape | 16x32bx2/16x64b/32x32b | 16x128b | 16x256b | + |=====================================================================| + | x1 | 1 | 2 | 4 | + | x2 | 2 | 4 | 8 | + | x4 | 4 | 8 | 16 | + | x8 | 8 | 16 | 32 | + | x16 | 16 | 32 | 64 | + | x32 | 32 | 64 | 128 | + | x64 | 64 | 128 | NA | + | x128 | 128 | NA | NA | + |=====================================================================| + ``` + + Example: + ```mlir + nvvm.tcgen05.ld %tmemAddr, %offset pack { + shape = #nvvm.tcgen05_ldst_shape, + } : <2xi32> + ``` + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-instructions-tcgen05-st) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace6:$tmemAddr, + Optional:$offset, + UnitAttr:$pack, + NVVM_Tcgen05LdStShapeAttr:$shape + ); + let results = (outs + NVVM_T32BitSignlessIntegerOrVector2:$res + ); +} +def NVVM_Tcgen05LdRedOp : NVVM_Op<"tcgen05.ld.red"> { + let description = [{ + Instruction `tcgen05.ld.red` asynchronously loads data from the Tensor + Memory at the location specified by the 32-bit address operand `addr` into + the destination register `data`, collectively across all threads of the + warp. The operation also performs reduction operation specified by `op` on + the loaded data across columns in each lane and stored into `redVal` + + The `shape` and the `num` attribute together determines the total + dimension of the data which is loaded from the Tensor Memory. The `shape` + attribute indicates the base dimension of data to be accessed as described + in the Data Movement Shape. The `num` attribute indicates the repeat + factor on the base dimension resulting in the total dimension of the data + that is accessed. + + The shape `16x32bx2` performs two accesses into Tensor Memory of the shape + `16x32b`. The base address of the first access is specified by `addr` + and the base address of the second access is specified by + `addr + offset`, where `offset` is an immediate argument. + + `isSigned` attribute is used to specify whether the integer reduction value is is signed, it is to false by default + + The following table describes the size of the vector for various combinations + of `num` and `shape` attributes: + ``` + |=============================================| + | num/shape | 16x32bx2/32x32b | + |=============================================| + | x2 | 2 | + | x4 | 4 | + | x8 | 8 | + | x16 | 16 | + | x32 | 32 | + | x64 | 64 | + | x128 | 128 | + |=============================================| + ``` + + Example: + ```mlir + // u32 + %data, %redval = nvvm.tcgen05.ld,red %addr, %offset { + shape = #nvvm.tcgen05_ldst_shape, + } : <2xi32>, i32 + + // s32 + %data, %redval = nvvm.tcgen05.ld,red %addr, %offset { + shape = #nvvm.tcgen05_ldst_shape, + isSigned = true + } : <2xi32>, i32 + + // f32 + %data, %redval = nvvm.tcgen05.ld,red %addr { + shape = #nvvm.tcgen05_ldst_shape, + } : <2xf32>, f32 + ``` + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-instructions-tcgen05-ld) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace6:$addr, + Optional:$offset, + NVVM_Tcgen05LdStShapeAttr:$shape, + NVVM_ReductionKindAttr:$op, + UnitAttr:$abs, + UnitAttr:$nan, + OptionalAttr:$isSigned + ); + let results = (outs + NVVM_VectorOf32BitSignlessInteger3:$data, + NVVM_T32BitSignlessIntegerOr32:$redVal + ); +} +def NVVM_Tcgen05LdRedSpcompressOp : NVVM_Op<"tcgen05.ld.red.spcompress"> { + let description = [{ + The `tcgen05.ld.red.spcompress` performs in-line sparse compression when loading data from Tensor Memory. + The operation uses a combination of `abs` and `opKind` parameters to select values for compression. + The NaN detection mode is always used for the compression operation, while the .NAN flag only applies + to the reduction portion. + + Arguments: + - `tmemAddr`: Tensor memory address to load from + - `abs`: Whether to use absolute value comparison (used with opKind for value selection) + - `opKind`: Comparison operator kind (min/max) for selecting values (used with abs for value selection) + - `shape`: Pattern used to construct the Matrix (used with repFactor for Matrix construction) + - `repFactor`: Repeat factor, number of times the pattern has to be repeated to form the Matrix (used with shape for Matrix construction) + - `compFactor`: Compression factor (2:4 sparsity, choose 2 pair of elems from every 4 pairs of elems) + - `nan`: Whether to use NaN detection mode + + Results: + - `metadata`: Vector storing sparse metadata + - `compressedData`: Vector storing compressed data + - `redVal`: Reduction value + + Examples: + ```mlir + // MIN operation with x4 repetition factor + %metadata, %compressed_data, %red_val = nvvm.tcgen05.ld.red.spcompress %taddr { + shape = #nvvm.tcgen05_ldst_shape, + opKind = #nvvm.spcomp_op_kind, + compFactor = #nvvm.spcomp_factor, + repFactor = #nvvm.spcomp_rep_factor + } : i32, vector<2xi32> + + // Max operation with absolute value comparison + %metadata, %compressed_data, %red_val = nvvm.tcgen05.ld.red.spcompress %taddr { + abs, + shape = #nvvm.tcgen05_ldst_shape, + opKind = #nvvm.spcomp_op_kind, + compFactor = #nvvm.spcomp_factor, + repFactor = #nvvm.spcomp_rep_factor + } : i32, vector<4xi32> + + // MIN operation with NaN detection for reduction + %metadata, %compressed_data, %red_val = nvvm.tcgen05.ld.red.spcompress %taddr { + nan, + shape = #nvvm.tcgen05_ldst_shape, + opKind = #nvvm.spcomp_op_kind, + compFactor = #nvvm.spcomp_factor, + repFactor = #nvvm.spcomp_rep_factor + } : i32, vector<8xi32> + + // MAX operation with abs and nan options + %metadata, %compressed_data, %red_val = nvvm.tcgen05.ld.red.spcompress %taddr { + abs, + nan, + shape = #nvvm.tcgen05_ldst_shape, + opKind = #nvvm.spcomp_op_kind, + compFactor = #nvvm.spcomp_factor, + repFactor = #nvvm.spcomp_rep_factor + } : vector<2xi32>, vector<32xi32> + + // MIN operation with maximum repetition factor + %metadata, %compressed_data, %red_val = nvvm.tcgen05.ld.red.spcompress %taddr { + shape = #nvvm.tcgen05_ldst_shape, + opKind = #nvvm.spcomp_op_kind, + compFactor = #nvvm.spcomp_factor, + repFactor = #nvvm.spcomp_rep_factor + } : vector<4xi32>, vector<64xi32> + ``` + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace6:$tmemAddr, + UnitAttr:$abs, + UnitAttr:$nan, + NVVM_Tcgen05LdStShapeAttr:$shape, + NVVM_SPCompressOpKindAttr:$opKind, + NVVM_SPCompressFactorTypeAttr:$compFactor, + NVVM_SPCompressRepFactorAttr:$repFactor + ); + let results = (outs + NVVM_T32BitSignlessIntegerOrVector3:$metadata, + NVVM_VectorOf32BitSignlessInteger4:$compressedData, + NVVM_T32BitSignlessInteger:$redVal + ); +} +def NVVM_Tcgen05LdSpcompressOp : NVVM_Op<"tcgen05.ld.spcompress"> { + let description = [{ + The `tcgen05.ld.spcompress` performs an in-line sparse compression when loading data from Tensor Memory. + The operation uses a combination of `abs` and `opKind` parameters to select values for compression. + + With a 2-to-4 sparsity factor (specified by `compFactor`), half as much data is written to the output + as is read from Tensor Memory. The sequence of sparse metadata is written to the `metadata` result, + while the compressed data is written to `compressedData`. + + Arguments: + - `tmemAddr`: Tensor memory address to load from + - `abs`: Whether to use absolute value comparison (used with opKind for value selection) + - `opKind`: Comparison operator kind (min/max) for selecting values (used with abs for value selection) + - `shape`: Pattern used to construct the Matrix (used with repFactor for Matrix construction) + - `repFactor`: Repeat factor, number of times the pattern has to be repeated to form the Matrix (used with shape for Matrix construction) + - `compFactor`: Compression factor (2:4 sparsity, choose 2 pair of elems from every 4 pairs of elems) + + Results: + - `metadata`: Vector storing sparse metadata + - `compressedData`: Vector storing compressed data + + Example: + ```mlir + // MIN operation with x4 repetition factor + %metadata, %compressed_data = nvvm.tcgen05.ld.spcompress %taddr { + shape = #nvvm.tcgen05_ldst_shape, + opKind = #nvvm.spcomp_op_kind, + compFactor = #nvvm.spcomp_factor, + repFactor = #nvvm.spcomp_rep_factor + } : i32, vector<2xi32> + + // MAX operation with abs option + %metadata, %compressed_data = nvvm.tcgen05.ld.spcompress %taddr { + abs, + shape = #nvvm.tcgen05_ldst_shape, + opKind = #nvvm.spcomp_op_kind, + compFactor = #nvvm.spcomp_factor, + repFactor = #nvvm.spcomp_rep_factor + } : i32, vector<4xi32> + + // MIN operation with x64 repetition factor + %metadata, %compressed_data = nvvm.tcgen05.ld.spcompress %taddr { + shape = #nvvm.tcgen05_ldst_shape, + opKind = #nvvm.spcomp_op_kind, + compFactor = #nvvm.spcomp_factor, + repFactor = #nvvm.spcomp_rep_factor + } : vector<2xi32>, vector<32xi32> + + // MAX operation with abs and x128 repetition factor + %metadata, %compressed_data = nvvm.tcgen05.ld.spcompress %taddr { + abs, + shape = #nvvm.tcgen05_ldst_shape, + opKind = #nvvm.spcomp_op_kind, + compFactor = #nvvm.spcomp_factor, + repFactor = #nvvm.spcomp_rep_factor + } : vector<4xi32>, vector<64xi32> + ``` + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace6:$tmemAddr, + UnitAttr:$abs, + NVVM_Tcgen05LdStShapeAttr:$shape, + NVVM_SPCompressOpKindAttr:$opKind, + NVVM_SPCompressFactorTypeAttr:$compFactor, + NVVM_SPCompressRepFactorAttr:$repFactor + ); + let results = (outs + NVVM_T32BitSignlessIntegerOrVector3:$metadata, + NVVM_VectorOf32BitSignlessInteger4:$compressedData + ); +} +def NVVM_Tcgen05MMAOp : NVVM_Op<"tcgen05.mma", [AttrSizedOperandSegments]> { + let description = [{ + The `tcgen05.mma` operation is an asynchronous tensor core instruction that + performs matrix multiplication, accumulation in a single fused operation. It + targets 5th-generation tensor cores, providing developers with fine-grained + control over execution and scheduling. + + ``` + D = A * B + (D * 2^ -scaleInputD) // if `scaleInputD` is provided + D = A * B // if `enableInputD` is false + D = A * B + D // otherwise + ``` + + where: + - A is an `M x K` matrix in tensor memory or described using shared memory descriptor + - B is a `K x N` matrix described using shared memory descriptor + - D is an `M x N` accumulator matrix in tensor memory + + The `shared memory descriptor` can be generated using `tcgen05.mma_smem_desc` Op + + - idesc is a 32-bit value representing the [Instruction Descriptor](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-instruction-descriptor) + + Optional Operands: + - `scaleInputD` is an Immediate value operand used for scaling D matrix by 2 ^ (-scaleInputD). The valid range is [0, 15] + + - `disableOutputLane` is a vector mask for selective output + * vector<4 x i32> when ctaGroup is CTA_1 + * vector<8 x i32> when ctaGroup is CTA_2 + + Required Attributes: + - `kind` is a Tcgen05MMAKind attribute + + - `ctaGroup` specifies CTA group configuration + * cta_1: MMA will be performed on the current thread's CTA + * cta_2: MMA will be performed on the current thread and it's peer CTA + + Default Attributes: + - collectorOp is a Tcgen05MMACollectorOp attribute with matrix A as the collector buffer + + - `aShift` shifts the rows of the A matrix down by one row and can only be + applied if A is in tensor memory + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-mma-instructions-mma) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace6:$matrixD, + NVVM_LLVMPointerInAddressSpace62:$matrixA, + NVVM_T64BitSignlessInteger:$matrixB, + NVVM_T32BitSignlessInteger:$idesc, + NVVM_T1BitSignlessInteger:$enableInputD, + Optional:$scaleInputD, + Optional:$disableOutputLane, + NVVM_Tcgen05MMAKindAttr:$kind, + NVVM_CTAGroupKindAttr:$ctaGroup, + OptionalAttr:$collectorOp, + OptionalAttr:$collectorOpB, + UnitAttr:$aShift + ); + let results = (outs); +} +def NVVM_Tcgen05MMABlockScaleOp : NVVM_Op<"tcgen05.mma.block_scale"> { + let description = [{ + The `tcgen05.mma.block_scale` operation is an asynchronous tensor core instruction + that performs matrix multiplication, accumulation with block scaling in a + single fused operation. It targets 5th-generation tensor cores, providing + developers with fine-grained control over execution and scheduling. + + ``` + D = (A * scale_a) * (B * scale_b)` // if `enableInputD` is false + D = (A * scale_a) * (B * scale_b) + D` + ``` + + where: + - A is an M x (K / 2) matrix in tensor memory or described using shared memory descriptor + - B is a K x N matrix described using shared memory descriptor + - D is an M x N accumulator matrix in tensor memory + - `scale_a` and `scale_b` are matrices in tensor memory used to scale `A` and `B` respectively + + The `shared memory descriptor` can be generated using `tcgen05.mma_smem_desc` Op + + - `idesc` is a 32 bit value representing the [Instruction Descriptor](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-instruction-descriptor) + + Required Attributes: + - `kind` is a Tcgen05MMAKind attribute restricted to mxf8f6f4, mxf4, or mxf4nvf4 + + - `ctaGroup` specifies CTA group configuration + * cta_1: MMA will be performed on the current thread's CTA + * cta_2: MMA will be performed on the current thread and it's peer CTA + + Default Attributes: + - collectorOp is a Tcgen05MMACollectorOp attribute with matrix A as the collector buffer + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-mma-instructions-mma) + }]; + let arguments = (ins + NVVM_AnyType:$matrixD, + NVVM_AnyType:$matrixA, + NVVM_AnyType:$matrixB, + NVVM_AnyType:$idesc, + NVVM_AnyType:$enableInputD, + NVVM_AnyType:$scaleA, + NVVM_AnyType:$scaleB, + NVVM_Tcgen05MMAKindAttr:$kind, + NVVM_CTAGroupKindAttr:$ctaGroup, + OptionalAttr:$blockScale, + OptionalAttr:$collectorOp, + OptionalAttr:$collectorOpB + ); + let results = (outs); +} +def NVVM_Tcgen05MMABlockScaleDecompressBOp : NVVM_Op<"tcgen05.mma.block_scale.decompress_b"> { + let description = [{ + The `tcgen05.mma.block_scale.decompress_b` operation is an asynchronous tensor core + instruction that decompresses the B matrix and performs matrix + multiplication, accumulation with block scaling in a single fused operation. + It targets 5th-generation tensor cores, providing developers with + fine-grained control over execution and scheduling. + + ``` + D = (A * scale_a) * (B * scale_b)` // if `enableInputD` is false + D = (A * scale_a) * (B * scale_b) + D` + ``` + + where: + - A is an M x (K / 2) matrix in tensor memory or described using shared memory descriptor + - B is a `K x N` matrix described using shared memory descriptor + - D is an `M x N` accumulator matrix in tensor memory + - `scale_a` and `scale_b` are matrices in tensor memory used to scale `A` and `B` respectively + + The `shared memory descriptor` can be generated using `tcgen05.mma_smem_desc` Op + + - `idesc` is a 32-bit value representing the [Instruction Descriptor](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-instruction-descriptor) + + - `decompressBMetadata` is a decompress metadata of B matrix + + Required Attributes: + - `ctaGroup` specifies CTA group configuration + * cta_1: MMA will be performed on the current thread's CTA + * cta_2: MMA will be performed on the current thread and it's peer CTA + + Default Attributes: + - collectorOpA is a Tcgen05MMACollectorOp attribute with matrix A as the collector buffer + - collectorOpB is a Tcgen05MMACollectorOp attribute with matrix B as the collector buffer + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-mma-instructions-mma) + }]; + let arguments = (ins + NVVM_AnyType:$matrixD, + NVVM_AnyType:$matrixA, + NVVM_AnyType:$matrixB, + NVVM_AnyType:$idesc, + NVVM_AnyType:$enableInputD, + NVVM_AnyType:$scaleA, + NVVM_AnyType:$scaleB, + NVVM_AnyType:$decompressBMetadata, + NVVM_Tcgen05MMAKindAttr:$kind, + NVVM_CTAGroupKindAttr:$ctaGroup, + OptionalAttr:$blockScale, + OptionalAttr:$collectorOpA, + OptionalAttr:$collectorOpB + ); + let results = (outs); +} +def NVVM_Tcgen05MMADecompressBOp : NVVM_Op<"tcgen05.mma.decompress_b"> { + let description = [{ + The `tcgen05.mma.decompress_b` operation is an asynchronous tensor core + instruction that decompresses the B matrix and performs matrix + multiplication, accumulation in a single fused operation. It targets + 5th-generation tensor cores, providing developers with fine-grained + control over execution and scheduling. + + ``` + D = A * B // if `enableInputD` is false + D = A * B + D // otherwise + ``` + + where: + - A is an `M x K` matrix in tensor memory or described using shared memory descriptor + - B is a `K x N` matrix described using shared memory descriptor + - D is an `M x N` accumulator matrix in tensor memory + + The `shared memory descriptor` can be generated using `tcgen05.mma_smem_desc` Op + + - `idesc` is a 32-bit value representing the [Instruction Descriptor](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-instruction-descriptor) + + - `decompressBMetadata` is a decompress metadata of B matrix + + Optional Operands: + - `disableOutputLane` is a vector mask for selective output + * vector<4 x i32> when ctaGroup is CTA_1 + * vector<8 x i32> when ctaGroup is CTA_2 + + Required Attributes: + - `ctaGroup` specifies CTA group configuration + * cta_1: MMA will be performed on the current thread's CTA + * cta_2: MMA will be performed on the current thread and it's peer CTA + + Default Attributes: + - collectorOAp is a Tcgen05MMACollectorOp attribute with matrix A as the collector buffer + - collectorOpB is a Tcgen05MMACollectorOp attribute with matrix B as the collector buffer + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-mma-instructions-mma) + }]; + let arguments = (ins + NVVM_AnyType:$matrixD, + NVVM_AnyType:$matrixA, + NVVM_AnyType:$matrixB, + NVVM_AnyType:$idesc, + NVVM_AnyType:$enableInputD, + NVVM_AnyType:$decompressBMetadata, + Optional:$disableOutputLane, + NVVM_Tcgen05MMAKindAttr:$kind, + NVVM_CTAGroupKindAttr:$ctaGroup, + OptionalAttr:$collectorOpA, + OptionalAttr:$collectorOpB + ); + let results = (outs); +} +def NVVM_Tcgen05MMASparseOp : NVVM_Op<"tcgen05.mma.sp", [AttrSizedOperandSegments]> { + let description = [{ + The `tcgen05.mma.sp` operation is an asynchronous tensor core instruction + that performs matrix multiplication, accumulation with sparse `A` matrix in + a single fused operation. It targets 5th-generation tensor cores, providing + developers with fine-grained control over execution and scheduling. + + ``` + D = A * B + (D * 2^ -scaleInputD) // if `scaleInputD` is provided + D = A * B // if `enableInputD` is false + D = A * B + D // otherwise + ``` + + where: + - A is an `M x (K / 2)` matrix in tensor memory or described using shared memory descriptor + - B is a `K x N` matrix described using shared memory descriptor + - D is an `M x N` accumulator matrix in tensor memory + - sparseMetadata located in tensor memory specifies the mapping of the `K / 2` + non-zero elements to the K elements before performing the MMA operation + + Other attributes and operands are similar to that of tcgen05.mma Op + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-mma-instructions-mma-sp) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace6:$matrixD, + NVVM_LLVMPointerInAddressSpace62:$matrixA, + NVVM_T64BitSignlessInteger:$matrixB, + NVVM_T32BitSignlessInteger:$idesc, + NVVM_T1BitSignlessInteger:$enableInputD, + NVVM_LLVMPointerInAddressSpace6:$sparseMetadata, + Optional:$scaleInputD, + Optional:$disableOutputLane, + NVVM_Tcgen05MMAKindAttr:$kind, + NVVM_CTAGroupKindAttr:$ctaGroup, + OptionalAttr:$collectorOp, + OptionalAttr:$collectorOpB, + UnitAttr:$aShift + ); + let results = (outs); +} +def NVVM_Tcgen05MMASparseBlockScaleOp : NVVM_Op<"tcgen05.mma.sp.block_scale"> { + let description = [{ + The `tcgen05.mma.sp.block_scale` operation is an asynchronous tensor core + instruction that performs matrix multiplication, accumulation with block + scaling, and sparse `A` matrix in a single fused operation. It targets + 5th-generation tensor cores, providing developers with fine-grained control + over execution, and scheduling. + + ``` + D = (A * scale_a) * (B * scale_b) // if `enableInputD` is specified + D = (A * scale_a) * (B * scale_b) + D // otherwise + ``` + + where: + - A is an M x (K / 2) matrix in tensor memory or described using shared memory descriptor + - B is a K x N matrix described using shared memory descriptor + - D is an M x N accumulator matrix in tensor memory + - `scale_a` and `scale_b` are matrices in tensor memory used to scale `A` and `B` respectively + + Other attributes and operands are similar to that of tcgen05.mma.block_scale Op + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-mma-instructions-mma-sp) + }]; + let arguments = (ins + NVVM_AnyType:$matrixD, + NVVM_AnyType:$matrixA, + NVVM_AnyType:$matrixB, + NVVM_AnyType:$idesc, + NVVM_AnyType:$enableInputD, + NVVM_AnyType:$sparseMetadata, + NVVM_AnyType:$scaleA, + NVVM_AnyType:$scaleB, + NVVM_Tcgen05MMAKindAttr:$kind, + NVVM_CTAGroupKindAttr:$ctaGroup, + OptionalAttr:$blockScale, + OptionalAttr:$collectorOp, + OptionalAttr:$collectorOpB + ); + let results = (outs); +} +def NVVM_Tcgen05MMAWsOp : NVVM_Op<"tcgen05.mma.ws"> { + let description = [{ + The `tcgen05.mma.ws` operation is an asynchronous tensor core instruction + that performs weight stationary convolution matrix multiplication, accumulation + in a single fused operation. It targets 5th-generation tensor cores, providing + developers with fine-grained control over execution, and scheduling. + + ``` + D = A * B` // if `enableInputD` is false + D = A * B + D` // otherwise + ``` + + where: + - A is an `M x K` matrix in tensor memory or described using shared memory descriptor + - B is a `K x N` matrix described using shared memory descriptor + - D is an `M x N` accumulator matrix in tensor memory + + The `shared memory descriptor` can be generated using `tcgen05.mma_smem_desc` Op + + - idesc is a 32-bit value representing the [Instruction Descriptor](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-instruction-descriptor) + + Optional Operands: + - zeroColMask is a 64 bit value representing the [Zero-column mask descriptor](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-zero-column-mask-descriptor) + + Required Attributes: + - `kind` is a Tcgen05MMAKind attribute + + Default Valued Attributes: + - collectorBBuffer specifies collector buffer for matrix B: b0 (default), b1, b2, b3 + + - collectorOp is a Tcgen05MMACollectorOp attribute with matrix B as the collector buffer + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-mma-instructions-mma-ws) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace6:$matrixD, + NVVM_LLVMPointerInAddressSpace62:$matrixA, + NVVM_T64BitSignlessInteger:$matrixB, + NVVM_T32BitSignlessInteger:$idesc, + NVVM_T1BitSignlessInteger:$enableInputD, + Optional:$zeroColMask, + NVVM_Tcgen05MMAKindAttr:$kind, + OptionalAttr:$collectorBBuffer, + OptionalAttr:$collectorOp + ); + let results = (outs); +} +def NVVM_Tcgen05MMAWsSparseOp : NVVM_Op<"tcgen05.mma.ws.sp"> { + let description = [{ + The `tcgen05.mma.ws.sp` operation is an asynchronous tensor core instruction + that performs weight stationary convolution matrix multiplication, accumulation + with sparse `A` matrix in a single fused operation. It targets 5th-generation + tensor cores, providing developers with fine-grained control over execution, + and scheduling. + + ``` + D = A * B` // if `enableInputD` is false + D = A * B + D` // otherwise + ``` + + where: + - A is an M x (K / 2) matrix in memory or descriptor format + - B is a K x N matrix + - D is an M x N accumulator matrix + - sparseMetadata located in tensor memory specifies the mapping of the `K / 2` + non-zero elements to the K elements before performing the MMA operation + + Other attributes and operands are similar to that of tcgen05.mma.ws Op + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-mma-instructions-mma-ws-sp) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace6:$matrixD, + NVVM_LLVMPointerInAddressSpace62:$matrixA, + NVVM_T64BitSignlessInteger:$matrixB, + NVVM_T32BitSignlessInteger:$idesc, + NVVM_T1BitSignlessInteger:$enableInputD, + NVVM_LLVMPointerInAddressSpace6:$sparseMetadata, + Optional:$zeroColMask, + NVVM_Tcgen05MMAKindAttr:$kind, + OptionalAttr:$collectorBBuffer, + OptionalAttr:$collectorOp + ); + let results = (outs); +} +def NVVM_Tcgen05MmaSmemDescOp : NVVM_Op<"tcgen05.mma_smem_desc", [MemoryEffects<[MemRead]>]> { + let description = [{ + The `nvvm.tcgen05_mma_smem_desc` constructs a Shared Memory descriptor + for tcgen05.mma. This descriptor is a 64-bit value which describes the + properties of multiplicand matrix in shared memory including its location + in the shared memory of the current CTA. + + ``` + +-----------+------+------------------------------------------------------+ + | Bit-field | Size | Description | + +-----------+------+------------------------------------------------------+ + | 0-13 | 14 | Matrix start address | + | 14-15 | 2 | Reserved | + | 16-29 | 14 | Leading dim relative-offset (or) absolute-address | + | 30-31 | 2 | Reserved | + | 32-45 | 14 | Stride dimension byte offset | + | 46-48 | 3 | Fixed constant value of 0b001 | + | 49-51 | 3 | Matrix base offset | + | 52 | 1 | Leading dimension stride mode: | + | | | 0: byte offset relative | + | | | 1: byte address absolute | + | 53-60 | 8 | Fixed constant value of 0xb00000000 | + | 61-63 | 3 | Swizzling mode: | + | | | 0: No swizzling | + | | | 1: 128-Byte with 32B atomic swizzling | + | | | 2: 128-Byte swizzling | + | | | 4: 64-Byte swizzling | + | | | 6: 32-Byte swizzling | + | | | (Values 3, 5 and 7 are invalid) | + +-----------+------+------------------------------------------------------+ + ``` + + Example: + ```mlir + %desc = nvvm.tcgen05.mma_smem_desc (%startAddr, %leadingDimOffset, %strideDimOffset, + %baseOffset, %leadingDimMode, %swizzleMode) : (i32, i32, i32, i8, i1, i8) -> i64 + ``` + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-shared-memory-descriptor) + }]; + let arguments = (ins + NVVM_T32BitSignlessInteger:$startAddr, + NVVM_T32BitSignlessInteger:$leadingDimOffset, + NVVM_T32BitSignlessInteger:$strideDimOffset, + NVVM_T8BitSignlessInteger2:$baseOffset, + NVVM_T1BitSignlessInteger:$leadingDimMode, + NVVM_T8BitSignlessInteger2:$swizzleMode + ); + let results = (outs + NVVM_T64BitSignlessInteger2:$res + ); +} +def NVVM_Tcgen05RelinquishAllocPermitOp : NVVM_Op<"tcgen05.relinquish_alloc_permit"> { + let description = [{ + The `tcgen05.relinquish_alloc_permit` Op specifies that the CTA + of the executing thread is relinquishing the right to allocate + Tensor Memory. So, it is illegal for a CTA to perform `tcgen05.alloc` + after any of its constituent threads execute `tcgen05.relinquish_alloc_permit`. + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-memory-alloc-manage-instructions) + }]; + let arguments = (ins + OptionalAttr:$group + ); + let results = (outs); +} +def NVVM_Tcgen05ShiftOp : NVVM_Op<"tcgen05.shift"> { + let description = [{ + The `tcgen05.shift` is an asynchronous instruction which initiates + the shifting of 32-byte elements downwards across all the rows, + except the last, by one row. The operand `taddr` specifies the base + address of the matrix in Tensor Memory whose rows must be down shifted. + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-instructions-tcgen05-shift) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace6:$taddr, + OptionalAttr:$group + ); + let results = (outs); +} +def NVVM_Tcgen05StOp : NVVM_Op<"tcgen05.st"> { + let description = [{ + Instruction `tcgen05.st` asynchronously stores data from the source register `r` + into the Tensor Memory at the location specified by the 32-bit address operand + `tmemAddr`, collectively across all threads of the warps. + + The `shape` and the `num` attribute together determines the total dimension of + the data which is stored to the Tensor Memory. The `shape` indicates the base + dimension of data to be accessed. The `num` attribute indicates the repeat + factor on the base dimension resulting in the total dimension of the data that + is accessed. + + The shape `16x32bx2` performs two accesses into Tensor Memory of the shape + `16x32b`. The base address of the first access is specified by `tmemAddr` + and the base address of the second access is specified by + `tmemAddr + offset`, where `offset` is an immediate argument. + + The unit attribute `unpack` can be used to unpack a 32-bit element + in the register into two 16-bit elements and store them in adjacent columns. + + The following table describes the size of the vector for various combinations + of `num` and `shape` attributes: + ``` + |=====================================================================| + | num/shape | 16x32bx2/16x64b/32x32b | 16x128b | 16x256b | + |=====================================================================| + | x1 | 1 | 2 | 4 | + | x2 | 2 | 4 | 8 | + | x4 | 4 | 8 | 16 | + | x8 | 8 | 16 | 32 | + | x16 | 16 | 32 | 64 | + | x32 | 32 | 64 | 128 | + | x64 | 64 | 128 | NA | + | x128 | 128 | NA | NA | + |=====================================================================| + ``` + + Example: + ```mlir + nvvm.tcgen05.st %tmemAddr, %val, %offset unpack { + shape = #nvvm.tcgen05_ldst_shape, + } : <2xi32> + ``` + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-instructions-tcgen05-st) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace6:$tmemAddr, + NVVM_T32BitSignlessIntegerOrVector2:$val, + Optional:$offset, + UnitAttr:$unpack, + NVVM_Tcgen05LdStShapeAttr:$shape + ); + let results = (outs); +} +def NVVM_Tcgen05WaitOp : NVVM_Op<"tcgen05.wait"> { + let description = [{ + The `tcgen05.wait` causes the executing thread to block until + all prior `tcgen05.ld` operations issued by the executing thread + have completed. Similarly, the `tcgen05.wait` causes the executing + thread to block until all prior `tcgen05.st` operations issued by the + executing thread have completed. + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-instructions-tcgen05-wait) + }]; + let arguments = (ins + NVVM_Tcgen05WaitKindAttr:$kind + ); + let results = (outs); +} +def NVVM_Tcgen05MmaSmemDescOpV2 : NVVM_Op<"tcgen05_mma_smem_desc_v2", [MemoryEffects<[MemRead]>]> { + let description = [{ + The `nvvm.tcgen05_mma_smem_desc_v2` constructs a Shared Memory descriptor for use as the MMA + Shared Memory operand. It configures the memory layout according to the provided parameters. + + The descriptor contains the following bitfields: + - Bits 63-61 (3 bits): Swizzle descriptor type + 0 = SWIZZLE_NONE matrix descriptor + 1 = SWIZZLE_128B_ATOM_32B + 2 = SWIZZLE_128B matrix descriptor + 3 = RESERVED + 4 = SWIZZLE_64B descriptor + 5 = RESERVED + 6 = SWIZZLE_32B descriptor + 7 = RESERVED + - Bits 60-54 (7 bits): Reserved + - Bit 53 (1 bit): UTCQMMA.LUTB Matrix B K_Segment start offset in K=128 (48B) block + 0: K[0:63] = GDESC.start_address + 0B + 1: K[64:127] = GDESC.start_address + 24B + - Bit 52 (1 bit): Matrix A or B leading dimension mode + 0 = BYTE_OFFSET (legacy) + 1 = NEXT_BUFFER_ADDR (circular buffer) + - Bits 51-49 (3 bits): Matrix A or B base offset (valid only for matrix descriptor 1, 2 or 3, 4) + - Bits 48-46 (3 bits): Matrix A or B descriptor version, needs to be set to 1 + - Bits 45-32 (14 bits): Matrix A or B stride dimension byte offset, 16 byte aligned, 4 LSBs not included + - Bit 31 (1 bit): Reserved + - Bits 30-16 (15 bits): Matrix A or B leading dimension byte offset or next buffer start address + If LEADING_DIM_MODE == BYTE_OFFSET: Matrix A or B leading dimension byte offset + If LEADING_DIM_MODE == NEXT_BUFFER_ADDR: Matrix A or B next buffer start address + Offset or address is 16 byte aligned, 4 LSBs not included + - Bit 15 (1 bit): Reserved + - Bits 14-0 (15 bits): Matrix A or B start address, 16 byte aligned, 4 LSBs not included + }]; + let arguments = (ins + NVVM_T32BitSignlessInteger:$startAddress, + NVVM_T32BitSignlessInteger:$leadingDimOffset, + NVVM_T32BitSignlessInteger:$strideDimOffset, + NVVM_T8BitSignlessInteger2:$descriptorVersion, + NVVM_T8BitSignlessInteger2:$baseOffset, + NVVM_T1BitSignlessInteger:$leadingDimMode, + NVVM_T1BitSignlessInteger:$kSegmentOffset, + NVVM_T8BitSignlessInteger2:$swizzleType + ); + let results = (outs + NVVM_T64BitSignlessInteger2:$res + ); +} +def NVVM_TensormapCpFenceProxyOp : NVVM_Op<"tensormap.cp_fenceproxy"> { + let description = [{ + The `tensormap.cp_fenceproxy` is a fused copy and fence operation. + Instructions perform the following operations in order: + + Copies data of size specified by the `size` argument, in bytes, from the + location specified by the address operand `src` in shared memory to the + location specified by the address operand `dst` in the global memory, in + the generic proxy. + + Establishes a uni-directional proxy release pattern on the ordering from + the copy operation to the subsequent access performed in the tensormap + proxy on the address `dst` + + The valid value of `size` operand is 128 and must be an immediate. + + The operands `src` and `dst` specify non-generic addresses in shared::cta and + global state space respectively + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace1:$dst, + NVVM_LLVMPointerInAddressSpace3:$src, + NVVM_T32BitSignlessInteger:$size, + NVVM_MemScopeKindAttr:$scope + ); + let results = (outs); +} +def NVVM_TensormapReplaceOp : NVVM_Op<"tensormap.replace"> { + let description = [{ + The `nvvm.tensormap.replace` replaces the specified field of the tensor-map + object at the location specified by `addr` with a new value (specified by + `new_value` or `new_value_attr`). + + The `field` argument specifies the field of the tensor-map object to + replace. + + `new_value` is an `i32`/`i64` argument that specifies the new value to + replace the `field` with for the `global_address`, `rank`, `box_dim`, + `global_dim`, `global_stride`, and `element_stride` fields. It must be an + `i64` for the `global_address` and `global_stride` fields and `i32` for the + remaining fields. + + For `rank`, `new_value` must be one less than the desired tensor rank as + this field uses zero-based numbering. + + `new_value_attr` is an attribute that specifies the new value to replace + the `field` with for the `elemtype`, `interleave_layout`, `swizzle_mode`, + `swizzle_atomicity`, and `fill_mode` fields. It takes the place of + `new_value` for these fields. It must be a valid attribute corresponding to + the `field` type. + + The ordinal `ord` is an immediate integer argument that specifies the + ordinal of the `field` across the tensor which needs to be replaced and is + required only for the `box_dim`, `global_dim`, `global_stride`, and + `element_stride` fields. + + [For more information, see PTX ISA.](https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-tensormap-replace) + }]; + let arguments = (ins + NVVM_LLVMPointerInAddressSpace14:$addr, + Optional:$new_value, + NVVM_TensormapFieldAttr:$field, + OptionalAttr:$ord, + OptionalAttr:$new_value_attr + ); + let results = (outs); +} +def NVVM_TraceMarkOp : NVVM_Op<"trace.mark", [AttrSizedOperandSegments]> { + let description = [{ + The `nvvm.trace.mark` operation introduces trace events for profiling + programs. + + It takes `event_type`, a 12-bit value in the range 0-4095, which denotes + the type of the event. Additionally, it takes `domain` and `event`, which + are strings that identify the domain and event name of the trace event. + Finally, it takes an optional 32-bit or 64-bit payload, an optional + payload descriptor (required when payload is provided), and a predicate to + conditionally execute the trace event. + + Examples: + + ```mlir + // Basic trace event + nvvm.trace.mark event_type = 42, domain = "myDomain", event = "myEvent" + + // With 32-bit payload + nvvm.trace.mark event_type = 42, payload = %p, + domain = "myDomain", event = "myEvent", payload_descriptor = "counter" + : i32 + + // Conditional trace event + nvvm.trace.mark event_type = 42, predicate = %pred, + domain = "myDomain", event = "myEvent" + ``` + }]; + let arguments = (ins + Optional:$payload, + Optional:$predicate, + I32Attr:$eventType, + StrAttr:$domain, + StrAttr:$event, + OptionalAttr:$payloadDescriptor + ); + let results = (outs); +} +def NVVM_VoteSyncOp : NVVM_InferredOp<"vote.sync"> { + let description = [{ + The `vote.sync` op will cause executing thread to wait until all non-exited + threads corresponding to membermask have executed `vote.sync` with the same + qualifiers and same membermask value before resuming execution. + + The vote operation kinds are: + - `any`: True if source predicate is True for some thread in membermask. + - `all`: True if source predicate is True for all non-exited threads in + membermask. + - `uni`: True if source predicate has the same value in all non-exited + threads in membermask. + - `ballot`: In the ballot form, the destination result is a 32 bit integer. + In this form, the predicate from each thread in membermask are copied into + the corresponding bit position of the result, where the bit position + corresponds to the thread's lane id. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-vote-sync) + }]; + let arguments = (ins + NVVM_T32BitSignlessInteger:$mask, + NVVM_T1BitSignlessInteger:$pred, + NVVM_VoteSyncKindAttr:$kind + ); + let results = (outs + NVVM_T32BitSignlessIntegerOr1:$res + ); +} +def NVVM_WgmmaGroupSyncAlignedOp : NVVM_Op<"wgmma.commit.group.sync.aligned"> { + let description = [{ + Commits all prior uncommitted warpgroup level matrix multiplication operations. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-instructions-wgmma-commit-group) + }]; + let arguments = (ins); + let results = (outs); +} +def NVVM_WgmmaFenceAlignedOp : NVVM_Op<"wgmma.fence.aligned"> { + let description = [{ + Enforce an ordering of register accesses between warpgroup level matrix + multiplication and other operations. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-instructions-wgmma-fence) + }]; + let arguments = (ins); + let results = (outs); +} +def NVVM_WgmmaMmaAsyncOp : NVVM_Op<"wgmma.mma_async"> { + let description = [{ + The warpgroup (128 threads) level matrix multiply and accumulate operation + has either of the following forms, where matrix D is called accumulator: + D = A * B + D + D = A * B, where the input from accumulator D is disabled. + + Supported shapes: + ``` + |--------------|--------------|------------|--------------|---------------| + | | | | |f16+=e4m3*e4m3 | + | | | | |f16+=e5m2*e5m2 | + |f32+=tf32*tf32|f16+=f16 *f16 | s32+=s8*s8 |s32 += b1 * b1|f16+=e5m2*e4m3 | + | |f32+=f16 *f16 | s32+=u8*u8 | |f16+=e4m3*e5m2 | + | |f32+=bf16*bf16| s32+=u8*u8 | |f16+=e4m3*e5m2 | + | |f32+=bf16*bf16| s32+=s8*u8 | |f32+=e4m3*e4m3 | + | | | s32+=u8*s8 | |f32+=e5m2*e5m2 | + | | | | |f32+=e4m3*e5m2 | + | | | | |f32+=e4m3*e5m2 | + |--------------|--------------|------------|--------------|---------------| + | .m64n8k8 | .m64n8k16 | .m64n8k32 | .m64n8k256 | .m64n8k32 | + | .m64n16k8 | .m64n16k16 | .m64n16k32 | .m64n16k256 | .m64n16k32 | + | .m64n24k8 | .m64n24k16 | .m64n24k32 | .m64n24k256 | .m64n24k32 | + | .m64n32k8 | .m64n32k16 | .m64n32k32 | .m64n32k256 | .m64n32k32 | + | .m64n40k8 | .m64n40k16 | .m64n48k32 | .m64n48k256 | .m64n40k32 | + | .m64n48k8 | .m64n48k16 | .m64n64k32 | .m64n64k256 | .m64n48k32 | + | .m64n56k8 | .m64n56k16 | .m64n80k32 | .m64n80k256 | .m64n56k32 | + | .m64n64k8 | .m64n64k16 | .m64n96k32 | .m64n96k256 | .m64n64k32 | + | .m64n72k8 | .m64n72k16 | .m64n112k32| .m64n112k256 | .m64n72k32 | + | .m64n80k8 | .m64n80k16 | .m64n128k32| .m64n128k256 | .m64n80k32 | + | .m64n88k8 | .m64n88k16 | .m64n144k32| .m64n144k256 | .m64n88k32 | + | .m64n96k8 | .m64n96k16 | .m64n160k32| .m64n160k256 | .m64n96k32 | + | .m64n104k8 | .m64n104k16 | .m64n176k32| .m64n176k256 | .m64n104k32 | + | .m64n112k8 | .m64n112k16 | .m64n192k32| .m64n192k256 | .m64n112k32 | + | .m64n120k8 | .m64n120k16 | .m64n208k32| .m64n208k256 | .m64n120k32 | + | .m64n128k8 | .m64n128k16 | .m64n224k32| .m64n224k256 | .m64n128k32 | + | .m64n136k8 | .m64n136k16 | .m64n240k32| .m64n240k256 | .m64n136k32 | + | .m64n144k8 | .m64n144k16 | .m64n256k32| .m64n256k256 | .m64n144k32 | + | .m64n152k8 | .m64n152k16 | | | .m64n152k32 | + | .m64n160k8 | .m64n160k16 | | | .m64n160k32 | + | .m64n168k8 | .m64n168k16 | | | .m64n168k32 | + | .m64n176k8 | .m64n176k16 | | | .m64n176k32 | + | .m64n184k8 | .m64n184k16 | | | .m64n184k32 | + | .m64n192k8 | .m64n192k16 | | | .m64n192k32 | + | .m64n200k8 | .m64n200k16 | | | .m64n200k32 | + | .m64n208k8 | .m64n208k16 | | | .m64n208k32 | + | .m64n216k8 | .m64n216k16 | | | .m64n216k32 | + | .m64n224k8 | .m64n224k16 | | | .m64n224k32 | + | .m64n232k8 | .m64n232k16 | | | .m64n232k32 | + | .m64n240k8 | .m64n240k16 | | | .m64n240k32 | + | .m64n248k8 | .m64n248k16 | | | .m64n248k32 | + | .m64n256k8 | .m64n256k16 | | | .m64n256k32 | + |--------------|--------------|------------|--------------|---------------| + ``` + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-instructions) + }]; + let arguments = (ins + NVVM_LLVMStructureType:$inouts, + NVVM_T64BitSignlessInteger:$descriptorA, + NVVM_T64BitSignlessInteger:$descriptorB, + NVVM_MMAShapeAttr:$shape, + NVVM_WGMMATypesAttr:$typeA, + NVVM_WGMMATypesAttr:$typeB, + NVVM_WGMMATypesAttr:$typeD, + NVVM_WGMMAScaleOutAttr:$scaleD, + NVVM_WGMMAScaleInAttr:$scaleA, + NVVM_WGMMAScaleInAttr:$scaleB, + NVVM_MMALayoutAttr:$layoutA, + NVVM_MMALayoutAttr:$layoutB, + OptionalAttr:$satfinite + ); + let results = (outs + NVVM_LLVMStructureType:$results_ + ); +} +def NVVM_WgmmaWaitGroupSyncOp : NVVM_Op<"wgmma.wait.group.sync.aligned"> { + let description = [{ + Signal the completion of a preceding warpgroup operation. + + [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-instructions-wgmma-wait-group) + }]; + let arguments = (ins + I64Attr:$group + ); + let results = (outs); +} +def NVVM_WMMALoadOp : NVVM_Op<"wmma.load"> { + let arguments = (ins + NVVM_LLVMPointerType:$ptr, + NVVM_T32BitSignlessInteger:$stride, + I32Attr:$m, + I32Attr:$n, + I32Attr:$k, + NVVM_MMALayoutAttr:$layout, + NVVM_MMATypesAttr:$eltype, + NVVM_MMAFragAttr:$frag + ); + let results = (outs + NVVM_LLVMStructureTypeOr64Bit:$res + ); +} +def NVVM_WMMAMmaOp : NVVM_Op<"wmma.mma"> { + let arguments = (ins + Variadic:$args, + I32Attr:$m, + I32Attr:$n, + I32Attr:$k, + NVVM_MMALayoutAttr:$layoutA, + NVVM_MMALayoutAttr:$layoutB, + NVVM_MMATypesAttr:$eltypeA, + NVVM_MMATypesAttr:$eltypeB + ); + let results = (outs + NVVM_LLVMStructureType:$res + ); +} +def NVVM_WMMAStoreOp : NVVM_Op<"wmma.store"> { + let arguments = (ins + NVVM_LLVMPointerType:$ptr, + Variadic:$args, + NVVM_T32BitSignlessInteger:$stride, + I32Attr:$m, + I32Attr:$n, + I32Attr:$k, + NVVM_MMALayoutAttr:$layout, + NVVM_MMATypesAttr:$eltype + ); + let results = (outs); +} + +#endif // MLIR_DIALECTS_NVVM_NVVMOPS_TD diff --git a/mlir/dialects/nvvm/capi.cc b/mlir/dialects/nvvm/capi.cc new file mode 100644 index 00000000..ff71af81 --- /dev/null +++ b/mlir/dialects/nvvm/capi.cc @@ -0,0 +1,1964 @@ +//===- capi.cc - C API for CuTe's nvvm dialect ----------------------------===// + +#include "capi.h" + +#include "mlir/dialects/nvvm/NVVMDialect.h" + +#include "mlir/CAPI/IR.h" +#include "mlir/CAPI/Registration.h" +#include "mlir/CAPI/Support.h" + +// Registration + +MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(CuteNVVM, cute_nvvm, + mlir::cutlass_compiler::nvvm::NVVMDialect) + +// Attributes + +namespace nvvm = mlir::cutlass_compiler::nvvm; + +// Unwraps `handle` into `out`: false when it is of another context or not a +// T, or when it is null and not optional. +template +static bool unwrapAs(mlir::MLIRContext *ctx, H handle, bool optional, T &out) { + auto value = unwrap(handle); + if (!value) + return optional; + if (value.getContext() != ctx) + return false; + out = llvm::dyn_cast(value); + return static_cast(out); +} + +bool mlirAttributeIsACuteNVVMAtomicOpKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMAtomicOpKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeAtomicOpKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::AtomicOpKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMAtomicOpKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMBarrierReduction(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMBarrierReductionAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeBarrierReduction(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::BarrierReductionAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMBarrierReductionAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMBarrierReduxKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMBarrierReduxKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeBarrierReduxKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::BarrierReduxKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMBarrierReduxKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMBlockScaleFormat(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMBlockScaleFormatAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeBlockScaleFormat(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::BlockScaleFormatAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMBlockScaleFormatAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMCTAGroupKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMCTAGroupKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeCTAGroupKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::CTAGroupKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMCTAGroupKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMCVTPackFloatKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMCVTPackFloatKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeCVTPackFloatKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::CVTPackFloatKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMCVTPackFloatKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMCacheEvictionPriority(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMCacheEvictionPriorityAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeCacheEvictionPriority(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::CacheEvictionPriorityAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMCacheEvictionPriorityAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMClusterLaunchControlQueryType(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute +mlirCuteNVVMClusterLaunchControlQueryTypeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeClusterLaunchControlQueryType( + value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::ClusterLaunchControlQueryTypeAttr::get(ctx, *valueValue)); +} +uint32_t +mlirCuteNVVMClusterLaunchControlQueryTypeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVVMCompareOpKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMCompareOpKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeCompareOpKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::CompareOpKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMCompareOpKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMConvertFP4Type(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMConvertFP4TypeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeConvertFP4Type(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::ConvertFP4TypeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMConvertFP4TypeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMConvertFP8Type(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMConvertFP8TypeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeConvertFP8Type(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::ConvertFP8TypeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMConvertFP8TypeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMConvertScaleKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMConvertScaleKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeConvertScaleKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::ConvertScaleKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMConvertScaleKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMDotAccumulateType(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMDotAccumulateTypeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeDotAccumulateType(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::DotAccumulateTypeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMDotAccumulateTypeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMEvictKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMEvictKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeEvictKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::EvictKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMEvictKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMFPRoundingMode(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMFPRoundingModeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeFPRoundingMode(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::FPRoundingModeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMFPRoundingModeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMGridDepActionKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMGridDepActionKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeGridDepActionKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::GridDepActionKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMGridDepActionKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMIntegerRoundingMode(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMIntegerRoundingModeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeIntegerRoundingMode(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::IntegerRoundingModeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMIntegerRoundingModeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVML2PrefetchSize(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVML2PrefetchSizeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeL2PrefetchSize(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::L2PrefetchSizeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVML2PrefetchSizeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMLdStMatrixEltType(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMLdStMatrixEltTypeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeLdStMatrixEltType(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::LdStMatrixEltTypeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMLdStMatrixEltTypeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMLoadCacheModifierExtKind(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute mlirCuteNVVMLoadCacheModifierExtKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeLoadCacheModifierExtKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::LoadCacheModifierExtKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMLoadCacheModifierExtKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMLoadCacheModifierKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMLoadCacheModifierKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeLoadCacheModifierKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::LoadCacheModifierKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMLoadCacheModifierKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMLoadShape(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMLoadShapeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeLoadShape(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::LoadShapeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMLoadShapeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMLoadSrcFormat(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMLoadSrcFormatAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeLoadSrcFormat(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::LoadSrcFormatAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMLoadSrcFormatAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMMBarrierLayout(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMMBarrierLayoutAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeMBarrierLayout(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::MBarrierLayoutAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMMBarrierLayoutAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMMBarrierPhase(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMMBarrierPhaseAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeMBarrierPhase(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::MBarrierPhaseAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMMBarrierPhaseAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMMBarrierScopeKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMMBarrierScopeKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeMBarrierScopeKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::MBarrierScopeKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMMBarrierScopeKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMMBarrierSpaceKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMMBarrierSpaceKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeMBarrierSpaceKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::MBarrierSpaceKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMMBarrierSpaceKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMMBarrierTxnKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMMBarrierTxnKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeMBarrierTxnKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::MBarrierTxnKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMMBarrierTxnKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMMBarrierWaitKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMMBarrierWaitKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeMBarrierWaitKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::MBarrierWaitKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMMBarrierWaitKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMMMAB1Op(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMMMAB1OpAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeMMAB1Op(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::MMAB1OpAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMMMAB1OpAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMMMABlockScaleKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMMMABlockScaleKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeMMABlockScaleKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::MMABlockScaleKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMMMABlockScaleKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMMMACtaCount(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMMMACtaCountAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeMMACtaCount(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::MMACtaCountAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMMMACtaCountAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMMMAFrag(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMMMAFragAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeMMAFrag(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::MMAFragAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMMMAFragAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMMMAIntOverflow(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMMMAIntOverflowAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeMMAIntOverflow(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::MMAIntOverflowAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMMMAIntOverflowAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMMMAKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMMMAKindAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeMMAKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::MMAKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMMMAKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMMMALayout(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMMMALayoutAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeMMALayout(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::MMALayoutAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMMMALayoutAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMMMATypes(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMMMATypesAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeMMATypes(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::MMATypesAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMMMATypesAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMMatchSyncKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMMatchSyncKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeMatchSyncKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::MatchSyncKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMMatchSyncKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMMemOrderKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMMemOrderKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeMemOrderKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::MemOrderKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMMemOrderKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMMemScopeKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMMemScopeKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeMemScopeKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::MemScopeKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMMemScopeKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMMulMode(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMMulModeAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeMulMode(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::MulModeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMMulModeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMNVVMMemorySpace(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMNVVMMemorySpaceAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeNVVMMemorySpace(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::NVVMMemorySpaceAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMNVVMMemorySpaceAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMPermuteMode(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMPermuteModeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizePermuteMode(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::PermuteModeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMPermuteModeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMPrefetchCacheLevel(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMPrefetchCacheLevelAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizePrefetchCacheLevel(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::PrefetchCacheLevelAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMPrefetchCacheLevelAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMProxyKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMProxyKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeProxyKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::ProxyKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMProxyKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMReductionKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMReductionKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeReductionKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::ReductionKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMReductionKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMReductionOp(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMReductionOpAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeReductionOp(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::ReductionOpAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMReductionOpAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMReductionType(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMReductionTypeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeReductionType(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::ReductionTypeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMReductionTypeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMSPCompressElemSize(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMSPCompressElemSizeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeSPCompressElemSize(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::SPCompressElemSizeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMSPCompressElemSizeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMSPCompressFactorType(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMSPCompressFactorTypeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeSPCompressFactorType(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::SPCompressFactorTypeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMSPCompressFactorTypeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMSPCompressIndexSize(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMSPCompressIndexSizeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeSPCompressIndexSize(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::SPCompressIndexSizeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMSPCompressIndexSizeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMSPCompressOpKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMSPCompressOpKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeSPCompressOpKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::SPCompressOpKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMSPCompressOpKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMSPCompressRepFactor(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMSPCompressRepFactorAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeSPCompressRepFactor(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::SPCompressRepFactorAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMSPCompressRepFactorAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMSPDecompressElemSize(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMSPDecompressElemSizeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeSPDecompressElemSize(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::SPDecompressElemSizeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMSPDecompressElemSizeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMSPDecompressFactorType(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMSPDecompressFactorTypeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeSPDecompressFactorType(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::SPDecompressFactorTypeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMSPDecompressFactorTypeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMSPDecompressIndexSize(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMSPDecompressIndexSizeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeSPDecompressIndexSize(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::SPDecompressIndexSizeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMSPDecompressIndexSizeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMSPDecompressRepFactor(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMSPDecompressRepFactorAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeSPDecompressRepFactor(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::SPDecompressRepFactorAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMSPDecompressRepFactorAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMSaturationMode(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMSaturationModeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeSaturationMode(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::SaturationModeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMSaturationModeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMSaturationModeKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMSaturationModeKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeSaturationModeKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::SaturationModeKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMSaturationModeKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMScaleVecSize(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMScaleVecSizeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeScaleVecSize(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::ScaleVecSizeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMScaleVecSizeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMSetMaxRegisterAction(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMSetMaxRegisterActionAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeSetMaxRegisterAction(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::SetMaxRegisterActionAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMSetMaxRegisterActionAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMSharedSpace(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMSharedSpaceAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeSharedSpace(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::SharedSpaceAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMSharedSpaceAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMShflKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMShflKindAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeShflKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::ShflKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMShflKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMSparsityFormat(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMSparsityFormatAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeSparsityFormat(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::SparsityFormatAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMSparsityFormatAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMStateSpace(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMStateSpaceAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeStateSpace(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::StateSpaceAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMStateSpaceAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMStoreCacheModifierKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMStoreCacheModifierKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeStoreCacheModifierKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::StoreCacheModifierKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMStoreCacheModifierKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMStoreShape(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMStoreShapeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeStoreShape(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::StoreShapeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMStoreShapeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMTCBarParam(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTCBarParamAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeTCBarParam(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::TCBarParamAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMTCBarParamAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMTMALoadMode(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTMALoadModeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeTMALoadMode(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::TMALoadModeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMTMALoadModeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMTMAReduxKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTMAReduxKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeTMAReduxKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::TMAReduxKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMTMAReduxKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMTMAStoreMode(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTMAStoreModeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeTMAStoreMode(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::TMAStoreModeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMTMAStoreModeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMTcgen05CpMulticast(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTcgen05CpMulticastAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeTcgen05CpMulticast(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::Tcgen05CpMulticastAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMTcgen05CpMulticastAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMTcgen05CpShape(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTcgen05CpShapeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeTcgen05CpShape(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::Tcgen05CpShapeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMTcgen05CpShapeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMTcgen05CpSrcFormat(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTcgen05CpSrcFormatAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeTcgen05CpSrcFormat(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::Tcgen05CpSrcFormatAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMTcgen05CpSrcFormatAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMTcgen05FenceKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTcgen05FenceKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeTcgen05FenceKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::Tcgen05FenceKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMTcgen05FenceKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMTcgen05LdStShape(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTcgen05LdStShapeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeTcgen05LdStShape(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::Tcgen05LdStShapeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMTcgen05LdStShapeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMTcgen05MMABlockScale(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTcgen05MMABlockScaleAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeTcgen05MMABlockScale(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::Tcgen05MMABlockScaleAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMTcgen05MMABlockScaleAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMTcgen05MMACollectorBBuffer(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTcgen05MMACollectorBBufferAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeTcgen05MMACollectorBBuffer( + value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::Tcgen05MMACollectorBBufferAttr::get(ctx, *valueValue)); +} +uint32_t +mlirCuteNVVMTcgen05MMACollectorBBufferAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsACuteNVVMTcgen05MMACollectorOp(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTcgen05MMACollectorOpAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeTcgen05MMACollectorOp(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::Tcgen05MMACollectorOpAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMTcgen05MMACollectorOpAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMTcgen05MMAKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTcgen05MMAKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeTcgen05MMAKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::Tcgen05MMAKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMTcgen05MMAKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMTcgen05WaitKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTcgen05WaitKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeTcgen05WaitKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::Tcgen05WaitKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMTcgen05WaitKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMTensormapElemtype(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTensormapElemtypeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeTensormapElemtype(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::TensormapElemtypeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMTensormapElemtypeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMTensormapField(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTensormapFieldAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeTensormapField(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::TensormapFieldAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMTensormapFieldAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMTensormapFillMode(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTensormapFillModeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeTensormapFillMode(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::TensormapFillModeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMTensormapFillModeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMTensormapInterleaveLayout(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTensormapInterleaveLayoutAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeTensormapInterleaveLayout(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::TensormapInterleaveLayoutAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMTensormapInterleaveLayoutAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMTensormapSwizzleAtomicity(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTensormapSwizzleAtomicityAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeTensormapSwizzleAtomicity(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::TensormapSwizzleAtomicityAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMTensormapSwizzleAtomicityAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMTensormapSwizzleMode(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTensormapSwizzleModeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeTensormapSwizzleMode(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::TensormapSwizzleModeAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMTensormapSwizzleModeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMTmemLayout(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTmemLayoutAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeTmemLayout(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::TmemLayoutAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMTmemLayoutAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMValidatePattern(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMValidatePatternAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeValidatePattern(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::ValidatePatternAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMValidatePatternAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMVoteSyncKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMVoteSyncKindAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeVoteSyncKind(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::VoteSyncKindAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMVoteSyncKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMWGMMAScaleIn(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMWGMMAScaleInAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeWGMMAScaleIn(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::WGMMAScaleInAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMWGMMAScaleInAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMWGMMAScaleOut(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMWGMMAScaleOutAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = + ::mlir::cutlass_compiler::nvvm::symbolizeWGMMAScaleOut(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::WGMMAScaleOutAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMWGMMAScaleOutAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMWGMMATypes(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMWGMMATypesAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeWGMMATypes(value); + if (!valueValue) + return {nullptr}; + return wrap(nvvm::WGMMATypesAttr::get(ctx, *valueValue)); +} +uint32_t mlirCuteNVVMWGMMATypesAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsACuteNVVMLdStMatrixShape(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMLdStMatrixShapeAttrGet(MlirContext context, int m, + int n) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(nvvm::LdStMatrixShapeAttr::get(ctx, m, n)); +} +int mlirCuteNVVMLdStMatrixShapeAttrGetM(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getM(); +} +int mlirCuteNVVMLdStMatrixShapeAttrGetN(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getN(); +} + +bool mlirAttributeIsACuteNVVMMMAShape(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMMMAShapeAttrGet(MlirContext context, int m, int n, + int k) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(nvvm::MMAShapeAttr::get(ctx, m, n, k)); +} +int mlirCuteNVVMMMAShapeAttrGetM(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getM(); +} +int mlirCuteNVVMMMAShapeAttrGetN(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getN(); +} +int mlirCuteNVVMMMAShapeAttrGetK(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getK(); +} + +bool mlirAttributeIsACuteNVVMTarget(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} +MlirAttribute mlirCuteNVVMTargetAttrGet(MlirContext context, int O, + MlirStringRef triple, + MlirStringRef chip, + MlirStringRef features, + MlirAttribute flags, MlirAttribute link, + bool verifyTarget) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + ctx->getOrLoadDialect(); + ::mlir::DictionaryAttr flagsValue; + if (!unwrapAs(ctx, flags, true, flagsValue)) + return {nullptr}; + ::mlir::ArrayAttr linkValue; + if (!unwrapAs(ctx, link, true, linkValue)) + return {nullptr}; + return wrap(nvvm::TargetAttr::getChecked( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, O, + unwrap(triple), unwrap(chip), unwrap(features), flagsValue, linkValue, + verifyTarget)); +} +int mlirCuteNVVMTargetAttrGetO(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getO(); +} +MlirStringRef mlirCuteNVVMTargetAttrGetTriple(MlirAttribute attr) { + return wrap(llvm::cast(unwrap(attr)).getTriple()); +} +MlirStringRef mlirCuteNVVMTargetAttrGetChip(MlirAttribute attr) { + return wrap(llvm::cast(unwrap(attr)).getChip()); +} +MlirStringRef mlirCuteNVVMTargetAttrGetFeatures(MlirAttribute attr) { + return wrap(llvm::cast(unwrap(attr)).getFeatures()); +} +MlirAttribute mlirCuteNVVMTargetAttrGetFlags(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getFlags())); +} +MlirAttribute mlirCuteNVVMTargetAttrGetLink(MlirAttribute attr) { + return wrap(static_cast( + llvm::cast(unwrap(attr)).getLink())); +} +bool mlirCuteNVVMTargetAttrGetVerifyTarget(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getVerifyTarget(); +} diff --git a/mlir/dialects/nvvm/capi.h b/mlir/dialects/nvvm/capi.h new file mode 100644 index 00000000..ee604770 --- /dev/null +++ b/mlir/dialects/nvvm/capi.h @@ -0,0 +1,785 @@ +//===- capi.h - C API for CuTe's nvvm dialect ---------------------*- C -*-===// + +#ifndef MLIR_DIALECTS_NVVM_CAPI_H +#define MLIR_DIALECTS_NVVM_CAPI_H + +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Registration + +MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(CuteNVVM, cute_nvvm); + +// Attributes + +// Constructors return null for invalid parameters or handles of another +// context; optional parameters take null handles. Enums are their integer +// values (the dialect's *Enums.td). Getters require a value accepted by the +// matching IsA. + +// `#nvvm.atomic_op`: operations supported by atom instruction +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMAtomicOpKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMAtomicOpKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMAtomicOpKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.reduction`: NVVM barrier reduction operation +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMBarrierReduction(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMBarrierReductionAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMBarrierReductionAttrGetValue(MlirAttribute attr); + +// `#nvvm.barrier_redux_kind`: NVVM barrier redux kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMBarrierReduxKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMBarrierReduxKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMBarrierReduxKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.block_scale_format`: MMA Block Scale Format +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMBlockScaleFormat(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMBlockScaleFormatAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMBlockScaleFormatAttrGetValue(MlirAttribute attr); + +// `#nvvm.cta_group`: NVVM CTA group kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMCTAGroupKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMCTAGroupKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMCTAGroupKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.packfloat_type`: NVVM CVT Pack Float kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMCVTPackFloatKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMCVTPackFloatKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMCVTPackFloatKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.cache_eviction_priority`: NVVM Cache Eviction Priority +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMCacheEvictionPriority(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMCacheEvictionPriorityAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMCacheEvictionPriorityAttrGetValue(MlirAttribute attr); + +// `#nvvm.cluster_launch_control_query_type`: NVVM ClusterLaunchControlQueryType +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMClusterLaunchControlQueryType(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMClusterLaunchControlQueryTypeAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMClusterLaunchControlQueryTypeAttrGetValue(MlirAttribute attr); + +// `#nvvm.op`: Comparison operator encoding +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMCompareOpKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMCompareOpKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMCompareOpKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.convert_fp4_type`: NVVM ConvertFP4Type kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMConvertFP4Type(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMConvertFP4TypeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMConvertFP4TypeAttrGetValue(MlirAttribute attr); + +// `#nvvm.convert_fp8_type`: NVVM ConvertFP8Type kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMConvertFP8Type(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMConvertFP8TypeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMConvertFP8TypeAttrGetValue(MlirAttribute attr); + +// `#nvvm.convert_scale_kind`: NVVM ConvertScale kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMConvertScaleKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMConvertScaleKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMConvertScaleKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.dot_accumulate_type`: NVVM DotAccumulateType +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMDotAccumulateType(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMDotAccumulateTypeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMDotAccumulateTypeAttrGetValue(MlirAttribute attr); + +// `#nvvm.evict_kind`: NVVM L2 Prefetch Size +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMEvictKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMEvictKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMEvictKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.fp_rnd_mode`: NVVM FPRoundingMode kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMFPRoundingMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMFPRoundingModeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMFPRoundingModeAttrGetValue(MlirAttribute attr); + +// `#nvvm.grid_dep_action`: Action kind for grid dependency control +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMGridDepActionKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMGridDepActionKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMGridDepActionKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.int_rnd_mode`: NVVM IntegerRoundingMode kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMIntegerRoundingMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMIntegerRoundingModeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMIntegerRoundingModeAttrGetValue(MlirAttribute attr); + +// `#nvvm.l2_prefetch`: NVVM L2 Prefetch Size +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVML2PrefetchSize(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVML2PrefetchSizeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVML2PrefetchSizeAttrGetValue(MlirAttribute attr); + +// `#nvvm.ld_st_matrix_elt_type`: Element type for ldmatrix and stmatrix +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMLdStMatrixEltType(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMLdStMatrixEltTypeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMLdStMatrixEltTypeAttrGetValue(MlirAttribute attr); + +// `#nvvm.load_cache_modifier_ext`: NVVM load cache modifier kind(Ext) +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMLoadCacheModifierExtKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMLoadCacheModifierExtKindAttrGet( + MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMLoadCacheModifierExtKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.load_cache_modifier`: NVVM load cache modifier kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMLoadCacheModifierKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMLoadCacheModifierKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMLoadCacheModifierKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.load_shape`: shape attribute for ldmatrix +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMLoadShape(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMLoadShapeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMLoadShapeAttrGetValue(MlirAttribute attr); + +// `#nvvm.load_src_format`: source format for ldmatrix +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMLoadSrcFormat(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMLoadSrcFormatAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMLoadSrcFormatAttrGetValue(MlirAttribute attr); + +// `#nvvm.mbarrier_layout`: NVVM MBarrier Layout +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMMBarrierLayout(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMMBarrierLayoutAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMMBarrierLayoutAttrGetValue(MlirAttribute attr); + +// `#nvvm.mbarrier_phase`: NVVM mbarrier phase type +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMMBarrierPhase(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMMBarrierPhaseAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMMBarrierPhaseAttrGetValue(MlirAttribute attr); + +// `#nvvm.mbar_scope`: NVVM MBarrier scope kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMMBarrierScopeKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMMBarrierScopeKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMMBarrierScopeKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.mbar_space`: NVVM MBarrier space kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMMBarrierSpaceKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMMBarrierSpaceKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMMBarrierSpaceKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.mbar_txn_kind`: NVVM MBarrier Transaction kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMMBarrierTxnKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMMBarrierTxnKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMMBarrierTxnKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.mbar_wait`: NVVM MBarrier wait kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMMBarrierWaitKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMMBarrierWaitKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMMBarrierWaitKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.mma_b1op`: MMA binary operations +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMMAB1Op(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMMMAB1OpAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMMMAB1OpAttrGetValue(MlirAttribute attr); + +// `#nvvm.block_scale_kind`: Block Scale Kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMMMABlockScaleKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMMMABlockScaleKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMMMABlockScaleKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.mma_cta_count`: MMA CTA count +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMMACtaCount(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMMMACtaCountAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMMMACtaCountAttrGetValue(MlirAttribute attr); + +// `#nvvm.mma_frag`: NVVM MMA frag type +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMMAFrag(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMMMAFragAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMMMAFragAttrGetValue(MlirAttribute attr); + +// `#nvvm.mma_int_overflow`: MMA overflow options +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMMMAIntOverflow(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMMMAIntOverflowAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMMMAIntOverflowAttrGetValue(MlirAttribute attr); + +// `#nvvm.mma_kind`: MMA operation kind +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMMAKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMMMAKindAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMMMAKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.mma_layout`: NVVM MMA layout +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMMALayout(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMMMALayoutAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMMMALayoutAttrGetValue(MlirAttribute attr); + +// `#nvvm.mma_type`: NVVM MMA types +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMMATypes(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMMMATypesAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMMMATypesAttrGetValue(MlirAttribute attr); + +// `#nvvm.match_sync_kind`: NVVM match sync kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMMatchSyncKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMMatchSyncKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMMatchSyncKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.mem_order`: NVVM Memory Ordering kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMMemOrderKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMMemOrderKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMMemOrderKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.mem_scope`: NVVM Memory Scope kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMMemScopeKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMMemScopeKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMMemScopeKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.mul_mode`: multiply mode attribute +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMulMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMMulModeAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMMulModeAttrGetValue(MlirAttribute attr); + +// `#nvvm.memory_space`: NVVM Memory Space +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMNVVMMemorySpace(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMNVVMMemorySpaceAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMNVVMMemorySpaceAttrGetValue(MlirAttribute attr); + +// `#nvvm.permute_mode`: NVVM permute mode +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMPermuteMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMPermuteModeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMPermuteModeAttrGetValue(MlirAttribute attr); + +// `#nvvm.prefetch_cache_level`: NVVM Prefetch Cache Level +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMPrefetchCacheLevel(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMPrefetchCacheLevelAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMPrefetchCacheLevelAttrGetValue(MlirAttribute attr); + +// `#nvvm.proxy_kind`: Proxy kind +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMProxyKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMProxyKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMProxyKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.reduction_kind`: NVVM Reduction Kind attribute +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMReductionKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMReductionKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMReductionKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.red_op`: Ops supported by red instruction +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMReductionOp(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMReductionOpAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMReductionOpAttrGetValue(MlirAttribute attr); + +// `#nvvm.red_type`: types supported by red instruction +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMReductionType(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMReductionTypeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMReductionTypeAttrGetValue(MlirAttribute attr); + +// `#nvvm.spcomp_elem_size`: Sparse tensor compression element size +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMSPCompressElemSize(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMSPCompressElemSizeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMSPCompressElemSizeAttrGetValue(MlirAttribute attr); + +// `#nvvm.spcomp_factor`: Sparse tensor compression factor type +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMSPCompressFactorType(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMSPCompressFactorTypeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMSPCompressFactorTypeAttrGetValue(MlirAttribute attr); + +// `#nvvm.spcomp_index_size`: Sparse tensor compression index size +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMSPCompressIndexSize(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMSPCompressIndexSizeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMSPCompressIndexSizeAttrGetValue(MlirAttribute attr); + +// `#nvvm.spcomp_op_kind`: spcompress operation kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMSPCompressOpKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMSPCompressOpKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMSPCompressOpKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.spcomp_rep_factor`: Sparse tensor compression repetition factor +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMSPCompressRepFactor(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMSPCompressRepFactorAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMSPCompressRepFactorAttrGetValue(MlirAttribute attr); + +// `#nvvm.spdecomp_elem_size`: Sparse tensor decompression element size +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMSPDecompressElemSize(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMSPDecompressElemSizeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMSPDecompressElemSizeAttrGetValue(MlirAttribute attr); + +// `#nvvm.spdecomp_factor`: Sparse tensor decompression factor type +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMSPDecompressFactorType(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMSPDecompressFactorTypeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMSPDecompressFactorTypeAttrGetValue(MlirAttribute attr); + +// `#nvvm.spdecomp_index_size`: Sparse tensor decompression index size +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMSPDecompressIndexSize(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMSPDecompressIndexSizeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMSPDecompressIndexSizeAttrGetValue(MlirAttribute attr); + +// `#nvvm.spdecomp_rep_factor`: Sparse tensor decompression repetition factor +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMSPDecompressRepFactor(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMSPDecompressRepFactorAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMSPDecompressRepFactorAttrGetValue(MlirAttribute attr); + +// `#nvvm.sat_mode`: NVVM SaturationMode kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMSaturationMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMSaturationModeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMSaturationModeAttrGetValue(MlirAttribute attr); + +// `#nvvm.sat`: NVVM SaturationMode kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMSaturationModeKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMSaturationModeKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMSaturationModeKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.scale_vec_size`: MMA Scale Vector Sizes +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMScaleVecSize(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMScaleVecSizeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMScaleVecSizeAttrGetValue(MlirAttribute attr); + +// `#nvvm.action`: NVVM set max register action +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMSetMaxRegisterAction(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMSetMaxRegisterActionAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMSetMaxRegisterActionAttrGetValue(MlirAttribute attr); + +// `#nvvm.shared_space`: Shared memory space +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMSharedSpace(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMSharedSpaceAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMSharedSpaceAttrGetValue(MlirAttribute attr); + +// `#nvvm.shfl_kind`: NVVM shuffle kind +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMShflKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMShflKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMShflKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.sparsity_format`: MMA Sparsity Format +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMSparsityFormat(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMSparsityFormatAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMSparsityFormatAttrGetValue(MlirAttribute attr); + +// `#nvvm.state_space`: NVVM State Space +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMStateSpace(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMStateSpaceAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMStateSpaceAttrGetValue(MlirAttribute attr); + +// `#nvvm.store_cache_modifier`: NVVM store cache modifier kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMStoreCacheModifierKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMStoreCacheModifierKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMStoreCacheModifierKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.store_shape`: shape attribute for stmatrix +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMStoreShape(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMStoreShapeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMStoreShapeAttrGetValue(MlirAttribute attr); + +// `#nvvm.TCBarParam`: Cluster MMA Barrier Parameter Type +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMTCBarParam(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMTCBarParamAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMTCBarParamAttrGetValue(MlirAttribute attr); + +// `#nvvm.tma_load_mode`: NVVM TMA Load Mode +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMTMALoadMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMTMALoadModeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMTMALoadModeAttrGetValue(MlirAttribute attr); + +// `#nvvm.tma_redux_kind`: NVVM TMA redux kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMTMAReduxKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMTMAReduxKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMTMAReduxKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.tma_store_mode`: NVVM TMA Store Mode +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMTMAStoreMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMTMAStoreModeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMTMAStoreModeAttrGetValue(MlirAttribute attr); + +// `#nvvm.tcgen05_cp_multicast`: tcgen05 cp multicast +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMTcgen05CpMulticast(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMTcgen05CpMulticastAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMTcgen05CpMulticastAttrGetValue(MlirAttribute attr); + +// `#nvvm.tcgen05_cp_shape`: tcgen05 cp shapes +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMTcgen05CpShape(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMTcgen05CpShapeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMTcgen05CpShapeAttrGetValue(MlirAttribute attr); + +// `#nvvm.tcgen05_cp_src_fmt`: tcgen05 cp source format +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMTcgen05CpSrcFormat(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMTcgen05CpSrcFormatAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMTcgen05CpSrcFormatAttrGetValue(MlirAttribute attr); + +// `#nvvm.tcgen05_fence`: NVVM Tcgen05 fence kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMTcgen05FenceKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMTcgen05FenceKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMTcgen05FenceKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.tcgen05_ldst_shape`: allowed 32-bit signless integer cases: 0, 1, 2, +// 3, 4 +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMTcgen05LdStShape(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMTcgen05LdStShapeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMTcgen05LdStShapeAttrGetValue(MlirAttribute attr); + +// `#nvvm.tcgen05_mma_block_scale`: tcgen05.mma block scale attribute +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMTcgen05MMABlockScale(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMTcgen05MMABlockScaleAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMTcgen05MMABlockScaleAttrGetValue(MlirAttribute attr); + +// `#nvvm.tcgen05_mma_collectorb`: tcgen05 MMA Collector Buffer B Attribute +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMTcgen05MMACollectorBBuffer(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMTcgen05MMACollectorBBufferAttrGet( + MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMTcgen05MMACollectorBBufferAttrGetValue(MlirAttribute attr); + +// `#nvvm.tcgen05_mma_collectorop`: tcgen05.mma Collector Buffer Operation +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMTcgen05MMACollectorOp(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMTcgen05MMACollectorOpAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMTcgen05MMACollectorOpAttrGetValue(MlirAttribute attr); + +// `#nvvm.tcgen05_mma_kind`: tcgen05 MMA Supported Types +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMTcgen05MMAKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMTcgen05MMAKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMTcgen05MMAKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.tcgen05_wait`: NVVM Tcgen05 wait kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMTcgen05WaitKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMTcgen05WaitKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMTcgen05WaitKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.tensormap_elemtype`: NVVM Tensormap Elemtype +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMTensormapElemtype(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMTensormapElemtypeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMTensormapElemtypeAttrGetValue(MlirAttribute attr); + +// `#nvvm.tensormap_field`: NVVM Tensormap Field Kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMTensormapField(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMTensormapFieldAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMTensormapFieldAttrGetValue(MlirAttribute attr); + +// `#nvvm.tensormap_fill_mode`: NVVM Tensormap Fill Mode +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMTensormapFillMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMTensormapFillModeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMTensormapFillModeAttrGetValue(MlirAttribute attr); + +// `#nvvm.tensormap_interleave_layout`: NVVM Tensormap Interleave Layout +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMTensormapInterleaveLayout(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMTensormapInterleaveLayoutAttrGet( + MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMTensormapInterleaveLayoutAttrGetValue(MlirAttribute attr); + +// `#nvvm.tensormap_swizzle_atomicity`: NVVM Tensormap Swizzle Atomicity +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMTensormapSwizzleAtomicity(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMTensormapSwizzleAtomicityAttrGet( + MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMTensormapSwizzleAtomicityAttrGetValue(MlirAttribute attr); + +// `#nvvm.tensormap_swizzle_mode`: NVVM Tensormap Swizzle Mode +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMTensormapSwizzleMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMTensormapSwizzleModeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMTensormapSwizzleModeAttrGetValue(MlirAttribute attr); + +// `#nvvm.TmemLayout`: Tensor Memory Layout Enumerated Type +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMTmemLayout(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMTmemLayoutAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMTmemLayoutAttrGetValue(MlirAttribute attr); + +// `#nvvm.validate_pattern`: NVVM validate data pattern +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMValidatePattern(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMValidatePatternAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMValidatePatternAttrGetValue(MlirAttribute attr); + +// `#nvvm.vote_sync_kind`: NVVM vote sync kind +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMVoteSyncKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMVoteSyncKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMVoteSyncKindAttrGetValue(MlirAttribute attr); + +// `#nvvm.wgmma_scale_in`: WGMMA overflow options +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMWGMMAScaleIn(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMWGMMAScaleInAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMWGMMAScaleInAttrGetValue(MlirAttribute attr); + +// `#nvvm.wgmma_scale_out`: WGMMA input predicate +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMWGMMAScaleOut(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMWGMMAScaleOutAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMWGMMAScaleOutAttrGetValue(MlirAttribute attr); + +// `#nvvm.wgmma_type`: NVVM WGMMA types +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMWGMMATypes(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMWGMMATypesAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirCuteNVVMWGMMATypesAttrGetValue(MlirAttribute attr); + +// `#nvvm.ld_st_matrix_shape`: Matrix shape of ldmatrix, stmatrix and movmatrix +MLIR_CAPI_EXPORTED bool +mlirAttributeIsACuteNVVMLdStMatrixShape(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMLdStMatrixShapeAttrGet(MlirContext context, int m, int n); +MLIR_CAPI_EXPORTED int mlirCuteNVVMLdStMatrixShapeAttrGetM(MlirAttribute attr); +MLIR_CAPI_EXPORTED int mlirCuteNVVMLdStMatrixShapeAttrGetN(MlirAttribute attr); + +// `#nvvm.shape`: Shape of an MMA operation +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMMAShape(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMMMAShapeAttrGet(MlirContext context, int m, int n, int k); +MLIR_CAPI_EXPORTED int mlirCuteNVVMMMAShapeAttrGetM(MlirAttribute attr); +MLIR_CAPI_EXPORTED int mlirCuteNVVMMMAShapeAttrGetN(MlirAttribute attr); +MLIR_CAPI_EXPORTED int mlirCuteNVVMMMAShapeAttrGetK(MlirAttribute attr); + +// `#nvvm.target`: GPU target of an NVVM module +MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMTarget(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMTargetAttrGet( + MlirContext context, int O, MlirStringRef triple, MlirStringRef chip, + MlirStringRef features, MlirAttribute flags, MlirAttribute link, + bool verifyTarget); +MLIR_CAPI_EXPORTED int mlirCuteNVVMTargetAttrGetO(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef +mlirCuteNVVMTargetAttrGetTriple(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef +mlirCuteNVVMTargetAttrGetChip(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef +mlirCuteNVVMTargetAttrGetFeatures(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMTargetAttrGetFlags(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirCuteNVVMTargetAttrGetLink(MlirAttribute attr); +MLIR_CAPI_EXPORTED bool +mlirCuteNVVMTargetAttrGetVerifyTarget(MlirAttribute attr); + +#ifdef __cplusplus +} +#endif + +#endif // MLIR_DIALECTS_NVVM_CAPI_H diff --git a/mlir/dialects/nvvm/nvvm.zig b/mlir/dialects/nvvm/nvvm.zig new file mode 100644 index 00000000..14a0e8e2 --- /dev/null +++ b/mlir/dialects/nvvm/nvvm.zig @@ -0,0 +1,311 @@ +//! Builders for the NVVM operations CuTe kernels emit, over the bindings of +//! NVIDIA's `nvvm` dialect (`mlir/dialects/nvvm`, `ops`): the CuTe-DSL +//! compiler's own NVVM, which the upstream dialect differs from. Each builder +//! fixes the attributes the kernels do not choose. + +const std = @import("std"); + +const mlir = @import("mlir"); + +pub const ops = @import("ops.zig"); + +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("cute_nvvm"); +} + +fn enumAttribute(comptime T: type, ctx: *mlir.Context, value: @FieldType(T.InitArgs, "value")) *const mlir.Attribute { + return (T.get(ctx, .{ .value = value }) catch unreachable).attribute(); +} + +/// `#nvvm.cta_group<...>`: CTAs cooperating on a tcgen05 operation. +pub const CtaGroup = ops.CTAGroupKind; +/// `#nvvm.tcgen05_fence<...>`: `tcgen05.fence::{before,after}_thread_sync`. +pub const Tcgen05FenceKind = ops.Tcgen05FenceKind; +/// `#nvvm.tcgen05_wait<...>`: `tcgen05.wait::{ld,st}`. +pub const Tcgen05WaitKind = ops.Tcgen05WaitKind; +/// `#nvvm.tcgen05_mma_kind<...>`: input types of `tcgen05.mma`. +pub const Tcgen05MmaKind = ops.Tcgen05MMAKind; +/// `#nvvm.tcgen05_ldst_shape<...>`: lane/bit shape of `tcgen05.ld` / `tcgen05.st`. +pub const Tcgen05LdStShape = ops.Tcgen05LdStShape; +/// `#nvvm.mem_scope<...>`: scope of a memory operation. +pub const MemScope = ops.MemScopeKind; +/// `#nvvm.mbar_wait<...>`: `test` (non-blocking) or `try` (may suspend) wait. +pub const MBarrierWaitKind = ops.MBarrierWaitKind; +/// `#nvvm.mbar_scope<...>`: scope of an mbarrier wait. +pub const MBarrierScope = ops.MBarrierScopeKind; +/// `#nvvm.proxy_kind<...>`: memory proxy of `fence.proxy`. +pub const ProxyKind = ops.ProxyKind; +/// `#nvvm.shared_space<...>`: `shared::cta` or `shared::cluster`. +pub const SharedSpace = ops.SharedSpace; + +// ============================================================================= +// Kernel attributes +// ============================================================================= + +/// `nvvm.reqntid`: the CTA shape (x, y, z) a kernel is always launched with. +pub fn reqntid(ctx: *mlir.Context, dims: [3]i32) mlir.NamedAttribute { + return .named(ctx, "nvvm.reqntid", .denseArray(ctx, .i32, &dims)); +} + +/// `nvvm.minctasm`: the number of CTAs a kernel must fit on one SM at once. +pub fn minctasm(ctx: *mlir.Context, ctas: i32) mlir.NamedAttribute { + return .named(ctx, "nvvm.minctasm", .int(ctx, .i32, ctas)); +} + +// ============================================================================= +// Special registers and thread synchronization +// ============================================================================= + +/// nvvm.read.ptx.sreg. — an i32 special register, e.g. `tid.x`, `ctaid.y`, +/// `cluster.ctarank`. +pub fn read_sreg(ctx: *mlir.Context, comptime name: []const u8, location: *const mlir.Location) *mlir.Operation { + const builder = comptime blk: { + var fn_name: [name.len]u8 = name[0..name.len].*; + std.mem.replaceScalar(u8, &fn_name, '.', '_'); + break :blk "read_ptx_sreg_" ++ fn_name; + }; + return @field(ops, builder)(ctx, .int(ctx, .i32), .{}, location); +} + +/// nvvm.barrier — `bar.sync 0` over the whole CTA. +pub fn barrier(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return ops.barrier(ctx, .{}, location); +} + +/// nvvm.bar.warp.sync — `bar.warp.sync mask` (i32 lane mask, -1 for the full warp). +pub fn bar_warp_sync(ctx: *mlir.Context, mask: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return ops.bar_warp_sync(ctx, mask, location); +} + +/// nvvm.elect.sync — i1, true in one elected lane of the warp. +pub fn elect_sync(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return ops.elect_sync(ctx, .int(ctx, .i1), .{}, location); +} + +// ============================================================================= +// mbarrier +// ============================================================================= + +/// nvvm.mbarrier.init — initialize the mbarrier at `ptr` (`!llvm.ptr<3>`) to expect +/// `count` (i32) arrivals per phase. +pub fn mbarrier_init(ctx: *mlir.Context, ptr: *const mlir.Value, count: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return ops.mbarrier_init(ctx, ptr, count, .{ .layout = enumAttribute(ops.MBarrierLayoutAttr, ctx, .v0) }, location); +} + +/// nvvm.mbarrier.arrive — arrive `count` (i32) times (release semantics). +pub fn mbarrier_arrive(ctx: *mlir.Context, ptr: *const mlir.Value, count: *const mlir.Value, scope: MemScope, location: *const mlir.Location) *mlir.Operation { + return ops.mbarrier_arrive( + ctx, + ptr, + .{ .count = count, .scope = enumAttribute(ops.MemScopeKindAttr, ctx, scope), .relaxed = .boolean(ctx, false) }, + location, + ); +} + +/// nvvm.mbarrier.arrive.expect_tx — arrive and expect `bytes` (i32) of asynchronous +/// transactions (SM90+). +pub fn mbarrier_arrive_expect_tx(ctx: *mlir.Context, ptr: *const mlir.Value, bytes: *const mlir.Value, scope: MemScope, location: *const mlir.Location) *mlir.Operation { + return ops.mbarrier_arrive_expect_tx( + ctx, + ptr, + bytes, + .{ .scope = enumAttribute(ops.MemScopeKindAttr, ctx, scope), .relaxed = .boolean(ctx, false) }, + location, + ); +} + +/// nvvm.mbarrier.wait.parity — i1, whether phase `parity` (i32) has completed +/// (`test`: never suspends; `try`: may suspend for a system-dependent time). +pub fn mbarrier_wait_parity( + ctx: *mlir.Context, + ptr: *const mlir.Value, + parity: *const mlir.Value, + kind: MBarrierWaitKind, + scope: MBarrierScope, + location: *const mlir.Location, +) *mlir.Operation { + return ops.mbarrier_wait_parity( + ctx, + ptr, + parity, + .int(ctx, .i1), + enumAttribute(ops.MBarrierWaitKindAttr, ctx, kind), + .{ .scope = enumAttribute(ops.MBarrierScopeKindAttr, ctx, scope) }, + location, + ); +} + +/// nvvm.mbarrier.try_wait.parity — wait (looping in PTX) until phase `parity` (i32) +/// of the mbarrier completes, with CTA acquire semantics. `suspend_time` (i32) +/// is a suspend-time hint in nanoseconds per try. +pub fn mbarrier_try_wait_parity( + ctx: *mlir.Context, + ptr: *const mlir.Value, + parity: *const mlir.Value, + suspend_time: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return ops.mbarrier_try_wait_parity(ctx, ptr, parity, suspend_time, .{ .useIntrinsic = .boolean(ctx, false) }, location); +} + +/// One potentially suspending parity test with acquire semantics. A false +/// result means the timeout expired; callers that need completion must retry. +pub fn mbarrier_try_wait_parity_timelimit( + ctx: *mlir.Context, + ptr: *const mlir.Value, + parity: *const mlir.Value, + suspend_time: *const mlir.Value, + scope: MBarrierScope, + location: *const mlir.Location, +) *mlir.Operation { + return ops.mbarrier_try_wait_parity_timelimit( + ctx, + ptr, + parity, + suspend_time, + .int(ctx, .i1), + .{ .scope = enumAttribute(ops.MBarrierScopeKindAttr, ctx, scope), .order = enumAttribute(ops.MemOrderKindAttr, ctx, .acquire) }, + location, + ); +} + +/// nvvm.fence.mbarrier.init — make mbarrier initialization visible to the cluster. +pub fn fence_mbarrier_init(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return ops.fence_mbarrier_init(ctx, location); +} + +/// nvvm.fence.proxy — order memory accesses across proxies. +pub fn fence_proxy(ctx: *mlir.Context, kind: ProxyKind, space: SharedSpace, location: *const mlir.Location) *mlir.Operation { + return ops.fence_proxy(ctx, enumAttribute(ops.ProxyKindAttr, ctx, kind), .{ .space = enumAttribute(ops.SharedSpaceAttr, ctx, space) }, location); +} + +// ============================================================================= +// Asynchronous copies +// ============================================================================= + +/// nvvm.cp.async.commit.group — commit the pending `cp.async` copies as a group. +pub fn cp_async_commit_group(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return ops.cp_async_commit_group(ctx, location); +} + +/// nvvm.cp.async.wait.group — wait until at most `pending` `cp.async` groups are in flight. +pub fn cp_async_wait_group(ctx: *mlir.Context, pending: u32, location: *const mlir.Location) *mlir.Operation { + return ops.cp_async_wait_group(ctx, .int(ctx, .i32, pending), location); +} + +/// nvvm.cp.async.bulk.commit.group — commit the pending bulk (TMA) copies as a group. +pub fn cp_async_bulk_commit_group(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return ops.cp_async_bulk_commit_group(ctx, location); +} + +/// nvvm.cp.async.bulk.wait_group — wait until at most `pending` bulk groups are in +/// flight; with `read`, only until their sources have been read. +pub fn cp_async_bulk_wait_group(ctx: *mlir.Context, pending: u32, read: bool, location: *const mlir.Location) *mlir.Operation { + return ops.cp_async_bulk_wait_group(ctx, .int(ctx, .i32, pending), .{ .read = if (read) .unit(ctx) else null }, location); +} + +// ============================================================================= +// tcgen05 (SM100 tensor cores and tensor memory) +// ============================================================================= + +/// nvvm.tcgen05.alloc — allocate `columns` (i32) tensor-memory columns and write the +/// address to `holder` (`!llvm.ptr<3>`). +pub fn tcgen05_alloc(ctx: *mlir.Context, holder: *const mlir.Value, columns: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return ops.tcgen05_alloc(ctx, holder, columns, .{}, location); +} + +/// nvvm.tcgen05.dealloc — free `columns` (i32) tensor-memory columns at `taddr` +/// (`!llvm.ptr<6>`). +pub fn tcgen05_dealloc(ctx: *mlir.Context, taddr: *const mlir.Value, columns: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return ops.tcgen05_dealloc(ctx, taddr, columns, .{}, location); +} + +/// nvvm.tcgen05.fence — order tcgen05 operations around a thread synchronization. +pub fn tcgen05_fence(ctx: *mlir.Context, kind: Tcgen05FenceKind, location: *const mlir.Location) *mlir.Operation { + return ops.tcgen05_fence(ctx, enumAttribute(ops.Tcgen05FenceKindAttr, ctx, kind), location); +} + +/// nvvm.tcgen05.wait — wait for the issued tensor-memory loads or stores. +pub fn tcgen05_wait(ctx: *mlir.Context, kind: Tcgen05WaitKind, location: *const mlir.Location) *mlir.Operation { + return ops.tcgen05_wait(ctx, enumAttribute(ops.Tcgen05WaitKindAttr, ctx, kind), location); +} + +/// nvvm.tcgen05.commit — arrive on the mbarrier at `barrier_ptr` when the +/// tcgen05 MMA/copy operations issued by the executing thread with `group` +/// complete. An optional mask selects CTAs in the cluster whose barriers at +/// the same shared-memory offset receive one arrival each. +pub fn tcgen05_commit(ctx: *mlir.Context, barrier_ptr: *const mlir.Value, group: CtaGroup, multicast_mask: ?*const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return ops.tcgen05_commit(ctx, barrier_ptr, .{ .multicastMask = multicast_mask, .group = enumAttribute(ops.CTAGroupKindAttr, ctx, group) }, location); +} + +/// nvvm.tcgen05.mma — D (tensor memory, `!llvm.ptr<6>`) = A (shared-memory +/// descriptor, i64) * B (descriptor, i64) [+ D when `accumulate` (i1)], with the +/// instruction descriptor `idesc` (i32). +pub fn tcgen05_mma( + ctx: *mlir.Context, + d: *const mlir.Value, + a_desc: *const mlir.Value, + b_desc: *const mlir.Value, + idesc: *const mlir.Value, + accumulate: *const mlir.Value, + kind: Tcgen05MmaKind, + cta_group: CtaGroup, + location: *const mlir.Location, +) *mlir.Operation { + return ops.tcgen05_mma( + ctx, + d, + a_desc, + b_desc, + idesc, + accumulate, + enumAttribute(ops.Tcgen05MMAKindAttr, ctx, kind), + enumAttribute(ops.CTAGroupKindAttr, ctx, cta_group), + .{}, + location, + ); +} + +/// nvvm.tcgen05.ld — load `result_type` (an i32 or vector of i32) from tensor memory +/// at `taddr` (`!llvm.ptr<6>`). +pub fn tcgen05_ld(ctx: *mlir.Context, taddr: *const mlir.Value, result_type: *const mlir.Type, shape: Tcgen05LdStShape, location: *const mlir.Location) *mlir.Operation { + return ops.tcgen05_ld(ctx, taddr, result_type, enumAttribute(ops.Tcgen05LdStShapeAttr, ctx, shape), .{}, location); +} + +/// nvvm.tcgen05.st — store `value` (an i32 or vector of i32) to tensor memory at `taddr`. +pub fn tcgen05_st(ctx: *mlir.Context, taddr: *const mlir.Value, value: *const mlir.Value, shape: Tcgen05LdStShape, location: *const mlir.Location) *mlir.Operation { + return ops.tcgen05_st(ctx, taddr, value, enumAttribute(ops.Tcgen05LdStShapeAttr, ctx, shape), .{}, location); +} + +// ============================================================================= +// Clusters +// ============================================================================= + +/// nvvm.mapa — the address of `ptr` in the shared memory of cluster CTA `rank` (i32). +pub fn mapa(ctx: *mlir.Context, ptr: *const mlir.Value, rank: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return ops.mapa(ctx, ptr, rank, result_type, location); +} + +/// nvvm.cluster.arrive — arrive on the cluster barrier. +pub fn cluster_arrive(ctx: *mlir.Context, aligned: bool, location: *const mlir.Location) *mlir.Operation { + return ops.cluster_arrive(ctx, .{ .aligned = if (aligned) .unit(ctx) else null }, location); +} + +/// nvvm.cluster.wait — wait on the cluster barrier. +pub fn cluster_wait(ctx: *mlir.Context, aligned: bool, location: *const mlir.Location) *mlir.Operation { + return ops.cluster_wait(ctx, .{ .aligned = if (aligned) .unit(ctx) else null }, location); +} + +// ============================================================================= +// Math +// ============================================================================= + +/// nvvm.ex2 — `ex2.approx[.ftz].f32`. +pub fn ex2(ctx: *mlir.Context, value: *const mlir.Value, ftz: bool, location: *const mlir.Location) *mlir.Operation { + return ops.ex2(ctx, value, .float(ctx, .f32), .{ .ftz = .boolean(ctx, ftz) }, location); +} + +/// nvvm.rcp.approx.ftz.f — `rcp.approx.ftz.f32`. +pub fn rcp_approx_ftz(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return ops.rcp_approx_ftz_f(ctx, value, .float(ctx, .f32), location); +} diff --git a/mlir/dialects/nvvm/ops.zig b/mlir/dialects/nvvm/ops.zig new file mode 100644 index 00000000..88c1c514 --- /dev/null +++ b/mlir/dialects/nvvm/ops.zig @@ -0,0 +1,8389 @@ +//! Zig bindings of NVIDIA's NVVM dialect of the CuTe DSL (nvidia-cutlass-dsl 4.8.0), generated +//! from the dialect's .td files; do not edit. + +const c = @import("c"); +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +/// The C dialect handle, `mlirGetDialectHandle__cute_nvvm__`. +pub const dialect_handle = "cute_nvvm"; + +// Enums, generated from the dialect's .td files; values are the compiler's. + +/// operations supported by atom instruction. +pub const AtomicOpKind = enum(u32) { + @"and" = 0, + @"or" = 1, + xor = 2, + cas = 3, + exch = 4, + add = 5, + inc = 6, + dec = 7, + min = 8, + max = 9, + umin = 10, + umax = 11, + fadd = 12, + sub = 13, +}; + +/// NVVM barrier reduction operation. +pub const BarrierReduction = enum(u32) { + popc = 0, + @"and" = 1, + @"or" = 2, +}; + +/// NVVM barrier redux kind. +pub const BarrierReduxKind = enum(u32) { + @"and" = 0, + @"or" = 1, + popc = 2, +}; + +/// MMA Block Scale Format. +pub const BlockScaleFormat = enum(u32) { + ue8m0 = 0, + ue4m3 = 1, +}; + +/// NVVM CTA group kind. +pub const CTAGroupKind = enum(u32) { + cta_1 = 0, + cta_2 = 1, +}; + +/// NVVM CVT Pack Float kind. +pub const CVTPackFloatKind = enum(u32) { + f32 = 0, + f16x2 = 1, + e4m3x2 = 2, + e5m2x2 = 3, + bf16x2 = 4, + e2m1x2 = 5, + e2m3x2 = 6, + e3m2x2 = 7, + ue8m0x2 = 8, +}; + +/// NVVM Cache Eviction Priority. +pub const CacheEvictionPriority = enum(u32) { + evict_normal = 0, + evict_first = 1, + evict_last = 2, + evict_unchanged = 3, + no_allocate = 4, +}; + +/// NVVM ClusterLaunchControlQueryType. +pub const ClusterLaunchControlQueryType = enum(u32) { + is_canceled = 0, + get_first_cta_id_x = 1, + get_first_cta_id_y = 2, + get_first_cta_id_z = 3, +}; + +/// Comparison operator encoding. +pub const CompareOpKind = enum(u32) { + eq = 0, + ne = 1, + ugt = 2, + uge = 3, + ult = 4, + ule = 5, + sgt = 6, + sge = 7, + slt = 8, + sle = 9, +}; + +/// NVVM ConvertFP4Type kind. +pub const ConvertFP4Type = enum(u32) { + e2m1 = 0, +}; + +/// NVVM ConvertFP8Type kind. +pub const ConvertFP8Type = enum(u32) { + e4m3 = 0, + e5m2 = 1, + ue8m0 = 2, + ue5m3 = 3, +}; + +/// NVVM ConvertScale kind. +pub const ConvertScaleKind = enum(u32) { + none = 0, + ue8m0 = 1, +}; + +/// NVVM DotAccumulateType. +pub const DotAccumulateType = enum(u32) { + signed = 1, + unsigned = 0, +}; + +/// NVVM L2 Prefetch Size. +pub const EvictKind = enum(u32) { + evict_normal = 0, + evict_first = 1, + evict_last = 2, + evict_unchanged = 3, + evict_no_allocate = 4, + none = 5, +}; + +/// NVVM FPRoundingMode kind. +pub const FPRoundingMode = enum(u32) { + none = 0, + rn = 1, + rm = 2, + rp = 3, + rz = 4, + rna = 5, + rs = 6, +}; + +/// Action kind for grid dependency control. +pub const GridDepActionKind = enum(u32) { + wait = 0, + launch_dependents = 1, +}; + +/// NVVM IntegerRoundingMode kind. +pub const IntegerRoundingMode = enum(u32) { + none = 0, + rni = 1, + rzi = 2, + rmi = 3, + rpi = 4, +}; + +/// NVVM L2 Prefetch Size. +pub const L2PrefetchSize = enum(u32) { + none = 0, + reserved = 1, + size_64b = 2, + size_128b = 3, + size_256b = 4, +}; + +/// Element type for ldmatrix and stmatrix. +pub const LdStMatrixEltType = enum(u32) { + b16 = 0, + b8 = 1, + @"b8x16.b6x16_p32" = 2, + @"b8x16.b4x16_p64" = 3, +}; + +/// NVVM load cache modifier kind(Ext). +pub const LoadCacheModifierExtKind = enum(u32) { + ca = 0, + cg = 1, + cs = 2, + lu = 3, + cv = 4, + none = 5, +}; + +/// NVVM load cache modifier kind. +pub const LoadCacheModifierKind = enum(u32) { + ca = 0, + cg = 1, + cs = 2, + lu = 3, + cv = 4, +}; + +/// shape attribute for ldmatrix. +pub const LoadShape = enum(u32) { + m8n8 = 0, + m8n16 = 1, + m16n16 = 3, +}; + +/// source format for ldmatrix. +pub const LoadSrcFormat = enum(u32) { + b6x16_p32 = 0, + b4x16_p64 = 1, + b8 = 2, +}; + +/// NVVM MBarrier Layout. +pub const MBarrierLayout = enum(u32) { + v0 = 0, + v1 = 1, +}; + +/// NVVM mbarrier phase type. +pub const MBarrierPhase = enum(u32) { + none = 0, + primary = 1, + conditional = 2, +}; + +/// NVVM MBarrier scope kind. +pub const MBarrierScopeKind = enum(u32) { + cta = 0, + cluster = 1, +}; + +/// NVVM MBarrier space kind. +pub const MBarrierSpaceKind = enum(u32) { + cta = 0, + cluster = 1, +}; + +/// NVVM MBarrier Transaction kind. +pub const MBarrierTxnKind = enum(u32) { + arrive = 0, + arrive_drop = 1, + arrive_expect_tx = 2, + arrive_drop_expect_tx = 3, + expect_tx = 4, + complete_tx = 5, +}; + +/// NVVM MBarrier wait kind. +pub const MBarrierWaitKind = enum(u32) { + @"test" = 0, + @"try" = 1, +}; + +/// MMA binary operations. +pub const MMAB1Op = enum(u32) { + none = 0, + xor_popc = 1, + and_popc = 2, +}; + +/// Block Scale Kind. +pub const MMABlockScaleKind = enum(u32) { + mxf8f6f4 = 0, + mxf4 = 1, + mxf4nvf4 = 2, +}; + +/// MMA CTA count. +pub const MMACtaCount = enum(u32) { + cta1 = 1, + cta2 = 2, +}; + +/// NVVM MMA frag type. +pub const MMAFrag = enum(u32) { + a = 0, + b = 1, + c = 2, +}; + +/// MMA overflow options. +pub const MMAIntOverflow = enum(u32) { + satfinite = 1, + wrapped = 0, +}; + +/// MMA operation kind. +pub const MMAKind = enum(u32) { + f8f6f4 = 0, +}; + +/// NVVM MMA layout. +pub const MMALayout = enum(u32) { + row = 0, + col = 1, +}; + +/// NVVM MMA types. +pub const MMATypes = enum(u32) { + f16 = 0, + f32 = 1, + tf32 = 2, + bf16 = 9, + s8 = 4, + u8 = 3, + s32 = 5, + s4 = 8, + u4 = 7, + b1 = 6, + f64 = 10, + e4m3 = 11, + e5m2 = 12, + e3m2 = 13, + e2m3 = 14, + e2m1 = 15, +}; + +/// NVVM match sync kind. +pub const MatchSyncKind = enum(u32) { + any = 0, + all = 1, +}; + +/// NVVM Memory Ordering kind. +pub const MemOrderKind = enum(u32) { + weak = 0, + relaxed = 1, + acquire = 2, + release = 3, + acq_rel = 4, + sc = 5, + mmio = 6, + @"volatile" = 8, + constant = 7, +}; + +/// NVVM Memory Scope kind. +pub const MemScopeKind = enum(u32) { + cta = 0, + cluster = 1, + gpu = 2, + sys = 3, +}; + +/// multiply mode attribute. +pub const MulMode = enum(u32) { + hi = 0, + lo = 1, + wide = 2, +}; + +/// NVVM Memory Space. +pub const NVVMMemorySpace = enum(u32) { + generic = 0, + global = 1, + shared = 3, + constant = 4, + local = 5, + tensor = 6, + shared_cluster = 7, +}; + +/// NVVM permute mode. +pub const PermuteMode = enum(u32) { + default = 0, + f4e = 1, + b4e = 2, + rc8 = 3, + ecl = 4, + ecr = 5, + rc16 = 6, +}; + +/// NVVM Prefetch Cache Level. +pub const PrefetchCacheLevel = enum(u32) { + L1 = 0, + L2 = 1, +}; + +/// Proxy kind. +pub const ProxyKind = enum(u32) { + alias = 0, + async = 1, + @"async.global" = 2, + @"async.shared" = 3, + tensormap = 4, + generic = 5, +}; + +/// NVVM Reduction Kind attribute. +pub const ReductionKind = enum(u32) { + add = 1, + @"and" = 2, + max = 3, + min = 4, + @"or" = 5, + umax = 6, + umin = 7, + xor = 8, + fmin = 9, + fmax = 10, +}; + +/// Ops supported by red instruction. +pub const ReductionOp = enum(u32) { + @"and" = 0, + @"or" = 1, + xor = 2, + add = 3, + inc = 4, + dec = 5, + min = 6, + max = 7, +}; + +/// types supported by red instruction. +pub const ReductionType = enum(u32) { + b32 = 0, + b64 = 1, + u32 = 2, + u64 = 3, + s32 = 4, + s64 = 5, + f32 = 6, + f64 = 7, + f16 = 8, + f16x2 = 9, + bf16 = 10, + bf16x2 = 11, +}; + +/// Sparse tensor compression element size. +pub const SPCompressElemSize = enum(u32) { + b4 = 4, + b8 = 8, + b16 = 16, +}; + +/// Sparse tensor compression factor type. +pub const SPCompressFactorType = enum(u32) { + sp2to4 = 0, +}; + +/// Sparse tensor compression index size. +pub const SPCompressIndexSize = enum(u32) { + b2 = 2, + b4 = 4, +}; + +/// spcompress operation kind. +pub const SPCompressOpKind = enum(u32) { + min = 0, + max = 1, +}; + +/// Sparse tensor compression repetition factor. +pub const SPCompressRepFactor = enum(u32) { + x1 = 1, + x2 = 2, + x4 = 4, + x8 = 8, + x16 = 16, + x32 = 32, + x64 = 64, + x128 = 128, +}; + +/// Sparse tensor decompression element size. +pub const SPDecompressElemSize = enum(u32) { + b4 = 4, + b8 = 8, + b16 = 16, +}; + +/// Sparse tensor decompression factor type. +pub const SPDecompressFactorType = enum(u32) { + sp1to2 = 0, + sp1to4 = 1, + sp1to8 = 2, + sp1to16 = 3, + sp2to4 = 4, + sp2to8 = 5, + sp2to16 = 6, + sp4to8 = 7, + sp4to16 = 8, + sp8to16 = 9, +}; + +/// Sparse tensor decompression index size. +pub const SPDecompressIndexSize = enum(u32) { + b2 = 2, + b4 = 4, +}; + +/// Sparse tensor decompression repetition factor. +pub const SPDecompressRepFactor = enum(u32) { + x1 = 1, + x2 = 2, + x4 = 4, + x8 = 8, + x16 = 16, + x32 = 32, + x64 = 64, +}; + +/// NVVM SaturationMode kind. +pub const SaturationMode = enum(u32) { + none = 0, + satfinite = 1, + sat = 2, +}; + +/// NVVM SaturationMode kind. +pub const SaturationModeKind = enum(u32) { + none = 0, + satfinite = 1, +}; + +/// MMA Scale Vector Sizes. +pub const ScaleVecSize = enum(u32) { + x1 = 0, + x2 = 1, + x4 = 2, +}; + +/// NVVM set max register action. +pub const SetMaxRegisterAction = enum(u32) { + decrease = 1, + increase = 0, +}; + +/// Shared memory space. +pub const SharedSpace = enum(u32) { + cta = 0, + cluster = 1, +}; + +/// NVVM shuffle kind. +pub const ShflKind = enum(u32) { + bfly = 0, + up = 1, + down = 2, + idx = 3, +}; + +/// MMA Sparsity Format. +pub const SparsityFormat = enum(u32) { + thread = 0, +}; + +/// NVVM State Space. +pub const StateSpace = enum(u32) { + generic = 0, + global = 1, + shared_cta = 2, + shared_cluster = 3, + constant = 4, + local = 5, + tensor = 6, +}; + +/// NVVM store cache modifier kind. +pub const StoreCacheModifierKind = enum(u32) { + wb = 0, + cg = 1, + cs = 2, + wt = 3, + none = 4, +}; + +/// shape attribute for stmatrix. +pub const StoreShape = enum(u32) { + m8n8 = 0, + m16n8 = 2, +}; + +/// Cluster MMA Barrier Parameter Type. +pub const TCBarParam = enum(u32) { + a1t0 = 0, + a0tx = 1, +}; + +/// NVVM TMA Load Mode. +pub const TMALoadMode = enum(u32) { + tile = 0, + im2col = 1, + im2col_w = 2, + im2col_w_128 = 3, + tile_gather4 = 4, +}; + +/// NVVM TMA redux kind. +pub const TMAReduxKind = enum(u32) { + add = 0, + max = 2, + min = 1, + inc = 3, + dec = 4, + @"and" = 5, + @"or" = 6, + xor = 7, +}; + +/// NVVM TMA Store Mode. +pub const TMAStoreMode = enum(u32) { + tile = 0, + im2col = 1, + tile_scatter4 = 2, + im2col_w = 3, +}; + +/// tcgen05 cp multicast. +pub const Tcgen05CpMulticast = enum(u32) { + none = 0, + warpx2_02_13 = 1, + warpx2_01_23 = 2, + warpx4 = 3, +}; + +/// tcgen05 cp shapes. +pub const Tcgen05CpShape = enum(u32) { + shape_128x256b = 0, + shape_4x256b = 1, + shape_128x128b = 2, + shape_64x128b = 3, + shape_32x128b = 4, +}; + +/// tcgen05 cp source format. +pub const Tcgen05CpSrcFormat = enum(u32) { + b6x16_p32 = 0, + b4x16_p64 = 1, +}; + +/// NVVM Tcgen05 fence kind. +pub const Tcgen05FenceKind = enum(u32) { + before = 0, + after = 1, +}; + +/// allowed 32-bit signless integer cases: 0, 1, 2, 3, 4. +pub const Tcgen05LdStShape = enum(u32) { + shape_16x64b = 0, + shape_16x128b = 1, + shape_16x256b = 2, + shape_32x32b = 3, + shape_16x32bx2 = 4, +}; + +/// tcgen05.mma block scale attribute. +pub const Tcgen05MMABlockScale = enum(u32) { + default = 0, + block16 = 1, + block32 = 2, +}; + +/// tcgen05 MMA Collector Buffer B Attribute. +pub const Tcgen05MMACollectorBBuffer = enum(u32) { + b0 = 0, + b1 = 1, + b2 = 2, + b3 = 3, +}; + +/// tcgen05.mma Collector Buffer Operation. +pub const Tcgen05MMACollectorOp = enum(u32) { + discard = 0, + lastuse = 1, + fill = 2, + use = 3, +}; + +/// tcgen05 MMA Supported Types. +pub const Tcgen05MMAKind = enum(u32) { + f16 = 0, + tf32 = 1, + f8f6f4 = 2, + i8 = 3, + mxf8f6f4 = 4, + mxf4 = 5, + mxf4nvf4 = 6, + ti16 = 7, +}; + +/// NVVM Tcgen05 wait kind. +pub const Tcgen05WaitKind = enum(u32) { + load = 0, + store = 1, +}; + +/// NVVM Tensormap Elemtype. +pub const TensormapElemtype = enum(u32) { + u8 = 0, + u16 = 1, + u32 = 2, + s32 = 3, + u64 = 4, + s64 = 5, + f16 = 6, + f32 = 7, + @"f32.ftz" = 8, + f64 = 9, + bf16 = 10, + tf32 = 11, + @"tf32.ftz" = 12, + b4x16 = 13, + b4x16_p64 = 14, + b6x16_p32 = 15, +}; + +/// NVVM Tensormap Field Kind. +pub const TensormapField = enum(u32) { + global_address = 0, + rank = 1, + box_dim = 2, + global_dim = 3, + global_stride = 4, + element_stride = 5, + elemtype = 6, + interleave_layout = 7, + swizzle_mode = 8, + swizzle_atomicity = 9, + fill_mode = 10, +}; + +/// NVVM Tensormap Fill Mode. +pub const TensormapFillMode = enum(u32) { + zero = 0, + oob_nan = 1, +}; + +/// NVVM Tensormap Interleave Layout. +pub const TensormapInterleaveLayout = enum(u32) { + no_interleave = 0, + b16 = 1, + b32 = 2, +}; + +/// NVVM Tensormap Swizzle Atomicity. +pub const TensormapSwizzleAtomicity = enum(u32) { + b16 = 0, + b32 = 1, + b32_flip_b8 = 2, + b64 = 3, +}; + +/// NVVM Tensormap Swizzle Mode. +pub const TensormapSwizzleMode = enum(u32) { + no_swizzling = 0, + b32 = 1, + b64 = 2, + b128 = 3, + b96 = 4, +}; + +/// Tensor Memory Layout Enumerated Type. +pub const TmemLayout = enum(u32) { + tmem_16dp_128bit = 0, + tmem_16dp_256bit = 1, + tmem_32dp_32bit = 2, + tmem_16dp_64bit = 3, + tmem_16dp_32bit_t0_t15 = 4, + tmem_16dp_32bit_t16_t31 = 5, +}; + +/// NVVM validate data pattern. +pub const ValidatePattern = enum(u32) { + none = 0, + per_16bytes_80000000 = 1, + per_16bytes_8000 = 2, + per_16bytes_80 = 3, + per_16bytes_8 = 4, + per_element_ff = 5, + per_element_80000000 = 6, + per_element_8000 = 7, + per_element_80 = 8, + per_element_8 = 9, + per_16bytes_80000000_paired = 10, +}; + +/// NVVM vote sync kind. +pub const VoteSyncKind = enum(u32) { + any = 0, + all = 1, + ballot = 2, + uni = 3, +}; + +/// WGMMA overflow options. +pub const WGMMAScaleIn = enum(u32) { + one = 1, + neg = 2, +}; + +/// WGMMA input predicate. +pub const WGMMAScaleOut = enum(u32) { + zero = 0, + one = 1, +}; + +/// NVVM WGMMA types. +pub const WGMMATypes = enum(u32) { + f16 = 0, + tf32 = 1, + u8 = 2, + s8 = 3, + b1 = 4, + bf16 = 5, + e4m3 = 6, + e5m2 = 7, + f32 = 8, + s32 = 9, +}; + +// Types and attributes, generated from the dialect's .td files. + +/// `#nvvm.atomic_op`: operations supported by atom instruction +pub const AtomicOpKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMAtomicOpKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "atomic_op"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: AtomicOpKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMAtomicOpKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) AtomicOpKind { + return @fromBackingInt(@intCast(c.mlirCuteNVVMAtomicOpKindAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.reduction`: NVVM barrier reduction operation +pub const BarrierReductionAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMBarrierReduction; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "reduction"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: BarrierReduction, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMBarrierReductionAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) BarrierReduction { + return @fromBackingInt(@intCast(c.mlirCuteNVVMBarrierReductionAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.barrier_redux_kind`: NVVM barrier redux kind +pub const BarrierReduxKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMBarrierReduxKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "barrier_redux_kind"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: BarrierReduxKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMBarrierReduxKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) BarrierReduxKind { + return @fromBackingInt(@intCast(c.mlirCuteNVVMBarrierReduxKindAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.block_scale_format`: MMA Block Scale Format +pub const BlockScaleFormatAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMBlockScaleFormat; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "block_scale_format"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: BlockScaleFormat, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMBlockScaleFormatAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) BlockScaleFormat { + return @fromBackingInt(@intCast(c.mlirCuteNVVMBlockScaleFormatAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.cta_group`: NVVM CTA group kind +pub const CTAGroupKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMCTAGroupKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "cta_group"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: CTAGroupKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMCTAGroupKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) CTAGroupKind { + return @fromBackingInt(@intCast(c.mlirCuteNVVMCTAGroupKindAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.packfloat_type`: NVVM CVT Pack Float kind +pub const CVTPackFloatKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMCVTPackFloatKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "packfloat_type"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: CVTPackFloatKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMCVTPackFloatKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) CVTPackFloatKind { + return @fromBackingInt(@intCast(c.mlirCuteNVVMCVTPackFloatKindAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.cache_eviction_priority`: NVVM Cache Eviction Priority +pub const CacheEvictionPriorityAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMCacheEvictionPriority; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "cache_eviction_priority"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: CacheEvictionPriority, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMCacheEvictionPriorityAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) CacheEvictionPriority { + return @fromBackingInt(@intCast(c.mlirCuteNVVMCacheEvictionPriorityAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.cluster_launch_control_query_type`: NVVM ClusterLaunchControlQueryType +pub const ClusterLaunchControlQueryTypeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMClusterLaunchControlQueryType; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "cluster_launch_control_query_type"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: ClusterLaunchControlQueryType, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMClusterLaunchControlQueryTypeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) ClusterLaunchControlQueryType { + return @fromBackingInt(@intCast(c.mlirCuteNVVMClusterLaunchControlQueryTypeAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.op`: Comparison operator encoding +pub const CompareOpKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMCompareOpKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "op"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: CompareOpKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMCompareOpKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) CompareOpKind { + return @fromBackingInt(@intCast(c.mlirCuteNVVMCompareOpKindAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.convert_fp4_type`: NVVM ConvertFP4Type kind +pub const ConvertFP4TypeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMConvertFP4Type; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "convert_fp4_type"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: ConvertFP4Type, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMConvertFP4TypeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) ConvertFP4Type { + return @fromBackingInt(@intCast(c.mlirCuteNVVMConvertFP4TypeAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.convert_fp8_type`: NVVM ConvertFP8Type kind +pub const ConvertFP8TypeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMConvertFP8Type; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "convert_fp8_type"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: ConvertFP8Type, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMConvertFP8TypeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) ConvertFP8Type { + return @fromBackingInt(@intCast(c.mlirCuteNVVMConvertFP8TypeAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.convert_scale_kind`: NVVM ConvertScale kind +pub const ConvertScaleKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMConvertScaleKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "convert_scale_kind"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: ConvertScaleKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMConvertScaleKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) ConvertScaleKind { + return @fromBackingInt(@intCast(c.mlirCuteNVVMConvertScaleKindAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.dot_accumulate_type`: NVVM DotAccumulateType +pub const DotAccumulateTypeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMDotAccumulateType; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "dot_accumulate_type"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: DotAccumulateType, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMDotAccumulateTypeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) DotAccumulateType { + return @fromBackingInt(@intCast(c.mlirCuteNVVMDotAccumulateTypeAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.evict_kind`: NVVM L2 Prefetch Size +pub const EvictKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMEvictKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "evict_kind"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: EvictKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMEvictKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) EvictKind { + return @fromBackingInt(@intCast(c.mlirCuteNVVMEvictKindAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.fp_rnd_mode`: NVVM FPRoundingMode kind +pub const FPRoundingModeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMFPRoundingMode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "fp_rnd_mode"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: FPRoundingMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMFPRoundingModeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) FPRoundingMode { + return @fromBackingInt(@intCast(c.mlirCuteNVVMFPRoundingModeAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.grid_dep_action`: Action kind for grid dependency control +pub const GridDepActionKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMGridDepActionKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "grid_dep_action"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: GridDepActionKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMGridDepActionKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) GridDepActionKind { + return @fromBackingInt(@intCast(c.mlirCuteNVVMGridDepActionKindAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.int_rnd_mode`: NVVM IntegerRoundingMode kind +pub const IntegerRoundingModeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMIntegerRoundingMode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "int_rnd_mode"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: IntegerRoundingMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMIntegerRoundingModeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) IntegerRoundingMode { + return @fromBackingInt(@intCast(c.mlirCuteNVVMIntegerRoundingModeAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.l2_prefetch`: NVVM L2 Prefetch Size +pub const L2PrefetchSizeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVML2PrefetchSize; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "l2_prefetch"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: L2PrefetchSize, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVML2PrefetchSizeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) L2PrefetchSize { + return @fromBackingInt(@intCast(c.mlirCuteNVVML2PrefetchSizeAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.ld_st_matrix_elt_type`: Element type for ldmatrix and stmatrix +pub const LdStMatrixEltTypeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMLdStMatrixEltType; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "ld_st_matrix_elt_type"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: LdStMatrixEltType, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMLdStMatrixEltTypeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) LdStMatrixEltType { + return @fromBackingInt(@intCast(c.mlirCuteNVVMLdStMatrixEltTypeAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.load_cache_modifier_ext`: NVVM load cache modifier kind(Ext) +pub const LoadCacheModifierExtKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMLoadCacheModifierExtKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "load_cache_modifier_ext"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: LoadCacheModifierExtKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMLoadCacheModifierExtKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) LoadCacheModifierExtKind { + return @fromBackingInt(@intCast(c.mlirCuteNVVMLoadCacheModifierExtKindAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.load_cache_modifier`: NVVM load cache modifier kind +pub const LoadCacheModifierKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMLoadCacheModifierKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "load_cache_modifier"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: LoadCacheModifierKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMLoadCacheModifierKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) LoadCacheModifierKind { + return @fromBackingInt(@intCast(c.mlirCuteNVVMLoadCacheModifierKindAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.load_shape`: shape attribute for ldmatrix +pub const LoadShapeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMLoadShape; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "load_shape"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: LoadShape, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMLoadShapeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) LoadShape { + return @fromBackingInt(@intCast(c.mlirCuteNVVMLoadShapeAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.load_src_format`: source format for ldmatrix +pub const LoadSrcFormatAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMLoadSrcFormat; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "load_src_format"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: LoadSrcFormat, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMLoadSrcFormatAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) LoadSrcFormat { + return @fromBackingInt(@intCast(c.mlirCuteNVVMLoadSrcFormatAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.mbarrier_layout`: NVVM MBarrier Layout +pub const MBarrierLayoutAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMMBarrierLayout; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "mbarrier_layout"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MBarrierLayout, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMMBarrierLayoutAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) MBarrierLayout { + return @fromBackingInt(@intCast(c.mlirCuteNVVMMBarrierLayoutAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.mbarrier_phase`: NVVM mbarrier phase type +pub const MBarrierPhaseAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMMBarrierPhase; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "mbarrier_phase"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MBarrierPhase, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMMBarrierPhaseAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) MBarrierPhase { + return @fromBackingInt(@intCast(c.mlirCuteNVVMMBarrierPhaseAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.mbar_scope`: NVVM MBarrier scope kind +pub const MBarrierScopeKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMMBarrierScopeKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "mbar_scope"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MBarrierScopeKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMMBarrierScopeKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) MBarrierScopeKind { + return @fromBackingInt(@intCast(c.mlirCuteNVVMMBarrierScopeKindAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.mbar_space`: NVVM MBarrier space kind +pub const MBarrierSpaceKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMMBarrierSpaceKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "mbar_space"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MBarrierSpaceKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMMBarrierSpaceKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) MBarrierSpaceKind { + return @fromBackingInt(@intCast(c.mlirCuteNVVMMBarrierSpaceKindAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.mbar_txn_kind`: NVVM MBarrier Transaction kind +pub const MBarrierTxnKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMMBarrierTxnKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "mbar_txn_kind"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MBarrierTxnKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMMBarrierTxnKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) MBarrierTxnKind { + return @fromBackingInt(@intCast(c.mlirCuteNVVMMBarrierTxnKindAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.mbar_wait`: NVVM MBarrier wait kind +pub const MBarrierWaitKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMMBarrierWaitKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "mbar_wait"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MBarrierWaitKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMMBarrierWaitKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) MBarrierWaitKind { + return @fromBackingInt(@intCast(c.mlirCuteNVVMMBarrierWaitKindAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.mma_b1op`: MMA binary operations +pub const MMAB1OpAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMMMAB1Op; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "mma_b1op"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MMAB1Op, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMMMAB1OpAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) MMAB1Op { + return @fromBackingInt(@intCast(c.mlirCuteNVVMMMAB1OpAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.block_scale_kind`: Block Scale Kind +pub const MMABlockScaleKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMMMABlockScaleKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "block_scale_kind"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MMABlockScaleKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMMMABlockScaleKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) MMABlockScaleKind { + return @fromBackingInt(@intCast(c.mlirCuteNVVMMMABlockScaleKindAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.mma_cta_count`: MMA CTA count +pub const MMACtaCountAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMMMACtaCount; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "mma_cta_count"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MMACtaCount, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMMMACtaCountAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) MMACtaCount { + return @fromBackingInt(@intCast(c.mlirCuteNVVMMMACtaCountAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.mma_frag`: NVVM MMA frag type +pub const MMAFragAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMMMAFrag; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "mma_frag"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MMAFrag, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMMMAFragAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) MMAFrag { + return @fromBackingInt(@intCast(c.mlirCuteNVVMMMAFragAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.mma_int_overflow`: MMA overflow options +pub const MMAIntOverflowAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMMMAIntOverflow; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "mma_int_overflow"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MMAIntOverflow, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMMMAIntOverflowAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) MMAIntOverflow { + return @fromBackingInt(@intCast(c.mlirCuteNVVMMMAIntOverflowAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.mma_kind`: MMA operation kind +pub const MMAKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMMMAKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "mma_kind"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MMAKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMMMAKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) MMAKind { + return @fromBackingInt(@intCast(c.mlirCuteNVVMMMAKindAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.mma_layout`: NVVM MMA layout +pub const MMALayoutAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMMMALayout; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "mma_layout"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MMALayout, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMMMALayoutAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) MMALayout { + return @fromBackingInt(@intCast(c.mlirCuteNVVMMMALayoutAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.mma_type`: NVVM MMA types +pub const MMATypesAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMMMATypes; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "mma_type"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MMATypes, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMMMATypesAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) MMATypes { + return @fromBackingInt(@intCast(c.mlirCuteNVVMMMATypesAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.match_sync_kind`: NVVM match sync kind +pub const MatchSyncKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMMatchSyncKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "match_sync_kind"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MatchSyncKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMMatchSyncKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) MatchSyncKind { + return @fromBackingInt(@intCast(c.mlirCuteNVVMMatchSyncKindAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.mem_order`: NVVM Memory Ordering kind +pub const MemOrderKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMMemOrderKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "mem_order"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MemOrderKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMMemOrderKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) MemOrderKind { + return @fromBackingInt(@intCast(c.mlirCuteNVVMMemOrderKindAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.mem_scope`: NVVM Memory Scope kind +pub const MemScopeKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMMemScopeKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "mem_scope"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MemScopeKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMMemScopeKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) MemScopeKind { + return @fromBackingInt(@intCast(c.mlirCuteNVVMMemScopeKindAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.mul_mode`: multiply mode attribute +pub const MulModeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMMulMode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "mul_mode"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: MulMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMMulModeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) MulMode { + return @fromBackingInt(@intCast(c.mlirCuteNVVMMulModeAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.memory_space`: NVVM Memory Space +pub const NVVMMemorySpaceAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMNVVMMemorySpace; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "memory_space"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: NVVMMemorySpace, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMNVVMMemorySpaceAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) NVVMMemorySpace { + return @fromBackingInt(@intCast(c.mlirCuteNVVMNVVMMemorySpaceAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.permute_mode`: NVVM permute mode +pub const PermuteModeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMPermuteMode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "permute_mode"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: PermuteMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMPermuteModeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) PermuteMode { + return @fromBackingInt(@intCast(c.mlirCuteNVVMPermuteModeAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.prefetch_cache_level`: NVVM Prefetch Cache Level +pub const PrefetchCacheLevelAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMPrefetchCacheLevel; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "prefetch_cache_level"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: PrefetchCacheLevel, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMPrefetchCacheLevelAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) PrefetchCacheLevel { + return @fromBackingInt(@intCast(c.mlirCuteNVVMPrefetchCacheLevelAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.proxy_kind`: Proxy kind +pub const ProxyKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMProxyKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "proxy_kind"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: ProxyKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMProxyKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) ProxyKind { + return @fromBackingInt(@intCast(c.mlirCuteNVVMProxyKindAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.reduction_kind`: NVVM Reduction Kind attribute +pub const ReductionKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMReductionKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "reduction_kind"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: ReductionKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMReductionKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) ReductionKind { + return @fromBackingInt(@intCast(c.mlirCuteNVVMReductionKindAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.red_op`: Ops supported by red instruction +pub const ReductionOpAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMReductionOp; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "red_op"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: ReductionOp, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMReductionOpAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) ReductionOp { + return @fromBackingInt(@intCast(c.mlirCuteNVVMReductionOpAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.red_type`: types supported by red instruction +pub const ReductionTypeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMReductionType; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "red_type"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: ReductionType, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMReductionTypeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) ReductionType { + return @fromBackingInt(@intCast(c.mlirCuteNVVMReductionTypeAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.spcomp_elem_size`: Sparse tensor compression element size +pub const SPCompressElemSizeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMSPCompressElemSize; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "spcomp_elem_size"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: SPCompressElemSize, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMSPCompressElemSizeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) SPCompressElemSize { + return @fromBackingInt(@intCast(c.mlirCuteNVVMSPCompressElemSizeAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.spcomp_factor`: Sparse tensor compression factor type +pub const SPCompressFactorTypeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMSPCompressFactorType; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "spcomp_factor"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: SPCompressFactorType, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMSPCompressFactorTypeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) SPCompressFactorType { + return @fromBackingInt(@intCast(c.mlirCuteNVVMSPCompressFactorTypeAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.spcomp_index_size`: Sparse tensor compression index size +pub const SPCompressIndexSizeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMSPCompressIndexSize; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "spcomp_index_size"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: SPCompressIndexSize, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMSPCompressIndexSizeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) SPCompressIndexSize { + return @fromBackingInt(@intCast(c.mlirCuteNVVMSPCompressIndexSizeAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.spcomp_op_kind`: spcompress operation kind +pub const SPCompressOpKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMSPCompressOpKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "spcomp_op_kind"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: SPCompressOpKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMSPCompressOpKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) SPCompressOpKind { + return @fromBackingInt(@intCast(c.mlirCuteNVVMSPCompressOpKindAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.spcomp_rep_factor`: Sparse tensor compression repetition factor +pub const SPCompressRepFactorAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMSPCompressRepFactor; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "spcomp_rep_factor"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: SPCompressRepFactor, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMSPCompressRepFactorAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) SPCompressRepFactor { + return @fromBackingInt(@intCast(c.mlirCuteNVVMSPCompressRepFactorAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.spdecomp_elem_size`: Sparse tensor decompression element size +pub const SPDecompressElemSizeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMSPDecompressElemSize; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "spdecomp_elem_size"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: SPDecompressElemSize, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMSPDecompressElemSizeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) SPDecompressElemSize { + return @fromBackingInt(@intCast(c.mlirCuteNVVMSPDecompressElemSizeAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.spdecomp_factor`: Sparse tensor decompression factor type +pub const SPDecompressFactorTypeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMSPDecompressFactorType; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "spdecomp_factor"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: SPDecompressFactorType, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMSPDecompressFactorTypeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) SPDecompressFactorType { + return @fromBackingInt(@intCast(c.mlirCuteNVVMSPDecompressFactorTypeAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.spdecomp_index_size`: Sparse tensor decompression index size +pub const SPDecompressIndexSizeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMSPDecompressIndexSize; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "spdecomp_index_size"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: SPDecompressIndexSize, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMSPDecompressIndexSizeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) SPDecompressIndexSize { + return @fromBackingInt(@intCast(c.mlirCuteNVVMSPDecompressIndexSizeAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.spdecomp_rep_factor`: Sparse tensor decompression repetition factor +pub const SPDecompressRepFactorAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMSPDecompressRepFactor; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "spdecomp_rep_factor"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: SPDecompressRepFactor, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMSPDecompressRepFactorAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) SPDecompressRepFactor { + return @fromBackingInt(@intCast(c.mlirCuteNVVMSPDecompressRepFactorAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.sat_mode`: NVVM SaturationMode kind +pub const SaturationModeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMSaturationMode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "sat_mode"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: SaturationMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMSaturationModeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) SaturationMode { + return @fromBackingInt(@intCast(c.mlirCuteNVVMSaturationModeAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.sat`: NVVM SaturationMode kind +pub const SaturationModeKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMSaturationModeKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "sat"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: SaturationModeKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMSaturationModeKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) SaturationModeKind { + return @fromBackingInt(@intCast(c.mlirCuteNVVMSaturationModeKindAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.scale_vec_size`: MMA Scale Vector Sizes +pub const ScaleVecSizeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMScaleVecSize; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "scale_vec_size"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: ScaleVecSize, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMScaleVecSizeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) ScaleVecSize { + return @fromBackingInt(@intCast(c.mlirCuteNVVMScaleVecSizeAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.action`: NVVM set max register action +pub const SetMaxRegisterActionAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMSetMaxRegisterAction; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "action"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: SetMaxRegisterAction, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMSetMaxRegisterActionAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) SetMaxRegisterAction { + return @fromBackingInt(@intCast(c.mlirCuteNVVMSetMaxRegisterActionAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.shared_space`: Shared memory space +pub const SharedSpaceAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMSharedSpace; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "shared_space"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: SharedSpace, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMSharedSpaceAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) SharedSpace { + return @fromBackingInt(@intCast(c.mlirCuteNVVMSharedSpaceAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.shfl_kind`: NVVM shuffle kind +pub const ShflKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMShflKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "shfl_kind"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: ShflKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMShflKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) ShflKind { + return @fromBackingInt(@intCast(c.mlirCuteNVVMShflKindAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.sparsity_format`: MMA Sparsity Format +pub const SparsityFormatAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMSparsityFormat; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "sparsity_format"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: SparsityFormat, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMSparsityFormatAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) SparsityFormat { + return @fromBackingInt(@intCast(c.mlirCuteNVVMSparsityFormatAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.state_space`: NVVM State Space +pub const StateSpaceAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMStateSpace; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "state_space"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: StateSpace, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMStateSpaceAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) StateSpace { + return @fromBackingInt(@intCast(c.mlirCuteNVVMStateSpaceAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.store_cache_modifier`: NVVM store cache modifier kind +pub const StoreCacheModifierKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMStoreCacheModifierKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "store_cache_modifier"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: StoreCacheModifierKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMStoreCacheModifierKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) StoreCacheModifierKind { + return @fromBackingInt(@intCast(c.mlirCuteNVVMStoreCacheModifierKindAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.store_shape`: shape attribute for stmatrix +pub const StoreShapeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMStoreShape; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "store_shape"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: StoreShape, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMStoreShapeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) StoreShape { + return @fromBackingInt(@intCast(c.mlirCuteNVVMStoreShapeAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.TCBarParam`: Cluster MMA Barrier Parameter Type +pub const TCBarParamAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTCBarParam; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "TCBarParam"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: TCBarParam, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTCBarParamAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) TCBarParam { + return @fromBackingInt(@intCast(c.mlirCuteNVVMTCBarParamAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.tma_load_mode`: NVVM TMA Load Mode +pub const TMALoadModeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTMALoadMode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "tma_load_mode"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: TMALoadMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTMALoadModeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) TMALoadMode { + return @fromBackingInt(@intCast(c.mlirCuteNVVMTMALoadModeAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.tma_redux_kind`: NVVM TMA redux kind +pub const TMAReduxKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTMAReduxKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "tma_redux_kind"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: TMAReduxKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTMAReduxKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) TMAReduxKind { + return @fromBackingInt(@intCast(c.mlirCuteNVVMTMAReduxKindAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.tma_store_mode`: NVVM TMA Store Mode +pub const TMAStoreModeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTMAStoreMode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "tma_store_mode"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: TMAStoreMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTMAStoreModeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) TMAStoreMode { + return @fromBackingInt(@intCast(c.mlirCuteNVVMTMAStoreModeAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.tcgen05_cp_multicast`: tcgen05 cp multicast +pub const Tcgen05CpMulticastAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTcgen05CpMulticast; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "tcgen05_cp_multicast"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: Tcgen05CpMulticast, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTcgen05CpMulticastAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) Tcgen05CpMulticast { + return @fromBackingInt(@intCast(c.mlirCuteNVVMTcgen05CpMulticastAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.tcgen05_cp_shape`: tcgen05 cp shapes +pub const Tcgen05CpShapeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTcgen05CpShape; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "tcgen05_cp_shape"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: Tcgen05CpShape, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTcgen05CpShapeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) Tcgen05CpShape { + return @fromBackingInt(@intCast(c.mlirCuteNVVMTcgen05CpShapeAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.tcgen05_cp_src_fmt`: tcgen05 cp source format +pub const Tcgen05CpSrcFormatAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTcgen05CpSrcFormat; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "tcgen05_cp_src_fmt"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: Tcgen05CpSrcFormat, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTcgen05CpSrcFormatAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) Tcgen05CpSrcFormat { + return @fromBackingInt(@intCast(c.mlirCuteNVVMTcgen05CpSrcFormatAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.tcgen05_fence`: NVVM Tcgen05 fence kind +pub const Tcgen05FenceKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTcgen05FenceKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "tcgen05_fence"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: Tcgen05FenceKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTcgen05FenceKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) Tcgen05FenceKind { + return @fromBackingInt(@intCast(c.mlirCuteNVVMTcgen05FenceKindAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.tcgen05_ldst_shape`: allowed 32-bit signless integer cases: 0, 1, 2, 3, 4 +pub const Tcgen05LdStShapeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTcgen05LdStShape; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "tcgen05_ldst_shape"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: Tcgen05LdStShape, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTcgen05LdStShapeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) Tcgen05LdStShape { + return @fromBackingInt(@intCast(c.mlirCuteNVVMTcgen05LdStShapeAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.tcgen05_mma_block_scale`: tcgen05.mma block scale attribute +pub const Tcgen05MMABlockScaleAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTcgen05MMABlockScale; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "tcgen05_mma_block_scale"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: Tcgen05MMABlockScale, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTcgen05MMABlockScaleAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) Tcgen05MMABlockScale { + return @fromBackingInt(@intCast(c.mlirCuteNVVMTcgen05MMABlockScaleAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.tcgen05_mma_collectorb`: tcgen05 MMA Collector Buffer B Attribute +pub const Tcgen05MMACollectorBBufferAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTcgen05MMACollectorBBuffer; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "tcgen05_mma_collectorb"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: Tcgen05MMACollectorBBuffer, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTcgen05MMACollectorBBufferAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) Tcgen05MMACollectorBBuffer { + return @fromBackingInt(@intCast(c.mlirCuteNVVMTcgen05MMACollectorBBufferAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.tcgen05_mma_collectorop`: tcgen05.mma Collector Buffer Operation +pub const Tcgen05MMACollectorOpAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTcgen05MMACollectorOp; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "tcgen05_mma_collectorop"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: Tcgen05MMACollectorOp, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTcgen05MMACollectorOpAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) Tcgen05MMACollectorOp { + return @fromBackingInt(@intCast(c.mlirCuteNVVMTcgen05MMACollectorOpAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.tcgen05_mma_kind`: tcgen05 MMA Supported Types +pub const Tcgen05MMAKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTcgen05MMAKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "tcgen05_mma_kind"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: Tcgen05MMAKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTcgen05MMAKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) Tcgen05MMAKind { + return @fromBackingInt(@intCast(c.mlirCuteNVVMTcgen05MMAKindAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.tcgen05_wait`: NVVM Tcgen05 wait kind +pub const Tcgen05WaitKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTcgen05WaitKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "tcgen05_wait"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: Tcgen05WaitKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTcgen05WaitKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) Tcgen05WaitKind { + return @fromBackingInt(@intCast(c.mlirCuteNVVMTcgen05WaitKindAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.tensormap_elemtype`: NVVM Tensormap Elemtype +pub const TensormapElemtypeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTensormapElemtype; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "tensormap_elemtype"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: TensormapElemtype, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTensormapElemtypeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) TensormapElemtype { + return @fromBackingInt(@intCast(c.mlirCuteNVVMTensormapElemtypeAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.tensormap_field`: NVVM Tensormap Field Kind +pub const TensormapFieldAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTensormapField; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "tensormap_field"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: TensormapField, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTensormapFieldAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) TensormapField { + return @fromBackingInt(@intCast(c.mlirCuteNVVMTensormapFieldAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.tensormap_fill_mode`: NVVM Tensormap Fill Mode +pub const TensormapFillModeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTensormapFillMode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "tensormap_fill_mode"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: TensormapFillMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTensormapFillModeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) TensormapFillMode { + return @fromBackingInt(@intCast(c.mlirCuteNVVMTensormapFillModeAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.tensormap_interleave_layout`: NVVM Tensormap Interleave Layout +pub const TensormapInterleaveLayoutAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTensormapInterleaveLayout; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "tensormap_interleave_layout"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: TensormapInterleaveLayout, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTensormapInterleaveLayoutAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) TensormapInterleaveLayout { + return @fromBackingInt(@intCast(c.mlirCuteNVVMTensormapInterleaveLayoutAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.tensormap_swizzle_atomicity`: NVVM Tensormap Swizzle Atomicity +pub const TensormapSwizzleAtomicityAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTensormapSwizzleAtomicity; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "tensormap_swizzle_atomicity"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: TensormapSwizzleAtomicity, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTensormapSwizzleAtomicityAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) TensormapSwizzleAtomicity { + return @fromBackingInt(@intCast(c.mlirCuteNVVMTensormapSwizzleAtomicityAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.tensormap_swizzle_mode`: NVVM Tensormap Swizzle Mode +pub const TensormapSwizzleModeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTensormapSwizzleMode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "tensormap_swizzle_mode"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: TensormapSwizzleMode, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTensormapSwizzleModeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) TensormapSwizzleMode { + return @fromBackingInt(@intCast(c.mlirCuteNVVMTensormapSwizzleModeAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.TmemLayout`: Tensor Memory Layout Enumerated Type +pub const TmemLayoutAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTmemLayout; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "TmemLayout"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: TmemLayout, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTmemLayoutAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) TmemLayout { + return @fromBackingInt(@intCast(c.mlirCuteNVVMTmemLayoutAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.validate_pattern`: NVVM validate data pattern +pub const ValidatePatternAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMValidatePattern; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "validate_pattern"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: ValidatePattern, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMValidatePatternAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) ValidatePattern { + return @fromBackingInt(@intCast(c.mlirCuteNVVMValidatePatternAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.vote_sync_kind`: NVVM vote sync kind +pub const VoteSyncKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMVoteSyncKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "vote_sync_kind"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: VoteSyncKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMVoteSyncKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) VoteSyncKind { + return @fromBackingInt(@intCast(c.mlirCuteNVVMVoteSyncKindAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.wgmma_scale_in`: WGMMA overflow options +pub const WGMMAScaleInAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMWGMMAScaleIn; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "wgmma_scale_in"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: WGMMAScaleIn, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMWGMMAScaleInAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) WGMMAScaleIn { + return @fromBackingInt(@intCast(c.mlirCuteNVVMWGMMAScaleInAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.wgmma_scale_out`: WGMMA input predicate +pub const WGMMAScaleOutAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMWGMMAScaleOut; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "wgmma_scale_out"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: WGMMAScaleOut, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMWGMMAScaleOutAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) WGMMAScaleOut { + return @fromBackingInt(@intCast(c.mlirCuteNVVMWGMMAScaleOutAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.wgmma_type`: NVVM WGMMA types +pub const WGMMATypesAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMWGMMATypes; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "wgmma_type"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: WGMMATypes, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMWGMMATypesAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) WGMMATypes { + return @fromBackingInt(@intCast(c.mlirCuteNVVMWGMMATypesAttrGetValue(self.ptr()))); + } +}; + +/// `#nvvm.ld_st_matrix_shape`: Matrix shape of ldmatrix, stmatrix and movmatrix +pub const LdStMatrixShapeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMLdStMatrixShape; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "ld_st_matrix_shape"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + m: c_int, + n: c_int, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMLdStMatrixShapeAttrGet(ctx.ptr(), args.m, args.n); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn m(self: *const Self) c_int { + return c.mlirCuteNVVMLdStMatrixShapeAttrGetM(self.ptr()); + } + pub fn n(self: *const Self) c_int { + return c.mlirCuteNVVMLdStMatrixShapeAttrGetN(self.ptr()); + } +}; + +/// `#nvvm.shape`: Shape of an MMA operation +pub const MMAShapeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMMMAShape; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "shape"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + m: c_int, + n: c_int, + k: c_int, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMMMAShapeAttrGet(ctx.ptr(), args.m, args.n, args.k); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn m(self: *const Self) c_int { + return c.mlirCuteNVVMMMAShapeAttrGetM(self.ptr()); + } + pub fn n(self: *const Self) c_int { + return c.mlirCuteNVVMMMAShapeAttrGetN(self.ptr()); + } + pub fn k(self: *const Self) c_int { + return c.mlirCuteNVVMMMAShapeAttrGetK(self.ptr()); + } +}; + +/// `#nvvm.target`: GPU target of an NVVM module +pub const TargetAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsACuteNVVMTarget; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "target"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + O: c_int = 2, + triple: []const u8 = "nvptx64-nvidia-cuda", + chip: []const u8 = "sm_75", + features: []const u8 = "", + flags: ?*const mlir.Attribute = null, + link: ?*const mlir.Attribute = null, + verifyTarget: bool = false, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirCuteNVVMTargetAttrGet(ctx.ptr(), args.O, mlir.stringRef(args.triple), mlir.stringRef(args.chip), mlir.stringRef(args.features), if (args.flags) |v| v.ptr() else .{ .ptr = null }, if (args.link) |v| v.ptr() else .{ .ptr = null }, args.verifyTarget); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn o(self: *const Self) c_int { + return c.mlirCuteNVVMTargetAttrGetO(self.ptr()); + } + pub fn triple(self: *const Self) []const u8 { + const s = c.mlirCuteNVVMTargetAttrGetTriple(self.ptr()); + return if (s.length == 0) "" else mlir.string(s); + } + pub fn chip(self: *const Self) []const u8 { + const s = c.mlirCuteNVVMTargetAttrGetChip(self.ptr()); + return if (s.length == 0) "" else mlir.string(s); + } + pub fn features(self: *const Self) []const u8 { + const s = c.mlirCuteNVVMTargetAttrGetFeatures(self.ptr()); + return if (s.length == 0) "" else mlir.string(s); + } + pub fn flags(self: *const Self) ?*const mlir.Attribute { + return @ptrCast(c.mlirCuteNVVMTargetAttrGetFlags(self.ptr()).ptr); + } + pub fn link(self: *const Self) ?*const mlir.Attribute { + return @ptrCast(c.mlirCuteNVVMTargetAttrGetLink(self.ptr()).ptr); + } + pub fn verifyTarget(self: *const Self) bool { + return c.mlirCuteNVVMTargetAttrGetVerifyTarget(self.ptr()); + } +}; + +// Operation builders, generated from the operations' .td. +/// `nvvm.add.packed.bf16x2.f32x2.f32x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn add_packed_bf16x2_f32x2_f32x2(ctx: *mlir.Context, srcA: *const mlir.Value, srcB: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.add.packed.bf16x2.f32x2.f32x2", .{ + .operands = .{ .flat = &.{ srcA, srcB } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.add.packed.f16x2.f32x2.f32x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn add_packed_f16x2_f32x2_f32x2(ctx: *mlir.Context, srcA: *const mlir.Value, srcB: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.add.packed.f16x2.f32x2.f32x2", .{ + .operands = .{ .flat = &.{ srcA, srcB } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.add.packed.f32x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn add_packed_f32x2( + ctx: *mlir.Context, + srcA: *const mlir.Value, + srcB: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + rnd: ?*const mlir.Attribute = null, + ftz: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); + return mlir.Operation.make(ctx, "nvvm.add.packed.f32x2", .{ + .operands = .{ .flat = &.{ srcA, srcB } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.add.packed.f32x2.bf16x2.f32x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn add_packed_f32x2_bf16x2_f32x2( + ctx: *mlir.Context, + srcA: *const mlir.Value, + srcB: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + rnd: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + return mlir.Operation.make(ctx, "nvvm.add.packed.f32x2.bf16x2.f32x2", .{ + .operands = .{ .flat = &.{ srcA, srcB } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.add.packed.f32x2.f16x2.f32x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn add_packed_f32x2_f16x2_f32x2( + ctx: *mlir.Context, + srcA: *const mlir.Value, + srcB: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + rnd: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + return mlir.Operation.make(ctx, "nvvm.add.packed.f32x2.f16x2.f32x2", .{ + .operands = .{ .flat = &.{ srcA, srcB } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.addf`. Result types are explicit; attributes use mlir.Attribute. +pub fn addf( + ctx: *mlir.Context, + lhs: *const mlir.Value, + rhs: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + rnd: ?*const mlir.Attribute = null, + sat: ?*const mlir.Attribute = null, + ftz: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); + if (opts.ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); + return mlir.Operation.make(ctx, "nvvm.addf", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.applypriority.async.bulk`. Result types are explicit; attributes use mlir.Attribute. +pub fn applypriority_async_bulk( + ctx: *mlir.Context, + ptr: *const mlir.Value, + size: *const mlir.Value, + opts: struct { + evict: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.evict) |value| attributes.appendAssumeCapacity(.named(ctx, "evict", value)); + return mlir.Operation.make(ctx, "nvvm.applypriority.async.bulk", .{ + .operands = .{ .flat = &.{ ptr, size } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.applypriority.async.bulk.tensor`. Result types are explicit; attributes use mlir.Attribute. +pub fn applypriority_async_bulk_tensor( + ctx: *mlir.Context, + tmaDesc: *const mlir.Value, + coordinates: []const *const mlir.Value, + im2colArgs: []const *const mlir.Value, + opts: struct { + mode: ?*const mlir.Attribute = null, + evict: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); + if (opts.evict) |value| attributes.appendAssumeCapacity(.named(ctx, "evict", value)); + return mlir.Operation.make(ctx, "nvvm.applypriority.async.bulk.tensor", .{ + .operands = .{ .variadic = &.{ + &.{tmaDesc}, + coordinates, + im2colArgs, + } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.applypriority.async.bulk.tensor.override`. Result types are explicit; attributes use mlir.Attribute. +pub fn applypriority_async_bulk_tensor_override( + ctx: *mlir.Context, + tmaDesc: *const mlir.Value, + overrideAdrr: *const mlir.Value, + coordinates: []const *const mlir.Value, + tensorSize: []const *const mlir.Value, + lowerStride: []const *const mlir.Value, + im2colArgs: []const *const mlir.Value, + opts: struct { + upperStride: ?*const mlir.Value = null, + mode: ?*const mlir.Attribute = null, + evict: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); + if (opts.evict) |value| attributes.appendAssumeCapacity(.named(ctx, "evict", value)); + return mlir.Operation.make(ctx, "nvvm.applypriority.async.bulk.tensor.override", .{ + .operands = .{ .variadic = &.{ + &.{tmaDesc}, + &.{overrideAdrr}, + coordinates, + tensorSize, + lowerStride, + if (opts.upperStride) |value| &.{value} else &.{}, + im2colArgs, + } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.atomicrmw`. Result types are explicit; attributes use mlir.Attribute. +pub fn atomicrmw( + ctx: *mlir.Context, + ptr: *const mlir.Value, + a: *const mlir.Value, + res_type: *const mlir.Type, + op: *const mlir.Attribute, + opts: struct { + b: ?*const mlir.Value = null, + memOrder: ?*const mlir.Attribute = null, + syncscope: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "op", op)); + if (opts.memOrder) |value| attributes.appendAssumeCapacity(.named(ctx, "memOrder", value)); + if (opts.syncscope) |value| attributes.appendAssumeCapacity(.named(ctx, "syncscope", value)); + return mlir.Operation.make(ctx, "nvvm.atomicrmw", .{ + .operands = .{ .flat = if (opts.b) |value| &.{ ptr, a, value } else &.{ ptr, a } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.bar.warp.sync`. Result types are explicit; attributes use mlir.Attribute. +pub fn bar_warp_sync(ctx: *mlir.Context, mask: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.bar.warp.sync", .{ + .operands = .{ .flat = &.{mask} }, + .location = location, + }); +} + +/// `nvvm.barrier`. Result types are explicit; attributes use mlir.Attribute. +pub fn barrier( + ctx: *mlir.Context, + opts: struct { + barrierId: ?*const mlir.Value = null, + numberOfThreads: ?*const mlir.Value = null, + reductionPredicate: ?*const mlir.Value = null, + res_type: ?*const mlir.Type = null, + reductionOp: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var results: stdx.BoundedArray(*const mlir.Type, 256) = .empty; + if (opts.res_type) |value| results.appendAssumeCapacity(value); + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.reductionOp) |value| attributes.appendAssumeCapacity(.named(ctx, "reductionOp", value)); + return mlir.Operation.make(ctx, "nvvm.barrier", .{ + .operands = .{ .variadic = &.{ + if (opts.barrierId) |value| &.{value} else &.{}, + if (opts.numberOfThreads) |value| &.{value} else &.{}, + if (opts.reductionPredicate) |value| &.{value} else &.{}, + } }, + .results = .{ .flat = results.constSlice() }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.barrier.arrive`. Result types are explicit; attributes use mlir.Attribute. +pub fn barrier_arrive( + ctx: *mlir.Context, + numberOfThreads: *const mlir.Value, + opts: struct { + barrierId: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.barrier.arrive", .{ + .operands = .{ .flat = if (opts.barrierId) |value| &.{ value, numberOfThreads } else &.{numberOfThreads} }, + .location = location, + }); +} + +/// `nvvm.barrier.cta.arrive`. Result types are explicit; attributes use mlir.Attribute. +pub fn barrier_cta_arrive( + ctx: *mlir.Context, + barrierId: *const mlir.Value, + threadCount: *const mlir.Value, + opts: struct { + aligned: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.aligned) |value| attributes.appendAssumeCapacity(.named(ctx, "aligned", value)); + return mlir.Operation.make(ctx, "nvvm.barrier.cta.arrive", .{ + .operands = .{ .flat = &.{ barrierId, threadCount } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.barrier.cta.red`. Result types are explicit; attributes use mlir.Attribute. +pub fn barrier_cta_red( + ctx: *mlir.Context, + pred: *const mlir.Value, + barrierId: *const mlir.Value, + res_type: *const mlir.Type, + kind: *const mlir.Attribute, + opts: struct { + threadCount: ?*const mlir.Value = null, + aligned: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + if (opts.aligned) |value| attributes.appendAssumeCapacity(.named(ctx, "aligned", value)); + return mlir.Operation.make(ctx, "nvvm.barrier.cta.red", .{ + .operands = .{ .flat = if (opts.threadCount) |value| &.{ pred, barrierId, value } else &.{ pred, barrierId } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.barrier.cta.sync`. Result types are explicit; attributes use mlir.Attribute. +pub fn barrier_cta_sync( + ctx: *mlir.Context, + barrierId: *const mlir.Value, + opts: struct { + threadCount: ?*const mlir.Value = null, + aligned: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.aligned) |value| attributes.appendAssumeCapacity(.named(ctx, "aligned", value)); + return mlir.Operation.make(ctx, "nvvm.barrier.cta.sync", .{ + .operands = .{ .flat = if (opts.threadCount) |value| &.{ barrierId, value } else &.{barrierId} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.barrier0`. Result types are explicit; attributes use mlir.Attribute. +pub fn barrier0(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.barrier0", .{ + .location = location, + }); +} + +/// `nvvm.breakpoint`. Result types are explicit; attributes use mlir.Attribute. +pub fn breakpoint(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.breakpoint", .{ + .location = location, + }); +} + +/// `nvvm.clmad`. Result types are explicit; attributes use mlir.Attribute. +pub fn clmad( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c_: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + high: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.high) |value| attributes.appendAssumeCapacity(.named(ctx, "high", value)); + return mlir.Operation.make(ctx, "nvvm.clmad", .{ + .operands = .{ .flat = &.{ a, b, c_ } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.cluster.arrive`. Result types are explicit; attributes use mlir.Attribute. +pub fn cluster_arrive( + ctx: *mlir.Context, + opts: struct { + aligned: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.aligned) |value| attributes.appendAssumeCapacity(.named(ctx, "aligned", value)); + return mlir.Operation.make(ctx, "nvvm.cluster.arrive", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.cluster.arrive.relaxed`. Result types are explicit; attributes use mlir.Attribute. +pub fn cluster_arrive_relaxed( + ctx: *mlir.Context, + opts: struct { + aligned: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.aligned) |value| attributes.appendAssumeCapacity(.named(ctx, "aligned", value)); + return mlir.Operation.make(ctx, "nvvm.cluster.arrive.relaxed", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.cluster.wait`. Result types are explicit; attributes use mlir.Attribute. +pub fn cluster_wait( + ctx: *mlir.Context, + opts: struct { + aligned: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.aligned) |value| attributes.appendAssumeCapacity(.named(ctx, "aligned", value)); + return mlir.Operation.make(ctx, "nvvm.cluster.wait", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.clusterlaunchcontrol.query.cancel`. Result types are explicit; attributes use mlir.Attribute. +pub fn clusterlaunchcontrol_query_cancel(ctx: *mlir.Context, try_cancel_response: *const mlir.Value, res_type: *const mlir.Type, query_type: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.clusterlaunchcontrol.query.cancel", .{ + .operands = .{ .flat = &.{try_cancel_response} }, + .results = .{ .flat = &.{res_type} }, + .attributes = &.{ + .named(ctx, "query_type", query_type), + }, + .location = location, + }); +} + +/// `nvvm.clusterlaunchcontrol.try.cancel`. Result types are explicit; attributes use mlir.Attribute. +pub fn clusterlaunchcontrol_try_cancel( + ctx: *mlir.Context, + smemAddress: *const mlir.Value, + mbarrier: *const mlir.Value, + opts: struct { + multicast: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.multicast) |value| attributes.appendAssumeCapacity(.named(ctx, "multicast", value)); + return mlir.Operation.make(ctx, "nvvm.clusterlaunchcontrol.try.cancel", .{ + .operands = .{ .flat = &.{ smemAddress, mbarrier } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.compare.and.set`. Result types are explicit; attributes use mlir.Attribute. +pub fn compare_and_set(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, result_type: *const mlir.Type, op: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.compare.and.set", .{ + .operands = .{ .flat = &.{ a, b } }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "op", op), + }, + .location = location, + }); +} + +/// `nvvm.convert.and.pack.integer`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_and_pack_integer( + ctx: *mlir.Context, + srcA: *const mlir.Value, + srcB: *const mlir.Value, + dst_type: *const mlir.Type, + convert_type: *const mlir.Attribute, + opts: struct { + srcC: ?*const mlir.Value = null, + is_signed: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.is_signed) |value| attributes.appendAssumeCapacity(.named(ctx, "is_signed", value)); + attributes.appendAssumeCapacity(.named(ctx, "convert_type", convert_type)); + return mlir.Operation.make(ctx, "nvvm.convert.and.pack.integer", .{ + .operands = .{ .flat = if (opts.srcC) |value| &.{ srcA, srcB, value } else &.{ srcA, srcB } }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.bf16x2.to.f4x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_bf16x2_to_f4x2( + ctx: *mlir.Context, + src: *const mlir.Value, + dst_type: *const mlir.Type, + dstTy: *const mlir.Attribute, + opts: struct { + scale_factor: ?*const mlir.Value = null, + isPZO: ?*const mlir.Attribute = null, + scale_factor_kind: ?*const mlir.Attribute = null, + rnd: ?*const mlir.Attribute = null, + relu: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + if (opts.isPZO) |value| attributes.appendAssumeCapacity(.named(ctx, "isPZO", value)); + if (opts.scale_factor_kind) |value| attributes.appendAssumeCapacity(.named(ctx, "scale_factor_kind", value)); + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + attributes.appendAssumeCapacity(.named(ctx, "dstTy", dstTy)); + return mlir.Operation.make(ctx, "nvvm.convert.bf16x2.to.f4x2", .{ + .operands = .{ .flat = if (opts.scale_factor) |value| &.{ src, value } else &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.bf16x2.to.f6x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_bf16x2_to_f6x2( + ctx: *mlir.Context, + src: *const mlir.Value, + dst_type: *const mlir.Type, + dstTy: *const mlir.Attribute, + opts: struct { + scale_factor: ?*const mlir.Value = null, + isPZO: ?*const mlir.Attribute = null, + scale_factor_kind: ?*const mlir.Attribute = null, + rnd: ?*const mlir.Attribute = null, + relu: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + if (opts.isPZO) |value| attributes.appendAssumeCapacity(.named(ctx, "isPZO", value)); + if (opts.scale_factor_kind) |value| attributes.appendAssumeCapacity(.named(ctx, "scale_factor_kind", value)); + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + attributes.appendAssumeCapacity(.named(ctx, "dstTy", dstTy)); + return mlir.Operation.make(ctx, "nvvm.convert.bf16x2.to.f6x2", .{ + .operands = .{ .flat = if (opts.scale_factor) |value| &.{ src, value } else &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.bf16x2.to.f8x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_bf16x2_to_f8x2( + ctx: *mlir.Context, + src: *const mlir.Value, + dst_type: *const mlir.Type, + dstTy: *const mlir.Attribute, + opts: struct { + scale_factor: ?*const mlir.Value = null, + isPZO: ?*const mlir.Attribute = null, + scale_factor_kind: ?*const mlir.Attribute = null, + rnd: ?*const mlir.Attribute = null, + sat: ?*const mlir.Attribute = null, + relu: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 6) = .empty; + if (opts.isPZO) |value| attributes.appendAssumeCapacity(.named(ctx, "isPZO", value)); + if (opts.scale_factor_kind) |value| attributes.appendAssumeCapacity(.named(ctx, "scale_factor_kind", value)); + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + attributes.appendAssumeCapacity(.named(ctx, "dstTy", dstTy)); + return mlir.Operation.make(ctx, "nvvm.convert.bf16x2.to.f8x2", .{ + .operands = .{ .flat = if (opts.scale_factor) |value| &.{ src, value } else &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.bf16x2.to.s2f6x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_bf16x2_to_s2f6x2( + ctx: *mlir.Context, + src: *const mlir.Value, + dst_type: *const mlir.Type, + opts: struct { + scaleFactor: ?*const mlir.Value = null, + relu: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + return mlir.Operation.make(ctx, "nvvm.convert.bf16x2.to.s2f6x2", .{ + .operands = .{ .flat = if (opts.scaleFactor) |value| &.{ src, value } else &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.f16x2.to.f4x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_f16x2_to_f4x2( + ctx: *mlir.Context, + src: *const mlir.Value, + dst_type: *const mlir.Type, + dstTy: *const mlir.Attribute, + opts: struct { + scale_factor: ?*const mlir.Value = null, + isPZO: ?*const mlir.Attribute = null, + scale_factor_kind: ?*const mlir.Attribute = null, + rnd: ?*const mlir.Attribute = null, + relu: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + if (opts.isPZO) |value| attributes.appendAssumeCapacity(.named(ctx, "isPZO", value)); + if (opts.scale_factor_kind) |value| attributes.appendAssumeCapacity(.named(ctx, "scale_factor_kind", value)); + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + attributes.appendAssumeCapacity(.named(ctx, "dstTy", dstTy)); + return mlir.Operation.make(ctx, "nvvm.convert.f16x2.to.f4x2", .{ + .operands = .{ .flat = if (opts.scale_factor) |value| &.{ src, value } else &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.f16x2.to.f6x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_f16x2_to_f6x2( + ctx: *mlir.Context, + src: *const mlir.Value, + dst_type: *const mlir.Type, + dstTy: *const mlir.Attribute, + opts: struct { + scale_factor: ?*const mlir.Value = null, + isPZO: ?*const mlir.Attribute = null, + scale_factor_kind: ?*const mlir.Attribute = null, + rnd: ?*const mlir.Attribute = null, + relu: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + if (opts.isPZO) |value| attributes.appendAssumeCapacity(.named(ctx, "isPZO", value)); + if (opts.scale_factor_kind) |value| attributes.appendAssumeCapacity(.named(ctx, "scale_factor_kind", value)); + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + attributes.appendAssumeCapacity(.named(ctx, "dstTy", dstTy)); + return mlir.Operation.make(ctx, "nvvm.convert.f16x2.to.f6x2", .{ + .operands = .{ .flat = if (opts.scale_factor) |value| &.{ src, value } else &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.f16x2.to.f8x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_f16x2_to_f8x2( + ctx: *mlir.Context, + a: *const mlir.Value, + dst_type: *const mlir.Type, + dstTy: *const mlir.Attribute, + opts: struct { + scale_factor: ?*const mlir.Value = null, + isPZO: ?*const mlir.Attribute = null, + scale_factor_kind: ?*const mlir.Attribute = null, + rnd: ?*const mlir.Attribute = null, + sat: ?*const mlir.Attribute = null, + relu: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 6) = .empty; + if (opts.isPZO) |value| attributes.appendAssumeCapacity(.named(ctx, "isPZO", value)); + if (opts.scale_factor_kind) |value| attributes.appendAssumeCapacity(.named(ctx, "scale_factor_kind", value)); + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + attributes.appendAssumeCapacity(.named(ctx, "dstTy", dstTy)); + return mlir.Operation.make(ctx, "nvvm.convert.f16x2.to.f8x2", .{ + .operands = .{ .flat = if (opts.scale_factor) |value| &.{ a, value } else &.{a} }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.f32x2.to.bf16x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_f32x2_to_bf16x2( + ctx: *mlir.Context, + src_hi: *const mlir.Value, + src_lo: *const mlir.Value, + dst_type: *const mlir.Type, + opts: struct { + random_bits: ?*const mlir.Value = null, + rnd: ?*const mlir.Attribute = null, + sat: ?*const mlir.Attribute = null, + relu: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + return mlir.Operation.make(ctx, "nvvm.convert.f32x2.to.bf16x2", .{ + .operands = .{ .flat = if (opts.random_bits) |value| &.{ src_hi, src_lo, value } else &.{ src_hi, src_lo } }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.f32x2.to.f16x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_f32x2_to_f16x2( + ctx: *mlir.Context, + src_hi: *const mlir.Value, + src_lo: *const mlir.Value, + dst_type: *const mlir.Type, + opts: struct { + random_bits: ?*const mlir.Value = null, + rnd: ?*const mlir.Attribute = null, + sat: ?*const mlir.Attribute = null, + relu: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + return mlir.Operation.make(ctx, "nvvm.convert.f32x2.to.f16x2", .{ + .operands = .{ .flat = if (opts.random_bits) |value| &.{ src_hi, src_lo, value } else &.{ src_hi, src_lo } }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.f32x2.to.f4x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_f32x2_to_f4x2( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + dst_type: *const mlir.Type, + dstType: *const mlir.Attribute, + opts: struct { + scale_factor: ?*const mlir.Value = null, + isPZO: ?*const mlir.Attribute = null, + scale_factor_kind: ?*const mlir.Attribute = null, + rnd: ?*const mlir.Attribute = null, + relu: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + if (opts.isPZO) |value| attributes.appendAssumeCapacity(.named(ctx, "isPZO", value)); + if (opts.scale_factor_kind) |value| attributes.appendAssumeCapacity(.named(ctx, "scale_factor_kind", value)); + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + attributes.appendAssumeCapacity(.named(ctx, "dstType", dstType)); + return mlir.Operation.make(ctx, "nvvm.convert.f32x2.to.f4x2", .{ + .operands = .{ .flat = if (opts.scale_factor) |value| &.{ a, b, value } else &.{ a, b } }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.f32x2.to.f6x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_f32x2_to_f6x2( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + dst_type: *const mlir.Type, + dstTy: *const mlir.Attribute, + opts: struct { + scale_factor: ?*const mlir.Value = null, + isPZO: ?*const mlir.Attribute = null, + scale_factor_kind: ?*const mlir.Attribute = null, + rnd: ?*const mlir.Attribute = null, + relu: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + if (opts.isPZO) |value| attributes.appendAssumeCapacity(.named(ctx, "isPZO", value)); + if (opts.scale_factor_kind) |value| attributes.appendAssumeCapacity(.named(ctx, "scale_factor_kind", value)); + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + attributes.appendAssumeCapacity(.named(ctx, "dstTy", dstTy)); + return mlir.Operation.make(ctx, "nvvm.convert.f32x2.to.f6x2", .{ + .operands = .{ .flat = if (opts.scale_factor) |value| &.{ a, b, value } else &.{ a, b } }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.f32x2.to.f8x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_f32x2_to_f8x2( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + dst_type: *const mlir.Type, + dstTy: *const mlir.Attribute, + opts: struct { + scale_factor: ?*const mlir.Value = null, + isPZO: ?*const mlir.Attribute = null, + scale_factor_kind: ?*const mlir.Attribute = null, + rnd: ?*const mlir.Attribute = null, + sat: ?*const mlir.Attribute = null, + relu: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 6) = .empty; + if (opts.isPZO) |value| attributes.appendAssumeCapacity(.named(ctx, "isPZO", value)); + if (opts.scale_factor_kind) |value| attributes.appendAssumeCapacity(.named(ctx, "scale_factor_kind", value)); + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + attributes.appendAssumeCapacity(.named(ctx, "dstTy", dstTy)); + return mlir.Operation.make(ctx, "nvvm.convert.f32x2.to.f8x2", .{ + .operands = .{ .flat = if (opts.scale_factor) |value| &.{ a, b, value } else &.{ a, b } }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.f32x2.to.s2f6x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_f32x2_to_s2f6x2( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + dst_type: *const mlir.Type, + opts: struct { + scaleFactor: ?*const mlir.Value = null, + relu: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + return mlir.Operation.make(ctx, "nvvm.convert.f32x2.to.s2f6x2", .{ + .operands = .{ .flat = if (opts.scaleFactor) |value| &.{ a, b, value } else &.{ a, b } }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.f32x4.to.f4x4`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_f32x4_to_f4x4( + ctx: *mlir.Context, + src: *const mlir.Value, + rbits: *const mlir.Value, + dst_type: *const mlir.Type, + dstTy: *const mlir.Attribute, + opts: struct { + relu: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + attributes.appendAssumeCapacity(.named(ctx, "dstTy", dstTy)); + return mlir.Operation.make(ctx, "nvvm.convert.f32x4.to.f4x4", .{ + .operands = .{ .flat = &.{ src, rbits } }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.f32x4.to.f6x4`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_f32x4_to_f6x4( + ctx: *mlir.Context, + src: *const mlir.Value, + rbits: *const mlir.Value, + dst_type: *const mlir.Type, + dstTy: *const mlir.Attribute, + opts: struct { + relu: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + attributes.appendAssumeCapacity(.named(ctx, "dstTy", dstTy)); + return mlir.Operation.make(ctx, "nvvm.convert.f32x4.to.f6x4", .{ + .operands = .{ .flat = &.{ src, rbits } }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.f32x4.to.f8x4`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_f32x4_to_f8x4( + ctx: *mlir.Context, + src: *const mlir.Value, + rbits: *const mlir.Value, + dst_type: *const mlir.Type, + dstTy: *const mlir.Attribute, + opts: struct { + relu: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + attributes.appendAssumeCapacity(.named(ctx, "dstTy", dstTy)); + return mlir.Operation.make(ctx, "nvvm.convert.f32x4.to.f8x4", .{ + .operands = .{ .flat = &.{ src, rbits } }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.f4x2.to.f16x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_f4x2_to_f16x2( + ctx: *mlir.Context, + src: *const mlir.Value, + dst_type: *const mlir.Type, + srcType: *const mlir.Attribute, + opts: struct { + relu: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + attributes.appendAssumeCapacity(.named(ctx, "srcType", srcType)); + return mlir.Operation.make(ctx, "nvvm.convert.f4x2.to.f16x2", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.f6x2.to.f16x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_f6x2_to_f16x2( + ctx: *mlir.Context, + src: *const mlir.Value, + dst_type: *const mlir.Type, + srcType: *const mlir.Attribute, + opts: struct { + relu: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + attributes.appendAssumeCapacity(.named(ctx, "srcType", srcType)); + return mlir.Operation.make(ctx, "nvvm.convert.f6x2.to.f16x2", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.f8x2.to.bf16x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_f8x2_to_bf16x2( + ctx: *mlir.Context, + src: *const mlir.Value, + dst_type: *const mlir.Type, + srcType: *const mlir.Attribute, + opts: struct { + scaleFactor: ?*const mlir.Value = null, + sat: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); + attributes.appendAssumeCapacity(.named(ctx, "srcType", srcType)); + return mlir.Operation.make(ctx, "nvvm.convert.f8x2.to.bf16x2", .{ + .operands = .{ .flat = if (opts.scaleFactor) |value| &.{ src, value } else &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.f8x2.to.f16x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_f8x2_to_f16x2( + ctx: *mlir.Context, + src: *const mlir.Value, + dst_type: *const mlir.Type, + srcType: *const mlir.Attribute, + opts: struct { + relu: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + attributes.appendAssumeCapacity(.named(ctx, "srcType", srcType)); + return mlir.Operation.make(ctx, "nvvm.convert.f8x2.to.f16x2", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.float.to.integer`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_float_to_integer( + ctx: *mlir.Context, + src: *const mlir.Value, + dst_type: *const mlir.Type, + opts: struct { + rnd: ?*const mlir.Attribute = null, + sat: ?*const mlir.Attribute = null, + ftz: ?*const mlir.Attribute = null, + is_signed: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); + if (opts.ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); + if (opts.is_signed) |value| attributes.appendAssumeCapacity(.named(ctx, "is_signed", value)); + return mlir.Operation.make(ctx, "nvvm.convert.float.to.integer", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.float.to.tf32`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_float_to_tf32( + ctx: *mlir.Context, + src: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + rnd: ?*const mlir.Attribute = null, + sat: ?*const mlir.Attribute = null, + relu: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + return mlir.Operation.make(ctx, "nvvm.convert.float.to.tf32", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.convert.s2f6x2.to.bf16x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn convert_s2f6x2_to_bf16x2( + ctx: *mlir.Context, + src: *const mlir.Value, + dst_type: *const mlir.Type, + opts: struct { + scaleFactor: ?*const mlir.Value = null, + sat: ?*const mlir.Attribute = null, + relu: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + return mlir.Operation.make(ctx, "nvvm.convert.s2f6x2.to.bf16x2", .{ + .operands = .{ .flat = if (opts.scaleFactor) |value| &.{ src, value } else &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.cos`. Result types are explicit; attributes use mlir.Attribute. +pub fn cos( + ctx: *mlir.Context, + src: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + ftz: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); + return mlir.Operation.make(ctx, "nvvm.cos", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.cp.async.bulk.commit.group`. Result types are explicit; attributes use mlir.Attribute. +pub fn cp_async_bulk_commit_group(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.commit.group", .{ + .location = location, + }); +} + +/// `nvvm.cp.async.bulk.global.shared.cta`. Result types are explicit; attributes use mlir.Attribute. +pub fn cp_async_bulk_global_shared_cta( + ctx: *mlir.Context, + dstMem: *const mlir.Value, + srcMem: *const mlir.Value, + size: *const mlir.Value, + opts: struct { + l2CacheHint: ?*const mlir.Value = null, + byteMask: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.global.shared.cta", .{ + .operands = .{ .variadic = &.{ + &.{dstMem}, + &.{srcMem}, + &.{size}, + if (opts.l2CacheHint) |value| &.{value} else &.{}, + if (opts.byteMask) |value| &.{value} else &.{}, + } }, + .location = location, + }); +} + +/// `nvvm.cp.async.bulk.prefetch`. Result types are explicit; attributes use mlir.Attribute. +pub fn cp_async_bulk_prefetch( + ctx: *mlir.Context, + srcMem: *const mlir.Value, + size: *const mlir.Value, + opts: struct { + l2CacheHint: ?*const mlir.Value = null, + evict: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.evict) |value| attributes.appendAssumeCapacity(.named(ctx, "evict", value)); + return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.prefetch", .{ + .operands = .{ .flat = if (opts.l2CacheHint) |value| &.{ srcMem, size, value } else &.{ srcMem, size } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.cp.async.bulk.shared.cluster.global`. Result types are explicit; attributes use mlir.Attribute. +pub fn cp_async_bulk_shared_cluster_global( + ctx: *mlir.Context, + dstMem: *const mlir.Value, + srcMem: *const mlir.Value, + mbar: *const mlir.Value, + size: *const mlir.Value, + opts: struct { + multicastMask: ?*const mlir.Value = null, + l2CacheHint: ?*const mlir.Value = null, + validatePattern: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.validatePattern) |value| attributes.appendAssumeCapacity(.named(ctx, "validatePattern", value)); + return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.shared.cluster.global", .{ + .operands = .{ .variadic = &.{ + &.{dstMem}, + &.{srcMem}, + &.{mbar}, + &.{size}, + if (opts.multicastMask) |value| &.{value} else &.{}, + if (opts.l2CacheHint) |value| &.{value} else &.{}, + } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.cp.async.bulk.shared.cluster.shared.cta`. Result types are explicit; attributes use mlir.Attribute. +pub fn cp_async_bulk_shared_cluster_shared_cta(ctx: *mlir.Context, dstMem: *const mlir.Value, srcMem: *const mlir.Value, mbar: *const mlir.Value, size: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.shared.cluster.shared.cta", .{ + .operands = .{ .flat = &.{ dstMem, srcMem, mbar, size } }, + .location = location, + }); +} + +/// `nvvm.cp.async.bulk.tensor.global.shared.cta`. Result types are explicit; attributes use mlir.Attribute. +pub fn cp_async_bulk_tensor_global_shared_cta( + ctx: *mlir.Context, + tmaDescriptor: *const mlir.Value, + srcMem: *const mlir.Value, + coordinates: []const *const mlir.Value, + opts: struct { + l2CacheHint: ?*const mlir.Value = null, + predicate: ?*const mlir.Value = null, + mode: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); + return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.tensor.global.shared.cta", .{ + .operands = .{ .variadic = &.{ + &.{tmaDescriptor}, + &.{srcMem}, + coordinates, + if (opts.l2CacheHint) |value| &.{value} else &.{}, + if (opts.predicate) |value| &.{value} else &.{}, + } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.cp.async.bulk.tensor.global.shared.cta.override`. Result types are explicit; attributes use mlir.Attribute. +pub fn cp_async_bulk_tensor_global_shared_cta_override( + ctx: *mlir.Context, + tmaDesc: *const mlir.Value, + srcMem: *const mlir.Value, + overrideAdrr: *const mlir.Value, + coordinates: []const *const mlir.Value, + tensorSize: []const *const mlir.Value, + lowerStride: []const *const mlir.Value, + opts: struct { + upperStride: ?*const mlir.Value = null, + l2CacheHint: ?*const mlir.Value = null, + mode: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); + return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.tensor.global.shared.cta.override", .{ + .operands = .{ .variadic = &.{ + &.{tmaDesc}, + &.{srcMem}, + &.{overrideAdrr}, + coordinates, + tensorSize, + lowerStride, + if (opts.upperStride) |value| &.{value} else &.{}, + if (opts.l2CacheHint) |value| &.{value} else &.{}, + } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.cp.async.bulk.tensor.prefetch`. Result types are explicit; attributes use mlir.Attribute. +pub fn cp_async_bulk_tensor_prefetch( + ctx: *mlir.Context, + tmaDescriptor: *const mlir.Value, + coordinates: []const *const mlir.Value, + im2colOffsets: []const *const mlir.Value, + opts: struct { + l2CacheHint: ?*const mlir.Value = null, + mode: ?*const mlir.Attribute = null, + evict: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); + if (opts.evict) |value| attributes.appendAssumeCapacity(.named(ctx, "evict", value)); + return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.tensor.prefetch", .{ + .operands = .{ .variadic = &.{ + &.{tmaDescriptor}, + coordinates, + im2colOffsets, + if (opts.l2CacheHint) |value| &.{value} else &.{}, + } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.cp.async.bulk.tensor.prefetch.override`. Result types are explicit; attributes use mlir.Attribute. +pub fn cp_async_bulk_tensor_prefetch_override( + ctx: *mlir.Context, + tmaDesc: *const mlir.Value, + overrideAdrr: *const mlir.Value, + coordinates: []const *const mlir.Value, + tensorSize: []const *const mlir.Value, + lowerStride: []const *const mlir.Value, + im2colOffsets: []const *const mlir.Value, + opts: struct { + upperStride: ?*const mlir.Value = null, + l2CacheHint: ?*const mlir.Value = null, + evict: ?*const mlir.Attribute = null, + mode: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.evict) |value| attributes.appendAssumeCapacity(.named(ctx, "evict", value)); + if (opts.mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); + return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.tensor.prefetch.override", .{ + .operands = .{ .variadic = &.{ + &.{tmaDesc}, + &.{overrideAdrr}, + coordinates, + tensorSize, + lowerStride, + if (opts.upperStride) |value| &.{value} else &.{}, + im2colOffsets, + if (opts.l2CacheHint) |value| &.{value} else &.{}, + } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.cp.async.bulk.tensor.reduce`. Result types are explicit; attributes use mlir.Attribute. +pub fn cp_async_bulk_tensor_reduce( + ctx: *mlir.Context, + tmaDescriptor: *const mlir.Value, + srcMem: *const mlir.Value, + coordinates: []const *const mlir.Value, + redKind: *const mlir.Attribute, + opts: struct { + l2CacheHint: ?*const mlir.Value = null, + mode: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "redKind", redKind)); + if (opts.mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); + return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.tensor.reduce", .{ + .operands = .{ .variadic = &.{ + &.{tmaDescriptor}, + &.{srcMem}, + coordinates, + if (opts.l2CacheHint) |value| &.{value} else &.{}, + } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.cp.async.bulk.tensor.reduce.override`. Result types are explicit; attributes use mlir.Attribute. +pub fn cp_async_bulk_tensor_reduce_override( + ctx: *mlir.Context, + tmaDesc: *const mlir.Value, + srcMem: *const mlir.Value, + overrideAdrr: *const mlir.Value, + coordinates: []const *const mlir.Value, + tensorSize: []const *const mlir.Value, + lowerStride: []const *const mlir.Value, + redKind: *const mlir.Attribute, + opts: struct { + upperStride: ?*const mlir.Value = null, + l2CacheHint: ?*const mlir.Value = null, + mode: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "redKind", redKind)); + if (opts.mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); + return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.tensor.reduce.override", .{ + .operands = .{ .variadic = &.{ + &.{tmaDesc}, + &.{srcMem}, + &.{overrideAdrr}, + coordinates, + tensorSize, + lowerStride, + if (opts.upperStride) |value| &.{value} else &.{}, + if (opts.l2CacheHint) |value| &.{value} else &.{}, + } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.cp.async.bulk.tensor.shared.cluster.global`. Result types are explicit; attributes use mlir.Attribute. +pub fn cp_async_bulk_tensor_shared_cluster_global( + ctx: *mlir.Context, + dstMem: *const mlir.Value, + tmaDescriptor: *const mlir.Value, + coordinates: []const *const mlir.Value, + mbar: *const mlir.Value, + im2colOffsets: []const *const mlir.Value, + opts: struct { + multicastMask: ?*const mlir.Value = null, + l2CacheHint: ?*const mlir.Value = null, + predicate: ?*const mlir.Value = null, + validatePattern: ?*const mlir.Attribute = null, + mode: ?*const mlir.Attribute = null, + isCTAOnly: ?*const mlir.Attribute = null, + group: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (opts.validatePattern) |value| attributes.appendAssumeCapacity(.named(ctx, "validatePattern", value)); + if (opts.mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); + if (opts.isCTAOnly) |value| attributes.appendAssumeCapacity(.named(ctx, "isCTAOnly", value)); + if (opts.group) |value| attributes.appendAssumeCapacity(.named(ctx, "group", value)); + return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.tensor.shared.cluster.global", .{ + .operands = .{ .variadic = &.{ + &.{dstMem}, + &.{tmaDescriptor}, + coordinates, + &.{mbar}, + im2colOffsets, + if (opts.multicastMask) |value| &.{value} else &.{}, + if (opts.l2CacheHint) |value| &.{value} else &.{}, + if (opts.predicate) |value| &.{value} else &.{}, + } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.cp.async.bulk.tensor.shared.cluster.global.override`. Result types are explicit; attributes use mlir.Attribute. +pub fn cp_async_bulk_tensor_shared_cluster_global_override( + ctx: *mlir.Context, + dstMem: *const mlir.Value, + tmaDesc: *const mlir.Value, + mbar: *const mlir.Value, + overrideAddr: *const mlir.Value, + coordinates: []const *const mlir.Value, + im2colArgs: []const *const mlir.Value, + tensorSize: []const *const mlir.Value, + lowerStride: []const *const mlir.Value, + opts: struct { + upperStride: ?*const mlir.Value = null, + multicastMask: ?*const mlir.Value = null, + l2CacheHint: ?*const mlir.Value = null, + group: ?*const mlir.Attribute = null, + validatePattern: ?*const mlir.Attribute = null, + mode: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.group) |value| attributes.appendAssumeCapacity(.named(ctx, "group", value)); + if (opts.validatePattern) |value| attributes.appendAssumeCapacity(.named(ctx, "validatePattern", value)); + if (opts.mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); + return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.tensor.shared.cluster.global.override", .{ + .operands = .{ .variadic = &.{ + &.{dstMem}, + &.{tmaDesc}, + &.{mbar}, + &.{overrideAddr}, + coordinates, + im2colArgs, + tensorSize, + lowerStride, + if (opts.upperStride) |value| &.{value} else &.{}, + if (opts.multicastMask) |value| &.{value} else &.{}, + if (opts.l2CacheHint) |value| &.{value} else &.{}, + } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.cp.async.bulk.wait_group`. Result types are explicit; attributes use mlir.Attribute. +pub fn cp_async_bulk_wait_group( + ctx: *mlir.Context, + group: *const mlir.Attribute, + opts: struct { + read: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "group", group)); + if (opts.read) |value| attributes.appendAssumeCapacity(.named(ctx, "read", value)); + return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.wait_group", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.cp.async.commit.group`. Result types are explicit; attributes use mlir.Attribute. +pub fn cp_async_commit_group(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.cp.async.commit.group", .{ + .location = location, + }); +} + +/// `nvvm.cp.async.mbarrier.arrive`. Result types are explicit; attributes use mlir.Attribute. +pub fn cp_async_mbarrier_arrive( + ctx: *mlir.Context, + addr: *const mlir.Value, + opts: struct { + noinc: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.noinc) |value| attributes.appendAssumeCapacity(.named(ctx, "noinc", value)); + return mlir.Operation.make(ctx, "nvvm.cp.async.mbarrier.arrive", .{ + .operands = .{ .flat = &.{addr} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.cp.async.shared.global`. Result types are explicit; attributes use mlir.Attribute. +pub fn cp_async_shared_global( + ctx: *mlir.Context, + dst: *const mlir.Value, + src: *const mlir.Value, + size: *const mlir.Attribute, + modifier: *const mlir.Attribute, + opts: struct { + cpSize: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.cp.async.shared.global", .{ + .operands = .{ .flat = if (opts.cpSize) |value| &.{ dst, src, value } else &.{ dst, src } }, + .attributes = &.{ + .named(ctx, "size", size), + .named(ctx, "modifier", modifier), + }, + .location = location, + }); +} + +/// `nvvm.cp.async.wait.group`. Result types are explicit; attributes use mlir.Attribute. +pub fn cp_async_wait_group(ctx: *mlir.Context, n: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.cp.async.wait.group", .{ + .attributes = &.{ + .named(ctx, "n", n), + }, + .location = location, + }); +} + +/// `nvvm.cvt.packfloat`. Result types are explicit; attributes use mlir.Attribute. +pub fn cvt_packfloat( + ctx: *mlir.Context, + srcA: *const mlir.Value, + srcC: *const mlir.Value, + res_type: *const mlir.Type, + from: *const mlir.Attribute, + to: *const mlir.Attribute, + opts: struct { + rnd: ?*const mlir.Attribute = null, + sat: ?*const mlir.Attribute = null, + relu: ?*const mlir.Attribute = null, + extractHi: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 6) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "from", from)); + attributes.appendAssumeCapacity(.named(ctx, "to", to)); + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + if (opts.extractHi) |value| attributes.appendAssumeCapacity(.named(ctx, "extractHi", value)); + return mlir.Operation.make(ctx, "nvvm.cvt.packfloat", .{ + .operands = .{ .flat = &.{ srcA, srcC } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.cvt.packfloat.f32`. Result types are explicit; attributes use mlir.Attribute. +pub fn cvt_packfloat_f32( + ctx: *mlir.Context, + srcA: *const mlir.Value, + srcB: *const mlir.Value, + srcC: *const mlir.Value, + res_type: *const mlir.Type, + to: *const mlir.Attribute, + opts: struct { + rnd: ?*const mlir.Attribute = null, + sat: ?*const mlir.Attribute = null, + relu: ?*const mlir.Attribute = null, + extractHi: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "to", to)); + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + if (opts.extractHi) |value| attributes.appendAssumeCapacity(.named(ctx, "extractHi", value)); + return mlir.Operation.make(ctx, "nvvm.cvt.packfloat.f32", .{ + .operands = .{ .flat = &.{ srcA, srcB, srcC } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.cvt.to.f4x8.packed`. Result types are explicit; attributes use mlir.Attribute. +pub fn cvt_to_f4x8_packed( + ctx: *mlir.Context, + srcs: []const *const mlir.Value, + dst_type: *const mlir.Type, + srcType: *const mlir.Attribute, + dstType: *const mlir.Attribute, + opts: struct { + rnd: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + attributes.appendAssumeCapacity(.named(ctx, "srcType", srcType)); + attributes.appendAssumeCapacity(.named(ctx, "dstType", dstType)); + return mlir.Operation.make(ctx, "nvvm.cvt.to.f4x8.packed", .{ + .operands = .{ .flat = srcs }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.dot.accumulate.2way`. Result types are explicit; attributes use mlir.Attribute. +pub fn dot_accumulate_2way(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, c_: *const mlir.Value, res_type: *const mlir.Type, a_type: *const mlir.Attribute, b_type: *const mlir.Attribute, b_hi: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.dot.accumulate.2way", .{ + .operands = .{ .flat = &.{ a, b, c_ } }, + .results = .{ .flat = &.{res_type} }, + .attributes = &.{ + .named(ctx, "a_type", a_type), + .named(ctx, "b_type", b_type), + .named(ctx, "b_hi", b_hi), + }, + .location = location, + }); +} + +/// `nvvm.dot.accumulate.4way`. Result types are explicit; attributes use mlir.Attribute. +pub fn dot_accumulate_4way(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, c_: *const mlir.Value, res_type: *const mlir.Type, a_type: *const mlir.Attribute, b_type: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.dot.accumulate.4way", .{ + .operands = .{ .flat = &.{ a, b, c_ } }, + .results = .{ .flat = &.{res_type} }, + .attributes = &.{ + .named(ctx, "a_type", a_type), + .named(ctx, "b_type", b_type), + }, + .location = location, + }); +} + +/// `nvvm.elect.sync`. Result types are explicit; attributes use mlir.Attribute. +pub fn elect_sync( + ctx: *mlir.Context, + pred_type: *const mlir.Type, + opts: struct { + membermask: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.elect.sync", .{ + .operands = .{ .flat = if (opts.membermask) |value| &.{value} else &.{} }, + .results = .{ .flat = &.{pred_type} }, + .location = location, + }); +} + +/// `nvvm.ex2`. Result types are explicit; attributes use mlir.Attribute. +pub fn ex2( + ctx: *mlir.Context, + src: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + ftz: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); + return mlir.Operation.make(ctx, "nvvm.ex2", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.exit`. Result types are explicit; attributes use mlir.Attribute. +pub fn exit(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.exit", .{ + .location = location, + }); +} + +/// `nvvm.fabs`. Result types are explicit; attributes use mlir.Attribute. +pub fn fabs( + ctx: *mlir.Context, + value_: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + ftz: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); + return mlir.Operation.make(ctx, "nvvm.fabs", .{ + .operands = .{ .flat = &.{value_} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.fence.mbarrier.init`. Result types are explicit; attributes use mlir.Attribute. +pub fn fence_mbarrier_init(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.fence.mbarrier.init", .{ + .location = location, + }); +} + +/// `nvvm.fence.proxy`. Result types are explicit; attributes use mlir.Attribute. +pub fn fence_proxy( + ctx: *mlir.Context, + kind: *const mlir.Attribute, + opts: struct { + space: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + if (opts.space) |value| attributes.appendAssumeCapacity(.named(ctx, "space", value)); + return mlir.Operation.make(ctx, "nvvm.fence.proxy", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.fence.proxy.acquire`. Result types are explicit; attributes use mlir.Attribute. +pub fn fence_proxy_acquire( + ctx: *mlir.Context, + addr: *const mlir.Value, + size: *const mlir.Value, + scope: *const mlir.Attribute, + opts: struct { + fromProxy: ?*const mlir.Attribute = null, + toProxy: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "scope", scope)); + if (opts.fromProxy) |value| attributes.appendAssumeCapacity(.named(ctx, "fromProxy", value)); + if (opts.toProxy) |value| attributes.appendAssumeCapacity(.named(ctx, "toProxy", value)); + return mlir.Operation.make(ctx, "nvvm.fence.proxy.acquire", .{ + .operands = .{ .flat = &.{ addr, size } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.fence.proxy.release`. Result types are explicit; attributes use mlir.Attribute. +pub fn fence_proxy_release( + ctx: *mlir.Context, + scope: *const mlir.Attribute, + opts: struct { + fromProxy: ?*const mlir.Attribute = null, + toProxy: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "scope", scope)); + if (opts.fromProxy) |value| attributes.appendAssumeCapacity(.named(ctx, "fromProxy", value)); + if (opts.toProxy) |value| attributes.appendAssumeCapacity(.named(ctx, "toProxy", value)); + return mlir.Operation.make(ctx, "nvvm.fence.proxy.release", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.fence.proxy.sync_restrict`. Result types are explicit; attributes use mlir.Attribute. +pub fn fence_proxy_sync_restrict( + ctx: *mlir.Context, + order: *const mlir.Attribute, + opts: struct { + fromProxy: ?*const mlir.Attribute = null, + toProxy: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "order", order)); + if (opts.fromProxy) |value| attributes.appendAssumeCapacity(.named(ctx, "fromProxy", value)); + if (opts.toProxy) |value| attributes.appendAssumeCapacity(.named(ctx, "toProxy", value)); + return mlir.Operation.make(ctx, "nvvm.fence.proxy.sync_restrict", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.fence.sc.cluster`. Result types are explicit; attributes use mlir.Attribute. +pub fn fence_sc_cluster(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.fence.sc.cluster", .{ + .location = location, + }); +} + +/// `nvvm.fence.sync_restrict`. Result types are explicit; attributes use mlir.Attribute. +pub fn fence_sync_restrict(ctx: *mlir.Context, order: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.fence.sync_restrict", .{ + .attributes = &.{ + .named(ctx, "order", order), + }, + .location = location, + }); +} + +/// `nvvm.fma`. Result types are explicit; attributes use mlir.Attribute. +pub fn fma( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c_: *const mlir.Value, + res_type: *const mlir.Type, + rnd: *const mlir.Attribute, + opts: struct { + sat: ?*const mlir.Attribute = null, + ftz: ?*const mlir.Attribute = null, + relu: ?*const mlir.Attribute = null, + oob: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "rnd", rnd)); + if (opts.sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); + if (opts.ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); + if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); + if (opts.oob) |value| attributes.appendAssumeCapacity(.named(ctx, "oob", value)); + return mlir.Operation.make(ctx, "nvvm.fma", .{ + .operands = .{ .flat = &.{ a, b, c_ } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.fma.packed.f32x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn fma_packed_f32x2( + ctx: *mlir.Context, + srcA: *const mlir.Value, + srcB: *const mlir.Value, + srcC: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + rnd: ?*const mlir.Attribute = null, + ftz: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); + return mlir.Operation.make(ctx, "nvvm.fma.packed.f32x2", .{ + .operands = .{ .flat = &.{ srcA, srcB, srcC } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.fma.packed.f32x2.bf16x2.f32x2.f32x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn fma_packed_f32x2_bf16x2_f32x2_f32x2( + ctx: *mlir.Context, + srcA: *const mlir.Value, + srcB: *const mlir.Value, + srcC: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + rnd: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + return mlir.Operation.make(ctx, "nvvm.fma.packed.f32x2.bf16x2.f32x2.f32x2", .{ + .operands = .{ .flat = &.{ srcA, srcB, srcC } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.fma.packed.f32x2.f16x2.f32x2.f32x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn fma_packed_f32x2_f16x2_f32x2_f32x2( + ctx: *mlir.Context, + srcA: *const mlir.Value, + srcB: *const mlir.Value, + srcC: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + rnd: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + return mlir.Operation.make(ctx, "nvvm.fma.packed.f32x2.f16x2.f32x2.f32x2", .{ + .operands = .{ .flat = &.{ srcA, srcB, srcC } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.fmax`. Result types are explicit; attributes use mlir.Attribute. +pub fn fmax( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + c_: ?*const mlir.Value = null, + ftz: ?*const mlir.Attribute = null, + nan: ?*const mlir.Attribute = null, + abs: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); + if (opts.nan) |value| attributes.appendAssumeCapacity(.named(ctx, "nan", value)); + if (opts.abs) |value| attributes.appendAssumeCapacity(.named(ctx, "abs", value)); + return mlir.Operation.make(ctx, "nvvm.fmax", .{ + .operands = .{ .flat = if (opts.c_) |value| &.{ a, b, value } else &.{ a, b } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.fmin`. Result types are explicit; attributes use mlir.Attribute. +pub fn fmin( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + c_: ?*const mlir.Value = null, + ftz: ?*const mlir.Attribute = null, + nan: ?*const mlir.Attribute = null, + abs: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); + if (opts.nan) |value| attributes.appendAssumeCapacity(.named(ctx, "nan", value)); + if (opts.abs) |value| attributes.appendAssumeCapacity(.named(ctx, "abs", value)); + return mlir.Operation.make(ctx, "nvvm.fmin", .{ + .operands = .{ .flat = if (opts.c_) |value| &.{ a, b, value } else &.{ a, b } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.griddepcontrol`. Result types are explicit; attributes use mlir.Attribute. +pub fn griddepcontrol(ctx: *mlir.Context, kind: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.griddepcontrol", .{ + .attributes = &.{ + .named(ctx, "kind", kind), + }, + .location = location, + }); +} + +/// `nvvm.inline_ptx`. Result types are explicit; attributes use mlir.Attribute. +pub fn inline_ptx( + ctx: *mlir.Context, + readOnlyArgs: []const *const mlir.Value, + readWriteArgs: []const *const mlir.Value, + ptxCode: *const mlir.Attribute, + opts: struct { + predicate: ?*const mlir.Value = null, + writeOnlyArgs_type: ?*const mlir.Type = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var results: stdx.BoundedArray(*const mlir.Type, 256) = .empty; + if (opts.writeOnlyArgs_type) |value| results.appendAssumeCapacity(value); + return mlir.Operation.make(ctx, "nvvm.inline_ptx", .{ + .operands = .{ .variadic = &.{ + readOnlyArgs, + readWriteArgs, + if (opts.predicate) |value| &.{value} else &.{}, + } }, + .results = .{ .flat = results.constSlice() }, + .attributes = &.{ + .named(ctx, "ptxCode", ptxCode), + }, + .location = location, + }); +} + +/// `nvvm.ldmatrix`. Result types are explicit; attributes use mlir.Attribute. +pub fn ldmatrix( + ctx: *mlir.Context, + ptr: *const mlir.Value, + res_type: *const mlir.Type, + num: *const mlir.Attribute, + layout: *const mlir.Attribute, + opts: struct { + shape: ?*const mlir.Attribute = null, + srcFormat: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "num", num)); + attributes.appendAssumeCapacity(.named(ctx, "layout", layout)); + if (opts.shape) |value| attributes.appendAssumeCapacity(.named(ctx, "shape", value)); + if (opts.srcFormat) |value| attributes.appendAssumeCapacity(.named(ctx, "srcFormat", value)); + return mlir.Operation.make(ctx, "nvvm.ldmatrix", .{ + .operands = .{ .flat = &.{ptr} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.load.ext`. Result types are explicit; attributes use mlir.Attribute. +pub fn load_ext( + ctx: *mlir.Context, + addr: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + l2CacheHint: ?*const mlir.Value = null, + order: ?*const mlir.Attribute = null, + scope: ?*const mlir.Attribute = null, + prefetch_: ?*const mlir.Attribute = null, + evict: ?*const mlir.Attribute = null, + cacheModifier: ?*const mlir.Attribute = null, + sharedSpace: ?*const mlir.Attribute = null, + unified: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 7) = .empty; + if (opts.order) |value| attributes.appendAssumeCapacity(.named(ctx, "order", value)); + if (opts.scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); + if (opts.prefetch_) |value| attributes.appendAssumeCapacity(.named(ctx, "prefetch", value)); + if (opts.evict) |value| attributes.appendAssumeCapacity(.named(ctx, "evict", value)); + if (opts.cacheModifier) |value| attributes.appendAssumeCapacity(.named(ctx, "cacheModifier", value)); + if (opts.sharedSpace) |value| attributes.appendAssumeCapacity(.named(ctx, "sharedSpace", value)); + if (opts.unified) |value| attributes.appendAssumeCapacity(.named(ctx, "unified", value)); + return mlir.Operation.make(ctx, "nvvm.load.ext", .{ + .operands = .{ .flat = if (opts.l2CacheHint) |value| &.{ addr, value } else &.{addr} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.log2`. Result types are explicit; attributes use mlir.Attribute. +pub fn log2( + ctx: *mlir.Context, + src: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + ftz: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); + return mlir.Operation.make(ctx, "nvvm.log2", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mapa`. Result types are explicit; attributes use mlir.Attribute. +pub fn mapa(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.mapa", .{ + .operands = .{ .flat = &.{ a, b } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.match.sync`. Result types are explicit; attributes use mlir.Attribute. +pub fn match_sync(ctx: *mlir.Context, thread_mask: *const mlir.Value, val: *const mlir.Value, res_type: *const mlir.Type, kind: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.match.sync", .{ + .operands = .{ .flat = &.{ thread_mask, val } }, + .results = .{ .flat = &.{res_type} }, + .attributes = &.{ + .named(ctx, "kind", kind), + }, + .location = location, + }); +} + +/// `nvvm.mbarrier.arrive`. Result types are explicit; attributes use mlir.Attribute. +pub fn mbarrier_arrive( + ctx: *mlir.Context, + addr: *const mlir.Value, + opts: struct { + count: ?*const mlir.Value = null, + multicastMask: ?*const mlir.Value = null, + res_type: ?*const mlir.Type = null, + scope: ?*const mlir.Attribute = null, + relaxed: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var results: stdx.BoundedArray(*const mlir.Type, 256) = .empty; + if (opts.res_type) |value| results.appendAssumeCapacity(value); + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); + if (opts.relaxed) |value| attributes.appendAssumeCapacity(.named(ctx, "relaxed", value)); + return mlir.Operation.make(ctx, "nvvm.mbarrier.arrive", .{ + .operands = .{ .variadic = &.{ + &.{addr}, + if (opts.count) |value| &.{value} else &.{}, + if (opts.multicastMask) |value| &.{value} else &.{}, + } }, + .results = .{ .flat = results.constSlice() }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mbarrier.arrive.expect_tx`. Result types are explicit; attributes use mlir.Attribute. +pub fn mbarrier_arrive_expect_tx( + ctx: *mlir.Context, + addr: *const mlir.Value, + txcount: *const mlir.Value, + opts: struct { + multicastMask: ?*const mlir.Value = null, + predicate: ?*const mlir.Value = null, + res_type: ?*const mlir.Type = null, + scope: ?*const mlir.Attribute = null, + relaxed: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var results: stdx.BoundedArray(*const mlir.Type, 256) = .empty; + if (opts.res_type) |value| results.appendAssumeCapacity(value); + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); + if (opts.relaxed) |value| attributes.appendAssumeCapacity(.named(ctx, "relaxed", value)); + return mlir.Operation.make(ctx, "nvvm.mbarrier.arrive.expect_tx", .{ + .operands = .{ .variadic = &.{ + &.{addr}, + &.{txcount}, + if (opts.multicastMask) |value| &.{value} else &.{}, + if (opts.predicate) |value| &.{value} else &.{}, + } }, + .results = .{ .flat = results.constSlice() }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mbarrier.arrive.nocomplete`. Result types are explicit; attributes use mlir.Attribute. +pub fn mbarrier_arrive_nocomplete(ctx: *mlir.Context, addr: *const mlir.Value, count: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.mbarrier.arrive.nocomplete", .{ + .operands = .{ .flat = &.{ addr, count } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.mbarrier.arrive_drop`. Result types are explicit; attributes use mlir.Attribute. +pub fn mbarrier_arrive_drop( + ctx: *mlir.Context, + addr: *const mlir.Value, + opts: struct { + count: ?*const mlir.Value = null, + multicastMask: ?*const mlir.Value = null, + res_type: ?*const mlir.Type = null, + scope: ?*const mlir.Attribute = null, + relaxed: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var results: stdx.BoundedArray(*const mlir.Type, 256) = .empty; + if (opts.res_type) |value| results.appendAssumeCapacity(value); + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); + if (opts.relaxed) |value| attributes.appendAssumeCapacity(.named(ctx, "relaxed", value)); + return mlir.Operation.make(ctx, "nvvm.mbarrier.arrive_drop", .{ + .operands = .{ .variadic = &.{ + &.{addr}, + if (opts.count) |value| &.{value} else &.{}, + if (opts.multicastMask) |value| &.{value} else &.{}, + } }, + .results = .{ .flat = results.constSlice() }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mbarrier.arrive_drop.expect_tx`. Result types are explicit; attributes use mlir.Attribute. +pub fn mbarrier_arrive_drop_expect_tx( + ctx: *mlir.Context, + addr: *const mlir.Value, + txcount: *const mlir.Value, + opts: struct { + multicastMask: ?*const mlir.Value = null, + res_type: ?*const mlir.Type = null, + scope: ?*const mlir.Attribute = null, + relaxed: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var results: stdx.BoundedArray(*const mlir.Type, 256) = .empty; + if (opts.res_type) |value| results.appendAssumeCapacity(value); + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); + if (opts.relaxed) |value| attributes.appendAssumeCapacity(.named(ctx, "relaxed", value)); + return mlir.Operation.make(ctx, "nvvm.mbarrier.arrive_drop.expect_tx", .{ + .operands = .{ .flat = if (opts.multicastMask) |value| &.{ addr, txcount, value } else &.{ addr, txcount } }, + .results = .{ .flat = results.constSlice() }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mbarrier.arrive_drop.nocomplete`. Result types are explicit; attributes use mlir.Attribute. +pub fn mbarrier_arrive_drop_nocomplete(ctx: *mlir.Context, addr: *const mlir.Value, count: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.mbarrier.arrive_drop.nocomplete", .{ + .operands = .{ .flat = &.{ addr, count } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.mbarrier.check.layout`. Result types are explicit; attributes use mlir.Attribute. +pub fn mbarrier_check_layout(ctx: *mlir.Context, addr: *const mlir.Value, res_type: *const mlir.Type, layout: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.mbarrier.check.layout", .{ + .operands = .{ .flat = &.{addr} }, + .results = .{ .flat = &.{res_type} }, + .attributes = &.{ + .named(ctx, "layout", layout), + }, + .location = location, + }); +} + +/// `nvvm.mbarrier.complete_tx`. Result types are explicit; attributes use mlir.Attribute. +pub fn mbarrier_complete_tx( + ctx: *mlir.Context, + addr: *const mlir.Value, + txcount: *const mlir.Value, + opts: struct { + report: ?*const mlir.Value = null, + multicastMask: ?*const mlir.Value = null, + scope: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); + return mlir.Operation.make(ctx, "nvvm.mbarrier.complete_tx", .{ + .operands = .{ .variadic = &.{ + &.{addr}, + &.{txcount}, + if (opts.report) |value| &.{value} else &.{}, + if (opts.multicastMask) |value| &.{value} else &.{}, + } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mbarrier.expect_tx`. Result types are explicit; attributes use mlir.Attribute. +pub fn mbarrier_expect_tx( + ctx: *mlir.Context, + addr: *const mlir.Value, + txcount: *const mlir.Value, + opts: struct { + multicastMask: ?*const mlir.Value = null, + scope: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); + return mlir.Operation.make(ctx, "nvvm.mbarrier.expect_tx", .{ + .operands = .{ .flat = if (opts.multicastMask) |value| &.{ addr, txcount, value } else &.{ addr, txcount } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mbarrier.init`. Result types are explicit; attributes use mlir.Attribute. +pub fn mbarrier_init( + ctx: *mlir.Context, + addr: *const mlir.Value, + count: *const mlir.Value, + opts: struct { + predicate: ?*const mlir.Value = null, + layout: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.layout) |value| attributes.appendAssumeCapacity(.named(ctx, "layout", value)); + return mlir.Operation.make(ctx, "nvvm.mbarrier.init", .{ + .operands = .{ .flat = if (opts.predicate) |value| &.{ addr, count, value } else &.{ addr, count } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mbarrier.inval`. Result types are explicit; attributes use mlir.Attribute. +pub fn mbarrier_inval(ctx: *mlir.Context, addr: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.mbarrier.inval", .{ + .operands = .{ .flat = &.{addr} }, + .location = location, + }); +} + +/// `nvvm.mbarrier.test.wait`. Result types are explicit; attributes use mlir.Attribute. +pub fn mbarrier_test_wait( + ctx: *mlir.Context, + addr: *const mlir.Value, + stateOrPhase: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + scope: ?*const mlir.Attribute = null, + relaxed: ?*const mlir.Attribute = null, + mbarrierPhase: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); + if (opts.relaxed) |value| attributes.appendAssumeCapacity(.named(ctx, "relaxed", value)); + if (opts.mbarrierPhase) |value| attributes.appendAssumeCapacity(.named(ctx, "mbarrierPhase", value)); + return mlir.Operation.make(ctx, "nvvm.mbarrier.test.wait", .{ + .operands = .{ .flat = &.{ addr, stateOrPhase } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mbarrier.try_wait`. Result types are explicit; attributes use mlir.Attribute. +pub fn mbarrier_try_wait( + ctx: *mlir.Context, + addr: *const mlir.Value, + stateOrPhase: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + ticks: ?*const mlir.Value = null, + scope: ?*const mlir.Attribute = null, + relaxed: ?*const mlir.Attribute = null, + mbarrierPhase: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); + if (opts.relaxed) |value| attributes.appendAssumeCapacity(.named(ctx, "relaxed", value)); + if (opts.mbarrierPhase) |value| attributes.appendAssumeCapacity(.named(ctx, "mbarrierPhase", value)); + return mlir.Operation.make(ctx, "nvvm.mbarrier.try_wait", .{ + .operands = .{ .flat = if (opts.ticks) |value| &.{ addr, stateOrPhase, value } else &.{ addr, stateOrPhase } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mbarrier.try_wait.parity`. Result types are explicit; attributes use mlir.Attribute. +pub fn mbarrier_try_wait_parity( + ctx: *mlir.Context, + addr: *const mlir.Value, + phase: *const mlir.Value, + ticks: *const mlir.Value, + opts: struct { + useIntrinsic: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.useIntrinsic) |value| attributes.appendAssumeCapacity(.named(ctx, "useIntrinsic", value)); + return mlir.Operation.make(ctx, "nvvm.mbarrier.try_wait.parity", .{ + .operands = .{ .flat = &.{ addr, phase, ticks } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mbarrier.try_wait.parity.timelimit`. Result types are explicit; attributes use mlir.Attribute. +pub fn mbarrier_try_wait_parity_timelimit( + ctx: *mlir.Context, + addr: *const mlir.Value, + phase: *const mlir.Value, + timeLimit: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + scope: ?*const mlir.Attribute = null, + order: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); + if (opts.order) |value| attributes.appendAssumeCapacity(.named(ctx, "order", value)); + return mlir.Operation.make(ctx, "nvvm.mbarrier.try_wait.parity.timelimit", .{ + .operands = .{ .flat = &.{ addr, phase, timeLimit } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mbarrier.try_wait.timelimit`. Result types are explicit; attributes use mlir.Attribute. +pub fn mbarrier_try_wait_timelimit( + ctx: *mlir.Context, + addr: *const mlir.Value, + state: *const mlir.Value, + timeLimit: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + scope: ?*const mlir.Attribute = null, + order: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); + if (opts.order) |value| attributes.appendAssumeCapacity(.named(ctx, "order", value)); + return mlir.Operation.make(ctx, "nvvm.mbarrier.try_wait.timelimit", .{ + .operands = .{ .flat = &.{ addr, state, timeLimit } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mbarrier.txn`. Result types are explicit; attributes use mlir.Attribute. +pub fn mbarrier_txn( + ctx: *mlir.Context, + addr: *const mlir.Value, + count: *const mlir.Value, + kind: *const mlir.Attribute, + opts: struct { + multicast: ?*const mlir.Value = null, + space: ?*const mlir.Attribute = null, + scope: ?*const mlir.Attribute = null, + order: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + if (opts.space) |value| attributes.appendAssumeCapacity(.named(ctx, "space", value)); + if (opts.scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); + if (opts.order) |value| attributes.appendAssumeCapacity(.named(ctx, "order", value)); + return mlir.Operation.make(ctx, "nvvm.mbarrier.txn", .{ + .operands = .{ .flat = if (opts.multicast) |value| &.{ addr, count, value } else &.{ addr, count } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mbarrier.txn.cta`. Result types are explicit; attributes use mlir.Attribute. +pub fn mbarrier_txn_cta( + ctx: *mlir.Context, + addr: *const mlir.Value, + count: *const mlir.Value, + res_type: *const mlir.Type, + kind: *const mlir.Attribute, + opts: struct { + scope: ?*const mlir.Attribute = null, + order: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + if (opts.scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); + if (opts.order) |value| attributes.appendAssumeCapacity(.named(ctx, "order", value)); + return mlir.Operation.make(ctx, "nvvm.mbarrier.txn.cta", .{ + .operands = .{ .flat = &.{ addr, count } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mbarrier.wait`. Result types are explicit; attributes use mlir.Attribute. +pub fn mbarrier_wait( + ctx: *mlir.Context, + addr: *const mlir.Value, + state: *const mlir.Value, + res_type: *const mlir.Type, + kind: *const mlir.Attribute, + opts: struct { + scope: ?*const mlir.Attribute = null, + order: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + if (opts.scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); + if (opts.order) |value| attributes.appendAssumeCapacity(.named(ctx, "order", value)); + return mlir.Operation.make(ctx, "nvvm.mbarrier.wait", .{ + .operands = .{ .flat = &.{ addr, state } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mbarrier.wait.parity`. Result types are explicit; attributes use mlir.Attribute. +pub fn mbarrier_wait_parity( + ctx: *mlir.Context, + addr: *const mlir.Value, + phase: *const mlir.Value, + res_type: *const mlir.Type, + kind: *const mlir.Attribute, + opts: struct { + scope: ?*const mlir.Attribute = null, + order: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + if (opts.scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); + if (opts.order) |value| attributes.appendAssumeCapacity(.named(ctx, "order", value)); + return mlir.Operation.make(ctx, "nvvm.mbarrier.wait.parity", .{ + .operands = .{ .flat = &.{ addr, phase } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.memory.barrier`. Result types are explicit; attributes use mlir.Attribute. +pub fn memory_barrier(ctx: *mlir.Context, scope: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.memory.barrier", .{ + .attributes = &.{ + .named(ctx, "scope", scope), + }, + .location = location, + }); +} + +/// `nvvm.mma.block_scale`. Result types are explicit; attributes use mlir.Attribute. +pub fn mma_block_scale( + ctx: *mlir.Context, + operandA: []const *const mlir.Value, + operandB: []const *const mlir.Value, + operandC: []const *const mlir.Value, + scaleAData: *const mlir.Value, + byteIdA: *const mlir.Value, + threadIdA: *const mlir.Value, + scaleBData: *const mlir.Value, + byteIdB: *const mlir.Value, + threadIdB: *const mlir.Value, + res_type: *const mlir.Type, + shape: *const mlir.Attribute, + scaleVecSize: *const mlir.Attribute, + blockScaleFormat: *const mlir.Attribute, + kind: *const mlir.Attribute, + opts: struct { + multiplicandAPtxType: ?*const mlir.Attribute = null, + multiplicandBPtxType: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 6) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); + if (opts.multiplicandAPtxType) |value| attributes.appendAssumeCapacity(.named(ctx, "multiplicandAPtxType", value)); + if (opts.multiplicandBPtxType) |value| attributes.appendAssumeCapacity(.named(ctx, "multiplicandBPtxType", value)); + attributes.appendAssumeCapacity(.named(ctx, "scaleVecSize", scaleVecSize)); + attributes.appendAssumeCapacity(.named(ctx, "blockScaleFormat", blockScaleFormat)); + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + return mlir.Operation.make(ctx, "nvvm.mma.block_scale", .{ + .operands = .{ .variadic = &.{ + operandA, + operandB, + operandC, + &.{scaleAData}, + &.{byteIdA}, + &.{threadIdA}, + &.{scaleBData}, + &.{byteIdB}, + &.{threadIdB}, + } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mma.block_scale.internal`. Result types are explicit; attributes use mlir.Attribute. +pub fn mma_block_scale_internal( + ctx: *mlir.Context, + operandA: *const mlir.Value, + scaleAData: *const mlir.Value, + byteIdA: *const mlir.Value, + threadIdA: *const mlir.Value, + operandB: *const mlir.Value, + scaleBData: *const mlir.Value, + byteIdB: *const mlir.Value, + threadIdB: *const mlir.Value, + operandC: *const mlir.Value, + res_type: *const mlir.Type, + shape: *const mlir.Attribute, + layoutA: *const mlir.Attribute, + layoutB: *const mlir.Attribute, + aType: *const mlir.Attribute, + bType: *const mlir.Attribute, + cType: *const mlir.Attribute, + scaleVecSize: *const mlir.Attribute, + blockScaleFormat: *const mlir.Attribute, + opts: struct { + kind: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 9) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); + attributes.appendAssumeCapacity(.named(ctx, "layoutA", layoutA)); + attributes.appendAssumeCapacity(.named(ctx, "layoutB", layoutB)); + attributes.appendAssumeCapacity(.named(ctx, "aType", aType)); + attributes.appendAssumeCapacity(.named(ctx, "bType", bType)); + attributes.appendAssumeCapacity(.named(ctx, "cType", cType)); + attributes.appendAssumeCapacity(.named(ctx, "scaleVecSize", scaleVecSize)); + attributes.appendAssumeCapacity(.named(ctx, "blockScaleFormat", blockScaleFormat)); + if (opts.kind) |value| attributes.appendAssumeCapacity(.named(ctx, "kind", value)); + return mlir.Operation.make(ctx, "nvvm.mma.block_scale.internal", .{ + .operands = .{ .flat = &.{ operandA, scaleAData, byteIdA, threadIdA, operandB, scaleBData, byteIdB, threadIdB, operandC } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mma.sp.block_scale`. Result types are explicit; attributes use mlir.Attribute. +pub fn mma_sp_block_scale( + ctx: *mlir.Context, + operandA: []const *const mlir.Value, + operandB: []const *const mlir.Value, + operandC: []const *const mlir.Value, + sparseMetadata: *const mlir.Value, + sparsitySelector: *const mlir.Value, + scaleAData: *const mlir.Value, + byteIdA: *const mlir.Value, + threadIdA: *const mlir.Value, + scaleBData: *const mlir.Value, + byteIdB: *const mlir.Value, + threadIdB: *const mlir.Value, + res_type: *const mlir.Type, + shape: *const mlir.Attribute, + scaleVecSize: *const mlir.Attribute, + blockScaleFormat: *const mlir.Attribute, + kind: *const mlir.Attribute, + opts: struct { + multiplicandAPtxType: ?*const mlir.Attribute = null, + multiplicandBPtxType: ?*const mlir.Attribute = null, + orderedMetadata: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 7) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); + if (opts.multiplicandAPtxType) |value| attributes.appendAssumeCapacity(.named(ctx, "multiplicandAPtxType", value)); + if (opts.multiplicandBPtxType) |value| attributes.appendAssumeCapacity(.named(ctx, "multiplicandBPtxType", value)); + attributes.appendAssumeCapacity(.named(ctx, "scaleVecSize", scaleVecSize)); + attributes.appendAssumeCapacity(.named(ctx, "blockScaleFormat", blockScaleFormat)); + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + if (opts.orderedMetadata) |value| attributes.appendAssumeCapacity(.named(ctx, "orderedMetadata", value)); + return mlir.Operation.make(ctx, "nvvm.mma.sp.block_scale", .{ + .operands = .{ .variadic = &.{ + operandA, + operandB, + operandC, + &.{sparseMetadata}, + &.{sparsitySelector}, + &.{scaleAData}, + &.{byteIdA}, + &.{threadIdA}, + &.{scaleBData}, + &.{byteIdB}, + &.{threadIdB}, + } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mma.sp.block_scale.internal`. Result types are explicit; attributes use mlir.Attribute. +pub fn mma_sp_block_scale_internal( + ctx: *mlir.Context, + operandA: *const mlir.Value, + scaleAData: *const mlir.Value, + byteIdA: *const mlir.Value, + threadIdA: *const mlir.Value, + operandB: *const mlir.Value, + scaleBData: *const mlir.Value, + byteIdB: *const mlir.Value, + threadIdB: *const mlir.Value, + operandC: *const mlir.Value, + sp_metadata: *const mlir.Value, + selector: *const mlir.Value, + res_type: *const mlir.Type, + shape: *const mlir.Attribute, + layoutA: *const mlir.Attribute, + layoutB: *const mlir.Attribute, + aType: *const mlir.Attribute, + bType: *const mlir.Attribute, + cType: *const mlir.Attribute, + scaleVecSize: *const mlir.Attribute, + blockScaleFormat: *const mlir.Attribute, + sparsityFormat: *const mlir.Attribute, + opts: struct { + b1Op: ?*const mlir.Attribute = null, + intOverflowBehavior: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 11) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); + if (opts.b1Op) |value| attributes.appendAssumeCapacity(.named(ctx, "b1Op", value)); + if (opts.intOverflowBehavior) |value| attributes.appendAssumeCapacity(.named(ctx, "intOverflowBehavior", value)); + attributes.appendAssumeCapacity(.named(ctx, "layoutA", layoutA)); + attributes.appendAssumeCapacity(.named(ctx, "layoutB", layoutB)); + attributes.appendAssumeCapacity(.named(ctx, "aType", aType)); + attributes.appendAssumeCapacity(.named(ctx, "bType", bType)); + attributes.appendAssumeCapacity(.named(ctx, "cType", cType)); + attributes.appendAssumeCapacity(.named(ctx, "scaleVecSize", scaleVecSize)); + attributes.appendAssumeCapacity(.named(ctx, "blockScaleFormat", blockScaleFormat)); + attributes.appendAssumeCapacity(.named(ctx, "sparsityFormat", sparsityFormat)); + return mlir.Operation.make(ctx, "nvvm.mma.sp.block_scale.internal", .{ + .operands = .{ .flat = &.{ operandA, scaleAData, byteIdA, threadIdA, operandB, scaleBData, byteIdB, threadIdB, operandC, sp_metadata, selector } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mma.sp.sync`. Result types are explicit; attributes use mlir.Attribute. +pub fn mma_sp_sync( + ctx: *mlir.Context, + operandA: []const *const mlir.Value, + operandB: []const *const mlir.Value, + operandC: []const *const mlir.Value, + sparseMetadata: *const mlir.Value, + sparsitySelector: *const mlir.Value, + res_type: *const mlir.Type, + shape: *const mlir.Attribute, + opts: struct { + intOverflowBehavior: ?*const mlir.Attribute = null, + multiplicandAPtxType: ?*const mlir.Attribute = null, + multiplicandBPtxType: ?*const mlir.Attribute = null, + orderedMetadata: ?*const mlir.Attribute = null, + kind: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 6) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); + if (opts.intOverflowBehavior) |value| attributes.appendAssumeCapacity(.named(ctx, "intOverflowBehavior", value)); + if (opts.multiplicandAPtxType) |value| attributes.appendAssumeCapacity(.named(ctx, "multiplicandAPtxType", value)); + if (opts.multiplicandBPtxType) |value| attributes.appendAssumeCapacity(.named(ctx, "multiplicandBPtxType", value)); + if (opts.orderedMetadata) |value| attributes.appendAssumeCapacity(.named(ctx, "orderedMetadata", value)); + if (opts.kind) |value| attributes.appendAssumeCapacity(.named(ctx, "kind", value)); + return mlir.Operation.make(ctx, "nvvm.mma.sp.sync", .{ + .operands = .{ .variadic = &.{ + operandA, + operandB, + operandC, + &.{sparseMetadata}, + &.{sparsitySelector}, + } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mma.sync`. Result types are explicit; attributes use mlir.Attribute. +pub fn mma_sync( + ctx: *mlir.Context, + operandA: []const *const mlir.Value, + operandB: []const *const mlir.Value, + operandC: []const *const mlir.Value, + res_type: *const mlir.Type, + shape: *const mlir.Attribute, + layoutA: *const mlir.Attribute, + layoutB: *const mlir.Attribute, + opts: struct { + b1Op: ?*const mlir.Attribute = null, + intOverflowBehavior: ?*const mlir.Attribute = null, + multiplicandAPtxType: ?*const mlir.Attribute = null, + multiplicandBPtxType: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 7) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); + if (opts.b1Op) |value| attributes.appendAssumeCapacity(.named(ctx, "b1Op", value)); + if (opts.intOverflowBehavior) |value| attributes.appendAssumeCapacity(.named(ctx, "intOverflowBehavior", value)); + attributes.appendAssumeCapacity(.named(ctx, "layoutA", layoutA)); + attributes.appendAssumeCapacity(.named(ctx, "layoutB", layoutB)); + if (opts.multiplicandAPtxType) |value| attributes.appendAssumeCapacity(.named(ctx, "multiplicandAPtxType", value)); + if (opts.multiplicandBPtxType) |value| attributes.appendAssumeCapacity(.named(ctx, "multiplicandBPtxType", value)); + return mlir.Operation.make(ctx, "nvvm.mma.sync", .{ + .operands = .{ .variadic = &.{ + operandA, + operandB, + operandC, + } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mma_smem_desc`. Result types are explicit; attributes use mlir.Attribute. +pub fn mma_smem_desc( + ctx: *mlir.Context, + pointer: *const mlir.Value, + ldm: *const mlir.Value, + stride: *const mlir.Value, + baseOffset: *const mlir.Value, + swizzle: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + mmaDescVersion: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.mmaDescVersion) |value| attributes.appendAssumeCapacity(.named(ctx, "mmaDescVersion", value)); + return mlir.Operation.make(ctx, "nvvm.mma_smem_desc", .{ + .operands = .{ .flat = &.{ pointer, ldm, stride, baseOffset, swizzle } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.movmatrix`. Result types are explicit; attributes use mlir.Attribute. +pub fn movmatrix( + ctx: *mlir.Context, + src: *const mlir.Value, + dst_type: *const mlir.Type, + shape: *const mlir.Attribute, + eltType: *const mlir.Attribute, + opts: struct { + layout: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); + if (opts.layout) |value| attributes.appendAssumeCapacity(.named(ctx, "layout", value)); + attributes.appendAssumeCapacity(.named(ctx, "eltType", eltType)); + return mlir.Operation.make(ctx, "nvvm.movmatrix", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{dst_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mul`. Result types are explicit; attributes use mlir.Attribute. +pub fn mul( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + res_type: *const mlir.Type, + mode: *const mlir.Attribute, + opts: struct { + isSigned: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "mode", mode)); + if (opts.isSigned) |value| attributes.appendAssumeCapacity(.named(ctx, "isSigned", value)); + return mlir.Operation.make(ctx, "nvvm.mul", .{ + .operands = .{ .flat = &.{ a, b } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.mul.packed.bf16x2.bf16x2.f16x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn mul_packed_bf16x2_bf16x2_f16x2(ctx: *mlir.Context, srcA: *const mlir.Value, srcB: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.mul.packed.bf16x2.bf16x2.f16x2", .{ + .operands = .{ .flat = &.{ srcA, srcB } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.mul.packed.bf16x2.f32x2.f32x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn mul_packed_bf16x2_f32x2_f32x2(ctx: *mlir.Context, srcA: *const mlir.Value, srcB: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.mul.packed.bf16x2.f32x2.f32x2", .{ + .operands = .{ .flat = &.{ srcA, srcB } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.mul.packed.f16x2.f16x2.bf16x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn mul_packed_f16x2_f16x2_bf16x2(ctx: *mlir.Context, srcA: *const mlir.Value, srcB: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.mul.packed.f16x2.f16x2.bf16x2", .{ + .operands = .{ .flat = &.{ srcA, srcB } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.mul.packed.f16x2.f32x2.f32x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn mul_packed_f16x2_f32x2_f32x2(ctx: *mlir.Context, srcA: *const mlir.Value, srcB: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.mul.packed.f16x2.f32x2.f32x2", .{ + .operands = .{ .flat = &.{ srcA, srcB } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.mul.packed.f32x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn mul_packed_f32x2( + ctx: *mlir.Context, + srcA: *const mlir.Value, + srcB: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + rnd: ?*const mlir.Attribute = null, + ftz: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); + return mlir.Operation.make(ctx, "nvvm.mul.packed.f32x2", .{ + .operands = .{ .flat = &.{ srcA, srcB } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.multimem.ld.reduce`. Result types are explicit; attributes use mlir.Attribute. +pub fn multimem_ld_reduce( + ctx: *mlir.Context, + addr: *const mlir.Value, + res_type: *const mlir.Type, + ordering: *const mlir.Attribute, + op: *const mlir.Attribute, + dataType: *const mlir.Attribute, + opts: struct { + accPrec: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "ordering", ordering)); + attributes.appendAssumeCapacity(.named(ctx, "op", op)); + attributes.appendAssumeCapacity(.named(ctx, "dataType", dataType)); + if (opts.accPrec) |value| attributes.appendAssumeCapacity(.named(ctx, "accPrec", value)); + return mlir.Operation.make(ctx, "nvvm.multimem.ld.reduce", .{ + .operands = .{ .flat = &.{addr} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.multimem.red`. Result types are explicit; attributes use mlir.Attribute. +pub fn multimem_red(ctx: *mlir.Context, val: *const mlir.Value, addr: *const mlir.Value, ordering: *const mlir.Attribute, scope: *const mlir.Attribute, op: *const mlir.Attribute, dataType: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.multimem.red", .{ + .operands = .{ .flat = &.{ val, addr } }, + .attributes = &.{ + .named(ctx, "ordering", ordering), + .named(ctx, "scope", scope), + .named(ctx, "op", op), + .named(ctx, "dataType", dataType), + }, + .location = location, + }); +} + +/// `nvvm.multimem.st`. Result types are explicit; attributes use mlir.Attribute. +pub fn multimem_st(ctx: *mlir.Context, val: *const mlir.Value, addr: *const mlir.Value, ordering: *const mlir.Attribute, scope: *const mlir.Attribute, dataType: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.multimem.st", .{ + .operands = .{ .flat = &.{ val, addr } }, + .attributes = &.{ + .named(ctx, "ordering", ordering), + .named(ctx, "scope", scope), + .named(ctx, "dataType", dataType), + }, + .location = location, + }); +} + +/// `nvvm.nanosleep`. Result types are explicit; attributes use mlir.Attribute. +pub fn nanosleep(ctx: *mlir.Context, duration: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.nanosleep", .{ + .operands = .{ .flat = &.{duration} }, + .location = location, + }); +} + +/// `nvvm.pmevent`. Result types are explicit; attributes use mlir.Attribute. +pub fn pmevent( + ctx: *mlir.Context, + opts: struct { + maskedEventId: ?*const mlir.Attribute = null, + eventId: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.maskedEventId) |value| attributes.appendAssumeCapacity(.named(ctx, "maskedEventId", value)); + if (opts.eventId) |value| attributes.appendAssumeCapacity(.named(ctx, "eventId", value)); + return mlir.Operation.make(ctx, "nvvm.pmevent", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.prefetch`. Result types are explicit; attributes use mlir.Attribute. +pub fn prefetch( + ctx: *mlir.Context, + addr: *const mlir.Value, + opts: struct { + predicate: ?*const mlir.Value = null, + cacheLevel: ?*const mlir.Attribute = null, + evictPriority: ?*const mlir.Attribute = null, + tensormap: ?*const mlir.Attribute = null, + uniform: ?*const mlir.Attribute = null, + in_param_space: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + if (opts.cacheLevel) |value| attributes.appendAssumeCapacity(.named(ctx, "cacheLevel", value)); + if (opts.evictPriority) |value| attributes.appendAssumeCapacity(.named(ctx, "evictPriority", value)); + if (opts.tensormap) |value| attributes.appendAssumeCapacity(.named(ctx, "tensormap", value)); + if (opts.uniform) |value| attributes.appendAssumeCapacity(.named(ctx, "uniform", value)); + if (opts.in_param_space) |value| attributes.appendAssumeCapacity(.named(ctx, "in_param_space", value)); + return mlir.Operation.make(ctx, "nvvm.prefetch", .{ + .operands = .{ .flat = if (opts.predicate) |value| &.{ addr, value } else &.{addr} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.prmt`. Result types are explicit; attributes use mlir.Attribute. +pub fn prmt( + ctx: *mlir.Context, + lo: *const mlir.Value, + selector: *const mlir.Value, + res_type: *const mlir.Type, + mode: *const mlir.Attribute, + opts: struct { + hi: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.prmt", .{ + .operands = .{ .flat = if (opts.hi) |value| &.{ lo, value, selector } else &.{ lo, selector } }, + .results = .{ .flat = &.{res_type} }, + .attributes = &.{ + .named(ctx, "mode", mode), + }, + .location = location, + }); +} + +/// `nvvm.rcp.approx.ftz.f`. Result types are explicit; attributes use mlir.Attribute. +pub fn rcp_approx_ftz_f(ctx: *mlir.Context, arg: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.rcp.approx.ftz.f", .{ + .operands = .{ .flat = &.{arg} }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.aggr.smem.size`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_aggr_smem_size(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.aggr.smem.size", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.clock`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_clock(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.clock", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.clock64`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_clock64(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.clock64", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.cluster.ctaid.x`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_cluster_ctaid_x( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.cluster.ctaid.x", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.cluster.ctaid.y`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_cluster_ctaid_y( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.cluster.ctaid.y", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.cluster.ctaid.z`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_cluster_ctaid_z( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.cluster.ctaid.z", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.cluster.ctarank`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_cluster_ctarank( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.cluster.ctarank", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.cluster.nctaid.x`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_cluster_nctaid_x( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.cluster.nctaid.x", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.cluster.nctaid.y`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_cluster_nctaid_y( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.cluster.nctaid.y", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.cluster.nctaid.z`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_cluster_nctaid_z( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.cluster.nctaid.z", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.cluster.nctarank`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_cluster_nctarank( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.cluster.nctarank", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.clusterid.x`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_clusterid_x( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.clusterid.x", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.clusterid.y`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_clusterid_y( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.clusterid.y", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.clusterid.z`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_clusterid_z( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.clusterid.z", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.ctaid.x`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_ctaid_x( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.ctaid.x", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.ctaid.y`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_ctaid_y( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.ctaid.y", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.ctaid.z`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_ctaid_z( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.ctaid.z", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.dynamic.smem.size`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_dynamic_smem_size(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.dynamic.smem.size", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg0`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg0(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg0", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg1`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg1(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg1", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg10`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg10(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg10", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg11`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg11(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg11", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg12`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg12(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg12", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg13`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg13(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg13", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg14`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg14(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg14", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg15`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg15(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg15", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg16`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg16(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg16", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg17`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg17(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg17", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg18`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg18(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg18", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg19`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg19(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg19", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg2`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg2(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg2", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg20`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg20(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg20", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg21`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg21(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg21", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg22`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg22(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg22", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg23`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg23(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg23", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg24`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg24(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg24", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg25`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg25(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg25", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg26`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg26(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg26", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg27`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg27(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg27", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg28`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg28(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg28", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg29`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg29(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg29", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg3`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg3(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg3", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg30`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg30(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg30", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg31`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg31(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg31", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg4`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg4(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg4", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg5`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg5(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg5", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg6`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg6(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg6", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg7`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg7(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg7", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg8`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg8(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg8", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.envreg9`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_envreg9(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg9", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.globaltimer`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_globaltimer(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.globaltimer", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.globaltimer.lo`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_globaltimer_lo(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.globaltimer.lo", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.gridid`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_gridid( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.gridid", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.laneid`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_laneid( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.laneid", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.lanemask.eq`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_lanemask_eq(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.lanemask.eq", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.lanemask.ge`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_lanemask_ge(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.lanemask.ge", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.lanemask.gt`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_lanemask_gt(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.lanemask.gt", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.lanemask.le`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_lanemask_le(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.lanemask.le", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.lanemask.lt`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_lanemask_lt(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.lanemask.lt", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.nclusterid.x`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_nclusterid_x( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.nclusterid.x", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.nclusterid.y`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_nclusterid_y( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.nclusterid.y", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.nclusterid.z`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_nclusterid_z( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.nclusterid.z", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.nctaid.x`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_nctaid_x( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.nctaid.x", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.nctaid.y`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_nctaid_y( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.nctaid.y", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.nctaid.z`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_nctaid_z( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.nctaid.z", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.nsmid`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_nsmid( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.nsmid", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.ntid.x`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_ntid_x( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.ntid.x", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.ntid.y`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_ntid_y( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.ntid.y", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.ntid.z`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_ntid_z( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.ntid.z", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.nwarpid`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_nwarpid( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.nwarpid", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.smid`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_smid( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.smid", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.tid.x`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_tid_x( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.tid.x", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.tid.y`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_tid_y( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.tid.y", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.tid.z`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_tid_z( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.tid.z", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.total.smem.size`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_total_smem_size(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.total.smem.size", .{ + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.warpid`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_warpid( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.warpid", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.read.ptx.sreg.warpsize`. Result types are explicit; attributes use mlir.Attribute. +pub fn read_ptx_sreg_warpsize( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); + return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.warpsize", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.red`. Result types are explicit; attributes use mlir.Attribute. +pub fn red( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + op: *const mlir.Attribute, + type_: *const mlir.Attribute, + opts: struct { + cacheHint: ?*const mlir.Value = null, + memOrder: ?*const mlir.Attribute = null, + memScope: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (opts.memOrder) |value| attributes.appendAssumeCapacity(.named(ctx, "memOrder", value)); + if (opts.memScope) |value| attributes.appendAssumeCapacity(.named(ctx, "memScope", value)); + attributes.appendAssumeCapacity(.named(ctx, "op", op)); + attributes.appendAssumeCapacity(.named(ctx, "type", type_)); + return mlir.Operation.make(ctx, "nvvm.red", .{ + .operands = .{ .flat = if (opts.cacheHint) |value| &.{ a, b, value } else &.{ a, b } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.redux.sync`. Result types are explicit; attributes use mlir.Attribute. +pub fn redux_sync( + ctx: *mlir.Context, + val: *const mlir.Value, + mask_and_clamp: *const mlir.Value, + res_type: *const mlir.Type, + kind: *const mlir.Attribute, + opts: struct { + abs: ?*const mlir.Attribute = null, + nan: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + if (opts.abs) |value| attributes.appendAssumeCapacity(.named(ctx, "abs", value)); + if (opts.nan) |value| attributes.appendAssumeCapacity(.named(ctx, "nan", value)); + return mlir.Operation.make(ctx, "nvvm.redux.sync", .{ + .operands = .{ .flat = &.{ val, mask_and_clamp } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.rsqrt`. Result types are explicit; attributes use mlir.Attribute. +pub fn rsqrt( + ctx: *mlir.Context, + src: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + ftz: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); + return mlir.Operation.make(ctx, "nvvm.rsqrt", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.setmaxregister`. Result types are explicit; attributes use mlir.Attribute. +pub fn setmaxregister(ctx: *mlir.Context, regCount: *const mlir.Attribute, action: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.setmaxregister", .{ + .attributes = &.{ + .named(ctx, "regCount", regCount), + .named(ctx, "action", action), + }, + .location = location, + }); +} + +/// `nvvm.shfl.sync`. Result types are explicit; attributes use mlir.Attribute. +pub fn shfl_sync( + ctx: *mlir.Context, + thread_mask: *const mlir.Value, + val: *const mlir.Value, + offset: *const mlir.Value, + mask_and_clamp: *const mlir.Value, + res_type: *const mlir.Type, + kind: *const mlir.Attribute, + opts: struct { + return_value_and_is_valid: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + if (opts.return_value_and_is_valid) |value| attributes.appendAssumeCapacity(.named(ctx, "return_value_and_is_valid", value)); + return mlir.Operation.make(ctx, "nvvm.shfl.sync", .{ + .operands = .{ .flat = &.{ thread_mask, val, offset, mask_and_clamp } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.sin`. Result types are explicit; attributes use mlir.Attribute. +pub fn sin( + ctx: *mlir.Context, + src: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + ftz: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); + return mlir.Operation.make(ctx, "nvvm.sin", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.spcompress`. Result types are explicit; attributes use mlir.Attribute. +pub fn spcompress(ctx: *mlir.Context, spDesc: *const mlir.Value, data: *const mlir.Value, metadata_type: *const mlir.Type, compressed_data_type: *const mlir.Type, compFactor: *const mlir.Attribute, indexSize: *const mlir.Attribute, elemSize: *const mlir.Attribute, repFactor: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.spcompress", .{ + .operands = .{ .flat = &.{ spDesc, data } }, + .results = .{ .flat = &.{ metadata_type, compressed_data_type } }, + .attributes = &.{ + .named(ctx, "compFactor", compFactor), + .named(ctx, "indexSize", indexSize), + .named(ctx, "elemSize", elemSize), + .named(ctx, "repFactor", repFactor), + }, + .location = location, + }); +} + +/// `nvvm.spdecompress`. Result types are explicit; attributes use mlir.Attribute. +pub fn spdecompress(ctx: *mlir.Context, metadata: *const mlir.Value, compressed_data: *const mlir.Value, data_type: *const mlir.Type, compFactor: *const mlir.Attribute, indexSize: *const mlir.Attribute, elemSize: *const mlir.Attribute, repFactor: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.spdecompress", .{ + .operands = .{ .flat = &.{ metadata, compressed_data } }, + .results = .{ .flat = &.{data_type} }, + .attributes = &.{ + .named(ctx, "compFactor", compFactor), + .named(ctx, "indexSize", indexSize), + .named(ctx, "elemSize", elemSize), + .named(ctx, "repFactor", repFactor), + }, + .location = location, + }); +} + +/// `nvvm.st.bulk`. Result types are explicit; attributes use mlir.Attribute. +pub fn st_bulk( + ctx: *mlir.Context, + addr: *const mlir.Value, + size: *const mlir.Value, + opts: struct { + initVal: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.initVal) |value| attributes.appendAssumeCapacity(.named(ctx, "initVal", value)); + return mlir.Operation.make(ctx, "nvvm.st.bulk", .{ + .operands = .{ .flat = &.{ addr, size } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.stmatrix`. Result types are explicit; attributes use mlir.Attribute. +pub fn stmatrix( + ctx: *mlir.Context, + ptr: *const mlir.Value, + sources: []const *const mlir.Value, + layout: *const mlir.Attribute, + opts: struct { + shape: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var operands: stdx.BoundedArray(*const mlir.Value, 256) = .empty; + operands.appendAssumeCapacity(ptr); + operands.appendSliceAssumeCapacity(sources); + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "layout", layout)); + if (opts.shape) |value| attributes.appendAssumeCapacity(.named(ctx, "shape", value)); + return mlir.Operation.make(ctx, "nvvm.stmatrix", .{ + .operands = .{ .flat = operands.constSlice() }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.store.async`. Result types are explicit; attributes use mlir.Attribute. +pub fn store_async(ctx: *mlir.Context, addr: *const mlir.Value, value_: *const mlir.Value, mbarrier: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.store.async", .{ + .operands = .{ .flat = &.{ addr, value_, mbarrier } }, + .location = location, + }); +} + +/// `nvvm.store.ext`. Result types are explicit; attributes use mlir.Attribute. +pub fn store_ext( + ctx: *mlir.Context, + value_: *const mlir.Value, + addr: *const mlir.Value, + opts: struct { + l2CacheHint: ?*const mlir.Value = null, + order: ?*const mlir.Attribute = null, + scope: ?*const mlir.Attribute = null, + evict: ?*const mlir.Attribute = null, + cacheModifier: ?*const mlir.Attribute = null, + sharedSpace: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + if (opts.order) |value| attributes.appendAssumeCapacity(.named(ctx, "order", value)); + if (opts.scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); + if (opts.evict) |value| attributes.appendAssumeCapacity(.named(ctx, "evict", value)); + if (opts.cacheModifier) |value| attributes.appendAssumeCapacity(.named(ctx, "cacheModifier", value)); + if (opts.sharedSpace) |value| attributes.appendAssumeCapacity(.named(ctx, "sharedSpace", value)); + return mlir.Operation.make(ctx, "nvvm.store.ext", .{ + .operands = .{ .flat = if (opts.l2CacheHint) |value| &.{ value_, addr, value } else &.{ value_, addr } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.sub.packed.bf16x2.f32x2.f32x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn sub_packed_bf16x2_f32x2_f32x2(ctx: *mlir.Context, srcA: *const mlir.Value, srcB: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.sub.packed.bf16x2.f32x2.f32x2", .{ + .operands = .{ .flat = &.{ srcA, srcB } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.sub.packed.f16x2.f32x2.f32x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn sub_packed_f16x2_f32x2_f32x2(ctx: *mlir.Context, srcA: *const mlir.Value, srcB: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.sub.packed.f16x2.f32x2.f32x2", .{ + .operands = .{ .flat = &.{ srcA, srcB } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.sub.packed.f32x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn sub_packed_f32x2( + ctx: *mlir.Context, + srcA: *const mlir.Value, + srcB: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + rnd: ?*const mlir.Attribute = null, + ftz: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); + return mlir.Operation.make(ctx, "nvvm.sub.packed.f32x2", .{ + .operands = .{ .flat = &.{ srcA, srcB } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.sub.packed.f32x2.bf16x2.f32x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn sub_packed_f32x2_bf16x2_f32x2( + ctx: *mlir.Context, + srcA: *const mlir.Value, + srcB: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + rnd: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + return mlir.Operation.make(ctx, "nvvm.sub.packed.f32x2.bf16x2.f32x2", .{ + .operands = .{ .flat = &.{ srcA, srcB } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.sub.packed.f32x2.f16x2.f32x2`. Result types are explicit; attributes use mlir.Attribute. +pub fn sub_packed_f32x2_f16x2_f32x2( + ctx: *mlir.Context, + srcA: *const mlir.Value, + srcB: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + rnd: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + return mlir.Operation.make(ctx, "nvvm.sub.packed.f32x2.f16x2.f32x2", .{ + .operands = .{ .flat = &.{ srcA, srcB } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.subf`. Result types are explicit; attributes use mlir.Attribute. +pub fn subf( + ctx: *mlir.Context, + lhs: *const mlir.Value, + rhs: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + rnd: ?*const mlir.Attribute = null, + sat: ?*const mlir.Attribute = null, + ftz: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); + if (opts.sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); + if (opts.ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); + return mlir.Operation.make(ctx, "nvvm.subf", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.tcgen05.alloc`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_alloc( + ctx: *mlir.Context, + addr: *const mlir.Value, + nCols: *const mlir.Value, + opts: struct { + isExclusive: ?*const mlir.Attribute = null, + group: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.isExclusive) |value| attributes.appendAssumeCapacity(.named(ctx, "isExclusive", value)); + if (opts.group) |value| attributes.appendAssumeCapacity(.named(ctx, "group", value)); + return mlir.Operation.make(ctx, "nvvm.tcgen05.alloc", .{ + .operands = .{ .flat = &.{ addr, nCols } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.tcgen05.commit`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_commit( + ctx: *mlir.Context, + addr: *const mlir.Value, + opts: struct { + multicastMask: ?*const mlir.Value = null, + smem_a_read: ?*const mlir.Attribute = null, + group: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.smem_a_read) |value| attributes.appendAssumeCapacity(.named(ctx, "smem_a_read", value)); + if (opts.group) |value| attributes.appendAssumeCapacity(.named(ctx, "group", value)); + return mlir.Operation.make(ctx, "nvvm.tcgen05.commit", .{ + .operands = .{ .flat = if (opts.multicastMask) |value| &.{ addr, value } else &.{addr} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.tcgen05.cp`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_cp( + ctx: *mlir.Context, + taddr: *const mlir.Value, + smem_desc: *const mlir.Value, + shape: *const mlir.Attribute, + opts: struct { + group: ?*const mlir.Attribute = null, + multicast: ?*const mlir.Attribute = null, + srcFormat: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); + if (opts.group) |value| attributes.appendAssumeCapacity(.named(ctx, "group", value)); + if (opts.multicast) |value| attributes.appendAssumeCapacity(.named(ctx, "multicast", value)); + if (opts.srcFormat) |value| attributes.appendAssumeCapacity(.named(ctx, "srcFormat", value)); + return mlir.Operation.make(ctx, "nvvm.tcgen05.cp", .{ + .operands = .{ .flat = &.{ taddr, smem_desc } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.tcgen05.dealloc`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_dealloc( + ctx: *mlir.Context, + taddr: *const mlir.Value, + nCols: *const mlir.Value, + opts: struct { + isExclusive: ?*const mlir.Attribute = null, + group: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.isExclusive) |value| attributes.appendAssumeCapacity(.named(ctx, "isExclusive", value)); + if (opts.group) |value| attributes.appendAssumeCapacity(.named(ctx, "group", value)); + return mlir.Operation.make(ctx, "nvvm.tcgen05.dealloc", .{ + .operands = .{ .flat = &.{ taddr, nCols } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.tcgen05.fence`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_fence(ctx: *mlir.Context, kind: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.tcgen05.fence", .{ + .attributes = &.{ + .named(ctx, "kind", kind), + }, + .location = location, + }); +} + +/// `nvvm.tcgen05.ld`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_ld( + ctx: *mlir.Context, + tmemAddr: *const mlir.Value, + res_type: *const mlir.Type, + shape: *const mlir.Attribute, + opts: struct { + offset: ?*const mlir.Value = null, + pack: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.pack) |value| attributes.appendAssumeCapacity(.named(ctx, "pack", value)); + attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); + return mlir.Operation.make(ctx, "nvvm.tcgen05.ld", .{ + .operands = .{ .flat = if (opts.offset) |value| &.{ tmemAddr, value } else &.{tmemAddr} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.tcgen05.ld.red`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_ld_red( + ctx: *mlir.Context, + addr: *const mlir.Value, + data_type: *const mlir.Type, + redVal_type: *const mlir.Type, + shape: *const mlir.Attribute, + op: *const mlir.Attribute, + opts: struct { + offset: ?*const mlir.Value = null, + abs: ?*const mlir.Attribute = null, + nan: ?*const mlir.Attribute = null, + isSigned: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); + attributes.appendAssumeCapacity(.named(ctx, "op", op)); + if (opts.abs) |value| attributes.appendAssumeCapacity(.named(ctx, "abs", value)); + if (opts.nan) |value| attributes.appendAssumeCapacity(.named(ctx, "nan", value)); + if (opts.isSigned) |value| attributes.appendAssumeCapacity(.named(ctx, "isSigned", value)); + return mlir.Operation.make(ctx, "nvvm.tcgen05.ld.red", .{ + .operands = .{ .flat = if (opts.offset) |value| &.{ addr, value } else &.{addr} }, + .results = .{ .flat = &.{ data_type, redVal_type } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.tcgen05.ld.red.spcompress`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_ld_red_spcompress( + ctx: *mlir.Context, + tmemAddr: *const mlir.Value, + metadata_type: *const mlir.Type, + compressedData_type: *const mlir.Type, + redVal_type: *const mlir.Type, + shape: *const mlir.Attribute, + opKind: *const mlir.Attribute, + compFactor: *const mlir.Attribute, + repFactor: *const mlir.Attribute, + opts: struct { + abs: ?*const mlir.Attribute = null, + nan: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 6) = .empty; + if (opts.abs) |value| attributes.appendAssumeCapacity(.named(ctx, "abs", value)); + if (opts.nan) |value| attributes.appendAssumeCapacity(.named(ctx, "nan", value)); + attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); + attributes.appendAssumeCapacity(.named(ctx, "opKind", opKind)); + attributes.appendAssumeCapacity(.named(ctx, "compFactor", compFactor)); + attributes.appendAssumeCapacity(.named(ctx, "repFactor", repFactor)); + return mlir.Operation.make(ctx, "nvvm.tcgen05.ld.red.spcompress", .{ + .operands = .{ .flat = &.{tmemAddr} }, + .results = .{ .flat = &.{ metadata_type, compressedData_type, redVal_type } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.tcgen05.ld.spcompress`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_ld_spcompress( + ctx: *mlir.Context, + tmemAddr: *const mlir.Value, + metadata_type: *const mlir.Type, + compressedData_type: *const mlir.Type, + shape: *const mlir.Attribute, + opKind: *const mlir.Attribute, + compFactor: *const mlir.Attribute, + repFactor: *const mlir.Attribute, + opts: struct { + abs: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + if (opts.abs) |value| attributes.appendAssumeCapacity(.named(ctx, "abs", value)); + attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); + attributes.appendAssumeCapacity(.named(ctx, "opKind", opKind)); + attributes.appendAssumeCapacity(.named(ctx, "compFactor", compFactor)); + attributes.appendAssumeCapacity(.named(ctx, "repFactor", repFactor)); + return mlir.Operation.make(ctx, "nvvm.tcgen05.ld.spcompress", .{ + .operands = .{ .flat = &.{tmemAddr} }, + .results = .{ .flat = &.{ metadata_type, compressedData_type } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.tcgen05.mma`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_mma( + ctx: *mlir.Context, + matrixD: *const mlir.Value, + matrixA: *const mlir.Value, + matrixB: *const mlir.Value, + idesc: *const mlir.Value, + enableInputD: *const mlir.Value, + kind: *const mlir.Attribute, + ctaGroup: *const mlir.Attribute, + opts: struct { + scaleInputD: ?*const mlir.Value = null, + disableOutputLane: ?*const mlir.Value = null, + collectorOp: ?*const mlir.Attribute = null, + collectorOpB: ?*const mlir.Attribute = null, + aShift: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + attributes.appendAssumeCapacity(.named(ctx, "ctaGroup", ctaGroup)); + if (opts.collectorOp) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOp", value)); + if (opts.collectorOpB) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOpB", value)); + if (opts.aShift) |value| attributes.appendAssumeCapacity(.named(ctx, "aShift", value)); + return mlir.Operation.make(ctx, "nvvm.tcgen05.mma", .{ + .operands = .{ .variadic = &.{ + &.{matrixD}, + &.{matrixA}, + &.{matrixB}, + &.{idesc}, + &.{enableInputD}, + if (opts.scaleInputD) |value| &.{value} else &.{}, + if (opts.disableOutputLane) |value| &.{value} else &.{}, + } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.tcgen05.mma.block_scale`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_mma_block_scale( + ctx: *mlir.Context, + matrixD: *const mlir.Value, + matrixA: *const mlir.Value, + matrixB: *const mlir.Value, + idesc: *const mlir.Value, + enableInputD: *const mlir.Value, + scaleA: *const mlir.Value, + scaleB: *const mlir.Value, + kind: *const mlir.Attribute, + ctaGroup: *const mlir.Attribute, + opts: struct { + blockScale: ?*const mlir.Attribute = null, + collectorOp: ?*const mlir.Attribute = null, + collectorOpB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + attributes.appendAssumeCapacity(.named(ctx, "ctaGroup", ctaGroup)); + if (opts.blockScale) |value| attributes.appendAssumeCapacity(.named(ctx, "blockScale", value)); + if (opts.collectorOp) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOp", value)); + if (opts.collectorOpB) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOpB", value)); + return mlir.Operation.make(ctx, "nvvm.tcgen05.mma.block_scale", .{ + .operands = .{ .flat = &.{ matrixD, matrixA, matrixB, idesc, enableInputD, scaleA, scaleB } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.tcgen05.mma.block_scale.decompress_b`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_mma_block_scale_decompress_b( + ctx: *mlir.Context, + matrixD: *const mlir.Value, + matrixA: *const mlir.Value, + matrixB: *const mlir.Value, + idesc: *const mlir.Value, + enableInputD: *const mlir.Value, + scaleA: *const mlir.Value, + scaleB: *const mlir.Value, + decompressBMetadata: *const mlir.Value, + kind: *const mlir.Attribute, + ctaGroup: *const mlir.Attribute, + opts: struct { + blockScale: ?*const mlir.Attribute = null, + collectorOpA: ?*const mlir.Attribute = null, + collectorOpB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + attributes.appendAssumeCapacity(.named(ctx, "ctaGroup", ctaGroup)); + if (opts.blockScale) |value| attributes.appendAssumeCapacity(.named(ctx, "blockScale", value)); + if (opts.collectorOpA) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOpA", value)); + if (opts.collectorOpB) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOpB", value)); + return mlir.Operation.make(ctx, "nvvm.tcgen05.mma.block_scale.decompress_b", .{ + .operands = .{ .flat = &.{ matrixD, matrixA, matrixB, idesc, enableInputD, scaleA, scaleB, decompressBMetadata } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.tcgen05.mma.decompress_b`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_mma_decompress_b( + ctx: *mlir.Context, + matrixD: *const mlir.Value, + matrixA: *const mlir.Value, + matrixB: *const mlir.Value, + idesc: *const mlir.Value, + enableInputD: *const mlir.Value, + decompressBMetadata: *const mlir.Value, + kind: *const mlir.Attribute, + ctaGroup: *const mlir.Attribute, + opts: struct { + disableOutputLane: ?*const mlir.Value = null, + collectorOpA: ?*const mlir.Attribute = null, + collectorOpB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + attributes.appendAssumeCapacity(.named(ctx, "ctaGroup", ctaGroup)); + if (opts.collectorOpA) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOpA", value)); + if (opts.collectorOpB) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOpB", value)); + return mlir.Operation.make(ctx, "nvvm.tcgen05.mma.decompress_b", .{ + .operands = .{ .flat = if (opts.disableOutputLane) |value| &.{ matrixD, matrixA, matrixB, idesc, enableInputD, decompressBMetadata, value } else &.{ matrixD, matrixA, matrixB, idesc, enableInputD, decompressBMetadata } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.tcgen05.mma.sp`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_mma_sp( + ctx: *mlir.Context, + matrixD: *const mlir.Value, + matrixA: *const mlir.Value, + matrixB: *const mlir.Value, + idesc: *const mlir.Value, + enableInputD: *const mlir.Value, + sparseMetadata: *const mlir.Value, + kind: *const mlir.Attribute, + ctaGroup: *const mlir.Attribute, + opts: struct { + scaleInputD: ?*const mlir.Value = null, + disableOutputLane: ?*const mlir.Value = null, + collectorOp: ?*const mlir.Attribute = null, + collectorOpB: ?*const mlir.Attribute = null, + aShift: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + attributes.appendAssumeCapacity(.named(ctx, "ctaGroup", ctaGroup)); + if (opts.collectorOp) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOp", value)); + if (opts.collectorOpB) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOpB", value)); + if (opts.aShift) |value| attributes.appendAssumeCapacity(.named(ctx, "aShift", value)); + return mlir.Operation.make(ctx, "nvvm.tcgen05.mma.sp", .{ + .operands = .{ .variadic = &.{ + &.{matrixD}, + &.{matrixA}, + &.{matrixB}, + &.{idesc}, + &.{enableInputD}, + &.{sparseMetadata}, + if (opts.scaleInputD) |value| &.{value} else &.{}, + if (opts.disableOutputLane) |value| &.{value} else &.{}, + } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.tcgen05.mma.sp.block_scale`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_mma_sp_block_scale( + ctx: *mlir.Context, + matrixD: *const mlir.Value, + matrixA: *const mlir.Value, + matrixB: *const mlir.Value, + idesc: *const mlir.Value, + enableInputD: *const mlir.Value, + sparseMetadata: *const mlir.Value, + scaleA: *const mlir.Value, + scaleB: *const mlir.Value, + kind: *const mlir.Attribute, + ctaGroup: *const mlir.Attribute, + opts: struct { + blockScale: ?*const mlir.Attribute = null, + collectorOp: ?*const mlir.Attribute = null, + collectorOpB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + attributes.appendAssumeCapacity(.named(ctx, "ctaGroup", ctaGroup)); + if (opts.blockScale) |value| attributes.appendAssumeCapacity(.named(ctx, "blockScale", value)); + if (opts.collectorOp) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOp", value)); + if (opts.collectorOpB) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOpB", value)); + return mlir.Operation.make(ctx, "nvvm.tcgen05.mma.sp.block_scale", .{ + .operands = .{ .flat = &.{ matrixD, matrixA, matrixB, idesc, enableInputD, sparseMetadata, scaleA, scaleB } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.tcgen05.mma.ws`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_mma_ws( + ctx: *mlir.Context, + matrixD: *const mlir.Value, + matrixA: *const mlir.Value, + matrixB: *const mlir.Value, + idesc: *const mlir.Value, + enableInputD: *const mlir.Value, + kind: *const mlir.Attribute, + opts: struct { + zeroColMask: ?*const mlir.Value = null, + collectorBBuffer: ?*const mlir.Attribute = null, + collectorOp: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + if (opts.collectorBBuffer) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorBBuffer", value)); + if (opts.collectorOp) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOp", value)); + return mlir.Operation.make(ctx, "nvvm.tcgen05.mma.ws", .{ + .operands = .{ .flat = if (opts.zeroColMask) |value| &.{ matrixD, matrixA, matrixB, idesc, enableInputD, value } else &.{ matrixD, matrixA, matrixB, idesc, enableInputD } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.tcgen05.mma.ws.sp`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_mma_ws_sp( + ctx: *mlir.Context, + matrixD: *const mlir.Value, + matrixA: *const mlir.Value, + matrixB: *const mlir.Value, + idesc: *const mlir.Value, + enableInputD: *const mlir.Value, + sparseMetadata: *const mlir.Value, + kind: *const mlir.Attribute, + opts: struct { + zeroColMask: ?*const mlir.Value = null, + collectorBBuffer: ?*const mlir.Attribute = null, + collectorOp: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); + if (opts.collectorBBuffer) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorBBuffer", value)); + if (opts.collectorOp) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOp", value)); + return mlir.Operation.make(ctx, "nvvm.tcgen05.mma.ws.sp", .{ + .operands = .{ .flat = if (opts.zeroColMask) |value| &.{ matrixD, matrixA, matrixB, idesc, enableInputD, sparseMetadata, value } else &.{ matrixD, matrixA, matrixB, idesc, enableInputD, sparseMetadata } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.tcgen05.mma_smem_desc`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_mma_smem_desc(ctx: *mlir.Context, startAddr: *const mlir.Value, leadingDimOffset: *const mlir.Value, strideDimOffset: *const mlir.Value, baseOffset: *const mlir.Value, leadingDimMode: *const mlir.Value, swizzleMode: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.tcgen05.mma_smem_desc", .{ + .operands = .{ .flat = &.{ startAddr, leadingDimOffset, strideDimOffset, baseOffset, leadingDimMode, swizzleMode } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.tcgen05.relinquish_alloc_permit`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_relinquish_alloc_permit( + ctx: *mlir.Context, + opts: struct { + group: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.group) |value| attributes.appendAssumeCapacity(.named(ctx, "group", value)); + return mlir.Operation.make(ctx, "nvvm.tcgen05.relinquish_alloc_permit", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.tcgen05.shift`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_shift( + ctx: *mlir.Context, + taddr: *const mlir.Value, + opts: struct { + group: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.group) |value| attributes.appendAssumeCapacity(.named(ctx, "group", value)); + return mlir.Operation.make(ctx, "nvvm.tcgen05.shift", .{ + .operands = .{ .flat = &.{taddr} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.tcgen05.st`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_st( + ctx: *mlir.Context, + tmemAddr: *const mlir.Value, + val: *const mlir.Value, + shape: *const mlir.Attribute, + opts: struct { + offset: ?*const mlir.Value = null, + unpack: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.unpack) |value| attributes.appendAssumeCapacity(.named(ctx, "unpack", value)); + attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); + return mlir.Operation.make(ctx, "nvvm.tcgen05.st", .{ + .operands = .{ .flat = if (opts.offset) |value| &.{ tmemAddr, val, value } else &.{ tmemAddr, val } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.tcgen05.wait`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_wait(ctx: *mlir.Context, kind: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.tcgen05.wait", .{ + .attributes = &.{ + .named(ctx, "kind", kind), + }, + .location = location, + }); +} + +/// `nvvm.tcgen05_mma_smem_desc_v2`. Result types are explicit; attributes use mlir.Attribute. +pub fn tcgen05_mma_smem_desc_v2(ctx: *mlir.Context, startAddress: *const mlir.Value, leadingDimOffset: *const mlir.Value, strideDimOffset: *const mlir.Value, descriptorVersion: *const mlir.Value, baseOffset: *const mlir.Value, leadingDimMode: *const mlir.Value, kSegmentOffset: *const mlir.Value, swizzleType: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.tcgen05_mma_smem_desc_v2", .{ + .operands = .{ .flat = &.{ startAddress, leadingDimOffset, strideDimOffset, descriptorVersion, baseOffset, leadingDimMode, kSegmentOffset, swizzleType } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `nvvm.tensormap.cp_fenceproxy`. Result types are explicit; attributes use mlir.Attribute. +pub fn tensormap_cp_fenceproxy(ctx: *mlir.Context, dst: *const mlir.Value, src: *const mlir.Value, size: *const mlir.Value, scope: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.tensormap.cp_fenceproxy", .{ + .operands = .{ .flat = &.{ dst, src, size } }, + .attributes = &.{ + .named(ctx, "scope", scope), + }, + .location = location, + }); +} + +/// `nvvm.tensormap.replace`. Result types are explicit; attributes use mlir.Attribute. +pub fn tensormap_replace( + ctx: *mlir.Context, + addr: *const mlir.Value, + field: *const mlir.Attribute, + opts: struct { + new_value: ?*const mlir.Value = null, + ord: ?*const mlir.Attribute = null, + new_value_attr: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "field", field)); + if (opts.ord) |value| attributes.appendAssumeCapacity(.named(ctx, "ord", value)); + if (opts.new_value_attr) |value| attributes.appendAssumeCapacity(.named(ctx, "new_value_attr", value)); + return mlir.Operation.make(ctx, "nvvm.tensormap.replace", .{ + .operands = .{ .flat = if (opts.new_value) |value| &.{ addr, value } else &.{addr} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.trace.mark`. Result types are explicit; attributes use mlir.Attribute. +pub fn trace_mark( + ctx: *mlir.Context, + eventType: *const mlir.Attribute, + domain: *const mlir.Attribute, + event: *const mlir.Attribute, + opts: struct { + payload: ?*const mlir.Value = null, + predicate: ?*const mlir.Value = null, + payloadDescriptor: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "eventType", eventType)); + attributes.appendAssumeCapacity(.named(ctx, "domain", domain)); + attributes.appendAssumeCapacity(.named(ctx, "event", event)); + if (opts.payloadDescriptor) |value| attributes.appendAssumeCapacity(.named(ctx, "payloadDescriptor", value)); + return mlir.Operation.make(ctx, "nvvm.trace.mark", .{ + .operands = .{ .variadic = &.{ + if (opts.payload) |value| &.{value} else &.{}, + if (opts.predicate) |value| &.{value} else &.{}, + } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.vote.sync`. Result types are explicit; attributes use mlir.Attribute. +pub fn vote_sync(ctx: *mlir.Context, mask: *const mlir.Value, pred: *const mlir.Value, res_type: *const mlir.Type, kind: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.vote.sync", .{ + .operands = .{ .flat = &.{ mask, pred } }, + .results = .{ .flat = &.{res_type} }, + .attributes = &.{ + .named(ctx, "kind", kind), + }, + .location = location, + }); +} + +/// `nvvm.wgmma.commit.group.sync.aligned`. Result types are explicit; attributes use mlir.Attribute. +pub fn wgmma_commit_group_sync_aligned(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.wgmma.commit.group.sync.aligned", .{ + .location = location, + }); +} + +/// `nvvm.wgmma.fence.aligned`. Result types are explicit; attributes use mlir.Attribute. +pub fn wgmma_fence_aligned(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.wgmma.fence.aligned", .{ + .location = location, + }); +} + +/// `nvvm.wgmma.mma_async`. Result types are explicit; attributes use mlir.Attribute. +pub fn wgmma_mma_async( + ctx: *mlir.Context, + inouts: *const mlir.Value, + descriptorA: *const mlir.Value, + descriptorB: *const mlir.Value, + results__type: *const mlir.Type, + shape: *const mlir.Attribute, + typeA: *const mlir.Attribute, + typeB: *const mlir.Attribute, + typeD: *const mlir.Attribute, + scaleD: *const mlir.Attribute, + scaleA: *const mlir.Attribute, + scaleB: *const mlir.Attribute, + layoutA: *const mlir.Attribute, + layoutB: *const mlir.Attribute, + opts: struct { + satfinite: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 10) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); + attributes.appendAssumeCapacity(.named(ctx, "typeA", typeA)); + attributes.appendAssumeCapacity(.named(ctx, "typeB", typeB)); + attributes.appendAssumeCapacity(.named(ctx, "typeD", typeD)); + attributes.appendAssumeCapacity(.named(ctx, "scaleD", scaleD)); + attributes.appendAssumeCapacity(.named(ctx, "scaleA", scaleA)); + attributes.appendAssumeCapacity(.named(ctx, "scaleB", scaleB)); + attributes.appendAssumeCapacity(.named(ctx, "layoutA", layoutA)); + attributes.appendAssumeCapacity(.named(ctx, "layoutB", layoutB)); + if (opts.satfinite) |value| attributes.appendAssumeCapacity(.named(ctx, "satfinite", value)); + return mlir.Operation.make(ctx, "nvvm.wgmma.mma_async", .{ + .operands = .{ .flat = &.{ inouts, descriptorA, descriptorB } }, + .results = .{ .flat = &.{results__type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `nvvm.wgmma.wait.group.sync.aligned`. Result types are explicit; attributes use mlir.Attribute. +pub fn wgmma_wait_group_sync_aligned(ctx: *mlir.Context, group: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.wgmma.wait.group.sync.aligned", .{ + .attributes = &.{ + .named(ctx, "group", group), + }, + .location = location, + }); +} + +/// `nvvm.wmma.load`. Result types are explicit; attributes use mlir.Attribute. +pub fn wmma_load(ctx: *mlir.Context, ptr: *const mlir.Value, stride: *const mlir.Value, res_type: *const mlir.Type, m: *const mlir.Attribute, n: *const mlir.Attribute, k: *const mlir.Attribute, layout: *const mlir.Attribute, eltype: *const mlir.Attribute, frag: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.wmma.load", .{ + .operands = .{ .flat = &.{ ptr, stride } }, + .results = .{ .flat = &.{res_type} }, + .attributes = &.{ + .named(ctx, "m", m), + .named(ctx, "n", n), + .named(ctx, "k", k), + .named(ctx, "layout", layout), + .named(ctx, "eltype", eltype), + .named(ctx, "frag", frag), + }, + .location = location, + }); +} + +/// `nvvm.wmma.mma`. Result types are explicit; attributes use mlir.Attribute. +pub fn wmma_mma(ctx: *mlir.Context, args: []const *const mlir.Value, res_type: *const mlir.Type, m: *const mlir.Attribute, n: *const mlir.Attribute, k: *const mlir.Attribute, layoutA: *const mlir.Attribute, layoutB: *const mlir.Attribute, eltypeA: *const mlir.Attribute, eltypeB: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "nvvm.wmma.mma", .{ + .operands = .{ .flat = args }, + .results = .{ .flat = &.{res_type} }, + .attributes = &.{ + .named(ctx, "m", m), + .named(ctx, "n", n), + .named(ctx, "k", k), + .named(ctx, "layoutA", layoutA), + .named(ctx, "layoutB", layoutB), + .named(ctx, "eltypeA", eltypeA), + .named(ctx, "eltypeB", eltypeB), + }, + .location = location, + }); +} + +/// `nvvm.wmma.store`. Result types are explicit; attributes use mlir.Attribute. +pub fn wmma_store(ctx: *mlir.Context, ptr: *const mlir.Value, args: []const *const mlir.Value, stride: *const mlir.Value, m: *const mlir.Attribute, n: *const mlir.Attribute, k: *const mlir.Attribute, layout: *const mlir.Attribute, eltype: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + var operands: stdx.BoundedArray(*const mlir.Value, 256) = .empty; + operands.appendAssumeCapacity(ptr); + operands.appendSliceAssumeCapacity(args); + operands.appendAssumeCapacity(stride); + return mlir.Operation.make(ctx, "nvvm.wmma.store", .{ + .operands = .{ .flat = operands.constSlice() }, + .attributes = &.{ + .named(ctx, "m", m), + .named(ctx, "n", n), + .named(ctx, "k", k), + .named(ctx, "layout", layout), + .named(ctx, "eltype", eltype), + }, + .location = location, + }); +} diff --git a/mlir/dialects/nvvm/test.zig b/mlir/dialects/nvvm/test.zig new file mode 100644 index 00000000..b908f6f4 --- /dev/null +++ b/mlir/dialects/nvvm/test.zig @@ -0,0 +1,481 @@ +const std = @import("std"); + +const mlir = @import("mlir"); +const nvvm = @import("mlir/dialects/nvvm"); + +const operation_names: []const []const u8 = &.{ + "nvvm.add.packed.bf16x2.f32x2.f32x2", + "nvvm.add.packed.f16x2.f32x2.f32x2", + "nvvm.add.packed.f32x2", + "nvvm.add.packed.f32x2.bf16x2.f32x2", + "nvvm.add.packed.f32x2.f16x2.f32x2", + "nvvm.addf", + "nvvm.applypriority.async.bulk", + "nvvm.applypriority.async.bulk.tensor", + "nvvm.applypriority.async.bulk.tensor.override", + "nvvm.atomicrmw", + "nvvm.bar.warp.sync", + "nvvm.barrier", + "nvvm.barrier.arrive", + "nvvm.barrier.cta.arrive", + "nvvm.barrier.cta.red", + "nvvm.barrier.cta.sync", + "nvvm.barrier0", + "nvvm.breakpoint", + "nvvm.clmad", + "nvvm.cluster.arrive", + "nvvm.cluster.arrive.relaxed", + "nvvm.cluster.wait", + "nvvm.clusterlaunchcontrol.query.cancel", + "nvvm.clusterlaunchcontrol.try.cancel", + "nvvm.compare.and.set", + "nvvm.convert.and.pack.integer", + "nvvm.convert.bf16x2.to.f4x2", + "nvvm.convert.bf16x2.to.f6x2", + "nvvm.convert.bf16x2.to.f8x2", + "nvvm.convert.bf16x2.to.s2f6x2", + "nvvm.convert.f16x2.to.f4x2", + "nvvm.convert.f16x2.to.f6x2", + "nvvm.convert.f16x2.to.f8x2", + "nvvm.convert.f32x2.to.bf16x2", + "nvvm.convert.f32x2.to.f16x2", + "nvvm.convert.f32x2.to.f4x2", + "nvvm.convert.f32x2.to.f6x2", + "nvvm.convert.f32x2.to.f8x2", + "nvvm.convert.f32x2.to.s2f6x2", + "nvvm.convert.f32x4.to.f4x4", + "nvvm.convert.f32x4.to.f6x4", + "nvvm.convert.f32x4.to.f8x4", + "nvvm.convert.f4x2.to.f16x2", + "nvvm.convert.f6x2.to.f16x2", + "nvvm.convert.f8x2.to.bf16x2", + "nvvm.convert.f8x2.to.f16x2", + "nvvm.convert.float.to.integer", + "nvvm.convert.float.to.tf32", + "nvvm.convert.s2f6x2.to.bf16x2", + "nvvm.cos", + "nvvm.cp.async.bulk.commit.group", + "nvvm.cp.async.bulk.global.shared.cta", + "nvvm.cp.async.bulk.prefetch", + "nvvm.cp.async.bulk.shared.cluster.global", + "nvvm.cp.async.bulk.shared.cluster.shared.cta", + "nvvm.cp.async.bulk.tensor.global.shared.cta", + "nvvm.cp.async.bulk.tensor.global.shared.cta.override", + "nvvm.cp.async.bulk.tensor.prefetch", + "nvvm.cp.async.bulk.tensor.prefetch.override", + "nvvm.cp.async.bulk.tensor.reduce", + "nvvm.cp.async.bulk.tensor.reduce.override", + "nvvm.cp.async.bulk.tensor.shared.cluster.global", + "nvvm.cp.async.bulk.tensor.shared.cluster.global.override", + "nvvm.cp.async.bulk.wait_group", + "nvvm.cp.async.commit.group", + "nvvm.cp.async.mbarrier.arrive", + "nvvm.cp.async.shared.global", + "nvvm.cp.async.wait.group", + "nvvm.cvt.packfloat", + "nvvm.cvt.packfloat.f32", + "nvvm.cvt.to.f4x8.packed", + "nvvm.dot.accumulate.2way", + "nvvm.dot.accumulate.4way", + "nvvm.elect.sync", + "nvvm.ex2", + "nvvm.exit", + "nvvm.fabs", + "nvvm.fence.mbarrier.init", + "nvvm.fence.proxy", + "nvvm.fence.proxy.acquire", + "nvvm.fence.proxy.release", + "nvvm.fence.proxy.sync_restrict", + "nvvm.fence.sc.cluster", + "nvvm.fence.sync_restrict", + "nvvm.fma", + "nvvm.fma.packed.f32x2", + "nvvm.fma.packed.f32x2.bf16x2.f32x2.f32x2", + "nvvm.fma.packed.f32x2.f16x2.f32x2.f32x2", + "nvvm.fmax", + "nvvm.fmin", + "nvvm.griddepcontrol", + "nvvm.inline_ptx", + "nvvm.ldmatrix", + "nvvm.load.ext", + "nvvm.log2", + "nvvm.mapa", + "nvvm.match.sync", + "nvvm.mbarrier.arrive", + "nvvm.mbarrier.arrive.expect_tx", + "nvvm.mbarrier.arrive.nocomplete", + "nvvm.mbarrier.arrive_drop", + "nvvm.mbarrier.arrive_drop.expect_tx", + "nvvm.mbarrier.arrive_drop.nocomplete", + "nvvm.mbarrier.check.layout", + "nvvm.mbarrier.complete_tx", + "nvvm.mbarrier.expect_tx", + "nvvm.mbarrier.init", + "nvvm.mbarrier.inval", + "nvvm.mbarrier.test.wait", + "nvvm.mbarrier.try_wait", + "nvvm.mbarrier.try_wait.parity", + "nvvm.mbarrier.try_wait.parity.timelimit", + "nvvm.mbarrier.try_wait.timelimit", + "nvvm.mbarrier.txn", + "nvvm.mbarrier.txn.cta", + "nvvm.mbarrier.wait", + "nvvm.mbarrier.wait.parity", + "nvvm.memory.barrier", + "nvvm.mma.block_scale", + "nvvm.mma.block_scale.internal", + "nvvm.mma.sp.block_scale", + "nvvm.mma.sp.block_scale.internal", + "nvvm.mma.sp.sync", + "nvvm.mma.sync", + "nvvm.mma_smem_desc", + "nvvm.movmatrix", + "nvvm.mul", + "nvvm.mul.packed.bf16x2.bf16x2.f16x2", + "nvvm.mul.packed.bf16x2.f32x2.f32x2", + "nvvm.mul.packed.f16x2.f16x2.bf16x2", + "nvvm.mul.packed.f16x2.f32x2.f32x2", + "nvvm.mul.packed.f32x2", + "nvvm.multimem.ld.reduce", + "nvvm.multimem.red", + "nvvm.multimem.st", + "nvvm.nanosleep", + "nvvm.pmevent", + "nvvm.prefetch", + "nvvm.prmt", + "nvvm.rcp.approx.ftz.f", + "nvvm.read.ptx.sreg.aggr.smem.size", + "nvvm.read.ptx.sreg.clock", + "nvvm.read.ptx.sreg.clock64", + "nvvm.read.ptx.sreg.cluster.ctaid.x", + "nvvm.read.ptx.sreg.cluster.ctaid.y", + "nvvm.read.ptx.sreg.cluster.ctaid.z", + "nvvm.read.ptx.sreg.cluster.ctarank", + "nvvm.read.ptx.sreg.cluster.nctaid.x", + "nvvm.read.ptx.sreg.cluster.nctaid.y", + "nvvm.read.ptx.sreg.cluster.nctaid.z", + "nvvm.read.ptx.sreg.cluster.nctarank", + "nvvm.read.ptx.sreg.clusterid.x", + "nvvm.read.ptx.sreg.clusterid.y", + "nvvm.read.ptx.sreg.clusterid.z", + "nvvm.read.ptx.sreg.ctaid.x", + "nvvm.read.ptx.sreg.ctaid.y", + "nvvm.read.ptx.sreg.ctaid.z", + "nvvm.read.ptx.sreg.dynamic.smem.size", + "nvvm.read.ptx.sreg.envreg0", + "nvvm.read.ptx.sreg.envreg1", + "nvvm.read.ptx.sreg.envreg10", + "nvvm.read.ptx.sreg.envreg11", + "nvvm.read.ptx.sreg.envreg12", + "nvvm.read.ptx.sreg.envreg13", + "nvvm.read.ptx.sreg.envreg14", + "nvvm.read.ptx.sreg.envreg15", + "nvvm.read.ptx.sreg.envreg16", + "nvvm.read.ptx.sreg.envreg17", + "nvvm.read.ptx.sreg.envreg18", + "nvvm.read.ptx.sreg.envreg19", + "nvvm.read.ptx.sreg.envreg2", + "nvvm.read.ptx.sreg.envreg20", + "nvvm.read.ptx.sreg.envreg21", + "nvvm.read.ptx.sreg.envreg22", + "nvvm.read.ptx.sreg.envreg23", + "nvvm.read.ptx.sreg.envreg24", + "nvvm.read.ptx.sreg.envreg25", + "nvvm.read.ptx.sreg.envreg26", + "nvvm.read.ptx.sreg.envreg27", + "nvvm.read.ptx.sreg.envreg28", + "nvvm.read.ptx.sreg.envreg29", + "nvvm.read.ptx.sreg.envreg3", + "nvvm.read.ptx.sreg.envreg30", + "nvvm.read.ptx.sreg.envreg31", + "nvvm.read.ptx.sreg.envreg4", + "nvvm.read.ptx.sreg.envreg5", + "nvvm.read.ptx.sreg.envreg6", + "nvvm.read.ptx.sreg.envreg7", + "nvvm.read.ptx.sreg.envreg8", + "nvvm.read.ptx.sreg.envreg9", + "nvvm.read.ptx.sreg.globaltimer", + "nvvm.read.ptx.sreg.globaltimer.lo", + "nvvm.read.ptx.sreg.gridid", + "nvvm.read.ptx.sreg.laneid", + "nvvm.read.ptx.sreg.lanemask.eq", + "nvvm.read.ptx.sreg.lanemask.ge", + "nvvm.read.ptx.sreg.lanemask.gt", + "nvvm.read.ptx.sreg.lanemask.le", + "nvvm.read.ptx.sreg.lanemask.lt", + "nvvm.read.ptx.sreg.nclusterid.x", + "nvvm.read.ptx.sreg.nclusterid.y", + "nvvm.read.ptx.sreg.nclusterid.z", + "nvvm.read.ptx.sreg.nctaid.x", + "nvvm.read.ptx.sreg.nctaid.y", + "nvvm.read.ptx.sreg.nctaid.z", + "nvvm.read.ptx.sreg.nsmid", + "nvvm.read.ptx.sreg.ntid.x", + "nvvm.read.ptx.sreg.ntid.y", + "nvvm.read.ptx.sreg.ntid.z", + "nvvm.read.ptx.sreg.nwarpid", + "nvvm.read.ptx.sreg.smid", + "nvvm.read.ptx.sreg.tid.x", + "nvvm.read.ptx.sreg.tid.y", + "nvvm.read.ptx.sreg.tid.z", + "nvvm.read.ptx.sreg.total.smem.size", + "nvvm.read.ptx.sreg.warpid", + "nvvm.read.ptx.sreg.warpsize", + "nvvm.red", + "nvvm.redux.sync", + "nvvm.rsqrt", + "nvvm.setmaxregister", + "nvvm.shfl.sync", + "nvvm.sin", + "nvvm.spcompress", + "nvvm.spdecompress", + "nvvm.st.bulk", + "nvvm.stmatrix", + "nvvm.store.async", + "nvvm.store.ext", + "nvvm.sub.packed.bf16x2.f32x2.f32x2", + "nvvm.sub.packed.f16x2.f32x2.f32x2", + "nvvm.sub.packed.f32x2", + "nvvm.sub.packed.f32x2.bf16x2.f32x2", + "nvvm.sub.packed.f32x2.f16x2.f32x2", + "nvvm.subf", + "nvvm.tcgen05.alloc", + "nvvm.tcgen05.commit", + "nvvm.tcgen05.cp", + "nvvm.tcgen05.dealloc", + "nvvm.tcgen05.fence", + "nvvm.tcgen05.ld", + "nvvm.tcgen05.ld.red", + "nvvm.tcgen05.ld.red.spcompress", + "nvvm.tcgen05.ld.spcompress", + "nvvm.tcgen05.mma", + "nvvm.tcgen05.mma.block_scale", + "nvvm.tcgen05.mma.block_scale.decompress_b", + "nvvm.tcgen05.mma.decompress_b", + "nvvm.tcgen05.mma.sp", + "nvvm.tcgen05.mma.sp.block_scale", + "nvvm.tcgen05.mma.ws", + "nvvm.tcgen05.mma.ws.sp", + "nvvm.tcgen05.mma_smem_desc", + "nvvm.tcgen05.relinquish_alloc_permit", + "nvvm.tcgen05.shift", + "nvvm.tcgen05.st", + "nvvm.tcgen05.wait", + "nvvm.tcgen05_mma_smem_desc_v2", + "nvvm.tensormap.cp_fenceproxy", + "nvvm.tensormap.replace", + "nvvm.trace.mark", + "nvvm.vote.sync", + "nvvm.wgmma.commit.group.sync.aligned", + "nvvm.wgmma.fence.aligned", + "nvvm.wgmma.mma_async", + "nvvm.wgmma.wait.group.sync.aligned", + "nvvm.wmma.load", + "nvvm.wmma.mma", + "nvvm.wmma.store", +}; + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(nvvm); + std.testing.refAllDecls(nvvm.ops); + inline for (comptime std.meta.declarations(nvvm.ops)) |decl| { + const T = @field(nvvm.ops, decl); + if (@TypeOf(T) == type) std.testing.refAllDecls(T); + } +} + +test "every enum attribute prints as the compiler does" { + @setEvalBranchQuota(1_000_000); + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + nvvm.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + inline for (.{ + .{ nvvm.ops.AtomicOpKindAttr, "#nvvm" }, + .{ nvvm.ops.BarrierReductionAttr, "#nvvm.reduction<{s}>" }, + .{ nvvm.ops.BarrierReduxKindAttr, "#nvvm.barrier_redux_kind<{s}>" }, + .{ nvvm.ops.BlockScaleFormatAttr, "#nvvm.block_scale_format<{s}>" }, + .{ nvvm.ops.CTAGroupKindAttr, "#nvvm.cta_group<{s}>" }, + .{ nvvm.ops.CVTPackFloatKindAttr, "#nvvm.packfloat_type<{s}>" }, + .{ nvvm.ops.CacheEvictionPriorityAttr, "#nvvm" }, + .{ nvvm.ops.ClusterLaunchControlQueryTypeAttr, "#nvvm" }, + .{ nvvm.ops.CompareOpKindAttr, "#nvvm.op<{s}>" }, + .{ nvvm.ops.ConvertFP4TypeAttr, "#nvvm.convert_fp4_type<{s}>" }, + .{ nvvm.ops.ConvertFP8TypeAttr, "#nvvm.convert_fp8_type<{s}>" }, + .{ nvvm.ops.ConvertScaleKindAttr, "#nvvm.convert_scale_kind<{s}>" }, + .{ nvvm.ops.DotAccumulateTypeAttr, "#nvvm.dot_accumulate_type<{s}>" }, + .{ nvvm.ops.EvictKindAttr, "#nvvm.evict_kind<{s}>" }, + .{ nvvm.ops.FPRoundingModeAttr, "#nvvm.fp_rnd_mode<{s}>" }, + .{ nvvm.ops.GridDepActionKindAttr, "#nvvm" }, + .{ nvvm.ops.IntegerRoundingModeAttr, "#nvvm.int_rnd_mode<{s}>" }, + .{ nvvm.ops.L2PrefetchSizeAttr, "#nvvm.l2_prefetch<{s}>" }, + .{ nvvm.ops.LdStMatrixEltTypeAttr, "#nvvm.ld_st_matrix_elt_type<{s}>" }, + .{ nvvm.ops.LoadCacheModifierExtKindAttr, "#nvvm.load_cache_modifier_ext<{s}>" }, + .{ nvvm.ops.LoadCacheModifierKindAttr, "#nvvm" }, + .{ nvvm.ops.LoadShapeAttr, "#nvvm.load_shape<{s}>" }, + .{ nvvm.ops.LoadSrcFormatAttr, "#nvvm.load_src_format<{s}>" }, + .{ nvvm.ops.MBarrierLayoutAttr, "#nvvm.mbarrier_layout<{s}>" }, + .{ nvvm.ops.MBarrierPhaseAttr, "#nvvm.mbarrier_phase<{s}>" }, + .{ nvvm.ops.MBarrierScopeKindAttr, "#nvvm.mbar_scope<{s}>" }, + .{ nvvm.ops.MBarrierSpaceKindAttr, "#nvvm.mbar_space<{s}>" }, + .{ nvvm.ops.MBarrierTxnKindAttr, "#nvvm.mbar_txn_kind<{s}>" }, + .{ nvvm.ops.MBarrierWaitKindAttr, "#nvvm.mbar_wait<{s}>" }, + .{ nvvm.ops.MMAB1OpAttr, "#nvvm.mma_b1op<{s}>" }, + .{ nvvm.ops.MMABlockScaleKindAttr, "#nvvm.block_scale_kind<{s}>" }, + .{ nvvm.ops.MMACtaCountAttr, "#nvvm.mma_cta_count<{s}>" }, + .{ nvvm.ops.MMAFragAttr, "#nvvm.mma_frag<{s}>" }, + .{ nvvm.ops.MMAIntOverflowAttr, "#nvvm.mma_int_overflow<{s}>" }, + .{ nvvm.ops.MMAKindAttr, "#nvvm.mma_kind<{s}>" }, + .{ nvvm.ops.MMALayoutAttr, "#nvvm.mma_layout<{s}>" }, + .{ nvvm.ops.MMATypesAttr, "#nvvm.mma_type<{s}>" }, + .{ nvvm.ops.MatchSyncKindAttr, "#nvvm" }, + .{ nvvm.ops.MemOrderKindAttr, "#nvvm.mem_order<{s}>" }, + .{ nvvm.ops.MemScopeKindAttr, "#nvvm.mem_scope<{s}>" }, + .{ nvvm.ops.MulModeAttr, "#nvvm" }, + .{ nvvm.ops.NVVMMemorySpaceAttr, "#nvvm.memory_space<{s}>" }, + .{ nvvm.ops.PermuteModeAttr, "#nvvm.permute_mode<{s}>" }, + .{ nvvm.ops.PrefetchCacheLevelAttr, "#nvvm" }, + .{ nvvm.ops.ProxyKindAttr, "#nvvm.proxy_kind<{s}>" }, + .{ nvvm.ops.ReductionKindAttr, "#nvvm" }, + .{ nvvm.ops.ReductionOpAttr, "#nvvm.red_op<{s}>" }, + .{ nvvm.ops.ReductionTypeAttr, "#nvvm.red_type<{s}>" }, + .{ nvvm.ops.SPCompressElemSizeAttr, "#nvvm.spcomp_elem_size<{s}>" }, + .{ nvvm.ops.SPCompressFactorTypeAttr, "#nvvm.spcomp_factor<{s}>" }, + .{ nvvm.ops.SPCompressIndexSizeAttr, "#nvvm.spcomp_index_size<{s}>" }, + .{ nvvm.ops.SPCompressOpKindAttr, "#nvvm.spcomp_op_kind<{s}>" }, + .{ nvvm.ops.SPCompressRepFactorAttr, "#nvvm.spcomp_rep_factor<{s}>" }, + .{ nvvm.ops.SPDecompressElemSizeAttr, "#nvvm.spdecomp_elem_size<{s}>" }, + .{ nvvm.ops.SPDecompressFactorTypeAttr, "#nvvm.spdecomp_factor<{s}>" }, + .{ nvvm.ops.SPDecompressIndexSizeAttr, "#nvvm.spdecomp_index_size<{s}>" }, + .{ nvvm.ops.SPDecompressRepFactorAttr, "#nvvm.spdecomp_rep_factor<{s}>" }, + .{ nvvm.ops.SaturationModeAttr, "#nvvm.sat_mode<{s}>" }, + .{ nvvm.ops.SaturationModeKindAttr, "#nvvm.sat<{s}>" }, + .{ nvvm.ops.ScaleVecSizeAttr, "#nvvm.scale_vec_size<{s}>" }, + .{ nvvm.ops.SetMaxRegisterActionAttr, "#nvvm" }, + .{ nvvm.ops.SharedSpaceAttr, "#nvvm.shared_space<{s}>" }, + .{ nvvm.ops.ShflKindAttr, "#nvvm" }, + .{ nvvm.ops.SparsityFormatAttr, "#nvvm.sparsity_format<{s}>" }, + .{ nvvm.ops.StateSpaceAttr, "#nvvm.state_space<{s}>" }, + .{ nvvm.ops.StoreCacheModifierKindAttr, "#nvvm.store_cache_modifier<{s}>" }, + .{ nvvm.ops.StoreShapeAttr, "#nvvm.store_shape<{s}>" }, + .{ nvvm.ops.TCBarParamAttr, "#nvvm.TCBarParam<{s}>" }, + .{ nvvm.ops.TMALoadModeAttr, "#nvvm.tma_load_mode<{s}>" }, + .{ nvvm.ops.TMAReduxKindAttr, "#nvvm.tma_redux_kind<{s}>" }, + .{ nvvm.ops.TMAStoreModeAttr, "#nvvm.tma_store_mode<{s}>" }, + .{ nvvm.ops.Tcgen05CpMulticastAttr, "#nvvm.tcgen05_cp_multicast<{s}>" }, + .{ nvvm.ops.Tcgen05CpShapeAttr, "#nvvm.tcgen05_cp_shape<{s}>" }, + .{ nvvm.ops.Tcgen05CpSrcFormatAttr, "#nvvm.tcgen05_cp_src_fmt<{s}>" }, + .{ nvvm.ops.Tcgen05FenceKindAttr, "#nvvm.tcgen05_fence<{s}>" }, + .{ nvvm.ops.Tcgen05LdStShapeAttr, "#nvvm.tcgen05_ldst_shape<{s}>" }, + .{ nvvm.ops.Tcgen05MMABlockScaleAttr, "#nvvm.tcgen05_mma_block_scale<{s}>" }, + .{ nvvm.ops.Tcgen05MMACollectorBBufferAttr, "#nvvm.tcgen05_mma_collectorb<{s}>" }, + .{ nvvm.ops.Tcgen05MMACollectorOpAttr, "#nvvm.tcgen05_mma_collectorop<{s}>" }, + .{ nvvm.ops.Tcgen05MMAKindAttr, "#nvvm.tcgen05_mma_kind<{s}>" }, + .{ nvvm.ops.Tcgen05WaitKindAttr, "#nvvm.tcgen05_wait<{s}>" }, + .{ nvvm.ops.TensormapElemtypeAttr, "#nvvm.tensormap_elemtype<{s}>" }, + .{ nvvm.ops.TensormapFieldAttr, "#nvvm" }, + .{ nvvm.ops.TensormapFillModeAttr, "#nvvm.tensormap_fill_mode<{s}>" }, + .{ nvvm.ops.TensormapInterleaveLayoutAttr, "#nvvm.tensormap_interleave_layout<{s}>" }, + .{ nvvm.ops.TensormapSwizzleAtomicityAttr, "#nvvm.tensormap_swizzle_atomicity<{s}>" }, + .{ nvvm.ops.TensormapSwizzleModeAttr, "#nvvm.tensormap_swizzle_mode<{s}>" }, + .{ nvvm.ops.TmemLayoutAttr, "#nvvm.TmemLayout<{s}>" }, + .{ nvvm.ops.ValidatePatternAttr, "#nvvm.validate_pattern<{s}>" }, + .{ nvvm.ops.VoteSyncKindAttr, "#nvvm" }, + .{ nvvm.ops.WGMMAScaleInAttr, "#nvvm.wgmma_scale_in<{s}>" }, + .{ nvvm.ops.WGMMAScaleOutAttr, "#nvvm.wgmma_scale_out<{s}>" }, + .{ nvvm.ops.WGMMATypesAttr, "#nvvm.wgmma_type<{s}>" }, + }) |example| { + const T = example[0]; + const E = @FieldType(T.InitArgs, "value"); + inline for (@typeInfo(E).@"enum".field_names) |name| { + const attr = try T.get(ctx, .{ .value = @field(E, name) }); + try std.testing.expectEqual(@field(E, name), attr.value()); + var buf: [256]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print("{f}", .{attr}); + try std.testing.expectEqualStrings(std.fmt.comptimePrint(example[1], .{name}), w.buffered()); + } + } +} + +fn testContext() !*mlir.Context { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + nvvm.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + return ctx; +} + +test "every bound operation is registered" { + const ctx = try testContext(); + defer ctx.deinit(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} + +test "nvvm types and attributes print as the compiler does" { + @setEvalBranchQuota(100_000); + const ctx = try testContext(); + defer ctx.deinit(); + // The compiler's printed forms, which these must print byte for byte. + inline for (.{ + .{ nvvm.ops.MMAShapeAttr, "#nvvm.shape" }, + .{ nvvm.ops.LdStMatrixShapeAttr, "#nvvm.ld_st_matrix_shape" }, + .{ nvvm.ops.TargetAttr, "#nvvm.target" }, + .{ nvvm.ops.TargetAttr, "#nvvm.target" }, + .{ nvvm.ops.TargetAttr, "#nvvm.target" }, + }) |example| { + const T = example[0]; + const is_type = @hasDecl(T, "type_"); + const parsed = if (is_type) try mlir.Type.parse(ctx, example[1]) else try mlir.Attribute.parse(ctx, example[1]); + const value = parsed.isA(T) orelse return error.TestUnexpectedResult; + try std.testing.expect((try rebuildEnums(T, ctx, value)).eql(value)); + var buf: [512]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print("{f}", .{value}); + try std.testing.expectEqualStrings(example[1], w.buffered()); + } +} + +test "reqntid and minctasm are kernel attributes of the nvvm dialect" { + const ctx = try testContext(); + defer ctx.deinit(); + var buf: [128]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print("{f}", .{mlir.Attribute.dict(ctx, &.{ nvvm.reqntid(ctx, .{ 128, 1, 1 }), nvvm.minctasm(ctx, 1) })}); + try std.testing.expectEqualStrings("{nvvm.minctasm = 1 : i32, nvvm.reqntid = array}", w.buffered()); +} + +test "nvvm target rejects what the compiler rejects" { + const ctx = try testContext(); + defer ctx.deinit(); + try std.testing.expectError(error.InvalidMlir, nvvm.ops.TargetAttr.get(ctx, .{ .O = 4 })); + try std.testing.expectError(error.InvalidMlir, nvvm.ops.TargetAttr.get(ctx, .{ .chip = "" })); + const not_strings = try mlir.Attribute.parse(ctx, "[\"a\", 2]"); + try std.testing.expectError(error.InvalidMlir, nvvm.ops.TargetAttr.get(ctx, .{ .link = not_strings })); +} + +/// `rebuild` for parameters that are slices of enums. +fn rebuildEnums(comptime T: type, ctx: *mlir.Context, value: *const T) !*const T { + var args: T.InitArgs = undefined; + var buffers: [4][16]u32 = undefined; + inline for (@typeInfo(T.InitArgs).@"struct".field_names, 0..) |field, i| { + const name = [_]u8{comptime std.ascii.toUpper(field[0])} ++ field[1..]; + const noun = [_]u8{comptime std.ascii.toLower(field[0])} ++ field[1..]; + const accessor = if (@hasDecl(T, noun ++ "_")) noun ++ "_" else noun; + const info = @typeInfo(@FieldType(T.InitArgs, field)); + if (info == .pointer and info.pointer.size == .slice and @typeInfo(info.pointer.child) == .@"enum") { + const E = info.pointer.child; + const count = @field(T, "num" ++ name)(value); + const elements: []E = @ptrCast(buffers[i][0..count]); + for (elements, 0..) |*element, pos| element.* = @field(T, noun[0 .. noun.len - 1])(value, pos); + @field(args, field) = elements; + } else { + @field(args, field) = @field(T, accessor)(value); + } + } + return T.get(ctx, args); +} diff --git a/mlir/dialects/rocdl/BUILD.bazel b/mlir/dialects/rocdl/BUILD.bazel new file mode 100644 index 00000000..ef0d7583 --- /dev/null +++ b/mlir/dialects/rocdl/BUILD.bazel @@ -0,0 +1,46 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +cc_library( + name = "capi", + srcs = ["capi.cc"], + hdrs = ["capi.h"], + tags = ["manual"], + deps = [ + "@llvm-project//mlir:CAPIIR", + "@llvm-project//mlir:CAPIROCDL", + "@llvm-project//mlir:ROCDLDialect", + ], +) + +zig_library( + name = "rocdl", + srcs = [ + "annotations.zig", + "attributes.zig", + "enums.zig", + "ops.zig", + "target.zig", + "types.zig", + ], + import_name = "mlir/dialects/rocdl", + main = "rocdl.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + ":capi", + "//mlir", + "//mlir/dialects/llvm", + "//stdx", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":capi", + ":rocdl", + "//mlir", + ], +) diff --git a/mlir/dialects/rocdl/README.md b/mlir/dialects/rocdl/README.md new file mode 100644 index 00000000..500e4657 --- /dev/null +++ b/mlir/dialects/rocdl/README.md @@ -0,0 +1,54 @@ +# ROCDL + +Depend on `//mlir/dialects/rocdl` and import `mlir/dialects/rocdl` for LLVM's +AMDGPU intrinsic dialect. The package links LLVM's `ROCDLDialect` and +`CAPIROCDL`. `fly_rocdl` is a separate dialect and package. + +The pinned LLVM revision supplies 374 operations, 15 attributes, and 12 enums. + +- `ops` provides every operation builder. Names replace dots with underscores, + such as `ops.raw_buffer_load` and `ops.mfma_f32_16x16x4f32`. Arguments are + operands, explicit result types, required attributes, an `opts` struct when + there are optional attributes, and the location, preceded by the MLIR context. + Attribute names match ODS. Optional and default-valued attributes are fields + of `opts` with `null` defaults; pass `.{}` to omit them and let MLIR supply + its defaults. Builders verify the resulting operation. +- Enum types and attribute wrappers are available at the module root and in + `enums`/`attributes`. Attribute constructors return `error.InvalidMlir` for + invalid enum values or flag bits. Bit enums support `combine` and `contains`. +- `TargetAttr` constructs and reads `#rocdl.target`, including optimization + level, triple, chip, features, ABI, flags, and link files. Defaults match LLVM. + Invalid fields and attributes from another context are rejected. +- `annotations` supplies function, module, and memory annotations. Apply + function annotations to `llvm.func`, and `xnack`/`sramecc` to LLVM-compatible + modules. A missing module option preserves the target default; `false` + explicitly disables the option. +- `types` supplies every builtin vector type ROCDL's TableGen records use, + named or inline, and LLVM pointers in ROCDL address spaces. ROCDL defines no independent + MLIR types. +- `registerDialect` registers ROCDL; loading it also loads its LLVM dependency. + The existing `cvtPkF32Fp8` convenience builder remains available. + +```zig +const rocdl = @import("mlir/dialects/rocdl"); + +const policy = try rocdl.Gfx942CachePolicyAttr.get(ctx, .{ .value = .combine(&.{ .sc0, .nt }) }); +const load = rocdl.ops.raw_buffer_load( + ctx, resource, offset, scalar_offset, + mlir.Type.float(ctx, .f32), .{ .aux = policy.attribute() }, location, +); +``` + +The `:capi` target exposes attribute constructors, predicates, and getters +through `capi.h`; dialect registration comes from upstream +`mlir-c/Dialect/ROCDL.h`. Declarations and implementations are grouped by +introspection (a test helper), attributes, and target configuration. Getters require a handle +accepted by the matching predicate. Contexts own all returned attributes and +strings. There are no additional C operation wrappers: Zig builders use MLIR's +standard operation API. + +Tests compare the generated operation list with the loaded dialect's registered +operation count, check every operation name, round-trip every named enum value, +reject invalid enum/target inputs, and verify and reparse representative barrier, +scheduling, buffer, and matrix operations in an LLVM function. The tests require +no AMD GPU or ROCm runtime. diff --git a/mlir/dialects/rocdl/REGENERATING.md b/mlir/dialects/rocdl/REGENERATING.md new file mode 100644 index 00000000..526cb17b --- /dev/null +++ b/mlir/dialects/rocdl/REGENERATING.md @@ -0,0 +1,62 @@ +# Regenerating the ROCDL bindings + +These are instructions for an agent. They describe what the generated ROCDL +bindings must contain after an LLVM bump, where to derive them from, and how to +check the result. Scripts and intermediate files are not committed; only the +generated bindings are. + +## Sources + +Use the LLVM revision Bazel pins, and read TableGen's resolved records rather +than the `.td` text: multiclasses instantiate many ROCDL operations. + +```sh +bazel build @llvm-project//llvm:llvm-tblgen +llvm_project="$(bazel info output_base)/external/+llvm+llvm-project" +bazel-bin/external/+llvm+llvm-project/llvm/llvm-tblgen --dump-json \ + -I "$llvm_project/mlir/include" -I "$llvm_project/llvm/include" \ + "$llvm_project/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td" \ + -o /tmp/rocdl-records.json +``` + +Every `Op`, `AttrDef` and `EnumInfo` record whose dialect is `ROCDL_Dialect` +belongs to the bindings. + +## What each file must contain + +Names, builder parameters, `opts`, wrappers, C API names and comments follow +`mlir/dialects/README.md`. Specific to ROCDL: + +- `ops.zig`: one builder per operation, listed in `test.zig`'s + `operation_names`. Follow argument constraints through `Arg` wrappers. +- `enums.zig`: one enum per ROCDL enum, with its bit width and values. Unlike + the README rule, a case whose IR spelling is not an identifier or is a + keyword takes the C++ symbol (`volatile` → `volatile_op`). Bit enums are + non-exhaustive and provide `combine` and `contains`. +- `attributes.zig`: one `Attr` wrapper per enum attribute, and + `enum_attributes` listing them. +- `types.zig`: `AddressSpace`, `pointerType`, and a `v` constructor + for every vector type the records use, named or inline. +- `capi.h`/`capi.cc`: `mlirROCDLGetOperationCount` under `// Introspection` + and, per enum attribute, the predicate, the getter (null for out-of-range + values) and `GetValue` under `// Attributes`. The `// Target` sections are + maintained by hand: in the + pinned revision `ROCDLTargetAttr::getChecked` delegates to `Base::get`, so the + C wrapper verifies target inputs first. + +Each generated Zig file starts with the "Generated from LLVM ROCDL TableGen +records" header. `target.zig`, `annotations.zig` and `rocdl.zig` are maintained +by hand; update their aliases and helpers when LLVM's surface changes. If the +dialect gains variadic operands, regions, successors, its own types, an enum +without an attribute, or new target fields or discardable attributes, extend +the bindings and this file. + +## Checks + +- Format and run the conformance checks of `mlir/dialects/README.md`. +- `bazel test //mlir/dialects/rocdl:test //dsl/fly:test //zml:test`. Beyond the + README's tests, the operation list is compared to + `mlirROCDLGetOperationCount`, and attribute tests exercise every named enum + case, named bit groups and combined bitfields. Add fixtures when a change + introduces a new operand layout or attribute representation. +- A regeneration without an LLVM change is byte-identical. diff --git a/mlir/dialects/rocdl/annotations.zig b/mlir/dialects/rocdl/annotations.zig new file mode 100644 index 00000000..85128473 --- /dev/null +++ b/mlir/dialects/rocdl/annotations.zig @@ -0,0 +1,56 @@ +const std = @import("std"); + +const mlir = @import("mlir"); + +/// Discardable ROCDL attributes for an `llvm.func` operation. +pub const FunctionOpts = struct { + kernel: bool = false, + reqdWorkGroupSize: ?[3]i32 = null, + flatWorkGroupSize: ?[]const u8 = null, + maxFlatWorkGroupSize: ?i32 = null, + wavesPerEu: ?WavesPerEu = null, + unsafeFpAtomics: ?bool = null, + + pub const WavesPerEu = struct { + min: u32, + max: ?u32 = null, + }; +}; + +pub fn annotateFunction(ctx: *mlir.Context, op: *mlir.Operation, opts: FunctionOpts) void { + if (opts.kernel) op.setAttributeByName("rocdl.kernel", .unit(ctx)); + if (opts.reqdWorkGroupSize) |size| op.setAttributeByName("rocdl.reqd_work_group_size", .denseArray(ctx, .i32, &size)); + if (opts.flatWorkGroupSize) |size| op.setAttributeByName("rocdl.flat_work_group_size", .string(ctx, size)); + if (opts.maxFlatWorkGroupSize) |size| op.setAttributeByName("rocdl.max_flat_work_group_size", .int(ctx, .i32, size)); + if (opts.wavesPerEu) |waves| { + var buffer: [24]u8 = undefined; + const value = if (waves.max) |max| std.fmt.bufPrint(&buffer, "{d},{d}", .{ waves.min, max }) else std.fmt.bufPrint(&buffer, "{d}", .{waves.min}); + op.setAttributeByName("rocdl.waves_per_eu", .string(ctx, value catch unreachable)); + } + if (opts.unsafeFpAtomics) |enabled| op.setAttributeByName("rocdl.unsafe_fp_atomics", .boolean(ctx, enabled)); +} + +/// A missing flag preserves the target's default; false explicitly disables it. +pub const ModuleOpts = struct { + xnack: ?bool = null, + sramecc: ?bool = null, +}; + +pub fn annotateModule(ctx: *mlir.Context, op: *mlir.Operation, opts: ModuleOpts) void { + if (opts.xnack) |enabled| op.setAttributeByName("rocdl.xnack", .boolean(ctx, enabled)); + if (opts.sramecc) |enabled| op.setAttributeByName("rocdl.sramecc", .boolean(ctx, enabled)); +} + +pub const MemoryOpts = struct { + lastUse: bool = false, + noRemoteMemory: bool = false, + noFineGrainedMemory: bool = false, + ignoreDenormalMode: bool = false, +}; + +pub fn annotateMemory(ctx: *mlir.Context, op: *mlir.Operation, opts: MemoryOpts) void { + if (opts.lastUse) op.setAttributeByName("rocdl.last_use", .unit(ctx)); + if (opts.noRemoteMemory) op.setAttributeByName("rocdl.no_remote_memory", .unit(ctx)); + if (opts.noFineGrainedMemory) op.setAttributeByName("rocdl.no_fine_grained_memory", .unit(ctx)); + if (opts.ignoreDenormalMode) op.setAttributeByName("rocdl.ignore_denormal_mode", .unit(ctx)); +} diff --git a/mlir/dialects/rocdl/attributes.zig b/mlir/dialects/rocdl/attributes.zig new file mode 100644 index 00000000..9e1b9863 --- /dev/null +++ b/mlir/dialects/rocdl/attributes.zig @@ -0,0 +1,417 @@ +// Generated from LLVM ROCDL TableGen records; see REGENERATING.md. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +const c = @import("c"); +const mlir = @import("mlir"); +const enums = @import("enums.zig"); + +pub const TargetAttr = @import("target.zig").TargetAttr; + +/// `#rocdl.buffer_oob_mode`. +pub const BufferOOBModeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const Value = enums.BufferOOBMode; + pub const isAFn = c.mlirAttributeIsAROCDLBufferOOBMode; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + value: Value, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirROCDLBufferOOBModeAttrGet(ctx.ptr(), @backingInt(args.value)).ptr orelse return error.InvalidMlir); + } + + pub fn value(self: *const Self) Value { + return @fromBackingInt(@intCast(c.mlirROCDLBufferOOBModeAttrGetValue(self.ptr()))); + } + + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +/// `#rocdl.buffer_oob_mode_flag`. +pub const BufferOOBModeModuleFlagAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const Value = enums.BufferOOBMode; + pub const isAFn = c.mlirAttributeIsAROCDLBufferOOBModeModuleFlag; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + value: Value, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirROCDLBufferOOBModeModuleFlagAttrGet(ctx.ptr(), @backingInt(args.value)).ptr orelse return error.InvalidMlir); + } + + pub fn value(self: *const Self) Value { + return @fromBackingInt(@intCast(c.mlirROCDLBufferOOBModeModuleFlagAttrGetValue(self.ptr()))); + } + + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +/// `#rocdl.gfx12_atomic_cache_policy`. +pub const Gfx12AtomicCachePolicyAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const Value = enums.Gfx12AtomicCachePolicy; + pub const isAFn = c.mlirAttributeIsAROCDLGfx12AtomicCachePolicy; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + value: Value, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirROCDLGfx12AtomicCachePolicyAttrGet(ctx.ptr(), @backingInt(args.value)).ptr orelse return error.InvalidMlir); + } + + pub fn value(self: *const Self) Value { + return @fromBackingInt(@intCast(c.mlirROCDLGfx12AtomicCachePolicyAttrGetValue(self.ptr()))); + } + + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +/// `#rocdl.gfx12_cache_policy`. +pub const Gfx12CachePolicyAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const Value = enums.Gfx12CachePolicy; + pub const isAFn = c.mlirAttributeIsAROCDLGfx12CachePolicy; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + value: Value, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirROCDLGfx12CachePolicyAttrGet(ctx.ptr(), @backingInt(args.value)).ptr orelse return error.InvalidMlir); + } + + pub fn value(self: *const Self) Value { + return @fromBackingInt(@intCast(c.mlirROCDLGfx12CachePolicyAttrGetValue(self.ptr()))); + } + + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +/// `#rocdl.gfx942_cache_policy`. +pub const Gfx942CachePolicyAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const Value = enums.Gfx942CachePolicy; + pub const isAFn = c.mlirAttributeIsAROCDLGfx942CachePolicy; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + value: Value, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirROCDLGfx942CachePolicyAttrGet(ctx.ptr(), @backingInt(args.value)).ptr orelse return error.InvalidMlir); + } + + pub fn value(self: *const Self) Value { + return @fromBackingInt(@intCast(c.mlirROCDLGfx942CachePolicyAttrGetValue(self.ptr()))); + } + + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +/// `#rocdl.mfma_neg_modifier`. +pub const MFMANegModifierAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const Value = enums.MFMANegModifier; + pub const isAFn = c.mlirAttributeIsAROCDLMFMANegModifier; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + value: Value, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirROCDLMFMANegModifierAttrGet(ctx.ptr(), @backingInt(args.value)).ptr orelse return error.InvalidMlir); + } + + pub fn value(self: *const Self) Value { + return @fromBackingInt(@intCast(c.mlirROCDLMFMANegModifierAttrGetValue(self.ptr()))); + } + + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +/// `#rocdl.mfma_perm_b`. +pub const MFMAPermBAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const Value = enums.MFMAPermB; + pub const isAFn = c.mlirAttributeIsAROCDLMFMAPermB; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + value: Value, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirROCDLMFMAPermBAttrGet(ctx.ptr(), @backingInt(args.value)).ptr orelse return error.InvalidMlir); + } + + pub fn value(self: *const Self) Value { + return @fromBackingInt(@intCast(c.mlirROCDLMFMAPermBAttrGetValue(self.ptr()))); + } + + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +/// `#rocdl.matrix_format`. +pub const MatrixFormatAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const Value = enums.MatrixFormat; + pub const isAFn = c.mlirAttributeIsAROCDLMatrixFormat; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + value: Value, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirROCDLMatrixFormatAttrGet(ctx.ptr(), @backingInt(args.value)).ptr orelse return error.InvalidMlir); + } + + pub fn value(self: *const Self) Value { + return @fromBackingInt(@intCast(c.mlirROCDLMatrixFormatAttrGetValue(self.ptr()))); + } + + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +/// `#rocdl.pre_gfx12_cache_policy`. +pub const PreGfx12CachePolicyAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const Value = enums.PreGfx12CachePolicy; + pub const isAFn = c.mlirAttributeIsAROCDLPreGfx12CachePolicy; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + value: Value, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirROCDLPreGfx12CachePolicyAttrGet(ctx.ptr(), @backingInt(args.value)).ptr orelse return error.InvalidMlir); + } + + pub fn value(self: *const Self) Value { + return @fromBackingInt(@intCast(c.mlirROCDLPreGfx12CachePolicyAttrGetValue(self.ptr()))); + } + + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +/// `#rocdl.sched_group_mask`. +pub const SchedGroupMaskAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const Value = enums.SchedGroupMask; + pub const isAFn = c.mlirAttributeIsAROCDLSchedGroupMask; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + value: Value, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirROCDLSchedGroupMaskAttrGet(ctx.ptr(), @backingInt(args.value)).ptr orelse return error.InvalidMlir); + } + + pub fn value(self: *const Self) Value { + return @fromBackingInt(@intCast(c.mlirROCDLSchedGroupMaskAttrGetValue(self.ptr()))); + } + + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +/// `#rocdl.tbuffer_oob_mode_flag`. +pub const TBufferOOBModeModuleFlagAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const Value = enums.BufferOOBMode; + pub const isAFn = c.mlirAttributeIsAROCDLTBufferOOBModeModuleFlag; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + value: Value, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirROCDLTBufferOOBModeModuleFlagAttrGet(ctx.ptr(), @backingInt(args.value)).ptr orelse return error.InvalidMlir); + } + + pub fn value(self: *const Self) Value { + return @fromBackingInt(@intCast(c.mlirROCDLTBufferOOBModeModuleFlagAttrGetValue(self.ptr()))); + } + + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +/// `#rocdl.wmma_c_modifier`. +pub const WMMACModifierAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const Value = enums.WMMACModifier; + pub const isAFn = c.mlirAttributeIsAROCDLWMMACModifier; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + value: Value, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirROCDLWMMACModifierAttrGet(ctx.ptr(), @backingInt(args.value)).ptr orelse return error.InvalidMlir); + } + + pub fn value(self: *const Self) Value { + return @fromBackingInt(@intCast(c.mlirROCDLWMMACModifierAttrGetValue(self.ptr()))); + } + + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +/// `#rocdl.wmma_matrix_scale`. +pub const WMMAMatrixScaleAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const Value = enums.WMMAMatrixScale; + pub const isAFn = c.mlirAttributeIsAROCDLWMMAMatrixScale; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + value: Value, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirROCDLWMMAMatrixScaleAttrGet(ctx.ptr(), @backingInt(args.value)).ptr orelse return error.InvalidMlir); + } + + pub fn value(self: *const Self) Value { + return @fromBackingInt(@intCast(c.mlirROCDLWMMAMatrixScaleAttrGetValue(self.ptr()))); + } + + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +/// `#rocdl.wmma_matrix_scale_format`. +pub const WMMAMatrixScaleFormatAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const Value = enums.WMMAMatrixScaleFormat; + pub const isAFn = c.mlirAttributeIsAROCDLWMMAMatrixScaleFormat; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + value: Value, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirROCDLWMMAMatrixScaleFormatAttrGet(ctx.ptr(), @backingInt(args.value)).ptr orelse return error.InvalidMlir); + } + + pub fn value(self: *const Self) Value { + return @fromBackingInt(@intCast(c.mlirROCDLWMMAMatrixScaleFormatAttrGetValue(self.ptr()))); + } + + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +pub const enum_attributes = .{ + BufferOOBModeAttr, + BufferOOBModeModuleFlagAttr, + Gfx12AtomicCachePolicyAttr, + Gfx12CachePolicyAttr, + Gfx942CachePolicyAttr, + MFMANegModifierAttr, + MFMAPermBAttr, + MatrixFormatAttr, + PreGfx12CachePolicyAttr, + SchedGroupMaskAttr, + TBufferOOBModeModuleFlagAttr, + WMMACModifierAttr, + WMMAMatrixScaleAttr, + WMMAMatrixScaleFormatAttr, +}; diff --git a/mlir/dialects/rocdl/capi.cc b/mlir/dialects/rocdl/capi.cc new file mode 100644 index 00000000..7a01bd56 --- /dev/null +++ b/mlir/dialects/rocdl/capi.cc @@ -0,0 +1,440 @@ +//===- capi.cc - C API for the rocdl dialect ------------------------------===// + +#include "capi.h" + +#include "mlir/CAPI/IR.h" +#include "mlir/CAPI/Support.h" +#include "mlir/Dialect/LLVMIR/ROCDLDialect.h" +#include "mlir/IR/Diagnostics.h" + +// Introspection + +intptr_t mlirROCDLGetOperationCount(MlirContext context) { + auto *ctx = unwrap(context); + if (!ctx) + return 0; + ctx->getOrLoadDialect(); + intptr_t count = 0; + for (auto op : ctx->getRegisteredOperations()) + if (op.getDialectNamespace() == "rocdl") + ++count; + return count; +} + +// Attributes + +bool mlirAttributeIsAROCDLBufferOOBMode(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirROCDLBufferOOBModeAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + auto symbol = + mlir::ROCDL::symbolizeBufferOOBMode(static_cast(value)); + if (!symbol) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(mlir::ROCDL::BufferOOBModeAttr::get(ctx, *symbol)); +} + +uint32_t mlirROCDLBufferOOBModeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsAROCDLBufferOOBModeModuleFlag(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} + +MlirAttribute mlirROCDLBufferOOBModeModuleFlagAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + auto symbol = + mlir::ROCDL::symbolizeBufferOOBMode(static_cast(value)); + if (!symbol) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(mlir::ROCDL::BufferOOBModeModuleFlagAttr::get(ctx, *symbol)); +} + +uint32_t mlirROCDLBufferOOBModeModuleFlagAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsAROCDLGfx12AtomicCachePolicy(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} + +MlirAttribute mlirROCDLGfx12AtomicCachePolicyAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + auto symbol = mlir::ROCDL::symbolizeGfx12AtomicCachePolicy( + static_cast(value)); + if (!symbol) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(mlir::ROCDL::Gfx12AtomicCachePolicyAttr::get(ctx, *symbol)); +} + +uint32_t mlirROCDLGfx12AtomicCachePolicyAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsAROCDLGfx12CachePolicy(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirROCDLGfx12CachePolicyAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + auto symbol = + mlir::ROCDL::symbolizeGfx12CachePolicy(static_cast(value)); + if (!symbol) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(mlir::ROCDL::Gfx12CachePolicyAttr::get(ctx, *symbol)); +} + +uint32_t mlirROCDLGfx12CachePolicyAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsAROCDLGfx942CachePolicy(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} + +MlirAttribute mlirROCDLGfx942CachePolicyAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + auto symbol = + mlir::ROCDL::symbolizeGfx942CachePolicy(static_cast(value)); + if (!symbol) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(mlir::ROCDL::Gfx942CachePolicyAttr::get(ctx, *symbol)); +} + +uint32_t mlirROCDLGfx942CachePolicyAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsAROCDLMFMANegModifier(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirROCDLMFMANegModifierAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + auto symbol = + mlir::ROCDL::symbolizeMFMANegModifier(static_cast(value)); + if (!symbol) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(mlir::ROCDL::MFMANegModifierAttr::get(ctx, *symbol)); +} + +uint32_t mlirROCDLMFMANegModifierAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsAROCDLMFMAPermB(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirROCDLMFMAPermBAttrGet(MlirContext context, uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + auto symbol = mlir::ROCDL::symbolizeMFMAPermB(static_cast(value)); + if (!symbol) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(mlir::ROCDL::MFMAPermBAttr::get(ctx, *symbol)); +} + +uint32_t mlirROCDLMFMAPermBAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsAROCDLMatrixFormat(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirROCDLMatrixFormatAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + auto symbol = + mlir::ROCDL::symbolizeMatrixFormat(static_cast(value)); + if (!symbol) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(mlir::ROCDL::MatrixFormatAttr::get(ctx, *symbol)); +} + +uint32_t mlirROCDLMatrixFormatAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsAROCDLPreGfx12CachePolicy(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} + +MlirAttribute mlirROCDLPreGfx12CachePolicyAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + auto symbol = + mlir::ROCDL::symbolizePreGfx12CachePolicy(static_cast(value)); + if (!symbol) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(mlir::ROCDL::PreGfx12CachePolicyAttr::get(ctx, *symbol)); +} + +uint32_t mlirROCDLPreGfx12CachePolicyAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsAROCDLSchedGroupMask(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirROCDLSchedGroupMaskAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + auto symbol = + mlir::ROCDL::symbolizeSchedGroupMask(static_cast(value)); + if (!symbol) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(mlir::ROCDL::SchedGroupMaskAttr::get(ctx, *symbol)); +} + +uint32_t mlirROCDLSchedGroupMaskAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsAROCDLTBufferOOBModeModuleFlag(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} + +MlirAttribute mlirROCDLTBufferOOBModeModuleFlagAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + auto symbol = + mlir::ROCDL::symbolizeBufferOOBMode(static_cast(value)); + if (!symbol) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(mlir::ROCDL::TBufferOOBModeModuleFlagAttr::get(ctx, *symbol)); +} + +uint32_t mlirROCDLTBufferOOBModeModuleFlagAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +bool mlirAttributeIsAROCDLWMMACModifier(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirROCDLWMMACModifierAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + if (value > 65535) + return {nullptr}; + auto symbol = + mlir::ROCDL::symbolizeWMMACModifier(static_cast(value)); + if (!symbol) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(mlir::ROCDL::WMMACModifierAttr::get(ctx, *symbol)); +} + +uint32_t mlirROCDLWMMACModifierAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsAROCDLWMMAMatrixScale(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirROCDLWMMAMatrixScaleAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + auto symbol = + mlir::ROCDL::symbolizeWMMAMatrixScale(static_cast(value)); + if (!symbol) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(mlir::ROCDL::WMMAMatrixScaleAttr::get(ctx, *symbol)); +} + +uint32_t mlirROCDLWMMAMatrixScaleAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsAROCDLWMMAMatrixScaleFormat(MlirAttribute attr) { + return llvm::isa_and_nonnull( + unwrap(attr)); +} + +MlirAttribute mlirROCDLWMMAMatrixScaleFormatAttrGet(MlirContext context, + uint32_t value) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + auto symbol = + mlir::ROCDL::symbolizeWMMAMatrixScaleFormat(static_cast(value)); + if (!symbol) + return {nullptr}; + ctx->getOrLoadDialect(); + return wrap(mlir::ROCDL::WMMAMatrixScaleFormatAttr::get(ctx, *symbol)); +} + +uint32_t mlirROCDLWMMAMatrixScaleFormatAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)) + .getValue()); +} + +// Target + +bool mlirAttributeIsAROCDLTarget(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirROCDLTargetAttrGet(MlirContext context, int optLevel, + MlirStringRef triple, MlirStringRef chip, + MlirStringRef features, MlirStringRef abi, + MlirAttribute flags, MlirAttribute link) { + auto *ctx = unwrap(context); + if (!ctx) + return {nullptr}; + for (auto s : {triple, chip, features, abi}) + if (!s.data && s.length) + return {nullptr}; + auto flagsValue = + llvm::dyn_cast_if_present(unwrap(flags)); + auto linkValue = llvm::dyn_cast_if_present(unwrap(link)); + if (unwrap(flags) && (!flagsValue || flagsValue.getContext() != ctx)) + return {nullptr}; + if (unwrap(link) && (!linkValue || linkValue.getContext() != ctx)) + return {nullptr}; + ctx->getOrLoadDialect(); + // The upstream custom getChecked builder delegates to unchecked Base::get. + // Verify first so invalid C API inputs return null instead of asserting. + if (mlir::failed(mlir::ROCDL::ROCDLTargetAttr::verifyInvariants( + [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, optLevel, + unwrap(triple), unwrap(chip), unwrap(features), unwrap(abi), + flagsValue, linkValue))) + return {nullptr}; + return wrap(mlir::ROCDL::ROCDLTargetAttr::get( + ctx, optLevel, unwrap(triple), unwrap(chip), unwrap(features), + unwrap(abi), flagsValue, linkValue)); +} + +int mlirROCDLTargetAttrGetOptLevel(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).getO(); +} + +MlirStringRef mlirROCDLTargetAttrGetTriple(MlirAttribute attr) { + return wrap( + llvm::cast(unwrap(attr)).getTriple()); +} + +MlirStringRef mlirROCDLTargetAttrGetChip(MlirAttribute attr) { + return wrap(llvm::cast(unwrap(attr)).getChip()); +} + +MlirStringRef mlirROCDLTargetAttrGetFeatures(MlirAttribute attr) { + return wrap( + llvm::cast(unwrap(attr)).getFeatures()); +} + +MlirStringRef mlirROCDLTargetAttrGetAbi(MlirAttribute attr) { + return wrap(llvm::cast(unwrap(attr)).getAbi()); +} + +MlirAttribute mlirROCDLTargetAttrGetFlags(MlirAttribute attr) { + return wrap( + llvm::cast(unwrap(attr)).getFlags()); +} + +MlirAttribute mlirROCDLTargetAttrGetLink(MlirAttribute attr) { + return wrap(llvm::cast(unwrap(attr)).getLink()); +} + +bool mlirROCDLTargetAttrHasFlag(MlirAttribute attr, MlirStringRef flag) { + return llvm::cast(unwrap(attr)) + .hasFlag(unwrap(flag)); +} + +bool mlirROCDLTargetAttrHasWave64(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).hasWave64(); +} + +bool mlirROCDLTargetAttrHasFastMath(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).hasFastMath(); +} + +bool mlirROCDLTargetAttrHasDaz(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).hasDaz(); +} + +bool mlirROCDLTargetAttrHasFiniteOnly(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).hasFiniteOnly(); +} + +bool mlirROCDLTargetAttrHasUnsafeMath(MlirAttribute attr) { + return llvm::cast(unwrap(attr)).hasUnsafeMath(); +} + +bool mlirROCDLTargetAttrHasCorrectSqrt(MlirAttribute attr) { + return llvm::cast(unwrap(attr)) + .hasCorrectSqrt(); +} diff --git a/mlir/dialects/rocdl/capi.h b/mlir/dialects/rocdl/capi.h new file mode 100644 index 00000000..b73a948d --- /dev/null +++ b/mlir/dialects/rocdl/capi.h @@ -0,0 +1,140 @@ +//===- capi.h - C API for the rocdl dialect -----------------------*- C -*-===// + +#ifndef MLIR_DIALECTS_ROCDL_CAPI_H +#define MLIR_DIALECTS_ROCDL_CAPI_H + +#include "mlir-c/Dialect/ROCDL.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Introspection + +MLIR_CAPI_EXPORTED intptr_t mlirROCDLGetOperationCount(MlirContext context); + +// Attributes + +MLIR_CAPI_EXPORTED bool mlirAttributeIsAROCDLBufferOOBMode(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirROCDLBufferOOBModeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirROCDLBufferOOBModeAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED bool +mlirAttributeIsAROCDLBufferOOBModeModuleFlag(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirROCDLBufferOOBModeModuleFlagAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirROCDLBufferOOBModeModuleFlagAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED bool +mlirAttributeIsAROCDLGfx12AtomicCachePolicy(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirROCDLGfx12AtomicCachePolicyAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirROCDLGfx12AtomicCachePolicyAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED bool +mlirAttributeIsAROCDLGfx12CachePolicy(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirROCDLGfx12CachePolicyAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirROCDLGfx12CachePolicyAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED bool +mlirAttributeIsAROCDLGfx942CachePolicy(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirROCDLGfx942CachePolicyAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirROCDLGfx942CachePolicyAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED bool +mlirAttributeIsAROCDLMFMANegModifier(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirROCDLMFMANegModifierAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirROCDLMFMANegModifierAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED bool mlirAttributeIsAROCDLMFMAPermB(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirROCDLMFMAPermBAttrGet(MlirContext context, + uint32_t value); +MLIR_CAPI_EXPORTED uint32_t mlirROCDLMFMAPermBAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED bool mlirAttributeIsAROCDLMatrixFormat(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirROCDLMatrixFormatAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirROCDLMatrixFormatAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED bool +mlirAttributeIsAROCDLPreGfx12CachePolicy(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirROCDLPreGfx12CachePolicyAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirROCDLPreGfx12CachePolicyAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED bool mlirAttributeIsAROCDLSchedGroupMask(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirROCDLSchedGroupMaskAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirROCDLSchedGroupMaskAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED bool +mlirAttributeIsAROCDLTBufferOOBModeModuleFlag(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirROCDLTBufferOOBModeModuleFlagAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirROCDLTBufferOOBModeModuleFlagAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED bool mlirAttributeIsAROCDLWMMACModifier(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirROCDLWMMACModifierAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirROCDLWMMACModifierAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED bool +mlirAttributeIsAROCDLWMMAMatrixScale(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirROCDLWMMAMatrixScaleAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirROCDLWMMAMatrixScaleAttrGetValue(MlirAttribute attr); + +MLIR_CAPI_EXPORTED bool +mlirAttributeIsAROCDLWMMAMatrixScaleFormat(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirROCDLWMMAMatrixScaleFormatAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirROCDLWMMAMatrixScaleFormatAttrGetValue(MlirAttribute attr); + +// Target + +MLIR_CAPI_EXPORTED bool mlirAttributeIsAROCDLTarget(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirROCDLTargetAttrGet( + MlirContext context, int optLevel, MlirStringRef triple, MlirStringRef chip, + MlirStringRef features, MlirStringRef abi, MlirAttribute flags, + MlirAttribute link); +MLIR_CAPI_EXPORTED int mlirROCDLTargetAttrGetOptLevel(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef +mlirROCDLTargetAttrGetTriple(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef mlirROCDLTargetAttrGetChip(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef +mlirROCDLTargetAttrGetFeatures(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirStringRef mlirROCDLTargetAttrGetAbi(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirROCDLTargetAttrGetFlags(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute mlirROCDLTargetAttrGetLink(MlirAttribute attr); +MLIR_CAPI_EXPORTED bool mlirROCDLTargetAttrHasFlag(MlirAttribute attr, + MlirStringRef flag); +MLIR_CAPI_EXPORTED bool mlirROCDLTargetAttrHasWave64(MlirAttribute attr); +MLIR_CAPI_EXPORTED bool mlirROCDLTargetAttrHasFastMath(MlirAttribute attr); +MLIR_CAPI_EXPORTED bool mlirROCDLTargetAttrHasDaz(MlirAttribute attr); +MLIR_CAPI_EXPORTED bool mlirROCDLTargetAttrHasFiniteOnly(MlirAttribute attr); +MLIR_CAPI_EXPORTED bool mlirROCDLTargetAttrHasUnsafeMath(MlirAttribute attr); +MLIR_CAPI_EXPORTED bool mlirROCDLTargetAttrHasCorrectSqrt(MlirAttribute attr); + +#ifdef __cplusplus +} +#endif + +#endif // MLIR_DIALECTS_ROCDL_CAPI_H diff --git a/mlir/dialects/rocdl/enums.zig b/mlir/dialects/rocdl/enums.zig new file mode 100644 index 00000000..fde24ff7 --- /dev/null +++ b/mlir/dialects/rocdl/enums.zig @@ -0,0 +1,208 @@ +// Generated from LLVM ROCDL TableGen records; see REGENERATING.md. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +/// ROCDL buffer out-of-bounds mode +pub const BufferOOBMode = enum(u32) { + any = 0, + relaxed = 1, + strict = 2, +}; + +/// gfx12 atomic cache policy bits +pub const Gfx12AtomicCachePolicy = enum(u32) { + none = 0, + keep_return = 1, + nt = 2, + cascade = 4, + scope_se = 8, + scope_dev = 16, + scope_sys = 24, + nv = 32, + volatile_op = 2147483648, + _, + + pub fn combine(values: []const Gfx12AtomicCachePolicy) Gfx12AtomicCachePolicy { + var bits: u32 = 0; + for (values) |value| bits |= @backingInt(value); + return @fromBackingInt(bits); + } + + pub fn contains(self: Gfx12AtomicCachePolicy, value: Gfx12AtomicCachePolicy) bool { + return @backingInt(self) & @backingInt(value) == @backingInt(value); + } +}; + +/// gfx12 cache policy bits +pub const Gfx12CachePolicy = enum(u32) { + none = 0, + nt = 1, + ht = 2, + lu = 3, + nt_rt = 4, + rt_nt = 5, + nt_ht = 6, + nt_wb = 7, + scope_se = 8, + scope_dev = 16, + scope_sys = 24, + nv = 32, + swz = 64, + scal = 2048, + volatile_op = 2147483648, + _, + + pub fn combine(values: []const Gfx12CachePolicy) Gfx12CachePolicy { + var bits: u32 = 0; + for (values) |value| bits |= @backingInt(value); + return @fromBackingInt(bits); + } + + pub fn contains(self: Gfx12CachePolicy, value: Gfx12CachePolicy) bool { + return @backingInt(self) & @backingInt(value) == @backingInt(value); + } +}; + +/// gfx942 buffer cache policy bits +pub const Gfx942CachePolicy = enum(u32) { + none = 0, + sc0 = 1, + nt = 2, + swz = 8, + sc1 = 16, + volatile_op = 2147483648, + _, + + pub fn combine(values: []const Gfx942CachePolicy) Gfx942CachePolicy { + var bits: u32 = 0; + for (values) |value| bits |= @backingInt(value); + return @fromBackingInt(bits); + } + + pub fn contains(self: Gfx942CachePolicy, value: Gfx942CachePolicy) bool { + return @backingInt(self) & @backingInt(value) == @backingInt(value); + } +}; + +/// negation modifier bitfield for gfx94x double-precision MFMA +pub const MFMANegModifier = enum(u32) { + none = 0, + neg_a = 1, + neg_b = 2, + neg_c = 4, + _, + + pub fn combine(values: []const MFMANegModifier) MFMANegModifier { + var bits: u32 = 0; + for (values) |value| bits |= @backingInt(value); + return @fromBackingInt(bits); + } + + pub fn contains(self: MFMANegModifier, value: MFMANegModifier) bool { + return @backingInt(self) & @backingInt(value) == @backingInt(value); + } +}; + +/// permutations of the lanes storing B in an MFMA +pub const MFMAPermB = enum(u32) { + none = 0, + bcast_first_32 = 1, + bcast_second_32 = 2, + rotate_16_right = 3, + bcast_first_16 = 4, + bcast_second_16 = 5, + bcast_third_16 = 6, + bcast_fourth_16 = 7, +}; + +/// matrix operand formats selected by scaled MFMA/WMMA format fields +pub const MatrixFormat = enum(u32) { + fp8_e4m3 = 0, + fp8_e5m2 = 1, + fp6_e2m3 = 2, + fp6_e3m2 = 3, + fp4_e2m1 = 4, +}; + +/// pre-gfx12 buffer cache policy bits +pub const PreGfx12CachePolicy = enum(u32) { + none = 0, + glc = 1, + slc = 2, + dlc = 4, + swz = 8, + scc = 16, + all = 23, + volatile_op = 2147483648, + _, + + pub fn combine(values: []const PreGfx12CachePolicy) PreGfx12CachePolicy { + var bits: u32 = 0; + for (values) |value| bits |= @backingInt(value); + return @fromBackingInt(bits); + } + + pub fn contains(self: PreGfx12CachePolicy, value: PreGfx12CachePolicy) bool { + return @backingInt(self) & @backingInt(value) == @backingInt(value); + } +}; + +/// instruction type mask for scheduling barriers +pub const SchedGroupMask = enum(u32) { + none = 0, + non_mem_non_sideeffect = 1, + valu = 2, + salu = 4, + mfma_wmma = 8, + all_vmem = 16, + vmem_read = 32, + vmem_write = 64, + all_ds = 128, + ds_read = 256, + ds_write = 512, + transcendental = 1024, + ldsdma = 2048, + all = 4095, + _, + + pub fn combine(values: []const SchedGroupMask) SchedGroupMask { + var bits: u32 = 0; + for (values) |value| bits |= @backingInt(value); + return @fromBackingInt(bits); + } + + pub fn contains(self: SchedGroupMask, value: SchedGroupMask) bool { + return @backingInt(self) & @backingInt(value) == @backingInt(value); + } +}; + +/// WMMA C operand modifiers +pub const WMMACModifier = enum(u16) { + none = 0, + neg = 1, + abs = 2, + neg_abs = 3, + _, + + pub fn combine(values: []const WMMACModifier) WMMACModifier { + var bits: u16 = 0; + for (values) |value| bits |= @backingInt(value); + return @fromBackingInt(bits); + } + + pub fn contains(self: WMMACModifier, value: WMMACModifier) bool { + return @backingInt(self) & @backingInt(value) == @backingInt(value); + } +}; + +/// matrix scale row selector +pub const WMMAMatrixScale = enum(u32) { + row0 = 0, + row1 = 1, +}; + +/// matrix scale exponent formats +pub const WMMAMatrixScaleFormat = enum(u32) { + e8 = 0, + e5m3 = 1, + e4m3 = 2, +}; diff --git a/mlir/dialects/rocdl/ops.zig b/mlir/dialects/rocdl/ops.zig new file mode 100644 index 00000000..b18a33cf --- /dev/null +++ b/mlir/dialects/rocdl/ops.zig @@ -0,0 +1,8600 @@ +// Generated from LLVM ROCDL TableGen records; see REGENERATING.md. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +/// `rocdl.asyncmark`. Mark the end of a group of asynchronous operations +pub fn asyncmark( + ctx: *mlir.Context, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.asyncmark", .{ + .location = location, + }); +} + +/// `rocdl.ballot`. Vote across thread group +pub fn ballot( + ctx: *mlir.Context, + pred: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.ballot", .{ + .operands = .{ .flat = &.{pred} }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.barrier`. +pub fn barrier( + ctx: *mlir.Context, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.barrier", .{ + .location = location, + }); +} + +/// `rocdl.cluster.id.x`. +pub fn cluster_id_x( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "range", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.cluster.id.x", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cluster.id.y`. +pub fn cluster_id_y( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "range", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.cluster.id.y", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cluster.id.z`. +pub fn cluster_id_z( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "range", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.cluster.id.z", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cluster.load.async.to.lds.b128`. +pub fn cluster_load_async_to_lds_b128( + ctx: *mlir.Context, + globalPtr: *const mlir.Value, + ldsPtr: *const mlir.Value, + mask: *const mlir.Value, + offset: *const mlir.Attribute, + opts: struct { + cpol: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "offset", offset)); + if (opts.cpol) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "cpol", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.cluster.load.async.to.lds.b128", .{ + .operands = .{ .flat = &.{ globalPtr, ldsPtr, mask } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cluster.load.async.to.lds.b32`. +pub fn cluster_load_async_to_lds_b32( + ctx: *mlir.Context, + globalPtr: *const mlir.Value, + ldsPtr: *const mlir.Value, + mask: *const mlir.Value, + offset: *const mlir.Attribute, + opts: struct { + cpol: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "offset", offset)); + if (opts.cpol) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "cpol", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.cluster.load.async.to.lds.b32", .{ + .operands = .{ .flat = &.{ globalPtr, ldsPtr, mask } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cluster.load.async.to.lds.b64`. +pub fn cluster_load_async_to_lds_b64( + ctx: *mlir.Context, + globalPtr: *const mlir.Value, + ldsPtr: *const mlir.Value, + mask: *const mlir.Value, + offset: *const mlir.Attribute, + opts: struct { + cpol: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "offset", offset)); + if (opts.cpol) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "cpol", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.cluster.load.async.to.lds.b64", .{ + .operands = .{ .flat = &.{ globalPtr, ldsPtr, mask } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cluster.load.async.to.lds.b8`. +pub fn cluster_load_async_to_lds_b8( + ctx: *mlir.Context, + globalPtr: *const mlir.Value, + ldsPtr: *const mlir.Value, + mask: *const mlir.Value, + offset: *const mlir.Attribute, + opts: struct { + cpol: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "offset", offset)); + if (opts.cpol) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "cpol", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.cluster.load.async.to.lds.b8", .{ + .operands = .{ .flat = &.{ globalPtr, ldsPtr, mask } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cluster.workgroup.id.x`. +pub fn cluster_workgroup_id_x( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "range", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.cluster.workgroup.id.x", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cluster.workgroup.id.y`. +pub fn cluster_workgroup_id_y( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "range", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.cluster.workgroup.id.y", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cluster.workgroup.id.z`. +pub fn cluster_workgroup_id_z( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "range", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.cluster.workgroup.id.z", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cos`. +pub fn cos( + ctx: *mlir.Context, + arg: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cos", .{ + .operands = .{ .flat = &.{arg} }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.f32.bf8`. Convert bf8 to f32 +pub fn cvt_f32_bf8( + ctx: *mlir.Context, + srcA: *const mlir.Value, + res_type: *const mlir.Type, + byteSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "byteSel", byteSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.f32.bf8", .{ + .operands = .{ .flat = &.{srcA} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.f32.fp8`. Convert fp8 to f32 +pub fn cvt_f32_fp8( + ctx: *mlir.Context, + srcA: *const mlir.Value, + res_type: *const mlir.Type, + byteSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "byteSel", byteSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.f32.fp8", .{ + .operands = .{ .flat = &.{srcA} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.pk.bf8.f32`. Convert two f32's to bf8 +pub fn cvt_pk_bf8_f32( + ctx: *mlir.Context, + srcA: *const mlir.Value, + srcB: *const mlir.Value, + old: *const mlir.Value, + res_type: *const mlir.Type, + wordSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "wordSel", wordSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.pk.bf8.f32", .{ + .operands = .{ .flat = &.{ srcA, srcB, old } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.pk.f32.bf8`. Convert packed bf8 to packed f32 +pub fn cvt_pk_f32_bf8( + ctx: *mlir.Context, + src: *const mlir.Value, + res_type: *const mlir.Type, + wordSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "wordSel", wordSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.pk.f32.bf8", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.pk.f32.fp8`. Convert packed fp8 to packed f32 +pub fn cvt_pk_f32_fp8( + ctx: *mlir.Context, + src: *const mlir.Value, + res_type: *const mlir.Type, + wordSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "wordSel", wordSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.pk.f32.fp8", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.pk.fp8.f32`. Convert two f32's to fp8 +pub fn cvt_pk_fp8_f32( + ctx: *mlir.Context, + srcA: *const mlir.Value, + srcB: *const mlir.Value, + old: *const mlir.Value, + res_type: *const mlir.Type, + wordSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "wordSel", wordSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.pk.fp8.f32", .{ + .operands = .{ .flat = &.{ srcA, srcB, old } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.pkrtz`. Convert two f32 input into a vector<2xf16> +pub fn cvt_pkrtz( + ctx: *mlir.Context, + srcA: *const mlir.Value, + srcB: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.pkrtz", .{ + .operands = .{ .flat = &.{ srcA, srcB } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scale.pk16.bf16.bf6`. Scales 16 bf6 and converts them to 16 bf16. +pub fn cvt_scale_pk16_bf16_bf6( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + scaleSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "scaleSel", scaleSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scale.pk16.bf16.bf6", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scale.pk16.bf16.fp6`. Scales 16 fp6 and converts them to 16 bf16. +pub fn cvt_scale_pk16_bf16_fp6( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + scaleSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "scaleSel", scaleSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scale.pk16.bf16.fp6", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scale.pk16.f16.bf6`. Scales 16 bf6 and converts them to 16 f16. +pub fn cvt_scale_pk16_f16_bf6( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + scaleSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "scaleSel", scaleSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scale.pk16.f16.bf6", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scale.pk16.f16.fp6`. Scales 16 fp6 and converts them to 16 f16. +pub fn cvt_scale_pk16_f16_fp6( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + scaleSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "scaleSel", scaleSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scale.pk16.f16.fp6", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scale.pk16.f32.bf6`. Scales 16 bf6 and converts them to 16 f32. +pub fn cvt_scale_pk16_f32_bf6( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + scaleSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "scaleSel", scaleSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scale.pk16.f32.bf6", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scale.pk16.f32.fp6`. Scales 16 fp6 and converts them to 16 f32. +pub fn cvt_scale_pk16_f32_fp6( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + scaleSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "scaleSel", scaleSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scale.pk16.f32.fp6", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scale.pk8.bf16.bf8`. Scales 8 bf8 and converts them to 8 bf16. +pub fn cvt_scale_pk8_bf16_bf8( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + scaleSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "scaleSel", scaleSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scale.pk8.bf16.bf8", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scale.pk8.bf16.fp4`. Scales 8 fp4 and converts them to 8 bf16. +pub fn cvt_scale_pk8_bf16_fp4( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + scaleSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "scaleSel", scaleSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scale.pk8.bf16.fp4", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scale.pk8.bf16.fp8`. Scales 8 fp8 and converts them to 8 bf16. +pub fn cvt_scale_pk8_bf16_fp8( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + scaleSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "scaleSel", scaleSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scale.pk8.bf16.fp8", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scale.pk8.f16.bf8`. Scales 8 bf8 and converts them to 8 f16. +pub fn cvt_scale_pk8_f16_bf8( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + scaleSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "scaleSel", scaleSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scale.pk8.f16.bf8", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scale.pk8.f16.fp4`. Scales 8 fp4 and converts them to 8 f16. +pub fn cvt_scale_pk8_f16_fp4( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + scaleSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "scaleSel", scaleSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scale.pk8.f16.fp4", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scale.pk8.f16.fp8`. Scales 8 fp8 and converts them to 8 f16. +pub fn cvt_scale_pk8_f16_fp8( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + scaleSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "scaleSel", scaleSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scale.pk8.f16.fp8", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scale.pk8.f32.bf8`. Scales 8 bf8 and converts them to 8 f32. +pub fn cvt_scale_pk8_f32_bf8( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + scaleSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "scaleSel", scaleSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scale.pk8.f32.bf8", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scale.pk8.f32.fp4`. Scales 8 fp4 and converts them to 8 f32. +pub fn cvt_scale_pk8_f32_fp4( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + scaleSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "scaleSel", scaleSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scale.pk8.f32.fp4", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scale.pk8.f32.fp8`. Scales 8 fp8 and converts them to 8 f32. +pub fn cvt_scale_pk8_f32_fp8( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + scaleSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "scaleSel", scaleSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scale.pk8.f32.fp8", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.2xpk16.bf6.f32`. Scale and convert two vector<16xf32> to 32 packed bf6 +pub fn cvt_scalef32_2xpk16_bf6_f32( + ctx: *mlir.Context, + src0: *const mlir.Value, + src1: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.2xpk16.bf6.f32", .{ + .operands = .{ .flat = &.{ src0, src1, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.2xpk16.fp6.f32`. Scale and convert two vector<16xf32> to 32 packed fp6 +pub fn cvt_scalef32_2xpk16_fp6_f32( + ctx: *mlir.Context, + src0: *const mlir.Value, + src1: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.2xpk16.fp6.f32", .{ + .operands = .{ .flat = &.{ src0, src1, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.f16.bf8`. Scaled convert bf8 from packed vector to f16, updating tied result +pub fn cvt_scalef32_f16_bf8( + ctx: *mlir.Context, + oldVdst: *const mlir.Value, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + srcSelIndex: *const mlir.Attribute, + dstLoHiSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "srcSelIndex", srcSelIndex)); + attributes.appendAssumeCapacity(.named(ctx, "dstLoHiSel", dstLoHiSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.f16.bf8", .{ + .operands = .{ .flat = &.{ oldVdst, src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.f16.fp8`. Scaled convert fp8 from packed vector to f16, updating tied result +pub fn cvt_scalef32_f16_fp8( + ctx: *mlir.Context, + oldVdst: *const mlir.Value, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + srcSelIndex: *const mlir.Attribute, + dstLoHiSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "srcSelIndex", srcSelIndex)); + attributes.appendAssumeCapacity(.named(ctx, "dstLoHiSel", dstLoHiSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.f16.fp8", .{ + .operands = .{ .flat = &.{ oldVdst, src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.f32.bf8`. Scaled convert bf8 from packed vector to f32 +pub fn cvt_scalef32_f32_bf8( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + srcSelIndex: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "srcSelIndex", srcSelIndex)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.f32.bf8", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.f32.fp8`. Scaled convert fp8 from packed vector to f32 +pub fn cvt_scalef32_f32_fp8( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + srcSelIndex: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "srcSelIndex", srcSelIndex)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.f32.fp8", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk.bf16.bf8`. Scaled convert two bf8to two bf16 +pub fn cvt_scalef32_pk_bf16_bf8( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + srcLoHiSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "srcLoHiSel", srcLoHiSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.bf16.bf8", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk.bf16.fp4`. Scale and convert two packed fp4 to packed bf16 +pub fn cvt_scalef32_pk_bf16_fp4( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + srcSelIndex: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "srcSelIndex", srcSelIndex)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.bf16.fp4", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk.bf16.fp8`. Scaled convert two fp8to two bf16 +pub fn cvt_scalef32_pk_bf16_fp8( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + srcLoHiSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "srcLoHiSel", srcLoHiSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.bf16.fp8", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk.bf8.bf16`. Scaled convert two bf16to two bf8, updating packed vector +pub fn cvt_scalef32_pk_bf8_bf16( + ctx: *mlir.Context, + oldVdst: *const mlir.Value, + src0: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + dstLoHiSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "dstLoHiSel", dstLoHiSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.bf8.bf16", .{ + .operands = .{ .flat = &.{ oldVdst, src0, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk.bf8.f16`. Scaled convert two f16to two bf8, updating packed vector +pub fn cvt_scalef32_pk_bf8_f16( + ctx: *mlir.Context, + oldVdst: *const mlir.Value, + src0: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + dstLoHiSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "dstLoHiSel", dstLoHiSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.bf8.f16", .{ + .operands = .{ .flat = &.{ oldVdst, src0, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk.bf8.f32`. Scaled convert two f32 to two bf8, updating packed vector +pub fn cvt_scalef32_pk_bf8_f32( + ctx: *mlir.Context, + oldVdst: *const mlir.Value, + src0: *const mlir.Value, + src1: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + dstLoHiSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "dstLoHiSel", dstLoHiSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.bf8.f32", .{ + .operands = .{ .flat = &.{ oldVdst, src0, src1, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk.f16.bf8`. Scaled convert two bf8to two f16 +pub fn cvt_scalef32_pk_f16_bf8( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + srcLoHiSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "srcLoHiSel", srcLoHiSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.f16.bf8", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk.f16.fp4`. Scale and convert two packed fp4 to packed f16 +pub fn cvt_scalef32_pk_f16_fp4( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + srcSelIndex: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "srcSelIndex", srcSelIndex)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.f16.fp4", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk.f16.fp8`. Scaled convert two fp8to two f16 +pub fn cvt_scalef32_pk_f16_fp8( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + srcLoHiSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "srcLoHiSel", srcLoHiSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.f16.fp8", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk.f32.bf8`. Scaled convert two bf8to two f32 +pub fn cvt_scalef32_pk_f32_bf8( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + srcLoHiSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "srcLoHiSel", srcLoHiSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.f32.bf8", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk.f32.fp4`. Scale and convert two packed fp4 to packed f32 +pub fn cvt_scalef32_pk_f32_fp4( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + srcSelIndex: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "srcSelIndex", srcSelIndex)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.f32.fp4", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk.f32.fp8`. Scaled convert two fp8to two f32 +pub fn cvt_scalef32_pk_f32_fp8( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + srcLoHiSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "srcLoHiSel", srcLoHiSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.f32.fp8", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk.fp4.bf16`. Scale and convert two bf16 to packed fp4, updating tied vector +pub fn cvt_scalef32_pk_fp4_bf16( + ctx: *mlir.Context, + oldVdst: *const mlir.Value, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + dstSelIndex: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "dstSelIndex", dstSelIndex)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.fp4.bf16", .{ + .operands = .{ .flat = &.{ oldVdst, src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk.fp4.f16`. Scale and convert two f16 to packed fp4, updating tied vector +pub fn cvt_scalef32_pk_fp4_f16( + ctx: *mlir.Context, + oldVdst: *const mlir.Value, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + dstSelIndex: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "dstSelIndex", dstSelIndex)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.fp4.f16", .{ + .operands = .{ .flat = &.{ oldVdst, src, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk.fp4.f32`. Scale and convert two f32 values to two packed fp4, updating tied vector +pub fn cvt_scalef32_pk_fp4_f32( + ctx: *mlir.Context, + oldVdst: *const mlir.Value, + src0: *const mlir.Value, + src1: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + dstSelIndex: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "dstSelIndex", dstSelIndex)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.fp4.f32", .{ + .operands = .{ .flat = &.{ oldVdst, src0, src1, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk.fp8.bf16`. Scaled convert two bf16to two fp8, updating packed vector +pub fn cvt_scalef32_pk_fp8_bf16( + ctx: *mlir.Context, + oldVdst: *const mlir.Value, + src0: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + dstLoHiSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "dstLoHiSel", dstLoHiSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.fp8.bf16", .{ + .operands = .{ .flat = &.{ oldVdst, src0, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk.fp8.f16`. Scaled convert two f16to two fp8, updating packed vector +pub fn cvt_scalef32_pk_fp8_f16( + ctx: *mlir.Context, + oldVdst: *const mlir.Value, + src0: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + dstLoHiSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "dstLoHiSel", dstLoHiSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.fp8.f16", .{ + .operands = .{ .flat = &.{ oldVdst, src0, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk.fp8.f32`. Scaled convert two f32 to two fp8, updating packed vector +pub fn cvt_scalef32_pk_fp8_f32( + ctx: *mlir.Context, + oldVdst: *const mlir.Value, + src0: *const mlir.Value, + src1: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + dstLoHiSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "dstLoHiSel", dstLoHiSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.fp8.f32", .{ + .operands = .{ .flat = &.{ oldVdst, src0, src1, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk16.bf6.bf16`. Scale and convert packed bf16 to packed bf6 +pub fn cvt_scalef32_pk16_bf6_bf16( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk16.bf6.bf16", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk16.bf6.f16`. Scale and convert packed f16 to packed bf6 +pub fn cvt_scalef32_pk16_bf6_f16( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk16.bf6.f16", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk16.bf6.f32`. Scale and convert packed f32 to packed bf6 +pub fn cvt_scalef32_pk16_bf6_f32( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk16.bf6.f32", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk16.fp6.bf16`. Scale and convert packed bf16 to packed fp6 +pub fn cvt_scalef32_pk16_fp6_bf16( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk16.fp6.bf16", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk16.fp6.f16`. Scale and convert packed f16 to packed fp6 +pub fn cvt_scalef32_pk16_fp6_f16( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk16.fp6.f16", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk16.fp6.f32`. Scale and convert packed f32 to packed fp6 +pub fn cvt_scalef32_pk16_fp6_f32( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk16.fp6.f32", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk32.bf16.bf6`. Scale and convert packed bf6 to packed bf16 +pub fn cvt_scalef32_pk32_bf16_bf6( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk32.bf16.bf6", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk32.bf16.fp6`. Scale and convert packed fp6 to packed bf16 +pub fn cvt_scalef32_pk32_bf16_fp6( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk32.bf16.fp6", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk32.bf6.bf16`. Scale and convert packed bf16 to packed bf6 +pub fn cvt_scalef32_pk32_bf6_bf16( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk32.bf6.bf16", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk32.bf6.f16`. Scale and convert packed f16 to packed bf6 +pub fn cvt_scalef32_pk32_bf6_f16( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk32.bf6.f16", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk32.f16.bf6`. Scale and convert packed bf6 to packed f16 +pub fn cvt_scalef32_pk32_f16_bf6( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk32.f16.bf6", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk32.f16.fp6`. Scale and convert packed fp6 to packed f16 +pub fn cvt_scalef32_pk32_f16_fp6( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk32.f16.fp6", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk32.f32.bf6`. Scale and convert packed bf6 to packed f32 +pub fn cvt_scalef32_pk32_f32_bf6( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk32.f32.bf6", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk32.f32.fp6`. Scale and convert packed fp6 to packed f32 +pub fn cvt_scalef32_pk32_f32_fp6( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk32.f32.fp6", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk32.fp6.bf16`. Scale and convert packed bf16 to packed fp6 +pub fn cvt_scalef32_pk32_fp6_bf16( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk32.fp6.bf16", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk32.fp6.f16`. Scale and convert packed f16 to packed fp6 +pub fn cvt_scalef32_pk32_fp6_f16( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk32.fp6.f16", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk8.bf8.bf16`. Scale and convert packed bf16 to packed bf8 +pub fn cvt_scalef32_pk8_bf8_bf16( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk8.bf8.bf16", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk8.bf8.f16`. Scale and convert packed f16 to packed bf8 +pub fn cvt_scalef32_pk8_bf8_f16( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk8.bf8.f16", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk8.bf8.f32`. Scale and convert packed f32 to packed bf8 +pub fn cvt_scalef32_pk8_bf8_f32( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk8.bf8.f32", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk8.fp4.bf16`. Scale and convert packed bf16 to packed fp4 +pub fn cvt_scalef32_pk8_fp4_bf16( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk8.fp4.bf16", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk8.fp4.f16`. Scale and convert packed f16 to packed fp4 +pub fn cvt_scalef32_pk8_fp4_f16( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk8.fp4.f16", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk8.fp4.f32`. Scale and convert packed f32 to packed fp4 +pub fn cvt_scalef32_pk8_fp4_f32( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk8.fp4.f32", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk8.fp8.bf16`. Scale and convert packed bf16 to packed fp8 +pub fn cvt_scalef32_pk8_fp8_bf16( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk8.fp8.bf16", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk8.fp8.f16`. Scale and convert packed f16 to packed fp8 +pub fn cvt_scalef32_pk8_fp8_f16( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk8.fp8.f16", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.pk8.fp8.f32`. Scale and convert packed f32 to packed fp8 +pub fn cvt_scalef32_pk8_fp8_f32( + ctx: *mlir.Context, + src: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk8.fp8.f32", .{ + .operands = .{ .flat = &.{ src, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.bf8.bf16`. Scaled convert bf16to bf8 with stochiastic rounding, updating packed vector +pub fn cvt_scalef32_sr_bf8_bf16( + ctx: *mlir.Context, + oldVdst: *const mlir.Value, + src0: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + dstSelIndex: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "dstSelIndex", dstSelIndex)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.bf8.bf16", .{ + .operands = .{ .flat = &.{ oldVdst, src0, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.bf8.f16`. Scaled convert f16to bf8 with stochiastic rounding, updating packed vector +pub fn cvt_scalef32_sr_bf8_f16( + ctx: *mlir.Context, + oldVdst: *const mlir.Value, + src0: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + dstSelIndex: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "dstSelIndex", dstSelIndex)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.bf8.f16", .{ + .operands = .{ .flat = &.{ oldVdst, src0, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.bf8.f32`. Scaled convert f32to bf8 with stochiastic rounding, updating packed vector +pub fn cvt_scalef32_sr_bf8_f32( + ctx: *mlir.Context, + oldVdst: *const mlir.Value, + src0: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + dstSelIndex: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "dstSelIndex", dstSelIndex)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.bf8.f32", .{ + .operands = .{ .flat = &.{ oldVdst, src0, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.fp8.bf16`. Scaled convert bf16to fp8 with stochiastic rounding, updating packed vector +pub fn cvt_scalef32_sr_fp8_bf16( + ctx: *mlir.Context, + oldVdst: *const mlir.Value, + src0: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + dstSelIndex: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "dstSelIndex", dstSelIndex)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.fp8.bf16", .{ + .operands = .{ .flat = &.{ oldVdst, src0, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.fp8.f16`. Scaled convert f16to fp8 with stochiastic rounding, updating packed vector +pub fn cvt_scalef32_sr_fp8_f16( + ctx: *mlir.Context, + oldVdst: *const mlir.Value, + src0: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + dstSelIndex: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "dstSelIndex", dstSelIndex)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.fp8.f16", .{ + .operands = .{ .flat = &.{ oldVdst, src0, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.fp8.f32`. Scaled convert f32to fp8 with stochiastic rounding, updating packed vector +pub fn cvt_scalef32_sr_fp8_f32( + ctx: *mlir.Context, + oldVdst: *const mlir.Value, + src0: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + dstSelIndex: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "dstSelIndex", dstSelIndex)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.fp8.f32", .{ + .operands = .{ .flat = &.{ oldVdst, src0, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk.fp4.bf16`. Scale and convert two bf16 to packed fp4 with stochiastic rounding, updating tied vector +pub fn cvt_scalef32_sr_pk_fp4_bf16( + ctx: *mlir.Context, + oldVdst: *const mlir.Value, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + dstSelIndex: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "dstSelIndex", dstSelIndex)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk.fp4.bf16", .{ + .operands = .{ .flat = &.{ oldVdst, src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk.fp4.f16`. Scale and convert two f16 to packed fp4 with stochiastic rounding, updating tied vector +pub fn cvt_scalef32_sr_pk_fp4_f16( + ctx: *mlir.Context, + oldVdst: *const mlir.Value, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + dstSelIndex: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "dstSelIndex", dstSelIndex)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk.fp4.f16", .{ + .operands = .{ .flat = &.{ oldVdst, src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk.fp4.f32`. Scale and convert two f32 to packed fp4 with stochiastic rounding, updating tied vector +pub fn cvt_scalef32_sr_pk_fp4_f32( + ctx: *mlir.Context, + oldVdst: *const mlir.Value, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + dstSelIndex: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "dstSelIndex", dstSelIndex)); + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk.fp4.f32", .{ + .operands = .{ .flat = &.{ oldVdst, src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk16.bf6.bf16`. Scale and convert packed bf16 to packed bf6 with stochastic rounding +pub fn cvt_scalef32_sr_pk16_bf6_bf16( + ctx: *mlir.Context, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk16.bf6.bf16", .{ + .operands = .{ .flat = &.{ src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk16.bf6.f16`. Scale and convert packed f16 to packed bf6 with stochastic rounding +pub fn cvt_scalef32_sr_pk16_bf6_f16( + ctx: *mlir.Context, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk16.bf6.f16", .{ + .operands = .{ .flat = &.{ src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk16.bf6.f32`. Scale and convert packed f32 to packed bf6 with stochastic rounding +pub fn cvt_scalef32_sr_pk16_bf6_f32( + ctx: *mlir.Context, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk16.bf6.f32", .{ + .operands = .{ .flat = &.{ src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk16.fp6.bf16`. Scale and convert packed bf16 to packed fp6 with stochastic rounding +pub fn cvt_scalef32_sr_pk16_fp6_bf16( + ctx: *mlir.Context, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk16.fp6.bf16", .{ + .operands = .{ .flat = &.{ src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk16.fp6.f16`. Scale and convert packed f16 to packed fp6 with stochastic rounding +pub fn cvt_scalef32_sr_pk16_fp6_f16( + ctx: *mlir.Context, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk16.fp6.f16", .{ + .operands = .{ .flat = &.{ src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk16.fp6.f32`. Scale and convert packed f32 to packed fp6 with stochastic rounding +pub fn cvt_scalef32_sr_pk16_fp6_f32( + ctx: *mlir.Context, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk16.fp6.f32", .{ + .operands = .{ .flat = &.{ src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk32.bf6.bf16`. Scale and convert packed bf16 to packed bf6 with stochiastic rounding +pub fn cvt_scalef32_sr_pk32_bf6_bf16( + ctx: *mlir.Context, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk32.bf6.bf16", .{ + .operands = .{ .flat = &.{ src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk32.bf6.f16`. Scale and convert packed f16 to packed bf6 with stochiastic rounding +pub fn cvt_scalef32_sr_pk32_bf6_f16( + ctx: *mlir.Context, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk32.bf6.f16", .{ + .operands = .{ .flat = &.{ src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk32.bf6.f32`. Scale and convert packed f32 to packed bf6 with stochiastic rounding +pub fn cvt_scalef32_sr_pk32_bf6_f32( + ctx: *mlir.Context, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk32.bf6.f32", .{ + .operands = .{ .flat = &.{ src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk32.fp6.bf16`. Scale and convert packed bf16 to packed fp6 with stochiastic rounding +pub fn cvt_scalef32_sr_pk32_fp6_bf16( + ctx: *mlir.Context, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk32.fp6.bf16", .{ + .operands = .{ .flat = &.{ src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk32.fp6.f16`. Scale and convert packed f16 to packed fp6 with stochiastic rounding +pub fn cvt_scalef32_sr_pk32_fp6_f16( + ctx: *mlir.Context, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk32.fp6.f16", .{ + .operands = .{ .flat = &.{ src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk32.fp6.f32`. Scale and convert packed f32 to packed fp6 with stochiastic rounding +pub fn cvt_scalef32_sr_pk32_fp6_f32( + ctx: *mlir.Context, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk32.fp6.f32", .{ + .operands = .{ .flat = &.{ src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk8.bf8.bf16`. Scale and convert packed bf16 to packed bf8 with stochastic rounding +pub fn cvt_scalef32_sr_pk8_bf8_bf16( + ctx: *mlir.Context, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk8.bf8.bf16", .{ + .operands = .{ .flat = &.{ src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk8.bf8.f16`. Scale and convert packed f16 to packed bf8 with stochastic rounding +pub fn cvt_scalef32_sr_pk8_bf8_f16( + ctx: *mlir.Context, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk8.bf8.f16", .{ + .operands = .{ .flat = &.{ src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk8.bf8.f32`. Scale and convert packed f32 to packed bf8 with stochastic rounding +pub fn cvt_scalef32_sr_pk8_bf8_f32( + ctx: *mlir.Context, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk8.bf8.f32", .{ + .operands = .{ .flat = &.{ src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk8.fp4.bf16`. Scale and convert packed bf16 to packed fp4 with stochastic rounding +pub fn cvt_scalef32_sr_pk8_fp4_bf16( + ctx: *mlir.Context, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk8.fp4.bf16", .{ + .operands = .{ .flat = &.{ src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk8.fp4.f16`. Scale and convert packed f16 to packed fp4 with stochastic rounding +pub fn cvt_scalef32_sr_pk8_fp4_f16( + ctx: *mlir.Context, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk8.fp4.f16", .{ + .operands = .{ .flat = &.{ src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk8.fp4.f32`. Scale and convert packed f32 to packed fp4 with stochastic rounding +pub fn cvt_scalef32_sr_pk8_fp4_f32( + ctx: *mlir.Context, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk8.fp4.f32", .{ + .operands = .{ .flat = &.{ src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk8.fp8.bf16`. Scale and convert packed bf16 to packed fp8 with stochastic rounding +pub fn cvt_scalef32_sr_pk8_fp8_bf16( + ctx: *mlir.Context, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk8.fp8.bf16", .{ + .operands = .{ .flat = &.{ src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk8.fp8.f16`. Scale and convert packed f16 to packed fp8 with stochastic rounding +pub fn cvt_scalef32_sr_pk8_fp8_f16( + ctx: *mlir.Context, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk8.fp8.f16", .{ + .operands = .{ .flat = &.{ src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.scalef32.sr.pk8.fp8.f32`. Scale and convert packed f32 to packed fp8 with stochastic rounding +pub fn cvt_scalef32_sr_pk8_fp8_f32( + ctx: *mlir.Context, + src: *const mlir.Value, + seed: *const mlir.Value, + scale: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk8.fp8.f32", .{ + .operands = .{ .flat = &.{ src, seed, scale } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.cvt.sr.bf8.f32`. Convert f32 to bf8, stochiastic rounding +pub fn cvt_sr_bf8_f32( + ctx: *mlir.Context, + srcA: *const mlir.Value, + srcB: *const mlir.Value, + old: *const mlir.Value, + res_type: *const mlir.Type, + byteSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "byteSel", byteSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.sr.bf8.f32", .{ + .operands = .{ .flat = &.{ srcA, srcB, old } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.cvt.sr.fp8.f32`. Convert f32 to fp8, stochiastic rounding +pub fn cvt_sr_fp8_f32( + ctx: *mlir.Context, + srcA: *const mlir.Value, + srcB: *const mlir.Value, + old: *const mlir.Value, + res_type: *const mlir.Type, + byteSel: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "byteSel", byteSel)); + return mlir.Operation.make(ctx, "rocdl.cvt.sr.fp8.f32", .{ + .operands = .{ .flat = &.{ srcA, srcB, old } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.dot4.f32.bf8.bf8`. +pub fn dot4_f32_bf8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.dot4.f32.bf8.bf8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.dot4.f32.bf8.fp8`. +pub fn dot4_f32_bf8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.dot4.f32.bf8.fp8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.dot4.f32.fp8.bf8`. +pub fn dot4_f32_fp8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.dot4.f32.fp8.bf8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.dot4.f32.fp8.fp8`. +pub fn dot4_f32_fp8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.dot4.f32.fp8.fp8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.ds.atomic.async.barrier.arrive.b64`. +pub fn ds_atomic_async_barrier_arrive_b64( + ctx: *mlir.Context, + barrierPtr: *const mlir.Value, + opts: struct { + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.ds.atomic.async.barrier.arrive.b64", .{ + .operands = .{ .flat = &.{barrierPtr} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.ds.atomic.barrier.arrive.rtn.b64`. +pub fn ds_atomic_barrier_arrive_rtn_b64( + ctx: *mlir.Context, + barrierPtr: *const mlir.Value, + val: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.ds.atomic.barrier.arrive.rtn.b64", .{ + .operands = .{ .flat = &.{ barrierPtr, val } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.ds.load.tr16.b128`. Loads and transposes a matrix from ds memory to registers (available in gfx1250+). +pub fn ds_load_tr16_b128( + ctx: *mlir.Context, + ptr: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.ds.load.tr16.b128", .{ + .operands = .{ .flat = &.{ptr} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.ds.load.tr4.b64`. Loads and transposes a matrix from ds memory to registers (available in gfx1250+). +pub fn ds_load_tr4_b64( + ctx: *mlir.Context, + ptr: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.ds.load.tr4.b64", .{ + .operands = .{ .flat = &.{ptr} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.ds.load.tr6.b96`. Loads and transposes a matrix from ds memory to registers (available in gfx1250+). +pub fn ds_load_tr6_b96( + ctx: *mlir.Context, + ptr: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.ds.load.tr6.b96", .{ + .operands = .{ .flat = &.{ptr} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.ds.load.tr8.b64`. Loads and transposes a matrix from ds memory to registers (available in gfx1250+). +pub fn ds_load_tr8_b64( + ctx: *mlir.Context, + ptr: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.ds.load.tr8.b64", .{ + .operands = .{ .flat = &.{ptr} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.ds.read.tr16.b64`. +pub fn ds_read_tr16_b64( + ctx: *mlir.Context, + ptr: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.ds.read.tr16.b64", .{ + .operands = .{ .flat = &.{ptr} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.ds.read.tr4.b64`. +pub fn ds_read_tr4_b64( + ctx: *mlir.Context, + ptr: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.ds.read.tr4.b64", .{ + .operands = .{ .flat = &.{ptr} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.ds.read.tr6.b96`. +pub fn ds_read_tr6_b96( + ctx: *mlir.Context, + ptr: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.ds.read.tr6.b96", .{ + .operands = .{ .flat = &.{ptr} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.ds.read.tr8.b64`. +pub fn ds_read_tr8_b64( + ctx: *mlir.Context, + ptr: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.ds.read.tr8.b64", .{ + .operands = .{ .flat = &.{ptr} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.ds_bpermute`. +pub fn ds_bpermute( + ctx: *mlir.Context, + index: *const mlir.Value, + src: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.ds_bpermute", .{ + .operands = .{ .flat = &.{ index, src } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.ds_swizzle`. +pub fn ds_swizzle( + ctx: *mlir.Context, + src: *const mlir.Value, + offset: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.ds_swizzle", .{ + .operands = .{ .flat = &.{ src, offset } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.exp`. +pub fn exp( + ctx: *mlir.Context, + arg: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.exp", .{ + .operands = .{ .flat = &.{arg} }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.exp2`. +pub fn exp2( + ctx: *mlir.Context, + arg: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.exp2", .{ + .operands = .{ .flat = &.{arg} }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.fdot2`. +pub fn fdot2( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + clamp: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.clamp) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.fdot2", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.fdot2.bf16.bf16`. +pub fn fdot2_bf16_bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.fdot2.bf16.bf16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.fdot2.f16.f16`. +pub fn fdot2_f16_f16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.fdot2.f16.f16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.fdot2.f32.bf16`. +pub fn fdot2_f32_bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + clamp: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.clamp) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.fdot2.f32.bf16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.flat.prefetch`. +pub fn flat_prefetch( + ctx: *mlir.Context, + ptr: *const mlir.Value, + opts: struct { + cachePolicy: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (opts.cachePolicy) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "cachePolicy", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.flat.prefetch", .{ + .operands = .{ .flat = &.{ptr} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.fmed3`. Median of three float/half values +pub fn fmed3( + ctx: *mlir.Context, + src0: *const mlir.Value, + src1: *const mlir.Value, + src2: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.fmed3", .{ + .operands = .{ .flat = &.{ src0, src1, src2 } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.global.load.async.lds`. Version of rocdl.load.async.to.lds specialized to global pointers +pub fn global_load_async_lds( + ctx: *mlir.Context, + globalPtr: *const mlir.Value, + ldsPtr: *const mlir.Value, + size: *const mlir.Attribute, + offset: *const mlir.Attribute, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 6) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "size", size)); + attributes.appendAssumeCapacity(.named(ctx, "offset", offset)); + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.global.load.async.lds", .{ + .operands = .{ .flat = &.{ globalPtr, ldsPtr } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.global.load.async.to.lds.b128`. +pub fn global_load_async_to_lds_b128( + ctx: *mlir.Context, + globalPtr: *const mlir.Value, + ldsPtr: *const mlir.Value, + offset: *const mlir.Attribute, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "offset", offset)); + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.global.load.async.to.lds.b128", .{ + .operands = .{ .flat = &.{ globalPtr, ldsPtr } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.global.load.async.to.lds.b32`. +pub fn global_load_async_to_lds_b32( + ctx: *mlir.Context, + globalPtr: *const mlir.Value, + ldsPtr: *const mlir.Value, + offset: *const mlir.Attribute, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "offset", offset)); + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.global.load.async.to.lds.b32", .{ + .operands = .{ .flat = &.{ globalPtr, ldsPtr } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.global.load.async.to.lds.b64`. +pub fn global_load_async_to_lds_b64( + ctx: *mlir.Context, + globalPtr: *const mlir.Value, + ldsPtr: *const mlir.Value, + offset: *const mlir.Attribute, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "offset", offset)); + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.global.load.async.to.lds.b64", .{ + .operands = .{ .flat = &.{ globalPtr, ldsPtr } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.global.load.async.to.lds.b8`. +pub fn global_load_async_to_lds_b8( + ctx: *mlir.Context, + globalPtr: *const mlir.Value, + ldsPtr: *const mlir.Value, + offset: *const mlir.Attribute, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "offset", offset)); + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.global.load.async.to.lds.b8", .{ + .operands = .{ .flat = &.{ globalPtr, ldsPtr } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.global.load.lds`. +pub fn global_load_lds( + ctx: *mlir.Context, + globalPtr: *const mlir.Value, + ldsPtr: *const mlir.Value, + size: *const mlir.Attribute, + offset: *const mlir.Attribute, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 6) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "size", size)); + attributes.appendAssumeCapacity(.named(ctx, "offset", offset)); + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.global.load.lds", .{ + .operands = .{ .flat = &.{ globalPtr, ldsPtr } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.global.load.tr.b128`. Loads and transposes a matrix from global memory to registers (available in gfx1250+). +pub fn global_load_tr_b128( + ctx: *mlir.Context, + ptr: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.global.load.tr.b128", .{ + .operands = .{ .flat = &.{ptr} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.global.load.tr.b64`. Loads and transposes a matrix from global memory to registers (available in gfx1250+). +pub fn global_load_tr_b64( + ctx: *mlir.Context, + ptr: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.global.load.tr.b64", .{ + .operands = .{ .flat = &.{ptr} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.global.load.tr4.b64`. Loads and transposes a matrix from global memory to registers (available in gfx1250+). +pub fn global_load_tr4_b64( + ctx: *mlir.Context, + ptr: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.global.load.tr4.b64", .{ + .operands = .{ .flat = &.{ptr} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.global.load.tr6.b96`. Loads and transposes a matrix from global memory to registers (available in gfx1250+). +pub fn global_load_tr6_b96( + ctx: *mlir.Context, + ptr: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.global.load.tr6.b96", .{ + .operands = .{ .flat = &.{ptr} }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.global.prefetch`. +pub fn global_prefetch( + ctx: *mlir.Context, + ptr: *const mlir.Value, + opts: struct { + cachePolicy: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (opts.cachePolicy) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "cachePolicy", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.global.prefetch", .{ + .operands = .{ .flat = &.{ptr} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.global.store.async.from.lds.b128`. +pub fn global_store_async_from_lds_b128( + ctx: *mlir.Context, + globalPtr: *const mlir.Value, + ldsPtr: *const mlir.Value, + offset: *const mlir.Attribute, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "offset", offset)); + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.global.store.async.from.lds.b128", .{ + .operands = .{ .flat = &.{ globalPtr, ldsPtr } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.global.store.async.from.lds.b32`. +pub fn global_store_async_from_lds_b32( + ctx: *mlir.Context, + globalPtr: *const mlir.Value, + ldsPtr: *const mlir.Value, + offset: *const mlir.Attribute, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "offset", offset)); + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.global.store.async.from.lds.b32", .{ + .operands = .{ .flat = &.{ globalPtr, ldsPtr } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.global.store.async.from.lds.b64`. +pub fn global_store_async_from_lds_b64( + ctx: *mlir.Context, + globalPtr: *const mlir.Value, + ldsPtr: *const mlir.Value, + offset: *const mlir.Attribute, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "offset", offset)); + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.global.store.async.from.lds.b64", .{ + .operands = .{ .flat = &.{ globalPtr, ldsPtr } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.global.store.async.from.lds.b8`. +pub fn global_store_async_from_lds_b8( + ctx: *mlir.Context, + globalPtr: *const mlir.Value, + ldsPtr: *const mlir.Value, + offset: *const mlir.Attribute, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "offset", offset)); + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.global.store.async.from.lds.b8", .{ + .operands = .{ .flat = &.{ globalPtr, ldsPtr } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.iglp.opt`. +pub fn iglp_opt( + ctx: *mlir.Context, + variant: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "variant", variant)); + return mlir.Operation.make(ctx, "rocdl.iglp.opt", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.load.async.to.lds`. Gathering load to LDS that requires explicit async memory tracking +pub fn load_async_to_lds( + ctx: *mlir.Context, + globalPtr: *const mlir.Value, + ldsPtr: *const mlir.Value, + size: *const mlir.Attribute, + offset: *const mlir.Attribute, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 6) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "size", size)); + attributes.appendAssumeCapacity(.named(ctx, "offset", offset)); + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.load.async.to.lds", .{ + .operands = .{ .flat = &.{ globalPtr, ldsPtr } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.load.to.lds`. +pub fn load_to_lds( + ctx: *mlir.Context, + globalPtr: *const mlir.Value, + ldsPtr: *const mlir.Value, + size: *const mlir.Attribute, + offset: *const mlir.Attribute, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 6) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "size", size)); + attributes.appendAssumeCapacity(.named(ctx, "offset", offset)); + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.load.to.lds", .{ + .operands = .{ .flat = &.{ globalPtr, ldsPtr } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.log`. +pub fn log( + ctx: *mlir.Context, + arg: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.log", .{ + .operands = .{ .flat = &.{arg} }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.make.buffer.rsrc`. +pub fn make_buffer_rsrc( + ctx: *mlir.Context, + base: *const mlir.Value, + stride: *const mlir.Value, + numRecords: *const mlir.Value, + flags: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.make.buffer.rsrc", .{ + .operands = .{ .flat = &.{ base, stride, numRecords, flags } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.mbcnt.hi`. +pub fn mbcnt_hi( + ctx: *mlir.Context, + in0: *const mlir.Value, + in1: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + arg_attrs: ?*const mlir.Attribute = null, + res_attrs: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.arg_attrs) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "arg_attrs", attribute)); + } + if (opts.res_attrs) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "res_attrs", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.mbcnt.hi", .{ + .operands = .{ .flat = &.{ in0, in1 } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mbcnt.lo`. +pub fn mbcnt_lo( + ctx: *mlir.Context, + in0: *const mlir.Value, + in1: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + arg_attrs: ?*const mlir.Attribute = null, + res_attrs: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.arg_attrs) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "arg_attrs", attribute)); + } + if (opts.res_attrs) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "res_attrs", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.mbcnt.lo", .{ + .operands = .{ .flat = &.{ in0, in1 } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.16x16x16bf16.1k`. +pub fn mfma_f32_16x16x16bf16_1k( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.16x16x16bf16.1k", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.16x16x16f16`. +pub fn mfma_f32_16x16x16f16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.16x16x16f16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.16x16x1f32`. +pub fn mfma_f32_16x16x1f32( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.16x16x1f32", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.16x16x2bf16`. +pub fn mfma_f32_16x16x2bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.16x16x2bf16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.16x16x32.bf16`. +pub fn mfma_f32_16x16x32_bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.16x16x32.bf16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.16x16x32.bf8.bf8`. +pub fn mfma_f32_16x16x32_bf8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.16x16x32.bf8.bf8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.16x16x32.bf8.fp8`. +pub fn mfma_f32_16x16x32_bf8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.16x16x32.bf8.fp8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.16x16x32.f16`. +pub fn mfma_f32_16x16x32_f16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.16x16x32.f16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.16x16x32.fp8.bf8`. +pub fn mfma_f32_16x16x32_fp8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.16x16x32.fp8.bf8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.16x16x32.fp8.fp8`. +pub fn mfma_f32_16x16x32_fp8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.16x16x32.fp8.fp8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.16x16x4bf16.1k`. +pub fn mfma_f32_16x16x4bf16_1k( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.16x16x4bf16.1k", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.16x16x4f16`. +pub fn mfma_f32_16x16x4f16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.16x16x4f16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.16x16x4f32`. +pub fn mfma_f32_16x16x4f32( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.16x16x4f32", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.16x16x8.xf32`. +pub fn mfma_f32_16x16x8_xf32( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.16x16x8.xf32", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.16x16x8bf16`. +pub fn mfma_f32_16x16x8bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.16x16x8bf16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.32x32x16.bf16`. +pub fn mfma_f32_32x32x16_bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.32x32x16.bf16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.32x32x16.bf8.bf8`. +pub fn mfma_f32_32x32x16_bf8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.32x32x16.bf8.bf8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.32x32x16.bf8.fp8`. +pub fn mfma_f32_32x32x16_bf8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.32x32x16.bf8.fp8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.32x32x16.f16`. +pub fn mfma_f32_32x32x16_f16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.32x32x16.f16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.32x32x16.fp8.bf8`. +pub fn mfma_f32_32x32x16_fp8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.32x32x16.fp8.bf8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.32x32x16.fp8.fp8`. +pub fn mfma_f32_32x32x16_fp8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.32x32x16.fp8.fp8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.32x32x1f32`. +pub fn mfma_f32_32x32x1f32( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.32x32x1f32", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.32x32x2bf16`. +pub fn mfma_f32_32x32x2bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.32x32x2bf16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.32x32x2f32`. +pub fn mfma_f32_32x32x2f32( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.32x32x2f32", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.32x32x4.xf32`. +pub fn mfma_f32_32x32x4_xf32( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.32x32x4.xf32", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.32x32x4bf16`. +pub fn mfma_f32_32x32x4bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.32x32x4bf16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.32x32x4bf16.1k`. +pub fn mfma_f32_32x32x4bf16_1k( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.32x32x4bf16.1k", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.32x32x4f16`. +pub fn mfma_f32_32x32x4f16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.32x32x4f16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.32x32x8bf16.1k`. +pub fn mfma_f32_32x32x8bf16_1k( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.32x32x8bf16.1k", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.32x32x8f16`. +pub fn mfma_f32_32x32x8f16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.32x32x8f16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.4x4x1f32`. +pub fn mfma_f32_4x4x1f32( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.4x4x1f32", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.4x4x2bf16`. +pub fn mfma_f32_4x4x2bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.4x4x2bf16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.4x4x4bf16.1k`. +pub fn mfma_f32_4x4x4bf16_1k( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.4x4x4bf16.1k", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f32.4x4x4f16`. +pub fn mfma_f32_4x4x4f16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f32.4x4x4f16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f64.16x16x4f64`. +pub fn mfma_f64_16x16x4f64( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f64.16x16x4f64", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.f64.4x4x4f64`. +pub fn mfma_f64_4x4x4f64( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.f64.4x4x4f64", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.i32.16x16x16i8`. +pub fn mfma_i32_16x16x16i8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.i32.16x16x16i8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.i32.16x16x32.i8`. +pub fn mfma_i32_16x16x32_i8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.i32.16x16x32.i8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.i32.16x16x4i8`. +pub fn mfma_i32_16x16x4i8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.i32.16x16x4i8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.i32.16x16x64.i8`. +pub fn mfma_i32_16x16x64_i8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.i32.16x16x64.i8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.i32.32x32x16.i8`. +pub fn mfma_i32_32x32x16_i8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.i32.32x32x16.i8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.i32.32x32x32.i8`. +pub fn mfma_i32_32x32x32_i8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.i32.32x32x32.i8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.i32.32x32x4i8`. +pub fn mfma_i32_32x32x4i8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.i32.32x32x4i8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.i32.32x32x8i8`. +pub fn mfma_i32_32x32x8i8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.i32.32x32x8i8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.i32.4x4x4i8`. +pub fn mfma_i32_4x4x4i8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + blgp: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + return mlir.Operation.make(ctx, "rocdl.mfma.i32.4x4x4i8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.scale.f32.16x16x128.f8f6f4`. +pub fn mfma_scale_f32_16x16x128_f8f6f4( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + scaleA: *const mlir.Value, + scaleB: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + blgp: *const mlir.Attribute, + opselA: *const mlir.Attribute, + opselB: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + attributes.appendAssumeCapacity(.named(ctx, "opselA", opselA)); + attributes.appendAssumeCapacity(.named(ctx, "opselB", opselB)); + return mlir.Operation.make(ctx, "rocdl.mfma.scale.f32.16x16x128.f8f6f4", .{ + .operands = .{ .flat = &.{ a, b, c, scaleA, scaleB } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.mfma.scale.f32.32x32x64.f8f6f4`. +pub fn mfma_scale_f32_32x32x64_f8f6f4( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + scaleA: *const mlir.Value, + scaleB: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + blgp: *const mlir.Attribute, + opselA: *const mlir.Attribute, + opselB: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); + attributes.appendAssumeCapacity(.named(ctx, "opselA", opselA)); + attributes.appendAssumeCapacity(.named(ctx, "opselB", opselB)); + return mlir.Operation.make(ctx, "rocdl.mfma.scale.f32.32x32x64.f8f6f4", .{ + .operands = .{ .flat = &.{ a, b, c, scaleA, scaleB } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.permlane16.swap`. +pub fn permlane16_swap( + ctx: *mlir.Context, + old: *const mlir.Value, + src: *const mlir.Value, + res_type: *const mlir.Type, + fi: *const mlir.Attribute, + boundControl: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "fi", fi)); + attributes.appendAssumeCapacity(.named(ctx, "boundControl", boundControl)); + return mlir.Operation.make(ctx, "rocdl.permlane16.swap", .{ + .operands = .{ .flat = &.{ old, src } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.permlane16.var`. +pub fn permlane16_var( + ctx: *mlir.Context, + old: *const mlir.Value, + src0: *const mlir.Value, + src1: *const mlir.Value, + res_type: *const mlir.Type, + fi: *const mlir.Attribute, + boundControl: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "fi", fi)); + attributes.appendAssumeCapacity(.named(ctx, "boundControl", boundControl)); + return mlir.Operation.make(ctx, "rocdl.permlane16.var", .{ + .operands = .{ .flat = &.{ old, src0, src1 } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.permlane32.swap`. +pub fn permlane32_swap( + ctx: *mlir.Context, + old: *const mlir.Value, + src: *const mlir.Value, + res_type: *const mlir.Type, + fi: *const mlir.Attribute, + boundControl: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "fi", fi)); + attributes.appendAssumeCapacity(.named(ctx, "boundControl", boundControl)); + return mlir.Operation.make(ctx, "rocdl.permlane32.swap", .{ + .operands = .{ .flat = &.{ old, src } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.permlanex16`. +pub fn permlanex16( + ctx: *mlir.Context, + old: *const mlir.Value, + src0: *const mlir.Value, + src1: *const mlir.Value, + src2: *const mlir.Value, + res_type: *const mlir.Type, + fi: *const mlir.Attribute, + boundControl: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "fi", fi)); + attributes.appendAssumeCapacity(.named(ctx, "boundControl", boundControl)); + return mlir.Operation.make(ctx, "rocdl.permlanex16", .{ + .operands = .{ .flat = &.{ old, src0, src1, src2 } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.permlanex16.var`. +pub fn permlanex16_var( + ctx: *mlir.Context, + old: *const mlir.Value, + src0: *const mlir.Value, + src1: *const mlir.Value, + res_type: *const mlir.Type, + fi: *const mlir.Attribute, + boundControl: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "fi", fi)); + attributes.appendAssumeCapacity(.named(ctx, "boundControl", boundControl)); + return mlir.Operation.make(ctx, "rocdl.permlanex16.var", .{ + .operands = .{ .flat = &.{ old, src0, src1 } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.ptr.s.buffer.load`. +pub fn ptr_s_buffer_load( + ctx: *mlir.Context, + rsrc: *const mlir.Value, + offset: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.ptr.s.buffer.load", .{ + .operands = .{ .flat = &.{ rsrc, offset } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.raw.buffer.atomic.cmpswap`. +pub fn raw_buffer_atomic_cmpswap( + ctx: *mlir.Context, + src: *const mlir.Value, + cmp: *const mlir.Value, + rsrc: *const mlir.Value, + offset: *const mlir.Value, + soffset: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + aux: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.raw.buffer.atomic.cmpswap", .{ + .operands = .{ .flat = &.{ src, cmp, rsrc, offset, soffset } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.raw.buffer.atomic.fadd`. +pub fn raw_buffer_atomic_fadd( + ctx: *mlir.Context, + vdata: *const mlir.Value, + rsrc: *const mlir.Value, + offset: *const mlir.Value, + soffset: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + aux: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.raw.buffer.atomic.fadd", .{ + .operands = .{ .flat = &.{ vdata, rsrc, offset, soffset } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.raw.buffer.atomic.fmax`. +pub fn raw_buffer_atomic_fmax( + ctx: *mlir.Context, + vdata: *const mlir.Value, + rsrc: *const mlir.Value, + offset: *const mlir.Value, + soffset: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + aux: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.raw.buffer.atomic.fmax", .{ + .operands = .{ .flat = &.{ vdata, rsrc, offset, soffset } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.raw.buffer.atomic.smax`. +pub fn raw_buffer_atomic_smax( + ctx: *mlir.Context, + vdata: *const mlir.Value, + rsrc: *const mlir.Value, + offset: *const mlir.Value, + soffset: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + aux: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.raw.buffer.atomic.smax", .{ + .operands = .{ .flat = &.{ vdata, rsrc, offset, soffset } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.raw.buffer.atomic.umin`. +pub fn raw_buffer_atomic_umin( + ctx: *mlir.Context, + vdata: *const mlir.Value, + rsrc: *const mlir.Value, + offset: *const mlir.Value, + soffset: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + aux: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.raw.buffer.atomic.umin", .{ + .operands = .{ .flat = &.{ vdata, rsrc, offset, soffset } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.raw.buffer.load`. +pub fn raw_buffer_load( + ctx: *mlir.Context, + rsrc: *const mlir.Value, + offset: *const mlir.Value, + soffset: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + aux: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.raw.buffer.load", .{ + .operands = .{ .flat = &.{ rsrc, offset, soffset } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.raw.buffer.store`. +pub fn raw_buffer_store( + ctx: *mlir.Context, + vdata: *const mlir.Value, + rsrc: *const mlir.Value, + offset: *const mlir.Value, + soffset: *const mlir.Value, + opts: struct { + aux: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.raw.buffer.store", .{ + .operands = .{ .flat = &.{ vdata, rsrc, offset, soffset } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.raw.ptr.buffer.atomic.cmpswap`. +pub fn raw_ptr_buffer_atomic_cmpswap( + ctx: *mlir.Context, + src: *const mlir.Value, + cmp: *const mlir.Value, + rsrc: *const mlir.Value, + offset: *const mlir.Value, + soffset: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.raw.ptr.buffer.atomic.cmpswap", .{ + .operands = .{ .flat = &.{ src, cmp, rsrc, offset, soffset } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.raw.ptr.buffer.atomic.fadd`. +pub fn raw_ptr_buffer_atomic_fadd( + ctx: *mlir.Context, + vdata: *const mlir.Value, + rsrc: *const mlir.Value, + offset: *const mlir.Value, + soffset: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.raw.ptr.buffer.atomic.fadd", .{ + .operands = .{ .flat = &.{ vdata, rsrc, offset, soffset } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.raw.ptr.buffer.atomic.fmax`. +pub fn raw_ptr_buffer_atomic_fmax( + ctx: *mlir.Context, + vdata: *const mlir.Value, + rsrc: *const mlir.Value, + offset: *const mlir.Value, + soffset: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.raw.ptr.buffer.atomic.fmax", .{ + .operands = .{ .flat = &.{ vdata, rsrc, offset, soffset } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.raw.ptr.buffer.atomic.smax`. +pub fn raw_ptr_buffer_atomic_smax( + ctx: *mlir.Context, + vdata: *const mlir.Value, + rsrc: *const mlir.Value, + offset: *const mlir.Value, + soffset: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.raw.ptr.buffer.atomic.smax", .{ + .operands = .{ .flat = &.{ vdata, rsrc, offset, soffset } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.raw.ptr.buffer.atomic.umin`. +pub fn raw_ptr_buffer_atomic_umin( + ctx: *mlir.Context, + vdata: *const mlir.Value, + rsrc: *const mlir.Value, + offset: *const mlir.Value, + soffset: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.raw.ptr.buffer.atomic.umin", .{ + .operands = .{ .flat = &.{ vdata, rsrc, offset, soffset } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.raw.ptr.buffer.load`. +pub fn raw_ptr_buffer_load( + ctx: *mlir.Context, + rsrc: *const mlir.Value, + offset: *const mlir.Value, + soffset: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.raw.ptr.buffer.load", .{ + .operands = .{ .flat = &.{ rsrc, offset, soffset } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.raw.ptr.buffer.load.async.lds`. Async variant of raw.ptr.buffer.load.lds +pub fn raw_ptr_buffer_load_async_lds( + ctx: *mlir.Context, + rsrc: *const mlir.Value, + ldsPtr: *const mlir.Value, + size: *const mlir.Value, + voffset: *const mlir.Value, + soffset: *const mlir.Value, + offset: *const mlir.Value, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.raw.ptr.buffer.load.async.lds", .{ + .operands = .{ .flat = &.{ rsrc, ldsPtr, size, voffset, soffset, offset } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.raw.ptr.buffer.load.lds`. +pub fn raw_ptr_buffer_load_lds( + ctx: *mlir.Context, + rsrc: *const mlir.Value, + ldsPtr: *const mlir.Value, + size: *const mlir.Value, + voffset: *const mlir.Value, + soffset: *const mlir.Value, + offset: *const mlir.Value, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.raw.ptr.buffer.load.lds", .{ + .operands = .{ .flat = &.{ rsrc, ldsPtr, size, voffset, soffset, offset } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.raw.ptr.buffer.store`. +pub fn raw_ptr_buffer_store( + ctx: *mlir.Context, + vdata: *const mlir.Value, + rsrc: *const mlir.Value, + offset: *const mlir.Value, + soffset: *const mlir.Value, + opts: struct { + aux: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (opts.aux) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.raw.ptr.buffer.store", .{ + .operands = .{ .flat = &.{ vdata, rsrc, offset, soffset } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.rcp`. +pub fn rcp( + ctx: *mlir.Context, + arg: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.rcp", .{ + .operands = .{ .flat = &.{arg} }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.readfirstlane`. Get the value in first active lane. +pub fn readfirstlane( + ctx: *mlir.Context, + src: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.readfirstlane", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.readlane`. Get the value in the specific lane. +pub fn readlane( + ctx: *mlir.Context, + src0: *const mlir.Value, + src1: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.readlane", .{ + .operands = .{ .flat = &.{ src0, src1 } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.rsq`. +pub fn rsq( + ctx: *mlir.Context, + arg: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.rsq", .{ + .operands = .{ .flat = &.{arg} }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.s.barrier`. +pub fn s_barrier( + ctx: *mlir.Context, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.s.barrier", .{ + .location = location, + }); +} + +/// `rocdl.s.barrier.init`. +pub fn s_barrier_init( + ctx: *mlir.Context, + ptr: *const mlir.Value, + memberCnt: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "memberCnt", memberCnt)); + return mlir.Operation.make(ctx, "rocdl.s.barrier.init", .{ + .operands = .{ .flat = &.{ptr} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.s.barrier.join`. +pub fn s_barrier_join( + ctx: *mlir.Context, + ptr: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.s.barrier.join", .{ + .operands = .{ .flat = &.{ptr} }, + .location = location, + }); +} + +/// `rocdl.s.barrier.leave`. +pub fn s_barrier_leave( + ctx: *mlir.Context, + id: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "id", id)); + return mlir.Operation.make(ctx, "rocdl.s.barrier.leave", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.s.barrier.signal`. +pub fn s_barrier_signal( + ctx: *mlir.Context, + id: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "id", id)); + return mlir.Operation.make(ctx, "rocdl.s.barrier.signal", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.s.barrier.signal.isfirst`. +pub fn s_barrier_signal_isfirst( + ctx: *mlir.Context, + res_type: *const mlir.Type, + id: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "id", id)); + return mlir.Operation.make(ctx, "rocdl.s.barrier.signal.isfirst", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.s.barrier.signal.var`. +pub fn s_barrier_signal_var( + ctx: *mlir.Context, + ptr: *const mlir.Value, + memberCnt: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "memberCnt", memberCnt)); + return mlir.Operation.make(ctx, "rocdl.s.barrier.signal.var", .{ + .operands = .{ .flat = &.{ptr} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.s.barrier.wait`. +pub fn s_barrier_wait( + ctx: *mlir.Context, + id: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "id", id)); + return mlir.Operation.make(ctx, "rocdl.s.barrier.wait", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.s.get.barrier.state`. +pub fn s_get_barrier_state( + ctx: *mlir.Context, + res_type: *const mlir.Type, + id: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "id", id)); + return mlir.Operation.make(ctx, "rocdl.s.get.barrier.state", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.s.get.named.barrier.state`. +pub fn s_get_named_barrier_state( + ctx: *mlir.Context, + ptr: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.s.get.named.barrier.state", .{ + .operands = .{ .flat = &.{ptr} }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.s.nop`. +pub fn s_nop( + ctx: *mlir.Context, + count: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "count", count)); + return mlir.Operation.make(ctx, "rocdl.s.nop", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.s.setprio`. +pub fn s_setprio( + ctx: *mlir.Context, + priority: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "priority", priority)); + return mlir.Operation.make(ctx, "rocdl.s.setprio", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.s.sleep`. +pub fn s_sleep( + ctx: *mlir.Context, + count: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "count", count)); + return mlir.Operation.make(ctx, "rocdl.s.sleep", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.s.wait.asynccnt`. Wait until ASYNCCNT is less than or equal to `count` +pub fn s_wait_asynccnt( + ctx: *mlir.Context, + count: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "count", count)); + return mlir.Operation.make(ctx, "rocdl.s.wait.asynccnt", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.s.wait.dscnt`. Wait until DSCNT is less than or equal to `count` +pub fn s_wait_dscnt( + ctx: *mlir.Context, + count: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "count", count)); + return mlir.Operation.make(ctx, "rocdl.s.wait.dscnt", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.s.wait.expcnt`. Wait until EXPCNT is less than or equal to `count` +pub fn s_wait_expcnt( + ctx: *mlir.Context, + count: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "count", count)); + return mlir.Operation.make(ctx, "rocdl.s.wait.expcnt", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.s.wait.loadcnt`. Wait until LOADCNT is less than or equal to `count` +pub fn s_wait_loadcnt( + ctx: *mlir.Context, + count: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "count", count)); + return mlir.Operation.make(ctx, "rocdl.s.wait.loadcnt", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.s.wait.storecnt`. Wait until STORECNT is less than or equal to `count` +pub fn s_wait_storecnt( + ctx: *mlir.Context, + count: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "count", count)); + return mlir.Operation.make(ctx, "rocdl.s.wait.storecnt", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.s.wait.tensorcnt`. Wait until TENSORCNT is less than or equal to `count` +pub fn s_wait_tensorcnt( + ctx: *mlir.Context, + count: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "count", count)); + return mlir.Operation.make(ctx, "rocdl.s.wait.tensorcnt", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.s.waitcnt`. +pub fn s_waitcnt( + ctx: *mlir.Context, + bitfield: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "bitfield", bitfield)); + return mlir.Operation.make(ctx, "rocdl.s.waitcnt", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.s.wakeup.barrier`. +pub fn s_wakeup_barrier( + ctx: *mlir.Context, + ptr: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.s.wakeup.barrier", .{ + .operands = .{ .flat = &.{ptr} }, + .location = location, + }); +} + +/// `rocdl.sched.barrier`. +pub fn sched_barrier( + ctx: *mlir.Context, + mask: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "mask", mask)); + return mlir.Operation.make(ctx, "rocdl.sched.barrier", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.sched.group.barrier`. +pub fn sched_group_barrier( + ctx: *mlir.Context, + mask: *const mlir.Attribute, + size: *const mlir.Attribute, + groupId: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "mask", mask)); + attributes.appendAssumeCapacity(.named(ctx, "size", size)); + attributes.appendAssumeCapacity(.named(ctx, "groupId", groupId)); + return mlir.Operation.make(ctx, "rocdl.sched.group.barrier", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.sdot2`. +pub fn sdot2( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + clamp: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.clamp) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.sdot2", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.sdot4`. +pub fn sdot4( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + clamp: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.clamp) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.sdot4", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.sdot8`. +pub fn sdot8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + clamp: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.clamp) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.sdot8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.sin`. +pub fn sin( + ctx: *mlir.Context, + arg: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.sin", .{ + .operands = .{ .flat = &.{arg} }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.smfmac.f32.16x16x128.bf8.bf8`. +pub fn smfmac_f32_16x16x128_bf8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.16x16x128.bf8.bf8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.16x16x128.bf8.fp8`. +pub fn smfmac_f32_16x16x128_bf8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.16x16x128.bf8.fp8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.16x16x128.fp8.bf8`. +pub fn smfmac_f32_16x16x128_fp8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.16x16x128.fp8.bf8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.16x16x128.fp8.fp8`. +pub fn smfmac_f32_16x16x128_fp8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.16x16x128.fp8.fp8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.16x16x32.bf16`. +pub fn smfmac_f32_16x16x32_bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.16x16x32.bf16", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.16x16x32.f16`. +pub fn smfmac_f32_16x16x32_f16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.16x16x32.f16", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.16x16x64.bf16`. +pub fn smfmac_f32_16x16x64_bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.16x16x64.bf16", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.16x16x64.bf8.bf8`. +pub fn smfmac_f32_16x16x64_bf8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.16x16x64.bf8.bf8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.16x16x64.bf8.fp8`. +pub fn smfmac_f32_16x16x64_bf8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.16x16x64.bf8.fp8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.16x16x64.f16`. +pub fn smfmac_f32_16x16x64_f16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.16x16x64.f16", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.16x16x64.fp8.bf8`. +pub fn smfmac_f32_16x16x64_fp8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.16x16x64.fp8.bf8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.16x16x64.fp8.fp8`. +pub fn smfmac_f32_16x16x64_fp8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.16x16x64.fp8.fp8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.32x32x16.bf16`. +pub fn smfmac_f32_32x32x16_bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.32x32x16.bf16", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.32x32x16.f16`. +pub fn smfmac_f32_32x32x16_f16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.32x32x16.f16", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.32x32x32.bf16`. +pub fn smfmac_f32_32x32x32_bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.32x32x32.bf16", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.32x32x32.bf8.bf8`. +pub fn smfmac_f32_32x32x32_bf8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.32x32x32.bf8.bf8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.32x32x32.bf8.fp8`. +pub fn smfmac_f32_32x32x32_bf8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.32x32x32.bf8.fp8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.32x32x32.f16`. +pub fn smfmac_f32_32x32x32_f16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.32x32x32.f16", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.32x32x32.fp8.bf8`. +pub fn smfmac_f32_32x32x32_fp8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.32x32x32.fp8.bf8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.32x32x32.fp8.fp8`. +pub fn smfmac_f32_32x32x32_fp8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.32x32x32.fp8.fp8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.32x32x64.bf8.bf8`. +pub fn smfmac_f32_32x32x64_bf8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.32x32x64.bf8.bf8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.32x32x64.bf8.fp8`. +pub fn smfmac_f32_32x32x64_bf8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.32x32x64.bf8.fp8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.32x32x64.fp8.bf8`. +pub fn smfmac_f32_32x32x64_fp8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.32x32x64.fp8.bf8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.f32.32x32x64.fp8.fp8`. +pub fn smfmac_f32_32x32x64_fp8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.f32.32x32x64.fp8.fp8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.i32.16x16x128.i8`. +pub fn smfmac_i32_16x16x128_i8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.i32.16x16x128.i8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.i32.16x16x64.i8`. +pub fn smfmac_i32_16x16x64_i8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.i32.16x16x64.i8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.i32.32x32x32.i8`. +pub fn smfmac_i32_32x32x32_i8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.i32.32x32x32.i8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.smfmac.i32.32x32x64.i8`. +pub fn smfmac_i32_32x32x64_i8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + cbsz: *const mlir.Attribute, + abid: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); + attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); + return mlir.Operation.make(ctx, "rocdl.smfmac.i32.32x32x64.i8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.sqrt`. +pub fn sqrt( + ctx: *mlir.Context, + arg: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.sqrt", .{ + .operands = .{ .flat = &.{arg} }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.sudot4`. +pub fn sudot4( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + signA: ?*const mlir.Attribute = null, + signB: ?*const mlir.Attribute = null, + clamp: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.signA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signA", attribute)); + } + if (opts.signB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signB", attribute)); + } + if (opts.clamp) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.sudot4", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.sudot8`. +pub fn sudot8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + signA: ?*const mlir.Attribute = null, + signB: ?*const mlir.Attribute = null, + clamp: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.signA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signA", attribute)); + } + if (opts.signB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signB", attribute)); + } + if (opts.clamp) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.sudot8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.swmmac.bf16.16x16x32.bf16`. +pub fn swmmac_bf16_16x16x32_bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.swmmac.bf16.16x16x32.bf16", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.swmmac.bf16.16x16x64.bf16`. +pub fn swmmac_bf16_16x16x64_bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + signA: ?*const mlir.Attribute = null, + signB: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (opts.signA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signA", attribute)); + } + if (opts.signB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signB", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.swmmac.bf16.16x16x64.bf16", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.swmmac.bf16f32.16x16x64.bf16`. +pub fn swmmac_bf16f32_16x16x64_bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + signA: ?*const mlir.Attribute = null, + signB: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (opts.signA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signA", attribute)); + } + if (opts.signB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signB", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.swmmac.bf16f32.16x16x64.bf16", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.swmmac.f16.16x16x128.bf8.bf8`. +pub fn swmmac_f16_16x16x128_bf8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.swmmac.f16.16x16x128.bf8.bf8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.swmmac.f16.16x16x128.bf8.fp8`. +pub fn swmmac_f16_16x16x128_bf8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.swmmac.f16.16x16x128.bf8.fp8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.swmmac.f16.16x16x128.fp8.bf8`. +pub fn swmmac_f16_16x16x128_fp8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.swmmac.f16.16x16x128.fp8.bf8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.swmmac.f16.16x16x128.fp8.fp8`. +pub fn swmmac_f16_16x16x128_fp8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.swmmac.f16.16x16x128.fp8.fp8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.swmmac.f16.16x16x32.f16`. +pub fn swmmac_f16_16x16x32_f16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.swmmac.f16.16x16x32.f16", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.swmmac.f16.16x16x64.f16`. +pub fn swmmac_f16_16x16x64_f16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + signA: ?*const mlir.Attribute = null, + signB: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (opts.signA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signA", attribute)); + } + if (opts.signB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signB", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.swmmac.f16.16x16x64.f16", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.swmmac.f32.16x16x128.bf8.bf8`. +pub fn swmmac_f32_16x16x128_bf8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.swmmac.f32.16x16x128.bf8.bf8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.swmmac.f32.16x16x128.bf8.fp8`. +pub fn swmmac_f32_16x16x128_bf8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.swmmac.f32.16x16x128.bf8.fp8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.swmmac.f32.16x16x128.fp8.bf8`. +pub fn swmmac_f32_16x16x128_fp8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.swmmac.f32.16x16x128.fp8.bf8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.swmmac.f32.16x16x128.fp8.fp8`. +pub fn swmmac_f32_16x16x128_fp8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.swmmac.f32.16x16x128.fp8.fp8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.swmmac.f32.16x16x32.bf16`. +pub fn swmmac_f32_16x16x32_bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.swmmac.f32.16x16x32.bf16", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.swmmac.f32.16x16x32.bf8.bf8`. +pub fn swmmac_f32_16x16x32_bf8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.swmmac.f32.16x16x32.bf8.bf8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.swmmac.f32.16x16x32.bf8.fp8`. +pub fn swmmac_f32_16x16x32_bf8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.swmmac.f32.16x16x32.bf8.fp8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.swmmac.f32.16x16x32.f16`. +pub fn swmmac_f32_16x16x32_f16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.swmmac.f32.16x16x32.f16", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.swmmac.f32.16x16x32.fp8.bf8`. +pub fn swmmac_f32_16x16x32_fp8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.swmmac.f32.16x16x32.fp8.bf8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.swmmac.f32.16x16x32.fp8.fp8`. +pub fn swmmac_f32_16x16x32_fp8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.swmmac.f32.16x16x32.fp8.fp8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.swmmac.f32.16x16x64.bf16`. +pub fn swmmac_f32_16x16x64_bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + signA: ?*const mlir.Attribute = null, + signB: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (opts.signA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signA", attribute)); + } + if (opts.signB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signB", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.swmmac.f32.16x16x64.bf16", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.swmmac.f32.16x16x64.f16`. +pub fn swmmac_f32_16x16x64_f16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + signA: ?*const mlir.Attribute = null, + signB: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (opts.signA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signA", attribute)); + } + if (opts.signB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signB", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.swmmac.f32.16x16x64.f16", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.swmmac.i32.16x16x128.iu8`. +pub fn swmmac_i32_16x16x128_iu8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + signA: ?*const mlir.Attribute = null, + signB: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + clamp: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + if (opts.signA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signA", attribute)); + } + if (opts.signB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signB", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + if (opts.clamp) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.swmmac.i32.16x16x128.iu8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.swmmac.i32.16x16x32.iu4`. +pub fn swmmac_i32_16x16x32_iu4( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + signA: ?*const mlir.Attribute = null, + signB: ?*const mlir.Attribute = null, + clamp: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.signA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signA", attribute)); + } + if (opts.signB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signB", attribute)); + } + if (opts.clamp) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.swmmac.i32.16x16x32.iu4", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.swmmac.i32.16x16x32.iu8`. +pub fn swmmac_i32_16x16x32_iu8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + signA: ?*const mlir.Attribute = null, + signB: ?*const mlir.Attribute = null, + clamp: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.signA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signA", attribute)); + } + if (opts.signB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signB", attribute)); + } + if (opts.clamp) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.swmmac.i32.16x16x32.iu8", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.swmmac.i32.16x16x64.iu4`. +pub fn swmmac_i32_16x16x64_iu4( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + index: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + signA: ?*const mlir.Attribute = null, + signB: ?*const mlir.Attribute = null, + clamp: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.signA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signA", attribute)); + } + if (opts.signB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signB", attribute)); + } + if (opts.clamp) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.swmmac.i32.16x16x64.iu4", .{ + .operands = .{ .flat = &.{ a, b, c, index } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.tanh`. +pub fn tanh( + ctx: *mlir.Context, + arg: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.tanh", .{ + .operands = .{ .flat = &.{arg} }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.tensor.load.to.lds`. Base class for ROCDL tensor load/store to/from LDS. +pub fn tensor_load_to_lds( + ctx: *mlir.Context, + dgroup0: *const mlir.Value, + dgroup1: *const mlir.Value, + dgroup2: *const mlir.Value, + dgroup3: *const mlir.Value, + dgroup4: *const mlir.Value, + opts: struct { + cachePolicy: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (opts.cachePolicy) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "cachePolicy", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.tensor.load.to.lds", .{ + .operands = .{ .flat = &.{ dgroup0, dgroup1, dgroup2, dgroup3, dgroup4 } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.tensor.store.from.lds`. Base class for ROCDL tensor load/store to/from LDS. +pub fn tensor_store_from_lds( + ctx: *mlir.Context, + dgroup0: *const mlir.Value, + dgroup1: *const mlir.Value, + dgroup2: *const mlir.Value, + dgroup3: *const mlir.Value, + dgroup4: *const mlir.Value, + opts: struct { + cachePolicy: ?*const mlir.Attribute = null, + alias_scopes: ?*const mlir.Attribute = null, + noalias_scopes: ?*const mlir.Attribute = null, + tbaa: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + if (opts.cachePolicy) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "cachePolicy", attribute)); + } + if (opts.alias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); + } + if (opts.noalias_scopes) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); + } + if (opts.tbaa) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.tensor.store.from.lds", .{ + .operands = .{ .flat = &.{ dgroup0, dgroup1, dgroup2, dgroup3, dgroup4 } }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.udot2`. +pub fn udot2( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + clamp: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.clamp) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.udot2", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.udot4`. +pub fn udot4( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + clamp: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.clamp) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.udot4", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.udot8`. +pub fn udot8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + clamp: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.clamp) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.udot8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.update.dpp`. +pub fn update_dpp( + ctx: *mlir.Context, + old: *const mlir.Value, + src: *const mlir.Value, + res_type: *const mlir.Type, + dppCtrl: *const mlir.Attribute, + rowMask: *const mlir.Attribute, + bankMask: *const mlir.Attribute, + boundCtrl: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "dppCtrl", dppCtrl)); + attributes.appendAssumeCapacity(.named(ctx, "rowMask", rowMask)); + attributes.appendAssumeCapacity(.named(ctx, "bankMask", bankMask)); + attributes.appendAssumeCapacity(.named(ctx, "boundCtrl", boundCtrl)); + return mlir.Operation.make(ctx, "rocdl.update.dpp", .{ + .operands = .{ .flat = &.{ old, src } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wait.asyncmark`. Wait until N or fewer async operation groups are unexecuted +pub fn wait_asyncmark( + ctx: *mlir.Context, + count: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + attributes.appendAssumeCapacity(.named(ctx, "count", count)); + return mlir.Operation.make(ctx, "rocdl.wait.asyncmark", .{ + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wave.barrier`. +pub fn wave_barrier( + ctx: *mlir.Context, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.wave.barrier", .{ + .location = location, + }); +} + +/// `rocdl.wave.id`. +pub fn wave_id( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "range", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wave.id", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wavefrontsize`. +pub fn wavefrontsize( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "range", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wavefrontsize", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.bf16.16x16x16.bf16`. +pub fn wmma_bf16_16x16x16_bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + opsel: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.opsel) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "opsel", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.bf16.16x16x16.bf16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.bf16.16x16x32.bf16`. +pub fn wmma_bf16_16x16x32_bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.bf16.16x16x32.bf16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.bf16f32.16x16x32.bf16`. +pub fn wmma_bf16f32_16x16x32_bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.bf16f32.16x16x32.bf16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.f16.16x16x128.bf8_bf8`. +pub fn wmma_f16_16x16x128_bf8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.f16.16x16x128.bf8_bf8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.f16.16x16x128.bf8_fp8`. +pub fn wmma_f16_16x16x128_bf8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.f16.16x16x128.bf8_fp8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.f16.16x16x128.fp8_bf8`. +pub fn wmma_f16_16x16x128_fp8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.f16.16x16x128.fp8_bf8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.f16.16x16x128.fp8_fp8`. +pub fn wmma_f16_16x16x128_fp8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.f16.16x16x128.fp8_fp8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.f16.16x16x16.f16`. +pub fn wmma_f16_16x16x16_f16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + opsel: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.opsel) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "opsel", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.f16.16x16x16.f16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.f16.16x16x32.f16`. +pub fn wmma_f16_16x16x32_f16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.f16.16x16x32.f16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.f16.16x16x64.bf8_bf8`. +pub fn wmma_f16_16x16x64_bf8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.f16.16x16x64.bf8_bf8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.f16.16x16x64.bf8_fp8`. +pub fn wmma_f16_16x16x64_bf8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.f16.16x16x64.bf8_fp8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.f16.16x16x64.fp8_bf8`. +pub fn wmma_f16_16x16x64_fp8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.f16.16x16x64.fp8_bf8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.f16.16x16x64.fp8_fp8`. +pub fn wmma_f16_16x16x64_fp8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.f16.16x16x64.fp8_fp8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.f32.16x16x128.bf8_bf8`. +pub fn wmma_f32_16x16x128_bf8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x128.bf8_bf8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.f32.16x16x128.bf8_fp8`. +pub fn wmma_f32_16x16x128_bf8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x128.bf8_fp8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.f32.16x16x128.fp8_bf8`. +pub fn wmma_f32_16x16x128_fp8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x128.fp8_bf8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.f32.16x16x128.fp8_fp8`. +pub fn wmma_f32_16x16x128_fp8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x128.fp8_fp8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.f32.16x16x16.bf16`. +pub fn wmma_f32_16x16x16_bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x16.bf16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.wmma.f32.16x16x16.bf8_bf8`. +pub fn wmma_f32_16x16x16_bf8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x16.bf8_bf8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.wmma.f32.16x16x16.bf8_fp8`. +pub fn wmma_f32_16x16x16_bf8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x16.bf8_fp8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.wmma.f32.16x16x16.f16`. +pub fn wmma_f32_16x16x16_f16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x16.f16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.wmma.f32.16x16x16.fp8_bf8`. +pub fn wmma_f32_16x16x16_fp8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x16.fp8_bf8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.wmma.f32.16x16x16.fp8_fp8`. +pub fn wmma_f32_16x16x16_fp8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x16.fp8_fp8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .location = location, + }); +} + +/// `rocdl.wmma.f32.16x16x32.bf16`. +pub fn wmma_f32_16x16x32_bf16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x32.bf16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.f32.16x16x32.f16`. +pub fn wmma_f32_16x16x32_f16( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x32.f16", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.f32.16x16x4.f32`. +pub fn wmma_f32_16x16x4_f32( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x4.f32", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.f32.16x16x64.bf8_bf8`. +pub fn wmma_f32_16x16x64_bf8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x64.bf8_bf8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.f32.16x16x64.bf8_fp8`. +pub fn wmma_f32_16x16x64_bf8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x64.bf8_fp8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.f32.16x16x64.fp8_bf8`. +pub fn wmma_f32_16x16x64_fp8_bf8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x64.fp8_bf8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.f32.16x16x64.fp8_fp8`. +pub fn wmma_f32_16x16x64_fp8_fp8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x64.fp8_fp8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.i32.16x16x16.iu4`. +pub fn wmma_i32_16x16x16_iu4( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + signA: ?*const mlir.Attribute = null, + signB: ?*const mlir.Attribute = null, + clamp: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.signA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signA", attribute)); + } + if (opts.signB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signB", attribute)); + } + if (opts.clamp) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.i32.16x16x16.iu4", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.i32.16x16x16.iu8`. +pub fn wmma_i32_16x16x16_iu8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + signA: ?*const mlir.Attribute = null, + signB: ?*const mlir.Attribute = null, + clamp: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.signA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signA", attribute)); + } + if (opts.signB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signB", attribute)); + } + if (opts.clamp) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.i32.16x16x16.iu8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.i32.16x16x32.iu4`. +pub fn wmma_i32_16x16x32_iu4( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + signA: ?*const mlir.Attribute = null, + signB: ?*const mlir.Attribute = null, + clamp: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + if (opts.signA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signA", attribute)); + } + if (opts.signB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signB", attribute)); + } + if (opts.clamp) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.i32.16x16x32.iu4", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.i32.16x16x64.iu8`. +pub fn wmma_i32_16x16x64_iu8( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + signA: ?*const mlir.Attribute = null, + signB: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + clamp: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + if (opts.signA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signA", attribute)); + } + if (opts.signB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "signB", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + if (opts.clamp) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.i32.16x16x64.iu8", .{ + .operands = .{ .flat = &.{ a, b, c } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.scale.f32.16x16x128.f8f6f4`. +pub fn wmma_scale_f32_16x16x128_f8f6f4( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + scaleA: *const mlir.Value, + scaleB: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + fmtA: ?*const mlir.Attribute = null, + fmtB: ?*const mlir.Attribute = null, + modC: ?*const mlir.Attribute = null, + scaleAType: ?*const mlir.Attribute = null, + fmtScaleA: ?*const mlir.Attribute = null, + scaleBType: ?*const mlir.Attribute = null, + fmtScaleB: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 9) = .empty; + if (opts.fmtA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "fmtA", attribute)); + } + if (opts.fmtB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "fmtB", attribute)); + } + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.scaleAType) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "scaleAType", attribute)); + } + if (opts.fmtScaleA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "fmtScaleA", attribute)); + } + if (opts.scaleBType) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "scaleBType", attribute)); + } + if (opts.fmtScaleB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "fmtScaleB", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.scale.f32.16x16x128.f8f6f4", .{ + .operands = .{ .flat = &.{ a, b, c, scaleA, scaleB } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.scale.f32.32x16x128.f4`. +pub fn wmma_scale_f32_32x16x128_f4( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + scaleA: *const mlir.Value, + scaleB: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + scaleAType: ?*const mlir.Attribute = null, + fmtScaleA: ?*const mlir.Attribute = null, + scaleBType: ?*const mlir.Attribute = null, + fmtScaleB: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 7) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.scaleAType) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "scaleAType", attribute)); + } + if (opts.fmtScaleA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "fmtScaleA", attribute)); + } + if (opts.scaleBType) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "scaleBType", attribute)); + } + if (opts.fmtScaleB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "fmtScaleB", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.scale.f32.32x16x128.f4", .{ + .operands = .{ .flat = &.{ a, b, c, scaleA, scaleB } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.scale16.f32.16x16x128.f8f6f4`. +pub fn wmma_scale16_f32_16x16x128_f8f6f4( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + scaleA: *const mlir.Value, + scaleB: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + fmtA: ?*const mlir.Attribute = null, + fmtB: ?*const mlir.Attribute = null, + modC: ?*const mlir.Attribute = null, + scaleAType: ?*const mlir.Attribute = null, + fmtScaleA: ?*const mlir.Attribute = null, + scaleBType: ?*const mlir.Attribute = null, + fmtScaleB: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 9) = .empty; + if (opts.fmtA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "fmtA", attribute)); + } + if (opts.fmtB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "fmtB", attribute)); + } + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.scaleAType) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "scaleAType", attribute)); + } + if (opts.fmtScaleA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "fmtScaleA", attribute)); + } + if (opts.scaleBType) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "scaleBType", attribute)); + } + if (opts.fmtScaleB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "fmtScaleB", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.scale16.f32.16x16x128.f8f6f4", .{ + .operands = .{ .flat = &.{ a, b, c, scaleA, scaleB } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.wmma.scale16.f32.32x16x128.f4`. +pub fn wmma_scale16_f32_32x16x128_f4( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c: *const mlir.Value, + scaleA: *const mlir.Value, + scaleB: *const mlir.Value, + res_type: *const mlir.Type, + opts: struct { + modC: ?*const mlir.Attribute = null, + scaleAType: ?*const mlir.Attribute = null, + fmtScaleA: ?*const mlir.Attribute = null, + scaleBType: ?*const mlir.Attribute = null, + fmtScaleB: ?*const mlir.Attribute = null, + reuseA: ?*const mlir.Attribute = null, + reuseB: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 7) = .empty; + if (opts.modC) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); + } + if (opts.scaleAType) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "scaleAType", attribute)); + } + if (opts.fmtScaleA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "fmtScaleA", attribute)); + } + if (opts.scaleBType) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "scaleBType", attribute)); + } + if (opts.fmtScaleB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "fmtScaleB", attribute)); + } + if (opts.reuseA) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); + } + if (opts.reuseB) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.wmma.scale16.f32.32x16x128.f4", .{ + .operands = .{ .flat = &.{ a, b, c, scaleA, scaleB } }, + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.workgroup.id.x`. +pub fn workgroup_id_x( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "range", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.workgroup.id.x", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.workgroup.id.y`. +pub fn workgroup_id_y( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "range", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.workgroup.id.y", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.workgroup.id.z`. +pub fn workgroup_id_z( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "range", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.workgroup.id.z", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.workitem.id.x`. +pub fn workitem_id_x( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "range", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.workitem.id.x", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.workitem.id.y`. +pub fn workitem_id_y( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "range", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.workitem.id.y", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} + +/// `rocdl.workitem.id.z`. +pub fn workitem_id_z( + ctx: *mlir.Context, + res_type: *const mlir.Type, + opts: struct { + range: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.range) |attribute| { + attributes.appendAssumeCapacity(.named(ctx, "range", attribute)); + } + return mlir.Operation.make(ctx, "rocdl.workitem.id.z", .{ + .results = .{ .flat = &.{res_type} }, + .attributes = attributes.constSlice(), + .location = location, + }); +} diff --git a/mlir/dialects/rocdl/rocdl.zig b/mlir/dialects/rocdl/rocdl.zig new file mode 100644 index 00000000..456ad494 --- /dev/null +++ b/mlir/dialects/rocdl/rocdl.zig @@ -0,0 +1,56 @@ +//! LLVM ROCDL operations, AMDGPU attributes, and GPU target configuration. +//! All operation builders from the pinned LLVM revision are available in `ops`. + +const mlir = @import("mlir"); + +pub const types = @import("types.zig"); +pub const annotations = @import("annotations.zig"); +pub const AddressSpace = types.AddressSpace; +pub const pointerType = types.pointerType; + +pub const ops = @import("ops.zig"); +pub const enums = @import("enums.zig"); +pub const attributes = @import("attributes.zig"); + +pub const BufferOOBModeAttr = attributes.BufferOOBModeAttr; +pub const BufferOOBModeModuleFlagAttr = attributes.BufferOOBModeModuleFlagAttr; +pub const Gfx12AtomicCachePolicyAttr = attributes.Gfx12AtomicCachePolicyAttr; +pub const Gfx12CachePolicyAttr = attributes.Gfx12CachePolicyAttr; +pub const Gfx942CachePolicyAttr = attributes.Gfx942CachePolicyAttr; +pub const MFMANegModifierAttr = attributes.MFMANegModifierAttr; +pub const MFMAPermBAttr = attributes.MFMAPermBAttr; +pub const MatrixFormatAttr = attributes.MatrixFormatAttr; +pub const PreGfx12CachePolicyAttr = attributes.PreGfx12CachePolicyAttr; +pub const SchedGroupMaskAttr = attributes.SchedGroupMaskAttr; +pub const TBufferOOBModeModuleFlagAttr = attributes.TBufferOOBModeModuleFlagAttr; +pub const TargetAttr = attributes.TargetAttr; +pub const WMMACModifierAttr = attributes.WMMACModifierAttr; +pub const WMMAMatrixScaleAttr = attributes.WMMAMatrixScaleAttr; +pub const WMMAMatrixScaleFormatAttr = attributes.WMMAMatrixScaleFormatAttr; + +pub const BufferOOBMode = enums.BufferOOBMode; +pub const Gfx12AtomicCachePolicy = enums.Gfx12AtomicCachePolicy; +pub const Gfx12CachePolicy = enums.Gfx12CachePolicy; +pub const Gfx942CachePolicy = enums.Gfx942CachePolicy; +pub const MFMANegModifier = enums.MFMANegModifier; +pub const MFMAPermB = enums.MFMAPermB; +pub const MatrixFormat = enums.MatrixFormat; +pub const PreGfx12CachePolicy = enums.PreGfx12CachePolicy; +pub const SchedGroupMask = enums.SchedGroupMask; +pub const WMMACModifier = enums.WMMACModifier; +pub const WMMAMatrixScale = enums.WMMAMatrixScale; +pub const WMMAMatrixScaleFormat = enums.WMMAMatrixScaleFormat; + +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("rocdl"); +} + +/// Converts two packed FP8 values from the selected word to vector<2xf32>. +pub fn cvtPkF32Fp8( + ctx: *mlir.Context, + bits: *const mlir.Value, + high: bool, + location: *const mlir.Location, +) *mlir.Operation { + return ops.cvt_pk_f32_fp8(ctx, bits, .vector(&.{2}, .float(ctx, .f32)), .boolean(ctx, high), location); +} diff --git a/mlir/dialects/rocdl/target.zig b/mlir/dialects/rocdl/target.zig new file mode 100644 index 00000000..30311dc5 --- /dev/null +++ b/mlir/dialects/rocdl/target.zig @@ -0,0 +1,100 @@ +const c = @import("c"); +const mlir = @import("mlir"); + +/// GPU compilation target, `#rocdl.target`. +pub const TargetAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsAROCDLTarget; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + optLevel: i32 = 2, + triple: []const u8 = "amdgcn-amd-amdhsa", + chip: []const u8 = "gfx900", + features: []const u8 = "", + abi: []const u8 = "600", + flags: ?*const mlir.Attribute = null, + link: ?*const mlir.Attribute = null, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + return @ptrCast(c.mlirROCDLTargetAttrGet( + ctx.ptr(), + args.optLevel, + mlir.stringRef(args.triple), + mlir.stringRef(args.chip), + mlir.stringRef(args.features), + mlir.stringRef(args.abi), + .{ .ptr = if (args.flags) |a| @constCast(a.ptr().ptr) else null }, + .{ .ptr = if (args.link) |a| @constCast(a.ptr().ptr) else null }, + ).ptr orelse return error.InvalidMlir); + } + + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + + pub fn optLevel(self: *const Self) i32 { + return c.mlirROCDLTargetAttrGetOptLevel(self.ptr()); + } + + pub fn triple(self: *const Self) []const u8 { + return string(c.mlirROCDLTargetAttrGetTriple(self.ptr())); + } + + pub fn chip(self: *const Self) []const u8 { + return string(c.mlirROCDLTargetAttrGetChip(self.ptr())); + } + + pub fn features(self: *const Self) []const u8 { + return string(c.mlirROCDLTargetAttrGetFeatures(self.ptr())); + } + + pub fn abi(self: *const Self) []const u8 { + return string(c.mlirROCDLTargetAttrGetAbi(self.ptr())); + } + + pub fn flags(self: *const Self) ?*const mlir.Attribute { + return @ptrCast(c.mlirROCDLTargetAttrGetFlags(self.ptr()).ptr); + } + + pub fn link(self: *const Self) ?*const mlir.Attribute { + return @ptrCast(c.mlirROCDLTargetAttrGetLink(self.ptr()).ptr); + } + + pub fn hasFlag(self: *const Self, flag: []const u8) bool { + return c.mlirROCDLTargetAttrHasFlag(self.ptr(), mlir.stringRef(flag)); + } + + pub fn hasWave64(self: *const Self) bool { + return c.mlirROCDLTargetAttrHasWave64(self.ptr()); + } + + pub fn hasFastMath(self: *const Self) bool { + return c.mlirROCDLTargetAttrHasFastMath(self.ptr()); + } + + pub fn hasDaz(self: *const Self) bool { + return c.mlirROCDLTargetAttrHasDaz(self.ptr()); + } + + pub fn hasFiniteOnly(self: *const Self) bool { + return c.mlirROCDLTargetAttrHasFiniteOnly(self.ptr()); + } + + pub fn hasUnsafeMath(self: *const Self) bool { + return c.mlirROCDLTargetAttrHasUnsafeMath(self.ptr()); + } + + pub fn hasCorrectSqrt(self: *const Self) bool { + return c.mlirROCDLTargetAttrHasCorrectSqrt(self.ptr()); + } +}; + +fn string(ref: c.MlirStringRef) []const u8 { + return if (ref.length == 0) "" else mlir.string(ref); +} diff --git a/mlir/dialects/rocdl/test.zig b/mlir/dialects/rocdl/test.zig new file mode 100644 index 00000000..81f0b646 --- /dev/null +++ b/mlir/dialects/rocdl/test.zig @@ -0,0 +1,592 @@ +const std = @import("std"); + +const c = @import("c"); +const mlir = @import("mlir"); +const rocdl = @import("mlir/dialects/rocdl"); + +const operation_names: []const []const u8 = &.{ + "rocdl.asyncmark", + "rocdl.ballot", + "rocdl.barrier", + "rocdl.cluster.id.x", + "rocdl.cluster.id.y", + "rocdl.cluster.id.z", + "rocdl.cluster.load.async.to.lds.b128", + "rocdl.cluster.load.async.to.lds.b32", + "rocdl.cluster.load.async.to.lds.b64", + "rocdl.cluster.load.async.to.lds.b8", + "rocdl.cluster.workgroup.id.x", + "rocdl.cluster.workgroup.id.y", + "rocdl.cluster.workgroup.id.z", + "rocdl.cos", + "rocdl.cvt.f32.bf8", + "rocdl.cvt.f32.fp8", + "rocdl.cvt.pk.bf8.f32", + "rocdl.cvt.pk.f32.bf8", + "rocdl.cvt.pk.f32.fp8", + "rocdl.cvt.pk.fp8.f32", + "rocdl.cvt.pkrtz", + "rocdl.cvt.scale.pk16.bf16.bf6", + "rocdl.cvt.scale.pk16.bf16.fp6", + "rocdl.cvt.scale.pk16.f16.bf6", + "rocdl.cvt.scale.pk16.f16.fp6", + "rocdl.cvt.scale.pk16.f32.bf6", + "rocdl.cvt.scale.pk16.f32.fp6", + "rocdl.cvt.scale.pk8.bf16.bf8", + "rocdl.cvt.scale.pk8.bf16.fp4", + "rocdl.cvt.scale.pk8.bf16.fp8", + "rocdl.cvt.scale.pk8.f16.bf8", + "rocdl.cvt.scale.pk8.f16.fp4", + "rocdl.cvt.scale.pk8.f16.fp8", + "rocdl.cvt.scale.pk8.f32.bf8", + "rocdl.cvt.scale.pk8.f32.fp4", + "rocdl.cvt.scale.pk8.f32.fp8", + "rocdl.cvt.scalef32.2xpk16.bf6.f32", + "rocdl.cvt.scalef32.2xpk16.fp6.f32", + "rocdl.cvt.scalef32.f16.bf8", + "rocdl.cvt.scalef32.f16.fp8", + "rocdl.cvt.scalef32.f32.bf8", + "rocdl.cvt.scalef32.f32.fp8", + "rocdl.cvt.scalef32.pk.bf16.bf8", + "rocdl.cvt.scalef32.pk.bf16.fp4", + "rocdl.cvt.scalef32.pk.bf16.fp8", + "rocdl.cvt.scalef32.pk.bf8.bf16", + "rocdl.cvt.scalef32.pk.bf8.f16", + "rocdl.cvt.scalef32.pk.bf8.f32", + "rocdl.cvt.scalef32.pk.f16.bf8", + "rocdl.cvt.scalef32.pk.f16.fp4", + "rocdl.cvt.scalef32.pk.f16.fp8", + "rocdl.cvt.scalef32.pk.f32.bf8", + "rocdl.cvt.scalef32.pk.f32.fp4", + "rocdl.cvt.scalef32.pk.f32.fp8", + "rocdl.cvt.scalef32.pk.fp4.bf16", + "rocdl.cvt.scalef32.pk.fp4.f16", + "rocdl.cvt.scalef32.pk.fp4.f32", + "rocdl.cvt.scalef32.pk.fp8.bf16", + "rocdl.cvt.scalef32.pk.fp8.f16", + "rocdl.cvt.scalef32.pk.fp8.f32", + "rocdl.cvt.scalef32.pk16.bf6.bf16", + "rocdl.cvt.scalef32.pk16.bf6.f16", + "rocdl.cvt.scalef32.pk16.bf6.f32", + "rocdl.cvt.scalef32.pk16.fp6.bf16", + "rocdl.cvt.scalef32.pk16.fp6.f16", + "rocdl.cvt.scalef32.pk16.fp6.f32", + "rocdl.cvt.scalef32.pk32.bf16.bf6", + "rocdl.cvt.scalef32.pk32.bf16.fp6", + "rocdl.cvt.scalef32.pk32.bf6.bf16", + "rocdl.cvt.scalef32.pk32.bf6.f16", + "rocdl.cvt.scalef32.pk32.f16.bf6", + "rocdl.cvt.scalef32.pk32.f16.fp6", + "rocdl.cvt.scalef32.pk32.f32.bf6", + "rocdl.cvt.scalef32.pk32.f32.fp6", + "rocdl.cvt.scalef32.pk32.fp6.bf16", + "rocdl.cvt.scalef32.pk32.fp6.f16", + "rocdl.cvt.scalef32.pk8.bf8.bf16", + "rocdl.cvt.scalef32.pk8.bf8.f16", + "rocdl.cvt.scalef32.pk8.bf8.f32", + "rocdl.cvt.scalef32.pk8.fp4.bf16", + "rocdl.cvt.scalef32.pk8.fp4.f16", + "rocdl.cvt.scalef32.pk8.fp4.f32", + "rocdl.cvt.scalef32.pk8.fp8.bf16", + "rocdl.cvt.scalef32.pk8.fp8.f16", + "rocdl.cvt.scalef32.pk8.fp8.f32", + "rocdl.cvt.scalef32.sr.bf8.bf16", + "rocdl.cvt.scalef32.sr.bf8.f16", + "rocdl.cvt.scalef32.sr.bf8.f32", + "rocdl.cvt.scalef32.sr.fp8.bf16", + "rocdl.cvt.scalef32.sr.fp8.f16", + "rocdl.cvt.scalef32.sr.fp8.f32", + "rocdl.cvt.scalef32.sr.pk.fp4.bf16", + "rocdl.cvt.scalef32.sr.pk.fp4.f16", + "rocdl.cvt.scalef32.sr.pk.fp4.f32", + "rocdl.cvt.scalef32.sr.pk16.bf6.bf16", + "rocdl.cvt.scalef32.sr.pk16.bf6.f16", + "rocdl.cvt.scalef32.sr.pk16.bf6.f32", + "rocdl.cvt.scalef32.sr.pk16.fp6.bf16", + "rocdl.cvt.scalef32.sr.pk16.fp6.f16", + "rocdl.cvt.scalef32.sr.pk16.fp6.f32", + "rocdl.cvt.scalef32.sr.pk32.bf6.bf16", + "rocdl.cvt.scalef32.sr.pk32.bf6.f16", + "rocdl.cvt.scalef32.sr.pk32.bf6.f32", + "rocdl.cvt.scalef32.sr.pk32.fp6.bf16", + "rocdl.cvt.scalef32.sr.pk32.fp6.f16", + "rocdl.cvt.scalef32.sr.pk32.fp6.f32", + "rocdl.cvt.scalef32.sr.pk8.bf8.bf16", + "rocdl.cvt.scalef32.sr.pk8.bf8.f16", + "rocdl.cvt.scalef32.sr.pk8.bf8.f32", + "rocdl.cvt.scalef32.sr.pk8.fp4.bf16", + "rocdl.cvt.scalef32.sr.pk8.fp4.f16", + "rocdl.cvt.scalef32.sr.pk8.fp4.f32", + "rocdl.cvt.scalef32.sr.pk8.fp8.bf16", + "rocdl.cvt.scalef32.sr.pk8.fp8.f16", + "rocdl.cvt.scalef32.sr.pk8.fp8.f32", + "rocdl.cvt.sr.bf8.f32", + "rocdl.cvt.sr.fp8.f32", + "rocdl.dot4.f32.bf8.bf8", + "rocdl.dot4.f32.bf8.fp8", + "rocdl.dot4.f32.fp8.bf8", + "rocdl.dot4.f32.fp8.fp8", + "rocdl.ds.atomic.async.barrier.arrive.b64", + "rocdl.ds.atomic.barrier.arrive.rtn.b64", + "rocdl.ds.load.tr16.b128", + "rocdl.ds.load.tr4.b64", + "rocdl.ds.load.tr6.b96", + "rocdl.ds.load.tr8.b64", + "rocdl.ds.read.tr16.b64", + "rocdl.ds.read.tr4.b64", + "rocdl.ds.read.tr6.b96", + "rocdl.ds.read.tr8.b64", + "rocdl.ds_bpermute", + "rocdl.ds_swizzle", + "rocdl.exp", + "rocdl.exp2", + "rocdl.fdot2", + "rocdl.fdot2.bf16.bf16", + "rocdl.fdot2.f16.f16", + "rocdl.fdot2.f32.bf16", + "rocdl.flat.prefetch", + "rocdl.fmed3", + "rocdl.global.load.async.lds", + "rocdl.global.load.async.to.lds.b128", + "rocdl.global.load.async.to.lds.b32", + "rocdl.global.load.async.to.lds.b64", + "rocdl.global.load.async.to.lds.b8", + "rocdl.global.load.lds", + "rocdl.global.load.tr.b128", + "rocdl.global.load.tr.b64", + "rocdl.global.load.tr4.b64", + "rocdl.global.load.tr6.b96", + "rocdl.global.prefetch", + "rocdl.global.store.async.from.lds.b128", + "rocdl.global.store.async.from.lds.b32", + "rocdl.global.store.async.from.lds.b64", + "rocdl.global.store.async.from.lds.b8", + "rocdl.iglp.opt", + "rocdl.load.async.to.lds", + "rocdl.load.to.lds", + "rocdl.log", + "rocdl.make.buffer.rsrc", + "rocdl.mbcnt.hi", + "rocdl.mbcnt.lo", + "rocdl.mfma.f32.16x16x16bf16.1k", + "rocdl.mfma.f32.16x16x16f16", + "rocdl.mfma.f32.16x16x1f32", + "rocdl.mfma.f32.16x16x2bf16", + "rocdl.mfma.f32.16x16x32.bf16", + "rocdl.mfma.f32.16x16x32.bf8.bf8", + "rocdl.mfma.f32.16x16x32.bf8.fp8", + "rocdl.mfma.f32.16x16x32.f16", + "rocdl.mfma.f32.16x16x32.fp8.bf8", + "rocdl.mfma.f32.16x16x32.fp8.fp8", + "rocdl.mfma.f32.16x16x4bf16.1k", + "rocdl.mfma.f32.16x16x4f16", + "rocdl.mfma.f32.16x16x4f32", + "rocdl.mfma.f32.16x16x8.xf32", + "rocdl.mfma.f32.16x16x8bf16", + "rocdl.mfma.f32.32x32x16.bf16", + "rocdl.mfma.f32.32x32x16.bf8.bf8", + "rocdl.mfma.f32.32x32x16.bf8.fp8", + "rocdl.mfma.f32.32x32x16.f16", + "rocdl.mfma.f32.32x32x16.fp8.bf8", + "rocdl.mfma.f32.32x32x16.fp8.fp8", + "rocdl.mfma.f32.32x32x1f32", + "rocdl.mfma.f32.32x32x2bf16", + "rocdl.mfma.f32.32x32x2f32", + "rocdl.mfma.f32.32x32x4.xf32", + "rocdl.mfma.f32.32x32x4bf16", + "rocdl.mfma.f32.32x32x4bf16.1k", + "rocdl.mfma.f32.32x32x4f16", + "rocdl.mfma.f32.32x32x8bf16.1k", + "rocdl.mfma.f32.32x32x8f16", + "rocdl.mfma.f32.4x4x1f32", + "rocdl.mfma.f32.4x4x2bf16", + "rocdl.mfma.f32.4x4x4bf16.1k", + "rocdl.mfma.f32.4x4x4f16", + "rocdl.mfma.f64.16x16x4f64", + "rocdl.mfma.f64.4x4x4f64", + "rocdl.mfma.i32.16x16x16i8", + "rocdl.mfma.i32.16x16x32.i8", + "rocdl.mfma.i32.16x16x4i8", + "rocdl.mfma.i32.16x16x64.i8", + "rocdl.mfma.i32.32x32x16.i8", + "rocdl.mfma.i32.32x32x32.i8", + "rocdl.mfma.i32.32x32x4i8", + "rocdl.mfma.i32.32x32x8i8", + "rocdl.mfma.i32.4x4x4i8", + "rocdl.mfma.scale.f32.16x16x128.f8f6f4", + "rocdl.mfma.scale.f32.32x32x64.f8f6f4", + "rocdl.permlane16.swap", + "rocdl.permlane16.var", + "rocdl.permlane32.swap", + "rocdl.permlanex16", + "rocdl.permlanex16.var", + "rocdl.ptr.s.buffer.load", + "rocdl.raw.buffer.atomic.cmpswap", + "rocdl.raw.buffer.atomic.fadd", + "rocdl.raw.buffer.atomic.fmax", + "rocdl.raw.buffer.atomic.smax", + "rocdl.raw.buffer.atomic.umin", + "rocdl.raw.buffer.load", + "rocdl.raw.buffer.store", + "rocdl.raw.ptr.buffer.atomic.cmpswap", + "rocdl.raw.ptr.buffer.atomic.fadd", + "rocdl.raw.ptr.buffer.atomic.fmax", + "rocdl.raw.ptr.buffer.atomic.smax", + "rocdl.raw.ptr.buffer.atomic.umin", + "rocdl.raw.ptr.buffer.load", + "rocdl.raw.ptr.buffer.load.async.lds", + "rocdl.raw.ptr.buffer.load.lds", + "rocdl.raw.ptr.buffer.store", + "rocdl.rcp", + "rocdl.readfirstlane", + "rocdl.readlane", + "rocdl.rsq", + "rocdl.s.barrier", + "rocdl.s.barrier.init", + "rocdl.s.barrier.join", + "rocdl.s.barrier.leave", + "rocdl.s.barrier.signal", + "rocdl.s.barrier.signal.isfirst", + "rocdl.s.barrier.signal.var", + "rocdl.s.barrier.wait", + "rocdl.s.get.barrier.state", + "rocdl.s.get.named.barrier.state", + "rocdl.s.nop", + "rocdl.s.setprio", + "rocdl.s.sleep", + "rocdl.s.wait.asynccnt", + "rocdl.s.wait.dscnt", + "rocdl.s.wait.expcnt", + "rocdl.s.wait.loadcnt", + "rocdl.s.wait.storecnt", + "rocdl.s.wait.tensorcnt", + "rocdl.s.waitcnt", + "rocdl.s.wakeup.barrier", + "rocdl.sched.barrier", + "rocdl.sched.group.barrier", + "rocdl.sdot2", + "rocdl.sdot4", + "rocdl.sdot8", + "rocdl.sin", + "rocdl.smfmac.f32.16x16x128.bf8.bf8", + "rocdl.smfmac.f32.16x16x128.bf8.fp8", + "rocdl.smfmac.f32.16x16x128.fp8.bf8", + "rocdl.smfmac.f32.16x16x128.fp8.fp8", + "rocdl.smfmac.f32.16x16x32.bf16", + "rocdl.smfmac.f32.16x16x32.f16", + "rocdl.smfmac.f32.16x16x64.bf16", + "rocdl.smfmac.f32.16x16x64.bf8.bf8", + "rocdl.smfmac.f32.16x16x64.bf8.fp8", + "rocdl.smfmac.f32.16x16x64.f16", + "rocdl.smfmac.f32.16x16x64.fp8.bf8", + "rocdl.smfmac.f32.16x16x64.fp8.fp8", + "rocdl.smfmac.f32.32x32x16.bf16", + "rocdl.smfmac.f32.32x32x16.f16", + "rocdl.smfmac.f32.32x32x32.bf16", + "rocdl.smfmac.f32.32x32x32.bf8.bf8", + "rocdl.smfmac.f32.32x32x32.bf8.fp8", + "rocdl.smfmac.f32.32x32x32.f16", + "rocdl.smfmac.f32.32x32x32.fp8.bf8", + "rocdl.smfmac.f32.32x32x32.fp8.fp8", + "rocdl.smfmac.f32.32x32x64.bf8.bf8", + "rocdl.smfmac.f32.32x32x64.bf8.fp8", + "rocdl.smfmac.f32.32x32x64.fp8.bf8", + "rocdl.smfmac.f32.32x32x64.fp8.fp8", + "rocdl.smfmac.i32.16x16x128.i8", + "rocdl.smfmac.i32.16x16x64.i8", + "rocdl.smfmac.i32.32x32x32.i8", + "rocdl.smfmac.i32.32x32x64.i8", + "rocdl.sqrt", + "rocdl.sudot4", + "rocdl.sudot8", + "rocdl.swmmac.bf16.16x16x32.bf16", + "rocdl.swmmac.bf16.16x16x64.bf16", + "rocdl.swmmac.bf16f32.16x16x64.bf16", + "rocdl.swmmac.f16.16x16x128.bf8.bf8", + "rocdl.swmmac.f16.16x16x128.bf8.fp8", + "rocdl.swmmac.f16.16x16x128.fp8.bf8", + "rocdl.swmmac.f16.16x16x128.fp8.fp8", + "rocdl.swmmac.f16.16x16x32.f16", + "rocdl.swmmac.f16.16x16x64.f16", + "rocdl.swmmac.f32.16x16x128.bf8.bf8", + "rocdl.swmmac.f32.16x16x128.bf8.fp8", + "rocdl.swmmac.f32.16x16x128.fp8.bf8", + "rocdl.swmmac.f32.16x16x128.fp8.fp8", + "rocdl.swmmac.f32.16x16x32.bf16", + "rocdl.swmmac.f32.16x16x32.bf8.bf8", + "rocdl.swmmac.f32.16x16x32.bf8.fp8", + "rocdl.swmmac.f32.16x16x32.f16", + "rocdl.swmmac.f32.16x16x32.fp8.bf8", + "rocdl.swmmac.f32.16x16x32.fp8.fp8", + "rocdl.swmmac.f32.16x16x64.bf16", + "rocdl.swmmac.f32.16x16x64.f16", + "rocdl.swmmac.i32.16x16x128.iu8", + "rocdl.swmmac.i32.16x16x32.iu4", + "rocdl.swmmac.i32.16x16x32.iu8", + "rocdl.swmmac.i32.16x16x64.iu4", + "rocdl.tanh", + "rocdl.tensor.load.to.lds", + "rocdl.tensor.store.from.lds", + "rocdl.udot2", + "rocdl.udot4", + "rocdl.udot8", + "rocdl.update.dpp", + "rocdl.wait.asyncmark", + "rocdl.wave.barrier", + "rocdl.wave.id", + "rocdl.wavefrontsize", + "rocdl.wmma.bf16.16x16x16.bf16", + "rocdl.wmma.bf16.16x16x32.bf16", + "rocdl.wmma.bf16f32.16x16x32.bf16", + "rocdl.wmma.f16.16x16x128.bf8_bf8", + "rocdl.wmma.f16.16x16x128.bf8_fp8", + "rocdl.wmma.f16.16x16x128.fp8_bf8", + "rocdl.wmma.f16.16x16x128.fp8_fp8", + "rocdl.wmma.f16.16x16x16.f16", + "rocdl.wmma.f16.16x16x32.f16", + "rocdl.wmma.f16.16x16x64.bf8_bf8", + "rocdl.wmma.f16.16x16x64.bf8_fp8", + "rocdl.wmma.f16.16x16x64.fp8_bf8", + "rocdl.wmma.f16.16x16x64.fp8_fp8", + "rocdl.wmma.f32.16x16x128.bf8_bf8", + "rocdl.wmma.f32.16x16x128.bf8_fp8", + "rocdl.wmma.f32.16x16x128.fp8_bf8", + "rocdl.wmma.f32.16x16x128.fp8_fp8", + "rocdl.wmma.f32.16x16x16.bf16", + "rocdl.wmma.f32.16x16x16.bf8_bf8", + "rocdl.wmma.f32.16x16x16.bf8_fp8", + "rocdl.wmma.f32.16x16x16.f16", + "rocdl.wmma.f32.16x16x16.fp8_bf8", + "rocdl.wmma.f32.16x16x16.fp8_fp8", + "rocdl.wmma.f32.16x16x32.bf16", + "rocdl.wmma.f32.16x16x32.f16", + "rocdl.wmma.f32.16x16x4.f32", + "rocdl.wmma.f32.16x16x64.bf8_bf8", + "rocdl.wmma.f32.16x16x64.bf8_fp8", + "rocdl.wmma.f32.16x16x64.fp8_bf8", + "rocdl.wmma.f32.16x16x64.fp8_fp8", + "rocdl.wmma.i32.16x16x16.iu4", + "rocdl.wmma.i32.16x16x16.iu8", + "rocdl.wmma.i32.16x16x32.iu4", + "rocdl.wmma.i32.16x16x64.iu8", + "rocdl.wmma.scale.f32.16x16x128.f8f6f4", + "rocdl.wmma.scale.f32.32x16x128.f4", + "rocdl.wmma.scale16.f32.16x16x128.f8f6f4", + "rocdl.wmma.scale16.f32.32x16x128.f4", + "rocdl.workgroup.id.x", + "rocdl.workgroup.id.y", + "rocdl.workgroup.id.z", + "rocdl.workitem.id.x", + "rocdl.workitem.id.y", + "rocdl.workitem.id.z", +}; + +fn testContext() !*mlir.Context { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + rocdl.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + return ctx; +} + +fn expectAttributeRoundTrip(ctx: *mlir.Context, attribute: *const mlir.Attribute) !void { + var buffer: [2048]u8 = undefined; + var writer = std.Io.Writer.fixed(&buffer); + try attribute.format(&writer); + try std.testing.expect(attribute.eql(try mlir.Attribute.parse(ctx, writer.buffered()))); +} + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(rocdl); + std.testing.refAllDecls(rocdl.ops); + std.testing.refAllDecls(rocdl.types); + std.testing.refAllDecls(rocdl.annotations); + inline for (rocdl.attributes.enum_attributes) |A| std.testing.refAllDecls(A); + std.testing.refAllDecls(rocdl.TargetAttr); +} + +test "packed FP8 conversion uses the registered LLVM ROCDL dialect" { + const ctx = try testContext(); + defer ctx.deinit(); + try std.testing.expect(ctx.isRegisteredOperation("rocdl.cvt.pk.f32.fp8")); + + const loc = mlir.Location.unknown(ctx); + const block = mlir.Block.init(&.{.int(ctx, .i32)}, &.{loc}); + defer block.deinit(); + for ([_]bool{ false, true }) |high| { + const op = rocdl.cvtPkF32Fp8(ctx, block.argument(0), high, loc); + defer op.deinit(); + try std.testing.expect(op.verify()); + try std.testing.expect(op.result(0).type_().eql(.vector(&.{2}, .float(ctx, .f32)))); + try std.testing.expect(op.attributeByName("wordSel").?.eql(.boolean(ctx, high))); + } +} + +test "every bound operation is registered" { + const ctx = try testContext(); + defer ctx.deinit(); + try std.testing.expectEqual(operation_names.len, @as(usize, @intCast(c.mlirROCDLGetOperationCount(ctx.ptr())))); + for (operation_names, 0..) |name, i| { + try std.testing.expect(ctx.isRegisteredOperation(name)); + for (operation_names[0..i]) |earlier| try std.testing.expect(!std.mem.eql(u8, name, earlier)); + } + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} + +test "all ROCDL enum attributes construct and round-trip every named value" { + @setEvalBranchQuota(100_000); + const ctx = try testContext(); + defer ctx.deinit(); + inline for (rocdl.attributes.enum_attributes) |A| { + for (std.enums.values(A.Value)) |value| { + const attribute = try A.get(ctx, .{ .value = value }); + try std.testing.expect(attribute.isA(A) != null); + try std.testing.expectEqual(value, attribute.value()); + try expectAttributeRoundTrip(ctx, attribute.attribute()); + } + } + const mask: rocdl.SchedGroupMask = .combine(&.{ .valu, .mfma_wmma, .ds_read }); + try std.testing.expect(mask.contains(.mfma_wmma)); + try std.testing.expect(!mask.contains(.all_vmem)); + const attribute = try rocdl.SchedGroupMaskAttr.get(ctx, .{ .value = mask }); + try std.testing.expectEqual(mask, attribute.value()); + try expectAttributeRoundTrip(ctx, attribute.attribute()); +} + +test "ROCDL enum constructors reject invalid values and flag bits" { + const ctx = try testContext(); + defer ctx.deinit(); + try std.testing.expect(c.mlirROCDLMatrixFormatAttrGet(ctx.ptr(), 5).ptr == null); + try std.testing.expect(c.mlirROCDLWMMACModifierAttrGet(ctx.ptr(), 65536).ptr == null); + try std.testing.expectError(error.InvalidMlir, rocdl.SchedGroupMaskAttr.get(ctx, .{ .value = @fromBackingInt(1 << 20) })); + try std.testing.expectError(error.InvalidMlir, rocdl.PreGfx12CachePolicyAttr.get(ctx, .{ .value = @fromBackingInt(1 << 6) })); +} + +test "target attributes expose all fields and reject invalid options" { + const ctx = try testContext(); + defer ctx.deinit(); + const defaults = try rocdl.TargetAttr.get(ctx, .{}); + try std.testing.expectEqual(@as(i32, 2), defaults.optLevel()); + try std.testing.expectEqualStrings("amdgcn-amd-amdhsa", defaults.triple()); + try std.testing.expectEqualStrings("gfx900", defaults.chip()); + try std.testing.expectEqualStrings("", defaults.features()); + try std.testing.expectEqualStrings("600", defaults.abi()); + try std.testing.expect(defaults.flags() == null and defaults.link() == null); + try std.testing.expect(defaults.hasWave64()); + try std.testing.expect(!defaults.hasFastMath()); + try std.testing.expect(defaults.hasCorrectSqrt()); + try expectAttributeRoundTrip(ctx, defaults.attribute()); + + const flags: *const mlir.Attribute = @ptrCast(mlir.DictionaryAttribute.init(ctx, &.{ + .named(ctx, "no_wave64", .unit(ctx)), + .named(ctx, "fast", .unit(ctx)), + .named(ctx, "daz", .unit(ctx)), + .named(ctx, "finite_only", .unit(ctx)), + .named(ctx, "unsafe_math", .unit(ctx)), + .named(ctx, "unsafe_sqrt", .unit(ctx)), + })); + const link: *const mlir.Attribute = @ptrCast(mlir.ArrayAttribute.init(ctx, &.{.string(ctx, "device.bc")})); + const target = try rocdl.TargetAttr.get(ctx, .{ .optLevel = 3, .chip = "gfx942", .features = "+xnack", .abi = "500", .flags = flags, .link = link }); + try std.testing.expectEqual(@as(i32, 3), target.optLevel()); + try std.testing.expectEqualStrings("gfx942", target.chip()); + try std.testing.expectEqualStrings("+xnack", target.features()); + try std.testing.expectEqualStrings("500", target.abi()); + try std.testing.expect(target.flags().?.eql(flags)); + try std.testing.expect(target.link().?.eql(link)); + try std.testing.expect(target.hasFlag("fast") and target.hasFastMath()); + try std.testing.expect(!target.hasWave64()); + try std.testing.expect(target.hasDaz() and target.hasFiniteOnly()); + try std.testing.expect(target.hasUnsafeMath() and !target.hasCorrectSqrt()); + try expectAttributeRoundTrip(ctx, target.attribute()); + + try std.testing.expectError(error.InvalidMlir, rocdl.TargetAttr.get(ctx, .{ .optLevel = 4 })); + try std.testing.expectError(error.InvalidMlir, rocdl.TargetAttr.get(ctx, .{ .triple = "" })); + try std.testing.expectError(error.InvalidMlir, rocdl.TargetAttr.get(ctx, .{ .chip = "" })); + try std.testing.expectError(error.InvalidMlir, rocdl.TargetAttr.get(ctx, .{ .abi = "700" })); + try std.testing.expectError(error.InvalidMlir, rocdl.TargetAttr.get(ctx, .{ .flags = .unit(ctx) })); + const badLink: *const mlir.Attribute = @ptrCast(mlir.ArrayAttribute.init(ctx, &.{.unit(ctx)})); + try std.testing.expectError(error.InvalidMlir, rocdl.TargetAttr.get(ctx, .{ .link = badLink })); + const other = try testContext(); + defer other.deinit(); + const otherFlags: *const mlir.Attribute = @ptrCast(mlir.DictionaryAttribute.init(other, &.{})); + try std.testing.expectError(error.InvalidMlir, rocdl.TargetAttr.get(ctx, .{ .flags = otherFlags })); +} + +test "ROCDL barriers buffer loads and matrix operations verify and reparse" { + const ctx = try testContext(); + defer ctx.deinit(); + const loc = mlir.Location.unknown(ctx); + const module = try mlir.Module.parse(ctx, + \\module { + \\ llvm.func @test(%rsrc: vector<4xi32>, %offset: i32, %a: f32, %c: vector<4xf32>) { + \\ llvm.return + \\ } + \\} + ); + defer module.deinit(); + const function = module.body().firstOperation().?; + const block = function.region(0).firstBlock().?; + const terminator = block.terminator().?; + const i32Type: *const mlir.Type = .int(ctx, .i32); + const f32Type: *const mlir.Type = .float(ctx, .f32); + const vec4f32 = rocdl.types.v4f32(ctx); + const zero: *const mlir.Attribute = .int(ctx, .i32, 0); + const opList = .{ + rocdl.ops.s_barrier(ctx, loc), + rocdl.ops.s_sleep(ctx, .int(ctx, .i32, 1), loc), + rocdl.ops.workitem_id_x(ctx, i32Type, .{}, loc), + rocdl.ops.sched_barrier(ctx, (try rocdl.SchedGroupMaskAttr.get(ctx, .{ .value = .combine(&.{ .valu, .mfma_wmma }) })).attribute(), loc), + rocdl.ops.raw_buffer_load(ctx, block.argument(0), block.argument(1), block.argument(1), f32Type, .{}, loc), + rocdl.ops.raw_buffer_load( + ctx, + block.argument(0), + block.argument(1), + block.argument(1), + f32Type, + .{ .aux = (try rocdl.Gfx942CachePolicyAttr.get(ctx, .{ .value = .combine(&.{ .sc0, .nt }) })).attribute() }, + loc, + ), + rocdl.ops.mfma_f32_16x16x4f32(ctx, block.argument(2), block.argument(2), block.argument(3), vec4f32, zero, zero, (try rocdl.MFMAPermBAttr.get(ctx, .{ .value = .none })).attribute(), loc), + }; + inline for (opList) |op| { + try std.testing.expect(op.verify()); + block.insertOwnedOperationBefore(terminator, op); + } + rocdl.annotations.annotateFunction(ctx, function, .{ + .kernel = true, + .reqdWorkGroupSize = .{ 64, 1, 1 }, + .flatWorkGroupSize = "64,64", + .maxFlatWorkGroupSize = 64, + .wavesPerEu = .{ .min = 1, .max = 2 }, + .unsafeFpAtomics = true, + }); + rocdl.annotations.annotateMemory(ctx, opList[4], .{ .lastUse = true, .noRemoteMemory = true, .noFineGrainedMemory = true, .ignoreDenormalMode = true }); + try std.testing.expect(opList[4].attributeByName("aux").?.eql(.int(ctx, .i32, 0))); + try std.testing.expect(opList[5].attributeByName("aux").?.isA(rocdl.Gfx942CachePolicyAttr) != null); + try std.testing.expect(function.attributeByName("rocdl.waves_per_eu").?.eql(.string(ctx, "1,2"))); + try std.testing.expect(module.operation().verify()); + var buffer: [16384]u8 = undefined; + var writer = std.Io.Writer.fixed(&buffer); + try module.format(&writer); + const parsed = try mlir.Module.parse(ctx, writer.buffered()); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "ROCDL annotations preserve module flag defaults and explicit false" { + const ctx = try testContext(); + defer ctx.deinit(); + const module = mlir.Module.init(.unknown(ctx)); + defer module.deinit(); + rocdl.annotations.annotateModule(ctx, module.operation(), .{}); + try std.testing.expect(module.operation().attributeByName("rocdl.xnack") == null); + rocdl.annotations.annotateModule(ctx, module.operation(), .{ .xnack = false, .sramecc = true }); + try std.testing.expect(module.operation().attributeByName("rocdl.xnack").?.eql(.boolean(ctx, false))); + try std.testing.expect(module.operation().attributeByName("rocdl.sramecc").?.eql(.boolean(ctx, true))); + try std.testing.expect(module.operation().verify()); + const pointer = rocdl.pointerType(ctx, .barrier); + try std.testing.expect(pointer.eql(try mlir.Type.parse(ctx, "!llvm.ptr<15>"))); + try std.testing.expect(rocdl.pointerType(ctx, .buffer_resource).eql(try mlir.Type.parse(ctx, "!llvm.ptr<8>"))); +} diff --git a/mlir/dialects/rocdl/types.zig b/mlir/dialects/rocdl/types.zig new file mode 100644 index 00000000..a91ede15 --- /dev/null +++ b/mlir/dialects/rocdl/types.zig @@ -0,0 +1,147 @@ +// Generated from LLVM ROCDL TableGen records; see REGENERATING.md. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +const mlir = @import("mlir"); +const llvm = @import("mlir/dialects/llvm"); + +/// ROCDL address spaces use LLVM pointer types. +pub const AddressSpace = enum(u32) { + generic = 0, + global = 1, + shared = 3, + constant = 4, + private = 5, + buffer_fat_pointer = 7, + buffer_resource = 8, + buffer_strided_pointer = 9, + barrier = 15, +}; + +pub fn pointerType(ctx: *mlir.Context, space: AddressSpace) *const mlir.Type { + return llvm.pointerType(ctx, space); +} + +/// `vector<16xbf16>`, as used by ROCDL operations. +pub fn v16bf16(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{16}, .float(ctx, .bf16)); +} + +/// `vector<16xf16>`, as used by ROCDL operations. +pub fn v16f16(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{16}, .float(ctx, .f16)); +} + +/// `vector<16xf32>`, as used by ROCDL operations. +pub fn v16f32(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{16}, .float(ctx, .f32)); +} + +/// `vector<16xi32>`, as used by ROCDL operations. +pub fn v16i32(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{16}, .int(ctx, .i32)); +} + +/// `vector<2xbf16>`, as used by ROCDL operations. +pub fn v2bf16(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{2}, .float(ctx, .bf16)); +} + +/// `vector<2xf16>`, as used by ROCDL operations. +pub fn v2f16(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{2}, .float(ctx, .f16)); +} + +/// `vector<2xf32>`, as used by ROCDL operations. +pub fn v2f32(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{2}, .float(ctx, .f32)); +} + +/// `vector<2xi16>`, as used by ROCDL operations. +pub fn v2i16(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{2}, .int(ctx, .i16)); +} + +/// `vector<2xi32>`, as used by ROCDL operations. +pub fn v2i32(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{2}, .int(ctx, .i32)); +} + +/// `vector<32xbf16>`, as used by ROCDL operations. +pub fn v32bf16(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{32}, .float(ctx, .bf16)); +} + +/// `vector<32xf16>`, as used by ROCDL operations. +pub fn v32f16(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{32}, .float(ctx, .f16)); +} + +/// `vector<32xf32>`, as used by ROCDL operations. +pub fn v32f32(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{32}, .float(ctx, .f32)); +} + +/// `vector<32xi32>`, as used by ROCDL operations. +pub fn v32i32(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{32}, .int(ctx, .i32)); +} + +/// `vector<3xi32>`, as used by ROCDL operations. +pub fn v3i32(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{3}, .int(ctx, .i32)); +} + +/// `vector<4xf16>`, as used by ROCDL operations. +pub fn v4f16(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{4}, .float(ctx, .f16)); +} + +/// `vector<4xf32>`, as used by ROCDL operations. +pub fn v4f32(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{4}, .float(ctx, .f32)); +} + +/// `vector<4xf64>`, as used by ROCDL operations. +pub fn v4f64(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{4}, .float(ctx, .f64)); +} + +/// `vector<4xi16>`, as used by ROCDL operations. +pub fn v4i16(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{4}, .int(ctx, .i16)); +} + +/// `vector<4xi32>`, as used by ROCDL operations. +pub fn v4i32(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{4}, .int(ctx, .i32)); +} + +/// `vector<6xi32>`, as used by ROCDL operations. +pub fn v6i32(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{6}, .int(ctx, .i32)); +} + +/// `vector<8xbf16>`, as used by ROCDL operations. +pub fn v8bf16(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{8}, .float(ctx, .bf16)); +} + +/// `vector<8xf16>`, as used by ROCDL operations. +pub fn v8f16(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{8}, .float(ctx, .f16)); +} + +/// `vector<8xf32>`, as used by ROCDL operations. +pub fn v8f32(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{8}, .float(ctx, .f32)); +} + +/// `vector<8xi16>`, as used by ROCDL operations. +pub fn v8i16(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{8}, .int(ctx, .i16)); +} + +/// `vector<8xi32>`, as used by ROCDL operations. +pub fn v8i32(ctx: *mlir.Context) *const mlir.Type { + return .vector(&.{8}, .int(ctx, .i32)); +} diff --git a/mlir/dialects/scf/BUILD.bazel b/mlir/dialects/scf/BUILD.bazel new file mode 100644 index 00000000..1d7a0b3c --- /dev/null +++ b/mlir/dialects/scf/BUILD.bazel @@ -0,0 +1,23 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "scf", + import_name = "mlir/dialects/scf", + main = "scf.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//mlir", + "//stdx", + "@llvm-project//mlir:CAPISCF", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":scf", + "//mlir", + ], +) diff --git a/mlir/dialects/scf/scf.zig b/mlir/dialects/scf/scf.zig new file mode 100644 index 00000000..1c82a9e0 --- /dev/null +++ b/mlir/dialects/scf/scf.zig @@ -0,0 +1,363 @@ +const std = @import("std"); + +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("scf"); +} + +pub const ForOpts = struct { + /// Per td: `scf.for` uses signed comparison by default. Setting this emits + /// the `unsignedCmp` UnitAttr so the induction variable advances with + /// unsigned comparison instead. + unsigned_cmp: bool = false, + num_stages: ?i32 = null, +}; + +/// scf.for — operands are (lower, upper, step, init_args...) as a flat list. +/// Result arity equals init_args arity; result types must match init_args types. +/// +/// The body block must have arguments `[iv_type, *init_arg_types]` and be +/// terminated with `scf.yield`. +pub fn for_( + ctx: *mlir.Context, + lower: *const mlir.Value, + upper: *const mlir.Value, + step: *const mlir.Value, + init_args: []const *const mlir.Value, + body: *mlir.Block, + opts: ForOpts, + location: *const mlir.Location, +) *mlir.Operation { + var operands: stdx.BoundedArray(*const mlir.Value, 64) = .empty; + operands.appendSliceAssumeCapacity(&.{ lower, upper, step }); + operands.appendSliceAssumeCapacity(init_args); + + var result_types: stdx.BoundedArray(*const mlir.Type, 64) = .empty; + for (init_args) |v| result_types.appendAssumeCapacity(v.type_()); + + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (opts.unsigned_cmp) { + attrs.appendAssumeCapacity(.named(ctx, "unsignedCmp", .unit(ctx))); + } + if (opts.num_stages) |n| { + attrs.appendAssumeCapacity(.named(ctx, "tt.num_stages", .int(ctx, .i32, n))); + } + + return mlir.Operation.make(ctx, "scf.for", .{ + .operands = .{ .flat = operands.constSlice() }, + .results = .{ .flat = result_types.constSlice() }, + .blocks = &.{body}, + .attributes = attrs.constSlice(), + // Per-op verify fails with "operand #0 does not dominate this use" when + // the scf.for references outer SSA values that only become reachable + // once it's appended to the parent block. Defer to module-level verify. + .verify = false, + .location = location, + }); +} + +/// scf.if — cond is i1; then/else blocks must be terminated with `scf.yield` +/// (or implicitly for no-result forms). +/// +/// Built via raw OperationState because it has two separate regions. +pub fn if_( + ctx: *mlir.Context, + cond: *const mlir.Value, + result_types: []const *const mlir.Type, + then_block: *mlir.Block, + opts: struct { + else_block: ?*mlir.Block = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + _ = ctx; + var state: mlir.OperationState = .init("scf.if", location); + state.addOperands(&.{cond}); + state.addResults(result_types); + + const then_region = mlir.Region.init(); + then_region.appendOwnedBlock(then_block); + const else_region = mlir.Region.init(); + if (opts.else_block) |b| else_region.appendOwnedBlock(b); + state.addOwnedRegions(&.{ then_region, else_region }); + + return mlir.Operation.init(&state) catch @panic("Failed to create scf.if"); +} + +/// scf.yield — variadic result terminator for scf.for / scf.if regions. +pub fn yield(ctx: *mlir.Context, values: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "scf.yield", .{ + .operands = .{ .flat = values }, + .verify = false, + .location = location, + }); +} + +/// scf.while — two regions. The `before` region computes the condition and +/// terminates with `scf.condition`. The `after` region runs when the condition +/// is true and terminates with `scf.yield`. The arguments of `before` have the +/// types of `inits`. The trailing operands of `scf.condition` feed both the +/// `after` region's arguments and the `scf.while` op's results. +pub fn while_( + ctx: *mlir.Context, + inits: []const *const mlir.Value, + result_types: []const *const mlir.Type, + before_block: *mlir.Block, + after_block: *mlir.Block, + location: *const mlir.Location, +) *mlir.Operation { + _ = ctx; + var state: mlir.OperationState = .init("scf.while", location); + state.addOperands(inits); + state.addResults(result_types); + + const before_region = mlir.Region.init(); + before_region.appendOwnedBlock(before_block); + const after_region = mlir.Region.init(); + after_region.appendOwnedBlock(after_block); + state.addOwnedRegions(&.{ before_region, after_region }); + + return mlir.Operation.init(&state) catch @panic("Failed to create scf.while"); +} + +/// scf.condition — terminator of the `before` region of scf.while. +/// First operand is i1 (the continuation condition). The remaining operands +/// are forwarded to either the `after` region's arguments (if true) or the +/// scf.while's results (if false). +pub fn condition( + ctx: *mlir.Context, + cond: *const mlir.Value, + args: []const *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + var buf: stdx.BoundedArray(*const mlir.Value, 64) = .empty; + buf.appendAssumeCapacity(cond); + buf.appendSliceAssumeCapacity(args); + return mlir.Operation.make(ctx, "scf.condition", .{ + .operands = .{ .flat = buf.constSlice() }, + .verify = false, + .location = location, + }); +} + +pub const ExecuteRegionOpts = struct { + no_inline: bool = false, +}; + +/// scf.execute_region — runs `body` exactly once, returns the yielded values. +/// `body` has no arguments but may span multiple blocks (unlike for/if). +pub fn execute_region( + ctx: *mlir.Context, + result_types: []const *const mlir.Type, + body: *mlir.Block, + opts: ExecuteRegionOpts, + location: *const mlir.Location, +) *mlir.Operation { + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.no_inline) { + attrs.appendAssumeCapacity(.named(ctx, "no_inline", .unit(ctx))); + } + return mlir.Operation.make(ctx, "scf.execute_region", .{ + .results = .{ .flat = result_types }, + .blocks = &.{body}, + .attributes = attrs.constSlice(), + // execute_region has its own region-verify; defer to the module-level + // verify so unresolved SSA uses don't block construction. + .verify = false, + .location = location, + }); +} + +/// scf.parallel — multi-dim parallel for loop with optional reductions. +/// Operands are (lbs, ubs, steps, inits). `body_block` takes one `index` arg +/// per loop dim and must be terminated with `scf.reduce` (or no-op if inits is +/// empty — the parser auto-inserts an empty reduce). +pub fn parallel( + ctx: *mlir.Context, + lbs: []const *const mlir.Value, + ubs: []const *const mlir.Value, + steps: []const *const mlir.Value, + inits: []const *const mlir.Value, + body_block: *mlir.Block, + result_types: []const *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + std.debug.assert(lbs.len == ubs.len); + std.debug.assert(lbs.len == steps.len); + + var operands: stdx.BoundedArray(*const mlir.Value, 64) = .empty; + operands.appendSliceAssumeCapacity(lbs); + operands.appendSliceAssumeCapacity(ubs); + operands.appendSliceAssumeCapacity(steps); + operands.appendSliceAssumeCapacity(inits); + + const seg_sizes = [4]i32{ + @intCast(lbs.len), + @intCast(ubs.len), + @intCast(steps.len), + @intCast(inits.len), + }; + return mlir.Operation.make(ctx, "scf.parallel", .{ + .operands = .{ .flat = operands.constSlice() }, + .results = .{ .flat = result_types }, + .blocks = &.{body_block}, + .attributes = &.{ + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), + }, + .verify = false, + .location = location, + }); +} + +/// scf.reduce — terminator of `scf.parallel`. Each region is a reduction body +/// taking two args (lhs, rhs) matching the corresponding operand's type, +/// terminated with `scf.reduce.return`. +pub fn reduce( + ctx: *mlir.Context, + operands: []const *const mlir.Value, + reduction_blocks: []const *mlir.Block, + location: *const mlir.Location, +) *mlir.Operation { + _ = ctx; + std.debug.assert(operands.len == reduction_blocks.len); + var state: mlir.OperationState = .init("scf.reduce", location); + state.addOperands(operands); + + var regions: stdx.BoundedArray(*mlir.Region, 16) = .empty; + for (reduction_blocks) |b| { + const r = mlir.Region.init(); + r.appendOwnedBlock(b); + regions.appendAssumeCapacity(r); + } + state.addOwnedRegions(regions.constSlice()); + + return mlir.Operation.init(&state) catch @panic("Failed to create scf.reduce"); +} + +pub fn reduce_return(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "scf.reduce.return", .{ + .operands = .{ .flat = &.{value} }, + .verify = false, + .location = location, + }); +} + +/// scf.forall — multi-dim parallel region, similar to `scf.parallel` but using +/// tensor-based `shared_outs` instead of init values. This wrapper builds the +/// fully-dynamic form (all lb/ub/step are runtime values). +/// +/// `body_block` must have `rank + shared_outs.len` arguments (one index per +/// loop dim, plus one per shared_out) and must be terminated with +/// `scf.forall.in_parallel`. +pub fn forall( + ctx: *mlir.Context, + lbs: []const *const mlir.Value, + ubs: []const *const mlir.Value, + steps: []const *const mlir.Value, + shared_outs: []const *const mlir.Value, + body_block: *mlir.Block, + result_types: []const *const mlir.Type, + opts: struct { + mapping: ?*const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + std.debug.assert(lbs.len == ubs.len); + std.debug.assert(lbs.len == steps.len); + + var operands: stdx.BoundedArray(*const mlir.Value, 64) = .empty; + operands.appendSliceAssumeCapacity(lbs); + operands.appendSliceAssumeCapacity(ubs); + operands.appendSliceAssumeCapacity(steps); + operands.appendSliceAssumeCapacity(shared_outs); + + const seg_sizes = [4]i32{ + @intCast(lbs.len), + @intCast(ubs.len), + @intCast(steps.len), + @intCast(shared_outs.len), + }; + + // Fully-dynamic form: the static arrays use the kDynamic sentinel + // (mlir::ShapedType::kDynamic == INT64_MIN). We build one slot per loop + // dim, all dynamic. This matches how MLIR printers emit + // `scf.forall (%iv) = (%lb) to (%ub) step (%step)` when no static + // constants are folded in. + const k_dynamic: i64 = std.math.minInt(i64); + var sentinel: stdx.BoundedArray(i64, 16) = .empty; + for (0..lbs.len) |_| sentinel.appendAssumeCapacity(k_dynamic); + const sentinel_slice = sentinel.constSlice(); + + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attrs.appendSliceAssumeCapacity(&.{ + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), + .named(ctx, "staticLowerBound", .denseArray(ctx, .i64, sentinel_slice)), + .named(ctx, "staticUpperBound", .denseArray(ctx, .i64, sentinel_slice)), + .named(ctx, "staticStep", .denseArray(ctx, .i64, sentinel_slice)), + }); + if (opts.mapping) |m| attrs.appendAssumeCapacity(.named(ctx, "mapping", m)); + + return mlir.Operation.make(ctx, "scf.forall", .{ + .operands = .{ .flat = operands.constSlice() }, + .results = .{ .flat = result_types }, + .blocks = &.{body_block}, + .attributes = attrs.constSlice(), + .verify = false, + .location = location, + }); +} + +/// scf.forall.in_parallel — terminator for scf.forall. The inner region is +/// a single block containing a flat list of ops (typically +/// tensor.parallel_insert_slice). `body_block` must be built by the caller. +pub fn forall_in_parallel( + ctx: *mlir.Context, + body_block: *mlir.Block, + location: *const mlir.Location, +) *mlir.Operation { + _ = ctx; + var state: mlir.OperationState = .init("scf.forall.in_parallel", location); + const region = mlir.Region.init(); + region.appendOwnedBlock(body_block); + state.addOwnedRegions(&.{region}); + return mlir.Operation.init(&state) catch @panic("Failed to create scf.forall.in_parallel"); +} + +/// scf.index_switch — branches to case_blocks[i] if arg == cases[i], else +/// falls through to default_block. Blocks must be terminated with scf.yield. +/// The op's results have the types yielded by each region. +pub fn index_switch( + ctx: *mlir.Context, + arg: *const mlir.Value, + cases: []const i64, + default_block: *mlir.Block, + case_blocks: []const *mlir.Block, + result_types: []const *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + std.debug.assert(cases.len == case_blocks.len); + var state: mlir.OperationState = .init("scf.index_switch", location); + state.addOperands(&.{arg}); + state.addResults(result_types); + state.addAttributes(&.{ + .named(ctx, "cases", .denseArray(ctx, .i64, cases)), + }); + + // First region is the default, then one per case — this is how + // `VariadicRegion>:$caseRegions` is positioned in the td. + const default_region = mlir.Region.init(); + default_region.appendOwnedBlock(default_block); + + var regions: stdx.BoundedArray(*mlir.Region, 64) = .empty; + regions.appendAssumeCapacity(default_region); + for (case_blocks) |b| { + const r = mlir.Region.init(); + r.appendOwnedBlock(b); + regions.appendAssumeCapacity(r); + } + state.addOwnedRegions(regions.constSlice()); + + return mlir.Operation.init(&state) catch @panic("Failed to create scf.index_switch"); +} diff --git a/mlir/dialects/scf/test.zig b/mlir/dialects/scf/test.zig new file mode 100644 index 00000000..dceac66e --- /dev/null +++ b/mlir/dialects/scf/test.zig @@ -0,0 +1,39 @@ +const std = @import("std"); + +const mlir = @import("mlir"); +const scf = @import("mlir/dialects/scf"); + +const operation_names: []const []const u8 = &.{ + "scf.condition", + "scf.execute_region", + "scf.for", + "scf.forall", + "scf.forall.in_parallel", + "scf.if", + "scf.index_switch", + "scf.parallel", + "scf.reduce", + "scf.reduce.return", + "scf.while", + "scf.yield", +}; + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(scf); + inline for (comptime std.meta.declarations(scf)) |decl| { + const T = @field(scf, decl); + if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); + } +} + +test "every bound operation is registered" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + scf.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} diff --git a/mlir/dialects/shardy/BUILD.bazel b/mlir/dialects/shardy/BUILD.bazel new file mode 100644 index 00000000..4077f902 --- /dev/null +++ b/mlir/dialects/shardy/BUILD.bazel @@ -0,0 +1,23 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "shardy", + import_name = "mlir/dialects/shardy", + main = "shardy.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//mlir", + "@shardy//shardy/integrations/c:sdy_capi", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":shardy", + "//mlir", + "//mlir/dialects/func", + ], +) diff --git a/mlir/dialects/shardy/shardy.zig b/mlir/dialects/shardy/shardy.zig new file mode 100644 index 00000000..d73bf81b --- /dev/null +++ b/mlir/dialects/shardy/shardy.zig @@ -0,0 +1,364 @@ +//! Bindings for OpenXLA shardy dialect +//! +//! - dialect overview: https://openxla.org/shardy/sdy_dialect +//! - C API: https://github.com/openxla/shardy/blob/main/shardy/integrations/c/attributes.h +const std = @import("std"); + +const c = @import("c"); +const mlir = @import("mlir"); + +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("sdy"); +} + +/// `sdy.sharding`, the sharding of a function argument or result. +pub fn shardingNamedAttr(ctx: *mlir.Context, sharding: *const TensorShardingAttr) mlir.NamedAttribute { + return .named(ctx, "sdy.sharding", sharding.attribute()); +} + +/// `sdy.mesh`. Not verified: it only verifies inside a module. +pub fn mesh(ctx: *mlir.Context, sym_name: []const u8, mesh_: *const MeshAttr, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "sdy.mesh", .{ + .attributes = &.{ + .named(ctx, "sym_name", .string(ctx, sym_name)), + .named(ctx, "mesh", mesh_.attribute()), + }, + .verify = false, + .location = location, + }); +} + +pub fn sharding_constraint(ctx: *mlir.Context, input: *const mlir.Value, sharding: *const TensorShardingAttr, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "sdy.sharding_constraint", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{input.type_()} }, + .attributes = &.{ + .named(ctx, "sharding", sharding.attribute()), + }, + .location = location, + }); +} + +pub fn manual_computation( + ctx: *mlir.Context, + tensors: []const *const mlir.Value, + result_types: []const *const mlir.Type, + body_block: *mlir.Block, + in_shardings: *const TensorShardingPerValueAttr, + out_shardings: *const TensorShardingPerValueAttr, + manual_axes: *const ManualAxesAttr, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "sdy.manual_computation", .{ + .operands = .{ .flat = tensors }, + .results = .{ .flat = result_types }, + .blocks = &.{body_block}, + .attributes = &.{ + .named(ctx, "in_shardings", in_shardings.attribute()), + .named(ctx, "out_shardings", out_shardings.attribute()), + .named(ctx, "manual_axes", manual_axes.attribute()), + }, + .location = location, + }); +} + +/// `sdy.return`. Not verified: a terminator only verifies inside its parent. +pub fn return_(ctx: *mlir.Context, results: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "sdy.return", .{ + .operands = .{ .flat = results }, + .verify = false, + .location = location, + }); +} + +pub const MeshAxisAttr = opaque { + const M = mlir.Methods(MeshAxisAttr, c.MlirAttribute); + + pub const isAFn = c.sdyAttributeIsAMeshAxisAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + name: []const u8, + size: i64, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) *const MeshAxisAttr { + return @ptrCast(c.sdyMeshAxisAttrGet(ctx.ptr(), mlir.stringRef(args.name), args.size).ptr); + } + + pub fn attribute(self: *const MeshAxisAttr) *const mlir.Attribute { + return @ptrCast(self); + } + + pub fn name(self: *const MeshAxisAttr) []const u8 { + return mlir.string(c.sdyMeshAxisAttrGetName(self.ptr())); + } + + pub fn size(self: *const MeshAxisAttr) i64 { + return c.sdyMeshAxisAttrGetSize(self.ptr()); + } +}; + +pub const MeshAttr = opaque { + const M = mlir.Methods(MeshAttr, c.MlirAttribute); + + pub const isAFn = c.sdyAttributeIsAMeshAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + axes: []const *const MeshAxisAttr, + device_ids: []const i64 = &.{}, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) *const MeshAttr { + const axes: []const c.struct_MlirAttribute = @ptrCast(args.axes); + return @ptrCast(c.sdyMeshAttrGet( + ctx.ptr(), + @intCast(axes.len), + axes.ptr, + @intCast(args.device_ids.len), + args.device_ids.ptr, + ).ptr); + } + + pub fn attribute(self: *const MeshAttr) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +pub const SubAxisInfoAttr = opaque { + const M = mlir.Methods(SubAxisInfoAttr, c.MlirAttribute); + + pub const isAFn = c.sdyAttributeIsASubAxisInfoAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + pre_size: i64, + size: i64, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) *const SubAxisInfoAttr { + return @ptrCast(c.sdySubAxisInfoAttrGet(ctx.ptr(), args.pre_size, args.size).ptr); + } + + pub fn attribute(self: *const SubAxisInfoAttr) *const mlir.Attribute { + return @ptrCast(self); + } + + pub fn preSize(self: *const SubAxisInfoAttr) i64 { + return c.sdySubAxisInfoAttrGetPreSize(self.ptr()); + } + + pub fn size(self: *const SubAxisInfoAttr) i64 { + return c.sdySubAxisInfoAttrGetSize(self.ptr()); + } +}; + +pub const AxisRefAttr = opaque { + const M = mlir.Methods(AxisRefAttr, c.MlirAttribute); + + pub const isAFn = c.sdyAttributeIsAnAxisRefAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub fn named(ctx: *mlir.Context, axis_name: []const u8) *const AxisRefAttr { + return @ptrCast(c.sdyAxisRefAttrGet(ctx.ptr(), mlir.stringRef(axis_name), .{ .ptr = null }).ptr); + } + + pub fn subAxis(ctx: *mlir.Context, axis_name: []const u8, pre_size: i64, size: i64) *const AxisRefAttr { + const sub_axis: *const SubAxisInfoAttr = .get(ctx, .{ .pre_size = pre_size, .size = size }); + return @ptrCast(c.sdyAxisRefAttrGet(ctx.ptr(), mlir.stringRef(axis_name), sub_axis.ptr()).ptr); + } + + pub fn attribute(self: *const AxisRefAttr) *const mlir.Attribute { + return @ptrCast(self); + } + + pub fn name(self: *const AxisRefAttr) []const u8 { + return mlir.string(c.sdyAxisRefAttrGetName(self.ptr())); + } + + pub fn subAxisInfo(self: *const AxisRefAttr) ?*const SubAxisInfoAttr { + const attr_ = c.sdyAxisRefAttrGetSubAxisInfo(self.ptr()); + return if (attr_.ptr == null) null else @ptrCast(attr_.ptr); + } +}; + +pub const DimensionShardingAttr = opaque { + const M = mlir.Methods(DimensionShardingAttr, c.MlirAttribute); + + pub const isAFn = c.sdyAttributeIsADimensionShardingAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + axes: []const *const AxisRefAttr, + is_closed: bool, + priority: ?i64 = null, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) *const DimensionShardingAttr { + const attrs: []const c.struct_MlirAttribute = @ptrCast(args.axes); + return @ptrCast(c.sdyDimensionShardingAttrGet(ctx.ptr(), @intCast(attrs.len), attrs.ptr, args.is_closed, args.priority orelse -1).ptr); + } + + pub fn closed(ctx: *mlir.Context, axes: []const *const AxisRefAttr) *const DimensionShardingAttr { + return get(ctx, .{ .axes = axes, .is_closed = true }); + } + + pub fn open(ctx: *mlir.Context, axes: []const *const AxisRefAttr) *const DimensionShardingAttr { + return get(ctx, .{ .axes = axes, .is_closed = false }); + } + + pub fn replicated(ctx: *mlir.Context) *const DimensionShardingAttr { + return closed(ctx, &.{}); + } + + pub fn attribute(self: *const DimensionShardingAttr) *const mlir.Attribute { + return @ptrCast(self); + } + + pub fn numAxes(self: *const DimensionShardingAttr) usize { + return @intCast(c.sdyDimensionShardingAttrGetAxesSize(self.ptr())); + } + + pub fn axis(self: *const DimensionShardingAttr, index: usize) *const AxisRefAttr { + return @ptrCast(c.sdyDimensionShardingAttrGetAxesElem(self.ptr(), @intCast(index)).ptr); + } + + pub fn isClosed(self: *const DimensionShardingAttr) bool { + return c.sdyDimensionShardingAttrGetIsClosed(self.ptr()); + } + + pub fn priority(self: *const DimensionShardingAttr) ?i64 { + const value = c.sdyDimensionShardingAttrGetPriority(self.ptr()); + return if (value == -1) null else value; + } +}; + +pub const TensorShardingAttr = opaque { + const M = mlir.Methods(TensorShardingAttr, c.MlirAttribute); + + pub const isAFn = c.sdyAttributeIsATensorShardingAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + mesh: []const u8, + dimensions: []const *const DimensionShardingAttr, + replicated_axes: []const *const AxisRefAttr = &.{}, + unreduced_axes: []const *const AxisRefAttr = &.{}, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) *const TensorShardingAttr { + comptime std.debug.assert(@sizeOf(c.struct_MlirAttribute) == @sizeOf(*const mlir.Attribute)); + const dimensions: []const c.struct_MlirAttribute = @ptrCast(args.dimensions); + const replicated: []const c.struct_MlirAttribute = @ptrCast(args.replicated_axes); + const unreduced: []const c.struct_MlirAttribute = @ptrCast(args.unreduced_axes); + return @ptrCast(c.sdyTensorShardingAttrGet( + ctx.ptr(), + mlir.Attribute.flatSymbolRef(ctx, args.mesh).ptr(), + @intCast(dimensions.len), + dimensions.ptr, + @intCast(replicated.len), + replicated.ptr, + @intCast(unreduced.len), + unreduced.ptr, + 0, + ).ptr); + } + + pub fn attribute(self: *const TensorShardingAttr) *const mlir.Attribute { + return @ptrCast(self); + } + + pub fn meshOrRef(self: *const TensorShardingAttr) *const mlir.Attribute { + return @ptrCast(c.sdyTensorShardingAttrGetMeshOrRef(self.ptr()).ptr); + } + + pub fn numDimensions(self: *const TensorShardingAttr) usize { + return @intCast(c.sdyTensorShardingAttrGetDimShardingsSize(self.ptr())); + } + + pub fn dimension(self: *const TensorShardingAttr, index: usize) *const DimensionShardingAttr { + return @ptrCast(c.sdyTensorShardingAttrGetDimShardingsElem(self.ptr(), @intCast(index)).ptr); + } + + pub fn numReplicatedAxes(self: *const TensorShardingAttr) usize { + return @intCast(c.sdyTensorShardingAttrGetReplicatedAxesSize(self.ptr())); + } + + pub fn replicatedAxis(self: *const TensorShardingAttr, index: usize) *const AxisRefAttr { + return @ptrCast(c.sdyTensorShardingAttrGetReplicatedAxesElem(self.ptr(), @intCast(index)).ptr); + } + + pub fn numUnreducedAxes(self: *const TensorShardingAttr) usize { + return @intCast(c.sdyTensorShardingAttrGetUnreducedAxesSize(self.ptr())); + } + + pub fn unreducedAxis(self: *const TensorShardingAttr, index: usize) *const AxisRefAttr { + return @ptrCast(c.sdyTensorShardingAttrGetUnreducedAxesElem(self.ptr(), @intCast(index)).ptr); + } +}; + +pub const TensorShardingPerValueAttr = opaque { + const M = mlir.Methods(TensorShardingPerValueAttr, c.MlirAttribute); + + pub const isAFn = c.sdyAttributeIsATensorShardingPerValueAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub fn get(ctx: *mlir.Context, shardings: []const *const TensorShardingAttr) *const TensorShardingPerValueAttr { + const attrs: []const c.struct_MlirAttribute = @ptrCast(shardings); + return @ptrCast(c.sdyTensorShardingPerValueAttrGet(ctx.ptr(), @intCast(attrs.len), attrs.ptr).ptr); + } + + pub fn attribute(self: *const TensorShardingPerValueAttr) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +pub const ManualAxesAttr = opaque { + const M = mlir.Methods(ManualAxesAttr, c.MlirAttribute); + + pub const isAFn = c.sdyAttributeIsAManualAxesAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub fn get(ctx: *mlir.Context, axes: []const *const mlir.StringAttribute) *const ManualAxesAttr { + comptime std.debug.assert(@sizeOf(c.struct_MlirAttribute) == @sizeOf(*const mlir.StringAttribute)); + const attrs: []const c.struct_MlirAttribute = @ptrCast(axes); + return @ptrCast(c.sdyManualAxesAttrGet(ctx.ptr(), @intCast(attrs.len), attrs.ptr).ptr); + } + + pub fn attribute(self: *const ManualAxesAttr) *const mlir.Attribute { + return @ptrCast(self); + } + + pub fn numAxes(self: *const ManualAxesAttr) usize { + return @intCast(c.sdyManualAxesAttrGetAxesSize(self.ptr())); + } + + pub fn axis(self: *const ManualAxesAttr, index: usize) []const u8 { + return mlir.string(c.sdyManualAxesAttrGetAxesElem(self.ptr(), @intCast(index))); + } +}; diff --git a/mlir/dialects/shardy/test.zig b/mlir/dialects/shardy/test.zig new file mode 100644 index 00000000..a4a757d2 --- /dev/null +++ b/mlir/dialects/shardy/test.zig @@ -0,0 +1,200 @@ +const std = @import("std"); + +const func = @import("mlir/dialects/func"); +const mlir = @import("mlir"); +const shardy = @import("mlir/dialects/shardy"); + +const ManualAxesAttr = shardy.ManualAxesAttr; +const TensorShardingAttr = shardy.TensorShardingAttr; +const TensorShardingPerValueAttr = shardy.TensorShardingPerValueAttr; + +const operation_names: []const []const u8 = &.{ + "sdy.manual_computation", + "sdy.mesh", + "sdy.return", + "sdy.sharding_constraint", +}; + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(shardy); + inline for (comptime std.meta.declarations(shardy)) |decl| { + const T = @field(shardy, decl); + if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); + } +} + +test "every bound operation is registered" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + shardy.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} + +test TensorShardingAttr { + const registry: *mlir.DialectRegistry = try .init(); + defer registry.deinit(); + shardy.registerDialect(registry); + func.registerExtensions(registry); + + var ctx: *mlir.Context = try .init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + + var w: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer w.deinit(); + + { + defer w.clearRetainingCapacity(); + const sharding: *const TensorShardingAttr = .get(ctx, .{ + .mesh = "folded_mesh", + .dimensions = &.{.replicated(ctx)}, + .replicated_axes = &.{ .named(ctx, "link_x"), .named(ctx, "link_y") }, + }); + try sharding.attribute().format(&w.writer); + try std.testing.expectEqualSlices(u8, "#sdy.sharding<@folded_mesh, [{}], replicated={\"link_x\", \"link_y\"}>", w.written()); + } + { + defer w.clearRetainingCapacity(); + const sharding: *const TensorShardingAttr = .get(ctx, .{ + .mesh = "suggested_mesh", + .dimensions = &.{ + .closed(ctx, &.{.named(ctx, "link_z")}), + .closed(ctx, &.{.named(ctx, "link_x")}), + }, + .replicated_axes = &.{.named(ctx, "link_y")}, + }); + try sharding.attribute().format(&w.writer); + try std.testing.expectEqualSlices(u8, "#sdy.sharding<@suggested_mesh, [{\"link_z\"}, {\"link_x\"}], replicated={\"link_y\"}>", w.written()); + } + { + defer w.clearRetainingCapacity(); + const sharding: *const TensorShardingAttr = .get(ctx, .{ + .mesh = "fold_mesh", + .dimensions = &.{ .closed(ctx, &.{.named(ctx, "link_x")}), .replicated(ctx) }, + }); + try sharding.attribute().format(&w.writer); + try std.testing.expectEqualSlices(u8, "#sdy.sharding<@fold_mesh, [{\"link_x\"}, {}]>", w.written()); + } + { + defer w.clearRetainingCapacity(); + const sharding: *const TensorShardingAttr = .get(ctx, .{ + .mesh = "folded_mesh", + .dimensions = &.{.closed(ctx, &.{ .named(ctx, "link_x"), .named(ctx, "link_y") })}, + }); + try sharding.attribute().format(&w.writer); + try std.testing.expectEqualSlices(u8, "#sdy.sharding<@folded_mesh, [{\"link_x\", \"link_y\"}]>", w.written()); + } + { + defer w.clearRetainingCapacity(); + const sharding: *const TensorShardingAttr = .get(ctx, .{ + .mesh = "strategy_fold", + .dimensions = &.{.closed(ctx, &.{.named(ctx, "link_x")})}, + .replicated_axes = &.{.named(ctx, "link_y")}, + }); + try sharding.attribute().format(&w.writer); + try std.testing.expectEqualSlices(u8, "#sdy.sharding<@strategy_fold, [{\"link_x\"}], replicated={\"link_y\"}>", w.written()); + } + { + defer w.clearRetainingCapacity(); + const sharding: *const TensorShardingAttr = .get(ctx, .{ + .mesh = "mix_mesh", + .dimensions = &.{ .open(ctx, &.{}), .replicated(ctx) }, + .replicated_axes = &.{ .named(ctx, "link_x"), .named(ctx, "link_y") }, + }); + try sharding.attribute().format(&w.writer); + try std.testing.expectEqualSlices(u8, "#sdy.sharding<@mix_mesh, [{?}, {}], replicated={\"link_x\", \"link_y\"}>", w.written()); + } + { + defer w.clearRetainingCapacity(); + const sharding: *const TensorShardingAttr = .get(ctx, .{ + .mesh = "3d_mesh", + .dimensions = &.{ + .closed(ctx, &.{.named(ctx, "link_x")}), + .closed(ctx, &.{.named(ctx, "link_y")}), + .closed(ctx, &.{.named(ctx, "link_z")}), + }, + }); + try sharding.attribute().format(&w.writer); + try std.testing.expectEqualSlices(u8, "#sdy.sharding<@\"3d_mesh\", [{\"link_x\"}, {\"link_y\"}, {\"link_z\"}]>", w.written()); + } +} + +test TensorShardingPerValueAttr { + const registry: *mlir.DialectRegistry = try .init(); + defer registry.deinit(); + shardy.registerDialect(registry); + func.registerExtensions(registry); + + var ctx: *mlir.Context = try .init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + + const shardings: [2]*const TensorShardingAttr = .{ + .get(ctx, .{ + .mesh = "mesh", + .dimensions = &.{.closed(ctx, &.{.named(ctx, "link_x")})}, + }), + .get(ctx, .{ + .mesh = "mesh", + .dimensions = &.{.replicated(ctx)}, + .replicated_axes = &.{.named(ctx, "link_y")}, + }), + }; + + const per_value = TensorShardingPerValueAttr.get(ctx, &shardings); + + var w: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer w.deinit(); + + try per_value.attribute().format(&w.writer); + try std.testing.expectEqualSlices( + u8, + "#sdy.sharding_per_value<[<@mesh, [{\"link_x\"}]>, <@mesh, [{}], replicated={\"link_y\"}>]>", + w.written(), + ); +} + +test ManualAxesAttr { + const registry: *mlir.DialectRegistry = try .init(); + defer registry.deinit(); + shardy.registerDialect(registry); + func.registerExtensions(registry); + + var ctx: *mlir.Context = try .init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + + const axes: [2]*const mlir.StringAttribute = .{ + .init(ctx, "link_x"), + .init(ctx, "link_y"), + }; + + const manual_axes: *const ManualAxesAttr = .get(ctx, &axes); + + try std.testing.expectEqual(2, manual_axes.numAxes()); + try std.testing.expectEqualStrings("link_x", manual_axes.axis(0)); + try std.testing.expectEqualStrings("link_y", manual_axes.axis(1)); + + var w: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer w.deinit(); + + try manual_axes.attribute().format(&w.writer); + try std.testing.expectEqualSlices(u8, "#sdy", w.written()); +} + +test "sub-axis references build" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + shardy.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + const ref = shardy.AxisRefAttr.subAxis(ctx, "x", 2, 4); + try std.testing.expectEqualStrings("x", ref.name()); + try std.testing.expectEqual(@as(i64, 4), ref.subAxisInfo().?.size()); +} diff --git a/mlir/dialects/stablehlo.zig b/mlir/dialects/stablehlo.zig deleted file mode 100644 index 9f3606e4..00000000 --- a/mlir/dialects/stablehlo.zig +++ /dev/null @@ -1,1212 +0,0 @@ -const std = @import("std"); - -const c = @import("c"); -const mlir = @import("mlir"); - -pub const abs = functors.unary_fn("stablehlo.abs").call; -pub const cosine = functors.unary_fn("stablehlo.cosine").call; -pub const sine = functors.unary_fn("stablehlo.sine").call; -pub const exponential = functors.unary_fn("stablehlo.exponential").call; -pub const exponential_minus_one = functors.unary_fn("stablehlo.exponential_minus_one").call; -pub const floor = functors.unary_fn("stablehlo.floor").call; -pub const log = functors.unary_fn("stablehlo.log").call; -pub const log_plus_one = functors.unary_fn("stablehlo.log_plus_one").call; -pub const not = functors.unary_fn("stablehlo.not").call; -pub const negate = functors.unary_fn("stablehlo.negate").call; -pub const sqrt = functors.unary_fn("stablehlo.sqrt").call; -pub const tanh = functors.unary_fn("stablehlo.tanh").call; -pub const cbrt = functors.unary_fn("stablehlo.cbrt").call; -pub const ceil = functors.unary_fn("stablehlo.ceil").call; -pub const rsqrt = functors.unary_fn("stablehlo.rsqrt").call; -pub const count_leading_zeros = functors.unary_fn("stablehlo.count_leading_zeros").call; -pub const is_finite = functors.unary_fn("stablehlo.is_finite").call; -pub const logistic = functors.unary_fn("stablehlo.logistic").call; -pub const popcnt = functors.unary_fn("stablehlo.popcnt").call; -pub const sign = functors.unary_fn("stablehlo.sign").call; -pub const real = functors.unary_fn("stablehlo.real").call; -pub const imag = functors.unary_fn("stablehlo.imag").call; - -pub const add = functors.binary_fn("stablehlo.add").call; -pub const multiply = functors.binary_fn("stablehlo.multiply").call; -pub const divide = functors.binary_fn("stablehlo.divide").call; -pub const subtract = functors.binary_fn("stablehlo.subtract").call; -pub const or_ = functors.binary_fn("stablehlo.or").call; -pub const xor = functors.binary_fn("stablehlo.xor").call; -pub const and_ = functors.binary_fn("stablehlo.and").call; -pub const atan2 = functors.binary_fn("stablehlo.atan2").call; -pub const maximum = functors.binary_fn("stablehlo.maximum").call; -pub const minimum = functors.binary_fn("stablehlo.minimum").call; -pub const power = functors.binary_fn("stablehlo.power").call; -pub const remainder = functors.binary_fn("stablehlo.remainder").call; -pub const shift_left = functors.binary_fn("stablehlo.shift_left").call; -pub const shift_right_arithmetic = functors.binary_fn("stablehlo.shift_right_arithmetic").call; -pub const shift_right_logical = functors.binary_fn("stablehlo.shift_right_logical").call; -pub const complex = functors.binary_fn("stablehlo.complex").call; - -const functors = struct { - fn unary_fn(comptime op_name: [:0]const u8) type { - return struct { - pub fn call(ctx: mlir.Context, value: mlir.Value, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, op_name, .{ - .operands = &.{value}, - .result_type_inference = true, - .location = location, - }); - } - }; - } - - pub fn binary_fn(comptime op_name: [:0]const u8) type { - return struct { - pub fn call(ctx: mlir.Context, lhs: mlir.Value, rhs: mlir.Value, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, op_name, .{ - .operands = &.{ lhs, rhs }, - .result_type_inference = true, - .location = location, - }); - } - }; - } -}; - -pub fn return_(ctx: mlir.Context, value: mlir.Value, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.return", .{ - .variadic_operands = &.{&.{value}}, - .verify = false, - .location = location, - }); -} - -pub fn returns_(ctx: mlir.Context, values: []const mlir.Value, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.return", .{ - .variadic_operands = &.{values}, - .verify = false, - .location = location, - }); -} - -pub fn bitcast_convert(ctx: mlir.Context, value: mlir.Value, result_type: mlir.Type, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.bitcast_convert", .{ - .operands = &.{value}, - .results = &.{result_type}, - .location = location, - }); -} - -pub fn cholesky(ctx: mlir.Context, value: mlir.Value, lower: bool, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.cholesky", .{ - .operands = &.{value}, - .result_type_inference = true, - .attributes = &.{ - .{ "lower", mlir.IntegerAttribute(.i1).init(ctx, @intCast(@intFromBool(lower))).as(mlir.Attribute).? }, - }, - .location = location, - }); -} - -pub fn clamp(ctx: mlir.Context, min: mlir.Value, value: mlir.Value, max: mlir.Value, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.clamp", .{ - .operands = &.{ min, value, max }, - .result_type_inference = true, - .location = location, - }); -} - -/// General matrix multiplication "a la Einstein sum" -/// Note: stablehlo doesn't do type inference for dot_general -pub fn dot_general(ctx: mlir.Context, lhs: mlir.Value, rhs: mlir.Value, result_type: mlir.Type, location: mlir.Location, opts: struct { - lhs_batching_dimensions: []const i64, - rhs_batching_dimensions: []const i64, - lhs_contracting_dimensions: []const i64, - rhs_contracting_dimensions: []const i64, - precision: []const PrecisionAttribute.Precision, -}) mlir.Operation { - var maxPrecisions: [10]mlir.Attribute = undefined; - for (opts.precision, 0..) |p, i| { - maxPrecisions[i] = PrecisionAttribute.init(ctx, p).as(mlir.Attribute).?; - } - - return mlir.Operation.make(ctx, "stablehlo.dot_general", .{ - .operands = &.{ lhs, rhs }, - .results = &.{result_type}, - .attributes = &.{ - .{ - "dot_dimension_numbers", DotDimensionNumbersAttribute.init(ctx, .{ - .lhs_batching_dimensions = opts.lhs_batching_dimensions, - .rhs_batching_dimensions = opts.rhs_batching_dimensions, - .lhs_contracting_dimensions = opts.lhs_contracting_dimensions, - .rhs_contracting_dimensions = opts.rhs_contracting_dimensions, - }).as(mlir.Attribute).?, - }, - .{ "precision_config", mlir.ArrayAttribute.init(ctx, maxPrecisions[0..opts.precision.len]).as(mlir.Attribute).? }, - }, - .location = location, - }); -} - -pub fn constant( - ctx: mlir.Context, - result_type: mlir.RankedTensorType, - elem_type: mlir.DenseElementsAttributeTypes, - raw_bytes: []const u8, - location: mlir.Location, -) mlir.Operation { - const attribute = switch (elem_type) { - inline else => |dt| mlir.DenseIntOrFPElementsAttribute(dt).init(result_type.as(mlir.Type).?, raw_bytes).as(mlir.Attribute).?, - }; - - return mlir.Operation.make(ctx, "stablehlo.constant", .{ - .operands = &.{}, - .results = &.{result_type.as(mlir.Type).?}, - .attributes = &.{.{ "value", attribute }}, - .location = location, - }); -} - -pub fn convert(ctx: mlir.Context, value: mlir.Value, result_type: mlir.Type, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.convert", .{ - .operands = &.{value}, - .results = &.{result_type}, - .location = location, - }); -} - -pub fn broadcast_in_dim(ctx: mlir.Context, operand: mlir.Value, dims: []const i64, result_type: mlir.Type, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.broadcast_in_dim", .{ - .operands = &.{operand}, - .results = &.{result_type}, - .attributes = &.{ - .{ "broadcast_dimensions", mlir.DenseArrayAttribute(.i64).init(ctx, dims).as(mlir.Attribute).? }, - }, - .location = location, - }); -} - -pub fn transpose(ctx: mlir.Context, value: mlir.Value, result_type: mlir.Type, location: mlir.Location, opts: struct { permutation: []const i64 }) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.transpose", .{ - .operands = &.{value}, - .results = &.{result_type}, - .attributes = &.{ - .{ "permutation", mlir.DenseArrayAttribute(.i64).init(ctx, opts.permutation).as(mlir.Attribute).? }, - }, - .location = location, - }); -} - -pub fn slice(ctx: mlir.Context, operand: mlir.Value, start_indices: []const i64, limit_indices: []const i64, strides: []const i64, result_type: mlir.Type, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.slice", .{ - .operands = &.{operand}, - .results = &.{result_type}, - .attributes = &.{ - .{ "start_indices", mlir.DenseArrayAttribute(.i64).init(ctx, start_indices).as(mlir.Attribute).? }, - .{ "limit_indices", mlir.DenseArrayAttribute(.i64).init(ctx, limit_indices).as(mlir.Attribute).? }, - .{ "strides", mlir.DenseArrayAttribute(.i64).init(ctx, strides).as(mlir.Attribute).? }, - }, - .location = location, - }); -} - -pub fn concatenate(ctx: mlir.Context, inputs: []const mlir.Value, dimension: i64, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.concatenate", .{ - .operands = inputs, - .result_type_inference = true, - .attributes = &.{ - .{ "dimension", mlir.IntegerAttribute(.i64).init(ctx, dimension).as(mlir.Attribute).? }, - }, - .location = location, - }); -} - -pub fn reshape(ctx: mlir.Context, value: mlir.Value, result_type: mlir.RankedTensorType, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.reshape", .{ - .operands = &.{value}, - .results = &.{result_type.as(mlir.Type).?}, - .location = location, - }); -} - -pub fn select(ctx: mlir.Context, condition: mlir.Value, then: mlir.Value, else_: mlir.Value, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.select", .{ - .operands = &.{ condition, then, else_ }, - .results = &.{then.getType()}, - .location = location, - }); -} - -pub fn gather( - ctx: mlir.Context, - value: mlir.Value, - indices: mlir.Value, - slice_sizes: []const i64, - location: mlir.Location, - args: struct { - offset_dims: []const i64, - collapsed_slice_dims: []const i64, - operand_batching_dims: []const i64, - start_indices_batching_dims: []const i64, - start_index_map: []const i64, - index_vector_dim: i64, - indices_are_sorted: bool = false, - }, -) mlir.Operation { - return mlir.Operation.make( - ctx, - "stablehlo.gather", - .{ - .operands = &.{ value, indices }, - .result_type_inference = true, - .attributes = &.{ - .{ "dimension_numbers", GatherDimensionNumbersAttribute.init( - ctx, - args.offset_dims, - args.collapsed_slice_dims, - args.operand_batching_dims, - args.start_indices_batching_dims, - args.start_index_map, - args.index_vector_dim, - ).as(mlir.Attribute).? }, - .{ "slice_sizes", mlir.DenseArrayAttribute(.i64).init(ctx, slice_sizes).as(mlir.Attribute).? }, - .{ "indices_are_sorted", mlir.BoolAttribute.init(ctx, args.indices_are_sorted).as(mlir.Attribute).? }, - }, - .location = location, - }, - ); -} - -fn elementTypeOrSelf(typ: mlir.Type) mlir.Type { - return if (typ.as(mlir.ShapedType)) |shaped| { - return shaped.elementType(); - } else typ; -} - -pub const ScatterArgs = struct { - update_window_dims: []const i64, - inserted_window_dims: []const i64, - input_batching_dims: []const i64, - scatter_indices_batching_dims: []const i64, - scatter_dims_to_operand_dims: []const i64, - index_vector_dim: i64, - indices_are_sorted: bool = false, - unique_indices: bool = false, - - pub fn getScatterDimensionNumbers(self: ScatterArgs, ctx: mlir.Context) mlir.Attribute { - return mlir.Attribute.wrap( - c.stablehloScatterDimensionNumbersGet( - ctx.inner(), - @intCast(self.update_window_dims.len), - self.update_window_dims.ptr, - @intCast(self.inserted_window_dims.len), - self.inserted_window_dims.ptr, - @intCast(self.input_batching_dims.len), - self.input_batching_dims.ptr, - @intCast(self.scatter_indices_batching_dims.len), - self.scatter_indices_batching_dims.ptr, - @intCast(self.scatter_dims_to_operand_dims.len), - self.scatter_dims_to_operand_dims.ptr, - self.index_vector_dim, - ), - ); - } -}; - -pub fn scatter( - ctx: mlir.Context, - inputs: []const mlir.Value, - scatter_indices: []const mlir.Value, - updates: []const mlir.Value, - update_block: mlir.Block, - args: ScatterArgs, - location: mlir.Location, -) mlir.Operation { - return mlir.Operation.make( - ctx, - "stablehlo.scatter", - .{ - .variadic_operands = &.{ inputs, scatter_indices, updates }, - .blocks = &.{update_block}, - .attributes = &.{ - .{ "scatter_dimension_numbers", args.getScatterDimensionNumbers(ctx) }, - .{ "indices_are_sorted", mlir.BoolAttribute.init(ctx, args.indices_are_sorted).as(mlir.Attribute).? }, - .{ "unique_indices", mlir.BoolAttribute.init(ctx, args.unique_indices).as(mlir.Attribute).? }, - }, - .result_type_inference = true, - .location = location, - }, - ); -} - -pub fn iota(ctx: mlir.Context, dimension: i64, result_type: mlir.Type, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.iota", .{ - .operands = &.{}, - .results = &.{result_type}, - .attributes = &.{ - .{ "iota_dimension", mlir.IntegerAttribute(.i64).init(ctx, dimension).as(mlir.Attribute).? }, - }, - .location = location, - }); -} - -pub fn reverse(ctx: mlir.Context, operand: mlir.Value, dimensions: []const i64, location: mlir.Location) mlir.Operation { - const result_type = operand.getType(); - return mlir.Operation.make(ctx, "stablehlo.reverse", .{ - .operands = &.{operand}, - .results = &.{result_type}, - .attributes = &.{ - .{ "dimensions", mlir.DenseArrayAttribute(.i64).init(ctx, dimensions).as(mlir.Attribute).? }, - }, - .location = location, - }); -} - -pub fn compare(ctx: mlir.Context, lhs: mlir.Value, rhs: mlir.Value, comparison_direction: ComparisonDirection, compare_type: CompareType, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.compare", .{ - .operands = &.{ lhs, rhs }, - .result_type_inference = true, - .attributes = &.{ - .{ "comparison_direction", comparison_direction.as(mlir.Attribute).? }, - .{ "compare_type", compare_type.as(mlir.Attribute).? }, - }, - .location = location, - }); -} - -pub fn reduce( - ctx: mlir.Context, - inputs: []const mlir.Value, - init_values: []const mlir.Value, - dimensions: []const i64, - blkctx: anytype, - blkfn: fn (anytype, mlir.Context, []const mlir.Value, []const mlir.Value) mlir.Operation, - location: mlir.Location, -) mlir.Operation { - const MaxBlockArguments = 32; - - const block_n_args = inputs.len + init_values.len; - const locations = ([_]mlir.Location{mlir.Location.unknown(ctx)} ** MaxBlockArguments)[0..block_n_args]; - var reduce_elem_types: [MaxBlockArguments]mlir.Type = undefined; - for (inputs, 0..) |input, i| { - const arg_type = mlir.RankedTensorType.init(&.{}, elementTypeOrSelf(input.getType())).as(mlir.Type).?; - reduce_elem_types[i] = arg_type; - reduce_elem_types[inputs.len + i] = arg_type; - } - var block = mlir.Block.open(reduce_elem_types[0..block_n_args], locations) catch unreachable; - { - defer block.close(); - - var block_inputs: [MaxBlockArguments / 2]mlir.Value = undefined; - var block_accs: [MaxBlockArguments / 2]mlir.Value = undefined; - for (0..inputs.len) |i| { - block_inputs[i] = block.argument(i); - block_accs[i] = block.argument(inputs.len + i); - } - _ = blkfn(blkctx, ctx, block_inputs[0..inputs.len], block_accs[0..init_values.len]); - } - - return mlir.Operation.make(ctx, "stablehlo.reduce", .{ - .variadic_operands = &.{ inputs, init_values }, - .result_type_inference = true, - .block = block, - .attributes = &.{ - .{ "dimensions", mlir.DenseArrayAttribute(.i64).init(ctx, dimensions).as(mlir.Attribute).? }, - }, - .location = location, - }); -} - -pub fn sort( - ctx: mlir.Context, - inputs: []const mlir.Value, - dimension: i64, - is_stable: bool, - blkctx: anytype, - compfn: fn (anytype, mlir.Context, []const mlir.Value) mlir.Operation, - location: mlir.Location, -) mlir.Operation { - const MaxBlockArguments = 32; - - const locations = ([_]mlir.Location{mlir.Location.unknown(ctx)} ** MaxBlockArguments)[0 .. inputs.len * 2]; - var sort_elem_types: [MaxBlockArguments]mlir.Type = undefined; - for (inputs, 0..) |input, i| { - const arg_type = mlir.RankedTensorType.init(&.{}, elementTypeOrSelf(input.getType())).as(mlir.Type).?; - sort_elem_types[i * 2] = arg_type; - sort_elem_types[i * 2 + 1] = arg_type; - } - var block = mlir.Block.init(sort_elem_types[0 .. inputs.len * 2], locations) catch unreachable; - - var block_inputs: [MaxBlockArguments]mlir.Value = undefined; - for (0..inputs.len * 2) |i| { - block_inputs[i] = block.argument(i); - } - _ = compfn(blkctx, ctx, block_inputs[0 .. inputs.len * 2]); - - return mlir.Operation.make(ctx, "stablehlo.sort", .{ - .variadic_operands = &.{inputs}, - .result_type_inference = true, - .block = block, - .attributes = &.{ - .{ "dimension", mlir.IntegerAttribute(.i64).init(ctx, dimension).as(mlir.Attribute).? }, - .{ "is_stable", mlir.BoolAttribute.init(ctx, is_stable).as(mlir.Attribute).? }, - }, - .location = location, - }); -} - -pub fn dynamicSlice(ctx: mlir.Context, operand: mlir.Value, new_dims: []const i64, start_indices: []const mlir.Value, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.dynamic_slice", .{ - .variadic_operands = &.{ &.{operand}, start_indices }, - .result_type_inference = true, - .attributes = &.{ - .{ "slice_sizes", mlir.DenseArrayAttribute(.i64).init(ctx, new_dims).as(mlir.Attribute).? }, - }, - .location = location, - }); -} - -pub fn round_nearest_afz(ctx: mlir.Context, value: mlir.Value, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.round_nearest_afz", .{ - .operands = &.{value}, - .result_type_inference = true, - .location = location, - }); -} - -pub fn round_nearest_even(ctx: mlir.Context, value: mlir.Value, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.round_nearest_even", .{ - .operands = &.{value}, - .result_type_inference = true, - .location = location, - }); -} - -pub const PadOpts = struct { - low: []const i64, - high: []const i64, - interior: ?[]const i64, -}; - -pub fn pad(ctx: mlir.Context, value: mlir.Value, padding_value: mlir.Value, opts: PadOpts, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.pad", .{ - .operands = &.{ value, padding_value }, - .result_type_inference = true, - .attributes = &.{ - .{ "edge_padding_low", mlir.DenseArrayAttribute(.i64).init(ctx, opts.low).as(mlir.Attribute).? }, - .{ "edge_padding_high", mlir.DenseArrayAttribute(.i64).init(ctx, opts.high).as(mlir.Attribute).? }, - .{ "interior_padding", mlir.DenseArrayAttribute(.i64).init(ctx, opts.interior.?).as(mlir.Attribute).? }, - }, - .location = location, - }); -} - -pub const TriangularSolveOpts = struct { - left_side: bool, - lower: bool, - unit_diagonal: bool, - transpose_a: Transpose.Type, -}; - -pub fn triangular_solve(ctx: mlir.Context, value: mlir.Value, other: mlir.Value, location: mlir.Location, opts: TriangularSolveOpts) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.triangular_solve", .{ - .operands = &.{ value, other }, - .result_type_inference = true, - .attributes = &.{ - .{ "left_side", mlir.IntegerAttribute(.i1).init(ctx, @intCast(@intFromBool(opts.left_side))).as(mlir.Attribute).? }, - .{ "lower", mlir.IntegerAttribute(.i1).init(ctx, @intCast(@intFromBool(opts.lower))).as(mlir.Attribute).? }, - .{ "unit_diagonal", mlir.IntegerAttribute(.i1).init(ctx, @intCast(@intFromBool(opts.unit_diagonal))).as(mlir.Attribute).? }, - .{ "transpose_a", Transpose.init(ctx, opts.transpose_a).as(mlir.Attribute).? }, - }, - .location = location, - }); -} - -pub const FftOpts = struct { - kind: FftType.Type, - length: []const i64, -}; - -pub fn fft(ctx: mlir.Context, value: mlir.Value, location: mlir.Location, opts: FftOpts) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.fft", .{ - .operands = &.{value}, - .result_type_inference = true, - .attributes = &.{ - .{ "fft_type", FftType.init(ctx, opts.kind).as(mlir.Attribute).? }, - .{ "fft_length", mlir.DenseArrayAttribute(.i64).init(ctx, opts.length).as(mlir.Attribute).? }, - }, - .location = location, - }); -} - -pub fn rng(ctx: mlir.Context, a: mlir.Value, b: mlir.Value, shape: mlir.Value, rng_distribution: RngDistribution.Type, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.rng", .{ - .operands = &.{ a, b, shape }, - .result_type_inference = true, - .attributes = &.{ - .{ "rng_distribution", RngDistribution.init(ctx, rng_distribution).as(mlir.Attribute).? }, - }, - .location = location, - }); -} - -pub fn rng_bit_generator(ctx: mlir.Context, rng_algorithm: RngAlgorithm.Type, initial_state: mlir.Value, res_state_type: mlir.Type, res_type: mlir.Type, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.rng_bit_generator", .{ - .operands = &.{initial_state}, - .results = &.{ res_state_type, res_type }, - .attributes = &.{ - .{ "rng_algorithm", RngAlgorithm.init(ctx, rng_algorithm).as(mlir.Attribute).? }, - }, - .location = location, - }); -} - -pub fn reduce_precision(ctx: mlir.Context, value: mlir.Value, exponent_bits: i32, mantissa_bits: i32, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.reduce_precision", .{ - .operands = &.{value}, - .result_type_inference = true, - .attributes = &.{ - .{ "exponent_bits", mlir.IntegerAttribute(.i32).init(ctx, exponent_bits).as(mlir.Attribute).? }, - .{ "mantissa_bits", mlir.IntegerAttribute(.i32).init(ctx, mantissa_bits).as(mlir.Attribute).? }, - }, - .location = location, - }); -} - -pub fn dynamic_update_slice(ctx: mlir.Context, operand: mlir.Value, update: mlir.Value, start_indices: []const mlir.Value, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.dynamic_update_slice", .{ - .variadic_operands = &.{ &.{operand}, &.{update}, start_indices }, - .result_type_inference = true, - .location = location, - }); -} - -pub fn tuple(ctx: mlir.Context, values: []const mlir.Value, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.tuple", .{ - .operands = values, - .result_type_inference = true, - .location = location, - }); -} - -pub fn get_tuple_element(ctx: mlir.Context, tuple_value: mlir.Value, index: i64, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.get_tuple_element", .{ - .operands = &.{tuple_value}, - .result_type_inference = true, - .attributes = &.{ - .{ "index", mlir.IntegerAttribute(.i32).init(ctx, index).as(mlir.Attribute).? }, - }, - .location = location, - }); -} - -pub const ConvolutionOpts = struct { - window_strides: []const i64, - pad_value: []const i64, - pad_shape: []const i64 = &.{}, - lhs_dilation: []const i64, - rhs_dilation: []const i64, - window_reversal: []const bool, - input_batch_dimension: i64, - input_feature_dimension: i64, - input_spatial_dimensions: []const i64, - kernel_input_feature_dimension: i64, - kernel_output_feature_dimension: i64, - kernel_spatial_dimensions: []const i64, - output_batch_dimension: i64, - output_feature_dimension: i64, - output_spatial_dimensions: []const i64, - feature_group_count: i64, - batch_group_count: i64, - precision_config: []const PrecisionAttribute.Precision = &.{}, -}; - -pub fn convolution( - ctx: mlir.Context, - lhs: mlir.Value, - rhs: mlir.Value, - opts: ConvolutionOpts, - res_type: mlir.Type, - location: mlir.Location, -) mlir.Operation { - var max_precisions: [2]mlir.Attribute = undefined; - for (opts.precision_config, 0..) |p, i| { - max_precisions[i] = PrecisionAttribute.init(ctx, p).as(mlir.Attribute).?; - } - var window_reversal: [3]i32 = undefined; - for (opts.window_reversal, 0..) |w, i| { - window_reversal[i] = @intCast(@intFromBool(w)); - } - const pad_type = mlir.IntegerType(.i64).init(ctx).as(mlir.Type).?; - const pad_shape = mlir.RankedTensorType.init(opts.pad_shape, pad_type).as(mlir.Type).?; - return mlir.Operation.make(ctx, "stablehlo.convolution", .{ - .operands = &.{ lhs, rhs }, - .results = &.{res_type}, - .attributes = &.{ - .{ "window_strides", mlir.DenseArrayAttribute(.i64).init(ctx, opts.window_strides).as(mlir.Attribute).? }, - .{ "padding", mlir.DenseIntOrFPElementsAttribute(.i64).init(pad_shape, std.mem.sliceAsBytes(opts.pad_value)).as(mlir.Attribute).? }, - .{ "lhs_dilation", mlir.DenseArrayAttribute(.i64).init(ctx, opts.lhs_dilation).as(mlir.Attribute).? }, - .{ "rhs_dilation", mlir.DenseArrayAttribute(.i64).init(ctx, opts.rhs_dilation).as(mlir.Attribute).? }, - .{ "window_reversal", mlir.DenseArrayAttribute(.bool).init(ctx, window_reversal[0..opts.window_reversal.len]).as(mlir.Attribute).? }, - .{ - "dimension_numbers", ConvDimensionNumbersAttribute.init(ctx, .{ - .input_batch_dimension = opts.input_batch_dimension, - .input_feature_dimension = opts.input_feature_dimension, - .input_spatial_dimensions = opts.input_spatial_dimensions, - .kernel_input_feature_dimension = opts.kernel_input_feature_dimension, - .kernel_output_feature_dimension = opts.kernel_output_feature_dimension, - .kernel_spatial_dimensions = opts.kernel_spatial_dimensions, - .output_batch_dimension = opts.output_batch_dimension, - .output_feature_dimension = opts.output_feature_dimension, - .output_spatial_dimensions = opts.output_spatial_dimensions, - }).as(mlir.Attribute).?, - }, - .{ "feature_group_count", mlir.IntegerAttribute(.i64).init(ctx, opts.feature_group_count).as(mlir.Attribute).? }, - .{ "batch_group_count", mlir.IntegerAttribute(.i64).init(ctx, opts.batch_group_count).as(mlir.Attribute).? }, - .{ "precision_config", mlir.ArrayAttribute.init(ctx, &max_precisions).as(mlir.Attribute).? }, - }, - .location = location, - }); -} - -pub const CustomCallOpts = struct { - call_target_name: [:0]const u8, - has_side_effect: bool, - backend_config: [:0]const u8 = &.{}, - api_version: i32, - output_operand_aliases: []const i64, -}; - -pub fn custom_call(ctx: mlir.Context, inputs: []const mlir.Value, opts: CustomCallOpts, res_types: []const mlir.Type, location: mlir.Location) mlir.Operation { - var buffer: [1024]u8 = undefined; - var fba = std.heap.FixedBufferAllocator.init(&buffer); - const allocator = fba.allocator(); - - const output_operand_aliases = allocator.alloc(mlir.Attribute, opts.output_operand_aliases.len) catch unreachable; - for (opts.output_operand_aliases, 0..) |alias, i| { - output_operand_aliases[i] = OutputOperandAliasAttribute.init(ctx, &.{}, alias, &.{}).as(mlir.Attribute).?; - } - - return mlir.Operation.make(ctx, "stablehlo.custom_call", .{ - .operands = inputs, - .results = res_types, - .attributes = &.{ - .{ "api_version", mlir.IntegerAttribute(.i32).init(ctx, opts.api_version).as(mlir.Attribute).? }, - .{ "call_target_name", mlir.StringAttribute.init(ctx, opts.call_target_name).as(mlir.Attribute).? }, - .{ "has_side_effect", mlir.BoolAttribute.init(ctx, opts.has_side_effect).as(mlir.Attribute).? }, - .{ "backend_config", mlir.StringAttribute.init(ctx, opts.backend_config).as(mlir.Attribute).? }, - .{ "output_operand_aliases", mlir.ArrayAttribute.init(ctx, output_operand_aliases).as(mlir.Attribute).? }, - }, - .location = location, - }); -} - -pub fn sharding(ctx: mlir.Context, inputs: []const mlir.Value, sharding_spec: mlir.StringAttribute, res_types: []const mlir.Type, location: mlir.Location) mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.custom_call", .{ - .operands = inputs, - .results = res_types, - .attributes = &.{ - .{ "api_version", mlir.IntegerAttribute(.i32).init(ctx, 1).asAttr() }, - .{ "call_target_name", mlir.StringAttribute.init(ctx, "Sharding").asAttr() }, - .{ "has_side_effect", mlir.BoolAttribute.init(ctx, false).asAttr() }, - .{ "backend_config", mlir.StringAttribute.init(ctx, &.{}).asAttr() }, - .{ "output_operand_aliases", mlir.ArrayAttribute.init(ctx, &.{}).asAttr() }, - .{ "mhlo.sharding", sharding_spec.asAttr() }, - }, - .location = location, - }); -} - -pub const DotDimensionNumbersAttribute = struct { - _inner: c.MlirAttribute, - - pub usingnamespace mlir.MlirHelpers(DotDimensionNumbersAttribute, .{ - .is_a_fn = c.stablehloAttributeIsADotDimensionNumbers, - .is_null_fn = c.mlirAttributeIsNull, - .dump_fn = c.mlirAttributeDump, - .equal_fn = c.mlirAttributeEqual, - }); - const Self = DotDimensionNumbersAttribute; - - pub fn init(ctx: mlir.Context, args: struct { - lhs_batching_dimensions: []const i64, - rhs_batching_dimensions: []const i64, - lhs_contracting_dimensions: []const i64, - rhs_contracting_dimensions: []const i64, - }) Self { - return Self.wrap( - c.stablehloDotDimensionNumbersGet( - ctx.inner(), - @intCast(args.lhs_batching_dimensions.len), - args.lhs_batching_dimensions.ptr, - @intCast(args.rhs_batching_dimensions.len), - args.rhs_batching_dimensions.ptr, - @intCast(args.lhs_contracting_dimensions.len), - args.lhs_contracting_dimensions.ptr, - @intCast(args.rhs_contracting_dimensions.len), - args.rhs_contracting_dimensions.ptr, - ), - ); - } - - pub fn getLhsBatchingDimensionsSize(self: Self) usize { - return @intCast(c.stablehloDotDimensionNumbersGetLhsBatchingDimensionsSize(self.inner())); - } - - pub fn getLhsBatchingDimensionsElem(self: Self, pos: usize) i64 { - return c.stablehloDotDimensionNumbersGetLhsBatchingDimensionsElem(self.inner(), @intCast(pos)); - } - - pub fn getRhsBatchingDimensionsSize(self: Self) usize { - return @intCast(c.stablehloDotDimensionNumbersGetRhsBatchingDimensionsSize(self.inner())); - } - - pub fn getRhsBatchingDimensionsElem(self: Self, pos: usize) i64 { - return c.stablehloDotDimensionNumbersGetRhsBatchingDimensionsElem(self.inner(), @intCast(pos)); - } - - pub fn getLhsContractingDimensionsSize(self: Self) usize { - return @intCast(c.stablehloDotDimensionNumbersGetLhsContractingDimensionsSize(self.inner())); - } - - pub fn getLhsContractingDimensionsElem(self: Self, pos: usize) i64 { - return c.stablehloDotDimensionNumbersGetLhsContractingDimensionsElem(self.inner(), @intCast(pos)); - } - - pub fn getRhsContractingDimensionsSize(self: Self) usize { - return @intCast(c.stablehloDotDimensionNumbersGetRhsContractingDimensionsSize(self.inner())); - } - - pub fn getRhsContractingDimensionsElem(self: Self, pos: usize) i64 { - return c.stablehloDotDimensionNumbersGetRhsContractingDimensionsElem(self.inner(), @intCast(pos)); - } -}; - -pub const GatherDimensionNumbersAttribute = struct { - _inner: c.MlirAttribute, - - pub usingnamespace mlir.MlirHelpers(GatherDimensionNumbersAttribute, .{ - .is_a_fn = c.stablehloAttributeIsAGatherDimensionNumbers, - .is_null_fn = c.mlirAttributeIsNull, - .dump_fn = c.mlirAttributeDump, - .equal_fn = c.mlirAttributeEqual, - }); - const Self = GatherDimensionNumbersAttribute; - - pub fn init( - ctx: mlir.Context, - offset_dims: []const i64, - collapsed_slice_dims: []const i64, - operand_batching_dims: []const i64, - start_indices_batching_dims: []const i64, - start_index_map: []const i64, - index_vector_dim: i64, - ) Self { - return Self.wrap( - c.stablehloGatherDimensionNumbersGet( - ctx.inner(), - @intCast(offset_dims.len), - offset_dims.ptr, - @intCast(collapsed_slice_dims.len), - collapsed_slice_dims.ptr, - @intCast(operand_batching_dims.len), - operand_batching_dims.ptr, - @intCast(start_indices_batching_dims.len), - start_indices_batching_dims.ptr, - @intCast(start_index_map.len), - start_index_map.ptr, - index_vector_dim, - ), - ); - } - - pub fn getOffsetDimsSize(self: Self) usize { - return @intCast(c.stablehloGatherDimensionNumbersGetOffsetDimsSize(self.inner())); - } - - pub fn getOffsetDimsElem(self: Self, pos: usize) i64 { - return c.stablehloGatherDimensionNumbersGetOffsetDimsElem(self.inner(), @intCast(pos)); - } - - pub fn getCollapsedSliceDimsSize(self: Self) usize { - return @intCast(c.stablehloGatherDimensionNumbersGetCollapsedSliceDimsSize(self.inner())); - } - - pub fn getCollapsedSliceDimsElem(self: Self, pos: usize) i64 { - return c.stablehloGatherDimensionNumbersGetCollapsedSliceDimsElem(self.inner(), @intCast(pos)); - } - - pub fn getStartIndexMapSize(self: Self) usize { - return @intCast(c.stablehloGatherDimensionNumbersGetStartIndexMapSize(self.inner())); - } - - pub fn getOperandBatchingDimsSize(self: Self) usize { - return @intCast(c.stablehloGatherDimensionNumbersGetOperandBatchingDimsSize(self.inner())); - } - - pub fn getOperandBatchingDimsElem(self: Self, pos: usize) i64 { - return c.stablehloGatherDimensionNumbersGetOperandBatchingDimsElem(self.inner(), @intCast(pos)); - } - - pub fn getStartIndicesBatchingDimsSize(self: Self) usize { - return @intCast(c.stablehloGatherDimensionNumbersGetStartIndicesBatchingDimsSize(self.inner())); - } - - pub fn getStartIndicesBatchingDimsElem(self: Self, pos: usize) i64 { - return c.stablehloGatherDimensionNumbersGetStartIndicesBatchingDimsElem(self.inner(), @intCast(pos)); - } - - pub fn getStartIndexMapElem(self: Self, pos: usize) i64 { - return c.stablehloGatherDimensionNumbersGetStartIndexMapElem(self.inner(), @intCast(pos)); - } - - pub fn getIndexVectorDim(self: Self) usize { - return @intCast(c.stablehloGatherDimensionNumbersGetIndexVectorDim(self.inner())); - } -}; - -pub const ConvDimensionNumbersAttribute = struct { - _inner: c.MlirAttribute, - - pub usingnamespace mlir.MlirHelpers(ConvDimensionNumbersAttribute, .{ - .is_a_fn = c.stablehloAttributeIsAConvDimensionNumbers, - .is_null_fn = c.mlirAttributeIsNull, - .dump_fn = c.mlirAttributeDump, - .equal_fn = c.mlirAttributeEqual, - }); - const Self = ConvDimensionNumbersAttribute; - - pub fn init(ctx: mlir.Context, args: struct { - input_batch_dimension: i64, - input_feature_dimension: i64, - input_spatial_dimensions: []const i64, - kernel_input_feature_dimension: i64, - kernel_output_feature_dimension: i64, - kernel_spatial_dimensions: []const i64, - output_batch_dimension: i64, - output_feature_dimension: i64, - output_spatial_dimensions: []const i64, - }) Self { - return Self.wrap( - c.stablehloConvDimensionNumbersGet( - ctx.inner(), - args.input_batch_dimension, - args.input_feature_dimension, - @intCast(args.input_spatial_dimensions.len), - args.input_spatial_dimensions.ptr, - args.kernel_input_feature_dimension, - args.kernel_output_feature_dimension, - @intCast(args.kernel_spatial_dimensions.len), - args.kernel_spatial_dimensions.ptr, - args.output_batch_dimension, - args.output_feature_dimension, - @intCast(args.output_spatial_dimensions.len), - args.output_spatial_dimensions.ptr, - ), - ); - } - - pub fn getInputBatchDimension(self: Self) i64 { - return c.stablehloConvDimensionNumbersGetInputBatchDimension(self.inner()); - } - - pub fn getInputFeatureDimension(self: Self) i64 { - return c.stablehloConvDimensionNumbersGetInputFeatureDimension(self.inner()); - } - - pub fn getInputSpatialDimensionsSize(self: Self) usize { - return @intCast(c.stablehloConvDimensionNumbersGetInputSpatialDimensionsSize(self.inner())); - } - - pub fn getInputSpatialDimensionsElem(self: Self, pos: usize) i64 { - return c.stablehloConvDimensionNumbersGetInputSpatialDimensionsElem(self.inner(), @intCast(pos)); - } - - pub fn getKernelInputFeatureDimension(self: Self) i64 { - return c.stablehloConvDimensionNumbersGetKernelInputFeatureDimension(self.inner()); - } - - pub fn getKernelOutputFeatureDimension(self: Self) i64 { - return c.stablehloConvDimensionNumbersGetKernelOutputFeatureDimension(self.inner()); - } - - pub fn getKernelSpatialDimensionsSize(self: Self) usize { - return @intCast(c.stablehloConvDimensionNumbersGetKernelSpatialDimensionsSize(self.inner())); - } - - pub fn getKernelSpatialDimensionsElem(self: Self, pos: usize) i64 { - return c.stablehloConvDimensionNumbersGetKernelSpatialDimensionsElem(self.inner(), @intCast(pos)); - } - - pub fn getOutputBatchDimension(self: Self) i64 { - return c.stablehloConvDimensionNumbersGetOutputBatchDimension(self.inner()); - } - - pub fn getOutputFeatureDimension(self: Self) i64 { - return c.stablehloConvDimensionNumbersGetOutputFeatureDimension(self.inner()); - } - - pub fn getOutputSpatialDimensionsSize(self: Self) usize { - return @intCast(c.stablehloConvDimensionNumbersGetOutputSpatialDimensionsSize(self.inner())); - } - - pub fn getOutputSpatialDimensionsElem(self: Self, pos: usize) i64 { - return c.stablehloConvDimensionNumbersGetOutputSpatialDimensionsElem(self.inner(), @intCast(pos)); - } -}; - -pub const OutputOperandAliasAttribute = struct { - _inner: c.MlirAttribute, - - pub usingnamespace mlir.MlirHelpers(OutputOperandAliasAttribute, .{ - .is_a_fn = c.stablehloAttributeIsAOutputOperandAlias, - .is_null_fn = c.mlirAttributeIsNull, - .dump_fn = c.mlirAttributeDump, - .equal_fn = c.mlirAttributeEqual, - }); - - pub fn init( - ctx: mlir.Context, - output_tuple_indices: []const i64, - operand_index: i64, - operand_tuple_indices: []const i64, - ) OutputOperandAliasAttribute { - return OutputOperandAliasAttribute.wrap(c.stablehloOutputOperandAliasGet( - ctx.inner(), - @intCast(output_tuple_indices.len), - output_tuple_indices.ptr, - @intCast(operand_index), - @intCast(operand_tuple_indices.len), - operand_tuple_indices.ptr, - )); - } -}; - -pub const PrecisionAttribute = struct { - _inner: c.MlirAttribute, - - pub usingnamespace mlir.MlirHelpers(PrecisionAttribute, .{ - .is_a_fn = c.stablehloAttributeIsAPrecisionAttr, - .is_null_fn = c.mlirAttributeIsNull, - .dump_fn = c.mlirAttributeDump, - .equal_fn = c.mlirAttributeEqual, - }); - const Self = PrecisionAttribute; - - pub const Precision = enum { - DEFAULT, - HIGH, - HIGHEST, - }; - - pub fn init(ctx: mlir.Context, value: Precision) Self { - return Self.wrap(c.stablehloPrecisionAttrGet(ctx.inner(), mlir.stringRef(@tagName(value)))); - } - - pub fn getValue(self: Self) Precision { - const value = mlir.fromStringRef(c.stablehloPrecisionAttrGetValue(self.inner())); - return std.meta.stringToEnum(Precision, value) orelse unreachable; - } -}; - -pub const ComparisonDirection = struct { - _inner: c.MlirAttribute, - - pub usingnamespace mlir.MlirHelpers(ComparisonDirection, .{ - .is_a_fn = c.stablehloAttributeIsAComparisonDirectionAttr, - .is_null_fn = c.mlirAttributeIsNull, - .dump_fn = c.mlirAttributeDump, - .equal_fn = c.mlirAttributeEqual, - }); - const Self = ComparisonDirection; - - pub const Direction = enum { - EQ, - NE, - GE, - GT, - LE, - LT, - }; - - pub fn init(ctx: mlir.Context, value: Direction) Self { - return Self.wrap(c.stablehloComparisonDirectionAttrGet(ctx.inner(), mlir.stringRef(@tagName(value)))); - } - - pub fn getValue(self: Self) Direction { - const value = mlir.fromStringRef(c.stablehloComparisonDirectionAttrGetValue(self.inner())); - return std.meta.stringToEnum(Direction, value) orelse unreachable; - } -}; - -pub const CompareType = struct { - _inner: c.MlirAttribute, - - pub usingnamespace mlir.MlirHelpers(CompareType, .{ - .is_a_fn = c.stablehloAttributeIsAComparisonTypeAttr, - .is_null_fn = c.mlirAttributeIsNull, - .dump_fn = c.mlirAttributeDump, - .equal_fn = c.mlirAttributeEqual, - }); - const Self = CompareType; - - pub const Type = enum { - SIGNED, - UNSIGNED, - FLOAT, - TOTALORDER, - }; - - pub fn init(ctx: mlir.Context, value: Type) Self { - return Self.wrap(c.stablehloComparisonTypeAttrGet(ctx.inner(), mlir.stringRef(@tagName(value)))); - } - - pub fn getValue(self: Self) Type { - const value = mlir.fromStringRef(c.stablehloComparisonTypeAttrGetValue(self.inner())); - return std.meta.stringToEnum(Type, value) orelse unreachable; - } -}; - -pub const Transpose = struct { - _inner: c.MlirAttribute, - - pub usingnamespace mlir.MlirHelpers(Transpose, .{ - .is_a_fn = c.stablehloAttributeIsATransposeAttr, - .is_null_fn = c.mlirAttributeIsNull, - .dump_fn = c.mlirAttributeDump, - .equal_fn = c.mlirAttributeEqual, - }); - const Self = Transpose; - - pub const Type = enum { - NO_TRANSPOSE, - TRANSPOSE, - ADJOINT, - }; - - pub fn init(ctx: mlir.Context, value: Type) Self { - return Self.wrap(c.stablehloTransposeAttrGet(ctx.inner(), mlir.stringRef(@tagName(value)))); - } - - pub fn getValue(self: Self) Type { - const value = mlir.fromStringRef(c.stablehloTransposeAttrGetValue(self.inner())); - return std.meta.stringToEnum(Type, value) orelse unreachable; - } -}; - -pub const FftType = struct { - _inner: c.MlirAttribute, - - pub usingnamespace mlir.MlirHelpers(FftType, .{ - .is_a_fn = c.stablehloAttributeIsAFftTypeAttr, - .is_null_fn = c.mlirAttributeIsNull, - .dump_fn = c.mlirAttributeDump, - .equal_fn = c.mlirAttributeEqual, - }); - const Self = FftType; - - pub const Type = enum { - FFT, - IFFT, - RFFT, - IRFFT, - }; - - pub fn init(ctx: mlir.Context, value: Type) Self { - return Self.wrap(c.stablehloFftTypeAttrGet(ctx.inner(), mlir.stringRef(@tagName(value)))); - } - - pub fn getValue(self: Self) Type { - const value = mlir.fromStringRef(c.stablehloFftTypeAttrGetValue(self.inner())); - return std.meta.stringToEnum(Type, value) orelse unreachable; - } -}; - -pub const RngDistribution = struct { - _inner: c.MlirAttribute, - - pub usingnamespace mlir.MlirHelpers(RngDistribution, .{ - .is_a_fn = c.stablehloAttributeIsARngDistributionAttr, - .is_null_fn = c.mlirAttributeIsNull, - .dump_fn = c.mlirAttributeDump, - .equal_fn = c.mlirAttributeEqual, - }); - const Self = RngDistribution; - - pub const Type = enum { - UNIFORM, - NORMAL, - }; - - pub fn init(ctx: mlir.Context, value: Type) Self { - return Self.wrap(c.stablehloRngDistributionAttrGet(ctx.inner(), mlir.stringRef(@tagName(value)))); - } - - pub fn getValue(self: Self) Type { - const value = mlir.fromStringRef(c.stablehloRngDistributionAttrGetValue(self.inner())); - return std.meta.stringToEnum(Type, value) orelse unreachable; - } -}; - -pub const RngAlgorithm = struct { - _inner: c.MlirAttribute, - - pub usingnamespace mlir.MlirHelpers(RngAlgorithm, .{ - .is_a_fn = c.stablehloAttributeIsARngAlgorithmAttr, - .is_null_fn = c.mlirAttributeIsNull, - .dump_fn = c.mlirAttributeDump, - .equal_fn = c.mlirAttributeEqual, - }); - const Self = RngAlgorithm; - - pub const Type = enum { - DEFAULT, - THREE_FRY, - PHILOX, - }; - - pub fn init(ctx: mlir.Context, value: Type) Self { - return Self.wrap(c.stablehloRngAlgorithmAttrGet(ctx.inner(), mlir.stringRef(@tagName(value)))); - } - - pub fn getValue(self: Self) Type { - const value = mlir.fromStringRef(c.stablehloRngAlgorithmAttrGetValue(self.inner())); - return std.meta.stringToEnum(Type, value) orelse unreachable; - } -}; - -pub fn stablehloVersionFromCompatibilityRequirement(requirement: c.MlirStablehloCompatibilityRequirement) []const u8 { - const Context = struct { - str: []const u8 = &.{}, - }; - var context = Context{}; - - c.stablehloVersionFromCompatibilityRequirement(requirement, (struct { - pub fn callback(mlir_str: c.MlirStringRef, userdata: ?*anyopaque) callconv(.C) void { - const inner_ctx: *Context = @ptrCast(@alignCast(userdata)); - inner_ctx.str = mlir.fromStringRef(mlir_str); - } - }).callback, &context); - - return context.str; -} - -pub fn stablehloGetMinimumVersion(writer: anytype) void { - var context = .{ .writer = writer }; - const WriterContext = @TypeOf(context); - - c.stablehloGetMinimumVersion((struct { - pub fn callback(mlir_str: c.MlirStringRef, userdata: ?*anyopaque) callconv(.C) void { - const inner_ctx: *WriterContext = @ptrCast(@alignCast(userdata)); - _ = inner_ctx.writer.write(mlir.fromStringRef(mlir_str)) catch unreachable; - } - }).callback, &context); -} - -pub fn serializePortableArtifact(module_str: []const u8, target_version: []const u8, writer: anytype) !void { - var context = .{ .writer = writer }; - const WriterContext = @TypeOf(context); - - try mlir.successOr(c.stablehloSerializePortableArtifactFromStringRef(mlir.stringRef(module_str), mlir.stringRef(target_version), (struct { - pub fn callback(mlir_str: c.MlirStringRef, userdata: ?*anyopaque) callconv(.C) void { - const inner_ctx: *WriterContext = @ptrCast(@alignCast(userdata)); - _ = inner_ctx.writer.write(mlir.fromStringRef(mlir_str)) catch unreachable; - } - }).callback, &context), error.InvalidMlirBytecodeVersion); -} diff --git a/mlir/dialects/stablehlo/BUILD.bazel b/mlir/dialects/stablehlo/BUILD.bazel new file mode 100644 index 00000000..d7214000 --- /dev/null +++ b/mlir/dialects/stablehlo/BUILD.bazel @@ -0,0 +1,23 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "stablehlo", + import_name = "mlir/dialects/stablehlo", + main = "stablehlo.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//mlir", + "//stdx", + "@stablehlo//:stablehlo_dialect_capi", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":stablehlo", + "//mlir", + ], +) diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig new file mode 100644 index 00000000..385a95b0 --- /dev/null +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -0,0 +1,1655 @@ +const std = @import("std"); + +const c = @import("c"); +const mlir = @import("mlir"); +const stringRef = mlir.stringRef; +const stdx = @import("stdx"); + +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("stablehlo"); +} + +pub const abs = unaryOp("stablehlo.abs").call; +pub const cosine = unaryOp("stablehlo.cosine").call; +pub const sine = unaryOp("stablehlo.sine").call; +pub const exponential = unaryOp("stablehlo.exponential").call; +pub const exponential_minus_one = unaryOp("stablehlo.exponential_minus_one").call; +pub const floor = unaryOp("stablehlo.floor").call; +pub const log = unaryOp("stablehlo.log").call; +pub const log_plus_one = unaryOp("stablehlo.log_plus_one").call; +pub const not = unaryOp("stablehlo.not").call; +pub const negate = unaryOp("stablehlo.negate").call; +pub const sqrt = unaryOp("stablehlo.sqrt").call; +pub const tanh = unaryOp("stablehlo.tanh").call; +pub const cbrt = unaryOp("stablehlo.cbrt").call; +pub const ceil = unaryOp("stablehlo.ceil").call; +pub const rsqrt = unaryOp("stablehlo.rsqrt").call; +pub const count_leading_zeros = unaryOp("stablehlo.count_leading_zeros").call; +pub const is_finite = unaryOp("stablehlo.is_finite").call; +pub const logistic = unaryOp("stablehlo.logistic").call; +pub const popcnt = unaryOp("stablehlo.popcnt").call; +pub const sign = unaryOp("stablehlo.sign").call; +pub const real = unaryOp("stablehlo.real").call; +pub const imag = unaryOp("stablehlo.imag").call; +pub const round_nearest_afz = unaryOp("stablehlo.round_nearest_afz").call; +pub const round_nearest_even = unaryOp("stablehlo.round_nearest_even").call; +pub const tuple = unaryOp("stablehlo.tuple").call; + +pub const add = binaryOp("stablehlo.add").call; +pub const multiply = binaryOp("stablehlo.multiply").call; +pub const divide = binaryOp("stablehlo.divide").call; +pub const subtract = binaryOp("stablehlo.subtract").call; +pub const or_ = binaryOp("stablehlo.or").call; +pub const xor = binaryOp("stablehlo.xor").call; +pub const and_ = binaryOp("stablehlo.and").call; +pub const atan2 = binaryOp("stablehlo.atan2").call; +pub const maximum = binaryOp("stablehlo.maximum").call; +pub const minimum = binaryOp("stablehlo.minimum").call; +pub const power = binaryOp("stablehlo.power").call; +pub const remainder = binaryOp("stablehlo.remainder").call; +pub const shift_left = binaryOp("stablehlo.shift_left").call; +pub const shift_right_arithmetic = binaryOp("stablehlo.shift_right_arithmetic").call; +pub const shift_right_logical = binaryOp("stablehlo.shift_right_logical").call; +pub const complex = binaryOp("stablehlo.complex").call; + +pub const bitcast_convert = castOp("stablehlo.bitcast_convert").call; +pub const convert = castOp("stablehlo.convert").call; +pub const reshape = castOp("stablehlo.reshape").call; + +fn castOp(comptime op_name: []const u8) type { + return struct { + pub fn call(ctx: *mlir.Context, value: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, op_name, .{ + .operands = .{ .flat = &.{value} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); + } + }; +} + +fn unaryOp(comptime op_name: []const u8) type { + return struct { + pub fn call(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, op_name, .{ + .operands = .{ .flat = &.{value} }, + .result_type_inference = true, + .location = location, + }); + } + }; +} + +pub fn binaryOp(comptime op_name: []const u8) type { + return struct { + pub fn call(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, op_name, .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .result_type_inference = true, + .location = location, + }); + } + }; +} + +pub fn cholesky(ctx: *mlir.Context, value: *const mlir.Value, lower: bool, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.cholesky", .{ + .operands = .{ .flat = &.{value} }, + .result_type_inference = true, + .attributes = &.{ + .named(ctx, "lower", .int(ctx, .i1, @intFromBool(lower))), + }, + .location = location, + }); +} + +pub fn clamp(ctx: *mlir.Context, min: *const mlir.Value, value: *const mlir.Value, max: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.clamp", .{ + .operands = .{ .flat = &.{ min, value, max } }, + .result_type_inference = true, + .location = location, + }); +} + +pub const CompositeOpts = struct { + name: []const u8, + decomposition: []const u8, + composite_attributes: []const mlir.NamedAttribute = &.{}, + version: i32 = 0, +}; + +pub fn composite( + ctx: *mlir.Context, + inputs: []const *const mlir.Value, + result_types: []const *const mlir.Type, + opts: CompositeOpts, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.composite", .{ + .operands = .{ .flat = inputs }, + .results = .{ .flat = result_types }, + .attributes = &.{ + .named(ctx, "name", .string(ctx, opts.name)), + .named(ctx, "decomposition", .flatSymbolRef(ctx, opts.decomposition)), + .named(ctx, "composite_attributes", .dict(ctx, opts.composite_attributes)), + .named(ctx, "version", .int(ctx, .i32, opts.version)), + }, + .verify = false, + .location = location, + }); +} + +pub const DotPrecision = union(enum) { + fast, + high, + highest, + algorithm: DotAlgorithm, + + pub fn precision(self: DotPrecision) PrecisionAttr.Precision { + return switch (self) { + .fast => .DEFAULT, + .high => .HIGH, + .highest => .HIGHEST, + // When we specify the dot algorithm, we should not specify the precision. + .algorithm => .DEFAULT, + }; + } + + pub fn algorithmAttr(self: DotPrecision, ctx: *mlir.Context, element_type: *const mlir.Type) ?*const mlir.Attribute { + return switch (self) { + .algorithm => |algo| dotAlgorithmAttr(ctx, algo, element_type), + else => null, + }; + } +}; + +pub const DotAlgorithm = struct { + accumulation: mlir.FloatTypes, + // Note stablehlo distinguish between left/right component_count + // but all the supported algorithm have the same component_count on both side. + component_count: u8 = 1, + num_primitive_operations: u8 = 1, + allow_imprecise_accumulation: bool = false, + + // bf16_6x: each input is decomposed to 3 bf16 components, then 6 dot operations are done on those components, and the result is accumulated in f32. + // not sure where this is available. + pub const bf16_6x: DotAlgorithm = .{ + .accumulation = .f32, + .component_count = 1, + .num_primitive_operations = 6, + .allow_imprecise_accumulation = false, + }; +}; + +pub fn dotAlgorithmAttr(ctx: *mlir.Context, dot_algorithm: DotAlgorithm, element_type: *const mlir.Type) *const mlir.Attribute { + return @ptrCast(c.stablehloDotAlgorithmGet( + ctx.ptr(), + element_type.ptr(), + element_type.ptr(), + mlir.Type.float(ctx, dot_algorithm.accumulation).ptr(), + dot_algorithm.component_count, + dot_algorithm.component_count, + dot_algorithm.num_primitive_operations, + dot_algorithm.allow_imprecise_accumulation, + ).ptr); +} + +/// General matrix multiplication "a la Einstein sum" +/// Note: stablehlo doesn't do type inference for dot_general +pub fn dot_general( + ctx: *mlir.Context, + lhs: *const mlir.Value, + rhs: *const mlir.Value, + result_type: *const mlir.Type, + opts: struct { + lhs_batching_dimensions: []const i64, + rhs_batching_dimensions: []const i64, + lhs_contracting_dimensions: []const i64, + rhs_contracting_dimensions: []const i64, + dot_precision: DotPrecision, + }, + location: *const mlir.Location, +) *mlir.Operation { + const precisions: [2]*const mlir.Attribute = @splat(precisionAttr(ctx, opts.dot_precision.precision())); + const attributes = [3]mlir.NamedAttribute{ + .named( + ctx, + "dot_dimension_numbers", + dotDimensionNumbersAttr(ctx, .{ + .lhs_batching_dimensions = opts.lhs_batching_dimensions, + .rhs_batching_dimensions = opts.rhs_batching_dimensions, + .lhs_contracting_dimensions = opts.lhs_contracting_dimensions, + .rhs_contracting_dimensions = opts.rhs_contracting_dimensions, + }), + ), + .named(ctx, "precision_config", .array(ctx, &precisions)), + // keep algorithm as the last attribute so we can omit it when it's not set. + .named(ctx, "algorithm", switch (opts.dot_precision) { + .algorithm => |v| dotAlgorithmAttr(ctx, v, lhs.type_()), + else => undefined, + }), + }; + const n_attributes = if (opts.dot_precision == .algorithm) attributes.len else attributes.len - 1; + return mlir.Operation.make(ctx, "stablehlo.dot_general", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{result_type} }, + .attributes = attributes[0..n_attributes], + .location = location, + }); +} + +pub fn constant( + ctx: *mlir.Context, + dims: []const i64, + elem_type: *const mlir.Type, + raw_bytes: []const u8, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.constant", .{ + .operands = .{ .flat = &.{} }, + .results = .{ .flat = &.{.rankedTensor(dims, elem_type)} }, + .attributes = &.{ + .named(ctx, "value", .denseElements(.rankedTensor(dims, elem_type), raw_bytes)), + }, + .location = location, + }); +} + +pub fn broadcast_in_dim( + ctx: *mlir.Context, + operand: *const mlir.Value, + dims: []const i64, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.broadcast_in_dim", .{ + .operands = .{ .flat = &.{operand} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "broadcast_dimensions", .denseArray(ctx, .i64, dims)), + }, + .location = location, + }); +} + +pub fn transpose( + ctx: *mlir.Context, + value: *const mlir.Value, + result_type: *const mlir.Type, + opts: struct { permutation: []const i64 }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.transpose", .{ + .operands = .{ .flat = &.{value} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "permutation", .denseArray(ctx, .i64, opts.permutation)), + }, + .location = location, + }); +} + +pub fn slice( + ctx: *mlir.Context, + operand: *const mlir.Value, + start_indices: []const i64, + limit_indices: []const i64, + strides: []const i64, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.slice", .{ + .operands = .{ .flat = &.{operand} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "start_indices", .denseArray(ctx, .i64, start_indices)), + .named(ctx, "limit_indices", .denseArray(ctx, .i64, limit_indices)), + .named(ctx, "strides", .denseArray(ctx, .i64, strides)), + }, + .location = location, + }); +} + +pub fn concatenate(ctx: *mlir.Context, inputs: []const *const mlir.Value, dimension: i64, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.concatenate", .{ + .operands = .{ .flat = inputs }, + .result_type_inference = true, + .attributes = &.{ + .named(ctx, "dimension", .int(ctx, .i64, dimension)), + }, + .location = location, + }); +} + +pub fn select( + ctx: *mlir.Context, + condition: *const mlir.Value, + then: *const mlir.Value, + else_: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.select", .{ + .operands = .{ .flat = &.{ condition, then, else_ } }, + .results = .{ .flat = &.{then.type_()} }, + .location = location, + }); +} + +pub fn gather( + ctx: *mlir.Context, + value: *const mlir.Value, + indices: *const mlir.Value, + slice_sizes: []const i64, + args: struct { + offset_dims: []const i64, + collapsed_slice_dims: []const i64, + operand_batching_dims: []const i64, + start_indices_batching_dims: []const i64, + start_index_map: []const i64, + index_vector_dim: i64, + indices_are_sorted: bool = false, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make( + ctx, + "stablehlo.gather", + .{ + .operands = .{ .flat = &.{ value, indices } }, + .result_type_inference = true, + .attributes = &.{ + .named(ctx, "dimension_numbers", gatherDimensionNumbersAttr( + ctx, + args.offset_dims, + args.collapsed_slice_dims, + args.operand_batching_dims, + args.start_indices_batching_dims, + args.start_index_map, + args.index_vector_dim, + )), + .named(ctx, "slice_sizes", .denseArray(ctx, .i64, slice_sizes)), + .named(ctx, "indices_are_sorted", .boolean(ctx, args.indices_are_sorted)), + }, + .location = location, + }, + ); +} + +pub const ScatterArgs = struct { + update_window_dims: []const i64, + inserted_window_dims: []const i64, + input_batching_dims: []const i64, + scatter_indices_batching_dims: []const i64, + scatter_dims_to_operand_dims: []const i64, + index_vector_dim: i64, + indices_are_sorted: bool = false, + unique_indices: bool = false, + + pub fn scatterDimensionNumbers(self: ScatterArgs, ctx: *mlir.Context) *const mlir.Attribute { + return @ptrCast(c.stablehloScatterDimensionNumbersGet( + ctx.ptr(), + @intCast(self.update_window_dims.len), + self.update_window_dims.ptr, + @intCast(self.inserted_window_dims.len), + self.inserted_window_dims.ptr, + @intCast(self.input_batching_dims.len), + self.input_batching_dims.ptr, + @intCast(self.scatter_indices_batching_dims.len), + self.scatter_indices_batching_dims.ptr, + @intCast(self.scatter_dims_to_operand_dims.len), + self.scatter_dims_to_operand_dims.ptr, + self.index_vector_dim, + ).ptr); + } +}; + +pub fn scatter( + ctx: *mlir.Context, + inputs: []const *const mlir.Value, + scatter_indices: []const *const mlir.Value, + updates: []const *const mlir.Value, + update_block: *mlir.Block, + args: ScatterArgs, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make( + ctx, + "stablehlo.scatter", + .{ + .operands = .{ .variadic = &.{ inputs, scatter_indices, updates } }, + .blocks = &.{update_block}, + .attributes = &.{ + .named(ctx, "scatter_dimension_numbers", args.scatterDimensionNumbers(ctx)), + .named(ctx, "indices_are_sorted", .boolean(ctx, args.indices_are_sorted)), + .named(ctx, "unique_indices", .boolean(ctx, args.unique_indices)), + }, + .result_type_inference = true, + .location = location, + }, + ); +} + +pub fn iota(ctx: *mlir.Context, dimension: i64, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.iota", .{ + .operands = .{ .flat = &.{} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "iota_dimension", .int(ctx, .i64, dimension)), + }, + .location = location, + }); +} + +pub fn reverse(ctx: *mlir.Context, operand: *const mlir.Value, dimensions: []const i64, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.reverse", .{ + .operands = .{ .flat = &.{operand} }, + .results = .{ .flat = &.{operand.type_()} }, + .attributes = &.{ + .named(ctx, "dimensions", .denseArray(ctx, .i64, dimensions)), + }, + .location = location, + }); +} + +pub fn compare( + ctx: *mlir.Context, + lhs: *const mlir.Value, + rhs: *const mlir.Value, + comparison_direction: ComparisonDirectionAttr.Direction, + compare_type: CompareTypeAttr.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.compare", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .result_type_inference = true, + .attributes = &.{ + .named(ctx, "comparison_direction", comparisonDirectionAttr(ctx, comparison_direction)), + .named(ctx, "compare_type", compareTypeAttr(ctx, compare_type)), + }, + .location = location, + }); +} + +pub fn dynamic_slice( + ctx: *mlir.Context, + operand: *const mlir.Value, + new_dims: []const i64, + start_indices: []const *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.dynamic_slice", .{ + .operands = .{ .variadic = &.{ &.{operand}, start_indices } }, + .result_type_inference = true, + .attributes = &.{ + .named(ctx, "slice_sizes", .denseArray(ctx, .i64, new_dims)), + }, + .location = location, + }); +} + +pub const PadOpts = struct { + low: []const i64, + high: []const i64, + interior: []const i64, +}; + +pub fn pad(ctx: *mlir.Context, value: *const mlir.Value, padding_value: *const mlir.Value, opts: PadOpts, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.pad", .{ + .operands = .{ .flat = &.{ value, padding_value } }, + .result_type_inference = true, + .attributes = &.{ + .named(ctx, "edge_padding_low", .denseArray(ctx, .i64, opts.low)), + .named(ctx, "edge_padding_high", .denseArray(ctx, .i64, opts.high)), + .named(ctx, "interior_padding", .denseArray(ctx, .i64, opts.interior)), + }, + .location = location, + }); +} + +pub const TriangularSolveOpts = struct { + left_side: bool, + lower: bool, + unit_diagonal: bool, + transpose_a: TransposeAttr.Type, +}; + +pub fn triangular_solve(ctx: *mlir.Context, value: *const mlir.Value, other: *const mlir.Value, opts: TriangularSolveOpts, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.triangular_solve", .{ + .operands = .{ .flat = &.{ value, other } }, + .result_type_inference = true, + .attributes = &.{ + .named(ctx, "left_side", .int(ctx, .i1, @intFromBool(opts.left_side))), + .named(ctx, "lower", .int(ctx, .i1, @intFromBool(opts.lower))), + .named(ctx, "unit_diagonal", .int(ctx, .i1, @intFromBool(opts.unit_diagonal))), + .named(ctx, "transpose_a", transposeAttr(ctx, opts.transpose_a)), + }, + .location = location, + }); +} + +pub const FftOpts = struct { + kind: FftTypeAttr.Type, + length: []const i64, +}; + +pub fn fft(ctx: *mlir.Context, value: *const mlir.Value, opts: FftOpts, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.fft", .{ + .operands = .{ .flat = &.{value} }, + .result_type_inference = true, + .attributes = &.{ + .named(ctx, "fft_type", fftTypeAttr(ctx, opts.kind)), + .named(ctx, "fft_length", .denseArray(ctx, .i64, opts.length)), + }, + .location = location, + }); +} + +pub const RngDistributionAttr = opaque { + const M = mlir.Methods(RngDistributionAttr, c.MlirAttribute); + + pub const isAFn = c.stablehloAttributeIsARngDistributionAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub const Type = enum { + UNIFORM, + NORMAL, + }; + + pub const InitArgs = struct { + value: Type, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) *const RngDistributionAttr { + return @ptrCast(c.stablehloRngDistributionAttrGet(ctx.ptr(), stringRef(@tagName(args.value))).ptr); + } + + pub fn value(self: *const RngDistributionAttr) Type { + return std.meta.stringToEnum( + Type, + mlir.string(c.stablehloRngDistributionAttrGetValue(self.ptr())), + ) orelse unreachable; + } + + pub fn attribute(self: *const RngDistributionAttr) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +pub fn rngDistributionAttr(ctx: *mlir.Context, value: RngDistributionAttr.Type) *const mlir.Attribute { + return RngDistributionAttr.get(ctx, .{ .value = value }).attribute(); +} + +pub const RngAlgorithmAttr = opaque { + const M = mlir.Methods(RngAlgorithmAttr, c.MlirAttribute); + + pub const isAFn = c.stablehloAttributeIsARngAlgorithmAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub const Type = enum { + DEFAULT, + THREE_FRY, + PHILOX, + }; + + pub const InitArgs = struct { + value: Type, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) *const RngAlgorithmAttr { + return @ptrCast(c.stablehloRngAlgorithmAttrGet(ctx.ptr(), stringRef(@tagName(args.value))).ptr); + } + + pub fn value(self: *const RngAlgorithmAttr) Type { + return std.meta.stringToEnum( + Type, + mlir.string(c.stablehloRngAlgorithmAttrGetValue(self.ptr())), + ) orelse unreachable; + } + + pub fn attribute(self: *const RngAlgorithmAttr) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +pub fn rngAlgorithmAttr(ctx: *mlir.Context, value: RngAlgorithmAttr.Type) *const mlir.Attribute { + return RngAlgorithmAttr.get(ctx, .{ .value = value }).attribute(); +} + +pub fn rng(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, shape: *const mlir.Value, rng_distribution: RngDistributionAttr.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.rng", .{ + .operands = .{ .flat = &.{ a, b, shape } }, + .result_type_inference = true, + .attributes = &.{ + .named(ctx, "rng_distribution", rngDistributionAttr(ctx, rng_distribution)), + }, + .location = location, + }); +} + +pub fn rng_bit_generator( + ctx: *mlir.Context, + rng_algorithm: RngAlgorithmAttr.Type, + initial_state: *const mlir.Value, + res_state_type: *const mlir.Type, + res_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.rng_bit_generator", .{ + .operands = .{ .flat = &.{initial_state} }, + .results = .{ .flat = &.{ res_state_type, res_type } }, + .attributes = &.{ + .named(ctx, "rng_algorithm", rngAlgorithmAttr(ctx, rng_algorithm)), + }, + .location = location, + }); +} + +pub fn reduce_precision(ctx: *mlir.Context, value: *const mlir.Value, exponent_bits: i32, mantissa_bits: i32, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.reduce_precision", .{ + .operands = .{ .flat = &.{value} }, + .result_type_inference = true, + .attributes = &.{ + .named(ctx, "exponent_bits", .int(ctx, .i32, exponent_bits)), + .named(ctx, "mantissa_bits", .int(ctx, .i32, mantissa_bits)), + }, + .location = location, + }); +} + +pub fn dynamic_update_slice(ctx: *mlir.Context, operand: *const mlir.Value, update: *const mlir.Value, start_indices: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.dynamic_update_slice", .{ + .operands = .{ .variadic = &.{ &.{operand}, &.{update}, start_indices } }, + .result_type_inference = true, + .location = location, + }); +} + +pub fn while_( + ctx: *mlir.Context, + operands: []const *const mlir.Value, + result_types: []const *const mlir.Type, + cond_block: *mlir.Block, + body_block: *mlir.Block, + location: *const mlir.Location, +) *mlir.Operation { + _ = ctx; + var state = mlir.OperationState.init("stablehlo.while", location); + state.addOperands(operands); + state.addResults(result_types); + + const cond_region = mlir.Region.init(); + cond_region.appendOwnedBlock(cond_block); + const body_region = mlir.Region.init(); + body_region.appendOwnedBlock(body_block); + state.addOwnedRegions(&.{ cond_region, body_region }); + + return mlir.Operation.init(&state) catch @panic("Failed to create stablehlo.while operation"); +} + +/// `stablehlo.if`. Not verified: its branches may use values defined +/// outside them, which only verify once the operation is in place. +pub fn if_( + ctx: *mlir.Context, + pred: *const mlir.Value, + true_branch_block: *mlir.Block, + false_branch_block: *mlir.Block, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.if", .{ + .operands = .{ .flat = &.{pred} }, + .result_type_inference = true, + .blocks = &.{ true_branch_block, false_branch_block }, + .verify = false, + .location = location, + }); +} + +pub fn reduce( + ctx: *mlir.Context, + inputs: []const *const mlir.Value, + init_values: []const *const mlir.Value, + body_block: *mlir.Block, + dimensions: []const i64, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.reduce", .{ + .operands = .{ .variadic = &.{ inputs, init_values } }, + .result_type_inference = true, + .blocks = &.{body_block}, + .attributes = &.{ + .named(ctx, "dimensions", .denseArray(ctx, .i64, dimensions)), + }, + .location = location, + }); +} + +pub fn reduce_window( + ctx: *mlir.Context, + inputs: []const *const mlir.Value, + init_values: []const *const mlir.Value, + body_block: *mlir.Block, + window_dimensions: []const i64, + opts: struct { + window_strides: ?[]const i64 = null, + base_dilations: ?[]const i64 = null, + window_dilations: ?[]const i64 = null, + /// (low, high) padding of each dimension. + padding: ?[]const [2]i64 = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; + attrs.appendAssumeCapacity(.named(ctx, "window_dimensions", .denseArray(ctx, .i64, window_dimensions))); + if (opts.window_strides) |v| attrs.appendAssumeCapacity(.named(ctx, "window_strides", .denseArray(ctx, .i64, v))); + if (opts.base_dilations) |v| attrs.appendAssumeCapacity(.named(ctx, "base_dilations", .denseArray(ctx, .i64, v))); + if (opts.window_dilations) |v| attrs.appendAssumeCapacity(.named(ctx, "window_dilations", .denseArray(ctx, .i64, v))); + if (opts.padding) |v| attrs.appendAssumeCapacity(.named(ctx, "padding", .denseElements( + .rankedTensor(&.{ @intCast(v.len), 2 }, .int(ctx, .i64)), + @as([]const i64, @ptrCast(v)), + ))); + return mlir.Operation.make(ctx, "stablehlo.reduce_window", .{ + .operands = .{ .variadic = &.{ inputs, init_values } }, + .result_type_inference = true, + .blocks = &.{body_block}, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +pub fn sort( + ctx: *mlir.Context, + inputs: []const *const mlir.Value, + comparator_block: *mlir.Block, + opts: struct { + dimension: i64 = -1, + is_stable: bool = false, + }, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.sort", .{ + .operands = .{ .variadic = &.{inputs} }, + .result_type_inference = true, + .blocks = &.{comparator_block}, + .attributes = &.{ + .named(ctx, "dimension", .int(ctx, .i64, opts.dimension)), + .named(ctx, "is_stable", .boolean(ctx, opts.is_stable)), + }, + .location = location, + }); +} + +pub fn get_tuple_element(ctx: *mlir.Context, tuple_value: *const mlir.Value, index: i64, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.get_tuple_element", .{ + .operands = .{ .flat = &.{tuple_value} }, + .result_type_inference = true, + .attributes = &.{ + .named(ctx, "index", .int(ctx, .i32, index)), + }, + .location = location, + }); +} + +pub const ConvolutionOpts = struct { + window_strides: []const i64, + pad_value: []const i64, + pad_shape: []const i64 = &.{}, + lhs_dilation: []const i64, + rhs_dilation: []const i64, + window_reversal: []const bool, + input_batch_dimension: i64, + input_feature_dimension: i64, + input_spatial_dimensions: []const i64, + kernel_input_feature_dimension: i64, + kernel_output_feature_dimension: i64, + kernel_spatial_dimensions: []const i64, + output_batch_dimension: i64, + output_feature_dimension: i64, + output_spatial_dimensions: []const i64, + feature_group_count: i64, + batch_group_count: i64, + precision_config: []const PrecisionAttr.Precision = &.{}, +}; + +pub fn convolution( + ctx: *mlir.Context, + lhs: *const mlir.Value, + rhs: *const mlir.Value, + res_type: *const mlir.Type, + opts: ConvolutionOpts, + location: *const mlir.Location, +) *mlir.Operation { + var max_precisions: [2]*const mlir.Attribute = undefined; + for (opts.precision_config, 0..) |p, i| { + max_precisions[i] = precisionAttr(ctx, p); + } + var window_reversal: [3]i32 = undefined; + for (opts.window_reversal, 0..) |w, i| { + window_reversal[i] = @intCast(@intFromBool(w)); + } + return mlir.Operation.make(ctx, "stablehlo.convolution", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{res_type} }, + .attributes = &.{ + .named(ctx, "window_strides", .denseArray(ctx, .i64, opts.window_strides)), + .named(ctx, "padding", .denseElements(.rankedTensor(opts.pad_shape, .int(ctx, .i64)), opts.pad_value)), + .named(ctx, "lhs_dilation", .denseArray(ctx, .i64, opts.lhs_dilation)), + .named(ctx, "rhs_dilation", .denseArray(ctx, .i64, opts.rhs_dilation)), + .named(ctx, "window_reversal", .denseArray(ctx, .bool, window_reversal[0..opts.window_reversal.len])), + .named(ctx, "dimension_numbers", convDimensionNumbersAttr(ctx, .{ + .input_batch_dimension = opts.input_batch_dimension, + .input_feature_dimension = opts.input_feature_dimension, + .input_spatial_dimensions = opts.input_spatial_dimensions, + .kernel_input_feature_dimension = opts.kernel_input_feature_dimension, + .kernel_output_feature_dimension = opts.kernel_output_feature_dimension, + .kernel_spatial_dimensions = opts.kernel_spatial_dimensions, + .output_batch_dimension = opts.output_batch_dimension, + .output_feature_dimension = opts.output_feature_dimension, + .output_spatial_dimensions = opts.output_spatial_dimensions, + })), + .named(ctx, "feature_group_count", .int(ctx, .i64, opts.feature_group_count)), + .named(ctx, "batch_group_count", .int(ctx, .i64, opts.batch_group_count)), + .named(ctx, "precision_config", .array(ctx, &max_precisions)), + }, + .location = location, + }); +} + +pub const CustomCallOpts = struct { + pub const MAX_OPERANDS = 64; + pub const MAX_RESULTS = 16; + + pub const ApiVersion = enum(i32) { + original = 1, + status_returning = 2, + status_returning_unified = 3, + typed_ffi = 4, + }; + + pub const BackendConfig = union(ApiVersion) { + original: []const u8, + status_returning: []const u8, + status_returning_unified: []const u8, + typed_ffi: *const mlir.Attribute, + }; + + pub const OutputOperandAlias = struct { + output_index: i64, + operand_index: i64, + }; + + call_target_name: []const u8, + has_side_effect: ?bool = null, + api_version: ?ApiVersion = null, + backend_config: ?BackendConfig = null, + operand_layouts: ?[]const []const usize = null, + result_layouts: ?[]const []const usize = null, + output_operand_aliases: ?[]const OutputOperandAlias = null, + additional_attributes: []const mlir.NamedAttribute = &.{}, +}; + +pub fn custom_call(ctx: *mlir.Context, inputs: []const *const mlir.Value, result_types: []const *const mlir.Type, opts: CustomCallOpts, location: *const mlir.Location) *mlir.Operation { + const MAX_OPERANDS = CustomCallOpts.MAX_OPERANDS; + const MAX_RESULTS = CustomCallOpts.MAX_RESULTS; + const MINOR_TO_MAJOR = comptime blk: { + var ret: [mlir.ShapedType.MAX_RANK]usize = undefined; + for (0..ret.len) |i| { + ret[i] = @intCast(mlir.ShapedType.MAX_RANK - i - 1); + } + break :blk ret; + }; + + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 32) = .empty; + attrs.appendSliceAssumeCapacity(&.{ + .named(ctx, "call_target_name", .string(ctx, opts.call_target_name)), + }); + attrs.appendSliceAssumeCapacity(opts.additional_attributes); + + if (opts.has_side_effect) |has_side_effects| { + attrs.appendAssumeCapacity( + .named(ctx, "has_side_effect", .boolean(ctx, has_side_effects)), + ); + } + + const backend_config_api_version: ?CustomCallOpts.ApiVersion = if (opts.backend_config) |bc| bc else null; + if (opts.api_version orelse backend_config_api_version) |v| { + attrs.appendAssumeCapacity( + .named(ctx, "api_version", .int(ctx, .i32, @backingInt(v))), + ); + } + if (opts.backend_config) |backend_config| { + attrs.appendAssumeCapacity( + .named(ctx, "backend_config", switch (backend_config) { + .typed_ffi => |v| @ptrCast(v), + inline else => |v| .string(ctx, v), + }), + ); + } + + if (opts.output_operand_aliases) |output_operand_aliases| { + var buffer: stdx.BoundedArray(*const mlir.Attribute, MAX_RESULTS) = .empty; + for (output_operand_aliases) |alias| { + const output_tuple_indices = if (result_types.len > 1) &[1]i64{alias.output_index} else &.{}; + buffer.appendAssumeCapacity( + outputOperandAliasAttr(ctx, .{ .operand_index = alias.operand_index, .output_tuple_indices = output_tuple_indices }), + ); + } + attrs.appendAssumeCapacity( + .named(ctx, "output_operand_aliases", .array(ctx, buffer.constSlice())), + ); + } + + var layouts_buffer: stdx.BoundedArray(*const mlir.Attribute, MAX_OPERANDS) = .empty; + if (opts.operand_layouts) |layouts| { + for (layouts) |ol| { + layouts_buffer.appendAssumeCapacity( + .denseElements( + .rankedTensor(&.{@intCast(ol.len)}, .index(ctx)), + ol, + ), + ); + } + } else { + for (inputs) |input| { + const shaped_type = input.type_().isA(mlir.ShapedType).?; + const ol = MINOR_TO_MAJOR[MINOR_TO_MAJOR.len - shaped_type.rank() ..]; + layouts_buffer.appendAssumeCapacity( + .denseElements( + .rankedTensor(&.{@intCast(ol.len)}, .index(ctx)), + ol, + ), + ); + } + } + attrs.appendAssumeCapacity( + .named(ctx, "operand_layouts", .array(ctx, layouts_buffer.constSlice())), + ); + + layouts_buffer.clear(); + if (opts.result_layouts) |layouts| { + for (layouts) |rl| { + layouts_buffer.appendAssumeCapacity( + .denseElements( + .rankedTensor(&.{@intCast(rl.len)}, .index(ctx)), + rl, + ), + ); + } + } else { + for (result_types) |result_type| { + const shaped_type = result_type.isA(mlir.ShapedType).?; + const rl = MINOR_TO_MAJOR[MINOR_TO_MAJOR.len - shaped_type.rank() ..]; + layouts_buffer.appendAssumeCapacity( + .denseElements( + .rankedTensor(&.{@intCast(rl.len)}, .index(ctx)), + rl, + ), + ); + } + } + attrs.appendAssumeCapacity( + .named(ctx, "result_layouts", .array(ctx, layouts_buffer.constSlice())), + ); + + return mlir.Operation.make(ctx, "stablehlo.custom_call", .{ + .operands = .{ .flat = inputs }, + .results = .{ .flat = result_types }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +pub fn optimizationBarrier(ctx: *mlir.Context, inputs: []const *const mlir.Value, result_types: []const *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.optimization_barrier", .{ + .operands = .{ .flat = inputs }, + .results = .{ .flat = result_types }, + .location = location, + }); +} + +pub fn createBuffer(ctx: *mlir.Context, value: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return custom_call(ctx, &.{}, &.{value}, .{ + .call_target_name = "CreateBuffer", + .api_version = .typed_ffi, + }, location); +} + +pub fn pin(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return custom_call( + ctx, + &.{value}, + &.{@ptrCast(mlir.MemRefType.fromShaped(value.type_().isA(mlir.ShapedType).?))}, + .{ + .call_target_name = "Pin", + .api_version = .typed_ffi, + }, + location, + ); +} + +pub fn unpin(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return custom_call( + ctx, + &.{value}, + &.{@ptrCast(mlir.RankedTensorType.fromShaped(value.type_().isA(mlir.ShapedType).?))}, + .{ + .call_target_name = "Unpin", + .api_version = .typed_ffi, + }, + location, + ); +} + +pub fn partition_id(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.partition_id", .{ + .results = .{ .flat = &.{.rankedTensor(&.{}, .int(ctx, .u32))} }, + .location = location, + }); +} + +pub const DotDimensionNumbersAttr = opaque { + const M = mlir.Methods(DotDimensionNumbersAttr, c.MlirAttribute); + + pub const isAFn = c.stablehloAttributeIsADotDimensionNumbers; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub const InitArgs = struct { + lhs_batching_dimensions: []const i64, + rhs_batching_dimensions: []const i64, + lhs_contracting_dimensions: []const i64, + rhs_contracting_dimensions: []const i64, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) *const DotDimensionNumbersAttr { + return @ptrCast(c.stablehloDotDimensionNumbersGet( + ctx.ptr(), + @intCast(args.lhs_batching_dimensions.len), + args.lhs_batching_dimensions.ptr, + @intCast(args.rhs_batching_dimensions.len), + args.rhs_batching_dimensions.ptr, + @intCast(args.lhs_contracting_dimensions.len), + args.lhs_contracting_dimensions.ptr, + @intCast(args.rhs_contracting_dimensions.len), + args.rhs_contracting_dimensions.ptr, + ).ptr); + } + + pub fn lhsBatchingDimensionsSize(self: *const DotDimensionNumbersAttr) usize { + return @intCast(c.stablehloDotDimensionNumbersGetLhsBatchingDimensionsSize(self.ptr())); + } + + pub fn lhsBatchingDimensionsElem(self: *const DotDimensionNumbersAttr, pos: usize) i64 { + return c.stablehloDotDimensionNumbersGetLhsBatchingDimensionsElem(self.ptr(), @intCast(pos)); + } + + pub fn rhsBatchingDimensionsSize(self: *const DotDimensionNumbersAttr) usize { + return @intCast(c.stablehloDotDimensionNumbersGetRhsBatchingDimensionsSize(self.ptr())); + } + + pub fn rhsBatchingDimensionsElem(self: *const DotDimensionNumbersAttr, pos: usize) i64 { + return c.stablehloDotDimensionNumbersGetRhsBatchingDimensionsElem(self.ptr(), @intCast(pos)); + } + + pub fn lhsContractingDimensionsSize(self: *const DotDimensionNumbersAttr) usize { + return @intCast(c.stablehloDotDimensionNumbersGetLhsContractingDimensionsSize(self.ptr())); + } + + pub fn lhsContractingDimensionsElem(self: *const DotDimensionNumbersAttr, pos: usize) i64 { + return c.stablehloDotDimensionNumbersGetLhsContractingDimensionsElem(self.ptr(), @intCast(pos)); + } + + pub fn rhsContractingDimensionsSize(self: *const DotDimensionNumbersAttr) usize { + return @intCast(c.stablehloDotDimensionNumbersGetRhsContractingDimensionsSize(self.ptr())); + } + + pub fn rhsContractingDimensionsElem(self: *const DotDimensionNumbersAttr, pos: usize) i64 { + return c.stablehloDotDimensionNumbersGetRhsContractingDimensionsElem(self.ptr(), @intCast(pos)); + } + + pub fn attribute(self: *const DotDimensionNumbersAttr) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +pub fn dotDimensionNumbersAttr(ctx: *mlir.Context, args: DotDimensionNumbersAttr.InitArgs) *const mlir.Attribute { + return @ptrCast(DotDimensionNumbersAttr.get(ctx, args)); +} + +pub const GatherDimensionNumbersAttr = opaque { + const M = mlir.Methods(GatherDimensionNumbersAttr, c.MlirAttribute); + + pub const isAFn = c.stablehloAttributeIsAGatherDimensionNumbers; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub fn get( + ctx: *mlir.Context, + offset_dims: []const i64, + collapsed_slice_dims: []const i64, + operand_batching_dims: []const i64, + start_indices_batching_dims: []const i64, + start_index_map: []const i64, + index_vector_dim: i64, + ) *const GatherDimensionNumbersAttr { + return @ptrCast(c.stablehloGatherDimensionNumbersGet( + ctx.ptr(), + @intCast(offset_dims.len), + offset_dims.ptr, + @intCast(collapsed_slice_dims.len), + collapsed_slice_dims.ptr, + @intCast(operand_batching_dims.len), + operand_batching_dims.ptr, + @intCast(start_indices_batching_dims.len), + start_indices_batching_dims.ptr, + @intCast(start_index_map.len), + start_index_map.ptr, + index_vector_dim, + ).ptr); + } + + pub fn offsetDimsSize(self: *const GatherDimensionNumbersAttr) usize { + return @intCast(c.stablehloGatherDimensionNumbersGetOffsetDimsSize(self.ptr())); + } + + pub fn offsetDimsElem(self: *const GatherDimensionNumbersAttr, pos: usize) i64 { + return c.stablehloGatherDimensionNumbersGetOffsetDimsElem(self.ptr(), @intCast(pos)); + } + + pub fn collapsedSliceDimsSize(self: *const GatherDimensionNumbersAttr) usize { + return @intCast(c.stablehloGatherDimensionNumbersGetCollapsedSliceDimsSize(self.ptr())); + } + + pub fn collapsedSliceDimsElem(self: *const GatherDimensionNumbersAttr, pos: usize) i64 { + return c.stablehloGatherDimensionNumbersGetCollapsedSliceDimsElem(self.ptr(), @intCast(pos)); + } + + pub fn startIndexMapSize(self: *const GatherDimensionNumbersAttr) usize { + return @intCast(c.stablehloGatherDimensionNumbersGetStartIndexMapSize(self.ptr())); + } + + pub fn operandBatchingDimsSize(self: *const GatherDimensionNumbersAttr) usize { + return @intCast(c.stablehloGatherDimensionNumbersGetOperandBatchingDimsSize(self.ptr())); + } + + pub fn operandBatchingDimsElem(self: *const GatherDimensionNumbersAttr, pos: usize) i64 { + return c.stablehloGatherDimensionNumbersGetOperandBatchingDimsElem(self.ptr(), @intCast(pos)); + } + + pub fn startIndicesBatchingDimsSize(self: *const GatherDimensionNumbersAttr) usize { + return @intCast(c.stablehloGatherDimensionNumbersGetStartIndicesBatchingDimsSize(self.ptr())); + } + + pub fn startIndicesBatchingDimsElem(self: *const GatherDimensionNumbersAttr, pos: usize) i64 { + return c.stablehloGatherDimensionNumbersGetStartIndicesBatchingDimsElem(self.ptr(), @intCast(pos)); + } + + pub fn startIndexMapElem(self: *const GatherDimensionNumbersAttr, pos: usize) i64 { + return c.stablehloGatherDimensionNumbersGetStartIndexMapElem(self.ptr(), @intCast(pos)); + } + + pub fn indexVectorDim(self: *const GatherDimensionNumbersAttr) usize { + return @intCast(c.stablehloGatherDimensionNumbersGetIndexVectorDim(self.ptr())); + } +}; + +pub fn gatherDimensionNumbersAttr( + ctx: *mlir.Context, + offset_dims: []const i64, + collapsed_slice_dims: []const i64, + operand_batching_dims: []const i64, + start_indices_batching_dims: []const i64, + start_index_map: []const i64, + index_vector_dim: i64, +) *const mlir.Attribute { + return @ptrCast(GatherDimensionNumbersAttr.get( + ctx, + offset_dims, + collapsed_slice_dims, + operand_batching_dims, + start_indices_batching_dims, + start_index_map, + index_vector_dim, + )); +} + +pub const ConvDimensionNumbersAttr = opaque { + const M = mlir.Methods(ConvDimensionNumbersAttr, c.MlirAttribute); + + pub const isAFn = c.stablehloAttributeIsAConvDimensionNumbers; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub const InitArgs = struct { + input_batch_dimension: i64, + input_feature_dimension: i64, + input_spatial_dimensions: []const i64, + kernel_input_feature_dimension: i64, + kernel_output_feature_dimension: i64, + kernel_spatial_dimensions: []const i64, + output_batch_dimension: i64, + output_feature_dimension: i64, + output_spatial_dimensions: []const i64, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) *const ConvDimensionNumbersAttr { + return @ptrCast(c.stablehloConvDimensionNumbersGet( + ctx.ptr(), + args.input_batch_dimension, + args.input_feature_dimension, + @intCast(args.input_spatial_dimensions.len), + args.input_spatial_dimensions.ptr, + args.kernel_input_feature_dimension, + args.kernel_output_feature_dimension, + @intCast(args.kernel_spatial_dimensions.len), + args.kernel_spatial_dimensions.ptr, + args.output_batch_dimension, + args.output_feature_dimension, + @intCast(args.output_spatial_dimensions.len), + args.output_spatial_dimensions.ptr, + ).ptr); + } + + pub fn inputBatchDimension(self: *const ConvDimensionNumbersAttr) i64 { + return c.stablehloConvDimensionNumbersGetInputBatchDimension(self.ptr()); + } + + pub fn inputFeatureDimension(self: *const ConvDimensionNumbersAttr) i64 { + return c.stablehloConvDimensionNumbersGetInputFeatureDimension(self.ptr()); + } + + pub fn inputSpatialDimensionsSize(self: *const ConvDimensionNumbersAttr) usize { + return @intCast(c.stablehloConvDimensionNumbersGetInputSpatialDimensionsSize(self.ptr())); + } + + pub fn inputSpatialDimensionsElem(self: *const ConvDimensionNumbersAttr, pos: usize) i64 { + return c.stablehloConvDimensionNumbersGetInputSpatialDimensionsElem(self.ptr(), @intCast(pos)); + } + + pub fn kernelInputFeatureDimension(self: *const ConvDimensionNumbersAttr) i64 { + return c.stablehloConvDimensionNumbersGetKernelInputFeatureDimension(self.ptr()); + } + + pub fn kernelOutputFeatureDimension(self: *const ConvDimensionNumbersAttr) i64 { + return c.stablehloConvDimensionNumbersGetKernelOutputFeatureDimension(self.ptr()); + } + + pub fn kernelSpatialDimensionsSize(self: *const ConvDimensionNumbersAttr) usize { + return @intCast(c.stablehloConvDimensionNumbersGetKernelSpatialDimensionsSize(self.ptr())); + } + + pub fn kernelSpatialDimensionsElem(self: *const ConvDimensionNumbersAttr, pos: usize) i64 { + return c.stablehloConvDimensionNumbersGetKernelSpatialDimensionsElem(self.ptr(), @intCast(pos)); + } + + pub fn outputBatchDimension(self: *const ConvDimensionNumbersAttr) i64 { + return c.stablehloConvDimensionNumbersGetOutputBatchDimension(self.ptr()); + } + + pub fn outputFeatureDimension(self: *const ConvDimensionNumbersAttr) i64 { + return c.stablehloConvDimensionNumbersGetOutputFeatureDimension(self.ptr()); + } + + pub fn outputSpatialDimensionsSize(self: *const ConvDimensionNumbersAttr) usize { + return @intCast(c.stablehloConvDimensionNumbersGetOutputSpatialDimensionsSize(self.ptr())); + } + + pub fn outputSpatialDimensionsElem(self: *const ConvDimensionNumbersAttr, pos: usize) i64 { + return c.stablehloConvDimensionNumbersGetOutputSpatialDimensionsElem(self.ptr(), @intCast(pos)); + } + + pub fn attribute(self: *const ConvDimensionNumbersAttr) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +pub fn convDimensionNumbersAttr(ctx: *mlir.Context, args: ConvDimensionNumbersAttr.InitArgs) *const mlir.Attribute { + return @ptrCast(ConvDimensionNumbersAttr.get(ctx, args)); +} + +pub const OutputOperandAliasAttr = opaque { + const M = mlir.Methods(OutputOperandAliasAttr, c.MlirAttribute); + + pub const isAFn = c.stablehloAttributeIsAOutputOperandAlias; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub const InitArgs = struct { + output_tuple_indices: []const i64 = &.{}, + operand_index: i64, + operand_tuple_indices: []const i64 = &.{}, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) *const OutputOperandAliasAttr { + return @ptrCast(c.stablehloOutputOperandAliasGet( + ctx.ptr(), + @intCast(args.output_tuple_indices.len), + @ptrCast(args.output_tuple_indices), + @intCast(args.operand_index), + @intCast(args.operand_tuple_indices.len), + @ptrCast(args.operand_tuple_indices), + ).ptr); + } + + pub fn attribute(self: *const OutputOperandAliasAttr) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +pub fn outputOperandAliasAttr(ctx: *mlir.Context, args: OutputOperandAliasAttr.InitArgs) *const mlir.Attribute { + return @ptrCast(OutputOperandAliasAttr.get(ctx, args)); +} + +pub const PrecisionAttr = opaque { + const M = mlir.Methods(PrecisionAttr, c.MlirAttribute); + + pub const isAFn = c.stablehloAttributeIsAPrecisionAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub const Precision = enum { + DEFAULT, + HIGH, + HIGHEST, + }; + + pub const InitArgs = struct { + value: Precision, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) *const PrecisionAttr { + return @ptrCast(c.stablehloPrecisionAttrGet(ctx.ptr(), stringRef(@tagName(args.value))).ptr); + } + + pub fn value(self: *const PrecisionAttr) Precision { + const name = mlir.string(c.stablehloPrecisionAttrGetValue(self.ptr())); + return std.meta.stringToEnum(Precision, name) orelse unreachable; + } + + pub fn attribute(self: *const PrecisionAttr) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +pub fn precisionAttr(ctx: *mlir.Context, precision: PrecisionAttr.Precision) *const mlir.Attribute { + return PrecisionAttr.get(ctx, .{ .value = precision }).attribute(); +} + +pub const ComparisonDirectionAttr = opaque { + const M = mlir.Methods(ComparisonDirectionAttr, c.MlirAttribute); + + pub const isAFn = c.stablehloAttributeIsAComparisonDirectionAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub const Direction = enum { + EQ, + NE, + GE, + GT, + LE, + LT, + }; + + pub const InitArgs = struct { + value: Direction, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) *const ComparisonDirectionAttr { + return @ptrCast(c.stablehloComparisonDirectionAttrGet(ctx.ptr(), mlir.stringRef(@tagName(args.value))).ptr); + } + + pub fn value(self: *const ComparisonDirectionAttr) Direction { + const name = mlir.string(c.stablehloComparisonDirectionAttrGetValue(self.ptr())); + return std.meta.stringToEnum(Direction, name) orelse unreachable; + } + + pub fn attribute(self: *const ComparisonDirectionAttr) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +pub fn comparisonDirectionAttr(ctx: *mlir.Context, value: ComparisonDirectionAttr.Direction) *const mlir.Attribute { + return ComparisonDirectionAttr.get(ctx, .{ .value = value }).attribute(); +} + +pub const CompareTypeAttr = opaque { + const M = mlir.Methods(CompareTypeAttr, c.MlirAttribute); + + pub const isAFn = c.stablehloAttributeIsAComparisonTypeAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub const Type = enum { + SIGNED, + UNSIGNED, + FLOAT, + TOTALORDER, + }; + + pub const InitArgs = struct { + value: Type, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) *const CompareTypeAttr { + return @ptrCast(c.stablehloComparisonTypeAttrGet(ctx.ptr(), stringRef(@tagName(args.value))).ptr); + } + + pub fn value(self: *const CompareTypeAttr) Type { + const name = mlir.string(c.stablehloComparisonTypeAttrGetValue(self.ptr())); + return std.meta.stringToEnum(Type, name) orelse unreachable; + } + + pub fn attribute(self: *const CompareTypeAttr) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +pub fn compareTypeAttr(ctx: *mlir.Context, value: CompareTypeAttr.Type) *const mlir.Attribute { + return CompareTypeAttr.get(ctx, .{ .value = value }).attribute(); +} + +pub const TransposeAttr = opaque { + const M = mlir.Methods(TransposeAttr, c.MlirAttribute); + + pub const isAFn = c.stablehloAttributeIsATransposeAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub const Type = enum { + NO_TRANSPOSE, + TRANSPOSE, + ADJOINT, + }; + + pub const InitArgs = struct { + value: Type, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) *const TransposeAttr { + return @ptrCast(c.stablehloTransposeAttrGet(ctx.ptr(), mlir.stringRef(@tagName(args.value))).ptr); + } + + pub fn value(self: *const TransposeAttr) Type { + const name = mlir.string(c.stablehloTransposeAttrGetValue(self.ptr())); + return std.meta.stringToEnum(Type, name) orelse unreachable; + } + + pub fn attribute(self: *const TransposeAttr) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +pub fn transposeAttr(ctx: *mlir.Context, value: TransposeAttr.Type) *const mlir.Attribute { + return TransposeAttr.get(ctx, .{ .value = value }).attribute(); +} + +pub const FftTypeAttr = opaque { + const M = mlir.Methods(FftTypeAttr, c.MlirAttribute); + + pub const isAFn = c.stablehloAttributeIsAFftTypeAttr; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub const Type = enum { + FFT, + IFFT, + RFFT, + IRFFT, + }; + + pub const InitArgs = struct { + value: Type, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) *const FftTypeAttr { + return @ptrCast(c.stablehloFftTypeAttrGet(ctx.ptr(), mlir.stringRef(@tagName(args.value))).ptr); + } + + pub fn value(self: *const FftTypeAttr) Type { + const name = mlir.string(c.stablehloFftTypeAttrGetValue(self.ptr())); + return std.meta.stringToEnum(Type, name) orelse unreachable; + } + + pub fn attribute(self: *const FftTypeAttr) *const mlir.Attribute { + return @ptrCast(self); + } +}; + +pub fn fftTypeAttr(ctx: *mlir.Context, value: FftTypeAttr.Type) *const mlir.Attribute { + return FftTypeAttr.get(ctx, .{ .value = value }).attribute(); +} + +fn stringFromStream(buf: []u8, streamFn: anytype, args: anytype) []const u8 { + var writer = std.Io.Writer.fixed(buf); + var sctx: mlir.StringCallbackCtx = .{ .writer = &writer }; + _ = @call(.auto, streamFn, args ++ .{ mlir.stringCallback, &sctx }); + return writer.buffered(); +} + +pub fn versionFromCompatibilityRequirement(buffer: []u8, requirement: c.MlirStablehloCompatibilityRequirement) []const u8 { + return stringFromStream(buffer, c.stablehloVersionFromCompatibilityRequirement, .{requirement}); +} + +pub fn smallerVersion(version1: []const u8, version2: []const u8) []const u8 { + var buf: [32]u8 = undefined; + const result = stringFromStream(&buf, c.stablehloGetSmallerVersion, .{ stringRef(version1), stringRef(version2) }); + return if (std.mem.eql(u8, result, version1)) version1 else version2; +} + +pub fn currentVersion() []const u8 { + const state = struct { + var buf: [32]u8 = undefined; + }; + return stringFromStream(&state.buf, c.stablehloGetCurrentVersion, .{}); +} + +pub fn minimumVersion() []const u8 { + const state = struct { + var buf: [32]u8 = undefined; + }; + return stringFromStream(&state.buf, c.stablehloGetMinimumVersion, .{}); +} + +pub fn serializePortableArtifact(bytecode: []const u8, target_version: []const u8, writer: *std.Io.Writer) !void { + var sctx: mlir.StringCallbackCtx = .{ .writer = writer }; + const result = c.stablehloSerializePortableArtifactFromStringRef( + stringRef(bytecode), + stringRef(target_version), + mlir.stringCallback, + &sctx, + ); + if (sctx.err) |err| { + return err; + } + if (result.value == 0) { + return error.InvalidMlirBytecodeVersion; + } +} + +pub fn serializePortableArtifact2(module: *mlir.Module, target_version: []const u8, writer: *std.Io.Writer) !void { + var sctx: mlir.StringCallbackCtx = .{ .writer = writer }; + const result = c.stablehloSerializePortableArtifactFromModule( + module.ptr(), + stringRef(target_version), + mlir.stringCallback, + &sctx, + true, + ); + if (sctx.err) |err| { + return err; + } + // TODO(Corentin): Find out why this crashes on macOS + //if (c.mlirLogicalResultIsFailure(result)) { + if (@as(c_int, result.value) == 0) { + return error.InvalidMlirBytecodeVersion; + } +} + +pub fn return_(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.return", .{ + .operands = .{ .flat = &.{value} }, + .location = location, + .verify = false, + }); +} + +pub fn returns(ctx: *mlir.Context, values: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.return", .{ + .operands = .{ .variadic = &.{values} }, + .verify = false, + .location = location, + }); +} + +pub const ChannelType = enum(i64) { + collective = 0, + send_recv = 1, +}; + +pub fn channelHandleAttr( + ctx: *mlir.Context, + handle: i64, + channel_type: ChannelType, +) *const mlir.Attribute { + return @ptrCast(c.stablehloChannelHandleGet( + ctx.ptr(), + handle, + @backingInt(channel_type), + ).ptr); +} + +pub fn allReduce( + ctx: *mlir.Context, + inputs: []const *const mlir.Value, + result_types: []const *const mlir.Type, + reducer_block: *mlir.Block, + replica_groups: *const mlir.Attribute, + channel_handle: *const mlir.Attribute, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "stablehlo.all_reduce", .{ + .operands = .{ .variadic = &.{inputs} }, + .results = .{ .flat = result_types }, + .blocks = &.{reducer_block}, + .attributes = &.{ + .named(ctx, "replica_groups", replica_groups), + .named(ctx, "channel_handle", channel_handle), + .named(ctx, "use_global_device_ids", .unit(ctx)), + }, + .verify = true, + .location = location, + }); +} diff --git a/mlir/dialects/stablehlo/test.zig b/mlir/dialects/stablehlo/test.zig new file mode 100644 index 00000000..c22b2ebd --- /dev/null +++ b/mlir/dialects/stablehlo/test.zig @@ -0,0 +1,106 @@ +const std = @import("std"); + +const mlir = @import("mlir"); +const stablehlo = @import("mlir/dialects/stablehlo"); + +const operation_names: []const []const u8 = &.{ + "stablehlo.abs", + "stablehlo.add", + "stablehlo.all_reduce", + "stablehlo.and", + "stablehlo.atan2", + "stablehlo.bitcast_convert", + "stablehlo.broadcast_in_dim", + "stablehlo.cbrt", + "stablehlo.ceil", + "stablehlo.cholesky", + "stablehlo.clamp", + "stablehlo.compare", + "stablehlo.complex", + "stablehlo.composite", + "stablehlo.concatenate", + "stablehlo.constant", + "stablehlo.convert", + "stablehlo.convolution", + "stablehlo.cosine", + "stablehlo.count_leading_zeros", + "stablehlo.custom_call", + "stablehlo.divide", + "stablehlo.dot_general", + "stablehlo.dynamic_slice", + "stablehlo.dynamic_update_slice", + "stablehlo.exponential", + "stablehlo.exponential_minus_one", + "stablehlo.fft", + "stablehlo.floor", + "stablehlo.gather", + "stablehlo.get_tuple_element", + "stablehlo.if", + "stablehlo.imag", + "stablehlo.iota", + "stablehlo.is_finite", + "stablehlo.log", + "stablehlo.log_plus_one", + "stablehlo.logistic", + "stablehlo.maximum", + "stablehlo.minimum", + "stablehlo.multiply", + "stablehlo.negate", + "stablehlo.not", + "stablehlo.optimization_barrier", + "stablehlo.or", + "stablehlo.pad", + "stablehlo.partition_id", + "stablehlo.popcnt", + "stablehlo.power", + "stablehlo.real", + "stablehlo.reduce", + "stablehlo.reduce_precision", + "stablehlo.reduce_window", + "stablehlo.remainder", + "stablehlo.reshape", + "stablehlo.return", + "stablehlo.reverse", + "stablehlo.rng", + "stablehlo.rng_bit_generator", + "stablehlo.round_nearest_afz", + "stablehlo.round_nearest_even", + "stablehlo.rsqrt", + "stablehlo.scatter", + "stablehlo.select", + "stablehlo.shift_left", + "stablehlo.shift_right_arithmetic", + "stablehlo.shift_right_logical", + "stablehlo.sign", + "stablehlo.sine", + "stablehlo.slice", + "stablehlo.sort", + "stablehlo.sqrt", + "stablehlo.subtract", + "stablehlo.tanh", + "stablehlo.transpose", + "stablehlo.triangular_solve", + "stablehlo.tuple", + "stablehlo.while", + "stablehlo.xor", +}; + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(stablehlo); + inline for (comptime std.meta.declarations(stablehlo)) |decl| { + const T = @field(stablehlo, decl); + if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); + } +} + +test "every bound operation is registered" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + stablehlo.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} diff --git a/mlir/dialects/tcl/BUILD.bazel b/mlir/dialects/tcl/BUILD.bazel new file mode 100644 index 00000000..d57b6f36 --- /dev/null +++ b/mlir/dialects/tcl/BUILD.bazel @@ -0,0 +1,172 @@ +load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library", "td_library") +load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +package(default_visibility = ["//visibility:public"]) + +cc_library( + name = "capi", + srcs = ["capi.cc"], + hdrs = ["capi.h"], + tags = ["manual"], + deps = [ + ":dialect", + ":headers", + "@llvm-project//mlir:CAPIIR", + ], +) + +zig_library( + name = "tcl", + import_name = "mlir/dialects/tcl", + main = "tcl.zig", + tags = ["manual"], + deps = [ + ":capi", + "//mlir", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":tcl", + "//mlir", + "//mlir/dialects/func", + ], +) + +# The native subset of the Furiosa XLA plugin's `tcl` dialect +# (xla/backends/furiosa/codegen/tcl/ir): what ZML emits and the plugin's custom +# call parses. Files mirror the plugin's one for one. +td_library( + name = "td_files", + srcs = [ + "tcl_attrs.td", + "tcl_dialect.td", + "tcl_ops.td", + "tcl_types.td", + ], + includes = ["/"], + deps = [ + "@llvm-project//mlir:OpBaseTdFiles", + "@llvm-project//mlir:SideEffectInterfacesTdFiles", + ], +) + +gentbl_cc_library( + name = "dialect_inc_gen", + tbl_outs = [ + ( + ["-gen-dialect-decls"], + "tcl_dialect.h.inc", + ), + ( + ["-gen-dialect-defs"], + "tcl_dialect.cc.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "tcl_dialect.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "types_inc_gen", + tbl_outs = [ + ( + [ + "-gen-typedef-decls", + "-typedefs-dialect=tcl", + ], + "tcl_types.h.inc", + ), + ( + [ + "-gen-typedef-defs", + "-typedefs-dialect=tcl", + ], + "tcl_types.cc.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "tcl_types.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "attrs_inc_gen", + tbl_outs = [ + ( + [ + "-gen-attrdef-decls", + "-attrdefs-dialect=tcl", + ], + "tcl_attrs.h.inc", + ), + ( + [ + "-gen-attrdef-defs", + "-attrdefs-dialect=tcl", + ], + "tcl_attrs.cc.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "tcl_attrs.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "ops_inc_gen", + tbl_outs = [ + ( + ["-gen-op-decls"], + "tcl_ops.h.inc", + ), + ( + ["-gen-op-defs"], + "tcl_ops.cc.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "tcl_ops.td", + deps = [":td_files"], +) + +cc_library( + name = "headers", + hdrs = [ + "tcl_attrs.h", + "tcl_dialect.h", + "tcl_ops.h", + "tcl_types.h", + ], + deps = [ + ":attrs_inc_gen", + ":dialect_inc_gen", + ":ops_inc_gen", + ":types_inc_gen", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:BytecodeOpInterface", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:SideEffectInterfaces", + ], +) + +cc_library( + name = "dialect", + srcs = [ + "tcl_attrs.cc", + "tcl_dialect.cc", + "tcl_ops.cc", + "tcl_types.cc", + ], + textual_hdrs = ["ve_instructions.inc"], + deps = [ + ":headers", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:IR", + ], +) diff --git a/mlir/dialects/tcl/capi.cc b/mlir/dialects/tcl/capi.cc new file mode 100644 index 00000000..70ee09d6 --- /dev/null +++ b/mlir/dialects/tcl/capi.cc @@ -0,0 +1,118 @@ +//===- capi.cc - C API for the Furiosa TCL dialect ------------------------===// + +#include "capi.h" + +#include "mlir/dialects/tcl/tcl_ops.h" + +#include "mlir/CAPI/IR.h" +#include "mlir/CAPI/Registration.h" +#include "mlir/CAPI/Support.h" + +#include + +using namespace mlir; +using namespace xla::furiosa::tcl; + +MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(Tcl, tcl, TclDialect) + +// `getChecked` with its diagnostic silenced: an invalid parameter is a null +// handle, which the caller reports. +template +static auto checked(MlirContext context, Args &&...args) { + MLIRContext *ctx = unwrap(context); + if (!ctx) + return T(); + ctx->getOrLoadDialect(); + ScopedDiagnosticHandler silence(ctx, [](Diagnostic &) { return success(); }); + return T::getChecked([ctx] { return emitError(UnknownLoc::get(ctx)); }, ctx, + std::forward(args)...); +} + +template static T as(MlirAttribute attr) { + return dyn_cast_if_present(unwrap(attr)); +} + +MlirAttribute mlirTclSymbolAttrGet(MlirContext context, MlirStringRef name) { + return wrap(checked(context, unwrap(name))); +} + +MlirAttribute mlirTclTacticAttrGet(MlirContext context, MlirStringRef value) { + return wrap(checked(context, unwrap(value))); +} + +MlirAttribute mlirTclVeOpcodeAttrGet(MlirContext context, MlirStringRef value) { + return wrap(checked(context, unwrap(value))); +} + +MlirAttribute mlirTclReduceModeAttrGet(MlirContext context, + MlirStringRef value) { + return wrap(checked(context, unwrap(value))); +} + +MlirAttribute mlirTclPredicateAttrGet(MlirContext context, + MlirStringRef value) { + return wrap(checked(context, unwrap(value))); +} + +MlirAttribute mlirTclContextAttrGet(MlirContext context, MlirAttribute fields) { + auto dict = as(fields); + return dict ? wrap(checked(context, dict)) + : MlirAttribute{nullptr}; +} + +MlirAttribute mlirTclReadOptionsAttrGet(MlirContext context, + MlirAttribute fields) { + auto dict = as(fields); + return dict ? wrap(checked(context, dict)) + : MlirAttribute{nullptr}; +} + +MlirAttribute mlirTclConfigAttrGet(MlirContext context, MlirAttribute fields) { + auto dict = as(fields); + return dict ? wrap(checked(context, dict)) + : MlirAttribute{nullptr}; +} + +MlirAttribute mlirTclExprAttrGet(MlirContext context, MlirStringRef kind, + MlirAttribute args) { + auto array = as(args); + return array ? wrap(checked(context, unwrap(kind), array)) + : MlirAttribute{nullptr}; +} + +MlirAttribute mlirTclDramMappingAttrGet(MlirContext context, MlirAttribute chip, + MlirAttribute inner, + MlirAttribute original) { + auto c = as(chip), i = as(inner), + o = as(original); + return c && i && o ? wrap(checked(context, c, i, o)) + : MlirAttribute{nullptr}; +} + +MlirType mlirTclLogicalTypeGet(MlirContext context, MlirType element, + MlirAttribute axes) { + auto array = as(axes); + if (!array || mlirTypeIsNull(element)) + return {nullptr}; + return wrap(checked(context, unwrap(element), array)); +} + +bool mlirTypeIsATclLogical(MlirType type) { + return isa_and_nonnull(unwrap(type)); +} + +MlirType mlirTclLogicalTypeGetElementType(MlirType type) { + return wrap(cast(unwrap(type)).getElementType()); +} + +MlirAttribute mlirTclLogicalTypeGetAxes(MlirType type) { + return wrap(cast(unwrap(type)).getAxes()); +} + +MlirType mlirTclMappedTypeGet(MlirContext context, MlirType element, + MlirAttribute mapping) { + auto m = as(mapping); + if (!m || mlirTypeIsNull(element)) + return {nullptr}; + return wrap(checked(context, unwrap(element), m)); +} diff --git a/mlir/dialects/tcl/capi.h b/mlir/dialects/tcl/capi.h new file mode 100644 index 00000000..31e72c3b --- /dev/null +++ b/mlir/dialects/tcl/capi.h @@ -0,0 +1,53 @@ +//===- capi.h - C API for the Furiosa TCL dialect -----------------*- C -*-===// + +#ifndef MLIR_DIALECTS_TCL_CAPI_H +#define MLIR_DIALECTS_TCL_CAPI_H + +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(Tcl, tcl); + +// Constructors verify their parameters and return a null handle when invalid. +MLIR_CAPI_EXPORTED MlirAttribute mlirTclSymbolAttrGet(MlirContext context, + MlirStringRef name); +MLIR_CAPI_EXPORTED MlirAttribute mlirTclExprAttrGet(MlirContext context, + MlirStringRef kind, + MlirAttribute args); +MLIR_CAPI_EXPORTED MlirAttribute mlirTclTacticAttrGet(MlirContext context, + MlirStringRef value); +MLIR_CAPI_EXPORTED MlirAttribute mlirTclVeOpcodeAttrGet(MlirContext context, + MlirStringRef value); +MLIR_CAPI_EXPORTED MlirAttribute mlirTclReduceModeAttrGet(MlirContext context, + MlirStringRef value); +MLIR_CAPI_EXPORTED MlirAttribute mlirTclPredicateAttrGet(MlirContext context, + MlirStringRef value); +MLIR_CAPI_EXPORTED MlirAttribute mlirTclContextAttrGet(MlirContext context, + MlirAttribute fields); +MLIR_CAPI_EXPORTED MlirAttribute +mlirTclReadOptionsAttrGet(MlirContext context, MlirAttribute fields); +MLIR_CAPI_EXPORTED MlirAttribute mlirTclConfigAttrGet(MlirContext context, + MlirAttribute fields); +MLIR_CAPI_EXPORTED MlirAttribute +mlirTclDramMappingAttrGet(MlirContext context, MlirAttribute chip, + MlirAttribute inner, MlirAttribute original); + +MLIR_CAPI_EXPORTED MlirType mlirTclLogicalTypeGet(MlirContext context, + MlirType element, + MlirAttribute axes); +MLIR_CAPI_EXPORTED bool mlirTypeIsATclLogical(MlirType type); +MLIR_CAPI_EXPORTED MlirType mlirTclLogicalTypeGetElementType(MlirType type); +MLIR_CAPI_EXPORTED MlirAttribute mlirTclLogicalTypeGetAxes(MlirType type); + +MLIR_CAPI_EXPORTED MlirType mlirTclMappedTypeGet(MlirContext context, + MlirType element, + MlirAttribute mapping); + +#ifdef __cplusplus +} +#endif + +#endif // MLIR_DIALECTS_TCL_CAPI_H diff --git a/mlir/dialects/tcl/tcl.zig b/mlir/dialects/tcl/tcl.zig new file mode 100644 index 00000000..a3eb4673 --- /dev/null +++ b/mlir/dialects/tcl/tcl.zig @@ -0,0 +1,364 @@ +const c = @import("c"); +const mlir = @import("mlir"); + +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("tcl"); +} + +fn stringRef(s: []const u8) c.MlirStringRef { + return .{ .data = s.ptr, .length = s.len }; +} + +fn attribute(a: c.MlirAttribute) mlir.Error!*const mlir.Attribute { + return if (a.ptr) |p| @ptrCast(p) else error.InvalidMlir; +} + +fn type_(t: c.MlirType) mlir.Error!*const mlir.Type { + return if (t.ptr) |p| @ptrCast(p) else error.InvalidMlir; +} + +// ============================================================================= +// Attributes +// ============================================================================= + +pub const Tactic = enum { + einsum_by_dpe, + reduce_by_ve, + interleaving, + elementwise, + tensor_operation, + einsum_by_ve, + filter_compaction, + + pub fn name(self: Tactic) []const u8 { + return switch (self) { + .einsum_by_dpe => "EinsumByDpe", + .reduce_by_ve => "ReduceByVe", + .interleaving => "Interleaving", + .elementwise => "Elementwise", + .tensor_operation => "TensorOperation", + .einsum_by_ve => "EinsumByVe", + .filter_compaction => "FilterCompaction", + }; + } +}; + +pub const ReduceMode = enum { + addi, + addf, + maxi, + maxf, + mini, + minf, + /// A running i32 sum that keeps its axes. + cumsum, + + pub fn name(self: ReduceMode) []const u8 { + return switch (self) { + .addi => "Addi", + .addf => "Addf", + .maxi => "Maxi", + .maxf => "Maxf", + .mini => "Mini", + .minf => "Minf", + .cumsum => "Cumsum", + }; + } +}; + +pub const Predicate = enum { eq, ne, lt, le, gt, ge }; + +/// An axis or another TCL identifier: `#tcl.symbol<"A">`. +pub fn symbolAttr(ctx: *mlir.Context, name: []const u8) mlir.Error!*const mlir.Attribute { + return attribute(c.mlirTclSymbolAttrGet(ctx.ptr(), stringRef(name))); +} + +/// `#tcl.expr`: `add`, `mul`, `stride`, `broadcast`, `arg`, ... +pub fn exprAttr(ctx: *mlir.Context, kind: []const u8, args: []const *const mlir.Attribute) mlir.Error!*const mlir.Attribute { + return attribute(c.mlirTclExprAttrGet(ctx.ptr(), stringRef(kind), mlir.Attribute.array(ctx, args).ptr())); +} + +pub fn tacticAttr(ctx: *mlir.Context, value: Tactic) *const mlir.Attribute { + return attribute(c.mlirTclTacticAttrGet(ctx.ptr(), stringRef(value.name()))) catch unreachable; +} + +/// A VE opcode of the SDK vocabulary (`exp`, `addf`, `to_f16`, ...). +pub fn veOpcodeAttr(ctx: *mlir.Context, opcode: []const u8) mlir.Error!*const mlir.Attribute { + return attribute(c.mlirTclVeOpcodeAttrGet(ctx.ptr(), stringRef(opcode))); +} + +pub fn reduceModeAttr(ctx: *mlir.Context, mode: ReduceMode) *const mlir.Attribute { + return attribute(c.mlirTclReduceModeAttrGet(ctx.ptr(), stringRef(mode.name()))) catch unreachable; +} + +pub fn predicateAttr(ctx: *mlir.Context, value: Predicate) *const mlir.Attribute { + return attribute(c.mlirTclPredicateAttrGet(ctx.ptr(), stringRef(@tagName(value)))) catch unreachable; +} + +/// `@ context(operator = {Chip: ...}, heuristic_hint = {...})`. +pub fn contextAttr(ctx: *mlir.Context, fields: []const mlir.NamedAttribute) mlir.Error!*const mlir.Attribute { + return attribute(c.mlirTclContextAttrGet(ctx.ptr(), mlir.Attribute.dict(ctx, fields).ptr())); +} + +/// `read(x, subtraction=, table_lookup=, broadcast_to=, typecast_to=, pad=)`. +pub fn readOptionsAttr(ctx: *mlir.Context, fields: []const mlir.NamedAttribute) mlir.Error!*const mlir.Attribute { + return attribute(c.mlirTclReadOptionsAttrGet(ctx.ptr(), mlir.Attribute.dict(ctx, fields).ptr())); +} + +/// `@compiler_config({...})` / `@auto_config({...})`, JSON-compatible fields. +pub fn configAttr(ctx: *mlir.Context, fields: []const mlir.NamedAttribute) mlir.Error!*const mlir.Attribute { + return attribute(c.mlirTclConfigAttrGet(ctx.ptr(), mlir.Attribute.dict(ctx, fields).ptr())); +} + +/// `dram(chip | inner)`, with the original axes of a padded mapping. +pub fn dramMappingAttr(ctx: *mlir.Context, chip: []const *const mlir.Attribute, inner: []const *const mlir.Attribute, original: []const *const mlir.Attribute) mlir.Error!*const mlir.Attribute { + return attribute(c.mlirTclDramMappingAttrGet( + ctx.ptr(), + mlir.Attribute.array(ctx, chip).ptr(), + mlir.Attribute.array(ctx, inner).ptr(), + mlir.Attribute.array(ctx, original).ptr(), + )); +} + +/// Discardable TCL attributes of a `func.func`. +pub const FunctionOpts = struct { + /// `tcl.updated_inputs`: per result, the argument it updates in place, or -1. + updated_inputs: ?[]const i64 = null, + /// `tcl.compiler_config`, a `configAttr`. + compiler_config: ?*const mlir.Attribute = null, + /// `tcl.auto_config`, a `configAttr`. + auto_config: ?*const mlir.Attribute = null, + /// `tcl.fusible`: the plugin may compile the function into the unit around it. + fusible: bool = false, +}; + +/// Sets `tcl.axes`, each axis's size or definition by name, and the +/// attributes of `opts` on `function`. +pub fn annotateFunction(ctx: *mlir.Context, function: *mlir.Operation, axes: []const mlir.NamedAttribute, opts: FunctionOpts) void { + if (opts.updated_inputs) |inputs| function.setAttributeByName("tcl.updated_inputs", .denseArray(ctx, .i64, inputs)); + function.setAttributeByName("tcl.axes", .dict(ctx, axes)); + if (opts.compiler_config) |config| function.setAttributeByName("tcl.compiler_config", config); + if (opts.auto_config) |config| function.setAttributeByName("tcl.auto_config", config); + if (opts.fusible) function.setAttributeByName("tcl.fusible", .unit(ctx)); +} + +// ============================================================================= +// Types +// ============================================================================= + +/// `[A, B]/bf16`: a logical tensor over named axes. +pub fn logicalType(ctx: *mlir.Context, element: *const mlir.Type, axes: []const *const mlir.Attribute) mlir.Error!*const mlir.Type { + return (try LogicalType.get(ctx, .{ .elementType = element, .axes = axes })).type_(); +} + +/// `dram(...)/bf16`: a tensor in an explicit DRAM mapping. +pub fn mappedType(ctx: *mlir.Context, element: *const mlir.Type, mapping: *const mlir.Attribute) mlir.Error!*const mlir.Type { + return type_(c.mlirTclMappedTypeGet(ctx.ptr(), element.ptr(), mapping.ptr())); +} + +pub const LogicalType = opaque { + const M = mlir.Methods(LogicalType, c.MlirType); + + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + pub const isAFn = c.mlirTypeIsATclLogical; + + pub const InitArgs = struct { + elementType: *const mlir.Type, + axes: []const *const mlir.Attribute, + }; + + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const LogicalType { + return @ptrCast(c.mlirTclLogicalTypeGet(ctx.ptr(), args.elementType.ptr(), mlir.Attribute.array(ctx, args.axes).ptr()).ptr orelse return error.InvalidMlir); + } + + pub fn type_(self: *const LogicalType) *const mlir.Type { + return @ptrCast(self); + } + + pub fn elementType(self: *const LogicalType) *const mlir.Type { + return @ptrCast(c.mlirTclLogicalTypeGetElementType(self.ptr()).ptr); + } + + pub fn axes(self: *const LogicalType) *const mlir.ArrayAttribute { + return @ptrCast(c.mlirTclLogicalTypeGetAxes(self.ptr()).ptr); + } +}; + +// ============================================================================= +// Operations +// ============================================================================= +// +// Builders do not verify: kernel instructions only verify inside a +// `tcl.kernel`, and a kernel or graph operation only once its operands +// dominate it. Verify each graph-level operation after inserting it. + +pub const KernelOpts = struct { + context: ?*const mlir.Attribute = null, +}; + +pub fn kernel(ctx: *mlir.Context, inputs: []const *const mlir.Value, output: *const mlir.Type, tactic: Tactic, body: *mlir.Block, opts: KernelOpts, location: *const mlir.Location) *mlir.Operation { + var attrs: [2]mlir.NamedAttribute = undefined; + attrs[0] = .named(ctx, "tactic", tacticAttr(ctx, tactic)); + if (opts.context) |a| attrs[1] = .named(ctx, "context", a); + return mlir.Operation.make(ctx, "tcl.kernel", .{ + .operands = .{ .flat = inputs }, + .results = .{ .flat = &.{output} }, + .attributes = attrs[0 .. @as(usize, 1) + @intFromBool(opts.context != null)], + .blocks = &.{body}, + .verify = false, + .location = location, + }); +} + +pub const ReadOpts = struct { + options: ?*const mlir.Attribute = null, +}; + +pub fn read(ctx: *mlir.Context, inputs: []const *const mlir.Value, output: *const mlir.Type, opts: ReadOpts, location: *const mlir.Location) *mlir.Operation { + var attrs: [1]mlir.NamedAttribute = undefined; + if (opts.options) |o| attrs[0] = .named(ctx, "options", o); + return mlir.Operation.make(ctx, "tcl.read", .{ + .operands = .{ .flat = inputs }, + .results = .{ .flat = &.{output} }, + .attributes = attrs[0..@intFromBool(opts.options != null)], + .verify = false, + .location = location, + }); +} + +pub fn dpe(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, output: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "tcl.dpe", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{output} }, + .verify = false, + .location = location, + }); +} + +pub fn ve(ctx: *mlir.Context, inputs: []const *const mlir.Value, output: *const mlir.Type, opcode: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "tcl.ve", .{ + .operands = .{ .flat = inputs }, + .results = .{ .flat = &.{output} }, + .attributes = &.{.named(ctx, "opcode", opcode)}, + .verify = false, + .location = location, + }); +} + +pub fn ve_reduce(ctx: *mlir.Context, input: *const mlir.Value, output: *const mlir.Type, mode: ReduceMode, axes: []const *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "tcl.ve_reduce", .{ + .operands = .{ .flat = &.{input} }, + .results = .{ .flat = &.{output} }, + .attributes = &.{ + .named(ctx, "mode", reduceModeAttr(ctx, mode)), + .named(ctx, "axes", .array(ctx, axes)), + }, + .verify = false, + .location = location, + }); +} + +pub fn ve_select(ctx: *mlir.Context, condition: *const mlir.Value, true_value: *const mlir.Value, false_value: *const mlir.Value, output: *const mlir.Type, predicate: Predicate, threshold: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "tcl.ve_select", .{ + .operands = .{ .flat = &.{ condition, true_value, false_value } }, + .results = .{ .flat = &.{output} }, + .attributes = &.{ + .named(ctx, "predicate", predicateAttr(ctx, predicate)), + .named(ctx, "threshold", threshold), + }, + .verify = false, + .location = location, + }); +} + +/// `tcl.filter_compaction`: keeps the elements of `input` along `axes` whose +/// `condition` satisfies `predicate` against `threshold`, packed into at most +/// `max_size` (an integer or axis attribute) elements. +pub fn filter_compaction(ctx: *mlir.Context, input: *const mlir.Value, condition: *const mlir.Value, output: *const mlir.Type, predicate: Predicate, threshold: *const mlir.Attribute, axes: []const *const mlir.Attribute, max_size: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "tcl.filter_compaction", .{ + .operands = .{ .flat = &.{ input, condition } }, + .results = .{ .flat = &.{output} }, + .attributes = &.{ + .named(ctx, "predicate", predicateAttr(ctx, predicate)), + .named(ctx, "threshold", threshold), + .named(ctx, "axes", .array(ctx, axes)), + .named(ctx, "max_size", max_size), + }, + .verify = false, + .location = location, + }); +} + +pub fn write(ctx: *mlir.Context, input: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "tcl.write", .{ + .operands = .{ .flat = &.{input} }, + .verify = false, + .location = location, + }); +} + +pub const GraphOp = enum { gather, all_gather, scatter, reshape, transmute, concat, slice, arange, vector, as_logical, as_dram, index_read, index_write, scratchpad, full, reduce_max_i32, sym_expr }; + +pub const GraphOpts = struct { + options: []const mlir.NamedAttribute = &.{}, + context: ?*const mlir.Attribute = null, +}; + +/// `tcl.graph.`: a graph-level TCL operation whose fields go in `options`. +pub fn graph(ctx: *mlir.Context, comptime op: GraphOp, inputs: []const *const mlir.Value, output: *const mlir.Type, opts: GraphOpts, location: *const mlir.Location) *mlir.Operation { + var attrs: [2]mlir.NamedAttribute = undefined; + attrs[0] = .named(ctx, "options", .dict(ctx, opts.options)); + if (opts.context) |a| attrs[1] = .named(ctx, "context", a); + return mlir.Operation.make(ctx, "tcl.graph." ++ @tagName(op), .{ + .operands = .{ .flat = inputs }, + .results = .{ .flat = &.{output} }, + .attributes = attrs[0 .. @as(usize, 1) + @intFromBool(opts.context != null)], + .verify = false, + .location = location, + }); +} + +/// `tcl.graph.copy`, `copy(src=value, dst=input)`: writes `src` into the +/// kernel input `dst` in place. +pub fn copy(ctx: *mlir.Context, src: *const mlir.Value, dst: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "tcl.graph.copy", .{ + .operands = .{ .flat = &.{ src, dst } }, + .results = .{ .flat = &.{} }, + .attributes = &.{.named(ctx, "options", .dict(ctx, &.{}))}, + .verify = false, + .location = location, + }); +} + +pub const LoopOpts = struct { + limit: ?*const mlir.Value = null, + bound: ?*const mlir.Attribute = null, +}; + +/// `tcl.graph.for`: runs `body` up to `limit` (a scalar i32 tensor) or `bound` +/// (an integer or axis attribute) times. The body's arguments are the index, +/// then one accumulator per `inits`; it ends with `tcl.graph.yield`. +pub fn loop(ctx: *mlir.Context, inits: []const *const mlir.Value, result_types: []const *const mlir.Type, body: *mlir.Block, opts: LoopOpts, location: *const mlir.Location) *mlir.Operation { + const limits: []const *const mlir.Value = if (opts.limit) |l| &.{l} else &.{}; + var attrs: [1]mlir.NamedAttribute = undefined; + if (opts.bound) |b| attrs[0] = .named(ctx, "bound", b); + return mlir.Operation.make(ctx, "tcl.graph.for", .{ + .operands = .{ .variadic = &.{ limits, inits } }, + .results = .{ .flat = result_types }, + .attributes = attrs[0..@intFromBool(opts.bound != null)], + .blocks = &.{body}, + .verify = false, + .location = location, + }); +} + +pub fn yield(ctx: *mlir.Context, values: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "tcl.graph.yield", .{ + .operands = .{ .flat = values }, + .verify = false, + .location = location, + }); +} diff --git a/mlir/dialects/tcl/tcl_attrs.cc b/mlir/dialects/tcl/tcl_attrs.cc new file mode 100644 index 00000000..5b2fe575 --- /dev/null +++ b/mlir/dialects/tcl/tcl_attrs.cc @@ -0,0 +1,266 @@ +//===- tcl_attrs.cc - Furiosa TCL attributes ------------------------------===// + +#include "mlir/dialects/tcl/tcl_attrs.h" + +#include "mlir/dialects/tcl/tcl_dialect.h" +#include "mlir/dialects/tcl/tcl_types.h" + +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/StringExtras.h" + +#include +namespace xla::furiosa::tcl { +using namespace mlir; +using Error = llvm::function_ref; +static bool Identifier(llvm::StringRef s) { + if (s.empty() || (!llvm::isAlpha(s.front()) && s.front() != '_')) + return false; + return llvm::all_of(s, [](char c) { return llvm::isAlnum(c) || c == '_'; }); +} +static bool Symbol(llvm::StringRef s) { + auto [base, view] = s.split('.'); + return Identifier(base) && + (view.empty() ? !s.contains('.') : Identifier(view)); +} +static bool Mapping(Attribute a) { + if (!a) + return false; + if (isa(a)) + return IsNumber(a); + if (isa(a)) + return true; + auto e = dyn_cast(a); + return e && + llvm::is_contained( + {"stride", "modulo", "padding", "resize", "pair", "broadcast"}, + e.getKind()) && + llvm::all_of(e.getArgs(), Mapping); +} +static bool Json(Attribute a) { + if (auto n = dyn_cast(a)) + return n.getValue().isFinite(); + if (IsNumber(a) || isa(a)) + return true; + if (auto arr = dyn_cast(a)) + return llvm::all_of(arr, Json); + if (auto dict = dyn_cast(a)) + return llvm::all_of(dict, + [](NamedAttribute f) { return Json(f.getValue()); }); + return false; +} +static LogicalResult Enum(Error error, llvm::StringRef value, + llvm::ArrayRef values) { + return llvm::is_contained(values, value) + ? success() + : error() << "invalid enumerator: " << value; +} + +bool IsNumber(Attribute a) { + if (!a) + return false; + if (auto i = dyn_cast(a)) + return i.getValue().getBitWidth() <= 64; + return isa(a); +} +bool IsExpression(Attribute a) { + if (!a) + return false; + if (isa(a)) + return IsNumber(a); + if (isa(a)) + return true; + if (auto e = dyn_cast(a)) + return e.getKind() != "broadcast" && e.getKind() != "pair" && + e.getKind() != "padding" && e.getKind() != "resize"; + return false; +} +bool IsAxis(Attribute a) { + if (!a) + return false; + auto s = dyn_cast(a); + return s && llvm::isUpper(s.getName().front()); +} +bool IsAxes(Attribute a) { + if (!a) + return false; + auto array = dyn_cast(a); + return array && llvm::all_of(array, IsAxis); +} +LogicalResult VerifyKeys(Error error, DictionaryAttr d, + llvm::ArrayRef keys) { + for (auto f : d) + if (!llvm::is_contained(keys, f.getName().strref())) + return error() << "unknown field: " << f.getName(); + return success(); +} +LogicalResult VerifyAxes(Error error, ArrayAttr axes) { + if (!IsAxes(axes)) + return error() << "axes must be uppercase TCL symbols"; + llvm::SmallDenseSet seen; + for (auto a : axes) + if (!seen.insert(a).second) + return error() << "duplicate logical axis"; + return success(); +} + +// opcode, emitted spelling, arity. Conversion widths are handled below. +static constexpr VeInstructionSpec kVeInstructions[] = { +#define VE(name, text, arity) {name, text, arity}, +#include "mlir/dialects/tcl/ve_instructions.inc" +#undef VE +}; +llvm::ArrayRef VeInstructions() { return kVeInstructions; } + +LogicalResult SymbolAttr::verify(Error error, llvm::StringRef name) { + return Symbol(name) ? success() + : error() << "expected a TCL identifier, optionally with " + "one view suffix"; +} +LogicalResult ExprAttr::verify(Error error, llvm::StringRef kind, + ArrayAttr args) { + if (kind == "broadcast") + return args.empty() ? success() : error() << "broadcast takes no arguments"; + if (kind == "arg") { + if (args.size() != 1 || !isa(args[0]) || !IsNumber(args[0]) || + cast(args[0]).getInt() < 0) + return error() << "arg requires one nonnegative operand index"; + return success(); + } + bool map = llvm::is_contained( + {"stride", "modulo", "padding", "resize", "pair"}, kind); + bool arithmetic = + llvm::is_contained({"add", "sub", "mul", "div", "rem", "exact_div", "eq", + "ne", "lt", "le", "gt", "ge"}, + kind); + if ((!map && !arithmetic) || args.size() != 2) + return error() << "expected a known binary expression"; + for (auto a : args) + if (!(map ? Mapping(a) : IsExpression(a))) + return error() << "invalid expression operand"; + if (llvm::is_contained({"div", "exact_div", "rem", "stride", "modulo"}, kind)) + if (auto n = dyn_cast(args[1]); n && n.getInt() == 0) + return error() << "division by zero"; + return success(); +} +LogicalResult TacticAttr::verify(Error e, llvm::StringRef v) { + return Enum(e, v, + {"EinsumByDpe", "ReduceByVe", "Interleaving", "Elementwise", + "TensorOperation", "EinsumByVe", "FilterCompaction"}); +} +LogicalResult VeOpcodeAttr::verify(Error e, llvm::StringRef v) { + for (const auto &spec : VeInstructions()) + if (spec.name == v) + return success(); + if (v.consume_front("to_f") || v.consume_front("to_i")) { + unsigned width; + if (!v.getAsInteger(10, width) && width <= 31 && std::to_string(width) == v) + return success(); + } + return e() << "unknown VE opcode"; +} +LogicalResult ReduceModeAttr::verify(Error e, llvm::StringRef v) { + return Enum(e, v, {"Addi", "Addf", "Maxi", "Maxf", "Mini", "Minf", "Cumsum"}); +} + +LogicalResult PredicateAttr::verify(Error e, llvm::StringRef v) { + return Enum(e, v, {"eq", "ne", "lt", "le", "gt", "ge"}); +} +LogicalResult ConfigAttr::verify(Error e, DictionaryAttr d) { + return Json(d) ? success() + : e() << "compiler/auto configuration requires " + "JSON-compatible attributes"; +} + +LogicalResult ReadOptionsAttr::verify(Error e, DictionaryAttr d) { + if (failed(VerifyKeys(e, d, + {"subtraction", "table_lookup", "broadcast_to", + "typecast_to", "pad", "slide"}))) + return failure(); + for (auto field : d) { + auto k = field.getName().strref(); + auto v = field.getValue(); + if (k == "subtraction" || k == "table_lookup") { + auto i = dyn_cast(v); + if (!i || !IsNumber(i) || i.getInt() < 1) + return e() << k << " must index an auxiliary read operand (>=1)"; + } else if (k == "broadcast_to") { + if (!IsAxes(v)) + return e() << "broadcast_to requires axes"; + } else if (k == "typecast_to") { + auto t = dyn_cast(v); + if (!t || ElementName(t.getValue()).empty()) + return e() << "invalid typecast element type"; + } else if (k == "slide") { + auto entries = dyn_cast(v); + if (!entries) + return e() << "slide requires an axis dictionary"; + for (auto item : entries) { + auto spec = dyn_cast(item.getValue()); + if (!Symbol(item.getName()) || + !llvm::isUpper(item.getName().strref().front()) || !spec || + failed(VerifyKeys(e, spec, + {"undilated_window", "frame_axis", "window_axis", + "stride", "dilation"})) || + !IsExpression(spec.get("undilated_window")) || + !IsAxis(spec.get("frame_axis")) || !IsAxis(spec.get("window_axis"))) + return e() << "slide requires undilated_window, frame_axis and " + "window_axis"; + for (auto key : {"stride", "dilation"}) + if (auto a = spec.get(key); a && !IsExpression(a)) + return e() << "invalid slide " << key; + } + } else { + auto entries = dyn_cast(v); + if (!entries) + return e() << k << " requires an axis dictionary"; + for (auto item : entries) { + if (!Symbol(item.getName()) || + !llvm::isUpper(item.getName().strref().front())) + return e() << "invalid axis"; + auto p = dyn_cast(item.getValue()); + if (!p || p.size() != 3 || !IsExpression(p[0]) || !IsExpression(p[1]) || + !IsNumber(p[2])) + return e() << "padding requires [left, right, fill]"; + } + } + } + return success(); +} +LogicalResult ContextAttr::verify(Error e, DictionaryAttr d) { + if (failed(VerifyKeys(e, d, {"operator", "heuristic_hint"}))) + return failure(); + if (auto a = d.get("operator")) { + auto layout = dyn_cast(a); + if (!layout || failed(VerifyKeys(e, layout, {"Chip", "Cluster", "Split"}))) + return e() << "invalid operator layout"; + if (!layout.get("Chip")) + return e() << "operator layout requires Chip"; + for (auto f : layout) { + if (f.getName() == "Split") { + if (!IsAxes(f.getValue())) + return e() << "Split requires axes"; + } else if (!IsAxis(f.getValue()) && + !(isa(f.getValue()) && + cast(f.getValue()).getKind() == "broadcast")) + return e() << "Chip/Cluster requires axis or Broadcast"; + } + } + if (auto a = d.get("heuristic_hint")) { + auto hints = dyn_cast(a); + if (!hints) + return e() << "heuristic_hint requires a dictionary"; + for (auto f : hints) + if (!Identifier(f.getName()) || + !(IsNumber(f.getValue()) || IsAxis(f.getValue()))) + return e() << "invalid heuristic hint"; + } + return success(); +} + +LogicalResult DramMappingAttr::verify(Error e, ArrayAttr chip, ArrayAttr inner, + ArrayAttr original) { + if (!llvm::all_of(chip, Mapping) || !llvm::all_of(inner, Mapping)) + return e() << "invalid DRAM mapping"; + return VerifyAxes(e, original); +} +} // namespace xla::furiosa::tcl diff --git a/mlir/dialects/tcl/tcl_attrs.h b/mlir/dialects/tcl/tcl_attrs.h new file mode 100644 index 00000000..816583b9 --- /dev/null +++ b/mlir/dialects/tcl/tcl_attrs.h @@ -0,0 +1,37 @@ +//===- tcl_attrs.h - Furiosa TCL attributes ---------------------*- C++ -*-===// + +#ifndef MLIR_DIALECTS_TCL_TCL_ATTRS_H +#define MLIR_DIALECTS_TCL_TCL_ATTRS_H + +#include "mlir/dialects/tcl/tcl_dialect.h" + +#include "mlir/IR/Attributes.h" +#include "mlir/IR/BuiltinAttributes.h" +#include "mlir/IR/Diagnostics.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/STLFunctionalExtras.h" +#include "llvm/ADT/StringRef.h" +#define GET_ATTRDEF_CLASSES +#include "mlir/dialects/tcl/tcl_attrs.h.inc" +namespace xla::furiosa::tcl { +struct VeInstructionSpec { + llvm::StringRef name; + llvm::StringRef spelling; + unsigned arity; +}; +llvm::ArrayRef VeInstructions(); + +// Schema predicates shared by attribute, type and operation verifiers. +bool IsNumber(mlir::Attribute a); +bool IsExpression(mlir::Attribute a); +bool IsAxis(mlir::Attribute a); +bool IsAxes(mlir::Attribute a); +mlir::LogicalResult +VerifyKeys(llvm::function_ref error, + mlir::DictionaryAttr d, llvm::ArrayRef keys); +mlir::LogicalResult +VerifyAxes(llvm::function_ref error, + mlir::ArrayAttr axes); +} // namespace xla::furiosa::tcl + +#endif // MLIR_DIALECTS_TCL_TCL_ATTRS_H diff --git a/mlir/dialects/tcl/tcl_attrs.td b/mlir/dialects/tcl/tcl_attrs.td new file mode 100644 index 00000000..f85c518e --- /dev/null +++ b/mlir/dialects/tcl/tcl_attrs.td @@ -0,0 +1,74 @@ +//===- tcl_attrs.td - Furiosa TCL attributes ---------------*- tablegen -*-===// + +#ifndef MLIR_DIALECTS_TCL_TCL_ATTRS_TD +#define MLIR_DIALECTS_TCL_TCL_ATTRS_TD + +include "mlir/IR/AttrTypeBase.td" +include "mlir/dialects/tcl/tcl_dialect.td" + +def Tcl_Symbol : AttrDef { + let mnemonic = "symbol"; + let parameters = (ins StringRefParameter<>:$name); + let assemblyFormat = "`<` $name `>`"; + let genVerifyDecl = 1; +} +def Tcl_Expr : AttrDef { + let mnemonic = "expr"; + let summary = "Checked symbolic arithmetic or DRAM mapping expression tree"; + let parameters = (ins StringRefParameter<>:$kind, "::mlir::ArrayAttr":$args); + let assemblyFormat = "`<` $kind `,` $args `>`"; + let genVerifyDecl = 1; +} +def Tcl_Tactic : AttrDef { + let mnemonic = "tactic"; + let parameters = (ins StringRefParameter<>:$value); + let assemblyFormat = "`<` $value `>`"; + let genVerifyDecl = 1; +} +def Tcl_VeOpcode : AttrDef { + let mnemonic = "ve_opcode"; + let parameters = (ins StringRefParameter<>:$value); + let assemblyFormat = "`<` $value `>`"; + let genVerifyDecl = 1; +} +def Tcl_ReduceMode : AttrDef { + let mnemonic = "reduce_mode"; + let parameters = (ins StringRefParameter<>:$value); + let assemblyFormat = "`<` $value `>`"; + let genVerifyDecl = 1; +} + +def Tcl_Predicate : AttrDef { + let mnemonic = "predicate"; + let parameters = (ins StringRefParameter<>:$value); + let assemblyFormat = "`<` $value `>`"; + let genVerifyDecl = 1; +} +def Tcl_Context : AttrDef { + let mnemonic = "context"; + let parameters = (ins "::mlir::DictionaryAttr":$fields); + let assemblyFormat = "`<` $fields `>`"; + let genVerifyDecl = 1; +} +def Tcl_ReadOptions : AttrDef { + let mnemonic = "read_options"; + let parameters = (ins "::mlir::DictionaryAttr":$fields); + let assemblyFormat = "`<` $fields `>`"; + let genVerifyDecl = 1; +} + +def Tcl_Config : AttrDef { + let mnemonic = "config"; + let parameters = (ins "::mlir::DictionaryAttr":$fields); + let assemblyFormat = "`<` $fields `>`"; + let genVerifyDecl = 1; +} +def Tcl_DramMapping : AttrDef { + let mnemonic = "dram_mapping"; + let parameters = (ins "::mlir::ArrayAttr":$chip, "::mlir::ArrayAttr":$inner, + "::mlir::ArrayAttr":$original); + let assemblyFormat = "`<` $chip `,` $inner `,` $original `>`"; + let genVerifyDecl = 1; +} + +#endif // MLIR_DIALECTS_TCL_TCL_ATTRS_TD diff --git a/mlir/dialects/tcl/tcl_dialect.cc b/mlir/dialects/tcl/tcl_dialect.cc new file mode 100644 index 00000000..2cea51a1 --- /dev/null +++ b/mlir/dialects/tcl/tcl_dialect.cc @@ -0,0 +1,34 @@ +//===- tcl_dialect.cc - Furiosa TCL dialect -------------------------------===// + +#include "mlir/dialects/tcl/tcl_dialect.h" + +#include "mlir/dialects/tcl/tcl_attrs.h" // IWYU pragma: keep +#include "mlir/dialects/tcl/tcl_ops.h" // IWYU pragma: keep +#include "mlir/dialects/tcl/tcl_types.h" // IWYU pragma: keep + +#include "mlir/IR/Builders.h" // IWYU pragma: keep +#include "mlir/IR/DialectImplementation.h" // IWYU pragma: keep +#include "llvm/ADT/TypeSwitch.h" // IWYU pragma: keep + +// Storage definitions must be complete where the dialect registers them. +#include "mlir/dialects/tcl/tcl_dialect.cc.inc" +#define GET_ATTRDEF_CLASSES +#include "mlir/dialects/tcl/tcl_attrs.cc.inc" +#define GET_TYPEDEF_CLASSES +#include "mlir/dialects/tcl/tcl_types.cc.inc" +namespace xla::furiosa::tcl { +void TclDialect::initialize() { + addAttributes< +#define GET_ATTRDEF_LIST +#include "mlir/dialects/tcl/tcl_attrs.cc.inc" + >(); + addTypes< +#define GET_TYPEDEF_LIST +#include "mlir/dialects/tcl/tcl_types.cc.inc" + >(); + addOperations< +#define GET_OP_LIST +#include "mlir/dialects/tcl/tcl_ops.cc.inc" + >(); +} +} // namespace xla::furiosa::tcl diff --git a/mlir/dialects/tcl/tcl_dialect.h b/mlir/dialects/tcl/tcl_dialect.h new file mode 100644 index 00000000..97267065 --- /dev/null +++ b/mlir/dialects/tcl/tcl_dialect.h @@ -0,0 +1,9 @@ +//===- tcl_dialect.h - Furiosa TCL dialect ----------------------*- C++ -*-===// + +#ifndef MLIR_DIALECTS_TCL_TCL_DIALECT_H +#define MLIR_DIALECTS_TCL_TCL_DIALECT_H + +#include "mlir/IR/Dialect.h" // IWYU pragma: keep +#include "mlir/dialects/tcl/tcl_dialect.h.inc" + +#endif // MLIR_DIALECTS_TCL_TCL_DIALECT_H diff --git a/mlir/dialects/tcl/tcl_dialect.td b/mlir/dialects/tcl/tcl_dialect.td new file mode 100644 index 00000000..7b86a825 --- /dev/null +++ b/mlir/dialects/tcl/tcl_dialect.td @@ -0,0 +1,19 @@ +//===- tcl_dialect.td - Furiosa TCL dialect ----------------*- tablegen -*-===// + +#ifndef MLIR_DIALECTS_TCL_TCL_DIALECT_TD +#define MLIR_DIALECTS_TCL_TCL_DIALECT_TD + +include "mlir/IR/DialectBase.td" + +// Native TCL: tensors over named axes, `tk.*` tensor operations and graph +// operations, one-to-one with the TCL source the Furiosa compiler takes. The +// subset of xla/backends/furiosa/codegen/tcl/ir that ZML emits. +def Tcl_Dialect : Dialect { + let name = "tcl"; + let cppNamespace = "::xla::furiosa::tcl"; + let summary = "Typed Furiosa Tensor Compute Language graph"; + let useDefaultTypePrinterParser = 1; + let useDefaultAttributePrinterParser = 1; +} + +#endif // MLIR_DIALECTS_TCL_TCL_DIALECT_TD diff --git a/mlir/dialects/tcl/tcl_ops.cc b/mlir/dialects/tcl/tcl_ops.cc new file mode 100644 index 00000000..9d569f0b --- /dev/null +++ b/mlir/dialects/tcl/tcl_ops.cc @@ -0,0 +1,473 @@ +//===- tcl_ops.cc - Furiosa TCL operations --------------------------------===// + +#include "mlir/dialects/tcl/tcl_ops.h" + +#include "mlir/IR/Builders.h" // IWYU pragma: keep +#include "mlir/IR/Matchers.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/STLExtras.h" +#define GET_OP_CLASSES +#include "mlir/dialects/tcl/tcl_ops.cc.inc" +namespace xla::furiosa::tcl { +using namespace mlir; +static bool Tensor(Type t) { return isa(t); } +static bool ScalarI32(Type t) { + auto l = dyn_cast(t); + return l && l.getAxes().empty() && l.getElementType().isInteger(32); +} +static bool LoopIndex(Value v) { + auto arg = dyn_cast(v); + return arg && arg.getArgNumber() == 0 && + isa_and_nonnull(arg.getOwner()->getParentOp()); +} +static LogicalResult InKernel(Operation *op) { + if (!op->getParentOfType()) + return op->emitOpError("requires a tcl.kernel region"); + return success(); +} +// Exact graph operation schema. Optional operands occupy documented positions; +// the options dictionary never changes operand ordering. +static LogicalResult VerifyGraph(Operation *op, DictionaryAttr options) { + auto e = [&] { return op->emitOpError(); }; + llvm::StringRef n = op->getName().getStringRef().drop_front(10); + int min = 1, max = 1, results = 1; + llvm::SmallVector fields; + if (n == "all_gather") { + fields = {"axis"}; + } else if (n == "gather" || n == "scatter") { + min = 2; + max = 3; + fields = {"axis", "batch_axis", "table_dram_layout"}; + } else if (n == "concat") { + max = -1; + } else if (n == "slice") { + fields = {"axis", "offset"}; + } else if (n == "arange") { + min = max = 0; + fields = {"start", "end", "step"}; + } else if (n == "vector") { + min = max = 0; + fields = {"values", "repeat_to"}; + } else if (n == "index_read") { + min = max = 2; + } else if (n == "index_write") { + min = max = 3; + } else if (n == "copy") { + min = max = 2; + results = 0; + } else if (n == "scratchpad") { + min = max = 0; + } else if (n == "full") { + min = max = 0; + fields = {"value"}; + } else if (n == "sym_expr") { + min = 0; + max = -1; + fields = {"expr"}; + } + if (op->getNumOperands() < min || (max >= 0 && op->getNumOperands() > max) || + op->getNumResults() != results) + return e() << "incorrect operand or result count"; + if (failed(VerifyKeys(e, options, fields))) + return failure(); + if (n == "gather" && options.get("batch_axis") && op->getNumOperands() == 3) + return e() << "SDK sparse gather cannot also use batch_axis"; + for (Type t : op->getOperandTypes()) + if (!Tensor(t)) + return e() << "expected a native TCL tensor operand"; + for (Type t : op->getResultTypes()) + if (!Tensor(t)) + return e() << "expected a native TCL tensor result"; + for (auto f : options) { + auto k = f.getName().strref(); + auto v = f.getValue(); + bool ok = false; + if (k.ends_with("axis") || k == "repeat_to") + ok = IsAxis(v); + else if (k == "axes") + ok = IsAxes(v); + else if (k == "value") + ok = IsNumber(v); + else if (k == "values") { + auto a = dyn_cast(v); + ok = a && !a.empty() && llvm::all_of(a, IsNumber); + } else if (k == "table_dram_layout") + ok = isa(v) && isa(cast(v).getValue()); + else + ok = IsExpression(v); + if (!ok) + return e() << "invalid field " << k; + } + for (auto k : fields) { + bool required = llvm::is_contained( + {"axis", "offset", "end", "values", "expr", "value"}, k); + if (required && !options.get(k)) + return e() << "missing required field " << k; + } + if (n == "as_logical" && (!isa(op->getOperand(0).getType()) || + !isa(op->getResult(0).getType()))) + return e() << "as_logical requires storage -> logical"; + if (n == "as_dram" && (!isa(op->getOperand(0).getType()) || + !isa(op->getResult(0).getType()))) + return e() << "as_dram requires logical -> mapped"; + if (n == "index_read" || n == "index_write") { + if (!LoopIndex(op->getOperand(1))) + return e() << "requires the index of an enclosing loop"; + auto table = dyn_cast(op->getOperand(0).getType()); + auto slice = dyn_cast( + (n == "index_read" ? op->getResult(0) : op->getOperand(2)).getType()); + if (!table || !slice || table.getAxes().empty() || + table.getElementType() != slice.getElementType() || + table.getAxes().getValue().drop_front() != slice.getAxes().getValue()) + return e() << "slice must be the table without its outermost axis"; + if (n == "index_write" && + (op->getResult(0).getType() != table || !op->getOperand(0).hasOneUse())) + return e() << "writes in place: the table must have no other use"; + } + if (n == "copy" && + (!isa(op->getOperand(1)) || + op->getOperand(0).getType() != op->getOperand(1).getType())) + return e() << "copies a value into a kernel input of its type"; + if ((n == "reduce_max_i32" || n == "sym_expr") && + !ScalarI32(op->getResult(0).getType())) + return e() << "requires a scalar i32 result"; + if (n == "sym_expr") { + for (Value v : op->getOperands()) + if (!ScalarI32(v.getType())) + return e() << "requires scalar i32 operands"; + llvm::SmallVector pending{options.get("expr")}; + while (!pending.empty()) { + auto x = dyn_cast(pending.pop_back_val()); + if (!x) + continue; + if (x.getKind() == "arg") { + if (cast(x.getArgs()[0]).getInt() >= op->getNumOperands()) + return e() << "expression operand out of range"; + continue; + } + pending.append(x.getArgs().begin(), x.getArgs().end()); + } + } + return success(); +} + +LogicalResult KernelOp::verify() { + if (getBody().empty() || getBody().front().empty() || + getBody().front().getNumArguments()) + return emitOpError("requires one body without block arguments"); + auto write = dyn_cast(getBody().front().getTerminator()); + if (!write) + return emitOpError("requires tcl.write terminator"); + for (auto input : getInputs()) + if (!Tensor(input.getType())) + return emitOpError("requires native tensor inputs"); + for (Operation &op : getBody().front()) { + if (!isa(op) && + op.getName().getStringRef() != "arith.constant") + return emitOpError("unsupported kernel instruction ") << op.getName(); + } + WalkResult captured = getBody().walk([&](Operation *op) { + for (Value v : op->getOperands()) { + if (getBody().isAncestor(v.getParentRegion())) + continue; + if (!isa(op) || !llvm::is_contained(getInputs(), v)) { + op->emitOpError( + "external values must be explicit kernel inputs consumed by read"); + return WalkResult::interrupt(); + } + } + return WalkResult::advance(); + }); + if (captured.wasInterrupted()) + return failure(); + return success(); +} +LogicalResult ReadOp::verify() { + if (failed(InKernel(*this))) + return failure(); + if (getInputs().empty() || + !llvm::all_of(getInputs(), [](Value v) { return Tensor(v.getType()); })) + return emitOpError("requires tensor inputs"); + if (getOptions()) + for (auto k : {"subtraction", "table_lookup"}) + if (auto a = getOptions()->getFields().getAs(k); + a && a.getInt() >= getInputs().size()) + return emitOpError("auxiliary read operand index out of bounds"); + if (auto input = dyn_cast(getInputs()[0].getType())) { + SmallVector axes(input.getAxes().begin(), input.getAxes().end()); + Type element = input.getElementType(); + if (getOptions()) { + auto fields = getOptions()->getFields(); + if (auto table = fields.getAs("table_lookup")) { + auto t = dyn_cast(getInputs()[table.getInt()].getType()); + if (!t || t.getAxes().size() != 1) + return emitOpError("lookup table must be a one-axis logical tensor"); + element = t.getElementType(); + } + if (auto cast = fields.getAs("typecast_to")) + element = cast.getValue(); + if (auto pads = fields.getAs("pad")) + for (auto pad : pads) + if (!llvm::is_contained(axes, + SymbolAttr::get(getContext(), pad.getName()))) + return emitOpError("padding axis missing from input"); + // A slide replaces its axis with the window and frame axes. + if (auto slides = fields.getAs("slide")) + for (auto slide : slides) { + auto it = + llvm::find(axes, SymbolAttr::get(getContext(), slide.getName())); + if (it == axes.end()) + return emitOpError("slide axis missing from input"); + auto spec = cast(slide.getValue()); + *it = spec.get("window_axis"); + axes.insert(it + 1, spec.get("frame_axis")); + } + if (auto broadcast = fields.getAs("broadcast_to")) { + for (auto axis : axes) + if (!llvm::is_contained(broadcast, axis)) + return emitOpError("broadcast must retain all read axes"); + axes.assign(broadcast.begin(), broadcast.end()); + } + } + if (getOutput().getType().getAxes().getValue() != + ArrayRef(axes) || + getOutput().getType().getElementType() != element) + return emitOpError("read result disagrees with its modifiers"); + } + return success(); +} +LogicalResult DpeOp::verify() { + if (failed(InKernel(*this))) + return failure(); + auto out = getOutput().getType(); + if (!out.getElementType().isF32() && + !out.getElementType().isSignlessInteger(32)) + return emitOpError("DPE accumulators require f32 or i32"); + for (auto axis : out.getAxes()) + if (!llvm::is_contained(getLhs().getType().getAxes(), axis) && + !llvm::is_contained(getRhs().getType().getAxes(), axis)) + return emitOpError("DPE result axis missing from both inputs"); + return success(); +} +LogicalResult VeOp::verify() { + if (failed(InKernel(*this))) + return failure(); + unsigned arity = 1; + for (auto spec : VeInstructions()) + if (spec.name == getOpcode().getValue()) + arity = spec.arity; + if (getInputs().size() != arity) + return emitOpError("incorrect VE operand count"); + llvm::SmallDenseSet axes; + for (Value v : getInputs()) { + if (auto type = dyn_cast(v.getType())) { + for (auto a : type.getAxes()) + axes.insert(a); + } else if (!isa(v.getType()) || + !matchPattern(v, m_Constant())) + return emitOpError("VE scalar operands must be constants"); + } + auto out = getOutput().getType(); + if (axes.size() != out.getAxes().size() || + !llvm::all_of(out.getAxes(), + [&](Attribute a) { return axes.contains(a); })) + return emitOpError("VE output axes must be the union of operand axes"); + auto code = getOpcode().getValue(); + llvm::StringRef spelling = code; + for (auto spec : VeInstructions()) + if (spec.name == code) + spelling = spec.spelling; + bool fp = spelling.starts_with("to_f") || spelling == "as_f" || + spelling.ends_with("_f") || + llvm::is_contained({"+", "-", "*", "/", "exp", "neg_exp", "sqrt", + "tanh", "sigmoid", "erf", "log", "sin", "cos"}, + spelling); + if (fp ? !out.getElementType().isF32() + : !out.getElementType().isSignlessInteger(32)) + return emitOpError("VE registers produce f32 or i32 according to opcode"); + return success(); +} +LogicalResult VeReduceOp::verify() { + if (failed(InKernel(*this)) || + failed(VerifyAxes([&] { return emitOpError(); }, getAxes()))) + return failure(); + // An integer cumulative sum runs along its axes and keeps them. + if (getMode().getValue() == "Cumsum") { + if (getAxes().empty() || + getInput().getType().getAxes() != getOutput().getType().getAxes() || + !llvm::all_of(getAxes(), + [&](Attribute a) { + return llvm::is_contained( + getInput().getType().getAxes(), a); + }) || + !getInput().getType().getElementType().isInteger(32)) + return emitOpError("cumsum keeps the axes of an i32 value"); + return success(); + } + llvm::SmallVector remaining; + for (Attribute a : getInput().getType().getAxes()) + if (!llvm::is_contained(getAxes(), a)) + remaining.push_back(a); + if (getAxes().empty() || + remaining.size() + getAxes().size() != + getInput().getType().getAxes().size() || + remaining != getOutput().getType().getAxes().getValue()) + return emitOpError("reduction result must remove exactly the reduced axes"); + return success(); +} + +LogicalResult VeSelectOp::verify() { + if (failed(InKernel(*this)) || !IsNumber(getThreshold())) + return emitOpError("requires kernel and numeric threshold"); + llvm::SmallDenseSet axes; + for (Value v : + SmallVector{getCondition(), getTrueValue(), getFalseValue()}) { + if (auto t = dyn_cast(v.getType())) { + for (auto a : t.getAxes()) + axes.insert(a); + } else if (!matchPattern(v, m_Constant())) + return emitOpError("scalar branches require constants"); + } + auto output = getOutput().getType().getAxes(); + if (axes.size() != output.size() || + !llvm::all_of(output, [&](Attribute a) { return axes.contains(a); })) + return emitOpError("select output axes must cover its operands"); + return success(); +} + +LogicalResult FilterCompactionOp::verify() { + if (failed(InKernel(*this)) || + failed(VerifyAxes([&] { return emitOpError(); }, getAxes()))) + return failure(); + auto size = dyn_cast(getMaxSize()); + if (!IsNumber(getThreshold()) || + (!IsAxis(getMaxSize()) && !(size && size.getInt() > 0)) || + getAxes().empty()) + return emitOpError( + "requires numeric threshold, positive maximum size, and axes"); + if (getInput().getType().getAxes() != getCondition().getType().getAxes() || + getInput().getType().getElementType() != + getOutput().getType().getElementType() || + !llvm::all_of(getAxes(), [&](Attribute axis) { + return llvm::is_contained(getInput().getType().getAxes(), axis); + })) + return emitOpError("incompatible compaction input, condition, or axes"); + if (!llvm::all_of(getOutput().getUsers(), + [](Operation *op) { return isa(op); })) + return emitOpError( + "filter_compaction must be the final kernel instruction"); + return success(); +} + +LogicalResult WriteOp::verify() { + auto kernel = dyn_cast((*this)->getParentOp()); + if (!kernel) + return emitOpError("requires a kernel parent"); + auto in = getInput().getType(), out = kernel.getOutput().getType(); + if (in.getAxes().size() != out.getAxes().size()) + return emitOpError("write must preserve the axis set"); + for (auto a : in.getAxes()) + if (!llvm::is_contained(out.getAxes(), a)) + return emitOpError("write must preserve the axis set"); + // TCL write.to implicitly consumes the preceding instruction. + auto previous = (*this)->getPrevNode(); + while (previous && previous->getName().getStringRef() == "arith.constant") + previous = previous->getPrevNode(); + if (getInput().getDefiningOp() != previous) + return emitOpError( + "write must consume the immediately preceding instruction"); + return success(); +} + +LogicalResult GraphGatherOp::verify() { + return VerifyGraph(*this, getOptions()); +} +LogicalResult GraphAllGatherOp::verify() { + if (failed(VerifyGraph(*this, getOptions()))) + return failure(); + auto input = dyn_cast(getInputs()[0].getType()); + auto output = dyn_cast(getOutputs()[0].getType()); + if (!input || !output || input != output) + return emitOpError("all_gather requires matching logical tensor types"); + if (!llvm::is_contained(input.getAxes(), getOptions().get("axis"))) + return emitOpError("collective axis must be present in the input"); + return success(); +} +LogicalResult GraphScatterOp::verify() { + return VerifyGraph(*this, getOptions()); +} +LogicalResult GraphReshapeOp::verify() { + return VerifyGraph(*this, getOptions()); +} +LogicalResult GraphTransmuteOp::verify() { + return VerifyGraph(*this, getOptions()); +} + +LogicalResult GraphConcatOp::verify() { + return VerifyGraph(*this, getOptions()); +} + +LogicalResult GraphSliceOp::verify() { + return VerifyGraph(*this, getOptions()); +} +LogicalResult GraphArangeOp::verify() { + return VerifyGraph(*this, getOptions()); +} +LogicalResult GraphVectorOp::verify() { + return VerifyGraph(*this, getOptions()); +} + +LogicalResult GraphAsLogicalOp::verify() { + return VerifyGraph(*this, getOptions()); +} +LogicalResult GraphAsDramOp::verify() { + return VerifyGraph(*this, getOptions()); +} +LogicalResult GraphCopyOp::verify() { return VerifyGraph(*this, getOptions()); } +LogicalResult GraphIndexReadOp::verify() { + return VerifyGraph(*this, getOptions()); +} +LogicalResult GraphIndexWriteOp::verify() { + return VerifyGraph(*this, getOptions()); +} +LogicalResult GraphScratchpadOp::verify() { + return VerifyGraph(*this, getOptions()); +} +LogicalResult GraphFullOp::verify() { return VerifyGraph(*this, getOptions()); } +LogicalResult GraphReduceMaxI32Op::verify() { + return VerifyGraph(*this, getOptions()); +} +LogicalResult GraphSymExprOp::verify() { + return VerifyGraph(*this, getOptions()); +} +LogicalResult GraphForOp::verify() { + if (!getLimit() == !getBound()) + return emitOpError("requires exactly one of a limit tensor or a bound"); + if (getLimit() && !ScalarI32(getLimit().getType())) + return emitOpError("limit must be a scalar i32 tensor"); + if (auto bound = getBoundAttr()) { + auto n = dyn_cast(bound); + if (!IsAxis(bound) && !(n && IsNumber(n) && n.getInt() > 0)) + return emitOpError("bound must be a positive integer or an axis"); + } + Block &body = getBody().front(); + if (body.getNumArguments() != getInits().size() + 1 || + !ScalarI32(body.getArgument(0).getType())) + return emitOpError("requires a scalar i32 index and one accumulator each"); + auto yield = dyn_cast(body.getTerminator()); + if (!yield) + return emitOpError("requires a tcl.graph.yield terminator"); + if (getOutputs().size() != getInits().size() || + yield.getValues().size() != getInits().size()) + return emitOpError("requires one result and one yield per accumulator"); + for (auto [i, init] : llvm::enumerate(getInits())) { + Type t = init.getType(); + if (!Tensor(t) || body.getArgument(i + 1).getType() != t || + getOutputs()[i].getType() != t || yield.getValues()[i].getType() != t) + return emitOpError("accumulator ") << i << " changes type"; + } + return success(); +} + +} // namespace xla::furiosa::tcl diff --git a/mlir/dialects/tcl/tcl_ops.h b/mlir/dialects/tcl/tcl_ops.h new file mode 100644 index 00000000..bce6ccae --- /dev/null +++ b/mlir/dialects/tcl/tcl_ops.h @@ -0,0 +1,19 @@ +//===- tcl_ops.h - Furiosa TCL operations -----------------------*- C++ -*-===// + +#ifndef MLIR_DIALECTS_TCL_TCL_OPS_H +#define MLIR_DIALECTS_TCL_TCL_OPS_H + +#include "mlir/dialects/tcl/tcl_attrs.h" // IWYU pragma: export +#include "mlir/dialects/tcl/tcl_dialect.h" // IWYU pragma: export +#include "mlir/dialects/tcl/tcl_types.h" // IWYU pragma: export + +#include "mlir/Bytecode/BytecodeOpInterface.h" // IWYU pragma: keep +#include "mlir/IR/BuiltinAttributes.h" // IWYU pragma: keep +#include "mlir/IR/BuiltinTypes.h" // IWYU pragma: keep +#include "mlir/IR/OpDefinition.h" // IWYU pragma: keep +#include "mlir/IR/OpImplementation.h" // IWYU pragma: keep +#include "mlir/Interfaces/SideEffectInterfaces.h" // IWYU pragma: keep +#define GET_OP_CLASSES +#include "mlir/dialects/tcl/tcl_ops.h.inc" + +#endif // MLIR_DIALECTS_TCL_TCL_OPS_H diff --git a/mlir/dialects/tcl/tcl_ops.td b/mlir/dialects/tcl/tcl_ops.td new file mode 100644 index 00000000..23676ec9 --- /dev/null +++ b/mlir/dialects/tcl/tcl_ops.td @@ -0,0 +1,122 @@ +//===- tcl_ops.td - Furiosa TCL operations -----------------*- tablegen -*-===// + +#ifndef MLIR_DIALECTS_TCL_TCL_OPS_TD +#define MLIR_DIALECTS_TCL_TCL_OPS_TD + +include "mlir/IR/OpBase.td" +include "mlir/Interfaces/SideEffectInterfaces.td" +include "mlir/dialects/tcl/tcl_attrs.td" +include "mlir/dialects/tcl/tcl_dialect.td" +include "mlir/dialects/tcl/tcl_types.td" + +class Tcl_Op traits = []> + : Op; + +// Options contain typed attributes, never snippets of TCL source. Their exact +// per-operation schema is verified in tcl_ops.cc. Generic assembly is +// intentional: all native constructs support standard MLIR parsing, printing +// and bytecode. +class Tcl_GraphOp traits = [MemoryEffects<[MemRead]>]> + : Tcl_Op { + let arguments = (ins Variadic:$inputs, + DefaultValuedAttr:$options, + OptionalAttr:$context); + let results = (outs Variadic:$outputs); + let hasVerifier = 1; +} +def Tcl_GraphGatherOp : Tcl_GraphOp<"gather">; +def Tcl_GraphAllGatherOp : Tcl_GraphOp<"all_gather">; +def Tcl_GraphScatterOp : Tcl_GraphOp<"scatter">; +def Tcl_GraphReshapeOp : Tcl_GraphOp<"reshape">; +def Tcl_GraphTransmuteOp : Tcl_GraphOp<"transmute">; + +def Tcl_GraphConcatOp : Tcl_GraphOp<"concat">; + +def Tcl_GraphSliceOp : Tcl_GraphOp<"slice">; +def Tcl_GraphArangeOp : Tcl_GraphOp<"arange">; +def Tcl_GraphVectorOp : Tcl_GraphOp<"vector">; + +def Tcl_GraphAsLogicalOp : Tcl_GraphOp<"as_logical">; +def Tcl_GraphAsDramOp : Tcl_GraphOp<"as_dram">; +// `copy(src=value, dst=input)` writes a value into a kernel input in place. +def Tcl_GraphCopyOp + : Tcl_GraphOp<"copy", [MemoryEffects<[MemRead, MemWrite]>]>; + +// Loop-level graph constructs. `x[i]` reads and `acc[i] = v` writes the slice +// of the outermost axis at the loop index; `sym_expr` evaluates an integer +// expression (`#tcl.expr` names its operands) on the device. +def Tcl_GraphIndexReadOp : Tcl_GraphOp<"index_read">; +def Tcl_GraphIndexWriteOp : Tcl_GraphOp<"index_write">; +def Tcl_GraphScratchpadOp : Tcl_GraphOp<"scratchpad">; +def Tcl_GraphFullOp : Tcl_GraphOp<"full">; +def Tcl_GraphReduceMaxI32Op : Tcl_GraphOp<"reduce_max_i32">; +def Tcl_GraphSymExprOp : Tcl_GraphOp<"sym_expr">; + +// `outputs = for i in 0..limit (acc = init, ...): body; yield ...;`. The limit +// is a static bound (integer or axis) or a scalar i32 tensor computed in the +// function. The block arguments are the loop index, then the accumulators. +def Tcl_GraphForOp : Tcl_Op<"graph.for", + [AttrSizedOperandSegments, RecursiveMemoryEffects, SingleBlock]> { + let arguments = (ins Optional:$limit, Variadic:$inits, + OptionalAttr:$bound); + let results = (outs Variadic:$outputs); + let regions = (region SizedRegion<1>:$body); + let hasVerifier = 1; +} +def Tcl_GraphYieldOp : Tcl_Op<"graph.yield", + [Pure, Terminator, HasParent<"GraphForOp">]> { + let arguments = (ins Variadic:$values); +} + +def Tcl_KernelOp : Tcl_Op<"kernel", [MemoryEffects<[MemRead]>, SingleBlock]> { + let arguments = (ins Variadic:$inputs, Tcl_Tactic:$tactic, + OptionalAttr:$context); + let results = (outs Tcl_Logical:$output); + let regions = (region AnyRegion:$body); + let hasVerifier = 1; +} +def Tcl_ReadOp : Tcl_Op<"read", [Pure]> { + let arguments = (ins Variadic:$inputs, + OptionalAttr:$options); + let results = (outs Tcl_Logical:$output); + let hasVerifier = 1; +} +def Tcl_DpeOp : Tcl_Op<"dpe", [Pure]> { + let arguments = (ins Tcl_Logical:$lhs, Tcl_Logical:$rhs); + let results = (outs Tcl_Logical:$output); + let hasVerifier = 1; +} +def Tcl_VeOp : Tcl_Op<"ve", [Pure]> { + let arguments = (ins Variadic:$inputs, Tcl_VeOpcode:$opcode); + let results = (outs Tcl_Logical:$output); + let hasVerifier = 1; +} +def Tcl_VeReduceOp : Tcl_Op<"ve_reduce", [Pure]> { + let arguments = (ins Tcl_Logical:$input, Tcl_ReduceMode:$mode, + ArrayAttr:$axes); + let results = (outs Tcl_Logical:$output); + let hasVerifier = 1; +} + +def Tcl_VeSelectOp : Tcl_Op<"ve_select", [Pure]> { + let arguments = (ins Tcl_Logical:$condition, Tcl_Predicate:$predicate, + AnyAttr:$threshold, AnyType:$true_value, + AnyType:$false_value); + let results = (outs Tcl_Logical:$output); + let hasVerifier = 1; +} + +def Tcl_FilterCompactionOp : Tcl_Op<"filter_compaction", [Pure]> { + let arguments = (ins Tcl_Logical:$input, Tcl_Logical:$condition, + Tcl_Predicate:$predicate, AnyAttr:$threshold, + ArrayAttr:$axes, AnyAttr:$max_size); + let results = (outs Tcl_Logical:$output); + let hasVerifier = 1; +} + +def Tcl_WriteOp : Tcl_Op<"write", [Pure, Terminator]> { + let arguments = (ins Tcl_Logical:$input); + let hasVerifier = 1; +} + +#endif // MLIR_DIALECTS_TCL_TCL_OPS_TD diff --git a/mlir/dialects/tcl/tcl_types.cc b/mlir/dialects/tcl/tcl_types.cc new file mode 100644 index 00000000..8b7f919a --- /dev/null +++ b/mlir/dialects/tcl/tcl_types.cc @@ -0,0 +1,68 @@ +//===- tcl_types.cc - Furiosa TCL types -----------------------------------===// + +#include "mlir/dialects/tcl/tcl_types.h" + +#include "mlir/dialects/tcl/tcl_attrs.h" +#include "mlir/dialects/tcl/tcl_dialect.h" + +#include "llvm/ADT/STLExtras.h" +namespace xla::furiosa::tcl { +using namespace mlir; +using Error = llvm::function_ref; +llvm::StringRef ElementName(Type t) { + if (t.isBF16()) + return "bf16"; + if (t.isF16()) + return "f16"; + if (t.isF32()) + return "f32"; + if (t.isF64()) + return "f64"; + if (isa(t)) + return "f4_e2"; + if (isa(t)) + return "f8_e4"; + if (isa(t)) + return "f8_e5"; + if (auto i = dyn_cast(t)) { + if (i.getWidth() == 1 && i.isSignless()) + return "bool"; + if (i.isUnsigned()) { + switch (i.getWidth()) { + case 8: + return "u8"; + case 16: + return "u16"; + case 32: + return "u32"; + case 64: + return "u64"; + } + } else { + switch (i.getWidth()) { + case 4: + return "i4"; + case 8: + return "i8"; + case 16: + return "i16"; + case 32: + return "i32"; + case 64: + return "i64"; + } + } + } + return {}; +} +LogicalResult LogicalType::verify(Error e, Type element, ArrayAttr axes) { + if (ElementName(element).empty()) + return e() << "unsupported TCL element type"; + return VerifyAxes(e, axes); +} +LogicalResult MappedType::verify(Error e, Type element, + DramMappingAttr mapping) { + return ElementName(element).empty() ? e() << "unsupported TCL element type" + : success(); +} +} // namespace xla::furiosa::tcl diff --git a/mlir/dialects/tcl/tcl_types.h b/mlir/dialects/tcl/tcl_types.h new file mode 100644 index 00000000..b1fdd99d --- /dev/null +++ b/mlir/dialects/tcl/tcl_types.h @@ -0,0 +1,18 @@ +//===- tcl_types.h - Furiosa TCL types --------------------------*- C++ -*-===// + +#ifndef MLIR_DIALECTS_TCL_TCL_TYPES_H +#define MLIR_DIALECTS_TCL_TCL_TYPES_H + +#include "mlir/dialects/tcl/tcl_attrs.h" +#include "mlir/dialects/tcl/tcl_dialect.h" + +#include "mlir/IR/BuiltinTypes.h" +#include "llvm/ADT/StringRef.h" +#define GET_TYPEDEF_CLASSES +#include "mlir/dialects/tcl/tcl_types.h.inc" +namespace xla::furiosa::tcl { +// Empty name denotes a type outside the SDK's 17 element types. +llvm::StringRef ElementName(mlir::Type type); +} // namespace xla::furiosa::tcl + +#endif // MLIR_DIALECTS_TCL_TCL_TYPES_H diff --git a/mlir/dialects/tcl/tcl_types.td b/mlir/dialects/tcl/tcl_types.td new file mode 100644 index 00000000..1cbe5e7f --- /dev/null +++ b/mlir/dialects/tcl/tcl_types.td @@ -0,0 +1,23 @@ +//===- tcl_types.td - Furiosa TCL types --------------------*- tablegen -*-===// + +#ifndef MLIR_DIALECTS_TCL_TCL_TYPES_TD +#define MLIR_DIALECTS_TCL_TCL_TYPES_TD + +include "mlir/IR/AttrTypeBase.td" +include "mlir/dialects/tcl/tcl_dialect.td" + +def Tcl_Logical : TypeDef { + let mnemonic = "logical"; + let parameters = (ins "::mlir::Type":$elementType, "::mlir::ArrayAttr":$axes); + let assemblyFormat = "`<` $elementType `,` $axes `>`"; + let genVerifyDecl = 1; +} +def Tcl_Mapped : TypeDef { + let mnemonic = "mapped"; + let parameters = (ins "::mlir::Type":$elementType, + "::xla::furiosa::tcl::DramMappingAttr":$mapping); + let assemblyFormat = "`<` $elementType `,` $mapping `>`"; + let genVerifyDecl = 1; +} + +#endif // MLIR_DIALECTS_TCL_TCL_TYPES_TD diff --git a/mlir/dialects/tcl/test.zig b/mlir/dialects/tcl/test.zig new file mode 100644 index 00000000..5adc34c0 --- /dev/null +++ b/mlir/dialects/tcl/test.zig @@ -0,0 +1,134 @@ +const std = @import("std"); + +const mlir = @import("mlir"); +const func = @import("mlir/dialects/func"); +const tcl = @import("mlir/dialects/tcl"); + +const operation_names: []const []const u8 = &.{ + "tcl.dpe", + "tcl.filter_compaction", + "tcl.graph.all_gather", + "tcl.graph.arange", + "tcl.graph.as_dram", + "tcl.graph.as_logical", + "tcl.graph.concat", + "tcl.graph.copy", + "tcl.graph.for", + "tcl.graph.full", + "tcl.graph.gather", + "tcl.graph.index_read", + "tcl.graph.index_write", + "tcl.graph.reduce_max_i32", + "tcl.graph.reshape", + "tcl.graph.scatter", + "tcl.graph.scratchpad", + "tcl.graph.slice", + "tcl.graph.sym_expr", + "tcl.graph.transmute", + "tcl.graph.vector", + "tcl.graph.yield", + "tcl.kernel", + "tcl.read", + "tcl.ve", + "tcl.ve_reduce", + "tcl.ve_select", + "tcl.write", +}; + +test { + std.testing.refAllDecls(tcl); + std.testing.refAllDecls(tcl.LogicalType); +} + +fn testContext() !*mlir.Context { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + inline for (.{ func, tcl }) |dialect| dialect.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + return ctx; +} + +test "every bound operation is registered" { + const ctx = try testContext(); + defer ctx.deinit(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} + +test "types and attributes verify" { + const ctx = try testContext(); + defer ctx.deinit(); + + const a = try tcl.symbolAttr(ctx, "A"); + const t = try tcl.logicalType(ctx, .float(ctx, .bf16), &.{a}); + try std.testing.expect(t.isA(tcl.LogicalType) != null); + try std.testing.expectError(error.InvalidMlir, tcl.symbolAttr(ctx, "not an axis")); + try std.testing.expectError(error.InvalidMlir, tcl.logicalType(ctx, .float(ctx, .bf16), &.{ a, a })); + try std.testing.expectError(error.InvalidMlir, tcl.veOpcodeAttr(ctx, "unknown")); + _ = try tcl.veOpcodeAttr(ctx, "to_f16"); +} + +/// `tcl.kernel(input) { read; filter_compaction along `axis`; write }`. +fn compactionKernel(ctx: *mlir.Context, entry: *mlir.Block, axis: *const mlir.Attribute) !*mlir.Operation { + const loc: *const mlir.Location = .unknown(ctx); + const n = try tcl.symbolAttr(ctx, "N"); + const k = try tcl.symbolAttr(ctx, "K"); + const input = try tcl.logicalType(ctx, .int(ctx, .i32), &.{n}); + const output = try tcl.logicalType(ctx, .int(ctx, .i32), &.{k}); + const body = mlir.Block.init(&.{}, &.{}); + const x = tcl.read(ctx, &.{entry.argument(0)}, input, .{}, loc).appendTo(body); + const compacted = tcl.filter_compaction(ctx, x.result(0), x.result(0), output, .ge, .int(ctx, .i32, 0), &.{axis}, k, loc).appendTo(body); + _ = tcl.write(ctx, compacted.result(0), loc).appendTo(body); + return tcl.kernel(ctx, &.{entry.argument(0)}, output, .filter_compaction, body, .{}, loc).appendTo(entry); +} + +test "filter_compaction verifies inside a kernel" { + const ctx = try testContext(); + defer ctx.deinit(); + const loc: *const mlir.Location = .unknown(ctx); + const n = try tcl.symbolAttr(ctx, "N"); + const entry = mlir.Block.init(&.{try tcl.logicalType(ctx, .int(ctx, .i32), &.{n})}, &.{loc}); + // The kernel verifier walks to its inputs' regions. + const function = func.func(ctx, .{ .name = "compact", .block = entry, .results = &.{}, .location = loc, .verify = false }); + defer function.deinit(); + try std.testing.expect((try compactionKernel(ctx, entry, n)).verify()); + try std.testing.expect(!(try compactionKernel(ctx, entry, try tcl.symbolAttr(ctx, "Missing"))).verify()); +} + +test "function attributes carry their tcl names" { + const ctx = try testContext(); + defer ctx.deinit(); + const loc: *const mlir.Location = .unknown(ctx); + const entry = mlir.Block.init(&.{}, &.{}); + _ = func.returns(ctx, &.{}, loc).appendTo(entry); + const function = func.func(ctx, .{ .name = "f", .block = entry, .location = loc }); + defer function.deinit(); + const config = try tcl.configAttr(ctx, &.{.named(ctx, "lowering_mode", .string(ctx, "Heuristic"))}); + tcl.annotateFunction(ctx, function, &.{.named(ctx, "A", .int(ctx, .i64, 4))}, .{ + .updated_inputs = &.{-1}, + .compiler_config = config, + .auto_config = config, + .fusible = true, + }); + try std.testing.expect(function.verify()); + var text: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer text.deinit(); + try text.writer.print("{f}", .{function}); + try std.testing.expect(std.mem.indexOf(u8, text.written(), "attributes {tcl.auto_config = #tcl.config<{lowering_mode = \"Heuristic\"}>, tcl.axes = {A = 4 : i64}, tcl.compiler_config = #tcl.config<{lowering_mode = \"Heuristic\"}>, tcl.fusible, tcl.updated_inputs = array}") != null); +} + +test "attributes and types print and reparse" { + const ctx = try testContext(); + defer ctx.deinit(); + const a = try tcl.symbolAttr(ctx, "A"); + const t = try tcl.logicalType(ctx, .float(ctx, .bf16), &.{a}); + var type_text: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer type_text.deinit(); + try type_text.writer.print("{f}", .{t}); + try std.testing.expect((try mlir.Type.parse(ctx, type_text.written())).eql(t)); + var attr_text: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer attr_text.deinit(); + try attr_text.writer.print("{f}", .{a}); + try std.testing.expect((try mlir.Attribute.parse(ctx, attr_text.written())).eql(a)); +} diff --git a/mlir/dialects/tcl/ve_instructions.inc b/mlir/dialects/tcl/ve_instructions.inc new file mode 100644 index 00000000..657eddca --- /dev/null +++ b/mlir/dialects/tcl/ve_instructions.inc @@ -0,0 +1,48 @@ +//===- ve_instructions.inc - Furiosa TCL VE instructions ------------------===// +// +// SDK 2026.3 VE opcode vocabulary; conversion widths 0..31 are parametric. + +VE("exp", "exp", 1) +VE("neg_exp", "neg_exp", 1) +VE("sqrt", "sqrt", 1) +VE("tanh", "tanh", 1) +VE("sigmoid", "sigmoid", 1) +VE("erf", "erf", 1) +VE("log", "log", 1) +VE("sin", "sin", 1) +VE("cos", "cos", 1) +VE("negf", "neg_f", 1) +VE("absf", "abs_f", 1) +VE("addi", "+_fxp", 2) +VE("addi_sat", "+_fxp_sat", 2) +VE("subi", "-_fxp", 2) +VE("subi_sat", "-_fxp_sat", 2) +VE("mul_q31", "*_fxp", 2) +VE("muli", "*_i", 2) +VE("shl", "<<", 2) +VE("shl_sat", "<<_sat", 2) +VE("shr_logical", ">>>", 2) +VE("shr_arith", ">>", 2) +VE("shr_arith_round", ">>_round", 2) +VE("bit_andi", "&_i", 2) +VE("bit_ori", "|_i", 2) +VE("bit_xori", "^_i", 2) +VE("mini", "min_i", 2) +VE("maxi", "max_i", 2) +VE("absmini", "|min|_i", 2) +VE("absmaxi", "|max|_i", 2) +VE("addf", "+", 2) +VE("subf", "-", 2) +VE("mulf", "*", 2) +VE("divf", "/", 2) +VE("bit_andf", "&_f", 2) +VE("bit_orf", "|_f", 2) +VE("bit_xorf", "^_f", 2) +VE("minf", "min_f", 2) +VE("maxf", "max_f", 2) +VE("absminf", "|min|_f", 2) +VE("absmaxf", "|max|_f", 2) +VE("reinterpret_as_float", "as_f", 1) +VE("reinterpret_as_int", "as_i", 1) +VE("to_float", "to_f31", 1) +VE("to_int", "to_i31", 1) diff --git a/mlir/dialects/tensor.zig b/mlir/dialects/tensor.zig deleted file mode 100644 index 8d7de344..00000000 --- a/mlir/dialects/tensor.zig +++ /dev/null @@ -1,36 +0,0 @@ -const std = @import("std"); -const mlir = @import("mlir"); - -pub fn empty(ctx: mlir.Context, args: struct { - result: mlir.Type, - location: mlir.Location, -}) mlir.Operation { - return mlir.Operation.make(ctx, "tensor.empty", .{ - .results = &.{args.result}, - .location = args.location, - }); -} - -pub fn splat(ctx: mlir.Context, args: struct { - value: mlir.Value, - result: mlir.Type, - location: mlir.Location, -}) mlir.Operation { - return mlir.Operation.make(ctx, "tensor.splat", .{ - .operands = &.{args.value}, - .results = &.{args.result}, - .location = args.location, - }); -} - -pub fn cast(ctx: mlir.Context, args: struct { - source: mlir.Value, - dest: mlir.Type, - location: mlir.Location, -}) mlir.Operation { - return mlir.Operation.make(ctx, "tensor.cast", .{ - .operands = &.{args.source}, - .results = &.{args.dest}, - .location = args.location, - }); -} diff --git a/mlir/dialects/ttir/BUILD.bazel b/mlir/dialects/ttir/BUILD.bazel new file mode 100644 index 00000000..8ab5598f --- /dev/null +++ b/mlir/dialects/ttir/BUILD.bazel @@ -0,0 +1,39 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +cc_library( + name = "capi", + srcs = ["capi.cc"], + hdrs = ["capi.h"], + tags = ["manual"], + deps = [ + "@llvm-project//mlir:CAPIIR", + "@triton//:TritonDialects", + ], +) + +zig_library( + name = "ttir", + import_name = "mlir/dialects/ttir", + main = "ttir.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + ":capi", + "//mlir", + "//stdx", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":ttir", + "//mlir", + "//mlir/dialects/arith", + "//mlir/dialects/func", + "//mlir/dialects/math", + "//mlir/dialects/scf", + ], +) diff --git a/mlir/dialects/ttir/capi.cc b/mlir/dialects/ttir/capi.cc new file mode 100644 index 00000000..0e7afb28 --- /dev/null +++ b/mlir/dialects/ttir/capi.cc @@ -0,0 +1,167 @@ +//===- capi.cc - C API for the Triton tt dialect --------------------------===// + +#include "capi.h" + +#include "triton/Dialect/Triton/IR/Dialect.h" +#include "triton/Dialect/Triton/IR/Types.h" + +#include "mlir/CAPI/IR.h" +#include "mlir/CAPI/Registration.h" + +// Note: the second arg `tt` controls the generated symbol name +// `mlirGetDialectHandle__tt__`, which `mlir.DialectHandle.fromString("tt")` on +// the Zig side resolves at comptime (see `DialectHandle.fromString` in +// mlir/mlir.zig). Must match the dialect's declared name (`let name = "tt"` in +// TritonDialect.td). +MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(Triton, tt, mlir::triton::TritonDialect) + +static mlir::MLIRContext *loadTriton(MlirContext context) { + mlir::MLIRContext *ctx = unwrap(context); + if (ctx) + ctx->getOrLoadDialect(); + return ctx; +} + +using mlir::cast; +using mlir::isa; + +MlirType mlirTritonPointerTypeGet(MlirType pointee, int32_t addressSpace) { + return wrap(mlir::triton::PointerType::get( + unwrap(pointee), static_cast(addressSpace))); +} + +MlirType mlirTritonPointerTypeGetPointee(MlirType type) { + return wrap(cast(unwrap(type)).getPointeeType()); +} + +int32_t mlirTritonPointerTypeGetAddressSpace(MlirType type) { + return static_cast( + cast(unwrap(type)).getAddressSpace()); +} + +bool mlirTypeIsATritonPointer(MlirType type) { + return isa(unwrap(type)); +} + +MlirType mlirTritonTensorDescTypeGet(intptr_t rank, const int64_t *shape, + MlirType elementType, + MlirAttribute sharedLayout) { + mlir::Attribute layoutAttr; + if (!mlirAttributeIsNull(sharedLayout)) { + layoutAttr = unwrap(sharedLayout); + } + // TensorDescType::get is a TypeBuilderWithInferredContext: the context is + // inferred from elementType. Signature: (shape, elementType, sharedLayout). + return wrap(mlir::triton::TensorDescType::get( + llvm::ArrayRef(shape, rank), unwrap(elementType), layoutAttr)); +} + +bool mlirTypeIsATritonTensorDesc(MlirType type) { + return isa(unwrap(type)); +} + +MlirAttribute mlirTritonProgramIDDimAttrGet(MlirContext context, + int32_t value) { + mlir::MLIRContext *ctx = loadTriton(context); + if (!ctx) + return {nullptr}; + return wrap(mlir::triton::ProgramIDDimAttr::get( + ctx, static_cast(value))); +} + +MlirAttribute mlirTritonCacheModifierAttrGet(MlirContext context, + int32_t value) { + mlir::MLIRContext *ctx = loadTriton(context); + if (!ctx) + return {nullptr}; + return wrap(mlir::triton::CacheModifierAttr::get( + ctx, static_cast(value))); +} + +MlirAttribute mlirTritonEvictionPolicyAttrGet(MlirContext context, + int32_t value) { + mlir::MLIRContext *ctx = loadTriton(context); + if (!ctx) + return {nullptr}; + return wrap(mlir::triton::EvictionPolicyAttr::get( + ctx, static_cast(value))); +} + +MlirAttribute mlirTritonPaddingOptionAttrGet(MlirContext context, + int32_t value) { + mlir::MLIRContext *ctx = loadTriton(context); + if (!ctx) + return {nullptr}; + return wrap(mlir::triton::PaddingOptionAttr::get( + ctx, static_cast(value))); +} + +MlirAttribute mlirTritonRoundingModeAttrGet(MlirContext context, + int32_t value) { + mlir::MLIRContext *ctx = loadTriton(context); + if (!ctx) + return {nullptr}; + return wrap(mlir::triton::RoundingModeAttr::get( + ctx, static_cast(value))); +} + +MlirAttribute mlirTritonInputPrecisionAttrGet(MlirContext context, + int32_t value) { + mlir::MLIRContext *ctx = loadTriton(context); + if (!ctx) + return {nullptr}; + return wrap(mlir::triton::InputPrecisionAttr::get( + ctx, static_cast(value))); +} + +MlirAttribute mlirTritonRMWOpAttrGet(MlirContext context, int32_t value) { + mlir::MLIRContext *ctx = loadTriton(context); + if (!ctx) + return {nullptr}; + return wrap(mlir::triton::RMWOpAttr::get( + ctx, static_cast(value))); +} + +MlirAttribute mlirTritonMemSemanticAttrGet(MlirContext context, int32_t value) { + mlir::MLIRContext *ctx = loadTriton(context); + if (!ctx) + return {nullptr}; + return wrap(mlir::triton::MemSemanticAttr::get( + ctx, static_cast(value))); +} + +MlirAttribute mlirTritonMemSyncScopeAttrGet(MlirContext context, + int32_t value) { + mlir::MLIRContext *ctx = loadTriton(context); + if (!ctx) + return {nullptr}; + return wrap(mlir::triton::MemSyncScopeAttr::get( + ctx, static_cast(value))); +} + +MlirAttribute mlirTritonPropagateNanAttrGet(MlirContext context, + int32_t value) { + mlir::MLIRContext *ctx = loadTriton(context); + if (!ctx) + return {nullptr}; + return wrap(mlir::triton::PropagateNanAttr::get( + ctx, static_cast(value))); +} + +MlirAttribute mlirTritonScaleDotElemTypeAttrGet(MlirContext context, + int32_t value) { + mlir::MLIRContext *ctx = loadTriton(context); + if (!ctx) + return {nullptr}; + return wrap(mlir::triton::ScaleDotElemTypeAttr::get( + ctx, static_cast(value))); +} + +MlirAttribute mlirTritonDescriptorReduceKindAttrGet(MlirContext context, + int32_t value) { + mlir::MLIRContext *ctx = loadTriton(context); + if (!ctx) + return {nullptr}; + return wrap(mlir::triton::DescriptorReduceKindAttr::get( + ctx, static_cast(value))); +} diff --git a/mlir/dialects/ttir/capi.h b/mlir/dialects/ttir/capi.h new file mode 100644 index 00000000..efd0aae9 --- /dev/null +++ b/mlir/dialects/ttir/capi.h @@ -0,0 +1,59 @@ +//===- capi.h - C API for the Triton tt dialect -------------------*- C -*-===// + +#ifndef MLIR_DIALECTS_TTIR_CAPI_H +#define MLIR_DIALECTS_TTIR_CAPI_H + +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(Triton, tt); + +// !tt.ptr +MLIR_CAPI_EXPORTED MlirType mlirTritonPointerTypeGet(MlirType pointee, + int32_t addressSpace); +MLIR_CAPI_EXPORTED MlirType mlirTritonPointerTypeGetPointee(MlirType type); +MLIR_CAPI_EXPORTED int32_t mlirTritonPointerTypeGetAddressSpace(MlirType type); +MLIR_CAPI_EXPORTED bool mlirTypeIsATritonPointer(MlirType type); + +// !tt.tensordesc> — result of tt.make_tensor_descriptor. +// `shape` is a `rank`-element array of int64_t block dims; `elementType` is +// the scalar element type; `sharedLayout` may be a null MlirAttribute for none. +MLIR_CAPI_EXPORTED MlirType +mlirTritonTensorDescTypeGet(intptr_t rank, const int64_t *shape, + MlirType elementType, MlirAttribute sharedLayout); +MLIR_CAPI_EXPORTED bool mlirTypeIsATritonTensorDesc(MlirType type); + +// I32EnumAttr constructors +MLIR_CAPI_EXPORTED MlirAttribute +mlirTritonProgramIDDimAttrGet(MlirContext context, int32_t value); +MLIR_CAPI_EXPORTED MlirAttribute +mlirTritonCacheModifierAttrGet(MlirContext context, int32_t value); +MLIR_CAPI_EXPORTED MlirAttribute +mlirTritonEvictionPolicyAttrGet(MlirContext context, int32_t value); +MLIR_CAPI_EXPORTED MlirAttribute +mlirTritonPaddingOptionAttrGet(MlirContext context, int32_t value); +MLIR_CAPI_EXPORTED MlirAttribute +mlirTritonRoundingModeAttrGet(MlirContext context, int32_t value); +MLIR_CAPI_EXPORTED MlirAttribute +mlirTritonInputPrecisionAttrGet(MlirContext context, int32_t value); +MLIR_CAPI_EXPORTED MlirAttribute mlirTritonRMWOpAttrGet(MlirContext context, + int32_t value); +MLIR_CAPI_EXPORTED MlirAttribute +mlirTritonMemSemanticAttrGet(MlirContext context, int32_t value); +MLIR_CAPI_EXPORTED MlirAttribute +mlirTritonMemSyncScopeAttrGet(MlirContext context, int32_t value); +MLIR_CAPI_EXPORTED MlirAttribute +mlirTritonPropagateNanAttrGet(MlirContext context, int32_t value); +MLIR_CAPI_EXPORTED MlirAttribute +mlirTritonScaleDotElemTypeAttrGet(MlirContext context, int32_t value); +MLIR_CAPI_EXPORTED MlirAttribute +mlirTritonDescriptorReduceKindAttrGet(MlirContext context, int32_t value); + +#ifdef __cplusplus +} +#endif + +#endif // MLIR_DIALECTS_TTIR_CAPI_H diff --git a/mlir/dialects/ttir/test.zig b/mlir/dialects/ttir/test.zig new file mode 100644 index 00000000..04361e4e --- /dev/null +++ b/mlir/dialects/ttir/test.zig @@ -0,0 +1,172 @@ +const std = @import("std"); + +const arith = @import("mlir/dialects/arith"); +const func = @import("mlir/dialects/func"); +const math = @import("mlir/dialects/math"); +const mlir = @import("mlir"); +const scf = @import("mlir/dialects/scf"); +const ttir = @import("mlir/dialects/ttir"); + +const operation_names: []const []const u8 = &.{ + "tt.addptr", + "tt.assert", + "tt.atomic_cas", + "tt.atomic_rmw", + "tt.bitcast", + "tt.broadcast", + "tt.call", + "tt.clampf", + "tt.descriptor_gather", + "tt.descriptor_load", + "tt.descriptor_reduce", + "tt.descriptor_scatter", + "tt.descriptor_store", + "tt.dot", + "tt.dot_scaled", + "tt.elementwise_inline_asm", + "tt.expand_dims", + "tt.extern_elementwise", + "tt.fp_to_fp", + "tt.func", + "tt.gather", + "tt.get_num_programs", + "tt.get_program_id", + "tt.histogram", + "tt.int_to_ptr", + "tt.join", + "tt.load", + "tt.make_range", + "tt.make_tensor_descriptor", + "tt.map_elementwise", + "tt.map_elementwise.return", + "tt.mulhiui", + "tt.precise_divf", + "tt.precise_sqrt", + "tt.print", + "tt.ptr_to_int", + "tt.reduce", + "tt.reduce.return", + "tt.reshape", + "tt.return", + "tt.scan", + "tt.scan.return", + "tt.splat", + "tt.split", + "tt.store", + "tt.trans", + "tt.unsplat", +}; + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(ttir); + inline for (comptime std.meta.declarations(ttir)) |decl| { + const T = @field(ttir, decl); + if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); + } +} + +test "tt.func round-trip" { + // Build: module { tt.func public @id(%arg0: !tt.ptr) { tt.return } } + // by constructing it with the Zig wrappers, printing to string, and re-parsing. + + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + func.registerDialect(registry); + arith.registerDialect(registry); + scf.registerDialect(registry); + math.registerDialect(registry); + ttir.registerDialect(registry); + + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + + try std.testing.expect(ctx.isRegisteredOperation("tt.func")); + try std.testing.expect(ctx.isRegisteredOperation("tt.return")); + try std.testing.expect(ctx.isRegisteredOperation("tt.load")); + try std.testing.expect(ctx.isRegisteredOperation("tt.store")); + try std.testing.expect(ctx.isRegisteredOperation("tt.get_program_id")); + try std.testing.expect(ctx.isRegisteredOperation("tt.splat")); + try std.testing.expect(ctx.isRegisteredOperation("tt.addptr")); + try std.testing.expect(ctx.isRegisteredOperation("scf.for")); + try std.testing.expect(ctx.isRegisteredOperation("arith.constant")); + + const loc: *const mlir.Location = .unknown(ctx); + const module: *mlir.Module = .init(loc); + defer module.deinit(); + + const f32_ty = mlir.Type.float(ctx, .f32); + const ptr_ty = ttir.pointerType(f32_ty, 1); + + const entry = mlir.Block.init(&.{ ptr_ty, ptr_ty }, &.{ loc, loc }); + + const arg0 = entry.argument(0); + const arg1 = entry.argument(1); + + // %0 = tt.load %arg0 : !tt.ptr + const loaded = ttir.load(ctx, arg0, f32_ty, .none, .normal, false, .{}, loc); + _ = loaded.appendTo(entry); + + // %1 = arith.constant 1.0 : f32 — built with Operation.make to avoid cross-dialect import + const one_attr: *const mlir.Attribute = .float(ctx, .f32, 1.0); + const one = mlir.Operation.make(ctx, "arith.constant", .{ + .results = .{ .flat = &.{f32_ty} }, + .attributes = &.{.named(ctx, "value", one_attr)}, + .location = loc, + }); + _ = one.appendTo(entry); + + // %2 = arith.addf %0, %1 : f32 + const summed = mlir.Operation.make(ctx, "arith.addf", .{ + .operands = .{ .flat = &.{ loaded.result(0), one.result(0) } }, + .result_type_inference = true, + .location = loc, + }); + _ = summed.appendTo(entry); + + // tt.store %arg1, %2 + _ = ttir.store(ctx, arg1, summed.result(0), .none, .normal, .{}, loc).appendTo(entry); + + // tt.return + _ = ttir.return_(ctx, &.{}, loc).appendTo(entry); + + const f = ttir.func(ctx, .{ + .name = "add_one", + .block = entry, + .location = loc, + }); + _ = f.appendTo(module.body()); + + // Verify the operation. + try std.testing.expect(module.operation().verify()); + + // Print IR and re-parse it to confirm it's valid text. + var al: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer al.deinit(); + try al.writer.print("{f}", .{module.operation()}); + + const parsed = try mlir.Module.parse(ctx, al.written()); + defer parsed.deinit(); + try std.testing.expect(parsed.operation().verify()); +} + +test "divisibility is an i32 attribute of the tt dialect" { + const ctx = try mlir.Context.init(.{}); + defer ctx.deinit(); + var buf: [64]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print("{f}", .{mlir.Attribute.dict(ctx, &.{ttir.divisibility(ctx, 16)})}); + try std.testing.expectEqualStrings("{tt.divisibility = 16 : i32}", w.buffered()); +} + +test "every bound operation is registered" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + ttir.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} diff --git a/mlir/dialects/ttir/ttir.zig b/mlir/dialects/ttir/ttir.zig new file mode 100644 index 00000000..ee7b5307 --- /dev/null +++ b/mlir/dialects/ttir/ttir.zig @@ -0,0 +1,1231 @@ +const std = @import("std"); + +const c = @import("c"); +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("tt"); +} + +// ============================================================================= +// Types — !tt.ptr +// ============================================================================= + +pub const PointerType = opaque { + const M = mlir.Methods(PointerType, c.MlirType); + + pub const isAFn = c.mlirTypeIsATritonPointer; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + pointee_type: *const mlir.Type, + address_space: i32, + }; + + pub fn get(args: InitArgs) *const PointerType { + return @ptrCast(c.mlirTritonPointerTypeGet(args.pointee_type.ptr(), args.address_space).ptr); + } + + pub fn pointee(self: *const PointerType) *const mlir.Type { + return @ptrCast(c.mlirTritonPointerTypeGetPointee(self.ptr()).ptr); + } + + pub fn addressSpace(self: *const PointerType) i32 { + return c.mlirTritonPointerTypeGetAddressSpace(self.ptr()); + } + + pub fn type_(self: *const PointerType) *const mlir.Type { + return @ptrCast(self); + } +}; + +/// Convenience alias: `!tt.ptr` as a plain *const mlir.Type. +pub fn pointerType(pointee_type: *const mlir.Type, address_space: i32) *const mlir.Type { + return @ptrCast(PointerType.get(.{ .pointee_type = pointee_type, .address_space = address_space })); +} + +// ----------------------------------------------------------------------------- +// !tt.tensordesc> — TMA descriptor type. +// ----------------------------------------------------------------------------- + +pub const TensorDescType = opaque { + const M = mlir.Methods(TensorDescType, c.MlirType); + + pub const isAFn = c.mlirTypeIsATritonTensorDesc; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + + pub const InitArgs = struct { + shape: []const i64, + element_type: *const mlir.Type, + /// May be null for an unlayout'd descriptor. + shared_layout: ?*const mlir.Attribute = null, + }; + + pub fn get(args: InitArgs) *const TensorDescType { + const attr_ptr = if (args.shared_layout) |a| a.ptr() else c.MlirAttribute{ .ptr = null }; + return @ptrCast(c.mlirTritonTensorDescTypeGet( + @intCast(args.shape.len), + args.shape.ptr, + args.element_type.ptr(), + attr_ptr, + ).ptr); + } + + pub fn type_(self: *const TensorDescType) *const mlir.Type { + return @ptrCast(self); + } +}; + +pub fn tensorDescType( + shape: []const i64, + element_type: *const mlir.Type, + opts: struct { + shared_layout: ?*const mlir.Attribute = null, + }, +) *const mlir.Type { + return @ptrCast(TensorDescType.get(.{ .shape = shape, .element_type = element_type, .shared_layout = opts.shared_layout })); +} + +// ============================================================================= +// Enum attributes +// ============================================================================= + +pub const ProgramIDDim = enum(i32) { x = 0, y = 1, z = 2 }; +pub const CacheModifier = enum(i32) { none = 1, ca = 2, cg = 3, wb = 4, cs = 5, wt = 6, cv = 7 }; +pub const EvictionPolicy = enum(i32) { normal = 1, evict_first = 2, evict_last = 3 }; +pub const PaddingOption = enum(i32) { pad_zero = 1, pad_nan = 2 }; +pub const RoundingMode = enum(i32) { rtz = 0, rtne = 1 }; +pub const InputPrecision = enum(i32) { tf32 = 0, tf32x3 = 1, ieee = 2, bf16x3 = 3, bf16x6 = 4 }; + +/// Atomic read-modify-write operations. +pub const RMWOp = enum(i32) { + @"and" = 1, + @"or" = 2, + xor = 3, + add = 4, + fadd = 5, + max = 6, + min = 7, + umax = 8, + umin = 9, + xchg = 10, +}; + +/// Memory ordering semantics. +pub const MemSemantic = enum(i32) { + relaxed = 1, + acquire = 2, + release = 3, + acq_rel = 4, +}; + +/// Memory sync scope. +pub const MemSyncScope = enum(i32) { + gpu = 1, + cta = 2, + sys = 3, +}; + +/// NaN propagation for clampf. 0xFFFF is the exact .td encoding. +pub const PropagateNan = enum(i32) { + none = 0, + all = 0xFFFF, +}; + +/// Packed FP element types for tt.dot_scaled. +pub const ScaleDotElemType = enum(i32) { + e4m3 = 0, + e5m2 = 1, + e2m3 = 2, + e3m2 = 3, + e2m1 = 4, + bf16 = 5, + fp16 = 6, +}; + +/// Reduction kind for tt.descriptor_reduce. +pub const DescriptorReduceKind = enum(i32) { + add = 1, + min = 2, + max = 3, + inc = 4, + dec = 5, + @"and" = 6, + @"or" = 7, + xor = 8, +}; + +pub fn programDimAttr(ctx: *mlir.Context, v: ProgramIDDim) *const mlir.Attribute { + return @ptrCast(c.mlirTritonProgramIDDimAttrGet(ctx.ptr(), @backingInt(v)).ptr); +} + +pub fn cacheModifierAttr(ctx: *mlir.Context, v: CacheModifier) *const mlir.Attribute { + return @ptrCast(c.mlirTritonCacheModifierAttrGet(ctx.ptr(), @backingInt(v)).ptr); +} + +pub fn evictionPolicyAttr(ctx: *mlir.Context, v: EvictionPolicy) *const mlir.Attribute { + return @ptrCast(c.mlirTritonEvictionPolicyAttrGet(ctx.ptr(), @backingInt(v)).ptr); +} + +pub fn paddingOptionAttr(ctx: *mlir.Context, v: PaddingOption) *const mlir.Attribute { + return @ptrCast(c.mlirTritonPaddingOptionAttrGet(ctx.ptr(), @backingInt(v)).ptr); +} + +pub fn roundingModeAttr(ctx: *mlir.Context, v: RoundingMode) *const mlir.Attribute { + return @ptrCast(c.mlirTritonRoundingModeAttrGet(ctx.ptr(), @backingInt(v)).ptr); +} + +pub fn inputPrecisionAttr(ctx: *mlir.Context, v: InputPrecision) *const mlir.Attribute { + return @ptrCast(c.mlirTritonInputPrecisionAttrGet(ctx.ptr(), @backingInt(v)).ptr); +} + +pub fn rmwOpAttr(ctx: *mlir.Context, v: RMWOp) *const mlir.Attribute { + return @ptrCast(c.mlirTritonRMWOpAttrGet(ctx.ptr(), @backingInt(v)).ptr); +} + +pub fn memSemanticAttr(ctx: *mlir.Context, v: MemSemantic) *const mlir.Attribute { + return @ptrCast(c.mlirTritonMemSemanticAttrGet(ctx.ptr(), @backingInt(v)).ptr); +} + +pub fn memSyncScopeAttr(ctx: *mlir.Context, v: MemSyncScope) *const mlir.Attribute { + return @ptrCast(c.mlirTritonMemSyncScopeAttrGet(ctx.ptr(), @backingInt(v)).ptr); +} + +pub fn propagateNanAttr(ctx: *mlir.Context, v: PropagateNan) *const mlir.Attribute { + return @ptrCast(c.mlirTritonPropagateNanAttrGet(ctx.ptr(), @backingInt(v)).ptr); +} + +pub fn scaleDotElemTypeAttr(ctx: *mlir.Context, v: ScaleDotElemType) *const mlir.Attribute { + return @ptrCast(c.mlirTritonScaleDotElemTypeAttrGet(ctx.ptr(), @backingInt(v)).ptr); +} + +pub fn descriptorReduceKindAttr(ctx: *mlir.Context, v: DescriptorReduceKind) *const mlir.Attribute { + return @ptrCast(c.mlirTritonDescriptorReduceKindAttrGet(ctx.ptr(), @backingInt(v)).ptr); +} + +/// `tt.divisibility`: an argument attribute promising that the argument (the +/// address, for a pointer) is a multiple of `divisor`. +pub fn divisibility(ctx: *mlir.Context, divisor: u32) mlir.NamedAttribute { + return .named(ctx, "tt.divisibility", .int(ctx, .i32, divisor)); +} + +// ============================================================================= +// Function ops — tt.func / tt.return +// ============================================================================= + +/// Args for building a `tt.func` op. +pub const FuncOpArgs = struct { + pub const Visibility = enum { public, private }; + + name: []const u8, + args: ?[]const *const mlir.Type = null, + args_attributes: ?[]const *const mlir.Attribute = null, + results: ?[]const *const mlir.Type = null, + results_attributes: ?[]const *const mlir.Attribute = null, + block: *mlir.Block, + location: *const mlir.Location, + no_inline: bool = false, + visibility: Visibility = .public, + /// `tt.func` has `HasParent<"ModuleOp">`, so verification before appending + /// to a module always fails. Defer to the module-level verify instead. + verify: bool = false, +}; + +pub fn func(ctx: *mlir.Context, args: FuncOpArgs) *mlir.Operation { + var args_buffer: stdx.BoundedArray(*const mlir.Type, 1024) = .empty; + var results_buffer: stdx.BoundedArray(*const mlir.Type, 32) = .empty; + + var attr_tuples_buffer: stdx.BoundedArray(mlir.NamedAttribute, 16) = .empty; + attr_tuples_buffer.appendSliceAssumeCapacity(&.{ + .named(ctx, "sym_name", .string(ctx, args.name)), + .named(ctx, "sym_visibility", .string(ctx, @tagName(args.visibility))), + .named(ctx, "function_type", .typeAttr(.function( + ctx, + args.args orelse args_: { + for (0..args.block.numArguments()) |i| { + args_buffer.appendAssumeCapacity(args.block.argument(i).type_()); + } + break :args_ args_buffer.constSlice(); + }, + args.results orelse results_: { + const terminator = args.block.terminator() orelse { + @panic("block has no terminator"); + }; + for (0..terminator.numOperands()) |i| { + results_buffer.appendAssumeCapacity(terminator.operand(i).type_()); + } + break :results_ results_buffer.constSlice(); + }, + ))), + }); + + if (args.args_attributes) |args_attributes| { + attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "arg_attrs", .array(ctx, args_attributes))); + } + if (args.results_attributes) |results_attributes| { + attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "res_attrs", .array(ctx, results_attributes))); + } + // Always emit noinline even when false — matches Triton's IR convention. + attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "noinline", .boolean(ctx, args.no_inline))); + + return .make(ctx, "tt.func", .{ + .blocks = &.{args.block}, + .attributes = attr_tuples_buffer.constSlice(), + .location = args.location, + .verify = args.verify, + }); +} + +pub fn return_(ctx: *mlir.Context, values: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return .make(ctx, "tt.return", .{ + .operands = .{ .flat = values }, + .verify = false, + .location = location, + }); +} + +// ============================================================================= +// SPMD ops +// ============================================================================= + +pub fn get_program_id(ctx: *mlir.Context, axis: ProgramIDDim, location: *const mlir.Location) *mlir.Operation { + return .make(ctx, "tt.get_program_id", .{ + .results = .{ .flat = &.{.int(ctx, .i32)} }, + .attributes = &.{ + .named(ctx, "axis", programDimAttr(ctx, axis)), + }, + .location = location, + }); +} + +pub fn get_num_programs(ctx: *mlir.Context, axis: ProgramIDDim, location: *const mlir.Location) *mlir.Operation { + return .make(ctx, "tt.get_num_programs", .{ + .results = .{ .flat = &.{.int(ctx, .i32)} }, + .attributes = &.{ + .named(ctx, "axis", programDimAttr(ctx, axis)), + }, + .location = location, + }); +} + +// ============================================================================= +// Range +// ============================================================================= + +pub fn make_range( + ctx: *mlir.Context, + start: i32, + end: i32, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return .make(ctx, "tt.make_range", .{ + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "start", .int(ctx, .i32, start)), + .named(ctx, "end", .int(ctx, .i32, end)), + }, + .location = location, + }); +} + +// ============================================================================= +// Shape manipulation +// ============================================================================= + +pub fn splat( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return .make(ctx, "tt.splat", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +pub fn expand_dims( + ctx: *mlir.Context, + src: *const mlir.Value, + axis: i32, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return .make(ctx, "tt.expand_dims", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "axis", .int(ctx, .i32, axis)), + }, + .location = location, + }); +} + +pub fn broadcast( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return .make(ctx, "tt.broadcast", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +pub fn reshape( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + allow_reorder: bool, + efficient_layout: bool, + location: *const mlir.Location, +) *mlir.Operation { + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + if (allow_reorder) { + attrs.appendAssumeCapacity(.named(ctx, "allow_reorder", .unit(ctx))); + } + if (efficient_layout) { + attrs.appendAssumeCapacity(.named(ctx, "efficient_layout", .unit(ctx))); + } + return .make(ctx, "tt.reshape", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +pub fn trans( + ctx: *mlir.Context, + src: *const mlir.Value, + order: []const i32, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return .make(ctx, "tt.trans", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "order", .denseArray(ctx, .i32, order)), + }, + .location = location, + }); +} + +// ============================================================================= +// Pointer arithmetic & memory +// ============================================================================= + +/// tt.addptr — result type matches ptr type (TypesMatchWith in TritonOps.td). +pub fn addptr( + ctx: *mlir.Context, + ptr_val: *const mlir.Value, + offset: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return .make(ctx, "tt.addptr", .{ + .operands = .{ .flat = &.{ ptr_val, offset } }, + .results = .{ .flat = &.{ptr_val.type_()} }, + .location = location, + }); +} + +/// tt.load — AttrSizedOperandSegments: (ptr, [mask], [other]). Uses flat operand list + +/// explicit operandSegmentSizes because Operation.make variadic coercion zeros mixed-length segments. +pub fn load( + ctx: *mlir.Context, + ptr_val: *const mlir.Value, + result_type: *const mlir.Type, + cache: CacheModifier, + evict: EvictionPolicy, + is_volatile: bool, + opts: struct { + mask: ?*const mlir.Value = null, + other: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 3) = .empty; + operands_buf.appendAssumeCapacity(ptr_val); + const mask_len: i32 = if (opts.mask) |m| blk: { + operands_buf.appendAssumeCapacity(m); + break :blk 1; + } else 0; + const other_len: i32 = if (opts.other) |o| blk: { + operands_buf.appendAssumeCapacity(o); + break :blk 1; + } else 0; + + const seg_sizes = [3]i32{ 1, mask_len, other_len }; + return .make(ctx, "tt.load", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), + .named(ctx, "cache", cacheModifierAttr(ctx, cache)), + .named(ctx, "evict", evictionPolicyAttr(ctx, evict)), + .named(ctx, "isVolatile", .boolean(ctx, is_volatile)), + }, + .location = location, + }); +} + +/// tt.store — operands: (ptr, value, [mask]). Flat operand list; mask is trailing variadic. +pub fn store( + ctx: *mlir.Context, + ptr_val: *const mlir.Value, + value: *const mlir.Value, + cache: CacheModifier, + evict: EvictionPolicy, + opts: struct { + mask: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var buf: stdx.BoundedArray(*const mlir.Value, 3) = .empty; + buf.appendSliceAssumeCapacity(&.{ ptr_val, value }); + if (opts.mask) |m| buf.appendAssumeCapacity(m); + return .make(ctx, "tt.store", .{ + .operands = .{ .flat = buf.constSlice() }, + .attributes = &.{ + .named(ctx, "cache", cacheModifierAttr(ctx, cache)), + .named(ctx, "evict", evictionPolicyAttr(ctx, evict)), + }, + .location = location, + }); +} + +// ============================================================================= +// Compute +// ============================================================================= + +pub fn dot( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c_acc: *const mlir.Value, + result_type: *const mlir.Type, + input_precision_: InputPrecision, + max_num_imprecise_acc: i32, + location: *const mlir.Location, +) *mlir.Operation { + return .make(ctx, "tt.dot", .{ + .operands = .{ .flat = &.{ a, b, c_acc } }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "inputPrecision", inputPrecisionAttr(ctx, input_precision_)), + .named(ctx, "maxNumImpreciseAcc", .int(ctx, .i32, max_num_imprecise_acc)), + }, + .location = location, + }); +} + +/// tt.reduce — combine_block must have args [elem, elem] matching srcs' element types, +/// terminated with tt.reduce.return. +pub fn reduce( + ctx: *mlir.Context, + srcs: []const *const mlir.Value, + axis: i32, + combine_block: *mlir.Block, + result_types: []const *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + // Single variadic operand group — flat list, no operandSegmentSizes needed. + return .make(ctx, "tt.reduce", .{ + .operands = .{ .flat = srcs }, + .results = .{ .flat = result_types }, + .blocks = &.{combine_block}, + .attributes = &.{ + .named(ctx, "axis", .int(ctx, .i32, axis)), + }, + .location = location, + }); +} + +pub fn reduce_return( + ctx: *mlir.Context, + values: []const *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return .make(ctx, "tt.reduce.return", .{ + .operands = .{ .flat = values }, + .verify = false, + .location = location, + }); +} + +// ============================================================================= +// Elementwise casts specific to Triton (arith.bitcast doesn't support ptrs) +// ============================================================================= + +/// tt.int_to_ptr — cast int-like to ptr-like (element-wise, same shape). +pub fn int_to_ptr( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return .make(ctx, "tt.int_to_ptr", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// tt.ptr_to_int — cast ptr-like to int-like (element-wise, same shape). +pub fn ptr_to_int( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return .make(ctx, "tt.ptr_to_int", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// tt.bitcast — cast between types of the same bit-width (also works on ptrs, +/// unlike arith.bitcast). +pub fn bitcast( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return .make(ctx, "tt.bitcast", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// tt.fp_to_fp — float to float cast, optionally with a rounding mode. +/// Pass `rounding = null` to omit the rounding attribute. +pub fn fp_to_fp( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + opts: struct { + rounding: ?RoundingMode = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; + if (opts.rounding) |rm| { + attrs.appendAssumeCapacity(.named(ctx, "rounding", roundingModeAttr(ctx, rm))); + } + return .make(ctx, "tt.fp_to_fp", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +// ============================================================================= +// Math ops +// ============================================================================= + +/// tt.clampf — clamp float tensor to [min, max] element-wise. +pub fn clampf( + ctx: *mlir.Context, + x: *const mlir.Value, + min: *const mlir.Value, + max: *const mlir.Value, + propagate_nan: PropagateNan, + location: *const mlir.Location, +) *mlir.Operation { + return .make(ctx, "tt.clampf", .{ + .operands = .{ .flat = &.{ x, min, max } }, + .results = .{ .flat = &.{x.type_()} }, + .attributes = &.{ + .named(ctx, "propagateNan", propagateNanAttr(ctx, propagate_nan)), + }, + .location = location, + }); +} + +/// tt.precise_sqrt — precise sqrt (SameOperandsAndResultType). +pub fn precise_sqrt( + ctx: *mlir.Context, + x: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return .make(ctx, "tt.precise_sqrt", .{ + .operands = .{ .flat = &.{x} }, + .results = .{ .flat = &.{x.type_()} }, + .location = location, + }); +} + +/// tt.precise_divf — precise float div (SameOperandsAndResultType). +pub fn precise_divf( + ctx: *mlir.Context, + x: *const mlir.Value, + y: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return .make(ctx, "tt.precise_divf", .{ + .operands = .{ .flat = &.{ x, y } }, + .results = .{ .flat = &.{x.type_()} }, + .location = location, + }); +} + +/// tt.mulhiui — most significant N bits of 2N-bit unsigned product. +pub fn mulhiui( + ctx: *mlir.Context, + x: *const mlir.Value, + y: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return .make(ctx, "tt.mulhiui", .{ + .operands = .{ .flat = &.{ x, y } }, + .results = .{ .flat = &.{x.type_()} }, + .location = location, + }); +} + +// ============================================================================= +// Shape manipulation — more +// ============================================================================= + +/// tt.unsplat — tensor-of-1-element → scalar. +pub fn unsplat( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return .make(ctx, "tt.unsplat", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// tt.join — join two tensors along a new minor dimension (shape gains +1 dim of size 2). +pub fn join( + ctx: *mlir.Context, + lhs: *const mlir.Value, + rhs: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return .make(ctx, "tt.join", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// tt.split — split a tensor along its last dimension (size must be 2). Returns (lhs, rhs). +pub fn split( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return .make(ctx, "tt.split", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{ result_type, result_type } }, + .location = location, + }); +} + +// ============================================================================= +// Gather / Histogram +// ============================================================================= + +/// tt.gather — gather elements from `src` using `indices` along `axis`. +/// Result shape matches `indices` shape. +pub fn gather( + ctx: *mlir.Context, + src: *const mlir.Value, + indices: *const mlir.Value, + axis: i32, + result_type: *const mlir.Type, + efficient_layout: bool, + location: *const mlir.Location, +) *mlir.Operation { + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + attrs.appendAssumeCapacity(.named(ctx, "axis", .int(ctx, .i32, axis))); + if (efficient_layout) { + attrs.appendAssumeCapacity(.named(ctx, "efficient_layout", .unit(ctx))); + } + return .make(ctx, "tt.gather", .{ + .operands = .{ .flat = &.{ src, indices } }, + .results = .{ .flat = &.{result_type} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +/// tt.histogram — build a histogram of `src`. Output size = number of bins. +pub fn histogram( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + opts: struct { + mask: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var buf: stdx.BoundedArray(*const mlir.Value, 2) = .empty; + buf.appendAssumeCapacity(src); + if (opts.mask) |m| buf.appendAssumeCapacity(m); + return .make(ctx, "tt.histogram", .{ + .operands = .{ .flat = buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +// ============================================================================= +// Scan — parallel scan with combine region (like reduce) +// ============================================================================= + +/// tt.scan — combine_block must already contain the tt.scan.return terminator +/// and have arguments with element types matching `srcs`. +pub fn scan( + ctx: *mlir.Context, + srcs: []const *const mlir.Value, + axis: i32, + reverse: bool, + combine_block: *mlir.Block, + result_types: []const *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return .make(ctx, "tt.scan", .{ + .operands = .{ .variadic = &.{srcs} }, + .results = .{ .flat = result_types }, + .blocks = &.{combine_block}, + .attributes = &.{ + .named(ctx, "axis", .int(ctx, .i32, axis)), + .named(ctx, "reverse", .boolean(ctx, reverse)), + }, + .location = location, + }); +} + +pub fn scan_return( + ctx: *mlir.Context, + values: []const *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return .make(ctx, "tt.scan.return", .{ + .operands = .{ .flat = values }, + .verify = false, + .location = location, + }); +} + +// ============================================================================= +// Atomics +// ============================================================================= + +/// tt.atomic_rmw — (ptr, val, [mask]). Result type matches val's type. +pub fn atomic_rmw( + ctx: *mlir.Context, + rmw: RMWOp, + ptr_val: *const mlir.Value, + val: *const mlir.Value, + sem: MemSemantic, + scope: MemSyncScope, + opts: struct { + mask: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 3) = .empty; + operands_buf.appendSliceAssumeCapacity(&.{ ptr_val, val }); + if (opts.mask) |m| operands_buf.appendAssumeCapacity(m); + + return .make(ctx, "tt.atomic_rmw", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{val.type_()} }, + .attributes = &.{ + .named(ctx, "atomic_rmw_op", rmwOpAttr(ctx, rmw)), + .named(ctx, "sem", memSemanticAttr(ctx, sem)), + .named(ctx, "scope", memSyncScopeAttr(ctx, scope)), + }, + .location = location, + }); +} + +/// tt.atomic_cas — compare-and-swap. Result type matches val's type. +pub fn atomic_cas( + ctx: *mlir.Context, + ptr_val: *const mlir.Value, + cmp: *const mlir.Value, + val: *const mlir.Value, + sem: MemSemantic, + scope: MemSyncScope, + location: *const mlir.Location, +) *mlir.Operation { + return .make(ctx, "tt.atomic_cas", .{ + .operands = .{ .flat = &.{ ptr_val, cmp, val } }, + .results = .{ .flat = &.{val.type_()} }, + .attributes = &.{ + .named(ctx, "sem", memSemanticAttr(ctx, sem)), + .named(ctx, "scope", memSyncScopeAttr(ctx, scope)), + }, + .location = location, + }); +} + +// ============================================================================= +// Debug +// ============================================================================= + +/// tt.assert — device-side assert. `condition` must be i1 or tensor<...xi1>. +pub fn assert_( + ctx: *mlir.Context, + condition: *const mlir.Value, + message: []const u8, + location: *const mlir.Location, +) *mlir.Operation { + return .make(ctx, "tt.assert", .{ + .operands = .{ .flat = &.{condition} }, + .attributes = &.{ + .named(ctx, "message", .string(ctx, message)), + }, + .location = location, + }); +} + +// ============================================================================= +// Call +// ============================================================================= + +/// tt.call — direct call to another tt.func in the same module. +/// `opts.arg_attrs` / `opts.res_attrs` propagate hints like tt.divisibility; +/// leave them unset to omit them. +pub fn call( + ctx: *mlir.Context, + callee: []const u8, + operands: []const *const mlir.Value, + result_types: []const *const mlir.Type, + opts: struct { + arg_attrs: ?[]const *const mlir.Attribute = null, + res_attrs: ?[]const *const mlir.Attribute = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; + attrs.appendAssumeCapacity(.named(ctx, "callee", .flatSymbolRef(ctx, callee))); + if (opts.arg_attrs) |aa| { + attrs.appendAssumeCapacity(.named(ctx, "arg_attrs", .array(ctx, aa))); + } + if (opts.res_attrs) |ra| { + attrs.appendAssumeCapacity(.named(ctx, "res_attrs", .array(ctx, ra))); + } + return .make(ctx, "tt.call", .{ + .operands = .{ .flat = operands }, + .results = .{ .flat = result_types }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +// ============================================================================= +// Scaled dot (microscaling spec). +// ============================================================================= + +/// tt.dot_scaled — dot with optional per-block scale factors. +/// a_scale/b_scale may be null. AttrSizedOperandSegments: (a, b, c, [a_scale], [b_scale]). +pub fn dot_scaled( + ctx: *mlir.Context, + a: *const mlir.Value, + b: *const mlir.Value, + c_acc: *const mlir.Value, + a_elem_type: ScaleDotElemType, + b_elem_type: ScaleDotElemType, + result_type: *const mlir.Type, + fast_math: bool, + lhs_k_pack: bool, + rhs_k_pack: bool, + opts: struct { + a_scale: ?*const mlir.Value = null, + b_scale: ?*const mlir.Value = null, + }, + location: *const mlir.Location, +) *mlir.Operation { + var buf: stdx.BoundedArray(*const mlir.Value, 5) = .empty; + buf.appendSliceAssumeCapacity(&.{ a, b, c_acc }); + const a_scale_len: i32 = if (opts.a_scale) |s| blk: { + buf.appendAssumeCapacity(s); + break :blk 1; + } else 0; + const b_scale_len: i32 = if (opts.b_scale) |s| blk: { + buf.appendAssumeCapacity(s); + break :blk 1; + } else 0; + + const seg_sizes = [5]i32{ 1, 1, 1, a_scale_len, b_scale_len }; + return .make(ctx, "tt.dot_scaled", .{ + .operands = .{ .flat = buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), + .named(ctx, "a_elem_type", scaleDotElemTypeAttr(ctx, a_elem_type)), + .named(ctx, "b_elem_type", scaleDotElemTypeAttr(ctx, b_elem_type)), + .named(ctx, "fastMath", .boolean(ctx, fast_math)), + .named(ctx, "lhs_k_pack", .boolean(ctx, lhs_k_pack)), + .named(ctx, "rhs_k_pack", .boolean(ctx, rhs_k_pack)), + }, + .location = location, + }); +} + +// ============================================================================= +// Extern elementwise (call library function pointwise) +// ============================================================================= + +pub fn extern_elementwise( + ctx: *mlir.Context, + srcs: []const *const mlir.Value, + result_type: *const mlir.Type, + libname: []const u8, + libpath: []const u8, + symbol: []const u8, + pure: bool, + location: *const mlir.Location, +) *mlir.Operation { + return .make(ctx, "tt.extern_elementwise", .{ + .operands = .{ .flat = srcs }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "libname", .string(ctx, libname)), + .named(ctx, "libpath", .string(ctx, libpath)), + .named(ctx, "symbol", .string(ctx, symbol)), + .named(ctx, "pure", .boolean(ctx, pure)), + }, + .location = location, + }); +} + +// ============================================================================= +// Device-side print +// ============================================================================= + +/// tt.print — device-side print. `is_signed` must be `args.len` long and +/// indicates signedness per arg (1 = signed, 0 = unsigned) for printf-style +/// integer formatting. +pub fn print( + ctx: *mlir.Context, + prefix: []const u8, + hex: bool, + args: []const *const mlir.Value, + is_signed: []const i32, + location: *const mlir.Location, +) *mlir.Operation { + std.debug.assert(args.len == is_signed.len); + return .make(ctx, "tt.print", .{ + .operands = .{ .flat = args }, + .attributes = &.{ + .named(ctx, "prefix", .string(ctx, prefix)), + .named(ctx, "hex", .boolean(ctx, hex)), + .named(ctx, "isSigned", .denseArray(ctx, .i32, is_signed)), + }, + .location = location, + }); +} + +// ============================================================================= +// Tensor descriptors (TMA) +// ============================================================================= + +/// tt.make_tensor_descriptor — build a TMA descriptor from a base ptr, runtime +/// shape/strides, and the compile-time block shape baked into `result_type`. +pub fn make_tensor_descriptor( + ctx: *mlir.Context, + base: *const mlir.Value, + shape: []const *const mlir.Value, + strides: []const *const mlir.Value, + padding: PaddingOption, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + var buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; + buf.appendAssumeCapacity(base); + buf.appendSliceAssumeCapacity(shape); + buf.appendSliceAssumeCapacity(strides); + // SameVariadicOperandSize — flatten all operands. + return .make(ctx, "tt.make_tensor_descriptor", .{ + .operands = .{ .flat = buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "padding", paddingOptionAttr(ctx, padding)), + }, + .location = location, + }); +} + +pub fn descriptor_load( + ctx: *mlir.Context, + desc: *const mlir.Value, + indices: []const *const mlir.Value, + result_type: *const mlir.Type, + cache: CacheModifier, + evict: EvictionPolicy, + location: *const mlir.Location, +) *mlir.Operation { + var buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; + buf.appendAssumeCapacity(desc); + buf.appendSliceAssumeCapacity(indices); + return .make(ctx, "tt.descriptor_load", .{ + .operands = .{ .flat = buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "cache", cacheModifierAttr(ctx, cache)), + .named(ctx, "evict", evictionPolicyAttr(ctx, evict)), + }, + .location = location, + }); +} + +pub fn descriptor_store( + ctx: *mlir.Context, + desc: *const mlir.Value, + src: *const mlir.Value, + indices: []const *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + var buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; + buf.appendSliceAssumeCapacity(&.{ desc, src }); + buf.appendSliceAssumeCapacity(indices); + return .make(ctx, "tt.descriptor_store", .{ + .operands = .{ .flat = buf.constSlice() }, + .location = location, + }); +} + +pub fn descriptor_reduce( + ctx: *mlir.Context, + kind: DescriptorReduceKind, + desc: *const mlir.Value, + src: *const mlir.Value, + indices: []const *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + var buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; + buf.appendSliceAssumeCapacity(&.{ desc, src }); + buf.appendSliceAssumeCapacity(indices); + return .make(ctx, "tt.descriptor_reduce", .{ + .operands = .{ .flat = buf.constSlice() }, + .attributes = &.{ + .named(ctx, "kind", descriptorReduceKindAttr(ctx, kind)), + }, + .location = location, + }); +} + +pub fn descriptor_gather( + ctx: *mlir.Context, + desc: *const mlir.Value, + x_offsets: *const mlir.Value, + y_offset: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return .make(ctx, "tt.descriptor_gather", .{ + .operands = .{ .flat = &.{ desc, x_offsets, y_offset } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +pub fn descriptor_scatter( + ctx: *mlir.Context, + desc: *const mlir.Value, + x_offsets: *const mlir.Value, + y_offset: *const mlir.Value, + src: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return .make(ctx, "tt.descriptor_scatter", .{ + .operands = .{ .flat = &.{ desc, x_offsets, y_offset, src } }, + .location = location, + }); +} + +// ============================================================================= +// Map elementwise — like a pointwise `map` with a user-defined scalar region. +// ============================================================================= + +/// tt.map_elementwise — `scalar_block` receives `pack * srcs.len` scalar args +/// (packed groups of `pack` elements per src tensor) and must be terminated +/// with `tt.map_elementwise.return`. +pub fn map_elementwise( + ctx: *mlir.Context, + srcs: []const *const mlir.Value, + pack: i32, + scalar_block: *mlir.Block, + result_types: []const *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return .make(ctx, "tt.map_elementwise", .{ + .operands = .{ .flat = srcs }, + .results = .{ .flat = result_types }, + .blocks = &.{scalar_block}, + .attributes = &.{ + .named(ctx, "pack", .int(ctx, .i32, pack)), + }, + .location = location, + }); +} + +pub fn map_elementwise_return( + ctx: *mlir.Context, + values: []const *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return .make(ctx, "tt.map_elementwise.return", .{ + .operands = .{ .flat = values }, + .verify = false, + .location = location, + }); +} + +// ============================================================================= +// Inline assembly +// ============================================================================= + +/// tt.elementwise_inline_asm — pointwise inline asm returning one or more +/// tensors. +pub fn elementwise_inline_asm( + ctx: *mlir.Context, + asm_string: []const u8, + constraints: []const u8, + args: []const *const mlir.Value, + result_types: []const *const mlir.Type, + pure: bool, + packed_element: i32, + location: *const mlir.Location, +) *mlir.Operation { + return .make(ctx, "tt.elementwise_inline_asm", .{ + .operands = .{ .flat = args }, + .results = .{ .flat = result_types }, + .attributes = &.{ + .named(ctx, "asm_string", .string(ctx, asm_string)), + .named(ctx, "constraints", .string(ctx, constraints)), + .named(ctx, "pure", .boolean(ctx, pure)), + .named(ctx, "packed_element", .int(ctx, .i32, packed_element)), + }, + .location = location, + }); +} diff --git a/mlir/dialects/ub/BUILD.bazel b/mlir/dialects/ub/BUILD.bazel new file mode 100644 index 00000000..039ca5ff --- /dev/null +++ b/mlir/dialects/ub/BUILD.bazel @@ -0,0 +1,34 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +cc_library( + name = "capi", + srcs = ["capi.cc"], + hdrs = ["capi.h"], + tags = ["manual"], + deps = [ + "@llvm-project//mlir:CAPIIR", + "@llvm-project//mlir:UBDialect", + ], +) + +zig_library( + name = "ub", + import_name = "mlir/dialects/ub", + main = "ub.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + ":capi", + "//mlir", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":ub", + "//mlir", + ], +) diff --git a/mlir/dialects/ub/capi.cc b/mlir/dialects/ub/capi.cc new file mode 100644 index 00000000..cf37a2c0 --- /dev/null +++ b/mlir/dialects/ub/capi.cc @@ -0,0 +1,8 @@ +//===- capi.cc - C API for the ub dialect ---------------------------------===// + +#include "capi.h" + +#include "mlir/CAPI/Registration.h" +#include "mlir/Dialect/UB/IR/UBOps.h" + +MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(UB, ub, mlir::ub::UBDialect) diff --git a/mlir/dialects/ub/capi.h b/mlir/dialects/ub/capi.h new file mode 100644 index 00000000..76765ad9 --- /dev/null +++ b/mlir/dialects/ub/capi.h @@ -0,0 +1,18 @@ +//===- capi.h - C API for the ub dialect --------------------------*- C -*-===// + +#ifndef MLIR_DIALECTS_UB_CAPI_H +#define MLIR_DIALECTS_UB_CAPI_H + +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(UB, ub); + +#ifdef __cplusplus +} +#endif + +#endif // MLIR_DIALECTS_UB_CAPI_H diff --git a/mlir/dialects/ub/test.zig b/mlir/dialects/ub/test.zig new file mode 100644 index 00000000..738d1378 --- /dev/null +++ b/mlir/dialects/ub/test.zig @@ -0,0 +1,23 @@ +const std = @import("std"); + +const mlir = @import("mlir"); +const ub = @import("mlir/dialects/ub"); + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(ub); + inline for (comptime std.meta.declarations(ub)) |decl| { + const T = @field(ub, decl); + if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); + } +} + +test "registerDialect registers ub" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + ub.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + try std.testing.expect(ctx.isRegisteredOperation("ub.poison")); +} diff --git a/mlir/dialects/ub/ub.zig b/mlir/dialects/ub/ub.zig new file mode 100644 index 00000000..b5352257 --- /dev/null +++ b/mlir/dialects/ub/ub.zig @@ -0,0 +1,5 @@ +const mlir = @import("mlir"); + +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("ub"); +} diff --git a/mlir/dialects/vector/BUILD.bazel b/mlir/dialects/vector/BUILD.bazel new file mode 100644 index 00000000..579fa599 --- /dev/null +++ b/mlir/dialects/vector/BUILD.bazel @@ -0,0 +1,36 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +cc_library( + name = "capi", + srcs = ["capi.cc"], + hdrs = ["capi.h"], + tags = ["manual"], + deps = [ + "@llvm-project//mlir:CAPIIR", + "@llvm-project//mlir:VectorDialect", + ], +) + +zig_library( + name = "vector", + import_name = "mlir/dialects/vector", + main = "vector.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + ":capi", + "//mlir", + "//stdx", + "@llvm-project//mlir:CAPIVector", + ], +) + +zig_test( + name = "test", + main = "test.zig", + deps = [ + ":vector", + "//mlir", + ], +) diff --git a/mlir/dialects/vector/capi.cc b/mlir/dialects/vector/capi.cc new file mode 100644 index 00000000..f22ad655 --- /dev/null +++ b/mlir/dialects/vector/capi.cc @@ -0,0 +1,51 @@ +//===- capi.cc - C API for the vector dialect -----------------------------===// + +#include "capi.h" + +#include "mlir/CAPI/IR.h" +#include "mlir/Dialect/Vector/IR/VectorOps.h" + +static mlir::MLIRContext *loadVector(MlirContext context) { + mlir::MLIRContext *ctx = unwrap(context); + if (ctx) + ctx->getOrLoadDialect(); + return ctx; +} + +bool mlirAttributeIsAVectorCombiningKind(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirVectorCombiningKindAttrGet(MlirContext context, + uint32_t value) { + mlir::MLIRContext *ctx = loadVector(context); + std::optional kind = + mlir::vector::symbolizeCombiningKind(value); + if (!ctx || !kind) + return {nullptr}; + return wrap(mlir::vector::CombiningKindAttr::get(ctx, *kind)); +} + +uint32_t mlirVectorCombiningKindAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} + +bool mlirAttributeIsAVectorIteratorType(MlirAttribute attr) { + return llvm::isa_and_nonnull(unwrap(attr)); +} + +MlirAttribute mlirVectorIteratorTypeAttrGet(MlirContext context, + uint32_t value) { + mlir::MLIRContext *ctx = loadVector(context); + std::optional type = + mlir::vector::symbolizeIteratorType(value); + if (!ctx || !type) + return {nullptr}; + return wrap(mlir::vector::IteratorTypeAttr::get(ctx, *type)); +} + +uint32_t mlirVectorIteratorTypeAttrGetValue(MlirAttribute attr) { + return static_cast( + llvm::cast(unwrap(attr)).getValue()); +} diff --git a/mlir/dialects/vector/capi.h b/mlir/dialects/vector/capi.h new file mode 100644 index 00000000..75e12cbd --- /dev/null +++ b/mlir/dialects/vector/capi.h @@ -0,0 +1,36 @@ +//===- capi.h - C API for the vector dialect ----------------------*- C -*-===// +// +// C entry points for the `vector` dialect's enum attributes, which upstream's +// C API leaves out. + +#ifndef MLIR_DIALECTS_VECTOR_CAPI_H +#define MLIR_DIALECTS_VECTOR_CAPI_H + +#include "mlir-c/IR.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Enum attributes take and return the C++ enum's integer value; `Get` returns +// a null attribute for a value outside the enum. + +// #vector.kind<...> +MLIR_CAPI_EXPORTED bool mlirAttributeIsAVectorCombiningKind(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirVectorCombiningKindAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirVectorCombiningKindAttrGetValue(MlirAttribute attr); + +// #vector.iterator_type<...> +MLIR_CAPI_EXPORTED bool mlirAttributeIsAVectorIteratorType(MlirAttribute attr); +MLIR_CAPI_EXPORTED MlirAttribute +mlirVectorIteratorTypeAttrGet(MlirContext context, uint32_t value); +MLIR_CAPI_EXPORTED uint32_t +mlirVectorIteratorTypeAttrGetValue(MlirAttribute attr); + +#ifdef __cplusplus +} +#endif + +#endif // MLIR_DIALECTS_VECTOR_CAPI_H diff --git a/mlir/dialects/vector/test.zig b/mlir/dialects/vector/test.zig new file mode 100644 index 00000000..ca737ae7 --- /dev/null +++ b/mlir/dialects/vector/test.zig @@ -0,0 +1,86 @@ +const std = @import("std"); + +const mlir = @import("mlir"); +const vector = @import("mlir/dialects/vector"); + +const operation_names: []const []const u8 = &.{ + "vector.bitcast", + "vector.broadcast", + "vector.compressstore", + "vector.constant_mask", + "vector.contract", + "vector.create_mask", + "vector.deinterleave", + "vector.expandload", + "vector.extract", + "vector.extract_strided_slice", + "vector.fma", + "vector.from_elements", + "vector.gather", + "vector.insert", + "vector.insert_strided_slice", + "vector.interleave", + "vector.load", + "vector.maskedload", + "vector.maskedstore", + "vector.multi_reduction", + "vector.reduction", + "vector.scatter", + "vector.shape_cast", + "vector.shuffle", + "vector.step", + "vector.store", + "vector.to_elements", + "vector.transfer_read", + "vector.transfer_write", + "vector.transpose", + "vector.type_cast", + "vector.vscale", + "vector.yield", +}; + +fn expectPrints(expected: []const u8, value: anytype) !void { + var buf: [128]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + try w.print("{f}", .{value}); + try std.testing.expectEqualStrings(expected, w.buffered()); +} + +test { + @setEvalBranchQuota(100_000); + std.testing.refAllDecls(vector); + inline for (comptime std.meta.declarations(vector)) |decl| { + const T = @field(vector, decl); + if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); + } +} + +test "vector enum attributes are built through the C API" { + const ctx = try mlir.Context.init(.{}); + defer ctx.deinit(); + + inline for (@typeInfo(vector.CombiningKind).@"enum".field_names) |name| { + const value: vector.CombiningKind = @field(vector.CombiningKind, name); + const attr = try vector.CombiningKindAttr.get(ctx, .{ .value = value }); + try std.testing.expectEqual(value, attr.value()); + try std.testing.expect(value.attribute(ctx).isA(vector.CombiningKindAttr) != null); + try expectPrints("#vector.kind<" ++ name ++ ">", attr); + } + inline for (@typeInfo(vector.IteratorType).@"enum".field_names) |name| { + const value: vector.IteratorType = @field(vector.IteratorType, name); + const attr = try vector.IteratorTypeAttr.get(ctx, .{ .value = value }); + try std.testing.expectEqual(value, attr.value()); + try expectPrints("#vector.iterator_type<" ++ name ++ ">", attr); + } +} + +test "every bound operation is registered" { + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + vector.registerDialect(registry); + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); + try std.testing.expect(!ctx.allowUnregisteredDialects()); +} diff --git a/mlir/dialects/vector/vector.zig b/mlir/dialects/vector/vector.zig new file mode 100644 index 00000000..1696d04b --- /dev/null +++ b/mlir/dialects/vector/vector.zig @@ -0,0 +1,820 @@ +const std = @import("std"); + +const c = @import("c"); +const mlir = @import("mlir"); +const stdx = @import("stdx"); + +pub fn registerDialect(registry: *mlir.DialectRegistry) void { + registry.registerDialect("vector"); +} + +// ============================================================================= +// Enum attributes +// ============================================================================= + +/// Kind of combining function for contractions and reductions. +pub const CombiningKind = enum(u32) { + add = 0, + mul = 1, + minui = 2, + minsi = 3, + minnumf = 4, + maxui = 5, + maxsi = 6, + maxnumf = 7, + @"and" = 8, + @"or" = 9, + xor = 10, + minimumf = 11, + maximumf = 12, + minimumnumf = 13, + maximumnumf = 14, + + pub fn attribute(self: CombiningKind, ctx: *mlir.Context) *const mlir.Attribute { + return (CombiningKindAttr.get(ctx, .{ .value = self }) catch unreachable).attribute(); + } +}; + +/// Iterator type of a `vector.contract` dimension. +pub const IteratorType = enum(u32) { + parallel = 0, + reduction = 1, + + pub fn attribute(self: IteratorType, ctx: *mlir.Context) *const mlir.Attribute { + return (IteratorTypeAttr.get(ctx, .{ .value = self }) catch unreachable).attribute(); + } +}; + +/// `#vector.kind<...>`. +pub const CombiningKindAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsAVectorCombiningKind; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "kind"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: CombiningKind, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirVectorCombiningKindAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) CombiningKind { + return @fromBackingInt(@intCast(c.mlirVectorCombiningKindAttrGetValue(self.ptr()))); + } +}; + +/// `#vector.iterator_type<...>`. +pub const IteratorTypeAttr = opaque { + const Self = @This(); + const M = mlir.Methods(Self, c.MlirAttribute); + pub const isAFn = c.mlirAttributeIsAVectorIteratorType; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + pub const mnemonic = "iterator_type"; + pub fn attribute(self: *const Self) *const mlir.Attribute { + return @ptrCast(self); + } + pub const InitArgs = struct { + value: IteratorType, + }; + pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { + const result = c.mlirVectorIteratorTypeAttrGet(ctx.ptr(), @backingInt(args.value)); + return @ptrCast(result.ptr orelse return error.InvalidMlir); + } + pub fn value(self: *const Self) IteratorType { + return @fromBackingInt(@intCast(c.mlirVectorIteratorTypeAttrGetValue(self.ptr()))); + } +}; + +// ============================================================================= +// Element / lane access — extract / insert / from_elements / to_elements +// ============================================================================= + +/// vector.extract — pull a single element or sub-vector out of `src`. Static +/// position entries equal to `kPoison` (== `INT64_MIN` per VectorOps.td) come +/// from `dynamic_position`. The result type is provided by the caller. +pub const kPoison: i64 = std.math.minInt(i64); + +pub fn extract( + ctx: *mlir.Context, + src: *const mlir.Value, + static_position: []const i64, + dynamic_position: []const *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; + operands_buf.appendAssumeCapacity(src); + operands_buf.appendSliceAssumeCapacity(dynamic_position); + + return mlir.Operation.make(ctx, "vector.extract", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "static_position", .denseArray(ctx, .i64, static_position)), + }, + .location = location, + }); +} + +/// vector.insert — write `value` into `dest` at the given position. The result +/// has the same type as `dest`. +pub fn insert( + ctx: *mlir.Context, + value: *const mlir.Value, + dest: *const mlir.Value, + static_position: []const i64, + dynamic_position: []const *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 17) = .empty; + operands_buf.appendSliceAssumeCapacity(&.{ value, dest }); + operands_buf.appendSliceAssumeCapacity(dynamic_position); + + return mlir.Operation.make(ctx, "vector.insert", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{dest.type_()} }, + .attributes = &.{ + .named(ctx, "static_position", .denseArray(ctx, .i64, static_position)), + }, + .location = location, + }); +} + +/// vector.from_elements — build a vector from an exact-size list of scalars. +pub fn from_elements( + ctx: *mlir.Context, + elements: []const *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "vector.from_elements", .{ + .operands = .{ .flat = elements }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// vector.to_elements — destructure a vector into its scalar lanes. +pub fn to_elements( + ctx: *mlir.Context, + src: *const mlir.Value, + result_types: []const *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "vector.to_elements", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = result_types }, + .location = location, + }); +} + +// ============================================================================= +// Strided slice — extract / insert with offsets/sizes/strides +// ============================================================================= + +pub const StridedSliceArgs = struct { + offsets: []const i64, + sizes: []const i64, + strides: []const i64, +}; + +pub fn extract_strided_slice( + ctx: *mlir.Context, + src: *const mlir.Value, + args: StridedSliceArgs, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "vector.extract_strided_slice", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "offsets", .intArray(ctx, i64, args.offsets)), + .named(ctx, "sizes", .intArray(ctx, i64, args.sizes)), + .named(ctx, "strides", .intArray(ctx, i64, args.strides)), + }, + .location = location, + }); +} + +pub fn insert_strided_slice( + ctx: *mlir.Context, + src: *const mlir.Value, + dest: *const mlir.Value, + offsets: []const i64, + strides: []const i64, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "vector.insert_strided_slice", .{ + .operands = .{ .flat = &.{ src, dest } }, + .results = .{ .flat = &.{dest.type_()} }, + .attributes = &.{ + .named(ctx, "offsets", .intArray(ctx, i64, offsets)), + .named(ctx, "strides", .intArray(ctx, i64, strides)), + }, + .location = location, + }); +} + +// ============================================================================= +// Shape manipulation +// ============================================================================= + +/// vector.broadcast — broadcast a scalar / lower-rank vector to a wider one. +pub fn broadcast( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "vector.broadcast", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// vector.shape_cast — reshape a vector preserving total number of lanes. +pub fn shape_cast( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "vector.shape_cast", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// vector.bitcast — same total bits, possibly different element type / shape. +pub fn bitcast( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "vector.bitcast", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// vector.type_cast — cast a `memref>` to `memref<...>` (and vice- +/// versa). Used in vector lowering / unrolling. +pub fn type_cast( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "vector.type_cast", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// vector.transpose — permute dims of a vector. +pub fn transpose( + ctx: *mlir.Context, + src: *const mlir.Value, + permutation: []const i64, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "vector.transpose", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "permutation", .denseArray(ctx, .i64, permutation)), + }, + .location = location, + }); +} + +// ============================================================================= +// Lane shuffle — shuffle / interleave / deinterleave +// ============================================================================= + +/// vector.shuffle — concat lhs and rhs into a logical vector (rank 1) and +/// pick `mask`-listed lanes to form the result. +pub fn shuffle( + ctx: *mlir.Context, + lhs: *const mlir.Value, + rhs: *const mlir.Value, + mask: []const i64, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "vector.shuffle", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "mask", .denseArray(ctx, .i64, mask)), + }, + .location = location, + }); +} + +/// vector.interleave — interleave even/odd lanes from lhs/rhs. +pub fn interleave( + ctx: *mlir.Context, + lhs: *const mlir.Value, + rhs: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "vector.interleave", .{ + .operands = .{ .flat = &.{ lhs, rhs } }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// vector.deinterleave — split a vector into (even-lanes, odd-lanes). +pub fn deinterleave( + ctx: *mlir.Context, + src: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "vector.deinterleave", .{ + .operands = .{ .flat = &.{src} }, + .results = .{ .flat = &.{ result_type, result_type } }, + .location = location, + }); +} + +// ============================================================================= +// Compute — fma / step / vscale +// ============================================================================= + +/// vector.fma — element-wise fused multiply-add `lhs * rhs + acc`. +pub fn fma( + ctx: *mlir.Context, + lhs: *const mlir.Value, + rhs: *const mlir.Value, + acc: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "vector.fma", .{ + .operands = .{ .flat = &.{ lhs, rhs, acc } }, + .result_type_inference = true, + .location = location, + }); +} + +/// vector.step — produce `<0, 1, ..., N-1>` of the result type. +pub fn step( + ctx: *mlir.Context, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "vector.step", .{ + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +/// vector.vscale — runtime scalable-vector size factor (index-typed). +pub fn vscale(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "vector.vscale", .{ + .results = .{ .flat = &.{.index(ctx)} }, + .location = location, + }); +} + +/// vector.yield — terminator for vector.mask / vector.warp regions. +pub fn yield(ctx: *mlir.Context, values: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { + return mlir.Operation.make(ctx, "vector.yield", .{ + .operands = .{ .flat = values }, + .verify = false, + .location = location, + }); +} + +// ============================================================================= +// Reductions — reduction / multi_reduction / contract +// ============================================================================= + +pub const ReductionOpts = struct { + /// Optional accumulator (scalar of result type). When provided the op is + /// `result = combine(reduce(src), acc)`. + acc: ?*const mlir.Value = null, +}; + +/// vector.reduction — reduce a 1-D vector to a scalar with `kind`. +pub fn reduction( + ctx: *mlir.Context, + kind: CombiningKind, + src: *const mlir.Value, + result_type: *const mlir.Type, + opts: ReductionOpts, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 2) = .empty; + operands_buf.appendAssumeCapacity(src); + if (opts.acc) |a| operands_buf.appendAssumeCapacity(a); + + return mlir.Operation.make(ctx, "vector.reduction", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "kind", kind.attribute(ctx)), + }, + .location = location, + }); +} + +/// vector.multi_reduction — reduce a multi-D vector along `reduction_dims`. +pub fn multi_reduction( + ctx: *mlir.Context, + kind: CombiningKind, + src: *const mlir.Value, + acc: *const mlir.Value, + reduction_dims: []const i64, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "vector.multi_reduction", .{ + .operands = .{ .flat = &.{ src, acc } }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "kind", kind.attribute(ctx)), + .named(ctx, "reduction_dims", .denseArray(ctx, .i64, reduction_dims)), + }, + .location = location, + }); +} + +pub const ContractOpts = struct { + kind: CombiningKind = .add, +}; + +/// vector.contract — generalized contraction (matmul, dot, batched matmul, +/// reductions). `indexing_maps` is one AffineMapAttr per (lhs, rhs, acc). +/// `iterator_types` is one IteratorType per dim of the iteration space. +pub fn contract( + ctx: *mlir.Context, + lhs: *const mlir.Value, + rhs: *const mlir.Value, + acc: *const mlir.Value, + indexing_maps: []const *const mlir.Attribute, + iterator_types: []const IteratorType, + result_type: *const mlir.Type, + opts: ContractOpts, + location: *const mlir.Location, +) *mlir.Operation { + var iter_attrs: stdx.BoundedArray(*const mlir.Attribute, 16) = .empty; + for (iterator_types) |it| iter_attrs.appendAssumeCapacity(it.attribute(ctx)); + + return mlir.Operation.make(ctx, "vector.contract", .{ + .operands = .{ .flat = &.{ lhs, rhs, acc } }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "indexing_maps", .array(ctx, indexing_maps)), + .named(ctx, "iterator_types", .array(ctx, iter_attrs.constSlice())), + .named(ctx, "kind", opts.kind.attribute(ctx)), + }, + .location = location, + }); +} + +// ============================================================================= +// Memory transfers — load / store / masked / gather / scatter / transfer_* +// ============================================================================= + +pub const LoadStoreOpts = struct { + nontemporal: bool = false, + alignment: ?i64 = null, +}; + +fn loadStoreAttrs(ctx: *mlir.Context, opts: LoadStoreOpts, dst: *stdx.BoundedArray(mlir.NamedAttribute, 2)) void { + if (opts.nontemporal) { + dst.appendAssumeCapacity(.named(ctx, "nontemporal", .boolean(ctx, true))); + } + if (opts.alignment) |a| { + dst.appendAssumeCapacity(.named(ctx, "alignment", .int(ctx, .i64, a))); + } +} + +/// vector.load — strided contiguous load from `base[indices...]` into the +/// result vector. +pub fn load( + ctx: *mlir.Context, + base: *const mlir.Value, + indices: []const *const mlir.Value, + result_type: *const mlir.Type, + opts: LoadStoreOpts, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; + operands_buf.appendAssumeCapacity(base); + operands_buf.appendSliceAssumeCapacity(indices); + + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + loadStoreAttrs(ctx, opts, &attrs); + + return mlir.Operation.make(ctx, "vector.load", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +/// vector.store — strided contiguous store of `value` to `base[indices...]`. +pub fn store( + ctx: *mlir.Context, + value: *const mlir.Value, + base: *const mlir.Value, + indices: []const *const mlir.Value, + opts: LoadStoreOpts, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 17) = .empty; + operands_buf.appendSliceAssumeCapacity(&.{ value, base }); + operands_buf.appendSliceAssumeCapacity(indices); + + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; + loadStoreAttrs(ctx, opts, &attrs); + + return mlir.Operation.make(ctx, "vector.store", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .attributes = attrs.constSlice(), + .location = location, + }); +} + +/// vector.maskedload — like load but lanes whose `mask` bit is 0 are read +/// from `passthru` instead. +pub fn masked_load( + ctx: *mlir.Context, + base: *const mlir.Value, + indices: []const *const mlir.Value, + mask: *const mlir.Value, + passthru: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 18) = .empty; + operands_buf.appendAssumeCapacity(base); + operands_buf.appendSliceAssumeCapacity(indices); + operands_buf.appendSliceAssumeCapacity(&.{ mask, passthru }); + + return mlir.Operation.make(ctx, "vector.maskedload", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +pub fn masked_store( + ctx: *mlir.Context, + base: *const mlir.Value, + indices: []const *const mlir.Value, + mask: *const mlir.Value, + value: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 18) = .empty; + operands_buf.appendAssumeCapacity(base); + operands_buf.appendSliceAssumeCapacity(indices); + operands_buf.appendSliceAssumeCapacity(&.{ mask, value }); + + return mlir.Operation.make(ctx, "vector.maskedstore", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .location = location, + }); +} + +/// vector.gather — gather scalar elements from `base[indices...]` plus per-lane +/// `index_vec` offsets. +pub fn gather( + ctx: *mlir.Context, + base: *const mlir.Value, + indices: []const *const mlir.Value, + index_vec: *const mlir.Value, + mask: *const mlir.Value, + passthru: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 20) = .empty; + operands_buf.appendAssumeCapacity(base); + operands_buf.appendSliceAssumeCapacity(indices); + operands_buf.appendSliceAssumeCapacity(&.{ index_vec, mask, passthru }); + + return mlir.Operation.make(ctx, "vector.gather", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +pub fn scatter( + ctx: *mlir.Context, + base: *const mlir.Value, + indices: []const *const mlir.Value, + index_vec: *const mlir.Value, + mask: *const mlir.Value, + value: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 20) = .empty; + operands_buf.appendAssumeCapacity(base); + operands_buf.appendSliceAssumeCapacity(indices); + operands_buf.appendSliceAssumeCapacity(&.{ index_vec, mask, value }); + + return mlir.Operation.make(ctx, "vector.scatter", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .location = location, + }); +} + +pub fn expand_load( + ctx: *mlir.Context, + base: *const mlir.Value, + indices: []const *const mlir.Value, + mask: *const mlir.Value, + passthru: *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 18) = .empty; + operands_buf.appendAssumeCapacity(base); + operands_buf.appendSliceAssumeCapacity(indices); + operands_buf.appendSliceAssumeCapacity(&.{ mask, passthru }); + + return mlir.Operation.make(ctx, "vector.expandload", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} + +pub fn compress_store( + ctx: *mlir.Context, + base: *const mlir.Value, + indices: []const *const mlir.Value, + mask: *const mlir.Value, + value: *const mlir.Value, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 18) = .empty; + operands_buf.appendAssumeCapacity(base); + operands_buf.appendSliceAssumeCapacity(indices); + operands_buf.appendSliceAssumeCapacity(&.{ mask, value }); + + return mlir.Operation.make(ctx, "vector.compressstore", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .location = location, + }); +} + +// ============================================================================= +// Transfer ops — vector.transfer_read / vector.transfer_write +// ============================================================================= + +pub const TransferReadArgs = struct { + /// `affine_map<(d0,...) -> (...)>` attribute. Build with + /// `mlir.parseAffineMapAttribute` or `mlir.Attribute.affineMap(...)`. + permutation_map: *const mlir.Attribute, + /// One bool per result-vector dim. Length must equal vector rank. + in_bounds: []const bool, + /// Optional i1 vector mask. + mask: ?*const mlir.Value = null, +}; + +/// vector.transfer_read — supervector load with permutation map and optional +/// mask. AttrSizedOperandSegments: (base, indices, padding, [mask]). +pub fn transfer_read( + ctx: *mlir.Context, + base: *const mlir.Value, + indices: []const *const mlir.Value, + padding: *const mlir.Value, + args: TransferReadArgs, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 20) = .empty; + operands_buf.appendAssumeCapacity(base); + operands_buf.appendSliceAssumeCapacity(indices); + operands_buf.appendAssumeCapacity(padding); + const mask_len: i32 = if (args.mask) |m| blk: { + operands_buf.appendAssumeCapacity(m); + break :blk 1; + } else 0; + + const seg_sizes = [4]i32{ 1, @intCast(indices.len), 1, mask_len }; + + var in_bounds_attrs: stdx.BoundedArray(*const mlir.Attribute, 16) = .empty; + for (args.in_bounds) |b| in_bounds_attrs.appendAssumeCapacity(.boolean(ctx, b)); + + return mlir.Operation.make(ctx, "vector.transfer_read", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), + .named(ctx, "permutation_map", args.permutation_map), + .named(ctx, "in_bounds", .array(ctx, in_bounds_attrs.constSlice())), + }, + .location = location, + }); +} + +pub const TransferWriteArgs = struct { + permutation_map: *const mlir.Attribute, + in_bounds: []const bool, + mask: ?*const mlir.Value = null, + /// When `dest` is a tensor, the op produces a new tensor — pass its type + /// here. For memref destinations leave null. + result_type: ?*const mlir.Type = null, +}; + +/// vector.transfer_write — supervector store with permutation map and optional +/// mask. AttrSizedOperandSegments: (vector, base, indices, [mask]). +pub fn transfer_write( + ctx: *mlir.Context, + value: *const mlir.Value, + base: *const mlir.Value, + indices: []const *const mlir.Value, + args: TransferWriteArgs, + location: *const mlir.Location, +) *mlir.Operation { + var operands_buf: stdx.BoundedArray(*const mlir.Value, 20) = .empty; + operands_buf.appendSliceAssumeCapacity(&.{ value, base }); + operands_buf.appendSliceAssumeCapacity(indices); + const mask_len: i32 = if (args.mask) |m| blk: { + operands_buf.appendAssumeCapacity(m); + break :blk 1; + } else 0; + + const seg_sizes = [4]i32{ 1, 1, @intCast(indices.len), mask_len }; + + var in_bounds_attrs: stdx.BoundedArray(*const mlir.Attribute, 16) = .empty; + for (args.in_bounds) |b| in_bounds_attrs.appendAssumeCapacity(.boolean(ctx, b)); + + var result_types: stdx.BoundedArray(*const mlir.Type, 1) = .empty; + if (args.result_type) |rt| result_types.appendAssumeCapacity(rt); + + return mlir.Operation.make(ctx, "vector.transfer_write", .{ + .operands = .{ .flat = operands_buf.constSlice() }, + .results = .{ .flat = result_types.constSlice() }, + .attributes = &.{ + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), + .named(ctx, "permutation_map", args.permutation_map), + .named(ctx, "in_bounds", .array(ctx, in_bounds_attrs.constSlice())), + }, + .location = location, + }); +} + +// ============================================================================= +// Masks — constant_mask / create_mask +// ============================================================================= + +/// vector.constant_mask — constant boolean mask. Each entry of +/// `mask_dim_sizes` is the number of leading true lanes along that dim. +pub fn constant_mask( + ctx: *mlir.Context, + mask_dim_sizes: []const i64, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "vector.constant_mask", .{ + .results = .{ .flat = &.{result_type} }, + .attributes = &.{ + .named(ctx, "mask_dim_sizes", .denseArray(ctx, .i64, mask_dim_sizes)), + }, + .location = location, + }); +} + +/// vector.create_mask — runtime mask from per-dim count operands. `operands` +/// length must equal the rank of `result_type`. +pub fn create_mask( + ctx: *mlir.Context, + operands: []const *const mlir.Value, + result_type: *const mlir.Type, + location: *const mlir.Location, +) *mlir.Operation { + return mlir.Operation.make(ctx, "vector.create_mask", .{ + .operands = .{ .flat = operands }, + .results = .{ .flat = &.{result_type} }, + .location = location, + }); +} diff --git a/mlir/mlir.h b/mlir/mlir.h new file mode 100644 index 00000000..b897c82b --- /dev/null +++ b/mlir/mlir.h @@ -0,0 +1,15 @@ +//===- mlir.h - MLIR C API headers bound by the mlir module -------*- C -*-===// + +#ifndef MLIR_MLIR_H +#define MLIR_MLIR_H + +#include "mlir-c/AffineExpr.h" +#include "mlir-c/AffineMap.h" +#include "mlir-c/BuiltinAttributes.h" +#include "mlir-c/BuiltinTypes.h" +#include "mlir-c/IR.h" +#include "mlir-c/IntegerSet.h" +#include "mlir-c/Pass.h" +#include "mlir-c/Transforms.h" + +#endif // MLIR_MLIR_H diff --git a/mlir/mlir.zig b/mlir/mlir.zig index 0c00cf54..57e57bd2 100644 --- a/mlir/mlir.zig +++ b/mlir/mlir.zig @@ -1,1008 +1,1473 @@ -const builtin = @import("builtin"); const std = @import("std"); const c = @import("c"); +const stdx = @import("stdx"); + +const log = std.log.scoped(.mlir); test { std.testing.refAllDecls(@This()); } -const Error = error{ +test "variadic operation groups preserve order and dense array segment sizes" { + const ctx = try Context.init(.{}); + defer ctx.deinit(); + ctx.setAllowUnregisteredDialects(true); + const loc = Location.unknown(ctx); + const i32_type = Type.int(ctx, .i32); + const i64_type = Type.int(ctx, .i64); + const block = Block.init(&.{ i32_type, i64_type, i32_type }, &.{ loc, loc, loc }); + defer block.deinit(); + const op = Operation.make(ctx, "test.segmented", .{ + .operands = .{ .variadic = &.{ + &.{block.argument(0)}, + &.{}, + &.{ block.argument(1), block.argument(2) }, + } }, + .results = .{ .variadic = &.{ + &.{}, + &.{ i64_type, i32_type }, + &.{i64_type}, + } }, + .location = loc, + }); + defer op.deinit(); + try std.testing.expectEqual(@as(usize, 3), op.numOperands()); + for (0..3) |i| try std.testing.expect(op.operand(i).eql(block.argument(i))); + try std.testing.expectEqual(@as(usize, 3), op.numResults()); + try std.testing.expect(op.result(0).type_().eql(i64_type)); + try std.testing.expect(op.result(1).type_().eql(i32_type)); + try std.testing.expect(op.result(2).type_().eql(i64_type)); + try std.testing.expect(op.attributeByName("operandSegmentSizes").?.eql(.denseArray(ctx, .i32, &.{ 1, 0, 2 }))); + try std.testing.expect(op.attributeByName("resultSegmentSizes").?.eql(.denseArray(ctx, .i32, &.{ 0, 2, 1 }))); +} + +test "unrealized_conversion_cast builds a verified operation" { + const ctx = try Context.init(.{}); + defer ctx.deinit(); + const loc = Location.unknown(ctx); + const block = Block.init(&.{Type.int(ctx, .i32)}, &.{loc}); + defer block.deinit(); + const cast = unrealized_conversion_cast(ctx, &.{block.argument(0)}, &.{Type.index(ctx)}, loc).appendTo(block); + try std.testing.expect(cast.verify()); + try std.testing.expect(cast.result(0).type_().eql(Type.index(ctx))); +} + +test "nested symbol references print their path" { + const ctx = try Context.init(.{}); + defer ctx.deinit(); + const symbol = Attribute.symbolRef(ctx, "kernels", &.{FlatSymbolRefAttribute.init(ctx, "add")}); + try std.testing.expect(symbol.eql(try Attribute.parse(ctx, "@kernels::@add"))); +} + +pub const Error = error{ /// Invalid Mlir was created. InvalidMlir, /// Another Mlir error. Check the log for more context. MlirUnexpected, /// A resource/executor was not found. NotFound, - OutOfMemory, -}; +} || std.mem.Allocator.Error; pub inline fn stringRef(str: []const u8) c.MlirStringRef { return .{ .data = str.ptr, .length = str.len }; } -pub inline fn fromStringRef(str: c.MlirStringRef) []const u8 { +pub inline fn string(str: c.MlirStringRef) []const u8 { // Note: mlir.StringRef need not to be null terminated. return str.data[0..str.length]; } +pub const StringCallbackCtx = struct { + writer: *std.Io.Writer, + err: ?std.Io.Writer.Error = null, +}; + +pub fn stringCallback(str: c.MlirStringRef, ctx_: ?*anyopaque) callconv(.c) void { + const ctx: *StringCallbackCtx = @ptrCast(@alignCast(ctx_)); + ctx.writer.writeAll(string(str)) catch |err| { + ctx.err = err; + }; +} + +fn stringFromStream(buf: []u8, streamFn: anytype, args: anytype) []const u8 { + var writer = std.Io.Writer.fixed(buf); + var sctx: StringCallbackCtx = .{ .writer = &writer }; + _ = @call(.auto, streamFn, args ++ .{ stringCallback, &sctx }); + return writer.buffered(); +} + pub fn registerPasses(comptime passes: []const u8) void { @field(c, "mlirRegister" ++ passes ++ "Passes")(); } -pub fn successOr(res: c.MlirLogicalResult, err: anytype) !void { - return if (res.value == 0) err; +fn isPtrConst(comptime T: type) bool { + return @typeInfo(T).pointer.attrs.@"const"; } -pub fn MlirTypeMethods(comptime InnerT: type) type { - return struct { - is_null_fn: ?fn (InnerT) callconv(.C) bool = null, - is_a_fn: ?fn (InnerT) callconv(.C) bool = null, - equal_fn: ?fn (InnerT, InnerT) callconv(.C) bool = null, - dump_fn: ?fn (InnerT) callconv(.C) void = null, - deinit_fn: ?fn (InnerT) callconv(.C) void = null, - }; +fn CastedPtr(comptime T1: type, comptime T2: type) type { + return if (@typeInfo(T1).pointer.attrs.@"const") *const T2 else *T2; } -/// Alternative to MlirWrapperType -pub const MlirStrCallback = fn (c.MlirStringRef, ?*anyopaque) callconv(.C) void; - -fn MlirHelpersMethods(OuterT: type) type { - switch (@typeInfo(OuterT)) { - .Struct => |info| { - if (info.fields.len != 1) @compileError("Mlir wrapper type can only wrap one Mlir value. Received: " ++ @typeName(OuterT)); - }, - else => @compileError("MlirHelpersMethods is only available on an Mlir wrapper struct. Received: " ++ @typeName(OuterT)), - } - +pub fn Methods(comptime T: type, comptime NativeT: type) type { return struct { - pub const InnerT = std.meta.FieldType(OuterT, ._inner); - comptime { - std.debug.assert(@sizeOf(InnerT) == @sizeOf(OuterT)); + pub const IsPtrConst = @typeInfo(@typeInfo(@FieldType(NativeT, "ptr")).optional.child).pointer.attrs.@"const"; + pub const Ptr = if (IsPtrConst) *const T else *T; + + pub fn ptr(self: anytype) NativeT { + return .{ .ptr = @ptrCast(@constCast(self)) }; } - is_null_fn: ?fn (InnerT) callconv(.C) bool = null, - is_a_fn: ?fn (InnerT) callconv(.C) bool = null, - equal_fn: ?fn (InnerT, InnerT) callconv(.C) bool = null, - deinit_fn: ?fn (InnerT) callconv(.C) void = null, - dump_fn: ?fn (InnerT) callconv(.C) void = null, - print_fn: ?fn (InnerT, ?*const MlirStrCallback, ?*anyopaque) callconv(.C) void = null, - }; -} - -pub fn MlirHelpers(comptime OuterT: type, comptime methods: MlirHelpersMethods(OuterT)) type { - const InnerT = @TypeOf(methods).InnerT; - return struct { - pub const Methods = methods; - - pub inline fn wrap(raw: InnerT) OuterT { - return .{ ._inner = raw }; - } - - pub inline fn inner(self: OuterT) InnerT { - return self._inner; - } - - pub inline fn innerPtr(self: *OuterT) *InnerT { - return &self._inner; - } - - pub inline fn is_a(self: OuterT, comptime otherT: type) bool { - if (otherT.Methods.is_a_fn) |is_a_fn| { - return is_a_fn(self.inner()); - } - return false; - } - - pub inline fn as(self: OuterT, comptime OtherT: type) ?OtherT { - if (OtherT.Methods.is_a_fn) |is_a_fn| { - return if (is_a_fn(self.inner())) OtherT.wrap(self.inner()) else null; - } - // if the other type doesn't have an is_a_fn, try. - return OtherT.wrap(self.inner()); - } - - pub usingnamespace if (Methods.is_null_fn) |is_null| struct { - pub inline fn wrapOr(raw: InnerT) ?OuterT { - return if (is_null(raw)) null else OuterT.wrap(raw); - } - } else struct {}; - - pub usingnamespace if (Methods.equal_fn) |equal| struct { - pub inline fn eql(self: OuterT, other: OuterT) bool { - return equal(self.inner(), other.inner()); - } - } else struct {}; - - pub usingnamespace if (Methods.deinit_fn) |_deinit| struct { - pub inline fn deinit(self: *OuterT) void { - _deinit(self.inner()); - self.* = undefined; - } - } else struct {}; - - pub usingnamespace if (Methods.dump_fn) |_dump| struct { - pub inline fn dump(self: OuterT) void { - return _dump(self.inner()); - } - } else struct {}; - - pub usingnamespace if (Methods.print_fn) |print| struct { - pub fn format( - self: OuterT, - comptime fmt: []const u8, - options: std.fmt.FormatOptions, - writer: anytype, - ) !void { - _ = fmt; - _ = options; - - const Writer = struct { - writer: @TypeOf(writer), - err: ?@TypeOf(writer).Error = null, - fn printCallback(mlir_str: c.MlirStringRef, opaque_ctx: ?*anyopaque) callconv(.C) void { - var ctx: *@This() = @alignCast(@ptrCast(opaque_ctx)); - if (ctx.err) |_| return; - _ = ctx.writer.write(mlir_str.data[0..mlir_str.length]) catch |err| { - ctx.err = err; - return; - }; + pub fn format(f: anytype) fn (self: *const T, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return struct { + fn format(self: *const T, writer: *std.Io.Writer) std.Io.Writer.Error!void { + var ctx: StringCallbackCtx = .{ .writer = writer }; + f(self.ptr(), stringCallback, &ctx); + if (ctx.err) |err| { + return err; } - }; - - var context: Writer = .{ .writer = writer }; - print(self.inner(), &Writer.printCallback, &context); - if (context.err) |err| return err; - } - } else struct {}; - }; -} - -pub const Registry = struct { - _inner: c.MlirDialectRegistry, - pub usingnamespace MlirHelpers(Registry, .{ - .is_null_fn = c.mlirDialectRegistryIsNull, - .deinit_fn = c.mlirDialectRegistryDestroy, - }); - const Self = Registry; - - pub fn init() !Self { - return Self.wrapOr(c.mlirDialectRegistryCreate()) orelse Error.MlirUnexpected; - } -}; - -pub const Context = struct { - _inner: c.MlirContext, - pub usingnamespace MlirHelpers(Context, .{ - .is_null_fn = c.mlirContextIsNull, - .deinit_fn = c.mlirContextDestroy, - }); - const Self = Context; - - pub fn init() !Self { - return Self.wrapOr(c.mlirContextCreate()) orelse Error.MlirUnexpected; - } - - pub fn initWithRegistry(registry: Registry, threadingEnabled: bool) !Self { - return Self.wrapOr( - c.mlirContextCreateWithRegistry(registry.inner(), threadingEnabled), - ) orelse Error.InvalidMlir; - } - - pub fn setMultiThreading(self: *Self, enabled: bool) void { - c.mlirContextEnableMultithreading(self.inner(), enabled); - } - - pub fn appendDialectRegistry(self: *Self, registry: Registry) void { - c.mlirContextAppendDialectRegistry(self.inner(), registry.inner()); - } - - pub fn loadAllAvailableDialects(self: *Self) void { - c.mlirContextLoadAllAvailableDialects(self.inner()); - } - - pub fn numRegisteredDialects(self: Self) usize { - return @intCast(c.mlirContextGetNumRegisteredDialects(self.inner())); - } - - pub fn numLoadedDialects(self: Self) usize { - return @intCast(c.mlirContextGetNumLoadedDialects(self.inner())); - } - - pub fn isRegisteredOperation(self: Self, op: [:0]const u8) bool { - return c.mlirContextIsRegisteredOperation(self.inner(), stringRef(op)); - } - - pub fn location(self: Self, src: std.builtin.SourceLocation) Location { - return Location.fromSrc(self, src); - } -}; - -pub const Module = struct { - _inner: c.MlirModule, - pub usingnamespace MlirHelpers(Module, .{ - .is_null_fn = c.mlirModuleIsNull, - .deinit_fn = c.mlirModuleDestroy, - }); - const Self = Module; - - pub fn init(loc: Location) Self { - return Self.wrap(c.mlirModuleCreateEmpty(loc.inner())); - } - - pub fn parse(ctx: Context, source: [:0]const u8) !Module { - return Module.wrapOr( - c.mlirModuleCreateParse(ctx.inner(), stringRef(source)), - ) orelse Error.InvalidMlir; - } - - pub fn fromOperation(operation: Operation) Module { - return Module.wrap(c.mlirModuleFromOperation(operation.inner())); - } - - pub fn context(self: Module) Context { - return Context.wrap(c.mlirModuleGetContext(self.inner())); - } - - pub fn getBody(self: Module) Block { - return Block.wrap(c.mlirModuleGetBody(self.inner())); - } - - pub fn op(self: Module) Operation { - return Operation.wrap(c.mlirModuleGetOperation(self.inner())); - } -}; - -pub const PassManager = struct { - _inner: c.MlirPassManager, - - pub usingnamespace MlirHelpers(PassManager, .{ - .is_null_fn = c.mlirPassManagerIsNull, - .deinit_fn = c.mlirPassManagerDestroy, - }); - const Self = PassManager; - - pub fn init(ctx: Context) !Self { - return Self.wrapOr( - c.mlirPassManagerCreate(ctx.inner()), - ) orelse Error.MlirUnexpected; - } - - pub fn initOnOperation(ctx: Context, op: [:0]const u8) !Self { - return Self.wrapOr( - c.mlirPassManagerCreateOnOperation(ctx.inner(), stringRef(op)), - ) orelse Error.MlirUnexpected; - } - - pub fn asOpPassManager(self: Self) OpPassManager { - return OpPassManager.wrap(c.mlirPassManagerGetAsOpPassManager(self.inner())); - } - - pub fn enableIRPrinting(self: *Self) void { - c.mlirPassManagerEnableIRPrinting(self.inner()); - } - - pub fn runOnOp(self: *Self, op: Operation) error{InvalidMlir}!void { - if (c.mlirPassManagerRunOnOp(self.inner(), op.inner()).value == 0) { - return Error.InvalidMlir; - } - } -}; - -fn _mlir_passpipeline_error(err: c.MlirStringRef, ctx: ?*anyopaque) callconv(.C) void { - _ = ctx; - std.debug.print(">>ERROR: {s}\n", .{err.data}); -} - -pub const OpPassManager = struct { - _inner: c.MlirOpPassManager, - pub usingnamespace MlirHelpers(OpPassManager, .{}); - - pub fn addPipeline(self: *OpPassManager, pipeline: [:0]const u8) error{OutOfMemory}!void { - if (c.mlirOpPassManagerAddPipeline( - self.inner(), - stringRef(pipeline), - &_mlir_passpipeline_error, - null, - ).value == 0) { - return Error.OutOfMemory; - } - } -}; - -pub const Identifier = struct { - _inner: c.MlirIdentifier, - pub usingnamespace MlirHelpers(Identifier, .{}); - const Self = Identifier; - - pub fn get(ctx: Context, str_: [:0]const u8) Self { - return Self.wrap(c.mlirIdentifierGet(ctx.inner(), stringRef(str_))); - } - - pub fn context(self: Self) Context { - return Context.wrap(c.mlirIdentifierGetContext(self.inner())); - } - - pub fn str(self: Self) []const u8 { - return fromStringRef(c.mlirIdentifierStr(self.inner())); - } - - pub fn equals(self: Self, other: Self) bool { - return c.mlirIdentifierEqual(self.inner(), other.inner()); - } -}; - -pub const Attribute = struct { - _inner: c.MlirAttribute, - pub usingnamespace MlirHelpers(Attribute, .{ - .is_null_fn = c.mlirAttributeIsNull, - .dump_fn = c.mlirAttributeDump, - .equal_fn = c.mlirAttributeEqual, - }); - const Self = Attribute; - - pub fn parse(ctx: Context, attr: [:0]const u8) !Attribute { - return Attribute.wrapOr( - c.mlirAttributeParseGet(ctx.inner(), stringRef(attr)), - ) orelse Error.InvalidMlir; - } - - pub fn getNull() Self { - return Self.wrap(c.mlirAttributeGetNull()); - } -}; - -pub const NamedAttribute = struct { - _inner: c.MlirNamedAttribute, - pub usingnamespace MlirHelpers(NamedAttribute, .{}); - const Self = NamedAttribute; - - pub fn init(name: Identifier, attr: Attribute) Self { - return Self.wrap(.{ - .name = name.inner(), - .attribute = attr.inner(), - }); - } -}; - -pub const StringAttribute = struct { - _inner: c.MlirAttribute, - pub usingnamespace MlirHelpers(StringAttribute, .{ - .is_a_fn = c.mlirAttributeIsAString, - .is_null_fn = c.mlirAttributeIsNull, - .dump_fn = c.mlirAttributeDump, - .equal_fn = c.mlirAttributeEqual, - }); - const Self = StringAttribute; - - pub fn init(ctx: Context, str: []const u8) Self { - return Self.wrap(c.mlirStringAttrGet(ctx.inner(), stringRef(str))); - } - - pub fn value(self: Self) []const u8 { - return fromStringRef(c.mlirStringAttrGetValue(self.inner())); - } - - pub fn asAttr(self: StringAttribute) Attribute { - return .{ ._inner = self._inner }; - } -}; - -pub const UnitAttribute = struct { - _inner: c.MlirAttribute, - pub usingnamespace MlirHelpers(UnitAttribute, .{ - .is_a_fn = c.mlirAttributeIsAUnit, - .is_null_fn = c.mlirAttributeIsNull, - .dump_fn = c.mlirAttributeDump, - .equal_fn = c.mlirAttributeEqual, - }); - const Self = UnitAttribute; - - pub fn init(ctx: Context) Self { - return Self.wrap(c.mlirUnitAttrGet(ctx.inner())); - } -}; - -pub const BoolAttribute = struct { - _inner: c.MlirAttribute, - pub usingnamespace MlirHelpers(BoolAttribute, .{ - .is_a_fn = c.mlirAttributeIsABool, - .is_null_fn = c.mlirAttributeIsNull, - .dump_fn = c.mlirAttributeDump, - .equal_fn = c.mlirAttributeEqual, - }); - const Self = BoolAttribute; - - pub fn init(ctx: Context, value_: bool) Self { - return Self.wrap(c.mlirBoolAttrGet(ctx.inner(), if (value_) 1 else 0)); - } - - pub fn value(self: Self) bool { - return c.mlirBoolAttrGetValue(self.inner()); - } - - pub fn asAttr(self: Self) Attribute { - return self.as(Attribute).?; - } -}; - -pub const TypeAttribute = struct { - _inner: c.MlirAttribute, - pub usingnamespace MlirHelpers(TypeAttribute, .{ - .is_a_fn = c.mlirAttributeIsAType, - .is_null_fn = c.mlirAttributeIsNull, - .dump_fn = c.mlirAttributeDump, - .equal_fn = c.mlirAttributeEqual, - }); - const Self = TypeAttribute; - - pub fn init(type_: Type) Self { - return Self.wrap(c.mlirTypeAttrGet(type_.inner())); - } - - pub fn typ(self: Self) Type { - return Type.wrap(c.mlirAttributeGetType(self.inner())); - } -}; - -pub const ArrayAttribute = struct { - _inner: c.MlirAttribute, - pub usingnamespace MlirHelpers(ArrayAttribute, .{ - .is_a_fn = c.mlirAttributeIsAArray, - .is_null_fn = c.mlirAttributeIsNull, - .dump_fn = c.mlirAttributeDump, - .equal_fn = c.mlirAttributeEqual, - }); - const Self = ArrayAttribute; - - pub fn init(ctx: Context, attrs: []const Attribute) Self { - return Self.wrap(c.mlirArrayAttrGet(ctx.inner(), @intCast(attrs.len), @ptrCast(attrs.ptr))); - } - - pub fn size(self: Self) usize { - return @intCast(c.mlirArrayAttrGetNumElements(self.inner())); - } - - pub fn get(self: Self, index: usize) Attribute { - return Attribute.wrap(c.mlirArrayAttrGetElement(self.inner(), @intCast(index))); - } - - pub fn asAttr(self: Self) Attribute { - return .{ ._inner = self._inner }; - } -}; - -pub fn IntegerAttribute(comptime it: IntegerTypes) type { - const ZigType, const getter = comptime switch (it) { - .i1, .i4, .i8, .i16, .i32, .i64 => .{ u64, c.mlirIntegerAttrGetValueInt }, - .si4, .si8, .si16, .si32, .si64 => .{ u64, c.mlirIntegerAttrGetValueSInt }, - .u4, .u8, .u16, .u32, .u64 => .{ u64, c.mlirIntegerAttrGetValueUInt }, - .unknown => @compileError("IntegerAttribute(unknown)"), - }; - - return struct { - _inner: c.MlirAttribute, - pub usingnamespace MlirHelpers(@This(), .{ - .is_a_fn = c.mlirAttributeIsAInteger, - .is_null_fn = c.mlirAttributeIsNull, - .dump_fn = c.mlirAttributeDump, - .equal_fn = c.mlirAttributeEqual, - }); - pub const IntegerTypeType = IntegerType(it); - const IntAttr = @This(); - - pub fn init(ctx: Context, value: i64) IntAttr { - return IntAttr.wrap(c.mlirIntegerAttrGet( - IntegerType(it).init(ctx).inner(), - value, - )); - } - - pub fn get(value: IntAttr) ZigType { - return @intCast(getter(value.inner())); - } - - pub fn asAttr(self: IntAttr) Attribute { - return .{ ._inner = self._inner }; - } - }; -} - -pub fn FloatAttribute(comptime ft: FloatTypes) type { - return struct { - _inner: c.MlirAttribute, - pub usingnamespace MlirHelpers(@This(), .{ - .is_a_fn = c.mlirAttributeIsAFloat, - .is_null_fn = c.mlirAttributeIsNull, - .dump_fn = c.mlirAttributeDump, - .equal_fn = c.mlirAttributeEqual, - }); - const FloatAttr = @This(); - pub fn init(ctx: Context, value: f64) FloatAttr { - return FloatAttr.wrap(c.mlirFloatAttrDoubleGet( - ctx.inner(), - FloatType(ft).init(ctx).inner(), - value, - )); - } - - pub fn get(value: FloatAttr) f64 { - return c.mlirFloatAttrGetValueDouble(value.inner()); - } - }; -} - -pub const DenseArrayTypes = enum { - bool, - i8, - i16, - i32, - i64, - f32, - f64, -}; - -pub fn DenseArrayAttribute(comptime dt: DenseArrayTypes) type { - const Config = switch (dt) { - .bool => .{ i32, c.mlirAttributeIsADenseBoolArray, c.mlirDenseBoolArrayGet, c.mlirDenseBoolArrayGetElement }, - .i8 => .{ i8, c.mlirAttributeIsADenseI8Array, c.mlirDenseI8ArrayGet, c.mlirDenseI8ArrayGetElement }, - .i16 => .{ i16, c.mlirAttributeIsADenseI16Array, c.mlirDenseI16ArrayGet, c.mlirDenseI16ArrayGetElement }, - .i32 => .{ i32, c.mlirAttributeIsADenseI32Array, c.mlirDenseI32ArrayGet, c.mlirDenseI32ArrayGetElement }, - .i64 => .{ i64, c.mlirAttributeIsADenseI64Array, c.mlirDenseI64ArrayGet, c.mlirDenseI64ArrayGetElement }, - .f32 => .{ f32, c.mlirAttributeIsADenseF32Array, c.mlirDenseF32ArrayGet, c.mlirDenseF32ArrayGetElement }, - .f64 => .{ f64, c.mlirAttributeIsADenseF64Array, c.mlirDenseF64ArrayGet, c.mlirDenseF64ArrayGetElement }, - }; - - return struct { - _inner: c.MlirAttribute, - pub usingnamespace MlirHelpers(@This(), .{ - .is_a_fn = Config[1], - .is_null_fn = c.mlirAttributeIsNull, - .dump_fn = c.mlirAttributeDump, - .equal_fn = c.mlirAttributeEqual, - }); - const Attr = @This(); - const ElementType = dt; - const ElementTypeZig = Config[0]; - - pub fn init(ctx: Context, values: []const ElementTypeZig) Attr { - const get_fn = Config[2]; - return Attr.wrap(get_fn(ctx.inner(), @intCast(values.len), @ptrCast(values.ptr))); - } - - pub fn get(self: Attr, pos: usize) ElementTypeZig { - const get_element_fn = Config[3]; - return get_element_fn(self.inner(), @intCast(pos)); - } - - pub fn len(self: Attr) usize { - return @intCast(c.mlirDenseArrayGetNumElements(self.inner())); - } - - pub usingnamespace switch (dt) { - .bool, .i64 => struct { - const DenseArray = DenseArrayAttribute(switch (dt) { - .bool => .bool, - .i64 => .i64, - else => @compileError("DenseArrayAttribute: unreachable"), - }); - - pub fn toElements(self: Attr) DenseArray { - return DenseArray.wrap(c.mlirDenseArrayToElements(self.inner())); } - }, - else => struct {}, - }; + }.format; + } + + pub fn eql(f: anytype) fn (self: *const T, other: *const T) bool { + return struct { + fn eql(self: *const T, other: *const T) bool { + return f(self.ptr(), other.ptr()); + } + }.eql; + } + + pub fn isA(self: *const T, OtherType: type) ?CastedPtr(@TypeOf(self), OtherType) { + return if (OtherType.isAFn(self.ptr())) @ptrCast(self) else null; + } }; } -pub const DenseElementsAttributeTypes = enum { - bool, +pub const DialectRegistry = opaque { + const M = Methods(DialectRegistry, c.MlirDialectRegistry); + + pub const ptr = M.ptr; + + pub fn init() std.mem.Allocator.Error!*DialectRegistry { + return @ptrCast(c.mlirDialectRegistryCreate().ptr orelse return std.mem.Allocator.Error.OutOfMemory); + } + + pub fn deinit(self: *DialectRegistry) void { + c.mlirDialectRegistryDestroy(self.ptr()); + } + + pub fn registerDialect(self: *DialectRegistry, comptime name: []const u8) void { + DialectHandle.fromString(name).insertDialect(self); + } +}; + +pub const PassManager = opaque { + const M = Methods(PassManager, c.MlirPassManager); + + pub const ptr = M.ptr; + + pub fn init(ctx: *Context) *PassManager { + return @ptrCast(c.mlirPassManagerCreate(ctx.ptr()).ptr); + } + + pub fn deinit(self: *const PassManager) void { + c.mlirPassManagerDestroy(self.ptr()); + } + + pub fn asOpPassManager(self: *PassManager) *OpPassManager { + return @ptrCast(c.mlirPassManagerGetAsOpPassManager(self.ptr()).ptr); + } + + pub fn runOnOp(self: *const PassManager, op: *Operation) !void { + const result = c.mlirPassManagerRunOnOp(self.ptr(), op.ptr()); + if (result.value == 0) { + return error.MlirUnexpected; + } + } + + pub const EnableIRPrintingOpts = struct { + printBeforeAll: bool = false, + printAfterAll: bool = false, + printModuleScope: bool = false, + printAfterOnlyOnChange: bool = false, + printAfterOnlyOnFailure: bool = false, + flags: Operation.PrintFlags = .{}, + treePrintingPath: []const u8 = "", + }; + + pub fn enableIRPrinting(self: *PassManager, opts: EnableIRPrintingOpts) void { + const flags = opts.flags.createNative(); + defer c.mlirOpPrintingFlagsDestroy(flags); + + c.mlirPassManagerEnableIRPrinting( + self.ptr(), + opts.printBeforeAll, + opts.printAfterAll, + opts.printModuleScope, + opts.printAfterOnlyOnChange, + opts.printAfterOnlyOnFailure, + flags, + stringRef(opts.treePrintingPath), + ); + } +}; + +pub const OpPassManager = opaque { + const M = Methods(OpPassManager, c.MlirOpPassManager); + + pub const ptr = M.ptr; + pub const format = M.format(c.mlirPrintPassPipeline); + + pub fn addPass(self: *OpPassManager, pass: *Pass) void { + c.mlirOpPassManagerAddOwnedPass(self.ptr(), pass.ptr()); + } + + pub fn addPasses(self: *OpPassManager, passes: []const *Pass) void { + for (passes) |pass| { + self.addPass(pass); + } + } + + pub fn addPipeline(self: *OpPassManager, pipeline_name: []const u8) !void { + var buffer: [1024]u8 = undefined; + const result = stringFromStream(&buffer, c.mlirOpPassManagerAddPipeline, .{ self.ptr(), stringRef(pipeline_name) }); + if (result.len != 0) { + log.err("mlir error: {s}", .{result}); + return error.MlirUnexpected; + } + } +}; + +pub const Pass = opaque { + const M = Methods(Pass, c.MlirPass); + + pub const ptr = M.ptr; + + pub fn create(comptime name: @EnumLiteral()) *Pass { + return @ptrCast(@field(c, "mlirCreate" ++ @tagName(name))().ptr); + } +}; + +pub const Context = opaque { + const M = Methods(Context, c.MlirContext); + + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirContextEqual); + + pub const InitOptions = struct { + registry: ?*const DialectRegistry = null, + threading: bool = false, + }; + + pub fn init(opts: InitOptions) std.mem.Allocator.Error!*Context { + const ctx = if (opts.registry) |r| c.mlirContextCreateWithRegistry( + r.ptr(), + opts.threading, + ) else c.mlirContextCreateWithThreading(opts.threading); + return @ptrCast(ctx.ptr orelse return std.mem.Allocator.Error.OutOfMemory); + } + + pub fn deinit(self: *Context) void { + c.mlirContextDestroy(self.ptr()); + } + + pub fn setMultiThreading(self: *Context, enabled: bool) void { + c.mlirContextEnableMultithreading(self.ptr(), enabled); + } + + pub fn appendDialectRegistry(self: *Context, registry: *const DialectRegistry) void { + c.mlirContextAppendDialectRegistry(self.ptr(), registry.ptr()); + } + + pub fn loadAllAvailableDialects(self: *Context) void { + c.mlirContextLoadAllAvailableDialects(self.ptr()); + } + + pub fn numRegisteredDialects(self: *const Context) usize { + return @intCast(c.mlirContextGetNumRegisteredDialects(self.ptr())); + } + + pub fn numLoadedDialects(self: *const Context) usize { + return @intCast(c.mlirContextGetNumLoadedDialects(self.ptr())); + } + + pub fn numThreads(self: *const Context) usize { + return @intCast(c.mlirContextGetNumThreads(self.ptr())); + } + + pub fn isRegisteredOperation(self: *const Context, op: []const u8) bool { + return c.mlirContextIsRegisteredOperation(self.ptr(), stringRef(op)); + } + + pub fn setAllowUnregisteredDialects(self: *Context, allow: bool) void { + c.mlirContextSetAllowUnregisteredDialects(self.ptr(), allow); + } + + pub fn allowUnregisteredDialects(self: *const Context) bool { + return c.mlirContextGetAllowUnregisteredDialects(self.ptr()); + } + + pub fn format(self: *const Context, writer: *std.Io.Writer) !void { + try writer.print("{s}{{ .numThreads = {d}, .numRegisteredDialects = {d}, .numLoadedDialects = {d} }}", .{ + @typeName(Context), + self.numThreads(), + self.numRegisteredDialects(), + self.numLoadedDialects(), + }); + } +}; + +pub const Location = opaque { + const M = Methods(Location, c.MlirLocation); + + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirLocationEqual); + pub const format = M.format(c.mlirLocationPrint); + + pub fn fromSrc(ctx: *Context, src: std.builtin.SourceLocation) *const Location { + return fileLineCol( + ctx, + src.file, + src.line, + src.column, + ); + } + + pub fn fileLineCol(ctx: *Context, file: []const u8, line: usize, column: usize) *const Location { + return @ptrCast(c.mlirLocationFileLineColGet( + ctx.ptr(), + stringRef(file), + @intCast(line), + @intCast(column), + ).ptr); + } + + pub fn callSite(callee: *const Location, caller: *const Location) *const Location { + return @ptrCast(c.mlirLocationCallSiteGet(callee.ptr(), caller.ptr()).ptr); + } + + pub fn fused(ctx: *Context, locs: []const *const Location, metadata: *const Attribute) *const Location { + return @ptrCast(c.mlirLocationFusedGet( + ctx.ptr(), + @intCast(locs.len), + @ptrCast(locs), + metadata.ptr(), + ).ptr); + } + + pub fn named(self: *const Location, ctx: *Context, loc_name: []const u8) *const Location { + return @ptrCast(c.mlirLocationNameGet(ctx.ptr(), stringRef(loc_name), self.ptr()).ptr); + } + + pub fn namedFmt(loc: *const Location, ctx: *Context, comptime fmt: []const u8, args: anytype) *const Location { + var buf: [256]u8 = undefined; + return loc.named(ctx, std.fmt.bufPrint(&buf, fmt, args) catch blk: { + buf[buf.len - 3 ..].* = "...".*; + break :blk &buf; + }); + } + + pub fn fromAttribute(attribute: *const Attribute) *const Location { + return @ptrCast(c.mlirLocationFromAttribute(attribute.ptr()).ptr); + } + + pub fn unknown(ctx: *Context) *const Location { + return @ptrCast(c.mlirLocationUnknownGet(ctx.ptr()).ptr); + } + + pub const Tagged = union(enum) { + unknown, + named: *const Named, + callsite: *const Callsite, + file_line_col: *const FileLineCol, + fused: *const Fused, + }; + + pub fn inspect(location: *const Location) Tagged { + const loc_ptr = location.ptr(); + return if (c.mlirLocationIsAUnknown(loc_ptr)) + .unknown + else if (c.mlirLocationIsAName(loc_ptr)) + .{ .named = @ptrCast(location) } + else if (c.mlirLocationIsACallSite(loc_ptr)) + .{ .callsite = @ptrCast(location) } + else if (c.mlirLocationIsAFileLineColRange(loc_ptr)) + .{ .file_line_col = @ptrCast(location) } + else if (c.mlirLocationIsAFused(loc_ptr)) + .{ .fused = @ptrCast(location) } + else + // Theoritically the C++ API has "OpaqueLoc" but it's not part of C bindings, + // I think in this case it's fine to fall back to .unknown + // https://mlir.llvm.org/docs/Dialects/Builtin/#opaqueloc + .unknown; + } + + pub const Named = opaque { + pub fn name(location: *const Named) []const u8 { + const identifier: *const Identifier = @ptrCast(c.mlirLocationNameGetName(.{ .ptr = location }).ptr); + return identifier.str(); + } + + pub fn childLoc(location: *const Named) *const Location { + return @ptrCast(c.mlirLocationNameGetChildLoc(.{ .ptr = location }).ptr); + } + }; + + pub const Callsite = opaque { + pub fn callee(location: *const Callsite) *const Location { + return @ptrCast(c.mlirLocationCallSiteGetCallee(.{ .ptr = location }).ptr); + } + + pub fn caller(location: *const Callsite) *const Location { + return @ptrCast(c.mlirLocationCallSiteGetCaller(.{ .ptr = location }).ptr); + } + }; + + pub const FileLineCol = opaque { + pub fn src(location: *const FileLineCol) *const std.builtin.SourceLocation { + const file_id: *const Identifier = @ptrCast(c.mlirLocationFileLineColRangeGetFilename(.{ .ptr = location }).ptr); + return .{ + .file = file_id.str(), + .line = @truncate(c.mlirLocationFileLineColRangeGetStartLine(.{ .ptr = location })), + .column = @truncate(c.mlirLocationFileLineColRangeGetStartColumn(.{ .ptr = location })), + }; + } + }; + + pub const Fused = opaque { + pub fn len(location: *const Fused) u32 { + return @intCast(c.mlirLocationFusedGetNumLocations(.{ .ptr = location })); + } + + pub fn locations(location: *const Fused, allocator: std.mem.Allocator) error{OutOfMemory}![]*const Location { + const result = try allocator.alloc(*const Location, location.len()); + c.mlirLocationFusedGetLocations(.{ .ptr = location }, @ptrCast(result.ptr)); + return result; + } + }; +}; + +pub const Type = opaque { + const M = Methods(Type, c.MlirType); + + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + + pub fn parse(ctx: *Context, str: []const u8) Error!*const Type { + return @ptrCast(c.mlirTypeParseGet(ctx.ptr(), stringRef(str)).ptr orelse return Error.InvalidMlir); + } + + pub fn isFloat(self: *const Type) bool { + return c.mlirTypeIsAFloat(self.ptr()); + } + + pub fn index(ctx: *Context) *const Type { + return @ptrCast(IndexType.get(ctx)); + } + + pub fn int(ctx: *Context, it: IntegerTypes) *const Type { + return @ptrCast(IntegerType.get(ctx, it)); + } + + pub fn float(ctx: *Context, ft: FloatTypes) *const Type { + return switch (ft) { + inline else => |v| @ptrCast(FloatType(v).get(ctx)), + }; + } + + pub fn complex(ctx: *Context, complex_type: ComplexTypes) *const Type { + return @ptrCast(ComplexType.get(ctx, complex_type)); + } + + pub fn rankedTensor(dimensions: []const i64, elem_type: *const Type) *const Type { + return @ptrCast(RankedTensorType.get(dimensions, elem_type, null)); + } + + pub fn rankedTensorWithEncoding(dimensions: []const i64, elem_type: *const Type, encoding: *Attribute) *const Type { + return @ptrCast(RankedTensorType.get(dimensions, elem_type, encoding)); + } + + pub fn function(ctx: *Context, args: []const *const Type, results: []const *const Type) *const Type { + return @ptrCast(FunctionType.init(ctx, args, results)); + } + + pub fn memRef(element_type: *const Type, shape: []const i64, layout: ?*const Attribute, memory_space: ?*const Attribute) *const Type { + return @ptrCast(MemRefType.init(element_type, shape, layout, memory_space)); + } + + pub fn element(self: *const VectorType) *const Type { + return @ptrCast(c.mlirShapedTypeGetElementType(self.ptr()).ptr); + } + + pub fn vector(shape: []const i64, element_type: *const Type) *const Type { + return @ptrCast(VectorType.init(shape, element_type)); + } + + pub fn unrankedMemRef(element_type: *const Type, memory_space: ?*const Attribute) *const Type { + return @ptrCast(UnrankedMemRefType.init(element_type, memory_space)); + } +}; + +pub const IndexType = opaque { + const M = Methods(IndexType, c.MlirType); + + pub const isAFn = c.mlirTypeIsAIndex; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + + pub fn get(ctx: *Context) *const IndexType { + return @ptrCast(c.mlirIndexTypeGet(ctx.ptr()).ptr); + } +}; + +pub const IntegerTypes = enum { + i1, + i2, + i4, i8, i16, i32, i64, + i128, + u1, + u2, + u4, u8, u16, u32, u64, + u128, + si1, + si4, + si8, + si16, + si32, + si64, + si128, + + pub fn bitwidth(self: IntegerTypes) usize { + return switch (self) { + .i1, .u1, .si1 => 1, + .i2, .u2 => 2, + .i4, .u4, .si4 => 4, + .i8, .u8, .si8 => 8, + .i16, .u16, .si16 => 16, + .i32, .u32, .si32 => 32, + .i64, .u64, .si64 => 64, + .i128, .u128, .si128 => 128, + }; + } + + pub fn signedness(self: IntegerTypes) IntegerType.Signedness { + return switch (self) { + .i1, .i2, .i4, .i8, .i16, .i32, .i64, .i128 => .signless, + .si1, .si4, .si8, .si16, .si32, .si64, .si128 => .signed, + .u1, .u2, .u4, .u8, .u16, .u32, .u64, .u128 => .unsigned, + }; + } +}; + +pub const IntegerType = opaque { + pub const Signedness = enum { + signless, + signed, + unsigned, + }; + + const M = Methods(IntegerType, c.MlirType); + + pub const isAFn = c.mlirTypeIsAInteger; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + + pub fn width(self: *const IntegerType) u32 { + return c.mlirIntegerTypeGetWidth(self.ptr()); + } + + fn get(ctx: *Context, it: IntegerTypes) *const IntegerType { + return exact(ctx, it.bitwidth(), it.signedness()); + } + + fn exact(ctx: *Context, bitwidth: usize, sign: Signedness) *const IntegerType { + return @ptrCast(switch (sign) { + .signless => c.mlirIntegerTypeGet(ctx.ptr(), @intCast(bitwidth)).ptr, + .signed => c.mlirIntegerTypeSignedGet(ctx.ptr(), @intCast(bitwidth)).ptr, + .unsigned => c.mlirIntegerTypeUnsignedGet(ctx.ptr(), @intCast(bitwidth)).ptr, + }); + } +}; + +pub const FloatTypes = enum { + f4e2m1fn, + f6e2m3fn, + f6e3m2fn, + f8e3m4, + f8e4m3, + f8e4m3b11fnuz, + f8e4m3fn, + f8e4m3fnuz, + f8e5m2, + f8e5m2fnuz, + f8e8m0fnu, bf16, f16, f32, f64, + // f80, + // f128, + tf32, }; -pub fn DenseIntOrFPElementsAttribute(comptime dt: DenseElementsAttributeTypes) type { - const ZigInDataType, const ZigOutDataType, const initFn, const getValue = switch (dt) { - .bool => .{ i32, bool, c.mlirDenseElementsAttrBoolGet, c.mlirDenseElementsAttrGetBoolValue }, - .i8 => .{ i8, i8, c.mlirDenseElementsAttrInt8Get, c.mlirDenseElementsAttrGetInt8Value }, - .i16 => .{ i16, i16, c.mlirDenseElementsAttrInt16Get, c.mlirDenseElementsAttrGetInt16Value }, - .i32 => .{ i32, i32, c.mlirDenseElementsAttrInt32Get, c.mlirDenseElementsAttrGetInt32Value }, - .i64 => .{ i64, i64, c.mlirDenseElementsAttrInt64Get, c.mlirDenseElementsAttrGetInt64Value }, - .u8 => .{ u8, u8, c.mlirDenseElementsAttrUInt8Get, c.mlirDenseElementsAttrGetUInt8Value }, - .u16 => .{ u16, u16, c.mlirDenseElementsAttrUInt16Get, c.mlirDenseElementsAttrGetUInt16Value }, - .u32 => .{ u32, u32, c.mlirDenseElementsAttrUInt32Get, c.mlirDenseElementsAttrGetUInt32Value }, - .u64 => .{ u64, u64, c.mlirDenseElementsAttrUInt64Get, c.mlirDenseElementsAttrGetUInt64Value }, - .bf16 => .{ u16, f32, c.mlirDenseElementsAttrBFloat16Get, c.mlirDenseElementsAttrGetFloatValue }, - .f16 => .{ f16, f32, c.mlirDenseElementsAttrFloat16Get, c.mlirDenseElementsAttrGetFloatValue }, - .f32 => .{ f32, f32, c.mlirDenseElementsAttrFloatGet, c.mlirDenseElementsAttrGetFloatValue }, - .f64 => .{ f64, f64, c.mlirDenseElementsAttrDoubleGet, c.mlirDenseElementsAttrGetDoubleValue }, +pub fn FloatType(comptime ft: FloatTypes) type { + const _isAFn, const getter = switch (ft) { + .f4e2m1fn => .{ c.mlirTypeIsAFloat4E2M1FN, c.mlirFloat4E2M1FNTypeGet }, + .f6e2m3fn => .{ c.mlirTypeIsAFloat6E2M3FN, c.mlirFloat6E2M3FNTypeGet }, + .f6e3m2fn => .{ c.mlirTypeIsAFloat6E3M2FN, c.mlirFloat6E3M2FNTypeGet }, + .f8e3m4 => .{ c.mlirTypeIsAFloat8E3M4, c.mlirFloat8E3M4TypeGet }, + .f8e4m3 => .{ c.mlirTypeIsAFloat8E4M3, c.mlirFloat8E4M3TypeGet }, + .f8e4m3b11fnuz => .{ c.mlirTypeIsAFloat8E4M3B11FNUZ, c.mlirFloat8E4M3B11FNUZTypeGet }, + .f8e4m3fn => .{ c.mlirTypeIsAFloat8E4M3FN, c.mlirFloat8E4M3FNTypeGet }, + .f8e4m3fnuz => .{ c.mlirTypeIsAFloat8E4M3FNUZ, c.mlirFloat8E4M3FNUZTypeGet }, + .f8e5m2 => .{ c.mlirTypeIsAFloat8E5M2, c.mlirFloat8E5M2TypeGet }, + .f8e5m2fnuz => .{ c.mlirTypeIsAFloat8E5M2FNUZ, c.mlirFloat8E5M2FNUZTypeGet }, + .f8e8m0fnu => .{ c.mlirTypeIsAFloat8E8M0FNU, c.mlirFloat8E8M0FNUTypeGet }, + .bf16 => .{ c.mlirTypeIsABF16, c.mlirBF16TypeGet }, + .f16 => .{ c.mlirTypeIsAF16, c.mlirF16TypeGet }, + .f32 => .{ c.mlirTypeIsAF32, c.mlirF32TypeGet }, + .f64 => .{ c.mlirTypeIsAF64, c.mlirF64TypeGet }, + .tf32 => .{ c.mlirTypeIsATF32, c.mlirTF32TypeGet }, }; - return struct { - _inner: c.MlirAttribute, - const Attr = @This(); - pub usingnamespace MlirHelpers(Attr, .{ - .is_a_fn = c.mlirAttributeIsADenseElements, - .is_null_fn = c.mlirAttributeIsNull, - .dump_fn = c.mlirAttributeDump, - .equal_fn = c.mlirAttributeEqual, - }); + return opaque { + const Self = @This(); + const M = Methods(Self, c.MlirType); - pub fn init(shaped_type: Type, raw_values: []const u8) Attr { - const values = std.mem.bytesAsSlice(ZigInDataType, raw_values); - return Attr.wrap(initFn(shaped_type.inner(), @intCast(values.len), @ptrCast(@alignCast(values.ptr)))); - } + pub const isAFn = _isAFn; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); - pub fn get(self: Attr, pos: usize) ZigOutDataType { - return getValue(self.inner(), @intCast(pos)); + pub fn get(ctx: *Context) *const Self { + return @ptrCast(getter(ctx.ptr()).ptr); } }; } -pub const FlatSymbolRefAttribute = struct { - _inner: c.MlirAttribute, - pub usingnamespace MlirHelpers(FlatSymbolRefAttribute, .{ - .is_a_fn = c.mlirAttributeIsAFlatSymbolRef, - .is_null_fn = c.mlirAttributeIsNull, - .dump_fn = c.mlirAttributeDump, - .equal_fn = c.mlirAttributeEqual, - }); +pub const ComplexTypes = enum { + c64, + c128, +}; - const Self = FlatSymbolRefAttribute; +pub const ComplexType = opaque { + const M = Methods(ComplexType, c.MlirType); - pub fn init(ctx: Context, str: [:0]const u8) Self { - return Self.wrap(c.mlirFlatSymbolRefAttrGet(ctx.inner(), stringRef(str))); + pub const isAFn = c.mlirTypeIsAComplex; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + + pub fn get(ctx: *Context, complex_type: ComplexTypes) *const ComplexType { + return @ptrCast(c.mlirComplexTypeGet(switch (complex_type) { + .c64 => Type.float(ctx, .f32).ptr(), + .c128 => Type.float(ctx, .f64).ptr(), + }).ptr); + } +}; + +pub const Attribute = opaque { + const M = Methods(Attribute, c.MlirAttribute); + + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + pub const isA = M.isA; + + pub fn parse(ctx: *Context, attr: []const u8) Error!*const Attribute { + return @ptrCast(c.mlirAttributeParseGet(ctx.ptr(), stringRef(attr)).ptr orelse return Error.InvalidMlir); } - pub fn value(self: Self) []const u8 { - return fromStringRef(c.mlirFlatSymbolRefAttrGetValue(self.inner())); + pub fn context(self: *const Context) *Context { + return @ptrCast(c.mlirAttributeGetContext(self.ptr()).ptr); + } + + pub fn dialect(self: *const Context) *const Dialect { + return @ptrCast(c.mlirAttributeGetDialect(self.ptr()).ptr); + } + + pub fn string(ctx: *Context, str: []const u8) *const Attribute { + return @ptrCast(StringAttribute.init(ctx, str)); + } + + pub fn int(ctx: *Context, it: IntegerTypes, value_: anytype) *const Attribute { + return @ptrCast(IntegerAttribute.init(ctx, it, value_)); + } + + pub fn intFromType(it: *const IntegerType, value_: i64) *const Attribute { + return @ptrCast(IntegerAttribute.fromType(it, value_)); + } + + pub fn float(ctx: *Context, comptime ft: FloatTypes, value_: anytype) *const Attribute { + return @ptrCast(FloatAttribute.init(ctx, ft, value_)); + } + pub fn boolean(ctx: *Context, value: bool) *const Attribute { + return @ptrCast(BoolAttribute.init(ctx, value)); + } + + pub fn typeAttr(type_: *const Type) *const Attribute { + return @ptrCast(TypeAttribute.init(type_)); + } + + pub fn dict(ctx: *Context, attributes: []const NamedAttribute) *const Attribute { + return @ptrCast(DictionaryAttribute.init(ctx, attributes)); + } + + pub fn denseElements(shaped_type: *const ShapedType, values: anytype) *const Attribute { + return @ptrCast(DenseElementsAttribute.init(shaped_type, values)); + } + + pub fn denseArray(ctx: *Context, comptime ElementType: DenseArrayTypes, values: []const ElementType.ZigType()) *const Attribute { + return @ptrCast(DenseArrayAttribute(ElementType).init(ctx, @ptrCast(values))); + } + pub fn array(ctx: *Context, attrs: []const *const Attribute) *const Attribute { + return @ptrCast(ArrayAttribute.init(ctx, attrs)); + } + + pub fn intArray(ctx: *Context, comptime T: type, values: []const T) *const Attribute { + const it: IntegerTypes = comptime @field(IntegerTypes, @typeName(T)); + var buf: stdx.BoundedArray(*const Attribute, ShapedType.MAX_RANK) = .empty; + for (values) |v| buf.appendAssumeCapacity(.int(ctx, it, v)); + return .array(ctx, buf.constSlice()); + } + + pub fn flatSymbolRef(ctx: *Context, symbol: []const u8) *const Attribute { + return @ptrCast(FlatSymbolRefAttribute.init(ctx, symbol)); + } + + pub fn symbolRef(ctx: *Context, root: []const u8, nested: []const *const FlatSymbolRefAttribute) *const Attribute { + return @ptrCast(SymbolRefAttribute.init(ctx, root, nested)); + } + + pub fn unit(ctx: *Context) *const Attribute { + return @ptrCast(UnitAttribute.get(ctx)); + } + + pub fn affineMap(map: *const AffineMap) *const Attribute { + return @ptrCast(AffineMapAttribute.init(map)); + } + + pub fn integerSet(set: *const IntegerSet) *const Attribute { + return @ptrCast(IntegerSetAttribute.init(set)); + } +}; + +pub const StringAttribute = opaque { + const M = Methods(StringAttribute, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsAString; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub fn init(ctx: *Context, str: []const u8) *const StringAttribute { + return @ptrCast(c.mlirStringAttrGet(ctx.ptr(), stringRef(str)).ptr); + } + + pub fn value(self: *const StringAttribute) []const u8 { + return string(c.mlirStringAttrGetValue(self.ptr())); + } +}; + +pub const IntegerAttribute = opaque { + const M = Methods(IntegerAttribute, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsAInteger; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub fn init(ctx: *Context, it: IntegerTypes, value_: anytype) *const IntegerAttribute { + return .fromType(.get(ctx, it), @intCast(value_)); + } + + pub fn fromType(it: *const IntegerType, value_: i64) *const IntegerAttribute { + return @ptrCast(c.mlirIntegerAttrGet(it.ptr(), value_).ptr); + } + + pub fn value(self: *const IntegerAttribute, comptime T: type) T { + const getValue = switch (@typeInfo(T).int.signedness) { + .signed => c.mlirIntegerAttrGetValueInt, + .unsigned => c.mlirIntegerAttrGetValueUInt, + }; + return @intCast(getValue(self.ptr())); + } +}; + +pub const FloatAttribute = opaque { + const M = Methods(FloatAttribute, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsAFloat; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub fn init(ctx: *Context, comptime ft: FloatTypes, value_: f64) *const FloatAttribute { + return @ptrCast(c.mlirFloatAttrDoubleGet( + ctx.ptr(), + FloatType(ft).get(ctx).ptr(), + value_, + ).ptr); + } +}; + +pub const BoolAttribute = opaque { + const M = Methods(BoolAttribute, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsABool; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub fn init(ctx: *Context, value_: bool) *const BoolAttribute { + return @ptrCast(c.mlirBoolAttrGet(ctx.ptr(), @intFromBool(value_)).ptr); + } + + pub fn value(self: *const BoolAttribute) bool { + return c.mlirBoolAttrGetValue(self.ptr()); + } +}; + +pub const TypeAttribute = opaque { + const M = Methods(TypeAttribute, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsAType; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub fn init(type_: *const Type) *const TypeAttribute { + return @ptrCast(c.mlirTypeAttrGet(type_.ptr()).ptr); + } + + pub fn value(self: *const TypeAttribute) *const Type { + return @ptrCast(c.mlirTypeAttrGetValue(self.ptr()).ptr); + } +}; + +pub const Dialect = struct { + const M = Methods(Dialect, c.MlirDialect); + + pub const ptr = M.ptr; + + pub fn context(self: *const Dialect) *Context { + return @ptrCast(c.mlirDialectGetContext(self.ptr()).ptr); + } + + pub fn namespace(self: *const Dialect) []const u8 { + return string(c.mlirDialectGetNamespace(self.ptr())); + } + + pub fn format(self: *const Dialect, writer: *std.Io.Writer) !void { + try writer.print("{s}{{ .namespace = \"{s}\" }}", .{ @typeName(Dialect), self.namespace() }); + } +}; + +pub const DialectHandle = opaque { + const M = Methods(DialectHandle, c.MlirDialectHandle); + + pub const ptr = M.ptr; + + pub fn format(self: *const DialectHandle, writer: *std.Io.Writer) !void { + try writer.print("{s}{{ .namespace = \"{s}\" }}", .{ @typeName(DialectHandle), self.namespace() }); + } + + pub fn namespace(self: *const DialectHandle) []const u8 { + return string(c.mlirDialectHandleGetNamespace(self.ptr())); + } + + pub fn insertDialect(self: *const DialectHandle, registry: *DialectRegistry) void { + c.mlirDialectHandleInsertDialect(self.ptr(), registry.ptr()); + } + + pub fn registerDialect(self: *const DialectHandle, ctx: *Context) void { + c.mlirDialectHandleRegisterDialect(self.ptr(), ctx.ptr()); + } + + pub fn loadDialect(self: *const DialectHandle, ctx: *Context) *Dialect { + return @ptrCast(c.mlirDialectHandleLoadDialect(self.ptr(), ctx.ptr()).ptr); + } + + pub fn fromString(comptime namespace_: []const u8) *const DialectHandle { + return @ptrCast(@field(c, "mlirGetDialectHandle__" ++ namespace_ ++ "__")().ptr); + } +}; + +pub const Identifier = opaque { + const M = Methods(Identifier, c.MlirIdentifier); + + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirIdentifierEqual); + + pub fn context(self: *const Identifier) *Context { + return @ptrCast(c.mlirIdentifierGetContext(self.ptr()).ptr); + } + + pub fn get(ctx: *Context, str_: []const u8) *const Identifier { + return @ptrCast(c.mlirIdentifierGet(ctx.ptr(), stringRef(str_)).ptr); + } + + pub fn str(self: *const Identifier) []const u8 { + return string(c.mlirIdentifierStr(self.ptr())); + } + + pub fn format(self: *const Identifier, writer: *std.Io.Writer) !void { + try writer.print("{s}{{ .str = \"{s}\" }}", .{ @typeName(Identifier), self.str() }); + } +}; + +pub const Module = opaque { + const M = Methods(Module, c.MlirModule); + + pub const ptr = M.ptr; + + pub fn init(loc: *const Location) *Module { + return @ptrCast(@constCast(c.mlirModuleCreateEmpty(loc.ptr()).ptr)); + } + + pub fn deinit(self: *Module) void { + c.mlirModuleDestroy(self.ptr()); + } + + pub fn format(self: *const Module, writer: *std.Io.Writer) !void { + return self.operation().format(writer); + } + + pub fn fromOperation(op: *Operation) *Module { + return @ptrCast(@constCast(c.mlirModuleFromOperation(op.ptr()).ptr)); + } + + pub fn operation(self: *const Module) *Operation { + return @ptrCast(c.mlirModuleGetOperation(self.ptr()).ptr); + } + + pub fn parse(ctx: *Context, source: []const u8) Error!*Module { + return @ptrCast(@constCast(c.mlirModuleCreateParse(ctx.ptr(), stringRef(source)).ptr orelse return Error.InvalidMlir)); + } + + pub fn context(self: *const Module) *Context { + return @ptrCast(c.mlirModuleGetContext(self.ptr()).ptr); + } + + pub fn body(self: *const Module) *Block { + return @ptrCast(c.mlirModuleGetBody(self.ptr()).ptr); + } +}; + +pub const Block = opaque { + const M = Methods(Block, c.MlirBlock); + + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirBlockEqual); + pub const format = M.format(c.mlirBlockPrint); + + pub fn init(args: []const *const Type, locs: []const *const Location) *Block { + return @ptrCast(c.mlirBlockCreate(@intCast(args.len), @ptrCast(args), @ptrCast(locs)).ptr); + } + + pub fn deinit(self: *Block) void { + std.debug.assert(self.parentOperation() == null); + c.mlirBlockDestroy(self.ptr()); + } + + pub fn argument(self: *const Block, index: usize) *const Value { + return @ptrCast(c.mlirBlockGetArgument(self.ptr(), @intCast(index)).ptr); + } + + pub fn numArguments(self: *const Block) usize { + return @intCast(c.mlirBlockGetNumArguments(self.ptr())); + } + + pub fn addArgument(self: *Block, type_: *const Type, loc: *const Location) *const Value { + return @ptrCast(c.mlirBlockAddArgument(self.ptr(), type_.ptr(), loc.ptr()).ptr); + } + + pub fn insertArgument(self: *Block, index: usize, type_: *const Type, loc: *const Location) *const Value { + return @ptrCast(c.mlirBlockInsertArgument(self.ptr(), @intCast(index), type_.ptr(), loc.ptr()).ptr); + } + + pub fn appendOwnedOperation(self: *Block, op: *Operation) void { + c.mlirBlockAppendOwnedOperation(self.ptr(), op.ptr()); + } + + pub fn appendOwnedOperations(self: *Block, ops: []const *Operation) void { + for (ops) |op| { + self.appendOwnedOperation(op); + } + } + + pub fn firstOperation(self: *const Block) ?*Operation { + const op = c.mlirBlockGetFirstOperation(self.ptr()); + if (op.ptr == null) return null; + return @ptrCast(op.ptr); + } + + pub fn insertOwnedOperationBefore(self: *Block, reference: *Operation, op: *Operation) void { + c.mlirBlockInsertOwnedOperationBefore(self.ptr(), reference.ptr(), op.ptr()); + } + + pub fn parentOperation(self: *const Block) ?*Operation { + return @ptrCast(c.mlirBlockGetParentOperation(self.ptr()).ptr); + } + + pub fn parentRegion(self: *const Block) *Region { + return @ptrCast(c.mlirBlockGetParentRegion(self.ptr()).ptr); + } + + pub fn detach(self: *Block) void { + c.mlirBlockDetach(self.ptr()); + } + + pub fn terminator(self: *const Block) ?*Operation { + return @ptrCast(c.mlirBlockGetTerminator(self.ptr()).ptr); + } +}; + +pub const NamedAttribute = struct { + inner: c.MlirNamedAttribute, + + pub fn ptr(self: NamedAttribute) c.MlirNamedAttribute { + return self.inner; + } + + pub fn name(self: NamedAttribute) *const Identifier { + return @ptrCast(self.inner.name.ptr); + } + + pub fn attribute(self: NamedAttribute) *const Attribute { + return @ptrCast(self.inner.attribute.ptr); + } + + pub fn named(ctx: *Context, name_: []const u8, attr: *const Attribute) NamedAttribute { + return .{ + .inner = .{ + .name = Identifier.get(ctx, name_).ptr(), + .attribute = attr.ptr(), + }, + }; + } + + pub fn init(name_: *const Identifier, attr: *const Attribute) NamedAttribute { + return .{ + .inner = .{ + .name = name_.ptr(), + .attribute = attr.ptr(), + }, + }; + } + + pub fn format(self: NamedAttribute, writer: *std.Io.Writer) std.Io.Writer.Error!void { + try writer.print("{s}{{ .{s} = \"{f}\" }}", .{ + @typeName(NamedAttribute), + self.name().str(), + self.attribute(), + }); + } +}; + +pub const DictionaryAttribute = opaque { + const M = Methods(DictionaryAttribute, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsADictionary; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub fn init(ctx: *Context, attributes: []const NamedAttribute) *const DictionaryAttribute { + return @ptrCast(c.mlirDictionaryAttrGet( + ctx.ptr(), + @intCast(attributes.len), + @ptrCast(attributes), + ).ptr); + } + + pub fn size(self: *const DictionaryAttribute) usize { + return @intCast(c.mlirDictionaryAttrGetNumElements(self.ptr())); + } + + pub fn get(self: *const DictionaryAttribute, pos: usize) NamedAttribute { + return .{ .inner = c.mlirDictionaryAttrGetElement(self.ptr(), @intCast(pos)) }; + } + + pub fn getByName(self: *const DictionaryAttribute, name: []const u8) ?*const Attribute { + return @ptrCast(c.mlirDictionaryAttrGetElementByName(self.ptr(), stringRef(name)).ptr); + } +}; + +pub const Value = opaque { + const M = Methods(Value, c.MlirValue); + + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirValueEqual); + pub const format = M.format(c.mlirValuePrint); + pub const isA = M.isA; + + pub fn type_(val: *const Value) *const Type { + return @ptrCast(c.mlirValueGetType(val.ptr()).ptr); + } + + pub fn setType(self: *Value, type__: *const Type) void { + c.mlirValueSetType(self.ptr(), type__.ptr()); + } + + pub fn firstUse(self: *const Value) *const OpOperand { + return @ptrCast(c.mlirValueGetFirstUse(self.ptr()).ptr); + } + + pub fn replaceAllUsesWith(self: *const Value, with: *const Value) void { + c.mlirValueReplaceAllUsesOfWith(self.ptr(), with.ptr()); + } + + pub fn owner(self: *const Value) *Operation { + return @ptrCast(c.mlirOpResultGetOwner(self.ptr()).ptr); + } +}; + +pub const OpOperand = opaque { + const M = Methods(OpOperand, c.MlirOpOperand); + + pub const ptr = M.ptr; + + pub fn owner(self: *const OpOperand) *Operation { + return @ptrCast(c.mlirOpOperandGetOwner(self.ptr()).ptr); + } + + pub fn number(self: *const OpOperand) usize { + return @intCast(c.mlirOpOperandGetOperandNumber(self.ptr())); + } + + pub fn nextUse(self: *const OpOperand) ?*const OpOperand { + return @ptrCast(c.mlirOpOperandGetNextUse(self.ptr()).ptr); + } +}; + +pub const OpResult = opaque { + const M = Methods(OpResult, c.MlirValue); + + pub const isAFn = c.mlirValueIsAOpResult; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirValueEqual); + pub const format = M.format(c.mlirValuePrint); + + pub fn owner(self: *const OpResult) *Operation { + return @ptrCast(c.mlirOpResultGetOwner(self.ptr()).ptr); + } + + pub fn resultNumber(self: *const OpResult) usize { + return @intCast(c.mlirOpResultGetResultNumber(self.ptr())); + } +}; + +pub const BlockArgument = opaque { + const M = Methods(BlockArgument, c.MlirValue); + + pub const isAFn = c.mlirValueIsABlockArgument; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirValueEqual); + pub const format = M.format(c.mlirValuePrint); + + pub fn block(arg: *const BlockArgument) *Block { + return @ptrCast(c.mlirBlockArgumentGetOwner(arg.ptr()).ptr); + } + + pub fn number(self: *const BlockArgument) usize { + return @intCast(c.mlirBlockArgumentGetArgNumber(self.ptr())); + } + + pub fn setType(self: *BlockArgument, type_: *const Type) void { + c.mlirBlockArgumentSetType(self.ptr(), type_.ptr()); } }; pub const OperationState = struct { - _inner: c.MlirOperationState, - pub usingnamespace MlirHelpers( - OperationState, - .{}, - ); + inner: c.MlirOperationState, - const Self = OperationState; - - pub fn init(name: [:0]const u8, loc: Location) Self { - return Self.wrap(c.mlirOperationStateGet(stringRef(name), loc.inner())); + pub fn init(name: []const u8, location: *const Location) OperationState { + return .{ .inner = c.mlirOperationStateGet(stringRef(name), location.ptr()) }; } - pub fn addResult(self: *Self, type_: Type) void { - c.mlirOperationStateAddResults(self.innerPtr(), 1, &[_]c.MlirType{type_.inner()}); + pub fn addResults(self: *OperationState, results: []const *const Type) void { + c.mlirOperationStateAddResults(&self.inner, @intCast(results.len), @ptrCast(results)); } - pub fn addResults(self: *Self, types: []const Type) void { - c.mlirOperationStateAddResults(self.innerPtr(), @intCast(types.len), @ptrCast(types.ptr)); + pub fn addOperands(self: *OperationState, operands: []const *const Value) void { + c.mlirOperationStateAddOperands(&self.inner, @intCast(operands.len), @ptrCast(operands)); } - pub fn addOperand(self: *Self, value: Value) void { - c.mlirOperationStateAddOperands(self.innerPtr(), 1, &[_]c.MlirValue{value.inner()}); + pub fn addOwnedRegions(self: *OperationState, regions: []const *Region) void { + c.mlirOperationStateAddOwnedRegions(&self.inner, @intCast(regions.len), @ptrCast(regions)); } - pub fn addOperands(self: *Self, values: []const Value) void { - c.mlirOperationStateAddOperands(self.innerPtr(), @intCast(values.len), @ptrCast(values.ptr)); + pub fn addSuccessors(self: *OperationState, successors: []*const Block) void { + c.mlirOperationStateAddSuccessors(&self.inner, @intCast(successors.len), @ptrCast(successors)); } - pub fn addRegion(self: *Self, region: *Region) void { - c.mlirOperationStateAddOwnedRegions(self.innerPtr(), 1, &[_]c.MlirRegion{region.inner()}); + pub fn addAttributes(self: *OperationState, attributes: []const NamedAttribute) void { + c.mlirOperationStateAddAttributes(&self.inner, @intCast(attributes.len), @ptrCast(attributes.ptr)); } - pub fn addRegions(self: *Self, regions: []const Region) void { - c.mlirOperationStateAddOwnedRegions(self.innerPtr(), @intCast(regions.len), @ptrCast(regions.ptr)); - } - - pub fn addAttribute(self: *Self, ctx: Context, name: [:0]const u8, attr: Attribute) void { - c.mlirOperationStateAddAttributes(self.innerPtr(), 1, @ptrCast(&.{ - .{ - .name = Identifier.get(ctx, name).inner(), - .attribute = attr.inner(), - }, - })); - } - - pub fn addAttributeRaw(self: *Self, name: Identifier, attr: Attribute) void { - c.mlirOperationStateAddAttributes(self.innerPtr(), 1, @ptrCast(&.{ - .{ - .name = name.inner(), - .attribute = attr.inner(), - }, - })); - } - - pub fn addAttributes(self: *Self, attributes: []const NamedAttribute) void { - c.mlirOperationStateAddAttributes(self.innerPtr(), @intCast(attributes.len), @ptrCast(attributes.ptr)); - } - - pub fn resultTypeInference(self: *Self, enabled: bool) void { - self.innerPtr().enableResultTypeInference = enabled; + pub fn enableResultTypeInference(self: *OperationState) void { + c.mlirOperationStateEnableResultTypeInference(&self.inner); } }; -pub const DictionaryAttribute = struct { - _inner: c.MlirAttribute, - pub usingnamespace MlirHelpers(DictionaryAttribute, .{ - .is_a_fn = c.mlirAttributeIsADictionary, - .is_null_fn = c.mlirAttributeIsNull, - .dump_fn = c.mlirAttributeDump, - .equal_fn = c.mlirAttributeEqual, - }); +pub const Operation = opaque { + const M = Methods(Operation, c.MlirOperation); - pub fn init(ctx: Context, attributes: []const NamedAttribute) DictionaryAttribute { - return DictionaryAttribute.wrap(c.mlirDictionaryAttrGet( - ctx.inner(), - @intCast(attributes.len), - @ptrCast(attributes.ptr), - )); + const MAX_SEGMENTS = 32; + + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirOperationEqual); + pub const format = M.format(c.mlirOperationPrint); + + pub fn init(state: *OperationState) !*Operation { + defer state.* = undefined; + return @ptrCast(c.mlirOperationCreate(&state.inner).ptr orelse return Error.InvalidMlir); } - pub fn size(self: DictionaryAttribute) usize { - return @intCast(c.mlirDictionaryAttrGetNumElements(self.inner())); + pub fn deinit(self: *Operation) void { + std.debug.assert(self.block() == null); + c.mlirOperationDestroy(self.ptr()); } - pub fn get(self: DictionaryAttribute, pos: usize) NamedAttribute { - return NamedAttribute.wrap(c.mlirDictionaryAttrGetElement(self.inner(), @intCast(pos))); + pub fn parse(ctx: *Context, source: []const u8, name_: []const u8) Error!*Operation { + return @ptrCast(c.mlirOperationCreateParse(ctx.ptr(), stringRef(source), stringRef(name_)).ptr orelse return Error.InvalidMlir); } - pub fn getByName(self: DictionaryAttribute, name: [:0]const u8) ?NamedAttribute { - return NamedAttribute.wrapOr(c.mlirDictionaryAttrGetElementByName(self.inner(), name)); + pub fn name(self: *const Operation) []const u8 { + return string(c.mlirIdentifierStr(c.mlirOperationGetName(self.ptr()))); } - pub fn asAttr(self: DictionaryAttribute) Attribute { - return .{ ._inner = self._inner }; - } -}; - -pub const Operation = struct { - const Self = Operation; - _inner: c.MlirOperation, - - pub usingnamespace MlirHelpers( - Operation, - .{ - .is_null_fn = c.mlirOperationIsNull, - .deinit_fn = c.mlirOperationDestroy, - .dump_fn = c.mlirOperationDump, - .equal_fn = c.mlirOperationEqual, - }, - ); - - pub fn init(state: *OperationState) !Self { - return Self.wrapOr( - c.mlirOperationCreate(state.innerPtr()), - ) orelse Error.InvalidMlir; + pub fn clone(self: *const Operation) *Operation { + return @ptrCast(c.mlirOperationClone(self.ptr()).ptr); } - pub fn make(ctx: Context, op_name: [:0]const u8, args: struct { - pub const AttrTuple = struct { [:0]const u8, Attribute }; + pub fn block(self: *const Operation) ?*Block { + return @ptrCast(c.mlirOperationGetBlock(self.ptr()).ptr); + } - operands: ?[]const Value = null, - variadic_operands: ?[]const []const Value = null, - results: ?[]const Type = null, - variadic_results: ?[]const []const Type = null, - result_type_inference: ?bool = null, - n_regions: usize = 0, - attributes: ?[]const AttrTuple = null, - blocks: ?[]const Block = null, - verify: bool = true, - location: Location, - }) Self { - var state = OperationState.init(op_name, args.location); - std.debug.assert(!(args.operands != null and args.variadic_operands != null)); - if (args.operands) |operands| { - state.addOperands(operands); - } else if (args.variadic_operands) |operands_segments| { - for (operands_segments) |operands| { - state.addOperands(operands); - } - } - if (args.result_type_inference) |enable| { - state.resultTypeInference(enable); - } - std.debug.assert(!(args.results != null and args.variadic_results != null)); - if (args.results) |results| { - state.addResults(results); - } else if (args.variadic_results) |result_segments| { - for (result_segments) |results| { - state.addResults(results); - } - } - for (0..args.n_regions) |_| { - var region_ = Region.init() catch { - @panic("Failed to create MLIR region"); - }; - state.addRegion(®ion_); - } - if (args.attributes) |attrs| { - for (attrs) |attr| { - state.addAttributeRaw( - Identifier.get(ctx, attr[0]), - attr[1], - ); - } - } - if (args.blocks) |blocks_| { - for (blocks_) |block_| { - var region_ = Region.init() catch { - @panic("Failed to create MLIR region"); - }; - region_.appendBlock(block_); - state.addRegion(®ion_); - } - } + pub fn appendTo(self: *Operation, block_: *Block) *Operation { + block_.appendOwnedOperation(self); + return self; + } - const new_op = Operation.init(&state) catch { - @panic("Failed to create MLIR operation"); + pub fn numRegions(self: *const Operation) usize { + return @intCast(c.mlirOperationGetNumRegions(self.ptr())); + } + + pub fn region(self: *const Operation, index: usize) *Region { + return @ptrCast(c.mlirOperationGetRegion(self.ptr(), @intCast(index)).ptr); + } + + pub fn parent(self: *const Operation) ?*Operation { + return @ptrCast(c.mlirOperationGetParentOperation(self.ptr()).ptr); + } + + pub fn removeFromParent(self: *Operation) void { + c.mlirOperationRemoveFromParent(self.ptr()); + } + + pub fn numOperands(self: *const Operation) usize { + return @intCast(c.mlirOperationGetNumOperands(self.ptr())); + } + + pub fn operand(self: *const Operation, index: usize) *const Value { + return @ptrCast(c.mlirOperationGetOperand(self.ptr(), @intCast(index)).ptr); + } + + pub fn setOperand(self: *Operation, index: usize, value: *const Value) void { + c.mlirOperationSetOperand(self.ptr(), @intCast(index), value.ptr()); + } + + pub fn numResults(self: *const Operation) usize { + return @intCast(c.mlirOperationGetNumResults(self.ptr())); + } + + pub fn result(self: *const Operation, index: usize) *const Value { + return @ptrCast(c.mlirOperationGetResult(self.ptr(), @intCast(index)).ptr); + } + + pub fn nextInBlock(self: *const Operation) *const Operation { + return @ptrCast(c.mlirOperationGetNextInBlock(self.ptr()).ptr); + } + + pub fn context(self: *const Operation) *Context { + return @ptrCast(c.mlirOperationGetContext(self.ptr()).ptr); + } + + pub fn setAttributeByName(self: *Operation, name_: []const u8, attribute: *const Attribute) void { + c.mlirOperationSetAttributeByName(self.ptr(), stringRef(name_), attribute.ptr()); + } + + /// Sets `attribute` under its name, as a dialect's attribute helper names it. + pub fn setAttribute(self: *Operation, attribute: NamedAttribute) void { + self.setAttributeByName(attribute.name().str(), attribute.attribute()); + } + + pub fn attributeByName(self: *const Operation, name_: []const u8) ?*const Attribute { + const attr = c.mlirOperationGetAttributeByName(self.ptr(), stringRef(name_)); + return if (attr.ptr == null) null else @ptrCast(attr.ptr); + } + + pub const PrintFlags = struct { + elide_large_elements_attrs: ?usize = null, + debug_info: bool = false, + debug_info_pretty_form: bool = true, + print_generic_op_form: bool = false, + use_local_scope: bool = false, + assume_verified: bool = false, + print_name_loc_as_prefix: bool = false, + + const Ctx = struct { + self: *const Operation, + flags: PrintFlags, }; + + fn createNative(self: PrintFlags) c.MlirOpPrintingFlags { + const flags = c.mlirOpPrintingFlagsCreate(); + if (self.elide_large_elements_attrs) |v| { + c.mlirOpPrintingFlagsElideLargeElementsAttrs(flags, @intCast(v)); + } + c.mlirOpPrintingFlagsEnableDebugInfo( + flags, + self.debug_info, + self.debug_info_pretty_form, + ); + if (self.print_generic_op_form) { + c.mlirOpPrintingFlagsPrintGenericOpForm(flags); + } + if (self.use_local_scope) { + c.mlirOpPrintingFlagsUseLocalScope(flags); + } + if (self.assume_verified) { + c.mlirOpPrintingFlagsAssumeVerified(flags); + } + if (self.print_name_loc_as_prefix) { + c.mlirOpPrintingFlagsPrintNameLocAsPrefix(flags); + } + return flags; + } + }; + + fn formatWithFlags(fctx: PrintFlags.Ctx, writer: *std.Io.Writer) std.Io.Writer.Error!void { + const flags = fctx.flags.createNative(); + defer c.mlirOpPrintingFlagsDestroy(flags); + var sctx: StringCallbackCtx = .{ .writer = writer }; + c.mlirOperationPrintWithFlags(fctx.self.ptr(), flags, stringCallback, &sctx); + if (sctx.err) |err| { + return err; + } + } + + pub fn fmt(self: *const Operation, flags: PrintFlags) std.fmt.Alt(PrintFlags.Ctx, formatWithFlags) { + return .{ .data = .{ + .self = self, + .flags = flags, + } }; + } + + pub const MakeArgs = struct { + pub const AttrTuple = struct { []const u8, *const Attribute }; + + pub const Operands = union(enum) { + flat: []const *const Value, + variadic: []const []const *const Value, + }; + + pub const ResultTypes = union(enum) { + flat: []const *const Type, + variadic: []const []const *const Type, + }; + + operands: ?Operands = null, + results: ?ResultTypes = null, + result_type_inference: bool = false, + attributes: ?[]const NamedAttribute = null, + blocks: ?[]const *Block = null, + verify: bool = true, + location: ?*const Location, + }; + + pub fn try_make(ctx: *Context, name_: []const u8, args: MakeArgs) !*Operation { + var state: OperationState = .init(name_, args.location orelse .unknown(ctx)); + if (args.result_type_inference) { + state.enableResultTypeInference(); + } + if (args.operands) |operands| switch (operands) { + .flat => |v| state.addOperands(v), + .variadic => |segments| { + var sizes: stdx.BoundedArray(i32, MAX_SEGMENTS) = .empty; + for (segments) |segment_operands| { + state.addOperands(segment_operands); + sizes.appendAssumeCapacity(@intCast(segment_operands.len)); + } + state.addAttributes(&.{ + .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, sizes.constSlice())), + }); + }, + }; + if (args.results) |results| switch (results) { + .flat => |v| state.addResults(v), + .variadic => |segments| { + var sizes: stdx.BoundedArray(i32, MAX_SEGMENTS) = .empty; + for (segments) |segment_results| { + state.addResults(segment_results); + sizes.appendAssumeCapacity(@intCast(segment_results.len)); + } + state.addAttributes(&.{ + .named(ctx, "resultSegmentSizes", .denseArray(ctx, .i32, sizes.constSlice())), + }); + }, + }; + if (args.attributes) |attrs| { + state.addAttributes(attrs); + } + if (args.blocks) |blocks| { + for (blocks) |block_| { + const body_region = Region.init(); + body_region.appendOwnedBlock(block_); + state.addOwnedRegions(&.{body_region}); + } + } + const new_op = try Operation.init(&state); + errdefer new_op.deinit(); if (args.verify and new_op.verify() == false) { - std.log.err("Failed to verify MLIR operation:\n{}", .{new_op.mlirFormatter(.{ .debug_info = true })}); - @panic("Failed to verify MLIR operation"); + log.err("Failed to verify MLIR operation:\n{f}", .{ + new_op.fmt(.{ + .print_generic_op_form = true, + .debug_info = true, + }), + }); + return Error.InvalidMlir; } return new_op; } - pub fn initParse(ctx: Context, str: [:0]const u8) !Self { - return Self.wrapOr( - c.mlirOperationCreateParse(ctx.inner(), stringRef(str), stringRef("pouet")), - ) orelse Error.InvalidMlir; + pub fn make(ctx: *Context, op_name: []const u8, args: MakeArgs) *Operation { + return try_make(ctx, op_name, args) catch { + @panic("Failed to create MLIR operation"); + }; } - pub fn clone(self: Self) !Self { - return Self.wrapOr( - c.mlirOperationClone(self.inner()), - ) orelse Error.InvalidMlir; + pub fn verify(self: *const Operation) bool { + return c.mlirOperationVerify(self.ptr()); } - pub fn name(self: Self) Identifier { - return Identifier.wrap(c.mlirOperationGetName(self.inner())); - } - - pub fn removeFromParent(self: *Self) void { - c.mlirOperationRemoveFromParent(self.inner()); - } - - pub fn numOperands(self: Self) usize { - return @intCast(c.mlirOperationGetNumOperands(self.inner())); - } - - pub fn operand(self: Self, index: usize) Value { - return Value.wrap(c.mlirOperationGetOperand(self.inner(), @intCast(index))); - } - - pub fn setOperand(self: *Self, index: usize, value: Value) void { - c.mlirOperationSetOperand(self.inner(), @intCast(index), value.inner()); - } - - pub fn numResults(self: Self) usize { - return @intCast(c.mlirOperationGetNumResults(self.inner())); - } - - pub fn result(self: Self, index: usize) Value { - return Value.wrap(c.mlirOperationGetResult(self.inner(), @intCast(index))); - } - - pub fn nextInBlock(self: Self) Self { - return Self.wrap(c.mlirOperationGetNextInBlock(self.inner())); - } - - // pub fn previousInBlock(self: Self) Self { - // return Self.wrap(c.mlirOperationGetPrevInBlock(self.inner())); - // } - - pub fn block(self: Self) ?Block { - return Block.wrapOr(c.mlirOperationGetBlock(self.inner())); - } - - pub fn parent(self: Self) ?Self { - return Self.wrapOr(c.mlirOperationGetParentOperation(self.inner())); - } - - pub fn region(self: Self, index: usize) Region { - return Region.wrap(c.mlirOperationGetRegion(self.inner(), @intCast(index))); - } - - pub fn context(self: Self) Context { - return Context.wrap(c.mlirOperationGetContext(self.inner())); - } - - pub fn writeBytecode(self: Self, writer: anytype) void { - var writer_context = .{ .writer = writer }; - const WriterContext = @TypeOf(writer_context); - - c.mlirOperationWriteBytecode( - self.inner(), - (struct { - pub fn callback(str: c.MlirStringRef, ctx_: ?*anyopaque) callconv(.C) void { - const inner_writer_context: *WriterContext = @ptrCast(@alignCast(ctx_)); - _ = inner_writer_context.writer.write(str.data[0..str.length]) catch unreachable; - } - }).callback, - &writer_context, - ); - } - - pub fn writeBytecodeWithConfig(self: Self, writer: anytype, config: struct { - desiredEmitedVersion: ?i64 = null, - }) !void { - const cfg = c.mlirBytecodeWriterConfigCreate(); - defer c.mlirBytecodeWriterConfigDestroy(cfg); - if (config.desiredEmitedVersion) |v| { - c.mlirBytecodeWriterConfigDesiredEmitVersion(cfg, v); - } - - var writer_context = .{ .writer = writer }; - const WriterContext = @TypeOf(writer_context); - - try successOr(c.mlirOperationWriteBytecodeWithConfig( - self.inner(), - cfg, - (struct { - pub fn callback(str: c.MlirStringRef, ctx_: ?*anyopaque) callconv(.C) void { - const inner_writer_context: *WriterContext = @ptrCast(@alignCast(ctx_)); - _ = inner_writer_context.writer.write(str.data[0..str.length]) catch unreachable; - } - }).callback, - &writer_context, - ), error.InvalidMlirBytecodeVersion); - } - - /// Enable a full dump of the IR. - /// Usage `std.log.debug("{}", .{ module.op().mlirFormatter(.{}) }); - pub fn mlirFormatter(self: Operation, flags: OpPrintingFlags) MlirFormatter { - return .{ .op = self, .flags = flags }; - } - - const MlirFormatter = struct { - op: Operation, - flags: OpPrintingFlags, - - pub fn format(self: @This(), comptime fmt: []const u8, options: std.fmt.FormatOptions, writer: anytype) !void { - _ = fmt; - _ = options; - self.op.print(writer, self.flags); - } - }; - - pub fn print(self: Self, writer: anytype, flags: OpPrintingFlags) void { - const pflags = flags.create(); - defer c.mlirOpPrintingFlagsDestroy(pflags); - - var writer_context = .{ .writer = writer }; - const WriterContext = @TypeOf(writer_context); - c.mlirOperationPrintWithFlags( - self.inner(), - pflags, - (struct { - pub fn callback(str: c.MlirStringRef, ctx_: ?*anyopaque) callconv(.C) void { - const inner_writer_context: *WriterContext = @ptrCast(@alignCast(ctx_)); - _ = inner_writer_context.writer.write(str.data[0..str.length]) catch unreachable; - } - }).callback, - &writer_context, - ); - } - - pub fn verify(self: Self) bool { - return c.mlirOperationVerify(self.inner()); - } - - pub fn getLocation(self: Self) Location { - return Location.wrap(c.mlirOperationGetLocation(self.inner())); + pub fn location(self: *const Operation) *const Location { + return @ptrCast(c.mlirOperationGetLocation(self.ptr()).ptr); } pub const WalkOrder = enum(c.MlirWalkOrder) { @@ -1016,666 +1481,635 @@ pub const Operation = struct { skip = c.MlirWalkResultSkip, }; - pub fn walk(self: Self, order: WalkOrder, ctx: anytype, walkfn: fn (ctx: anytype, op: Operation) WalkResult) void { + pub fn walk(self: *Operation, order: WalkOrder, ctx: anytype, walkfn: fn (ctx_: @TypeOf(ctx), op: *Operation) WalkResult) void { var inner_ctx = .{ .ctx = ctx }; const ContextType = @TypeOf(inner_ctx); c.mlirOperationWalk( - self.inner(), + self._inner, (struct { - pub fn callback(op: c.MlirOperation, ctx_: ?*anyopaque) callconv(.C) c.MlirWalkResult { + pub fn callback(op: c.MlirOperation, ctx_: ?*anyopaque) callconv(.c) c.MlirWalkResult { const inner_ctx_: *ContextType = @ptrCast(@alignCast(ctx_)); - return @intFromEnum(walkfn(inner_ctx_.ctx, Operation.wrap(op))); + return @backingInt(walkfn(inner_ctx_.ctx, @ptrCast(op.ptr))); } }).callback, &inner_ctx, - @intFromEnum(order), + @backingInt(order), ); } - pub fn getAttribute(self: Self, pos: usize) NamedAttribute { - return NamedAttribute.wrap(c.mlirOperationGetAttribute(self.inner(), @intCast(pos))); - } + pub const BytecodeWriterConfig = struct { + desired_emit_version: ?usize = null, - pub fn getAttributeByName(self: Self, name_: [:0]const u8) ?Attribute { - return Attribute.wrapOr(c.mlirOperationGetAttributeByName(self.inner(), stringRef(name_))); - } - - pub fn setAttributeByName(self: Self, name_: [:0]const u8, attr: Attribute) void { - c.mlirOperationSetAttributeByName(self.inner(), stringRef(name_), attr.inner()); - } - - pub fn removeAttributeByName(self: Self, name_: [:0]const u8) bool { - return c.mlirOperationRemoveAttributeByName(self.inner(), stringRef(name_)); - } -}; - -pub const OpPrintingFlags = struct { - elide_large_elements_attrs: ?usize = null, - debug_info: bool = false, - debug_info_pretty_form: bool = false, - print_generic_op_form: bool = false, - use_local_scope: bool = false, - assume_verified: bool = false, - - pub fn create(self: OpPrintingFlags) c.MlirOpPrintingFlags { - const pflags = c.mlirOpPrintingFlagsCreate(); - if (self.elide_large_elements_attrs) |v| { - c.mlirOpPrintingFlagsElideLargeElementsAttrs(pflags, @intCast(v)); - } - c.mlirOpPrintingFlagsEnableDebugInfo(pflags, self.debug_info, self.debug_info_pretty_form); - if (self.print_generic_op_form) { - c.mlirOpPrintingFlagsPrintGenericOpForm(pflags); - } - if (self.use_local_scope) { - c.mlirOpPrintingFlagsUseLocalScope(pflags); - } - if (self.assume_verified) { - c.mlirOpPrintingFlagsAssumeVerified(pflags); - } - return pflags; - } -}; - -pub const OpOperand = struct { - _inner: c.MlirOpOperand, - pub usingnamespace MlirHelpers(OpOperand, .{ - .is_null_fn = c.mlirOpOperandIsNull, - }); - - const Self = OpOperand; - - pub fn owner(self: Self) Operation { - return Operation.wrap(c.mlirOpOperandGetOwner(self.inner())); - } - - pub fn number(self: Self) usize { - return @intCast(c.mlirOpOperandGetOperandNumber(self.inner())); - } - - pub fn nextUse(self: Self) ?Self { - return Self.wrapOr( - c.mlirOpOperandGetNextUse(self.inner()), - ); - } -}; - -pub const Region = struct { - _inner: c.MlirRegion, - pub usingnamespace MlirHelpers(Region, .{ - .is_null_fn = c.mlirRegionIsNull, - .deinit_fn = c.mlirRegionDestroy, - .equal_fn = c.mlirRegionEqual, - }); - - const Self = Region; - - pub fn init() !Self { - return Self.wrapOr( - c.mlirRegionCreate(), - ) orelse Error.InvalidMlir; - } - - pub fn appendBlock(self: *Self, block: Block) void { - c.mlirRegionAppendOwnedBlock(self.inner(), block.inner()); - } - - pub fn insertBlock(self: *Self, index: isize, block: Block) void { - c.mlirRegionInsertOwnedBlock(self.inner(), index, block.inner()); - } - - pub fn insertBlockBefore(self: *Self, reference: Block, block: Block) void { - c.mlirRegionInsertOwnedBlockBefore(self.inner(), reference.inner(), block.inner()); - } - - pub fn insertBlockAfter(self: *Self, reference: Block, block: Block) void { - c.mlirRegionInsertOwnedBlockAfter(self.inner(), reference.inner(), block.inner()); - } - - pub fn firstBlock(self: Self) Block { - return Block.wrap(c.mlirRegionGetFirstBlock(self.inner())); - } -}; - -pub const Value = struct { - _inner: c.MlirValue, - - pub usingnamespace MlirHelpers(Value, .{ - .is_null_fn = c.mlirValueIsNull, - .equal_fn = c.mlirValueEqual, - .dump_fn = c.mlirValueDump, - .print_fn = c.mlirValuePrint, - }); - - pub fn getType(val: Value) Type { - return Type.wrap(c.mlirValueGetType(val.inner())); - } - - pub fn setType(val: *Value, typ: Type) void { - c.mlirValueSetType(val.inner(), typ.inner()); - } - - pub fn firstUse(val: Value) OpOperand { - return OpOperand.wrap(c.mlirValueGetFirstUse(val.inner())); - } - - pub fn replaceAllUsesWith(val: Value, with: Value) void { - c.mlirValueReplaceAllUsesOfWith(val.inner(), with.inner()); - } - - pub fn owner(val: Value) Operation { - return Operation.wrap(c.mlirOpResultGetOwner(val.inner())); - } - - pub fn isABlockArgument(val: Value) bool { - return c.mlirValueIsABlockArgument(val.inner()); - } - - pub fn isAOpResult(val: Value) bool { - return c.mlirValueIsAOpResult(val.inner()); - } - - pub const Kind = enum { - unknown, - block_argument, - op_result, + const Ctx = struct { + self: *const Operation, + config: ?BytecodeWriterConfig, + }; }; - pub fn kind(val: Value) Kind { - if (val.isAOpResult()) { - return .op_result; + const WriteBytecodeError = error{InvalidMlirBytecodeVersion} || std.Io.Writer.Error; + + pub fn writeBytecode(self: *const Operation, config: ?BytecodeWriterConfig, writer: *std.Io.Writer) WriteBytecodeError!void { + if (config) |config_| { + const bc_config = c.mlirBytecodeWriterConfigCreate(); + defer c.mlirBytecodeWriterConfigDestroy(bc_config); + if (config_.desired_emit_version) |version| { + c.mlirBytecodeWriterConfigDesiredEmitVersion(bc_config, @intCast(version)); + } + var sctx: StringCallbackCtx = .{ .writer = writer }; + const logical_result = c.mlirOperationWriteBytecodeWithConfig(self.ptr(), bc_config, stringCallback, &sctx); + + if (logical_result.value == 0) { + return WriteBytecodeError.InvalidMlirBytecodeVersion; + } + + if (sctx.err) |err| { + return err; + } + return; } - if (val.isABlockArgument()) { - return .block_argument; + + var sctx: StringCallbackCtx = .{ .writer = writer }; + c.mlirOperationWriteBytecode(self.ptr(), stringCallback, &sctx); + if (sctx.err) |err| { + return err; } - return .unknown; + } + + fn fmtBytecodeImpl(ctx: BytecodeWriterConfig.Ctx, writer: *std.Io.Writer) std.Io.Writer.Error!void { + try ctx.self.writeBytecode(ctx.config, writer); + } + + pub fn fmtBytecode(self: *const Operation, config: ?BytecodeWriterConfig) std.fmt.Alt(BytecodeWriterConfig.Ctx, fmtBytecodeImpl) { + return .{ .data = .{ + .self = self, + .config = config, + } }; } }; -pub const Type = struct { - _inner: c.MlirType, - - pub usingnamespace MlirHelpers(Type, .{ - .is_null_fn = c.mlirTypeIsNull, - .dump_fn = c.mlirTypeDump, - .equal_fn = c.mlirTypeEqual, - .print_fn = c.mlirTypePrint, +/// builtin.unrealized_conversion_cast — `inputs` as values of `outputs_types`, +/// a cast between type systems that a later conversion resolves. +pub fn unrealized_conversion_cast(ctx: *Context, inputs: []const *const Value, outputs_types: []const *const Type, location: *const Location) *Operation { + return Operation.make(ctx, "builtin.unrealized_conversion_cast", .{ + .operands = .{ .flat = inputs }, + .results = .{ .flat = outputs_types }, + .location = location, }); +} - pub fn parse(ctx: Context, str: [:0]const u8) !Type { - return Type.wrapOr( - c.mlirTypeParseGet(ctx.inner(), stringRef(str)), - ) orelse Error.InvalidMlir; +pub const Region = opaque { + const M = Methods(Region, c.MlirRegion); + + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirRegionEqual); + + pub fn init() *Region { + return @ptrCast(c.mlirRegionCreate().ptr); + } + + pub fn deinit(self: *Region) void { + c.mlirRegionDestroy(self.ptr()); + } + + pub fn appendOwnedBlock(self: *Region, block: *Block) void { + c.mlirRegionAppendOwnedBlock(self.ptr(), block.ptr()); + } + + pub fn insertOwnedBlock(self: *Region, index: usize, block: *Block) void { + c.mlirRegionInsertOwnedBlock(self.ptr(), @intCast(index), block.ptr()); + } + + pub fn insertOwnedBlockBefore(self: *Region, reference: *const Block, block: *Block) void { + c.mlirRegionInsertOwnedBlockBefore(self.ptr(), reference.ptr(), block.ptr()); + } + + pub fn insertOwnedBlockAfter(self: *Region, reference: *const Block, block: *Block) void { + c.mlirRegionInsertOwnedBlockAfter(self.ptr(), reference.ptr(), block.ptr()); + } + + pub fn firstBlock(self: *const Region) ?*Block { + return @ptrCast(c.mlirRegionGetFirstBlock(self.ptr()).ptr); + } + + pub fn takeBody(self: *Region, from: *Region) void { + c.mlirRegionTakeBody(self.ptr(), from.ptr()); + } + + pub fn nextInOperation(self: *const Region) *Region { + return @ptrCast(c.mlirRegionGetNextInOperation(self.ptr()).ptr); } }; -pub const IntegerTypes = enum { - i1, - i4, +pub const ShapedType = opaque { + const M = Methods(ShapedType, c.MlirType); + + pub const MAX_RANK = 16; + + pub const isAFn = c.mlirTypeIsAShaped; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + + pub fn elementType(self: *const ShapedType) *const Type { + return @ptrCast(c.mlirShapedTypeGetElementType(self.ptr()).ptr); + } + + pub fn rank(self: *const ShapedType) usize { + return @intCast(c.mlirShapedTypeGetRank(self.ptr())); + } + + pub fn dimension(self: *const ShapedType, dim: usize) i64 { + return c.mlirShapedTypeGetDimSize(self.ptr(), @intCast(dim)); + } + + pub fn rankedTensor(dimensions: []const i64, elem_type: *const Type) *const ShapedType { + return @ptrCast(RankedTensorType.get(dimensions, elem_type, null)); + } + + pub fn rankedTensorWithEncoding(dimensions: []const i64, elem_type: *const Type, encoding: *Attribute) *const ShapedType { + return @ptrCast(RankedTensorType.get(dimensions, elem_type, encoding)); + } +}; + +pub const RankedTensorType = opaque { + const M = Methods(RankedTensorType, c.MlirType); + + pub const isAFn = c.mlirTypeIsARankedTensor; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + + pub fn get(dimensions: []const i64, elem_type: *const Type, encoding: ?*Attribute) *const RankedTensorType { + return @ptrCast(c.mlirRankedTensorTypeGet( + @intCast(dimensions.len), + @ptrCast(dimensions), + elem_type.ptr(), + if (encoding) |e| e.ptr() else c.mlirAttributeGetNull(), + ).ptr); + } + + pub fn fromShaped(other: *const ShapedType) *const RankedTensorType { + var dims: stdx.BoundedArray(i64, ShapedType.MAX_RANK) = .empty; + for (0..other.rank()) |i| { + dims.appendAssumeCapacity(other.dimension(i)); + } + return get( + dims.constSlice(), + other.elementType(), + null, + ); + } + + pub fn elementType(self: *const RankedTensorType) *const Type { + return @ptrCast(c.mlirShapedTypeGetElementType(self.ptr()).ptr); + } + + pub fn rank(self: *const RankedTensorType) usize { + return @intCast(c.mlirShapedTypeGetRank(self.ptr())); + } + + pub fn dimension(self: *const RankedTensorType, dim: usize) i64 { + return c.mlirShapedTypeGetDimSize(self.ptr(), @intCast(dim)); + } + + pub fn shaped(self: *const RankedTensorType) *const ShapedType { + return @ptrCast(self); + } +}; + +pub const DenseElementsAttribute = opaque { + const M = Methods(DenseElementsAttribute, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsADenseElements; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub fn init(shaped_type: *const ShapedType, values: anytype) *const DenseElementsAttribute { + const bytes = std.mem.sliceAsBytes(values); + return @ptrCast(c.mlirDenseElementsAttrRawBufferGet( + shaped_type.ptr(), + @intCast(bytes.len), + @ptrCast(bytes), + ).ptr orelse unreachable); + } +}; + +pub const DenseArrayTypes = enum { + bool, i8, i16, i32, i64, - si4, - si8, - si16, - si32, - si64, - u4, - u8, - u16, - u32, - u64, - - unknown, -}; - -pub fn IntegerType(comptime it: IntegerTypes) type { - const Config = switch (it) { - .i1 => .{ 1, c.mlirIntegerTypeGet, c.mlirIntegerTypeIsSignless }, - .i4 => .{ 4, c.mlirIntegerTypeGet, c.mlirIntegerTypeIsSignless }, - .i8 => .{ 8, c.mlirIntegerTypeGet, c.mlirIntegerTypeIsSignless }, - .i16 => .{ 16, c.mlirIntegerTypeGet, c.mlirIntegerTypeIsSignless }, - .i32 => .{ 32, c.mlirIntegerTypeGet, c.mlirIntegerTypeIsSignless }, - .i64 => .{ 64, c.mlirIntegerTypeGet, c.mlirIntegerTypeIsSignless }, - .si4 => .{ 4, c.mlirIntegerTypeSignedGet, c.mlirIntegerTypeIsSigned }, - .si8 => .{ 8, c.mlirIntegerTypeSignedGet, c.mlirIntegerTypeIsSigned }, - .si16 => .{ 16, c.mlirIntegerTypeSignedGet, c.mlirIntegerTypeIsSigned }, - .si32 => .{ 32, c.mlirIntegerTypeSignedGet, c.mlirIntegerTypeIsSigned }, - .si64 => .{ 64, c.mlirIntegerTypeSignedGet, c.mlirIntegerTypeIsSigned }, - .u4 => .{ 4, c.mlirIntegerTypeUnsignedGet, c.mlirIntegerTypeIsUnsigned }, - .u8 => .{ 8, c.mlirIntegerTypeUnsignedGet, c.mlirIntegerTypeIsUnsigned }, - .u16 => .{ 16, c.mlirIntegerTypeUnsignedGet, c.mlirIntegerTypeIsUnsigned }, - .u32 => .{ 32, c.mlirIntegerTypeUnsignedGet, c.mlirIntegerTypeIsUnsigned }, - .u64 => .{ 64, c.mlirIntegerTypeUnsignedGet, c.mlirIntegerTypeIsUnsigned }, - .unknown => .{ 0, null, null }, - }; - - return struct { - _inner: c.MlirType, - - const Int = @This(); - pub usingnamespace MlirHelpers(Int, .{ - .is_a_fn = switch (it) { - .unknown => c.mlirTypeIsAInteger, - else => typeIsAIntegerExact, - }, - .is_null_fn = c.mlirTypeIsNull, - .dump_fn = c.mlirTypeDump, - .equal_fn = c.mlirTypeEqual, - }); - const IntegerTypeType = it; - - fn typeIsAIntegerExact(typ: c.MlirType) callconv(.C) bool { - const bit_width = Config[0]; - const is_sign = Config[2]; - return c.mlirTypeIsAInteger(typ) and (c.mlirIntegerTypeGetWidth(typ) == bit_width) and is_sign(typ); - } - pub usingnamespace if (it != .unknown) struct { - pub const BitWidth = Config[0]; - - pub fn init(ctx: Context) Int { - const type_get = Config[1]; - return Int.wrap(type_get(ctx.inner(), BitWidth)); - } - } else struct {}; - }; -} - -pub const FloatTypes = enum { - f8e4m3b11fnuz, - f8e4m3fn, - f8e4m3fnuz, - f8e5m2, - f8e5m2fnuz, - bf16, - f16, f32, f64, - unknown, -}; - -pub fn FloatType(comptime ft: FloatTypes) type { - const Config = switch (ft) { - .f8e4m3b11fnuz => .{ c.mlirTypeIsAFloat8E4M3B11FNUZ, c.mlirFloat8E4M3B11FNUZTypeGet }, - .f8e4m3fn => .{ c.mlirTypeIsAFloat8E4M3FN, c.mlirFloat8E4M3FNTypeGet }, - .f8e4m3fnuz => .{ c.mlirTypeIsAFloat8E4M3FNUZ, c.mlirFloat8E4M3FNUZTypeGet }, - .f8e5m2 => .{ c.mlirTypeIsAFloat8E5M2, c.mlirFloat8E5M2TypeGet }, - .f8e5m2fnuz => .{ c.mlirTypeIsAFloat8E5M2FNUZ, c.mlirFloat8E5M2FNUZTypeGet }, - .bf16 => .{ c.mlirTypeIsABF16, c.mlirBF16TypeGet }, - .f16 => .{ c.mlirTypeIsAF16, c.mlirF16TypeGet }, - .f32 => .{ c.mlirTypeIsAF32, c.mlirF32TypeGet }, - .f64 => .{ c.mlirTypeIsAF64, c.mlirF64TypeGet }, - .unknown => .{ null, null }, - }; - - return struct { - _inner: c.MlirType, - const Float = @This(); - pub usingnamespace MlirHelpers(Float, .{ - .is_a_fn = switch (ft) { - .unknown => typeIsAUnknownFloat, - else => Config[0], - }, - .is_null_fn = c.mlirTypeIsNull, - .dump_fn = c.mlirTypeDump, - .equal_fn = c.mlirTypeEqual, - }); - - pub usingnamespace if (ft != .unknown) struct { - pub const FloatTypeType = ft; - - pub fn init(ctx: Context) Float { - const type_get = Config[1]; - return Float.wrap(type_get(ctx.inner())); - } - } else struct {}; - - fn typeIsAUnknownFloat(typ: c.MlirType) callconv(.C) bool { - const is_a_fns = .{ - c.mlirTypeIsABF16, - c.mlirTypeIsAF16, - c.mlirTypeIsAF32, - c.mlirTypeIsF64, - }; - inline for (is_a_fns) |is_a_fn| { - if (is_a_fn(typ)) { - return true; - } - } - return false; - } - }; -} - -pub const ComplexTypes = enum { - c64, - c128, - - unknown, -}; - -pub fn ComplexType(comptime ct: ComplexTypes) type { - return struct { - _inner: c.MlirType, - const Complex = @This(); - - fn mlirC64TypeGet(ctx: c.MlirContext) callconv(.C) c.MlirType { - return c.mlirComplexTypeGet(c.mlirF32TypeGet(ctx)); - } - - fn mlirC128TypeGet(ctx: c.MlirContext) callconv(.C) c.MlirType { - return c.mlirComplexTypeGet(c.mlirF64TypeGet(ctx)); - } - - fn mlirTypeIsAC64(typ: c.MlirType) callconv(.C) bool { - const element_type: c.MlirType = c.mlirComplexTypeGetElementType(typ); - return c.mlirTypeIsAF32(element_type); - } - - fn mlirTypeIsAC128(typ: c.MlirType) callconv(.C) bool { - const element_type: c.MlirType = c.mlirComplexTypeGetElementType(typ); - return c.mlirTypeIsAF64(element_type); - } - - const Config = switch (ct) { - .c64 => .{ mlirTypeIsAC64, mlirC64TypeGet }, - .c128 => .{ mlirTypeIsAC128, mlirC128TypeGet }, - .unknown => .{ c.mlirTypeIsAComplex, null }, + fn ZigType(comptime ElementType: @This()) type { + return switch (ElementType) { + .bool => i32, + .i8 => i8, + .i16 => i16, + .i32 => i32, + .i64 => i64, + .f32 => f32, + .f64 => f64, }; - - fn typeIsAUnknownComplex(typ: c.MlirType) callconv(.C) bool { - return c.mlirTypeIsAComplex(typ); - } - - pub usingnamespace MlirHelpers(@This(), .{ - .is_a_fn = Config[0], - .is_null_fn = c.mlirTypeIsNull, - .dump_fn = c.mlirTypeDump, - .equal_fn = c.mlirTypeEqual, - }); - - pub usingnamespace if (ct != .unknown) struct { - pub const ComplexTypeType = ct; - - pub fn init(ctx: Context) Complex { - const type_get = Config[1]; - return Complex.wrap(type_get(ctx.inner())); - } - } else struct {}; - }; -} - -pub const TupleType = struct { - _inner: c.MlirType, - pub usingnamespace MlirHelpers(TupleType, .{ - .is_a_fn = c.mlirTypeIsATuple, - .is_null_fn = c.mlirTypeIsNull, - .dump_fn = c.mlirTypeDump, - .equal_fn = c.mlirTypeEqual, - }); - - const Self = TupleType; - - pub fn init(ctx: Context, elements: []const Type) !Self { - return Self.wrapOr(c.mlirTupleTypeGet( - ctx.inner(), - @intCast(elements.len), - @ptrCast(elements.ptr), - )) orelse Error.InvalidMlir; - } - - pub fn getNumTypes(self: Self) usize { - return @intCast(c.mlirTupleTypeGetNumTypes(self.inner())); - } - - pub fn getElementType(self: Self, index: usize) Type { - return Type.wrap(c.mlirTupleTypeGetType(self.inner(), @intCast(index))); } }; -pub const FunctionType = struct { - _inner: c.MlirType, - pub usingnamespace MlirHelpers(FunctionType, .{ - .is_a_fn = c.mlirTypeIsAFunction, - .is_null_fn = c.mlirTypeIsNull, - .dump_fn = c.mlirTypeDump, - .equal_fn = c.mlirTypeEqual, - }); +pub fn DenseArrayAttribute(comptime ElementType: DenseArrayTypes) type { + const ElementTypeZig = ElementType.ZigType(); - const Self = FunctionType; + const _isAFn, const getFn, const getElementFn = switch (ElementType) { + .bool => .{ c.mlirAttributeIsADenseBoolArray, c.mlirDenseBoolArrayGet, c.mlirDenseBoolArrayGetElement }, + .i8 => .{ c.mlirAttributeIsADenseI8Array, c.mlirDenseI8ArrayGet, c.mlirDenseI8ArrayGetElement }, + .i16 => .{ c.mlirAttributeIsADenseI16Array, c.mlirDenseI16ArrayGet, c.mlirDenseI16ArrayGetElement }, + .i32 => .{ c.mlirAttributeIsADenseI32Array, c.mlirDenseI32ArrayGet, c.mlirDenseI32ArrayGetElement }, + .i64 => .{ c.mlirAttributeIsADenseI64Array, c.mlirDenseI64ArrayGet, c.mlirDenseI64ArrayGetElement }, + .f32 => .{ c.mlirAttributeIsADenseF32Array, c.mlirDenseF32ArrayGet, c.mlirDenseF32ArrayGetElement }, + .f64 => .{ c.mlirAttributeIsADenseF64Array, c.mlirDenseF64ArrayGet, c.mlirDenseF64ArrayGetElement }, + }; - pub fn init(ctx: Context, args: []const Type, results: []const Type) !Self { - return Self.wrapOr(c.mlirFunctionTypeGet( - ctx.inner(), + return struct { + const Self = @This(); + + const M = Methods(@This(), c.MlirAttribute); + + pub const isAFn = _isAFn; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub fn init(ctx: *Context, values: []const ElementTypeZig) *const Self { + return @ptrCast(getFn(ctx.ptr(), @intCast(values.len), @ptrCast(values)).ptr); + } + + pub fn element(self: *const Self, pos: usize) ElementTypeZig { + return getElementFn(self.ptr(), @intCast(pos)); + } + + pub fn numElements(self: *const Self) usize { + return @intCast(c.mlirDenseArrayGetNumElements(self.ptr())); + } + }; +} + +pub const ArrayAttribute = opaque { + const M = Methods(ArrayAttribute, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsAArray; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub fn init(ctx: *Context, attrs: []const *const Attribute) *const ArrayAttribute { + return @ptrCast(c.mlirArrayAttrGet(ctx.ptr(), @intCast(attrs.len), @ptrCast(attrs)).ptr); + } + + pub fn element(self: *const ArrayAttribute, index: usize) Attribute { + return @ptrCast(c.mlirArrayAttrGetElement(self.ptr(), @intCast(index)).ptr); + } + + pub fn numElements(self: *const ArrayAttribute) usize { + return @intCast(c.mlirArrayAttrGetNumElements(self.ptr())); + } +}; + +pub const FunctionType = opaque { + const M = Methods(FunctionType, c.MlirType); + pub const isAFn = c.mlirTypeIsAFunction; + + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + + pub fn init(ctx: *Context, args: []const *const Type, results: []const *const Type) *const FunctionType { + return @ptrCast(c.mlirFunctionTypeGet( + ctx.ptr(), @intCast(args.len), @ptrCast(args.ptr), @intCast(results.len), @ptrCast(results.ptr), - )) orelse Error.InvalidMlir; + ).ptr); } }; -pub const RankedTensorType = struct { - _inner: c.MlirType, - pub usingnamespace MlirHelpers(RankedTensorType, .{ - .is_a_fn = c.mlirTypeIsARankedTensor, - .is_null_fn = c.mlirTypeIsNull, - .dump_fn = c.mlirTypeDump, - .equal_fn = c.mlirTypeEqual, - .print_fn = c.mlirTypePrint, - }); +pub const FlatSymbolRefAttribute = opaque { + const M = Methods(FlatSymbolRefAttribute, c.MlirAttribute); - pub fn init(dimensions: []const i64, elemType: Type) RankedTensorType { - return RankedTensorType.wrap( - c.mlirRankedTensorTypeGet( - @intCast(dimensions.len), - @ptrCast(dimensions.ptr), - elemType.inner(), - c.mlirAttributeGetNull(), - ), + pub const isAFn = c.mlirAttributeIsAFlatSymbolRef; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub fn init(ctx: *Context, symbol_: []const u8) *const FlatSymbolRefAttribute { + return @ptrCast(c.mlirFlatSymbolRefAttrGet(ctx.ptr(), stringRef(symbol_)).ptr); + } + + pub fn symbol(self: *const FlatSymbolRefAttribute) []const u8 { + return string(c.mlirFlatSymbolRefAttrGetValue(self.ptr())); + } +}; + +/// `@root::@nested...`. +pub const SymbolRefAttribute = opaque { + const M = Methods(SymbolRefAttribute, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsASymbolRef; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub fn init(ctx: *Context, root: []const u8, nested: []const *const FlatSymbolRefAttribute) *const SymbolRefAttribute { + return @ptrCast(c.mlirSymbolRefAttrGet(ctx.ptr(), stringRef(root), @intCast(nested.len), @ptrCast(nested)).ptr); + } +}; + +pub const UnitAttribute = opaque { + const M = Methods(UnitAttribute, c.MlirAttribute); + + pub const isAFn = c.mlirAttributeIsAUnit; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub fn get(ctx: *Context) *const UnitAttribute { + return @ptrCast(c.mlirUnitAttrGet(ctx.ptr()).ptr); + } +}; + +pub const MemRefType = opaque { + const M = Methods(MemRefType, c.MlirType); + + pub const isAFn = c.mlirTypeIsAMemRef; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; + + pub fn init(element_type: *const Type, shape: []const i64, layout_attr: ?*const Attribute, memory_space: ?*const Attribute) *const MemRefType { + return @ptrCast(c.mlirMemRefTypeGet( + element_type.ptr(), + @intCast(shape.len), + @ptrCast(shape), + if (layout_attr) |l| l.ptr() else .{ .ptr = null }, + if (memory_space) |m| m.ptr() else .{ .ptr = null }, + ).ptr); + } + + pub fn fromShaped(other: *const ShapedType) *const MemRefType { + var dims: stdx.BoundedArray(i64, ShapedType.MAX_RANK) = .empty; + for (0..other.rank()) |i| { + dims.appendAssumeCapacity(other.dimension(i)); + } + return init( + other.elementType(), + dims.constSlice(), + null, + null, ); } - pub fn getElementType(self: RankedTensorType) Type { - return Type.wrap(c.mlirShapedTypeGetElementType(self.inner())); + pub fn elementType(self: *const MemRefType) *const Type { + return @ptrCast(c.mlirShapedTypeGetElementType(self.ptr()).ptr); } - pub fn getRank(self: RankedTensorType) usize { - return @intCast(c.mlirShapedTypeGetRank(self.inner())); + pub fn rank(self: *const MemRefType) usize { + return @intCast(c.mlirShapedTypeGetRank(self.ptr())); } - pub fn getDimension(self: RankedTensorType, dim: usize) i64 { - return c.mlirShapedTypeGetDimSize(self.inner(), @intCast(dim)); + pub fn dimension(self: *const MemRefType, dim: usize) i64 { + return c.mlirShapedTypeGetDimSize(self.ptr(), @intCast(dim)); + } + + pub fn layout(self: *const MemRefType) *const Attribute { + return @ptrCast(c.mlirMemRefTypeGetLayout(self.ptr()).ptr); + } + + pub fn memorySpace(self: *const MemRefType) ?*const Attribute { + const a = c.mlirMemRefTypeGetMemorySpace(self.ptr()); + return if (a.ptr == null) null else @ptrCast(a.ptr); + } + + pub fn shaped(self: *const MemRefType) *const ShapedType { + return @ptrCast(self); } }; -pub const Dialect = struct { - _inner: c.MlirDialect, - pub usingnamespace MlirHelpers(Dialect, .{ - .equal_fn = c.mlirDialectEqual, - .is_null_fn = c.mlirDialectIsNull, - }); +pub const VectorType = opaque { + const M = Methods(VectorType, c.MlirType); - const Self = Dialect; + pub const isAFn = c.mlirTypeIsAVector; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; - pub fn getContext(self: Self) Context { - return Context.wrap(c.mlirDialectGetContext(self.inner())); + /// Plain (non-scalable) vector type. + pub fn init(shape: []const i64, element_type: *const Type) *const VectorType { + return @ptrCast(c.mlirVectorTypeGet( + @intCast(shape.len), + @ptrCast(shape), + element_type.ptr(), + ).ptr); } - pub fn getNamespace(self: Self) []const u8 { - return fromStringRef(c.mlirDialectGetNamespace(self.inner())); + /// Vector type with explicit per-dim scalability flags. `scalable.len` + /// must equal `shape.len`. + pub fn initScalable( + shape: []const i64, + scalable: []const bool, + element_type: *const Type, + ) *const VectorType { + std.debug.assert(shape.len == scalable.len); + return @ptrCast(c.mlirVectorTypeGetScalable( + @intCast(shape.len), + @ptrCast(shape), + @ptrCast(scalable), + element_type.ptr(), + ).ptr); + } + + pub fn rank(self: *const VectorType) usize { + return @intCast(c.mlirShapedTypeGetRank(self.ptr())); + } + + pub fn dimension(self: *const VectorType, dim: usize) i64 { + return c.mlirShapedTypeGetDimSize(self.ptr(), @intCast(dim)); + } + + pub fn shaped(self: *const VectorType) *const ShapedType { + return @ptrCast(self); } }; -pub const DialectHandle = struct { - _inner: c.MlirDialectHandle, - pub usingnamespace MlirHelpers( - DialectHandle, - .{}, - ); +pub const UnrankedMemRefType = opaque { + const M = Methods(UnrankedMemRefType, c.MlirType); - pub fn getNamespace(self: DialectHandle) []const u8 { - return fromStringRef(c.mlirDialectHandleGetNamespace(self.inner())); - } + pub const isAFn = c.mlirTypeIsAUnrankedMemRef; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirTypeEqual); + pub const format = M.format(c.mlirTypePrint); + pub const isA = M.isA; - pub fn insertDialect(self: DialectHandle, registry: Registry) void { - c.mlirDialectHandleInsertDialect(self.inner(), registry.inner()); - } - - pub fn registerDialect(self: DialectHandle, ctx: Context) void { - c.mlirDialectHandleRegisterDialect(self.inner(), ctx.inner()); - } - - pub fn loadDialect(self: DialectHandle, ctx: Context) Dialect { - return Dialect.wrap(c.mlirDialectHandleLoadDialect(self.inner(), ctx.inner())); - } - - pub fn fromString(comptime namespace: []const u8) DialectHandle { - return DialectHandle.wrap(@field(c, "mlirGetDialectHandle__" ++ namespace ++ "__")()); + pub fn init(element_type: *const Type, memory_space: ?*const Attribute) *const UnrankedMemRefType { + return @ptrCast(c.mlirUnrankedMemRefTypeGet( + element_type.ptr(), + if (memory_space) |m| m.ptr() else .{ .ptr = null }, + ).ptr); } }; -pub const ShapedType = struct { - _inner: c.MlirType, - pub usingnamespace MlirHelpers(ShapedType, .{ - .is_a_fn = c.mlirTypeIsAShaped, - .is_null_fn = c.mlirTypeIsNull, - .dump_fn = c.mlirTypeDump, - .equal_fn = c.mlirTypeEqual, - }); - const Self = ShapedType; +pub const AffineExpr = opaque { + const M = Methods(AffineExpr, c.MlirAffineExpr); - pub fn rank(self: Self) usize { - return @intCast(c.mlirShapedTypeGetRank(self.inner())); + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAffineExprEqual); + pub const format = M.format(c.mlirAffineExprPrint); + + pub fn dim(ctx: *Context, position: usize) *const AffineExpr { + return @ptrCast(c.mlirAffineDimExprGet(ctx.ptr(), @intCast(position)).ptr); } - pub fn elementType(self: Self) Type { - return Type.wrap(c.mlirShapedTypeGetElementType(self.inner())); + pub fn symbol(ctx: *Context, position: usize) *const AffineExpr { + return @ptrCast(c.mlirAffineSymbolExprGet(ctx.ptr(), @intCast(position)).ptr); } - pub fn dimension(self: Self, dim: usize) usize { - return @intCast(c.mlirShapedTypeGetDimSize(self.inner(), @intCast(dim))); + pub fn constant(ctx: *Context, value: i64) *const AffineExpr { + return @ptrCast(c.mlirAffineConstantExprGet(ctx.ptr(), value).ptr); + } + + pub fn add(self: *const AffineExpr, other: *const AffineExpr) *const AffineExpr { + return @ptrCast(c.mlirAffineAddExprGet(self.ptr(), other.ptr()).ptr); + } + + pub fn mul(self: *const AffineExpr, other: *const AffineExpr) *const AffineExpr { + return @ptrCast(c.mlirAffineMulExprGet(self.ptr(), other.ptr()).ptr); + } + + pub fn floorDiv(self: *const AffineExpr, other: *const AffineExpr) *const AffineExpr { + return @ptrCast(c.mlirAffineFloorDivExprGet(self.ptr(), other.ptr()).ptr); + } + + pub fn ceilDiv(self: *const AffineExpr, other: *const AffineExpr) *const AffineExpr { + return @ptrCast(c.mlirAffineCeilDivExprGet(self.ptr(), other.ptr()).ptr); + } + + pub fn mod(self: *const AffineExpr, other: *const AffineExpr) *const AffineExpr { + return @ptrCast(c.mlirAffineModExprGet(self.ptr(), other.ptr()).ptr); } }; -pub const Location = struct { - _inner: c.MlirLocation, +pub const AffineMap = opaque { + const M = Methods(AffineMap, c.MlirAffineMap); - pub usingnamespace MlirHelpers(Location, .{ - .is_null_fn = c.mlirLocationIsNull, - .equal_fn = c.mlirLocationEqual, - .print_fn = c.mlirLocationPrint, - }); + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAffineMapEqual); + pub const format = M.format(c.mlirAffineMapPrint); - pub fn fromSrc(ctx: Context, src: std.builtin.SourceLocation) Location { - return Location.wrap(c.mlirLocationFileLineColGet( - ctx.inner(), - stringRef(src.file), - @intCast(src.line), - @intCast(src.column), - )); + /// Empty `() -> ()` map. + pub fn empty(ctx: *Context) *const AffineMap { + return @ptrCast(c.mlirAffineMapEmptyGet(ctx.ptr()).ptr); } - pub fn fileLineCol(ctx: Context, file: []const u8, line: usize, column: usize) Location { - return Location.wrap(c.mlirLocationFileLineColGet( - ctx.inner(), - stringRef(file), - @intCast(line), - @intCast(column), - )); + /// Zero-result map: `(d0, d1, ...) -> ()`. + pub fn zeroResult(ctx: *Context, dim_count: usize, symbol_count: usize) *const AffineMap { + return @ptrCast(c.mlirAffineMapZeroResultGet( + ctx.ptr(), + @intCast(dim_count), + @intCast(symbol_count), + ).ptr); } - pub fn callSite(callee: Location, caller: Location) Location { - return Location.wrap(c.mlirLocationCallSiteGet(callee.inner(), caller.inner())); + /// Identity map: `(d0, d1, ..., dN-1) -> (d0, d1, ..., dN-1)`. + pub fn multiDimIdentity(ctx: *Context, num_dims: usize) *const AffineMap { + return @ptrCast(c.mlirAffineMapMultiDimIdentityGet(ctx.ptr(), @intCast(num_dims)).ptr); } - pub fn fused(ctx: Context, locations: []const Location, metadata: Attribute) Location { - return Location.wrap(c.mlirLocationFusedGet( - ctx.inner(), - @intCast(locations.len), - @ptrCast(locations.ptr), - metadata.inner(), - )); + /// Build a map from explicit results. + pub fn get( + ctx: *Context, + dim_count: usize, + symbol_count: usize, + results: []const *const AffineExpr, + ) *const AffineMap { + return @ptrCast(c.mlirAffineMapGet( + ctx.ptr(), + @intCast(dim_count), + @intCast(symbol_count), + @intCast(results.len), + @ptrCast(@constCast(results.ptr)), + ).ptr); } - pub fn named(loc: Location, ctx: Context, loc_name: [:0]const u8) Location { - return Location.wrap(c.mlirLocationNameGet(ctx.inner(), stringRef(loc_name), loc.inner())); + pub fn numDims(self: *const AffineMap) usize { + return @intCast(c.mlirAffineMapGetNumDims(self.ptr())); } - pub fn namedFmt(loc: Location, ctx: Context, comptime fmt: [:0]const u8, args: anytype) Location { - var buf: [256]u8 = undefined; - var stream = std.io.fixedBufferStream(&buf); - std.fmt.format(stream.writer(), fmt, args) catch { - buf[256 - 3 ..].* = "...".*; - }; - return loc.named(ctx, @ptrCast(stream.getWritten())); + pub fn numSymbols(self: *const AffineMap) usize { + return @intCast(c.mlirAffineMapGetNumSymbols(self.ptr())); } - pub fn unknown(ctx: Context) Location { - return Location.wrap(c.mlirLocationUnknownGet(ctx.inner())); + pub fn numResults(self: *const AffineMap) usize { + return @intCast(c.mlirAffineMapGetNumResults(self.ptr())); } }; -pub const Block = struct { - _inner: c.MlirBlock, - pub usingnamespace MlirHelpers(Block, .{ - .is_null_fn = c.mlirBlockIsNull, - .deinit_fn = c.mlirBlockDestroy, - .equal_fn = c.mlirBlockEqual, - }); +/// AffineMapAttr — `affine_map<(d0,...) -> (...)>` wrapped as an attribute. +pub const AffineMapAttribute = opaque { + const M = Methods(AffineMapAttribute, c.MlirAttribute); - pub fn init(args: []const Type, locs: []const Location) !Block { - const block = Block.wrapOr( - c.mlirBlockCreate(@intCast(args.len), @ptrCast(args.ptr), @ptrCast(locs.ptr)), - ); - return block orelse error.InvalidMlir; + pub const isAFn = c.mlirAttributeIsAAffineMap; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); + + pub fn init(map: *const AffineMap) *const AffineMapAttribute { + return @ptrCast(c.mlirAffineMapAttrGet(map.ptr()).ptr); } - pub fn argument(self: Block, index: usize) Value { - return Value.wrap(c.mlirBlockGetArgument(self.inner(), @intCast(index))); + pub fn value(self: *const AffineMapAttribute) *const AffineMap { + return @ptrCast(c.mlirAffineMapAttrGetValue(self.ptr()).ptr); } +}; - pub fn numArguments(self: Block) usize { - return @intCast(c.mlirBlockGetNumArguments(self.inner())); - } - - pub fn addArgument(self: *Block, typ: Type, loc: Location) Value { - return Value.wrap(c.mlirBlockAddArgument(self.inner(), typ.inner(), loc.inner())); - } +pub const IntegerSet = opaque { + const M = Methods(IntegerSet, c.MlirIntegerSet); - pub fn insertArgument(self: *Block, index: usize, typ: Type, loc: Location) Value { - return Value.wrap(c.mlirBlockInsertArgument(self.inner(), @intCast(index), typ.inner(), loc.inner())); - } + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirIntegerSetEqual); + pub const format = M.format(c.mlirIntegerSetPrint); - pub fn equals(self: Block, other: Block) bool { - return c.mlirBlockEqual(self.inner(), other.inner()); + pub fn get( + ctx: *Context, + num_dims: usize, + num_symbols: usize, + constraints: []const *const AffineExpr, + eqs: []const bool, + ) *const IntegerSet { + std.debug.assert(constraints.len == eqs.len); + return @ptrCast(c.mlirIntegerSetGet( + ctx.ptr(), + @intCast(num_dims), + @intCast(num_symbols), + @intCast(constraints.len), + @ptrCast(constraints), + @ptrCast(eqs), + ).ptr); } +}; - pub fn appendOperation(self: Block, op: Operation) void { - c.mlirBlockAppendOwnedOperation(self.inner(), op.inner()); - } +pub const IntegerSetAttribute = opaque { + const M = Methods(IntegerSetAttribute, c.MlirAttribute); - pub fn appendOperations(self: *Block, ops: []const Operation) void { - for (ops) |op| { - c.mlirBlockAppendOwnedOperation(self.inner(), op.inner()); - } - } + pub const isAFn = c.mlirAttributeIsAIntegerSet; + pub const ptr = M.ptr; + pub const eql = M.eql(c.mlirAttributeEqual); + pub const format = M.format(c.mlirAttributePrint); - pub fn addOperationsRecursive(block: *Block, op_or_result: anytype) void { - const op: Operation = switch (@TypeOf(op_or_result)) { - Operation => op_or_result, - Value => if (op_or_result.kind() == .op_result) op_or_result.owner() else return, - else => |t| @compileError("can either be an operation or a value, not " ++ @typeName(t)), - }; - if (op.block()) |prev_block| { - std.debug.assert(prev_block.equals(block.*)); - return; - } - for (0..op.numOperands()) |i| { - block.addOperationsRecursive(op.operand(i)); - } - block.appendOperation(op); + pub fn init(set: *const IntegerSet) *const IntegerSetAttribute { + return @ptrCast(c.mlirIntegerSetAttrGet(set.ptr()).ptr); } }; diff --git a/mlir/mlirx.cc b/mlir/mlirx.cc deleted file mode 100644 index f5fd45f5..00000000 --- a/mlir/mlirx.cc +++ /dev/null @@ -1,27 +0,0 @@ -#include "mlir/IR/Attributes.h" -#include "mlir/IR/BuiltinAttributes.h" -#include "mlir/CAPI/IR.h" -#include "mlir/CAPI/Support.h" -#include "mlirx.h" - -namespace mlirx { - - static mlir::Attribute ArrayToElements(mlir::Attribute attr) { - if (auto array = attr.dyn_cast()) { - return mlir::DenseIntElementsAttr::get( - mlir::RankedTensorType::get(array.size(), array.getElementType()), - array.asArrayRef()); - } - if (auto array = attr.dyn_cast()) { - return mlir::DenseIntElementsAttr::get( - mlir::RankedTensorType::get(array.size(), array.getElementType()), - array.asArrayRef()); - } - return attr; - } - -} - -MlirAttribute mlirDenseArrayToElements(MlirAttribute attr) { - return wrap(mlirx::ArrayToElements(unwrap(attr))); -} diff --git a/mlir/mlirx.h b/mlir/mlirx.h deleted file mode 100644 index 9946477f..00000000 --- a/mlir/mlirx.h +++ /dev/null @@ -1,16 +0,0 @@ -#ifndef MLIRX_CC_H -#define MLIRX_CC_H - -#include "mlir-c/IR.h" - -#ifdef __cplusplus -extern "C" { -#endif - -MLIR_CAPI_EXPORTED MlirAttribute mlirDenseArrayToElements(MlirAttribute attr); - -#ifdef __cplusplus -} -#endif - -#endif // MLIRX_CC_H diff --git a/pjrt/BUILD.bazel b/pjrt/BUILD.bazel index 4882d877..5370afdc 100644 --- a/pjrt/BUILD.bazel +++ b/pjrt/BUILD.bazel @@ -1,30 +1,24 @@ load("@rules_zig//zig:defs.bzl", "zig_library") -load("//bazel:zig_proto_library.bzl", "zig_proto_library") - -cc_library( - name = "dlfcn", - hdrs = ["dlfcn.h"], -) zig_library( name = "pjrt", - srcs = ["profiler.zig"], + srcs = ["ffi.zig"], main = "pjrt.zig", + tags = ["manual"], visibility = ["//visibility:public"], deps = [ - ":profiler_options_proto", - "//runtimes", + "@xla//xla/ffi/api:c_api", + # TODO: fix non conformance to C standard + # "@xla//xla/pjrt/c:pjrt_c_api_callback_extension_hdrs", + "@xla//xla/pjrt/c:pjrt_c_api_custom_partitioner_extension_hdrs", + "@xla//xla/pjrt/c:pjrt_c_api_ffi_extension_hdrs", "@xla//xla/pjrt/c:pjrt_c_api_gpu_extension_hdrs", "@xla//xla/pjrt/c:pjrt_c_api_hdrs", + "@xla//xla/pjrt/c:pjrt_c_api_layouts_extension_hdrs", + "@xla//xla/pjrt/c:pjrt_c_api_memory_descriptions_extension_hdrs", + "@xla//xla/pjrt/c:pjrt_c_api_phase_compile_extension_hdrs", "@xla//xla/pjrt/c:pjrt_c_api_profiler_extension_hdrs", - ] + select({ - "@platforms//os:linux": [":dlfcn"], - "//conditions:default": [], - }), -) - -zig_proto_library( - name = "profiler_options_proto", - import_name = "//tsl:profiler_options_proto", - deps = ["@tsl//tsl/profiler/protobuf:profiler_options_proto"], + "@xla//xla/pjrt/c:pjrt_c_api_raw_buffer_extension_hdrs", + "@xla//xla/pjrt/c:pjrt_c_api_stream_extension_hdrs", + ], ) diff --git a/pjrt/dlfcn.h b/pjrt/dlfcn.h deleted file mode 100644 index 9e446a2b..00000000 --- a/pjrt/dlfcn.h +++ /dev/null @@ -1 +0,0 @@ -#include diff --git a/pjrt/ffi.zig b/pjrt/ffi.zig new file mode 100644 index 00000000..cebec841 --- /dev/null +++ b/pjrt/ffi.zig @@ -0,0 +1,526 @@ +/// Bindings for PJRT custom call declaration / execution. +const std = @import("std"); + +const c = @import("c"); +pub const TypeId = c.XLA_FFI_TypeId; +const stdx = @import("stdx"); + +const pjrt = @import("pjrt.zig"); +const Stream = @import("pjrt.zig").Stream; + +const log = std.log.scoped(.pjrt); + +comptime { + if (@typeInfo(TypeId).@"struct".field_names.len != 1) @compileError("TypeId has changed"); +} + +/// The signature of a generic custom call. +pub const Handler = fn (*CallFrame) callconv(.c) ?*Error; + +pub const ApiVersion = extern struct { + pub const major = c.XLA_FFI_API_MAJOR; + pub const minor = c.XLA_FFI_API_MINOR; + + struct_size: usize, + extension_start: ?*ExtensionBase, + major_version: i32, + minor_version: i32, +}; + +pub const ExtensionType = enum(c.XLA_FFI_InternalExtension_Type) { + metadata = c.XLA_FFI_Extension_Metadata, +}; + +pub const ExtensionBase = extern struct { + struct_size: usize, + type: ExtensionType, + next: ?*ExtensionBase, +}; + +// Based of https://github.com/openxla/xla/blob/145f836bd5175dc5dd262f716a0c59af2b0297a0/xla/ffi/api/c_api.h#L449 +pub const HandlerTraits = packed struct(c_uint) { + /// Calls to FFI handler are safe to trace into the command buffer. + /// It means that calls to FFI handler always launch exactly the same device operations (can depend on attribute values) + /// that can be captured and then replayed. + command_buffer_compatible: bool, + + __unassigned__: u31 = 0, +}; + +pub const Metadata = extern struct { + struct_size: usize, + api_version: ApiVersion, + traits: HandlerTraits, +}; + +pub const MetadataExtension = extern struct { + extension_base: ExtensionBase, + metadata: ?*Metadata, +}; + +fn TransmuteMixin(comptime T: type, comptime InnerT: type) type { + return struct { + pub fn to(self: anytype) switch (@TypeOf(self)) { + *T => *InnerT, + *const T => *const InnerT, + else => unreachable, + } { + return @ptrCast(@alignCast(self)); + } + + pub fn from(self: anytype) switch (@TypeOf(self)) { + *InnerT => *T, + *const InnerT => *const T, + else => unreachable, + } { + return @ptrCast(@alignCast(self)); + } + }; +} + +pub const Api = opaque { + pub const inner = TransmuteMixin(Api, c.XLA_FFI_Api).to; + + pub fn stream(self: *const Api, context: *const ExecutionContext) *pjrt.Stream { + var ret: pjrt.meta.Struct(c.XLA_FFI_Stream_Get_Args) = .{ + .ctx = @constCast(context.inner()), + }; + + const result = self.inner().XLA_FFI_Stream_Get.?(@ptrCast(&ret)); + if (result) |ffi_error| { + const err = Error.fromInner(ffi_error); + defer err.destroy(self); + log.err("[Api.getStream] {s}", .{err.getMessage(self)}); + + @panic("failed to get stream"); + } + + return @ptrCast(ret.stream.?); + } + + pub fn allocateDeviceMemory(self: *const Api, context: *const ExecutionContext, size: usize, alignment: usize) pjrt.ApiError!*anyopaque { + var ret: pjrt.meta.Struct(c.XLA_FFI_DeviceMemory_Allocate_Args) = .{ + .ctx = @constCast(context.inner()), + .size = size, + .alignment = alignment, + }; + const result = self.inner().XLA_FFI_DeviceMemory_Allocate.?(@ptrCast(&ret)); + + if (result) |ffi_error| { + const err = Error.fromInner(ffi_error); + defer err.destroy(self); + log.err("[Api.allocateDeviceMemory] {s}", .{err.getMessage(self)}); + + // TODO(Corentin): Retrieve error code from Error when implemented in XLA. + return error.Unknown; + } + + return ret.data.?; + } + + pub fn freeDeviceMemory(self: *const Api, context: *const ExecutionContext, data: *anyopaque, size: usize) pjrt.ApiError!void { + var ret: pjrt.meta.Struct(c.XLA_FFI_DeviceMemory_Free_Args) = .{ + .ctx = @constCast(context.inner()), + .size = size, + .data = data, + }; + const result = self.inner().XLA_FFI_DeviceMemory_Free.?(@ptrCast(&ret)); + + if (result) |ffi_error| { + const err = Error.fromInner(ffi_error); + defer err.destroy(self); + log.err("[Api.freeDeviceMemory] {s}", .{err.getMessage(self)}); + + // TODO(Corentin): Retrieve error code from Error when implemented in XLA. + return error.Unknown; + } + } + + // TODO(Corentin): Implement remaining methods if needed: + // * XLA_FFI_ThreadPool_Schedule + // * XLA_FFI_Handler_Register + // * XLA_FFI_TypeId_Register + // * XLA_FFI_State_Set + // * XLA_FFI_State_Get +}; + +pub const ExecutionStage = enum(c.XLA_FFI_ExecutionStage) { + instantiate = c.XLA_FFI_ExecutionStage_INSTANTIATE, + prepare = c.XLA_FFI_ExecutionStage_PREPARE, + initialize = c.XLA_FFI_ExecutionStage_INITIALIZE, + execute = c.XLA_FFI_ExecutionStage_EXECUTE, +}; + +pub const ExecutionContext = opaque { + pub const inner = TransmuteMixin(ExecutionContext, c.XLA_FFI_InvokeContext).to; + + pub fn getContext(self: *const ExecutionContext, type_id: TypeId, api: *const Api) pjrt.ApiError!*anyopaque { + var ret: c.XLA_FFI_InvokeContext_Get_Args = .{ + .struct_size = pjrt.meta.structSize(c.XLA_FFI_InvokeContext_Get_Args), + .extension_start = api.inner().extension_start, + .ctx = @ptrCast(@constCast(self)), + .type_id = @constCast(&type_id), + .data = undefined, // set by XLA_FFI_InvokeContext_Get. + }; + const maybe_err = api.inner().XLA_FFI_InvokeContext_Get.?(&ret); + + if (maybe_err) |ffi_error| { + const err = Error.fromInner(ffi_error); + defer err.destroy(api); + log.err("[ExecutionContext.get] {s}", .{err.getMessage(api)}); + + // TODO(Corentin): Retrieve error code from Error when implemented in XLA. + return error.Unknown; + } + + if (ret.data == null) return error.NotFound; + return ret.data.?; + } + + pub fn getDeviceOrdinal(self: *const ExecutionContext, api: *const Api) pjrt.ApiError!i32 { + var ret: pjrt.meta.Struct(c.XLA_FFI_DeviceOrdinal_Get_Args) = .{ + .ctx = @constCast(self.inner()), + }; + const result = api.inner().XLA_FFI_DeviceOrdinal_Get.?(@ptrCast(&ret)); + + if (result) |ffi_error| { + const err = Error.fromInner(ffi_error); + defer err.destroy(api); + log.err("[ExecutionContext.getDeviceOrdinal] {s}", .{err.getMessage(api)}); + + // TODO(Corentin): Retrieve error code from Error when implemented in XLA. + return error.Unknown; + } + + return ret.device_ordinal; + } + + const Task = fn (*anyopaque) void; + + pub fn scheduleTask(self: *const ExecutionContext, api: *const Api, task: *const Task, data: *anyopaque) pjrt.ApiError!void { + var ret: pjrt.meta.Struct(c.XLA_FFI_ThreadPool_Schedule_Args) = .{ + .ctx = @constCast(self.inner()), + .task = @as(?*const c.XLA_FFI_Task, @ptrCast(@alignCast(task))), + .data = data, + }; + + const result = api.inner().XLA_FFI_ThreadPool_Schedule.?(@ptrCast(&ret)); + + if (result) |ffi_error| { + const err = Error.fromInner(ffi_error); + defer err.destroy(api); + std.debug.print("error: {any} \n", .{err}); + log.err("[ExecutionContext.get] {s}", .{err.getMessage(api)}); + + // TODO(Corentin): Retrieve error code from Error when implemented in XLA. + return error.Unknown; + } + } +}; + +pub const ByteSpan = extern struct { + ptr: [*]const u8, + len: usize, + + pub fn slice(self: ByteSpan) []const u8 { + return self.ptr[0..self.len]; + } +}; + +pub const DataType = enum(c.XLA_FFI_DataType) { + invalid = c.XLA_FFI_DataType_INVALID, + bool = c.XLA_FFI_DataType_PRED, + i8 = c.XLA_FFI_DataType_S8, + i16 = c.XLA_FFI_DataType_S16, + i32 = c.XLA_FFI_DataType_S32, + i64 = c.XLA_FFI_DataType_S64, + u8 = c.XLA_FFI_DataType_U8, + u16 = c.XLA_FFI_DataType_U16, + u32 = c.XLA_FFI_DataType_U32, + u64 = c.XLA_FFI_DataType_U64, + f16 = c.XLA_FFI_DataType_F16, + f32 = c.XLA_FFI_DataType_F32, + f64 = c.XLA_FFI_DataType_F64, + bf16 = c.XLA_FFI_DataType_BF16, + c64 = c.XLA_FFI_DataType_C64, + c128 = c.XLA_FFI_DataType_C128, + token = c.XLA_FFI_DataType_TOKEN, + f8e5m2 = c.XLA_FFI_DataType_F8E5M2, + f8e3m4 = c.XLA_FFI_DataType_F8E3M4, + f8e4m3 = c.XLA_FFI_DataType_F8E4M3, + f8e4m3fn = c.XLA_FFI_DataType_F8E4M3FN, + f8e4m3b11fnuz = c.XLA_FFI_DataType_F8E4M3B11FNUZ, + f8e5m2fnuz = c.XLA_FFI_DataType_F8E5M2FNUZ, + f8e4m3fnuz = c.XLA_FFI_DataType_F8E4M3FNUZ, + f4e2m1 = c.XLA_FFI_DataType_F4E2M1FN, +}; + +pub const Buffer = extern struct { + struct_size: usize, + extension_start: ?*c.XLA_FFI_InternalExtension, + dtype: DataType, + data: [*]u8, + rank: u64, + _dims: [*]const i64, + + pub fn dims(self: Buffer) []const i64 { + return self._dims[0..self.rank]; + } + + pub fn format(buffer: Buffer, writer: *std.Io.Writer) !void { + try writer.print("FfiBuffer({any}, .{t})@0x{x}", .{ buffer.dims(), buffer.dtype, @intFromPtr(buffer.data) }); + } +}; + +pub const Args = extern struct { + struct_size: usize, + extension_start: ?*const c.XLA_FFI_InternalExtension, + len: u64, + types: [*]const Type, + ptr: [*]*const Buffer, + + pub const Type = enum(c.XLA_FFI_ArgType) { + buffer = c.XLA_FFI_ArgType_BUFFER, + }; + + pub fn buffers(self: Args) []*const Buffer { + return self.ptr[0..self.len]; + } +}; + +pub const Rets = extern struct { + struct_size: usize, + extension_start: ?*const c.XLA_FFI_InternalExtension, + len: u64, + types: [*]const Type, + ptr: [*]*const Buffer, + + pub const Type = enum(c.XLA_FFI_RetType) { + buffer = c.XLA_FFI_RetType_BUFFER, + }; + + pub fn buffers(self: Rets) []*const Buffer { + return self.ptr[0..self.len]; + } +}; + +pub const AttrType = enum(c.XLA_FFI_AttrType) { + array = c.XLA_FFI_AttrType_ARRAY, + dictionary = c.XLA_FFI_AttrType_DICTIONARY, + scalar = c.XLA_FFI_AttrType_SCALAR, + string = c.XLA_FFI_AttrType_STRING, +}; + +pub const Attrs = extern struct { + struct_size: usize, + extension_start: ?*ExtensionBase, + len: u64, + types: [*]const AttrType, + names: [*]const *const ByteSpan, + ptr: [*]const *const Attr, + + const Attr = extern union { + scalar: Scalar, + array: Array, + string: ByteSpan, + }; + + pub const Scalar = extern struct { + dtype: DataType, + value: *const anyopaque, + + pub fn get(self: Scalar, T: type) T { + const ptr: *const T = @ptrCast(@alignCast(self.value)); + return ptr.*; + } + }; + + pub const Array = extern struct { + dtype: DataType, + len: usize, + data: [*]const u8, + + pub fn slice(self: Array, T: type) []const T { + const ptr: [*]const T = @ptrCast(@alignCast(self.data)); + return ptr[0..self.len]; + } + }; + + pub fn slice(self: Array, T: type) []const T { + const ptr: [*]const T = @ptrCast(@alignCast(self.data)); + return ptr[0..self.len]; + } + + pub fn getByIndex(self: Attrs, comptime attr_type: AttrType, index: usize) ?*const @FieldType(Attr, @tagName(attr_type)) { + const attr = self.ptr[0..self.len][index]; + const actual_type = self.types[index]; + if (actual_type != attr_type) return null; + return @ptrCast(attr); + } + + pub fn getByName(self: Attrs, comptime attr_type: AttrType, name: []const u8) ?*const @FieldType(Attr, @tagName(attr_type)) { + const names = self.names[0..self.len]; + for (0.., names) |i, attr_name| { + if (std.mem.eql(u8, attr_name.slice(), name)) { + return self.getByIndex(attr_type, i); + } + } + + return null; + } +}; + +/// All informations needed by the user callback, +/// including the list of input/ouput buffers to work on. +pub const CallFrame = extern struct { + struct_size: usize, + extension_start: ?*ExtensionBase, + api: *const Api, + ctx: *const ExecutionContext, + stage: ExecutionStage, + args: Args, + results: Rets, + attrs: Attrs, + future: ?*Future, + + /// The registery mechanism will first call the custom call in registration mode, + /// and expects us to indicate which version of XLA we have been compiled against. + /// Returns true if we registered ourselves and if the caller custom call should return early. + pub fn registeringHook(call_frame: *CallFrame) bool { + if (call_frame.extension_start != null and call_frame.extension_start.?.type == .metadata) { + const metadata_extension: *MetadataExtension = @fieldParentPtr("extension_base", call_frame.extension_start.?); + metadata_extension.metadata.?.api_version.major_version = ApiVersion.major; + metadata_extension.metadata.?.api_version.minor_version = ApiVersion.minor; + return true; + } + return false; + } + + pub fn stream(call_frame: CallFrame) ?*const pjrt.Stream { + return call_frame.api.stream(call_frame.ctx); + } +}; + +pub const ErrorCode = enum(c.XLA_FFI_Error_Code) { + cancelled = c.XLA_FFI_Error_Code_CANCELLED, + unknown = c.XLA_FFI_Error_Code_UNKNOWN, + invalid_argument = c.XLA_FFI_Error_Code_INVALID_ARGUMENT, + deadline_exceeded = c.XLA_FFI_Error_Code_DEADLINE_EXCEEDED, + not_found = c.XLA_FFI_Error_Code_NOT_FOUND, + already_exists = c.XLA_FFI_Error_Code_ALREADY_EXISTS, + permission_denied = c.XLA_FFI_Error_Code_PERMISSION_DENIED, + resource_exhausted = c.XLA_FFI_Error_Code_RESOURCE_EXHAUSTED, + failed_precondition = c.XLA_FFI_Error_Code_FAILED_PRECONDITION, + aborted = c.XLA_FFI_Error_Code_ABORTED, + out_of_range = c.XLA_FFI_Error_Code_OUT_OF_RANGE, + unimplemented = c.XLA_FFI_Error_Code_UNIMPLEMENTED, + internal = c.XLA_FFI_Error_Code_INTERNAL, + unavailable = c.XLA_FFI_Error_Code_UNAVAILABLE, + data_loss = c.XLA_FFI_Error_Code_DATA_LOSS, + unauthenticated = c.XLA_FFI_Error_Code_UNAUTHENTICATED, + + pub fn toApiError(code: ErrorCode) pjrt.ApiError { + return switch (code) { + .cancelled => error.Cancelled, + .unknown => error.Unknown, + .invalid_argument => error.InvalidArgument, + .deadline_exceeded => error.DeadlineExceeded, + .not_found => error.NotFound, + .already_exists => error.AlreadyExists, + .permission_denied => error.PermissionDenied, + .resource_exhausted => error.ResourceExhausted, + .failed_precondition => error.FailedPrecondition, + .aborted => error.Aborted, + .out_of_range => error.OutOfRange, + .unimplemented => error.Unimplemented, + .internal => error.Internal, + .unavailable => error.Unavailable, + .data_loss => error.DataLoss, + .unauthenticated => error.Unauthenticated, + }; + } +}; + +pub const Error = opaque { + pub const inner = TransmuteMixin(Error, c.XLA_FFI_Error).to; + pub const fromInner = TransmuteMixin(Error, c.XLA_FFI_Error).from; + + pub const ok: ?*Error = null; + + pub fn create(api: *const Api, error_code: ErrorCode, message: []const u8) *Error { + var ret: pjrt.meta.Struct(c.XLA_FFI_Error_Create_Args) = .{ + .message = message.ptr, + .errc = @backingInt(error_code), + }; + return fromInner(api.inner().XLA_FFI_Error_Create.?(@ptrCast(&ret)).?); + } + + pub fn destroy(err: *Error, api: *const Api) void { + var ret: pjrt.meta.Struct(c.XLA_FFI_Error_Destroy_Args) = .{ .@"error" = err.inner() }; + api.inner().XLA_FFI_Error_Destroy.?(@ptrCast(&ret)); + } + + pub fn getMessage(err: *Error, api: *const Api) [:0]const u8 { + var ret: pjrt.meta.Struct(c.XLA_FFI_Error_GetDetails_Args) = .{ + .@"error" = err.inner(), + }; + api.inner().XLA_FFI_Error_GetDetails.?(@ptrCast(&ret)); + return std.mem.span(ret.message); + } +}; + +pub const Future = opaque { + pub const inner = TransmuteMixin(Future, c.XLA_FFI_Future).to; + pub const fromInner = TransmuteMixin(Future, c.XLA_FFI_Future).from; + + pub fn create(api: *const Api) pjrt.ApiError!*Future { + var ret: pjrt.meta.Struct(c.XLA_FFI_Future_Create_Args) = .{}; + const result = api.inner().XLA_FFI_Future_Create.?(@ptrCast(&ret)); + + if (result) |ffi_error| { + const err = Error.fromInner(ffi_error); + defer err.destroy(api); + log.err("[Future.create] {s}", .{err.getMessage(api)}); + + // TODO(Corentin): Retrieve error code from Error when implemented in XLA. + return error.Unknown; + } + + return fromInner(ret.future.?); + } + + pub fn setAvailable(self: *Future, api: *const Api) pjrt.ApiError!void { + var ret: pjrt.meta.Struct(c.XLA_FFI_Future_SetAvailable_Args) = .{ + .future = self.inner(), + }; + + const result = api.inner().XLA_FFI_Future_SetAvailable.?(@ptrCast(&ret)); + + if (result) |ffi_error| { + const err = Error.fromInner(ffi_error); + defer err.destroy(api); + log.err("[Future.setAvailable] {s}", .{err.getMessage(api)}); + + // TODO(Corentin): Retrieve error code from Error when implemented in XLA. + return error.Unknown; + } + } + + pub fn setError(self: *Future, api: *const Api, err: *Error) pjrt.ApiError!void { + var ret: pjrt.meta.Struct(c.XLA_FFI_Future_SetError_Args) = .{ + .future = self.inner(), + .@"error" = err.inner(), + }; + + const result = api.inner().XLA_FFI_Future_SetError.?(@ptrCast(&ret)); + + if (result) |ffi_error| { + const err2 = Error.fromInner(ffi_error); + defer err2.destroy(api); + log.err("[Future.setError] {s}", .{err2.getMessage(api)}); + + // TODO(Corentin): Retrieve error code from Error when implemented in XLA. + return error.Unknown; + } + } +}; diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 5a4aaecb..3715361f 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -1,39 +1,60 @@ -const builtin = @import("builtin"); const std = @import("std"); +const builtin = @import("builtin"); const c = @import("c"); -const log = std.log.scoped(.pjrt); +pub const ffi = @import("ffi.zig"); -pub const Profiler = @import("profiler.zig").Profiler; +const log_scope = .pjrt; +const log = std.log.scoped(log_scope); test { std.testing.refAllDecls(@This()); } -// We could calculate it like PJRT does, but it turns out that some of those -// were wrong in PJRT itself [1], which gets propagated to binary plugins. In -// order to mirror that, we just the value as computed by PJRT itself, through -// comptime reflection. We could make the argument to remove that one day since -// [1] has been fixed. The problem is that this problem could happen again in -// as the way PJRT does it is not very robust. -// -// 1. https://github.com/openxla/xla/issues/10032 -fn pjrtStructSize(comptime T: type) usize { - // unsafe on purpose, we want this to fail if that ever changes - const typedef_name = comptime blk: { - const needle = ".struct_"; - const idx = std.mem.indexOf(u8, @typeName(T), needle).?; - break :blk @typeName(T)[idx + needle.len ..]; - }; - return @field(c, typedef_name ++ "_STRUCT_SIZE"); -} +pub const meta = struct { + // We could calculate it like PJRT does, but it turns out that some of those + // were wrong in PJRT itself [1], which gets propagated to binary plugins. In + // order to mirror that, we just the value as computed by PJRT itself, through + // comptime reflection. We could make the argument to remove that one day since + // [1] has been fixed. The problem is that this problem could happen again in + // as the way PJRT does it is not very robust. + // + // 1. https://github.com/openxla/xla/issues/10032 + pub fn structSize(comptime T: type) usize { + // unsafe on purpose, we want this to fail if that ever changes + const typedef_name = comptime blk: { + const needle = ".struct_"; + const idx = std.mem.indexOf(u8, @typeName(T), needle).?; + break :blk @typeName(T)[idx + needle.len ..]; + }; + return @field(c, typedef_name ++ "_STRUCT_SIZE"); + } -inline fn pjrtStruct(v: anytype) @TypeOf(v) { - var ret = v; - ret.struct_size = pjrtStructSize(@TypeOf(v)); - return ret; -} + pub fn Struct(comptime T: type) type { + const info = @typeInfo(T).@"struct"; + var names: [info.field_names.len][]const u8 = undefined; + var types: [info.field_names.len]type = undefined; + var attributes: [info.field_names.len]std.builtin.Type.Struct.FieldAttributes = undefined; + for (info.field_names, info.field_types, &names, &types, &attributes) |field_name, Field, *name, *type_, *attr| { + name.* = field_name; + type_.* = Field; + attr.* = .{ + .default_value_ptr = @ptrCast(if (std.mem.eql(u8, field_name, "struct_size")) + &structSize(T) + else + &std.mem.zeroes(Field)), + }; + } + return @Struct( + .@"extern", + null, + &names, + &types, + &attributes, + ); + } +}; pub const ApiError = error{ Cancelled, @@ -54,10 +75,19 @@ pub const ApiError = error{ Unauthenticated, }; +inline fn interpretPjrtError(api: *const Api, pjrt_error: *Error, context: []const u8, comptime log_level: std.log.Level) ApiError { + defer pjrt_error.deinit(api); + const err_code = pjrt_error.getCode(api).toApiError(); + if (comptime std.log.logEnabled(log_level, log_scope)) { + std.options.logFn(log_level, log_scope, "[{s}] {t}: {s}", .{ context, err_code, pjrt_error.getMessage(api) }); + } + return err_code; +} + fn InnerMixin(comptime innerT: type) type { return struct { fn inner(self: anytype) *innerT { - return @ptrCast(@constCast(@alignCast(self))); + return @ptrCast(@alignCast(@constCast(self))); } }; } @@ -66,70 +96,164 @@ pub const Api = struct { pub const Version = struct { major: i64, minor: i64, + + pub fn format(self: Version, writer: *std.Io.Writer) std.Io.Writer.Error!void { + try writer.print("{d}.{d}", .{ self.major, self.minor }); + } }; const Funcs = std.meta.FieldEnum(c.PJRT_Api); inner: c.PJRT_Api, - pub fn loadFrom(library: []const u8) !*const Api { + pub fn loadFrom(library: [:0]const u8) !*const Api { + const basename = std.Io.Dir.path.basename(library); + log.info("Loading: {s}...", .{basename}); + var lib: std.DynLib = switch (builtin.os.tag) { - .linux => blk: { - const library_c = try std.posix.toPosixPath(library); + .linux, .macos => blk: { + const rtld: std.c.RTLD = switch (builtin.os.tag) { + // We use RTLD_GLOBAL so that symbols from NEEDED libraries are available in the global namespace. + .linux => .{ .LAZY = true, .GLOBAL = true, .NODELETE = true }, + .macos => .{ .LAZY = true, .LOCAL = true }, + else => unreachable, + }; break :blk .{ .inner = .{ - .handle = c.dlopen(&library_c, c.RTLD_LAZY | c.RTLD_LOCAL | c.RTLD_NODELETE) orelse { + .handle = std.c.dlopen(library, rtld) orelse { + log.err("Unable to dlopen plugin {s}\n{s}", .{ library, std.c.dlerror() orelse "" }); return error.FileNotFound; }, }, }; }, - else => try std.DynLib.open(library), + else => std.DynLib.open(library) catch |err| { + log.err("Unable to dlopen plugin {s}: {}", .{ library, err }); + return err; + }, }; - const DynGetPjrtApi = lib.lookup(*const fn () callconv(.C) *const Api, "GetPjrtApi") orelse { - std.debug.panic("Unable to find GetPjrtApi symbol in library: {s}", .{library}); + + const api = fromDynLib(&lib) catch |err| { + log.err("Unable to load PJRT API from plugin {s}: {}", .{ library, err }); + return err; + }; + log.info("Loaded: {s}", .{basename}); + return api; + } + + pub fn fromDynLib(lib: *std.DynLib) !*const Api { + const DynGetPjrtApi = lib.lookup(*const fn () callconv(.c) *const Api, "GetPjrtApi") orelse { + return error.MissingGetPjrtApi; }; const api = DynGetPjrtApi(); - log.info("Loaded library: {s}", .{library}); - _ = api.call(.PJRT_Plugin_Initialize, .{}) catch unreachable; + _ = try api.call(.PJRT_Plugin_Initialize, .{}); return api; } - fn CallFnArgType(comptime func: Funcs) type { - const fti = @typeInfo(std.meta.FieldType(c.PJRT_Api, func)); - const fn_ptr = @typeInfo(fti.Optional.child); - const fn_type_info = @typeInfo(fn_ptr.Pointer.child); - const arg_array_type_info = @typeInfo(fn_type_info.Fn.params[0].type.?); - return arg_array_type_info.Pointer.child; + fn PJRTFnArg(comptime func: Funcs) type { + const fti = @typeInfo(@FieldType(c.PJRT_Api, @tagName(func))); + const fn_ptr = @typeInfo(fti.optional.child); + const fn_type_info = @typeInfo(fn_ptr.pointer.child); + const arg_array_type_info = @typeInfo(fn_type_info.@"fn".param_types[0].?); + return arg_array_type_info.pointer.child; } - inline fn call(self: *const Api, comptime method: Funcs, arg: CallFnArgType(method)) ApiError!@TypeOf(arg) { - var ret = pjrtStruct(arg); - const fn_ptr = @field(&self.inner, @tagName(method)).?; - const result = fn_ptr(&ret); - if (@TypeOf(result) == void) { - return ret; - } + fn PJRTFnArgWithDefault(comptime func: Funcs) type { + const argT = PJRTFnArg(func); + return switch (@typeInfo(argT)) { + .@"struct" => meta.Struct(argT), + else => argT, + }; + } + inline fn innerCall(self: *const Api, comptime method: Funcs, arg: *PJRTFnArg(method)) ?*Error { + if (@offsetOf(c.PJRT_Api, @tagName(method)) > self.inner.struct_size) { + std.debug.panic("PJRT Api method {s} not available in this plugin", .{@tagName(method)}); + } + const fn_ptr = @field(&self.inner, @tagName(method)).?; + const result = fn_ptr(arg); + if (@TypeOf(result) == void) { + return null; + } if (result) |pjrt_c_error| { const pjrt_error: *Error = @ptrCast(pjrt_c_error); - log.err("[{s}] {s}", .{ @tagName(method), pjrt_error.getMessage(self) }); - return pjrt_error.getCode(self).toApiError(); + return pjrt_error; } + return null; + } + inline fn call(self: *const Api, comptime method: Funcs, arg: PJRTFnArgWithDefault(method)) ApiError!PJRTFnArgWithDefault(method) { + var ret = arg; + const pjrt_error = innerCall(self, method, @ptrCast(&ret)); + if (pjrt_error) |err| return interpretPjrtError(self, err, @tagName(method), .warn); return ret; } - pub fn lookupExtension(self: *const Api, comptime ExtensionT: type, ext_id: c_int) ?*const ExtensionT { - var cur: [*c]const c.PJRT_Extension_Base = @alignCast(@ptrCast(self.inner.extension_start)); - while (cur != null) : (cur = cur.*.next) { - if (cur.*.type == ext_id) { - return @alignCast(@ptrCast(cur)); + inline fn callSilent(self: *const Api, comptime method: Funcs, arg: PJRTFnArgWithDefault(method)) ApiError!PJRTFnArgWithDefault(method) { + var ret = arg; + const pjrt_error = innerCall(self, method, @ptrCast(&ret)); + if (pjrt_error) |err| return interpretPjrtError(self, err, @tagName(method), .debug); + return ret; + } + + pub const Extensions = struct { + pub const Type = enum(c.PJRT_Extension_Type) { + custom_call = c.PJRT_Extension_Type_Gpu_Custom_Call, + profiler = c.PJRT_Extension_Type_Profiler, + stream = c.PJRT_Extension_Type_Stream, + layouts = c.PJRT_Extension_Type_Layouts, + ffi = c.PJRT_Extension_Type_FFI, + memory_descriptions = c.PJRT_Extension_Type_MemoryDescriptions, + raw_buffer = c.PJRT_Extension_Type_RawBuffer, + phase_compile = c.PJRT_Extension_Type_PhaseCompile, + unknown = c.PJRT_Extension_Type_Unknown, + }; + + pub const Extension = union(Type) { + custom_call: *const c.PJRT_Gpu_Custom_Call, + profiler: *const c.PJRT_Profiler_Extension, + stream: *const c.PJRT_Stream_Extension, + layouts: *const c.PJRT_Layouts_Extension, + ffi: *const c.PJRT_FFI_Extension, + memory_descriptions: *const c.PJRT_MemoryDescriptions_Extension, + raw_buffer: *const c.PJRT_RawBuffer_Extension, + phase_compile: *const c.PJRT_PhaseCompile_Extension, + unknown: *const c.PJRT_Extension_Base, + }; + + pub const Iterator = struct { + current: ?*const c.PJRT_Extension_Base, + + pub fn next(self: *Iterator) ?Extension { + defer if (self.current) |cur| { + self.current = cur.next; + }; + if (self.current) |cur| { + if (std.enums.fromInt(Extensions.Type, cur.type)) |e| { + return switch (e) { + inline else => |t| @unionInit(Extension, @tagName(t), @ptrCast(self.current)), + }; + } + return .{ .unknown = cur }; + } + return null; + } + }; + }; + + pub fn extensions(self: *const Api) Extensions.Iterator { + return .{ .current = @ptrCast(@alignCast(self.inner.extension_start)) }; + } + + pub fn extension(self: *const Api, ext_type: Extensions.Type) ?Extensions.Extension { + var it = self.extensions(); + while (it.next()) |ext| { + if (std.meta.activeTag(ext) == ext_type) { + return ext; } } - return null; } @@ -139,6 +263,65 @@ pub const Api = struct { .minor = @intCast(self.inner.pjrt_api_version.minor_version), }; } + + pub fn stablehloCurrentVersion(self: *const Api) ?[]const u8 { + const state = struct { + var buf: [32]u8 = undefined; + var str: ?[]const u8 = null; + }; + if (state.str) |str| { + return str; + } + if (self.pluginAttribute("stablehlo_current_version")) |nv| { + switch (nv.value()) { + .int64list => |v| { + state.str = std.mem.print(&state.buf, "{d}.{d}.{d}", .{ v[0], v[1], v[2] }) catch unreachable; + }, + else => unreachable, + } + } + return state.str; + } + + pub fn createExecuteContext(api: *const Api) ApiError!*ExecuteContext { + const ret = try api.call(.PJRT_ExecuteContext_Create, .{}); + return @ptrCast(ret.context.?); + } + + pub fn ffi(api: *const Api) ?Ffi { + if (api.extension(.ffi)) |ext| { + return .{ .inner = ext.ffi }; + } + return null; + } + + pub fn profiler(self: *const Api, options_pb: []const u8) ApiError!?Profiler { + if (self.extension(.profiler)) |ext| { + return try Profiler.init(self, ext.profiler, options_pb); + } + + return null; + } + + pub fn pluginAttribute(api: *const Api, key: []const u8) ?NamedValue { + const attributes = api.pluginAttributes(); + for (attributes) |attr| { + if (std.mem.eql(u8, attr.name(), key)) { + return attr; + } + } + return null; + } + + pub fn pluginAttributes(api: *const Api) []const NamedValue { + const ret = api.call(.PJRT_Plugin_Attributes, .{ + .extension_start = null, + }) catch unreachable; + + if (ret.attributes == null) return &.{}; + + return @ptrCast(ret.attributes[0..ret.num_attributes]); + } }; pub const ErrorCode = enum(c.PJRT_Error_Code) { @@ -161,43 +344,45 @@ pub const ErrorCode = enum(c.PJRT_Error_Code) { pub fn toApiError(code: ErrorCode) ApiError { return switch (code) { - .cancelled => error.Cancelled, - .unknown => error.Unknown, - .invalid_argument => error.InvalidArgument, - .deadline_exceeded => error.DeadlineExceeded, - .not_found => error.NotFound, - .already_exists => error.AlreadyExists, - .permission_denied => error.PermissionDenied, - .resource_exhausted => error.ResourceExhausted, - .failed_precondition => error.FailedPrecondition, - .aborted => error.Aborted, - .out_of_range => error.OutOfRange, - .unimplemented => error.Unimplemented, - .internal => error.Internal, - .unavailable => error.Unavailable, - .data_loss => error.DataLoss, - .unauthenticated => error.Unauthenticated, + .cancelled => ApiError.Cancelled, + .unknown => ApiError.Unknown, + .invalid_argument => ApiError.InvalidArgument, + .deadline_exceeded => ApiError.DeadlineExceeded, + .not_found => ApiError.NotFound, + .already_exists => ApiError.AlreadyExists, + .permission_denied => ApiError.PermissionDenied, + .resource_exhausted => ApiError.ResourceExhausted, + .failed_precondition => ApiError.FailedPrecondition, + .aborted => ApiError.Aborted, + .out_of_range => ApiError.OutOfRange, + .unimplemented => ApiError.Unimplemented, + .internal => ApiError.Internal, + .unavailable => ApiError.Unavailable, + .data_loss => ApiError.DataLoss, + .unauthenticated => ApiError.Unauthenticated, }; } }; pub const Error = opaque { + const inner = InnerMixin(c.PJRT_Error).inner; + pub fn deinit(self: *Error, api: *const Api) void { _ = api.call(.PJRT_Error_Destroy, .{ - .@"error" = @ptrCast(self), + .@"error" = self.inner(), }) catch unreachable; } pub fn getCode(self: *Error, api: *const Api) ErrorCode { const ret = api.call(.PJRT_Error_GetCode, .{ - .@"error" = @ptrCast(self), + .@"error" = self.inner(), }) catch unreachable; - return @enumFromInt(ret.code); + return @fromBackingInt(@intCast(ret.code)); } pub fn getMessage(self: *Error, api: *const Api) []const u8 { const ret = api.call(.PJRT_Error_Message, .{ - .@"error" = @ptrCast(self), + .@"error" = self.inner(), }) catch unreachable; return ret.message[0..ret.message_size]; } @@ -205,6 +390,34 @@ pub const Error = opaque { pub const ClientInitError = error{LoadingFailed} || ApiError; +pub const ShapeSpec = extern struct { + comptime { + std.debug.assert(@sizeOf(ShapeSpec) == @sizeOf(c.PJRT_ShapeSpec)); + } + + inner: meta.Struct(c.PJRT_ShapeSpec), + + pub fn init(dims_: []const i64, bt: BufferType) ShapeSpec { + return .{ + .inner = .{ + .dims = @ptrCast(@constCast(dims_)), + .num_dims = dims_.len, + .element_type = @backingInt(bt), + }, + }; + } + + pub fn dims(self: ShapeSpec) []const i64 { + return self.inner.dims[0..self.inner.num_dims]; + } + + pub fn bufferType(self: ShapeSpec) BufferType { + return @fromBackingInt(@intCast(self.inner.element_type)); + } +}; + +pub const Stream = opaque {}; + pub const Client = opaque { const inner = InnerMixin(c.PJRT_Client).inner; @@ -216,12 +429,8 @@ pub const Client = opaque { pub fn init(api: *const Api, create_options: []const NamedValue) ClientInitError!*Client { // log.info("Loaded PJRT runtime plugin: {s}", .{api.Platform}); const ret = try api.call(.PJRT_Client_Create, .{ - .create_options = @ptrCast(create_options.ptr), + .create_options = @ptrCast(create_options), .num_options = create_options.len, - .kv_get_callback = null, - .kv_put_callback = null, - .kv_put_user_arg = null, - .kv_get_user_arg = null, }); return @ptrCast(ret.client.?); } @@ -232,21 +441,21 @@ pub const Client = opaque { }) catch {}; } - pub fn getPlatformName(self: *const Client, api: *const Api) []const u8 { + pub fn platformName(self: *const Client, api: *const Api) []const u8 { const ret = api.call(.PJRT_Client_PlatformName, .{ .client = self.inner(), }) catch unreachable; return ret.platform_name[0..ret.platform_name_size]; } - pub fn getDevices(self: *const Client, api: *const Api) []const *Device { + pub fn devices(self: *const Client, api: *const Api) []const *Device { const ret = api.call(.PJRT_Client_Devices, .{ .client = self.inner(), }) catch unreachable; return @ptrCast(ret.devices[0..ret.num_devices]); } - pub fn getAddressableDevices(self: *const Client, api: *const Api) []const *Device { + pub fn addressableDevices(self: *const Client, api: *const Api) []const *Device { const ret = api.call(.PJRT_Client_AddressableDevices, .{ .client = self.inner(), }) catch unreachable; @@ -259,66 +468,61 @@ pub const Client = opaque { compile_options_pb: []const u8, }; - pub fn compile(self: *const Client, api: *const Api, args: CompileArgs) ApiError!*LoadedExecutable { + pub fn compileRaw(self: *const Client, api: *const Api, args: CompileArgs) ApiError!*LoadedExecutable { const bytecode_format_ = @tagName(args.bytecode_format); const ret = try api.call(.PJRT_Client_Compile, .{ - .program = &pjrtStruct(c.PJRT_Program{ - .code = @ptrCast(@constCast(args.bytecode.ptr)), + .program = @ptrCast(&meta.Struct(c.PJRT_Program){ + .code = @ptrCast(@constCast(args.bytecode)), .code_size = args.bytecode.len, - .format = @ptrCast(@constCast(bytecode_format_.ptr)), + .format = @ptrCast(@constCast(bytecode_format_)), .format_size = bytecode_format_.len, }), - .compile_options = @ptrCast(@constCast(args.compile_options_pb.ptr)), + .compile_options = @ptrCast(@constCast(args.compile_options_pb)), .compile_options_size = args.compile_options_pb.len, .client = self.inner(), }); return @ptrCast(ret.executable.?); } + pub fn compile(self: *const Client, api: *const Api, io: std.Io, args: CompileArgs) ApiError!*LoadedExecutable { + var future = io.async(compileRaw, .{ self, api, args }); + return future.await(io); + } + pub const BufferFromHostBufferArgs = struct { - data: []const u8, + data: [*]const u8, buffer_type: BufferType, dims: []const i64, byte_strides: ?[]const i64, - device: *const Device, + layout: MemoryLayout, host_buffer_semantics: HostBufferSemantics, + dst: union(enum) { + device: *const Device, + memory: *const Memory, + }, }; - pub fn bufferFromHostBuffer(self: *const Client, api: *const Api, args: BufferFromHostBufferArgs) ApiError!struct { *Buffer, *Event } { + pub fn bufferFromHostBuffer(self: *const Client, api: *const Api, args: BufferFromHostBufferArgs) ApiError!struct { *Buffer, ?*Event } { const ret = try api.call(.PJRT_Client_BufferFromHostBuffer, .{ .client = self.inner(), - .data = @ptrCast(@constCast(args.data.ptr)), - .type = @intFromEnum(args.buffer_type), - .dims = @ptrCast(@constCast(args.dims.ptr)), + .data = @constCast(args.data), + .type = @backingInt(args.buffer_type), + .dims = @ptrCast(@constCast(args.dims)), .num_dims = args.dims.len, - .byte_strides = if (args.byte_strides) |bs| @ptrCast(@constCast(bs.ptr)) else null, + .byte_strides = if (args.byte_strides) |bs| @ptrCast(@constCast(bs)) else null, .num_byte_strides = if (args.byte_strides) |bs| bs.len else 0, - .host_buffer_semantics = @intFromEnum(args.host_buffer_semantics), - .device = @ptrCast(@constCast(args.device)), - .memory = null, // TODO - .device_layout = null, // TODO - .done_with_host_buffer = null, - .buffer = null, + .device_layout = @ptrCast(@constCast(&args.layout.toCStruct())), + .host_buffer_semantics = @backingInt(args.host_buffer_semantics), + .device = if (args.dst == .device) @ptrCast(@constCast(args.dst.device)) else null, + .memory = if (args.dst == .memory) args.dst.memory.inner() else null, }); + return .{ @ptrCast(ret.buffer.?), - @ptrCast(ret.done_with_host_buffer.?), + @ptrCast(ret.done_with_host_buffer), }; } - /// Returns the Profiler for this API. - /// Not all platform have a profiling api, for those the profiler object will do nothing. - /// Platforms with known profiler extensions: cuda, xpu - pub fn getProfiler(self: *const Client, api: *const Api, options: Profiler.Options) Profiler { - if (api.version().minor >= 45) { - if (api.lookupExtension(c.PJRT_Profiler_Extension, c.PJRT_Extension_Type_Profiler)) |ext| { - return Profiler.init(ext.profiler_api.*, options); - } - } - log.warn("No profiler found for platform: {}", .{self}); - return Profiler.init(null, options); - } - pub fn deserializeAndLoad(self: *const Client, api: *const Api, bytes: []const u8) ApiError!*LoadedExecutable { const ret = try api.call(.PJRT_Executable_DeserializeAndLoad, .{ .client = self.inner(), @@ -329,29 +533,159 @@ pub const Client = opaque { } pub const CreateViewOfDeviceBufferArgs = struct { - data: []const u8, + device_buffer_ptr: *anyopaque, dims: []const i64, element_type: BufferType, layout: MemoryLayout, - device: *const Device, - on_delete_callback: ?*const fn (device_buffer_ptr: ?*anyopaque, ctx: ?*anyopaque) callconv(.C) void = null, + device: ?*const Device = null, + /// The memory holding `device_buffer_ptr`. Takes precedence over `device`, + /// which designates its default memory. + memory: ?*const Memory = null, + on_delete_callback: *const fn (device_buffer_ptr: ?*anyopaque, ctx: ?*anyopaque) callconv(.c) void = &struct { + fn call(_: ?*anyopaque, _: ?*anyopaque) callconv(.c) void {} + }.call, on_delete_callback_arg: ?*anyopaque = null, - stream: ?isize = null, + stream: ?*const Stream = null, }; pub fn createViewOfDeviceBuffer(self: *const Client, api: *const Api, args: CreateViewOfDeviceBufferArgs) ApiError!*Buffer { const layout = args.layout.toCStruct(); const ret = try api.call(.PJRT_Client_CreateViewOfDeviceBuffer, .{ .client = self.inner(), - .device_buffer_ptr = @ptrCast(@constCast(args.data.ptr)), + .device_buffer_ptr = @constCast(args.device_buffer_ptr), .dims = args.dims.ptr, .num_dims = args.dims.len, - .element_type = @intFromEnum(args.element_type), + .element_type = @backingInt(args.element_type), .layout = @ptrCast(@constCast(&layout)), .device = @ptrCast(@constCast(args.device)), .on_delete_callback = args.on_delete_callback, .on_delete_callback_arg = args.on_delete_callback_arg, - .stream = if (args.stream) |stream| stream else 0, + .stream = @bitCast(@intFromPtr(args.stream)), + .memory = if (args.memory) |memory| memory.inner() else null, + }); + return @ptrCast(ret.buffer.?); + } + + pub fn addressableMemories(self: *const Client, api: *const Api) []*const Memory { + const ret = api.call(.PJRT_Client_AddressableMemories, .{ + .client = self.inner(), + }) catch return &.{}; + if (ret.addressable_memories) |memories| { + return @ptrCast(@constCast(memories[0..ret.num_addressable_memories])); + } + return &.{}; + } + + pub fn dmaMap(self: *const Client, api: *const Api, data: []const u8) ApiError!void { + _ = try api.call(.PJRT_Client_DmaMap, .{ + .client = self.inner(), + .data = @ptrCast(@constCast(data)), + .size = @intCast(data.len), + }); + } + + pub fn dmaUnmap(self: *const Client, api: *const Api, data: []const u8) ApiError!void { + _ = try api.call(.PJRT_Client_DmaUnmap, .{ + .client = self.inner(), + .data = @ptrCast(@constCast(data)), + }); + } + + pub const CreateBuffersForAsyncHostToDeviceArgs = struct { + shape_specs: []const ShapeSpec, + device_layouts: ?[]*const MemoryLayout = null, + memory: *const Memory, + }; + + pub fn createBuffersForAsyncHostToDevice(self: *const Client, api: *const Api, args: CreateBuffersForAsyncHostToDeviceArgs) ApiError!*AsyncHostToDeviceTransferManager { + const ret = try api.call(.PJRT_Client_CreateBuffersForAsyncHostToDevice, .{ + .client = self.inner(), + .shape_specs = @ptrCast(@constCast(args.shape_specs)), + .num_shape_specs = args.shape_specs.len, + .device_layouts = if (args.device_layouts) |layouts| @ptrCast(@constCast(layouts)) else null, + .num_device_layouts = if (args.device_layouts) |layouts| layouts.len else 0, + .memory = args.memory.inner(), + }); + return @ptrCast(ret.transfer_manager.?); + } + + pub fn defaultMemoryLayout( + self: *const Client, + api: *const Api, + element_type: BufferType, + dims: []const i64, + ) !DefaultMemoryLayout { + const ext = if (api.extension(.layouts)) |e| e.layouts else return error.LayoutsExtensionUnavailable; + + var get_args: c.PJRT_Layouts_PJRT_Client_GetDefaultLayout_Args = .{ + .struct_size = c.PJRT_Layouts_PJRT_Client_GetDefaultLayout_Args_STRUCT_SIZE, + .extension_start = null, + .client = self.inner(), + .type = @backingInt(element_type), + .dims = dims.ptr, + .num_dims = dims.len, + .layout = null, + }; + if (ext.PJRT_Layouts_PJRT_Client_GetDefaultLayout.?(&get_args)) |pjrt_c_error| { + const pjrt_error: *Error = @ptrCast(pjrt_c_error); + defer pjrt_error.deinit(api); + return pjrt_error.getCode(api).toApiError(); + } + defer { + var destroy_args: c.PJRT_Layouts_MemoryLayout_Destroy_Args = .{ + .struct_size = c.PJRT_Layouts_MemoryLayout_Destroy_Args_STRUCT_SIZE, + .extension_start = null, + .layout = get_args.layout, + }; + if (ext.PJRT_Layouts_MemoryLayout_Destroy.?(&destroy_args)) |pjrt_c_error| { + const pjrt_error: *Error = @ptrCast(pjrt_c_error); + defer pjrt_error.deinit(api); + log.err("[PJRT_Layouts_MemoryLayout_Destroy] {s}", .{pjrt_error.getMessage(api)}); + } + } + + var serialize_args: c.PJRT_Layouts_MemoryLayout_Serialize_Args = .{ + .struct_size = c.PJRT_Layouts_MemoryLayout_Serialize_Args_STRUCT_SIZE, + .extension_start = null, + .layout = get_args.layout, + .serialized_bytes = null, + .serialized_bytes_size = 0, + .serialized_layout = null, + .serialized_layout_deleter = null, + }; + if (ext.PJRT_Layouts_MemoryLayout_Serialize.?(&serialize_args)) |pjrt_c_error| { + const pjrt_error: *Error = @ptrCast(pjrt_c_error); + defer pjrt_error.deinit(api); + return pjrt_error.getCode(api).toApiError(); + } + defer if (serialize_args.serialized_layout_deleter) |deleter| { + deleter(serialize_args.serialized_layout); + }; + + const serialized = serialize_args.serialized_bytes[0..serialize_args.serialized_bytes_size]; + return try DefaultMemoryLayout.parseSerialized(serialized); + } + + pub const CreateUninitializedBufferArgs = struct { + dims: []const i64, + element_type: BufferType, + layout: MemoryLayout, + dst: union(enum) { + device: *const Device, + memory: *const Memory, + }, + }; + + pub fn createUninitializedBuffer(self: *const Client, api: *const Api, args: CreateUninitializedBufferArgs) ApiError!*Buffer { + var layout = args.layout.toCStruct(); + const ret = try api.call(.PJRT_Client_CreateUninitializedBuffer, .{ + .client = self.inner(), + .shape_dims = args.dims.ptr, + .shape_num_dims = args.dims.len, + .shape_element_type = @backingInt(args.element_type), + .shape_layout = @ptrCast(&layout), + .device = if (args.dst == .device) @ptrCast(@constCast(args.dst.device)) else null, + .memory = if (args.dst == .memory) args.dst.memory.inner() else null, }); return @ptrCast(ret.buffer.?); } @@ -360,6 +694,40 @@ pub const Client = opaque { pub const Device = opaque { const inner = InnerMixin(c.PJRT_Device).inner; + pub const Attributes = struct { + values: []const NamedValue, + state: *c.PJRT_Device_Attributes, + deleter: *const fn (?*c.PJRT_Device_Attributes) callconv(.c) void, + + pub fn deinit(self: Attributes) void { + self.deleter(self.state); + } + + /// String and list values remain valid until this owner is deinitialized. + pub fn get(self: Attributes, name: []const u8) ?NamedValue.Value { + for (self.values) |attribute| { + if (std.mem.eql(u8, attribute.name(), name)) return attribute.value(); + } + return null; + } + }; + + pub fn attributes(self: *const Device, api: *const Api) ApiError!Attributes { + const ret = try api.call(.PJRT_Device_GetAttributes, .{ .device = self.inner() }); + + const deleter = ret.attributes_deleter.?; + errdefer deleter(ret.device_attributes); + + const state = ret.device_attributes.?; + std.debug.assert((ret.attributes == null) == (ret.num_attributes == 0)); + + return .{ + .values = if (ret.attributes == null) &.{} else @ptrCast(ret.attributes[0..ret.num_attributes]), + .state = state, + .deleter = deleter, + }; + } + pub fn getDescription(self: *const Device, api: *const Api) *const DeviceDescription { const ret = api.call(.PJRT_Device_GetDescription, .{ .device = self.inner(), @@ -374,32 +742,107 @@ pub const Device = opaque { return ret.is_addressable; } - pub fn getLocalHardwareId(self: *const Device, api: *const Api) usize { + pub fn localHardwareId(self: *const Device, api: *const Api) usize { const ret = api.call(.PJRT_Device_LocalHardwareId, .{ .device = self.inner(), }) catch unreachable; return @intCast(ret.local_hardware_id); } + + pub fn addressableMemories(self: *const Device, api: *const Api) []const *Memory { + const ret = api.call( + .PJRT_Device_AddressableMemories, + .{ .device = self.inner() }, + ) catch return &.{}; + return @ptrCast(ret.memories[0..ret.num_memories]); + } + + pub fn addressableMemory(client: *const Client, api: *const Api, kind: Memory.Kind) ?*const Memory { + for (client.addressableMemories(api)) |mem| { + if (mem.kind(api) == kind) { + return mem; + } + } + return null; + } + + pub fn defaultMemory(self: *const Device, api: *const Api) *const Memory { + const ret = api.call(.PJRT_Device_DefaultMemory, .{ + .device = self.inner(), + }) catch unreachable; + return @ptrCast(ret.memory); + } + + pub const MemoryStats = struct { + /// Number of bytes in use. + bytes_in_use: u64, + /// The peak bytes in use. + peak_bytes_in_use: ?u64, + /// Number of allocations. + num_allocs: ?u64, + /// The largest single allocation seen. + largest_alloc_size: ?u64, + /// The upper limit of user-allocatable device memory in bytes. + bytes_limit: ?u64, + /// Number of bytes reserved. + bytes_reserved: ?u64, + /// The peak number of bytes reserved. + peak_bytes_reserved: ?u64, + /// The upper limit on the number bytes of reservable memory. + bytes_reservable_limit: ?u64, + /// Largest free block size in bytes. + largest_free_block_bytes: ?u64, + /// Number of bytes of memory held by the allocator. This may be higher than + /// bytes_in_use if the allocator holds a pool of memory (e.g. BFCAllocator). + pool_bytes: ?u64, + peak_pool_bytes: ?u64, + + pub const zeroes = std.mem.zeroes(MemoryStats); + + fn fromCStruct(v: meta.Struct(c.PJRT_Device_MemoryStats_Args)) MemoryStats { + return .{ + .bytes_in_use = @intCast(v.bytes_in_use), + .peak_bytes_in_use = if (v.peak_bytes_in_use_is_set) @intCast(v.peak_bytes_in_use) else null, + .num_allocs = if (v.num_allocs_is_set) @intCast(v.num_allocs) else null, + .largest_alloc_size = if (v.largest_alloc_size_is_set) @intCast(v.largest_alloc_size) else null, + .bytes_limit = if (v.bytes_limit_is_set) @intCast(v.bytes_limit) else null, + .bytes_reserved = if (v.bytes_reserved_is_set) @intCast(v.bytes_reserved) else null, + .peak_bytes_reserved = if (v.peak_bytes_reserved_is_set) @intCast(v.peak_bytes_reserved) else null, + .bytes_reservable_limit = if (v.bytes_reservable_limit_is_set) @intCast(v.bytes_reservable_limit) else null, + .largest_free_block_bytes = if (v.largest_free_block_bytes_is_set) @intCast(v.largest_free_block_bytes) else null, + .pool_bytes = if (v.pool_bytes_is_set) @intCast(v.pool_bytes) else null, + .peak_pool_bytes = if (v.peak_pool_bytes_is_set) @intCast(v.peak_pool_bytes) else null, + }; + } + }; + + pub fn memoryStats(self: *const Device, api: *const Api) ApiError!MemoryStats { + // callSilent: If this API is not available no point in printing it at every call + const ret = try api.callSilent(.PJRT_Device_MemoryStats, .{ + .device = self.inner(), + }); + return .fromCStruct(ret); + } }; pub const DeviceDescription = opaque { const inner = InnerMixin(c.PJRT_DeviceDescription).inner; - pub fn getId(self: *const DeviceDescription, api: *const Api) usize { + pub fn id(self: *const DeviceDescription, api: *const Api) usize { const ret = api.call(.PJRT_DeviceDescription_Id, .{ .device_description = self.inner(), }) catch unreachable; return @intCast(ret.id); } - pub fn getProcessIndex(self: *const DeviceDescription, api: *const Api) usize { + pub fn processIndex(self: *const DeviceDescription, api: *const Api) usize { const ret = api.call(.PJRT_DeviceDescription_ProcessIndex, .{ .device_description = self.inner(), }) catch unreachable; return @intCast(ret.process_index); } - pub fn getKind(self: *const DeviceDescription, api: *const Api) []const u8 { + pub fn kind(self: *const DeviceDescription, api: *const Api) []const u8 { const ret = api.call(.PJRT_DeviceDescription_Kind, .{ .device_description = self.inner(), }) catch unreachable; @@ -419,6 +862,25 @@ pub const DeviceDescription = opaque { }) catch unreachable; return ret.to_string[0..ret.to_string_size]; } + + pub fn attributes(self: *const DeviceDescription, api: *const Api) []const NamedValue { + const ret = api.call(.PJRT_DeviceDescription_Attributes, .{ + .device_description = self.inner(), + }) catch unreachable; + + if (ret.attributes == null) return &.{}; + + return @ptrCast(ret.attributes[0..ret.num_attributes]); + } + + pub fn attribute(self: *const DeviceDescription, api: *const Api, name: []const u8) ?NamedValue.Value { + for (self.attributes(api)) |attr| { + if (std.mem.eql(u8, attr.name(), name)) { + return attr.value(); + } + } + return null; + } }; pub const GetCostAnalysisError = std.mem.Allocator.Error || ApiError; @@ -426,7 +888,7 @@ pub const GetCostAnalysisError = std.mem.Allocator.Error || ApiError; pub const SerializeResult = struct { bytes: []const u8, handle: *anyopaque, - deleter: *const fn (?*anyopaque) callconv(.C) void, + deleter: *const fn (?*anyopaque) callconv(.c) void, pub fn deinit(self: *SerializeResult) void { self.deleter(self.handle); @@ -435,6 +897,14 @@ pub const SerializeResult = struct { } }; +pub const ExecuteContext = opaque { + pub fn deinit(self: *ExecuteContext, api: *const Api) void { + _ = api.call(.PJRT_ExecuteContext_Destroy, .{ + .context = @ptrCast(self), + }) catch {}; + } +}; + pub const Executable = opaque { const inner = InnerMixin(c.PJRT_Executable).inner; @@ -444,11 +914,11 @@ pub const Executable = opaque { }) catch unreachable; } - pub fn getCostAnalysis(self: *const Executable, api: *const Api) GetCostAnalysisError![]*const NamedValue { + pub fn costAnalysis(self: *const Executable, api: *const Api) GetCostAnalysisError![]const NamedValue { const ret = try api.call(.PJRT_Executable_GetCostAnalysis, .{ .executable = self.inner(), }); - const values: [*]*const NamedValue = @ptrCast(ret.properties); + const values: [*]const NamedValue = @ptrCast(ret.properties); return values[0..ret.num_properties]; } @@ -463,6 +933,42 @@ pub const Executable = opaque { .deleter = @ptrCast(ret.serialized_executable_deleter.?), }; } + + pub const CompiledMemoryStats = struct { + /// Mirrors xla::CompiledMemoryStats. + /// Device default memory (e.g., HBM for GPU/TPU) usage stats. + generated_code_size_in_bytes: u64, + argument_size_in_bytes: u64, + output_size_in_bytes: u64, + /// much: How argument is reused for output. + alias_size_in_bytes: u64, + temp_size_in_bytes: u64, + + /// memory: Host usage stats. + host_generated_code_size_in_bytes: u64, + host_argument_size_in_bytes: u64, + host_output_size_in_bytes: u64, + host_alias_size_in_bytes: u64, + host_temp_size_in_bytes: u64, + }; + + pub fn getCompiledMemoryStats(self: *const Executable, api: *const Api) ApiError!CompiledMemoryStats { + const ret = try api.call(.PJRT_Executable_GetCompiledMemoryStats, .{ + .executable = self.inner(), + }); + return .{ + .generated_code_size_in_bytes = @intCast(ret.generated_code_size_in_bytes), + .argument_size_in_bytes = @intCast(ret.argument_size_in_bytes), + .output_size_in_bytes = @intCast(ret.output_size_in_bytes), + .alias_size_in_bytes = @intCast(ret.alias_size_in_bytes), + .temp_size_in_bytes = @intCast(ret.temp_size_in_bytes), + .host_generated_code_size_in_bytes = @intCast(ret.host_generated_code_size_in_bytes), + .host_argument_size_in_bytes = @intCast(ret.host_argument_size_in_bytes), + .host_output_size_in_bytes = @intCast(ret.host_output_size_in_bytes), + .host_alias_size_in_bytes = @intCast(ret.host_alias_size_in_bytes), + .host_temp_size_in_bytes = @intCast(ret.host_temp_size_in_bytes), + }; + } }; pub const LoadedExecutable = opaque { @@ -472,7 +978,6 @@ pub const LoadedExecutable = opaque { _ = api.call(.PJRT_LoadedExecutable_Destroy, .{ .executable = self.inner(), }) catch {}; - self.* = undefined; } pub fn delete(self: *LoadedExecutable, api: *const Api) void { @@ -488,42 +993,40 @@ pub const LoadedExecutable = opaque { return ret.is_deleted; } - pub fn getAddressableDevices(self: *const LoadedExecutable, api: *const Api) []Device { + pub fn addressableDevices(self: *const LoadedExecutable, api: *const Api) []const *Device { const ret = api.call(.PJRT_LoadedExecutable_AddressableDevices, .{ .executable = self.inner(), }) catch unreachable; - return @ptrCast(ret.addressable_devices); + return @ptrCast(ret.addressable_devices[0..ret.num_addressable_devices]); } - pub fn execute(self: *const LoadedExecutable, api: *const Api, args: struct { + pub const ExecuteArgs = struct { num_args: usize, arguments: []const [*]const *const Buffer, results: []const [*]*Buffer, - events: []*Event, + events: ?[]?*Event, non_donatable_input_indices: []const i64 = &.{}, - }) ApiError!void { - var options = pjrtStruct(c.PJRT_ExecuteOptions{ - .send_callbacks = null, - .recv_callbacks = null, - .num_send_ops = 0, - .num_recv_ops = 0, - .launch_id = 0, - .non_donatable_input_indices = @ptrCast(args.non_donatable_input_indices.ptr), + context: ?*ExecuteContext, + }; + + pub fn execute(self: *const LoadedExecutable, api: *const Api, args: ExecuteArgs) ApiError!void { + var options: meta.Struct(c.PJRT_ExecuteOptions) = .{ + .non_donatable_input_indices = @ptrCast(args.non_donatable_input_indices), .num_non_donatable_input_indices = args.non_donatable_input_indices.len, - }); + .context = @ptrCast(args.context), + }; _ = try api.call(.PJRT_LoadedExecutable_Execute, .{ .executable = self.inner(), .options = @ptrCast(&options), - .argument_lists = @ptrCast(args.arguments.ptr), - .num_devices = @intCast(args.arguments.len), + .argument_lists = @ptrCast(args.arguments), + .num_devices = args.arguments.len, .num_args = args.num_args, - .output_lists = @ptrCast(args.results.ptr), - .device_complete_events = @ptrCast(args.events.ptr), - .execute_device = null, + .output_lists = @ptrCast(args.results), + .device_complete_events = if (args.events) |ev| @ptrCast(ev) else null, }); } - pub fn getExecutable(self: *LoadedExecutable, api: *const Api) ApiError!*Executable { + pub fn executable(self: *const LoadedExecutable, api: *const Api) ApiError!*Executable { const ret = try api.call(.PJRT_LoadedExecutable_GetExecutable, .{ .loaded_executable = self.inner(), }); @@ -532,34 +1035,40 @@ pub const LoadedExecutable = opaque { }; pub const BufferType = enum(c.PJRT_Buffer_Type) { - INVALID = c.PJRT_Buffer_Type_INVALID, - PRED = c.PJRT_Buffer_Type_PRED, - S8 = c.PJRT_Buffer_Type_S8, - S16 = c.PJRT_Buffer_Type_S16, - S32 = c.PJRT_Buffer_Type_S32, - S64 = c.PJRT_Buffer_Type_S64, - U8 = c.PJRT_Buffer_Type_U8, - U16 = c.PJRT_Buffer_Type_U16, - U32 = c.PJRT_Buffer_Type_U32, - U64 = c.PJRT_Buffer_Type_U64, - F16 = c.PJRT_Buffer_Type_F16, - F32 = c.PJRT_Buffer_Type_F32, - F64 = c.PJRT_Buffer_Type_F64, - BF16 = c.PJRT_Buffer_Type_BF16, - C64 = c.PJRT_Buffer_Type_C64, - C128 = c.PJRT_Buffer_Type_C128, - F8E5M2 = c.PJRT_Buffer_Type_F8E5M2, - F8E4M3FN = c.PJRT_Buffer_Type_F8E4M3FN, - F8E4M3B11FNUZ = c.PJRT_Buffer_Type_F8E4M3B11FNUZ, - F8E5M2FNUZ = c.PJRT_Buffer_Type_F8E5M2FNUZ, - F8E4M3FNUZ = c.PJRT_Buffer_Type_F8E4M3FNUZ, - S4 = c.PJRT_Buffer_Type_S4, - U4 = c.PJRT_Buffer_Type_U4, + invalid = c.PJRT_Buffer_Type_INVALID, + bool = c.PJRT_Buffer_Type_PRED, + i2 = c.PJRT_Buffer_Type_S2, + i4 = c.PJRT_Buffer_Type_S4, + i8 = c.PJRT_Buffer_Type_S8, + i16 = c.PJRT_Buffer_Type_S16, + i32 = c.PJRT_Buffer_Type_S32, + i64 = c.PJRT_Buffer_Type_S64, + u2 = c.PJRT_Buffer_Type_U2, + u4 = c.PJRT_Buffer_Type_U4, + u8 = c.PJRT_Buffer_Type_U8, + u16 = c.PJRT_Buffer_Type_U16, + u32 = c.PJRT_Buffer_Type_U32, + u64 = c.PJRT_Buffer_Type_U64, + f16 = c.PJRT_Buffer_Type_F16, + f32 = c.PJRT_Buffer_Type_F32, + f64 = c.PJRT_Buffer_Type_F64, + bf16 = c.PJRT_Buffer_Type_BF16, + c64 = c.PJRT_Buffer_Type_C64, + c128 = c.PJRT_Buffer_Type_C128, + f8e5m2 = c.PJRT_Buffer_Type_F8E5M2, + f8e4m3fn = c.PJRT_Buffer_Type_F8E4M3FN, + f8e4m3b11fnuz = c.PJRT_Buffer_Type_F8E4M3B11FNUZ, + f8e5m2fnuz = c.PJRT_Buffer_Type_F8E5M2FNUZ, + f8e4m3fnuz = c.PJRT_Buffer_Type_F8E4M3FNUZ, + f8e4m3 = c.PJRT_Buffer_Type_F8E4M3, + f8e3m4 = c.PJRT_Buffer_Type_F8E3M4, + f8e8m0 = c.PJRT_Buffer_Type_F8E8M0FNU, + f4e2m1 = c.PJRT_Buffer_Type_F4E2M1FN, }; pub const MemoryLayoutType = enum(c.PJRT_Buffer_MemoryLayout_Type) { - Tiled = c.PJRT_Buffer_MemoryLayout_Type_Tiled, - Strides = c.PJRT_Buffer_MemoryLayout_Type_Strides, + tiled = c.PJRT_Buffer_MemoryLayout_Type_Tiled, + strides = c.PJRT_Buffer_MemoryLayout_Type_Strides, }; pub const MemoryLayout = union(MemoryLayoutType) { @@ -575,15 +1084,19 @@ pub const MemoryLayout = union(MemoryLayoutType) { byte_strides: []const i64, }; - Tiled: Tiled, - Strides: Strides, + tiled: Tiled, + strides: Strides, fn toCStruct(self: MemoryLayout) c.PJRT_Buffer_MemoryLayout { - return pjrtStruct(switch (self) { - .Tiled => |v| c.PJRT_Buffer_MemoryLayout{ + return switch (self) { + .tiled => |v| c.PJRT_Buffer_MemoryLayout{ + .struct_size = c.PJRT_Buffer_MemoryLayout_STRUCT_SIZE, + .extension_start = null, .type = c.PJRT_Buffer_MemoryLayout_Type_Tiled, .unnamed_0 = .{ .tiled = c.PJRT_Buffer_MemoryLayout_Tiled{ + .struct_size = c.PJRT_Buffer_MemoryLayout_Tiled_STRUCT_SIZE, + .extension_start = null, .minor_to_major = v.minor_to_major.ptr, .minor_to_major_size = v.minor_to_major.len, .tile_dims = v.tile_dims.ptr, @@ -592,16 +1105,142 @@ pub const MemoryLayout = union(MemoryLayoutType) { }, }, }, - .Strides => |v| c.PJRT_Buffer_MemoryLayout{ + .strides => |v| c.PJRT_Buffer_MemoryLayout{ + .struct_size = c.PJRT_Buffer_MemoryLayout_STRUCT_SIZE, + .extension_start = null, .type = c.PJRT_Buffer_MemoryLayout_Type_Strides, .unnamed_0 = .{ .strides = c.PJRT_Buffer_MemoryLayout_Strides{ + .struct_size = c.PJRT_Buffer_MemoryLayout_Strides_STRUCT_SIZE, + .extension_start = null, .byte_strides = v.byte_strides.ptr, .num_byte_strides = v.byte_strides.len, }, }, }, - }); + }; + } + + fn fromCStruct(self: c.PJRT_Buffer_MemoryLayout) MemoryLayout { + return switch (self.type) { + c.PJRT_Buffer_MemoryLayout_Type_Tiled => b: { + const tiled = self.unnamed_0.tiled; + + const minor_to_major: []const i64 = if (tiled.minor_to_major == null) &.{} else tiled.minor_to_major[0..tiled.minor_to_major_size]; + const tile_dims_sizes: []const usize = if (tiled.tile_dim_sizes == null) &.{} else tiled.tile_dim_sizes[0..tiled.num_tiles]; + + const tile_dims: []const i64 = if (tiled.tile_dims == null) &.{} else td: { + var tile_dim_len: usize = 0; + for (tile_dims_sizes) |size| tile_dim_len += size; + break :td tiled.tile_dims[0..tile_dim_len]; + }; + + break :b .{ + .tiled = .{ + .minor_to_major = minor_to_major, + .tile_dims = tile_dims, + .tile_dims_sizes = tile_dims_sizes, + }, + }; + }, + c.PJRT_Buffer_MemoryLayout_Type_Strides => .{ + .strides = .{ .byte_strides = self.unnamed_0.strides.byte_strides[0..self.unnamed_0.strides.num_byte_strides] }, + }, + else => unreachable, + }; + } +}; + +pub const DefaultMemoryLayout = struct { + pub const MAX_MINOR_TO_MAJOR: usize = 16; + pub const MAX_TILE_DIMS: usize = 128; + pub const MAX_NUM_TILES: usize = 16; + + minor_to_major_storage: [MAX_MINOR_TO_MAJOR]i64 = undefined, + minor_to_major_len: usize = 0, + + tile_dims_storage: [MAX_TILE_DIMS]i64 = undefined, + tile_dims_len: usize = 0, + + tile_dims_sizes_storage: [MAX_NUM_TILES]usize = undefined, + tile_dims_sizes_len: usize = 0, + + pub fn toMemoryLayout(self: *const DefaultMemoryLayout) MemoryLayout { + return .{ + .tiled = .{ + .minor_to_major = self.minor_to_major_storage[0..self.minor_to_major_len], + .tile_dims = self.tile_dims_storage[0..self.tile_dims_len], + .tile_dims_sizes = self.tile_dims_sizes_storage[0..self.tile_dims_sizes_len], + }, + }; + } + + pub fn parseSerialized(serialized: []const u8) !DefaultMemoryLayout { + var parsed: DefaultMemoryLayout = .{}; + try parseSerializedInto(serialized, &parsed); + return parsed; + } + + fn parseSerializedInto(serialized: []const u8, out: *DefaultMemoryLayout) !void { + out.* = .{}; + const raw = std.mem.trim(u8, serialized, " \t\n\r\x00"); + + if (raw.len < 2 or raw[0] != '{' or raw[raw.len - 1] != '}') { + return error.InvalidLayoutString; + } + + const body = raw[1 .. raw.len - 1]; + const first_colon = std.mem.indexOfScalar(u8, body, ':'); + + const minor_to_major_str = std.mem.trim(u8, if (first_colon) |index| body[0..index] else body, " \t\n\r"); + + if (minor_to_major_str.len > 0) { + var mtm_iter = std.mem.splitScalar(u8, minor_to_major_str, ','); + while (mtm_iter.next()) |part_raw| { + if (out.minor_to_major_len >= MAX_MINOR_TO_MAJOR) return error.LayoutTooLarge; + out.minor_to_major_storage[out.minor_to_major_len] = try parseI64Token(part_raw); + out.minor_to_major_len += 1; + } + } + + if (first_colon) |index| { + const attrs = body[index + 1 ..]; + var cursor: usize = 0; + while (cursor < attrs.len) { + const t_pos = std.mem.indexOfScalarPos(u8, attrs, cursor, 'T') orelse break; + cursor = t_pos + 1; + while (cursor < attrs.len and std.ascii.isWhitespace(attrs[cursor])) : (cursor += 1) {} + if (cursor >= attrs.len or attrs[cursor] != '(') continue; + + while (cursor < attrs.len and attrs[cursor] == '(') { + const tuple_start = cursor + 1; + const tuple_end = std.mem.indexOfScalarPos(u8, attrs, tuple_start, ')') orelse return error.InvalidLayoutString; + const tuple = attrs[tuple_start..tuple_end]; + + var dims_in_tile: usize = 0; + var tuple_iter = std.mem.splitScalar(u8, tuple, ','); + while (tuple_iter.next()) |dim_raw| { + if (out.tile_dims_len >= MAX_TILE_DIMS) return error.LayoutTooLarge; + out.tile_dims_storage[out.tile_dims_len] = try parseI64Token(dim_raw); + out.tile_dims_len += 1; + dims_in_tile += 1; + } + if (dims_in_tile == 0) return error.InvalidLayoutString; + + if (out.tile_dims_sizes_len >= MAX_NUM_TILES) return error.LayoutTooLarge; + out.tile_dims_sizes_storage[out.tile_dims_sizes_len] = dims_in_tile; + out.tile_dims_sizes_len += 1; + + cursor = tuple_end + 1; + } + } + } + } + + fn parseI64Token(token_raw: []const u8) !i64 { + const token = std.mem.trim(u8, token_raw, " \t\n\r"); + if (token.len == 0) return error.InvalidLayoutString; + return std.fmt.parseInt(i64, token, 10); } }; @@ -621,7 +1260,7 @@ pub const Buffer = opaque { }) catch unreachable; } - pub fn getDevice(self: *const Buffer, api: *const Api) ApiError!*Device { + pub fn device(self: *const Buffer, api: *const Api) ApiError!*Device { const ret = try api.call(.PJRT_Buffer_Device, .{ .buffer = self.inner(), }); @@ -648,23 +1287,23 @@ pub const Buffer = opaque { return ret.is_on_cpu; } - pub fn toHostBuffer(self: *const Buffer, api: *const Api, dst: []u8) ApiError!*Event { + pub fn toHostBuffer(self: *const Buffer, api: *const Api, dst: []u8) ApiError!?*Event { const ret = try api.call(.PJRT_Buffer_ToHostBuffer, .{ .src = self.inner(), - .dst = @ptrCast(dst.ptr), + .dst = dst.ptr, .dst_size = dst.len, }); - return @ptrCast(ret.event.?); + return @ptrCast(ret.event); } - pub fn getElementType(self: *const Buffer, api: *const Api) BufferType { + pub fn elementType(self: *const Buffer, api: *const Api) BufferType { const ret = api.call(.PJRT_Buffer_ElementType, .{ .buffer = self.inner(), }) catch unreachable; - return @enumFromInt(ret.type); + return @fromBackingInt(@intCast(ret.type)); } - pub fn getDimensions(self: *const Buffer, api: *const Api) []const i64 { + pub fn dimensions(self: *const Buffer, api: *const Api) []const i64 { const ret = api.call(.PJRT_Buffer_Dimensions, .{ .buffer = self.inner(), }) catch unreachable; @@ -674,40 +1313,85 @@ pub const Buffer = opaque { return ret.dims[0..ret.num_dims]; } - pub fn getUnpaddedDimensions(self: *const Buffer, api: *const Api) ApiError![]const i64 { + pub fn unpaddedDimensions(self: *const Buffer, api: *const Api) ApiError![]const i64 { const ret = try api.call(.PJRT_Buffer_UnpaddedDimensions, .{ .buffer = self.inner(), }); - return ret.dims[0..ret.num_dims]; + return ret.unpadded_dims[0..ret.num_dims]; } - pub fn getOnDeviceSizeInBytes(self: *const Buffer, api: *const Api) ApiError!usize { + pub fn onDeviceSizeInBytes(self: *const Buffer, api: *const Api) ApiError!usize { const ret = try api.call(.PJRT_Buffer_OnDeviceSizeInBytes, .{ .buffer = self.inner(), }); - return ret.on_device_size_in_bytes; + return @intCast(ret.on_device_size_in_bytes); } - pub fn copyToDevice(self: *const Buffer, api: *const Api, device: Device) ApiError!Buffer { + pub fn copyToDevice(self: *const Buffer, api: *const Api, dst_device: *Device) ApiError!*Buffer { const ret = try api.call(.PJRT_Buffer_CopyToDevice, .{ .buffer = self.inner(), - .dst_device = device.inner, + .dst_device = dst_device.inner(), }); return @ptrCast(ret.dst_buffer.?); } - pub fn getReadyEvent(self: *const Buffer, api: *const Api) *Event { + pub fn readyEvent(self: *const Buffer, api: *const Api) *Event { const ret = api.call(.PJRT_Buffer_ReadyEvent, .{ .buffer = self.inner(), }) catch unreachable; return @ptrCast(ret.event.?); } - pub fn getOpaqueDeviceMemoryDataPointer(self: *const Buffer, api: *const Api) ApiError!*anyopaque { + pub fn opaqueDeviceMemoryDataPointer(self: *const Buffer, api: *const Api) ApiError!*anyopaque { const ret = try api.call(.PJRT_Buffer_OpaqueDeviceMemoryDataPointer, .{ .buffer = self.inner(), }); - return ret.device_memory_ptr.?; + return ret.device_memory_ptr orelse { + log.err("[PJRT_Buffer_OpaqueDeviceMemoryDataPointer] plugin returned success with null device_memory_ptr", .{}); + return error.Internal; + }; + } + + pub fn copyRawToHost(self: *const Buffer, api: *const Api, dst: []u8, offset: i64) ApiError!?*Event { + const ret = try api.call(.PJRT_Buffer_CopyRawToHost, .{ + .buffer = self.inner(), + .dst = @ptrCast(dst), + .offset = offset, + .transfer_size = @intCast(dst.len), + }); + return @ptrCast(ret.event); + } + + pub fn copyToMemory(self: *const Buffer, api: *const Api, dst_memory: *const Memory) ApiError!*Buffer { + const ret = try api.call(.PJRT_Buffer_CopyToMemory, .{ + .buffer = self.inner(), + .dst_memory = dst_memory.inner(), + }); + return @ptrCast(ret.dst_buffer); + } + + pub fn memory(self: *const Buffer, api: *const Api) *const Memory { + const ret = api.call(.PJRT_Buffer_Memory, .{ + .buffer = self.inner(), + }) catch unreachable; + return @ptrCast(ret.memory); + } + + pub fn increaseExternalReferenceCount(self: *const Buffer, api: *const Api) ApiError!void { + _ = try api.call(.PJRT_Buffer_IncreaseExternalReferenceCount, .{ + .buffer = self.inner(), + }); + } + + pub fn decreaseExternalReferenceCount(self: *const Buffer, api: *const Api) ApiError!void { + _ = try api.call(.PJRT_Buffer_DecreaseExternalReferenceCount, .{ + .buffer = self.inner(), + }); + } + + pub fn memoryLayout(self: *const Buffer, api: *const Api) ApiError!MemoryLayout { + const ret = try api.call(.PJRT_Buffer_GetMemoryLayout, .{ .buffer = self.inner() }); + return .fromCStruct(ret.layout); } }; @@ -728,27 +1412,200 @@ pub const Event = opaque { } pub fn getEventError(self: *const Event, api: *const Api) ?*Error { - var args: Api.CallFnArgType(.PJRT_Event_Error) = .{ .event = self.inner() }; - args = pjrtStruct(args); - const result: ?*c.PJRT_Error = api.inner.PJRT_Event_Error.?(&args); + var args: meta.Struct(c.PJRT_Event_Error_Args) = .{ + .event = self.inner(), + }; + const result: ?*c.PJRT_Error = api.inner.PJRT_Event_Error.?(@ptrCast(&args)); return @ptrCast(result); } - pub fn await_(self: *const Event, api: *const Api) ApiError!void { + pub fn await(self: *Event, api: *const Api, io: std.Io) ApiError!void { + if (self.isReady(api)) { + const err = self.getEventError(api) orelse return; + return interpretPjrtError(api, err, "PJRT_Event_Await", .warn); + } + + const Ctx = struct { + err: ?*Error = null, + event: std.Io.Event = .unset, + io: std.Io, + }; + var ctx: Ctx = .{ .io = io }; + try self.onReady(api, Ctx, struct { + fn call(err: ?*Error, ctx_: *Ctx) void { + ctx_.err = err; + ctx_.event.set(ctx_.io); + } + }.call, &ctx); + ctx.event.waitUncancelable(io); + + if (ctx.err) |err| return interpretPjrtError(api, err, "PJRT_Event_OnReady", .warn); + } + + pub fn awaitRaw(self: *const Event, api: *const Api) ApiError!void { _ = try api.call(.PJRT_Event_Await, .{ .event = self.inner(), }); } - pub fn onReady(self: *Event, api: *const Api, func: *const fn (err: ?*Error, user_arg: ?*anyopaque) callconv(.C) void, user_arg: ?*anyopaque) ApiError!void { + pub fn onReady(self: *Event, api: *const Api, comptime T: type, func: fn (err: ?*Error, user_arg: *T) void, user_arg: *T) ApiError!void { _ = try api.call(.PJRT_Event_OnReady, .{ .event = self.inner(), - .callback = @ptrCast(func), + .callback = @ptrCast(&struct { + fn call(err: ?*Error, user_arg_: ?*anyopaque) callconv(.c) void { + func(err, @ptrCast(@alignCast(user_arg_.?))); + } + }.call), .user_arg = user_arg, }); } }; +pub const Memory = opaque { + pub const Kind = enum { + device, + host_pinned, + host_unpinned, + + pub fn pjrtName(k: Kind) []const u8 { + return switch (k) { + .device => "device", + .host_pinned => "pinned_host", + .host_unpinned => "unpinned_host", + }; + } + }; + + const inner = InnerMixin(c.PJRT_Memory).inner; + + pub fn id(self: *const Memory, api: *const Api) usize { + const ret = api.call(.PJRT_Memory_Id, .{ .memory = self.inner() }) catch unreachable; + return @intCast(ret.id); + } + + pub fn kind(self: *const Memory, api: *const Api) Kind { + const ret = api.call(.PJRT_Memory_Kind, .{ .memory = self.inner() }) catch unreachable; + return switch (ret.kind_size) { + "device".len => .device, + "pinned_host".len => .host_pinned, + "unpinned_host".len => .host_unpinned, + else => @panic("Memory kind not supported"), + }; + } + + pub fn kind_(self: *const Memory, api: *const Api) []const u8 { + const ret = api.call(.PJRT_Memory_Kind, .{ .memory = self.inner() }) catch unreachable; + return ret.kind[0..ret.kind_size]; + } + + pub fn kindId(self: *const Memory, api: *const Api) u32 { + const ret = api.call(.PJRT_Memory_Kind_Id, .{ + .memory = self.inner(), + }) catch unreachable; + return @bitCast(ret.kind_id); + } + + pub fn debugString(self: *const Memory, api: *const Api) []const u8 { + const ret = api.call(.PJRT_Memory_DebugString, .{ + .memory = self.inner(), + }) catch unreachable; + if (ret.debug_string) |debug_string| { + return debug_string[0..ret.debug_string_size]; + } + return &.{}; + } + + pub fn toString(self: *const Memory, api: *const Api) []const u8 { + const ret = api.call(.PJRT_Memory_ToString, .{ + .memory = self.inner(), + }) catch unreachable; + if (ret.to_string) |to_string| { + return to_string[0..ret.to_string_size]; + } + return &.{}; + } + + pub fn addressableByDevices(self: *const Memory, api: *const Api) []const *Device { + const ret = api.call(.PJRT_Memory_AddressableByDevices, .{ + .memory = self.inner(), + }) catch unreachable; + if (ret.devices) |devices| { + return @ptrCast(devices[0..ret.num_devices]); + } + return &.{}; + } +}; + +pub const AsyncHostToDeviceTransferManager = opaque { + const inner = InnerMixin(c.PJRT_AsyncHostToDeviceTransferManager).inner; + + pub fn deinit(self: *AsyncHostToDeviceTransferManager, api: *const Api) void { + _ = api.call(.PJRT_AsyncHostToDeviceTransferManager_Destroy, .{ + .transfer_manager = self.inner(), + }) catch unreachable; + } + + pub fn transferData(self: *AsyncHostToDeviceTransferManager, api: *const Api, buffer_index: usize, data: []const u8, offset: i64, is_last_transfer: bool) ApiError!*Event { + const ret = try api.call(.PJRT_AsyncHostToDeviceTransferManager_TransferData, .{ + .transfer_manager = self.inner(), + .buffer_index = @intCast(buffer_index), + .data = data.ptr, + .offset = offset, + .transfer_size = @intCast(data.len), + .is_last_transfer = is_last_transfer, + }); + return @ptrCast(ret.done_with_h2d_transfer.?); + } + + pub fn retrieveBuffer(self: *AsyncHostToDeviceTransferManager, api: *const Api, buffer_index: usize) ApiError!*Buffer { + const ret = try api.call(.PJRT_AsyncHostToDeviceTransferManager_RetrieveBuffer, .{ + .transfer_manager = self.inner(), + .buffer_index = @intCast(buffer_index), + }); + return @ptrCast(ret.buffer_out.?); + } + + pub fn device(self: *AsyncHostToDeviceTransferManager, api: *const Api) ApiError!*Device { + const ret = try api.call(.PJRT_AsyncHostToDeviceTransferManager_Device, .{ + .transfer_manager = self.inner(), + }); + return @ptrCast(ret.device_out.?); + } + + pub fn bufferCount(self: *AsyncHostToDeviceTransferManager, api: *const Api) ApiError!usize { + const ret = try api.call(.PJRT_AsyncHostToDeviceTransferManager_BufferCount, .{ + .transfer_manager = self.inner(), + }); + return ret.buffer_count; + } + + pub fn bufferSize(self: *AsyncHostToDeviceTransferManager, api: *const Api, buffer_index: usize) ApiError!usize { + const ret = try api.call(.PJRT_AsyncHostToDeviceTransferManager_BufferSize, .{ + .transfer_manager = self.inner(), + .buffer_index = @intCast(buffer_index), + }); + return ret.buffer_size; + } + + pub fn setBufferError(self: *AsyncHostToDeviceTransferManager, api: *const Api, buffer_index: usize, error_code: c.PJRT_Error_Code, error_message: []const u8) ApiError!void { + _ = try api.call(.PJRT_AsyncHostToDeviceTransferManager_SetBufferError, .{ + .transfer_manager = self.inner(), + .buffer_index = @intCast(buffer_index), + .error_code = error_code, + .error_message = error_message.ptr, + .error_message_size = error_message.len, + }); + } + + pub fn addMetadata(self: *AsyncHostToDeviceTransferManager, api: *const Api, transfer_metadata: []const NamedValue) ApiError!void { + _ = try api.call(.PJRT_AsyncHostToDeviceTransferManager_AddMetadata, .{ + .transfer_manager = self.inner(), + .transfer_metadata = @ptrCast(transfer_metadata), + .num_metadata = transfer_metadata.len, + }); + } +}; + pub const NamedValue = extern struct { comptime { std.debug.assert(@sizeOf(NamedValue) == @sizeOf(c.PJRT_NamedValue)); @@ -764,92 +1621,244 @@ pub const NamedValue = extern struct { bool = c.PJRT_NamedValue_kBool, }; + pub const Value = union(Kind) { + string: []const u8, + int64: i64, + int64list: []const i64, + float: f32, + bool: bool, + }; + pub fn kind(self: NamedValue) Kind { - return @enumFromInt(self.inner.type); + return @fromBackingInt(@intCast(self.inner.type)); } pub fn name(self: NamedValue) []const u8 { return self.inner.name[0..self.inner.name_size]; } - pub fn from(name_: []const u8, value: anytype) NamedValue { - return switch (@TypeOf(value)) { - []u8, []const u8 => fromString(name_, value), - i64 => fromInt64(name_, value), - []i64, []const i64 => fromInt64List(name_, value), - f32 => fromFloat(name_, value), - bool => fromBool(name_, value), - else => unreachable, + pub fn value(self: NamedValue) Value { + return switch (self.kind()) { + .string => .{ .string = self.inner.unnamed_0.string_value[0..self.inner.value_size] }, + .int64 => .{ .int64 = self.inner.unnamed_0.int64_value }, + .int64list => .{ .int64list = self.inner.unnamed_0.int64_array_value[0..self.inner.value_size] }, + .float => .{ .float = self.inner.unnamed_0.float_value }, + .bool => .{ .bool = self.inner.unnamed_0.bool_value }, }; } - pub fn fromString(name_: []const u8, value: []const u8) NamedValue { - return .{ .inner = pjrtStruct(c.PJRT_NamedValue{ - .name = @ptrCast(@constCast(name_.ptr)), - .name_size = name_.len, - .type = c.PJRT_NamedValue_kString, - .unnamed_0 = .{ .string_value = @ptrCast(@constCast(value.ptr)) }, - .value_size = value.len, - }) }; + pub fn init(comptime kind_: Kind, name_: []const u8, value_: @FieldType(Value, @tagName(kind_))) NamedValue { + return .{ + .inner = .{ + .struct_size = c.PJRT_NamedValue_STRUCT_SIZE, + .extension_start = null, + .name = @ptrCast(name_), + .name_size = name_.len, + .type = @backingInt(kind_), + .unnamed_0 = switch (kind_) { + .string => .{ .string_value = @ptrCast(@constCast(value_)) }, + .int64 => .{ .int64_value = value_ }, + .int64list => .{ .int64_array_value = @ptrCast(@constCast(value_)) }, + .float => .{ .float_value = value_ }, + .bool => .{ .bool_value = value_ }, + }, + .value_size = switch (kind_) { + .string, .int64list => value_.len, + inline else => 1, + }, + }, + }; } - pub fn fromInt64(name_: []const u8, value: i64) NamedValue { - return .{ .inner = pjrtStruct(c.PJRT_NamedValue{ - .name = @ptrCast(@constCast(name_.ptr)), - .name_size = name_.len, - .type = c.PJRT_NamedValue_kInt64, - .unnamed_0 = .{ .int64_value = value }, - .value_size = 1, - }) }; - } - - pub fn fromInt64List(name_: []const u8, value: []const i64) NamedValue { - return .{ .inner = pjrtStruct(c.PJRT_NamedValue{ - .name = @ptrCast(@constCast(name_.ptr)), - .name_size = name_.len, - .type = c.PJRT_NamedValue_kInt64List, - .unnamed_0 = .{ .int64_array_value = @ptrCast(@constCast(value.ptr)) }, - .value_size = value.len, - }) }; - } - - pub fn fromFloat(name_: []const u8, value: f32) NamedValue { - return .{ .inner = pjrtStruct(c.PJRT_NamedValue{ - .name = @ptrCast(@constCast(name_.ptr)), - .name_size = name_.len, - .type = c.PJRT_NamedValue_kFloat, - .unnamed_0 = .{ .float_value = value }, - .value_size = 1, - }) }; - } - - pub fn fromBool(name_: []const u8, value: bool) NamedValue { - return .{ .inner = pjrtStruct(c.PJRT_NamedValue{ - .name = @ptrCast(@constCast(name_.ptr)), - .name_size = name_.len, - .type = c.PJRT_NamedValue_kBool, - .unnamed_0 = .{ .bool_value = value }, - .value_size = 1, - }) }; - } - - pub fn format( - self: NamedValue, - comptime fmt: []const u8, - options: std.fmt.FormatOptions, - writer: anytype, - ) !void { - _ = fmt; - _ = options; - try writer.print("{s}{{ .name = {s},", .{ @typeName(NamedValue), self.inner.name[0..self.inner.name_size] }); - const u = self.inner.unnamed_0; - switch (self.kind()) { - .string => try writer.print(" .string = {s} ", .{u.string_value[0..self.inner.value_size]}), - .int64 => try writer.print(" .int64 = {d} ", .{u.int64_value}), - .int64list => try writer.print(" .int64list = {d} ", .{u.int64_array_value[0..self.inner.value_size]}), - .float => try writer.print(" .float = {d} ", .{u.float_value}), - .bool => try writer.print(" .bool = {} ", .{u.bool_value}), + pub fn format(self: NamedValue, writer: *std.Io.Writer) !void { + try writer.print("{s}{{ .name = {s},", .{ @typeName(NamedValue), self.name() }); + switch (self.value()) { + .string => |v| try writer.print(" .string = \"{s}\" ", .{v}), + .int64 => |v| try writer.print(" .int64 = {d} ", .{v}), + .int64list => |v| try writer.print(" .int64list = {any} ", .{v}), + .float => |v| try writer.print(" .float = {d} ", .{v}), + .bool => |v| try writer.print(" .bool = {} ", .{v}), } try writer.writeAll("}"); } }; + +pub const Ffi = extern struct { + inner: *const c.PJRT_FFI, + + pub const UserData = extern struct { + type_id: i64, + user_data: *anyopaque, + + fn toCStruct(self: UserData) c.PJRT_FFI_UserData { + return .{ + .type_id = self.type_id, + .data = self.user_data, + }; + } + }; + + pub const TypeInfo = struct { + deleter: ?*const fn (*anyopaque) callconv(.c) void = null, + serialize: ?*const fn () callconv(.c) void = null, + deserialize: ?*const fn () callconv(.c) void = null, + + pub fn toCStruct(self: TypeInfo) c.PJRT_FFI_Type_Info { + return .{ + .deleter = @ptrCast(self.deleter), + .serialize = @ptrCast(self.serialize), + .deserialize = @ptrCast(self.deserialize), + }; + } + }; + + // todo : support all missing handlers available in GPU plugin extension: handler_instantiate, handler_prepare, handler_initialize + // introduced by https://github.com/openxla/xla/commit/ef85a7bcc308313492ebc50295a8a08b4e51b8f5 + pub fn register( + self: *const Ffi, + api: *const Api, + target_name: []const u8, + platform_name: []const u8, + func: *const ffi.Handler, + traits: ffi.HandlerTraits, + ) ApiError!void { + var ret: meta.Struct(c.PJRT_FFI_Register_Handler_Args) = .{ + .target_name = target_name.ptr, + .target_name_size = target_name.len, + .handler = @as(?*anyopaque, @ptrCast(@constCast(func))), + .platform_name = platform_name.ptr, + .platform_name_size = platform_name.len, + .traits = @bitCast(traits), + }; + const result = self.inner.register_handler.?(@ptrCast(&ret)); + if (result) |pjrt_c_error| { + const pjrt_error: *Error = @ptrCast(pjrt_c_error); + log.err("registerFfi error: {s}", .{pjrt_error.getMessage(api)}); + return pjrt_error.getCode(api).toApiError(); + } + } + + pub fn registerTypeId(self: *const Ffi, api: *const Api, type_name: []const u8, type_info: ?*const c.PJRT_FFI_Type_Info) ApiError!ffi.TypeId { + var ret: meta.Struct(c.PJRT_FFI_Type_Register_Args) = .{ + .type_name = type_name.ptr, + .type_name_size = type_name.len, + .type_id = 0, // let the plugin assign a unique type ID + .type_info = @ptrCast(@constCast(type_info)), + }; + const result = self.inner.type_register.?(&ret); + if (result) |pjrt_c_error| { + const pjrt_error: *Error = @ptrCast(pjrt_c_error); + return pjrt_error.getCode(api).toApiError(); + } + + return .{ .type_id = ret.type_id }; + } + + pub fn addUserData(self: *const Ffi, api: *const Api, context: *ExecuteContext, user_data: UserData) ApiError!void { + var ret: meta.Struct(c.PJRT_FFI_UserData_Add_Args) = .{ + .context = @ptrCast(context), + .user_data = user_data.toCStruct(), + }; + const result = self.inner.user_data_add.?(&ret); + if (result) |pjrt_c_error| { + const pjrt_error: *Error = @ptrCast(pjrt_c_error); + log.err("addUserData error: {s}", .{pjrt_error.getMessage(api)}); + return pjrt_error.getCode(api).toApiError(); + } + } +}; + +pub const Profiler = struct { + pjrt_api: *const Api, + api: *const c.PLUGIN_Profiler_Api, + inner: *c.PLUGIN_Profiler, + + pub fn init(api_: *const Api, prof_ext: *const c.PJRT_Profiler_Extension, options_pb: []const u8) ApiError!Profiler { + var args: c.PLUGIN_Profiler_Create_Args = .{ + .struct_size = meta.structSize(c.PLUGIN_Profiler_Create_Args), + .options = options_pb.ptr, + .options_size = options_pb.len, + .profiler = null, + }; + + const profiler_api: *const c.PLUGIN_Profiler_Api = @ptrCast(prof_ext.profiler_api); + if (profiler_api.create.?(&args)) |err| { + const pjrt_err: *Error = @ptrCast(err); + return pjrt_err.getCode(api_).toApiError(); + } + + return .{ + .pjrt_api = api_, + .api = profiler_api, + .inner = args.profiler.?, + }; + } + + pub fn start(self: *Profiler) ApiError!void { + var args: c.PLUGIN_Profiler_Start_Args = .{ + .struct_size = meta.structSize(c.PLUGIN_Profiler_Start_Args), + .profiler = self.inner, + }; + + if (self.api.start.?(&args)) |err| { + const pjrt_err: *Error = @ptrCast(err); + return pjrt_err.getCode(self.pjrt_api).toApiError(); + } + } + + pub fn stop(self: *Profiler) ApiError!void { + var args: c.PLUGIN_Profiler_Stop_Args = .{ + .struct_size = meta.structSize(c.PLUGIN_Profiler_Stop_Args), + .profiler = self.inner, + }; + + if (self.api.stop.?(&args)) |err| { + const pjrt_err: *Error = @ptrCast(err); + return pjrt_err.getCode(self.pjrt_api).toApiError(); + } + } + + pub fn collectData(self: *Profiler, allocator: std.mem.Allocator) ![]u8 { + var args: c.PLUGIN_Profiler_CollectData_Args = .{ + .struct_size = meta.structSize(c.PLUGIN_Profiler_CollectData_Args), + .profiler = self.inner, + .buffer = null, + .buffer_size_in_bytes = 0, + }; + + if (self.api.collect_data.?(&args)) |err| { + const pjrt_err: *Error = @ptrCast(err); + return pjrt_err.getCode(self.pjrt_api).toApiError(); + } + + if (args.buffer != null) { + var data = args.buffer[0..args.buffer_size_in_bytes]; + if (data.len > 0 and data[data.len - 1] == 0) { + data = data[0 .. data.len - 1]; + } + return allocator.dupe(u8, data); + } + + if (args.buffer_size_in_bytes == 0) return &[_]u8{}; + + const buffer = try allocator.alloc(u8, args.buffer_size_in_bytes); + errdefer allocator.free(buffer); + + args.buffer = buffer.ptr; + if (self.api.collect_data.?(&args)) |err| { + const pjrt_err: *Error = @ptrCast(err); + return pjrt_err.getCode(self.pjrt_api).toApiError(); + } + + // Trim trailing null byte logic + var data = buffer[0..args.buffer_size_in_bytes]; + if (data.len > 0 and data[data.len - 1] == 0) { + data = data[0 .. data.len - 1]; + } + + const pb = try allocator.dupe(u8, data); + allocator.free(buffer); + + return pb; + } +}; diff --git a/pjrt/profiler.zig b/pjrt/profiler.zig deleted file mode 100644 index e148bb53..00000000 --- a/pjrt/profiler.zig +++ /dev/null @@ -1,147 +0,0 @@ -const std = @import("std"); -const c = @import("c"); -const tsl_proto = @import("//tsl:profiler_options_proto"); - -const log = std.log.scoped(.zml_profiler); - -/// Pjrt Profiler extension -pub const Profiler = struct { - api: ?c.PLUGIN_Profiler_Api, - inner: *c.PLUGIN_Profiler, - last_error: ?*Error = null, - status: Status = .ready, - - pub const Status = enum { ready, started, stopped, done }; - pub const Error = c.PLUGIN_Profiler_Error; - pub const Options = tsl_proto.ProfileOptions; - - pub fn init(api: ?c.PLUGIN_Profiler_Api, options: Options) Profiler { - if (api == null) { - return .{ .api = null, .inner = undefined }; - } - - var buffer: [std.fs.max_path_bytes + @sizeOf(Options) * 4]u8 = undefined; - var fba = std.heap.FixedBufferAllocator.init(&buffer); - const byte_options = options.encode(fba.allocator()) catch unreachable; - var res: Profiler = .{ .api = api, .inner = undefined }; - var args: c.PLUGIN_Profiler_Create_Args = .{ - .options = byte_options.ptr, - .options_size = byte_options.len, - .profiler = undefined, // out - }; - res.check(api.?.create.?(&args)) catch unreachable; - - res.inner = args.profiler.?; - return res; - } - - fn transition(self: *Profiler, fn_name: []const u8, expected: Status, next: Status) void { - if (self.status == expected) { - self.status = next; - return; - } - std.debug.panic("Profiler can't `{s}()`. Current status: {}, expected: {}", .{ fn_name, self.status, expected }); - } - - pub fn start(self: *Profiler) void { - self.transition("start", .ready, .started); - if (self.api == null) return; - var args: c.PLUGIN_Profiler_Start_Args = .{ .profiler = self.inner }; - self.check(self.api.?.start.?(&args)) catch unreachable; - } - - pub fn stop(self: *Profiler) void { - self.transition("stop", .started, .stopped); - if (self.api == null) return; - - var args: c.PLUGIN_Profiler_Stop_Args = .{ .profiler = self.inner }; - self.check(self.api.?.stop.?(&args)) catch unreachable; - } - - pub fn collectData(self: *Profiler, allocator: std.mem.Allocator) !ProfilingData { - self.transition("collect_data", .stopped, .done); - if (self.api == null) return .{ .external = &.{} }; - - var args: c.PLUGIN_Profiler_CollectData_Args = .{ - .struct_size = c.PLUGIN_Profiler_CollectData_Args_STRUCT_SIZE, - .profiler = self.inner, - .buffer = null, - .buffer_size_in_bytes = 0, - }; - try self.check(self.api.?.collect_data.?(&args)); - std.debug.assert(args.buffer_size_in_bytes > 0); - const buffer: ProfilingData = if (args.buffer == null) blk: { - std.log.debug("Plugin profiler wants us to allocate {d} bytes for profile data", .{args.buffer_size_in_bytes}); - // The plugin want us to allocate memory for it: - const buffer = try allocator.alloc(u8, args.buffer_size_in_bytes); - args.buffer = buffer.ptr; - try self.check(self.api.?.collect_data.?(&args)); - break :blk .{ .owned = buffer }; - } else blk: { - std.log.debug("Plugin profiler has {d} bytes of profile data", .{args.buffer_size_in_bytes}); - // Drop sentinel. The profiler plugin returns a null terminated string. - // But this is creating issues if we save the sentinel on disk, - // because it will trip up protobuf readers. - var data = args.buffer[0..args.buffer_size_in_bytes]; - data = if (data.len > 0 and data[data.len - 1] == 0) data[0 .. data.len - 1] else data; - break :blk .{ .external = data }; - }; - - // printDataAsXSpace(allocator, buffer.items()); - return buffer; - } - - pub fn dumpDataTo( - self: *Profiler, - allocator: std.mem.Allocator, - dir: std.fs.Dir, - file_name: []const u8, - ) !void { - const profile_data = try self.collectData(allocator); - defer profile_data.free(allocator); - - if (profile_data.items().len == 0) return; - - const file = try dir.createFile(file_name, .{ .truncate = true }); - defer file.close(); - log.info("Writing profiling data to {s} ({} bytes)", .{ file_name, profile_data.items().len }); - return try file.writeAll(profile_data.items()); - } - - fn check(self: *Profiler, c_error: ?*Error) !void { - if (c_error) |err| { - self.last_error = err; - return error.PjrtProfilerError; - } - } - - pub fn deinit(self: Profiler) void { - switch (self.status) { - .started => log.warn("Profiler was never stopped", .{}), - .stopped => log.warn("Profiler data was never collected", .{}), - else => {}, - } - if (self.api == null) return; - - var args: c.PLUGIN_Profiler_Destroy_Args = .{ .profiler = self.inner }; - _ = self.api.?.destroy.?(&args); - } -}; - -const ProfilingData = union(enum) { - owned: []const u8, - external: []const u8, - - pub fn items(self: ProfilingData) []const u8 { - return switch (self) { - inline else => |x| x, - }; - } - - pub fn free(self: ProfilingData, allocator: std.mem.Allocator) void { - switch (self) { - .owned => |data| allocator.free(data), - .external => {}, - } - } -}; diff --git a/platform_mappings b/platform_mappings deleted file mode 120000 index 4a67a02f..00000000 --- a/platform_mappings +++ /dev/null @@ -1 +0,0 @@ -examples/platform_mappings \ No newline at end of file diff --git a/platforms/BUILD.bazel b/platforms/BUILD.bazel index 05a143a2..5ee2f176 100644 --- a/platforms/BUILD.bazel +++ b/platforms/BUILD.bazel @@ -1,8 +1,112 @@ +load("@bazel_skylib//rules:common_settings.bzl", "bool_flag") +load("@rules_zig//zig:defs.bzl", "zig_library") +load("//bazel:zig_options.bzl", "zig_option", "zig_options") + +PLATFORMS = { + "cpu": True, + "cuda": False, + "rocm": False, + "tpu": False, + "neuron": False, + "oneapi": False, + "metal": False, + "furiosa": False, +} + +[ + bool_flag( + name = runtime, + build_setting_default = default, + ) + for runtime, default in PLATFORMS.items() +] + +[ + zig_option( + name = "{}_enabled".format(runtime), + bool = select({ + "//platforms:{}.enabled".format(runtime): True, + "//conditions:default": False, + }), + visibility = ["//platforms:__pkg__"], + ) + for runtime in PLATFORMS +] + +zig_options( + name = "options", + import_name = "platforms/options", + settings = [":{}_enabled".format(runtime) for runtime in PLATFORMS], + visibility = ["//platforms:__subpackages__"], +) + +# x86_64 and aarch64 +[ + config_setting( + name = "{}.enabled".format(runtime), + flag_values = {":{}".format(runtime): "True"}, + visibility = ["//visibility:public"], + ) + for runtime in [ + "cpu", + "cuda", + ] +] + +# x86_64 only +[ + config_setting( + name = "{}.enabled".format(runtime), + constraint_values = [ + "@platforms//cpu:x86_64", + "@platforms//os:linux", + ], + flag_values = {":{}".format(runtime): "True"}, + visibility = ["//visibility:public"], + ) + for runtime in [ + "rocm", + "tpu", + "neuron", + "oneapi", + "furiosa", + ] +] + +config_setting( + name = "metal.enabled", + constraint_values = [ + "@platforms//cpu:aarch64", + "@platforms//os:macos", + ], + flag_values = {":metal": "True"}, + visibility = ["//visibility:public"], +) + +zig_library( + name = "platforms", + main = "platforms.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + ":options", + "//pjrt", + ] + [ + "//platforms/{}".format(runtime) + for runtime in PLATFORMS.keys() + ], +) + platform( name = "linux_amd64", constraint_values = [ "@platforms//cpu:x86_64", "@platforms//os:linux", + "@llvm//constraints/libc:gnu.2.28", + ], + flags = [ + "--force_pic=true", # required for protobuf upb on Linux + "--cpu=x86_64", ], visibility = ["//visibility:public"], ) @@ -12,6 +116,20 @@ platform( constraint_values = [ "@platforms//cpu:aarch64", "@platforms//os:linux", + "@llvm//constraints/libc:gnu.2.28", + ], + flags = [ + "--force_pic=true", # required for protobuf upb on Linux + "--cpu=aarch64", + ], + visibility = ["//visibility:public"], +) + +platform( + name = "macos_amd64", + constraint_values = [ + "@platforms//cpu:x86_64", + "@platforms//os:macos", ], visibility = ["//visibility:public"], ) @@ -24,3 +142,26 @@ platform( ], visibility = ["//visibility:public"], ) + +constraint_setting(name = "execution_location") + +constraint_value( + name = "local_execution", + constraint_setting = ":execution_location", + visibility = ["//visibility:public"], +) + +# Special constrained host platform that toolchain can use as exec_compatible_with +# if they want to only be used when executing on the local machine. +# +# This is currently used by Zig toolchains so that they are only resolved for +# the host platform and thus can be used for no-remote-exec actions. +platform( + name = "local_execution_platform", + constraint_values = [":local_execution"], + exec_properties = { + "no-remote-exec": "1", + }, + parents = ["@platforms//host"], + visibility = ["//visibility:public"], +) diff --git a/platforms/cpu/BUILD.bazel b/platforms/cpu/BUILD.bazel new file mode 100644 index 00000000..418c7837 --- /dev/null +++ b/platforms/cpu/BUILD.bazel @@ -0,0 +1,60 @@ +load("@bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") +load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library") + +config_setting( + name = "darwin_arm64", + constraint_values = [ + "@platforms//os:macos", + "@platforms//cpu:aarch64", + ], +) + +config_setting( + name = "darwin_amd64", + constraint_values = [ + "@platforms//os:macos", + "@platforms//cpu:x86_64", + ], +) + +copy_to_directory( + name = "sandbox", + srcs = select({ + ":darwin_arm64": ["@libzml_cpu_darwin_arm64//:libzml_cpu"], + ":darwin_amd64": ["@libzml_cpu_darwin_amd64//:libzml_cpu"], + "@platforms//os:linux": ["@libzml_cpu_linux_amd64//:libzml_cpu"], + }), + out = "sandbox", + include_external_repositories = ["**"], + replace_prefixes = { + "libzml_cpu_so": "lib", + "usr/lib/x86_64-linux-gnu": "lib", + }, +) + +cc_library( + name = "libzml_cpu", + data = [":sandbox"], + tags = ["manual"], +) + +zig_library( + name = "cpu", + import_name = "platforms/cpu", + main = "cpu.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//pjrt", + "//platforms:options", + ] + select({ + "//platforms:cpu.enabled": [ + ":libzml_cpu", + "//bazel", + "//stdx", + "@rules_zig//zig/runfiles", + ], + "//conditions:default": [], + }), +) diff --git a/platforms/cpu/cpu.bzl b/platforms/cpu/cpu.bzl new file mode 100644 index 00000000..08817afe --- /dev/null +++ b/platforms/cpu/cpu.bzl @@ -0,0 +1,75 @@ +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("//bazel:http_deb_archive.bzl", "http_deb_archive") +load("//platforms:packages.bzl", "packages") + +_BUILD_FILE_DEFAULT_VISIBILITY = """\ +package(default_visibility = ["//visibility:public"]) +""" + +_BUILD_LINUX = "\n".join([ + packages.load_("@zml//bazel:patchelf.bzl", "patchelf"), + packages.patchelf( + name = "libzml_cpu_so", + src = "lib/libzml_cpu.so", + set_rpath = "$ORIGIN", + ), + packages.filegroup( + name = "libzml_cpu", + srcs = [":libzml_cpu_so", "@llvm-libunwind1//:libunwind"], + visibility = ["@zml//platforms/cpu:__subpackages__"], + ), +]) + +_BUILD_DARWIN = packages.filegroup( + name = "libzml_cpu", + srcs = ["lib/libzml_cpu.dylib"], + visibility = ["@zml//platforms/cpu:__subpackages__"], +) + +def _cpu_plugin_impl(mctx): + loaded_packages = packages.read(mctx, ["@zml//platforms/cpu:packages.lock.json"]) + pkg = loaded_packages["llvm-libunwind1"]["amd64"] + http_deb_archive( + name = "llvm-libunwind1", + urls = pkg["urls"], + sha256 = pkg["sha256"], + build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + packages.filegroup( + name = "libunwind", + srcs = ["usr/lib/x86_64-linux-gnu/libunwind.so.1"], + ), + ) + + http_archive( + name = "libzml_cpu_linux_amd64", + build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_LINUX, + sha256 = "3d6a6d1b047a0943b50fd9e63e5f6dff6b7cf184b186c0cc1052de40f0dbec92", + url = "https://mirror.zml.ai/plugins/202611101651.363.1.d46c7471c419/zml-cpu-linux-amd64.tar.zst", + ) + + http_archive( + name = "libzml_cpu_darwin_amd64", + build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, + sha256 = "d6b4b15b13a2d1a3e0b84f143f568cd572bef5db91784f9f7a826173ed626c7e", + url = "https://mirror.zml.ai/plugins/202611101651.363.1.d46c7471c419/zml-cpu-darwin-amd64.tar.zst", + ) + + http_archive( + name = "libzml_cpu_darwin_arm64", + build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, + sha256 = "c06babc001edeab313bc63daab3aa3d40e087266a3a29634d182628fb65fad14", + url = "https://mirror.zml.ai/plugins/202611101651.363.1.d46c7471c419/zml-cpu-darwin-arm64.tar.zst", + ) + + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = [ + "libzml_cpu_linux_amd64", + "libzml_cpu_darwin_amd64", + "libzml_cpu_darwin_arm64", + ], + root_module_direct_dev_deps = [], + ) + +cpu_plugin = module_extension( + implementation = _cpu_plugin_impl, +) diff --git a/platforms/cpu/cpu.zig b/platforms/cpu/cpu.zig new file mode 100644 index 00000000..523cd119 --- /dev/null +++ b/platforms/cpu/cpu.zig @@ -0,0 +1,43 @@ +const std = @import("std"); +const builtin = @import("builtin"); + +const bazel = @import("bazel"); +const bazel_builtin = @import("bazel_builtin"); +const pjrt = @import("pjrt"); +const platforms_options = @import("platforms/options"); +const runfiles = @import("runfiles"); +const stdx = @import("stdx"); + +const log = std.log.scoped(.@"zml/platforms/cpu"); + +pub fn isEnabled() bool { + return platforms_options.cpu_enabled; +} + +pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { + _ = io; + _ = allocator; + if (comptime !isEnabled()) { + return error.Unavailable; + } + + const r = try bazel.runfiles(bazel_builtin.current_repository); + + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const sandbox_path = try r.rlocation("zml/platforms/cpu/sandbox", &path_buf) orelse { + log.err("Failed to find sandbox path for CPU runtime", .{}); + return error.FileNotFound; + }; + + return blk: { + const ext = switch (builtin.os.tag) { + .windows => ".dll", + .macos, .ios, .watchos => ".dylib", + else => ".so", + }; + + var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libzml_cpu" ++ ext }); + break :blk .loadFrom(path); + }; +} diff --git a/platforms/cpu/packages.lock.json b/platforms/cpu/packages.lock.json new file mode 100644 index 00000000..7e9963e3 --- /dev/null +++ b/platforms/cpu/packages.lock.json @@ -0,0 +1,65 @@ +{ + "packages": [ + { + "arch": "amd64", + "dependencies": [ + { + "key": "libc6_2.36-9-p-deb12u10_amd64", + "name": "libc6", + "version": "2.36-9+deb12u10" + }, + { + "key": "libgcc-s1_12.2.0-14-p-deb12u1_amd64", + "name": "libgcc-s1", + "version": "12.2.0-14+deb12u1" + }, + { + "key": "gcc-12-base_12.2.0-14-p-deb12u1_amd64", + "name": "gcc-12-base", + "version": "12.2.0-14+deb12u1" + } + ], + "key": "llvm-libunwind1_1-22.1.8_-p--p-20260714014842-p-ca7933e47d3a-1_exp1_20260714134950.86_amd64", + "name": "llvm-libunwind1", + "sha256": "04cb76d70da569e5bcf2395f28dd7e9b9e9f0bf4521cf3b008b0f62ae9d2b712", + "urls": [ + "https://apt.llvm.org/bookworm/pool/main/l/llvm-toolchain-22/llvm-libunwind1_22.1.8~++20260714014842+ca7933e47d3a-1~exp1~20260714134950.86_amd64.deb" + ], + "version": "1:22.1.8~++20260714014842+ca7933e47d3a-1~exp1~20260714134950.86" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libc6_2.36-9-p-deb12u10_amd64", + "name": "libc6", + "sha256": "5dc83256f10ca4d0f2a53dd6583ffd0d0e319af30074ea6c82fb0ae77bd16365", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z/pool/main/g/glibc/libc6_2.36-9+deb12u10_amd64.deb" + ], + "version": "2.36-9+deb12u10" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libgcc-s1_12.2.0-14-p-deb12u1_amd64", + "name": "libgcc-s1", + "sha256": "3016e62cb4b7cd8038822870601f5ed131befe942774d0f745622cc77d8a88f7", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z/pool/main/g/gcc-12/libgcc-s1_12.2.0-14+deb12u1_amd64.deb" + ], + "version": "12.2.0-14+deb12u1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "gcc-12-base_12.2.0-14-p-deb12u1_amd64", + "name": "gcc-12-base", + "sha256": "1896a2aacf4ad681ff5eacc24a5b0ca4d5d9c9b9c9e4b6de5197bc1e116ea619", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z/pool/main/g/gcc-12/gcc-12-base_12.2.0-14+deb12u1_amd64.deb" + ], + "version": "12.2.0-14+deb12u1" + } + ], + "version": 1 +} \ No newline at end of file diff --git a/platforms/cpu/packages.yaml b/platforms/cpu/packages.yaml new file mode 100644 index 00000000..d6f97d05 --- /dev/null +++ b/platforms/cpu/packages.yaml @@ -0,0 +1,16 @@ +# +# bazel run @apt_cpu//:lock +# +version: 1 + +sources: + - channel: bookworm main + url: https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z + - channel: llvm-toolchain-bookworm-22 main + url: https://apt.llvm.org/bookworm + +archs: + - "amd64" + +packages: + - "llvm-libunwind1" # LLVM 22 runtime from llvm-toolchain-bookworm-22. diff --git a/platforms/cuda/BUILD.bazel b/platforms/cuda/BUILD.bazel new file mode 100644 index 00000000..7f7aed55 --- /dev/null +++ b/platforms/cuda/BUILD.bazel @@ -0,0 +1,66 @@ +load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library", "zig_shared_library") + +zig_binary( + name = "compat_probe", + main = "compat_probe.zig", + visibility = ["//visibility:public"], +) + +zig_shared_library( + name = "zmlxcuda", + main = "zmlxcuda.zig", + shared_lib_name = "libzmlxcuda.so.0", + visibility = ["//visibility:public"], + # Use Clang's compiler-rt, but disable stack checking + # to avoid requiring on the _zig_probe_stack symbol. + zigopts = ["-fno-stack-check"], + deps = ["//stdx"], +) + +cc_library( + name = "libzml_cuda_repo", + target_compatible_with = ["@platforms//os:linux"], + deps = select({ + "@llvm//platforms/config:linux_x86_64": ["@libzml_cuda_linux_amd64//:libzml_cuda"], + "@llvm//platforms/config:linux_aarch64": ["@libzml_cuda_linux_arm64//:libzml_cuda"], + }), +) + +cc_library( + name = "cuda_nvtx_headers", + target_compatible_with = ["@platforms//os:linux"], + visibility = ["//visibility:public"], + deps = select({ + "@llvm//platforms/config:linux_x86_64": ["@cuda_nvtx_linux_x86_64//:headers"], + "@llvm//platforms/config:linux_aarch64": ["@cuda_nvtx_linux_sbsa//:headers"], + }), +) + +cc_library( + name = "libzml_cuda", + hdrs = ["libzml_cuda.h"], + tags = ["manual"], + deps = [":libzml_cuda_repo"], +) + +zig_library( + name = "cuda", + srcs = ["compat_probe.zig"], + import_name = "platforms/cuda", + main = "cuda.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//pjrt", + "//platforms:options", + ] + select({ + "//platforms:cuda.enabled": [ + ":libzml_cuda", + "//bazel", + "//stdx", + "@rules_zig//zig/runfiles", + ], + "//conditions:default": [], + }), +) diff --git a/platforms/cuda/compat_probe.zig b/platforms/cuda/compat_probe.zig new file mode 100644 index 00000000..3f890c6a --- /dev/null +++ b/platforms/cuda/compat_probe.zig @@ -0,0 +1,39 @@ +const std = @import("std"); + +pub const ExitCode = enum(u8) { + Success = 0, + CompatNotSupportedOnDevice = 1, + SystemDriverMismatch = 2, + UnexpectedError = 3, +}; + +const CUDA_SUCCESS = 0; +const CUDA_ERROR_SYSTEM_DRIVER_MISMATCH = 803; +const CUDA_ERROR_COMPAT_NOT_SUPPORTED_ON_DEVICE = 804; + +pub fn main(init: std.process.Init) !u8 { + const arena = init.arena; + const io = init.io; + + const self_exe_dir = try std.process.executableDirPathAlloc(io, arena.allocator()); + const libcuda_path = try std.Io.Dir.path.joinZ(arena.allocator(), &.{ self_exe_dir, "..", "lib", "compat", "libcuda.so.1" }); + + var libcuda = try std.DynLib.open(libcuda_path); + const cuInit = libcuda.lookup(*const fn (c_uint) callconv(.c) c_int, "cuInit").?; + const cuGetErrorString = libcuda.lookup(*const fn (c_int, *[*c]const u8) callconv(.c) c_int, "cuGetErrorString").?; + + return @backingInt(switch (cuInit(0)) { + CUDA_SUCCESS => ExitCode.Success, + CUDA_ERROR_SYSTEM_DRIVER_MISMATCH => ExitCode.SystemDriverMismatch, + CUDA_ERROR_COMPAT_NOT_SUPPORTED_ON_DEVICE => ExitCode.CompatNotSupportedOnDevice, + else => |v| blk: { + var err_str_ptr: [*c]const u8 = undefined; + if (cuGetErrorString(v, &err_str_ptr) == CUDA_SUCCESS) { + std.log.err("cuInit returned unexpected error code: {d}: {s}", .{ v, err_str_ptr }); + } else { + std.log.err("cuInit returned unexpected error code: {d}", .{v}); + } + break :blk ExitCode.UnexpectedError; + }, + }); +} diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl new file mode 100644 index 00000000..8482e52a --- /dev/null +++ b/platforms/cuda/cuda.bzl @@ -0,0 +1,384 @@ +load("@bazel_skylib//lib:paths.bzl", "paths") +load("@llvm//:http_bsdtar_archive.bzl", http_archive = "http_bsdtar_archive") +load("//bazel:http_deb_archive.bzl", "http_deb_archive") +load("//platforms:packages.bzl", "packages") + +ARCHS = ["linux-x86_64", "linux-sbsa"] + +CUDA_REDIST_PREFIX = "https://developer.download.nvidia.com/compute/cuda/redist/" +CUDA_VERSION = "13.3.0" +CUDA_VARIANT = "cuda13.3" +CUDA_REDIST_JSON_SHA256 = "507eddaab1360336bc0fe17b77552e0b7dfe1e74da888671c3a2f5fad7775db1" + +CUDNN_REDIST_PREFIX = "https://developer.download.nvidia.com/compute/cudnn/redist/" +CUDNN_VERSION = "9.24.0" +CUDNN_REDIST_JSON_SHA256 = "89a0626f1925d9f4ceb8d6bc0b2de160b30e92afaae5471e6cfbb50234ae5488" + +NVSHMEM_REDIST_PREFIX = "https://developer.download.nvidia.com/compute/nvshmem/redist/" +NVSHMEM_VERSION = "3.6.5" +NVSHMEM_REDIST_JSON_SHA256 = "afbf1ad5c4174c25a66cadd5b925d6814e679e8f836ef1fee1442afb6cb7fdd3" + +_BUILD_FILE_DEFAULT_VISIBILITY = """\ +package(default_visibility = ["//visibility:public"]) + +load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("@rules_cc//cc:cc_import.bzl", "cc_import") +load("@zml//bazel:patchelf.bzl", "patchelf") +""" + +CUDA_COMPAT_FILES = [ + "libcuda.so.1", + "libcudadebugger.so.1", + "libnvidia-nvvm.so.4", + "libnvidia-nvvm70.so.4", + "libnvidia-ptxjitcompiler.so.1", +] + +CUDA_PACKAGES = { + "cuda_nvml_dev": [ + packages.cc_library( + name = "nvml", + hdrs = ["include/nvml.h"], + includes = ["include"], + visibility = ["//visibility:public"], + ), + ], + "cuda_cudart": [ + # Driver API only + packages.cc_library( + name = "cuda", + hdrs = ["include/cuda.h"], + includes = ["include"], + ), + #TODO: Remove me as soon we use the Driver API in tracer.zig + packages.filegroup( + name = "cuda_cudart", + srcs = ["lib/libcudart.so.13"], + ), + ], + "cuda_cupti": [ + packages.filegroup( + name = "cuda_cupti", + srcs = ["lib/libcupti.so.13"], + ), + ], + "cuda_nvtx": [ + packages.cc_library_hdrs_glob( + name = "headers", + hdrs_glob = ["include/nvtx3/**"], + includes = ["include"], + ), + packages.filegroup( + name = "cuda_nvtx", + srcs = ["lib/libnvtx3interop.so"], + ), + ], + "cuda_compat": [ + packages.patchelf( + name = "{}.patchelf".format(file), + set_rpath = "$ORIGIN", + src = "compat/{}".format(file), + soname = file, + ) + for file in CUDA_COMPAT_FILES + ] + [ + """filegroup( + name = "cuda_compat", + srcs = [ + "compat/libnvidia-gpucomp.so.610.43.02", + "compat/libnvidia-tileiras.so.610.43.02", + ] + {} + select({{ + "@llvm//platforms/config:linux_x86_64": [ + "compat/libnvidia-pkcs11-openssl3.so.610.43.02", + ], + "@llvm//platforms/config:linux_aarch64": [], + }}), + )""".format(repr([ + ":{}.patchelf".format(file) + for file in CUDA_COMPAT_FILES + ])), + ], + "libcufft": [ + packages.filegroup( + name = "libcufft", + srcs = ["lib/libcufft.so.12"], + ), + ], + "libcusolver": [ + packages.filegroup( + name = "libcusolver", + srcs = ["lib/libcusolver.so.12"], + ), + ], + "libcusparse": [ + packages.filegroup( + name = "libcusparse", + srcs = ["lib/libcusparse.so.12"], + ), + ], + "libnvjitlink": [ + packages.filegroup( + name = "libnvjitlink", + srcs = ["lib/libnvJitLink.so.13"], + ), + ], + "cuda_nvcc": [ + packages.filegroup( + name = "cuda_nvcc", + srcs = [ + "bin/ptxas", + "bin/nvlink", + ], + ), + ], + "cuda_tileiras": [ + packages.filegroup( + name = "cuda_tileiras", + srcs = ["bin/tileiras"], + ), + ], + "libnvvm": [ + packages.filegroup( + name = "libnvvm", + srcs = [ + "nvvm/bin/cicc", + "nvvm/libdevice/libdevice.10.bc", + "nvvm/lib64/libnvvm.so", + "nvvm/lib64/libnvvm.so.4", + "nvvm/lib64/libnvvm.so.4.0.0", + ], + ), + ], + "cuda_nvrtc": [ + packages.filegroup( + name = "cuda_nvrtc", + srcs = [ + "lib/libnvrtc.so.13", + "lib/libnvrtc-builtins.so.13.3", + ], + ), + ], + "libcublas": [ + packages.filegroup( + name = "libcublas", + srcs = [ + "lib/libcublasLt.so.13", + "lib/libcublas.so.13", + ], + ), + ], +} + +CUDNN_PACKAGES = { + "cudnn": [ + packages.filegroup( + name = "cudnn", + srcs = [ + "lib/libcudnn.so.9", + "lib/libcudnn_adv.so.9", + "lib/libcudnn_ops.so.9", + "lib/libcudnn_cnn.so.9", + "lib/libcudnn_graph.so.9", + "lib/libcudnn_engines_precompiled.so.9", + "lib/libcudnn_engines_runtime_compiled.so.9", + "lib/libcudnn_engines_tensor_ir.so.9", + "lib/libcudnn_heuristic.so.9", + ], + ), + ], +} + +NVSHMEM_PACKAGES = { + "libnvshmem": [ + packages.filegroup( + name = "libnvshmem", + srcs = [ + "lib/libnvshmem_host.so.3", + "lib/nvshmem_bootstrap_uid.so.3", + "lib/nvshmem_transport_ibrc.so.5", + ], + ), + ], +} + +_UBUNTU_PACKAGES = { + "llvm-libunwind1": [ + """filegroup( + name = "libunwind", + srcs = select({ + "@llvm//platforms/config:linux_x86_64": ["usr/lib/x86_64-linux-gnu/libunwind.so.1"], + "@llvm//platforms/config:linux_aarch64": ["usr/lib/aarch64-linux-gnu/libunwind.so.1"], + }), + )""", + ], + "zlib1g": [ + """filegroup( + name = "zlib1g", + srcs = select({ + "@llvm//platforms/config:linux_x86_64": ["lib/x86_64-linux-gnu/libz.so.1"], + "@llvm//platforms/config:linux_aarch64": ["lib/aarch64-linux-gnu/libz.so.1"], + }), + )""", + ], +} + +_PLUGINS = { + "amd64": { + "sha256": "72b1359384ac50e727b800e81c56d7134f9d6a061a982c4e36b7fef62cf7c331", + "url": "https://mirror.zml.ai/plugins/202611101651.363.1.d46c7471c419/zml-cuda-linux-amd64.tar.zst", + }, + "arm64": { + "sha256": "caeefd717960fa2acec6304cd8296299e262abdba669ecf4d487e3af76d73a7b", + "url": "https://mirror.zml.ai/plugins/202611101651.363.1.d46c7471c419/zml-cuda-linux-arm64.tar.zst", + }, +} + +_NCCL_ASSETS = { + "amd64": { + "url": "https://pypi.nvidia.com/nvidia-nccl-cu13/nvidia_nccl_cu13-2.30.4-py3-none-manylinux_2_18_x86_64.whl", + "sha256": "534dbf3058cadb625f08ab0d17f1dffad3b961a2bfa360d66633fcf21be53f57", + }, + "arm64": { + "url": "https://pypi.nvidia.com/nvidia-nccl-cu13/nvidia_nccl_cu13-2.30.4-py3-none-manylinux_2_18_aarch64.whl", + "sha256": "e99308a3a89fba78918d50886e81072a6c8b0b4199feb02c3903e63713a6525a", + }, +} + +def _repo_suffix(arch): + return "linux_{}".format(arch) + +def _repo_name(name, arch): + return "{}_{}".format(name, _repo_suffix(arch)) + +def _read_redist_json(mctx, url, sha256): + fname = ".{}.json".format(sha256) + mctx.download( + url = url, + output = fname, + sha256 = sha256, + ) + return json.decode(mctx.read(fname)) + +def _cuda_impl(mctx): + loaded_packages = packages.read(mctx, [ + "@zml//platforms/cuda:packages.lock.json", + ]) + + CUDA_REDIST = _read_redist_json( + mctx, + url = CUDA_REDIST_PREFIX + "redistrib_{}.json".format(CUDA_VERSION), + sha256 = CUDA_REDIST_JSON_SHA256, + ) + + NVSHMEM_REDIST = _read_redist_json( + mctx, + url = NVSHMEM_REDIST_PREFIX + "redistrib_{}.json".format(NVSHMEM_VERSION), + sha256 = NVSHMEM_REDIST_JSON_SHA256, + ) + + CUDNN_REDIST = _read_redist_json( + mctx, + url = CUDNN_REDIST_PREFIX + "redistrib_{}.json".format(CUDNN_VERSION), + sha256 = CUDNN_REDIST_JSON_SHA256, + ) + + #TODO(cerisier): for each architecture + for pkg_name, build_file_content in _UBUNTU_PACKAGES.items(): + pkgs = loaded_packages[pkg_name] + for arch, pkg in pkgs.items(): + http_deb_archive( + name = _repo_name(pkg_name, arch), + urls = pkg["urls"], + sha256 = pkg["sha256"], + build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + "\n".join(build_file_content), + ) + + for pkg, build_file_content in CUDA_PACKAGES.items(): + pkg_data = CUDA_REDIST[pkg] + for arch in ARCHS: + pkg_repo_name = pkg + "_" + arch.replace("-", "_") + arch_data = pkg_data.get(arch) + if not arch_data: + fail("CUDA redist package {} does not have data for architecture {}".format(pkg, arch)) + arch_data = arch_data.get(CUDA_VARIANT, None) or arch_data + http_archive( + name = pkg_repo_name, + build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + "\n".join(build_file_content), + url = CUDA_REDIST_PREFIX + arch_data["relative_path"], + sha256 = arch_data["sha256"], + strip_prefix = paths.basename(arch_data["relative_path"]).replace(".tar.xz", ""), + ) + + for pkg, build_file_content in CUDNN_PACKAGES.items(): + for arch in ARCHS: + pkg_repo_name = pkg + "_" + arch.replace("-", "_") + pkg_data = CUDNN_REDIST[pkg] + arch_data = pkg_data.get(arch) + if not arch_data: + fail("CUDA redist package {} does not have data for architecture {}".format(pkg, arch)) + arch_data = arch_data.get("cuda13", arch_data) + http_archive( + name = pkg_repo_name, + build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + "\n".join(build_file_content), + url = CUDNN_REDIST_PREFIX + arch_data["relative_path"], + sha256 = arch_data["sha256"], + strip_prefix = paths.basename(arch_data["relative_path"]).replace(".tar.xz", ""), + ) + + for pkg, build_file_content in NVSHMEM_PACKAGES.items(): + for arch in ARCHS: + pkg_repo_name = pkg + "_" + arch.replace("-", "_") + pkg_data = NVSHMEM_REDIST[pkg] + arch_data = pkg_data.get(arch) + if not arch_data: + fail("CUDA redist package {} does not have data for architecture {}".format(pkg, arch)) + arch_data = arch_data.get("cuda13", arch_data) + http_archive( + name = pkg_repo_name, + build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + "\n".join(build_file_content), + url = NVSHMEM_REDIST_PREFIX + arch_data["relative_path"], + sha256 = arch_data["sha256"], + strip_prefix = paths.basename(arch_data["relative_path"]).replace(".tar.xz", ""), + ) + + for arch, arch_config in _NCCL_ASSETS.items(): + http_archive( + name = _repo_name("nccl", arch), + urls = [arch_config["url"]], + type = "zip", + sha256 = arch_config["sha256"], + build_file_content = "\n".join([ + _BUILD_FILE_DEFAULT_VISIBILITY, + packages.filegroup( + name = "nccl", + srcs = ["nvidia/nccl/lib/libnccl.so.2"], + ), + ]), + ) + + for arch, arch_config in _PLUGINS.items(): + http_archive( + name = _repo_name("libzml_cuda", arch), + build_file = "libzml_cuda.BUILD.bazel", + url = arch_config["url"], + sha256 = arch_config["sha256"], + ) + + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = [ + "libzml_cuda_linux_amd64", + "libzml_cuda_linux_arm64", + "cuda_nvml_dev_linux_x86_64", + "cuda_nvml_dev_linux_sbsa", + "zlib1g_linux_arm64", + "zlib1g_linux_amd64", + "cuda_nvtx_linux_x86_64", + "cuda_nvtx_linux_sbsa", + ], + root_module_direct_dev_deps = [], + ) + +cuda_packages = module_extension( + implementation = _cuda_impl, +) diff --git a/platforms/cuda/cuda.zig b/platforms/cuda/cuda.zig new file mode 100644 index 00000000..692ea226 --- /dev/null +++ b/platforms/cuda/cuda.zig @@ -0,0 +1,132 @@ +const std = @import("std"); +const builtin = @import("builtin"); + +const bazel = @import("bazel"); +const bazel_builtin = @import("bazel_builtin"); +const c = @import("c"); +const pjrt = @import("pjrt"); +const platforms_options = @import("platforms/options"); +const stdx = @import("stdx"); + +const compat_probe = @import("compat_probe.zig"); + +const nvidiaLibsPath = "/cuda/"; + +const log = std.log.scoped(.@"zml/platforms/cuda"); + +fn findCudaSandbox( + r: anytype, + buffer: *[std.Io.Dir.max_path_bytes]u8, +) !?[]const u8 { + const candidate = switch (builtin.cpu.arch) { + .aarch64 => "libzml_cuda_linux_arm64/sandbox", + .x86_64 => "libzml_cuda_linux_amd64/sandbox", + else => return null, + }; + return try r.rlocation(candidate, buffer); +} + +pub fn isEnabled() bool { + return platforms_options.cuda_enabled; +} + +pub fn needsCudaCompat(io: std.Io, sandbox_path: []const u8) !bool { + var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const nvidia_compat_path = try stdx.Io.Dir.path.bufJoinZ(&buf, &.{ sandbox_path, "bin", "compat_probe" }); + + var child = try std.process.spawn(io, .{ + .argv = &[_][]const u8{nvidia_compat_path}, + .cwd = .{ .path = sandbox_path }, + .stdin = .ignore, + .stdout = .inherit, + .stderr = .inherit, + }); + defer child.kill(io); + + const res = child.wait(io) catch |err| { + log.err("Failed to run CUDA compatibility probe: {any}", .{err}); + return err; + }; + const result: compat_probe.ExitCode = @fromBackingInt(@intCast(res.exited)); + + return switch (result) { + .Success => true, + .SystemDriverMismatch, .CompatNotSupportedOnDevice => false, + .UnexpectedError => blk: { + log.err("CUDA compatibility probe returned unexpected error code", .{}); + break :blk false; + }, + }; +} + +fn hasNvidiaDevice(io: std.Io) bool { + for (&[_][]const u8{ "/dev/nvidiactl", "/dev/dxg" }) |dev| { + std.Io.Dir.accessAbsolute(io, dev, .{ .read = true }) catch continue; + return true; + } + return false; +} + +fn hasCudaPathInLDPath() bool { + const ldLibraryPath = std.c.getenv("LD_LIBRARY_PATH") orelse return false; + return std.ascii.findIgnoreCase(std.mem.span(ldLibraryPath), nvidiaLibsPath) != null; +} + +fn setupXlaGpuCudaDirFlag(allocator: std.mem.Allocator, sandbox: []const u8) !void { + const xla_flags = std.c.getenv("XLA_FLAGS") orelse ""; + const new_xla_flagsZ = try std.fmt.allocPrintSentinel(allocator, "{s} --xla_gpu_cuda_data_dir={s}", .{ xla_flags, sandbox }, 0); + _ = c.setenv("XLA_FLAGS", new_xla_flagsZ, 1); +} + +pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { + if (comptime !isEnabled()) { + return error.Unavailable; + } + if (comptime builtin.os.tag != .linux) { + return error.Unavailable; + } + if (!hasNvidiaDevice(io)) { + return error.Unavailable; + } + if (hasCudaPathInLDPath()) { + log.warn("Detected {s} in LD_LIBRARY_PATH. This can lead to undefined behaviors and crashes", .{nvidiaLibsPath}); + } + + var arena = std.heap.ArenaAllocator.init(allocator); + defer arena.deinit(); + + const r = try bazel.runfiles(bazel_builtin.current_repository); + + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const sandbox_path = try findCudaSandbox(r, &path_buf) orelse { + log.err("Failed to find sandbox path for CUDA runtime", .{}); + return error.FileNotFound; + }; + + // CUDA path has to be set _before_ loading the PJRT plugin. + // See https://github.com/openxla/xla/issues/21428 + try setupXlaGpuCudaDirFlag(arena.allocator(), sandbox_path); + + { + const cudaCompat = needsCudaCompat(io, sandbox_path) catch |err| blk: { + log.err("Unable to determine wether or not to use CUDA Compat, disabling: {any}", .{err}); + break :blk false; + }; + + if (cudaCompat) { + log.info("Detected NVIDIA GPU that requires CUDA compatibility libraries.", .{}); + var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "compat", "libcuda.so.1" }); + _ = std.c.dlopen(path, .{ .NOW = true }) orelse { + log.warn("Failed to load CUDA compatibility library from {s}: {any}", .{ path, std.mem.span(std.c.dlerror()) }); + }; + log.info("Loaded CUDA compatibility libraries.", .{}); + } + } + + return blk: { + var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libzml_cuda.so" }); + break :blk .loadFrom(path); + }; +} diff --git a/platforms/cuda/flashinfer_cutlass_moe/BUILD.bazel b/platforms/cuda/flashinfer_cutlass_moe/BUILD.bazel new file mode 100644 index 00000000..8a61e394 --- /dev/null +++ b/platforms/cuda/flashinfer_cutlass_moe/BUILD.bazel @@ -0,0 +1,48 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +_DEPS = [ + "//bazel", + "@rules_zig//zig/runfiles", +] + select({ + "@llvm//platforms/config:linux_x86_64": [ + "@flashinfer_cutlass_moe_linux_amd64//:c", + ], + "@llvm//platforms/config:linux_aarch64": [ + "@flashinfer_cutlass_moe_linux_arm64//:c", + ], + "//conditions:default": [], +}) + +_DATA = select({ + "@llvm//platforms/config:linux_x86_64": [ + "@flashinfer_cutlass_moe_linux_amd64//:libraries", + ], + "@llvm//platforms/config:linux_aarch64": [ + "@flashinfer_cutlass_moe_linux_arm64//:libraries", + ], + "//conditions:default": [], +}) + +zig_library( + name = "flashinfer_cutlass_moe", + data = _DATA, + import_name = "platforms/cuda/flashinfer_cutlass_moe", + main = "flashinfer_cutlass_moe.zig", + tags = ["manual"], + target_compatible_with = [ + "@platforms//os:linux", + ], + visibility = ["//visibility:public"], + deps = _DEPS, +) + +zig_test( + name = "test", + data = _DATA, + main = "flashinfer_cutlass_moe.zig", + tags = ["manual"], + target_compatible_with = [ + "@platforms//os:linux", + ], + deps = _DEPS, +) diff --git a/platforms/cuda/flashinfer_cutlass_moe/flashinfer_cutlass_moe.zig b/platforms/cuda/flashinfer_cutlass_moe/flashinfer_cutlass_moe.zig new file mode 100644 index 00000000..0b4360cf --- /dev/null +++ b/platforms/cuda/flashinfer_cutlass_moe/flashinfer_cutlass_moe.zig @@ -0,0 +1,165 @@ +const std = @import("std"); + +const bazel = @import("bazel"); +const bazel_builtin = @import("bazel_builtin"); +const builtin = @import("builtin"); +pub const c = @import("c"); + +pub const Status = c.zml_fi_cutlass_moe_status; +pub const DType = c.zml_fi_cutlass_moe_dtype; +pub const Activation = c.zml_fi_cutlass_moe_activation; +pub const RunnerOptions = c.zml_fi_cutlass_moe_runner_options; +pub const Context = c.zml_fi_cutlass_moe_context; +pub const Io = c.zml_fi_cutlass_moe_io; +pub const Workspace = c.zml_fi_cutlass_moe_workspace; +pub const WorkspaceRequirements = c.zml_fi_cutlass_moe_workspace_requirements; +pub const Runner = c.zml_fi_cutlass_moe_runner; + +pub const abiVersionExpected = c.ZML_FI_CUTLASS_MOE_ABI_VERSION; + +const AbiVersionFn = @TypeOf(&c.zml_fi_cutlass_moe_abi_version); +const CompiledSmFn = @TypeOf(&c.zml_fi_cutlass_moe_compiled_sm); +const LastErrorFn = @TypeOf(&c.zml_fi_cutlass_moe_last_error); +const DeviceIsSupportedFn = @TypeOf(&c.zml_fi_cutlass_moe_device_is_supported); +const RunnerCreateFn = @TypeOf(&c.zml_fi_cutlass_moe_runner_create); +const RunnerDestroyFn = @TypeOf(&c.zml_fi_cutlass_moe_runner_destroy); +const GetTacticCountsFn = @TypeOf(&c.zml_fi_cutlass_moe_get_tactic_counts); +const GetTacticOccupancyFn = @TypeOf(&c.zml_fi_cutlass_moe_get_tactic_occupancy); +const GetWorkspaceRequirementsFn = @TypeOf(&c.zml_fi_cutlass_moe_get_workspace_requirements); +const RunFn = @TypeOf(&c.zml_fi_cutlass_moe_run); + +pub const Api = struct { + library: std.DynLib, + abiVersion: AbiVersionFn, + compiledSm: CompiledSmFn, + lastError: LastErrorFn, + deviceIsSupported: DeviceIsSupportedFn, + runnerCreate: RunnerCreateFn, + runnerDestroy: RunnerDestroyFn, + getTacticCounts: GetTacticCountsFn, + getTacticOccupancy: GetTacticOccupancyFn, + getWorkspaceRequirements: GetWorkspaceRequirementsFn, + run: RunFn, +}; + +var loaded_api: ?Api = null; + +fn libraryRunfile(sm: u16, buffer: []u8) ![]const u8 { + if (comptime builtin.os.tag != .linux) return error.UnsupportedPlatform; + const cpu_arch = comptime switch (builtin.cpu.arch) { + .aarch64 => "arm64", + .x86_64 => "amd64", + else => return error.UnsupportedPlatform, + }; + return try std.fmt.bufPrint( + buffer, + "flashinfer_cutlass_moe_linux_{s}/lib/libflashinfer_cutlass_moe_sm{d}.so", + .{ cpu_arch, sm }, + ); +} + +fn loadApi(path: []const u8) !Api { + var library = std.DynLib.open(path) catch |err| { + std.log.err("Failed to open FlashInfer CUTLASS MoE library {s}: {any}", .{ path, err }); + return err; + }; + errdefer library.close(); + + const api: Api = .{ + .library = library, + .abiVersion = library.lookup(AbiVersionFn, "zml_fi_cutlass_moe_abi_version") orelse + return error.SymbolNotFound, + .compiledSm = library.lookup(CompiledSmFn, "zml_fi_cutlass_moe_compiled_sm") orelse + return error.SymbolNotFound, + .lastError = library.lookup(LastErrorFn, "zml_fi_cutlass_moe_last_error") orelse + return error.SymbolNotFound, + .deviceIsSupported = library.lookup(DeviceIsSupportedFn, "zml_fi_cutlass_moe_device_is_supported") orelse + return error.SymbolNotFound, + .runnerCreate = library.lookup(RunnerCreateFn, "zml_fi_cutlass_moe_runner_create") orelse + return error.SymbolNotFound, + .runnerDestroy = library.lookup(RunnerDestroyFn, "zml_fi_cutlass_moe_runner_destroy") orelse + return error.SymbolNotFound, + .getTacticCounts = library.lookup(GetTacticCountsFn, "zml_fi_cutlass_moe_get_tactic_counts") orelse + return error.SymbolNotFound, + .getTacticOccupancy = library.lookup(GetTacticOccupancyFn, "zml_fi_cutlass_moe_get_tactic_occupancy") orelse + return error.SymbolNotFound, + .getWorkspaceRequirements = library.lookup( + GetWorkspaceRequirementsFn, + "zml_fi_cutlass_moe_get_workspace_requirements", + ) orelse return error.SymbolNotFound, + .run = library.lookup(RunFn, "zml_fi_cutlass_moe_run") orelse + return error.SymbolNotFound, + }; + if (api.abiVersion() != abiVersionExpected) return error.AbiVersionMismatch; + return api; +} + +pub fn load(allocator: std.mem.Allocator, io: std.Io, sm: u16) !void { + _ = allocator; + if (loaded_api) |*api| { + if (api.compiledSm() != sm) return error.ArchitectureAlreadyLoaded; + return; + } + + const runfiles = try bazel.runfiles(bazel_builtin.current_repository); + var runfile_name_buffer: [160]u8 = undefined; + const runfile_name = try libraryRunfile(sm, &runfile_name_buffer); + var runfile_path_buffer: [std.Io.Dir.max_path_bytes]u8 = undefined; + const runfile_path = (try runfiles.rlocation(runfile_name, &runfile_path_buffer)) orelse + return error.NotFound; + + // Preserve the producer library's $ORIGIN-relative dependency lookup when + // the development repository exposes it through Bazel symlinks. + var canonical_buffer: [std.Io.Dir.max_path_bytes]u8 = undefined; + const canonical_length = if (std.fs.path.isAbsolute(runfile_path)) + try std.Io.Dir.realPathFileAbsolute(io, runfile_path, &canonical_buffer) + else + try std.Io.Dir.cwd().realPathFile(io, runfile_path, &canonical_buffer); + var api = try loadApi(canonical_buffer[0..canonical_length]); + errdefer api.library.close(); + if (api.compiledSm() != sm) return error.WrongArchitectureLibrary; + loaded_api = api; +} + +pub fn apiForDevice(device: i32) !*Api { + const api = if (loaded_api) |*value| value else return error.BackendNotLoaded; + var supported: u8 = 0; + const status = api.deviceIsSupported(device, &supported); + if (status != c.ZML_FI_CUTLASS_MOE_STATUS_SUCCESS) { + if (api.lastError()) |message| { + std.log.err("FlashInfer CUTLASS MoE device probe failed: {s}", .{std.mem.span(message)}); + } + return error.DeviceProbeFailed; + } + if (supported != 0) return api; + return error.UnsupportedArchitecture; +} + +test "select host and GPU architecture-specific FlashInfer CUTLASS MoE library" { + const test_case = &.{ + .{ + .compat_version = 120, + .expected = switch (builtin.cpu.arch) { + .aarch64 => "flashinfer_cutlass_moe_linux_arm64/lib/libflashinfer_cutlass_moe_sm120.so", + .x86_64 => "flashinfer_cutlass_moe_linux_amd64/lib/libflashinfer_cutlass_moe_sm120.so", + }, + }, + .{ + .compat_version = 103, + .expected = switch (builtin.cpu.arch) { + .aarch64 => "flashinfer_cutlass_moe_linux_arm64/lib/libflashinfer_cutlass_moe_sm103.so", + .x86_64 => "flashinfer_cutlass_moe_linux_amd64/lib/libflashinfer_cutlass_moe_sm103.so", + }, + }, + }; + + inline for (test_case) |t| { + var buffer: [160]u8 = undefined; + const path = try libraryRunfile(t.compat_version, &buffer); + try std.testing.expectEqualStrings(t.expected, path); + + try load(std.testing.allocator, std.testing.io, 120); + const api = if (loaded_api) |*value| value else return error.NotLoaded; + try std.testing.expectEqual(@as(i32, 120), api.compiledSm()); + } +} diff --git a/platforms/cuda/libzml_cuda.BUILD.bazel b/platforms/cuda/libzml_cuda.BUILD.bazel new file mode 100644 index 00000000..e79ab900 --- /dev/null +++ b/platforms/cuda/libzml_cuda.BUILD.bazel @@ -0,0 +1,108 @@ +load("@bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") +load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("@zml//bazel:patchelf.bzl", "patchelf") +load("@zml//platforms/cuda:cuda.bzl", "CUDA_COMPAT_FILES") + +patchelf( + name = "libzml_cuda_so", + src = "lib/libzml_cuda.so", + add_needed = [ + "libzmlxcuda.so.0", + ], + rename_dynamic_symbols = { + "dlopen": "zmlxcuda_dlopen", + }, + set_rpath = "$ORIGIN", +) + +copy_to_directory( + name = "sandbox", + srcs = [ + "lib/cutlass_ir.so", + ":libzml_cuda_so", + "@zml//platforms/cuda:compat_probe", + "@zml//platforms/cuda:zmlxcuda", + ] + select({ + "@llvm//platforms/config:linux_x86_64": [ + "@cuda_compat_linux_x86_64//:cuda_compat", + "@cuda_cudart_linux_x86_64//:cuda_cudart", + "@cuda_cupti_linux_x86_64//:cuda_cupti", + "@cuda_nvcc_linux_x86_64//:cuda_nvcc", + "@cuda_nvrtc_linux_x86_64//:cuda_nvrtc", + "@cuda_nvtx_linux_x86_64//:cuda_nvtx", + "@cuda_tileiras_linux_x86_64//:cuda_tileiras", + "@cudnn_linux_x86_64//:cudnn", + "@libcublas_linux_x86_64//:libcublas", + "@libcufft_linux_x86_64//:libcufft", + "@libcusolver_linux_x86_64//:libcusolver", + "@libcusparse_linux_x86_64//:libcusparse", + "@libnvjitlink_linux_x86_64//:libnvjitlink", + "@libnvshmem_linux_x86_64//:libnvshmem", + "@libnvvm_linux_x86_64//:libnvvm", + "@llvm-libunwind1_linux_amd64//:libunwind", + "@nccl_linux_amd64//:nccl", + "@zlib1g_linux_amd64//:zlib1g", + ], + "@llvm//platforms/config:linux_aarch64": [ + "@cuda_compat_linux_sbsa//:cuda_compat", + "@cuda_cudart_linux_sbsa//:cuda_cudart", + "@cuda_cupti_linux_sbsa//:cuda_cupti", + "@cuda_nvcc_linux_sbsa//:cuda_nvcc", + "@cuda_nvrtc_linux_sbsa//:cuda_nvrtc", + "@cuda_nvtx_linux_sbsa//:cuda_nvtx", + "@cuda_tileiras_linux_sbsa//:cuda_tileiras", + "@cudnn_linux_sbsa//:cudnn", + "@libcublas_linux_sbsa//:libcublas", + "@libcufft_linux_sbsa//:libcufft", + "@libcusolver_linux_sbsa//:libcusolver", + "@libcusparse_linux_sbsa//:libcusparse", + "@libnvjitlink_linux_sbsa//:libnvjitlink", + "@libnvshmem_linux_sbsa//:libnvshmem", + "@libnvvm_linux_sbsa//:libnvvm", + "@llvm-libunwind1_linux_arm64//:libunwind", + "@nccl_linux_arm64//:nccl", + "@zlib1g_linux_arm64//:zlib1g", + ], + }), + add_directory_to_runfiles = False, + include_external_repositories = ["**"], + replace_prefixes = { + "compat": "lib/compat", + "nvidia/nccl/lib": "lib", + "nvvm/lib64": "lib", + "libzml_cuda_so": "lib", + "lib/x86_64-linux-gnu": "lib", + "lib/aarch64-linux-gnu": "lib", + "usr/lib/x86_64-linux-gnu": "lib", + "usr/lib/aarch64-linux-gnu": "lib", + "platforms/cuda/compat_probe": "bin/compat_probe", + "platforms/cuda/libzmlxcuda": "lib/libzmlxcuda", + } | { + "{}.patchelf".format(file): "lib/compat" + for file in CUDA_COMPAT_FILES + }, +) + +cc_library( + name = "libzml_cuda", + data = [":sandbox"], + linkopts = [ + # Defer function call resolution until the function is called + # (lazy loading) rather than at load time. + # + # This is required because we want to let downstream use weak CUDA symbols. + # + # We force it here because -z,now (which resolve all symbols at load time), + # is the default in most bazel CC toolchains as well as in certain linkers. + "-Wl,-z,lazy", + ], + visibility = ["@zml//platforms/cuda:__subpackages__"], + deps = select({ + "@llvm//platforms/config:linux_x86_64": [ + "@cuda_cudart_linux_x86_64//:cuda", + ], + "@llvm//platforms/config:linux_aarch64": [ + "@cuda_cudart_linux_sbsa//:cuda", + ], + }), +) diff --git a/platforms/cuda/libzml_cuda.h b/platforms/cuda/libzml_cuda.h new file mode 100644 index 00000000..c2d9f0da --- /dev/null +++ b/platforms/cuda/libzml_cuda.h @@ -0,0 +1 @@ +#include diff --git a/platforms/cuda/packages.lock.json b/platforms/cuda/packages.lock.json new file mode 100755 index 00000000..3563bc61 --- /dev/null +++ b/platforms/cuda/packages.lock.json @@ -0,0 +1,287 @@ +{ + "packages": [ + { + "arch": "amd64", + "dependencies": [ + { + "key": "libc6_2.36-9-p-deb12u10_amd64", + "name": "libc6", + "version": "2.36-9+deb12u10" + }, + { + "key": "libgcc-s1_12.2.0-14-p-deb12u1_amd64", + "name": "libgcc-s1", + "version": "12.2.0-14+deb12u1" + }, + { + "key": "gcc-12-base_12.2.0-14-p-deb12u1_amd64", + "name": "gcc-12-base", + "version": "12.2.0-14+deb12u1" + } + ], + "key": "llvm-libunwind1_1-22.1.8_-p--p-20260714014842-p-ca7933e47d3a-1_exp1_20260714134950.86_amd64", + "name": "llvm-libunwind1", + "sha256": "04cb76d70da569e5bcf2395f28dd7e9b9e9f0bf4521cf3b008b0f62ae9d2b712", + "urls": [ + "https://apt.llvm.org/bookworm/pool/main/l/llvm-toolchain-22/llvm-libunwind1_22.1.8~++20260714014842+ca7933e47d3a-1~exp1~20260714134950.86_amd64.deb" + ], + "version": "1:22.1.8~++20260714014842+ca7933e47d3a-1~exp1~20260714134950.86" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libc6_2.36-9-p-deb12u10_amd64", + "name": "libc6", + "sha256": "5dc83256f10ca4d0f2a53dd6583ffd0d0e319af30074ea6c82fb0ae77bd16365", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z/pool/main/g/glibc/libc6_2.36-9+deb12u10_amd64.deb" + ], + "version": "2.36-9+deb12u10" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libgcc-s1_12.2.0-14-p-deb12u1_amd64", + "name": "libgcc-s1", + "sha256": "3016e62cb4b7cd8038822870601f5ed131befe942774d0f745622cc77d8a88f7", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z/pool/main/g/gcc-12/libgcc-s1_12.2.0-14+deb12u1_amd64.deb" + ], + "version": "12.2.0-14+deb12u1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "gcc-12-base_12.2.0-14-p-deb12u1_amd64", + "name": "gcc-12-base", + "sha256": "1896a2aacf4ad681ff5eacc24a5b0ca4d5d9c9b9c9e4b6de5197bc1e116ea619", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z/pool/main/g/gcc-12/gcc-12-base_12.2.0-14+deb12u1_amd64.deb" + ], + "version": "12.2.0-14+deb12u1" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "libc6_2.36-9-p-deb12u10_amd64", + "name": "libc6", + "version": "2.36-9+deb12u10" + }, + { + "key": "libgcc-s1_12.2.0-14-p-deb12u1_amd64", + "name": "libgcc-s1", + "version": "12.2.0-14+deb12u1" + }, + { + "key": "gcc-12-base_12.2.0-14-p-deb12u1_amd64", + "name": "gcc-12-base", + "version": "12.2.0-14+deb12u1" + } + ], + "key": "zlib1g_1-1.2.13.dfsg-1_amd64", + "name": "zlib1g", + "sha256": "d7dd1d1411fedf27f5e27650a6eff20ef294077b568f4c8c5e51466dc7c08ce4", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z/pool/main/z/zlib/zlib1g_1.2.13.dfsg-1_amd64.deb" + ], + "version": "1:1.2.13.dfsg-1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "cuda-compat-13-4_615.71.09-2ubuntu1_amd64", + "name": "cuda-compat-13-4", + "sha256": "1b814aa705b4dfe955420c967bd545f39ef7b67f32bd0f285a005ea296b02831", + "urls": [ + "https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/./cuda-compat-13-4_615.71.09-2ubuntu1_amd64.deb" + ], + "version": "615.71.09-2ubuntu1" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "libstdc-p--p-6_12.2.0-14-p-deb12u1_amd64", + "name": "libstdc++6", + "version": "12.2.0-14+deb12u1" + }, + { + "key": "libgcc-s1_12.2.0-14-p-deb12u1_amd64", + "name": "libgcc-s1", + "version": "12.2.0-14+deb12u1" + }, + { + "key": "libc6_2.36-9-p-deb12u10_amd64", + "name": "libc6", + "version": "2.36-9+deb12u10" + }, + { + "key": "gcc-12-base_12.2.0-14-p-deb12u1_amd64", + "name": "gcc-12-base", + "version": "12.2.0-14+deb12u1" + } + ], + "key": "libnccl2_2.32.3-1-p-cuda13.4_amd64", + "name": "libnccl2", + "sha256": "55a3e32a3f560f20e2f423cd38f1f284951e92093d1e8fbaabb4c42ac76f3195", + "urls": [ + "https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/./libnccl2_2.32.3-1+cuda13.4_amd64.deb" + ], + "version": "2.32.3-1+cuda13.4" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libstdc-p--p-6_12.2.0-14-p-deb12u1_amd64", + "name": "libstdc++6", + "sha256": "5cd3171216d4ab0fc911cfe9c35509bf2dd8f47761c43b7f6a4296701551a24d", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z/pool/main/g/gcc-12/libstdc++6_12.2.0-14+deb12u1_amd64.deb" + ], + "version": "12.2.0-14+deb12u1" + }, + { + "arch": "arm64", + "dependencies": [ + { + "key": "libc6_2.36-9-p-deb12u10_arm64", + "name": "libc6", + "version": "2.36-9+deb12u10" + }, + { + "key": "libgcc-s1_12.2.0-14-p-deb12u1_arm64", + "name": "libgcc-s1", + "version": "12.2.0-14+deb12u1" + }, + { + "key": "gcc-12-base_12.2.0-14-p-deb12u1_arm64", + "name": "gcc-12-base", + "version": "12.2.0-14+deb12u1" + } + ], + "key": "llvm-libunwind1_1-22.1.8_-p--p-20260714014842-p-ca7933e47d3a-1_exp1_20260714134950.86_arm64", + "name": "llvm-libunwind1", + "sha256": "05890076ab166d0fe3eab3ab08f9076bca1edc7322d123df8970760ae26c25e8", + "urls": [ + "https://apt.llvm.org/bookworm/pool/main/l/llvm-toolchain-22/llvm-libunwind1_22.1.8~++20260714014842+ca7933e47d3a-1~exp1~20260714134950.86_arm64.deb" + ], + "version": "1:22.1.8~++20260714014842+ca7933e47d3a-1~exp1~20260714134950.86" + }, + { + "arch": "arm64", + "dependencies": [], + "key": "libc6_2.36-9-p-deb12u10_arm64", + "name": "libc6", + "sha256": "9b25c7c2aaa9328b2cfd92b47cc48b24631bfdff0c61afe05cba937911fcf239", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z/pool/main/g/glibc/libc6_2.36-9+deb12u10_arm64.deb" + ], + "version": "2.36-9+deb12u10" + }, + { + "arch": "arm64", + "dependencies": [], + "key": "libgcc-s1_12.2.0-14-p-deb12u1_arm64", + "name": "libgcc-s1", + "sha256": "576926b283613db80168ddf76380a3bd877602778cf0d226caa7bfbfa71eacf3", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z/pool/main/g/gcc-12/libgcc-s1_12.2.0-14+deb12u1_arm64.deb" + ], + "version": "12.2.0-14+deb12u1" + }, + { + "arch": "arm64", + "dependencies": [], + "key": "gcc-12-base_12.2.0-14-p-deb12u1_arm64", + "name": "gcc-12-base", + "sha256": "674cf6cba6d432bd200c45fe866c1652c7a53523cc2e7a613e05bc4abf7b5440", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z/pool/main/g/gcc-12/gcc-12-base_12.2.0-14+deb12u1_arm64.deb" + ], + "version": "12.2.0-14+deb12u1" + }, + { + "arch": "arm64", + "dependencies": [ + { + "key": "libc6_2.36-9-p-deb12u10_arm64", + "name": "libc6", + "version": "2.36-9+deb12u10" + }, + { + "key": "libgcc-s1_12.2.0-14-p-deb12u1_arm64", + "name": "libgcc-s1", + "version": "12.2.0-14+deb12u1" + }, + { + "key": "gcc-12-base_12.2.0-14-p-deb12u1_arm64", + "name": "gcc-12-base", + "version": "12.2.0-14+deb12u1" + } + ], + "key": "zlib1g_1-1.2.13.dfsg-1_arm64", + "name": "zlib1g", + "sha256": "52b8b8a145bbe1956bba82034f77022cbef0c3d0885c9e32d9817a7932fe1913", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z/pool/main/z/zlib/zlib1g_1.2.13.dfsg-1_arm64.deb" + ], + "version": "1:1.2.13.dfsg-1" + }, + { + "arch": "arm64", + "dependencies": [], + "key": "cuda-compat-13-4_615.71.09-2ubuntu1_arm64", + "name": "cuda-compat-13-4", + "sha256": "83cdca90db73a36784a077693e3b48565d852158c63f604194c1f0d42c2e5458", + "urls": [ + "https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/sbsa/./cuda-compat-13-4_615.71.09-2ubuntu1_arm64.deb" + ], + "version": "615.71.09-2ubuntu1" + }, + { + "arch": "arm64", + "dependencies": [ + { + "key": "libstdc-p--p-6_12.2.0-14-p-deb12u1_arm64", + "name": "libstdc++6", + "version": "12.2.0-14+deb12u1" + }, + { + "key": "libgcc-s1_12.2.0-14-p-deb12u1_arm64", + "name": "libgcc-s1", + "version": "12.2.0-14+deb12u1" + }, + { + "key": "libc6_2.36-9-p-deb12u10_arm64", + "name": "libc6", + "version": "2.36-9+deb12u10" + }, + { + "key": "gcc-12-base_12.2.0-14-p-deb12u1_arm64", + "name": "gcc-12-base", + "version": "12.2.0-14+deb12u1" + } + ], + "key": "libnccl2_2.32.3-1-p-cuda13.4_arm64", + "name": "libnccl2", + "sha256": "809a6eea2f8a570050229bc218e30d4280829cbe01e824e052c96ee9203a7fe5", + "urls": [ + "https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/sbsa/./libnccl2_2.32.3-1+cuda13.4_arm64.deb" + ], + "version": "2.32.3-1+cuda13.4" + }, + { + "arch": "arm64", + "dependencies": [], + "key": "libstdc-p--p-6_12.2.0-14-p-deb12u1_arm64", + "name": "libstdc++6", + "sha256": "26e138c677a985775331373828a6c286c551ff397cb735d00e2383cb273d1cb2", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z/pool/main/g/gcc-12/libstdc++6_12.2.0-14+deb12u1_arm64.deb" + ], + "version": "12.2.0-14+deb12u1" + } + ], + "version": 1 +} \ No newline at end of file diff --git a/platforms/cuda/packages.yaml b/platforms/cuda/packages.yaml new file mode 100644 index 00000000..010ca572 --- /dev/null +++ b/platforms/cuda/packages.yaml @@ -0,0 +1,26 @@ +# +# bazel run @apt_cuda//:lock +# +version: 1 + +sources: + - channel: bookworm main + url: https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z + - channel: llvm-toolchain-bookworm-22 main + url: https://apt.llvm.org/bookworm + - channel: / + url: https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64 + archs: [amd64] + - channel: / + url: https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/sbsa + archs: [arm64] + +archs: + - "amd64" + - "arm64" + +packages: + - "llvm-libunwind1" # LLVM 22 runtime from llvm-toolchain-bookworm-22. + - "zlib1g" + - "cuda-compat-13-4" + - "libnccl2" diff --git a/platforms/cuda/zmlxcuda.zig b/platforms/cuda/zmlxcuda.zig new file mode 100644 index 00000000..5834cb6b --- /dev/null +++ b/platforms/cuda/zmlxcuda.zig @@ -0,0 +1,28 @@ +const std = @import("std"); + +const stdx = @import("stdx"); + +pub export fn zmlxcuda_dlopen(filename: [*c]const u8, flags: c_int) ?*anyopaque { + const replacements: std.StaticStringMap([:0]const u8) = .initComptime(.{ + .{ "libcublas.so", "libcublas.so.13" }, + .{ "libcublasLt.so", "libcublasLt.so.13" }, + .{ "libcudart.so", "libcudart.so.13" }, + .{ "libcudnn.so", "libcudnn.so.9" }, + .{ "libcufft.so", "libcufft.so.12" }, + .{ "libcupti.so", "libcupti.so.13" }, + .{ "libcusolver.so", "libcusolver.so.12" }, + .{ "libcusparse.so", "libcusparse.so.12" }, + .{ "libnccl.so", "libnccl.so.2" }, + }); + + var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const new_filename: [*c]const u8 = if (filename) |f| blk: { + const replacement = replacements.get(std.Io.Dir.path.basename(std.mem.span(f))) orelse break :blk f; + break :blk stdx.Io.Dir.path.bufJoinZ(&buf, &.{ + stdx.process.selfSharedObjectDirPath(), + replacement, + }) catch unreachable; + } else null; + + return std.c.dlopen(new_filename, @bitCast(flags)); +} diff --git a/platforms/furiosa/BUILD.bazel b/platforms/furiosa/BUILD.bazel new file mode 100644 index 00000000..3118e6d0 --- /dev/null +++ b/platforms/furiosa/BUILD.bazel @@ -0,0 +1,21 @@ +load("@rules_shell//shell:sh_test.bzl", "sh_test") +load("@rules_zig//zig:defs.bzl", "zig_library") + +exports_files(["libzml_furiosa.BUILD.bazel"]) + +zig_library( + name = "furiosa", + import_name = "platforms/furiosa", + main = "furiosa.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//bazel", + "//pjrt", + "//platforms:options", + "//stdx", + ] + select({ + "//platforms:furiosa.enabled": ["@libzml_furiosa"], + "//conditions:default": [], + }), +) diff --git a/platforms/furiosa/furiosa.bzl b/platforms/furiosa/furiosa.bzl new file mode 100644 index 00000000..afb26864 --- /dev/null +++ b/platforms/furiosa/furiosa.bzl @@ -0,0 +1,21 @@ +"""Consume the complete Furiosa sandbox produced by XLA.""" + +load("@llvm//:http_bsdtar_archive.bzl", http_archive = "http_bsdtar_archive") + +def _furiosa_impl(mctx): + http_archive( + name = "libzml_furiosa", + build_file = "libzml_furiosa.BUILD.bazel", + url = "https://mirror.zml.ai/plugins/202611131503.373.1.be246d178d7d/zml-furiosa-linux-amd64.tar.zst", + sha256 = "c7cdf0120818589aee3b53d8a6813e2b5e244447701acc34eb697b30f46e22f9", + ) + + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = ["libzml_furiosa"], + root_module_direct_dev_deps = [], + ) + +furiosa_packages = module_extension( + implementation = _furiosa_impl, +) diff --git a/platforms/furiosa/furiosa.zig b/platforms/furiosa/furiosa.zig new file mode 100644 index 00000000..ee33fd58 --- /dev/null +++ b/platforms/furiosa/furiosa.zig @@ -0,0 +1,57 @@ +const std = @import("std"); +const builtin = @import("builtin"); + +const bazel = @import("bazel"); +const bazel_builtin = @import("bazel_builtin"); +const pjrt = @import("pjrt"); +const platforms_options = @import("platforms/options"); +const stdx = @import("stdx"); + +const log = std.log.scoped(.@"zml/platforms/furiosa"); + +extern fn setenv(name: [*:0]const u8, value: [*:0]const u8, overwrite: c_int) c_int; + +fn setEnv(name: [*:0]const u8, value: [:0]const u8, overwrite: c_int) !void { + if (setenv(name, value, overwrite) != 0) return error.SetEnvFailed; +} + +pub fn isEnabled() bool { + return platforms_options.furiosa_enabled; +} + +fn hasFuriosaDevice(io: std.Io) bool { + std.Io.Dir.accessAbsolute(io, "/dev/rngd", .{ .read = true }) catch return false; + return true; +} + +fn hasDmaHeapAccess(io: std.Io) bool { + std.Io.Dir.accessAbsolute(io, "/dev/dma_heap/system", .{ .read = true }) catch return false; + return true; +} + +pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { + if (comptime !isEnabled() or builtin.os.tag != .linux or builtin.cpu.arch != .x86_64) return error.Unavailable; + if (!hasFuriosaDevice(io)) { + return error.Unavailable; + } + if (!hasDmaHeapAccess(io)) { + std.debug.panic("Furiosa needs read access to /dev/dma_heap/system for pinned host memory", .{}); + } + + const r = try bazel.runfiles(bazel_builtin.current_repository); + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const sandbox = try r.rlocation("libzml_furiosa/sandbox", &path_buf) orelse { + log.err("Missing Furiosa sandbox runfile", .{}); + return error.FileNotFound; + }; + + const root = try std.Io.Dir.cwd().realPathFileAlloc(io, sandbox, allocator); + defer allocator.free(root); + + var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + try setEnv("FURIOSA_SDK_ROOT", try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{root}), 1); + + const library = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ root, "lib", "libzml_furiosa.so" }); + log.info("Loading Furiosa plugin: {s}", .{library}); + return .loadFrom(library); +} diff --git a/platforms/furiosa/libzml_furiosa.BUILD.bazel b/platforms/furiosa/libzml_furiosa.BUILD.bazel new file mode 100644 index 00000000..3a2da19a --- /dev/null +++ b/platforms/furiosa/libzml_furiosa.BUILD.bazel @@ -0,0 +1,23 @@ +load("@bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") +load("@rules_cc//cc:cc_library.bzl", "cc_library") + +# XLA owns the full payload. ZML only places it in the platform's runfiles. +copy_to_directory( + name = "sandbox", + srcs = glob( + [ + "bin/**", + "include/**", + "lib/**", + ], + allow_empty = False, + ), + add_directory_to_runfiles = True, + visibility = ["//visibility:public"], +) + +cc_library( + name = "libzml_furiosa", + data = [":sandbox"], + visibility = ["//visibility:public"], +) diff --git a/platforms/metal/BUILD.bazel b/platforms/metal/BUILD.bazel new file mode 100644 index 00000000..48bca771 --- /dev/null +++ b/platforms/metal/BUILD.bazel @@ -0,0 +1,45 @@ +load("@bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") +load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library") + +copy_to_directory( + name = "metal_toolchain", + srcs = [ + "@metal_toolchain//:metal_compile_runtime", + "@metal_toolchain//:metal_compile_tools", + ], + out = "metal_toolchain", + include_external_repositories = ["**"], + replace_prefixes = { + "Metal.xctoolchain/usr/metal/32023": ".", + }, + target_compatible_with = ["@platforms//os:macos"], +) + +cc_library( + name = "libzml_metal", + hdrs = ["metal.h"], + data = [":metal_toolchain"], + tags = ["manual"], + deps = ["@libzml_metal"], +) + +zig_library( + name = "metal", + import_name = "platforms/metal", + main = "metal.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//pjrt", + "//platforms:options", + ] + select({ + "//platforms:metal.enabled": [ + ":libzml_metal", + "//bazel", + "//stdx", + "@rules_zig//zig/runfiles", + ], + "//conditions:default": [], + }), +) diff --git a/platforms/metal/libzml_metal.BUILD.bazel b/platforms/metal/libzml_metal.BUILD.bazel new file mode 100644 index 00000000..7bac767d --- /dev/null +++ b/platforms/metal/libzml_metal.BUILD.bazel @@ -0,0 +1,17 @@ +load("@bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") +load("@rules_cc//cc:cc_library.bzl", "cc_library") + +copy_to_directory( + name = "sandbox", + srcs = [ + "lib/libzml_metal.dylib", + ], + add_directory_to_runfiles = False, + include_external_repositories = ["**"], +) + +cc_library( + name = "libzml_metal", + data = [":sandbox"], + visibility = ["@zml//platforms/metal:__subpackages__"], +) diff --git a/platforms/metal/metal.bzl b/platforms/metal/metal.bzl new file mode 100644 index 00000000..71e1858a --- /dev/null +++ b/platforms/metal/metal.bzl @@ -0,0 +1,19 @@ +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +def _metal_impl(mctx): + http_archive( + name = "libzml_metal", + build_file = "libzml_metal.BUILD.bazel", + sha256 = "89881d1a1db336e3337a9373db2c4e94f71ddfb77619dee75c0c12e2903913ae", + url = "https://mirror.zml.ai/plugins/202611101651.363.1.d46c7471c419/zml-metal-darwin-arm64.tar.zst", + ) + + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = ["libzml_metal"], + root_module_direct_dev_deps = [], + ) + +metal_packages = module_extension( + implementation = _metal_impl, +) diff --git a/platforms/metal/metal.h b/platforms/metal/metal.h new file mode 100644 index 00000000..933e4f31 --- /dev/null +++ b/platforms/metal/metal.h @@ -0,0 +1 @@ + #include diff --git a/platforms/metal/metal.zig b/platforms/metal/metal.zig new file mode 100644 index 00000000..ddde2a30 --- /dev/null +++ b/platforms/metal/metal.zig @@ -0,0 +1,53 @@ +const std = @import("std"); +const builtin = @import("builtin"); + +const bazel = @import("bazel"); +const bazel_builtin = @import("bazel_builtin"); +const c = @import("c"); +const pjrt = @import("pjrt"); +const platforms_options = @import("platforms/options"); +const stdx = @import("stdx"); + +const log = std.log.scoped(.@"zml/platforms/metal"); + +pub fn isEnabled() bool { + return platforms_options.metal_enabled; +} + +fn setMetalToolchainEnv(r: anytype) !void { + var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const root = try r.rlocation("zml/platforms/metal/metal_toolchain", &buf) orelse + return error.ToolchainNotInRunfiles; + + var bindir_z: [std.Io.Dir.max_path_bytes]u8 = undefined; + _ = c.setenv("METAL_TOOLCHAIN", try stdx.Io.Dir.path.bufJoinZ(&bindir_z, &.{ root, "bin" }), 0); +} + +pub fn load(_: std.mem.Allocator, io: std.Io) !*const pjrt.Api { + _ = io; + if (comptime !isEnabled()) { + return error.Unavailable; + } + + if (comptime builtin.os.tag != .macos) { + return error.Unavailable; + } + + const r = try bazel.runfiles(bazel_builtin.current_repository); + + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const sandbox_path = try r.rlocation("libzml_metal/sandbox", &path_buf) orelse { + log.err("Failed to find sandbox path for Metal runtime", .{}); + return error.FileNotFound; + }; + + setMetalToolchainEnv(r) catch |err| { + log.err("Bundled Metal toolchain not found in runfiles ({any})", .{err}); + }; + + return blk: { + var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libzml_metal.dylib" }); + break :blk pjrt.Api.loadFrom(path); + }; +} diff --git a/platforms/metal/metal_toolchain.BUILD.bazel b/platforms/metal/metal_toolchain.BUILD.bazel new file mode 100644 index 00000000..ca70fff0 --- /dev/null +++ b/platforms/metal/metal_toolchain.BUILD.bazel @@ -0,0 +1,382 @@ +package(default_visibility = ["//visibility:public"]) + +# Static BUILD file for the 17B54 MetalToolchain asset. + +filegroup( + name = "toolchain_runtime", + srcs = glob(["Metal.xctoolchain/**"]), +) + +exports_files([ + "Metal.xctoolchain/ToolchainInfo.plist", + "Metal.xctoolchain/usr/bin/air-ar", + "Metal.xctoolchain/usr/bin/air-arch", + "Metal.xctoolchain/usr/bin/air-as", + "Metal.xctoolchain/usr/bin/air-binary-perf", + "Metal.xctoolchain/usr/bin/air-config", + "Metal.xctoolchain/usr/bin/air-dsymutil", + "Metal.xctoolchain/usr/bin/air-libtool", + "Metal.xctoolchain/usr/bin/air-link", + "Metal.xctoolchain/usr/bin/air-lipo", + "Metal.xctoolchain/usr/bin/air-lld", + "Metal.xctoolchain/usr/bin/air-nm", + "Metal.xctoolchain/usr/bin/air-nt", + "Metal.xctoolchain/usr/bin/air-objcopy", + "Metal.xctoolchain/usr/bin/air-objdump", + "Metal.xctoolchain/usr/bin/air-opt", + "Metal.xctoolchain/usr/bin/air-pack", + "Metal.xctoolchain/usr/bin/air-package", + "Metal.xctoolchain/usr/bin/air-ranlib", + "Metal.xctoolchain/usr/bin/air-readobj", + "Metal.xctoolchain/usr/bin/air-size", + "Metal.xctoolchain/usr/bin/air-strip", + "Metal.xctoolchain/usr/bin/air-tt", + "Metal.xctoolchain/usr/bin/air-vtool", + "Metal.xctoolchain/usr/bin/amdgpu-nt", + "Metal.xctoolchain/usr/bin/applegpu-nt", + "Metal.xctoolchain/usr/bin/intelgpu-nt", + "Metal.xctoolchain/usr/bin/metal", + "Metal.xctoolchain/usr/bin/metal-ar", + "Metal.xctoolchain/usr/bin/metal-arch", + "Metal.xctoolchain/usr/bin/metal-as", + "Metal.xctoolchain/usr/bin/metal-binary-perf", + "Metal.xctoolchain/usr/bin/metal-config", + "Metal.xctoolchain/usr/bin/metal-dsymutil", + "Metal.xctoolchain/usr/bin/metal-libtool", + "Metal.xctoolchain/usr/bin/metal-link", + "Metal.xctoolchain/usr/bin/metal-lipo", + "Metal.xctoolchain/usr/bin/metal-lld", + "Metal.xctoolchain/usr/bin/metal-nm", + "Metal.xctoolchain/usr/bin/metal-nt", + "Metal.xctoolchain/usr/bin/metal-objcopy", + "Metal.xctoolchain/usr/bin/metal-objdump", + "Metal.xctoolchain/usr/bin/metal-opt", + "Metal.xctoolchain/usr/bin/metal-pack", + "Metal.xctoolchain/usr/bin/metal-package", + "Metal.xctoolchain/usr/bin/metal-ranlib", + "Metal.xctoolchain/usr/bin/metal-readobj", + "Metal.xctoolchain/usr/bin/metal-size", + "Metal.xctoolchain/usr/bin/metal-source", + "Metal.xctoolchain/usr/bin/metal-strip", + "Metal.xctoolchain/usr/bin/metal-tt", + "Metal.xctoolchain/usr/bin/metal-vtool", + "Metal.xctoolchain/usr/bin/metallib", + "Metal.xctoolchain/usr/metal/32023/lib/AMDShared.bundle/Contents/PlugIns/libAMDIL902.dylib", + "Metal.xctoolchain/usr/metal/32023/lib/AMDShared.bundle/Contents/PlugIns/libGCNASM.dylib", + "Metal.xctoolchain/usr/metal/32023/lib/AMDShared.bundle/Contents/PlugIns/libSC.dylib", + "Metal.xctoolchain/usr/metal/32023/lib/amd_13/AMDShared.bundle/Contents/PlugIns/libAMDIL902.dylib", + "Metal.xctoolchain/usr/metal/32023/lib/amd_13/AMDShared.bundle/Contents/PlugIns/libGCNASM.dylib", + "Metal.xctoolchain/usr/metal/32023/lib/amd_13/AMDShared.bundle/Contents/PlugIns/libSC.dylib", + "Metal.xctoolchain/usr/metal/32023/lib/amd_13/libAMDNTPlugin.dylib", + "Metal.xctoolchain/usr/metal/32023/lib/clang/32023.830/lib/darwin/libtracepoint_rt_ios.metallib", + "Metal.xctoolchain/usr/metal/32023/lib/clang/32023.830/lib/darwin/libtracepoint_rt_iosmac.metallib", + "Metal.xctoolchain/usr/metal/32023/lib/clang/32023.830/lib/darwin/libtracepoint_rt_iossim.metallib", + "Metal.xctoolchain/usr/metal/32023/lib/clang/32023.830/lib/darwin/libtracepoint_rt_osx.metallib", + "Metal.xctoolchain/usr/metal/32023/lib/clang/32023.830/lib/darwin/libtracepoint_rt_tvos.metallib", + "Metal.xctoolchain/usr/metal/32023/lib/clang/32023.830/lib/darwin/libtracepoint_rt_tvossim.metallib", + "Metal.xctoolchain/usr/metal/32023/lib/clang/32023.830/lib/darwin/libtracepoint_rt_watchos.metallib", + "Metal.xctoolchain/usr/metal/32023/lib/clang/32023.830/lib/darwin/libtracepoint_rt_watchossim.metallib", + "Metal.xctoolchain/usr/metal/32023/lib/clang/32023.830/lib/darwin/libtracepoint_rt_xros.metallib", + "Metal.xctoolchain/usr/metal/32023/lib/clang/32023.830/lib/darwin/libtracepoint_rt_xrossim.metallib", + "Metal.xctoolchain/usr/metal/32023/lib/libAMDNTPlugin.dylib", + "Metal.xctoolchain/usr/metal/32023/lib/libMTLIGCCompilerPlugin.dylib", + "Metal.xctoolchain/usr/metal/32023/lib/libMTLPasses.dylib", + "Metal.xctoolchain/usr/metal/32023/lib/libapplegpu-nt.dylib", + "Metal.xctoolchain/usr/metal/32023/lib/libapplegpu23-nt.dylib", + "Metal.xctoolchain/usr/metal/32023/lib/libapplegpu24-nt.dylib", + "Metal.xctoolchain/usr/metal/32023/lib/libapplegpuG9G12-nt.dylib", + "Metal.xctoolchain/usr/metal/32023/lib/libigc.dylib", + "Metal.xctoolchain/usr/metal/32023/lib/metalfe/libmetal.dylib", + "Metal.xctoolchain/usr/metal/32023/lib/runtime.gpuencode.split", + "Metal.xctoolchain/usr/metal/32023/lib/runtime.igc64.split", + "Metal.xctoolchain/usr/metal/32023/lib/wrapper-nt.dylib", +]) + +alias(name = "toolchain_info", actual = "Metal.xctoolchain/ToolchainInfo.plist") + +alias( + name = "air-ar", + actual = "Metal.xctoolchain/usr/bin/air-ar", +) + +alias( + name = "air-arch", + actual = "Metal.xctoolchain/usr/bin/air-arch", +) + +alias( + name = "air-as", + actual = "Metal.xctoolchain/usr/bin/air-as", +) + +alias( + name = "air-binary-perf", + actual = "Metal.xctoolchain/usr/bin/air-binary-perf", +) + +alias( + name = "air-config", + actual = "Metal.xctoolchain/usr/bin/air-config", +) + +alias( + name = "air-dsymutil", + actual = "Metal.xctoolchain/usr/bin/air-dsymutil", +) + +alias( + name = "air-libtool", + actual = "Metal.xctoolchain/usr/bin/air-libtool", +) + +alias( + name = "air-link", + actual = "Metal.xctoolchain/usr/bin/air-link", +) + +alias( + name = "air-lipo", + actual = "Metal.xctoolchain/usr/bin/air-lipo", +) + +alias( + name = "air-lld", + actual = "Metal.xctoolchain/usr/bin/air-lld", +) + +alias( + name = "air-nm", + actual = "Metal.xctoolchain/usr/bin/air-nm", +) + +alias( + name = "air-nt", + actual = "Metal.xctoolchain/usr/bin/air-nt", +) + +alias( + name = "air-objcopy", + actual = "Metal.xctoolchain/usr/bin/air-objcopy", +) + +alias( + name = "air-objdump", + actual = "Metal.xctoolchain/usr/bin/air-objdump", +) + +alias( + name = "air-opt", + actual = "Metal.xctoolchain/usr/bin/air-opt", +) + +alias( + name = "air-pack", + actual = "Metal.xctoolchain/usr/bin/air-pack", +) + +alias( + name = "air-package", + actual = "Metal.xctoolchain/usr/bin/air-package", +) + +alias( + name = "air-ranlib", + actual = "Metal.xctoolchain/usr/bin/air-ranlib", +) + +alias( + name = "air-readobj", + actual = "Metal.xctoolchain/usr/bin/air-readobj", +) + +alias( + name = "air-size", + actual = "Metal.xctoolchain/usr/bin/air-size", +) + +alias( + name = "air-strip", + actual = "Metal.xctoolchain/usr/bin/air-strip", +) + +alias( + name = "air-tt", + actual = "Metal.xctoolchain/usr/bin/air-tt", +) + +alias( + name = "air-vtool", + actual = "Metal.xctoolchain/usr/bin/air-vtool", +) + +alias( + name = "amdgpu-nt", + actual = "Metal.xctoolchain/usr/bin/amdgpu-nt", +) + +alias( + name = "applegpu-nt", + actual = "Metal.xctoolchain/usr/bin/applegpu-nt", +) + +alias( + name = "intelgpu-nt", + actual = "Metal.xctoolchain/usr/bin/intelgpu-nt", +) + +alias( + name = "metal", + actual = "Metal.xctoolchain/usr/bin/metal", +) + +alias( + name = "metal-ar", + actual = "Metal.xctoolchain/usr/bin/metal-ar", +) + +alias( + name = "metal-arch", + actual = "Metal.xctoolchain/usr/bin/metal-arch", +) + +alias( + name = "metal-as", + actual = "Metal.xctoolchain/usr/bin/metal-as", +) + +alias( + name = "metal-binary-perf", + actual = "Metal.xctoolchain/usr/bin/metal-binary-perf", +) + +alias( + name = "metal-config", + actual = "Metal.xctoolchain/usr/bin/metal-config", +) + +alias( + name = "metal-dsymutil", + actual = "Metal.xctoolchain/usr/bin/metal-dsymutil", +) + +alias( + name = "metal-libtool", + actual = "Metal.xctoolchain/usr/bin/metal-libtool", +) + +alias( + name = "metal-link", + actual = "Metal.xctoolchain/usr/bin/metal-link", +) + +alias( + name = "metal-lipo", + actual = "Metal.xctoolchain/usr/bin/metal-lipo", +) + +alias( + name = "metal-lld", + actual = "Metal.xctoolchain/usr/bin/metal-lld", +) + +alias( + name = "metal-nm", + actual = "Metal.xctoolchain/usr/bin/metal-nm", +) + +alias( + name = "metal-nt", + actual = "Metal.xctoolchain/usr/bin/metal-nt", +) + +alias( + name = "metal-objcopy", + actual = "Metal.xctoolchain/usr/bin/metal-objcopy", +) + +alias( + name = "metal-objdump", + actual = "Metal.xctoolchain/usr/bin/metal-objdump", +) + +alias( + name = "metal-opt", + actual = "Metal.xctoolchain/usr/bin/metal-opt", +) + +alias( + name = "metal-pack", + actual = "Metal.xctoolchain/usr/bin/metal-pack", +) + +alias( + name = "metal-package", + actual = "Metal.xctoolchain/usr/bin/metal-package", +) + +alias( + name = "metal-ranlib", + actual = "Metal.xctoolchain/usr/bin/metal-ranlib", +) + +alias( + name = "metal-readobj", + actual = "Metal.xctoolchain/usr/bin/metal-readobj", +) + +alias( + name = "metal-size", + actual = "Metal.xctoolchain/usr/bin/metal-size", +) + +alias( + name = "metal-source", + actual = "Metal.xctoolchain/usr/bin/metal-source", +) + +alias( + name = "metal-strip", + actual = "Metal.xctoolchain/usr/bin/metal-strip", +) + +alias( + name = "metal-tt", + actual = "Metal.xctoolchain/usr/bin/metal-tt", +) + +alias( + name = "metal-vtool", + actual = "Metal.xctoolchain/usr/bin/metal-vtool", +) + +alias( + name = "metallib", + actual = "Metal.xctoolchain/usr/bin/metallib", +) + +filegroup( + name = "metal_headers", + srcs = glob(["Metal.xctoolchain/usr/metal/*/lib/clang/*/include/**/*.h"]), +) + +filegroup( + name = "tracepoint_runtime_metallibs", + srcs = glob(["Metal.xctoolchain/usr/metal/*/lib/clang/*/lib/darwin/*.metallib"]), +) + +filegroup( + name = "metal_compile_tools", + srcs = [ + "Metal.xctoolchain/usr/metal/32023/bin/metal", + "Metal.xctoolchain/usr/metal/32023/bin/air-opt", + "Metal.xctoolchain/usr/metal/32023/bin/air-as", + "Metal.xctoolchain/usr/metal/32023/bin/metallib", + ], +) + +filegroup( + name = "metal_compile_runtime", + srcs = glob([ + "Metal.xctoolchain/usr/metal/32023/lib/clang/32023.830/**", + ]), +) diff --git a/platforms/neuron/BUILD.bazel b/platforms/neuron/BUILD.bazel new file mode 100644 index 00000000..ad537191 --- /dev/null +++ b/platforms/neuron/BUILD.bazel @@ -0,0 +1,192 @@ +load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("@rules_python//python/uv:lock.bzl", uv_lock = "lock") +load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library", "zig_shared_library") +load("@zml//bazel:cc_import.bzl", "cc_import") +load("@zml//bazel:runfiles.bzl", "runfiles_to_default") + +exports_files([ + "neuronx_cc_shim.py", + "nki_kernel_compiler.py", +]) + +# A proxy PJRT Plugin that loads the Neuron PJRT Plugin +# and returns the instance from nested GetPjrtApi. +# +# The proxy also initializes the Neuron runtime environment and exposes the +# sandboxed compiler binaries to libneuronxla 3.0. +zig_shared_library( + name = "libzml_neuron", + srcs = ["neuron.zig"], + main = "libzml_neuron.zig", + target_compatible_with = ["@platforms//os:linux"], + visibility = ["@libzml_neuron//:__subpackages__"], + zigopts = ["-fno-stack-check"], + deps = [ + ":libnrt_headers", + ":neuron_nrt", + "//bazel", + "//pjrt", + "//platforms:options", + "//stdx", + "@rules_zig//zig/runfiles", + "@xla//xla/pjrt/c:pjrt_c_api_hdrs", + ], +) + +zig_binary( + name = "neuronx-cc", + data = ["@neuron_py_deps//neuronx_cc"], + linkopts = ["-Wl,-rpath,$$ORIGIN/../lib"], + main = "neuronxcc_compiler_launcher.zig", + tags = ["manual"], + target_compatible_with = ["@platforms//os:linux"], + deps = [ + ":libpython", + ":python_launcher", + ], +) + +runfiles_to_default( + name = "neuronx-cc_files", + target_compatible_with = ["@platforms//os:linux"], + visibility = ["@libzml_neuron//:__subpackages__"], + deps = [":neuronx-cc"], +) + +runfiles_to_default( + name = "nki-cc_files", + target_compatible_with = ["@platforms//os:linux"], + visibility = ["@libzml_neuron//:__subpackages__"], + deps = [":nki-cc"], +) + +cc_library( + name = "libpython", + hdrs = ["libpython.h"], + tags = ["manual"], + visibility = ["//platforms/neuron:__subpackages__"], + deps = [ + "@rules_python//python/cc:current_py_cc_headers", + "@rules_python//python/cc:current_py_cc_libs", + ], +) + +cc_library( + name = "libnrt_headers", + hdrs = ["nrt.h"], + tags = ["manual"], + visibility = [ + "//bin/zml-smi:__subpackages__", + ], + deps = ["@libzml_neuron//:libnrt_headers"], +) + +cc_import( + name = "neuron_ncfw", + shared_library = "@aws-neuronx-runtime-lib//:libncfw.patchelf", + tags = ["manual"], + target_compatible_with = [ + "@platforms//cpu:x86_64", + "@platforms//os:linux", + ], +) + +cc_import( + name = "neuron_nrt", + shared_library = "@aws-neuronx-runtime-lib//:libnrt.patchelf", + tags = ["manual"], + target_compatible_with = [ + "@platforms//cpu:x86_64", + "@platforms//os:linux", + ], + deps = [":neuron_ncfw"], +) + +zig_library( + name = "python_launcher", + import_name = "platforms/neuron/python_launcher", + main = "python_launcher.zig", + tags = ["manual"], + visibility = ["//platforms/neuron:__subpackages__"], + deps = [":libpython"], +) + +zig_binary( + name = "nki-cc", + data = [ + "nki_kernel_compiler.py", + "@neuron_py_deps//neuronx_cc", + "@neuron_py_deps//nki", + ], + linkopts = ["-Wl,-rpath,$$ORIGIN/../lib"], + main = "nki_compiler_launcher.zig", + tags = ["manual"], + target_compatible_with = ["@platforms//os:linux"], + visibility = ["//platforms/neuron:__pkg__"], + deps = [ + ":libpython", + ":python_launcher", + ], +) + +zig_library( + name = "nki_kernel", + import_name = "platforms/neuron/nki_kernel", + main = "nki_kernel.zig", + tags = ["manual"], + visibility = ["//zml:__pkg__"], + deps = [ + ":neuron", + "//bazel", + "//stdx", + "@rules_zig//zig/runfiles", + ], +) + +uv_lock( + name = "requirements", + srcs = ["requirements.in"], + out = "requirements.lock.txt", + args = [ + "--emit-index-url", + "--emit-find-links", + "--index-strategy=unsafe-best-match", + "--upgrade", + "--python-platform=x86_64-unknown-linux-gnu", + ], + tags = ["manual"], +) + +zig_library( + name = "neuron", + import_name = "platforms/neuron", + main = "neuron.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//pjrt", + "//platforms:options", + ] + select({ + "//platforms:neuron.enabled": [ + ":libnrt_headers", + ":neuron_nrt", + "//bazel", + "//stdx", + "@libzml_neuron", + "@rules_zig//zig/runfiles", + ], + "//conditions:default": [], + }), +) + +zig_library( + name = "topology", + data = [":libzml_neuron"], + import_name = "platforms/neuron/topology", + main = "topology.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + ":neuron", + ], +) diff --git a/platforms/neuron/libpython.h b/platforms/neuron/libpython.h new file mode 100644 index 00000000..605f9fc5 --- /dev/null +++ b/platforms/neuron/libpython.h @@ -0,0 +1,2 @@ +#define PY_SSIZE_T_CLEAN +#include diff --git a/platforms/neuron/libzml_neuron.BUILD.bazel b/platforms/neuron/libzml_neuron.BUILD.bazel new file mode 100644 index 00000000..8d8a7119 --- /dev/null +++ b/platforms/neuron/libzml_neuron.BUILD.bazel @@ -0,0 +1,136 @@ +load("@bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") +load("@bazel_skylib//rules:select_file.bzl", "select_file") +load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("@zml//bazel:patchelf.bzl", "patchelf") +load("@zml//bazel:runfiles.bzl", "runfiles_to_default") + +runfiles_to_default( + name = "libneuronxla_files", + deps = ["@neuron_py_deps//libneuronxla:pkg"], +) + +select_file( + name = "libneuronpjrt_so_orig", + srcs = ":libneuronxla_files", + subpath = "site-packages/libneuronxla/libneuronpjrt.so", +) + +patchelf( + name = "libneuronpjrt_so", + src = ":libneuronpjrt_so_orig", + set_rpath = "$ORIGIN", + soname = "libneuronpjrt.so", +) + +patchelf( + name = "libzml_neuron_so", + src = "@zml//platforms/neuron:libzml_neuron", + set_rpath = "$ORIGIN", + soname = "libzml_neuron.so", +) + +patchelf( + name = "neuron_explorer", + src = "@aws-neuronx-tools//:neuron-explorer", + add_needed = ["libzml_neuron.so"], + set_rpath = "$ORIGIN/../lib", +) + +# This is a compiler-enabled sandbox. Python stays packaged because AWS exposes +# neuronx-cc and NKI compilation through Python entrypoints; ZML wraps those as +# native ELF launchers in bin/ so the image does not need shell scripts or host +# Python. Keep AWS compiler/runtime payloads intact and only trim Python +# files that are not part of Neuron compilation. +copy_to_directory( + name = "sandbox", + srcs = [ + ":libneuronpjrt_so", + ":libneuronxla_files", + ":libzml_neuron_so", + ":neuron_explorer", + "@aws-neuronx-collectives//:libnccom", + "@aws-neuronx-runtime-lib//:libncfw.patchelf", + "@aws-neuronx-runtime-lib//:libnrt.patchelf", + "@aws-neuronx-runtime-lib//:libnrtucode_extisa", + "@libgcc_s1", + "@libgomp1", + "@libstdcpp6", + "@rules_python//python:current_py_toolchain", + "@zlib1g", + "@zml//platforms/neuron:neuronx-cc_files", + "@zml//platforms/neuron:neuronx_cc_shim.py", + "@zml//platforms/neuron:nki-cc_files", + "@zml//platforms/neuron:nki_kernel_compiler.py", + ], + add_directory_to_runfiles = False, + allow_overwrites = True, + exclude_srcs_patterns = [ + "**/__pycache__/**", + "**/*.pyc", + "**/tests/**", + "lib/libtcl*.so*", + "lib/libtk*.so*", + "lib/python*/idlelib/**", + "lib/python*/tkinter/**", + "lib/python*/turtledemo/**", + "lib/python*/ensurepip/**", + # "**/include/**", + "**/_solib_*/**", + ], + hardlink = "on", + include_external_repositories = ["**"], + include_srcs_patterns = [ + "platforms/neuron/neuronx-cc", + "platforms/neuron/neuronx_cc_shim.py", + "platforms/neuron/nki-cc", + "platforms/neuron/nki_kernel_compiler.py", + "neuron_explorer/**", + "**/lib**", + "lib/python*/**", + "site-packages/**", + ], + replace_prefixes = { + "platforms/neuron/neuronx-cc": "bin/neuronx-cc", + "platforms/neuron/neuronx_cc_shim.py": "bin/python-shims/neuronx-cc", + "platforms/neuron/nki-cc": "bin/nki-cc", + "platforms/neuron/nki_kernel_compiler.py": "site-packages/nki_kernel_compiler.py", + "neuron_explorer": "bin", + "libneuronpjrt_so": "lib", + "libzml_neuron_so": "lib", + "libnrt.patchelf": "lib", + "libncfw.patchelf": "lib", + "libnrtucode_extisa": "lib", + "lib/x86_64-linux-gnu": "lib", + "usr/lib/x86_64-linux-gnu": "lib", + "opt/neuron": "lib", + }, + visibility = [ + "@zml//tools/neuron:__pkg__", + "@zml//zml:__pkg__", + "@zml//zml/kernels/flash_attention/nki:__pkg__", + ], +) + +cc_library( + name = "libnrt_headers", + linkopts = [ + # Defer function call resolution until the function is called + # (lazy loading) rather than at load time. + # + # This is required because we want to let downstream use weak NRT symbols. + # + # We force it here because -z,now (which resolve all symbols at load time), + # is the default in most bazel CC toolchains as well as in certain linkers. + "-Wl,-z,lazy", + ], + visibility = ["@zml//platforms/neuron:__subpackages__"], + deps = [ + "@aws-neuronx-runtime-lib//:libnrt_headers", + ], +) + +cc_library( + name = "libzml_neuron", + data = [":sandbox"], + visibility = ["@zml//platforms/neuron:__subpackages__"], +) diff --git a/platforms/neuron/libzml_neuron.zig b/platforms/neuron/libzml_neuron.zig new file mode 100644 index 00000000..404d8cc0 --- /dev/null +++ b/platforms/neuron/libzml_neuron.zig @@ -0,0 +1,180 @@ +const std = @import("std"); + +const bazel_builtin = @import("bazel_builtin"); +const c = @import("c"); +const neuron = @import("neuron.zig"); +const runfiles = @import("runfiles"); +const stdx = @import("stdx"); + +const log = std.log.scoped(.@"zml/platforms/neuron"); + +fn findFreeTcpPort(io: std.Io) !u16 { + const addr: std.Io.net.IpAddress = .{ .ip4 = .{ .bytes = .{ 127, 0, 0, 1 }, .port = 0 } }; + var server = try std.Io.net.IpAddress.listen( + &addr, + io, + .{}, + ); + defer server.deinit(io); + return server.socket.address.getPort(); +} + +pub export fn zmlxneuron_dlopen(filename: [*c]const u8, flags: c_int) ?*anyopaque { + const replacements: std.StaticStringMap([:0]const u8) = .initComptime(.{ + .{ "libnccom.so", "libnccom.so.2" }, + .{ "libnrt.so", "libnrt.so.1" }, + .{ "libncfw.so", "libncfw.so.2" }, + }); + + var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const new_filename: [*c]const u8 = if (filename) |f| blk: { + const replacement = replacements.get(std.Io.Dir.path.basename(std.mem.span(f))) orelse break :blk f; + break :blk stdx.Io.Dir.path.bufJoinZ(&buf, &.{ + stdx.process.selfSharedObjectDirPath(), + replacement, + }) catch unreachable; + } else null; + + return std.c.dlopen(new_filename, @bitCast(flags)); +} + +extern fn setenv(name: [*:0]const u8, value: [*:0]const u8, overwrite: c_int) c_int; + +fn setupNeuronEnv(io: std.Io, sandbox_path: []const u8) !void { + const scratchpad_page_size = "1024"; + + var root_comm_buf: [256]u8 = undefined; + _ = setenv( + "NEURON_RT_ROOT_COMM_ID", + try std.mem.printSentinel(&root_comm_buf, "127.0.0.1:{d}", .{try findFreeTcpPort(io)}, 0), + 1, + ); + _ = setenv( + "NEURON_INTERNAL_PJRT_C_API_VERSION", + std.fmt.comptimePrint("{d}.{d}", .{ + c.PJRT_API_MAJOR, + c.PJRT_API_MINOR, + }), + 1, + ); + _ = setenv("NEURON_RT_LOG_LEVEL", "error", 0); + _ = setenv("NEURON_RT_DISABLE_EXECUTION_BARRIER", "1", 0); + _ = setenv("NEURON_RT_STOCHASTIC_ROUNDING_EN", "1", 1); + _ = setenv("NEURON_RT_ASYNC_EXEC_MAX_INFLIGHT_REQUESTS", "3", 0); + _ = setenv("NEURON_SCRATCHPAD_PAGE_SIZE", scratchpad_page_size, 0); + + var bin_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const bin_path = try stdx.Io.Dir.path.bufJoin(&bin_path_buf, &.{ sandbox_path, "bin" }); + + const old_path = if (std.c.getenv("PATH")) |path| std.mem.span(path) else ""; + var new_path_buf: [std.posix.PATH_MAX:0]u8 = undefined; + const new_path = if (old_path.len == 0) + try std.mem.printSentinel(&new_path_buf, "{s}", .{bin_path}, 0) + else + try std.mem.printSentinel(&new_path_buf, "{s}:{s}", .{ bin_path, old_path }, 0); + if (setenv("PATH", new_path.ptr, 1) != 0) return error.SetEnvFailed; + + if (std.c.getenv("NEURON_CC_FLAGS") != null) return; + + const log_level = if (std.c.getenv("NEURON_RT_LOG_LEVEL")) |level| std.mem.span(level) else "error"; + + var flags_buf: [512:0]u8 = undefined; + const instance = try neuron.instance(); + const target = @tagName(instance.compilerTarget()); + const flags = try std.mem.printSentinel( + &flags_buf, + "--target={s} " ++ + "--optlevel=1 " ++ + "--model-type=transformer " ++ + "--enable-fast-loading-neuron-binaries " ++ + "--enable-fast-context-switch " ++ + "--hbm-scratchpad-page-size={s} " ++ + "--verbose {s} " ++ + "--logfile-verbose {s} " ++ + "--logfile=./log-neuron-cc.txt", + .{ + target, + scratchpad_page_size, + log_level, + log_level, + }, + 0, + ); + + _ = setenv("NEURON_CC_FLAGS", flags.ptr, 0); +} + +// Duplicates a PJRT Api object while being careful about struct size differences +fn dupePjrtApi(api: *c.PJRT_Api) c.PJRT_Api { + var ret: c.PJRT_Api = undefined; + const struct_size = @min(@sizeOf(c.PJRT_Api), api.struct_size); + @memcpy( + std.mem.asBytes(&ret)[0..struct_size], + std.mem.asBytes(api)[0..struct_size], + ); + return ret; +} + +fn getPjrtApi() !*c.PJRT_Api { + var threaded: std.Io.Threaded = .init(std.heap.c_allocator, .{}); + defer threaded.deinit(); + const io = threaded.io(); + + const Static = struct { + var inner: *c.PJRT_Api = undefined; + var proxy: c.PJRT_Api = undefined; + }; + + var sandbox_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const sandbox_path = try stdx.Io.Dir.path.bufJoin(&sandbox_path_buf, &.{ + stdx.process.selfSharedObjectDirPath(), + "..", + }); + + try setupNeuronEnv(io, sandbox_path); + + Static.inner = blk: { + const GetPjrtApi_inner = GetPjrtApi_blk: { + var lib: std.DynLib = .{ + .inner = .{ + .handle = handle_blk: { + var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const library = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libneuronpjrt.so" }); + break :handle_blk std.c.dlopen(library, .{ .LAZY = true, .NODELETE = true }) orelse { + log.err("Unable to dlopen plugin: {?s}", .{std.mem.span(std.c.dlerror())}); + return error.DlOpenFailed; + }; + }, + }, + }; + + break :GetPjrtApi_blk lib.lookup(*const fn () callconv(.c) *c.PJRT_Api, "GetPjrtApi") orelse { + log.err("Unable to find symbol GetPjrtApi in plugin: {?s}", .{std.mem.span(std.c.dlerror())}); + return error.SymbolNotFound; + }; + }; + + break :blk GetPjrtApi_inner(); + }; + + Static.proxy = dupePjrtApi(Static.inner); + // Setup the API proxy functions + // Static.proxy.PJRT_Plugin_Attributes = &struct { + // const STRUCT_SIZE = 24; // according to the failing assertion + + // fn call(args: [*c]c.PJRT_Plugin_Attributes_Args) callconv(.c) ?*c.PJRT_Error { + // var new_args = args.*; + // new_args.struct_size = @min(new_args.struct_size, STRUCT_SIZE); + // return Static.inner.PJRT_Plugin_Attributes.?(&new_args); + // } + // }.call; + + return &Static.proxy; +} + +pub export fn GetPjrtApi() ?*c.PJRT_Api { + return getPjrtApi() catch |err| { + log.err("Failed to get PJRT API: {}", .{err}); + return null; + }; +} diff --git a/platforms/neuron/neuron.bzl b/platforms/neuron/neuron.bzl new file mode 100644 index 00000000..8b947c04 --- /dev/null +++ b/platforms/neuron/neuron.bzl @@ -0,0 +1,134 @@ +load("@python_versions//3.12:defs.bzl", _py_binary = "py_binary") +load("@rules_python//python:defs.bzl", "PyInfo") +load("@with_cfg.bzl", "with_cfg") +load("//bazel:http_deb_archive.bzl", "http_deb_archive") +load("//bazel:simple_repository.bzl", "simple_repository") +load("//platforms:packages.bzl", "packages") + +BASE_URL = "https://apt.repos.neuron.amazonaws.com" +STRIP_PREFIX = "./opt/aws/neuron" + +_BUILD_FILE_PRELUDE = """\ +package(default_visibility = ["//visibility:public"]) + +load("@rules_cc//cc:cc_library.bzl", "cc_library") +""" + +_UBUNTU_PACKAGES = { + "libgcc_s1": { + "package": "libgcc-s1", + "build": packages.filegroup(name = "libgcc_s1", srcs = ["lib/x86_64-linux-gnu/libgcc_s.so.1"]), + }, + "libgomp1": { + "package": "libgomp1", + "build": packages.filegroup(name = "libgomp1", srcs = ["usr/lib/x86_64-linux-gnu/libgomp.so.1"]), + }, + "libstdcpp6": { + "package": "libstdc++6", + "build": packages.filegroup(name = "libstdcpp6", srcs = ["usr/lib/x86_64-linux-gnu/libstdc++.so.6"]), + }, + "zlib1g": { + "package": "zlib1g", + "build": packages.filegroup(name = "zlib1g", srcs = ["lib/x86_64-linux-gnu/libz.so.1"]), + }, +} + +_NEURON_PACKAGES = { + "aws-neuronx-runtime-lib": "\n".join([ + packages.load_("@zml//bazel:patchelf.bzl", "patchelf"), + packages.cc_library_hdrs_glob( + name = "libnrt_headers", + hdrs_glob = [ + "include/ndl/**/*.h", + "include/nrt/**/*.h", + ], + includes = ["include"], + visibility = ["//visibility:public"], + ), + packages.patchelf( + name = "libnrt.patchelf", + src = "lib/libnrt.so.1", + set_rpath = "$ORIGIN", + add_needed = [ + # readelf -d ./opt/aws/neuron/libl/libncfw.so + "libncfw.so.2", + ], + rename_dynamic_symbols = { + "dlopen": "zmlxneuron_dlopen", + }, + ), + packages.patchelf( + name = "libncfw.patchelf", + src = "lib/libncfw.so", + soname = "libncfw.so.2", + ), + packages.filegroup( + name = "libnrtucode_extisa", + srcs = ["lib/libnrtucode_extisa.so"], + ), + ]), + "aws-neuronx-collectives": "\n".join([ + packages.filegroup( + name = "libnccom", + srcs = ["lib/libnccom.so.2"], + ), + ]), + "aws-neuronx-tools": "\n".join([ + packages.filegroup( + name = "neuron-explorer", + srcs = ["bin/neuron-explorer"], + ), + ]), +} + +def _neuron_impl(mctx): + loaded_packages = packages.read(mctx, [ + "@zml//platforms/neuron:packages.lock.json", + ]) + + simple_repository( + name = "libzml_neuron", + build_file = ":libzml_neuron.BUILD.bazel", + ) + + for repo_name, package in _UBUNTU_PACKAGES.items(): + pkg = loaded_packages[package["package"]]["amd64"] + http_deb_archive( + name = repo_name, + urls = pkg["urls"], + sha256 = pkg["sha256"], + build_file_content = _BUILD_FILE_PRELUDE + package["build"], + ) + + for pkg_name, build_file_content in _NEURON_PACKAGES.items(): + pkg = loaded_packages[pkg_name]["amd64"] + http_deb_archive( + name = pkg_name, + urls = pkg["urls"], + sha256 = pkg["sha256"], + strip_prefix = STRIP_PREFIX, + build_file_content = _BUILD_FILE_PRELUDE + build_file_content, + ) + + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = [ + "aws-neuronx-collectives", + "aws-neuronx-runtime-lib", + "aws-neuronx-tools", + "libgcc_s1", + "libgomp1", + "libstdcpp6", + "libzml_neuron", + ], + root_module_direct_dev_deps = [], + ) + +neuron_packages = module_extension( + implementation = _neuron_impl, +) + +py_binary_with_script, _py_binary_internal = with_cfg( + kind = _py_binary, + extra_providers = [PyInfo], +).build() diff --git a/platforms/neuron/neuron.zig b/platforms/neuron/neuron.zig new file mode 100644 index 00000000..e0365fe9 --- /dev/null +++ b/platforms/neuron/neuron.zig @@ -0,0 +1,150 @@ +const std = @import("std"); +const builtin = @import("builtin"); + +const bazel = @import("bazel"); +const bazel_builtin = @import("bazel_builtin"); +const c = @import("c"); +const pjrt = @import("pjrt"); +const platforms_options = @import("platforms/options"); +const stdx = @import("stdx"); + +const log = std.log.scoped(.@"zml/platforms/neuron"); + +pub const CompilerTarget = enum { + trn1, + inf2, + trn1n, + trn2, + trn2n, + trn3, +}; + +pub const Family = enum(u32) { + unknown = 0, + inf1 = 1, + trn1 = 2, + trn1n = 3, + inf2 = 4, + trn2 = 5, + trn2n = 6, + inf2e = 7, + trn2p = 8, + trn2u = 9, + trn2e = 10, + trn2eu = 11, + trn2ac = 12, + trn2uac = 13, + trn3 = 14, + trn3pds98 = 15, + + pub fn compilerTarget(self: Family) CompilerTarget { + return switch (self) { + .unknown, .inf1 => unreachable, + .trn1 => .trn1, + .trn1n => .trn1n, + .inf2, .inf2e => .inf2, + .trn2, .trn2n, .trn2p, .trn2u, .trn2e, .trn2eu, .trn2ac, .trn2uac => .trn2, + .trn3, .trn3pds98 => .trn3, + }; + } +}; + +pub const Size = enum(u32) { + xl1 = 0, + xl2 = 1, + xl4 = 2, + xl6 = 3, + xl8 = 4, + xl24 = 5, + xl32 = 6, + xl48 = 7, + xl3 = 8, + unknown = 9, +}; + +pub const Instance = struct { + family: Family, + size: Size, + + pub fn compilerTarget(self: Instance) CompilerTarget { + return self.family.compilerTarget(); + } + + pub fn coresPerChip(self: Instance) !usize { + return switch (self.family) { + .inf1 => 4, + .trn1, .trn1n, .inf2, .inf2e => 2, + .trn2, .trn2n, .trn2p, .trn2u, .trn2e, .trn2eu, .trn2ac, .trn2uac, .trn3, .trn3pds98 => 8, + .unknown => unreachable, + }; + } +}; + +pub fn instance() !Instance { + var info = std.mem.zeroes(c.struct_nrt_instance_info); + info.family = @backingInt(Family.unknown); + info.size = @backingInt(Size.unknown); + + if (c.nrt_get_instance_info(&info, @sizeOf(c.struct_nrt_instance_info)) != c.NRT_SUCCESS) { + return error.Unavailable; + } + + return .{ + .family = @fromBackingInt(@intCast(info.family)), + .size = @fromBackingInt(@intCast(info.size)), + }; +} + +pub fn isEnabled() bool { + return platforms_options.neuron_enabled; +} + +fn hasNeuronDevice(io: std.Io) bool { + std.Io.Dir.accessAbsolute(io, "/dev/neuron0", .{ .read = true }) catch return false; + return true; +} + +fn isRunningOnEC2(io: std.Io) !bool { + const AmazonEC2 = "Amazon EC2"; + var buffer: [AmazonEC2.len]u8 = undefined; + return std.mem.eql( + u8, + AmazonEC2, + try std.Io.Dir.readFile( + .cwd(), + io, + "/sys/devices/virtual/dmi/id/sys_vendor", + &buffer, + ), + ); +} + +pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { + _ = allocator; + if (comptime !isEnabled()) { + return error.Unavailable; + } + if (comptime builtin.os.tag != .linux) { + return error.Unavailable; + } + if (!(isRunningOnEC2(io) catch false)) { + return error.Unavailable; + } + if (!hasNeuronDevice(io)) { + return error.Unavailable; + } + + const r = try bazel.runfiles(bazel_builtin.current_repository); + + var sandbox_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const sandbox_path = try r.rlocation("libzml_neuron/sandbox", &sandbox_path_buf) orelse { + log.err("Failed to find sandbox path for NEURON runtime", .{}); + return error.FileNotFound; + }; + + return blk: { + var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libzml_neuron.so" }); + break :blk .loadFrom(path); + }; +} diff --git a/platforms/neuron/neuronx_cc_shim.py b/platforms/neuron/neuronx_cc_shim.py new file mode 100755 index 00000000..deff0b5d --- /dev/null +++ b/platforms/neuron/neuronx_cc_shim.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python3 +"""Python-compatible shim for harness NKI standalone compilation.""" + +import os +import sys + + +def main() -> None: + native = os.path.join( + os.path.dirname(os.path.realpath(__file__)), "..", "neuronx-cc" + ) + os.execv(native, [native, *sys.argv[1:]]) + + +if __name__ == "__main__": + main() diff --git a/platforms/neuron/neuronxcc_compiler_launcher.zig b/platforms/neuron/neuronxcc_compiler_launcher.zig new file mode 100644 index 00000000..066923e2 --- /dev/null +++ b/platforms/neuron/neuronxcc_compiler_launcher.zig @@ -0,0 +1,9 @@ +const std = @import("std"); + +const python_launcher = @import("platforms/neuron/python_launcher"); + +pub fn main(init: std.process.Init) !void { + try python_launcher.runModuleEntrypoint(init, .{ + .module_name = "neuronxcc.driver.CommandDriver", + }); +} diff --git a/platforms/neuron/nki_compiler_launcher.zig b/platforms/neuron/nki_compiler_launcher.zig new file mode 100644 index 00000000..f8a6b884 --- /dev/null +++ b/platforms/neuron/nki_compiler_launcher.zig @@ -0,0 +1,12 @@ +const std = @import("std"); + +const python_launcher = @import("platforms/neuron/python_launcher"); + +pub fn main(init: std.process.Init) !void { + _ = std; + try python_launcher.runScriptMain(init, &.{ + "..", + "site-packages", + "nki_kernel_compiler.py", + }); +} diff --git a/platforms/neuron/nki_kernel.zig b/platforms/neuron/nki_kernel.zig new file mode 100644 index 00000000..4d897be8 --- /dev/null +++ b/platforms/neuron/nki_kernel.zig @@ -0,0 +1,160 @@ +const std = @import("std"); + +const bazel = @import("bazel"); +const bazel_builtin = @import("bazel_builtin"); +const stdx = @import("stdx"); + +const neuron = @import("platforms/neuron"); + +const log = std.log.scoped(.@"zml/platforms/neuron/nki_kernel"); + +pub const TensorSignature = struct { + dtype: []const u8, + dims: []const i64, +}; + +pub const Kernel = struct { + name: []const u8, + entrypoint: []const u8, + source_path: []const u8, + compiler_target: []const u8, + inputs: []const TensorSignature, + outputs: []const TensorSignature, +}; + +const CompilerRequest = struct { + entrypoint: []const u8, + source: []const u8, + target: []const u8, + tool_bin_dir: []const u8, + neuronx_cc_args: []const []const u8, + inputs: []const TensorSignature, + outputs: []const TensorSignature, +}; + +const CompilerResponse = struct { + backend_config_b64: []const u8, +}; + +pub fn compilerTargetFromInstance() neuron.CompilerTarget { + const instance = neuron.instance() catch |err| { + std.debug.panic("failed to query Neuron instance for NKI compiler target: {}", .{err}); + }; + return instance.compilerTarget(); +} + +pub fn compileNkiKernel(allocator: std.mem.Allocator, io: std.Io, kernel: Kernel) ![]const u8 { + const compiler_flags = std.mem.span(std.c.getenv("NEURON_CC_FLAGS") orelse return error.MissingNeuronCompilerFlags); + var compiler_args: std.ArrayList([]const u8) = .empty; + defer compiler_args.deinit(allocator); + var compiler_flags_iter = std.mem.tokenizeScalar(u8, compiler_flags, ' '); + while (compiler_flags_iter.next()) |arg| { + try compiler_args.append(allocator, arg); + } + + const tmp_root_path = if (std.c.getenv("TMPDIR")) |tmpdir| std.mem.span(tmpdir) else "/tmp"; + var tmp_root = try std.Io.Dir.openDir(.cwd(), io, tmp_root_path, .{}); + defer tmp_root.close(io); + + var dir_name_buf: [128]u8 = undefined; + const dir_name = try std.fmt.bufPrint(&dir_name_buf, "zml-nki-{d}", .{std.Io.Timestamp.now(io, .real).nanoseconds}); + + var tmp_dir = try tmp_root.createDirPathOpen(io, dir_name, .{ .permissions = .fromMode(0o700) }); + defer tmp_dir.close(io); + + var cwd_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const cwd = try tmp_dir.realPath(io, &cwd_buf); + + const request_file = try tmp_dir.createFile(io, "kernel.request.json", .{ .read = true }); + defer request_file.close(io); + + const result_file = try tmp_dir.createFile(io, "kernel.result.json", .{ .read = true }); + defer result_file.close(io); + + var request_file_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const request_file_path = try request_file.realPath(io, &request_file_buf); + + var result_file_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const result_file_path = try result_file.realPath(io, &result_file_buf); + + const r = try bazel.runfiles(bazel_builtin.current_repository); + + var sandbox_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const sandbox_path = try r.rlocation("libzml_neuron/sandbox", &sandbox_path_buf) orelse return error.FileNotFound; + + var source_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const source_path = try r.rlocation(kernel.source_path, &source_path_buf) orelse return error.FileNotFound; + const source = try std.Io.Dir.cwd().readFileAlloc(io, source_path, allocator, .unlimited); + defer allocator.free(source); + + var nki_cc_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const nki_cc_path = try stdx.Io.Dir.path.bufJoin(&nki_cc_buf, &.{ sandbox_path, "bin", "nki-cc" }); + + const bin_dirpath = std.fs.path.dirname(nki_cc_path) orelse return error.InvalidSandboxPath; + + const request: CompilerRequest = .{ + .entrypoint = kernel.entrypoint, + .source = source, + .target = kernel.compiler_target, + .tool_bin_dir = bin_dirpath, + .neuronx_cc_args = compiler_args.items, + .inputs = kernel.inputs, + .outputs = kernel.outputs, + }; + + const request_json = try std.fmt.allocPrint(allocator, "{f}", .{std.json.fmt(request, .{ .emit_null_optional_fields = false })}); + defer allocator.free(request_json); + + try request_file.writePositionalAll(io, request_json, 0); + + var spawn_threaded: std.Io.Threaded = .init(std.heap.c_allocator, .{}); + defer spawn_threaded.deinit(); + + const spawn_io = spawn_threaded.io(); + + var child = try std.process.spawn(spawn_io, .{ + .argv = &.{ + nki_cc_path, + "--request", + request_file_buf[0..request_file_path], + "--result", + result_file_buf[0..result_file_path], + }, + .stdin = .ignore, + .stdout = .inherit, + .stderr = .inherit, + .cwd = .{ .path = cwd_buf[0..cwd] }, + }); + + const term = try child.wait(spawn_io); + switch (term) { + .exited => |code| { + if (code != 0) { + log.err("nki-cc exited with code {}", .{code}); + return error.NkiCcFailed; + } + }, + .signal => |sig| { + log.err("nki-cc terminated by signal {}", .{sig}); + return error.NkiCcFailed; + }, + else => |status| { + log.err("nki-cc terminated unexpectedly: {}", .{status}); + return error.NkiCcFailed; + }, + } + + var result_reader = result_file.reader(io, &.{}); + const data = try result_reader.interface.readAlloc(allocator, try result_file.length(io)); + defer allocator.free(data); + + const parsed_response = try std.json.parseFromSlice(CompilerResponse, allocator, data, .{ + .ignore_unknown_fields = true, + }); + defer parsed_response.deinit(); + + const backend_config = try allocator.dupe(u8, parsed_response.value.backend_config_b64); + errdefer allocator.free(backend_config); + + return backend_config; +} diff --git a/platforms/neuron/nki_kernel_compiler.py b/platforms/neuron/nki_kernel_compiler.py new file mode 100755 index 00000000..207d0c26 --- /dev/null +++ b/platforms/neuron/nki_kernel_compiler.py @@ -0,0 +1,179 @@ +"""Compile inline ZML NKI kernels into Neuron custom-native backend configs. + +The `nki_kernel.zig` helper writes one JSON request per inline kernel. +This script asks NKI to produce BIR and returns the backend_config expected by +`AwsNeuronCustomNativeKernel`. +""" + +import argparse +import inspect +import json +import re +import sys +import tempfile +import types +from dataclasses import dataclass +from pathlib import Path + +import ml_dtypes +import numpy as np + + +@dataclass(frozen=True) +class TensorSpec: + dtype: str + shape: tuple[int, ...] + + +def dtype_from_name(name: str): + mapping = { + "bool": np.bool_, + "i8": np.int8, + "i16": np.int16, + "i32": np.int32, + "i64": np.int64, + "u8": np.uint8, + "u16": np.uint16, + "u32": np.uint32, + "u64": np.uint64, + "f16": np.float16, + "f32": np.float32, + "f64": np.float64, + } + + if name == "bf16": + return ml_dtypes.bfloat16 + + try: + return mapping[name] + except KeyError as exc: + raise ValueError(f"unsupported dtype: {name}") from exc + + +def parse_tensor_specs(raw_specs: list[dict]) -> tuple[TensorSpec, ...]: + return tuple( + TensorSpec( + dtype=spec["dtype"], + shape=tuple(spec.get("shape", spec["dims"])), + ) + for spec in raw_specs + ) + + +def load_kernel(source_path: Path, source: str, entrypoint: str): + module_name = re.sub(r"\W", "_", source_path.stem) or "kernel" + if module_name[0].isdigit(): + module_name = f"kernel_{module_name}" + + module = types.ModuleType(module_name) + module.__file__ = str(source_path) + module.__dict__["__builtins__"] = __builtins__ + sys.modules[module_name] = module + exec(compile(source, str(source_path), "exec"), module.__dict__) + + try: + return module.__dict__[entrypoint] + except KeyError as exc: + raise KeyError(f"entrypoint {entrypoint!r} not found in {source_path}") from exc + + +def make_inputs(inputs: tuple[TensorSpec, ...]) -> list[np.ndarray]: + """Create zero-filled specialization inputs matching the declared ZML signature.""" + arrays: list[np.ndarray] = [] + for input_spec in inputs: + arrays.append( + np.zeros( + shape=input_spec.shape, + dtype=dtype_from_name(input_spec.dtype), + ) + ) + return arrays + + +def compile_kernel( + source: str, + entrypoint: str, + target: str, + tool_bin_dir: str, + neuronx_cc_args: tuple[str, ...], + inputs: tuple[TensorSpec, ...], + outputs: tuple[TensorSpec, ...], +) -> str: + """Run the full embedded-kernel pipeline for one custom-call.""" + tmp_dir = Path(tempfile.mkdtemp(prefix="zml-nki-", dir=".")) + + source_path = tmp_dir / "kernel.py" + artifacts_dir = tmp_dir / "artifacts" + artifacts_dir.mkdir(parents=True, exist_ok=False) + + source_path.write_text(source, encoding="utf-8") + + kernel = load_kernel(source_path, source, entrypoint) + + input_names = [ + parameter.name + for parameter in inspect.signature(kernel).parameters.values() + if parameter.kind + in (inspect.Parameter.POSITIONAL_ONLY, inspect.Parameter.POSITIONAL_OR_KEYWORD) + ] + + if len(input_names) != len(inputs): + raise ValueError( + f"{entrypoint} expects {len(input_names)} inputs, ZML declared {len(inputs)}" + ) + + from nki.compiler.driver import compile_to_bir + from nki.compiler.frontend import ParserFrontend + from nki.compiler.ncc_driver import CompilationMode, CompileOptions + + compile_opts = CompileOptions( + target=target, + artifacts_dir=str(artifacts_dir), + output_path=str(artifacts_dir / "unused.neff"), + neuronx_cc_path=str(Path(tool_bin_dir) / "neuronx-cc"), + neuronx_cc_args=neuronx_cc_args, + mode=CompilationMode.INTEGRATION, + ).disable_backend_optimizations() + + nir = compile_to_bir( + kernel, + frontend=ParserFrontend(), + inputs=dict(zip(input_names, make_inputs(inputs), strict=True)), + compile_opts=compile_opts, + output_names=[f"output{i}" for i in range(len(outputs))], + ) + + return nir.build_config().backend_config_b64.decode("utf-8") + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Compile a ZML NKI kernel to a custom-native backend config" + ) + parser.add_argument("--request", required=True) + parser.add_argument("--result", required=True) + args = parser.parse_args() + + request_path = Path(args.request) + result_path = Path(args.result) + + request = json.loads(request_path.read_text(encoding="utf-8")) + + backend_config_b64 = compile_kernel( + source=request["source"], + entrypoint=request["entrypoint"], + target=request["target"], + tool_bin_dir=request["tool_bin_dir"], + neuronx_cc_args=tuple(request["neuronx_cc_args"]), + inputs=parse_tensor_specs(request["inputs"]), + outputs=parse_tensor_specs(request["outputs"]), + ) + + result_path.write_text( + json.dumps({"backend_config_b64": backend_config_b64}), + encoding="utf-8", + ) + + +if __name__ == "__main__": + main() diff --git a/platforms/neuron/nrt.h b/platforms/neuron/nrt.h new file mode 100644 index 00000000..9970d0bd --- /dev/null +++ b/platforms/neuron/nrt.h @@ -0,0 +1,5 @@ +#include +#include +#include +#include +#include diff --git a/platforms/neuron/packages.lock.json b/platforms/neuron/packages.lock.json new file mode 100755 index 00000000..2f275bba --- /dev/null +++ b/platforms/neuron/packages.lock.json @@ -0,0 +1,579 @@ +{ + "packages": [ + { + "arch": "amd64", + "dependencies": [ + { + "key": "libc6_2.36-9-p-deb12u10_amd64", + "name": "libc6", + "version": "2.36-9+deb12u10" + }, + { + "key": "gcc-12-base_12.2.0-14-p-deb12u1_amd64", + "name": "gcc-12-base", + "version": "12.2.0-14+deb12u1" + } + ], + "key": "libgcc-s1_12.2.0-14-p-deb12u1_amd64", + "name": "libgcc-s1", + "sha256": "3016e62cb4b7cd8038822870601f5ed131befe942774d0f745622cc77d8a88f7", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/g/gcc-12/libgcc-s1_12.2.0-14+deb12u1_amd64.deb" + ], + "version": "12.2.0-14+deb12u1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libc6_2.36-9-p-deb12u10_amd64", + "name": "libc6", + "sha256": "5dc83256f10ca4d0f2a53dd6583ffd0d0e319af30074ea6c82fb0ae77bd16365", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/g/glibc/libc6_2.36-9+deb12u10_amd64.deb" + ], + "version": "2.36-9+deb12u10" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "gcc-12-base_12.2.0-14-p-deb12u1_amd64", + "name": "gcc-12-base", + "sha256": "1896a2aacf4ad681ff5eacc24a5b0ca4d5d9c9b9c9e4b6de5197bc1e116ea619", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/g/gcc-12/gcc-12-base_12.2.0-14+deb12u1_amd64.deb" + ], + "version": "12.2.0-14+deb12u1" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "libc6_2.36-9-p-deb12u10_amd64", + "name": "libc6", + "version": "2.36-9+deb12u10" + }, + { + "key": "libgcc-s1_12.2.0-14-p-deb12u1_amd64", + "name": "libgcc-s1", + "version": "12.2.0-14+deb12u1" + }, + { + "key": "gcc-12-base_12.2.0-14-p-deb12u1_amd64", + "name": "gcc-12-base", + "version": "12.2.0-14+deb12u1" + } + ], + "key": "libgomp1_12.2.0-14-p-deb12u1_amd64", + "name": "libgomp1", + "sha256": "48fec46bda7f5b1638b9e959889bfbc20491247d402d120bb152687eb48143d7", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/g/gcc-12/libgomp1_12.2.0-14+deb12u1_amd64.deb" + ], + "version": "12.2.0-14+deb12u1" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "libgcc-s1_12.2.0-14-p-deb12u1_amd64", + "name": "libgcc-s1", + "version": "12.2.0-14+deb12u1" + }, + { + "key": "libc6_2.36-9-p-deb12u10_amd64", + "name": "libc6", + "version": "2.36-9+deb12u10" + }, + { + "key": "gcc-12-base_12.2.0-14-p-deb12u1_amd64", + "name": "gcc-12-base", + "version": "12.2.0-14+deb12u1" + } + ], + "key": "libstdc-p--p-6_12.2.0-14-p-deb12u1_amd64", + "name": "libstdc++6", + "sha256": "5cd3171216d4ab0fc911cfe9c35509bf2dd8f47761c43b7f6a4296701551a24d", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/g/gcc-12/libstdc++6_12.2.0-14+deb12u1_amd64.deb" + ], + "version": "12.2.0-14+deb12u1" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "libc6_2.36-9-p-deb12u10_amd64", + "name": "libc6", + "version": "2.36-9+deb12u10" + }, + { + "key": "libgcc-s1_12.2.0-14-p-deb12u1_amd64", + "name": "libgcc-s1", + "version": "12.2.0-14+deb12u1" + }, + { + "key": "gcc-12-base_12.2.0-14-p-deb12u1_amd64", + "name": "gcc-12-base", + "version": "12.2.0-14+deb12u1" + } + ], + "key": "zlib1g_1-1.2.13.dfsg-1_amd64", + "name": "zlib1g", + "sha256": "d7dd1d1411fedf27f5e27650a6eff20ef294077b568f4c8c5e51466dc7c08ce4", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/z/zlib/zlib1g_1.2.13.dfsg-1_amd64.deb" + ], + "version": "1:1.2.13.dfsg-1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "aws-neuronx-runtime-lib_2.32.31.0-0234f5ed2_amd64", + "name": "aws-neuronx-runtime-lib", + "sha256": "ed69354674b926490d917a6aeb7dea8b6300278a37958ee9a412e2cd89c41792", + "urls": [ + "https://apt.repos.neuron.amazonaws.com/pool/main/a/aws-neuronx-runtime-lib/aws-neuronx-runtime-lib_2.32.31.0-0234f5ed2_amd64.deb" + ], + "version": "2.32.31.0-0234f5ed2" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "libhwloc-dev_2.9.0-1_amd64", + "name": "libhwloc-dev", + "version": "2.9.0-1" + }, + { + "key": "libltdl-dev_2.4.7-7_deb12u1_amd64", + "name": "libltdl-dev", + "version": "2.4.7-7~deb12u1" + }, + { + "key": "libltdl7_2.4.7-7_deb12u1_amd64", + "name": "libltdl7", + "version": "2.4.7-7~deb12u1" + }, + { + "key": "libc6_2.36-9-p-deb12u10_amd64", + "name": "libc6", + "version": "2.36-9+deb12u10" + }, + { + "key": "libgcc-s1_12.2.0-14-p-deb12u1_amd64", + "name": "libgcc-s1", + "version": "12.2.0-14+deb12u1" + }, + { + "key": "gcc-12-base_12.2.0-14-p-deb12u1_amd64", + "name": "gcc-12-base", + "version": "12.2.0-14+deb12u1" + }, + { + "key": "libnuma-dev_2.0.16-1_amd64", + "name": "libnuma-dev", + "version": "2.0.16-1" + }, + { + "key": "libc6-dev_2.36-9-p-deb12u10_amd64", + "name": "libc6-dev", + "version": "2.36-9+deb12u10" + }, + { + "key": "rpcsvc-proto_1.4.3-1_amd64", + "name": "rpcsvc-proto", + "version": "1.4.3-1" + }, + { + "key": "libnsl-dev_1.3.0-2_amd64", + "name": "libnsl-dev", + "version": "1.3.0-2" + }, + { + "key": "libtirpc-dev_1.3.3-p-ds-1_amd64", + "name": "libtirpc-dev", + "version": "1.3.3+ds-1" + }, + { + "key": "libtirpc3_1.3.3-p-ds-1_amd64", + "name": "libtirpc3", + "version": "1.3.3+ds-1" + }, + { + "key": "libtirpc-common_1.3.3-p-ds-1_amd64", + "name": "libtirpc-common", + "version": "1.3.3+ds-1" + }, + { + "key": "libgssapi-krb5-2_1.20.1-2-p-deb12u3_amd64", + "name": "libgssapi-krb5-2", + "version": "1.20.1-2+deb12u3" + }, + { + "key": "libkrb5support0_1.20.1-2-p-deb12u3_amd64", + "name": "libkrb5support0", + "version": "1.20.1-2+deb12u3" + }, + { + "key": "libkrb5-3_1.20.1-2-p-deb12u3_amd64", + "name": "libkrb5-3", + "version": "1.20.1-2+deb12u3" + }, + { + "key": "libssl3_3.0.16-1_deb12u1_amd64", + "name": "libssl3", + "version": "3.0.16-1~deb12u1" + }, + { + "key": "libkeyutils1_1.6.3-2_amd64", + "name": "libkeyutils1", + "version": "1.6.3-2" + }, + { + "key": "libk5crypto3_1.20.1-2-p-deb12u3_amd64", + "name": "libk5crypto3", + "version": "1.20.1-2+deb12u3" + }, + { + "key": "libcom-err2_1.47.0-2_amd64", + "name": "libcom-err2", + "version": "1.47.0-2" + }, + { + "key": "libnsl2_1.3.0-2_amd64", + "name": "libnsl2", + "version": "1.3.0-2" + }, + { + "key": "libcrypt-dev_1-4.4.33-2_amd64", + "name": "libcrypt-dev", + "version": "1:4.4.33-2" + }, + { + "key": "libcrypt1_1-4.4.33-2_amd64", + "name": "libcrypt1", + "version": "1:4.4.33-2" + }, + { + "key": "linux-libc-dev_6.1.137-1_amd64", + "name": "linux-libc-dev", + "version": "6.1.137-1" + }, + { + "key": "libc-dev-bin_2.36-9-p-deb12u10_amd64", + "name": "libc-dev-bin", + "version": "2.36-9+deb12u10" + }, + { + "key": "libnuma1_2.0.16-1_amd64", + "name": "libnuma1", + "version": "2.0.16-1" + }, + { + "key": "libhwloc15_2.9.0-1_amd64", + "name": "libhwloc15", + "version": "2.9.0-1" + }, + { + "key": "libudev1_252.36-1_deb12u1_amd64", + "name": "libudev1", + "version": "252.36-1~deb12u1" + } + ], + "key": "aws-neuronx-collectives_2.32.28.0-452cba8de_amd64", + "name": "aws-neuronx-collectives", + "sha256": "ec737c6647b6fbd38550e11f25e27d0f1a96dbb2cd28ae5bbf3f827faba5bdc7", + "urls": [ + "https://apt.repos.neuron.amazonaws.com/pool/main/a/aws-neuronx-collectives/aws-neuronx-collectives_2.32.28.0-452cba8de_amd64.deb" + ], + "version": "2.32.28.0-452cba8de" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libhwloc-dev_2.9.0-1_amd64", + "name": "libhwloc-dev", + "sha256": "0b12593b41cd3c270548509c680192059c7b124261536db2630252559c865fad", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/h/hwloc/libhwloc-dev_2.9.0-1_amd64.deb" + ], + "version": "2.9.0-1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libltdl-dev_2.4.7-7_deb12u1_amd64", + "name": "libltdl-dev", + "sha256": "60ad26b3007d4304a1141649d4e764a27735ff4f1835a62c845d51bd734932b0", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/libt/libtool/libltdl-dev_2.4.7-7~deb12u1_amd64.deb" + ], + "version": "2.4.7-7~deb12u1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libltdl7_2.4.7-7_deb12u1_amd64", + "name": "libltdl7", + "sha256": "8070b550321d4bd1c298f52f75374898d39112b709026d2ea5362ef1277b775a", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/libt/libtool/libltdl7_2.4.7-7~deb12u1_amd64.deb" + ], + "version": "2.4.7-7~deb12u1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libnuma-dev_2.0.16-1_amd64", + "name": "libnuma-dev", + "sha256": "5893d10043918ed5d730ac1c6a54dd4e3f781166b809cbd43e43b2e576e4a5db", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/n/numactl/libnuma-dev_2.0.16-1_amd64.deb" + ], + "version": "2.0.16-1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libc6-dev_2.36-9-p-deb12u10_amd64", + "name": "libc6-dev", + "sha256": "5597f588e682c0d8a02b5445eab3b61e5530c9a32db3527698cb963669b27527", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/g/glibc/libc6-dev_2.36-9+deb12u10_amd64.deb" + ], + "version": "2.36-9+deb12u10" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "rpcsvc-proto_1.4.3-1_amd64", + "name": "rpcsvc-proto", + "sha256": "32ac0692694f8a34cc90c895f4fc739680fb2ef0e2d4870a68833682bf1c81a3", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/r/rpcsvc-proto/rpcsvc-proto_1.4.3-1_amd64.deb" + ], + "version": "1.4.3-1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libnsl-dev_1.3.0-2_amd64", + "name": "libnsl-dev", + "sha256": "bb81a188c119cd7fdebae723cbc95887b6c549b2fe4fb7e268a9c8846444da99", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/libn/libnsl/libnsl-dev_1.3.0-2_amd64.deb" + ], + "version": "1.3.0-2" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libtirpc-dev_1.3.3-p-ds-1_amd64", + "name": "libtirpc-dev", + "sha256": "03326473eed54ffa27efae19aa5d6aeb402930968f869f318445513093691d55", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/libt/libtirpc/libtirpc-dev_1.3.3+ds-1_amd64.deb" + ], + "version": "1.3.3+ds-1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libtirpc3_1.3.3-p-ds-1_amd64", + "name": "libtirpc3", + "sha256": "2a46d5a5e9486da11ffeff5740931740d6deae4f92cd6098df060dc5dff1e1c7", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/libt/libtirpc/libtirpc3_1.3.3+ds-1_amd64.deb" + ], + "version": "1.3.3+ds-1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libtirpc-common_1.3.3-p-ds-1_amd64", + "name": "libtirpc-common", + "sha256": "3e3ef129b4bf61513144236e15e1b4ec57fa5ae3dc8a72137abdbefb7a63af85", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/libt/libtirpc/libtirpc-common_1.3.3+ds-1_all.deb" + ], + "version": "1.3.3+ds-1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libgssapi-krb5-2_1.20.1-2-p-deb12u3_amd64", + "name": "libgssapi-krb5-2", + "sha256": "34a3fcd0e5aeeb01e895b8ce563b06b12b0d763f1b5943b97feffed07a3f56f1", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/k/krb5/libgssapi-krb5-2_1.20.1-2+deb12u3_amd64.deb" + ], + "version": "1.20.1-2+deb12u3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libkrb5support0_1.20.1-2-p-deb12u3_amd64", + "name": "libkrb5support0", + "sha256": "8862ae9677f2361d888fcefb2734e4ddfb2678f82535d21c4c0cc77d8e406e9f", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/k/krb5/libkrb5support0_1.20.1-2+deb12u3_amd64.deb" + ], + "version": "1.20.1-2+deb12u3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libkrb5-3_1.20.1-2-p-deb12u3_amd64", + "name": "libkrb5-3", + "sha256": "e3c228ea202bf5a278329f5a96cbe3ca48927e0ed7fea849680b0996fece4879", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/k/krb5/libkrb5-3_1.20.1-2+deb12u3_amd64.deb" + ], + "version": "1.20.1-2+deb12u3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libssl3_3.0.16-1_deb12u1_amd64", + "name": "libssl3", + "sha256": "eaa2bab2130820f09361dc8186ddeb11d2a18ec5e5e3806f24414d5d8065a57a", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/o/openssl/libssl3_3.0.16-1~deb12u1_amd64.deb" + ], + "version": "3.0.16-1~deb12u1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libkeyutils1_1.6.3-2_amd64", + "name": "libkeyutils1", + "sha256": "cfac89e6a7a54ff3c6a4f843310e25efeddaa771baeae470bd98bd588c373563", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/k/keyutils/libkeyutils1_1.6.3-2_amd64.deb" + ], + "version": "1.6.3-2" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libk5crypto3_1.20.1-2-p-deb12u3_amd64", + "name": "libk5crypto3", + "sha256": "6aff44ea189ad062eb7a83b9fac99861f1efa35bf66bc7605c2ade644a08bb94", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/k/krb5/libk5crypto3_1.20.1-2+deb12u3_amd64.deb" + ], + "version": "1.20.1-2+deb12u3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libcom-err2_1.47.0-2_amd64", + "name": "libcom-err2", + "sha256": "8010e4285276bb344c05ae780deae2fffb45e237116c3a78481365c5954125ec", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/e/e2fsprogs/libcom-err2_1.47.0-2_amd64.deb" + ], + "version": "1.47.0-2" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libnsl2_1.3.0-2_amd64", + "name": "libnsl2", + "sha256": "c0d83437fdb016cb289436f49f28a36be44b3e8f1f2498c7e3a095f709c0d6f8", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/libn/libnsl/libnsl2_1.3.0-2_amd64.deb" + ], + "version": "1.3.0-2" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libcrypt-dev_1-4.4.33-2_amd64", + "name": "libcrypt-dev", + "sha256": "81ccd29130f75a9e3adabc80e61921abff42f76761e1f792fa2d1bb69af7f52f", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/libx/libxcrypt/libcrypt-dev_4.4.33-2_amd64.deb" + ], + "version": "1:4.4.33-2" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libcrypt1_1-4.4.33-2_amd64", + "name": "libcrypt1", + "sha256": "f5f60a5cdfd4e4eaa9438ade5078a57741a7a78d659fcb0c701204f523e8bd29", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/libx/libxcrypt/libcrypt1_4.4.33-2_amd64.deb" + ], + "version": "1:4.4.33-2" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "linux-libc-dev_6.1.137-1_amd64", + "name": "linux-libc-dev", + "sha256": "0fc664e6e81677598e8bfa41cf377ebd353a6e50ad2099bc4603b9f66c9b854b", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/l/linux/linux-libc-dev_6.1.137-1_amd64.deb" + ], + "version": "6.1.137-1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libc-dev-bin_2.36-9-p-deb12u10_amd64", + "name": "libc-dev-bin", + "sha256": "db898f3f85e19d4d54a277a0ab479b57566be5b902e6e8ebe814410aa3149e72", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/g/glibc/libc-dev-bin_2.36-9+deb12u10_amd64.deb" + ], + "version": "2.36-9+deb12u10" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libnuma1_2.0.16-1_amd64", + "name": "libnuma1", + "sha256": "639e1ab6bd66ead40db8a22c332d7199679fa22db261cac34444eb8eb4c17dda", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/n/numactl/libnuma1_2.0.16-1_amd64.deb" + ], + "version": "2.0.16-1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libhwloc15_2.9.0-1_amd64", + "name": "libhwloc15", + "sha256": "4e7194b5d9adbb1a7d11a9501a4be4a4fa22876b69c73b7998056f552962c0e4", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/h/hwloc/libhwloc15_2.9.0-1_amd64.deb" + ], + "version": "2.9.0-1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libudev1_252.36-1_deb12u1_amd64", + "name": "libudev1", + "sha256": "ed03696385df7ba1b18b224175b120083875deae0e2f4369857f260b18533ea2", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/s/systemd/libudev1_252.36-1~deb12u1_amd64.deb" + ], + "version": "252.36-1~deb12u1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "aws-neuronx-tools_2.30.10.0-39feaa591_amd64", + "name": "aws-neuronx-tools", + "sha256": "b972e4db5e84a429aa237d1eb9347fcaa78c19e848049a3495efef734a0b240f", + "urls": [ + "https://apt.repos.neuron.amazonaws.com/pool/main/a/aws-neuronx-tools/aws-neuronx-tools_2.30.10.0-39feaa591_amd64.deb" + ], + "version": "2.30.10.0-39feaa591" + } + ], + "version": 1 +} \ No newline at end of file diff --git a/platforms/neuron/packages.yaml b/platforms/neuron/packages.yaml new file mode 100644 index 00000000..eb399f49 --- /dev/null +++ b/platforms/neuron/packages.yaml @@ -0,0 +1,27 @@ +# Packages for examples/debian_snapshot. +# +# Anytime this file is changed, the lockfile needs to be regenerated. +# +# To generate the ubuntu_base.lock.json run the following command +# +# bazel run @apt_neuron//:lock +# +version: 1 + +sources: + - channel: bookworm main + url: https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/ + - channel: jammy main + url: https://apt.repos.neuron.amazonaws.com + +archs: + - "amd64" + +packages: + - "libgcc-s1" # implicit dep of libneuronpjrt/libstdc++ + - "libgomp1" # implicit dep of neuronx-cc + - "libstdc++6" # implicit dep of libneuronpjrt/libnccom + - "zlib1g" # implicit dep of the sandboxed Python runtime + - "aws-neuronx-runtime-lib" + - "aws-neuronx-collectives" + - "aws-neuronx-tools" diff --git a/platforms/neuron/python_launcher.zig b/platforms/neuron/python_launcher.zig new file mode 100644 index 00000000..075034ab --- /dev/null +++ b/platforms/neuron/python_launcher.zig @@ -0,0 +1,155 @@ +const std = @import("std"); + +const c = @import("c"); + +// Shared isolated-Python launcher used by Neuron helper binaries. The sandbox +// contains the Python stdlib, site-packages, and these tiny Zig executables; the +// helpers configure Python from their own location so they are hermetic. + +pub const ModuleEntrypoint = struct { + module_name: [:0]const u8, + attr_name: [:0]const u8 = "main", +}; + +// Run `module.attr()` inside the sandboxed Python interpreter. +pub fn runModuleEntrypoint(init: std.process.Init, entrypoint: ModuleEntrypoint) !void { + var config: c.PyConfig = undefined; + _ = try configureInterpreter(init, &config); + defer c.PyConfig_Clear(&config); + + pyStatusCheck(c.Py_InitializeFromConfig(&config)); + defer c.Py_Finalize(); + + const module = c.PyImport_ImportModule(entrypoint.module_name.ptr) orelse { + std.log.err("Failed to import {s}", .{entrypoint.module_name}); + c.PyErr_Print(); + return error.FailedToImportModule; + }; + defer c.Py_DecRef(module); + + const main_fn = c.PyObject_GetAttrString(module, entrypoint.attr_name.ptr) orelse { + std.log.err("Failed to resolve {s}.{s}", .{ entrypoint.module_name, entrypoint.attr_name }); + c.PyErr_Print(); + return error.FailedToResolveEntrypoint; + }; + defer c.Py_DecRef(main_fn); + + const result = c.PyObject_CallNoArgs(main_fn); + if (result == null) { + c.PyErr_Print(); + std.process.exit(1); + } + defer c.Py_DecRef(result); +} + +// Run a Python file relative to the helper binary and call its `main()`. +pub fn runScriptMain(init: std.process.Init, relative_script_path: []const []const u8) !void { + const arena = init.arena; + + var config: c.PyConfig = undefined; + const self_exe_dir = try configureInterpreter(init, &config); + defer c.PyConfig_Clear(&config); + + pyStatusCheck(c.Py_InitializeFromConfig(&config)); + defer c.Py_Finalize(); + + const script_path = try std.Io.Dir.path.join(arena.allocator(), relative_script_path); + const full_script_path = try std.Io.Dir.path.join(arena.allocator(), &.{ self_exe_dir, script_path }); + const bootstrap = try std.fmt.allocPrint(arena.allocator(), + \\import importlib.util + \\spec = importlib.util.spec_from_file_location("_zml_neuron_python_launcher", "{s}") + \\module = importlib.util.module_from_spec(spec) + \\assert spec.loader is not None + \\spec.loader.exec_module(module) + \\module.main() + , .{full_script_path}); + + if (c.PyRun_SimpleStringFlags(bootstrap.ptr, null) != 0) { + c.PyErr_Print(); + return error.CompilerExecutionFailed; + } +} + +fn pyStatusCheck(status: c.PyStatus) void { + if (c.PyStatus_Exception(status) != 0) { + if (c.PyStatus_IsExit(status) != 0) { + std.process.exit(@intCast(status.exitcode)); + } + c.Py_ExitStatusException(status); + } +} + +fn toPosixPathW(file_path: []const u8) error{ InvalidUtf8, NameTooLong, CodepointTooLarge }![std.posix.PATH_MAX - 1:0]c.wchar_t { + var view = std.unicode.Utf8View.init(file_path) catch return error.InvalidUtf8; + var it = view.iterator(); + + var path_with_null: [std.posix.PATH_MAX - 1:0]c.wchar_t = undefined; + var len: usize = 0; + while (it.nextCodepoint()) |cp| { + if (len + 1 >= path_with_null.len) return error.NameTooLong; + path_with_null[len] = std.math.cast(c.wchar_t, cp) orelse return error.CodepointTooLarge; + len += 1; + } + path_with_null[len] = 0; + return path_with_null; +} + +fn configureInterpreter(init: std.process.Init, config: *c.PyConfig) ![]const u8 { + const arena = init.arena; + const io = init.io; + + { + var preconfig: c.PyPreConfig = undefined; + c.PyPreConfig_InitIsolatedConfig(&preconfig); + preconfig.utf8_mode = 1; + pyStatusCheck(c.Py_PreInitialize(&preconfig)); + } + + c.PyConfig_InitIsolatedConfig(config); + config.module_search_paths_set = 1; + config.optimization_level = 2; + config.write_bytecode = 0; + + const args = try init.minimal.args.toSlice(arena.allocator()); + const c_args = try arena.allocator().alloc([*:0]const u8, args.len); + for (args, 0..) |arg, i| { + c_args[i] = (try arena.allocator().dupeSentinel(u8, arg, 0)).ptr; + } + pyStatusCheck(c.PyConfig_SetBytesArgv(config, @intCast(c_args.len), @ptrCast(c_args.ptr))); + + var self_exe_dir_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const self_exe_dir = blk: { + const n = try std.process.executableDirPath(io, &self_exe_dir_buf); + break :blk self_exe_dir_buf[0..n]; + }; + + { + var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const home = try std.mem.printSentinel(&buf, "{f}{d}.{d}", .{ + std.Io.Dir.path.fmtJoin(&.{ + self_exe_dir, + "..", + "lib", + "python", + }), + c.PY_MAJOR_VERSION, + c.PY_MINOR_VERSION, + }, 0); + pyStatusCheck(c.PyConfig_SetBytesString(config, &config.home, home)); + pyStatusCheck(c.PyWideStringList_Append(&config.module_search_paths, &try toPosixPathW(home))); + } + + { + var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const site_packages = try std.fmt.bufPrint(&buf, "{f}", .{ + std.Io.Dir.path.fmtJoin(&.{ + self_exe_dir, + "..", + "site-packages", + }), + }); + pyStatusCheck(c.PyWideStringList_Append(&config.module_search_paths, &try toPosixPathW(site_packages))); + } + + return self_exe_dir; +} diff --git a/platforms/neuron/requirements.in b/platforms/neuron/requirements.in new file mode 100644 index 00000000..51174303 --- /dev/null +++ b/platforms/neuron/requirements.in @@ -0,0 +1,5 @@ +--find-links https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html +--extra-index-url=https://pip.repos.neuron.amazonaws.com +libneuronxla>=3.0 +neuronx-cc>=2.25 +nki>=0.4 diff --git a/platforms/neuron/requirements.lock.txt b/platforms/neuron/requirements.lock.txt new file mode 100644 index 00000000..0945aa7d --- /dev/null +++ b/platforms/neuron/requirements.lock.txt @@ -0,0 +1,490 @@ +# This file was autogenerated by uv via the following command: +# bazel run //platforms/neuron:requirements.update +--index-url https://pypi.org/simple +--extra-index-url https://pip.repos.neuron.amazonaws.com +--find-links https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html + +boto3==1.43.14 \ + --hash=sha256:574335744656cfed0b362a0a0467aaf2eb2bf15526edcd02d31d3c661f4b09e4 \ + --hash=sha256:5c0a994b3182061ee101812e721100717a4d664f9f4ceaf4a86b6d032ce9fc2d + # via libneuronxla +botocore==1.43.14 \ + --hash=sha256:1f4a2a95ea78c10398e78431e98c1fe47adb54a7b10a32975144c1f541186658 \ + --hash=sha256:b9e500737e43d2f147c9d4e23b54360335e77d4c0ba90a318f51b65e06cb8516 + # via + # boto3 + # libneuronxla + # s3transfer +certifi==2026.5.20 \ + --hash=sha256:3c52e209ba0a4ad7aebe60436a4ab349c39e1e602e8c134221e546902ad25897 \ + --hash=sha256:69dea482ab64caa7b9f6aba1c6bf48bb6a5448d1c0f1b17ab42ad8c763a5344d + # via requests +charset-normalizer==3.4.7 \ + --hash=sha256:007d05ec7321d12a40227aae9e2bc6dca73f3cb21058999a1df9e193555a9dcc \ + --hash=sha256:03853ed82eeebbce3c2abfdbc98c96dc205f32a79627688ac9a27370ea61a49c \ + --hash=sha256:07d9e39b01743c3717745f4c530a6349eadbfa043c7577eef86c502c15df2c67 \ + --hash=sha256:08e721811161356f97b4059a9ba7bafb23ea5ee2255402c42881c214e173c6b4 \ + --hash=sha256:0c96c3b819b5c3e9e165495db84d41914d6894d55181d2d108cc1a69bfc9cce0 \ + --hash=sha256:0ea948db76d31190bf08bd371623927ee1339d5f2a0b4b1b4a4439a65298703c \ + --hash=sha256:0f7eb884681e3938906ed0434f20c63046eacd0111c4ba96f27b76084cd679f5 \ + --hash=sha256:12a6fff75f6bc66711b73a2f0addfc4c8c15a20e805146a02d147a318962c444 \ + --hash=sha256:12d8baf840cc7889b37c7c770f478adea7adce3dcb3944d02ec87508e2dcf153 \ + --hash=sha256:14265bfe1f09498b9d8ec91e9ec9fa52775edf90fcbde092b25f4a33d444fea9 \ + --hash=sha256:16d971e29578a5e97d7117866d15889a4a07befe0e87e703ed63cd90cb348c01 \ + --hash=sha256:177a0ba5f0211d488e295aaf82707237e331c24788d8d76c96c5a41594723217 \ + --hash=sha256:1a87ca9d5df6fe460483d9a5bbf2b18f620cbed41b432e2bddb686228282d10b \ + --hash=sha256:1c2a768fdd44ee4a9339a9b0b130049139b8ce3c01d2ce09f67f5a68048d477c \ + --hash=sha256:1c2aed2e5e41f24ea8ef1590b8e848a79b56f3a5564a65ceec43c9d692dc7d8a \ + --hash=sha256:1dc8b0ea451d6e69735094606991f32867807881400f808a106ee1d963c46a83 \ + --hash=sha256:1efde3cae86c8c273f1eb3b287be7d8499420cf2fe7585c41d370d3e790054a5 \ + --hash=sha256:202389074300232baeb53ae2569a60901f7efadd4245cf3a3bf0617d60b439d7 \ + --hash=sha256:203104ed3e428044fd943bc4bf45fa73c0730391f9621e37fe39ecf477b128cb \ + --hash=sha256:2257141f39fe65a3fdf38aeccae4b953e5f3b3324f4ff0daf9f15b8518666a2c \ + --hash=sha256:298930cec56029e05497a76988377cbd7457ba864beeea92ad7e844fe74cd1f1 \ + --hash=sha256:2cd4a60d0e2fb04537162c62bbbb4182f53541fe0ede35cdf270a1c1e723cc42 \ + --hash=sha256:2d6eb928e13016cea4f1f21d1e10c1cebd5a421bc57ddf5b1142ae3f86824fab \ + --hash=sha256:2fe249cb4651fd12605b7288b24751d8bfd46d35f12a20b1ba33dea122e690df \ + --hash=sha256:30b8d1d8c52a48c2c5690e152c169b673487a2a58de1ec7393196753063fcd5e \ + --hash=sha256:320ade88cfb846b8cd6b4ddf5ee9e80ee0c1f52401f2456b84ae1ae6a1a5f207 \ + --hash=sha256:3534e7dcbdcf757da6b85a0bbf5b6868786d5982dd959b065e65481644817a18 \ + --hash=sha256:36836d6ff945a00b88ba1e4572d721e60b5b8c98c155d465f56ad19d68f23734 \ + --hash=sha256:38c0109396c4cfc574d502df99742a45c72c08eff0a36158b6f04000043dbf38 \ + --hash=sha256:3946fa46a0cf3e4c8cb1cc52f56bb536310d34f25f01ca9b6c16afa767dab110 \ + --hash=sha256:3bec022aec2c514d9cf199522a802bd007cd588ab17ab2525f20f9c34d067c18 \ + --hash=sha256:3c9a494bc5ec77d43cea229c4f6db1e4d8fe7e1bbffa8b6f0f0032430ff8ab44 \ + --hash=sha256:3dce51d0f5e7951f8bb4900c257dad282f49190fdbebecd4ba99bcc41fef404d \ + --hash=sha256:3dedcc22d73ec993f42055eff4fcfed9318d1eeb9a6606c55892a26964964e48 \ + --hash=sha256:4042d5c8f957e15221d423ba781e85d553722fc4113f523f2feb7b188cc34c5e \ + --hash=sha256:481551899c856c704d58119b5025793fa6730adda3571971af568f66d2424bb5 \ + --hash=sha256:4dc1e73c36828f982bfe79fadf5919923f8a6f4df2860804db9a98c48824ce8d \ + --hash=sha256:4e5163c14bffd570ef2affbfdd77bba66383890797df43dc8b4cc7d6f500bf53 \ + --hash=sha256:511ef87c8aec0783e08ac18565a16d435372bc1ac25a91e6ac7f5ef2b0bff790 \ + --hash=sha256:532bc9bf33a68613fd7d65e4b1c71a6a38d7d42604ecf239c77392e9b4e8998c \ + --hash=sha256:54523e136b8948060c0fa0bc7b1b50c32c186f2fceee897a495406bb6e311d2b \ + --hash=sha256:5649fd1c7bade02f320a462fdefd0b4bd3ce036065836d4f42e0de958038e116 \ + --hash=sha256:56be790f86bfb2c98fb742ce566dfb4816e5a83384616ab59c49e0604d49c51d \ + --hash=sha256:5b77459df20e08151cd6f8b9ef8ef1f961ef73d85c21a555c7eed5b79410ec10 \ + --hash=sha256:5ed6ab538499c8644b8a3e18debabcd7ce684f3fa91cf867521a7a0279cab2d6 \ + --hash=sha256:6178f72c5508bfc5fd446a5905e698c6212932f25bcdd4b47a757a50605a90e2 \ + --hash=sha256:6370e8686f662e6a3941ee48ed4742317cafbe5707e36406e9df792cdb535776 \ + --hash=sha256:64f02c6841d7d83f832cd97ccf8eb8a906d06eb95d5276069175c696b024b60a \ + --hash=sha256:65bcd23054beab4d166035cabbc868a09c1a49d1efe458fe8e4361215df40265 \ + --hash=sha256:66671f93accb62ed07da56613636f3641f1a12c13046ce91ffc923721f23c008 \ + --hash=sha256:6696b7688f54f5af4462118f0bfa7c1621eeb87154f77fa04b9295ce7a8f2943 \ + --hash=sha256:6785f414ae0f3c733c437e0f3929197934f526d19dfaa75e18fdb4f94c6fb374 \ + --hash=sha256:67f6279d125ca0046a7fd386d01b311c6363844deac3e5b069b514ba3e63c246 \ + --hash=sha256:6c114670c45346afedc0d947faf3c7f701051d2518b943679c8ff88befe14f8e \ + --hash=sha256:6e0d51f618228538a3e8f46bd246f87a6cd030565e015803691603f55e12afb5 \ + --hash=sha256:6ed74185b2db44f41ef35fd1617c5888e59792da9bbc9190d6c7300617182616 \ + --hash=sha256:708838739abf24b2ceb208d0e22403dd018faeef86ddac04319a62ae884c4f15 \ + --hash=sha256:715479b9a2802ecac752a3b0efa2b0b60285cf962ee38414211abdfccc233b41 \ + --hash=sha256:733784b6d6def852c814bce5f318d25da2ee65dd4839a0718641c696e09a2960 \ + --hash=sha256:750e02e074872a3fad7f233b47734166440af3cdea0add3e95163110816d6752 \ + --hash=sha256:752a45dc4a6934060b3b0dab47e04edc3326575f82be64bc4fc293914566503e \ + --hash=sha256:7579e913a5339fb8fa133f6bbcfd8e6749696206cf05acdbdca71a1b436d8e72 \ + --hash=sha256:7641bb8895e77f921102f72833904dcd9901df5d6d72a2ab8f31d04b7e51e4e7 \ + --hash=sha256:7804338df6fcc08105c7745f1502ba68d900f45fd770d5bdd5288ddccb8a42d8 \ + --hash=sha256:80d04837f55fc81da168b98de4f4b797ef007fc8a79ab71c6ec9bc4dd662b15b \ + --hash=sha256:813c0e0132266c08eb87469a642cb30aaff57c5f426255419572aaeceeaa7bf4 \ + --hash=sha256:82b271f5137d07749f7bf32f70b17ab6eaabedd297e75dce75081a24f76eb545 \ + --hash=sha256:84c018e49c3bf790f9c2771c45e9313a08c2c2a6342b162cd650258b57817706 \ + --hash=sha256:8751d2787c9131302398b11e6c8068053dcb55d5a8964e114b6e196cf16cb366 \ + --hash=sha256:8778f0c7a52e56f75d12dae53ae320fae900a8b9b4164b981b9c5ce059cd1fcb \ + --hash=sha256:87fad7d9ba98c86bcb41b2dc8dbb326619be2562af1f8ff50776a39e55721c5a \ + --hash=sha256:8d828b6667a32a728a1ad1d93957cdf37489c57b97ae6c4de2860fa749b8fc1e \ + --hash=sha256:8e385e4267ab76874ae30db04c627faaaf0b509e1ccc11a95b3fc3e83f855c00 \ + --hash=sha256:92a0a01ead5e668468e952e4238cccd7c537364eb7d851ab144ab6627dbbe12f \ + --hash=sha256:94e1885b270625a9a828c9793b4d52a64445299baa1fea5a173bf1d3dd9a1a5a \ + --hash=sha256:a180c5e59792af262bf263b21a3c49353f25945d8d9f70628e73de370d55e1e1 \ + --hash=sha256:a277ab8928b9f299723bc1a2dabb1265911b1a76341f90a510368ca44ad9ab66 \ + --hash=sha256:a5fe03b42827c13cdccd08e6c0247b6a6d4b5e3cdc53fd1749f5896adcdc2356 \ + --hash=sha256:a6c5863edfbe888d9eff9c8b8087354e27618d9da76425c119293f11712a6319 \ + --hash=sha256:a89c23ef8d2c6b27fd200a42aa4ac72786e7c60d40efdc76e6011260b6e949c4 \ + --hash=sha256:adb2597b428735679446b46c8badf467b4ca5f5056aae4d51a19f9570301b1ad \ + --hash=sha256:ae196f021b5e7c78e918242d217db021ed2a6ace2bc6ae94c0fc596221c7f58d \ + --hash=sha256:ae89db9e5f98a11a4bf50407d4363e7b09b31e55bc117b4f7d80aab97ba009e5 \ + --hash=sha256:aed52fea0513bac0ccde438c188c8a471c4e0f457c2dd20cdbf6ea7a450046c7 \ + --hash=sha256:aef65cd602a6d0e0ff6f9930fcb1c8fec60dd2cfcb6facaf4bdb0e5873042db0 \ + --hash=sha256:af21eb4409a119e365397b2adbaca4c9ccab56543a65d5dbd9f920d6ac29f686 \ + --hash=sha256:b14b2d9dac08e28bb8046a1a0434b1750eb221c8f5b87a68f4fa11a6f97b5e34 \ + --hash=sha256:bb6d88045545b26da47aa879dd4a89a71d1dce0f0e549b1abcb31dfe4a8eac49 \ + --hash=sha256:bb8cc7534f51d9a017b93e3e85b260924f909601c3df002bcdb58ddb4dc41a5c \ + --hash=sha256:bc17a677b21b3502a21f66a8cc64f5bfad4df8a0b8434d661666f8ce90ac3af1 \ + --hash=sha256:bd6c2a1c7573c64738d716488d2cdd3c00e340e4835707d8fdb8dc1a66ef164e \ + --hash=sha256:bd9b23791fe793e4968dba0c447e12f78e425c59fc0e3b97f6450f4781f3ee60 \ + --hash=sha256:c03a41a8784091e67a39648f70c5f97b5b6a37f216896d44d2cdcb82615339a0 \ + --hash=sha256:c0f081d69a6e58272819b70288d3221a6ee64b98df852631c80f293514d3b274 \ + --hash=sha256:c35abb8bfff0185efac5878da64c45dafd2b37fb0383add1be155a763c1f083d \ + --hash=sha256:c36c333c39be2dbca264d7803333c896ab8fa7d4d6f0ab7edb7dfd7aea6e98c0 \ + --hash=sha256:c45e9440fb78f8ddabcf714b68f936737a121355bf59f3907f4e17721b9d1aae \ + --hash=sha256:c593052c465475e64bbfe5dbd81680f64a67fdc752c56d7a0ae205dc8aeefe0f \ + --hash=sha256:cdd68a1fb318e290a2077696b7eb7a21a49163c455979c639bf5a5dcdc46617d \ + --hash=sha256:ce3412fbe1e31eb81ea42f4169ed94861c56e643189e1e75f0041f3fe7020abe \ + --hash=sha256:cf1493cd8607bec4d8a7b9b004e699fcf8f9103a9284cc94962cb73d20f9d4a3 \ + --hash=sha256:cf29836da5119f3c8a8a70667b0ef5fdca3bb12f80fd06487cfa575b3909b393 \ + --hash=sha256:d4a48e5b3c2a489fae013b7589308a40146ee081f6f509e047e0e096084ceca1 \ + --hash=sha256:d560742f3c0d62afaccf9f41fe485ed69bd7661a241f86a3ef0f0fb8b1a397af \ + --hash=sha256:d6038d37043bced98a66e68d3aa2b6a35505dc01328cd65217cefe82f25def44 \ + --hash=sha256:d61f00a0869d77422d9b2aba989e2d24afa6ffd552af442e0e58de4f35ea6d00 \ + --hash=sha256:d635aab80466bc95771bb78d5370e74d36d1fe31467b6b29b8b57b2a3cd7d22c \ + --hash=sha256:dca4bbc466a95ba9c0234ef56d7dd9509f63da22274589ebd4ed7f1f4d4c54e3 \ + --hash=sha256:dd915403e231e6b1809fe9b6d9fc55cf8fb5e02765ac625d9cd623342a7905d7 \ + --hash=sha256:e044c39e41b92c845bc815e5ae4230804e8e7bc29e399b0437d64222d92809dd \ + --hash=sha256:e060d01aec0a910bdccb8be71faf34e7799ce36950f8294c8bf612cba65a2c9e \ + --hash=sha256:e1421b502d83040e6d7fb2fb18dff63957f720da3d77b2fbd3187ceb63755d7b \ + --hash=sha256:e17b8d5d6a8c47c85e68ca8379def1303fd360c3e22093a807cd34a71cd082b8 \ + --hash=sha256:e5f4d355f0a2b1a31bc3edec6795b46324349c9cb25eed068049e4f472fb4259 \ + --hash=sha256:e712b419df8ba5e42b226c510472b37bd57b38e897d3eca5e8cfd410a29fa859 \ + --hash=sha256:e74327fb75de8986940def6e8dee4f127cc9752bee7355bb323cc5b2659b6d46 \ + --hash=sha256:e80c8378d8f3d83cd3164da1ad2df9e37a666cdde7b1cb2298ed0b558064be30 \ + --hash=sha256:e8ac484bf18ce6975760921bb6148041faa8fef0547200386ea0b52b5d27bf7b \ + --hash=sha256:eca9705049ad3c7345d574e3510665cb2cf844c2f2dcfe675332677f081cbd46 \ + --hash=sha256:ed065083d0898c9d5b4bbec7b026fd755ff7454e6e8b73a67f8c744b13986e24 \ + --hash=sha256:edac0f1ab77644605be2cbba52e6b7f630731fc42b34cb0f634be1a6eface56a \ + --hash=sha256:effc3f449787117233702311a1b7d8f59cba9ced946ba727bdc329ec69028e24 \ + --hash=sha256:f22dec1690b584cea26fade98b2435c132c1b5f68e39f5a0b7627cd7ae31f1dc \ + --hash=sha256:f495a1652cf3fbab2eb0639776dad966c2fb874d79d87ca07f9d5f059b8bd215 \ + --hash=sha256:f496c9c3cc02230093d8330875c4c3cdfc3b73612a5fd921c65d39cbcef08063 \ + --hash=sha256:f59099f9b66f0d7145115e6f80dd8b1d847176df89b234a5a6b3f00437aa0832 \ + --hash=sha256:f59ad4c0e8f6bba240a9bb85504faa1ab438237199d4cce5f622761507b8f6a6 \ + --hash=sha256:fbccdc05410c9ee21bbf16a35f4c1d16123dcdeb8a1d38f33654fa21d0234f79 \ + --hash=sha256:fea24543955a6a729c45a73fe90e08c743f0b3334bbf3201e6c4bc1b0c7fa464 + # via requests +ec2-metadata==2.17.0 \ + --hash=sha256:a696a899cb8ee1f0101eb9ec7bc2050e1ffc8574ee4cd748a52d9c748fb19d9e \ + --hash=sha256:ad98a5afd8f4f49f688e4899dc505257da3237348761eceeb0fc7fc7d00cc90d + # via neuronx-cc +idna==3.16 \ + --hash=sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5 \ + --hash=sha256:d7a6da03db833450fca25d2358ac9ff06cd624577a4aea3a596d5c0f77b8e03d + # via requests +islpy==2023.2.5 \ + --hash=sha256:06446395028b88701fd84b685b1c82b9de493728906f96f576a4dca69d12dd1f \ + --hash=sha256:0c1689d9ef2ccfb3fcb3108a37c8fa21561dac3615ec05508021628a3197d81c \ + --hash=sha256:0f723e6d8fda7608e6329ee9a786b2812afefec01f6f45e4ac812594608d2990 \ + --hash=sha256:20484f2ef8d8f5827e18eaac064bfbd4d0e9cb877030ee1297a544bc80e126db \ + --hash=sha256:21571052bac1a319ef4937886559a41c85eddfc29389c7f9be6d8ddaa08dbd4a \ + --hash=sha256:24b469e1e390860cb78a13e5a933727fed42fd991eb6dbb758754410224a28a2 \ + --hash=sha256:2a64abecd9a966e6223633315e20b3abd9eb1faa8887122c675a35a7f4885212 \ + --hash=sha256:31c48b641824bd6b50e97659a6842c445feeb359115148c3debcbb862cb85c1f \ + --hash=sha256:3ee7815b1d49ac1efca0a42a5c038f88e87be01b9ddf598aadea0ef84b930a46 \ + --hash=sha256:3f50ec187bbd61cef4400af6885e65afd4092e3c4dc84b5d5c98554de6d06988 \ + --hash=sha256:4e97e23ba815afa37b8919ebb0d1c7ec7a9cb2c1491ea7ba89a886a8d213169c \ + --hash=sha256:58b5b86a9f5f58d0e8165c7e2215e7441eeea3466277e44f24788aef8b2b8311 \ + --hash=sha256:662a1c6f83feb77397eaef0027c5a57b98e249ecc454df6131c8cfe1541964dc \ + --hash=sha256:679431e17ad40d3c496d5fd3e9cbb824047775c00ade143dd59f848110b287f7 \ + --hash=sha256:69a13fa79fe9e901376e2c75bc49770e9027c55d9699077f8417e5bbe9e8b137 \ + --hash=sha256:6a336af9106eb92c3b7babe919d9e1e0a629143941d32d474e47cc55cf64f647 \ + --hash=sha256:6ed6c49803e53db5b0dbba9ec2fd3f896a8b9ea68f4a1cd203e5e7490f8e1c8e \ + --hash=sha256:773e85ba8768e14bd2b813d4a7e8e8140fe628cddddbd4fcb3452b023ed57f84 \ + --hash=sha256:7acbb16a6f55a6020cfcbd5823bbd3dd36ca03ef0713f9f7a47404b0aad7cba4 \ + --hash=sha256:80674c7a9aef8d2657872264dc59aeb66ce92067603ead43be51a56a6efc058a \ + --hash=sha256:819b16c4add84b42f227b451c2f773b8ae06489313d482771fc54c8806c1a2af \ + --hash=sha256:831b7a8f290af99e24addcb5cc5f4e73f7d047d50202ad84c7d208cadbe24f15 \ + --hash=sha256:a66c7baf86d6fcef8e2170483f45a241dca0811e0c78cc71efa30bc74bfc6bc8 \ + --hash=sha256:b27f1aa0ec0ab8182e47aaab4e363a42221ae00f9cba1f7cdde4d687120965f3 \ + --hash=sha256:b3582a7d19ee47f29c8c6b40643f69e5a52c0f0489ed179408cfd4d9616210c9 \ + --hash=sha256:b45c69990d0f7a9fbbba4a28b552c3b93c80a75d17b52032851f2dde13ebd661 \ + --hash=sha256:c0d67b1b8644ac8cf5dee3b2efbf8029891f5db1a625ee5259e2fcf6c33a2d55 \ + --hash=sha256:c2fdf13ae6465c511d75b42ed18b2289b49ecc7f1764e7fa24340b05d0392e80 \ + --hash=sha256:c6665d90beb3f37b2dcb95570bf8700e58884c5ad03dd131568287a639c03a5e \ + --hash=sha256:cea357471c345e804fa31c9f1eacfb3f67eda26b3e8fd417c33556822076565b \ + --hash=sha256:d35750ea2a3964325a2e67196806cf02ed771c2d2b2d47fb2bf60f58bf933cc4 \ + --hash=sha256:d5bedd79939dac4403b573abe5e13f6fc6a4502ebe8ed7de79b2fbfdd7856f3f \ + --hash=sha256:d75b4521f3291b92fc6796ebb7eabf5c6e318d44a578390f678d7bfaedde13b8 \ + --hash=sha256:d77acfe6653d044d14b8e93482e0282f42b947d40f5a98a13ee6b705936aaa18 \ + --hash=sha256:dd74398baf36938abb7c2a9d9828cc1a2e549c6c80485cec8b06b29abfaad18f \ + --hash=sha256:e33812a9922d967da814b17bc778a2c0dd1dabcd7ff685d86677bd745e8a3d65 \ + --hash=sha256:e372461fb5ba22f9955cdf180d6187ac691f0ffa972874af181bf60501c738cd \ + --hash=sha256:ea705d8b5244409fdc32affc689b010e88efbc8d378072bcba8edcf51c797de7 \ + --hash=sha256:ed0c62e64d160735ed1a7fc314d4c0f501711c282bb84a00ceaf590b4c4bd61f \ + --hash=sha256:eeec97588bd5282c983bd73f9010156e5e251e9e5307af547e2cb53b02d4639a + # via neuronx-cc +jmespath==1.1.0 \ + --hash=sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d \ + --hash=sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64 + # via + # boto3 + # botocore +libneuronxla==3.0.2891.0+e2a4b1f5 \ + --hash=sha256:b453709b37565902acc85bcfb4011d55e99381b81d831a5559847bba71f9cec3 + # via -r platforms/neuron/requirements.in +lockfile==0.12.2 \ + --hash=sha256:6aed02de03cba24efabcd600b30540140634fc06cfa603822d508d5361e9f799 \ + --hash=sha256:6c3cb24f344923d30b2785d5ad75182c8ea7ac1b6171b08657258ec7429d50fa + # via python-daemon +ml-dtypes==0.5.4 \ + --hash=sha256:0d2ffd05a2575b1519dc928c0b93c06339eb67173ff53acb00724502cda231cf \ + --hash=sha256:11942cbf2cf92157db91e5022633c0d9474d4dfd813a909383bd23ce828a4b7d \ + --hash=sha256:14a4fd3228af936461db66faccef6e4f41c1d82fcc30e9f8d58a08916b1d811f \ + --hash=sha256:19b9a53598f21e453ea2fbda8aa783c20faff8e1eeb0d7ab899309a0053f1483 \ + --hash=sha256:2314892cdc3fcf05e373d76d72aaa15fda9fb98625effa73c1d646f331fcecb7 \ + --hash=sha256:2b857d3af6ac0d39db1de7c706e69c7f9791627209c3d6dedbfca8c7e5faec22 \ + --hash=sha256:304ad47faa395415b9ccbcc06a0350800bc50eda70f0e45326796e27c62f18b6 \ + --hash=sha256:35f29491a3e478407f7047b8a4834e4640a77d2737e0b294d049746507af5175 \ + --hash=sha256:388d399a2152dd79a3f0456a952284a99ee5c93d3e2f8dfe25977511e0515270 \ + --hash=sha256:3bbbe120b915090d9dd1375e4684dd17a20a2491ef25d640a908281da85e73f1 \ + --hash=sha256:3d277bf3637f2a62176f4575512e9ff9ef51d00e39626d9fe4a161992f355af2 \ + --hash=sha256:4381fe2f2452a2d7589689693d3162e876b3ddb0a832cde7a414f8e1adf7eab1 \ + --hash=sha256:4ff7f3e7ca2972e7de850e7b8fcbb355304271e2933dd90814c1cb847414d6e2 \ + --hash=sha256:531eff30e4d368cb6255bc2328d070e35836aa4f282a0fb5f3a0cd7260257298 \ + --hash=sha256:533ce891ba774eabf607172254f2e7260ba5f57bdd64030c9a4fcfbd99815d0d \ + --hash=sha256:557a31a390b7e9439056644cb80ed0735a6e3e3bb09d67fd5687e4b04238d1de \ + --hash=sha256:5a0f68ca8fd8d16583dfa7793973feb86f2fbb56ce3966daf9c9f748f52a2049 \ + --hash=sha256:6a0df4223b514d799b8a1629c65ddc351b3efa833ccf7f8ea0cf654a61d1e35d \ + --hash=sha256:6c7ecb74c4bd71db68a6bea1edf8da8c34f3d9fe218f038814fd1d310ac76c90 \ + --hash=sha256:7c23c54a00ae43edf48d44066a7ec31e05fdc2eee0be2b8b50dd1903a1db94bb \ + --hash=sha256:805cef3a38f4eafae3a5bf9ebdcdb741d0bcfd9e1bd90eb54abd24f928cd2465 \ + --hash=sha256:88c982aac7cb1cbe8cbb4e7f253072b1df872701fcaf48d84ffbb433b6568f24 \ + --hash=sha256:8ab06a50fb9bf9666dd0fe5dfb4676fa2b0ac0f31ecff72a6c3af8e22c063453 \ + --hash=sha256:8c6a2dcebd6f3903e05d51960a8058d6e131fe69f952a5397e5dbabc841b6d56 \ + --hash=sha256:8c760d85a2f82e2bed75867079188c9d18dae2ee77c25a54d60e9cc79be1bc48 \ + --hash=sha256:9ad459e99793fa6e13bd5b7e6792c8f9190b4e5a1b45c63aba14a4d0a7f1d5ff \ + --hash=sha256:9bad06436568442575beb2d03389aa7456c690a5b05892c471215bfd8cf39460 \ + --hash=sha256:a174837a64f5b16cab6f368171a1a03a27936b31699d167684073ff1c4237dac \ + --hash=sha256:a7f7c643e8b1320fd958bf098aa7ecf70623a42ec5154e3be3be673f4c34d900 \ + --hash=sha256:a9b61c19040397970d18d7737375cffd83b1f36a11dd4ad19f83a016f736c3ef \ + --hash=sha256:b4b801ebe0b477be666696bda493a9be8356f1f0057a57f1e35cd26928823e5a \ + --hash=sha256:b95e97e470fe60ed493fd9ae3911d8da4ebac16bd21f87ffa2b7c588bf22ea2c \ + --hash=sha256:bc11d7e8c44a65115d05e2ab9989d1e045125d7be8e05a071a48bc76eb6d6040 \ + --hash=sha256:bfc534409c5d4b0bf945af29e5d0ab075eae9eecbb549ff8a29280db822f34f9 \ + --hash=sha256:c1a953995cccb9e25a4ae19e34316671e4e2edaebe4cf538229b1fc7109087b7 \ + --hash=sha256:cb73dccfc991691c444acc8c0012bee8f2470da826a92e3a20bb333b1a7894e6 \ + --hash=sha256:ce756d3a10d0c4067172804c9cc276ba9cc0ff47af9078ad439b075d1abdc29b \ + --hash=sha256:d81fdb088defa30eb37bf390bb7dde35d3a83ec112ac8e33d75ab28cc29dd8b0 \ + --hash=sha256:f21c9219ef48ca5ee78402d5cc831bd58ea27ce89beda894428bc67a52da5328 + # via + # neuronx-cc + # nki +networkx==2.8.8 \ + --hash=sha256:230d388117af870fce5647a3c52401fcf753e94720e6ea6b4197a5355648885e \ + --hash=sha256:e435dfa75b1d7195c7b8378c3859f0445cd88c6b0375c181ed66823a9ceb7524 + # via neuronx-cc +neuronx-cc==2.25.3371.0+f524f7f8 \ + --hash=sha256:c71cb7333d5320b6ea91e13e0168eec01be5e7ddfd5515bf62aa47088ee274c2 + # via + # -r platforms/neuron/requirements.in + # libneuronxla +nki==0.4.0+25940409122.gd30719f9 \ + --hash=sha256:09b520140e55790ba00ebe3e7d53da1e5f221ff3d6c1823414acfebe4b794879 + # via + # -r platforms/neuron/requirements.in + # neuronx-cc +numpy==2.4.6 \ + --hash=sha256:001fbb8e08d942dd57599e781f2472269ee7f2755fae407b4f67b2f0b17da3f1 \ + --hash=sha256:0280e0356c0829a18d9de1cb7eee50ec22ca639878d7240307ca0943d73cd2c4 \ + --hash=sha256:043191bfa8eab18c776647b62723ac9dddece59743b13f49b2016094129c2b3f \ + --hash=sha256:06ca2f61ec4385a07a6977c55ba998a4466c123642b4a32694d3128fce18c079 \ + --hash=sha256:0a041d3d761dc3c35cc56ce0351506a02bcbc25f7b169f652435141a17db9096 \ + --hash=sha256:0ab0a9c4ffb1a6d95ef519fe4247dba8eb6b18ad93999f76b7f657039acabd47 \ + --hash=sha256:0c9136e14ed34a9e343a31c533d78a9813a69a3148332bce5e9821cb2f996e66 \ + --hash=sha256:110f8b71aacb688ec69062bb7f6938a0f8acb01b7c1c4beb453c65b6d234584d \ + --hash=sha256:112b06a867b235ef466ed3508ddf0238050df9c727cafb5301ac385b899189a1 \ + --hash=sha256:17f9ade344e7d9b464a084d69bcf18fc691cb1db67c62ed80820bf4926d78f0e \ + --hash=sha256:1e254a00cdf42b1e4d5b3d68d33af63268d41340d8885df2ab6470f2e1500147 \ + --hash=sha256:1e978ec1e8bd0e0e4de6bb75de9d30cbb74db6b6a2bb727618613703ca0167dd \ + --hash=sha256:25c692919ac5a01f170a3bfcd62d745b24fd095c353d50812637d6fcab442e75 \ + --hash=sha256:260a5d70215b61ab4fadf5c7baacd64821842975eea312125ed3c39a6391b063 \ + --hash=sha256:2803abfebfc990042cd494d8ce2d5f82e9d847af6d35ec486923aa19dbad5e73 \ + --hash=sha256:29a287e0cf63ff528da061de6b9f64a4618da591ca1046aafc54062e40ca7eab \ + --hash=sha256:29cb7f67d10b479ff07c17d33e39f78c07f71c40ef30d63c153d340e96cd3fb4 \ + --hash=sha256:3213d622a0283a39a93d188f3cf72b26862df52fbb4ca3697f51705016523d41 \ + --hash=sha256:33111801a01c12a8a1e3721f0a9232f8cfc8ae2c6b7098167e6f623c6073f402 \ + --hash=sha256:357cc07a6d7b0b182ff02249616a03742827ebb1277546b5c7cd7f7620a45698 \ + --hash=sha256:38efbc8de75c7a0fc1ac190162d892787f3f47b57cc291231aafee36b80982b7 \ + --hash=sha256:4081eb135ac24158bd51cdfbef16f1c64df7063b1143f24731387137c092bec8 \ + --hash=sha256:40fdc1ae7125e518ea98e53e69a4ebc27e1fd50510c47b7ea130cf21e5e1d42b \ + --hash=sha256:4cfe66903cc32a9921a6733d96b19bb6abf310397581bbad89c228f5abaf0ee8 \ + --hash=sha256:511dbaf848decaaaf4b4ca48032619fb3138710c4bf7da7617765edad1ef96b0 \ + --hash=sha256:55cced7c52e981362f708ad635198e97a752dfba412cc03c23bbf3bd8d5cd662 \ + --hash=sha256:56b39e5e0622a09a25bf5baf62f4bcf0cb8a41ae6e2819cf49bbc5a74c083f91 \ + --hash=sha256:5dbbdb29840ca3d91ee0fece42fc29278886d908280bfec0a5846c6f901a3eb0 \ + --hash=sha256:5f9fb9157b4ce2971008323afe46053787b526ef624fea915b261468a8421a0f \ + --hash=sha256:6180d8b35af935aed8ece3a85e0a43f87393ae0ac87c8d2c8bd2c993f7270ef3 \ + --hash=sha256:68a5124b13fa6cc2086764a20005d30bc0548146f7f5322f02fce212ca14317f \ + --hash=sha256:68bb27509ac1b9a3443094260f6326150663b06abe40b73a2f81160623da5b67 \ + --hash=sha256:6f41ae150c4e32db4f3310cdaf64b1593a03dbabe29eec77fc9b50fe64061df6 \ + --hash=sha256:7265a2f3d436e54ef9f2b52b5c937e6be778781bd97a590319d7348f1c1ca997 \ + --hash=sha256:72fbe16c6fac95aedf5937fa873445cec2110be35d8a4e9433d7501fd98dae6b \ + --hash=sha256:7d92c3819208a60205a12a245c91ad70cb0a85336659b19b834205573ac8456e \ + --hash=sha256:8155154c7c691289fe18f510b5d4657c68c67989f293f0535a91360392ff6538 \ + --hash=sha256:81a1cca95ed5bb92aa8b10dd2cdc9a0d3853a50fad926c28b5d7e8ea54389627 \ + --hash=sha256:89cd468399cfd2504718f0ba50e410dca55a170b61a02ad92bb18c8a65186e93 \ + --hash=sha256:8ad03c0965fb3c692200e74d458ca28c1dbb4ce96f9a479a8aa041ad5fabca02 \ + --hash=sha256:90f9849678c75fe7afa2d348ac842c168b0a4d3d61919687216dfc547976d853 \ + --hash=sha256:948424b06129ce883307e8cff868c31396d8dc7630a59c61d70d98dbe70f222c \ + --hash=sha256:9cd5ffd25db4e7ba6a375693b3fc0fc1791ec636c17db3720da19bde7180ec43 \ + --hash=sha256:a0df0043bdb289bde1f62da130d20df23d58b45429f752bc7a8fc5325a225ecd \ + --hash=sha256:a2c306dea656c12c68f51f4cea133cbe78ca7435eb28c735eac1d3ebe73be6e8 \ + --hash=sha256:a7830bab239b79cda9c08c2da014761cafb48da6150e1da17ac06283f43b6089 \ + --hash=sha256:a7c711e21628b52034bb5ab8d1bce291f752fcc5e92accc615778acee1ff4778 \ + --hash=sha256:aaf159caa35993cb1f56fb9b8e4610d35758e7ca005412eb1daa856a78c9c4b1 \ + --hash=sha256:ae506e6902902557576a26ff33eda8695e7ecb3cb36c3b573a0765dee114ebdb \ + --hash=sha256:b507f5c4c1d508876d1819b6bf9a49d365b96320b5d4993426b33a23ca4b8261 \ + --hash=sha256:bf162abab1c1a736333192707cef898e735a5ca00f38f27eeedf44b39d9e85eb \ + --hash=sha256:c1a2af6c6ef86344a6b0db6b97834208bf598db514f2b155042439b62605601a \ + --hash=sha256:c2d37ab77531417474168eb79d6d80b14f821a966818505d03013d0833edb7a8 \ + --hash=sha256:c4fc99836233ea196540b17ab0983aff60ed07941751930f5f4d05bc3b3b7359 \ + --hash=sha256:d581b735e177fdcdce6fed8e7e8880a3fb6ee4e3653a3ac6af01c6f4c03effc5 \ + --hash=sha256:d6da64deb6b8ed903e7560180a92f2d804ee1ba5eeb849ac2748b8c1aba1f6d7 \ + --hash=sha256:d8e8286dd7cea7895157318d1b91cdacac64c479f3cbc8dce548331728484751 \ + --hash=sha256:ddea102b48f9e339f3948bf22040944184627a30fdf7f858667673b9c5f033c8 \ + --hash=sha256:dfa20cc6ca228e6b155b11da03825975ce66aea520985dbbddf0f2a5a495c605 \ + --hash=sha256:e3e5193ef5a3dc73bceee50f7fdc2c90dbb76c42df8d8fae3d1067a583df579e \ + --hash=sha256:e3eeb0aabd6bd5ce64faae67e9935203a6991b4bc2a485a767fbafb2c5125f45 \ + --hash=sha256:e5805d5a22fd19c8ccff10a9561f9df94436b0545619ea579db2d3c35294bce2 \ + --hash=sha256:e85b752a1e912b70eaad4fafbd4d1238007ab221de2009b9a2f5ae7461239895 \ + --hash=sha256:eaf7fa2de5c0be8ae6ff8e9bea2ccd725e980541244521d8d4b5f3354a27babe \ + --hash=sha256:ebfb099f8dcf083deef3ac1ca4c1503f387cf76296fcb3816b66f5ecb5f54fdb \ + --hash=sha256:ece3d2cfe132e7d51f44a832b303895e6f2d499c5e74dfbdb06ee246147a304a \ + --hash=sha256:ed9749eef4cbd126da3dc1d6bcb3a57f5eb7ac6a6484146bdbf743f552dfc577 \ + --hash=sha256:ede83e07a75dd06bc501566c1eca2afc0d61677c1472ac9ad93fdee6e638a48d \ + --hash=sha256:ef4aea96ce4d3b074422cb4f2f64e216bf9e213004bb58ecfdf50ea02ea8eb9a \ + --hash=sha256:f3a3570c4a2a16746ac2c31a7c7c7b0c186b95ce902e33db6f28094ed7387dda \ + --hash=sha256:f407cb6b8e9d6d8c626bc73c945db1706035af8fd632295547bf1c9e46d092d6 \ + --hash=sha256:f74a575920ab21fe304421a3fc28793d82e299cae9eccb37084e9fc7f3617c20 + # via + # ml-dtypes + # neuronx-cc + # nki + # scipy +pgzip==0.4.0 \ + --hash=sha256:41e44c03a9f1bbbd390dd6b6e37a11484113f8c159a049fa81cd8a9ddde82a21 \ + --hash=sha256:91f6526f0c6e1f6c2d3522707777e5843fad7c791401745593fd01bd869b7a05 + # via neuronx-cc +protobuf==7.35.0 \ + --hash=sha256:4c4617b83ade0e279d1d2bfe04025a1adb87f9ed657de038620dc0ff959357f6 \ + --hash=sha256:4cbf5cc286130e06a6c9bbefac442431173906dfcc979712183d4adcc01b37ee \ + --hash=sha256:66be6c513931c794fa92c080ffee41671390da3d79da219cf9c0c0907f035dda \ + --hash=sha256:6c0f98f10c8a05ea30f8993dfef2de093d27b490fdae78bb60c8343795d55011 \ + --hash=sha256:a2efd84605f41e559f1881b0912b44099d0a2ac9bf46b3474823f10fb393b0e6 \ + --hash=sha256:c13f325cf242bad135c350629eeb5d54b24228eb472fb3e2e9ebbd4c5dc20ca0 \ + --hash=sha256:f05bcadf9a2a6b8dda047007075135fb7d08c73d9177aabc067e1be46881a201 \ + --hash=sha256:fcbe42a4ac09d3ec9c987ddfcd956afd0b15f1ff613bd8371bde9405ffd5c8e5 + # via neuronx-cc +psutil==7.2.2 \ + --hash=sha256:0746f5f8d406af344fd547f1c8daa5f5c33dbc293bb8d6a16d80b4bb88f59372 \ + --hash=sha256:076a2d2f923fd4821644f5ba89f059523da90dc9014e85f8e45a5774ca5bc6f9 \ + --hash=sha256:11fe5a4f613759764e79c65cf11ebdf26e33d6dd34336f8a337aa2996d71c841 \ + --hash=sha256:1a571f2330c966c62aeda00dd24620425d4b0cc86881c89861fbc04549e5dc63 \ + --hash=sha256:1a7b04c10f32cc88ab39cbf606e117fd74721c831c98a27dc04578deb0c16979 \ + --hash=sha256:1fa4ecf83bcdf6e6c8f4449aff98eefb5d0604bf88cb883d7da3d8d2d909546a \ + --hash=sha256:2edccc433cbfa046b980b0df0171cd25bcaeb3a68fe9022db0979e7aa74a826b \ + --hash=sha256:7b6d09433a10592ce39b13d7be5a54fbac1d1228ed29abc880fb23df7cb694c9 \ + --hash=sha256:8c233660f575a5a89e6d4cb65d9f938126312bca76d8fe087b947b3a1aaac9ee \ + --hash=sha256:917e891983ca3c1887b4ef36447b1e0873e70c933afc831c6b6da078ba474312 \ + --hash=sha256:ab486563df44c17f5173621c7b198955bd6b613fb87c71c161f827d3fb149a9b \ + --hash=sha256:ae0aefdd8796a7737eccea863f80f81e468a1e4cf14d926bd9b6f5f2d5f90ca9 \ + --hash=sha256:b0726cecd84f9474419d67252add4ac0cd9811b04d61123054b9fb6f57df6e9e \ + --hash=sha256:b58fabe35e80b264a4e3bb23e6b96f9e45a3df7fb7eed419ac0e5947c61e47cc \ + --hash=sha256:c7663d4e37f13e884d13994247449e9f8f574bc4655d509c3b95e9ec9e2b9dc1 \ + --hash=sha256:e452c464a02e7dc7822a05d25db4cde564444a67e58539a00f929c51eddda0cf \ + --hash=sha256:e78c8603dcd9a04c7364f1a3e670cea95d51ee865e4efb3556a3a63adef958ea \ + --hash=sha256:eb7e81434c8d223ec4a219b5fc1c47d0417b12be7ea866e24fb5ad6e84b3d988 \ + --hash=sha256:ed0cace939114f62738d808fdcecd4c869222507e266e574799e9c0faa17d486 \ + --hash=sha256:eed63d3b4d62449571547b60578c5b2c4bcccc5387148db46e0c2313dad0ee00 \ + --hash=sha256:fd04ef36b4a6d599bbdb225dd1d3f51e00105f6d48a28f006da7f9822f2606d8 + # via neuronx-cc +python-daemon==3.1.2 \ + --hash=sha256:b906833cef63502994ad48e2eab213259ed9bb18d54fa8774dcba2ff7864cec6 \ + --hash=sha256:f7b04335adc473de877f5117e26d5f1142f4c9f7cd765408f0877757be5afbf4 + # via neuronx-cc +python-dateutil==2.9.0.post0 \ + --hash=sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3 \ + --hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427 + # via botocore +requests==2.34.2 \ + --hash=sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0 \ + --hash=sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed + # via + # ec2-metadata + # requests-unixsocket +requests-unixsocket==0.4.1 \ + --hash=sha256:60c4942e9dbecc2f64d611039fb1dfc25da382083c6434ac0316dca3ff908f4d \ + --hash=sha256:b2596158c356ecee68d27ba469a52211230ac6fb0cde8b66afb19f0ed47a1995 + # via neuronx-cc +s3transfer==0.17.0 \ + --hash=sha256:9edeb6d1c3c2f89d6050348548834ad8289610d886e5bf7b7207728bd43ce33a \ + --hash=sha256:ce3801712acf4ad3e89fb9990df97b4972e93f4b3b0004d214be5bce12814c20 + # via boto3 +scipy==1.17.1 \ + --hash=sha256:010f4333c96c9bb1a4516269e33cb5917b08ef2166d5556ca2fd9f082a9e6ea0 \ + --hash=sha256:02ae3b274fde71c5e92ac4d54bc06c42d80e399fec704383dcd99b301df37458 \ + --hash=sha256:08b900519463543aa604a06bec02461558a6e1cef8fdbb8098f77a48a83c8118 \ + --hash=sha256:131f5aaea57602008f9822e2115029b55d4b5f7c070287699fe45c661d051e39 \ + --hash=sha256:158dd96d2207e21c966063e1635b1063cd7787b627b6f07305315dd73d9c679e \ + --hash=sha256:1cc682cea2ae55524432f3cdff9e9a3be743d52a7443d0cba9017c23c87ae2f6 \ + --hash=sha256:1f95b894f13729334fb990162e911c9e5dc1ab390c58aa6cbecb389c5b5e28ec \ + --hash=sha256:200e1050faffacc162be6a486a984a0497866ec54149a01270adc8a59b7c7d21 \ + --hash=sha256:2040ad4d1795a0ae89bfc7e8429677f365d45aa9fd5e4587cf1ea737f927b4a1 \ + --hash=sha256:2b64ca7d4aee0102a97f3ba22124052b4bd2152522355073580bf4845e2550b6 \ + --hash=sha256:2ceb2d3e01c5f1d83c4189737a42d9cb2fc38a6eeed225e7515eef71ad301dce \ + --hash=sha256:35c3a56d2ef83efc372eaec584314bd0ef2e2f0d2adb21c55e6ad5b344c0dcb8 \ + --hash=sha256:37425bc9175607b0268f493d79a292c39f9d001a357bebb6b88fdfaff13f6448 \ + --hash=sha256:3877ac408e14da24a6196de0ddcace62092bfc12a83823e92e49e40747e52c19 \ + --hash=sha256:3fd1fcdab3ea951b610dc4cef356d416d5802991e7e32b5254828d342f7b7e0b \ + --hash=sha256:41b71f4a3a4cab9d366cd9065b288efc4d4f3c0b37a91a8e0947fb5bd7f31d87 \ + --hash=sha256:43af8d1f3bea642559019edfe64e9b11192a8978efbd1539d7bc2aaa23d92de4 \ + --hash=sha256:45abad819184f07240d8a696117a7aacd39787af9e0b719d00285549ed19a1e9 \ + --hash=sha256:4b400bdc6f79fa02a4d86640310dde87a21fba0c979efff5248908c6f15fad1b \ + --hash=sha256:4eb6c25dd62ee8d5edf68a8e1c171dd71c292fdae95d8aeb3dd7d7de4c364082 \ + --hash=sha256:581b2264fc0aa555f3f435a5944da7504ea3a065d7029ad60e7c3d1ae09c5464 \ + --hash=sha256:5cf36e801231b6a2059bf354720274b7558746f3b1a4efb43fcf557ccd484a87 \ + --hash=sha256:5e3c5c011904115f88a39308379c17f91546f77c1667cea98739fe0fccea804c \ + --hash=sha256:6609bc224e9568f65064cfa72edc0f24ee6655b47575954ec6339534b2798369 \ + --hash=sha256:6e3dcd57ab780c741fde8dc68619de988b966db759a3c3152e8e9142c26295ad \ + --hash=sha256:6fac755ca3d2c3edcb22f479fceaa241704111414831ddd3bc6056e18516892f \ + --hash=sha256:744b2bf3640d907b79f3fd7874efe432d1cf171ee721243e350f55234b4cec4c \ + --hash=sha256:74cbb80d93260fe2ffa334efa24cb8f2f0f622a9b9febf8b483c0b865bfb3475 \ + --hash=sha256:766e0dc5a616d026a3a1cffa379af959671729083882f50307e18175797b3dfd \ + --hash=sha256:7bdf2da170b67fdf10bca777614b1c7d96ae3ca5794fd9587dce41eb2966e866 \ + --hash=sha256:7ff200bf9d24f2e4d5dc6ee8c3ac64d739d3a89e2326ba68aaf6c4a2b838fd7d \ + --hash=sha256:844e165636711ef41f80b4103ed234181646b98a53c8f05da12ca5ca289134f6 \ + --hash=sha256:8a604bae87c6195d8b1045eddece0514d041604b14f2727bbc2b3020172045eb \ + --hash=sha256:94055a11dfebe37c656e70317e1996dc197e1a15bbcc351bcdd4610e128fe1ca \ + --hash=sha256:95d8e012d8cb8816c226aef832200b1d45109ed4464303e997c5b13122b297c0 \ + --hash=sha256:9cdc1a2fcfd5c52cfb3045feb399f7b3ce822abdde3a193a6b9a60b3cb5854ca \ + --hash=sha256:9ecb4efb1cd6e8c4afea0daa91a87fbddbce1b99d2895d151596716c0b2e859d \ + --hash=sha256:a3472cfbca0a54177d0faa68f697d8ba4c80bbdc19908c3465556d9f7efce9ee \ + --hash=sha256:a4328d245944d09fd639771de275701ccadf5f781ba0ff092ad141e017eccda4 \ + --hash=sha256:a48a72c77a310327f6a3a920092fa2b8fd03d7deaa60f093038f22d98e096717 \ + --hash=sha256:a720477885a9d2411f94a93d16f9d89bad0f28ca23c3f8daa521e2dcc3f44d49 \ + --hash=sha256:a77cbd07b940d326d39a1d1b37817e2ee4d79cb30e7338f3d0cddffae70fcaa2 \ + --hash=sha256:a9956e4d4f4a301ebf6cde39850333a6b6110799d470dbbb1e25326ac447f52a \ + --hash=sha256:adb2642e060a6549c343603a3851ba76ef0b74cc8c079a9a58121c7ec9fe2350 \ + --hash=sha256:beeda3d4ae615106d7094f7e7cef6218392e4465cc95d25f900bebabfded0950 \ + --hash=sha256:c80be5ede8f3f8eded4eff73cc99a25c388ce98e555b17d31da05287015ffa5b \ + --hash=sha256:cc90d2e9c7e5c7f1a482c9875007c095c3194b1cfedca3c2f3291cdc2bc7c086 \ + --hash=sha256:cd96a1898c0a47be4520327e01f874acfd61fb48a9420f8aa9f6483412ffa444 \ + --hash=sha256:d2650c1fb97e184d12d8ba010493ee7b322864f7d3d00d3f9bb97d9c21de4068 \ + --hash=sha256:d30e57c72013c2a4fe441c2fcb8e77b14e152ad48b5464858e07e2ad9fbfceff \ + --hash=sha256:d59c30000a16d8edc7e64152e30220bfbd724c9bbb08368c054e24c651314f0a \ + --hash=sha256:dbc12c9f3d185f5c737d801da555fb74b3dcfa1a50b66a1a93e09190f41fab50 \ + --hash=sha256:e18f12c6b0bc5a592ed23d3f7b891f68fd7f8241d69b7883769eb5d5dfb52696 \ + --hash=sha256:e19ebea31758fac5893a2ac360fedd00116cbb7628e650842a6691ba7ca28a21 \ + --hash=sha256:e30bdeaa5deed6bc27b4cc490823cd0347d7dae09119b8803ae576ea0ce52e4c \ + --hash=sha256:eb092099205ef62cd1782b006658db09e2fed75bffcae7cc0d44052d8aa0f484 \ + --hash=sha256:eee2cfda04c00a857206a4330f0c5e3e56535494e30ca445eb19ec624ae75118 \ + --hash=sha256:f4115102802df98b2b0db3cce5cb9b92572633a1197c77b7553e5203f284a5b3 \ + --hash=sha256:f590cd684941912d10becc07325a3eeb77886fe981415660d9265c4c418d0bea \ + --hash=sha256:f8885db0bc2bffa59d5c1b72fad7a6a92d3e80e7257f967dd81abb553a90d293 \ + --hash=sha256:fcb310ddb270a06114bb64bbe53c94926b943f5b7f0842194d585c65eb4edd76 + # via neuronx-cc +six==1.17.0 \ + --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \ + --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81 + # via python-dateutil +tqdm==4.67.3 \ + --hash=sha256:7d825f03f89244ef73f1d4ce193cb1774a8179fd96f31d7e1dcde62092b960bb \ + --hash=sha256:ee1e4c0e59148062281c49d80b25b67771a127c85fc9676d3be5f243206826bf + # via neuronx-cc +typing-extensions==4.15.0 \ + --hash=sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466 \ + --hash=sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548 + # via neuronx-cc +urllib3==2.7.0 \ + --hash=sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c \ + --hash=sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897 + # via + # botocore + # requests diff --git a/platforms/neuron/topology.zig b/platforms/neuron/topology.zig new file mode 100644 index 00000000..201db15f --- /dev/null +++ b/platforms/neuron/topology.zig @@ -0,0 +1,222 @@ +const std = @import("std"); + +const neuron = @import("platforms/neuron"); + +// References: +// - `neuron.Instance` exposes the NRT instance family and size used below. +// - Neuron instance families and sizes are documented here: +// https://awsdocs-neuron.readthedocs-hosted.com/en/latest/about-neuron/arch/index.html +// - Inf1: +// https://awsdocs-neuron.readthedocs-hosted.com/en/latest/about-neuron/arch/neuron-hardware/inf1-arch.html +// - Inf2: +// https://awsdocs-neuron.readthedocs-hosted.com/en/latest/about-neuron/arch/neuron-hardware/inf2-arch.html +// - Trn1 / Trn1n: +// https://awsdocs-neuron.readthedocs-hosted.com/en/latest/about-neuron/arch/neuron-hardware/trn1-arch.html +// - Trn2 / Trn2 UltraServer: +// https://awsdocs-neuron.readthedocs-hosted.com/en/latest/about-neuron/arch/neuron-hardware/trn2-arch.html +// - Trn3: +// https://awsdocs-neuron.readthedocs-hosted.com/en/latest/about-neuron/arch/neuron-hardware/trn3-arch.html +// +// The same instance-architecture pages are also the basis for the inter-chip +// topology inference below, including 2D torus and ring-over-torus fabrics. +pub const InterChipTopology = union(enum) { + none: void, + linear: usize, + ring: usize, + torus2d: struct { + x: usize, + y: usize, + }, + torus2d_ring: struct { + x: usize, + y: usize, + z: usize, + }, + point_to_point: usize, +}; + +pub const Axis = struct { + fabric: Fabric, + size: usize, + geometry: Geometry, +}; + +pub const Fabric = enum { + core, + chip_x, + chip_y, + chip_z, +}; + +pub const Geometry = enum { + linear_ring, + closed_ring, + torus, + point_to_point, +}; + +pub const Placement = struct { + input_index: usize, + nc_id: usize, + chip: usize, + core: usize, +}; + +pub const VisibleMesh = struct { + instance: neuron.Instance, + chips: usize, + cores_per_chip: usize, + inter_chip_topology: InterChipTopology, + placements: []Placement, + axes_buffer: [4]Axis, + axes_len: usize, + + pub fn deinit(self: *VisibleMesh, allocator: std.mem.Allocator) void { + allocator.free(self.placements); + self.* = undefined; + } + + pub fn axes(self: *const VisibleMesh) []const Axis { + return self.axes_buffer[0..self.axes_len]; + } + + pub fn format(self: VisibleMesh, writer: *std.Io.Writer) !void { + try writer.print( + "VisibleMesh(family={s} size={s} chips={d} cores_per_chip={d} inter_chip_topology=", + .{ + @tagName(self.instance.family), + @tagName(self.instance.size), + self.chips, + self.cores_per_chip, + }, + ); + switch (self.inter_chip_topology) { + .none => try writer.writeAll("none"), + .linear => |size| try writer.print("linear({d})", .{size}), + .ring => |size| try writer.print("ring({d})", .{size}), + .torus2d => |dims| try writer.print("torus2d({d}x{d})", .{ dims.x, dims.y }), + .torus2d_ring => |dims| try writer.print("torus2d_ring({d}x{d}x{d})", .{ dims.x, dims.y, dims.z }), + .point_to_point => |size| try writer.print("point_to_point({d})", .{size}), + } + try writer.writeAll(")"); + } +}; + +pub fn visibleMeshFromNcIds(allocator: std.mem.Allocator, nc_ids: []const usize) !VisibleMesh { + if (nc_ids.len == 0) return error.InvalidDeviceTopology; + + const instance = try neuron.instance(); + const physical_cores_per_chip = try instance.coresPerChip(); + const placements = try allocator.alloc(Placement, nc_ids.len); + errdefer allocator.free(placements); + + for (nc_ids, placements, 0..) |nc_id, *placement, input_index| { + placement.* = .{ + .input_index = input_index, + .nc_id = nc_id, + .chip = chipOf(nc_id, physical_cores_per_chip), + .core = coreOf(nc_id, physical_cores_per_chip), + }; + } + + const Sort = struct { + fn lessThan(_: void, a: Placement, b: Placement) bool { + return if (a.chip == b.chip) a.core < b.core else a.chip < b.chip; + } + }; + std.mem.sort(Placement, placements, {}, Sort.lessThan); + + const first_chip = placements[0].chip; + const last_chip = placements[placements.len - 1].chip; + const selected_chip_count = last_chip - first_chip + 1; + const selected_cores_per_chip = try std.math.divExact(usize, placements.len, selected_chip_count); + const inter_chip_topology = interChipTopology(instance, selected_chip_count); + + var res: VisibleMesh = .{ + .instance = instance, + .chips = selected_chip_count, + .cores_per_chip = selected_cores_per_chip, + .inter_chip_topology = inter_chip_topology, + .placements = placements, + .axes_buffer = undefined, + .axes_len = 0, + }; + + switch (inter_chip_topology) { + .none => {}, + .linear => |size| appendAxis(&res, .chip_x, size, .linear_ring), + .ring => |size| appendAxis(&res, .chip_x, size, .closed_ring), + .torus2d => |dims| { + appendAxis(&res, .chip_x, dims.x, .torus); + appendAxis(&res, .chip_y, dims.y, .torus); + }, + .torus2d_ring => |dims| { + appendAxis(&res, .chip_x, dims.x, .torus); + appendAxis(&res, .chip_y, dims.y, .torus); + appendAxis(&res, .chip_z, dims.z, .closed_ring); + }, + .point_to_point => |size| appendAxis(&res, .chip_x, size, .point_to_point), + } + + // Inter-chip topology only describes NeuronLink fabric between chips. Add a + // local core axis for multi-core chips, and keep single-chip meshes non-empty. + if (res.cores_per_chip > 1 or res.axes_len == 0) { + appendAxis(&res, .core, res.cores_per_chip, .point_to_point); + } + + return res; +} + +pub fn interChipTopology(instance: neuron.Instance, chip_count: usize) InterChipTopology { + if (chip_count == 1) return .none; + + // References for chip-count and fabric-shape inference: + // - Inf1 instance sizes and 1/4/16-chip layouts + // - Inf2 instance sizes and 1/6/12-chip layouts + // - Trn1 / Trn1n 16-chip 2D torus + // - Trn2 16-chip torus and Trn2 UltraServer 64-chip ring-over-torus + // - Trn3 all-to-all / point-to-point scale-up + return switch (instance.family) { + .inf1 => switch (chip_count) { + 4 => .{ .linear = 4 }, + else => .{ .ring = chip_count }, + }, + .inf2, .inf2e => switch (chip_count) { + 6 => .{ .torus2d = .{ .x = 3, .y = 2 } }, + 12 => .{ .torus2d = .{ .x = 4, .y = 3 } }, + else => switch (instance.size) { + .xl24 => .{ .torus2d = .{ .x = 3, .y = 2 } }, + .xl48 => .{ .torus2d = .{ .x = 4, .y = 3 } }, + else => .{ .ring = chip_count }, + }, + }, + .trn1, .trn1n => switch (chip_count) { + 16 => .{ .torus2d = .{ .x = 4, .y = 4 } }, + else => .{ .ring = chip_count }, + }, + .trn2, .trn2n, .trn2p, .trn2u, .trn2e, .trn2eu, .trn2ac, .trn2uac => switch (chip_count) { + 16 => .{ .torus2d = .{ .x = 4, .y = 4 } }, + 64 => .{ .torus2d_ring = .{ .x = 4, .y = 4, .z = 4 } }, + else => .{ .ring = chip_count }, + }, + .trn3, .trn3pds98 => .{ .point_to_point = chip_count }, + .unknown => .{ .ring = chip_count }, + }; +} + +pub fn chipOf(nc_id: usize, physical_cores_per_chip: usize) usize { + return @divFloor(nc_id, physical_cores_per_chip); +} + +pub fn coreOf(nc_id: usize, physical_cores_per_chip: usize) usize { + return @mod(nc_id, physical_cores_per_chip); +} + +fn appendAxis(self: *VisibleMesh, fabric: Fabric, size: usize, geometry: Geometry) void { + self.axes_buffer[self.axes_len] = .{ + .fabric = fabric, + .size = size, + .geometry = geometry, + }; + self.axes_len += 1; +} diff --git a/platforms/oneapi/BUILD.bazel b/platforms/oneapi/BUILD.bazel new file mode 100644 index 00000000..aaa06989 --- /dev/null +++ b/platforms/oneapi/BUILD.bazel @@ -0,0 +1,39 @@ +load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_shared_library") + +zig_shared_library( + name = "zmlxoneapi", + main = "zmlxoneapi.zig", + shared_lib_name = "libzmlxoneapi.so.0", + target_compatible_with = ["@platforms//os:linux"], + visibility = ["@libzml_oneapi//:__subpackages__"], + zigopts = ["-fno-stack-check"], + deps = ["//stdx"], +) + +cc_library( + name = "libzml_oneapi", + hdrs = ["oneapi.h"], + tags = ["manual"], + deps = ["@libzml_oneapi"], +) + +zig_library( + name = "oneapi", + import_name = "platforms/oneapi", + main = "oneapi.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//bazel", + "//pjrt", + "//platforms:options", + "//stdx", + "@rules_zig//zig/runfiles", + ] + select({ + "//platforms:oneapi.enabled": [ + ":libzml_oneapi", + ], + "//conditions:default": [], + }), +) diff --git a/platforms/oneapi/README.md b/platforms/oneapi/README.md new file mode 100644 index 00000000..d450bba6 --- /dev/null +++ b/platforms/oneapi/README.md @@ -0,0 +1,54 @@ +# EXPERIMENTAL: oneAPI + + +# About + +This directory contains the oneAPI backend for ZML. It is currently in an early stage of development, and only supports a subset of ZML's features. The oneAPI backend is only available on Linux, and requires an Intel GPU. + +The sharding is not yet supported, so only models that fit in the GPU memory can be run. + +# Getting Started + +## Prerequisites + +- An Intel GPU +- bazel see [Getting Started](../README.md#getting-started) for installation instructions. + +### Linux Only + +One GPU is supported on Linux. +ONEAPI_DEVICE_SELECTOR environment variable must be set to select the GPU device. + +For example, to select the first GPU: +```bash +export ONEAPI_DEVICE_SELECTOR=level_zero:0 +``` + +#### Run + + +To run the LLM example with oneAPI and the Llama 3.2 1B model: +```bash +ONEAPI_DEVICE_SELECTOR=level_zero:0 \ +bazel run //examples/llm \ + --config=release \ + --@zml//platforms:cpu=false \ + --@zml//platforms:oneapi=true \ + -- \ + --model=hf://meta-llama/Llama-3.2-1B-Instruct \ + --topk=1 \ + --prompt="Tell me a story about a cat in 2 lines" +``` + +To run the LLM example with oneAPI and a local model: +```bash +ONEAPI_DEVICE_SELECTOR=level_zero:0 \ +bazel run //examples/llm \ + --config=release \ + --@zml//platforms:cpu=false \ + --@zml//platforms:oneapi=true \ + -- \ + --model=path/to/your/model/Llama-3.1-8B-Instruct \ + --topk=1 \ + --prompt="Tell me a story about a cat in 2 lines" +``` diff --git a/platforms/oneapi/libzml_oneapi.BUILD.bazel b/platforms/oneapi/libzml_oneapi.BUILD.bazel new file mode 100644 index 00000000..eb08e652 --- /dev/null +++ b/platforms/oneapi/libzml_oneapi.BUILD.bazel @@ -0,0 +1,154 @@ +load("@bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") +load("@zml//bazel:patchelf.bzl", "patchelf") + +patchelf( + name = "libzml_oneapi_so", + src = "lib/libzml_oneapi.so", + add_needed = [ + "libzmlxoneapi.so.0", + ], + rename_dynamic_symbols = { + "dlopen": "zmlxoneapi_dlopen", + }, + set_rpath = "$ORIGIN", + soname = "libzml_oneapi.so", +) + +patchelf( + name = "libdnnl_so_3", + src = "@intel-oneapi-dnnl-2026.0//:libdnnl_so_3", + set_rpath = "$ORIGIN", + soname = "libdnnl.so.3", +) + +patchelf( + name = "libtbb_so_12", + src = "@intel-oneapi-tbb-2023.1//:libtbb_so_12", + set_rpath = "$ORIGIN", + soname = "libtbb.so.12", +) + +patchelf( + name = "libsycl_so", + src = "@intel-oneapi-compiler-dpcpp-cpp-runtime-2026.1//:libsycl_so", + set_rpath = "$ORIGIN", +) + +patchelf( + name = "libccl_legacy_so", + src = "@intel-oneapi-ccl-2022.1//:libccl_legacy_so", + set_rpath = "$ORIGIN", +) + +patchelf( + name = "libze_intel_gpu_so", + src = "@libze-intel-gpu1//:libze_intel_gpu_so", + set_rpath = "$ORIGIN", + soname = "libze_intel_gpu.so.1", +) + +patchelf( + name = "libigc_so_2", + src = "@libigc2//:libigc_so_2", + set_rpath = "$ORIGIN", + soname = "libigc.so.2", +) + +patchelf( + name = "libiga64_so_2", + src = "@libigc2//:libiga64_so_2", + set_rpath = "$ORIGIN", + soname = "libiga64.so.2", +) + +patchelf( + name = "libigdfcl_so_2", + src = "@libigdfcl2//:libigdfcl_so_2", + set_rpath = "$ORIGIN", + soname = "libigdfcl.so.2", +) + +ONEAPI_VERSION = "2026.1" + +ONEAPI_CCL_LIB = "ccl/2022.1/lib" + +ONEAPI_MPI_LIB = "mpi/2021.18/lib" + +ONEAPI_MPI_LIBFABRIC_LIB = "mpi/2021.18/opt/mpi/libfabric/lib" + +ONEAPI_TCM_VERSION = "1.5" + +ONEAPI_UMF_VERSION = "1.1" + +ONEAPI_COMPILER_LIB = "compiler/{}/lib".format(ONEAPI_VERSION) + +ONEAPI_MKL_LIB = "mkl/{}/lib".format(ONEAPI_VERSION) + +ONEAPI_TCM_LIB = "tcm/{}/lib".format(ONEAPI_TCM_VERSION) + +ONEAPI_UMF_LIB = "umf/{}/lib".format(ONEAPI_UMF_VERSION) + +copy_to_directory( + name = "sandbox", + srcs = [ + ":libdnnl_so_3", + ":libiga64_so_2", + ":libigc_so_2", + ":libigdfcl_so_2", + ":libsycl_so", + ":libtbb_so_12", + ":libze_intel_gpu_so", + ":libzml_oneapi_so", + "@intel-oneapi-ccl-2022.1//:ccl_runtime", + "@intel-oneapi-compiler-dpcpp-cpp-runtime-2026.1//:sycl_runtime", + "@intel-oneapi-compiler-shared-runtime-2026.1//:compiler_runtime", + "@intel-oneapi-mkl-core-2026.1//:mkl_core_runtime", + "@intel-oneapi-mkl-sycl-blas-2026.1//:mkl_sycl_blas", + "@intel-oneapi-mkl-sycl-dft-2026.1//:mkl_sycl_dft", + "@intel-oneapi-mkl-sycl-lapack-2026.1//:mkl_sycl_lapack", + "@intel-oneapi-mkl-sycl-rng-2026.1//:mkl_sycl_rng", + "@intel-oneapi-mkl-sycl-sparse-2026.1//:mkl_sycl_sparse", + "@intel-oneapi-mpi-2021.18//:mpi_runtime", + "@intel-oneapi-tcm-1.5//:hwloc", + "@intel-oneapi-umf-1.1//:umf", + "@libigdfcl2//:libopencl_clang", + "@libigdgmm12", + "@libnl-3-200//:libnl_3", + "@libnl-genl-3-200//:libnl_genl_3", + "@libnl-route-3-200//:libnl_route_3", + "@libnuma1", + "@libze1", + "@zlib1g", + "@zml//platforms/oneapi:zmlxoneapi", + ], + add_directory_to_runfiles = False, + allow_overwrites = True, + hardlink = "auto", + include_external_repositories = ["**"], + replace_prefixes = { + "libiga64_so_2": "lib", + "libigdfcl_so_2": "lib", + "libigc_so_2": "lib", + "libdnnl_so_3": "lib", + "libzml_oneapi_so": "lib", + "libsycl_so": "lib", + "libtbb_so_12": "lib", + ONEAPI_CCL_LIB: "lib", + ONEAPI_COMPILER_LIB: "lib", + ONEAPI_MPI_LIB: "lib", + ONEAPI_MPI_LIBFABRIC_LIB: "lib", + ONEAPI_MKL_LIB: "lib", + ONEAPI_TCM_LIB: "lib", + ONEAPI_UMF_LIB: "lib", + "lib/x86_64-linux-gnu": "lib", + "libze_intel_gpu_so": "lib", + "platforms/oneapi/libzmlxoneapi": "lib/libzmlxoneapi", + }, + root_paths = ["."], +) + +cc_library( + name = "libzml_oneapi", + data = [":sandbox"], + visibility = ["@zml//platforms/oneapi:__subpackages__"], +) diff --git a/platforms/oneapi/oneapi.bzl b/platforms/oneapi/oneapi.bzl new file mode 100644 index 00000000..a3c968d3 --- /dev/null +++ b/platforms/oneapi/oneapi.bzl @@ -0,0 +1,325 @@ +load("@llvm//:http_bsdtar_archive.bzl", http_archive = "http_bsdtar_archive") +load("//bazel:http_deb_archive.bzl", "http_deb_archive") + +_PLUGIN_ONEAPI_URL = "https://mirror.zml.ai/plugins/202611132003.375.1.7d765f57a407/zml-oneapi-linux-amd64.tar.zst" +_PLUGIN_ONEAPI_SHA256 = "294722670832c543c8ac4c074c30f636f2a3b0854820f7c5a070fa82c0d6f3d5" + +ONEAPI_VERSION = "2026.1" + +# www.intel.com/content/www/us/en/developer/tools/oneapi/onednn-download.html +ONEAPI_DNNL_VERSION = "2026.0" + +# https://github.com/uxlfoundation/onetbb +ONEAPI_TBB_VERSION = "2023.1" +ONEAPI_TCM_VERSION = "1.5" +ONEAPI_UMF_VERSION = "1.1" + +ONEAPI_CCL_LIB = "ccl/2022.1/lib" +ONEAPI_COMPILER_LIB = "compiler/{}/lib".format(ONEAPI_VERSION) +ONEAPI_DNNL_LIB = "dnnl/{}/lib".format(ONEAPI_DNNL_VERSION) +ONEAPI_MPI_LIB = "mpi/2021.18/lib" +ONEAPI_MPI_LIBFABRIC_LIB = "mpi/2021.18/opt/mpi/libfabric/lib" +ONEAPI_MKL_LIB = "mkl/{}/lib".format(ONEAPI_VERSION) +ONEAPI_TBB_LIB = "tbb/{}/lib".format(ONEAPI_TBB_VERSION) +ONEAPI_TCM_LIB = "tcm/{}/lib".format(ONEAPI_TCM_VERSION) +ONEAPI_UMF_LIB = "umf/{}/lib".format(ONEAPI_UMF_VERSION) + +_BUILD_FILE_DEFAULT_VISIBILITY = """\ +package(default_visibility = ["//visibility:public"]) +""" + +_UBUNTU_STRIP_PREFIX = "./usr" +_ONEAPI_STRIP_PREFIX = "./opt/intel/oneapi" + +def _read_packages(mctx, labels): + ret = {} + for label in labels: + data = json.decode(mctx.read(Label(label))) + for pkg in data["packages"]: + ret.setdefault(pkg["name"], {})[pkg["arch"]] = pkg + return ret + +_UBUNTU_PACKAGES = { + "libnl-3-200": """ +filegroup( + name = "libnl_3", + srcs = ["lib/x86_64-linux-gnu/libnl-3.so.200"], +)""", + "libnl-genl-3-200": """ +filegroup( + name = "libnl_genl_3", + srcs = ["lib/x86_64-linux-gnu/libnl-genl-3.so.200"], +)""", + "libnl-route-3-200": """ +filegroup( + name = "libnl_route_3", + srcs = ["lib/x86_64-linux-gnu/libnl-route-3.so.200"], +)""", + "libnuma1": """ +filegroup( + name = "libnuma1", + srcs = ["lib/x86_64-linux-gnu/libnuma.so.1"], +)""", + "libigdgmm12": """ +filegroup( + name = "libigdgmm12", + srcs = ["lib/x86_64-linux-gnu/libigdgmm.so.12"], +)""", + "libigc2": """ +filegroup( + name = "libigc_so_2", + srcs = ["lib/x86_64-linux-gnu/libigc.so.2.41.15+0"], +) +filegroup( + name = "libiga64_so_2", + srcs = ["lib/x86_64-linux-gnu/libiga64.so.2.41.15+0"], +) +filegroup( + name = "libigc2", + srcs = [ + ":libiga64_so_2", + ":libigc_so_2", + ], +)""", + "libigdfcl2": """ +filegroup( + name = "libigdfcl_so_2", + srcs = ["lib/x86_64-linux-gnu/libigdfcl.so.2.41.15+0"], +) +filegroup( + name = "libopencl_clang", + srcs = ["lib/x86_64-linux-gnu/libopencl-clang.so.22.1"], +) +filegroup( + name = "libigdfcl2", + srcs = [ + ":libigdfcl_so_2", + ":libopencl_clang", + ], +)""", + "libze-intel-gpu1": """ +filegroup( + name = "libze_intel_gpu_so", + srcs = ["lib/x86_64-linux-gnu/libze_intel_gpu.so.1"], +) +filegroup( + name = "libze_intel_gpu", + srcs = ["lib/x86_64-linux-gnu/libze_intel_gpu.so.1"], +)""", + "libze1": """ +filegroup( + name = "libze1", + srcs = [ + "lib/x86_64-linux-gnu/libze_loader.so.1", + "lib/x86_64-linux-gnu/libze_tracing_layer.so.1", + "lib/x86_64-linux-gnu/libze_validation_layer.so.1", + ], +)""", + "zlib1g": """ +filegroup( + name = "zlib1g", + srcs = ["lib/x86_64-linux-gnu/libz.so.1"], +)""", +} + +_ONEAPI_PACKAGES = { + "intel-oneapi-ccl-2022.1": """ +filegroup( + name = "libccl_legacy_so", + srcs = ["{ONEAPI_CCL_LIB}/libccl_legacy.so"], +) +filegroup( + name = "ccl_runtime", + srcs = [ + "{ONEAPI_CCL_LIB}/ccl/cpu/lib/libccl.so", + "{ONEAPI_CCL_LIB}/ccl/cpu/lib/libccl.so.1", + "{ONEAPI_CCL_LIB}/ccl/cpu/lib/libccl.so.1.0", + "{ONEAPI_CCL_LIB}/ccl/cpu/lib/libccl_openmp.so", + "{ONEAPI_CCL_LIB}/ccl/cpu/lib/libccl_openmp.so.0", + "{ONEAPI_CCL_LIB}/ccl/cpu/lib/libccl_openmp.so.0.1", + "{ONEAPI_CCL_LIB}/ccl/kernels/kernels.spv", + "{ONEAPI_CCL_LIB}/libccl.so", + "{ONEAPI_CCL_LIB}/libccl.so.1", + "{ONEAPI_CCL_LIB}/libccl.so.1.0", + "{ONEAPI_CCL_LIB}/libccl.so.2", + "{ONEAPI_CCL_LIB}/libccl.so.2.0", + "{ONEAPI_CCL_LIB}/libccl_legacy_cpu.so", + "{ONEAPI_CCL_LIB}/libccl_openmp.so", + "{ONEAPI_CCL_LIB}/libccl_openmp.so.0", + "{ONEAPI_CCL_LIB}/libccl_openmp.so.0.1", + ], +)""".format(ONEAPI_CCL_LIB = ONEAPI_CCL_LIB), + "intel-oneapi-mpi-2021.18": """ +filegroup( + name = "mpi_runtime", + srcs = [ + "{ONEAPI_MPI_LIB}/libmpi.so", + "{ONEAPI_MPI_LIB}/libmpi.so.12", + "{ONEAPI_MPI_LIB}/libmpi.so.12.0", + "{ONEAPI_MPI_LIB}/libmpi.so.12.0.0", + "{ONEAPI_MPI_LIB}/libmpi_abi.so", + "{ONEAPI_MPI_LIB}/libmpi_abi.so.1", + "{ONEAPI_MPI_LIB}/libmpi_abi.so.1.0", + "{ONEAPI_MPI_LIB}/libmpi_ilp64.so", + "{ONEAPI_MPI_LIB}/libmpi_ilp64.so.4", + "{ONEAPI_MPI_LIB}/libmpi_ilp64.so.4.1", + "{ONEAPI_MPI_LIB}/libmpi_shm_heap_proxy.so", + "{ONEAPI_MPI_LIB}/libmpicxx.so", + "{ONEAPI_MPI_LIB}/libmpicxx.so.12", + "{ONEAPI_MPI_LIB}/libmpicxx.so.12.0", + "{ONEAPI_MPI_LIB}/libmpicxx.so.12.0.0", + "{ONEAPI_MPI_LIB}/libmpifort.so", + "{ONEAPI_MPI_LIB}/libmpifort.so.12", + "{ONEAPI_MPI_LIB}/libmpifort.so.12.0", + "{ONEAPI_MPI_LIB}/libmpifort.so.12.0.0", + "{ONEAPI_MPI_LIB}/libmpifort_abi.so", + "{ONEAPI_MPI_LIB}/libmpifort_abi.so.1", + "{ONEAPI_MPI_LIB}/libmpifort_abi.so.1.0", + "{ONEAPI_MPI_LIB}/libmpijava.so", + "{ONEAPI_MPI_LIB}/libmpijava.so.1", + "{ONEAPI_MPI_LIB}/libmpijava.so.1.0", + "{ONEAPI_MPI_LIB}/mpi/libmpi_ze_hooks.so", + "{ONEAPI_MPI_LIBFABRIC_LIB}/libfabric.so", + "{ONEAPI_MPI_LIBFABRIC_LIB}/libfabric.so.1", + "{ONEAPI_MPI_LIBFABRIC_LIB}/prov/libefa-fi.so", + "{ONEAPI_MPI_LIBFABRIC_LIB}/prov/libmlx-fi.so", + "{ONEAPI_MPI_LIBFABRIC_LIB}/prov/libpsm3-fi.so", + "{ONEAPI_MPI_LIBFABRIC_LIB}/prov/libpsmx2-fi.so", + "{ONEAPI_MPI_LIBFABRIC_LIB}/prov/librxm-fi.so", + "{ONEAPI_MPI_LIBFABRIC_LIB}/prov/libshm-fi.so", + "{ONEAPI_MPI_LIBFABRIC_LIB}/prov/libtcp-fi.so", + "{ONEAPI_MPI_LIBFABRIC_LIB}/prov/libverbs-1.1-fi.so", + "{ONEAPI_MPI_LIBFABRIC_LIB}/prov/libverbs-1.12-fi.so", + ], +)""".format( + ONEAPI_MPI_LIB = ONEAPI_MPI_LIB, + ONEAPI_MPI_LIBFABRIC_LIB = ONEAPI_MPI_LIBFABRIC_LIB, + ), + "intel-oneapi-dnnl-2026.0": """ +filegroup( + name = "libdnnl_so_3", + srcs = ["{ONEAPI_DNNL_LIB}/libdnnl.so.3.11"], +)""".format(ONEAPI_DNNL_LIB = ONEAPI_DNNL_LIB), + "intel-oneapi-tbb-2023.1": """ +filegroup( + name = "libtbb_so_12", + srcs = ["{ONEAPI_TBB_LIB}/libtbb.so.12.19"], +)""".format(ONEAPI_TBB_LIB = ONEAPI_TBB_LIB), + "intel-oneapi-tcm-1.5": """ +filegroup( + name = "hwloc", + srcs = ["{ONEAPI_TCM_LIB}/libhwloc.so.15"], +)""".format(ONEAPI_TCM_LIB = ONEAPI_TCM_LIB), + "intel-oneapi-umf-1.1": """ +filegroup( + name = "umf", + srcs = ["{ONEAPI_UMF_LIB}/libumf.so.1"], +)""".format(ONEAPI_UMF_LIB = ONEAPI_UMF_LIB), + "intel-oneapi-compiler-dpcpp-cpp-runtime-2026.1": """ +filegroup( + name = "libsycl_so", + srcs = ["{ONEAPI_COMPILER_LIB}/libsycl.so.9"], +) +filegroup( + name = "sycl_runtime", + srcs = [ + "{ONEAPI_COMPILER_LIB}/libur_adapter_level_zero.so.0", + "{ONEAPI_COMPILER_LIB}/libur_adapter_level_zero_v2.so.0", + "{ONEAPI_COMPILER_LIB}/libur_adapter_opencl.so.0", + "{ONEAPI_COMPILER_LIB}/libur_loader.so.0", + ], +)""".format(ONEAPI_COMPILER_LIB = ONEAPI_COMPILER_LIB), + "intel-oneapi-compiler-shared-runtime-2026.1": """ +filegroup( + name = "compiler_runtime", + srcs = [ + "{ONEAPI_COMPILER_LIB}/libOpenCL.so", + "{ONEAPI_COMPILER_LIB}/libOpenCL.so.1", + "{ONEAPI_COMPILER_LIB}/libOpenCL.so.1.0.0", + "{ONEAPI_COMPILER_LIB}/libimf.so", + "{ONEAPI_COMPILER_LIB}/libintlc.so.5", + "{ONEAPI_COMPILER_LIB}/libirc.so", + "{ONEAPI_COMPILER_LIB}/libirng.so", + "{ONEAPI_COMPILER_LIB}/libsvml.so", + ], +)""".format(ONEAPI_COMPILER_LIB = ONEAPI_COMPILER_LIB), + "intel-oneapi-mkl-core-2026.1": """ +filegroup( + name = "mkl_core_runtime", + srcs = [ + "{ONEAPI_MKL_LIB}/libmkl_core.so.3", + "{ONEAPI_MKL_LIB}/libmkl_def.so.3", + "{ONEAPI_MKL_LIB}/libmkl_intel_ilp64.so.3", + "{ONEAPI_MKL_LIB}/libmkl_sequential.so.3", + ], +)""".format(ONEAPI_MKL_LIB = ONEAPI_MKL_LIB), + "intel-oneapi-mkl-sycl-blas-2026.1": """ +filegroup( + name = "mkl_sycl_blas", + srcs = ["{ONEAPI_MKL_LIB}/libmkl_sycl_blas.so.6"], +)""".format(ONEAPI_MKL_LIB = ONEAPI_MKL_LIB), + "intel-oneapi-mkl-sycl-dft-2026.1": """ +filegroup( + name = "mkl_sycl_dft", + srcs = ["{ONEAPI_MKL_LIB}/libmkl_sycl_dft.so.6"], +)""".format(ONEAPI_MKL_LIB = ONEAPI_MKL_LIB), + "intel-oneapi-mkl-sycl-lapack-2026.1": """ +filegroup( + name = "mkl_sycl_lapack", + srcs = ["{ONEAPI_MKL_LIB}/libmkl_sycl_lapack.so.6"], +)""".format(ONEAPI_MKL_LIB = ONEAPI_MKL_LIB), + "intel-oneapi-mkl-sycl-rng-2026.1": """ +filegroup( + name = "mkl_sycl_rng", + srcs = ["{ONEAPI_MKL_LIB}/libmkl_sycl_rng.so.6"], +)""".format(ONEAPI_MKL_LIB = ONEAPI_MKL_LIB), + "intel-oneapi-mkl-sycl-sparse-2026.1": """ +filegroup( + name = "mkl_sycl_sparse", + srcs = ["{ONEAPI_MKL_LIB}/libmkl_sycl_sparse.so.6"], +)""".format(ONEAPI_MKL_LIB = ONEAPI_MKL_LIB), +} + +_ROOT_MODULE_DIRECT_DEPS = ["libzml_oneapi"] + +def _oneapi_impl(mctx): + loaded_packages = _read_packages(mctx, [ + "@zml//platforms/oneapi:packages.lock.json", + ]) + + http_archive( + name = "libzml_oneapi", + build_file = "libzml_oneapi.BUILD.bazel", + sha256 = _PLUGIN_ONEAPI_SHA256, + url = _PLUGIN_ONEAPI_URL, + ) + + for pkg_name, build_file_content in _UBUNTU_PACKAGES.items(): + pkg = loaded_packages[pkg_name]["amd64"] + http_deb_archive( + name = pkg_name, + urls = pkg["urls"], + sha256 = pkg["sha256"], + strip_prefix = _UBUNTU_STRIP_PREFIX, + build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + build_file_content, + ) + + for pkg_name, build_file_content in _ONEAPI_PACKAGES.items(): + pkg = loaded_packages[pkg_name]["amd64"] + http_deb_archive( + name = pkg_name, + urls = pkg["urls"], + sha256 = pkg["sha256"], + strip_prefix = _ONEAPI_STRIP_PREFIX, + build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + build_file_content, + ) + + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = _ROOT_MODULE_DIRECT_DEPS, + root_module_direct_dev_deps = [], + ) + +oneapi_packages = module_extension( + implementation = _oneapi_impl, +) diff --git a/platforms/oneapi/oneapi.h b/platforms/oneapi/oneapi.h new file mode 100644 index 00000000..933e4f31 --- /dev/null +++ b/platforms/oneapi/oneapi.h @@ -0,0 +1 @@ + #include diff --git a/platforms/oneapi/oneapi.zig b/platforms/oneapi/oneapi.zig new file mode 100644 index 00000000..c5654741 --- /dev/null +++ b/platforms/oneapi/oneapi.zig @@ -0,0 +1,73 @@ +const std = @import("std"); +const builtin = @import("builtin"); + +const bazel = @import("bazel"); +const bazel_builtin = @import("bazel_builtin"); +const c = @import("c"); +const pjrt = @import("pjrt"); +const platforms_options = @import("platforms/options"); +const stdx = @import("stdx"); + +const log = std.log.scoped(.@"zml/platforms/oneapi"); + +pub fn isEnabled() bool { + return platforms_options.oneapi_enabled; +} + +fn hasOneApiDevice(io: std.Io) bool { + var dir = std.Io.Dir.openDirAbsolute(io, "/dev/dri", .{ .iterate = true }) catch return false; + defer dir.close(io); + + var it = dir.iterate(); + while (it.next(io) catch null) |entry| { + if (std.mem.startsWith(u8, entry.name, "renderD")) { + return true; + } + } + + return false; +} + +fn setupOneAPIEnv(driver_path: [:0]const u8) void { + _ = c.setenv("CCL_LOG_LEVEL", c.getenv("CCL_LOG_LEVEL") orelse "error", 1); + _ = c.setenv("CCL_ATL_TRANSPORT", "ofi", 1); + _ = c.setenv("FI_PROVIDER", "shm", 1); + // collective-permute through oneccl recv/send often relies on sycl d2d memcpy underneath, + // that doesn't work because of peer residency likely this: + // https://github.com/intel/compute-runtime/issues/953 + _ = c.setenv("XLA_ONECCL_COLLECTIVE_PERMUTE_BYPASS_SYCL_P2P", "1", 1); + _ = c.setenv("ONEAPI_DEVICE_SELECTOR", "level_zero:*", 0); + _ = c.setenv("ZE_ENABLE_ALT_DRIVERS", driver_path, 0); +} + +pub fn load(_: std.mem.Allocator, io: std.Io) !*const pjrt.Api { + if (comptime !isEnabled()) { + return error.Unavailable; + } + + if (comptime builtin.os.tag != .linux) { + return error.Unavailable; + } + + if (!hasOneApiDevice(io)) { + return error.Unavailable; + } + + const r = try bazel.runfiles(bazel_builtin.current_repository); + + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const sandbox_path = try r.rlocation("libzml_oneapi/sandbox", &path_buf) orelse { + log.err("Failed to find sandbox path for oneAPI runtime", .{}); + return error.FileNotFound; + }; + + var driver_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const driver_path = try stdx.Io.Dir.path.bufJoinZ(&driver_path_buf, &.{ sandbox_path, "lib", "libze_intel_gpu.so.1" }); + setupOneAPIEnv(driver_path); + + return blk: { + var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libzml_oneapi.so" }); + break :blk pjrt.Api.loadFrom(path); + }; +} diff --git a/platforms/oneapi/packages.lock.json b/platforms/oneapi/packages.lock.json new file mode 100644 index 00000000..140ce102 --- /dev/null +++ b/platforms/oneapi/packages.lock.json @@ -0,0 +1,269 @@ +{ + "packages": [ + { + "arch": "amd64", + "dependencies": [], + "key": "intel-oneapi-ccl-2022.1_2022.1.2-9_amd64", + "name": "intel-oneapi-ccl-2022.1", + "sha256": "432e62469978f60a9b35b55216cfd495b0adc76c334707d3bbc81f69b7cc2948", + "urls": [ + "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-ccl-2022.1-2022.1.2-9_amd64.deb" + ], + "version": "2022.1.2-9" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "intel-oneapi-mpi-2021.18_2021.18.1-9_amd64", + "name": "intel-oneapi-mpi-2021.18", + "sha256": "f6f76f39755329bbac890deeb72fb105df064dc5dcb67cde1396e4d6ef0f6706", + "urls": [ + "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-mpi-2021.18-2021.18.1-9_amd64.deb" + ], + "version": "2021.18.1-9" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "intel-oneapi-compiler-dpcpp-cpp-runtime-2026.1_2026.1.1-325_amd64", + "name": "intel-oneapi-compiler-dpcpp-cpp-runtime-2026.1", + "sha256": "df0413d57cff5a9c67bd02c1e958d7e0512f77878bbcf976c9e08af7c0b1069c", + "urls": [ + "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-compiler-dpcpp-cpp-runtime-2026.1-2026.1.1-325_amd64.deb" + ], + "version": "2026.1.1-325" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "intel-oneapi-compiler-shared-runtime-2026.1_2026.1.1-325_amd64", + "name": "intel-oneapi-compiler-shared-runtime-2026.1", + "sha256": "c0c77700243af0413f7fbefd26649943c4959171ead91b6ace36ef0c8dfb2630", + "urls": [ + "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-compiler-shared-runtime-2026.1-2026.1.1-325_amd64.deb" + ], + "version": "2026.1.1-325" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "intel-oneapi-dnnl-2026.0_2026.0.2-40_amd64", + "name": "intel-oneapi-dnnl-2026.0", + "sha256": "4b0f7e23415a695f204f8080328977225759534158d813c2225403f8851433e8", + "urls": [ + "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-dnnl-2026.0-2026.0.2-40_amd64.deb" + ], + "version": "2026.0.2-40" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "intel-oneapi-mkl-core-2026.1_2026.1.0-236_amd64", + "name": "intel-oneapi-mkl-core-2026.1", + "sha256": "3b230c02fc0ac44f8f71062c20cd883b37c6c1e00940dc0a7f13b02d9c47bd93", + "urls": [ + "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-mkl-core-2026.1-2026.1.0-236_amd64.deb" + ], + "version": "2026.1.0-236" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "intel-oneapi-mkl-sycl-blas-2026.1_2026.1.0-236_amd64", + "name": "intel-oneapi-mkl-sycl-blas-2026.1", + "sha256": "9668d51a7cf68ae4b8944a8cdaa24f6d4ef72604c048f68fd8603b2b792bd783", + "urls": [ + "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-mkl-sycl-blas-2026.1-2026.1.0-236_amd64.deb" + ], + "version": "2026.1.0-236" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "intel-oneapi-mkl-sycl-dft-2026.1_2026.1.0-236_amd64", + "name": "intel-oneapi-mkl-sycl-dft-2026.1", + "sha256": "31862dd325a31ccacc1e5e9c93140f4c8bd8a0353e0b3658e2b87e329e58d373", + "urls": [ + "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-mkl-sycl-dft-2026.1-2026.1.0-236_amd64.deb" + ], + "version": "2026.1.0-236" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "intel-oneapi-mkl-sycl-lapack-2026.1_2026.1.0-236_amd64", + "name": "intel-oneapi-mkl-sycl-lapack-2026.1", + "sha256": "f701832f5e49b29731c15a27a094122bebe6a1fbd90ddf14c431b7923b8fd921", + "urls": [ + "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-mkl-sycl-lapack-2026.1-2026.1.0-236_amd64.deb" + ], + "version": "2026.1.0-236" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "intel-oneapi-mkl-sycl-rng-2026.1_2026.1.0-236_amd64", + "name": "intel-oneapi-mkl-sycl-rng-2026.1", + "sha256": "b0748e152f2042dedd1d21c784fb2a8b551c573b7e7ed0361ada2423deb275bf", + "urls": [ + "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-mkl-sycl-rng-2026.1-2026.1.0-236_amd64.deb" + ], + "version": "2026.1.0-236" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "intel-oneapi-mkl-sycl-sparse-2026.1_2026.1.0-236_amd64", + "name": "intel-oneapi-mkl-sycl-sparse-2026.1", + "sha256": "6a499dc1c5f049ad522a5288151ff0c93f535b8e8ab34edcd773785338e07d45", + "urls": [ + "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-mkl-sycl-sparse-2026.1-2026.1.0-236_amd64.deb" + ], + "version": "2026.1.0-236" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "intel-oneapi-tbb-2023.1_2023.1.0-151_amd64", + "name": "intel-oneapi-tbb-2023.1", + "sha256": "f6e4ae469436a8c5ee2250203268344164c9ff455a2207219c75c463aab99a8b", + "urls": [ + "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-tbb-2023.1-2023.1.0-151_amd64.deb" + ], + "version": "2023.1.0-151" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "intel-oneapi-tcm-1.5_1.5.0-489_amd64", + "name": "intel-oneapi-tcm-1.5", + "sha256": "7122ed7748d54342b24f0a2838a64f24b54e430ac4d06b2961a8eefee4ffb089", + "urls": [ + "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-tcm-1.5-1.5.0-489_amd64.deb" + ], + "version": "1.5.0-489" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "intel-oneapi-umf-1.1_1.1.0-340_amd64", + "name": "intel-oneapi-umf-1.1", + "sha256": "8705a997042c76f67228b4135184754e1cd3f8f68b6629bcb261c1d5b958ce78", + "urls": [ + "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-umf-1.1-1.1.0-340_amd64.deb" + ], + "version": "1.1.0-340" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libnl-3-200_3.7.0-0.3build1.1_amd64", + "name": "libnl-3-200", + "sha256": "15b3f61db753dc3db6f62f4259f8b822299343e500886776486612a13a2ed247", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20260715T000000Z/pool/main/libn/libnl3/libnl-3-200_3.7.0-0.3build1.1_amd64.deb" + ], + "version": "3.7.0-0.3build1.1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libnl-genl-3-200_3.7.0-0.3build1.1_amd64", + "name": "libnl-genl-3-200", + "sha256": "9ca0cb63f295d00e6346b9924278c07cfc61311cbd288b805b06e25aa08c9fd8", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20260715T000000Z/pool/main/libn/libnl3/libnl-genl-3-200_3.7.0-0.3build1.1_amd64.deb" + ], + "version": "3.7.0-0.3build1.1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libnl-route-3-200_3.7.0-0.3build1.1_amd64", + "name": "libnl-route-3-200", + "sha256": "c9dbd29cb7fdcf1a0587726cbc5432a7d3e81242108dfcfa2dd6716c44c74d4d", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20260715T000000Z/pool/main/libn/libnl3/libnl-route-3-200_3.7.0-0.3build1.1_amd64.deb" + ], + "version": "3.7.0-0.3build1.1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libnuma1_2.0.18-1ubuntu0.24.04.1_amd64", + "name": "libnuma1", + "sha256": "f333b8edf6f0b705c19f2a67008194df66f7aab0fa2350dc98510569a033ff29", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20260715T000000Z/pool/main/n/numactl/libnuma1_2.0.18-1ubuntu0.24.04.1_amd64.deb" + ], + "version": "2.0.18-1ubuntu0.24.04.1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libigdgmm12_22.10.1-1_24.04_ppa1_amd64", + "name": "libigdgmm12", + "sha256": "2c68fcb150cba1b679063266cdf0054f26f18af90de71d6d3b614eaf95c98f80", + "urls": [ + "https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20260930T000000Z/pool/main/i/intel-gmmlib/libigdgmm12_22.10.1-1~24.04~ppa1_amd64.deb" + ], + "version": "22.10.1-1~24.04~ppa1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libigc2_2.41.15-1_24.04_amd64", + "name": "libigc2", + "sha256": "c3e5f181eecd40dc8b656c3ff5b7e3d4f690e93181255726516416e61ade1d1f", + "urls": [ + "https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20261029T000000Z/pool/main/i/intel-graphics-compiler/libigc2_2.41.15-1~24.04_amd64.deb" + ], + "version": "2.41.15-1~24.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libigdfcl2_2.41.15-1_24.04_amd64", + "name": "libigdfcl2", + "sha256": "8a2058fd9f5e7d0a4b05001ff861780d2f72529135fa70fb7fbba0f0ff8cd74f", + "urls": [ + "https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20261029T000000Z/pool/main/i/intel-graphics-compiler/libigdfcl2_2.41.15-1~24.04_amd64.deb" + ], + "version": "2.41.15-1~24.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libze-intel-gpu1_26.31.39395.14-1_24.04_ppa1_amd64", + "name": "libze-intel-gpu1", + "sha256": "39f10740e40cf0de1015997d0de094a3c3dc6371c3b9ea1dcd15771d64559c14", + "urls": [ + "https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20260930T000000Z/pool/main/i/intel-compute-runtime/libze-intel-gpu1_26.31.39395.14-1~24.04~ppa1_amd64.deb" + ], + "version": "26.31.39395.14-1~24.04~ppa1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libze1_1.32.0-1_24.04_ppa1_amd64", + "name": "libze1", + "sha256": "d4001dc474d3cb74ded50a449b9f646cfdfbb31e69e70b8af38f2d18273dfce1", + "urls": [ + "https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20261029T000000Z/pool/main/l/level-zero-loader/libze1_1.32.0-1~24.04~ppa1_amd64.deb" + ], + "version": "1.32.0-1~24.04~ppa1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "zlib1g_1-1.3.dfsg-3.1ubuntu2.1_amd64", + "name": "zlib1g", + "sha256": "7074b6a2f6367a10d280c00a1cb02e74277709180bab4f2491a2f355ab2d6c20", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20260715T000000Z/pool/main/z/zlib/zlib1g_1.3.dfsg-3.1ubuntu2.1_amd64.deb" + ], + "version": "1:1.3.dfsg-3.1ubuntu2.1" + } + ], + "version": 1 +} diff --git a/platforms/oneapi/packages.yaml b/platforms/oneapi/packages.yaml new file mode 100644 index 00000000..5a829ac5 --- /dev/null +++ b/platforms/oneapi/packages.yaml @@ -0,0 +1,53 @@ +# +# bazel run @apt_oneapi//:lock +# +version: 1 + +sources: + - channel: all main + url: https://apt.repos.intel.com/oneapi + - channel: noble main + url: https://snapshot.ubuntu.com/ubuntu/20260715T000000Z + - channel: noble-security main + url: https://snapshot.ubuntu.com/ubuntu/20260715T000000Z + - channel: noble-updates main + url: https://snapshot.ubuntu.com/ubuntu/20260715T000000Z + - channel: noble main + url: https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20260930T000000Z + + # IGC 2.41.15 is newer than the graphics snapshot above; the other graphics + # packages are pinned below to keep their versions from it. Use snapshots, + # not the live PPA: the PPA deletes superseded packages from its pool. + - channel: noble main + url: https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20261029T000000Z + +archs: + - "amd64" + +packages: + # Do not use "intel-oneapi-toolkit" package + # It will install an opaque deb file that will at preinst download the sdk toolkit + - "intel-oneapi-ccl-2022.1" + - "intel-oneapi-mpi-2021.18" + - "intel-oneapi-compiler-dpcpp-cpp-runtime-2026.1" + - "intel-oneapi-compiler-shared-runtime-2026.1" + - "intel-oneapi-dnnl-2026.0" + - "intel-oneapi-mkl-core-2026.1" + - "intel-oneapi-mkl-sycl-blas-2026.1" + - "intel-oneapi-mkl-sycl-dft-2026.1" + - "intel-oneapi-mkl-sycl-lapack-2026.1" + - "intel-oneapi-mkl-sycl-rng-2026.1" + - "intel-oneapi-mkl-sycl-sparse-2026.1" + - "intel-oneapi-tbb-2023.1" + - "intel-oneapi-tcm-1.5" + - "intel-oneapi-umf-1.1" + - "libnl-3-200" + - "libnl-genl-3-200" + - "libnl-route-3-200" + - "libnuma1" + - "libigdgmm12 (= 22.10.1-1~24.04~ppa1)" + - "libigc2 (= 2.41.15-1~24.04)" + - "libigdfcl2 (= 2.41.15-1~24.04)" + - "libze-intel-gpu1 (= 26.31.39395.14-1~24.04~ppa1)" + - "libze1" + - "zlib1g" diff --git a/platforms/oneapi/zmlxoneapi.zig b/platforms/oneapi/zmlxoneapi.zig new file mode 100644 index 00000000..6be2505a --- /dev/null +++ b/platforms/oneapi/zmlxoneapi.zig @@ -0,0 +1,31 @@ +const std = @import("std"); + +const stdx = @import("stdx"); + +pub export fn zmlxoneapi_dlopen(filename: [*c]const u8, flags: c_int) ?*anyopaque { + const replacements: std.StaticStringMap([:0]const u8) = .initComptime(.{ + .{ "libOpenCL.so", "libOpenCL.so.1" }, + .{ "libze_loader.so", "libze_loader.so.1" }, + .{ "libumf.so", "libumf.so.1" }, + .{ "libur_loader.so", "libur_loader.so.0" }, + .{ "libur_adapter_level_zero.so", "libur_adapter_level_zero.so.0" }, + .{ "libur_adapter_level_zero_v2.so", "libur_adapter_level_zero_v2.so.0" }, + .{ "libur_adapter_opencl.so", "libur_adapter_opencl.so.0" }, + .{ "libsycl.so", "libsycl.so.9" }, + .{ "libmkl_core.so", "libmkl_core.so.3" }, + .{ "libmkl_intel_ilp64.so", "libmkl_intel_ilp64.so.3" }, + .{ "libmkl_sequential.so", "libmkl_sequential.so.3" }, + .{ "libmkl_sycl_blas.so", "libmkl_sycl_blas.so.6" }, + }); + + var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const new_filename: [*c]const u8 = if (filename) |f| blk: { + const replacement = replacements.get(std.Io.Dir.path.basename(std.mem.span(f))) orelse break :blk f; + break :blk stdx.Io.Dir.path.bufJoinZ(&buf, &.{ + stdx.process.selfSharedObjectDirPath(), + replacement, + }) catch unreachable; + } else null; + + return std.c.dlopen(new_filename, @bitCast(flags)); +} diff --git a/platforms/packages.bzl b/platforms/packages.bzl new file mode 100644 index 00000000..6d3117cd --- /dev/null +++ b/platforms/packages.bzl @@ -0,0 +1,61 @@ +_BUILD_FILE_DEFAULT_VISIBILITY = """\ +package(default_visibility = ["//visibility:public"]) +""" + +def _kwargs(**kwargs): + return repr(struct(**kwargs))[len("struct("):-1] + +def _cc_import(**kwargs): + return """cc_import({})""".format(_kwargs(**kwargs)) + +def _cc_library(**kwargs): + return """cc_library({})""".format(_kwargs(**kwargs)) + +def _cc_library_hdrs_glob(name, hdrs_glob, deps = [], **kwargs): + return """\ +cc_library( + name = "{name}", + hdrs = glob({hdrs_glob}), + deps = {deps}, + {kwargs} +) +""".format(name = name, hdrs_glob = repr(hdrs_glob), deps = repr(deps), kwargs = _kwargs(**kwargs)) + +def _cc_import_glob_hdrs(name, hdrs_glob, shared_library, deps = [], **kwargs): + return """\ +cc_import( + name = "{name}", + shared_library = {shared_library}, + hdrs = glob({hdrs_glob}), + deps = {deps}, + {kwargs} +) +""".format(name = name, hdrs_glob = repr(hdrs_glob), shared_library = repr(shared_library), deps = repr(deps), kwargs = _kwargs(**kwargs)) + +def _filegroup(**kwargs): + return """filegroup({})""".format(_kwargs(**kwargs)) + +def _patchelf(**kwargs): + return """patchelf({})""".format(_kwargs(**kwargs)) + +def _load(bzl, name): + return """load({}, {})""".format(repr(bzl), repr(name)) + +def _read(mctx, labels): + ret = {} + for label in labels: + data = json.decode(mctx.read(Label(label))) + for pkg in data["packages"]: + ret.setdefault(pkg["name"], {})[pkg["arch"]] = pkg + return ret + +packages = struct( + read = _read, + cc_import = _cc_import, + cc_import_glob_hdrs = _cc_import_glob_hdrs, + cc_library = _cc_library, + cc_library_hdrs_glob = _cc_library_hdrs_glob, + filegroup = _filegroup, + load_ = _load, + patchelf = _patchelf, +) diff --git a/platforms/platforms.zig b/platforms/platforms.zig new file mode 100644 index 00000000..789c9a10 --- /dev/null +++ b/platforms/platforms.zig @@ -0,0 +1,50 @@ +const std = @import("std"); + +const cpu = @import("platforms/cpu"); +const cuda = @import("platforms/cuda"); +const furiosa = @import("platforms/furiosa"); +const metal = @import("platforms/metal"); +const neuron = @import("platforms/neuron"); +const oneapi = @import("platforms/oneapi"); +const pjrt = @import("pjrt"); +const platforms_options = @import("platforms/options"); +const rocm = @import("platforms/rocm"); +const tpu = @import("platforms/tpu"); + +const platforms = @This(); + +pub const Platform = enum { + cpu, + cuda, + rocm, + tpu, + neuron, + oneapi, + metal, + furiosa, + + pub fn selectFirstAcceleratorEnabled() error{Unavailable}!Platform { + inline for (@typeInfo(Platform).@"enum".field_names) |field| { + const target: Platform = @field(Platform, field); + if (comptime (target != .cpu and target.isEnabled())) return target; + } + + if (!comptime Platform.cpu.isEnabled()) @compileError("No platform was enabled, use --@zml//platforms:cuda=true to eg enable Cuda"); + return .cpu; + } + + pub fn isEnabled(target: Platform) bool { + return switch (target) { + inline else => |tag| @field(platforms_options, @tagName(tag) ++ "_enabled"), + }; + } + + pub fn load(target: Platform, allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { + return switch (target) { + inline else => |tag| @field(platforms, @tagName(tag)).load(allocator, io), + }; + } +}; + +pub const load = Platform.load; +pub const isEnabled = Platform.isEnabled; diff --git a/platforms/rocm/BUILD.bazel b/platforms/rocm/BUILD.bazel new file mode 100644 index 00000000..91651372 --- /dev/null +++ b/platforms/rocm/BUILD.bazel @@ -0,0 +1,49 @@ +load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_shared_library") + +exports_files( + ["merge_amdgpu_ids.zig"], + visibility = ["@libzml_rocm//:__subpackages__"], +) + +zig_shared_library( + name = "zmlxrocm", + main = "zmlxrocm.zig", + shared_lib_name = "libzmlxrocm.so.0", + target_compatible_with = ["@platforms//os:linux"], + visibility = [ + "//bin/zml-smi:__pkg__", + "@libzml_rocm//:__subpackages__", + ], + # Use Clang's compiler-rt, but disable stack checking + # to avoid requiring on the _zig_probe_stack symbol. + zigopts = ["-fno-stack-check"], + deps = ["//stdx"], +) + +cc_library( + name = "libzml_rocm", + hdrs = ["rocm.h"], + tags = ["manual"], + deps = ["@libzml_rocm"], +) + +zig_library( + name = "rocm", + import_name = "platforms/rocm", + main = "rocm.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//pjrt", + "//platforms:options", + ] + select({ + "//platforms:rocm.enabled": [ + ":libzml_rocm", + "//bazel", + "//stdx", + "@rules_zig//zig/runfiles", + ], + "//conditions:default": [], + }), +) diff --git a/platforms/rocm/libzml_rocm.BUILD.bazel b/platforms/rocm/libzml_rocm.BUILD.bazel new file mode 100644 index 00000000..7996b001 --- /dev/null +++ b/platforms/rocm/libzml_rocm.BUILD.bazel @@ -0,0 +1,122 @@ +load("@bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") +load("@bazel_skylib//rules:run_binary.bzl", "run_binary") +load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_binary") +load("@zml//bazel:patchelf.bzl", "patchelf") + +patchelf( + name = "libzml_rocm_so", + src = "lib/libzml_rocm.so", + add_needed = [ + "libzmlxrocm.so.0", + ], + rename_dynamic_symbols = { + "dlopen": "zmlxrocm_dlopen", + }, + set_rpath = "$ORIGIN:$ORIGIN/rocm_sysdeps/lib", +) + +zig_binary( + name = "merge_amdgpu_ids", + main = "@zml//platforms/rocm:merge_amdgpu_ids.zig", +) + +run_binary( + name = "amdgpu_ids", + srcs = [ + "@libdrm-common//:amdgpu_ids", + "@libdrm_mesa_amdgpu_ids//file:amdgpu.ids", + ], + outs = ["amdgpu.ids"], + args = [ + "$(location @libdrm-common//:amdgpu_ids)", + "$(location @libdrm_mesa_amdgpu_ids//file:amdgpu.ids)", + "$(location amdgpu.ids)", + ], + tool = ":merge_amdgpu_ids", + visibility = ["@zml//bin/zml-smi:__pkg__"], +) + +copy_to_directory( + name = "sandbox", + srcs = [ + ":amdgpu_ids", + ":libzml_rocm_so", + "@amdrocm-amdsmi//:libamd_smi", + "@amdrocm-base//:rocm-core", + "@amdrocm-base//:rocprofiler-register", + "@amdrocm-blas//:hipblaslt", + "@amdrocm-blas//:hipblaslt_runfiles", + "@amdrocm-blas//:rocblas", + "@amdrocm-blas//:rocblas_runfiles", + "@amdrocm-dnn//:MIOpen", + "@amdrocm-fft//:hipfft", + "@amdrocm-fft//:rocfft", + "@amdrocm-llvm//:llvm_libs", + "@amdrocm-llvm//:rocm_device_libs_runfiles", + "@amdrocm-math-common//:host_math_libs", + "@amdrocm-profiler-base//:hsa-amd-aqlprofile", + "@amdrocm-profiler-base//:rocprofiler-sdk", + "@amdrocm-profiler-base//:rocprofv3_tool", + "@amdrocm-profiler-base//:roctracer", + "@amdrocm-profiler-base//:roctx", + "@amdrocm-rccl//:rccl", + "@amdrocm-runtime//:amd_comgr", + "@amdrocm-runtime//:amdhip", + "@amdrocm-runtime//:hiprtc", + "@amdrocm-runtime//:hsa-runtime", + "@amdrocm-solver//:hipsolver", + "@amdrocm-solver//:rocsolver", + "@amdrocm-sparse//:rocsparse", + "@amdrocm-sysdeps//:system_libs", + "@libatomic1", + "@llvm-libunwind1//:libunwind", + "@zml//platforms/rocm:zmlxrocm", + ], + add_directory_to_runfiles = True, + include_external_repositories = ["**"], + replace_prefixes = { + "MIOpen": "lib", + "amdhip_so": "lib", + "amdhip": "lib", + "amd_comgr": "lib", + "hipdnn_backend": "lib", + "hipblas": "lib", + "hiprtc_so": "lib", + "hipfft": "lib", + "hiprtc": "lib", + "hipsolver": "lib", + "hipsparse": "lib", + "hipsparselt": "lib", + "hsa-amd-aqlprofile": "lib", + "hsa-runtime": "lib", + "hipblaslt": "lib", + "lib/host-math/lib": "lib", + "lib/llvm/lib": "lib", + "lib/llvm/amdgcn": "amdgcn", + "libzml_rocm_so": "lib", + "amdgpu.ids": "share/libdrm/amdgpu.ids", + "rccl": "lib", + "rocblas": "lib", + "rocfft": "lib", + "rocm-core": "lib", + "rocprofiler-register": "lib", + "rocprofiler-sdk_so": "lib", + "rocprofiler-sdk": "lib", + "rocm_smi": "lib", + "rocsolver": "lib", + "rocroller": "lib", + "rocsparse": "lib", + "roctracer": "lib", + "roctx": "lib", + "platforms/rocm": "lib", + "usr/lib/x86_64-linux-gnu": "lib", + }, + visibility = ["//visibility:public"], +) + +cc_library( + name = "libzml_rocm", + data = [":sandbox"], + visibility = ["@zml//platforms/rocm:__subpackages__"], +) diff --git a/platforms/rocm/merge_amdgpu_ids.zig b/platforms/rocm/merge_amdgpu_ids.zig new file mode 100644 index 00000000..b939df82 --- /dev/null +++ b/platforms/rocm/merge_amdgpu_ids.zig @@ -0,0 +1,73 @@ +const std = @import("std"); + +pub fn main(init: std.process.Init) !void { + const io = init.io; + const arena = init.arena.allocator(); + + const args = try init.minimal.args.toSlice(arena); + + if (args.len != 4) { + try std.Io.File.stderr().writeStreamingAll(io, "Usage: merge_amdgpu_ids \n"); + std.process.exit(1); + } + + var entries: std.StringHashMapUnmanaged([]const u8) = .empty; + defer entries.deinit(arena); + + const cwd = std.Io.Dir.cwd(); + + try parseFile(io, cwd, arena, args[2], &entries); + try parseFile(io, cwd, arena, args[1], &entries); + + const out = try cwd.createFile(io, args[3], .{}); + defer out.close(io); + + var buf: [4096]u8 = undefined; + var writer = out.writer(io, &buf); + + try writer.interface.print("# List of AMDGPU IDs\n" ++ + "#\n" ++ + "# Syntax:\n" ++ + "# device_id,\trevision_id,\tproduct_name <-- single tab after comma\n" ++ + "\n" ++ + "1.0.0\n", .{}); + + var it = entries.valueIterator(); + while (it.next()) |v| { + try writer.interface.print("{s}\n", .{v.*}); + } + + try writer.flush(); +} + +fn parseFile( + io: std.Io, + cwd: std.Io.Dir, + arena: std.mem.Allocator, + path: []const u8, + entries: *std.StringHashMapUnmanaged([]const u8), +) !void { + const file = try cwd.openFile(io, path, .{ .mode = .read_only }); + defer file.close(io); + + var buf: [4096]u8 = undefined; + var reader = file.reader(io, &buf); + + while (true) { + const raw = try reader.interface.takeDelimiter('\n') orelse break; + + const line = std.mem.trimEnd(u8, raw, " \t\r"); + if (line.len == 0 or line[0] == '#') continue; + + var iter = std.mem.splitSequence(u8, line, ","); + + const device = std.mem.trim(u8, iter.next() orelse continue, " \t"); + const revision = std.mem.trim(u8, iter.next() orelse continue, " \t"); + const product = std.mem.trim(u8, iter.rest(), " \t"); + + const key = try std.fmt.allocPrint(arena, "{s},{s}", .{ device, revision }); + const value = try std.fmt.allocPrint(arena, "{s},\t{s},\t{s}", .{ device, revision, product }); + + try entries.put(arena, key, value); + } +} diff --git a/platforms/rocm/packages.lock.json b/platforms/rocm/packages.lock.json new file mode 100644 index 00000000..2b2a4836 --- /dev/null +++ b/platforms/rocm/packages.lock.json @@ -0,0 +1,8467 @@ +{ + "packages": [ + { + "arch": "amd64", + "dependencies": [ + { + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" + } + ], + "key": "llvm-libunwind1_1-22.1.8_-p--p-20260613092327-p-e80beda6e255-1_exp1_20260613092437.81_amd64", + "name": "llvm-libunwind1", + "sha256": "0da2b5596faee4e53456b7bdf51458e7bd5ec06852af93f2538e74c122032df2", + "urls": [ + "https://apt.llvm.org/jammy/pool/main/l/llvm-toolchain-22/llvm-libunwind1_22.1.8~++20260613092327+e80beda6e255-1~exp1~20260613092437.81_amd64.deb" + ], + "version": "1:22.1.8~++20260613092327+e80beda6e255-1~exp1~20260613092437.81" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "sha256": "d8e36ba6cc55719d728c38166bf753650710878e02e30af2aa79aefa244eb275", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/glibc/libc6_2.35-0ubuntu3.10_amd64.deb" + ], + "version": "2.35-0ubuntu3.10" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "sha256": "3fa566e9f861a08736cbc5a97562d9d6e4f0c00450fbeafcb6d7583423b04a98", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libx/libxcrypt/libcrypt1_4.4.27-1_amd64.deb" + ], + "version": "1:4.4.27-1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "sha256": "8ca7b573446d22ef46a97b4ec537c38acc8653f76391127253040acc7b74af5f", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/libgcc-s1_12.3.0-1ubuntu1~22.04_amd64.deb" + ], + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "sha256": "80a1f5df097af3978f7df2808006ca77197865e5eb7a923712aee6876e7a14cb", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/gcc-12-base_12.3.0-1ubuntu1~22.04_amd64.deb" + ], + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" + } + ], + "key": "libatomic1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libatomic1", + "sha256": "56ec991983c7c89a2c15bd5134534531d722ec5b51a60679f7ceaa1a09b37a09", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/libatomic1_12.3.0-1ubuntu1~22.04_amd64.deb" + ], + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libdrm-common_2.4.113-2_ubuntu0.22.04.1_amd64", + "name": "libdrm-common", + "sha256": "35a306712d8b15b30c42ecd73ec087813eb01c0b3125dc8f7ca2b5134e133522", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libd/libdrm/libdrm-common_2.4.113-2~ubuntu0.22.04.1_all.deb" + ], + "version": "2.4.113-2~ubuntu0.22.04.1" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "amdrocm-sysdeps10.1_10.1.0-3_amd64", + "name": "amdrocm-sysdeps10.1", + "version": "10.1.0-3" + }, + { + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, + { + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "version": "3.0.2-0ubuntu1.19" + }, + { + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "version": "1.5.79ubuntu1" + }, + { + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "version": "5.34.0-3ubuntu1.4" + }, + { + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "version": "1.21.1ubuntu2.3" + }, + { + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "version": "3.3-1build2" + }, + { + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "version": "10.39-3ubuntu0.1" + }, + { + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "version": "2.3.1-1" + }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + }, + { + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "version": "1.6.1-2ubuntu3" + }, + { + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "version": "1.46.5-2ubuntu1.2" + }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" + }, + { + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" + }, + { + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" + }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" + }, + { + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" + }, + { + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + } + ], + "key": "amdrocm-amdsmi10.1_10.1.0-3_amd64", + "name": "amdrocm-amdsmi10.1", + "sha256": "994e715d38a16d9c408eebd508c66123ff2f64cf4d4f4ca15780c4500e85103a", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-amdsmi10.1_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sysdeps10.1_10.1.0-3_amd64", + "name": "amdrocm-sysdeps10.1", + "sha256": "a2493d98144a586df49489c4b4b7a7449c9eb123597515929699b1b5fccfd916", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sysdeps10.1_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "sha256": "43dbcc12d790ed9360be4a1dc690ac0a7464a5e7c3170785886df84811a3f5a5", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/python3-defaults/python3_3.10.6-1~22.04.1_amd64.deb" + ], + "version": "3.10.6-1~22.04.1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "sha256": "41b8f285f0a1661ece778a63592dcb556710cce3c559d83114fad245bfdf56ea", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/python3-defaults/libpython3-stdlib_3.10.6-1~22.04.1_amd64.deb" + ], + "version": "3.10.6-1~22.04.1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "sha256": "123b338b5857a35e145c2bd7ce890434dbed2e7f6cd03505e86377f9e0aac40b", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/python3.10/libpython3.10-stdlib_3.10.12-1~22.04.10_amd64.deb" + ], + "version": "3.10.12-1~22.04.10" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "sha256": "3a88f3c8c58a32c623b79f1caace557f63cb997c03fa18eaa31ca0285f635919", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/u/util-linux/libuuid1_2.37.2-4ubuntu3.4_amd64.deb" + ], + "version": "2.37.2-4ubuntu3.4" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "sha256": "f2f07d1ab8ae8a8a6e893975df0bca8683caec516e7a30e8dadbb87d33aa52c4", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libt/libtirpc/libtirpc3_1.3.2-2ubuntu0.1_amd64.deb" + ], + "version": "1.3.2-2ubuntu0.1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "sha256": "cd7b220447999402e87d75f7cedcfad159a2e5d4236927a414d77f2932219bee", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libt/libtirpc/libtirpc-common_1.3.2-2ubuntu0.1_all.deb" + ], + "version": "1.3.2-2ubuntu0.1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "sha256": "2bc13d97c9cd1045271bf600c53ca3ae7516c61d9102f1b9c9d2fa722c35183b", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/k/krb5/libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64.deb" + ], + "version": "1.19.2-2ubuntu0.7" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "sha256": "fd2f8c23945590b38f576f7a58dcba7eb63bb6dc97d37271a15b9d8b2c29210f", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/k/krb5/libkrb5support0_1.19.2-2ubuntu0.7_amd64.deb" + ], + "version": "1.19.2-2ubuntu0.7" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "sha256": "75fc4ab539697f7df8afa2e226b39515b6425111a80aecce1ea41ce392896224", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/k/krb5/libkrb5-3_1.19.2-2ubuntu0.7_amd64.deb" + ], + "version": "1.19.2-2ubuntu0.7" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "sha256": "3a4b984a91fb0fda4d287f1aceeea0f8f1d8a4ed6c9fe7dac114208be92e0f1f", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/o/openssl/libssl3_3.0.2-0ubuntu1.19_amd64.deb" + ], + "version": "3.0.2-0ubuntu1.19" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "sha256": "395a65b3a9539304391456d4d0e5531f4249f297b6e67eb0fcaf2ff77ddc3dc3", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/d/debconf/debconf_1.5.79ubuntu1_all.deb" + ], + "version": "1.5.79ubuntu1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "sha256": "465f2523d9bbbd64c1dc9ffdb894885214723d6e8e6942b4bae6b4b8811fdaa5", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/perl/perl-base_5.34.0-3ubuntu1.4_amd64.deb" + ], + "version": "5.34.0-3ubuntu1.4" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "sha256": "92c21b47281c1bf24d788fe02d717c058fd9701b9c224b0b5c2c1179fb5d152d", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/d/dpkg/dpkg_1.21.1ubuntu2.3_amd64.deb" + ], + "version": "1.21.1ubuntu2.3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "sha256": "151e3a61137aeb04cd714656852877a00d47d918246806f9e5b08d1cef32b816", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/t/tar/tar_1.34+dfsg-1ubuntu0.1.22.04.2_amd64.deb" + ], + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "sha256": "b96c6b40ee2388bd51341cb11c0f1d5bcca29b9180b6e3a77a06b881f2913f7e", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libs/libselinux/libselinux1_3.3-1build2_amd64.deb" + ], + "version": "3.3-1build2" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "sha256": "01a5a74f3d305cad32fffffdc200c2a92fd8e5a01bff97de76ac598fb444d53e", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/pcre2/libpcre2-8-0_10.39-3ubuntu0.1_amd64.deb" + ], + "version": "10.39-3ubuntu0.1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "sha256": "4db2c64ec74f673ed022e92cce7b83d0cbe0b779e02ca60a56ba59ae07754e05", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/a/acl/libacl1_2.3.1-1_amd64.deb" + ], + "version": "2.3.1-1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "sha256": "9dc17e51a1be2d9ed63b7b84ef0e4e29c5abe6f1bc62cb03e7181483cce8a2f2", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/z/zlib/zlib1g_1.2.11.dfsg-2ubuntu9.2_amd64.deb" + ], + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "sha256": "ae7db00ce8b093e50c994518b90203544e063b4bc574836a048bb142b950b2c9", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libz/libzstd/libzstd1_1.4.8+dfsg-3build1_amd64.deb" + ], + "version": "1.4.8+dfsg-3build1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "sha256": "8f1c46e7d3f5102a5e4fdca7c949728a343ba71c2a7c124118df2c13d4c444f7", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/x/xz-utils/liblzma5_5.2.5-2ubuntu1_amd64.deb" + ], + "version": "5.2.5-2ubuntu1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "sha256": "3bfeaf4259eadbb7faa09feee86cd6cad172cd95907d7465afd0eb5aebb5433f", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/b/bzip2/libbz2-1.0_1.0.8-5build1_amd64.deb" + ], + "version": "1.0.8-5build1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "sha256": "940daf78ee0229549b4eabc92ffd79dd038ed96a44e2e912305a23540cf22a0a", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/k/keyutils/libkeyutils1_1.6.1-2ubuntu3_amd64.deb" + ], + "version": "1.6.1-2ubuntu3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "sha256": "76174ecc0948b587a0e92a08b023c9f8655f581aaff476c52f50299e80e92056", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/k/krb5/libk5crypto3_1.19.2-2ubuntu0.7_amd64.deb" + ], + "version": "1.19.2-2ubuntu0.7" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "sha256": "552bc8169451cfdcf174a8da845b13ced0a55777149af5cf47491193161ed791", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/e/e2fsprogs/libcom-err2_1.46.5-2ubuntu1.2_amd64.deb" + ], + "version": "1.46.5-2ubuntu1.2" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "sha256": "1ec03fe34a7dfd8640a0564a3cad385a10b4f68ebf2190a4c08ccadb395133d3", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/n/ncurses/libtinfo6_6.3-2ubuntu0.1_amd64.deb" + ], + "version": "6.3-2ubuntu0.1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "sha256": "5bdd1506004599dca5f659e43b09fa419565168d2b4a5ea4ba11c9e830eae53f", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/s/sqlite3/libsqlite3-0_3.37.2-2ubuntu0.4_amd64.deb" + ], + "version": "3.37.2-2ubuntu0.4" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "sha256": "51fd16f608efa710c987e4b071ffba0bec63a9ac8e55eddd5770f5e5252edc09", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/r/readline/libreadline8_8.1.2-1_amd64.deb" + ], + "version": "8.1.2-1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "sha256": "a78d02935f6107a12d3d349c25ce4f92fcdcbf6b65e37f1f5a13a9054b5b9e94", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/r/readline/readline-common_8.1.2-1_all.deb" + ], + "version": "8.1.2-1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "sha256": "cfeef478f96ace59617f4f93c2497776b98a33c99bf3602af46844ccf9cba9d3", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libn/libnsl/libnsl2_1.3.0-2build2_amd64.deb" + ], + "version": "1.3.0-2build2" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "sha256": "88e5178ccdd4317e263dbe03db1a54a6e35ae1f1fd057585655c2b06ab487f6e", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/n/ncurses/libncursesw6_6.3-2ubuntu0.1_amd64.deb" + ], + "version": "6.3-2ubuntu0.1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "sha256": "941ca0b2e73d26522f75a801b1bf529afa5ceb2ac9b00cbf324b59f474cae813", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/m/mpdecimal/libmpdec3_2.5.1-2build2_amd64.deb" + ], + "version": "2.5.1-2build2" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "sha256": "83873058e692936a09649ede2f8e70b87dde1f3f5488db53da8081b81c79a5d9", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/libstdc++6_12.3.0-1ubuntu1~22.04_amd64.deb" + ], + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "sha256": "b4f88c91fa6f4c942097be6abfc365fb133c5e147640168cbb7704fd855eac10", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libf/libffi/libffi8_3.4.2-4_amd64.deb" + ], + "version": "3.4.2-4" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "sha256": "ce60e446c9cb0e94bb4b4d0fe4f969e772edec250deb186bacf955e4d196548b", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/d/db5.3/libdb5.3_5.3.28+dfsg1-0.8ubuntu3_amd64.deb" + ], + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "sha256": "a993af16a8ccdcefaf75eef360f295f8a3a91a78fd5e5ea1e3fd4f2376df6615", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/m/media-types/media-types_7.0.0_all.deb" + ], + "version": "7.0.0" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "sha256": "9ccaf10c03370cdaaad5182f33a10ae02d65a1aa292486d431efbccf1a207078", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/python3.10/libpython3.10-minimal_3.10.12-1~22.04.10_amd64.deb" + ], + "version": "3.10.12-1~22.04.10" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "sha256": "72189c001d158fd4088b83389b0b0aead43ddde72176c17ead0dd72c8fcb5722", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/python3.10/python3.10_3.10.12-1~22.04.10_amd64.deb" + ], + "version": "3.10.12-1~22.04.10" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "sha256": "2b78a48f72f3cbef21afdfd21a92423dcce4a4596413b85a8ae56cfb6bcc3e5d", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/python3.10/python3.10-minimal_3.10.12-1~22.04.10_amd64.deb" + ], + "version": "3.10.12-1~22.04.10" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "sha256": "b3f93e3fc3f15ca134fd02ba6c59df80a8673f41b04dec9f66c63ce7183fc046", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/e/expat/libexpat1_2.4.7-1ubuntu0.6_amd64.deb" + ], + "version": "2.4.7-1ubuntu0.6" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "sha256": "c2f9ebfa720344485d192f9b493065bee4a29d5021b093f9300c8101483de9dd", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/python3-defaults/python3-minimal_3.10.6-1~22.04.1_amd64.deb" + ], + "version": "3.10.6-1~22.04.1" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "amdrocm-runtime10.1_10.1.0-3_amd64", + "name": "amdrocm-runtime10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-llvm10.1_10.1.0-3_amd64", + "name": "amdrocm-llvm10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sysdeps10.1_10.1.0-3_amd64", + "name": "amdrocm-sysdeps10.1", + "version": "10.1.0-3" + }, + { + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libgcc-11-dev", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libquadmath0_12.3.0-1ubuntu1_22.04_amd64", + "name": "libquadmath0", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libubsan1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libubsan1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libtsan0_11.4.0-1ubuntu1_22.04_amd64", + "name": "libtsan0", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "gcc-11-base_11.4.0-1ubuntu1_22.04_amd64", + "name": "gcc-11-base", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "liblsan0_12.3.0-1ubuntu1_22.04_amd64", + "name": "liblsan0", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libasan6_11.4.0-1ubuntu1_22.04_amd64", + "name": "libasan6", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libatomic1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libatomic1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libitm1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libitm1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libgomp1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgomp1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libstdc-p--p--11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libstdc++-11-dev", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libc6-dev_2.35-0ubuntu3.10_amd64", + "name": "libc6-dev", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libnsl-dev_1.3.0-2build2_amd64", + "name": "libnsl-dev", + "version": "1.3.0-2build2" + }, + { + "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-dev", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "version": "3.0.2-0ubuntu1.19" + }, + { + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "version": "1.5.79ubuntu1" + }, + { + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "version": "5.34.0-3ubuntu1.4" + }, + { + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "version": "1.21.1ubuntu2.3" + }, + { + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "version": "3.3-1build2" + }, + { + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "version": "10.39-3ubuntu0.1" + }, + { + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "version": "2.3.1-1" + }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + }, + { + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "version": "1.6.1-2ubuntu3" + }, + { + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "version": "1.46.5-2ubuntu1.2" + }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, + { + "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", + "name": "rpcsvc-proto", + "version": "1.4.2-0ubuntu6" + }, + { + "key": "libcrypt-dev_1-4.4.27-1_amd64", + "name": "libcrypt-dev", + "version": "1:4.4.27-1" + }, + { + "key": "linux-libc-dev_5.15.0-143.153_amd64", + "name": "linux-libc-dev", + "version": "5.15.0-143.153" + }, + { + "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", + "name": "libc-dev-bin", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" + }, + { + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" + }, + { + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" + }, + { + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" + }, + { + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + } + ], + "key": "amdrocm-base10.1_10.1.0-3_amd64", + "name": "amdrocm-base10.1", + "sha256": "a3d48032edee53c51016b28884a1fe81e2d2f84e087c2546dd20753402ad53c9", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-base10.1_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-runtime10.1_10.1.0-3_amd64", + "name": "amdrocm-runtime10.1", + "sha256": "2cf4fa3b82f9b37be9fc18ca6886e1791585c1b8041758ccba9e3c5dadb7c6b1", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-runtime10.1_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "amdrocm-runtime10.1_10.1.0-3_amd64", + "name": "amdrocm-runtime10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sysdeps10.1_10.1.0-3_amd64", + "name": "amdrocm-sysdeps10.1", + "version": "10.1.0-3" + }, + { + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libgcc-11-dev", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libquadmath0_12.3.0-1ubuntu1_22.04_amd64", + "name": "libquadmath0", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libubsan1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libubsan1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libtsan0_11.4.0-1ubuntu1_22.04_amd64", + "name": "libtsan0", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "gcc-11-base_11.4.0-1ubuntu1_22.04_amd64", + "name": "gcc-11-base", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "liblsan0_12.3.0-1ubuntu1_22.04_amd64", + "name": "liblsan0", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libasan6_11.4.0-1ubuntu1_22.04_amd64", + "name": "libasan6", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libatomic1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libatomic1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libitm1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libitm1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libgomp1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgomp1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libstdc-p--p--11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libstdc++-11-dev", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libc6-dev_2.35-0ubuntu3.10_amd64", + "name": "libc6-dev", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libnsl-dev_1.3.0-2build2_amd64", + "name": "libnsl-dev", + "version": "1.3.0-2build2" + }, + { + "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-dev", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "version": "3.0.2-0ubuntu1.19" + }, + { + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "version": "1.5.79ubuntu1" + }, + { + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "version": "5.34.0-3ubuntu1.4" + }, + { + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "version": "1.21.1ubuntu2.3" + }, + { + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "version": "3.3-1build2" + }, + { + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "version": "10.39-3ubuntu0.1" + }, + { + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "version": "2.3.1-1" + }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + }, + { + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "version": "1.6.1-2ubuntu3" + }, + { + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "version": "1.46.5-2ubuntu1.2" + }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, + { + "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", + "name": "rpcsvc-proto", + "version": "1.4.2-0ubuntu6" + }, + { + "key": "libcrypt-dev_1-4.4.27-1_amd64", + "name": "libcrypt-dev", + "version": "1:4.4.27-1" + }, + { + "key": "linux-libc-dev_5.15.0-143.153_amd64", + "name": "linux-libc-dev", + "version": "5.15.0-143.153" + }, + { + "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", + "name": "libc-dev-bin", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "amdrocm-base10.1_10.1.0-3_amd64", + "name": "amdrocm-base10.1", + "version": "10.1.0-3" + }, + { + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" + }, + { + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" + }, + { + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" + }, + { + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" + }, + { + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + } + ], + "key": "amdrocm-llvm10.1_10.1.0-3_amd64", + "name": "amdrocm-llvm10.1", + "sha256": "6a4727102d830f6c97b911351192b15623189278afad9b4558514a6fe51ca58a", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-llvm10.1_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libgcc-11-dev", + "sha256": "06faa64479f1987e4f870ff9c1ccfdf3c0c441bd3886136d6e24c11316c9f3d6", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-11/libgcc-11-dev_11.4.0-1ubuntu1~22.04_amd64.deb" + ], + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libquadmath0_12.3.0-1ubuntu1_22.04_amd64", + "name": "libquadmath0", + "sha256": "6bf564c2bb191aefcf96758502a7ac24b2af4d644c87b3678e4b1107c5494b13", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/libquadmath0_12.3.0-1ubuntu1~22.04_amd64.deb" + ], + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libubsan1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libubsan1", + "sha256": "becb1726d3b8a45cfd336e502fc29d7c43e024689857c670ecf75a5362732c63", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/libubsan1_12.3.0-1ubuntu1~22.04_amd64.deb" + ], + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libtsan0_11.4.0-1ubuntu1_22.04_amd64", + "name": "libtsan0", + "sha256": "7b14e81317fce5e62d6cc2454d975d3ec4eac90710a1a536d63c9ee84a50b6f2", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-11/libtsan0_11.4.0-1ubuntu1~22.04_amd64.deb" + ], + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "gcc-11-base_11.4.0-1ubuntu1_22.04_amd64", + "name": "gcc-11-base", + "sha256": "5448bba72109dbf2df6ee30463b5f06c074138006cd1d5730fb5189b93e6d563", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-11/gcc-11-base_11.4.0-1ubuntu1~22.04_amd64.deb" + ], + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "liblsan0_12.3.0-1ubuntu1_22.04_amd64", + "name": "liblsan0", + "sha256": "606e212c3cbc8fb622313fdb32f78a024b4b5b9fe54b1cc5a8f731297023b899", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/liblsan0_12.3.0-1ubuntu1~22.04_amd64.deb" + ], + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libasan6_11.4.0-1ubuntu1_22.04_amd64", + "name": "libasan6", + "sha256": "0580e3209722dd0e6d9e57c4149a0f8f6073f975cf8b5ede26ac54289c2fc99a", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-11/libasan6_11.4.0-1ubuntu1~22.04_amd64.deb" + ], + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libitm1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libitm1", + "sha256": "ac3c96ed50a42b4f57c410576bdf816710684e3e32e34f704a7295014565b724", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/libitm1_12.3.0-1ubuntu1~22.04_amd64.deb" + ], + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libgomp1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgomp1", + "sha256": "072a7ee2dc00bc92a816e11ae62ad5a497c86fc1f0473c0a095f29e13003da72", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/libgomp1_12.3.0-1ubuntu1~22.04_amd64.deb" + ], + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libstdc-p--p--11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libstdc++-11-dev", + "sha256": "5ab9c245d4c65a3986d567c54e2503251c92bd369c9f42e48a2c4b1f1835dcf5", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-11/libstdc++-11-dev_11.4.0-1ubuntu1~22.04_amd64.deb" + ], + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libc6-dev_2.35-0ubuntu3.10_amd64", + "name": "libc6-dev", + "sha256": "c2e3b50af761886617462ca47ebc9303a83968129acca2add42b8eba0a7abca3", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/glibc/libc6-dev_2.35-0ubuntu3.10_amd64.deb" + ], + "version": "2.35-0ubuntu3.10" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libnsl-dev_1.3.0-2build2_amd64", + "name": "libnsl-dev", + "sha256": "e93aa448fe708a4bbfcc40b2a8cab6f5f88896174a301caab2719f734e08a621", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libn/libnsl/libnsl-dev_1.3.0-2build2_amd64.deb" + ], + "version": "1.3.0-2build2" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-dev", + "sha256": "f4254aff5a32ef3c9fca06d3f4f2eff71c93b9393cc1ecc718f7478a5d6a83da", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libt/libtirpc/libtirpc-dev_1.3.2-2ubuntu0.1_amd64.deb" + ], + "version": "1.3.2-2ubuntu0.1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", + "name": "rpcsvc-proto", + "sha256": "447c6d8f7624a3d728a12232440d3f10e6e53bbf60cc202b7beb8a3bc9f7acc9", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/r/rpcsvc-proto/rpcsvc-proto_1.4.2-0ubuntu6_amd64.deb" + ], + "version": "1.4.2-0ubuntu6" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libcrypt-dev_1-4.4.27-1_amd64", + "name": "libcrypt-dev", + "sha256": "a79be2f6e45823dcc09e04d5e98c88ec88d07d5b8895d05b875a8ade8b345efa", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libx/libxcrypt/libcrypt-dev_4.4.27-1_amd64.deb" + ], + "version": "1:4.4.27-1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "linux-libc-dev_5.15.0-143.153_amd64", + "name": "linux-libc-dev", + "sha256": "f534bb052e96efb9f9e08279b18950d1b1f2a7958f3a06c426fed7d2c1500840", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/l/linux/linux-libc-dev_5.15.0-143.153_amd64.deb" + ], + "version": "5.15.0-143.153" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", + "name": "libc-dev-bin", + "sha256": "d93f29bc5ad18f69a4fe292fd392dc17e0a9d9bf0a22ee11fcf43e6395b6ecf8", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/glibc/libc-dev-bin_2.35-0ubuntu3.10_amd64.deb" + ], + "version": "2.35-0ubuntu3.10" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "amdrocm-blas10.1-gfx950_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx950", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx950_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx950", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx950_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx950", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse-host10.1_10.1.0-3_amd64", + "name": "amdrocm-sparse-host10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-runtime10.1_10.1.0-3_amd64", + "name": "amdrocm-runtime10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-llvm10.1_10.1.0-3_amd64", + "name": "amdrocm-llvm10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sysdeps10.1_10.1.0-3_amd64", + "name": "amdrocm-sysdeps10.1", + "version": "10.1.0-3" + }, + { + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libgcc-11-dev", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libquadmath0_12.3.0-1ubuntu1_22.04_amd64", + "name": "libquadmath0", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libubsan1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libubsan1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libtsan0_11.4.0-1ubuntu1_22.04_amd64", + "name": "libtsan0", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "gcc-11-base_11.4.0-1ubuntu1_22.04_amd64", + "name": "gcc-11-base", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "liblsan0_12.3.0-1ubuntu1_22.04_amd64", + "name": "liblsan0", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libasan6_11.4.0-1ubuntu1_22.04_amd64", + "name": "libasan6", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libatomic1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libatomic1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libitm1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libitm1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libgomp1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgomp1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libstdc-p--p--11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libstdc++-11-dev", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libc6-dev_2.35-0ubuntu3.10_amd64", + "name": "libc6-dev", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libnsl-dev_1.3.0-2build2_amd64", + "name": "libnsl-dev", + "version": "1.3.0-2build2" + }, + { + "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-dev", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "version": "3.0.2-0ubuntu1.19" + }, + { + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "version": "1.5.79ubuntu1" + }, + { + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "version": "5.34.0-3ubuntu1.4" + }, + { + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "version": "1.21.1ubuntu2.3" + }, + { + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "version": "3.3-1build2" + }, + { + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "version": "10.39-3ubuntu0.1" + }, + { + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "version": "2.3.1-1" + }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + }, + { + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "version": "1.6.1-2ubuntu3" + }, + { + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "version": "1.46.5-2ubuntu1.2" + }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, + { + "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", + "name": "rpcsvc-proto", + "version": "1.4.2-0ubuntu6" + }, + { + "key": "libcrypt-dev_1-4.4.27-1_amd64", + "name": "libcrypt-dev", + "version": "1:4.4.27-1" + }, + { + "key": "linux-libc-dev_5.15.0-143.153_amd64", + "name": "linux-libc-dev", + "version": "5.15.0-143.153" + }, + { + "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", + "name": "libc-dev-bin", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "amdrocm-base10.1_10.1.0-3_amd64", + "name": "amdrocm-base10.1", + "version": "10.1.0-3" + }, + { + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" + }, + { + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" + }, + { + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" + }, + { + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" + }, + { + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "amdrocm-profiler-base10.1_10.1.0-3_amd64", + "name": "amdrocm-profiler-base10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver-host10.1_10.1.0-3_amd64", + "name": "amdrocm-solver-host10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-math-common10.1_10.1.0-3_amd64", + "name": "amdrocm-math-common10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas-host10.1_10.1.0-3_amd64", + "name": "amdrocm-blas-host10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx942_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx942", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx942_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx942", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx942_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx942", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx90a_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx90a", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx90a_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx90a", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx90a_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx90a", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx908_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx908", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx908_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx908", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx908_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx908", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1250_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1250", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1250_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1250", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1250_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1250", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1201_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1201", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1201_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1201", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1201_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1201", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1200_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1200", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1200_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1200", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1200_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1200", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1153_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1153", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1153_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1153", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1153_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1153", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1152_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1152", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1152_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1152", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1152_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1152", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1151_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1151", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1151_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1151", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1151_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1151", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1150_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1150", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1150_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1150", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1150_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1150", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1103_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1103", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1103_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1103", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1103_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1103", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1102_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1102", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1102_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1102", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1102_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1102", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1101_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1101", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1101_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1101", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1101_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1101", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1100_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1100", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1100_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1100", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1100_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1100", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1036_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1036", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1036_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1036", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1036_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1036", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1035_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1035", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1035_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1035", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1035_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1035", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1034_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1034", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1034_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1034", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1034_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1034", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1033_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1033", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1033_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1033", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1033_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1033", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1032_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1032", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1032_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1032", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1032_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1032", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1031_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1031", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1031_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1031", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1031_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1031", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1030_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1030", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1030_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1030", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1030_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1030", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1012_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1012", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1012_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1012", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1012_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1012", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1011_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1011", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1011_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1011", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1011_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1011", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1010_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1010", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1010_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1010", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1010_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1010", + "version": "10.1.0-3" + } + ], + "key": "amdrocm-blas10.1_10.1.0-3_amd64", + "name": "amdrocm-blas10.1", + "sha256": "2a8a63f7c3e5b5e11974433c9cb6abdd297c9939c07cd061f83e0a8d3867e785", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas10.1-gfx950_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx950", + "sha256": "ba944cc7b634d098344315357b126a6ab7fff39ab34970f798d20a3e8b1177d2", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx950_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver10.1-gfx950_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx950", + "sha256": "089bf10ade33b38698d8eb02af7cd8ec9768ad1946afd2e76b18175d7dbbaa26", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx950_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse10.1-gfx950_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx950", + "sha256": "b0714db7f27322097d7d1940e9c95a431d9ec11ff7ea93081ca549f8b72a379c", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx950_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse-host10.1_10.1.0-3_amd64", + "name": "amdrocm-sparse-host10.1", + "sha256": "44faf2440577e65c51b9b6474c1b70a96503a44c7939e6b8ec75512ab928f08e", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse-host10.1_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "amdrocm-sysdeps10.1_10.1.0-3_amd64", + "name": "amdrocm-sysdeps10.1", + "version": "10.1.0-3" + }, + { + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "amdrocm-runtime10.1_10.1.0-3_amd64", + "name": "amdrocm-runtime10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-llvm10.1_10.1.0-3_amd64", + "name": "amdrocm-llvm10.1", + "version": "10.1.0-3" + }, + { + "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libgcc-11-dev", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libquadmath0_12.3.0-1ubuntu1_22.04_amd64", + "name": "libquadmath0", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libubsan1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libubsan1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libtsan0_11.4.0-1ubuntu1_22.04_amd64", + "name": "libtsan0", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "gcc-11-base_11.4.0-1ubuntu1_22.04_amd64", + "name": "gcc-11-base", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "liblsan0_12.3.0-1ubuntu1_22.04_amd64", + "name": "liblsan0", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libasan6_11.4.0-1ubuntu1_22.04_amd64", + "name": "libasan6", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libatomic1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libatomic1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libitm1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libitm1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libgomp1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgomp1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libstdc-p--p--11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libstdc++-11-dev", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libc6-dev_2.35-0ubuntu3.10_amd64", + "name": "libc6-dev", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libnsl-dev_1.3.0-2build2_amd64", + "name": "libnsl-dev", + "version": "1.3.0-2build2" + }, + { + "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-dev", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "version": "3.0.2-0ubuntu1.19" + }, + { + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "version": "1.5.79ubuntu1" + }, + { + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "version": "5.34.0-3ubuntu1.4" + }, + { + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "version": "1.21.1ubuntu2.3" + }, + { + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "version": "3.3-1build2" + }, + { + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "version": "10.39-3ubuntu0.1" + }, + { + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "version": "2.3.1-1" + }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + }, + { + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "version": "1.6.1-2ubuntu3" + }, + { + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "version": "1.46.5-2ubuntu1.2" + }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, + { + "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", + "name": "rpcsvc-proto", + "version": "1.4.2-0ubuntu6" + }, + { + "key": "libcrypt-dev_1-4.4.27-1_amd64", + "name": "libcrypt-dev", + "version": "1:4.4.27-1" + }, + { + "key": "linux-libc-dev_5.15.0-143.153_amd64", + "name": "linux-libc-dev", + "version": "5.15.0-143.153" + }, + { + "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", + "name": "libc-dev-bin", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "amdrocm-base10.1_10.1.0-3_amd64", + "name": "amdrocm-base10.1", + "version": "10.1.0-3" + }, + { + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" + }, + { + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" + }, + { + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" + }, + { + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" + }, + { + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + } + ], + "key": "amdrocm-profiler-base10.1_10.1.0-3_amd64", + "name": "amdrocm-profiler-base10.1", + "sha256": "5f027faf16f293f66fe42cd794ec6b984377183417f49c49c8c2daf3a5675d23", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-profiler-base10.1_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver-host10.1_10.1.0-3_amd64", + "name": "amdrocm-solver-host10.1", + "sha256": "1ec2e8677941eadcb4b87f48c5cfae18511b1fc2ac08794d61740843eabba022", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver-host10.1_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-math-common10.1_10.1.0-3_amd64", + "name": "amdrocm-math-common10.1", + "sha256": "1e4b3e1fc77bc8431e3f5bf81b9193376c9025dd22b6c8a28f87d448cd3d7eaf", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-math-common10.1_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas-host10.1_10.1.0-3_amd64", + "name": "amdrocm-blas-host10.1", + "sha256": "9c35dc27bc27b797f80d42d01cf3d8e042c5f6cc1db66f1a54180d5f21ccf3a3", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas-host10.1_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas10.1-gfx942_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx942", + "sha256": "f988f42ba63f83ca7e65086a5439ac693a0c76c5a963e871b1d8440c7e892c34", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx942_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver10.1-gfx942_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx942", + "sha256": "c7964290972e15f3db7d73a91c460332ecbb19fa1c187cbdf5836c73804c2748", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx942_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse10.1-gfx942_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx942", + "sha256": "0dadc1402ca074a627918c52680d77dfe99b22abdad643ad36fdfb964ef3fa16", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx942_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas10.1-gfx90a_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx90a", + "sha256": "a0799f0aad9e9378f245288ce5ae653ff1b4f716962fe0f8d6eb160820fb9870", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx90a_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver10.1-gfx90a_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx90a", + "sha256": "f329191044bc2aee16a5eb33ab989f393dfbef42c4705542e4d5a0112e406b8d", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx90a_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse10.1-gfx90a_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx90a", + "sha256": "200cae46a18b621385e50b191f9acbacceea04b8c7757148442661c65ea58c40", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx90a_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas10.1-gfx908_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx908", + "sha256": "da0bf3ce9c571e9c2a60cc1964e287ff0c2496ebb156003962212650c8a3a42c", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx908_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver10.1-gfx908_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx908", + "sha256": "0c7e23e418f4892d6d341d9c89f774477d52e105595204801272f2834be1f40b", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx908_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse10.1-gfx908_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx908", + "sha256": "54d6ddf65e98722e9e5c0b39909d7ccd689ec273c6d7020f6095ba7274404152", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx908_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas10.1-gfx1250_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1250", + "sha256": "44db7d9e96ecddd26bd8b069a0a5a89610276c8db859ca7cad9f619f0bf0dd60", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1250_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver10.1-gfx1250_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1250", + "sha256": "7e76b6e1b314d87b55c268f5b6f0b9e300f571eb34de551524bc6302b9ab276b", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1250_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse10.1-gfx1250_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1250", + "sha256": "d555ef3296a63874af1fa99fc7f8f48e57d932abce3c57ec16b2d4cb24ee027e", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1250_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas10.1-gfx1201_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1201", + "sha256": "1906724e7d338bcbf8013e6f02a96618120d39d5d7c8f3b5a40da185be236bc2", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1201_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver10.1-gfx1201_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1201", + "sha256": "43214b4df226aac08647646f71c13dec6cd3be561f9278f0a9fa36b3a5827337", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1201_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse10.1-gfx1201_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1201", + "sha256": "4612811ba76d77f1faa19b1f4da9db829a473253058aa2b170a2ee555de26a15", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1201_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas10.1-gfx1200_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1200", + "sha256": "b4df7e69d7c5c0e6e0ebe262411574153cca723c1ef4e9d445dbb539e1724a16", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1200_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver10.1-gfx1200_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1200", + "sha256": "757a0c725ebcf7701ced94331d1cb4d615aef9e34849c96a84c3aa9dc0e4a754", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1200_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse10.1-gfx1200_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1200", + "sha256": "740dd6b691abe24651412c7d5eac34f10a3878eec939759b5dc39f720d530084", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1200_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas10.1-gfx1153_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1153", + "sha256": "dfaec5607aad9e14282c27c1d1f7440cedbde1d147d2e42c11aada69cfb5c30d", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1153_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver10.1-gfx1153_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1153", + "sha256": "b9cfe52bea768867d858a4b0db6de109b95455c1470528cfb51ba5ef4a62f58d", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1153_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse10.1-gfx1153_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1153", + "sha256": "fac6655f10a9e857c844e0f0c9941213ad543c375335272e97ca9a05c3adf64a", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1153_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas10.1-gfx1152_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1152", + "sha256": "3cb059af8f1ce850cf6fabe3182a1904ada1f2460f638476ad2fd57c43423988", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1152_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver10.1-gfx1152_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1152", + "sha256": "93cccffec257c00cbc45e0970f980fcefe06b2a55893dda128bd97b1b6be29cb", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1152_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse10.1-gfx1152_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1152", + "sha256": "2b348c1379a9ffbfafea1d606d47574d6b009fb7e729eeaa586a4bcac19830b5", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1152_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas10.1-gfx1151_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1151", + "sha256": "9de48ee15f185b471fc6b0e836abea882153e7d3cced6e3157a5f091f21ea4cd", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1151_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver10.1-gfx1151_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1151", + "sha256": "d22416ee83fc82dddf4d094eebe28a1a699b09153ed4d996b7763d8686365ab4", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1151_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse10.1-gfx1151_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1151", + "sha256": "8ab4582f863ceeb2bd0f9bc607786f57d03d34515bed3fb38e4bcaa89b05e8f2", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1151_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas10.1-gfx1150_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1150", + "sha256": "7f2fa46ad609d5cf9b8ef181cf0f76e288880d8916530e33b74df33efe508d37", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1150_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver10.1-gfx1150_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1150", + "sha256": "ffd9c28d8313d64a83401614d6164e4308ade4f7eca52209206c7974f605e541", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1150_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse10.1-gfx1150_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1150", + "sha256": "81626b7367523f6bb0ba918aa284c9ddcbe59093fad6110462d76cf51dfe06a5", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1150_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas10.1-gfx1103_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1103", + "sha256": "2bda07dcfbf333c35dd9b063e8eb4436bb653afea0aa0ef23fac58ebc54ae021", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1103_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver10.1-gfx1103_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1103", + "sha256": "b20f76bd6424de48b4a3d7881bc7ee4dc7c0e060970b237a8db53297b46c2acc", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1103_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse10.1-gfx1103_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1103", + "sha256": "1089660ba5ab633ac0c6f7acc0da10e39926713e71c581f3e27bb7add04cfa59", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1103_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas10.1-gfx1102_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1102", + "sha256": "626a668f339768a2842a9679a417009ad5c885e9a727e0e34ff7386ab9262f71", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1102_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver10.1-gfx1102_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1102", + "sha256": "33d78823a628a6b0bf77f4a36cf977c1d1c816f1aa7e1b55bd2af86370bd3f6c", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1102_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse10.1-gfx1102_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1102", + "sha256": "fcabe97aadc69b9a6f4d006191d0e4fb77df22b734b688763b2c437071c9e11a", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1102_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas10.1-gfx1101_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1101", + "sha256": "736385257ca8d3680f49500667fd7bc3fe483e249e80b550da6386e17ac92cb8", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1101_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver10.1-gfx1101_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1101", + "sha256": "49610fda5782cbf81a0ade0ecea1e4cda4a23ea5f619bbeb19954fd3159bc511", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1101_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse10.1-gfx1101_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1101", + "sha256": "28470e954ec30f8a6cbfff23b85fcdbc0c486b839f904d61a2c922e3157349d0", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1101_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas10.1-gfx1100_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1100", + "sha256": "7998acee523f9ac9e2ce6c36514c8343fb9a651629275f0354404e47e080f076", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1100_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver10.1-gfx1100_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1100", + "sha256": "71d7fdbe3619d77d01db2f7895f73330365ca3e1fb792ccc80f8accfb067db7b", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1100_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse10.1-gfx1100_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1100", + "sha256": "15f0a8421d49acdc19365063b7ce4d3b85102bbfbb517e83bc888e4107061694", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1100_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas10.1-gfx1036_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1036", + "sha256": "dcf5d14b3781669526b279a40866068d60e4d119aa5b628f38fd69e065fba450", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1036_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver10.1-gfx1036_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1036", + "sha256": "7e65239be07b34e784e22f1fd98aa1b02a89a713aad05c3640ee88b08cf38a7d", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1036_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse10.1-gfx1036_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1036", + "sha256": "028a6598a0f90384b08bdfb0ee42ca2031206e8bf5b65730eeef2a77c201cf87", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1036_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas10.1-gfx1035_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1035", + "sha256": "5e0397e891522100a5432d433039e8877479b5111f7226401ed185700a56d2ef", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1035_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver10.1-gfx1035_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1035", + "sha256": "2763f459121dcca48edc4997dc861763cb3c237d102a0cf771209f5d4e0865db", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1035_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse10.1-gfx1035_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1035", + "sha256": "2aed6082214e2ef53cc472d169128238a64ec7ec9300291759a5b6416e173d20", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1035_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas10.1-gfx1034_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1034", + "sha256": "370b7644ad5031bde2dcbb0fdc32441a741869d44d953ff315abe7ba263dbb72", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1034_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver10.1-gfx1034_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1034", + "sha256": "aa24693000c9b848f66135dde49722d19673ef1d3385736b91b5ebe298509b28", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1034_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse10.1-gfx1034_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1034", + "sha256": "0ab0d5238ba6a8d2c0b46e57ace593316c10d93759aa1328937e170c348d1a1f", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1034_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas10.1-gfx1033_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1033", + "sha256": "cf3d658fb0a76fbead063bad0c9c892d7624f9b3c40e41de67cbe49169dfae58", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1033_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver10.1-gfx1033_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1033", + "sha256": "208976c7e784551983664f24330c8ba9734b4bb505e25ae6dac233c98a711052", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1033_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse10.1-gfx1033_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1033", + "sha256": "ced0d2f76cf3113b6bf7246ce1ddd90c638293a61f0c78df1b8099f3cc89a5d7", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1033_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas10.1-gfx1032_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1032", + "sha256": "03aa4d150527f6fb4ab0c84ed650e8ce63a132935ec50641e595693d378529da", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1032_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver10.1-gfx1032_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1032", + "sha256": "9a79590868ea114964ce9ca634b73213600f7800c0cd2d5b26f3e964bf496d28", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1032_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse10.1-gfx1032_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1032", + "sha256": "4182454edef4240bbc911ec01f00deae83e71a8ff878fbcceff70db3d52eaefe", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1032_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas10.1-gfx1031_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1031", + "sha256": "2553208d7552421a7e0100334a0135320dfd07d70746a04d6a75ac49c7c5514d", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1031_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver10.1-gfx1031_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1031", + "sha256": "dc2453d80d4e7690f918ff3dbe9eff6663b1f837aaae3975144282126d7b1527", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1031_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse10.1-gfx1031_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1031", + "sha256": "3f0bb24ff77b733eee320eba671a0b169b1527fc8bf2dfae6c0b4cc5c363ec3f", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1031_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas10.1-gfx1030_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1030", + "sha256": "4b56e03309b8339d2584a7af18609b0f5f8d041dbf08653c1a32f4a36fa9e73b", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1030_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver10.1-gfx1030_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1030", + "sha256": "8c3802bb51b012a36a9012d86c020694f26ebea07492f9bdb3189248b87ec2d1", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1030_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse10.1-gfx1030_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1030", + "sha256": "e8c255c31835bcf6a5193f9ff8ed91b6a94369097ed8c364168dd71d2c163e6e", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1030_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas10.1-gfx1012_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1012", + "sha256": "95c5aa73aaa87b4fb944d5cd05088b1ce500d71f012d3ee1eae76d7db105b906", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1012_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver10.1-gfx1012_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1012", + "sha256": "11afcd2eebbb54385d0334e126a2ff63b90ccf820b04ed0f81ae4042273bfd66", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1012_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse10.1-gfx1012_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1012", + "sha256": "ade96cb7aa4f0a16f23920eafb683686efabee5e5aaaac023d5b11434037d181", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1012_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas10.1-gfx1011_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1011", + "sha256": "7142eefd24ec5363036bb03267e49caf7af820a8f275df4ae63afcb53471cee6", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1011_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver10.1-gfx1011_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1011", + "sha256": "b689b157fd60e30ba111dbea21077086020e0e1eda146995e83b5b44cd117b43", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1011_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse10.1-gfx1011_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1011", + "sha256": "4a4e94348c97148c7332ceb32073cbf330623f507a6022ae398ccfeb6ddc2878", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1011_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-blas10.1-gfx1010_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1010", + "sha256": "052bdaa575e9a47e59ec21d492b5288ee289999d0121c7d0b82484f4a4f3c19f", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1010_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-solver10.1-gfx1010_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1010", + "sha256": "969e5b48158bb841dbdc3db2388e7b13d58016649104b138e2ce307a087f4afa", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1010_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-sparse10.1-gfx1010_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1010", + "sha256": "905dca5b90e3e301512e422eb0d5ae5631e341cb4b8dab4f5f2324ec2947a33f", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1010_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "amdrocm-hipblas-common-dev10.1_10.1.0-3_amd64", + "name": "amdrocm-hipblas-common-dev10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-runtime-dev10.1_10.1.0-3_amd64", + "name": "amdrocm-runtime-dev10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-llvm-dev10.1_10.1.0-3_amd64", + "name": "amdrocm-llvm-dev10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-llvm10.1_10.1.0-3_amd64", + "name": "amdrocm-llvm10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-runtime10.1_10.1.0-3_amd64", + "name": "amdrocm-runtime10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-base10.1_10.1.0-3_amd64", + "name": "amdrocm-base10.1", + "version": "10.1.0-3" + }, + { + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, + { + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "version": "3.0.2-0ubuntu1.19" + }, + { + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "version": "1.5.79ubuntu1" + }, + { + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "version": "5.34.0-3ubuntu1.4" + }, + { + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "version": "1.21.1ubuntu2.3" + }, + { + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "version": "3.3-1build2" + }, + { + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "version": "10.39-3ubuntu0.1" + }, + { + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "version": "2.3.1-1" + }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + }, + { + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "version": "1.6.1-2ubuntu3" + }, + { + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "version": "1.46.5-2ubuntu1.2" + }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" + }, + { + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" + }, + { + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" + }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" + }, + { + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" + }, + { + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "amdrocm-sysdeps10.1_10.1.0-3_amd64", + "name": "amdrocm-sysdeps10.1", + "version": "10.1.0-3" + }, + { + "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libgcc-11-dev", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libquadmath0_12.3.0-1ubuntu1_22.04_amd64", + "name": "libquadmath0", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libubsan1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libubsan1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libtsan0_11.4.0-1ubuntu1_22.04_amd64", + "name": "libtsan0", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "gcc-11-base_11.4.0-1ubuntu1_22.04_amd64", + "name": "gcc-11-base", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "liblsan0_12.3.0-1ubuntu1_22.04_amd64", + "name": "liblsan0", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libasan6_11.4.0-1ubuntu1_22.04_amd64", + "name": "libasan6", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libatomic1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libatomic1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libitm1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libitm1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libgomp1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgomp1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libstdc-p--p--11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libstdc++-11-dev", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libc6-dev_2.35-0ubuntu3.10_amd64", + "name": "libc6-dev", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libnsl-dev_1.3.0-2build2_amd64", + "name": "libnsl-dev", + "version": "1.3.0-2build2" + }, + { + "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-dev", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", + "name": "rpcsvc-proto", + "version": "1.4.2-0ubuntu6" + }, + { + "key": "libcrypt-dev_1-4.4.27-1_amd64", + "name": "libcrypt-dev", + "version": "1:4.4.27-1" + }, + { + "key": "linux-libc-dev_5.15.0-143.153_amd64", + "name": "linux-libc-dev", + "version": "5.15.0-143.153" + }, + { + "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", + "name": "libc-dev-bin", + "version": "2.35-0ubuntu3.10" + } + ], + "key": "amdrocm-blas-dev10.1_10.1.0-3_amd64", + "name": "amdrocm-blas-dev10.1", + "sha256": "c7b6b25d7f1143742a7b7a2bc6819cab56a166995b60ebdc5b282d5ddc67770f", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas-dev10.1_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-hipblas-common-dev10.1_10.1.0-3_amd64", + "name": "amdrocm-hipblas-common-dev10.1", + "sha256": "e8456d1a340533ae73e5a7cbe9aa66feea1f8cedef72971f00930167b80037a3", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-hipblas-common-dev10.1_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "amdrocm-llvm-dev10.1_10.1.0-3_amd64", + "name": "amdrocm-llvm-dev10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-llvm10.1_10.1.0-3_amd64", + "name": "amdrocm-llvm10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-runtime10.1_10.1.0-3_amd64", + "name": "amdrocm-runtime10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-base10.1_10.1.0-3_amd64", + "name": "amdrocm-base10.1", + "version": "10.1.0-3" + }, + { + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, + { + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "version": "3.0.2-0ubuntu1.19" + }, + { + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "version": "1.5.79ubuntu1" + }, + { + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "version": "5.34.0-3ubuntu1.4" + }, + { + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "version": "1.21.1ubuntu2.3" + }, + { + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "version": "3.3-1build2" + }, + { + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "version": "10.39-3ubuntu0.1" + }, + { + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "version": "2.3.1-1" + }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + }, + { + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "version": "1.6.1-2ubuntu3" + }, + { + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "version": "1.46.5-2ubuntu1.2" + }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" + }, + { + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" + }, + { + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" + }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" + }, + { + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" + }, + { + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "amdrocm-sysdeps10.1_10.1.0-3_amd64", + "name": "amdrocm-sysdeps10.1", + "version": "10.1.0-3" + }, + { + "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libgcc-11-dev", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libquadmath0_12.3.0-1ubuntu1_22.04_amd64", + "name": "libquadmath0", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libubsan1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libubsan1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libtsan0_11.4.0-1ubuntu1_22.04_amd64", + "name": "libtsan0", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "gcc-11-base_11.4.0-1ubuntu1_22.04_amd64", + "name": "gcc-11-base", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "liblsan0_12.3.0-1ubuntu1_22.04_amd64", + "name": "liblsan0", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libasan6_11.4.0-1ubuntu1_22.04_amd64", + "name": "libasan6", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libatomic1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libatomic1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libitm1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libitm1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libgomp1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgomp1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libstdc-p--p--11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libstdc++-11-dev", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libc6-dev_2.35-0ubuntu3.10_amd64", + "name": "libc6-dev", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libnsl-dev_1.3.0-2build2_amd64", + "name": "libnsl-dev", + "version": "1.3.0-2build2" + }, + { + "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-dev", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", + "name": "rpcsvc-proto", + "version": "1.4.2-0ubuntu6" + }, + { + "key": "libcrypt-dev_1-4.4.27-1_amd64", + "name": "libcrypt-dev", + "version": "1:4.4.27-1" + }, + { + "key": "linux-libc-dev_5.15.0-143.153_amd64", + "name": "linux-libc-dev", + "version": "5.15.0-143.153" + }, + { + "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", + "name": "libc-dev-bin", + "version": "2.35-0ubuntu3.10" + } + ], + "key": "amdrocm-runtime-dev10.1_10.1.0-3_amd64", + "name": "amdrocm-runtime-dev10.1", + "sha256": "452b3a3b31a4fa4da8d4c08976a5698aa05732b52ab88daa85cb4b4d2c84754c", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-runtime-dev10.1_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-llvm-dev10.1_10.1.0-3_amd64", + "name": "amdrocm-llvm-dev10.1", + "sha256": "351db702fd37be1fda7e5221aedf19484e99d27f2a3e7d2df5f0f531bcebcfd5", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-llvm-dev10.1_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "amdrocm-dnn10.1-gfx950_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx950", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx950_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx950", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx950_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx950", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx950_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx950", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse-host10.1_10.1.0-3_amd64", + "name": "amdrocm-sparse-host10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-runtime10.1_10.1.0-3_amd64", + "name": "amdrocm-runtime10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-llvm10.1_10.1.0-3_amd64", + "name": "amdrocm-llvm10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sysdeps10.1_10.1.0-3_amd64", + "name": "amdrocm-sysdeps10.1", + "version": "10.1.0-3" + }, + { + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libgcc-11-dev", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libquadmath0_12.3.0-1ubuntu1_22.04_amd64", + "name": "libquadmath0", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libubsan1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libubsan1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libtsan0_11.4.0-1ubuntu1_22.04_amd64", + "name": "libtsan0", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "gcc-11-base_11.4.0-1ubuntu1_22.04_amd64", + "name": "gcc-11-base", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "liblsan0_12.3.0-1ubuntu1_22.04_amd64", + "name": "liblsan0", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libasan6_11.4.0-1ubuntu1_22.04_amd64", + "name": "libasan6", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libatomic1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libatomic1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libitm1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libitm1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libgomp1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgomp1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libstdc-p--p--11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libstdc++-11-dev", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libc6-dev_2.35-0ubuntu3.10_amd64", + "name": "libc6-dev", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libnsl-dev_1.3.0-2build2_amd64", + "name": "libnsl-dev", + "version": "1.3.0-2build2" + }, + { + "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-dev", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "version": "3.0.2-0ubuntu1.19" + }, + { + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "version": "1.5.79ubuntu1" + }, + { + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "version": "5.34.0-3ubuntu1.4" + }, + { + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "version": "1.21.1ubuntu2.3" + }, + { + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "version": "3.3-1build2" + }, + { + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "version": "10.39-3ubuntu0.1" + }, + { + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "version": "2.3.1-1" + }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + }, + { + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "version": "1.6.1-2ubuntu3" + }, + { + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "version": "1.46.5-2ubuntu1.2" + }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, + { + "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", + "name": "rpcsvc-proto", + "version": "1.4.2-0ubuntu6" + }, + { + "key": "libcrypt-dev_1-4.4.27-1_amd64", + "name": "libcrypt-dev", + "version": "1:4.4.27-1" + }, + { + "key": "linux-libc-dev_5.15.0-143.153_amd64", + "name": "linux-libc-dev", + "version": "5.15.0-143.153" + }, + { + "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", + "name": "libc-dev-bin", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "amdrocm-base10.1_10.1.0-3_amd64", + "name": "amdrocm-base10.1", + "version": "10.1.0-3" + }, + { + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" + }, + { + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" + }, + { + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" + }, + { + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" + }, + { + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "amdrocm-profiler-base10.1_10.1.0-3_amd64", + "name": "amdrocm-profiler-base10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver-host10.1_10.1.0-3_amd64", + "name": "amdrocm-solver-host10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-math-common10.1_10.1.0-3_amd64", + "name": "amdrocm-math-common10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas-host10.1_10.1.0-3_amd64", + "name": "amdrocm-blas-host10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-dnn-host10.1_10.1.0-3_amd64", + "name": "amdrocm-dnn-host10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-dnn10.1-gfx942_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx942", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx942_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx942", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx942_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx942", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx942_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx942", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-dnn10.1-gfx90a_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx90a", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx90a_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx90a", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx90a_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx90a", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx90a_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx90a", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-dnn10.1-gfx908_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx908", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx908_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx908", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx908_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx908", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx908_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx908", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-dnn10.1-gfx1250_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1250", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1250_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1250", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1250_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1250", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1250_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1250", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-dnn10.1-gfx1201_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1201", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1201_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1201", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1201_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1201", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1201_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1201", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-dnn10.1-gfx1200_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1200", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1200_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1200", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1200_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1200", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1200_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1200", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-dnn10.1-gfx1153_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1153", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1153_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1153", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1153_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1153", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1153_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1153", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-dnn10.1-gfx1152_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1152", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1152_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1152", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1152_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1152", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1152_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1152", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-dnn10.1-gfx1151_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1151", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1151_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1151", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1151_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1151", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1151_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1151", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-dnn10.1-gfx1150_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1150", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1150_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1150", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1150_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1150", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1150_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1150", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-dnn10.1-gfx1103_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1103", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1103_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1103", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1103_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1103", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1103_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1103", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-dnn10.1-gfx1102_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1102", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1102_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1102", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1102_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1102", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1102_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1102", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-dnn10.1-gfx1101_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1101", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1101_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1101", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1101_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1101", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1101_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1101", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-dnn10.1-gfx1100_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1100", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1100_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1100", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1100_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1100", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1100_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1100", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-dnn10.1-gfx1036_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1036", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1036_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1036", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1036_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1036", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1036_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1036", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-dnn10.1-gfx1035_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1035", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1035_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1035", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1035_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1035", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1035_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1035", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-dnn10.1-gfx1034_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1034", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1034_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1034", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1034_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1034", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1034_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1034", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-dnn10.1-gfx1033_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1033", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1033_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1033", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1033_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1033", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1033_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1033", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-dnn10.1-gfx1032_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1032", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1032_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1032", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1032_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1032", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1032_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1032", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-dnn10.1-gfx1031_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1031", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1031_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1031", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1031_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1031", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1031_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1031", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-dnn10.1-gfx1030_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1030", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1030_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1030", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1030_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1030", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1030_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1030", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-dnn10.1-gfx1012_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1012", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1012_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1012", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1012_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1012", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1012_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1012", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-dnn10.1-gfx1011_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1011", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1011_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1011", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1011_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1011", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1011_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1011", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-dnn10.1-gfx1010_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1010", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1010_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1010", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1010_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1010", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1010_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1010", + "version": "10.1.0-3" + } + ], + "key": "amdrocm-dnn10.1_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1", + "sha256": "2a65817ed4bd283ff908a6789c1beaa95e1232043b9c0261edb4f7b031e1880e", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx950_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx950", + "sha256": "b945f4d2f35c53226a43ef798c036e0ea23eb4844c12a9f917fb450a41e756d4", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx950_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn-host10.1_10.1.0-3_amd64", + "name": "amdrocm-dnn-host10.1", + "sha256": "199c66da1e43484d21dfe1581bcf84deec3ee82d106605058b932d8c3dc86359", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn-host10.1_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx942_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx942", + "sha256": "78e43e7ecf4a52b895fc9500e778abc0d4882a7ad5f60caff8830bb2afd64119", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx942_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx90a_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx90a", + "sha256": "6fa5ee1edecb61cc9434659cfece977c636da00b63d9886646848b876a60cba6", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx90a_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx908_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx908", + "sha256": "349a3f6c0c9cf7981d5cc2476be98e0893a5b54033a541660e084a3011efba17", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx908_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx1250_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1250", + "sha256": "bffbfd01dd43330cdc2a6957b4100c64983d9e94e0fecda43dd799ce6b545534", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1250_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx1201_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1201", + "sha256": "d53c6f605df4faa6cfbc4ea9a3f3e75549dbec3400ab899ab502bb8d68681d3f", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1201_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx1200_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1200", + "sha256": "e8e27e5fe95a8092af0c7b287e1c8b2ebb4a4099677cb274a56d09d04f6effd6", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1200_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx1153_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1153", + "sha256": "39609097b14134dfce5937940c0d6118f089f8963f41b88aaf82c6dbe36d8d88", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1153_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx1152_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1152", + "sha256": "f9383ce30973bca37538eab93173f7a6f277189dc1f2c32285902284524b9f6c", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1152_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx1151_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1151", + "sha256": "daf6239c37012e979b4b31612cc27c1f6b5df16c4d0a3eb9ffdb7149f6c0c88e", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1151_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx1150_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1150", + "sha256": "57370823faa258ee6c28650dbaa1afe4a983f0d431c0a4f59b45c52bfd5aa776", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1150_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx1103_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1103", + "sha256": "31103c9e996fb78bb9fbee801b67401fb757bf0f909d35d22ee3c555f6f2419c", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1103_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx1102_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1102", + "sha256": "ba17cd8597c96637216596edc2146297bf3606e32e9bac7dce06147a8d171a06", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1102_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx1101_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1101", + "sha256": "25844db830121a1bb8cb3e64e1d04eb8fbda654a14c7e08b96eafce54891c789", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1101_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx1100_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1100", + "sha256": "7b742fc6696895809f20edaf373a22cd50fd19f17086c56d6cb144d025b32712", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1100_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx1036_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1036", + "sha256": "46658d6cf72bfc08aae94e13c7d7c34209714e700dd033b2c109203addc158b1", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1036_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx1035_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1035", + "sha256": "9d2165f2a2771936204ab97105a427c3b489ddc6984cebf8d3786822fbd566e9", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1035_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx1034_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1034", + "sha256": "d3c75ffa1b9f79165bfcad631d009149742780114e132b3cd3e4912eb0f23632", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1034_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx1033_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1033", + "sha256": "ebb48929bd2faf317aaa457096d4cefd6a9a2f8b9cbe2492bffb2e52135bdea3", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1033_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx1032_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1032", + "sha256": "ee57ec63c768f031899bb630b42ca780bd753ce5c2543d04aea25bd86a04bf33", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1032_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx1031_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1031", + "sha256": "6ce6789fcbaaac92404402adeec89e1170ccd04cccc7340240338f506f3e0e27", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1031_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx1030_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1030", + "sha256": "e33aeb4b4acd112e37a278e006fe214a19927451e63d8bfd8db7cf4fb04a03a6", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1030_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx1012_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1012", + "sha256": "21723b94ee22e0cc5caf839dab16c431e0b6b7f91d9a06cb22e58cc486ae7a5f", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1012_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx1011_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1011", + "sha256": "4b1722d77fc4e0509802a1c56466cc72a8e208ade66f83b92c32cb54ec9c357e", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1011_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-dnn10.1-gfx1010_10.1.0-3_amd64", + "name": "amdrocm-dnn10.1-gfx1010", + "sha256": "c9e57b5b1c75dcbf41dc69310adb3cdac7ef807e3119dd51a0357f2a9429f97a", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1010_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "amdrocm-fft10.1-gfx950_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx950", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-fft-host10.1_10.1.0-3_amd64", + "name": "amdrocm-fft-host10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sysdeps10.1_10.1.0-3_amd64", + "name": "amdrocm-sysdeps10.1", + "version": "10.1.0-3" + }, + { + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "amdrocm-runtime10.1_10.1.0-3_amd64", + "name": "amdrocm-runtime10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-llvm10.1_10.1.0-3_amd64", + "name": "amdrocm-llvm10.1", + "version": "10.1.0-3" + }, + { + "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libgcc-11-dev", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libquadmath0_12.3.0-1ubuntu1_22.04_amd64", + "name": "libquadmath0", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libubsan1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libubsan1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libtsan0_11.4.0-1ubuntu1_22.04_amd64", + "name": "libtsan0", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "gcc-11-base_11.4.0-1ubuntu1_22.04_amd64", + "name": "gcc-11-base", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "liblsan0_12.3.0-1ubuntu1_22.04_amd64", + "name": "liblsan0", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libasan6_11.4.0-1ubuntu1_22.04_amd64", + "name": "libasan6", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libatomic1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libatomic1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libitm1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libitm1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libgomp1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgomp1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libstdc-p--p--11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libstdc++-11-dev", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libc6-dev_2.35-0ubuntu3.10_amd64", + "name": "libc6-dev", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libnsl-dev_1.3.0-2build2_amd64", + "name": "libnsl-dev", + "version": "1.3.0-2build2" + }, + { + "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-dev", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "version": "3.0.2-0ubuntu1.19" + }, + { + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "version": "1.5.79ubuntu1" + }, + { + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "version": "5.34.0-3ubuntu1.4" + }, + { + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "version": "1.21.1ubuntu2.3" + }, + { + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "version": "3.3-1build2" + }, + { + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "version": "10.39-3ubuntu0.1" + }, + { + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "version": "2.3.1-1" + }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + }, + { + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "version": "1.6.1-2ubuntu3" + }, + { + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "version": "1.46.5-2ubuntu1.2" + }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, + { + "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", + "name": "rpcsvc-proto", + "version": "1.4.2-0ubuntu6" + }, + { + "key": "libcrypt-dev_1-4.4.27-1_amd64", + "name": "libcrypt-dev", + "version": "1:4.4.27-1" + }, + { + "key": "linux-libc-dev_5.15.0-143.153_amd64", + "name": "linux-libc-dev", + "version": "5.15.0-143.153" + }, + { + "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", + "name": "libc-dev-bin", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "amdrocm-base10.1_10.1.0-3_amd64", + "name": "amdrocm-base10.1", + "version": "10.1.0-3" + }, + { + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" + }, + { + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" + }, + { + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" + }, + { + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" + }, + { + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "amdrocm-fft10.1-gfx942_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx942", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-fft10.1-gfx90a_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx90a", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-fft10.1-gfx908_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx908", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-fft10.1-gfx1250_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1250", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-fft10.1-gfx1201_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1201", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-fft10.1-gfx1200_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1200", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-fft10.1-gfx1153_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1153", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-fft10.1-gfx1152_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1152", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-fft10.1-gfx1151_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1151", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-fft10.1-gfx1150_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1150", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-fft10.1-gfx1103_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1103", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-fft10.1-gfx1102_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1102", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-fft10.1-gfx1101_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1101", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-fft10.1-gfx1100_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1100", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-fft10.1-gfx1036_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1036", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-fft10.1-gfx1035_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1035", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-fft10.1-gfx1034_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1034", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-fft10.1-gfx1033_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1033", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-fft10.1-gfx1032_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1032", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-fft10.1-gfx1031_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1031", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-fft10.1-gfx1030_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1030", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-fft10.1-gfx1012_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1012", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-fft10.1-gfx1011_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1011", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-fft10.1-gfx1010_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1010", + "version": "10.1.0-3" + } + ], + "key": "amdrocm-fft10.1_10.1.0-3_amd64", + "name": "amdrocm-fft10.1", + "sha256": "0246b0530ab01388018cafba38ee64736933c3b5eeea0e21dcd9dafb44a5950b", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx950_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx950", + "sha256": "3be1ec8ef93a0d7c4d3c463fa92a0ab0f7422f09e5c278d486e42424897704c8", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx950_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft-host10.1_10.1.0-3_amd64", + "name": "amdrocm-fft-host10.1", + "sha256": "5eb075b4dd89ea9d500921959be55a469c60675f53c89e846e186da1d353c27e", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft-host10.1_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx942_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx942", + "sha256": "415771e4caefb3af0e91264a6ac232ab0a1401aa68d0e9c9b836cc177c609ac2", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx942_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx90a_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx90a", + "sha256": "c626c38512d954de19bcc298cfde38e419d4fc21d57665c10d92523c18419265", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx90a_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx908_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx908", + "sha256": "3310f7fe026228c270df2b9b8100c2a64ef7aaae783a54e55ca5099fe57a718b", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx908_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx1250_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1250", + "sha256": "7b8cb472a25096913a4329f5dfc2e124e98621764aff2e04b79db5d7f6ce3ea8", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1250_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx1201_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1201", + "sha256": "eb10f46c03ffdbc1be901e2fefc16fcd3dcd36079acc17bf2a991dda010ca22e", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1201_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx1200_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1200", + "sha256": "04f88bce83b753730025bc352c61b16027c2a6d565f85763ce8990faf155231f", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1200_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx1153_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1153", + "sha256": "cd7270fd487b7abea0498a832122c11186ac234981822274af84520b8980c0c3", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1153_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx1152_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1152", + "sha256": "67250ae45a6190b8e0f0bea35a8019a8fd5a5da070432206a32c4ae3388f2e0e", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1152_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx1151_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1151", + "sha256": "f70434a361d0207188a0feff30d677164c1fdcdb0af970acbc55561b3bfc42f5", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1151_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx1150_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1150", + "sha256": "26b23859562047c84e8eb4ff7387bcab69b475eeb8655e577b0c95e436e6c2f4", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1150_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx1103_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1103", + "sha256": "0cd8464f08f8b6b08ed10773352a34d1d1efea4de3c039a36e6f3bb4fdfb0f9d", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1103_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx1102_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1102", + "sha256": "ec4b9439df683ee782241ec4551daa4e0ec4e4ae4a8c2b92f1bd41285cad596e", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1102_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx1101_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1101", + "sha256": "2ed4b2fbabb310fdb7fdf8ed01c9ca1f0f37fe97b5e3027451737be4815492a8", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1101_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx1100_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1100", + "sha256": "df323baa9cfb3a2c2dd81bd81510021db4780eb074717b22abb9dbd80d992dc0", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1100_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx1036_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1036", + "sha256": "6eb45f4a815e10d2119370e0ed68fe44bc34c2999c3b0a799d15d23d96f17ed8", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1036_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx1035_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1035", + "sha256": "1da409970e160027c8e5fbf9d446394fa9108b92163ba69796b7d7f598f65d18", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1035_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx1034_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1034", + "sha256": "58351fe6a54cc374a3d94a61416c7c4079303e2b38e974d54381bd4f5b53f83e", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1034_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx1033_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1033", + "sha256": "98bb8653582f9840ba723b0344204be8f6c55c758e756acec6ee372e9233119d", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1033_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx1032_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1032", + "sha256": "e4b0fc5dab2eaac470a54811d19bef3b9c1f903e94d5dfa56a4f3f05b8d5977d", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1032_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx1031_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1031", + "sha256": "998eae13396245c496da21504fff6832654b50323cc40cb694060a79f943a51c", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1031_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx1030_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1030", + "sha256": "e321ced70752e27d4c8b70eaf3b8590b9ce6019d64896cb5cfca75d4ad26bf89", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1030_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx1012_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1012", + "sha256": "b8910ca5eea38a9b9e68d3752637e214ded07e84b4ac83da39f15157cc56bf64", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1012_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx1011_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1011", + "sha256": "f6cc9a8657250ba79d739ca65bc8cb13491629695b9c486060652ea9fb508656", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1011_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-fft10.1-gfx1010_10.1.0-3_amd64", + "name": "amdrocm-fft10.1-gfx1010", + "sha256": "378078f09766726732a4f192397b6f1d9fee346b68efd519b58238f9c41f895a", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1010_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "amdrocm-rccl10.1-gfx950_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx950", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-rccl-host10.1_10.1.0-3_amd64", + "name": "amdrocm-rccl-host10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-profiler-base10.1_10.1.0-3_amd64", + "name": "amdrocm-profiler-base10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sysdeps10.1_10.1.0-3_amd64", + "name": "amdrocm-sysdeps10.1", + "version": "10.1.0-3" + }, + { + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "amdrocm-runtime10.1_10.1.0-3_amd64", + "name": "amdrocm-runtime10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-llvm10.1_10.1.0-3_amd64", + "name": "amdrocm-llvm10.1", + "version": "10.1.0-3" + }, + { + "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libgcc-11-dev", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libquadmath0_12.3.0-1ubuntu1_22.04_amd64", + "name": "libquadmath0", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libubsan1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libubsan1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libtsan0_11.4.0-1ubuntu1_22.04_amd64", + "name": "libtsan0", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "gcc-11-base_11.4.0-1ubuntu1_22.04_amd64", + "name": "gcc-11-base", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "liblsan0_12.3.0-1ubuntu1_22.04_amd64", + "name": "liblsan0", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libasan6_11.4.0-1ubuntu1_22.04_amd64", + "name": "libasan6", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libatomic1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libatomic1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libitm1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libitm1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libgomp1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgomp1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libstdc-p--p--11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libstdc++-11-dev", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libc6-dev_2.35-0ubuntu3.10_amd64", + "name": "libc6-dev", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libnsl-dev_1.3.0-2build2_amd64", + "name": "libnsl-dev", + "version": "1.3.0-2build2" + }, + { + "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-dev", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "version": "3.0.2-0ubuntu1.19" + }, + { + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "version": "1.5.79ubuntu1" + }, + { + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "version": "5.34.0-3ubuntu1.4" + }, + { + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "version": "1.21.1ubuntu2.3" + }, + { + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "version": "3.3-1build2" + }, + { + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "version": "10.39-3ubuntu0.1" + }, + { + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "version": "2.3.1-1" + }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + }, + { + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "version": "1.6.1-2ubuntu3" + }, + { + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "version": "1.46.5-2ubuntu1.2" + }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, + { + "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", + "name": "rpcsvc-proto", + "version": "1.4.2-0ubuntu6" + }, + { + "key": "libcrypt-dev_1-4.4.27-1_amd64", + "name": "libcrypt-dev", + "version": "1:4.4.27-1" + }, + { + "key": "linux-libc-dev_5.15.0-143.153_amd64", + "name": "linux-libc-dev", + "version": "5.15.0-143.153" + }, + { + "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", + "name": "libc-dev-bin", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "amdrocm-base10.1_10.1.0-3_amd64", + "name": "amdrocm-base10.1", + "version": "10.1.0-3" + }, + { + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" + }, + { + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" + }, + { + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" + }, + { + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" + }, + { + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "amdrocm-amdsmi10.1_10.1.0-3_amd64", + "name": "amdrocm-amdsmi10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-rccl10.1-gfx942_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx942", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-rccl10.1-gfx90a_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx90a", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-rccl10.1-gfx908_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx908", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-rccl10.1-gfx1250_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1250", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-rccl10.1-gfx1201_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1201", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-rccl10.1-gfx1200_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1200", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-rccl10.1-gfx1151_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1151", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-rccl10.1-gfx1102_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1102", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-rccl10.1-gfx1101_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1101", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-rccl10.1-gfx1100_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1100", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-rccl10.1-gfx1036_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1036", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-rccl10.1-gfx1035_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1035", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-rccl10.1-gfx1034_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1034", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-rccl10.1-gfx1033_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1033", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-rccl10.1-gfx1032_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1032", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-rccl10.1-gfx1031_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1031", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-rccl10.1-gfx1030_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1030", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-rccl10.1-gfx1012_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1012", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-rccl10.1-gfx1011_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1011", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-rccl10.1-gfx1010_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1010", + "version": "10.1.0-3" + } + ], + "key": "amdrocm-rccl10.1_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1", + "sha256": "18b5e340191a729524ede9ec9d1810f3673d035b4940743307189e1b4a96d293", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx950_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx950", + "sha256": "d9a77415ca9392af357d188df8e805f86c8b0f7574705133ee919b88fec66f4d", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx950_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl-host10.1_10.1.0-3_amd64", + "name": "amdrocm-rccl-host10.1", + "sha256": "b38172c360fc8fdf89d038edfa3cb5c3cb874804f089a4c9ba91c384be7c9707", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl-host10.1_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx942_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx942", + "sha256": "c0c4be063b49364858d76af509cbf88e4642146489a1fcde2088f66a2e1f31a9", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx942_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx90a_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx90a", + "sha256": "ffcc6bce6b2271b3c6cd1c5d7b568fe2773a0022a2ba4d24c177f7df5d6c5225", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx90a_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx908_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx908", + "sha256": "2eebb6a869970c182dd64105da18f956801ed7a048693f47f99a05a087b01f59", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx908_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx1250_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1250", + "sha256": "941da91fc0f827732d0c4e0010403723643a27dd3ca2494270aa97d0a8a38f35", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1250_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx1201_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1201", + "sha256": "4c589250c3c587d4865551b15f32adb46cfbb5aaa07e147eb82b3d6634cff213", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1201_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx1200_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1200", + "sha256": "33086fcdf5aa8e353c90ac38effddca41cb3d8ae88272333df2c580b08826abe", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1200_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx1151_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1151", + "sha256": "4d7b24fc17351d69c63028867b5bdf4b9015e563175c868a814c8ca34b7bbcce", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1151_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx1102_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1102", + "sha256": "935b928d8f2901861021d3f9cb2c43953d309975c4e5803b4ac496d6bd4ee007", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1102_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx1101_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1101", + "sha256": "4d418a9fb372ec15bd9e42faa02f9bf945adc5de2853cb77db54cb658286c67c", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1101_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx1100_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1100", + "sha256": "ceddf293510c927d2a5a8fee30cc76fce6cd0d3cc6977422396514bedbcc597e", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1100_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx1036_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1036", + "sha256": "8fb0c4dfa528a82c872df49b0feeb7205e9664ce06d9760d0fa85a1892d97f79", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1036_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx1035_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1035", + "sha256": "174385d65d68e5c5aa7c51f5cfcf90fae9005c37944f4515883f370760830b74", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1035_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx1034_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1034", + "sha256": "c3e4727c061c15aa67b89b61539d58be182dca850df13defe1790b7b5b7ffd4d", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1034_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx1033_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1033", + "sha256": "521d5e2bc7c7d8f4d941044472fd9f800d5f172d4c9a3c015b035e2454c2dba1", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1033_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx1032_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1032", + "sha256": "198345dc05c10db202080744337fee857c367786a315004e662dc24340cff787", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1032_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx1031_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1031", + "sha256": "91761516a93977fcc70cb771e5ad2e7f958f8147905ca86adf592e5f7d79dab5", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1031_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx1030_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1030", + "sha256": "076013938bd6ef83e31d5c4f444d7fb136d0733a4edfdfccf05d8108776c3388", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1030_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx1012_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1012", + "sha256": "473c3be9bf38e2f30dac5f6019a521c3aad2b96e10e28389f542c799141b8f09", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1012_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx1011_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1011", + "sha256": "842068f0334f6c3d4977394c9ee5098f8a8254012206b5f66f5b2aaa0e510ce1", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1011_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "amdrocm-rccl10.1-gfx1010_10.1.0-3_amd64", + "name": "amdrocm-rccl10.1-gfx1010", + "sha256": "09c782685d0d6a93e767a580dc5e35183cf9f742859953271d8abaa977a74c6e", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1010_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "amdrocm-solver10.1-gfx950_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx950", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx950_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx950", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas-host10.1_10.1.0-3_amd64", + "name": "amdrocm-blas-host10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-profiler-base10.1_10.1.0-3_amd64", + "name": "amdrocm-profiler-base10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sysdeps10.1_10.1.0-3_amd64", + "name": "amdrocm-sysdeps10.1", + "version": "10.1.0-3" + }, + { + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "amdrocm-runtime10.1_10.1.0-3_amd64", + "name": "amdrocm-runtime10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-llvm10.1_10.1.0-3_amd64", + "name": "amdrocm-llvm10.1", + "version": "10.1.0-3" + }, + { + "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libgcc-11-dev", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libquadmath0_12.3.0-1ubuntu1_22.04_amd64", + "name": "libquadmath0", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libubsan1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libubsan1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libtsan0_11.4.0-1ubuntu1_22.04_amd64", + "name": "libtsan0", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "gcc-11-base_11.4.0-1ubuntu1_22.04_amd64", + "name": "gcc-11-base", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "liblsan0_12.3.0-1ubuntu1_22.04_amd64", + "name": "liblsan0", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libasan6_11.4.0-1ubuntu1_22.04_amd64", + "name": "libasan6", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libatomic1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libatomic1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libitm1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libitm1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libgomp1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgomp1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libstdc-p--p--11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libstdc++-11-dev", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libc6-dev_2.35-0ubuntu3.10_amd64", + "name": "libc6-dev", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libnsl-dev_1.3.0-2build2_amd64", + "name": "libnsl-dev", + "version": "1.3.0-2build2" + }, + { + "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-dev", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "version": "3.0.2-0ubuntu1.19" + }, + { + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "version": "1.5.79ubuntu1" + }, + { + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "version": "5.34.0-3ubuntu1.4" + }, + { + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "version": "1.21.1ubuntu2.3" + }, + { + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "version": "3.3-1build2" + }, + { + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "version": "10.39-3ubuntu0.1" + }, + { + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "version": "2.3.1-1" + }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + }, + { + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "version": "1.6.1-2ubuntu3" + }, + { + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "version": "1.46.5-2ubuntu1.2" + }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, + { + "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", + "name": "rpcsvc-proto", + "version": "1.4.2-0ubuntu6" + }, + { + "key": "libcrypt-dev_1-4.4.27-1_amd64", + "name": "libcrypt-dev", + "version": "1:4.4.27-1" + }, + { + "key": "linux-libc-dev_5.15.0-143.153_amd64", + "name": "linux-libc-dev", + "version": "5.15.0-143.153" + }, + { + "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", + "name": "libc-dev-bin", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "amdrocm-base10.1_10.1.0-3_amd64", + "name": "amdrocm-base10.1", + "version": "10.1.0-3" + }, + { + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" + }, + { + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" + }, + { + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" + }, + { + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" + }, + { + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "amdrocm-sparse10.1-gfx950_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx950", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse-host10.1_10.1.0-3_amd64", + "name": "amdrocm-sparse-host10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver-host10.1_10.1.0-3_amd64", + "name": "amdrocm-solver-host10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-math-common10.1_10.1.0-3_amd64", + "name": "amdrocm-math-common10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx942_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx942", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx942_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx942", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx942_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx942", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx90a_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx90a", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx90a_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx90a", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx90a_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx90a", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx908_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx908", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx908_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx908", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx908_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx908", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1250_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1250", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1250_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1250", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1250_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1250", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1201_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1201", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1201_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1201", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1201_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1201", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1200_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1200", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1200_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1200", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1200_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1200", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1153_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1153", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1153_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1153", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1153_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1153", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1152_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1152", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1152_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1152", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1152_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1152", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1151_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1151", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1151_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1151", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1151_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1151", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1150_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1150", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1150_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1150", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1150_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1150", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1103_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1103", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1103_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1103", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1103_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1103", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1102_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1102", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1102_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1102", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1102_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1102", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1101_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1101", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1101_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1101", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1101_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1101", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1100_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1100", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1100_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1100", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1100_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1100", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1036_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1036", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1036_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1036", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1036_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1036", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1035_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1035", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1035_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1035", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1035_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1035", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1034_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1034", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1034_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1034", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1034_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1034", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1033_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1033", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1033_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1033", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1033_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1033", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1032_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1032", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1032_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1032", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1032_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1032", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1031_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1031", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1031_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1031", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1031_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1031", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1030_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1030", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1030_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1030", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1030_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1030", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1012_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1012", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1012_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1012", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1012_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1012", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1011_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1011", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1011_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1011", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1011_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1011", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1010_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1010", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1010_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1010", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1010_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1010", + "version": "10.1.0-3" + } + ], + "key": "amdrocm-solver10.1_10.1.0-3_amd64", + "name": "amdrocm-solver10.1", + "sha256": "66f914aa374d13a700da9006af9dcd4aeae20c02a50b4807c45d3ed990da0711", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "amdrocm-sparse10.1-gfx950_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx950", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx950_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx950", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx950_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx950", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver-host10.1_10.1.0-3_amd64", + "name": "amdrocm-solver-host10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-math-common10.1_10.1.0-3_amd64", + "name": "amdrocm-math-common10.1", + "version": "10.1.0-3" + }, + { + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "amdrocm-blas-host10.1_10.1.0-3_amd64", + "name": "amdrocm-blas-host10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-profiler-base10.1_10.1.0-3_amd64", + "name": "amdrocm-profiler-base10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sysdeps10.1_10.1.0-3_amd64", + "name": "amdrocm-sysdeps10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-runtime10.1_10.1.0-3_amd64", + "name": "amdrocm-runtime10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-llvm10.1_10.1.0-3_amd64", + "name": "amdrocm-llvm10.1", + "version": "10.1.0-3" + }, + { + "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libgcc-11-dev", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libquadmath0_12.3.0-1ubuntu1_22.04_amd64", + "name": "libquadmath0", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libubsan1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libubsan1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libtsan0_11.4.0-1ubuntu1_22.04_amd64", + "name": "libtsan0", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "gcc-11-base_11.4.0-1ubuntu1_22.04_amd64", + "name": "gcc-11-base", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "liblsan0_12.3.0-1ubuntu1_22.04_amd64", + "name": "liblsan0", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libasan6_11.4.0-1ubuntu1_22.04_amd64", + "name": "libasan6", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libatomic1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libatomic1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libitm1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libitm1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libgomp1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgomp1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libstdc-p--p--11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libstdc++-11-dev", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libc6-dev_2.35-0ubuntu3.10_amd64", + "name": "libc6-dev", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libnsl-dev_1.3.0-2build2_amd64", + "name": "libnsl-dev", + "version": "1.3.0-2build2" + }, + { + "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-dev", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "version": "3.0.2-0ubuntu1.19" + }, + { + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "version": "1.5.79ubuntu1" + }, + { + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "version": "5.34.0-3ubuntu1.4" + }, + { + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "version": "1.21.1ubuntu2.3" + }, + { + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "version": "3.3-1build2" + }, + { + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "version": "10.39-3ubuntu0.1" + }, + { + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "version": "2.3.1-1" + }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + }, + { + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "version": "1.6.1-2ubuntu3" + }, + { + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "version": "1.46.5-2ubuntu1.2" + }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, + { + "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", + "name": "rpcsvc-proto", + "version": "1.4.2-0ubuntu6" + }, + { + "key": "libcrypt-dev_1-4.4.27-1_amd64", + "name": "libcrypt-dev", + "version": "1:4.4.27-1" + }, + { + "key": "linux-libc-dev_5.15.0-143.153_amd64", + "name": "linux-libc-dev", + "version": "5.15.0-143.153" + }, + { + "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", + "name": "libc-dev-bin", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "amdrocm-base10.1_10.1.0-3_amd64", + "name": "amdrocm-base10.1", + "version": "10.1.0-3" + }, + { + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" + }, + { + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" + }, + { + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" + }, + { + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" + }, + { + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "amdrocm-sparse-host10.1_10.1.0-3_amd64", + "name": "amdrocm-sparse-host10.1", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx942_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx942", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx942_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx942", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx942_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx942", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx90a_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx90a", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx90a_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx90a", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx90a_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx90a", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx908_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx908", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx908_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx908", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx908_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx908", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1250_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1250", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1250_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1250", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1250_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1250", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1201_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1201", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1201_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1201", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1201_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1201", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1200_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1200", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1200_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1200", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1200_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1200", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1153_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1153", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1153_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1153", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1153_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1153", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1152_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1152", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1152_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1152", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1152_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1152", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1151_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1151", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1151_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1151", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1151_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1151", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1150_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1150", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1150_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1150", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1150_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1150", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1103_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1103", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1103_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1103", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1103_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1103", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1102_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1102", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1102_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1102", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1102_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1102", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1101_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1101", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1101_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1101", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1101_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1101", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1100_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1100", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1100_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1100", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1100_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1100", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1036_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1036", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1036_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1036", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1036_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1036", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1035_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1035", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1035_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1035", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1035_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1035", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1034_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1034", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1034_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1034", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1034_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1034", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1033_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1033", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1033_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1033", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1033_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1033", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1032_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1032", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1032_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1032", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1032_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1032", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1031_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1031", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1031_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1031", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1031_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1031", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1030_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1030", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1030_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1030", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1030_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1030", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1012_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1012", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1012_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1012", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1012_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1012", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1011_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1011", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1011_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1011", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1011_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1011", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-sparse10.1-gfx1010_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1-gfx1010", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-blas10.1-gfx1010_10.1.0-3_amd64", + "name": "amdrocm-blas10.1-gfx1010", + "version": "10.1.0-3" + }, + { + "key": "amdrocm-solver10.1-gfx1010_10.1.0-3_amd64", + "name": "amdrocm-solver10.1-gfx1010", + "version": "10.1.0-3" + } + ], + "key": "amdrocm-sparse10.1_10.1.0-3_amd64", + "name": "amdrocm-sparse10.1", + "sha256": "71a68bcbb89fdac799535916b602a7f456269a24e2e40d0151abd003c7ad1b58", + "urls": [ + "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1_10.1.0-3_amd64.deb" + ], + "version": "10.1.0-3" + } + ], + "version": 1 +} \ No newline at end of file diff --git a/platforms/rocm/packages.yaml b/platforms/rocm/packages.yaml new file mode 100644 index 00000000..c9116561 --- /dev/null +++ b/platforms/rocm/packages.yaml @@ -0,0 +1,36 @@ +# +# bazel run @apt_rocm//:lock +# +version: 1 + +sources: + - channel: llvm-toolchain-jammy-22 main + url: https://apt.llvm.org/jammy + - channel: stable main + url: https://stable.repo.amd.com/rocm/core/packages/ubuntu2204 + - channel: jammy main + url: https://snapshot.ubuntu.com/ubuntu/20250711T030400Z + - channel: jammy-security main + url: https://snapshot.ubuntu.com/ubuntu/20250711T030400Z + - channel: jammy-updates main + url: https://snapshot.ubuntu.com/ubuntu/20250711T030400Z + +archs: + - "amd64" + +packages: + - "llvm-libunwind1" # LLVM 22 runtime from llvm-toolchain-jammy-22. + - "libatomic1" + - "libdrm-common" + - "amdrocm-amdsmi10.1" + - "amdrocm-base10.1" + - "amdrocm-blas10.1" + - "amdrocm-blas-dev10.1" + - "amdrocm-dnn10.1" + - "amdrocm-fft10.1" + - "amdrocm-llvm10.1" + - "amdrocm-profiler-base10.1" + - "amdrocm-rccl10.1" + - "amdrocm-runtime-dev10.1" + - "amdrocm-solver10.1" + - "amdrocm-sparse10.1" diff --git a/platforms/rocm/rocm.bzl b/platforms/rocm/rocm.bzl new file mode 100644 index 00000000..41b01b60 --- /dev/null +++ b/platforms/rocm/rocm.bzl @@ -0,0 +1,298 @@ +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file") +load("@llvm//:http_bsdtar_archive.bzl", http_archive = "http_bsdtar_archive") +load("//bazel:http_deb_archive.bzl", "http_deb_archive") +load("//platforms:packages.bzl", "packages") + +_BUILD_FILE_DEFAULT_VISIBILITY = """\ +package(default_visibility = ["//visibility:public"]) + +load("@rules_cc//cc:cc_library.bzl", "cc_library") +""" + +_ROCM_VERSION = "10.1" +_ROCM_STRIP_PREFIX = "./opt/rocm/core-" + _ROCM_VERSION +_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202611101651.363.1.d46c7471c419/zml-rocm-linux-amd64.tar.zst" +_PLUGIN_ROCM_SHA256 = "cdc4434f2023842f0e1a34c08aaed87f082df19f92c4ae21c9d4452a9739b604" + +def _rocm_package_name(name): + return name + _ROCM_VERSION + +def _rocm_repo_name(package_name): + return package_name.replace(_ROCM_VERSION, "") + +_UBUNTU_PACKAGES = { + "llvm-libunwind1": packages.filegroup(name = "libunwind", srcs = ["usr/lib/x86_64-linux-gnu/libunwind.so.1"]), + "libatomic1": packages.filegroup(name = "libatomic1", srcs = ["usr/lib/x86_64-linux-gnu/libatomic.so.1"]), + "libdrm-common": packages.filegroup(name = "amdgpu_ids", srcs = ["usr/share/libdrm/amdgpu.ids"]), +} + +def _rocm_dlopen_patchelf(name, src): + return "\n".join([ + packages.load_("@zml//bazel:patchelf.bzl", "patchelf"), + packages.patchelf( + name = name, + src = src, + add_needed = ["libzmlxrocm.so.0"], + set_rpath = "$ORIGIN:$ORIGIN/rocm_sysdeps/lib", + rename_dynamic_symbols = { + "dlopen": "zmlxrocm_dlopen", + "pthread_attr_setstacksize": "zmlxrocm_pthread_attr_setstacksize", + }, + ), + ]) + +def _glob_filegroup(name, srcs): + return """filegroup( + name = "{name}", + srcs = glob({srcs}), +)""".format(name = name, srcs = repr(srcs)) + +def _family_runfile_package_names(loaded_packages, family): + prefix = "amdrocm-{}{}-gfx".format(family, _ROCM_VERSION) + return sorted([ + pkg_name + for pkg_name in loaded_packages.keys() + if pkg_name.startswith(prefix) + ]) + +def _family_runfile_labels(loaded_packages, family): + return [ + "@{}//:runfiles".format(_rocm_repo_name(pkg_name)) + for pkg_name in _family_runfile_package_names(loaded_packages, family) + ] + +def _rocm_runfile_build_files(loaded_packages): + rocm_packages = {} + + for pkg_name in _family_runfile_package_names(loaded_packages, "blas"): + rocm_packages[pkg_name] = _glob_filegroup("runfiles", [ + ".kpack/**", + "lib/hipblaslt/library/**", + "lib/rocblas/library/**", + ]) + + for pkg_name in _family_runfile_package_names(loaded_packages, "dnn"): + rocm_packages[pkg_name] = _glob_filegroup("runfiles", [ + ".kpack/**", + "lib/libMIOpenCKGroupedConv_*.so", + "share/miopen/db/**", + ]) + + for pkg_name in _family_runfile_package_names(loaded_packages, "fft"): + rocm_packages[pkg_name] = _glob_filegroup("runfiles", [".kpack/**"]) + + for pkg_name in _family_runfile_package_names(loaded_packages, "rccl"): + rocm_packages[pkg_name] = _glob_filegroup("runfiles", [".kpack/**"]) + + return rocm_packages + +def _rocm_base_build_files(loaded_packages): + return { + _rocm_package_name("amdrocm-amdsmi"): "\n\n".join([ + packages.cc_library(name = "amdsmi", hdrs = ["include/amd_smi/amdsmi.h"], includes = ["include/amd_smi"]), + packages.filegroup(name = "libamd_smi", srcs = ["lib/libamd_smi.so.27"]), + ]), + _rocm_package_name("amdrocm-base"): "\n\n".join([ + _rocm_dlopen_patchelf(name = "rocm-core", src = "lib/librocm-core.so.1"), + _rocm_dlopen_patchelf(name = "rocprofiler-register", src = "lib/librocprofiler-register.so.0"), + ]), + _rocm_package_name("amdrocm-blas-dev"): _glob_filegroup("headers", ["include/**"]), + _rocm_package_name("amdrocm-blas-host"): "\n\n".join([ + _rocm_dlopen_patchelf(name = "hipblas", src = "lib/libhipblas.so.3"), + packages.filegroup(name = "hipblaslt", srcs = [ + "lib/libhipblaslt.so.1", + "lib/liborigami.so.1", + ]), + packages.filegroup(name = "rocblas", srcs = ["lib/librocblas.so.5"]), + _rocm_dlopen_patchelf(name = "rocroller", src = "lib/librocroller.so.1"), + _glob_filegroup("hipblaslt_support", [ + "lib/hipdnn_plugins/engines/libhipblaslt_plugin.so", + "share/hipblaslt/**", + ]), + ]), + _rocm_package_name("amdrocm-blas"): "\n\n".join([ + packages.filegroup(name = "hipblaslt", srcs = [ + "@amdrocm-blas-host//:hipblaslt", + "@amdrocm-blas-host//:rocroller", + ]), + packages.filegroup(name = "hipblaslt_runfiles", srcs = _family_runfile_labels(loaded_packages, "blas") + [ + "@amdrocm-blas-host//:hipblaslt_support", + ]), + packages.filegroup(name = "rocblas", srcs = [ + "@amdrocm-blas-host//:hipblas", + "@amdrocm-blas-host//:rocblas", + ]), + packages.filegroup(name = "rocblas_runfiles", srcs = _family_runfile_labels(loaded_packages, "blas")), + ]), + _rocm_package_name("amdrocm-dnn-host"): "\n\n".join([ + _rocm_dlopen_patchelf(name = "MIOpen", src = "lib/libMIOpen.so.1"), + _rocm_dlopen_patchelf(name = "hipdnn_backend", src = "lib/libhipdnn_backend.so"), + _glob_filegroup("runfiles", [ + "lib/hipdnn_plugins/**", + "share/miopen/**", + ]), + ]), + _rocm_package_name("amdrocm-dnn"): packages.filegroup(name = "MIOpen", srcs = [ + "@amdrocm-dnn-host//:MIOpen", + "@amdrocm-dnn-host//:hipdnn_backend", + "@amdrocm-dnn-host//:runfiles", + ] + _family_runfile_labels(loaded_packages, "dnn")), + _rocm_package_name("amdrocm-fft-host"): "\n\n".join([ + _rocm_dlopen_patchelf(name = "hipfft", src = "lib/libhipfft.so.0"), + _rocm_dlopen_patchelf(name = "hipfftw", src = "lib/libhipfftw.so.0"), + packages.filegroup(name = "rocfft", srcs = ["lib/librocfft.so.0"]), + ]), + _rocm_package_name("amdrocm-fft"): "\n\n".join([ + packages.filegroup(name = "hipfft", srcs = [ + "@amdrocm-fft-host//:hipfft", + "@amdrocm-fft-host//:hipfftw", + ]), + packages.filegroup(name = "rocfft", srcs = [ + "@amdrocm-fft-host//:rocfft", + ] + _family_runfile_labels(loaded_packages, "fft")), + ]), + _rocm_package_name("amdrocm-llvm"): "\n\n".join([ + packages.filegroup(name = "llvm_libs", srcs = [ + "lib/llvm/lib/libLLVM.so.24.0git", + "lib/llvm/lib/libclang-cpp.so.24.0git", + ]), + _glob_filegroup("rocm_device_libs_runfiles", ["lib/llvm/amdgcn/**"]), + ]), + _rocm_package_name("amdrocm-math-common"): _glob_filegroup("host_math_libs", ["lib/host-math/lib/*.so*"]), + _rocm_package_name("amdrocm-profiler-base"): "\n\n".join([ + _rocm_dlopen_patchelf(name = "hsa-amd-aqlprofile", src = "lib/libhsa-amd-aqlprofile64.so.1"), + _rocm_dlopen_patchelf(name = "rocprofiler-sdk_so", src = "lib/librocprofiler-sdk.so.1"), + packages.filegroup(name = "rocprofiler-sdk-attach", srcs = ["lib/librocprofiler-sdk-attach.so.1"]), + packages.filegroup(name = "rocprofiler-sdk-rocattach", srcs = ["lib/librocprofiler-sdk-rocattach.so.1"]), + _rocm_dlopen_patchelf(name = "roctracer", src = "lib/libroctracer64.so.4"), + _rocm_dlopen_patchelf(name = "roctx", src = "lib/libroctx64.so.4"), + packages.cc_library_hdrs_glob( + name = "roctx_headers", + hdrs_glob = ["include/rocprofiler-sdk-roctx/**/*.h"], + includes = ["include"], + ), + packages.filegroup(name = "rocprofiler-sdk", srcs = [ + ":rocprofiler-sdk_so", + ":rocprofiler-sdk-attach", + ":rocprofiler-sdk-rocattach", + ]), + _glob_filegroup("rocprofv3_tool", [ + "bin/rocprof*", + "bin/rocpd*", + "libexec/rocprofiler-sdk/**", + "share/rocprofiler-sdk/**", + ]), + ]), + _rocm_package_name("amdrocm-rccl-host"): packages.filegroup(name = "rccl", srcs = ["lib/librccl.so.1"]), + _rocm_package_name("amdrocm-rccl"): packages.filegroup(name = "rccl", srcs = [ + "@amdrocm-rccl-host//:rccl", + ] + _family_runfile_labels(loaded_packages, "rccl")), + _rocm_package_name("amdrocm-runtime"): "\n\n".join([ + _rocm_dlopen_patchelf(name = "amd_comgr", src = "lib/libamd_comgr.so.3"), + _rocm_dlopen_patchelf(name = "amdhip_so", src = "lib/libamdhip64.so.7"), + _rocm_dlopen_patchelf(name = "hiprtc_so", src = "lib/libhiprtc.so.7"), + _rocm_dlopen_patchelf(name = "hsa-runtime", src = "lib/libhsa-runtime64.so.1"), + packages.filegroup(name = "amdhip", srcs = [ + ":amdhip_so", + "lib/librocm_kpack.so.0", + ]), + packages.filegroup(name = "hiprtc", srcs = [ + ":hiprtc_so", + "lib/libhiprtc-builtins.so.7", + ]), + ]), + _rocm_package_name("amdrocm-runtime-dev"): _glob_filegroup("headers", ["include/**"]), + _rocm_package_name("amdrocm-solver-host"): "\n\n".join([ + _rocm_dlopen_patchelf(name = "hipsolver", src = "lib/libhipsolver.so.1"), + packages.filegroup(name = "rocsolver", srcs = ["lib/librocsolver.so.0"]), + ]), + _rocm_package_name("amdrocm-solver"): "\n\n".join([ + packages.filegroup(name = "hipsolver", srcs = ["@amdrocm-solver-host//:hipsolver"]), + packages.filegroup(name = "rocsolver", srcs = [ + "@amdrocm-solver-host//:rocsolver", + "@amdrocm-math-common//:host_math_libs", + ]), + ]), + _rocm_package_name("amdrocm-sparse-host"): "\n\n".join([ + _rocm_dlopen_patchelf(name = "hipsparse", src = "lib/libhipsparse.so.4"), + packages.filegroup(name = "rocsparse", srcs = ["lib/librocsparse.so.1"]), + ]), + _rocm_package_name("amdrocm-sparse"): packages.filegroup(name = "rocsparse", srcs = [ + "@amdrocm-sparse-host//:hipsparse", + "@amdrocm-sparse-host//:rocsparse", + ]), + _rocm_package_name("amdrocm-sysdeps"): _glob_filegroup("system_libs", [ + "lib/rocm_sysdeps/lib/*.so*", + "lib/rocm_sysdeps/share/**", + ]), + } + +def _rocm_build_files(loaded_packages): + rocm_packages = _rocm_base_build_files(loaded_packages) + rocm_packages.update(_rocm_runfile_build_files(loaded_packages)) + return rocm_packages + +def _rocm_impl(mctx): + loaded_packages = packages.read(mctx, [ + "@zml//platforms/rocm:packages.lock.json", + ]) + + for pkg_name, build_file_content in _UBUNTU_PACKAGES.items(): + pkg = loaded_packages[pkg_name]["amd64"] + http_deb_archive( + name = pkg_name, + urls = pkg["urls"], + sha256 = pkg["sha256"], + build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + build_file_content, + ) + + for pkg_name, build_file_content in _rocm_build_files(loaded_packages).items(): + pkg = loaded_packages[pkg_name]["amd64"] + http_deb_archive( + name = _rocm_repo_name(pkg_name), + urls = pkg["urls"], + sha256 = pkg["sha256"], + strip_prefix = _ROCM_STRIP_PREFIX, + build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + build_file_content, + ) + + http_file( + name = "libdrm_mesa_amdgpu_ids", + url = "https://gitlab.freedesktop.org/mesa/libdrm/-/raw/979f607906ad64f629967ac1f3ba3590e756442c/data/amdgpu.ids?inline=false", + sha256 = "ffd2a8f1bfa755f4d90f537b4969fc4676f116e5af051ce2f18ef93a96d8beb6", + downloaded_file_path = "amdgpu.ids", + ) + + http_archive( + name = "libzml_rocm", + build_file = "libzml_rocm.BUILD.bazel", + url = _PLUGIN_ROCM_URL, + sha256 = _PLUGIN_ROCM_SHA256, + ) + + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = [ + "amdrocm-amdsmi", + "amdrocm-base", + "amdrocm-blas", + "amdrocm-dnn", + "amdrocm-fft", + "amdrocm-llvm", + "amdrocm-math-common", + "amdrocm-profiler-base", + "amdrocm-rccl", + "amdrocm-runtime", + "amdrocm-solver", + "amdrocm-sparse", + "amdrocm-sysdeps", + "libatomic1", + "libdrm-common", + "libzml_rocm", + ], + root_module_direct_dev_deps = [], + ) + +rocm_packages = module_extension( + implementation = _rocm_impl, +) diff --git a/platforms/rocm/rocm.h b/platforms/rocm/rocm.h new file mode 100644 index 00000000..933e4f31 --- /dev/null +++ b/platforms/rocm/rocm.h @@ -0,0 +1 @@ + #include diff --git a/platforms/rocm/rocm.zig b/platforms/rocm/rocm.zig new file mode 100644 index 00000000..ee31bd83 --- /dev/null +++ b/platforms/rocm/rocm.zig @@ -0,0 +1,79 @@ +const std = @import("std"); +const builtin = @import("builtin"); + +const bazel = @import("bazel"); +const bazel_builtin = @import("bazel_builtin"); +const c = @import("c"); +const pjrt = @import("pjrt"); +const platforms_options = @import("platforms/options"); +const runfiles = @import("runfiles"); +const stdx = @import("stdx"); + +const log = std.log.scoped(.@"zml/platforms/rocm"); + +pub fn isEnabled() bool { + return platforms_options.rocm_enabled; +} + +fn hasRocmDevices(io: std.Io) bool { + inline for (&.{ "/dev/kfd", "/dev/dri" }) |path| { + std.Io.Dir.accessAbsolute(io, path, .{ .read = true }) catch return false; + } + return true; +} + +fn setupRocmEnv(rocm_data_dir: []const u8) !void { + var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + _ = c.setenv("ROCM_PATH", try stdx.Io.Dir.path.bufJoinZ(&buf, &.{rocm_data_dir}), 1); // must be zero terminated + + // Use one HIP graph stream to reduce cross-stream synchronization overhead. + _ = c.setenv("DEBUG_HIP_FORCE_GRAPH_QUEUES", "1", 0); + // Share one hardware queue per device so auxiliary barriers stay on the kernel queue. + _ = c.setenv("GPU_MAX_HW_QUEUES", "1", 0); + // Disable rocprofiler queue interposition to avoid its dispatch overhead outside profiling. + _ = c.setenv("ROCPROFILER_QUEUE_INTERPOSITION", "0", 0); + + if (c.getenv("ROCPROFILER_LOG_LEVEL") == null) { + // rocprofiler defaults to the `warning` log level, which can produce unnecessary log output. + // Set the log level to `fatal` to suppress all but fatal error messages. + _ = c.setenv("ROCPROFILER_LOG_LEVEL", "error", 0); + } +} + +pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { + _ = allocator; + if (comptime !isEnabled()) { + return error.Unavailable; + } + if (comptime builtin.os.tag != .linux) { + return error.Unavailable; + } + if (!hasRocmDevices(io)) { + return error.Unavailable; + } + + const r = try bazel.runfiles(bazel_builtin.current_repository); + + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const sandbox_path = try r.rlocation("libzml_rocm/sandbox", &path_buf) orelse { + log.err("Failed to find sandbox path for ROCm runtime", .{}); + return error.FileNotFound; + }; + + try setupRocmEnv(sandbox_path); + + // We must load the PJRT plugin from the main thread. + // + // This is because libamdhip64.so use thread local storage as part of the static destructors... + // + // This destructor accesses a thread-local variable. If the destructor is + // executed in a different thread than the one that originally called dlopen() + // on the library, the thread-local storage (TLS) offset may be resolved + // relative to the TLS base of the main thread, rather than the thread actually + // executing the destructor. Accessing this variable results in a segmentation fault... + return blk: { + var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const lib_path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libzml_rocm.so" }); + break :blk .loadFrom(lib_path); + }; +} diff --git a/platforms/rocm/zmlxrocm.zig b/platforms/rocm/zmlxrocm.zig new file mode 100644 index 00000000..069c08bb --- /dev/null +++ b/platforms/rocm/zmlxrocm.zig @@ -0,0 +1,67 @@ +const std = @import("std"); + +const stdx = @import("stdx"); + +pub export fn zmlxrocm_dlopen(filename: [*c]const u8, flags: c_int) ?*anyopaque { + const replacements: std.StaticStringMap([:0]const u8) = .initComptime(.{ + .{ "libamdhip64.so", "libamdhip64.so.7" }, + .{ "libhsa-amd-aqlprofile64.so", "libhsa-amd-aqlprofile64.so.1" }, + .{ "librocprofiler-sdk-attach.so", "librocprofiler-sdk-attach.so.1" }, + .{ "librocprofiler-sdk.so", "librocprofiler-sdk.so.1" }, + .{ "libroctx64.so", "libroctx64.so.4" }, + }); + + var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const new_filename: [*c]const u8 = if (filename) |f| blk: { + const replacement = replacements.get(std.Io.Dir.path.basename(std.mem.span(f))) orelse break :blk f; + break :blk stdx.Io.Dir.path.bufJoinZ(&buf, &.{ + stdx.process.selfSharedObjectDirPath(), + replacement, + }) catch unreachable; + } else null; + + return std.c.dlopen(new_filename, @bitCast(flags)); +} + +pub export fn zmlxrocm_fopen64(pathname: [*c]const u8, mode: [*c]const u8) ?*std.c.FILE { + const replacements: std.StaticStringMap([]const u8) = .initComptime(.{ + .{ "/opt/amdgpu/share/libdrm/amdgpu.ids", "../share/libdrm/amdgpu.ids" }, + }); + + var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const new_pathname: [*c]const u8 = blk: { + const replacement = replacements.get(std.mem.span(pathname)) orelse break :blk pathname; + break :blk stdx.Io.Dir.path.bufJoinZ(&buf, &.{ + stdx.process.selfSharedObjectDirPath(), + replacement, + }) catch unreachable; + }; + + return std.c.fopen64(new_pathname, mode); +} + +extern fn pthread_attr_setstacksize(attr: ?*anyopaque, stacksize: usize) c_int; + +// ROCclr asks for small helper-thread stacks: CQ_THREAD_STACK_SIZE is 256 KiB, +// then ROCclr adds the pthread guard size before calling pthread_attr_setstacksize. +// https://github.com/ROCm/clr/blob/1736f3eba9134a5ee4b56475acf15665f065ab05/rocclr/utils/flags.hpp#L25 +// https://github.com/ROCm/clr/blob/1736f3eba9134a5ee4b56475acf15665f065ab05/rocclr/os/os_posix.cpp#L378-L385 +// glibc 2.28 accepts stack sizes >= PTHREAD_STACK_MIN at attribute-set time, +// but pthread_create can still reject stacks that do not leave room for the +// guard, static TLS, and MINIMAL_REST_STACK: +// https://github.com/bminor/glibc/blob/glibc-2.28/nptl/pthread_attr_setstacksize.c#L35-L38 +// https://github.com/bminor/glibc/blob/glibc-2.28/nptl/allocatestack.c#L532-L544 +// The 1 MiB floor below is a local compatibility margin for that glibc path; +// it is intentionally larger than ROCclr's request and not a ROCm ABI value. +pub export fn zmlxrocm_pthread_attr_setstacksize(attr: ?*anyopaque, stacksize: usize) c_int { + const min_stack_size = 1024 * 1024; + const adjusted_stacksize = blk: { + if (stacksize < min_stack_size) { + break :blk min_stack_size; + } else { + break :blk stacksize; + } + }; + + return pthread_attr_setstacksize(attr, adjusted_stacksize); +} diff --git a/platforms/tpu/BUILD.bazel b/platforms/tpu/BUILD.bazel new file mode 100644 index 00000000..1aab4031 --- /dev/null +++ b/platforms/tpu/BUILD.bazel @@ -0,0 +1,39 @@ +load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library") + +cc_library( + name = "libzml_tpu", + tags = ["manual"], + deps = ["@libzml_tpu"], +) + +zig_library( + name = "tpu", + import_name = "platforms/tpu", + main = "tpu.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//bazel", + "//pjrt", + "//platforms:options", + "//stdx", + ] + select({ + "//platforms:tpu.enabled": [ + ":libzml_tpu", + ], + "//conditions:default": [], + }), +) + +zig_library( + name = "ragged_paged", + import_name = "platforms/tpu/ragged_paged", + main = "ragged_paged.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//dsl/mosaic_tpu:builder", + "//mlir", + ], +) diff --git a/platforms/tpu/libzml_tpu.BUILD.bazel b/platforms/tpu/libzml_tpu.BUILD.bazel new file mode 100644 index 00000000..1186510c --- /dev/null +++ b/platforms/tpu/libzml_tpu.BUILD.bazel @@ -0,0 +1,35 @@ +load("@bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") +load("@bazel_skylib//rules:copy_file.bzl", "copy_file") + +copy_file( + name = "libzml_tpu.so", + src = "libtpu/libtpu.so", + out = "lib/libzml_tpu.so", +) + +copy_to_directory( + name = "sandbox", + srcs = [ + "lib/libzml_tpu.so", + ], + add_directory_to_runfiles = False, + allow_overwrites = True, + exclude_srcs_patterns = [ + "**/include/**", + "**/_solib_*/**", + ], + hardlink = "auto", + include_external_repositories = ["**"], + include_srcs_patterns = [ + "**/lib**", + ], + replace_prefixes = { + "lib/x86_64-linux-gnu": "lib", + }, +) + +cc_library( + name = "libzml_tpu", + data = [":sandbox"], + visibility = ["@zml//platforms/tpu:__subpackages__"], +) diff --git a/platforms/tpu/ragged_paged.zig b/platforms/tpu/ragged_paged.zig new file mode 100644 index 00000000..6a0fdfae --- /dev/null +++ b/platforms/tpu/ragged_paged.zig @@ -0,0 +1,715 @@ +const std = @import("std"); + +const mlir = @import("mlir"); +const mtt = @import("dsl/mosaic_tpu/builder"); + +pub const Cfg = struct { + // Shapes that vary at compile-of-kernel time. + num_q_tokens: i64, + num_q_heads: i64, + num_kv_heads: i64, + head_dim: i64, + total_num_pages: i64, + page_size: i64, + max_num_seqs: i64, + pages_per_seq: i64, + + // Block-level tuning knobs. + num_kv_pages_per_block: i64, + num_queries_per_block: i64, + + // Element types. + q_dtype: mtt.DType = .bf16, + kv_dtype: mtt.DType = .bf16, + + // Numeric knobs. + sm_scale: f32, + mask_value: f32 = -2.38197633e+38, + /// Pre-mask softcap: `qk = soft_cap * tanh(qk / soft_cap)`. + soft_cap: ?f32 = null, + /// K dequant scale applied right after `stridedLoadKv` (mirrors Pallas's + /// `k.astype(f32) * k_scale → k.astype(q_dtype)`). + k_scale: ?f32 = null, + v_scale: ?f32 = null, + + /// Whether to mask keys beyond the current query row. + is_causal: bool = true, + + /// Sliding-window mask added on top of the validity/causal mask: any + /// `(row, col)` with `row - sliding_window >= col` is masked out. Mirrors + /// `kernel.py:391-393` in causal mode. + sliding_window: ?i64 = null, + vmem_limit_bytes: ?i64 = null, + + // Derived shape helpers — keeps the body readable. + pub fn numCombinedKvHeads(self: Cfg) i64 { + return 2 * self.num_kv_heads; + } + pub fn numQHeadsPerKvHead(self: Cfg) i64 { + return @divExact(self.num_q_heads, self.num_kv_heads); + } + pub fn numQHeadsPerBlk(self: Cfg) i64 { + return self.num_q_heads; + } + pub fn numCombinedKvHeadsPerBlk(self: Cfg) i64 { + return self.numCombinedKvHeads(); + } + pub fn numKvHeadsPerBlk(self: Cfg) i64 { + return @divExact(self.numCombinedKvHeadsPerBlk(), 2); + } + pub fn numQPerBlk(self: Cfg) i64 { + return self.num_queries_per_block; + } + pub fn numKvPerBlk(self: Cfg) i64 { + return self.num_kv_pages_per_block * self.page_size; + } + pub fn numHeadsBlks(self: Cfg) i64 { + return @divExact(self.num_q_heads, self.numQHeadsPerBlk()); + } + pub fn numQBlks(self: Cfg) i64 { + return @divExact(self.num_q_tokens, self.numQPerBlk()); + } + pub fn flashQRows(self: Cfg) i64 { + return self.numQPerBlk() * self.numQHeadsPerKvHead(); + } + pub fn flashKRows(self: Cfg) i64 { + return self.numKvPerBlk(); + } + + /// Pallas's `get_dtype_packing(dtype) = 32 // bitwidth(dtype)`. Mirrors + /// `kernel.py:get_dtype_packing` exactly. + pub fn kvPackingFactor(self: Cfg) i64 { + return switch (self.kv_dtype) { + .i32, .f32 => 1, + .i16, .f16, .bf16 => 2, + .i8, .f8e4m3fn, .f8e5m2 => 4, + else => std.debug.panic("ragged_paged: unsupported KV dtype {s}", .{@tagName(self.kv_dtype)}), + }; + } + /// Same packing rule applied to `q_dtype` — used by `fold_on_2nd_minor` + /// to decide whether to cast q to f32 before the trailing-dim reshape. + pub fn qPackingFactor(self: Cfg) i64 { + return switch (self.q_dtype) { + .f32 => 1, + .f16, .bf16 => 2, + else => std.debug.panic("ragged_paged: unsupported Q dtype {s}", .{@tagName(self.q_dtype)}), + }; + } + /// `kv_load_step = max(1, kv_packing // 2)` — number of (k, v) pairs the + /// caller unpacks from a single `b` strided-load (Pallas's outer-loop step). + pub fn kvLoadStep(self: Cfg) i64 { + const p = self.kvPackingFactor(); + return @max(@as(i64, 1), @divFloor(p, 2)); + } + pub fn kvFlatRows(self: Cfg) i64 { + return self.num_kv_pages_per_block * self.page_size * self.numCombinedKvHeadsPerBlk(); + } + pub fn kvFlatI32Rows(self: Cfg) i64 { + return @divExact(self.kvFlatRows(), self.kvPackingFactor()); + } +}; + +fn maskedStoreLM( + k: *mtt.Builder, + ref: mtt.Value, + val: mtt.Value, + kv_head_idx_idx: mtt.Value, + start: mtt.Value, + end: mtt.Value, + group: i64, +) void { + const shape = val.shape().constSlice(); + const iota = k.iota(shape, .i32, &.{0}); + const iota_div = k.divFloor(iota, @as(i32, @intCast(group))); + const ge = k.cmpi(.sge, iota_div, k.broadcastTo(start, shape)); + const lt = k.cmpi(.slt, iota_div, k.broadcastTo(end, shape)); + const mask = k.andi(ge, lt); + const slot_shape = [_]i64{ 1, shape[0], shape[1] }; + const val_3d = k.shapeCast(val, &slot_shape); + const mask_3d = k.shapeCast(mask, &slot_shape); + k.refStoreOpts(ref, val_3d, &.{ kv_head_idx_idx, k.constIndex(0), k.constIndex(0) }, .{ .mask = mask_3d, .strides = &.{} }); +} + +/// `pltpu.store(acc_ref, val, mask=…)` — group=1 so no floor-div. +fn maskedStoreAcc( + k: *mtt.Builder, + ref: mtt.Value, + val: mtt.Value, + q_head_offset_idx: mtt.Value, + start: mtt.Value, + end: mtt.Value, +) void { + const shape = val.shape().constSlice(); + const iota = k.iota(shape, .i32, &.{0}); + const ge = k.cmpi(.sge, iota, k.broadcastTo(start, shape)); + const lt = k.cmpi(.slt, iota, k.broadcastTo(end, shape)); + const mask = k.andi(ge, lt); + k.refStoreOpts(ref, val, &.{ k.constIndex(0), q_head_offset_idx, k.constIndex(0) }, .{ .mask = mask, .strides = &.{} }); +} + +/// `MultiPageAsyncCopyDescriptor.__init__` — `tpu.memref_slice → squeeze +/// → enqueue_dma` chain that Pallas emits for one async page-copy. +fn createKvAsyncCopyDescriptors( + k: *mtt.Builder, + cfg: Cfg, + a: anytype, + heads_blk_idx: mtt.Value, // i32 + seq_idx: mtt.Value, // i32 + kv_blk_idx: mtt.Value, // i32 — the block-relative page index + buf_idx: mtt.Value, // i32 +) void { + const start_kv_page_idx = k.muli(kv_blk_idx, k.constant(.i32, cfg.num_kv_pages_per_block)); + const kv_at_seq = k.scalarLoad(a.kv_lens, &.{k.toIndex(seq_idx)}); + const end_kv_page_idx = k.minsi( + k.divsi(k.addi(kv_at_seq, k.constant(.i32, cfg.page_size - 1)), k.constant(.i32, cfg.page_size)), + k.constant(.i32, cfg.pages_per_seq), + ); + const heads_start = k.muli(heads_blk_idx, k.constant(.i32, cfg.numCombinedKvHeadsPerBlk())); + + const within = k.cmpi(.slt, start_kv_page_idx, end_kv_page_idx); + const page_idx_safe = k.select(within, start_kv_page_idx, k.constant(.i32, 0)); + const page_table = k.scalarLoad(a.page_indices, &.{ k.toIndex(seq_idx), k.toIndex(page_idx_safe) }); + + const src0 = k.memRefSlice(a.kv_pages, &.{ k.constant(.i32, 0), k.constant(.i32, 0), heads_start, k.constant(.i32, 0) }, &.{ cfg.total_num_pages, cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }, &.{}); + const src1 = k.memRefSlice(src0, &.{ page_table, k.constant(.i32, 0), k.constant(.i32, 0), k.constant(.i32, 0) }, &.{ 1, cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }, &.{}); + const src = k.memRefSqueeze(src1, &.{ cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }); + + const dst0 = k.memRefSlice(a.kv_bufs, &.{ buf_idx, k.constant(.i32, 0), k.constant(.i32, 0), k.constant(.i32, 0), k.constant(.i32, 0) }, &.{ 1, cfg.num_kv_pages_per_block, cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }, &.{}); + const dst1 = k.memRefSqueeze(dst0, &.{ cfg.num_kv_pages_per_block, cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }); + const dst = k.memRefSqueeze(dst1, &.{ cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }); + + const sem_slice = k.memRefSlice(a.sems, &.{buf_idx}, &.{1}, &.{}); + const sem = k.memRefSqueeze(sem_slice, &.{}); + + k.enqueueDma(src, dst, sem, .{}); +} + +fn waitKvDma( + k: *mtt.Builder, + cfg: Cfg, + a: anytype, + heads_blk_idx: mtt.Value, + seq_idx: mtt.Value, + kv_blk_idx: mtt.Value, + buf_idx: mtt.Value, +) void { + const start_kv_page_idx = k.muli(kv_blk_idx, k.constant(.i32, cfg.num_kv_pages_per_block)); + const kv_at_seq = k.scalarLoad(a.kv_lens, &.{k.toIndex(seq_idx)}); + const end_kv_page_idx = k.minsi( + k.divsi(k.addi(kv_at_seq, k.constant(.i32, cfg.page_size - 1)), k.constant(.i32, cfg.page_size)), + k.constant(.i32, cfg.pages_per_seq), + ); + const heads_start = k.muli(heads_blk_idx, k.constant(.i32, cfg.numCombinedKvHeadsPerBlk())); + const within = k.cmpi(.slt, start_kv_page_idx, end_kv_page_idx); + const page_idx_safe = k.select(within, start_kv_page_idx, k.constant(.i32, 0)); + const page_table = k.scalarLoad(a.page_indices, &.{ k.toIndex(seq_idx), k.toIndex(page_idx_safe) }); + + const src0 = k.memRefSlice(a.kv_pages, &.{ k.constant(.i32, 0), k.constant(.i32, 0), heads_start, k.constant(.i32, 0) }, &.{ cfg.total_num_pages, cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }, &.{}); + const src1 = k.memRefSlice(src0, &.{ page_table, k.constant(.i32, 0), k.constant(.i32, 0), k.constant(.i32, 0) }, &.{ 1, cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }, &.{}); + const src = k.memRefSqueeze(src1, &.{ cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }); + + const dst0 = k.memRefSlice(a.kv_bufs, &.{ buf_idx, k.constant(.i32, 0), k.constant(.i32, 0), k.constant(.i32, 0), k.constant(.i32, 0) }, &.{ 1, cfg.num_kv_pages_per_block, cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }, &.{}); + const dst1 = k.memRefSqueeze(dst0, &.{ cfg.num_kv_pages_per_block, cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }); + const dst = k.memRefSqueeze(dst1, &.{ cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }); + + const sem_slice = k.memRefSlice(a.sems, &.{buf_idx}, &.{1}, &.{}); + const sem = k.memRefSqueeze(sem_slice, &.{}); + + k.waitDma2(sem, src, dst, .{}); +} + +const KvPair = struct { k: mtt.Value, v: mtt.Value }; + +/// Mirrors Pallas's `b_ref[b_start::b_step, :]` lowering: +/// - `b_step == 1` → `vector.load` (the slice is contiguous). +/// - `last_dim ≤ 128` and `b_step > 1` → `tpu.strided_load`. +/// - `last_dim > 128` → per-row `vector.load` + `tpu.concatenate` +/// (`jnp.stack([ref[start + i*step, :] for i in range(num_rows)])`). +fn loadStridedOrStack( + k: *mtt.Builder, + ref: mtt.Value, + start_row: i64, + stride: i32, + out_rows: i64, + out_last_dim: i64, +) mtt.Value { + if (out_last_dim <= 128) { + if (stride == 1) { + return k.vectorLoadShape(ref, &.{ k.constIndex(start_row), k.constIndex(0) }, &.{ out_rows, out_last_dim }); + } + return k.stridedLoadShape(ref, &.{ k.constIndex(start_row), k.constIndex(0) }, &.{ stride, 1 }, &.{ out_rows, out_last_dim }); + } + const rows_count: usize = @intCast(out_rows); + const rows = k.arena.allocator().alloc(mtt.Value, rows_count) catch @panic("loadStridedOrStack OOM"); + for (0..rows_count) |i| { + const row_idx: i64 = start_row + @as(i64, @intCast(i)) * @as(i64, stride); + rows[i] = k.vectorLoadShape(ref, &.{ k.constIndex(row_idx), k.constIndex(0) }, &.{ 1, out_last_dim }); + } + return k.concatenate(rows, 0); +} + +/// Pallas's `broadcast_to_shape(arr, shape)` — emit `tpu.concatenate` of +/// `shape[1] / arr.shape[1]` copies when expanding axis 1 (e.g., +/// `` → `` for `head_dim > 128`). +fn broadcastToShape(k: *mtt.Builder, arr: mtt.Value, shape: []const i64) mtt.Value { + const arr_shape = arr.shape(); + std.debug.assert(arr_shape.len == 2 and shape.len == 2); + if (arr_shape.get(1) == shape[1]) return arr; + std.debug.assert(arr_shape.get(0) == shape[0]); + std.debug.assert(@mod(shape[1], arr_shape.get(1)) == 0); + const repeats: usize = @intCast(@divExact(shape[1], arr_shape.get(1))); + const sources = k.arena.allocator().alloc(mtt.Value, repeats) catch @panic("broadcastToShape OOM"); + for (sources) |*s| s.* = arr; + return k.concatenate(sources, 1); +} + +/// `strided_load_kv(ref, kv_head_chunk_idx * 2, num_combined_kv_heads_per_blk)` +/// — full Pallas parity. +/// +/// - `packing == 1` (i32/f32): two strided loads in original dtype. +/// - `packing == 2` bf16: special-case shli + and + tpu.bitcast(f32) + truncf. +/// - `packing == 2` f16/i16, `packing == 4` (i8/u8/f8): generic shrui + trunci +/// + tpu.bitcast unpack producing `kv_load_step` (k, v) pairs. +fn stridedLoadKv( + k: *mtt.Builder, + cfg: Cfg, + a: anytype, + buf_idx: mtt.Value, + kv_head_chunk_idx: i64, +) []const KvPair { + const packing = cfg.kvPackingFactor(); + const start: i64 = kv_head_chunk_idx * 2; + const step: i32 = @intCast(cfg.numCombinedKvHeadsPerBlk()); + + const dst0 = k.memRefSlice(a.kv_bufs, &.{ buf_idx, k.constant(.i32, 0), k.constant(.i32, 0), k.constant(.i32, 0), k.constant(.i32, 0) }, &.{ 1, cfg.num_kv_pages_per_block, cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }, &.{}); + const dst1 = k.memRefSqueeze(dst0, &.{ cfg.num_kv_pages_per_block, cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }); + const flat = k.memRefReshape(dst1, &.{ cfg.kvFlatRows(), cfg.head_dim }); + + if (packing == 1) { + const k_v = loadStridedOrStack(k, flat, start, step, cfg.flashKRows(), cfg.head_dim); + const v_v = loadStridedOrStack(k, flat, start + 1, step, cfg.flashKRows(), cfg.head_dim); + const result = k.arena.allocator().alloc(KvPair, 1) catch @panic("stridedLoadKv OOM"); + result[0] = .{ .k = k_v, .v = v_v }; + return result; + } + + std.debug.assert(packing == 2 or packing == 4 or packing == 8); + std.debug.assert(@mod(step, packing) == 0); + + const ref_i32 = k.memRefBitcastShape(flat, &.{ cfg.kvFlatI32Rows(), cfg.head_dim }, .i32); + const b_stride: i32 = @intCast(@divExact(cfg.numCombinedKvHeadsPerBlk(), packing)); + const b_start: i64 = @divExact(start, packing); + const b = loadStridedOrStack(k, ref_i32, b_start, b_stride, cfg.flashKRows(), cfg.head_dim); + + const splat_shape = &[_]i64{ cfg.flashKRows(), cfg.head_dim }; + + if (cfg.kv_dtype == .bf16) { + const bk = k.shli(b, k.splat(@as(i32, 16), splat_shape, .i32)); + const bv = k.andi(b, k.splat(@as(i32, @bitCast(@as(u32, 0xFFFF0000))), splat_shape, .i32)); + const k_v = k.arithTruncf(bk.bitCast(.f32), cfg.kv_dtype); + const v_v = k.arithTruncf(bv.bitCast(.f32), cfg.kv_dtype); + const result = k.arena.allocator().alloc(KvPair, 1) catch @panic("stridedLoadKv OOM"); + result[0] = .{ .k = k_v, .v = v_v }; + return result; + } + + const bitwidth: i64 = @divExact(@as(i64, 32), packing); + const trunc_dtype: mtt.DType = switch (bitwidth) { + 16 => .i16, + 8 => .i8, + else => std.debug.panic("ragged_paged: unsupported packing bitwidth {d}", .{bitwidth}), + }; + const kv_load_step: usize = @intCast(cfg.kvLoadStep()); + const result = k.arena.allocator().alloc(KvPair, kv_load_step) catch @panic("stridedLoadKv OOM"); + var i: usize = 0; + while (i < kv_load_step) : (i += 1) { + const shift_k: i32 = @intCast(@as(i64, @intCast(i)) * 2 * bitwidth); + const shift_v: i32 = @intCast((@as(i64, @intCast(i)) * 2 + 1) * bitwidth); + const bk = k.shrui(b, k.splat(shift_k, splat_shape, .i32)); + const bv = k.shrui(b, k.splat(shift_v, splat_shape, .i32)); + const bk_trunc = k.trunci(bk, trunc_dtype); + const bv_trunc = k.trunci(bv, trunc_dtype); + result[i] = .{ + .k = bk_trunc.bitCast(cfg.kv_dtype), + .v = bv_trunc.bitCast(cfg.kv_dtype), + }; + } + return result; +} + +/// `flash_attention(q, k, v, ...)` — one online-softmax step over `num_kv_per_blk` +/// keys. +fn flashAttention( + k: *mtt.Builder, + cfg: Cfg, + a: anytype, + q: mtt.Value, + kv_k: mtt.Value, + v: mtt.Value, + /// Element type of `kv_k` as currently held — equal to `cfg.kv_dtype` + /// unless `cfg.k_scale` was applied upstream (then `cfg.q_dtype`). + /// Used for the masked-zero round-trip Pallas's `.astype(k.dtype)` does. + k_input_dtype: mtt.DType, + /// Same role as `k_input_dtype`, for `v`. + v_input_dtype: mtt.DType, + kv_head_idx: i64, + q_head_idx: i64, + kv_blk_idx: mtt.Value, + kv_len: mtt.Value, + q_start: mtt.Value, + q_end: mtt.Value, + q_len: mtt.Value, + q_len_start: mtt.Value, +) void { + const flash_q: i64 = cfg.flashQRows(); + const flash_k: i64 = cfg.flashKRows(); + const head_dim: i64 = cfg.head_dim; + const num_q_per_blk: i64 = cfg.numQPerBlk(); + const num_q_heads_per_kv_head: i64 = cfg.numQHeadsPerKvHead(); + + const kv_len_start = k.muli(kv_blk_idx, k.constant(.i32, cfg.numKvPerBlk())); + + const kv_mask = k.cmpi( + .slt, + k.iota(&.{ flash_k, head_dim }, .i32, &.{0}), + k.broadcastTo(k.subi(kv_len, kv_len_start), &.{ flash_k, head_dim }), + ); + const k_zeroed = k.select(kv_mask, kv_k.to(.f32), k.zeros(&.{ flash_k, head_dim }, .f32)).to(k_input_dtype); + const v_zeroed = k.select(kv_mask, v.to(.f32), k.zeros(&.{ flash_k, head_dim }, .f32)).to(v_input_dtype); + + k.traceStart(10, "nd,md->nm"); + const qk_raw = k.matmulOpts( + q, + k_zeroed, + k.zeros(&.{ flash_q, flash_k }, .f32), + .{ .dimension_numbers = k.dotDimensionNumbers(&.{1}, &.{1}, &.{0}, &.{0}, &.{ 0, 0, 1, 0 }, &.{}, &.{}) }, + ); + k.traceStop(); + var qk = k.mulf(qk_raw, k.splat(cfg.sm_scale, &.{ flash_q, flash_k }, .f32)); + + const store_start = k.maxsi(k.subi(q_start, q_len_start), k.constant(.i32, 0)); + const store_end = k.minsi(k.subi(q_end, q_len_start), k.constant(.i32, num_q_per_blk)); + + const row_offset = k.subi(k.addi(k.subi(kv_len, q_len), q_len_start), q_start); + const row_iota = k.iota(&.{ flash_q, flash_k }, .i32, &.{0}); + const row_q = k.divFloor(row_iota, @as(i32, @intCast(num_q_heads_per_kv_head))); + const row_ids = k.addi(k.broadcastTo(row_offset, &.{ flash_q, flash_k }), row_q); + const col_ids = k.addi( + k.broadcastTo(kv_len_start, &.{ flash_q, flash_k }), + k.iota(&.{ flash_q, flash_k }, .i32, &.{1}), + ); + var score_mask = if (cfg.is_causal) + k.cmpi(.slt, row_ids, col_ids) + else + // `row_ids < row_ids` is an all-false mask with the same bool shape + // as the causal mask; sliding-window masking can still be OR-ed below. + k.cmpi(.slt, row_ids, row_ids); + if (cfg.sliding_window) |sw| { + const sw_splat = k.splat(@as(i32, @intCast(sw)), &.{ flash_q, flash_k }, .i32); + const window_mask = k.cmpi(.sge, k.subi(row_ids, sw_splat), col_ids); + score_mask = k.ori(score_mask, window_mask); + } + if (cfg.soft_cap) |sc| { + const sc_splat = k.splat(sc, &.{ flash_q, flash_k }, .f32); + qk = k.mulf(sc_splat, k.tanh(k.divf(qk, sc_splat))); + } + const qk_masked = k.addf( + qk, + k.select(score_mask, k.splat(cfg.mask_value, &.{ flash_q, flash_k }, .f32), k.zeros(&.{ flash_q, flash_k }, .f32)), + ); + + // jnp.max → maximumf (NaN-propagating). + const m_curr_v = k.multiReduction(.maximumf, qk_masked, k.splat(@as(f32, -std.math.inf(f32)), &.{flash_q}, .f32), &.{1}); + const m_curr_2d = k.shapeCast(m_curr_v, &.{ flash_q, 1 }); + const s_curr = k.exp(k.subf(qk_masked, k.broadcastTo(m_curr_2d, &.{ flash_q, flash_k }))); + + const qkv = k.matmulOpts( + s_curr, + v_zeroed, + k.zeros(&.{ flash_q, head_dim }, .f32), + .{ .dimension_numbers = k.dotDimensionNumbers(&.{1}, &.{0}, &.{0}, &.{1}, &.{ 0, 0, 1, 1 }, &.{}, &.{}) }, + ); + + // Pallas hardcodes the m/l-ref trailing dim to 128 (the TPU lane width) + // regardless of `head_dim`. Broadcast m/l values to `head_dim` only + // immediately before mixing with `qkv` / `o_curr`. + const lm_lane: i64 = 128; + + const m_curr_bc = k.broadcastTo(m_curr_2d, &.{ flash_q, lm_lane }); + const l_curr_v = k.multiReduction(.add, s_curr, k.zeros(&.{flash_q}, .f32), &.{1}); + const l_curr_bc = k.broadcastTo(k.shapeCast(l_curr_v, &.{ flash_q, 1 }), &.{ flash_q, lm_lane }); + + const head_l_ref_idx = k.constIndex(kv_head_idx); + const m_prev_load = k.shapeCast( + k.vectorLoadShape(a.m_ref, &.{ head_l_ref_idx, k.constIndex(0), k.constIndex(0) }, &.{ 1, flash_q, lm_lane }), + &.{ flash_q, lm_lane }, + ); + const m_prev = k.select(k.cmpi(.eq, kv_blk_idx, k.constant(.i32, 0)), k.splat(@as(f32, -std.math.inf(f32)), &.{ flash_q, lm_lane }, .f32), m_prev_load); + const l_prev_load = k.shapeCast( + k.vectorLoadShape(a.l_ref, &.{ head_l_ref_idx, k.constIndex(0), k.constIndex(0) }, &.{ 1, flash_q, lm_lane }), + &.{ flash_q, lm_lane }, + ); + const l_prev = k.select(k.cmpi(.eq, kv_blk_idx, k.constant(.i32, 0)), k.zeros(&.{ flash_q, lm_lane }, .f32), l_prev_load); + + const m_next = k.maximumf(m_prev, m_curr_bc); + maskedStoreLM(k, a.m_ref, m_next, head_l_ref_idx, store_start, store_end, num_q_heads_per_kv_head); + + const alpha = k.exp(k.subf(m_prev, m_next)); + const beta = k.exp(k.subf(m_curr_bc, m_next)); + const l_alpha = k.mulf(alpha, l_prev); + const l_next = k.addf(l_alpha, k.mulf(beta, l_curr_bc)); + const l_next_safe = k.select( + k.cmpf(.oeq, l_next, k.zeros(&.{ flash_q, lm_lane }, .f32)), + k.splat(@as(f32, 1.0), &.{ flash_q, lm_lane }, .f32), + l_next, + ); + maskedStoreLM(k, a.l_ref, l_next_safe, head_l_ref_idx, store_start, store_end, num_q_heads_per_kv_head); + + const q_head_idx_idx = k.constIndex(q_head_idx); + const o_curr_3d = k.vectorLoadShape(a.acc_ref, &.{ k.constIndex(0), q_head_idx_idx, k.constIndex(0) }, &.{ num_q_per_blk, num_q_heads_per_kv_head, head_dim }); + const o_curr_pre = k.select( + k.cmpi(.eq, kv_blk_idx, k.constant(.i32, 0)), + k.zeros(&.{ num_q_per_blk, num_q_heads_per_kv_head, head_dim }, .f32), + o_curr_3d, + ); + const o_curr = k.shapeCast(o_curr_pre, &.{ flash_q, head_dim }); + const l_alpha_bc = broadcastToShape(k, l_alpha, &.{ flash_q, head_dim }); + const beta_bc = broadcastToShape(k, beta, &.{ flash_q, head_dim }); + const l_next_safe_bc = broadcastToShape(k, l_next_safe, &.{ flash_q, head_dim }); + const out = k.divf(k.addf(k.mulf(l_alpha_bc, o_curr), k.mulf(beta_bc, qkv)), l_next_safe_bc); + const out_3d = k.shapeCast(out, &.{ num_q_per_blk, num_q_heads_per_kv_head, head_dim }); + maskedStoreAcc(k, a.acc_ref, out_3d, q_head_idx_idx, store_start, store_end); +} + +/// Emit the kernel body into a pre-opened `mtt.Builder`. The factory in +/// `zml/kernel/mosaic_tpu/mosaic_tpu.zig` opens the builder, calls this `run`, and finalizes the +/// IR with `b.finishOpts(&.{}, .{ .canonicalize = true })`. +pub fn run(b: *mtt.Builder, cfg: Cfg) mtt.FinishError!void { + const q_transform: []const mtt.ArgSpec.TransformReturn = &.{ + .{ .program_id = 1 }, + .{ .program_id = 0 }, + .zero, + }; + + const a = try b.declareArgsOpts(.{ + .heads_blk_idx = .{ .scalar = .i32 }, + .q_blk_idx = .{ .scalar = .i32 }, + + .kv_lens = .{ .ref = .{ .shape = &.{cfg.max_num_seqs}, .dtype = .i32, .memory_space = .smem, .role = .scalar_prefetch } }, + .page_indices = .{ .ref = .{ .shape = &.{ cfg.max_num_seqs, cfg.pages_per_seq }, .dtype = .i32, .memory_space = .smem, .role = .scalar_prefetch } }, + .cu_q_lens = .{ .ref = .{ .shape = &.{cfg.max_num_seqs + 1}, .dtype = .i32, .memory_space = .smem, .role = .scalar_prefetch } }, + .seq_buf_idx = .{ .ref = .{ .shape = &.{2}, .dtype = .i32, .memory_space = .smem, .role = .scalar_prefetch } }, + .num_seqs = .{ .ref = .{ .shape = &.{1}, .dtype = .i32, .memory_space = .smem, .role = .scalar_prefetch } }, + + .q = .{ .ref = .{ + .shape = &.{ cfg.numQPerBlk(), cfg.numQHeadsPerBlk(), cfg.head_dim }, + .dtype = cfg.q_dtype, + .window = mtt.ArgSpec.WindowSpec{ .transform_returns = q_transform }, + } }, + .kv_pages = .{ .ref = .{ + .shape = &.{ cfg.total_num_pages, cfg.page_size, cfg.numCombinedKvHeads(), cfg.head_dim }, + .dtype = cfg.kv_dtype, + .memory_space = .any, + } }, + .o = .{ .ref = .{ + .shape = &.{ cfg.numQPerBlk(), cfg.numQHeadsPerBlk(), cfg.head_dim }, + .dtype = cfg.q_dtype, + .role = .output, + .window = mtt.ArgSpec.WindowSpec{ .transform_returns = q_transform }, + } }, + + .kv_bufs = .{ .ref = .{ + .shape = &.{ 2, cfg.num_kv_pages_per_block, cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }, + .dtype = cfg.kv_dtype, + .role = .scratch, + } }, + .sems = .{ .sem_array = .{ .shape = &.{2}, .kind = .dma } }, + // Pallas hardcodes the trailing dim to 128 (TPU lane width); see + // `kernel.py:lm_scratch`. + .l_ref = .{ .ref = .{ .shape = &.{ cfg.numKvHeadsPerBlk(), cfg.flashQRows(), 128 }, .dtype = .f32, .role = .scratch } }, + .m_ref = .{ .ref = .{ .shape = &.{ cfg.numKvHeadsPerBlk(), cfg.flashQRows(), 128 }, .dtype = .f32, .role = .scratch } }, + .acc_ref = .{ .ref = .{ .shape = &.{ cfg.numQPerBlk(), cfg.numQHeadsPerBlk(), cfg.head_dim }, .dtype = .f32, .role = .scratch } }, + }, &.{}, .{ + .dimension_semantics = &.{ .arbitrary, .arbitrary }, + .iteration_bounds = &.{ cfg.numHeadsBlks(), cfg.numQBlks() }, + .scalar_prefetch = 5, + .scratch_operands = 5, + .pallas_window_params = true, + }); + + const k = b; + + const num_seqs = k.scalarLoad(a.num_seqs, &.{k.constIndex(0)}); + const init_seq_idx = k.scalarLoad(a.seq_buf_idx, &.{k.constIndex(0)}); + const init_buf_idx = k.scalarLoad(a.seq_buf_idx, &.{k.constIndex(1)}); + const q_len_start = k.muli(a.q_blk_idx, k.constant(.i32, cfg.numQPerBlk())); + const q_len_end = k.addi(q_len_start, k.constant(.i32, cfg.numQPerBlk())); + + var i = k.openIf(k.cmpi(.eq, k.addi(a.heads_blk_idx, a.q_blk_idx), k.constant(.i32, 0))); + { + const first_seq_idx = nextNonemptyKvSequence(k, cfg, a, init_seq_idx, num_seqs); + var nonempty = k.openIf(k.cmpi(.slt, first_seq_idx, num_seqs)); + { + createKvAsyncCopyDescriptors(k, cfg, a, a.heads_blk_idx, first_seq_idx, k.constant(.i32, 0), init_buf_idx); + nonempty.close(); + } + i.close(); + } + + // Empty rows skip flashAttention, so their accumulator must already be zero. + k.vectorStoreAt(a.acc_ref, k.zeros(&.{ cfg.numQPerBlk(), cfg.numQHeadsPerBlk(), cfg.head_dim }, .f32), &.{ k.constIndex(0), k.constIndex(0), k.constIndex(0) }); + + var q_loop = k.openWhile(.{ k.constant(.i32, 0), init_seq_idx, init_buf_idx }, .{ .i32, .i32, .i32 }); + { + const done = q_loop.before_carried[0]; + const cur_seq_idx = q_loop.before_carried[1]; + const cur_buf_idx = q_loop.before_carried[2]; + const cu_at_num_seqs = k.scalarLoad(a.cu_q_lens, &.{k.toIndex(num_seqs)}); + const should_run = k.andi( + k.cmpi(.slt, q_len_start, cu_at_num_seqs), + k.cmpi(.slt, cur_seq_idx, num_seqs), + ); + const cond = k.andi(k.cmpi(.eq, done, k.constant(.i32, 0)), should_run); + q_loop.yieldBefore(cond, .{ done, cur_seq_idx, cur_buf_idx }); + } + { + const done = q_loop.after_carried[0]; + const cur_seq_idx = q_loop.after_carried[1]; + const cur_buf_idx = q_loop.after_carried[2]; + + const q_start = k.scalarLoad(a.cu_q_lens, &.{k.toIndex(cur_seq_idx)}); + const q_end = k.scalarLoad(a.cu_q_lens, &.{k.toIndex(k.addi(cur_seq_idx, k.constant(.i32, 1)))}); + const q_len = k.subi(q_end, q_start); + const kv_len = k.scalarLoad(a.kv_lens, &.{k.toIndex(cur_seq_idx)}); + + var kv_loop = k.openWhile(.{ k.constant(.i32, 0), cur_buf_idx }, .{ .i32, .i32 }); + { + const kv_blk = kv_loop.before_carried[0]; + const buf = kv_loop.before_carried[1]; + const cond = k.cmpi(.slt, k.muli(kv_blk, k.constant(.i32, cfg.numKvPerBlk())), kv_len); + kv_loop.yieldBefore(cond, .{ kv_blk, buf }); + } + { + const kv_blk = kv_loop.after_carried[0]; + const cur_buf = kv_loop.after_carried[1]; + + const next_kv_blk_raw = k.addi(kv_blk, k.constant(.i32, 1)); + const is_last_kv = k.cmpi(.sge, k.muli(next_kv_blk_raw, k.constant(.i32, cfg.numKvPerBlk())), kv_len); + const next_kv_blk = k.select(is_last_kv, k.constant(.i32, 0), next_kv_blk_raw); + const cur_seq_end = k.cmpi(.sle, q_end, q_len_end); + const next_nonempty_seq = blk: { + // Only sequence transitions need to scan for empty KV rows. + var next_seq = k.openIfElse(k.andi(is_last_kv, cur_seq_end), .{.i32}); + { + const start_seq_idx = k.addi(cur_seq_idx, k.constant(.i32, 1)); + next_seq.yieldThen(.{nextNonemptyKvSequence(k, cfg, a, start_seq_idx, num_seqs)}); + } + { + next_seq.yieldElse(.{cur_seq_idx}); + } + break :blk next_seq.results[0]; + }; + const is_last_seq = k.cmpi(.eq, next_nonempty_seq, num_seqs); + const next_seq_idx = k.select(is_last_seq, k.constant(.i32, 0), next_nonempty_seq); + const next_heads_blk_idx = k.select(is_last_seq, k.addi(a.heads_blk_idx, k.constant(.i32, 1)), a.heads_blk_idx); + const next_buf = k.extui(k.cmpi(.eq, cur_buf, k.constant(.i32, 0)), .i32); + + var ii = k.openIf(k.cmpi(.slt, next_heads_blk_idx, k.constant(.i32, cfg.numHeadsBlks()))); + { + createKvAsyncCopyDescriptors(k, cfg, a, next_heads_blk_idx, next_seq_idx, next_kv_blk, next_buf); + ii.close(); + } + + waitKvDma(k, cfg, a, a.heads_blk_idx, cur_seq_idx, kv_blk, cur_buf); + + // for kv_head_chunk_idx in range(0, num_kv_heads_per_blk, kv_load_step) — unrolled at IR-emit time. + const num_kv_heads_per_blk: usize = @intCast(cfg.numKvHeadsPerBlk()); + const kv_load_step: usize = @intCast(cfg.kvLoadStep()); + var kv_head_chunk_idx: usize = 0; + while (kv_head_chunk_idx < num_kv_heads_per_blk) : (kv_head_chunk_idx += kv_load_step) { + const kv_pairs = stridedLoadKv(k, cfg, a, cur_buf, @intCast(kv_head_chunk_idx)); + var step_idx: usize = 0; + while (step_idx < kv_load_step) : (step_idx += 1) { + var kv_k_val = kv_pairs[step_idx].k; + var kv_v_val = kv_pairs[step_idx].v; + var k_input_dtype = cfg.kv_dtype; + var v_input_dtype = cfg.kv_dtype; + if (cfg.k_scale) |ks| { + const ks_splat = k.splat(ks, &.{ cfg.flashKRows(), cfg.head_dim }, .f32); + kv_k_val = k.mulf(kv_k_val.to(.f32), ks_splat).to(cfg.q_dtype); + k_input_dtype = cfg.q_dtype; + } + if (cfg.v_scale) |vs| { + const vs_splat = k.splat(vs, &.{ cfg.flashKRows(), cfg.head_dim }, .f32); + kv_v_val = k.mulf(kv_v_val.to(.f32), vs_splat).to(cfg.q_dtype); + v_input_dtype = cfg.q_dtype; + } + const kv_head_idx: i64 = @intCast(kv_head_chunk_idx + step_idx); + const q_head_idx = kv_head_idx * cfg.numQHeadsPerKvHead(); + + // `fold_on_2nd_minor` (kernel.py:255-262): when the + // 2nd-to-last dim is not a multiple of q's packing + // factor, cast q to f32 before the trailing-dim reshape + // so the layout pass can fold it cleanly. + var q_block = k.vectorLoadShape(a.q, &.{ k.constIndex(0), k.constIndex(q_head_idx), k.constIndex(0) }, &.{ cfg.numQPerBlk(), cfg.numQHeadsPerKvHead(), cfg.head_dim }); + if (@mod(cfg.numQHeadsPerKvHead(), cfg.qPackingFactor()) != 0) { + q_block = q_block.to(.f32); + } + const q_2d = k.shapeCast(q_block, &.{ cfg.flashQRows(), cfg.head_dim }); + flashAttention(k, cfg, a, q_2d, kv_k_val, kv_v_val, k_input_dtype, v_input_dtype, kv_head_idx, q_head_idx, kv_blk, kv_len, q_start, q_end, q_len, q_len_start); + } + } + + kv_loop.yieldAfter(.{ k.addi(kv_blk, k.constant(.i32, 1)), next_buf }); + } + const next_buf_idx = kv_loop.results[1]; + + const next_seq_idx_outer = k.select(k.cmpi(.sle, q_end, q_len_end), k.addi(cur_seq_idx, k.constant(.i32, 1)), cur_seq_idx); + const next_done = k.select(k.cmpi(.slt, q_end, q_len_end), done, k.constant(.i32, 1)); + q_loop.yieldAfter(.{ next_done, next_seq_idx_outer, next_buf_idx }); + } + const seq_idx_out = q_loop.results[1]; + const buf_idx_out = q_loop.results[2]; + + k.scalarStore(a.seq_buf_idx, k.select(k.cmpi(.slt, seq_idx_out, num_seqs), seq_idx_out, k.constant(.i32, 0)), &.{k.constIndex(0)}); + k.scalarStore(a.seq_buf_idx, buf_idx_out, &.{k.constIndex(1)}); + + k.vectorStoreAt(a.o, k.refLoad(a.acc_ref).to(cfg.q_dtype), &.{ k.constIndex(0), k.constIndex(0), k.constIndex(0) }); +} + +// Fixed drafter blocks can have query tokens without visible KV tokens. +// Keep their query positions, but never schedule a DMA for an empty KV row. +fn nextNonemptyKvSequence( + k: *mtt.Builder, + cfg: Cfg, + a: anytype, + start_seq_idx: mtt.Value, + num_seqs: mtt.Value, +) mtt.Value { + var loop = k.openWhile(.{start_seq_idx}, .{.i32}); + { + const seq_idx = loop.before_carried[0]; + // The terminal index can equal the metadata array length. + const safe_seq_idx = k.minsi(seq_idx, k.constant(.i32, cfg.max_num_seqs - 1)); + const kv_len = k.scalarLoad(a.kv_lens, &.{k.toIndex(safe_seq_idx)}); + const skip = k.andi( + k.cmpi(.slt, seq_idx, num_seqs), + k.cmpi(.sle, kv_len, k.constant(.i32, 0)), + ); + loop.yieldBefore(skip, .{seq_idx}); + } + { + loop.yieldAfter(.{k.addi(loop.after_carried[0], k.constant(.i32, 1))}); + } + return loop.results[0]; +} + +/// Open a builder, run the kernel body, and return the canonicalized MLIR IR. +/// Used by offline tooling (`examples/mosaic_ragged_paged`); production model +/// code goes through `zml.kernel.mosaic_tpu.Kernel(...).emit(...)` instead. +pub fn buildIr(allocator: std.mem.Allocator, ctx: *mlir.Context, cfg: Cfg) ![:0]const u8 { + var b = try mtt.Builder.open(allocator, ctx, "ragged_paged_attention_kernel"); + defer b.deinit(); + try run(&b, cfg); + return b.finishOpts(&.{}, .{ .canonicalize = true }); +} diff --git a/runtimes/tpu/tpu.bzl b/platforms/tpu/tpu.bzl similarity index 50% rename from runtimes/tpu/tpu.bzl rename to platforms/tpu/tpu.bzl index 23b484b6..2fbc2a9b 100644 --- a/runtimes/tpu/tpu.bzl +++ b/platforms/tpu/tpu.bzl @@ -1,17 +1,17 @@ -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("@llvm//:http_bsdtar_archive.bzl", http_archive = "http_bsdtar_archive") def _tpu_impl(mctx): # https://storage.googleapis.com/jax-releases/libtpu_releases.html http_archive( - name = "libpjrt_tpu", - url = "https://storage.googleapis.com/libtpu-nightly-releases/wheels/libtpu-nightly/libtpu_nightly-0.1.dev20240915+nightly-py3-none-any.whl", + name = "libzml_tpu", + url = "https://storage.googleapis.com/libtpu-nightly-releases/wheels/libtpu/libtpu-0.0.50.dev20261005+nightly-cp314-cp314t-manylinux_2_31_x86_64.whl", type = "zip", - sha256 = "4eee6c7dc92e7e60334c5b0261308a0a07c2f5b6235c7c60f465263576c602bf", - build_file = "libpjrt_tpu.BUILD.bazel", + sha256 = "a1e8615a33f60e1bfce15ec9d570642f882a8bec1d46763531e3b831edb2bb42", + build_file = "libzml_tpu.BUILD.bazel", ) return mctx.extension_metadata( reproducible = True, - root_module_direct_deps = ["libpjrt_tpu"], + root_module_direct_deps = ["libzml_tpu"], root_module_direct_dev_deps = [], ) diff --git a/platforms/tpu/tpu.zig b/platforms/tpu/tpu.zig new file mode 100644 index 00000000..3c8143bb --- /dev/null +++ b/platforms/tpu/tpu.zig @@ -0,0 +1,60 @@ +const std = @import("std"); +const builtin = @import("builtin"); + +const bazel = @import("bazel"); +const bazel_builtin = @import("bazel_builtin"); +const pjrt = @import("pjrt"); +const platforms_options = @import("platforms/options"); +const stdx = @import("stdx"); + +const log = std.log.scoped(.@"zml/platforms/tpu"); + +pub fn isEnabled() bool { + return platforms_options.tpu_enabled; +} + +/// Check if running on Google Compute Engine, because TPUs will poll the +/// metadata server, hanging the process. So only do it on GCP. +/// Do it using the official method at: +/// https://cloud.google.com/compute/docs/instances/detect-compute-engine?hl=en#use_operating_system_tools_to_detect_if_a_vm_is_running_in +fn isOnGCP(io: std.Io) !bool { + const GoogleComputeEngine = "Google Compute Engine"; + var buffer: [GoogleComputeEngine.len]u8 = undefined; + return std.mem.eql( + u8, + GoogleComputeEngine, + try std.Io.Dir.readFile( + .cwd(), + io, + "/sys/devices/virtual/dmi/id/product_name", + &buffer, + ), + ); +} + +pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { + _ = allocator; + if (comptime !isEnabled()) { + return error.Unavailable; + } + if (comptime builtin.os.tag != .linux) { + return error.Unavailable; + } + if (!(isOnGCP(io) catch false)) { + return error.Unavailable; + } + + const r = try bazel.runfiles(bazel_builtin.current_repository); + + var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const sandbox_path = try r.rlocation("libzml_tpu/sandbox", &path_buf) orelse { + log.err("Failed to find sandbox path for TPU runtime", .{}); + return error.FileNotFound; + }; + + return blk: { + var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libzml_tpu.so" }); + break :blk .loadFrom(path); + }; +} diff --git a/runtimes/BUILD.bazel b/runtimes/BUILD.bazel deleted file mode 100644 index 291651c6..00000000 --- a/runtimes/BUILD.bazel +++ /dev/null @@ -1,42 +0,0 @@ -load("@bazel_skylib//rules:common_settings.bzl", "bool_flag") - -RUNTIMES = { - "cpu": True, - "cuda": False, - "rocm": False, - "tpu": False, -} - -[ - bool_flag( - name = runtime, - build_setting_default = default, - ) - for runtime, default in RUNTIMES.items() -] - -[ - config_setting( - name = "_{}".format(runtime), - flag_values = {":{}".format(runtime): "True"}, - ) - for runtime in RUNTIMES.keys() -] - -cc_library( - name = "runtimes", - visibility = ["//visibility:public"], - deps = select({ - ":_cpu": ["//runtimes/cpu"], - "//conditions:default": [], - }) + select({ - ":_cuda": ["//runtimes/cuda"], - "//conditions:default": [], - }) + select({ - ":_rocm": ["//runtimes/rocm"], - "//conditions:default": [], - }) + select({ - ":_tpu": ["//runtimes/tpu"], - "//conditions:default": [], - }), -) diff --git a/runtimes/cpu/BUILD.bazel b/runtimes/cpu/BUILD.bazel deleted file mode 100644 index ac3ac6d8..00000000 --- a/runtimes/cpu/BUILD.bazel +++ /dev/null @@ -1,8 +0,0 @@ -alias( - name = "cpu", - actual = select({ - "@platforms//os:macos": "@libpjrt_cpu_darwin_arm64//:libpjrt_cpu", - "@platforms//os:linux": "@libpjrt_cpu_linux_amd64//:libpjrt_cpu", - }), - visibility = ["//visibility:public"], -) diff --git a/runtimes/cpu/cpu.bzl b/runtimes/cpu/cpu.bzl deleted file mode 100644 index 8cd1599f..00000000 --- a/runtimes/cpu/cpu.bzl +++ /dev/null @@ -1,34 +0,0 @@ -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") - -_BUILD = """\ -cc_import( - name = "libpjrt_cpu", - shared_library = "libpjrt_cpu.{ext}", - visibility = ["//visibility:public"], -) -""" - -def _cpu_pjrt_plugin_impl(mctx): - http_archive( - name = "libpjrt_cpu_linux_amd64", - build_file_content = _BUILD.format(ext = "so"), - sha256 = "2058c999a4866716f1dae0c42476c09da0f6deff7e77e34c5223b61f5e0027fb", - url = "https://github.com/zml/pjrt-artifacts/releases/download/v0.2.2/pjrt-cpu_linux-amd64.tar.gz", - ) - - http_archive( - name = "libpjrt_cpu_darwin_arm64", - build_file_content = _BUILD.format(ext = "dylib"), - sha256 = "727b0380a577b2759468a4e0b3574e1d81e1b4348c3942d23284d590c7ca91a5", - url = "https://github.com/zml/pjrt-artifacts/releases/download/v0.2.2/pjrt-cpu_darwin-arm64.tar.gz", - ) - - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -cpu_pjrt_plugin = module_extension( - implementation = _cpu_pjrt_plugin_impl, -) diff --git a/runtimes/cuda/BUILD.bazel b/runtimes/cuda/BUILD.bazel deleted file mode 100644 index 530c10d1..00000000 --- a/runtimes/cuda/BUILD.bazel +++ /dev/null @@ -1,5 +0,0 @@ -alias( - name = "cuda", - actual = "@libpjrt_cuda", - visibility = ["//visibility:public"], -) diff --git a/runtimes/cuda/cuda.bzl b/runtimes/cuda/cuda.bzl deleted file mode 100644 index aa85b8e1..00000000 --- a/runtimes/cuda/cuda.bzl +++ /dev/null @@ -1,197 +0,0 @@ -load("@bazel_skylib//lib:paths.bzl", "paths") -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -load("//bazel:http_deb_archive.bzl", "http_deb_archive") - -ARCH = "linux-x86_64" - -CUDA_VERSION = "12.6.2" -CUDNN_VERSION = "9.4.0" - -_CC_IMPORT_TPL = """\ -cc_import( - name = "{name}", - shared_library = "lib/{shared_library}", - visibility = ["@libpjrt_cuda//:__subpackages__"], -) -""" - -CUDA_PACKAGES = { - "cuda_cudart": _CC_IMPORT_TPL.format(name = "cudart", shared_library = "libcudart.so.12"), - "cuda_cupti": _CC_IMPORT_TPL.format(name = "cupti", shared_library = "libcupti.so.12"), - "libcufft": _CC_IMPORT_TPL.format(name = "cufft", shared_library = "libcufft.so.11"), - "libcusolver": _CC_IMPORT_TPL.format(name = "cusolver", shared_library = "libcusolver.so.11"), - "libcusparse": _CC_IMPORT_TPL.format(name = "cusparse", shared_library = "libcusparse.so.12"), - "libnvjitlink": _CC_IMPORT_TPL.format(name = "nvjitlink", shared_library = "libnvJitLink.so.12"), - "cuda_nvcc": """\ -filegroup( - name = "ptxas", - srcs = ["bin/ptxas"], - visibility = ["@libpjrt_cuda//:__subpackages__"], -) - -filegroup( - name = "libdevice", - srcs = ["nvvm/libdevice/libdevice.10.bc"], - visibility = ["@libpjrt_cuda//:__subpackages__"], -) - -cc_import( - name = "nvvm", - shared_library = "nvvm/lib64/libnvvm.so.4", - visibility = ["@libpjrt_cuda//:__subpackages__"], -) -""", - "cuda_nvrtc": """\ -cc_import( - name = "nvrtc", - shared_library = "lib/libnvrtc.so.12", - visibility = ["@libpjrt_cuda//:__subpackages__"], - deps = [":nvrtc_builtins"], -) - -cc_import( - name = "nvrtc_builtins", - shared_library = "lib/libnvrtc-builtins.so.12.6", -) -""", - "libcublas": """\ -cc_import( - name = "cublasLt", - shared_library = "lib/libcublasLt.so.12", -) - -cc_import( - name = "cublas", - shared_library = "lib/libcublas.so.12", - visibility = ["@libpjrt_cuda//:__subpackages__"], - deps = [":cublasLt"], -) -""", -} - -CUDNN_PACKAGES = { - "cudnn": """\ -cc_import( - name = "cudnn", - shared_library = "lib/libcudnn.so.9", - visibility = ["@libpjrt_cuda//:__subpackages__"], - deps = [ - ":cudnn_adv", - ":cudnn_ops", - ":cudnn_cnn", - ":cudnn_graph", - ":cudnn_engines_precompiled", - ":cudnn_engines_runtime_compiled", - ":cudnn_heuristic", - ], -) - -cc_import( - name = "cudnn_adv", - shared_library = "lib/libcudnn_adv.so.9", -) - -cc_import( - name = "cudnn_ops", - shared_library = "lib/libcudnn_ops.so.9", -) - -cc_import( - name = "cudnn_cnn", - shared_library = "lib/libcudnn_cnn.so.9", - deps = [":cudnn_ops"], -) - -cc_import( - name = "cudnn_graph", - shared_library = "lib/libcudnn_graph.so.9", -) - -cc_import( - name = "cudnn_engines_precompiled", - shared_library = "lib/libcudnn_engines_precompiled.so.9", -) - -cc_import( - name = "cudnn_engines_runtime_compiled", - shared_library = "lib/libcudnn_engines_runtime_compiled.so.9", -) - -cc_import( - name = "cudnn_heuristic", - shared_library = "lib/libcudnn_heuristic.so.9", -) -""", -} - -def _cuda_impl(mctx): - CUDA_REDIST = json.decode(mctx.read(Label("@zml//runtimes/cuda:cuda.redistrib_{}.json".format(CUDA_VERSION)))) - CUDNN_REDIST = json.decode(mctx.read(Label("@zml//runtimes/cuda:cudnn.redistrib_{}.json".format(CUDNN_VERSION)))) - - for pkg, build_file_content in CUDA_PACKAGES.items(): - pkg_data = CUDA_REDIST[pkg] - arch_data = pkg_data.get(ARCH) - if not arch_data: - continue - http_archive( - name = pkg, - build_file_content = build_file_content, - url = "https://developer.download.nvidia.com/compute/cuda/redist/" + arch_data["relative_path"], - sha256 = arch_data["sha256"], - strip_prefix = paths.basename(arch_data["relative_path"]).replace(".tar.xz", ""), - ) - - for pkg, build_file_content in CUDNN_PACKAGES.items(): - pkg_data = CUDNN_REDIST[pkg] - arch_data = pkg_data.get(ARCH) - if not arch_data: - continue - arch_data = arch_data.get("cuda12", arch_data) - http_archive( - name = pkg, - build_file_content = build_file_content, - url = "https://developer.download.nvidia.com/compute/cudnn/redist/" + arch_data["relative_path"], - sha256 = arch_data["sha256"], - strip_prefix = paths.basename(arch_data["relative_path"]).replace(".tar.xz", ""), - ) - - http_deb_archive( - name = "libnccl", - urls = ["https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/libnccl2_2.22.3-1+cuda12.6_amd64.deb"], - sha256 = "2f64685bcd503150ab45d00503236a56da58a15eac5fd36508045a74f4e10678", - build_file_content = """\ -cc_import( - name = "nccl", - shared_library = "usr/lib/x86_64-linux-gnu/libnccl.so.2", - visibility = ["@libpjrt_cuda//:__subpackages__"], -) -""", - ) - http_deb_archive( - name = "zlib", - urls = ["http://archive.ubuntu.com/ubuntu/pool/main/z/zlib/zlib1g_1.3.dfsg-3.1ubuntu2.1_amd64.deb"], - sha256 = "7074b6a2f6367a10d280c00a1cb02e74277709180bab4f2491a2f355ab2d6c20", - build_file_content = """\ -cc_import( - name = "zlib", - shared_library = "usr/lib/x86_64-linux-gnu/libz.so.1", - visibility = ["@libpjrt_cuda//:__subpackages__"], -) -""", - ) - http_archive( - name = "libpjrt_cuda", - build_file = "libpjrt_cuda.BUILD.bazel", - url = "https://github.com/zml/pjrt-artifacts/releases/download/v0.2.3/pjrt-cuda_linux-amd64.tar.gz", - sha256 = "14f39ffef0c9ac529b1a8957750b0b5f5d2f6d310c0d997436051c53a9eb1618", - ) - - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = ["libpjrt_cuda"], - root_module_direct_dev_deps = [], - ) - -cuda_packages = module_extension( - implementation = _cuda_impl, -) diff --git a/runtimes/cuda/cuda.redistrib_12.6.2.json b/runtimes/cuda/cuda.redistrib_12.6.2.json deleted file mode 100644 index 28814969..00000000 --- a/runtimes/cuda/cuda.redistrib_12.6.2.json +++ /dev/null @@ -1,1037 +0,0 @@ -{ - "release_date": "2024-10-01", - "release_label": "12.6.2", - "release_product": "cuda", - "cuda_cccl": { - "name": "CXX Core Compute Libraries", - "license": "CUDA Toolkit", - "license_path": "cuda_cccl/LICENSE.txt", - "version": "12.6.77", - "linux-x86_64": { - "relative_path": "cuda_cccl/linux-x86_64/cuda_cccl-linux-x86_64-12.6.77-archive.tar.xz", - "sha256": "9c3145ef01f73e50c0f5fcf923f0899c847f487c529817daa8f8b1a3ecf20925", - "md5": "93c6af9115fe841e0b70c61313fd3784", - "size": "934952" - }, - "linux-sbsa": { - "relative_path": "cuda_cccl/linux-sbsa/cuda_cccl-linux-sbsa-12.6.77-archive.tar.xz", - "sha256": "cea5f1487b9ff78a01cadd96e950bfe6fe4bc638849fae81b8739adfdde47146", - "md5": "d45c87f6a996db35c91561b1be0a0b19", - "size": "934808" - }, - "windows-x86_64": { - "relative_path": "cuda_cccl/windows-x86_64/cuda_cccl-windows-x86_64-12.6.77-archive.zip", - "sha256": "469b783b6731964170b3d27ae214e97e2a63ec93c57ace3ed4052e6d01261f2e", - "md5": "6e472a3821a5eb3e6a4d52aaf87f7757", - "size": "2929962" - }, - "linux-aarch64": { - "relative_path": "cuda_cccl/linux-aarch64/cuda_cccl-linux-aarch64-12.6.77-archive.tar.xz", - "sha256": "ab8bd5c346ca69c96ac9afb9d317e8cbc0ec8fc32629ca17b38c14b3dede2fb7", - "md5": "0726a103aec56a5f726e5868ee6e29c2", - "size": "935196" - } - }, - "cuda_compat": { - "name": "CUDA compat L4T", - "license": "CUDA Toolkit", - "license_path": "cuda_compat/LICENSE.txt", - "version": "12.6.36890662", - "linux-aarch64": { - "relative_path": "cuda_compat/linux-aarch64/cuda_compat-linux-aarch64-12.6.36890662-archive.tar.xz", - "sha256": "007cdab536bf0a1ec274faff73cde2233f3ccfe0e6dba2b015fb1b527e7e231f", - "md5": "de7405e0e2e570b52d16dedc14ebc44e", - "size": "19628624" - } - }, - "cuda_cudart": { - "name": "CUDA Runtime (cudart)", - "license": "CUDA Toolkit", - "license_path": "cuda_cudart/LICENSE.txt", - "version": "12.6.77", - "linux-x86_64": { - "relative_path": "cuda_cudart/linux-x86_64/cuda_cudart-linux-x86_64-12.6.77-archive.tar.xz", - "sha256": "f74689258a60fd9c5bdfa7679458527a55e22442691ba678dcfaeffbf4391ef9", - "md5": "868af7d55bed8ce473f6a9c1416f846e", - "size": "1126072" - }, - "linux-sbsa": { - "relative_path": "cuda_cudart/linux-sbsa/cuda_cudart-linux-sbsa-12.6.77-archive.tar.xz", - "sha256": "c73c8e5bfe8fcd7468d012c9eebff15063005a3bba44423d541d573dc058de58", - "md5": "e0d2f3ee94af8d68bb7e40affd70cb75", - "size": "1112288" - }, - "windows-x86_64": { - "relative_path": "cuda_cudart/windows-x86_64/cuda_cudart-windows-x86_64-12.6.77-archive.zip", - "sha256": "7a313bc0c93b1a50bb03aa9783a199ae70c3b66e2d8084da65e8254a8577b925", - "md5": "27da2c088df06222d204c88f3d7e6831", - "size": "2512007" - }, - "linux-aarch64": { - "relative_path": "cuda_cudart/linux-aarch64/cuda_cudart-linux-aarch64-12.6.77-archive.tar.xz", - "sha256": "d5b8f6f5a2ab34066c267daf5f73f25d2f24ea25d8fa3dce427c196ff07504e7", - "md5": "f99fdf8e2a33ec402cc209d650ddb60d", - "size": "1175528" - } - }, - "cuda_cuobjdump": { - "name": "cuobjdump", - "license": "CUDA Toolkit", - "license_path": "cuda_cuobjdump/LICENSE.txt", - "version": "12.6.77", - "linux-x86_64": { - "relative_path": "cuda_cuobjdump/linux-x86_64/cuda_cuobjdump-linux-x86_64-12.6.77-archive.tar.xz", - "sha256": "54d0edf14284249baf8e2db1c26c71d079ad7afd1f41779e22f239ff9e8bf60f", - "md5": "dd84a355927fff7134e968074d40f82e", - "size": "219068" - }, - "linux-sbsa": { - "relative_path": "cuda_cuobjdump/linux-sbsa/cuda_cuobjdump-linux-sbsa-12.6.77-archive.tar.xz", - "sha256": "a55dcccc834b018b994a9592ca20245e5e14449dbb8c61d76e43347e4bfa21b9", - "md5": "429926be33ad495c506c5a8036973bf7", - "size": "208736" - }, - "windows-x86_64": { - "relative_path": "cuda_cuobjdump/windows-x86_64/cuda_cuobjdump-windows-x86_64-12.6.77-archive.zip", - "sha256": "b9207c09a5d6b34e6bdd8ff2093b3d101ef85f08b34c8ba8d710050b522661ae", - "md5": "bee233537c8ea01ca1e22e398d5e97c2", - "size": "4248722" - }, - "linux-aarch64": { - "relative_path": "cuda_cuobjdump/linux-aarch64/cuda_cuobjdump-linux-aarch64-12.6.77-archive.tar.xz", - "sha256": "c75d148dca4d4672a81a83601f06ae7466367ea5c9a4e61cf74fa7396a437eb4", - "md5": "a3c6e47e36427d6e6ffd892b86e174f7", - "size": "196188" - } - }, - "cuda_cupti": { - "name": "CUPTI", - "license": "CUDA Toolkit", - "license_path": "cuda_cupti/LICENSE.txt", - "version": "12.6.80", - "linux-x86_64": { - "relative_path": "cuda_cupti/linux-x86_64/cuda_cupti-linux-x86_64-12.6.80-archive.tar.xz", - "sha256": "5ef9aa8229f81e16f243ebe5b710a8b6a871374bbe1aeb5b95c0ed35791592b7", - "md5": "c7b643aa8744ed25065233e660ce159a", - "size": "13922028" - }, - "linux-sbsa": { - "relative_path": "cuda_cupti/linux-sbsa/cuda_cupti-linux-sbsa-12.6.80-archive.tar.xz", - "sha256": "990b04b1e17c0c29612438b0f31bef94b4c5151f51384a51873a50cb23f88c75", - "md5": "216065a718351644ae1eb9010bdd8266", - "size": "10877748" - }, - "windows-x86_64": { - "relative_path": "cuda_cupti/windows-x86_64/cuda_cupti-windows-x86_64-12.6.80-archive.zip", - "sha256": "6b69bad461cd54f6555a11785b6e5878407df5bdc3caeadf2ff729cf45376128", - "md5": "c285a8a43b40ae31f258a6b9da95dc36", - "size": "11216576" - }, - "linux-aarch64": { - "relative_path": "cuda_cupti/linux-aarch64/cuda_cupti-linux-aarch64-12.6.80-archive.tar.xz", - "sha256": "9a3945219e5cf98c8df612681bd1984a9fdad6d86226bb0e99a515d18f736b4b", - "md5": "4e825a435ed237ee97370d8a4f9f669c", - "size": "4093644" - } - }, - "cuda_cuxxfilt": { - "name": "CUDA cuxxfilt (demangler)", - "license": "CUDA Toolkit", - "license_path": "cuda_cuxxfilt/LICENSE.txt", - "version": "12.6.77", - "linux-x86_64": { - "relative_path": "cuda_cuxxfilt/linux-x86_64/cuda_cuxxfilt-linux-x86_64-12.6.77-archive.tar.xz", - "sha256": "f8c4c9434621597367ce826f6ad3c0267cf06ca50625b36ff32777604013235d", - "md5": "1cd4eb0e893fc3c148e0e435e1ed7cef", - "size": "189148" - }, - "linux-sbsa": { - "relative_path": "cuda_cuxxfilt/linux-sbsa/cuda_cuxxfilt-linux-sbsa-12.6.77-archive.tar.xz", - "sha256": "b8f82d3dd97fed289f40e8552ead4ddd88db70775074fc3ed06fe89dd17be320", - "md5": "161cd522f27d7f213fc7245d2d852d2e", - "size": "176760" - }, - "windows-x86_64": { - "relative_path": "cuda_cuxxfilt/windows-x86_64/cuda_cuxxfilt-windows-x86_64-12.6.77-archive.zip", - "sha256": "e771414a1d96eb4b89c0759cd2d2f127b929c5c2a0961f176c3a8f0736e94b64", - "md5": "8478f284513a59e15e32cfc032f64468", - "size": "170746" - }, - "linux-aarch64": { - "relative_path": "cuda_cuxxfilt/linux-aarch64/cuda_cuxxfilt-linux-aarch64-12.6.77-archive.tar.xz", - "sha256": "f860a51d5baf19ada53ed9af6746ab7a256290ac9c579d3f9de93756b66e4734", - "md5": "ceba05046b5056fa274747cb91fbf3e7", - "size": "170252" - } - }, - "cuda_demo_suite": { - "name": "CUDA Demo Suite", - "license": "CUDA Toolkit", - "license_path": "cuda_demo_suite/LICENSE.txt", - "version": "12.6.77", - "linux-x86_64": { - "relative_path": "cuda_demo_suite/linux-x86_64/cuda_demo_suite-linux-x86_64-12.6.77-archive.tar.xz", - "sha256": "8911660314f06d9e1876be7c4d5ebd432ee7ac19df34b9866bde3ce9be0ac4c6", - "md5": "8807586a517743187f0afa4078372196", - "size": "4016948" - }, - "windows-x86_64": { - "relative_path": "cuda_demo_suite/windows-x86_64/cuda_demo_suite-windows-x86_64-12.6.77-archive.zip", - "sha256": "829480fff285de7668b39d821703bce2f75d564690886ac6970545b4749f7fc3", - "md5": "3ac5275c99c7d6696fb84bef9f5c9858", - "size": "5065609" - } - }, - "cuda_documentation": { - "name": "CUDA Documentation", - "license": "CUDA Toolkit", - "license_path": "cuda_documentation/LICENSE.txt", - "version": "12.6.77", - "linux-x86_64": { - "relative_path": "cuda_documentation/linux-x86_64/cuda_documentation-linux-x86_64-12.6.77-archive.tar.xz", - "sha256": "438a9875891b197ce3129a4dfc52e4d30e13b876ddd8ae0dbeb522969a4066c6", - "md5": "ac71a1bba443e0c3f53b79af99462356", - "size": "67324" - }, - "linux-sbsa": { - "relative_path": "cuda_documentation/linux-sbsa/cuda_documentation-linux-sbsa-12.6.77-archive.tar.xz", - "sha256": "b02f326f3e96b847358ecb3dc1de5d0d83fee512b8aea33fe30a5c8b63647eaa", - "md5": "18b1833075c0b3c995cc2114dceacf6e", - "size": "67144" - }, - "windows-x86_64": { - "relative_path": "cuda_documentation/windows-x86_64/cuda_documentation-windows-x86_64-12.6.77-archive.zip", - "sha256": "397931b1bf6981ecbfb5afbbb203e243e6af950a97913577c3ddbcbf9f7eaffb", - "md5": "d80708e0d0a696c5e78d84ba876f57d5", - "size": "105658" - }, - "linux-aarch64": { - "relative_path": "cuda_documentation/linux-aarch64/cuda_documentation-linux-aarch64-12.6.77-archive.tar.xz", - "sha256": "7f05a2d72ea16b75ffffe4389ae295a55a33d98d05bbcf4a987f1fc72b620d09", - "md5": "908512080538115b91f70a685c5e56d0", - "size": "67140" - } - }, - "cuda_gdb": { - "name": "CUDA GDB", - "license": "CUDA Toolkit", - "license_path": "cuda_gdb/LICENSE.txt", - "version": "12.6.77", - "linux-x86_64": { - "relative_path": "cuda_gdb/linux-x86_64/cuda_gdb-linux-x86_64-12.6.77-archive.tar.xz", - "sha256": "458cb1c88eb0af9d986071782c48134b9871633b1c7b29b1f73df194b5ae70db", - "md5": "314d69883a38d39d0aa7710ff279627e", - "size": "64688388" - }, - "linux-sbsa": { - "relative_path": "cuda_gdb/linux-sbsa/cuda_gdb-linux-sbsa-12.6.77-archive.tar.xz", - "sha256": "914e6e44ec4174c4961275644e9fa675d3cb7a6962761772d48e414f95dce575", - "md5": "2abf0f4d63483c86cca66c898d68d240", - "size": "43505780" - }, - "linux-aarch64": { - "relative_path": "cuda_gdb/linux-aarch64/cuda_gdb-linux-aarch64-12.6.77-archive.tar.xz", - "sha256": "2cd9a9e949ccd1c5f3aea55fb023eb7fc87d4c1d9ac5bd2ba6301e1aed7631a1", - "md5": "cbfdc9bb202a819e48a7336f3a715abe", - "size": "43422588" - } - }, - "cuda_nsight": { - "name": "Nsight Eclipse Edition Plugin", - "license": "CUDA Toolkit", - "license_path": "cuda_nsight/LICENSE.txt", - "version": "12.6.77", - "linux-x86_64": { - "relative_path": "cuda_nsight/linux-x86_64/cuda_nsight-linux-x86_64-12.6.77-archive.tar.xz", - "sha256": "9e13b2784eed7c6edad031d5f8561a675370a39c2f6f1e6fb915af31ccfc11d1", - "md5": "9b48cfe6dcaef7ed91a162e81b60677a", - "size": "118682264" - } - }, - "cuda_nvcc": { - "name": "CUDA NVCC", - "license": "CUDA Toolkit", - "license_path": "cuda_nvcc/LICENSE.txt", - "version": "12.6.77", - "linux-x86_64": { - "relative_path": "cuda_nvcc/linux-x86_64/cuda_nvcc-linux-x86_64-12.6.77-archive.tar.xz", - "sha256": "c2b319299ead62980b6978670498bfb70aad2309fdb581de197a682648ff3b72", - "md5": "fa9d2c9ee18b2fb71b8fc3cc57d917c1", - "size": "49861812" - }, - "linux-sbsa": { - "relative_path": "cuda_nvcc/linux-sbsa/cuda_nvcc-linux-sbsa-12.6.77-archive.tar.xz", - "sha256": "fcee340e703c8dec0b7fc4748a70c4b676f1946f92f0b6584fefd7b1a4c0bd63", - "md5": "86ce697b86ccb311ba48c55c030f8482", - "size": "43928232" - }, - "windows-x86_64": { - "relative_path": "cuda_nvcc/windows-x86_64/cuda_nvcc-windows-x86_64-12.6.77-archive.zip", - "sha256": "f537b70b04f41bf0c3b2f4cc063ae448367accc246aa53f9d40559bcd41d42cb", - "md5": "64f0208d9fb56a64ba18444517a23829", - "size": "80138996" - }, - "linux-aarch64": { - "relative_path": "cuda_nvcc/linux-aarch64/cuda_nvcc-linux-aarch64-12.6.77-archive.tar.xz", - "sha256": "19972955983cdb8eb01b786c7c03b1f5c92a5dcf27304eda7fcdecab77224c85", - "md5": "165c50d3bd8eaa89df5fd95064a8ef8d", - "size": "45417876" - } - }, - "cuda_nvdisasm": { - "name": "CUDA nvdisasm", - "license": "CUDA Toolkit", - "license_path": "cuda_nvdisasm/LICENSE.txt", - "version": "12.6.77", - "linux-x86_64": { - "relative_path": "cuda_nvdisasm/linux-x86_64/cuda_nvdisasm-linux-x86_64-12.6.77-archive.tar.xz", - "sha256": "aeae4d2bdeaaa0b9149f1c6f486d40b248e8f4f5f1fccba42de5ab69337f7252", - "md5": "837df3229ef75698f2cafb4cf8431705", - "size": "49896188" - }, - "linux-sbsa": { - "relative_path": "cuda_nvdisasm/linux-sbsa/cuda_nvdisasm-linux-sbsa-12.6.77-archive.tar.xz", - "sha256": "aeed7512466923f91f339573c110dd8187a21c110f70fc391510372148de47b5", - "md5": "3657a465fd33820850b6aed6452b24fa", - "size": "49812236" - }, - "windows-x86_64": { - "relative_path": "cuda_nvdisasm/windows-x86_64/cuda_nvdisasm-windows-x86_64-12.6.77-archive.zip", - "sha256": "f3e52fc4cb3bab0e097c7f5d1ee19ff90bf40f3304aef02281dfde989b5d008c", - "md5": "5ab1d53d0a54ff7a01a271a82a466f71", - "size": "50158269" - }, - "linux-aarch64": { - "relative_path": "cuda_nvdisasm/linux-aarch64/cuda_nvdisasm-linux-aarch64-12.6.77-archive.tar.xz", - "sha256": "5f9616270bb35bf4bfa519c8ec1097e51e786646800077df2ed0409b3d728b68", - "md5": "fe319e2ff5084624209f7348879bcb3e", - "size": "49821656" - } - }, - "cuda_nvml_dev": { - "name": "CUDA NVML Headers", - "license": "CUDA Toolkit", - "license_path": "cuda_nvml_dev/LICENSE.txt", - "version": "12.6.77", - "linux-x86_64": { - "relative_path": "cuda_nvml_dev/linux-x86_64/cuda_nvml_dev-linux-x86_64-12.6.77-archive.tar.xz", - "sha256": "bf297acf16ddc9340859180c5395689fcd890c1e3295915851de3977f306d601", - "md5": "2153a3e64008e638ce2d5e298534ee1b", - "size": "145364" - }, - "linux-sbsa": { - "relative_path": "cuda_nvml_dev/linux-sbsa/cuda_nvml_dev-linux-sbsa-12.6.77-archive.tar.xz", - "sha256": "3083c53ca6fcde4f5a41eebee647fe6b49a408431e1bd53ab5b0a38e4fd2268e", - "md5": "7f502ebe3aba5cfae3043922533e218f", - "size": "147848" - }, - "windows-x86_64": { - "relative_path": "cuda_nvml_dev/windows-x86_64/cuda_nvml_dev-windows-x86_64-12.6.77-archive.zip", - "sha256": "45f2466ff154a44701388a90913b7d1c4c0ec591f0ec33b3d30464d5c2ffbe8d", - "md5": "50774ef4ccf9253357b27389647d2509", - "size": "129922" - }, - "linux-aarch64": { - "relative_path": "cuda_nvml_dev/linux-aarch64/cuda_nvml_dev-linux-aarch64-12.6.77-archive.tar.xz", - "sha256": "56abe6ff8e45f1d07433963093557a82ed92689a4ce9bc6f9a5e1c076141c5bb", - "md5": "93ad8f14c5ca47c30b3e35b39185db28", - "size": "148564" - } - }, - "cuda_nvprof": { - "name": "CUDA nvprof", - "license": "CUDA Toolkit", - "license_path": "cuda_nvprof/LICENSE.txt", - "version": "12.6.80", - "linux-x86_64": { - "relative_path": "cuda_nvprof/linux-x86_64/cuda_nvprof-linux-x86_64-12.6.80-archive.tar.xz", - "sha256": "c206de5394cd7411f7793fad077f39be235533323b7dc4c1d463f3cdc1cc838d", - "md5": "025440d214b5e3a227abd900567afccf", - "size": "2438040" - }, - "windows-x86_64": { - "relative_path": "cuda_nvprof/windows-x86_64/cuda_nvprof-windows-x86_64-12.6.80-archive.zip", - "sha256": "a7aba7f1a24d721e659bf058cc1abfeceaad50bdf1cc9f88e1f67d4de5584663", - "md5": "f43132255cb5ab3dcdd0241b7c91671b", - "size": "1703106" - } - }, - "cuda_nvprune": { - "name": "CUDA nvprune", - "license": "CUDA Toolkit", - "license_path": "cuda_nvprune/LICENSE.txt", - "version": "12.6.77", - "linux-x86_64": { - "relative_path": "cuda_nvprune/linux-x86_64/cuda_nvprune-linux-x86_64-12.6.77-archive.tar.xz", - "sha256": "c633bf85f0263ff01fd2c34bd8341e65363faef775b3127dfdac48d74e6d2b8b", - "md5": "7f5b2a558dbc2e98e30f7809b7f469ce", - "size": "57080" - }, - "linux-sbsa": { - "relative_path": "cuda_nvprune/linux-sbsa/cuda_nvprune-linux-sbsa-12.6.77-archive.tar.xz", - "sha256": "94e0fb67ac1329b22f1e459aad5bdf0188a65c9f03d8635601fc88180d089199", - "md5": "057dd355a4eacf501085b35cfcd63cee", - "size": "48972" - }, - "windows-x86_64": { - "relative_path": "cuda_nvprune/windows-x86_64/cuda_nvprune-windows-x86_64-12.6.77-archive.zip", - "sha256": "b586c315e32731794f8bc7b3cbccbeb7f535d361f0191b456f730c70ad74612e", - "md5": "a42de97cfad819da9f8a9fb7a82a3281", - "size": "146651" - }, - "linux-aarch64": { - "relative_path": "cuda_nvprune/linux-aarch64/cuda_nvprune-linux-aarch64-12.6.77-archive.tar.xz", - "sha256": "d5f69d4b4b88d6da0175c9f8ec44b428867a6675dac166de93995b21b69735d7", - "md5": "651d49f5add932ee6a631ef99dc7ba0f", - "size": "50800" - } - }, - "cuda_nvrtc": { - "name": "CUDA NVRTC", - "license": "CUDA Toolkit", - "license_path": "cuda_nvrtc/LICENSE.txt", - "version": "12.6.77", - "linux-x86_64": { - "relative_path": "cuda_nvrtc/linux-x86_64/cuda_nvrtc-linux-x86_64-12.6.77-archive.tar.xz", - "sha256": "9bc74afbd5027d565631c8cdf6e2b9ddb1815d87137d8fdb724ca71596e1a8e3", - "md5": "8ecf112732d1851292b0c27eaf9c5d33", - "size": "34138444" - }, - "linux-sbsa": { - "relative_path": "cuda_nvrtc/linux-sbsa/cuda_nvrtc-linux-sbsa-12.6.77-archive.tar.xz", - "sha256": "3bd7a77c8c0a7b57c8814127721bea7998b7c7e6f3750cea99f69899ddf302b8", - "md5": "8b8ca2262c83aaa95bbca0b3b51be530", - "size": "31899772" - }, - "windows-x86_64": { - "relative_path": "cuda_nvrtc/windows-x86_64/cuda_nvrtc-windows-x86_64-12.6.77-archive.zip", - "sha256": "99631d3d918ae426739388ddb1b05a3edae0b13a69e37ae5c517c60947b8bbf5", - "md5": "fcbd846651415ba0d69c14274ddfa335", - "size": "187234763" - }, - "linux-aarch64": { - "relative_path": "cuda_nvrtc/linux-aarch64/cuda_nvrtc-linux-aarch64-12.6.77-archive.tar.xz", - "sha256": "aafeadb7108b49ed6d432cc471262eafd9c585dfa3317b57815c59a727f872d4", - "md5": "bb37e68368db642d7bbe23d2fef1ea3c", - "size": "33079920" - } - }, - "cuda_nvtx": { - "name": "CUDA NVTX", - "license": "CUDA Toolkit", - "license_path": "cuda_nvtx/LICENSE.txt", - "version": "12.6.77", - "linux-x86_64": { - "relative_path": "cuda_nvtx/linux-x86_64/cuda_nvtx-linux-x86_64-12.6.77-archive.tar.xz", - "sha256": "d97dd4f3bbc23e57302990c1573f737bea55f3174b2c8e3d19067d9b189a446b", - "md5": "7064b8cfa7085a9892c47e8e98ac4f7a", - "size": "48536" - }, - "linux-sbsa": { - "relative_path": "cuda_nvtx/linux-sbsa/cuda_nvtx-linux-sbsa-12.6.77-archive.tar.xz", - "sha256": "498b10e525d4721c127ddefbd81fd97942c011cebf5fe5cbbd7f71814e038c56", - "md5": "d1050b70467d8fc6ab0cb09a678689b7", - "size": "49076" - }, - "windows-x86_64": { - "relative_path": "cuda_nvtx/windows-x86_64/cuda_nvtx-windows-x86_64-12.6.77-archive.zip", - "sha256": "d7726a135f5dc25b4222d684aac6ae98e4cc8a53a49a1235c7c6ca175f8e1e0a", - "md5": "a1c5ffd55791dc60212bb54ec55cdbe3", - "size": "65837" - }, - "linux-aarch64": { - "relative_path": "cuda_nvtx/linux-aarch64/cuda_nvtx-linux-aarch64-12.6.77-archive.tar.xz", - "sha256": "38cafee61f9cf1001e3c2b03de3c85fbea1b4c1b5d84f7fdac72ed7c98cb314a", - "md5": "d285db42a0e148ee2ded26bce43226c3", - "size": "51720" - } - }, - "cuda_nvvp": { - "name": "CUDA NVVP", - "license": "CUDA Toolkit", - "license_path": "cuda_nvvp/LICENSE.txt", - "version": "12.6.80", - "linux-x86_64": { - "relative_path": "cuda_nvvp/linux-x86_64/cuda_nvvp-linux-x86_64-12.6.80-archive.tar.xz", - "sha256": "ff2bd6d98af535e85d8e216b3110f13fb203130fbe132c0a49b02c872fbdbff5", - "md5": "327376a9f604166f5bf2f5175fbc7e7e", - "size": "117718608" - }, - "windows-x86_64": { - "relative_path": "cuda_nvvp/windows-x86_64/cuda_nvvp-windows-x86_64-12.6.80-archive.zip", - "sha256": "30145b90dd0439ca1389f2eed67d40fd03888798a7853ffac3d1221bd618e860", - "md5": "78dd9a6c2cc1965f5e549b60eb66bad1", - "size": "120343280" - } - }, - "cuda_opencl": { - "name": "CUDA OpenCL", - "license": "CUDA Toolkit", - "license_path": "cuda_opencl/LICENSE.txt", - "version": "12.6.77", - "linux-x86_64": { - "relative_path": "cuda_opencl/linux-x86_64/cuda_opencl-linux-x86_64-12.6.77-archive.tar.xz", - "sha256": "aaf8aedb89ad5e4a63ccf564a6ec1073ac17863a7ae2a211a5d8127b3e1d09f4", - "md5": "5344e01d090f74d5926c45723dabf849", - "size": "92304" - }, - "windows-x86_64": { - "relative_path": "cuda_opencl/windows-x86_64/cuda_opencl-windows-x86_64-12.6.77-archive.zip", - "sha256": "dc4555a8920f02d2142d266a8516b26cda7cc2609a20adedb8b268c2c4660971", - "md5": "f1f052a1c59d63d981ae47c055424313", - "size": "137329" - } - }, - "cuda_profiler_api": { - "name": "CUDA Profiler API", - "license": "CUDA Toolkit", - "license_path": "cuda_profiler_api/LICENSE.txt", - "version": "12.6.77", - "linux-x86_64": { - "relative_path": "cuda_profiler_api/linux-x86_64/cuda_profiler_api-linux-x86_64-12.6.77-archive.tar.xz", - "sha256": "0807ec2335ea358a96f984ce08cc0b3d572b5cc486f9082fc3afa9a1773f2a58", - "md5": "78ff6eced1fc8d26c94c41fb71af7d71", - "size": "16188" - }, - "linux-sbsa": { - "relative_path": "cuda_profiler_api/linux-sbsa/cuda_profiler_api-linux-sbsa-12.6.77-archive.tar.xz", - "sha256": "a8401ac3a2a4da39febd73b5008a2f0388f4facf52168b15dd5aefbed46bd481", - "md5": "fff49b98490bc2aa06b62bee1ba73f60", - "size": "16180" - }, - "windows-x86_64": { - "relative_path": "cuda_profiler_api/windows-x86_64/cuda_profiler_api-windows-x86_64-12.6.77-archive.zip", - "sha256": "d0ed383bac1a84d5c1e1d6a3076b80bc581e488093fef5655d55c39a198aea29", - "md5": "c66247462b1f770fc9d2798d911e54dd", - "size": "20222" - }, - "linux-aarch64": { - "relative_path": "cuda_profiler_api/linux-aarch64/cuda_profiler_api-linux-aarch64-12.6.77-archive.tar.xz", - "sha256": "f09f8b865fa07093400faaf1ca24133cf9014d5eebe9596d667687a9e16400f9", - "md5": "d0210f5dd6811fb211e2622fe3b7e340", - "size": "16176" - } - }, - "cuda_sanitizer_api": { - "name": "CUDA Compute Sanitizer API", - "license": "CUDA Toolkit", - "license_path": "cuda_sanitizer_api/LICENSE.txt", - "version": "12.6.77", - "linux-x86_64": { - "relative_path": "cuda_sanitizer_api/linux-x86_64/cuda_sanitizer_api-linux-x86_64-12.6.77-archive.tar.xz", - "sha256": "6839933c940fe023c133f0e2b41a5d02d4068359a1487ec8bcb82ea9a7237d57", - "md5": "d93bcc3e55e23e7c90adb583ca3f854b", - "size": "9756336" - }, - "linux-sbsa": { - "relative_path": "cuda_sanitizer_api/linux-sbsa/cuda_sanitizer_api-linux-sbsa-12.6.77-archive.tar.xz", - "sha256": "686eb7bf05f3dbff6700606f2d65aba7ad413843f7c62cccd7c1433399280c2f", - "md5": "b026e2007ae409af19ad6854dd6ea24a", - "size": "7252516" - }, - "windows-x86_64": { - "relative_path": "cuda_sanitizer_api/windows-x86_64/cuda_sanitizer_api-windows-x86_64-12.6.77-archive.zip", - "sha256": "b35cd306e7c61ca224eaf5a4d3a0ee2efb1c8fa6b0fb90f5e2315dcd4c356f71", - "md5": "8de75d81500d27768e039efd74fb582e", - "size": "9755411" - }, - "linux-aarch64": { - "relative_path": "cuda_sanitizer_api/linux-aarch64/cuda_sanitizer_api-linux-aarch64-12.6.77-archive.tar.xz", - "sha256": "dbf5b1c1cf51327c449123a80f5b4b3551da3470ff8aa8a11a11902887a6dff8", - "md5": "75cda6f13274536fb7f83190739ecd6d", - "size": "4105012" - } - }, - "fabricmanager": { - "name": "NVIDIA Fabric Manager", - "license": "NVIDIA Driver", - "license_path": "fabricmanager/LICENSE.txt", - "version": "560.35.03", - "linux-x86_64": { - "relative_path": "fabricmanager/linux-x86_64/fabricmanager-linux-x86_64-560.35.03-archive.tar.xz", - "sha256": "9fbb881948f524af417b46e9b17f2146d682d011ded3aa0e821309ed23df3aa5", - "md5": "a18c407a4379849b8ed0b38028797522", - "size": "6065756" - }, - "linux-sbsa": { - "relative_path": "fabricmanager/linux-sbsa/fabricmanager-linux-sbsa-560.35.03-archive.tar.xz", - "sha256": "e36fd543964eb3977d1086a59210b8a953afff03c1a9b91ccbe2681356bfa0bd", - "md5": "011d67ac6414fa3d2a4b04255d69a884", - "size": "5479452" - } - }, - "imex": { - "name": "Nvidia-Imex", - "license": "NVIDIA Proprietary", - "license_path": "imex/LICENSE.txt", - "version": "560.35.03", - "linux-x86_64": { - "relative_path": "imex/linux-x86_64/nvidia-imex-linux-x86_64-560.35.03-archive.tar.xz", - "sha256": "502f1579136741ea0e89e2b74e0303f671336e9fd9dafa6b0312a0f891a1559a", - "md5": "fd25e88f744d937e5d7e336dda15b633", - "size": "7419540" - }, - "linux-sbsa": { - "relative_path": "imex/linux-sbsa/nvidia-imex-linux-sbsa-560.35.03-archive.tar.xz", - "sha256": "5219e343fdcdef1be626a0f8632e3da23711cb63ef18b6b64cf3089a1b6f2025", - "md5": "01dfaec839ee13a2eafea256f2632fdb", - "size": "6617284" - } - }, - "libcublas": { - "name": "CUDA cuBLAS", - "license": "CUDA Toolkit", - "license_path": "libcublas/LICENSE.txt", - "version": "12.6.3.3", - "linux-x86_64": { - "relative_path": "libcublas/linux-x86_64/libcublas-linux-x86_64-12.6.3.3-archive.tar.xz", - "sha256": "82e62a45a622922615a5b0f49462bb47e434a307a6857a9c1505df607c1b80df", - "md5": "f71b5d92dde96bf80c2dc23979fbeab0", - "size": "522864584" - }, - "linux-sbsa": { - "relative_path": "libcublas/linux-sbsa/libcublas-linux-sbsa-12.6.3.3-archive.tar.xz", - "sha256": "1d184ca49c315ffef887e949edd3ac40fa14cf6b52c66ac96875b36009e81be1", - "md5": "47b56b30bc5a958ea1dfe6aae7ed7ca5", - "size": "521104044" - }, - "windows-x86_64": { - "relative_path": "libcublas/windows-x86_64/libcublas-windows-x86_64-12.6.3.3-archive.zip", - "sha256": "05ae7a77008ce0ad3be214d128d8bdb23b88387408e7bdf318a0f01c54841806", - "md5": "d95c91517f7e43b3547fd5bf31df5233", - "size": "428590136" - }, - "linux-aarch64": { - "relative_path": "libcublas/linux-aarch64/libcublas-linux-aarch64-12.6.3.3-archive.tar.xz", - "sha256": "fb9be60ac1a366cbc2c4675bf8eb4cd332a65f341caaa506952ef71aac80db31", - "md5": "0c264d005ab6f3d868717f5bad7f81aa", - "size": "186081352" - } - }, - "libcudla": { - "name": "cuDLA", - "license": "CUDA Toolkit", - "license_path": "libcudla/LICENSE.txt", - "version": "12.6.77", - "linux-aarch64": { - "relative_path": "libcudla/linux-aarch64/libcudla-linux-aarch64-12.6.77-archive.tar.xz", - "sha256": "d255c6b436e46d514af4d2899a5166b07204808eaafa6a37539819e360e9cfd1", - "md5": "f165e8fb65aa83476b4a3792d5a64eb8", - "size": "38464" - } - }, - "libcufft": { - "name": "CUDA cuFFT", - "license": "CUDA Toolkit", - "license_path": "libcufft/LICENSE.txt", - "version": "11.3.0.4", - "linux-x86_64": { - "relative_path": "libcufft/linux-x86_64/libcufft-linux-x86_64-11.3.0.4-archive.tar.xz", - "sha256": "63a046d51a45388e10612c3fd423bb7fa5127496aa9bb3951a609e8b9d996852", - "md5": "de19c7d0e525761fad7b3554ff5dda0c", - "size": "476376920" - }, - "linux-sbsa": { - "relative_path": "libcufft/linux-sbsa/libcufft-linux-sbsa-11.3.0.4-archive.tar.xz", - "sha256": "bce2ae627c9fee47d59850f622e28213aa7dc39fec6759505b9195c858f8701d", - "md5": "883ad3ffae02618cb111022028faae25", - "size": "476669696" - }, - "windows-x86_64": { - "relative_path": "libcufft/windows-x86_64/libcufft-windows-x86_64-11.3.0.4-archive.zip", - "sha256": "7b3c75e90a907b3a2d215f7c846d67b684c6ddb76369d1bced67d2cdd411014b", - "md5": "336ebef70375322fd258796f831b1c8d", - "size": "197843242" - }, - "linux-aarch64": { - "relative_path": "libcufft/linux-aarch64/libcufft-linux-aarch64-11.3.0.4-archive.tar.xz", - "sha256": "beafc176148563833a4271a6913bc318a004037a8b4f5b73904667a33fe6ecb1", - "md5": "2a101674bea1630906205187097dd408", - "size": "476526352" - } - }, - "libcufile": { - "name": "CUDA cuFile", - "license": "CUDA Toolkit", - "license_path": "libcufile/LICENSE.txt", - "version": "1.11.1.6", - "linux-x86_64": { - "relative_path": "libcufile/linux-x86_64/libcufile-linux-x86_64-1.11.1.6-archive.tar.xz", - "sha256": "99ee42e0ebf826f13cea9d15d63f54f225ddc42ff87be1427c58be9692ae25bc", - "md5": "1cb6fc8741517038ab956e6eb141a262", - "size": "41841440" - }, - "linux-sbsa": { - "relative_path": "libcufile/linux-sbsa/libcufile-linux-sbsa-1.11.1.6-archive.tar.xz", - "sha256": "fda8d0491f9e3311adb317841893a531737b82316187619e2116d86d7a84faaf", - "md5": "9a3dcdf43d42e78ac29d366c46c17764", - "size": "41290172" - }, - "linux-aarch64": { - "relative_path": "libcufile/linux-aarch64/libcufile-linux-aarch64-1.11.1.6-archive.tar.xz", - "sha256": "2d2e461830748ebb508be8cc93c804486618dbd6a53de13fb7976d71a481013a", - "md5": "aed61fb0c12b25f8dcb8c80c82c6923f", - "size": "41280828" - } - }, - "libcurand": { - "name": "CUDA cuRAND", - "license": "CUDA Toolkit", - "license_path": "libcurand/LICENSE.txt", - "version": "10.3.7.77", - "linux-x86_64": { - "relative_path": "libcurand/linux-x86_64/libcurand-linux-x86_64-10.3.7.77-archive.tar.xz", - "sha256": "981339cc86d7b8779e9a3c17e72d8c5e1a8a2d06c24db692eecabed8e746a3c7", - "md5": "d4b265d626d59f852a41a73b1fea9dc9", - "size": "81729748" - }, - "linux-sbsa": { - "relative_path": "libcurand/linux-sbsa/libcurand-linux-sbsa-10.3.7.77-archive.tar.xz", - "sha256": "a189b8456e86a5f53c73f7fd392bf158d81d72a4e94f6853d2056cd67454e858", - "md5": "3f815e6953516eaa3f51b47fea151537", - "size": "81711072" - }, - "windows-x86_64": { - "relative_path": "libcurand/windows-x86_64/libcurand-windows-x86_64-10.3.7.77-archive.zip", - "sha256": "f4554a9bbc933118ff7cde51db2664e17b23ffb5d32715157bee4564bba8bfd7", - "md5": "245d5b349e7b7a5deea05e032a7d62ac", - "size": "55075353" - }, - "linux-aarch64": { - "relative_path": "libcurand/linux-aarch64/libcurand-linux-aarch64-10.3.7.77-archive.tar.xz", - "sha256": "eab273ca1d30659f026600d15cd0492d511408ae2141349cdf2edadd3da2661d", - "md5": "8739d2194ad5d16db1b90d9210dd51cc", - "size": "83949508" - } - }, - "libcusolver": { - "name": "CUDA cuSOLVER", - "license": "CUDA Toolkit", - "license_path": "libcusolver/LICENSE.txt", - "version": "11.7.1.2", - "linux-x86_64": { - "relative_path": "libcusolver/linux-x86_64/libcusolver-linux-x86_64-11.7.1.2-archive.tar.xz", - "sha256": "7cf794bf21c4947d0922e378984fa7939f0811bfc355121be1a0d44cd8695c09", - "md5": "0de2d9376f0ada4792dc931a1313ff60", - "size": "159512244" - }, - "linux-sbsa": { - "relative_path": "libcusolver/linux-sbsa/libcusolver-linux-sbsa-11.7.1.2-archive.tar.xz", - "sha256": "96bdff422ac078049e70442404a5f4027c64d58881efdfd47eb32d3bcfdd3b44", - "md5": "f055a72d4f87ecd4c624110713bd286f", - "size": "158302060" - }, - "windows-x86_64": { - "relative_path": "libcusolver/windows-x86_64/libcusolver-windows-x86_64-11.7.1.2-archive.zip", - "sha256": "9999cbbf5d3789bb766ed3336dfb8d9be02832ffc5f36b1c37b9aa099ab47604", - "md5": "14cccf5792c2f8ea6dc5a6accb421d3c", - "size": "147388290" - }, - "linux-aarch64": { - "relative_path": "libcusolver/linux-aarch64/libcusolver-linux-aarch64-11.7.1.2-archive.tar.xz", - "sha256": "d7e578143ebd90582873cb4c84f0f29ae1b3f086104a7660f5a63a28a115b9e6", - "md5": "9aed9bb2e94efb2aa03c8db542003adb", - "size": "174581012" - } - }, - "libcusparse": { - "name": "CUDA cuSPARSE", - "license": "CUDA Toolkit", - "license_path": "libcusparse/LICENSE.txt", - "version": "12.5.4.2", - "linux-x86_64": { - "relative_path": "libcusparse/linux-x86_64/libcusparse-linux-x86_64-12.5.4.2-archive.tar.xz", - "sha256": "a9ad797756a7bb9a92bf8a63768dec837af38064ee8be3c32e21382e03604bd8", - "md5": "e7e5312dbf7d472fec153dfead67c876", - "size": "236134716" - }, - "linux-sbsa": { - "relative_path": "libcusparse/linux-sbsa/libcusparse-linux-sbsa-12.5.4.2-archive.tar.xz", - "sha256": "27f5743ca677a8b0144a824f95f6674f39f3effdac50fdd15e050a9a87069d8b", - "md5": "ef319172b2adaace463a08db5a63e209", - "size": "235710908" - }, - "windows-x86_64": { - "relative_path": "libcusparse/windows-x86_64/libcusparse-windows-x86_64-12.5.4.2-archive.zip", - "sha256": "ab409eee74ab63d5f2e78473d5901c0ce4d414161a1b30838d4e0dc857b2452f", - "md5": "2978fdfe0029c439441ae6e574fc1edf", - "size": "211361778" - }, - "linux-aarch64": { - "relative_path": "libcusparse/linux-aarch64/libcusparse-linux-aarch64-12.5.4.2-archive.tar.xz", - "sha256": "fc790c3d0d194fc96212bde94c8a22016e63a07d048ac6e9e49fdc8cf2a2dfcc", - "md5": "4ab706313af2f1a77f0ee89796fe0f3e", - "size": "253390908" - } - }, - "libnpp": { - "name": "CUDA NPP", - "license": "CUDA Toolkit", - "license_path": "libnpp/LICENSE.txt", - "version": "12.3.1.54", - "linux-x86_64": { - "relative_path": "libnpp/linux-x86_64/libnpp-linux-x86_64-12.3.1.54-archive.tar.xz", - "sha256": "0a7f5bb81e65e60775f831990a567394682a209694b396a4e9233a6c91db65e3", - "md5": "f1a61e38a71c751a7c904a66362ccfea", - "size": "182688996" - }, - "linux-sbsa": { - "relative_path": "libnpp/linux-sbsa/libnpp-linux-sbsa-12.3.1.54-archive.tar.xz", - "sha256": "0901bdf3abc7c3d4143acebb058ba5f95c1dfd092f25ce0c52d6b6974585a564", - "md5": "3fbbf44ca4a36f6040ffe3b77fa82534", - "size": "182191412" - }, - "windows-x86_64": { - "relative_path": "libnpp/windows-x86_64/libnpp-windows-x86_64-12.3.1.54-archive.zip", - "sha256": "9a5271ce10478945eaa730360226e0c6f408425bb5689d5cf48bb4ca8085a008", - "md5": "4c9546d1e16b00cd400871dcfc17a537", - "size": "154933759" - }, - "linux-aarch64": { - "relative_path": "libnpp/linux-aarch64/libnpp-linux-aarch64-12.3.1.54-archive.tar.xz", - "sha256": "eb0473643b991be77b64b31f5e63437203e1167bbc9274d7f6a6ad5114d048fd", - "md5": "40c771b024287b7509ada380aad87a15", - "size": "200333628" - } - }, - "libnvfatbin": { - "name": "NVIDIA compiler library for fatbin interaction", - "license": "CUDA Toolkit", - "license_path": "libnvfatbin/LICENSE.txt", - "version": "12.6.77", - "linux-x86_64": { - "relative_path": "libnvfatbin/linux-x86_64/libnvfatbin-linux-x86_64-12.6.77-archive.tar.xz", - "sha256": "a4046f5337656af57bb3861dd80207143dc928c3ff9a12670fb982f1e3e17ee5", - "md5": "b0385eef134ed932f3295ed9e13e9839", - "size": "909740" - }, - "linux-sbsa": { - "relative_path": "libnvfatbin/linux-sbsa/libnvfatbin-linux-sbsa-12.6.77-archive.tar.xz", - "sha256": "0311c19b6e08b173d915fccfe70a9250b7cac4e456878a0e3bedfc81d878cf2f", - "md5": "0fa0ed520efcb7e4228b7f91b2f8fe0a", - "size": "818636" - }, - "windows-x86_64": { - "relative_path": "libnvfatbin/windows-x86_64/libnvfatbin-windows-x86_64-12.6.77-archive.zip", - "sha256": "d54d37d51301f3d478e0f148e1755ab357905876d75e675aa3168c815bf2ed1d", - "md5": "d421dfd34e0d2f70613fc9c6b4b66e87", - "size": "2181182" - }, - "linux-aarch64": { - "relative_path": "libnvfatbin/linux-aarch64/libnvfatbin-linux-aarch64-12.6.77-archive.tar.xz", - "sha256": "4d6c69acf31fee099d6065cb4c6fee2fe9bff9299c55e0a957c567377d1c2d64", - "md5": "7bd654173af88b4dd0acc38eb21d5929", - "size": "792100" - } - }, - "libnvidia_nscq": { - "name": "NVIDIA NSCQ API", - "license": "NVIDIA Driver", - "license_path": "libnvidia_nscq/LICENSE.txt", - "version": "560.35.03", - "linux-x86_64": { - "relative_path": "libnvidia_nscq/linux-x86_64/libnvidia_nscq-linux-x86_64-560.35.03-archive.tar.xz", - "sha256": "66349840f41caeb16b1b77f1ac4956abe787bf1bbd54ad19a30e7d904f06c942", - "md5": "f593eed9bea4e09980969a25525c6e03", - "size": "356820" - }, - "linux-sbsa": { - "relative_path": "libnvidia_nscq/linux-sbsa/libnvidia_nscq-linux-sbsa-560.35.03-archive.tar.xz", - "sha256": "5b851b42776367500c58b6481a0599daa7fd79f174f60dcc03630920c058eccf", - "md5": "038cb3140d08fb683cb51f0a02a98d80", - "size": "350984" - } - }, - "libnvjitlink": { - "name": "NVIDIA compiler library for JIT LTO functionality", - "license": "CUDA Toolkit", - "license_path": "libnvjitlink/LICENSE.txt", - "version": "12.6.77", - "linux-x86_64": { - "relative_path": "libnvjitlink/linux-x86_64/libnvjitlink-linux-x86_64-12.6.77-archive.tar.xz", - "sha256": "2f6930cc8e2a022723c5a787784b52292e9334fb1487cf743b90b82edea12b55", - "md5": "2b5cf637cc9013f79c31f8c93f689384", - "size": "28954888" - }, - "linux-sbsa": { - "relative_path": "libnvjitlink/linux-sbsa/libnvjitlink-linux-sbsa-12.6.77-archive.tar.xz", - "sha256": "cb8dc3244b2f501d3b7d4097682ac6a10d1c8ad81611c47e86cb9e6526e4a39d", - "md5": "c238a725564e2611e9df2644fbf0c58b", - "size": "26745764" - }, - "windows-x86_64": { - "relative_path": "libnvjitlink/windows-x86_64/libnvjitlink-windows-x86_64-12.6.77-archive.zip", - "sha256": "58f5819c82a98d7c5418a1e91a622c397f9a30eb99805a2af2571986c614e2c0", - "md5": "1bff4308db18238be5359dd1394093f2", - "size": "153330057" - }, - "linux-aarch64": { - "relative_path": "libnvjitlink/linux-aarch64/libnvjitlink-linux-aarch64-12.6.77-archive.tar.xz", - "sha256": "f21bd06242ed75345ee24d3f567c1fdc4bb422a8179ed890ec0d12cf801d0b1c", - "md5": "94d46bfcafc4f7a77d1980f7ad3253fa", - "size": "27883404" - } - }, - "libnvjpeg": { - "name": "CUDA nvJPEG", - "license": "CUDA Toolkit", - "license_path": "libnvjpeg/LICENSE.txt", - "version": "12.3.3.54", - "linux-x86_64": { - "relative_path": "libnvjpeg/linux-x86_64/libnvjpeg-linux-x86_64-12.3.3.54-archive.tar.xz", - "sha256": "f74f6aee35ef4fb1619bc2a67be943c3d1a84d6746b41788c44a7d3b0d9ae3b4", - "md5": "e21f0dc4f94bc875ef99e1afbfd85bbe", - "size": "2579560" - }, - "linux-sbsa": { - "relative_path": "libnvjpeg/linux-sbsa/libnvjpeg-linux-sbsa-12.3.3.54-archive.tar.xz", - "sha256": "6970db1c9757f3ae30adba21d7c1306fd5afc4cb034b0739c711691a481d3ed4", - "md5": "01811faaa7d3de0353169948247aeb02", - "size": "2418632" - }, - "windows-x86_64": { - "relative_path": "libnvjpeg/windows-x86_64/libnvjpeg-windows-x86_64-12.3.3.54-archive.zip", - "sha256": "15c1b28e623598eb75f48d02bfd34efa83b35e3f8d1d5241c93a385d0bea9d18", - "md5": "63f0c5626a07b2850e6eadb704685d69", - "size": "2829340" - }, - "linux-aarch64": { - "relative_path": "libnvjpeg/linux-aarch64/libnvjpeg-linux-aarch64-12.3.3.54-archive.tar.xz", - "sha256": "e81f56682cb4d6e837d5527aaf1acd26ab8eace470b5e7a32f9a8d685e7ef2bc", - "md5": "6e90b73e124be49f09b9fe517dff5bb6", - "size": "2562184" - } - }, - "libnvsdm": { - "name": "LIBNVSDM", - "license": "NVIDIA", - "license_path": "libnvsdm/LICENSE.txt", - "version": "560.35.03", - "linux-x86_64": { - "relative_path": "libnvsdm/linux-x86_64/libnvsdm-linux-x86_64-560.35.03-archive.tar.xz", - "sha256": "807f63491c351fb459876f39ee4455d6fb81b5cc6b17100e04c6f0c7f5a24970", - "md5": "bd9facf361fc729bf3676200b3223023", - "size": "550380" - } - }, - "nsight_compute": { - "name": "Nsight Compute", - "license": "NVIDIA SLA", - "license_path": "nsight_compute/LICENSE.txt", - "version": "2024.3.2.3", - "linux-x86_64": { - "relative_path": "nsight_compute/linux-x86_64/nsight_compute-linux-x86_64-2024.3.2.3-archive.tar.xz", - "sha256": "c37b4e45582a5eef39f5f3b4aee89e8805eac5ce236c960d7a42b97106ceb4d9", - "md5": "a14bdec76126b4a642aa636eca3e3f76", - "size": "429295908" - }, - "linux-sbsa": { - "relative_path": "nsight_compute/linux-sbsa/nsight_compute-linux-sbsa-2024.3.2.3-archive.tar.xz", - "sha256": "213e080e2b7ccbbc11ae55dffd316cf24d6e46811b23a2b3596190b4cd12f919", - "md5": "9c9abcc88a967d7fab4e9503bf6953b0", - "size": "218632392" - }, - "windows-x86_64": { - "relative_path": "nsight_compute/windows-x86_64/nsight_compute-windows-x86_64-2024.3.2.3-archive.zip", - "sha256": "7a7d8b33aa928f5e1fe130e239ca869ff019e4e0e2103d6f0b8bbdecfa642987", - "md5": "2500473f4975306dd0f7ee06ab8e49b4", - "size": "462207543" - }, - "linux-aarch64": { - "relative_path": "nsight_compute/linux-aarch64/nsight_compute-linux-aarch64-2024.3.2.3-archive.tar.xz", - "sha256": "ae5d005c398a97f410b57ddc9eb12b5bf74502e861678b9067959bf2d98cfcff", - "md5": "94c85a947f3f8c00b95c34766297ae9c", - "size": "429331864" - } - }, - "nsight_systems": { - "name": "Nsight Systems", - "license": "NVIDIA SLA", - "license_path": "nsight_systems/LICENSE.txt", - "version": "2024.5.1.113", - "linux-x86_64": { - "relative_path": "nsight_systems/linux-x86_64/nsight_systems-linux-x86_64-2024.5.1.113-archive.tar.xz", - "sha256": "b34b829866bcd2c38432ea9c3f4f2feaca52dead50391e3f3bcce654fafdcc2b", - "md5": "3579478a0393dbfdc4d0b854f4a55212", - "size": "254397476" - }, - "linux-sbsa": { - "relative_path": "nsight_systems/linux-sbsa/nsight_systems-linux-sbsa-2024.5.1.113-archive.tar.xz", - "sha256": "c19dc46fc7846abd8dfb4058bb9877232937238682c5d212f61adb39ca2eaadc", - "md5": "a800f94276f04817cdce5c7fbeafa61a", - "size": "215840944" - }, - "windows-x86_64": { - "relative_path": "nsight_systems/windows-x86_64/nsight_systems-windows-x86_64-2024.5.1.113-archive.zip", - "sha256": "5bc27dba701c9dadc0659f7a471866d6d7af717db0c741bba87fb9ec18bd130e", - "md5": "ab5730ccd853fbfe0bca3ab0d3cbed52", - "size": "350421706" - } - }, - "nsight_vse": { - "name": "Nsight Visual Studio Edition (VSE)", - "license": "NVIDIA SLA", - "license_path": "nsight_vse/LICENSE.txt", - "version": "2024.3.0.24164", - "windows-x86_64": { - "relative_path": "nsight_vse/windows-x86_64/nsight_vse-windows-x86_64-2024.3.0.24164-archive.zip", - "sha256": "f06b1b9d9e4bcdf6cb44a85c506ecbf55482c4557df02c1906c1d469986b371a", - "md5": "e86b066ca8aa0922c2db274757e40dad", - "size": "471792358" - } - }, - "nvidia_driver": { - "name": "NVIDIA Linux Driver", - "license": "NVIDIA Driver", - "license_path": "nvidia_driver/LICENSE.txt", - "version": "560.35.03", - "linux-x86_64": { - "relative_path": "nvidia_driver/linux-x86_64/nvidia_driver-linux-x86_64-560.35.03-archive.tar.xz", - "sha256": "301e7bfb7868eeae74b076068f0ebc00b3732d718084a78261f9c862975d1058", - "md5": "74e3f5e5cb10326b3236cb3b5d1fb731", - "size": "374384124" - }, - "linux-sbsa": { - "relative_path": "nvidia_driver/linux-sbsa/nvidia_driver-linux-sbsa-560.35.03-archive.tar.xz", - "sha256": "6c41cc106562ba8969e00785eff1f91be2c0eb8bd3ee0fde78fa32b3dfc45a03", - "md5": "7a01f16712f0ac284db1addb86d95a3e", - "size": "289359252" - } - }, - "nvidia_fs": { - "name": "NVIDIA filesystem", - "license": "CUDA Toolkit", - "license_path": "nvidia_fs/LICENSE.txt", - "version": "2.22.3", - "linux-x86_64": { - "relative_path": "nvidia_fs/linux-x86_64/nvidia_fs-linux-x86_64-2.22.3-archive.tar.xz", - "sha256": "67ac2e05a6388db2824c454e0bdd808d662aa1fe63ad3292471b144fb0c7fad1", - "md5": "605ff67650a8ae1f72a9bd8540ed0ee0", - "size": "59284" - }, - "linux-sbsa": { - "relative_path": "nvidia_fs/linux-sbsa/nvidia_fs-linux-sbsa-2.22.3-archive.tar.xz", - "sha256": "1c73cfb016ce4c6b0c99d46084a678a5e062875e1180b556bda624f7648653f6", - "md5": "8debe062baaa33cd4283237d112633c4", - "size": "59284" - }, - "linux-aarch64": { - "relative_path": "nvidia_fs/linux-aarch64/nvidia_fs-linux-aarch64-2.22.3-archive.tar.xz", - "sha256": "7d8db9cf65b6c29198a6c29749ffb5aea3367c1adee71c6bd10169fcaca488db", - "md5": "076dbd175b9dbacd9a199f29069161e5", - "size": "59288" - } - }, - "visual_studio_integration": { - "name": "CUDA Visual Studio Integration", - "license": "CUDA Toolkit", - "license_path": "visual_studio_integration/LICENSE.txt", - "version": "12.6.77", - "windows-x86_64": { - "relative_path": "visual_studio_integration/windows-x86_64/visual_studio_integration-windows-x86_64-12.6.77-archive.zip", - "sha256": "79345a5f54a2dcd72b8324586a22a3545d948567fa3fd144038a6261866818a3", - "md5": "f59a994431512c0040cd381a66287086", - "size": "533182" - } - } -} \ No newline at end of file diff --git a/runtimes/cuda/cudnn.redistrib_9.4.0.json b/runtimes/cuda/cudnn.redistrib_9.4.0.json deleted file mode 100644 index 68e13181..00000000 --- a/runtimes/cuda/cudnn.redistrib_9.4.0.json +++ /dev/null @@ -1,77 +0,0 @@ -{ - "release_date": "2024-09-05", - "release_label": "9.4.0", - "release_product": "cudnn", - "cudnn": { - "name": "NVIDIA CUDA Deep Neural Network library", - "license": "cudnn", - "license_path": "cudnn/LICENSE.txt", - "version": "9.4.0.58", - "linux-x86_64": { - "cuda11": { - "relative_path": "cudnn/linux-x86_64/cudnn-linux-x86_64-9.4.0.58_cuda11-archive.tar.xz", - "sha256": "34a3c798764b8fc10c7145065798fb530d62073e88a3b0cfdcec458e2ace4467", - "md5": "cabfb7d1acee9dbe1eef8e5960e7b207", - "size": "737638524" - }, - "cuda12": { - "relative_path": "cudnn/linux-x86_64/cudnn-linux-x86_64-9.4.0.58_cuda12-archive.tar.xz", - "sha256": "1ae128159a8c01b1fd8709050a30813ed9a4dd95824c0d3ddf841c52c8e25218", - "md5": "6f3dc24117b9ba640f3a8b6582dd0f2c", - "size": "745465960" - } - }, - "cuda_variant": [ - "11", - "12" - ], - "linux-sbsa": { - "cuda11": { - "relative_path": "cudnn/linux-sbsa/cudnn-linux-sbsa-9.4.0.58_cuda11-archive.tar.xz", - "sha256": "aa39d5a1d52fbf8c92a5a0c0cdb9b8d2772f99be15ee91c1a22932ac81d0f0dd", - "md5": "28be699ed21c03e05ef8dbece0bff43f", - "size": "736652544" - }, - "cuda12": { - "relative_path": "cudnn/linux-sbsa/cudnn-linux-sbsa-9.4.0.58_cuda12-archive.tar.xz", - "sha256": "07c1b686b3dfa3a976dfc03e886fe44190c49c55f05795ea912d49518bf7d0cd", - "md5": "6346daaf3e4e730e2e710ce6080b7b73", - "size": "743942524" - } - }, - "windows-x86_64": { - "cuda11": { - "relative_path": "cudnn/windows-x86_64/cudnn-windows-x86_64-9.4.0.58_cuda11-archive.zip", - "sha256": "2f5389d9769fc7b13fcc08ae050f8f911506e76a3b785abcc53cde97d4b3a5a5", - "md5": "2b32d2393d92b95fd172913b972ab590", - "size": "557332411" - }, - "cuda12": { - "relative_path": "cudnn/windows-x86_64/cudnn-windows-x86_64-9.4.0.58_cuda12-archive.zip", - "sha256": "c7c642c8db950d32c283acbb6a3d2aaeb484ae3fcef2eb7fee5523b72303a452", - "md5": "4783043b45542528c4af17af659fe26d", - "size": "560885858" - } - }, - "linux-aarch64": { - "cuda12": { - "relative_path": "cudnn/linux-aarch64/cudnn-linux-aarch64-9.4.0.58_cuda12-archive.tar.xz", - "sha256": "5914e8845e5dcf41fd8963483f3d54db1f04f2c0be725ed012dde27b9fca2488", - "md5": "96678293921e3b9a35f059cd4f7401d2", - "size": "787931272" - } - } - }, - "cudnn_samples": { - "name": "NVIDIA cuDNN samples", - "license": "cudnn", - "license_path": "cudnn_samples/LICENSE.txt", - "version": "9.4.0.58", - "source": { - "relative_path": "cudnn_samples/source/cudnn_samples-source-9.4.0.58-archive.tar.xz", - "sha256": "623f7351b51ca8799e18026341f04bf20fc74d283a2f301763906a7bd12af735", - "md5": "f73b4203e9fff4d2df002bb1886f401f", - "size": "1664768" - } - } -} \ No newline at end of file diff --git a/runtimes/cuda/libpjrt_cuda.BUILD.bazel b/runtimes/cuda/libpjrt_cuda.BUILD.bazel deleted file mode 100644 index 6b609a31..00000000 --- a/runtimes/cuda/libpjrt_cuda.BUILD.bazel +++ /dev/null @@ -1,32 +0,0 @@ -load("@aspect_bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") -load("@zml//bazel:cc_import.bzl", "cc_import") - -copy_to_directory( - name = "sandbox", - srcs = [ - "@cuda_nvcc//:libdevice", - "@cuda_nvcc//:ptxas", - ], - include_external_repositories = ["**"], -) - -cc_import( - name = "libpjrt_cuda", - data = [":sandbox"], - shared_library = "libpjrt_cuda.so", - visibility = ["@zml//runtimes/cuda:__subpackages__"], - deps = [ - "@cuda_cudart//:cudart", - "@cuda_cupti//:cupti", - "@cuda_nvcc//:nvvm", - "@cuda_nvrtc//:nvrtc", - "@cudnn//:cudnn", - "@libcublas//:cublas", - "@libcufft//:cufft", - "@libcusolver//:cusolver", - "@libcusparse//:cusparse", - "@libnccl//:nccl", - "@libnvjitlink//:nvjitlink", - "@zlib", - ], -) diff --git a/runtimes/rocm/BUILD.bazel b/runtimes/rocm/BUILD.bazel deleted file mode 100644 index a75fc83c..00000000 --- a/runtimes/rocm/BUILD.bazel +++ /dev/null @@ -1,21 +0,0 @@ -filegroup( - name = "zmlrocmhooks_srcs", - srcs = ["zmlrocmhooks.cc"], - visibility = ["@libpjrt_rocm//:__subpackages__"], -) - -alias( - name = "hipblaslt", - actual = "@libpjrt_rocm//:hipblaslt", -) - -alias( - name = "gfx", - actual = "@libpjrt_rocm//:gfx", -) - -alias( - name = "rocm", - actual = "@libpjrt_rocm", - visibility = ["//visibility:public"], -) diff --git a/runtimes/rocm/gfx.bzl b/runtimes/rocm/gfx.bzl deleted file mode 100644 index d63095cc..00000000 --- a/runtimes/rocm/gfx.bzl +++ /dev/null @@ -1,49 +0,0 @@ -load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") - -_ALL_GFX = ["gfx900", "gfx906", "gfx908", "gfx90a", "gfx940", "gfx941", "gfx942", "gfx1010", "gfx1012", "gfx1030", "gfx1100", "gfx1101", "gfx1102"] - -def _compute_enabled_gfx(values): - ret = {} - for v in values: - if (v == "all"): - ret = {gfx: True for gfx in _ALL_GFX} - elif (v == "none"): - ret = {} - else: - ret[v] = True - return ret - -def _gfx_from_file(file): - return file.basename[:-len(file.extension) - 1].rpartition("_")[-1].partition("-")[0] - -def _is_file_enabled(file, enabled_gfx): - gfx = _gfx_from_file(file) - return gfx in enabled_gfx or gfx == "fallback" - -def _bytecode_select_impl(ctx): - enabled_gfx = _compute_enabled_gfx(ctx.attr.enabled_gfx[BuildSettingInfo].value) - return [ - DefaultInfo( - files = depset([ - file - for file in ctx.files.bytecodes - if _is_file_enabled(file, enabled_gfx) - ]), - ), - ] - -bytecode_select = rule( - implementation = _bytecode_select_impl, - attrs = { - "bytecodes": attr.label_list(allow_files = True), - "enabled_gfx": attr.label(mandatory = True), - }, -) - - -def if_gfx(gfx, value): - return select({ - "@zml//runtimes/rocm:_{}".format(gfx): value, - "//conditions:default": [], - }) - diff --git a/runtimes/rocm/libpjrt_rocm.BUILD.bazel b/runtimes/rocm/libpjrt_rocm.BUILD.bazel deleted file mode 100644 index e6e333ad..00000000 --- a/runtimes/rocm/libpjrt_rocm.BUILD.bazel +++ /dev/null @@ -1,88 +0,0 @@ -load("@aspect_bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") -load("@bazel_skylib//rules:common_settings.bzl", "bool_flag", "string_list_flag") -load("@zml//bazel:cc_import.bzl", "cc_import") - -string_list_flag( - name = "gfx", - build_setting_default = ["all"], - visibility = [ - "@rocblas//:__subpackages__", - "@hipblaslt-dev//:__subpackages__", - ], -) - -bool_flag( - name = "hipblaslt", - build_setting_default = True, -) - -config_setting( - name = "_hipblaslt", - flag_values = {":hipblaslt": "True"}, -) - -copy_to_directory( - name = "sandbox", - srcs = [ - "@rocm-device-libs//:runfiles", - "@rocm-llvm//:lld", - ], - include_external_repositories = ["*"], -) - -cc_library( - name = "zmlrocmhooks_lib", - data = ["@rocblas//:runfiles"], - srcs = ["@zml//runtimes/rocm:zmlrocmhooks_srcs"], - linkopts = [ - "-lc", - "-ldl", - ], - deps = ["@bazel_tools//tools/cpp/runfiles"], -) - -cc_shared_library( - name = "zmlrocmhooks_so", - shared_lib_name = "libzmlrocmhooks.so.0", - deps = [":zmlrocmhooks_lib"], -) - -cc_import( - name = "zmlrocmhooks", - shared_library = ":zmlrocmhooks_so", -) - -cc_import( - name = "libpjrt_rocm", - data = [ - ":sandbox", - "@rocblas//:runfiles", - ] + select({ - ":_hipblaslt": ["@hipblaslt-dev//:runfiles"], - "//conditions:default": [], - }), - shared_library = "libpjrt_rocm.so", - visibility = ["//visibility:public"], - deps = [ - ":zmlrocmhooks", - "@comgr//:amd_comgr", - "@hip-runtime-amd//:amdhip", - "@hipblaslt", - "@hsa-amd-aqlprofile//:hsa-amd-aqlprofile", - "@hsa-rocr//:hsa-runtime", - "@miopen-hip//:MIOpen", - "@rccl", - "@rocblas", - "@rocm-core", - "@rocm-smi-lib//:rocm_smi", - "@rocprofiler-register", - "@roctracer", - "@libelf", - "@libdrm", - "@libnuma", - "@libzstd", - "@libdrm-amdgpu", - "@libtinfo", - "@zlib1g", - ], -) diff --git a/runtimes/rocm/packages.amd64.txt b/runtimes/rocm/packages.amd64.txt deleted file mode 100644 index 09ab6a1d..00000000 --- a/runtimes/rocm/packages.amd64.txt +++ /dev/null @@ -1,11376 +0,0 @@ -Package: amd-smi-lib -Architecture: amd64 -Depends: sudo, python3 (>= 3.6.8), python3-pip, rocm-core -Priority: optional -Section: devel -Filename: pool/main/a/amd-smi-lib/amd-smi-lib_24.6.3.60202-116~20.04_amd64.deb -Size: 1326346 -SHA256: 72004d9a2fcc6657b8dd1f591514f68e987ea946ab6f3ba7b7c88be5947fda98 -SHA1: bf9a6242eb4f66f720ba752a64b52294aea3f692 -MD5sum: 9895ce249a7ae883888e2261523d4270 -Description: AMD System Management libraries -Maintainer: AMD-SMILib Support -Provides: amd-smi -Recommends: python3-argcomplete, libdrm-dev, python3-yaml -Version: 24.6.3.60202-116~20.04 -Installed-Size: 7700 - -Package: amd-smi-lib-asan -Architecture: amd64 -Depends: sudo, python3 (>= 3.6.8), python3-pip, rocm-core-asan -Priority: optional -Section: devel -Filename: pool/main/a/amd-smi-lib-asan/amd-smi-lib-asan_24.6.3.60202-116~20.04_amd64.deb -Size: 1169710 -SHA256: e03a40470c46dfe618164d8754d9b84aa10527b500474f9debd4f414c1961386 -SHA1: 29a8bf6ad757ede46e432182ab90a8ce4ffc79cc -MD5sum: bbc0555728c61343a794481f520814d1 -Description: AMD System Management libraries -Maintainer: AMD-SMILib Support -Provides: amd-smi-lib-asan -Recommends: python3-argcomplete, libdrm-dev, python3-yaml -Version: 24.6.3.60202-116~20.04 -Installed-Size: 11612 - -Package: amd-smi-lib-asan-dbgsym -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 4d454b877efbaf1644eb7d6ca5044a05c13c32e7 -Depends: amd-smi-lib-asan (= 24.6.3.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/a/amd-smi-lib-asan-dbgsym/amd-smi-lib-asan-dbgsym_24.6.3.60202-116~20.04_amd64.deb -Size: 5753582 -SHA256: 1a8eb46ba6af20bce516683676ef8ad1631264738c7198d51592d9cb0b842bad -SHA1: c5a0d77d678ecf549c5f33c6b888094c7a00b396 -MD5sum: 53cde186cb7717e1284bab4d3d7ce563 -Description: debug symbols for amd-smi-lib-asan -Maintainer: AMD-SMILib Support -Package-Type: ddeb -Version: 24.6.3.60202-116~20.04 -Installed-Size: 18280 - -Package: amd-smi-lib-asan-dbgsym-rpath6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 4d454b877efbaf1644eb7d6ca5044a05c13c32e7 -Depends: amd-smi-lib-asan-rpath6.2.2 (= 24.6.3.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/a/amd-smi-lib-asan-dbgsym-rpath6.2.2/amd-smi-lib-asan-dbgsym-rpath6.2.2_24.6.3.60202-116~20.04_amd64.deb -Size: 3472940 -SHA256: 9d8c53e52b891ee36e83ae623a15c88975f49201f9c0399aeb2b8b934f124ec5 -SHA1: 067559535546fb8c81e0f89e5b6a113fbb7cefa2 -MD5sum: cccc710c580ee6e1d2d4b526fe75def5 -Description: debug symbols for amd-smi-lib-asan -Maintainer: AMD-SMILib Support -Package-Type: ddeb -Version: 24.6.3.60202-116~20.04 -Installed-Size: 18280 - -Package: amd-smi-lib-asan-dbgsym6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 4d454b877efbaf1644eb7d6ca5044a05c13c32e7 -Depends: amd-smi-lib-asan6.2.2 (= 24.6.3.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/a/amd-smi-lib-asan-dbgsym6.2.2/amd-smi-lib-asan-dbgsym6.2.2_24.6.3.60202-116~20.04_amd64.deb -Size: 3473844 -SHA256: d7f03d9a151e5eb17f6064f4fac4d31ae7bf6014f1f3aa97f4dec5578ba86b07 -SHA1: cd36cd47a86cd2d8341cb350608c1c1fa909b738 -MD5sum: a3655d25a3e40528ae2871cf634868e2 -Description: debug symbols for amd-smi-lib-asan -Maintainer: AMD-SMILib Support -Package-Type: ddeb -Version: 24.6.3.60202-116~20.04 -Installed-Size: 18280 - -Package: amd-smi-lib-asan-rpath6.2.2 -Architecture: amd64 -Depends: sudo, python3 (>= 3.6.8), python3-pip, rocm-core-asan-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/a/amd-smi-lib-asan-rpath6.2.2/amd-smi-lib-asan-rpath6.2.2_24.6.3.60202-116~20.04_amd64.deb -Size: 794032 -SHA256: 5bcb46eb16bb03140490e8c92f9c5af210d2c160a49beae5d2215bb3a97dbfab -SHA1: 4b310f60ee013152e936f484a39e0c4e28a0bbbb -MD5sum: 23ad65f7cb47c7621f917bff307666ce -Description: AMD System Management libraries -Maintainer: AMD-SMILib Support -Provides: amd-smi-lib-asan-rpath6.2.2 -Recommends: python3-argcomplete, libdrm-dev, python3-yaml -Version: 24.6.3.60202-116~20.04 -Installed-Size: 11612 - -Package: amd-smi-lib-asan6.2.2 -Architecture: amd64 -Depends: sudo, python3 (>= 3.6.8), python3-pip, rocm-core-asan6.2.2 -Priority: optional -Section: devel -Filename: pool/main/a/amd-smi-lib-asan6.2.2/amd-smi-lib-asan6.2.2_24.6.3.60202-116~20.04_amd64.deb -Size: 794936 -SHA256: b504d897fa239bc53a0fc541ec3ae4c720dbd1206c56b75408f32e8b14debe4c -SHA1: 4cb1d056dd3c884f4189b79fbe08a151b9980dd3 -MD5sum: 39603f41cd6e208840627c7d6f7f3ffa -Description: AMD System Management libraries -Maintainer: AMD-SMILib Support -Provides: amd-smi-lib-asan6.2.2 -Recommends: python3-argcomplete, libdrm-dev, python3-yaml -Version: 24.6.3.60202-116~20.04 -Installed-Size: 11612 - -Package: amd-smi-lib-dbgsym -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: bc7da17826f624c2b29de0de166b44ef87102ee7 bc7da17826f624c2b29de0de166b44ef87102ee7 -Depends: amd-smi-lib (= 24.6.3.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/a/amd-smi-lib-dbgsym/amd-smi-lib-dbgsym_24.6.3.60202-116~20.04_amd64.deb -Size: 10153342 -SHA256: ae88979ce5c1844ecd0d71cd8a7497f251ba7fa07b26eaccfa479ab9b3086a94 -SHA1: 3e3b4fb8a41b46f2ebe4064e3df7447ceb3d30d7 -MD5sum: cccf2e2e2740bb7738c0de12759e06ce -Description: debug symbols for amd-smi-lib -Maintainer: AMD-SMILib Support -Package-Type: ddeb -Version: 24.6.3.60202-116~20.04 -Installed-Size: 39320 - -Package: amd-smi-lib-dbgsym-rpath6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: bc7da17826f624c2b29de0de166b44ef87102ee7 bc7da17826f624c2b29de0de166b44ef87102ee7 -Depends: amd-smi-lib-rpath6.2.2 (= 24.6.3.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/a/amd-smi-lib-dbgsym-rpath6.2.2/amd-smi-lib-dbgsym-rpath6.2.2_24.6.3.60202-116~20.04_amd64.deb -Size: 6651668 -SHA256: fecf8fc9298223803c857d4e571153da160ffbae672cd51441f272994fcc7a4b -SHA1: 1f9e02d3d1c7a03a0f564f502e726f6f85b3da22 -MD5sum: c49aff04646d68fd01ac41b295dfb64d -Description: debug symbols for amd-smi-lib -Maintainer: AMD-SMILib Support -Package-Type: ddeb -Version: 24.6.3.60202-116~20.04 -Installed-Size: 39320 - -Package: amd-smi-lib-dbgsym6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: bc7da17826f624c2b29de0de166b44ef87102ee7 bc7da17826f624c2b29de0de166b44ef87102ee7 -Depends: amd-smi-lib6.2.2 (= 24.6.3.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/a/amd-smi-lib-dbgsym6.2.2/amd-smi-lib-dbgsym6.2.2_24.6.3.60202-116~20.04_amd64.deb -Size: 6651296 -SHA256: 5cec802a00e0ea6503302ca28b41f25cf2ec2943c26f87d6e25164d834552903 -SHA1: 443f11649ac679eab61e7f33bb9c21273938dff7 -MD5sum: 3ee0d7c614366ae601907ed304f1a729 -Description: debug symbols for amd-smi-lib -Maintainer: AMD-SMILib Support -Package-Type: ddeb -Version: 24.6.3.60202-116~20.04 -Installed-Size: 39320 - -Package: amd-smi-lib-rpath6.2.2 -Architecture: amd64 -Depends: sudo, python3 (>= 3.6.8), python3-pip, rocm-core-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/a/amd-smi-lib-rpath6.2.2/amd-smi-lib-rpath6.2.2_24.6.3.60202-116~20.04_amd64.deb -Size: 559132 -SHA256: e717db83fa431e46a94e7114593884b066393901740635a010f5295c0d615786 -SHA1: 13421bc515f4ad19fbfb0b2fb60618ab8ae53bb9 -MD5sum: 501f0587f48196c8fb062fa6a6b1ce85 -Description: AMD System Management libraries -Maintainer: AMD-SMILib Support -Provides: amd-smi-rpath6.2.2 -Recommends: python3-argcomplete, libdrm-dev, python3-yaml -Version: 24.6.3.60202-116~20.04 -Installed-Size: 7700 - -Package: amd-smi-lib6.2.2 -Architecture: amd64 -Depends: sudo, python3 (>= 3.6.8), python3-pip, rocm-core6.2.2 -Priority: optional -Section: devel -Filename: pool/main/a/amd-smi-lib6.2.2/amd-smi-lib6.2.2_24.6.3.60202-116~20.04_amd64.deb -Size: 560084 -SHA256: 2eb41af8cd09f90c028b629032a8f203319dd1f4e8af71d0cff28c1fe558ceb1 -SHA1: 1b245dbf8bf0033010babcfd5c706a8883b46260 -MD5sum: aec70dc488385b324fca1866e3b2511f -Description: AMD System Management libraries -Maintainer: AMD-SMILib Support -Provides: amd-smi6.2.2 -Recommends: python3-argcomplete, libdrm-dev, python3-yaml -Version: 24.6.3.60202-116~20.04 -Installed-Size: 7700 - -Package: comgr -Architecture: amd64 -Depends: libtinfo-dev, rocm-core -Priority: optional -Section: devel -Filename: pool/main/c/comgr/comgr_2.8.0.60202-116~20.04_amd64.deb -Size: 54876024 -SHA256: 7dc8edaca45ee7633ed664032696e29286a86943993282c4005194bb7b197b36 -SHA1: 8a7b546de79b4036c1dda68d968b6d656ed270d0 -MD5sum: 3e88146422d537a187c51e5bc2f09b9c -Description: Library to provide support functions for ROCm code objects. -Homepage: https://github.com/RadeonOpenCompute/ROCm-CompilerSupport -Maintainer: ROCm Compiler Support -Version: 2.8.0.60202-116~20.04 -Installed-Size: 447714 - -Package: comgr-asan -Architecture: amd64 -Depends: libtinfo-dev, rocm-core-asan -Priority: optional -Section: devel -Filename: pool/main/c/comgr-asan/comgr-asan_2.8.0.60202-116~20.04_amd64.deb -Size: 55329874 -SHA256: ce572dfaeb839b8d63c46635187bb9b5d9734a477a7b0c8858ad0a2afb6e92c5 -SHA1: 733b197bf930891e91fb33e095b973f8356fba33 -MD5sum: 5082b34bb4b033b172e0f7e66c97f58b -Description: AddressSanitizer Instrumented Libraries to provide support functions for ROCm code objects. -Homepage: https://github.com/RadeonOpenCompute/ROCm-CompilerSupport -Maintainer: ROCm Compiler Support -Version: 2.8.0.60202-116~20.04 -Installed-Size: 450576 - -Package: comgr-asan-dbgsym -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 008bbe3657c9a924022f341b3b49e46ae3aff60b -Depends: comgr-asan (= 2.8.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/c/comgr-asan-dbgsym/comgr-asan-dbgsym_2.8.0.60202-116~20.04_amd64.deb -Size: 6679774 -SHA256: d6b23d10868d11386384815ba518b9d4391fa13a5741e4a599d51ebb09e06eea -SHA1: 6711a1669fa0b6627b3e0358cc135ba72650e949 -MD5sum: 4c8694bf9796cded4485495184020680 -Description: debug symbols for comgr-asan -Maintainer: ROCm Compiler Support -Package-Type: ddeb -Version: 2.8.0.60202-116~20.04 -Installed-Size: 32182 - -Package: comgr-asan-dbgsym-rpath6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 008bbe3657c9a924022f341b3b49e46ae3aff60b -Depends: comgr-asan-rpath6.2.2 (= 2.8.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/c/comgr-asan-dbgsym-rpath6.2.2/comgr-asan-dbgsym-rpath6.2.2_2.8.0.60202-116~20.04_amd64.deb -Size: 4272876 -SHA256: 0734a6e4d514394bd6df1405a557d87c12b35c024c991d2600a237b9f487309c -SHA1: 3abb2bd14b0c3f4780c0f8f472d402a289e30da3 -MD5sum: 4e69e8ef1076c97f88fe6d0c1d38e242 -Description: debug symbols for comgr-asan -Maintainer: ROCm Compiler Support -Package-Type: ddeb -Version: 2.8.0.60202-116~20.04 -Installed-Size: 32182 - -Package: comgr-asan-dbgsym6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 008bbe3657c9a924022f341b3b49e46ae3aff60b -Depends: comgr-asan6.2.2 (= 2.8.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/c/comgr-asan-dbgsym6.2.2/comgr-asan-dbgsym6.2.2_2.8.0.60202-116~20.04_amd64.deb -Size: 4272148 -SHA256: fb3c3c731ff45360a6e22a87b0b764d69cd06219f5cebeffd359def19a684808 -SHA1: eefdc80548990e1c6a4f116a433138e6116e29c9 -MD5sum: b724dc430545a33a600783d88baecb16 -Description: debug symbols for comgr-asan -Maintainer: ROCm Compiler Support -Package-Type: ddeb -Version: 2.8.0.60202-116~20.04 -Installed-Size: 32182 - -Package: comgr-asan-rpath6.2.2 -Architecture: amd64 -Depends: libtinfo-dev, rocm-core-asan-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/c/comgr-asan-rpath6.2.2/comgr-asan-rpath6.2.2_2.8.0.60202-116~20.04_amd64.deb -Size: 37831024 -SHA256: 152d5f7087fa0a8124af3ccfcd3c978484ae92158c95290165b4dab1a80b00e0 -SHA1: 18cac0966dd98109afe68391c66a844bf851ac97 -MD5sum: e2a4e18715bc07c02a5ab7aeacae362a -Description: AddressSanitizer Instrumented Libraries to provide support functions for ROCm code objects. -Homepage: https://github.com/RadeonOpenCompute/ROCm-CompilerSupport -Maintainer: ROCm Compiler Support -Version: 2.8.0.60202-116~20.04 -Installed-Size: 450576 - -Package: comgr-asan6.2.2 -Architecture: amd64 -Depends: libtinfo-dev, rocm-core-asan6.2.2 -Priority: optional -Section: devel -Filename: pool/main/c/comgr-asan6.2.2/comgr-asan6.2.2_2.8.0.60202-116~20.04_amd64.deb -Size: 37842816 -SHA256: 274ed7e8d41a0fb70bb619abbfc945c778d3bd23fb8132c678a778475daeffb2 -SHA1: b489b29d1e32bf41fd26ab8a9e8f40210e1fa1d1 -MD5sum: ec8dd2683f6047964715b36fd7b6f3c1 -Description: AddressSanitizer Instrumented Libraries to provide support functions for ROCm code objects. -Homepage: https://github.com/RadeonOpenCompute/ROCm-CompilerSupport -Maintainer: ROCm Compiler Support -Version: 2.8.0.60202-116~20.04 -Installed-Size: 450576 - -Package: comgr-dbgsym -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: a9b54f2f44db5fddfac4d350040c5c36b5fbf311 -Depends: comgr (= 2.8.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/c/comgr-dbgsym/comgr-dbgsym_2.8.0.60202-116~20.04_amd64.deb -Size: 15159082 -SHA256: e5e539ff8ed8e6a340fc85b0a363e6c113c4ce1e242e59a7d01810cbb724584c -SHA1: 8d15ab5b131b245e86054af541f6c2e96cc76097 -MD5sum: 1e342557b748d4606fac6d241475ffac -Description: debug symbols for comgr -Maintainer: ROCm Compiler Support -Package-Type: ddeb -Version: 2.8.0.60202-116~20.04 -Installed-Size: 57307 - -Package: comgr-dbgsym-rpath6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: a9b54f2f44db5fddfac4d350040c5c36b5fbf311 -Depends: comgr-rpath6.2.2 (= 2.8.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/c/comgr-dbgsym-rpath6.2.2/comgr-dbgsym-rpath6.2.2_2.8.0.60202-116~20.04_amd64.deb -Size: 9137176 -SHA256: 641e633b1fc368e12931c00519548d97674dfe4ce47ed5caffb7675dc3afeaaf -SHA1: d912ccf037e87c91441701aa742940ee83f4351b -MD5sum: 36f0ab38288f81c6d2bdecd7fd061724 -Description: debug symbols for comgr -Maintainer: ROCm Compiler Support -Package-Type: ddeb -Version: 2.8.0.60202-116~20.04 -Installed-Size: 57307 - -Package: comgr-dbgsym6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: a9b54f2f44db5fddfac4d350040c5c36b5fbf311 -Depends: comgr6.2.2 (= 2.8.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/c/comgr-dbgsym6.2.2/comgr-dbgsym6.2.2_2.8.0.60202-116~20.04_amd64.deb -Size: 9135488 -SHA256: fd1ed269d29a61561048d307bb97ccf6350a33d5f714b8f39f1b13cf7c138e07 -SHA1: 1514dc600055fc887a7495f89061be0fc6998fcc -MD5sum: f3881f358e0c00f1f28b8e037a22aed1 -Description: debug symbols for comgr -Maintainer: ROCm Compiler Support -Package-Type: ddeb -Version: 2.8.0.60202-116~20.04 -Installed-Size: 57307 - -Package: comgr-rpath6.2.2 -Architecture: amd64 -Depends: libtinfo-dev, rocm-core-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/c/comgr-rpath6.2.2/comgr-rpath6.2.2_2.8.0.60202-116~20.04_amd64.deb -Size: 37565584 -SHA256: 22cfedff4796e366b57cb29bf115a8d3b02467079df356f55f2bf15a0c238e6d -SHA1: 929f0c504b3e746bee3f5b1f1d9b5d81bfbc8a24 -MD5sum: 9464cdd3e43ef7b61c8be91e952b4896 -Description: Library to provide support functions for ROCm code objects. -Homepage: https://github.com/RadeonOpenCompute/ROCm-CompilerSupport -Maintainer: ROCm Compiler Support -Version: 2.8.0.60202-116~20.04 -Installed-Size: 447714 - -Package: comgr6.2.2 -Architecture: amd64 -Depends: libtinfo-dev, rocm-core6.2.2 -Priority: optional -Section: devel -Filename: pool/main/c/comgr6.2.2/comgr6.2.2_2.8.0.60202-116~20.04_amd64.deb -Size: 37562240 -SHA256: 20654ba990c1956155310cfb22c7a0c5d7157a4122bb6dc7a92e73f2cac85fed -SHA1: 0fe3dbff4f62d01a433074d9d7764a266ef9712d -MD5sum: 1c406e50ce49a7f3b983cf5074338df5 -Description: Library to provide support functions for ROCm code objects. -Homepage: https://github.com/RadeonOpenCompute/ROCm-CompilerSupport -Maintainer: ROCm Compiler Support -Version: 2.8.0.60202-116~20.04 -Installed-Size: 447714 - -Package: composablekernel-ckprofiler_gfx10 -Architecture: amd64 -Depends: rocm-core -Priority: optional -Section: devel -Filename: pool/main/c/composablekernel-ckprofiler_gfx10/composablekernel-ckprofiler_gfx10_1.1.0.60202-116~20.04_amd64.deb -Size: 12608880 -SHA256: 9f69d24760a76cdf02b7f359b5ca08a72d201d825f9020ddf63dcf0aeb6e47ff -SHA1: 4fa2aa954530b2550c3b64d4a5089f605ca5520c -MD5sum: 4ea936582b317cfab1ce48d94e55d311 -Description: High Performance Composable Kernel for AMD GPUs -Maintainer: MIOpen Kernels Dev Team -Version: 1.1.0.60202-116~20.04 -Installed-Size: 296617 - -Package: composablekernel-ckprofiler_gfx10-rpath6.2.2 -Architecture: amd64 -Depends: rocm-core-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/c/composablekernel-ckprofiler_gfx10-rpath6.2.2/composablekernel-ckprofiler_gfx10-rpath6.2.2_1.1.0.60202-116~20.04_amd64.deb -Size: 12596148 -SHA256: 53ee7766af90d589844aac358af5131ee43d650c13bfd82ea4f498a30681120f -SHA1: d58492c1188e8267bdbf2b6e186be76e16aee4f3 -MD5sum: 50ed5136474e54e1e4e5b7345f21c7bf -Description: High Performance Composable Kernel for AMD GPUs -Maintainer: MIOpen Kernels Dev Team -Version: 1.1.0.60202-116~20.04 -Installed-Size: 296617 - -Package: composablekernel-ckprofiler_gfx106.2.2 -Architecture: amd64 -Depends: rocm-core6.2.2 -Priority: optional -Section: devel -Filename: pool/main/c/composablekernel-ckprofiler_gfx106.2.2/composablekernel-ckprofiler_gfx106.2.2_1.1.0.60202-116~20.04_amd64.deb -Size: 12596380 -SHA256: 856138ba947b8cf9250a3fafe49d6f1e6b9935218c97075c4e285c7cc741f4b7 -SHA1: e09fb5c327c7a95ce37a311e39f597e09b4c20c4 -MD5sum: 0bae26ce6662c9261d0201a21ca7101e -Description: High Performance Composable Kernel for AMD GPUs -Maintainer: MIOpen Kernels Dev Team -Version: 1.1.0.60202-116~20.04 -Installed-Size: 296617 - -Package: composablekernel-ckprofiler_gfx11 -Architecture: amd64 -Depends: rocm-core -Priority: optional -Section: devel -Filename: pool/main/c/composablekernel-ckprofiler_gfx11/composablekernel-ckprofiler_gfx11_1.1.0.60202-116~20.04_amd64.deb -Size: 28675578 -SHA256: 6ac95e750706588756fc17d75b0f1330bd335ae2d39da71b696f6d2b4a9bf969 -SHA1: 0c5439bdf732d0ce08b44d94b56e0a4deed76b25 -MD5sum: de3aeec04fadd43e2b39dd131095cd73 -Description: High Performance Composable Kernel for AMD GPUs -Maintainer: MIOpen Kernels Dev Team -Version: 1.1.0.60202-116~20.04 -Installed-Size: 790278 - -Package: composablekernel-ckprofiler_gfx11-rpath6.2.2 -Architecture: amd64 -Depends: rocm-core-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/c/composablekernel-ckprofiler_gfx11-rpath6.2.2/composablekernel-ckprofiler_gfx11-rpath6.2.2_1.1.0.60202-116~20.04_amd64.deb -Size: 28703832 -SHA256: 23648d0f8bdf60b71ff96700a3a6bd9e09332bc7da388ff9a1c7db4b89d51328 -SHA1: 320a6ec05d87deb6aee2420c822b74f4b145740f -MD5sum: 7ef6ab91cb6b13c6a9ddaaba6e406d2d -Description: High Performance Composable Kernel for AMD GPUs -Maintainer: MIOpen Kernels Dev Team -Version: 1.1.0.60202-116~20.04 -Installed-Size: 790278 - -Package: composablekernel-ckprofiler_gfx116.2.2 -Architecture: amd64 -Depends: rocm-core6.2.2 -Priority: optional -Section: devel -Filename: pool/main/c/composablekernel-ckprofiler_gfx116.2.2/composablekernel-ckprofiler_gfx116.2.2_1.1.0.60202-116~20.04_amd64.deb -Size: 28703796 -SHA256: c3c3017365a49031cda2dd80b23c4bc7d6e1d55e67100b96b9d69fd63ee90e14 -SHA1: 3b511fd4559aff0b726a24ad041d321f6993854e -MD5sum: f20885f3135141023640dda51439934b -Description: High Performance Composable Kernel for AMD GPUs -Maintainer: MIOpen Kernels Dev Team -Version: 1.1.0.60202-116~20.04 -Installed-Size: 790278 - -Package: composablekernel-ckprofiler_gfx90 -Architecture: amd64 -Depends: rocm-core -Priority: optional -Section: devel -Filename: pool/main/c/composablekernel-ckprofiler_gfx90/composablekernel-ckprofiler_gfx90_1.1.0.60202-116~20.04_amd64.deb -Size: 97689952 -SHA256: f14fafe63f6a861d768c7539f90fbec9d9e5136e4e7e55bee5bb6fd967819612 -SHA1: 98be9286ef2fdf2a89e515da74e2209d63e8b845 -MD5sum: eb188d14d5a488f9e68d289b3d73438e -Description: High Performance Composable Kernel for AMD GPUs -Maintainer: MIOpen Kernels Dev Team -Version: 1.1.0.60202-116~20.04 -Installed-Size: 2017912 - -Package: composablekernel-ckprofiler_gfx90-rpath6.2.2 -Architecture: amd64 -Depends: rocm-core-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/c/composablekernel-ckprofiler_gfx90-rpath6.2.2/composablekernel-ckprofiler_gfx90-rpath6.2.2_1.1.0.60202-116~20.04_amd64.deb -Size: 97681168 -SHA256: 24f79fb1e667a0276740eb9615d98d893ffc9d782bfb06fb73615f384f7a76c5 -SHA1: 8497df40e7c93125df539eb1cf2da167daac18d1 -MD5sum: ba6a761686b9d68f06a06e427e06e247 -Description: High Performance Composable Kernel for AMD GPUs -Maintainer: MIOpen Kernels Dev Team -Version: 1.1.0.60202-116~20.04 -Installed-Size: 2017912 - -Package: composablekernel-ckprofiler_gfx906.2.2 -Architecture: amd64 -Depends: rocm-core6.2.2 -Priority: optional -Section: devel -Filename: pool/main/c/composablekernel-ckprofiler_gfx906.2.2/composablekernel-ckprofiler_gfx906.2.2_1.1.0.60202-116~20.04_amd64.deb -Size: 97684368 -SHA256: 27024d367cb56565e907da9bfe4d26d0b52c03aebd284ec5b2bd3f30de4ead71 -SHA1: 8a89fc20cd61f96f13a18dbe598c019d2999afc5 -MD5sum: 71f2985490516f555300f89d578bf7dc -Description: High Performance Composable Kernel for AMD GPUs -Maintainer: MIOpen Kernels Dev Team -Version: 1.1.0.60202-116~20.04 -Installed-Size: 2017912 - -Package: composablekernel-ckprofiler_gfx94 -Architecture: amd64 -Depends: rocm-core -Priority: optional -Section: devel -Filename: pool/main/c/composablekernel-ckprofiler_gfx94/composablekernel-ckprofiler_gfx94_1.1.0.60202-116~20.04_amd64.deb -Size: 59066938 -SHA256: db404b876705188ee3f310bdb1cd91060c365524f4463b3db63c54497a7268ff -SHA1: 0ce552701b5e6f6fcaf3394726a4aefd2451933d -MD5sum: 6c7f8716a667430f66538309aa4a7836 -Description: High Performance Composable Kernel for AMD GPUs -Maintainer: MIOpen Kernels Dev Team -Version: 1.1.0.60202-116~20.04 -Installed-Size: 2139356 - -Package: composablekernel-ckprofiler_gfx94-rpath6.2.2 -Architecture: amd64 -Depends: rocm-core-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/c/composablekernel-ckprofiler_gfx94-rpath6.2.2/composablekernel-ckprofiler_gfx94-rpath6.2.2_1.1.0.60202-116~20.04_amd64.deb -Size: 59011936 -SHA256: 972d3e32aca69a9c092f2eb04ddc26cdfe2fc7942b8fa873220beb6f614a11b9 -SHA1: 3fba495ebfb185b5187f9ec3879bf2494eb72e7c -MD5sum: 926a41fb67b2a4be0236ddddbdaca2f5 -Description: High Performance Composable Kernel for AMD GPUs -Maintainer: MIOpen Kernels Dev Team -Version: 1.1.0.60202-116~20.04 -Installed-Size: 2139356 - -Package: composablekernel-ckprofiler_gfx946.2.2 -Architecture: amd64 -Depends: rocm-core6.2.2 -Priority: optional -Section: devel -Filename: pool/main/c/composablekernel-ckprofiler_gfx946.2.2/composablekernel-ckprofiler_gfx946.2.2_1.1.0.60202-116~20.04_amd64.deb -Size: 59017448 -SHA256: 9424d5379f35fef946bcff98f0731ad8d9b6c0e5768ef7dfbfaefa3d35ea56dd -SHA1: ef810227393c9b616aaf5bbb8f92001d99519a03 -MD5sum: 7ff2e6a9bfacf49f0877e1768d0e29dd -Description: High Performance Composable Kernel for AMD GPUs -Maintainer: MIOpen Kernels Dev Team -Version: 1.1.0.60202-116~20.04 -Installed-Size: 2139356 - -Package: composablekernel-dev -Architecture: amd64 -Depends: rocm-core -Priority: optional -Section: devel -Filename: pool/main/c/composablekernel-dev/composablekernel-dev_1.1.0.60202-116~20.04_amd64.deb -Size: 198770410 -SHA256: 1e70bf8040ca0fdfed7f0c64d6b7d5a6bb705e93904c32a1e4cdd2a32f56fbd9 -SHA1: 7baa56a77589554964ed2cfa647f8344028b1069 -MD5sum: 54ccf8f151e0fd0ee15d2897d9396563 -Description: High Performance Composable Kernel for AMD GPUs -Maintainer: MIOpen Kernels Dev Team -Provides: CPACK_DEBIAN_PACKAGE_PROVIDES, composablekernel (= 1.1.0.60202) -Version: 1.1.0.60202-116~20.04 -Installed-Size: 5639543 - -Package: composablekernel-dev-rpath6.2.2 -Architecture: amd64 -Depends: rocm-core-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/c/composablekernel-dev-rpath6.2.2/composablekernel-dev-rpath6.2.2_1.1.0.60202-116~20.04_amd64.deb -Size: 198757340 -SHA256: caedf4ece5744a61f64c9b4a5f46977512c57b9928f68f253a16ea23df32d6c5 -SHA1: 128e4db01084711811f9ecd4410725d811f31e51 -MD5sum: 7f3b3e231949a8b940d2da93c5ad2fd5 -Description: High Performance Composable Kernel for AMD GPUs -Maintainer: MIOpen Kernels Dev Team -Provides: CPACK_DEBIAN_PACKAGE_PROVIDES, composablekernel-rpath6.2.2 (= 1.1.0.60202) -Version: 1.1.0.60202-116~20.04 -Installed-Size: 5639543 - -Package: composablekernel-dev6.2.2 -Architecture: amd64 -Depends: rocm-core6.2.2 -Priority: optional -Section: devel -Filename: pool/main/c/composablekernel-dev6.2.2/composablekernel-dev6.2.2_1.1.0.60202-116~20.04_amd64.deb -Size: 198757288 -SHA256: 0dbf03b791e202308e5ebb3ca19c1d4c2575e5854fb9dc7a50d5237d0bb71fe1 -SHA1: c4dcd2fbadb1869a6960ed232807908998946950 -MD5sum: eac6e5c916af23d9b03faa25d9db6d29 -Description: High Performance Composable Kernel for AMD GPUs -Maintainer: MIOpen Kernels Dev Team -Provides: CPACK_DEBIAN_PACKAGE_PROVIDES, composablekernel6.2.2 (= 1.1.0.60202) -Version: 1.1.0.60202-116~20.04 -Installed-Size: 5639543 - -Package: half -Architecture: amd64 -Depends: rocm-core -Priority: optional -Section: devel -Filename: pool/main/h/half/half_1.12.0.60202-116~20.04_amd64.deb -Size: 19528 -SHA256: 38fa841f58199d778a5e18b2986ee8447049e771a43899bc827b02fa22833058 -SHA1: d91d6ee4720f426cce5bfd965c5aeac7ef893e6e -MD5sum: 4f89ad4227fe4285c43af7bbcef037cf -Description: HALF-PRECISION FLOATING POINT LIBRARY -Maintainer: Paul Fultz II -Version: 1.12.0.60202-116~20.04 -Installed-Size: 175 - -Package: half-rpath6.2.2 -Architecture: amd64 -Depends: rocm-core-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/half-rpath6.2.2/half-rpath6.2.2_1.12.0.60202-116~20.04_amd64.deb -Size: 19788 -SHA256: 387fc9d8085dca765ff9c2813e7d4cb73ac87a5dfec26f36e84fb8d59e7cdc96 -SHA1: 22cf3e7e2e8da583cd48f6b785d4f010cca82361 -MD5sum: 8cf2d5fdc06dd727eb8823951a41b687 -Description: HALF-PRECISION FLOATING POINT LIBRARY -Maintainer: Paul Fultz II -Version: 1.12.0.60202-116~20.04 -Installed-Size: 175 - -Package: half6.2.2 -Architecture: amd64 -Depends: rocm-core6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/half6.2.2/half6.2.2_1.12.0.60202-116~20.04_amd64.deb -Size: 19764 -SHA256: 82c681235783cec80ffb4e6a6a9110b9c76ce2c2ae735c09dcb52887f24970fa -SHA1: 3a81edf987a03545e46acc82cc57192de8e69df7 -MD5sum: 62bc240c35fd24be9c1faf20a2c5e5e0 -Description: HALF-PRECISION FLOATING POINT LIBRARY -Maintainer: Paul Fultz II -Version: 1.12.0.60202-116~20.04 -Installed-Size: 175 - -Package: hip-dev -Architecture: amd64 -Depends: perl (>= 5.0), libfile-copy-recursive-perl, libfile-listing-perl, libfile-which-perl, liburi-perl, libc6, file, hip-runtime-amd, rocm-llvm, rocm-core, hsa-rocr-dev -Priority: optional -Section: devel -Filename: pool/main/h/hip-dev/hip-dev_6.2.41134.60202-116~20.04_amd64.deb -Size: 307026 -SHA256: da0d6433a6e110067decd42c5e781cdd11ac44ae25861846ef6b63a0369cf0ee -SHA1: 3a62ff03d7adf6c8780b3d36d5ba44f5c970efb0 -MD5sum: 197b6b5dab426c799abf000a4ba72761 -Description: HIP:Heterogenous-computing Interface for Portability - HIP: Heterogenous-computing Interface for Portability [DEVELOPMENT] -Maintainer: HIP Support -Provides: hip-base -Replaces: hip-base -Version: 6.2.41134.60202-116~20.04 -Installed-Size: 2276 - -Package: hip-dev-rpath6.2.2 -Architecture: amd64 -Depends: perl (>= 5.0), libfile-copy-recursive-perl, libfile-listing-perl, libfile-which-perl, liburi-perl, libc6, file, hip-runtime-amd-rpath6.2.2, rocm-llvm-rpath6.2.2, rocm-core-rpath6.2.2, hsa-rocr-dev-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/hip-dev-rpath6.2.2/hip-dev-rpath6.2.2_6.2.41134.60202-116~20.04_amd64.deb -Size: 222952 -SHA256: d92aaf6d0539df285ea600224d07ea01a1f1e19b989f2bce21114ac2ad3beadd -SHA1: 25368c30ddc82847e3a826d8e98e700e75d82da0 -MD5sum: e9ff002824f1c07358cc122c3fd07e56 -Description: HIP:Heterogenous-computing Interface for Portability - HIP: Heterogenous-computing Interface for Portability [DEVELOPMENT] -Maintainer: HIP Support -Provides: hip-base-rpath6.2.2 -Replaces: hip-base-rpath6.2.2 -Version: 6.2.41134.60202-116~20.04 -Installed-Size: 2276 - -Package: hip-dev6.2.2 -Architecture: amd64 -Depends: perl (>= 5.0), libfile-copy-recursive-perl, libfile-listing-perl, libfile-which-perl, liburi-perl, libc6, file, hip-runtime-amd6.2.2, rocm-llvm6.2.2, rocm-core6.2.2, hsa-rocr-dev6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/hip-dev6.2.2/hip-dev6.2.2_6.2.41134.60202-116~20.04_amd64.deb -Size: 222940 -SHA256: 2a3fbcb03e0fd500b193960b67feaf0f011455f5496b6691691dc313ed1cae20 -SHA1: d29dec0bded767f47f95ce8049b96ddb6c9d2770 -MD5sum: 62ada02978dd93957e3c898bfb1d6eef -Description: HIP:Heterogenous-computing Interface for Portability - HIP: Heterogenous-computing Interface for Portability [DEVELOPMENT] -Maintainer: HIP Support -Provides: hip-base6.2.2 -Replaces: hip-base6.2.2 -Version: 6.2.41134.60202-116~20.04 -Installed-Size: 2276 - -Package: hip-doc -Architecture: amd64 -Depends: hip-dev (= 6.2.41134.60202-116~20.04), rocm-core -Priority: optional -Section: devel -Filename: pool/main/h/hip-doc/hip-doc_6.2.41134.60202-116~20.04_amd64.deb -Size: 89098 -SHA256: d0e92efd6e27b1e275f4e78bd4fa889ade9854f47da862ba2ac10747ab55dc68 -SHA1: d7b8510314e9af1d06e2739e9e15d8fcd2b5dcd3 -MD5sum: 1fc26a3749b8915c349752735329f811 -Description: HIP:Heterogenous-computing Interface for Portability - HIP: Heterogenous-computing Interface for Portability [DOCUMENTATION] -Maintainer: HIP Support -Provides: hip-doc -Version: 6.2.41134.60202-116~20.04 -Installed-Size: 294 - -Package: hip-doc-rpath6.2.2 -Architecture: amd64 -Depends: hip-dev-rpath6.2.2 (= 6.2.41134.60202-116~20.04), rocm-core-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/hip-doc-rpath6.2.2/hip-doc-rpath6.2.2_6.2.41134.60202-116~20.04_amd64.deb -Size: 79172 -SHA256: 01d3bc6e99f1d708b16904e6467e7a87814d031fa512146e8c68c59da76571ee -SHA1: 7ef0bdd027f6b832b1c73cafa3f1d90a861aa0ed -MD5sum: aeaa0289613430970fc15ff1d2b8666b -Description: HIP:Heterogenous-computing Interface for Portability - HIP: Heterogenous-computing Interface for Portability [DOCUMENTATION] -Maintainer: HIP Support -Provides: hip-doc-rpath6.2.2 -Version: 6.2.41134.60202-116~20.04 -Installed-Size: 294 - -Package: hip-doc6.2.2 -Architecture: amd64 -Depends: hip-dev6.2.2 (= 6.2.41134.60202-116~20.04), rocm-core6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/hip-doc6.2.2/hip-doc6.2.2_6.2.41134.60202-116~20.04_amd64.deb -Size: 79168 -SHA256: c31fa0c0f90bf9322f8f7bf6ed88ba82891a60c80c6c3c182293fa7ae3fd7834 -SHA1: 54affe81369dc4948a7eb8aae0893acfe02d1084 -MD5sum: 029507a6dda15bd98f2c5dd4e86c3056 -Description: HIP:Heterogenous-computing Interface for Portability - HIP: Heterogenous-computing Interface for Portability [DOCUMENTATION] -Maintainer: HIP Support -Provides: hip-doc6.2.2 -Version: 6.2.41134.60202-116~20.04 -Installed-Size: 294 - -Package: hip-runtime-amd -Architecture: amd64 -Depends: hsa-rocr, rocminfo, comgr, rocm-core, rocprofiler-register, libnuma1, libstdc++6, libgcc-s1, libc6 -Priority: optional -Section: devel -Filename: pool/main/h/hip-runtime-amd/hip-runtime-amd_6.2.41134.60202-116~20.04_amd64.deb -Size: 13955042 -SHA256: 62a5e057fb25cdfabbbb403046475ad7720a97d009f9c1df3ee1d34f020d6334 -SHA1: 194357db52942e179bfc95fb2e0845851835d63d -MD5sum: 40c23d03c9a591fef0ab507d9f7a0521 -Description: HIP:Heterogenous-computing Interface for Portability - HIP:Heterogenous-computing Interface for Portability [RUNTIME - AMD] -Maintainer: HIP Support -Provides: hip-rocclr (= 6.2.41134.60202) -Replaces: hip-rocclr (= 6.2.41134.60202) -Version: 6.2.41134.60202-116~20.04 -Installed-Size: 81182 - -Package: hip-runtime-amd-asan -Architecture: amd64 -Depends: hsa-rocr-asan, rocminfo, comgr-asan, rocm-llvm, rocm-core-asan, libnuma1, libstdc++6, libgcc-s1, libc6 -Priority: optional -Section: devel -Filename: pool/main/h/hip-runtime-amd-asan/hip-runtime-amd-asan_6.2.41134.60202-116~20.04_amd64.deb -Size: 15865878 -SHA256: dfa661e6c44986ba76f906b752f713c14e0722f83d07f0f825846517ad6cace0 -SHA1: 333eef1bfb72d0c8f3fcc1ac5d8a81afb9b8115e -MD5sum: 680c3fd0f1947f9f048bd82af911087f -Description: HIP:Heterogenous-computing Interface for Portability - HIP:Heterogenous-computing Interface for Portability [AddressSanitizer libraries] -Maintainer: HIP Support -Version: 6.2.41134.60202-116~20.04 -Installed-Size: 104286 - -Package: hip-runtime-amd-asan-dbgsym -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: dd18b13ba7768e31327b7aef08357ca3e7d1ca88 ce9dd10c3da8036813e32a2ed87f5e2243a414da 73809b3d19c0d591feb9202c10e82881ca47a8b6 -Depends: hip-runtime-amd-asan (= 6.2.41134.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/h/hip-runtime-amd-asan-dbgsym/hip-runtime-amd-asan-dbgsym_6.2.41134.60202-116~20.04_amd64.deb -Size: 16386384 -SHA256: 9fcc160b656d12c923937194bef6ab081c7f7b9f7da4802dc92ad0155f80b29d -SHA1: de7592fb42e81f9b89da7858fc3c0925889d1fd9 -MD5sum: c46d657de9a7fd44c24f0a239b8e40de -Description: debug symbols for hip-runtime-amd-asan -Maintainer: HIP Support -Package-Type: ddeb -Version: 6.2.41134.60202-116~20.04 -Installed-Size: 51595 - -Package: hip-runtime-amd-asan-dbgsym-rpath6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: dd18b13ba7768e31327b7aef08357ca3e7d1ca88 ce9dd10c3da8036813e32a2ed87f5e2243a414da 73809b3d19c0d591feb9202c10e82881ca47a8b6 -Depends: hip-runtime-amd-asan-rpath6.2.2 (= 6.2.41134.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/h/hip-runtime-amd-asan-dbgsym-rpath6.2.2/hip-runtime-amd-asan-dbgsym-rpath6.2.2_6.2.41134.60202-116~20.04_amd64.deb -Size: 9515604 -SHA256: 8f3f2453cba29ab0a4ecbfbcdfa187b12956e51c3231dc29e6570c26ca57e3fc -SHA1: 94967426a6ebe4820518dc8ed21d3d810cbfe37a -MD5sum: 08f26a296c15c558de475f0f6f5a35a4 -Description: debug symbols for hip-runtime-amd-asan -Maintainer: HIP Support -Package-Type: ddeb -Version: 6.2.41134.60202-116~20.04 -Installed-Size: 51595 - -Package: hip-runtime-amd-asan-dbgsym6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: dd18b13ba7768e31327b7aef08357ca3e7d1ca88 ce9dd10c3da8036813e32a2ed87f5e2243a414da 73809b3d19c0d591feb9202c10e82881ca47a8b6 -Depends: hip-runtime-amd-asan6.2.2 (= 6.2.41134.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/h/hip-runtime-amd-asan-dbgsym6.2.2/hip-runtime-amd-asan-dbgsym6.2.2_6.2.41134.60202-116~20.04_amd64.deb -Size: 9515136 -SHA256: 3d57e6ce53e5b041aa2d5c31b9b78ae978b8712f51b1c24c21fdd00ec2d5a3e4 -SHA1: ae6b81a2a2c82e72178a8bf2d1204385e8cd5db0 -MD5sum: a41bfcde1b5d5e917d256775eefcc988 -Description: debug symbols for hip-runtime-amd-asan -Maintainer: HIP Support -Package-Type: ddeb -Version: 6.2.41134.60202-116~20.04 -Installed-Size: 51595 - -Package: hip-runtime-amd-asan-rpath6.2.2 -Architecture: amd64 -Depends: hsa-rocr-asan-rpath6.2.2, rocminfo, comgr-asan-rpath6.2.2, rocm-llvm, rocm-core-asan-rpath6.2.2, libnuma1, libstdc++6, libgcc-s1, libc6 -Priority: optional -Section: devel -Filename: pool/main/h/hip-runtime-amd-asan-rpath6.2.2/hip-runtime-amd-asan-rpath6.2.2_6.2.41134.60202-116~20.04_amd64.deb -Size: 11611476 -SHA256: 4da7c16c66815b1c24dc74348ec06de747c81b500770db0cd5900576e1f4ddcd -SHA1: 9dd0e1b4b10ef9e710eec80d70befa1e79707d6c -MD5sum: e8eaa0b09b7c40adcd369e6f382118f9 -Description: HIP:Heterogenous-computing Interface for Portability - HIP:Heterogenous-computing Interface for Portability [AddressSanitizer libraries] -Maintainer: HIP Support -Version: 6.2.41134.60202-116~20.04 -Installed-Size: 104286 - -Package: hip-runtime-amd-asan6.2.2 -Architecture: amd64 -Depends: hsa-rocr-asan6.2.2, rocminfo, comgr-asan6.2.2, rocm-llvm, rocm-core-asan6.2.2, libnuma1, libstdc++6, libgcc-s1, libc6 -Priority: optional -Section: devel -Filename: pool/main/h/hip-runtime-amd-asan6.2.2/hip-runtime-amd-asan6.2.2_6.2.41134.60202-116~20.04_amd64.deb -Size: 11610460 -SHA256: 0714b00b9a50d7481c3b8d95b5c699a3ce708ca2ed89151e9f5170f7e4da6351 -SHA1: 971dc916b29e614f0ef1dc33120abb9be4188b0e -MD5sum: 6a732c3cf04c19c25bf73e13dfd7514b -Description: HIP:Heterogenous-computing Interface for Portability - HIP:Heterogenous-computing Interface for Portability [AddressSanitizer libraries] -Maintainer: HIP Support -Version: 6.2.41134.60202-116~20.04 -Installed-Size: 104286 - -Package: hip-runtime-amd-dbgsym -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: db3df50ab7491c7957c70874e1c97f110e37e130 18f18c89fda46b6b126803b84e3bf66325a64160 0af4a1de7da3c798fdc68e42f607bae1e8df3bc3 -Depends: hip-runtime-amd (= 6.2.41134.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/h/hip-runtime-amd-dbgsym/hip-runtime-amd-dbgsym_6.2.41134.60202-116~20.04_amd64.deb -Size: 32278788 -SHA256: b53dfc87fcaa8c6ef80fa97295c6fb7b159f1178d1b3dfd59b985c139508410e -SHA1: c4bdc808c017732c9386981303ff1f2015b50055 -MD5sum: 7d8a848e3d62eb341f6beb16e30162c8 -Description: debug symbols for hip-runtime-amd -Maintainer: HIP Support -Package-Type: ddeb -Version: 6.2.41134.60202-116~20.04 -Installed-Size: 115402 - -Package: hip-runtime-amd-dbgsym-rpath6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: db3df50ab7491c7957c70874e1c97f110e37e130 18f18c89fda46b6b126803b84e3bf66325a64160 0af4a1de7da3c798fdc68e42f607bae1e8df3bc3 -Depends: hip-runtime-amd-rpath6.2.2 (= 6.2.41134.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/h/hip-runtime-amd-dbgsym-rpath6.2.2/hip-runtime-amd-dbgsym-rpath6.2.2_6.2.41134.60202-116~20.04_amd64.deb -Size: 20418612 -SHA256: 13c1f0e0dc875632bc9d4c981c960d537ea5b7726fcba293b56e61eafb729cc4 -SHA1: 9769c7a7f0391c01e242fd29d8ac13d433c687b0 -MD5sum: 400e5f861d6de0bb11d00a3bf5a07ec1 -Description: debug symbols for hip-runtime-amd -Maintainer: HIP Support -Package-Type: ddeb -Version: 6.2.41134.60202-116~20.04 -Installed-Size: 115402 - -Package: hip-runtime-amd-dbgsym6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: db3df50ab7491c7957c70874e1c97f110e37e130 18f18c89fda46b6b126803b84e3bf66325a64160 0af4a1de7da3c798fdc68e42f607bae1e8df3bc3 -Depends: hip-runtime-amd6.2.2 (= 6.2.41134.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/h/hip-runtime-amd-dbgsym6.2.2/hip-runtime-amd-dbgsym6.2.2_6.2.41134.60202-116~20.04_amd64.deb -Size: 20413640 -SHA256: 29bff99cca7c614659505a198bc95395884f68e72de7942af4410f0cf5b96270 -SHA1: 23d6cb4e10b1e12d56a6158c1e6507361a7fa3a0 -MD5sum: b25c12cdfbfaf98c5a4683dccd6d6a98 -Description: debug symbols for hip-runtime-amd -Maintainer: HIP Support -Package-Type: ddeb -Version: 6.2.41134.60202-116~20.04 -Installed-Size: 115402 - -Package: hip-runtime-amd-rpath6.2.2 -Architecture: amd64 -Depends: hsa-rocr-rpath6.2.2, rocminfo-rpath6.2.2, comgr-rpath6.2.2, rocm-core-rpath6.2.2, rocprofiler-register-rpath6.2.2, libnuma1, libstdc++6, libgcc-s1, libc6 -Priority: optional -Section: devel -Filename: pool/main/h/hip-runtime-amd-rpath6.2.2/hip-runtime-amd-rpath6.2.2_6.2.41134.60202-116~20.04_amd64.deb -Size: 10540464 -SHA256: cc405c7dd1883e89c0915f5f48d6dd2085ad294907e372c3ec36ae5f8f505fc0 -SHA1: 3b197a21beb8bcea3ccbc51bb2324b93f9f1d321 -MD5sum: 254ceec124a9546da272630e42a728a8 -Description: HIP:Heterogenous-computing Interface for Portability - HIP:Heterogenous-computing Interface for Portability [RUNTIME - AMD] -Maintainer: HIP Support -Provides: hip-rocclr-rpath6.2.2 (= 6.2.41134.60202) -Replaces: hip-rocclr-rpath6.2.2 (= 6.2.41134.60202) -Version: 6.2.41134.60202-116~20.04 -Installed-Size: 81182 - -Package: hip-runtime-amd6.2.2 -Architecture: amd64 -Depends: hsa-rocr6.2.2, rocminfo6.2.2, comgr6.2.2, rocm-core6.2.2, rocprofiler-register6.2.2, libnuma1, libstdc++6, libgcc-s1, libc6 -Priority: optional -Section: devel -Filename: pool/main/h/hip-runtime-amd6.2.2/hip-runtime-amd6.2.2_6.2.41134.60202-116~20.04_amd64.deb -Size: 10544072 -SHA256: 9b48d2f8b282dffbf6e5e9306f174db7e40efa2a2c18f360a0b3f1648884238a -SHA1: cc1db30599d213c7329c2623803f6c51a1cbdc5f -MD5sum: 449ccb89de522ed31b697558a97c8735 -Description: HIP:Heterogenous-computing Interface for Portability - HIP:Heterogenous-computing Interface for Portability [RUNTIME - AMD] -Maintainer: HIP Support -Provides: hip-rocclr6.2.2 (= 6.2.41134.60202) -Replaces: hip-rocclr6.2.2 (= 6.2.41134.60202) -Version: 6.2.41134.60202-116~20.04 -Installed-Size: 81182 - -Package: hip-runtime-nvidia -Architecture: amd64 -Depends: cuda (>= 7.5), rocm-core, hipcc-nvidia -Priority: optional -Section: devel -Filename: pool/main/h/hip-runtime-nvidia/hip-runtime-nvidia_6.2.41134.60202-116~20.04_amd64.deb -Size: 694 -SHA256: d53d8632e8ba6c8e0de299e8c6f313e4c2907be511ddf3dd301f44008e9acde8 -SHA1: 5963f5a25260175958aeb14b24a9f02171fb9e14 -MD5sum: 09c100e2df8b451fbe5c1307fb1d8616 -Description: HIP:Heterogenous-computing Interface for Portability - HIP: Heterogenous-computing Interface for Portability [RUNTIME-NVIDIA] -Maintainer: HIP Support -Provides: hip-nvcc -Replaces: hip-nvcc -Version: 6.2.41134.60202-116~20.04 -Installed-Size: 8 - -Package: hip-runtime-nvidia-rpath6.2.2 -Architecture: amd64 -Depends: cuda (>= 7.5), rocm-core-rpath6.2.2, hipcc-nvidia-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/hip-runtime-nvidia-rpath6.2.2/hip-runtime-nvidia-rpath6.2.2_6.2.41134.60202-116~20.04_amd64.deb -Size: 916 -SHA256: da4627f0e65e77f25a83b0d325d1e266c6f7c5eed2321ad151852c327cf63903 -SHA1: e91fc21d85537b54366c1fecc437d26eb8d42b3f -MD5sum: 15bd4698e53ec5df0be4fc3478b3d239 -Description: HIP:Heterogenous-computing Interface for Portability - HIP: Heterogenous-computing Interface for Portability [RUNTIME-NVIDIA] -Maintainer: HIP Support -Provides: hip-nvcc-rpath6.2.2 -Replaces: hip-nvcc-rpath6.2.2 -Version: 6.2.41134.60202-116~20.04 -Installed-Size: 8 - -Package: hip-runtime-nvidia6.2.2 -Architecture: amd64 -Depends: cuda (>= 7.5), rocm-core6.2.2, hipcc-nvidia6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/hip-runtime-nvidia6.2.2/hip-runtime-nvidia6.2.2_6.2.41134.60202-116~20.04_amd64.deb -Size: 904 -SHA256: ab577b987460cab11a3559eba7310d564ffd2d30a25526639f96a6aa8313f08c -SHA1: 4b9394d3c8a5cd38e947905898cc060dda4fb470 -MD5sum: e9f03a1c2d88c86d21c9bcc56bb988d5 -Description: HIP:Heterogenous-computing Interface for Portability - HIP: Heterogenous-computing Interface for Portability [RUNTIME-NVIDIA] -Maintainer: HIP Support -Provides: hip-nvcc6.2.2 -Replaces: hip-nvcc6.2.2 -Version: 6.2.41134.60202-116~20.04 -Installed-Size: 8 - -Package: hip-samples -Architecture: amd64 -Depends: rocm-core, hip-dev, hipcc -Priority: optional -Section: devel -Filename: pool/main/h/hip-samples/hip-samples_6.2.41134.60202-116~20.04_amd64.deb -Size: 51330 -SHA256: 4b49f2cebb134db46a87ef8514d63574d9b2e3361647d89699635939833d602f -SHA1: cedea8a6f9ff345bea29253f278ff8c43faf6598 -MD5sum: e705b4557375c27ba9ff8e51cb4dc127 -Description: HIP: Heterogenous-computing Interface for Portability [HIP SAMPLES] - HIP: - Heterogenous-computing Interface for Portability [HIP SAMPLES] -Maintainer: HIP Support -Provides: hip-samples -Version: 6.2.41134.60202-116~20.04 -Installed-Size: 461 - -Package: hip-samples-rpath6.2.2 -Architecture: amd64 -Depends: rocm-core-rpath6.2.2, hip-dev-rpath6.2.2, hipcc-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/hip-samples-rpath6.2.2/hip-samples-rpath6.2.2_6.2.41134.60202-116~20.04_amd64.deb -Size: 38912 -SHA256: f315226e740cb962a40f4336fcb8bf0f5a52b239a8619497bf23ba182e66d236 -SHA1: 7d0b2397203ee5087b80d0edd62208bf4be76729 -MD5sum: ba6b3f7acbd9d0b73cdd0a5047936c3f -Description: HIP: Heterogenous-computing Interface for Portability [HIP SAMPLES] - HIP: - Heterogenous-computing Interface for Portability [HIP SAMPLES] -Maintainer: HIP Support -Provides: hip-samples-rpath6.2.2 -Version: 6.2.41134.60202-116~20.04 -Installed-Size: 461 - -Package: hip-samples6.2.2 -Architecture: amd64 -Depends: rocm-core6.2.2, hip-dev6.2.2, hipcc6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/hip-samples6.2.2/hip-samples6.2.2_6.2.41134.60202-116~20.04_amd64.deb -Size: 38916 -SHA256: 36e09abb304ba331b493efc43f41a22ccae171662593c8379683a0155657a224 -SHA1: 284273e309e646cebd6bdb525166be822e8509fb -MD5sum: fccf128eec72130a71fb78a01058f6f6 -Description: HIP: Heterogenous-computing Interface for Portability [HIP SAMPLES] - HIP: - Heterogenous-computing Interface for Portability [HIP SAMPLES] -Maintainer: HIP Support -Provides: hip-samples6.2.2 -Version: 6.2.41134.60202-116~20.04 -Installed-Size: 461 - -Package: hipblas -Architecture: amd64 -Depends: rocblas (>= 4.2.0), rocsolver (>= 3.26.0), rocm-core -Priority: optional -Section: devel -Filename: pool/main/h/hipblas/hipblas_2.2.0.60202-116~20.04_amd64.deb -Size: 134530 -SHA256: 5dd490b88214cba7c4d2a0c9a4eddefa608870ee494f21d3a588949287f3acba -SHA1: dd49da385b47a726d26f846d8367ec596f947fd9 -MD5sum: 1d3be0c44eff9838a62776a9c2c468b9 -Description: Radeon Open Compute BLAS marshalling library -Maintainer: hipBLAS Maintainer -Recommends: hipblas-dev (>=2.2.0.60202) -Version: 2.2.0.60202-116~20.04 -Installed-Size: 2090 - -Package: hipblas-asan -Architecture: amd64 -Depends: rocblas (>= 4.2.0), rocsolver (>= 3.26.0), rocm-core-asan -Priority: optional -Section: devel -Filename: pool/main/h/hipblas-asan/hipblas-asan_2.2.0.60202-116~20.04_amd64.deb -Size: 215342 -SHA256: a6aac687c50bf44eacada4c45fd02b1a27dd71bd9167c16d160e6151e9ce4ef7 -SHA1: af01543fbee42a4eb19c0f0eb29daa4b28023633 -MD5sum: 7d787719c1dc9a6b2d7aa01c58258d7a -Description: Radeon Open Compute BLAS marshalling library -Maintainer: hipBLAS Maintainer -Recommends: hipblas-asan-dev (>=2.2.0.60202) -Version: 2.2.0.60202-116~20.04 -Installed-Size: 4850 - -Package: hipblas-asan-dbgsym -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 3babd0aecde14c20ce237f7892725bcb72ab085f -Depends: hipblas-asan (= 2.2.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/h/hipblas-asan-dbgsym/hipblas-asan-dbgsym_2.2.0.60202-116~20.04_amd64.deb -Size: 636452 -SHA256: a3555bd29dbc026c1150cdb83da03e20fab5af4f8f36d38d5d82de2a466fcbce -SHA1: 05264b7c440a5d653ccf7336c08901a861c52269 -MD5sum: bb64dfe1d27ef797828a8258b9c4959f -Description: debug symbols for hipblas-asan -Maintainer: hipBLAS Maintainer -Package-Type: ddeb -Version: 2.2.0.60202-116~20.04 -Installed-Size: 1202 - -Package: hipblas-asan-dbgsym-rpath6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 3babd0aecde14c20ce237f7892725bcb72ab085f -Depends: hipblas-asan-rpath6.2.2 (= 2.2.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/h/hipblas-asan-dbgsym-rpath6.2.2/hipblas-asan-dbgsym-rpath6.2.2_2.2.0.60202-116~20.04_amd64.deb -Size: 636392 -SHA256: d66494819a0b8e8e7ba9738bd348c3c55b22a070b911ffa51a0ffe4c17368fe9 -SHA1: cc7d44be11cb72c5b0aac345768232cac1f2bd8d -MD5sum: 0135b30d3850e6de57ff0936d1e42167 -Description: debug symbols for hipblas-asan -Maintainer: hipBLAS Maintainer -Package-Type: ddeb -Version: 2.2.0.60202-116~20.04 -Installed-Size: 1202 - -Package: hipblas-asan-dbgsym6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 3babd0aecde14c20ce237f7892725bcb72ab085f -Depends: hipblas-asan6.2.2 (= 2.2.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/h/hipblas-asan-dbgsym6.2.2/hipblas-asan-dbgsym6.2.2_2.2.0.60202-116~20.04_amd64.deb -Size: 637280 -SHA256: 9c975dc6ef690bee5cd67030a2bb842da5e55675b0db47bae22b38dd5f2c493a -SHA1: 7af773bca8544859dbd4e39f5e3510867d8e9b4e -MD5sum: d3a1d165f6a5f1fdd63dfed3da755ee4 -Description: debug symbols for hipblas-asan -Maintainer: hipBLAS Maintainer -Package-Type: ddeb -Version: 2.2.0.60202-116~20.04 -Installed-Size: 1202 - -Package: hipblas-asan-rpath6.2.2 -Architecture: amd64 -Depends: rocblas (>= 4.2.0), rocsolver (>= 3.26.0), rocm-core-asan-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/hipblas-asan-rpath6.2.2/hipblas-asan-rpath6.2.2_2.2.0.60202-116~20.04_amd64.deb -Size: 215080 -SHA256: c2f0a48bff54ec3512198c40a328f36e9f0a42bf13af342909a0064ff786fa0c -SHA1: 507d9b3ab9a1a8595a815b4be17f564e77b2bba3 -MD5sum: c9e5e6fa22d72bc9e65be3cbab69a8b7 -Description: Radeon Open Compute BLAS marshalling library -Maintainer: hipBLAS Maintainer -Recommends: hipblas-asan-dev (>=2.2.0.60202) -Version: 2.2.0.60202-116~20.04 -Installed-Size: 4850 - -Package: hipblas-asan6.2.2 -Architecture: amd64 -Depends: rocblas (>= 4.2.0), rocsolver (>= 3.26.0), rocm-core-asan6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/hipblas-asan6.2.2/hipblas-asan6.2.2_2.2.0.60202-116~20.04_amd64.deb -Size: 214744 -SHA256: 13212b8058d5356546d131f20406172840a4e8c7fd70e039b2f4fef119c6eb0d -SHA1: 8c0183a1d951cf7cb1717b3c6dc411b80409c0b5 -MD5sum: 842569f53e02dfb51e01d013a77464d4 -Description: Radeon Open Compute BLAS marshalling library -Maintainer: hipBLAS Maintainer -Recommends: hipblas-asan-dev (>=2.2.0.60202) -Version: 2.2.0.60202-116~20.04 -Installed-Size: 4850 - -Package: hipblas-dbgsym -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: eb99e1e65a906b3d017151d7c5d149e038ed188f -Depends: hipblas (= 2.2.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/h/hipblas-dbgsym/hipblas-dbgsym_2.2.0.60202-116~20.04_amd64.deb -Size: 706714 -SHA256: 15add6b537b773b6ebc74d4b90a6bf18fef5a42a2c3233529f1702ec6a0d7bb5 -SHA1: ef3e9b073c2040dbd48cfd63175364d0a331b771 -MD5sum: a7a152079082aea713e7c06ea36dca42 -Description: debug symbols for hipblas -Maintainer: hipBLAS Maintainer -Package-Type: ddeb -Version: 2.2.0.60202-116~20.04 -Installed-Size: 5208 - -Package: hipblas-dbgsym-rpath6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: eb99e1e65a906b3d017151d7c5d149e038ed188f -Depends: hipblas-rpath6.2.2 (= 2.2.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/h/hipblas-dbgsym-rpath6.2.2/hipblas-dbgsym-rpath6.2.2_2.2.0.60202-116~20.04_amd64.deb -Size: 703676 -SHA256: a0be30abcb54e769d63ae5cdd160517d6ae3cc7812b5b9a3b198ef7dcb657bac -SHA1: f8d3d4432da23f3d0c700e12a3005cdb2f8d49d1 -MD5sum: 7d1b8f3ded781f2f15f52bd7bd6ae750 -Description: debug symbols for hipblas -Maintainer: hipBLAS Maintainer -Package-Type: ddeb -Version: 2.2.0.60202-116~20.04 -Installed-Size: 5208 - -Package: hipblas-dbgsym6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: eb99e1e65a906b3d017151d7c5d149e038ed188f -Depends: hipblas6.2.2 (= 2.2.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/h/hipblas-dbgsym6.2.2/hipblas-dbgsym6.2.2_2.2.0.60202-116~20.04_amd64.deb -Size: 703532 -SHA256: f0cd5f503afb75b9e991301c4437002d0c3a23afa0a36e6d84eb6486828ee20e -SHA1: fed9ae16bd041949a4874a725bad7689bda2cc14 -MD5sum: 5a9291b1824831b7df659a8c6ea66301 -Description: debug symbols for hipblas -Maintainer: hipBLAS Maintainer -Package-Type: ddeb -Version: 2.2.0.60202-116~20.04 -Installed-Size: 5208 - -Package: hipblas-dev -Architecture: amd64 -Depends: hipblas (>= 2.2.0.60202) -Priority: optional -Section: devel -Filename: pool/main/h/hipblas-dev/hipblas-dev_2.2.0.60202-116~20.04_amd64.deb -Size: 89416 -SHA256: 25ecec4bf2e9bbe2d60136641132afc49e6be06d1a83f6883c52d13885a46002 -SHA1: 7ecb381c8d89f0b8f977145ee6531e6c3218cbc9 -MD5sum: 9db2de104fbfcd2ec77938bda09fc997 -Description: Radeon Open Compute BLAS marshalling library -Maintainer: hipBLAS Maintainer -Version: 2.2.0.60202-116~20.04 -Installed-Size: 2784 - -Package: hipblas-dev-rpath6.2.2 -Architecture: amd64 -Depends: hipblas-rpath6.2.2 (>= 2.2.0.60202) -Priority: optional -Section: devel -Filename: pool/main/h/hipblas-dev-rpath6.2.2/hipblas-dev-rpath6.2.2_2.2.0.60202-116~20.04_amd64.deb -Size: 89624 -SHA256: 34737a96ca2e099e3c84646310c4f450ce9835f436fc0ee72d22f43f2eb62261 -SHA1: da44632441be319455c1f7e504d0442373f4f2e3 -MD5sum: 05c2f9108977f8d4f5de7881e31714a5 -Description: Radeon Open Compute BLAS marshalling library -Maintainer: hipBLAS Maintainer -Version: 2.2.0.60202-116~20.04 -Installed-Size: 2784 - -Package: hipblas-dev6.2.2 -Architecture: amd64 -Depends: hipblas6.2.2 (>= 2.2.0.60202) -Priority: optional -Section: devel -Filename: pool/main/h/hipblas-dev6.2.2/hipblas-dev6.2.2_2.2.0.60202-116~20.04_amd64.deb -Size: 89668 -SHA256: 2cc54906cc9bdecebbacd6d4c5c4aa0f3f6e63e6e272f70267b1c78331f05eb8 -SHA1: b2c89b73dd8fc4ae8783356a65947cec45c8a4a6 -MD5sum: 1ded48be4a8e7377853210819765dfe1 -Description: Radeon Open Compute BLAS marshalling library -Maintainer: hipBLAS Maintainer -Version: 2.2.0.60202-116~20.04 -Installed-Size: 2784 - -Package: hipblas-rpath6.2.2 -Architecture: amd64 -Depends: rocblas-rpath6.2.2 (>= 4.2.0), rocsolver-rpath6.2.2 (>= 3.26.0), rocm-core-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/hipblas-rpath6.2.2/hipblas-rpath6.2.2_2.2.0.60202-116~20.04_amd64.deb -Size: 132892 -SHA256: 3a651c331a77f55859a079caa78376d1d008d5f67f322ac2c920f4aa5358bced -SHA1: 8bbc4dcc9f9e81fa697fa7613c2ecf2cec1dc9d1 -MD5sum: ee43e8978dc2a478505b04d0811f2674 -Description: Radeon Open Compute BLAS marshalling library -Maintainer: hipBLAS Maintainer -Recommends: hipblas-dev-rpath6.2.2 (>=2.2.0.60202) -Version: 2.2.0.60202-116~20.04 -Installed-Size: 2090 - -Package: hipblas6.2.2 -Architecture: amd64 -Depends: rocblas6.2.2 (>= 4.2.0), rocsolver6.2.2 (>= 3.26.0), rocm-core6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/hipblas6.2.2/hipblas6.2.2_2.2.0.60202-116~20.04_amd64.deb -Size: 134752 -SHA256: 207fe52dfb750da3add2f3c40e3e9f80aa6ce39d49e93458d830786f180741d0 -SHA1: 9bf61d0ec17743640459af9503454cf16066b907 -MD5sum: f4b400f1235740b2773aa468a91f8bd5 -Description: Radeon Open Compute BLAS marshalling library -Maintainer: hipBLAS Maintainer -Recommends: hipblas-dev6.2.2 (>=2.2.0.60202) -Version: 2.2.0.60202-116~20.04 -Installed-Size: 2090 - -Package: hipblaslt -Architecture: amd64 -Depends: hipblas (>= 0.50.0), rocm-core -Priority: optional -Section: devel -Filename: pool/main/h/hipblaslt/hipblaslt_0.8.0.60202-116~20.04_amd64.deb -Size: 994274 -SHA256: 24751862eebcb346b87fff2fd26565ce7d04d8c847f9bd927e1d13afc6a4f581 -SHA1: 1348a47e9d48e8d30256f190b33e29c3ecdfa609 -MD5sum: 82ae5fdf5324234f2a47e811131ed623 -Description: Radeon Open Compute BLAS marshalling library -Maintainer: hipBLASLt Maintainer -Recommends: hipblaslt-dev (>=0.8.0.60202) -Version: 0.8.0.60202-116~20.04 -Installed-Size: 20243 - -Package: hipblaslt-asan -Architecture: amd64 -Depends: hipblas (>= 0.50.0), rocm-core-asan -Priority: optional -Section: devel -Filename: pool/main/h/hipblaslt-asan/hipblaslt-asan_0.8.0.60202-116~20.04_amd64.deb -Size: 18059948 -SHA256: ad922f8aff6dec0228ce12c485e2e950a4f7cd87f71ae8d31a4555cfa82acd94 -SHA1: f4dd183498e432948375562c8e57e7075f03465b -MD5sum: 23fe7e66af4c046a00d3f171faa1f0c9 -Description: Radeon Open Compute BLAS marshalling library -Maintainer: hipBLASLt Maintainer -Recommends: hipblaslt-asan-dev (>=0.8.0.60202) -Version: 0.8.0.60202-116~20.04 -Installed-Size: 69407 - -Package: hipblaslt-asan-rpath6.2.2 -Architecture: amd64 -Depends: hipblas (>= 0.50.0), rocm-core-asan-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/hipblaslt-asan-rpath6.2.2/hipblaslt-asan-rpath6.2.2_0.8.0.60202-116~20.04_amd64.deb -Size: 18060020 -SHA256: 34b10a71e9e6c3a20c67bfdcf935be62bb06ed0b0b60f1e6a128487ba972ec1e -SHA1: 71249408a8bbabc902a5bf992675daabaae77227 -MD5sum: fd1ee6824c842dd77c278d78c4f017e7 -Description: Radeon Open Compute BLAS marshalling library -Maintainer: hipBLASLt Maintainer -Recommends: hipblaslt-asan-dev (>=0.8.0.60202) -Version: 0.8.0.60202-116~20.04 -Installed-Size: 69407 - -Package: hipblaslt-asan6.2.2 -Architecture: amd64 -Depends: hipblas (>= 0.50.0), rocm-core-asan6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/hipblaslt-asan6.2.2/hipblaslt-asan6.2.2_0.8.0.60202-116~20.04_amd64.deb -Size: 18059960 -SHA256: 61b33743faa019761b182604e0bb921b26ceda50026ce000bc0f7ffa3f47f315 -SHA1: 399f0252080d1a6c95eeab93058b744024926685 -MD5sum: 8bd506ca868321309666298378d15f6b -Description: Radeon Open Compute BLAS marshalling library -Maintainer: hipBLASLt Maintainer -Recommends: hipblaslt-asan-dev (>=0.8.0.60202) -Version: 0.8.0.60202-116~20.04 -Installed-Size: 69407 - -Package: hipblaslt-dev -Architecture: amd64 -Depends: hipblaslt (>= 0.8.0.60202) -Priority: optional -Section: devel -Filename: pool/main/h/hipblaslt-dev/hipblaslt-dev_0.8.0.60202-116~20.04_amd64.deb -Size: 171387994 -SHA256: 7486bf1129c6333f68e6fd48b1f9b5df9d9f2d7c6f3ec98cc04c886a935f52e8 -SHA1: 659e65ab9af5c874e5abc776dbb0ee67e1b7f409 -MD5sum: 15066f00f1ae21fefb5f2f8fae50809b -Description: Radeon Open Compute BLAS marshalling library -Maintainer: hipBLASLt Maintainer -Version: 0.8.0.60202-116~20.04 -Installed-Size: 4938335 - -Package: hipblaslt-dev-rpath6.2.2 -Architecture: amd64 -Depends: hipblaslt-rpath6.2.2 (>= 0.8.0.60202) -Priority: optional -Section: devel -Filename: pool/main/h/hipblaslt-dev-rpath6.2.2/hipblaslt-dev-rpath6.2.2_0.8.0.60202-116~20.04_amd64.deb -Size: 171288008 -SHA256: 2f441b4ce6b40462203097d16a25392519569258d5cdf4b4f2f4992f1f133f4d -SHA1: 26c50c9b024252dd342ffa980973466828eb0d7c -MD5sum: d023939e6de70e3c21e101fbdc6e8d00 -Description: Radeon Open Compute BLAS marshalling library -Maintainer: hipBLASLt Maintainer -Version: 0.8.0.60202-116~20.04 -Installed-Size: 4938335 - -Package: hipblaslt-dev6.2.2 -Architecture: amd64 -Depends: hipblaslt6.2.2 (>= 0.8.0.60202) -Priority: optional -Section: devel -Filename: pool/main/h/hipblaslt-dev6.2.2/hipblaslt-dev6.2.2_0.8.0.60202-116~20.04_amd64.deb -Size: 171286732 -SHA256: 3a434de1322644862bee15e030983eeaa6e0f25626ac8b7a82f25fd906f0b163 -SHA1: ea91ccb2b767af019d0710f54d6c917e93e1e89a -MD5sum: 93fb778155675e995e8bd03d74b90231 -Description: Radeon Open Compute BLAS marshalling library -Maintainer: hipBLASLt Maintainer -Version: 0.8.0.60202-116~20.04 -Installed-Size: 4938335 - -Package: hipblaslt-rpath6.2.2 -Architecture: amd64 -Depends: hipblas-rpath6.2.2 (>= 0.50.0), rocm-core-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/hipblaslt-rpath6.2.2/hipblaslt-rpath6.2.2_0.8.0.60202-116~20.04_amd64.deb -Size: 995648 -SHA256: 8a8ebd940cdfa230e62f8734c034d0b0f78d529676d89758b526a7f54d581186 -SHA1: 3145fd940a004a5de419773705d740be45c4c21e -MD5sum: 95dbdff627835dc65860950dffd9ccd0 -Description: Radeon Open Compute BLAS marshalling library -Maintainer: hipBLASLt Maintainer -Recommends: hipblaslt-dev-rpath6.2.2 (>=0.8.0.60202) -Version: 0.8.0.60202-116~20.04 -Installed-Size: 20243 - -Package: hipblaslt6.2.2 -Architecture: amd64 -Depends: hipblas6.2.2 (>= 0.50.0), rocm-core6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/hipblaslt6.2.2/hipblaslt6.2.2_0.8.0.60202-116~20.04_amd64.deb -Size: 995504 -SHA256: 619802a36e574228288712ee31f272805c90eb994367ff2e2d7a3f2f690c7ee2 -SHA1: 24c8bc4e4626366019696ea923d3513d9754838d -MD5sum: 1b5a286e8a4379a414188618dfc47190 -Description: Radeon Open Compute BLAS marshalling library -Maintainer: hipBLASLt Maintainer -Recommends: hipblaslt-dev6.2.2 (>=0.8.0.60202) -Version: 0.8.0.60202-116~20.04 -Installed-Size: 20243 - -Package: hipcc -Architecture: amd64 -Depends: perl (>= 5.0), hip-dev, rocm-core, rocm-llvm -Priority: optional -Section: devel -Filename: pool/main/h/hipcc/hipcc_1.1.1.60202-116~20.04_amd64.deb -Size: 231062 -SHA256: c1fae6d1c71c55ce072cf62de4c78b0cb67f5cd5cb2c85217acf0e8676d7109e -SHA1: 236dae6f5496cac69e15f9fa36ddb5a67bb10d06 -MD5sum: 1a9b012d7141f383e90f807cdb4f2630 -Description: hipcc built using CMake -Homepage: https://github.com/ROCm-Developer-Tools/HIPCC -Maintainer: ROCm Compiler Support -Version: 1.1.1.60202-116~20.04 -Installed-Size: 602 - -Package: hipcc-dbgsym -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 5666e2a9aaf860e3f0d84a2fabcac37c4d8d2110 5666e2a9aaf860e3f0d84a2fabcac37c4d8d2110 -Depends: hipcc (= 1.1.1.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/h/hipcc-dbgsym/hipcc-dbgsym_1.1.1.60202-116~20.04_amd64.deb -Size: 1540018 -SHA256: d3680b53d9a7a6336651e0eebd6cae4e046de776c266ce8fb10f47c4d811597a -SHA1: f61513e702cf1bf3c50c95f7d28d1095264687c5 -MD5sum: e27b66a7945fe340833d4fd4a4bbe7eb -Description: debug symbols for hipcc -Maintainer: ROCm Compiler Support -Package-Type: ddeb -Version: 1.1.1.60202-116~20.04 -Installed-Size: 6951 - -Package: hipcc-dbgsym-rpath6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 5666e2a9aaf860e3f0d84a2fabcac37c4d8d2110 5666e2a9aaf860e3f0d84a2fabcac37c4d8d2110 -Depends: hipcc-rpath6.2.2 (= 1.1.1.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/h/hipcc-dbgsym-rpath6.2.2/hipcc-dbgsym-rpath6.2.2_1.1.1.60202-116~20.04_amd64.deb -Size: 1068604 -SHA256: 99bf2230e9d1a82a700580de393e2b89a0ffa9bc090f7a5dcafb973fcaade5c7 -SHA1: 336dc331a14bd38f6015137af55d13204918da8f -MD5sum: 523c5d90bdc6fc02a9a1a9e1c2ad33d5 -Description: debug symbols for hipcc -Maintainer: ROCm Compiler Support -Package-Type: ddeb -Version: 1.1.1.60202-116~20.04 -Installed-Size: 6951 - -Package: hipcc-dbgsym6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 5666e2a9aaf860e3f0d84a2fabcac37c4d8d2110 5666e2a9aaf860e3f0d84a2fabcac37c4d8d2110 -Depends: hipcc6.2.2 (= 1.1.1.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/h/hipcc-dbgsym6.2.2/hipcc-dbgsym6.2.2_1.1.1.60202-116~20.04_amd64.deb -Size: 1068964 -SHA256: 7b5bdccb12877643813fe5f95d604532f61af3f4c0f8d9c3457d209b7a4f66fc -SHA1: b9479a3bd96a0c6c69efedc21aea821e21f28e31 -MD5sum: fa27f545c07a6c2142ce45d8f8f37e12 -Description: debug symbols for hipcc -Maintainer: ROCm Compiler Support -Package-Type: ddeb -Version: 1.1.1.60202-116~20.04 -Installed-Size: 6951 - -Package: hipcc-nvidia -Architecture: amd64 -Depends: perl (>= 5.0), hip-dev, rocm-core -Priority: optional -Section: devel -Filename: pool/main/h/hipcc-nvidia/hipcc-nvidia_1.1.1.60202-116~20.04_amd64.deb -Size: 231066 -SHA256: 4285573933b35bb7bd33e851cc696bf5f6c66098ac5da939658d09fc045687ae -SHA1: 30f390d52a725581577a9c4b0e80d691ecd4f754 -MD5sum: 1936c1fe5151c9234ddde550802cccec -Description: hipcc built using CMake -Homepage: https://github.com/ROCm-Developer-Tools/HIPCC -Maintainer: ROCm Compiler Support -Version: 1.1.1.60202-116~20.04 -Installed-Size: 602 - -Package: hipcc-nvidia-dbgsym -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 5666e2a9aaf860e3f0d84a2fabcac37c4d8d2110 5666e2a9aaf860e3f0d84a2fabcac37c4d8d2110 -Depends: hipcc-nvidia (= 1.1.1.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/h/hipcc-nvidia-dbgsym/hipcc-nvidia-dbgsym_1.1.1.60202-116~20.04_amd64.deb -Size: 1540024 -SHA256: e16b1c2315a93de958639425f556a3694353287e6cd7d143f525eafed72d880c -SHA1: 9e7af5392d8111176c6f103bb781d6f371e2c0f7 -MD5sum: 63b8d80b1b225c2238e0cde0fa6f4074 -Description: debug symbols for hipcc-nvidia -Maintainer: ROCm Compiler Support -Package-Type: ddeb -Version: 1.1.1.60202-116~20.04 -Installed-Size: 6951 - -Package: hipcc-nvidia-dbgsym-rpath6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 5666e2a9aaf860e3f0d84a2fabcac37c4d8d2110 5666e2a9aaf860e3f0d84a2fabcac37c4d8d2110 -Depends: hipcc-nvidia-rpath6.2.2 (= 1.1.1.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/h/hipcc-nvidia-dbgsym-rpath6.2.2/hipcc-nvidia-dbgsym-rpath6.2.2_1.1.1.60202-116~20.04_amd64.deb -Size: 1070020 -SHA256: ac4774cae5771db1be75452f16c76644a180188c739a590e0d49238ad37f9f85 -SHA1: 42f04d8937fab84594cb3903935942ef19bb78a3 -MD5sum: 3a7cb6278a7defc6586bd49a926b5e99 -Description: debug symbols for hipcc-nvidia -Maintainer: ROCm Compiler Support -Package-Type: ddeb -Version: 1.1.1.60202-116~20.04 -Installed-Size: 6951 - -Package: hipcc-nvidia-dbgsym6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 5666e2a9aaf860e3f0d84a2fabcac37c4d8d2110 5666e2a9aaf860e3f0d84a2fabcac37c4d8d2110 -Depends: hipcc-nvidia6.2.2 (= 1.1.1.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/h/hipcc-nvidia-dbgsym6.2.2/hipcc-nvidia-dbgsym6.2.2_1.1.1.60202-116~20.04_amd64.deb -Size: 1069396 -SHA256: 3b50f92c0b87c5ac730d11538f00d4fccff6268d85134d668d4cfa4fb52035f5 -SHA1: 8d8778fb8ce3b61fa63492f5e2f02cad598d77ca -MD5sum: 55941ac0e3a29006e890c0373c7fda5e -Description: debug symbols for hipcc-nvidia -Maintainer: ROCm Compiler Support -Package-Type: ddeb -Version: 1.1.1.60202-116~20.04 -Installed-Size: 6951 - -Package: hipcc-nvidia-rpath6.2.2 -Architecture: amd64 -Depends: perl (>= 5.0), hip-dev-rpath6.2.2, rocm-core-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/hipcc-nvidia-rpath6.2.2/hipcc-nvidia-rpath6.2.2_1.1.1.60202-116~20.04_amd64.deb -Size: 98572 -SHA256: 0e0e8a1b3df0e7ab6b9ba764f388f1d6cbf67df4cc5466ccd4e2aa9255761077 -SHA1: 945f9f2820ed01d5dd2abee09dc75fa402e8cbb4 -MD5sum: c342474baac048179104c40ae92cd0c5 -Description: hipcc built using CMake -Homepage: https://github.com/ROCm-Developer-Tools/HIPCC -Maintainer: ROCm Compiler Support -Version: 1.1.1.60202-116~20.04 -Installed-Size: 602 - -Package: hipcc-nvidia6.2.2 -Architecture: amd64 -Depends: perl (>= 5.0), hip-dev6.2.2, rocm-core6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/hipcc-nvidia6.2.2/hipcc-nvidia6.2.2_1.1.1.60202-116~20.04_amd64.deb -Size: 98568 -SHA256: 011cbaa7abec55ee2b86faf15d5c6f939140cba01a73a2048731cab9c0757faa -SHA1: bb10a33db7c8ffe32453d205d05fe2b6d148244e -MD5sum: 180b3314ca221113f875bbe5c7089205 -Description: hipcc built using CMake -Homepage: https://github.com/ROCm-Developer-Tools/HIPCC -Maintainer: ROCm Compiler Support -Version: 1.1.1.60202-116~20.04 -Installed-Size: 602 - -Package: hipcc-rpath6.2.2 -Architecture: amd64 -Depends: perl (>= 5.0), hip-dev-rpath6.2.2, rocm-core-rpath6.2.2, rocm-llvm-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/hipcc-rpath6.2.2/hipcc-rpath6.2.2_1.1.1.60202-116~20.04_amd64.deb -Size: 98612 -SHA256: 01ae9f26d6735a9209c62304ae64111f5166026099711dd84a57156b894161d2 -SHA1: fe49c35970bedef9df34b5ab234da41eb93801a9 -MD5sum: a57c0a675865e8465379f28423c9cf87 -Description: hipcc built using CMake -Homepage: https://github.com/ROCm-Developer-Tools/HIPCC -Maintainer: ROCm Compiler Support -Version: 1.1.1.60202-116~20.04 -Installed-Size: 602 - -Package: hipcc6.2.2 -Architecture: amd64 -Depends: perl (>= 5.0), hip-dev6.2.2, rocm-core6.2.2, rocm-llvm6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/hipcc6.2.2/hipcc6.2.2_1.1.1.60202-116~20.04_amd64.deb -Size: 98556 -SHA256: 849385b9cb8835907c6bfc11edacf469b9271c0c36c7f704fa889abaef55072e -SHA1: f98843cd1beb3d43f37cff9b63bccadefc59ee84 -MD5sum: 49fd57bf38e944e269e038c455a1f91d -Description: hipcc built using CMake -Homepage: https://github.com/ROCm-Developer-Tools/HIPCC -Maintainer: ROCm Compiler Support -Version: 1.1.1.60202-116~20.04 -Installed-Size: 602 - -Package: hipcub-dev -Architecture: amd64 -Depends: rocprim-dev (>= 2.10.1), rocm-core -Priority: optional -Section: devel -Filename: pool/main/h/hipcub-dev/hipcub-dev_3.2.0.60202-116~20.04_amd64.deb -Size: 73346 -SHA256: 5d8e2d36fe81dea8677f383d54f6bc009e9209ae7bfeb6ad607a41fef3f53b80 -SHA1: c7fd9f8156927c3300add45a08dbfe8c7e64ba46 -MD5sum: 5f2ead12042bb586778b8b153c5d5f2a -Description: hipCUB (rocPRIM backend) -Maintainer: hipcub-maintainer@amd.com -Provides: CPACK_DEBIAN_PACKAGE_PROVIDES, hipcub (= 3.2.0.60202) -Replaces: cub-hip -Version: 3.2.0.60202-116~20.04 -Installed-Size: 1253 - -Package: hipcub-dev-rpath6.2.2 -Architecture: amd64 -Depends: rocprim-dev-rpath6.2.2 (>= 2.10.1), rocm-core-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/hipcub-dev-rpath6.2.2/hipcub-dev-rpath6.2.2_3.2.0.60202-116~20.04_amd64.deb -Size: 73012 -SHA256: 0c5e645fd966a41653d6dc5d829c360328234ae517b65202fecd337884438dfc -SHA1: 83287cebb8101678c101014ef86ef0e369f67129 -MD5sum: de0edc9d9792f251bf9c67cb9d5fc26c -Description: hipCUB (rocPRIM backend) -Maintainer: hipcub-maintainer@amd.com -Provides: CPACK_DEBIAN_PACKAGE_PROVIDES, hipcub-rpath6.2.2 (= 3.2.0.60202) -Replaces: cub-hip-rpath6.2.2 -Version: 3.2.0.60202-116~20.04 -Installed-Size: 1253 - -Package: hipcub-dev6.2.2 -Architecture: amd64 -Depends: rocprim-dev6.2.2 (>= 2.10.1), rocm-core6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/hipcub-dev6.2.2/hipcub-dev6.2.2_3.2.0.60202-116~20.04_amd64.deb -Size: 73016 -SHA256: b9b56570650852dac180e8d097f00387c6036c92fe2840a7621eb291adbd5d2e -SHA1: 9253e9475b381b90bed22db76ca5d7ae16aff891 -MD5sum: b7102c08a61d9feac5649b816d919d0d -Description: hipCUB (rocPRIM backend) -Maintainer: hipcub-maintainer@amd.com -Provides: CPACK_DEBIAN_PACKAGE_PROVIDES, hipcub6.2.2 (= 3.2.0.60202) -Replaces: cub-hip6.2.2 -Version: 3.2.0.60202-116~20.04 -Installed-Size: 1253 - -Package: hipfft -Architecture: amd64 -Depends: rocfft (>= 1.0.21), rocm-core -Priority: optional -Section: devel -Filename: pool/main/h/hipfft/hipfft_1.0.15.60202-116~20.04_amd64.deb -Size: 25048 -SHA256: a3f380fe3ec90fbc2bbaf9e2f2108e035fda8d18381e352f16261f27a222d771 -SHA1: 6974f8ef863df505634d64dd2b9220e3e513441d -MD5sum: 4fce443ba5f41ffa00854ddf2f608ce8 -Description: ROCm FFT marshalling library -Maintainer: hipfft-maintainer@amd.com -Recommends: hipfft-dev (>=1.0.15.60202) -Version: 1.0.15.60202-116~20.04 -Installed-Size: 151 - -Package: hipfft-asan -Architecture: amd64 -Depends: rocfft (>= 1.0.21), rocm-core-asan -Priority: optional -Section: devel -Filename: pool/main/h/hipfft-asan/hipfft-asan_1.0.15.60202-116~20.04_amd64.deb -Size: 45298 -SHA256: 19e16e0fc96447657cfbd1a42079f5de66aa5b1a4fc405f633220f18a56d5aa4 -SHA1: b8bf0e1781a57cc45a169b2b9d96a23b946568c8 -MD5sum: 3601f05a4b7dceef89d162fa68dfa84f -Description: ROCm FFT marshalling library -Maintainer: hipfft-maintainer@amd.com -Recommends: hipfft-asan-dev (>=1.0.15.60202) -Version: 1.0.15.60202-116~20.04 -Installed-Size: 302 - -Package: hipfft-asan-dbgsym -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 03284ab0799a05cd11fa6166be4c247c7371a5ed -Depends: hipfft-asan (= 1.0.15.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/h/hipfft-asan-dbgsym/hipfft-asan-dbgsym_1.0.15.60202-116~20.04_amd64.deb -Size: 176280 -SHA256: 7d0cc20533557256cd7c1f4735ad3a152804d60732d36732b6bb4851f1e079f8 -SHA1: 142cb9f4929e59fd880f2b5891cea40f122ba7d7 -MD5sum: 425333cd840dabd478eeaf1934add69e -Description: debug symbols for hipfft-asan -Maintainer: hipfft-maintainer@amd.com -Package-Type: ddeb -Version: 1.0.15.60202-116~20.04 -Installed-Size: 230 - -Package: hipfft-asan-dbgsym-rpath6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 03284ab0799a05cd11fa6166be4c247c7371a5ed -Depends: hipfft-asan-rpath6.2.2 (= 1.0.15.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/h/hipfft-asan-dbgsym-rpath6.2.2/hipfft-asan-dbgsym-rpath6.2.2_1.0.15.60202-116~20.04_amd64.deb -Size: 176460 -SHA256: 6bab120d359cdf415830e41f274da44c26e76f81b7c7b29c9c8ee4d93246f4cd -SHA1: 90b4b4ec17c13c9265e607636626dca9551abb34 -MD5sum: 1b0e60d21f122be0e67a59886eacdd60 -Description: debug symbols for hipfft-asan -Maintainer: hipfft-maintainer@amd.com -Package-Type: ddeb -Version: 1.0.15.60202-116~20.04 -Installed-Size: 230 - -Package: hipfft-asan-dbgsym6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 03284ab0799a05cd11fa6166be4c247c7371a5ed -Depends: hipfft-asan6.2.2 (= 1.0.15.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/h/hipfft-asan-dbgsym6.2.2/hipfft-asan-dbgsym6.2.2_1.0.15.60202-116~20.04_amd64.deb -Size: 176444 -SHA256: 3682d5e6e33e1897a295c29f75dc4d49e03a094fcf266bd6ea3cbef191248a63 -SHA1: 0a59163ca93f042bf506f330fa0a17af8d739afa -MD5sum: 793e62e3d511de05c5b7a8632bf0011c -Description: debug symbols for hipfft-asan -Maintainer: hipfft-maintainer@amd.com -Package-Type: ddeb -Version: 1.0.15.60202-116~20.04 -Installed-Size: 230 - -Package: hipfft-asan-rpath6.2.2 -Architecture: amd64 -Depends: rocfft (>= 1.0.21), rocm-core-asan-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/hipfft-asan-rpath6.2.2/hipfft-asan-rpath6.2.2_1.0.15.60202-116~20.04_amd64.deb -Size: 45572 -SHA256: 49821a40ad8d71ea0c5b0ce527a8ad2b4c3f94f38c46e3281f4d6d090101e7ec -SHA1: bdfda146d3aca002976ead3b034805a44f550dfe -MD5sum: 803ad095c32a1e7d6e3f5f0f9ef033d1 -Description: ROCm FFT marshalling library -Maintainer: hipfft-maintainer@amd.com -Recommends: hipfft-asan-dev (>=1.0.15.60202) -Version: 1.0.15.60202-116~20.04 -Installed-Size: 302 - -Package: hipfft-asan6.2.2 -Architecture: amd64 -Depends: rocfft (>= 1.0.21), rocm-core-asan6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/hipfft-asan6.2.2/hipfft-asan6.2.2_1.0.15.60202-116~20.04_amd64.deb -Size: 45448 -SHA256: 7113c35e055369a476e45006f1034b6b71c718a437cd889e94b5bf8105011d81 -SHA1: f86300ff7dfd614456011604b4303f0bd13c187f -MD5sum: b87e91fa357338dbbebbdec6873b5d50 -Description: ROCm FFT marshalling library -Maintainer: hipfft-maintainer@amd.com -Recommends: hipfft-asan-dev (>=1.0.15.60202) -Version: 1.0.15.60202-116~20.04 -Installed-Size: 302 - -Package: hipfft-dbgsym -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: b685e0badb0e4eca6d9ecae28edb4ccef036a4f3 -Depends: hipfft (= 1.0.15.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/h/hipfft-dbgsym/hipfft-dbgsym_1.0.15.60202-116~20.04_amd64.deb -Size: 101504 -SHA256: ee66b31fbed6e6d1589bc30e548bd649b8b46e3c5852f1496b39c462f6ed493c -SHA1: 41ac0e3f082c3b78805c2b00aee5345c9c7ec7b2 -MD5sum: 35fc4fd61330234049acc4f6f5e6ceba -Description: debug symbols for hipfft -Maintainer: hipfft-maintainer@amd.com -Package-Type: ddeb -Version: 1.0.15.60202-116~20.04 -Installed-Size: 387 - -Package: hipfft-dbgsym-rpath6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: b685e0badb0e4eca6d9ecae28edb4ccef036a4f3 -Depends: hipfft-rpath6.2.2 (= 1.0.15.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/h/hipfft-dbgsym-rpath6.2.2/hipfft-dbgsym-rpath6.2.2_1.0.15.60202-116~20.04_amd64.deb -Size: 101796 -SHA256: 7c3464b6ef537d0275fcf206fafa7a7315ca74708227431d5fc46c50002e8a2c -SHA1: d407d6359efdfb0499f24c929613861241a5c569 -MD5sum: 4830233036b961032a4b0173cb5a3d30 -Description: debug symbols for hipfft -Maintainer: hipfft-maintainer@amd.com -Package-Type: ddeb -Version: 1.0.15.60202-116~20.04 -Installed-Size: 387 - -Package: hipfft-dbgsym6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: b685e0badb0e4eca6d9ecae28edb4ccef036a4f3 -Depends: hipfft6.2.2 (= 1.0.15.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/h/hipfft-dbgsym6.2.2/hipfft-dbgsym6.2.2_1.0.15.60202-116~20.04_amd64.deb -Size: 101736 -SHA256: 3fb48689b5841e70e3388d357d3fbf25317aee7234e2334865deba5ce7dcf1e5 -SHA1: c8082c8ef94f896e435a0d7d614005fecddde0ab -MD5sum: f4310028015eaaa9a0b76bb8a5ce9712 -Description: debug symbols for hipfft -Maintainer: hipfft-maintainer@amd.com -Package-Type: ddeb -Version: 1.0.15.60202-116~20.04 -Installed-Size: 387 - -Package: hipfft-dev -Architecture: amd64 -Depends: hipfft (>= 1.0.15.60202) -Priority: optional -Section: devel -Filename: pool/main/h/hipfft-dev/hipfft-dev_1.0.15.60202-116~20.04_amd64.deb -Size: 11154 -SHA256: 45f5f41a0d72526fcf3daf7874967e6c7d066bfdc1bfeae56a159b48e6f74150 -SHA1: b979c6adc6e3a40128b4e43c3b21e581d381ff8f -MD5sum: ceebb9aa330f1a951fd02ec562d259e1 -Description: ROCm FFT marshalling library -Maintainer: hipfft-maintainer@amd.com -Version: 1.0.15.60202-116~20.04 -Installed-Size: 90 - -Package: hipfft-dev-rpath6.2.2 -Architecture: amd64 -Depends: hipfft-rpath6.2.2 (>= 1.0.15.60202) -Priority: optional -Section: devel -Filename: pool/main/h/hipfft-dev-rpath6.2.2/hipfft-dev-rpath6.2.2_1.0.15.60202-116~20.04_amd64.deb -Size: 11304 -SHA256: 7487fda025ca62c42498419a2fbec5efcb7c8108e61e915209bbc1d358e41567 -SHA1: 0b891e76127f34430a5f40fbe3556118d5287e28 -MD5sum: aeb3825665c1d76b0cc410cdb6a8b3ee -Description: ROCm FFT marshalling library -Maintainer: hipfft-maintainer@amd.com -Version: 1.0.15.60202-116~20.04 -Installed-Size: 90 - -Package: hipfft-dev6.2.2 -Architecture: amd64 -Depends: hipfft6.2.2 (>= 1.0.15.60202) -Priority: optional -Section: devel -Filename: pool/main/h/hipfft-dev6.2.2/hipfft-dev6.2.2_1.0.15.60202-116~20.04_amd64.deb -Size: 11300 -SHA256: 70d73d653f8691b76b32198d9c12a52b242c9dd4eb2c4f0d3feb5a77bb61b07a -SHA1: 3cf9ae3d2f7234d02968eca52f2210c943bf533a -MD5sum: 156b71d95eb53e442ce172b712013f3d -Description: ROCm FFT marshalling library -Maintainer: hipfft-maintainer@amd.com -Version: 1.0.15.60202-116~20.04 -Installed-Size: 90 - -Package: hipfft-rpath6.2.2 -Architecture: amd64 -Depends: rocfft-rpath6.2.2 (>= 1.0.21), rocm-core-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/hipfft-rpath6.2.2/hipfft-rpath6.2.2_1.0.15.60202-116~20.04_amd64.deb -Size: 25232 -SHA256: 7fa3c5940f92c04408228e984bd2597d59bf7a597fc9c88fe04028ed3a8a5e0b -SHA1: 933609b8f6f10acd3eb32783491ec2c39d948338 -MD5sum: e63d2aa5e679d436c924709291c5936c -Description: ROCm FFT marshalling library -Maintainer: hipfft-maintainer@amd.com -Recommends: hipfft-dev-rpath6.2.2 (>=1.0.15.60202) -Version: 1.0.15.60202-116~20.04 -Installed-Size: 151 - -Package: hipfft6.2.2 -Architecture: amd64 -Depends: rocfft6.2.2 (>= 1.0.21), rocm-core6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/hipfft6.2.2/hipfft6.2.2_1.0.15.60202-116~20.04_amd64.deb -Size: 25208 -SHA256: 7588d82e82b3f27f4ea6b95e9449aed37493377c0900a3bf1b1110d91c9c6b29 -SHA1: 93ccf509e4b67fa3da61b1a6c273b31105df5227 -MD5sum: fd2d36ed1612fb33979653b175b1a6f1 -Description: ROCm FFT marshalling library -Maintainer: hipfft-maintainer@amd.com -Recommends: hipfft-dev6.2.2 (>=1.0.15.60202) -Version: 1.0.15.60202-116~20.04 -Installed-Size: 151 - -Package: hipfort-dev -Architecture: amd64 -Conflicts: hipfort -Depends: hip-runtime-amd (>= 4.5.0), rocm-core -Priority: optional -Section: devel -Filename: pool/main/h/hipfort-dev/hipfort-dev_0.4.0.60202-116~20.04_amd64.deb -Size: 6667782 -SHA256: f98fe59b8e27da23f0cca2bff6b4f0e56af0194d1a0b933603a265bca4a75d73 -SHA1: 5dd08debeb07698af43d0b43044558d70781f580 -MD5sum: b62dd4b6c47eb8d9e43732e860fb9ffc -Description: Fortran Interface For GPU Kernel Libraries -Maintainer: Hipfort maintainer -Provides: CPACK_DEBIAN_PACKAGE_PROVIDES, hipfort (= 0.4.0.60202) -Replaces: hipfort -Version: 0.4.0.60202-116~20.04 -Installed-Size: 88514 - -Package: hipfort-dev-rpath6.2.2 -Architecture: amd64 -Conflicts: hipfort-rpath6.2.2 -Depends: hip-runtime-amd-rpath6.2.2 (>= 4.5.0), rocm-core-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/hipfort-dev-rpath6.2.2/hipfort-dev-rpath6.2.2_0.4.0.60202-116~20.04_amd64.deb -Size: 6669468 -SHA256: 81f912dcd563f8850270a4756aff883296bd05b6ad48c5801c70291cfd3c2a07 -SHA1: 0e07d90ede7ef8254e80270c3050c6badeaac89d -MD5sum: fd5e7310aa3809b59b716741c322d67e -Description: Fortran Interface For GPU Kernel Libraries -Maintainer: Hipfort maintainer -Provides: CPACK_DEBIAN_PACKAGE_PROVIDES, hipfort-rpath6.2.2 (= 0.4.0.60202) -Replaces: hipfort-rpath6.2.2 -Version: 0.4.0.60202-116~20.04 -Installed-Size: 88514 - -Package: hipfort-dev6.2.2 -Architecture: amd64 -Conflicts: hipfort6.2.2 -Depends: hip-runtime-amd6.2.2 (>= 4.5.0), rocm-core6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/hipfort-dev6.2.2/hipfort-dev6.2.2_0.4.0.60202-116~20.04_amd64.deb -Size: 6667776 -SHA256: 0c0f1a02bd683819507567c785225d4851798eaab3b1f383a025e9e6773377a0 -SHA1: 3babf807b2bb683bd7fb34b22857112bc3da491e -MD5sum: 6b87a6d862dc6e06f25c0e2c0abbc2d4 -Description: Fortran Interface For GPU Kernel Libraries -Maintainer: Hipfort maintainer -Provides: CPACK_DEBIAN_PACKAGE_PROVIDES, hipfort6.2.2 (= 0.4.0.60202) -Replaces: hipfort6.2.2 -Version: 0.4.0.60202-116~20.04 -Installed-Size: 88514 - -Package: hipify-clang -Architecture: amd64 -Depends: rocm-core -Priority: optional -Section: devel -Filename: pool/main/h/hipify-clang/hipify-clang_18.0.0.60202-116~20.04_amd64.deb -Size: 21049282 -SHA256: e969771c5bbba984f1bb03cf123ffea02b2b095ca9b34af99003030ec78b0428 -SHA1: e72f385f9c6783b2ec395c7788c78515c96c1923 -MD5sum: d6a6ab4294a2cedb7dfff7cd0d65e0b4 -Description: Hipify CUDA source - This tool translates CUDA source code into portable HIP C++ automatically -Maintainer: ROCm Compiler Support -Version: 18.0.0.60202-116~20.04 -Installed-Size: 53971 - -Package: hipify-clang-rpath6.2.2 -Architecture: amd64 -Depends: rocm-core-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/hipify-clang-rpath6.2.2/hipify-clang-rpath6.2.2_18.0.0.60202-116~20.04_amd64.deb -Size: 14659584 -SHA256: 37240891d8490b6f16d09300d96af93d5055d6d533ea6ba6364f30b96d31bf1d -SHA1: ad44ef4a237d50235c843f62bedddcfbac261658 -MD5sum: 62b5259256f578705d34ee77ffa007ca -Description: Hipify CUDA source - This tool translates CUDA source code into portable HIP C++ automatically -Maintainer: ROCm Compiler Support -Version: 18.0.0.60202-116~20.04 -Installed-Size: 53971 - -Package: hipify-clang6.2.2 -Architecture: amd64 -Depends: rocm-core6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/hipify-clang6.2.2/hipify-clang6.2.2_18.0.0.60202-116~20.04_amd64.deb -Size: 14657860 -SHA256: 9dd2921f64f76da2ee524e96ad6c1e0e60da0553f56307f149cfbbfa43d686eb -SHA1: 36c22ca8db9e8edb14e656588b85ff06708572a8 -MD5sum: 5498db682b916aadbf2b61710a46685b -Description: Hipify CUDA source - This tool translates CUDA source code into portable HIP C++ automatically -Maintainer: ROCm Compiler Support -Version: 18.0.0.60202-116~20.04 -Installed-Size: 53971 - -Package: hiprand -Architecture: amd64 -Depends: rocm-core -Priority: optional -Section: devel -Filename: pool/main/h/hiprand/hiprand_2.11.0.60202-116~20.04_amd64.deb -Size: 5010 -SHA256: 56071d9e2d196320dfadda147f7a54a2a3185f91b7ef8db556682b57db01f936 -SHA1: 33b263c701db4deb3e919916f15e60d395439268 -MD5sum: b804581175ccbce1463c3ac2623b7349 -Description: Radeon Open Compute RAND library -Maintainer: hipRAND Maintainer -Recommends: hiprand-dev (>=2.11.0.60202) -Version: 2.11.0.60202-116~20.04 -Installed-Size: 54 - -Package: hiprand-asan -Architecture: amd64 -Depends: rocm-core-asan -Priority: optional -Section: devel -Filename: pool/main/h/hiprand-asan/hiprand-asan_2.11.0.60202-116~20.04_amd64.deb -Size: 7024 -SHA256: d4de9f14fab0f6acbd3b395c76321a894dbba8e0535b0a0268b66ebf50e44af4 -SHA1: ad6852d35932022f0c3f66be1f3aa49097eb8560 -MD5sum: 8dd2e7f51014128d62a426da1c480fb4 -Description: Radeon Open Compute RAND library -Maintainer: hipRAND Maintainer -Recommends: hiprand-asan-dev (>=2.11.0.60202) -Version: 2.11.0.60202-116~20.04 -Installed-Size: 81 - -Package: hiprand-asan-dbgsym -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 3e0e788c6b0f28e3ce580247f480f34cd0adf599 -Depends: hiprand-asan (= 2.11.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/h/hiprand-asan-dbgsym/hiprand-asan-dbgsym_2.11.0.60202-116~20.04_amd64.deb -Size: 30658 -SHA256: c7a9454aa50738f31c42c2d55653467342d8d5f35c396d557d2d5fe83366c292 -SHA1: 86421a647b82688fcd5c318027cebcedd376415e -MD5sum: ec0eb81dcade92918f0b2bd6c224bd50 -Description: debug symbols for hiprand-asan -Maintainer: hipRAND Maintainer -Package-Type: ddeb -Version: 2.11.0.60202-116~20.04 -Installed-Size: 74 - -Package: hiprand-asan-dbgsym-rpath6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 3e0e788c6b0f28e3ce580247f480f34cd0adf599 -Depends: hiprand-asan-rpath6.2.2 (= 2.11.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/h/hiprand-asan-dbgsym-rpath6.2.2/hiprand-asan-dbgsym-rpath6.2.2_2.11.0.60202-116~20.04_amd64.deb -Size: 30812 -SHA256: add021a036a19623564cf40107996cb4d0a0d6b6a3fe918c73ead802eae21dad -SHA1: e6f53c0b1d06356e1759e00e32e9d4bffc7483bb -MD5sum: 528dc8e52beea41157152e78f7d80b56 -Description: debug symbols for hiprand-asan -Maintainer: hipRAND Maintainer -Package-Type: ddeb -Version: 2.11.0.60202-116~20.04 -Installed-Size: 74 - -Package: hiprand-asan-dbgsym6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 3e0e788c6b0f28e3ce580247f480f34cd0adf599 -Depends: hiprand-asan6.2.2 (= 2.11.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/h/hiprand-asan-dbgsym6.2.2/hiprand-asan-dbgsym6.2.2_2.11.0.60202-116~20.04_amd64.deb -Size: 30800 -SHA256: 503ac0f36106bed5af5878255c5750f9f6ae57f1fe9fd663033c698c17714424 -SHA1: 08eeef8dfc28f34b5e8dafc9da76eed7ded3d925 -MD5sum: 8c2d13fe31b4710928cef7c5ea6fe384 -Description: debug symbols for hiprand-asan -Maintainer: hipRAND Maintainer -Package-Type: ddeb -Version: 2.11.0.60202-116~20.04 -Installed-Size: 74 - -Package: hiprand-asan-rpath6.2.2 -Architecture: amd64 -Depends: rocm-core-asan-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/hiprand-asan-rpath6.2.2/hiprand-asan-rpath6.2.2_2.11.0.60202-116~20.04_amd64.deb -Size: 7216 -SHA256: f01be0a94896a92ac599c29348aaef1e8ec01fba4cc9034018c2072a683acf42 -SHA1: 4146a17f7f889cb6e2016294ec513d4bb0ae2dff -MD5sum: f6928d152e866d0019d407bc9be27884 -Description: Radeon Open Compute RAND library -Maintainer: hipRAND Maintainer -Recommends: hiprand-asan-dev (>=2.11.0.60202) -Version: 2.11.0.60202-116~20.04 -Installed-Size: 81 - -Package: hiprand-asan6.2.2 -Architecture: amd64 -Depends: rocm-core-asan6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/hiprand-asan6.2.2/hiprand-asan6.2.2_2.11.0.60202-116~20.04_amd64.deb -Size: 7216 -SHA256: 2effe215673412409875ce18f0cf8c11da4f1198f5d26fba923a55b8d000efd1 -SHA1: acad109674734c659e1ff8e75c499751f09b66cb -MD5sum: 1b9d182cb75ead8c5a1ca01d932f5b9d -Description: Radeon Open Compute RAND library -Maintainer: hipRAND Maintainer -Recommends: hiprand-asan-dev (>=2.11.0.60202) -Version: 2.11.0.60202-116~20.04 -Installed-Size: 81 - -Package: hiprand-dbgsym -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 7956f482ff0ff40e592f82dd54121daccf4c6742 -Depends: hiprand (= 2.11.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/h/hiprand-dbgsym/hiprand-dbgsym_2.11.0.60202-116~20.04_amd64.deb -Size: 18214 -SHA256: af174f57050a55c409e5998b38735789e1788daff087bcdc77386673f5608669 -SHA1: beaa71a43d2e87061be29715728bdd251b2f3cac -MD5sum: b1c41eafbed1e1b8c7a7d31966b9ec48 -Description: debug symbols for hiprand -Maintainer: hipRAND Maintainer -Package-Type: ddeb -Version: 2.11.0.60202-116~20.04 -Installed-Size: 77 - -Package: hiprand-dbgsym-rpath6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 7956f482ff0ff40e592f82dd54121daccf4c6742 -Depends: hiprand-rpath6.2.2 (= 2.11.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/h/hiprand-dbgsym-rpath6.2.2/hiprand-dbgsym-rpath6.2.2_2.11.0.60202-116~20.04_amd64.deb -Size: 18340 -SHA256: acf688611d5defd5938e5ce2f0fb8026c7bf5d6bbdf870a0c0ce9792ad23c7a7 -SHA1: 64e28c5b88e2c1dc7748d915005343815d4bb5dc -MD5sum: b559d9591a325d42097aaead1e045aef -Description: debug symbols for hiprand -Maintainer: hipRAND Maintainer -Package-Type: ddeb -Version: 2.11.0.60202-116~20.04 -Installed-Size: 77 - -Package: hiprand-dbgsym6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 7956f482ff0ff40e592f82dd54121daccf4c6742 -Depends: hiprand6.2.2 (= 2.11.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/h/hiprand-dbgsym6.2.2/hiprand-dbgsym6.2.2_2.11.0.60202-116~20.04_amd64.deb -Size: 18324 -SHA256: 56a8699f74d5b4bec032d900732a87710ed08e260b206286be93c419de38700e -SHA1: 238f1b0648747e081770ef5e68057335b90213a0 -MD5sum: bfa4bc8fb0df7e044566ea6913b96252 -Description: debug symbols for hiprand -Maintainer: hipRAND Maintainer -Package-Type: ddeb -Version: 2.11.0.60202-116~20.04 -Installed-Size: 77 - -Package: hiprand-dev -Architecture: amd64 -Depends: hiprand (>= 2.11.0.60202) -Priority: optional -Section: devel -Filename: pool/main/h/hiprand-dev/hiprand-dev_2.11.0.60202-116~20.04_amd64.deb -Size: 21152 -SHA256: 86200e35925b658988bf15fb7d1964e220ce44a8b25e65cbcc95b389aad4ebb2 -SHA1: 996fd66a37459eb314cdce8db18b337bc976f3b9 -MD5sum: 5c6ae528c260db088d5d3a8e9e1700a2 -Description: Radeon Open Compute RAND library -Maintainer: hipRAND Maintainer -Version: 2.11.0.60202-116~20.04 -Installed-Size: 209 - -Package: hiprand-dev-rpath6.2.2 -Architecture: amd64 -Depends: hiprand-rpath6.2.2 (>= 2.11.0.60202) -Priority: optional -Section: devel -Filename: pool/main/h/hiprand-dev-rpath6.2.2/hiprand-dev-rpath6.2.2_2.11.0.60202-116~20.04_amd64.deb -Size: 21284 -SHA256: 3f8e8bda103ec7b61c6b2cfb1672e28504ea23bcaed6a3402d1cfcbcc77fa3d4 -SHA1: 6f2e49f4f5624fcea24b1b78660007ee64166c4f -MD5sum: 13901163c5e29b26f95ac44ff1654462 -Description: Radeon Open Compute RAND library -Maintainer: hipRAND Maintainer -Version: 2.11.0.60202-116~20.04 -Installed-Size: 209 - -Package: hiprand-dev6.2.2 -Architecture: amd64 -Depends: hiprand6.2.2 (>= 2.11.0.60202) -Priority: optional -Section: devel -Filename: pool/main/h/hiprand-dev6.2.2/hiprand-dev6.2.2_2.11.0.60202-116~20.04_amd64.deb -Size: 21284 -SHA256: 1886446172f6b965501e900bd31ab85461fbf7e197c95904d1d4869d1aa170ac -SHA1: f7a9778314e278f2a4f45c167b30a00ef1587c14 -MD5sum: d67cf206c746e3bf5319766cebe128f2 -Description: Radeon Open Compute RAND library -Maintainer: hipRAND Maintainer -Version: 2.11.0.60202-116~20.04 -Installed-Size: 209 - -Package: hiprand-rpath6.2.2 -Architecture: amd64 -Depends: rocm-core-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/hiprand-rpath6.2.2/hiprand-rpath6.2.2_2.11.0.60202-116~20.04_amd64.deb -Size: 5164 -SHA256: 587c7717d5b6c144c8caab3558f1e0397b69c540abeaa53317554db34d3d3413 -SHA1: 38d457819ff26a43e121808aabc5fce07f87111b -MD5sum: 833fc7826035fc102a6b89f967c56fc7 -Description: Radeon Open Compute RAND library -Maintainer: hipRAND Maintainer -Recommends: hiprand-dev-rpath6.2.2 (>=2.11.0.60202) -Version: 2.11.0.60202-116~20.04 -Installed-Size: 54 - -Package: hiprand6.2.2 -Architecture: amd64 -Depends: rocm-core6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/hiprand6.2.2/hiprand6.2.2_2.11.0.60202-116~20.04_amd64.deb -Size: 5156 -SHA256: fa234de790bb525d4a7a6dc6ea1de788b69cf9e739baf2a41a5be865e4f30e1b -SHA1: f33cf37c1e4512d5e51a874450e4b0da5c993f76 -MD5sum: 1b6c4ffddef168d5fb6c001341bc99ad -Description: Radeon Open Compute RAND library -Maintainer: hipRAND Maintainer -Recommends: hiprand-dev6.2.2 (>=2.11.0.60202) -Version: 2.11.0.60202-116~20.04 -Installed-Size: 54 - -Package: hipsolver -Architecture: amd64 -Depends: rocblas (>= 4.2.0), rocsolver (>= 3.26.0), rocsparse (>= 2.3.0), libcholmod3, libsuitesparseconfig5, rocm-core -Priority: optional -Section: devel -Filename: pool/main/h/hipsolver/hipsolver_2.2.0.60202-116~20.04_amd64.deb -Size: 56286 -SHA256: 3feaa120ebb7ebbccf397b02590ef1afcf0a8db59c0f588e2abb7c6df52573e5 -SHA1: 34d373fac67c809b1f69312a3f7a84cf258d96f8 -MD5sum: 3d0bd719b2bb8a561f24edf395f11deb -Description: Radeon Open Compute LAPACK marshalling library -Maintainer: hipSOLVER Maintainer -Recommends: hipsolver-dev (>=2.2.0.60202) -Version: 2.2.0.60202-116~20.04 -Installed-Size: 715 - -Package: hipsolver-asan -Architecture: amd64 -Depends: rocblas (>= 4.2.0), rocsolver (>= 3.26.0), rocsparse (>= 2.3.0), libcholmod3, libsuitesparseconfig5, rocm-core-asan -Priority: optional -Section: devel -Filename: pool/main/h/hipsolver-asan/hipsolver-asan_2.2.0.60202-116~20.04_amd64.deb -Size: 78042 -SHA256: 3e24594b883c1ddbc47563d390815079f5055fbd9dfa8812eb53b58b8539da27 -SHA1: 4e7fad90393ae1fcc37e1570e2b4e631cdfe2026 -MD5sum: dceccab90cbde0ee0e878dc4f6589e02 -Description: Radeon Open Compute LAPACK marshalling library -Maintainer: hipSOLVER Maintainer -Recommends: hipsolver-asan-dev (>=2.2.0.60202) -Version: 2.2.0.60202-116~20.04 -Installed-Size: 900 - -Package: hipsolver-asan-dbgsym -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 064cb87f7499b241f15e49bce029c597f13a735e -Depends: hipsolver-asan (= 2.2.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/h/hipsolver-asan-dbgsym/hipsolver-asan-dbgsym_2.2.0.60202-116~20.04_amd64.deb -Size: 288266 -SHA256: f6ddedfda331d211347b89bcb0eb8473877341a66620231f4c4d0c7c163339a6 -SHA1: 1efb5c2f8fe36fecb9c1b0bf75a5117c45e0e0b4 -MD5sum: 8e13f5728f926ac584b0fcb37a159171 -Description: debug symbols for hipsolver-asan -Maintainer: hipSOLVER Maintainer -Package-Type: ddeb -Version: 2.2.0.60202-116~20.04 -Installed-Size: 373 - -Package: hipsolver-asan-dbgsym-rpath6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 064cb87f7499b241f15e49bce029c597f13a735e -Depends: hipsolver-asan-rpath6.2.2 (= 2.2.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/h/hipsolver-asan-dbgsym-rpath6.2.2/hipsolver-asan-dbgsym-rpath6.2.2_2.2.0.60202-116~20.04_amd64.deb -Size: 288260 -SHA256: 7e136fa9502b07348c9f5fac00d175e6d120e68b866ea39619abbe97560feb40 -SHA1: 932e3c435579bee270402c1d938f8c412490f6f6 -MD5sum: 27d416a0fd36ad09dd381f3fa83defbc -Description: debug symbols for hipsolver-asan -Maintainer: hipSOLVER Maintainer -Package-Type: ddeb -Version: 2.2.0.60202-116~20.04 -Installed-Size: 373 - -Package: hipsolver-asan-dbgsym6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 064cb87f7499b241f15e49bce029c597f13a735e -Depends: hipsolver-asan6.2.2 (= 2.2.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/h/hipsolver-asan-dbgsym6.2.2/hipsolver-asan-dbgsym6.2.2_2.2.0.60202-116~20.04_amd64.deb -Size: 288332 -SHA256: 3ed6f9c0b36b3d204c6a836487a92a651eeacf2dccbd9eee374a9411af605ad4 -SHA1: 2a6db633417d61352b00c49ac8a236788579644c -MD5sum: f1218255437195d610c5c08e0384541f -Description: debug symbols for hipsolver-asan -Maintainer: hipSOLVER Maintainer -Package-Type: ddeb -Version: 2.2.0.60202-116~20.04 -Installed-Size: 373 - -Package: hipsolver-asan-rpath6.2.2 -Architecture: amd64 -Depends: rocblas (>= 4.2.0), rocsolver (>= 3.26.0), rocsparse (>= 2.3.0), libcholmod3, libsuitesparseconfig5, rocm-core-asan-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/hipsolver-asan-rpath6.2.2/hipsolver-asan-rpath6.2.2_2.2.0.60202-116~20.04_amd64.deb -Size: 77872 -SHA256: 4c75c7c2bd5b325c2179352b75586a0577127edec426861b31ee8dbdfdc7e9f7 -SHA1: c15d319d354535c0ba52a8b16972f5f67fcca09e -MD5sum: 1a501d2ee40caf2614e4678d0eb8df6d -Description: Radeon Open Compute LAPACK marshalling library -Maintainer: hipSOLVER Maintainer -Recommends: hipsolver-asan-dev (>=2.2.0.60202) -Version: 2.2.0.60202-116~20.04 -Installed-Size: 900 - -Package: hipsolver-asan6.2.2 -Architecture: amd64 -Depends: rocblas (>= 4.2.0), rocsolver (>= 3.26.0), rocsparse (>= 2.3.0), libcholmod3, libsuitesparseconfig5, rocm-core-asan6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/hipsolver-asan6.2.2/hipsolver-asan6.2.2_2.2.0.60202-116~20.04_amd64.deb -Size: 78308 -SHA256: c2be45246431e27ddad1d634977f7a4f161b399b704c76c415d134792047c54c -SHA1: ec1d0ff85d08b794ca9f526ca0b5be0173da5c7a -MD5sum: 77267d23bcff3020f1baf51929a71ed2 -Description: Radeon Open Compute LAPACK marshalling library -Maintainer: hipSOLVER Maintainer -Recommends: hipsolver-asan-dev (>=2.2.0.60202) -Version: 2.2.0.60202-116~20.04 -Installed-Size: 900 - -Package: hipsolver-dbgsym -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: d7a5177b35e72e8b711b6902d895887f371d0e4a -Depends: hipsolver (= 2.2.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/h/hipsolver-dbgsym/hipsolver-dbgsym_2.2.0.60202-116~20.04_amd64.deb -Size: 189338 -SHA256: 4fb3c87a2a0e24b9660450b66e382f604ebbd926386696fd435200b5fcbe9b61 -SHA1: 1bba95fd1f90df236f7aaf2fb99a0ce696b3eac3 -MD5sum: ed839de89fdcf6440bf6492e592314f0 -Description: debug symbols for hipsolver -Maintainer: hipSOLVER Maintainer -Package-Type: ddeb -Version: 2.2.0.60202-116~20.04 -Installed-Size: 756 - -Package: hipsolver-dbgsym-rpath6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: d7a5177b35e72e8b711b6902d895887f371d0e4a -Depends: hipsolver-rpath6.2.2 (= 2.2.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/h/hipsolver-dbgsym-rpath6.2.2/hipsolver-dbgsym-rpath6.2.2_2.2.0.60202-116~20.04_amd64.deb -Size: 189260 -SHA256: d53febe9a3e23648bd0003182bb9ec23a7e7d6a5515b7fd87bf44449134ef35a -SHA1: 841aa1d7c49934b68d130f2c7a83d9022f852dff -MD5sum: e1482dbf56f39060fe74db7328466ca3 -Description: debug symbols for hipsolver -Maintainer: hipSOLVER Maintainer -Package-Type: ddeb -Version: 2.2.0.60202-116~20.04 -Installed-Size: 756 - -Package: hipsolver-dbgsym6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: d7a5177b35e72e8b711b6902d895887f371d0e4a -Depends: hipsolver6.2.2 (= 2.2.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/h/hipsolver-dbgsym6.2.2/hipsolver-dbgsym6.2.2_2.2.0.60202-116~20.04_amd64.deb -Size: 189256 -SHA256: f0ae9f07a84b02cf0ded479017aa0f93c05822365895bc907f61d3f08827415c -SHA1: d97781bbc75c6657b16c010513b742284e311ce2 -MD5sum: 9346aea17561224af5b0a328cf244916 -Description: debug symbols for hipsolver -Maintainer: hipSOLVER Maintainer -Package-Type: ddeb -Version: 2.2.0.60202-116~20.04 -Installed-Size: 756 - -Package: hipsolver-dev -Architecture: amd64 -Depends: hipsolver (>= 2.2.0.60202) -Priority: optional -Section: devel -Filename: pool/main/h/hipsolver-dev/hipsolver-dev_2.2.0.60202-116~20.04_amd64.deb -Size: 18686 -SHA256: 3b11f1f95d4db9d57a89b231c7ed41687be7a3f629dd2952dfbff6ee3fa523a7 -SHA1: 8ab2c3076f4d78f03ea34647d6d3257337a91c57 -MD5sum: cb17ce5a1bf492a9a94da54e77b5784b -Description: Radeon Open Compute LAPACK marshalling library -Maintainer: hipSOLVER Maintainer -Version: 2.2.0.60202-116~20.04 -Installed-Size: 460 - -Package: hipsolver-dev-rpath6.2.2 -Architecture: amd64 -Depends: hipsolver-rpath6.2.2 (>= 2.2.0.60202) -Priority: optional -Section: devel -Filename: pool/main/h/hipsolver-dev-rpath6.2.2/hipsolver-dev-rpath6.2.2_2.2.0.60202-116~20.04_amd64.deb -Size: 18816 -SHA256: 4ff2560201f782abdc441d2362d31e4038d6353f20742a162f7fcf4ca4dada36 -SHA1: a33ac69f29faa26a2cf4c5b8d1a871d51c295d88 -MD5sum: bbc1e5d4cfe8b5c59c570c118b50dd61 -Description: Radeon Open Compute LAPACK marshalling library -Maintainer: hipSOLVER Maintainer -Version: 2.2.0.60202-116~20.04 -Installed-Size: 460 - -Package: hipsolver-dev6.2.2 -Architecture: amd64 -Depends: hipsolver6.2.2 (>= 2.2.0.60202) -Priority: optional -Section: devel -Filename: pool/main/h/hipsolver-dev6.2.2/hipsolver-dev6.2.2_2.2.0.60202-116~20.04_amd64.deb -Size: 18820 -SHA256: 7317cba16363e35363595178a918220ffe890eaa0c8d8261995176bb0c01685b -SHA1: 7e662a5cb1dcc7a3e2ad7e2b5cd9bb54e60345e2 -MD5sum: dae98e2a9618570e498762fb83c41b6d -Description: Radeon Open Compute LAPACK marshalling library -Maintainer: hipSOLVER Maintainer -Version: 2.2.0.60202-116~20.04 -Installed-Size: 460 - -Package: hipsolver-rpath6.2.2 -Architecture: amd64 -Depends: rocblas-rpath6.2.2 (>= 4.2.0), rocsolver-rpath6.2.2 (>= 3.26.0), rocsparse-rpath6.2.2 (>= 2.3.0), libcholmod3, libsuitesparseconfig5, rocm-core-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/hipsolver-rpath6.2.2/hipsolver-rpath6.2.2_2.2.0.60202-116~20.04_amd64.deb -Size: 56848 -SHA256: bb3655c1b24189006d8b90839ce057595c9709664eb715e99ef7a87cf1ba02d1 -SHA1: 4a9e87cb56faab6b5f2ce81b8a98ce1abe2e6183 -MD5sum: d38e085017af916ae920ac4743ecb10d -Description: Radeon Open Compute LAPACK marshalling library -Maintainer: hipSOLVER Maintainer -Recommends: hipsolver-dev-rpath6.2.2 (>=2.2.0.60202) -Version: 2.2.0.60202-116~20.04 -Installed-Size: 715 - -Package: hipsolver6.2.2 -Architecture: amd64 -Depends: rocblas6.2.2 (>= 4.2.0), rocsolver6.2.2 (>= 3.26.0), rocsparse6.2.2 (>= 2.3.0), libcholmod3, libsuitesparseconfig5, rocm-core6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/hipsolver6.2.2/hipsolver6.2.2_2.2.0.60202-116~20.04_amd64.deb -Size: 56408 -SHA256: e1894a342300ef7843378a0fd691c8336de16abac99061a1d80a086bef93ed05 -SHA1: d81f12b3525eaae98e14893342d626b321d1c6ee -MD5sum: 3781d26c57283fa70570b54255bf37ab -Description: Radeon Open Compute LAPACK marshalling library -Maintainer: hipSOLVER Maintainer -Recommends: hipsolver-dev6.2.2 (>=2.2.0.60202) -Version: 2.2.0.60202-116~20.04 -Installed-Size: 715 - -Package: hipsparse -Architecture: amd64 -Depends: rocsparse (>= 1.12.10), rocm-core -Priority: optional -Section: devel -Filename: pool/main/h/hipsparse/hipsparse_3.1.1.60202-116~20.04_amd64.deb -Size: 45610 -SHA256: 5c4ef7683ee74ed6d1ef98fdc770f0abf93020c1a2c8edbb6c6278191368542b -SHA1: 5d22ce11c39fce7755827c7cc25c48b74ebdca77 -MD5sum: 479f7dfae83ea91f701c2e0d452c81ba -Description: ROCm SPARSE library -Maintainer: hipSPARSE Maintainer -Recommends: hipsparse-dev (>=3.1.1.60202) -Version: 3.1.1.60202-116~20.04 -Installed-Size: 518 - -Package: hipsparse-asan -Architecture: amd64 -Depends: rocsparse (>= 1.12.10), rocm-core-asan -Priority: optional -Section: devel -Filename: pool/main/h/hipsparse-asan/hipsparse-asan_3.1.1.60202-116~20.04_amd64.deb -Size: 71286 -SHA256: ef766ea516e3ca06be60bc8de0c4ce46fad41c82b395f707dd99ab8996c5a7ee -SHA1: f00c41f3c3d0e82ce9fb555330d47bc8e94c4428 -MD5sum: 44ff03aae63ef022381786e0f0ea065a -Description: ROCm SPARSE library -Maintainer: hipSPARSE Maintainer -Recommends: hipsparse-asan-dev (>=3.1.1.60202) -Version: 3.1.1.60202-116~20.04 -Installed-Size: 1015 - -Package: hipsparse-asan-dbgsym -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 12fad6d4bcab05473afbe7b215a8ab16abd30063 -Depends: hipsparse-asan (= 3.1.1.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/h/hipsparse-asan-dbgsym/hipsparse-asan-dbgsym_3.1.1.60202-116~20.04_amd64.deb -Size: 242524 -SHA256: 3ceb3d2d97974f97d13ce3f7fdb06fff9a657ed92449bbcc6780173780438e7c -SHA1: eef9537af0069ebe4bdb3e8610785c2e13972d62 -MD5sum: 9b9b2fd9c5cb53ab704cc0a819ea431c -Description: debug symbols for hipsparse-asan -Maintainer: hipSPARSE Maintainer -Package-Type: ddeb -Version: 3.1.1.60202-116~20.04 -Installed-Size: 397 - -Package: hipsparse-asan-dbgsym-rpath6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 12fad6d4bcab05473afbe7b215a8ab16abd30063 -Depends: hipsparse-asan-rpath6.2.2 (= 3.1.1.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/h/hipsparse-asan-dbgsym-rpath6.2.2/hipsparse-asan-dbgsym-rpath6.2.2_3.1.1.60202-116~20.04_amd64.deb -Size: 242688 -SHA256: 33b1295070700b0a1d763d8abf708a910d92559ed7257722ae92cce943f14b48 -SHA1: 066313dff559ac988817fd14f63f3faca801ae8a -MD5sum: c83a0942d0ebf72b6b4d01b3b69adb95 -Description: debug symbols for hipsparse-asan -Maintainer: hipSPARSE Maintainer -Package-Type: ddeb -Version: 3.1.1.60202-116~20.04 -Installed-Size: 397 - -Package: hipsparse-asan-dbgsym6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 12fad6d4bcab05473afbe7b215a8ab16abd30063 -Depends: hipsparse-asan6.2.2 (= 3.1.1.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/h/hipsparse-asan-dbgsym6.2.2/hipsparse-asan-dbgsym6.2.2_3.1.1.60202-116~20.04_amd64.deb -Size: 242644 -SHA256: d8d5940ada5d026fe3db137121f01907fd8d11c8b5b532c07ed7094b0c62df04 -SHA1: b0bb5c103784221fcfd6aa3fb87fcdfda4db12a8 -MD5sum: 9acd7efb74741ab93312e6bd98bc841d -Description: debug symbols for hipsparse-asan -Maintainer: hipSPARSE Maintainer -Package-Type: ddeb -Version: 3.1.1.60202-116~20.04 -Installed-Size: 397 - -Package: hipsparse-asan-rpath6.2.2 -Architecture: amd64 -Depends: rocsparse (>= 1.12.10), rocm-core-asan-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/hipsparse-asan-rpath6.2.2/hipsparse-asan-rpath6.2.2_3.1.1.60202-116~20.04_amd64.deb -Size: 71880 -SHA256: 2ab83d4b1d078292443974a9a8970610c7a2bc369982c4f0d9dafcd5500f20c8 -SHA1: d309ff0b94146a008039096202376f07e241d228 -MD5sum: 4f1eea7e98de1d050dd6d67899ab71b8 -Description: ROCm SPARSE library -Maintainer: hipSPARSE Maintainer -Recommends: hipsparse-asan-dev (>=3.1.1.60202) -Version: 3.1.1.60202-116~20.04 -Installed-Size: 1015 - -Package: hipsparse-asan6.2.2 -Architecture: amd64 -Depends: rocsparse (>= 1.12.10), rocm-core-asan6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/hipsparse-asan6.2.2/hipsparse-asan6.2.2_3.1.1.60202-116~20.04_amd64.deb -Size: 71964 -SHA256: 23d90e4df129175fc9632ccfab56f01253d88cd4d174dfa5ac5915f45872a074 -SHA1: b1421cac2307b948239b5bb69232cbe5c6168d38 -MD5sum: e65a7135ef4836fa886b9e21aaa11689 -Description: ROCm SPARSE library -Maintainer: hipSPARSE Maintainer -Recommends: hipsparse-asan-dev (>=3.1.1.60202) -Version: 3.1.1.60202-116~20.04 -Installed-Size: 1015 - -Package: hipsparse-dbgsym -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 0a9eea7c035f9f30e28e0b97fbe237424930af34 -Depends: hipsparse (= 3.1.1.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/h/hipsparse-dbgsym/hipsparse-dbgsym_3.1.1.60202-116~20.04_amd64.deb -Size: 178088 -SHA256: 518ce2cc2771b13332366f1c17e2166d8065c307c406abc9bfa95e07957c5253 -SHA1: 06667f4b498885f7dae45d1002c5a57ba57d2165 -MD5sum: e48ad0110f628e607056765138810dbd -Description: debug symbols for hipsparse -Maintainer: hipSPARSE Maintainer -Package-Type: ddeb -Version: 3.1.1.60202-116~20.04 -Installed-Size: 1096 - -Package: hipsparse-dbgsym-rpath6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 0a9eea7c035f9f30e28e0b97fbe237424930af34 -Depends: hipsparse-rpath6.2.2 (= 3.1.1.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/h/hipsparse-dbgsym-rpath6.2.2/hipsparse-dbgsym-rpath6.2.2_3.1.1.60202-116~20.04_amd64.deb -Size: 177748 -SHA256: 689890d399304ea118b340ffe76b6d301816ec4f48854bd6ad2befca7d6287e8 -SHA1: afacfe1b3855faf9ab79b9c519ec67d20bda1ef9 -MD5sum: c6cb241da606bf7b976d099b1d31ed26 -Description: debug symbols for hipsparse -Maintainer: hipSPARSE Maintainer -Package-Type: ddeb -Version: 3.1.1.60202-116~20.04 -Installed-Size: 1096 - -Package: hipsparse-dbgsym6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 0a9eea7c035f9f30e28e0b97fbe237424930af34 -Depends: hipsparse6.2.2 (= 3.1.1.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/h/hipsparse-dbgsym6.2.2/hipsparse-dbgsym6.2.2_3.1.1.60202-116~20.04_amd64.deb -Size: 178564 -SHA256: 8fd11af97e2ac5c4ab29041f1f173828ecfde0570fc7c01bd027a7f8a5d2dfb2 -SHA1: e1e55b1cf4e9ce01bbe412c6952d52956cda37e1 -MD5sum: 782ffa56793bbdc610f0b983c6ab8920 -Description: debug symbols for hipsparse -Maintainer: hipSPARSE Maintainer -Package-Type: ddeb -Version: 3.1.1.60202-116~20.04 -Installed-Size: 1096 - -Package: hipsparse-dev -Architecture: amd64 -Depends: hipsparse (>= 3.1.1.60202) -Priority: optional -Section: devel -Filename: pool/main/h/hipsparse-dev/hipsparse-dev_3.1.1.60202-116~20.04_amd64.deb -Size: 40932 -SHA256: 4efdf6f96a32fbbcd22eb0280e82a55aba538ec2b7ecdffcc8b4db97c80e490c -SHA1: a67127a552d4e70b64cd32a77b3e6c18f78aefcc -MD5sum: d309d44e0488ed5a6a0b5e4bbd3f62cd -Description: ROCm SPARSE library -Maintainer: hipSPARSE Maintainer -Version: 3.1.1.60202-116~20.04 -Installed-Size: 749 - -Package: hipsparse-dev-rpath6.2.2 -Architecture: amd64 -Depends: hipsparse-rpath6.2.2 (>= 3.1.1.60202) -Priority: optional -Section: devel -Filename: pool/main/h/hipsparse-dev-rpath6.2.2/hipsparse-dev-rpath6.2.2_3.1.1.60202-116~20.04_amd64.deb -Size: 41092 -SHA256: a8738b858a6b25eca0a32acf1550aa9d36a2bacc858578803b4f7e9dc9809148 -SHA1: 4f6d1339239c27575ca8e538eb5e27ba2add5603 -MD5sum: 04761a8d743d66df7b9b534864794cb1 -Description: ROCm SPARSE library -Maintainer: hipSPARSE Maintainer -Version: 3.1.1.60202-116~20.04 -Installed-Size: 749 - -Package: hipsparse-dev6.2.2 -Architecture: amd64 -Depends: hipsparse6.2.2 (>= 3.1.1.60202) -Priority: optional -Section: devel -Filename: pool/main/h/hipsparse-dev6.2.2/hipsparse-dev6.2.2_3.1.1.60202-116~20.04_amd64.deb -Size: 41068 -SHA256: 4edc6771d3a830a8329c6f156a6454a6663651cd70960d5b3a7d8640b762ca87 -SHA1: 403c969dc33a202f52e93ae874d9b885e2f3979f -MD5sum: a639f0d48ebffff5e0339fe95edd990b -Description: ROCm SPARSE library -Maintainer: hipSPARSE Maintainer -Version: 3.1.1.60202-116~20.04 -Installed-Size: 749 - -Package: hipsparse-rpath6.2.2 -Architecture: amd64 -Depends: rocsparse-rpath6.2.2 (>= 1.12.10), rocm-core-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/hipsparse-rpath6.2.2/hipsparse-rpath6.2.2_3.1.1.60202-116~20.04_amd64.deb -Size: 46140 -SHA256: 441f7a0d64795bbbbc21954a727320b8a19af8096698577b5c9af2edd46c04d6 -SHA1: 0331b19d1972b95029bde870022d2f443be22abd -MD5sum: 79dd724d3785e3216e06f85e2675baf7 -Description: ROCm SPARSE library -Maintainer: hipSPARSE Maintainer -Recommends: hipsparse-dev-rpath6.2.2 (>=3.1.1.60202) -Version: 3.1.1.60202-116~20.04 -Installed-Size: 518 - -Package: hipsparse6.2.2 -Architecture: amd64 -Depends: rocsparse6.2.2 (>= 1.12.10), rocm-core6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/hipsparse6.2.2/hipsparse6.2.2_3.1.1.60202-116~20.04_amd64.deb -Size: 46168 -SHA256: 0dfa5900717fcf6c146c172671dc842b232a69f85d5915a34a289dd7082fb7db -SHA1: fe36211fdc7c7057da0ce118537d4e095d292660 -MD5sum: 63e6afe64f320131b1fc11431ff8bad9 -Description: ROCm SPARSE library -Maintainer: hipSPARSE Maintainer -Recommends: hipsparse-dev6.2.2 (>=3.1.1.60202) -Version: 3.1.1.60202-116~20.04 -Installed-Size: 518 - -Package: hipsparselt -Architecture: amd64 -Depends: hipsparse (>= 2.1.0), rocm-core -Priority: optional -Section: devel -Filename: pool/main/h/hipsparselt/hipsparselt_0.2.1.60202-116~20.04_amd64.deb -Size: 6542844 -SHA256: 83c384d7ca1b063099dfc93daa5b5cb0fb5566d31c3dd7d4cdc3b7bf11d75e2c -SHA1: d493e788e8ed1f2abb5e5d0e02b51c97d228bea0 -MD5sum: 48a871f944fef9fcbb57f21143dc49de -Description: Radeon Open Compute Structured Sparsity Matrix Multiplication marshalling library -Maintainer: hipSPARSELt Maintainer -Recommends: hipsparselt-dev (>=0.2.1.60202) -Version: 0.2.1.60202-116~20.04 -Installed-Size: 232236 - -Package: hipsparselt-asan -Architecture: amd64 -Depends: hipsparse (>= 2.1.0), rocm-core-asan -Priority: optional -Section: devel -Filename: pool/main/h/hipsparselt-asan/hipsparselt-asan_0.2.1.60202-116~20.04_amd64.deb -Size: 16481500 -SHA256: 7775073b89cf1342d1fab3c1156bce866d5b3a9297daaef323e44a98127fab68 -SHA1: 000b654005e72a6ff5d59eef410ca82be4141a3e -MD5sum: 9e129960835a196678c320d5dde19a13 -Description: Radeon Open Compute Structured Sparsity Matrix Multiplication marshalling library -Maintainer: hipSPARSELt Maintainer -Recommends: hipsparselt-asan-dev (>=0.2.1.60202) -Version: 0.2.1.60202-116~20.04 -Installed-Size: 59812 - -Package: hipsparselt-asan-rpath6.2.2 -Architecture: amd64 -Depends: hipsparse (>= 2.1.0), rocm-core-asan-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/hipsparselt-asan-rpath6.2.2/hipsparselt-asan-rpath6.2.2_0.2.1.60202-116~20.04_amd64.deb -Size: 16482888 -SHA256: 1c8626c5e4a1acac4bb26769644cd6c1c1f45497d3654b11bc69bd48e2dbe6f7 -SHA1: 54f470ad7f86342fc8311f534e3c814cb014fce1 -MD5sum: 2eea6a2e697844570244fa5ba5c8bbd4 -Description: Radeon Open Compute Structured Sparsity Matrix Multiplication marshalling library -Maintainer: hipSPARSELt Maintainer -Recommends: hipsparselt-asan-dev (>=0.2.1.60202) -Version: 0.2.1.60202-116~20.04 -Installed-Size: 59812 - -Package: hipsparselt-asan6.2.2 -Architecture: amd64 -Depends: hipsparse (>= 2.1.0), rocm-core-asan6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/hipsparselt-asan6.2.2/hipsparselt-asan6.2.2_0.2.1.60202-116~20.04_amd64.deb -Size: 16482696 -SHA256: 0535eef6e4a9c9255efa6a88b8f39ee3ec1778d54083c3c095c5d95be20ab299 -SHA1: 91a2d0576d9ba26f1bea36aa1f623b3f69d9b152 -MD5sum: 4dea0c053fc0756edeacc53221497baf -Description: Radeon Open Compute Structured Sparsity Matrix Multiplication marshalling library -Maintainer: hipSPARSELt Maintainer -Recommends: hipsparselt-asan-dev (>=0.2.1.60202) -Version: 0.2.1.60202-116~20.04 -Installed-Size: 59812 - -Package: hipsparselt-dev -Architecture: amd64 -Depends: hipsparselt (>= 0.2.1.60202) -Priority: optional -Section: devel -Filename: pool/main/h/hipsparselt-dev/hipsparselt-dev_0.2.1.60202-116~20.04_amd64.deb -Size: 11612 -SHA256: cfd39548c6614e6d3be90ad95d9ca1cae78ffb7ad1a00737ffc6251bad755645 -SHA1: be7e4fd5d82577ef45fb8a80b6c979f932ccf5c1 -MD5sum: d63a963615c5676401d02617ef48829b -Description: Radeon Open Compute Structured Sparsity Matrix Multiplication marshalling library -Maintainer: hipSPARSELt Maintainer -Version: 0.2.1.60202-116~20.04 -Installed-Size: 94 - -Package: hipsparselt-dev-rpath6.2.2 -Architecture: amd64 -Depends: hipsparselt-rpath6.2.2 (>= 0.2.1.60202) -Priority: optional -Section: devel -Filename: pool/main/h/hipsparselt-dev-rpath6.2.2/hipsparselt-dev-rpath6.2.2_0.2.1.60202-116~20.04_amd64.deb -Size: 11764 -SHA256: f8cc80ac57c701c86175e6b1ad5255fd19fc97362e1813ab0b3686e7372ad351 -SHA1: b677ceb29c5508e105f088150ca6a84370d5215f -MD5sum: e2dc8cbf03d96d7f9af18dfe4ba8a7df -Description: Radeon Open Compute Structured Sparsity Matrix Multiplication marshalling library -Maintainer: hipSPARSELt Maintainer -Version: 0.2.1.60202-116~20.04 -Installed-Size: 94 - -Package: hipsparselt-dev6.2.2 -Architecture: amd64 -Depends: hipsparselt6.2.2 (>= 0.2.1.60202) -Priority: optional -Section: devel -Filename: pool/main/h/hipsparselt-dev6.2.2/hipsparselt-dev6.2.2_0.2.1.60202-116~20.04_amd64.deb -Size: 11756 -SHA256: 432160e47268d8fc7c3540bf213b8ea4e2f264c7d4cc7be73f1b9dc358378fb8 -SHA1: 4fd879c769985c587e4a9e158d64e477278c3966 -MD5sum: 002a10072598b381f4d26cbe01e33ba3 -Description: Radeon Open Compute Structured Sparsity Matrix Multiplication marshalling library -Maintainer: hipSPARSELt Maintainer -Version: 0.2.1.60202-116~20.04 -Installed-Size: 94 - -Package: hipsparselt-rpath6.2.2 -Architecture: amd64 -Depends: hipsparse-rpath6.2.2 (>= 2.1.0), rocm-core-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/hipsparselt-rpath6.2.2/hipsparselt-rpath6.2.2_0.2.1.60202-116~20.04_amd64.deb -Size: 6535332 -SHA256: ac7c9fefc9bfc131b278ee00067b064a25399f374bccbb41afdaf7092bfa2bd0 -SHA1: db4bffab5199c44d5583f35892041ae8f73dde78 -MD5sum: 264be4f9cf6d8665a733717647e91fe1 -Description: Radeon Open Compute Structured Sparsity Matrix Multiplication marshalling library -Maintainer: hipSPARSELt Maintainer -Recommends: hipsparselt-dev-rpath6.2.2 (>=0.2.1.60202) -Version: 0.2.1.60202-116~20.04 -Installed-Size: 232236 - -Package: hipsparselt6.2.2 -Architecture: amd64 -Depends: hipsparse6.2.2 (>= 2.1.0), rocm-core6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/hipsparselt6.2.2/hipsparselt6.2.2_0.2.1.60202-116~20.04_amd64.deb -Size: 6535312 -SHA256: 84b1b6e9077c4690a438739c23bec046c711a100e18eb76df63b3b6125eb457f -SHA1: 2864d358c62f53862929289da7230b33d4f89414 -MD5sum: 4725f3272f10f79f0e5fffaeb0c79a25 -Description: Radeon Open Compute Structured Sparsity Matrix Multiplication marshalling library -Maintainer: hipSPARSELt Maintainer -Recommends: hipsparselt-dev6.2.2 (>=0.2.1.60202) -Version: 0.2.1.60202-116~20.04 -Installed-Size: 232236 - -Package: hiptensor -Architecture: amd64 -Depends: rocm-core -Priority: optional -Section: devel -Filename: pool/main/h/hiptensor/hiptensor_1.3.0.60202-116~20.04_amd64.deb -Size: 39764192 -SHA256: 3577bdd767fe612a6d90e8eb6225119f5d7c83dbb4bacbca9fabf7e21a807b9c -SHA1: d00c9463d803351bc2ddc0b104bbe48cc9e550f7 -MD5sum: da2ef60d51cd4538f9e504ed20fabc76 -Description: AMD high-performance HIP library for tensor primitives -Maintainer: hiptensor Maintainer -Recommends: hiptensor-dev (>=1.3.0.60202) -Version: 1.3.0.60202-116~20.04 -Installed-Size: 1112023 - -Package: hiptensor-asan -Architecture: amd64 -Depends: rocm-core-asan -Priority: optional -Section: devel -Filename: pool/main/h/hiptensor-asan/hiptensor-asan_1.3.0.60202-116~20.04_amd64.deb -Size: 1447374834 -SHA256: 7f76413a2785d88b62d4860272455fcc766e4f85429b8034fd25cfce4791c090 -SHA1: 96f27543c7b1f8cf50b526a9c165316150f6fa3c -MD5sum: 9c557c7da488176488a81fa15ee63543 -Description: AMD high-performance HIP library for tensor primitives -Maintainer: hiptensor Maintainer -Recommends: hiptensor-asan-dev (>=1.3.0.60202) -Version: 1.3.0.60202-116~20.04 -Installed-Size: 5192496 - -Package: hiptensor-asan-rpath6.2.2 -Architecture: amd64 -Depends: rocm-core-asan-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/hiptensor-asan-rpath6.2.2/hiptensor-asan-rpath6.2.2_1.3.0.60202-116~20.04_amd64.deb -Size: 1447384964 -SHA256: 602c5c31caa5035bc415cfdadcd9737b5475777c150995db97e7370ba5d74abe -SHA1: 9386262275d541ddc0f09324ba6bf5e3d2b9dd81 -MD5sum: 9276ea1efed6cc67bef5089764d0fa79 -Description: AMD high-performance HIP library for tensor primitives -Maintainer: hiptensor Maintainer -Recommends: hiptensor-asan-dev (>=1.3.0.60202) -Version: 1.3.0.60202-116~20.04 -Installed-Size: 5192496 - -Package: hiptensor-asan6.2.2 -Architecture: amd64 -Depends: rocm-core-asan6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/hiptensor-asan6.2.2/hiptensor-asan6.2.2_1.3.0.60202-116~20.04_amd64.deb -Size: 1447377784 -SHA256: d1f21c3ba14e737f8a75c48741077a818169c17fe2bf49dfcd313711bf9131dc -SHA1: b1fe28b883c8db5aeb4053c37b855edd8fba47c0 -MD5sum: 469e09c4b5c8ee7d1931fcbd8f541288 -Description: AMD high-performance HIP library for tensor primitives -Maintainer: hiptensor Maintainer -Recommends: hiptensor-asan-dev (>=1.3.0.60202) -Version: 1.3.0.60202-116~20.04 -Installed-Size: 5192496 - -Package: hiptensor-dev -Architecture: amd64 -Depends: hiptensor (>= 1.3.0.60202) -Priority: optional -Section: devel -Filename: pool/main/h/hiptensor-dev/hiptensor-dev_1.3.0.60202-116~20.04_amd64.deb -Size: 12428 -SHA256: 05e09b9182266d40c145478127da2bfc813dc36c1cf15a64b9abe1eea12da711 -SHA1: 75b3987a5c1cefa0c47597a3e6d76f4dff29ba5f -MD5sum: 4ac02225b46b3de2e77507ae2f0ae76a -Description: AMD high-performance HIP library for tensor primitives -Maintainer: hiptensor Maintainer -Version: 1.3.0.60202-116~20.04 -Installed-Size: 98 - -Package: hiptensor-dev-rpath6.2.2 -Architecture: amd64 -Depends: hiptensor-rpath6.2.2 (>= 1.3.0.60202) -Priority: optional -Section: devel -Filename: pool/main/h/hiptensor-dev-rpath6.2.2/hiptensor-dev-rpath6.2.2_1.3.0.60202-116~20.04_amd64.deb -Size: 12580 -SHA256: 7e10ff746750670df839485c7759c4fa2815ed7b42a9ec7a5918bce0c4d51b67 -SHA1: ea66168c36c2b30b274b7ecc93a72f01478ee1dc -MD5sum: b594afd710116b71279b4ec4f7c6eb73 -Description: AMD high-performance HIP library for tensor primitives -Maintainer: hiptensor Maintainer -Version: 1.3.0.60202-116~20.04 -Installed-Size: 98 - -Package: hiptensor-dev6.2.2 -Architecture: amd64 -Depends: hiptensor6.2.2 (>= 1.3.0.60202) -Priority: optional -Section: devel -Filename: pool/main/h/hiptensor-dev6.2.2/hiptensor-dev6.2.2_1.3.0.60202-116~20.04_amd64.deb -Size: 12564 -SHA256: 6adc2508deee0d0d7822e2520ee6ba4fa4da8d0d6fcd55764e38c45a7d483a96 -SHA1: b49d2aec5c866f315ae9d8d4703258467a2d7531 -MD5sum: d7b2f27a1691bade6f5b534ad689afb4 -Description: AMD high-performance HIP library for tensor primitives -Maintainer: hiptensor Maintainer -Version: 1.3.0.60202-116~20.04 -Installed-Size: 98 - -Package: hiptensor-rpath6.2.2 -Architecture: amd64 -Depends: rocm-core-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/hiptensor-rpath6.2.2/hiptensor-rpath6.2.2_1.3.0.60202-116~20.04_amd64.deb -Size: 39772448 -SHA256: a4d8064c448533431e43d2c2469aa135383f4e9e1463ff1a2a2a605465ede36b -SHA1: 4b92f89490aa787f5e96c8afeeafb9c9eab02322 -MD5sum: 7fa799a431b1ba4bd135c05b01c369d2 -Description: AMD high-performance HIP library for tensor primitives -Maintainer: hiptensor Maintainer -Recommends: hiptensor-dev-rpath6.2.2 (>=1.3.0.60202) -Version: 1.3.0.60202-116~20.04 -Installed-Size: 1112023 - -Package: hiptensor6.2.2 -Architecture: amd64 -Depends: rocm-core6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/hiptensor6.2.2/hiptensor6.2.2_1.3.0.60202-116~20.04_amd64.deb -Size: 39759296 -SHA256: 47429337f076dbe7a98c4d5fedc371979b33a8af296fd38ec45ba1f8a88474fa -SHA1: a1b0090327add2d3749a6c37cbe96091987ff1b9 -MD5sum: 870263ccee2ab674cbd8bb7c7cd3f380 -Description: AMD high-performance HIP library for tensor primitives -Maintainer: hiptensor Maintainer -Recommends: hiptensor-dev6.2.2 (>=1.3.0.60202) -Version: 1.3.0.60202-116~20.04 -Installed-Size: 1112023 - -Package: hsa-amd-aqlprofile -Architecture: amd64 -Depends: rocm-core -Priority: optional -Section: devel -Filename: pool/main/h/hsa-amd-aqlprofile/hsa-amd-aqlprofile_1.0.0.60202.60202-116~20.04_amd64.deb -Size: 459174 -SHA256: f89c02d187dd20b8142fcf33c235746cc0c5ede58e240dbff2424a8b8f04fbe9 -SHA1: a9cd7aff4b428c3c5568e39faebc452c850b2fc6 -MD5sum: f1d460690ebcb9f3fdee4bb797da1e87 -Description: AQLPROFILE library for AMD HSA runtime API extension support - Dynamic libraries for the AQLProfile -Homepage: https://github.com/RadeonOpenCompute/HSA-AqlProfile-AMD-extension -Maintainer: ROCm Profiler Support -Version: 1.0.0.60202.60202-116~20.04 -Installed-Size: 2748 - -Package: hsa-amd-aqlprofile-asan -Architecture: amd64 -Depends: rocm-core-asan -Priority: optional -Section: devel -Filename: pool/main/h/hsa-amd-aqlprofile-asan/hsa-amd-aqlprofile-asan_1.0.0.60202.60202-116~20.04_amd64.deb -Size: 6428678 -SHA256: e28b2c0feaff4708e216806b08c6d5280f1d52de7dd7d9d6a09614b5b2ff36ed -SHA1: e511eb261ddc1b32dd05ed36b93907b9d38dca47 -MD5sum: 702646c130b037ec3d55d240224cc953 -Description: AQLPROFILE library for AMD HSA runtime API extension support - ASAN libraries for the AQLProfile -Homepage: https://github.com/RadeonOpenCompute/HSA-AqlProfile-AMD-extension -Maintainer: ROCm Profiler Support -Version: 1.0.0.60202.60202-116~20.04 -Installed-Size: 30650 - -Package: hsa-amd-aqlprofile-asan-rpath6.2.2 -Architecture: amd64 -Depends: rocm-core-asan-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/hsa-amd-aqlprofile-asan-rpath6.2.2/hsa-amd-aqlprofile-asan-rpath6.2.2_1.0.0.60202.60202-116~20.04_amd64.deb -Size: 3816684 -SHA256: 36a49e1b02e9b9fed8411ca7ea91884df89fc9733d9d9746edcb4c411a2091c0 -SHA1: d7ef2e5f2d3e739833f7af0257ab91153c91da2d -MD5sum: f62af78c5156ab417e7a64d9dd0e65b9 -Description: AQLPROFILE library for AMD HSA runtime API extension support - ASAN libraries for the AQLProfile -Homepage: https://github.com/RadeonOpenCompute/HSA-AqlProfile-AMD-extension -Maintainer: ROCm Profiler Support -Version: 1.0.0.60202.60202-116~20.04 -Installed-Size: 30650 - -Package: hsa-amd-aqlprofile-asan6.2.2 -Architecture: amd64 -Depends: rocm-core-asan6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/hsa-amd-aqlprofile-asan6.2.2/hsa-amd-aqlprofile-asan6.2.2_1.0.0.60202.60202-116~20.04_amd64.deb -Size: 3816168 -SHA256: 3c3706f9ab203062530188e7d0f671a93ad1c8c96aa133f801d376e57ef3e2b4 -SHA1: ccfe9187fbed2200856e773b48d5c3d200ed694a -MD5sum: 68ea573ef6d882a37ec85d5f7a059235 -Description: AQLPROFILE library for AMD HSA runtime API extension support - ASAN libraries for the AQLProfile -Homepage: https://github.com/RadeonOpenCompute/HSA-AqlProfile-AMD-extension -Maintainer: ROCm Profiler Support -Version: 1.0.0.60202.60202-116~20.04 -Installed-Size: 30650 - -Package: hsa-amd-aqlprofile-rpath6.2.2 -Architecture: amd64 -Depends: rocm-core-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/hsa-amd-aqlprofile-rpath6.2.2/hsa-amd-aqlprofile-rpath6.2.2_1.0.0.60202.60202-116~20.04_amd64.deb -Size: 267252 -SHA256: 5d793342e5aab0359c9f881507b6e667ecf2b55edc2fd8863e244693c730fa3b -SHA1: 6a769dec75fa2b51c9656369cc9c3339f98441cf -MD5sum: 38a2bb235c096bfbcabd083410b0d876 -Description: AQLPROFILE library for AMD HSA runtime API extension support - Dynamic libraries for the AQLProfile -Homepage: https://github.com/RadeonOpenCompute/HSA-AqlProfile-AMD-extension -Maintainer: ROCm Profiler Support -Version: 1.0.0.60202.60202-116~20.04 -Installed-Size: 2748 - -Package: hsa-amd-aqlprofile6.2.2 -Architecture: amd64 -Depends: rocm-core6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/hsa-amd-aqlprofile6.2.2/hsa-amd-aqlprofile6.2.2_1.0.0.60202.60202-116~20.04_amd64.deb -Size: 267228 -SHA256: 4467bee59d5560c9f154dd374f83d51fa1c5b4f3e827129149b93f4337dd26e6 -SHA1: 8ec44193e936f99264780f1809ddc0e098aea2de -MD5sum: c2579792cd8c8f8192ce1e2189570cfa -Description: AQLPROFILE library for AMD HSA runtime API extension support - Dynamic libraries for the AQLProfile -Homepage: https://github.com/RadeonOpenCompute/HSA-AqlProfile-AMD-extension -Maintainer: ROCm Profiler Support -Version: 1.0.0.60202.60202-116~20.04 -Installed-Size: 2748 - -Package: hsa-rocr -Architecture: amd64 -Breaks: hsa-ext-rocr-dev -Depends: libdrm-amdgpu-amdgpu1 | libdrm-amdgpu1, libnuma1, libelf1, rocm-core, rocprofiler-register -Priority: optional -Section: devel -Filename: pool/main/h/hsa-rocr/hsa-rocr_1.14.0.60202-116~20.04_amd64.deb -Size: 935194 -SHA256: 66cd708ebad4dc99b031d79c0882c280abbc4555891ac630ad09ecd919caefea -SHA1: efd42000a1662202740bc6fc3cc1aa22574f2fdf -MD5sum: ecb576c560ca92e59d574aaa51278d0e -Description: AMD Heterogeneous System Architecture HSA - Linux HSA Runtime for Boltzmann (ROCm) platforms -Homepage: https://github.com/RadeonOpenCompute/ROCR-Runtime -Maintainer: AMD HSA Support -Recommends: libdrm-amdgpu-amdgpu1 -Replaces: hsa-ext-rocr-dev -Version: 1.14.0.60202-116~20.04 -Installed-Size: 9535 - -Package: hsa-rocr-asan -Architecture: amd64 -Breaks: hsa-ext-rocr-dev -Depends: libdrm-amdgpu-amdgpu1 | libdrm-amdgpu1, libnuma1, libelf1, rocm-core-asan -Priority: optional -Section: devel -Filename: pool/main/h/hsa-rocr-asan/hsa-rocr-asan_1.14.0.60202-116~20.04_amd64.deb -Size: 1643782 -SHA256: ab3f5350aed5723e6e7f32a4aac9310c5463fe04e641bb4f962fc77290f05cc9 -SHA1: e1c8c0add3461c33538d74ae4231878fe0789eaa -MD5sum: 5ced42341abd1123ccb552638558f68e -Description: AMD Heterogeneous System Architecture HSA - Linux HSA instrumented libraries for Boltzmann (ROCm) platforms -Homepage: https://github.com/RadeonOpenCompute/ROCR-Runtime -Maintainer: AMD HSA Support -Replaces: hsa-ext-rocr-dev -Version: 1.14.0.60202-116~20.04 -Installed-Size: 17194 - -Package: hsa-rocr-asan-dbgsym -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 3cd73428a57c654e904b32077497e59f0c951270 -Depends: hsa-rocr-asan (= 1.14.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/h/hsa-rocr-asan-dbgsym/hsa-rocr-asan-dbgsym_1.14.0.60202-116~20.04_amd64.deb -Size: 4980264 -SHA256: 917c4b503f9574def7d19eca6d3617dd734ff4196d6ff824c753bbca4fc8b56d -SHA1: d4bc198df1a4c23eb6013e78fd068b95d2fb0df3 -MD5sum: cbbbf132e345fa813396401b7c80aa0a -Description: debug symbols for hsa-rocr-asan -Maintainer: AMD HSA Support -Package-Type: ddeb -Version: 1.14.0.60202-116~20.04 -Installed-Size: 13781 - -Package: hsa-rocr-asan-dbgsym-rpath6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 3cd73428a57c654e904b32077497e59f0c951270 -Depends: hsa-rocr-asan-rpath6.2.2 (= 1.14.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/h/hsa-rocr-asan-dbgsym-rpath6.2.2/hsa-rocr-asan-dbgsym-rpath6.2.2_1.14.0.60202-116~20.04_amd64.deb -Size: 3469464 -SHA256: f06410dba492b7fbe85ff791b349da6ca85b4589831f1269692c60aa246a8865 -SHA1: 6e8f096789f3061bc4180d3727638a7c85983e69 -MD5sum: c07b572e3e6c2b7e40f84472aa4e9d88 -Description: debug symbols for hsa-rocr-asan -Maintainer: AMD HSA Support -Package-Type: ddeb -Version: 1.14.0.60202-116~20.04 -Installed-Size: 13781 - -Package: hsa-rocr-asan-dbgsym6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 3cd73428a57c654e904b32077497e59f0c951270 -Depends: hsa-rocr-asan6.2.2 (= 1.14.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/h/hsa-rocr-asan-dbgsym6.2.2/hsa-rocr-asan-dbgsym6.2.2_1.14.0.60202-116~20.04_amd64.deb -Size: 3469560 -SHA256: 15bd29bba0f53b167026c64de5979f0a6761ddc84eb26682501239a0b90f5ee6 -SHA1: 5625ea9fa6d9d68b8586ccefc61fc95950d7cf19 -MD5sum: a5b10ec22a12ee7983225a5748d30272 -Description: debug symbols for hsa-rocr-asan -Maintainer: AMD HSA Support -Package-Type: ddeb -Version: 1.14.0.60202-116~20.04 -Installed-Size: 13781 - -Package: hsa-rocr-asan-rpath6.2.2 -Architecture: amd64 -Breaks: hsa-ext-rocr-dev-rpath6.2.2 -Depends: libdrm-amdgpu-amdgpu1 | libdrm-amdgpu1, libnuma1, libelf1, rocm-core-asan-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/hsa-rocr-asan-rpath6.2.2/hsa-rocr-asan-rpath6.2.2_1.14.0.60202-116~20.04_amd64.deb -Size: 1031380 -SHA256: d539a87c6f5a964d9b54c8efac9b1c5d637aa4a5c7b24162bb86edb25fe71048 -SHA1: 7cc0453e9c3ee88e65a34391ff4c6b1a93f0a756 -MD5sum: 9dbb3e0196c665e39e49a9814167a0e8 -Description: AMD Heterogeneous System Architecture HSA - Linux HSA instrumented libraries for Boltzmann (ROCm) platforms -Homepage: https://github.com/RadeonOpenCompute/ROCR-Runtime -Maintainer: AMD HSA Support -Replaces: hsa-ext-rocr-dev-rpath6.2.2 -Version: 1.14.0.60202-116~20.04 -Installed-Size: 17194 - -Package: hsa-rocr-asan6.2.2 -Architecture: amd64 -Breaks: hsa-ext-rocr-dev6.2.2 -Depends: libdrm-amdgpu-amdgpu1 | libdrm-amdgpu1, libnuma1, libelf1, rocm-core-asan6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/hsa-rocr-asan6.2.2/hsa-rocr-asan6.2.2_1.14.0.60202-116~20.04_amd64.deb -Size: 1030928 -SHA256: c5140073518b3a733041f100c474da1b3eafd8f375a8b442a55ea1780ed22a3b -SHA1: ec181224d6bc9da511403a17f4fc1a1d8315c59c -MD5sum: 13b4cd679bfa8dba4e98d035562a11c0 -Description: AMD Heterogeneous System Architecture HSA - Linux HSA instrumented libraries for Boltzmann (ROCm) platforms -Homepage: https://github.com/RadeonOpenCompute/ROCR-Runtime -Maintainer: AMD HSA Support -Replaces: hsa-ext-rocr-dev6.2.2 -Version: 1.14.0.60202-116~20.04 -Installed-Size: 17194 - -Package: hsa-rocr-dbgsym -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 2712d2d5f96b5573d48e6b2eafbed56df250c28e -Depends: hsa-rocr (= 1.14.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/h/hsa-rocr-dbgsym/hsa-rocr-dbgsym_1.14.0.60202-116~20.04_amd64.deb -Size: 10283614 -SHA256: 50e84b0609af2893e8d9653b8e98775769747eed6a6dd97e3a87dee3ce4f1735 -SHA1: 82e0bd03c7aec391bfc93a6334607f53b8482301 -MD5sum: ba17bb3bf4029266074442dc0f8b1b34 -Description: debug symbols for hsa-rocr -Maintainer: AMD HSA Support -Package-Type: ddeb -Version: 1.14.0.60202-116~20.04 -Installed-Size: 34610 - -Package: hsa-rocr-dbgsym-rpath6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 2712d2d5f96b5573d48e6b2eafbed56df250c28e -Depends: hsa-rocr-rpath6.2.2 (= 1.14.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/h/hsa-rocr-dbgsym-rpath6.2.2/hsa-rocr-dbgsym-rpath6.2.2_1.14.0.60202-116~20.04_amd64.deb -Size: 6890816 -SHA256: f83a89d295f94b0c3f9710344cdb84fe61cc57a0873e90a8c2b2b8bb45f93cae -SHA1: b59910d940687c4c69a9158bdab13168a08f198d -MD5sum: a0d1ef4a67d4c9f2084e762fd61932d4 -Description: debug symbols for hsa-rocr -Maintainer: AMD HSA Support -Package-Type: ddeb -Version: 1.14.0.60202-116~20.04 -Installed-Size: 34610 - -Package: hsa-rocr-dbgsym6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 2712d2d5f96b5573d48e6b2eafbed56df250c28e -Depends: hsa-rocr6.2.2 (= 1.14.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/h/hsa-rocr-dbgsym6.2.2/hsa-rocr-dbgsym6.2.2_1.14.0.60202-116~20.04_amd64.deb -Size: 6890668 -SHA256: 91efbd6bf971ff45e476cf3c1ea705ee9c1010d2011be16f3cbec1119f5cab9c -SHA1: 043d890be4d29fef2725095c417b118e15110737 -MD5sum: 2e0df9475e1f5f3d5de2ba9752bf4ba5 -Description: debug symbols for hsa-rocr -Maintainer: AMD HSA Support -Package-Type: ddeb -Version: 1.14.0.60202-116~20.04 -Installed-Size: 34610 - -Package: hsa-rocr-dev -Architecture: amd64 -Breaks: hsa-ext-rocr-dev -Depends: hsa-rocr, hsakmt-roct-dev -Priority: optional -Section: devel -Filename: pool/main/h/hsa-rocr-dev/hsa-rocr-dev_1.14.0.60202-116~20.04_amd64.deb -Size: 105938 -SHA256: 87b868469a9ccfd457134e41abf2d198d263b67e3b852415cb04a871cd1ee5ca -SHA1: c73f73e671a97b68b14ef164aaa3c93c0792f0f6 -MD5sum: 47256f56300018102c8ffa931a9390f7 -Description: AMD Heterogeneous System Architecture HSA development package. - This package contains the headers and cmake files for the hsa-rocr package. -Homepage: https://github.com/RadeonOpenCompute/ROCR-Runtime -Maintainer: AMD HSA Support -Replaces: hsa-ext-rocr-dev -Version: 1.14.0.60202-116~20.04 -Installed-Size: 584 - -Package: hsa-rocr-dev-rpath6.2.2 -Architecture: amd64 -Breaks: hsa-ext-rocr-dev-rpath6.2.2 -Depends: hsa-rocr-rpath6.2.2, hsakmt-roct-dev-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/hsa-rocr-dev-rpath6.2.2/hsa-rocr-dev-rpath6.2.2_1.14.0.60202-116~20.04_amd64.deb -Size: 81156 -SHA256: 1a2850495a1ba39e4aff20f9f0c814bc1496b14d4daeca96e0d88f88529fcc4d -SHA1: ffadf87fd8626a70003783a9bfb0e7849538a5f0 -MD5sum: 53b352f2673f817358bba6a455a05e4e -Description: AMD Heterogeneous System Architecture HSA development package. - This package contains the headers and cmake files for the hsa-rocr package. -Homepage: https://github.com/RadeonOpenCompute/ROCR-Runtime -Maintainer: AMD HSA Support -Replaces: hsa-ext-rocr-dev-rpath6.2.2 -Version: 1.14.0.60202-116~20.04 -Installed-Size: 584 - -Package: hsa-rocr-dev6.2.2 -Architecture: amd64 -Breaks: hsa-ext-rocr-dev6.2.2 -Depends: hsa-rocr6.2.2, hsakmt-roct-dev6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/hsa-rocr-dev6.2.2/hsa-rocr-dev6.2.2_1.14.0.60202-116~20.04_amd64.deb -Size: 81192 -SHA256: 6453db78f1221a245e5aa72ee6faebb8be43e317e72648497be1a687b6aaaabd -SHA1: 451e2583f52e1991e9ce4d4dc5b6b4f03825875a -MD5sum: 24578fb91a5e86a0f7986f234d0d4637 -Description: AMD Heterogeneous System Architecture HSA development package. - This package contains the headers and cmake files for the hsa-rocr package. -Homepage: https://github.com/RadeonOpenCompute/ROCR-Runtime -Maintainer: AMD HSA Support -Replaces: hsa-ext-rocr-dev6.2.2 -Version: 1.14.0.60202-116~20.04 -Installed-Size: 584 - -Package: hsa-rocr-rpath6.2.2 -Architecture: amd64 -Breaks: hsa-ext-rocr-dev-rpath6.2.2 -Depends: libdrm-amdgpu-amdgpu1 | libdrm-amdgpu1, libnuma1, libelf1, rocm-core-rpath6.2.2, rocprofiler-register-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/hsa-rocr-rpath6.2.2/hsa-rocr-rpath6.2.2_1.14.0.60202-116~20.04_amd64.deb -Size: 554548 -SHA256: 484e68e1548658ab80505e457822107c51eeeb4a10b3ca6829154f079bcc62bf -SHA1: 42beb7e23f62c9c8e226d3894db068ae3e49a9d6 -MD5sum: aea725b247796004272dc1f1f16cc426 -Description: AMD Heterogeneous System Architecture HSA - Linux HSA Runtime for Boltzmann (ROCm) platforms -Homepage: https://github.com/RadeonOpenCompute/ROCR-Runtime -Maintainer: AMD HSA Support -Recommends: libdrm-amdgpu-amdgpu1 -Replaces: hsa-ext-rocr-dev-rpath6.2.2 -Version: 1.14.0.60202-116~20.04 -Installed-Size: 9535 - -Package: hsa-rocr6.2.2 -Architecture: amd64 -Breaks: hsa-ext-rocr-dev6.2.2 -Depends: libdrm-amdgpu-amdgpu1 | libdrm-amdgpu1, libnuma1, libelf1, rocm-core6.2.2, rocprofiler-register6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/hsa-rocr6.2.2/hsa-rocr6.2.2_1.14.0.60202-116~20.04_amd64.deb -Size: 554860 -SHA256: d247e9b78c09f87cbb70d0a9d34822f8fa600032c841514e99987b76e26512e7 -SHA1: 015dad4a6797863779aae4829d2e5cf1bfc070a7 -MD5sum: 9759a60ad3b5f6ac0d99209d842403cf -Description: AMD Heterogeneous System Architecture HSA - Linux HSA Runtime for Boltzmann (ROCm) platforms -Homepage: https://github.com/RadeonOpenCompute/ROCR-Runtime -Maintainer: AMD HSA Support -Recommends: libdrm-amdgpu-amdgpu1 -Replaces: hsa-ext-rocr-dev6.2.2 -Version: 1.14.0.60202-116~20.04 -Installed-Size: 9535 - -Package: hsakmt-roct-asan -Architecture: amd64 -Depends: libdrm-amdgpu-dev | libdrm-dev, rocm-core-asan -Priority: optional -Section: devel -Filename: pool/main/h/hsakmt-roct-asan/hsakmt-roct-asan_20240607.4.05.60202-116~20.04_amd64.deb -Size: 447520 -SHA256: 4ff56cbf3c646bad4f0f1292a2482dce6cae595fac8f6864da2d5eb2bb2bf57a -SHA1: 0c11ad39186771edd2df5a096e602015b03e9e59 -MD5sum: 41b771ea7ce794589485b906facd7eab -Description: ASAN libraries for the LIBHSAKMT -Homepage: https://github.com/ROCm/ROCT-Thunk-Interface -Maintainer: AMD GFX mailing list -Provides: hsakmt-roct -Recommends: libdrm-amdgpu-dev -Replaces: hsakmt-roct -Version: 20240607.4.05.60202-116~20.04 -Installed-Size: 1502 - -Package: hsakmt-roct-asan-rpath6.2.2 -Architecture: amd64 -Depends: libdrm-amdgpu-dev | libdrm-dev, rocm-core-asan-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/hsakmt-roct-asan-rpath6.2.2/hsakmt-roct-asan-rpath6.2.2_20240607.4.05.60202-116~20.04_amd64.deb -Size: 300196 -SHA256: 3cda018daaaed707c17d55e9e48e5e0b0728c957feb205a4a9cc3b34963a3083 -SHA1: 3c8503ee2b86fc31098fcd5c543f43a809fc6558 -MD5sum: 0752b175e48bbfe9e28fa43421bfb342 -Description: ASAN libraries for the LIBHSAKMT -Homepage: https://github.com/ROCm/ROCT-Thunk-Interface -Maintainer: AMD GFX mailing list -Provides: hsakmt-roct-rpath6.2.2 -Recommends: libdrm-amdgpu-dev -Replaces: hsakmt-roct-rpath6.2.2 -Version: 20240607.4.05.60202-116~20.04 -Installed-Size: 1502 - -Package: hsakmt-roct-asan6.2.2 -Architecture: amd64 -Depends: libdrm-amdgpu-dev | libdrm-dev, rocm-core-asan6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/hsakmt-roct-asan6.2.2/hsakmt-roct-asan6.2.2_20240607.4.05.60202-116~20.04_amd64.deb -Size: 300184 -SHA256: 5f89b9603f423d7df35db5331ca1b0d2f248aff933e63623951a0299cc57b860 -SHA1: 97015b5b191b77d3d1744a048d8832a723b46ffa -MD5sum: 5b22ab4347b5febb68c2028e44064592 -Description: ASAN libraries for the LIBHSAKMT -Homepage: https://github.com/ROCm/ROCT-Thunk-Interface -Maintainer: AMD GFX mailing list -Provides: hsakmt-roct6.2.2 -Recommends: libdrm-amdgpu-dev -Replaces: hsakmt-roct6.2.2 -Version: 20240607.4.05.60202-116~20.04 -Installed-Size: 1502 - -Package: hsakmt-roct-dev -Architecture: amd64 -Depends: libdrm-amdgpu-dev | libdrm-dev, rocm-core -Priority: optional -Section: devel -Filename: pool/main/h/hsakmt-roct-dev/hsakmt-roct-dev_20240607.4.05.60202-116~20.04_amd64.deb -Size: 782244 -SHA256: d368523e2454da1b42ed53bca62fef5acc33279525894783120b01b3077ae887 -SHA1: 7d03ffb25a596e3ce544e7f439a663e7b60a21d3 -MD5sum: 23e4a5922b2cd0fd527e003e795b2c87 -Description: hsakmt built using CMake -Homepage: https://github.com/ROCm/ROCT-Thunk-Interface -Maintainer: AMD GFX mailing list -Provides: hsakmt-roct -Recommends: libdrm-amdgpu-dev -Replaces: hsakmt-roct -Version: 20240607.4.05.60202-116~20.04 -Installed-Size: 2884 - -Package: hsakmt-roct-dev-rpath6.2.2 -Architecture: amd64 -Depends: libdrm-amdgpu-dev | libdrm-dev, rocm-core-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/hsakmt-roct-dev-rpath6.2.2/hsakmt-roct-dev-rpath6.2.2_20240607.4.05.60202-116~20.04_amd64.deb -Size: 278424 -SHA256: 9448e8fef9a9432b3c038f73aef5ac1ceda3c4b475e3ed4dfbccfeb18ee8361c -SHA1: bbcb2dab97ccae63b5b1c94d497d876c0393693d -MD5sum: e1fa0c8ee6817d1b8902b411916457b6 -Description: hsakmt built using CMake -Homepage: https://github.com/ROCm/ROCT-Thunk-Interface -Maintainer: AMD GFX mailing list -Provides: hsakmt-roct-rpath6.2.2 -Recommends: libdrm-amdgpu-dev -Replaces: hsakmt-roct-rpath6.2.2 -Version: 20240607.4.05.60202-116~20.04 -Installed-Size: 2884 - -Package: hsakmt-roct-dev6.2.2 -Architecture: amd64 -Depends: libdrm-amdgpu-dev | libdrm-dev, rocm-core6.2.2 -Priority: optional -Section: devel -Filename: pool/main/h/hsakmt-roct-dev6.2.2/hsakmt-roct-dev6.2.2_20240607.4.05.60202-116~20.04_amd64.deb -Size: 278524 -SHA256: 6527134032a5284f46e555bdfe177e64a14446cff744960ff1b7dc6ab6f8762c -SHA1: 34bde359076a5a8fbe548d5b88cdffcc51e4079f -MD5sum: 34ac28d1a2010e30f997badf63ca7dec -Description: hsakmt built using CMake -Homepage: https://github.com/ROCm/ROCT-Thunk-Interface -Maintainer: AMD GFX mailing list -Provides: hsakmt-roct6.2.2 -Recommends: libdrm-amdgpu-dev -Replaces: hsakmt-roct6.2.2 -Version: 20240607.4.05.60202-116~20.04 -Installed-Size: 2884 - -Package: migraphx -Architecture: amd64 -Depends: hip-dev, hip-runtime-amd, half, miopen-hip, rocblas, rocm-core -Priority: optional -Section: devel -Filename: pool/main/m/migraphx/migraphx_2.10.0.60202-116~20.04_amd64.deb -Size: 42147982 -SHA256: 73bceaee4b0e35b2323b5524b615cccdbcbf957b0d047db297b241303c0bdfa5 -SHA1: 595e82213da520e40c880d11177bfb1f1695c822 -MD5sum: dda57d400802d25ff25ec70b1ed9ceab -Description: AMD graph optimizer -Maintainer: AMDMIGraphX Maintainer -Recommends: migraphx-dev (>=2.10.0.60202) -Version: 2.10.0.60202-116~20.04 -Installed-Size: 794562 - -Package: migraphx-asan -Architecture: amd64 -Depends: hip-dev, hip-runtime-amd, half, miopen-hip, rocblas, rocm-core-asan -Priority: optional -Section: devel -Filename: pool/main/m/migraphx-asan/migraphx-asan_2.10.0.60202-116~20.04_amd64.deb -Size: 85403278 -SHA256: b975e215fd95e3456574a098bf6777c4d5feacf803a8f81ce7551638d98a8048 -SHA1: 5d85ca4d31b1afc4018e92e81a9e61532591945f -MD5sum: d07596e6e5103e3a9374e6fa6574b3a2 -Description: AMD graph optimizer -Maintainer: AMDMIGraphX Maintainer -Recommends: migraphx-asan-dev (>=2.10.0.60202) -Version: 2.10.0.60202-116~20.04 -Installed-Size: 1195805 - -Package: migraphx-asan-dbgsym -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 49d334f2acf12813aae74b5aed83a65f8fe9620b 1f259b2f8a5e26a2832e1abe9e133361adf32f74 c28b098e30f33a5e516cd9db6a8118dd83b19a1e baddce3cedc39005f38e6af3376a28adbd6b06fb 90a5698374d7ab0a77693f3d1b4a6f72fa571f92 53f21872603322527a5cd64cd35e79b525380725 442a3bdf4a19f95bcf59af5d8a96539847ff6f5b 1794a2248968ca827aaa19ae3d33c315a9461ab1 9c1af99be77770b64e08e89f62d775dfb2bb4bed 339c5035dd60e19689e5e0a1b96c4e424d57928d 92bc3f2cdc1f510dfaecb42fa1d1062ee4eb1711 -Depends: migraphx-asan (= 2.10.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/m/migraphx-asan-dbgsym/migraphx-asan-dbgsym_2.10.0.60202-116~20.04_amd64.deb -Size: 690021802 -SHA256: 4fd3bbde062d94983b9426b40402060851fa11a25f1419dd6912f17a5022e1f5 -SHA1: 2c6a7d812adadfc9fa86d650b3b476740202ce36 -MD5sum: 851890be1ec0e4814c847f08aefb5780 -Description: debug symbols for migraphx-asan -Maintainer: AMDMIGraphX Maintainer -Package-Type: ddeb -Version: 2.10.0.60202-116~20.04 -Installed-Size: 779379 - -Package: migraphx-asan-dbgsym-rpath6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 49d334f2acf12813aae74b5aed83a65f8fe9620b 1f259b2f8a5e26a2832e1abe9e133361adf32f74 c28b098e30f33a5e516cd9db6a8118dd83b19a1e baddce3cedc39005f38e6af3376a28adbd6b06fb 90a5698374d7ab0a77693f3d1b4a6f72fa571f92 53f21872603322527a5cd64cd35e79b525380725 442a3bdf4a19f95bcf59af5d8a96539847ff6f5b 1794a2248968ca827aaa19ae3d33c315a9461ab1 9c1af99be77770b64e08e89f62d775dfb2bb4bed 339c5035dd60e19689e5e0a1b96c4e424d57928d 92bc3f2cdc1f510dfaecb42fa1d1062ee4eb1711 -Depends: migraphx-asan-rpath6.2.2 (= 2.10.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/m/migraphx-asan-dbgsym-rpath6.2.2/migraphx-asan-dbgsym-rpath6.2.2_2.10.0.60202-116~20.04_amd64.deb -Size: 690023812 -SHA256: 281573c21a1f50a870bed502c52793821f2e9f422394e8d79e3fef2582c2d9d8 -SHA1: 41dd70d1a28777f6b759aaf7ad96d4bdeef9c2d4 -MD5sum: da84344b96c7d63e7fd92092a90d6d75 -Description: debug symbols for migraphx-asan -Maintainer: AMDMIGraphX Maintainer -Package-Type: ddeb -Version: 2.10.0.60202-116~20.04 -Installed-Size: 779379 - -Package: migraphx-asan-dbgsym6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 49d334f2acf12813aae74b5aed83a65f8fe9620b 1f259b2f8a5e26a2832e1abe9e133361adf32f74 c28b098e30f33a5e516cd9db6a8118dd83b19a1e baddce3cedc39005f38e6af3376a28adbd6b06fb 90a5698374d7ab0a77693f3d1b4a6f72fa571f92 53f21872603322527a5cd64cd35e79b525380725 442a3bdf4a19f95bcf59af5d8a96539847ff6f5b 1794a2248968ca827aaa19ae3d33c315a9461ab1 9c1af99be77770b64e08e89f62d775dfb2bb4bed 339c5035dd60e19689e5e0a1b96c4e424d57928d 92bc3f2cdc1f510dfaecb42fa1d1062ee4eb1711 -Depends: migraphx-asan6.2.2 (= 2.10.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/m/migraphx-asan-dbgsym6.2.2/migraphx-asan-dbgsym6.2.2_2.10.0.60202-116~20.04_amd64.deb -Size: 690023804 -SHA256: 0a5dae2b6e7a14ee3f944aab647917d359e8f5c2aa0147a13a900fb853ace6b2 -SHA1: ce081fdd0a0451a5a57c826fdab976c8ad1d72ea -MD5sum: c12fa04b3c1bc9c83f2f70480ebd3b91 -Description: debug symbols for migraphx-asan -Maintainer: AMDMIGraphX Maintainer -Package-Type: ddeb -Version: 2.10.0.60202-116~20.04 -Installed-Size: 779379 - -Package: migraphx-asan-rpath6.2.2 -Architecture: amd64 -Depends: hip-dev, hip-runtime-amd, half, miopen-hip, rocblas, rocm-core-asan-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/m/migraphx-asan-rpath6.2.2/migraphx-asan-rpath6.2.2_2.10.0.60202-116~20.04_amd64.deb -Size: 85406264 -SHA256: 50169aee2cd7560177f218d48ff7f22049cc4c8b50b2031f1955d3f21aed4f49 -SHA1: dcc0208db95350163dd135d34c5a6fefa45a6d68 -MD5sum: ea39caba31f33df6c896928d2e67e842 -Description: AMD graph optimizer -Maintainer: AMDMIGraphX Maintainer -Recommends: migraphx-asan-dev (>=2.10.0.60202) -Version: 2.10.0.60202-116~20.04 -Installed-Size: 1195805 - -Package: migraphx-asan6.2.2 -Architecture: amd64 -Depends: hip-dev, hip-runtime-amd, half, miopen-hip, rocblas, rocm-core-asan6.2.2 -Priority: optional -Section: devel -Filename: pool/main/m/migraphx-asan6.2.2/migraphx-asan6.2.2_2.10.0.60202-116~20.04_amd64.deb -Size: 85406256 -SHA256: 515c2701b6deb0f39967572bcf3ac931a7b7f071b4735047f3f5b010824a12b2 -SHA1: 5fc73eadf2b6b06d4436257111916e2996651661 -MD5sum: 241f46defef3aa2c24cf61cce691bcef -Description: AMD graph optimizer -Maintainer: AMDMIGraphX Maintainer -Recommends: migraphx-asan-dev (>=2.10.0.60202) -Version: 2.10.0.60202-116~20.04 -Installed-Size: 1195805 - -Package: migraphx-dbgsym -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 431254178447e7f44f2c7ef98df162266f8620ed 4677cc6faa169e231db0ccc7a5f43defba274d25 3123ef0bf2aaa2a7e1e9dea569d64ff1011f5a07 152d6f39b95bf157ee1611d89dceb4718508e2b3 0ff037991584d5c31553d697bb01df103a18d217 6ea6e73e004eb61b47970170c6f07eae3739256b 134fc9ded56073047c23a527e4d052a7519c4e16 314dd91f62d27ecd7d6e7a30de89623610a9ee1b 1681042e667fdbf6377e1d330ee03ca8dde0bce6 5c95d2548571c62fc39216e458832e0c987cd61d 54c43bbcff45f89b493fca5b4f68c6cdde40495e -Depends: migraphx (= 2.10.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/m/migraphx-dbgsym/migraphx-dbgsym_2.10.0.60202-116~20.04_amd64.deb -Size: 64840372 -SHA256: a6f0a96e399dc011462eb695acf6e9d46293e9e06a470bf91a8774b7d3f2da7b -SHA1: a56fe0734d1c5e148b8d5a381036b098bd0f9f19 -MD5sum: f85159357884a61f32d1eb7409335811 -Description: debug symbols for migraphx -Maintainer: AMDMIGraphX Maintainer -Package-Type: ddeb -Version: 2.10.0.60202-116~20.04 -Installed-Size: 495607 - -Package: migraphx-dbgsym-rpath6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 431254178447e7f44f2c7ef98df162266f8620ed 4677cc6faa169e231db0ccc7a5f43defba274d25 3123ef0bf2aaa2a7e1e9dea569d64ff1011f5a07 152d6f39b95bf157ee1611d89dceb4718508e2b3 0ff037991584d5c31553d697bb01df103a18d217 6ea6e73e004eb61b47970170c6f07eae3739256b 134fc9ded56073047c23a527e4d052a7519c4e16 314dd91f62d27ecd7d6e7a30de89623610a9ee1b 1681042e667fdbf6377e1d330ee03ca8dde0bce6 5c95d2548571c62fc39216e458832e0c987cd61d 54c43bbcff45f89b493fca5b4f68c6cdde40495e -Depends: migraphx-rpath6.2.2 (= 2.10.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/m/migraphx-dbgsym-rpath6.2.2/migraphx-dbgsym-rpath6.2.2_2.10.0.60202-116~20.04_amd64.deb -Size: 64781768 -SHA256: 842b82d06b950a936584a4e24dbe755fd56e7b8f77b441dc3e02e78e129c5f6f -SHA1: e35c6d029cb1f2e4b57adb5c3adea94b8d6e2157 -MD5sum: 375b4c933743f8ae2b253046753b8193 -Description: debug symbols for migraphx -Maintainer: AMDMIGraphX Maintainer -Package-Type: ddeb -Version: 2.10.0.60202-116~20.04 -Installed-Size: 495607 - -Package: migraphx-dbgsym6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 431254178447e7f44f2c7ef98df162266f8620ed 4677cc6faa169e231db0ccc7a5f43defba274d25 3123ef0bf2aaa2a7e1e9dea569d64ff1011f5a07 152d6f39b95bf157ee1611d89dceb4718508e2b3 0ff037991584d5c31553d697bb01df103a18d217 6ea6e73e004eb61b47970170c6f07eae3739256b 134fc9ded56073047c23a527e4d052a7519c4e16 314dd91f62d27ecd7d6e7a30de89623610a9ee1b 1681042e667fdbf6377e1d330ee03ca8dde0bce6 5c95d2548571c62fc39216e458832e0c987cd61d 54c43bbcff45f89b493fca5b4f68c6cdde40495e -Depends: migraphx6.2.2 (= 2.10.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/m/migraphx-dbgsym6.2.2/migraphx-dbgsym6.2.2_2.10.0.60202-116~20.04_amd64.deb -Size: 64794076 -SHA256: 01e6e9daf72dc26588f2140d53a2537092d3c30ebae80f3c1d32f82f87a26202 -SHA1: a217ef298f625081408187a1359cc05ccf623aa8 -MD5sum: b2ef84746887f47b83d0869d3d3bbb53 -Description: debug symbols for migraphx -Maintainer: AMDMIGraphX Maintainer -Package-Type: ddeb -Version: 2.10.0.60202-116~20.04 -Installed-Size: 495607 - -Package: migraphx-dev -Architecture: amd64 -Depends: migraphx (>= 2.10.0.60202) -Priority: optional -Section: devel -Filename: pool/main/m/migraphx-dev/migraphx-dev_2.10.0.60202-116~20.04_amd64.deb -Size: 158080 -SHA256: 837555aa40966a44168b080e7131066acd90fd96d752a73f01bc3783f3f48368 -SHA1: 32c361bc5d7e22a7400ff39437b0a230ca36a32a -MD5sum: 8f306019b88c3ad03f98abecf8bf2708 -Description: AMD graph optimizer -Maintainer: AMDMIGraphX Maintainer -Version: 2.10.0.60202-116~20.04 -Installed-Size: 1555 - -Package: migraphx-dev-rpath6.2.2 -Architecture: amd64 -Depends: migraphx-rpath6.2.2 (>= 2.10.0.60202) -Priority: optional -Section: devel -Filename: pool/main/m/migraphx-dev-rpath6.2.2/migraphx-dev-rpath6.2.2_2.10.0.60202-116~20.04_amd64.deb -Size: 156792 -SHA256: 98d1ef88aeb7c9d8d4badb06f5beb5201797576f010d185cf1dd0657d6f99f10 -SHA1: e6c1877dda4e88e294c9fbca6bba78b14309aafa -MD5sum: 36b7b9536e984c8c71f363f132d4955c -Description: AMD graph optimizer -Maintainer: AMDMIGraphX Maintainer -Version: 2.10.0.60202-116~20.04 -Installed-Size: 1555 - -Package: migraphx-dev6.2.2 -Architecture: amd64 -Depends: migraphx6.2.2 (>= 2.10.0.60202) -Priority: optional -Section: devel -Filename: pool/main/m/migraphx-dev6.2.2/migraphx-dev6.2.2_2.10.0.60202-116~20.04_amd64.deb -Size: 156768 -SHA256: 16df80c6c93732a95cc0ceae755e3c7775deebc86823ea11ceee4d9256a33cc9 -SHA1: 89fbcc87889c73da8883c514bcd1ad03e9eacba5 -MD5sum: 0fa096def642f19c37d5027bc0e412e4 -Description: AMD graph optimizer -Maintainer: AMDMIGraphX Maintainer -Version: 2.10.0.60202-116~20.04 -Installed-Size: 1555 - -Package: migraphx-rpath6.2.2 -Architecture: amd64 -Depends: hip-dev-rpath6.2.2, hip-runtime-amd-rpath6.2.2, half-rpath6.2.2, miopen-hip-rpath6.2.2, rocblas-rpath6.2.2, rocm-core-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/m/migraphx-rpath6.2.2/migraphx-rpath6.2.2_2.10.0.60202-116~20.04_amd64.deb -Size: 42247140 -SHA256: 77d9425c10cb50a8bea69bb4a4272297b78ec7df38f8bfc8acc4c643e9981340 -SHA1: 0a6acf2dbca6638ee5a18193dd4d34fb4f4e592b -MD5sum: e0b56b696f73ea299267ddd01806e0a3 -Description: AMD graph optimizer -Maintainer: AMDMIGraphX Maintainer -Recommends: migraphx-dev-rpath6.2.2 (>=2.10.0.60202) -Version: 2.10.0.60202-116~20.04 -Installed-Size: 794562 - -Package: migraphx6.2.2 -Architecture: amd64 -Depends: hip-dev6.2.2, hip-runtime-amd6.2.2, half6.2.2, miopen-hip6.2.2, rocblas6.2.2, rocm-core6.2.2 -Priority: optional -Section: devel -Filename: pool/main/m/migraphx6.2.2/migraphx6.2.2_2.10.0.60202-116~20.04_amd64.deb -Size: 42247860 -SHA256: 3e01365133505295a5088bd6f28f458692b78b849a13c2a7f3f7659c13c3b453 -SHA1: 50ac8d9a0e25b0b451b2d7e6bf400df6274a198d -MD5sum: 536679787ec63074c0ad69ee9b56272d -Description: AMD graph optimizer -Maintainer: AMDMIGraphX Maintainer -Recommends: migraphx-dev6.2.2 (>=2.10.0.60202) -Version: 2.10.0.60202-116~20.04 -Installed-Size: 794562 - -Package: miopen-hip -Architecture: amd64 -Conflicts: miopen-opencl -Depends: hip-runtime-amd, comgr, roctracer, rocblas, rocm-core, rocrand, rocm-core -Priority: optional -Section: devel -Filename: pool/main/m/miopen-hip/miopen-hip_3.2.0.60202-116~20.04_amd64.deb -Size: 113650750 -SHA256: d09d1242da0d10be4b0a43b38ec9a2755b447e5c53088f632ea5e9c82d8d0b3a -SHA1: eb00d89463655947346989bf04aefb66f8ff35d9 -MD5sum: adf4d7447e3d70a8d0cd08df7d81f618 -Description: AMD DNN Library -Maintainer: MIOpen Maintainer -Recommends: miopen-hip-dev (>=3.2.0.60202) -Version: 3.2.0.60202-116~20.04 -Installed-Size: 3386725 - -Package: miopen-hip-asan -Architecture: amd64 -Conflicts: miopen-opencl -Depends: hip-runtime-amd-asan, comgr, roctracer, rocblas, rocm-core, rocrand, rocm-core-asan -Priority: optional -Section: devel -Filename: pool/main/m/miopen-hip-asan/miopen-hip-asan_3.2.0.60202-116~20.04_amd64.deb -Size: 78156770 -SHA256: d6f67713ac2e24abbe2776774df3486afb87460765a988d1d95f1aab4a39a6c4 -SHA1: 9a184200acaf19d7865eeace3782ee89018b27cc -MD5sum: 059a736d18453900a37e783d129a7dbd -Description: AMD DNN Library -Maintainer: MIOpen Maintainer -Recommends: miopen-hip-asan-dev (>=3.2.0.60202) -Version: 3.2.0.60202-116~20.04 -Installed-Size: 3054008 - -Package: miopen-hip-asan-rpath6.2.2 -Architecture: amd64 -Conflicts: miopen-opencl-rpath6.2.2 -Depends: hip-runtime-amd-asan-rpath6.2.2, comgr, roctracer, rocblas, rocm-core, rocrand, rocm-core-asan-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/m/miopen-hip-asan-rpath6.2.2/miopen-hip-asan-rpath6.2.2_3.2.0.60202-116~20.04_amd64.deb -Size: 78169200 -SHA256: ed8e01df803219ff0c2769dff36e50415f6ce3accd6cf5b70a58d8bac255950d -SHA1: 2bbc45618cd05eab264f6039112d13973d7dbab3 -MD5sum: 340451846a8c1c54c48bfb25632935a7 -Description: AMD DNN Library -Maintainer: MIOpen Maintainer -Recommends: miopen-hip-asan-dev (>=3.2.0.60202) -Version: 3.2.0.60202-116~20.04 -Installed-Size: 3054008 - -Package: miopen-hip-asan6.2.2 -Architecture: amd64 -Conflicts: miopen-opencl6.2.2 -Depends: hip-runtime-amd-asan6.2.2, comgr, roctracer, rocblas, rocm-core, rocrand, rocm-core-asan6.2.2 -Priority: optional -Section: devel -Filename: pool/main/m/miopen-hip-asan6.2.2/miopen-hip-asan6.2.2_3.2.0.60202-116~20.04_amd64.deb -Size: 78164376 -SHA256: 5ee4af760381bde3464be90b290808f5bfb9e954c1db329728aafeda6d636dcc -SHA1: f3ca7bd0dc31cb6c1cde376af4d40cd515481de2 -MD5sum: 730be93858a2d08e69426a6070634173 -Description: AMD DNN Library -Maintainer: MIOpen Maintainer -Recommends: miopen-hip-asan-dev (>=3.2.0.60202) -Version: 3.2.0.60202-116~20.04 -Installed-Size: 3054008 - -Package: miopen-hip-client -Architecture: amd64 -Conflicts: miopen-opencl -Depends: hip-runtime-amd, comgr, roctracer, rocblas, rocm-core, rocrand, rocm-core -Priority: optional -Section: devel -Filename: pool/main/m/miopen-hip-client/miopen-hip-client_3.2.0.60202-116~20.04_amd64.deb -Size: 744 -SHA256: 01105d41ece8a1d185d8de2bf9ae5a34aa14e33086ddd24be05415f0bc4b9f99 -SHA1: 3aaa8f2dc7219549a76d275769027fddf528b36b -MD5sum: c6913c214a3e320b027accd2bcc5ee74 -Description: AMD DNN Library -Maintainer: MIOpen Maintainer -Version: 3.2.0.60202-116~20.04 -Installed-Size: 8 - -Package: miopen-hip-client-rpath6.2.2 -Architecture: amd64 -Conflicts: miopen-opencl-rpath6.2.2 -Depends: hip-runtime-amd-rpath6.2.2, comgr-rpath6.2.2, roctracer-rpath6.2.2, rocblas-rpath6.2.2, rocm-core-rpath6.2.2, rocrand-rpath6.2.2, rocm-core-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/m/miopen-hip-client-rpath6.2.2/miopen-hip-client-rpath6.2.2_3.2.0.60202-116~20.04_amd64.deb -Size: 904 -SHA256: 1347a35d02a2c081447aa18a76d8cd10ba105bab961ff7f93403eb5b31310fd9 -SHA1: 7c66dc1b17df2e2dbb7938d8f84f7e1232630163 -MD5sum: dc6617d946f9d90d6b292ffdab4652c8 -Description: AMD DNN Library -Maintainer: MIOpen Maintainer -Version: 3.2.0.60202-116~20.04 -Installed-Size: 8 - -Package: miopen-hip-client6.2.2 -Architecture: amd64 -Conflicts: miopen-opencl6.2.2 -Depends: hip-runtime-amd6.2.2, comgr6.2.2, roctracer6.2.2, rocblas6.2.2, rocm-core6.2.2, rocrand6.2.2, rocm-core6.2.2 -Priority: optional -Section: devel -Filename: pool/main/m/miopen-hip-client6.2.2/miopen-hip-client6.2.2_3.2.0.60202-116~20.04_amd64.deb -Size: 896 -SHA256: 28ce50eb0b4c8e1b1433621f74a1faaddac65f2e2a10b7638569d12228befc72 -SHA1: 21a2176cedebda233e60244dc4cc0a20357943ba -MD5sum: d03853532d52b1117fb7f5d45c77af55 -Description: AMD DNN Library -Maintainer: MIOpen Maintainer -Version: 3.2.0.60202-116~20.04 -Installed-Size: 8 - -Package: miopen-hip-dev -Architecture: amd64 -Conflicts: miopen-opencl -Depends: miopen-hip (>= 3.2.0.60202) -Priority: optional -Section: devel -Filename: pool/main/m/miopen-hip-dev/miopen-hip-dev_3.2.0.60202-116~20.04_amd64.deb -Size: 44108 -SHA256: 079092642b3f21eb9aef1f491b35d8f9544711c66c807de9296d6e46f224ccc6 -SHA1: ed989bf0eaf3d76c31392fd1fdeec3cf33dd839f -MD5sum: 7869ddcd624d5e9a65f4e43c39ced92c -Description: AMD DNN Library -Maintainer: MIOpen Maintainer -Version: 3.2.0.60202-116~20.04 -Installed-Size: 409 - -Package: miopen-hip-dev-rpath6.2.2 -Architecture: amd64 -Conflicts: miopen-opencl-rpath6.2.2 -Depends: miopen-hip-rpath6.2.2 (>= 3.2.0.60202) -Priority: optional -Section: devel -Filename: pool/main/m/miopen-hip-dev-rpath6.2.2/miopen-hip-dev-rpath6.2.2_3.2.0.60202-116~20.04_amd64.deb -Size: 44256 -SHA256: a86606c9ac2bcd406fd5587f5b8ce1dae2c84df6355c89a4f0b9d4375b322dec -SHA1: 68ed813e197563a41094e58a0200f9a1ca12b51a -MD5sum: 2c1beaa724ad8752809daa9165e0db69 -Description: AMD DNN Library -Maintainer: MIOpen Maintainer -Version: 3.2.0.60202-116~20.04 -Installed-Size: 409 - -Package: miopen-hip-dev6.2.2 -Architecture: amd64 -Conflicts: miopen-opencl6.2.2 -Depends: miopen-hip6.2.2 (>= 3.2.0.60202) -Priority: optional -Section: devel -Filename: pool/main/m/miopen-hip-dev6.2.2/miopen-hip-dev6.2.2_3.2.0.60202-116~20.04_amd64.deb -Size: 44268 -SHA256: de5f2e02f5951a909851321511c65b09579f46a04f89f690f7393113f713302b -SHA1: 19ea7c437ef28ff9b78b806ef77cdcf5ec503406 -MD5sum: e5a1325cb0b697ed13acfc038bee63e2 -Description: AMD DNN Library -Maintainer: MIOpen Maintainer -Version: 3.2.0.60202-116~20.04 -Installed-Size: 409 - -Package: miopen-hip-gfx1030kdb -Architecture: amd64 -Conflicts: miopen-opencl -Depends: hip-runtime-amd, comgr, roctracer, rocblas, rocm-core, rocrand, rocm-core -Priority: optional -Section: devel -Filename: pool/main/m/miopen-hip-gfx1030kdb/miopen-hip-gfx1030kdb_3.2.0.60202-116~20.04_amd64.deb -Size: 390901138 -SHA256: f5fc7334ad33a9ff1f1aa6ff8fabbb52d923c32cb9870a1cf3e0a1719d02fc44 -SHA1: 56ebb66ee09168b2ed3b26247f8fc65a3416bff3 -MD5sum: 2a1887c24df76cb3d7d5d01af6faf91f -Description: AMD DNN Library -Maintainer: MIOpen Maintainer -Version: 3.2.0.60202-116~20.04 -Installed-Size: 727956 - -Package: miopen-hip-gfx1030kdb-rpath6.2.2 -Architecture: amd64 -Conflicts: miopen-opencl-rpath6.2.2 -Depends: hip-runtime-amd-rpath6.2.2, comgr-rpath6.2.2, roctracer-rpath6.2.2, rocblas-rpath6.2.2, rocm-core-rpath6.2.2, rocrand-rpath6.2.2, rocm-core-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/m/miopen-hip-gfx1030kdb-rpath6.2.2/miopen-hip-gfx1030kdb-rpath6.2.2_3.2.0.60202-116~20.04_amd64.deb -Size: 390904624 -SHA256: 19120b8c834c96d8c9b014b1c4fbcc0f6f91ce09824f9b29c1d23c779c2671d2 -SHA1: 2d8541e0b3faf68c0e9afbd818ef456c74ca9599 -MD5sum: 2335c040fc779108194daca69669f8e1 -Description: AMD DNN Library -Maintainer: MIOpen Maintainer -Version: 3.2.0.60202-116~20.04 -Installed-Size: 727956 - -Package: miopen-hip-gfx1030kdb6.2.2 -Architecture: amd64 -Conflicts: miopen-opencl6.2.2 -Depends: hip-runtime-amd6.2.2, comgr6.2.2, roctracer6.2.2, rocblas6.2.2, rocm-core6.2.2, rocrand6.2.2, rocm-core6.2.2 -Priority: optional -Section: devel -Filename: pool/main/m/miopen-hip-gfx1030kdb6.2.2/miopen-hip-gfx1030kdb6.2.2_3.2.0.60202-116~20.04_amd64.deb -Size: 390902808 -SHA256: 3b940fa822249be2eb3cb662c36522350fbbf0630467ca8af2cd7449584edcd5 -SHA1: aa8784e260502505640a2b50774744ab46a20901 -MD5sum: 647dcb11cc9f5591e0013619f4df5323 -Description: AMD DNN Library -Maintainer: MIOpen Maintainer -Version: 3.2.0.60202-116~20.04 -Installed-Size: 727956 - -Package: miopen-hip-gfx900kdb -Architecture: amd64 -Conflicts: miopen-opencl -Depends: hip-runtime-amd, comgr, roctracer, rocblas, rocm-core, rocrand, rocm-core -Priority: optional -Section: devel -Filename: pool/main/m/miopen-hip-gfx900kdb/miopen-hip-gfx900kdb_3.2.0.60202-116~20.04_amd64.deb -Size: 272267180 -SHA256: ed990ceeb79db2282a58dafd9e4fb45c14ab9bbd5df6d53fbb5d40925818c8d2 -SHA1: e594c631857f86bd7ecda5154ec09fc01184c87c -MD5sum: 182ff592b06267bcb4b749fd338585ae -Description: AMD DNN Library -Maintainer: MIOpen Maintainer -Version: 3.2.0.60202-116~20.04 -Installed-Size: 518104 - -Package: miopen-hip-gfx900kdb-rpath6.2.2 -Architecture: amd64 -Conflicts: miopen-opencl-rpath6.2.2 -Depends: hip-runtime-amd-rpath6.2.2, comgr-rpath6.2.2, roctracer-rpath6.2.2, rocblas-rpath6.2.2, rocm-core-rpath6.2.2, rocrand-rpath6.2.2, rocm-core-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/m/miopen-hip-gfx900kdb-rpath6.2.2/miopen-hip-gfx900kdb-rpath6.2.2_3.2.0.60202-116~20.04_amd64.deb -Size: 272265436 -SHA256: 4980ddb6343da52f72867534e335f9240cd6061987ddfeaf654c596fd2750ed1 -SHA1: d841e317940b3890262f2b4f6f2707242cf5e0ec -MD5sum: 32f6bc04ac7f48e1edb5f041d0110d18 -Description: AMD DNN Library -Maintainer: MIOpen Maintainer -Version: 3.2.0.60202-116~20.04 -Installed-Size: 518104 - -Package: miopen-hip-gfx900kdb6.2.2 -Architecture: amd64 -Conflicts: miopen-opencl6.2.2 -Depends: hip-runtime-amd6.2.2, comgr6.2.2, roctracer6.2.2, rocblas6.2.2, rocm-core6.2.2, rocrand6.2.2, rocm-core6.2.2 -Priority: optional -Section: devel -Filename: pool/main/m/miopen-hip-gfx900kdb6.2.2/miopen-hip-gfx900kdb6.2.2_3.2.0.60202-116~20.04_amd64.deb -Size: 272266140 -SHA256: c5fbf2161400127bd742fafd713dae60fa8572f6e596091dec1526776d32eefd -SHA1: 60707f9031c0068261c3627a210f0728f8c597d0 -MD5sum: 88958acff2c4d52d4615a06a5647e607 -Description: AMD DNN Library -Maintainer: MIOpen Maintainer -Version: 3.2.0.60202-116~20.04 -Installed-Size: 518104 - -Package: miopen-hip-gfx906kdb -Architecture: amd64 -Conflicts: miopen-opencl -Depends: hip-runtime-amd, comgr, roctracer, rocblas, rocm-core, rocrand, rocm-core -Priority: optional -Section: devel -Filename: pool/main/m/miopen-hip-gfx906kdb/miopen-hip-gfx906kdb_3.2.0.60202-116~20.04_amd64.deb -Size: 321529736 -SHA256: 3a8efe4c950ec7ac032e2ca799ed5d5d01783c4ab856576327882ceadd614aa7 -SHA1: 1afe3d72bfcc96a7d1f6028497855ebf8ab9937b -MD5sum: 177570d4ee2d49849d4e394ce87d1cc4 -Description: AMD DNN Library -Maintainer: MIOpen Maintainer -Version: 3.2.0.60202-116~20.04 -Installed-Size: 608416 - -Package: miopen-hip-gfx906kdb-rpath6.2.2 -Architecture: amd64 -Conflicts: miopen-opencl-rpath6.2.2 -Depends: hip-runtime-amd-rpath6.2.2, comgr-rpath6.2.2, roctracer-rpath6.2.2, rocblas-rpath6.2.2, rocm-core-rpath6.2.2, rocrand-rpath6.2.2, rocm-core-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/m/miopen-hip-gfx906kdb-rpath6.2.2/miopen-hip-gfx906kdb-rpath6.2.2_3.2.0.60202-116~20.04_amd64.deb -Size: 321529900 -SHA256: c60381752265e588c698579ee6fbebab3b5601d05e011f44fbe9e800fb8f9a87 -SHA1: 134599f2b1c31148dd8bceb419bade852fd4b08e -MD5sum: 06692f7bcb1f466033946d049a7a1c8b -Description: AMD DNN Library -Maintainer: MIOpen Maintainer -Version: 3.2.0.60202-116~20.04 -Installed-Size: 608416 - -Package: miopen-hip-gfx906kdb6.2.2 -Architecture: amd64 -Conflicts: miopen-opencl6.2.2 -Depends: hip-runtime-amd6.2.2, comgr6.2.2, roctracer6.2.2, rocblas6.2.2, rocm-core6.2.2, rocrand6.2.2, rocm-core6.2.2 -Priority: optional -Section: devel -Filename: pool/main/m/miopen-hip-gfx906kdb6.2.2/miopen-hip-gfx906kdb6.2.2_3.2.0.60202-116~20.04_amd64.deb -Size: 321530188 -SHA256: c878bc7fd299d303006381880bfc0f3ca1acc9827034863e41b12006bdbf640c -SHA1: fe70bf4bf039e0603c2969efdaed6de06a0b5176 -MD5sum: 9627b5a38b4a560f2fd2a28f1c51020e -Description: AMD DNN Library -Maintainer: MIOpen Maintainer -Version: 3.2.0.60202-116~20.04 -Installed-Size: 608416 - -Package: miopen-hip-gfx908kdb -Architecture: amd64 -Conflicts: miopen-opencl -Depends: hip-runtime-amd, comgr, roctracer, rocblas, rocm-core, rocrand, rocm-core -Priority: optional -Section: devel -Filename: pool/main/m/miopen-hip-gfx908kdb/miopen-hip-gfx908kdb_3.2.0.60202-116~20.04_amd64.deb -Size: 241882930 -SHA256: d0a14b6a632ea8f044faeee00e6cb6c5d9646dd68f71164aa026ac18158cd66a -SHA1: 04d07328fbe8a51ba19b869c7d1d0bfa7357d6f2 -MD5sum: c24932a139c574c40deeada53af223c6 -Description: AMD DNN Library -Maintainer: MIOpen Maintainer -Version: 3.2.0.60202-116~20.04 -Installed-Size: 629992 - -Package: miopen-hip-gfx908kdb-rpath6.2.2 -Architecture: amd64 -Conflicts: miopen-opencl-rpath6.2.2 -Depends: hip-runtime-amd-rpath6.2.2, comgr-rpath6.2.2, roctracer-rpath6.2.2, rocblas-rpath6.2.2, rocm-core-rpath6.2.2, rocrand-rpath6.2.2, rocm-core-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/m/miopen-hip-gfx908kdb-rpath6.2.2/miopen-hip-gfx908kdb-rpath6.2.2_3.2.0.60202-116~20.04_amd64.deb -Size: 241885716 -SHA256: 39a65fa071add3a3aebddd7bc54004eaa58a9be925509a39d66b271e88385ca8 -SHA1: e637f9df506df798a37362fa52d1799e24c8b809 -MD5sum: c959c0b753bfd6d4fd387c932e0fd4bd -Description: AMD DNN Library -Maintainer: MIOpen Maintainer -Version: 3.2.0.60202-116~20.04 -Installed-Size: 629992 - -Package: miopen-hip-gfx908kdb6.2.2 -Architecture: amd64 -Conflicts: miopen-opencl6.2.2 -Depends: hip-runtime-amd6.2.2, comgr6.2.2, roctracer6.2.2, rocblas6.2.2, rocm-core6.2.2, rocrand6.2.2, rocm-core6.2.2 -Priority: optional -Section: devel -Filename: pool/main/m/miopen-hip-gfx908kdb6.2.2/miopen-hip-gfx908kdb6.2.2_3.2.0.60202-116~20.04_amd64.deb -Size: 241883660 -SHA256: 5e8bdc6223e9b3eb5adba2e7056bfe96603e779bbafd7265fede25aa53ab266d -SHA1: 502f489e043a4e8b690dca89a511fb5421edb28d -MD5sum: c5fe0fbbdbe223e0fbd3881ad6d6b4cd -Description: AMD DNN Library -Maintainer: MIOpen Maintainer -Version: 3.2.0.60202-116~20.04 -Installed-Size: 629992 - -Package: miopen-hip-gfx90akdb -Architecture: amd64 -Conflicts: miopen-opencl -Depends: hip-runtime-amd, comgr, roctracer, rocblas, rocm-core, rocrand, rocm-core -Priority: optional -Section: devel -Filename: pool/main/m/miopen-hip-gfx90akdb/miopen-hip-gfx90akdb_3.2.0.60202-116~20.04_amd64.deb -Size: 134604752 -SHA256: 35bbb7679389280361d47c8a8768dcd7ed0088551e04b1df98a1c7894e50c358 -SHA1: 3b7ad03313bce3d01444eeda81322fc4ad44adab -MD5sum: d95e84833e44228ab77660ff58e80254 -Description: AMD DNN Library -Maintainer: MIOpen Maintainer -Version: 3.2.0.60202-116~20.04 -Installed-Size: 342828 - -Package: miopen-hip-gfx90akdb-rpath6.2.2 -Architecture: amd64 -Conflicts: miopen-opencl-rpath6.2.2 -Depends: hip-runtime-amd-rpath6.2.2, comgr-rpath6.2.2, roctracer-rpath6.2.2, rocblas-rpath6.2.2, rocm-core-rpath6.2.2, rocrand-rpath6.2.2, rocm-core-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/m/miopen-hip-gfx90akdb-rpath6.2.2/miopen-hip-gfx90akdb-rpath6.2.2_3.2.0.60202-116~20.04_amd64.deb -Size: 134605024 -SHA256: ea8d193771b01586ed8e3dc53acffc4a5f6f2ead643e6862e250a5ed0bfb6679 -SHA1: d12a713bec5f135b93ff63206ce4731a05a674b2 -MD5sum: 2237e76ac590c46fb1f1e8d4501fba9a -Description: AMD DNN Library -Maintainer: MIOpen Maintainer -Version: 3.2.0.60202-116~20.04 -Installed-Size: 342828 - -Package: miopen-hip-gfx90akdb6.2.2 -Architecture: amd64 -Conflicts: miopen-opencl6.2.2 -Depends: hip-runtime-amd6.2.2, comgr6.2.2, roctracer6.2.2, rocblas6.2.2, rocm-core6.2.2, rocrand6.2.2, rocm-core6.2.2 -Priority: optional -Section: devel -Filename: pool/main/m/miopen-hip-gfx90akdb6.2.2/miopen-hip-gfx90akdb6.2.2_3.2.0.60202-116~20.04_amd64.deb -Size: 134604952 -SHA256: 17495b39d1b48375389faafef70f6e3a49b1f734e6385980319e9589f1f4c032 -SHA1: 33cac5c671f2d1a5155c58a26837d0f7f51369b8 -MD5sum: fb811ad9f75b83cc040c1a96386de40e -Description: AMD DNN Library -Maintainer: MIOpen Maintainer -Version: 3.2.0.60202-116~20.04 -Installed-Size: 342828 - -Package: miopen-hip-rpath6.2.2 -Architecture: amd64 -Conflicts: miopen-opencl-rpath6.2.2 -Depends: hip-runtime-amd-rpath6.2.2, comgr-rpath6.2.2, roctracer-rpath6.2.2, rocblas-rpath6.2.2, rocm-core-rpath6.2.2, rocrand-rpath6.2.2, rocm-core-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/m/miopen-hip-rpath6.2.2/miopen-hip-rpath6.2.2_3.2.0.60202-116~20.04_amd64.deb -Size: 113643344 -SHA256: e34349228bea0cdf3396dde20b6ddc6ca42808b269af50a4293cab4cd22744b8 -SHA1: d14dc568718fa65ff85badc2f8ef1e9312111341 -MD5sum: b3cacf1631fe91b3a48c60d558728e9c -Description: AMD DNN Library -Maintainer: MIOpen Maintainer -Recommends: miopen-hip-dev-rpath6.2.2 (>=3.2.0.60202) -Version: 3.2.0.60202-116~20.04 -Installed-Size: 3386725 - -Package: miopen-hip6.2.2 -Architecture: amd64 -Conflicts: miopen-opencl6.2.2 -Depends: hip-runtime-amd6.2.2, comgr6.2.2, roctracer6.2.2, rocblas6.2.2, rocm-core6.2.2, rocrand6.2.2, rocm-core6.2.2 -Priority: optional -Section: devel -Filename: pool/main/m/miopen-hip6.2.2/miopen-hip6.2.2_3.2.0.60202-116~20.04_amd64.deb -Size: 113650316 -SHA256: 61a861edd5e1c5b65cc67de0d937b1e0e847d475cf5a6452328d40155ffd8caa -SHA1: c54f6a61093246d858838e9ec191e71bcfb95b9b -MD5sum: a570a86eda045b2f6c3eaaae1d1ba49d -Description: AMD DNN Library -Maintainer: MIOpen Maintainer -Recommends: miopen-hip-dev6.2.2 (>=3.2.0.60202) -Version: 3.2.0.60202-116~20.04 -Installed-Size: 3386725 - -Package: mivisionx -Architecture: amd64 -Depends: rocm-core, rocm-hip-runtime, half, rpp, rocblas, miopen-hip, migraphx, rocdecode -Priority: optional -Section: devel -Filename: pool/main/m/mivisionx/mivisionx_3.0.0.60202-116~20.04_amd64.deb -Size: 43642404 -SHA256: c5e7052414e6c9b4150209a71e75f7d7c1962ad2e8c6b19df004dd852ac45353 -SHA1: 359b09582b148972475056c2dd4136c5613b069a -MD5sum: cfb50177a02101c4d7fffb8747a4b6bb -Description: AMD MIVisionX is a comprehensive Computer Vision and ML Inference Toolkit. MIVisionX runtime package provides MIVisionX libraries and license.txt -Homepage: https://github.com/ROCm/MIVisionX -Maintainer: MIVisionX Support -Version: 3.0.0.60202-116~20.04 -Installed-Size: 266541 - -Package: mivisionx-asan -Architecture: amd64 -Depends: rocm-core-asan, rocm-hip-runtime, half, rpp, rocblas, miopen-hip, migraphx, rocdecode -Priority: optional -Section: devel -Filename: pool/main/m/mivisionx-asan/mivisionx-asan_3.0.0.60202-116~20.04_amd64.deb -Size: 40856064 -SHA256: 6652a59003c3566f426aa7ba75cf34752304cc7f0f58106d92e6851cb03701ad -SHA1: 3e26d2bc5e91c6ce98ea0fb19bfb89532a74ede2 -MD5sum: 5cdf2e62fc01224c3df61057879a5a10 -Description: AMD MIVisionX is a comprehensive Computer Vision and ML Inference Toolkit. MIVisionX ASAN package provides MIVisionX ASAN libraries -Homepage: https://github.com/ROCm/MIVisionX -Maintainer: MIVisionX Support -Version: 3.0.0.60202-116~20.04 -Installed-Size: 282749 - -Package: mivisionx-asan-dbgsym -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: ed2cc8b5b77a33504f3b003cc546f746bfaddb03 a73359fc0ed549f2390b0bd6ace19980a7178372 98fb1db0f99de9e314c62fca92243794d79863ce 573f758000a0972328cf4eb1467b5f06d6c398ff 225f3ee910220e5a34e91b35eb507519fbb82243 e559000ef79d15d8b5f22b69ad34133b7279aeab c831f5d777f326f09461f6f83738e6f1cd85d51d f0c461ec9db8c9c84146657dead6d630e5a776a9 -Depends: mivisionx-asan (= 3.0.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/m/mivisionx-asan-dbgsym/mivisionx-asan-dbgsym_3.0.0.60202-116~20.04_amd64.deb -Size: 7340956 -SHA256: 9c37970203757041f357d294dee96f56baa67147436a22929e48744132463f3d -SHA1: 11efe210feca870966d65c41518f6919d6ef2477 -MD5sum: 0abec14b8d853390f3429e1297b425df -Description: debug symbols for mivisionx-asan -Maintainer: MIVisionX Support -Package-Type: ddeb -Version: 3.0.0.60202-116~20.04 -Installed-Size: 8487 - -Package: mivisionx-asan-dbgsym-rpath6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: ed2cc8b5b77a33504f3b003cc546f746bfaddb03 a73359fc0ed549f2390b0bd6ace19980a7178372 98fb1db0f99de9e314c62fca92243794d79863ce 573f758000a0972328cf4eb1467b5f06d6c398ff 225f3ee910220e5a34e91b35eb507519fbb82243 e559000ef79d15d8b5f22b69ad34133b7279aeab c831f5d777f326f09461f6f83738e6f1cd85d51d f0c461ec9db8c9c84146657dead6d630e5a776a9 -Depends: mivisionx-asan-rpath6.2.2 (= 3.0.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/m/mivisionx-asan-dbgsym-rpath6.2.2/mivisionx-asan-dbgsym-rpath6.2.2_3.0.0.60202-116~20.04_amd64.deb -Size: 7176168 -SHA256: 20d4cf3ba886afe20c0723a3e75bdbad00a7d96b557a05d0d9ad3d93bbf1b218 -SHA1: b84b486ffe572ae2e416e6a528825e334998acf9 -MD5sum: f46801241e15846ab4c5aba59575432e -Description: debug symbols for mivisionx-asan -Maintainer: MIVisionX Support -Package-Type: ddeb -Version: 3.0.0.60202-116~20.04 -Installed-Size: 8487 - -Package: mivisionx-asan-dbgsym6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: ed2cc8b5b77a33504f3b003cc546f746bfaddb03 a73359fc0ed549f2390b0bd6ace19980a7178372 98fb1db0f99de9e314c62fca92243794d79863ce 573f758000a0972328cf4eb1467b5f06d6c398ff 225f3ee910220e5a34e91b35eb507519fbb82243 e559000ef79d15d8b5f22b69ad34133b7279aeab c831f5d777f326f09461f6f83738e6f1cd85d51d f0c461ec9db8c9c84146657dead6d630e5a776a9 -Depends: mivisionx-asan6.2.2 (= 3.0.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/m/mivisionx-asan-dbgsym6.2.2/mivisionx-asan-dbgsym6.2.2_3.0.0.60202-116~20.04_amd64.deb -Size: 7176164 -SHA256: db35dcb49e5d60b278b40c8955e1c837a81b95ce2d9051dc1494ee73ef1603a8 -SHA1: b7a671c94910a40b67af1496eb863382602c0e44 -MD5sum: 492e40a3171049b684fe90d264fce85a -Description: debug symbols for mivisionx-asan -Maintainer: MIVisionX Support -Package-Type: ddeb -Version: 3.0.0.60202-116~20.04 -Installed-Size: 8487 - -Package: mivisionx-asan-rpath6.2.2 -Architecture: amd64 -Depends: rocm-core-asan-rpath6.2.2, rocm-hip-runtime, half, rpp, rocblas, miopen-hip, migraphx, rocdecode -Priority: optional -Section: devel -Filename: pool/main/m/mivisionx-asan-rpath6.2.2/mivisionx-asan-rpath6.2.2_3.0.0.60202-116~20.04_amd64.deb -Size: 18491356 -SHA256: 20f64b0c42134831b4dec731be1bbb6809ebdcdb32d79c784c58dc73a3aa25c3 -SHA1: 52f51fe338a44b815a4cd7c0b82d7fef263b48f7 -MD5sum: 9480034edac7d6726ede062e435afae9 -Description: AMD MIVisionX is a comprehensive Computer Vision and ML Inference Toolkit. MIVisionX ASAN package provides MIVisionX ASAN libraries -Homepage: https://github.com/ROCm/MIVisionX -Maintainer: MIVisionX Support -Version: 3.0.0.60202-116~20.04 -Installed-Size: 282749 - -Package: mivisionx-asan6.2.2 -Architecture: amd64 -Depends: rocm-core-asan6.2.2, rocm-hip-runtime, half, rpp, rocblas, miopen-hip, migraphx, rocdecode -Priority: optional -Section: devel -Filename: pool/main/m/mivisionx-asan6.2.2/mivisionx-asan6.2.2_3.0.0.60202-116~20.04_amd64.deb -Size: 18490868 -SHA256: 5c9c792750d2809d999a90633bc8b17f868f1d21ea13ef7fe9302807ec7fd7a8 -SHA1: 9d2623a06fe31fb52bb78f3f6b0c4b6947651494 -MD5sum: 1d855157eef35d7a25719b9407cf5e93 -Description: AMD MIVisionX is a comprehensive Computer Vision and ML Inference Toolkit. MIVisionX ASAN package provides MIVisionX ASAN libraries -Homepage: https://github.com/ROCm/MIVisionX -Maintainer: MIVisionX Support -Version: 3.0.0.60202-116~20.04 -Installed-Size: 282749 - -Package: mivisionx-dbgsym -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 5515c365a55d977b989a9a78b6dce53a6ad734c0 06e81f74b3808d260af14db37fd7152bde58a51c d52bc5ff99c8364cdd20192d9f879364de27abf6 46462528a11539a3fd1236aa08bc6caa60d00ffd 917c36ca6652adc23a8f3b26d8052f67daafacc3 96d74e856dad3f2f0ea83739d418c8fb769f6240 c9938d0502c294089fc0bfbeb9b36ff9db3fd6a0 4c19faee3747f8ed8a08da268c053cf75de86daf 3ac5215c62270e7c9fc861f6a94ec1a6f3bd4b56 94f0c8a5ec4764806d27a71ae9dbacbf47ab48cd -Depends: mivisionx (= 3.0.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/m/mivisionx-dbgsym/mivisionx-dbgsym_3.0.0.60202-116~20.04_amd64.deb -Size: 16582476 -SHA256: cfaaa90cc10ceb1f735e35da270e9cc4db05b1f5eaceac3fed6a89d31877fa4a -SHA1: 96ee2cb17fb46a977dfc1b5c7ade3050d46ee9c1 -MD5sum: 9181fdc574fa481dd4ee4b8d4f3dfa82 -Description: debug symbols for mivisionx -Maintainer: MIVisionX Support -Package-Type: ddeb -Version: 3.0.0.60202-116~20.04 -Installed-Size: 47247 - -Package: mivisionx-dbgsym-rpath6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 5515c365a55d977b989a9a78b6dce53a6ad734c0 06e81f74b3808d260af14db37fd7152bde58a51c d52bc5ff99c8364cdd20192d9f879364de27abf6 46462528a11539a3fd1236aa08bc6caa60d00ffd 917c36ca6652adc23a8f3b26d8052f67daafacc3 96d74e856dad3f2f0ea83739d418c8fb769f6240 c9938d0502c294089fc0bfbeb9b36ff9db3fd6a0 4c19faee3747f8ed8a08da268c053cf75de86daf 3ac5215c62270e7c9fc861f6a94ec1a6f3bd4b56 94f0c8a5ec4764806d27a71ae9dbacbf47ab48cd -Depends: mivisionx-rpath6.2.2 (= 3.0.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/m/mivisionx-dbgsym-rpath6.2.2/mivisionx-dbgsym-rpath6.2.2_3.0.0.60202-116~20.04_amd64.deb -Size: 6984136 -SHA256: 3d9f3926cd56a84af18a581cd6ccbef1fc723cf9805c3f2317b34b3be075bfbf -SHA1: 1ebe6ab6513d54c13a3bac89729dbd1f67c353dc -MD5sum: c760623db882d03879771caeafeaf6ac -Description: debug symbols for mivisionx -Maintainer: MIVisionX Support -Package-Type: ddeb -Version: 3.0.0.60202-116~20.04 -Installed-Size: 47247 - -Package: mivisionx-dbgsym6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 5515c365a55d977b989a9a78b6dce53a6ad734c0 06e81f74b3808d260af14db37fd7152bde58a51c d52bc5ff99c8364cdd20192d9f879364de27abf6 46462528a11539a3fd1236aa08bc6caa60d00ffd 917c36ca6652adc23a8f3b26d8052f67daafacc3 96d74e856dad3f2f0ea83739d418c8fb769f6240 c9938d0502c294089fc0bfbeb9b36ff9db3fd6a0 4c19faee3747f8ed8a08da268c053cf75de86daf 3ac5215c62270e7c9fc861f6a94ec1a6f3bd4b56 94f0c8a5ec4764806d27a71ae9dbacbf47ab48cd -Depends: mivisionx6.2.2 (= 3.0.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/m/mivisionx-dbgsym6.2.2/mivisionx-dbgsym6.2.2_3.0.0.60202-116~20.04_amd64.deb -Size: 6985756 -SHA256: ee848be3448dedc325d0c2afe4ca125a1c7a6b2c7b85281b97dbe07b0d162c3b -SHA1: c6dd32d04cb14157c3861b995aa9e7b065e188a2 -MD5sum: 7c0e0dc1c579c593cf75fb935a83fe27 -Description: debug symbols for mivisionx -Maintainer: MIVisionX Support -Package-Type: ddeb -Version: 3.0.0.60202-116~20.04 -Installed-Size: 47247 - -Package: mivisionx-dev -Architecture: amd64 -Depends: rocm-core, mivisionx, rocm-hip-runtime-dev, rpp-dev, rocblas-dev, miopen-hip-dev, migraphx-dev, ffmpeg, libavcodec-dev, libavformat-dev, libavutil-dev, libswscale-dev, rocdecode-dev -Priority: optional -Section: devel -Filename: pool/main/m/mivisionx-dev/mivisionx-dev_3.0.0.60202-116~20.04_amd64.deb -Size: 23597340 -SHA256: a601a15d5d288899e91a2a7e35874e5e86ca5f36b8d0811b547e16717668861d -SHA1: 598117e79caef9d6bccc1c7e4fd1431e515b593d -MD5sum: d2deac5ac7d70c77b09a0f307aff7edc -Description: AMD MIVisionX is a comprehensive Computer Vision and ML Inference Toolkit. MIVisionX develop package provides MIVisionX libraries, header files, samples, and license.txt -Homepage: https://github.com/ROCm/MIVisionX -Maintainer: MIVisionX Support -Version: 3.0.0.60202-116~20.04 -Installed-Size: 41263 - -Package: mivisionx-dev-rpath6.2.2 -Architecture: amd64 -Depends: rocm-core-rpath6.2.2, mivisionx-rpath6.2.2, rocm-hip-runtime-dev-rpath6.2.2, rpp-dev-rpath6.2.2, rocblas-dev-rpath6.2.2, miopen-hip-dev-rpath6.2.2, migraphx-dev-rpath6.2.2, ffmpeg, libavcodec-dev, libavformat-dev, libavutil-dev, libswscale-dev, rocdecode-dev-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/m/mivisionx-dev-rpath6.2.2/mivisionx-dev-rpath6.2.2_3.0.0.60202-116~20.04_amd64.deb -Size: 19278376 -SHA256: 6aef5d1b87046e37757c27b4cde70a2a50607aeeaea1a269334c2d03f5a696b5 -SHA1: 6824ee6b272f5d4733c11e87437736c6848881d6 -MD5sum: 1a641ad9f2217ba8aa64608ad7eb241b -Description: AMD MIVisionX is a comprehensive Computer Vision and ML Inference Toolkit. MIVisionX develop package provides MIVisionX libraries, header files, samples, and license.txt -Homepage: https://github.com/ROCm/MIVisionX -Maintainer: MIVisionX Support -Version: 3.0.0.60202-116~20.04 -Installed-Size: 41263 - -Package: mivisionx-dev6.2.2 -Architecture: amd64 -Depends: rocm-core6.2.2, mivisionx6.2.2, rocm-hip-runtime-dev6.2.2, rpp-dev6.2.2, rocblas-dev6.2.2, miopen-hip-dev6.2.2, migraphx-dev6.2.2, ffmpeg, libavcodec-dev, libavformat-dev, libavutil-dev, libswscale-dev, rocdecode-dev6.2.2 -Priority: optional -Section: devel -Filename: pool/main/m/mivisionx-dev6.2.2/mivisionx-dev6.2.2_3.0.0.60202-116~20.04_amd64.deb -Size: 19278268 -SHA256: b8b8952556014429370a8933500fe33189c9d26d6b3ef1e32cf454e107e0379c -SHA1: e129c303bd055e8bfbbd537f6ecd3dc7c89be5f5 -MD5sum: 5eb9ed8dfbf817f3f4ad9f9053c766eb -Description: AMD MIVisionX is a comprehensive Computer Vision and ML Inference Toolkit. MIVisionX develop package provides MIVisionX libraries, header files, samples, and license.txt -Homepage: https://github.com/ROCm/MIVisionX -Maintainer: MIVisionX Support -Version: 3.0.0.60202-116~20.04 -Installed-Size: 41263 - -Package: mivisionx-rpath6.2.2 -Architecture: amd64 -Depends: rocm-core-rpath6.2.2, rocm-hip-runtime-rpath6.2.2, half-rpath6.2.2, rpp-rpath6.2.2, rocblas-rpath6.2.2, miopen-hip-rpath6.2.2, migraphx-rpath6.2.2, rocdecode-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/m/mivisionx-rpath6.2.2/mivisionx-rpath6.2.2_3.0.0.60202-116~20.04_amd64.deb -Size: 11081232 -SHA256: faf7ce875482df440121274ab7ecea1de4c0b31b3fd9c1e0ec9d7a4869bbea62 -SHA1: 094cedbc17f90954e79e596e2abcf770d72dfe5c -MD5sum: 2ef7ad5fefe83bfabc1c59c6915b428e -Description: AMD MIVisionX is a comprehensive Computer Vision and ML Inference Toolkit. MIVisionX runtime package provides MIVisionX libraries and license.txt -Homepage: https://github.com/ROCm/MIVisionX -Maintainer: MIVisionX Support -Version: 3.0.0.60202-116~20.04 -Installed-Size: 266541 - -Package: mivisionx-test -Architecture: amd64 -Depends: rocm-core, mivisionx-dev -Priority: optional -Section: devel -Filename: pool/main/m/mivisionx-test/mivisionx-test_3.0.0.60202-116~20.04_amd64.deb -Size: 10595674 -SHA256: 80138228f1b56152291153cb7848f5f549b44682b8d05a6d759dc3577d0fae20 -SHA1: 25e1f9e5cea0bebc12d114853377e577e368ee53 -MD5sum: 797ec13dfbc55858c64bec3f33c64ef9 -Description: AMD MIVisionX is a comprehensive Computer Vision and ML Inference Toolkit. MIVisionX Test package provides MIVisionX Test Components -Homepage: https://github.com/ROCm/MIVisionX -Maintainer: MIVisionX Support -Version: 3.0.0.60202-116~20.04 -Installed-Size: 18030 - -Package: mivisionx-test-rpath6.2.2 -Architecture: amd64 -Depends: rocm-core-rpath6.2.2, mivisionx-dev-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/m/mivisionx-test-rpath6.2.2/mivisionx-test-rpath6.2.2_3.0.0.60202-116~20.04_amd64.deb -Size: 7458148 -SHA256: 02df3932d8d313571bc06db1ab6bdf117f95387a918016393efd9abbccb6ac41 -SHA1: a3aadf6c6539dfb7371efb25876a3926fe3e0b81 -MD5sum: d176390192a578615a31fc0c6ba9c3f3 -Description: AMD MIVisionX is a comprehensive Computer Vision and ML Inference Toolkit. MIVisionX Test package provides MIVisionX Test Components -Homepage: https://github.com/ROCm/MIVisionX -Maintainer: MIVisionX Support -Version: 3.0.0.60202-116~20.04 -Installed-Size: 18030 - -Package: mivisionx-test6.2.2 -Architecture: amd64 -Depends: rocm-core6.2.2, mivisionx-dev6.2.2 -Priority: optional -Section: devel -Filename: pool/main/m/mivisionx-test6.2.2/mivisionx-test6.2.2_3.0.0.60202-116~20.04_amd64.deb -Size: 7466700 -SHA256: 438e1f2c5c29e5663b106320ae711b602a29d9c4b8d004d9e64623481555316d -SHA1: 2b823c579b04f86f4fd60431ebcac6506eb4caed -MD5sum: 20f795edfc9bba3021aa2e2cb538f607 -Description: AMD MIVisionX is a comprehensive Computer Vision and ML Inference Toolkit. MIVisionX Test package provides MIVisionX Test Components -Homepage: https://github.com/ROCm/MIVisionX -Maintainer: MIVisionX Support -Version: 3.0.0.60202-116~20.04 -Installed-Size: 18030 - -Package: mivisionx6.2.2 -Architecture: amd64 -Depends: rocm-core6.2.2, rocm-hip-runtime6.2.2, half6.2.2, rpp6.2.2, rocblas6.2.2, miopen-hip6.2.2, migraphx6.2.2, rocdecode6.2.2 -Priority: optional -Section: devel -Filename: pool/main/m/mivisionx6.2.2/mivisionx6.2.2_3.0.0.60202-116~20.04_amd64.deb -Size: 11079536 -SHA256: 996a094b7bb1c976e2f084e8d843bb601f7ad3ccc9ad44146fd8d437ecd1e29b -SHA1: 94cde353b0f42e3dd14b02745325d88caddb02b1 -MD5sum: 9b1bd5f215176f9a30b134406adf8649 -Description: AMD MIVisionX is a comprehensive Computer Vision and ML Inference Toolkit. MIVisionX runtime package provides MIVisionX libraries and license.txt -Homepage: https://github.com/ROCm/MIVisionX -Maintainer: MIVisionX Support -Version: 3.0.0.60202-116~20.04 -Installed-Size: 266541 - -Package: omniperf -Architecture: amd64 -Depends: rocprofiler -Priority: optional -Section: devel -Filename: pool/main/o/omniperf/omniperf_2.0.1.60202-116~20.04_amd64.deb -Size: 2620068 -SHA256: f141dbb2778585d0494a6e50ad19e841e83347490b37a96fce0e49d2bd327c29 -SHA1: e7d20ada00a2d71cd48a27a6b124803001a6befd -MD5sum: be2df303f6f53c9d10bdb4e75e249701 -Description: Omniperf: tool for GPU performance profiling -Homepage: https://github.com/ROCm/omniperf -Maintainer: https://github.com/ROCm/omniperf -Version: 2.0.1.60202-116~20.04 -Installed-Size: 12687 - -Package: omniperf-rpath6.2.2 -Architecture: amd64 -Depends: rocprofiler-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/o/omniperf-rpath6.2.2/omniperf-rpath6.2.2_2.0.1.60202-116~20.04_amd64.deb -Size: 1663392 -SHA256: 1f4503006b2b2bf8f00b4a5e65cbe40f831a259c1593909aed26f35564e5de01 -SHA1: 83cfdc41a532006d142dfbb3340410af0788d1d3 -MD5sum: f5691a128227695584b4f6cb70465067 -Description: Omniperf: tool for GPU performance profiling -Homepage: https://github.com/ROCm/omniperf -Maintainer: https://github.com/ROCm/omniperf -Version: 2.0.1.60202-116~20.04 -Installed-Size: 12687 - -Package: omniperf6.2.2 -Architecture: amd64 -Depends: rocprofiler6.2.2 -Priority: optional -Section: devel -Filename: pool/main/o/omniperf6.2.2/omniperf6.2.2_2.0.1.60202-116~20.04_amd64.deb -Size: 1663936 -SHA256: c9b6e83edd7d1593e87895715fd617960779f0e6517453600a3a97c61dc6b9e0 -SHA1: edeb908da33138976e85ecf377b6837db263a5a0 -MD5sum: 67df5d99f1f959f81536d9963018ac28 -Description: Omniperf: tool for GPU performance profiling -Homepage: https://github.com/ROCm/omniperf -Maintainer: https://github.com/ROCm/omniperf -Version: 2.0.1.60202-116~20.04 -Installed-Size: 12687 - -Package: omnitrace -Architecture: amd64 -Depends: libgomp1, rocm-smi-lib (>= 6.0.0.60202), roctracer-dev (>= 1.0.0.60202), rocprofiler-dev (>= 1.0.0.60202), libbz2-1.0, libc6 (>= 2.30), libgcc-s1 (>= 3.0), libgomp1 (>= 6), liblzma5 (>= 5.1.1alpha+20120614), libstdc++6 (>= 9), zlib1g (>= 1:1.2.2.3) -Priority: optional -Section: devel -Filename: pool/main/o/omnitrace/omnitrace_1.11.2.60202-116~20.04_amd64.deb -Size: 40312816 -SHA256: da642d4f0d84ddef53b3348e3179416560b57120e6e5feeee42fabe90e042bab -SHA1: 98254d24f0ae699544bc5c7bb84f58302b888975 -MD5sum: d72f3317431f3b62d9cfb5e9f92c9a40 -Description: Runtime instrumentation and binary rewriting for Perfetto via Dyninst -Homepage: https://github.com/ROCm/omnitrace -Maintainer: jonathan.madsen@amd.com -Version: 1.11.2.60202-116~20.04 -Installed-Size: 247179 - -Package: omnitrace-dbgsym -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: eac6018245ae1ab6a190826381d54153e26a0333 90367c8b66d2890c93370198e15ea17fcbc8802e e46786ac82bde169a1a2439bb741ab3509d402c2 e90d37a85ef322a2bb830e15e802d5c995ad0583 c5659842a2fc6c0500be26b3644e4bca2c291676 2a14778353135340141f9f2bf2277891a60c7a34 b1c0ba4b1b1e282ba18eeea9da2b721bc25778b9 b4053a9e681f77f68a20e864048cd3608b8e0eb7 d4752efac240904543eff29cc9b7b1d401d491b6 b674b1f00bde8499ef4ad317f20049c1658a97c6 c0c0d879c4659580bffe1ac674da3092bc86a9f1 03f1f4487a07016d970b9fe73850158a5b8ff614 571c29a98604f0998baf065ff377e50e107b66e9 7e26e5020a2665496c4f0bf393c5e360b2896d52 10cbea4ee1a63a5f3ff0ebf60975f763833b7f42 71b301a0cd95aea5ebb650b8b6131f39b5fbcf64 5fe9076b6106191c259e65e6064cc18bc5bc2234 4eb469c832aca954b04fe0def8c4f46d001586aa 1d6ff80ce6a89fbf5419a8113f370101e4dacf8d 1f79009801f5925ff6c5794a4bab6d3eb760941c c657487b15b6913ec6d600e7d070ece8a6385f1c c19463bc8cf86ac9038edeac60845a303d6d5be6 36d6e98f567ef617872abcb5103f08f6c647da53 824bccace94f66f9fccdd3e7c7e7c4472405d36d dc5c3734dd59cb9f6e9894bfc6e7ec9fa3fe49bf 4a62d200925c4f02f1e1c99b52b8aa24e24f5346 c323d5999292b93e0e1645f661b7492195baf3c1 0e760371e572ed657abe47253fbaf53627e6ebed c174f94221af80cbee0b5ed9cbd764e944374a27 d622ba94fcb15809e803c73456449f7465e81309 c19463bc8cf86ac9038edeac60845a303d6d5be6 99ca4257dbd41ac17a17a81842c119a24228362f 7e534a2cd50b1b309d90b95b16f5068f0ee532aa c7a630225000b86ce3e18d41225640780c7c1f17 21b4811cfe3b68bda7b9c94b232e5682cbcc91bb 1e440a4c4b2d3bc4ea190648190506fefe3ca6a5 6bdffbb68823670516d1063da1010bb651ece1e9 e3a1a11e75eb616ec3125ad788ffce5ee040f307 de930c9567f15ab1896a3be37fc767592555bb78 e68255fb37dfb7044c3f8bfe232efd4a2be9aaf2 -Depends: omnitrace (= 1.11.2.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/o/omnitrace-dbgsym/omnitrace-dbgsym_1.11.2.60202-116~20.04_amd64.deb -Size: 818803122 -SHA256: b69c9065f8acc255e972e8ca536ecd9426a3d7f1772ffdc12e86260e03825bc3 -SHA1: 66d6042dbae87635087f93d8a13a97694f60d1d9 -MD5sum: 401370003a411130433ca59b0dc29afd -Description: debug symbols for omnitrace -Maintainer: jonathan.madsen@amd.com -Package-Type: ddeb -Version: 1.11.2.60202-116~20.04 -Installed-Size: 3083713 - -Package: omnitrace-dbgsym-rpath6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: eac6018245ae1ab6a190826381d54153e26a0333 90367c8b66d2890c93370198e15ea17fcbc8802e e46786ac82bde169a1a2439bb741ab3509d402c2 e90d37a85ef322a2bb830e15e802d5c995ad0583 c5659842a2fc6c0500be26b3644e4bca2c291676 2a14778353135340141f9f2bf2277891a60c7a34 b1c0ba4b1b1e282ba18eeea9da2b721bc25778b9 b4053a9e681f77f68a20e864048cd3608b8e0eb7 d4752efac240904543eff29cc9b7b1d401d491b6 b674b1f00bde8499ef4ad317f20049c1658a97c6 c0c0d879c4659580bffe1ac674da3092bc86a9f1 03f1f4487a07016d970b9fe73850158a5b8ff614 571c29a98604f0998baf065ff377e50e107b66e9 7e26e5020a2665496c4f0bf393c5e360b2896d52 10cbea4ee1a63a5f3ff0ebf60975f763833b7f42 71b301a0cd95aea5ebb650b8b6131f39b5fbcf64 5fe9076b6106191c259e65e6064cc18bc5bc2234 4eb469c832aca954b04fe0def8c4f46d001586aa 1d6ff80ce6a89fbf5419a8113f370101e4dacf8d 1f79009801f5925ff6c5794a4bab6d3eb760941c c657487b15b6913ec6d600e7d070ece8a6385f1c c19463bc8cf86ac9038edeac60845a303d6d5be6 36d6e98f567ef617872abcb5103f08f6c647da53 824bccace94f66f9fccdd3e7c7e7c4472405d36d dc5c3734dd59cb9f6e9894bfc6e7ec9fa3fe49bf 4a62d200925c4f02f1e1c99b52b8aa24e24f5346 c323d5999292b93e0e1645f661b7492195baf3c1 0e760371e572ed657abe47253fbaf53627e6ebed c174f94221af80cbee0b5ed9cbd764e944374a27 d622ba94fcb15809e803c73456449f7465e81309 c19463bc8cf86ac9038edeac60845a303d6d5be6 99ca4257dbd41ac17a17a81842c119a24228362f 7e534a2cd50b1b309d90b95b16f5068f0ee532aa c7a630225000b86ce3e18d41225640780c7c1f17 21b4811cfe3b68bda7b9c94b232e5682cbcc91bb 1e440a4c4b2d3bc4ea190648190506fefe3ca6a5 6bdffbb68823670516d1063da1010bb651ece1e9 e3a1a11e75eb616ec3125ad788ffce5ee040f307 de930c9567f15ab1896a3be37fc767592555bb78 e68255fb37dfb7044c3f8bfe232efd4a2be9aaf2 -Depends: omnitrace-rpath6.2.2 (= 1.11.2.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/o/omnitrace-dbgsym-rpath6.2.2/omnitrace-dbgsym-rpath6.2.2_1.11.2.60202-116~20.04_amd64.deb -Size: 514334544 -SHA256: 7e2f9840d33205e3f1dfa3b54c641cd8e24b2d2bbf479959bd6ccea2b7d28c42 -SHA1: 69bfac39ac0afd2199803815af0c7ef65590e931 -MD5sum: edc5c84b4df77b615deb4376fcc7d94f -Description: debug symbols for omnitrace -Maintainer: jonathan.madsen@amd.com -Package-Type: ddeb -Version: 1.11.2.60202-116~20.04 -Installed-Size: 3083713 - -Package: omnitrace-dbgsym6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: eac6018245ae1ab6a190826381d54153e26a0333 90367c8b66d2890c93370198e15ea17fcbc8802e e46786ac82bde169a1a2439bb741ab3509d402c2 e90d37a85ef322a2bb830e15e802d5c995ad0583 c5659842a2fc6c0500be26b3644e4bca2c291676 2a14778353135340141f9f2bf2277891a60c7a34 b1c0ba4b1b1e282ba18eeea9da2b721bc25778b9 b4053a9e681f77f68a20e864048cd3608b8e0eb7 d4752efac240904543eff29cc9b7b1d401d491b6 b674b1f00bde8499ef4ad317f20049c1658a97c6 c0c0d879c4659580bffe1ac674da3092bc86a9f1 03f1f4487a07016d970b9fe73850158a5b8ff614 571c29a98604f0998baf065ff377e50e107b66e9 7e26e5020a2665496c4f0bf393c5e360b2896d52 10cbea4ee1a63a5f3ff0ebf60975f763833b7f42 71b301a0cd95aea5ebb650b8b6131f39b5fbcf64 5fe9076b6106191c259e65e6064cc18bc5bc2234 4eb469c832aca954b04fe0def8c4f46d001586aa 1d6ff80ce6a89fbf5419a8113f370101e4dacf8d 1f79009801f5925ff6c5794a4bab6d3eb760941c c657487b15b6913ec6d600e7d070ece8a6385f1c c19463bc8cf86ac9038edeac60845a303d6d5be6 36d6e98f567ef617872abcb5103f08f6c647da53 824bccace94f66f9fccdd3e7c7e7c4472405d36d dc5c3734dd59cb9f6e9894bfc6e7ec9fa3fe49bf 4a62d200925c4f02f1e1c99b52b8aa24e24f5346 c323d5999292b93e0e1645f661b7492195baf3c1 0e760371e572ed657abe47253fbaf53627e6ebed c174f94221af80cbee0b5ed9cbd764e944374a27 d622ba94fcb15809e803c73456449f7465e81309 c19463bc8cf86ac9038edeac60845a303d6d5be6 99ca4257dbd41ac17a17a81842c119a24228362f 7e534a2cd50b1b309d90b95b16f5068f0ee532aa c7a630225000b86ce3e18d41225640780c7c1f17 21b4811cfe3b68bda7b9c94b232e5682cbcc91bb 1e440a4c4b2d3bc4ea190648190506fefe3ca6a5 6bdffbb68823670516d1063da1010bb651ece1e9 e3a1a11e75eb616ec3125ad788ffce5ee040f307 de930c9567f15ab1896a3be37fc767592555bb78 e68255fb37dfb7044c3f8bfe232efd4a2be9aaf2 -Depends: omnitrace6.2.2 (= 1.11.2.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/o/omnitrace-dbgsym6.2.2/omnitrace-dbgsym6.2.2_1.11.2.60202-116~20.04_amd64.deb -Size: 514333492 -SHA256: 426dd469e6ef27d8e35392826091c32794bedda53d4c6b3c308e52464b52c5ba -SHA1: 8875a10dbd6f48abbc13898f3921f8fec9601745 -MD5sum: ff695050fef6bd87904b9a507ebf1766 -Description: debug symbols for omnitrace -Maintainer: jonathan.madsen@amd.com -Package-Type: ddeb -Version: 1.11.2.60202-116~20.04 -Installed-Size: 3083713 - -Package: omnitrace-rpath6.2.2 -Architecture: amd64 -Depends: libgomp1, rocm-smi-lib-rpath6.2.2 (>= 6.0.0.60202), roctracer-dev-rpath6.2.2 (>= 1.0.0.60202), rocprofiler-dev-rpath6.2.2 (>= 1.0.0.60202), libbz2-1.0, libc6 (>= 2.30), libgcc-s1 (>= 3.0), libgomp1 (>= 6), liblzma5 (>= 5.1.1alpha+20120614), libstdc++6 (>= 9), zlib1g (>= 1:1.2.2.3) -Priority: optional -Section: devel -Filename: pool/main/o/omnitrace-rpath6.2.2/omnitrace-rpath6.2.2_1.11.2.60202-116~20.04_amd64.deb -Size: 22429944 -SHA256: 6df51f04f75c85717e1a0f1adf9d6f30b73013ca0fbb245c4a7b7916b32096dd -SHA1: 00bef216db9cafe704222c889e092e91161dc6f1 -MD5sum: 573116c97d9aac3217f2e5d9d27c9c75 -Description: Runtime instrumentation and binary rewriting for Perfetto via Dyninst -Homepage: https://github.com/ROCm/omnitrace -Maintainer: jonathan.madsen@amd.com -Version: 1.11.2.60202-116~20.04 -Installed-Size: 247179 - -Package: omnitrace6.2.2 -Architecture: amd64 -Depends: libgomp1, rocm-smi-lib6.2.2 (>= 6.0.0.60202), roctracer-dev6.2.2 (>= 1.0.0.60202), rocprofiler-dev6.2.2 (>= 1.0.0.60202), libbz2-1.0, libc6 (>= 2.30), libgcc-s1 (>= 3.0), libgomp1 (>= 6), liblzma5 (>= 5.1.1alpha+20120614), libstdc++6 (>= 9), zlib1g (>= 1:1.2.2.3) -Priority: optional -Section: devel -Filename: pool/main/o/omnitrace6.2.2/omnitrace6.2.2_1.11.2.60202-116~20.04_amd64.deb -Size: 22436056 -SHA256: 641e2d1a2aeafd76f921645fc1d78801eb13395261163937e5664b3c7b8c2d01 -SHA1: 8dc82beef27f06c6bdbc0c834ba5bfccf020ec7d -MD5sum: 960bb52612b9bfa4aea58d5351b5dcda -Description: Runtime instrumentation and binary rewriting for Perfetto via Dyninst -Homepage: https://github.com/ROCm/omnitrace -Maintainer: jonathan.madsen@amd.com -Version: 1.11.2.60202-116~20.04 -Installed-Size: 247179 - -Package: openmp-extras-asan -Architecture: amd64 -Maintainer: Openmp Extras Support -Version: 18.62.0.60202-116~20.04 -Depends: hsa-rocr-asan, rocm-core-asan -Recommends: gcc, g++ -Priority: optional -Section: devel -Filename: pool/main/o/openmp-extras-asan/openmp-extras-asan_18.62.0.60202-116~20.04_amd64.deb -Size: 188725204 -SHA256: 6e7e3bd64a6a257456b80e7f4575df6caff2ed87508126764ccfb0b036bab578 -SHA1: 9515cb6ee6783f466b9114ad4d7d982850f9e841 -MD5sum: b565f11390abbf2cc40ab1f3a56e0bfe -Description: AddressSanitizer OpenMP Extras provides instrumented openmp and flang libraries. - openmp-extras 18.62.0.60202 is based on LLVM 17 and is used for offloading to Radeon GPUs. - -Package: openmp-extras-asan-rpath6.2.2 -Architecture: amd64 -Maintainer: Openmp Extras Support -Version: 18.62.0.60202-116~20.04 -Depends: hsa-rocr-asan-rpath6.2.2, rocm-core-asan-rpath6.2.2 -Recommends: gcc, g++ -Priority: optional -Section: devel -Filename: pool/main/o/openmp-extras-asan-rpath6.2.2/openmp-extras-asan-rpath6.2.2_18.62.0.60202-116~20.04_amd64.deb -Size: 114726764 -SHA256: d380e732a91dfea3f2b0250e92e7f25a86c5f61b3016cd979739748363f628d9 -SHA1: 60cb7a10ddd51bed939be6d6f7137f778ffed7f6 -MD5sum: 2675e7d1bf7348c637959fd1370b8424 -Description: AddressSanitizer OpenMP Extras provides instrumented openmp and flang libraries. - openmp-extras 18.62.0.60202 is based on LLVM 17 and is used for offloading to Radeon GPUs. - -Package: openmp-extras-asan6.2.2 -Architecture: amd64 -Maintainer: Openmp Extras Support -Version: 18.62.0.60202-116~20.04 -Depends: hsa-rocr-asan6.2.2, rocm-core-asan6.2.2 -Recommends: gcc, g++ -Priority: optional -Section: devel -Filename: pool/main/o/openmp-extras-asan6.2.2/openmp-extras-asan6.2.2_18.62.0.60202-116~20.04_amd64.deb -Size: 114728836 -SHA256: e964bc93293746c83809a41e5168a40a64ea0e584772b5e3e0244944523e25be -SHA1: 7d461b03d6f30e5951b5bcd18221a088e0e94fcb -MD5sum: c5e985269c6a44130308f3093e661254 -Description: AddressSanitizer OpenMP Extras provides instrumented openmp and flang libraries. - openmp-extras 18.62.0.60202 is based on LLVM 17 and is used for offloading to Radeon GPUs. - -Package: openmp-extras-dev -Architecture: amd64 -Maintainer: Openmp Extras Support -Version: 18.62.0.60202-116~20.04 -Depends: rocm-llvm, rocm-device-libs, rocm-core, openmp-extras-runtime, hsa-rocr-dev -Recommends: gcc, g++ -Provides: openmp-extras -Conflicts: openmp-extras -Replaces: openmp-extras -Priority: optional -Section: devel -Filename: pool/main/o/openmp-extras-dev/openmp-extras-dev_18.62.0.60202-116~20.04_amd64.deb -Size: 49046206 -SHA256: 40e0dbcacab017720dcf93f0df31c2bb11534566aea0d99b2cee51a6829529a1 -SHA1: f86de86f1fdf239385a13b5e718018a6183ebc8c -MD5sum: 00a5bed991ad975bb11953c50c57d78d -Description: OpenMP Extras provides openmp and flang libraries. - openmp-extras 18.62.0.60202 is based on LLVM 17 and is used for offloading to Radeon GPUs. - -Package: openmp-extras-dev-rpath6.2.2 -Architecture: amd64 -Maintainer: Openmp Extras Support -Version: 18.62.0.60202-116~20.04 -Depends: rocm-llvm-rpath6.2.2, rocm-device-libs-rpath6.2.2, rocm-core-rpath6.2.2, openmp-extras-runtime-rpath6.2.2, hsa-rocr-dev-rpath6.2.2 -Recommends: gcc, g++ -Provides: openmp-extras-rpath6.2.2 -Conflicts: openmp-extras-rpath6.2.2 -Replaces: openmp-extras-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/o/openmp-extras-dev-rpath6.2.2/openmp-extras-dev-rpath6.2.2_18.62.0.60202-116~20.04_amd64.deb -Size: 25650896 -SHA256: befa76da6ca8a68358e0191d588b3a51fed39fcb924c22ff37fa266c02595c0d -SHA1: d0945869e0a5be318fec5a138fcf0b97bb7b886a -MD5sum: 878c31295cc4e99dcb26ee7f1aae4777 -Description: OpenMP Extras provides openmp and flang libraries. - openmp-extras 18.62.0.60202 is based on LLVM 17 and is used for offloading to Radeon GPUs. - -Package: openmp-extras-dev6.2.2 -Architecture: amd64 -Maintainer: Openmp Extras Support -Version: 18.62.0.60202-116~20.04 -Depends: rocm-llvm6.2.2, rocm-device-libs6.2.2, rocm-core6.2.2, openmp-extras-runtime6.2.2, hsa-rocr-dev6.2.2 -Recommends: gcc, g++ -Provides: openmp-extras6.2.2 -Conflicts: openmp-extras6.2.2 -Replaces: openmp-extras6.2.2 -Priority: optional -Section: devel -Filename: pool/main/o/openmp-extras-dev6.2.2/openmp-extras-dev6.2.2_18.62.0.60202-116~20.04_amd64.deb -Size: 25650248 -SHA256: abf9efb248dbde93c9949e635e8052cc5c82734381dd64a44f3eee0ec9f6c259 -SHA1: fdfe0e4fed8c7b71c6495d04d511546029fc076f -MD5sum: 31e353c61709b204531f857bf2afa404 -Description: OpenMP Extras provides openmp and flang libraries. - openmp-extras 18.62.0.60202 is based on LLVM 17 and is used for offloading to Radeon GPUs. - -Package: openmp-extras-runtime -Architecture: amd64 -Maintainer: Openmp Extras Support -Version: 18.62.0.60202-116~20.04 -Depends: rocm-core, hsa-rocr -Recommends: gcc, g++ -Priority: optional -Section: devel -Filename: pool/main/o/openmp-extras-runtime/openmp-extras-runtime_18.62.0.60202-116~20.04_amd64.deb -Size: 150997604 -SHA256: 0a09d0722f7ab193d6cf181c30ee8b2eaaae798142bfdc10ce88d5e0e2f52adb -SHA1: 9d42b9a51a15f55f32a21fddda81e624034e650c -MD5sum: 160f20dd27450ab08c0c9d3751069701 -Description: OpenMP Extras provides openmp and flang libraries. - openmp-extras 18.62.0.60202 is based on LLVM 17 and is used for offloading to Radeon GPUs. - -Package: openmp-extras-runtime-rpath6.2.2 -Architecture: amd64 -Maintainer: Openmp Extras Support -Version: 18.62.0.60202-116~20.04 -Depends: rocm-core-rpath6.2.2, hsa-rocr-rpath6.2.2 -Recommends: gcc, g++ -Priority: optional -Section: devel -Filename: pool/main/o/openmp-extras-runtime-rpath6.2.2/openmp-extras-runtime-rpath6.2.2_18.62.0.60202-116~20.04_amd64.deb -Size: 98105056 -SHA256: c9af80c29400dbb945a529428d077b3ffd0d87370574515623689a712193afe7 -SHA1: a2bc447966e328aafb745bd327f7c05be761e1a7 -MD5sum: 7158e4f665c4c22ddf691f360f7bcd3f -Description: OpenMP Extras provides openmp and flang libraries. - openmp-extras 18.62.0.60202 is based on LLVM 17 and is used for offloading to Radeon GPUs. - -Package: openmp-extras-runtime6.2.2 -Architecture: amd64 -Maintainer: Openmp Extras Support -Version: 18.62.0.60202-116~20.04 -Depends: rocm-core6.2.2, hsa-rocr6.2.2 -Recommends: gcc, g++ -Priority: optional -Section: devel -Filename: pool/main/o/openmp-extras-runtime6.2.2/openmp-extras-runtime6.2.2_18.62.0.60202-116~20.04_amd64.deb -Size: 98102096 -SHA256: baefdfef254e8e97b2f6a3c25c79292f8240d33e73eb094a81c565d465108a62 -SHA1: 39a9d63c3b77440e629e4bc8012d8be2b2e31410 -MD5sum: 65d6ef5909a4a1eb39f422e97b16aee9 -Description: OpenMP Extras provides openmp and flang libraries. - openmp-extras 18.62.0.60202 is based on LLVM 17 and is used for offloading to Radeon GPUs. - -Package: rccl -Architecture: amd64 -Depends: hip-runtime-amd (>= 4.5.0), rocm-smi-lib (>= 4.0.0), rocm-core, libc6 (>= 2.17), libgcc-s1 (>= 3.0), libstdc++6 (>= 9) -Priority: optional -Section: devel -Filename: pool/main/r/rccl/rccl_2.20.5.60202-116~20.04_amd64.deb -Size: 62974104 -SHA256: f18137e0b5eb771a199e66873748a25baa9c8e12c64dd292b586b3e682cf0899 -SHA1: a248c2239d85fbd8da33b4eb6bd374129daee87a -MD5sum: 33ae8846038601cbe17fd68c2deb8c04 -Description: ROCm Communication Collectives Library - ROCm Communication Collectives Library - Optimized primitives for collective multi-GPU communication -Maintainer: RCCL Maintainer -Recommends: rccl-dev (>=2.20.5.60202) -Version: 2.20.5.60202-116~20.04 -Installed-Size: 1895293 - -Package: rccl-asan -Architecture: amd64 -Depends: hip-runtime-amd-asan (>= 4.5.0), rocm-smi-lib (>= 4.0.0), rocm-core-asan, libc6 (>= 2.10), libgcc-s1 (>= 3.0), libstdc++6 (>= 9) -Priority: optional -Section: devel -Filename: pool/main/r/rccl-asan/rccl-asan_2.20.5.60202-116~20.04_amd64.deb -Size: 122407786 -SHA256: e38a6bf8d37b92e5d5f29d8df1b6ba843aac1b20b20b40170afc43c1c37edabe -SHA1: cb44b7565966e3f0b280bfa742f0476a0634c76c -MD5sum: 43ae0a64244ff52d9f85dfd5704a559b -Description: ROCm Communication Collectives Library - ROCm Communication Collectives Library - Optimized primitives for collective multi-GPU communication -Maintainer: RCCL Maintainer -Recommends: rccl-asan-dev (>=2.20.5.60202) -Version: 2.20.5.60202-116~20.04 -Installed-Size: 1124607 - -Package: rccl-asan-dbgsym -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 40bba56f353372f3488a1656384691f0edbb645b -Depends: rccl-asan (= 2.20.5.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rccl-asan-dbgsym/rccl-asan-dbgsym_2.20.5.60202-116~20.04_amd64.deb -Size: 2345962 -SHA256: e042c8aab01c4b89f56f443482ad9e58615a28499bf8e4323d6917b4dc206975 -SHA1: 86bd324ade9fb5755f6e2b1f985fb65b46799f5d -MD5sum: 98baded1618b3f35e9473b99ff377887 -Description: debug symbols for rccl-asan -Maintainer: RCCL Maintainer -Package-Type: ddeb -Version: 2.20.5.60202-116~20.04 -Installed-Size: 2758 - -Package: rccl-asan-dbgsym-rpath6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 40bba56f353372f3488a1656384691f0edbb645b -Depends: rccl-asan-rpath6.2.2 (= 2.20.5.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rccl-asan-dbgsym-rpath6.2.2/rccl-asan-dbgsym-rpath6.2.2_2.20.5.60202-116~20.04_amd64.deb -Size: 2346176 -SHA256: 8aaeacef465211185d61888fe009475d13ea684de707567b43b3a9e8b1f7ad0a -SHA1: 2b01c53128b177388bf168d40dbd3d4470083a71 -MD5sum: a596d4fe6f6f5e1dd0f1cd36c90fb643 -Description: debug symbols for rccl-asan -Maintainer: RCCL Maintainer -Package-Type: ddeb -Version: 2.20.5.60202-116~20.04 -Installed-Size: 2758 - -Package: rccl-asan-dbgsym6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 40bba56f353372f3488a1656384691f0edbb645b -Depends: rccl-asan6.2.2 (= 2.20.5.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rccl-asan-dbgsym6.2.2/rccl-asan-dbgsym6.2.2_2.20.5.60202-116~20.04_amd64.deb -Size: 2346152 -SHA256: 5eb461adca674e54cbcd8baf9e3d463165a06d48c60050ca3d1fc0916104d709 -SHA1: 64029049a71db1e8e7f102b95dd338f85d940ab8 -MD5sum: d3052197f319b632df547a493fd10474 -Description: debug symbols for rccl-asan -Maintainer: RCCL Maintainer -Package-Type: ddeb -Version: 2.20.5.60202-116~20.04 -Installed-Size: 2758 - -Package: rccl-asan-rpath6.2.2 -Architecture: amd64 -Depends: hip-runtime-amd-asan-rpath6.2.2 (>= 4.5.0), rocm-smi-lib (>= 4.0.0), rocm-core-asan-rpath6.2.2, libc6 (>= 2.10), libgcc-s1 (>= 3.0), libstdc++6 (>= 9) -Priority: optional -Section: devel -Filename: pool/main/r/rccl-asan-rpath6.2.2/rccl-asan-rpath6.2.2_2.20.5.60202-116~20.04_amd64.deb -Size: 122407516 -SHA256: d776dcdd1d0b7cc2bb71bb117471f4fd2db62e784220ea7d368b4e05b3e5234d -SHA1: d1522306c57b3f44bbca4e2329450ded99e76f3c -MD5sum: 89e9f6fe88a3ec63d9296c826d80e213 -Description: ROCm Communication Collectives Library - ROCm Communication Collectives Library - Optimized primitives for collective multi-GPU communication -Maintainer: RCCL Maintainer -Recommends: rccl-asan-dev (>=2.20.5.60202) -Version: 2.20.5.60202-116~20.04 -Installed-Size: 1124607 - -Package: rccl-asan6.2.2 -Architecture: amd64 -Depends: hip-runtime-amd-asan6.2.2 (>= 4.5.0), rocm-smi-lib (>= 4.0.0), rocm-core-asan6.2.2, libc6 (>= 2.10), libgcc-s1 (>= 3.0), libstdc++6 (>= 9) -Priority: optional -Section: devel -Filename: pool/main/r/rccl-asan6.2.2/rccl-asan6.2.2_2.20.5.60202-116~20.04_amd64.deb -Size: 122407664 -SHA256: 5fa8d3d3d3eb1cc3b94dda78dc43e9391eead2eaa7c8c533350247664d0c6c4f -SHA1: c2cad26feb45e09d544666ff85278b0e052b4f3e -MD5sum: 72e1bedf70828e66de62f63bb47163dc -Description: ROCm Communication Collectives Library - ROCm Communication Collectives Library - Optimized primitives for collective multi-GPU communication -Maintainer: RCCL Maintainer -Recommends: rccl-asan-dev (>=2.20.5.60202) -Version: 2.20.5.60202-116~20.04 -Installed-Size: 1124607 - -Package: rccl-dbgsym -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 3959a90132c3a2abb08ab8e0c10129bb6df7a8c6 -Depends: rccl (= 2.20.5.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rccl-dbgsym/rccl-dbgsym_2.20.5.60202-116~20.04_amd64.deb -Size: 1250920 -SHA256: f38dbfea41d27a7a2639de72670bdebfb8ca59c710e6a2fade95699b05979b9e -SHA1: 2326a60909e727127f5f333cedd9e37d17372fa5 -MD5sum: 5908791b26480228803f12371e603949 -Description: debug symbols for rccl -Maintainer: RCCL Maintainer -Package-Type: ddeb -Version: 2.20.5.60202-116~20.04 -Installed-Size: 6346 - -Package: rccl-dbgsym-rpath6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 3959a90132c3a2abb08ab8e0c10129bb6df7a8c6 -Depends: rccl-rpath6.2.2 (= 2.20.5.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rccl-dbgsym-rpath6.2.2/rccl-dbgsym-rpath6.2.2_2.20.5.60202-116~20.04_amd64.deb -Size: 1251320 -SHA256: 7bcfddbe68c39658080d8dd85cba971e20fa5f0cce2d8f1772e177c413d3266f -SHA1: 1685001eb40bc2a486f1c8a4f8e7fd47e86796e1 -MD5sum: 1f89c4429116cb51ba2cc8f9b200693b -Description: debug symbols for rccl -Maintainer: RCCL Maintainer -Package-Type: ddeb -Version: 2.20.5.60202-116~20.04 -Installed-Size: 6346 - -Package: rccl-dbgsym6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 3959a90132c3a2abb08ab8e0c10129bb6df7a8c6 -Depends: rccl6.2.2 (= 2.20.5.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rccl-dbgsym6.2.2/rccl-dbgsym6.2.2_2.20.5.60202-116~20.04_amd64.deb -Size: 1251468 -SHA256: 7f6ece55d66e8e19fdf6a663aae7ad6828ab304c7bbfc8d570d96f847dd2f797 -SHA1: 0ce9b2ee176c26105fd0cbddf946f66a81c125d0 -MD5sum: bced38f7b736f473a7bb780052f5dc64 -Description: debug symbols for rccl -Maintainer: RCCL Maintainer -Package-Type: ddeb -Version: 2.20.5.60202-116~20.04 -Installed-Size: 6346 - -Package: rccl-dev -Architecture: amd64 -Depends: rccl (>= 2.20.5.60202) -Priority: optional -Section: devel -Filename: pool/main/r/rccl-dev/rccl-dev_2.20.5.60202-116~20.04_amd64.deb -Size: 160254 -SHA256: ea92d0cc461015c83657f01558f3639127b60fde546d1b56472f846af1d4bd47 -SHA1: 4152cbafa452af38a450689bc497fa9d696d1f78 -MD5sum: 07a26ca91e47fec18eee05c1b1f5f609 -Description: ROCm Communication Collectives Library - ROCm Communication Collectives Library - Optimized primitives for collective multi-GPU communication -Maintainer: RCCL Maintainer -Version: 2.20.5.60202-116~20.04 -Installed-Size: 17430 - -Package: rccl-dev-rpath6.2.2 -Architecture: amd64 -Depends: rccl-rpath6.2.2 (>= 2.20.5.60202) -Priority: optional -Section: devel -Filename: pool/main/r/rccl-dev-rpath6.2.2/rccl-dev-rpath6.2.2_2.20.5.60202-116~20.04_amd64.deb -Size: 160248 -SHA256: 1e536ccea06c1c8049e01ea7108e656971065e0b0ecdfb1f028779ba910953bb -SHA1: 4ce5826eaad64e1d8c1d6fe85e7078e00417c0de -MD5sum: 65997557c3815a50e301d9933eaeb4ba -Description: ROCm Communication Collectives Library - ROCm Communication Collectives Library - Optimized primitives for collective multi-GPU communication -Maintainer: RCCL Maintainer -Version: 2.20.5.60202-116~20.04 -Installed-Size: 17430 - -Package: rccl-dev6.2.2 -Architecture: amd64 -Depends: rccl6.2.2 (>= 2.20.5.60202) -Priority: optional -Section: devel -Filename: pool/main/r/rccl-dev6.2.2/rccl-dev6.2.2_2.20.5.60202-116~20.04_amd64.deb -Size: 160200 -SHA256: 909087b53bfebc7ee844d4b573d03e6a9b36061cd2565e0edd8509b60db35c1b -SHA1: 8b738d5f73ff586dbb4c89cc1a03b5d1f9dddb07 -MD5sum: 4b7fbbc21fb31afff1c1bf14ee3f22cc -Description: ROCm Communication Collectives Library - ROCm Communication Collectives Library - Optimized primitives for collective multi-GPU communication -Maintainer: RCCL Maintainer -Version: 2.20.5.60202-116~20.04 -Installed-Size: 17430 - -Package: rccl-rdma-sharp-plugins -Version: 1.0.60202-116~20.04 -Architecture: amd64 -Maintainer: rccl-maintainer@amd.com -Installed-Size: 334 -Depends: libc6 (>= 2.14), libibverbs1 (>= 28), sharp, ucx -Homepage: http://www.amd.com -Priority: extra -Section: libs -Filename: pool/main/r/rccl-rdma-sharp-plugins/rccl-rdma-sharp-plugins_1.0.60202-116~20.04_amd64.deb -Size: 111336 -SHA256: 72d8dc7f606023f27236a8bee22bb3ce104ebbda358652dece938b2cb9026f8d -SHA1: 3a230b10ba9414a0eb8afcad1d925cd451bbdd76 -MD5sum: dfb0fa82077abbcb45ea76f9548840cc -Description: RDMA and SHARP plugin for NCCL - Plugin enabled RDMA and switch collectives(SHARP) in RCCL - -Package: rccl-rdma-sharp-plugins-rpath6.2.2 -Version: 1.0.60202-116~20.04 -Architecture: amd64 -Maintainer: rccl-maintainer@amd.com -Installed-Size: 334 -Depends: libc6 (>= 2.14), libibverbs1 (>= 28), sharp, ucx -Homepage: http://www.amd.com -Priority: extra -Section: libs -Filename: pool/main/r/rccl-rdma-sharp-plugins-rpath6.2.2/rccl-rdma-sharp-plugins-rpath6.2.2_1.0.60202-116~20.04_amd64.deb -Size: 69268 -SHA256: ab15e21746922cce830b571b33219bde150c67ade5837c12341113adfb342419 -SHA1: 6ce3fbe22547552ce3eefa666ee998976b8919e0 -MD5sum: 8c89bcb7df03cc704ed43aa9dc79761e -Description: RDMA and SHARP plugin for NCCL - Plugin enabled RDMA and switch collectives(SHARP) in RCCL - -Package: rccl-rdma-sharp-plugins6.2.2 -Version: 1.0.60202-116~20.04 -Architecture: amd64 -Maintainer: rccl-maintainer@amd.com -Installed-Size: 334 -Depends: libc6 (>= 2.14), libibverbs1 (>= 28), sharp, ucx -Homepage: http://www.amd.com -Priority: extra -Section: libs -Filename: pool/main/r/rccl-rdma-sharp-plugins6.2.2/rccl-rdma-sharp-plugins6.2.2_1.0.60202-116~20.04_amd64.deb -Size: 69104 -SHA256: f0410d6699bfcf8f2feba53fa89c9a93136323cb63853ffccdff885b554fbf57 -SHA1: bca493d9d446e3d17b37b72bf337c10f30eae330 -MD5sum: a2c8cf570f171d974eccc1b936e91f4d -Description: RDMA and SHARP plugin for NCCL - Plugin enabled RDMA and switch collectives(SHARP) in RCCL - -Package: rccl-rpath6.2.2 -Architecture: amd64 -Depends: hip-runtime-amd-rpath6.2.2 (>= 4.5.0), rocm-smi-lib-rpath6.2.2 (>= 4.0.0), rocm-core-rpath6.2.2, libc6 (>= 2.17), libgcc-s1 (>= 3.0), libstdc++6 (>= 9) -Priority: optional -Section: devel -Filename: pool/main/r/rccl-rpath6.2.2/rccl-rpath6.2.2_2.20.5.60202-116~20.04_amd64.deb -Size: 62974136 -SHA256: 9b42c26cac56ef0303dad89cbf766834d0bbb5301596a210a205180084cc3681 -SHA1: 2d606e9bdf4c5b79ce4daa76202af54b55d91c34 -MD5sum: f00384e4f3ce94ea5aa0e2cc2d44699b -Description: ROCm Communication Collectives Library - ROCm Communication Collectives Library - Optimized primitives for collective multi-GPU communication -Maintainer: RCCL Maintainer -Recommends: rccl-dev-rpath6.2.2 (>=2.20.5.60202) -Version: 2.20.5.60202-116~20.04 -Installed-Size: 1895293 - -Package: rccl-unittests -Architecture: amd64 -Depends: rccl (>= 2.20.5.60202), libc6 (>= 2.14), libgcc-s1 (>= 3.0), libstdc++6 (>= 9) -Priority: optional -Section: devel -Filename: pool/main/r/rccl-unittests/rccl-unittests_2.20.5.60202-116~20.04_amd64.deb -Size: 354106 -SHA256: 50a452e52525a022c809472f87abde88b61b399c04eaf62129c7792d9ef9bf72 -SHA1: b84fe605dcd8252dbc24dd109f63b552abfc2091 -MD5sum: 7b1424223354a0ca70f73cb98a163362 -Description: ROCm Communication Collectives Library - ROCm Communication Collectives Library - Optimized primitives for collective multi-GPU communication -Maintainer: RCCL Maintainer -Version: 2.20.5.60202-116~20.04 -Installed-Size: 1619 - -Package: rccl-unittests-dbgsym -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 171c4b54ebac1e09e26d86c29bed9a52f2ae2b86 -Depends: rccl-unittests (= 2.20.5.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rccl-unittests-dbgsym/rccl-unittests-dbgsym_2.20.5.60202-116~20.04_amd64.deb -Size: 856524 -SHA256: 70be890691aa7079237c4ea648686e2140759cd0758737774711bbb33045b7bd -SHA1: a65320bfe130bcca169b01b768aab88a76b5cd5b -MD5sum: 2ad07554eab3ffa006d4ec81f2b69bac -Description: debug symbols for rccl-unittests -Maintainer: RCCL Maintainer -Package-Type: ddeb -Version: 2.20.5.60202-116~20.04 -Installed-Size: 4748 - -Package: rccl-unittests-dbgsym-rpath6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 171c4b54ebac1e09e26d86c29bed9a52f2ae2b86 -Depends: rccl-unittests-rpath6.2.2 (= 2.20.5.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rccl-unittests-dbgsym-rpath6.2.2/rccl-unittests-dbgsym-rpath6.2.2_2.20.5.60202-116~20.04_amd64.deb -Size: 856436 -SHA256: 5aa95fa28c6a38e357194e453d6d8c8e26ae5e7de3ffc82a613830953d61104c -SHA1: ecb277bb12dc6c22d3f2f537962aac8af1378838 -MD5sum: 67cbf3fdf580730795b8de0759381de7 -Description: debug symbols for rccl-unittests -Maintainer: RCCL Maintainer -Package-Type: ddeb -Version: 2.20.5.60202-116~20.04 -Installed-Size: 4748 - -Package: rccl-unittests-dbgsym6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 171c4b54ebac1e09e26d86c29bed9a52f2ae2b86 -Depends: rccl-unittests6.2.2 (= 2.20.5.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rccl-unittests-dbgsym6.2.2/rccl-unittests-dbgsym6.2.2_2.20.5.60202-116~20.04_amd64.deb -Size: 857164 -SHA256: 6922156e4d3fe49447e6642f5a7d6cbf7e5ffe3b3d5f89accb35db797ba8ddaf -SHA1: 5a6f5041abb64ee2c7a5c402fc8d69f07f8dc882 -MD5sum: 3d47c99b45563ddfc1b49ec8b43353a6 -Description: debug symbols for rccl-unittests -Maintainer: RCCL Maintainer -Package-Type: ddeb -Version: 2.20.5.60202-116~20.04 -Installed-Size: 4748 - -Package: rccl-unittests-rpath6.2.2 -Architecture: amd64 -Depends: rccl-rpath6.2.2 (>= 2.20.5.60202), libc6 (>= 2.14), libgcc-s1 (>= 3.0), libstdc++6 (>= 9) -Priority: optional -Section: devel -Filename: pool/main/r/rccl-unittests-rpath6.2.2/rccl-unittests-rpath6.2.2_2.20.5.60202-116~20.04_amd64.deb -Size: 352716 -SHA256: 8dd87ab480ec51fcd2996524c5ca1827005c701611009f8c737c07ef4838c140 -SHA1: 28325aa048eee30b6f9c447915621710b36d5db8 -MD5sum: 7bf1715deab07540c444fc38f781442f -Description: ROCm Communication Collectives Library - ROCm Communication Collectives Library - Optimized primitives for collective multi-GPU communication -Maintainer: RCCL Maintainer -Version: 2.20.5.60202-116~20.04 -Installed-Size: 1619 - -Package: rccl-unittests6.2.2 -Architecture: amd64 -Depends: rccl6.2.2 (>= 2.20.5.60202), libc6 (>= 2.14), libgcc-s1 (>= 3.0), libstdc++6 (>= 9) -Priority: optional -Section: devel -Filename: pool/main/r/rccl-unittests6.2.2/rccl-unittests6.2.2_2.20.5.60202-116~20.04_amd64.deb -Size: 352632 -SHA256: ee45f79635898bc6a17e78e5801265293db3b896026b1d794c664cdd4990c841 -SHA1: 257083fd7326a1113e79cf06d5c2ea8f59e44155 -MD5sum: 546e1a8d28a00125fe232ba150081792 -Description: ROCm Communication Collectives Library - ROCm Communication Collectives Library - Optimized primitives for collective multi-GPU communication -Maintainer: RCCL Maintainer -Version: 2.20.5.60202-116~20.04 -Installed-Size: 1619 - -Package: rccl6.2.2 -Architecture: amd64 -Depends: hip-runtime-amd6.2.2 (>= 4.5.0), rocm-smi-lib6.2.2 (>= 4.0.0), rocm-core6.2.2, libc6 (>= 2.17), libgcc-s1 (>= 3.0), libstdc++6 (>= 9) -Priority: optional -Section: devel -Filename: pool/main/r/rccl6.2.2/rccl6.2.2_2.20.5.60202-116~20.04_amd64.deb -Size: 62974816 -SHA256: 967cae620d293734f2f8c19d87e2906f99374fe0a7899284320f38e8e3a2f558 -SHA1: 1343a3e818a98290edd965ac96a29ef9a3fefcf4 -MD5sum: 2e7f38b0d0aa0f32ba3752219b8ebdc9 -Description: ROCm Communication Collectives Library - ROCm Communication Collectives Library - Optimized primitives for collective multi-GPU communication -Maintainer: RCCL Maintainer -Recommends: rccl-dev6.2.2 (>=2.20.5.60202) -Version: 2.20.5.60202-116~20.04 -Installed-Size: 1895293 - -Package: rdc -Architecture: amd64 -Depends: amd-smi-lib, libc6 -Priority: optional -Section: devel -Filename: pool/main/r/rdc/rdc_0.3.0.60202-116~20.04_amd64.deb -Size: 17998194 -SHA256: 6a708d3ceda8a37569f3659c26324b7f3be5bb25ccc428e9f121e12d317786cc -SHA1: d0f02494c33f209b8e7cb8c47e7708e7674ac422 -MD5sum: 6075c3e0d667f0c75908d3911d0d5bd5 -Description: Radeon Data Center Tools -Homepage: https://github.com/RadeonOpenCompute/rdc -Maintainer: RDC Support -Version: 0.3.0.60202-116~20.04 -Installed-Size: 135988 - -Package: rdc-rpath6.2.2 -Architecture: amd64 -Depends: amd-smi-lib-rpath6.2.2, libc6 -Priority: optional -Section: devel -Filename: pool/main/r/rdc-rpath6.2.2/rdc-rpath6.2.2_0.3.0.60202-116~20.04_amd64.deb -Size: 11338344 -SHA256: bcdf783e6018e1415d0bf71c3e15a9795c7eb1d58481ce6a3908866fde37de30 -SHA1: c21b79e058894018e3d6b7d1b9229818402b7228 -MD5sum: 7ef809ccc6a25ee3691e12bd7948dc44 -Description: Radeon Data Center Tools -Homepage: https://github.com/RadeonOpenCompute/rdc -Maintainer: RDC Support -Version: 0.3.0.60202-116~20.04 -Installed-Size: 135988 - -Package: rdc6.2.2 -Architecture: amd64 -Depends: amd-smi-lib6.2.2, libc6 -Priority: optional -Section: devel -Filename: pool/main/r/rdc6.2.2/rdc6.2.2_0.3.0.60202-116~20.04_amd64.deb -Size: 11335204 -SHA256: e7302a95dc8ca04c7d3373c62d38ee2899a668fac5989ee6f17d5dc611134423 -SHA1: 4e76a982870157ef1afbf05c23b296bba9fa4813 -MD5sum: 404da910a9c4b97dfacba6a2bb483e1c -Description: Radeon Data Center Tools -Homepage: https://github.com/RadeonOpenCompute/rdc -Maintainer: RDC Support -Version: 0.3.0.60202-116~20.04 -Installed-Size: 135988 - -Package: rocal -Architecture: amd64 -Depends: rpp, mivisionx, rocdecode -Priority: optional -Section: devel -Filename: pool/main/r/rocal/rocal_2.0.0.60202-116~20.04_amd64.deb -Size: 1394762 -SHA256: 5b3b2fd0892bde79e8f1f1aad74cf118d09bc7a55d5a98e821dd29dc5ade1ad4 -SHA1: bdef20bbeb4de97808c3f689f15d23dbc72f5006 -MD5sum: 4868870f2cc60c2acb875cb3e3346442 -Description: AMD rocAL is a comprehensive augmentation library. rocAL runtime package provides rocAL libraries and license.txt -Homepage: https://github.com/ROCm/rocAL -Maintainer: rocAL Support -Version: 2.0.0.60202-116~20.04 -Installed-Size: 8513 - -Package: rocal-dev -Architecture: amd64 -Depends: rocal, rpp-dev, mivisionx-dev, liblmdb-dev, libprotobuf-dev, rocdecode-dev -Priority: optional -Section: devel -Filename: pool/main/r/rocal-dev/rocal-dev_2.0.0.60202-116~20.04_amd64.deb -Size: 22748 -SHA256: 373f510cf4a2325abba3b31801673cda614bd41be3b75b4d80359bb65afeb11f -SHA1: dcacc7a112930bd80518e4906a5c47532b3e578a -MD5sum: 6d15c03343baa4501813ab2eee26a5be -Description: AMD rocAL is a comprehensive augmentation library. rocAL develop package provides rocAL libraries, header files, samples, and license.txt -Homepage: https://github.com/ROCm/rocAL -Maintainer: rocAL Support -Version: 2.0.0.60202-116~20.04 -Installed-Size: 240 - -Package: rocal-dev-rpath6.2.2 -Architecture: amd64 -Depends: rocal-rpath6.2.2, rpp-dev-rpath6.2.2, mivisionx-dev-rpath6.2.2, liblmdb-dev, libprotobuf-dev, rocdecode-dev-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocal-dev-rpath6.2.2/rocal-dev-rpath6.2.2_2.0.0.60202-116~20.04_amd64.deb -Size: 18800 -SHA256: 69e736f6574ee6d4ecd0998abf1c3f083060c58431e898df91484c25cb9b0f84 -SHA1: d034d5bfacace8a95475315586dced8abd29462c -MD5sum: 4945d9a819a552e53c74c91b4af4da33 -Description: AMD rocAL is a comprehensive augmentation library. rocAL develop package provides rocAL libraries, header files, samples, and license.txt -Homepage: https://github.com/ROCm/rocAL -Maintainer: rocAL Support -Version: 2.0.0.60202-116~20.04 -Installed-Size: 240 - -Package: rocal-dev6.2.2 -Architecture: amd64 -Depends: rocal6.2.2, rpp-dev6.2.2, mivisionx-dev6.2.2, liblmdb-dev, libprotobuf-dev, rocdecode-dev6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocal-dev6.2.2/rocal-dev6.2.2_2.0.0.60202-116~20.04_amd64.deb -Size: 18796 -SHA256: d2c886749567074390aa6321c401cf43480795ac32a8b79767b3ba1916cc3e2d -SHA1: 139a79e63e6b6167283694597c524da702367a63 -MD5sum: 7ac209b878cc3ad4cb2f08f2292e6bf5 -Description: AMD rocAL is a comprehensive augmentation library. rocAL develop package provides rocAL libraries, header files, samples, and license.txt -Homepage: https://github.com/ROCm/rocAL -Maintainer: rocAL Support -Version: 2.0.0.60202-116~20.04 -Installed-Size: 240 - -Package: rocal-rpath6.2.2 -Architecture: amd64 -Depends: rpp-rpath6.2.2, mivisionx-rpath6.2.2, rocdecode-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocal-rpath6.2.2/rocal-rpath6.2.2_2.0.0.60202-116~20.04_amd64.deb -Size: 965524 -SHA256: 5da411e326e1bf8c6cbf28e753527ea50ec7efd37f274c881bd2e555956f2bb3 -SHA1: f8fcb39fa5b255c28fb969f641e30c72c798f495 -MD5sum: 82eb06793b94afaf61e79ce45d0b4a23 -Description: AMD rocAL is a comprehensive augmentation library. rocAL runtime package provides rocAL libraries and license.txt -Homepage: https://github.com/ROCm/rocAL -Maintainer: rocAL Support -Version: 2.0.0.60202-116~20.04 -Installed-Size: 8513 - -Package: rocal-test -Architecture: amd64 -Depends: rocal-dev -Priority: optional -Section: devel -Filename: pool/main/r/rocal-test/rocal-test_2.0.0.60202-116~20.04_amd64.deb -Size: 32040300 -SHA256: 5f907bf5edd75a5c143af5c36e6fd1956f92072081d803222bccdc27d1865835 -SHA1: 6b625652d58d1fed5a155734fcd2cf944d8c3d36 -MD5sum: c223684bde69e5a91fbc2e906ea8560e -Description: AMD rocAL is a comprehensive augmentation library. rocAL Test package provides rocAL Test Components -Homepage: https://github.com/ROCm/rocAL -Maintainer: rocAL Support -Version: 2.0.0.60202-116~20.04 -Installed-Size: 34931 - -Package: rocal-test-rpath6.2.2 -Architecture: amd64 -Depends: rocal-dev-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocal-test-rpath6.2.2/rocal-test-rpath6.2.2_2.0.0.60202-116~20.04_amd64.deb -Size: 31891192 -SHA256: 69f82fc9cbb9df1f0c29d9789cda22dd86c5f067edec150a0477db9e83fe0682 -SHA1: 5a6ab6e6776f4e3c42432a3b59b944a3d96e5377 -MD5sum: bbccc9bd7977fdfbc569dd494b0772fe -Description: AMD rocAL is a comprehensive augmentation library. rocAL Test package provides rocAL Test Components -Homepage: https://github.com/ROCm/rocAL -Maintainer: rocAL Support -Version: 2.0.0.60202-116~20.04 -Installed-Size: 34931 - -Package: rocal-test6.2.2 -Architecture: amd64 -Depends: rocal-dev6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocal-test6.2.2/rocal-test6.2.2_2.0.0.60202-116~20.04_amd64.deb -Size: 31891312 -SHA256: 3984bbd0ec5704ac4f525331319b2393a6e0bde05cd185ff2adcb5ec270b54d5 -SHA1: 34698ca5d5e77294243108061be55f3eff2602a6 -MD5sum: be094622504ce6a89dd2e213766cd4fc -Description: AMD rocAL is a comprehensive augmentation library. rocAL Test package provides rocAL Test Components -Homepage: https://github.com/ROCm/rocAL -Maintainer: rocAL Support -Version: 2.0.0.60202-116~20.04 -Installed-Size: 34931 - -Package: rocal6.2.2 -Architecture: amd64 -Depends: rpp6.2.2, mivisionx6.2.2, rocdecode6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocal6.2.2/rocal6.2.2_2.0.0.60202-116~20.04_amd64.deb -Size: 965164 -SHA256: 7102158796740fbf1a90759f9d1e92b9850d13507603b2e878f880f20a34aca3 -SHA1: 8aff9dce3fb43b979fec8c9df1aba6de06ea3c8c -MD5sum: 3a61331c666593dba978d2cb6be0568f -Description: AMD rocAL is a comprehensive augmentation library. rocAL runtime package provides rocAL libraries and license.txt -Homepage: https://github.com/ROCm/rocAL -Maintainer: rocAL Support -Version: 2.0.0.60202-116~20.04 -Installed-Size: 8513 - -Package: rocalution -Architecture: amd64 -Depends: hip-runtime-amd (>= 4.5.0), rocsparse (>= 1.12.10), rocblas (>= 2.22.0), rocrand (>= 2.1.0), rocm-core -Priority: optional -Section: devel -Filename: pool/main/r/rocalution/rocalution_3.2.0.60202-116~20.04_amd64.deb -Size: 4577776 -SHA256: aa0c0707eccee21e4302cac062f868e7ee4add1b16b6647158a2b4ecdd323a4e -SHA1: 5d847e368d44403475e0769b39486e13f0fa441f -MD5sum: 415ead06113b40c53f3541272bcd48b5 -Description: ROCm library for sparse linear systems -Maintainer: rocALUTION Maintainer -Recommends: rocalution-dev (>=3.2.0.60202) -Version: 3.2.0.60202-116~20.04 -Installed-Size: 167584 - -Package: rocalution-asan -Architecture: amd64 -Depends: hip-runtime-amd-asan (>= 4.5.0), rocsparse (>= 1.12.10), rocblas (>= 2.22.0), rocrand (>= 2.1.0), rocm-core-asan -Priority: optional -Section: devel -Filename: pool/main/r/rocalution-asan/rocalution-asan_3.2.0.60202-116~20.04_amd64.deb -Size: 6260026 -SHA256: 766a8561555fd04caa133f4c9778b49b29535444efc118da5410158f44a9a848 -SHA1: cf44df9a075537f4fbbd574c162e4cae34c017ce -MD5sum: beb5a1e784cdf2326cbb92d1c1005d1a -Description: ROCm library for sparse linear systems -Maintainer: rocALUTION Maintainer -Recommends: rocalution-asan-dev (>=3.2.0.60202) -Version: 3.2.0.60202-116~20.04 -Installed-Size: 197608 - -Package: rocalution-asan-dbgsym -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: fd383b5c1a0cbc9337210935bb6750ca9a4ff63a fa9fd32fa46261960435eecb759f061a9a3405ec -Depends: rocalution-asan (= 3.2.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocalution-asan-dbgsym/rocalution-asan-dbgsym_3.2.0.60202-116~20.04_amd64.deb -Size: 10878958 -SHA256: db6af28ff7be46771da37b82741f2af23e1ce89b778a5dd686acd11717a85f71 -SHA1: a76e399889657d77eb2d4454c7621846603b1108 -MD5sum: 067b034a583b9c129d2deb744164cbd2 -Description: debug symbols for rocalution-asan -Maintainer: rocALUTION Maintainer -Package-Type: ddeb -Version: 3.2.0.60202-116~20.04 -Installed-Size: 15216 - -Package: rocalution-asan-dbgsym-rpath6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: fd383b5c1a0cbc9337210935bb6750ca9a4ff63a fa9fd32fa46261960435eecb759f061a9a3405ec -Depends: rocalution-asan-rpath6.2.2 (= 3.2.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocalution-asan-dbgsym-rpath6.2.2/rocalution-asan-dbgsym-rpath6.2.2_3.2.0.60202-116~20.04_amd64.deb -Size: 10880400 -SHA256: a8b6743bf7de038975f7eb6dbe8077610008063ff78772075c3cbe1203ad4f6d -SHA1: 07bca115ed05e6063347e75ebd2cfe7696a284a1 -MD5sum: c6adf028a1da033be3589661f864a98e -Description: debug symbols for rocalution-asan -Maintainer: rocALUTION Maintainer -Package-Type: ddeb -Version: 3.2.0.60202-116~20.04 -Installed-Size: 15216 - -Package: rocalution-asan-dbgsym6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: fd383b5c1a0cbc9337210935bb6750ca9a4ff63a fa9fd32fa46261960435eecb759f061a9a3405ec -Depends: rocalution-asan6.2.2 (= 3.2.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocalution-asan-dbgsym6.2.2/rocalution-asan-dbgsym6.2.2_3.2.0.60202-116~20.04_amd64.deb -Size: 10878992 -SHA256: 2e0c29755de0cada06d316e220f60a4f6fa9c5c4af69d829a1a90f467a25c80a -SHA1: 23c165e8931916060c91a32e90eb0ac9daad0cd6 -MD5sum: bcf69b90ef159cb78167ece8f47482ea -Description: debug symbols for rocalution-asan -Maintainer: rocALUTION Maintainer -Package-Type: ddeb -Version: 3.2.0.60202-116~20.04 -Installed-Size: 15216 - -Package: rocalution-asan-rpath6.2.2 -Architecture: amd64 -Depends: hip-runtime-amd-asan-rpath6.2.2 (>= 4.5.0), rocsparse (>= 1.12.10), rocblas (>= 2.22.0), rocrand (>= 2.1.0), rocm-core-asan-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocalution-asan-rpath6.2.2/rocalution-asan-rpath6.2.2_3.2.0.60202-116~20.04_amd64.deb -Size: 6259544 -SHA256: 645c7e383deeaea1a4092d4adafb2dd4f5ed66e2788ac9cf6bfdd981b8d4acc0 -SHA1: fe86b3566dc5028d89d05242d36337e9b1ba1add -MD5sum: a9de506c930a2df5594b50720b0e1dee -Description: ROCm library for sparse linear systems -Maintainer: rocALUTION Maintainer -Recommends: rocalution-asan-dev (>=3.2.0.60202) -Version: 3.2.0.60202-116~20.04 -Installed-Size: 197608 - -Package: rocalution-asan6.2.2 -Architecture: amd64 -Depends: hip-runtime-amd-asan6.2.2 (>= 4.5.0), rocsparse (>= 1.12.10), rocblas (>= 2.22.0), rocrand (>= 2.1.0), rocm-core-asan6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocalution-asan6.2.2/rocalution-asan6.2.2_3.2.0.60202-116~20.04_amd64.deb -Size: 6261016 -SHA256: 81bc4b6a28d6c0fe56f6b45ccad525a530d63cda703764e9dd16a13fde375339 -SHA1: 416a39c52b409824419b4fd1abbba9f55ad17ef6 -MD5sum: c15222fafe0a02d28689450a0e73ff1a -Description: ROCm library for sparse linear systems -Maintainer: rocALUTION Maintainer -Recommends: rocalution-asan-dev (>=3.2.0.60202) -Version: 3.2.0.60202-116~20.04 -Installed-Size: 197608 - -Package: rocalution-dbgsym -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: f8d8e3034d496581278547e8c554de953478d502 d7369600dd69d60c780e7b7980a23559e2089cc9 -Depends: rocalution (= 3.2.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocalution-dbgsym/rocalution-dbgsym_3.2.0.60202-116~20.04_amd64.deb -Size: 11227570 -SHA256: 4a7268886b295abf83dbe6168edec2dbc7384ae25018998c307bb57194f4b079 -SHA1: 870eedbaf4470a86820c3c76842f5dc8997bed77 -MD5sum: 6c77a63518977193f8e38cca22ae8afe -Description: debug symbols for rocalution -Maintainer: rocALUTION Maintainer -Package-Type: ddeb -Version: 3.2.0.60202-116~20.04 -Installed-Size: 78585 - -Package: rocalution-dbgsym-rpath6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: f8d8e3034d496581278547e8c554de953478d502 d7369600dd69d60c780e7b7980a23559e2089cc9 -Depends: rocalution-rpath6.2.2 (= 3.2.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocalution-dbgsym-rpath6.2.2/rocalution-dbgsym-rpath6.2.2_3.2.0.60202-116~20.04_amd64.deb -Size: 11235792 -SHA256: a059d92ec05de49c2b7d6fce82e390975e7faeb6ceefb1f75a334bdd2c0011d8 -SHA1: 0786e75e3a92b5b12220b9a0316d9bcb7dccff07 -MD5sum: 4de8af83982e1e2e89e847a02a743b48 -Description: debug symbols for rocalution -Maintainer: rocALUTION Maintainer -Package-Type: ddeb -Version: 3.2.0.60202-116~20.04 -Installed-Size: 78585 - -Package: rocalution-dbgsym6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: f8d8e3034d496581278547e8c554de953478d502 d7369600dd69d60c780e7b7980a23559e2089cc9 -Depends: rocalution6.2.2 (= 3.2.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocalution-dbgsym6.2.2/rocalution-dbgsym6.2.2_3.2.0.60202-116~20.04_amd64.deb -Size: 11253328 -SHA256: 103b048e9fc9543d54ada88b8058ca50e5b1df89446f59b69e446fd8d1b8b530 -SHA1: 59d43437bbccc53e4a92533824d9f65693f47499 -MD5sum: 39a22e598b416f1e9e8ee943785ce874 -Description: debug symbols for rocalution -Maintainer: rocALUTION Maintainer -Package-Type: ddeb -Version: 3.2.0.60202-116~20.04 -Installed-Size: 78585 - -Package: rocalution-dev -Architecture: amd64 -Depends: rocalution (>= 3.2.0.60202) -Priority: optional -Section: devel -Filename: pool/main/r/rocalution-dev/rocalution-dev_3.2.0.60202-116~20.04_amd64.deb -Size: 42974 -SHA256: e0720d9cbb87d242759b2e5086b60e68f1f02d5c51a926b5f731895156d3e54b -SHA1: 8c1a899b2b3803f45724375511e2172eb4a24898 -MD5sum: 1df676e6c865ad6e6d2a95c5ab2afbaa -Description: ROCm library for sparse linear systems -Maintainer: rocALUTION Maintainer -Version: 3.2.0.60202-116~20.04 -Installed-Size: 422 - -Package: rocalution-dev-rpath6.2.2 -Architecture: amd64 -Depends: rocalution-rpath6.2.2 (>= 3.2.0.60202) -Priority: optional -Section: devel -Filename: pool/main/r/rocalution-dev-rpath6.2.2/rocalution-dev-rpath6.2.2_3.2.0.60202-116~20.04_amd64.deb -Size: 42948 -SHA256: cc8fd1b495c948a3ccffea15390115166eeeec485767b792f8e5e59af6328636 -SHA1: 4f3cf662a17156b78a5788f32661f5218d36e486 -MD5sum: b291a50a77a6b7fd02751c977c409d6c -Description: ROCm library for sparse linear systems -Maintainer: rocALUTION Maintainer -Version: 3.2.0.60202-116~20.04 -Installed-Size: 422 - -Package: rocalution-dev6.2.2 -Architecture: amd64 -Depends: rocalution6.2.2 (>= 3.2.0.60202) -Priority: optional -Section: devel -Filename: pool/main/r/rocalution-dev6.2.2/rocalution-dev6.2.2_3.2.0.60202-116~20.04_amd64.deb -Size: 42948 -SHA256: fc5650bd10a7a62d7cca0d353ea922cec37fb878ddec99b2b67b399447102c0a -SHA1: 07e0197fa9acaac12f09df5b4fef35dcf12b85a7 -MD5sum: 4dca39fa80690a493ed3baacb7345c22 -Description: ROCm library for sparse linear systems -Maintainer: rocALUTION Maintainer -Version: 3.2.0.60202-116~20.04 -Installed-Size: 422 - -Package: rocalution-rpath6.2.2 -Architecture: amd64 -Depends: hip-runtime-amd-rpath6.2.2 (>= 4.5.0), rocsparse-rpath6.2.2 (>= 1.12.10), rocblas-rpath6.2.2 (>= 2.22.0), rocrand-rpath6.2.2 (>= 2.1.0), rocm-core-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocalution-rpath6.2.2/rocalution-rpath6.2.2_3.2.0.60202-116~20.04_amd64.deb -Size: 4578680 -SHA256: 0be55ffbc4bb710f6a8993048abbb760f2fb89a178693116fd577b9aaf4e3db9 -SHA1: 929d5649d9d910a1a5eb28fe63d87ec3d6b61d7c -MD5sum: 7d3eea89a381b8f0dcac49f448da8c89 -Description: ROCm library for sparse linear systems -Maintainer: rocALUTION Maintainer -Recommends: rocalution-dev-rpath6.2.2 (>=3.2.0.60202) -Version: 3.2.0.60202-116~20.04 -Installed-Size: 167584 - -Package: rocalution6.2.2 -Architecture: amd64 -Depends: hip-runtime-amd6.2.2 (>= 4.5.0), rocsparse6.2.2 (>= 1.12.10), rocblas6.2.2 (>= 2.22.0), rocrand6.2.2 (>= 2.1.0), rocm-core6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocalution6.2.2/rocalution6.2.2_3.2.0.60202-116~20.04_amd64.deb -Size: 4573788 -SHA256: 8511bed49ae58dee12186a667a7a2a6ae60a032793538c4788227cd9b3b46774 -SHA1: 329d70208ea4386fc5190e1aaf25262d26f45d16 -MD5sum: 7ef33736dfea662ee41500c463b6e0d0 -Description: ROCm library for sparse linear systems -Maintainer: rocALUTION Maintainer -Recommends: rocalution-dev6.2.2 (>=3.2.0.60202) -Version: 3.2.0.60202-116~20.04 -Installed-Size: 167584 - -Package: rocblas -Architecture: amd64 -Depends: hip-runtime-amd (>= 4.5.0), rocm-core -Priority: optional -Section: devel -Filename: pool/main/r/rocblas/rocblas_4.2.1.60202-116~20.04_amd64.deb -Size: 144341310 -SHA256: 04f1d773078f3e3ece3ddd3728ab927403d61049a8f009be5d94f8f8cb42e15c -SHA1: df7dac09ea8db23704b86ccce60ae0bef1e05891 -MD5sum: d9bd34440448cea6c1479c8f1e8733cc -Description: rocBLAS is the AMD library for BLAS in ROCm. Implemented using the HIP language and optimized for AMD GPUs -Maintainer: rocBLAS Maintainer -Recommends: rocblas-dev (>=4.2.1.60202) -Version: 4.2.1.60202-116~20.04 -Installed-Size: 5844976 - -Package: rocblas-asan -Architecture: amd64 -Depends: hip-runtime-amd-asan (>= 4.5.0), rocm-core-asan -Priority: optional -Section: devel -Filename: pool/main/r/rocblas-asan/rocblas-asan_4.2.1.60202-116~20.04_amd64.deb -Size: 285993780 -SHA256: ddea65049d29e6e315a8d1a1313765be13c926cd29cd142244c67ef91f26791b -SHA1: c6215574e58bf6e960f7ed56f38006d55b6984f4 -MD5sum: 76450a7f4e2a7f11d4ce6bc5ecc0bbe3 -Description: rocBLAS is the AMD library for BLAS in ROCm. Implemented using the HIP language and optimized for AMD GPUs -Maintainer: rocBLAS Maintainer -Recommends: rocblas-asan-dev (>=4.2.1.60202) -Version: 4.2.1.60202-116~20.04 -Installed-Size: 2902247 - -Package: rocblas-asan-rpath6.2.2 -Architecture: amd64 -Depends: hip-runtime-amd-asan-rpath6.2.2 (>= 4.5.0), rocm-core-asan-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocblas-asan-rpath6.2.2/rocblas-asan-rpath6.2.2_4.2.1.60202-116~20.04_amd64.deb -Size: 285986744 -SHA256: 7a3c99eefb194f956435213a6781d8549c2fda5d98a76f1b5b60285af1263733 -SHA1: 263b2cea39ad3c48683c2caa6f48df823188cc5b -MD5sum: e49b26b24345fe7e6e354fb64732601b -Description: rocBLAS is the AMD library for BLAS in ROCm. Implemented using the HIP language and optimized for AMD GPUs -Maintainer: rocBLAS Maintainer -Recommends: rocblas-asan-dev (>=4.2.1.60202) -Version: 4.2.1.60202-116~20.04 -Installed-Size: 2902247 - -Package: rocblas-asan6.2.2 -Architecture: amd64 -Depends: hip-runtime-amd-asan6.2.2 (>= 4.5.0), rocm-core-asan6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocblas-asan6.2.2/rocblas-asan6.2.2_4.2.1.60202-116~20.04_amd64.deb -Size: 286010988 -SHA256: 4151ea3b33d980e8297bbd8176edd939d061ea2da0fea89d6e5545b4316f9307 -SHA1: 2880cedf15ae08050cf5bd70e1bca4a027614e4a -MD5sum: c329efa38c77d09d5b938725bb341b0d -Description: rocBLAS is the AMD library for BLAS in ROCm. Implemented using the HIP language and optimized for AMD GPUs -Maintainer: rocBLAS Maintainer -Recommends: rocblas-asan-dev (>=4.2.1.60202) -Version: 4.2.1.60202-116~20.04 -Installed-Size: 2902247 - -Package: rocblas-dev -Architecture: amd64 -Depends: rocblas (>= 4.2.1.60202) -Priority: optional -Section: devel -Filename: pool/main/r/rocblas-dev/rocblas-dev_4.2.1.60202-116~20.04_amd64.deb -Size: 91614 -SHA256: 93a6c284a84ee8d8bf59c5473836574a6974af59b317c339ade91e39488eb17b -SHA1: a16a6c09a721eda5a5b4534f3886eaf202c790ad -MD5sum: e39bd6fc1011cc03e1dec010298b36da -Description: rocBLAS is the AMD library for BLAS in ROCm. Implemented using the HIP language and optimized for AMD GPUs -Maintainer: rocBLAS Maintainer -Version: 4.2.1.60202-116~20.04 -Installed-Size: 2530 - -Package: rocblas-dev-rpath6.2.2 -Architecture: amd64 -Depends: rocblas-rpath6.2.2 (>= 4.2.1.60202) -Priority: optional -Section: devel -Filename: pool/main/r/rocblas-dev-rpath6.2.2/rocblas-dev-rpath6.2.2_4.2.1.60202-116~20.04_amd64.deb -Size: 91756 -SHA256: bea5c7dcf64e3a6b1d2ae058c0e17496ed3a193e1f23c2c61245f10f359b7591 -SHA1: 4a982221fc1ca165b78a4b9678ef6a787298d8df -MD5sum: d368e7ddea44b5d1e157f594f95fa8e7 -Description: rocBLAS is the AMD library for BLAS in ROCm. Implemented using the HIP language and optimized for AMD GPUs -Maintainer: rocBLAS Maintainer -Version: 4.2.1.60202-116~20.04 -Installed-Size: 2530 - -Package: rocblas-dev6.2.2 -Architecture: amd64 -Depends: rocblas6.2.2 (>= 4.2.1.60202) -Priority: optional -Section: devel -Filename: pool/main/r/rocblas-dev6.2.2/rocblas-dev6.2.2_4.2.1.60202-116~20.04_amd64.deb -Size: 91744 -SHA256: ed6ffdbb5fb0f9420319fbed74ebb38b293db73efdb94de8e88c3d82c0bd0276 -SHA1: e9949e3bb8ef7a37743b99bf255a15befa8c7090 -MD5sum: 011a7578dcef34821b8b30cf3b8eb99c -Description: rocBLAS is the AMD library for BLAS in ROCm. Implemented using the HIP language and optimized for AMD GPUs -Maintainer: rocBLAS Maintainer -Version: 4.2.1.60202-116~20.04 -Installed-Size: 2530 - -Package: rocblas-rpath6.2.2 -Architecture: amd64 -Depends: hip-runtime-amd-rpath6.2.2 (>= 4.5.0), rocm-core-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocblas-rpath6.2.2/rocblas-rpath6.2.2_4.2.1.60202-116~20.04_amd64.deb -Size: 144295048 -SHA256: 270389553f25e5da1ed58d9e5b8f93b4b477363482d93893fa58ed25d1b44c5d -SHA1: 6501576f08c8acf29eb059fb8a8e03528ea49024 -MD5sum: 3743a21173d649eea65740517a9e1590 -Description: rocBLAS is the AMD library for BLAS in ROCm. Implemented using the HIP language and optimized for AMD GPUs -Maintainer: rocBLAS Maintainer -Recommends: rocblas-dev-rpath6.2.2 (>=4.2.1.60202) -Version: 4.2.1.60202-116~20.04 -Installed-Size: 5844976 - -Package: rocblas6.2.2 -Architecture: amd64 -Depends: hip-runtime-amd6.2.2 (>= 4.5.0), rocm-core6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocblas6.2.2/rocblas6.2.2_4.2.1.60202-116~20.04_amd64.deb -Size: 144295224 -SHA256: bbedfd27c104ce00856ad62469fce083929c1f2dc3248ec6331d67add9e28e7b -SHA1: c79fd2e180f2c480c0e237e914cf551fff89a24f -MD5sum: d5de9e2f17d28acb0b49b1bf4d636df9 -Description: rocBLAS is the AMD library for BLAS in ROCm. Implemented using the HIP language and optimized for AMD GPUs -Maintainer: rocBLAS Maintainer -Recommends: rocblas-dev6.2.2 (>=4.2.1.60202) -Version: 4.2.1.60202-116~20.04 -Installed-Size: 5844976 - -Package: rocdecode -Architecture: amd64 -Depends: rocm-core, rocm-hip-runtime, libva2, libdrm-amdgpu1, mesa-amdgpu-va-drivers -Priority: optional -Section: devel -Filename: pool/main/r/rocdecode/rocdecode_0.6.0.60202-116~20.04_amd64.deb -Size: 164820 -SHA256: 3b9f6edfc772bd9fe75b3b2c888d483fb5acb4f9a45a60e8479f63a991475226 -SHA1: 9fc39efe05653d77e83b6d21c8a7e1db3c013f21 -MD5sum: a649d69b384901e277088839fe44f635 -Description: AMD rocDecode is a high performance video decode SDK for AMD GPUs. rocDecode runtime package provides rocDecode library and license.txt -Homepage: https://github.com/ROCm/rocDecode -Maintainer: rocDecode Support -Version: 0.6.0.60202-116~20.04 -Installed-Size: 860 - -Package: rocdecode-dev -Architecture: amd64 -Depends: rocm-core, rocdecode, rocm-hip-runtime-dev, libva-dev, pkg-config, ffmpeg, libavcodec-dev, libavformat-dev, libavutil-dev -Priority: optional -Section: devel -Filename: pool/main/r/rocdecode-dev/rocdecode-dev_0.6.0.60202-116~20.04_amd64.deb -Size: 34708504 -SHA256: 767b71d7147d9d0a0fc01dba71c899e9976053938e7d882fbe086c51e66f1391 -SHA1: 059af7797cfbf1f3923954758db6f9fb7021fbd1 -MD5sum: 607ba82aab4edc8ce4a2e6d31898dc28 -Description: AMD rocDecode is a high performance video decode SDK for AMD GPUs. rocDecode develop package provides rocDecode library, header files, samples, and license.txt -Homepage: https://github.com/ROCm/rocDecode -Maintainer: rocDecode Support -Version: 0.6.0.60202-116~20.04 -Installed-Size: 35013 - -Package: rocdecode-dev-rpath6.2.2 -Architecture: amd64 -Depends: rocm-core-rpath6.2.2, rocdecode-rpath6.2.2, rocm-hip-runtime-dev-rpath6.2.2, libva-dev, pkg-config, ffmpeg, libavcodec-dev, libavformat-dev, libavutil-dev -Priority: optional -Section: devel -Filename: pool/main/r/rocdecode-dev-rpath6.2.2/rocdecode-dev-rpath6.2.2_0.6.0.60202-116~20.04_amd64.deb -Size: 34913388 -SHA256: f53383d37512a25c27dba38e4822ab4ec84659e2d3cfe39a8cecd7f6186db520 -SHA1: 7f16e720a29d9768a3e5448f77553322cec7607e -MD5sum: a30f59301ca9077acc07edf6c7fccc9c -Description: AMD rocDecode is a high performance video decode SDK for AMD GPUs. rocDecode develop package provides rocDecode library, header files, samples, and license.txt -Homepage: https://github.com/ROCm/rocDecode -Maintainer: rocDecode Support -Version: 0.6.0.60202-116~20.04 -Installed-Size: 35013 - -Package: rocdecode-dev6.2.2 -Architecture: amd64 -Depends: rocm-core6.2.2, rocdecode6.2.2, rocm-hip-runtime-dev6.2.2, libva-dev, pkg-config, ffmpeg, libavcodec-dev, libavformat-dev, libavutil-dev -Priority: optional -Section: devel -Filename: pool/main/r/rocdecode-dev6.2.2/rocdecode-dev6.2.2_0.6.0.60202-116~20.04_amd64.deb -Size: 34913380 -SHA256: 4366c12381a5ed729c1b32c8857bca4f13aa4542892d896be4e8468ce8d386af -SHA1: f17916bd177623d21e1088f2de6e1efc4c73b967 -MD5sum: 11eb763946a2d8144095bd88138c3db9 -Description: AMD rocDecode is a high performance video decode SDK for AMD GPUs. rocDecode develop package provides rocDecode library, header files, samples, and license.txt -Homepage: https://github.com/ROCm/rocDecode -Maintainer: rocDecode Support -Version: 0.6.0.60202-116~20.04 -Installed-Size: 35013 - -Package: rocdecode-rpath6.2.2 -Architecture: amd64 -Depends: rocm-core-rpath6.2.2, rocm-hip-runtime-rpath6.2.2, libva2, libdrm-amdgpu1, mesa-amdgpu-va-drivers -Priority: optional -Section: devel -Filename: pool/main/r/rocdecode-rpath6.2.2/rocdecode-rpath6.2.2_0.6.0.60202-116~20.04_amd64.deb -Size: 123056 -SHA256: a572b8fa44cc87036dcd06d90e97c096573b1ab7f7372be8242acdf6dc77992f -SHA1: bd00edf84dc6c792e1dffff48256db8c741fa459 -MD5sum: 27b031e573e6d7b14ca7dd8dfb0b78db -Description: AMD rocDecode is a high performance video decode SDK for AMD GPUs. rocDecode runtime package provides rocDecode library and license.txt -Homepage: https://github.com/ROCm/rocDecode -Maintainer: rocDecode Support -Version: 0.6.0.60202-116~20.04 -Installed-Size: 860 - -Package: rocdecode-test -Architecture: amd64 -Depends: rocm-core, rocdecode-dev -Priority: optional -Section: devel -Filename: pool/main/r/rocdecode-test/rocdecode-test_0.6.0.60202-116~20.04_amd64.deb -Size: 6626 -SHA256: 3b6c3d4eb13a270f2eb198a3bbb9ca5a9fb3ee0bda0968fc425deb1387253fd1 -SHA1: 71a7a5205c36c485d220d6f0e8a80ab677076ae2 -MD5sum: b8ff26a998dc57a79fde82d301a5f82b -Description: AMD rocDecode is a high performance video decode SDK for AMD GPUs. rocDecode Test package provides rocDecode Test Components -Homepage: https://github.com/ROCm/rocDecode -Maintainer: rocDecode Support -Version: 0.6.0.60202-116~20.04 -Installed-Size: 58 - -Package: rocdecode-test-rpath6.2.2 -Architecture: amd64 -Depends: rocm-core-rpath6.2.2, rocdecode-dev-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocdecode-test-rpath6.2.2/rocdecode-test-rpath6.2.2_0.6.0.60202-116~20.04_amd64.deb -Size: 6528 -SHA256: ab8689f812818e6cd897ec4c6ce81fe711797098df5b7b92fa59abd325c81aaa -SHA1: c2cc4b482bc28b0ea4477133069821c7e6223fac -MD5sum: 47cddf3b00250029dbec93c48a99b76f -Description: AMD rocDecode is a high performance video decode SDK for AMD GPUs. rocDecode Test package provides rocDecode Test Components -Homepage: https://github.com/ROCm/rocDecode -Maintainer: rocDecode Support -Version: 0.6.0.60202-116~20.04 -Installed-Size: 58 - -Package: rocdecode-test6.2.2 -Architecture: amd64 -Depends: rocm-core6.2.2, rocdecode-dev6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocdecode-test6.2.2/rocdecode-test6.2.2_0.6.0.60202-116~20.04_amd64.deb -Size: 6524 -SHA256: f53e4a679796a0e61e301f20260196ab1223292159296b4f2f758afa71ced63b -SHA1: e7cb502e4f5bb431da02a7f258cb6421a399fd2e -MD5sum: 72b3f16e2eaa2e000dd55e2700303243 -Description: AMD rocDecode is a high performance video decode SDK for AMD GPUs. rocDecode Test package provides rocDecode Test Components -Homepage: https://github.com/ROCm/rocDecode -Maintainer: rocDecode Support -Version: 0.6.0.60202-116~20.04 -Installed-Size: 58 - -Package: rocdecode6.2.2 -Architecture: amd64 -Depends: rocm-core6.2.2, rocm-hip-runtime6.2.2, libva2, libdrm-amdgpu1, mesa-amdgpu-va-drivers -Priority: optional -Section: devel -Filename: pool/main/r/rocdecode6.2.2/rocdecode6.2.2_0.6.0.60202-116~20.04_amd64.deb -Size: 123136 -SHA256: 680be8eac981c75495660c9bcdf139703570937ce2082bc794a9a75fe8561c5d -SHA1: 9eb7dc5e35769b790bbbedd360cc5c70fa373dfe -MD5sum: 838f6f92c7ddc6603ea72c708a44f7bd -Description: AMD rocDecode is a high performance video decode SDK for AMD GPUs. rocDecode runtime package provides rocDecode library and license.txt -Homepage: https://github.com/ROCm/rocDecode -Maintainer: rocDecode Support -Version: 0.6.0.60202-116~20.04 -Installed-Size: 860 - -Package: rocfft -Architecture: amd64 -Depends: rocm-core -Priority: optional -Section: devel -Filename: pool/main/r/rocfft/rocfft_1.0.29.60202-116~20.04_amd64.deb -Size: 100619296 -SHA256: abf0efb69e5ef7292a127f1bc48bd4008347d1b16731471e5f816665aa3824b0 -SHA1: cb65f8fe04b6720c7806596df9d583ca61ffacdd -MD5sum: 412650961e78f996965d4c3c57ee238a -Description: ROCm FFT library -Maintainer: rocfft-maintainer@amd.com -Recommends: rocfft-dev (>=1.0.29.60202) -Version: 1.0.29.60202-116~20.04 -Installed-Size: 2083197 - -Package: rocfft-asan -Architecture: amd64 -Depends: rocm-core-asan -Priority: optional -Section: devel -Filename: pool/main/r/rocfft-asan/rocfft-asan_1.0.29.60202-116~20.04_amd64.deb -Size: 4694932 -SHA256: 463bba5d57d23c501d70b5107a3f7f4cf6ce1cbff69832faefc4d2fcf9123dcf -SHA1: 6d01b0c97c1cdb59cc3632aad5d152a03bc25d55 -MD5sum: af0b962205d85083e9a179f0b023f995 -Description: ROCm FFT library -Maintainer: rocfft-maintainer@amd.com -Recommends: rocfft-asan-dev (>=1.0.29.60202) -Version: 1.0.29.60202-116~20.04 -Installed-Size: 40988 - -Package: rocfft-asan-dbgsym -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 5190a68ffb5acf0f2c2e99b725aeacd25d1e7ac4 -Depends: rocfft-asan (= 1.0.29.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocfft-asan-dbgsym/rocfft-asan-dbgsym_1.0.29.60202-116~20.04_amd64.deb -Size: 21665926 -SHA256: 69439a8a15c2e659f2e68e129883db9c975b0dcd051dc9f1dbf54ce4fed39c26 -SHA1: 0b0a559f4d5ca5b76400cde25e071d2a4b462388 -MD5sum: 15607550c37d2e5b5b8b4f3f938aae4e -Description: debug symbols for rocfft-asan -Maintainer: rocfft-maintainer@amd.com -Package-Type: ddeb -Version: 1.0.29.60202-116~20.04 -Installed-Size: 24795 - -Package: rocfft-asan-dbgsym-rpath6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 5190a68ffb5acf0f2c2e99b725aeacd25d1e7ac4 -Depends: rocfft-asan-rpath6.2.2 (= 1.0.29.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocfft-asan-dbgsym-rpath6.2.2/rocfft-asan-dbgsym-rpath6.2.2_1.0.29.60202-116~20.04_amd64.deb -Size: 21666048 -SHA256: 6278e69bcc5ff4dccfde0518aa9d1425aaa210d8e278b02f0e41c5bdcf4758a1 -SHA1: 33090f6c6494e03e9811db943496dd5ea628f8a4 -MD5sum: 347be4582f64834aa525621a4bd801ef -Description: debug symbols for rocfft-asan -Maintainer: rocfft-maintainer@amd.com -Package-Type: ddeb -Version: 1.0.29.60202-116~20.04 -Installed-Size: 24795 - -Package: rocfft-asan-dbgsym6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 5190a68ffb5acf0f2c2e99b725aeacd25d1e7ac4 -Depends: rocfft-asan6.2.2 (= 1.0.29.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocfft-asan-dbgsym6.2.2/rocfft-asan-dbgsym6.2.2_1.0.29.60202-116~20.04_amd64.deb -Size: 21665568 -SHA256: ea624ef6fe3299d036e862aac5dac34d0d6218830371704b75af43affa4f9c5c -SHA1: a9b2f5d1bcfd8ab3c444d1a8d3f53ce58402ff75 -MD5sum: 4da44c67af27840fe58875a159dadbc6 -Description: debug symbols for rocfft-asan -Maintainer: rocfft-maintainer@amd.com -Package-Type: ddeb -Version: 1.0.29.60202-116~20.04 -Installed-Size: 24795 - -Package: rocfft-asan-rpath6.2.2 -Architecture: amd64 -Depends: rocm-core-asan-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocfft-asan-rpath6.2.2/rocfft-asan-rpath6.2.2_1.0.29.60202-116~20.04_amd64.deb -Size: 4686888 -SHA256: 9cc99786f0bd22fe089642dfd43d4b7445c83dac3bf9dd0607e96a325334a2ae -SHA1: 46ce74d3664de56bec7329044345a44c012561b4 -MD5sum: a1ed3ee71d7e518eb26e176df72e356f -Description: ROCm FFT library -Maintainer: rocfft-maintainer@amd.com -Recommends: rocfft-asan-dev (>=1.0.29.60202) -Version: 1.0.29.60202-116~20.04 -Installed-Size: 40988 - -Package: rocfft-asan6.2.2 -Architecture: amd64 -Depends: rocm-core-asan6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocfft-asan6.2.2/rocfft-asan6.2.2_1.0.29.60202-116~20.04_amd64.deb -Size: 4688376 -SHA256: c9bf50d7d9cbce5a67b56e0c7afd7131405f0997cc3d90d6df786adfe9f985ed -SHA1: 08cbdb87e9cb38777a487a86318ffe2bbeb86660 -MD5sum: b46d962ee136ccc27db4b4a7dba36e77 -Description: ROCm FFT library -Maintainer: rocfft-maintainer@amd.com -Recommends: rocfft-asan-dev (>=1.0.29.60202) -Version: 1.0.29.60202-116~20.04 -Installed-Size: 40988 - -Package: rocfft-dbgsym -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 126abc910a9659a1b1b7fe468968d6f0dd685152 a067c448ad8dd97e851d90c68f32c23669b5a430 -Depends: rocfft (= 1.0.29.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocfft-dbgsym/rocfft-dbgsym_1.0.29.60202-116~20.04_amd64.deb -Size: 9987516 -SHA256: e54afa38c77a36116a153a1af83c9b13cf249b2909d04ba0c99ef8e49c2db28d -SHA1: 133b295951f1f34e18961e755d36749944348333 -MD5sum: a79c210745e3eebeea7d0d2d16e76ebb -Description: debug symbols for rocfft -Maintainer: rocfft-maintainer@amd.com -Package-Type: ddeb -Version: 1.0.29.60202-116~20.04 -Installed-Size: 65778 - -Package: rocfft-dbgsym-rpath6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 126abc910a9659a1b1b7fe468968d6f0dd685152 a067c448ad8dd97e851d90c68f32c23669b5a430 -Depends: rocfft-rpath6.2.2 (= 1.0.29.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocfft-dbgsym-rpath6.2.2/rocfft-dbgsym-rpath6.2.2_1.0.29.60202-116~20.04_amd64.deb -Size: 9996824 -SHA256: 7ed6a9462ac14fee888aab89875eff51b816a16709501d5a17e91c0916f171ab -SHA1: ec103a2d549b62da3a03f8d68b282b928a2e24f8 -MD5sum: 76cfef9285991236fb95a46a5ca40379 -Description: debug symbols for rocfft -Maintainer: rocfft-maintainer@amd.com -Package-Type: ddeb -Version: 1.0.29.60202-116~20.04 -Installed-Size: 65778 - -Package: rocfft-dbgsym6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 126abc910a9659a1b1b7fe468968d6f0dd685152 a067c448ad8dd97e851d90c68f32c23669b5a430 -Depends: rocfft6.2.2 (= 1.0.29.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocfft-dbgsym6.2.2/rocfft-dbgsym6.2.2_1.0.29.60202-116~20.04_amd64.deb -Size: 9995728 -SHA256: a2bffd651d714121daf34158329d8d95ab69b093c7152adfa0b18aee27cd5248 -SHA1: ffaaf33ef0a78e72026968cece9a9d1468275038 -MD5sum: 3e6dd8a822ca5eb62d77916758d6e525 -Description: debug symbols for rocfft -Maintainer: rocfft-maintainer@amd.com -Package-Type: ddeb -Version: 1.0.29.60202-116~20.04 -Installed-Size: 65778 - -Package: rocfft-dev -Architecture: amd64 -Depends: rocfft (>= 1.0.29.60202) -Priority: optional -Section: devel -Filename: pool/main/r/rocfft-dev/rocfft-dev_1.0.29.60202-116~20.04_amd64.deb -Size: 10490 -SHA256: b4a65b704be3eabddc9c4fbebaf42f29ab4afd43d852b6835a880a9cdc91b16c -SHA1: fb8796477cdc8b6dbbdec839e894b37348a0a88d -MD5sum: 5469ccebedb4fa9bd8a41595529b5b13 -Description: ROCm FFT library -Maintainer: rocfft-maintainer@amd.com -Version: 1.0.29.60202-116~20.04 -Installed-Size: 70 - -Package: rocfft-dev-rpath6.2.2 -Architecture: amd64 -Depends: rocfft-rpath6.2.2 (>= 1.0.29.60202) -Priority: optional -Section: devel -Filename: pool/main/r/rocfft-dev-rpath6.2.2/rocfft-dev-rpath6.2.2_1.0.29.60202-116~20.04_amd64.deb -Size: 10648 -SHA256: fd7879f2b00c34e9314ece7ef6a4d99abf9c45add1b3b86b0c085b413f991d58 -SHA1: d860e5d6de99c280647d67b52055492de85df376 -MD5sum: 35a10bd2fb94978a30c21730f7fc2aba -Description: ROCm FFT library -Maintainer: rocfft-maintainer@amd.com -Version: 1.0.29.60202-116~20.04 -Installed-Size: 70 - -Package: rocfft-dev6.2.2 -Architecture: amd64 -Depends: rocfft6.2.2 (>= 1.0.29.60202) -Priority: optional -Section: devel -Filename: pool/main/r/rocfft-dev6.2.2/rocfft-dev6.2.2_1.0.29.60202-116~20.04_amd64.deb -Size: 10644 -SHA256: d8df1396be031851ce4786ef7e9db8083b1faf8caa8563363c56893af2f074e5 -SHA1: b276158dbe2757b867a3730013539eaac0c40e30 -MD5sum: ee33e3bcc4fc18274bc557f3b0f083f2 -Description: ROCm FFT library -Maintainer: rocfft-maintainer@amd.com -Version: 1.0.29.60202-116~20.04 -Installed-Size: 70 - -Package: rocfft-rpath6.2.2 -Architecture: amd64 -Depends: rocm-core-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocfft-rpath6.2.2/rocfft-rpath6.2.2_1.0.29.60202-116~20.04_amd64.deb -Size: 100613636 -SHA256: dff71adc36128873525c087e18c0d9fa600618aac60c519d31c219fea3eed82e -SHA1: 7b3623214ea9b49ab51843428ebd591cfd4adfab -MD5sum: 584bf36ed3fb5418b89ec6958513e0bb -Description: ROCm FFT library -Maintainer: rocfft-maintainer@amd.com -Recommends: rocfft-dev-rpath6.2.2 (>=1.0.29.60202) -Version: 1.0.29.60202-116~20.04 -Installed-Size: 2083197 - -Package: rocfft6.2.2 -Architecture: amd64 -Depends: rocm-core6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocfft6.2.2/rocfft6.2.2_1.0.29.60202-116~20.04_amd64.deb -Size: 100615608 -SHA256: 262de5e8c9e95f88fa35c22d17468f09b89976e69a50a887eb655aa399a515c7 -SHA1: 5d6526227973600366d8ed4cd62964f07182f5ef -MD5sum: cc23d4b26b1a0dc7820c0de9ec2e5745 -Description: ROCm FFT library -Maintainer: rocfft-maintainer@amd.com -Recommends: rocfft-dev6.2.2 (>=1.0.29.60202) -Version: 1.0.29.60202-116~20.04 -Installed-Size: 2083197 - -Package: rocm -Architecture: amd64 -Depends: rocm-utils (= 6.2.2.60202-116~20.04), rocm-developer-tools (= 6.2.2.60202-116~20.04), rocm-openmp-sdk (= 6.2.2.60202-116~20.04), rocm-opencl-sdk (= 6.2.2.60202-116~20.04), rocm-ml-sdk (= 6.2.2.60202-116~20.04), mivisionx (= 3.0.0.60202-116~20.04), migraphx (= 2.10.0.60202-116~20.04), rpp (= 1.8.0.60202-116~20.04), rocm-core (= 6.2.2.60202-116~20.04), migraphx-dev (= 2.10.0.60202-116~20.04), mivisionx-dev (= 3.0.0.60202-116~20.04), rpp-dev (= 1.8.0.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm/rocm_6.2.2.60202-116~20.04_amd64.deb -Size: 2070 -SHA256: 88f25e6319a83d69a99c01fa9e48f77c87e1f920b883999f37f64e89d3717413 -SHA1: f77845d4d8746136c85315bfc2db1dd097711e5c -MD5sum: 4d2d217e683d0aa90b6f1b53b8594de8 -Description: Radeon Open Compute (ROCm) software stack meta package -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm dev support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-asan -Architecture: amd64 -Depends: rocm (= 6.2.2.60202-116~20.04), rocm-core-asan (= 6.2.2.60202-116~20.04), rocm-developer-tools-asan (= 6.2.2.60202-116~20.04), rocm-language-runtime-asan (= 6.2.2.60202-116~20.04), rocm-opencl-runtime-asan (= 6.2.2.60202-116~20.04), rocm-ml-libraries-asan (= 6.2.2.60202-116~20.04), mivisionx-asan (= 3.0.0.60202-116~20.04), migraphx-asan (= 2.10.0.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-asan/rocm-asan_6.2.2.60202-116~20.04_amd64.deb -Size: 866 -SHA256: 4340e538b16dbd09b104d903d69e5848a8029a22cb9c39524659a0019394c0de -SHA1: e4422ffc30c935117ca3a39994533495c6153474 -MD5sum: d7dece40da9a777ab0e8a0f0a178f70d -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-asan-rpath6.2.2 -Architecture: amd64 -Depends: rocm (= 6.2.2.60202-116~20.04), rocm-core-asan-rpath6.2.2 (= 6.2.2.60202-116~20.04), rocm-developer-tools-asan-rpath6.2.2 (= 6.2.2.60202-116~20.04), rocm-language-runtime-asan-rpath6.2.2 (= 6.2.2.60202-116~20.04), rocm-opencl-runtime-asan-rpath6.2.2 (= 6.2.2.60202-116~20.04), rocm-ml-libraries-asan-rpath6.2.2 (= 6.2.2.60202-116~20.04), mivisionx-asan-rpath6.2.2 (= 3.0.0.60202-116~20.04), migraphx-asan-rpath6.2.2 (= 2.10.0.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-asan-rpath6.2.2/rocm-asan-rpath6.2.2_6.2.2.60202-116~20.04_amd64.deb -Size: 1064 -SHA256: 4f74c49171251721df69fd70296d5414d0a844265e7c6d6ed350536a56f43d99 -SHA1: 54ae89a5b4c13d095f9082314c87c77bbe1148fb -MD5sum: 33c625f293b9d7c082537dfba6571315 -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-asan6.2.2 -Architecture: amd64 -Depends: rocm (= 6.2.2.60202-116~20.04), rocm-core-asan6.2.2 (= 6.2.2.60202-116~20.04), rocm-developer-tools-asan6.2.2 (= 6.2.2.60202-116~20.04), rocm-language-runtime-asan6.2.2 (= 6.2.2.60202-116~20.04), rocm-opencl-runtime-asan6.2.2 (= 6.2.2.60202-116~20.04), rocm-ml-libraries-asan6.2.2 (= 6.2.2.60202-116~20.04), mivisionx-asan6.2.2 (= 3.0.0.60202-116~20.04), migraphx-asan6.2.2 (= 2.10.0.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-asan6.2.2/rocm-asan6.2.2_6.2.2.60202-116~20.04_amd64.deb -Size: 1056 -SHA256: c9cbc95680d8e6f3703ba332ce36950ef19164366ad86b934659c736ef3d9f99 -SHA1: 0423fef60d2e1490309a391139ca42ad621ed264 -MD5sum: 35daeb7a00cc73c9054bbcb5b5aadf2b -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-bandwidth-test -Architecture: amd64 -Depends: libstdc++6, hsa-rocr -Priority: optional -Section: devel -Filename: pool/main/r/rocm-bandwidth-test/rocm-bandwidth-test_1.4.0.60202-116~20.04_amd64.deb -Size: 61130 -SHA256: 557a5448e348c2afbcac0fb245e07ecf14f1c8096acec1b585cc51a55c2ae536 -SHA1: 6e4b62169261c77482bffb7690d16aeb0df311d7 -MD5sum: 37ed3d1f3fd97e2a2d0616c61b91cd1d -Description: Diagnostic utility tool to measure PCIe bandwidth on ROCm platforms -Homepage: https://github.com/RadeonOpenCompute/rocm_bandwidth_test -Maintainer: TODO -Version: 1.4.0.60202-116~20.04 -Installed-Size: 177 - -Package: rocm-bandwidth-test-rpath6.2.2 -Architecture: amd64 -Depends: libstdc++6, hsa-rocr-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocm-bandwidth-test-rpath6.2.2/rocm-bandwidth-test-rpath6.2.2_1.4.0.60202-116~20.04_amd64.deb -Size: 49716 -SHA256: e8a944b01979019839ad50da4dfce048852f883070ffb9e44b1e60a2344354c4 -SHA1: 025354cffd40b920c156b99d09c791eebcbba465 -MD5sum: aa4963ea17ccb8772164d7238fcb4f73 -Description: Diagnostic utility tool to measure PCIe bandwidth on ROCm platforms -Homepage: https://github.com/RadeonOpenCompute/rocm_bandwidth_test -Maintainer: TODO -Version: 1.4.0.60202-116~20.04 -Installed-Size: 177 - -Package: rocm-bandwidth-test6.2.2 -Architecture: amd64 -Depends: libstdc++6, hsa-rocr6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocm-bandwidth-test6.2.2/rocm-bandwidth-test6.2.2_1.4.0.60202-116~20.04_amd64.deb -Size: 49676 -SHA256: c3c6ea3ba0dc81e4d5c9c9f958840497f5431e57a53f376fbb22d24766666c33 -SHA1: 2fd1b0b7cbb3261bb7c5983eb3f0fbd30bb2535f -MD5sum: 9f830634d0e84161e67d1b36d50d4f22 -Description: Diagnostic utility tool to measure PCIe bandwidth on ROCm platforms -Homepage: https://github.com/RadeonOpenCompute/rocm_bandwidth_test -Maintainer: TODO -Version: 1.4.0.60202-116~20.04 -Installed-Size: 177 - -Package: rocm-cmake -Architecture: amd64 -Depends: rocm-core -Priority: optional -Section: devel -Filename: pool/main/r/rocm-cmake/rocm-cmake_0.13.0.60202-116~20.04_amd64.deb -Size: 24624 -SHA256: 31207fc63787c40320d8b3688a39d21c3566bf5bcde69e1772cabc8995caac1d -SHA1: 62206dbb5c2a54fa9483f9cc394a51662947ee89 -MD5sum: 5a35a11be0535a0cc9fe821b55b1ea0a -Description: rocm-cmake built using CMake -Maintainer: Paul Fultz II pfultz@amd.com -Version: 0.13.0.60202-116~20.04 -Installed-Size: 157 - -Package: rocm-cmake-rpath6.2.2 -Architecture: amd64 -Depends: rocm-core-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocm-cmake-rpath6.2.2/rocm-cmake-rpath6.2.2_0.13.0.60202-116~20.04_amd64.deb -Size: 24744 -SHA256: 641c636e610f6cb3539387c69ac06bd91254c5bd4e95c02c3dc7882c4f8988f6 -SHA1: aa8b284dc66bd452e74934831d9776c68602e49d -MD5sum: b0ad646508ffe470267af5f220cf5cf8 -Description: rocm-cmake built using CMake -Maintainer: Paul Fultz II pfultz@amd.com -Version: 0.13.0.60202-116~20.04 -Installed-Size: 157 - -Package: rocm-cmake6.2.2 -Architecture: amd64 -Depends: rocm-core6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocm-cmake6.2.2/rocm-cmake6.2.2_0.13.0.60202-116~20.04_amd64.deb -Size: 24748 -SHA256: 6dea74b5449f98e8eb68bd0600fe3091147f0c9b5a24d4173550944879204ba5 -SHA1: 6249f511fea69e2392e438462a32319ac48bc42c -MD5sum: e3ed4a0bd04d8a1a5235a19d92810ada -Description: rocm-cmake built using CMake -Maintainer: Paul Fultz II pfultz@amd.com -Version: 0.13.0.60202-116~20.04 -Installed-Size: 157 - -Package: rocm-core -Architecture: amd64 -Priority: optional -Section: devel -Filename: pool/main/r/rocm-core/rocm-core_6.2.2.60202-116~20.04_amd64.deb -Size: 10732 -SHA256: 0eb844163e0f1a7f71c593ff3331a6c4cc6bae2d2ca775802d2ad79fcf7f10e2 -SHA1: 86ebd4bca8b9126b0d5aaa3ed3b47ca8197e4f78 -MD5sum: 7aa94008ad13576cd9b10b33f738b4c3 -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 110 - -Package: rocm-core-asan -Architecture: amd64 -Depends: rocm-core -Priority: optional -Section: devel -Filename: pool/main/r/rocm-core-asan/rocm-core-asan_6.2.2.60202-116~20.04_amd64.deb -Size: 22902 -SHA256: 4ed319063477ddec5b5a9da08f43bbe5a7d346f84c31ae4ee2f893fab2efb139 -SHA1: 151bc2c786bd92d9c42c40f988b371759f3ea4ec -MD5sum: b89fe9ae200ac8d5b2f6a28838ca1a6c -Description: Radeon Open Compute (ROCm) AddressSanitizer Instrumented Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 263 - -Package: rocm-core-asan-rpath6.2.2 -Architecture: amd64 -Depends: rocm-core -Priority: optional -Section: devel -Filename: pool/main/r/rocm-core-asan-rpath6.2.2/rocm-core-asan-rpath6.2.2_6.2.2.60202-116~20.04_amd64.deb -Size: 16596 -SHA256: 31cbe5391dc0b1561dfd8ca05af376bc028062b4c51b557ea6cd655ce5b8080b -SHA1: 40d745c69f335fc066b7f321be96c1b33565d50b -MD5sum: 180c5856230a44807cc70ccd280e8c5c -Description: Radeon Open Compute (ROCm) AddressSanitizer Instrumented Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 263 - -Package: rocm-core-asan6.2.2 -Architecture: amd64 -Depends: rocm-core -Priority: optional -Section: devel -Filename: pool/main/r/rocm-core-asan6.2.2/rocm-core-asan6.2.2_6.2.2.60202-116~20.04_amd64.deb -Size: 16528 -SHA256: f1753e41837855bfc23696e85e49a0709a59d74d7f0343e737cc5c9244939f73 -SHA1: 4b23898ae69da7008472d89434b729a0c54a9b51 -MD5sum: 65d604fa37c56515fe4fd6ecfb60a397 -Description: Radeon Open Compute (ROCm) AddressSanitizer Instrumented Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 263 - -Package: rocm-core-rpath6.2.2 -Architecture: amd64 -Priority: optional -Section: devel -Filename: pool/main/r/rocm-core-rpath6.2.2/rocm-core-rpath6.2.2_6.2.2.60202-116~20.04_amd64.deb -Size: 9800 -SHA256: bb1ad1b2b589a9f4023e244f9644f8a67d18963d84f2673a00e623d16fea7212 -SHA1: 2af716b5a92ce5a6a679d67d273645217817bf50 -MD5sum: 475ad31680dbe7d3ec672e460499f8d5 -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 110 - -Package: rocm-core6.2.2 -Architecture: amd64 -Priority: optional -Section: devel -Filename: pool/main/r/rocm-core6.2.2/rocm-core6.2.2_6.2.2.60202-116~20.04_amd64.deb -Size: 9796 -SHA256: 4a58d1630f8afa5a43b4f0d3e10202b45e3d3230d8bddbce7a10906ba5e5584a -SHA1: 18fd6d66028d086bf049df5e3be8bcc277061787 -MD5sum: 08ed3233fd2ee39d51685d2a494ed4b4 -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 110 - -Package: rocm-dbgapi -Architecture: amd64 -Depends: comgr(>=1.2.0), rocm-core, pci.ids -Priority: optional -Section: devel -Filename: pool/main/r/rocm-dbgapi/rocm-dbgapi_0.76.0.60202-116~20.04_amd64.deb -Size: 1730956 -SHA256: 9231e0ddf9118057cac4456dfc10219c3bd9058f44ff883b5c92d9cb14c183d5 -SHA1: 88829100ea95063ba0a757bfe7fb5fe7ef23bc98 -MD5sum: 0c18badb7a280ca614c5c98269dfdfc8 -Description: Library to provide AMD GPU debugger API - Non ASAN libraries, include files for the ROCM-DBGAPI -Maintainer: ROCm Debugger Support -Version: 0.76.0.60202-116~20.04 -Installed-Size: 7367 - -Package: rocm-dbgapi-asan -Architecture: amd64 -Depends: comgr-asan(>=1.2.0), rocm-core-asan, pci.ids -Priority: optional -Section: devel -Filename: pool/main/r/rocm-dbgapi-asan/rocm-dbgapi-asan_0.76.0.60202-116~20.04_amd64.deb -Size: 754514 -SHA256: e61c7603d321f85e63ce06871132d3beeef8c75efc4321d6d3f17ba0ad6eb5d7 -SHA1: f1b3d5037f6a37f13c7bcb2ad2a85b234b7593a8 -MD5sum: 9abb8821540d851fae7134c0d6167093 -Description: Library to provide AMD GPU debugger API - ASAN libraries for the ROCM-DBGAPI -Maintainer: ROCm Debugger Support -Version: 0.76.0.60202-116~20.04 -Installed-Size: 7901 - -Package: rocm-dbgapi-asan-dbgsym -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: fa60551eb22cd4746235981d2084d160d542b299 -Depends: rocm-dbgapi-asan (= 0.76.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocm-dbgapi-asan-dbgsym/rocm-dbgapi-asan-dbgsym_0.76.0.60202-116~20.04_amd64.deb -Size: 4740596 -SHA256: 2a17bdb5fb803dd69e46eecf839daa30546ccb3709eafea6417436d1c86b2c6d -SHA1: 163abd48df4f3f7882d2b39e0d23e0abfcafcf51 -MD5sum: a37b29d96e6cf44a8114a24385dc7379 -Description: debug symbols for rocm-dbgapi-asan -Maintainer: ROCm Debugger Support -Package-Type: ddeb -Version: 0.76.0.60202-116~20.04 -Installed-Size: 18280 - -Package: rocm-dbgapi-asan-dbgsym-rpath6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: fa60551eb22cd4746235981d2084d160d542b299 -Depends: rocm-dbgapi-asan-rpath6.2.2 (= 0.76.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocm-dbgapi-asan-dbgsym-rpath6.2.2/rocm-dbgapi-asan-dbgsym-rpath6.2.2_0.76.0.60202-116~20.04_amd64.deb -Size: 2808484 -SHA256: d389a061f6e13b1adb60e034b2f47c22085354b88f47ae39ffb5b686786437a5 -SHA1: 96a3b59fce20b1239cd33cf4d6617ce80b899c25 -MD5sum: f03a7856ca47245ffa4d9973824e4a11 -Description: debug symbols for rocm-dbgapi-asan -Maintainer: ROCm Debugger Support -Package-Type: ddeb -Version: 0.76.0.60202-116~20.04 -Installed-Size: 18280 - -Package: rocm-dbgapi-asan-dbgsym6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: fa60551eb22cd4746235981d2084d160d542b299 -Depends: rocm-dbgapi-asan6.2.2 (= 0.76.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocm-dbgapi-asan-dbgsym6.2.2/rocm-dbgapi-asan-dbgsym6.2.2_0.76.0.60202-116~20.04_amd64.deb -Size: 2807632 -SHA256: f5d6e8337b2c815a000a5e2c0dcb4405dc5e106616b5f77ac92d5c88124a51b6 -SHA1: e08bcdcf5ac570157b76c7a478769dd3f361a57d -MD5sum: 24ea4ba78ea8c72f5a5c1297848a0242 -Description: debug symbols for rocm-dbgapi-asan -Maintainer: ROCm Debugger Support -Package-Type: ddeb -Version: 0.76.0.60202-116~20.04 -Installed-Size: 18280 - -Package: rocm-dbgapi-asan-rpath6.2.2 -Architecture: amd64 -Depends: comgr-asan-rpath6.2.2(>=1.2.0), rocm-core-asan-rpath6.2.2, pci.ids -Priority: optional -Section: devel -Filename: pool/main/r/rocm-dbgapi-asan-rpath6.2.2/rocm-dbgapi-asan-rpath6.2.2_0.76.0.60202-116~20.04_amd64.deb -Size: 451744 -SHA256: 7245e2e81f2d02c4ea0baa21495121e47caff9c5ea6b104d9ceb6ee0daf5cfd4 -SHA1: a4f555c6c52ae8f53def1a77199114383c9a8366 -MD5sum: 50234f5192f33ee92c3d5a17848ee070 -Description: Library to provide AMD GPU debugger API - ASAN libraries for the ROCM-DBGAPI -Maintainer: ROCm Debugger Support -Version: 0.76.0.60202-116~20.04 -Installed-Size: 7901 - -Package: rocm-dbgapi-asan6.2.2 -Architecture: amd64 -Depends: comgr-asan6.2.2(>=1.2.0), rocm-core-asan6.2.2, pci.ids -Priority: optional -Section: devel -Filename: pool/main/r/rocm-dbgapi-asan6.2.2/rocm-dbgapi-asan6.2.2_0.76.0.60202-116~20.04_amd64.deb -Size: 451668 -SHA256: d75156af306fa5d357b2d62a13cf74a3853f111c54b65753a15bb74fbd46d7cc -SHA1: 08a372020dc08f26c95ab3a574c399fffb7e97c1 -MD5sum: 6227ed8c260e94fa6a8f8b8e61904559 -Description: Library to provide AMD GPU debugger API - ASAN libraries for the ROCM-DBGAPI -Maintainer: ROCm Debugger Support -Version: 0.76.0.60202-116~20.04 -Installed-Size: 7901 - -Package: rocm-dbgapi-dbgsym -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: b50eb0c252172d4e19e0f058fe3ae57d359a9865 -Depends: rocm-dbgapi (= 0.76.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocm-dbgapi-dbgsym/rocm-dbgapi-dbgsym_0.76.0.60202-116~20.04_amd64.deb -Size: 11620730 -SHA256: d0d0fc1387f071adf73bd2b74ff0c45f2ebb78c203c5a90c74b78653d1f74920 -SHA1: 032b8d1106a5018b12c12715bbb7a0c8677d5e5b -MD5sum: db33aba58df29027cf270706a1a0ec08 -Description: debug symbols for rocm-dbgapi -Maintainer: ROCm Debugger Support -Package-Type: ddeb -Version: 0.76.0.60202-116~20.04 -Installed-Size: 47932 - -Package: rocm-dbgapi-dbgsym-rpath6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: b50eb0c252172d4e19e0f058fe3ae57d359a9865 -Depends: rocm-dbgapi-rpath6.2.2 (= 0.76.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocm-dbgapi-dbgsym-rpath6.2.2/rocm-dbgapi-dbgsym-rpath6.2.2_0.76.0.60202-116~20.04_amd64.deb -Size: 7260420 -SHA256: bdfb25f0f6ac51b12373c071d289f981537b9224784280a8586a8b1e23869584 -SHA1: 509d55c44753e191a4e566fc2c33850b8982a6c5 -MD5sum: 36b0107af8fa5fbe307f6684a2ff1664 -Description: debug symbols for rocm-dbgapi -Maintainer: ROCm Debugger Support -Package-Type: ddeb -Version: 0.76.0.60202-116~20.04 -Installed-Size: 47932 - -Package: rocm-dbgapi-dbgsym6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: b50eb0c252172d4e19e0f058fe3ae57d359a9865 -Depends: rocm-dbgapi6.2.2 (= 0.76.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocm-dbgapi-dbgsym6.2.2/rocm-dbgapi-dbgsym6.2.2_0.76.0.60202-116~20.04_amd64.deb -Size: 7260416 -SHA256: 27b57193f687a794dc9144a6549f34ce85ce742422f7cde4122e4bedb1320468 -SHA1: f9ff724f74ac749a83ac6fa70f5eb70b4e048269 -MD5sum: 0a0ed14c9c3a02fb09b551d8ce803dff -Description: debug symbols for rocm-dbgapi -Maintainer: ROCm Debugger Support -Package-Type: ddeb -Version: 0.76.0.60202-116~20.04 -Installed-Size: 47932 - -Package: rocm-dbgapi-rpath6.2.2 -Architecture: amd64 -Depends: comgr-rpath6.2.2(>=1.2.0), rocm-core-rpath6.2.2, pci.ids -Priority: optional -Section: devel -Filename: pool/main/r/rocm-dbgapi-rpath6.2.2/rocm-dbgapi-rpath6.2.2_0.76.0.60202-116~20.04_amd64.deb -Size: 1350048 -SHA256: bfecda202682b6be0264da4afa356463587a66a763ee7f6d10559f7a0fcf181d -SHA1: 1db9067605f4d937136a50b9a0a30e058c042520 -MD5sum: 50e780f3e6b0dfa2dc97701038b8af86 -Description: Library to provide AMD GPU debugger API - Non ASAN libraries, include files for the ROCM-DBGAPI -Maintainer: ROCm Debugger Support -Version: 0.76.0.60202-116~20.04 -Installed-Size: 7367 - -Package: rocm-dbgapi6.2.2 -Architecture: amd64 -Depends: comgr6.2.2(>=1.2.0), rocm-core6.2.2, pci.ids -Priority: optional -Section: devel -Filename: pool/main/r/rocm-dbgapi6.2.2/rocm-dbgapi6.2.2_0.76.0.60202-116~20.04_amd64.deb -Size: 1350292 -SHA256: 60e9695c0a910d3175f3920a69f7cd0efdb9b1d201631b0e5d2a18a3176c4ee5 -SHA1: e497327610aefef8aad235b6c837e928d21a1d23 -MD5sum: 80bf366d2f98db548560d9c8e29d4cc7 -Description: Library to provide AMD GPU debugger API - Non ASAN libraries, include files for the ROCM-DBGAPI -Maintainer: ROCm Debugger Support -Version: 0.76.0.60202-116~20.04 -Installed-Size: 7367 - -Package: rocm-debug-agent -Architecture: amd64 -Depends: rocm-dbgapi, rocm-core -Priority: optional -Section: devel -Filename: pool/main/r/rocm-debug-agent/rocm-debug-agent_2.0.3.60202-116~20.04_amd64.deb -Size: 54122 -SHA256: b420823cae5049a6325c3e14278cdef663096fb4fc9a9dd68058297ec2582aa3 -SHA1: 27cef8cf8ed9e9d7613b82913711b88a7a52269f -MD5sum: 095e473b2bfc2a33235284eff3a56c97 -Description: Radeon Open Compute Debug Agent (ROCdebug-agent) - Dynamic libraries for the ROCdebug-agent -Maintainer: ROCm Debugger Support -Version: 2.0.3.60202-116~20.04 -Installed-Size: 263 - -Package: rocm-debug-agent-asan -Architecture: amd64 -Depends: rocm-dbgapi-asan, rocm-core-asan -Priority: optional -Section: devel -Filename: pool/main/r/rocm-debug-agent-asan/rocm-debug-agent-asan_2.0.3.60202-116~20.04_amd64.deb -Size: 88596 -SHA256: b165f819a19576dfd453ed2f88759aaa55d4c80d96db30f285879e8a5d550c83 -SHA1: 5c0e47f1e1ebb889a22d862291fdb1be8637cc68 -MD5sum: efc9c7c553c43af30f6dba389f2c3468 -Description: Radeon Open Compute Debug Agent (ROCdebug-agent) - AddressSanitizer libraries for the ROCdebug-agent -Maintainer: ROCm Debugger Support -Version: 2.0.3.60202-116~20.04 -Installed-Size: 499 - -Package: rocm-debug-agent-asan-dbgsym -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: dc000cdfcbd0c98135ae29385a2869a5aec33714 -Depends: rocm-debug-agent-asan (= 2.0.3.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocm-debug-agent-asan-dbgsym/rocm-debug-agent-asan-dbgsym_2.0.3.60202-116~20.04_amd64.deb -Size: 439176 -SHA256: c3786755669e17127f417203fa026357134c27728d20e9f4ef3e2094e1e7c53f -SHA1: 9cdc7d3dc84e079fc0e7c053d417608c3a434dc7 -MD5sum: 0010b1963b2e18d9c6366c7412a75d0b -Description: debug symbols for rocm-debug-agent-asan -Maintainer: ROCm Debugger Support -Package-Type: ddeb -Version: 2.0.3.60202-116~20.04 -Installed-Size: 1436 - -Package: rocm-debug-agent-asan-dbgsym-rpath6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: dc000cdfcbd0c98135ae29385a2869a5aec33714 -Depends: rocm-debug-agent-asan-rpath6.2.2 (= 2.0.3.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocm-debug-agent-asan-dbgsym-rpath6.2.2/rocm-debug-agent-asan-dbgsym-rpath6.2.2_2.0.3.60202-116~20.04_amd64.deb -Size: 325780 -SHA256: cff63facaaa4b6e3e8dad7479fbfdfc105a3577ce9ca660c83dfbce499ca1df2 -SHA1: 61fdac7582800ffb3a26321cfc03b349f64a5762 -MD5sum: b74bf3307af3f90401ae5bc66e49e1ec -Description: debug symbols for rocm-debug-agent-asan -Maintainer: ROCm Debugger Support -Package-Type: ddeb -Version: 2.0.3.60202-116~20.04 -Installed-Size: 1436 - -Package: rocm-debug-agent-asan-dbgsym6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: dc000cdfcbd0c98135ae29385a2869a5aec33714 -Depends: rocm-debug-agent-asan6.2.2 (= 2.0.3.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocm-debug-agent-asan-dbgsym6.2.2/rocm-debug-agent-asan-dbgsym6.2.2_2.0.3.60202-116~20.04_amd64.deb -Size: 325700 -SHA256: 30d076ee19b796bfab49b59a065b8bc002fdf6cbb892117686d546bfbf19bdac -SHA1: fdcc170e2ee6e953fc0832b38372aaa91070c285 -MD5sum: 87caf76e486a14bf5113df9b666234ab -Description: debug symbols for rocm-debug-agent-asan -Maintainer: ROCm Debugger Support -Package-Type: ddeb -Version: 2.0.3.60202-116~20.04 -Installed-Size: 1436 - -Package: rocm-debug-agent-asan-rpath6.2.2 -Architecture: amd64 -Depends: rocm-dbgapi-asan-rpath6.2.2, rocm-core-asan-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocm-debug-agent-asan-rpath6.2.2/rocm-debug-agent-asan-rpath6.2.2_2.0.3.60202-116~20.04_amd64.deb -Size: 71076 -SHA256: 995220681abcfc0801cd67bbea02e48e1136ed9be658fa64375e3ceeb7af232f -SHA1: 1a3a289f8be8561173b6f08e01b5eb8f8e816710 -MD5sum: 654e37d0b84582ebeb87979137279722 -Description: Radeon Open Compute Debug Agent (ROCdebug-agent) - AddressSanitizer libraries for the ROCdebug-agent -Maintainer: ROCm Debugger Support -Version: 2.0.3.60202-116~20.04 -Installed-Size: 499 - -Package: rocm-debug-agent-asan6.2.2 -Architecture: amd64 -Depends: rocm-dbgapi-asan6.2.2, rocm-core-asan6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocm-debug-agent-asan6.2.2/rocm-debug-agent-asan6.2.2_2.0.3.60202-116~20.04_amd64.deb -Size: 71012 -SHA256: 5d9319d012c016c445895dd6a253bfbfa09d8c9760a4cac72100b1c4208f3c6c -SHA1: d78fd52634c1f6fb1f61f064144b519a968b164a -MD5sum: c4f673c5f2f6c59c62e2d24de95f17bd -Description: Radeon Open Compute Debug Agent (ROCdebug-agent) - AddressSanitizer libraries for the ROCdebug-agent -Maintainer: ROCm Debugger Support -Version: 2.0.3.60202-116~20.04 -Installed-Size: 499 - -Package: rocm-debug-agent-dbgsym -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 5559ca3618ae1b760af3f6aa17f447a6647d8b43 -Depends: rocm-debug-agent (= 2.0.3.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocm-debug-agent-dbgsym/rocm-debug-agent-dbgsym_2.0.3.60202-116~20.04_amd64.deb -Size: 706068 -SHA256: 473ce424f91b90790e5bbaa65c3792a68399d0e296e570eaa02597e97cc0f83e -SHA1: 91403aa58c56378961ee283d89b7cb39755fd4c6 -MD5sum: 0a57b766628521cfc9d6981171f86fda -Description: debug symbols for rocm-debug-agent -Maintainer: ROCm Debugger Support -Package-Type: ddeb -Version: 2.0.3.60202-116~20.04 -Installed-Size: 2552 - -Package: rocm-debug-agent-dbgsym-rpath6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 5559ca3618ae1b760af3f6aa17f447a6647d8b43 -Depends: rocm-debug-agent-rpath6.2.2 (= 2.0.3.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocm-debug-agent-dbgsym-rpath6.2.2/rocm-debug-agent-dbgsym-rpath6.2.2_2.0.3.60202-116~20.04_amd64.deb -Size: 530004 -SHA256: cba481b6e7fa1144c9cce98769ad22f7d4a6f862cb7270a1141ef8ec7dae85e6 -SHA1: b56c83e608f8917e905937ee696850ccf28e0df8 -MD5sum: 49b669f561096fdf0be3aaa1c94af7af -Description: debug symbols for rocm-debug-agent -Maintainer: ROCm Debugger Support -Package-Type: ddeb -Version: 2.0.3.60202-116~20.04 -Installed-Size: 2552 - -Package: rocm-debug-agent-dbgsym6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 5559ca3618ae1b760af3f6aa17f447a6647d8b43 -Depends: rocm-debug-agent6.2.2 (= 2.0.3.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocm-debug-agent-dbgsym6.2.2/rocm-debug-agent-dbgsym6.2.2_2.0.3.60202-116~20.04_amd64.deb -Size: 530012 -SHA256: 1597d205dc3acf756a8bfba914ca5cd3c28f4a7af318a49bb8d525c9e3a7428e -SHA1: 921edc3de263f6eba4eb98aec0e39e480cf8de13 -MD5sum: 186b3e22ecc231fafd0d24413068fa41 -Description: debug symbols for rocm-debug-agent -Maintainer: ROCm Debugger Support -Package-Type: ddeb -Version: 2.0.3.60202-116~20.04 -Installed-Size: 2552 - -Package: rocm-debug-agent-rpath6.2.2 -Architecture: amd64 -Depends: rocm-dbgapi-rpath6.2.2, rocm-core-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocm-debug-agent-rpath6.2.2/rocm-debug-agent-rpath6.2.2_2.0.3.60202-116~20.04_amd64.deb -Size: 45100 -SHA256: 801a1997e6f40699ef0692bbcf7788fd8ca7375d6728fd0e21369b57dbcfcf41 -SHA1: 9249230a0cf7af3b3f83a6c9efff2174d12bff65 -MD5sum: 980019d403e10e8a19cb2e308b17a248 -Description: Radeon Open Compute Debug Agent (ROCdebug-agent) - Dynamic libraries for the ROCdebug-agent -Maintainer: ROCm Debugger Support -Version: 2.0.3.60202-116~20.04 -Installed-Size: 263 - -Package: rocm-debug-agent6.2.2 -Architecture: amd64 -Depends: rocm-dbgapi6.2.2, rocm-core6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocm-debug-agent6.2.2/rocm-debug-agent6.2.2_2.0.3.60202-116~20.04_amd64.deb -Size: 45088 -SHA256: 2dade24867c82278adef95e8e628b2574dc4298c85b2653d2b39439cc566c74e -SHA1: 7aea5bfd3b0cee3e29f04ad235f1383f9bb55985 -MD5sum: 6cc8cd6e664e97bbab1f70d884eee106 -Description: Radeon Open Compute Debug Agent (ROCdebug-agent) - Dynamic libraries for the ROCdebug-agent -Maintainer: ROCm Debugger Support -Version: 2.0.3.60202-116~20.04 -Installed-Size: 263 - -Package: rocm-dev -Architecture: amd64 -Depends: amd-smi-lib (= 24.6.3.60202-116~20.04), comgr (= 2.8.0.60202-116~20.04), hip-doc (= 6.2.41134.60202-116~20.04), hipify-clang (= 18.0.0.60202-116~20.04), hip-runtime-amd (= 6.2.41134.60202-116~20.04), hip-samples (= 6.2.41134.60202-116~20.04), hipcc (= 1.1.1.60202-116~20.04), hsa-rocr (= 1.14.0.60202-116~20.04), hsa-amd-aqlprofile (= 1.0.0.60202.60202-116~20.04), rocm-llvm (= 18.0.0.24355.60202-116~20.04), openmp-extras-runtime (= 18.62.0.60202-116~20.04), rocm-cmake (= 0.13.0.60202-116~20.04), rocm-dbgapi (= 0.76.0.60202-116~20.04), rocm-debug-agent (= 2.0.3.60202-116~20.04), rocm-device-libs (= 1.0.0.60202-116~20.04), rocm-gdb (= 14.2.60202-116~20.04), rocm-smi-lib (= 7.3.0.60202-116~20.04), rocm-utils (= 6.2.2.60202-116~20.04), rocm-core (= 6.2.2.60202-116~20.04), rocm-opencl (= 2.0.0.60202-116~20.04), rocm-opencl-icd-loader (= 1.2.60202-116~20.04), rocprofiler-register (= 0.4.0.60202-116~20.04), rocprofiler (= 2.0.60202.60202-116~20.04), rocprofiler-plugins (= 2.0.60202.60202-116~20.04), roctracer (= 4.1.60202.60202-116~20.04), rocprofiler-sdk (= 0.4.0-116~20.04), rocprofiler-sdk-roctx (= 0.4.0-116~20.04), hip-dev (= 6.2.41134.60202-116~20.04), hsa-rocr-dev (= 1.14.0.60202-116~20.04), hsakmt-roct-dev (= 20240607.4.05.60202-116~20.04), rocprofiler-register (= 0.4.0.60202-116~20.04), rocprofiler-dev (= 2.0.60202.60202-116~20.04), roctracer-dev (= 4.1.60202.60202-116~20.04), openmp-extras-dev (= 18.62.0.60202-116~20.04), rocm-opencl-dev (= 2.0.0.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-dev/rocm-dev_6.2.2.60202-116~20.04_amd64.deb -Size: 2482 -SHA256: 80ad1f6e43216b3eee65fcca9fddc29602dee6c9c57e0308cbb9549aceb149ce -SHA1: dc49a241679eba21cbf109908a0eff1eb821fe99 -MD5sum: cba5be5a642167dd5306f41fc287218c -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-dev-asan -Architecture: amd64 -Depends: rocm-dev (= 6.2.2.60202-116~20.04), comgr-asan (= 2.8.0.60202-116~20.04), hip-runtime-amd-asan (= 6.2.41134.60202-116~20.04), hsa-rocr-asan (= 1.14.0.60202-116~20.04), hsa-amd-aqlprofile-asan (= 1.0.0.60202.60202-116~20.04), openmp-extras-asan (= 18.62.0.60202-116~20.04), rocm-dbgapi-asan (= 0.76.0.60202-116~20.04), rocm-debug-agent-asan (= 2.0.3.60202-116~20.04), rocm-smi-lib-asan (= 7.3.0.60202-116~20.04), rocm-core-asan (= 6.2.2.60202-116~20.04), rocm-opencl-asan (= 2.0.0.60202-116~20.04), rocprofiler-asan (= 2.0.60202.60202-116~20.04), roctracer-asan (= 4.1.60202.60202-116~20.04), hsakmt-roct-asan (= 20240607.4.05.60202-116~20.04), amd-smi-lib-asan (= 24.6.3.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-dev-asan/rocm-dev-asan_6.2.2.60202-116~20.04_amd64.deb -Size: 974 -SHA256: 55343f5dee7bb35785b35ce84d04c840043d2b85ab5df1917ef823f7ba4040dc -SHA1: 9c1e2431689e6dda3c9d62ac369a4d3f43220156 -MD5sum: 9c75d0f2864b39232a8be90de2a870ac -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-dev-asan-rpath6.2.2 -Architecture: amd64 -Depends: rocm-dev (= 6.2.2.60202-116~20.04), comgr-asan-rpath6.2.2 (= 2.8.0.60202-116~20.04), hip-runtime-amd-asan-rpath6.2.2 (= 6.2.41134.60202-116~20.04), hsa-rocr-asan-rpath6.2.2 (= 1.14.0.60202-116~20.04), hsa-amd-aqlprofile-asan-rpath6.2.2 (= 1.0.0.60202.60202-116~20.04), openmp-extras-asan-rpath6.2.2 (= 18.62.0.60202-116~20.04), rocm-dbgapi-asan-rpath6.2.2 (= 0.76.0.60202-116~20.04), rocm-debug-agent-asan-rpath6.2.2 (= 2.0.3.60202-116~20.04), rocm-smi-lib-asan-rpath6.2.2 (= 7.3.0.60202-116~20.04), rocm-core-asan-rpath6.2.2 (= 6.2.2.60202-116~20.04), rocm-opencl-asan-rpath6.2.2 (= 2.0.0.60202-116~20.04), rocprofiler-asan-rpath6.2.2 (= 2.0.60202.60202-116~20.04), roctracer-asan-rpath6.2.2 (= 4.1.60202.60202-116~20.04), hsakmt-roct-asan-rpath6.2.2 (= 20240607.4.05.60202-116~20.04), amd-smi-lib-asan-rpath6.2.2 (= 24.6.3.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-dev-asan-rpath6.2.2/rocm-dev-asan-rpath6.2.2_6.2.2.60202-116~20.04_amd64.deb -Size: 1180 -SHA256: 38f520b676dc3a16baad5ee5bd7e7a12bd9e566029f83eb2190edde5a9c8ce14 -SHA1: 4b4588b272673293cc79bc17fe1abd840b9ee349 -MD5sum: 667949a874117d6fd2c970f9593c0709 -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-dev-asan6.2.2 -Architecture: amd64 -Depends: rocm-dev (= 6.2.2.60202-116~20.04), comgr-asan6.2.2 (= 2.8.0.60202-116~20.04), hip-runtime-amd-asan6.2.2 (= 6.2.41134.60202-116~20.04), hsa-rocr-asan6.2.2 (= 1.14.0.60202-116~20.04), hsa-amd-aqlprofile-asan6.2.2 (= 1.0.0.60202.60202-116~20.04), openmp-extras-asan6.2.2 (= 18.62.0.60202-116~20.04), rocm-dbgapi-asan6.2.2 (= 0.76.0.60202-116~20.04), rocm-debug-agent-asan6.2.2 (= 2.0.3.60202-116~20.04), rocm-smi-lib-asan6.2.2 (= 7.3.0.60202-116~20.04), rocm-core-asan6.2.2 (= 6.2.2.60202-116~20.04), rocm-opencl-asan6.2.2 (= 2.0.0.60202-116~20.04), rocprofiler-asan6.2.2 (= 2.0.60202.60202-116~20.04), roctracer-asan6.2.2 (= 4.1.60202.60202-116~20.04), hsakmt-roct-asan6.2.2 (= 20240607.4.05.60202-116~20.04), amd-smi-lib-asan6.2.2 (= 24.6.3.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-dev-asan6.2.2/rocm-dev-asan6.2.2_6.2.2.60202-116~20.04_amd64.deb -Size: 1172 -SHA256: d3db3bbb321b42974786040c70af9e9cef00fd0aa882d1c27d32c4797eb0f65a -SHA1: 61f0b8397dfd0ef37f1f7e67dd085fe1b63cb881 -MD5sum: d8bf2fb8f31dda83b25511ae6cc0c8f0 -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-dev-rpath6.2.2 -Architecture: amd64 -Depends: amd-smi-lib-rpath6.2.2 (= 24.6.3.60202-116~20.04), comgr-rpath6.2.2 (= 2.8.0.60202-116~20.04), hip-doc-rpath6.2.2 (= 6.2.41134.60202-116~20.04), hipify-clang-rpath6.2.2 (= 18.0.0.60202-116~20.04), hip-runtime-amd-rpath6.2.2 (= 6.2.41134.60202-116~20.04), hip-samples-rpath6.2.2 (= 6.2.41134.60202-116~20.04), hipcc-rpath6.2.2 (= 1.1.1.60202-116~20.04), hsa-rocr-rpath6.2.2 (= 1.14.0.60202-116~20.04), hsa-amd-aqlprofile-rpath6.2.2 (= 1.0.0.60202.60202-116~20.04), rocm-llvm-rpath6.2.2 (= 18.0.0.24355.60202-116~20.04), openmp-extras-runtime-rpath6.2.2 (= 18.62.0.60202-116~20.04), rocm-cmake-rpath6.2.2 (= 0.13.0.60202-116~20.04), rocm-dbgapi-rpath6.2.2 (= 0.76.0.60202-116~20.04), rocm-debug-agent-rpath6.2.2 (= 2.0.3.60202-116~20.04), rocm-device-libs-rpath6.2.2 (= 1.0.0.60202-116~20.04), rocm-gdb-rpath6.2.2 (= 14.2.60202-116~20.04), rocm-smi-lib-rpath6.2.2 (= 7.3.0.60202-116~20.04), rocm-utils-rpath6.2.2 (= 6.2.2.60202-116~20.04), rocm-core-rpath6.2.2 (= 6.2.2.60202-116~20.04), rocm-opencl-rpath6.2.2 (= 2.0.0.60202-116~20.04), rocm-opencl-icd-loader-rpath6.2.2 (= 1.2.60202-116~20.04), rocprofiler-register-rpath6.2.2 (= 0.4.0.60202-116~20.04), rocprofiler-rpath6.2.2 (= 2.0.60202.60202-116~20.04), rocprofiler-plugins-rpath6.2.2 (= 2.0.60202.60202-116~20.04), roctracer-rpath6.2.2 (= 4.1.60202.60202-116~20.04), rocprofiler-sdk-rpath6.2.2 (= 0.4.0-116~20.04), rocprofiler-sdk-roctx-rpath6.2.2 (= 0.4.0-116~20.04), hip-dev-rpath6.2.2 (= 6.2.41134.60202-116~20.04), hsa-rocr-dev-rpath6.2.2 (= 1.14.0.60202-116~20.04), hsakmt-roct-dev-rpath6.2.2 (= 20240607.4.05.60202-116~20.04), rocprofiler-register-rpath6.2.2 (= 0.4.0.60202-116~20.04), rocprofiler-dev-rpath6.2.2 (= 2.0.60202.60202-116~20.04), roctracer-dev-rpath6.2.2 (= 4.1.60202.60202-116~20.04), openmp-extras-dev-rpath6.2.2 (= 18.62.0.60202-116~20.04), rocm-opencl-dev-rpath6.2.2 (= 2.0.0.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-dev-rpath6.2.2/rocm-dev-rpath6.2.2_6.2.2.60202-116~20.04_amd64.deb -Size: 2640 -SHA256: 264769227a845786edd99d4be9bc9f8639e180000ee82646a8d6471e2af80d9e -SHA1: 60aaffbf5288d303863e0430860d8912c4b7fe78 -MD5sum: 8ce93c2a6d9858b45d927b7fe79511ea -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-dev6.2.2 -Architecture: amd64 -Depends: amd-smi-lib6.2.2 (= 24.6.3.60202-116~20.04), comgr6.2.2 (= 2.8.0.60202-116~20.04), hip-doc6.2.2 (= 6.2.41134.60202-116~20.04), hipify-clang6.2.2 (= 18.0.0.60202-116~20.04), hip-runtime-amd6.2.2 (= 6.2.41134.60202-116~20.04), hip-samples6.2.2 (= 6.2.41134.60202-116~20.04), hipcc6.2.2 (= 1.1.1.60202-116~20.04), hsa-rocr6.2.2 (= 1.14.0.60202-116~20.04), hsa-amd-aqlprofile6.2.2 (= 1.0.0.60202.60202-116~20.04), rocm-llvm6.2.2 (= 18.0.0.24355.60202-116~20.04), openmp-extras-runtime6.2.2 (= 18.62.0.60202-116~20.04), rocm-cmake6.2.2 (= 0.13.0.60202-116~20.04), rocm-dbgapi6.2.2 (= 0.76.0.60202-116~20.04), rocm-debug-agent6.2.2 (= 2.0.3.60202-116~20.04), rocm-device-libs6.2.2 (= 1.0.0.60202-116~20.04), rocm-gdb6.2.2 (= 14.2.60202-116~20.04), rocm-smi-lib6.2.2 (= 7.3.0.60202-116~20.04), rocm-utils6.2.2 (= 6.2.2.60202-116~20.04), rocm-core6.2.2 (= 6.2.2.60202-116~20.04), rocm-opencl6.2.2 (= 2.0.0.60202-116~20.04), rocm-opencl-icd-loader6.2.2 (= 1.2.60202-116~20.04), rocprofiler-register6.2.2 (= 0.4.0.60202-116~20.04), rocprofiler6.2.2 (= 2.0.60202.60202-116~20.04), rocprofiler-plugins6.2.2 (= 2.0.60202.60202-116~20.04), roctracer6.2.2 (= 4.1.60202.60202-116~20.04), rocprofiler-sdk6.2.2 (= 0.4.0-116~20.04), rocprofiler-sdk-roctx6.2.2 (= 0.4.0-116~20.04), hip-dev6.2.2 (= 6.2.41134.60202-116~20.04), hsa-rocr-dev6.2.2 (= 1.14.0.60202-116~20.04), hsakmt-roct-dev6.2.2 (= 20240607.4.05.60202-116~20.04), rocprofiler-register6.2.2 (= 0.4.0.60202-116~20.04), rocprofiler-dev6.2.2 (= 2.0.60202.60202-116~20.04), roctracer-dev6.2.2 (= 4.1.60202.60202-116~20.04), openmp-extras-dev6.2.2 (= 18.62.0.60202-116~20.04), rocm-opencl-dev6.2.2 (= 2.0.0.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-dev6.2.2/rocm-dev6.2.2_6.2.2.60202-116~20.04_amd64.deb -Size: 2636 -SHA256: 462c328cf0635dc8a25162aa730dd6cd99f3d2b72f13dc9849a4df2f19f1357d -SHA1: e73306f12cd5325b10a96e17fa5909faaf91e6c1 -MD5sum: cbe82554655953c74d8560618d250f1d -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-developer-tools -Architecture: amd64 -Depends: amd-smi-lib (= 24.6.3.60202-116~20.04), rocm-core (= 6.2.2.60202-116~20.04), rocm-language-runtime (= 6.2.2.60202-116~20.04), rocm-dbgapi (= 0.76.0.60202-116~20.04), rocm-debug-agent (= 2.0.3.60202-116~20.04), rocm-gdb (= 14.2.60202-116~20.04), hsa-amd-aqlprofile (= 1.0.0.60202.60202-116~20.04), rocm-smi-lib (= 7.3.0.60202-116~20.04), rocprofiler-register (= 0.4.0.60202-116~20.04), rocprofiler (= 2.0.60202.60202-116~20.04), rocprofiler-plugins (= 2.0.60202.60202-116~20.04), roctracer (= 4.1.60202.60202-116~20.04), rocprofiler-sdk (= 0.4.0-116~20.04), rocprofiler-sdk-roctx (= 0.4.0-116~20.04), rocprofiler-dev (= 2.0.60202.60202-116~20.04), roctracer-dev (= 4.1.60202.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-developer-tools/rocm-developer-tools_6.2.2.60202-116~20.04_amd64.deb -Size: 2176 -SHA256: 0b5bf0ec579556c24f73eed69f234d9286e54d122441220b7ff14627422fff64 -SHA1: 921737b00479e03d59d44fa48da10a07c13e5ab8 -MD5sum: ad413f59e0a66808e205c17d28414b88 -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-developer-tools-asan -Architecture: amd64 -Depends: rocm-developer-tools (= 6.2.2.60202-116~20.04), rocm-core-asan (= 6.2.2.60202-116~20.04), rocm-language-runtime-asan (= 6.2.2.60202-116~20.04), rocm-dbgapi-asan (= 0.76.0.60202-116~20.04), rocm-debug-agent-asan (= 2.0.3.60202-116~20.04), hsa-amd-aqlprofile-asan (= 1.0.0.60202.60202-116~20.04), rocm-smi-lib-asan (= 7.3.0.60202-116~20.04), amd-smi-lib-asan (= 24.6.3.60202-116~20.04), rocprofiler-asan (= 2.0.60202.60202-116~20.04), roctracer-asan (= 4.1.60202.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-developer-tools-asan/rocm-developer-tools-asan_6.2.2.60202-116~20.04_amd64.deb -Size: 924 -SHA256: 883db6bf8c67d8170724c9a3b77b1fc5aace048f8562117d2347cb563f0c22bb -SHA1: db7d2f548bb4a23cdacda85568f6e027a1006f76 -MD5sum: 2df1ee3be6574f470d174e03be6fa917 -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-developer-tools-asan-rpath6.2.2 -Architecture: amd64 -Depends: rocm-developer-tools (= 6.2.2.60202-116~20.04), rocm-core-asan-rpath6.2.2 (= 6.2.2.60202-116~20.04), rocm-language-runtime-asan-rpath6.2.2 (= 6.2.2.60202-116~20.04), rocm-dbgapi-asan-rpath6.2.2 (= 0.76.0.60202-116~20.04), rocm-debug-agent-asan-rpath6.2.2 (= 2.0.3.60202-116~20.04), hsa-amd-aqlprofile-asan-rpath6.2.2 (= 1.0.0.60202.60202-116~20.04), rocm-smi-lib-asan-rpath6.2.2 (= 7.3.0.60202-116~20.04), amd-smi-lib-asan-rpath6.2.2 (= 24.6.3.60202-116~20.04), rocprofiler-asan-rpath6.2.2 (= 2.0.60202.60202-116~20.04), roctracer-asan-rpath6.2.2 (= 4.1.60202.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-developer-tools-asan-rpath6.2.2/rocm-developer-tools-asan-rpath6.2.2_6.2.2.60202-116~20.04_amd64.deb -Size: 1128 -SHA256: fc0324f4fe38435d028bd9178813dfc921be73ea4ccccfd0a24da14123db4215 -SHA1: 8754cdf1e164f6b87813f38adddbadf514565369 -MD5sum: 8a50cc65d8a20e10f4c2702e04ce9724 -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-developer-tools-asan6.2.2 -Architecture: amd64 -Depends: rocm-developer-tools (= 6.2.2.60202-116~20.04), rocm-core-asan6.2.2 (= 6.2.2.60202-116~20.04), rocm-language-runtime-asan6.2.2 (= 6.2.2.60202-116~20.04), rocm-dbgapi-asan6.2.2 (= 0.76.0.60202-116~20.04), rocm-debug-agent-asan6.2.2 (= 2.0.3.60202-116~20.04), hsa-amd-aqlprofile-asan6.2.2 (= 1.0.0.60202.60202-116~20.04), rocm-smi-lib-asan6.2.2 (= 7.3.0.60202-116~20.04), amd-smi-lib-asan6.2.2 (= 24.6.3.60202-116~20.04), rocprofiler-asan6.2.2 (= 2.0.60202.60202-116~20.04), roctracer-asan6.2.2 (= 4.1.60202.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-developer-tools-asan6.2.2/rocm-developer-tools-asan6.2.2_6.2.2.60202-116~20.04_amd64.deb -Size: 1120 -SHA256: 08b1c080d77770c15cc0c93267c841fc3d35da96bbd3221326b66c13a900cd7e -SHA1: 56023064d7c335620e43e29b777b238636a124bc -MD5sum: f3be38fe632d013fa72fa1dc057a8262 -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-developer-tools-rpath6.2.2 -Architecture: amd64 -Depends: amd-smi-lib-rpath6.2.2 (= 24.6.3.60202-116~20.04), rocm-core-rpath6.2.2 (= 6.2.2.60202-116~20.04), rocm-language-runtime-rpath6.2.2 (= 6.2.2.60202-116~20.04), rocm-dbgapi-rpath6.2.2 (= 0.76.0.60202-116~20.04), rocm-debug-agent-rpath6.2.2 (= 2.0.3.60202-116~20.04), rocm-gdb-rpath6.2.2 (= 14.2.60202-116~20.04), hsa-amd-aqlprofile-rpath6.2.2 (= 1.0.0.60202.60202-116~20.04), rocm-smi-lib-rpath6.2.2 (= 7.3.0.60202-116~20.04), rocprofiler-register-rpath6.2.2 (= 0.4.0.60202-116~20.04), rocprofiler-rpath6.2.2 (= 2.0.60202.60202-116~20.04), rocprofiler-plugins-rpath6.2.2 (= 2.0.60202.60202-116~20.04), roctracer-rpath6.2.2 (= 4.1.60202.60202-116~20.04), rocprofiler-sdk-rpath6.2.2 (= 0.4.0-116~20.04), rocprofiler-sdk-roctx-rpath6.2.2 (= 0.4.0-116~20.04), rocprofiler-dev-rpath6.2.2 (= 2.0.60202.60202-116~20.04), roctracer-dev-rpath6.2.2 (= 4.1.60202.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-developer-tools-rpath6.2.2/rocm-developer-tools-rpath6.2.2_6.2.2.60202-116~20.04_amd64.deb -Size: 2332 -SHA256: d7a326d5c4aaf6105934bba43bc00281484363a3bcd0e620acd37196cfc7b21b -SHA1: e1aca54427e4608a3d74cbe3b3b251f7be78d889 -MD5sum: 9e8cc7bb810811e919fa36dc8bfa6be3 -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-developer-tools6.2.2 -Architecture: amd64 -Depends: amd-smi-lib6.2.2 (= 24.6.3.60202-116~20.04), rocm-core6.2.2 (= 6.2.2.60202-116~20.04), rocm-language-runtime6.2.2 (= 6.2.2.60202-116~20.04), rocm-dbgapi6.2.2 (= 0.76.0.60202-116~20.04), rocm-debug-agent6.2.2 (= 2.0.3.60202-116~20.04), rocm-gdb6.2.2 (= 14.2.60202-116~20.04), hsa-amd-aqlprofile6.2.2 (= 1.0.0.60202.60202-116~20.04), rocm-smi-lib6.2.2 (= 7.3.0.60202-116~20.04), rocprofiler-register6.2.2 (= 0.4.0.60202-116~20.04), rocprofiler6.2.2 (= 2.0.60202.60202-116~20.04), rocprofiler-plugins6.2.2 (= 2.0.60202.60202-116~20.04), roctracer6.2.2 (= 4.1.60202.60202-116~20.04), rocprofiler-sdk6.2.2 (= 0.4.0-116~20.04), rocprofiler-sdk-roctx6.2.2 (= 0.4.0-116~20.04), rocprofiler-dev6.2.2 (= 2.0.60202.60202-116~20.04), roctracer-dev6.2.2 (= 4.1.60202.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-developer-tools6.2.2/rocm-developer-tools6.2.2_6.2.2.60202-116~20.04_amd64.deb -Size: 2328 -SHA256: 8462b9b602a48a09d2ae47d70bfe7fa12150708cca4fde949867d92c2e483a69 -SHA1: 4ff8ff622f6dc153b2a2e0fcf492232f336ab322 -MD5sum: 42bf05069dbf4d37d80bf4d07b9deedf -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-device-libs -Architecture: amd64 -Depends: rocm-core -Priority: optional -Section: devel -Filename: pool/main/r/rocm-device-libs/rocm-device-libs_1.0.0.60202-116~20.04_amd64.deb -Size: 717744 -SHA256: c06570df2de10c233f6d8af26a4fc74373fbab7ff5c0f0a3ff0e9854e41068cc -SHA1: 29deb99526d011ff758f9fbf4d2c7223597d5d58 -MD5sum: d9ce86726dd0ea03f03ad04dfa9f1594 -Description: Radeon Open Compute - device libraries - This package includes LLVM bitcode libraries. -Homepage: https://github.com/RadeonOpenCompute/ROCm-Device-Libs -Maintainer: ROCm Compiler Support -Version: 1.0.0.60202-116~20.04 -Installed-Size: 3326 - -Package: rocm-device-libs-rpath6.2.2 -Architecture: amd64 -Depends: rocm-core-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocm-device-libs-rpath6.2.2/rocm-device-libs-rpath6.2.2_1.0.0.60202-116~20.04_amd64.deb -Size: 467264 -SHA256: 9951a0af8b2044e71da0b900ed4cc119c34dcc2582782600f10174e8e6e7d426 -SHA1: 23e881c4d6b10099aef7ba090c5e090d17adeb2f -MD5sum: 9562e7119ecf351bceb957dfcbf6caab -Description: Radeon Open Compute - device libraries - This package includes LLVM bitcode libraries. -Homepage: https://github.com/RadeonOpenCompute/ROCm-Device-Libs -Maintainer: ROCm Compiler Support -Version: 1.0.0.60202-116~20.04 -Installed-Size: 3326 - -Package: rocm-device-libs6.2.2 -Architecture: amd64 -Depends: rocm-core6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocm-device-libs6.2.2/rocm-device-libs6.2.2_1.0.0.60202-116~20.04_amd64.deb -Size: 467240 -SHA256: ce3eee024a6de77072208738a45998b058171e7fa12d3045a3f66c698e35a0b4 -SHA1: 1d9886c160676be9c2738566d01fec15a65339ea -MD5sum: 06f3c3627082409dec44e127a8228197 -Description: Radeon Open Compute - device libraries - This package includes LLVM bitcode libraries. -Homepage: https://github.com/RadeonOpenCompute/ROCm-Device-Libs -Maintainer: ROCm Compiler Support -Version: 1.0.0.60202-116~20.04 -Installed-Size: 3326 - -Package: rocm-gdb -Version: 14.2.60202-116~20.04 -Maintainer: ROCm Debugger Support -Priority: optional -Section: utils -Filename: pool/main/r/rocm-gdb/rocm-gdb_14.2.60202-116~20.04_amd64.deb -Size: 110134884 -SHA256: 6df50aa0d2e1327cfea792ae1140cd50a82bf59cac648fc1189d47370fce527b -SHA1: d835ec228b078ae5bc7d66dafdd20eb459b6dc7a -MD5sum: f77d83f611284d9462cc008e47daae03 -Description: ROCgdb - This is ROCgdb, the AMD ROCm source-level debugger for Linux, - based on GDB, the GNU source-level debugger. -Architecture: amd64 -Essential: no -Depends: libc6 (>= 2.29), libexpat1 (>= 2.0.1), libgcc-s1 (>= 4.2), libgmp10, liblzma5 (>= 5.1.1alpha+20110809), libmpfr6 (>= 3.1.3), libncursesw6 (>= 6), libpython3.8 (>= 3.8.2), libstdc++6 (>= 7), libtinfo6 (>= 6), zlib1g (>= 1:1.2.0), rocm-dbgapi, rocm-core - -Package: rocm-gdb-rpath6.2.2 -Version: 14.2.60202-116~20.04 -Maintainer: ROCm Debugger Support -Priority: optional -Section: utils -Filename: pool/main/r/rocm-gdb-rpath6.2.2/rocm-gdb-rpath6.2.2_14.2.60202-116~20.04_amd64.deb -Size: 66404204 -SHA256: d4aa7f72c8412a66e2191fa4e8ad865bdc9743b19776ed45de79bf3500460692 -SHA1: 8fe3f9751791edeba189877372d378a92f2f34e6 -MD5sum: 8e1726eb679abf0bcb4f60147ab35ab6 -Description: ROCgdb - This is ROCgdb, the AMD ROCm source-level debugger for Linux, - based on GDB, the GNU source-level debugger. -Architecture: amd64 -Essential: no -Depends: libc6 (>= 2.29), libexpat1 (>= 2.0.1), libgcc-s1 (>= 4.2), libgmp10, liblzma5 (>= 5.1.1alpha+20110809), libmpfr6 (>= 3.1.3), libncursesw6 (>= 6), libpython3.8 (>= 3.8.2), libstdc++6 (>= 7), libtinfo6 (>= 6), zlib1g (>= 1:1.2.0), rocm-dbgapi-rpath6.2.2, rocm-core-rpath6.2.2 - -Package: rocm-gdb6.2.2 -Version: 14.2.60202-116~20.04 -Maintainer: ROCm Debugger Support -Priority: optional -Section: utils -Filename: pool/main/r/rocm-gdb6.2.2/rocm-gdb6.2.2_14.2.60202-116~20.04_amd64.deb -Size: 66402800 -SHA256: 516452d407f2bb972603ac9faa810949b36d8703543e7805f129d2c44a5afeed -SHA1: 4e122b5a60100bf843772dceb48352d230b8a787 -MD5sum: 6a82b38af9e85e0f5f8a3b04be3987c8 -Description: ROCgdb - This is ROCgdb, the AMD ROCm source-level debugger for Linux, - based on GDB, the GNU source-level debugger. -Architecture: amd64 -Essential: no -Depends: libc6 (>= 2.29), libexpat1 (>= 2.0.1), libgcc-s1 (>= 4.2), libgmp10, liblzma5 (>= 5.1.1alpha+20110809), libmpfr6 (>= 3.1.3), libncursesw6 (>= 6), libpython3.8 (>= 3.8.2), libstdc++6 (>= 7), libtinfo6 (>= 6), zlib1g (>= 1:1.2.0), rocm-dbgapi6.2.2, rocm-core6.2.2 - -Package: rocm-hip-libraries -Architecture: amd64 -Depends: rocm-core (= 6.2.2.60202-116~20.04), rocm-smi-lib (= 7.3.0.60202-116~20.04), rocm-hip-runtime (= 6.2.2.60202-116~20.04), hipblas (= 2.2.0.60202-116~20.04), hipblaslt (= 0.8.0.60202-116~20.04), hipfft (= 1.0.15.60202-116~20.04), hipsparse (= 3.1.1.60202-116~20.04), hipsolver (= 2.2.0.60202-116~20.04), hiptensor (= 1.3.0.60202-116~20.04), rccl (= 2.20.5.60202-116~20.04), rocalution (= 3.2.0.60202-116~20.04), rocblas (= 4.2.1.60202-116~20.04), rocfft (= 1.0.29.60202-116~20.04), rocrand (= 3.1.0.60202-116~20.04), hiprand (= 2.11.0.60202-116~20.04), rocsolver (= 3.26.0.60202-116~20.04), rocsparse (= 3.2.0.60202-116~20.04), hipsparselt (= 0.2.1.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-hip-libraries/rocm-hip-libraries_6.2.2.60202-116~20.04_amd64.deb -Size: 948 -SHA256: b6d5fd6ed7de139ce8e892152f720275925b7138ca90632b00b4b201c8c41d62 -SHA1: 5f55ff4ae3c4d41c7acd2853cf9458d458cb7f7a -MD5sum: 7004192668c98004026f6119b6a69191 -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-hip-libraries-asan -Architecture: amd64 -Depends: rocm-hip-libraries (= 6.2.2.60202-116~20.04), rocm-core-asan (= 6.2.2.60202-116~20.04), rocm-smi-lib-asan (= 7.3.0.60202-116~20.04), rocm-hip-runtime-asan (= 6.2.2.60202-116~20.04), hipblas-asan (= 2.2.0.60202-116~20.04), hipblaslt-asan (= 0.8.0.60202-116~20.04), hipfft-asan (= 1.0.15.60202-116~20.04), hipsparse-asan (= 3.1.1.60202-116~20.04), hipsolver-asan (= 2.2.0.60202-116~20.04), rocalution-asan (= 3.2.0.60202-116~20.04), rocblas-asan (= 4.2.1.60202-116~20.04), rocfft-asan (= 1.0.29.60202-116~20.04), rocsolver-asan (= 3.26.0.60202-116~20.04), rocsparse-asan (= 3.2.0.60202-116~20.04), hipsparselt-asan (= 0.2.1.60202-116~20.04), rocrand-asan (= 3.1.0.60202-116~20.04), hiprand-asan (= 2.11.0.60202-116~20.04), hiptensor-asan (= 1.3.0.60202-116~20.04), rccl-asan (= 2.20.5.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-hip-libraries-asan/rocm-hip-libraries-asan_6.2.2.60202-116~20.04_amd64.deb -Size: 966 -SHA256: 68549aa7d596a3a811288259d9af17277cb5318917f93b91ee450fe93febd1d0 -SHA1: e749ce20f5a21ee09ab022bd20bf0da6b582672f -MD5sum: d5bc122c3554748984e42d354e76c4d4 -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-hip-libraries-asan-rpath6.2.2 -Architecture: amd64 -Depends: rocm-hip-libraries (= 6.2.2.60202-116~20.04), rocm-core-asan-rpath6.2.2 (= 6.2.2.60202-116~20.04), rocm-smi-lib-asan-rpath6.2.2 (= 7.3.0.60202-116~20.04), rocm-hip-runtime-asan-rpath6.2.2 (= 6.2.2.60202-116~20.04), hipblas-asan-rpath6.2.2 (= 2.2.0.60202-116~20.04), hipblaslt-asan-rpath6.2.2 (= 0.8.0.60202-116~20.04), hipfft-asan-rpath6.2.2 (= 1.0.15.60202-116~20.04), hipsparse-asan-rpath6.2.2 (= 3.1.1.60202-116~20.04), hipsolver-asan-rpath6.2.2 (= 2.2.0.60202-116~20.04), rocalution-asan-rpath6.2.2 (= 3.2.0.60202-116~20.04), rocblas-asan-rpath6.2.2 (= 4.2.1.60202-116~20.04), rocfft-asan-rpath6.2.2 (= 1.0.29.60202-116~20.04), rocsolver-asan-rpath6.2.2 (= 3.26.0.60202-116~20.04), rocsparse-asan-rpath6.2.2 (= 3.2.0.60202-116~20.04), hipsparselt-asan-rpath6.2.2 (= 0.2.1.60202-116~20.04), rocrand-asan-rpath6.2.2 (= 3.1.0.60202-116~20.04), hiprand-asan-rpath6.2.2 (= 2.11.0.60202-116~20.04), hiptensor-asan-rpath6.2.2 (= 1.3.0.60202-116~20.04), rccl-asan-rpath6.2.2 (= 2.20.5.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-hip-libraries-asan-rpath6.2.2/rocm-hip-libraries-asan-rpath6.2.2_6.2.2.60202-116~20.04_amd64.deb -Size: 1164 -SHA256: cafcc83aa7b85fd6ca4c004717ac6850714d5bda13fc5b3402221e0d9613f13a -SHA1: be9b9fe419aa0ded09dd54ec2c887b9f251a9a7b -MD5sum: f6ad4629f743e9813c79cb8c7887d4f4 -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-hip-libraries-asan6.2.2 -Architecture: amd64 -Depends: rocm-hip-libraries (= 6.2.2.60202-116~20.04), rocm-core-asan6.2.2 (= 6.2.2.60202-116~20.04), rocm-smi-lib-asan6.2.2 (= 7.3.0.60202-116~20.04), rocm-hip-runtime-asan6.2.2 (= 6.2.2.60202-116~20.04), hipblas-asan6.2.2 (= 2.2.0.60202-116~20.04), hipblaslt-asan6.2.2 (= 0.8.0.60202-116~20.04), hipfft-asan6.2.2 (= 1.0.15.60202-116~20.04), hipsparse-asan6.2.2 (= 3.1.1.60202-116~20.04), hipsolver-asan6.2.2 (= 2.2.0.60202-116~20.04), rocalution-asan6.2.2 (= 3.2.0.60202-116~20.04), rocblas-asan6.2.2 (= 4.2.1.60202-116~20.04), rocfft-asan6.2.2 (= 1.0.29.60202-116~20.04), rocsolver-asan6.2.2 (= 3.26.0.60202-116~20.04), rocsparse-asan6.2.2 (= 3.2.0.60202-116~20.04), hipsparselt-asan6.2.2 (= 0.2.1.60202-116~20.04), rocrand-asan6.2.2 (= 3.1.0.60202-116~20.04), hiprand-asan6.2.2 (= 2.11.0.60202-116~20.04), hiptensor-asan6.2.2 (= 1.3.0.60202-116~20.04), rccl-asan6.2.2 (= 2.20.5.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-hip-libraries-asan6.2.2/rocm-hip-libraries-asan6.2.2_6.2.2.60202-116~20.04_amd64.deb -Size: 1164 -SHA256: fbd7f73e3780a9ee7be1a81c4dc221a7e7eefb9a791b01a40218b07e37cd9607 -SHA1: 8048d569edb3ee0d08de328cf651e5dd2e782d7e -MD5sum: 693d37d6e9986bcb48a2b2a38281aaea -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-hip-libraries-rpath6.2.2 -Architecture: amd64 -Depends: rocm-core-rpath6.2.2 (= 6.2.2.60202-116~20.04), rocm-smi-lib-rpath6.2.2 (= 7.3.0.60202-116~20.04), rocm-hip-runtime-rpath6.2.2 (= 6.2.2.60202-116~20.04), hipblas-rpath6.2.2 (= 2.2.0.60202-116~20.04), hipblaslt-rpath6.2.2 (= 0.8.0.60202-116~20.04), hipfft-rpath6.2.2 (= 1.0.15.60202-116~20.04), hipsparse-rpath6.2.2 (= 3.1.1.60202-116~20.04), hipsolver-rpath6.2.2 (= 2.2.0.60202-116~20.04), hiptensor-rpath6.2.2 (= 1.3.0.60202-116~20.04), rccl-rpath6.2.2 (= 2.20.5.60202-116~20.04), rocalution-rpath6.2.2 (= 3.2.0.60202-116~20.04), rocblas-rpath6.2.2 (= 4.2.1.60202-116~20.04), rocfft-rpath6.2.2 (= 1.0.29.60202-116~20.04), rocrand-rpath6.2.2 (= 3.1.0.60202-116~20.04), hiprand-rpath6.2.2 (= 2.11.0.60202-116~20.04), rocsolver-rpath6.2.2 (= 3.26.0.60202-116~20.04), rocsparse-rpath6.2.2 (= 3.2.0.60202-116~20.04), hipsparselt-rpath6.2.2 (= 0.2.1.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-hip-libraries-rpath6.2.2/rocm-hip-libraries-rpath6.2.2_6.2.2.60202-116~20.04_amd64.deb -Size: 1152 -SHA256: 7637ec70cc5d3d27024b0dd16027a1a726417661787ef265cc03e54ffa09d193 -SHA1: 6bff408c78cbc0d9bcff4770b77a29017d48ce42 -MD5sum: f5647e18710e8ce3a72752adb0c4dda2 -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-hip-libraries6.2.2 -Architecture: amd64 -Depends: rocm-core6.2.2 (= 6.2.2.60202-116~20.04), rocm-smi-lib6.2.2 (= 7.3.0.60202-116~20.04), rocm-hip-runtime6.2.2 (= 6.2.2.60202-116~20.04), hipblas6.2.2 (= 2.2.0.60202-116~20.04), hipblaslt6.2.2 (= 0.8.0.60202-116~20.04), hipfft6.2.2 (= 1.0.15.60202-116~20.04), hipsparse6.2.2 (= 3.1.1.60202-116~20.04), hipsolver6.2.2 (= 2.2.0.60202-116~20.04), hiptensor6.2.2 (= 1.3.0.60202-116~20.04), rccl6.2.2 (= 2.20.5.60202-116~20.04), rocalution6.2.2 (= 3.2.0.60202-116~20.04), rocblas6.2.2 (= 4.2.1.60202-116~20.04), rocfft6.2.2 (= 1.0.29.60202-116~20.04), rocrand6.2.2 (= 3.1.0.60202-116~20.04), hiprand6.2.2 (= 2.11.0.60202-116~20.04), rocsolver6.2.2 (= 3.26.0.60202-116~20.04), rocsparse6.2.2 (= 3.2.0.60202-116~20.04), hipsparselt6.2.2 (= 0.2.1.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-hip-libraries6.2.2/rocm-hip-libraries6.2.2_6.2.2.60202-116~20.04_amd64.deb -Size: 1144 -SHA256: 10038af95ed40c96375e3a948a6fe6e5f84b14e2b30be18d404f3c9d520bf36b -SHA1: a925b7b4dcbc2f73b518faabae52091117005cdc -MD5sum: d8282b0a92c0fe154383afb2275497f8 -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-hip-runtime -Architecture: amd64 -Depends: rocm-core (= 6.2.2.60202-116~20.04), rocm-language-runtime (= 6.2.2.60202-116~20.04), rocminfo (= 1.0.0.60202-116~20.04), hip-runtime-amd (= 6.2.41134.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-hip-runtime/rocm-hip-runtime_6.2.2.60202-116~20.04_amd64.deb -Size: 2030 -SHA256: 5788c38d2f3599d5e5cceaeef2a3233e6527cefd8c8f3d861735100a8bba3030 -SHA1: 77f70999e42b4e0d5841430cec6c5ac726759d28 -MD5sum: 196b31d2eee7a500a1500905822862bd -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-hip-runtime-asan -Architecture: amd64 -Depends: rocm-hip-runtime (= 6.2.2.60202-116~20.04), rocm-core-asan (= 6.2.2.60202-116~20.04), rocm-language-runtime-asan (= 6.2.2.60202-116~20.04), hip-runtime-amd-asan (= 6.2.41134.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-hip-runtime-asan/rocm-hip-runtime-asan_6.2.2.60202-116~20.04_amd64.deb -Size: 836 -SHA256: 9dfc11140cdc1b0413d044f8fc2c7c5b955bc2e00c4ce32c8262ba455b3b9fda -SHA1: 0cee691fadc679765df6bb32a3a5f37961f9c57a -MD5sum: 476e8176e3f997b6bab2984d261cbffe -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-hip-runtime-asan-rpath6.2.2 -Architecture: amd64 -Depends: rocm-hip-runtime (= 6.2.2.60202-116~20.04), rocm-core-asan-rpath6.2.2 (= 6.2.2.60202-116~20.04), rocm-language-runtime-asan-rpath6.2.2 (= 6.2.2.60202-116~20.04), hip-runtime-amd-asan-rpath6.2.2 (= 6.2.41134.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-hip-runtime-asan-rpath6.2.2/rocm-hip-runtime-asan-rpath6.2.2_6.2.2.60202-116~20.04_amd64.deb -Size: 1040 -SHA256: 354aeca06d7c1668e58348c2e811a287aa4f0aa8edecdba677016d9cb11def2e -SHA1: 57f1b3a8f49ae202c30b4535462ce604bfa2026f -MD5sum: a7064697b37947d3a67c7ef9a62471f9 -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-hip-runtime-asan6.2.2 -Architecture: amd64 -Depends: rocm-hip-runtime (= 6.2.2.60202-116~20.04), rocm-core-asan6.2.2 (= 6.2.2.60202-116~20.04), rocm-language-runtime-asan6.2.2 (= 6.2.2.60202-116~20.04), hip-runtime-amd-asan6.2.2 (= 6.2.41134.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-hip-runtime-asan6.2.2/rocm-hip-runtime-asan6.2.2_6.2.2.60202-116~20.04_amd64.deb -Size: 1044 -SHA256: 80fb28d0363c1472b6917dc2d85ee4221cb08e9153a8a151122175091b92656e -SHA1: ade65d17bd22367750ae37ec03955e920da2202a -MD5sum: bd363f004e53112394fc274f014ae6d5 -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-hip-runtime-dev -Architecture: amd64 -Depends: rocm-core (= 6.2.2.60202-116~20.04), rocm-device-libs (= 1.0.0.60202-116~20.04), rocm-hip-runtime (= 6.2.2.60202-116~20.04), rocm-cmake (= 0.13.0.60202-116~20.04), rocm-llvm (= 18.0.0.24355.60202-116~20.04), hipcc (= 1.1.1.60202-116~20.04), hipify-clang (= 18.0.0.60202-116~20.04), hip-doc (= 6.2.41134.60202-116~20.04), hip-samples (= 6.2.41134.60202-116~20.04), hip-dev (= 6.2.41134.60202-116~20.04), hsa-rocr-dev (= 1.14.0.60202-116~20.04), hsakmt-roct-dev (= 20240607.4.05.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-hip-runtime-dev/rocm-hip-runtime-dev_6.2.2.60202-116~20.04_amd64.deb -Size: 2222 -SHA256: d852bd16cdffa5b1b9ab2bf267e715e876798f6091436288a9f1233ec8b423d8 -SHA1: 99c57d965f3743b1efe362889e04dd1b23d27596 -MD5sum: f75a1a905e7f16b3bc1bef5791ecacab -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-hip-runtime-dev-rpath6.2.2 -Architecture: amd64 -Depends: rocm-core-rpath6.2.2 (= 6.2.2.60202-116~20.04), rocm-device-libs-rpath6.2.2 (= 1.0.0.60202-116~20.04), rocm-hip-runtime-rpath6.2.2 (= 6.2.2.60202-116~20.04), rocm-cmake-rpath6.2.2 (= 0.13.0.60202-116~20.04), rocm-llvm-rpath6.2.2 (= 18.0.0.24355.60202-116~20.04), hipcc-rpath6.2.2 (= 1.1.1.60202-116~20.04), hipify-clang-rpath6.2.2 (= 18.0.0.60202-116~20.04), hip-doc-rpath6.2.2 (= 6.2.41134.60202-116~20.04), hip-samples-rpath6.2.2 (= 6.2.41134.60202-116~20.04), hip-dev-rpath6.2.2 (= 6.2.41134.60202-116~20.04), hsa-rocr-dev-rpath6.2.2 (= 1.14.0.60202-116~20.04), hsakmt-roct-dev-rpath6.2.2 (= 20240607.4.05.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-hip-runtime-dev-rpath6.2.2/rocm-hip-runtime-dev-rpath6.2.2_6.2.2.60202-116~20.04_amd64.deb -Size: 2380 -SHA256: e8d0233bcce376eeb04119dc20caf39fb861f3a54a06a92573a0d0b99c6d3a1a -SHA1: 185e7502fb628bb7542915fec7a8fd4e316daabd -MD5sum: 7526c67c43a3f707f047e55042e7713b -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-hip-runtime-dev6.2.2 -Architecture: amd64 -Depends: rocm-core6.2.2 (= 6.2.2.60202-116~20.04), rocm-device-libs6.2.2 (= 1.0.0.60202-116~20.04), rocm-hip-runtime6.2.2 (= 6.2.2.60202-116~20.04), rocm-cmake6.2.2 (= 0.13.0.60202-116~20.04), rocm-llvm6.2.2 (= 18.0.0.24355.60202-116~20.04), hipcc6.2.2 (= 1.1.1.60202-116~20.04), hipify-clang6.2.2 (= 18.0.0.60202-116~20.04), hip-doc6.2.2 (= 6.2.41134.60202-116~20.04), hip-samples6.2.2 (= 6.2.41134.60202-116~20.04), hip-dev6.2.2 (= 6.2.41134.60202-116~20.04), hsa-rocr-dev6.2.2 (= 1.14.0.60202-116~20.04), hsakmt-roct-dev6.2.2 (= 20240607.4.05.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-hip-runtime-dev6.2.2/rocm-hip-runtime-dev6.2.2_6.2.2.60202-116~20.04_amd64.deb -Size: 2372 -SHA256: bd02bc902f58a94f68b995df830e35449fe6c3beeb6d9227ca9963cb6c74bafe -SHA1: eb5e90e9188b5ddcb929ed68a6a9e3963532e01e -MD5sum: 48eb9b7cbd5e38f94fc91239cbb2bc41 -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-hip-runtime-rpath6.2.2 -Architecture: amd64 -Depends: rocm-core-rpath6.2.2 (= 6.2.2.60202-116~20.04), rocm-language-runtime-rpath6.2.2 (= 6.2.2.60202-116~20.04), rocminfo-rpath6.2.2 (= 1.0.0.60202-116~20.04), hip-runtime-amd-rpath6.2.2 (= 6.2.41134.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-hip-runtime-rpath6.2.2/rocm-hip-runtime-rpath6.2.2_6.2.2.60202-116~20.04_amd64.deb -Size: 2204 -SHA256: 351958619c24ce0f75b5ea75ea64c2159fff3d901bf302d1671ebd519c538126 -SHA1: a2734eb510f777d6f009c59326ffb0e14c435045 -MD5sum: 6fa73df2a788ea24ceb8add5241b91d5 -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-hip-runtime6.2.2 -Architecture: amd64 -Depends: rocm-core6.2.2 (= 6.2.2.60202-116~20.04), rocm-language-runtime6.2.2 (= 6.2.2.60202-116~20.04), rocminfo6.2.2 (= 1.0.0.60202-116~20.04), hip-runtime-amd6.2.2 (= 6.2.41134.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-hip-runtime6.2.2/rocm-hip-runtime6.2.2_6.2.2.60202-116~20.04_amd64.deb -Size: 2188 -SHA256: 1ce6cae08e08ee905f83013e68e74bc03fe945a2208c3fc7138578bc544f1aa6 -SHA1: fe6a602347d450607d92ffcdf9dfd3cbfd77c951 -MD5sum: 12558d77b62895c4080c95ae332c5b2c -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-hip-sdk -Architecture: amd64 -Depends: rocm-hip-libraries (= 6.2.2.60202-116~20.04), rocm-core (= 6.2.2.60202-116~20.04), rocm-hip-runtime-dev (= 6.2.2.60202-116~20.04), composablekernel-dev (= 1.1.0.60202-116~20.04), hipblas-dev (= 2.2.0.60202-116~20.04), hipblaslt-dev (= 0.8.0.60202-116~20.04), hipcub-dev (= 3.2.0.60202-116~20.04), hipfft-dev (= 1.0.15.60202-116~20.04), hipsparse-dev (= 3.1.1.60202-116~20.04), hipsolver-dev (= 2.2.0.60202-116~20.04), hipfort-dev (= 0.4.0.60202-116~20.04), hiptensor-dev (= 1.3.0.60202-116~20.04), rccl-dev (= 2.20.5.60202-116~20.04), rocalution-dev (= 3.2.0.60202-116~20.04), rocblas-dev (= 4.2.1.60202-116~20.04), rocfft-dev (= 1.0.29.60202-116~20.04), rocprim-dev (= 3.2.0.60202-116~20.04), rocrand-dev (= 3.1.0.60202-116~20.04), hiprand-dev (= 2.11.0.60202-116~20.04), rocsolver-dev (= 3.26.0.60202-116~20.04), rocsparse-dev (= 3.2.0.60202-116~20.04), rocthrust-dev (= 3.1.0.60202-116~20.04), rocwmma-dev (= 1.5.0.60202-116~20.04), hipsparselt-dev (= 0.2.1.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-hip-sdk/rocm-hip-sdk_6.2.2.60202-116~20.04_amd64.deb -Size: 2192 -SHA256: 83c415aac6323fd82179b77faf5478640952157a9ffb340f0526465067f320a6 -SHA1: 1432b94bc318b800a56b0e1daa9f6bb860a2b2f9 -MD5sum: eb55b6972b9fa3e57abb2d2f9d5e606f -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-hip-sdk-rpath6.2.2 -Architecture: amd64 -Depends: rocm-hip-libraries-rpath6.2.2 (= 6.2.2.60202-116~20.04), rocm-core-rpath6.2.2 (= 6.2.2.60202-116~20.04), rocm-hip-runtime-dev-rpath6.2.2 (= 6.2.2.60202-116~20.04), composablekernel-dev-rpath6.2.2 (= 1.1.0.60202-116~20.04), hipblas-dev-rpath6.2.2 (= 2.2.0.60202-116~20.04), hipblaslt-dev-rpath6.2.2 (= 0.8.0.60202-116~20.04), hipcub-dev-rpath6.2.2 (= 3.2.0.60202-116~20.04), hipfft-dev-rpath6.2.2 (= 1.0.15.60202-116~20.04), hipsparse-dev-rpath6.2.2 (= 3.1.1.60202-116~20.04), hipsolver-dev-rpath6.2.2 (= 2.2.0.60202-116~20.04), hipfort-dev-rpath6.2.2 (= 0.4.0.60202-116~20.04), hiptensor-dev-rpath6.2.2 (= 1.3.0.60202-116~20.04), rccl-dev-rpath6.2.2 (= 2.20.5.60202-116~20.04), rocalution-dev-rpath6.2.2 (= 3.2.0.60202-116~20.04), rocblas-dev-rpath6.2.2 (= 4.2.1.60202-116~20.04), rocfft-dev-rpath6.2.2 (= 1.0.29.60202-116~20.04), rocprim-dev-rpath6.2.2 (= 3.2.0.60202-116~20.04), rocrand-dev-rpath6.2.2 (= 3.1.0.60202-116~20.04), hiprand-dev-rpath6.2.2 (= 2.11.0.60202-116~20.04), rocsolver-dev-rpath6.2.2 (= 3.26.0.60202-116~20.04), rocsparse-dev-rpath6.2.2 (= 3.2.0.60202-116~20.04), rocthrust-dev-rpath6.2.2 (= 3.1.0.60202-116~20.04), rocwmma-dev-rpath6.2.2 (= 1.5.0.60202-116~20.04), hipsparselt-dev-rpath6.2.2 (= 0.2.1.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-hip-sdk-rpath6.2.2/rocm-hip-sdk-rpath6.2.2_6.2.2.60202-116~20.04_amd64.deb -Size: 2340 -SHA256: 0237f336c030bd785584e3a389c0542643adabca925047a20bc7ece840c3f59a -SHA1: 3bbca8eeca8552715665e0a8164332426f8586a1 -MD5sum: 5caabfc33aba9e13d9092e356c4a8a91 -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-hip-sdk6.2.2 -Architecture: amd64 -Depends: rocm-hip-libraries6.2.2 (= 6.2.2.60202-116~20.04), rocm-core6.2.2 (= 6.2.2.60202-116~20.04), rocm-hip-runtime-dev6.2.2 (= 6.2.2.60202-116~20.04), composablekernel-dev6.2.2 (= 1.1.0.60202-116~20.04), hipblas-dev6.2.2 (= 2.2.0.60202-116~20.04), hipblaslt-dev6.2.2 (= 0.8.0.60202-116~20.04), hipcub-dev6.2.2 (= 3.2.0.60202-116~20.04), hipfft-dev6.2.2 (= 1.0.15.60202-116~20.04), hipsparse-dev6.2.2 (= 3.1.1.60202-116~20.04), hipsolver-dev6.2.2 (= 2.2.0.60202-116~20.04), hipfort-dev6.2.2 (= 0.4.0.60202-116~20.04), hiptensor-dev6.2.2 (= 1.3.0.60202-116~20.04), rccl-dev6.2.2 (= 2.20.5.60202-116~20.04), rocalution-dev6.2.2 (= 3.2.0.60202-116~20.04), rocblas-dev6.2.2 (= 4.2.1.60202-116~20.04), rocfft-dev6.2.2 (= 1.0.29.60202-116~20.04), rocprim-dev6.2.2 (= 3.2.0.60202-116~20.04), rocrand-dev6.2.2 (= 3.1.0.60202-116~20.04), hiprand-dev6.2.2 (= 2.11.0.60202-116~20.04), rocsolver-dev6.2.2 (= 3.26.0.60202-116~20.04), rocsparse-dev6.2.2 (= 3.2.0.60202-116~20.04), rocthrust-dev6.2.2 (= 3.1.0.60202-116~20.04), rocwmma-dev6.2.2 (= 1.5.0.60202-116~20.04), hipsparselt-dev6.2.2 (= 0.2.1.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-hip-sdk6.2.2/rocm-hip-sdk6.2.2_6.2.2.60202-116~20.04_amd64.deb -Size: 2332 -SHA256: f0110990239781c5de411ed404850a6d9d9a5eb6e09d4cd28f8d5f1204b15cc9 -SHA1: 010a99fd23f4c56bff7995f9b4dd6624d855b0c9 -MD5sum: b01dbb40a6596094a73f2837b0e10862 -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-khronos-cts -Architecture: amd64 -Depends: rocm-core, rocm-opencl-icd-loader -Priority: optional -Section: devel -Filename: pool/main/r/rocm-khronos-cts/rocm-khronos-cts_3.0.0.60202-116~20.04_amd64.deb -Size: 8309264 -SHA256: 9a6102cb97cd7417ceeb262458e5b600251239c1cba5014e663b91cdb8eee6c5 -SHA1: d8939d11706094b5c92f9f6ac9aa6d16b9d285be -MD5sum: b132e8929f1d7665751b768471ec7abf -Description: CLConform built using CMake -Maintainer: ROCm OpenCL Support -Version: 3.0.0.60202-116~20.04 -Installed-Size: 24079 - -Package: rocm-khronos-cts-dbgsym -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 94e9917906b3ce3d0ca64e31e75247644c9a46ce 9c37af5c1c6d62d20844a1f13b33c45e415a3055 98b06959a1d9de91d7c65a6491d461280d524b60 5366c00d77471b19badf16b8538bd8b74bdbbb91 c8cc61352a916478de98236803c6ca44605b3a66 68f4f8a8bf9bb317c67ac1322f1aa41680422df8 732d242066175d6f183fcc01caf1637e7f136be6 1bbdf6e4ce5a5d854e5f1b472af2c0ce58a4cba5 45b66ce9dc4a938fb93e94f69741581a24422784 045693d56269ce19aadf568552f22d46133f9632 714b4d6b129ba80811ebf5f449c238fbe5912f93 7cda9e0482c9280818978a5d1cba225c00ccb645 857a0dbc0d5684074269ba2862d9e7b9d00a45d0 e75fb162b1b7ceeae1e8dad92ed7ede977e05c8b 81bb62eb33d0952e982828c06eac7281ec35ca33 56a8425f61d034eddd7a3bdbb479038f135b9b96 b0bfdc5b73ebb8005b9148ee759b16033b458290 5267465a18ef8ff7bb24554a0e0ed4470fb3d0a6 2a4c642ffe919adb48a4f2058ad08f88bf335f57 489e092eb67f21cd078a326726ee0f36d726b1dc 7f9613c11119ee83427dd12fb45138f633261e89 fe5cbfece7bf1615b428255aa71b11fc6b4ecfcd b58a7f822e726b70acc4e7823edefd820514b25a 564610a456f0f4ab8d493a2b12f1ae13231caded 5db3a42379e42caa7f744437084aed5d19760566 7781e23422823e38e19bad64b995826e1c429363 4c9dfa4efe2d5ecfbce8b7ff8b1dff1d6cbd4f84 727fa19224b7e5305f20fdf1713a32e35acc1b49 f6cc343dd946f75d1c31a33cf674adbd93ae402a b88b5d1a1820aab4fa1b841584b60c5ed11b89c4 80b6e472fb5ad023ba99a5ba97205bdcfc99f7c5 99e52da9e711c4a0ba4a8d4183817af6c6c46d32 a2b170517b8425c8abdd3aa628405c0b753d6ce7 bb2af8446cf57726e4a900ca63e7bf9f82ad5639 9491e7c8905060234d37707cf00d0e74e455c26b fb33d11866ac9fc0e291ee3f0039c39970ba5bb6 1eea99c5cda0bdffbec92ce946e42699b8e147d7 1ffd33ddd7b7e8a58a28c12ff664e1963faf399d 10c268330940980c139aad706c6268cf8d85c3d3 49970265e0f28910457feadfe07424d8e1702eae 6863af6c17608d60c16868b35e7ef477a64260d3 2a5e889f9d3e7ae8d584e986a1c63844bf9b0ddb b55b56eda641a927ec21309b8d209f75fe373ac3 b916b3ba12c8268c2dd0d4174aa9c8417a207a7c 7ffe5de18e09db0356896173c72e081c75cde566 828ba05644f03f264b6a5419ecec7f00b1cbfdb2 -Depends: rocm-khronos-cts (= 3.0.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocm-khronos-cts-dbgsym/rocm-khronos-cts-dbgsym_3.0.0.60202-116~20.04_amd64.deb -Size: 85608890 -SHA256: 765f8fd255ba18fb7326f32c54553470700c2f25f1135f7fb390d5f9fa3a175b -SHA1: 42f8cda256ddcfedf3e6711b92abb04ddf532b38 -MD5sum: b8fced7edadcbbf3b53ca0b57ab6416c -Description: debug symbols for rocm-khronos-cts -Maintainer: ROCm OpenCL Support -Package-Type: ddeb -Version: 3.0.0.60202-116~20.04 -Installed-Size: 341286 - -Package: rocm-khronos-cts-dbgsym-rpath6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 94e9917906b3ce3d0ca64e31e75247644c9a46ce 9c37af5c1c6d62d20844a1f13b33c45e415a3055 98b06959a1d9de91d7c65a6491d461280d524b60 5366c00d77471b19badf16b8538bd8b74bdbbb91 c8cc61352a916478de98236803c6ca44605b3a66 68f4f8a8bf9bb317c67ac1322f1aa41680422df8 732d242066175d6f183fcc01caf1637e7f136be6 1bbdf6e4ce5a5d854e5f1b472af2c0ce58a4cba5 45b66ce9dc4a938fb93e94f69741581a24422784 045693d56269ce19aadf568552f22d46133f9632 714b4d6b129ba80811ebf5f449c238fbe5912f93 7cda9e0482c9280818978a5d1cba225c00ccb645 857a0dbc0d5684074269ba2862d9e7b9d00a45d0 e75fb162b1b7ceeae1e8dad92ed7ede977e05c8b 81bb62eb33d0952e982828c06eac7281ec35ca33 56a8425f61d034eddd7a3bdbb479038f135b9b96 b0bfdc5b73ebb8005b9148ee759b16033b458290 5267465a18ef8ff7bb24554a0e0ed4470fb3d0a6 2a4c642ffe919adb48a4f2058ad08f88bf335f57 489e092eb67f21cd078a326726ee0f36d726b1dc 7f9613c11119ee83427dd12fb45138f633261e89 fe5cbfece7bf1615b428255aa71b11fc6b4ecfcd b58a7f822e726b70acc4e7823edefd820514b25a 564610a456f0f4ab8d493a2b12f1ae13231caded 5db3a42379e42caa7f744437084aed5d19760566 7781e23422823e38e19bad64b995826e1c429363 4c9dfa4efe2d5ecfbce8b7ff8b1dff1d6cbd4f84 727fa19224b7e5305f20fdf1713a32e35acc1b49 f6cc343dd946f75d1c31a33cf674adbd93ae402a b88b5d1a1820aab4fa1b841584b60c5ed11b89c4 80b6e472fb5ad023ba99a5ba97205bdcfc99f7c5 99e52da9e711c4a0ba4a8d4183817af6c6c46d32 a2b170517b8425c8abdd3aa628405c0b753d6ce7 bb2af8446cf57726e4a900ca63e7bf9f82ad5639 9491e7c8905060234d37707cf00d0e74e455c26b fb33d11866ac9fc0e291ee3f0039c39970ba5bb6 1eea99c5cda0bdffbec92ce946e42699b8e147d7 1ffd33ddd7b7e8a58a28c12ff664e1963faf399d 10c268330940980c139aad706c6268cf8d85c3d3 49970265e0f28910457feadfe07424d8e1702eae 6863af6c17608d60c16868b35e7ef477a64260d3 2a5e889f9d3e7ae8d584e986a1c63844bf9b0ddb b55b56eda641a927ec21309b8d209f75fe373ac3 b916b3ba12c8268c2dd0d4174aa9c8417a207a7c 7ffe5de18e09db0356896173c72e081c75cde566 828ba05644f03f264b6a5419ecec7f00b1cbfdb2 -Depends: rocm-khronos-cts-rpath6.2.2 (= 3.0.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocm-khronos-cts-dbgsym-rpath6.2.2/rocm-khronos-cts-dbgsym-rpath6.2.2_3.0.0.60202-116~20.04_amd64.deb -Size: 46597724 -SHA256: a1bebc6614b5e6d30b16b9135cdc27c295bb08f5c5ae6771f30ffb39848cd200 -SHA1: 4883f9932443ce2852b6e5145370663378561314 -MD5sum: e556c80b1b825980466d045bfcda270a -Description: debug symbols for rocm-khronos-cts -Maintainer: ROCm OpenCL Support -Package-Type: ddeb -Version: 3.0.0.60202-116~20.04 -Installed-Size: 341286 - -Package: rocm-khronos-cts-dbgsym6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 94e9917906b3ce3d0ca64e31e75247644c9a46ce 9c37af5c1c6d62d20844a1f13b33c45e415a3055 98b06959a1d9de91d7c65a6491d461280d524b60 5366c00d77471b19badf16b8538bd8b74bdbbb91 c8cc61352a916478de98236803c6ca44605b3a66 68f4f8a8bf9bb317c67ac1322f1aa41680422df8 732d242066175d6f183fcc01caf1637e7f136be6 1bbdf6e4ce5a5d854e5f1b472af2c0ce58a4cba5 45b66ce9dc4a938fb93e94f69741581a24422784 045693d56269ce19aadf568552f22d46133f9632 714b4d6b129ba80811ebf5f449c238fbe5912f93 7cda9e0482c9280818978a5d1cba225c00ccb645 857a0dbc0d5684074269ba2862d9e7b9d00a45d0 e75fb162b1b7ceeae1e8dad92ed7ede977e05c8b 81bb62eb33d0952e982828c06eac7281ec35ca33 56a8425f61d034eddd7a3bdbb479038f135b9b96 b0bfdc5b73ebb8005b9148ee759b16033b458290 5267465a18ef8ff7bb24554a0e0ed4470fb3d0a6 2a4c642ffe919adb48a4f2058ad08f88bf335f57 489e092eb67f21cd078a326726ee0f36d726b1dc 7f9613c11119ee83427dd12fb45138f633261e89 fe5cbfece7bf1615b428255aa71b11fc6b4ecfcd b58a7f822e726b70acc4e7823edefd820514b25a 564610a456f0f4ab8d493a2b12f1ae13231caded 5db3a42379e42caa7f744437084aed5d19760566 7781e23422823e38e19bad64b995826e1c429363 4c9dfa4efe2d5ecfbce8b7ff8b1dff1d6cbd4f84 727fa19224b7e5305f20fdf1713a32e35acc1b49 f6cc343dd946f75d1c31a33cf674adbd93ae402a b88b5d1a1820aab4fa1b841584b60c5ed11b89c4 80b6e472fb5ad023ba99a5ba97205bdcfc99f7c5 99e52da9e711c4a0ba4a8d4183817af6c6c46d32 a2b170517b8425c8abdd3aa628405c0b753d6ce7 bb2af8446cf57726e4a900ca63e7bf9f82ad5639 9491e7c8905060234d37707cf00d0e74e455c26b fb33d11866ac9fc0e291ee3f0039c39970ba5bb6 1eea99c5cda0bdffbec92ce946e42699b8e147d7 1ffd33ddd7b7e8a58a28c12ff664e1963faf399d 10c268330940980c139aad706c6268cf8d85c3d3 49970265e0f28910457feadfe07424d8e1702eae 6863af6c17608d60c16868b35e7ef477a64260d3 2a5e889f9d3e7ae8d584e986a1c63844bf9b0ddb b55b56eda641a927ec21309b8d209f75fe373ac3 b916b3ba12c8268c2dd0d4174aa9c8417a207a7c 7ffe5de18e09db0356896173c72e081c75cde566 828ba05644f03f264b6a5419ecec7f00b1cbfdb2 -Depends: rocm-khronos-cts6.2.2 (= 3.0.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocm-khronos-cts-dbgsym6.2.2/rocm-khronos-cts-dbgsym6.2.2_3.0.0.60202-116~20.04_amd64.deb -Size: 46599844 -SHA256: c2ef89fac02a01d0bf31cd5779cfc9b349bf521da542a8bb6f1de0e752e45e19 -SHA1: bd58d39a7d252c664cc45cc4d4f7e84a0a885cce -MD5sum: 078be9726562359627b5efa3379e6415 -Description: debug symbols for rocm-khronos-cts -Maintainer: ROCm OpenCL Support -Package-Type: ddeb -Version: 3.0.0.60202-116~20.04 -Installed-Size: 341286 - -Package: rocm-khronos-cts-rpath6.2.2 -Architecture: amd64 -Depends: rocm-core-rpath6.2.2, rocm-opencl-icd-loader-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocm-khronos-cts-rpath6.2.2/rocm-khronos-cts-rpath6.2.2_3.0.0.60202-116~20.04_amd64.deb -Size: 3529220 -SHA256: e0429568803a6a5dd29fafca6e09b1c85bb3f1d957e1880cc2ca8b4822af01a0 -SHA1: 54a937327717dddb93f28f7a769cbb0815d753d6 -MD5sum: 1688cdbe2e9571b64970fb943052254e -Description: CLConform built using CMake -Maintainer: ROCm OpenCL Support -Version: 3.0.0.60202-116~20.04 -Installed-Size: 24079 - -Package: rocm-khronos-cts6.2.2 -Architecture: amd64 -Depends: rocm-core6.2.2, rocm-opencl-icd-loader6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocm-khronos-cts6.2.2/rocm-khronos-cts6.2.2_3.0.0.60202-116~20.04_amd64.deb -Size: 3531420 -SHA256: 72e5620d89a1a3efbe8cd4b0c1bd37f738d8394aef494d2051603bc328a1a38a -SHA1: eaa9e02d9ecb87c03bc84ff801f111240ff24168 -MD5sum: 18801dcb3b4d7396fd05f34aa581d475 -Description: CLConform built using CMake -Maintainer: ROCm OpenCL Support -Version: 3.0.0.60202-116~20.04 -Installed-Size: 24079 - -Package: rocm-language-runtime -Architecture: amd64 -Depends: comgr (= 2.8.0.60202-116~20.04), hsa-rocr (= 1.14.0.60202-116~20.04), rocm-core (= 6.2.2.60202-116~20.04), openmp-extras-runtime (= 18.62.0.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-language-runtime/rocm-language-runtime_6.2.2.60202-116~20.04_amd64.deb -Size: 834 -SHA256: d70626b42ee6951a18efe3875e36beb04ccecde77153d98a538802b1950334ad -SHA1: f1a4064e9ce55bf5320b81d546de0863972a5cc3 -MD5sum: 910f8fa2171e5f9e8c212c3f42705e25 -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-language-runtime-asan -Architecture: amd64 -Depends: rocm-language-runtime (= 6.2.2.60202-116~20.04), comgr-asan (= 2.8.0.60202-116~20.04), hsakmt-roct-asan (= 20240607.4.05.60202-116~20.04), hsa-rocr-asan (= 1.14.0.60202-116~20.04), openmp-extras-asan (= 18.62.0.60202-116~20.04), rocm-core-asan (= 6.2.2.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-language-runtime-asan/rocm-language-runtime-asan_6.2.2.60202-116~20.04_amd64.deb -Size: 874 -SHA256: be77f52763eda8f8509a4b9f865129052f2d391959d039aaae9aa5de7b759b88 -SHA1: 49f39d8432964941fb01db3ba76600f5d84bcdca -MD5sum: 2ca183d12214a71e0376216a73264bde -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-language-runtime-asan-rpath6.2.2 -Architecture: amd64 -Depends: rocm-language-runtime (= 6.2.2.60202-116~20.04), comgr-asan-rpath6.2.2 (= 2.8.0.60202-116~20.04), hsakmt-roct-asan-rpath6.2.2 (= 20240607.4.05.60202-116~20.04), hsa-rocr-asan-rpath6.2.2 (= 1.14.0.60202-116~20.04), openmp-extras-asan-rpath6.2.2 (= 18.62.0.60202-116~20.04), rocm-core-asan-rpath6.2.2 (= 6.2.2.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-language-runtime-asan-rpath6.2.2/rocm-language-runtime-asan-rpath6.2.2_6.2.2.60202-116~20.04_amd64.deb -Size: 1080 -SHA256: b0b24c2e7a66cbd917bcc17348bdfbd7d2d5e45e65d0a28c77727d0c0851d776 -SHA1: d9ba2089aec2b5545084a0b312075a804eaad503 -MD5sum: 7ffc9d4334ab8f7d744e9157653d5f30 -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-language-runtime-asan6.2.2 -Architecture: amd64 -Depends: rocm-language-runtime (= 6.2.2.60202-116~20.04), comgr-asan6.2.2 (= 2.8.0.60202-116~20.04), hsakmt-roct-asan6.2.2 (= 20240607.4.05.60202-116~20.04), hsa-rocr-asan6.2.2 (= 1.14.0.60202-116~20.04), openmp-extras-asan6.2.2 (= 18.62.0.60202-116~20.04), rocm-core-asan6.2.2 (= 6.2.2.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-language-runtime-asan6.2.2/rocm-language-runtime-asan6.2.2_6.2.2.60202-116~20.04_amd64.deb -Size: 1080 -SHA256: 2f3f10bcbea978dcfe2323aed80777f678e3b74445c381fa44488a50bfba884a -SHA1: 6d8066345c06ed6e9a5342705b2d19f5eabef8e5 -MD5sum: 6ca3a247541cf01677a7c966cc6e09a2 -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-language-runtime-rpath6.2.2 -Architecture: amd64 -Depends: comgr-rpath6.2.2 (= 2.8.0.60202-116~20.04), hsa-rocr-rpath6.2.2 (= 1.14.0.60202-116~20.04), rocm-core-rpath6.2.2 (= 6.2.2.60202-116~20.04), openmp-extras-runtime-rpath6.2.2 (= 18.62.0.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-language-runtime-rpath6.2.2/rocm-language-runtime-rpath6.2.2_6.2.2.60202-116~20.04_amd64.deb -Size: 1040 -SHA256: 3f5ae4b9ff1efeac8918634ffd5e922cb979d3dc96d028c0e6386ac7f001b5f4 -SHA1: 8a49bb39c9ec3de2947d5b4e08b12caeea85eba8 -MD5sum: be944d4f69c9af0e6ecdc60e201fa85b -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-language-runtime6.2.2 -Architecture: amd64 -Depends: comgr6.2.2 (= 2.8.0.60202-116~20.04), hsa-rocr6.2.2 (= 1.14.0.60202-116~20.04), rocm-core6.2.2 (= 6.2.2.60202-116~20.04), openmp-extras-runtime6.2.2 (= 18.62.0.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-language-runtime6.2.2/rocm-language-runtime6.2.2_6.2.2.60202-116~20.04_amd64.deb -Size: 1024 -SHA256: fd925ded8c216eab36bf131d851fcefa5018a7872a2b0f89a811159533fb3b15 -SHA1: 8023795166b439a6737e6a97a866346635b33124 -MD5sum: c25f4a647725933c270e627b8deefe25 -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-libs -Architecture: amd64 -Depends: hipblas (= 2.2.0.60202-116~20.04), hipblaslt (= 0.8.0.60202-116~20.04), hipfft (= 1.0.15.60202-116~20.04), hipsolver (= 2.2.0.60202-116~20.04), hipsparse (= 3.1.1.60202-116~20.04), hiptensor (= 1.3.0.60202-116~20.04), miopen-hip (= 3.2.0.60202-116~20.04), half (= 1.12.0.60202-116~20.04), rccl (= 2.20.5.60202-116~20.04), rocalution (= 3.2.0.60202-116~20.04), rocblas (= 4.2.1.60202-116~20.04), rocfft (= 1.0.29.60202-116~20.04), rocrand (= 3.1.0.60202-116~20.04), hiprand (= 2.11.0.60202-116~20.04), rocsolver (= 3.26.0.60202-116~20.04), rocsparse (= 3.2.0.60202-116~20.04), rocm-core (= 6.2.2.60202-116~20.04), hipsparselt (= 0.2.1.60202-116~20.04), composablekernel-dev (= 1.1.0.60202-116~20.04), hipblas-dev (= 2.2.0.60202-116~20.04), hipblaslt-dev (= 0.8.0.60202-116~20.04), hipcub-dev (= 3.2.0.60202-116~20.04), hipfft-dev (= 1.0.15.60202-116~20.04), hipsolver-dev (= 2.2.0.60202-116~20.04), hipsparse-dev (= 3.1.1.60202-116~20.04), hiptensor-dev (= 1.3.0.60202-116~20.04), miopen-hip-dev (= 3.2.0.60202-116~20.04), rccl-dev (= 2.20.5.60202-116~20.04), rocalution-dev (= 3.2.0.60202-116~20.04), rocblas-dev (= 4.2.1.60202-116~20.04), rocfft-dev (= 1.0.29.60202-116~20.04), rocprim-dev (= 3.2.0.60202-116~20.04), rocrand-dev (= 3.1.0.60202-116~20.04), hiprand-dev (= 2.11.0.60202-116~20.04), rocsolver-dev (= 3.26.0.60202-116~20.04), rocsparse-dev (= 3.2.0.60202-116~20.04), rocthrust-dev (= 3.1.0.60202-116~20.04), rocwmma-dev (= 1.5.0.60202-116~20.04), hipsparselt-dev (= 0.2.1.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-libs/rocm-libs_6.2.2.60202-116~20.04_amd64.deb -Size: 1064 -SHA256: 94781697b92f126c381331ef849a92cb1c5b8badb2d552e22e36216f6ef57ea0 -SHA1: 37fc15b1569772b60992949f48637185d783321f -MD5sum: 8d9d2fe65e76cdfbfb28d81322b5f8ef -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-libs-rpath6.2.2 -Architecture: amd64 -Depends: hipblas-rpath6.2.2 (= 2.2.0.60202-116~20.04), hipblaslt-rpath6.2.2 (= 0.8.0.60202-116~20.04), hipfft-rpath6.2.2 (= 1.0.15.60202-116~20.04), hipsolver-rpath6.2.2 (= 2.2.0.60202-116~20.04), hipsparse-rpath6.2.2 (= 3.1.1.60202-116~20.04), hiptensor-rpath6.2.2 (= 1.3.0.60202-116~20.04), miopen-hip-rpath6.2.2 (= 3.2.0.60202-116~20.04), half-rpath6.2.2 (= 1.12.0.60202-116~20.04), rccl-rpath6.2.2 (= 2.20.5.60202-116~20.04), rocalution-rpath6.2.2 (= 3.2.0.60202-116~20.04), rocblas-rpath6.2.2 (= 4.2.1.60202-116~20.04), rocfft-rpath6.2.2 (= 1.0.29.60202-116~20.04), rocrand-rpath6.2.2 (= 3.1.0.60202-116~20.04), hiprand-rpath6.2.2 (= 2.11.0.60202-116~20.04), rocsolver-rpath6.2.2 (= 3.26.0.60202-116~20.04), rocsparse-rpath6.2.2 (= 3.2.0.60202-116~20.04), rocm-core-rpath6.2.2 (= 6.2.2.60202-116~20.04), hipsparselt-rpath6.2.2 (= 0.2.1.60202-116~20.04), composablekernel-dev-rpath6.2.2 (= 1.1.0.60202-116~20.04), hipblas-dev-rpath6.2.2 (= 2.2.0.60202-116~20.04), hipblaslt-dev-rpath6.2.2 (= 0.8.0.60202-116~20.04), hipcub-dev-rpath6.2.2 (= 3.2.0.60202-116~20.04), hipfft-dev-rpath6.2.2 (= 1.0.15.60202-116~20.04), hipsolver-dev-rpath6.2.2 (= 2.2.0.60202-116~20.04), hipsparse-dev-rpath6.2.2 (= 3.1.1.60202-116~20.04), hiptensor-dev-rpath6.2.2 (= 1.3.0.60202-116~20.04), miopen-hip-dev-rpath6.2.2 (= 3.2.0.60202-116~20.04), rccl-dev-rpath6.2.2 (= 2.20.5.60202-116~20.04), rocalution-dev-rpath6.2.2 (= 3.2.0.60202-116~20.04), rocblas-dev-rpath6.2.2 (= 4.2.1.60202-116~20.04), rocfft-dev-rpath6.2.2 (= 1.0.29.60202-116~20.04), rocprim-dev-rpath6.2.2 (= 3.2.0.60202-116~20.04), rocrand-dev-rpath6.2.2 (= 3.1.0.60202-116~20.04), hiprand-dev-rpath6.2.2 (= 2.11.0.60202-116~20.04), rocsolver-dev-rpath6.2.2 (= 3.26.0.60202-116~20.04), rocsparse-dev-rpath6.2.2 (= 3.2.0.60202-116~20.04), rocthrust-dev-rpath6.2.2 (= 3.1.0.60202-116~20.04), rocwmma-dev-rpath6.2.2 (= 1.5.0.60202-116~20.04), hipsparselt-dev-rpath6.2.2 (= 0.2.1.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-libs-rpath6.2.2/rocm-libs-rpath6.2.2_6.2.2.60202-116~20.04_amd64.deb -Size: 1252 -SHA256: a3f6d02f6c0840062693f704ed8a0603a1abb79566721a808954d118027e6827 -SHA1: 637d0af951ad671e81c905faca11889cb5274140 -MD5sum: 77f12d38cc591076eb33a1f3f0fd8ea0 -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-libs6.2.2 -Architecture: amd64 -Depends: hipblas6.2.2 (= 2.2.0.60202-116~20.04), hipblaslt6.2.2 (= 0.8.0.60202-116~20.04), hipfft6.2.2 (= 1.0.15.60202-116~20.04), hipsolver6.2.2 (= 2.2.0.60202-116~20.04), hipsparse6.2.2 (= 3.1.1.60202-116~20.04), hiptensor6.2.2 (= 1.3.0.60202-116~20.04), miopen-hip6.2.2 (= 3.2.0.60202-116~20.04), half6.2.2 (= 1.12.0.60202-116~20.04), rccl6.2.2 (= 2.20.5.60202-116~20.04), rocalution6.2.2 (= 3.2.0.60202-116~20.04), rocblas6.2.2 (= 4.2.1.60202-116~20.04), rocfft6.2.2 (= 1.0.29.60202-116~20.04), rocrand6.2.2 (= 3.1.0.60202-116~20.04), hiprand6.2.2 (= 2.11.0.60202-116~20.04), rocsolver6.2.2 (= 3.26.0.60202-116~20.04), rocsparse6.2.2 (= 3.2.0.60202-116~20.04), rocm-core6.2.2 (= 6.2.2.60202-116~20.04), hipsparselt6.2.2 (= 0.2.1.60202-116~20.04), composablekernel-dev6.2.2 (= 1.1.0.60202-116~20.04), hipblas-dev6.2.2 (= 2.2.0.60202-116~20.04), hipblaslt-dev6.2.2 (= 0.8.0.60202-116~20.04), hipcub-dev6.2.2 (= 3.2.0.60202-116~20.04), hipfft-dev6.2.2 (= 1.0.15.60202-116~20.04), hipsolver-dev6.2.2 (= 2.2.0.60202-116~20.04), hipsparse-dev6.2.2 (= 3.1.1.60202-116~20.04), hiptensor-dev6.2.2 (= 1.3.0.60202-116~20.04), miopen-hip-dev6.2.2 (= 3.2.0.60202-116~20.04), rccl-dev6.2.2 (= 2.20.5.60202-116~20.04), rocalution-dev6.2.2 (= 3.2.0.60202-116~20.04), rocblas-dev6.2.2 (= 4.2.1.60202-116~20.04), rocfft-dev6.2.2 (= 1.0.29.60202-116~20.04), rocprim-dev6.2.2 (= 3.2.0.60202-116~20.04), rocrand-dev6.2.2 (= 3.1.0.60202-116~20.04), hiprand-dev6.2.2 (= 2.11.0.60202-116~20.04), rocsolver-dev6.2.2 (= 3.26.0.60202-116~20.04), rocsparse-dev6.2.2 (= 3.2.0.60202-116~20.04), rocthrust-dev6.2.2 (= 3.1.0.60202-116~20.04), rocwmma-dev6.2.2 (= 1.5.0.60202-116~20.04), hipsparselt-dev6.2.2 (= 0.2.1.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-libs6.2.2/rocm-libs6.2.2_6.2.2.60202-116~20.04_amd64.deb -Size: 1244 -SHA256: cf1097a840bc86f0b8fe89a2c3810b63d6522a9aa88f680e0538589df66413f4 -SHA1: cfd6203eb56e9ccc442f46bdb0dc63c9b0639bd7 -MD5sum: a2156c47f49a270b0cf965a7ad562f21 -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-llvm -Architecture: amd64 -Maintainer: ROCm Compiler Support -Version: 18.0.0.24355.60202-116~20.04 -Release: 116~20.04 -Depends: python3, libc6, libstdc++6|libstdc++8, libstdc++-5-dev|libstdc++-7-dev|libstdc++-11-dev, libgcc-5-dev|libgcc-7-dev|libgcc-11-dev, rocm-core -Recommends: gcc, g++, gcc-multilib, g++-multilib -Priority: optional -Section: devel -Filename: pool/main/r/rocm-llvm/rocm-llvm_18.0.0.24355.60202-116~20.04_amd64.deb -Size: 327564920 -SHA256: a80f3fa79d97459481876f6f1c14790a3a9267b0bde380f4851b33cd61af0d2f -SHA1: dc36c437538714e653e016d45b389a5bc03effdd -MD5sum: e3cc584792933ec7983e941e1134f720 -Description: ROCm core compiler - ROCm core compiler based on LLVM 18.0.0 - -Package: rocm-llvm-dev -Architecture: amd64 -Maintainer: ROCm Compiler Support -Version: 18.0.0.24355.60202-116~20.04 -Release: 116~20.04 -Depends: python3, libc6, libstdc++6|libstdc++8, libstdc++-5-dev|libstdc++-7-dev|libstdc++-11-dev, libgcc-5-dev|libgcc-7-dev|libgcc-11-dev, rocm-core, rocm-llvm -Recommends: gcc, g++, gcc-multilib, g++-multilib -Priority: optional -Section: devel -Filename: pool/main/r/rocm-llvm-dev/rocm-llvm-dev_18.0.0.24355.60202-116~20.04_amd64.deb -Size: 1012488664 -SHA256: 0046becdb84e7fca8c2e18694f020daa1ec425f8f7a671f5ea8f842a09897753 -SHA1: 316f7d63dbe361665b774e676792dfda79a63fe6 -MD5sum: e3f6db6b727f46aad8215a9a58258e48 -Description: ROCm compiler dev tools - ROCm compiler dev tools and documentation, based on LLVM 18.0.0 - -Package: rocm-llvm-dev-rpath6.2.2 -Architecture: amd64 -Maintainer: ROCm Compiler Support -Version: 18.0.0.24355.60202-116~20.04 -Release: 116~20.04 -Depends: python3, libc6, libstdc++6|libstdc++8, libstdc++-5-dev|libstdc++-7-dev|libstdc++-11-dev, libgcc-5-dev|libgcc-7-dev|libgcc-11-dev, rocm-core-rpath6.2.2, rocm-llvm-rpath6.2.2 -Recommends: gcc, g++, gcc-multilib, g++-multilib -Priority: optional -Section: devel -Filename: pool/main/r/rocm-llvm-dev-rpath6.2.2/rocm-llvm-dev-rpath6.2.2_18.0.0.24355.60202-116~20.04_amd64.deb -Size: 653705492 -SHA256: 107c50cd875671a47d4e865dcd232647e5912b4ae1d2fe72f1955aa786721f4a -SHA1: f175b5f97a4ffd268b48704825c36131e982a70d -MD5sum: 3a81815a9674904bf27d71f66db7a369 -Description: ROCm compiler dev tools - ROCm compiler dev tools and documentation, based on LLVM 18.0.0 - -Package: rocm-llvm-dev6.2.2 -Architecture: amd64 -Maintainer: ROCm Compiler Support -Version: 18.0.0.24355.60202-116~20.04 -Release: 116~20.04 -Depends: python3, libc6, libstdc++6|libstdc++8, libstdc++-5-dev|libstdc++-7-dev|libstdc++-11-dev, libgcc-5-dev|libgcc-7-dev|libgcc-11-dev, rocm-core6.2.2, rocm-llvm6.2.2 -Recommends: gcc, g++, gcc-multilib, g++-multilib -Priority: optional -Section: devel -Filename: pool/main/r/rocm-llvm-dev6.2.2/rocm-llvm-dev6.2.2_18.0.0.24355.60202-116~20.04_amd64.deb -Size: 653704248 -SHA256: daf5e14c527d27abdc1ae5d80ee5938b22db0387c5a6b05622b4fc46069c75e4 -SHA1: 6e405178628f3e8701a6c7d263020b26b8fabfd2 -MD5sum: a30eea81b82deebf5683fc268ad39a75 -Description: ROCm compiler dev tools - ROCm compiler dev tools and documentation, based on LLVM 18.0.0 - -Package: rocm-llvm-docs -Architecture: amd64 -Maintainer: ROCm Compiler Support -Version: 18.0.0.24355.60202-116~20.04 -Release: 116~20.04 -Depends: rocm-core -Recommends: -Priority: optional -Section: devel -Filename: pool/main/r/rocm-llvm-docs/rocm-llvm-docs_18.0.0.24355.60202-116~20.04_amd64.deb -Size: 12956632 -SHA256: 272869b7982e8be76deacb6c099935e41857b4fa2dbc5e0d22afdd43010c5bd0 -SHA1: b9ba41b0f9481f67b00699acfd2e80226f2d8e61 -MD5sum: 92c1a88002ec27cc8903f584f9f7b9d0 -Description: ROCm LLVM compiler documentation - Documenation for LLVM 18.0.0 - -Package: rocm-llvm-docs-rpath6.2.2 -Architecture: amd64 -Maintainer: ROCm Compiler Support -Version: 18.0.0.24355.60202-116~20.04 -Release: 116~20.04 -Depends: rocm-core-rpath6.2.2 -Recommends: -Priority: optional -Section: devel -Filename: pool/main/r/rocm-llvm-docs-rpath6.2.2/rocm-llvm-docs-rpath6.2.2_18.0.0.24355.60202-116~20.04_amd64.deb -Size: 9544768 -SHA256: 8deed48a88b0cb372e02c07ef3e2949c02cd2d39360159445779d8bc49e346c6 -SHA1: e979050ab2ddcc33b533a286b017ef7d48b7bc17 -MD5sum: d24fc086f91515a127a921090d398496 -Description: ROCm LLVM compiler documentation - Documenation for LLVM 18.0.0 - -Package: rocm-llvm-docs6.2.2 -Architecture: amd64 -Maintainer: ROCm Compiler Support -Version: 18.0.0.24355.60202-116~20.04 -Release: 116~20.04 -Depends: rocm-core6.2.2 -Recommends: -Priority: optional -Section: devel -Filename: pool/main/r/rocm-llvm-docs6.2.2/rocm-llvm-docs6.2.2_18.0.0.24355.60202-116~20.04_amd64.deb -Size: 9544944 -SHA256: 9a54aa9b88be30c673ff3a3cd07e4296f16964bcbaf62697808f74d18cc7086c -SHA1: 649af2a9e54aa69a1b37f611b7f94a695fda6982 -MD5sum: b421a8cb87dca18d3bb5505687d9dd19 -Description: ROCm LLVM compiler documentation - Documenation for LLVM 18.0.0 - -Package: rocm-llvm-rpath6.2.2 -Architecture: amd64 -Maintainer: ROCm Compiler Support -Version: 18.0.0.24355.60202-116~20.04 -Release: 116~20.04 -Depends: python3, libc6, libstdc++6|libstdc++8, libstdc++-5-dev|libstdc++-7-dev|libstdc++-11-dev, libgcc-5-dev|libgcc-7-dev|libgcc-11-dev, rocm-core-rpath6.2.2 -Recommends: gcc, g++, gcc-multilib, g++-multilib -Priority: optional -Section: devel -Filename: pool/main/r/rocm-llvm-rpath6.2.2/rocm-llvm-rpath6.2.2_18.0.0.24355.60202-116~20.04_amd64.deb -Size: 215539620 -SHA256: 5a4eb6fa0893fb4090412e68cc60727ad425ee765c90a90667be464c0154fee5 -SHA1: a9c767b1653b7274e6cc7fda2d3d4f7d542ec2d6 -MD5sum: 4d82db5cc5e5dd7c92fa2633ef1594a0 -Description: ROCm core compiler - ROCm core compiler based on LLVM 18.0.0 - -Package: rocm-llvm6.2.2 -Architecture: amd64 -Maintainer: ROCm Compiler Support -Version: 18.0.0.24355.60202-116~20.04 -Release: 116~20.04 -Depends: python3, libc6, libstdc++6|libstdc++8, libstdc++-5-dev|libstdc++-7-dev|libstdc++-11-dev, libgcc-5-dev|libgcc-7-dev|libgcc-11-dev, rocm-core6.2.2 -Recommends: gcc, g++, gcc-multilib, g++-multilib -Priority: optional -Section: devel -Filename: pool/main/r/rocm-llvm6.2.2/rocm-llvm6.2.2_18.0.0.24355.60202-116~20.04_amd64.deb -Size: 215534508 -SHA256: cd75c714026dd11be0340edd0fe4e23bdba453d596df8b72fc2c172aa9c5b147 -SHA1: 9c7359a68540db7920192cf91832a29083f088ce -MD5sum: bf2617f96b9c23fdb5a39eda87011e82 -Description: ROCm core compiler - ROCm core compiler based on LLVM 18.0.0 - -Package: rocm-ml-libraries -Architecture: amd64 -Depends: rocm-hip-libraries (= 6.2.2.60202-116~20.04), miopen-hip (= 3.2.0.60202-116~20.04), half (= 1.12.0.60202-116~20.04), rocm-llvm (= 18.0.0.24355.60202-116~20.04), rocm-core (= 6.2.2.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-ml-libraries/rocm-ml-libraries_6.2.2.60202-116~20.04_amd64.deb -Size: 850 -SHA256: 414f9da69a8e38e7d57c63df9c703754f23321de046ea7d03e3b403c9cf40d64 -SHA1: 11cde7b5535da212c01841335b8ee63202c80eb4 -MD5sum: 8cf8a628e98e9dfcdb4baf108b8433d7 -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-ml-libraries-asan -Architecture: amd64 -Depends: rocm-ml-libraries (= 6.2.2.60202-116~20.04), rocm-core-asan (= 6.2.2.60202-116~20.04), rocm-hip-libraries-asan (= 6.2.2.60202-116~20.04), miopen-hip-asan (= 3.2.0.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-ml-libraries-asan/rocm-ml-libraries-asan_6.2.2.60202-116~20.04_amd64.deb -Size: 840 -SHA256: d30b086cb9c304a57387314e73f0ef03a194cc8f53a8d97d8d6e15d1a69072c3 -SHA1: c975fb93d04a46d8bd4b04e2773d316635db3d3b -MD5sum: 03ad7270d28de19b477439a63c8ba5b2 -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-ml-libraries-asan-rpath6.2.2 -Architecture: amd64 -Depends: rocm-ml-libraries (= 6.2.2.60202-116~20.04), rocm-core-asan-rpath6.2.2 (= 6.2.2.60202-116~20.04), rocm-hip-libraries-asan-rpath6.2.2 (= 6.2.2.60202-116~20.04), miopen-hip-asan-rpath6.2.2 (= 3.2.0.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-ml-libraries-asan-rpath6.2.2/rocm-ml-libraries-asan-rpath6.2.2_6.2.2.60202-116~20.04_amd64.deb -Size: 1044 -SHA256: c0dae7ceaf479c8224c7e7bab98c456be88a2fd12e3ac99301a4114f39a3824f -SHA1: 40203c3be76d49b16a38cf80139767bc8b9356db -MD5sum: a87ba56e997ac84a470a7a8d9a4757ab -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-ml-libraries-asan6.2.2 -Architecture: amd64 -Depends: rocm-ml-libraries (= 6.2.2.60202-116~20.04), rocm-core-asan6.2.2 (= 6.2.2.60202-116~20.04), rocm-hip-libraries-asan6.2.2 (= 6.2.2.60202-116~20.04), miopen-hip-asan6.2.2 (= 3.2.0.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-ml-libraries-asan6.2.2/rocm-ml-libraries-asan6.2.2_6.2.2.60202-116~20.04_amd64.deb -Size: 1044 -SHA256: 63a67f10d5c920d6fe929349c117c27a46dd55cb96513ea20cbfebdd693af75e -SHA1: 18c1d77f340523ce243dd13f9325082352d79cca -MD5sum: 8a29d2a290aba510d6e87ab1385ae66c -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-ml-libraries-rpath6.2.2 -Architecture: amd64 -Depends: rocm-hip-libraries-rpath6.2.2 (= 6.2.2.60202-116~20.04), miopen-hip-rpath6.2.2 (= 3.2.0.60202-116~20.04), half-rpath6.2.2 (= 1.12.0.60202-116~20.04), rocm-llvm-rpath6.2.2 (= 18.0.0.24355.60202-116~20.04), rocm-core-rpath6.2.2 (= 6.2.2.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-ml-libraries-rpath6.2.2/rocm-ml-libraries-rpath6.2.2_6.2.2.60202-116~20.04_amd64.deb -Size: 1052 -SHA256: 32d0e7377c7dacf0ea1e1086e58da147c7bd1c4a135cb44e2461ff7ef14db984 -SHA1: beec11d090828f388647da15912bfc04db120b57 -MD5sum: 16d0c58a60954936b0cfc97e7999de7e -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-ml-libraries6.2.2 -Architecture: amd64 -Depends: rocm-hip-libraries6.2.2 (= 6.2.2.60202-116~20.04), miopen-hip6.2.2 (= 3.2.0.60202-116~20.04), half6.2.2 (= 1.12.0.60202-116~20.04), rocm-llvm6.2.2 (= 18.0.0.24355.60202-116~20.04), rocm-core6.2.2 (= 6.2.2.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-ml-libraries6.2.2/rocm-ml-libraries6.2.2_6.2.2.60202-116~20.04_amd64.deb -Size: 1048 -SHA256: 38171cd7dadc944d894cdc05baa6a6282950b5fbad268f3c4568297659ed9765 -SHA1: 4fd92f7cd307475984b60eb1dd28ca6cb75b3cab -MD5sum: 291fc31fd70e39cddac10c0e3631ae6a -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-ml-sdk -Architecture: amd64 -Depends: rocm-ml-libraries (= 6.2.2.60202-116~20.04), rocm-hip-sdk (= 6.2.2.60202-116~20.04), rocm-core (= 6.2.2.60202-116~20.04), miopen-hip-dev (= 3.2.0.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-ml-sdk/rocm-ml-sdk_6.2.2.60202-116~20.04_amd64.deb -Size: 828 -SHA256: 8527c4d090d2161b05d6226a8f3f900d0f4c1e664c3f1986521322f3f32986f9 -SHA1: da1e11bb7dc8471943aff4bbb4dcd4353a3caed1 -MD5sum: 70c84e26f5a9bfca6648c995a1b44e5f -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-ml-sdk-asan -Architecture: amd64 -Depends: rocm-ml-sdk (= 6.2.2.60202-116~20.04), rocm-core-asan (= 6.2.2.60202-116~20.04), rocm-ml-libraries-asan (= 6.2.2.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-ml-sdk-asan/rocm-ml-sdk-asan_6.2.2.60202-116~20.04_amd64.deb -Size: 822 -SHA256: a030495846c5542a77d1528131cbf4dac3a5fda7b45f4c999f4ccfb0f3127c27 -SHA1: 234f3e70c8ebfc9a8bf3989be3629663f4c30a0f -MD5sum: a79ba97ed282d36c689d554f35f67f27 -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-ml-sdk-asan-rpath6.2.2 -Architecture: amd64 -Depends: rocm-ml-sdk (= 6.2.2.60202-116~20.04), rocm-core-asan-rpath6.2.2 (= 6.2.2.60202-116~20.04), rocm-ml-libraries-asan-rpath6.2.2 (= 6.2.2.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-ml-sdk-asan-rpath6.2.2/rocm-ml-sdk-asan-rpath6.2.2_6.2.2.60202-116~20.04_amd64.deb -Size: 1016 -SHA256: 15b9851e62b8c49f5b07b8b5cb7e8da7e18853e0e14bce8aae1b9fc39e357724 -SHA1: 486c18417cf476834161094d19f7cd7f00581553 -MD5sum: d0fe513b45f4637b38175d4cc3182186 -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-ml-sdk-asan6.2.2 -Architecture: amd64 -Depends: rocm-ml-sdk (= 6.2.2.60202-116~20.04), rocm-core-asan6.2.2 (= 6.2.2.60202-116~20.04), rocm-ml-libraries-asan6.2.2 (= 6.2.2.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-ml-sdk-asan6.2.2/rocm-ml-sdk-asan6.2.2_6.2.2.60202-116~20.04_amd64.deb -Size: 1012 -SHA256: 78f71aa3daecf6ac41d7c1ddef1df9c13fca1863aee3555c5a2a3c1d5b338e13 -SHA1: 9e9ae75c43e9445d2bccc3181988a5b0a5ac89e7 -MD5sum: da23e25c6f951e827b2bc7f5e4859144 -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-ml-sdk-rpath6.2.2 -Architecture: amd64 -Depends: rocm-ml-libraries-rpath6.2.2 (= 6.2.2.60202-116~20.04), rocm-hip-sdk-rpath6.2.2 (= 6.2.2.60202-116~20.04), rocm-core-rpath6.2.2 (= 6.2.2.60202-116~20.04), miopen-hip-dev-rpath6.2.2 (= 3.2.0.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-ml-sdk-rpath6.2.2/rocm-ml-sdk-rpath6.2.2_6.2.2.60202-116~20.04_amd64.deb -Size: 1032 -SHA256: b17ec0ade45a058bcb9e744b0502af228c9d63eec152df4c5d57e57d6178a6ed -SHA1: 09f5faf058c122415775495472a5193eeb70194f -MD5sum: 628cc03e463baaa899f8f270c97a71b2 -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-ml-sdk6.2.2 -Architecture: amd64 -Depends: rocm-ml-libraries6.2.2 (= 6.2.2.60202-116~20.04), rocm-hip-sdk6.2.2 (= 6.2.2.60202-116~20.04), rocm-core6.2.2 (= 6.2.2.60202-116~20.04), miopen-hip-dev6.2.2 (= 3.2.0.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-ml-sdk6.2.2/rocm-ml-sdk6.2.2_6.2.2.60202-116~20.04_amd64.deb -Size: 1012 -SHA256: 96c7c1284c0d22cbd3acd3e4bf0edfcde382b978a02e71fb56c324308004e3fa -SHA1: 711e8be9bd20599c17fd6f5b6aec786b9c18c8e6 -MD5sum: a58cea04def2475f5977c74d3dae0b49 -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-ocltst -Architecture: amd64 -Priority: optional -Section: devel -Filename: pool/main/r/rocm-ocltst/rocm-ocltst_2.0.0.60202-116~20.04_amd64.deb -Size: 677812 -SHA256: 36df6aec7734cf7b5ecd1b674a3a1d6d62abb04965ba33433dee120cbf627a41 -SHA1: 3570d0e4f452e0a10b24b38dd64069f9191cdd98 -MD5sum: 2ead6c0e4211a379e5b46e3c4a2c2712 -Description: clr built using CMake -Maintainer: ROCm OpenCL Support -Version: 2.0.0.60202-116~20.04 -Installed-Size: 2209 - -Package: rocm-ocltst-dbgsym -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 7b00dd458c5bf4acf47307d02ad600fb4da2a7c3 1a7bbc156b190024eb70dfd4e10fad9dfba801b8 c0657ade7075fe3c1992d05dd5c2dfcab23d33e8 25ebf6a0784117e306d7e37527abe83a90575f79 -Depends: rocm-ocltst (= 2.0.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocm-ocltst-dbgsym/rocm-ocltst-dbgsym_2.0.0.60202-116~20.04_amd64.deb -Size: 8649782 -SHA256: 9337498f9dd9a4a6e92c2bdd716e557a8221f5cb18917e4d78922f2d965bc14d -SHA1: 9881cc506d670ca002f4ee479ee25307e4e6a687 -MD5sum: bcf4acf118f774b9f6a04d113501e96f -Description: debug symbols for rocm-ocltst -Maintainer: ROCm OpenCL Support -Package-Type: ddeb -Version: 2.0.0.60202-116~20.04 -Installed-Size: 29495 - -Package: rocm-ocltst-dbgsym-rpath6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 7b00dd458c5bf4acf47307d02ad600fb4da2a7c3 1a7bbc156b190024eb70dfd4e10fad9dfba801b8 c0657ade7075fe3c1992d05dd5c2dfcab23d33e8 25ebf6a0784117e306d7e37527abe83a90575f79 -Depends: rocm-ocltst-rpath6.2.2 (= 2.0.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocm-ocltst-dbgsym-rpath6.2.2/rocm-ocltst-dbgsym-rpath6.2.2_2.0.0.60202-116~20.04_amd64.deb -Size: 4871348 -SHA256: fec5d5685375968856446f4192b9131092ff72f52bf4dfd9688b255500e3e239 -SHA1: e776b16b38ef70c44af61c3933fc4b8ccefd641d -MD5sum: b989270b387e1e630ccd0fe1b5336b6c -Description: debug symbols for rocm-ocltst -Maintainer: ROCm OpenCL Support -Package-Type: ddeb -Version: 2.0.0.60202-116~20.04 -Installed-Size: 29495 - -Package: rocm-ocltst-dbgsym6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 7b00dd458c5bf4acf47307d02ad600fb4da2a7c3 1a7bbc156b190024eb70dfd4e10fad9dfba801b8 c0657ade7075fe3c1992d05dd5c2dfcab23d33e8 25ebf6a0784117e306d7e37527abe83a90575f79 -Depends: rocm-ocltst6.2.2 (= 2.0.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocm-ocltst-dbgsym6.2.2/rocm-ocltst-dbgsym6.2.2_2.0.0.60202-116~20.04_amd64.deb -Size: 4869036 -SHA256: 3d23ff97a3c29867518b1ef0bc19fbcdce77eb7ff58a2c04feef22d3fd83e3a2 -SHA1: 34096b8c444f7f3b2937a751cf1a87da982531d6 -MD5sum: 6c3c500dec9410a3b060d6efeea32f48 -Description: debug symbols for rocm-ocltst -Maintainer: ROCm OpenCL Support -Package-Type: ddeb -Version: 2.0.0.60202-116~20.04 -Installed-Size: 29495 - -Package: rocm-ocltst-rpath6.2.2 -Architecture: amd64 -Priority: optional -Section: devel -Filename: pool/main/r/rocm-ocltst-rpath6.2.2/rocm-ocltst-rpath6.2.2_2.0.0.60202-116~20.04_amd64.deb -Size: 453312 -SHA256: 3829cabf131c3f3ab08d9c5ac4ee9c35c41f96dc8825864ef71a1c51259d32b3 -SHA1: ce07921654be1d7286d83fb7b3b014d6200fc406 -MD5sum: a437fdcae67067d0beb0b15a3a5cb305 -Description: clr built using CMake -Maintainer: ROCm OpenCL Support -Version: 2.0.0.60202-116~20.04 -Installed-Size: 2209 - -Package: rocm-ocltst6.2.2 -Architecture: amd64 -Priority: optional -Section: devel -Filename: pool/main/r/rocm-ocltst6.2.2/rocm-ocltst6.2.2_2.0.0.60202-116~20.04_amd64.deb -Size: 453164 -SHA256: 8e0ad516c4a78ef6a61cce9067ac2f662c5efdf06ec2a854ac0611d3e779da42 -SHA1: b11b212a1c8079f6576a32699aee94a8d998eb0a -MD5sum: 4912342b52b634f20a78537447a4f9f5 -Description: clr built using CMake -Maintainer: ROCm OpenCL Support -Version: 2.0.0.60202-116~20.04 -Installed-Size: 2209 - -Package: rocm-opencl -Architecture: amd64 -Depends: libelf-dev, comgr, hsa-rocr, rocm-core -Priority: optional -Section: devel -Filename: pool/main/r/rocm-opencl/rocm-opencl_2.0.0.60202-116~20.04_amd64.deb -Size: 589190 -SHA256: e6273d47749fd301526ae4fba559f65651c6df449c724ca27ab83d0e2d0ca2e6 -SHA1: d5e6eff01f8034220d79d32603de908c2a616684 -MD5sum: 58c868d8680350372f2d98dce27490a9 -Description: clr built using CMake -Maintainer: ROCm OpenCL Support -Version: 2.0.0.60202-116~20.04 -Installed-Size: 4351 - -Package: rocm-opencl-asan -Architecture: amd64 -Depends: libelf-dev, comgr-asan, hsa-rocr-asan, rocm-core-asan -Priority: optional -Section: devel -Filename: pool/main/r/rocm-opencl-asan/rocm-opencl-asan_2.0.0.60202-116~20.04_amd64.deb -Size: 1542798 -SHA256: 2e517e7ffe752e7bb22ecf1af414cac081b1cd5d5a4f61d41abea9a2d34000e1 -SHA1: 202468b660438c9c4eb3f29ef0dfc9e1c5668965 -MD5sum: e28db0e96dc11c8d79603ee3e3cda386 -Description: clr built using CMake -Maintainer: ROCm OpenCL Support -Version: 2.0.0.60202-116~20.04 -Installed-Size: 15345 - -Package: rocm-opencl-asan-dbgsym -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: eef7ad2aed0961302b2d20de7af165eee5fb1da8 -Depends: rocm-opencl-asan (= 2.0.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocm-opencl-asan-dbgsym/rocm-opencl-asan-dbgsym_2.0.0.60202-116~20.04_amd64.deb -Size: 8195724 -SHA256: d592feee75938cdd21a887e0db101e8185ad11cbbf027ac3717d0d3ab2731397 -SHA1: b9e1772e7588e5e7966a6e394cbb6f25b024b2d6 -MD5sum: 58c03c41ee24c75e6b357443a8746282 -Description: debug symbols for rocm-opencl-asan -Maintainer: ROCm OpenCL Support -Package-Type: ddeb -Version: 2.0.0.60202-116~20.04 -Installed-Size: 24929 - -Package: rocm-opencl-asan-dbgsym-rpath6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: eef7ad2aed0961302b2d20de7af165eee5fb1da8 -Depends: rocm-opencl-asan-rpath6.2.2 (= 2.0.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocm-opencl-asan-dbgsym-rpath6.2.2/rocm-opencl-asan-dbgsym-rpath6.2.2_2.0.0.60202-116~20.04_amd64.deb -Size: 4669352 -SHA256: 7dc512b20d18c99697c895081443c68ab8b109c88706bf51760c923e6758a73a -SHA1: bd8be1eed4968d1786efa3718e696196903b4df9 -MD5sum: 691f1164ae5a0c8d5a3c24c6c8c591dc -Description: debug symbols for rocm-opencl-asan -Maintainer: ROCm OpenCL Support -Package-Type: ddeb -Version: 2.0.0.60202-116~20.04 -Installed-Size: 24929 - -Package: rocm-opencl-asan-dbgsym6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: eef7ad2aed0961302b2d20de7af165eee5fb1da8 -Depends: rocm-opencl-asan6.2.2 (= 2.0.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocm-opencl-asan-dbgsym6.2.2/rocm-opencl-asan-dbgsym6.2.2_2.0.0.60202-116~20.04_amd64.deb -Size: 4667600 -SHA256: e2ee9f12804f0357a82fe3e95957e3efc56256c6e13951228a2aa0309b4448b7 -SHA1: 409ab69ce6239e6377334a697dee56d806443f76 -MD5sum: c4782c1fe8dbe03f6f8bb1b16cc6ad39 -Description: debug symbols for rocm-opencl-asan -Maintainer: ROCm OpenCL Support -Package-Type: ddeb -Version: 2.0.0.60202-116~20.04 -Installed-Size: 24929 - -Package: rocm-opencl-asan-rpath6.2.2 -Architecture: amd64 -Depends: libelf-dev, comgr-asan-rpath6.2.2, hsa-rocr-asan-rpath6.2.2, rocm-core-asan-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocm-opencl-asan-rpath6.2.2/rocm-opencl-asan-rpath6.2.2_2.0.0.60202-116~20.04_amd64.deb -Size: 938532 -SHA256: e1e6f0acce6644225dc3f3a112955c9f9469316425ee8da80e413fa7f8e444ec -SHA1: 5f4219fc945345aaa239847f4161dc87cc425cd2 -MD5sum: 3d88eac9757584e54b0083af201db6d4 -Description: clr built using CMake -Maintainer: ROCm OpenCL Support -Version: 2.0.0.60202-116~20.04 -Installed-Size: 15345 - -Package: rocm-opencl-asan6.2.2 -Architecture: amd64 -Depends: libelf-dev, comgr-asan6.2.2, hsa-rocr-asan6.2.2, rocm-core-asan6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocm-opencl-asan6.2.2/rocm-opencl-asan6.2.2_2.0.0.60202-116~20.04_amd64.deb -Size: 941016 -SHA256: 7ce43cf280c5542c00e73d570d8fc8b0af82447c3db6eb595eac4691209e2c8c -SHA1: d04f15fa3de150567425dcd114c221478e6ba5b6 -MD5sum: 743d9861f2c9efc9070512fed7e5b1de -Description: clr built using CMake -Maintainer: ROCm OpenCL Support -Version: 2.0.0.60202-116~20.04 -Installed-Size: 15345 - -Package: rocm-opencl-dbgsym -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: ad8d7102dab3181e2c606998a729eef74c074b1a b33aa63193be865497dc932332ae65c3627a3ff2 -Depends: rocm-opencl (= 2.0.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocm-opencl-dbgsym/rocm-opencl-dbgsym_2.0.0.60202-116~20.04_amd64.deb -Size: 15213280 -SHA256: 2ea553fcff5a942c7dd251c7f358ced77c0f7b819b0ad6a91576b944cfacabd3 -SHA1: fd45d857f8f6c2259c7b237701ed84fd5efe7e8a -MD5sum: d2486256247f951adecbe2a9bebb1274 -Description: debug symbols for rocm-opencl -Maintainer: ROCm OpenCL Support -Package-Type: ddeb -Version: 2.0.0.60202-116~20.04 -Installed-Size: 47985 - -Package: rocm-opencl-dbgsym-rpath6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: ad8d7102dab3181e2c606998a729eef74c074b1a b33aa63193be865497dc932332ae65c3627a3ff2 -Depends: rocm-opencl-rpath6.2.2 (= 2.0.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocm-opencl-dbgsym-rpath6.2.2/rocm-opencl-dbgsym-rpath6.2.2_2.0.0.60202-116~20.04_amd64.deb -Size: 9381320 -SHA256: 1c5ddd5735ea60f6942d7ea0d82f06da6ae3053d58d600aedc4f74d336347b7e -SHA1: 401823f906927764e9adbf228adaa4591b5b6ac3 -MD5sum: b09d383a20fe6cfced9de045995c9348 -Description: debug symbols for rocm-opencl -Maintainer: ROCm OpenCL Support -Package-Type: ddeb -Version: 2.0.0.60202-116~20.04 -Installed-Size: 47985 - -Package: rocm-opencl-dbgsym6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: ad8d7102dab3181e2c606998a729eef74c074b1a b33aa63193be865497dc932332ae65c3627a3ff2 -Depends: rocm-opencl6.2.2 (= 2.0.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocm-opencl-dbgsym6.2.2/rocm-opencl-dbgsym6.2.2_2.0.0.60202-116~20.04_amd64.deb -Size: 9376060 -SHA256: 200a114e740722922baa83f93e18e3cea781cadc1872eae0eaac3b7ab6a95780 -SHA1: c8ee841992125bfeb19d4656889c4f50fbbaa2a5 -MD5sum: 1bbf02f2b80ac10646db9f4c1651a572 -Description: debug symbols for rocm-opencl -Maintainer: ROCm OpenCL Support -Package-Type: ddeb -Version: 2.0.0.60202-116~20.04 -Installed-Size: 47985 - -Package: rocm-opencl-dev -Architecture: amd64 -Depends: mesa-common-dev, rocm-opencl, hsa-rocr-dev, rocm-core -Priority: optional -Section: devel -Filename: pool/main/r/rocm-opencl-dev/rocm-opencl-dev_2.0.0.60202-116~20.04_amd64.deb -Size: 121372 -SHA256: 9fa32e075a591e6823d75543e1b93a070d9054301091ae977bdedc03d975d061 -SHA1: 55b46bcba1f42e85b978e26795f4ded551922634 -MD5sum: daf6cfd1611249a64632b6803d5c1fad -Description: clr built using CMake -Maintainer: ROCm OpenCL Support -Version: 2.0.0.60202-116~20.04 -Installed-Size: 854 - -Package: rocm-opencl-dev-rpath6.2.2 -Architecture: amd64 -Depends: mesa-common-dev, rocm-opencl-rpath6.2.2, hsa-rocr-dev-rpath6.2.2, rocm-core-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocm-opencl-dev-rpath6.2.2/rocm-opencl-dev-rpath6.2.2_2.0.0.60202-116~20.04_amd64.deb -Size: 77064 -SHA256: 3287b20310729d2813a50cb7ef5dc67c2427fb3519a404dcdae9a8bdd75614af -SHA1: 0a1de5bde7ad7724ab993c510f29cacbd84bbf06 -MD5sum: 1188a9af75883fd2f8b759c976c5c634 -Description: clr built using CMake -Maintainer: ROCm OpenCL Support -Version: 2.0.0.60202-116~20.04 -Installed-Size: 854 - -Package: rocm-opencl-dev6.2.2 -Architecture: amd64 -Depends: mesa-common-dev, rocm-opencl6.2.2, hsa-rocr-dev6.2.2, rocm-core6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocm-opencl-dev6.2.2/rocm-opencl-dev6.2.2_2.0.0.60202-116~20.04_amd64.deb -Size: 77028 -SHA256: a8f51bb843439796190be9deb5be2a4cbd6faeb1af50c49ff0a8009a10aac233 -SHA1: b4d6f896b256088f19baf10f1661d595042a1f3d -MD5sum: faeda29bcef63283d9b1b57addd69d87 -Description: clr built using CMake -Maintainer: ROCm OpenCL Support -Version: 2.0.0.60202-116~20.04 -Installed-Size: 854 - -Package: rocm-opencl-icd-loader -Architecture: amd64 -Conflicts: rocm-ocl-icd -Depends: rocm-core -Priority: optional -Section: devel -Filename: pool/main/r/rocm-opencl-icd-loader/rocm-opencl-icd-loader_1.2.60202-116~20.04_amd64.deb -Size: 17146 -SHA256: 59168ff07c34b28d196cacf62c12310404481eac542cbf138437db03503aa932 -SHA1: f01398a678f0c4e40065f346ad9c2f00f6019319 -MD5sum: 06fef64089f4452410ad9769eff344df -Description: OpenCL-ICD-Loader built using CMake -Maintainer: ROCm OpenCL Support -Provides: rocm-ocl-icd -Replaces: rocm-ocl-icd -Version: 1.2.60202-116~20.04 -Installed-Size: 139 - -Package: rocm-opencl-icd-loader-dbgsym -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: e6d2e7b5acb9ca5975340c9cc608bb78d14dd90c -Depends: rocm-opencl-icd-loader (= 1.2.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocm-opencl-icd-loader-dbgsym/rocm-opencl-icd-loader-dbgsym_1.2.60202-116~20.04_amd64.deb -Size: 56178 -SHA256: 6623f7085090cfad7f63b7568388bc9b08cea6b8d7f3bc1763cea406769c0afc -SHA1: 85a85f22c331efe51180d4aae76f9bca3fdba30f -MD5sum: ff8ef65377ab896d9ddd05d4221d9562 -Description: debug symbols for rocm-opencl-icd-loader -Maintainer: ROCm OpenCL Support -Package-Type: ddeb -Version: 1.2.60202-116~20.04 -Installed-Size: 197 - -Package: rocm-opencl-icd-loader-dbgsym-rpath6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: e6d2e7b5acb9ca5975340c9cc608bb78d14dd90c -Depends: rocm-opencl-icd-loader-rpath6.2.2 (= 1.2.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocm-opencl-icd-loader-dbgsym-rpath6.2.2/rocm-opencl-icd-loader-dbgsym-rpath6.2.2_1.2.60202-116~20.04_amd64.deb -Size: 39064 -SHA256: b347b1a01b7cb647b2418416d1b3671b9c9a15f2e67ac4f6f9a486503994e61d -SHA1: a1f0aa93d499891737516db40ee713d806340800 -MD5sum: 9fea83aa1ceff5a935a938db5d22898e -Description: debug symbols for rocm-opencl-icd-loader -Maintainer: ROCm OpenCL Support -Package-Type: ddeb -Version: 1.2.60202-116~20.04 -Installed-Size: 197 - -Package: rocm-opencl-icd-loader-dbgsym6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: e6d2e7b5acb9ca5975340c9cc608bb78d14dd90c -Depends: rocm-opencl-icd-loader6.2.2 (= 1.2.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocm-opencl-icd-loader-dbgsym6.2.2/rocm-opencl-icd-loader-dbgsym6.2.2_1.2.60202-116~20.04_amd64.deb -Size: 39108 -SHA256: 580795e8201fe1d4b204f5533944b51d6c84d448b7f4fe2cdc2acea73e31ce3b -SHA1: 46529ee817f4e554d0f52eb5641313e74d7bc83a -MD5sum: 75c47c801e1c08c333cc95afea494982 -Description: debug symbols for rocm-opencl-icd-loader -Maintainer: ROCm OpenCL Support -Package-Type: ddeb -Version: 1.2.60202-116~20.04 -Installed-Size: 197 - -Package: rocm-opencl-icd-loader-rpath6.2.2 -Architecture: amd64 -Conflicts: rocm-ocl-icd-rpath6.2.2 -Depends: rocm-core-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocm-opencl-icd-loader-rpath6.2.2/rocm-opencl-icd-loader-rpath6.2.2_1.2.60202-116~20.04_amd64.deb -Size: 14324 -SHA256: 4105b37605fdf59eeabad29233d6da0f1c77a58650cfb1c80cac66571afa1978 -SHA1: 8719cc31b8ce1079270703b774d3d602934b1885 -MD5sum: 88240ad8353bc5c1ef3cb4e8337033f1 -Description: OpenCL-ICD-Loader built using CMake -Maintainer: ROCm OpenCL Support -Provides: rocm-ocl-icd-rpath6.2.2 -Replaces: rocm-ocl-icd-rpath6.2.2 -Version: 1.2.60202-116~20.04 -Installed-Size: 139 - -Package: rocm-opencl-icd-loader6.2.2 -Architecture: amd64 -Conflicts: rocm-ocl-icd6.2.2 -Depends: rocm-core6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocm-opencl-icd-loader6.2.2/rocm-opencl-icd-loader6.2.2_1.2.60202-116~20.04_amd64.deb -Size: 14320 -SHA256: 741dd8437d8d1200d8c3a0e4a7983891d3e798ccbb43ecfbc38f42d30699c793 -SHA1: 0b774c3b6493b6fe77e5c23fbc6e27b09c2dc11c -MD5sum: 9cdfd43fe4b09176da3526b5bc9ba627 -Description: OpenCL-ICD-Loader built using CMake -Maintainer: ROCm OpenCL Support -Provides: rocm-ocl-icd6.2.2 -Replaces: rocm-ocl-icd6.2.2 -Version: 1.2.60202-116~20.04 -Installed-Size: 139 - -Package: rocm-opencl-rpath6.2.2 -Architecture: amd64 -Depends: libelf-dev, comgr-rpath6.2.2, hsa-rocr-rpath6.2.2, rocm-core-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocm-opencl-rpath6.2.2/rocm-opencl-rpath6.2.2_2.0.0.60202-116~20.04_amd64.deb -Size: 452160 -SHA256: 1b3aee547ccda785db607416a23ce618bd9127f78326a65a7e8dde32f7656136 -SHA1: 1454000263e7a63b6b230cd4f337b5ab9df4512e -MD5sum: b7e23c62fea6c0913261e53d4d32d658 -Description: clr built using CMake -Maintainer: ROCm OpenCL Support -Version: 2.0.0.60202-116~20.04 -Installed-Size: 4351 - -Package: rocm-opencl-runtime -Architecture: amd64 -Depends: rocm-core (= 6.2.2.60202-116~20.04), rocm-language-runtime (= 6.2.2.60202-116~20.04), rocm-opencl (= 2.0.0.60202-116~20.04), rocm-opencl-icd-loader (= 1.2.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-opencl-runtime/rocm-opencl-runtime_6.2.2.60202-116~20.04_amd64.deb -Size: 2016 -SHA256: 6e1bc51a2ffefc6ed0262c22bf99d06d7b620530d8a901836211de60a9dced5b -SHA1: 3b3ebdf82daaf9b064d338cba44b9cf14cd2bffc -MD5sum: d9fc0aaeee5fb964a3fd14ac41a75b18 -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-opencl-runtime-asan -Architecture: amd64 -Depends: rocm-opencl-runtime (= 6.2.2.60202-116~20.04), rocm-core-asan (= 6.2.2.60202-116~20.04), rocm-language-runtime-asan (= 6.2.2.60202-116~20.04), rocm-opencl-asan (= 2.0.0.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-opencl-runtime-asan/rocm-opencl-runtime-asan_6.2.2.60202-116~20.04_amd64.deb -Size: 830 -SHA256: a686e0551c6e85a3f0446a1e0b8f90e0176d65c81746fe26b2f5a51e30fbcb42 -SHA1: 01edf1f1a243aeb6977532f1b3e797c62887f858 -MD5sum: eca906006f3a08509f89ef0d80ff5020 -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-opencl-runtime-asan-rpath6.2.2 -Architecture: amd64 -Depends: rocm-opencl-runtime (= 6.2.2.60202-116~20.04), rocm-core-asan-rpath6.2.2 (= 6.2.2.60202-116~20.04), rocm-language-runtime-asan-rpath6.2.2 (= 6.2.2.60202-116~20.04), rocm-opencl-asan-rpath6.2.2 (= 2.0.0.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-opencl-runtime-asan-rpath6.2.2/rocm-opencl-runtime-asan-rpath6.2.2_6.2.2.60202-116~20.04_amd64.deb -Size: 1036 -SHA256: efe4d4baf6c34ccbd3849156bc5b10d489c36f58d98adf353ab0eefcae189622 -SHA1: 486dba0dea37158b97f161588a21d29491fa2bf3 -MD5sum: 72898b1789bc32bbca4cd1efe3a4790a -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-opencl-runtime-asan6.2.2 -Architecture: amd64 -Depends: rocm-opencl-runtime (= 6.2.2.60202-116~20.04), rocm-core-asan6.2.2 (= 6.2.2.60202-116~20.04), rocm-language-runtime-asan6.2.2 (= 6.2.2.60202-116~20.04), rocm-opencl-asan6.2.2 (= 2.0.0.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-opencl-runtime-asan6.2.2/rocm-opencl-runtime-asan6.2.2_6.2.2.60202-116~20.04_amd64.deb -Size: 1032 -SHA256: 3d23ebd7c10ea706df25a9d224dd04551ace137cc21a780b60ea18a8efc345b6 -SHA1: aef0580b83adf6b7c8e6b00e6656cac2e2a5a3f9 -MD5sum: 86238ef8b70bf7237b91d19152ed1eda -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-opencl-runtime-rpath6.2.2 -Architecture: amd64 -Depends: rocm-core-rpath6.2.2 (= 6.2.2.60202-116~20.04), rocm-language-runtime-rpath6.2.2 (= 6.2.2.60202-116~20.04), rocm-opencl-rpath6.2.2 (= 2.0.0.60202-116~20.04), rocm-opencl-icd-loader-rpath6.2.2 (= 1.2.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-opencl-runtime-rpath6.2.2/rocm-opencl-runtime-rpath6.2.2_6.2.2.60202-116~20.04_amd64.deb -Size: 2188 -SHA256: 4855065aa337d1f58767a9dbbb7cf872435a4f44fb89f0ad54072891fa3f108e -SHA1: 586f8b557e149e626e7c770634d11e423da3fb8a -MD5sum: c86cf725d94d021bed921017a6a338a5 -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-opencl-runtime6.2.2 -Architecture: amd64 -Depends: rocm-core6.2.2 (= 6.2.2.60202-116~20.04), rocm-language-runtime6.2.2 (= 6.2.2.60202-116~20.04), rocm-opencl6.2.2 (= 2.0.0.60202-116~20.04), rocm-opencl-icd-loader6.2.2 (= 1.2.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-opencl-runtime6.2.2/rocm-opencl-runtime6.2.2_6.2.2.60202-116~20.04_amd64.deb -Size: 2172 -SHA256: d144306b429ff8cf0fccf9f07fe2a8986230eeefeff0f2c0feea11f4a368b1e2 -SHA1: bbb738828210836c46ff13257293c03225400cdb -MD5sum: c432d002092fa49c3363bef14787adb9 -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-opencl-sdk -Architecture: amd64 -Depends: rocm-core (= 6.2.2.60202-116~20.04), rocm-opencl-runtime (= 6.2.2.60202-116~20.04), rocm-opencl-dev (= 2.0.0.60202-116~20.04), hsa-rocr-dev (= 1.14.0.60202-116~20.04), hsakmt-roct-dev (= 20240607.4.05.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-opencl-sdk/rocm-opencl-sdk_6.2.2.60202-116~20.04_amd64.deb -Size: 852 -SHA256: ea7d70f85e14e54cef39dd9b662e9773508aff106344ae4a6bd62eacbb470fed -SHA1: f0671eac47e84d3f8a77036f22de180703b9540a -MD5sum: 7b35b2d35ece5f80ec85aa2f150e3836 -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-opencl-sdk-rpath6.2.2 -Architecture: amd64 -Depends: rocm-core-rpath6.2.2 (= 6.2.2.60202-116~20.04), rocm-opencl-runtime-rpath6.2.2 (= 6.2.2.60202-116~20.04), rocm-opencl-dev-rpath6.2.2 (= 2.0.0.60202-116~20.04), hsa-rocr-dev-rpath6.2.2 (= 1.14.0.60202-116~20.04), hsakmt-roct-dev-rpath6.2.2 (= 20240607.4.05.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-opencl-sdk-rpath6.2.2/rocm-opencl-sdk-rpath6.2.2_6.2.2.60202-116~20.04_amd64.deb -Size: 1056 -SHA256: f626fba282cd2507c4d4ac0f2a169918aa101a6b83a5cbb2caeb65bb3fd7fd51 -SHA1: 1a2ae3eedd04f01e8d2710f0e6fbf7950e4b2633 -MD5sum: 8f8f4b3ad17e1f8814bcbb4cd7dd071d -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-opencl-sdk6.2.2 -Architecture: amd64 -Depends: rocm-core6.2.2 (= 6.2.2.60202-116~20.04), rocm-opencl-runtime6.2.2 (= 6.2.2.60202-116~20.04), rocm-opencl-dev6.2.2 (= 2.0.0.60202-116~20.04), hsa-rocr-dev6.2.2 (= 1.14.0.60202-116~20.04), hsakmt-roct-dev6.2.2 (= 20240607.4.05.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-opencl-sdk6.2.2/rocm-opencl-sdk6.2.2_6.2.2.60202-116~20.04_amd64.deb -Size: 1044 -SHA256: e1692bbfbddf3e9f05ac3e59a2accf71f01f310e1792f51bd722a255e563f570 -SHA1: 4dce4a513236c6fc1d3045390d97af13f11bab37 -MD5sum: dda564408c6ad9dbc2c6b9cbd651c520 -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-opencl6.2.2 -Architecture: amd64 -Depends: libelf-dev, comgr6.2.2, hsa-rocr6.2.2, rocm-core6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocm-opencl6.2.2/rocm-opencl6.2.2_2.0.0.60202-116~20.04_amd64.deb -Size: 452084 -SHA256: a36b6fa904b983ef12df4f619faab5f874d79a66112f4880aecc265f55f8c134 -SHA1: 42a53125d0de4e155b48a3ce01df34c00e7321c1 -MD5sum: 01430c52d7e6f6f4b7baf251833f4749 -Description: clr built using CMake -Maintainer: ROCm OpenCL Support -Version: 2.0.0.60202-116~20.04 -Installed-Size: 4351 - -Package: rocm-openmp-sdk -Architecture: amd64 -Depends: rocm-language-runtime (= 6.2.2.60202-116~20.04), rocm-device-libs (= 1.0.0.60202-116~20.04), rocm-llvm (= 18.0.0.24355.60202-116~20.04), rocm-core (= 6.2.2.60202-116~20.04), openmp-extras-dev (= 18.62.0.60202-116~20.04), hsa-rocr-dev (= 1.14.0.60202-116~20.04), hsakmt-roct-dev (= 20240607.4.05.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-openmp-sdk/rocm-openmp-sdk_6.2.2.60202-116~20.04_amd64.deb -Size: 894 -SHA256: 2032da4d1d9dba8d798dacb55b56cbc866f7955f9ed21fe776037a054e677ba5 -SHA1: 8ca0c6e49f9fafef6d6e65bd933a4482d4badadc -MD5sum: ff9e15fe137306f5081578accb4221c9 -Description: Radeon Open Compute (ROCm) OpenMP Software development Kit. -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-openmp-sdk-rpath6.2.2 -Architecture: amd64 -Depends: rocm-language-runtime-rpath6.2.2 (= 6.2.2.60202-116~20.04), rocm-device-libs-rpath6.2.2 (= 1.0.0.60202-116~20.04), rocm-llvm-rpath6.2.2 (= 18.0.0.24355.60202-116~20.04), rocm-core-rpath6.2.2 (= 6.2.2.60202-116~20.04), openmp-extras-dev-rpath6.2.2 (= 18.62.0.60202-116~20.04), hsa-rocr-dev-rpath6.2.2 (= 1.14.0.60202-116~20.04), hsakmt-roct-dev-rpath6.2.2 (= 20240607.4.05.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-openmp-sdk-rpath6.2.2/rocm-openmp-sdk-rpath6.2.2_6.2.2.60202-116~20.04_amd64.deb -Size: 1096 -SHA256: deb77e6a87dd6ff412d6575518d5ffa8605f36b712493f2ed5fcba5fd0798029 -SHA1: 1833aebc7f3d64bd12d23d8932c545e4530000cb -MD5sum: b2fa8ed2c84decd31b32553391ea5363 -Description: Radeon Open Compute (ROCm) OpenMP Software development Kit. -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-openmp-sdk6.2.2 -Architecture: amd64 -Depends: rocm-language-runtime6.2.2 (= 6.2.2.60202-116~20.04), rocm-device-libs6.2.2 (= 1.0.0.60202-116~20.04), rocm-llvm6.2.2 (= 18.0.0.24355.60202-116~20.04), rocm-core6.2.2 (= 6.2.2.60202-116~20.04), openmp-extras-dev6.2.2 (= 18.62.0.60202-116~20.04), hsa-rocr-dev6.2.2 (= 1.14.0.60202-116~20.04), hsakmt-roct-dev6.2.2 (= 20240607.4.05.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-openmp-sdk6.2.2/rocm-openmp-sdk6.2.2_6.2.2.60202-116~20.04_amd64.deb -Size: 1096 -SHA256: 9713d7106edebb66d675928a3fdf9d1ede6d6b25f45d5e1dd75dc941282a2e27 -SHA1: 9e4d9df63aaa80d9fe1d0840111471b4c175a05d -MD5sum: b74f6d2e12d6941251316eeed4098713 -Description: Radeon Open Compute (ROCm) OpenMP Software development Kit. -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-rpath6.2.2 -Architecture: amd64 -Depends: rocm-utils-rpath6.2.2 (= 6.2.2.60202-116~20.04), rocm-developer-tools-rpath6.2.2 (= 6.2.2.60202-116~20.04), rocm-openmp-sdk-rpath6.2.2 (= 6.2.2.60202-116~20.04), rocm-opencl-sdk-rpath6.2.2 (= 6.2.2.60202-116~20.04), rocm-ml-sdk-rpath6.2.2 (= 6.2.2.60202-116~20.04), mivisionx-rpath6.2.2 (= 3.0.0.60202-116~20.04), migraphx-rpath6.2.2 (= 2.10.0.60202-116~20.04), rpp-rpath6.2.2 (= 1.8.0.60202-116~20.04), rocm-core-rpath6.2.2 (= 6.2.2.60202-116~20.04), migraphx-dev-rpath6.2.2 (= 2.10.0.60202-116~20.04), mivisionx-dev-rpath6.2.2 (= 3.0.0.60202-116~20.04), rpp-dev-rpath6.2.2 (= 1.8.0.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-rpath6.2.2/rocm-rpath6.2.2_6.2.2.60202-116~20.04_amd64.deb -Size: 2232 -SHA256: ebea9a11a53bca6ace0970e59d740ae6e4856535540002f40492533c5a05db67 -SHA1: a9bc492c05ac69185434d9d64e6c343400a41855 -MD5sum: 18e4db9d4b73bab1e2d31e6a39faea10 -Description: Radeon Open Compute (ROCm) software stack meta package -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm dev support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-smi-lib -Architecture: amd64 -Depends: python3, rocm-core -Priority: optional -Section: devel -Filename: pool/main/r/rocm-smi-lib/rocm-smi-lib_7.3.0.60202-116~20.04_amd64.deb -Size: 1002264 -SHA256: 8733acec9d1330ef09322249bb5ac6af2c273cc9f631133a0d5ecc4ded922037 -SHA1: 44af78fd819637804bebc740af43664881e8025b -MD5sum: b33ea3adff75b2200382816d6c7315f8 -Description: AMD System Management libraries - Development needed header files for ROCM-SMI -Maintainer: RocmSMILib Support -Suggests: sudo -Version: 7.3.0.60202-116~20.04 -Installed-Size: 8304 - -Package: rocm-smi-lib-asan -Architecture: amd64 -Depends: python3, rocm-core-asan -Priority: optional -Section: devel -Filename: pool/main/r/rocm-smi-lib-asan/rocm-smi-lib-asan_7.3.0.60202-116~20.04_amd64.deb -Size: 1943834 -SHA256: b7f2402068f0c6a40f4f0311fb986ee02c2e83c2884afc8b741be2fe70398c48 -SHA1: 9b3e33b181605b5fd6ea2fc9e813ad629dc433d6 -MD5sum: 6677216ff4dfd1bec29624a0f1f2250f -Description: AMD System Management libraries - ASAN libraries for the ROCM-SMI -Maintainer: RocmSMILib Support -Suggests: sudo -Version: 7.3.0.60202-116~20.04 -Installed-Size: 18451 - -Package: rocm-smi-lib-asan-dbgsym -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 67401c9968a014cb3690775ccdaa94454dc264ad 2f019d2b003fadab56bfeb48e2113c714b9abf6a -Depends: rocm-smi-lib-asan (= 7.3.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocm-smi-lib-asan-dbgsym/rocm-smi-lib-asan-dbgsym_7.3.0.60202-116~20.04_amd64.deb -Size: 9100068 -SHA256: 8fa6f45a6e8272e4c69b401ca28e6e84a5642ef9ddeba029814634385f49ca39 -SHA1: 4c52441840e9e4728faa6d84436f5b9dc8dd014b -MD5sum: 7aa7405acdb94c0fa83bcdb6cbb3b4f2 -Description: debug symbols for rocm-smi-lib-asan -Maintainer: RocmSMILib Support -Package-Type: ddeb -Version: 7.3.0.60202-116~20.04 -Installed-Size: 29935 - -Package: rocm-smi-lib-asan-dbgsym-rpath6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 67401c9968a014cb3690775ccdaa94454dc264ad 2f019d2b003fadab56bfeb48e2113c714b9abf6a -Depends: rocm-smi-lib-asan-rpath6.2.2 (= 7.3.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocm-smi-lib-asan-dbgsym-rpath6.2.2/rocm-smi-lib-asan-dbgsym-rpath6.2.2_7.3.0.60202-116~20.04_amd64.deb -Size: 5598564 -SHA256: 660a8e9e98feb5166ed4b183b525dc88508aa7f19c7334a4bc3d4401ef258be7 -SHA1: fa98512a1ffd89c91d9dc83d65301d569299ae12 -MD5sum: a35dc4e89add54a83e0c771a8c19449c -Description: debug symbols for rocm-smi-lib-asan -Maintainer: RocmSMILib Support -Package-Type: ddeb -Version: 7.3.0.60202-116~20.04 -Installed-Size: 29935 - -Package: rocm-smi-lib-asan-dbgsym6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 67401c9968a014cb3690775ccdaa94454dc264ad 2f019d2b003fadab56bfeb48e2113c714b9abf6a -Depends: rocm-smi-lib-asan6.2.2 (= 7.3.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocm-smi-lib-asan-dbgsym6.2.2/rocm-smi-lib-asan-dbgsym6.2.2_7.3.0.60202-116~20.04_amd64.deb -Size: 5597524 -SHA256: 64c173123c19b59b071d730232538f07013a06521588e9cafc3030e23edf09e7 -SHA1: 24077402160a3caedc1e4978536b279af672a2be -MD5sum: 1fa071223bc74bb7dc1bcfdc55589867 -Description: debug symbols for rocm-smi-lib-asan -Maintainer: RocmSMILib Support -Package-Type: ddeb -Version: 7.3.0.60202-116~20.04 -Installed-Size: 29935 - -Package: rocm-smi-lib-asan-rpath6.2.2 -Architecture: amd64 -Depends: python3, rocm-core-asan-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocm-smi-lib-asan-rpath6.2.2/rocm-smi-lib-asan-rpath6.2.2_7.3.0.60202-116~20.04_amd64.deb -Size: 883076 -SHA256: f1220cff2f79734c0dd0d01138fababf5eaf2635c481a75a4a5941d6b8bf044b -SHA1: 45f8eb6124a794385899924d011219afe3790f68 -MD5sum: 675cef74cb894013f569a17770ee8014 -Description: AMD System Management libraries - ASAN libraries for the ROCM-SMI -Maintainer: RocmSMILib Support -Suggests: sudo -Version: 7.3.0.60202-116~20.04 -Installed-Size: 18451 - -Package: rocm-smi-lib-asan6.2.2 -Architecture: amd64 -Depends: python3, rocm-core-asan6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocm-smi-lib-asan6.2.2/rocm-smi-lib-asan6.2.2_7.3.0.60202-116~20.04_amd64.deb -Size: 882856 -SHA256: 10f26cf893ce7937bfaadd06c10a520b1c5d14c9e6236c0cfe5a2e9813c80c9e -SHA1: c61ac30d1b1928936f5a70fdbae9fb6331e99ee7 -MD5sum: 4c506d6fada62e3dae57ff9eede3b26b -Description: AMD System Management libraries - ASAN libraries for the ROCM-SMI -Maintainer: RocmSMILib Support -Suggests: sudo -Version: 7.3.0.60202-116~20.04 -Installed-Size: 18451 - -Package: rocm-smi-lib-dbgsym -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: e736ecddecbf1488cf53c283cba00ef64b3e01fb 9210c855bcea4aa551c9e7e245fc54a178fcf0b4 -Depends: rocm-smi-lib (= 7.3.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocm-smi-lib-dbgsym/rocm-smi-lib-dbgsym_7.3.0.60202-116~20.04_amd64.deb -Size: 16069244 -SHA256: 56936051edc6f16593d965c630e15e7ceb7d319f1df9ca333ffccfb118b023cf -SHA1: 736fcb4840927199acd65e1aa6e5cbc58fc54cb5 -MD5sum: bbe391d616d4a038b093421deff985c3 -Description: debug symbols for rocm-smi-lib -Maintainer: RocmSMILib Support -Package-Type: ddeb -Version: 7.3.0.60202-116~20.04 -Installed-Size: 63561 - -Package: rocm-smi-lib-dbgsym-rpath6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: e736ecddecbf1488cf53c283cba00ef64b3e01fb 9210c855bcea4aa551c9e7e245fc54a178fcf0b4 -Depends: rocm-smi-lib-rpath6.2.2 (= 7.3.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocm-smi-lib-dbgsym-rpath6.2.2/rocm-smi-lib-dbgsym-rpath6.2.2_7.3.0.60202-116~20.04_amd64.deb -Size: 10508424 -SHA256: 0c096b9faf394638975675d29c45617269ca18e0198db998e4b3c1d20aeca187 -SHA1: 52b9f825d84fee17c5fdf6dcf5532c77cbb5acf9 -MD5sum: 8d5424f0ee430d9826a4d0bc2eb2558c -Description: debug symbols for rocm-smi-lib -Maintainer: RocmSMILib Support -Package-Type: ddeb -Version: 7.3.0.60202-116~20.04 -Installed-Size: 63561 - -Package: rocm-smi-lib-dbgsym6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: e736ecddecbf1488cf53c283cba00ef64b3e01fb 9210c855bcea4aa551c9e7e245fc54a178fcf0b4 -Depends: rocm-smi-lib6.2.2 (= 7.3.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocm-smi-lib-dbgsym6.2.2/rocm-smi-lib-dbgsym6.2.2_7.3.0.60202-116~20.04_amd64.deb -Size: 10508580 -SHA256: 79ced8e6d052e48ad002057d71bc8d800012d203942b3781de7f1973b80a74a0 -SHA1: 1d764b7de51b0af3f68547aa4bdedda6fe7441b0 -MD5sum: abcad4e99505b6acb785498859d38f95 -Description: debug symbols for rocm-smi-lib -Maintainer: RocmSMILib Support -Package-Type: ddeb -Version: 7.3.0.60202-116~20.04 -Installed-Size: 63561 - -Package: rocm-smi-lib-rpath6.2.2 -Architecture: amd64 -Depends: python3, rocm-core-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocm-smi-lib-rpath6.2.2/rocm-smi-lib-rpath6.2.2_7.3.0.60202-116~20.04_amd64.deb -Size: 524324 -SHA256: 86d3248eb4782eaa49bdba33d7345c6dce839bc399c5a74b9319558d90125028 -SHA1: 7b86335e73eee0517fc951b049dc74826b14d46a -MD5sum: 2ddf9e6f7d367860d3b3a9df3c6e62d6 -Description: AMD System Management libraries - Development needed header files for ROCM-SMI -Maintainer: RocmSMILib Support -Suggests: sudo -Version: 7.3.0.60202-116~20.04 -Installed-Size: 8304 - -Package: rocm-smi-lib6.2.2 -Architecture: amd64 -Depends: python3, rocm-core6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocm-smi-lib6.2.2/rocm-smi-lib6.2.2_7.3.0.60202-116~20.04_amd64.deb -Size: 524388 -SHA256: ade47b6efbe9bc16a4b70aa553bf6681cdb09a9d8fd3867544555de27cd70694 -SHA1: 8085c2ff557b2bc83fbdb245c693b2d18df4bea3 -MD5sum: 0896bba16c169866e5154e432b08ffa9 -Description: AMD System Management libraries - Development needed header files for ROCM-SMI -Maintainer: RocmSMILib Support -Suggests: sudo -Version: 7.3.0.60202-116~20.04 -Installed-Size: 8304 - -Package: rocm-utils -Architecture: amd64 -Depends: rocminfo (= 1.0.0.60202-116~20.04), rocm-cmake (= 0.13.0.60202-116~20.04), rocm-core (= 6.2.2.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-utils/rocm-utils_6.2.2.60202-116~20.04_amd64.deb -Size: 814 -SHA256: 127461de76d2bee64cadb9e3536eb788fb1802e843c1677a87c50688be9ee78d -SHA1: 088feaeff0284d50aa07a19767c5921c8a8c0783 -MD5sum: e1bd775bfe1ea7203a036d7c66b134d2 -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-utils-rpath6.2.2 -Architecture: amd64 -Depends: rocminfo-rpath6.2.2 (= 1.0.0.60202-116~20.04), rocm-cmake-rpath6.2.2 (= 0.13.0.60202-116~20.04), rocm-core-rpath6.2.2 (= 6.2.2.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-utils-rpath6.2.2/rocm-utils-rpath6.2.2_6.2.2.60202-116~20.04_amd64.deb -Size: 1008 -SHA256: f13b047f74cf2a6e5e72fa37fb311411cc89d40a9155601f31d202e58baeb4e0 -SHA1: 71f720fcc3aa23e03a7385c815a325caced6a70b -MD5sum: 316082732a66f9536f1389f041c482fd -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-utils6.2.2 -Architecture: amd64 -Depends: rocminfo6.2.2 (= 1.0.0.60202-116~20.04), rocm-cmake6.2.2 (= 0.13.0.60202-116~20.04), rocm-core6.2.2 (= 6.2.2.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm-utils6.2.2/rocm-utils6.2.2_6.2.2.60202-116~20.04_amd64.deb -Size: 1004 -SHA256: 142b02d63af4b5a4957f160362de5b4d875e4de04157c9eed9e8e74602f45380 -SHA1: 8970f63d9f3058544650d2dbc063d7a165f25e3a -MD5sum: 0c365d18196e6075af12c2efdfe71daa -Description: Radeon Open Compute (ROCm) Runtime software stack -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm Dev Support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocm-validation-suite -Architecture: amd64 -Depends: hip-runtime-amd, comgr, hsa-rocr, rocblas, rocm-smi-lib, rocm-core, libpci3, libyaml-cpp-dev -Priority: optional -Section: devel -Filename: pool/main/r/rocm-validation-suite/rocm-validation-suite_1.0.60202.60202-116~20.04_amd64.deb -Size: 1594682 -SHA256: 21041b254c82ee764cac198ad829fbd0f6fd2347604e4bb7442ea66f66070850 -SHA1: 6f86784809c06ed04d75fe63585843222e260654 -MD5sum: 2cff62992ae67eec81284631730bda41 -Description: The ROCm Validation Suite (RVS) is a system validation and diagnostics tool for monitoring, stress testing, detecting and troubleshooting issues that affects the functionality and performance of AMD GPU(s) operating in a high-performance/AI/ML computing environment. - ROCm Validation Suite -Maintainer: ROCM Validation Suite Support -Version: 1.0.60202.60202-116~20.04 -Installed-Size: 13422 - -Package: rocm-validation-suite-dbgsym -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 45cf12437d892a7fc12a662b86e018cf7f3919d9 327a64b6a183b952ea9cd22cbec565116961df54 d909b285bd0fce2c5499bd65d93b27fd2ee9ba19 32de5f0f03b206323027786842e4d719d1aa0499 e247e8ef7625be5747be0ad823abb0824d213803 c4b64695bfa438458772fcb289f40cf6dc70f15d bed2f27a0d47483b0743827f15a2ae604b211b42 12655854c1f2ed9d3db4753cb5ff84a40604251c 75dc9126d165ef1e3e208ecbf2f450c588d73408 e834c0ee65d8ad14a550147cff05444d67287c0f f192d65ce54fa50d3b5e748c9cbb7eee87b9609c b7bf0949cc2ac31c887a53f62ffa43e485244840 f56c4abe40a613d20c31116c99274853932da8fb dcfc7e4475f51ff717d16963163bab8281d32fa3 434a198537c9e9c244f45c4c7bed7b0bee675f98 1db2f7cfebb1ea76b18caf698bb4d6973646ce50 -Depends: rocm-validation-suite (= 1.0.60202.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocm-validation-suite-dbgsym/rocm-validation-suite-dbgsym_1.0.60202.60202-116~20.04_amd64.deb -Size: 5023188 -SHA256: 897f3e99323cdc07cce62f73bdd0549e442dcbcda297f7acc578a84abfd7c7f0 -SHA1: 2faaa3272573c327607863a8144dd260a798cd7b -MD5sum: d5d2003448af3bc8e7a048d95fe3f836 -Description: debug symbols for rocm-validation-suite -Maintainer: ROCM Validation Suite Support -Package-Type: ddeb -Version: 1.0.60202.60202-116~20.04 -Installed-Size: 16438 - -Package: rocm-validation-suite-dbgsym-rpath6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 45cf12437d892a7fc12a662b86e018cf7f3919d9 327a64b6a183b952ea9cd22cbec565116961df54 d909b285bd0fce2c5499bd65d93b27fd2ee9ba19 32de5f0f03b206323027786842e4d719d1aa0499 e247e8ef7625be5747be0ad823abb0824d213803 c4b64695bfa438458772fcb289f40cf6dc70f15d bed2f27a0d47483b0743827f15a2ae604b211b42 12655854c1f2ed9d3db4753cb5ff84a40604251c 75dc9126d165ef1e3e208ecbf2f450c588d73408 e834c0ee65d8ad14a550147cff05444d67287c0f f192d65ce54fa50d3b5e748c9cbb7eee87b9609c b7bf0949cc2ac31c887a53f62ffa43e485244840 f56c4abe40a613d20c31116c99274853932da8fb dcfc7e4475f51ff717d16963163bab8281d32fa3 434a198537c9e9c244f45c4c7bed7b0bee675f98 1db2f7cfebb1ea76b18caf698bb4d6973646ce50 -Depends: rocm-validation-suite-rpath6.2.2 (= 1.0.60202.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocm-validation-suite-dbgsym-rpath6.2.2/rocm-validation-suite-dbgsym-rpath6.2.2_1.0.60202.60202-116~20.04_amd64.deb -Size: 3131192 -SHA256: 3cbdbe63365fb54bba0daa30720dae4aedf4d72796c3fcc477116ea596a06735 -SHA1: e8ca01a48749144335400225f38949a127399b87 -MD5sum: f245042ed8e17fe5f40bbb1e4c7e0352 -Description: debug symbols for rocm-validation-suite -Maintainer: ROCM Validation Suite Support -Package-Type: ddeb -Version: 1.0.60202.60202-116~20.04 -Installed-Size: 16438 - -Package: rocm-validation-suite-dbgsym6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 45cf12437d892a7fc12a662b86e018cf7f3919d9 327a64b6a183b952ea9cd22cbec565116961df54 d909b285bd0fce2c5499bd65d93b27fd2ee9ba19 32de5f0f03b206323027786842e4d719d1aa0499 e247e8ef7625be5747be0ad823abb0824d213803 c4b64695bfa438458772fcb289f40cf6dc70f15d bed2f27a0d47483b0743827f15a2ae604b211b42 12655854c1f2ed9d3db4753cb5ff84a40604251c 75dc9126d165ef1e3e208ecbf2f450c588d73408 e834c0ee65d8ad14a550147cff05444d67287c0f f192d65ce54fa50d3b5e748c9cbb7eee87b9609c b7bf0949cc2ac31c887a53f62ffa43e485244840 f56c4abe40a613d20c31116c99274853932da8fb dcfc7e4475f51ff717d16963163bab8281d32fa3 434a198537c9e9c244f45c4c7bed7b0bee675f98 1db2f7cfebb1ea76b18caf698bb4d6973646ce50 -Depends: rocm-validation-suite6.2.2 (= 1.0.60202.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocm-validation-suite-dbgsym6.2.2/rocm-validation-suite-dbgsym6.2.2_1.0.60202.60202-116~20.04_amd64.deb -Size: 3130088 -SHA256: 00d9edf3b41aa79c938c7577ccc1afcc975034fdecac3a409dfd52e1ada9beef -SHA1: dace3076c0e678ec411d03fd1ea3c97591620de9 -MD5sum: f737d2220f0416bc3a73f189401debd3 -Description: debug symbols for rocm-validation-suite -Maintainer: ROCM Validation Suite Support -Package-Type: ddeb -Version: 1.0.60202.60202-116~20.04 -Installed-Size: 16438 - -Package: rocm-validation-suite-rpath6.2.2 -Architecture: amd64 -Depends: hip-runtime-amd-rpath6.2.2, comgr-rpath6.2.2, hsa-rocr-rpath6.2.2, rocblas-rpath6.2.2, rocm-smi-lib-rpath6.2.2, rocm-core-rpath6.2.2, libpci3, libyaml-cpp-dev -Priority: optional -Section: devel -Filename: pool/main/r/rocm-validation-suite-rpath6.2.2/rocm-validation-suite-rpath6.2.2_1.0.60202.60202-116~20.04_amd64.deb -Size: 741376 -SHA256: 916bf7894148d765b3a25993333d3bf3dda40bcf2f86789986b4e8bd6fba728b -SHA1: bd9ef2f306f27834accc447c991fdf4acd11257f -MD5sum: 497faceb0dd8701ab8a80def38960611 -Description: The ROCm Validation Suite (RVS) is a system validation and diagnostics tool for monitoring, stress testing, detecting and troubleshooting issues that affects the functionality and performance of AMD GPU(s) operating in a high-performance/AI/ML computing environment. - ROCm Validation Suite -Maintainer: ROCM Validation Suite Support -Version: 1.0.60202.60202-116~20.04 -Installed-Size: 13422 - -Package: rocm-validation-suite6.2.2 -Architecture: amd64 -Depends: hip-runtime-amd6.2.2, comgr6.2.2, hsa-rocr6.2.2, rocblas6.2.2, rocm-smi-lib6.2.2, rocm-core6.2.2, libpci3, libyaml-cpp-dev -Priority: optional -Section: devel -Filename: pool/main/r/rocm-validation-suite6.2.2/rocm-validation-suite6.2.2_1.0.60202.60202-116~20.04_amd64.deb -Size: 743056 -SHA256: e76335c8dd362abb999b32311fae5e538e49430f4e96286f1151a10783a20be3 -SHA1: c78c408dee001467a17c6391795db8fee25b313a -MD5sum: b45d4f3b835c59a3efde9b599b946957 -Description: The ROCm Validation Suite (RVS) is a system validation and diagnostics tool for monitoring, stress testing, detecting and troubleshooting issues that affects the functionality and performance of AMD GPU(s) operating in a high-performance/AI/ML computing environment. - ROCm Validation Suite -Maintainer: ROCM Validation Suite Support -Version: 1.0.60202.60202-116~20.04 -Installed-Size: 13422 - -Package: rocm6.2.2 -Architecture: amd64 -Depends: rocm-utils6.2.2 (= 6.2.2.60202-116~20.04), rocm-developer-tools6.2.2 (= 6.2.2.60202-116~20.04), rocm-openmp-sdk6.2.2 (= 6.2.2.60202-116~20.04), rocm-opencl-sdk6.2.2 (= 6.2.2.60202-116~20.04), rocm-ml-sdk6.2.2 (= 6.2.2.60202-116~20.04), mivisionx6.2.2 (= 3.0.0.60202-116~20.04), migraphx6.2.2 (= 2.10.0.60202-116~20.04), rpp6.2.2 (= 1.8.0.60202-116~20.04), rocm-core6.2.2 (= 6.2.2.60202-116~20.04), migraphx-dev6.2.2 (= 2.10.0.60202-116~20.04), mivisionx-dev6.2.2 (= 3.0.0.60202-116~20.04), rpp-dev6.2.2 (= 1.8.0.60202-116~20.04) -Priority: optional -Section: devel -Filename: pool/main/r/rocm6.2.2/rocm6.2.2_6.2.2.60202-116~20.04_amd64.deb -Size: 2224 -SHA256: baa61f7a355bc75214a9a9e338b81580153bccdb568b2b41540716a2212329ea -SHA1: 57c8d2600e5b06b8dabf9408305b6f20dce053ae -MD5sum: dd21a94b091ee48e1c07a231fbda1195 -Description: Radeon Open Compute (ROCm) software stack meta package -Homepage: https://github.com/RadeonOpenCompute/ROCm -Maintainer: ROCm dev support -Version: 6.2.2.60202-116~20.04 -Installed-Size: 13 - -Package: rocminfo -Architecture: amd64 -Depends: hsa-rocr, kmod, pciutils, python3, libc6, libgcc-s1, libstdc++6, rocm-core -Priority: optional -Section: devel -Filename: pool/main/r/rocminfo/rocminfo_1.0.0.60202-116~20.04_amd64.deb -Size: 28986 -SHA256: 11e2b0b197406a6e3f451d344f080f7af287b9d0c5286b93eaed63a98ce93e7c -SHA1: 8727a96566e12c40f3eecda60263327e6b115ac9 -MD5sum: b56a104f360ab2d0d6a1c09f34a38afe -Description: Radeon Open Compute (ROCm) Runtime rocminfo tool -Maintainer: AMD Rocminfo Support -Version: 1.0.0.60202-116~20.04 -Installed-Size: 106 - -Package: rocminfo-dbgsym -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 30d8001dfe25c0a3113298feaf282072c203a06b -Depends: rocminfo (= 1.0.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocminfo-dbgsym/rocminfo-dbgsym_1.0.0.60202-116~20.04_amd64.deb -Size: 57240 -SHA256: a142e89644e1bb7ea8d04935e1148e071c94e30cad45bac0d07f22eae179a69d -SHA1: 821147cbec4a0f10b3cb633c74eafe0c560d05ba -MD5sum: 570f57711d527bcb89d3575981346171 -Description: debug symbols for rocminfo -Maintainer: AMD Rocminfo Support -Package-Type: ddeb -Version: 1.0.0.60202-116~20.04 -Installed-Size: 192 - -Package: rocminfo-dbgsym-rpath6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 30d8001dfe25c0a3113298feaf282072c203a06b -Depends: rocminfo-rpath6.2.2 (= 1.0.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocminfo-dbgsym-rpath6.2.2/rocminfo-dbgsym-rpath6.2.2_1.0.0.60202-116~20.04_amd64.deb -Size: 47896 -SHA256: 66d4d828ac7c62622b070710905c414c10f01c1bd544840c7cc5fd1814383bc4 -SHA1: c5303e339df14bde7844029e709fc6d3eb629fbd -MD5sum: 78afb0fbe55807a209a27de13519906a -Description: debug symbols for rocminfo -Maintainer: AMD Rocminfo Support -Package-Type: ddeb -Version: 1.0.0.60202-116~20.04 -Installed-Size: 192 - -Package: rocminfo-dbgsym6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 30d8001dfe25c0a3113298feaf282072c203a06b -Depends: rocminfo6.2.2 (= 1.0.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocminfo-dbgsym6.2.2/rocminfo-dbgsym6.2.2_1.0.0.60202-116~20.04_amd64.deb -Size: 47872 -SHA256: 9c27e8f5b94f9eee5fb518369872394b794c8e8ab1ec974c838228c7708e10af -SHA1: 4273920ace173ad6148955d4f665ac29b1ebe635 -MD5sum: a10c3bf82fec054ef003d4555b714993 -Description: debug symbols for rocminfo -Maintainer: AMD Rocminfo Support -Package-Type: ddeb -Version: 1.0.0.60202-116~20.04 -Installed-Size: 192 - -Package: rocminfo-rpath6.2.2 -Architecture: amd64 -Depends: hsa-rocr-rpath6.2.2, kmod, pciutils, python3, libc6, libgcc-s1, libstdc++6, rocm-core-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocminfo-rpath6.2.2/rocminfo-rpath6.2.2_1.0.0.60202-116~20.04_amd64.deb -Size: 23180 -SHA256: d42eb140672e3299c4c10be82e357911b126f06d944d94affc7ff7427a222846 -SHA1: 3ac3e5aa54c52c15fa8c0789eb79649eab8ae6a4 -MD5sum: b8c6b63a58f7dea0eab6a5b4bb39e9ba -Description: Radeon Open Compute (ROCm) Runtime rocminfo tool -Maintainer: AMD Rocminfo Support -Version: 1.0.0.60202-116~20.04 -Installed-Size: 106 - -Package: rocminfo6.2.2 -Architecture: amd64 -Depends: hsa-rocr6.2.2, kmod, pciutils, python3, libc6, libgcc-s1, libstdc++6, rocm-core6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocminfo6.2.2/rocminfo6.2.2_1.0.0.60202-116~20.04_amd64.deb -Size: 23212 -SHA256: 13eb5fb53c41c979f77526540f4c06b8ac78351cbb2734491599141dd708bc6a -SHA1: 65ab23a31709957148adc62cda82c7cf945b28af -MD5sum: 712e846b3b4e6656ecabd94d27a2cc9c -Description: Radeon Open Compute (ROCm) Runtime rocminfo tool -Maintainer: AMD Rocminfo Support -Version: 1.0.0.60202-116~20.04 -Installed-Size: 106 - -Package: rocprim-dev -Architecture: amd64 -Depends: hip-runtime-amd (>= 4.5.0), rocm-core -Priority: optional -Section: devel -Filename: pool/main/r/rocprim-dev/rocprim-dev_3.2.0.60202-116~20.04_amd64.deb -Size: 211424 -SHA256: 61b3371289ad26073aeaad72f0a4f057c3d35de99d758dc28781e4ac26073640 -SHA1: f3bcab5674e42268690d89b6cf687f81532aa799 -MD5sum: c40877431bfd778745dc0d448554c35a -Description: Radeon Open Compute Parallel Primitives Library -Maintainer: rocPRIM Maintainer -Provides: CPACK_DEBIAN_PACKAGE_PROVIDES, rocprim (= 3.2.0.60202) -Version: 3.2.0.60202-116~20.04 -Installed-Size: 3787 - -Package: rocprim-dev-rpath6.2.2 -Architecture: amd64 -Depends: hip-runtime-amd-rpath6.2.2 (>= 4.5.0), rocm-core-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocprim-dev-rpath6.2.2/rocprim-dev-rpath6.2.2_3.2.0.60202-116~20.04_amd64.deb -Size: 211008 -SHA256: 0ab1ffcb88bf9356fda583da7140abf828030143d5cdc18d8f0cf2e49ee1dc7c -SHA1: c4560b30f03d8a1f7e612aa8c8a506b7ee160b09 -MD5sum: 02963572f058d52b361216af8f085cc5 -Description: Radeon Open Compute Parallel Primitives Library -Maintainer: rocPRIM Maintainer -Provides: CPACK_DEBIAN_PACKAGE_PROVIDES, rocprim-rpath6.2.2 (= 3.2.0.60202) -Version: 3.2.0.60202-116~20.04 -Installed-Size: 3787 - -Package: rocprim-dev6.2.2 -Architecture: amd64 -Depends: hip-runtime-amd6.2.2 (>= 4.5.0), rocm-core6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocprim-dev6.2.2/rocprim-dev6.2.2_3.2.0.60202-116~20.04_amd64.deb -Size: 211032 -SHA256: 4e00158236f4747d50642f3fd40f58e5dc8a411c3f172bbf3d32db365b3c9f81 -SHA1: 99e863fae4a001309fa8b5cce002e8b36aa0fa21 -MD5sum: 8c178e5d4e8c68fcbb7962343d048124 -Description: Radeon Open Compute Parallel Primitives Library -Maintainer: rocPRIM Maintainer -Provides: CPACK_DEBIAN_PACKAGE_PROVIDES, rocprim6.2.2 (= 3.2.0.60202) -Version: 3.2.0.60202-116~20.04 -Installed-Size: 3787 - -Package: rocprofiler -Architecture: amd64 -Depends: rocminfo, hsa-rocr, rocm-core, libsystemd-dev, libelf-dev, libnuma-dev, libpciaccess-dev, libxml2-dev -Priority: optional -Section: devel -Filename: pool/main/r/rocprofiler/rocprofiler_2.0.60202.60202-116~20.04_amd64.deb -Size: 906372 -SHA256: c259d89c6b1221a9d8eccfbcb10901b179632fd40fc95ebde26e57e0d4b8bae6 -SHA1: a4b1ef043b18d61c5446f876bb621e35bf9afb0f -MD5sum: cb0452e622dca8f25d49badbb2f7fa2c -Description: ROCPROFILER library for AMD HSA runtime API extension support - Dynamic libraries for the ROCProfiler -Maintainer: ROCm Profiler Support -Version: 2.0.60202.60202-116~20.04 -Installed-Size: 4105 - -Package: rocprofiler-asan -Architecture: amd64 -Depends: hsa-rocr-asan, rocm-core-asan -Priority: optional -Section: devel -Filename: pool/main/r/rocprofiler-asan/rocprofiler-asan_2.0.60202.60202-116~20.04_amd64.deb -Size: 1513414 -SHA256: e365fbebfc0125d2ba2fd3e2e3be9b06de7c1820c8b31ad434352e7d39d34462 -SHA1: d680c2babaffc636be159e3ec4db1252aef267eb -MD5sum: 974fd3609d6f4b8dce6d016d222ebd20 -Description: ROCPROFILER library for AMD HSA runtime API extension support - ASAN libraries for the ROCPROFILER -Maintainer: ROCm Profiler Support -Version: 2.0.60202.60202-116~20.04 -Installed-Size: 10588 - -Package: rocprofiler-asan-dbgsym -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 179819000e8bc640140b96596db3911269f27475 b0a46bb51454151eeccfb510a199780d956bf55d 62b2cdf0d08b7098672fb3d85b0bb7fdf8937fad 5c1fe666e7464f918bf5bf2208429fb93b1e7f0c 5e11225aac0e0e34e1508d483fda33a6bbc1e7f7 eab537f660621a5f1f194c11a7a42c808fd410e9 50f4874cfde557a555c9195b277537e675861045 4a5ef97f7cda0b940f270843e1b55c82341bd24d -Depends: rocprofiler-asan (= 2.0.60202.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocprofiler-asan-dbgsym/rocprofiler-asan-dbgsym_2.0.60202.60202-116~20.04_amd64.deb -Size: 7693986 -SHA256: 847d67490662b8cceee5945d4501d12b07918ee81e96748a6fe461bb34e3ffcc -SHA1: 070abbcd0f55f1eadb1452aa7187a2b42a7d52d7 -MD5sum: 6211a531bb0a9c3e0e070414af1d723e -Description: debug symbols for rocprofiler-asan -Maintainer: ROCm Profiler Support -Package-Type: ddeb -Version: 2.0.60202.60202-116~20.04 -Installed-Size: 23460 - -Package: rocprofiler-asan-dbgsym-rpath6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 179819000e8bc640140b96596db3911269f27475 b0a46bb51454151eeccfb510a199780d956bf55d 62b2cdf0d08b7098672fb3d85b0bb7fdf8937fad 5c1fe666e7464f918bf5bf2208429fb93b1e7f0c 5e11225aac0e0e34e1508d483fda33a6bbc1e7f7 eab537f660621a5f1f194c11a7a42c808fd410e9 50f4874cfde557a555c9195b277537e675861045 4a5ef97f7cda0b940f270843e1b55c82341bd24d -Depends: rocprofiler-asan-rpath6.2.2 (= 2.0.60202.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocprofiler-asan-dbgsym-rpath6.2.2/rocprofiler-asan-dbgsym-rpath6.2.2_2.0.60202.60202-116~20.04_amd64.deb -Size: 4672832 -SHA256: c34791891e894497d6edc44b25b17b919db6399020248601169f7ffe8758811e -SHA1: 96601b34ac61c169c2cda0651faedbdb969b5925 -MD5sum: 2f62ed5b0dfc782fc62a2389a99fa6e2 -Description: debug symbols for rocprofiler-asan -Maintainer: ROCm Profiler Support -Package-Type: ddeb -Version: 2.0.60202.60202-116~20.04 -Installed-Size: 23460 - -Package: rocprofiler-asan-dbgsym6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 179819000e8bc640140b96596db3911269f27475 b0a46bb51454151eeccfb510a199780d956bf55d 62b2cdf0d08b7098672fb3d85b0bb7fdf8937fad 5c1fe666e7464f918bf5bf2208429fb93b1e7f0c 5e11225aac0e0e34e1508d483fda33a6bbc1e7f7 eab537f660621a5f1f194c11a7a42c808fd410e9 50f4874cfde557a555c9195b277537e675861045 4a5ef97f7cda0b940f270843e1b55c82341bd24d -Depends: rocprofiler-asan6.2.2 (= 2.0.60202.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocprofiler-asan-dbgsym6.2.2/rocprofiler-asan-dbgsym6.2.2_2.0.60202.60202-116~20.04_amd64.deb -Size: 4674772 -SHA256: fcfaa87aaafec02ff0c6d456e81f1fd6fc28097903190970bd17c756fd7d97d8 -SHA1: 2e94b1aa70a21a5273dc500f4b1284b049b38791 -MD5sum: aa2147c058582697596309bcdb7a8447 -Description: debug symbols for rocprofiler-asan -Maintainer: ROCm Profiler Support -Package-Type: ddeb -Version: 2.0.60202.60202-116~20.04 -Installed-Size: 23460 - -Package: rocprofiler-asan-rpath6.2.2 -Architecture: amd64 -Depends: hsa-rocr-asan-rpath6.2.2, rocm-core-asan-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocprofiler-asan-rpath6.2.2/rocprofiler-asan-rpath6.2.2_2.0.60202.60202-116~20.04_amd64.deb -Size: 884600 -SHA256: d6eaf31ec9578acf9e805398184c726ec48304af486bf6a4a444f6947382dfe4 -SHA1: 29eb7f0913f29e2f8511dd163ad01b1dd34d216a -MD5sum: 2bc3248ca80511bca8ec72f8f5460eb4 -Description: ROCPROFILER library for AMD HSA runtime API extension support - ASAN libraries for the ROCPROFILER -Maintainer: ROCm Profiler Support -Version: 2.0.60202.60202-116~20.04 -Installed-Size: 10588 - -Package: rocprofiler-asan6.2.2 -Architecture: amd64 -Depends: hsa-rocr-asan6.2.2, rocm-core-asan6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocprofiler-asan6.2.2/rocprofiler-asan6.2.2_2.0.60202.60202-116~20.04_amd64.deb -Size: 883224 -SHA256: c5808460da88a1075deb26a5b53dad530048db713da2d91f8d1f6877ccfa7454 -SHA1: 36735369e6beb039ccd889e2fe68ea02534aa170 -MD5sum: a5cc174b8b626d926aaf502285e81077 -Description: ROCPROFILER library for AMD HSA runtime API extension support - ASAN libraries for the ROCPROFILER -Maintainer: ROCm Profiler Support -Version: 2.0.60202.60202-116~20.04 -Installed-Size: 10588 - -Package: rocprofiler-dbgsym -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 974e2f790e32fa43004212e824e55f6e1c90ee0d d5fd45c8dfd91fa5506b8a305a2a6efe51927966 0856ce137bb3d5b3c9f6af8a516d54fa1daa5deb 55fc95b4c3f4cb1d2f298ad711d31214e5a4987d a56e332b72ac8edda270274d80cc3b7dad639809 338039c1f90a130459c3bfb353dd81ee11fbabf9 af7252f9887b44138a579162b9980aa766be4f3e aca552cd112ef86c606ae12f2b8a1f7e53f8c38c e0e6d1c0afa204472cca68231f233292c3078087 2ceb822dc01d982c7a816f271c4c6ad1b7605eb4 557859e8d6582e94a4cf166c95514e87dc5a89fb -Depends: rocprofiler (= 2.0.60202.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocprofiler-dbgsym/rocprofiler-dbgsym_2.0.60202.60202-116~20.04_amd64.deb -Size: 19057118 -SHA256: 1cb2f25746ad0a6fdde727d1a999ddf16cb27e236ab21eae7915e936efcb99cf -SHA1: 40cf99c8eda164268029ecf9ae382725ed3ddfda -MD5sum: 367920391dfb68b25320508bd6576a0d -Description: debug symbols for rocprofiler -Maintainer: ROCm Profiler Support -Package-Type: ddeb -Version: 2.0.60202.60202-116~20.04 -Installed-Size: 65566 - -Package: rocprofiler-dbgsym-rpath6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 974e2f790e32fa43004212e824e55f6e1c90ee0d d5fd45c8dfd91fa5506b8a305a2a6efe51927966 0856ce137bb3d5b3c9f6af8a516d54fa1daa5deb 55fc95b4c3f4cb1d2f298ad711d31214e5a4987d a56e332b72ac8edda270274d80cc3b7dad639809 338039c1f90a130459c3bfb353dd81ee11fbabf9 af7252f9887b44138a579162b9980aa766be4f3e aca552cd112ef86c606ae12f2b8a1f7e53f8c38c e0e6d1c0afa204472cca68231f233292c3078087 2ceb822dc01d982c7a816f271c4c6ad1b7605eb4 557859e8d6582e94a4cf166c95514e87dc5a89fb -Depends: rocprofiler-rpath6.2.2 (= 2.0.60202.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocprofiler-dbgsym-rpath6.2.2/rocprofiler-dbgsym-rpath6.2.2_2.0.60202.60202-116~20.04_amd64.deb -Size: 12099748 -SHA256: 22dc29385ff97e144358789aa0f0e7ed9b89dd1b559ac74e17e506316056642d -SHA1: 531b5d5fc3346fc8680da416675891f740e4dd37 -MD5sum: bcb378f364785c969ae74e2c3275ceb4 -Description: debug symbols for rocprofiler -Maintainer: ROCm Profiler Support -Package-Type: ddeb -Version: 2.0.60202.60202-116~20.04 -Installed-Size: 65566 - -Package: rocprofiler-dbgsym6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 974e2f790e32fa43004212e824e55f6e1c90ee0d d5fd45c8dfd91fa5506b8a305a2a6efe51927966 0856ce137bb3d5b3c9f6af8a516d54fa1daa5deb 55fc95b4c3f4cb1d2f298ad711d31214e5a4987d a56e332b72ac8edda270274d80cc3b7dad639809 338039c1f90a130459c3bfb353dd81ee11fbabf9 af7252f9887b44138a579162b9980aa766be4f3e aca552cd112ef86c606ae12f2b8a1f7e53f8c38c e0e6d1c0afa204472cca68231f233292c3078087 2ceb822dc01d982c7a816f271c4c6ad1b7605eb4 557859e8d6582e94a4cf166c95514e87dc5a89fb -Depends: rocprofiler6.2.2 (= 2.0.60202.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocprofiler-dbgsym6.2.2/rocprofiler-dbgsym6.2.2_2.0.60202.60202-116~20.04_amd64.deb -Size: 12100220 -SHA256: 89f34caaa40c2240fc602463e0fd4b870ab4ebc2ecc80c3277d0926226d9cc3a -SHA1: f846b95ed1a00630b28f9bb96654a8b34dc80744 -MD5sum: 554355f7ef3aec122ca05a871526c61c -Description: debug symbols for rocprofiler -Maintainer: ROCm Profiler Support -Package-Type: ddeb -Version: 2.0.60202.60202-116~20.04 -Installed-Size: 65566 - -Package: rocprofiler-dev -Architecture: amd64 -Depends: rocprofiler, hsa-rocr-dev, rocm-core -Priority: optional -Section: devel -Filename: pool/main/r/rocprofiler-dev/rocprofiler-dev_2.0.60202.60202-116~20.04_amd64.deb -Size: 23888 -SHA256: 204544eb3391989983f2e4161ba479603d0f17546bb8e96f0d74d984996cd4a6 -SHA1: ab6ec19fbf91fccbe4232edc6199ee76aa05798b -MD5sum: ad7b56a41501f6baefe15db51931942b -Description: ROCPROFILER library for AMD HSA runtime API extension support - Development needed header files for ROCProfiler -Maintainer: ROCm Profiler Support -Version: 2.0.60202.60202-116~20.04 -Installed-Size: 133 - -Package: rocprofiler-dev-rpath6.2.2 -Architecture: amd64 -Depends: rocprofiler-rpath6.2.2, hsa-rocr-dev-rpath6.2.2, rocm-core-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocprofiler-dev-rpath6.2.2/rocprofiler-dev-rpath6.2.2_2.0.60202.60202-116~20.04_amd64.deb -Size: 21064 -SHA256: 593ad6d05b161ec444e44460c28e761ab054397d22985352898cf274e2cd0585 -SHA1: f8a5019adeaff60c009841fe12a8d154d7bc2fc2 -MD5sum: 9fc7578ecf51689cf274e4c1a31dce6c -Description: ROCPROFILER library for AMD HSA runtime API extension support - Development needed header files for ROCProfiler -Maintainer: ROCm Profiler Support -Version: 2.0.60202.60202-116~20.04 -Installed-Size: 133 - -Package: rocprofiler-dev6.2.2 -Architecture: amd64 -Depends: rocprofiler6.2.2, hsa-rocr-dev6.2.2, rocm-core6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocprofiler-dev6.2.2/rocprofiler-dev6.2.2_2.0.60202.60202-116~20.04_amd64.deb -Size: 21056 -SHA256: c1335387d2e5a0e48610d57511d1704c9327373391ae0f40276b3e8176cdb5ea -SHA1: 1d4edb5a07254e5a83d3e100bac77fae4f068855 -MD5sum: 73c85c4416f1fc10190af5be89e5e512 -Description: ROCPROFILER library for AMD HSA runtime API extension support - Development needed header files for ROCProfiler -Maintainer: ROCm Profiler Support -Version: 2.0.60202.60202-116~20.04 -Installed-Size: 133 - -Package: rocprofiler-docs -Architecture: amd64 -Depends: rocprofiler-dev, hsa-rocr-dev, rocm-core -Priority: optional -Section: devel -Filename: pool/main/r/rocprofiler-docs/rocprofiler-docs_2.0.60202.60202-116~20.04_amd64.deb -Size: 734 -SHA256: c95b18a5f1c42a8a548737434c83ff9fa1293610c1c02cbed65de8e6ea6d7b93 -SHA1: 62bc0876d7421b83aef42033bd0794e6798cc4bc -MD5sum: 773c87f616ccfd03556c36d48fab8586 -Description: ROCPROFILER library for AMD HSA runtime API extension support - Documentation for the ROCProfiler API -Maintainer: ROCm Profiler Support -Version: 2.0.60202.60202-116~20.04 -Installed-Size: 20 - -Package: rocprofiler-docs-rpath6.2.2 -Architecture: amd64 -Depends: rocprofiler-dev-rpath6.2.2, hsa-rocr-dev-rpath6.2.2, rocm-core-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocprofiler-docs-rpath6.2.2/rocprofiler-docs-rpath6.2.2_2.0.60202.60202-116~20.04_amd64.deb -Size: 944 -SHA256: a51824ba1e47265fbc4412850d66662ea0adf8c9d7381d4ef59bde635551ffb7 -SHA1: 2d28092d270da3ef0e2a333822282dbebfeaafd0 -MD5sum: c32f6cbe3ce6c6ded1431c08a62f8f0a -Description: ROCPROFILER library for AMD HSA runtime API extension support - Documentation for the ROCProfiler API -Maintainer: ROCm Profiler Support -Version: 2.0.60202.60202-116~20.04 -Installed-Size: 20 - -Package: rocprofiler-docs6.2.2 -Architecture: amd64 -Depends: rocprofiler-dev6.2.2, hsa-rocr-dev6.2.2, rocm-core6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocprofiler-docs6.2.2/rocprofiler-docs6.2.2_2.0.60202.60202-116~20.04_amd64.deb -Size: 932 -SHA256: c28f5492f816f7329db80e2a225efee69e11f477909f97a083aed8afd0d416ac -SHA1: d1d0986545ac7cabb6298b5b10680e503117a3e6 -MD5sum: f70fcacd5c190250ced71685af065102 -Description: ROCPROFILER library for AMD HSA runtime API extension support - Documentation for the ROCProfiler API -Maintainer: ROCm Profiler Support -Version: 2.0.60202.60202-116~20.04 -Installed-Size: 20 - -Package: rocprofiler-plugins -Architecture: amd64 -Depends: rocprofiler, hsa-rocr-dev, rocm-core -Priority: optional -Section: devel -Filename: pool/main/r/rocprofiler-plugins/rocprofiler-plugins_2.0.60202.60202-116~20.04_amd64.deb -Size: 1032224 -SHA256: 063553c3c9aac322ccaaf0c5817e59f5ccfc828750133c8d0b5dea24d511ccf7 -SHA1: 28b3a1e69c3f305463df307fa846bbca27c721bd -MD5sum: 5b87c82e6e568fd4e6b9285ac112b7f4 -Description: ROCPROFILER library for AMD HSA runtime API extension support - Plugins for handling ROCProfiler data output -Maintainer: ROCm Profiler Support -Version: 2.0.60202.60202-116~20.04 -Installed-Size: 4258 - -Package: rocprofiler-plugins-dbgsym -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: c152c2ca27ad5ea978364a3cce516da4212f082a c48a77ed9bcc1569020b3c30eefe2213b080e507 fa64e6d90c8a16a1f2f352f56317a156d3f5fc14 -Depends: rocprofiler-plugins (= 2.0.60202.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocprofiler-plugins-dbgsym/rocprofiler-plugins-dbgsym_2.0.60202.60202-116~20.04_amd64.deb -Size: 20630470 -SHA256: d6f9a59582f98e96af92456da57d7865eac60f3744c64be65db91f8192c888b0 -SHA1: a59d453bc50548771ba6e159fd922b25c2a89661 -MD5sum: 35daae8fd2145c71bd4f7770ab8b689e -Description: debug symbols for rocprofiler-plugins -Maintainer: ROCm Profiler Support -Package-Type: ddeb -Version: 2.0.60202.60202-116~20.04 -Installed-Size: 84043 - -Package: rocprofiler-plugins-dbgsym-rpath6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: c152c2ca27ad5ea978364a3cce516da4212f082a c48a77ed9bcc1569020b3c30eefe2213b080e507 fa64e6d90c8a16a1f2f352f56317a156d3f5fc14 -Depends: rocprofiler-plugins-rpath6.2.2 (= 2.0.60202.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocprofiler-plugins-dbgsym-rpath6.2.2/rocprofiler-plugins-dbgsym-rpath6.2.2_2.0.60202.60202-116~20.04_amd64.deb -Size: 13543620 -SHA256: 7169700aedcd9ea352c9dad956a612d5445c22d9cf27797769e8f0ff4abfa244 -SHA1: 06b68456fb149d023e04ac79d3c25a81b28b9c08 -MD5sum: 898392ceae398756c3ba04f4d876febf -Description: debug symbols for rocprofiler-plugins -Maintainer: ROCm Profiler Support -Package-Type: ddeb -Version: 2.0.60202.60202-116~20.04 -Installed-Size: 84043 - -Package: rocprofiler-plugins-dbgsym6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: c152c2ca27ad5ea978364a3cce516da4212f082a c48a77ed9bcc1569020b3c30eefe2213b080e507 fa64e6d90c8a16a1f2f352f56317a156d3f5fc14 -Depends: rocprofiler-plugins6.2.2 (= 2.0.60202.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocprofiler-plugins-dbgsym6.2.2/rocprofiler-plugins-dbgsym6.2.2_2.0.60202.60202-116~20.04_amd64.deb -Size: 13537796 -SHA256: 8f93fb32481659086c74b0df2bca055b6dba04862ab96eb437d4f4b4a63dc4d6 -SHA1: 01089ea8f1dc1c0b7ada786713c7061355bddfe8 -MD5sum: 549cac5fa0a3714f32d077d16565ca31 -Description: debug symbols for rocprofiler-plugins -Maintainer: ROCm Profiler Support -Package-Type: ddeb -Version: 2.0.60202.60202-116~20.04 -Installed-Size: 84043 - -Package: rocprofiler-plugins-rpath6.2.2 -Architecture: amd64 -Depends: rocprofiler-rpath6.2.2, hsa-rocr-dev-rpath6.2.2, rocm-core-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocprofiler-plugins-rpath6.2.2/rocprofiler-plugins-rpath6.2.2_2.0.60202.60202-116~20.04_amd64.deb -Size: 714032 -SHA256: 0d476e92c0683814caa1d4daeb62ce6032b5044c3bf44ab5e39afd73b8b98993 -SHA1: 978abbb5519396daa0a38b573e19147c67a2fce4 -MD5sum: 6a4d6b8a38d1db287ffed1086e53b142 -Description: ROCPROFILER library for AMD HSA runtime API extension support - Plugins for handling ROCProfiler data output -Maintainer: ROCm Profiler Support -Version: 2.0.60202.60202-116~20.04 -Installed-Size: 4258 - -Package: rocprofiler-plugins6.2.2 -Architecture: amd64 -Depends: rocprofiler6.2.2, hsa-rocr-dev6.2.2, rocm-core6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocprofiler-plugins6.2.2/rocprofiler-plugins6.2.2_2.0.60202.60202-116~20.04_amd64.deb -Size: 713420 -SHA256: 973a0ec68986dcf9d1edcc741f4b9cd53a1bd703c821a7c85c099771c214aed9 -SHA1: b4bb7dcbcfab7361b603879a281e61a6af88b001 -MD5sum: 51f212f53660aa059ab2d5c6aac04439 -Description: ROCPROFILER library for AMD HSA runtime API extension support - Plugins for handling ROCProfiler data output -Maintainer: ROCm Profiler Support -Version: 2.0.60202.60202-116~20.04 -Installed-Size: 4258 - -Package: rocprofiler-register -Architecture: amd64 -Depends: rocm-core, libc6 (>= 2.14), libgcc-s1 (>= 3.3), libstdc++6 (>= 9) -Priority: optional -Section: devel -Filename: pool/main/r/rocprofiler-register/rocprofiler-register_0.4.0.60202-116~20.04_amd64.deb -Size: 221060 -SHA256: 8e4f58f503ba6b96b11b802c089b8a61e5e1d61435a2c16b4f46c1f1ae93de45 -SHA1: bf3b5dd78f6b7668ef1792cd818ac586904120b7 -MD5sum: 1f57116ca4f39f653a382d019e6780d5 -Description: Registration library for rocprofiler -Homepage: https://github.com/ROCm/rocprofiler-register-internal -Maintainer: ROCm Profiler Support -Version: 0.4.0.60202-116~20.04 -Installed-Size: 1713 - -Package: rocprofiler-register-rpath6.2.2 -Architecture: amd64 -Depends: rocm-core-rpath6.2.2, libc6 (>= 2.14), libgcc-s1 (>= 3.3), libstdc++6 (>= 9) -Priority: optional -Section: devel -Filename: pool/main/r/rocprofiler-register-rpath6.2.2/rocprofiler-register-rpath6.2.2_0.4.0.60202-116~20.04_amd64.deb -Size: 178912 -SHA256: 0a2244ab74b46726f2e05f01690c9411061ad251790aaa1681649c535d3c54e0 -SHA1: 7a2f8fa5cb21ada1374bff1aba1170a6eb5acd88 -MD5sum: 46d2bc4975578dcba8cad9e96f1c2143 -Description: Registration library for rocprofiler -Homepage: https://github.com/ROCm/rocprofiler-register-internal -Maintainer: ROCm Profiler Support -Version: 0.4.0.60202-116~20.04 -Installed-Size: 1713 - -Package: rocprofiler-register6.2.2 -Architecture: amd64 -Depends: rocm-core6.2.2, libc6 (>= 2.14), libgcc-s1 (>= 3.3), libstdc++6 (>= 9) -Priority: optional -Section: devel -Filename: pool/main/r/rocprofiler-register6.2.2/rocprofiler-register6.2.2_0.4.0.60202-116~20.04_amd64.deb -Size: 179064 -SHA256: 126a87ce383bbdadbf50265b57691935353f595a286bc2fa1d1e4fbd55d5a5ad -SHA1: b7dc6b650e8dd7225919d21a5b2f8cf23dfdf914 -MD5sum: 1ae6e6df6e05a427062932ca6fb05770 -Description: Registration library for rocprofiler -Homepage: https://github.com/ROCm/rocprofiler-register-internal -Maintainer: ROCm Profiler Support -Version: 0.4.0.60202-116~20.04 -Installed-Size: 1713 - -Package: rocprofiler-rpath6.2.2 -Architecture: amd64 -Depends: rocminfo-rpath6.2.2, hsa-rocr-rpath6.2.2, rocm-core-rpath6.2.2, libsystemd-dev, libelf-dev, libnuma-dev, libpciaccess-dev, libxml2-dev -Priority: optional -Section: devel -Filename: pool/main/r/rocprofiler-rpath6.2.2/rocprofiler-rpath6.2.2_2.0.60202.60202-116~20.04_amd64.deb -Size: 574008 -SHA256: 446aff24ec1c8ad500baa188ea415f9389f536506952abf65f824984195d69c2 -SHA1: 0b50506b15210733067a1e3c20eb3c49053a83f2 -MD5sum: b01cc9f268982d356f1e0b355d34e54d -Description: ROCPROFILER library for AMD HSA runtime API extension support - Dynamic libraries for the ROCProfiler -Maintainer: ROCm Profiler Support -Version: 2.0.60202.60202-116~20.04 -Installed-Size: 4105 - -Package: rocprofiler-sdk -Architecture: amd64 -Depends: rocm-core -Priority: optional -Section: devel -Filename: pool/main/r/rocprofiler-sdk/rocprofiler-sdk_0.4.0-116~20.04_amd64.deb -Size: 2963142 -SHA256: f5fda75523fb4a4609a82543a6b7ffedc7e29b0d0db6d78e4a7099c719f46050 -SHA1: 6428e7834f35adaacca21f6211a49f3035a55cb6 -MD5sum: 01a7ebc01abd11e1c2b4bc63ba853658 -Description: ROCm GPU performance analysis SDK -Homepage: https://github.com/ROCm/rocprofiler-sdk -Maintainer: ROCm Profiler Support -Version: 0.4.0-116~20.04 -Installed-Size: 28487 - -Package: rocprofiler-sdk-dbgsym -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: baa4214b9a23bd6568d0021bc1ee09f6cdc222c0 964ddf6f5dbdc1acb9fb4045692585e364e7db58 43d66679e69ad8a5adf060d6e7bb5eb79db0ec8d -Depends: rocprofiler-sdk (= 0.4.0-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocprofiler-sdk-dbgsym/rocprofiler-sdk-dbgsym_0.4.0-116~20.04_amd64.deb -Size: 60250376 -SHA256: efc10b9cd8e823acbc3506d41b68cee49154473a5b28b44b53e869b3cc0cb81e -SHA1: 963afd205f8726a3835a84101a06005f54d07987 -MD5sum: 6ca1a8f4ed973f6aaf208d8f970413b9 -Description: debug symbols for rocprofiler-sdk -Maintainer: ROCm Profiler Support -Package-Type: ddeb -Version: 0.4.0-116~20.04 -Installed-Size: 248893 - -Package: rocprofiler-sdk-dbgsym-rpath6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: baa4214b9a23bd6568d0021bc1ee09f6cdc222c0 964ddf6f5dbdc1acb9fb4045692585e364e7db58 43d66679e69ad8a5adf060d6e7bb5eb79db0ec8d -Depends: rocprofiler-sdk-rpath6.2.2 (= 0.4.0-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocprofiler-sdk-dbgsym-rpath6.2.2/rocprofiler-sdk-dbgsym-rpath6.2.2_0.4.0-116~20.04_amd64.deb -Size: 38870740 -SHA256: 2e95c443d9341b55c5f63fa1ca7162dee359db086dd16762b84c18ff5195dbf6 -SHA1: 5b9be91fe50d76039e3b5d2c22b4c6f64d1e5ef9 -MD5sum: b9b7b84e2aa912eb7d1ca80ed101db75 -Description: debug symbols for rocprofiler-sdk -Maintainer: ROCm Profiler Support -Package-Type: ddeb -Version: 0.4.0-116~20.04 -Installed-Size: 248893 - -Package: rocprofiler-sdk-dbgsym6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: baa4214b9a23bd6568d0021bc1ee09f6cdc222c0 964ddf6f5dbdc1acb9fb4045692585e364e7db58 43d66679e69ad8a5adf060d6e7bb5eb79db0ec8d -Depends: rocprofiler-sdk6.2.2 (= 0.4.0-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocprofiler-sdk-dbgsym6.2.2/rocprofiler-sdk-dbgsym6.2.2_0.4.0-116~20.04_amd64.deb -Size: 38867456 -SHA256: 42bcba22a6d1a72441157bf8032b24094016993611423e08a45e96516bcee137 -SHA1: 8bea0bdd761041864c3fa6da4251bfb5734e9f1b -MD5sum: fa934298097d3bab5e02b2940d92567b -Description: debug symbols for rocprofiler-sdk -Maintainer: ROCm Profiler Support -Package-Type: ddeb -Version: 0.4.0-116~20.04 -Installed-Size: 248893 - -Package: rocprofiler-sdk-roctx -Architecture: amd64 -Depends: rocm-core -Priority: optional -Section: devel -Filename: pool/main/r/rocprofiler-sdk-roctx/rocprofiler-sdk-roctx_0.4.0-116~20.04_amd64.deb -Size: 204782 -SHA256: 3d7e667b4c0cbed78642a3b272f4f917e3d7fc67199ca75b64e87677103feca0 -SHA1: d284c83a4c36cf793363a237e5f06620ef5c4135 -MD5sum: d260e541238cdd11aea9f4a7b0c4c149 -Description: ROCm GPU performance analysis SDK -Homepage: https://github.com/ROCm/rocprofiler-sdk -Maintainer: ROCm Profiler Support -Version: 0.4.0-116~20.04 -Installed-Size: 1513 - -Package: rocprofiler-sdk-roctx-dbgsym -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 9893677c91fdf2a0d301ff6662184655830bbdca -Depends: rocprofiler-sdk-roctx (= 0.4.0-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocprofiler-sdk-roctx-dbgsym/rocprofiler-sdk-roctx-dbgsym_0.4.0-116~20.04_amd64.deb -Size: 2585796 -SHA256: d9b5b5a10651aa991896a03a06162b04c485ca8a7e9fe8f07774ac731dd675e5 -SHA1: ca9b29c76afee3c027cf1c17d3b36f87e31e38aa -MD5sum: 1406d8b77124ceb3e1c33836617f3799 -Description: debug symbols for rocprofiler-sdk-roctx -Maintainer: ROCm Profiler Support -Package-Type: ddeb -Version: 0.4.0-116~20.04 -Installed-Size: 9277 - -Package: rocprofiler-sdk-roctx-dbgsym-rpath6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 9893677c91fdf2a0d301ff6662184655830bbdca -Depends: rocprofiler-sdk-roctx-rpath6.2.2 (= 0.4.0-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocprofiler-sdk-roctx-dbgsym-rpath6.2.2/rocprofiler-sdk-roctx-dbgsym-rpath6.2.2_0.4.0-116~20.04_amd64.deb -Size: 1792696 -SHA256: bd94ae8356a635177ecc7d188836171776b65c417bc7f61ed6ec4704e75c7076 -SHA1: 3d8e7fbe828d668664b78b1fd1758471295e29a7 -MD5sum: b7725bae5558115d5d048e63d34ee43f -Description: debug symbols for rocprofiler-sdk-roctx -Maintainer: ROCm Profiler Support -Package-Type: ddeb -Version: 0.4.0-116~20.04 -Installed-Size: 9277 - -Package: rocprofiler-sdk-roctx-dbgsym6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 9893677c91fdf2a0d301ff6662184655830bbdca -Depends: rocprofiler-sdk-roctx6.2.2 (= 0.4.0-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocprofiler-sdk-roctx-dbgsym6.2.2/rocprofiler-sdk-roctx-dbgsym6.2.2_0.4.0-116~20.04_amd64.deb -Size: 1793464 -SHA256: c7e48ea7d4800e9cd2759f20f645d6757809c91caf26b991d158d9b323f9fcb1 -SHA1: 52809f9283f6841754ccae71813813af639cd459 -MD5sum: 05e656722631dd542ed6b02a5c73cb7f -Description: debug symbols for rocprofiler-sdk-roctx -Maintainer: ROCm Profiler Support -Package-Type: ddeb -Version: 0.4.0-116~20.04 -Installed-Size: 9277 - -Package: rocprofiler-sdk-roctx-rpath6.2.2 -Architecture: amd64 -Depends: rocm-core-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocprofiler-sdk-roctx-rpath6.2.2/rocprofiler-sdk-roctx-rpath6.2.2_0.4.0-116~20.04_amd64.deb -Size: 168124 -SHA256: ab0e706a11a7e5fe167828714f16785ce0f82956868a178a763674294f6bec98 -SHA1: 2abca78562b874e6e31abd5cda005f5ead14b404 -MD5sum: 89871f0b7e6a680e861e5f00bcab3424 -Description: ROCm GPU performance analysis SDK -Homepage: https://github.com/ROCm/rocprofiler-sdk -Maintainer: ROCm Profiler Support -Version: 0.4.0-116~20.04 -Installed-Size: 1513 - -Package: rocprofiler-sdk-roctx6.2.2 -Architecture: amd64 -Depends: rocm-core6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocprofiler-sdk-roctx6.2.2/rocprofiler-sdk-roctx6.2.2_0.4.0-116~20.04_amd64.deb -Size: 168152 -SHA256: ca80c9bd3c0c497f38bdfaa8cdc4453ee9a541be4f4da24f7300ad04033a3b08 -SHA1: ddc067a02cefbc1b630e15a2ad95214a1d6b288f -MD5sum: 72130ab04b2faa0e3dc2a2a0f8bacfd5 -Description: ROCm GPU performance analysis SDK -Homepage: https://github.com/ROCm/rocprofiler-sdk -Maintainer: ROCm Profiler Support -Version: 0.4.0-116~20.04 -Installed-Size: 1513 - -Package: rocprofiler-sdk-rpath6.2.2 -Architecture: amd64 -Depends: rocm-core-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocprofiler-sdk-rpath6.2.2/rocprofiler-sdk-rpath6.2.2_0.4.0-116~20.04_amd64.deb -Size: 1919476 -SHA256: 693654d2282a3e123647467bfdde92c40537d805cbf34485d3ae41dd8259a866 -SHA1: a6eeb25432e23828b8da9543c5a9cfde328ced6d -MD5sum: 5f111ffbe2cd116b6d483531a3913bf4 -Description: ROCm GPU performance analysis SDK -Homepage: https://github.com/ROCm/rocprofiler-sdk -Maintainer: ROCm Profiler Support -Version: 0.4.0-116~20.04 -Installed-Size: 28487 - -Package: rocprofiler-sdk6.2.2 -Architecture: amd64 -Depends: rocm-core6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocprofiler-sdk6.2.2/rocprofiler-sdk6.2.2_0.4.0-116~20.04_amd64.deb -Size: 1919960 -SHA256: 6014979516ec691c4f436d7fe90b0668b8f7e4efaed062dedab21ac0b772141e -SHA1: cffa80efad95649b4f878a4cb91512bb116c3a64 -MD5sum: c5de5ca9d92c5c8b8296f73d3e802f77 -Description: ROCm GPU performance analysis SDK -Homepage: https://github.com/ROCm/rocprofiler-sdk -Maintainer: ROCm Profiler Support -Version: 0.4.0-116~20.04 -Installed-Size: 28487 - -Package: rocprofiler6.2.2 -Architecture: amd64 -Depends: rocminfo6.2.2, hsa-rocr6.2.2, rocm-core6.2.2, libsystemd-dev, libelf-dev, libnuma-dev, libpciaccess-dev, libxml2-dev -Priority: optional -Section: devel -Filename: pool/main/r/rocprofiler6.2.2/rocprofiler6.2.2_2.0.60202.60202-116~20.04_amd64.deb -Size: 573852 -SHA256: 421273c0f1a463485b525fb5ba15be11a725c66cee58600f34f455918d689d07 -SHA1: 96ae159c59446cb0ee351089e2b55c02ebfa4ac1 -MD5sum: 26f7c6730fd55d647af5cb13d0b3398c -Description: ROCPROFILER library for AMD HSA runtime API extension support - Dynamic libraries for the ROCProfiler -Maintainer: ROCm Profiler Support -Version: 2.0.60202.60202-116~20.04 -Installed-Size: 4105 - -Package: rocrand -Architecture: amd64 -Depends: rocm-core -Priority: optional -Section: devel -Filename: pool/main/r/rocrand/rocrand_3.1.0.60202-116~20.04_amd64.deb -Size: 21785558 -SHA256: ead09f5f71311e32f4c24258e4d2566b5179f3d54c60d0172b8dc69cbb3e83fb -SHA1: 9bfbab700a04340aeb3dee75ef85861f90972263 -MD5sum: aa4d041989aa7cbe0afce5a9169c03b1 -Description: Radeon Open Compute RAND library -Maintainer: rocRAND Maintainer -Recommends: rocrand-dev (>=3.1.0.60202) -Version: 3.1.0.60202-116~20.04 -Installed-Size: 301186 - -Package: rocrand-asan -Architecture: amd64 -Depends: rocm-core-asan -Priority: optional -Section: devel -Filename: pool/main/r/rocrand-asan/rocrand-asan_3.1.0.60202-116~20.04_amd64.deb -Size: 28712162 -SHA256: 5e44d6a0afe388923834957ff2e3ae05cc67aa803df11538077a2b29fc192565 -SHA1: 4524194df356cddbbd593985700cc81972d68bb6 -MD5sum: 085315bb9097693dc77771fbe0c94ff8 -Description: Radeon Open Compute RAND library -Maintainer: rocRAND Maintainer -Recommends: rocrand-asan-dev (>=3.1.0.60202) -Version: 3.1.0.60202-116~20.04 -Installed-Size: 228593 - -Package: rocrand-asan-dbgsym -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 8cb2667c3aa0fb0b71ba0d1c78148b308b1997ee -Depends: rocrand-asan (= 3.1.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocrand-asan-dbgsym/rocrand-asan-dbgsym_3.1.0.60202-116~20.04_amd64.deb -Size: 4506518 -SHA256: 392efdb70931ac7c8f3930b910d4245ed8a85547a895b785126e44829badee1b -SHA1: b00def4b2d31dd0a616b81701556a840569d504d -MD5sum: 9b931293bea98e94187d1a8336b3022d -Description: debug symbols for rocrand-asan -Maintainer: rocRAND Maintainer -Package-Type: ddeb -Version: 3.1.0.60202-116~20.04 -Installed-Size: 5760 - -Package: rocrand-asan-dbgsym-rpath6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 8cb2667c3aa0fb0b71ba0d1c78148b308b1997ee -Depends: rocrand-asan-rpath6.2.2 (= 3.1.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocrand-asan-dbgsym-rpath6.2.2/rocrand-asan-dbgsym-rpath6.2.2_3.1.0.60202-116~20.04_amd64.deb -Size: 4506700 -SHA256: 21acc0661a06d7505e187b0b15f6bbae362d8d76f74747670f40c13bd504db33 -SHA1: 4969dc46468d7e6fbbdfae2a23b588f6dbd1224d -MD5sum: 3a6d8f71abcea5683cc19615110b1a4a -Description: debug symbols for rocrand-asan -Maintainer: rocRAND Maintainer -Package-Type: ddeb -Version: 3.1.0.60202-116~20.04 -Installed-Size: 5760 - -Package: rocrand-asan-dbgsym6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 8cb2667c3aa0fb0b71ba0d1c78148b308b1997ee -Depends: rocrand-asan6.2.2 (= 3.1.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocrand-asan-dbgsym6.2.2/rocrand-asan-dbgsym6.2.2_3.1.0.60202-116~20.04_amd64.deb -Size: 4506888 -SHA256: ef203593bd93c6d5e0afca832e47fee653b8ce359efeabfccf269e5559d097ea -SHA1: 87f5d9a8a8d16fd7577276bf4318ba6ed12f607d -MD5sum: a2f0b928f2a4347a3e169f3330fd4891 -Description: debug symbols for rocrand-asan -Maintainer: rocRAND Maintainer -Package-Type: ddeb -Version: 3.1.0.60202-116~20.04 -Installed-Size: 5760 - -Package: rocrand-asan-rpath6.2.2 -Architecture: amd64 -Depends: rocm-core-asan-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocrand-asan-rpath6.2.2/rocrand-asan-rpath6.2.2_3.1.0.60202-116~20.04_amd64.deb -Size: 28815016 -SHA256: d1bf9fb0c3942d6bb5e4066eee2acf36718b499e44471d10d2cd9c3893b0cd4b -SHA1: afd9dac223a824a5a1b95d086c69ae91dd4a8b76 -MD5sum: b7f4dc22dad5e3c926e1c8a41d32aa40 -Description: Radeon Open Compute RAND library -Maintainer: rocRAND Maintainer -Recommends: rocrand-asan-dev (>=3.1.0.60202) -Version: 3.1.0.60202-116~20.04 -Installed-Size: 228593 - -Package: rocrand-asan6.2.2 -Architecture: amd64 -Depends: rocm-core-asan6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocrand-asan6.2.2/rocrand-asan6.2.2_3.1.0.60202-116~20.04_amd64.deb -Size: 28815056 -SHA256: 90f78b32b29609e41d4ee0dc1b8ef332c868b4b307f854c10e52645c9b9257bb -SHA1: a2bda93ccd19e504cbc574b593d323b67c71b642 -MD5sum: 9a75ffa2fba235adb9df17018c214410 -Description: Radeon Open Compute RAND library -Maintainer: rocRAND Maintainer -Recommends: rocrand-asan-dev (>=3.1.0.60202) -Version: 3.1.0.60202-116~20.04 -Installed-Size: 228593 - -Package: rocrand-dbgsym -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 5dedcfd30c92ca38e445da59b18b60a81d051dc6 -Depends: rocrand (= 3.1.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocrand-dbgsym/rocrand-dbgsym_3.1.0.60202-116~20.04_amd64.deb -Size: 1932244 -SHA256: b9c9130f156c90e71160902eba1e13a06721ea536a2e0e0cd5aea3bd6b079671 -SHA1: c58146f96551864abe36d7e2bcc03ee8cddb6925 -MD5sum: 0dd3dd51e4387e8274368cbd8a721fa1 -Description: debug symbols for rocrand -Maintainer: rocRAND Maintainer -Package-Type: ddeb -Version: 3.1.0.60202-116~20.04 -Installed-Size: 14848 - -Package: rocrand-dbgsym-rpath6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 5dedcfd30c92ca38e445da59b18b60a81d051dc6 -Depends: rocrand-rpath6.2.2 (= 3.1.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocrand-dbgsym-rpath6.2.2/rocrand-dbgsym-rpath6.2.2_3.1.0.60202-116~20.04_amd64.deb -Size: 1926472 -SHA256: 0819a61b8713009da60191c595e96ce77109b0d48967bf6a0009231a72543530 -SHA1: a9418109b0d85321d074aa8cd3369bc84e043dae -MD5sum: c9e49cf6b27840f0b860b079934bd011 -Description: debug symbols for rocrand -Maintainer: rocRAND Maintainer -Package-Type: ddeb -Version: 3.1.0.60202-116~20.04 -Installed-Size: 14848 - -Package: rocrand-dbgsym6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 5dedcfd30c92ca38e445da59b18b60a81d051dc6 -Depends: rocrand6.2.2 (= 3.1.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocrand-dbgsym6.2.2/rocrand-dbgsym6.2.2_3.1.0.60202-116~20.04_amd64.deb -Size: 1931220 -SHA256: d9e90a577fc7e1ba3cc75455368776b8a6d38b64184da5ba1eefdadea4a365cb -SHA1: 6a3559f1c9c2a86b0a304e4b6518fac9e4ad2ce2 -MD5sum: 4409c9d9913a59e7160fd3f2b59f6d11 -Description: debug symbols for rocrand -Maintainer: rocRAND Maintainer -Package-Type: ddeb -Version: 3.1.0.60202-116~20.04 -Installed-Size: 14848 - -Package: rocrand-dev -Architecture: amd64 -Depends: rocrand (>= 3.1.0.60202) -Priority: optional -Section: devel -Filename: pool/main/r/rocrand-dev/rocrand-dev_3.1.0.60202-116~20.04_amd64.deb -Size: 543388 -SHA256: 08ab02a3342c71893c7f81b56277863855975dc838463b95fa7f914f912690d9 -SHA1: 692368e85a87972f9905d16fd517711fc5c7c06e -MD5sum: 23afbd103a112ea2641a32afdd26baad -Description: Radeon Open Compute RAND library -Maintainer: rocRAND Maintainer -Version: 3.1.0.60202-116~20.04 -Installed-Size: 3794 - -Package: rocrand-dev-rpath6.2.2 -Architecture: amd64 -Depends: rocrand-rpath6.2.2 (>= 3.1.0.60202) -Priority: optional -Section: devel -Filename: pool/main/r/rocrand-dev-rpath6.2.2/rocrand-dev-rpath6.2.2_3.1.0.60202-116~20.04_amd64.deb -Size: 543376 -SHA256: 3aefd6f046cb090afb0f57da7d9ddfd585dbca522fca964cd8447a803877072e -SHA1: d64ea7656037fa4fe563084cccb681cfc5f4fb68 -MD5sum: b16a8e4869236951ced9a2b1fd836438 -Description: Radeon Open Compute RAND library -Maintainer: rocRAND Maintainer -Version: 3.1.0.60202-116~20.04 -Installed-Size: 3794 - -Package: rocrand-dev6.2.2 -Architecture: amd64 -Depends: rocrand6.2.2 (>= 3.1.0.60202) -Priority: optional -Section: devel -Filename: pool/main/r/rocrand-dev6.2.2/rocrand-dev6.2.2_3.1.0.60202-116~20.04_amd64.deb -Size: 544888 -SHA256: 5b63b076f3d7f2d046919de82e8e4cdfc552e0900fd5b369918bd14e4d47c778 -SHA1: 65d018fec47a52fb26e6781887a148c71b0255c4 -MD5sum: 012abcb6f41d98e7696375683889d28f -Description: Radeon Open Compute RAND library -Maintainer: rocRAND Maintainer -Version: 3.1.0.60202-116~20.04 -Installed-Size: 3794 - -Package: rocrand-rpath6.2.2 -Architecture: amd64 -Depends: rocm-core-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocrand-rpath6.2.2/rocrand-rpath6.2.2_3.1.0.60202-116~20.04_amd64.deb -Size: 21713060 -SHA256: 4e35b0953f020eae0e963c6efa0fbf3f71b735894fcc2cb21548a972d38fdc67 -SHA1: 0e11d72a111732eee4af4bcf6409b8db3b11da3b -MD5sum: 053e0bd6101e6f45bbc3f02ad4df87d6 -Description: Radeon Open Compute RAND library -Maintainer: rocRAND Maintainer -Recommends: rocrand-dev-rpath6.2.2 (>=3.1.0.60202) -Version: 3.1.0.60202-116~20.04 -Installed-Size: 301186 - -Package: rocrand6.2.2 -Architecture: amd64 -Depends: rocm-core6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocrand6.2.2/rocrand6.2.2_3.1.0.60202-116~20.04_amd64.deb -Size: 21691444 -SHA256: 6aae5b7bb19a826b28b454162b511da13143cbb38497e9591b4719da0e1449c4 -SHA1: b09cad065d3304540315eabf8f5f75d294372f66 -MD5sum: 6526e14ca39c83d551d15dc0869f3c7c -Description: Radeon Open Compute RAND library -Maintainer: rocRAND Maintainer -Recommends: rocrand-dev6.2.2 (>=3.1.0.60202) -Version: 3.1.0.60202-116~20.04 -Installed-Size: 301186 - -Package: rocsolver -Architecture: amd64 -Depends: rocblas (>= 4.2), rocsparse (>= 2.2), rocm-core -Priority: optional -Section: devel -Filename: pool/main/r/rocsolver/rocsolver_3.26.0.60202-116~20.04_amd64.deb -Size: 251150948 -SHA256: e96465d620b4818b00a2c48144c50caca1e9c15cfde12acad18750885cf324ca -SHA1: abb58a6cc07e1b4374c28ba5c2ab0630156b6b03 -MD5sum: bfe820518192be0c8115a292fc9e62c1 -Description: AMD ROCm SOLVER library -Maintainer: RocSOLVER maintainer -Recommends: rocsolver-dev (>=3.26.0.60202) -Version: 3.26.0.60202-116~20.04 -Installed-Size: 3383329 - -Package: rocsolver-asan -Architecture: amd64 -Depends: rocblas (>= 4.2), rocsparse (>= 2.2), rocm-core-asan -Priority: optional -Section: devel -Filename: pool/main/r/rocsolver-asan/rocsolver-asan_3.26.0.60202-116~20.04_amd64.deb -Size: 268058028 -SHA256: 3891cd9913fe87cf70fe2c804574f8953faf8da0caffa310b4f8657ceb482f11 -SHA1: 59f1f2112374d1947fcd8e295e9ebe3b0e455ae1 -MD5sum: a9618c5663a2c053d0df53c2e1087d9d -Description: AMD ROCm SOLVER library -Maintainer: RocSOLVER maintainer -Recommends: rocsolver-asan-dev (>=3.26.0.60202) -Version: 3.26.0.60202-116~20.04 -Installed-Size: 3641644 - -Package: rocsolver-asan-dbgsym -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 34a236b61698beef1179a396999403ace810fcc9 -Depends: rocsolver-asan (= 3.26.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocsolver-asan-dbgsym/rocsolver-asan-dbgsym_3.26.0.60202-116~20.04_amd64.deb -Size: 126663428 -SHA256: 48e513863d466e3cf45c05a723976705c12262a8bab3e1249e09f082aecc0009 -SHA1: 4c93a34616bb11c5bcb4c3daf86606371001519a -MD5sum: b1d5e42ef2cb7d809c3d17827cb14f0b -Description: debug symbols for rocsolver-asan -Maintainer: RocSOLVER maintainer -Package-Type: ddeb -Version: 3.26.0.60202-116~20.04 -Installed-Size: 140468 - -Package: rocsolver-asan-dbgsym-rpath6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 34a236b61698beef1179a396999403ace810fcc9 -Depends: rocsolver-asan-rpath6.2.2 (= 3.26.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocsolver-asan-dbgsym-rpath6.2.2/rocsolver-asan-dbgsym-rpath6.2.2_3.26.0.60202-116~20.04_amd64.deb -Size: 126662544 -SHA256: d23679eecaaf96c8252eb48de4bcd6721f01839e37f0c23e0efa02c6e1c829cb -SHA1: 864608de0cf29ff15997dc622545fcb8fa0b3a4b -MD5sum: de174668cd49865e0c19b7d3a8965a7b -Description: debug symbols for rocsolver-asan -Maintainer: RocSOLVER maintainer -Package-Type: ddeb -Version: 3.26.0.60202-116~20.04 -Installed-Size: 140468 - -Package: rocsolver-asan-dbgsym6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 34a236b61698beef1179a396999403ace810fcc9 -Depends: rocsolver-asan6.2.2 (= 3.26.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocsolver-asan-dbgsym6.2.2/rocsolver-asan-dbgsym6.2.2_3.26.0.60202-116~20.04_amd64.deb -Size: 126663064 -SHA256: fd268e86ade5fe5c8520f0711f15c44be37b7b430c1411f81e6b421b468e48e7 -SHA1: 88fe68a417fbf9c5343dcf1193f82c22237c3111 -MD5sum: c9a39b2fd1f500884d24140e8a3e9e07 -Description: debug symbols for rocsolver-asan -Maintainer: RocSOLVER maintainer -Package-Type: ddeb -Version: 3.26.0.60202-116~20.04 -Installed-Size: 140468 - -Package: rocsolver-asan-rpath6.2.2 -Architecture: amd64 -Depends: rocblas (>= 4.2), rocsparse (>= 2.2), rocm-core-asan-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocsolver-asan-rpath6.2.2/rocsolver-asan-rpath6.2.2_3.26.0.60202-116~20.04_amd64.deb -Size: 268057152 -SHA256: 38566bb7465ea2ae446c729bb8e6c3f4edee738154bd5089d62842356a5c5e71 -SHA1: 67a2e0ff4f20fad5f3dd7f4506b217e1b531eb62 -MD5sum: 0d8790ddaf9a2b4ed99c30b1de8fdf3d -Description: AMD ROCm SOLVER library -Maintainer: RocSOLVER maintainer -Recommends: rocsolver-asan-dev (>=3.26.0.60202) -Version: 3.26.0.60202-116~20.04 -Installed-Size: 3641644 - -Package: rocsolver-asan6.2.2 -Architecture: amd64 -Depends: rocblas (>= 4.2), rocsparse (>= 2.2), rocm-core-asan6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocsolver-asan6.2.2/rocsolver-asan6.2.2_3.26.0.60202-116~20.04_amd64.deb -Size: 268058712 -SHA256: 9e3de9c140be7d6654bb0d57bf11b3f79b100b78a176faf2f3a7717d51cb6a64 -SHA1: ff011c4dee8d2b5c5f9758ea70a3ad12fb45766d -MD5sum: 931b94ba2dfc6de2adc7194d5b772fd6 -Description: AMD ROCm SOLVER library -Maintainer: RocSOLVER maintainer -Recommends: rocsolver-asan-dev (>=3.26.0.60202) -Version: 3.26.0.60202-116~20.04 -Installed-Size: 3641644 - -Package: rocsolver-dbgsym -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 3fc7e046426f28b44df45dd32f3ce3b1acc34e7c -Depends: rocsolver (= 3.26.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocsolver-dbgsym/rocsolver-dbgsym_3.26.0.60202-116~20.04_amd64.deb -Size: 34743916 -SHA256: 28ad5475b044c3f56470f16e044ccb0e461e2326061bd0cc00c4aaaba2032e43 -SHA1: d650c0b927332af1eff2eef1aab4bef7ee665e00 -MD5sum: 0fcb19cfa68fdfd234baedd4079d6e35 -Description: debug symbols for rocsolver -Maintainer: RocSOLVER maintainer -Package-Type: ddeb -Version: 3.26.0.60202-116~20.04 -Installed-Size: 304262 - -Package: rocsolver-dbgsym-rpath6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 3fc7e046426f28b44df45dd32f3ce3b1acc34e7c -Depends: rocsolver-rpath6.2.2 (= 3.26.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocsolver-dbgsym-rpath6.2.2/rocsolver-dbgsym-rpath6.2.2_3.26.0.60202-116~20.04_amd64.deb -Size: 34724608 -SHA256: 9476c849d2830036de533aaf2b93d54eadf0730732ffafb8810696bf3a5314b7 -SHA1: 86ba2496a83fbbf4fe868cc2cd1b9e2b3845eaf5 -MD5sum: 686b5a6b8664c1bc025e0f88469301b1 -Description: debug symbols for rocsolver -Maintainer: RocSOLVER maintainer -Package-Type: ddeb -Version: 3.26.0.60202-116~20.04 -Installed-Size: 304262 - -Package: rocsolver-dbgsym6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 3fc7e046426f28b44df45dd32f3ce3b1acc34e7c -Depends: rocsolver6.2.2 (= 3.26.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocsolver-dbgsym6.2.2/rocsolver-dbgsym6.2.2_3.26.0.60202-116~20.04_amd64.deb -Size: 34723520 -SHA256: 631b35f3ada72066c4e1b35612f4b2baa36e8ef2953def9d8e6900155d08c914 -SHA1: 655290872a8dfeb2c22b6943b8a032222f55aee8 -MD5sum: 561b06fe3f652840b18c28ff25cfcb3b -Description: debug symbols for rocsolver -Maintainer: RocSOLVER maintainer -Package-Type: ddeb -Version: 3.26.0.60202-116~20.04 -Installed-Size: 304262 - -Package: rocsolver-dev -Architecture: amd64 -Depends: rocsolver (>= 3.26.0.60202) -Priority: optional -Section: devel -Filename: pool/main/r/rocsolver-dev/rocsolver-dev_3.26.0.60202-116~20.04_amd64.deb -Size: 50976 -SHA256: b87ec77e4b8e1de1e97021b54bc6ed538eb2a54f1c6e61d2196fe67cf7b4c23f -SHA1: f023b4fcc49ba7270d522cefcc453456a867df93 -MD5sum: 38ad250ad8437391d04538ca90de3722 -Description: AMD ROCm SOLVER library -Maintainer: RocSOLVER maintainer -Version: 3.26.0.60202-116~20.04 -Installed-Size: 1572 - -Package: rocsolver-dev-rpath6.2.2 -Architecture: amd64 -Depends: rocsolver-rpath6.2.2 (>= 3.26.0.60202) -Priority: optional -Section: devel -Filename: pool/main/r/rocsolver-dev-rpath6.2.2/rocsolver-dev-rpath6.2.2_3.26.0.60202-116~20.04_amd64.deb -Size: 51100 -SHA256: 5154b5c2a02e215ebf19643813075ae46f265fb4944bb0e8f1bf75cc8c009fae -SHA1: 294bde2dff84de779407ce62a53d195e05f73396 -MD5sum: c459125342c5011844bebd2d533e918b -Description: AMD ROCm SOLVER library -Maintainer: RocSOLVER maintainer -Version: 3.26.0.60202-116~20.04 -Installed-Size: 1572 - -Package: rocsolver-dev6.2.2 -Architecture: amd64 -Depends: rocsolver6.2.2 (>= 3.26.0.60202) -Priority: optional -Section: devel -Filename: pool/main/r/rocsolver-dev6.2.2/rocsolver-dev6.2.2_3.26.0.60202-116~20.04_amd64.deb -Size: 51112 -SHA256: 92af3484de723db25da5a455f04a00ed52faddc7e35d9104ef17d3ebf4c1f793 -SHA1: bf5e75c4fc8107679bdaf36402952ecd88567b4a -MD5sum: f5b4b779eb4791024e46c273ccbb6a07 -Description: AMD ROCm SOLVER library -Maintainer: RocSOLVER maintainer -Version: 3.26.0.60202-116~20.04 -Installed-Size: 1572 - -Package: rocsolver-rpath6.2.2 -Architecture: amd64 -Depends: rocblas-rpath6.2.2 (>= 4.2), rocsparse-rpath6.2.2 (>= 2.2), rocm-core-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocsolver-rpath6.2.2/rocsolver-rpath6.2.2_3.26.0.60202-116~20.04_amd64.deb -Size: 251148860 -SHA256: a126c106fd083b6c017cc6533951900c80cd1f8928ad2b93ab602d21f5e2446f -SHA1: c83145277432ddc057c2ea0a29d983318e5a4ff5 -MD5sum: 8d9a6e21c7e73a811915d9df5c81c402 -Description: AMD ROCm SOLVER library -Maintainer: RocSOLVER maintainer -Recommends: rocsolver-dev-rpath6.2.2 (>=3.26.0.60202) -Version: 3.26.0.60202-116~20.04 -Installed-Size: 3383329 - -Package: rocsolver6.2.2 -Architecture: amd64 -Depends: rocblas6.2.2 (>= 4.2), rocsparse6.2.2 (>= 2.2), rocm-core6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocsolver6.2.2/rocsolver6.2.2_3.26.0.60202-116~20.04_amd64.deb -Size: 251150368 -SHA256: 2092362b9e8a9d909a009159a759f94f30e6bf76e912ad1f7a91d39c2e2b5447 -SHA1: 174ba7c74d9d8eb4ce65fa099bd9f83f759e9ece -MD5sum: 7f6af9402e280ef625db61a841bf56ea -Description: AMD ROCm SOLVER library -Maintainer: RocSOLVER maintainer -Recommends: rocsolver-dev6.2.2 (>=3.26.0.60202) -Version: 3.26.0.60202-116~20.04 -Installed-Size: 3383329 - -Package: rocsparse -Architecture: amd64 -Depends: hip-runtime-amd (>= 4.5.0), rocm-core -Priority: optional -Section: devel -Filename: pool/main/r/rocsparse/rocsparse_3.2.0.60202-116~20.04_amd64.deb -Size: 172433726 -SHA256: a63f1daf1deea6d4c1856119034cba9cdc63c897dcfcaeec6f086ba81033eb4b -SHA1: f73b81830bbb41843b6ab7b8fe1b759d97507a64 -MD5sum: 65a2d9c0348bb32b0c6722b01497eded -Description: Radeon Open Compute SPARSE library -Maintainer: rocSPARSE Maintainer -Recommends: rocsparse-dev (>=3.2.0.60202) -Version: 3.2.0.60202-116~20.04 -Installed-Size: 2809937 - -Package: rocsparse-asan -Architecture: amd64 -Depends: hip-runtime-amd-asan (>= 4.5.0), rocm-core-asan -Priority: optional -Section: devel -Filename: pool/main/r/rocsparse-asan/rocsparse-asan_3.2.0.60202-116~20.04_amd64.deb -Size: 196496860 -SHA256: cad66b9d46b8327cec567f43a79da308f28a19d677bdc38b5af91df1a522b2d7 -SHA1: 3e4448e5185f46c84099799a0fad7e56632be4cb -MD5sum: 5ba21c0450111093bf4e6b27d89f3bde -Description: Radeon Open Compute SPARSE library -Maintainer: rocSPARSE Maintainer -Recommends: rocsparse-asan-dev (>=3.2.0.60202) -Version: 3.2.0.60202-116~20.04 -Installed-Size: 3236868 - -Package: rocsparse-asan-dbgsym -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 035a9ddb99045605f4185f7cc5631e7188eb9341 -Depends: rocsparse-asan (= 3.2.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocsparse-asan-dbgsym/rocsparse-asan-dbgsym_3.2.0.60202-116~20.04_amd64.deb -Size: 40845960 -SHA256: e024766fb773b68a242bea77ae43ba93422c883e4b213f3ae578382f148bf4ac -SHA1: 28b593290861e6ede0f6b057dd1ae5c22b735943 -MD5sum: aa25fea199f6fdc068794c2e252fb03e -Description: debug symbols for rocsparse-asan -Maintainer: rocSPARSE Maintainer -Package-Type: ddeb -Version: 3.2.0.60202-116~20.04 -Installed-Size: 56488 - -Package: rocsparse-asan-dbgsym-rpath6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 035a9ddb99045605f4185f7cc5631e7188eb9341 -Depends: rocsparse-asan-rpath6.2.2 (= 3.2.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocsparse-asan-dbgsym-rpath6.2.2/rocsparse-asan-dbgsym-rpath6.2.2_3.2.0.60202-116~20.04_amd64.deb -Size: 40838212 -SHA256: 7e79275f18f2101cc716066f893055dd9bfa0c8b13349ada3eee2c82fbfe9585 -SHA1: 6a9a0af58edbcfec5069092425ea2b10d6b87ddb -MD5sum: ce9e8cfa74959ef4c67ddad5b515e63a -Description: debug symbols for rocsparse-asan -Maintainer: rocSPARSE Maintainer -Package-Type: ddeb -Version: 3.2.0.60202-116~20.04 -Installed-Size: 56488 - -Package: rocsparse-asan-dbgsym6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 035a9ddb99045605f4185f7cc5631e7188eb9341 -Depends: rocsparse-asan6.2.2 (= 3.2.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocsparse-asan-dbgsym6.2.2/rocsparse-asan-dbgsym6.2.2_3.2.0.60202-116~20.04_amd64.deb -Size: 40838220 -SHA256: 9b88c4fe6408b78f66c24c2e0653365818033ec344fce0e06425710fb1fdaea2 -SHA1: 9b5dc2b3b4bf2ade51a151a9345bcd5fd75982ac -MD5sum: d45e003ba5b311d5ce0e7b0bdf915e89 -Description: debug symbols for rocsparse-asan -Maintainer: rocSPARSE Maintainer -Package-Type: ddeb -Version: 3.2.0.60202-116~20.04 -Installed-Size: 56488 - -Package: rocsparse-asan-rpath6.2.2 -Architecture: amd64 -Depends: hip-runtime-amd-asan-rpath6.2.2 (>= 4.5.0), rocm-core-asan-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocsparse-asan-rpath6.2.2/rocsparse-asan-rpath6.2.2_3.2.0.60202-116~20.04_amd64.deb -Size: 196494108 -SHA256: a4a4710e7cff249a682f3a45d8f92b95e03f9e4dfe8451697c0a3a9bb74dd350 -SHA1: 5e5d4db8f8dbbd14ccdb337288bc6be00c28888b -MD5sum: b2a13c095c9b9e7aad18c8770fce6ea0 -Description: Radeon Open Compute SPARSE library -Maintainer: rocSPARSE Maintainer -Recommends: rocsparse-asan-dev (>=3.2.0.60202) -Version: 3.2.0.60202-116~20.04 -Installed-Size: 3236868 - -Package: rocsparse-asan6.2.2 -Architecture: amd64 -Depends: hip-runtime-amd-asan6.2.2 (>= 4.5.0), rocm-core-asan6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocsparse-asan6.2.2/rocsparse-asan6.2.2_3.2.0.60202-116~20.04_amd64.deb -Size: 196490584 -SHA256: 57dbc50a989c94968c5e28e3f81b4da53db08b9f49354beefacb3131dba0042d -SHA1: 524ce0081d1d463036743cd825bce24618150909 -MD5sum: ef95eed2fd2bfcc9b1cfdb7094ac9b7d -Description: Radeon Open Compute SPARSE library -Maintainer: rocSPARSE Maintainer -Recommends: rocsparse-asan-dev (>=3.2.0.60202) -Version: 3.2.0.60202-116~20.04 -Installed-Size: 3236868 - -Package: rocsparse-dbgsym -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: a12631a2023809a9d6fdb40004454d9390343722 -Depends: rocsparse (= 3.2.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocsparse-dbgsym/rocsparse-dbgsym_3.2.0.60202-116~20.04_amd64.deb -Size: 16622428 -SHA256: bf02d32a5b8cea9b5729dab1a5e681fa02ce495e3e13ecef6f435183eb413468 -SHA1: faa533fc45206a4c74d1fe0826181e2750f55608 -MD5sum: a384b0e3ecda0795665979221d8866c6 -Description: debug symbols for rocsparse -Maintainer: rocSPARSE Maintainer -Package-Type: ddeb -Version: 3.2.0.60202-116~20.04 -Installed-Size: 135539 - -Package: rocsparse-dbgsym-rpath6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: a12631a2023809a9d6fdb40004454d9390343722 -Depends: rocsparse-rpath6.2.2 (= 3.2.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocsparse-dbgsym-rpath6.2.2/rocsparse-dbgsym-rpath6.2.2_3.2.0.60202-116~20.04_amd64.deb -Size: 16632584 -SHA256: 8b9669fc9293e6ad7d01a7081eb987863ea75bbb69f2a3ad90d3d71802ed53f7 -SHA1: c45289eb3dcc8ccbdc5846178e75e981873680fb -MD5sum: 038a1531666b9560ef9ad786d9253b58 -Description: debug symbols for rocsparse -Maintainer: rocSPARSE Maintainer -Package-Type: ddeb -Version: 3.2.0.60202-116~20.04 -Installed-Size: 135539 - -Package: rocsparse-dbgsym6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: a12631a2023809a9d6fdb40004454d9390343722 -Depends: rocsparse6.2.2 (= 3.2.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rocsparse-dbgsym6.2.2/rocsparse-dbgsym6.2.2_3.2.0.60202-116~20.04_amd64.deb -Size: 16624956 -SHA256: 142c405aa442021b5e68d7a129439a6751694a373474837ac1b010924ddedcec -SHA1: d4bed365e4be6fc298efb7ff919044580a69e8a8 -MD5sum: 7af719c3bfcbb7d7af5959021852afce -Description: debug symbols for rocsparse -Maintainer: rocSPARSE Maintainer -Package-Type: ddeb -Version: 3.2.0.60202-116~20.04 -Installed-Size: 135539 - -Package: rocsparse-dev -Architecture: amd64 -Depends: rocsparse (>= 3.2.0.60202) -Priority: optional -Section: devel -Filename: pool/main/r/rocsparse-dev/rocsparse-dev_3.2.0.60202-116~20.04_amd64.deb -Size: 92920 -SHA256: ae2c5ab73f7aaf5bb1b7817a785d6cc831861f37ae0518002881a4bb63ce9376 -SHA1: 778c5613588e76116b86eb8bbcef846f43eddfd9 -MD5sum: 1f5ba466dad6bf6d0cd3caf4871ce779 -Description: Radeon Open Compute SPARSE library -Maintainer: rocSPARSE Maintainer -Version: 3.2.0.60202-116~20.04 -Installed-Size: 1932 - -Package: rocsparse-dev-rpath6.2.2 -Architecture: amd64 -Depends: rocsparse-rpath6.2.2 (>= 3.2.0.60202) -Priority: optional -Section: devel -Filename: pool/main/r/rocsparse-dev-rpath6.2.2/rocsparse-dev-rpath6.2.2_3.2.0.60202-116~20.04_amd64.deb -Size: 92692 -SHA256: beee65ca66843b710ccb2990e5f3b9ac4061e8b725bf603092d76c2edf2db38e -SHA1: f3ed5f5c156617293fcbbe4eb419680a9ca30092 -MD5sum: d1c87c576c65bec5594adac0c60e9fd5 -Description: Radeon Open Compute SPARSE library -Maintainer: rocSPARSE Maintainer -Version: 3.2.0.60202-116~20.04 -Installed-Size: 1932 - -Package: rocsparse-dev6.2.2 -Architecture: amd64 -Depends: rocsparse6.2.2 (>= 3.2.0.60202) -Priority: optional -Section: devel -Filename: pool/main/r/rocsparse-dev6.2.2/rocsparse-dev6.2.2_3.2.0.60202-116~20.04_amd64.deb -Size: 92664 -SHA256: f8fe9d6efbe72e5aca22820ac14d545210dbbdf26579a87f65bc1107001dab28 -SHA1: 461fe428772ea2bb55779c7dd8df88ac73f8d77b -MD5sum: 6fbb71a0dcd1209d2301155becff2bea -Description: Radeon Open Compute SPARSE library -Maintainer: rocSPARSE Maintainer -Version: 3.2.0.60202-116~20.04 -Installed-Size: 1932 - -Package: rocsparse-rpath6.2.2 -Architecture: amd64 -Depends: hip-runtime-amd-rpath6.2.2 (>= 4.5.0), rocm-core-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocsparse-rpath6.2.2/rocsparse-rpath6.2.2_3.2.0.60202-116~20.04_amd64.deb -Size: 172432296 -SHA256: a8ba87c6a012bb077e790e58e5395f0da368104796acb914851de6aff076b6bd -SHA1: f68a924448a41980ecf8919bc8ba4a9a2a262a66 -MD5sum: f4ac2a6b70486dc22d25410066da169a -Description: Radeon Open Compute SPARSE library -Maintainer: rocSPARSE Maintainer -Recommends: rocsparse-dev-rpath6.2.2 (>=3.2.0.60202) -Version: 3.2.0.60202-116~20.04 -Installed-Size: 2809937 - -Package: rocsparse6.2.2 -Architecture: amd64 -Depends: hip-runtime-amd6.2.2 (>= 4.5.0), rocm-core6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocsparse6.2.2/rocsparse6.2.2_3.2.0.60202-116~20.04_amd64.deb -Size: 172433900 -SHA256: 2ac61174dae9e5b7c5196a08eb1a88a7cbda457ac8375927dda1eba5967bb6c2 -SHA1: e941fdffda51fb93398a80621a619d9b1094e134 -MD5sum: 893d46572b4be54fd22cae0a770bf2ab -Description: Radeon Open Compute SPARSE library -Maintainer: rocSPARSE Maintainer -Recommends: rocsparse-dev6.2.2 (>=3.2.0.60202) -Version: 3.2.0.60202-116~20.04 -Installed-Size: 2809937 - -Package: rocthrust-dev -Architecture: amd64 -Conflicts: hip-thrust, thrust -Depends: rocprim-dev (>= 2.10.1), rocm-core -Priority: optional -Section: devel -Filename: pool/main/r/rocthrust-dev/rocthrust-dev_3.1.0.60202-116~20.04_amd64.deb -Size: 416506 -SHA256: af3b9991279d516c8b472f37a1fa59507761c0c3dd080830e5981376112e7457 -SHA1: 01deb9176659bd58502487616f8cf6084047cdd9 -MD5sum: 257d0eb6ef83994b0ab7c3f758af05bc -Description: Radeon Open Compute Thrust library -Maintainer: rocthrust-maintainer@amd.com -Provides: hipstdpar, CPACK_DEBIAN_PACKAGE_PROVIDES, rocthrust (= 3.1.0.60202) -Version: 3.1.0.60202-116~20.04 -Installed-Size: 5630 - -Package: rocthrust-dev-rpath6.2.2 -Architecture: amd64 -Conflicts: hip-thrust-rpath6.2.2, thrust-rpath6.2.2 -Depends: rocprim-dev-rpath6.2.2 (>= 2.10.1), rocm-core-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocthrust-dev-rpath6.2.2/rocthrust-dev-rpath6.2.2_3.1.0.60202-116~20.04_amd64.deb -Size: 412960 -SHA256: 203203a6ade0a843a5accf00a232f03d96ba670fc567a0de02cd468036bf7fd6 -SHA1: a36f29626b32f55132b839e723845aad31cb9315 -MD5sum: 58e66f028937ffae98243e05434f43f5 -Description: Radeon Open Compute Thrust library -Maintainer: rocthrust-maintainer@amd.com -Provides: hipstdpar, CPACK_DEBIAN_PACKAGE_PROVIDES, rocthrust-rpath6.2.2 (= 3.1.0.60202) -Version: 3.1.0.60202-116~20.04 -Installed-Size: 5630 - -Package: rocthrust-dev6.2.2 -Architecture: amd64 -Conflicts: hip-thrust6.2.2, thrust6.2.2 -Depends: rocprim-dev6.2.2 (>= 2.10.1), rocm-core6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocthrust-dev6.2.2/rocthrust-dev6.2.2_3.1.0.60202-116~20.04_amd64.deb -Size: 413000 -SHA256: efb48ddc326ce104f07881d350764a4627c95bddbf7cbc565918eb24626e8513 -SHA1: d513d1a21e94bf058d0f2dd0b6749088f64bc43b -MD5sum: e9324b11362bbde9ab1076741ee9fdc8 -Description: Radeon Open Compute Thrust library -Maintainer: rocthrust-maintainer@amd.com -Provides: hipstdpar, CPACK_DEBIAN_PACKAGE_PROVIDES, rocthrust6.2.2 (= 3.1.0.60202) -Version: 3.1.0.60202-116~20.04 -Installed-Size: 5630 - -Package: roctracer -Architecture: amd64 -Depends: rocm-core -Priority: optional -Section: devel -Filename: pool/main/r/roctracer/roctracer_4.1.60202.60202-116~20.04_amd64.deb -Size: 451964 -SHA256: c734fc2a76b2faf70dede04730d18feb4ddcf2ff445a058c921f8dc44f42816d -SHA1: fe3d6e4e2535b20fbd748a45ca462ee4cd879556 -MD5sum: ccc07249386d11679f7a5008ecb40cdc -Description: AMD ROCTRACER library - Dynamic libraries for the ROCtracer -Maintainer: ROCm Profiler Support -Version: 4.1.60202.60202-116~20.04 -Installed-Size: 1481 - -Package: roctracer-asan -Architecture: amd64 -Depends: rocm-core-asan -Priority: optional -Section: devel -Filename: pool/main/r/roctracer-asan/roctracer-asan_4.1.60202.60202-116~20.04_amd64.deb -Size: 565572 -SHA256: 0780b08a387207b4f6d0e1b064a87aaabc675deb04b9981118304e01d5ada157 -SHA1: 07b87d7c4fbe35e0855f4073b40a796ab0380398 -MD5sum: 15f620a1b0eda6b69c41563267ff2263 -Description: AMD ROCTRACER library - ASAN libraries for the ROCtracer -Maintainer: ROCm Profiler Support -Version: 4.1.60202.60202-116~20.04 -Installed-Size: 3119 - -Package: roctracer-asan-dbgsym -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: e5a6abf5ebf60cb096ab49d958e2ff3c90507891 e2de70955d2536f021d0c74ceac7fcdc6995536d 401d7092c4878ead973952ddd05b6b1a63b9f97b 1994bbbe5b612a5d21e254ef88d74060cec02f0e df26785417eb3f19df4a14cb2ecda96fa7654e09 -Depends: roctracer-asan (= 4.1.60202.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/roctracer-asan-dbgsym/roctracer-asan-dbgsym_4.1.60202.60202-116~20.04_amd64.deb -Size: 1741496 -SHA256: 41c9962754664fc2dfe6f73f1263ecc094c7395fbca9280549c4ea4f8bde2b8a -SHA1: 85079037425f1c114579cc4cbb266759cec4779b -MD5sum: 84837f4d9e8a0aed6d037d2cf0da4e68 -Description: debug symbols for roctracer-asan -Maintainer: ROCm Profiler Support -Package-Type: ddeb -Version: 4.1.60202.60202-116~20.04 -Installed-Size: 5642 - -Package: roctracer-asan-dbgsym-rpath6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: e5a6abf5ebf60cb096ab49d958e2ff3c90507891 e2de70955d2536f021d0c74ceac7fcdc6995536d 401d7092c4878ead973952ddd05b6b1a63b9f97b 1994bbbe5b612a5d21e254ef88d74060cec02f0e df26785417eb3f19df4a14cb2ecda96fa7654e09 -Depends: roctracer-asan-rpath6.2.2 (= 4.1.60202.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/roctracer-asan-dbgsym-rpath6.2.2/roctracer-asan-dbgsym-rpath6.2.2_4.1.60202.60202-116~20.04_amd64.deb -Size: 1108104 -SHA256: fd1b2b2400ac5cb3ca3bb6cb69f5dd680583b279016ed39f8ef519d39fe4696f -SHA1: 71a951249e2009e3a48a125f4d140445ab46c0fe -MD5sum: 8198a6f56cfe0a9a85070c44ae4d3592 -Description: debug symbols for roctracer-asan -Maintainer: ROCm Profiler Support -Package-Type: ddeb -Version: 4.1.60202.60202-116~20.04 -Installed-Size: 5642 - -Package: roctracer-asan-dbgsym6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: e5a6abf5ebf60cb096ab49d958e2ff3c90507891 e2de70955d2536f021d0c74ceac7fcdc6995536d 401d7092c4878ead973952ddd05b6b1a63b9f97b 1994bbbe5b612a5d21e254ef88d74060cec02f0e df26785417eb3f19df4a14cb2ecda96fa7654e09 -Depends: roctracer-asan6.2.2 (= 4.1.60202.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/roctracer-asan-dbgsym6.2.2/roctracer-asan-dbgsym6.2.2_4.1.60202.60202-116~20.04_amd64.deb -Size: 1106948 -SHA256: 129f52f9e6e2d9feff343cce061b3954d0fec7cdb606e8bea4f307ec78a40a2b -SHA1: 745b3fda91bfd9a3da0f25482c7ddc0c5b6e3ea3 -MD5sum: a0004d7a45de1beeb65a08b21f326925 -Description: debug symbols for roctracer-asan -Maintainer: ROCm Profiler Support -Package-Type: ddeb -Version: 4.1.60202.60202-116~20.04 -Installed-Size: 5642 - -Package: roctracer-asan-rpath6.2.2 -Architecture: amd64 -Depends: rocm-core-asan-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/roctracer-asan-rpath6.2.2/roctracer-asan-rpath6.2.2_4.1.60202.60202-116~20.04_amd64.deb -Size: 349108 -SHA256: c0098966b5de6f1420b3e41f5238b2bce2db324d6321f95c8834edc18e03073f -SHA1: 7d7127711ceaf3f42b78a882d545e9b92ebf95bf -MD5sum: 9038b4c3d014d13aba959619da768e1e -Description: AMD ROCTRACER library - ASAN libraries for the ROCtracer -Maintainer: ROCm Profiler Support -Version: 4.1.60202.60202-116~20.04 -Installed-Size: 3119 - -Package: roctracer-asan6.2.2 -Architecture: amd64 -Depends: rocm-core-asan6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/roctracer-asan6.2.2/roctracer-asan6.2.2_4.1.60202.60202-116~20.04_amd64.deb -Size: 350636 -SHA256: 28634cd28693d6a88165020b9299779409e537ce2a6e8bd6f5b111c1f519302c -SHA1: 16fe88c1560f4508b71631651583838c7af25caa -MD5sum: 6aa721a43622f24aedebe8b733322b58 -Description: AMD ROCTRACER library - ASAN libraries for the ROCtracer -Maintainer: ROCm Profiler Support -Version: 4.1.60202.60202-116~20.04 -Installed-Size: 3119 - -Package: roctracer-dbgsym -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 2f74f94a8300d7c696b6fa7100a767e7126da1b4 c4a7b146ad8b0028be1a054c72e2b50260ca10ad b9dc3ec29e2bedfdbf3258c3947fc92fa6f0fc08 7b33698dc3ec7656b98a0c345ee10a4b3fe54b26 fd586d27f838047b200c63518efc9e0b345d1eb3 -Depends: roctracer (= 4.1.60202.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/roctracer-dbgsym/roctracer-dbgsym_4.1.60202.60202-116~20.04_amd64.deb -Size: 2989458 -SHA256: 254c1a6fd4287d4e80185b8ba3f5d9f6a5149e6cff5ad8772cd9fcd25b7b9a1b -SHA1: 10c8495087fff5eddda459c4a060a156f9c678cf -MD5sum: dc1ca69b940fecb5a89fe4a81406bbc1 -Description: debug symbols for roctracer -Maintainer: ROCm Profiler Support -Package-Type: ddeb -Version: 4.1.60202.60202-116~20.04 -Installed-Size: 11493 - -Package: roctracer-dbgsym-rpath6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 2f74f94a8300d7c696b6fa7100a767e7126da1b4 c4a7b146ad8b0028be1a054c72e2b50260ca10ad b9dc3ec29e2bedfdbf3258c3947fc92fa6f0fc08 7b33698dc3ec7656b98a0c345ee10a4b3fe54b26 fd586d27f838047b200c63518efc9e0b345d1eb3 -Depends: roctracer-rpath6.2.2 (= 4.1.60202.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/roctracer-dbgsym-rpath6.2.2/roctracer-dbgsym-rpath6.2.2_4.1.60202.60202-116~20.04_amd64.deb -Size: 1983536 -SHA256: 923935f7b8f0d6abe2464a293dc202bee00ea2eec4ab37ad7edbfb70d492b5dd -SHA1: 6946f73ed33d4fad0d6c224552d01c890ebd111f -MD5sum: ca7e193bb5bd48c2e23e5c214273050b -Description: debug symbols for roctracer -Maintainer: ROCm Profiler Support -Package-Type: ddeb -Version: 4.1.60202.60202-116~20.04 -Installed-Size: 11493 - -Package: roctracer-dbgsym6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 2f74f94a8300d7c696b6fa7100a767e7126da1b4 c4a7b146ad8b0028be1a054c72e2b50260ca10ad b9dc3ec29e2bedfdbf3258c3947fc92fa6f0fc08 7b33698dc3ec7656b98a0c345ee10a4b3fe54b26 fd586d27f838047b200c63518efc9e0b345d1eb3 -Depends: roctracer6.2.2 (= 4.1.60202.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/roctracer-dbgsym6.2.2/roctracer-dbgsym6.2.2_4.1.60202.60202-116~20.04_amd64.deb -Size: 1985452 -SHA256: 2870247d5fc39fd2280cb069458a702cdd9bacd56bc8593480506d25d403c06d -SHA1: 67b533532112ac7fdd7df025e167ef68f236a6fb -MD5sum: 3a85d4f4e65628c70d9190d3156547a9 -Description: debug symbols for roctracer -Maintainer: ROCm Profiler Support -Package-Type: ddeb -Version: 4.1.60202.60202-116~20.04 -Installed-Size: 11493 - -Package: roctracer-dev -Architecture: amd64 -Depends: roctracer, rocm-core -Priority: optional -Section: devel -Filename: pool/main/r/roctracer-dev/roctracer-dev_4.1.60202.60202-116~20.04_amd64.deb -Size: 454194 -SHA256: 8afacf8c7a7aa2ca7c01af426b1aa46b445b33b8a88f7d5542256f7f15856451 -SHA1: 9e53df34152c3ff1c00c7284683fa5b4a226c1c9 -MD5sum: 456c29ae298a50b42f404a277f2e2790 -Description: AMD ROCTRACER library - Header files and documentation for ROCtracer -Maintainer: ROCm Profiler Support -Version: 4.1.60202.60202-116~20.04 -Installed-Size: 1447 - -Package: roctracer-dev-rpath6.2.2 -Architecture: amd64 -Depends: roctracer-rpath6.2.2, rocm-core-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/roctracer-dev-rpath6.2.2/roctracer-dev-rpath6.2.2_4.1.60202.60202-116~20.04_amd64.deb -Size: 395396 -SHA256: 2e4f47fb5d8bd959d401ef424b9d3899ce01dfa4355e9b416cff60dae2f8809d -SHA1: cb25b954c2e7ce325de5df48df1ccd6677d8dba5 -MD5sum: 88b76bd98245b763163a22cf11a4c07b -Description: AMD ROCTRACER library - Header files and documentation for ROCtracer -Maintainer: ROCm Profiler Support -Version: 4.1.60202.60202-116~20.04 -Installed-Size: 1447 - -Package: roctracer-dev6.2.2 -Architecture: amd64 -Depends: roctracer6.2.2, rocm-core6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/roctracer-dev6.2.2/roctracer-dev6.2.2_4.1.60202.60202-116~20.04_amd64.deb -Size: 395400 -SHA256: e94f42477774969e835cf77507fb954d85f984d52e7d1dbe7085094679025f44 -SHA1: f684da54336dd3983657834636ec45a308ee90e9 -MD5sum: fbc260c9f5b914933a84f77f02ea0c3e -Description: AMD ROCTRACER library - Header files and documentation for ROCtracer -Maintainer: ROCm Profiler Support -Version: 4.1.60202.60202-116~20.04 -Installed-Size: 1447 - -Package: roctracer-rpath6.2.2 -Architecture: amd64 -Depends: rocm-core-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/roctracer-rpath6.2.2/roctracer-rpath6.2.2_4.1.60202.60202-116~20.04_amd64.deb -Size: 262204 -SHA256: cde35dffe5da36d278c500578d7d8048cb6a76010a7f58acfa7d0918f763a3e5 -SHA1: 757d0824a5bc8a6c89d9c58fbef1fab468545dc9 -MD5sum: 2abec0da227f42c46c6a9b828f64b29c -Description: AMD ROCTRACER library - Dynamic libraries for the ROCtracer -Maintainer: ROCm Profiler Support -Version: 4.1.60202.60202-116~20.04 -Installed-Size: 1481 - -Package: roctracer6.2.2 -Architecture: amd64 -Depends: rocm-core6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/roctracer6.2.2/roctracer6.2.2_4.1.60202.60202-116~20.04_amd64.deb -Size: 262244 -SHA256: b98f1dd4a9044ba3f969d9289d0bb1f05d19d337b96060d3d37e55d91fbe6085 -SHA1: bee50fb6da22dfc4cdb25702bc4e7fbf7d25da64 -MD5sum: d5d3524b43a2b219f31ab1b41a742758 -Description: AMD ROCTRACER library - Dynamic libraries for the ROCtracer -Maintainer: ROCm Profiler Support -Version: 4.1.60202.60202-116~20.04 -Installed-Size: 1481 - -Package: rocwmma-dev -Architecture: amd64 -Depends: rocm-core -Priority: optional -Section: devel -Filename: pool/main/r/rocwmma-dev/rocwmma-dev_1.5.0.60202-116~20.04_amd64.deb -Size: 71886 -SHA256: 63b82cd2e81506e2848a488e68681b06aec899457d6d668ec5942ad1b777a006 -SHA1: 6eb21c0f66be1fb32f72a34b0a66567edb727402 -MD5sum: 99917bdf5b79b41ac0d366d82424d210 -Description: AMD C++ library for facilitating GEMM, or GEMM-like 2D matrix multiplications on GPU leveraging MFMA instructions executing on matrix cores. -Maintainer: rocWMMA Maintainer -Provides: CPACK_DEBIAN_PACKAGE_PROVIDES, rocwmma (= 1.5.0.60202) -Version: 1.5.0.60202-116~20.04 -Installed-Size: 817 - -Package: rocwmma-dev-rpath6.2.2 -Architecture: amd64 -Depends: rocm-core-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocwmma-dev-rpath6.2.2/rocwmma-dev-rpath6.2.2_1.5.0.60202-116~20.04_amd64.deb -Size: 71840 -SHA256: 392528f4b9df27f14d6f184cb71e524eec9b42f555551868990c47bf4624baad -SHA1: d080141c1d8c13c11313ad621052aeb13a59e2ca -MD5sum: e42765bfbf3626d439b997ed385c6fc3 -Description: AMD C++ library for facilitating GEMM, or GEMM-like 2D matrix multiplications on GPU leveraging MFMA instructions executing on matrix cores. -Maintainer: rocWMMA Maintainer -Provides: CPACK_DEBIAN_PACKAGE_PROVIDES, rocwmma-rpath6.2.2 (= 1.5.0.60202) -Version: 1.5.0.60202-116~20.04 -Installed-Size: 817 - -Package: rocwmma-dev6.2.2 -Architecture: amd64 -Depends: rocm-core6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rocwmma-dev6.2.2/rocwmma-dev6.2.2_1.5.0.60202-116~20.04_amd64.deb -Size: 71792 -SHA256: cbf83dfc3afde3a42495d7b3ec42826e296f1c396e7a72b34db490a2153a845e -SHA1: 122db691db6e870d48a8ebfa033fc9a1a9c759be -MD5sum: 2035b3dfebe770e50dcc23931a35e824 -Description: AMD C++ library for facilitating GEMM, or GEMM-like 2D matrix multiplications on GPU leveraging MFMA instructions executing on matrix cores. -Maintainer: rocWMMA Maintainer -Provides: CPACK_DEBIAN_PACKAGE_PROVIDES, rocwmma6.2.2 (= 1.5.0.60202) -Version: 1.5.0.60202-116~20.04 -Installed-Size: 817 - -Package: rpp -Architecture: amd64 -Depends: rocm-hip-runtime -Priority: optional -Section: devel -Filename: pool/main/r/rpp/rpp_1.8.0.60202-116~20.04_amd64.deb -Size: 81559734 -SHA256: 1dc244ed66d0f9aa299079f75397f22a3ed79f9aec1f57168d85aa480d4a5e2c -SHA1: b347b004c7e0f8b3777111413a815e57f438e121 -MD5sum: 7895bd20f24959e2b3bbde439bfc596b -Description: ROCm Performance Primitives library is a comprehensive high performance computer vision library for AMD CPUs and GPUs with HOST/HIP/OpenCL back-ends. - RPP runtime package provides rpp library and license.txt -Homepage: https://github.com/ROCm/rpp -Maintainer: mivisionx support -Version: 1.8.0.60202-116~20.04 -Installed-Size: 559581 - -Package: rpp-asan -Architecture: amd64 -Depends: rocm-core-asan -Priority: optional -Section: devel -Filename: pool/main/r/rpp-asan/rpp-asan_1.8.0.60202-116~20.04_amd64.deb -Size: 85408466 -SHA256: e649ba4fda4bb44e658d4df13d547c4587e3c8d51c42caf337ba8dfd23b32ea7 -SHA1: e01879527462f9e9bbfd0ccbac39681161666e3a -MD5sum: 97706dea0baa461f4bb4efe31dc912e3 -Description: ROCm Performance Primitives library is a comprehensive high performance computer vision library for AMD CPUs and GPUs with HOST/HIP/OpenCL back-ends. RPP ASAN package provides rpp ASAN libraries -Homepage: https://github.com/ROCm/rpp -Maintainer: mivisionx support -Version: 1.8.0.60202-116~20.04 -Installed-Size: 552016 - -Package: rpp-asan-dbgsym -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 35a40bcaeb6b91840e0ffadeefe9b74952a128d0 -Depends: rpp-asan (= 1.8.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rpp-asan-dbgsym/rpp-asan-dbgsym_1.8.0.60202-116~20.04_amd64.deb -Size: 8421756 -SHA256: cc6ad72e28a9d7f479ec894c68717971ff03224bf03f5589964c676d322b95e4 -SHA1: 63d2005f121ff22809911c38b2fadd7d1a95aa35 -MD5sum: 8cbb56048decf98dbda13139ea79d9c7 -Description: debug symbols for rpp-asan -Maintainer: mivisionx support -Package-Type: ddeb -Version: 1.8.0.60202-116~20.04 -Installed-Size: 9515 - -Package: rpp-asan-dbgsym-rpath6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 35a40bcaeb6b91840e0ffadeefe9b74952a128d0 -Depends: rpp-asan-rpath6.2.2 (= 1.8.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rpp-asan-dbgsym-rpath6.2.2/rpp-asan-dbgsym-rpath6.2.2_1.8.0.60202-116~20.04_amd64.deb -Size: 8287260 -SHA256: 3cc4dcb90cb3bbe231c0173550132a3baabf298f09d53103f4e8abb97dea9e62 -SHA1: 85024ce87176975de80e985631f9f747df10d75c -MD5sum: 48a3ee952799231903b33fd3b829cbef -Description: debug symbols for rpp-asan -Maintainer: mivisionx support -Package-Type: ddeb -Version: 1.8.0.60202-116~20.04 -Installed-Size: 9515 - -Package: rpp-asan-dbgsym6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 35a40bcaeb6b91840e0ffadeefe9b74952a128d0 -Depends: rpp-asan6.2.2 (= 1.8.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rpp-asan-dbgsym6.2.2/rpp-asan-dbgsym6.2.2_1.8.0.60202-116~20.04_amd64.deb -Size: 8287264 -SHA256: 62e480c373f6f08ca06e249073dac113ae36be9e113497f00862e91fd94972bb -SHA1: 1ab81a01998070ef8da40c1a6ecb08921612fa38 -MD5sum: 6dd2f39c3a8400e32323a6f9cf22922c -Description: debug symbols for rpp-asan -Maintainer: mivisionx support -Package-Type: ddeb -Version: 1.8.0.60202-116~20.04 -Installed-Size: 9515 - -Package: rpp-asan-rpath6.2.2 -Architecture: amd64 -Depends: rocm-core-asan-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rpp-asan-rpath6.2.2/rpp-asan-rpath6.2.2_1.8.0.60202-116~20.04_amd64.deb -Size: 39588176 -SHA256: b1424138953e81f77a618f5740b40eed63e79c0e413cd42c85efaf2089ecc959 -SHA1: d0cf83e5a9461c68e2500399b562cfd458ef374b -MD5sum: 9107a71b4da99a34934db2a20318beba -Description: ROCm Performance Primitives library is a comprehensive high performance computer vision library for AMD CPUs and GPUs with HOST/HIP/OpenCL back-ends. RPP ASAN package provides rpp ASAN libraries -Homepage: https://github.com/ROCm/rpp -Maintainer: mivisionx support -Version: 1.8.0.60202-116~20.04 -Installed-Size: 552016 - -Package: rpp-asan6.2.2 -Architecture: amd64 -Depends: rocm-core-asan6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rpp-asan6.2.2/rpp-asan6.2.2_1.8.0.60202-116~20.04_amd64.deb -Size: 39586764 -SHA256: 4cb58f7d8889ffc3f68af90e3f527377af0ee51e922a8f9abbabba167c488a8d -SHA1: 2bb1264dfe3b3d6be51701ed071d97f22fdb1cb0 -MD5sum: 4f9ba9beeca9ea6d6ded49237b09b9c6 -Description: ROCm Performance Primitives library is a comprehensive high performance computer vision library for AMD CPUs and GPUs with HOST/HIP/OpenCL back-ends. RPP ASAN package provides rpp ASAN libraries -Homepage: https://github.com/ROCm/rpp -Maintainer: mivisionx support -Version: 1.8.0.60202-116~20.04 -Installed-Size: 552016 - -Package: rpp-dbgsym -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 9bf48693abcf0e13bfc6da824c40ab1e623e0e0b -Depends: rpp (= 1.8.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rpp-dbgsym/rpp-dbgsym_1.8.0.60202-116~20.04_amd64.deb -Size: 5520754 -SHA256: 7d9e0cec2f3f4ac24201cac5068348abc62b494b96a572b7a03f6550df0c403f -SHA1: 82c7a6b43d23e0a45f5f064b2899956db9070012 -MD5sum: 7b038af7ee7dc5e67b2b469d0e939fa8 -Description: debug symbols for rpp -Maintainer: mivisionx support -Package-Type: ddeb -Version: 1.8.0.60202-116~20.04 -Installed-Size: 16203 - -Package: rpp-dbgsym-rpath6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 9bf48693abcf0e13bfc6da824c40ab1e623e0e0b -Depends: rpp-rpath6.2.2 (= 1.8.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rpp-dbgsym-rpath6.2.2/rpp-dbgsym-rpath6.2.2_1.8.0.60202-116~20.04_amd64.deb -Size: 3495688 -SHA256: 64b7034c3f0f19346f9cf9e1affe31571a4fcf5739ed41e775190a0269f14a56 -SHA1: 0b16f8432ef7b4f7ffb2e16e4712a67e46164254 -MD5sum: ac05f3d7743df7f7060e8fc84c23790a -Description: debug symbols for rpp -Maintainer: mivisionx support -Package-Type: ddeb -Version: 1.8.0.60202-116~20.04 -Installed-Size: 16203 - -Package: rpp-dbgsym6.2.2 -Architecture: amd64 -Auto-Built-Package: debug-symbols -Build-Ids: 9bf48693abcf0e13bfc6da824c40ab1e623e0e0b -Depends: rpp6.2.2 (= 1.8.0.60202-116~20.04) -Priority: optional -Section: debug -Filename: pool/main/r/rpp-dbgsym6.2.2/rpp-dbgsym6.2.2_1.8.0.60202-116~20.04_amd64.deb -Size: 3495796 -SHA256: 1dad9d5e21ea988369415dd89b27c4031ae75bb6091845035c78d7ebb155de90 -SHA1: 4806b512a0bc506e65295a37e4ebccbd68157fc4 -MD5sum: 4c52777502b355eb47c786b78ec6e693 -Description: debug symbols for rpp -Maintainer: mivisionx support -Package-Type: ddeb -Version: 1.8.0.60202-116~20.04 -Installed-Size: 16203 - -Package: rpp-dev -Architecture: amd64 -Depends: rpp, rocm-hip-runtime-dev, half -Priority: optional -Section: devel -Filename: pool/main/r/rpp-dev/rpp-dev_1.8.0.60202-116~20.04_amd64.deb -Size: 47558 -SHA256: 1a68849342bd627af4b225b00be80e192bb3171ed1f67889943ec4ad7701309e -SHA1: 0af750239db44088ee2880a1c3be04cc002c09f8 -MD5sum: 2afaf7e794d209673226cad44a478505 -Description: ROCm Performance Primitives library is a comprehensive high performance computer vision library for AMD CPUs and GPUs with HOST/HIP/OpenCL back-ends. RPP develop package provides rpp library, header files, and license.txt -Homepage: https://github.com/ROCm/rpp -Maintainer: mivisionx support -Version: 1.8.0.60202-116~20.04 -Installed-Size: 626 - -Package: rpp-dev-rpath6.2.2 -Architecture: amd64 -Depends: rpp-rpath6.2.2, rocm-hip-runtime-dev-rpath6.2.2, half-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rpp-dev-rpath6.2.2/rpp-dev-rpath6.2.2_1.8.0.60202-116~20.04_amd64.deb -Size: 34924 -SHA256: 67eeeff47df05b708b3b3461a217b0180a547272f274f1371a5c3850971e8afa -SHA1: 85cba24cda103f0930eac0835df7e366ca93f305 -MD5sum: c600c1f2d2d5acd8460417c813640913 -Description: ROCm Performance Primitives library is a comprehensive high performance computer vision library for AMD CPUs and GPUs with HOST/HIP/OpenCL back-ends. RPP develop package provides rpp library, header files, and license.txt -Homepage: https://github.com/ROCm/rpp -Maintainer: mivisionx support -Version: 1.8.0.60202-116~20.04 -Installed-Size: 626 - -Package: rpp-dev6.2.2 -Architecture: amd64 -Depends: rpp6.2.2, rocm-hip-runtime-dev6.2.2, half6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rpp-dev6.2.2/rpp-dev6.2.2_1.8.0.60202-116~20.04_amd64.deb -Size: 34908 -SHA256: dd8469c0330f4fcdce16e23168af4f0848c32dee4b87eb0032f6c7c0fadeee9b -SHA1: 34266f034306576c84c58b5fa2efe640bf5960c3 -MD5sum: 1a4333d0ecbceca1f01ac20a588ba1d9 -Description: ROCm Performance Primitives library is a comprehensive high performance computer vision library for AMD CPUs and GPUs with HOST/HIP/OpenCL back-ends. RPP develop package provides rpp library, header files, and license.txt -Homepage: https://github.com/ROCm/rpp -Maintainer: mivisionx support -Version: 1.8.0.60202-116~20.04 -Installed-Size: 626 - -Package: rpp-rpath6.2.2 -Architecture: amd64 -Depends: rocm-hip-runtime-rpath6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rpp-rpath6.2.2/rpp-rpath6.2.2_1.8.0.60202-116~20.04_amd64.deb -Size: 24334016 -SHA256: 56fb6f27890b084b03105ed35c8a42ed8fb1b04ec981d6faae8b51826306d1c7 -SHA1: ef17c428834f840d6cb963a613d043fcb14c6250 -MD5sum: fde9b3133e8fb6d93d81e45606da2e5b -Description: ROCm Performance Primitives library is a comprehensive high performance computer vision library for AMD CPUs and GPUs with HOST/HIP/OpenCL back-ends. - RPP runtime package provides rpp library and license.txt -Homepage: https://github.com/ROCm/rpp -Maintainer: mivisionx support -Version: 1.8.0.60202-116~20.04 -Installed-Size: 559581 - -Package: rpp-test -Architecture: amd64 -Depends: rpp-dev, clang -Priority: optional -Section: devel -Filename: pool/main/r/rpp-test/rpp-test_1.8.0.60202-116~20.04_amd64.deb -Size: 36448216 -SHA256: 9d81f56ea128aaa26d7466e00858d3d7e290f70d47539205f26f9c9a8f0be1e8 -SHA1: 9719ade9097055bb71c9fbc8f90aa7b10a93804d -MD5sum: ac03f196e3362db70a64a0a72e032a66 -Description: ROCm Performance Primitives library is a comprehensive high performance computer vision library for AMD CPUs and GPUs with HOST/HIP/OpenCL back-ends. RPP test package provides rpp test suite -Homepage: https://github.com/ROCm/rpp -Maintainer: mivisionx support -Version: 1.8.0.60202-116~20.04 -Installed-Size: 93869 - -Package: rpp-test-rpath6.2.2 -Architecture: amd64 -Depends: rpp-dev-rpath6.2.2, clang -Priority: optional -Section: devel -Filename: pool/main/r/rpp-test-rpath6.2.2/rpp-test-rpath6.2.2_1.8.0.60202-116~20.04_amd64.deb -Size: 29752380 -SHA256: 2c8b35671610c7b3d5649e8d9b8a37e74172e28c29dfda502dd3775ec838013f -SHA1: 770288853b991b0bc7a7f0969ec91286d595357a -MD5sum: 7188f8b0c6cc1dd56007d0210cb08a55 -Description: ROCm Performance Primitives library is a comprehensive high performance computer vision library for AMD CPUs and GPUs with HOST/HIP/OpenCL back-ends. RPP test package provides rpp test suite -Homepage: https://github.com/ROCm/rpp -Maintainer: mivisionx support -Version: 1.8.0.60202-116~20.04 -Installed-Size: 93869 - -Package: rpp-test6.2.2 -Architecture: amd64 -Depends: rpp-dev6.2.2, clang -Priority: optional -Section: devel -Filename: pool/main/r/rpp-test6.2.2/rpp-test6.2.2_1.8.0.60202-116~20.04_amd64.deb -Size: 29747996 -SHA256: 5e4f11592a55e046c829d49b715baa55df5a9d4e416e0522893d0b1c7471aa74 -SHA1: 91d03c029554ef54a5a78db3e10ee01e249f19b2 -MD5sum: 3de6727c421fda74657f95633eed8a0e -Description: ROCm Performance Primitives library is a comprehensive high performance computer vision library for AMD CPUs and GPUs with HOST/HIP/OpenCL back-ends. RPP test package provides rpp test suite -Homepage: https://github.com/ROCm/rpp -Maintainer: mivisionx support -Version: 1.8.0.60202-116~20.04 -Installed-Size: 93869 - -Package: rpp6.2.2 -Architecture: amd64 -Depends: rocm-hip-runtime6.2.2 -Priority: optional -Section: devel -Filename: pool/main/r/rpp6.2.2/rpp6.2.2_1.8.0.60202-116~20.04_amd64.deb -Size: 24334500 -SHA256: dfa2a255fceb71b5ebd34d9e0e3e6e2572e8cdb37b0f36152fc1a0628d715e0a -SHA1: b1327226d0935d338f024d6e225504715ed57842 -MD5sum: 41a9fec99bee27b4ad211714b75ff1db -Description: ROCm Performance Primitives library is a comprehensive high performance computer vision library for AMD CPUs and GPUs with HOST/HIP/OpenCL back-ends. - RPP runtime package provides rpp library and license.txt -Homepage: https://github.com/ROCm/rpp -Maintainer: mivisionx support -Version: 1.8.0.60202-116~20.04 -Installed-Size: 559581 - diff --git a/runtimes/rocm/rocm.bzl b/runtimes/rocm/rocm.bzl deleted file mode 100644 index d261be1a..00000000 --- a/runtimes/rocm/rocm.bzl +++ /dev/null @@ -1,242 +0,0 @@ -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -load("//bazel:http_deb_archive.bzl", "http_deb_archive") - -ROCM_VERSION = "6.2.2" -BASE_URL = "https://repo.radeon.com/rocm/apt/{}".format(ROCM_VERSION) -STRIP_PREFIX = "opt/rocm-6.2.2" - -def pkg_kwargs(pkg, packages): - return { - "name": pkg, - "urls": [BASE_URL + "/" + packages[pkg]["Filename"]], - "sha256": packages[pkg]["SHA256"], - "strip_prefix": STRIP_PREFIX, - } - -def _ubuntu_package(path, deb_path, sha256, name, shared_library): - return { - "urls": ["http://archive.ubuntu.com/ubuntu/pool/main/{}".format(path)], - "sha256": sha256, - "build_file_content": """\ -cc_import( - name = {name}, - shared_library = "{deb_path}{shared_library}", - visibility = ["//visibility:public"], -) -""".format(name = repr(name), shared_library = shared_library, deb_path = deb_path), - } - -_UBUNTU_PACKAGES = { - "libdrm": _ubuntu_package( - path = "libd/libdrm/libdrm2_2.4.107-8ubuntu1~20.04.2_amd64.deb", - deb_path = "usr/lib/x86_64-linux-gnu/", - sha256 = "9b01d73313841abe8e3f24c2715edced675fbe329bbd10be912a5b135cd51fb6", - name = "libdrm", - shared_library = "libdrm.so.2", - ), - "libelf": _ubuntu_package( - path = "e/elfutils/libelf1_0.176-1.1build1_amd64.deb", - deb_path = "usr/lib/x86_64-linux-gnu/", - sha256 = "78a8761227efc04a1e37527f2f33ba608c6fb5d6c911616346ada5d7b9b72ee3", - name = "libelf", - shared_library = "libelf.so.1", - ), - "libnuma": _ubuntu_package( - path = "n/numactl/libnuma1_2.0.12-1_amd64.deb", - deb_path = "usr/lib/x86_64-linux-gnu/", - sha256 = "0b1edf08cf9befecd21fe94e298ac25e476f87fd876ddd4adf42ef713449e637", - name = "libnuma", - shared_library = "libnuma.so.1", - ), - "libzstd": _ubuntu_package( - path = "libz/libzstd/libzstd1_1.4.4+dfsg-3ubuntu0.1_amd64.deb", - deb_path = "usr/lib/x86_64-linux-gnu/", - sha256 = "7a4422dadb90510dc90765c308d65e61a3e244ceb3886394335e48cff7559e69", - name = "libzstd", - shared_library = "libzstd.so.1", - ), - "libdrm-amdgpu": _ubuntu_package( - path = "libd/libdrm/libdrm-amdgpu1_2.4.107-8ubuntu1~20.04.2_amd64.deb", - deb_path = "usr/lib/x86_64-linux-gnu/", - sha256 = "0d95779b581f344e3d658e0f21f6e4b57da6eb3606c0bcb8cb874c12f5754bf2", - name = "libdrm-amdgpu", - shared_library = "libdrm_amdgpu.so.1", - ), - "libtinfo": _ubuntu_package( - path = "n/ncurses/libtinfo6_6.2-0ubuntu2.1_amd64.deb", - deb_path = "lib/x86_64-linux-gnu/", - sha256 = "711a3a901c3a71561565558865699efa9c07a99fdc810ffe086a5636f89c6431", - name = "libtinfo", - shared_library = "libtinfo.so.6", - ), - "zlib1g": _ubuntu_package( - path = "z/zlib/zlib1g_1.2.11.dfsg-2ubuntu1.5_amd64.deb", - deb_path = "lib/x86_64-linux-gnu/", - sha256 = "bf67018f5303466eb468680b637a5d3f3bb17b9d44decf3d82d40b35babcd3e0", - name = "zlib1g", - shared_library = "libz.so.1", - ), -} - -_CC_IMPORT_TPL = """\ -cc_import( - name = "{name}", - shared_library = "lib/{shared_library}", - visibility = ["@libpjrt_rocm//:__subpackages__"], -) -""" - -_RUNFILES_TPL = """\ -filegroup( - name = "{name}", - srcs = glob({glob}), - visibility = ["@libpjrt_rocm//:__subpackages__"], -) -""" - -_PACKAGES = { - "rocm-core": _CC_IMPORT_TPL.format(name = "rocm-core", shared_library = "librocm-core.so.1"), - "rocm-smi-lib": _CC_IMPORT_TPL.format(name = "rocm_smi", shared_library = "librocm_smi64.so.7"), - "hsa-rocr": _CC_IMPORT_TPL.format(name = "hsa-runtime", shared_library = "libhsa-runtime64.so.1"), - "hsa-amd-aqlprofile": _CC_IMPORT_TPL.format(name = "hsa-amd-aqlprofile", shared_library = "libhsa-amd-aqlprofile64.so.1"), - "comgr": _CC_IMPORT_TPL.format(name = "amd_comgr", shared_library = "libamd_comgr.so.2"), - "rocprofiler-register": _CC_IMPORT_TPL.format(name = "rocprofiler-register", shared_library = "librocprofiler-register.so.0"), - "miopen-hip": "".join([ - _CC_IMPORT_TPL.format(name = "MIOpen", shared_library = "libMIOpen.so.1"), - _RUNFILES_TPL.format(name = "runfiles", glob = repr(["share/miopen/**"])), - ]), - "rccl": "".join([ - _CC_IMPORT_TPL.format(name = "rccl", shared_library = "librccl.so.1"), - _RUNFILES_TPL.format(name = "runfiles", glob = repr(["share/rccl/msccl-algorithms/**"])), - ]), - "rocm-device-libs": _RUNFILES_TPL.format(name = "runfiles", glob = repr(["amdgcn/**"])), - "hip-dev": _RUNFILES_TPL.format(name = "runfiles", glob = repr(["share/**"])), - "rocblas": """\ -load("@zml//bazel:cc_import.bzl", "cc_import") -load("@zml//runtimes/rocm:gfx.bzl", "bytecode_select") - -cc_import( - name = "rocblas", - shared_library = "lib/librocblas.so.4", - add_needed = ["libzmlrocmhooks.so.0"], - visibility = ["@libpjrt_rocm//:__subpackages__"], -) - -bytecode_select( - name = "runfiles", - bytecodes = glob(["lib/rocblas/library/*"]), - enabled_gfx = "@libpjrt_rocm//:gfx", - visibility = ["@libpjrt_rocm//:__subpackages__"], -) -""", - "roctracer": """\ -cc_import( - name = "roctracer", - shared_library = "lib/libroctracer64.so.4", - visibility = ["@libpjrt_rocm//:__subpackages__"], - deps = [":roctx"], -) - -cc_import( - name = "roctx", - shared_library = "lib/libroctx64.so.4", -) -""", - "hipblaslt": """\ -load("@zml//bazel:cc_import.bzl", "cc_import") -cc_import( - name = "hipblaslt", - shared_library = "lib/libhipblaslt.so.0", - add_needed = ["libzmlrocmhooks.so.0"], - visibility = ["@libpjrt_rocm//:__subpackages__"], -) -""", - "hipblaslt-dev": """\ -load("@zml//runtimes/rocm:gfx.bzl", "bytecode_select") - -bytecode_select( - name = "bytecodes", - bytecodes = glob( - include = ["lib/hipblaslt/library/*"], - exclude = ["lib/hipblaslt/library/hipblasltExtOpLibrary.dat"], - ), - enabled_gfx = "@libpjrt_rocm//:gfx", -) - -filegroup( - name = "runfiles", - srcs = [ - "lib/hipblaslt/library/hipblasltExtOpLibrary.dat", - ":bytecodes", - ], - visibility = ["@libpjrt_rocm//:__subpackages__"], -) -""", - "hip-runtime-amd": """\ -cc_import( - name = "amdhip", - shared_library = "lib/libamdhip64.so.6", - visibility = ["@libpjrt_rocm//:__subpackages__"], - deps = [":hiprtc"], -) -cc_import( - name = "hiprtc", - shared_library = "lib/libhiprtc.so.6", -) -""", - "rocm-llvm": """\ -filegroup( - name = "lld", - srcs = ["llvm/bin/ld.lld"], - visibility = ["@libpjrt_rocm//:__subpackages__"], -) -""", -} - -def _packages_to_dict(txt): - packages = {} - current_pkg = {} - for line in txt.splitlines(): - if line == "": - if current_pkg: - packages[current_pkg["Package"]] = current_pkg - current_pkg = {} - continue - if line.startswith(" "): - current_pkg[key] += line - continue - split = line.split(": ", 1) - key = split[0] - value = len(split) > 1 and split[1] or "" - current_pkg[key] = value - return packages - -def _rocm_impl(mctx): - data = mctx.read(Label("@zml//runtimes/rocm:packages.amd64.txt")) - PACKAGES = _packages_to_dict(data) - - for pkg, build_file_content in _PACKAGES.items(): - http_deb_archive( - build_file_content = build_file_content, - **pkg_kwargs(pkg, PACKAGES) - ) - - for repository, kwargs in _UBUNTU_PACKAGES.items(): - http_deb_archive(name = repository, **kwargs) - - http_archive( - name = "libpjrt_rocm", - build_file = "libpjrt_rocm.BUILD.bazel", - url = "https://github.com/zml/pjrt-artifacts/releases/download/v0.2.2/pjrt-rocm_linux-amd64.tar.gz", - sha256 = "dcb2f8e1fd29e3d7ba8d3018d97a060888e5bcf4847a683cb11686caa6ad9fa2", - ) - - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = ["libpjrt_rocm"], - root_module_direct_dev_deps = [], - ) - -rocm_packages = module_extension( - implementation = _rocm_impl, -) diff --git a/runtimes/rocm/zmlrocmhooks.cc b/runtimes/rocm/zmlrocmhooks.cc deleted file mode 100644 index 0c0fc60a..00000000 --- a/runtimes/rocm/zmlrocmhooks.cc +++ /dev/null @@ -1,103 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include "tools/cpp/runfiles/runfiles.h" - -namespace zml -{ - using bazel::tools::cpp::runfiles::Runfiles; - - std::unique_ptr runfiles; - std::string ROCBLAS_TENSILE_LIBPATH; - std::string HIPBLASLT_TENSILE_LIBPATH; - std::string HIPBLASLT_EXT_OP_LIBRARY_PATH; - std::string ROCM_PATH; - - typedef void *(*dlopen_func)(const char *filename, int flags); - dlopen_func dlopen_orig = nullptr; - - __attribute__((constructor)) static void setup(int argc, char **argv) - { - runfiles = std::unique_ptr(Runfiles::Create(argv[0], BAZEL_CURRENT_REPOSITORY)); - - HIPBLASLT_EXT_OP_LIBRARY_PATH = runfiles->Rlocation("hipblaslt-dev/lib/hipblaslt/library/hipblasltExtOpLibrary.dat"); - if (HIPBLASLT_EXT_OP_LIBRARY_PATH != "") - { - setenv("HIPBLASLT_EXT_OP_LIBRARY_PATH", HIPBLASLT_EXT_OP_LIBRARY_PATH.c_str(), 1); - } - - HIPBLASLT_TENSILE_LIBPATH = runfiles->Rlocation("hipblaslt-dev/lib/hipblaslt/library"); - if (HIPBLASLT_TENSILE_LIBPATH != "") - { - setenv("HIPBLASLT_TENSILE_LIBPATH", HIPBLASLT_TENSILE_LIBPATH.c_str(), 1); - } - - ROCBLAS_TENSILE_LIBPATH = runfiles->Rlocation("rocblas/lib/rocblas/library"); - setenv("ROCBLAS_TENSILE_LIBPATH", ROCBLAS_TENSILE_LIBPATH.c_str(), 1); - - ROCM_PATH = runfiles->Rlocation("libpjrt_rocm/sandbox"); - setenv("ROCM_PATH", ROCM_PATH.c_str(), 1); - } - - static void *rocm_dlopen(const char *filename, int flags) - { - if (filename != NULL) - { - char *replacements[] = { - "librocm-core.so", - "librocm-core.so.1", - "librocm_smi64.so", - "librocm_smi64.so.7", - "libhsa-runtime64.so", - "libhsa-runtime64.so.1", - "libhsa-amd-aqlprofile64.so", - "libhsa-amd-aqlprofile64.so.1", - "libamd_comgr.so", - "libamd_comgr.so.2", - "librocprofiler-register.so", - "librocprofiler-register.so.0", - "libMIOpen.so", - "libMIOpen.so.1", - "librccl.so", - "librccl.so.1", - "librocblas.so", - "librocblas.so.4", - "libroctracer64.so", - "libroctracer64.so.4", - "libroctx64.so", - "libroctx64.so.4", - "libhipblaslt.so", - "libhipblaslt.so.0", - "libamdhip64.so", - "libamdhip64.so.6", - "libhiprtc.so", - "libhiprtc.so.6", - NULL, - NULL, - }; - for (int i = 0; replacements[i] != NULL; i += 2) - { - if (strcmp(filename, replacements[i]) == 0) - { - filename = replacements[i + 1]; - break; - } - } - } - return dlopen_orig(filename, flags); - } -} - -extern "C" -{ - zml::dlopen_func _zml_rocm_resolve_dlopen() - { - zml::dlopen_orig = (zml::dlopen_func)dlsym(RTLD_NEXT, "dlopen"); - return zml::rocm_dlopen; - } - - extern void *dlopen(const char *filename, int flags) __attribute__((ifunc("_zml_rocm_resolve_dlopen"))); -} diff --git a/runtimes/tpu/BUILD.bazel b/runtimes/tpu/BUILD.bazel deleted file mode 100644 index 6bda66e6..00000000 --- a/runtimes/tpu/BUILD.bazel +++ /dev/null @@ -1,5 +0,0 @@ -alias( - name = "tpu", - actual = "@libpjrt_tpu", - visibility = ["//visibility:public"], -) diff --git a/runtimes/tpu/libpjrt_tpu.BUILD.bazel b/runtimes/tpu/libpjrt_tpu.BUILD.bazel deleted file mode 100644 index b4385aa8..00000000 --- a/runtimes/tpu/libpjrt_tpu.BUILD.bazel +++ /dev/null @@ -1,14 +0,0 @@ -load("@bazel_skylib//rules:copy_file.bzl", "copy_file") - -copy_file( - name = "libpjrt_tpu_so", - src = "libtpu/libtpu.so", - out = "libpjrt_tpu.so", - allow_symlink = True, -) - -cc_import( - name = "libpjrt_tpu", - shared_library = ":libpjrt_tpu_so", - visibility = ["@zml//runtimes/tpu:__subpackages__"], -) diff --git a/stdx/BUILD.bazel b/stdx/BUILD.bazel new file mode 100644 index 00000000..d7c94d82 --- /dev/null +++ b/stdx/BUILD.bazel @@ -0,0 +1,38 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "stdx", + srcs = [ + "Io.zig", + "bounded_array.zig", + "crypto.zig", + "debug.zig", + "flags.zig", + "fmt.zig", + "cpu.zig", + "json.zig", + "math.zig", + "meta.zig", + "process.zig", + "queue.zig", + "segmented_list.zig", + "time.zig", + "unicode.zig", + ], + main = "stdx.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + zigopts = ["-lc"], +) + +zig_test( + name = "test", + testonly = False, + deps = [":stdx"], +) + +zig_test( + name = "cpu_test", + main = "cpu.zig", + zigopts = ["-lc"], +) diff --git a/stdx/Io.zig b/stdx/Io.zig new file mode 100644 index 00000000..5b5a35d8 --- /dev/null +++ b/stdx/Io.zig @@ -0,0 +1,151 @@ +const std = @import("std"); + +pub const Dir = struct { + pub const path = struct { + pub fn bufJoin(buf: []u8, paths: []const []const u8) ![]u8 { + var fa: std.heap.FixedBufferAllocator = .init(buf); + return try std.Io.Dir.path.join(fa.allocator(), paths); + } + + pub fn bufJoinZ(buf: []u8, paths: []const []const u8) ![:0]u8 { + var fa: std.heap.FixedBufferAllocator = .init(buf); + return try std.Io.Dir.path.joinZ(fa.allocator(), paths); + } + }; + + pub fn readFileAlloc(dir: std.Io.Dir, io: std.Io, sub_path: []const u8, gpa: std.mem.Allocator, limit: std.Io.Limit) ![]u8 { + const stat = try std.Io.Dir.statFile(dir, io, sub_path, .{}); + const buffer = try gpa.alloc(u8, limit.minInt64(stat.size)); + errdefer gpa.free(buffer); + _ = try std.Io.Dir.readFile(.cwd(), io, sub_path, buffer); + return buffer; + } +}; + +pub const LimitedGroup = struct { + limit: usize, + in_flight: std.atomic.Value(usize) = .init(0), + group: std.Io.Group = .init, + cond: std.Io.Condition = .init, + mutex: std.Io.Mutex = .init, + + fn Wrapper(comptime function: anytype) type { + return struct { + fn wrapper(self: *LimitedGroup, io: std.Io, args: std.meta.ArgsTuple(@TypeOf(function))) std.Io.Cancelable!void { + while (true) { + var in_flight = self.in_flight.load(.acquire); + while (in_flight < self.limit) { + if (self.in_flight.cmpxchgWeak(in_flight, in_flight + 1, .release, .acquire)) |actual| { + in_flight = actual; + continue; + } + + defer { + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + _ = self.in_flight.fetchSub(1, .release); + self.cond.signal(io); + } + return @call(.auto, function, args); + } + + try self.mutex.lock(io); + defer self.mutex.unlock(io); + while (self.in_flight.load(.acquire) >= self.limit) { + try self.cond.wait(io, &self.mutex); + } + } + } + }; + } + + pub fn init(limit: usize) LimitedGroup { + return .{ .limit = limit }; + } + + pub fn async(self: *LimitedGroup, io: std.Io, comptime function: anytype, args: std.meta.ArgsTuple(@TypeOf(function))) void { + self.group.async(io, Wrapper(function).wrapper, .{ self, io, args }); + } + + pub fn concurrent(self: *LimitedGroup, io: std.Io, comptime function: anytype, args: std.meta.ArgsTuple(@TypeOf(function))) std.Io.ConcurrentError!void { + try self.group.concurrent(io, Wrapper(function).wrapper, .{ self, io, args }); + } + + pub fn await(self: *LimitedGroup, io: std.Io) std.Io.Cancelable!void { + try self.group.await(io); + } + + pub fn cancel(self: *LimitedGroup, io: std.Io) void { + self.group.cancel(io); + } +}; + +pub const AllocatingLimitedConcurrentGroup = struct { + allocator: std.mem.Allocator, + + queue_buffer: []bool = &[_]bool{}, + limit: usize, + + group: std.Io.Group = .init, + queue: std.Io.Queue(bool) = undefined, + + pub fn init(allocator: std.mem.Allocator, limit: usize) std.mem.Allocator.Error!AllocatingLimitedConcurrentGroup { + const buffer = try allocator.alloc(bool, limit); + @memset(buffer, false); + var self: AllocatingLimitedConcurrentGroup = .{ + .allocator = allocator, + .queue_buffer = buffer, + .limit = limit, + .group = .init, + .queue = undefined, + }; + self.queue = std.Io.Queue(bool).init(self.queue_buffer); + return self; + } + + pub fn deinit(self: *AllocatingLimitedConcurrentGroup) void { + self.allocator.free(self.queue_buffer); + } + + pub fn concurrent( + self: *AllocatingLimitedConcurrentGroup, + io: std.Io, + comptime function: anytype, + args: std.meta.ArgsTuple(@TypeOf(function)), + ) !void { + self.queue.putOneUncancelable(io, true) catch unreachable; + + try self.group.concurrent(io, struct { + fn wrapper( + io_: std.Io, + queue: *std.Io.Queue(bool), + inner_args: std.meta.ArgsTuple(@TypeOf(function)), + ) !void { + defer _ = queue.getOneUncancelable(io_) catch unreachable; + return @call(.auto, function, inner_args); + } + }.wrapper, .{ io, &self.queue, args }); + } + + pub fn await(self: *AllocatingLimitedConcurrentGroup, io: std.Io) std.Io.Cancelable!void { + try self.group.await(io); + } + + pub fn cancel(self: *AllocatingLimitedConcurrentGroup, io: std.Io) void { + self.group.cancel(io); + } +}; + +pub const Duration = struct { + pub fn div(self: std.Io.Duration, rhs: u64) std.Io.Duration { + return .fromNanoseconds(@divTrunc(self.nanoseconds, @as(i96, @intCast(rhs)))); + } + + pub fn hz(self: std.Io.Duration) u64 { + return @intCast(std.time.ns_per_s / self.nanoseconds); + } + + pub fn hzFloat(self: std.Io.Duration) f64 { + return (1 * std.time.ns_per_s) / @as(f64, @floatFromInt(self.nanoseconds)); + } +}; diff --git a/stdx/bounded_array.zig b/stdx/bounded_array.zig new file mode 100644 index 00000000..1f77a14d --- /dev/null +++ b/stdx/bounded_array.zig @@ -0,0 +1,415 @@ +const std = @import("std"); +const assert = std.debug.assert; +const mem = std.mem; +const testing = std.testing; +const Alignment = std.mem.Alignment; + +/// A structure with an array and a length, that can be used as a slice. +/// +/// Useful to pass around small arrays whose exact size is only known at +/// runtime, but whose maximum size is known at comptime, without requiring +/// an `Allocator`. +/// +/// ```zig +/// var actual_size = 32; +/// var a = try BoundedArray(u8, 64).init(actual_size); +/// var slice = a.slice(); // a slice of the 64-byte array +/// var a_clone = a; // creates a copy - the structure doesn't use any internal pointers +/// ``` +pub fn BoundedArray(comptime T: type, comptime buffer_capacity: usize) type { + return BoundedArrayAligned(T, .of(T), buffer_capacity); +} + +/// A structure with an array, length and alignment, that can be used as a +/// slice. +/// +/// Useful to pass around small explicitly-aligned arrays whose exact size is +/// only known at runtime, but whose maximum size is known at comptime, without +/// requiring an `Allocator`. +/// ```zig +/// var a: BoundedArrayAligned(u8, 16, 2) = .empty; +/// try a.append(255); +/// try a.append(255); +/// const b: *const [1]u16 = @ptrCast(a.constSlice().ptr); +/// try testing.expectEqual(@as(u16, 65535), b[0]); +/// ``` +/// +/// Note: there is no init anymore. It's generally unsafe because it makes `slice()` return undefined.; +/// If you used `try init(0)` use `.empty` +/// If you used `try init(n)` use `.{ .buffer = undefined, .len = n }` and fill `buffer` yourself. +pub fn BoundedArrayAligned( + comptime T: type, + comptime alignment: Alignment, + comptime buffer_capacity: usize, +) type { + return struct { + const Self = @This(); + buffer: [buffer_capacity]T align(alignment.toByteUnits()), + len: usize, + + pub const max_len: usize = buffer_capacity; + + pub const empty: Self = .{ .buffer = undefined, .len = 0 }; + + pub fn init(m: []const T) Self { + var buf: Self = .{ .buffer = undefined, .len = m.len }; + @memcpy(buf.buffer[0..m.len], m); + return buf; + } + + pub fn repeat(value: T, len: usize) Self { + std.debug.assert(len <= buffer_capacity); + var buf: Self = .{ .buffer = undefined, .len = len }; + @memset(buf.buffer[0..len], value); + return buf; + } + + /// View the internal array as a slice whose size was previously set. + pub fn slice(self: anytype) switch (@TypeOf(&self.buffer)) { + *align(alignment.toByteUnits()) [buffer_capacity]T => []align(alignment.toByteUnits()) T, + *align(alignment.toByteUnits()) const [buffer_capacity]T => []align(alignment.toByteUnits()) const T, + else => unreachable, + } { + return self.buffer[0..self.len]; + } + + /// View the internal array as a constant slice whose size was previously set. + pub fn constSlice(self: *const Self) []align(alignment.toByteUnits()) const T { + return self.slice(); + } + + /// Adjust the slice's length to `len`. + /// Does not initialize added items if any. + pub fn resize(self: *Self, len: usize) error{Overflow}!void { + if (len > buffer_capacity) return error.Overflow; + self.len = len; + } + + /// Remove all elements from the slice. + pub fn clear(self: *Self) void { + self.len = 0; + } + + /// Copy the content of an existing slice. + pub fn fromSlice(m: []const T) error{Overflow}!Self { + if (m.len > buffer_capacity) return error.Overflow; + return .init(m); + } + + /// Return the element at index `i` of the slice. + pub fn get(self: Self, i: usize) T { + std.debug.assert(i < self.len); + return self.buffer[i]; + } + + /// Set the value of the element at index `i` of the slice. + pub fn set(self: *Self, i: usize, item: T) void { + std.debug.assert(i < self.len); + self.buffer[i] = item; + } + + /// Return the maximum length of a slice. + pub fn capacity(self: Self) usize { + return self.buffer.len; + } + + /// Check that the slice can hold at least `additional_count` items. + pub fn ensureUnusedCapacity(self: Self, additional_count: usize) error{Overflow}!void { + if (self.len + additional_count > buffer_capacity) { + return error.Overflow; + } + } + + /// Increase length by 1, returning a pointer to the new item. + pub fn addOne(self: *Self) error{Overflow}!*T { + try self.ensureUnusedCapacity(1); + return self.addOneAssumeCapacity(); + } + + /// Increase length by 1, returning pointer to the new item. + /// Asserts that there is space for the new item. + pub fn addOneAssumeCapacity(self: *Self) *T { + assert(self.len < buffer_capacity); + self.len += 1; + return &self.slice()[self.len - 1]; + } + + /// Resize the slice, adding `n` new elements, which have `undefined` values. + /// The return value is a pointer to the array of uninitialized elements. + pub fn addManyAsArray(self: *Self, comptime n: usize) error{Overflow}!*align(alignment.toByteUnits()) [n]T { + const prev_len = self.len; + try self.resize(self.len + n); + return self.slice()[prev_len..][0..n]; + } + + /// Resize the slice, adding `n` new elements, which have `undefined` values. + /// The return value is a slice pointing to the uninitialized elements. + pub fn addManyAsSlice(self: *Self, n: usize) error{Overflow}![]align(alignment.toByteUnits()) T { + const prev_len = self.len; + try self.resize(self.len + n); + return self.slice()[prev_len..][0..n]; + } + + /// Remove and return the last element from the slice, or return `null` if the slice is empty. + pub fn pop(self: *Self) ?T { + if (self.len == 0) return null; + const item = self.get(self.len - 1); + self.len -= 1; + return item; + } + + /// Return a slice of only the extra capacity after items. + /// This can be useful for writing directly into it. + /// Note that such an operation must be followed up with a + /// call to `resize()` + pub fn unusedCapacitySlice(self: *Self) []align(alignment.toByteUnits()) T { + return self.buffer[self.len..]; + } + + /// Insert `item` at index `i` by moving `slice[n .. slice.len]` to make room. + /// This operation is O(N). + pub fn insert( + self: *Self, + i: usize, + item: T, + ) error{Overflow}!void { + if (i > self.len) { + return error.Overflow; + } + _ = try self.addOne(); + var s = self.slice(); + mem.copyBackwards(T, s[i + 1 .. s.len], s[i .. s.len - 1]); + self.buffer[i] = item; + } + + /// Insert slice `items` at index `i` by moving `slice[i .. slice.len]` to make room. + /// This operation is O(N). + pub fn insertSlice(self: *Self, i: usize, items: []const T) error{Overflow}!void { + try self.ensureUnusedCapacity(items.len); + self.len += items.len; + mem.copyBackwards(T, self.slice()[i + items.len .. self.len], self.constSlice()[i .. self.len - items.len]); + @memcpy(self.slice()[i..][0..items.len], items); + } + + /// Replace range of elements `slice[start..][0..len]` with `new_items`. + /// Grows slice if `len < new_items.len`. + /// Shrinks slice if `len > new_items.len`. + pub fn replaceRange( + self: *Self, + start: usize, + len: usize, + new_items: []const T, + ) error{Overflow}!void { + const after_range = start + len; + var range = self.slice()[start..after_range]; + + if (range.len == new_items.len) { + @memcpy(range[0..new_items.len], new_items); + } else if (range.len < new_items.len) { + const first = new_items[0..range.len]; + const rest = new_items[range.len..]; + @memcpy(range[0..first.len], first); + try self.insertSlice(after_range, rest); + } else { + @memcpy(range[0..new_items.len], new_items); + const after_subrange = start + new_items.len; + for (self.constSlice()[after_range..], 0..) |item, i| { + self.slice()[after_subrange..][i] = item; + } + self.len -= len - new_items.len; + } + } + + /// Extend the slice by 1 element. + pub fn append(self: *Self, item: T) error{Overflow}!void { + const new_item_ptr = try self.addOne(); + new_item_ptr.* = item; + } + + /// Extend the slice by 1 element, asserting the capacity is already + /// enough to store the new item. + pub fn appendAssumeCapacity(self: *Self, item: T) void { + const new_item_ptr = self.addOneAssumeCapacity(); + new_item_ptr.* = item; + } + + /// Remove the element at index `i`, shift elements after index + /// `i` forward, and return the removed element. + /// Asserts the slice has at least one item. + /// This operation is O(N). + pub fn orderedRemove(self: *Self, i: usize) T { + const newlen = self.len - 1; + if (newlen == i) return self.pop().?; + const old_item = self.get(i); + for (self.slice()[i..newlen], 0..) |*b, j| b.* = self.get(i + 1 + j); + self.set(newlen, undefined); + self.len = newlen; + return old_item; + } + + /// Remove the element at the specified index and return it. + /// The empty slot is filled from the end of the slice. + /// This operation is O(1). + pub fn swapRemove(self: *Self, i: usize) T { + if (self.len - 1 == i) return self.pop().?; + const old_item = self.get(i); + self.set(i, self.pop().?); + return old_item; + } + + /// Append the slice of items to the slice. + pub fn appendSlice(self: *Self, items: []const T) error{Overflow}!void { + try self.ensureUnusedCapacity(items.len); + self.appendSliceAssumeCapacity(items); + } + + /// Append the slice of items to the slice, asserting the capacity is already + /// enough to store the new items. + pub fn appendSliceAssumeCapacity(self: *Self, items: []const T) void { + const old_len = self.len; + self.len += items.len; + @memcpy(self.slice()[old_len..][0..items.len], items); + } + + /// Append a value to the slice `n` times. + /// Allocates more memory as necessary. + pub fn appendNTimes(self: *Self, value: T, n: usize) error{Overflow}!void { + const old_len = self.len; + try self.resize(old_len + n); + @memset(self.slice()[old_len..self.len], value); + } + + /// Append a value to the slice `n` times. + /// Asserts the capacity is enough. + pub fn appendNTimesAssumeCapacity(self: *Self, value: T, n: usize) void { + const old_len = self.len; + self.len += n; + assert(self.len <= buffer_capacity); + @memset(self.slice()[old_len..self.len], value); + } + + /// Same as `appendSlice` except it returns the number of bytes written, which is always the same + /// as `m.len`. The purpose of this function existing is to match `std.io.GenericWriter` API. + fn appendWrite(self: *Self, m: []const u8) error{Overflow}!usize { + try self.appendSlice(m); + return m.len; + } + }; +} + +test BoundedArray { + var a: BoundedArray(u8, 64) = .{ .buffer = undefined, .len = 32 }; + + try testing.expectEqual(a.capacity(), 64); + try testing.expectEqual(a.slice().len, 32); + try testing.expectEqual(a.constSlice().len, 32); + + try a.resize(48); + try testing.expectEqual(a.len, 48); + + const x: [10]u8 = @splat(1); + a = try .fromSlice(&x); + try testing.expectEqualSlices(u8, &x, a.constSlice()); + + var a2 = a; + try testing.expectEqualSlices(u8, a.constSlice(), a2.constSlice()); + a2.set(0, 0); + try testing.expect(a.get(0) != a2.get(0)); + + try testing.expectError(error.Overflow, a.resize(100)); + try testing.expectError(error.Overflow, BoundedArray(u8, x.len - 1).fromSlice(&x)); + + try a.resize(0); + try a.ensureUnusedCapacity(a.capacity()); + (try a.addOne()).* = 0; + try a.ensureUnusedCapacity(a.capacity() - 1); + try testing.expectEqual(a.len, 1); + + const uninitialized = try a.addManyAsArray(4); + try testing.expectEqual(uninitialized.len, 4); + try testing.expectEqual(a.len, 5); + + try a.append(0xff); + try testing.expectEqual(a.len, 6); + try testing.expectEqual(a.pop(), 0xff); + + a.appendAssumeCapacity(0xff); + try testing.expectEqual(a.len, 6); + try testing.expectEqual(a.pop(), 0xff); + + try a.resize(1); + try testing.expectEqual(a.pop(), 0); + try testing.expectEqual(a.pop(), null); + var unused = a.unusedCapacitySlice(); + @memset(unused[0..8], 2); + unused[8] = 3; + unused[9] = 4; + try testing.expectEqual(unused.len, a.capacity()); + try a.resize(10); + + try a.insert(5, 0xaa); + try testing.expectEqual(a.len, 11); + try testing.expectEqual(a.get(5), 0xaa); + try testing.expectEqual(a.get(9), 3); + try testing.expectEqual(a.get(10), 4); + + try a.insert(11, 0xbb); + try testing.expectEqual(a.len, 12); + try testing.expectEqual(a.pop(), 0xbb); + + try a.appendSlice(&x); + try testing.expectEqual(a.len, 11 + x.len); + + try a.appendNTimes(0xbb, 5); + try testing.expectEqual(a.len, 11 + x.len + 5); + try testing.expectEqual(a.pop(), 0xbb); + + a.appendNTimesAssumeCapacity(0xcc, 5); + try testing.expectEqual(a.len, 11 + x.len + 5 - 1 + 5); + try testing.expectEqual(a.pop(), 0xcc); + + try testing.expectEqual(a.len, 29); + try a.replaceRange(1, 20, &x); + try testing.expectEqual(a.len, 29 + x.len - 20); + + try a.insertSlice(0, &x); + try testing.expectEqual(a.len, 29 + x.len - 20 + x.len); + + try a.replaceRange(1, 5, &x); + try testing.expectEqual(a.len, 29 + x.len - 20 + x.len + x.len - 5); + + try a.append(10); + try testing.expectEqual(a.pop(), 10); + + try a.append(20); + const removed = a.orderedRemove(5); + try testing.expectEqual(removed, 1); + try testing.expectEqual(a.len, 34); + + a.set(0, 0xdd); + a.set(a.len - 1, 0xee); + const swapped = a.swapRemove(0); + try testing.expectEqual(swapped, 0xdd); + try testing.expectEqual(a.get(0), 0xee); + + const added_slice = try a.addManyAsSlice(3); + try testing.expectEqual(added_slice.len, 3); + try testing.expectEqual(a.len, 36); + + while (a.pop()) |_| {} + const s = "hello, this is a test string"; + _ = try a.appendWrite(s); + try testing.expectEqualStrings(s, a.constSlice()); +} + +test "BoundedArrayAligned" { + var a: BoundedArrayAligned(u8, .@"16", 4) = .empty; + try a.append(0); + try a.append(0); + try a.append(255); + try a.append(255); + + const b = @as(*const [2]u16, @ptrCast(a.constSlice().ptr)); + try testing.expectEqual(@as(u16, 0), b[0]); + try testing.expectEqual(@as(u16, 65535), b[1]); +} diff --git a/stdx/cpu.zig b/stdx/cpu.zig new file mode 100644 index 00000000..c51b32d4 --- /dev/null +++ b/stdx/cpu.zig @@ -0,0 +1,411 @@ +const std = @import("std"); +const builtin = @import("builtin"); + +/// Restricts the calling thread and existing threads to the CPUs of a NUMA +/// node within the caller's current affinity, preserving each thread's narrower +/// affinity. Threads with no overlap are left alone. Returns the number of +/// threads whose affinity changed. +/// Call during startup, before threads can concurrently create other threads. +/// On failure, previously changed affinities remain in effect. +pub fn pinProcessToNumaNode(io: std.Io, node: u32) !usize { + if (builtin.os.tag != .linux) return error.Unsupported; + + var path_buf: [64]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buf, "/sys/devices/system/node/node{d}/cpulist", .{node}); + + var cpulist_buf: [4096]u8 = undefined; + const cpulist = try std.Io.Dir.cwd().readFile(io, path, &cpulist_buf); + + var set: std.os.linux.cpu_set_t = @splat(0); + try parseCpuList(cpulist, &set); + + return pinProcessToCpuSet(io, set); +} + +/// CPUs the calling thread may run on, ordered for worker pools. +pub const AllowedCores = struct { + /// Allowed CPU ids: one per physical core first, then the remaining SMT + /// siblings, round by round. + ids: []const usize, + /// `ids[0..physical_count]` holds exactly one CPU per physical core. Equals + /// `ids.len` when the topology is unknown. + physical_count: usize, + + /// One CPU per physical core, skipping SMT siblings. + pub fn physical(self: AllowedCores) []const usize { + return self.ids[0..self.physical_count]; + } + + pub fn deinit(self: AllowedCores, allocator: std.mem.Allocator) void { + allocator.free(self.ids); + } +}; + +/// Returns the CPUs the calling thread may run on. Linux uses sysfs topology so +/// that a second logical CPU of a physical core comes only after every allowed +/// physical core got one. +pub fn allowedCores(allocator: std.mem.Allocator, io: std.Io) !AllowedCores { + if (builtin.os.tag != .linux) return naturalOrder(allocator, try std.Thread.getCpuCount()); + + const cpus = linuxAllowedCoreIds(allocator) catch |err| switch (err) { + error.OutOfMemory => return err, + else => return naturalOrder(allocator, try std.Thread.getCpuCount()), + }; + errdefer allocator.free(cpus); + + const physical_count = linuxOrderBySiblings(allocator, io, cpus) catch |err| switch (err) { + error.OutOfMemory => return err, + else => cpus.len, + }; + return .{ .ids = cpus, .physical_count = physical_count }; +} + +fn pinProcessToCpuSet(io: std.Io, node_set: std.os.linux.cpu_set_t) !usize { + const linux = std.os.linux; + // Keep the caller's taskset/numactl choice as the process-wide limit, and + // intersect each thread's own affinity below so private bindings stay narrow. + const set = intersectCpuSets(node_set, try threadAffinity(0)); + if (std.mem.allEqual(usize, &set, 0)) return 0; + + var tasks = try std.Io.Dir.cwd().openDir(io, "/proc/self/task", .{ .iterate = true }); + defer tasks.close(io); + + // Pin the caller first so threads it creates inherit the target mask. + var pinned: usize = @intFromBool(try pinThreadToCpuSet(0, set)); + var it = tasks.iterate(); + while (try it.next(io)) |entry| { + const tid = std.fmt.parseInt(linux.pid_t, entry.name, 10) catch continue; + // Threads may exit while the list is being walked. + const changed = pinThreadToCpuSet(tid, set) catch |err| switch (err) { + error.ThreadGone => continue, + else => return err, + }; + pinned += @intFromBool(changed); + } + + return pinned; +} + +fn intersectCpuSets(a: std.os.linux.cpu_set_t, b: std.os.linux.cpu_set_t) std.os.linux.cpu_set_t { + var set = a; + for (&set, b) |*word, other| word.* &= other; + return set; +} + +fn threadAffinity(tid: std.os.linux.pid_t) error{ ThreadGone, GetAffinityFailed }!std.os.linux.cpu_set_t { + const linux = std.os.linux; + var set: linux.cpu_set_t = @splat(0); + switch (linux.errno(linux.sched_getaffinity(tid, @sizeOf(linux.cpu_set_t), &set))) { + .SUCCESS => return set, + .SRCH => return error.ThreadGone, + else => return error.GetAffinityFailed, + } +} + +fn setThreadAffinity(tid: std.os.linux.pid_t, set: *const std.os.linux.cpu_set_t) error{ ThreadGone, SetAffinityFailed }!void { + const linux = std.os.linux; + const rc = linux.syscall3(.sched_setaffinity, @as(usize, @bitCast(@as(isize, tid))), @sizeOf(linux.cpu_set_t), @intFromPtr(set)); + switch (linux.errno(rc)) { + .SUCCESS => {}, + .SRCH => return error.ThreadGone, + else => return error.SetAffinityFailed, + } +} + +fn pinThreadToCpuSet(tid: std.os.linux.pid_t, target: std.os.linux.cpu_set_t) !bool { + const current = try threadAffinity(tid); + const set = intersectCpuSets(target, current); + if (std.mem.allEqual(usize, &set, 0) or std.mem.eql(usize, &set, ¤t)) return false; + try setThreadAffinity(tid, &set); + return true; +} + +/// Parses Linux CPU lists, including singleton CPUs and inclusive ranges. +fn parseCpuList(cpulist: []const u8, set: *std.os.linux.cpu_set_t) !void { + const trimmed = std.mem.trim(u8, cpulist, " \t\r\n"); + if (trimmed.len == 0) return; + + const bits_per_word = @bitSizeOf(usize); + + var ranges = std.mem.splitScalar(u8, trimmed, ','); + while (ranges.next()) |raw_range| { + const range = std.mem.trim(u8, raw_range, " \t\r\n"); + var ends = std.mem.splitScalar(u8, range, '-'); + const first = std.fmt.parseInt(usize, ends.first(), 10) catch return error.InvalidCpuList; + + const last = if (ends.next()) |end| + std.fmt.parseInt(usize, end, 10) catch return error.InvalidCpuList + else + first; + + if (last < first or ends.next() != null or last >= set.len * bits_per_word) return error.InvalidCpuList; + for (first..last + 1) |cpu| { + set[cpu / bits_per_word] |= @as(usize, 1) << @intCast(cpu % bits_per_word); + } + } +} + +fn linuxAllowedCoreIds(allocator: std.mem.Allocator) ![]usize { + const set = try threadAffinity(0); + + var core_ids: std.ArrayList(usize) = .empty; + errdefer core_ids.deinit(allocator); + + var core_id: usize = 0; + while (core_id < @bitSizeOf(std.os.linux.cpu_set_t)) : (core_id += 1) { + if (cpuSetContains(set, core_id)) { + try core_ids.append(allocator, core_id); + } + } + + if (core_ids.items.len == 0) return error.MissingCpuTopology; + + return core_ids.toOwnedSlice(allocator); +} + +fn cpuSetContains(set: std.os.linux.cpu_set_t, core_id: usize) bool { + const word_bit_count = @bitSizeOf(usize); + const word_index = core_id / word_bit_count; + const bit_index: std.math.Log2Int(usize) = @intCast(core_id % word_bit_count); + return word_index < set.len and (set[word_index] & (@as(usize, 1) << bit_index)) != 0; +} + +fn linuxOrderBySiblings(allocator: std.mem.Allocator, io: std.Io, cpus: []usize) !usize { + const leaders = try allocator.alloc(usize, cpus.len); + defer allocator.free(leaders); + for (cpus, leaders) |cpu, *leader| leader.* = try linuxCoreLeader(io, cpu); + return orderBySiblings(allocator, cpus, leaders); +} + +/// Returns the lowest CPU id sharing a physical core with `cpu`. +fn linuxCoreLeader(io: std.Io, cpu: usize) !usize { + var cpulist_buf: [4096]u8 = undefined; + // `core_cpus_list` replaced `thread_siblings_list` in Linux 5.7. + const cpulist = linuxReadTopology(io, cpu, "core_cpus_list", &cpulist_buf) catch |err| switch (err) { + error.FileNotFound => try linuxReadTopology(io, cpu, "thread_siblings_list", &cpulist_buf), + else => return err, + }; + + var set: std.os.linux.cpu_set_t = @splat(0); + try parseCpuList(cpulist, &set); + for (set, 0..) |word, word_index| { + if (word != 0) return word_index * @bitSizeOf(usize) + @ctz(word); + } + return error.MissingCpuTopology; +} + +fn linuxReadTopology(io: std.Io, cpu: usize, name: []const u8, buf: []u8) ![]const u8 { + var path_buf: [96]u8 = undefined; + const path = try std.fmt.bufPrint(&path_buf, "/sys/devices/system/cpu/cpu{d}/topology/{s}", .{ cpu, name }); + return std.Io.Dir.cwd().readFile(io, path, buf); +} + +/// Reorders `cpus` so that each round holds at most one CPU per physical core, +/// in ascending order, before the next round of SMT siblings. `leaders[i]` +/// identifies the physical core of `cpus[i]`. Returns the number of physical +/// cores, which is also the size of the first round. +fn orderBySiblings(allocator: std.mem.Allocator, cpus: []usize, leaders: []const usize) !usize { + const Entry = struct { + round: usize, + cpu: usize, + + fn lessThan(_: void, lhs: @This(), rhs: @This()) bool { + return if (lhs.round != rhs.round) lhs.round < rhs.round else lhs.cpu < rhs.cpu; + } + }; + + const entries = try allocator.alloc(Entry, cpus.len); + defer allocator.free(entries); + + var physical_count: usize = 0; + for (entries, cpus, leaders, 0..) |*entry, cpu, leader, index| { + var round: usize = 0; + for (leaders[0..index]) |previous| round += @intFromBool(previous == leader); + entry.* = .{ .round = round, .cpu = cpu }; + physical_count += @intFromBool(round == 0); + } + + std.mem.sort(Entry, entries, {}, Entry.lessThan); + for (cpus, entries) |*cpu, entry| cpu.* = entry.cpu; + return physical_count; +} + +fn naturalOrder(allocator: std.mem.Allocator, cpu_count: usize) !AllowedCores { + const ids = try allocator.alloc(usize, cpu_count); + for (ids, 0..) |*id, index| id.* = index; + return .{ .ids = ids, .physical_count = ids.len }; +} + +test "CPU lists respect mask boundaries" { + var set: std.os.linux.cpu_set_t = @splat(0); + try parseCpuList("0-2,64,66-67\n", &set); + try std.testing.expectEqual(@as(usize, 0b111), set[0]); + try std.testing.expectEqual(@as(usize, 0b1101), set[1]); + const capacity = @bitSizeOf(std.os.linux.cpu_set_t); + var text: [32]u8 = undefined; + try parseCpuList(try std.fmt.bufPrint(&text, "{d}", .{capacity - 1}), &set); + try std.testing.expect((set[set.len - 1] & (@as(usize, 1) << (@bitSizeOf(usize) - 1))) != 0); + try std.testing.expectError(error.InvalidCpuList, parseCpuList(try std.fmt.bufPrint(&text, "{d}", .{capacity}), &set)); + try std.testing.expectError(error.InvalidCpuList, parseCpuList("3-1", &set)); + try std.testing.expectError(error.InvalidCpuList, parseCpuList("1-2-3", &set)); +} + +const AffinityTest = struct { + const CpuSet = std.os.linux.cpu_set_t; + + fn available(count: usize) !CpuSet { + if (builtin.os.tag != .linux) return error.SkipZigTest; + const current = try threadAffinity(0); + var selected: CpuSet = @splat(0); + var remaining = count; + for (0..@bitSizeOf(CpuSet)) |core_id| { + if (!cpuSetContains(current, core_id)) continue; + selected[core_id / @bitSizeOf(usize)] |= @as(usize, 1) << @intCast(core_id % @bitSizeOf(usize)); + remaining -= 1; + if (remaining == 0) break; + } + if (remaining != 0) return error.SkipZigTest; + return selected; + } + + fn check(caller: CpuSet, worker: CpuSet, node: CpuSet, expected: CpuSet) !void { + const io = std.testing.io; + const Worker = struct { + io: std.Io, + set: CpuSet, + tid: std.os.linux.pid_t = undefined, + ready: std.Io.Event = .unset, + stop: std.Io.Event = .unset, + failure: ?anyerror = null, + + fn run(self: *@This()) void { + self.tid = @intCast(std.Thread.getCurrentId()); + setThreadAffinity(0, &self.set) catch |err| { + self.failure = err; + }; + self.ready.set(self.io); + self.stop.waitUncancelable(self.io); + } + }; + var state: Worker = .{ .io = io, .set = worker }; + const thread = try std.Thread.spawn(.{}, Worker.run, .{&state}); + defer { + state.stop.set(io); + thread.join(); + } + try state.ready.wait(io); + if (state.failure) |err| return err; + + // Restore every thread touched by the process-wide helper, including + // any test runner IO workers, before the next test executes. + var saved: std.ArrayList(struct { tid: std.os.linux.pid_t, set: CpuSet }) = .empty; + defer { + for (saved.items) |entry| setThreadAffinity(entry.tid, &entry.set) catch |err| switch (err) { + error.ThreadGone => {}, + error.SetAffinityFailed => @panic("failed to restore test thread affinity"), + }; + saved.deinit(std.testing.allocator); + } + var tasks = try std.Io.Dir.cwd().openDir(io, "/proc/self/task", .{ .iterate = true }); + defer tasks.close(io); + var it = tasks.iterate(); + while (try it.next(io)) |entry| { + const tid = std.fmt.parseInt(std.os.linux.pid_t, entry.name, 10) catch continue; + const set = threadAffinity(tid) catch |err| switch (err) { + error.ThreadGone => continue, + else => return err, + }; + try saved.append(std.testing.allocator, .{ .tid = tid, .set = set }); + } + + try setThreadAffinity(0, &caller); + _ = try pinProcessToCpuSet(io, node); + try std.testing.expectEqualSlices(usize, &expected, &(try threadAffinity(state.tid))); + const expected_caller = intersectCpuSets(caller, node); + try std.testing.expectEqualSlices(usize, &expected_caller, &(try threadAffinity(0))); + } +}; + +test "process NUMA affinity preserves a narrower worker binding" { + const caller = try AffinityTest.available(3); + const node = try AffinityTest.available(2); + const worker = try AffinityTest.available(1); + try AffinityTest.check(caller, worker, node, worker); +} + +test "process NUMA affinity visits workers when the caller is already pinned" { + const caller = try AffinityTest.available(1); + const worker = try AffinityTest.available(2); + try AffinityTest.check(caller, worker, worker, caller); +} + +test "process NUMA affinity leaves a worker with no overlapping CPUs alone" { + const caller = try AffinityTest.available(2); + const node = try AffinityTest.available(1); + var worker = caller; + for (&worker, node) |*word, excluded| word.* &= ~excluded; + try AffinityTest.check(caller, worker, node, worker); +} + +test "CPU lists accept whitespace and empty node CPU sets" { + var set: std.os.linux.cpu_set_t = @splat(0); + try parseCpuList(" \t\r\n", &set); + try std.testing.expect(std.mem.allEqual(usize, &set, 0)); + try parseCpuList(" \t0-2, 64,66-67\r\n", &set); + try std.testing.expectEqual(@as(usize, 0b111), set[0]); + try std.testing.expectEqual(@as(usize, 0b1101), set[1]); +} + +test "CPU lists reject malformed ranges" { + for ([_][]const u8{ "3-1", "x", "-1", "1-", "1-2-3", "1,,2", "1,", ",1", "999999999999999999999999999999" }) |text| { + var set: std.os.linux.cpu_set_t = @splat(0); + try std.testing.expectError(error.InvalidCpuList, parseCpuList(text, &set)); + } +} + +test "natural order treats every CPU as physical" { + const cores = try naturalOrder(std.testing.allocator, 4); + defer cores.deinit(std.testing.allocator); + + try std.testing.expectEqualSlices(usize, &.{ 0, 1, 2, 3 }, cores.ids); + try std.testing.expectEqualSlices(usize, &.{ 0, 1, 2, 3 }, cores.physical()); +} + +test "sibling order puts one CPU per physical core first" { + const Case = struct { cpus: []const usize, leaders: []const usize, expected: []const usize, physical_count: usize }; + const cases = [_]Case{ + // Siblings numbered after all cores (typical Intel). + .{ .cpus = &.{ 0, 1, 2, 3 }, .leaders = &.{ 0, 1, 0, 1 }, .expected = &.{ 0, 1, 2, 3 }, .physical_count = 2 }, + // Adjacent siblings (typical AMD, some ARM). + .{ .cpus = &.{ 0, 1, 2, 3 }, .leaders = &.{ 0, 0, 2, 2 }, .expected = &.{ 0, 2, 1, 3 }, .physical_count = 2 }, + // Affinity allows only some siblings of cores {0,4}, {1,5}, {2,6}. + .{ .cpus = &.{ 0, 1, 2, 5 }, .leaders = &.{ 0, 1, 2, 1 }, .expected = &.{ 0, 1, 2, 5 }, .physical_count = 3 }, + // Affinity allows only the second sibling of some cores. + .{ .cpus = &.{ 1, 4, 5 }, .leaders = &.{ 1, 0, 1 }, .expected = &.{ 1, 4, 5 }, .physical_count = 2 }, + // No SMT. + .{ .cpus = &.{ 0, 1, 2 }, .leaders = &.{ 0, 1, 2 }, .expected = &.{ 0, 1, 2 }, .physical_count = 3 }, + }; + + for (cases) |case| { + const cpus = try std.testing.allocator.dupe(usize, case.cpus); + defer std.testing.allocator.free(cpus); + + const physical_count = try orderBySiblings(std.testing.allocator, cpus, case.leaders); + try std.testing.expectEqualSlices(usize, case.expected, cpus); + try std.testing.expectEqual(case.physical_count, physical_count); + } +} + +test "allowed cores cover the thread affinity" { + const cores = try allowedCores(std.testing.allocator, std.testing.io); + defer cores.deinit(std.testing.allocator); + + try std.testing.expect(cores.physical_count > 0 and cores.physical_count <= cores.ids.len); + if (builtin.os.tag != .linux) return; + + const affinity = try threadAffinity(0); + var expected_count: usize = 0; + for (affinity) |word| expected_count += @popCount(word); + try std.testing.expectEqual(expected_count, cores.ids.len); + for (cores.ids) |cpu| try std.testing.expect(cpuSetContains(affinity, cpu)); +} diff --git a/stdx/crypto.zig b/stdx/crypto.zig new file mode 100644 index 00000000..eb5729b6 --- /dev/null +++ b/stdx/crypto.zig @@ -0,0 +1,41 @@ +const std = @import("std"); + +pub fn MacWriter(comptime T: type) type { + return struct { + const Self = @This(); + + mac: *T, + interface: std.Io.Writer, + + pub fn init(mac_: *T) MacWriter(T) { + return .{ + .mac = mac_, + .interface = .{ + .buffer = &.{}, + .vtable = &.{ + .drain = drain, + }, + }, + }; + } + + pub fn drain(w: *std.Io.Writer, data: []const []const u8, splat: usize) std.Io.Writer.Error!usize { + const self: *Self = @alignCast(@fieldParentPtr("interface", w)); + var total: usize = 0; + for (data) |chunk| { + self.mac.update(chunk); + total += chunk.len; + } + const last = data[data.len - 1]; + for (0..splat - 1) |_| { + self.mac.update(last); + total += last.len; + } + return total; + } + }; +} + +pub fn hmacWriter(hmac: anytype) MacWriter(std.meta.Child(@TypeOf(hmac))) { + return .init(hmac); +} diff --git a/stdx/debug.zig b/stdx/debug.zig new file mode 100644 index 00000000..a217eca1 --- /dev/null +++ b/stdx/debug.zig @@ -0,0 +1,34 @@ +const std = @import("std"); + +pub inline fn guard(check: bool, src: std.builtin.SourceLocation) void { + assert(check, "Invalid inputs {s}@{s}:{d}", .{ src.file, src.fn_name, src.line }); +} + +pub inline fn internalAssert(check: bool, comptime msg: []const u8, args: anytype) void { + assert(check, "internal error: " ++ msg, args); +} + +pub inline fn assert(check: bool, comptime msg: []const u8, args: anytype) void { + if (!check) { + @branchHint(.cold); + panic(msg, args); + } +} + +pub fn panic(comptime format: []const u8, args: anytype) noreturn { + std.debug.panic(format, args); +} + +pub inline fn compileLog(comptime msg: []const u8, comptime args: anytype) void { + @compileLog(std.fmt.comptimePrint(msg, args)); +} + +pub inline fn compileError(comptime msg: []const u8, comptime args: anytype) noreturn { + @compileError(std.fmt.comptimePrint(msg, args)); +} + +pub inline fn assertComptime(comptime check: bool, comptime msg: []const u8, comptime args: anytype) void { + if (check == false) { + compileError(msg, args); + } +} diff --git a/examples/third_party/tigerbeetle/flags.zig b/stdx/flags.zig similarity index 59% rename from examples/third_party/tigerbeetle/flags.zig rename to stdx/flags.zig index f4bc8d40..d0c4352d 100644 --- a/examples/third_party/tigerbeetle/flags.zig +++ b/stdx/flags.zig @@ -41,14 +41,15 @@ //! caller to manage the lifetime. The caller should be skipping program name. const std = @import("std"); -const builtin = @import("builtin"); const assert = std.debug.assert; +const builtin = @import("builtin"); + +const debug = @import("debug.zig"); /// Format and print an error message to stderr, then exit with an exit code of 1. pub fn fatal(comptime fmt_string: []const u8, args: anytype) noreturn { - const stderr = std.io.getStdErr().writer(); - stderr.print("error: " ++ fmt_string ++ "\n", args) catch {}; - std.posix.exit(1); + std.debug.print("error: " ++ fmt_string ++ "\n", args); + std.process.exit(1); } /// Parse CLI arguments for subcommands specified as Zig `struct` or `union(enum)`: @@ -74,19 +75,27 @@ pub fn fatal(comptime fmt_string: []const u8, args: anytype) noreturn { /// `positional` field is treated specially, it designates positional arguments. /// /// If `pub const help` declaration is present, it is used to implement `-h/--help` argument. -pub fn parse(args: *std.process.ArgIterator, comptime CliArgs: type) CliArgs { - assert(args.skip()); // Discard executable name. +pub fn parse(args: std.process.Args, comptime CliArgs: type) CliArgs { + var it = args.iterate(); + assert(it.skip()); // Discard executable name. return switch (@typeInfo(CliArgs)) { - .Union => parse_commands(args, CliArgs), - .Struct => parse_flags(args, CliArgs), + .@"union" => parse_commands(&it, CliArgs), + .@"struct" => parse_flags(&it, CliArgs), else => unreachable, }; } -fn parse_commands(args: *std.process.ArgIterator, comptime Commands: type) Commands { - comptime assert(@typeInfo(Commands) == .Union); - comptime assert(std.meta.fields(Commands).len >= 2); +/// Parse CLI arguments for current process. +/// See `stdx.flags.parse` documentation for more. +pub fn parseProcessArgs(init_: std.process.Init.Minimal, comptime CliArgs: type) CliArgs { + return parse(init_.args, CliArgs); +} + +fn parse_commands(args: *std.process.Args.Iterator, comptime Commands: type) Commands { + comptime assert(@typeInfo(Commands) == .@"union"); + const field_names = std.meta.fieldNames(Commands); + comptime assert(field_names.len >= 2); const first_arg = args.next() orelse fatal( "subcommand required, expected {s}", @@ -96,21 +105,21 @@ fn parse_commands(args: *std.process.ArgIterator, comptime Commands: type) Comma // NB: help must be declared as *pub* const to be visible here. if (@hasDecl(Commands, "help")) { if (std.mem.eql(u8, first_arg, "-h") or std.mem.eql(u8, first_arg, "--help")) { - std.io.getStdOut().writeAll(Commands.help) catch std.posix.exit(1); - std.posix.exit(0); + std.debug.print(Commands.help, .{}); + std.process.exit(0); } } - inline for (comptime std.meta.fields(Commands)) |field| { - comptime assert(std.mem.indexOf(u8, field.name, "_") == null); - if (std.mem.eql(u8, first_arg, field.name)) { - return @unionInit(Commands, field.name, parse_flags(args, field.type)); + inline for (field_names, std.meta.fieldTypes(Commands)) |field_name, FieldType| { + comptime assert(std.mem.indexOf(u8, field_name, "_") == null); + if (std.mem.eql(u8, first_arg, field_name)) { + return @unionInit(Commands, field_name, parse_flags(args, FieldType)); } } fatal("unknown subcommand: '{s}'", .{first_arg}); } -fn parse_flags(args: *std.process.ArgIterator, comptime Flags: type) Flags { +fn parse_flags(args: *std.process.Args.Iterator, comptime Flags: type) Flags { @setEvalBranchQuota(5_000); if (Flags == void) { @@ -120,85 +129,96 @@ fn parse_flags(args: *std.process.ArgIterator, comptime Flags: type) Flags { return {}; } - assert(@typeInfo(Flags) == .Struct); + const flags_info = @typeInfo(Flags).@"struct"; - comptime var fields: [std.meta.fields(Flags).len]std.builtin.Type.StructField = undefined; + const total_num_fields = flags_info.field_names.len; + comptime var field_names: [total_num_fields][]const u8 = undefined; + comptime var field_types: [total_num_fields]type = undefined; + comptime var field_defaults: [total_num_fields]?*const anyopaque = undefined; comptime var field_count = 0; - comptime var positional_fields: []const std.builtin.Type.StructField = &.{}; + const positional_infos: ?std.lang.Type.Struct = if (@hasField(Flags, "positional")) + @typeInfo(@FieldType(Flags, "positional")).@"struct" + else + null; - comptime for (std.meta.fields(Flags)) |field| { - if (std.mem.eql(u8, field.name, "positional")) { - assert(@typeInfo(field.type) == .Struct); - positional_fields = std.meta.fields(field.type); + comptime for (flags_info.field_names, flags_info.field_types, flags_info.field_attrs) |field_name, FieldType, field_attr| { + if (std.mem.eql(u8, field_name, "positional")) { var optional_tail = false; - for (positional_fields) |positional_field| { - if (default_value(positional_field) == null) { + for (positional_infos.?.field_types, positional_infos.?.field_attrs) |PosType, pos_attr| { + const default: ?PosType = pos_attr.defaultValue(PosType); + if (default == null) { if (optional_tail) @panic("optional positional arguments must be last"); } else { optional_tail = true; } - switch (@typeInfo(positional_field.type)) { - .Optional => |optional| { + switch (@typeInfo(PosType)) { + .optional => |optional| { // optional flags should have a default - assert(default_value(positional_field) != null); - assert(default_value(positional_field).? == null); + assert(default != null); + assert(default.? == null); assert_valid_value_type(optional.child); }, else => { - assert_valid_value_type(positional_field.type); + assert_valid_value_type(PosType); }, } } } else { - fields[field_count] = field; + field_names[field_count] = field_name; + field_types[field_count] = FieldType; + field_defaults[field_count] = field_attr.default_value_ptr; field_count += 1; - switch (@typeInfo(field.type)) { - .Bool => { - // boolean flags should have a default - assert(default_value(field) != null); - assert(default_value(field).? == false); - }, - .Optional => |optional| { - // optional flags should have a default - assert(default_value(field) != null); - assert(default_value(field).? == null); + const default: ?FieldType = field_attr.defaultValue(FieldType); + switch (@typeInfo(FieldType)) { + .bool => { + // boolean flags should have a default + debug.assertComptime(default != null and default.? == false, "boolean flag --{s} should default to false", .{field_name}); + }, + .optional => |optional| { + // optional flags should have a default + debug.assertComptime(default != null and default.? == null, "optional flag --{s} should have a null default value", .{field_name}); assert_valid_value_type(optional.child); }, else => { - assert_valid_value_type(field.type); + assert_valid_value_type(FieldType); }, } } }; var result: Flags = undefined; - // Would use std.enums.EnumFieldStruct(Flags, u32, 0) here but Flags is a Struct not an Enum. + // Would use std.enums.EnumFieldStruct(Flags, u32, 0) here but Flags is a struct not an Enum. var counts = comptime blk: { - var count_fields = std.meta.fields(Flags)[0..std.meta.fields(Flags).len].*; - for (&count_fields) |*field| { - field.type = u32; - field.alignment = @alignOf(u32); - field.default_value = @ptrCast(&@as(u32, 0)); - } - break :blk @Type(.{ .Struct = .{ - .layout = .auto, - .fields = &count_fields, - .decls = &.{}, - .is_tuple = false, - } }){}; + const count_field_types: [total_num_fields]type = @splat(u32); + const count_field_attrs: [total_num_fields]std.builtin.Type.Struct.FieldAttributes = @splat(.{ + .@"align" = @alignOf(u32), + .default_value_ptr = @ptrCast(&@as(u32, 0)), + }); + break :blk @Struct(.auto, null, flags_info.field_names, &count_field_types, &count_field_attrs){}; }; // When parsing arguments, we must consider longer arguments first, such that `--foo-bar=92` is // not confused for a misspelled `--foo=92`. Using `std.sort` for comptime-only values does not // work, so open-code insertion sort, and comptime assert order during the actual parsing. comptime { - for (fields[0..field_count], 0..) |*field_right, i| { - for (fields[0..i]) |*field_left| { - if (field_left.name.len < field_right.name.len) { - std.mem.swap(std.builtin.Type.StructField, field_left, field_right); + for ( + 0.., + field_names[0..field_count], + field_types[0..field_count], + field_defaults[0..field_count], + ) |i, *name_right, *type_right, *default_right| { + for ( + field_names[0..i], + field_types[0..i], + field_defaults[0..i], + ) |*name_left, *type_left, *default_left| { + if (name_left.len < name_right.len) { + std.mem.swap([]const u8, name_left, name_right); + std.mem.swap(type, type_left, type_right); + std.mem.swap(?*const anyopaque, default_left, default_right); } } } @@ -206,30 +226,38 @@ fn parse_flags(args: *std.process.ArgIterator, comptime Flags: type) Flags { var parsed_positional = false; next_arg: while (args.next()) |arg| { - comptime var field_len_prev = std.math.maxInt(usize); - inline for (fields[0..field_count]) |field| { - const flag = comptime flag_name(field); + // NB: help must be declared as *pub* const to be visible here. + if (@hasDecl(Flags, "help")) { + if (std.mem.eql(u8, arg, "-h") or std.mem.eql(u8, arg, "--help")) { + std.debug.print(Flags.help, .{}); + std.process.exit(0); + } + } - comptime assert(field_len_prev >= field.name.len); - field_len_prev = field.name.len; + comptime var field_len_prev = std.math.maxInt(usize); + inline for (field_names[0..field_count], field_types[0..field_count]) |field_name, FieldType| { + const flag = comptime flag_name(field_name); + + comptime assert(field_len_prev >= field_name.len); + field_len_prev = field_name.len; if (std.mem.startsWith(u8, arg, flag)) { if (parsed_positional) { fatal("unexpected trailing option: '{s}'", .{arg}); } - @field(counts, field.name) += 1; - const flag_value = parse_flag(field.type, flag, arg); - @field(result, field.name) = flag_value; + @field(counts, field_name) += 1; + const flag_value = parse_flag(FieldType, flag, arg); + @field(result, field_name) = flag_value; continue :next_arg; } } - if (@hasField(Flags, "positional")) { + if (positional_infos) |pos_infos| { counts.positional += 1; switch (counts.positional - 1) { - inline 0...positional_fields.len - 1 => |positional_index| { - const positional_field = positional_fields[positional_index]; - const flag = comptime flag_name_positional(positional_field); + inline 0...pos_infos.field_names.len - 1 => |positional_index| { + const positional_field_name = pos_infos.field_names[positional_index]; + const flag = comptime flag_name_positional(positional_field_name); if (arg.len == 0) fatal("{s}: empty argument", .{flag}); // Prevent ambiguity between a flag and positional argument value. We could add @@ -238,8 +266,8 @@ fn parse_flags(args: *std.process.ArgIterator, comptime Flags: type) Flags { if (arg[0] == '-') fatal("unexpected argument: '{s}'", .{arg}); parsed_positional = true; - @field(result.positional, positional_field.name) = - parse_value(positional_field.type, flag, arg); + @field(result.positional, positional_field_name) = + parse_value(pos_infos.field_types[positional_index], flag, arg); continue :next_arg; }, else => {}, // Fall-through to the unexpected argument error. @@ -249,11 +277,15 @@ fn parse_flags(args: *std.process.ArgIterator, comptime Flags: type) Flags { fatal("unexpected argument: '{s}'", .{arg}); } - inline for (fields[0..field_count]) |field| { - const flag = flag_name(field); - switch (@field(counts, field.name)) { - 0 => if (default_value(field)) |default| { - @field(result, field.name) = default; + inline for ( + field_names[0..field_count], + field_types[0..field_count], + field_defaults[0..field_count], + ) |field_name, FieldType, default_ptr| { + const flag = flag_name(field_name); + switch (@field(counts, field_name)) { + 0 => if (defaultValue(FieldType, default_ptr)) |default| { + @field(result, field_name) = default; } else { fatal("{s}: argument is required", .{flag}); }, @@ -262,15 +294,19 @@ fn parse_flags(args: *std.process.ArgIterator, comptime Flags: type) Flags { } } - if (@hasField(Flags, "positional")) { - assert(counts.positional <= positional_fields.len); - inline for (positional_fields, 0..) |positional_field, positional_index| { + if (positional_infos) |pos_infos| { + assert(counts.positional <= pos_infos.field_names.len); + inline for ( + 0.., + pos_infos.field_names, + pos_infos.field_types, + pos_infos.field_attrs, + ) |positional_index, name, PosFieldType, attr| { if (positional_index >= counts.positional) { - const flag = comptime flag_name_positional(positional_field); - if (default_value(positional_field)) |default| { - @field(result.positional, positional_field.name) = default; + if (attr.defaultValue(PosFieldType)) |default| { + @field(result.positional, name) = default; } else { - fatal("{s}: argument is required", .{flag}); + fatal("<{s}>: argument is required", .{name}); } } } @@ -281,12 +317,12 @@ fn parse_flags(args: *std.process.ArgIterator, comptime Flags: type) Flags { fn assert_valid_value_type(comptime T: type) void { comptime { - if (T == []const u8 or T == [:0]const u8 or T == ByteSize or @typeInfo(T) == .Int) return; + if (T == []const u8 or T == [:0]const u8 or T == ByteSize or @typeInfo(T) == .int or @typeInfo(T) == .float) return; - if (@typeInfo(T) == .Enum) { - const info = @typeInfo(T).Enum; - assert(info.is_exhaustive); - assert(info.fields.len >= 2); + if (@typeInfo(T) == .@"enum") { + const info = @typeInfo(T).@"enum"; + assert(info.mode == .exhaustive); + assert(info.field_names.len >= 2); return; } @@ -336,14 +372,15 @@ fn parse_value(comptime T: type, flag: []const u8, value: [:0]const u8) T { assert(value.len > 0); const V = switch (@typeInfo(T)) { - .Optional => |optional| optional.child, + .optional => |optional| optional.child, else => T, }; if (V == []const u8 or V == [:0]const u8) return value; if (V == ByteSize) return parse_value_size(flag, value); - if (@typeInfo(V) == .Int) return parse_value_int(V, flag, value); - if (@typeInfo(V) == .Enum) return parse_value_enum(V, flag, value); + if (@typeInfo(V) == .int) return parse_value_int(V, flag, value); + if (@typeInfo(V) == .float) return parse_value_float(V, flag, value); + if (@typeInfo(V) == .@"enum") return parse_value_enum(V, flag, value); comptime unreachable; } @@ -434,7 +471,7 @@ pub const ByteSize = struct { else ByteUnit.bytes; - _ = std.math.mul(u64, amount, @intFromEnum(unit)) catch { + _ = std.math.mul(u64, amount, @backingInt(unit)) catch { return ByteSizeParseError.BytesOverflow; }; @@ -445,7 +482,7 @@ pub const ByteSize = struct { return std.math.mul( u64, size.value, - @intFromEnum(size.unit), + @backingInt(size.unit), ) catch unreachable; } @@ -501,7 +538,7 @@ fn parse_value_int(comptime T: type, flag: []const u8, value: [:0]const u8) T { switch (err) { error.Overflow => fatal( "{s}: value exceeds {d}-bit {s} integer: '{s}'", - .{ flag, @typeInfo(T).Int.bits, @tagName(@typeInfo(T).Int.signedness), value }, + .{ flag, @typeInfo(T).int.bits, @tagName(@typeInfo(T).int.signedness), value }, ), error.InvalidCharacter => fatal( "{s}: expected an integer value, but found '{s}' (invalid digit)", @@ -511,9 +548,23 @@ fn parse_value_int(comptime T: type, flag: []const u8, value: [:0]const u8) T { }; } +/// Parse string value into a float, providing a nice error message for the user. +fn parse_value_float(comptime T: type, flag: []const u8, value: [:0]const u8) T { + assert((flag[0] == '-' and flag[1] == '-') or flag[0] == '<'); + + return std.fmt.parseFloat(T, value) catch |err| { + switch (err) { + error.InvalidCharacter => fatal( + "{s}: expected a decimal value, but found '{s}' (invalid character)", + .{ flag, value }, + ), + } + }; +} + fn parse_value_enum(comptime E: type, flag: []const u8, value: [:0]const u8) E { assert((flag[0] == '-' and flag[1] == '-') or flag[0] == '<'); - comptime assert(@typeInfo(E).Enum.is_exhaustive); + comptime assert(@typeInfo(E).@"enum".mode == .exhaustive); return std.meta.stringToEnum(E, value) orelse fatal( "{s}: expected one of {s}, but found '{s}'", @@ -523,139 +574,44 @@ fn parse_value_enum(comptime E: type, flag: []const u8, value: [:0]const u8) E { fn fields_to_comma_list(comptime E: type) []const u8 { comptime { - const field_count = std.meta.fields(E).len; + const field_names = std.meta.fieldNames(E); + const field_count = field_names.len; assert(field_count >= 2); var result: []const u8 = ""; - for (std.meta.fields(E), 0..) |field, field_index| { + for (field_names, 0..) |field_name, field_index| { const separator = switch (field_index) { 0 => "", else => ", ", field_count - 1 => if (field_count == 2) " or " else ", or ", }; - result = result ++ separator ++ "'" ++ field.name ++ "'"; + result = result ++ separator ++ "'" ++ field_name ++ "'"; } return result; } } -pub fn flag_name(comptime field: std.builtin.Type.StructField) []const u8 { - // TODO(Zig): Cleanup when this is fixed after Zig 0.11. - // Without comptime blk, the compiler thinks the result is a runtime slice returning a UAF. - return comptime blk: { - assert(!std.mem.eql(u8, field.name, "positional")); - - var result: []const u8 = "--"; - var index = 0; - while (std.mem.indexOf(u8, field.name[index..], "_")) |i| { - result = result ++ field.name[index..][0..i] ++ "-"; - index = index + i + 1; - } - result = result ++ field.name[index..]; - break :blk result; - }; +/// Maps a field name to its CLI flag, e.g. `model_path` to `--model-path`. +pub fn flag_name(comptime field_name: []const u8) []const u8 { + comptime assert(!std.mem.eql(u8, field_name, "positional")); + comptime var name = field_name[0..field_name.len].*; + comptime std.mem.replaceScalar(u8, &name, '_', '-'); + const dashed = name; + return "--" ++ dashed; } test flag_name { - const field = @typeInfo(struct { statsd: bool }).Struct.fields[0]; - try std.testing.expectEqualStrings(flag_name(field), "--statsd"); + try std.testing.expectEqualStrings("--statsd", flag_name("statsd")); + try std.testing.expectEqualStrings("--model-path", flag_name("model_path")); + try std.testing.expectEqualStrings("--a-b-c", flag_name("a_b_c")); } -fn flag_name_positional(comptime field: std.builtin.Type.StructField) []const u8 { - comptime assert(std.mem.indexOf(u8, field.name, "_") == null); - return "<" ++ field.name ++ ">"; +fn flag_name_positional(comptime field_name: []const u8) []const u8 { + comptime assert(std.mem.indexOf(u8, field_name, "_") == null); + return "<" ++ field_name ++ ">"; } -/// This is essentially `field.default_value`, but with a useful type instead of `?*anyopaque`. -pub fn default_value(comptime field: std.builtin.Type.StructField) ?field.type { - return if (field.default_value) |default_opaque| - @as(*const field.type, @ptrCast(@alignCast(default_opaque))).* - else - null; +pub fn defaultValue(comptime FieldType: type, default_value_ptr: ?*const anyopaque) ?FieldType { + const dp: *const FieldType = @ptrCast(@alignCast(default_value_ptr orelse return null)); + return dp.*; } - -// CLI parsing makes a liberal use of `fatal`, so testing it within the process is impossible. We -// test it out of process by: -// - using Zig compiler to build this very file as an executable in a temporary directory, -// - running the following main with various args and capturing stdout, stderr, and the exit code. -// - asserting that the captured values are correct. -pub const main = if (@import("root") != @This()) - // For production builds, don't include the main function. - // This is `if __name__ == "__main__":` at comptime! -{} else struct { - const CliArgs = union(enum) { - empty, - prefix: struct { - foo: u8 = 0, - foo_bar: u8 = 0, - opt: bool = false, - option: bool = false, - }, - pos: struct { flag: bool = false, positional: struct { - p1: []const u8, - p2: []const u8, - p3: ?u32 = null, - p4: ?u32 = null, - } }, - required: struct { - foo: u8, - bar: u8, - }, - values: struct { - int: u32 = 0, - size: ByteSize = .{ .value = 0 }, - boolean: bool = false, - path: []const u8 = "not-set", - optional: ?[]const u8 = null, - choice: enum { marlowe, shakespeare } = .marlowe, - }, - - pub const help = - \\ flags-test-program [flags] - \\ - ; - }; - - pub fn main() !void { - var gpa_allocator = std.heap.GeneralPurposeAllocator(.{}){}; - const gpa = gpa_allocator.allocator(); - - var args = try std.process.argsWithAllocator(gpa); - defer args.deinit(); - - const cli_args = parse(&args, CliArgs); - - const stdout = std.io.getStdOut(); - const out_stream = stdout.writer(); - switch (cli_args) { - .empty => try out_stream.print("empty\n", .{}), - .prefix => |values| { - try out_stream.print("foo: {}\n", .{values.foo}); - try out_stream.print("foo-bar: {}\n", .{values.foo_bar}); - try out_stream.print("opt: {}\n", .{values.opt}); - try out_stream.print("option: {}\n", .{values.option}); - }, - .pos => |values| { - try out_stream.print("p1: {s}\n", .{values.positional.p1}); - try out_stream.print("p2: {s}\n", .{values.positional.p2}); - try out_stream.print("p3: {?}\n", .{values.positional.p3}); - try out_stream.print("p4: {?}\n", .{values.positional.p4}); - try out_stream.print("flag: {}\n", .{values.flag}); - }, - .required => |required| { - try out_stream.print("foo: {}\n", .{required.foo}); - try out_stream.print("bar: {}\n", .{required.bar}); - }, - .values => |values| { - try out_stream.print("int: {}\n", .{values.int}); - try out_stream.print("size: {}\n", .{values.size.bytes()}); - try out_stream.print("boolean: {}\n", .{values.boolean}); - try out_stream.print("path: {s}\n", .{values.path}); - try out_stream.print("optional: {?s}\n", .{values.optional}); - try out_stream.print("choice: {?s}\n", .{@tagName(values.choice)}); - }, - } - } -}.main; - -// Note: I removed tests to not pull too many deps diff --git a/stdx/fmt.zig b/stdx/fmt.zig new file mode 100644 index 00000000..a62d4f7d --- /dev/null +++ b/stdx/fmt.zig @@ -0,0 +1,239 @@ +const std = @import("std"); + +/// Properly format a slice of numbers. +pub fn slice(any_slice: anytype) FmtSlice(std.meta.Elem(@TypeOf(any_slice))) { + return .{ .slice = any_slice }; +} + +/// Properly format a slice of []const u8. +pub fn strings(strs: []const []const u8) FmtStrings { + return .{ .strings = strs }; +} + +/// Properly format a slice of [*:0]const u8. +pub fn stringsZ(strs: []const [*:0]const u8) FmtZeroStrings { + return .{ .strings = strs }; +} + +fn FmtSlice(T: type) type { + return struct { + slice: []const T, + + pub fn format(f: @This(), writer: *std.Io.Writer) std.Io.Writer.Error!void { + return try formatSliceAny(f.slice, .{}, 1, writer); + } + + pub fn formatNumber(f: @This(), writer: *std.Io.Writer, n: std.fmt.Number) std.Io.Writer.Error!void { + return switch (@typeInfo(T)) { + .comptime_float, .float => try formatFloatSlice(f.slice, n, 1, writer), + .comptime_int, .int => try formatIntSlice(f.slice, n, 1, writer), + .bool => try formatBoolSlice(f.slice, n, 1, writer), + .@"struct" => if (@hasField(T, "re") and @hasField(T, "im")) { + try formatComplexSlice(f.slice, n, 1, writer); + } else if (@hasDecl(T, "toF32")) { + try formatFloatSlice(f.slice, n, 1, writer); + } else { + try formatSliceAny(f.slice, n, 1, writer); + }, + else => @compileError("FmtSlice doesn't support type: " ++ @typeName(T)), + }; + } + }; +} + +pub const FmtStrings = struct { + strings: []const []const u8, + + pub fn format(f: FmtStrings, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return try formatSliceCustom(formatStr, f.strings, .{}, 1, writer); + } + + pub fn formatNumber(f: FmtStrings, writer: *std.Io.Writer, n: std.fmt.Number) std.Io.Writer.Error!void { + return try formatIntSlice(f.strings, n, 1, writer); + } +}; + +pub const FmtZeroStrings = struct { + strings: []const [*:0]const u8, + + pub fn format(f: FmtZeroStrings, writer: *std.Io.Writer) std.Io.Writer.Error!void { + return try formatSliceCustom(formatStrZ, f.strings, .{}, 1, writer); + } +}; + +pub fn formatFloat(value: anytype, spec: std.fmt.Number, writer: *std.Io.Writer) !void { + const x = switch (@typeInfo(@TypeOf(value))) { + .@"struct" => value.toF32(), + .float => value, + else => @compileError("formatFloat expects a float, got: " ++ @typeName(@TypeOf(value))), + }; + switch (@typeInfo(@TypeOf(x))) { + .float => try writer.printFloat(x, spec), + // Handle F4.toF32() being a [2]f32 + .array => inline for (0.., x) |i, xi| { + try writer.printFloat(xi, spec); + if (i < x.len - 1) try writer.writeByte(','); + }, + else => @compileError("formatFloat expects a float, got: " ++ @typeName(@TypeOf(value))), + } +} + +pub fn formatInt(value: anytype, spec: std.fmt.Number, writer: *std.Io.Writer) !void { + return switch (@typeInfo(@TypeOf(value))) { + .int => try writer.printInt(value, spec.mode.base().?, spec.case, .{ .alignment = spec.alignment, .fill = spec.fill }), + .vector => { + const vector = @typeInfo(@TypeOf(value)).vector; + const array: [vector.len]vector.child = value; + for (&array, 0..) |elem, i| { + try writer.printInt(elem, spec.mode.base().?, spec.case, .{ .alignment = spec.alignment, .fill = spec.fill }); + if (i < array.len - 1) { + try writer.writeAll(", "); + } + } + }, + else => @compileError("formatInt expects an int, got: " ++ @typeName(@TypeOf(value))), + }; +} + +pub fn formatComplex(value: anytype, spec: std.fmt.Number, writer: *std.Io.Writer) !void { + try writer.writeAll(".{.re="); + try writer.printFloat(value.re, spec); + try writer.writeAll(", .im="); + try writer.printFloat(value.im, spec); + try writer.writeAll("}"); +} + +pub fn formatBool(value: bool, spec: std.fmt.Number, writer: *std.Io.Writer) !void { + try writer.alignBufferOptions(if (value) "1" else "0", .{ .alignment = spec.alignment, .fill = spec.fill }); +} + +pub fn formatStr(value: []const u8, _: std.fmt.Number, writer: *std.Io.Writer) !void { + try writer.writeAll(value); +} + +pub fn formatStrZ(value: [*:0]const u8, _: std.fmt.Number, writer: *std.Io.Writer) !void { + try writer.writeAll(std.mem.span(value)); +} + +pub fn formatAny(value: anytype, spec: std.fmt.Number, writer: *std.Io.Writer) !void { + var buf: [48]u8 = undefined; + const T = @TypeOf(value); + const fmt = switch (@typeInfo(T)) { + .@"struct", .@"union", .@"enum", .@"opaque" => if (@hasDecl(T, "formatNumber")) + "{d}" + else + "{f}", + else => "{f}", + }; + + const s = std.fmt.bufPrint(&buf, fmt, .{value}) catch blk: { + buf[45..].* = "...".*; + break :blk buf[0..]; + }; + return try writer.alignBufferOptions(s, .{ .alignment = spec.alignment, .fill = spec.fill }); +} + +pub fn formatSliceCustom(fmt_func: anytype, values: anytype, spec: std.fmt.Number, stride: i64, writer: *std.Io.Writer) !void { + // use the format "width" for the number of columns instead of individual width. + const num_cols: usize = spec.width orelse 12; + var my_options = spec; + my_options.width = null; + // TODO: handle negative strides + const strd: usize = @intCast(stride); + const n: usize = @divTrunc(values.len, strd); + + _ = try writer.write("{"); + if (n <= num_cols) { + for (0..n) |i| { + // Force inlining so that the switch and the buffer can be done once. + try @call(.always_inline, fmt_func, .{ values[i * strd], my_options, writer }); + if (i < n - 1) _ = try writer.write(","); + } + } else { + const half = @divFloor(num_cols, 2); + for (0..half) |i| { + try @call(.always_inline, fmt_func, .{ values[i * strd], my_options, writer }); + _ = try writer.write(","); + } + _ = try writer.write(" ..., "); + for (n - half..n) |i| { + try @call(.always_inline, fmt_func, .{ values[i * strd], my_options, writer }); + if (i < n - 1) _ = try writer.write(","); + } + } + _ = try writer.write("}"); +} + +pub fn formatSliceAny(values: anytype, spec: std.fmt.Number, stride: i64, writer: *std.Io.Writer) !void { + return try formatSliceCustom(formatAny, values, spec, stride, writer); +} + +pub fn formatFloatSlice(values: anytype, spec: std.fmt.Number, stride: i64, writer: *std.Io.Writer) !void { + return try formatSliceCustom(formatFloat, values, spec, stride, writer); +} + +pub fn formatIntSlice(values: anytype, spec: std.fmt.Number, stride: i64, writer: *std.Io.Writer) !void { + return try formatSliceCustom(formatInt, values, spec, stride, writer); +} + +pub fn formatComplexSlice(values: anytype, spec: std.fmt.Number, stride: i64, writer: *std.Io.Writer) !void { + return try formatSliceCustom(formatComplex, values, spec, stride, writer); +} + +pub fn formatBoolSlice(values: anytype, spec: std.fmt.Number, stride: i64, writer: *std.Io.Writer) !void { + return try formatSliceCustom(formatBool, values, spec, stride, writer); +} + +/// Format a struct using `format` method of subfields when possible. +pub fn any(any_val: anytype) FmtAny(@TypeOf(any_val)) { + return .{ .data = any_val }; +} + +fn FmtAny(Data: type) type { + return struct { + data: Data, + + pub inline fn format(self: @This(), writer: *std.Io.Writer) std.Io.Writer.Error!void { + try printValue(writer, .{}, self.data, std.options.fmt_max_depth); + } + }; +} + +/// Fix up of std.Io.Writer.printValue that uses `format` method of subfields when possible +fn printValue( + w: *std.Io.Writer, + options: std.fmt.Options, + value: anytype, + max_depth: usize, +) std.Io.Writer.Error!void { + const T = @TypeOf(value); + if (std.meta.hasMethod(T, "format")) { + return try value.format(w); + } + if (std.meta.hasMethod(T, "formatNumber")) { + return try value.formatNumber(w, options.toNumber(.decimal, .lower)); + } + + if (max_depth == 0) { + try w.writeAll(".{ ... }"); + return; + } + + switch (@typeInfo(T)) { + .@"struct" => |info| { + try w.writeAll(".{ "); + inline for (info.field_names, 0..) |field_name, i| { + if (i > 0) try w.writeAll(", "); + + if (!info.is_tuple) { + try w.writeByte('.'); + try w.writeAll(field_name); + try w.writeAll(" = "); + } + try printValue(w, options, @field(value, field_name), max_depth - 1); + } + try w.writeAll(" }"); + }, + inline else => try w.printValue("any", options, value, max_depth - 1), + } +} diff --git a/stdx/json.zig b/stdx/json.zig new file mode 100644 index 00000000..c1f71a4a --- /dev/null +++ b/stdx/json.zig @@ -0,0 +1,376 @@ +pub const std = @import("std"); +const ParseFromValueError = std.json.ParseFromValueError; + +/// Handle json fields that can have different Zig types depending on the message. +/// Each union field should have a unique Zig type. +/// +/// Example json: +/// +/// ```json +/// [ +/// { "question": "How old are you ?", "answer": 5 }, +/// { "question": "Count to three.", "answer": [1, 2, 3] }, +/// ] +/// ``` +/// +/// Corresponding Zig code: +/// +/// ```zig +/// const Answer = union { +/// number: i32, +/// numbers: []const i32, +/// +/// const Helpers = stdx.json.UnionHelpers(@This()); +/// pub const jsonParse = Helpers.jsonParse; +/// pub const jsonParseFromValue = Helpers.jsonParseFromValue; +/// pub const jsonStringify = Helpers.jsonStringify; +/// }; +/// +/// const Message = struct { +/// question: []const u8; +/// answer: Answer; +/// } +/// ``` +pub fn UnionHelpers(comptime T: type) type { + return struct { + pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) std.json.ParseError(@TypeOf(source.*))!T { + return jsonParseFromValue( + allocator, + try std.json.innerParse(std.json.Value, allocator, source, options), + options, + ); + } + + pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) ParseFromValueError!T { + inline for (comptime std.meta.fieldNames(T), comptime std.meta.fieldTypes(T)) |field_name, Field| { + switch (Field) { + bool => if (source == .bool) return @unionInit(T, field_name, source.bool), + []const u8 => switch (source) { + .string, .number_string => |v| { + const dupe = try allocator.dupe(u8, v); + return @unionInit(T, field_name, dupe); + }, + else => {}, + }, + else => switch (@typeInfo(Field)) { + .int => if (source == .integer) return @unionInit(T, field_name, @intCast(source.integer)), + .float => if (source == .float) return @unionInit(T, field_name, @floatCast(source.float)), + .@"struct" => if (source == .object) return @unionInit(T, field_name, try std.json.innerParseFromValue(Field, allocator, source, options)), + inline else => switch (source) { + .number_string, .array => return @unionInit(T, field_name, try std.json.innerParseFromValue(Field, allocator, source, options)), + else => {}, + }, + }, + } + } + return error.UnknownField; + } + + pub fn jsonStringify(self: T, jw: anytype) !void { + return switch (self) { + inline else => |v| jw.write(v), + }; + } + }; +} + +/// Deprecated: prefer importing `stdx.json.UnionHelpers` method into your union. +/// This will remove the nesting required by `stdx.json.Union`. +pub fn Union(comptime T: type) type { + return struct { + const Self = @This(); + + value: T, + + const Helpers = UnionHelpers(T); + + pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) std.json.ParseError(@TypeOf(source.*))!Self { + return .{ .value = try Helpers.jsonParse(allocator, source, options) }; + } + + pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) ParseFromValueError!Self { + return .{ .value = try Helpers.jsonParseFromValue(allocator, source, options) }; + } + + pub fn jsonStringify(self: Self, jw: anytype) !void { + return try Helpers.jsonStringify(self.value, jw); + } + }; +} + +/// Handle json fields that can have different Zig types depending on another field in the same message. +/// This is translated to a Zig tagged union. +/// +/// Example json: +/// +/// ```json +/// [ +/// { "type": "faq", "question": "How old are you ?", "answer": 5 }, +/// { "type": "address", "city": "NYC", "zipcode": "49130"}, +/// ] +/// ``` +/// +/// Corresponding Zig struct: +/// +/// ```zig +/// const Entry = union { +/// faq: struct { question: []const u8, answer: u32 }, +/// address: struct { city: []const u8, zipcode: []const u8 }, +/// +/// const Helpers = stdx.json.TaggedUnionHelpers(@This(), "type"); +/// pub const jsonParse = Helpers.jsonParse; +/// pub const jsonParseFromValue = Helpers.jsonParseFromValue; +/// pub const jsonStringify = Helpers.jsonStringify; +/// }; +/// +/// const Message = []const Entry; +/// ``` +pub fn TaggedUnionHelpers(comptime T: type, comptime tag_name: [:0]const u8) type { + comptime { + if (@typeInfo(T) != .@"union") { + @compileError("TaggedUnion expects a union type, found " ++ @typeName(T)); + } + for (std.meta.fieldNames(T), std.meta.fieldTypes(T)) |field_name, Field| { + if (@typeInfo(Field) != .@"struct") { + @compileError("TaggedUnion member '" ++ field_name ++ "' must be a struct, found " ++ @typeName(Field)); + } + } + } + + return struct { + pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) std.json.ParseError(@TypeOf(source.*))!T { + return jsonParseFromValue( + allocator, + try std.json.innerParse(std.json.Value, allocator, source, options), + options, + ); + } + + pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) ParseFromValueError!T { + errdefer std.log.warn("failed to parse: {} as {s}", .{ source, @typeName(T) }); + if (source != .object) return error.UnexpectedToken; + var o = source.object; + const tag = (o.fetchSwapRemove(tag_name) orelse return error.MissingField).value; + if (tag != .string) return error.LengthMismatch; + inline for (comptime std.meta.fieldNames(T), comptime std.meta.fieldTypes(T)) |field_name, Field| { + if (std.mem.eql(u8, field_name, tag.string)) { + const inner: Field = std.json.parseFromValueLeaky(Field, allocator, .{ .object = o }, options) catch |err| { + std.log.warn("failed to interpret {s} as a {s}: {}", .{ tag.string, @typeName(Field), err }); + return err; + }; + return @unionInit(T, field_name, inner); + } + } + return error.InvalidEnumTag; + } + + pub fn jsonStringify(self: T, jw: anytype) !void { + try jw.beginObject(); + switch (self) { + inline else => |v, tag| { + // emit a field containing the enum tag. + try jw.objectField(tag_name); + try jw.write(@tagName(tag)); + switch (@typeInfo(@TypeOf(v))) { + .@"struct" => inline for (comptime std.meta.fieldNames(@TypeOf(v))) |name| { + try jw.objectField(name); + try jw.write(@field(v, name)); + }, + else => unreachable, // Would have failed with comptime check. + } + }, + } + try jw.endObject(); + } + }; +} + +/// Deprecated: prefer importing `stdx.json.TaggedUnionHelpers` method into your union. +/// This will remove the nesting required by `stdx.json.TaggedUnion`. +pub fn TaggedUnion(comptime T: type, comptime tag_name: [:0]const u8) type { + return struct { + const Self = @This(); + + value: T, + + const Helpers = TaggedUnionHelpers(T, tag_name); + + pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) std.json.ParseError(@TypeOf(source.*))!Self { + return .{ .value = try Helpers.jsonParse(allocator, source, options) }; + } + + pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) ParseFromValueError!Self { + return .{ .value = try Helpers.jsonParseFromValue(allocator, source, options) }; + } + + pub fn jsonStringify(self: Self, jw: anytype) !void { + return try Helpers.jsonStringify(self.value, jw); + } + }; +} + +pub fn NeverNull(comptime T: type, comptime default_value: T) type { + return struct { + const Self = @This(); + + value: T = default_value, + + pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) !Self { + return .{ .value = (try std.json.innerParse(?T, allocator, source, options)) orelse default_value }; + } + + pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) !Self { + return .{ .value = (try std.json.innerParseFromValue(?T, allocator, source, options)) orelse default_value }; + } + + pub fn jsonStringify(self: Self, jw: anytype) !void { + try jw.write(self.value); + } + }; +} + +pub fn fillDefaultStructValues(comptime T: type, r: *T) !void { + const info = @typeInfo(T).@"struct"; + inline for (info.field_names, info.field_types, info.field_attrs) |field_name, Field, attrs| { + if (attrs.defaultValue(Field)) |default| { + if (@field(r, field_name) == null) { + @field(r, field_name) = default; + } + } + } +} + +test "union" { + const Answer = union(enum) { + number: i32, + numbers: []const i32, + flag: bool, + ratio: f64, + text: []const u8, + + const Helpers = UnionHelpers(@This()); + pub const jsonParse = Helpers.jsonParse; + pub const jsonParseFromValue = Helpers.jsonParseFromValue; + pub const jsonStringify = Helpers.jsonStringify; + }; + + const Message = struct { + question: []const u8, + answer: Answer, + }; + + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const allocator = arena.allocator(); + + const number_case = "{\"question\":\"How old are you?\",\"answer\":5}"; + const parsed_number = try std.json.parseFromSliceLeaky(Message, allocator, number_case, .{}); + try std.testing.expectEqualStrings("How old are you?", parsed_number.question); + try std.testing.expectEqual(@as(i32, 5), parsed_number.answer.number); + try std.testing.expectFmt(number_case, "{f}", .{std.json.fmt(parsed_number, .{})}); + + const numbers_case = "{\"question\":\"Count to three.\",\"answer\":[1,2,3]}"; + const parsed_numbers = try std.json.parseFromSliceLeaky(Message, allocator, numbers_case, .{}); + try std.testing.expectEqual(@as(usize, 3), parsed_numbers.answer.numbers.len); + try std.testing.expectEqual(@as(i32, 1), parsed_numbers.answer.numbers[0]); + try std.testing.expectEqual(@as(i32, 2), parsed_numbers.answer.numbers[1]); + try std.testing.expectEqual(@as(i32, 3), parsed_numbers.answer.numbers[2]); + try std.testing.expectFmt(numbers_case, "{f}", .{std.json.fmt(parsed_numbers, .{})}); + + const flag_case = "{\"question\":\"Are you ready?\",\"answer\":true}"; + const parsed_flag = try std.json.parseFromSliceLeaky(Message, allocator, flag_case, .{}); + try std.testing.expectEqual(true, parsed_flag.answer.flag); + try std.testing.expectFmt(flag_case, "{f}", .{std.json.fmt(parsed_flag, .{})}); + + const ratio_case = "{\"question\":\"What is pi?\",\"answer\":3.5}"; + const parsed_ratio = try std.json.parseFromSliceLeaky(Message, allocator, ratio_case, .{}); + try std.testing.expectEqual(@as(f64, 3.5), parsed_ratio.answer.ratio); + try std.testing.expectFmt(ratio_case, "{f}", .{std.json.fmt(parsed_ratio, .{})}); + + const text_case = "{\"question\":\"Your name?\",\"answer\":\"zml\"}"; + const parsed_text = try std.json.parseFromSliceLeaky(Message, allocator, text_case, .{}); + try std.testing.expectEqualStrings("zml", parsed_text.answer.text); + try std.testing.expectFmt(text_case, "{f}", .{std.json.fmt(parsed_text, .{})}); + + const null_value = try std.json.parseFromSliceLeaky(std.json.Value, allocator, "null", .{}); + try std.testing.expectError(error.UnknownField, Answer.jsonParseFromValue(allocator, null_value, .{})); +} + +test "strings in union can be freed" { + const Answer = union(enum) { + string: []const u8, + number: u32, + + const Helpers = UnionHelpers(@This()); + pub const jsonParse = Helpers.jsonParse; + pub const jsonParseFromValue = Helpers.jsonParseFromValue; + pub const jsonStringify = Helpers.jsonStringify; + }; + const Struct = struct { + answer1: Answer, + answer2: Answer, + }; + const struct_json = + \\{"answer1":"I'm 29","answer2": 1e29} + ; + var parsed_value = try std.json.parseFromSlice(std.json.Value, std.testing.allocator, struct_json, .{ .parse_numbers = false }); + defer parsed_value.deinit(); + + const parsed = try std.json.parseFromValueLeaky(Struct, std.testing.allocator, parsed_value.value, .{}); + + try std.testing.expect(parsed.answer1 == .string); + try std.testing.expectEqualStrings("I'm 29", parsed.answer1.string); + std.testing.allocator.free(parsed.answer1.string); + + try std.testing.expect(parsed.answer2 == .string); + try std.testing.expectEqualStrings("1e29", parsed.answer2.string); + std.testing.allocator.free(parsed.answer2.string); +} + +test "tagged union" { + const Message = union(enum) { + faq: struct { + question: []const u8, + answer: u32, + }, + address: struct { + city: []const u8, + zipcode: []const u8, + }, + + const Helpers = TaggedUnionHelpers(@This(), "type"); + pub const jsonParse = Helpers.jsonParse; + pub const jsonParseFromValue = Helpers.jsonParseFromValue; + pub const jsonStringify = Helpers.jsonStringify; + }; + + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const allocator = arena.allocator(); + + const faq_case = "{\"type\":\"faq\",\"question\":\"How old are you?\",\"answer\":5}"; + const parsed_faq = try std.json.parseFromSliceLeaky(Message, allocator, faq_case, .{}); + try std.testing.expectEqualStrings("How old are you?", parsed_faq.faq.question); + try std.testing.expectEqual(@as(u32, 5), parsed_faq.faq.answer); + try std.testing.expectFmt(faq_case, "{f}", .{std.json.fmt(parsed_faq, .{})}); + + const address_case = "{\"type\":\"address\",\"city\":\"NYC\",\"zipcode\":\"49130\"}"; + const parsed_address = try std.json.parseFromSliceLeaky(Message, allocator, address_case, .{}); + try std.testing.expectEqualStrings("NYC", parsed_address.address.city); + try std.testing.expectEqualStrings("49130", parsed_address.address.zipcode); + try std.testing.expectFmt(address_case, "{f}", .{std.json.fmt(parsed_address, .{})}); +} + +test "never null" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const allocator = arena.allocator(); + + const non_null = try std.json.parseFromSliceLeaky(NeverNull(i32, 10), allocator, "3", .{}); + try std.testing.expectEqual(@as(i32, 3), non_null.value); + try std.testing.expectFmt("3", "{f}", .{std.json.fmt(non_null, .{})}); + + const was_null = try std.json.parseFromSliceLeaky(NeverNull(i32, 10), allocator, "null", .{}); + try std.testing.expectEqual(@as(i32, 10), was_null.value); + try std.testing.expectFmt("10", "{f}", .{std.json.fmt(was_null, .{})}); +} diff --git a/stdx/math.zig b/stdx/math.zig new file mode 100644 index 00000000..da0bec79 --- /dev/null +++ b/stdx/math.zig @@ -0,0 +1,23 @@ +pub inline fn divFloat(comptime T: type, numerator: anytype, denominator: anytype) T { + return floatCast(T, numerator) / floatCast(T, denominator); +} + +pub inline fn floatCast(comptime T: type, x: anytype) T { + return switch (@typeInfo(@TypeOf(x))) { + .float => @floatCast(x), + else => @floatFromInt(x), + }; +} + +pub inline fn intCast(comptime T: type, x: anytype) T { + return switch (@typeInfo(@TypeOf(x))) { + .int => @intCast(x), + else => @intFromFloat(x), + }; +} + +pub inline fn roundeven(x: anytype) @TypeOf(x) { + return struct { + extern fn @"llvm.roundeven"(@TypeOf(x)) @TypeOf(x); + }.@"llvm.roundeven"(x); +} diff --git a/stdx/meta.zig b/stdx/meta.zig new file mode 100644 index 00000000..9b1c05af --- /dev/null +++ b/stdx/meta.zig @@ -0,0 +1,210 @@ +const std = @import("std"); + +const debug = @import("debug.zig"); +const compileError = debug.compileError; + +pub fn isStruct(comptime T: type) bool { + return switch (@typeInfo(T)) { + .@"struct" => true, + else => false, + }; +} + +pub fn isTuple(comptime T: type) bool { + return switch (@typeInfo(T)) { + .@"struct" => |info| info.is_tuple, + else => false, + }; +} + +pub fn isStructOf(comptime T: type, comptime Elem: type) bool { + return switch (@typeInfo(T)) { + .@"struct" => |info| blk: { + inline for (info.field_types) |FieldType| { + if (FieldType != Elem) { + break :blk false; + } + } + break :blk true; + }, + else => false, + }; +} + +pub fn isStructOfAny(comptime T: type, comptime f: fn (comptime type) bool) bool { + return switch (@typeInfo(T)) { + .@"struct" => |info| blk: { + inline for (info.field_types) |FieldType| { + if (f(FieldType) == false) { + break :blk false; + } + } + break :blk true; + }, + else => false, + }; +} + +pub fn isTupleOf(comptime T: type, comptime Elem: type) bool { + return isTuple(T) and isStructOf(T, Elem); +} + +pub fn isTupleOfAny(comptime T: type, comptime f: fn (comptime type) bool) bool { + return isTuple(T) and isStructOfAny(T, f); +} + +pub fn isSliceOf(comptime T: type, comptime Elem: type) bool { + return switch (@typeInfo(T)) { + .pointer => |info| switch (info.size) { + .slice => info.child == Elem, + .one => switch (@typeInfo(info.child)) { + // As Zig, convert pointer to Array as a slice. + .array => |arr_info| arr_info.child == Elem, + else => false, + }, + else => false, + }, + else => false, + }; +} + +pub fn isInteger(comptime T: type) bool { + return switch (@typeInfo(T)) { + .int, .comptime_int => true, + else => false, + }; +} + +pub fn isSliceOfAny(comptime T: type, comptime f: fn (comptime type) bool) bool { + return switch (@typeInfo(T)) { + .pointer => |info| switch (info.size) { + .one => info.child == @TypeOf(.{}), + .slice => f(info.child), + else => false, + }, + else => false, + }; +} + +pub fn DeclEnum(comptime T: type) type { + const field_infos = std.meta.declarations(T); + if (field_infos.len == 0) { + compileError("Struct {} has no declarations", .{T}); + } + return std.meta.DeclEnum(UnwrapPtr(T)); +} + +pub fn UnwrapPtr(comptime T: type) type { + return switch (@typeInfo(T)) { + .pointer => |info| switch (info.size) { + .one => info.child, + else => T, + }, + else => T, + }; +} + +pub fn asSlice(comptime T: type) type { + const err_msg = "Type " ++ @typeName(T) ++ " can't be interpreted as a slice"; + return switch (@typeInfo(T)) { + .pointer => |info| switch (info.size) { + .slice => info.child, + .one => switch (@typeInfo(info.child)) { + // As Zig, convert pointer to Array as a slice. + .array => |arr_info| arr_info.child, + else => @compileError(err_msg), + }, + else => @compileError(err_msg), + }, + else => @compileError(err_msg), + }; +} + +pub fn TupleRange(comptime T: type, comptime start: ?usize, comptime end: ?usize) type { + const all_field_types = std.meta.fieldTypes(T); + const start_ = start orelse 0; + const end_ = end orelse all_field_types.len; + + if (start_ == end_) { + return @Tuple(&.{}); + } + + var field_types: [end_ - start_]type = undefined; + inline for (start_..end_, 0..) |i, j| { + field_types[j] = all_field_types[i]; + } + return @Tuple(&field_types); +} + +pub fn FnParam(comptime func: anytype, comptime n: comptime_int) type { + const params = @typeInfo(@TypeOf(func)).@"fn".param_types; + if (n >= params.len) { + @compileError("param doesn't exist in func"); + } + return params[n] orelse @compileError("anytype is not supported"); +} + +pub fn FnReturn(comptime func: anytype) type { + return @typeInfo(@TypeOf(func)).@"fn".return_type orelse @compileError("anytype is not supported"); +} + +pub fn FnReturnPayload(comptime func: anytype) type { + const rt = @typeInfo(@TypeOf(func)).@"fn".return_type orelse @compileError("anytype is not supported"); + return switch (@typeInfo(rt)) { + .error_union => |u| u.payload, + else => rt, + }; +} + +pub fn FnReturnErrorSet(comptime func: anytype) ?type { + const rt = @typeInfo(@TypeOf(func)).@"fn".return_type orelse @compileError("anytype is not supported"); + return switch (@typeInfo(rt)) { + .error_union => |u| u.error_set, + else => null, + }; +} + +pub fn Head(comptime Tuple: type) type { + return switch (@typeInfo(Tuple)) { + .@"struct" => |struct_info| { + if (struct_info.field_types.len == 0) @compileError("Can't tail empty tuple"); + return struct_info.field_types[0]; + }, + else => @compileError("Head works on tuple type"), + }; +} + +pub fn Tail(comptime Tuple: type) type { + return switch (@typeInfo(Tuple)) { + .@"struct" => |struct_info| { + if (struct_info.field_types.len == 0) @compileError("Can't tail empty tuple"); + var types: [struct_info.field_types.len - 1]type = undefined; + for (struct_info.field_types[1..], &types) |FieldType, *type_| { + type_.* = FieldType; + } + return @Tuple(&types); + }, + else => @compileError("Tail works on tuple type"), + }; +} + +pub fn fnName(comptime func: anytype) []const u8 { + const T = @TypeOf(func); + std.debug.assert(@typeInfo(T) == .@"fn"); + const wrapper_name = @typeName(Wrapper(func)); + + var it = std.mem.splitSequence(u8, wrapper_name, ".Wrapper((function '"); + _ = it.first(); + const rest = it.next().?; + return rest[0 .. rest.len - 3]; +} + +fn Wrapper(comptime arg: anytype) type { + return struct { + const stuff = arg; + }; +} + +test fnName { + try std.testing.expectEqualStrings("Tail", fnName(Tail)); +} diff --git a/stdx/process.zig b/stdx/process.zig new file mode 100644 index 00000000..91cf5586 --- /dev/null +++ b/stdx/process.zig @@ -0,0 +1,24 @@ +const std = @import("std"); + +extern fn dladdr(addr: *anyopaque, info: *Dl_info) c_int; + +const Dl_info = extern struct { + dli_fname: [*c]const u8, + dli_fbase: *anyopaque, + dli_sname: [*c]const u8, + dli_saddr: *anyopaque, +}; + +fn selfSharedObjectPathImpl(addr: usize) []const u8 { + var info: Dl_info = undefined; + _ = dladdr(@ptrFromInt(addr), &info); + return std.mem.span(info.dli_fname); +} + +pub noinline fn selfSharedObjectPath() []const u8 { + return selfSharedObjectPathImpl(@returnAddress()); +} + +pub noinline fn selfSharedObjectDirPath() []const u8 { + return std.fs.path.dirname(selfSharedObjectPathImpl(@returnAddress())).?; +} diff --git a/stdx/queue.zig b/stdx/queue.zig new file mode 100644 index 00000000..ebbf0589 --- /dev/null +++ b/stdx/queue.zig @@ -0,0 +1,305 @@ +const std = @import("std"); +const assert = std.debug.assert; + +/// An intrusive queue implementation. The type T must have a field +/// "next" of type `?*T`. +/// +/// For those unaware, an intrusive variant of a data structure is one in which +/// the data type in the list has the pointer to the next element, rather +/// than a higher level "node" or "container" type. The primary benefit +/// of this (and the reason we implement this) is that it defers all memory +/// management to the caller: the data structure implementation doesn't need +/// to allocate "nodes" to contain each element. Instead, the caller provides +/// the element and how its allocated is up to them. +pub fn SPSC(comptime T: type) type { + return struct { + const Self = @This(); + + /// Head is the front of the queue and tail is the back of the queue. + head: ?*T = null, + tail: ?*T = null, + + /// Enqueue a new element to the back of the queue. + pub fn push(self: *Self, v: *T) void { + assert(v.next == null); + + if (self.tail) |tail| { + // If we have elements in the queue, then we add a new tail. + tail.next = v; + self.tail = v; + } else { + // No elements in the queue we setup the initial state. + self.head = v; + self.tail = v; + } + } + + pub fn pushAll(self: *Self, other: Self) void { + if (self.tail) |tail| { + tail.next = other.head; + } else { + self.head = other.head; + } + self.tail = other.tail; + } + + /// Dequeue the next element from the queue. + pub fn pop(self: *Self) ?*T { + // The next element is in "head". + const next = self.head orelse return null; + + // If the head and tail are equal this is the last element + // so we also set tail to null so we can now be empty. + if (self.head == self.tail) self.tail = null; + + // Head is whatever is next (if we're the last element, + // this will be null); + self.head = next.next; + + // We set the "next" field to null so that this element + // can be inserted again. + next.next = null; + return next; + } + + pub fn len(self: Self) usize { + var ret: usize = 0; + var current = self.head; + while (current) |elem| : (current = elem.next) { + ret += 1; + } + return ret; + } + + /// Returns true if the queue is empty. + pub fn empty(self: *const Self) bool { + return self.head == null; + } + }; +} + +test SPSC { + const testing = std.testing; + + // Types + const Elem = struct { + const Self = @This(); + next: ?*Self = null, + }; + const Queue = SPSC(Elem); + var q: Queue = .{}; + try testing.expect(q.empty()); + + // Elems + var elems: [10]Elem = @splat(.{}); + + // One + try testing.expect(q.pop() == null); + q.push(&elems[0]); + try testing.expect(!q.empty()); + try testing.expect(q.pop().? == &elems[0]); + try testing.expect(q.pop() == null); + try testing.expect(q.empty()); + + // Two + try testing.expect(q.pop() == null); + q.push(&elems[0]); + q.push(&elems[1]); + try testing.expect(q.pop().? == &elems[0]); + try testing.expect(q.pop().? == &elems[1]); + try testing.expect(q.pop() == null); + + // Interleaved + try testing.expect(q.pop() == null); + q.push(&elems[0]); + try testing.expect(q.pop().? == &elems[0]); + q.push(&elems[1]); + try testing.expect(q.pop().? == &elems[1]); + try testing.expect(q.pop() == null); +} + +/// An intrusive MPSC (multi-provider, single consumer) queue implementation. +/// The type T must have a field "next" of type `?*T`. +/// +/// This is an implementatin of a Vyukov Queue[1]. +/// TODO(mitchellh): I haven't audited yet if I got all the atomic operations +/// correct. I was short term more focused on getting something that seemed +/// to work; I need to make sure it actually works. +/// +/// For those unaware, an intrusive variant of a data structure is one in which +/// the data type in the list has the pointer to the next element, rather +/// than a higher level "node" or "container" type. The primary benefit +/// of this (and the reason we implement this) is that it defers all memory +/// management to the caller: the data structure implementation doesn't need +/// to allocate "nodes" to contain each element. Instead, the caller provides +/// the element and how its allocated is up to them. +/// +/// [1]: https://www.1024cores.net/home/lock-free-algorithms/queues/intrusive-mpsc-node-based-queue +pub fn MPSC(comptime T: type) type { + return struct { + const Self = @This(); + + /// Head is the front of the queue and tail is the back of the queue. + head: *T, + tail: *T, + stub: T, + + /// Initialize the queue. This requires a stable pointer to itself. + /// This must be called before the queue is used concurrently. + pub fn init(self: *Self) void { + self.head = &self.stub; + self.tail = &self.stub; + self.stub.next = null; + } + + /// Push an item onto the queue. This can be called by any number + /// of producers. + pub fn push(self: *Self, v: *T) void { + @atomicStore(?*T, &v.next, null, .unordered); + const prev = @atomicRmw(*T, &self.head, .Xchg, v, .acq_rel); + @atomicStore(?*T, &prev.next, v, .release); + } + + /// Pop the first in element from the queue. This must be called + /// by only a single consumer at any given time. + pub fn pop(self: *Self) ?*T { + var tail = @atomicLoad(*T, &self.tail, .unordered); + var next_ = @atomicLoad(?*T, &tail.next, .acquire); + if (tail == &self.stub) { + const next = next_ orelse return null; + @atomicStore(*T, &self.tail, next, .unordered); + tail = next; + next_ = @atomicLoad(?*T, &tail.next, .acquire); + } + + if (next_) |next| { + @atomicStore(*T, &self.tail, next, .release); + tail.next = null; + return tail; + } + + const head = @atomicLoad(*T, &self.head, .unordered); + if (tail != head) return null; + self.push(&self.stub); + + next_ = @atomicLoad(?*T, &tail.next, .acquire); + if (next_) |next| { + @atomicStore(*T, &self.tail, next, .unordered); + tail.next = null; + return tail; + } + + return null; + } + }; +} + +test MPSC { + const testing = std.testing; + + // Types + const Elem = struct { + const Self = @This(); + next: ?*Self = null, + }; + const Queue = MPSC(Elem); + var q: Queue = undefined; + q.init(); + + // Elems + var elems: [10]Elem = @splat(.{}); + + // One + try testing.expect(q.pop() == null); + q.push(&elems[0]); + try testing.expect(q.pop().? == &elems[0]); + try testing.expect(q.pop() == null); + + // Two + try testing.expect(q.pop() == null); + q.push(&elems[0]); + q.push(&elems[1]); + try testing.expect(q.pop().? == &elems[0]); + try testing.expect(q.pop().? == &elems[1]); + try testing.expect(q.pop() == null); + + // // Interleaved + try testing.expect(q.pop() == null); + q.push(&elems[0]); + try testing.expect(q.pop().? == &elems[0]); + q.push(&elems[1]); + try testing.expect(q.pop().? == &elems[1]); + try testing.expect(q.pop() == null); +} + +pub fn ArrayQueue(comptime T: type, comptime size: usize) type { + return struct { + const Self = @This(); + + vals: [size]T = undefined, + head: ?usize = null, + tail: ?usize = null, + + pub fn len(self: Self) usize { + switch (self.state()) { + .empty => return 0, + .one => return 1, + .many => { + const head = self.head.?; + const tail = self.tail.?; + if (tail > head) { + return tail - head + 1; + } + return size - head + tail + 1; + }, + } + } + + pub fn available(self: Self) usize { + return size - self.len(); + } + + pub fn push(self: *Self, val: T) !void { + if (self.len() == size) { + return error.QueueFull; + } + switch (self.state()) { + .empty => { + self.head = 0; + self.tail = 0; + self.vals[0] = val; + }, + .one, .many => { + const tail = self.tail.?; + const new_tail = (tail + 1) % size; + self.vals[new_tail] = val; + self.tail = new_tail; + }, + } + } + + pub fn pop(self: *Self) ?T { + switch (self.state()) { + .empty => return null, + .one => { + const out = self.vals[self.head.?]; + self.head = null; + self.tail = null; + return out; + }, + .many => { + const out = self.vals[self.head.?]; + self.head = (self.head.? + 1) % size; + return out; + }, + } + } + + const State = enum { empty, one, many }; + inline fn state(self: Self) State { + if (self.head == null) return .empty; + if (self.head.? == self.tail.?) return .one; + return .many; + } + }; +} diff --git a/stdx/segmented_list.zig b/stdx/segmented_list.zig new file mode 100644 index 00000000..567534be --- /dev/null +++ b/stdx/segmented_list.zig @@ -0,0 +1,539 @@ +// Origin: https://github.com/ziglang/std-lib-orphanage/ + +// SPDX-License-Identifier: MIT +// Copyright (c) 2015-2020 Zig Contributors +// This file is part of [zig](https://ziglang.org/), which is MIT licensed. +// The MIT license requires this copyright notice to be included in all copies +// and substantial portions of the software. + +const std = @import("std"); +const assert = std.debug.assert; +const testing = std.testing; +const mem = std.mem; +const Allocator = std.mem.Allocator; + +// Imagine that `fn at(self: *Self, index: usize) &T` is a customer asking for a box +// from a warehouse, based on a flat array, boxes ordered from 0 to N - 1. +// But the warehouse actually stores boxes in shelves of increasing powers of 2 sizes. +// So when the customer requests a box index, we have to translate it to shelf index +// and box index within that shelf. Illustration: +// +// customer indexes: +// shelf 0: 0 +// shelf 1: 1 2 +// shelf 2: 3 4 5 6 +// shelf 3: 7 8 9 10 11 12 13 14 +// shelf 4: 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 +// shelf 5: 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 +// ... +// +// warehouse indexes: +// shelf 0: 0 +// shelf 1: 0 1 +// shelf 2: 0 1 2 3 +// shelf 3: 0 1 2 3 4 5 6 7 +// shelf 4: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 +// shelf 5: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 +// ... +// +// With this arrangement, here are the equations to get the shelf index and +// box index based on customer box index: +// +// shelf_index = floor(log2(customer_index + 1)) +// shelf_count = ceil(log2(box_count + 1)) +// box_index = customer_index + 1 - 2 ** shelf +// shelf_size = 2 ** shelf_index +// +// Now we complicate it a little bit further by adding a preallocated shelf, which must be +// a power of 2: +// prealloc=4 +// +// customer indexes: +// prealloc: 0 1 2 3 +// shelf 0: 4 5 6 7 8 9 10 11 +// shelf 1: 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 +// shelf 2: 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 +// ... +// +// warehouse indexes: +// prealloc: 0 1 2 3 +// shelf 0: 0 1 2 3 4 5 6 7 +// shelf 1: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 +// shelf 2: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 +// ... +// +// Now the equations are: +// +// shelf_index = floor(log2(customer_index + prealloc)) - log2(prealloc) - 1 +// shelf_count = ceil(log2(box_count + prealloc)) - log2(prealloc) - 1 +// box_index = customer_index + prealloc - 2 ** (log2(prealloc) + 1 + shelf) +// shelf_size = prealloc * 2 ** (shelf_index + 1) + +/// This is a stack data structure where pointers to indexes have the same lifetime as the data structure +/// itself, unlike ArrayList where append() invalidates all existing element pointers. +/// The tradeoff is that elements are not guaranteed to be contiguous. For that, use ArrayList. +/// Note however that most elements are contiguous, making this data structure cache-friendly. +/// +/// Because it never has to copy elements from an old location to a new location, it does not require +/// its elements to be copyable, and it avoids wasting memory when backed by an ArenaAllocator. +/// Note that the append() and pop() convenience methods perform a copy, but you can instead use +/// addOne(), at(), setCapacity(), and shrinkCapacity() to avoid copying items. +/// +/// This data structure has O(1) append and O(1) pop. +/// +/// It supports preallocated elements, making it especially well suited when the expected maximum +/// size is small. `prealloc_item_count` must be 0, or a power of 2. +pub fn SegmentedList(comptime T: type, comptime prealloc_item_count: usize) type { + return struct { + const Self = @This(); + const ShelfIndex = std.math.Log2Int(usize); + + const prealloc_exp: ShelfIndex = blk: { + // we don't use the prealloc_exp constant when prealloc_item_count is 0 + // but lazy-init may still be triggered by other code so supply a value + if (prealloc_item_count == 0) { + break :blk 0; + } else { + assert(std.math.isPowerOfTwo(prealloc_item_count)); + const value = std.math.log2_int(usize, prealloc_item_count); + break :blk value; + } + }; + + prealloc_segment: [prealloc_item_count]T = undefined, + dynamic_segments: [][*]T = &[_][*]T{}, + len: usize = 0, + + pub const prealloc_count = prealloc_item_count; + + fn AtType(comptime SelfType: type) type { + if (@typeInfo(SelfType).pointer.attrs.@"const") { + return *const T; + } else { + return *T; + } + } + + pub fn deinit(self: *Self, allocator: Allocator) void { + self.freeShelves(allocator, @as(ShelfIndex, @intCast(self.dynamic_segments.len)), 0); + allocator.free(self.dynamic_segments); + self.* = undefined; + } + + pub fn at(self: anytype, i: usize) AtType(@TypeOf(self)) { + assert(i < self.len); + return self.uncheckedAt(i); + } + + pub fn count(self: Self) usize { + return self.len; + } + + pub fn append(self: *Self, allocator: Allocator, item: T) Allocator.Error!void { + const new_item_ptr = try self.addOne(allocator); + new_item_ptr.* = item; + } + + pub fn appendSlice(self: *Self, allocator: Allocator, items: []const T) Allocator.Error!void { + for (items) |item| { + try self.append(allocator, item); + } + } + + pub fn pop(self: *Self) ?T { + if (self.len == 0) return null; + + const index = self.len - 1; + const result = uncheckedAt(self, index).*; + self.len = index; + return result; + } + + pub fn addOne(self: *Self, allocator: Allocator) Allocator.Error!*T { + const new_length = self.len + 1; + try self.growCapacity(allocator, new_length); + const result = uncheckedAt(self, self.len); + self.len = new_length; + return result; + } + + /// Reduce length to `new_len`. + /// Invalidates pointers for the elements at index new_len and beyond. + pub fn shrinkRetainingCapacity(self: *Self, new_len: usize) void { + assert(new_len <= self.len); + self.len = new_len; + } + + /// Invalidates all element pointers. + pub fn clearRetainingCapacity(self: *Self) void { + self.len = 0; + } + + /// Invalidates all element pointers. + pub fn clearAndFree(self: *Self, allocator: Allocator) void { + self.setCapacity(allocator, 0) catch unreachable; + self.len = 0; + } + + /// Grows or shrinks capacity to match usage. + /// TODO update this and related methods to match the conventions set by ArrayList + pub fn setCapacity(self: *Self, allocator: Allocator, new_capacity: usize) Allocator.Error!void { + if (prealloc_item_count != 0) { + if (new_capacity <= @as(usize, 1) << (prealloc_exp + @as(ShelfIndex, @intCast(self.dynamic_segments.len)))) { + return self.shrinkCapacity(allocator, new_capacity); + } + } + return self.growCapacity(allocator, new_capacity); + } + + /// Only grows capacity, or retains current capacity. + pub fn growCapacity(self: *Self, allocator: Allocator, new_capacity: usize) Allocator.Error!void { + const new_cap_shelf_count = shelfCount(new_capacity); + const old_shelf_count = @as(ShelfIndex, @intCast(self.dynamic_segments.len)); + if (new_cap_shelf_count <= old_shelf_count) return; + + const new_dynamic_segments = try allocator.alloc([*]T, new_cap_shelf_count); + errdefer allocator.free(new_dynamic_segments); + + var i: ShelfIndex = 0; + while (i < old_shelf_count) : (i += 1) { + new_dynamic_segments[i] = self.dynamic_segments[i]; + } + errdefer while (i > old_shelf_count) : (i -= 1) { + allocator.free(new_dynamic_segments[i][0..shelfSize(i)]); + }; + while (i < new_cap_shelf_count) : (i += 1) { + new_dynamic_segments[i] = (try allocator.alloc(T, shelfSize(i))).ptr; + } + + allocator.free(self.dynamic_segments); + self.dynamic_segments = new_dynamic_segments; + } + + /// Only shrinks capacity or retains current capacity. + /// It may fail to reduce the capacity in which case the capacity will remain unchanged. + pub fn shrinkCapacity(self: *Self, allocator: Allocator, new_capacity: usize) void { + if (new_capacity <= prealloc_item_count) { + const len = @as(ShelfIndex, @intCast(self.dynamic_segments.len)); + self.freeShelves(allocator, len, 0); + allocator.free(self.dynamic_segments); + self.dynamic_segments = &[_][*]T{}; + return; + } + + const new_cap_shelf_count = shelfCount(new_capacity); + const old_shelf_count = @as(ShelfIndex, @intCast(self.dynamic_segments.len)); + assert(new_cap_shelf_count <= old_shelf_count); + if (new_cap_shelf_count == old_shelf_count) return; + + // freeShelves() must be called before resizing the dynamic + // segments, but we don't know if resizing the dynamic segments + // will work until we try it. So we must allocate a fresh memory + // buffer in order to reduce capacity. + const new_dynamic_segments = allocator.alloc([*]T, new_cap_shelf_count) catch return; + self.freeShelves(allocator, old_shelf_count, new_cap_shelf_count); + if (allocator.resize(self.dynamic_segments, new_cap_shelf_count)) { + // We didn't need the new memory allocation after all. + self.dynamic_segments = self.dynamic_segments[0..new_cap_shelf_count]; + allocator.free(new_dynamic_segments); + } else { + // Good thing we allocated that new memory slice. + @memcpy(new_dynamic_segments, self.dynamic_segments[0..new_cap_shelf_count]); + allocator.free(self.dynamic_segments); + self.dynamic_segments = new_dynamic_segments; + } + } + + pub fn shrink(self: *Self, new_len: usize) void { + assert(new_len <= self.len); + // TODO take advantage of the new realloc semantics + self.len = new_len; + } + + pub fn writeToSlice(self: *Self, dest: []T, start: usize) void { + const end = start + dest.len; + assert(end <= self.len); + + var i = start; + if (end <= prealloc_item_count) { + const src = self.prealloc_segment[i..end]; + @memcpy(dest[i - start ..][0..src.len], src); + return; + } else if (i < prealloc_item_count) { + const src = self.prealloc_segment[i..]; + @memcpy(dest[i - start ..][0..src.len], src); + i = prealloc_item_count; + } + + while (i < end) { + const shelf_index = shelfIndex(i); + const copy_start = boxIndex(i, shelf_index); + const copy_end = @min(shelfSize(shelf_index), copy_start + end - i); + const src = self.dynamic_segments[shelf_index][copy_start..copy_end]; + @memcpy(dest[i - start ..][0..src.len], src); + i += (copy_end - copy_start); + } + } + + pub fn uncheckedAt(self: anytype, index: usize) AtType(@TypeOf(self)) { + if (index < prealloc_item_count) { + return &self.prealloc_segment[index]; + } + const shelf_index = shelfIndex(index); + const box_index = boxIndex(index, shelf_index); + return &self.dynamic_segments[shelf_index][box_index]; + } + + fn shelfCount(box_count: usize) ShelfIndex { + if (prealloc_item_count == 0) { + return log2_int_ceil(usize, box_count + 1); + } + return log2_int_ceil(usize, box_count + prealloc_item_count) - prealloc_exp - 1; + } + + fn shelfSize(shelf_index: ShelfIndex) usize { + if (prealloc_item_count == 0) { + return @as(usize, 1) << shelf_index; + } + return @as(usize, 1) << (shelf_index + (prealloc_exp + 1)); + } + + fn shelfIndex(list_index: usize) ShelfIndex { + if (prealloc_item_count == 0) { + return std.math.log2_int(usize, list_index + 1); + } + return std.math.log2_int(usize, list_index + prealloc_item_count) - prealloc_exp - 1; + } + + fn boxIndex(list_index: usize, shelf_index: ShelfIndex) usize { + if (prealloc_item_count == 0) { + return (list_index + 1) - (@as(usize, 1) << shelf_index); + } + return list_index + prealloc_item_count - (@as(usize, 1) << ((prealloc_exp + 1) + shelf_index)); + } + + fn freeShelves(self: *Self, allocator: Allocator, from_count: ShelfIndex, to_count: ShelfIndex) void { + var i = from_count; + while (i != to_count) { + i -= 1; + allocator.free(self.dynamic_segments[i][0..shelfSize(i)]); + } + } + + pub const Iterator = BaseIterator(*Self, *T); + pub const ConstIterator = BaseIterator(*const Self, *const T); + fn BaseIterator(comptime SelfType: type, comptime ElementPtr: type) type { + return struct { + list: SelfType, + index: usize, + box_index: usize, + shelf_index: ShelfIndex, + shelf_size: usize, + + pub fn next(it: *@This()) ?ElementPtr { + if (it.index >= it.list.len) return null; + if (it.index < prealloc_item_count) { + const ptr = &it.list.prealloc_segment[it.index]; + it.index += 1; + if (it.index == prealloc_item_count) { + it.box_index = 0; + it.shelf_index = 0; + it.shelf_size = prealloc_item_count * 2; + } + return ptr; + } + + const ptr = &it.list.dynamic_segments[it.shelf_index][it.box_index]; + it.index += 1; + it.box_index += 1; + if (it.box_index == it.shelf_size) { + it.shelf_index += 1; + it.box_index = 0; + it.shelf_size *= 2; + } + return ptr; + } + + pub fn prev(it: *@This()) ?ElementPtr { + if (it.index == 0) return null; + + it.index -= 1; + if (it.index < prealloc_item_count) return &it.list.prealloc_segment[it.index]; + + if (it.box_index == 0) { + it.shelf_index -= 1; + it.shelf_size /= 2; + it.box_index = it.shelf_size - 1; + } else { + it.box_index -= 1; + } + + return &it.list.dynamic_segments[it.shelf_index][it.box_index]; + } + + pub fn peek(it: *@This()) ?ElementPtr { + if (it.index >= it.list.len) + return null; + if (it.index < prealloc_item_count) + return &it.list.prealloc_segment[it.index]; + + return &it.list.dynamic_segments[it.shelf_index][it.box_index]; + } + + pub fn set(it: *@This(), index: usize) void { + it.index = index; + if (index < prealloc_item_count) return; + it.shelf_index = shelfIndex(index); + it.box_index = boxIndex(index, it.shelf_index); + it.shelf_size = shelfSize(it.shelf_index); + } + }; + } + + pub fn iterator(self: *Self, start_index: usize) Iterator { + var it = Iterator{ + .list = self, + .index = undefined, + .shelf_index = undefined, + .box_index = undefined, + .shelf_size = undefined, + }; + it.set(start_index); + return it; + } + + pub fn constIterator(self: *const Self, start_index: usize) ConstIterator { + var it = ConstIterator{ + .list = self, + .index = undefined, + .shelf_index = undefined, + .box_index = undefined, + .shelf_size = undefined, + }; + it.set(start_index); + return it; + } + }; +} + +test "basic usage" { + try testSegmentedList(0); + try testSegmentedList(1); + try testSegmentedList(2); + try testSegmentedList(4); + try testSegmentedList(8); + try testSegmentedList(16); +} + +fn testSegmentedList(comptime prealloc: usize) !void { + var list = SegmentedList(i32, prealloc){}; + defer list.deinit(testing.allocator); + + { + var i: usize = 0; + while (i < 100) : (i += 1) { + try list.append(testing.allocator, @as(i32, @intCast(i + 1))); + try testing.expect(list.len == i + 1); + } + } + + { + var i: usize = 0; + while (i < 100) : (i += 1) { + try testing.expect(list.at(i).* == @as(i32, @intCast(i + 1))); + } + } + + { + var it = list.iterator(0); + var x: i32 = 0; + while (it.next()) |item| { + x += 1; + try testing.expect(item.* == x); + } + try testing.expect(x == 100); + while (it.prev()) |item| : (x -= 1) { + try testing.expect(item.* == x); + } + try testing.expect(x == 0); + } + + { + var it = list.constIterator(0); + var x: i32 = 0; + while (it.next()) |item| { + x += 1; + try testing.expect(item.* == x); + } + try testing.expect(x == 100); + while (it.prev()) |item| : (x -= 1) { + try testing.expect(item.* == x); + } + try testing.expect(x == 0); + } + + try testing.expect(list.pop().? == 100); + try testing.expect(list.len == 99); + + try list.appendSlice(testing.allocator, &[_]i32{ 1, 2, 3 }); + try testing.expect(list.len == 102); + try testing.expect(list.pop().? == 3); + try testing.expect(list.pop().? == 2); + try testing.expect(list.pop().? == 1); + try testing.expect(list.len == 99); + + try list.appendSlice(testing.allocator, &[_]i32{}); + try testing.expect(list.len == 99); + + { + var i: i32 = 99; + while (list.pop()) |item| : (i -= 1) { + try testing.expect(item == i); + list.shrinkCapacity(testing.allocator, list.len); + } + } + + { + var control: [100]i32 = undefined; + var dest: [100]i32 = undefined; + + var i: i32 = 0; + while (i < 100) : (i += 1) { + try list.append(testing.allocator, i + 1); + control[@as(usize, @intCast(i))] = i + 1; + } + + @memset(dest[0..], 0); + list.writeToSlice(dest[0..], 0); + try testing.expect(mem.eql(i32, control[0..], dest[0..])); + + @memset(dest[0..], 0); + list.writeToSlice(dest[50..], 50); + try testing.expect(mem.eql(i32, control[50..], dest[50..])); + } + + try list.setCapacity(testing.allocator, 0); +} + +test "clearRetainingCapacity" { + var list = SegmentedList(i32, 1){}; + defer list.deinit(testing.allocator); + + try list.appendSlice(testing.allocator, &[_]i32{ 4, 5 }); + list.clearRetainingCapacity(); + try list.append(testing.allocator, 6); + try testing.expect(list.at(0).* == 6); + try testing.expect(list.len == 1); + list.clearRetainingCapacity(); + try testing.expect(list.len == 0); +} + +/// TODO look into why this std.math function was changed in +/// fc9430f56798a53f9393a697f4ccd6bf9981b970. +fn log2_int_ceil(comptime T: type, x: T) std.math.Log2Int(T) { + assert(x != 0); + const log2_val = std.math.log2_int(T, x); + if (@as(T, 1) << log2_val == x) + return log2_val; + return log2_val + 1; +} diff --git a/stdx/stdx.zig b/stdx/stdx.zig new file mode 100644 index 00000000..1b19e638 --- /dev/null +++ b/stdx/stdx.zig @@ -0,0 +1,145 @@ +const std = @import("std"); +const builtin = @import("builtin"); + +pub const BoundedArray = @import("bounded_array.zig").BoundedArray; +pub const BoundedArrayAligned = @import("bounded_array.zig").BoundedArrayAligned; +pub const cpu = @import("cpu.zig"); +pub const crypto = @import("crypto.zig"); +pub const debug = @import("debug.zig"); +pub const flags = @import("flags.zig"); +pub const fmt = @import("fmt.zig"); +pub const Io = @import("Io.zig"); +pub const json = @import("json.zig"); +pub const math = @import("math.zig"); +pub const meta = @import("meta.zig"); +pub const process = @import("process.zig"); +pub const queue = @import("queue.zig"); +pub const SegmentedList = @import("segmented_list.zig").SegmentedList; +pub const time = @import("time.zig"); +pub const unicode = @import("unicode.zig"); + +test { + std.testing.refAllDecls(@This()); +} + +pub inline fn stackSlice(comptime max_len: usize, T: type, len: usize) []T { + debug.assert(len <= max_len, "stackSlice can only create a slice of up to {} elements, got: {}", .{ max_len, len }); + var storage: [max_len]T = undefined; + return storage[0..len]; +} + +pub const noalloc: std.mem.Allocator = if (builtin.mode == .fast) undefined else std.testing.failing_allocator; + +pub fn arenaWithCapacity(parent: std.mem.Allocator, initial_capacity: usize) std.mem.Allocator.Error!std.heap.ArenaAllocator { + var a: std.heap.ArenaAllocator = .init(parent); + + _ = try a.allocator().alloc(u8, initial_capacity); + std.debug.assert(a.state.used_list.?.end_index == initial_capacity); + a.state.used_list.?.end_index = 0; + return a; +} + +pub fn pinToCore(core_id: usize) void { + if (builtin.os.tag == .linux) { + const CPUSet = std.bit_set.ArrayBitSet(usize, std.os.linux.CPU_SETSIZE * @sizeOf(usize)); + + var set: CPUSet = .empty; + set.set(core_id); + std.os.linux.sched_setaffinity(0, @ptrCast(&set.masks)) catch {}; + } +} + +pub fn once(comptime f: anytype) once(f) { + return .{}; +} + +/// An object that executes the function `f` just once. +/// It is undefined behavior if `f` re-enters the same Once instance. +pub fn Once(comptime f: anytype) type { + const Args = std.meta.ArgsTuple(@TypeOf(f)); + return struct { + done: bool = false, + mutex: std.Io.Mutex = .init, + + /// Call the function `f`. + /// If `call` is invoked multiple times `f` will be executed only the + /// first time. + /// The invocations are thread-safe. + pub fn call(self: *@This(), io: std.Io, args: Args) void { + if (@atomicLoad(bool, &self.done, .acquire)) + return; + + self.callSlow(io, args); + } + + fn callSlow(self: *@This(), io: std.Io, args: Args) void { + @branchHint(.cold); + + self.mutex.lock(io); + defer self.mutex.unlock(io); + + // The first thread to acquire the mutex gets to run the initializer + if (!self.done) { + @call(.auto, f, args); + @atomicStore(bool, &self.done, true, .release); + } + } + }; +} + +/// Packed storage of sub-byte values +/// +/// Note: before Zig 0.17, @Vector(u2, 4) was always one byte and could be used as a storage type, +/// But that's not true anymore. +pub fn Packed(T: type) type { + return switch (@bitSizeOf(T)) { + 2 => packed struct(u8) { + x: T, + y: T, + z: T, + w: T, + + const P = @This(); + pub fn pack(x: T, y: T, z: T, w: T) P { + return .{ .x = x, .y = y, .z = z, .w = w }; + } + + pub fn unpack(p: P) [4]T { + return .{ p.x, p.y, p.z, p.w }; + } + + pub fn formatNumber(p: P, w: *std.Io.Writer, n: std.fmt.Number) std.Io.Writer.Error!void { + try formatPackedNumber(T, &p.unpack(), w, n); + } + }, + 4 => packed struct(u8) { + x: T, + y: T, + + const P = @This(); + pub fn pack(x: T, y: T) P { + return .{ .x = x, .y = y }; + } + + pub fn unpack(p: P) [2]T { + return .{ p.x, p.y }; + } + + pub fn formatNumber(p: P, w: *std.Io.Writer, n: std.fmt.Number) std.Io.Writer.Error!void { + try formatPackedNumber(T, &p.unpack(), w, n); + } + }, + else => debug.compileError("stdx.Packed expects a bitsize of 2 or 4, got type {} with bitsize {d}", .{ T, @bitSizeOf(T) }), + }; +} + +fn formatPackedNumber(T: type, values: []const T, w: *std.Io.Writer, n: std.fmt.Number) std.Io.Writer.Error!void { + for (0.., values) |i, val| { + if (i > 0) try w.writeByte(','); + switch (@typeInfo(T)) { + .@"struct" => try val.formatNumber(w, n), + .int => try fmt.formatInt(val, n, w), + else => try w.print("{}", .{val}), + } + } +} diff --git a/stdx/time.zig b/stdx/time.zig new file mode 100644 index 00000000..d724c0c2 --- /dev/null +++ b/stdx/time.zig @@ -0,0 +1,43 @@ +const std = @import("std"); + +pub const Duration = struct { + ns: u64 = 0, + + pub fn div(self: Duration, rhs: u64) Duration { + return .{ .ns = self.ns / rhs }; + } + + pub fn hz(self: Duration) u64 { + return (1 * std.time.ns_per_s) / self.ns; + } + + pub fn hzFloat(self: Duration) f64 { + return (1 * std.time.ns_per_s) / @as(f64, @floatFromInt(self.ns)); + } + + pub fn formatDuration(duration: Duration, writer: *std.Io.Writer) std.Io.Writer.Error!void { + try writer.printDuration(duration.ns, .{}); + } + + pub const format = formatDuration; +}; + +pub const Timer = struct { + inner: std.time.Timer, + + pub fn start() !Timer { + return .{ .inner = try std.time.Timer.start() }; + } + + pub fn lap(self: *Timer) Duration { + return .{ .ns = self.inner.lap() }; + } + + pub fn read(self: *Timer) Duration { + return .{ .ns = self.inner.read() }; + } + + pub fn reset(self: *Timer) void { + self.inner.reset(); + } +}; diff --git a/stdx/unicode.zig b/stdx/unicode.zig new file mode 100644 index 00000000..89c65a28 --- /dev/null +++ b/stdx/unicode.zig @@ -0,0 +1,24 @@ +const std = @import("std"); + +pub fn calcUtf32LeLen(utf8: []const u8) !usize { + return (try std.unicode.utf8CountCodepoints(utf8)) * @sizeOf(u32); +} + +pub fn utf8ToUtf32Le(utf32le: []u32, utf8: []const u8) !usize { + var view: std.unicode.Utf8View = .initUnchecked(utf8); + var it = view.iterator(); + var idx: usize = 0; + while (it.nextCodepoint()) |cp| : (idx += 1) { + if (idx >= utf32le.len) { + return error.BufferTooSmall; + } + utf32le[idx] = @intCast(cp); + } + return idx; +} + +pub fn utf8ToUtf32LeZ(utf32le: []u32, utf8: []const u8) !usize { + const n = try utf8ToUtf32Le(utf32le, utf8); + utf32le[n] = 0; + return n + 1; +} diff --git a/third_party/abseil-cpp/bmi2intrin.patch b/third_party/abseil-cpp/bmi2intrin.patch new file mode 100644 index 00000000..8273731b --- /dev/null +++ b/third_party/abseil-cpp/bmi2intrin.patch @@ -0,0 +1,13 @@ +diff --git a/absl/container/internal/raw_hash_set.h b/absl/container/internal/raw_hash_set.h +index 0d05ad7..85ed32d 100644 +--- a/absl/container/internal/raw_hash_set.h ++++ b/absl/container/internal/raw_hash_set.h +@@ -226,7 +226,7 @@ static_assert(std::numeric_limits::is_iec559, + #endif + + #ifdef __BMI2__ +-#include ++#include + #endif // __BMI2__ + + #if defined(__SSE2__) || defined(__aarch64__) || \ \ No newline at end of file diff --git a/third_party/modules/llvm-raw/20240823.0-f142f8a/overlay/BUILD.bazel b/third_party/arocc/BUILD.bazel similarity index 100% rename from third_party/modules/llvm-raw/20240823.0-f142f8a/overlay/BUILD.bazel rename to third_party/arocc/BUILD.bazel diff --git a/third_party/arocc/arocc.bazel b/third_party/arocc/arocc.bazel new file mode 100644 index 00000000..a5de1d98 --- /dev/null +++ b/third_party/arocc/arocc.bazel @@ -0,0 +1,148 @@ +load("@bazel_skylib//rules:run_binary.bzl", "run_binary") +load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library") +load("@zml//bazel:zig_options.bzl", "zig_option", "zig_options") + +zig_option( + name = "enable_linker_build_id", + bool = False, +) + +zig_option( + name = "linker", + string = "", +) + +zig_option( + name = "sysroot", + string = "", +) + +zig_option( + name = "gcc_install_prefix", + string = "", +) + +zig_option( + name = "rtlib", + string = "", +) + +zig_option( + name = "unwindlib", + string = "", +) + +zig_options( + name = "system_defaults", + settings = [ + ":enable_linker_build_id", + ":linker", + ":sysroot", + ":gcc_install_prefix", + ":rtlib", + ":unwindlib", + ], +) + +zig_option( + name = "enable_tracy", + bool = False, +) + +zig_option( + name = "enable_tracy_callstack", + bool = False, +) + +zig_option( + name = "enable_tracy_allocation", + bool = False, +) + +zig_option( + name = "debug_allocations", + bool = False, +) + +zig_option( + name = "version_str", + string = "", +) + +zig_options( + name = "build_options", + settings = [ + ":enable_tracy", + ":enable_tracy_callstack", + ":enable_tracy_allocation", + ":debug_allocations", + ":version_str", + ], +) + +zig_library( + name = "backend", + main = "src/backend.zig", + deps = [":build_options"], +) + +zig_library( + name = "aro", + srcs = glob(["src/**/*.zig"]), + main = "src/aro.zig", + visibility = ["//visibility:public"], + deps = [ + ":backend", + ":build_options", + ":system_defaults", + ] + [ + ":{}".format(name.replace("src/aro/", "")) + for name in glob(["src/aro/**/*.def"]) + ], +) + +zig_library( + name = "assembly_backend", + main = "src/assembly_backend.zig", + deps = [ + ":aro", + ], +) + +zig_binary( + name = "arocc", + srcs = glob(["src/**/*.zig"]), + main = "src/main.zig", + deps = [ + ":aro", + ":assembly_backend", + ":build_options", + ], +) + +zig_binary( + name = "generate-def", + main = "build/generate_def.zig", +) + +[ + run_binary( + name = "def_{}".format(name), + srcs = [name], + outs = ["{}.zig".format(name)], + args = [ + "$(location {})".format(name), + "$(location {}.zig)".format(name), + ], + tool = ":generate-def", + ) + for name in glob(["src/aro/**/*.def"]) +] + +[ + zig_library( + name = "{}".format(name.replace("src/aro/", "")), + main = ":def_{}".format(name), + ) + for name in glob(["src/aro/**/*.def"]) +] diff --git a/third_party/arocc/repo.bzl b/third_party/arocc/repo.bzl new file mode 100644 index 00000000..15ff6d7f --- /dev/null +++ b/third_party/arocc/repo.bzl @@ -0,0 +1,10 @@ +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") + +def repo(): + # https://github.com/zml/arocc/releases/tag/0.17.0 + git_repository( + name = "arocc", + remote = "https://github.com/zml/arocc", + commit = "d0c8c4d9c55daa7ef6e40cf0f630a5b5e900989b", + build_file = Label("//third_party/arocc:arocc.bazel"), + ) diff --git a/third_party/modules/llvm-raw/20240919.0-94c024a/overlay/BUILD.bazel b/third_party/cloud_accelerator_diagnostics/BUILD.bazel similarity index 100% rename from third_party/modules/llvm-raw/20240919.0-94c024a/overlay/BUILD.bazel rename to third_party/cloud_accelerator_diagnostics/BUILD.bazel diff --git a/third_party/cloud_accelerator_diagnostics/cloud_accelerator_diagnostics.BUILD.bazel b/third_party/cloud_accelerator_diagnostics/cloud_accelerator_diagnostics.BUILD.bazel new file mode 100644 index 00000000..563999d2 --- /dev/null +++ b/third_party/cloud_accelerator_diagnostics/cloud_accelerator_diagnostics.BUILD.bazel @@ -0,0 +1,34 @@ +load("@com_google_protobuf//bazel:cc_proto_library.bzl", "cc_proto_library") +load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library") +load("@grpc//bazel:cc_grpc_library.bzl", "cc_grpc_library") + +proto_library( + name = "tpu_telemetry_proto", + srcs = ["tpu_telemetry.proto"], + import_prefix = "tpu_info/proto", + visibility = ["//visibility:public"], +) + +proto_library( + name = "tpu_metric_service_proto", + srcs = ["tpu_metric_service.proto"], + deps = [ + ":tpu_telemetry_proto", + "@com_google_protobuf//:timestamp_proto", + ], + visibility = ["//visibility:public"], +) + +cc_proto_library( + name = "tpu_metric_service_cc_proto", + deps = [":tpu_metric_service_proto"], + visibility = ["//visibility:public"], +) + +cc_grpc_library( + name = "tpu_metric_service_cc_grpc", + srcs = [":tpu_metric_service_proto"], + grpc_only = True, + deps = [":tpu_metric_service_cc_proto"], + visibility = ["//visibility:public"], +) diff --git a/third_party/cloud_accelerator_diagnostics/repo.bzl b/third_party/cloud_accelerator_diagnostics/repo.bzl new file mode 100644 index 00000000..661e45ec --- /dev/null +++ b/third_party/cloud_accelerator_diagnostics/repo.bzl @@ -0,0 +1,29 @@ +_COMMIT = "6eccaf47364904d1c55209dd8795e3bf9cac21e1" +_URL_PREFIX = "https://raw.githubusercontent.com/AI-Hypercomputer/cloud-accelerator-diagnostics/" + _COMMIT + "/tpu_info/tpu_info/proto/" + +_PROTOS = { + "tpu_metric_service.proto": "a89d80938c75177ff45fb4c6130f5d71bc5164df473a77948e31b7dedd96a1bb", + "tpu_telemetry.proto": "4ad6096e40de5ba00b6c7badea1218c801069b82be829ec56dc0ce693b5a2505", +} + +def _impl(ctx): + for output, sha256 in _PROTOS.items(): + ctx.download( + url = _URL_PREFIX + output, + sha256 = sha256, + output = output, + ) + ctx.template("BUILD.bazel", ctx.attr.build_file) + +_cloud_accelerator_diagnostics = repository_rule( + implementation = _impl, + attrs = { + "build_file": attr.label(mandatory = True), + }, +) + +def repo(): + _cloud_accelerator_diagnostics( + name = "cloud_accelerator_diagnostics", + build_file = "//third_party/cloud_accelerator_diagnostics:cloud_accelerator_diagnostics.BUILD.bazel", + ) diff --git a/third_party/com_google_sentencepiece/BUILD.bazel b/third_party/com_google_sentencepiece/BUILD.bazel new file mode 100644 index 00000000..598dce0b --- /dev/null +++ b/third_party/com_google_sentencepiece/BUILD.bazel @@ -0,0 +1,39 @@ +load("@rules_cc//cc:cc_library.bzl", "cc_library") +load(":fwd.bzl", "include_fwd") + +include_fwd( + name = "absl_hdrs", + includes = [ + "absl/container/flat_hash_map.h", + "absl/container/flat_hash_set.h", + "absl/flags/flag.h", + "absl/flags/parse.h", + "absl/flags/usage.h", + "absl/strings/match.h", + "absl/strings/numbers.h", + "absl/strings/str_cat.h", + "absl/strings/str_format.h", + "absl/strings/str_join.h", + "absl/strings/str_replace.h", + "absl/strings/str_split.h", + "absl/strings/string_view.h", + "absl/strings/strip.h", + ], +) + +cc_library( + name = "absl", + hdrs = [":absl_hdrs"], + include_prefix = "third_party", + includes = ["."], + visibility = ["//visibility:public"], + deps = [ + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/flags:flag", + "@com_google_absl//absl/flags:parse", + "@com_google_absl//absl/flags:usage", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:string_view", + ], +) diff --git a/third_party/com_google_sentencepiece/fwd.bzl b/third_party/com_google_sentencepiece/fwd.bzl new file mode 100644 index 00000000..dd4836f5 --- /dev/null +++ b/third_party/com_google_sentencepiece/fwd.bzl @@ -0,0 +1,14 @@ +def _include_fwd_impl(ctx): + files = [] + for include in ctx.attr.includes: + f = ctx.actions.declare_file(include) + ctx.actions.write(f, '#include "{}"'.format(include)) + files.append(f) + return [DefaultInfo(files = depset(files))] + +include_fwd = rule( + implementation = _include_fwd_impl, + attrs = { + "includes": attr.string_list(), + }, +) diff --git a/third_party/com_google_sentencepiece/repo.bzl b/third_party/com_google_sentencepiece/repo.bzl new file mode 100644 index 00000000..8b2a11aa --- /dev/null +++ b/third_party/com_google_sentencepiece/repo.bzl @@ -0,0 +1,10 @@ +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +def repo(): + http_archive( + name = "com_google_sentencepiece", + url = "https://github.com/google/sentencepiece/releases/download/v0.2.1/sentencepiece-0.2.1.tar.gz", + strip_prefix = "sentencepiece-0.2.1/sentencepiece", + sha256 = "8138cec27c2f2282f4a34d9a016e3374cd40e5c6e9cb335063db66a0a3b71fad", + build_file = "//third_party/com_google_sentencepiece:sentencepiece.bazel", + ) diff --git a/third_party/com_google_sentencepiece/sentencepiece.bazel b/third_party/com_google_sentencepiece/sentencepiece.bazel new file mode 100644 index 00000000..9d200b0e --- /dev/null +++ b/third_party/com_google_sentencepiece/sentencepiece.bazel @@ -0,0 +1,97 @@ +load("@bazel_skylib//rules:copy_file.bzl", "copy_file") +load("@com_google_protobuf//bazel:cc_proto_library.bzl", "cc_proto_library") +load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library") +load("@rules_cc//cc:cc_library.bzl", "cc_library") + +package( + default_visibility = ["//visibility:public"], + features = [ + "layering_check", + "parse_headers", + ], +) + +licenses(["notice"]) # Apache 2, BSD, MIT + +proto_library( + name = "sentencepiece_proto", + srcs = ["src/sentencepiece.proto"], +) + +cc_proto_library( + name = "sentencepiece_cc_proto", + deps = [":sentencepiece_proto"], +) + +proto_library( + name = "sentencepiece_model_proto", + srcs = ["src/sentencepiece_model.proto"], +) + +cc_proto_library( + name = "sentencepiece_model_cc_proto", + deps = [":sentencepiece_model_proto"], +) + +copy_file( + name = "config_h", + src = "config.h.in", + out = "config.h", + allow_symlink = True, +) + +cc_library( + name = "darts_clone", + hdrs = glob([ + "third_party/darts_clone/*.h", + ]), + includes = ["."], +) + +cc_library( + name = "sentencepiece_processor", + srcs = [ + "src/bpe_model.cc", + "src/char_model.cc", + "src/error.cc", + "src/filesystem.cc", + "src/model_factory.cc", + "src/model_interface.cc", + "src/normalizer.cc", + "src/sentencepiece_processor.cc", + "src/unigram_model.cc", + "src/util.cc", + "src/word_model.cc", + ], + hdrs = [ + ":config_h", + "src/common.h", + "src/bpe_model.h", + "src/char_model.h", + "src/filesystem.h", + "src/freelist.h", + "src/init.h", + "src/model_factory.h", + "src/model_interface.h", + "src/normalizer.h", + "src/sentencepiece_processor.h", + "src/trainer_interface.h", + "src/unigram_model.h", + "src/util.h", + "src/word_model.h", + ], + defines = [ + "_USE_EXTERNAL_PROTOBUF", + "_USE_EXTERNAL_ABSL", + ], + includes = [ + "src", + ], + deps = [ + ":darts_clone", + ":sentencepiece_cc_proto", + ":sentencepiece_model_cc_proto", + "@com_google_protobuf//:protobuf_lite", + "@zml//third_party/com_google_sentencepiece:absl", + ], +) diff --git a/examples/build.zig b/third_party/cuda_tile/BUILD.bazel similarity index 100% rename from examples/build.zig rename to third_party/cuda_tile/BUILD.bazel diff --git a/third_party/cuda_tile/cuda_tile.bazel b/third_party/cuda_tile/cuda_tile.bazel new file mode 100644 index 00000000..bfa768e9 --- /dev/null +++ b/third_party/cuda_tile/cuda_tile.bazel @@ -0,0 +1,450 @@ +# XLA's third_party/cuda_tile/cuda_tile.BUILD, plus the C API library it does +# not build (cuda_tile_capi at the bottom). +load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library", "td_library") +load("@rules_cc//cc:cc_binary.bzl", "cc_binary") +load("@rules_cc//cc:cc_library.bzl", "cc_library") + +cc_binary( + name = "cuda-tile-tblgen", + srcs = glob([ + "tools/cuda-tile-tblgen/*.cpp", + "tools/cuda-tile-tblgen/*.h", + ]), + deps = [ + "@llvm-project//llvm:Support", + "@llvm-project//llvm:TableGen", + "@llvm-project//mlir:MlirTableGenMain", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:TableGen", + ], +) + +exports_files(["LICENSE"]) + +td_library( + name = "CudaTileTdFiles", + srcs = glob(["include/cuda_tile/Dialect/CudaTile/IR/*.td"]), + includes = ["include"], + deps = [ + "@llvm-project//mlir:BuiltinDialectTdFiles", + "@llvm-project//mlir:ControlFlowInterfacesTdFiles", + "@llvm-project//mlir:FunctionInterfacesTdFiles", + "@llvm-project//mlir:InferTypeOpInterfaceTdFiles", + "@llvm-project//mlir:OpBaseTdFiles", + "@llvm-project//mlir:SideEffectInterfacesTdFiles", + "@llvm-project//mlir:ViewLikeInterfaceTdFiles", + ], +) + +gentbl_cc_library( + name = "CudaTileDialectIncGen", + tbl_outs = [ + ( + [ + "-gen-dialect-decls", + "-dialect=cuda_tile", + ], + "include/cuda_tile/Dialect/CudaTile/IR/Dialect.h.inc", + ), + ( + [ + "-gen-dialect-defs", + "-dialect=cuda_tile", + ], + "include/cuda_tile/Dialect/CudaTile/IR/Dialect.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/cuda_tile/Dialect/CudaTile/IR/Dialect.td", + deps = [":CudaTileTdFiles"], +) + +gentbl_cc_library( + name = "CudaTileInterfacesIncGen", + tbl_outs = [ + ( + ["-gen-attr-interface-decls"], + "include/cuda_tile/Dialect/CudaTile/IR/AttrInterfaces.h.inc", + ), + ( + ["-gen-attr-interface-defs"], + "include/cuda_tile/Dialect/CudaTile/IR/AttrInterfaces.cpp.inc", + ), + ( + ["-gen-type-interface-decls"], + "include/cuda_tile/Dialect/CudaTile/IR/TypeInterfaces.h.inc", + ), + ( + ["-gen-type-interface-defs"], + "include/cuda_tile/Dialect/CudaTile/IR/TypeInterfaces.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/cuda_tile/Dialect/CudaTile/IR/Interfaces.td", + deps = [":CudaTileTdFiles"], +) + +gentbl_cc_library( + name = "CudaTileTypesIncGen", + tbl_outs = [ + ( + [ + "-gen-typedef-decls", + "-typedefs-dialect=cuda_tile", + ], + "include/cuda_tile/Dialect/CudaTile/IR/Types.h.inc", + ), + ( + [ + "-gen-typedef-defs", + "-typedefs-dialect=cuda_tile", + ], + "include/cuda_tile/Dialect/CudaTile/IR/Types.cpp.inc", + ), + ( + [ + "-gen-type-constraint-decls", + "-dialect=cuda_tile", + ], + "include/cuda_tile/Dialect/CudaTile/IR/TypeConstraints.h.inc", + ), + ( + [ + "-gen-type-constraint-defs", + "-dialect=cuda_tile", + ], + "include/cuda_tile/Dialect/CudaTile/IR/TypeConstraints.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/cuda_tile/Dialect/CudaTile/IR/Types.td", + deps = [":CudaTileTdFiles"], +) + +gentbl_cc_library( + name = "CudaTileAttrDefsIncGen", + tbl_outs = [ + ( + [ + "-gen-attrdef-decls", + "-attrdefs-dialect=cuda_tile", + ], + "include/cuda_tile/Dialect/CudaTile/IR/AttrDefs.h.inc", + ), + ( + [ + "-gen-attrdef-defs", + "-attrdefs-dialect=cuda_tile", + ], + "include/cuda_tile/Dialect/CudaTile/IR/AttrDefs.cpp.inc", + ), + ( + [ + "-gen-enum-decls", + "-dialect=cuda_tile", + ], + "include/cuda_tile/Dialect/CudaTile/IR/Enums.h.inc", + ), + ( + [ + "-gen-enum-defs", + "-dialect=cuda_tile", + ], + "include/cuda_tile/Dialect/CudaTile/IR/Enums.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/cuda_tile/Dialect/CudaTile/IR/AttrDefs.td", + deps = [":CudaTileTdFiles"], +) + +gentbl_cc_library( + name = "CudaTileAttrCodeGenIncGen", + tbl_outs = [ + ( + ["-gen-enum-attr-list"], + "include/cuda_tile/Dialect/CudaTile/IR/CudaTileEnumAttrs.inc", + ), + ( + ["-gen-gpu-arch-def"], + "include/cuda_tile/Dialect/CudaTile/IR/GpuArchitectures.inc", + ), + ( + ["-gen-gpu-arch-constants"], + "include/cuda_tile/Dialect/CudaTile/IR/GpuArchConstants.inc", + ), + ( + ["-gen-hint-key-accessors"], + "include/cuda_tile/Dialect/CudaTile/IR/HintKeyAccessors.inc", + ), + ( + ["-gen-hint-key-impl"], + "include/cuda_tile/Dialect/CudaTile/IR/HintKeyImpl.inc", + ), + ], + tblgen = ":cuda-tile-tblgen", + td_file = "include/cuda_tile/Dialect/CudaTile/IR/AttrDefs.td", + deps = [":CudaTileTdFiles"], +) + +gentbl_cc_library( + name = "CudaTileRemarksIncGen", + tbl_outs = [ + ( + ["-gen-enum-decls"], + "include/cuda_tile/Dialect/CudaTile/IR/TileIRRemarks.h.inc", + ), + ( + ["-gen-enum-defs"], + "include/cuda_tile/Dialect/CudaTile/IR/TileIRRemarks.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/cuda_tile/Dialect/CudaTile/IR/Remarks.td", + deps = [":CudaTileTdFiles"], +) + +gentbl_cc_library( + name = "CudaTileOpsIncGen", + tbl_outs = [ + ( + ["-gen-op-decls"], + "include/cuda_tile/Dialect/CudaTile/IR/Ops.h.inc", + ), + ( + ["-gen-op-defs"], + "include/cuda_tile/Dialect/CudaTile/IR/Ops.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/cuda_tile/Dialect/CudaTile/IR/Ops.td", + deps = [":CudaTileTdFiles"], +) + +gentbl_cc_library( + name = "CudaTileOpsCanonicalizationIncGen", + tbl_outs = [ + ( + ["-gen-rewriters"], + "lib/Dialect/CudaTile/IR/OpsCanonicalization.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "lib/Dialect/CudaTile/IR/OpsCanonicalization.td", + deps = [ + ":CudaTileTdFiles", + "@llvm-project//mlir:OpBaseTdFiles", + ], +) + +cc_library( + name = "CudaTileDialect", + srcs = glob( + ["lib/Dialect/CudaTile/IR/*.cpp"], + exclude = ["lib/Dialect/CudaTile/IR/CudaTileTesting.cpp"], + ), + hdrs = glob(["include/cuda_tile/Dialect/CudaTile/IR/*.h"]), + includes = [ + "include", + "lib/Dialect/CudaTile/IR", + ], + visibility = ["//visibility:public"], + deps = [ + ":CudaTileAttrCodeGenIncGen", + ":CudaTileAttrDefsIncGen", + ":CudaTileDialectIncGen", + ":CudaTileInterfacesIncGen", + ":CudaTileOpsCanonicalizationIncGen", + ":CudaTileOpsIncGen", + ":CudaTileRemarksIncGen", + ":CudaTileTypesIncGen", + "@llvm-project//llvm:Core", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:ArithDialect", + "@llvm-project//mlir:BytecodeOpInterface", + "@llvm-project//mlir:ControlFlowInterfaces", + "@llvm-project//mlir:FunctionInterfaces", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:InferTypeOpInterface", + "@llvm-project//mlir:InliningUtils", + "@llvm-project//mlir:SideEffectInterfaces", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:ViewLikeInterface", + ], +) + +td_library( + name = "CudaTileTransformsTdFiles", + srcs = ["include/cuda_tile/Dialect/CudaTile/Transforms/Passes.td"], + includes = ["include"], + deps = [ + "@llvm-project//mlir:PassBaseTdFiles", + ], +) + +gentbl_cc_library( + name = "CudaTileTransformsIncGen", + tbl_outs = [ + ( + [ + "-gen-pass-decls", + "-name=CudaTile", + ], + "include/cuda_tile/Dialect/CudaTile/Transforms/Passes.h.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/cuda_tile/Dialect/CudaTile/Transforms/Passes.td", + deps = [":CudaTileTransformsTdFiles"], +) + +cc_library( + name = "CudaTileTransforms", + srcs = glob(["lib/Dialect/CudaTile/Transforms/*.cpp"]), + hdrs = glob(["include/cuda_tile/Dialect/CudaTile/Transforms/*.h"]), + includes = ["include"], + visibility = ["//visibility:public"], + deps = [ + ":CudaTileDialect", + ":CudaTileTransformsIncGen", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:SideEffectInterfaces", + "@llvm-project//mlir:TransformUtils", + ], +) + +gentbl_cc_library( + name = "CudaTileBytecodeOpsIncGen", + tbl_outs = [ + ( + ["-gen-cuda-tile-bytecode"], + "include/cuda_tile/Bytecode/Writer/Bytecode.inc", + ), + ( + ["-gen-cuda-tile-bytecode-reader"], + "include/cuda_tile/Bytecode/Reader/BytecodeReader.inc", + ), + ], + tblgen = ":cuda-tile-tblgen", + td_file = "include/cuda_tile/Dialect/CudaTile/IR/Ops.td", + deps = [":CudaTileTdFiles"], +) + +gentbl_cc_library( + name = "CudaTileBytecodeOpcodesIncGen", + tbl_outs = [ + ( + ["-gen-cuda-tile-opcodes"], + "include/cuda_tile/Bytecode/Common/StaticOpcodes.inc", + ), + ], + tblgen = ":cuda-tile-tblgen", + td_file = "include/cuda_tile/Dialect/CudaTile/IR/BytecodeOpcodes.td", + deps = [":CudaTileTdFiles"], +) + +gentbl_cc_library( + name = "CudaTileBytecodeTypeIncGen", + tbl_outs = [ + ( + ["-gen-cuda-tile-type-bytecode"], + "include/cuda_tile/Bytecode/Writer/TypeBytecode.inc", + ), + ( + ["-gen-cuda-tile-type-bytecode-reader"], + "include/cuda_tile/Bytecode/Reader/TypeBytecodeReader.inc", + ), + ], + tblgen = ":cuda-tile-tblgen", + td_file = "include/cuda_tile/Dialect/CudaTile/IR/BytecodeTypeOpcodes.td", + deps = [":CudaTileTdFiles"], +) + +gentbl_cc_library( + name = "CudaTileBytecodeAttrIncGen", + tbl_outs = [ + ( + ["-gen-cuda-tile-attr-bytecode"], + "include/cuda_tile/Bytecode/Writer/AttrBytecode.inc", + ), + ], + tblgen = ":cuda-tile-tblgen", + td_file = "include/cuda_tile/Dialect/CudaTile/IR/BytecodeAttrOpcodes.td", + deps = [":CudaTileTdFiles"], +) + +cc_library( + name = "CudaTileBytecode", + srcs = glob([ + "lib/Bytecode/**/*.cpp", + "lib/Bytecode/**/*.h", + ]), + hdrs = glob([ + "include/cuda_tile/Bytecode/**/*.h", + ]), + includes = [ + "include", + "include/cuda_tile/Bytecode/Common", + "include/cuda_tile/Bytecode/Reader", + "include/cuda_tile/Bytecode/Writer", + ], + visibility = ["//visibility:public"], + deps = [ + ":CudaTileBytecodeAttrIncGen", + ":CudaTileBytecodeOpcodesIncGen", + ":CudaTileBytecodeOpsIncGen", + ":CudaTileBytecodeTypeIncGen", + ":CudaTileDialect", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:FunctionInterfaces", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:TranslateLib", + ], +) + +cc_library( + name = "CudaTileOptimizer", + srcs = ["lib/Dialect/CudaTile/Optimizer/CudaTileOptimizer.cpp"], + hdrs = ["include/cuda_tile/Dialect/CudaTile/Optimizer/CudaTileOptimizer.h"], + includes = ["include"], + visibility = ["//visibility:public"], + deps = [ + ":CudaTileBytecode", + ":CudaTileDialect", + ":CudaTileTransforms", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Parser", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:Transforms", + ], +) + +# The MLIR C API for the dialect: types, enum attributes, bytecode, passes, and +# the dialect handle `mlirGetDialectHandle__cuda_tile__` that ZML resolves at +# comptime. Mirrors lib/CAPI/CMakeLists.txt + lib/CAPI/Dialect/CMakeLists.txt. +cc_library( + name = "cuda_tile_capi", + srcs = [ + "lib/CAPI/Dialect/CudaTileDialect.cpp", + "lib/CAPI/Dialect/CudaTileOptimizer.cpp", + "lib/CAPI/Registration.cpp", + ], + hdrs = glob(["include/cuda_tile-c/**/*.h"]), + includes = ["include"], + visibility = ["//visibility:public"], + deps = [ + ":CudaTileBytecode", + ":CudaTileDialect", + ":CudaTileOptimizer", + ":CudaTileTransforms", + "@llvm-project//mlir:CAPIIR", + ], + # MLIR_DEFINE_CAPI_DIALECT_REGISTRATION expands to a global static + # initializer that the linker would otherwise dead-strip. + alwayslink = True, +) diff --git a/third_party/cuda_tile/repo.bzl b/third_party/cuda_tile/repo.bzl new file mode 100644 index 00000000..79076c51 --- /dev/null +++ b/third_party/cuda_tile/repo.bzl @@ -0,0 +1,10 @@ +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") + +# v13.4.0 with upstream LLVM compatibility fixes +def repo(): + git_repository( + name = "cuda_tile", + remote = "https://github.com/NVIDIA/cuda-tile.git", + commit = "5b3fa4c0dd04d90edeb935a1887c20bf14a86ebe", + build_file = "//third_party/cuda_tile:cuda_tile.bazel", + ) diff --git a/third_party/cute_ir/BUILD.bazel b/third_party/cute_ir/BUILD.bazel new file mode 100644 index 00000000..e69de29b diff --git a/third_party/cute_ir/cute_ir.bazel b/third_party/cute_ir/cute_ir.bazel new file mode 100644 index 00000000..0c0aed19 --- /dev/null +++ b/third_party/cute_ir/cute_ir.bazel @@ -0,0 +1,104 @@ +load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library", "td_library") +load("@rules_cc//cc:defs.bzl", "cc_library") + +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) # BSD-3-Clause + +# The CuTe dialect of the cutlass_compiler/ tree, as released. The tree that +# builds it (//mlir/dialects/cute_ir) adds the DSL compiler's own attributes, +# types and operations and compiles these sources with them. +CUTE_IR_INCLUDE = "cute_ir/include" + +exports_files([ + "LICENSE.txt", + CUTE_IR_INCLUDE + "/cute_ir/Dialect/Cute/IR/CuteDialect.h", + "cute_ir/lib/Dialect/IR/CuteAttributes.cpp", + "cute_ir/lib/Dialect/IR/CuteDialect.cpp", +]) + + +# cutegen, the CuTe layout algebra, with the `?{div=N}` leaf added by +# cutegen_dynamic_divisibility.patch: a dynamic extent the DSL knows to be a +# multiple of N, which the release's cutegen cannot read. +cc_library( + name = "cutegen", + hdrs = glob(["cutegen/include/cutegen/*.hpp"]), + includes = ["cutegen/include"], + deps = [ + "@llvm-project//llvm:Support", + "@llvm-project//mlir:ArithDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:LLVMDialect", + "@llvm-project//mlir:MathDialect", + "@llvm-project//mlir:NVVMDialect", + "@llvm-project//mlir:SCFDialect", + "@llvm-project//mlir:UBDialect", + ], +) + +td_library( + name = "cute_ir_dialect_td", + srcs = [ + CUTE_IR_INCLUDE + "/cute_ir/Dialect/Cute/IR/CuteAttrs.td", + CUTE_IR_INCLUDE + "/cute_ir/Dialect/Cute/IR/CuteDialect.td", + CUTE_IR_INCLUDE + "/cute_ir/Dialect/Cute/IR/CuteTypeInterfaces.td", + CUTE_IR_INCLUDE + "/cute_ir/Dialect/Cute/IR/CuteTypes.td", + ], + includes = [CUTE_IR_INCLUDE], + deps = [ + "@llvm-project//mlir:BuiltinDialectTdFiles", + "@llvm-project//mlir:DataLayoutInterfacesTdFiles", + "@llvm-project//mlir:InferTypeOpInterfaceTdFiles", + "@llvm-project//mlir:OpBaseTdFiles", + "@llvm-project//mlir:SideEffectInterfacesTdFiles", + ], +) + +gentbl_cc_library( + name = "cute_ir_dialect_inc_gen", + includes = [CUTE_IR_INCLUDE], + tbl_outs = [ + ( + ["-gen-dialect-decls", "-dialect=cute"], + CUTE_IR_INCLUDE + "/cute_ir/Dialect/Cute/IR/CuteDialect.h.inc", + ), + ( + ["-gen-dialect-defs", "-dialect=cute"], + CUTE_IR_INCLUDE + "/cute_ir/Dialect/Cute/IR/CuteDialect.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = CUTE_IR_INCLUDE + "/cute_ir/Dialect/Cute/IR/CuteDialect.td", + deps = [":cute_ir_dialect_td"], +) + +gentbl_cc_library( + name = "cute_ir_type_interfaces_inc_gen", + includes = [CUTE_IR_INCLUDE], + tbl_outs = [ + ( + ["-gen-type-interface-decls"], + CUTE_IR_INCLUDE + "/cute_ir/Dialect/Cute/IR/CuteTypeInterfaces.h.inc", + ), + ( + ["-gen-type-interface-defs"], + CUTE_IR_INCLUDE + "/cute_ir/Dialect/Cute/IR/CuteTypeInterfaces.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = CUTE_IR_INCLUDE + "/cute_ir/Dialect/Cute/IR/CuteTypeInterfaces.td", + deps = [":cute_ir_dialect_td"], +) + +# The header alone: the attribute, type and operation class declarations it +# includes are generated by the building tree, which adds the compiler's own. +cc_library( + name = "cute_ir_dialect_header", + hdrs = [CUTE_IR_INCLUDE + "/cute_ir/Dialect/Cute/IR/CuteDialect.h"], + includes = [CUTE_IR_INCLUDE], + deps = [ + ":cute_ir_dialect_inc_gen", + ":cute_ir_type_interfaces_inc_gen", + ], +) diff --git a/third_party/cute_ir/cutegen_dynamic_divisibility.patch b/third_party/cute_ir/cutegen_dynamic_divisibility.patch new file mode 100644 index 00000000..4f479d60 --- /dev/null +++ b/third_party/cute_ir/cutegen_dynamic_divisibility.patch @@ -0,0 +1,243 @@ +--- a/cutlass_compiler/cutegen/include/cutegen/dynamic.hpp ++++ b/cutlass_compiler/cutegen/include/cutegen/dynamic.hpp +@@ -73,13 +73,20 @@ + * @brief Width (in bits) of the associated dynamic integer + */ + int32_t width = default_dynamic_int_bitwidth(); ++ /** ++ * @brief Known positive divisor of the associated dynamic integer; 1 when ++ * nothing is known. Spelled `div=N` in the dynamic leaf's properties. ++ */ ++ static_int_t divisibility = 1; + /// Constructor + constexpr dynamic_int_properties_t() = default; + explicit dynamic_int_properties_t(int32_t w); ++ dynamic_int_properties_t(int32_t w, static_int_t div); + /// Equality operators + bool operator==(const dynamic_int_properties_t& rhs) const; + bool operator!=(const dynamic_int_properties_t& rhs) const; + dynamic_int_properties_t with_width(int32_t w) const; ++ dynamic_int_properties_t with_divisibility(static_int_t div) const; + bool is_default() const; + bool are_compatible(static_int_t v) const; + static_int_t minimum() const; +@@ -475,11 +482,25 @@ + static std::string to_str(const dynamic_t& d) + { + std::string s("?"); +- if(d.get_width() != default_dynamic_int_bitwidth()) ++ const bool has_width = d.get_width() != default_dynamic_int_bitwidth(); ++ const bool has_div = d.get_properties().divisibility != 1; ++ if(has_width || has_div) + { + s.append("{"); +- s.append("i"); +- s.append(std::to_string(d.get_width())); ++ if(has_width) ++ { ++ s.append("i"); ++ s.append(std::to_string(d.get_width())); ++ } ++ if(has_div) ++ { ++ if(has_width) ++ { ++ s.append(" "); ++ } ++ s.append("div="); ++ s.append(std::to_string(d.get_properties().divisibility)); ++ } + s.append("}"); + } + return s; +@@ -553,6 +574,22 @@ + return; + } + } ++ else if(kw.value() == "div") ++ { ++ auto eq = parse_type(); ++ if(!eq.has_value()) ++ { ++ // Error: missing '=' after 'div' ++ return; ++ } ++ auto div = parse_type(); ++ if(!div.has_value() || div.value() <= 0) ++ { ++ // Error: no positive value for the divisor ++ return; ++ } ++ prop.divisibility = div.value(); ++ } + else + { + // Error: an unexpected keyword was found +@@ -572,6 +609,7 @@ + static void encode_value(const dynamic_t& d, TBuffer& buffer) + { + encode(buffer, d.get_width()); ++ encode(buffer, d.get_properties().divisibility); + } + }; + +@@ -586,7 +624,10 @@ + if(!w.has_value() || + (widths.end() == std::find(widths.begin(), widths.end(), w.value()))) + return std::nullopt; +- return dynamic_t(w.value()); ++ const auto div = decode(it, end); ++ if(!div.has_value() || div.value() <= 0) ++ return std::nullopt; ++ return dynamic_t(dynamic_int_properties_t(w.value(), div.value())); + } + }; + +@@ -816,45 +857,72 @@ + struct property_policy_c_t + { + using int_t = static_int_t; ++ // The divisibility a result keeps: a sum or difference is a multiple of ++ // the gcd of its operands' divisors, a product of their product, and a ++ // static operand counts as its own absolute value. ++ static static_int_t gcd_div(static_int_t a, static_int_t b) ++ { ++ a = a < 0 ? -a : a; ++ b = b < 0 ? -b : b; ++ while(b != 0) ++ { ++ static_int_t t = a % b; ++ a = b; ++ b = t; ++ } ++ return a == 0 ? 1 : a; ++ } ++ static dynamic_int_properties_t with_div(int32_t width, static_int_t div) ++ { ++ return dynamic_int_properties_t(width, div <= 0 ? 1 : div); ++ } + //-------------------------------------------------------------------------- + // resolve_add() + static arith_resolve_t resolve_add(const dynamic_t& lhs, const dynamic_t& rhs) + { + auto res_width = std::max(lhs.get_width(), rhs.get_width()); +- return arith_op_desc_t(dynamic_int_properties_t(res_width), res_width); ++ auto div = gcd_div(lhs.get_properties().divisibility, ++ rhs.get_properties().divisibility); ++ return arith_op_desc_t(with_div(res_width, div), res_width); + } + template , bool> = true> + static arith_resolve_t resolve_add(const dynamic_t& lhs, TInt rhs) + { + auto res_width = std::max(lhs.get_width(), in_range(rhs) ? 32 : 64); +- return arith_op_desc_t(dynamic_int_properties_t(res_width), res_width); ++ auto div = gcd_div(lhs.get_properties().divisibility, static_int_t(rhs)); ++ return arith_op_desc_t(with_div(res_width, div), res_width); + } + static arith_resolve_t resolve_sub(const dynamic_t& lhs, const dynamic_t& rhs) + { + auto res_width = std::max(lhs.get_width(), rhs.get_width()); +- return arith_op_desc_t(dynamic_int_properties_t(res_width), res_width); ++ auto div = gcd_div(lhs.get_properties().divisibility, ++ rhs.get_properties().divisibility); ++ return arith_op_desc_t(with_div(res_width, div), res_width); + } + template , bool> = true> + static arith_resolve_t resolve_sub(const dynamic_t& lhs, TInt rhs) + { + auto res_width = std::max(lhs.get_width(), in_range(rhs) ? 32 : 64); +- return arith_op_desc_t(dynamic_int_properties_t(res_width), res_width); ++ auto div = gcd_div(lhs.get_properties().divisibility, static_int_t(rhs)); ++ return arith_op_desc_t(with_div(res_width, div), res_width); + } + template , bool> = true> + static arith_resolve_t resolve_sub(TInt lhs, const dynamic_t& rhs) + { + auto res_width = std::max(rhs.get_width(), in_range(lhs) ? 32 : 64); +- return arith_op_desc_t(dynamic_int_properties_t(res_width), res_width); ++ auto div = gcd_div(rhs.get_properties().divisibility, static_int_t(lhs)); ++ return arith_op_desc_t(with_div(res_width, div), res_width); + } + //-------------------------------------------------------------------------- + // resolve_mul() + static arith_resolve_t resolve_mul(const dynamic_t& lhs, const dynamic_t& rhs) + { + auto res_width = std::max(lhs.get_width(), rhs.get_width()); +- return arith_op_desc_t(dynamic_int_properties_t(res_width), res_width); ++ auto div = lhs.get_properties().divisibility * rhs.get_properties().divisibility; ++ return arith_op_desc_t(with_div(res_width, div), res_width); + } + template , bool> = true> +@@ -870,7 +938,9 @@ + return src_operand_index_t(0); + } + auto res_width = std::max(lhs.get_width(), in_range(rhs) ? 32 : 64); +- return arith_op_desc_t(dynamic_int_properties_t(res_width), res_width); ++ auto div = lhs.get_properties().divisibility * ++ static_int_t(rhs < 0 ? -rhs : rhs); ++ return arith_op_desc_t(with_div(res_width, div), res_width); + } + //-------------------------------------------------------------------------- + // resolve_mod() +@@ -939,7 +1009,9 @@ + return src_operand_index_t(0); + } + auto res_width = std::max(lhs.get_width(), in_range(rhs) ? 32 : 64); +- return arith_op_desc_t(dynamic_int_properties_t(res_width), res_width); ++ auto d = lhs.get_properties().divisibility; ++ auto r = static_int_t(rhs < 0 ? -rhs : rhs); ++ return arith_op_desc_t(with_div(res_width, d % r == 0 ? d / r : 1), res_width); + } + template , bool> = true> +@@ -1259,12 +1331,18 @@ + + inline dynamic_int_properties_t::dynamic_int_properties_t(int32_t w) : + width(w) ++{ ++} ++ ++inline dynamic_int_properties_t::dynamic_int_properties_t(int32_t w, static_int_t div) : ++ width(w), ++ divisibility(div) + { + } + + inline bool dynamic_int_properties_t::operator==(const dynamic_int_properties_t& rhs) const + { +- return (width == rhs.width); ++ return (width == rhs.width) && (divisibility == rhs.divisibility); + } + + inline bool dynamic_int_properties_t::operator!=(const dynamic_int_properties_t& rhs) const +@@ -1284,9 +1362,16 @@ + return r; + } + ++inline dynamic_int_properties_t dynamic_int_properties_t::with_divisibility(static_int_t div) const ++{ ++ dynamic_int_properties_t r = *this; ++ r.divisibility = div; ++ return r; ++} ++ + inline bool dynamic_int_properties_t::are_compatible(static_int_t v) const + { +- return in_range_of_width(v, width); ++ return in_range_of_width(v, width) && (v % divisibility == 0); + } + + inline static_int_t dynamic_int_properties_t::minimum() const +@@ -1313,7 +1398,7 @@ + + inline bool dynamic_int_properties_t::is_constrained() const + { +- return false; ++ return divisibility != 1; + } + + inline dynamic_t::dynamic_t(int32_t width_) : diff --git a/third_party/cute_ir/repo.bzl b/third_party/cute_ir/repo.bzl new file mode 100644 index 00000000..6beb5df0 --- /dev/null +++ b/third_party/cute_ir/repo.bzl @@ -0,0 +1,22 @@ +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") + +def repo(): + # The cutlass_compiler/ tree of the CUTLASS v4.8.0 release: the open-source + # part of the CuTe dialect that mlir/dialects/cute_ir builds on. + git_repository( + name = "cute_ir", + remote = "https://github.com/NVIDIA/cutlass.git", + commit = "098de2a652cf8f00fd70b2df54051c7eccbb855a", + build_file = Label("//third_party/cute_ir:cute_ir.bazel"), + strip_prefix = "cutlass_compiler", + patches = [ + # `cute.slice` upstream only infers tuple and layout results; the + # MoE kernels slice memrefs and coordinate tensors with explicit + # result types. + Label("//third_party/cute_ir:slice_tensor.patch"), + # The DSL spells a dynamic leaf it knows to be a multiple of N as + # `?{div=N}`; the release's cutegen reads only the width. + Label("//third_party/cute_ir:cutegen_dynamic_divisibility.patch"), + ], + patch_args = ["-p2"], + ) diff --git a/third_party/cute_ir/slice_tensor.patch b/third_party/cute_ir/slice_tensor.patch new file mode 100644 index 00000000..173a795b --- /dev/null +++ b/third_party/cute_ir/slice_tensor.patch @@ -0,0 +1,45 @@ +diff --git a/cutlass_compiler/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteOps.td b/cutlass_compiler/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteOps.td +--- a/cutlass_compiler/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteOps.td ++++ b/cutlass_compiler/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteOps.td +@@ -2990,7 +2990,7 @@ + + def Cute_SliceOp : Cute_Op<"slice", [ + Pure, +- DeclareOpInterfaceMethods ++ DeclareOpInterfaceMethods + ]> { + let cuteSpecGroup = "LayoutAlgebra"; + let summary = "Slice a value, keeping modes where the coordinate is `_`"; +@@ -3154,11 +3154,11 @@ + }]; + + let arguments = (ins +- Cute_Arg:$input, ++ Cute_Arg:$input, + Cute_Arg:$coord + ); + let results = (outs +- Cute_Arg:$result ++ Cute_Arg:$result + ); + let assemblyFormat = [{ `(` operands `)` attr-dict `:` type(operands) }]; + } +diff --git a/cutlass_compiler/cute_ir/lib/Dialect/IR/CuteOps.cpp b/cutlass_compiler/cute_ir/lib/Dialect/IR/CuteOps.cpp +--- a/cutlass_compiler/cute_ir/lib/Dialect/IR/CuteOps.cpp ++++ b/cutlass_compiler/cute_ir/lib/Dialect/IR/CuteOps.cpp +@@ -1327,6 +1327,15 @@ + }); + } + ++LogicalResult SliceOp::refineReturnTypes( ++ MLIRContext *, std::optional, ValueRange, DictionaryAttr, ++ PropertyRef, RegionRange, SmallVectorImpl &) { ++ // Tensor views are private CuTe-DSL types. Their result type is supplied ++ // explicitly by the front end, so no public layout-algebra refinement is ++ // required here. ++ return success(); ++} ++ + //===----------------------------------------------------------------------===// + // DiceOp (cute.dice) + //===----------------------------------------------------------------------===// diff --git a/third_party/cutlass/cutlass.bazel b/third_party/cutlass/cutlass.bazel new file mode 100644 index 00000000..e009a6d4 --- /dev/null +++ b/third_party/cutlass/cutlass.bazel @@ -0,0 +1,111 @@ +load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library", "td_library") +load("@rules_cc//cc:defs.bzl", "cc_library") + +td_library( + name = "dialect_td", + srcs = [ + "include/cute_ir/Dialect/Cute/IR/CuteAttrs.td", + "include/cute_ir/Dialect/Cute/IR/CuteDialect.td", + "include/cute_ir/Dialect/Cute/IR/CuteOps.td", + "include/cute_ir/Dialect/Cute/IR/CuteTypeInterfaces.td", + "include/cute_ir/Dialect/Cute/IR/CuteTypes.td", + ], + includes = ["include"], + deps = [ + "@llvm-project//mlir:BuiltinDialectTdFiles", + "@llvm-project//mlir:DataLayoutInterfacesTdFiles", + "@llvm-project//mlir:InferTypeOpInterfaceTdFiles", + "@llvm-project//mlir:OpBaseTdFiles", + "@llvm-project//mlir:SideEffectInterfacesTdFiles", + ], +) + +gentbl_cc_library( + name = "dialect_inc_gen", + strip_include_prefix = ".", + tbl_outs = [ + ( + ["-gen-dialect-decls", "-dialect=cute"], + "include/cute_ir/Dialect/Cute/IR/CuteDialect.h.inc", + ), + ( + ["-gen-dialect-defs", "-dialect=cute"], + "include/cute_ir/Dialect/Cute/IR/CuteDialect.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/cute_ir/Dialect/Cute/IR/CuteDialect.td", + deps = [":dialect_td"], +) + +gentbl_cc_library( + name = "type_interfaces_inc_gen", + strip_include_prefix = ".", + tbl_outs = [ + ( + ["-gen-type-interface-decls"], + "include/cute_ir/Dialect/Cute/IR/CuteTypeInterfaces.h.inc", + ), + ( + ["-gen-type-interface-defs"], + "include/cute_ir/Dialect/Cute/IR/CuteTypeInterfaces.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/cute_ir/Dialect/Cute/IR/CuteTypeInterfaces.td", + deps = [":dialect_td"], +) + +gentbl_cc_library( + name = "attrs_inc_gen", + strip_include_prefix = ".", + tbl_outs = [ + ( + ["-gen-attrdef-decls", "-attrdefs-dialect=cute"], + "include/cute_ir/Dialect/Cute/IR/CuteAttrs.h.inc", + ), + ( + ["-gen-attrdef-defs", "-attrdefs-dialect=cute"], + "include/cute_ir/Dialect/Cute/IR/CuteAttrs.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/cute_ir/Dialect/Cute/IR/CuteAttrs.td", + deps = [":dialect_td"], +) + +gentbl_cc_library( + name = "ops_inc_gen", + strip_include_prefix = ".", + tbl_outs = [ + ( + ["-gen-op-decls"], + "include/cute_ir/Dialect/Cute/IR/CuteOps.h.inc", + ), + ( + ["-gen-op-defs"], + "include/cute_ir/Dialect/Cute/IR/CuteOps.cpp.inc", + ), + ], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/cute_ir/Dialect/Cute/IR/CuteOps.td", + deps = [":dialect_td"], +) + +cc_library( + name = "headers", + hdrs = [ + hdr for hdr in [ + "include/cute_ir/Dialect/Cute/IR/CuteDialect.h", + "include/cute_ir/Dialect/Cute/IR/CuteOps.h", + "include/cute_ir/Dialect/Cute/IR/CuteTypeInterfaces.h", + ] + ], + strip_include_prefix = "include", + deps = [ + ":attrs_inc_gen", + ":dialect_inc_gen", + ":ops_inc_gen", + ":type_interfaces_inc_gen", + ], +) diff --git a/third_party/flashinfer_cutlass_moe/BUILD.bazel b/third_party/flashinfer_cutlass_moe/BUILD.bazel new file mode 100644 index 00000000..b2ddeede --- /dev/null +++ b/third_party/flashinfer_cutlass_moe/BUILD.bazel @@ -0,0 +1,4 @@ +exports_files([ + "flashinfer_cutlass_moe.BUILD.bazel", + "repo.bzl", +]) diff --git a/third_party/flashinfer_cutlass_moe/flashinfer_cutlass_moe.BUILD.bazel b/third_party/flashinfer_cutlass_moe/flashinfer_cutlass_moe.BUILD.bazel new file mode 100644 index 00000000..780ac0d2 --- /dev/null +++ b/third_party/flashinfer_cutlass_moe/flashinfer_cutlass_moe.BUILD.bazel @@ -0,0 +1,33 @@ +load("@rules_cc//cc:cc_library.bzl", "cc_library") + +package(default_visibility = ["//visibility:public"]) + +filegroup( + name = "lib_sm100", + srcs = ["lib/libflashinfer_cutlass_moe_sm100.so"], +) + +filegroup( + name = "lib_sm103", + srcs = ["lib/libflashinfer_cutlass_moe_sm103.so"], +) + +filegroup( + name = "lib_sm120", + srcs = ["lib/libflashinfer_cutlass_moe_sm120.so"], +) + +filegroup( + name = "libraries", + srcs = [ + ":lib_sm100", + ":lib_sm103", + ":lib_sm120", + ], +) + +cc_library( + name = "c", + hdrs = ["include/flashinfer_cutlass_moe.h"], + includes = ["include"], +) diff --git a/third_party/flashinfer_cutlass_moe/repo.bzl b/third_party/flashinfer_cutlass_moe/repo.bzl new file mode 100644 index 00000000..2f897c2d --- /dev/null +++ b/third_party/flashinfer_cutlass_moe/repo.bzl @@ -0,0 +1,38 @@ +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +_RELEASE = "cutlass-moe-v0.2.0" + +_ASSETS = { + "amd64": { + "sha256": "5c3a29da0fd91e2281d046eb2e467a7a40aeb24a0be8ba0487006356d45fecc4", + "url": "https://github.com/zml/flashinfer/releases/download/{release}/flashinfer-cutlass-moe_linux-amd64.tar.gz", + }, + "arm64": { + "sha256": "60b6fdbd193e11c24589add1eb206f59a3900cde4760e53719d6b2add492343e", + "url": "https://github.com/zml/flashinfer/releases/download/{release}/flashinfer-cutlass-moe_linux-arm64.tar.gz", + }, +} + +def _archive(name, arch): + asset = _ASSETS[arch] + + http_archive( + name = name, + urls = [ + asset["url"].format(release = _RELEASE), + ], + sha256 = asset["sha256"], + build_file = Label( + "//third_party/flashinfer_cutlass_moe:flashinfer_cutlass_moe.BUILD.bazel", + ), + ) + +def repo(): + _archive( + name = "flashinfer_cutlass_moe_linux_amd64", + arch = "amd64", + ) + _archive( + name = "flashinfer_cutlass_moe_linux_arm64", + arch = "arm64", + ) diff --git a/third_party/flydsl/BUILD.bazel b/third_party/flydsl/BUILD.bazel new file mode 100644 index 00000000..e69de29b diff --git a/third_party/flydsl/atomic_exchange.patch b/third_party/flydsl/atomic_exchange.patch new file mode 100644 index 00000000..7de76db1 --- /dev/null +++ b/third_party/flydsl/atomic_exchange.patch @@ -0,0 +1,25 @@ +diff --git a/include/flydsl/Dialect/Fly/IR/FlyAttrDefs.td b/include/flydsl/Dialect/Fly/IR/FlyAttrDefs.td +--- a/include/flydsl/Dialect/Fly/IR/FlyAttrDefs.td ++++ b/include/flydsl/Dialect/Fly/IR/FlyAttrDefs.td +@@ -43,7 +43,8 @@ + I32EnumAttrCase<"And", 3, "and">, + I32EnumAttrCase<"Or", 4, "or">, + I32EnumAttrCase<"Inc", 5, "inc">, +- I32EnumAttrCase<"Dec", 6, "dec"> ++ I32EnumAttrCase<"Dec", 6, "dec">, ++ I32EnumAttrCase<"Exchange", 7, "exchange"> + ]>; + def Fly_AtomicOpAttr : Fly_EnumAttr {} + +diff --git a/lib/Dialect/Fly/IR/FlyUniversalOps.cpp b/lib/Dialect/Fly/IR/FlyUniversalOps.cpp +--- a/lib/Dialect/Fly/IR/FlyUniversalOps.cpp ++++ b/lib/Dialect/Fly/IR/FlyUniversalOps.cpp +@@ -228,6 +228,8 @@ + return isFloat ? std::nullopt : std::optional(LLVM::AtomicBinOp::uinc_wrap); + case AtomicOp::Dec: + return isFloat ? std::nullopt : std::optional(LLVM::AtomicBinOp::udec_wrap); ++ case AtomicOp::Exchange: ++ return LLVM::AtomicBinOp::xchg; + } + return std::nullopt; + } diff --git a/third_party/flydsl/flydsl.bazel b/third_party/flydsl/flydsl.bazel new file mode 100644 index 00000000..7c8cb7e9 --- /dev/null +++ b/third_party/flydsl/flydsl.bazel @@ -0,0 +1,423 @@ +load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library", "td_library") +load("@rules_cc//cc:cc_library.bzl", "cc_library") + +package( + default_visibility = ["//visibility:public"], + features = [ + "-parse_headers", + "-use_header_modules", + ], + licenses = ["notice"], +) + +exports_files(["LICENSE"]) + +td_library( + name = "td_files", + srcs = glob(["include/flydsl/**/*.td"]), + includes = ["include"], + deps = [ + "@llvm-project//mlir:ArithOpsTdFiles", + "@llvm-project//mlir:BuiltinDialectBytecodeTdFiles", + "@llvm-project//mlir:BytecodeOpInterfaceTdFiles", + "@llvm-project//mlir:FunctionInterfacesTdFiles", + "@llvm-project//mlir:GPUOpsTdFiles", + "@llvm-project//mlir:InferTypeOpInterfaceTdFiles", + "@llvm-project//mlir:LLVMOpsTdFiles", + "@llvm-project//mlir:OpBaseTdFiles", + "@llvm-project//mlir:PassBaseTdFiles", + "@llvm-project//mlir:ROCDLOpsTdFiles", + "@llvm-project//mlir:SCFTdFiles", + "@llvm-project//mlir:SideEffectInterfacesTdFiles", + "@llvm-project//mlir:VectorOpsTdFiles", + ], +) + +gentbl_cc_library( + name = "fly_dialect_inc_gen", + tbl_outs = { + "include/flydsl/Dialect/Fly/IR/FlyDialect.cpp.inc": ["--gen-dialect-defs"], + "include/flydsl/Dialect/Fly/IR/FlyDialect.h.inc": ["--gen-dialect-decls"], + }, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/flydsl/Dialect/Fly/IR/FlyDialect.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "fly_interfaces_inc_gen", + tbl_outs = { + "include/flydsl/Dialect/Fly/IR/FlyAttrInterfaces.cpp.inc": ["--gen-attr-interface-defs"], + "include/flydsl/Dialect/Fly/IR/FlyAttrInterfaces.h.inc": ["--gen-attr-interface-decls"], + "include/flydsl/Dialect/Fly/IR/FlyOpInterfaces.cpp.inc": ["--gen-op-interface-defs"], + "include/flydsl/Dialect/Fly/IR/FlyOpInterfaces.h.inc": ["--gen-op-interface-decls"], + "include/flydsl/Dialect/Fly/IR/FlyTypeInterfaces.cpp.inc": ["--gen-type-interface-defs"], + "include/flydsl/Dialect/Fly/IR/FlyTypeInterfaces.h.inc": ["--gen-type-interface-decls"], + }, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/flydsl/Dialect/Fly/IR/FlyInterfaces.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "fly_types_inc_gen", + tbl_outs = { + "include/flydsl/Dialect/Fly/IR/FlyTypeConstraints.cpp.inc": [ + "--gen-type-constraint-defs", + "--typedefs-dialect=fly", + ], + "include/flydsl/Dialect/Fly/IR/FlyTypeConstraints.h.inc": [ + "--gen-type-constraint-decls", + "--typedefs-dialect=fly", + ], + "include/flydsl/Dialect/Fly/IR/FlyTypeDefs.cpp.inc": [ + "--gen-typedef-defs", + "--typedefs-dialect=fly", + ], + "include/flydsl/Dialect/Fly/IR/FlyTypeDefs.h.inc": [ + "--gen-typedef-decls", + "--typedefs-dialect=fly", + ], + }, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/flydsl/Dialect/Fly/IR/FlyTypeDefs.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "fly_attrs_inc_gen", + tbl_outs = { + "include/flydsl/Dialect/Fly/IR/FlyAttrConstraints.cpp.inc": [ + "--gen-attr-constraint-defs", + "--attrdefs-dialect=fly", + ], + "include/flydsl/Dialect/Fly/IR/FlyAttrConstraints.h.inc": [ + "--gen-attr-constraint-decls", + "--attrdefs-dialect=fly", + ], + "include/flydsl/Dialect/Fly/IR/FlyAttrDefs.cpp.inc": [ + "--gen-attrdef-defs", + "--attrdefs-dialect=fly", + ], + "include/flydsl/Dialect/Fly/IR/FlyAttrDefs.h.inc": [ + "--gen-attrdef-decls", + "--attrdefs-dialect=fly", + ], + "include/flydsl/Dialect/Fly/IR/FlyEnums.cpp.inc": ["--gen-enum-defs"], + "include/flydsl/Dialect/Fly/IR/FlyEnums.h.inc": ["--gen-enum-decls"], + }, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/flydsl/Dialect/Fly/IR/FlyAttrDefs.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "fly_ops_inc_gen", + tbl_outs = { + "include/flydsl/Dialect/Fly/IR/FlyOps.cpp.inc": ["--gen-op-defs"], + "include/flydsl/Dialect/Fly/IR/FlyOps.h.inc": ["--gen-op-decls"], + }, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/flydsl/Dialect/Fly/IR/FlyOps.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "fly_passes_inc_gen", + tbl_outs = { + "include/flydsl/Dialect/Fly/Transforms/Passes.h.inc": [ + "--gen-pass-decls", + "--name=Fly", + ], + }, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/flydsl/Dialect/Fly/Transforms/Passes.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "fly_int_tuple_lowering_inc_gen", + tbl_outs = { + "include/flydsl/Dialect/Fly/Transforms/IntTupleLowering.cpp.inc": ["--gen-rewriters"], + }, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/flydsl/Dialect/Fly/Transforms/IntTupleLowering.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "fly_layout_lowering_inc_gen", + tbl_outs = { + "include/flydsl/Dialect/Fly/Transforms/LayoutLowering.cpp.inc": ["--gen-rewriters"], + }, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/flydsl/Dialect/Fly/Transforms/LayoutLowering.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "fly_memref_lowering_inc_gen", + tbl_outs = { + "include/flydsl/Dialect/Fly/Transforms/MemrefLowering.cpp.inc": ["--gen-rewriters"], + }, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/flydsl/Dialect/Fly/Transforms/MemrefLowering.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "fly_rocdl_dialect_inc_gen", + tbl_outs = { + "include/flydsl/Dialect/FlyROCDL/IR/Dialect.cpp.inc": ["--gen-dialect-defs"], + "include/flydsl/Dialect/FlyROCDL/IR/Dialect.h.inc": ["--gen-dialect-decls"], + }, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/flydsl/Dialect/FlyROCDL/IR/Dialect.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "fly_rocdl_atom_inc_gen", + tbl_outs = { + "include/flydsl/Dialect/FlyROCDL/IR/Atom.cpp.inc": [ + "--gen-typedef-defs", + "--typedefs-dialect=fly_rocdl", + ], + "include/flydsl/Dialect/FlyROCDL/IR/Atom.h.inc": [ + "--gen-typedef-decls", + "--typedefs-dialect=fly_rocdl", + ], + "include/flydsl/Dialect/FlyROCDL/IR/AtomStateEnums.cpp.inc": [ + "--gen-enum-defs", + "--typedefs-dialect=fly_rocdl", + ], + "include/flydsl/Dialect/FlyROCDL/IR/AtomStateEnums.h.inc": [ + "--gen-enum-decls", + "--typedefs-dialect=fly_rocdl", + ], + }, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/flydsl/Dialect/FlyROCDL/IR/Atom.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "fly_rocdl_attrs_inc_gen", + tbl_outs = { + "include/flydsl/Dialect/FlyROCDL/IR/AttrDefs.cpp.inc": [ + "--gen-attrdef-defs", + "--attrdefs-dialect=fly_rocdl", + ], + "include/flydsl/Dialect/FlyROCDL/IR/AttrDefs.h.inc": [ + "--gen-attrdef-decls", + "--attrdefs-dialect=fly_rocdl", + ], + "include/flydsl/Dialect/FlyROCDL/IR/AttrEnums.cpp.inc": [ + "--gen-enum-defs", + "--attrdefs-dialect=fly_rocdl", + ], + "include/flydsl/Dialect/FlyROCDL/IR/AttrEnums.h.inc": [ + "--gen-enum-decls", + "--attrdefs-dialect=fly_rocdl", + ], + }, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/flydsl/Dialect/FlyROCDL/IR/AttrDefs.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "fly_rocdl_ops_inc_gen", + tbl_outs = { + "include/flydsl/Dialect/FlyROCDL/IR/Ops.cpp.inc": ["--gen-op-defs"], + "include/flydsl/Dialect/FlyROCDL/IR/Ops.h.inc": ["--gen-op-decls"], + }, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/flydsl/Dialect/FlyROCDL/IR/Ops.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "fly_to_rocdl_passes_inc_gen", + tbl_outs = { + "include/flydsl/Conversion/FlyToROCDL/Passes.h.inc": [ + "--gen-pass-decls", + "--name=FlyToROCDL", + ], + }, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/flydsl/Conversion/FlyToROCDL/Passes.td", + deps = [":td_files"], +) + +gentbl_cc_library( + name = "fly_rocdl_passes_inc_gen", + tbl_outs = { + "include/flydsl/Dialect/FlyROCDL/Transforms/Passes.h.inc": [ + "--gen-pass-decls", + "--name=FlyROCDL", + ], + }, + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "include/flydsl/Dialect/FlyROCDL/Transforms/Passes.td", + deps = [":td_files"], +) + +cc_library( + name = "FlyDialect", + srcs = [ + "lib/Dialect/Fly/IR/FlyAttrDefs.cpp", + "lib/Dialect/Fly/IR/FlyDialect.cpp", + "lib/Dialect/Fly/IR/FlyOps.cpp", + "lib/Dialect/Fly/IR/FlyTypeDefs.cpp", + "lib/Dialect/Fly/IR/FlyUniversalOps.cpp", + "lib/Dialect/Fly/Utils/IntTupleUtils.cpp", + "lib/Dialect/Fly/Utils/IntUtils.cpp", + "lib/Dialect/Fly/Utils/NormalForm.cpp", + "lib/Dialect/Fly/Utils/PointerUtils.cpp", + ], + hdrs = glob( + [ + "include/flydsl/Dialect/Fly/IR/*.h", + "include/flydsl/Dialect/Fly/Utils/*.h", + "include/flydsl/Dialect/Fly/Utils/*.inc", + ], + exclude = ["include/flydsl/Dialect/Fly/IR/FlyLLVMTranslation.h"], + ), + includes = ["include"], + deps = [ + ":fly_attrs_inc_gen", + ":fly_dialect_inc_gen", + ":fly_interfaces_inc_gen", + ":fly_ops_inc_gen", + ":fly_types_inc_gen", + "@llvm-project//llvm:Support", + "@llvm-project//llvm:ir_headers", + "@llvm-project//mlir:ArithDialect", + "@llvm-project//mlir:BytecodeOpInterface", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:GPUDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:InferTypeOpInterface", + "@llvm-project//mlir:LLVMDialect", + "@llvm-project//mlir:SCFDialect", + "@llvm-project//mlir:SideEffectInterfaces", + ], +) + +cc_library( + name = "FlyTransforms", + srcs = glob(["lib/Dialect/Fly/Transforms/*.cpp"]), + hdrs = glob(["include/flydsl/Dialect/Fly/Transforms/*.h"]), + includes = ["include"], + deps = [ + ":FlyDialect", + ":fly_int_tuple_lowering_inc_gen", + ":fly_layout_lowering_inc_gen", + ":fly_memref_lowering_inc_gen", + ":fly_passes_inc_gen", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:ArithDialect", + "@llvm-project//mlir:ArithUtils", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:FuncTransforms", + "@llvm-project//mlir:GPUDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:LLVMDialect", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:SCFDialect", + "@llvm-project//mlir:SCFTransforms", + "@llvm-project//mlir:Transforms", + "@llvm-project//mlir:UBDialect", + "@llvm-project//mlir:VectorDialect", + ], +) + +cc_library( + name = "FlyROCDLDialect", + srcs = [ + "lib/Dialect/FlyROCDL/CDNA3/CopyAtom.cpp", + "lib/Dialect/FlyROCDL/CDNA3/MmaAtom.cpp", + "lib/Dialect/FlyROCDL/CDNA4/CopyAtom.cpp", + "lib/Dialect/FlyROCDL/CDNA4/MmaAtom.cpp", + "lib/Dialect/FlyROCDL/CDNA5/CopyAtom.cpp", + "lib/Dialect/FlyROCDL/CDNA5/TdmAtomBuilder.cpp", + "lib/Dialect/FlyROCDL/CDNA5/TdmGeometry.cpp", + "lib/Dialect/FlyROCDL/Dialect.cpp", + "lib/Dialect/FlyROCDL/GFX11/MmaAtom.cpp", + "lib/Dialect/FlyROCDL/GFX120X/MmaAtom.cpp", + "lib/Dialect/FlyROCDL/GFX1250/CopyAtom.cpp", + "lib/Dialect/FlyROCDL/GFX1250/MmaAtom.cpp", + "lib/Dialect/FlyROCDL/GFX1250/MmaAtomScale.cpp", + "lib/Dialect/FlyROCDL/Ops.cpp", + "lib/Dialect/FlyROCDL/Transforms/ClusterAttr.cpp", + "lib/Dialect/FlyROCDL/Transforms/ExpandOps.cpp", + ], + hdrs = glob([ + "include/flydsl/Dialect/FlyROCDL/**/*.h", + "lib/Dialect/FlyROCDL/**/*.h", + ]), + includes = ["include"], + deps = [ + ":FlyDialect", + ":fly_rocdl_atom_inc_gen", + ":fly_rocdl_attrs_inc_gen", + ":fly_rocdl_dialect_inc_gen", + ":fly_rocdl_ops_inc_gen", + ":fly_rocdl_passes_inc_gen", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:ArithDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:LLVMDialect", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:ROCDLDialect", + "@llvm-project//mlir:SCFDialect", + "@llvm-project//mlir:VectorDialect", + ], +) + +cc_library( + name = "FlyToROCDL", + srcs = [ + "lib/Conversion/FlyToROCDL/FastMathOps.cpp", + "lib/Conversion/FlyToROCDL/FlyToROCDL.cpp", + ], + hdrs = glob(["include/flydsl/Conversion/**/*.h"]), + includes = ["include"], + deps = [ + ":FlyDialect", + ":FlyROCDLDialect", + ":fly_to_rocdl_passes_inc_gen", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:ArithDialect", + "@llvm-project//mlir:ArithUtils", + "@llvm-project//mlir:ControlFlowTransforms", + "@llvm-project//mlir:FuncDialect", + "@llvm-project//mlir:FuncTransforms", + "@llvm-project//mlir:GPUDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:LLVMDialect", + "@llvm-project//mlir:MathDialect", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:ROCDLDialect", + "@llvm-project//mlir:SCFDialect", + "@llvm-project//mlir:SCFTransforms", + "@llvm-project//mlir:Transforms", + "@llvm-project//mlir:VectorDialect", + ], +) + +# ZML addition: the per-type accessors FlyDSL's Python bindings use to derive +# tiled TV layouts and MMA tile sizes. They live under lib/Bindings/Python but +# depend only on the dialect (no nanobind). +cc_library( + name = "TiledOpTraits", + srcs = ["lib/Bindings/Python/TiledOpTraits.cpp"], + hdrs = ["lib/Bindings/Python/TiledOpTraits.h"], + includes = ["lib/Bindings/Python"], + deps = [ + ":FlyDialect", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:IR", + ], +) diff --git a/third_party/flydsl/mlir_legacy_assembly.patch b/third_party/flydsl/mlir_legacy_assembly.patch new file mode 100644 index 00000000..0739e489 --- /dev/null +++ b/third_party/flydsl/mlir_legacy_assembly.patch @@ -0,0 +1,39 @@ +--- a/include/flydsl/Dialect/Fly/IR/FlyDialect.td ++++ b/include/flydsl/Dialect/Fly/IR/FlyDialect.td +@@ -19,6 +19,11 @@ + + let useDefaultTypePrinterParser = 1; + let useDefaultAttributePrinterParser = 1; ++ ++ // ZML custom-call inputs use Fly assembly printed by a separately pinned ++ // MLIR/Fly build. Keep accepting its inherent attributes in attr-dict until ++ // the producer and consumer can migrate to prop-dict together. ++ let useStrictPropertiesInAssemblyFormat = 0; + } + + class Fly_Type traits = []> +@@ -38,7 +43,10 @@ + } + + class Fly_EnumAttr traits = []> +- : EnumAttr; ++ : EnumAttr { ++ // Keep the Fly text ABI independent of MLIR's default enum delimiters. ++ let assemblyFormat = "$value"; ++} + + class Fly_Op traits = []> + : Op { +diff --git a/include/flydsl/Dialect/FlyROCDL/IR/Ops.td b/include/flydsl/Dialect/FlyROCDL/IR/Ops.td +index 2b745065..875d8a69 100644 +--- a/include/flydsl/Dialect/FlyROCDL/IR/Ops.td ++++ b/include/flydsl/Dialect/FlyROCDL/IR/Ops.td +@@ -44,7 +44,7 @@ class FlyROCDL_MakeTiledTdmAtomOp + let results = (outs AnyType:$atom, AnyType:$coordTensor); + + let assemblyFormat = [{ +- $tensor `,` $smemLayout `,` $tiler attr-dict `:` type(operands) ++ $tensor `,` $smemLayout `,` $tiler prop-dict attr-dict `:` type(operands) + }]; + } + diff --git a/third_party/flydsl/mlir_property_ref.patch b/third_party/flydsl/mlir_property_ref.patch new file mode 100644 index 00000000..145bcc1b --- /dev/null +++ b/third_party/flydsl/mlir_property_ref.patch @@ -0,0 +1,128 @@ +diff --git a/lib/Conversion/FlyToROCDL/FlyToROCDL.cpp b/lib/Conversion/FlyToROCDL/FlyToROCDL.cpp +index 80d2fb3b..7981077c 100644 +--- a/lib/Conversion/FlyToROCDL/FlyToROCDL.cpp ++++ b/lib/Conversion/FlyToROCDL/FlyToROCDL.cpp +@@ -13,6 +13,7 @@ + #include "mlir/Dialect/SCF/Transforms/Patterns.h" + #include "mlir/Dialect/Vector/IR/VectorOps.h" + #include "mlir/IR/BuiltinAttributes.h" ++#include "mlir/IR/BuiltinOps.h" + #include "mlir/IR/BuiltinTypes.h" + #include "mlir/Pass/Pass.h" + #include "mlir/Transforms/DialectConversion.h" +@@ -85,10 +86,10 @@ static unsigned getLLVMAlignment(fly::PointerType ptrTy) { + // chosen to avoid collisions with existing globals in the module. Shared by + // the static-LDS `make_ptr` and dynamic-LDS `get_dyn_shared` lowerings. + static LLVM::GlobalOp createLDSGlobal(ConversionPatternRewriter &rewriter, +- gpu::GPUModuleOp moduleOp, Location loc, StringRef prefix, ++ Block *moduleBody, Location loc, StringRef prefix, + int64_t nbytes, int64_t align, unsigned addrSpace) { + llvm::StringSet<> existingNames; +- for (auto globalOp : moduleOp.getBody()->getOps()) ++ for (auto globalOp : moduleBody->getOps()) + existingNames.insert(globalOp.getSymName()); + + unsigned counter = 0; +@@ -96,7 +97,7 @@ static LLVM::GlobalOp createLDSGlobal(ConversionPatternRewriter &rewriter, + prefix, [&](StringRef candidate) { return existingNames.contains(candidate); }, counter); + + OpBuilder::InsertionGuard guard(rewriter); +- rewriter.setInsertionPointToStart(moduleOp.getBody()); ++ rewriter.setInsertionPointToStart(moduleBody); + + auto arrayTy = LLVM::LLVMArrayType::get(IntegerType::get(rewriter.getContext(), 8), nbytes); + auto globalOp = LLVM::GlobalOp::create(rewriter, loc, arrayTy, +@@ -149,12 +150,17 @@ public: + return rewriter.notifyMatchFailure( + op, "shared make_ptr requires allocBytes, allocAlign in dictAttrs"); + +- auto moduleOp = op->getParentOfType(); +- if (!moduleOp) ++ Block *moduleBody = nullptr; ++ if (auto gpuModule = op->getParentOfType()) { ++ moduleBody = gpuModule.getBody(); ++ } else if (auto builtinModule = op->getParentOfType()) { ++ moduleBody = builtinModule.getBody(); ++ } ++ if (!moduleBody) + return op->emitError("shared make_ptr must be inside a gpu.module"); + + LLVM::GlobalOp global = +- createLDSGlobal(rewriter, moduleOp, loc, "__shared_alloc", allocBytesAttr.getInt(), ++ createLDSGlobal(rewriter, moduleBody, loc, "__shared_alloc", allocBytesAttr.getInt(), + allocAlignAttr.getInt(), /*addrSpace=*/3); + rewriter.replaceOpWithNewOp(op, global); + return success(); +@@ -191,15 +197,20 @@ public: + auto flyPtrTy = cast(op.getResult().getType()); + unsigned addrSpace = mapAttrToLLVMAddressSpace(flyPtrTy.getAddressSpace()); + +- auto moduleOp = op->getParentOfType(); +- if (!moduleOp) ++ Block *moduleBody = nullptr; ++ if (auto gpuModule = op->getParentOfType()) { ++ moduleBody = gpuModule.getBody(); ++ } else if (auto builtinModule = op->getParentOfType()) { ++ moduleBody = builtinModule.getBody(); ++ } ++ if (!moduleBody) + return op->emitError("get_dyn_shared must be inside a gpu.module"); + + // Dynamic shared memory has a single logical region per kernel; reuse an + // existing `[0 x i8]` global if one is already present so multiple + // `get_dyn_shared` ops resolve to the same base symbol. + LLVM::GlobalOp sharedGlobal; +- for (auto globalOp : moduleOp.getBody()->getOps()) { ++ for (auto globalOp : moduleBody->getOps()) { + if (auto arrayType = dyn_cast(globalOp.getType())) { + if (globalOp.getAddrSpace() == addrSpace && arrayType.getNumElements() == 0 && + globalOp.getAlignment().value_or(0) == 1024) { +@@ -209,7 +220,7 @@ public: + } + } + if (!sharedGlobal) { +- sharedGlobal = createLDSGlobal(rewriter, moduleOp, loc, "__dynamic_shared", ++ sharedGlobal = createLDSGlobal(rewriter, moduleBody, loc, "__dynamic_shared", + /*nbytes=*/0, /*align=*/1024, addrSpace); + } + +@@ -810,10 +821,6 @@ public: + if (Value tok = op.getAsyncToken()) + asyncTokenType = tok.getType(); + +- // There are two relevant builder signatures in this MLIR: +- // - (kernel, ..., asyncTokenType, asyncDependencies, clusterSize) +- // - (kernel, ..., asyncObject, clusterSize) +- // Pick the one that matches the original op structure. + if (Value asyncObj = adaptor.getAsyncObject()) { + if (!adaptor.getAsyncDependencies().empty()) + return rewriter.notifyMatchFailure( +@@ -821,13 +828,14 @@ public: + + rewriter.replaceOpWithNewOp( + op, kernelRef, grid, block, adaptor.getDynamicSharedMemorySize(), +- adaptor.getKernelOperands(), asyncObj, clusterSize); ++ adaptor.getKernelOperands(), asyncTokenType, ValueRange{}, asyncObj, clusterSize); + return success(); + } + + rewriter.replaceOpWithNewOp( + op, kernelRef, grid, block, adaptor.getDynamicSharedMemorySize(), +- adaptor.getKernelOperands(), asyncTokenType, adaptor.getAsyncDependencies(), clusterSize); ++ adaptor.getKernelOperands(), asyncTokenType, adaptor.getAsyncDependencies(), ++ /*asyncObject=*/nullptr, clusterSize); + return success(); + } + }; +diff --git a/lib/Dialect/Fly/Transforms/LayoutLowering.cpp b/lib/Dialect/Fly/Transforms/LayoutLowering.cpp +--- a/lib/Dialect/Fly/Transforms/LayoutLowering.cpp ++++ b/lib/Dialect/Fly/Transforms/LayoutLowering.cpp +@@ -2994,7 +2994,7 @@ + // A value reached twice on one walk (cyclic chain) cannot be bounded here. + if (!visited.insert(def).second) + return std::nullopt; +- auto pop = llvm::make_scope_exit([&] { visited.erase(def); }); ++ auto pop = llvm::scope_exit([&] { visited.erase(def); }); + + // Launch coordinates: ask the op for its own range. Every gpu index op + // implements InferIntRangeInterface, and that implementation already prefers diff --git a/third_party/flydsl/repo.bzl b/third_party/flydsl/repo.bzl new file mode 100644 index 00000000..144e5af5 --- /dev/null +++ b/third_party/flydsl/repo.bzl @@ -0,0 +1,17 @@ +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") + +def repo(): + git_repository( + name = "flydsl", + remote = "https://github.com/ROCm/FlyDSL.git", + commit = "1941889400621f1fc3d8f03bada1a4e7380cdf7e", + build_file = "//third_party/flydsl:flydsl.bazel", + patches = [ + "//third_party/flydsl:atomic_exchange.patch", + "//third_party/flydsl:mlir_property_ref.patch", + "//third_party/flydsl:mlir_legacy_assembly.patch", + "//third_party/flydsl:vector_atomic.patch", + "//third_party/flydsl:xf32_mfma.patch", + ], + patch_args = ["-p1"], + ) diff --git a/third_party/flydsl/vector_atomic.patch b/third_party/flydsl/vector_atomic.patch new file mode 100644 index 00000000..edd23c40 --- /dev/null +++ b/third_party/flydsl/vector_atomic.patch @@ -0,0 +1,44 @@ +diff --git a/lib/Dialect/Fly/IR/FlyUniversalOps.cpp b/lib/Dialect/Fly/IR/FlyUniversalOps.cpp +--- a/lib/Dialect/Fly/IR/FlyUniversalOps.cpp ++++ b/lib/Dialect/Fly/IR/FlyUniversalOps.cpp +@@ -35,28 +35,36 @@ + return FxLayout(FxShape(FxC(1), FxC(getBitSize())), FxStride(FxC(1), FxC(1))); + } + ++static int32_t getAtomicValueBitWidth(Type type) { ++ if (auto vectorType = dyn_cast(type)) { ++ return vectorType.getNumElements() * ++ vectorType.getElementType().getIntOrFloatBitWidth(); ++ } ++ return type.getIntOrFloatBitWidth(); ++} ++ + bool CopyOpUniversalAtomicType::isStatic() const { return true; } + + Value CopyOpUniversalAtomicType::rebuildStaticValue(OpBuilder &builder, Location loc, + Value currentValue) const { + if (currentValue && isa(currentValue.getDefiningOp())) + return nullptr; +- int32_t bits = getValType().getIntOrFloatBitWidth(); ++ int32_t bits = getAtomicValueBitWidth(getValType()); + return MakeCopyAtomOp::create(builder, loc, CopyAtomType::get(*this, bits), bits); + } + + Attribute CopyOpUniversalAtomicType::getThrLayout() const { return FxLayout(FxC(1), FxC(1)); } + + Attribute CopyOpUniversalAtomicType::getThrBitLayoutSrc() const { +- int32_t bits = getValType().getIntOrFloatBitWidth(); ++ int32_t bits = getAtomicValueBitWidth(getValType()); + return FxLayout(FxShape(FxC(1), FxC(bits)), FxStride(FxC(1), FxC(1))); + } + Attribute CopyOpUniversalAtomicType::getThrBitLayoutDst() const { +- int32_t bits = getValType().getIntOrFloatBitWidth(); ++ int32_t bits = getAtomicValueBitWidth(getValType()); + return FxLayout(FxShape(FxC(1), FxC(bits)), FxStride(FxC(1), FxC(1))); + } + Attribute CopyOpUniversalAtomicType::getThrBitLayoutRef() const { +- int32_t bits = getValType().getIntOrFloatBitWidth(); ++ int32_t bits = getAtomicValueBitWidth(getValType()); + return FxLayout(FxShape(FxC(1), FxC(bits)), FxStride(FxC(1), FxC(1))); + } + diff --git a/third_party/flydsl/xf32_mfma.patch b/third_party/flydsl/xf32_mfma.patch new file mode 100644 index 00000000..bc6126a0 --- /dev/null +++ b/third_party/flydsl/xf32_mfma.patch @@ -0,0 +1,25 @@ +diff --git a/lib/Dialect/FlyROCDL/CDNA3/MmaAtom.cpp b/lib/Dialect/FlyROCDL/CDNA3/MmaAtom.cpp +--- a/lib/Dialect/FlyROCDL/CDNA3/MmaAtom.cpp ++++ b/lib/Dialect/FlyROCDL/CDNA3/MmaAtom.cpp +@@ -109,8 +109,12 @@ + static bool isBF8(Type ty) { return isa(ty); } + + static Type getMfmaABType(MLIRContext *ctx, Type elemTy, int32_t mn, int32_t k = 0) { +- if (elemTy.isF32()) ++ if (elemTy.isF32()) { ++ // v_mfma_f32_32x32x4_xf32 consumes two packed FP32 inputs per lane. ++ if (mn == 32 && k == 4) ++ return VectorType::get({2}, Float32Type::get(ctx)); + return Float32Type::get(ctx); ++ } + if (elemTy.isF16()) + return VectorType::get({mn * k / 64}, Float16Type::get(ctx)); + if (elemTy.isBF16()) { +@@ -179,6 +183,7 @@ + DISPATCH_MFMA_SSA(4, 1, elemTyA.isF32(), mfma_f32_4x4x1f32) + DISPATCH_MFMA_SSA(32, 2, elemTyA.isF32(), mfma_f32_32x32x2f32) + DISPATCH_MFMA_SSA(16, 4, elemTyA.isF32(), mfma_f32_16x16x4f32) ++ DISPATCH_MFMA_SSA(32, 4, elemTyA.isF32(), mfma_f32_32x32x4_xf32) + + DISPATCH_MFMA_SSA(32, 4, elemTyA.isF16(), mfma_f32_32x32x4f16) + DISPATCH_MFMA_SSA(16, 4, elemTyA.isF16(), mfma_f32_16x16x4f16) diff --git a/third_party/grpc/grpc++_unsecure-visibility.patch b/third_party/grpc/grpc++_unsecure-visibility.patch new file mode 100644 index 00000000..3b4013ab --- /dev/null +++ b/third_party/grpc/grpc++_unsecure-visibility.patch @@ -0,0 +1,13 @@ +diff --git a/BUILD b/BUILD +index 0e4bda7e16..dbe1708421 100644 +--- a/BUILD ++++ b/BUILD +@@ -1236,7 +1236,7 @@ grpc_cc_library( + "avoid_dep", + "nofixdeps", + ], +- visibility = ["//bazel:grpc++_unsecure"], ++ visibility = ["//visibility:public"], + deps = [ + "channel_arg_names", + "global_callback_hook", diff --git a/third_party/grpc/plugin-flags.patch b/third_party/grpc/plugin-flags.patch new file mode 100644 index 00000000..b69a77f5 --- /dev/null +++ b/third_party/grpc/plugin-flags.patch @@ -0,0 +1,19 @@ +# Match XLA's cc_grpc_library extension for services_namespace. +--- a/bazel/cc_grpc_library.bzl ++++ b/bazel/cc_grpc_library.bzl +@@ -29,6 +29,7 @@ + allow_deprecated = False, + use_external = False, # @unused + grpc_only = False, ++ plugin_flags = [], + **kwargs): + """Generates C++ grpc classes for services defined in a proto file. + +@@ -110,6 +111,7 @@ + well_known_protos = well_known_protos, + generate_mocks = generate_mocks, + allow_deprecated = allow_deprecated, ++ flags = plugin_flags, + **kwargs + ) + diff --git a/third_party/grpc/protobuf-35-upb-load.patch b/third_party/grpc/protobuf-35-upb-load.patch new file mode 100644 index 00000000..7311b981 --- /dev/null +++ b/third_party/grpc/protobuf-35-upb-load.patch @@ -0,0 +1,14 @@ +diff --git a/bazel/grpc_build_system.bzl b/bazel/grpc_build_system.bzl +index e0d47a29e1..e33f40d758 100644 +--- a/bazel/grpc_build_system.bzl ++++ b/bazel/grpc_build_system.bzl +@@ -31,7 +31,8 @@ load("@build_bazel_apple_support//rules:universal_binary.bzl", "universal_binary + load("@build_bazel_rules_apple//apple:ios.bzl", "ios_unit_test") + load("@build_bazel_rules_apple//apple/testing/default_runner:ios_test_runner.bzl", "ios_test_runner") + load("@com_google_protobuf//bazel:cc_proto_library.bzl", "cc_proto_library") +-load("@com_google_protobuf//bazel:upb_proto_library.bzl", "upb_proto_library", "upb_proto_reflection_library") ++load("@com_google_protobuf//bazel:upb_c_proto_library.bzl", upb_proto_library = "upb_c_proto_library") ++load("@com_google_protobuf//bazel:upb_proto_reflection_library.bzl", "upb_proto_reflection_library") + load("@rules_cc//cc:cc_binary.bzl", "cc_binary") + load("@rules_cc//cc:cc_library.bzl", "cc_library") + load("@rules_cc//cc:cc_test.bzl", "cc_test") diff --git a/third_party/iree/BUILD.bazel b/third_party/iree/BUILD.bazel new file mode 100644 index 00000000..e69de29b diff --git a/third_party/iree/fix-added-token-matching.patch b/third_party/iree/fix-added-token-matching.patch new file mode 100644 index 00000000..2a418780 --- /dev/null +++ b/third_party/iree/fix-added-token-matching.patch @@ -0,0 +1,123 @@ +diff --git a/runtime/src/iree/tokenizer/format/huggingface/tokenizer_json.c b/runtime/src/iree/tokenizer/format/huggingface/tokenizer_json.c +index 47d2a04c4d..66f1474c81 100644 +--- a/runtime/src/iree/tokenizer/format/huggingface/tokenizer_json.c ++++ b/runtime/src/iree/tokenizer/format/huggingface/tokenizer_json.c +@@ -48,11 +48,12 @@ static const iree_string_view_t kTopLevelAllowedKeys[] = { + + // Builds special_tokens collections from added_tokens. + // +-// ALL added tokens with `normalized=true` are included (not just those with +-// `special=true`) because they must be matched BEFORE the segmenter runs. +-// Without this, ByteLevel segmentation transforms the input before these +-// tokens can match, breaking tokenizers like GPT-NeoX that have multi-space +-// tokens (e.g., " " at ID 50276). ++// ALL added tokens are included (regardless of their `special` flag) because ++// they must be matched BEFORE the segmenter runs, mirroring HuggingFace's ++// AddedVocabulary. Without this, ByteLevel segmentation transforms the input ++// before these tokens can match, breaking tokenizers like GPT-NeoX that have ++// multi-space tokens (e.g., " " at ID 50276) and dropping non-special added ++// tokens such as `` or ``. + // + // Tokens are split by their normalized flag: + // - out_special_tokens: normalized=false, matched in raw input before +@@ -70,22 +71,10 @@ static iree_status_t iree_tokenizer_huggingface_build_special_tokens( + iree_tokenizer_special_tokens_initialize(out_special_tokens); + iree_tokenizer_special_tokens_initialize(out_special_tokens_post_norm); + +- // Count tokens to process: special tokens OR tokens with normalized=true. +- // Non-special tokens with normalized=true must be matched before +- // segmentation. +- iree_host_size_t match_count = 0; +- for (iree_host_size_t i = 0; i < added_tokens->count; ++i) { +- const iree_tokenizer_huggingface_added_token_t* token = +- iree_tokenizer_huggingface_added_tokens_get(added_tokens, i); +- bool is_special = iree_any_bit_set( +- token->flags, IREE_TOKENIZER_HUGGINGFACE_ADDED_TOKEN_FLAG_SPECIAL); +- bool is_normalized = iree_any_bit_set( +- token->flags, IREE_TOKENIZER_HUGGINGFACE_ADDED_TOKEN_FLAG_NORMALIZED); +- // Include if special OR if normalized (needs pre-segmentation matching). +- if (is_special || is_normalized) { +- ++match_count; +- } +- } ++ // All added tokens are matched before segmentation, mirroring HuggingFace's ++ // AddedVocabulary which matches every added token regardless of its `special` ++ // flag. They are routed by their `normalized` flag below. ++ iree_host_size_t match_count = added_tokens->count; + + // Early exit if nothing to process. + if (match_count == 0) { +@@ -108,45 +97,38 @@ static iree_status_t iree_tokenizer_huggingface_build_special_tokens( + i < added_tokens->count && iree_status_is_ok(status); ++i) { + const iree_tokenizer_huggingface_added_token_t* token = + iree_tokenizer_huggingface_added_tokens_get(added_tokens, i); +- bool is_special = iree_any_bit_set( +- token->flags, IREE_TOKENIZER_HUGGINGFACE_ADDED_TOKEN_FLAG_SPECIAL); + bool is_normalized = iree_any_bit_set( + token->flags, IREE_TOKENIZER_HUGGINGFACE_ADDED_TOKEN_FLAG_NORMALIZED); + +- // Process if special OR if normalized (needs pre-segmentation matching). +- if (is_special || is_normalized) { +- iree_string_view_t content = +- iree_tokenizer_huggingface_added_token_content(added_tokens, token); +- +- // Convert HuggingFace flags to special_tokens flags. +- iree_tokenizer_special_token_flags_t special_flags = +- IREE_TOKENIZER_SPECIAL_TOKEN_FLAG_NONE; +- if (iree_any_bit_set( +- token->flags, +- IREE_TOKENIZER_HUGGINGFACE_ADDED_TOKEN_FLAG_LSTRIP)) { +- special_flags |= IREE_TOKENIZER_SPECIAL_TOKEN_FLAG_LSTRIP; +- } +- if (iree_any_bit_set( +- token->flags, +- IREE_TOKENIZER_HUGGINGFACE_ADDED_TOKEN_FLAG_RSTRIP)) { +- special_flags |= IREE_TOKENIZER_SPECIAL_TOKEN_FLAG_RSTRIP; +- } +- if (iree_any_bit_set( +- token->flags, +- IREE_TOKENIZER_HUGGINGFACE_ADDED_TOKEN_FLAG_SINGLE_WORD)) { +- special_flags |= IREE_TOKENIZER_SPECIAL_TOKEN_FLAG_SINGLE_WORD; +- } ++ iree_string_view_t content = ++ iree_tokenizer_huggingface_added_token_content(added_tokens, token); + +- // Route to appropriate builder based on normalized flag. +- if (is_normalized) { +- // normalized=true: match after normalization but before segmentation. +- status = iree_tokenizer_special_tokens_builder_add( +- &builder_post_norm, content, token->id, special_flags); +- } else { +- // normalized=false (default): match before normalization. +- status = iree_tokenizer_special_tokens_builder_add( +- &builder_pre_norm, content, token->id, special_flags); +- } ++ // Convert HuggingFace flags to special_tokens flags. ++ iree_tokenizer_special_token_flags_t special_flags = ++ IREE_TOKENIZER_SPECIAL_TOKEN_FLAG_NONE; ++ if (iree_any_bit_set(token->flags, ++ IREE_TOKENIZER_HUGGINGFACE_ADDED_TOKEN_FLAG_LSTRIP)) { ++ special_flags |= IREE_TOKENIZER_SPECIAL_TOKEN_FLAG_LSTRIP; ++ } ++ if (iree_any_bit_set(token->flags, ++ IREE_TOKENIZER_HUGGINGFACE_ADDED_TOKEN_FLAG_RSTRIP)) { ++ special_flags |= IREE_TOKENIZER_SPECIAL_TOKEN_FLAG_RSTRIP; ++ } ++ if (iree_any_bit_set( ++ token->flags, ++ IREE_TOKENIZER_HUGGINGFACE_ADDED_TOKEN_FLAG_SINGLE_WORD)) { ++ special_flags |= IREE_TOKENIZER_SPECIAL_TOKEN_FLAG_SINGLE_WORD; ++ } ++ ++ // Route to appropriate builder based on normalized flag. ++ if (is_normalized) { ++ // normalized=true: match after normalization but before segmentation. ++ status = iree_tokenizer_special_tokens_builder_add( ++ &builder_post_norm, content, token->id, special_flags); ++ } else { ++ // normalized=false (default): match before normalization. ++ status = iree_tokenizer_special_tokens_builder_add( ++ &builder_pre_norm, content, token->id, special_flags); + } + } + diff --git a/third_party/iree/fix-gemma4-encode.patch b/third_party/iree/fix-gemma4-encode.patch new file mode 100644 index 00000000..18f493e6 --- /dev/null +++ b/third_party/iree/fix-gemma4-encode.patch @@ -0,0 +1,149 @@ +From 96151c5c077dc56c00e2b608c4f39b5553089515 Mon Sep 17 00:00:00 2001 +From: Benjamin Rabier +Date: Thu, 16 Apr 2026 09:07:02 +0000 +Subject: [PATCH] Fix tokenization + +Signed-off-by: Benjamin Rabier +--- + runtime/src/iree/tokenizer/tokenizer.c | 13 ++- + .../iree/tokenizer/tokenizer_encode_test.cc | 100 ++++++++++++++++++ + 2 files changed, 109 insertions(+), 4 deletions(-) + +diff --git a/runtime/src/iree/tokenizer/tokenizer.c b/runtime/src/iree/tokenizer/tokenizer.c +index ff9fe6242e..37df477c9d 100644 +--- a/runtime/src/iree/tokenizer/tokenizer.c ++++ b/runtime/src/iree/tokenizer/tokenizer.c +@@ -2606,10 +2606,15 @@ static iree_status_t iree_tokenizer_encode_state_pump( + state->segmenter_view_start = state->read_position; + } + } +- // When a forced-complete drains the ring, exit partial mode so fresh +- // text after the special token goes through the normal segmenter path. +- if (force_complete && segment_complete && +- state->read_position >= state->write_position) { ++ // When a forced-complete segment finishes, retire the entire ring region ++ // that was just fed directly to the model. Unlike the partial reclaim path ++ // above, a non-partial completion resets the model to SEGMENT_START, so ++ // state_reclaim() returns 0 even though all bytes were fully processed. ++ // If we leave the ring positions unchanged, the same bytes are re-encoded ++ // on the next pump iteration and a pending special token can never flush. ++ if (force_complete && segment_complete) { ++ state->read_position = state->write_position; ++ state->segmenter_view_start = state->write_position; + state->has_partial_segment = false; + } + if (reclaimed > 0 || tokens_written > 0) { +diff --git a/runtime/src/iree/tokenizer/tokenizer_encode_test.cc b/runtime/src/iree/tokenizer/tokenizer_encode_test.cc +index a28811fe9b..0eb3f68dc8 100644 +--- a/runtime/src/iree/tokenizer/tokenizer_encode_test.cc ++++ b/runtime/src/iree/tokenizer/tokenizer_encode_test.cc +@@ -3152,6 +3152,106 @@ TEST_F(TokenizerSpecialTokenStreamingTest, FinalizeDoesNotSilentlyDropPartial) { + iree_tokenizer_free(tokenizer); + } + ++// Regression test for a deferred pre-norm special token that causes an infinite ++// loop when the pump enters partial-segment mode. ++// ++// The bug: after a force-complete of ring content (triggered by a pending ++// special token), state_reclaim() returns 0 because the model resets to ++// SEGMENT_START on non-partial completion. The old code required ++// read_position >= write_position to clear partial mode and advance positions, ++// but since reclaim never advanced read_position, this condition was always ++// false — creating an infinite loop re-encoding the same bytes. ++// ++// This test reproduces the real-world trigger: a Replace normalizer that ++// expands " " → "▁" (1 byte → 3 bytes). With a Split segmenter on space ++// (MergedWithPrevious), each word+space is a small segment, but the normalizer ++// expansion means N bytes of input produce up to 3N bytes of normalized output. ++// When enough text with spaces precedes a special token, the normalized output ++// overflows the ring buffer's logical capacity, forcing partial-segment mode. ++// The pending special token then triggers force_complete, exposing the bug. ++// ++// With a 128-byte ring (64-byte logical capacity), ~40 bytes of space-heavy ++// input normalizes to >64 bytes, reliably triggering the overflow. ++TEST_F(TokenizerSpecialTokenStreamingTest, ++ DeferredSpecialTokenDoesNotReplayForcedCompleteSegment) { ++ // ▁ (U+2581) = 0xE2 0x96 0x81 (3 bytes in UTF-8). ++ static const char kMetaspace[] = "\xE2\x96\x81"; ++ ++ ScopedVocabBuilder vocab_builder(257); ++ for (int i = 0; i < 256; ++i) { ++ char byte_token[2] = {static_cast(i), '\0'}; ++ vocab_builder.AddToken(i, byte_token); ++ } ++ // Add ▁ as a token so the BPE model can encode normalized spaces. ++ vocab_builder.AddToken(256, kMetaspace); ++ vocab_builder.AddToken(50256, "<|tool>"); ++ ScopedVocab vocab = vocab_builder.Build(); ++ ++ iree_tokenizer_special_tokens_builder_t st_builder; ++ iree_tokenizer_special_tokens_builder_initialize(iree_allocator_system(), ++ &st_builder); ++ IREE_ASSERT_OK(iree_tokenizer_special_tokens_builder_add( ++ &st_builder, IREE_SV("<|tool>"), 50256, ++ IREE_TOKENIZER_SPECIAL_TOKEN_FLAG_NONE)); ++ iree_tokenizer_special_tokens_t special_tokens; ++ IREE_ASSERT_OK(iree_tokenizer_special_tokens_builder_build( ++ &st_builder, iree_allocator_system(), &special_tokens)); ++ iree_tokenizer_special_tokens_builder_deinitialize(&st_builder); ++ ++ // Replace normalizer: " " → "▁" (1→3 byte expansion, matching Gemma4). ++ iree_tokenizer_normalizer_t* replace_normalizer = nullptr; ++ IREE_ASSERT_OK(iree_tokenizer_normalizer_replace_allocate( ++ IREE_SV(" "), iree_make_string_view(kMetaspace, 3), ++ iree_allocator_system(), &replace_normalizer)); ++ ++ // Split on literal space with MergedWithPrevious (matching Gemma4 config). ++ iree_tokenizer_segmenter_t* segmenter = CreateSplitSegmenterWithBehavior( ++ " ", IREE_TOKENIZER_UTIL_REGEX_SPLIT_MERGED_WITH_PREVIOUS); ++ ASSERT_NE(segmenter, nullptr); ++ ++ ScopedBuilder builder; ++ iree_tokenizer_builder_set_normalizer(builder.get(), replace_normalizer); ++ iree_tokenizer_builder_set_segmenter(builder.get(), segmenter); ++ iree_tokenizer_builder_set_model(builder.get(), CreateBPEModel(vocab.get())); ++ iree_tokenizer_builder_set_special_tokens(builder.get(), &special_tokens); ++ iree_tokenizer_builder_set_vocab(builder.get(), vocab.release()); ++ ++ iree_tokenizer_t* tokenizer = BuildTokenizer(builder.get()); ++ ASSERT_NE(tokenizer, nullptr); ++ ++ // Normal text with many spaces. Each space expands from 1→3 bytes after ++ // normalization. With a 128-byte ring (64-byte logical capacity), even this ++ // short text overflows the ring due to normalizer expansion, forcing ++ // partial-segment mode. The special token then triggers force_complete. ++ std::string input = ++ "A knight of powder-horn and shot " ++ "Once fill'd his bag as I would not " ++ "Unless the feelings of my breast " ++ "By poverty were sorely press'd " ++ "With birds and squirrels for the spits " ++ "Of certain gormandizing cits " ++ "With merry heart the fellow went "; ++ input += "<|tool>d"; ++ ++ IREE_ASSERT_OK_AND_ASSIGN(auto large_tokens, ++ EncodeWithBufferSize(tokenizer, input, 1024)); ++ IREE_ASSERT_OK_AND_ASSIGN(auto small_tokens, ++ EncodeWithBufferSize(tokenizer, input, 256)); ++ ++ EXPECT_EQ(small_tokens, large_tokens) ++ << "Small-buffer streaming must match large-buffer streaming when a " ++ "deferred special token follows text with normalizer expansion"; ++ ++ // Verify the special token and trailing byte are present at the end. ++ ASSERT_GE(small_tokens.size(), 2u); ++ EXPECT_EQ(small_tokens[small_tokens.size() - 2], 50256u); ++ EXPECT_EQ(small_tokens[small_tokens.size() - 1], ++ static_cast('d')); ++ ++ iree_tokenizer_special_tokens_deinitialize(&special_tokens); ++ iree_tokenizer_free(tokenizer); ++} ++ + //===----------------------------------------------------------------------===// + // Strip Normalizer + Special Token Integration Tests + //===----------------------------------------------------------------------===// +-- +2.53.0 + diff --git a/third_party/iree/fix-normalizer-lifetime-and-boundaries.patch b/third_party/iree/fix-normalizer-lifetime-and-boundaries.patch new file mode 100644 index 00000000..194a7b87 --- /dev/null +++ b/third_party/iree/fix-normalizer-lifetime-and-boundaries.patch @@ -0,0 +1,74 @@ +diff --git a/runtime/src/iree/tokenizer/normalizer/sequence.c b/runtime/src/iree/tokenizer/normalizer/sequence.c +--- a/runtime/src/iree/tokenizer/normalizer/sequence.c ++++ b/runtime/src/iree/tokenizer/normalizer/sequence.c +@@ -188,6 +188,14 @@ + "%" PRIhsz " exceeds maximum %" PRIu32, + flattened_count, + IREE_TOKENIZER_NORMALIZER_SEQUENCE_MAX_DEPTH); ++ } ++ ++ // Allocate before transferring ownership, including flattening nested ++ // sequences. On failure the caller must retain all original children. ++ iree_tokenizer_normalizer_sequence_t* normalizer = NULL; ++ if (flattened_count != 1) { ++ IREE_RETURN_IF_ERROR(iree_allocator_malloc( ++ allocator, sizeof(*normalizer), (void**)&normalizer)); + } + + // Build flattened children array, transferring ownership as we go. +@@ -232,18 +240,6 @@ + state_size = iree_host_align(state_size, iree_max_align_t); + child_state_offsets[i] = state_size; + state_size += iree_tokenizer_normalizer_state_size(flattened_children[i]); +- } +- +- // Allocate the sequence normalizer. +- iree_tokenizer_normalizer_sequence_t* normalizer = NULL; +- iree_status_t status = iree_allocator_malloc(allocator, sizeof(*normalizer), +- (void**)&normalizer); +- if (!iree_status_is_ok(status)) { +- // Cleanup: we've taken ownership of the children, so free them on failure. +- for (iree_host_size_t i = 0; i < flattened_count; ++i) { +- iree_tokenizer_normalizer_free(flattened_children[i]); +- } +- return status; + } + + // Initialize base. +@@ -566,6 +562,15 @@ + if (tile_size > IREE_TOKENIZER_NORMALIZER_SEQUENCE_MIN_BATCH) { + // Limit tile to what we can handle in scratch. + tile_size = IREE_TOKENIZER_NORMALIZER_SEQUENCE_MIN_BATCH; ++ // Never split a UTF-8 codepoint between child process calls. ++ while (tile_size > 0 && ++ ((uint8_t)remaining_input.data[tile_size] & 0xC0) == 0x80) { ++ --tile_size; ++ } ++ } ++ iree_tokenizer_normalizer_flags_t tile_flags = flags; ++ if (tile_size < remaining_input.size) { ++ tile_flags &= ~IREE_TOKENIZER_NORMALIZER_FLAG_SEGMENT_END; + } + + // Process one tile through all children. +@@ -574,7 +579,7 @@ + iree_host_size_t tile_consumed = 0; + iree_host_size_t tile_written = 0; + IREE_RETURN_IF_ERROR(iree_tokenizer_normalizer_sequence_process_tile( +- state, tile_input, scratch, scratch_size, remaining_output, flags, ++ state, tile_input, scratch, scratch_size, remaining_output, tile_flags, + &tile_consumed, &tile_written)); + + if (tile_consumed == 0 && tile_written == 0) { +diff --git a/runtime/src/iree/tokenizer/normalizer/strip.c b/runtime/src/iree/tokenizer/normalizer/strip.c +--- a/runtime/src/iree/tokenizer/normalizer/strip.c ++++ b/runtime/src/iree/tokenizer/normalizer/strip.c +@@ -353,7 +353,7 @@ + } + + // Phase 3: Handle segment end. +- if (is_segment_end) { ++ if (is_segment_end && in_ptr == in_end) { + // Pending whitespace is trailing - discard it. + iree_tokenizer_normalizer_strip_clear_pending(state); + state->at_start = true; diff --git a/third_party/iree/futex-public-api-header.patch b/third_party/iree/futex-public-api-header.patch new file mode 100644 index 00000000..7623f422 --- /dev/null +++ b/third_party/iree/futex-public-api-header.patch @@ -0,0 +1,17 @@ +diff --git a/runtime/src/iree/base/threading/futex.h b/runtime/src/iree/base/threading/futex.h +index 9b42d3cfd0..f2509b77e9 100644 +--- a/runtime/src/iree/base/threading/futex.h ++++ b/runtime/src/iree/base/threading/futex.h +@@ -30,11 +30,8 @@ + + #include + +-#include "iree/base/attributes.h" +-#include "iree/base/config.h" +-#include "iree/base/status.h" ++#include "iree/base/api.h" + #include "iree/base/target_platform.h" +-#include "iree/base/time.h" + + //===----------------------------------------------------------------------===// + // Platform detection diff --git a/third_party/iree/laguna-tokenizer-compat.patch b/third_party/iree/laguna-tokenizer-compat.patch new file mode 100644 index 00000000..c978c591 --- /dev/null +++ b/third_party/iree/laguna-tokenizer-compat.patch @@ -0,0 +1,78 @@ +diff --git a/runtime/src/iree/tokenizer/format/huggingface/segmenter_json.c b/runtime/src/iree/tokenizer/format/huggingface/segmenter_json.c +--- a/runtime/src/iree/tokenizer/format/huggingface/segmenter_json.c ++++ b/runtime/src/iree/tokenizer/format/huggingface/segmenter_json.c +@@ -217,0 +218,10 @@ ++ // Some code-oriented tokenizers use this lookahead split only to attach ++ // single-newline runs to the next segment. The IREE regex compiler does not ++ // support lookahead; the following ByteLevel/GPT split still preserves the ++ // input bytes, so treat this grouping hint as a no-op instead of rejecting ++ // the tokenizer at load time. ++ if (iree_string_view_equal( ++ pattern, IREE_SV("(?:\\r?\\n)+(?!\\r?\\n)")) && ++ behavior == IREE_TOKENIZER_UTIL_REGEX_SPLIT_MERGED_WITH_NEXT && !invert) { ++ return iree_ok_status(); ++ } +diff --git a/runtime/src/iree/tokenizer/format/huggingface/model_json.c b/runtime/src/iree/tokenizer/format/huggingface/model_json.c +--- a/runtime/src/iree/tokenizer/format/huggingface/model_json.c ++++ b/runtime/src/iree/tokenizer/format/huggingface/model_json.c +@@ -444,0 +445,46 @@ ++static char iree_tokenizer_ascii_upper(char c) { ++ return c >= 'a' && c <= 'z' ? (char)(c - ('a' - 'A')) : c; ++} ++ ++static bool iree_tokenizer_string_view_contains_ascii_case( ++ iree_string_view_t value, iree_string_view_t needle) { ++ if (needle.size == 0 || needle.size > value.size) return false; ++ for (iree_host_size_t i = 0; i <= value.size - needle.size; ++i) { ++ bool match = true; ++ for (iree_host_size_t j = 0; j < needle.size; ++j) { ++ if (iree_tokenizer_ascii_upper(value.data[i + j]) != ++ iree_tokenizer_ascii_upper(needle.data[j])) { ++ match = false; ++ break; ++ } ++ } ++ if (match) return true; ++ } ++ return false; ++} ++ ++static bool iree_tokenizer_huggingface_find_added_unk_token( ++ const iree_tokenizer_huggingface_added_tokens_t* added_tokens, ++ iree_tokenizer_token_id_t* out_token_id) { ++ *out_token_id = IREE_TOKENIZER_TOKEN_ID_INVALID; ++ if (!added_tokens) return false; ++ for (iree_host_size_t i = 0; i < added_tokens->count; ++i) { ++ const iree_tokenizer_huggingface_added_token_t* token = ++ iree_tokenizer_huggingface_added_tokens_get(added_tokens, i); ++ if (!iree_any_bit_set( ++ token->flags, ++ IREE_TOKENIZER_HUGGINGFACE_ADDED_TOKEN_FLAG_SPECIAL)) { ++ continue; ++ } ++ iree_string_view_t content = ++ iree_tokenizer_huggingface_added_token_content(added_tokens, token); ++ if (!iree_tokenizer_string_view_contains_ascii_case(content, ++ IREE_SV("unk"))) { ++ continue; ++ } ++ *out_token_id = token->id; ++ return true; ++ } ++ return false; ++} ++ +@@ -584,3 +630,10 @@ +- status = iree_make_status(IREE_STATUS_NOT_FOUND, +- "unk_token '%.*s' not found in vocabulary", +- (int)unk_length, unk_buffer); ++ // Some tokenizer.json files keep a stale default `unk_token` in the BPE ++ // model section while declaring the real unknown token in added_tokens ++ // (for example, custom bracketed forms such as `〈|UNK|〉`). Prefer that ++ // explicit special added token before failing the whole tokenizer load. ++ if (!iree_tokenizer_huggingface_find_added_unk_token(added_tokens, ++ &unk_token_id)) { ++ status = iree_make_status(IREE_STATUS_NOT_FOUND, ++ "unk_token '%.*s' not found in vocabulary", ++ (int)unk_length, unk_buffer); ++ } diff --git a/third_party/iree/match-hf-tokenizer.patch b/third_party/iree/match-hf-tokenizer.patch new file mode 100644 index 00000000..4cbab21a --- /dev/null +++ b/third_party/iree/match-hf-tokenizer.patch @@ -0,0 +1,1643 @@ +diff --git a/runtime/src/iree/tokenizer/model/bpe.c b/runtime/src/iree/tokenizer/model/bpe.c +index 309f1e261f..02fb055eee 100644 +--- a/runtime/src/iree/tokenizer/model/bpe.c ++++ b/runtime/src/iree/tokenizer/model/bpe.c +@@ -179,16 +179,17 @@ iree_status_t iree_tokenizer_bpe_model_allocate( + // Backtracking path capacities. + // Segments up to this size use the O(n) backtracking algorithm. + // Larger segments fall back to the O(n log L) window+heap path. +- // Capped at 4095 so the bitfield (ceil_div(4096, 64) = 64 words) fits in +- // a single uint64_t dirty mask for O(1) amortized per-segment init. ++ // Capped at 32768 to keep exact backtracking state bounded while covering ++ // large normalized prompts that still need HuggingFace-compatible ++ // suffix/prefix merge validation. + iree_host_size_t min_backtrack = 0; + if (!iree_host_size_checked_mul(16, model->max_token_length, + &min_backtrack)) { + // Overflow implies very large token length; clamp to maximum. +- min_backtrack = 4095; ++ min_backtrack = 32768; + } +- if (min_backtrack < 2048) min_backtrack = 2048; +- if (min_backtrack > 4095) min_backtrack = 4095; ++ if (min_backtrack < 32768) min_backtrack = 32768; ++ if (min_backtrack > 32768) min_backtrack = 32768; + model->max_backtrack_segment_bytes = min_backtrack; + // Stack holds at most one token per byte (worst case: all single-byte + // tokens). +@@ -311,6 +312,7 @@ static void iree_tokenizer_bpe_model_destroy( + iree_tokenizer_vocab_trie_free(model->trie); + iree_tokenizer_vocab_merge_hash_free(model->merge_hash); + iree_allocator_free(allocator, model->backtrack_tables.slab); ++ iree_allocator_free(allocator, model->backtrack_tables.alternate_suffix_slab); + iree_allocator_free(allocator, model); + + IREE_TRACE_ZONE_END(z0); +diff --git a/runtime/src/iree/tokenizer/model/bpe_backtrack.c b/runtime/src/iree/tokenizer/model/bpe_backtrack.c +index 7f2cab797c..02b272838f 100644 +--- a/runtime/src/iree/tokenizer/model/bpe_backtrack.c ++++ b/runtime/src/iree/tokenizer/model/bpe_backtrack.c +@@ -81,10 +81,9 @@ typedef struct iree_tokenizer_bpe_validation_frame_t { + static bool iree_tokenizer_bpe_is_valid_token_pair( + const iree_tokenizer_bpe_model_t* model, uint32_t token1, uint32_t token2, + uint32_t deferred_merge_rank) { +- const iree_tokenizer_vocab_t* vocab = model->vocab; +- const uint32_t* effective_rank = model->backtrack_tables.effective_rank; + const iree_tokenizer_bpe_split_entry_t* split_table = + model->backtrack_tables.split_table; ++ const iree_tokenizer_vocab_t* vocab = model->vocab; + + iree_tokenizer_bpe_validation_frame_t + stack[IREE_TOKENIZER_BPE_PAIR_VALIDATION_STACK_CAPACITY]; +@@ -111,7 +110,7 @@ static bool iree_tokenizer_bpe_is_valid_token_pair( + (int32_t)frame->token1, + (int32_t)frame->token2); + if (merge.result_id >= 0) { +- uint32_t combined_rank = effective_rank[(uint32_t)merge.result_id]; ++ uint32_t combined_rank = merge.rank + 1; + // A merge is invalid if it should have fired (rank < limit) UNLESS + // it was intentionally deferred for a better suffix merge. When + // suffix_blocked triggers, it reports the rank of the blocked token; +@@ -140,7 +139,8 @@ static bool iree_tokenizer_bpe_is_valid_token_pair( + // Need to split at least one token. Determine order based on rank. + // Higher-ranked tokens (formed later in BPE) are split first. + bool try_token1_first = +- effective_rank[frame->token1] > effective_rank[frame->token2]; ++ iree_tokenizer_bpe_token_formation_rank(model, frame->token1) > ++ iree_tokenizer_bpe_token_formation_rank(model, frame->token2); + + if (try_token1_first) { + frame->side = +@@ -172,16 +172,26 @@ static bool iree_tokenizer_bpe_is_valid_token_pair( + if (split.left_id != decompose_token) { + // Non-base token: split_table provides (left_id, right_id) directly. + // By construction, merge(left_id, right_id) == decompose_token. ++ uint32_t decomposition_rank = ++ iree_tokenizer_bpe_token_formation_rank(model, decompose_token); ++ iree_tokenizer_merge_hash_result_t split_merge = ++ iree_tokenizer_vocab_merge_hash_lookup( ++ model->merge_hash, (int32_t)split.left_id, ++ (int32_t)split.right_id); ++ if (split_merge.result_id == (int32_t)decompose_token) { ++ decomposition_rank = split_merge.rank + 1; ++ } ++ + uint32_t new_limit; + uint32_t new_token1, new_token2; + if (frame->side == 1) { + new_token1 = split.right_id; + new_token2 = frame->token2; +- new_limit = effective_rank[frame->token1]; ++ new_limit = decomposition_rank; + } else { + new_token1 = frame->token1; + new_token2 = split.left_id; +- new_limit = effective_rank[frame->token2] + 1; ++ new_limit = decomposition_rank + 1; + } + + if (stack_count < IREE_TOKENIZER_BPE_PAIR_VALIDATION_STACK_CAPACITY) { +@@ -243,16 +253,17 @@ static bool iree_tokenizer_bpe_is_valid_token_pair( + if (merge.result_id != (int32_t)decompose_token) continue; + + // Valid decomposition found. Compute new limit and push child frame. ++ uint32_t decomposition_rank = merge.rank + 1; + uint32_t new_limit; + uint32_t new_token1, new_token2; + if (frame->side == 1) { + new_token1 = (uint32_t)right_id; + new_token2 = frame->token2; +- new_limit = effective_rank[frame->token1]; ++ new_limit = decomposition_rank; + } else { + new_token1 = frame->token1; + new_token2 = (uint32_t)left_id; +- new_limit = effective_rank[frame->token2] + 1; ++ new_limit = decomposition_rank + 1; + } + + // Save resume position for backtracking. +@@ -381,160 +392,202 @@ static int32_t iree_tokenizer_bpe_find_first_token_at( + static int32_t iree_tokenizer_bpe_token_at_position( + const iree_tokenizer_bpe_model_t* model, const uint8_t* data, + iree_host_size_t size, iree_host_size_t* out_length) { +- int32_t token = iree_tokenizer_bpe_single_char_token(model, data[0]); +- if (token >= 0) { +- if (out_length) *out_length = 1; +- return token; ++ return iree_tokenizer_bpe_base_token_at_position(model, data, size, ++ out_length); ++} ++ ++static bool iree_tokenizer_bpe_is_suffix_blocked( ++ const iree_tokenizer_bpe_model_t* model, int32_t token, ++ const uint8_t* remaining_data, iree_host_size_t remaining_size, ++ const char* suffix_data, iree_host_size_t suffix_length, ++ uint32_t* out_blocking_rank); ++static bool iree_tokenizer_bpe_is_suffix_blocked_impl( ++ const iree_tokenizer_bpe_model_t* model, int32_t token, ++ const uint8_t* remaining_data, iree_host_size_t remaining_size, ++ const char* suffix_data, iree_host_size_t suffix_length, ++ uint32_t* out_blocking_rank, int depth); ++ ++// Recursive implementation for iree_tokenizer_bpe_is_token_consumed_rightward. ++// ++// |ready_rank_plus_one| tracks when the right compound is available. Base ++// tokens are available at rank 0; a compound built by an internal merge at ++// rank N is available starting at N+1, so an outer merge at rank R can use it ++// iff ready_rank_plus_one <= R. ++static bool iree_tokenizer_bpe_is_token_consumed_rightward_impl( ++ const iree_tokenizer_bpe_model_t* model, int32_t token, ++ const uint8_t* remaining_data, iree_host_size_t remaining_size, ++ iree_host_size_t token_end, uint32_t max_rank, int depth) { ++ if (token_end >= remaining_size) return false; ++ if (depth >= 8) return false; ++ ++ const iree_tokenizer_bpe_split_entry_t* split_table = ++ model->backtrack_tables.split_table; ++ const bool byte_level = ++ iree_all_bits_set(model->flags, IREE_TOKENIZER_BPE_FLAG_BYTE_LEVEL_INPUT); ++ ++ iree_tokenizer_trie_cursor_t cursor; ++ iree_tokenizer_trie_cursor_reset(&cursor, model->trie); ++ ++ for (iree_host_size_t i = token_end; i < remaining_size; ++i) { ++ if (!iree_tokenizer_bpe_trie_advance_byte(&cursor, remaining_data[i], ++ byte_level)) { ++ break; ++ } ++ ++ int32_t compound_token = iree_tokenizer_trie_cursor_token_id(&cursor); ++ if (compound_token < 0) continue; ++ ++ iree_host_size_t compound_end = i + 1; ++ iree_tokenizer_merge_hash_result_t merge = ++ iree_tokenizer_vocab_merge_hash_lookup(model->merge_hash, token, ++ compound_token); ++ if (!iree_tokenizer_merge_hash_result_is_valid(merge) || ++ merge.rank >= max_rank) { ++ continue; ++ } ++ ++ uint32_t compound = (uint32_t)compound_token; ++ bool compound_is_base = split_table[compound].left_id == compound; ++ uint32_t compound_ready_rank = ++ iree_tokenizer_bpe_token_formation_rank(model, compound); ++ if (compound_ready_rank > merge.rank) continue; ++ if (!compound_is_base && ++ !iree_tokenizer_bpe_is_first_token_reachable(model, compound)) { ++ continue; ++ } ++ ++ if (!compound_is_base && compound_end < remaining_size) { ++ uint32_t rightmost_base = ++ iree_tokenizer_bpe_rightmost_base_token(model, compound); ++ uint32_t right_consumed = ++ iree_tokenizer_bpe_right_boundary_consumed_rank(model, compound); ++ if (right_consumed > 0 && ++ iree_tokenizer_bpe_is_token_consumed_rightward_impl( ++ model, (int32_t)rightmost_base, remaining_data, remaining_size, ++ compound_end, right_consumed - 1, depth + 1)) { ++ continue; ++ } ++ } ++ ++ uint32_t result_token = (uint32_t)merge.result_id; ++ if (compound_end < remaining_size && depth < 4) { ++ uint32_t blocking_rank = 0; ++ if (iree_tokenizer_bpe_is_suffix_blocked_impl( ++ model, (int32_t)result_token, remaining_data + compound_end, ++ remaining_size - compound_end, NULL, 0, &blocking_rank, ++ depth + 1)) { ++ continue; ++ } ++ } ++ ++ // merge(token, compound) would consume token if the compound is not itself ++ // consumed to the right before that merge rank. ++ if (!iree_tokenizer_bpe_is_token_consumed_rightward_impl( ++ model, compound_token, remaining_data, remaining_size, compound_end, ++ merge.rank, depth + 1)) { ++ return true; ++ } + } +- return iree_tokenizer_bpe_find_first_token_at(model, data, size, 8, +- out_length); ++ return false; + } + + // Checks whether a token at a given position is consumed by a rightward merge + // at rank lower than |max_rank|. This means merge(token, following) exists and + // fires before any merge at |max_rank| would. + // +-// To verify the rightward merge actually fires, also checks that the following +-// token isn't itself consumed by an even lower-rank merge. This handles chains +-// like: merge(c,h) would consume c, but merge(h,X) at lower rank consumes h +-// first — so merge(c,h) can't fire and c survives. ++// The following token may itself be a compound formed by lower-rank rightward ++// merges (for example t+ry after r+y has formed). This bounded search mirrors ++// the short compound lookahead used by suffix-blocking preemption. + static bool iree_tokenizer_bpe_is_token_consumed_rightward( + const iree_tokenizer_bpe_model_t* model, int32_t token, + const uint8_t* remaining_data, iree_host_size_t remaining_size, + iree_host_size_t token_end, uint32_t max_rank) { +- if (token_end >= remaining_size) return false; +- iree_host_size_t following_length = 1; +- int32_t following_token = iree_tokenizer_bpe_token_at_position( +- model, remaining_data + token_end, remaining_size - token_end, +- &following_length); +- if (following_token < 0) return false; +- +- iree_tokenizer_merge_hash_result_t merge = +- iree_tokenizer_vocab_merge_hash_lookup(model->merge_hash, token, +- following_token); +- if (!iree_tokenizer_merge_hash_result_is_valid(merge) || +- merge.rank >= max_rank) { +- return false; +- } +- +- // merge(token, following) would fire. Verify following is available: if +- // merge(following, next_after) fires at even lower rank, following is +- // consumed and merge(token, following) can't fire. +- iree_host_size_t next_after_end = token_end + following_length; +- if (next_after_end < remaining_size) { +- int32_t next_after_token = iree_tokenizer_bpe_token_at_position( +- model, remaining_data + next_after_end, remaining_size - next_after_end, +- NULL); +- if (next_after_token >= 0) { +- iree_tokenizer_merge_hash_result_t following_merge = +- iree_tokenizer_vocab_merge_hash_lookup( +- model->merge_hash, following_token, next_after_token); +- if (iree_tokenizer_merge_hash_result_is_valid(following_merge) && +- following_merge.rank < merge.rank) { +- return false; // Following consumed first, token survives. +- } +- } +- } +- return true; ++ return iree_tokenizer_bpe_is_token_consumed_rightward_impl( ++ model, token, remaining_data, remaining_size, token_end, max_rank, 0); + } + + // Checks if a suffix merge is preempted by lower-rank merges in the following + // input. Returns true if the suffix merge would NOT actually fire because the + // prefix token would be consumed first. + // +-// Two preemption paths: +-// Direct: merge(prefix, next) at rank < suffix_merge_rank, where the next +-// token is available (not consumed by a lower-rank rightward merge). +-// Compound: merge(prefix, compound) where compound is built by progressively +-// merging characters: next+third->c1, c1+fourth->c2, etc. This handles +-// cases where the direct merge can't fire but the prefix is still consumed +-// via a deeper compound (e.g., merge(l, ish) when i+s fires first). +-// +-// For non-byte-level tokenizers (e.g., SentencePiece), uses trie lookup to find +-// tokens when single-char lookup fails. This handles multi-byte base tokens +-// like metaspace (E2 96 81). ++// This delegates to the same bounded rightward-consumption search used to ++// validate prefix availability. That search handles direct and compound merges ++// while checking that each compound component is actually available at the rank ++// where it would be used. + static bool iree_tokenizer_bpe_is_suffix_merge_preempted( + const iree_tokenizer_bpe_model_t* model, int32_t prefix_token, + uint32_t suffix_merge_rank, const uint8_t* remaining_data, + iree_host_size_t remaining_size, iree_host_size_t prefix_end) { +- if (prefix_end >= remaining_size) return false; +- +- iree_host_size_t next_length = 1; +- int32_t next_token = iree_tokenizer_bpe_token_at_position( +- model, remaining_data + prefix_end, remaining_size - prefix_end, +- &next_length); +- if (next_token < 0) return false; +- +- // Direct preemption: merge(prefix, next) at rank < suffix_merge_rank. +- // Only valid if next is actually available (not consumed rightward). +- iree_tokenizer_merge_hash_result_t prefix_next = +- iree_tokenizer_vocab_merge_hash_lookup(model->merge_hash, prefix_token, +- next_token); +- if (iree_tokenizer_merge_hash_result_is_valid(prefix_next) && +- prefix_next.rank < suffix_merge_rank) { +- if (!iree_tokenizer_bpe_is_token_consumed_rightward( +- model, next_token, remaining_data, remaining_size, +- prefix_end + next_length, prefix_next.rank)) { ++ return iree_tokenizer_bpe_is_token_consumed_rightward( ++ model, prefix_token, remaining_data, remaining_size, prefix_end, ++ suffix_merge_rank); ++} ++ ++static bool iree_tokenizer_bpe_suffix_already_collected( ++ const uint32_t* suffixes, const uint32_t* suffix_consumed_at, ++ iree_host_size_t suffix_count, uint32_t suffix, uint32_t consumed_at) { ++ for (iree_host_size_t i = 0; i < suffix_count; ++i) { ++ if (suffixes[i] == suffix && suffix_consumed_at[i] == consumed_at) { + return true; + } + } ++ return false; ++} + +- // Compound preemption: build progressively longer compounds from the +- // characters following the prefix, checking merge(prefix, compound) at each +- // step. This catches cases where the direct merge can't fire but the prefix +- // merges with a multi-character compound that forms at lower rank. +- // +- // Example (Whisper "establish"): +- // prefix=l, next=i, merge(l,i) can't fire (i consumed by i+s at rank 15) +- // compound: i+s->is (rank 15), is+h->ish (rank 486) +- // merge(l, ish) at rank 1677 < suffix_merge_rank -> preempted +- iree_host_size_t compound_end = prefix_end + next_length; +- iree_host_size_t extension_length = 1; +- int32_t extension_token = -1; +- if (compound_end < remaining_size) { +- extension_token = iree_tokenizer_bpe_token_at_position( +- model, remaining_data + compound_end, remaining_size - compound_end, +- &extension_length); +- } +- if (extension_token < 0) return false; ++static void iree_tokenizer_bpe_collect_reachable_suffixes( ++ const iree_tokenizer_bpe_model_t* model, uint32_t token, ++ uint32_t* suffixes, uint32_t* suffix_consumed_at, ++ iree_host_size_t* suffix_count, iree_host_size_t suffix_capacity, ++ int depth) { ++ if (*suffix_count >= suffix_capacity || depth >= 16) return; + +- iree_tokenizer_merge_hash_result_t compound_merge = +- iree_tokenizer_vocab_merge_hash_lookup(model->merge_hash, next_token, +- extension_token); +- if (!iree_tokenizer_merge_hash_result_is_valid(compound_merge) || +- compound_merge.rank >= suffix_merge_rank) { +- return false; +- } ++ const iree_tokenizer_vocab_t* vocab = model->vocab; ++ const iree_tokenizer_bpe_split_entry_t* split_table = ++ model->backtrack_tables.split_table; + +- // Build the compound progressively, checking at each level. +- int32_t compound_token = compound_merge.result_id; +- compound_end += extension_length; +- for (int depth = 0; depth < 4; ++depth) { +- iree_tokenizer_merge_hash_result_t prefix_compound = +- iree_tokenizer_vocab_merge_hash_lookup(model->merge_hash, prefix_token, +- compound_token); +- if (iree_tokenizer_merge_hash_result_is_valid(prefix_compound) && +- prefix_compound.rank < suffix_merge_rank) { +- return true; ++ if (split_table[token].left_id == token) return; // Base token. ++ ++ iree_string_view_t token_text = ++ iree_tokenizer_vocab_token_text(vocab, (int32_t)token); ++ ++ for (iree_host_size_t split_pos = 1; ++ split_pos < token_text.size && *suffix_count < suffix_capacity; ++ ++split_pos) { ++ iree_string_view_t left_text = ++ iree_make_string_view(token_text.data, split_pos); ++ iree_string_view_t right_text = iree_make_string_view( ++ token_text.data + split_pos, token_text.size - split_pos); ++ ++ int32_t left_id = iree_tokenizer_vocab_lookup(vocab, left_text); ++ int32_t right_id = iree_tokenizer_vocab_lookup(vocab, right_text); ++ if (left_id < 0 || right_id < 0) continue; ++ ++ iree_tokenizer_merge_hash_result_t merge = ++ iree_tokenizer_vocab_merge_hash_lookup(model->merge_hash, left_id, ++ right_id); ++ if (merge.result_id != (int32_t)token) continue; ++ ++ if (!iree_tokenizer_bpe_is_decomposition_reachable( ++ model, (uint32_t)left_id, (uint32_t)right_id, token)) { ++ continue; + } + +- // Extend compound by one more character. +- if (compound_end >= remaining_size) break; +- extension_token = iree_tokenizer_bpe_token_at_position( +- model, remaining_data + compound_end, remaining_size - compound_end, +- &extension_length); +- if (extension_token < 0) break; +- +- compound_merge = iree_tokenizer_vocab_merge_hash_lookup( +- model->merge_hash, compound_token, extension_token); +- if (!iree_tokenizer_merge_hash_result_is_valid(compound_merge) || +- compound_merge.rank >= suffix_merge_rank) { +- break; ++ uint32_t consumed_at = merge.rank + 1; ++ if (!iree_tokenizer_bpe_suffix_already_collected( ++ suffixes, suffix_consumed_at, *suffix_count, (uint32_t)right_id, ++ consumed_at)) { ++ suffixes[*suffix_count] = (uint32_t)right_id; ++ suffix_consumed_at[*suffix_count] = consumed_at; ++ ++*suffix_count; + } +- compound_token = compound_merge.result_id; +- compound_end += extension_length; ++ ++ // Walk deeper suffixes of the right child. This generalizes the old ++ // rightmost split_table walk to alternate reachable decompositions, e.g. ++ // recon can split as rec+on or re+con and both suffixes can matter. ++ iree_tokenizer_bpe_collect_reachable_suffixes( ++ model, (uint32_t)right_id, suffixes, suffix_consumed_at, suffix_count, ++ suffix_capacity, depth + 1); + } +- return false; + } + + // Checks if a candidate token is "suffix-blocked" - meaning a proper suffix of +@@ -566,21 +619,25 @@ static bool iree_tokenizer_bpe_is_suffix_merge_preempted( + // This allows pair validation to accept pairs whose merge rank >= + // blocking_rank, since those merges were intentionally deferred for better + // suffix merges. +-static bool iree_tokenizer_bpe_is_suffix_blocked( ++static bool iree_tokenizer_bpe_is_suffix_blocked_impl( + const iree_tokenizer_bpe_model_t* model, int32_t token, + const uint8_t* remaining_data, iree_host_size_t remaining_size, + const char* suffix_data, iree_host_size_t suffix_length, +- uint32_t* out_blocking_rank) { ++ uint32_t* out_blocking_rank, int depth) { + *out_blocking_rank = 0; ++ if (depth >= 8) return false; + if (remaining_size == 0 && suffix_length == 0) return false; + + const iree_tokenizer_bpe_split_entry_t* split_table = + model->backtrack_tables.split_table; +- const uint32_t* effective_rank = model->backtrack_tables.effective_rank; ++ const uint64_t* token_has_reachable_alternates = ++ model->backtrack_tables.token_has_reachable_alternates; ++ const iree_tokenizer_vocab_t* vocab = model->vocab; + const bool byte_level = + iree_all_bits_set(model->flags, IREE_TOKENIZER_BPE_FLAG_BYTE_LEVEL_INPUT); + +- uint32_t token_rank = effective_rank[(uint32_t)token]; ++ uint32_t token_rank = ++ iree_tokenizer_bpe_token_formation_rank(model, (uint32_t)token); + if (token_rank <= 1) return false; // Base token, no suffix to check. + + // Collect all rightmost suffixes with their consumption ranks. +@@ -601,29 +658,64 @@ static bool iree_tokenizer_bpe_is_suffix_blocked( + uint32_t suffix_consumed_at[32]; + iree_host_size_t suffix_count = 0; + ++ bool needs_alternate_suffixes = false; + for (uint32_t current = (uint32_t)token; +- suffix_count < 32 && split_table[current].left_id != current; ++ suffix_count < IREE_ARRAYSIZE(suffixes) && ++ split_table[current].left_id != current; + current = split_table[current].right_id) { +- // Check if this decomposition is actually reachable via BPE. If the +- // split_table decomposition is blocked by a boundary merge, skip this +- // suffix - it won't exist in the actual BPE token sequence. ++ if (iree_any_bit_set(token_has_reachable_alternates[current / 64], ++ 1ull << (current % 64))) { ++ needs_alternate_suffixes = true; ++ break; ++ } + uint32_t left = split_table[current].left_id; + uint32_t right = split_table[current].right_id; + bool reachable = iree_tokenizer_bpe_is_decomposition_reachable( + model, left, right, current); + if (!reachable) { +- // This decomposition is blocked. The actual BPE path uses a different +- // decomposition, so we can't rely on this suffix chain. Stop collecting +- // suffixes here - the actual suffixes depend on the alternate path which +- // we don't have easy access to. ++ needs_alternate_suffixes = true; + break; + } + suffixes[suffix_count] = right; +- suffix_consumed_at[suffix_count] = effective_rank[current]; ++ suffix_consumed_at[suffix_count] = ++ iree_tokenizer_bpe_token_formation_rank(model, current); + suffix_count++; ++ iree_tokenizer_bpe_collect_reachable_suffixes( ++ model, right, suffixes, suffix_consumed_at, &suffix_count, ++ IREE_ARRAYSIZE(suffixes), 0); ++ } ++ if (needs_alternate_suffixes) { ++ suffix_count = 0; ++ if (model->backtrack_tables.alternate_suffix_offsets) { ++ uint32_t token_id = (uint32_t)token; ++ uint32_t start = ++ model->backtrack_tables.alternate_suffix_offsets[token_id]; ++ uint32_t end = ++ model->backtrack_tables.alternate_suffix_offsets[token_id + 1]; ++ for (uint32_t i = start; ++ i < end && suffix_count < IREE_ARRAYSIZE(suffixes); ++i) { ++ suffixes[suffix_count] = ++ model->backtrack_tables.alternate_suffixes[i]; ++ suffix_consumed_at[suffix_count] = ++ model->backtrack_tables.alternate_suffix_consumed_at[i]; ++ ++suffix_count; ++ } ++ iree_host_size_t loaded_suffix_count = suffix_count; ++ for (iree_host_size_t i = 0; i < loaded_suffix_count; ++i) { ++ iree_tokenizer_bpe_collect_reachable_suffixes( ++ model, suffixes[i], suffixes, suffix_consumed_at, &suffix_count, ++ IREE_ARRAYSIZE(suffixes), 0); ++ } ++ } else { ++ iree_tokenizer_bpe_collect_reachable_suffixes( ++ model, (uint32_t)token, suffixes, suffix_consumed_at, &suffix_count, ++ IREE_ARRAYSIZE(suffixes), 0); ++ } ++ } else { ++ IREE_ASSERT(suffix_count < IREE_ARRAYSIZE(suffixes), ++ "suffix chain depth exceeds capacity"); + } +- IREE_ASSERT(suffix_count < IREE_ARRAYSIZE(suffixes), +- "suffix chain depth exceeds capacity"); ++ + if (suffix_count == 0) return false; + + // Walk the trie to enumerate all prefix tokens of the remaining input. +@@ -652,18 +744,33 @@ static bool iree_tokenizer_bpe_is_suffix_blocked( + + // Skip if merge rank is too high to matter. + uint32_t merge_effective_rank = merge.rank + 1; ++ uint32_t suffix_available_at = ++ iree_tokenizer_bpe_token_formation_rank(model, suffixes[s]); ++ uint32_t prefix_available_at = ++ iree_tokenizer_bpe_token_formation_rank(model, ++ (uint32_t)prefix_token); ++ if (merge_effective_rank < suffix_available_at) continue; ++ if (merge_effective_rank < prefix_available_at) continue; + if (merge_effective_rank >= token_rank) continue; + + // Skip if the suffix is consumed before this merge could fire. + if (merge_effective_rank >= suffix_consumed_at[s]) continue; + +- // Skip when suffix == prefix (repeating pattern case). +- // When the suffix token equals the prefix token, we have a repeating +- // pattern like consecutive metaspaces. In this case, any shorter token +- // would also face suffix blocking by the same pattern, creating an +- // infinite loop. The suffix blocking optimization doesn't help here +- // because there's no "better" tokenization to defer to. +- if ((int32_t)suffixes[s] == prefix_token) continue; ++ // If the prefix token would itself be split to enable a better suffix ++ // merge with following bytes, it is not available for this boundary ++ // merge. This handles chains such as ge|uffer|t where b+uffer would ++ // form "buffer" before ge+b, except "uffer" is blocked by er+t. ++ if (i + 1 < remaining_size && depth < 7 && ++ iree_tokenizer_bpe_token_formation_rank( ++ model, (uint32_t)prefix_token) > 1) { ++ uint32_t prefix_blocking_rank = 0; ++ if (iree_tokenizer_bpe_is_suffix_blocked_impl( ++ model, prefix_token, remaining_data + i + 1, ++ remaining_size - (i + 1), NULL, 0, &prefix_blocking_rank, ++ depth + 1)) { ++ continue; ++ } ++ } + + // Check if the prefix would be consumed by lower-rank merges first. + if (iree_tokenizer_bpe_is_suffix_merge_preempted( +@@ -678,7 +785,8 @@ static bool iree_tokenizer_bpe_is_suffix_blocked( + // the rightmost base of the prefix merges with the following + // character at a rank lower than the prefix's internal consumption + // of that base, the prefix cannot form here. +- if (effective_rank[(uint32_t)prefix_token] > 1 && ++ if (iree_tokenizer_bpe_token_formation_rank( ++ model, (uint32_t)prefix_token) > 1 && + i + 1 < remaining_size) { + uint32_t rightmost_base = iree_tokenizer_bpe_rightmost_base_token( + model, (uint32_t)prefix_token); +@@ -725,14 +833,18 @@ static bool iree_tokenizer_bpe_is_suffix_blocked( + + // Skip if merge rank is too high to matter. + uint32_t merge_effective_rank = merge.rank + 1; ++ uint32_t suffix_available_at = ++ iree_tokenizer_bpe_token_formation_rank(model, suffixes[s]); ++ uint32_t prefix_available_at = ++ iree_tokenizer_bpe_token_formation_rank(model, ++ (uint32_t)prefix_token); ++ if (merge_effective_rank < suffix_available_at) continue; ++ if (merge_effective_rank < prefix_available_at) continue; + if (merge_effective_rank >= token_rank) continue; + + // Skip if the token suffix is consumed before this merge could fire. + if (merge_effective_rank >= suffix_consumed_at[s]) continue; + +- // Skip when suffix == prefix (repeating pattern case). +- if ((int32_t)suffixes[s] == prefix_token) continue; +- + // Suffixed tokens consume all remaining input + suffix bytes, so no + // preemption check is needed. The prefix_token represents the complete + // remaining segment (remaining_data + partial suffix) and nothing +@@ -745,6 +857,16 @@ static bool iree_tokenizer_bpe_is_suffix_blocked( + return false; + } + ++static bool iree_tokenizer_bpe_is_suffix_blocked( ++ const iree_tokenizer_bpe_model_t* model, int32_t token, ++ const uint8_t* remaining_data, iree_host_size_t remaining_size, ++ const char* suffix_data, iree_host_size_t suffix_length, ++ uint32_t* out_blocking_rank) { ++ return iree_tokenizer_bpe_is_suffix_blocked_impl( ++ model, token, remaining_data, remaining_size, suffix_data, suffix_length, ++ out_blocking_rank, 0); ++} ++ + //===----------------------------------------------------------------------===// + // BPE Backtracking: Core Algorithm + //===----------------------------------------------------------------------===// +@@ -774,18 +896,11 @@ void iree_tokenizer_bpe_backtrack_encode( + iree_tokenizer_bpe_state_pair_cache(state, model); + const uint32_t pair_cache_mask = (uint32_t)(model->pair_cache_capacity - 1); + +- // Reset only the bitfield words dirtied by the previous segment's +- // backtracking. On the first call, dirty_mask is pre-set to all-ones by +- // state_initialize, so this performs the equivalent of a full init. +- // On subsequent calls with no backtracking, dirty_mask is 0 and this +- // loop body executes zero times. +- uint64_t dirty = state->backtrack.dirty_mask; +- while (dirty) { +- iree_host_size_t word_index = +- (iree_host_size_t)iree_math_count_trailing_zeros_u64(dirty); +- bitfield[word_index] = UINT64_MAX; +- dirty &= dirty - 1; // Clear lowest set bit. +- } ++ // Reset the reachability bitfield for this segment. The exact backtracking ++ // path supports more than 64 bitfield words, so a single dirty mask is not ++ // expressive enough. ++ memset(bitfield, 0xFF, ++ model->backtrack_bitfield_capacity * sizeof(uint64_t)); + state->backtrack.dirty_mask = 0; + + iree_host_size_t stack_count = 0; +@@ -984,7 +1099,8 @@ void iree_tokenizer_bpe_backtrack_encode( + // fires first, blocking the path to "hello" as a single token. + bool token_unreachable = + token_raw_length > 1 && +- model->backtrack_tables.effective_rank[(uint32_t)token] > 1 && ++ iree_tokenizer_bpe_token_formation_rank(model, (uint32_t)token) > ++ 1 && + !iree_any_bit_set( + model->backtrack_tables.token_reachable[(uint32_t)token / 64], + 1ull << ((uint32_t)token % 64)); +diff --git a/runtime/src/iree/tokenizer/model/bpe_encode.c b/runtime/src/iree/tokenizer/model/bpe_encode.c +index bf140ae97c..4fa090026f 100644 +--- a/runtime/src/iree/tokenizer/model/bpe_encode.c ++++ b/runtime/src/iree/tokenizer/model/bpe_encode.c +@@ -280,39 +280,29 @@ static iree_status_t iree_tokenizer_bpe_encode_segment( + byte_position < segment.size; ++byte_position) { + uint8_t input_byte = (uint8_t)segment.data[byte_position]; + +- // Look up single-byte token. For ByteLevel mode, this is the UTF-8 +- // encoding of the ByteLevel codepoint (e.g., space -> "Ġ"). +- int32_t token_id = model->byte_to_token[input_byte]; ++ // Seed with the shortest base vocabulary token. Some base symbols (for ++ // example SentencePiece's U+2581 metaspace) span multiple UTF-8 bytes, ++ // and byte-fallback vocabularies may also have tokens for each raw byte. ++ // The BPE heap must start from the base symbol, not the byte fallback. + iree_host_size_t token_byte_length = 1; ++ int32_t token_id = iree_tokenizer_bpe_base_token_at_position( ++ model, (const uint8_t*)segment.data + byte_position, ++ segment.size - byte_position, &token_byte_length); + if (token_id < 0) { +- // No direct single-byte token. Try a multi-byte trie match first: +- // some base vocabulary tokens (e.g., SentencePiece's ▁ = U+2581, +- // 3 bytes) span multiple raw bytes but are never produced by a merge. +- // The trie walk finds the longest such token; effective_rank filtering +- // inside backtrack_longest_match ensures we only accept tokens that +- // participate in BPE (not stray added tokens). +- iree_host_size_t trie_raw_length = 0; +- iree_tokenizer_bpe_backtrack_longest_match( +- model, (const uint8_t*)segment.data + byte_position, +- segment.size - byte_position, &token_id, &trie_raw_length); +- if (token_id >= 0) { +- token_byte_length = trie_raw_length; +- } else { +- // No trie match. Fall back to byte-fallback (<0xNN>) or UNK. +- // For FUSE_UNK, check the last token in the window (not yet emitted) +- // before falling back to last_emitted_token_id. +- int32_t previous_token = +- (state->window.count > 0) +- ? iree_tokenizer_bpe_window_at(state, model, +- state->window.count - 1) +- ->token_id +- : state->last_emitted_token_id; +- token_id = iree_tokenizer_bpe_handle_unknown_byte(model, input_byte, +- previous_token); +- if (token_id < 0) { +- // Byte was fused with previous UNK. Skip to next byte. +- continue; +- } ++ // No trie match. Fall back to byte-fallback (<0xNN>) or UNK. For ++ // FUSE_UNK, check the last token in the window (not yet emitted) ++ // before falling back to last_emitted_token_id. ++ int32_t previous_token = ++ (state->window.count > 0) ++ ? iree_tokenizer_bpe_window_at(state, model, ++ state->window.count - 1) ++ ->token_id ++ : state->last_emitted_token_id; ++ token_id = iree_tokenizer_bpe_handle_unknown_byte(model, input_byte, ++ previous_token); ++ if (token_id < 0) { ++ // Byte was fused with previous UNK. Skip to next byte. ++ continue; + } + } + +@@ -620,13 +610,14 @@ iree_host_size_t iree_tokenizer_bpe_state_reclaim( + window[index].end_byte -= (uint32_t)committed; + } + +- // Adjust heap entry byte positions. +- // After BYTE_LOOP processes all segment bytes, apply_pending_merges has +- // drained most entries. Any remaining entries reference current window tokens +- // (left_start_byte >= committed). Stale entries that would underflow are +- // discarded by lazy invalidation when popped. +- for (iree_host_size_t i = 0; i < bpe_state->heap.size; ++i) { +- bpe_state->heap.entries[i].left_start_byte -= (uint32_t)committed; ++ // Rebuild heap entries for the rebased window. The heap can contain stale ++ // entries for tokens that were already emitted; subtracting |committed| from ++ // those would underflow their start byte and leave noisy stale entries. The ++ // window is bounded, so rebuilding is cheap and keeps all held-back merges ++ // visible after reclaim. ++ iree_tokenizer_bpe_heap_reset(&bpe_state->heap); ++ for (iree_host_size_t i = 0; i + 1 < bpe_state->window.count; ++i) { ++ iree_tokenizer_bpe_maybe_add_merge(bpe_state, model, i); + } + + return committed; +diff --git a/runtime/src/iree/tokenizer/model/bpe_internal.h b/runtime/src/iree/tokenizer/model/bpe_internal.h +index 69546df3f0..4e9873e653 100644 +--- a/runtime/src/iree/tokenizer/model/bpe_internal.h ++++ b/runtime/src/iree/tokenizer/model/bpe_internal.h +@@ -45,50 +45,42 @@ typedef struct iree_tokenizer_bpe_split_entry_t { + uint32_t right_id; + } iree_tokenizer_bpe_split_entry_t; + +-// Backtracking stack entry: token ID and the byte position where it starts. +-// Bit-packed backtrack entry to maintain 8-byte size while supporting large +-// vocabs. start_byte is bounded by max_backtrack_segment_bytes (<=4095), so +-// we use 12 bits for it and 20 bits for deferred_merge_rank (supports 1M +-// tokens, covering all realistic vocabularies). +-#define IREE_TOKENIZER_BPE_BACKTRACK_START_BYTE_BITS 12 +-#define IREE_TOKENIZER_BPE_BACKTRACK_START_BYTE_MASK \ +- ((1u << IREE_TOKENIZER_BPE_BACKTRACK_START_BYTE_BITS) - 1) +-#define IREE_TOKENIZER_BPE_BACKTRACK_DEFERRED_RANK_BITS 20 ++// Backtracking stack entry: token ID, byte start, and deferred merge rank. ++// Keep deferred_merge_rank as a full 19-bit value (Gemma-sized vocabularies ++// need it) and store start_byte separately so exact backtracking can cover ++// larger normalized segments without falling back to the approximate heap path. ++#define IREE_TOKENIZER_BPE_BACKTRACK_DEFERRED_RANK_BITS 19 + #define IREE_TOKENIZER_BPE_BACKTRACK_DEFERRED_RANK_MAX \ + ((1u << IREE_TOKENIZER_BPE_BACKTRACK_DEFERRED_RANK_BITS) - 1) + + typedef struct iree_tokenizer_bpe_backtrack_entry_t { + int32_t token_id; +- // Packed field: lower 12 bits = start_byte, upper 20 bits = deferred_rank. ++ uint32_t start_byte; + // The deferred_merge_rank is the effective_rank that was active when this + // token was pushed. It tells pair validation at the next position which + // merges were intentionally deferred due to suffix blocking. +- uint32_t start_byte_and_deferred_rank; ++ uint32_t deferred_rank; + } iree_tokenizer_bpe_backtrack_entry_t; + + static inline uint32_t iree_tokenizer_bpe_backtrack_entry_start_byte( + const iree_tokenizer_bpe_backtrack_entry_t* entry) { +- return entry->start_byte_and_deferred_rank & +- IREE_TOKENIZER_BPE_BACKTRACK_START_BYTE_MASK; ++ return entry->start_byte; + } + + static inline uint32_t iree_tokenizer_bpe_backtrack_entry_deferred_rank( + const iree_tokenizer_bpe_backtrack_entry_t* entry) { +- return entry->start_byte_and_deferred_rank >> +- IREE_TOKENIZER_BPE_BACKTRACK_START_BYTE_BITS; ++ return entry->deferred_rank; + } + + static inline void iree_tokenizer_bpe_backtrack_entry_set( + iree_tokenizer_bpe_backtrack_entry_t* entry, int32_t token_id, + uint32_t start_byte, uint32_t deferred_rank) { + entry->token_id = token_id; +- // Cap deferred_rank at 20-bit max (supports vocabs up to 1M tokens). + if (deferred_rank > IREE_TOKENIZER_BPE_BACKTRACK_DEFERRED_RANK_MAX) { + deferred_rank = IREE_TOKENIZER_BPE_BACKTRACK_DEFERRED_RANK_MAX; + } +- entry->start_byte_and_deferred_rank = +- (start_byte & IREE_TOKENIZER_BPE_BACKTRACK_START_BYTE_MASK) | +- (deferred_rank << IREE_TOKENIZER_BPE_BACKTRACK_START_BYTE_BITS); ++ entry->start_byte = start_byte; ++ entry->deferred_rank = deferred_rank; + } + + //===----------------------------------------------------------------------===// +@@ -101,11 +93,19 @@ static inline void iree_tokenizer_bpe_backtrack_entry_set( + typedef struct iree_tokenizer_bpe_backtrack_tables_t { + // Single allocation for all tables below. + void* slab; ++ void* alternate_suffix_slab; + iree_tokenizer_bpe_split_entry_t* split_table; // Inverse merge table. + // Longest proper prefix token, or UINT32_MAX. + uint32_t* next_prefix_match; + uint32_t* effective_rank; // 0 = non-participant, >0 = BPE-reachable. + uint64_t* token_reachable; // Precomputed reachability bits. ++ // Tokens with more than one reachable decomposition. Suffix blocking must ++ // inspect alternate decompositions for these tokens instead of only walking ++ // split_table's right spine. ++ uint64_t* token_has_reachable_alternates; ++ uint32_t* alternate_suffix_offsets; // vocab_capacity + 1 entries. ++ uint32_t* alternate_suffixes; ++ uint32_t* alternate_suffix_consumed_at; + } iree_tokenizer_bpe_backtrack_tables_t; + + //===----------------------------------------------------------------------===// +@@ -118,10 +118,9 @@ typedef struct iree_tokenizer_bpe_backtrack_state_t { + iree_host_size_t stack_count; // Tokens in the stack. + // Next token to emit from stack. + iree_host_size_t emit_index; +- // Dirty mask for lazy bitfield reset. Each bit corresponds to a uint64_t +- // word in the backtrack bitfield that was modified (cleared) during the +- // previous segment's backtracking. On segment start, only these words are +- // reset to UINT64_MAX, giving O(backtracks) init instead of O(capacity). ++ // Retained for state layout compatibility with older code paths. The ++ // backtracking bitfield is reset as a whole at segment start because the ++ // exact path now supports more than 64 bitfield words. + uint64_t dirty_mask; + } iree_tokenizer_bpe_backtrack_state_t; + +@@ -210,6 +209,22 @@ typedef struct iree_tokenizer_bpe_model_t { + iree_host_size_t pair_cache_offset; + } iree_tokenizer_bpe_model_t; + ++static inline uint32_t iree_tokenizer_bpe_token_formation_rank( ++ const iree_tokenizer_bpe_model_t* model, uint32_t token) { ++ iree_tokenizer_bpe_split_entry_t split = ++ model->backtrack_tables.split_table[token]; ++ if (split.left_id == token) return 0; // Base token. ++ ++ iree_tokenizer_merge_hash_result_t merge = ++ iree_tokenizer_vocab_merge_hash_lookup(model->merge_hash, ++ (int32_t)split.left_id, ++ (int32_t)split.right_id); ++ if (merge.result_id == (int32_t)token) { ++ return merge.rank + 1; ++ } ++ return model->backtrack_tables.effective_rank[token]; ++} ++ + //===----------------------------------------------------------------------===// + // Segment Context (State-Side) + //===----------------------------------------------------------------------===// +@@ -589,9 +604,9 @@ static inline bool iree_tokenizer_bpe_bitfield_is_set( + + static inline void iree_tokenizer_bpe_bitfield_clear( + uint64_t* bitfield, uint64_t* dirty_mask, iree_host_size_t position) { ++ (void)dirty_mask; + iree_host_size_t word_index = position / 64; + bitfield[word_index] &= ~((uint64_t)1 << (position % 64)); +- *dirty_mask |= (uint64_t)1 << word_index; + } + + //===----------------------------------------------------------------------===// +@@ -679,6 +694,73 @@ iree_tokenizer_bpe_backtrack_longest_match( + } + } + ++// Finds the shortest base vocabulary token matching raw input bytes at data[0]. ++// BPE starts from base symbols and applies merge rules by rank. For ++// SentencePiece-style tokenizers, some base symbols (for example U+2581 ++// metaspace) span multiple UTF-8 bytes and cannot be found through byte_to_token. ++// The window/heap path must seed those base symbols, not the longest reachable ++// merged token, otherwise rank-priority merges can be skipped. ++static inline int32_t iree_tokenizer_bpe_base_token_at_position( ++ const iree_tokenizer_bpe_model_t* model, const uint8_t* data, ++ iree_host_size_t size, iree_host_size_t* out_raw_length) { ++ if (size == 0) return -1; ++ ++ int32_t byte_token = model->byte_to_token[data[0]]; ++ ++ const bool byte_level = ++ iree_all_bits_set(model->flags, IREE_TOKENIZER_BPE_FLAG_BYTE_LEVEL_INPUT); ++ ++ iree_tokenizer_trie_cursor_t cursor; ++ iree_tokenizer_trie_cursor_reset(&cursor, model->trie); ++ ++ int32_t fallback_base_token = -1; ++ iree_host_size_t fallback_base_length = 0; ++ for (iree_host_size_t raw_consumed = 0; raw_consumed < size; ++ ++raw_consumed) { ++ if (!iree_tokenizer_bpe_trie_advance_byte(&cursor, data[raw_consumed], ++ byte_level)) { ++ break; ++ } ++ ++ int32_t token = iree_tokenizer_trie_cursor_token_id(&cursor); ++ if (token < 0) continue; ++ ++ const uint32_t token_id = (uint32_t)token; ++ if (token_id >= model->vocab_capacity) continue; ++ ++ iree_tokenizer_bpe_split_entry_t split = ++ model->backtrack_tables.split_table[token_id]; ++ const bool is_base_token = ++ split.left_id == token_id && split.right_id == token_id; ++ const bool is_matchable = ++ model->backtrack_tables.effective_rank[token_id] > 0; ++ if (is_base_token && is_matchable) { ++ iree_string_view_t token_text = ++ iree_tokenizer_vocab_token_text(model->vocab, token); ++ if (raw_consumed == 0 && data[0] >= 0x80 && token_text.size == 1 && ++ raw_consumed + 1 < size) { ++ fallback_base_token = token; ++ fallback_base_length = raw_consumed + 1; ++ continue; ++ } ++ if (out_raw_length) *out_raw_length = raw_consumed + 1; ++ return token; ++ } ++ } ++ ++ if (fallback_base_token >= 0) { ++ if (out_raw_length) *out_raw_length = fallback_base_length; ++ return fallback_base_token; ++ } ++ ++ if (byte_token >= 0) { ++ if (out_raw_length) *out_raw_length = 1; ++ return byte_token; ++ } ++ ++ return -1; ++} ++ + //===----------------------------------------------------------------------===// + // Table Construction (bpe_tables.c) + //===----------------------------------------------------------------------===// +diff --git a/runtime/src/iree/tokenizer/model/bpe_tables.c b/runtime/src/iree/tokenizer/model/bpe_tables.c +index c44021d3e1..480d7cc4e8 100644 +--- a/runtime/src/iree/tokenizer/model/bpe_tables.c ++++ b/runtime/src/iree/tokenizer/model/bpe_tables.c +@@ -13,6 +13,28 @@ + #include "iree/tokenizer/vocab/vocab_merge_hash.h" + #include "iree/tokenizer/vocab/vocab_trie.h" + ++static bool iree_tokenizer_bpe_is_repeated_initial_symbol( ++ iree_string_view_t text) { ++ if (text.size < 2) return false; ++ ++ bool all_newlines = true; ++ bool all_tabs = true; ++ for (iree_host_size_t i = 0; i < text.size; ++i) { ++ all_newlines &= text.data[i] == '\n'; ++ all_tabs &= text.data[i] == '\t'; ++ } ++ if (all_newlines || all_tabs) return true; ++ ++ static const char kMetaspace[] = "\xE2\x96\x81"; ++ if (text.size % (sizeof(kMetaspace) - 1) != 0) return false; ++ for (iree_host_size_t i = 0; i < text.size; i += sizeof(kMetaspace) - 1) { ++ if (memcmp(text.data + i, kMetaspace, sizeof(kMetaspace) - 1) != 0) { ++ return false; ++ } ++ } ++ return true; ++} ++ + //===----------------------------------------------------------------------===// + // BPE Backtracking: First Token Reachability + //===----------------------------------------------------------------------===// +@@ -56,7 +78,6 @@ uint32_t iree_tokenizer_bpe_right_boundary_consumed_rank( + const iree_tokenizer_bpe_model_t* model, uint32_t token) { + const iree_tokenizer_bpe_split_entry_t* split_table = + model->backtrack_tables.split_table; +- const uint32_t* effective_rank = model->backtrack_tables.effective_rank; + + // Walk down the right spine until we find where the rightmost base token + // is consumed (i.e., where the right child is a base token). +@@ -64,7 +85,7 @@ uint32_t iree_tokenizer_bpe_right_boundary_consumed_rank( + uint32_t right = split_table[token].right_id; + if (split_table[right].left_id == right) { + // Right child is a base token - it's consumed at this token's merge. +- return effective_rank[token]; ++ return iree_tokenizer_bpe_token_formation_rank(model, token); + } + // Recurse into right subtree. + token = right; +@@ -81,7 +102,6 @@ static uint32_t iree_tokenizer_bpe_left_boundary_consumed_rank( + const iree_tokenizer_bpe_model_t* model, uint32_t token) { + const iree_tokenizer_bpe_split_entry_t* split_table = + model->backtrack_tables.split_table; +- const uint32_t* effective_rank = model->backtrack_tables.effective_rank; + + // Walk down the left spine until we find where the leftmost base token + // is consumed (i.e., where the left child is a base token). +@@ -89,7 +109,7 @@ static uint32_t iree_tokenizer_bpe_left_boundary_consumed_rank( + uint32_t left = split_table[token].left_id; + if (split_table[left].left_id == left) { + // Left child is a base token - it's consumed at this token's merge. +- return effective_rank[token]; ++ return iree_tokenizer_bpe_token_formation_rank(model, token); + } + // Recurse into left subtree. + token = left; +@@ -121,6 +141,67 @@ bool iree_tokenizer_bpe_is_first_token_reachable( + 1ull << (token % 64)); + } + ++static bool iree_tokenizer_bpe_decomposition_has_suffix_prefix_block( ++ const iree_tokenizer_bpe_model_t* model, uint32_t left, uint32_t right, ++ uint32_t token, uint32_t token_rank) { ++ const iree_tokenizer_bpe_split_entry_t* split_table = ++ model->backtrack_tables.split_table; ++ ++ uint32_t suffixes[32]; ++ uint32_t suffix_consumed_at[32]; ++ iree_host_size_t suffix_count = 0; ++ for (uint32_t current = left; ++ suffix_count < IREE_ARRAYSIZE(suffixes) && ++ split_table[current].left_id != current; ++ current = split_table[current].right_id) { ++ suffixes[suffix_count] = split_table[current].right_id; ++ suffix_consumed_at[suffix_count] = ++ iree_tokenizer_bpe_token_formation_rank(model, current); ++ suffix_count++; ++ } ++ if (suffix_count == 0) return false; ++ ++ uint32_t prefixes[32]; ++ uint32_t prefix_consumed_at[32]; ++ iree_host_size_t prefix_count = 0; ++ prefixes[prefix_count] = right; ++ prefix_consumed_at[prefix_count] = UINT32_MAX; ++ ++prefix_count; ++ for (uint32_t current = right; ++ prefix_count < IREE_ARRAYSIZE(prefixes) && ++ split_table[current].left_id != current; ++ current = split_table[current].left_id) { ++ prefixes[prefix_count] = split_table[current].left_id; ++ prefix_consumed_at[prefix_count] = ++ iree_tokenizer_bpe_token_formation_rank(model, current); ++ ++prefix_count; ++ } ++ ++ for (iree_host_size_t p = 0; p < prefix_count; ++p) { ++ for (iree_host_size_t s = 0; s < suffix_count; ++s) { ++ iree_tokenizer_merge_hash_result_t merge = ++ iree_tokenizer_vocab_merge_hash_lookup(model->merge_hash, ++ (int32_t)suffixes[s], ++ (int32_t)prefixes[p]); ++ if (!iree_tokenizer_merge_hash_result_is_valid(merge)) continue; ++ if ((uint32_t)merge.result_id == token) continue; ++ ++ uint32_t merge_effective_rank = merge.rank + 1; ++ uint32_t suffix_available_at = ++ iree_tokenizer_bpe_token_formation_rank(model, suffixes[s]); ++ uint32_t prefix_available_at = ++ iree_tokenizer_bpe_token_formation_rank(model, prefixes[p]); ++ if (merge_effective_rank < suffix_available_at) continue; ++ if (merge_effective_rank < prefix_available_at) continue; ++ if (merge_effective_rank >= token_rank) continue; ++ if (merge_effective_rank >= suffix_consumed_at[s]) continue; ++ if (merge_effective_rank > prefix_consumed_at[p]) continue; ++ return true; ++ } ++ } ++ return false; ++} ++ + // Checks if a specific decomposition (left + right) of a token is reachable. + // This is separate from is_first_token_reachable to allow checking alternative + // decompositions when a token has multiple merges producing it. +@@ -128,6 +209,22 @@ bool iree_tokenizer_bpe_is_decomposition_reachable( + const iree_tokenizer_bpe_model_t* model, uint32_t left, uint32_t right, + uint32_t token) { + const uint32_t* effective_rank = model->backtrack_tables.effective_rank; ++ uint32_t decomposition_rank = effective_rank[token]; ++ iree_tokenizer_merge_hash_result_t decomposition_merge = ++ iree_tokenizer_vocab_merge_hash_lookup(model->merge_hash, (int32_t)left, ++ (int32_t)right); ++ if (decomposition_merge.result_id == (int32_t)token) { ++ decomposition_rank = decomposition_merge.rank + 1; ++ } ++ ++ uint32_t left_available_at = ++ iree_tokenizer_bpe_token_formation_rank(model, left); ++ uint32_t right_available_at = ++ iree_tokenizer_bpe_token_formation_rank(model, right); ++ if (decomposition_rank < left_available_at || ++ decomposition_rank < right_available_at) { ++ return false; ++ } + + // Check for blocking merge at the split boundary. + // +@@ -158,17 +255,18 @@ bool iree_tokenizer_bpe_is_decomposition_reachable( + if (boundary_merge.result_id >= 0 && + (uint32_t)boundary_merge.result_id != token) { + // There's a different merge at this boundary. It blocks only if it fires +- // before EITHER boundary token is consumed by internal merges. +- uint32_t boundary_rank = effective_rank[(uint32_t)boundary_merge.result_id]; ++ // before the left boundary token is consumed by an internal merge and ++ // before or tied with the right boundary token's internal merge. Ties are ++ // resolved left-to-right: an internal merge in the left token starts before ++ // this boundary and wins the tie, while an internal merge in the right ++ // token starts after this boundary and loses the tie. ++ uint32_t boundary_rank = boundary_merge.rank + 1; + uint32_t left_boundary_consumed = + iree_tokenizer_bpe_right_boundary_consumed_rank(model, left); + uint32_t right_boundary_consumed = + iree_tokenizer_bpe_left_boundary_consumed_rank(model, right); +- uint32_t min_boundary_consumed = +- left_boundary_consumed < right_boundary_consumed +- ? left_boundary_consumed +- : right_boundary_consumed; +- if (boundary_rank < min_boundary_consumed) { ++ if (boundary_rank < left_boundary_consumed && ++ boundary_rank <= right_boundary_consumed) { + return false; // Blocking merge fires before either boundary is consumed. + } + } +@@ -180,7 +278,74 @@ bool iree_tokenizer_bpe_is_decomposition_reachable( + iree_tokenizer_bpe_is_first_token_reachable(model, left); + bool right_reachable = + iree_tokenizer_bpe_is_first_token_reachable(model, right); +- return left_reachable && right_reachable; ++ if (!left_reachable || !right_reachable) return false; ++ ++ return !iree_tokenizer_bpe_decomposition_has_suffix_prefix_block( ++ model, left, right, token, decomposition_rank); ++} ++ ++static bool iree_tokenizer_bpe_table_suffix_already_collected( ++ const uint32_t* suffixes, const uint32_t* suffix_consumed_at, ++ iree_host_size_t suffix_count, uint32_t suffix, uint32_t consumed_at) { ++ for (iree_host_size_t i = 0; i < suffix_count; ++i) { ++ if (suffixes[i] == suffix && suffix_consumed_at[i] == consumed_at) { ++ return true; ++ } ++ } ++ return false; ++} ++ ++static void iree_tokenizer_bpe_table_collect_reachable_suffixes( ++ const iree_tokenizer_bpe_model_t* model, uint32_t token, ++ uint32_t* suffixes, uint32_t* suffix_consumed_at, ++ iree_host_size_t* suffix_count, iree_host_size_t suffix_capacity, ++ int depth) { ++ if (*suffix_count >= suffix_capacity || depth >= 16) return; ++ ++ const iree_tokenizer_vocab_t* vocab = model->vocab; ++ const iree_tokenizer_bpe_split_entry_t* split_table = ++ model->backtrack_tables.split_table; ++ ++ if (split_table[token].left_id == token) return; // Base token. ++ ++ iree_string_view_t token_text = ++ iree_tokenizer_vocab_token_text(vocab, (int32_t)token); ++ ++ for (iree_host_size_t split_pos = 1; ++ split_pos < token_text.size && *suffix_count < suffix_capacity; ++ ++split_pos) { ++ iree_string_view_t left_text = ++ iree_make_string_view(token_text.data, split_pos); ++ iree_string_view_t right_text = iree_make_string_view( ++ token_text.data + split_pos, token_text.size - split_pos); ++ ++ int32_t left_id = iree_tokenizer_vocab_lookup(vocab, left_text); ++ int32_t right_id = iree_tokenizer_vocab_lookup(vocab, right_text); ++ if (left_id < 0 || right_id < 0) continue; ++ ++ iree_tokenizer_merge_hash_result_t merge = ++ iree_tokenizer_vocab_merge_hash_lookup(model->merge_hash, left_id, ++ right_id); ++ if (merge.result_id != (int32_t)token) continue; ++ ++ if (!iree_tokenizer_bpe_is_decomposition_reachable( ++ model, (uint32_t)left_id, (uint32_t)right_id, token)) { ++ continue; ++ } ++ ++ uint32_t consumed_at = merge.rank + 1; ++ if (!iree_tokenizer_bpe_table_suffix_already_collected( ++ suffixes, suffix_consumed_at, *suffix_count, (uint32_t)right_id, ++ consumed_at)) { ++ suffixes[*suffix_count] = (uint32_t)right_id; ++ suffix_consumed_at[*suffix_count] = consumed_at; ++ ++*suffix_count; ++ } ++ ++ iree_tokenizer_bpe_table_collect_reachable_suffixes( ++ model, (uint32_t)right_id, suffixes, suffix_consumed_at, suffix_count, ++ suffix_capacity, depth + 1); ++ } + } + + //===----------------------------------------------------------------------===// +@@ -213,6 +378,7 @@ iree_status_t iree_tokenizer_bpe_build_backtrack_tables( + iree_host_size_t next_prefix_offset = 0; + iree_host_size_t effective_rank_offset = 0; + iree_host_size_t token_reachable_offset = 0; ++ iree_host_size_t token_has_reachable_alternates_offset = 0; + IREE_RETURN_AND_END_ZONE_IF_ERROR( + z0, + IREE_STRUCT_LAYOUT( +@@ -222,7 +388,9 @@ iree_status_t iree_tokenizer_bpe_build_backtrack_tables( + IREE_STRUCT_FIELD(vocab_capacity, uint32_t, &next_prefix_offset), + IREE_STRUCT_FIELD(vocab_capacity, uint32_t, &effective_rank_offset), + IREE_STRUCT_FIELD(reachable_words, uint64_t, +- &token_reachable_offset))); ++ &token_reachable_offset), ++ IREE_STRUCT_FIELD(reachable_words, uint64_t, ++ &token_has_reachable_alternates_offset))); + + IREE_RETURN_AND_END_ZONE_IF_ERROR( + z0, iree_allocator_malloc(model->allocator, slab_size, +@@ -237,6 +405,8 @@ iree_status_t iree_tokenizer_bpe_build_backtrack_tables( + (uint32_t*)(slab + effective_rank_offset); + model->backtrack_tables.token_reachable = + (uint64_t*)(slab + token_reachable_offset); ++ model->backtrack_tables.token_has_reachable_alternates = ++ (uint64_t*)(slab + token_has_reachable_alternates_offset); + + // Initialize: base tokens map to (self, self), rank 0, no prefix. + for (iree_host_size_t i = 0; i < vocab_capacity; ++i) { +@@ -309,6 +479,30 @@ iree_status_t iree_tokenizer_bpe_build_backtrack_tables( + } + } + ++ // Gemma-style BPE vocabularies contain repeated initial-symbol tokens ++ // (metaspace, newline, and tab runs) whose merge entries are not a ++ // rank-ordered DAG: longer runs can appear before the shorter runs needed to ++ // build them. HuggingFace still matches these as direct vocabulary symbols. ++ // Treat them as base tokens so reachability and suffix blocking do not try ++ // to validate impossible synthetic merge trees. ++ for (iree_host_size_t token_id = 0; token_id < vocab_capacity; ++token_id) { ++ iree_tokenizer_token_attr_t attrs = ++ iree_tokenizer_vocab_token_attrs(vocab, (int32_t)token_id); ++ if (attrs & IREE_TOKENIZER_TOKEN_ATTR_SPECIAL) continue; ++ ++ iree_string_view_t text = ++ iree_tokenizer_vocab_token_text(vocab, (int32_t)token_id); ++ if (!iree_tokenizer_bpe_is_repeated_initial_symbol(text)) continue; ++ ++ model->backtrack_tables.split_table[token_id].left_id = ++ (uint32_t)token_id; ++ model->backtrack_tables.split_table[token_id].right_id = ++ (uint32_t)token_id; ++ if (model->backtrack_tables.effective_rank[token_id] == 0) { ++ model->backtrack_tables.effective_rank[token_id] = 1; ++ } ++ } ++ + // Mark multi-byte merge participants as matchable. + // + // Some base vocabulary tokens (e.g., SentencePiece's ▁ character, U+2581) +@@ -433,6 +627,8 @@ iree_status_t iree_tokenizer_bpe_build_backtrack_tables( + // and no blocking boundary merge. + memset(model->backtrack_tables.token_reachable, 0, + reachable_words * sizeof(uint64_t)); ++ memset(model->backtrack_tables.token_has_reachable_alternates, 0, ++ reachable_words * sizeof(uint64_t)); + + // Initialize: mark single-codepoint base tokens as reachable. + for (iree_host_size_t token_id = 0; token_id < vocab_capacity; ++token_id) { +@@ -448,7 +644,8 @@ iree_status_t iree_tokenizer_bpe_build_backtrack_tables( + codepoint_count++; + } + } +- if (codepoint_count <= 1) { ++ if (codepoint_count <= 1 || ++ iree_tokenizer_bpe_is_repeated_initial_symbol(token_text)) { + model->backtrack_tables.token_reachable[token_id / 64] |= + 1ull << (token_id % 64); + } +@@ -491,36 +688,10 @@ iree_status_t iree_tokenizer_bpe_build_backtrack_tables( + continue; + } + +- // Check for blocking merge at the boundary. +- uint32_t left_boundary = +- iree_tokenizer_bpe_rightmost_base_token(model, merge.left_id); +- uint32_t right_boundary = +- iree_tokenizer_bpe_leftmost_base_token(model, merge.right_id); +- +- iree_tokenizer_merge_hash_result_t boundary_merge = +- iree_tokenizer_vocab_merge_hash_lookup(model->merge_hash, +- (int32_t)left_boundary, +- (int32_t)right_boundary); +- if (boundary_merge.result_id >= 0 && +- (uint32_t)boundary_merge.result_id != (uint32_t)result_id) { +- // There's a different merge at this boundary. It blocks only if it +- // fires before EITHER boundary token is consumed by internal merges. +- uint32_t boundary_rank = +- model->backtrack_tables +- .effective_rank[(uint32_t)boundary_merge.result_id]; +- uint32_t left_boundary_consumed = +- iree_tokenizer_bpe_right_boundary_consumed_rank(model, +- merge.left_id); +- uint32_t right_boundary_consumed = +- iree_tokenizer_bpe_left_boundary_consumed_rank(model, +- merge.right_id); +- uint32_t min_boundary_consumed = +- left_boundary_consumed < right_boundary_consumed +- ? left_boundary_consumed +- : right_boundary_consumed; +- if (boundary_rank < min_boundary_consumed) { +- continue; // Blocking merge fires first. +- } ++ if (!iree_tokenizer_bpe_is_decomposition_reachable( ++ model, (uint32_t)merge.left_id, (uint32_t)merge.right_id, ++ (uint32_t)result_id)) { ++ continue; + } + + // This decomposition is reachable! +@@ -549,6 +720,175 @@ iree_status_t iree_tokenizer_bpe_build_backtrack_tables( + } + } + ++ // Mark tokens that have reachable decompositions besides the canonical ++ // split_table decomposition. Most tokens have exactly one reachable path and ++ // can use the cheap right-spine suffix walk during encode; ambiguous tokens ++ // need the slower all-splits suffix collection for HuggingFace parity. ++ for (iree_host_size_t rank = 0; rank < merge_count; ++rank) { ++ iree_tokenizer_merge_t merge = iree_tokenizer_vocab_merge(vocab, rank); ++ iree_tokenizer_merge_hash_result_t merge_result = ++ iree_tokenizer_vocab_merge_hash_lookup(model->merge_hash, ++ merge.left_id, merge.right_id); ++ int32_t result_id = merge_result.result_id; ++ if (result_id < 0 || (iree_host_size_t)result_id >= vocab_capacity) { ++ continue; ++ } ++ ++ uint32_t result = (uint32_t)result_id; ++ if (!iree_any_bit_set(model->backtrack_tables.token_reachable[result / 64], ++ 1ull << (result % 64))) { ++ continue; ++ } ++ if (!iree_tokenizer_bpe_is_decomposition_reachable( ++ model, (uint32_t)merge.left_id, (uint32_t)merge.right_id, ++ result)) { ++ continue; ++ } ++ ++ iree_tokenizer_bpe_split_entry_t canonical = ++ model->backtrack_tables.split_table[result]; ++ if (canonical.left_id != (uint32_t)merge.left_id || ++ canonical.right_id != (uint32_t)merge.right_id) { ++ model->backtrack_tables.token_has_reachable_alternates[result / 64] |= ++ 1ull << (result % 64); ++ } ++ } ++ ++ // Propagate the ambiguity bit to any token whose right-spine suffixes pass ++ // through an ambiguous token. Those parent tokens also need the precomputed ++ // all-splits suffix list during suffix blocking. ++ for (iree_host_size_t token_id = 0; token_id < vocab_capacity; ++token_id) { ++ for (uint32_t current = (uint32_t)token_id; ++ model->backtrack_tables.split_table[current].left_id != current; ++ current = model->backtrack_tables.split_table[current].right_id) { ++ if (iree_any_bit_set( ++ model->backtrack_tables ++ .token_has_reachable_alternates[current / 64], ++ 1ull << (current % 64))) { ++ model->backtrack_tables ++ .token_has_reachable_alternates[token_id / 64] |= ++ 1ull << (token_id % 64); ++ break; ++ } ++ } ++ } ++ ++ for (iree_host_size_t token_id = 0; token_id < vocab_capacity; ++token_id) { ++ uint32_t cheap_suffixes[32]; ++ uint32_t cheap_consumed_at[32]; ++ iree_host_size_t cheap_count = 0; ++ for (uint32_t current = (uint32_t)token_id; ++ cheap_count < IREE_ARRAYSIZE(cheap_suffixes) && ++ model->backtrack_tables.split_table[current].left_id != current; ++ current = model->backtrack_tables.split_table[current].right_id) { ++ cheap_suffixes[cheap_count] = ++ model->backtrack_tables.split_table[current].right_id; ++ cheap_consumed_at[cheap_count] = ++ iree_tokenizer_bpe_token_formation_rank(model, current); ++ ++cheap_count; ++ } ++ ++ uint32_t all_suffixes[32]; ++ uint32_t all_consumed_at[32]; ++ iree_host_size_t all_count = 0; ++ iree_tokenizer_bpe_table_collect_reachable_suffixes( ++ model, (uint32_t)token_id, all_suffixes, all_consumed_at, &all_count, ++ IREE_ARRAYSIZE(all_suffixes), 0); ++ ++ bool differs = cheap_count != all_count; ++ for (iree_host_size_t i = 0; i < all_count && !differs; ++i) { ++ bool found = false; ++ for (iree_host_size_t j = 0; j < cheap_count; ++j) { ++ if (all_suffixes[i] == cheap_suffixes[j] && ++ all_consumed_at[i] == cheap_consumed_at[j]) { ++ found = true; ++ break; ++ } ++ } ++ differs = !found; ++ } ++ if (differs) { ++ model->backtrack_tables.token_has_reachable_alternates[token_id / 64] |= ++ 1ull << (token_id % 64); ++ } ++ } ++ ++ iree_host_size_t total_alternate_suffixes = 0; ++ for (iree_host_size_t token_id = 0; token_id < vocab_capacity; ++token_id) { ++ if (!iree_any_bit_set( ++ model->backtrack_tables ++ .token_has_reachable_alternates[token_id / 64], ++ 1ull << (token_id % 64))) { ++ continue; ++ } ++ uint32_t suffixes[32]; ++ uint32_t suffix_consumed_at[32]; ++ iree_host_size_t suffix_count = 0; ++ iree_tokenizer_bpe_table_collect_reachable_suffixes( ++ model, (uint32_t)token_id, suffixes, suffix_consumed_at, &suffix_count, ++ IREE_ARRAYSIZE(suffixes), 0); ++ total_alternate_suffixes += suffix_count; ++ } ++ ++ if (total_alternate_suffixes > 0) { ++ iree_host_size_t alt_slab_size = 0; ++ iree_host_size_t alt_offsets_offset = 0; ++ iree_host_size_t alt_suffixes_offset = 0; ++ iree_host_size_t alt_consumed_offset = 0; ++ IREE_RETURN_AND_END_ZONE_IF_ERROR( ++ z0, ++ IREE_STRUCT_LAYOUT( ++ /*base_size=*/0, &alt_slab_size, ++ IREE_STRUCT_FIELD(vocab_capacity + 1, uint32_t, ++ &alt_offsets_offset), ++ IREE_STRUCT_FIELD(total_alternate_suffixes, uint32_t, ++ &alt_suffixes_offset), ++ IREE_STRUCT_FIELD(total_alternate_suffixes, uint32_t, ++ &alt_consumed_offset))); ++ ++ IREE_RETURN_AND_END_ZONE_IF_ERROR( ++ z0, iree_allocator_malloc(model->allocator, alt_slab_size, ++ &model->backtrack_tables ++ .alternate_suffix_slab)); ++ ++ uint8_t* alt_slab = ++ (uint8_t*)model->backtrack_tables.alternate_suffix_slab; ++ model->backtrack_tables.alternate_suffix_offsets = ++ (uint32_t*)(alt_slab + alt_offsets_offset); ++ model->backtrack_tables.alternate_suffixes = ++ (uint32_t*)(alt_slab + alt_suffixes_offset); ++ model->backtrack_tables.alternate_suffix_consumed_at = ++ (uint32_t*)(alt_slab + alt_consumed_offset); ++ ++ iree_host_size_t write_index = 0; ++ for (iree_host_size_t token_id = 0; token_id < vocab_capacity; ++ ++token_id) { ++ model->backtrack_tables.alternate_suffix_offsets[token_id] = ++ (uint32_t)write_index; ++ if (!iree_any_bit_set( ++ model->backtrack_tables ++ .token_has_reachable_alternates[token_id / 64], ++ 1ull << (token_id % 64))) { ++ continue; ++ } ++ ++ uint32_t suffixes[32]; ++ uint32_t suffix_consumed_at[32]; ++ iree_host_size_t suffix_count = 0; ++ iree_tokenizer_bpe_table_collect_reachable_suffixes( ++ model, (uint32_t)token_id, suffixes, suffix_consumed_at, ++ &suffix_count, IREE_ARRAYSIZE(suffixes), 0); ++ for (iree_host_size_t i = 0; i < suffix_count; ++i) { ++ model->backtrack_tables.alternate_suffixes[write_index] = suffixes[i]; ++ model->backtrack_tables.alternate_suffix_consumed_at[write_index] = ++ suffix_consumed_at[i]; ++ ++write_index; ++ } ++ } ++ model->backtrack_tables.alternate_suffix_offsets[vocab_capacity] = ++ (uint32_t)write_index; ++ } ++ + IREE_TRACE_ZONE_END(z0); + return iree_ok_status(); + } +diff --git a/runtime/src/iree/tokenizer/model/bpe_test.cc b/runtime/src/iree/tokenizer/model/bpe_test.cc +index f2b82260d5..5bac3f0f44 100644 +--- a/runtime/src/iree/tokenizer/model/bpe_test.cc ++++ b/runtime/src/iree/tokenizer/model/bpe_test.cc +@@ -82,13 +82,13 @@ TEST_F(BPEModelTest, StateSizeIsReasonable) { + // tokens. For max_token_length=1, vocab_capacity=1: + // window: 2 * 12 = 24 + // heap: 3 * 8 = 24 +- // backtrack stack: 2048 * 8 = 16,384 +- // backtrack bitfield: 33 * 8 = 264 ++ // backtrack stack: 32768 * 12 = 393,216 ++ // backtrack bitfield: 513 * 8 = 4,104 + // pair cache: 4096 * 8 = 32,768 + // word cache: 0 (disabled, vocab < 256) + // struct: ~128 +- // total: ~49,592 +- EXPECT_LE(state_size, 65536u); ++ // total: ~430,240 ++ EXPECT_LE(state_size, 512 * 1024u); + } + + //===----------------------------------------------------------------------===// +@@ -2738,6 +2738,43 @@ TEST_F(BPEModelTest, MergePriorityWithPrecedingToken) { + /*expect_pending_after_encode=*/false); + } + ++// Gemma-style tokenizers use U+2581 metaspace as a multi-byte base token. Very ++// long segments bypass the backtracking path and use the window/heap encoder, ++// which must still seed base tokens before applying rank-priority merges. ++TEST_F(BPEModelTest, LongSegmentMergePriorityAfterMultiByteBaseToken) { ++ ScopedVocabBuilder builder; ++ ++ builder.AddToken(0, "▁"); ++ builder.AddToken(1, "z"); ++ builder.AddToken(2, "m"); ++ builder.AddToken(3, "l"); ++ builder.AddToken(4, "x"); ++ builder.AddToken(5, "▁z"); ++ builder.AddToken(6, "ml"); ++ builder.AddToken(7, "▁zm"); ++ builder.AddToken(8, "zm"); ++ ++ builder.AddMerge(0, 1); // Rank 0: ▁ + z -> ▁z ++ builder.AddMerge(2, 3); // Rank 1: m + l -> ml ++ builder.AddMerge(5, 2); // Rank 2: ▁z + m -> ▁zm ++ builder.AddMerge(0, 8); // Rank 3: ▁ + zm -> ▁zm (alternate path) ++ builder.AddMerge(1, 2); // Rank 4: z + m -> zm ++ ++ CreateModel(builder); ++ ++ std::string input = "▁zml"; ++ input.append(9000, 'x'); ++ auto tokens = EncodeAndFinalize(model(), input, ++ /*expect_pending_after_encode=*/false); ++ ++ ASSERT_EQ(tokens.size(), 9002u); ++ EXPECT_EQ(tokens[0], 5); // ▁z ++ EXPECT_EQ(tokens[1], 6); // ml ++ for (size_t i = 2; i < tokens.size(); ++i) { ++ EXPECT_EQ(tokens[i], 4); ++ } ++} ++ + // Tests ByteLevel transformation of multi-byte UTF-8 sequences. + // This is the Qwen unicode bug scenario where 'él' (UTF-8: c3 a9 6c) should + // transform to 'él' (UTF-8: c3 83 c2 a9 6c) and match as a single token. +diff --git a/runtime/src/iree/tokenizer/tokenizer.c b/runtime/src/iree/tokenizer/tokenizer.c +index 2299f0de7e..76011353f9 100644 +--- a/runtime/src/iree/tokenizer/tokenizer.c ++++ b/runtime/src/iree/tokenizer/tokenizer.c +@@ -1571,6 +1571,19 @@ static iree_status_t iree_tokenizer_encode_ring_as_segment( + state->transform_buffer.data, physical_write); + } + ++ if (is_partial) { ++ iree_host_size_t incomplete_tail = ++ iree_unicode_utf8_incomplete_tail_length( ++ (const char*)state->transform_buffer.data + physical_read, ++ data_length); ++ data_length -= incomplete_tail; ++ if (data_length == 0) { ++ *out_tokens_written = 0; ++ *out_segment_complete = false; ++ return iree_ok_status(); ++ } ++ } ++ + iree_tokenizer_segment_t segment = { + .start = physical_read, + .end = physical_read + data_length, +diff --git a/runtime/src/iree/tokenizer/tokenizer.h b/runtime/src/iree/tokenizer/tokenizer.h +index 098e6fef87..40db2bb459 100644 +--- a/runtime/src/iree/tokenizer/tokenizer.h ++++ b/runtime/src/iree/tokenizer/tokenizer.h +@@ -32,16 +32,17 @@ extern "C" { + // Maximum transform buffer allocation. + // Recommended values: + // 8KB: Minimal memory, suitable for most English text +-// 16KB: Good balance for mixed content and cache locality ++// 64KB: Keeps a 32KB logical ring so large metaspace-normalized prompts can ++// stay on the exact BPE path instead of the approximate streaming path. + // + // The transform buffer determines the maximum segment size that the segmenter + // pipeline processes. Larger buffers allow longer segments, but segments that +-// exceed L1 data cache (typically 32KB) cause DFA cache misses during +-// multi-stage pre-tokenization. 16KB (8KB logical capacity after ring buffer +-// halving) keeps the working set cache-friendly for segmenters with multiple +-// children (e.g., DeepSeek V3's 4-stage Sequence pre-tokenizer). ++// exceed L1 data cache (typically 32KB) can cause DFA cache misses during ++// multi-stage pre-tokenization. 64KB (32KB logical capacity after ring buffer ++// halving) favors HuggingFace-compatible exact BPE for long normalized ++// segments. + #ifndef IREE_TOKENIZER_TRANSFORM_BUFFER_MAX_SIZE +-#define IREE_TOKENIZER_TRANSFORM_BUFFER_MAX_SIZE (16 * 1024) ++#define IREE_TOKENIZER_TRANSFORM_BUFFER_MAX_SIZE (64 * 1024) + #endif + + // Normalizer expansion factor for transform buffer sizing. diff --git a/third_party/iree/mistral-right-aligned-numbers.patch b/third_party/iree/mistral-right-aligned-numbers.patch new file mode 100644 index 00000000..ca6e3fe5 --- /dev/null +++ b/third_party/iree/mistral-right-aligned-numbers.patch @@ -0,0 +1,303 @@ +diff --git a/runtime/src/iree/tokenizer/format/huggingface/segmenter_json.c b/runtime/src/iree/tokenizer/format/huggingface/segmenter_json.c +--- a/runtime/src/iree/tokenizer/format/huggingface/segmenter_json.c ++++ b/runtime/src/iree/tokenizer/format/huggingface/segmenter_json.c +@@ -225,6 +225,29 @@ + return iree_ok_status(); + } + ++ // Tekken groups numbers from the right using a positive lookahead. Match the ++ // entire number run with the DFA, then subdivide it in the Split segmenter. ++ // Restrict this lowering to the exact known pattern and isolated behavior. ++ bool right_aligned_numbers = false; ++ if (behavior == IREE_TOKENIZER_UTIL_REGEX_SPLIT_ISOLATED && !invert && ++ iree_string_view_equal( ++ pattern, ++ IREE_SV("[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*" ++ "[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|" ++ "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+" ++ "[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|" ++ "\\p{N}{1,3}(?=(?:\\p{N}{3})*(?!\\p{N}))|\\p{N}{1,2}|" ++ " ?[^\\s\\p{L}\\p{N}]+[\\r\\n/" ++ "]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"))) { ++ pattern = IREE_SV( ++ "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*" ++ "[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|" ++ "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+" ++ "[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}+|" ++ " ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"); ++ right_aligned_numbers = true; ++ } ++ + // Compile the regex pattern to a DFA. + iree_tokenizer_regex_dfa_t dfa; + uint8_t* dfa_data = NULL; +@@ -239,8 +262,13 @@ + } + + // Create the Split segmenter. On failure, free DFA data. +- status = iree_tokenizer_segmenter_split_allocate( +- dfa, dfa_data, behavior, invert, allocator, out_segmenter); ++ if (right_aligned_numbers) { ++ status = iree_tokenizer_segmenter_split_right_aligned_numbers_allocate( ++ dfa, dfa_data, allocator, out_segmenter); ++ } else { ++ status = iree_tokenizer_segmenter_split_allocate( ++ dfa, dfa_data, behavior, invert, allocator, out_segmenter); ++ } + if (!iree_status_is_ok(status)) { + iree_allocator_free(allocator, dfa_data); + } +diff --git a/runtime/src/iree/tokenizer/segmenter/BUILD.bazel b/runtime/src/iree/tokenizer/segmenter/BUILD.bazel +--- a/runtime/src/iree/tokenizer/segmenter/BUILD.bazel ++++ b/runtime/src/iree/tokenizer/segmenter/BUILD.bazel +@@ -188,6 +188,7 @@ + hdrs = ["split.h"], + deps = [ + "//runtime/src/iree/base", ++ "//runtime/src/iree/base/internal:unicode", + "//runtime/src/iree/tokenizer:segmenter", + "//runtime/src/iree/tokenizer/regex:exec", + ], +diff --git a/runtime/src/iree/tokenizer/segmenter/CMakeLists.txt b/runtime/src/iree/tokenizer/segmenter/CMakeLists.txt +--- a/runtime/src/iree/tokenizer/segmenter/CMakeLists.txt ++++ b/runtime/src/iree/tokenizer/segmenter/CMakeLists.txt +@@ -201,6 +201,7 @@ + "split.c" + DEPS + iree::base ++ iree::base::internal::unicode + iree::tokenizer::regex::exec + iree::tokenizer::segmenter + PUBLIC +diff --git a/runtime/src/iree/tokenizer/segmenter/split.c b/runtime/src/iree/tokenizer/segmenter/split.c +--- a/runtime/src/iree/tokenizer/segmenter/split.c ++++ b/runtime/src/iree/tokenizer/segmenter/split.c +@@ -7,6 +7,8 @@ + #include "iree/tokenizer/segmenter/split.h" + + #include ++ ++#include "iree/base/internal/unicode.h" + + //===----------------------------------------------------------------------===// + // Types +@@ -19,6 +21,7 @@ + bool invert; + // false = regex mode, true = literal mode. + bool is_literal; ++ bool right_aligned_numbers; + union { + struct { + iree_tokenizer_regex_dfa_t dfa; +@@ -361,7 +364,46 @@ + iree_tokenizer_regex_split_behavior_t behavior; + bool invert; + bool absolute; ++ bool right_aligned_numbers; ++ iree_string_view_t input; + } iree_tokenizer_split_callback_context_t; ++ ++// The caller has recognized a number-only match. Count codepoints, rather than ++// bytes, so mixed Unicode number categories retain the same grouping as \p{N}. ++static iree_status_t iree_tokenizer_split_emit_number_match( ++ iree_tokenizer_split_callback_context_t* context, ++ iree_tokenizer_regex_match_t match, iree_string_view_t text) { ++ iree_host_size_t position = 0; ++ iree_host_size_t count = 0; ++ while (position < text.size) { ++ iree_unicode_utf8_decode(text, &position); ++ ++count; ++ } ++ ++ iree_tokenizer_split_emit(context->emitter, context->last_emit_end, ++ match.start, context->absolute); ++ position = 0; ++ iree_host_size_t start = match.start; ++ iree_host_size_t group_size = (count - 1) % 3 + 1; ++ while (position < text.size) { ++ for (iree_host_size_t i = 0; i < group_size; ++i) { ++ iree_unicode_utf8_decode(text, &position); ++ } ++ iree_host_size_t end = match.start + position; ++ // finalize() can retry the same match after partially filling the output. ++ if (end > context->last_emit_end) { ++ iree_tokenizer_split_emit(context->emitter, start, end, ++ context->absolute); ++ if (context->emitter->full) { ++ return iree_status_from_code(IREE_STATUS_RESOURCE_EXHAUSTED); ++ } ++ } ++ start = end; ++ group_size = 3; ++ } ++ context->last_emit_end = match.end; ++ return iree_ok_status(); ++} + + // Processes each regex match inline: computes behavior result, emits segments, + // and commits state. Returns RESOURCE_EXHAUSTED if output capacity is +@@ -370,6 +412,20 @@ + void* user_data, iree_tokenizer_regex_match_t match) { + iree_tokenizer_split_callback_context_t* context = + (iree_tokenizer_split_callback_context_t*)user_data; ++ ++ if (context->right_aligned_numbers && match.end > match.start) { ++ iree_host_size_t base = context->emitter->chunk_base; ++ if (match.start < base || match.end - base > context->input.size) { ++ return iree_make_status(IREE_STATUS_INTERNAL, ++ "number match outside retained input"); ++ } ++ iree_string_view_t text = iree_make_string_view( ++ context->input.data + match.start - base, match.end - match.start); ++ iree_host_size_t position = 0; ++ if (iree_unicode_is_number(iree_unicode_utf8_decode(text, &position))) { ++ return iree_tokenizer_split_emit_number_match(context, match, text); ++ } ++ } + + // Compute behavior result (pure, no side effects). + // Gap is the region between the last emit and the current match start. +@@ -590,6 +646,7 @@ + segmenter->behavior = behavior; + segmenter->invert = invert; + segmenter->is_literal = false; ++ segmenter->right_aligned_numbers = false; + segmenter->regex.dfa = dfa; + segmenter->regex.dfa_data = dfa_data; + +@@ -605,6 +662,17 @@ + } + IREE_TRACE_ZONE_END(z0); + return status; ++} ++ ++iree_status_t iree_tokenizer_segmenter_split_right_aligned_numbers_allocate( ++ iree_tokenizer_regex_dfa_t dfa, uint8_t* dfa_data, ++ iree_allocator_t allocator, iree_tokenizer_segmenter_t** out_segmenter) { ++ IREE_RETURN_IF_ERROR(iree_tokenizer_segmenter_split_allocate( ++ dfa, dfa_data, IREE_TOKENIZER_UTIL_REGEX_SPLIT_ISOLATED, false, allocator, ++ out_segmenter)); ++ ((iree_tokenizer_segmenter_split_t*)*out_segmenter)->right_aligned_numbers = ++ true; ++ return iree_ok_status(); + } + + iree_status_t iree_tokenizer_segmenter_split_literal_allocate( +@@ -641,6 +709,7 @@ + segmenter->behavior = behavior; + segmenter->invert = invert; + segmenter->is_literal = true; ++ segmenter->right_aligned_numbers = false; + segmenter->literal.pattern_data = pattern_data; + segmenter->literal.pattern = + iree_make_string_view((const char*)pattern_data, pattern.size); +@@ -739,6 +808,8 @@ + .behavior = self->behavior, + .invert = self->invert, + .absolute = false, ++ .right_aligned_numbers = segmenter->right_aligned_numbers, ++ .input = input, + }; + + // Feed pattern matcher; matches are processed inline via callback. +@@ -882,6 +953,8 @@ + .behavior = self->behavior, + .invert = self->invert, + .absolute = false, ++ .right_aligned_numbers = segmenter->right_aligned_numbers, ++ .input = remaining_input, + }; + + iree_host_size_t input_end = chunk_base + remaining_input.size; +diff --git a/runtime/src/iree/tokenizer/segmenter/split.h b/runtime/src/iree/tokenizer/segmenter/split.h +--- a/runtime/src/iree/tokenizer/segmenter/split.h ++++ b/runtime/src/iree/tokenizer/segmenter/split.h +@@ -81,6 +81,14 @@ + iree_tokenizer_regex_split_behavior_t behavior, bool invert, + iree_allocator_t allocator, iree_tokenizer_segmenter_t** out_segmenter); + ++// Like ISOLATED splitting, but subdivides number-only matches into groups of ++// three Unicode numbers from the right. The DFA must match entire number runs ++// separately from other text. This implements Tekken's numeric lookahead ++// without requiring general positive lookahead support in the regex compiler. ++iree_status_t iree_tokenizer_segmenter_split_right_aligned_numbers_allocate( ++ iree_tokenizer_regex_dfa_t dfa, uint8_t* dfa_data, ++ iree_allocator_t allocator, iree_tokenizer_segmenter_t** out_segmenter); ++ + // Allocates a segmenter that splits text on literal string matches. + // Unlike the regex version, this uses efficient byte-by-byte substring search. + // The |pattern| is copied and owned by the segmenter. +diff --git a/runtime/src/iree/tokenizer/segmenter/split_test.cc b/runtime/src/iree/tokenizer/segmenter/split_test.cc +--- a/runtime/src/iree/tokenizer/segmenter/split_test.cc ++++ b/runtime/src/iree/tokenizer/segmenter/split_test.cc +@@ -28,7 +28,8 @@ + public: + ScopedSplitSegmenter(const char* pattern, + iree_tokenizer_regex_split_behavior_t behavior, +- bool invert = false) { ++ bool invert = false, ++ bool right_aligned_numbers = false) { + iree_tokenizer_regex_compile_error_t error = {0}; + status_ = iree_tokenizer_regex_compile_and_load( + iree_make_cstring_view(pattern), +@@ -37,8 +38,13 @@ + if (!iree_status_is_ok(status_)) return; + + iree_tokenizer_segmenter_t* raw = nullptr; +- status_ = iree_tokenizer_segmenter_split_allocate( +- dfa_, dfa_storage_, behavior, invert, iree_allocator_system(), &raw); ++ if (right_aligned_numbers) { ++ status_ = iree_tokenizer_segmenter_split_right_aligned_numbers_allocate( ++ dfa_, dfa_storage_, iree_allocator_system(), &raw); ++ } else { ++ status_ = iree_tokenizer_segmenter_split_allocate( ++ dfa_, dfa_storage_, behavior, invert, iree_allocator_system(), &raw); ++ } + if (!iree_status_is_ok(status_)) return; + + segmenter_ = ScopedSegmenter(raw); +@@ -72,6 +78,44 @@ + IREE_TOKENIZER_UTIL_REGEX_SPLIT_ISOLATED); + IREE_ASSERT_OK(segmenter.status()); + ASSERT_NE(segmenter.get(), nullptr); ++} ++ ++TEST(SplitSegmenterTest, RightAlignedNumbers) { ++ ScopedSplitSegmenter segmenter("\\p{N}+|\\p{L}+|\\s+", ++ IREE_TOKENIZER_UTIL_REGEX_SPLIT_ISOLATED, ++ false, true); ++ IREE_ASSERT_OK(segmenter.status()); ++ auto input = IREE_SV("a1 12 123 1234 12345 123456 1234567z"); ++ std::vector expected = { ++ "a", "1", " ", "12", " ", "123", " ", "1", "234", " ", ++ "12", "345", " ", "123", "456", " ", "1", "234", "567", "z"}; ++ TestWithAllChunkSizes(segmenter.get(), input, expected, false); ++ TestLimitedOutputCapacity(segmenter.get(), input, expected); ++ TestZeroCapacityOutput(segmenter.get(), input); ++} ++ ++TEST(SplitSegmenterTest, RightAlignedUnicodeNumbers) { ++ ScopedSplitSegmenter segmenter("\\p{N}+|\\p{L}+|\\s+", ++ IREE_TOKENIZER_UTIL_REGEX_SPLIT_ISOLATED, ++ false, true); ++ IREE_ASSERT_OK(segmenter.status()); ++ // Nd, Nl and No, including mixed UTF-8 lengths within a number run. ++ auto input = IREE_SV("١٢٣٤ 12345 Ⅷ²3४"); ++ std::vector expected = {"١", "٢٣٤", " ", "12", ++ "345", " ", "Ⅷ", "²3४"}; ++ TestWithAllChunkSizesUtf8(segmenter.get(), input, expected, false); ++ TestLimitedOutputCapacity(segmenter.get(), input, expected); ++} ++ ++TEST(SplitSegmenterTest, RightAlignedNumbersLongRun) { ++ ScopedSplitSegmenter segmenter( ++ "\\p{N}+", IREE_TOKENIZER_UTIL_REGEX_SPLIT_ISOLATED, false, true); ++ IREE_ASSERT_OK(segmenter.status()); ++ std::string input(1000, '1'); ++ std::vector expected(334, "111"); ++ expected[0] = "1"; ++ TestLimitedOutputCapacity(segmenter.get(), ++ iree_make_cstring_view(input.c_str()), expected); + } + + TEST(SplitSegmenterTest, StateSizeIsReasonable) { diff --git a/third_party/iree/repo.bzl b/third_party/iree/repo.bzl new file mode 100644 index 00000000..ee89fb6d --- /dev/null +++ b/third_party/iree/repo.bzl @@ -0,0 +1,27 @@ +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") +load("@bazel_tools//tools/build_defs/repo:local.bzl", "new_local_repository") + +def repo(): + git_repository( + name = "iree", + remote = "https://github.com/iree-org/iree.git", + commit = "4d4e97d00f099a21f38eeff26f82a6d9e3643a11", + sparse_checkout_patterns = [ + "runtime/src/**", + "build_tools/bazel/**", + "build_tools/third_party/libbacktrace/**", + "BUILD.bazel", + ], + patches = [ + "//third_party/iree:tokenizer-only.patch", + "//third_party/iree:fix-added-token-matching.patch", + "//third_party/iree:match-hf-tokenizer.patch", + "//third_party/iree:laguna-tokenizer-compat.patch", + "//third_party/iree:fix-normalizer-lifetime-and-boundaries.patch", + "//third_party/iree:mistral-right-aligned-numbers.patch", + ], + patch_args = ["-p1"], + ) + + # Use this if you want to use a local copy of IREE instead of the git repository. Make sure to update the path to point to your local IREE checkout. + #new_local_repository(name = "iree", build_file="//:third_party/iree/BUILD.bazel", path="/home/corendos/iree/") diff --git a/third_party/iree/tokenizer-only.patch b/third_party/iree/tokenizer-only.patch new file mode 100644 index 00000000..c6be27da --- /dev/null +++ b/third_party/iree/tokenizer-only.patch @@ -0,0 +1,50 @@ +diff --git a/build_tools/bazel/build_defs.oss.bzl b/build_tools/bazel/build_defs.oss.bzl +--- a/build_tools/bazel/build_defs.oss.bzl ++++ b/build_tools/bazel/build_defs.oss.bzl +@@ -4,24 +4,35 @@ + # See https://llvm.org/LICENSE.txt for license information. + # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +-load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library", "gentbl_filegroup", "td_library") ++# load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library", "gentbl_filegroup", "td_library") + + # All load statements must come first in Starlark. + load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_test") +-load("@rules_shell//shell:sh_test.bzl", "sh_test") +-load( +- "//build_tools/bazel:iree_cc_fuzz.bzl", +- _iree_cc_fuzz = "iree_cc_fuzz", +- _iree_compiler_cc_fuzz = "iree_compiler_cc_fuzz", +- _iree_runtime_cc_fuzz = "iree_runtime_cc_fuzz", +-) ++# load("@rules_shell//shell:sh_test.bzl", "sh_test") ++# load( ++# "//build_tools/bazel:iree_cc_fuzz.bzl", ++# _iree_cc_fuzz = "iree_cc_fuzz", ++# _iree_compiler_cc_fuzz = "iree_compiler_cc_fuzz", ++# _iree_runtime_cc_fuzz = "iree_runtime_cc_fuzz", ++# ) + + """Common Bazel definitions for IREE.""" + + # Re-export fuzz rules for external use. +-iree_cc_fuzz = _iree_cc_fuzz +-iree_compiler_cc_fuzz = _iree_compiler_cc_fuzz +-iree_runtime_cc_fuzz = _iree_runtime_cc_fuzz ++# iree_cc_fuzz = _iree_cc_fuzz ++# iree_compiler_cc_fuzz = _iree_compiler_cc_fuzz ++# iree_runtime_cc_fuzz = _iree_runtime_cc_fuzz ++ ++def generic_pass(**kwargs): ++ pass ++ ++gentbl_cc_library = generic_pass ++gentbl_filegroup = generic_pass ++td_library = generic_pass ++sh_test = generic_pass ++iree_runtime_cc_fuzz = generic_pass ++iree_compiler_cc_fuzz = generic_pass ++iree_cc_fuzz = generic_pass + + def defaulting_select(selector): + """Pass through to select() with special semantics when converting to CMake. diff --git a/third_party/libvaxis/BUILD.bazel b/third_party/libvaxis/BUILD.bazel new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/third_party/libvaxis/BUILD.bazel @@ -0,0 +1 @@ + diff --git a/third_party/libvaxis/fixes.patch b/third_party/libvaxis/fixes.patch new file mode 100644 index 00000000..effbb2aa --- /dev/null +++ b/third_party/libvaxis/fixes.patch @@ -0,0 +1,202 @@ +diff --git a/src/Loop.zig b/src/Loop.zig +--- a/src/Loop.zig ++++ b/src/Loop.zig +@@ -352,7 +352,7 @@ + } + }, + .cap_da1 => { +- std.Io.futexWake(vx.io, std.atomic.Value(u32), &vx.query_futex, 10); ++ std.Io.futexWake(vx.io, u32, &vx.query_futex.raw, 10); + vx.queries_done.store(true, .unordered); + }, + .mouse => |mouse| { +@@ -484,7 +484,7 @@ + vx.caps.program_status = true; + }, + .cap_da1 => { +- std.Io.futexWake(vx.io, std.atomic.Value(u32), &vx.query_futex, 10); ++ std.Io.futexWake(vx.io, u32, &vx.query_futex.raw, 10); + vx.queries_done.store(true, .unordered); + }, + .winsize => |winsize| { +diff --git a/src/Vaxis.zig b/src/Vaxis.zig +--- a/src/Vaxis.zig ++++ b/src/Vaxis.zig +@@ -139,14 +139,13 @@ + } + } + +-/// resets enabled features, sends cursor to home and clears below cursor +-pub fn resetState(self: *Vaxis, tty: *std.Io.Writer) !void { +- // always show the cursor on state reset ++/// Resets enabled terminal features without clearing screen content. ++/// Use this for non-interactive / single-shot rendering where the ++/// output should remain visible after exit. ++pub fn resetModes(self: *Vaxis, tty: *std.Io.Writer) !void { + tty.writeAll(ctlseqs.show_cursor) catch {}; + tty.writeAll(ctlseqs.sgr_reset) catch {}; + if (self.screen.cursor_shape != .default) { +- // In many terminals, `.default` will set to the configured cursor shape. Others, it will +- // change to a blinking block. + tty.print(ctlseqs.cursor_shape, .{@backingInt(Cell.CursorShape.default)}) catch {}; + } + if (self.state.kitty_keyboard) { +@@ -159,6 +158,32 @@ + if (self.state.bracketed_paste) { + try self.setBracketedPaste(tty, false); + } ++ if (self.state.color_scheme_updates) { ++ try tty.writeAll(ctlseqs.color_scheme_reset); ++ self.state.color_scheme_updates = false; ++ } ++ if (self.state.in_band_resize) { ++ try tty.writeAll(ctlseqs.in_band_resize_reset); ++ self.state.in_band_resize = false; ++ } ++ if (self.state.changed_default_fg) { ++ try tty.writeAll(ctlseqs.osc10_reset); ++ self.state.changed_default_fg = false; ++ } ++ if (self.state.changed_default_bg) { ++ try tty.writeAll(ctlseqs.osc11_reset); ++ self.state.changed_default_bg = false; ++ } ++ if (self.state.changed_cursor_color) { ++ try tty.writeAll(ctlseqs.osc12_reset); ++ self.state.changed_cursor_color = false; ++ } ++ try tty.flush(); ++} ++ ++/// resets enabled features, sends cursor to home and clears below cursor ++pub fn resetState(self: *Vaxis, tty: *std.Io.Writer) !void { ++ self.resetModes(tty) catch {}; + if (self.state.alt_screen) { + try tty.writeAll(ctlseqs.home); + try tty.writeAll(ctlseqs.erase_below_cursor); +@@ -171,27 +196,6 @@ + } + try tty.writeAll(ctlseqs.erase_below_cursor); + } +- if (self.state.color_scheme_updates) { +- try tty.writeAll(ctlseqs.color_scheme_reset); +- self.state.color_scheme_updates = false; +- } +- if (self.state.in_band_resize) { +- try tty.writeAll(ctlseqs.in_band_resize_reset); +- self.state.in_band_resize = false; +- } +- if (self.state.changed_default_fg) { +- try tty.writeAll(ctlseqs.osc10_reset); +- self.state.changed_default_fg = false; +- } +- if (self.state.changed_default_bg) { +- try tty.writeAll(ctlseqs.osc11_reset); +- self.state.changed_default_bg = false; +- } +- if (self.state.changed_cursor_color) { +- try tty.writeAll(ctlseqs.osc12_reset); +- self.state.changed_cursor_color = false; +- } +- + try tty.flush(); + } + +@@ -1101,19 +1105,19 @@ + switch (format) { + .rgb => { + try tty.print( +- "\x1b_Gf=24,s={d},v={d},i={d},t={c};{s}\x1b\\", ++ "\x1b_Gf=24,s={d},v={d},i={d},t={c},q=2;{s}\x1b\\", + .{ width, height, id, medium_char, encoded }, + ); + }, + .rgba => { + try tty.print( +- "\x1b_Gf=32,s={d},v={d},i={d},t={c};{s}\x1b\\", ++ "\x1b_Gf=32,s={d},v={d},i={d},t={c},q=2;{s}\x1b\\", + .{ width, height, id, medium_char, encoded }, + ); + }, + .png => { + try tty.print( +- "\x1b_Gf=100,i={d},t={c};{s}\x1b\\", ++ "\x1b_Gf=100,i={d},t={c},q=2;{s}\x1b\\", + .{ id, medium_char, encoded }, + ); + }, +@@ -1149,7 +1153,7 @@ + + if (bytes.len < 4096) { + try tty.print( +- "\x1b_Gf={d},s={d},v={d},i={d};{s}\x1b\\", ++ "\x1b_Gf={d},s={d},v={d},i={d},q=2;{s}\x1b\\", + .{ + fmt, + width, +@@ -1162,14 +1166,14 @@ + var n: usize = 4096; + + try tty.print( +- "\x1b_Gf={d},s={d},v={d},i={d},m=1;{s}\x1b\\", ++ "\x1b_Gf={d},s={d},v={d},i={d},m=1,q=2;{s}\x1b\\", + .{ fmt, width, height, id, bytes[0..n] }, + ); + while (n < bytes.len) : (n += 4096) { + const end: usize = @min(n + 4096, bytes.len); + const m: u2 = if (end == bytes.len) 0 else 1; + try tty.print( +- "\x1b_Gm={d};{s}\x1b\\", ++ "\x1b_Gm={d},q=2;{s}\x1b\\", + .{ + m, + bytes[n..end], +diff --git a/src/c_api.zig b/src/c_api.zig +--- a/src/c_api.zig ++++ b/src/c_api.zig +@@ -965,7 +965,7 @@ + .cap_multi_cursor => vx.caps.multi_cursor = true, + .cap_program_status => vx.caps.program_status = true, + .cap_da1 => { +- std.Io.futexWake(vx.io, std.atomic.Value(u32), &vx.query_futex, 10); ++ std.Io.futexWake(vx.io, u32, &vx.query_futex.raw, 10); + vx.queries_done.store(true, .unordered); + }, + .winsize => { +diff --git a/src/ctlseqs.zig b/src/ctlseqs.zig +--- a/src/ctlseqs.zig ++++ b/src/ctlseqs.zig +@@ -140,8 +140,8 @@ + pub const osc52_clipboard_request = "\x1b]52;c;?\x1b\\"; + + // Kitty graphics +-pub const kitty_graphics_clear = "\x1b_Ga=d\x1b\\"; +-pub const kitty_graphics_preamble = "\x1b_Ga=p,i={d}"; ++pub const kitty_graphics_clear = "\x1b_Ga=d,q=2\x1b\\"; ++pub const kitty_graphics_preamble = "\x1b_Ga=p,q=2,i={d}"; + pub const kitty_graphics_closing = ",C=1\x1b\\"; + + // Color control sequences +diff --git a/src/vxfw/Padding.zig b/src/vxfw/Padding.zig +--- a/src/vxfw/Padding.zig ++++ b/src/vxfw/Padding.zig +@@ -6,7 +6,7 @@ + const vxfw = @import("vxfw.zig"); + + const Padding = @This(); +-const PadValues = struct { ++pub const PadValues = struct { + left: u16 = 0, + right: u16 = 0, + top: u16 = 0, +@@ -54,10 +54,6 @@ + + pub fn draw(self: *const Padding, ctx: vxfw.DrawContext) Allocator.Error!vxfw.Surface { + const pad = self.padding; +- if (pad.left > 0 or pad.right > 0) +- std.debug.assert(ctx.max.width != null); +- if (pad.top > 0 or pad.bottom > 0) +- std.debug.assert(ctx.max.height != null); + const inner_min: vxfw.Size = .{ + .width = ctx.min.width -| (pad.right + pad.left), + .height = ctx.min.height -| (pad.top + pad.bottom), diff --git a/third_party/modules/libxev/20240910.0-a2d9b31/overlay/BUILD.bazel b/third_party/libvaxis/libvaxis.bazel similarity index 51% rename from third_party/modules/libxev/20240910.0-a2d9b31/overlay/BUILD.bazel rename to third_party/libvaxis/libvaxis.bazel index 872ef7fb..122e25a1 100644 --- a/third_party/modules/libxev/20240910.0-a2d9b31/overlay/BUILD.bazel +++ b/third_party/libvaxis/libvaxis.bazel @@ -1,13 +1,17 @@ load("@rules_zig//zig:defs.bzl", "zig_library") zig_library( - name = "xev", + name = "vaxis", srcs = glob([ "src/*.zig", - "src/backend/*.zig", - "src/linux/*.zig", - "src/watcher/*.zig", + "src/vxfw/*.zig", + "src/widgets/*.zig", + "src/widgets/terminal/*.zig", ]), main = "src/main.zig", + deps = [ + "@zigimg", + "@uucode", + ], visibility = ["//visibility:public"], ) diff --git a/third_party/libvaxis/repo.bzl b/third_party/libvaxis/repo.bzl new file mode 100644 index 00000000..e10982a4 --- /dev/null +++ b/third_party/libvaxis/repo.bzl @@ -0,0 +1,11 @@ +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") + +def repo(): + git_repository( + name = "libvaxis", + remote = "https://github.com/rockorager/libvaxis.git", + commit = "14b7684272234a3d66316c0d1bbdc876d4e8997c", + build_file = "//third_party/libvaxis:libvaxis.bazel", + patches = ["//third_party/libvaxis:fixes.patch"], + patch_args = ["-p1"], + ) diff --git a/third_party/linenoise/BUILD.bazel b/third_party/linenoise/BUILD.bazel new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/third_party/linenoise/BUILD.bazel @@ -0,0 +1 @@ + diff --git a/third_party/linenoise/linenoise.bazel b/third_party/linenoise/linenoise.bazel new file mode 100644 index 00000000..cd4dbbc9 --- /dev/null +++ b/third_party/linenoise/linenoise.bazel @@ -0,0 +1,11 @@ +load("@rules_cc//cc:cc_library.bzl", "cc_library") + +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) # BSD-2-Clause + +cc_library( + name = "linenoise", + srcs = ["linenoise.c"], + hdrs = ["linenoise.h"], +) diff --git a/third_party/linenoise/repo.bzl b/third_party/linenoise/repo.bzl new file mode 100644 index 00000000..1d7a1678 --- /dev/null +++ b/third_party/linenoise/repo.bzl @@ -0,0 +1,9 @@ +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") + +def repo(): + git_repository( + name = "linenoise", + remote = "https://github.com/antirez/linenoise.git", + commit = "dc83cc373ac2058030eb3cf5e404959a26fef112", + build_file = "//third_party/linenoise:linenoise.bazel", + ) diff --git a/third_party/mnist/BUILD.bazel b/third_party/mnist/BUILD.bazel new file mode 100644 index 00000000..e69de29b diff --git a/third_party/mnist/repo.bzl b/third_party/mnist/repo.bzl new file mode 100644 index 00000000..fdabe71a --- /dev/null +++ b/third_party/mnist/repo.bzl @@ -0,0 +1,9 @@ +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +def repo(): + http_archive( + name = "mnist", + sha256 = "9c7d9a5ef9c245084996f6d2ec66ef176e51186e6a5b22efdcc3828d644941ca", + url = "https://mirror.zml.ai/data/mnist_safetensors.tar.zst", + build_file_content = """exports_files(glob(["**"]), visibility = ["//visibility:public"])""", + ) diff --git a/third_party/modules/aspect_bazel_lib/2.8.1.1/MODULE.bazel b/third_party/modules/aspect_bazel_lib/2.8.1.1/MODULE.bazel deleted file mode 100644 index 7f1750b4..00000000 --- a/third_party/modules/aspect_bazel_lib/2.8.1.1/MODULE.bazel +++ /dev/null @@ -1,78 +0,0 @@ -"aspect-build/bazel-lib" - -module( - name = "aspect_bazel_lib", - version = "2.8.1.1", - bazel_compatibility = [">=6.0.0"], - compatibility_level = 1, -) - -# Lower-bounds (minimum) versions for direct runtime dependencies -bazel_dep(name = "bazel_skylib", version = "1.5.0") -bazel_dep(name = "platforms", version = "0.0.8") - -# 0.5.4 is the first version with bzlmod support -bazel_dep(name = "stardoc", version = "0.6.2", repo_name = "io_bazel_stardoc") - -bazel_lib_toolchains = use_extension("@aspect_bazel_lib//lib:extensions.bzl", "toolchains") -bazel_lib_toolchains.copy_directory() -bazel_lib_toolchains.copy_to_directory() -bazel_lib_toolchains.jq() -bazel_lib_toolchains.yq() -bazel_lib_toolchains.coreutils() -bazel_lib_toolchains.tar() -bazel_lib_toolchains.zstd() -bazel_lib_toolchains.expand_template() -bazel_lib_toolchains.bats() -use_repo(bazel_lib_toolchains, "bats_toolchains", "bsd_tar_toolchains", "copy_directory_toolchains", "copy_to_directory_toolchains", "coreutils_toolchains", "expand_template_toolchains", "jq_toolchains", "yq_toolchains", "zstd_toolchains") - -register_toolchains( - "@copy_directory_toolchains//:all", - "@copy_to_directory_toolchains//:all", - "@jq_toolchains//:all", - "@yq_toolchains//:all", - "@coreutils_toolchains//:all", - "@expand_template_toolchains//:all", - "@bats_toolchains//:all", - "@bsd_tar_toolchains//:all", - "@zstd_toolchains//:all", -) - -####### Dev dependencies ######## - -# To allow /tools to be built from source -# NOTE: when publishing to BCR, we patch this to be dev_dependency, as we publish pre-built binaries -# along with our releases. - -bazel_dep( - name = "gazelle", - version = "0.36.0", - dev_dependency = True, -) -bazel_dep( - name = "rules_go", - version = "0.46.0", - repo_name = "io_bazel_rules_go", - dev_dependency = True, -) - -go_deps = use_extension( - "@gazelle//:extensions.bzl", - "go_deps", - dev_dependency = True, -) -go_deps.from_file(go_mod = "//:go.mod") -use_repo( - go_deps, - "com_github_bmatcuk_doublestar_v4", - "org_golang_x_exp", - "org_golang_x_sys", -) - -host = use_extension("@aspect_bazel_lib//lib:extensions.bzl", "host", dev_dependency = True) -host.host() -use_repo(host, "aspect_bazel_lib_host") - -bazel_dep(name = "bazel_skylib_gazelle_plugin", version = "1.5.0", dev_dependency = True) -bazel_dep(name = "buildifier_prebuilt", version = "6.4.0", dev_dependency = True) -bazel_dep(name = "bazel_features", version = "0.2.0", dev_dependency = True) diff --git a/third_party/modules/aspect_bazel_lib/2.8.1.1/patches/0001-fix-add-repo-mapping-to-tar-archive.patch b/third_party/modules/aspect_bazel_lib/2.8.1.1/patches/0001-fix-add-repo-mapping-to-tar-archive.patch deleted file mode 100644 index 96aa8dfc..00000000 --- a/third_party/modules/aspect_bazel_lib/2.8.1.1/patches/0001-fix-add-repo-mapping-to-tar-archive.patch +++ /dev/null @@ -1,50 +0,0 @@ -From 5dbe6a2604e440b684add7b44531898acc8631b4 Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Sun, 8 Sep 2024 21:17:29 +0200 -Subject: [PATCH] fix: add repo mapping to tar archive - -When using bzlmod, runfiles lookup will fail without it. ---- - lib/private/tar.bzl | 17 +++++++++++++---- - 1 file changed, 13 insertions(+), 4 deletions(-) - -diff --git a/lib/private/tar.bzl b/lib/private/tar.bzl -index 733ff60..29434f6 100644 ---- a/lib/private/tar.bzl -+++ b/lib/private/tar.bzl -@@ -147,12 +147,19 @@ def _tar_impl(ctx): - args.add(ctx.file.mtree, format = "@%s") - inputs.append(ctx.file.mtree) - -+ src_runfiles = [] -+ for src in ctx.attr.srcs: -+ src_di = src[DefaultInfo] -+ if getattr(src_di.files_to_run, "repo_mapping_manifest", None) != None: -+ src_runfiles.append(depset( -+ direct = [src_di.files_to_run.repo_mapping_manifest], -+ transitive = [src_di.default_runfiles.files], -+ )) -+ else: -+ src_runfiles.append(src_di.default_runfiles.files) - ctx.actions.run( - executable = bsdtar.tarinfo.binary, -- inputs = depset(direct = inputs, transitive = [bsdtar.default.files] + [ -- src[DefaultInfo].default_runfiles.files -- for src in ctx.attr.srcs -- ]), -+ inputs = depset(direct = inputs, transitive = [bsdtar.default.files] + src_runfiles), - outputs = [out], - arguments = [args], - mnemonic = "Tar", -@@ -234,6 +241,8 @@ def _mtree_impl(ctx): - workspace_name = str(ctx.workspace_name) - - content.add(_mtree_line(runfiles_dir, type = "dir")) -+ if getattr(default_info.files_to_run, "repo_mapping_manifest", None) != None: -+ content.add(_mtree_line("{}/_repo_mapping".format(runfiles_dir), type = "file", content = default_info.files_to_run.repo_mapping_manifest.path)) - content.add_all( - s.default_runfiles.files, - expand_directories = True, --- -2.39.3 (Apple Git-146) - diff --git a/third_party/modules/aspect_bazel_lib/2.8.1.1/patches/go_dev_dep.patch b/third_party/modules/aspect_bazel_lib/2.8.1.1/patches/go_dev_dep.patch deleted file mode 100644 index ffe1c1d3..00000000 --- a/third_party/modules/aspect_bazel_lib/2.8.1.1/patches/go_dev_dep.patch +++ /dev/null @@ -1,27 +0,0 @@ -diff --git a/MODULE.bazel b/MODULE.bazel -index e63fa5b..9d78a88 100644 ---- a/MODULE.bazel -+++ b/MODULE.bazel -@@ -50,19 +50,19 @@ use_repo(host, "aspect_bazel_lib_host") - bazel_dep( - name = "gazelle", - version = "0.36.0", -- # In released versions: dev_dependency = True -+ dev_dependency = True, - ) - bazel_dep( - name = "rules_go", - version = "0.46.0", - repo_name = "io_bazel_rules_go", -- # In released versions: dev_dependency = True -+ dev_dependency = True, - ) - - go_deps = use_extension( - "@gazelle//:extensions.bzl", - "go_deps", -- # In released versions: dev_dependency = True -+ dev_dependency = True, - ) - go_deps.from_file(go_mod = "//:go.mod") - use_repo( diff --git a/third_party/modules/aspect_bazel_lib/2.8.1.1/patches/module_dot_bazel_version.patch b/third_party/modules/aspect_bazel_lib/2.8.1.1/patches/module_dot_bazel_version.patch deleted file mode 100644 index fe11475e..00000000 --- a/third_party/modules/aspect_bazel_lib/2.8.1.1/patches/module_dot_bazel_version.patch +++ /dev/null @@ -1,14 +0,0 @@ -=================================================================== ---- a/MODULE.bazel -+++ b/MODULE.bazel -@@ -1,9 +1,9 @@ - "aspect-build/bazel-lib" - - module( - name = "aspect_bazel_lib", -- version = "0.0.0", -+ version = "2.8.1", - bazel_compatibility = [">=6.0.0"], - compatibility_level = 1, - ) - diff --git a/third_party/modules/aspect_bazel_lib/2.8.1.1/source.json b/third_party/modules/aspect_bazel_lib/2.8.1.1/source.json deleted file mode 100644 index 3b7d500e..00000000 --- a/third_party/modules/aspect_bazel_lib/2.8.1.1/source.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "integrity": "sha256-aINU7mvuunGUJD1z6wmSuaEujt7u7FtlRPS1MaMRIjc=", - "strip_prefix": "bazel-lib-2.8.1", - "url": "https://github.com/aspect-build/bazel-lib/releases/download/v2.8.1/bazel-lib-v2.8.1.tar.gz", - "patches": { - "go_dev_dep.patch": "sha256-DTc/hk+etl4D50M0BLRik2vHbrgDb6rds+Dj4xphWb4=", - "module_dot_bazel_version.patch": "sha256-cEMv6bY7Sc5dERv8YG0lq45zuZCsVPNn4oAN9aOkf40=", - "0001-fix-add-repo-mapping-to-tar-archive.patch": "" - }, - "patch_strip": 1 -} diff --git a/third_party/modules/aspect_bazel_lib/metadata.json b/third_party/modules/aspect_bazel_lib/metadata.json deleted file mode 100644 index db6bde15..00000000 --- a/third_party/modules/aspect_bazel_lib/metadata.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "homepage": "https://docs.aspect.build/rules/aspect_bazel_lib", - "maintainers": [ - { - "name": "Alex Eagle", - "email": "alex@aspect.dev", - "github": "alexeagle" - }, - { - "name": "Derek Cormier", - "email": "derek@aspect.dev", - "github": "kormide" - } - ], - "repository": [ - "github:aspect-build/bazel-lib" - ], - "versions": [ - "2.8.1.1" - ], - "yanked_versions": { - "1.31.0": "1.31.0 has a breaking change to the default yq version", - "1.34.2": "The version field has a leading 'v' due to a release automation bug" - } -} diff --git a/third_party/modules/libxev/20240825.0-dbe2291/MODULE.bazel b/third_party/modules/libxev/20240825.0-dbe2291/MODULE.bazel deleted file mode 100644 index 8f8806c6..00000000 --- a/third_party/modules/libxev/20240825.0-dbe2291/MODULE.bazel +++ /dev/null @@ -1,7 +0,0 @@ -module( - name = "libxev", - version = "20240825.0-dbe2291", - compatibility_level = 1, -) - -bazel_dep(name = "rules_zig", version = "20240904.0-010da15") diff --git a/third_party/modules/libxev/20240825.0-dbe2291/overlay/BUILD.bazel b/third_party/modules/libxev/20240825.0-dbe2291/overlay/BUILD.bazel deleted file mode 100644 index 872ef7fb..00000000 --- a/third_party/modules/libxev/20240825.0-dbe2291/overlay/BUILD.bazel +++ /dev/null @@ -1,13 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") - -zig_library( - name = "xev", - srcs = glob([ - "src/*.zig", - "src/backend/*.zig", - "src/linux/*.zig", - "src/watcher/*.zig", - ]), - main = "src/main.zig", - visibility = ["//visibility:public"], -) diff --git a/third_party/modules/libxev/20240825.0-dbe2291/overlay/MODULE.bazel b/third_party/modules/libxev/20240825.0-dbe2291/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/libxev/20240825.0-dbe2291/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/libxev/20240825.0-dbe2291/source.json b/third_party/modules/libxev/20240825.0-dbe2291/source.json deleted file mode 100644 index 81950288..00000000 --- a/third_party/modules/libxev/20240825.0-dbe2291/source.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "strip_prefix": "libxev-dbe22910a43e9e8ec9948d3cbd73d8488a074967", - "url": "https://github.com/mitchellh/libxev/archive/dbe22910a43e9e8ec9948d3cbd73d8488a074967.tar.gz", - "integrity": "sha256-JM70bkRUyuAQXKV1piJ6I0IClzCrXV43sdLfIKVC9Lo=", - "overlay": { - "MODULE.bazel": "", - "BUILD.bazel": "" - } -} diff --git a/third_party/modules/libxev/20240910.0-a2d9b31/MODULE.bazel b/third_party/modules/libxev/20240910.0-a2d9b31/MODULE.bazel deleted file mode 100644 index 99daae70..00000000 --- a/third_party/modules/libxev/20240910.0-a2d9b31/MODULE.bazel +++ /dev/null @@ -1,7 +0,0 @@ -module( - name = "libxev", - version = "20240910.0-a2d9b31", - compatibility_level = 1, -) - -bazel_dep(name = "rules_zig", version = "20240904.0-010da15") diff --git a/third_party/modules/libxev/20240910.0-a2d9b31/overlay/MODULE.bazel b/third_party/modules/libxev/20240910.0-a2d9b31/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/libxev/20240910.0-a2d9b31/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/libxev/20240910.0-a2d9b31/patches/0001-Change-Backend-Linux.patch b/third_party/modules/libxev/20240910.0-a2d9b31/patches/0001-Change-Backend-Linux.patch deleted file mode 100644 index ed524e67..00000000 --- a/third_party/modules/libxev/20240910.0-a2d9b31/patches/0001-Change-Backend-Linux.patch +++ /dev/null @@ -1,22 +0,0 @@ -From 4db5de34f70d991fedbe28915c8239b97ba7a064 Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Mon, 18 Mar 2024 17:17:34 +0100 -Subject: [PATCH 3/3] [PJRT C API] Ensure C compliance for Profiler Extension - ---- - xla/pjrt/c/pjrt_c_api_profiler_extension.h | 2 ++ - 1 file changed, 2 insertions(+) - -diff --git a/src/main.zig b/src/main.zig -index 74f451a..1cfb61c 100644 ---- a/src/main.zig -+++ b/src/main.zig -@@ -36,7 +36,7 @@ pub const Backend = enum { - /// Returns a recommend default backend from inspecting the system. - pub fn default() Backend { - return @as(?Backend, switch (builtin.os.tag) { -- .linux => .io_uring, -+ .linux => .epoll, - .ios, .macos => .kqueue, - .wasi => .wasi_poll, - .windows => .iocp, diff --git a/third_party/modules/libxev/20240910.0-a2d9b31/source.json b/third_party/modules/libxev/20240910.0-a2d9b31/source.json deleted file mode 100644 index c38d9c27..00000000 --- a/third_party/modules/libxev/20240910.0-a2d9b31/source.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "strip_prefix": "libxev-a2d9b3154f1e5ed463c9f2fb98a0d739057796ce", - "url": "https://github.com/zml/libxev/archive/a2d9b3154f1e5ed463c9f2fb98a0d739057796ce.tar.gz", - "integrity": "sha256-xdMjrGSB3telt52iRqHgem0WSRPjfGlTKBRyDMXLlaQ=", - "overlay": { - "MODULE.bazel": "", - "BUILD.bazel": "" - }, - "patch_strip": 1, - "patches": { - "0001-Change-Backend-Linux.patch": "" - } -} diff --git a/third_party/modules/libxev/metadata.json b/third_party/modules/libxev/metadata.json deleted file mode 100644 index 3b334be8..00000000 --- a/third_party/modules/libxev/metadata.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "homepage": "https://github.com/mitchellh/libxev", - "maintainers": [ - { - "email": "bzlmod@zml.ai", - "github": "zml", - "name": "ZML Engineering Team" - } - ], - "repository": [ - "github:mitchellh/libxev" - ], - "versions": [ - "20240825.0-dbe2291", - "20240910.0-a2d9b31" - ], - "yanked_versions": {} -} diff --git a/third_party/modules/llvm-raw/20240823.0-f142f8a/MODULE.bazel b/third_party/modules/llvm-raw/20240823.0-f142f8a/MODULE.bazel deleted file mode 100644 index 6bf421a8..00000000 --- a/third_party/modules/llvm-raw/20240823.0-f142f8a/MODULE.bazel +++ /dev/null @@ -1,10 +0,0 @@ -module( - name = "llvm-raw", - version = "20240823.0-f142f8a", - compatibility_level = 1, -) - -bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "platforms", version = "0.0.10") -bazel_dep(name = "zstd", version = "1.5.6", repo_name = "llvm_zstd") -bazel_dep(name = "zlib", version = "1.3.1.bcr.3", repo_name = "llvm_zlib") diff --git a/third_party/modules/llvm-raw/20240823.0-f142f8a/overlay/MODULE.bazel b/third_party/modules/llvm-raw/20240823.0-f142f8a/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/llvm-raw/20240823.0-f142f8a/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/llvm-raw/20240823.0-f142f8a/source.json b/third_party/modules/llvm-raw/20240823.0-f142f8a/source.json deleted file mode 100644 index 6edc73dc..00000000 --- a/third_party/modules/llvm-raw/20240823.0-f142f8a/source.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "strip_prefix": "llvm-project-f142f8afe21bceb00fb495468aa0b5043e98c419", - "url": "https://github.com/llvm/llvm-project/archive/f142f8afe21bceb00fb495468aa0b5043e98c419.tar.gz", - "integrity": "sha256-8ftYOnMGq99igt2I1n9SOZpYLq79+F7HiKL/7lOwcHs=", - "overlay": { - "BUILD.bazel": "", - "MODULE.bazel": "", - "utils/bazel/extension.bzl": "" - } -} diff --git a/third_party/modules/llvm-raw/20240919.0-94c024a/MODULE.bazel b/third_party/modules/llvm-raw/20240919.0-94c024a/MODULE.bazel deleted file mode 100644 index c9130b80..00000000 --- a/third_party/modules/llvm-raw/20240919.0-94c024a/MODULE.bazel +++ /dev/null @@ -1,10 +0,0 @@ -module( - name = "llvm-raw", - version = "20240919.0-94c024a", - compatibility_level = 1, -) - -bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "platforms", version = "0.0.10") -bazel_dep(name = "zstd", version = "1.5.6", repo_name = "llvm_zstd") -bazel_dep(name = "zlib", version = "1.3.1.bcr.3", repo_name = "llvm_zlib") diff --git a/third_party/modules/llvm-raw/20240919.0-94c024a/overlay/MODULE.bazel b/third_party/modules/llvm-raw/20240919.0-94c024a/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/llvm-raw/20240919.0-94c024a/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/llvm-raw/20240919.0-94c024a/overlay/utils/bazel/extension.bzl b/third_party/modules/llvm-raw/20240919.0-94c024a/overlay/utils/bazel/extension.bzl deleted file mode 100644 index f247ed4b..00000000 --- a/third_party/modules/llvm-raw/20240919.0-94c024a/overlay/utils/bazel/extension.bzl +++ /dev/null @@ -1,28 +0,0 @@ -load("//utils/bazel:configure.bzl", _llvm_configure = "llvm_configure") - -def _llvm_impl(mctx): - _targets = {} - for mod in mctx.modules: - for conf in mod.tags.configure: - for target in conf.targets: - _targets[target] = True - _llvm_configure( - name = "llvm-project", - targets = _targets.keys(), - ) - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -llvm = module_extension( - implementation = _llvm_impl, - tag_classes = { - "configure": tag_class( - attrs = { - "targets": attr.string_list(mandatory = True), - }, - ), - }, -) diff --git a/third_party/modules/llvm-raw/20240919.0-94c024a/source.json b/third_party/modules/llvm-raw/20240919.0-94c024a/source.json deleted file mode 100644 index 2aec201a..00000000 --- a/third_party/modules/llvm-raw/20240919.0-94c024a/source.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "strip_prefix": "llvm-project-94c024adedcb53059c29d7c2d62982053b60e86a", - "url": "https://github.com/llvm/llvm-project/archive/94c024adedcb53059c29d7c2d62982053b60e86a.tar.gz", - "integrity": "sha256-IEzt6quG8GXvZMs4id0ukt3UqPXVtrwctLJ2aU+2p5g=", - "overlay": { - "BUILD.bazel": "", - "MODULE.bazel": "", - "utils/bazel/extension.bzl": "" - } -} diff --git a/third_party/modules/llvm-raw/metadata.json b/third_party/modules/llvm-raw/metadata.json deleted file mode 100644 index 077104b7..00000000 --- a/third_party/modules/llvm-raw/metadata.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "homepage": "https://github.com/llvm/llvm-project", - "maintainers": [ - { - "email": "bzlmod@zml.ai", - "github": "zml", - "name": "ZML Engineering Team" - } - ], - "repository": [ - "github:llvm/llvm-project" - ], - "versions": [ - "20240823.0-f142f8a", - "20240919.0-94c024a", - ], - "yanked_versions": {} -} diff --git a/third_party/modules/rules_zig/20240904.0-010da15/MODULE.bazel b/third_party/modules/rules_zig/20240904.0-010da15/MODULE.bazel deleted file mode 100644 index 723c6a98..00000000 --- a/third_party/modules/rules_zig/20240904.0-010da15/MODULE.bazel +++ /dev/null @@ -1,68 +0,0 @@ -module( - name = "rules_zig", - version = "20240904.0-010da15", - compatibility_level = 1, -) - -bazel_dep(name = "aspect_bazel_lib", version = "2.8.1") -bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "platforms", version = "0.0.10") - -zig = use_extension("//zig:extensions.bzl", "zig") -zig.index(file = "//zig/private:versions.json") -use_repo(zig, "zig_toolchains") - -register_toolchains("@rules_zig//zig/target:all") - -register_toolchains("@zig_toolchains//:all") - -zig_dev = use_extension( - "//zig:extensions.bzl", - "zig", - dev_dependency = True, -) -zig_dev.toolchain(zig_version = "0.13.0") -zig_dev.toolchain(zig_version = "0.12.1") -zig_dev.toolchain(zig_version = "0.12.0") -zig_dev.toolchain(zig_version = "0.11.0") - -bazel_dep(name = "rules_cc", version = "0.0.9", dev_dependency = True) -bazel_dep(name = "stardoc", version = "0.7.0", dev_dependency = True, repo_name = "io_bazel_stardoc") -bazel_dep(name = "gazelle", version = "0.38.0", dev_dependency = True, repo_name = "bazel_gazelle") -bazel_dep(name = "bazel_skylib_gazelle_plugin", version = "1.7.1", dev_dependency = True) -bazel_dep( - name = "buildifier_prebuilt", - version = "7.3.1", - dev_dependency = True, -) -bazel_dep(name = "rules_multirun", version = "0.9.0", dev_dependency = True) -bazel_dep(name = "rules_python", version = "0.35.0", dev_dependency = True) -bazel_dep( - name = "rules_bazel_integration_test", - version = "0.25.0", - dev_dependency = True, -) - -bazel_binaries = use_extension( - "@rules_bazel_integration_test//:extensions.bzl", - "bazel_binaries", - dev_dependency = True, -) - -# NOTE: Keep in sync with WORKSPACE. -bazel_binaries.download(version_file = "//:.bazelversion") -bazel_binaries.download(version = "7.0.0") -use_repo( - bazel_binaries, - "bazel_binaries", - "bazel_binaries_bazelisk", - "build_bazel_bazel_.bazelversion", - "build_bazel_bazel_7_0_0", -) - -# TODO[AH] Should be an implicit transitive dependency through rules_bazel_integration_test. -# However, if we do not include it explicitly, then the runfiles resolution for -# cgrindel_bazel_starlib/shlib/lib/message.sh fails in -# rules_bazel_integration_test/tools/update_deleted_packages.sh when invoked -# through the rules_multirun target //util:update. -bazel_dep(name = "cgrindel_bazel_starlib", version = "0.21.0", dev_dependency = True) diff --git a/third_party/modules/rules_zig/20240904.0-010da15/source.json b/third_party/modules/rules_zig/20240904.0-010da15/source.json deleted file mode 100644 index 253af196..00000000 --- a/third_party/modules/rules_zig/20240904.0-010da15/source.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "type": "git_repository", - "remote": "https://github.com/zml/rules_zig.git", - "commit": "010da15abb4335479778d6b4fb2ca752a0ab80e3" -} diff --git a/third_party/modules/rules_zig/20240909.0-37f17ff/MODULE.bazel b/third_party/modules/rules_zig/20240909.0-37f17ff/MODULE.bazel deleted file mode 100644 index 4ded2bb6..00000000 --- a/third_party/modules/rules_zig/20240909.0-37f17ff/MODULE.bazel +++ /dev/null @@ -1,68 +0,0 @@ -module( - name = "rules_zig", - version = "20240909.0-37f17ff", - compatibility_level = 1, -) - -bazel_dep(name = "aspect_bazel_lib", version = "2.8.1") -bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "platforms", version = "0.0.10") - -zig = use_extension("//zig:extensions.bzl", "zig") -zig.index(file = "//zig/private:versions.json") -use_repo(zig, "zig_toolchains") - -register_toolchains("@rules_zig//zig/target:all") - -register_toolchains("@zig_toolchains//:all") - -zig_dev = use_extension( - "//zig:extensions.bzl", - "zig", - dev_dependency = True, -) -zig_dev.toolchain(zig_version = "0.13.0") -zig_dev.toolchain(zig_version = "0.12.1") -zig_dev.toolchain(zig_version = "0.12.0") -zig_dev.toolchain(zig_version = "0.11.0") - -bazel_dep(name = "rules_cc", version = "0.0.9", dev_dependency = True) -bazel_dep(name = "stardoc", version = "0.7.0", dev_dependency = True, repo_name = "io_bazel_stardoc") -bazel_dep(name = "gazelle", version = "0.38.0", dev_dependency = True, repo_name = "bazel_gazelle") -bazel_dep(name = "bazel_skylib_gazelle_plugin", version = "1.7.1", dev_dependency = True) -bazel_dep( - name = "buildifier_prebuilt", - version = "7.3.1", - dev_dependency = True, -) -bazel_dep(name = "rules_multirun", version = "0.9.0", dev_dependency = True) -bazel_dep(name = "rules_python", version = "0.35.0", dev_dependency = True) -bazel_dep( - name = "rules_bazel_integration_test", - version = "0.25.0", - dev_dependency = True, -) - -bazel_binaries = use_extension( - "@rules_bazel_integration_test//:extensions.bzl", - "bazel_binaries", - dev_dependency = True, -) - -# NOTE: Keep in sync with WORKSPACE. -bazel_binaries.download(version_file = "//:.bazelversion") -bazel_binaries.download(version = "7.0.0") -use_repo( - bazel_binaries, - "bazel_binaries", - "bazel_binaries_bazelisk", - "build_bazel_bazel_.bazelversion", - "build_bazel_bazel_7_0_0", -) - -# TODO[AH] Should be an implicit transitive dependency through rules_bazel_integration_test. -# However, if we do not include it explicitly, then the runfiles resolution for -# cgrindel_bazel_starlib/shlib/lib/message.sh fails in -# rules_bazel_integration_test/tools/update_deleted_packages.sh when invoked -# through the rules_multirun target //util:update. -bazel_dep(name = "cgrindel_bazel_starlib", version = "0.21.0", dev_dependency = True) diff --git a/third_party/modules/rules_zig/20240909.0-37f17ff/source.json b/third_party/modules/rules_zig/20240909.0-37f17ff/source.json deleted file mode 100644 index 19d05aef..00000000 --- a/third_party/modules/rules_zig/20240909.0-37f17ff/source.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "type": "git_repository", - "remote": "https://github.com/zml/rules_zig.git", - "commit": "37f17ffd2f6d04fc4bf9b583f9a1b117856a88d1" -} diff --git a/third_party/modules/rules_zig/20240912.0-41bfe84/MODULE.bazel b/third_party/modules/rules_zig/20240912.0-41bfe84/MODULE.bazel deleted file mode 100644 index e288be56..00000000 --- a/third_party/modules/rules_zig/20240912.0-41bfe84/MODULE.bazel +++ /dev/null @@ -1,68 +0,0 @@ -module( - name = "rules_zig", - version = "20240912.0-41bfe84", - compatibility_level = 1, -) - -bazel_dep(name = "aspect_bazel_lib", version = "2.8.1") -bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "platforms", version = "0.0.10") - -zig = use_extension("//zig:extensions.bzl", "zig") -zig.index(file = "//zig/private:versions.json") -use_repo(zig, "zig_toolchains") - -register_toolchains("@rules_zig//zig/target:all") - -register_toolchains("@zig_toolchains//:all") - -zig_dev = use_extension( - "//zig:extensions.bzl", - "zig", - dev_dependency = True, -) -zig_dev.toolchain(zig_version = "0.13.0") -zig_dev.toolchain(zig_version = "0.12.1") -zig_dev.toolchain(zig_version = "0.12.0") -zig_dev.toolchain(zig_version = "0.11.0") - -bazel_dep(name = "rules_cc", version = "0.0.9") -bazel_dep(name = "stardoc", version = "0.7.0", dev_dependency = True, repo_name = "io_bazel_stardoc") -bazel_dep(name = "gazelle", version = "0.38.0", dev_dependency = True, repo_name = "bazel_gazelle") -bazel_dep(name = "bazel_skylib_gazelle_plugin", version = "1.7.1", dev_dependency = True) -bazel_dep( - name = "buildifier_prebuilt", - version = "7.3.1", - dev_dependency = True, -) -bazel_dep(name = "rules_multirun", version = "0.9.0", dev_dependency = True) -bazel_dep(name = "rules_python", version = "0.35.0", dev_dependency = True) -bazel_dep( - name = "rules_bazel_integration_test", - version = "0.25.0", - dev_dependency = True, -) - -bazel_binaries = use_extension( - "@rules_bazel_integration_test//:extensions.bzl", - "bazel_binaries", - dev_dependency = True, -) - -# NOTE: Keep in sync with WORKSPACE. -bazel_binaries.download(version_file = "//:.bazelversion") -bazel_binaries.download(version = "7.0.0") -use_repo( - bazel_binaries, - "bazel_binaries", - "bazel_binaries_bazelisk", - "build_bazel_bazel_.bazelversion", - "build_bazel_bazel_7_0_0", -) - -# TODO[AH] Should be an implicit transitive dependency through rules_bazel_integration_test. -# However, if we do not include it explicitly, then the runfiles resolution for -# cgrindel_bazel_starlib/shlib/lib/message.sh fails in -# rules_bazel_integration_test/tools/update_deleted_packages.sh when invoked -# through the rules_multirun target //util:update. -bazel_dep(name = "cgrindel_bazel_starlib", version = "0.21.0", dev_dependency = True) diff --git a/third_party/modules/rules_zig/20240912.0-41bfe84/source.json b/third_party/modules/rules_zig/20240912.0-41bfe84/source.json deleted file mode 100644 index 3d576fb1..00000000 --- a/third_party/modules/rules_zig/20240912.0-41bfe84/source.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "type": "git_repository", - "remote": "https://github.com/zml/rules_zig.git", - "commit": "41bfe84e4d9a43cbe55281dddc80b683cc6fc6eb" -} diff --git a/third_party/modules/rules_zig/20240913.0-1957d05/MODULE.bazel b/third_party/modules/rules_zig/20240913.0-1957d05/MODULE.bazel deleted file mode 100644 index f2ff768e..00000000 --- a/third_party/modules/rules_zig/20240913.0-1957d05/MODULE.bazel +++ /dev/null @@ -1,68 +0,0 @@ -module( - name = "rules_zig", - version = "20240913.0-1957d05", - compatibility_level = 1, -) - -bazel_dep(name = "aspect_bazel_lib", version = "2.8.1") -bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "platforms", version = "0.0.10") - -zig = use_extension("//zig:extensions.bzl", "zig") -zig.index(file = "//zig/private:versions.json") -use_repo(zig, "zig_toolchains") - -register_toolchains("@rules_zig//zig/target:all") - -register_toolchains("@zig_toolchains//:all") - -zig_dev = use_extension( - "//zig:extensions.bzl", - "zig", - dev_dependency = True, -) -zig_dev.toolchain(zig_version = "0.13.0") -zig_dev.toolchain(zig_version = "0.12.1") -zig_dev.toolchain(zig_version = "0.12.0") -zig_dev.toolchain(zig_version = "0.11.0") - -bazel_dep(name = "rules_cc", version = "0.0.9") -bazel_dep(name = "stardoc", version = "0.7.0", dev_dependency = True, repo_name = "io_bazel_stardoc") -bazel_dep(name = "gazelle", version = "0.38.0", dev_dependency = True, repo_name = "bazel_gazelle") -bazel_dep(name = "bazel_skylib_gazelle_plugin", version = "1.7.1", dev_dependency = True) -bazel_dep( - name = "buildifier_prebuilt", - version = "7.3.1", - dev_dependency = True, -) -bazel_dep(name = "rules_multirun", version = "0.9.0", dev_dependency = True) -bazel_dep(name = "rules_python", version = "0.35.0", dev_dependency = True) -bazel_dep( - name = "rules_bazel_integration_test", - version = "0.25.0", - dev_dependency = True, -) - -bazel_binaries = use_extension( - "@rules_bazel_integration_test//:extensions.bzl", - "bazel_binaries", - dev_dependency = True, -) - -# NOTE: Keep in sync with WORKSPACE. -bazel_binaries.download(version_file = "//:.bazelversion") -bazel_binaries.download(version = "7.0.0") -use_repo( - bazel_binaries, - "bazel_binaries", - "bazel_binaries_bazelisk", - "build_bazel_bazel_.bazelversion", - "build_bazel_bazel_7_0_0", -) - -# TODO[AH] Should be an implicit transitive dependency through rules_bazel_integration_test. -# However, if we do not include it explicitly, then the runfiles resolution for -# cgrindel_bazel_starlib/shlib/lib/message.sh fails in -# rules_bazel_integration_test/tools/update_deleted_packages.sh when invoked -# through the rules_multirun target //util:update. -bazel_dep(name = "cgrindel_bazel_starlib", version = "0.21.0", dev_dependency = True) diff --git a/third_party/modules/rules_zig/20240913.0-1957d05/source.json b/third_party/modules/rules_zig/20240913.0-1957d05/source.json deleted file mode 100644 index 9da3ae38..00000000 --- a/third_party/modules/rules_zig/20240913.0-1957d05/source.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "type": "git_repository", - "remote": "https://github.com/zml/rules_zig.git", - "commit": "1957d0572193fb859e721a0fab8bd8f0fb57f3ff" -} diff --git a/third_party/modules/rules_zig/metadata.json b/third_party/modules/rules_zig/metadata.json deleted file mode 100644 index c6a894f2..00000000 --- a/third_party/modules/rules_zig/metadata.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "homepage": "https://github.com/zml/rules_zig", - "maintainers": [ - { - "email": "bzlmod@zml.ai", - "github": "zml", - "name": "ZML Engineering Team" - } - ], - "repository": [ - "github:zml/rules_zig" - ], - "versions": [ - "20240904.0-010da15", - "20240909.0-37f17ff", - "20240912.0-41bfe84", - "20240913.0-1957d05" - ], - "yanked_versions": {} -} diff --git a/third_party/modules/sentencepiece/20240618.0-d7ace0a/MODULE.bazel b/third_party/modules/sentencepiece/20240618.0-d7ace0a/MODULE.bazel deleted file mode 100644 index af94d93f..00000000 --- a/third_party/modules/sentencepiece/20240618.0-d7ace0a/MODULE.bazel +++ /dev/null @@ -1,7 +0,0 @@ -module( - name = "sentencepiece", - version = "20240618.0-d7ace0a", - compatibility_level = 1, -) - -bazel_dep(name = "rules_proto", version = "6.0.0-rc1") diff --git a/third_party/modules/sentencepiece/20240618.0-d7ace0a/overlay/BUILD.bazel b/third_party/modules/sentencepiece/20240618.0-d7ace0a/overlay/BUILD.bazel deleted file mode 100644 index 43639e89..00000000 --- a/third_party/modules/sentencepiece/20240618.0-d7ace0a/overlay/BUILD.bazel +++ /dev/null @@ -1,7 +0,0 @@ -load("@rules_proto//proto:defs.bzl", "proto_library") - -proto_library( - name = "sentencepiece_model_proto", - srcs = ["src/sentencepiece_model.proto"], - visibility = ["//visibility:public"], -) diff --git a/third_party/modules/sentencepiece/20240618.0-d7ace0a/overlay/MODULE.bazel b/third_party/modules/sentencepiece/20240618.0-d7ace0a/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/sentencepiece/20240618.0-d7ace0a/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/sentencepiece/20240618.0-d7ace0a/source.json b/third_party/modules/sentencepiece/20240618.0-d7ace0a/source.json deleted file mode 100644 index 588c9bcb..00000000 --- a/third_party/modules/sentencepiece/20240618.0-d7ace0a/source.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "strip_prefix": "sentencepiece-d7ace0a4df17af17d73dc8fe6ff92cfd8d08e2b8", - "url": "https://github.com/google/sentencepiece/archive/d7ace0a4df17af17d73dc8fe6ff92cfd8d08e2b8.tar.gz", - "integrity": "sha256-bt8QFNNIpqcJoCNMUrmWfUf0uNobtk8pW+XbHL+ZRLg=", - "overlay": { - "MODULE.bazel": "", - "BUILD.bazel": "" - } -} diff --git a/third_party/modules/sentencepiece/metadata.json b/third_party/modules/sentencepiece/metadata.json deleted file mode 100644 index 2c9758c9..00000000 --- a/third_party/modules/sentencepiece/metadata.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "homepage": "https://github.com/google/sentencepiece", - "maintainers": [ - { - "email": "bzlmod@zml.ai", - "github": "zml", - "name": "ZML Engineering Team" - } - ], - "repository": [ - "google/sentencepiece" - ], - "versions": [ - "20240618.0-d7ace0a" - ], - "yanked_versions": {} -} diff --git a/third_party/modules/stablehlo/20240829.0-54aa1a5/MODULE.bazel b/third_party/modules/stablehlo/20240829.0-54aa1a5/MODULE.bazel deleted file mode 100644 index 29e45e04..00000000 --- a/third_party/modules/stablehlo/20240829.0-54aa1a5/MODULE.bazel +++ /dev/null @@ -1,15 +0,0 @@ -module( - name = "stablehlo", - version = "20240829.0-54aa1a5", - compatibility_level = 1, -) - -bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "rules_cc", version = "0.0.9") -bazel_dep(name = "llvm-raw", version = "20240823.0-94c024a") - -llvm = use_extension("@llvm-raw//utils/bazel:extension.bzl", "llvm") -llvm.configure( - targets = ["AArch64", "X86", "NVPTX"], -) -use_repo(llvm, "llvm-project") diff --git a/third_party/modules/stablehlo/20240829.0-54aa1a5/overlay/MODULE.bazel b/third_party/modules/stablehlo/20240829.0-54aa1a5/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/stablehlo/20240829.0-54aa1a5/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/stablehlo/20240829.0-54aa1a5/patches/0001-Remove-duplicated-symbols-in-StablehloApi.h.patch b/third_party/modules/stablehlo/20240829.0-54aa1a5/patches/0001-Remove-duplicated-symbols-in-StablehloApi.h.patch deleted file mode 100644 index 82ca0c4d..00000000 --- a/third_party/modules/stablehlo/20240829.0-54aa1a5/patches/0001-Remove-duplicated-symbols-in-StablehloApi.h.patch +++ /dev/null @@ -1,83 +0,0 @@ -From f11d9bd7639f63cba681caa39cea27af114a4a71 Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Mon, 2 Sep 2024 23:02:28 +0200 -Subject: [PATCH 1/2] Remove duplicated symbols in StablehloApi.h - -This causes C compilers to choke. - -Refs #2494 ---- - stablehlo/integrations/c/StablehloApi.cpp | 2 +- - stablehlo/integrations/c/StablehloApi.h | 15 +-------------- - stablehlo/integrations/python/StablehloApi.cpp | 2 +- - 3 files changed, 3 insertions(+), 16 deletions(-) - -diff --git a/stablehlo/integrations/c/StablehloApi.cpp b/stablehlo/integrations/c/StablehloApi.cpp -index 8d922198..9b41ff21 100644 ---- a/stablehlo/integrations/c/StablehloApi.cpp -+++ b/stablehlo/integrations/c/StablehloApi.cpp -@@ -98,7 +98,7 @@ MlirLogicalResult stablehloSerializePortableArtifact( - return mlirLogicalResultSuccess(); - } - --MlirLogicalResult stablehloDeserializePortableArtifact( -+MlirLogicalResult stablehloDeserializePortableArtifactToBytecode( - MlirStringRef artifactStr, MlirStringCallback callback, void *userData) { - mlir::detail::CallbackOstream stream(callback, userData); - if (failed(mlir::stablehlo::deserializePortableArtifact(unwrap(artifactStr), -diff --git a/stablehlo/integrations/c/StablehloApi.h b/stablehlo/integrations/c/StablehloApi.h -index 4c542508..f94dca8d 100644 ---- a/stablehlo/integrations/c/StablehloApi.h -+++ b/stablehlo/integrations/c/StablehloApi.h -@@ -76,16 +76,6 @@ MLIR_CAPI_EXPORTED MlirLogicalResult stablehloSerializePortableArtifact( - MlirStringRef moduleStr, MlirStringRef targetVersion, - MlirStringCallback callback, void* userData); - --// Write a StableHLO program expressed as a string (either prettyprinted MLIR --// module or MLIR bytecode) to a portable artifact. --// Can fail if `moduleStr` cannot be parsed, or if it cannot be expressed in the --// `targetVersion` version of StableHLO, e.g. if it's using new or removed --// features, or if it involves unsupported dialects. --// Returns false on failure. --MLIR_CAPI_EXPORTED MlirLogicalResult stablehloSerializePortableArtifact( -- MlirModule moduleStr, MlirStringRef targetVersion, -- MlirStringCallback callback, void* userData); -- - // Read a StableHLO program from a portable artifact, returning the module as - // MLIR bytecode. Note, this bytecode returned is not a portable artifact, - // and has the stability of returning textual assembly format. Bytecode is -@@ -93,7 +83,7 @@ MLIR_CAPI_EXPORTED MlirLogicalResult stablehloSerializePortableArtifact( - // Can fail if `artifactStr` cannot be expressed in the current version of - // StableHLO, e.g. if it's using incompatible features. - // Returns false on failure. --MLIR_CAPI_EXPORTED MlirLogicalResult stablehloDeserializePortableArtifact( -+MLIR_CAPI_EXPORTED MlirLogicalResult stablehloDeserializePortableArtifactAsBytecode( - MlirStringRef artifactStr, MlirStringCallback callback, void* userData); - - // Read a StableHLO program from a portable artifact, returning the module as -@@ -109,9 +99,6 @@ MLIR_CAPI_EXPORTED MlirModule stablehloDeserializePortableArtifact( - - // Call the Interpreter, returns MlirArrayAttr of dense element - // MlirAttribute results --MLIR_CAPI_EXPORTED MlirModule stablehloDeserializePortableArtifact( -- MlirStringRef artifactStr, MlirContext ctx); -- - // Entrypoint for calling the StableHLO reference interpreter. - // Returns an array attribute of dense element attributes for results. - // Sets error code to non-zero on failure. -diff --git a/stablehlo/integrations/python/StablehloApi.cpp b/stablehlo/integrations/python/StablehloApi.cpp -index 46a640e1..4229ef76 100644 ---- a/stablehlo/integrations/python/StablehloApi.cpp -+++ b/stablehlo/integrations/python/StablehloApi.cpp -@@ -213,7 +213,7 @@ void AddPortableApi(py::module &m) { - "deserialize_portable_artifact_str", - [](std::string_view artifact) -> py::bytes { - StringWriterHelper accumulator; -- if (mlirLogicalResultIsFailure(stablehloDeserializePortableArtifact( -+ if (mlirLogicalResultIsFailure(stablehloDeserializePortableArtifactToBytecode( - toMlirStringRef(artifact), accumulator.getMlirStringCallback(), - accumulator.getUserData()))) { - PyErr_SetString(PyExc_ValueError, "failed to deserialize module"); --- -2.39.3 (Apple Git-146) - diff --git a/third_party/modules/stablehlo/20240829.0-54aa1a5/patches/temporary.patch b/third_party/modules/stablehlo/20240829.0-54aa1a5/patches/temporary.patch deleted file mode 100644 index 5240a14c..00000000 --- a/third_party/modules/stablehlo/20240829.0-54aa1a5/patches/temporary.patch +++ /dev/null @@ -1,401 +0,0 @@ -diff --ruN a/stablehlo/BUILD.bazel b/stablehlo/BUILD.bazel ---- stablehlo/BUILD.bazel -+++ stablehlo/BUILD.bazel -@@ -340,6 +340,21 @@ - ], - ) - -+gentbl_cc_library( -+ name = "stablehlo_create_compatibility_expander_inc_gen", -+ tbl_outs = [ -+ ( -+ ["--gen-rewriters"], -+ "stablehlo/transforms/StablehloCreateCompatibilityExpanderPatterns.h.inc", -+ ), -+ ], -+ tblgen = "@llvm-project//mlir:mlir-tblgen", -+ td_file = "stablehlo/transforms/StablehloCreateCompatibilityExpanderPatterns.td", -+ deps = [ -+ ":stablehlo_ops_td_files", -+ ], -+) -+ - cc_library( - name = "interpreter_ops", - srcs = [ -@@ -1086,6 +1101,7 @@ - "stablehlo/transforms/StablehloAggressiveSimplification.cpp", - "stablehlo/transforms/StablehloCanonicalizeDynamism.cpp", - "stablehlo/transforms/StablehloConvertToSignless.cpp", -+ "stablehlo/transforms/StablehloCreateCompatibilityExpander.cpp", - "stablehlo/transforms/StablehloLegalizeCompositeToCall.cpp", - "stablehlo/transforms/StablehloLegalizeDeprecatedOps.cpp", - "stablehlo/transforms/StablehloLegalizeQDQToQuantizedOp.cpp", -@@ -1109,6 +1125,7 @@ - ":chlo_ops", - ":chlo_rewriters_inc_gen", - ":linalg_passes", -+ ":stablehlo_create_compatibility_expander_inc_gen", - ":stablehlo_legalize_deprecated_ops_inc_gen", - ":stablehlo_ops", - ":stablehlo_ops_inc_gen", -diff --ruN a/stablehlo/stablehlo/dialect/Version.cpp b/stablehlo/stablehlo/dialect/Version.cpp ---- stablehlo/stablehlo/dialect/Version.cpp -+++ stablehlo/stablehlo/dialect/Version.cpp -@@ -82,7 +82,7 @@ - case CompatibilityRequirement::WEEK_4: - return Version(1, 3, 0); // v1.3.0 - Jul 15, 2024 - case CompatibilityRequirement::WEEK_12: -- return Version(1, 0, 0); // v1.0.0 - May 14, 2024 -+ return Version(1, 1, 0); // v1.1.0 - May 30, 2024 - case CompatibilityRequirement::MAX: - return Version::getMinimumVersion(); - } -diff --ruN a/stablehlo/stablehlo/tests/transforms/stablehlo_create_compatibility_expander.mlir b/stablehlo/stablehlo/tests/transforms/stablehlo_create_compatibility_expander.mlir ---- stablehlo/stablehlo/tests/transforms/stablehlo_create_compatibility_expander.mlir -+++ stablehlo/stablehlo/tests/transforms/stablehlo_create_compatibility_expander.mlir -@@ -0,0 +1,43 @@ -+// RUN: stablehlo-opt %s -verify-diagnostics -split-input-file -allow-unregistered-dialect --stablehlo-create-compatibility-expander='target=1.0.0' | FileCheck %s --check-prefixes=CHECK -+// RUN: stablehlo-opt %s -verify-diagnostics -split-input-file --stablehlo-create-compatibility-expander='target=1.6.0' | FileCheck %s --check-prefixes=CHECK-NO-DOWNGRADE -+ -+// ----- -+ -+// CHECK-LABEL @tan_op_non_complex -+// CHECK: %[[sine0:.*]] = stablehlo.sine %arg0 : tensor<4xf64> -+// CHECK-NEXT: %[[cosine1:.*]] = stablehlo.cosine %arg0 : tensor<4xf64> -+// CHECK-NEXT: %[[div2:.*]] = stablehlo.divide %[[sine0]], %[[cosine1]] : tensor<4xf64> -+// CHECK-NEXT: return %[[div2]] : tensor<4xf64> -+func.func @tan_op_non_complex(%arg0: tensor<4xf64>) -> tensor<4xf64> { -+ // CHECK-NO-DOWNGRADE: stablehlo.tan %arg0 : tensor<4xf64> -+ %1 = stablehlo.tan %arg0 : tensor<4xf64> -+ func.return %1 : tensor<4xf64> -+} -+ -+// ----- -+ -+// CHECK-LABEL: @tan_op_complex -+// CHECK: %[[cst:.*]] = stablehlo.constant dense<1.000000e+00> : tensor<4xf64> -+// CHECK: %[[complex:.*]] = stablehlo.complex %arg0, %arg1 : tensor<4xcomplex> -+// CHECK: %[[real:.*]] = stablehlo.real %[[complex]] : (tensor<4xcomplex>) -> tensor<4xf64> -+// CHECK: %[[sine:.*]] = stablehlo.sine %[[real]] : tensor<4xf64> -+// CHECK: %[[cosine:.*]] = stablehlo.cosine %[[real]] : tensor<4xf64> -+// CHECK: %[[divide1:.*]] = stablehlo.divide %[[sine]], %[[cosine]] : tensor<4xf64> -+// CHECK: %[[imag:.*]] = stablehlo.imag %[[complex]] : (tensor<4xcomplex>) -> tensor<4xf64> -+// CHECK: %[[tanh:.*]] = stablehlo.tanh %[[imag]] : tensor<4xf64> -+// CHECK: %[[complex2:.*]] = stablehlo.complex %[[divide1]], %[[tanh]] : tensor<4xcomplex> -+// CHECK: %[[multiply:.*]] = stablehlo.multiply %[[divide1]], %[[tanh]] : tensor<4xf64> -+// CHECK: %[[negate:.*]] = stablehlo.negate %[[multiply]] : tensor<4xf64> -+// CHECK: %[[complex3:.*]] = stablehlo.complex %[[cst]], %[[negate]] : tensor<4xcomplex> -+// CHECK: %[[divide2:.*]] = stablehlo.divide %[[complex2]], %[[complex3]] : tensor<4xcomplex> -+// CHECK: %[[real2:.*]] = stablehlo.real %[[divide2]] : (tensor<4xcomplex>) -> tensor<4xf64> -+// CHECK: %[[imag2:.*]] = stablehlo.imag %[[divide2]] : (tensor<4xcomplex>) -> tensor<4xf64> -+// CHECK: return %[[real2]], %[[imag2]] : tensor<4xf64>, tensor<4xf64> -+func.func @tan_op_complex(%arg0: tensor<4xf64>, %arg1: tensor<4xf64>) -> (tensor<4xf64>, tensor<4xf64>) { -+ %0 = stablehlo.complex %arg0, %arg1 : tensor<4xcomplex> -+ // CHECK-NO-DOWNGRADE: stablehlo.tan %0 : tensor<4xcomplex> -+ %1 = stablehlo.tan %0 : tensor<4xcomplex> -+ %2 = stablehlo.real %1 : (tensor<4xcomplex>) -> tensor<4xf64> -+ %3 = stablehlo.imag %1 : (tensor<4xcomplex>) -> tensor<4xf64> -+ func.return %2, %3 : tensor<4xf64>, tensor<4xf64> -+} -diff --ruN a/stablehlo/stablehlo/transforms/CMakeLists.txt b/stablehlo/stablehlo/transforms/CMakeLists.txt ---- stablehlo/stablehlo/transforms/CMakeLists.txt -+++ stablehlo/stablehlo/transforms/CMakeLists.txt -@@ -20,6 +20,10 @@ - mlir_tablegen(ChloDecompositionPatterns.h.inc --gen-rewriters) - add_public_tablegen_target(ChloDecompositionPatternsIncGen) - -+set(LLVM_TARGET_DEFINITIONS StablehloCreateCompatibilityExpanderPatterns.td) -+mlir_tablegen(StablehloCreateCompatibilityExpanderPatterns.h.inc --gen-rewriters) -+add_public_tablegen_target(StablehloCreateCompatibilityExpanderPatternsIncGen) -+ - set(LLVM_TARGET_DEFINITIONS StablehloLegalizeDeprecatedOpsPatterns.td) - mlir_tablegen(StablehloLegalizeDeprecatedOpsPatterns.h.inc --gen-rewriters) - add_public_tablegen_target(StablehloLegalizeDeprecatedOpsPatternsIncGen) -@@ -27,6 +31,7 @@ - set(LLVM_TARGET_DEFINITIONS VhloToVersionPatterns.td) - mlir_tablegen(VhloToVersionPatterns.h.inc --gen-rewriters) - add_public_tablegen_target(VhloToVersionPatterns) -+ - - add_mlir_dialect_library(StablehloPasses - PARTIAL_SOURCES_INTENDED -@@ -37,6 +42,7 @@ - StablehloAggressiveSimplification.cpp - StablehloCanonicalizeDynamism.cpp - StablehloConvertToSignless.cpp -+ StablehloCreateCompatibilityExpander.cpp - StablehloLegalizeCompositeToCall.cpp - StablehloLegalizeDeprecatedOps.cpp - StablehloLegalizeQuantToMath.cpp -@@ -53,6 +59,7 @@ - StablehloLegalizeDeprecatedOpsPatternsIncGen - PassesIncGen - VhloToVersionPatterns -+ StablehloCreateCompatibilityExpanderPatternsIncGen - - LINK_LIBS PUBLIC - ChloOps -diff --ruN a/stablehlo/stablehlo/transforms/Passes.h b/stablehlo/stablehlo/transforms/Passes.h ---- stablehlo/stablehlo/transforms/Passes.h -+++ stablehlo/stablehlo/transforms/Passes.h -@@ -25,6 +25,7 @@ - #include "mlir/Pass/Pass.h" - #include "mlir/Support/LogicalResult.h" - #include "mlir/Transforms/DialectConversion.h" -+#include "stablehlo/dialect/Version.h" - - namespace mlir { - namespace stablehlo { -@@ -96,6 +97,12 @@ - void populateShapeToStablehloPatterns(MLIRContext *context, - RewritePatternSet *patterns); - -+/// Collection of patterns to create compatibility expander for StableHLO -+/// operations. -+void populateStablehloCreateCompatibilityExpanderPatterns( -+ RewritePatternSet *patterns, MLIRContext *context, -+ vhlo::Version targetVersion); -+ - //// Additional pass constructors //// - - std::unique_ptr> createStablehloRefineArgumentsPass( -diff --ruN a/stablehlo/stablehlo/transforms/Passes.td b/stablehlo/stablehlo/transforms/Passes.td ---- stablehlo/stablehlo/transforms/Passes.td -+++ stablehlo/stablehlo/transforms/Passes.td -@@ -292,3 +292,51 @@ - "mlir::stablehlo::StablehloDialect", - ]; - } -+ -+def StablehloCreateCompatibilityExpanderPass : Pass<"stablehlo-create-compatibility-expander", "mlir::func::FuncOp"> { -+ let summary = "Create compatibility expander for StableHLO operations."; -+ -+ let description = [{ -+ StableHLO ops gets updates or new op is introduced in the latest versions. -+ This opt-in pass expands backward compatibility with older StableHLO -+ versions by decomposing newer StableHLO operations into equivalent -+ operations supported by those older versions. -+ -+ Why is this an opt-in pass? -+ -+ Occasionally, StableHLO op enhancements are used to greatly simplify the -+ handling of certain common patterns in the OpenXLA ecosystem. This -+ includes things like TanOp, which has high framework and compiler support, -+ as well as gather/scatter batching dimensions, which can be represented -+ using slices, but makes sharding much more difficult. For this category of -+ new features, we do not offer automatic downgrade, since it may throw away -+ important information used in subsequent optimizations. This pass can be -+ used to expand these ops based on a target version to maximize compatibility -+ at the expense of potentially less optimal compilation. -+ -+ ```mlir -+ func.func @tan_op_non_complex(%arg0: tensor<4xf64>) -> tensor<4xf64> { -+ %1 = stablehlo.tan %arg0 : tensor<4xf64> -+ func.return %1 : tensor<4xf64> -+ } -+ ``` -+ -+ will become: -+ -+ ```mlir -+ func.func @tan_op_non_complex(%arg0: tensor<4xf64>) -> tensor<4xf64> { -+ %0 = stablehlo.sine %arg0 : tensor<4xf64> -+ %1 = stablehlo.cosine %arg0 : tensor<4xf64> -+ %2 = stablehlo.divide %0, %1 : tensor<4xf64> -+ return %2 : tensor<4xf64> -+ } -+ ``` -+ }]; -+ let options = [ -+ Option<"targetVersionOption", "target", "std::string", "", -+ "The target version. Must be a version of the form #.#.#.">, -+ ]; -+ let dependentDialects = [ -+ "mlir::stablehlo::StablehloDialect", -+ ]; -+} -diff --ruN a/stablehlo/stablehlo/transforms/StablehloCreateCompatibilityExpander.cpp b/stablehlo/stablehlo/transforms/StablehloCreateCompatibilityExpander.cpp ---- stablehlo/stablehlo/transforms/StablehloCreateCompatibilityExpander.cpp -+++ stablehlo/stablehlo/transforms/StablehloCreateCompatibilityExpander.cpp -@@ -0,0 +1,128 @@ -+/* Copyright 2024 The StableHLO Authors. All Rights Reserved. -+Licensed under the Apache License, Version 2.0 (the "License"); -+you may not use this file except in compliance with the License. -+You may obtain a copy of the License at -+ http://www.apache.org/licenses/LICENSE-2.0 -+Unless required by applicable law or agreed to in writing, software -+distributed under the License is distributed on an "AS IS" BASIS, -+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -+See the License for the specific language governing permissions and -+limitations under the License. -+==============================================================================*/ -+ -+#include -+ -+#include -+ -+#include "llvm/ADT/APFloat.h" -+#include "llvm/Support/ErrorHandling.h" -+#include "mlir/Dialect/Func/IR/FuncOps.h" -+#include "mlir/IR/BuiltinAttributes.h" -+#include "mlir/IR/PatternMatch.h" -+#include "mlir/Support/LLVM.h" -+#include "mlir/Transforms/DialectConversion.h" -+#include "mlir/Transforms/GreedyPatternRewriteDriver.h" -+#include "stablehlo/dialect/StablehloOps.h" -+#include "stablehlo/dialect/Version.h" -+#include "stablehlo/transforms/Passes.h" -+ -+namespace mlir { -+namespace stablehlo { -+#define GEN_PASS_DEF_STABLEHLOCREATECOMPATIBILITYEXPANDERPASS -+#include "stablehlo/transforms/Passes.h.inc" -+ -+namespace { -+ -+//===----------------------------------------------------------------------===// -+// Helpers. -+//===----------------------------------------------------------------------===// -+ -+// Creates a constant with all ones. -+static Value createConstantWithAllOnes(OpBuilder &b, Location loc, Value val) { -+ auto shapedTy = dyn_cast(val.getType()); -+ if (!shapedTy) llvm_unreachable("Unsupported shaped type."); -+ -+ mlir::DenseElementsAttr elementsAttr = -+ mlir::DenseElementsAttr::get(shapedTy, 1.0); -+ -+ return b.create(loc, val.getType(), -+ elementsAttr); -+} -+ -+// Check user-specified target version. -+vhlo::Version validateTargetVersion(llvm::StringRef versionRef) { -+ auto failOrVersion = vhlo::Version::fromString(versionRef); -+ if (failed(failOrVersion)) { -+ assert(!versionRef.empty() && -+ "No target version specified. Target version must be of the form " -+ "`#.#.#`."); -+ assert(versionRef.empty() && -+ "Invalid target version argument. Target version must be of the " -+ "form `#.#.#`."); -+ } -+ vhlo::Version targetVersion = *failOrVersion; -+ assert((vhlo::Version::getMinimumVersion() <= targetVersion) && -+ "target version is less than minimum supported."); -+ assert((targetVersion <= vhlo::Version::getCurrentVersion()) && -+ "target version is greater than current version."); -+ return targetVersion; -+} -+ -+//===----------------------------------------------------------------------===// -+// Pass -+//===----------------------------------------------------------------------===// -+ -+struct StablehloCreateCompatibilityExpanderPass -+ : public impl::StablehloCreateCompatibilityExpanderPassBase< -+ StablehloCreateCompatibilityExpanderPass> { -+ StablehloCreateCompatibilityExpanderPass() -+ : StablehloCreateCompatibilityExpanderPassBase< -+ StablehloCreateCompatibilityExpanderPass>() {} -+ StablehloCreateCompatibilityExpanderPass( -+ const StablehloCreateCompatibilityExpanderPassOptions &opts) -+ : StablehloCreateCompatibilityExpanderPassBase< -+ StablehloCreateCompatibilityExpanderPass>(opts) {} -+ -+ public: -+ LogicalResult initialize(MLIRContext *context) override { -+ auto targetVersion = validateTargetVersion(targetVersionOption); -+ -+ config.useTopDownTraversal = true; -+ RewritePatternSet patterns_(context); -+ populateStablehloCreateCompatibilityExpanderPatterns(&patterns_, context, -+ targetVersion); -+ patterns = std::move(patterns_); -+ return success(); -+ } -+ -+ void runOnOperation() override { -+ auto func = getOperation(); -+ if (failed(applyPatternsAndFoldGreedily(func, patterns, config))) { -+ func.emitError( -+ "Failed to converge StableHLOCreateCompatibilityExpanderPass in ") -+ << config.maxIterations << " iterations"; -+ signalPassFailure(); -+ } -+ } -+ -+ private: -+ FrozenRewritePatternSet patterns; -+ GreedyRewriteConfig config; -+}; -+ -+#include "stablehlo/transforms/StablehloCreateCompatibilityExpanderPatterns.h.inc" -+ -+} // namespace -+ -+void populateStablehloCreateCompatibilityExpanderPatterns( -+ RewritePatternSet *patterns, MLIRContext *context, -+ vhlo::Version targetVersion) { -+ // StableHLO TanOp is introduced in v1.4.0. -+ if (targetVersion < vhlo::Version(1, 4, 0)) { -+ patterns->add(context); -+ patterns->add(context); -+ } -+} -+ -+} // namespace stablehlo -+} // namespace mlir -diff --ruN a/stablehlo/stablehlo/transforms/StablehloCreateCompatibilityExpanderPatterns.td b/stablehlo/stablehlo/transforms/StablehloCreateCompatibilityExpanderPatterns.td ---- stablehlo/stablehlo/transforms/StablehloCreateCompatibilityExpanderPatterns.td -+++ stablehlo/stablehlo/transforms/StablehloCreateCompatibilityExpanderPatterns.td -@@ -0,0 +1,47 @@ -+/* Copyright 2022 The StableHLO Authors. -+ -+Licensed under the Apache License, Version 2.0 (the "License"); -+you may not use this file except in compliance with the License. -+You may obtain a copy of the License at -+ -+ http://www.apache.org/licenses/LICENSE-2.0 -+ -+Unless required by applicable law or agreed to in writing, software -+distributed under the License is distributed on an "AS IS" BASIS, -+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -+See the License for the specific language governing permissions and -+limitations under the License. -+==============================================================================*/ -+ -+include "mlir/IR/OpBase.td" -+include "stablehlo/dialect/StablehloOps.td" -+ -+def ComplexElementType : Type< -+ CPred<"isa(cast($_self).getElementType())">, -+ "Complex element type">; -+ -+def NonComplexElementType : Type< -+ CPred<"!isa(cast($_self).getElementType())">, -+ "Non-complex element type">; -+ -+def createConstantWithAllOnes : NativeCodeCall<"createConstantWithAllOnes($_builder, $_loc, $0)">; -+ -+// Express `tan` as -+// sine(x) / cosine(x) -+def TanOp_CompatiblityExpander : Pat<(StableHLO_TanOp NonComplexElementType:$input), -+ (StableHLO_DivOp -+ (StableHLO_SineOp $input), -+ (StableHLO_CosineOp $input) -+ )>; -+ -+// Express `tan(a + bi)` as -+// (tan(a) + i tanh(b)) / (1 - i tan(a) * tanh(b)) -+def TanOp_ComplexElementType_CompatiblityExpander : Pat<(StableHLO_TanOp ComplexElementType:$input), -+ (StableHLO_DivOp -+ (StableHLO_ComplexOp -+ (StableHLO_TanOp:$tan (StableHLO_RealOp $input)), -+ (StableHLO_TanhOp:$tanh (StableHLO_ImagOp $input))), -+ (StableHLO_ComplexOp -+ (createConstantWithAllOnes $tan), -+ (StableHLO_NegOp (StableHLO_MulOp $tan, $tanh))) -+ )>; - diff --git a/third_party/modules/stablehlo/20240829.0-54aa1a5/source.json b/third_party/modules/stablehlo/20240829.0-54aa1a5/source.json deleted file mode 100644 index 6d271a53..00000000 --- a/third_party/modules/stablehlo/20240829.0-54aa1a5/source.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "strip_prefix": "stablehlo-54aa1a57178251981da616b877dda1a88d840d11", - "url": "https://github.com/openxla/stablehlo/archive/54aa1a57178251981da616b877dda1a88d840d11.tar.gz", - "integrity": "sha256-PFXXs3vQ+WlMiCC8twbGEFbsBEF/67Z4qtmq8CtUSUU=", - "overlay": { - "MODULE.bazel": "" - }, - "patch_strip": 1, - "patches": { - "0001-Remove-duplicated-symbols-in-StablehloApi.h.patch": "" - } -} diff --git a/third_party/modules/stablehlo/20240917.0-78c753a/MODULE.bazel b/third_party/modules/stablehlo/20240917.0-78c753a/MODULE.bazel deleted file mode 100644 index 7cbfb936..00000000 --- a/third_party/modules/stablehlo/20240917.0-78c753a/MODULE.bazel +++ /dev/null @@ -1,15 +0,0 @@ -module( - name = "stablehlo", - version = "20240917.0-78c753a", - compatibility_level = 1, -) - -bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "rules_cc", version = "0.0.9") -bazel_dep(name = "llvm-raw", version = "20240919.0-94c024a") - -llvm = use_extension("@llvm-raw//utils/bazel:extension.bzl", "llvm") -llvm.configure( - targets = ["AArch64", "X86", "NVPTX"], -) -use_repo(llvm, "llvm-project") diff --git a/third_party/modules/stablehlo/20240917.0-78c753a/overlay/MODULE.bazel b/third_party/modules/stablehlo/20240917.0-78c753a/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/stablehlo/20240917.0-78c753a/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/stablehlo/20240917.0-78c753a/source.json b/third_party/modules/stablehlo/20240917.0-78c753a/source.json deleted file mode 100644 index cd677bd8..00000000 --- a/third_party/modules/stablehlo/20240917.0-78c753a/source.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "strip_prefix": "stablehlo-78c753ad13ad8205cacc5fcc12418c1ac97276c7", - "url": "https://github.com/openxla/stablehlo/archive/78c753ad13ad8205cacc5fcc12418c1ac97276c7.tar.gz", - "integrity": "sha256-2qFcPLXs7glf7RFDUiqbKXiBBamW7n8a30egR4Yu/bc=", - "overlay": { - "MODULE.bazel": "" - } -} diff --git a/third_party/modules/stablehlo/metadata.json b/third_party/modules/stablehlo/metadata.json deleted file mode 100644 index 1fb46175..00000000 --- a/third_party/modules/stablehlo/metadata.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "homepage": "https://github.com/openxla/stablehlo", - "maintainers": [ - { - "email": "bzlmod@zml.ai", - "github": "zml", - "name": "ZML Engineering Team" - } - ], - "repository": [ - "github:openxla/stablehlo" - ], - "versions": [ - "20240829.0-54aa1a5", - "20240917.0-78c753a", - ], - "yanked_versions": {} -} diff --git a/third_party/modules/xla/20240902.0-d18cd64/MODULE.bazel b/third_party/modules/xla/20240902.0-d18cd64/MODULE.bazel deleted file mode 100644 index 739fe835..00000000 --- a/third_party/modules/xla/20240902.0-d18cd64/MODULE.bazel +++ /dev/null @@ -1,34 +0,0 @@ -module( - name = "xla", - version = "20240902.0-d18cd64", - compatibility_level = 1, -) - -bazel_dep(name = "platforms", version = "0.0.8") -bazel_dep(name = "bazel_skylib", version = "1.5.0") -bazel_dep(name = "rules_cc", version = "0.0.9") -bazel_dep(name = "rules_apple", version = "3.2.1", repo_name = "build_bazel_rules_apple") -bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "com_google_absl") -bazel_dep(name = "rules_python", version = "0.29.0") -bazel_dep(name = "rules_proto", version = "6.0.0-rc1") -bazel_dep(name = "rules_java", version = "7.3.2") -bazel_dep(name = "rules_pkg", version = "0.9.1") -bazel_dep(name = "zlib", version = "1.2.13") -bazel_dep(name = "re2", version = "2024-02-01", repo_name = "com_googlesource_code_re2") -bazel_dep(name = "rules_license", version = "0.0.8") - -bazel_dep(name = "stablehlo", version = "20240829.0-54aa1a5") - -tsl = use_extension("//:tsl.bzl", "tsl") -use_repo(tsl, "tsl") - -xla_workspace = use_extension("//:workspace.bzl", "xla_workspace") -use_repo( - xla_workspace, - "com_github_grpc_grpc", - "com_google_protobuf", - "local_config_cuda", - "local_config_remote_execution", - "local_config_rocm", - "local_config_tensorrt", -) diff --git a/third_party/modules/xla/20240902.0-d18cd64/overlay/MODULE.bazel b/third_party/modules/xla/20240902.0-d18cd64/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/xla/20240902.0-d18cd64/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/xla/20240902.0-d18cd64/overlay/tsl.bzl b/third_party/modules/xla/20240902.0-d18cd64/overlay/tsl.bzl deleted file mode 100644 index 50c1df12..00000000 --- a/third_party/modules/xla/20240902.0-d18cd64/overlay/tsl.bzl +++ /dev/null @@ -1,13 +0,0 @@ -load("//third_party:repo.bzl", "tf_vendored") - -def _tsl_impl(mctx): - tf_vendored(name = "tsl", relpath = "third_party/tsl") - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -tsl = module_extension( - implementation = _tsl_impl, -) diff --git a/third_party/modules/xla/20240902.0-d18cd64/overlay/workspace.bzl b/third_party/modules/xla/20240902.0-d18cd64/overlay/workspace.bzl deleted file mode 100644 index 87c035b7..00000000 --- a/third_party/modules/xla/20240902.0-d18cd64/overlay/workspace.bzl +++ /dev/null @@ -1,52 +0,0 @@ -load("@tsl//third_party:repo.bzl", "tf_http_archive", "tf_mirror_urls") -load("@tsl//third_party/gpus:cuda_configure.bzl", "cuda_configure") -load("@tsl//third_party/gpus:rocm_configure.bzl", "rocm_configure") -load("@tsl//third_party/tensorrt:tensorrt_configure.bzl", "tensorrt_configure") -load("@tsl//tools/toolchains/remote:configure.bzl", "remote_execution_configure") - -def _xla_workspace_impl(mctx): - cuda_configure(name = "local_config_cuda") - remote_execution_configure(name = "local_config_remote_execution") - rocm_configure(name = "local_config_rocm") - tensorrt_configure(name = "local_config_tensorrt") - tf_http_archive( - name = "com_github_grpc_grpc", - sha256 = "b956598d8cbe168b5ee717b5dafa56563eb5201a947856a6688bbeac9cac4e1f", - strip_prefix = "grpc-b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd", - system_build_file = "@tsl//third_party/systemlibs:grpc.BUILD", - patch_file = [ - "@tsl//third_party/grpc:generate_cc_env_fix.patch", - "@tsl//third_party/grpc:register_go_toolchain.patch", - ], - system_link_files = { - "@tsl//third_party/systemlibs:BUILD": "bazel/BUILD", - "@tsl//third_party/systemlibs:grpc.BUILD": "src/compiler/BUILD", - "@tsl//third_party/systemlibs:grpc.bazel.grpc_deps.bzl": "bazel/grpc_deps.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.grpc_extra_deps.bzl": "bazel/grpc_extra_deps.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.cc_grpc_library.bzl": "bazel/cc_grpc_library.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.generate_cc.bzl": "bazel/generate_cc.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.protobuf.bzl": "bazel/protobuf.bzl", - }, - urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd.tar.gz"), - ) - tf_http_archive( - name = "com_google_protobuf", - patch_file = ["@tsl//third_party/protobuf:protobuf.patch"], - sha256 = "f66073dee0bc159157b0bd7f502d7d1ee0bc76b3c1eac9836927511bdc4b3fc1", - strip_prefix = "protobuf-3.21.9", - system_build_file = "@tsl//third_party/systemlibs:protobuf.BUILD", - system_link_files = { - "@tsl//third_party/systemlibs:protobuf.bzl": "protobuf.bzl", - "@tsl//third_party/systemlibs:protobuf_deps.bzl": "protobuf_deps.bzl", - }, - urls = tf_mirror_urls("https://github.com/protocolbuffers/protobuf/archive/v3.21.9.zip"), - ) - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -xla_workspace = module_extension( - implementation = _xla_workspace_impl, -) diff --git a/third_party/modules/xla/20240902.0-d18cd64/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch b/third_party/modules/xla/20240902.0-d18cd64/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch deleted file mode 100644 index 0f58f107..00000000 --- a/third_party/modules/xla/20240902.0-d18cd64/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch +++ /dev/null @@ -1,27 +0,0 @@ -From 4db5de34f70d991fedbe28915c8239b97ba7a064 Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Mon, 18 Mar 2024 17:17:34 +0100 -Subject: [PATCH 3/3] [PJRT C API] Ensure C compliance for Profiler Extension - ---- - xla/pjrt/c/pjrt_c_api_profiler_extension.h | 2 ++ - 1 file changed, 2 insertions(+) - -diff --git a/xla/pjrt/c/pjrt_c_api_profiler_extension.h b/xla/pjrt/c/pjrt_c_api_profiler_extension.h -index c821916ad..89a596123 100644 ---- a/xla/pjrt/c/pjrt_c_api_profiler_extension.h -+++ b/xla/pjrt/c/pjrt_c_api_profiler_extension.h -@@ -16,8 +16,10 @@ limitations under the License. - #ifndef XLA_PJRT_C_PJRT_C_API_PROFILER_EXTENSION_H_ - #define XLA_PJRT_C_PJRT_C_API_PROFILER_EXTENSION_H_ - -+#ifdef __cplusplus - #include - #include -+#endif - - #include "xla/backends/profiler/plugin/profiler_c_api.h" - #include "xla/pjrt/c/pjrt_c_api.h" --- -2.39.3 (Apple Git-146) - diff --git a/third_party/modules/xla/20240902.0-d18cd64/source.json b/third_party/modules/xla/20240902.0-d18cd64/source.json deleted file mode 100644 index 359f9b12..00000000 --- a/third_party/modules/xla/20240902.0-d18cd64/source.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "strip_prefix": "xla-d18cd64b7cd61a2ade10089665ac104f639101b1", - "url": "https://github.com/openxla/xla/archive/d18cd64b7cd61a2ade10089665ac104f639101b1.tar.gz", - "integrity": "sha256-EtKhjU91STBceUmg0TUE6cPeRkeSz3LI2S1i3EFMj/E=", - "overlay": { - "tsl.bzl": "", - "workspace.bzl": "", - "MODULE.bazel": "" - }, - "patch_strip": 1, - "patches": { - "0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch": "" - } -} diff --git a/third_party/modules/xla/20240919.0-1b18dd6/MODULE.bazel b/third_party/modules/xla/20240919.0-1b18dd6/MODULE.bazel deleted file mode 100644 index bd3a0046..00000000 --- a/third_party/modules/xla/20240919.0-1b18dd6/MODULE.bazel +++ /dev/null @@ -1,34 +0,0 @@ -module( - name = "xla", - version = "20240919.0-1b18dd6", - compatibility_level = 1, -) - -bazel_dep(name = "platforms", version = "0.0.8") -bazel_dep(name = "bazel_skylib", version = "1.5.0") -bazel_dep(name = "rules_cc", version = "0.0.9") -bazel_dep(name = "rules_apple", version = "3.2.1", repo_name = "build_bazel_rules_apple") -bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "com_google_absl") -bazel_dep(name = "rules_python", version = "0.29.0") -bazel_dep(name = "rules_proto", version = "6.0.0-rc1") -bazel_dep(name = "rules_java", version = "7.3.2") -bazel_dep(name = "rules_pkg", version = "0.9.1") -bazel_dep(name = "zlib", version = "1.2.13") -bazel_dep(name = "re2", version = "2024-02-01", repo_name = "com_googlesource_code_re2") -bazel_dep(name = "rules_license", version = "0.0.8") - -bazel_dep(name = "stablehlo", version = "20240917.0-78c753a") - -tsl = use_extension("//:tsl.bzl", "tsl") -use_repo(tsl, "tsl") - -xla_workspace = use_extension("//:workspace.bzl", "xla_workspace") -use_repo( - xla_workspace, - "com_github_grpc_grpc", - "com_google_protobuf", - "local_config_cuda", - "local_config_remote_execution", - "local_config_rocm", - "local_config_tensorrt", -) diff --git a/third_party/modules/xla/20240919.0-1b18dd6/overlay/MODULE.bazel b/third_party/modules/xla/20240919.0-1b18dd6/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/xla/20240919.0-1b18dd6/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/xla/20240919.0-1b18dd6/overlay/tsl.bzl b/third_party/modules/xla/20240919.0-1b18dd6/overlay/tsl.bzl deleted file mode 100644 index 50c1df12..00000000 --- a/third_party/modules/xla/20240919.0-1b18dd6/overlay/tsl.bzl +++ /dev/null @@ -1,13 +0,0 @@ -load("//third_party:repo.bzl", "tf_vendored") - -def _tsl_impl(mctx): - tf_vendored(name = "tsl", relpath = "third_party/tsl") - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -tsl = module_extension( - implementation = _tsl_impl, -) diff --git a/third_party/modules/xla/20240919.0-1b18dd6/overlay/workspace.bzl b/third_party/modules/xla/20240919.0-1b18dd6/overlay/workspace.bzl deleted file mode 100644 index 87c035b7..00000000 --- a/third_party/modules/xla/20240919.0-1b18dd6/overlay/workspace.bzl +++ /dev/null @@ -1,52 +0,0 @@ -load("@tsl//third_party:repo.bzl", "tf_http_archive", "tf_mirror_urls") -load("@tsl//third_party/gpus:cuda_configure.bzl", "cuda_configure") -load("@tsl//third_party/gpus:rocm_configure.bzl", "rocm_configure") -load("@tsl//third_party/tensorrt:tensorrt_configure.bzl", "tensorrt_configure") -load("@tsl//tools/toolchains/remote:configure.bzl", "remote_execution_configure") - -def _xla_workspace_impl(mctx): - cuda_configure(name = "local_config_cuda") - remote_execution_configure(name = "local_config_remote_execution") - rocm_configure(name = "local_config_rocm") - tensorrt_configure(name = "local_config_tensorrt") - tf_http_archive( - name = "com_github_grpc_grpc", - sha256 = "b956598d8cbe168b5ee717b5dafa56563eb5201a947856a6688bbeac9cac4e1f", - strip_prefix = "grpc-b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd", - system_build_file = "@tsl//third_party/systemlibs:grpc.BUILD", - patch_file = [ - "@tsl//third_party/grpc:generate_cc_env_fix.patch", - "@tsl//third_party/grpc:register_go_toolchain.patch", - ], - system_link_files = { - "@tsl//third_party/systemlibs:BUILD": "bazel/BUILD", - "@tsl//third_party/systemlibs:grpc.BUILD": "src/compiler/BUILD", - "@tsl//third_party/systemlibs:grpc.bazel.grpc_deps.bzl": "bazel/grpc_deps.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.grpc_extra_deps.bzl": "bazel/grpc_extra_deps.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.cc_grpc_library.bzl": "bazel/cc_grpc_library.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.generate_cc.bzl": "bazel/generate_cc.bzl", - "@tsl//third_party/systemlibs:grpc.bazel.protobuf.bzl": "bazel/protobuf.bzl", - }, - urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd.tar.gz"), - ) - tf_http_archive( - name = "com_google_protobuf", - patch_file = ["@tsl//third_party/protobuf:protobuf.patch"], - sha256 = "f66073dee0bc159157b0bd7f502d7d1ee0bc76b3c1eac9836927511bdc4b3fc1", - strip_prefix = "protobuf-3.21.9", - system_build_file = "@tsl//third_party/systemlibs:protobuf.BUILD", - system_link_files = { - "@tsl//third_party/systemlibs:protobuf.bzl": "protobuf.bzl", - "@tsl//third_party/systemlibs:protobuf_deps.bzl": "protobuf_deps.bzl", - }, - urls = tf_mirror_urls("https://github.com/protocolbuffers/protobuf/archive/v3.21.9.zip"), - ) - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -xla_workspace = module_extension( - implementation = _xla_workspace_impl, -) diff --git a/third_party/modules/xla/20240919.0-1b18dd6/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch b/third_party/modules/xla/20240919.0-1b18dd6/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch deleted file mode 100644 index 0f58f107..00000000 --- a/third_party/modules/xla/20240919.0-1b18dd6/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch +++ /dev/null @@ -1,27 +0,0 @@ -From 4db5de34f70d991fedbe28915c8239b97ba7a064 Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Mon, 18 Mar 2024 17:17:34 +0100 -Subject: [PATCH 3/3] [PJRT C API] Ensure C compliance for Profiler Extension - ---- - xla/pjrt/c/pjrt_c_api_profiler_extension.h | 2 ++ - 1 file changed, 2 insertions(+) - -diff --git a/xla/pjrt/c/pjrt_c_api_profiler_extension.h b/xla/pjrt/c/pjrt_c_api_profiler_extension.h -index c821916ad..89a596123 100644 ---- a/xla/pjrt/c/pjrt_c_api_profiler_extension.h -+++ b/xla/pjrt/c/pjrt_c_api_profiler_extension.h -@@ -16,8 +16,10 @@ limitations under the License. - #ifndef XLA_PJRT_C_PJRT_C_API_PROFILER_EXTENSION_H_ - #define XLA_PJRT_C_PJRT_C_API_PROFILER_EXTENSION_H_ - -+#ifdef __cplusplus - #include - #include -+#endif - - #include "xla/backends/profiler/plugin/profiler_c_api.h" - #include "xla/pjrt/c/pjrt_c_api.h" --- -2.39.3 (Apple Git-146) - diff --git a/third_party/modules/xla/20240919.0-1b18dd6/source.json b/third_party/modules/xla/20240919.0-1b18dd6/source.json deleted file mode 100644 index 9853728a..00000000 --- a/third_party/modules/xla/20240919.0-1b18dd6/source.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "strip_prefix": "xla-d391119197eab771a84c1f8a59a7f50b7da4b43d", - "url": "https://github.com/openxla/xla/archive/d391119197eab771a84c1f8a59a7f50b7da4b43d.tar.gz", - "integrity": "sha256-ToPcvPhoGX6Ny3yLx6kJlQSwj7k5Xy4T+BjIp3DAj/s=", - "overlay": { - "tsl.bzl": "", - "workspace.bzl": "", - "MODULE.bazel": "" - }, - "patch_strip": 1, - "patches": { - "0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch": "" - } -} diff --git a/third_party/modules/xla/metadata.json b/third_party/modules/xla/metadata.json deleted file mode 100644 index e0dc3250..00000000 --- a/third_party/modules/xla/metadata.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "homepage": "https://github.com/openxla/xla", - "maintainers": [ - { - "email": "bzlmod@zml.ai", - "github": "zml", - "name": "ZML Engineering Team" - } - ], - "repository": [ - "github:openxla/xla" - ], - "versions": [ - "20240902.0-d18cd64", - "20240919.0-1b18dd6", - ], - "yanked_versions": {} -} diff --git a/third_party/modules/zig-protobuf/20240722.0-c644d11/MODULE.bazel b/third_party/modules/zig-protobuf/20240722.0-c644d11/MODULE.bazel deleted file mode 100644 index 4fb39921..00000000 --- a/third_party/modules/zig-protobuf/20240722.0-c644d11/MODULE.bazel +++ /dev/null @@ -1,8 +0,0 @@ -module( - name = "zig-protobuf", - version = "20240722.0-c644d11", - compatibility_level = 1, -) - -bazel_dep(name = "rules_zig", version = "20240904.0-010da15") -bazel_dep(name = "rules_proto", version = "6.0.0-rc1") diff --git a/third_party/modules/zig-protobuf/20240722.0-c644d11/overlay/BUILD.bazel b/third_party/modules/zig-protobuf/20240722.0-c644d11/overlay/BUILD.bazel deleted file mode 100644 index 832c7591..00000000 --- a/third_party/modules/zig-protobuf/20240722.0-c644d11/overlay/BUILD.bazel +++ /dev/null @@ -1,32 +0,0 @@ -load("@rules_proto//proto:defs.bzl", "proto_lang_toolchain") -load("@rules_zig//zig:defs.bzl", "BINARY_KIND", "zig_binary", "zig_library") - -zig_library( - name = "protobuf", - import_name = "protobuf", - main = "src/protobuf.zig", - visibility = ["//visibility:public"], -) - -zig_binary( - name = "generator", - srcs = [ - "bootstrapped-generator/FullName.zig", - "bootstrapped-generator/google/protobuf/compiler/plugin.pb.zig", - "bootstrapped-generator/google/protobuf/descriptor.pb.zig", - ], - kind = BINARY_KIND.exe, - main = "bootstrapped-generator/main.zig", - visibility = ["//visibility:public"], - deps = [":protobuf"], -) - -proto_lang_toolchain( - name = "zig_toolchain", - command_line = "--zig_out=$(OUT)", - output_files = "multiple", - plugin = ":generator", - plugin_format_flag = "--plugin=protoc-gen-zig=%s", - runtime = ":protobuf", - visibility = ["//visibility:public"], -) diff --git a/third_party/modules/zig-protobuf/20240722.0-c644d11/overlay/MODULE.bazel b/third_party/modules/zig-protobuf/20240722.0-c644d11/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/zig-protobuf/20240722.0-c644d11/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/zig-protobuf/20240722.0-c644d11/patches/0001-USE_MODULES-true.patch b/third_party/modules/zig-protobuf/20240722.0-c644d11/patches/0001-USE_MODULES-true.patch deleted file mode 100644 index 67353922..00000000 --- a/third_party/modules/zig-protobuf/20240722.0-c644d11/patches/0001-USE_MODULES-true.patch +++ /dev/null @@ -1,11 +0,0 @@ ---- a/bootstrapped-generator/main.zig -+++ b/bootstrapped-generator/main.zig -@@ -6,7 +6,7 @@ const descriptor = @import("google/protobuf/descriptor.pb.zig"); - const mem = std.mem; - const FullName = @import("./FullName.zig").FullName; - --const USE_MODULES = false; -+const USE_MODULES = true; - - const allocator = std.heap.page_allocator; - diff --git a/third_party/modules/zig-protobuf/20240722.0-c644d11/source.json b/third_party/modules/zig-protobuf/20240722.0-c644d11/source.json deleted file mode 100644 index f985b249..00000000 --- a/third_party/modules/zig-protobuf/20240722.0-c644d11/source.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "strip_prefix": "zig-protobuf-c644d1105caaca4d9a48d3536e253687052f74c8", - "url": "https://github.com/gwenzek/zig-protobuf/archive/c644d1105caaca4d9a48d3536e253687052f74c8.tar.gz", - "integrity": "sha256-KoZp8MLLEQQgS7Uip4sdjw7TYmhzKPdpilQYKp5iYV4=", - "overlay": { - "MODULE.bazel": "", - "BUILD.bazel": "" - }, - "patch_strip": 1, - "patches": { - "0001-USE_MODULES-true.patch": "" - } -} diff --git a/third_party/modules/zig-protobuf/metadata.json b/third_party/modules/zig-protobuf/metadata.json deleted file mode 100644 index 17e897b6..00000000 --- a/third_party/modules/zig-protobuf/metadata.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "homepage": "https://github.com/gwenzek/zig-protobuf", - "maintainers": [ - { - "email": "bzlmod@zml.ai", - "github": "zml", - "name": "ZML Engineering Team" - } - ], - "repository": [ - "gwenzek/zig-protobuf" - ], - "versions": [ - "20240722.0-c644d11" - ], - "yanked_versions": {} -} diff --git a/third_party/modules/zig-yaml/20240903.0-83d5fdf/MODULE.bazel b/third_party/modules/zig-yaml/20240903.0-83d5fdf/MODULE.bazel deleted file mode 100644 index 151f3337..00000000 --- a/third_party/modules/zig-yaml/20240903.0-83d5fdf/MODULE.bazel +++ /dev/null @@ -1,7 +0,0 @@ -module( - name = "zig-yaml", - version = "20240903.0-83d5fdf", - compatibility_level = 1, -) - -bazel_dep(name = "rules_zig", version = "20240904.0-010da15") diff --git a/third_party/modules/zig-yaml/20240903.0-83d5fdf/overlay/BUILD.bazel b/third_party/modules/zig-yaml/20240903.0-83d5fdf/overlay/BUILD.bazel deleted file mode 100644 index 94897f78..00000000 --- a/third_party/modules/zig-yaml/20240903.0-83d5fdf/overlay/BUILD.bazel +++ /dev/null @@ -1,12 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") - -zig_library( - name = "zig-yaml", - srcs = glob([ - "src/*.zig", - "src/parse/*.zig", - "src/yaml/*.zig", - ]), - main = "src/yaml.zig", - visibility = ["//visibility:public"], -) diff --git a/third_party/modules/zig-yaml/20240903.0-83d5fdf/overlay/MODULE.bazel b/third_party/modules/zig-yaml/20240903.0-83d5fdf/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/zig-yaml/20240903.0-83d5fdf/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/zig-yaml/20240903.0-83d5fdf/source.json b/third_party/modules/zig-yaml/20240903.0-83d5fdf/source.json deleted file mode 100644 index 643aa238..00000000 --- a/third_party/modules/zig-yaml/20240903.0-83d5fdf/source.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "strip_prefix": "zig-yaml-83d5fdfdd6c847b4bd2a8cad73b05a5c6447984b", - "url": "https://github.com/gwenzek/zig-yaml/archive/83d5fdfdd6c847b4bd2a8cad73b05a5c6447984b.tar.gz", - "integrity": "sha256-uusblAIZQgyKlg0qVmuHWD2JG+dMM+N6oKNGZsBdpEk=", - "overlay": { - "MODULE.bazel": "", - "BUILD.bazel": "" - } -} diff --git a/third_party/modules/zig-yaml/metadata.json b/third_party/modules/zig-yaml/metadata.json deleted file mode 100644 index 3aa2c43c..00000000 --- a/third_party/modules/zig-yaml/metadata.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "homepage": "https://github.com/gwenzek/zig-yaml", - "maintainers": [ - { - "email": "bzlmod@zml.ai", - "github": "zml", - "name": "ZML Engineering Team" - } - ], - "repository": [ - "github:gwenzek/zig-yaml" - ], - "versions": [ - "20240903.0-83d5fdf" - ], - "yanked_versions": {} -} diff --git a/third_party/modules/zigcoro/20240829.0-fc1db29/MODULE.bazel b/third_party/modules/zigcoro/20240829.0-fc1db29/MODULE.bazel deleted file mode 100644 index a59b2d87..00000000 --- a/third_party/modules/zigcoro/20240829.0-fc1db29/MODULE.bazel +++ /dev/null @@ -1,8 +0,0 @@ -module( - name = "zigcoro", - version = "20240829.0-fc1db29", - compatibility_level = 1, -) - -bazel_dep(name = "rules_zig", version = "20240904.0-010da15") -bazel_dep(name = "libxev", version = "20240825.0-dbe2291") diff --git a/third_party/modules/zigcoro/20240829.0-fc1db29/overlay/BUILD.bazel b/third_party/modules/zigcoro/20240829.0-fc1db29/overlay/BUILD.bazel deleted file mode 100644 index 29227e97..00000000 --- a/third_party/modules/zigcoro/20240829.0-fc1db29/overlay/BUILD.bazel +++ /dev/null @@ -1,29 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") - -genrule( - name = "libcoro_options_gen", - outs = ["options.zig"], - cmd = """\ -cat < $@ -pub const default_stack_size: usize = 4096; -pub const debug_log_level: usize = 0; -EOF -""", -) - -zig_library( - name = "libcoro_options", - main = ":libcoro_options_gen", -) - -zig_library( - name = "libcoro", - srcs = glob(["src/*.zig"]), - extra_srcs = glob(["src/asm/*.s"]), - main = "src/main.zig", - visibility = ["//visibility:public"], - deps = [ - ":libcoro_options", - "@libxev//:xev", - ], -) diff --git a/third_party/modules/zigcoro/20240829.0-fc1db29/overlay/MODULE.bazel b/third_party/modules/zigcoro/20240829.0-fc1db29/overlay/MODULE.bazel deleted file mode 120000 index 9b599e3a..00000000 --- a/third_party/modules/zigcoro/20240829.0-fc1db29/overlay/MODULE.bazel +++ /dev/null @@ -1 +0,0 @@ -../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/zigcoro/20240829.0-fc1db29/patches/0001-Handle-generic-function-in-Signature.patch b/third_party/modules/zigcoro/20240829.0-fc1db29/patches/0001-Handle-generic-function-in-Signature.patch deleted file mode 100644 index 6072de07..00000000 --- a/third_party/modules/zigcoro/20240829.0-fc1db29/patches/0001-Handle-generic-function-in-Signature.patch +++ /dev/null @@ -1,48 +0,0 @@ -From e2416ff5ec9cad5572e6a5edac3f949b75024c00 Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Tue, 6 Aug 2024 11:34:35 +0200 -Subject: [PATCH 1/2] Handle generic function in Signature - ---- - src/coro.zig | 7 +++++-- - 1 file changed, 5 insertions(+), 2 deletions(-) - -diff --git a/src/coro.zig b/src/coro.zig -index 9285489..8a34717 100644 ---- a/src/coro.zig -+++ b/src/coro.zig -@@ -266,6 +266,7 @@ const CoroT = struct { - YieldT: type = void, - InjectT: type = void, - ArgsT: type, -+ _ReturnT: type, - - // If the function this signature represents is compile-time known, - // it can be held here. -@@ -273,19 +274,21 @@ const CoroT = struct { - - fn init(comptime Func: anytype, comptime options: CoroT.Options) @This() { - const FuncT = if (@TypeOf(Func) == type) Func else @TypeOf(Func); -+ const ArgsT = options.ArgsT orelse ArgsTuple(FuncT); - return .{ - .Func = FuncT, - .YieldT = options.YieldT, - .InjectT = options.InjectT, -- .ArgsT = options.ArgsT orelse ArgsTuple(FuncT), -+ .ArgsT = ArgsT, - .func_ptr = if (@TypeOf(Func) == type) null else struct { - const val = Func; - }, -+ ._ReturnT = @TypeOf(@call(.auto, Func, @as(ArgsT, undefined))), - }; - } - - pub fn ReturnT(comptime self: @This()) type { -- return @typeInfo(self.Func).Fn.return_type.?; -+ return self._ReturnT; - } - }; - --- -2.39.3 (Apple Git-146) - diff --git a/third_party/modules/zigcoro/20240829.0-fc1db29/patches/0002-asyncio-return-error-super-set-in-pread-pwrite.patch b/third_party/modules/zigcoro/20240829.0-fc1db29/patches/0002-asyncio-return-error-super-set-in-pread-pwrite.patch deleted file mode 100644 index f1e68622..00000000 --- a/third_party/modules/zigcoro/20240829.0-fc1db29/patches/0002-asyncio-return-error-super-set-in-pread-pwrite.patch +++ /dev/null @@ -1,34 +0,0 @@ -From d8a4773ee9fa0ae5c5fd91e84c322e75a634a077 Mon Sep 17 00:00:00 2001 -From: Steeve Morin -Date: Tue, 6 Aug 2024 13:14:59 +0200 -Subject: [PATCH 2/2] asyncio: return error super set in pread/pwrite - ---- - src/asyncio.zig | 4 ++-- - 1 file changed, 2 insertions(+), 2 deletions(-) - -diff --git a/src/asyncio.zig b/src/asyncio.zig -index 68cc7d3..8487754 100644 ---- a/src/asyncio.zig -+++ b/src/asyncio.zig -@@ -354,7 +354,7 @@ pub const File = struct { - } - - const PReadResult = xev.ReadError!usize; -- pub fn pread(self: Self, buf: xev.ReadBuffer, offset: u64) PReadResult { -+ pub fn pread(self: Self, buf: xev.ReadBuffer, offset: u64) !usize { - const ResultT = PReadResult; - const Data = struct { - result: ResultT = undefined, -@@ -391,7 +391,7 @@ pub const File = struct { - } - - const PWriteResult = xev.WriteError!usize; -- pub fn pwrite(self: Self, buf: xev.WriteBuffer, offset: u64) PWriteResult { -+ pub fn pwrite(self: Self, buf: xev.WriteBuffer, offset: u64) !usize { - const ResultT = PWriteResult; - const Data = struct { - result: ResultT = undefined, --- -2.39.3 (Apple Git-146) - diff --git a/third_party/modules/zigcoro/20240829.0-fc1db29/source.json b/third_party/modules/zigcoro/20240829.0-fc1db29/source.json deleted file mode 100644 index 17e66d4e..00000000 --- a/third_party/modules/zigcoro/20240829.0-fc1db29/source.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "strip_prefix": "zigcoro-fc1db29dee049ce572a516f1586cce0eca4fcb2a", - "url": "https://github.com/rsepassi/zigcoro/archive/fc1db29dee049ce572a516f1586cce0eca4fcb2a.tar.gz", - "integrity": "sha256-2ynCPLSRFPto3HIOhDTBDyUS770PNMI9ioMkj8Z7Y+s=", - "overlay": { - "MODULE.bazel": "", - "BUILD.bazel": "" - }, - "patch_strip": 1, - "patches": { - "0001-Handle-generic-function-in-Signature.patch": "", - "0002-asyncio-return-error-super-set-in-pread-pwrite.patch": "" - } -} diff --git a/third_party/modules/zigcoro/metadata.json b/third_party/modules/zigcoro/metadata.json deleted file mode 100644 index 533957ce..00000000 --- a/third_party/modules/zigcoro/metadata.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "homepage": "https://github.com/rsepassi/zigcoro", - "maintainers": [ - { - "email": "bzlmod@zml.ai", - "github": "zml", - "name": "ZML Engineering Team" - } - ], - "repository": [ - "github:rsepassi/zigcoro" - ], - "versions": [ - "20240829.0-fc1db29" - ], - "yanked_versions": {} -} diff --git a/third_party/non_module_deps.bzl b/third_party/non_module_deps.bzl new file mode 100644 index 00000000..876b0db2 --- /dev/null +++ b/third_party/non_module_deps.bzl @@ -0,0 +1,48 @@ +load("//third_party/arocc:repo.bzl", arocc = "repo") +load("//third_party/cloud_accelerator_diagnostics:repo.bzl", cloud_accelerator_diagnostics = "repo") +load("//third_party/com_google_sentencepiece:repo.bzl", com_google_sentencepiece = "repo") +load("//third_party/cuda_tile:repo.bzl", cuda_tile = "repo") +load("//third_party/cute_ir:repo.bzl", cute_ir = "repo") +load("//third_party/flashinfer_cutlass_moe:repo.bzl", flashinfer_cutlass_moe = "repo") +load("//third_party/flydsl:repo.bzl", flydsl = "repo") +load("//third_party/iree:repo.bzl", iree = "repo") +load("//third_party/libvaxis:repo.bzl", libvaxis = "repo") +load("//third_party/linenoise:repo.bzl", linenoise = "repo") +load("//third_party/mnist:repo.bzl", mnist = "repo") +load("//third_party/org_swig_swig:repo.bzl", org_swig_swig = "repo") +load("//third_party/stb:repo.bzl", stb = "repo") +load("//third_party/translate-c:repo.bzl", translate_c = "repo") +load("//third_party/uucode:repo.bzl", uucode = "repo") +load("//third_party/xla:repo.bzl", xla = "repo") +load("//third_party/zigimg:repo.bzl", zigimg = "repo") +load("//third_party/zio:repo.bzl", zio = "repo") +load("//third_party/zls:repo.bzl", zls = "repo") + +def _non_module_deps_impl(mctx): + cloud_accelerator_diagnostics() + com_google_sentencepiece() + org_swig_swig() + mnist() + xla() + arocc() + translate_c() + flashinfer_cutlass_moe() + linenoise() + stb() + zigimg() + uucode() + libvaxis() + iree() + cuda_tile() + flydsl() + cute_ir() + zio() + zls() + + return mctx.extension_metadata( + reproducible = True, + ) + +non_module_deps = module_extension( + implementation = _non_module_deps_impl, +) diff --git a/third_party/org_swig_swig/BUILD.bazel b/third_party/org_swig_swig/BUILD.bazel new file mode 100644 index 00000000..e69de29b diff --git a/third_party/org_swig_swig/repo.bzl b/third_party/org_swig_swig/repo.bzl new file mode 100644 index 00000000..9a6c55bb --- /dev/null +++ b/third_party/org_swig_swig/repo.bzl @@ -0,0 +1,13 @@ +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +def repo(): + http_archive( + name = "org_swig_swig", + urls = [ + "https://downloads.sourceforge.net/project/swig/swig/swig-4.4.1/swig-4.4.1.tar.gz", + "https://prdownloads.sourceforge.net/swig/swig-4.4.1.tar.gz", + ], + sha256 = "40162a706c56f7592d08fd52ef5511cb7ac191f3593cf07306a0a554c6281fcf", + strip_prefix = "swig-4.4.1", + build_file = "//:third_party/org_swig_swig/swig.bazel", + ) diff --git a/third_party/org_swig_swig/swig.bazel b/third_party/org_swig_swig/swig.bazel new file mode 100644 index 00000000..c47ed381 --- /dev/null +++ b/third_party/org_swig_swig/swig.bazel @@ -0,0 +1,111 @@ +load("@rules_cc//cc:cc_binary.bzl", "cc_binary") + +licenses(["restricted"]) # GPLv3 + +exports_files(["LICENSE"]) + +filegroup( + name = "lib", + srcs = glob([ + "Lib/*.*", + "Lib/c/*.*", + "Lib/std/*.*", + "Lib/typemaps/*.*", + ]), + visibility = ["//visibility:public"], +) + +cc_binary( + name = "swig", + srcs = glob([ + "Source/CParse/*.h", + "Source/CParse/*.c", + "Source/DOH/*.h", + "Source/DOH/*.c", + "Source/Include/*.h", + "Source/Preprocessor/*.h", + "Source/Preprocessor/*.c", + "Source/Swig/*.h", + "Source/Swig/*.c", + ]) + [ + "Source/Include/swigconfig.h", + "Source/Modules/allocate.cxx", + "Source/Modules/c.cxx", + "Source/Modules/contract.cxx", + "Source/Modules/directors.cxx", + "Source/Modules/emit.cxx", + "Source/Modules/interface.cxx", + "Source/Modules/lang.cxx", + "Source/Modules/main.cxx", + "Source/Modules/nested.cxx", + "Source/Modules/overload.cxx", + "Source/Modules/swigmain-lite.cxx", + "Source/Modules/swigmod.h", + "Source/Modules/typepass.cxx", + "Source/Modules/utils.cxx", + "Source/Modules/xml.cxx", + ], + includes = [ + "Source/CParse", + "Source/DOH", + "Source/Include", + "Source/Modules", + "Source/Preprocessor", + "Source/Swig", + ], + data = [":lib"], + output_licenses = ["unencumbered"], + visibility = ["//visibility:public"], + deps = ["@pcre2"], +) + +genrule( + name = "swigconfig", + outs = ["Source/Include/swigconfig.h"], + cmd = """\ +cat <$@ +#define HAVE_BOOL +#define HAVE_PCRE +#define HAVE_POPEN +#define PACKAGE_BUGREPORT \"http://www.swig.org\" +#define PACKAGE_VERSION \"4.3.0\" +#define STDC_HEADERS +#define SWIG_CXX \"bazel4lyfe\" +#define SWIG_LIB \"external/org_swig_swig/Lib\" +#define SWIG_LIB_WIN_UNIX \"\" +#define SWIG_PLATFORM \"bazel4lyfe\" +EOF +""", +) + +genrule( + name = "get_rid_of_stuff_we_dont_need_yet", + srcs = ["Source/Modules/swigmain.cxx"], + outs = ["Source/Modules/swigmain-lite.cxx"], + cmd = """\ +sed -e '/swig_allegrocl/d' \ + -e '/swig_chicken/d' \ + -e '/swig_clisp/d' \ + -e '/swig_csharp/d' \ + -e '/swig_d/d' \ + -e '/swig_guile/d' \ + -e '/swig_go/d' \ + -e '/swig_java/d' \ + -e '/swig_lua/d' \ + -e '/swig_modula3/d' \ + -e '/swig_mzscheme/d' \ + -e '/swig_ocaml/d' \ + -e '/swig_octave/d' \ + -e '/swig_perl/d' \ + -e '/swig_php/d' \ + -e '/swig_pike/d' \ + -e '/swig_python/d' \ + -e '/swig_r/d' \ + -e '/swig_ruby/d' \ + -e '/swig_scilab/d' \ + -e '/swig_sexp/d' \ + -e '/swig_tcl/d' \ + -e '/swig_uffi/d' \ + $< >$@ +""", +) diff --git a/third_party/rules_distroless/accept-nospace-version-constraints.patch b/third_party/rules_distroless/accept-nospace-version-constraints.patch new file mode 100644 index 00000000..62973749 --- /dev/null +++ b/third_party/rules_distroless/accept-nospace-version-constraints.patch @@ -0,0 +1,28 @@ +diff --git a/apt/private/version_constraint.bzl b/apt/private/version_constraint.bzl +index 531fa01..b3a3d30 100644 +--- a/apt/private/version_constraint.bzl ++++ b/apt/private/version_constraint.bzl +@@ -2,11 +2,19 @@ + + load(":version.bzl", version_lib = "version") + ++_VERSION_CONSTRAINT_OPERATORS = ["<<", ">>", "<=", ">=", "="] ++ + def _parse_version_constraint(rawv): +- vconst_i = rawv.find(" ") +- if vconst_i == -1: +- fail('invalid version string %s expected a version constraint ">=", "=", ">=", "<<", ">>"' % rawv) +- return (rawv[:vconst_i], rawv[vconst_i + 1:]) ++ rawv = rawv.strip() ++ for op in _VERSION_CONSTRAINT_OPERATORS: ++ if rawv.startswith(op): ++ version = rawv[len(op):].strip() ++ if version: ++ return (op, version) ++ ++ break ++ ++ fail('invalid version string %s expected a version constraint ">=", "=", "<=", "<<", ">>"' % rawv) + + def _parse_dep(raw): + raw = raw.strip() # remove leading & trailing whitespace diff --git a/third_party/rules_distroless/flat-repositories.patch b/third_party/rules_distroless/flat-repositories.patch new file mode 100644 index 00000000..e4ba73bc --- /dev/null +++ b/third_party/rules_distroless/flat-repositories.patch @@ -0,0 +1,59 @@ +diff --git a/apt/private/deb_resolve.bzl b/apt/private/deb_resolve.bzl +--- a/apt/private/deb_resolve.bzl ++++ b/apt/private/deb_resolve.bzl +@@ -46,7 +46,10 @@ + sources = [] + + for src in manifest["sources"]: +- distr, components = src["channel"].split(" ", 1) ++ if src["channel"].endswith("/"): ++ distr, components = src["channel"], "" ++ else: ++ distr, components = src["channel"].split(" ", 1) + for comp in components.split(" "): + # TODO: only support urls before 1.0 + if "urls" in src: +@@ -60,6 +63,7 @@ + urls, + distr, + comp, ++ src.get("archs", manifest["archs"]), + )) + + repository = deb_repository.new(rctx, sources = sources, archs = manifest["archs"]) +diff --git a/apt/private/apt_deb_repository.bzl b/apt/private/apt_deb_repository.bzl +--- a/apt/private/apt_deb_repository.bzl ++++ b/apt/private/apt_deb_repository.bzl +@@ -13,7 +13,7 @@ + return use_netrc(netrc, urls, {}) + + def _fetch_package_index(rctx, urls, dist, comp, arch, integrity): +- target_triple = "{dist}/{comp}/{arch}".format(dist = dist, comp = comp, arch = arch) ++ target_triple = "{dist}/{comp}/{arch}".format(dist = dist.strip("/") or ".", comp = comp, arch = arch) + + # See https://linux.die.net/man/1/xz , https://linux.die.net/man/1/gzip , and https://linux.die.net/man/1/bzip2 + # --keep -> keep the original file (Bazel might be still committing the output to the cache) +@@ -35,7 +35,10 @@ + download = None + for (ext, cmd) in supported_extensions: + output = "{}/Packages{}".format(target_triple, ext) +- dist_url = "{}/dists/{}/{}/binary-{}/Packages{}".format(url, dist, comp, arch, ext) ++ if dist.endswith("/"): ++ dist_url = "{}/{}Packages{}".format(url, dist.lstrip("/"), ext) ++ else: ++ dist_url = "{}/dists/{}/{}/binary-{}/Packages{}".format(url, dist, comp, arch, ext) + auth = {} + if url in base_auth: + auth = {dist_url: base_auth[url]} +@@ -167,7 +170,10 @@ + ) + + for arch in archs: +- for (urls, dist, comp) in sources: ++ for (urls, dist, comp, source_archs) in sources: ++ if arch not in source_archs: ++ continue ++ + # We assume that `url` does not contain a trailing forward slash when passing to + # functions below. If one is present, remove it. Some HTTP servers do not handle + # redirects properly when a path contains "//" diff --git a/third_party/stb/BUILD.bazel b/third_party/stb/BUILD.bazel new file mode 100644 index 00000000..f91653e8 --- /dev/null +++ b/third_party/stb/BUILD.bazel @@ -0,0 +1 @@ +exports_files(["stb.BUILD"]) diff --git a/third_party/stb/repo.bzl b/third_party/stb/repo.bzl new file mode 100644 index 00000000..44645c46 --- /dev/null +++ b/third_party/stb/repo.bzl @@ -0,0 +1,9 @@ +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") + +def repo(): + git_repository( + name = "stb", + remote = "https://github.com/nothings/stb.git", + build_file = "//third_party/stb:stb.BUILD", + commit = "f1c79c02822848a9bed4315b12c8c8f3761e1296", + ) diff --git a/third_party/stb/stb.BUILD b/third_party/stb/stb.BUILD new file mode 100644 index 00000000..b59ef43f --- /dev/null +++ b/third_party/stb/stb.BUILD @@ -0,0 +1,35 @@ +cc_import( + name = "stb_image_write", + hdrs = ["stb_image_write.h"], + includes = ["."], + visibility = ["//visibility:public"], +) + +genrule( + name = "impl", + outs = ["impl.c"], + cmd = """\ +cat < $(@) +#include "stb_image.h" +#include "stb_image_resize2.h" +#include "stb_image_write.h" +EOF +""", +) + +cc_library( + name = "stb", + local_defines = [ + "STB_IMAGE_IMPLEMENTATION", + "STB_IMAGE_RESIZE_IMPLEMENTATION", + "STB_IMAGE_WRITE_IMPLEMENTATION", + ], + srcs = [":impl"], + hdrs = [ + "stb_image.h", + "stb_image_resize2.h", + "stb_image_write.h", + ], + includes = ["."], + visibility = ["//visibility:public"], +) diff --git a/third_party/translate-c/BUILD.bazel b/third_party/translate-c/BUILD.bazel new file mode 100644 index 00000000..e69de29b diff --git a/third_party/translate-c/repo.bzl b/third_party/translate-c/repo.bzl new file mode 100644 index 00000000..dfd8835d --- /dev/null +++ b/third_party/translate-c/repo.bzl @@ -0,0 +1,11 @@ +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") + +def repo(): + git_repository( + name = "translate-c", + # Use the ZML GitHub mirror because Codeberg is unreliable. + remote = "https://github.com/zml/translate-c", + # 2.0.0 + commit = "0da7a16c3235b935b82421646076e0657cda21f6", + build_file = Label("//third_party/translate-c:translate-c.bazel"), + ) diff --git a/third_party/translate-c/translate-c.bazel b/third_party/translate-c/translate-c.bazel new file mode 100644 index 00000000..7de2a5eb --- /dev/null +++ b/third_party/translate-c/translate-c.bazel @@ -0,0 +1,49 @@ +load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_configure_binary", "zig_library") +load("@zml//bazel:zig_options.bzl", "zig_option", "zig_options") + +zig_option(name = "debug_allocations", bool = False) + +zig_options( + name = "build_options", + settings = [":debug_allocations"], +) + +zig_library( + name = "c_builtins", + main = "lib/c_builtins.zig", + visibility = ["//visibility:public"], +) + +zig_library( + name = "helpers", + main = "lib/helpers.zig", + visibility = ["//visibility:public"], +) + +zig_binary( + name = "translate-c_raw", + data = glob(["lib/**/*.zig"]), + srcs = glob(["src/**/*.zig"]), + main = "src/main.zig", + linkopts = ["-lc"], + deps = [ + ":build_options", + ":c_builtins", + ":helpers", + "@arocc//:aro", + ], + visibility = ["//visibility:public"], +) + +# TODO(cerisier): Generate 1 translate-c toolchain per target platform +# So that the choice of doing this for a given platform is given to users. +# +# Right now we do this because the arocc source code has the same support +# for platforms than the official zig compiler and in the case of using a +# custom zig version with support for newer chips, arocc cannot be built. +zig_configure_binary( + name = "translate-c", + actual = ":translate-c_raw", + bootstrapped = 0, + visibility = ["//visibility:public"], +) diff --git a/third_party/uucode/BUILD.bazel b/third_party/uucode/BUILD.bazel new file mode 100644 index 00000000..49e3a399 --- /dev/null +++ b/third_party/uucode/BUILD.bazel @@ -0,0 +1,4 @@ +exports_files([ + "build_config.zig", + "uucode.bazel", +]) diff --git a/third_party/uucode/build_config.zig b/third_party/uucode/build_config.zig new file mode 100644 index 00000000..a4e76bff --- /dev/null +++ b/third_party/uucode/build_config.zig @@ -0,0 +1,16 @@ +const config = @import("config.zig"); + +pub const fields = config.fields; +pub const build_components = config.build_components; +pub const get_components = config.get_components; + +pub const tables: []const config.Table = &.{ + .{ + .fields = &.{ + "east_asian_width", + "grapheme_break", + "general_category", + "is_emoji_presentation", + }, + }, +}; diff --git a/third_party/uucode/config-storage-module.patch b/third_party/uucode/config-storage-module.patch new file mode 100644 index 00000000..d390986c --- /dev/null +++ b/third_party/uucode/config-storage-module.patch @@ -0,0 +1,33 @@ +diff --git a/src/config.zig b/src/config.zig +index 2cadab4..aa46c5d 100644 +--- a/src/config.zig ++++ b/src/config.zig +@@ -1,5 +1,5 @@ + const std = @import("std"); +-const storage = @import("storage.zig"); ++pub const storage = @import("storage.zig"); + const multi_slice = @import("multi_slice.zig"); + pub const quirks = @import("quirks.zig"); + pub const components = @import("components.zig"); +diff --git a/src/generate.zig b/src/generate.zig +index 6f4dcf5..8ef1d83 100644 +--- a/src/generate.zig ++++ b/src/generate.zig +@@ -1,7 +1,7 @@ + const std = @import("std"); +-const storage = @import("storage.zig"); + const config = @import("config.zig"); + const build_config = @import("build_config"); ++const storage = config.storage; + const inlineAssert = config.quirks.inlineAssert; + + pub const std_options: std.Options = .{ +@@ -339,7 +339,7 @@ pub fn main(init: std.process.Init.Minimal) !void { + \\const std = @import("std"); + \\const config = @import("config.zig"); + \\const types = config.types; +- \\const storage = @import("storage.zig"); ++ \\const storage = config.storage; + \\const build_config = @import("build_config"); + \\ + \\pub const get_components = build_config.get_components; diff --git a/third_party/uucode/repo.bzl b/third_party/uucode/repo.bzl new file mode 100644 index 00000000..5f7b1cbf --- /dev/null +++ b/third_party/uucode/repo.bzl @@ -0,0 +1,29 @@ +def _uucode_repo_impl(ctx): + ctx.download_and_extract( + url = "https://github.com/jacobsandlund/uucode/archive/{}.tar.gz".format(ctx.attr.commit), + stripPrefix = "uucode-{}".format(ctx.attr.commit), + ) + ctx.patch(ctx.attr.config_storage_patch, strip = 1) + + ctx.symlink(ctx.attr.build_config, "build_config.zig") + ctx.symlink(ctx.attr.build_file, "BUILD.bazel") + +_uucode_repo = repository_rule( + implementation = _uucode_repo_impl, + attrs = { + "commit": attr.string(mandatory = True), + "build_config": attr.label(mandatory = True, allow_single_file = True), + "build_file": attr.label(mandatory = True, allow_single_file = True), + "config_storage_patch": attr.label(mandatory = True, allow_single_file = True), + }, +) + +def repo(): + _uucode_repo( + name = "uucode", + # main branch + commit = "1fb73433bba5d93366c57f23ff2e9d7939746500", + build_config = "//third_party/uucode:build_config.zig", + build_file = "//third_party/uucode:uucode.bazel", + config_storage_patch = "//third_party/uucode:config-storage-module.patch", + ) diff --git a/third_party/uucode/uucode.bazel b/third_party/uucode/uucode.bazel new file mode 100644 index 00000000..4a3710f2 --- /dev/null +++ b/third_party/uucode/uucode.bazel @@ -0,0 +1,91 @@ +load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library") + +# Generator + +zig_library( + name = "_config_gen", + import_name = "config.zig", + main = "src/config.zig", + srcs = [ + "src/components.zig", + "src/ucd_inputs.zig", + "src/fields.zig", + "src/multi_slice.zig", + "src/quirks.zig", + "src/storage.zig", + "src/types.zig", + ], +) + +zig_library( + name = "_build_config_gen", + import_name = "build_config", + main = "build_config.zig", + deps = [":_config_gen"], +) + +zig_binary( + name = "gen_tables", + main = "src/generate.zig", + deps = [":_build_config_gen", ":_config_gen"], +) + +genrule( + name = "tables", + srcs = glob(["ucd/**"]), + outs = ["tables.zig"], + cmd = "OUT=$$PWD/$@ && TOOL=$$PWD/$(execpath :gen_tables) && cd $$(dirname $(location ucd/UnicodeData.txt))/.. && $$TOOL $$OUT", + tools = [":gen_tables"], +) + +# Library + +zig_library( + name = "types", + import_name = "types.zig", + main = "src/types.zig", +) + +zig_library( + name = "config", + import_name = "config.zig", + main = "src/config.zig", + srcs = [ + "src/components.zig", + "src/ucd_inputs.zig", + "src/fields.zig", + "src/multi_slice.zig", + "src/quirks.zig", + "src/storage.zig", + ], + deps = [":types"], +) + +zig_library( + name = "build_config", + import_name = "build_config", + main = "build_config.zig", + deps = [":config"], +) + +zig_library( + name = "tables_lib", + import_name = "tables", + main = ":tables", + deps = [":build_config", ":config"], +) + +zig_library( + name = "uucode", + import_name = "uucode", + main = "src/root.zig", + srcs = [ + "src/ascii.zig", + "src/code_point.zig", + "src/get.zig", + "src/grapheme.zig", + "src/utf8.zig", + ], + deps = [":config", ":tables_lib", ":types"], + visibility = ["//visibility:public"], +) diff --git a/third_party/xla/BUILD.bazel b/third_party/xla/BUILD.bazel new file mode 100644 index 00000000..75f731ff --- /dev/null +++ b/third_party/xla/BUILD.bazel @@ -0,0 +1 @@ +# Package marker for XLA patches diff --git a/third_party/xla/cuda-root-path-local-defines.patch b/third_party/xla/cuda-root-path-local-defines.patch new file mode 100644 index 00000000..c9ca6aa8 --- /dev/null +++ b/third_party/xla/cuda-root-path-local-defines.patch @@ -0,0 +1,25 @@ +diff --git a/xla/tsl/platform/default/BUILD b/xla/tsl/platform/default/BUILD +index f01bdfc67e..18a30d44a 100644 +--- a/xla/tsl/platform/default/BUILD ++++ b/xla/tsl/platform/default/BUILD +@@ -82,13 +82,13 @@ cc_library( + if_false = ["@cuda_nvcc//:nvvm"], + if_true = ["@cuda_nvvm//:nvvm"], + )), +- local_defines = [ +- "CUDA_NVCC_REPO_NAME=\\\"" + get_canonical_repo_name("cuda_nvcc") + "\\\"", +- "CUDA_NVDISASM_REPO_NAME=\\\"" + get_canonical_repo_name("cuda_nvdisasm") + "\\\"", +- "NVIDIA_NVSHMEM_REPO_NAME=\\\"" + get_canonical_repo_name("nvidia_nvshmem") + "\\\"", +- "CUDA_NVVM_REPO_NAME=\\\"" + get_canonical_repo_name("cuda_nvvm") + "\\\"", +- "CUDA_CUDART_REPO_NAME=\\\"" + get_canonical_repo_name("cuda_cudart") + "\\\"", +- ], ++ # local_defines = [ ++ # "CUDA_NVCC_REPO_NAME=\\\"" + get_canonical_repo_name("cuda_nvcc") + "\\\"", ++ # "CUDA_NVDISASM_REPO_NAME=\\\"" + get_canonical_repo_name("cuda_nvdisasm") + "\\\"", ++ # "NVIDIA_NVSHMEM_REPO_NAME=\\\"" + get_canonical_repo_name("nvidia_nvshmem") + "\\\"", ++ # "CUDA_NVVM_REPO_NAME=\\\"" + get_canonical_repo_name("cuda_nvvm") + "\\\"", ++ # "CUDA_CUDART_REPO_NAME=\\\"" + get_canonical_repo_name("cuda_cudart") + "\\\"", ++ # ], + tags = [ + "manual", + "no_oss", diff --git a/third_party/modules/llvm-raw/20240823.0-f142f8a/overlay/utils/bazel/extension.bzl b/third_party/xla/llvm.bzl similarity index 77% rename from third_party/modules/llvm-raw/20240823.0-f142f8a/overlay/utils/bazel/extension.bzl rename to third_party/xla/llvm.bzl index f247ed4b..b4a2fe4f 100644 --- a/third_party/modules/llvm-raw/20240823.0-f142f8a/overlay/utils/bazel/extension.bzl +++ b/third_party/xla/llvm.bzl @@ -1,4 +1,4 @@ -load("//utils/bazel:configure.bzl", _llvm_configure = "llvm_configure") +load("@llvm-raw//utils/bazel:configure.bzl", _llvm_configure = "llvm_configure") def _llvm_impl(mctx): _targets = {} @@ -21,7 +21,9 @@ llvm = module_extension( tag_classes = { "configure": tag_class( attrs = { - "targets": attr.string_list(mandatory = True), + "targets": attr.string_list( + default = [], + ), }, ), }, diff --git a/third_party/xla/llvm_raw_only.bzl b/third_party/xla/llvm_raw_only.bzl new file mode 100644 index 00000000..f2758994 --- /dev/null +++ b/third_party/xla/llvm_raw_only.bzl @@ -0,0 +1,13 @@ +load("@xla//third_party/llvm:workspace.bzl", llvm = "repo") + +def _llvm_raw_only_impl(mctx): + llvm("llvm-raw") + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = ["llvm-raw"], + root_module_direct_dev_deps = [], + ) + +llvm_raw_only = module_extension( + implementation = _llvm_raw_only_impl, +) diff --git a/third_party/xla/repo.bzl b/third_party/xla/repo.bzl new file mode 100644 index 00000000..0e1d9620 --- /dev/null +++ b/third_party/xla/repo.bzl @@ -0,0 +1,12 @@ +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") + +def repo(): + git_repository( + name = "xla", + remote = "https://github.com/openxla/xla.git", + commit = "65aed3e0ed7b0f965825676dd27e312965c846c8", + patches = [ + "//third_party/xla:cuda-root-path-local-defines.patch", + ], + patch_args = ["-p1"], + ) diff --git a/third_party/xla/xla.bzl b/third_party/xla/xla.bzl new file mode 100644 index 00000000..6634fa15 --- /dev/null +++ b/third_party/xla/xla.bzl @@ -0,0 +1,232 @@ +load("@xla//third_party:repo.bzl", "tf_http_archive", "tf_mirror_urls", "tf_vendored") +load("@xla//third_party/eigen3:workspace.bzl", eigen3 = "repo") +load("@xla//third_party/farmhash:workspace.bzl", farmhash = "repo") +load("@xla//third_party/highwayhash:workspace.bzl", highwayhash = "repo") +load("@xla//third_party/hwloc:workspace.bzl", hwloc = "repo") +load("@xla//third_party/llvm:workspace.bzl", llvm = "repo") +load("@xla//third_party/py/ml_dtypes:workspace.bzl", ml_dtypes = "repo") +load("@xla//third_party/shardy:workspace.bzl", shardy = "repo") +load("@xla//third_party/stablehlo:workspace.bzl", stablehlo = "repo") +load("@xla//third_party/triton:workspace.bzl", triton = "repo") + +_BZL_HELPERS = """\ +always_newer_than = lambda wanted_ver, if_true, if_false = []: if_true +always_if_true = lambda if_true, if_false = []: if_true +always_if_false = lambda if_true, if_false = []: if_false +always_false = lambda *args, **kwargs: False +""" + +def _simple_files_impl(rctx): + rctx.file("BUILD.bazel", "") + for f, content in rctx.attr.files.items(): + rctx.file(f, content) + +simple_files = repository_rule( + implementation = _simple_files_impl, + attrs = { + "files": attr.string_dict(), + }, +) + +def _dummy_repos(mctx): + simple_files(name = "local_config_cuda", files = { + "BUILD.bazel": """\ +package(default_visibility = ["//visibility:public"]) + +config_setting( + name = "is_cuda_enabled", + define_values = {"local_config_cuda_enabled": "true"}, +) + +config_setting( + name = "is_cuda_compiler_nvcc", + define_values = {"local_config_cuda_compiler_nvcc": "true"}, +) +""", + "cuda/BUILD.bazel": """\ +package(default_visibility = ["//visibility:public"]) + +exports_files(["build_defs.bzl"]) + +filegroup( + name = "build_defs_bzl", + srcs = ["build_defs.bzl"], +) + +config_setting( + name = "using_config_cuda", + define_values = {"local_config_cuda_using_config": "true"}, +) + +config_setting( + name = "FALSE", + define_values = {"local_config_cuda_false": "true"}, +) +""", + "cuda/build_defs.bzl": _BZL_HELPERS + """\ +cuda_library = always_false +if_cuda = always_if_false +if_cuda_is_configured = always_if_false +if_cuda_newer_than = always_newer_than +is_cuda_configured = always_false +""", + }) + simple_files(name = "local_config_rocm", files = { + "BUILD.bazel": "", + "rocm/BUILD.bazel": """\ +package(default_visibility = ["//visibility:public"]) + +exports_files(["build_defs.bzl"]) + +filegroup( + name = "build_defs_bzl", + srcs = ["build_defs.bzl"], +) + +config_setting( + name = "using_hipcc", + define_values = {"local_config_rocm_using_hipcc": "true"}, +) +""", + "rocm/build_defs.bzl": _BZL_HELPERS + """\ +if_rocm = always_if_false +if_rocm_is_configured = always_if_false +if_rocm_newer_than = always_newer_than +is_rocm_configured = always_false +if_gpu_is_configured = always_if_false +if_cuda_or_rocm = always_if_false +get_rbe_amdgpu_pool = always_false +""", + }) + simple_files(name = "local_config_sycl", files = { + "BUILD.bazel": "", + "sycl/BUILD.bazel": """\ +package(default_visibility = ["//visibility:public"]) + +exports_files(["build_defs.bzl"]) + +filegroup( + name = "build_defs_bzl", + srcs = ["build_defs.bzl"], +) +""", + "crosstool/BUILD.bazel": "", + "sycl/build_defs.bzl": """\ +load("@rules_cc//cc:cc_library.bzl", "cc_library") + +""" + _BZL_HELPERS + """\ +if_sycl = always_if_false +if_sycl_is_configured = always_if_false +sycl_library = cc_library +""", + }) + simple_files(name = "local_config_remote_execution", files = { + "remote_execution.bzl": """gpu_test_tags = lambda: []""", + }) + simple_files(name = "local_config_tensorrt", files = { + "BUILD.bazel": """\ +package(default_visibility = ["//visibility:public"]) + +exports_files(["build_defs.bzl"]) + +filegroup( + name = "build_defs_bzl", + srcs = ["build_defs.bzl"], +) +""", + "build_defs.bzl": _BZL_HELPERS + """if_tensorrt = always_if_false""", + }) + simple_files(name = "python_version_repo", files = { + "py_version.bzl": """USE_PYWRAP_RULES = False""", + }) + simple_files(name = "rules_ml_toolchain", files = { + "py/rules_pywrap/BUILD.bazel": """\ +package(default_visibility = ["//visibility:public"]) + +exports_files([ + "pywrap.default.bzl", + "pywrap.impl.bzl", +]) + +filegroup( + name = "pywrap_bzl", + srcs = [ + "pywrap.default.bzl", + "pywrap.impl.bzl", + ], +) +""", + "py/rules_pywrap/pywrap.default.bzl": """\ +def use_pywrap_rules(): + return False + +def _unsupported(*args, **kwargs): + fail("rules_ml_toolchain pywrap rules are not available in this workspace") + +pybind_extension = _unsupported +pywrap_aware_cc_import = _unsupported +pywrap_aware_filegroup = _unsupported +pywrap_aware_genrule = _unsupported +pywrap_binaries = _unsupported +pywrap_common_library = _unsupported +pywrap_library = _unsupported +stripped_cc_info = _unsupported +""", + "py/rules_pywrap/pywrap.impl.bzl": """\ +def _unsupported(*args, **kwargs): + fail("rules_ml_toolchain pywrap rules are not available in this workspace") + +collected_pywrap_infos = _unsupported +generated_common_win_def_file = _unsupported +pybind_extension = _unsupported +python_extension = _unsupported +pywrap_binaries = _unsupported +pywrap_common_library = _unsupported +pywrap_library = _unsupported +stripped_cc_info = _unsupported +""", + "third_party/gpus/BUILD.bazel": "", + "third_party/gpus/nvidia_common_rules.bzl": """cuda_rpath_flags = lambda *args, **kwargs: []""", + "third_party/extensions/sycl_configure.bzl": "", + }) + simple_files(name = "sycl_configure_ext", files = {}) + simple_files(name = "sycl_configure", files = {}) + +def _xla_impl(mctx): + llvm("llvm-raw") + stablehlo() + shardy() + triton() + farmhash() + highwayhash() + hwloc() + eigen3() + ml_dtypes() + + tf_http_archive( + name = "com_github_grpc_grpc", + sha256 = "41b695614b26652ff9e97ce50cfd4a6c7a3d45a9fe598d1454407746499bbf2c", + strip_prefix = "grpc-1.81.0", + patch_file = ["//third_party/grpc:grpc.patch"], + urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/refs/tags/v1.81.0.tar.gz"), + ) + tf_vendored(name = "tsl", path = "third_party/tsl") + + tf_http_archive( + name = "snappy", + build_file = "//third_party:snappy.BUILD", + sha256 = "90f74bc1fbf78a6c56b3c4a082a05103b3a56bb17bca1a27e052ea11723292dc", + strip_prefix = "snappy-1.2.2", + urls = tf_mirror_urls("https://github.com/google/snappy/archive/refs/tags/1.2.2.tar.gz"), + ) + _dummy_repos(mctx) + + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = "all", + root_module_direct_dev_deps = [], + ) + +xla = module_extension( + implementation = _xla_impl, +) diff --git a/third_party/zigimg/BUILD.bazel b/third_party/zigimg/BUILD.bazel new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/third_party/zigimg/BUILD.bazel @@ -0,0 +1 @@ + diff --git a/third_party/zigimg/repo.bzl b/third_party/zigimg/repo.bzl new file mode 100644 index 00000000..a4bb178f --- /dev/null +++ b/third_party/zigimg/repo.bzl @@ -0,0 +1,10 @@ +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") + +def repo(): + git_repository( + name = "zigimg", + remote = "https://github.com/zigimg/zigimg.git", + # Match libvaxis build.zig.zon. + commit = "734fa919fb15b7917d4eb124f53600348f4ffa30", + build_file = Label("//third_party/zigimg:zigimg.bazel"), + ) diff --git a/third_party/zigimg/zigimg.bazel b/third_party/zigimg/zigimg.bazel new file mode 100644 index 00000000..f02d9380 --- /dev/null +++ b/third_party/zigimg/zigimg.bazel @@ -0,0 +1,18 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "zigimg", + srcs = glob([ + "src/*.zig", + "src/compressions/*.zig", + "src/compressions/deflate/*.zig", + "src/formats/*.zig", + "src/formats/jpeg/*.zig", + "src/formats/png/*.zig", + "src/formats/tiff/*.zig", + "src/Image/*.zig", + "tests/**/*.zig", + ]), + main = "zigimg.zig", + visibility = ["//visibility:public"], +) diff --git a/third_party/zio/BUILD.bazel b/third_party/zio/BUILD.bazel new file mode 100644 index 00000000..e69de29b diff --git a/third_party/zio/repo.bzl b/third_party/zio/repo.bzl new file mode 100644 index 00000000..96133ef0 --- /dev/null +++ b/third_party/zio/repo.bzl @@ -0,0 +1,9 @@ +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") + +def repo(): + git_repository( + name = "zio", + remote = "https://github.com/lalinsky/zio", + commit = "1b4e9787f8a4a67d91372a603b23258364f89bf0", + build_file = Label("//third_party/zio:zio.bazel"), + ) diff --git a/third_party/zio/zio.bazel b/third_party/zio/zio.bazel new file mode 100644 index 00000000..39805936 --- /dev/null +++ b/third_party/zio/zio.bazel @@ -0,0 +1,46 @@ +load("@bazel_skylib//rules:common_settings.bzl", "string_flag") +load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library") +load("@zml//bazel:zig_options.bzl", "zig_option", "zig_options") + +string_flag( + name = "backend", + build_setting_default = "", + values = ["", "poll", "linux", "epoll", "kqueue", "io_uring", "iocp"], +) + +string_flag( + name = "scheduling", + build_setting_default = "work_stealing", + values = ["single_executor", "pinned", "work_stealing"], +) + +string_flag( + name = "resolve_beneath_mode", + build_setting_default = "strict", + values = ["strict", "best_effort"], +) + +zig_option(name = "backend_option", flag = ":backend", nullable = True) +zig_option(name = "scheduling_option", flag = ":scheduling") +zig_option(name = "resolve_beneath_mode_option", flag = ":resolve_beneath_mode") +zig_option(name = "no_hacks", bool = False) +zig_option(name = "scheduler_metrics", bool = False) + +zig_options( + name = "zio_build_options", + settings = [ + ":backend_option", + ":scheduling_option", + ":resolve_beneath_mode_option", + ":no_hacks", + ":scheduler_metrics", + ], +) + +zig_library( + name = "zio", + main = "src/zio.zig", + srcs = glob(["src/**/*.zig"]), + deps = [":zio_build_options"], + visibility = ["//visibility:public"], +) diff --git a/third_party/zls/BUILD.bazel b/third_party/zls/BUILD.bazel index 223284e4..69cc6128 100644 --- a/third_party/zls/BUILD.bazel +++ b/third_party/zls/BUILD.bazel @@ -1,14 +1,36 @@ -load(":zls.bzl", "targets", "zig_runner", "zls_runner") +load("@rules_zig//zig:defs.bzl", "zig_library") +load("@rules_zig//zig/zls:toolchain.bzl", "zls_toolchain") -toolchain_type(name = "toolchain_type") +exports_files(glob(["*.bazel"])) -targets() - -zig_runner( - name = "zig", +zig_library( + name = "build_options", + main = "build_options.zig", + visibility = ["@zls//:__pkg__"], ) -zls_runner( +zig_library( + name = "exe_options", + main = "exe_options.zig", + visibility = ["@zls//:__pkg__"], +) + +zig_library( + name = "tracy_options", + import_name = "options", + main = "tracy_options.zig", + visibility = ["@zls//:__pkg__"], +) + +zls_toolchain( name = "zls", - zig = ":zig", + bin = "@zls//:zls", + zls_version = "0.17.0", +) + +toolchain( + name = "toolchain", + target_settings = ["@zig_toolchains//:any_0.17.release"], + toolchain = ":zls", + toolchain_type = "@rules_zig//zig/zls:toolchain_type", ) diff --git a/third_party/zls/build_options.zig b/third_party/zls/build_options.zig new file mode 100644 index 00000000..b50cb2d8 --- /dev/null +++ b/third_party/zls/build_options.zig @@ -0,0 +1,5 @@ +const std = @import("std"); + +pub const version_string = "0.17.0-dev+eab2be0fd744"; +pub const version = std.SemanticVersion.parse(version_string) catch unreachable; +pub const minimum_runtime_zig_version_string = "0.17.0"; diff --git a/third_party/zls/diffz.bazel b/third_party/zls/diffz.bazel new file mode 100644 index 00000000..fb4b85fa --- /dev/null +++ b/third_party/zls/diffz.bazel @@ -0,0 +1,8 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "diffz", + srcs = glob(["**/*.zig"]), + main = "DiffMatchPatch.zig", + visibility = ["@zls//:__pkg__"], +) diff --git a/third_party/zls/exe_options.zig b/third_party/zls/exe_options.zig new file mode 100644 index 00000000..eaad61b5 --- /dev/null +++ b/third_party/zls/exe_options.zig @@ -0,0 +1,3 @@ +pub const enable_failing_allocator = false; +pub const enable_failing_allocator_likelihood: u32 = 256; +pub const debug_gpa = false; diff --git a/third_party/zls/known_folders.bazel b/third_party/zls/known_folders.bazel new file mode 100644 index 00000000..234db4b0 --- /dev/null +++ b/third_party/zls/known_folders.bazel @@ -0,0 +1,8 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "known-folders", + srcs = glob(["**/*.zig"]), + main = "known-folders.zig", + visibility = ["@zls//:__pkg__"], +) diff --git a/third_party/zls/lsp.bazel b/third_party/zls/lsp.bazel new file mode 100644 index 00000000..b7c6b6dd --- /dev/null +++ b/third_party/zls/lsp.bazel @@ -0,0 +1,51 @@ +load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library") + +# @embedFile resolves a module's root file without parsing it as Zig. +genrule( + name = "meta_model_source", + srcs = ["metaModel.json"], + outs = ["meta_model.zig"], + cmd = "cp $(<) $(@)", +) + +zig_library( + name = "meta-model", + main = ":meta_model.zig", +) + +zig_binary( + name = "codegen", + srcs = glob(["src/codegen/*.zig"]), + extra_srcs = ["src/codegen/config.zon"], + main = "src/codegen/codegen.zig", + deps = [":meta-model"], +) + +genrule( + name = "generate_types", + outs = ["lsp_types.zig"], + cmd = "$(location :codegen) $(@)", + tools = [":codegen"], +) + +zig_library( + name = "parser", + main = "src/parser.zig", +) + +zig_library( + name = "types", + main = ":lsp_types.zig", + deps = [":parser"], +) + +zig_library( + name = "lsp", + srcs = glob(["src/*.zig"]), + main = "src/lsp.zig", + visibility = ["@zls//:__pkg__"], + deps = [ + ":parser", + ":types", + ], +) diff --git a/third_party/zls/repo.bzl b/third_party/zls/repo.bzl new file mode 100644 index 00000000..27f05c5c --- /dev/null +++ b/third_party/zls/repo.bzl @@ -0,0 +1,28 @@ +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") + +def repo(): + # ZLS 0.17 development revision; dependencies match its build.zig.zon. + git_repository( + name = "zls", + remote = "https://github.com/zigtools/zls.git", + commit = "eab2be0fd74443a27662da809b771c4ad0d2afcf", + build_file = Label("//third_party/zls:zls.bazel"), + ) + git_repository( + name = "zls_lsp", + remote = "https://github.com/zigtools/lsp-kit.git", + commit = "d148676ace1eaecf3c703ce7d350d0e94e502ed7", + build_file = Label("//third_party/zls:lsp.bazel"), + ) + git_repository( + name = "zls_diffz", + remote = "https://github.com/ziglibs/diffz.git", + commit = "aac8aa99c436ab8277b0711922aad062c0167b12", + build_file = Label("//third_party/zls:diffz.bazel"), + ) + git_repository( + name = "zls_known_folders", + remote = "https://github.com/ziglibs/known-folders.git", + commit = "6da0d0c41b78b9ed2d34fa364fcb81b5ebece6c4", + build_file = Label("//third_party/zls:known_folders.bazel"), + ) diff --git a/third_party/zls/tracy_options.zig b/third_party/zls/tracy_options.zig new file mode 100644 index 00000000..01dfcb1d --- /dev/null +++ b/third_party/zls/tracy_options.zig @@ -0,0 +1,3 @@ +pub const enable = false; +pub const enable_allocation = false; +pub const enable_callstack = false; diff --git a/third_party/zls/zls.bazel b/third_party/zls/zls.bazel new file mode 100644 index 00000000..56ccbff8 --- /dev/null +++ b/third_party/zls/zls.bazel @@ -0,0 +1,53 @@ +load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library") + +zig_binary( + name = "config_gen", + extra_srcs = ["src/tools/config.json"], + main = "src/tools/config_gen.zig", +) + +genrule( + name = "generate_version_data", + srcs = ["src/tools/langref.html.in"], + outs = ["version_data.zig"], + cmd = "$(location :config_gen) --langref-version master --langref-path $(<) --generate-version-data $(@)", + tools = [":config_gen"], +) + +zig_library( + name = "version_data", + main = ":version_data.zig", +) + +zig_library( + name = "tracy", + main = "src/tracy.zig", + deps = ["@zml//third_party/zls:tracy_options"], +) + +zig_library( + name = "zls_lib", + srcs = glob(["src/**/*.zig"]), + import_name = "zls", + main = "src/zls.zig", + deps = [ + ":tracy", + ":version_data", + "@zls_diffz//:diffz", + "@zls_lsp//:lsp", + "@zml//third_party/zls:build_options", + ], +) + +zig_binary( + name = "zls", + srcs = glob(["src/**/*.zig"]), + main = "src/main.zig", + visibility = ["//visibility:public"], + deps = [ + ":tracy", + ":zls_lib", + "@zls_known_folders//:known-folders", + "@zml//third_party/zls:exe_options", + ], +) diff --git a/third_party/zls/zls.bzl b/third_party/zls/zls.bzl deleted file mode 100644 index 816c1d08..00000000 --- a/third_party/zls/zls.bzl +++ /dev/null @@ -1,238 +0,0 @@ -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") - -_VERSION = "0.13.0" - -_ARCH = { - "x86_64-linux": struct( - sha256 = "ec4c1b45caf88e2bcb9ebb16c670603cc596e4f621b96184dfbe837b39cd8410", - exec_compatible_with = [ - "@platforms//os:linux", - "@platforms//cpu:x86_64", - ], - ), - "aarch64-macos": struct( - sha256 = "9848514524f5e5d33997ac280b7d92388407209d4b8d4be3866dc3cf30ca6ca8", - exec_compatible_with = [ - "@platforms//os:macos", - "@platforms//cpu:aarch64", - ], - ), -} - -ZlsInfo = provider( - fields = { - "bin": "ZLS binary", - }, -) - -def _zls_toolchain_impl(ctx): - default = DefaultInfo( - files = depset(direct = [ctx.file.zls]), - ) - zlsinfo = ZlsInfo( - bin = ctx.file.zls, - ) - toolchain_info = platform_common.ToolchainInfo( - default = default, - zlsinfo = zlsinfo, - ) - - return [ - default, - zlsinfo, - toolchain_info, - ] - -zls_toolchain = rule( - implementation = _zls_toolchain_impl, - attrs = { - "zls": attr.label( - executable = True, - allow_single_file = True, - cfg = "exec", - ), - }, -) - -def _repo_impl(mctx): - for arch, config in _ARCH.items(): - http_archive( - name = "zls_{}".format(arch), - url = "https://github.com/zigtools/zls/releases/download/{version}/zls-{arch}.tar.xz".format( - version = _VERSION, - arch = arch, - ), - sha256 = config.sha256, - build_file_content = """\ -load("@zml//third_party/zls:zls.bzl", "zls_toolchain") -zls_toolchain(name = "toolchain", zls = "zls") -""", - ) - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = "all", - root_module_direct_dev_deps = [], - ) - -repo = module_extension( - implementation = _repo_impl, -) - -def targets(): - for arch, config in _ARCH.items(): - native.toolchain( - name = "toolchain_{}".format(arch), - exec_compatible_with = config.exec_compatible_with, - target_compatible_with = config.exec_compatible_with, - toolchain = "@zls_{}//:toolchain".format(arch), - toolchain_type = "@zml//third_party/zls:toolchain_type", - ) - -_ZIG_RUNNER_TPL = """\ -#!/bin/bash - -if [[ "${{1}}" == "build" ]]; then - for arg in "${{@:2}}"; do - if [[ "${{arg}}" == "-Dcmd="* ]]; then - cd ${{BUILD_WORKSPACE_DIRECTORY}} - exec ${{arg/-Dcmd=/}} - fi - done -fi - -export ZIG_GLOBAL_CACHE_DIR="$(realpath {zig_cache})" -export ZIG_LOCAL_CACHE_DIR="$(realpath {zig_cache})" -export ZIG_LIB_DIR="$(realpath {zig_lib_path})" -exec {zig_exe_path} "${{@}}" -""" - -_RUNNER_TPL = """\ -#!/bin/bash -set -eo pipefail - -zig() {{ - if [[ "${{1}}" == "build" ]]; then - for arg in "${{@:2}}"; do - if [[ "${{arg}}" == "-Dcmd="* ]]; then - cd ${{BUILD_WORKSPACE_DIRECTORY}} - exec ${{arg/-Dcmd=/}} - fi - done - fi - - export ZIG_GLOBAL_CACHE_DIR="$(realpath {zig_cache})" - export ZIG_LOCAL_CACHE_DIR="$(realpath {zig_cache})" - export ZIG_LIB_DIR="$(realpath {zig_lib_path})" - exec {zig_exe_path} "${{@}}" -}} - -zls() {{ - json_config="$(mktemp)" - ZLS_ARGS=("--config-path" "${{json_config}}") - - cat < ${{json_config}} -{{ - "zig_lib_path": "$(realpath {zig_lib_path})", - "zig_exe_path": "$(realpath {zig_exe_path})", - "global_cache_path": "$(realpath {zig_cache})" -}} -EOF - - while ((${{#}})); do - case "${{1}}" in - --config-path) - cat "${{2}}" >> "${{json_config}}" - {jq} -s add "${{json_config}}" > "${{json_config}}.tmp" - mv "${{json_config}}.tmp" "${{json_config}}" - shift 2 - ;; - *) - ZLS_ARGS+=("${{1}}") - shift - ;; - esac - done - - exec {zls} "${{ZLS_ARGS[@]}}" -}} - -case $1 in - zig) - shift - zig "${{@}}" - ;; - zls) - shift - zls "${{@}}" - ;; -esac -""" - -def _zls_runner_impl(ctx): - jqinfo = ctx.toolchains["@aspect_bazel_lib//lib:jq_toolchain_type"].jqinfo - zigtoolchaininfo = ctx.toolchains["@rules_zig//zig:toolchain_type"].zigtoolchaininfo - zlsinfo = ctx.toolchains["@zml//third_party/zls:toolchain_type"].zlsinfo - - zls_runner = ctx.actions.declare_file(ctx.label.name + ".zls_runner.sh") - ctx.actions.write(zls_runner, _RUNNER_TPL.format( - jq = jqinfo.bin.short_path, - zig_cache = zigtoolchaininfo.zig_cache, - zig_exe_path = zigtoolchaininfo.zig_exe.short_path, - zig_lib_path = zigtoolchaininfo.zig_lib.short_path, - zls = zlsinfo.bin.short_path, - )) - - return [ - DefaultInfo( - files = depset([zls_runner]), - executable = zls_runner, - runfiles = ctx.runfiles( - files = [ - ctx.executable.zig, - jqinfo.bin, - zlsinfo.bin, - ], - transitive_files = zigtoolchaininfo.zig_files, - ), - ), - ] - -zls_runner = rule( - implementation = _zls_runner_impl, - attrs = { - "zig": attr.label(mandatory = True, executable = True, cfg = "exec"), - }, - executable = True, - toolchains = [ - "@rules_zig//zig:toolchain_type", - "@aspect_bazel_lib//lib:jq_toolchain_type", - "@zml//third_party/zls:toolchain_type", - ], -) - -def _zig_runner_impl(ctx): - zigtoolchaininfo = ctx.toolchains["@rules_zig//zig:toolchain_type"].zigtoolchaininfo - - zig_runner = ctx.actions.declare_file(ctx.label.name + ".zig_runner.sh") - ctx.actions.write(zig_runner, _ZIG_RUNNER_TPL.format( - zig_cache = zigtoolchaininfo.zig_cache, - zig_exe_path = zigtoolchaininfo.zig_exe.short_path, - zig_lib_path = zigtoolchaininfo.zig_lib.short_path, - )) - - return [ - DefaultInfo( - files = depset([zig_runner]), - executable = zig_runner, - runfiles = ctx.runfiles( - files = [zig_runner], - transitive_files = zigtoolchaininfo.zig_files, - ), - ), - ] - -zig_runner = rule( - implementation = _zig_runner_impl, - executable = True, - toolchains = ["@rules_zig//zig:toolchain_type"], -) diff --git a/tools/BUILD.bazel b/tools/BUILD.bazel new file mode 100644 index 00000000..844778f5 --- /dev/null +++ b/tools/BUILD.bazel @@ -0,0 +1,26 @@ +load("@rules_cc//cc:cc_binary.bzl", "cc_binary") +load("@rules_python//python:py_library.bzl", "py_library") +load("@rules_shell//shell:sh_binary.bzl", "sh_binary") + +py_library( + name = "zml_utils", + srcs = ["zml_utils.py"], + visibility = ["//visibility:public"], +) + +sh_binary( + name = "nsys", + srcs = ["nsys.sh"], + visibility = ["//visibility:public"], +) + +sh_binary( + name = "rocprofv3", + srcs = ["rocprofv3.sh"], + data = [ + "@amdrocm-profiler-base//:rocprofv3_tool", + "@libzml_rocm//:sandbox", + ], + target_compatible_with = ["@platforms//os:linux"], + visibility = ["//visibility:public"], +) diff --git a/tools/buildifier.sh b/tools/buildifier.sh index 3e1b0930..0e2b7ad7 100755 --- a/tools/buildifier.sh +++ b/tools/buildifier.sh @@ -1,3 +1,4 @@ #!/bin/bash cd "$(dirname "${BASH_SOURCE[0]}")" +cd "$(bazel info workspace)" exec bazel run -- @buildifier_prebuilt//:buildifier "$@" diff --git a/tools/hf/BUILD.bazel b/tools/hf/BUILD.bazel new file mode 100644 index 00000000..21a5315f --- /dev/null +++ b/tools/hf/BUILD.bazel @@ -0,0 +1,16 @@ +load("@rules_python//python/entry_points:py_console_script_binary.bzl", "py_console_script_binary") +load("@rules_python//python/uv:lock.bzl", uv_lock = "lock") + +py_console_script_binary( + name = "hf", + pkg = "@huggingface_hub//huggingface_hub", + tags = ["manual"], +) + +uv_lock( + name = "requirements", + srcs = ["requirements.in"], + out = "requirements.lock.txt", + args = ["--upgrade"], + tags = ["manual"], +) diff --git a/tools/hf/requirements.in b/tools/hf/requirements.in new file mode 100644 index 00000000..96187ec3 --- /dev/null +++ b/tools/hf/requirements.in @@ -0,0 +1,2 @@ +huggingface_hub +hf_transfer diff --git a/tools/hf/requirements.lock.txt b/tools/hf/requirements.lock.txt new file mode 100644 index 00000000..9d6c0dea --- /dev/null +++ b/tools/hf/requirements.lock.txt @@ -0,0 +1,269 @@ +# This file was autogenerated by uv via the following command: +# bazel run //tools/hf:requirements.update +certifi==2025.10.5 \ + --hash=sha256:0f212c2744a9bb6de0c56639a6f68afe01ecd92d91f14ae897c4fe7bbeeef0de \ + --hash=sha256:47c09d31ccf2acf0be3f701ea53595ee7e0b8fa08801c6624be771df09ae7b43 + # via requests +charset-normalizer==3.4.4 \ + --hash=sha256:027f6de494925c0ab2a55eab46ae5129951638a49a34d87f4c3eda90f696b4ad \ + --hash=sha256:077fbb858e903c73f6c9db43374fd213b0b6a778106bc7032446a8e8b5b38b93 \ + --hash=sha256:0a98e6759f854bd25a58a73fa88833fba3b7c491169f86ce1180c948ab3fd394 \ + --hash=sha256:0d3d8f15c07f86e9ff82319b3d9ef6f4bf907608f53fe9d92b28ea9ae3d1fd89 \ + --hash=sha256:0f04b14ffe5fdc8c4933862d8306109a2c51e0704acfa35d51598eb45a1e89fc \ + --hash=sha256:11d694519d7f29d6cd09f6ac70028dba10f92f6cdd059096db198c283794ac86 \ + --hash=sha256:194f08cbb32dc406d6e1aea671a68be0823673db2832b38405deba2fb0d88f63 \ + --hash=sha256:1bee1e43c28aa63cb16e5c14e582580546b08e535299b8b6158a7c9c768a1f3d \ + --hash=sha256:21d142cc6c0ec30d2efee5068ca36c128a30b0f2c53c1c07bd78cb6bc1d3be5f \ + --hash=sha256:2437418e20515acec67d86e12bf70056a33abdacb5cb1655042f6538d6b085a8 \ + --hash=sha256:244bfb999c71b35de57821b8ea746b24e863398194a4014e4c76adc2bbdfeff0 \ + --hash=sha256:2677acec1a2f8ef614c6888b5b4ae4060cc184174a938ed4e8ef690e15d3e505 \ + --hash=sha256:277e970e750505ed74c832b4bf75dac7476262ee2a013f5574dd49075879e161 \ + --hash=sha256:2aaba3b0819274cc41757a1da876f810a3e4d7b6eb25699253a4effef9e8e4af \ + --hash=sha256:2b7d8f6c26245217bd2ad053761201e9f9680f8ce52f0fcd8d0755aeae5b2152 \ + --hash=sha256:2c9d3c380143a1fedbff95a312aa798578371eb29da42106a29019368a475318 \ + --hash=sha256:3162d5d8ce1bb98dd51af660f2121c55d0fa541b46dff7bb9b9f86ea1d87de72 \ + --hash=sha256:31fd66405eaf47bb62e8cd575dc621c56c668f27d46a61d975a249930dd5e2a4 \ + --hash=sha256:362d61fd13843997c1c446760ef36f240cf81d3ebf74ac62652aebaf7838561e \ + --hash=sha256:376bec83a63b8021bb5c8ea75e21c4ccb86e7e45ca4eb81146091b56599b80c3 \ + --hash=sha256:44c2a8734b333e0578090c4cd6b16f275e07aa6614ca8715e6c038e865e70576 \ + --hash=sha256:47cc91b2f4dd2833fddaedd2893006b0106129d4b94fdb6af1f4ce5a9965577c \ + --hash=sha256:4902828217069c3c5c71094537a8e623f5d097858ac6ca8252f7b4d10b7560f1 \ + --hash=sha256:4bd5d4137d500351a30687c2d3971758aac9a19208fc110ccb9d7188fbe709e8 \ + --hash=sha256:4fe7859a4e3e8457458e2ff592f15ccb02f3da787fcd31e0183879c3ad4692a1 \ + --hash=sha256:542d2cee80be6f80247095cc36c418f7bddd14f4a6de45af91dfad36d817bba2 \ + --hash=sha256:554af85e960429cf30784dd47447d5125aaa3b99a6f0683589dbd27e2f45da44 \ + --hash=sha256:5833d2c39d8896e4e19b689ffc198f08ea58116bee26dea51e362ecc7cd3ed26 \ + --hash=sha256:5947809c8a2417be3267efc979c47d76a079758166f7d43ef5ae8e9f92751f88 \ + --hash=sha256:5ae497466c7901d54b639cf42d5b8c1b6a4fead55215500d2f486d34db48d016 \ + --hash=sha256:5bd2293095d766545ec1a8f612559f6b40abc0eb18bb2f5d1171872d34036ede \ + --hash=sha256:5bfbb1b9acf3334612667b61bd3002196fe2a1eb4dd74d247e0f2a4d50ec9bbf \ + --hash=sha256:5cb4d72eea50c8868f5288b7f7f33ed276118325c1dfd3957089f6b519e1382a \ + --hash=sha256:5dbe56a36425d26d6cfb40ce79c314a2e4dd6211d51d6d2191c00bed34f354cc \ + --hash=sha256:5f819d5fe9234f9f82d75bdfa9aef3a3d72c4d24a6e57aeaebba32a704553aa0 \ + --hash=sha256:64b55f9dce520635f018f907ff1b0df1fdc31f2795a922fb49dd14fbcdf48c84 \ + --hash=sha256:6515f3182dbe4ea06ced2d9e8666d97b46ef4c75e326b79bb624110f122551db \ + --hash=sha256:65e2befcd84bc6f37095f5961e68a6f077bf44946771354a28ad434c2cce0ae1 \ + --hash=sha256:6aee717dcfead04c6eb1ce3bd29ac1e22663cdea57f943c87d1eab9a025438d7 \ + --hash=sha256:6b39f987ae8ccdf0d2642338faf2abb1862340facc796048b604ef14919e55ed \ + --hash=sha256:6e1fcf0720908f200cd21aa4e6750a48ff6ce4afe7ff5a79a90d5ed8a08296f8 \ + --hash=sha256:74018750915ee7ad843a774364e13a3db91682f26142baddf775342c3f5b1133 \ + --hash=sha256:74664978bb272435107de04e36db5a9735e78232b85b77d45cfb38f758efd33e \ + --hash=sha256:74bb723680f9f7a6234dcf67aea57e708ec1fbdf5699fb91dfd6f511b0a320ef \ + --hash=sha256:752944c7ffbfdd10c074dc58ec2d5a8a4cd9493b314d367c14d24c17684ddd14 \ + --hash=sha256:778d2e08eda00f4256d7f672ca9fef386071c9202f5e4607920b86d7803387f2 \ + --hash=sha256:780236ac706e66881f3b7f2f32dfe90507a09e67d1d454c762cf642e6e1586e0 \ + --hash=sha256:798d75d81754988d2565bff1b97ba5a44411867c0cf32b77a7e8f8d84796b10d \ + --hash=sha256:799a7a5e4fb2d5898c60b640fd4981d6a25f1c11790935a44ce38c54e985f828 \ + --hash=sha256:7a32c560861a02ff789ad905a2fe94e3f840803362c84fecf1851cb4cf3dc37f \ + --hash=sha256:7c308f7e26e4363d79df40ca5b2be1c6ba9f02bdbccfed5abddb7859a6ce72cf \ + --hash=sha256:7fa17817dc5625de8a027cb8b26d9fefa3ea28c8253929b8d6649e705d2835b6 \ + --hash=sha256:81d5eb2a312700f4ecaa977a8235b634ce853200e828fbadf3a9c50bab278328 \ + --hash=sha256:82004af6c302b5d3ab2cfc4cc5f29db16123b1a8417f2e25f9066f91d4411090 \ + --hash=sha256:837c2ce8c5a65a2035be9b3569c684358dfbf109fd3b6969630a87535495ceaa \ + --hash=sha256:840c25fb618a231545cbab0564a799f101b63b9901f2569faecd6b222ac72381 \ + --hash=sha256:8a6562c3700cce886c5be75ade4a5db4214fda19fede41d9792d100288d8f94c \ + --hash=sha256:8af65f14dc14a79b924524b1e7fffe304517b2bff5a58bf64f30b98bbc5079eb \ + --hash=sha256:8ef3c867360f88ac904fd3f5e1f902f13307af9052646963ee08ff4f131adafc \ + --hash=sha256:94537985111c35f28720e43603b8e7b43a6ecfb2ce1d3058bbe955b73404e21a \ + --hash=sha256:99ae2cffebb06e6c22bdc25801d7b30f503cc87dbd283479e7b606f70aff57ec \ + --hash=sha256:9a26f18905b8dd5d685d6d07b0cdf98a79f3c7a918906af7cc143ea2e164c8bc \ + --hash=sha256:9b35f4c90079ff2e2edc5b26c0c77925e5d2d255c42c74fdb70fb49b172726ac \ + --hash=sha256:9cd98cdc06614a2f768d2b7286d66805f94c48cde050acdbbb7db2600ab3197e \ + --hash=sha256:9d1bb833febdff5c8927f922386db610b49db6e0d4f4ee29601d71e7c2694313 \ + --hash=sha256:9f7fcd74d410a36883701fafa2482a6af2ff5ba96b9a620e9e0721e28ead5569 \ + --hash=sha256:a59cb51917aa591b1c4e6a43c132f0cdc3c76dbad6155df4e28ee626cc77a0a3 \ + --hash=sha256:a61900df84c667873b292c3de315a786dd8dac506704dea57bc957bd31e22c7d \ + --hash=sha256:a79cfe37875f822425b89a82333404539ae63dbdddf97f84dcbc3d339aae9525 \ + --hash=sha256:a8a8b89589086a25749f471e6a900d3f662d1d3b6e2e59dcecf787b1cc3a1894 \ + --hash=sha256:a8bf8d0f749c5757af2142fe7903a9df1d2e8aa3841559b2bad34b08d0e2bcf3 \ + --hash=sha256:a9768c477b9d7bd54bc0c86dbaebdec6f03306675526c9927c0e8a04e8f94af9 \ + --hash=sha256:ac1c4a689edcc530fc9d9aa11f5774b9e2f33f9a0c6a57864e90908f5208d30a \ + --hash=sha256:af2d8c67d8e573d6de5bc30cdb27e9b95e49115cd9baad5ddbd1a6207aaa82a9 \ + --hash=sha256:b435cba5f4f750aa6c0a0d92c541fb79f69a387c91e61f1795227e4ed9cece14 \ + --hash=sha256:b5b290ccc2a263e8d185130284f8501e3e36c5e02750fc6b6bdeb2e9e96f1e25 \ + --hash=sha256:b5d84d37db046c5ca74ee7bb47dd6cbc13f80665fdde3e8040bdd3fb015ecb50 \ + --hash=sha256:b7cf1017d601aa35e6bb650b6ad28652c9cd78ee6caff19f3c28d03e1c80acbf \ + --hash=sha256:bc7637e2f80d8530ee4a78e878bce464f70087ce73cf7c1caf142416923b98f1 \ + --hash=sha256:c0463276121fdee9c49b98908b3a89c39be45d86d1dbaa22957e38f6321d4ce3 \ + --hash=sha256:c4ef880e27901b6cc782f1b95f82da9313c0eb95c3af699103088fa0ac3ce9ac \ + --hash=sha256:c8ae8a0f02f57a6e61203a31428fa1d677cbe50c93622b4149d5c0f319c1d19e \ + --hash=sha256:ca5862d5b3928c4940729dacc329aa9102900382fea192fc5e52eb69d6093815 \ + --hash=sha256:cb01158d8b88ee68f15949894ccc6712278243d95f344770fa7593fa2d94410c \ + --hash=sha256:cb6254dc36b47a990e59e1068afacdcd02958bdcce30bb50cc1700a8b9d624a6 \ + --hash=sha256:cc00f04ed596e9dc0da42ed17ac5e596c6ccba999ba6bd92b0e0aef2f170f2d6 \ + --hash=sha256:cd09d08005f958f370f539f186d10aec3377d55b9eeb0d796025d4886119d76e \ + --hash=sha256:cd4b7ca9984e5e7985c12bc60a6f173f3c958eae74f3ef6624bb6b26e2abbae4 \ + --hash=sha256:ce8a0633f41a967713a59c4139d29110c07e826d131a316b50ce11b1d79b4f84 \ + --hash=sha256:cead0978fc57397645f12578bfd2d5ea9138ea0fac82b2f63f7f7c6877986a69 \ + --hash=sha256:d055ec1e26e441f6187acf818b73564e6e6282709e9bcb5b63f5b23068356a15 \ + --hash=sha256:d1f13550535ad8cff21b8d757a3257963e951d96e20ec82ab44bc64aeb62a191 \ + --hash=sha256:d9c7f57c3d666a53421049053eaacdd14bbd0a528e2186fcb2e672effd053bb0 \ + --hash=sha256:d9e45d7faa48ee908174d8fe84854479ef838fc6a705c9315372eacbc2f02897 \ + --hash=sha256:da3326d9e65ef63a817ecbcc0df6e94463713b754fe293eaa03da99befb9a5bd \ + --hash=sha256:de00632ca48df9daf77a2c65a484531649261ec9f25489917f09e455cb09ddb2 \ + --hash=sha256:e1f185f86a6f3403aa2420e815904c67b2f9ebc443f045edd0de921108345794 \ + --hash=sha256:e824f1492727fa856dd6eda4f7cee25f8518a12f3c4a56a74e8095695089cf6d \ + --hash=sha256:e912091979546adf63357d7e2ccff9b44f026c075aeaf25a52d0e95ad2281074 \ + --hash=sha256:eaabd426fe94daf8fd157c32e571c85cb12e66692f15516a83a03264b08d06c3 \ + --hash=sha256:ebf3e58c7ec8a8bed6d66a75d7fb37b55e5015b03ceae72a8e7c74495551e224 \ + --hash=sha256:ecaae4149d99b1c9e7b88bb03e3221956f68fd6d50be2ef061b2381b61d20838 \ + --hash=sha256:eecbc200c7fd5ddb9a7f16c7decb07b566c29fa2161a16cf67b8d068bd21690a \ + --hash=sha256:f155a433c2ec037d4e8df17d18922c3a0d9b3232a396690f17175d2946f0218d \ + --hash=sha256:f1e34719c6ed0b92f418c7c780480b26b5d9c50349e9a9af7d76bf757530350d \ + --hash=sha256:f34be2938726fc13801220747472850852fe6b1ea75869a048d6f896838c896f \ + --hash=sha256:f820802628d2694cb7e56db99213f930856014862f3fd943d290ea8438d07ca8 \ + --hash=sha256:f8bf04158c6b607d747e93949aa60618b61312fe647a6369f88ce2ff16043490 \ + --hash=sha256:f8e160feb2aed042cd657a72acc0b481212ed28b1b9a95c0cee1621b524e1966 \ + --hash=sha256:f9d332f8c2a2fcbffe1378594431458ddbef721c1769d78e2cbc06280d8155f9 \ + --hash=sha256:fa09f53c465e532f4d3db095e0c55b615f010ad81803d383195b6b5ca6cbf5f3 \ + --hash=sha256:faa3a41b2b66b6e50f84ae4a68c64fcd0c44355741c6374813a800cd6695db9e \ + --hash=sha256:fd44c878ea55ba351104cb93cc85e74916eb8fa440ca7903e57575e97394f608 + # via requests +filelock==3.20.0 \ + --hash=sha256:339b4732ffda5cd79b13f4e2711a31b0365ce445d95d243bb996273d072546a2 \ + --hash=sha256:711e943b4ec6be42e1d4e6690b48dc175c822967466bb31c0c293f34334c13f4 + # via huggingface-hub +fsspec==2025.9.0 \ + --hash=sha256:19fd429483d25d28b65ec68f9f4adc16c17ea2c7c7bf54ec61360d478fb19c19 \ + --hash=sha256:530dc2a2af60a414a832059574df4a6e10cce927f6f4a78209390fe38955cfb7 + # via huggingface-hub +hf-transfer==0.1.9 \ + --hash=sha256:035572865dab29d17e783fbf1e84cf1cb24f3fcf8f1b17db1cfc7fdf139f02bf \ + --hash=sha256:0d991376f0eac70a60f0cbc95602aa708a6f7c8617f28b4945c1431d67b8e3c8 \ + --hash=sha256:16f208fc678911c37e11aa7b586bc66a37d02e636208f18b6bc53d29b5df40ad \ + --hash=sha256:1a6bd16c667ebe89a069ca163060127a794fa3a3525292c900b8c8cc47985b0d \ + --hash=sha256:2c7fc1b85f4d0f76e452765d7648c9f4bfd0aedb9ced2ae1ebfece2d8cfaf8e2 \ + --hash=sha256:3a736dfbb2c84f5a2c975478ad200c0c8bfcb58a25a35db402678fb87ce17fa4 \ + --hash=sha256:3ebc4ab9023414880c8b1d3c38174d1c9989eb5022d37e814fa91a3060123eb0 \ + --hash=sha256:435cc3cdc8524ce57b074032b8fd76eed70a4224d2091232fa6a8cef8fd6803e \ + --hash=sha256:504b8427fd785dd8546d53b9fafe6e436bd7a3adf76b9dce556507650a7b4567 \ + --hash=sha256:57fd9880da1ee0f47250f735f791fab788f0aa1ee36afc49f761349869c8b4d9 \ + --hash=sha256:5828057e313de59300dd1abb489444bc452efe3f479d3c55b31a8f680936ba42 \ + --hash=sha256:5d561f0520f493c66b016d99ceabe69c23289aa90be38dd802d2aef279f15751 \ + --hash=sha256:6e94e8822da79573c9b6ae4d6b2f847c59a7a06c5327d7db20751b68538dc4f6 \ + --hash=sha256:8669dbcc7a3e2e8d61d42cd24da9c50d57770bd74b445c65123291ca842a7e7a \ + --hash=sha256:8674026f21ed369aa2a0a4b46000aca850fc44cd2b54af33a172ce5325b4fc82 \ + --hash=sha256:89a23f58b7b7effbc047b8ca286f131b17728c99a9f972723323003ffd1bb916 \ + --hash=sha256:8fd0167c4407a3bc4cdd0307e65ada2294ec04f1813d8a69a5243e379b22e9d8 \ + --hash=sha256:a5b366d34cd449fe9b20ef25941e6eef0460a2f74e7389f02e673e1f88ebd538 \ + --hash=sha256:cdca9bfb89e6f8f281890cc61a8aff2d3cecaff7e1a4d275574d96ca70098557 \ + --hash=sha256:d2fde99d502093ade3ab1b53f80da18480e9902aa960dab7f74fb1b9e5bc5746 \ + --hash=sha256:dc7fff1345980d6c0ebb92c811d24afa4b98b3e07ed070c8e38cc91fd80478c5 \ + --hash=sha256:e66acf91df4a8b72f60223059df3003062a5ae111757187ed1a06750a30e911b \ + --hash=sha256:e6ac4eddcd99575ed3735ed911ddf9d1697e2bd13aa3f0ad7e3904dd4863842e \ + --hash=sha256:ee8b10afedcb75f71091bcc197c526a6ebf5c58bbbadb34fdeee6160f55f619f \ + --hash=sha256:fc6bd19e1cc177c66bdef15ef8636ad3bde79d5a4f608c158021153b4573509d + # via -r tools/hf/requirements.in +hf-xet==1.1.10 \ + --hash=sha256:0a0005fd08f002180f7a12d4e13b22be277725bc23ed0529f8add5c7a6309c06 \ + --hash=sha256:408aef343800a2102374a883f283ff29068055c111f003ff840733d3b715bb97 \ + --hash=sha256:5f54b19cc347c13235ae7ee98b330c26dd65ef1df47e5316ffb1e87713ca7045 \ + --hash=sha256:686083aca1a6669bc85c21c0563551cbcdaa5cf7876a91f3d074a030b577231d \ + --hash=sha256:6b6bceb6361c80c1cc42b5a7b4e3efd90e64630bcf11224dcac50ef30a47e435 \ + --hash=sha256:71081925383b66b24eedff3013f8e6bbd41215c3338be4b94ba75fd75b21513b \ + --hash=sha256:eae7c1fc8a664e54753ffc235e11427ca61f4b0477d757cc4eb9ae374b69f09c \ + --hash=sha256:f900481cf6e362a6c549c61ff77468bd59d6dd082f3170a36acfef2eb6a6793f + # via huggingface-hub +huggingface-hub==0.35.3 \ + --hash=sha256:0e3a01829c19d86d03793e4577816fe3bdfc1602ac62c7fb220d593d351224ba \ + --hash=sha256:350932eaa5cc6a4747efae85126ee220e4ef1b54e29d31c3b45c5612ddf0b32a + # via -r tools/hf/requirements.in +idna==3.11 \ + --hash=sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea \ + --hash=sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902 + # via requests +packaging==25.0 \ + --hash=sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484 \ + --hash=sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f + # via huggingface-hub +pyyaml==6.0.3 \ + --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \ + --hash=sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a \ + --hash=sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3 \ + --hash=sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956 \ + --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \ + --hash=sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c \ + --hash=sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65 \ + --hash=sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a \ + --hash=sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0 \ + --hash=sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b \ + --hash=sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1 \ + --hash=sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6 \ + --hash=sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7 \ + --hash=sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e \ + --hash=sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007 \ + --hash=sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310 \ + --hash=sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4 \ + --hash=sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9 \ + --hash=sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295 \ + --hash=sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea \ + --hash=sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0 \ + --hash=sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e \ + --hash=sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac \ + --hash=sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9 \ + --hash=sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7 \ + --hash=sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35 \ + --hash=sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb \ + --hash=sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b \ + --hash=sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69 \ + --hash=sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5 \ + --hash=sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b \ + --hash=sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c \ + --hash=sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369 \ + --hash=sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd \ + --hash=sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824 \ + --hash=sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198 \ + --hash=sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065 \ + --hash=sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c \ + --hash=sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c \ + --hash=sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764 \ + --hash=sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196 \ + --hash=sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b \ + --hash=sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00 \ + --hash=sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac \ + --hash=sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8 \ + --hash=sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e \ + --hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 \ + --hash=sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3 \ + --hash=sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5 \ + --hash=sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4 \ + --hash=sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b \ + --hash=sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf \ + --hash=sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5 \ + --hash=sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702 \ + --hash=sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8 \ + --hash=sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788 \ + --hash=sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da \ + --hash=sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d \ + --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \ + --hash=sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c \ + --hash=sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba \ + --hash=sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f \ + --hash=sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917 \ + --hash=sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5 \ + --hash=sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26 \ + --hash=sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f \ + --hash=sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b \ + --hash=sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be \ + --hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c \ + --hash=sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3 \ + --hash=sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6 \ + --hash=sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926 \ + --hash=sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0 + # via huggingface-hub +requests==2.32.5 \ + --hash=sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6 \ + --hash=sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf + # via huggingface-hub +tqdm==4.67.1 \ + --hash=sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2 \ + --hash=sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2 + # via huggingface-hub +typing-extensions==4.15.0 \ + --hash=sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466 \ + --hash=sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548 + # via huggingface-hub +urllib3==2.5.0 \ + --hash=sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760 \ + --hash=sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc + # via requests diff --git a/tools/neuron/BUILD.bazel b/tools/neuron/BUILD.bazel new file mode 100644 index 00000000..8bfcabed --- /dev/null +++ b/tools/neuron/BUILD.bazel @@ -0,0 +1,51 @@ +load("@rules_shell//shell:sh_binary.bzl", "sh_binary") + +sh_binary( + name = "profile", + srcs = ["profile.sh"], + data = ["@libzml_neuron//:sandbox"], + visibility = ["//visibility:public"], +) + +sh_binary( + name = "server", + srcs = ["server.sh"], + args = [ + "/tmp/zml-neuron-explorer", + "3001", + ], + data = ["@libzml_neuron//:sandbox"], + visibility = ["//visibility:public"], +) + +sh_binary( + name = "ingest", + srcs = ["ingest.sh"], + args = [ + "/tmp/zml-neuron-explorer", + "/tmp/zml-neuron-profile", + ], + data = ["@libzml_neuron//:sandbox"], + visibility = ["//visibility:public"], +) + +sh_binary( + name = "summary-txt", + srcs = ["summary.sh"], + data = ["@libzml_neuron//:sandbox"], + visibility = ["//visibility:public"], +) + +sh_binary( + name = "summary-json", + srcs = ["summary.sh"], + data = ["@libzml_neuron//:sandbox"], + visibility = ["//visibility:public"], +) + +sh_binary( + name = "summary-perfetto", + srcs = ["summary.sh"], + data = ["@libzml_neuron//:sandbox"], + visibility = ["//visibility:public"], +) diff --git a/tools/neuron/ingest.sh b/tools/neuron/ingest.sh new file mode 100755 index 00000000..bb03850d --- /dev/null +++ b/tools/neuron/ingest.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +set -euo pipefail +shopt -s nullglob + +neuron_explorer_bin="$0.runfiles/+neuron_packages+libzml_neuron/sandbox/bin/neuron-explorer" +data_path="$1" +default_profile_root="$2" +shift 2 + +mkdir -p "${data_path}" + +ingest_run() { + local run_dir="$1" + local execution_dir="${run_dir}/execution" + local run_id + run_id="$(basename "${run_dir}")" + + if [[ ! -d "${execution_dir}" ]]; then + echo "skipping ${run_dir}: missing execution directory" >&2 + return + fi + + for profile_session in "${execution_dir}"/*/*; do + [[ -f "${profile_session}/ntrace.pb" ]] || continue + + local system_dir + system_dir="$(mktemp -d)" + for file in ntrace.pb trace_info.pb cpu_util.pb host_mem.pb; do + [[ -f "${profile_session}/${file}" ]] && cp "${profile_session}/${file}" "${system_dir}/${file}" + done + + "${neuron_explorer_bin}" view \ + --ingest-only \ + --force \ + --data-path "${data_path}" \ + --display-name "${run_id}:system" \ + -d "${system_dir}" + + rm -rf "${system_dir}" + done + + for neff in "${execution_dir}"/*/*/*.neff; do + local neff_name + local program_name + local ntff_prefix + neff_name="$(basename "${neff}" .neff)" + program_name="$(grep -aEom1 'MODULE_[A-Za-z0-9_]+' "${neff}" | sed -E 's/^MODULE_//; s/_[0-9]+$//')" + if [[ -z "${program_name}" ]]; then + echo "failed to find program name in ${neff}" >&2 + exit 1 + fi + ntff_prefix="$(dirname "${neff}")/${neff_name}" + + if ! compgen -G "${ntff_prefix}_rank_*.ntff" >/dev/null && [[ ! -f "${ntff_prefix}.ntff" ]]; then + "${neuron_explorer_bin}" capture \ + -n "${neff}" \ + -s "${ntff_prefix}.ntff" \ + --num-exec=2 \ + --profile-nth-exec=2 \ + --io-from=neff + fi + + for ntff in "${ntff_prefix}"_rank_*.ntff "${ntff_prefix}".ntff; do + [[ -f "${ntff}" ]] || continue + + local core_suffix + local display_name + core_suffix="$(basename "${ntff}" .ntff | sed -E 's/^.*_rank_([0-9]+).*$/nc\1/; s/^.*_(vnc_[0-9]+)$/\1/')" + display_name="${run_id}:${program_name}_${core_suffix}" + + "${neuron_explorer_bin}" view \ + --ingest-only \ + --force \ + --data-path "${data_path}" \ + --display-name "${display_name}" \ + -n "${neff}" \ + -s "${ntff}" + done + done +} + +[[ $# -eq 0 ]] && set -- "${default_profile_root}" + +for path in "$@"; do + if [[ -d "${path}/execution" ]]; then + ingest_run "${path}" + else + for run_dir in "${path}"/*; do + [[ -d "${run_dir}" ]] && ingest_run "${run_dir}" + done + fi +done diff --git a/tools/neuron/profile.sh b/tools/neuron/profile.sh new file mode 100755 index 00000000..e1118d56 --- /dev/null +++ b/tools/neuron/profile.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +set -euo pipefail + +profile_root="$1" +shift + +run_dir="${profile_root}/$(date -u '+%Y%m%dT%H%M%SZ')" +execution_dir="${run_dir}/execution" +compile_dir="${run_dir}/compile" + +mkdir -p "${execution_dir}" "${compile_dir}" + +neuron_explorer_bin="$0.runfiles/+neuron_packages+libzml_neuron/sandbox/bin/neuron-explorer" + +xla_flags="--xla_dump_to=${compile_dir} --xla_dump_hlo_as_text --xla_dump_hlo_as_proto --xla_dump_hlo_pass_re=.*" + +set +e +env \ + NEURON_RT_INSPECT_ENABLE=1 \ + NEURON_RT_INSPECT_SYSTEM_PROFILE=1 \ + NEURON_RT_INSPECT_DEVICE_PROFILE=1 \ + NEURON_RT_INSPECT_OUTPUT_DIR="${execution_dir}" \ + SKIP_PJRT_PROFILER=true \ + XLA_IR_DEBUG=1 \ + XLA_HLO_DEBUG=1 \ + XLA_FLAGS="${xla_flags}" \ + "${neuron_explorer_bin}" inspect -o "${execution_dir}" "$@" +status=$? +set -e + +echo "Neuron profile dump path: ${run_dir}" >&2 + +exit "${status}" diff --git a/tools/neuron/server.sh b/tools/neuron/server.sh new file mode 100755 index 00000000..3d0babe1 --- /dev/null +++ b/tools/neuron/server.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euo pipefail + +data_path="$1" +port="$2" +neuron_explorer_bin="$0.runfiles/+neuron_packages+libzml_neuron/sandbox/bin/neuron-explorer" + +mkdir -p "${data_path}" + +echo "Neuron Explorer server: http://127.0.0.1:${port}" + +exec "${neuron_explorer_bin}" view \ + --data-path "${data_path}" \ + --port "${port}" diff --git a/tools/neuron/summary.sh b/tools/neuron/summary.sh new file mode 100755 index 00000000..5a1b1461 --- /dev/null +++ b/tools/neuron/summary.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +set -euo pipefail + +neuron_explorer_bin="$0.runfiles/+neuron_packages+libzml_neuron/sandbox/bin/neuron-explorer" + +run_dir="$1" +execution_dir="${run_dir}/execution" + +case "$(basename "$0")" in + summary-txt) + output_format="summary-text" + output_file="summary.txt" + ;; + summary-json) + output_format="summary-json" + output_file="summary.json" + ;; + summary-perfetto) + output_format="perfetto" + output_file="summary.pftrace" + ;; +esac + +if [[ "${output_format}" == "summary-text" || "${output_format}" == "summary-json" ]]; then + "${neuron_explorer_bin}" view \ + -d "${execution_dir}" \ + --output-format "${output_format}" \ + >"${run_dir}/${output_file}" +else + "${neuron_explorer_bin}" view \ + -d "${execution_dir}" \ + --output-format "${output_format}" \ + --output-file "${run_dir}/${output_file}" +fi + +echo "Neuron summary path: ${run_dir}/${output_file}" >&2 diff --git a/tools/nsys.sh b/tools/nsys.sh new file mode 100755 index 00000000..b8d38e38 --- /dev/null +++ b/tools/nsys.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -lt 1 ]]; then + echo "usage: $0 [args...]" >&2 + exit 1 +fi + +sudo_cmd=() +if [[ "${ZML_PROFILE_NO_SUDO:-0}" != "1" ]]; then + sudo_cmd=(sudo -E) +fi + +declare -a nsys_args +if [[ -n "${ZML_NSYS_ARGS:-}" ]]; then + nsys_args=(${ZML_NSYS_ARGS}) +else + nsys_args=( + profile + -t + cuda,syscall,nvtx,cublas,cublas-verbose,cusparse,cusparse-verbose,cudnn,osrt + --inherit-environment=true + --cuda-memory-usage + true + --cuda-event-trace=false + --backtrace=dwarf + --cuda-graph-trace=node + ) +fi + +exec env SKIP_PJRT_PROFILER=true "${sudo_cmd[@]}" nsys "${nsys_args[@]}" "$@" diff --git a/tools/rocprofv3.sh b/tools/rocprofv3.sh new file mode 100755 index 00000000..cc7e70bb --- /dev/null +++ b/tools/rocprofv3.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -lt 1 ]]; then + echo "usage: $0 [args...]" >&2 + exit 1 +fi + +if [[ -n "${RUNFILES_DIR:-}" ]]; then + runfiles_root="${RUNFILES_DIR}" +elif [[ -d "$0.runfiles" ]]; then + runfiles_root="$0.runfiles" +else + echo "failed to locate Bazel runfiles for sandboxed rocprofv3" >&2 + exit 1 +fi + +rocm_root="${runfiles_root}/+rocm_packages+libzml_rocm/sandbox" +if [[ ! -d "${rocm_root}" ]]; then + echo "sandboxed ROCm runtime root not found at ${rocm_root}" >&2 + exit 1 +fi + +rocprofv3_bin="${rocm_root}/bin/rocprofv3" +if [[ ! -x "${rocprofv3_bin}" ]]; then + echo "sandboxed rocprofv3 not found at ${rocprofv3_bin}" >&2 + exit 1 +fi + +declare -a rocprof_args +if [[ -n "${ZML_ROCPROFV3_ARGS:-}" ]]; then + rocprof_args=(${ZML_ROCPROFV3_ARGS}) +else + rocprof_args=( + --sys-trace + --runtime-trace + --hip-trace + --kernel-trace + --memory-copy-trace + --hip-runtime-trace + --hsa-core-trace + --scratch-memory-trace + --marker-trace + --kernel-rename + --stats + --output-format + pftrace + ) +fi + +exec env SKIP_PJRT_PROFILER=true "${rocprofv3_bin}" --rocm-root "${rocm_root}" "${rocprof_args[@]}" -- "$@" diff --git a/tools/safetensors/BUILD.bazel b/tools/safetensors/BUILD.bazel new file mode 100644 index 00000000..aff4954d --- /dev/null +++ b/tools/safetensors/BUILD.bazel @@ -0,0 +1,13 @@ +load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_test") + +zig_binary( + name = "safetensors", + main = "main.zig", + visibility = ["//visibility:public"], + zigopts = ["-lc"], + deps = [ + "//stdx", + "//vfs", + "//zml", + ], +) diff --git a/tools/safetensors/main.zig b/tools/safetensors/main.zig new file mode 100644 index 00000000..cf902eef --- /dev/null +++ b/tools/safetensors/main.zig @@ -0,0 +1,91 @@ +const std = @import("std"); + +const stdx = @import("stdx"); +const VFS = @import("vfs"); +const zml = @import("zml"); + +const log = std.log.scoped(.safetensors_convert); + +const Args = struct { + output: ?[]const u8 = null, + positional: struct { model: []const u8 }, + + pub const help = + \\Usage: [--output=] + \\ + \\Read safetensors headers and write model.safetensors.full_index.json in the working directory + \\Example: bazel run //tools/safetensors -- --output="model.safetensors.full_index.json" hf://deepseek-ai/DeepSeek-V4.1-Flash + ; +}; + +pub fn main(init: std.process.Init) !void { + const args = stdx.flags.parse(init.minimal.args, Args); + const allocator = init.gpa; + + const start = std.Io.Clock.now(.awake, init.io); + var client: std.http.Client = .{ .allocator = allocator, .io = init.io }; + defer client.deinit(); + try client.initDefaultProxies(allocator, init.environ_map); + + const input = args.positional.model; + + var file_vfs: VFS.File = .init(allocator, init.io, .{ .direct_io = false }); + defer file_vfs.deinit(); + var http_vfs: VFS.HTTP = try .init(allocator, init.io, &client, .http); + defer http_vfs.deinit(); + var https_vfs: VFS.HTTP = try .init(allocator, init.io, &client, .https); + defer https_vfs.deinit(); + var hf_vfs: ?VFS.HF = if (std.mem.startsWith(u8, input, "hf://")) + try .auto(allocator, init.io, &client, init.environ_map) + else + null; + defer if (hf_vfs) |*backend| backend.deinit(); + var s3_vfs: ?VFS.S3 = if (std.mem.startsWith(u8, input, "s3://")) + try .auto(allocator, init.io, &client, init.environ_map) + else + null; + defer if (s3_vfs) |*backend| backend.deinit(); + var gcs_vfs: ?VFS.GCS = if (std.mem.startsWith(u8, input, "gs://")) + try .auto(allocator, init.io, &client, init.environ_map) + else + null; + defer if (gcs_vfs) |*backend| backend.deinit(); + + var vfs: VFS = try .init(allocator, init.io); + const io = vfs.io(); + defer vfs.deinit(); + try vfs.register("file", file_vfs.io()); + try vfs.register("http", http_vfs.io()); + try vfs.register("https", https_vfs.io()); + if (hf_vfs) |*backend| try vfs.register("hf", backend.io()); + if (s3_vfs) |*backend| try vfs.register("s3", backend.io()); + if (gcs_vfs) |*backend| try vfs.register("gs", backend.io()); + + var arena: std.heap.ArenaAllocator = .init(allocator); + defer arena.deinit(); + + const repo = try zml.safetensors.resolveModelRepo(io, input); + var registry: zml.safetensors.TensorRegistry = try .fromRepo(allocator, io, repo); + defer registry.deinit(); + const loaded = std.Io.Clock.now(.awake, init.io); + + const index_json_file = if (args.output) |output| + try std.Io.Dir.cwd().createFile(io, output, .{}) + else + try repo.createFile(io, full_index_path, .{}); + const out_buffer = try allocator.alloc(u8, 4096); + defer allocator.free(out_buffer); + var out_writer = index_json_file.writer(io, out_buffer); + try registry.saveAsFullIndexJson(io, allocator, repo, &out_writer.interface); + const done = std.Io.Clock.now(.awake, init.io); + + log.info("Wrote {s}: {d} tensors; load {f}s, write {f}s, total {f}s", .{ + args.output orelse full_index_path, + registry.tensors.count(), + start.durationTo(loaded), + loaded.durationTo(done), + start.durationTo(done), + }); +} + +const full_index_path = "model.safetensors.full_index.json"; diff --git a/tools/workspace_status.sh b/tools/workspace_status.sh new file mode 100755 index 00000000..05f02bed --- /dev/null +++ b/tools/workspace_status.sh @@ -0,0 +1,8 @@ +#!/bin/bash +if git diff --quiet; then + echo BUILD_SCM_REVISION "$(git rev-parse --short HEAD)" +else + echo BUILD_SCM_REVISION "dev" +fi + +echo STABLE_ZML_SMI_VERSION "$(git describe --tags --dirty --match 'zml-smi-v*' 2>/dev/null || echo "dev")" diff --git a/tools/xprof/BUILD.bazel b/tools/xprof/BUILD.bazel new file mode 100644 index 00000000..285f7003 --- /dev/null +++ b/tools/xprof/BUILD.bazel @@ -0,0 +1,19 @@ +load("@rules_python//python/uv:lock.bzl", uv_lock = "lock") +load("@rules_python//python:py_binary.bzl", "py_binary") + +py_binary( + name = "xprof", + srcs = ["main.py"], + main = "main.py", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = ["@xprof//xprof:pkg"], +) + +uv_lock( + name = "requirements", + srcs = ["requirements.in"], + out = "requirements.lock.txt", + args = ["--upgrade"], + tags = ["manual"], +) diff --git a/tools/xprof/main.py b/tools/xprof/main.py new file mode 100644 index 00000000..79ed3ccf --- /dev/null +++ b/tools/xprof/main.py @@ -0,0 +1,42 @@ +import os +import sys +from pathlib import Path + +from xprof import profile_plugin +from xprof.cli.xprof_cli import main as xprof_main + + +def _configure_static_dir() -> None: + static_dir = Path(profile_plugin.__file__).parent / "static" + # Bazel runfiles contain a real directory whose files are symlinks into the + # repository cache. Resolve a file so XProf's traversal check sees the same + # base path as the assets it opens. + resolved_static_dir = (static_dir / "index.html").resolve().parent + os.environ.setdefault("XPROF_STATIC_DIR", str(resolved_static_dir)) + + +def _argv() -> list[str]: + user_args = sys.argv[1:] + if user_args and not user_args[0].startswith("-"): + logdir = user_args[0] + trailing_args = user_args[1:] + else: + logdir = "." + trailing_args = user_args + return [ + sys.argv[0], + "--logdir", + logdir, + "--max_concurrent_worker_requests", + "32", + "--port", + "6006", + "--hide_capture_profile_button", + *trailing_args, + ] + + +if __name__ == "__main__": + _configure_static_dir() + sys.argv = _argv() + raise SystemExit(xprof_main()) diff --git a/tools/xprof/requirements.in b/tools/xprof/requirements.in new file mode 100644 index 00000000..7a238fea --- /dev/null +++ b/tools/xprof/requirements.in @@ -0,0 +1 @@ +xprof==2.23.2 diff --git a/tools/xprof/requirements.lock.txt b/tools/xprof/requirements.lock.txt new file mode 100644 index 00000000..f3ac6542 --- /dev/null +++ b/tools/xprof/requirements.lock.txt @@ -0,0 +1,1453 @@ +# This file was autogenerated by uv via the following command: +# bazel run //tools/xprof:requirements.update +absl-py==2.5.0 \ + --hash=sha256:0c996f25c0490700fadabe6351630f6111534fa0ae252cc6d2014ea3b141135f \ + --hash=sha256:0f17b89f2a4eaaedc4f28c622998aa690564b3012a396a4ffad0821007fe03ba + # via xprof +aiohappyeyeballs==2.7.1 \ + --hash=sha256:065665c041c42a5938ed220bdcd7230f22527fbec085e1853d2402c8a3615d9d \ + --hash=sha256:9243213661e29250eb41368e5daa826fc017156c3b8a11440826b2e3ed376472 + # via aiohttp +aiohttp==3.14.3 \ + --hash=sha256:03cd2bde3d7f085b64e549c985f4bb928cad7e8ecf5323bfca320db548d81b39 \ + --hash=sha256:041badb8f84396357c4d3ad26de6afd7a32b112f43d3c63045c0c8278cfd2043 \ + --hash=sha256:0a5ff2dfbb9ce645fa5b8ef3e02c6c0b9cc3f6030ff863d0c51fffc50cb5541b \ + --hash=sha256:0fdea2281997af69da84c77ffa6f5938a0285f21fb3887c249d67419ca865b3d \ + --hash=sha256:11fb37ef075669eee52ab1928fbf6e1741fada40409fa309ebde9607a962aebf \ + --hash=sha256:134ac5ddcf61c6fad984b9a5727d83492ada43d63471db20fb73042c13fca62f \ + --hash=sha256:152516815ef926786a0b6ae2b8f1fd2e0c71582dee0b435636865316fd4891b7 \ + --hash=sha256:1576145bdceeb92382d899751e12743a3a5b8e460a841e3e50543859e54864dc \ + --hash=sha256:16100ad3ab8d649fdfbee87602d9d2dcdca9df0b9eda8a1b5fdc0d41f96da559 \ + --hash=sha256:16ea7e24c309fb7c0bbd505d149abe4fe4dccfb8db911db7dbec0921bc889a6f \ + --hash=sha256:18c441d0a8fca6de8d1f546849b9f0ab20d435993e2c5b59562b2fae6be2f929 \ + --hash=sha256:18cb43369747b2ae007bd2655fb8e63a099c2ff1d207962943636dac989b3147 \ + --hash=sha256:1b59533861b70a2185c8f4f350f791f39d64358ef6944ce71c5240c9ec0982c9 \ + --hash=sha256:1c5281acc88b92396f88c7e1e2748f8466689df22b80170e4f51efa712fb47a8 \ + --hash=sha256:1c5ec8fb1bcc31a8466f74aaf26c345d5c386fa4bd08a3f0eb9c7a4a3fe8b5bf \ + --hash=sha256:1caa7b0d05f3e3a36f87788c59e970a7ee1cefcfcbb924a9f138c4a6551c9cb7 \ + --hash=sha256:21c016079415ed3fd676963e9793700a566d85dbbd6bfc564b9b2d209147dcc8 \ + --hash=sha256:2498f0fe69ead802f9675beca44a7c21c62fdaa4ec5145ea1c3ad6edbee29f85 \ + --hash=sha256:25bd2708db6bdf6a6630dd37bdcdfcb47c4434d22ac69c64665b802910140b30 \ + --hash=sha256:270d3dace9ca2f10f0da5d8ebe519b7a310fc6112ed916e32df5866df0888553 \ + --hash=sha256:2e1161602f45a54de2ce0905243a95f58cb42dcd378402f3697f5e0b21e9d2e7 \ + --hash=sha256:2e9878ae68e4a5f1c0abe4dd497dbc3d51946f5837b56759e2a02e78fa90ef86 \ + --hash=sha256:30402d03a7c0ff52bce290b57e564e9079fd9d0cb545c8aba73f86a103162d2e \ + --hash=sha256:33a2d7c28d33797a2e99923dffa63f83d908a19b6bf26cfe80fa790aa5e1a75a \ + --hash=sha256:362a3fd481769cac1a824514bcd86fda51c65e8fe6e051099e008fddde6db17c \ + --hash=sha256:38901a84da3ce22249f6e860bf8f90d141bcab7da090cc398f8bb58c0e44b7da \ + --hash=sha256:39aded8c7f3b935b54aab1d8d73c70ec0ee2d3ec3b943e0e86611bc150ba47f5 \ + --hash=sha256:3a26434dafe408229ff3403458ca58de24fb51936504decac49ce6755f77e59d \ + --hash=sha256:3ae5b3a59436d089b5395d910121a390feed4d00578eb95a0fd1a329fe963100 \ + --hash=sha256:3d4f72af88ac2474bb5bca640030320e3d38a0163a1d7533500e87be458eef71 \ + --hash=sha256:3f42e9b78301f11c8f861746175d8b9c1ccef713fcad9eab396e2f6db8ed4a22 \ + --hash=sha256:42a67efc36300d052fb4508a53e8b6901b9284b599ae63945c377569c5fcc1e1 \ + --hash=sha256:48d67b87db6279c044760787eb01f6413032c2e6f3ba1cafaa492b1c8e578479 \ + --hash=sha256:498c6c623134f8e09a3c4e60bcd607a0b4590dd7dbf08dd40851b27cbb520ccb \ + --hash=sha256:49f7325beb0f85ef4aef5f48f490269575f83e6e2acad00a1d80b807eb027062 \ + --hash=sha256:4e3ac92d90e92773b2362d506068e9a948192bd553e743c5b2429e28527c8661 \ + --hash=sha256:530125ee1163c4219af35dc3aa1206e541e7b31b6efc1a3f93b70a136f65d427 \ + --hash=sha256:5373dc80ad1aa2fb9ad95c83f24eef418bbda3a61375f128e5b0192e4f3f9b32 \ + --hash=sha256:53e5179d8abb5710f8e83ba207c41c8d1261fcffd4616500e15ca2b7a33be10a \ + --hash=sha256:53e7b4ce82b54a8bcc71b3b67a5cbd177ca1d7f592cbc92cd38b7349f73482db \ + --hash=sha256:543906c127fb1d929b95076db19b83fa2d46751006ff1e23b093aa5ac4d8db42 \ + --hash=sha256:54cfcdee2770dac994417cbb0ee1f3eb0e7cb6b30c79bf44f2c02ff79ec5124a \ + --hash=sha256:55bdcc472aafe2de4a253045cc128007a64f1e0264fb675791e132ea5edaa3bd \ + --hash=sha256:56f355e79f71aef2a85c80305cc915f894b170dba76de5fe84f6351939b83c06 \ + --hash=sha256:5895ef58c4620afe02fa16044f023dc4dafec08158f9d08874a46a7dbc0341b8 \ + --hash=sha256:5bcb6ff3fdab1258a192679ff1a05d44f59626430aa05cd1a9d2447423599228 \ + --hash=sha256:5f08ec777f35ee70720233b8b9811d3bb5d728137f30ac91b7457709c3261ac0 \ + --hash=sha256:614c61d478b83953e261d02bb2df750f17227cd33ef8002945bf5aebbde21919 \ + --hash=sha256:617105e2c3018ee38d0c8ce5ee3c84f621a6d8b9f723202aacaff28449ca91ee \ + --hash=sha256:6debfa7312ff9d4c124dc71d72e9a0a4b9e0879e48ba6fcb42bef5c3300289e2 \ + --hash=sha256:7041d52c3a7fa20c9e8c182b534704abb19502c8bdcbde7ab23bfda6f642394f \ + --hash=sha256:70c987b27534f9ae1a723f47ae921571d616da21d3208282bf4c52af5164ac43 \ + --hash=sha256:74ab5b6a9fb13e873e5a90946588baecaf488745e1db1a4a5c433f971f035098 \ + --hash=sha256:78253b573e6ffab5028924fc98bc281aae05445969982a10864bc360dea2016c \ + --hash=sha256:7a75aa63cbf9b21cfaf60dc2657e19df2c2867d91707d653fee171ffeedd1371 \ + --hash=sha256:8800c996b01c2772a783e3e46f3e1abd5823029adca0df54231960de9bfefa5b \ + --hash=sha256:89176250f686cb9853c0fb7ead90e639e915b84a6f43eedc2a4e7ec21f1037f0 \ + --hash=sha256:8a5fd34f7f7410d1730d5c2ba873cacb2eed3fede366feb268a70ba22581ed8f \ + --hash=sha256:8b3b60de05f3dcb6f6a00f818bb2ec781cee4de0645f59ccaf99b1d1823b6100 \ + --hash=sha256:8f2f1c4c032c7cedd7d8da6f54c97b70266c6570c3108d3fdffee7188bb70529 \ + --hash=sha256:9491196535a88924a60afd5b5f434b5b203b6cc616250878dbdb223a8f7844bc \ + --hash=sha256:9aa6e61fdf20105c4144e755bd586008ff450791d67b1c8146fdc15959c4d51c \ + --hash=sha256:9d9edccfe496b476db5f398d97b865e9a6752bcf8aec4eef8390ce20fb64bb41 \ + --hash=sha256:9fc7b5bfec6573f3ae844f457fdde5adeb713f8b8e4a81ad64fc207b49383716 \ + --hash=sha256:a0dc483c00da8b673abbb367eb6f8d8f4bcec30eb58529ea13cb42e7fd2dfa33 \ + --hash=sha256:a3a8296e7ab5c295f53f1041487cb088e1480775aafbf7fe545d93b770a0f96f \ + --hash=sha256:a3e22975f905b89a55a488c2a08f2fdb2186175349e917d48985cc468a3d4c6e \ + --hash=sha256:a4af35c443e0b1a1bd6a8af3f3485d7fda15c142751a00f3ff8090f0b93346fa \ + --hash=sha256:a94dbaae5ae27bd849c93570669bff91e0510f33a80805738e3de72a7be0447b \ + --hash=sha256:ac74facc01463f138b0da5580329cfcc82818dea5656e83ddcd11268fc12ff80 \ + --hash=sha256:ad4c8b7488d745d2ca4838ebd8ae5ba9b56341d30b1da43640e4ce87f9f49646 \ + --hash=sha256:b014a6ed7cf912e787149fdc529166d3ceabac23f26efeea3158c9aba2354e7e \ + --hash=sha256:b20032766aedf6261c7a566585a40867d092ac03a0d81592d5370ef9b054f99b \ + --hash=sha256:b2466434105a4e03113c36ec775cc2ebe6676b62eae326fa670bb607ef788c1c \ + --hash=sha256:b304db572b4368edd8dda8a2274f73156fe15558fca4a917cb8a09fc47af5963 \ + --hash=sha256:ba59d59aba08ac02fc03b0c8983ccd5ee39a199d0552ce9e6d2b4845b34d59ae \ + --hash=sha256:bd52f811e65f6fb634b1047159657c98f52b407f8efec907bcfc09da9a4c0a25 \ + --hash=sha256:bdd0e2834dce1a26c1bbe26464861e16bbe217042cbff619247c11594472518c \ + --hash=sha256:c23ec8ee9d5ab2f5421f9c7fffce208435607af27fd46d4a44e031954352838f \ + --hash=sha256:c39846c3aad97a8530c89d7a3869a8f8e9e3762c6ac0504481e5c80948f7e807 \ + --hash=sha256:c3c200cf9757edd785051dc699c7ecbec22110dbfcb3fefc7a9f9695eda8ea7a \ + --hash=sha256:c7d3a97c678d34fc5b59da671ee9cd630096ddc643e7b5a30d54a2a6f3574d3f \ + --hash=sha256:c8653fd547c93a61aadc612007790f5555cdd18946fa48cf45e26d8ea4ea473d \ + --hash=sha256:cc7cb243a68167172f48c1fd43cee91ec4b1d40cefd190edd43369d1a6bc9c82 \ + --hash=sha256:ccd4893707b3e2a13e39c90d43cf80edf2e4d0457935bcc103bf2346214c3f15 \ + --hash=sha256:cd817772b2fcf2b8c0905795318485f9ec16eae60b29feb7f4c77085311637f0 \ + --hash=sha256:cda5fd5c95ad7a125a2e8464acc78b98b94c475a3780d6aa0aa157c93f470f4d \ + --hash=sha256:cef89a58e628c4efcac3275c2d68083f82426dcdc89c1492a6f654f9f7ea6ab9 \ + --hash=sha256:d1558173930a5a8d3069cee5c92fc91c87c4dbcb099debbb3622053717145a19 \ + --hash=sha256:d6088ec9894113802bddb3c09e974929aed2c7b3a8c456219b8aab4481f1a239 \ + --hash=sha256:d6218d92e450824e9b4881f44e8c09f1853b490f9a64130801024a4793b1b3b0 \ + --hash=sha256:d77640cc618c1d99fc4f8589c0f24a730adfa54eb1e57ef7bf0c8dfb78da898c \ + --hash=sha256:d7d2deec16eeedf55f2c7cf75b521ea3856a5177e123844f8fd0f114ce252cb5 \ + --hash=sha256:db332af25642007330fca8be5c4d194caf2bea7a7fc84415aff3497af5dfee6b \ + --hash=sha256:dd54d0e8717de95939766febac482ac0474d8ac3b048115f9f2b1d23a16e7db4 \ + --hash=sha256:ddcac3c6b382e81f1dd0499199d4136b877beb4cb5ef770bbbfba56c4b8f55d2 \ + --hash=sha256:df82f3787c940c94986b34222d59c9e38843fba85139f36e85255a82ad5355a9 \ + --hash=sha256:dfa68deb2a443bdaa3ea5297b0699c1464f08aef3812b486d1348eee61b07dc0 \ + --hash=sha256:dff9461ec275f22135650d5ba4b4931a11f3958df7dfbb8db630000d4dee0883 \ + --hash=sha256:e1e74298bab6ee0d6e749ed4fd1901c7e604bdda32c03d787a2cc71c46d0433d \ + --hash=sha256:e2667f0bbe7eb6c74eae5e9691441ad186e5845ca3cff63230fc09c4e7514f5d \ + --hash=sha256:e3be98a7c30b8c25d573dafba7171d66dfb05ee6a9070fc46535464ff97700a6 \ + --hash=sha256:e568e14940c09955aa51f4e645b6daa18a581c5dcfcd73744dcc86a856e3ced3 \ + --hash=sha256:e72ee89e28d907a18f46959b4eb0bb06701cc7f8cf4366e00029e2ccfaaf5924 \ + --hash=sha256:e92eb8acc45eb6a9f4935071a77edf5b85cc6f8dfad5cd99e97653c26593cdde \ + --hash=sha256:ea05e1f97ceea523942d9b2a7d7c0359d781d683d6b043f5943a602b14da4787 \ + --hash=sha256:eac645b09bcfdf73df7536331f0678c1086ea250981118ddb5199e17ccef72bb \ + --hash=sha256:eb0495d778817619273c108784292be161a924b9f5ae5cbbc70a2caa6838250b \ + --hash=sha256:ebe8e504f058fe91223351cecd2d9d6946c9d241bb0250d898ffbdf584cc72b0 \ + --hash=sha256:ed099d105449c4f9e84f24af203cd131349d4761d8813fa7e02c32e7128cd910 \ + --hash=sha256:f0f177d1b195b9e06376cfd7d308d8a1b920909a609d03ac82a8c73bbb16d3b9 \ + --hash=sha256:f3d2669fe7dec7fc359ecdb5984b29b50d85d5d00f8c1cb61de4f4a24ee42627 \ + --hash=sha256:f4e05329faa0ea1a404b37de4f034fd2c2defcca06a68dc6745e4e56c88e8a48 \ + --hash=sha256:f53bcd52f585e1ac3e590d61434eb61f9a88c38df041b4ea126d97144344a77b \ + --hash=sha256:f55119f7bf25f49ed210f6096090715da24f2943c62102448915fde3c62877ce \ + --hash=sha256:f631fe87a6f30df5fbe6d79640b25e4cffb38c31c7fb6f10871517b84b0f8c1a \ + --hash=sha256:f8fb78a83c9e5f741ca3a68cfb455c1f5bb83b4e7249a3848b3cd78d0a8563b0 \ + --hash=sha256:fa9467a8113aa69d3d7c55a70ef0b7c636010a40993f3df9d9d0d73b3eb7ef24 \ + --hash=sha256:fd51ebf9d3a00c074df4ede271023f4d2dba289bcc740b88191872716014e3c5 + # via gcsfs +aiosignal==1.4.0 \ + --hash=sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e \ + --hash=sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7 + # via aiohttp +attrs==26.1.0 \ + --hash=sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309 \ + --hash=sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32 + # via aiohttp +certifi==2026.7.22 \ + --hash=sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775 \ + --hash=sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55 + # via requests +cffi==2.1.1 \ + --hash=sha256:046bfc24911b37851ee1b51aab8bffe713d89c68c6a057b09484ce9fd5f69b4e \ + --hash=sha256:06c72bb76605a4b0cd0aad6930b69d4baf7dd5d806cfc409b824191099700e66 \ + --hash=sha256:0beceaabe56af686895136a2de78db54ecd8e4046b236b8fd6d6cb61389e9bf2 \ + --hash=sha256:154852545011f779917b11c78db2358d095da62a9a172b78ad0a583ee5adc0d0 \ + --hash=sha256:194cffa889098ced9976c3fc6340305e43f6303657d298da55366907c05c22d6 \ + --hash=sha256:19ee6127ee34de7d83ce3d371ebc5ed91addbdcc39f9ab15ce4eb35a4e534971 \ + --hash=sha256:1a18a57b58cfb21fc28d72e876acf10eaed67a1ed96226f92af4df681d571c4c \ + --hash=sha256:1aa5645c30469b09530c4ebca77ebf8f17618293c58f8549cb1a543a50236e7d \ + --hash=sha256:1dea0e4d7d4f11f619fe8c1d76caf49e24405b4b5743c0e3be16a500ecd930c9 \ + --hash=sha256:208f941bb9d18e768138677f0a6d2ce01f590df56043dda1df1535ac57c88517 \ + --hash=sha256:210019b6c7cf07f081b4c54635c8cf744377001350e29cc0f81c4377b4797735 \ + --hash=sha256:246fa40ce8645a614ff682e0b70f37134e460eaf93a775e0cbe3cca585a67a80 \ + --hash=sha256:25792eac27877609e7bb06d42ff88278a6624fff2ba9bbb523c09616b117e80f \ + --hash=sha256:27350daa11d4f10c540e6e89dada4c54feb7256ad03e9a4dc075ebad7ba360d1 \ + --hash=sha256:28907ab9bfb6aa13184cfc17c6b8e1023c5ab6fd7076d8c20a35e59fe04f8f29 \ + --hash=sha256:2ae64be792b8966f2c69538199728b290e34726562896df1e5dc8ffd8d8188e8 \ + --hash=sha256:31348097ff5bbe827ccc41795d4dd099d9f0625e7def00ee653c137a490c2a6c \ + --hash=sha256:3143d81e29e1e20a9ce10901ec369012947876596f75a222235965f2b7ae832e \ + --hash=sha256:3222ba5d678f80a030e6afbcc33dc1ae5cb45facabb61cee2c7016b8432fde48 \ + --hash=sha256:3311ed60d36f83378794e1009ac6258bafbf81f7888b4caa7b35a521e3f95813 \ + --hash=sha256:334644fbac4eff73d985a17a91226df55d0f394160c4cfb880e084c8f7161cac \ + --hash=sha256:34e261f78cb6ceaaa36f42f2613f4380d94d9c759a9c73c769ee6e0247364632 \ + --hash=sha256:363e05fa78e15116c3c32c210ee36884fd6b9afa6d440e47112c3bd511d64cb6 \ + --hash=sha256:398aff33cee2767e3e781d2554c54bd0dff386bb437581e0d8011fde1a942ec1 \ + --hash=sha256:3d22a20b1fb1632cc72c22f95f7b0d2961c3e1c235f245ba4c606c4771035659 \ + --hash=sha256:42a494cee34437f05546455144f2b5d9ac09b1face62bcfce597d2e521066688 \ + --hash=sha256:42e2f76b9455f5a9a844f770bf3e200ed3da0e15f5df3db9c31fe80b04b3d004 \ + --hash=sha256:42f6930c31dc7f50732c9ae793c2786c7b6b044195967bbdde40bb9be81c4cc0 \ + --hash=sha256:456a61fa52d579ebf9df2e9552ead5129855dbaff6c1e5a9b1bc408809bdc062 \ + --hash=sha256:471cee653ae88de62096552e6d24ccb4a5adb8c8c9f10b5054d0122c15bf2779 \ + --hash=sha256:49cbc70e6542d4ccccb936558d1064a8012541e78f821f955cff24e357776c94 \ + --hash=sha256:4a7c934f7360e8cd64fe9efadcbd10c7c6364f531e432b9a4bf5ccbc9e0e8b50 \ + --hash=sha256:4be96343e422f2dfcd12ab5c9f5aebe03f82f737c6bffeca6830b3875cb44aab \ + --hash=sha256:4f42141fc14250de6dde5ee7ea4432be017252d91f19c5ad043c084cea629cac \ + --hash=sha256:507a24c282e0f42f8ed737cf048572cbf580468da5555764a8331735e9c736b6 \ + --hash=sha256:51b31d1c98274844cfd7838ce00bfc27c7423a4dc00fc0772fc3331c2cc90676 \ + --hash=sha256:58acb8ab8e295e6c5ea12f888cbb13cf21511ef2a3303a23f4325c29d17fe5c1 \ + --hash=sha256:5a59cc1c4442bc3d5c703bf720b51138d0bfc173618807c9ee2490a7541dd3d9 \ + --hash=sha256:5bb4e7ea95dcd6a014a6fef62e62467d67d8e582326443f3d68e71d6320a9fcf \ + --hash=sha256:5c58fe613dc5e5336357eff555824a314d8e43282600435c8d1cb6a7a2fedd13 \ + --hash=sha256:5e7cecbaadb83884793e05828cee59b210b24583b9c7425d0ba6a754fe22eb4e \ + --hash=sha256:616f097f2fe415bc92a247f02e11f634e1f9e9a83d327e3c915c15089c87869e \ + --hash=sha256:63bbfd5ded17c4840ac07cd8f1c21ba9d9708141f840b324f422f41b207e3973 \ + --hash=sha256:64faea20f4e2613363a1a9b9c7dd73058f3ecd00133a511e72ad7c511658f527 \ + --hash=sha256:661c298b4821edebead0c91edd2b00374d67ad7c5a1f7a91d4442633b79d6a72 \ + --hash=sha256:68e62fe11f30d5ca8289242866f0a5291402d8529ca2178ab8afc5c9694ae890 \ + --hash=sha256:6a8dddef476fab96d066d578fc88526767b836ab5ab21754e1d5bf3879c31c7c \ + --hash=sha256:6e192623c49c94421616a5778fba35cf0d5a8d000650c1967ef4448ee5cdd990 \ + --hash=sha256:7225e4514edb64eb6740324353e0da0711954fd8d7da4576755b1c6e09b697cd \ + --hash=sha256:75f80557d1389eddbd0de2681f6a390a0c5338c31ddaa821381c203fc3fd50d9 \ + --hash=sha256:770de9db11e84213beec501cfcaa013b019820ca881e03344dea5844f7876d94 \ + --hash=sha256:7750c6449dff7864bb9bb27ddfb0267756189201a3afc911d82b3caacd70dfc3 \ + --hash=sha256:7bde5e4cc5c10140859842b9d383af292b22639a4dffb725314baf45968cef80 \ + --hash=sha256:7ce713ace7c0e4520535b42b77eaa742c16dab813978064913e5a3cf82973b41 \ + --hash=sha256:7da0c5eff80f0197f3b3d1232ec5a682a9325f4ae9016a78f5f5ca35f9ced1f5 \ + --hash=sha256:7dbb61fe3a7699468030f71bbe5f8a0e326a151daa91beb11a6fc1f980c55e1c \ + --hash=sha256:811bd1e21d32de12efca32393a0ab3f5133b54fce9bd44b8bd77ab07da14bf6a \ + --hash=sha256:8ef53b2de9bcb9197d31854256575d59dbac0cba72ac627bb291ef5eceb74be4 \ + --hash=sha256:937c0052c05a31ca1daf18de3158eed4dbfcb9cc107adbea227728d647be701e \ + --hash=sha256:9d2055050ea716bd38b7f7f1579c275386646b4894c155a3e2f3cd62ed41b7c6 \ + --hash=sha256:9f8d177621de5cb38ee3e731eda45d421db093ec0739f46a5594babda7987a98 \ + --hash=sha256:a2d7755bef5a12ed488f4ef1f1b69ee9191d7396083b755a5d2295f6edb4768b \ + --hash=sha256:a48d62ab9d6f4f98c983223a547af44be6ca3691074c31cecced6facd3ba2dc1 \ + --hash=sha256:a4f00aa42f75d6e4595e8866e748cc1705adc0cddfeb2ca86d0d03993d63ba03 \ + --hash=sha256:a6e721d4b0e45d5b65e87534470e67b18dcd092c83f68fba09f152b9cbc061af \ + --hash=sha256:a730a083190634c65cca36ba5f489531576ebd79bcd5c8e172130f6453127231 \ + --hash=sha256:a931079504ecc49efed7744c476a5c343a92fabf66dec2db95edb1b2fdc770e2 \ + --hash=sha256:aa9511c62d14da7aacc9b4bf51f3f697a621e83b2d6919008243c3aad168eea3 \ + --hash=sha256:ab36d55f9ed2d067327667c2fea18dda018eb628dd6347aa01dda6cf1f5d3836 \ + --hash=sha256:ad2c86c495b899d862ea0f4b42891b8713a3bd45dd4105c7fd51c2a72f39f3a5 \ + --hash=sha256:aeae0e330c9f6acd681f647d46cefd30c29f93e3392882e792e82080c9691399 \ + --hash=sha256:b0431303acaea1089ad4b3e9ce4e6518193def1118d4073ca848635ee4ea2e96 \ + --hash=sha256:b5bdfd1c873d4e093aabc0ca84c4ca6dbc4f752afb5c86f146d9742580c9da2e \ + --hash=sha256:baed1e86cc735622097354b9d1281406caf42ff42a886d29faa8e8d1630333be \ + --hash=sha256:c1453022f490d2459a11819d83ad1d586e9ff65a12ac3e705ffebd46d3685dcf \ + --hash=sha256:c26608d2222fb1e94487e4a387d85f13eb55d5ed725cb25a0c589ac4ee60e7bc \ + --hash=sha256:c7659f22557c5a0bc4855cd635f55edec690cc008a40768527762cb9fb263455 \ + --hash=sha256:c8c69575568085ba0b1b10c0249d779a214aea6f6522e949a0fc9fb0fcb449d0 \ + --hash=sha256:c8d2c9fd1f2d16f780d15127abb050d13d1a76c03a4bd87d7e4980e45e511e12 \ + --hash=sha256:ca82be1a1d406ecfe1d25dc16cb33488e5a16bf4438c9fb590484ea29d92478b \ + --hash=sha256:cc572dace3f60ef98d7b12ff411d20f5362feb31a0439eab0085bbfd349982d7 \ + --hash=sha256:d18e5ac0f2f03f4f518d3e23db0f0cad7faa1da8620e9c09461d443bbf6e6692 \ + --hash=sha256:d28630f5854ab07ab1fd4aba756de52326c82e6be15d414b12793f1975048b54 \ + --hash=sha256:d9c275eaacd24aa73f94ffd6de08fc3f932424d8b6c376f4bed7cde376fe7bc3 \ + --hash=sha256:da0e573f9f97159390c89d9f1a9e41908b66d408cc5b58d08cf3847d844c531b \ + --hash=sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be \ + --hash=sha256:dddad92b554513a31f272570678ba307fb9f618f05e3d4a5eacafff9eae03e1d \ + --hash=sha256:df423d40ee8654634421812bc3b196da3f9bd7d32929da813f8394c4348a5358 \ + --hash=sha256:df913725b79db7bcf03448f36b7bf8815363417d5b58deecf9305e3e30f0f21a \ + --hash=sha256:e0bcb7e0f677f543555d2adff3bf19c05f66cdb4796e5ff602442ab2fe3c4ef7 \ + --hash=sha256:e2d65b31f36619cda3999b78b2aa9632e76b78448e7a56fc4240824200e7c4fc \ + --hash=sha256:e6e8cff14d6fb0be70a09c0bdc58096f501952d04624ebf867e0e56da2df8960 \ + --hash=sha256:f16c709686a78c727bbbf059f92b0bf41c6fc60deec706d2dc19f529175a6125 \ + --hash=sha256:f24fb43132a4c6b4cb4eb029492919b2db645be6808d738f244fd146c03c32cb \ + --hash=sha256:f53e442b08449d42821fa4a4fba000095af9f62742a500f978a9f557ec44339a \ + --hash=sha256:f5cfbc5fe74540d335175b656c725d74d90e3730c626d92575eea35029d9afaa \ + --hash=sha256:f81b3b8f3d4e343550fa4baa0e479bba9f2d29ce9c2e9b51d1ce1718d7442fcf \ + --hash=sha256:f8ec5e643a9a937f64e1999eb9f75d072263751912dc5cd06d3c85f8f44be7c3 \ + --hash=sha256:fb92203a88b3d3053034db775110081c49d28be6551923805e039924093761e4 \ + --hash=sha256:fcd22650c908d7b7da162bbfaab594a1227a15d1643a98c68b122ac642fa2264 + # via cryptography +charset-normalizer==3.5.1 \ + --hash=sha256:00668ebb0609751758682eb0b5857e7c35b9f00e84dfdef062e103244ec94d45 \ + --hash=sha256:012a22b88a77ca2e59b98ac5889b0deb604147666032f45e6d6e217634d2550d \ + --hash=sha256:01e93745f7f219b703b60ba7afead36cfc4242782be5af484673fc500df12da5 \ + --hash=sha256:04368edf83514385ffc3e1cfd4546e595f4f1272dd23ba437a93a9cc3741d47b \ + --hash=sha256:0722590aabf9dc6a6c0343d523c05458fa2b5047dbe6302fd526bb570600753f \ + --hash=sha256:07ffd07412fc5d5e84cd8952acf9ff7e4ed7a708e69d1bada19d8ba91711353f \ + --hash=sha256:09a7bba9f739468c8e78c36a75c33768e53cb1959fc638f510454c14683f00d5 \ + --hash=sha256:0b2b1b3fa5670c127b246df1d0c059defd41f689a868a3b9d79df9b1cac42d22 \ + --hash=sha256:0c6dfb5ca6723eeed15aa8e564a014d69fcb8812f94eef11fe3631e0508199f5 \ + --hash=sha256:0d929fc574b4d6fd9e7c0f5c2ede8716a41911923aa7fa5fce38e0818aa4a1ac \ + --hash=sha256:13e3afe97712e8887cd516e960c63f0b93122971e5b5e4b2622fe7701771e838 \ + --hash=sha256:15f024313246a4ed976c60f440bb8d257815513a681d212ff74fd46f7d715a90 \ + --hash=sha256:195ce897c6153c0700078142cf8efe3e6454ca4cf4357499e4078dfd83396626 \ + --hash=sha256:19a3dd5aa73cef1c99687c4fc57db016a9c17104ae1185da88ba566a5d3bebe4 \ + --hash=sha256:1d1c7a53a6c2103925cdd6d7229f8c567379f211c869793df679f2e9f738c369 \ + --hash=sha256:1f5883d77fd409a261abb5dc8ccbe335720d798b1de4abb3b1d47ccbbc76b53b \ + --hash=sha256:21b82d8082f6f5e7f456ef0bd16323d08de1266efbfeb476e64b2a91d1471a4e \ + --hash=sha256:252d099029bcbea642f2a06c4ed5046bdf8b5a8150b64afa5e027e88b106e5ee \ + --hash=sha256:256dd4d85d9e4dc595e2bc983c980e73f62ddeb3165c58b4c3dfe78c5c8548c1 \ + --hash=sha256:26422d45fd13551cf564c58932f7d72b4f58b93b0fcf18c35ba6be12b46bb102 \ + --hash=sha256:2679de311c7946dde5d3b6f44941844133ff5c7cb86099c0061ab1e8901c20a8 \ + --hash=sha256:29880d17a8eb0b5cfdfd8944b468322928059aa35f1f5fa8ff22b149ec0b42f8 \ + --hash=sha256:2bced4061f000f7187254a02ad3433ae17eaf991747ceea2f478422590a5bba9 \ + --hash=sha256:2e9cf9253119d8e5d111f05d71626786fd3d6193817316eab1ca088cdb8593cf \ + --hash=sha256:2f06b7eae9dbe77fe1d644ca244dad508de8d302870a43f3c559b521270938a0 \ + --hash=sha256:2f293479cce755c75f1697e87c409b7ae4c555c7dfecb6e988ad13abba943031 \ + --hash=sha256:329fc3ccb63ad22d867d84c2adea759a64079a37ba4a343433b02c7a2816871e \ + --hash=sha256:343fb4f2821043bd87095f7b08a1a181febc8e36ac64212143bbfd0a0e1bc235 \ + --hash=sha256:3588e376b3ea2eea84976f67273d679f229e24c66dce7b82ae45aef04ff6e072 \ + --hash=sha256:35aea775dc2bd5f54cd84a1cd2696cc3207c479cb9cf0bd346f0d343e4300ddb \ + --hash=sha256:35fe081843b35aad20ffeccec3eeffbe637b15d14f3fb22cc1b59cd8ec17e93c \ + --hash=sha256:36047af20e17097c3bb9476c2b7655f2f7aa51322c0ba58c07695bedf755a950 \ + --hash=sha256:3617ac3cfd8b9888f145ad89dd6e692285834b0201c6074a5eeaad3fd4d668c2 \ + --hash=sha256:366ec70f5547c640d3ce1985722490f23faf4eb5216a7eeba78277490e78dacb \ + --hash=sha256:394fea06235c8543390050ed5f529187074b029fb027213f6c46ac11ab5d950e \ + --hash=sha256:3d27167433c0d5f18dc850f07d0b3816221984fecdc405d6c157a6f0b8f8e9e6 \ + --hash=sha256:3e5e1224c0a6a90e05843e07adfec669edebec17801c67072f51e59561d63c0b \ + --hash=sha256:41876ee62a3dddf48ff1121ad8f0798032aa03f2fd35f21f34a4cab14f18d8d2 \ + --hash=sha256:433c5a81eade63b47e522303bad236f59dba55ea6951746f5558355eeed8c75d \ + --hash=sha256:4582c27e8c889d64811987b5967fbd3ae0c823fe1fd933b543d55ac20bb475fa \ + --hash=sha256:485a0d363cafefcd2538a73c7c838daa2035f09b2c9f9b5e3133f80c6aeb84c2 \ + --hash=sha256:494b70049a4d69aec6e8137c13af4cf8db8c9f9820a1392ac293b0dd2987a818 \ + --hash=sha256:496846868fea80e479324862fa877f02411f2fd0f83b79ccee2607aa68b2a032 \ + --hash=sha256:4abdc5f9ad448c1ecbfae2974b820535d6bc6e7eef63babbab3d81cf46968c71 \ + --hash=sha256:4b599739b93b2cbeded49645ae3c8d1405c29ddfbceac1545c87a3f9580a9e96 \ + --hash=sha256:4bea7f8ebe90bbd7f0e4a2de42ca6924ba23e3e76418c408ff82f1d46fabd687 \ + --hash=sha256:4c4fb141a727957c93edfe5c32a26ceb6b5f6461d67146e2d39f51e16170bea8 \ + --hash=sha256:4c9548dc78002099910abaebc0a72ac58b7d30931869e0351c09b507dff4ece3 \ + --hash=sha256:4d26f14f041e83dd8edfd61f4cd4fa7285d31798b5bf1f28e70c367ba6c41d61 \ + --hash=sha256:4f298bdadb8f0b9e5672877f647d1be9373ef5320c9e2f049795e26cad28b6a9 \ + --hash=sha256:52ec005752a56ae79547a05c0139ca2501a0c866390b6115008456b9f0e7cde1 \ + --hash=sha256:55261ac0d2941c42f196dd576f543d87a8ee03cd6f5e30dfb4d807b2e3b9121a \ + --hash=sha256:56490c595a28b1bb27dfc583e816152a9767721ef58b2c03b13f954d2f707420 \ + --hash=sha256:58d3e12c88e0950bca850ae1f7c256055c097639c2edb9eb123af9807d8b15e4 \ + --hash=sha256:58d4aa13a59c969dbfdf9e6a9560e242cbfd9e8a8f50c2747714df1a423adf65 \ + --hash=sha256:59171c6e45bf07d0d5cab3b0bf81d945035530f6873398b3b531c31184d46663 \ + --hash=sha256:5b6d1386bf0096d26d3a863dc0a487a5b4eb9aa93cf5ba69683d29dde6b9d60f \ + --hash=sha256:5c0ea61a470e070686aa30892fed79e297d2c8d0ab46b8bcdf027d38c51da591 \ + --hash=sha256:5c84bec0ab5ae0c64bfe73a7d2adcb5ce73b467523fc27fd6a28ab2aa6cbe35a \ + --hash=sha256:5ca0555312ae2fe82715cada7fac375530c2f3349e1eaa1bcb33d0283ac79a18 \ + --hash=sha256:5d8531a6569d025f68e2321e7638fb7978f23db58e5f69f56913837aae03816e \ + --hash=sha256:5e2d0e146dcb57034f8b97dc58d2d512cb90aba253960ce449f695fec6a82c6f \ + --hash=sha256:5fc45d653ea8c9a20479167e11d4a0f8cb2fa3470737ab6f9c827532313187b7 \ + --hash=sha256:6117b84ea48435e5356dc737f5121485c30920ba43375fa7b434fd753df0eac3 \ + --hash=sha256:6199d5606e2bbf2b096cf64d03f8b6790c91081d5ac866b8e7bb6422738cc60c \ + --hash=sha256:62b55f6722735a6c472f88361cde6640608773d9443cebdbb51abf436a1fcdd3 \ + --hash=sha256:687c9ca3035544b113bea2055e180af96fb63c0c476e22a9180f51925186e7b7 \ + --hash=sha256:6b7430cf5728e68f6c462254009a6ef4086e1bea43cf2f57aa9c55fb4f50ff96 \ + --hash=sha256:6ba32c4d2abf1d2fe7cf27d280f4cca5664233b0f885549c7761719eb977f486 \ + --hash=sha256:6c9cdde8becb25a7fde49924511aa2644d6f8081cc8df8e9452724303348d8e3 \ + --hash=sha256:6df0ec430f9a831772c23ca5a224cba36517a58a84bb32c32bb59a9fa67c47f6 \ + --hash=sha256:6e2912d4babbc65196ac13c2f53468dc57fb8b9c25ef913e8c59ddf7c6dc0e1b \ + --hash=sha256:6e5e4d73d588ca5ed09df1b7dcd1b203d1df3c542e3f50d126c947d432b10731 \ + --hash=sha256:70055ff39b97c99e7ae40ea3e393fb62aa2e44dbd9b29f8d14f42fb0025c3959 \ + --hash=sha256:706bfd38730a5ac7a365793269a00f4e988178cec121391f4248d84ad8c972e9 \ + --hash=sha256:7235dc28fc6dd9d832ac7c7bce95367dedb85929f17368a0c2bee1e080b9acbf \ + --hash=sha256:774d157f112367ff4abd29019f38f023c24e00e56edc7829c20e358a5a913ad8 \ + --hash=sha256:77efcff2b23071c349402ac1066667a3d011f62398d81408c9b88ad991747c9e \ + --hash=sha256:789b8982559ae28dad2356519f841655756cdcd96616410590ae0b17454ee64f \ + --hash=sha256:7ac76cf9afd34929d76eb7fcb63be476a4853d8a96f0dcf2d0db68a0cbdf9885 \ + --hash=sha256:7c0c10730342b0c9b35dd1d619beb8214e520bd96a1f870f452680b238aab3e0 \ + --hash=sha256:823f82903d189af463d7df250ef1f7f696f3cee08cc8d91deb565e8d425f6506 \ + --hash=sha256:838648accb3a7fd9803fd45c87bce8509648eb0c11bc34e216141300977244f2 \ + --hash=sha256:854066be00447fa8de2ccbbe893e2ffc4b123ef16d897af794c1e18bd4a714b0 \ + --hash=sha256:85d5855daafc240cc045c026d7a15fd198a09b0fc8ff6f5ecbb5297b509cb11e \ + --hash=sha256:85de3134b5379856e323ba37c19c9256d39425f7b76a63af52b09fb4664c2e8f \ + --hash=sha256:87e4f41d375c0b9be2fb5251aee4b8a689169e134535aed81bf085c3b647451e \ + --hash=sha256:88ca277405c2d3b71c4e1c2ee0e7966e807bcba86a69d11e19ba199d18ae4491 \ + --hash=sha256:88e85ab89cb822c1e635f51d6d32e488f94e002e70e2f492bdb8b945543f345a \ + --hash=sha256:8ac8c94b6539074e0f40899301273ac8402b9b3e01c7b7ba269ff30340aaaf20 \ + --hash=sha256:8fe532b3c966d1fb794e0698e4589d0444017ae77fc0b31edea13c0e35bcc449 \ + --hash=sha256:9085f87b0e38a2b92b8923059b4e8789fe40d9279712d15dcc670048d77079af \ + --hash=sha256:90b7481fb62fbe172c558bc6fd1c4c98d82004a54a7551f20e11ac9bf0b8708c \ + --hash=sha256:92caef967d287a407085d61176fce4012b1dd62daed4eb6d5ceb26d3d2538712 \ + --hash=sha256:9362dd90aa7dab48c0054a21187791ccf05473f7dba5d92b8033ae62164675e7 \ + --hash=sha256:94d78ecec2605a8d0398b0f365d5f12a63248438516f5dac536a5eff7337df4a \ + --hash=sha256:94fbf1c0c6cc0d3d5e50f9a9313a8cdca90dd696d34b381cd1704f8c9e939f20 \ + --hash=sha256:950f23cb393f85543777b0433f082cddd25b51ab398eac7971146495679efe5f \ + --hash=sha256:96eefc178f8636b9c760c5829345307fd81cfae9ab1e80997dbddeb0f54ee9a3 \ + --hash=sha256:96fef3e886d6a9874b14f27fc193fbdc69d5d8035783d86aa4e1cea594e695f9 \ + --hash=sha256:977cdbd483a9cff38179bea4fd754289a6f2195c7abd414aba85410b3e66cc5e \ + --hash=sha256:978eab16f55b4ab2c2a745be9a0a840bf8f09a7f227d9c76eb30214d078865a5 \ + --hash=sha256:994e883d17c559cdfd38c84003c8b27d25424a1077272a17e7cd27bfe0bf57b2 \ + --hash=sha256:9ac4444d8d4fd4c4bd08bf451ed3167aa9e7ec6cdb41b648794f1d1103652e36 \ + --hash=sha256:9b5db6052055d34d41230fb78d7c439c23dc536a9896f6cb039e8dd92cfc1263 \ + --hash=sha256:9d9a0dc7cbe9bec24c3f767c9122c41fe5a1bc43f47cd099d00d393e09769de4 \ + --hash=sha256:9dbdd9205662134957cf0c324f639bdc5031c0ca056e2369e238db75187c0f11 \ + --hash=sha256:9eea3ab2597a5e65fe65296e2d6a84570845a6b55532d90333d740d48bbc850a \ + --hash=sha256:a2028475ba855475b8b4d3cfeb4994269c967aea8b9892dfba907f4263a863a3 \ + --hash=sha256:a3a370082ce34d0612f421e15fe011c53bb1feff21a26d06ad4fb244dab5a375 \ + --hash=sha256:a545775cfe815855ea32d7c27731d79da358ef2055b4a25830231b1622dd18aa \ + --hash=sha256:a5cbd90ecf0fc62e64726917ad083b73001f0563657a87ec3c0b504e277dc90d \ + --hash=sha256:a6d095662e73e74f0a49988e0593373e243e3a52e27bfeea0a859e88acf4a0f5 \ + --hash=sha256:a6dac12ff6b846103483683f60c5f8fee205121adc58ffd87e90a90a3af69e99 \ + --hash=sha256:a951ad59cad9145664a730d3036b40b844e74d2d3683da40111463cd3a83845d \ + --hash=sha256:aa1099b956fb795e686d073568f6dc002a0bb89765ea6d5b055dd7d9bf1b116c \ + --hash=sha256:aa2bb0b37202dca27175591f761108b5d34096ade1191ffe4808bdf6b1571488 \ + --hash=sha256:aae2ee51122d3ae968a3837d97dc24a0aeebb0dea23694422cd172bd30017cd6 \ + --hash=sha256:ab743e9bc90c1f73552ec33e10e3331315acd2c397b36065b591b0181de533cc \ + --hash=sha256:ac00177c4831ffa650f8609e4bdddd5fe09c03b1c0c47acece7e6ea20421598b \ + --hash=sha256:ac13b004224fb341e1e25a1ed5e19d32f57cdb2a403e01f003b46f051a550f6f \ + --hash=sha256:acaf604462bf330b0d07e7a07c1d6e4adac79e5fb13e9c5140590542cafacc00 \ + --hash=sha256:ae31a1a1db2ee6cc2942fccaf695c934bc7f3db9f2133a3fef1f367cf1a4ab10 \ + --hash=sha256:ae4a097991662cd4fff0ddc74e0fe7874f82e00042fa0ea00855645ed0c79598 \ + --hash=sha256:aea996a6aba25260827c9ea511d1addfde2da9eb686ac961838509086188b7e6 \ + --hash=sha256:b39b69b347e5e47a3b5b8cfc005c68c1ba347474e3960236c4944a8ecd174962 \ + --hash=sha256:b54e7e13267d49ffbfe68e25b3cbd774dab38fa37238f71265e91b36146eb21c \ + --hash=sha256:b9af956078716df40d985fb0dfeb2c2120c5ca92ba4ff4b388acfd01cdc14d08 \ + --hash=sha256:ba2f37ee79e6338845261a3c5b1784e5d1acdff2c0785b284f1b633033d136ab \ + --hash=sha256:ba501e667c17d8411f98e67a022d9604ef179aff0e459b7e292c796837c13573 \ + --hash=sha256:baf3775a2635e5a11fbd5e4e64ee69c7e86875d224a5c72aca4c141064589a90 \ + --hash=sha256:bb57753e36e4855b8ca375069482250a6246372331a3e4f3407eaebb007443f5 \ + --hash=sha256:bd6c173f04743d483881bffa1478d5a4624475b8cd1d2194956a75548e191c18 \ + --hash=sha256:be47f99644b208bff7766314013f9acf57b056b04191d570d68ad14022cf5b1d \ + --hash=sha256:c010f5581d9c612804cc59fcf7b524b707fbcb72828551237ab545bb5c7034af \ + --hash=sha256:c1dcc36dcb96abc02236e182d17e0f71430152a6c2c7447421da2d2dc144edea \ + --hash=sha256:c428c6c31eb5f4277d7f8eccaf767fbd548ddd5ce3c8b4f4cbbfab3d96b5904c \ + --hash=sha256:c658c50ac0c98cd755a2dd50b7977d3bca7df401dcc47fbdfa87db53ef7d4e8b \ + --hash=sha256:c71fb0d56c920c269cd3e2e3fe7c610e3f1fdb21a6ce60efa6430ff63676cea6 \ + --hash=sha256:c7b742bf31c88566b4bb6335a7f393bb322e580b6bb98df7bd0c25e6e3519ce8 \ + --hash=sha256:cc0329df4caaceb950d2f580b5ac716a377f7059624a0bafaeaf8a218c6ed774 \ + --hash=sha256:cc5d36d96478aa9c60654bd932525bf32964c62a7281eafdf16d85003a8d6004 \ + --hash=sha256:ce854f5f478050ade5a238731c4ca985a7d3b3cb53ff600a9b5c3b689b5f0a7a \ + --hash=sha256:ced3fdd71aaa83ce593746c2edb42b7a59cb4c19c8b5c407781c72e493aae55a \ + --hash=sha256:cee5dd7c6fb5dd52a0fe2a740f9bc6e3593f5f8b1788bde49de02086f30182b2 \ + --hash=sha256:cfa1c0cc3a8f9f53f1243a5a99ac36fd003880199383b37672e86ddda9cb07e2 \ + --hash=sha256:d1ee1e296209fdce05b81b663250eefa02213a2da7b41bf26f7829b8ba3545aa \ + --hash=sha256:d59b75732e9b6f27388e10c14b0259cc5f2e48c78627d185e6a177b58ad3cffe \ + --hash=sha256:d63600d620ad0064c3a748b950ac5ea38a80190e5498532efefa4b7b3f1da1f3 \ + --hash=sha256:dd732602a7009217f658d5863d12d79d373a4de0eebc111094bcdd3bb8e0a6cc \ + --hash=sha256:e06efa066f7dbadbc84ebc126a97c452a6451dfcf589d89d788484949e1cf795 \ + --hash=sha256:e199fb99720074809a7720f1c0b4d919eea8b87e88713e0f8f602f7bef543d9d \ + --hash=sha256:e4b018dc5a0eee4676e38fe84a47a427816c590b93b55d9025274ec4d6ffc2dc \ + --hash=sha256:e6621fb2a4988d6e53eedc455e5903e2679f3967b8acb3d639f1b63c14a2e893 \ + --hash=sha256:e71c909f353863b2b89c83de2ebed71ea6d0df8a6ef65a128193c5e650766bef \ + --hash=sha256:e90251c0c7bdd54a100a0dce3c07b7e637278c93af29dbf78ebb89a58c4bac7d \ + --hash=sha256:e9fbdce1e47394b09bc9f26ab117dfc8d6491977a11d86f592bb42c779db2fda \ + --hash=sha256:eb12fb2ba69ffa05f8695f61c69e591dc4b4a12ac3757ac8af8adb259bf56d17 \ + --hash=sha256:eda059b6bc8bc0812d626fd91a7ce01bf583df0a61296eff390fd94141a34e30 \ + --hash=sha256:f03ac127268b43ef4fe9e6ab6794a6794b49485a0cc0c1db79876d2f33f75bc7 \ + --hash=sha256:f298e218441525d3794428b4c8b8fb8662c6d3ea79925d4807ee6b9a96a3bca5 \ + --hash=sha256:f5542f9b941279d82d41eb0aa9f98eba36fe4df5c7086c651df7944935b37182 \ + --hash=sha256:f6f7deae3feb4edfa2efaf7c574fe88cbf055038a6abdb40188e4fff66d5699f \ + --hash=sha256:f9b1e28d0e8dbfa858abdba91d6b547beaf2df1a59bec6da6faae7b96a4991a9 \ + --hash=sha256:f9f8405c2c758532c74fed975dbee57be1f31a6e865c031870c79a6ed3212ada \ + --hash=sha256:fa48b1b63d639f9483e0633e092f5851e2348c352f1f9bb6c8182f87884ef876 \ + --hash=sha256:fb78f6e7fcd8ad785d28cd577168bc1aaee827b25bb8755638f694794ea98f0a \ + --hash=sha256:fbc597639158fd7c14d55e808718848319540f51b0e6746e3eefa59723a4a348 \ + --hash=sha256:fce8cbd4997efeb450bd298b54f755dcdff18d496f7a5ddbb4867c6d7c88fdc3 \ + --hash=sha256:fd0350afdc3aabd5576f60ea109228bd5538139713c7b094c5cd27c73a98bc6f \ + --hash=sha256:fd0a274c0e5f9a21565cd9d3dd749b61f96b7aa1e20a93aa1ba4029518f2e5c0 \ + --hash=sha256:fdb8a068947befafba9952162645dc2fecaeb400e64584829ed5e9b2fbe21a7f + # via requests +cheroot==11.1.2 \ + --hash=sha256:0f6c0ba05c00fbc869fb46b1de4ec2384e1d85418ae963d3bc10ae83b688dbfa \ + --hash=sha256:bfb70c49663f63b0440f2b54dbc6b0d1650e56dfe4e2641f59b2c6f727b44aca + # via xprof +cryptography==50.0.1 \ + --hash=sha256:01f41478cf33fc605a6a089cd56d28b45c6c0b45a1928b61797f2621a04bac71 \ + --hash=sha256:05ba322c4da95b262a212c345af888ef2c37c88c0509756ea00a0e6d68850f23 \ + --hash=sha256:16c5ecd954b3330ebfb6605eca4fd952da8bef376551d5cc264534e3770a9ee6 \ + --hash=sha256:2a93d05e34d5f67fba6f891fe85d929999baa7195e853923ea6d7576c9e68c5e \ + --hash=sha256:2b34d76a652ea2b6faf777c35df230c5637842cd904e04f16230c3f9f03e4361 \ + --hash=sha256:2ebbfb0f1fed745e91796e3e1080a1440423fdae8ece1b995a1d80883a409054 \ + --hash=sha256:30a125032e5642a21ff816e021152bd4e7e94f03eff3f4b7fca41cd22bc3110f \ + --hash=sha256:330fbb252391c596f1ae42c5754449dc924e6ad012dca8efe0d703f9f2d12ec6 \ + --hash=sha256:359e62deae718bce96170e223fdcb6357e4fbd3bb7a3a75f4430763532560e49 \ + --hash=sha256:407fe2b6db00939c05c0e945e9914238f2f0a430974839429dafc82b1ee6bee5 \ + --hash=sha256:42be3bb70596b3abe4ac097b75be223e8b3ab614a0e5de068e3dcc54d71d6149 \ + --hash=sha256:4c4188f7c0cf655be5c06342b817ed0f9595b69ffa2b12026e5353eed29dea88 \ + --hash=sha256:51593d180cf6d179bde5c5d065bed81386b1f381656ae7d042b7ffc87a9895ad \ + --hash=sha256:51afcfceb15597cf2635068e4ac9a56b2abde622edde17f37d85fd7b5306497a \ + --hash=sha256:53e279950892dc102c6b4e52af03ae5ea92fac572a1ddab78ca73a997f62b69f \ + --hash=sha256:55d16b1ef3ee0958d893a977b19777887e546c9954ea81b200c3301a864013f2 \ + --hash=sha256:5dd9bda1c12b4162f6ff568eeb5e0ff956c28d14406e875cfe8a63a2d414ff20 \ + --hash=sha256:5fe002589592ed749ce77fe0695fcbd3500dd61d7d6db5858a7544c612fa8e45 \ + --hash=sha256:5fe939deeb161024a6be98229c953b6591fef1f41214497a78fe793a244c017f \ + --hash=sha256:693c99b49bd37d0d096e4334c10232c77248c415b98d35236094cdf96d57258b \ + --hash=sha256:76de83fbd91ac49c0feaaa983d0748fd7a53176afac5fb3bf7478d244f0eb527 \ + --hash=sha256:79bf008d1f9af6071c797ad133e39915dfee7614f18f18f4db9072eb715064a3 \ + --hash=sha256:804728ce710890870f3aaa344b2e161172d258d768ac139d02cfd9092d0d94e6 \ + --hash=sha256:8921d58f426793c5f1b47f0b59575780de9a095214958d0eb37d909593db8367 \ + --hash=sha256:8df2de9102026855887e4587084f6eabd80ed0f345b8ad8a7ac27ab9bf4723e0 \ + --hash=sha256:9cb3cb952cf5a8abd50c782a98a89d71699715e802fe349704b47f2425b42a94 \ + --hash=sha256:9dde0a357190eb3b1da1bb9ab750e9c85cba82ca5977aa0836cbb94e92611239 \ + --hash=sha256:9ebcdd5519be9b652a46f507817a74591774fc3d6923ac364e4dfa64e36b291b \ + --hash=sha256:a0b1a59e3a089064a0ec309e9428c8e3ae4e161419d20ac33600767e83fc658a \ + --hash=sha256:a255449073358275b64b67d3f595f268bbef70e72b6edb65e0c70c735bf739c9 \ + --hash=sha256:a8f40ea47330e71b594a7e246898f93177c259490c63183dbaf9e571d71ed9a5 \ + --hash=sha256:ac02b07824d4d1001bd4367599f839c19cb171924c796e52c23508ac14c2c0cc \ + --hash=sha256:aed8db4f6d71c51efb89530e12d9464e7bf2923d46c3205dc794a2a93f8c0648 \ + --hash=sha256:b8f852c65863251b9e3a1b8c150ce21e59b522dbb6a7d4bc80e680d38388e986 \ + --hash=sha256:be224a65493ec5b74a158ff22a5522ce4a5ca1e543c647a3a4730d4a09e5f959 \ + --hash=sha256:ca83d00d9e69cd5eb63f2e69c3a5a59e0cecae5ae14c6ae0b35830fe3b37bad0 \ + --hash=sha256:cbf74a81765ee67413503ca6e26dcc4f6f5a519822436cc0a1b97aab6c1b8a17 \ + --hash=sha256:d63ae8f6481fec907ac0f588eee8a90aefde112c633131fe540e5711ddbb5a4e \ + --hash=sha256:e22dfed744bd4002e909464cb23d2f0b05c6f3113a79ef2e9864a53db737c733 \ + --hash=sha256:e2ca8fd1b6b4b82a1c4cb02841d0837e3c12336c2e24b520ab8ab3b969733d8f \ + --hash=sha256:e74591e283fe6eb956416c929eb58262a719fe0311fd9054c62c3350ed8760d8 \ + --hash=sha256:f74455bb086a85d5e81246412602aaa97ed095e504cd40dd261ef50be42205bf \ + --hash=sha256:fb4b9672d389c738b175c4166e78310f8a70358886aacd9173ee03a85ffdc671 \ + --hash=sha256:fc3ed7ebd2a8c96f5b166de0ab9b624996bef3b07bbeb19364dfb78222c22c80 \ + --hash=sha256:fd3718b960d0b5dd213cdf03f3bcb7000e69dda0de8b956061947ff6bcff5558 \ + --hash=sha256:ff838d62ec1bfce4f9ba7fa16f4a7b554cd8d0c299e6be37502161a660c84eef + # via google-auth +decorator==5.3.1 \ + --hash=sha256:4cbcdd55a6efadb9dbea26b858f4fb3264567b52d69ca0d25b721b553f60ea82 \ + --hash=sha256:f47fe6fdbd2edd623ecfe36875d37aba411624e2670dd395dddae1358689bb3c + # via gcsfs +etils[epath]==1.14.0 \ + --hash=sha256:8136e7f4c4173cd0af0ca5481c4475152f0b8686192951eefa60ee8711e1ede4 \ + --hash=sha256:b5df7341f54dbe1405a4450b2741207b4a8c279780402b45f87202b94dfc52b4 + # via xprof +fire==0.7.1 \ + --hash=sha256:3b208f05c736de98fb343310d090dcc4d8c78b2a89ea4f32b837c586270a9cbf \ + --hash=sha256:e43fd8a5033a9001e7e2973bab96070694b9f12f2e0ecf96d4683971b5ab1882 + # via xprof +frozenlist==1.8.0 \ + --hash=sha256:0325024fe97f94c41c08872db482cf8ac4800d80e79222c6b0b7b162d5b13686 \ + --hash=sha256:032efa2674356903cd0261c4317a561a6850f3ac864a63fc1583147fb05a79b0 \ + --hash=sha256:03ae967b4e297f58f8c774c7eabcce57fe3c2434817d4385c50661845a058121 \ + --hash=sha256:06be8f67f39c8b1dc671f5d83aaefd3358ae5cdcf8314552c57e7ed3e6475bdd \ + --hash=sha256:073f8bf8becba60aa931eb3bc420b217bb7d5b8f4750e6f8b3be7f3da85d38b7 \ + --hash=sha256:07cdca25a91a4386d2e76ad992916a85038a9b97561bf7a3fd12d5d9ce31870c \ + --hash=sha256:09474e9831bc2b2199fad6da3c14c7b0fbdd377cce9d3d77131be28906cb7d84 \ + --hash=sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d \ + --hash=sha256:0f96534f8bfebc1a394209427d0f8a63d343c9779cda6fc25e8e121b5fd8555b \ + --hash=sha256:102e6314ca4da683dca92e3b1355490fed5f313b768500084fbe6371fddfdb79 \ + --hash=sha256:11847b53d722050808926e785df837353bd4d75f1d494377e59b23594d834967 \ + --hash=sha256:119fb2a1bd47307e899c2fac7f28e85b9a543864df47aa7ec9d3c1b4545f096f \ + --hash=sha256:13d23a45c4cebade99340c4165bd90eeb4a56c6d8a9d8aa49568cac19a6d0dc4 \ + --hash=sha256:154e55ec0655291b5dd1b8731c637ecdb50975a2ae70c606d100750a540082f7 \ + --hash=sha256:168c0969a329b416119507ba30b9ea13688fafffac1b7822802537569a1cb0ef \ + --hash=sha256:17c883ab0ab67200b5f964d2b9ed6b00971917d5d8a92df149dc2c9779208ee9 \ + --hash=sha256:1a7607e17ad33361677adcd1443edf6f5da0ce5e5377b798fba20fae194825f3 \ + --hash=sha256:1a7fa382a4a223773ed64242dbe1c9c326ec09457e6b8428efb4118c685c3dfd \ + --hash=sha256:1aa77cb5697069af47472e39612976ed05343ff2e84a3dcf15437b232cbfd087 \ + --hash=sha256:1b9290cf81e95e93fdf90548ce9d3c1211cf574b8e3f4b3b7cb0537cf2227068 \ + --hash=sha256:20e63c9493d33ee48536600d1a5c95eefc870cd71e7ab037763d1fbb89cc51e7 \ + --hash=sha256:21900c48ae04d13d416f0e1e0c4d81f7931f73a9dfa0b7a8746fb2fe7dd970ed \ + --hash=sha256:229bf37d2e4acdaf808fd3f06e854a4a7a3661e871b10dc1f8f1896a3b05f18b \ + --hash=sha256:2552f44204b744fba866e573be4c1f9048d6a324dfe14475103fd51613eb1d1f \ + --hash=sha256:27c6e8077956cf73eadd514be8fb04d77fc946a7fe9f7fe167648b0b9085cc25 \ + --hash=sha256:28bd570e8e189d7f7b001966435f9dac6718324b5be2990ac496cf1ea9ddb7fe \ + --hash=sha256:294e487f9ec720bd8ffcebc99d575f7eff3568a08a253d1ee1a0378754b74143 \ + --hash=sha256:29548f9b5b5e3460ce7378144c3010363d8035cea44bc0bf02d57f5a685e084e \ + --hash=sha256:2c5dcbbc55383e5883246d11fd179782a9d07a986c40f49abe89ddf865913930 \ + --hash=sha256:2dc43a022e555de94c3b68a4ef0b11c4f747d12c024a520c7101709a2144fb37 \ + --hash=sha256:2f05983daecab868a31e1da44462873306d3cbfd76d1f0b5b69c473d21dbb128 \ + --hash=sha256:33139dc858c580ea50e7e60a1b0ea003efa1fd42e6ec7fdbad78fff65fad2fd2 \ + --hash=sha256:332db6b2563333c5671fecacd085141b5800cb866be16d5e3eb15a2086476675 \ + --hash=sha256:33f48f51a446114bc5d251fb2954ab0164d5be02ad3382abcbfe07e2531d650f \ + --hash=sha256:34187385b08f866104f0c0617404c8eb08165ab1272e884abc89c112e9c00746 \ + --hash=sha256:342c97bf697ac5480c0a7ec73cd700ecfa5a8a40ac923bd035484616efecc2df \ + --hash=sha256:3462dd9475af2025c31cc61be6652dfa25cbfb56cbbf52f4ccfe029f38decaf8 \ + --hash=sha256:39ecbc32f1390387d2aa4f5a995e465e9e2f79ba3adcac92d68e3e0afae6657c \ + --hash=sha256:3e0761f4d1a44f1d1a47996511752cf3dcec5bbdd9cc2b4fe595caf97754b7a0 \ + --hash=sha256:3ede829ed8d842f6cd48fc7081d7a41001a56f1f38603f9d49bf3020d59a31ad \ + --hash=sha256:3ef2d026f16a2b1866e1d86fc4e1291e1ed8a387b2c333809419a2f8b3a77b82 \ + --hash=sha256:405e8fe955c2280ce66428b3ca55e12b3c4e9c336fb2103a4937e891c69a4a29 \ + --hash=sha256:42145cd2748ca39f32801dad54aeea10039da6f86e303659db90db1c4b614c8c \ + --hash=sha256:4314debad13beb564b708b4a496020e5306c7333fa9a3ab90374169a20ffab30 \ + --hash=sha256:433403ae80709741ce34038da08511d4a77062aa924baf411ef73d1146e74faf \ + --hash=sha256:44389d135b3ff43ba8cc89ff7f51f5a0bb6b63d829c8300f79a2fe4fe61bcc62 \ + --hash=sha256:48e6d3f4ec5c7273dfe83ff27c91083c6c9065af655dc2684d2c200c94308bb5 \ + --hash=sha256:494a5952b1c597ba44e0e78113a7266e656b9794eec897b19ead706bd7074383 \ + --hash=sha256:4970ece02dbc8c3a92fcc5228e36a3e933a01a999f7094ff7c23fbd2beeaa67c \ + --hash=sha256:4e0c11f2cc6717e0a741f84a527c52616140741cd812a50422f83dc31749fb52 \ + --hash=sha256:50066c3997d0091c411a66e710f4e11752251e6d2d73d70d8d5d4c76442a199d \ + --hash=sha256:517279f58009d0b1f2e7c1b130b377a349405da3f7621ed6bfae50b10adf20c1 \ + --hash=sha256:54b2077180eb7f83dd52c40b2750d0a9f175e06a42e3213ce047219de902717a \ + --hash=sha256:5500ef82073f599ac84d888e3a8c1f77ac831183244bfd7f11eaa0289fb30714 \ + --hash=sha256:581ef5194c48035a7de2aefc72ac6539823bb71508189e5de01d60c9dcd5fa65 \ + --hash=sha256:59a6a5876ca59d1b63af8cd5e7ffffb024c3dc1e9cf9301b21a2e76286505c95 \ + --hash=sha256:5a3a935c3a4e89c733303a2d5a7c257ea44af3a56c8202df486b7f5de40f37e1 \ + --hash=sha256:5c1c8e78426e59b3f8005e9b19f6ff46e5845895adbde20ece9218319eca6506 \ + --hash=sha256:5d63a068f978fc69421fb0e6eb91a9603187527c86b7cd3f534a5b77a592b888 \ + --hash=sha256:667c3777ca571e5dbeb76f331562ff98b957431df140b54c85fd4d52eea8d8f6 \ + --hash=sha256:6da155091429aeba16851ecb10a9104a108bcd32f6c1642867eadaee401c1c41 \ + --hash=sha256:6dc4126390929823e2d2d9dc79ab4046ed74680360fc5f38b585c12c66cdf459 \ + --hash=sha256:7398c222d1d405e796970320036b1b563892b65809d9e5261487bb2c7f7b5c6a \ + --hash=sha256:74c51543498289c0c43656701be6b077f4b265868fa7f8a8859c197006efb608 \ + --hash=sha256:776f352e8329135506a1d6bf16ac3f87bc25b28e765949282dcc627af36123aa \ + --hash=sha256:778a11b15673f6f1df23d9586f83c4846c471a8af693a22e066508b77d201ec8 \ + --hash=sha256:78f7b9e5d6f2fdb88cdde9440dc147259b62b9d3b019924def9f6478be254ac1 \ + --hash=sha256:799345ab092bee59f01a915620b5d014698547afd011e691a208637312db9186 \ + --hash=sha256:7bf6cdf8e07c8151fba6fe85735441240ec7f619f935a5205953d58009aef8c6 \ + --hash=sha256:8009897cdef112072f93a0efdce29cd819e717fd2f649ee3016efd3cd885a7ed \ + --hash=sha256:80f85f0a7cc86e7a54c46d99c9e1318ff01f4687c172ede30fd52d19d1da1c8e \ + --hash=sha256:8585e3bb2cdea02fc88ffa245069c36555557ad3609e83be0ec71f54fd4abb52 \ + --hash=sha256:878be833caa6a3821caf85eb39c5ba92d28e85df26d57afb06b35b2efd937231 \ + --hash=sha256:8a76ea0f0b9dfa06f254ee06053d93a600865b3274358ca48a352ce4f0798450 \ + --hash=sha256:8b7b94a067d1c504ee0b16def57ad5738701e4ba10cec90529f13fa03c833496 \ + --hash=sha256:8d92f1a84bb12d9e56f818b3a746f3efba93c1b63c8387a73dde655e1e42282a \ + --hash=sha256:908bd3f6439f2fef9e85031b59fd4f1297af54415fb60e4254a95f75b3cab3f3 \ + --hash=sha256:92db2bf818d5cc8d9c1f1fc56b897662e24ea5adb36ad1f1d82875bd64e03c24 \ + --hash=sha256:940d4a017dbfed9daf46a3b086e1d2167e7012ee297fef9e1c545c4d022f5178 \ + --hash=sha256:957e7c38f250991e48a9a73e6423db1bb9dd14e722a10f6b8bb8e16a0f55f695 \ + --hash=sha256:96153e77a591c8adc2ee805756c61f59fef4cf4073a9275ee86fe8cba41241f7 \ + --hash=sha256:96f423a119f4777a4a056b66ce11527366a8bb92f54e541ade21f2374433f6d4 \ + --hash=sha256:97260ff46b207a82a7567b581ab4190bd4dfa09f4db8a8b49d1a958f6aa4940e \ + --hash=sha256:974b28cf63cc99dfb2188d8d222bc6843656188164848c4f679e63dae4b0708e \ + --hash=sha256:9ff15928d62a0b80bb875655c39bf517938c7d589554cbd2669be42d97c2cb61 \ + --hash=sha256:a6483e309ca809f1efd154b4d37dc6d9f61037d6c6a81c2dc7a15cb22c8c5dca \ + --hash=sha256:a88f062f072d1589b7b46e951698950e7da00442fc1cacbe17e19e025dc327ad \ + --hash=sha256:ac913f8403b36a2c8610bbfd25b8013488533e71e62b4b4adce9c86c8cea905b \ + --hash=sha256:adbeebaebae3526afc3c96fad434367cafbfd1b25d72369a9e5858453b1bb71a \ + --hash=sha256:b2a095d45c5d46e5e79ba1e5b9cb787f541a8dee0433836cea4b96a2c439dcd8 \ + --hash=sha256:b3210649ee28062ea6099cfda39e147fa1bc039583c8ee4481cb7811e2448c51 \ + --hash=sha256:b37f6d31b3dcea7deb5e9696e529a6aa4a898adc33db82da12e4c60a7c4d2011 \ + --hash=sha256:b4dec9482a65c54a5044486847b8a66bf10c9cb4926d42927ec4e8fd5db7fed8 \ + --hash=sha256:b4f3b365f31c6cd4af24545ca0a244a53688cad8834e32f56831c4923b50a103 \ + --hash=sha256:b6db2185db9be0a04fecf2f241c70b63b1a242e2805be291855078f2b404dd6b \ + --hash=sha256:b9be22a69a014bc47e78072d0ecae716f5eb56c15238acca0f43d6eb8e4a5bda \ + --hash=sha256:bac9c42ba2ac65ddc115d930c78d24ab8d4f465fd3fc473cdedfccadb9429806 \ + --hash=sha256:bf0a7e10b077bf5fb9380ad3ae8ce20ef919a6ad93b4552896419ac7e1d8e042 \ + --hash=sha256:c23c3ff005322a6e16f71bf8692fcf4d5a304aaafe1e262c98c6d4adc7be863e \ + --hash=sha256:c4c800524c9cd9bac5166cd6f55285957fcfc907db323e193f2afcd4d9abd69b \ + --hash=sha256:c7366fe1418a6133d5aa824ee53d406550110984de7637d65a178010f759c6ef \ + --hash=sha256:c8d1634419f39ea6f5c427ea2f90ca85126b54b50837f31497f3bf38266e853d \ + --hash=sha256:c9a63152fe95756b85f31186bddf42e4c02c6321207fd6601a1c89ebac4fe567 \ + --hash=sha256:cb89a7f2de3602cfed448095bab3f178399646ab7c61454315089787df07733a \ + --hash=sha256:cba69cb73723c3f329622e34bdbf5ce1f80c21c290ff04256cff1cd3c2036ed2 \ + --hash=sha256:cee686f1f4cadeb2136007ddedd0aaf928ab95216e7691c63e50a8ec066336d0 \ + --hash=sha256:cf253e0e1c3ceb4aaff6df637ce033ff6535fb8c70a764a8f46aafd3d6ab798e \ + --hash=sha256:d1eaff1d00c7751b7c6662e9c5ba6eb2c17a2306ba5e2a37f24ddf3cc953402b \ + --hash=sha256:d3bb933317c52d7ea5004a1c442eef86f426886fba134ef8cf4226ea6ee1821d \ + --hash=sha256:d4d3214a0f8394edfa3e303136d0575eece0745ff2b47bd2cb2e66dd92d4351a \ + --hash=sha256:d6a5df73acd3399d893dafc71663ad22534b5aa4f94e8a2fabfe856c3c1b6a52 \ + --hash=sha256:d8b7138e5cd0647e4523d6685b0eac5d4be9a184ae9634492f25c6eb38c12a47 \ + --hash=sha256:db1e72ede2d0d7ccb213f218df6a078a9c09a7de257c2fe8fcef16d5925230b1 \ + --hash=sha256:e25ac20a2ef37e91c1b39938b591457666a0fa835c7783c3a8f33ea42870db94 \ + --hash=sha256:e2de870d16a7a53901e41b64ffdf26f2fbb8917b3e6ebf398098d72c5b20bd7f \ + --hash=sha256:e4a3408834f65da56c83528fb52ce7911484f0d1eaf7b761fc66001db1646eff \ + --hash=sha256:eaa352d7047a31d87dafcacbabe89df0aa506abb5b1b85a2fb91bc3faa02d822 \ + --hash=sha256:eab8145831a0d56ec9c4139b6c3e594c7a83c2c8be25d5bcf2d86136a532287a \ + --hash=sha256:ec3cc8c5d4084591b4237c0a272cc4f50a5b03396a47d9caaf76f5d7b38a4f11 \ + --hash=sha256:edee74874ce20a373d62dc28b0b18b93f645633c2943fd90ee9d898550770581 \ + --hash=sha256:eefdba20de0d938cec6a89bd4d70f346a03108a19b9df4248d3cf0d88f1b0f51 \ + --hash=sha256:ef2b7b394f208233e471abc541cc6991f907ffd47dc72584acee3147899d6565 \ + --hash=sha256:f21f00a91358803399890ab167098c131ec2ddd5f8f5fd5fe9c9f2c6fcd91e40 \ + --hash=sha256:f4be2e3d8bc8aabd566f8d5b8ba7ecc09249d74ba3c9ed52e54dc23a293f0b92 \ + --hash=sha256:f57fb59d9f385710aa7060e89410aeb5058b99e62f4d16b08b91986b9a2140c2 \ + --hash=sha256:f6292f1de555ffcc675941d65fffffb0a5bcd992905015f85d0592201793e0e5 \ + --hash=sha256:f833670942247a14eafbb675458b4e61c82e002a148f49e68257b79296e865c4 \ + --hash=sha256:fa47e444b8ba08fffd1c18e8cdb9a75db1b6a27f17507522834ad13ed5922b93 \ + --hash=sha256:fb30f9626572a76dfe4293c7194a09fb1fe93ba94c7d4f720dfae3b646b45027 \ + --hash=sha256:fe3c58d2f5db5fbd18c2987cba06d51b0529f52bc3a6cdc33d3f4eab725104bd + # via + # aiohttp + # aiosignal +fsspec[gcs]==2026.7.0 \ + --hash=sha256:b57ddbafedfaef7018c1ecab32aa200a9d7ca26b77965f64e48b70061249d279 \ + --hash=sha256:c803c40f4cf860b49dea58ee3e1c33cb9c790520e233537e1340049f89b82a88 + # via + # etils + # gcsfs + # xprof +gcsfs==2026.8.1 \ + --hash=sha256:66e6316690b17e1887a4f4bc0902eee53f5b3fd860cc3f0377d65c5be3313b1f \ + --hash=sha256:738da7dcc74817c7668d032f932598482a23cce1b061c537e54278be44d2fff8 + # via fsspec +google-api-core[grpc]==2.37.0 \ + --hash=sha256:cf58f220aa797f1ffdda52194c4c7d72efeced09297c2976529f8135f6d85b9a \ + --hash=sha256:d84042a0034cce9c4304e17d2e5e9966c2fcb1a851b73799291f687a67470c8b + # via + # google-cloud-core + # google-cloud-storage + # google-cloud-storage-control +google-auth==2.58.0 \ + --hash=sha256:55e30cf15e737de92c5323d78cda8a83fcd57e7ffbaf900c4600039fd60a80fd \ + --hash=sha256:8a9c4645bb4c8e91668fb1934b95ae6a8687084232753639220ba9bf04a1610d + # via + # gcsfs + # google-api-core + # google-auth-oauthlib + # google-cloud-core + # google-cloud-storage + # google-cloud-storage-control +google-auth-oauthlib==1.4.1 \ + --hash=sha256:1a83f5f2a8421dedadaa3caf25b3a710dddf85a33a63144be41c2fc79174b106 \ + --hash=sha256:a1be43ec69fe563ac9b2c4d6fc4334b323b21cbdc59a638b5fa34dd4d5a2a348 + # via gcsfs +google-cloud-core==2.7.0 \ + --hash=sha256:874aaf89765db87a9b911b7a2ca7c5068554868eed9e75c7766affe342a2913d \ + --hash=sha256:c18a250904cfdda021eb3ae8b8238c9f9ca272a4cbbfb5cba946b3fe3022eed1 + # via google-cloud-storage +google-cloud-storage==3.14.1 \ + --hash=sha256:8f0fe2b74bd80ddcbd67247a3b47b81035a0a4f448569ed358fa40a46c940118 \ + --hash=sha256:b24e74b493c60b19b83462933a83bb831e45b6ca4924b0d75eac8e176b58b3a7 + # via + # gcsfs + # xprof +google-cloud-storage-control==1.15.0 \ + --hash=sha256:46c8c801ff0425a1bd59bc28899a8242e15f6bbb06a4f2304cbd169c7d26cb29 \ + --hash=sha256:e3d50276f0aed0c33ebcf3c6221e933889e042df70a56886ddffe59c66889183 + # via gcsfs +google-crc32c==1.8.0 \ + --hash=sha256:014a7e68d623e9a4222d663931febc3033c5c7c9730785727de2a81f87d5bab8 \ + --hash=sha256:01f126a5cfddc378290de52095e2c7052be2ba7656a9f0caf4bcd1bfb1833f8a \ + --hash=sha256:0470b8c3d73b5f4e3300165498e4cf25221c7eb37f1159e221d1825b6df8a7ff \ + --hash=sha256:119fcd90c57c89f30040b47c211acee231b25a45d225e3225294386f5d258288 \ + --hash=sha256:14f87e04d613dfa218d6135e81b78272c3b904e2a7053b841481b38a7d901411 \ + --hash=sha256:17446feb05abddc187e5441a45971b8394ea4c1b6efd88ab0af393fd9e0a156a \ + --hash=sha256:19b40d637a54cb71e0829179f6cb41835f0fbd9e8eb60552152a8b52c36cbe15 \ + --hash=sha256:2a3dc3318507de089c5384cc74d54318401410f82aa65b2d9cdde9d297aca7cb \ + --hash=sha256:3b9776774b24ba76831609ffbabce8cdf6fa2bd5e9df37b594221c7e333a81fa \ + --hash=sha256:3cc0c8912038065eafa603b238abf252e204accab2a704c63b9e14837a854962 \ + --hash=sha256:3d488e98b18809f5e322978d4506373599c0c13e6c5ad13e53bb44758e18d215 \ + --hash=sha256:3ebb04528e83b2634857f43f9bb8ef5b2bbe7f10f140daeb01b58f972d04736b \ + --hash=sha256:450dc98429d3e33ed2926fc99ee81001928d63460f8538f21a5d6060912a8e27 \ + --hash=sha256:4b8286b659c1335172e39563ab0a768b8015e88e08329fa5321f774275fc3113 \ + --hash=sha256:57a50a9035b75643996fbf224d6661e386c7162d1dfdab9bc4ca790947d1007f \ + --hash=sha256:61f58b28e0b21fcb249a8247ad0db2e64114e201e2e9b4200af020f3b6242c9f \ + --hash=sha256:6f35aaffc8ccd81ba3162443fabb920e65b1f20ab1952a31b13173a67811467d \ + --hash=sha256:71734788a88f551fbd6a97be9668a0020698e07b2bf5b3aa26a36c10cdfb27b2 \ + --hash=sha256:864abafe7d6e2c4c66395c1eb0fe12dc891879769b52a3d56499612ca93b6092 \ + --hash=sha256:86cfc00fe45a0ac7359e5214a1704e51a99e757d0272554874f419f79838c5f7 \ + --hash=sha256:87b0072c4ecc9505cfa16ee734b00cd7721d20a0f595be4d40d3d21b41f65ae2 \ + --hash=sha256:87fa445064e7db928226b2e6f0d5304ab4cd0339e664a4e9a25029f384d9bb93 \ + --hash=sha256:89c17d53d75562edfff86679244830599ee0a48efc216200691de8b02ab6b2b8 \ + --hash=sha256:8b3f68782f3cbd1bce027e48768293072813469af6a61a86f6bb4977a4380f21 \ + --hash=sha256:a428e25fb7691024de47fecfbff7ff957214da51eddded0da0ae0e0f03a2cf79 \ + --hash=sha256:b0d1a7afc6e8e4635564ba8aa5c0548e3173e41b6384d7711a9123165f582de2 \ + --hash=sha256:ba6aba18daf4d36ad4412feede6221414692f44d17e5428bdd81ad3fc1eee5dc \ + --hash=sha256:cb5c869c2923d56cb0c8e6bcdd73c009c36ae39b652dbe46a05eb4ef0ad01454 \ + --hash=sha256:d511b3153e7011a27ab6ee6bb3a5404a55b994dc1a7322c0b87b29606d9790e2 \ + --hash=sha256:db3fe8eaf0612fc8b20fa21a5f25bd785bc3cd5be69f8f3412b0ac2ffd49e733 \ + --hash=sha256:e6584b12cb06796d285d09e33f63309a09368b9d806a551d8036a4207ea43697 \ + --hash=sha256:f4b51844ef67d6cf2e9425983274da75f18b1597bb2c998e1c0a0e8d46f8f651 \ + --hash=sha256:f639065ea2042d5c034bf258a9f085eaa7af0cd250667c0635a3118e8f92c69c + # via + # google-cloud-storage + # google-resumable-media +google-resumable-media==2.10.2 \ + --hash=sha256:1de441703cd298d75a419bfdc0066e9fc7b0a1de630df96eea8ce8f5c759358c \ + --hash=sha256:e3cedc827a4ea41e216582d74346f1fb9fceb625a8c3c53912f2ca1d663334d7 + # via google-cloud-storage +googleapis-common-protos[grpc]==1.75.3 \ + --hash=sha256:57c435ac2c68b108999b6db075d9053e4d7a936ba57b4a3d45667b1346f1738a \ + --hash=sha256:a018d2bf098ca9fb6faa08d5bb780e2a2c2f73c566f069761331386c9596d3f2 + # via + # google-api-core + # grpc-google-iam-v1 + # grpcio-status +grpc-google-iam-v1==0.14.5 \ + --hash=sha256:07fd3a9fafb586588e771831fbfc8f6597050181d0c3b45e039d18b8fdc1aab5 \ + --hash=sha256:0f5e680b20aa0a9441e68c769da04d94d70fca4e43751a82d8abb8aa6a7181ca + # via google-cloud-storage-control +grpcio==1.84.0 \ + --hash=sha256:026d757df86c5b7a41de8200b9a2cda454aaa5004cb0c7e3374c66eb82f61499 \ + --hash=sha256:06619ba1515e5ee69fb2a514e95dd8be05ce74cb3928d5b34f87f87c86fe3c27 \ + --hash=sha256:08735e3d08d24ab3132cf87e2e5dea8746cabcc7d676c2b0b7362f195feef9d9 \ + --hash=sha256:0d532ade4486dad9b302ffa4d4683d67561051c26d17c4023322845e9fa10140 \ + --hash=sha256:158c1c11cfb61b4849c3caf4d52de6f5ecd376e14446feb4a90dc95a90d616f5 \ + --hash=sha256:15bb76489e337fc492685c9758e2fd4d4ab516b901ad830dc5a91987decf00be \ + --hash=sha256:19aaf172fc2edbefccce3f6e92c5150975dbe56c45744e9e87cf72ebdf85bfbe \ + --hash=sha256:209414080da8c20af94df1395b635da52dd57b5edc9e917e1deca0dc1c4bb55e \ + --hash=sha256:210e4c32f907045eb8158273e60c6ab69a3947697df6245dbda381f26c59485b \ + --hash=sha256:23e6e8e8a75cff88e0a793bfd3becea03a13e2763ae90c1ff573bc19ca5b429a \ + --hash=sha256:27b8b36200a9fbee6e120246f4a8a41657549107ef19fb2c819c4b2fd524f39a \ + --hash=sha256:28d2609691da93051e998495108bbddd2a9f7a561253bae94828d81290f30c15 \ + --hash=sha256:2c024da73b296f040b8360e60bd73a659b230093684a438da0e1260f34cc724e \ + --hash=sha256:393d8a78bff6731ecc5ad2151a821f8fbc1709b137ebb9c25a4ef399fbdcc914 \ + --hash=sha256:3d6a82c4fc6c85f2fb7572c86bdb86f84c97b6580e5f6599f711800bac48a5d8 \ + --hash=sha256:3de427b05f244ba2c2a9bdc67e7a6731c8340811524ecc4435466549f8af1d17 \ + --hash=sha256:406583b4e8fb2282ebd392e12b963e601c1f82e07125a8c2cb5b144e7e024796 \ + --hash=sha256:4119efa6519871719ad81f33bc95ab87857dcb1c5801f30a6e592f2c41164169 \ + --hash=sha256:42959bd50dd660ffc3f2a9bec15a6da4f9aaa0dda555d59ff2d2e80b908456a8 \ + --hash=sha256:455ed6083353b8e938f1d58c765eab2fbb165731e5b507be30fee344915a2a11 \ + --hash=sha256:465eef3d17e59ad22a556fc0138f7c7c799df426734344daec42c797d49fda99 \ + --hash=sha256:47ecf0d9b81d981f07b61bd89eced9d2582f5eaacc3aaa36ad27f81aef70a27f \ + --hash=sha256:49717e857899f4136d7657bf5aded61ac479110a075438290923a4d86af7cd02 \ + --hash=sha256:4aaeceeb7fa7d824c322d1ec3208c8495c88478a927295553235435fc49043ad \ + --hash=sha256:57dc36a5ab0e676f5f6e171de2917fd0aef73f32a9aaf23956bfe19997a30bd1 \ + --hash=sha256:5933a052946873d01a42119a05420d669bdca436aeba2d1851988ccb12b421c0 \ + --hash=sha256:5deda5b4bf62769eb98c119cca43d40e1231e34846b19db5cdea821d446a2253 \ + --hash=sha256:61386101ecaa096b694d0dd278caf99a56aeec78440cc17e918eef0b50f2d567 \ + --hash=sha256:659728f20fc7a0933ed7b1945435e31014b97ab8a5a7edcbaa70da4794aeb191 \ + --hash=sha256:70bb4ce8be0c5606bec259cbd7152374470396413b7863a658a08c849e6b29ff \ + --hash=sha256:71fd60e6e426d293d0a2f685115ad0a0845117602cf13605a4be7524fb5f7bba \ + --hash=sha256:756ea5c2da00fa65c930284892d2a9706828704ca3ba40b4c51c4834eb39fcfd \ + --hash=sha256:800b7e00d92553313c0463c200087930aa78678ec1d528193aeb50906f55989b \ + --hash=sha256:82da34ae4f639c73ac46e521e00c0a49bf86f717b9fb1f405f133e98731e38dc \ + --hash=sha256:8e1a45d174b6b8589f51dce1cea804aa6c1f72c9c80cba91ae2caabeb6d90540 \ + --hash=sha256:8e3f508d0e9e6236ba2f08d56e33355e434e785e813149a1b8477d3edf69779d \ + --hash=sha256:986e9751d416d7a6eaa2fecdac38da63153d63a4b340ba7d624889c490451500 \ + --hash=sha256:9b73836ba0e16fcbb57c31cf6cbc2907c8d8c790b83679df454b74bd15e0be04 \ + --hash=sha256:9bab4cf571653a8afffb83ce21aa27b51dfe629b526b7b6adec35491fe1fc2ea \ + --hash=sha256:a71d24f40b0cc6798feaa978c7411dc1135b7018e9fc0442db611c139bf58344 \ + --hash=sha256:a9383401d9f116f98cacd4eba6c505a6edb80ba65badfc8e8ed8ae64983bcc44 \ + --hash=sha256:b44f0a0fc7bc6677d38cc80bca1a32814ce6c8f200fb8b3c1a61c9d77eaefbf3 \ + --hash=sha256:b5c6f20d657ae09ae4e30d9d3a21edd13f1219d58cc6f999b9d1bb63be9c1baa \ + --hash=sha256:b61692f0069b3eee2fc8a3a1b7f6c044df9e03fede6ce69b3ca832e1c39f26c5 \ + --hash=sha256:b8c62888c3e49debf37ad9773e3c02f77b0c1e811f8fb0962f2b6c3bbab5b97a \ + --hash=sha256:bd8ea8eb3817b226057cc1c0e7ec4b378dcda52043b972b6ff12b1152178967d \ + --hash=sha256:c5559b492007dc09b4de9b95dab05f0b5e53547aad230cf07e46c7dd017a3be5 \ + --hash=sha256:d0fdd25faece8a1f95e8a3a8006e29701b5cf8dadb4a8132e68f3134637004a5 \ + --hash=sha256:e094dd21f077af8194923fc263cad872eaa1802bb0156fd7e5ae18e99cd86715 \ + --hash=sha256:e41c3993eee896c617dbd8a505085d28b6e84a0445ed9a1f40f95808473cf678 \ + --hash=sha256:e88d304f094f4937bc27ec6a435e218a084168f11ec630c8d5d39b431d08d81d \ + --hash=sha256:e90e3bdf7b5eac005fef631adae9cafde16f922def207b80a7c46b253c18ad20 \ + --hash=sha256:ed2c1493c44d0932f1e55fdb5d1ead658c68288ec5d51b8c4928422d98633ef9 \ + --hash=sha256:edb6f87fc60ff438557291501b3e16c7a77c3b01a52d782cf276dccc7c5dd89c \ + --hash=sha256:efb29f8633bf6630dc89de4fe0353ac3d7e4b70ef7b6e29fb40f00e68c127fa5 \ + --hash=sha256:f6c972474ce691aca74e58d17625450cef153dc4760364cadeb167983ea6d589 \ + --hash=sha256:f6d178ba6dc8e82976c184b65fddde172d054c17237993a3e083efe4f134d55b \ + --hash=sha256:f9a456bdbed52a01c9ab8423bdebab04a5363c78676edc55ab9b58bd13bdf9e1 \ + --hash=sha256:fbdbcd06986ede3ce584083b1dc2afe6808e8943e5cf50ad11183c03aceda25a \ + --hash=sha256:fc66cb50c93554b86db0b6625ab5c6e9051dbf8847c08d93c84918e02e413fb7 \ + --hash=sha256:fff5ef3fe1bba7d6147e5f19e01e5e122ac2c076486887ddcb8d42e663400fbe + # via + # google-api-core + # google-cloud-storage-control + # googleapis-common-protos + # grpc-google-iam-v1 + # grpcio-status +grpcio-status==1.84.0 \ + --hash=sha256:0c182ca0d6e60acbfd0e14499cf39a155e4827a1c3fd9f7638e49af15a74c30a \ + --hash=sha256:5caf28ba7184b81f618b5f7f094859fd2541bf429d2189bbbcd715c9c2cdcee2 + # via google-api-core +gviz-api==1.10.0 \ + --hash=sha256:846692dd8cc73224fc31b18e41589bd934e1cc05090c6576af4b4b26c2e71b90 \ + --hash=sha256:a05055fed8c279f34f4b496eace7648c7fe9c1b06851e8a36e748541f1adbb05 + # via xprof +idna==3.19 \ + --hash=sha256:5e0811a4383b21dc5838069f801c4fb62113b7447663d2530d2bd6e77b49bf15 \ + --hash=sha256:815e7be7a7806d54abb586dc943addc79e8b2ee16915059658cbeff4b1b43bf4 + # via + # requests + # yarl +jaraco-functools==4.6.0 \ + --hash=sha256:880c577ec9720b3a052d5bc611fb9f2269b3d87902ef42440df443b88e443280 \ + --hash=sha256:99e3dc0060c5cbe8fcd1cdb36258e2a65ca40f1566b2033b12abb1bb44dd3c30 + # via cheroot +markupsafe==3.0.3 \ + --hash=sha256:0303439a41979d9e74d18ff5e2dd8c43ed6c6001fd40e5bf2e43f7bd9bbc523f \ + --hash=sha256:068f375c472b3e7acbe2d5318dea141359e6900156b5b2ba06a30b169086b91a \ + --hash=sha256:0bf2a864d67e76e5c9a34dc26ec616a66b9888e25e7b9460e1c76d3293bd9dbf \ + --hash=sha256:0db14f5dafddbb6d9208827849fad01f1a2609380add406671a26386cdf15a19 \ + --hash=sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf \ + --hash=sha256:0f4b68347f8c5eab4a13419215bdfd7f8c9b19f2b25520968adfad23eb0ce60c \ + --hash=sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175 \ + --hash=sha256:116bb52f642a37c115f517494ea5feb03889e04df47eeff5b130b1808ce7c219 \ + --hash=sha256:12c63dfb4a98206f045aa9563db46507995f7ef6d83b2f68eda65c307c6829eb \ + --hash=sha256:133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6 \ + --hash=sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab \ + --hash=sha256:15d939a21d546304880945ca1ecb8a039db6b4dc49b2c5a400387cdae6a62e26 \ + --hash=sha256:177b5253b2834fe3678cb4a5f0059808258584c559193998be2601324fdeafb1 \ + --hash=sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce \ + --hash=sha256:1b4b79e8ebf6b55351f0d91fe80f893b4743f104bff22e90697db1590e47a218 \ + --hash=sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634 \ + --hash=sha256:1ba88449deb3de88bd40044603fafffb7bc2b055d626a330323a9ed736661695 \ + --hash=sha256:1cc7ea17a6824959616c525620e387f6dd30fec8cb44f649e31712db02123dad \ + --hash=sha256:218551f6df4868a8d527e3062d0fb968682fe92054e89978594c28e642c43a73 \ + --hash=sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c \ + --hash=sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe \ + --hash=sha256:2a15a08b17dd94c53a1da0438822d70ebcd13f8c3a95abe3a9ef9f11a94830aa \ + --hash=sha256:2f981d352f04553a7171b8e44369f2af4055f888dfb147d55e42d29e29e74559 \ + --hash=sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa \ + --hash=sha256:3524b778fe5cfb3452a09d31e7b5adefeea8c5be1d43c4f810ba09f2ceb29d37 \ + --hash=sha256:3537e01efc9d4dccdf77221fb1cb3b8e1a38d5428920e0657ce299b20324d758 \ + --hash=sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f \ + --hash=sha256:38664109c14ffc9e7437e86b4dceb442b0096dfe3541d7864d9cbe1da4cf36c8 \ + --hash=sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d \ + --hash=sha256:3b562dd9e9ea93f13d53989d23a7e775fdfd1066c33494ff43f5418bc8c58a5c \ + --hash=sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97 \ + --hash=sha256:4bd4cd07944443f5a265608cc6aab442e4f74dff8088b0dfc8238647b8f6ae9a \ + --hash=sha256:4e885a3d1efa2eadc93c894a21770e4bc67899e3543680313b09f139e149ab19 \ + --hash=sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9 \ + --hash=sha256:509fa21c6deb7a7a273d629cf5ec029bc209d1a51178615ddf718f5918992ab9 \ + --hash=sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc \ + --hash=sha256:591ae9f2a647529ca990bc681daebdd52c8791ff06c2bfa05b65163e28102ef2 \ + --hash=sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4 \ + --hash=sha256:69c0b73548bc525c8cb9a251cddf1931d1db4d2258e9599c28c07ef3580ef354 \ + --hash=sha256:6b5420a1d9450023228968e7e6a9ce57f65d148ab56d2313fcd589eee96a7a50 \ + --hash=sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698 \ + --hash=sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9 \ + --hash=sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b \ + --hash=sha256:795e7751525cae078558e679d646ae45574b47ed6e7771863fcc079a6171a0fc \ + --hash=sha256:7be7b61bb172e1ed687f1754f8e7484f1c8019780f6f6b0786e76bb01c2ae115 \ + --hash=sha256:7c3fb7d25180895632e5d3148dbdc29ea38ccb7fd210aa27acbd1201a1902c6e \ + --hash=sha256:7e68f88e5b8799aa49c85cd116c932a1ac15caaa3f5db09087854d218359e485 \ + --hash=sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f \ + --hash=sha256:8485f406a96febb5140bfeca44a73e3ce5116b2501ac54fe953e488fb1d03b12 \ + --hash=sha256:8709b08f4a89aa7586de0aadc8da56180242ee0ada3999749b183aa23df95025 \ + --hash=sha256:8f71bc33915be5186016f675cd83a1e08523649b0e33efdb898db577ef5bb009 \ + --hash=sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d \ + --hash=sha256:949b8d66bc381ee8b007cd945914c721d9aba8e27f71959d750a46f7c282b20b \ + --hash=sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a \ + --hash=sha256:9a1abfdc021a164803f4d485104931fb8f8c1efd55bc6b748d2f5774e78b62c5 \ + --hash=sha256:9b79b7a16f7fedff2495d684f2b59b0457c3b493778c9eed31111be64d58279f \ + --hash=sha256:a320721ab5a1aba0a233739394eb907f8c8da5c98c9181d1161e77a0c8e36f2d \ + --hash=sha256:a4afe79fb3de0b7097d81da19090f4df4f8d3a2b3adaa8764138aac2e44f3af1 \ + --hash=sha256:ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287 \ + --hash=sha256:b8512a91625c9b3da6f127803b166b629725e68af71f8184ae7e7d54686a56d6 \ + --hash=sha256:bc51efed119bc9cfdf792cdeaa4d67e8f6fcccab66ed4bfdd6bde3e59bfcbb2f \ + --hash=sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581 \ + --hash=sha256:bdd37121970bfd8be76c5fb069c7751683bdf373db1ed6c010162b2a130248ed \ + --hash=sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b \ + --hash=sha256:c0c0b3ade1c0b13b936d7970b1d37a57acde9199dc2aecc4c336773e1d86049c \ + --hash=sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026 \ + --hash=sha256:c4ffb7ebf07cfe8931028e3e4c85f0357459a3f9f9490886198848f4fa002ec8 \ + --hash=sha256:ccfcd093f13f0f0b7fdd0f198b90053bf7b2f02a3927a30e63f3ccc9df56b676 \ + --hash=sha256:d2ee202e79d8ed691ceebae8e0486bd9a2cd4794cec4824e1c99b6f5009502f6 \ + --hash=sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e \ + --hash=sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d \ + --hash=sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d \ + --hash=sha256:de8a88e63464af587c950061a5e6a67d3632e36df62b986892331d4620a35c01 \ + --hash=sha256:df2449253ef108a379b8b5d6b43f4b1a8e81a061d6537becd5582fba5f9196d7 \ + --hash=sha256:e1c1493fb6e50ab01d20a22826e57520f1284df32f2d8601fdd90b6304601419 \ + --hash=sha256:e1cf1972137e83c5d4c136c43ced9ac51d0e124706ee1c8aa8532c1287fa8795 \ + --hash=sha256:e2103a929dfa2fcaf9bb4e7c091983a49c9ac3b19c9061b6d5427dd7d14d81a1 \ + --hash=sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5 \ + --hash=sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d \ + --hash=sha256:e8fc20152abba6b83724d7ff268c249fa196d8259ff481f3b1476383f8f24e42 \ + --hash=sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe \ + --hash=sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda \ + --hash=sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e \ + --hash=sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737 \ + --hash=sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523 \ + --hash=sha256:f42d0984e947b8adf7dd6dde396e720934d12c506ce84eea8476409563607591 \ + --hash=sha256:f71a396b3bf33ecaa1626c255855702aca4d3d9fea5e051b41ac59a9c1c41edc \ + --hash=sha256:f9e130248f4462aaa8e2552d547f36ddadbeaa573879158d721bbd33dfe4743a \ + --hash=sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50 + # via werkzeug +more-itertools==11.1.0 \ + --hash=sha256:48e8f4d9e7e5878571ecf6f2b4e57634f93cd474cc8cfbd2376f2d11b396e30d \ + --hash=sha256:4b65538ae22f6fed0ce4874efd317463a7489796a0939fa66824dd542125a192 + # via + # cheroot + # jaraco-functools +multidict==6.8.0 \ + --hash=sha256:003a3bddb32915c3f67096ea41d24e53edf710edb65a1f5d0c70ab40b0e4d20b \ + --hash=sha256:00be37bde741bf60871082cd347a093218c44886e99231b7516671c70f2c280d \ + --hash=sha256:029897732a9c798737457e382bf84e8c64237eff224a90aea2639f4413c45e4e \ + --hash=sha256:05c2e90c5289c5f7436ba2c25812a5fbdaa1c1bc11c8d8d3bbf64f5cd7c633dd \ + --hash=sha256:071da134651b04a8507dfb331ac0988f376337c2aea59486bf20989fb5b5a64e \ + --hash=sha256:088b04a66b3c1fce6fe4d771ec184a0426262d0b86709c908477b4ac7965df40 \ + --hash=sha256:093167d22a8c95af30f597b8a5686f20a14512989942d4be804d119899caca20 \ + --hash=sha256:0935971bffd0b479fc90c4811ca787703e93fcb6afea939a375dfc80285ab368 \ + --hash=sha256:095f62ea4e7a3be2f6c567ab695ce10e950f2adb905c1bec82281593e0b2d2ad \ + --hash=sha256:0b143d53590e89f43153d81d505a8448d4d57354354385aef8a51d67ffefa27e \ + --hash=sha256:0c1c4debad7337627b86837abdf0237ca3cb3d7e17de7eab0177c263878546d4 \ + --hash=sha256:0eca15d627e942ce186a935061f1568cc46c02e97c419c8da802df2be9f917d8 \ + --hash=sha256:0ef606c15cac6c90279acf34120784b6f36662cbf382defd3955cd8f1115336b \ + --hash=sha256:10456943903744ae1249728161c96bd9d2f7eb5ee17fcc2ffda2dc32e1bb36c7 \ + --hash=sha256:11d71490bf4bbff1141b14b93af419ad68c56b60bea9277fcb3f94dcca4796eb \ + --hash=sha256:122adc7c46ac1e31ecfc7f81b2530533dccafdba70f5d741649f87e336c63384 \ + --hash=sha256:13967dca8b2f33230a1427b52438326bb1c9101a1df22a3309ed3fcbbb3c96f0 \ + --hash=sha256:13e26f59f0eecfc5f67c663ad550ffdaf62c0f657547cde387f6c86af1c9449e \ + --hash=sha256:15db8e6cab5f4cc9241bc56e69fdf3452cf49c10ee3c7977c742e68a275b3786 \ + --hash=sha256:18f0e06360c3e451a3ab800355773c8d125a758238d780c800b0ee5e90ee903c \ + --hash=sha256:1969971900b0871530f9b62280dcc2d75688e74d2a69262bc01faf2b96c78f04 \ + --hash=sha256:1b8986d4313dcee7c932837d16a535f1840b827bac1ea7c5c4c80751d0423794 \ + --hash=sha256:1bdb9b8fba5a9aef673ec90db3f55b1ce743f2fbdea4d37dc04d14ccdfc153ff \ + --hash=sha256:1f57c414be82490bc0e0305fdb834186229b2d9b6a35fa0afd1eb1a772d125ab \ + --hash=sha256:1f66fe6a021173d0d47968491791966b9f3e6d61115f2491744aa0c07a6e67af \ + --hash=sha256:202436df907c15adbb94360296c425ea53cf8968a5d2cff9b5b9790ae1972b33 \ + --hash=sha256:2196ba6df392c3574acadd14ef87550f3611349c8618564de324b806a7a31cee \ + --hash=sha256:22a310ad37672a261e55a8b5e28d0ae08cfb68abb1f46418ccd19835c3b8e836 \ + --hash=sha256:23c9ee89967b6a9b4048acb3b93b660ed714ce9c8bf3bbe652959bc120dc02dc \ + --hash=sha256:2622fe114c0bd66ca5c461859357587f5a5e35ee5ff49fc5643d1bc78dbb41c6 \ + --hash=sha256:26a7aafc992e78872e2c8c1f7248c0e01139cf9020a7781b0c064fa566832712 \ + --hash=sha256:27747162712e85c84598d364425dbf1714ff335bdb6ba3171c4e5081196e8916 \ + --hash=sha256:29631224698de1e42abc8fa7658d830e0aed0029785144b5832b695da5adef2f \ + --hash=sha256:29b6e7bc4442a56cf8e0dc1cabf3fdc77cd533568d6829fc76a1effd2ce332ec \ + --hash=sha256:29be9fd289e9ab8f480996ea2f686e1654b80242033843cb11691688329423f1 \ + --hash=sha256:2ba9933e8f35fe4a70f540b837254c4055da82dc3a9e500a8f95e61498083a15 \ + --hash=sha256:2cc66abb85e2108c9ff8a1c0d20fa260bf690bbb33caef4ff3ecb2c2cbdfff5d \ + --hash=sha256:2cd560498ae8e1bcc955643c1d78eb8e338226d07a983c656ea8c4443d3eec0f \ + --hash=sha256:2f79cc3e8039a8cf5c77e0811b0807953fd52d0863b9b76970b20d696dc64a78 \ + --hash=sha256:2f8a4b0b4d639d525928c7f30de527bfdf9ead6e44a5e8cb9c50aced5e4590cb \ + --hash=sha256:307c1acd812fe897e7fbe10c6758822e8c04be4e7c60a9f54901cdf8b5ab8bc3 \ + --hash=sha256:3126f2a96704505aa4e92a72d6e8a5d7f29d40a987ced8bf69e29d71dfc71fbc \ + --hash=sha256:31e8901637e20ccb3cf8f8848b5d0f7a00462bf5b34f7cf3dcbb2753b18e8b39 \ + --hash=sha256:346ac52e56bcda320c0dcdfdd081947ed7cada33afea4e2284bef7b0733bff9b \ + --hash=sha256:348bb85e2038b40c007383616d73f734869063772372519549ebd7da1723d1a4 \ + --hash=sha256:3533a03e4e789baf6a286e7b0b1b6da3f3d7c3eab569686ee29ee1d8b52e2cb4 \ + --hash=sha256:35977263d9bf506dbc65349f63b3b8c91606d4abc110990945e3b94bc671319c \ + --hash=sha256:397599503b718f0137f26d3f6532d6955069cd2e5917c47ef581495bc2529ff8 \ + --hash=sha256:3bafff8598f0528017ddc74194e5451d5c22d046c98935f8f86247b0f286e4f8 \ + --hash=sha256:3d1f48582686a0a3b81e9b43234766cc96697df72081af3f48107bd3f34d34e5 \ + --hash=sha256:4261863fc8b5ab1b815ede94e592e94c6af5b04616014929057e61859e7382a9 \ + --hash=sha256:43a4b56555bbcf8af161e7c7682bd93eec10f068c95844511864c018c8e5e13b \ + --hash=sha256:45cc39ba50fb0754a4359b90f8229ae08598fe2266abe3521b4e5a9ba916534a \ + --hash=sha256:46029e6e27a3ec0dc55b53f58df82d10f04c5e111f78248279b530bedad2c30a \ + --hash=sha256:48ea524a25a1cd5972cf293bc95713918cba0bcd6fa9b992d906c857c546abe2 \ + --hash=sha256:4ee953a5ebaeed38dc21cc032ed17a9d9782802e00042200497ab4b01b0bf7c0 \ + --hash=sha256:54af1266710cb0f305127ae0b970aff8d208057f8a29cd6e1db99b0114947035 \ + --hash=sha256:560b211fc3bd4a1e1c6de44f6d38113bf5b410dfc89a4c0d2a3c0edbf1a0dfb8 \ + --hash=sha256:563661919f603374c40cf45ffcd25535c12b8954203569a2ab1cee5265871cf4 \ + --hash=sha256:563d6500ca80dac7bba6f48a78e0ffd87e21a7d4d24642c6503a2ddccd70c110 \ + --hash=sha256:59e539c4eb4d3a53b0e630a6ba2b2f2824732b5e73f90e30a280f12fde157b15 \ + --hash=sha256:5bbbb696c8024475b1877d14ce20d5f1cc05b8f6d786cea0fe3aa7fedc02e891 \ + --hash=sha256:5caf684986a2490628f059a99dd107b566a2d34cf947f8eb8387e0500a1f90c5 \ + --hash=sha256:5cd4637ce76312ba1e05eb9c5193fec231f64fee0944e135fa1e951242355b37 \ + --hash=sha256:610c7637bc36b90f39e6c66f710f93d57018f83d53e1e187caaa218c6892b95f \ + --hash=sha256:628ff11e6720f90acd0c305dfa3339f04a783a20de8cda6ac333ba46447261e8 \ + --hash=sha256:62b8e291a4f7edbf7cde7a43d831d893ba443a1b627498b53581943b0e348feb \ + --hash=sha256:6300d5176647145ba1e22991c924fb29743e54b4d7b8bc85a0d3ec0e55e189cb \ + --hash=sha256:64eaeda36ee8d88f9e8616a587a8c66a663283cf6e0dcf013c1ddd8c758e4aef \ + --hash=sha256:658f5a1895b804423d97b22d06fc0d0b171c7c01dcc3aa9c8faf0c0e26a249a5 \ + --hash=sha256:65c85c79f5a2c04fbbc18f006c014674dc5fdf270cb978d8862c82c6f694e60c \ + --hash=sha256:68186a2d4051c8ffd17be33553bea2ec9bbc8ef860fe2980a221d96126296f31 \ + --hash=sha256:68d40b2bace413f3231f5729d3fcfb1837fd31c4907e241b5d43211bfd76f3c2 \ + --hash=sha256:69708fecaa88bcb2341397b49fc95057a835b02a3670c551b37f95dd79e64e3a \ + --hash=sha256:69b3e519a132bb943b0daae15fc8c2168706b17f826481d32a32a5e784b129e3 \ + --hash=sha256:6b62b7e0025aa48dec11e125e655d1157985a5fdcec04b1ad500101ad072b891 \ + --hash=sha256:714597cb5d5e15a8a449d2ae23c45b486a9e8fa33c462c7a33d7f35b65d92943 \ + --hash=sha256:758233648ac47b07c575224c4eadd73c8929c3b4c31e2afcfea935fde1cda735 \ + --hash=sha256:75daa15ca16d6285eb2e104b2f05ee6f8d9836c68da3ce5c85f615a0450eed0e \ + --hash=sha256:77745725125d01fd613b6db043362aa7c6bfbfdb23d45dbfc3d92bf58160af62 \ + --hash=sha256:7941ef106ca1f2c62314a13c7ed913bcf49641f3efdc12864d588e17870920ac \ + --hash=sha256:7a2573d0fd34f361a4a14e54d8cda3a91ac4e55fbf0d719698024f3b09c5b147 \ + --hash=sha256:7a62e302fc8cd6aa8972207e7e951d1fdee7c1dda18568305041d19f0e2c00f5 \ + --hash=sha256:7bb0dad75068fee80fcb60f88569722c199d8656a16706702dc6e3b786819c90 \ + --hash=sha256:7bc7003991ebd368a20d05228137a37b3d3066751f3ea1e4f7b8efe8e752f2f5 \ + --hash=sha256:7d26dc8f070c0ec5579e987fa615ffd6883086106eefdff9e10d160fc5630630 \ + --hash=sha256:8125e60f3c70e323ac07dd8b3635f7b3bbc5c3a9ac04ae5988f668ff7ae28a18 \ + --hash=sha256:8180b635290a75af8478f1b3e9810135381ae24833293fe77b85c1c21ff842ab \ + --hash=sha256:82780eb8bf59e8fb25dd081fde6e058805045d6374a7f2f877effc826ca4434b \ + --hash=sha256:835d5a90b11d1f5f8200ff3cc8316bded76eebebc92436398947a27657e645e7 \ + --hash=sha256:83ff054b04915be5c15680da6c6012474a2cc2bf534129a0e8c6a99f17ba7238 \ + --hash=sha256:8457aff3c12a89a8e1c4674de5c777857fbc429f40fe117a3d29538547cbc364 \ + --hash=sha256:847d6082ae694dc95e548acb201bc100e1cfa96513bc71fdcb86f709dad6c435 \ + --hash=sha256:883284137e25318ed9735b742ae46341a864888fae28e8b6314c4f84da080f08 \ + --hash=sha256:887f9a975996032c686719eb7b3e1e7942fab5079c2b778bbd9afe9a9d78244f \ + --hash=sha256:8890c89d662560e51c55ac1304d6f919b23942abe9ae1127cb1de9aa6132fa52 \ + --hash=sha256:88a6df88567680504ae28bfa7a1f2f64243d91e79a40b2c92ef42efc531e23da \ + --hash=sha256:8d1046b5427dcafe6e8a0e07527dd74f1ee694006160162f53f3a17f15aad3b4 \ + --hash=sha256:8daafaa0b2eb43f76898ced78b1e0fb91b38c4fa50da516c18067f2a2d578c20 \ + --hash=sha256:8dc2d9c3a924ed14166e63650b2cf9f59e7821743bdd50b23802bd97ca09bde5 \ + --hash=sha256:90c10b22860dbd09982d0b8993b66231a861bea2993d4a817ff35273f6ea285a \ + --hash=sha256:91fa75d0a693832106d98f66c849f034f21c828d14437f1fb97d3784aab89e84 \ + --hash=sha256:930c6058047410e3edff445f5a6e4457f2e089042dede00e2d18ce06f3ceae2e \ + --hash=sha256:9442b14eec262a1f74369bbd07e75bc5155105164649a4b9fbc1ebc7b8fb0b14 \ + --hash=sha256:95c27b4f3f04320fc44e338573f40c5c956b504a7fcf081a157fd0b02579311c \ + --hash=sha256:9606f583e7acaf61e7b3f56074e14037b9af7cb194590edfc0114b3ae5931ff7 \ + --hash=sha256:962f18c59a000f30b084ea2e6b8001521bb315efd4e5f10acf9fb36f366b7882 \ + --hash=sha256:9caef53b20a105c0d66518a34be2f71b2783de8d091767575ef86f6ea422236d \ + --hash=sha256:9e37024b41d7a7e7e9cce14b248d54707c21c2a2ea30a47b71bdcefcafec00f2 \ + --hash=sha256:a5a7ee1217949ddd43c6b7bcf70d5c22193bb50e8c695386de5905325e93ce9f \ + --hash=sha256:a5e1583c14775580da05641240ce0d93f36ce3ddef3d5083a827468b0bcfe874 \ + --hash=sha256:a9e246f67ac038568b854ed7c5578e4c6af1f742359901a8fcc3603ff1358df6 \ + --hash=sha256:ab83fdd8cf307353edba9c427c17a3a021c2522d690f5633dd9f72d28b48ccca \ + --hash=sha256:ac746cb365bac1c462da9e3e6ab8904a8efe2217a56b0b2e3d9480f41d2b2602 \ + --hash=sha256:ad474c11d851b6fc97cb625e4822bc0cbd567fc07dc2602e28faec5a36b42bbb \ + --hash=sha256:b03ca066b47b18b205cc080dca6f76cbd159f8cdd33a02a0700164c13b37e463 \ + --hash=sha256:b1cd4d66ce894a45482e1ac2837c31d0bd447df35065e542b60055aa2d00404b \ + --hash=sha256:b25426f9f6ed402835617c8f23609a47045f91ecff365eb6734817e039a8ed25 \ + --hash=sha256:b367c342327717d644db4c0ddb37ceb655c84822215ea0773a3a36911b74b71d \ + --hash=sha256:b7e62b8fc7bd6cad007b9f2e0ad9c8d4854c06350d5f51e1a439dd18b510ecac \ + --hash=sha256:b8b7aa75146266fd3e2a2437cf69ae188688c04ab8665b163d4257b46c1e0c83 \ + --hash=sha256:bb36381e1f9f9d06eba2f10bdd438e5d20c07d5b55e1a3eee30b9f44cbf52316 \ + --hash=sha256:bb8c7da8c861391f7ae48e3593762be2dabe405109e01aec520fbe1a6d15d14b \ + --hash=sha256:bb9a60b7faa5d37c426fa91cf4d6738182a1f2755b9fab7c9c64cd466c4ce51e \ + --hash=sha256:be007d1aee2cbd530347dcafedb400891a3b5f1bd7135f95cf5d5b330b5219ee \ + --hash=sha256:be569fff1d85cd29391c431c5641c8772acb75bbdc61e60a8e82fceb9023d385 \ + --hash=sha256:bea7df027015856ba5d0a88e3b4777ff8cb5c66b58fc108050fe79d4dd9d4d2d \ + --hash=sha256:c0fe437a6d2f36aac2b49517057776575b5bf359df314cca20d230a6e139c089 \ + --hash=sha256:c2b2a96cf1dd99fe7867be4c013314225f4d5786e6685906e29932d42aca6f11 \ + --hash=sha256:c2c5fd0fd39574ccd58e1a52565b341aff522c5c836f1b3eb7605c371e61f52c \ + --hash=sha256:c46a08bf070d6849fed483e9d9833f9d06aecb8382ed985be0b38508b3ae958e \ + --hash=sha256:c5f3a2af441670d80ce5fdf13b6c1b421fc1fc7fc5182d58ac7486738bb2b742 \ + --hash=sha256:c60e50bc5b07faac92fd3a20fa21cc8cf3e3f7204d2867b206c73293ebc19101 \ + --hash=sha256:c68e0c0649d17c2d0339e3674e86a4aeba4a7e6b21c1e394cf947a95433b31d0 \ + --hash=sha256:c9c98d2f0126ba84cb45601eed97ff67ff767e19ae6eb3c31b02827b54d700e5 \ + --hash=sha256:ca52b9ec80851366197577154c862c4c4c7036ca76ae94cef5cb59c5cfeab944 \ + --hash=sha256:cbd86f9787c5e2f5fd27d8b21458222f107347c6731c4e93dde68f554b466a2d \ + --hash=sha256:d0264f8d5cb0a803f650a6a8572dfa0cd1e099a2234c588dc8fb220b415b865f \ + --hash=sha256:d0be2b832435001bc623ca7f1499ca1a853d4f082fb61221a80ce71132f50b26 \ + --hash=sha256:d244cf6b52b5ba1c34c3832f4652a668ebb36d95949b96eed9a1c54d916a90dd \ + --hash=sha256:d2d236b8a44ae91536a12ebcb996bdb31cf27425f36b4d05c87f2ba2716050ba \ + --hash=sha256:d3da668e903c934ed0b587ecacfed6901f6ae6384a6e975887592b61845e78bc \ + --hash=sha256:d6dc7804c50fabd28644d4d18a4b20aad3681b3e64f3acd3182b330ca73f7a32 \ + --hash=sha256:d7e5ba0a0153e35fbce9c51df530c8b4cb0c3012b46a04ff9a048441a269c2ed \ + --hash=sha256:d8a5ac357ac283490a8d1899b0383355fd1f8634b14ba0d59e4c0dd97db85556 \ + --hash=sha256:da1c112c5784ccd9d32cd90be6739fee32644e874eff6ae8f0497cba3e352e58 \ + --hash=sha256:dc911ae6152e455b16a2a1a626aa6cd612fa01efb9d0a4ab3f5cf328b911483d \ + --hash=sha256:e0db3a4d1e264e225037a6023888972c25206a96e016021a5bea41c9a939f2a9 \ + --hash=sha256:e192018b732f7b168e6604cbdf40fa8e05c996693b9eb445a0d8a73f4b77c5d3 \ + --hash=sha256:e37b744849fb631bb52e3dadde35ffeee365a6c41cf71257b5b7acc9cd83fd38 \ + --hash=sha256:e41226ecf607f062fe34a2f4cf64ad3a89e3a0180dc800b463b6b14c06dd10dc \ + --hash=sha256:e418ec99574ca24365ca96546af285c2b021a1a072478a79f0e3cc3b08837154 \ + --hash=sha256:e6ec7d37841609a691b96a10b4fde386c7cd93ebbb939f59c9f23325ee788395 \ + --hash=sha256:e886ef8c9879105fe4fc99417447b3a5f35d1131412ce839470bd2089fe2043f \ + --hash=sha256:e8e1e895e23818d343e4ae7dd95a0a556fdeaf8b471acf1c0a39b93c6f54d478 \ + --hash=sha256:e9dc7b4ff6ef184504b49ef9a4113d49a646653b2ce89f5f48c1f57cdf6ba081 \ + --hash=sha256:ea880d441be7c510106bc56064be39266d948aef94ad4955e8784690019a5d9f \ + --hash=sha256:eabb03dc3e4ed6333ecd1cc9826ec80e7a98b5506deeb832d7260c8e44166d23 \ + --hash=sha256:ec0a4d066356054d569a66e0a94691a2058b680be5e710298f61db11a3c4609f \ + --hash=sha256:edda19aff836ec515caafc09ea53d2ab144a041f09ee9a7cefcbd3ae4e976256 \ + --hash=sha256:f1f4a220db6ed7c8fd16b6d644ffd1f082651693204daf3275e049fadc849e39 \ + --hash=sha256:f25b61a708bd276e8cbb6afcbbf1b8e793a3be70ba0a842d0b8692020f83b706 \ + --hash=sha256:f2fa3d3b1c933d4bcb8fd2018700d5e7235c52f2ab8c88d22286965c5c0f00f8 \ + --hash=sha256:f3071e6515cc63714d014da8f738ae9fa3997c476203f3cd46de380c2376ed7b \ + --hash=sha256:f3a0a31189acf6703307397c6139ddabd734c20c5ef92649fc93e473df6615a3 \ + --hash=sha256:f7eefd0233a7c33ca980a5cfef26f1e9b5e2137839e752a99963696729f12d91 \ + --hash=sha256:f8b09b25e0f4dc2ea9e2adbb1cc3ba11a94d6fa3dd978ae659c8743052e1afbc \ + --hash=sha256:f8d7b66c9e09c0bb0add2b5895e646b62a0849e71155066f215523de6b95cbe6 \ + --hash=sha256:fa6c2880709c84457de104385b704fc28860f27e442ad13966fc4af8e714fe9c \ + --hash=sha256:fc5460940f50dff00731b4132366840ba9685286ea88ea104b661899084f3fea \ + --hash=sha256:fd789a294d8e098528be29b2669b83005ce569339f8cef167fc0274c3115c34c + # via + # aiohttp + # yarl +oauthlib==3.3.1 \ + --hash=sha256:0f0f8aa759826a193cf66c12ea1af1637f87b9b4622d46e866952bb022e538c9 \ + --hash=sha256:88119c938d2b8fb88561af5f6ee0eec8cc8d552b7bb1f712743136eb7523b7a1 + # via requests-oauthlib +opentelemetry-api==1.44.0 \ + --hash=sha256:67647e5e9566edcf421166fdf022b3537f818635daa852b289e34604dc6fb33a \ + --hash=sha256:94b98c893a91b88657eaac1e3ba89618cdb85be6918196705354f34728b2cdef + # via google-api-core +propcache==0.5.4 \ + --hash=sha256:004e685b315646c410771836e72a44f143bbe624f29653a42687815069a303d5 \ + --hash=sha256:02c0a34f16889cf800f10f0247a564d8ce6eeab6ffcd7c87198f769067eb8432 \ + --hash=sha256:03969626faf0783a592dfa17e28eac06018bd0b44dafae6943d53b92421a7f72 \ + --hash=sha256:03b229037d25b801e7af53fd52b9fc49d9439b036fca1e087e02780631adfa97 \ + --hash=sha256:0951315a6b3142ee2167404d707743f0157c110091342b1aa0accac5cf0e4acf \ + --hash=sha256:0a095db8e15a6020db149ecbed6461939fe74f6acaa3ae8b702a1fe8c38cd983 \ + --hash=sha256:0c889f6fa84957bc7e8b4eab71fd16a0455068d5045e3aa40c733071d2b2fd77 \ + --hash=sha256:0d21d0d2c82bbfeb1677a9711f38df968f9837576102bb4add1bd449d28d88f1 \ + --hash=sha256:10ef33a68a61ce317e095fd2e202a592ea92392b90944a78c993f0d9a73ab06c \ + --hash=sha256:12682126712ddc19b70ff819debbd279e58adf1f0c8f8f8138c18ade2044b284 \ + --hash=sha256:135036c5cfc93864affb0f9af9a27e5d7a71cb7bd745e7b6dbfc2d56cc30e827 \ + --hash=sha256:13e52b6e0bde97dee98ab66552dbff2931649c96f1ac432eac299fe689ec373b \ + --hash=sha256:141fdbd73748db0cf7636035030aaac383d2efde8f34e7bc24594cc776d225b8 \ + --hash=sha256:146f48a9e4812611a7581003b1a39de56c34967046310c4171a68ef908c9a745 \ + --hash=sha256:174507f82d3594622acb1dd2dafecf2d899d6d506335494e7107767bf05f3aae \ + --hash=sha256:1783582065a1f07f9d9ee1e992e13f15d7dc8fb1eb3a7476d43eb3f2e69d26bb \ + --hash=sha256:17a7400cec0256f0a71ae71f9da398f9894c956ff6668a1c9d317b3367316320 \ + --hash=sha256:1b2f3bec4261a94019575481c726c29850f72e27907773c75b1de421e20e9f9d \ + --hash=sha256:1d759d05634f1b038fb625a66662a8c85e5a8fec912da381b5149ddac107482b \ + --hash=sha256:1df8d8561b21465c5dd56110a01caf897e026d065b4b84e98a488209094272ec \ + --hash=sha256:213bb68d9ced5cf2bf717b1071bf2b09b4b04c426256f9fe6d054c60318424c4 \ + --hash=sha256:23278f808cd81d5ada7184a76606b925fb3389c60e1077b2cd7da7b1fcf0553c \ + --hash=sha256:251c63dd46a0659bb875cb254dc4c1e79ee91a847c737cd62373295afc2235dc \ + --hash=sha256:279655a16973f1ee2bd2fe79973137681642fd9ae0d89215bba263726eb0dc3a \ + --hash=sha256:2814ecd8e818f487bee4b0f921bc4d1c176cc5fc71ac0f072d0fa67eda4ac14b \ + --hash=sha256:286867fb156488c251a3721766e380ac4495e4fd6b51aaa1403d89ce7f4359d9 \ + --hash=sha256:2dba2f02d2d5c09ef8a0e6c1a42aeaa451f4be9898cb00b04fe98717da2eb23b \ + --hash=sha256:30cc1cebaf9aef49db06357a50398323ae04d70460c0491837d026ab7d6452ea \ + --hash=sha256:31eb43ba2edc704ab2ec27815315dd8a19def0fb16215be4cfe8d32fe78ffd51 \ + --hash=sha256:350b272b2279f4135a64fc0c304a5d08e28a137c9573442c606152446638a831 \ + --hash=sha256:36c0d9db44b523ef93d03341b1c42d69ff01d673c053d1b1c6c3a363bcaa39ba \ + --hash=sha256:3af0c8642b2da4815d86e631232ac8286e17644fad907c19508aa8e7cb4ba8ad \ + --hash=sha256:3cd3a7edb6b95b9b33998135ebfa18d709da82290fb8f27c858970b5a12c8b56 \ + --hash=sha256:3d605bb239b796e82a81c6709548b2bd460ab73b4590cb0c83de8a2dd9694d0f \ + --hash=sha256:3e413d7a4a9b4866b7a761d6060d434b64d23cd35122eda3b026a0bbe8196b25 \ + --hash=sha256:3eb2e820e8e2101407da93f17c57cbb7d225461955fc60105daaba14cd421ee2 \ + --hash=sha256:3fa15757fea1dfcd5b7745cad9f4638929605531bd4018ab2adff7955f1a403d \ + --hash=sha256:3fc24f209c1b7f7f688b66b98293954f5504279760999b58920ee12dd8471c1d \ + --hash=sha256:4054acf80d40456a0537f2913b349718649d8d6458a14ab7f48d0ce28c30869d \ + --hash=sha256:40e94adb1e7d39ff28a8bd8d8b8fbd1df6b9f40976dbe379134f1ce058e532dd \ + --hash=sha256:420162a77f94eb1cf5ef7893f500016dabd548e73de956785a1dd899cc73006a \ + --hash=sha256:425f8cc86ab5018b4b8d4a23bc8e74d964bd3d757c3702e301aa79be76c53f6c \ + --hash=sha256:44149f46500a0a41b95b4d99c2e586a77319539730607b9892974a092788b111 \ + --hash=sha256:445ee3bfb46e85838387fb3c536a73cc0b994dc192b004e40e170adc54aa2a7e \ + --hash=sha256:45488d1a5f9ab5bd90aaa1ca20f50fe1922b8ffad71a2009d2adf41355897aac \ + --hash=sha256:45bebbe252550fec975ba3b62bc6f931643cfd3b5464ef47619cf3fef154e01c \ + --hash=sha256:45bf2e730ab8905d0527fe05a86500f406e64305c34cc81ebe64b4617cab9760 \ + --hash=sha256:48cb48c5346a97de792254af77715aa2529c2a1ebc5f586aa0aae44a02f1fe57 \ + --hash=sha256:4a1f4f5ffa55dce6307631f3cb2948e117e665966ea512e0d502b16c24f567e7 \ + --hash=sha256:4cfe0a92ae30151869e67a4b5f5e105e4e03ad30b3f38e5211b5bf77d0881993 \ + --hash=sha256:4d86476a935c88963d9b8e1a9a0d38188790e9622169bfbafa173046846709d3 \ + --hash=sha256:4e985382be6d15da8d0c2710a6fa7b9070fc9ecdeefb7f580e88373984ec8be3 \ + --hash=sha256:4f2d880ff60f45898f4acfa152aac8d04e3ee627d90ff4003491bf92239d5757 \ + --hash=sha256:4fbc1a15dc8cd1689508758d626b372b1f09d28d9577667feaf9e6bfcd8efcbc \ + --hash=sha256:50e337653721d20ead710da33bf44487fbe8a0db8782714b60306481e9f95b51 \ + --hash=sha256:53eaa697c4d0422ff4cb714d00231b43352064d97b944033b30c1d57cc506ec0 \ + --hash=sha256:56fc3f7599528db40b1efa0889a620116e2704144495273d66066e8164e45838 \ + --hash=sha256:58134228927cee6c047d626c08e60a81be604a20578a12ce752cc5c9a84d4826 \ + --hash=sha256:594eb4c6ec35e7179b058481f4e9f02521b56de16fa577c4b85c76fb1bf8a9f8 \ + --hash=sha256:5cacf3c9efd09df409dc33654dd077e1c245ba8fb747b0f0236ef41b7c49b589 \ + --hash=sha256:60a64cbccaa11b7760ce705a14ada17ba459e7ca9f23ba587eb013821032d7ef \ + --hash=sha256:62530ca89187827e4a4fe733f971abe81a7542eeea48ff61995f19b64d7199c8 \ + --hash=sha256:62c60aec739ed00124573cce1178138fd690c7676352d67a37328c1cf51d7468 \ + --hash=sha256:69fc35c0779522da366c563e5faf203ffc1f8ff0021d5b1337fa4efa5be73177 \ + --hash=sha256:6af4693716bfb03f1752ef1b30faa593db2c01d5272e9b8564a1549452a979ab \ + --hash=sha256:6c7599df2b57ebeea8de011b5f2f7b85de95e76037d43d34b95e328430275487 \ + --hash=sha256:6e9368e87a3efc285e559131092c5db643eb8e56de4ee42064d5baec22ef2bb5 \ + --hash=sha256:6f0093ac3e9daada202c2082439d414a625c57184727a46e112a3fb2a81cb788 \ + --hash=sha256:7177c43eddf10a0893c4fec52ebb408fdcd7f7d63962caace9180d8f81b14ece \ + --hash=sha256:720cf832eb2d0b0dfee129cb3335a26f6ce3cc45ee1187e8f0731758caa16792 \ + --hash=sha256:770e8209d018175fc0063936fa9583b6d27e88c5ad31543f3383d66080efdd62 \ + --hash=sha256:7a8d5ff04eb1f85698a78d20c62a14676e7b960dcafde09a388d60ad377d355d \ + --hash=sha256:7b9100a93b372418d8688f3f2a3e5b45c64d70ca4d6176e121aca1e3bfc1e32f \ + --hash=sha256:7cc528e760a8af06f2b13e9b9f362cd90c7c718ea61228a96dbd31ba16ed7f47 \ + --hash=sha256:7ffafcbfc7b549ab940047e505c831eabac5e67de53e1bc174adbc5285c55944 \ + --hash=sha256:87a3caecf8095e48dc72f84bfa42e23a848cf410cc9cc13031fba4869b706a21 \ + --hash=sha256:886b59c4d28ca97dd23b025fdfc50a0356be934efbbbca89ad26230067f86fe5 \ + --hash=sha256:8876b39961e33d912afe3c1bee18ee564fdad0206f873cc15d522756b7f50737 \ + --hash=sha256:897d1ddf6716e8f47200f7aad9a0efa6cc7586df66c6defa572f9eab379c078e \ + --hash=sha256:8a1fc236528c457cd739c88abe823da851b7ab645d72792f88658114cc340c12 \ + --hash=sha256:8a235f73d6e020855dc29dff012d920c02ee0feab8d73a24185a7569f4be1161 \ + --hash=sha256:8f911c395cef73c510bac566da9507bb6a43e7763d0c79138dc60ee53f11207e \ + --hash=sha256:96f7c5c15656040ddcbc51e56dc59b58aa25999d743c126abd425b9766ab43e9 \ + --hash=sha256:978f28401afbc76cdc3df9e1717b4229a06b626a1dcc75db4e1f2beb3884c3e9 \ + --hash=sha256:98914de2c4d7f0f9f4a8c6ea4bf05841f4175796941e3ef7d47eb718f22311fb \ + --hash=sha256:9a2a8a50a93dee0268a860a07fa3b4bd968f8ce4dbd794957da772f395368526 \ + --hash=sha256:9cbfff4423eef4cc6cafc021469641a2b835f610b2647a6c5281903e21b8670d \ + --hash=sha256:9e9ab13760aa8b6d0881ae7cb04fd891d8d490cd2554ea8e79bb278399169bcc \ + --hash=sha256:9f3551b8a35c1df3e7ea4d2d86edee15f0dde1bddd434a71744048683544d0ef \ + --hash=sha256:9f86f7259efe2c951f43e57d471c9b41daa5bfc7db9f67189059cf1ae6d77fd9 \ + --hash=sha256:9fb0a5be8d9aa213150e8d8148a42aca4984b285bcad1e69587dc4298edd929b \ + --hash=sha256:a219f0ac59817a9114dd2aa57c13180f993e819ba658c7ddab4b66ed1ee0d370 \ + --hash=sha256:a419ee85e654927baabda3929c03c0cc1112bf472ff0dfd6142f4e3a81ca4162 \ + --hash=sha256:a4d7a54719b67338a305dca2ce6aafe366817df94ddfd4b5514374356f5ca546 \ + --hash=sha256:a5793c7698a53f56f4a1889a4737c7eeb1b7ad0842fa6b1abca22913ff79c8c1 \ + --hash=sha256:a5e8ef588c109725dc713ba69aadcac00a1ef90c2ce9c0a8c7075128f569f47f \ + --hash=sha256:a74bfa37147cc08fb29df10bd9c16f40fa7f860cd3a6d2fff853323a94f6e17f \ + --hash=sha256:ada748108a43d29b7c328ba7db3755327cd94f028bcc1a7ee3f0addcfacd9c38 \ + --hash=sha256:ae58f361bd5dae942717c65d3413b478c70aea9c462599e7b9adad3731db3894 \ + --hash=sha256:b28f41fa3b8c6900457f858ec5b03998f3a6d535fbc1bb2edec5961ea05ec429 \ + --hash=sha256:b3083bfe87f95c756e610bd8025f26cbd1cd4aaa03a422f2d65efb7a97cd53d8 \ + --hash=sha256:b61805357d966680acf68b3b6d49772631ed9df44ebece10ff1460e117a7da8a \ + --hash=sha256:b77c313314524ca9c38fbd70f73515d04597ac58c40c939bc0e71eeb4abff680 \ + --hash=sha256:bee7d3aed13d56f54e681df38c3a23031bc9e3863f687d9d598825c9146acd7d \ + --hash=sha256:c02c0e570c5c7e077b0181a9f3cdb7d4c3617d1cda6b5c95bd5d34022923d82c \ + --hash=sha256:c174bfd1c48a1b51a3078e95586dde718374bac79719ab3541ec9e74aec40574 \ + --hash=sha256:c2ba30a89035b57b73e00475de948521602f543d79ce01db10b04b36c4c76fc8 \ + --hash=sha256:c3e98c55bde2bcf7db3c70d1aed7ae9aa8aebbf19a250c66645cde44cdb8b867 \ + --hash=sha256:c3ef2818d63bc86071e9d2989ae75a1bc32b8f7059cfd9f5abbbee70c32e2ed6 \ + --hash=sha256:c83acbce9f2b5e3f5f5eda9e53d2001fed22fcdfef81274a9e02d8fd53b70a30 \ + --hash=sha256:c9281e922c072158c91974d4589f1dbe0fee6d467f284c28e463f9f5a4d933f4 \ + --hash=sha256:cc07876cfb079b6f6f36d21ce75784ad6c2c6b563eeac0ed26c2fa2669b85df9 \ + --hash=sha256:ccf4f7a79e26bb7efb06ecd50c177833b71df05cbc748701372325e6bcc17f6f \ + --hash=sha256:cdee8205a44d0be91bbac4c41b95d86641b72dfc7aef1279400e4fda3f26a937 \ + --hash=sha256:ceb3e879afac028f93d272c957814695dc5569e4904262dbee92f6c41bd5e4a3 \ + --hash=sha256:d1f5a500bfcbb2c0ab85e98a0dcd70f5899d34efe365a0187700369a79603031 \ + --hash=sha256:d42a9a856a4a6e2f6c10f1318c07e7daa498d6593abe745c71dae4521a26ca39 \ + --hash=sha256:d83b12902eb8bce151259c86c03ba746600b2d994543de46e370cecf96c452f2 \ + --hash=sha256:d8e017eeb7482bed34cdb0d61cf2bcfc88d104bbab296a17cd16a6af8aabc70e \ + --hash=sha256:db3ae52ccc150dbc84704e9d642743897f3e1c54742ff34cacb661e52e3818a9 \ + --hash=sha256:dbab5f5ff6897c81f355d079010cdae85b02e5a0b518b5251523b8ad8ae9ac3c \ + --hash=sha256:dc4242ca653c9b30ab51c5f8193323e7bc0928f897ee9103201e59a43abcb72e \ + --hash=sha256:dcbf346a318a5e30063f547630b02bb787ce2f45b6368d5da143660b6a3835d8 \ + --hash=sha256:dd2ac8f5b643454c2cc6b6118b13da16e88f4a6434fc3ba61aca384029f04f36 \ + --hash=sha256:e1d52a05dc417279f7e5c7618c5dfbbc29923aaf9bc0a5c1802ddcebf54c61a0 \ + --hash=sha256:e6720ba44ad7e72174314d0e1fb0172494cff5c73a3a8a2159c3d2402ff15565 \ + --hash=sha256:e738ab81179510ce79b2eac9a6ecf47feffd9e76d1c72e403005dddb6e36c06c \ + --hash=sha256:e904d4d01f36bd6e197590be1533c44e06058771e0746dd073a8ebb3ef880858 \ + --hash=sha256:e9f165403b81fea7e89c932d89046a1e3d9a3a60e8d7ef2f249dccdcb0982bf5 \ + --hash=sha256:ec6a85f424afa8d23e0d9a094e5dbb6eda01da91c92b9183cd433768247ffc97 \ + --hash=sha256:ee19113bce2f3acd46432050688b70f61acd6857d75abb9ec96341b7e9ced123 \ + --hash=sha256:ef3b928d9c984322b5c44e6964d8dbc653da87d2d8ee1647fa6da43072e650a9 \ + --hash=sha256:f273dcf7149a50527c4fd1f55cfe9eac0f60753f5af544b4c9352578e20c0874 \ + --hash=sha256:f5470694918830da62fac9e69133b53d23b736d7070e587b27a4a2be37e08e68 \ + --hash=sha256:f574e460d1c8a08384a016fdb09ccf3543433263ed6b2f97104f979e64ea57c2 \ + --hash=sha256:f85915e00dcb1cd9f2f890ead064ed40a27df06f0db65be427b29482ae357572 \ + --hash=sha256:fc2461ecc45f17893f8207e73b46ea8ba93e33630e51cf4af3fbc21d47462b1a \ + --hash=sha256:ff6b113f50bc066a698db5d944d2c6dc7507168dd3341e255a8892fd0715a558 + # via + # aiohttp + # yarl +proto-plus==1.28.4 \ + --hash=sha256:4b01341272f8a348db3f003b6143109f83ab43091019d5181b3fcdf500ab32aa \ + --hash=sha256:5ff7ecad828e032a491fcb86947801768e32237f99dd049b649965b892ae9a63 + # via + # google-api-core + # google-cloud-storage-control +protobuf==7.36.1 \ + --hash=sha256:0b53ce95272aad50ad25d7ff03373743209822e8ba42ea7fad27d2bee1547d00 \ + --hash=sha256:39c518c05586c016d7874ff6079ee115bcec1ea5fbb1d177fbf7867ef4c67e44 \ + --hash=sha256:3cf2ee25d006cee57294a1196ea43b37feb78e0dcd1e8af5c1aeddb777655aca \ + --hash=sha256:43d3d37b1eb24c113b9b7d02008cac44e423f00b611b7781ae998d7623972969 \ + --hash=sha256:51139351435d9b43d88a55eaa49fb6f737fbb478fb0cbf2cf694d1a04a9d3363 \ + --hash=sha256:7d951e46b3f963d6c264c367c437921de9d5aedd9c3f9612b9077736b4e3ad5c \ + --hash=sha256:97198b77e369a0abd8e262b8f6c7266c55ddb796a3a12c76d7b8881188ed83aa \ + --hash=sha256:d0f6470f0ce2b84e3feaea2d4b816378b37ba4d4aa08a274305373de93e2d524 + # via + # google-api-core + # google-cloud-storage-control + # googleapis-common-protos + # grpc-google-iam-v1 + # grpcio-status + # proto-plus + # xprof +pyasn1==0.6.4 \ + --hash=sha256:9c447d8431c947fe4c8febc4ed9e760bc29011a5b01e5c74b67025bd9fb8ce81 \ + --hash=sha256:deda9277cfd454080ec40b207fb6df82206a3a2688735233cdcd8d3d565f088b + # via pyasn1-modules +pyasn1-modules==0.4.2 \ + --hash=sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a \ + --hash=sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6 + # via google-auth +pycparser==3.0 \ + --hash=sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29 \ + --hash=sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992 + # via cffi +requests==2.34.2 \ + --hash=sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0 \ + --hash=sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed + # via + # gcsfs + # google-api-core + # google-cloud-storage + # requests-oauthlib +requests-oauthlib==2.0.0 \ + --hash=sha256:7dd8a5c40426b779b0868c404bdef9768deccf22749cde15852df527e6269b36 \ + --hash=sha256:b3dffaebd884d8cd778494369603a9e7b58d29111bf6b41bdc2dcd87203af4e9 + # via google-auth-oauthlib +setuptools==69.5.1 \ + --hash=sha256:6c1fccdac05a97e598fb0ae3bbed5904ccb317337a51139dcd51453611bbb987 \ + --hash=sha256:c636ac361bc47580504644275c9ad802c50415c7522212252c033bd15f301f32 + # via xprof +six==1.17.0 \ + --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \ + --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81 + # via + # gviz-api + # xprof +termcolor==3.3.0 \ + --hash=sha256:348871ca648ec6a9a983a13ab626c0acce02f515b9e1983332b17af7979521c5 \ + --hash=sha256:cf642efadaf0a8ebbbf4bc7a31cec2f9b5f21a9f726f4ccbb08192c9c26f43a5 + # via fire +typing-extensions==4.16.0 \ + --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 \ + --hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5 + # via + # aiohttp + # aiosignal + # etils + # grpcio + # opentelemetry-api +urllib3==2.8.0 \ + --hash=sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3 \ + --hash=sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63 + # via + # requests + # xprof +werkzeug==3.1.8 \ + --hash=sha256:63a77fb8892bf28ebc3178683445222aa500e48ebad5ec77b0ad80f8726b1f50 \ + --hash=sha256:9bad61a4268dac112f1c5cd4630a56ede601b6ed420300677a869083d70a4c44 + # via xprof +xprof==2.23.2 \ + --hash=sha256:38e809a29b8226183143d7adf59321f96f6b09ce5a0db0810162a2d9742187fa \ + --hash=sha256:921bc9b1d28ea78d7c63614e28b88dadbe535cf5b6c07c7b65a829a329635a9e \ + --hash=sha256:cdd3387cc357f08713df10b1032977d01c478690e7601f833de13a8ef763aa7b \ + --hash=sha256:d2b802594cf853c509e502a63403c481e607630a98d9ace4cb5fb275c707b5f5 + # via -r tools/xprof/requirements.in +yarl==1.25.1 \ + --hash=sha256:0136d640dfa9b0523853e411430a99f8a91eca85774c6420285a33b755bc6de3 \ + --hash=sha256:03dd38de09bc213e9a8b29761eec33ee1d5318dac0e49d8af36e4d27830e23a7 \ + --hash=sha256:0a191bfdb30a79b98e5d175d75285f9fcb78bf0e46ba5efda042e1c72071a0de \ + --hash=sha256:0a66db89ea473abeac4b70523cafd94db3772380e565f9d28af7a179b7af71fa \ + --hash=sha256:0ae12ff2b805fa02c4dab838005caef735e39986322698c48588d3beacb65c62 \ + --hash=sha256:0f12afda4eea8c8994a76d4df1875c765194f5fbe8a9d197929ea303caee29ec \ + --hash=sha256:10b2fd95332f0d716d5eee3c9fb2ce8eada19082de7fee83d32e37992fd75c26 \ + --hash=sha256:126a2533570c554719ca40a1288fdee1700b6bc82e7131aa69fa85252d92e651 \ + --hash=sha256:12b6bc4906e11f5e1a1cdcb12296e7afbd366c783cc8073403cd2fb74334e453 \ + --hash=sha256:142c06c4d6a35ee3ec5da08499805e879cb3ca7c1fbfbecb0140fe72403818d6 \ + --hash=sha256:14b79a30a93a3ce2e8832603fd0ab780ada281b0ba5110b519a634f2d7d7d1fc \ + --hash=sha256:17c9877a89fb6e2bca6f9087eb24cd7fb434653946ef5075e470d23d49b52287 \ + --hash=sha256:192a866877a49993949ef1975864ad8728bea28ee810f6abe1a0729c2b500426 \ + --hash=sha256:1ab7618921a93767387a4b83776f751588f5b5ae9bb5bc96620e2e2e00bca868 \ + --hash=sha256:1e80dcf1446e1b080b1932b0d103c464a04112f5bc31f0f983ad418172063cde \ + --hash=sha256:1f51020b2eb8a003c84925638ec63c21a750a4bddd3a22ec8eac6a742dadf1b9 \ + --hash=sha256:1fb2a01ba8cd9c5d2c5dc1ec35e0fc951d04b4f037541d4ac090c993ce58b3d7 \ + --hash=sha256:2239a02249d9326655419e0168a28ca9008938eaab31dc29fc875c217927a6c0 \ + --hash=sha256:23bf5b403c879a54964e0feac7285688e04bb220074878d737d331522da0a5bf \ + --hash=sha256:24ce942011a61953e7d313438038f4d32ff21387b775f58a957f7a07dd55ef95 \ + --hash=sha256:25868beca8b6765f8f7d0e11fe6dd7c66dd4b0793b9500286d20cc92352126a5 \ + --hash=sha256:287e99ff5aa4dc1c7630bfc683ded6f106d756c99dec432a2d7f197a784f51c6 \ + --hash=sha256:29273edf1530e397bd07cb784db1fbe0d2590b77569f2e24679a9c0a2d763b94 \ + --hash=sha256:2b49375d22299b0a834c2bca72f39aaecc270d96fb24c30424899676f487b22a \ + --hash=sha256:30eec96e8a91bd588ce897c9543f6d5d8d34b28fbcba28a4dedf20ebeae9fe57 \ + --hash=sha256:319e070a01db9920fb63761843f96a104c8e2b9427266731810dc1e22595b17c \ + --hash=sha256:35dcbea443fafb3eece757ad4e514560ddeb6c34cfae1582c620d7b293d7feee \ + --hash=sha256:3f4d48a6112712973e676bd792121fee470e432d749177162d9949d5c9460a1b \ + --hash=sha256:3feb99222553a8cbedfa52c2f59dd84c3f50d5b582c728d522caf8d72769a54b \ + --hash=sha256:419f392a1da624877975709e3864dfe833af6cc7671b39318086d456e288380c \ + --hash=sha256:42a66563d8cc056ee32e6191e05097a7b2b3bc302e0bc3133daf8710eb18bd26 \ + --hash=sha256:48796ea00a303961507dc6c8437c4b325a6fc3f95f7c36c71b91ea9a8150963c \ + --hash=sha256:4bd6340d20ae2c7ca719b87b426e808e90743b676d05d4c26c4fb5ca71f41184 \ + --hash=sha256:4ca89e4e21854ed27ec753297dde84b16c9f8e53b14a4866fb44457d643c19f8 \ + --hash=sha256:4d781294bb815ecb5ea57ff6bbf8038e0a31a95fdf3e1788f66e0dc100d64b58 \ + --hash=sha256:4f1c91f5a5980a937ff8e238e98e6897e1ad74a4b1e2c0d68c73b5ffbb3f5c0b \ + --hash=sha256:564fdc7085d2245ab84f88882fdb1d6ac0723124bff6ded35bfb1c00f812630d \ + --hash=sha256:59ba3a6e1aa8cfe5adf4bd270fd965db21955401b7ca6f1696010c55ed4daec2 \ + --hash=sha256:5df89f769cc8ff94c3d7e7603386fba309d25ce5240132d26c15baa8d0e96c4c \ + --hash=sha256:632da579b2d879f6bad20f2cfa35ded1efe2f4f77f8abb26a6234a5b236acd2f \ + --hash=sha256:65b5b2066651b7432d389e9799d979c703bcc6ef44266bb8153ef54e91e4aab3 \ + --hash=sha256:664ec6a520b74a1df2810666eb67695fcb77fa663e6ea0a25aaf2e529cb24dfa \ + --hash=sha256:681c758b0490f9e96b78e5fa8e8dc6e648e9185bb6eaebe73183c33ea0c445f3 \ + --hash=sha256:683e362b8ba453080f7489c66f4ea794e751c35b72e7eab3575ef784c2fbc7fb \ + --hash=sha256:68782fdb4027b8d1eee25ec35e9a6db05e863b899eb0310b3a33b6c3fef55707 \ + --hash=sha256:6efaf45df6a849cef613a03a94c845647456662f85438c886bb67a9c027c8c2c \ + --hash=sha256:71f42c5b9a948c113bbdebfa544598321431d064ff959d32e99b1feb61d68345 \ + --hash=sha256:72849d892954be4d09e569b8b831ac39ce58417fedc767d4308a0fe542018a40 \ + --hash=sha256:72c34ac7ad4314c19362d5ce27626dcc8429bd30bbf8c179f4234078851f9492 \ + --hash=sha256:734f6e5400352ac4254456003d462866c684703570929cff7a7bde015d0cb371 \ + --hash=sha256:75baa6cf9b6d1c52f3e111a130e202fd8cf0a5b3a066c3f73d615e885092e4ec \ + --hash=sha256:77716e245c90f058466a05e6a465bb8600f767a8f4b18b4d40f3aff958e5f73c \ + --hash=sha256:77e5099b99b37f3cf79c246998ca9f7313a78054cd1809ec46bc1afad47e1c4c \ + --hash=sha256:783dd1467083f4d3f7722ad6a313f24c173e7571372738fcb7a6e6d1ba48df25 \ + --hash=sha256:7a5c3115595995779ee21f2567035793911c3802a43c74f3fbb0314929ec67ac \ + --hash=sha256:7c88edaec8c349ad4c5ad4c486a3defcc4b80ceb2f074436ffa0a87caf5e76a6 \ + --hash=sha256:7cb414a73e21a7ab58254926073f2930cb22f5b4314ea4260a687e2b3fd4dce3 \ + --hash=sha256:7d42e7e3ca399555578b4d617e3a6ecf13371b3743a115995fa010c7bf341459 \ + --hash=sha256:7d575b54cb3863ef9bc290ea4b009999d55dc237326131e4853cf33e888fee03 \ + --hash=sha256:7e4de3ac4adbad3d0bc7c6f4360a7dbff5de2f15e3b723be3198074e17fd9c40 \ + --hash=sha256:7efc9f082dfed77c316edffa9deb52888e1bc6789171887cc1f68e06d65465c8 \ + --hash=sha256:80a063f8297fc796296f00f100be520f209b23dc98f93ce8eba6ee7122598209 \ + --hash=sha256:80e47012e730da131c9f059c80936783f9659aae22dc31c03c0595590d11ed54 \ + --hash=sha256:83d4a37e4b95da4d8bda930d6d35b75b4cdadbacbb4980cae290ea3100b5d51d \ + --hash=sha256:83e9f4a25085bd4b7214701a0794ff1f50fc633ffb8bdfebf07abdd81c2db126 \ + --hash=sha256:85a18376073f8a39aa07be34f9fc77e2869aa72c55c441efdd2cf79a0407504d \ + --hash=sha256:87796fedc3ba97ec14fab55acb48584276e6c1e4c1e89c422bda62c838e754a9 \ + --hash=sha256:882569ff613758cac762a457a5d72d6e211b28d4bcfea89d1d71ea942b02eac0 \ + --hash=sha256:8ce4d6ccafb33d39bd78444612d14938ead674c25702ded2ee9c54a47735d225 \ + --hash=sha256:8e7d98cdbb6d71e726f7d525952867096053d1f290dd4e3c50d7d313a136f414 \ + --hash=sha256:8ee202350cf57abf0e9502a41601841019c25d3db7ff52d980aaf31446254059 \ + --hash=sha256:8fb0eb4955adf0579001581f2f71a126e8781ba61bcd120f127b0401163c6c2d \ + --hash=sha256:90c30ed53546da833c700115c0064c22120d1b1560f474699fd31f22dd668233 \ + --hash=sha256:9489e6abf47ba37f332075a91444c7cfedb03e6ce99fbb2f116bfe1ce810da3b \ + --hash=sha256:94d7aa6debf92a1dd14cb5280b083a764169a13cfb23a452111160274ed989f4 \ + --hash=sha256:98d370568f393215d605304cdb77b3d5539bd192c75b623c7304c42c8d6d8273 \ + --hash=sha256:9b1bdaae98bc016825dd3c9d8ee1832f829b3341f9cc6ebd1a1b0a7fef7367cc \ + --hash=sha256:9d693bf4bf534e9ba3ae2780cfd577f5135629f7b5ac653490859d0b77864865 \ + --hash=sha256:9d6ed3d17bccce4c05343e1ca8da13bc5c02c812a4e7282ddd05e8769322d3fc \ + --hash=sha256:9e23c82b63cd7652fc24d33ed6cc17099d607aa3b4fc4ddc75e95062f3d82df4 \ + --hash=sha256:a1daf47cd95a7c3a63456336bc5aaa8c86dd3a47d07ed3d0e76132ae4666a5a1 \ + --hash=sha256:a1e32763e641a1566507d90a8d3b19bfc3cc04a9d4e5ae3e32189874ed4b58a3 \ + --hash=sha256:a2059a2d891bd156bc5184e7ab7a56e78a84dfcfdeac8c501b552533ad1c36ee \ + --hash=sha256:a2ed0ba415ccdf08f14bf544cb78346d0f76086707ffee24921a2c84dbf1305a \ + --hash=sha256:a3faadac7d812ddac258feb57b9846b60c1b437c4f4b9ad42595c6f6fe4390df \ + --hash=sha256:a5877f2255aab518ebe528289037699201d5dc5f045f2396cb30aa02db22f57f \ + --hash=sha256:a78b50b4f7918a3de71105d5c0b93bbc57bb8339a4d03a9dfd449f9068e76f3d \ + --hash=sha256:a8c2b841478068440d8b733005d13a5ef535b9928cbc05f17182d410f32ba449 \ + --hash=sha256:a9ca696eb02e5c02a8afd872ada510eba9b7fe6e68b9572c2e9a9b1941e31e2e \ + --hash=sha256:aa4ed3dd308548f9e707d9caaf005d2d7f8c1e7868f858dfeb47fe76e16b391d \ + --hash=sha256:ab2054c5531af2a9ba7b69b8ec91e4f884420e83a8c5e579b013084cb57e5e5d \ + --hash=sha256:abb1384477f5901d436b5d2e5465954de46ea6098f59163d243660b5c4461d35 \ + --hash=sha256:acae6b45d1ace09b6ba3876da43b88366ef368f73b988c7f57e14231753d4420 \ + --hash=sha256:acfa7e22aa6c6e7a5996a41d275bfa01efa7ea56ab890590280e9063e2cf5c1b \ + --hash=sha256:af4ea5b37403ef4e30f3927eaed540db942bde01d8d3ff083527c0704d1c9c68 \ + --hash=sha256:b10dd0557ba422715b5206b3743192135a6022acca8baec51aa127d0a75db8fe \ + --hash=sha256:b13b88747769537f3d32e89e3a735da10c0a9e35d7322928c701b5f93d3afffd \ + --hash=sha256:b51c159a9794633f5e0db7ecec7b2b6e3734eca1f5d17dc989ff3552a43ff78b \ + --hash=sha256:b5402a340723fa7da00b5cff987ddab61276be6d11251ea71ae02bcac54890d8 \ + --hash=sha256:b7abffdf37af1cec6a2ad69b827aa84320db5894791bc8ed932dc93fb274b7e9 \ + --hash=sha256:b8075fe90bc08e40b8b8a1874fab42ee4c7b56af05c5886e9cc841397f916908 \ + --hash=sha256:bc3ac7bf569f6b64dad04dd7808c7872dae8a97df657856eac05e9b7e3614a85 \ + --hash=sha256:bd0912757081f89b107d6c00b2ff8a194401b0b87eadcf4481de2b865a8fd44f \ + --hash=sha256:bdc8d8b8c22e9e43ac68316b5e6cf083dec537f4ec213cb4aa967b583bc3fa64 \ + --hash=sha256:be80550d9bfe83d9b62398a37081a90434e6df2d978ec345c3d2820de6beddab \ + --hash=sha256:c6f117789d22dce188e5754e8bc65b7e6ebf8cb73963b9fa761f672a5883769d \ + --hash=sha256:ca32926d7d77bcc8838425c4c95e040a3ace1cb7dfdae599013458dcda2607ca \ + --hash=sha256:cce0727fd5ac04d372fa9bbfde9febc2bcf209aadfcf0468e45dec72719895d1 \ + --hash=sha256:d0f1489233a254bb3643d2f05de7d59019254d81daeca6b9162fe9edef57e0c7 \ + --hash=sha256:d1c557dfd5e3db046053a0bdc72261ade790ebe8e2c7a41b36b0ca1f14cb95f3 \ + --hash=sha256:d21f0fa80a02d05299207eeaafef345d812ace96d5306e4ef265e1d419a615fa \ + --hash=sha256:d45673badd08456d0340e9364eddafe1c53a9d2896424294de4d7dd71ad3ee57 \ + --hash=sha256:d5add7b4ca7afeea91d52e4d4e4db3b1fe9885b71f07054560d8c4296b7441a2 \ + --hash=sha256:d5f90e44653c4e0f78501ed9bb7d3fce835a8d62b7c6ed0cb16557534087e743 \ + --hash=sha256:d7306dee25b8a0e737363f347362b875094b4dc4e367311470656ae420fdbf8e \ + --hash=sha256:dbcef5a9119ef653653132cccaf999b30a0af6f33bb0a4ba80bec30056868487 \ + --hash=sha256:def538065f9e4d4cf1ae164bd59aba00dfa84f03923e0de4c3788f252d6bcd17 \ + --hash=sha256:df23df54b5114a17c2d0ef192433e2e5a9f0c5178c32375e90b7cfc965f349d0 \ + --hash=sha256:dfbf531053a0935f2e871bcd4753f90313688772ff8c017f5ea402e315a78c1f \ + --hash=sha256:e029648f9c951db30e98a7d7ec90835db88ec4b32820efe2a9bdc2287e032eb6 \ + --hash=sha256:e07595c7d6f4db270ceede356a1bd1c07a34f1c26f958d1ed0cd7b48e0d2bba3 \ + --hash=sha256:e12c538e00e7c1b286a07061046b90e8124e6a9793efae2c70db6a4aad07faad \ + --hash=sha256:e546fe1d4a93ebc2910f0d768baff19faa09843ab3f2036a67ed6e69fae4419d \ + --hash=sha256:e5637ca8d0bd7fb72648a6c7934af4baaccb697657f7438c9d264fc2abb8b0b1 \ + --hash=sha256:e636b64d24fd9c38053c5e389a1174c66361fa49dcfd220f4dd35b4abde7cb89 \ + --hash=sha256:e7011b8fb8c4054bf0c12e5edc6cd83778b0028e99ce59b18586ed036f92cfdc \ + --hash=sha256:e80f557716fd765439577131e526b8942ffc2c07bdbc5e39fa62f660ba1e963f \ + --hash=sha256:e92b6bcc741b86d67606c40d3cb9c7cc8e6c737f81e31f4a94efc204456c92e3 \ + --hash=sha256:eb96ed1ae6c7d072d60840c0434aef07a2df611812810807fbc54263a6053e9a \ + --hash=sha256:eda19ea5ee88742f47a2340816e6f2d40b53bed3ab5b69794769f36af9f35bb4 \ + --hash=sha256:ef74070ac553c59eb4f04258722066d6c6135b7baa03b2e9f2da65c096e96d98 \ + --hash=sha256:efb01a106f971cb3752856bca2318bbdf7f01bd8823779c461586cbe5ffd5258 \ + --hash=sha256:f074e8d4aa0a5798920ddb6de3d08b228c614ff3724c3e8bd7577f4bafea867b \ + --hash=sha256:f38a70074041d3b7e138e452799f5174198bae5bd5ab2000917badf403908c5f \ + --hash=sha256:f41753a76f4f63927d03a0d8ba8f5ce0f2083bec29a8cfaccc55371b1564b96b \ + --hash=sha256:f53dcd26694f148f738edc052b5a69234833e739f10f4c3287bdfd8ec0f7b326 \ + --hash=sha256:f61964f235a43738bfac50da46fc4254943a7eea3051aeb0b6fc7c992c29fadc \ + --hash=sha256:fe01645169a2112aa1d4ebc3e4c5f029c5c8f97adfc32e5d37c993b39a994d75 + # via aiohttp +zipp==4.1.0 \ + --hash=sha256:25ad4e16390cd314347dd8f1de67a2ac538ae658ed4ab9db16029c07c188e97f \ + --hash=sha256:4cb57381f544315db7688e976e922a2b18cdb513d21cc194eb42232ba2a3e602 + # via etils diff --git a/tools/xspace_to_perfetto/BUILD.bazel b/tools/xspace_to_perfetto/BUILD.bazel new file mode 100644 index 00000000..5de164d6 --- /dev/null +++ b/tools/xspace_to_perfetto/BUILD.bazel @@ -0,0 +1,32 @@ +load("@com_google_protobuf//bazel:upb_c_proto_library.bzl", "upb_c_proto_library") +load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library") + +upb_c_proto_library( + name = "xplane_upb", + deps = ["@tsl//tsl/profiler/protobuf:xplane_proto"], + tags = ["manual"], +) + +zig_library( + name = "xspace_to_perfetto_lib", + tags = ["manual"], + import_name = "tools/xspace_to_perfetto", + main = "xspace_to_perfetto.zig", + visibility = ["//visibility:public"], + zigopts = ["-lc"], + deps = [ + ":xplane_upb", + "//upb", + ], +) + +zig_binary( + name = "xspace_to_perfetto", + main = "cli.zig", + visibility = ["//visibility:public"], + zigopts = ["-lc"], + deps = [ + ":xspace_to_perfetto_lib", + "//stdx", + ], +) diff --git a/tools/xspace_to_perfetto/cli.zig b/tools/xspace_to_perfetto/cli.zig new file mode 100644 index 00000000..f338855a --- /dev/null +++ b/tools/xspace_to_perfetto/cli.zig @@ -0,0 +1,34 @@ +const std = @import("std"); + +const stdx = @import("stdx"); +const xspace_to_perfetto = @import("tools/xspace_to_perfetto"); + +pub fn main(init: std.process.Init) !void { + const allocator = init.gpa; + + var threaded: std.Io.Threaded = .init(allocator, .{}); + defer threaded.deinit(); + const io = threaded.io(); + + var upb_arena = std.heap.ArenaAllocator.init(allocator); + defer upb_arena.deinit(); + + const args = stdx.flags.parseProcessArgs(init.minimal, xspace_to_perfetto.Args); + const input_path = args.positional.input; + const output_path = args.positional.output orelse try std.fmt.allocPrint(allocator, "{s}.trace.json", .{input_path}); + defer if (args.positional.output == null) allocator.free(output_path); + + const stat = try std.Io.Dir.statFile(.cwd(), io, input_path, .{}); + + var file = try std.Io.Dir.openFile(.cwd(), io, input_path, .{ .mode = .read_only }); + defer file.close(io); + + const xspace = try allocator.alloc(u8, stat.size); + defer allocator.free(xspace); + _ = try file.readPositionalAll(io, xspace, 0); + + try xspace_to_perfetto.dumpXSpaceProtoToTraceJsonFile(allocator, io, xspace, output_path); + + var stdout = std.Io.File.stdout().writer(io, &.{}); + try stdout.interface.print("{s}\n", .{output_path}); +} diff --git a/tools/xspace_to_perfetto/xspace_to_perfetto.zig b/tools/xspace_to_perfetto/xspace_to_perfetto.zig new file mode 100644 index 00000000..46aec099 --- /dev/null +++ b/tools/xspace_to_perfetto/xspace_to_perfetto.zig @@ -0,0 +1,712 @@ +const std = @import("std"); + +const c = @import("c"); +const upb = @import("upb"); + +pub const Args = struct { + positional: struct { + input: []const u8, + output: ?[]const u8 = null, + }, + + pub const help = + \\Usage: xspace_to_perfetto [output_trace.json] + \\ + \\Converts an XSpace protobuf profile into trace-viewer JSON. + \\ + ; +}; + +// XPlane schema source: +// ../../xla/third_party/tsl/tsl/profiler/protobuf/xplane.proto +const kHostThreadsPlaneName = "/host:CPU"; +const kGpuPlanePrefix = "/device:GPU:"; +const kTpuPlanePrefix = "/device:TPU:"; +const kCustomPlanePrefix = "/device:CUSTOM:"; +const kXlaAsyncOpLineName = "Async XLA Ops"; +const kOpaqueBytesText = ""; + +const kPicosecondsPerNanosecond: u64 = 1_000; +const kPicosecondsPerMicrosecond: f64 = 1_000_000.0; +const kOutputBufferSizeBytes = 1024 * 1024; +const kHexDigits = "0123456789abcdef"; + +const kFirstDeviceId: u32 = 1; +const kLastDeviceId: u32 = 500; +const kFirstCustomPlaneDeviceId: u32 = kLastDeviceId + 1; +const kMaxCustomPlaneDevicesPerHost: u32 = 200; +const kLastCustomPlaneDeviceId: u32 = kFirstCustomPlaneDeviceId + kMaxCustomPlaneDevicesPerHost - 1; +const kHostThreadsDeviceId: u32 = kLastCustomPlaneDeviceId + 1; + +const StringValue = union(enum) { + text: []const u8, + int64: i64, + uint64: u64, + double: f64, + opaque_bytes, +}; + +const Arg = struct { + name: []const u8, + value: StringValue, +}; + +const StatBehavior = enum { + emit, + step_name, + skip, +}; + +const JsonArrayWriter = struct { + needs_comma: bool = false, + + fn nextItem(self: *JsonArrayWriter, writer: *std.Io.Writer) !void { + if (self.needs_comma) { + try writer.writeByte(','); + } + self.needs_comma = true; + } +}; + +const EventScratch = struct { + args: std.ArrayListUnmanaged(Arg) = .empty, + + fn reset(self: *EventScratch) void { + self.args.clearRetainingCapacity(); + } + + fn deinit(self: *EventScratch, allocator: std.mem.Allocator) void { + self.args.deinit(allocator); + } +}; + +const PlaneContext = struct { + plane: *const c.tensorflow_profiler_XPlane, + device_id: u32, + is_host: bool, + event_metadata_by_id: std.AutoHashMapUnmanaged(i64, *const c.tensorflow_profiler_XEventMetadata) = .{}, + stat_metadata_by_id: std.AutoHashMapUnmanaged(i64, *const c.tensorflow_profiler_XStatMetadata) = .{}, + event_is_internal_by_id: std.AutoHashMapUnmanaged(i64, bool) = .{}, + stat_behavior_by_id: std.AutoHashMapUnmanaged(i64, StatBehavior) = .{}, + + // Metadata caching mirrors the role of XPlaneVisitor lookups in: + // ../../xla/xla/tsl/profiler/utils/xplane_visitor.h + // ../../xla/xla/tsl/profiler/utils/xplane_visitor.cc + fn init(plane: *const c.tensorflow_profiler_XPlane, device_id: u32, is_host: bool) PlaneContext { + return .{ + .plane = plane, + .device_id = device_id, + .is_host = is_host, + }; + } + + fn deinit(self: *PlaneContext, allocator: std.mem.Allocator) void { + self.event_metadata_by_id.deinit(allocator); + self.stat_metadata_by_id.deinit(allocator); + self.event_is_internal_by_id.deinit(allocator); + self.stat_behavior_by_id.deinit(allocator); + } + + fn getEventMetadata( + self: *PlaneContext, + allocator: std.mem.Allocator, + metadata_id: i64, + ) !?*const c.tensorflow_profiler_XEventMetadata { + if (self.event_metadata_by_id.get(metadata_id)) |metadata| { + return metadata; + } + + var metadata: ?*c.tensorflow_profiler_XEventMetadata = null; + if (!c.tensorflow_profiler_XPlane_event_metadata_get(self.plane, metadata_id, &metadata)) { + return null; + } + const resolved = metadata orelse return null; + const result: *const c.tensorflow_profiler_XEventMetadata = resolved; + try self.event_metadata_by_id.put(allocator, metadata_id, result); + return result; + } + + fn getStatMetadata( + self: *PlaneContext, + allocator: std.mem.Allocator, + metadata_id: i64, + ) !?*const c.tensorflow_profiler_XStatMetadata { + if (self.stat_metadata_by_id.get(metadata_id)) |metadata| { + return metadata; + } + + var metadata: ?*c.tensorflow_profiler_XStatMetadata = null; + if (!c.tensorflow_profiler_XPlane_stat_metadata_get(self.plane, metadata_id, &metadata)) { + return null; + } + const resolved = metadata orelse return null; + const result: *const c.tensorflow_profiler_XStatMetadata = resolved; + try self.stat_metadata_by_id.put(allocator, metadata_id, result); + return result; + } + + fn isInternalEvent( + self: *PlaneContext, + allocator: std.mem.Allocator, + metadata_id: i64, + ) !bool { + if (self.event_is_internal_by_id.get(metadata_id)) |is_internal| { + return is_internal; + } + + const is_internal = if (try self.getEventMetadata(allocator, metadata_id)) |metadata| + isInternalEventName(stringView(c.tensorflow_profiler_XEventMetadata_name(metadata))) + else + false; + try self.event_is_internal_by_id.put(allocator, metadata_id, is_internal); + return is_internal; + } + + fn statBehavior( + self: *PlaneContext, + allocator: std.mem.Allocator, + metadata_id: i64, + ) !StatBehavior { + if (self.stat_behavior_by_id.get(metadata_id)) |behavior| { + return behavior; + } + + const behavior = if (try self.getStatMetadata(allocator, metadata_id)) |metadata| + classifyStatName(stringView(c.tensorflow_profiler_XStatMetadata_name(metadata))) + else + .emit; + try self.stat_behavior_by_id.put(allocator, metadata_id, behavior); + return behavior; + } +}; + +pub fn dumpXSpaceProtoToTraceJsonFile( + allocator: std.mem.Allocator, + io: std.Io, + xspace_proto: []const u8, + output_path: []const u8, +) !void { + var upb_backing_arena = std.heap.ArenaAllocator.init(allocator); + defer upb_backing_arena.deinit(); + + var upb_allocator: upb.Allocator = .init(upb_backing_arena.allocator()); + const upb_arena = c.upb_Arena_Init(null, 0, upb_allocator.inner()); + defer c.upb_Arena_Free(upb_arena); + + const xspace = c.tensorflow_profiler_XSpace_parse(xspace_proto.ptr, xspace_proto.len, upb_arena) orelse + return error.MalformedXSpace; + try writeTraceJsonFile(allocator, io, xspace, output_path); +} + +fn writeTraceJsonFile( + allocator: std.mem.Allocator, + io: std.Io, + xspace: *const c.tensorflow_profiler_XSpace, + output_path: []const u8, +) !void { + var output_file = try std.Io.Dir.createFile(.cwd(), io, output_path, .{}); + defer output_file.close(io); + + const output_buffer = try allocator.alloc(u8, kOutputBufferSizeBytes); + defer allocator.free(output_buffer); + + var file_writer = output_file.writer(io, output_buffer); + defer file_writer.interface.flush() catch {}; + + try streamXSpaceToTraceJson(allocator, xspace, &file_writer.interface); + try file_writer.interface.flush(); +} + +fn streamXSpaceToTraceJson( + allocator: std.mem.Allocator, + xspace: *const c.tensorflow_profiler_XSpace, + writer: *std.Io.Writer, +) !void { + // Keep the same trace JSON envelope as the streaming C++ implementation in: + // ../../xla/xla/tsl/profiler/convert/trace_events_to_json.cc + var planes = try collectSelectedPlanes(allocator, xspace); + defer { + for (planes.items) |*plane| { + plane.deinit(allocator); + } + planes.deinit(allocator); + } + + try writer.writeAll("{\"displayTimeUnit\":\"ns\",\"metadata\":{\"highres-ticks\":true},\"traceEvents\":["); + var array_writer: JsonArrayWriter = .{}; + + for (planes.items) |*plane| { + try streamPlaneMetadata(writer, &array_writer, plane); + } + for (planes.items) |*plane| { + try streamPlaneEvents(allocator, writer, &array_writer, plane); + } + + try writer.writeAll("]}"); +} + +fn collectSelectedPlanes( + allocator: std.mem.Allocator, + xspace: *const c.tensorflow_profiler_XSpace, +) !std.ArrayListUnmanaged(PlaneContext) { + // Plane selection intentionally follows: + // ../../xla/xla/tsl/profiler/convert/xplane_to_trace_events.cc + var result: std.ArrayListUnmanaged(PlaneContext) = .empty; + errdefer { + for (result.items) |*plane| { + plane.deinit(allocator); + } + result.deinit(allocator); + } + + var plane_count: usize = 0; + const planes = c.tensorflow_profiler_XSpace_planes(xspace, &plane_count) orelse return result; + const plane_slice = planes[0..plane_count]; + + var host_plane: ?*const c.tensorflow_profiler_XPlane = null; + for (plane_slice) |plane_c| { + const plane: *const c.tensorflow_profiler_XPlane = plane_c orelse continue; + if (std.mem.eql(u8, stringView(c.tensorflow_profiler_XPlane_name(plane)), kHostThreadsPlaneName)) { + host_plane = plane; + break; + } + } + + if (host_plane) |plane| { + try result.append(allocator, PlaneContext.init(plane, kHostThreadsDeviceId, true)); + } + + if (try appendPlanesWithPrefix(allocator, &result, plane_slice, kGpuPlanePrefix)) { + return result; + } + if (try appendPlanesWithPrefix(allocator, &result, plane_slice, kTpuPlanePrefix)) { + return result; + } + _ = try appendPlanesWithPrefix(allocator, &result, plane_slice, kCustomPlanePrefix); + return result; +} + +fn appendPlanesWithPrefix( + allocator: std.mem.Allocator, + result: *std.ArrayListUnmanaged(PlaneContext), + planes: []const [*c]const c.tensorflow_profiler_XPlane, + prefix: []const u8, +) !bool { + var found = false; + for (planes) |plane_c| { + const plane: *const c.tensorflow_profiler_XPlane = plane_c orelse continue; + const name = stringView(c.tensorflow_profiler_XPlane_name(plane)); + if (!std.mem.startsWith(u8, name, prefix)) continue; + found = true; + const device_ordinal = wrapI64ToU32(c.tensorflow_profiler_XPlane_id(plane)); + try result.append(allocator, PlaneContext.init(plane, kFirstDeviceId + device_ordinal, false)); + } + return found; +} + +fn streamPlaneMetadata( + writer: *std.Io.Writer, + array_writer: *JsonArrayWriter, + plane: *const PlaneContext, +) !void { + // Metadata event shape matches the resource/process metadata emitted in: + // ../../xla/xla/tsl/profiler/convert/xplane_to_trace_events.cc + const plane_name = stringView(c.tensorflow_profiler_XPlane_name(plane.plane)); + if (plane_name.len != 0) { + try array_writer.nextItem(writer); + try writer.writeAll("{\"ph\":\"M\",\"pid\":"); + try writer.print("{d}", .{plane.device_id}); + try writer.writeAll(",\"name\":\"process_name\",\"args\":{\"name\":"); + try writeJsonString(writer, plane_name); + try writer.writeAll("}}"); + } + + try array_writer.nextItem(writer); + try writer.writeAll("{\"ph\":\"M\",\"pid\":"); + try writer.print("{d}", .{plane.device_id}); + try writer.writeAll(",\"name\":\"process_sort_index\",\"args\":{\"sort_index\":"); + try writer.print("{d}", .{plane.device_id}); + try writer.writeAll("}}"); + + var ordinal: u32 = 0; + var line_count: usize = 0; + const lines = c.tensorflow_profiler_XPlane_lines(plane.plane, &line_count) orelse return; + for (lines[0..line_count]) |line_c| { + const line: *const c.tensorflow_profiler_XLine = line_c orelse continue; + const resource_id = try lineResourceId(line); + const resource_name = lineDisplayName(line); + const sort_index = if (plane.is_host) blk: { + ordinal += 1; + break :blk ordinal; + } else resource_id; + try streamResourceMetadata(writer, array_writer, plane.device_id, resource_id, resource_name, sort_index); + } +} + +fn streamResourceMetadata( + writer: *std.Io.Writer, + array_writer: *JsonArrayWriter, + device_id: u32, + resource_id: u32, + resource_name: []const u8, + sort_index: u32, +) !void { + if (resource_name.len != 0) { + try array_writer.nextItem(writer); + try writer.writeAll("{\"ph\":\"M\",\"pid\":"); + try writer.print("{d}", .{device_id}); + try writer.writeAll(",\"tid\":"); + try writer.print("{d}", .{resource_id}); + try writer.writeAll(",\"name\":\"thread_name\",\"args\":{\"name\":"); + try writeJsonString(writer, resource_name); + try writer.writeAll("}}"); + } + + try array_writer.nextItem(writer); + try writer.writeAll("{\"ph\":\"M\",\"pid\":"); + try writer.print("{d}", .{device_id}); + try writer.writeAll(",\"tid\":"); + try writer.print("{d}", .{resource_id}); + try writer.writeAll(",\"name\":\"thread_sort_index\",\"args\":{\"sort_index\":"); + try writer.print("{d}", .{sort_index}); + try writer.writeAll("}}"); +} + +fn streamPlaneEvents( + allocator: std.mem.Allocator, + writer: *std.Io.Writer, + array_writer: *JsonArrayWriter, + plane: *PlaneContext, +) !void { + var scratch: EventScratch = .{}; + defer scratch.deinit(allocator); + + var line_count: usize = 0; + const lines = c.tensorflow_profiler_XPlane_lines(plane.plane, &line_count) orelse return; + for (lines[0..line_count]) |line_c| { + const line: *const c.tensorflow_profiler_XLine = line_c orelse continue; + if (std.mem.eql(u8, lineDisplayName(line), kXlaAsyncOpLineName)) continue; + + const resource_id = try lineResourceId(line); + var event_count: usize = 0; + const events = c.tensorflow_profiler_XLine_events(line, &event_count) orelse continue; + for (events[0..event_count]) |event_c| { + const event: *const c.tensorflow_profiler_XEvent = event_c orelse continue; + try streamEvent(allocator, writer, array_writer, plane, line, event, resource_id, &scratch); + } + } +} + +fn streamEvent( + allocator: std.mem.Allocator, + writer: *std.Io.Writer, + array_writer: *JsonArrayWriter, + plane: *PlaneContext, + line: *const c.tensorflow_profiler_XLine, + event: *const c.tensorflow_profiler_XEvent, + resource_id: u32, + scratch: *EventScratch, +) !void { + const metadata_id = c.tensorflow_profiler_XEvent_metadata_id(event); + if (try plane.isInternalEvent(allocator, metadata_id)) return; + + // Display name, long_name, stat merge order, and step_name override mirror: + // ../../xla/xla/tsl/profiler/convert/xplane_to_trace_events.cc + const metadata = try plane.getEventMetadata(allocator, metadata_id); + const metadata_name = if (metadata) |value| stringView(c.tensorflow_profiler_XEventMetadata_name(value)) else ""; + const metadata_display_name = if (metadata) |value| stringView(c.tensorflow_profiler_XEventMetadata_display_name(value)) else ""; + + var event_name: StringValue = .{ + .text = if (metadata_display_name.len != 0) metadata_display_name else metadata_name, + }; + + scratch.reset(); + const args = &scratch.args; + + if (metadata_display_name.len != 0) { + try putArg(args, allocator, "long_name", .{ .text = metadata_name }); + } + + if (metadata) |value| { + try appendStats(allocator, plane, args, &event_name, metadataStats(value)); + } + try appendStats(allocator, plane, args, &event_name, eventStats(event)); + + std.mem.sort(Arg, args.items, {}, argLessThan); + + try array_writer.nextItem(writer); + try writer.writeAll("{\"ph\":\"X\",\"pid\":"); + try writer.print("{d}", .{plane.device_id}); + try writer.writeAll(",\"tid\":"); + try writer.print("{d}", .{resource_id}); + try writer.writeAll(",\"ts\":"); + var number_buf: [64]u8 = undefined; + try writer.writeAll(formatPicosToMicros(&number_buf, eventTimestampPs(line, event))); + try writer.writeAll(",\"dur\":"); + try writer.writeAll(formatPicosToMicros(&number_buf, eventDurationPs(event))); + try writer.writeAll(",\"name\":"); + try writeJsonStringValue(writer, event_name); + + if (args.items.len != 0) { + try writer.writeAll(",\"args\":{"); + for (args.items, 0..) |arg, index| { + if (index != 0) try writer.writeByte(','); + try writeJsonString(writer, arg.name); + try writer.writeByte(':'); + try writeJsonStringValue(writer, arg.value); + } + try writer.writeByte('}'); + } + + try writer.writeByte('}'); +} + +fn appendStats( + allocator: std.mem.Allocator, + plane: *PlaneContext, + args: *std.ArrayListUnmanaged(Arg), + event_name: *StringValue, + stats: []const [*c]const c.tensorflow_profiler_XStat, +) !void { + // Stat filtering and name lookup correspond to: + // ../../xla/xla/tsl/profiler/utils/xplane_schema.cc + for (stats) |stat_c| { + const stat: *const c.tensorflow_profiler_XStat = stat_c orelse continue; + const value_case = c.tensorflow_profiler_XStat_value_case(stat); + if (value_case == c.tensorflow_profiler_XStat_value_NOT_SET) continue; + + const metadata_id = c.tensorflow_profiler_XStat_metadata_id(stat); + const behavior = try plane.statBehavior(allocator, metadata_id); + if (behavior == .skip) continue; + + const stat_value = try stringValueFromStat(allocator, plane, stat); + if (behavior == .step_name) { + event_name.* = stat_value; + } + + const stat_name = if (try plane.getStatMetadata(allocator, metadata_id)) |metadata| + stringView(c.tensorflow_profiler_XStatMetadata_name(metadata)) + else + ""; + try putArg(args, allocator, stat_name, stat_value); + } +} + +fn putArg( + args: *std.ArrayListUnmanaged(Arg), + allocator: std.mem.Allocator, + name: []const u8, + value: StringValue, +) !void { + for (args.items) |*arg| { + if (std.mem.eql(u8, arg.name, name)) { + arg.value = value; + return; + } + } + try args.append(allocator, .{ .name = name, .value = value }); +} + +fn metadataStats(metadata: *const c.tensorflow_profiler_XEventMetadata) []const [*c]const c.tensorflow_profiler_XStat { + var stat_count: usize = 0; + const stats = c.tensorflow_profiler_XEventMetadata_stats(metadata, &stat_count) orelse return &.{}; + return stats[0..stat_count]; +} + +fn eventStats(event: *const c.tensorflow_profiler_XEvent) []const [*c]const c.tensorflow_profiler_XStat { + var stat_count: usize = 0; + const stats = c.tensorflow_profiler_XEvent_stats(event, &stat_count) orelse return &.{}; + return stats[0..stat_count]; +} + +fn stringValueFromStat( + allocator: std.mem.Allocator, + plane: *PlaneContext, + stat: *const c.tensorflow_profiler_XStat, +) !StringValue { + // Stringification intentionally matches XStatVisitor::ToString(): + // ../../xla/xla/tsl/profiler/utils/xplane_visitor.cc + return switch (c.tensorflow_profiler_XStat_value_case(stat)) { + c.tensorflow_profiler_XStat_value_int64_value => .{ .int64 = c.tensorflow_profiler_XStat_int64_value(stat) }, + c.tensorflow_profiler_XStat_value_uint64_value => .{ .uint64 = c.tensorflow_profiler_XStat_uint64_value(stat) }, + c.tensorflow_profiler_XStat_value_double_value => .{ .double = c.tensorflow_profiler_XStat_double_value(stat) }, + c.tensorflow_profiler_XStat_value_str_value => .{ .text = stringView(c.tensorflow_profiler_XStat_str_value(stat)) }, + c.tensorflow_profiler_XStat_value_bytes_value => .opaque_bytes, + c.tensorflow_profiler_XStat_value_ref_value => blk: { + const ref_metadata_id = std.math.cast(i64, c.tensorflow_profiler_XStat_ref_value(stat)) orelse return error.InvalidStatMetadataId; + const text = if (try plane.getStatMetadata(allocator, ref_metadata_id)) |metadata| + stringView(c.tensorflow_profiler_XStatMetadata_name(metadata)) + else + ""; + break :blk .{ .text = text }; + }, + else => unreachable, + }; +} + +fn writeJsonStringValue(writer: *std.Io.Writer, value: StringValue) !void { + switch (value) { + .text => |text| try writeJsonString(writer, text), + .opaque_bytes => try writeJsonString(writer, kOpaqueBytesText), + .int64 => |number| { + var buffer: [64]u8 = undefined; + const text = try std.fmt.bufPrint(&buffer, "{d}", .{number}); + try writeJsonString(writer, text); + }, + .uint64 => |number| { + var buffer: [64]u8 = undefined; + const text = try std.fmt.bufPrint(&buffer, "{d}", .{number}); + try writeJsonString(writer, text); + }, + .double => |number| { + var buffer: [64]u8 = undefined; + try writeJsonString(writer, formatDouble(&buffer, "%g", number)); + }, + } +} + +fn writeJsonString(writer: *std.Io.Writer, text: []const u8) !void { + try writer.writeByte('"'); + + var chunk_start: usize = 0; + for (text, 0..) |byte, index| { + const escape = switch (byte) { + '"' => "\\\"", + '\\' => "\\\\", + '\n' => "\\n", + '\r' => "\\r", + '\t' => "\\t", + 0x08 => "\\b", + 0x0c => "\\f", + else => null, + }; + if (escape) |escaped| { + if (chunk_start < index) { + try writer.writeAll(text[chunk_start..index]); + } + try writer.writeAll(escaped); + chunk_start = index + 1; + continue; + } + if (byte < 0x20) { + if (chunk_start < index) { + try writer.writeAll(text[chunk_start..index]); + } + try writeAsciiControlEscape(writer, byte); + chunk_start = index + 1; + } + } + + if (chunk_start < text.len) { + try writer.writeAll(text[chunk_start..]); + } + + try writer.writeByte('"'); +} + +fn lineResourceId(line: *const c.tensorflow_profiler_XLine) !u32 { + const display_id = c.tensorflow_profiler_XLine_display_id(line); + const effective_id = if (display_id != 0) display_id else c.tensorflow_profiler_XLine_id(line); + return wrapI64ToU32(effective_id); +} + +fn lineDisplayName(line: *const c.tensorflow_profiler_XLine) []const u8 { + const display_name = stringView(c.tensorflow_profiler_XLine_display_name(line)); + if (display_name.len != 0) return display_name; + return stringView(c.tensorflow_profiler_XLine_name(line)); +} + +fn eventTimestampPs(line: *const c.tensorflow_profiler_XLine, event: *const c.tensorflow_profiler_XEvent) u64 { + const line_timestamp_ns = reinterpretSignedAsUnsigned64(c.tensorflow_profiler_XLine_timestamp_ns(line)); + const event_offset_ps = reinterpretSignedAsUnsigned64(c.tensorflow_profiler_XEvent_offset_ps(event)); + return line_timestamp_ns *% kPicosecondsPerNanosecond +% event_offset_ps; +} + +fn eventDurationPs(event: *const c.tensorflow_profiler_XEvent) u64 { + const duration_ps = reinterpretSignedAsUnsigned64(c.tensorflow_profiler_XEvent_duration_ps(event)); + return @max(duration_ps, 1); +} + +fn formatPicosToMicros(buffer: *[64]u8, picoseconds: u64) []const u8 { + // The conversion and "%.17g" formatting are kept identical to: + // ../../xla/xla/tsl/profiler/utils/math_utils.h + // ../../xla/xla/tsl/profiler/utils/format_utils.h + const microseconds = @as(f64, @floatFromInt(picoseconds)) / kPicosecondsPerMicrosecond; + return formatDouble(buffer, "%.17g", microseconds); +} + +fn formatDouble(buffer: *[64]u8, comptime fmt: [*:0]const u8, value: f64) []const u8 { + const written = c.snprintf(buffer.ptr, buffer.len, fmt, value); + if (written <= 0 or written >= buffer.len) unreachable; + return buffer[0..@intCast(written)]; +} + +fn stringView(value: c.upb_StringView) []const u8 { + const text = upb.slice(value); + return text orelse ""; +} + +fn reinterpretSignedAsUnsigned64(value: i64) u64 { + const same_bits: u64 = @bitCast(value); + return same_bits; +} + +fn wrapI64ToU32(value: i64) u32 { + const unsigned_value = reinterpretSignedAsUnsigned64(value); + return @truncate(unsigned_value); +} + +fn writeAsciiControlEscape(writer: *std.Io.Writer, byte: u8) !void { + const high_nibble = @as(usize, byte / 16); + const low_nibble = @as(usize, byte % 16); + + try writer.writeAll("\\u00"); + try writer.writeByte(kHexDigits[high_nibble]); + try writer.writeByte(kHexDigits[low_nibble]); +} + +fn argLessThan(_: void, lhs: Arg, rhs: Arg) bool { + return std.mem.order(u8, lhs.name, rhs.name) == .lt; +} + +fn isInternalEventName(name: []const u8) bool { + // Internal host-event filtering is derived from: + // ../../xla/xla/tsl/profiler/utils/xplane_schema.cc + inline for ([_][]const u8{ + "MemoryAllocation", + "MemoryDeallocation", + "PrefetchProduce", + "PrefetchConsume", + "ParallelInterleaveProduce", + "ParallelInterleaveConsume", + "ParallelInterleaveInitializeInput", + "ParallelMapProduce", + "ParallelMapConsume", + "MapAndBatchProduce", + "MapAndBatchConsume", + "ParseExampleProduce", + "ParseExampleConsume", + }) |candidate| { + if (std.mem.eql(u8, name, candidate)) return true; + } + return false; +} + +fn classifyStatName(name: []const u8) StatBehavior { + // Internal stat filtering and step_name handling are derived from: + // ../../xla/xla/tsl/profiler/utils/xplane_schema.cc + if (std.mem.eql(u8, name, "step_name")) return .step_name; + inline for ([_][]const u8{ + "_pt", + "_ct", + "_p", + "_c", + "_r", + "flops", + "program_id", + "symbol_id", + }) |candidate| { + if (std.mem.eql(u8, name, candidate)) return .skip; + } + return .emit; +} diff --git a/tools/zig.sh b/tools/zig.sh deleted file mode 100755 index a9b746cd..00000000 --- a/tools/zig.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -cd "$(dirname "${BASH_SOURCE[0]}")" -exec bazel run --config=silent @zml//third_party/zls:zls -- zig "${@}" diff --git a/tools/zls.sh b/tools/zls.sh index 9fefc368..2b83f7b6 100755 --- a/tools/zls.sh +++ b/tools/zls.sh @@ -1,3 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash cd "$(dirname "${BASH_SOURCE[0]}")" -exec ../examples/bazel.sh run --config=silent @zml//third_party/zls:zls -- zls "${@}" +cd "$(bazel info workspace)" +exec bazel run -- @zml//:completion "${@}" diff --git a/tools/zml_utils.py b/tools/zml_utils.py index a0053e77..9bddc023 100644 --- a/tools/zml_utils.py +++ b/tools/zml_utils.py @@ -21,16 +21,6 @@ import torch log = logging.getLogger(__name__) -builtin_open = builtins.open - - -def log_and_open(file, *args, **kwargs): - print("open:", file, args, kwargs) - return builtin_open(file, *args, **kwargs) - - -builtins.open = log_and_open - class ActivationCollector: """Wrap a given torch.nn.Module and collect all its intermediary activations. diff --git a/upb/BUILD.bazel b/upb/BUILD.bazel new file mode 100644 index 00000000..9189fc8f --- /dev/null +++ b/upb/BUILD.bazel @@ -0,0 +1,19 @@ +load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library") + +cc_library( + name = "upb_c", + tags = ["manual"], + hdrs = ["upb.h"], + deps = [ + "@com_google_protobuf//upb/message:copy", + ], +) + +zig_library( + name = "upb", + tags = ["manual"], + main = "upb.zig", + visibility = ["//visibility:public"], + deps = [":upb_c"], +) diff --git a/upb/upb.h b/upb/upb.h new file mode 100644 index 00000000..dbec3577 --- /dev/null +++ b/upb/upb.h @@ -0,0 +1 @@ +#include "upb/message/copy.h" diff --git a/upb/upb.zig b/upb/upb.zig new file mode 100644 index 00000000..f07d003a --- /dev/null +++ b/upb/upb.zig @@ -0,0 +1,156 @@ +const std = @import("std"); + +const c = @import("c"); + +export fn _upb_MiniTable_StrongReference_dont_copy_me__upb_internal_use_only(mt: *c.upb_MiniTable) callconv(.c) void { + const unused: *volatile c.upb_MiniTable = mt; + _ = &unused; +} + +pub const SerializeOptions = packed struct(c_int) { + deterministic: bool = false, + skip_unknown: bool = false, + check_required: bool = false, + _ignored: u29 = 0, +}; + +pub const SerializeError = error{ + MaxDepthExceeded, + MissingRequired, + Unknown, +} || std.mem.Allocator.Error; + +pub const ParseOptions = packed struct { + alias_string: bool = false, + check_required: bool = false, + experimental_allow_unlinked: bool = false, + always_validate_utf8: bool = false, + disable_fast_table: bool = false, + _ignored: u27 = 0, +}; + +pub const ParseError = error{ + Malformed, + BadUtf8, + MaxDepthExceeded, + MissingRequired, + Unknown, +} || std.mem.Allocator.Error; + +pub fn deepClone(comptime UpbType: type, arena: *c.upb_Arena, msg: *const UpbType) std.mem.Allocator.Error!*UpbType { + return @ptrCast(c.upb_Message_DeepClone(@ptrCast(msg), Minitable(UpbType), arena) orelse return std.mem.Allocator.Error.OutOfMemory); +} + +pub fn shallowClone(comptime UpbType: type, arena: *c.upb_Arena, msg: *const UpbType) std.mem.Allocator.Error!*UpbType { + return @ptrCast(c.upb_Message_ShallowClone(@ptrCast(msg), Minitable(UpbType), arena) orelse return std.mem.Allocator.Error.OutOfMemory); +} + +pub fn stringView(data: ?[]const u8) c.upb_StringView { + return if (data) |d| c.upb_StringView_FromDataAndSize(d.ptr, d.len) else .{ .data = null, .size = 0 }; +} + +pub fn slice(sv: c.upb_StringView) ?[]const u8 { + return if (sv.data) |d| d[0..sv.size] else null; +} + +fn ProtoName(comptime UpbType: type) []const u8 { + const needle = ".struct_"; + const type_name = @typeName(UpbType); + const idx = std.mem.indexOf(u8, type_name, needle) orelse @compileError("Type name is invalid"); + return type_name[idx + needle.len ..]; +} + +fn Minitable(comptime UpbType: type) *const c.upb_MiniTable { + const field_name = comptime blk: { + const name = ProtoName(UpbType); + var it = std.mem.tokenizeScalar(u8, name, '_'); + while (it.next()) |_| { + const new_name = name[0..it.index] ++ "_" ++ name[it.index..] ++ "_msg_init"; + if (@hasDecl(c, new_name)) { + break :blk new_name; + } + } else { + @compileError("Unable to find minitable for type:" ++ @typeName(UpbType)); + } + }; + return &@field(c, field_name); +} + +pub fn serializeEx(ptr: anytype, arena: *c.upb_Arena, opts: SerializeOptions) SerializeError![]const u8 { + var buf: [*c]u8 = undefined; + var size: usize = undefined; + return switch (c.upb_Encode(@ptrCast(ptr), Minitable(@TypeOf(ptr.*)), @bitCast(opts), arena, &buf, &size)) { + c.kUpb_EncodeStatus_Ok => buf[0..size], + c.kUpb_EncodeStatus_OutOfMemory => std.mem.Allocator.Error.OutOfMemory, + c.kUpb_EncodeStatus_MaxDepthExceeded => SerializeError.MaxDepthExceeded, + c.kUpb_EncodeStatus_MissingRequired => SerializeError.MissingRequired, + else => return SerializeError.Unknown, + }; +} + +pub fn serialize(ptr: anytype, arena: *c.upb_Arena) SerializeError![]const u8 { + return try serializeEx(ptr, arena, .{}); +} + +pub fn parseEx(comptime UpbType: type, arena: *c.upb_Arena, data: []const u8, opts: ParseOptions) ParseError!*UpbType { + const obj = try new(UpbType, arena); + return switch (c.upb_Decode(@ptrCast(@constCast(data)), data.len, @ptrCast(@alignCast(obj)), Minitable(UpbType), null, @bitCast(opts), arena)) { + c.kUpb_DecodeStatus_Ok => obj, + c.kUpb_DecodeStatus_Malformed => ParseError.Malformed, + c.kUpb_DecodeStatus_OutOfMemory => std.mem.Allocator.Error.OutOfMemory, + c.kUpb_DecodeStatus_BadUtf8 => ParseError.BadUtf8, + c.kUpb_DecodeStatus_MaxDepthExceeded => ParseError.MaxDepthExceeded, + c.kUpb_DecodeStatus_MissingRequired => ParseError.MissingRequired, + else => ParseError.Unknown, + }; +} + +pub fn parse(comptime UpbType: type, arena: *c.upb_Arena, data: []const u8) ParseError!*UpbType { + return parseEx(UpbType, arena, data, .{}); +} + +pub fn new(comptime UpbType: type, arena: *c.upb_Arena) std.mem.Allocator.Error!*UpbType { + const new_fn = @field(c, ProtoName(UpbType) ++ "_new"); + return @ptrCast(new_fn(arena) orelse return std.mem.Allocator.Error.OutOfMemory); +} + +pub const Allocator = struct { + upb_alloc: c.upb_alloc, + allocator: std.mem.Allocator, + + pub fn init(allocator: std.mem.Allocator) Allocator { + return .{ + .upb_alloc = .{ + .func = &alloc_impl, + }, + .allocator = allocator, + }; + } + + pub fn inner(self: *Allocator) *c.upb_alloc { + return &self.upb_alloc; + } + + fn alloc_impl(alloc: [*c]c.upb_alloc, ptr: ?*anyopaque, oldsize: usize, size: usize, actual_size: [*c]usize) callconv(.c) ?*anyopaque { + const PointerAlignedSlice = [*c]align(@alignOf(*anyopaque)) u8; + const upb_alloc: *c.upb_alloc = alloc orelse return null; + const self: *Allocator = @fieldParentPtr("upb_alloc", upb_alloc); + defer { + if (actual_size) |as| { + as.* = size; + } + } + if (ptr) |ptr_| { + const ptr_as_slice: []u8 = @as(PointerAlignedSlice, @ptrCast(@alignCast(ptr_)))[0..oldsize]; + if (size == 0) { + self.allocator.free(ptr_as_slice); + return null; + } else if (size != oldsize) { + return (self.allocator.realloc(ptr_as_slice, size) catch return null).ptr; + } + @panic("Unsupported case"); + } + + return @ptrCast(self.allocator.alignedAlloc(u8, std.mem.Alignment.of(*anyopaque), size) catch return null); + } +}; diff --git a/vfs/BUILD.bazel b/vfs/BUILD.bazel new file mode 100644 index 00000000..c8de1e0c --- /dev/null +++ b/vfs/BUILD.bazel @@ -0,0 +1,25 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "vfs", + srcs = [ + "base.zig", + "file.zig", + "gcs.zig", + "hf.zig", + "http.zig", + "parallel_read.zig", + "s3.zig", + ], + main = "vfs.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//stdx", + ], +) + +zig_test( + name = "test", + deps = [":vfs"], +) diff --git a/vfs/base.zig b/vfs/base.zig new file mode 100644 index 00000000..1e66ca89 --- /dev/null +++ b/vfs/base.zig @@ -0,0 +1,660 @@ +const std = @import("std"); + +const stdx = @import("stdx"); + +pub const VFSBase = struct { + inner: std.Io, + + pub fn init(io: std.Io) VFSBase { + return .{ .inner = io }; + } + + pub fn vtable(overrides: anytype) std.Io.VTable { + var new_vtable: std.Io.VTable = .{ + .crashHandler = crashHandler, + .async = async, + .concurrent = concurrent, + .await = await, + .cancel = cancel, + .groupAsync = groupAsync, + .groupConcurrent = groupConcurrent, + .groupAwait = groupAwait, + .groupCancel = groupCancel, + .recancel = recancel, + .swapCancelProtection = swapCancelProtection, + .checkCancel = checkCancel, + .futexWait = futexWait, + .futexWaitUncancelable = futexWaitUncancelable, + .futexWake = futexWake, + .operate = operate, + .batchAwaitAsync = batchAwaitAsync, + .batchAwaitConcurrent = batchAwaitConcurrent, + .batchCancel = batchCancel, + .dirCreateDir = dirCreateDir, + .dirCreateDirPath = dirCreateDirPath, + .dirCreateDirPathOpen = dirCreateDirPathOpen, + .dirOpenDir = dirOpenDir, + .dirStat = dirStat, + .dirStatFile = dirStatFile, + .dirAccess = dirAccess, + .dirCreateFile = dirCreateFile, + .dirCreateFileAtomic = dirCreateFileAtomic, + .dirOpenFile = dirOpenFile, + .dirClose = dirClose, + .dirRead = dirRead, + .dirRealPath = dirRealPath, + .dirRealPathFile = dirRealPathFile, + .dirDeleteFile = dirDeleteFile, + .dirDeleteDir = dirDeleteDir, + .dirRename = dirRename, + .dirRenamePreserve = dirRenamePreserve, + .dirSymLink = dirSymLink, + .dirReadLink = dirReadLink, + .dirSetOwner = dirSetOwner, + .dirSetFileOwner = dirSetFileOwner, + .dirSetPermissions = dirSetPermissions, + .dirSetFilePermissions = dirSetFilePermissions, + .dirSetTimestamps = dirSetTimestamps, + .dirHardLink = dirHardLink, + .fileStat = fileStat, + .fileLength = fileLength, + .fileClose = fileClose, + .fileWritePositional = fileWritePositional, + .fileWriteFileStreaming = fileWriteFileStreaming, + .fileWriteFilePositional = fileWriteFilePositional, + .fileReadPositional = fileReadPositional, + .fileSeekBy = fileSeekBy, + .fileSeekTo = fileSeekTo, + .fileSync = fileSync, + .fileIsTty = fileIsTty, + .fileEnableAnsiEscapeCodes = fileEnableAnsiEscapeCodes, + .fileSupportsAnsiEscapeCodes = fileSupportsAnsiEscapeCodes, + .fileSetLength = fileSetLength, + .fileSetOwner = fileSetOwner, + .fileSetPermissions = fileSetPermissions, + .fileSetTimestamps = fileSetTimestamps, + .fileLock = fileLock, + .fileTryLock = fileTryLock, + .fileUnlock = fileUnlock, + .fileDowngradeLock = fileDowngradeLock, + .fileRealPath = fileRealPath, + .fileHardLink = fileHardLink, + .fileMemoryMapCreate = fileMemoryMapCreate, + .fileMemoryMapDestroy = fileMemoryMapDestroy, + .fileMemoryMapSetLength = fileMemoryMapSetLength, + .fileMemoryMapRead = fileMemoryMapRead, + .fileMemoryMapWrite = fileMemoryMapWrite, + .processExecutableOpen = processExecutableOpen, + .processExecutablePath = processExecutablePath, + .lockStderr = lockStderr, + .tryLockStderr = tryLockStderr, + .unlockStderr = unlockStderr, + .processCurrentPath = processCurrentPath, + .processSetCurrentDir = processSetCurrentDir, + .processSetCurrentPath = processSetCurrentPath, + .processReplace = processReplace, + .processSpawn = processSpawn, + .childWait = childWait, + .childKill = childKill, + .progressParentFile = progressParentFile, + .inheritParentDir = inheritParentDir, + .inheritParentFile = inheritParentFile, + .now = now, + .clockResolution = clockResolution, + .sleep = sleep, + .random = random, + .randomSecure = randomSecure, + .netListenIp = netListenIp, + .netAccept = netAccept, + .netBindIp = netBindIp, + .netConnectIp = netConnectIp, + .netListenUnix = netListenUnix, + .netConnectUnix = netConnectUnix, + .netSocketCreatePair = netSocketCreatePair, + .netWriteFile = netWriteFile, + .netClose = netClose, + .netShutdown = netShutdown, + .netInterfaceNameResolve = netInterfaceNameResolve, + .netInterfaceName = netInterfaceName, + .netLookup = netLookup, + }; + for (std.meta.fieldNames(@TypeOf(overrides))) |field_name| { + @field(new_vtable, field_name) = @field(overrides, field_name); + } + return new_vtable; + } + + pub fn as(userdata: ?*anyopaque) *VFSBase { + return @ptrCast(@alignCast(userdata.?)); + } + + pub fn crashHandler(userdata: ?*anyopaque) void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + self.inner.vtable.crashHandler(self.inner.userdata); + } + + pub fn async(userdata: ?*anyopaque, result: []u8, result_alignment: std.mem.Alignment, context: []const u8, context_alignment: std.mem.Alignment, start: *const fn (context: *const anyopaque, result: *anyopaque) void) ?*std.Io.AnyFuture { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.async(self.inner.userdata, result, result_alignment, context, context_alignment, start); + } + + pub fn concurrent(userdata: ?*anyopaque, result_len: usize, result_alignment: std.mem.Alignment, context: []const u8, context_alignment: std.mem.Alignment, start: *const fn (context: *const anyopaque, result: *anyopaque) void) std.Io.ConcurrentError!*std.Io.AnyFuture { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.concurrent(self.inner.userdata, result_len, result_alignment, context, context_alignment, start); + } + + pub fn await(userdata: ?*anyopaque, any_future: *std.Io.AnyFuture, result: []u8, result_alignment: std.mem.Alignment) void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.await(self.inner.userdata, any_future, result, result_alignment); + } + + pub fn cancel(userdata: ?*anyopaque, any_future: *std.Io.AnyFuture, result: []u8, result_alignment: std.mem.Alignment) void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.cancel(self.inner.userdata, any_future, result, result_alignment); + } + + pub fn groupAsync(userdata: ?*anyopaque, group: *std.Io.Group, context: []const u8, context_alignment: std.mem.Alignment, start: *const fn (context: *const anyopaque) void) void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.groupAsync(self.inner.userdata, group, context, context_alignment, start); + } + + pub fn groupConcurrent(userdata: ?*anyopaque, group: *std.Io.Group, context: []const u8, context_alignment: std.mem.Alignment, start: *const fn (context: *const anyopaque) void) std.Io.ConcurrentError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.groupConcurrent(self.inner.userdata, group, context, context_alignment, start); + } + + pub fn groupAwait(userdata: ?*anyopaque, group: *std.Io.Group, token: *anyopaque) std.Io.Cancelable!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.groupAwait(self.inner.userdata, group, token); + } + + pub fn groupCancel(userdata: ?*anyopaque, group: *std.Io.Group, token: *anyopaque) void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.groupCancel(self.inner.userdata, group, token); + } + + pub fn recancel(userdata: ?*anyopaque) void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.recancel(self.inner.userdata); + } + + pub fn swapCancelProtection(userdata: ?*anyopaque, new: std.Io.CancelProtection) std.Io.CancelProtection { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.swapCancelProtection(self.inner.userdata, new); + } + + pub fn checkCancel(userdata: ?*anyopaque) std.Io.Cancelable!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.checkCancel(self.inner.userdata); + } + + pub fn futexWait(userdata: ?*anyopaque, ptr: *const u32, expected: u32, timeout: std.Io.Timeout) std.Io.Cancelable!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.futexWait(self.inner.userdata, ptr, expected, timeout); + } + + pub fn futexWaitUncancelable(userdata: ?*anyopaque, ptr: *const u32, expected: u32) void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.futexWaitUncancelable(self.inner.userdata, ptr, expected); + } + + pub fn futexWake(userdata: ?*anyopaque, ptr: *const u32, max_waiters: u32) void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.futexWake(self.inner.userdata, ptr, max_waiters); + } + + pub fn operate(userdata: ?*anyopaque, operation: std.Io.Operation) std.Io.Cancelable!std.Io.Operation.Result { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.operate(self.inner.userdata, operation); + } + + pub fn batchAwaitAsync(userdata: ?*anyopaque, batch: *std.Io.Batch) std.Io.Cancelable!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.batchAwaitAsync(self.inner.userdata, batch); + } + + pub fn batchAwaitConcurrent(userdata: ?*anyopaque, batch: *std.Io.Batch, timeout: std.Io.Timeout) std.Io.Batch.AwaitConcurrentError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.batchAwaitConcurrent(self.inner.userdata, batch, timeout); + } + + pub fn batchCancel(userdata: ?*anyopaque, batch: *std.Io.Batch) void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + self.inner.vtable.batchCancel(self.inner.userdata, batch); + } + + pub fn dirCreateDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, permissions: std.Io.Dir.Permissions) std.Io.Dir.CreateDirError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.dirCreateDir(self.inner.userdata, dir, sub_path, permissions); + } + + pub fn dirCreateDirPath(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, permissions: std.Io.Dir.Permissions) std.Io.Dir.CreateDirPathError!std.Io.Dir.CreatePathStatus { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.dirCreateDirPath(self.inner.userdata, dir, sub_path, permissions); + } + + pub fn dirCreateDirPathOpen(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, permissions: std.Io.Dir.Permissions, options: std.Io.Dir.OpenOptions) std.Io.Dir.CreateDirPathOpenError!std.Io.Dir { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.dirCreateDirPathOpen(self.inner.userdata, dir, sub_path, permissions, options); + } + + pub fn dirOpenDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.OpenOptions) std.Io.Dir.OpenError!std.Io.Dir { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.dirOpenDir(self.inner.userdata, dir, sub_path, options); + } + + pub fn dirStat(userdata: ?*anyopaque, dir: std.Io.Dir) std.Io.Dir.StatError!std.Io.Dir.Stat { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.dirStat(self.inner.userdata, dir); + } + + pub fn dirStatFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.StatFileOptions) std.Io.Dir.StatFileError!std.Io.File.Stat { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.dirStatFile(self.inner.userdata, dir, sub_path, options); + } + + pub fn dirAccess(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.dirAccess(self.inner.userdata, dir, sub_path, options); + } + + pub fn dirCreateFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.Dir.CreateFileOptions) std.Io.File.OpenError!std.Io.File { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.dirCreateFile(self.inner.userdata, dir, sub_path, flags); + } + + pub fn dirCreateFileAtomic(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.CreateFileAtomicOptions) std.Io.Dir.CreateFileAtomicError!std.Io.File.Atomic { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.dirCreateFileAtomic(self.inner.userdata, dir, sub_path, options); + } + + pub fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!std.Io.File { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.dirOpenFile(self.inner.userdata, dir, sub_path, flags); + } + + pub fn dirClose(userdata: ?*anyopaque, dirs: []const std.Io.Dir) void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + self.inner.vtable.dirClose(self.inner.userdata, dirs); + } + + pub fn dirRead(userdata: ?*anyopaque, reader: *std.Io.Dir.Reader, entries: []std.Io.Dir.Entry) std.Io.Dir.Reader.Error!usize { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.dirRead(self.inner.userdata, reader, entries); + } + + pub fn dirRealPath(userdata: ?*anyopaque, dir: std.Io.Dir, out_buffer: []u8) std.Io.Dir.RealPathError!usize { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.dirRealPath(self.inner.userdata, dir, out_buffer); + } + + pub fn dirRealPathFile(userdata: ?*anyopaque, dir: std.Io.Dir, path_name: []const u8, out_buffer: []u8) std.Io.Dir.RealPathFileError!usize { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.dirRealPathFile(self.inner.userdata, dir, path_name, out_buffer); + } + + pub fn dirDeleteFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8) std.Io.Dir.DeleteFileError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.dirDeleteFile(self.inner.userdata, dir, sub_path); + } + + pub fn dirDeleteDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8) std.Io.Dir.DeleteDirError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.dirDeleteDir(self.inner.userdata, dir, sub_path); + } + + pub fn dirRename(userdata: ?*anyopaque, old_dir: std.Io.Dir, old_sub_path: []const u8, new_dir: std.Io.Dir, new_sub_path: []const u8) std.Io.Dir.RenameError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.dirRename(self.inner.userdata, old_dir, old_sub_path, new_dir, new_sub_path); + } + + pub fn dirRenamePreserve(userdata: ?*anyopaque, old_dir: std.Io.Dir, old_sub_path: []const u8, new_dir: std.Io.Dir, new_sub_path: []const u8) std.Io.Dir.RenamePreserveError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.dirRenamePreserve(self.inner.userdata, old_dir, old_sub_path, new_dir, new_sub_path); + } + + pub fn dirSymLink(userdata: ?*anyopaque, dir: std.Io.Dir, target_path: []const u8, sym_link_path: []const u8, flags: std.Io.Dir.SymLinkFlags) std.Io.Dir.SymLinkError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.dirSymLink(self.inner.userdata, dir, target_path, sym_link_path, flags); + } + + pub fn dirReadLink(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, buffer: []u8) std.Io.Dir.ReadLinkError!usize { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.dirReadLink(self.inner.userdata, dir, sub_path, buffer); + } + + pub fn dirSetOwner(userdata: ?*anyopaque, dir: std.Io.Dir, uid: ?std.Io.File.Uid, gid: ?std.Io.File.Gid) std.Io.Dir.SetOwnerError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.dirSetOwner(self.inner.userdata, dir, uid, gid); + } + + pub fn dirSetFileOwner(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, uid: ?std.Io.File.Uid, gid: ?std.Io.File.Gid, options: std.Io.Dir.SetFileOwnerOptions) std.Io.Dir.SetFileOwnerError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.dirSetFileOwner(self.inner.userdata, dir, sub_path, uid, gid, options); + } + + pub fn dirSetPermissions(userdata: ?*anyopaque, dir: std.Io.Dir, permissions: std.Io.Dir.Permissions) std.Io.Dir.SetPermissionsError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.dirSetPermissions(self.inner.userdata, dir, permissions); + } + + pub fn dirSetFilePermissions(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, permissions: std.Io.File.Permissions, options: std.Io.Dir.SetFilePermissionsOptions) std.Io.Dir.SetFilePermissionsError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.dirSetFilePermissions(self.inner.userdata, dir, sub_path, permissions, options); + } + + pub fn dirSetTimestamps(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.SetTimestampsOptions) std.Io.Dir.SetTimestampsError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.dirSetTimestamps(self.inner.userdata, dir, sub_path, options); + } + + pub fn dirHardLink(userdata: ?*anyopaque, old_dir: std.Io.Dir, old_sub_path: []const u8, new_dir: std.Io.Dir, new_sub_path: []const u8, options: std.Io.Dir.HardLinkOptions) std.Io.Dir.HardLinkError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.dirHardLink(self.inner.userdata, old_dir, old_sub_path, new_dir, new_sub_path, options); + } + + pub fn fileStat(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.StatError!std.Io.File.Stat { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.fileStat(self.inner.userdata, file); + } + + pub fn fileLength(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.LengthError!u64 { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.fileLength(self.inner.userdata, file); + } + + pub fn fileClose(userdata: ?*anyopaque, files: []const std.Io.File) void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + self.inner.vtable.fileClose(self.inner.userdata, files); + } + + pub fn fileWritePositional(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, data: []const []const u8, splat: usize, offset: u64) std.Io.File.WritePositionalError!usize { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.fileWritePositional(self.inner.userdata, file, header, data, splat, offset); + } + + pub fn fileWriteFileStreaming(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, reader: *std.Io.File.Reader, limit: std.Io.Limit) std.Io.File.Writer.WriteFileError!usize { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.fileWriteFileStreaming(self.inner.userdata, file, header, reader, limit); + } + + pub fn fileWriteFilePositional(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, reader: *std.Io.File.Reader, limit: std.Io.Limit, offset: u64) std.Io.File.WriteFilePositionalError!usize { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.fileWriteFilePositional(self.inner.userdata, file, header, reader, limit, offset); + } + + pub fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.fileReadPositional(self.inner.userdata, file, data, offset); + } + + pub fn fileSeekBy(userdata: ?*anyopaque, file: std.Io.File, relative_offset: i64) std.Io.File.SeekError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.fileSeekBy(self.inner.userdata, file, relative_offset); + } + + pub fn fileSeekTo(userdata: ?*anyopaque, file: std.Io.File, absolute_offset: u64) std.Io.File.SeekError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.fileSeekTo(self.inner.userdata, file, absolute_offset); + } + + pub fn fileSync(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.SyncError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.fileSync(self.inner.userdata, file); + } + + pub fn fileIsTty(userdata: ?*anyopaque, file: std.Io.File) std.Io.Cancelable!bool { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.fileIsTty(self.inner.userdata, file); + } + + pub fn fileEnableAnsiEscapeCodes(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.EnableAnsiEscapeCodesError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.fileEnableAnsiEscapeCodes(self.inner.userdata, file); + } + + pub fn fileSupportsAnsiEscapeCodes(userdata: ?*anyopaque, file: std.Io.File) std.Io.Cancelable!bool { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.fileSupportsAnsiEscapeCodes(self.inner.userdata, file); + } + + pub fn fileSetLength(userdata: ?*anyopaque, file: std.Io.File, length: u64) std.Io.File.SetLengthError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.fileSetLength(self.inner.userdata, file, length); + } + + pub fn fileSetOwner(userdata: ?*anyopaque, file: std.Io.File, uid: ?std.Io.File.Uid, gid: ?std.Io.File.Gid) std.Io.File.SetOwnerError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.fileSetOwner(self.inner.userdata, file, uid, gid); + } + + pub fn fileSetPermissions(userdata: ?*anyopaque, file: std.Io.File, permissions: std.Io.File.Permissions) std.Io.File.SetPermissionsError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.fileSetPermissions(self.inner.userdata, file, permissions); + } + + pub fn fileSetTimestamps(userdata: ?*anyopaque, file: std.Io.File, options: std.Io.File.SetTimestampsOptions) std.Io.File.SetTimestampsError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.fileSetTimestamps(self.inner.userdata, file, options); + } + + pub fn fileLock(userdata: ?*anyopaque, file: std.Io.File, lock: std.Io.File.Lock) std.Io.File.LockError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.fileLock(self.inner.userdata, file, lock); + } + + pub fn fileTryLock(userdata: ?*anyopaque, file: std.Io.File, lock: std.Io.File.Lock) std.Io.File.LockError!bool { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.fileTryLock(self.inner.userdata, file, lock); + } + + pub fn fileUnlock(userdata: ?*anyopaque, file: std.Io.File) void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.fileUnlock(self.inner.userdata, file); + } + + pub fn fileDowngradeLock(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.DowngradeLockError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.fileDowngradeLock(self.inner.userdata, file); + } + + pub fn fileRealPath(userdata: ?*anyopaque, file: std.Io.File, out_buffer: []u8) std.Io.File.RealPathError!usize { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.fileRealPath(self.inner.userdata, file, out_buffer); + } + + pub fn fileHardLink(userdata: ?*anyopaque, file: std.Io.File, dir: std.Io.Dir, path: []const u8, options: std.Io.File.HardLinkOptions) std.Io.File.HardLinkError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.fileHardLink(self.inner.userdata, file, dir, path, options); + } + + pub fn fileMemoryMapCreate(userdata: ?*anyopaque, file: std.Io.File, options: std.Io.File.MemoryMap.CreateOptions) std.Io.File.MemoryMap.CreateError!std.Io.File.MemoryMap { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.fileMemoryMapCreate(self.inner.userdata, file, options); + } + + pub fn fileMemoryMapDestroy(userdata: ?*anyopaque, memory_map: *std.Io.File.MemoryMap) void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + self.inner.vtable.fileMemoryMapDestroy(self.inner.userdata, memory_map); + } + + pub fn fileMemoryMapSetLength(userdata: ?*anyopaque, memory_map: *std.Io.File.MemoryMap, length: usize) std.Io.File.MemoryMap.SetLengthError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.fileMemoryMapSetLength(self.inner.userdata, memory_map, length); + } + + pub fn fileMemoryMapRead(userdata: ?*anyopaque, memory_map: *std.Io.File.MemoryMap) std.Io.File.ReadPositionalError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.fileMemoryMapRead(self.inner.userdata, memory_map); + } + + pub fn fileMemoryMapWrite(userdata: ?*anyopaque, memory_map: *std.Io.File.MemoryMap) std.Io.File.WritePositionalError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.fileMemoryMapWrite(self.inner.userdata, memory_map); + } + + pub fn processExecutableOpen(userdata: ?*anyopaque, flags: std.Io.Dir.OpenFileOptions) std.process.OpenExecutableError!std.Io.File { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.processExecutableOpen(self.inner.userdata, flags); + } + + pub fn processExecutablePath(userdata: ?*anyopaque, buffer: []u8) std.process.ExecutablePathError!usize { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.processExecutablePath(self.inner.userdata, buffer); + } + + pub fn lockStderr(userdata: ?*anyopaque, mode: ?std.Io.Terminal.Mode) std.Io.Cancelable!std.Io.LockedStderr { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.lockStderr(self.inner.userdata, mode); + } + + pub fn tryLockStderr(userdata: ?*anyopaque, mode: ?std.Io.Terminal.Mode) std.Io.Cancelable!?std.Io.LockedStderr { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.tryLockStderr(self.inner.userdata, mode); + } + + pub fn unlockStderr(userdata: ?*anyopaque) void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.unlockStderr(self.inner.userdata); + } + + pub fn processCurrentPath(userdata: ?*anyopaque, buffer: []u8) std.process.CurrentPathError!usize { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.processCurrentPath(self.inner.userdata, buffer); + } + + pub fn processSetCurrentDir(userdata: ?*anyopaque, dir: std.Io.Dir) std.process.SetCurrentDirError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.processSetCurrentDir(self.inner.userdata, dir); + } + + pub fn processSetCurrentPath(userdata: ?*anyopaque, path: []const u8) std.process.SetCurrentPathError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.processSetCurrentPath(self.inner.userdata, path); + } + + pub fn processReplace(userdata: ?*anyopaque, options: std.process.ReplaceOptions) std.process.ReplaceError { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.processReplace(self.inner.userdata, options); + } + + pub fn processSpawn(userdata: ?*anyopaque, options: std.process.SpawnOptions) std.process.SpawnError!std.process.Child { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.processSpawn(self.inner.userdata, options); + } + + pub fn childWait(userdata: ?*anyopaque, child: *std.process.Child) std.process.Child.WaitError!std.process.Child.Term { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.childWait(self.inner.userdata, child); + } + + pub fn childKill(userdata: ?*anyopaque, child: *std.process.Child) void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + self.inner.vtable.childKill(self.inner.userdata, child); + } + + pub fn progressParentFile(userdata: ?*anyopaque) std.Progress.ParentFileError!std.Io.File { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.progressParentFile(self.inner.userdata); + } + + pub fn inheritParentDir(userdata: ?*anyopaque, handle: std.Io.Dir.Handle) std.Io.InheritParentHandleError!std.Io.Dir { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.inheritParentDir(self.inner.userdata, handle); + } + + pub fn inheritParentFile(userdata: ?*anyopaque, handle: std.Io.File.Handle, flags: std.Io.File.Flags) std.Io.InheritParentHandleError!std.Io.File { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.inheritParentFile(self.inner.userdata, handle, flags); + } + + pub fn now(userdata: ?*anyopaque, clock: std.Io.Clock) std.Io.Timestamp { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.now(self.inner.userdata, clock); + } + + pub fn clockResolution(userdata: ?*anyopaque, clock: std.Io.Clock) std.Io.Clock.ResolutionError!std.Io.Duration { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.clockResolution(self.inner.userdata, clock); + } + + pub fn sleep(userdata: ?*anyopaque, timeout: std.Io.Timeout) std.Io.Cancelable!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.sleep(self.inner.userdata, timeout); + } + + pub fn random(userdata: ?*anyopaque, buffer: []u8) void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + self.inner.vtable.random(self.inner.userdata, buffer); + } + + pub fn randomSecure(userdata: ?*anyopaque, buffer: []u8) std.Io.RandomSecureError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.randomSecure(self.inner.userdata, buffer); + } + + pub fn netListenIp(userdata: ?*anyopaque, address: *const std.Io.net.IpAddress, options: std.Io.net.IpAddress.ListenOptions) std.Io.net.IpAddress.ListenError!std.Io.net.Socket { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.netListenIp(self.inner.userdata, address, options); + } + + pub fn netAccept(userdata: ?*anyopaque, server: std.Io.net.Socket.Handle, options: std.Io.net.Server.AcceptOptions) std.Io.net.Server.AcceptError!std.Io.net.Socket { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.netAccept(self.inner.userdata, server, options); + } + + pub fn netBindIp(userdata: ?*anyopaque, address: *const std.Io.net.IpAddress, options: std.Io.net.IpAddress.BindOptions) std.Io.net.IpAddress.BindError!std.Io.net.Socket { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.netBindIp(self.inner.userdata, address, options); + } + + pub fn netConnectIp(userdata: ?*anyopaque, address: *const std.Io.net.IpAddress, options: std.Io.net.IpAddress.ConnectOptions) std.Io.net.IpAddress.ConnectError!std.Io.net.Socket { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.netConnectIp(self.inner.userdata, address, options); + } + + pub fn netListenUnix(userdata: ?*anyopaque, address: *const std.Io.net.UnixAddress, options: std.Io.net.UnixAddress.ListenOptions) std.Io.net.UnixAddress.ListenError!std.Io.net.Socket.Handle { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.netListenUnix(self.inner.userdata, address, options); + } + + pub fn netConnectUnix(userdata: ?*anyopaque, address: *const std.Io.net.UnixAddress) std.Io.net.UnixAddress.ConnectError!std.Io.net.Socket.Handle { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.netConnectUnix(self.inner.userdata, address); + } + + pub fn netSocketCreatePair(userdata: ?*anyopaque, options: std.Io.net.Socket.CreatePairOptions) std.Io.net.Socket.CreatePairError![2]std.Io.net.Socket { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.netSocketCreatePair(self.inner.userdata, options); + } + + pub fn netWriteFile(userdata: ?*anyopaque, handle: std.Io.net.Socket.Handle, header: []const u8, reader: *std.Io.File.Reader, limit: std.Io.Limit) std.Io.net.Stream.Writer.WriteFileError!usize { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.netWriteFile(self.inner.userdata, handle, header, reader, limit); + } + + pub fn netClose(userdata: ?*anyopaque, sockets: []const std.Io.net.Socket) void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + self.inner.vtable.netClose(self.inner.userdata, sockets); + } + + pub fn netShutdown(userdata: ?*anyopaque, handle: std.Io.net.Socket.Handle, how: std.Io.net.ShutdownHow) std.Io.net.ShutdownError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.netShutdown(self.inner.userdata, handle, how); + } + + pub fn netInterfaceNameResolve(userdata: ?*anyopaque, name: *const std.Io.net.Interface.Name) std.Io.net.Interface.Name.ResolveError!std.Io.net.Interface { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.netInterfaceNameResolve(self.inner.userdata, name); + } + + pub fn netInterfaceName(userdata: ?*anyopaque, iface: std.Io.net.Interface) std.Io.net.Interface.NameError!std.Io.net.Interface.Name { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.netInterfaceName(self.inner.userdata, iface); + } + + pub fn netLookup(userdata: ?*anyopaque, host: std.Io.net.HostName, q: *std.Io.Queue(std.Io.net.HostName.LookupResult), options: std.Io.net.HostName.LookupOptions) std.Io.net.HostName.LookupError!void { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.netLookup(self.inner.userdata, host, q, options); + } +}; diff --git a/vfs/file.zig b/vfs/file.zig new file mode 100644 index 00000000..1b077c33 --- /dev/null +++ b/vfs/file.zig @@ -0,0 +1,430 @@ +const std = @import("std"); +const builtin = @import("builtin"); + +const stdx = @import("stdx"); + +const VFSBase = @import("base.zig").VFSBase; + +const log = std.log.scoped(.@"zml/vfs/file"); + +pub const DirectIoError = error{ + UnsupportedPlatform, + MisalignedBuffer, + BufferTooSmall, + UnexpectedFcntlResult, +}; + +fn canUseDirectIO() bool { + if (builtin.target.os.tag == .linux) { + return @hasField(std.posix.O, "DIRECT"); + } + return false; +} + +fn getStatusFlags(file: std.Io.File) DirectIoError!c_int { + while (true) { + const result: c_int = std.posix.system.fcntl(file.handle, std.posix.F.GETFL, @as(c_int, 0)); + switch (std.posix.errno(result)) { + .SUCCESS => return result, + .INTR => continue, + else => return DirectIoError.UnexpectedFcntlResult, + } + } +} + +fn setStatusFlags(file: std.Io.File, flags: c_int) DirectIoError!void { + while (true) { + switch (std.posix.errno(std.posix.system.fcntl(file.handle, std.posix.F.SETFL, flags))) { + .SUCCESS => return, + .INTR => continue, + else => return DirectIoError.UnexpectedFcntlResult, + } + } +} + +fn useDirectIO(file: std.Io.File) DirectIoError!bool { + if (comptime canUseDirectIO()) { + const flags = try getStatusFlags(file); + const direct_flag: c_int = @bitCast(std.posix.O{ .DIRECT = true }); + return (flags & direct_flag) != 0; + } else { + return DirectIoError.UnsupportedPlatform; + } +} + +fn switchToBufferedIO(file: std.fs.File) DirectIoError!void { + if (comptime canUseDirectIO()) { + const flags = try getStatusFlags(file); + const direct_flag: c_int = @bitCast(std.posix.O{ .DIRECT = true }); + if ((flags & direct_flag) == 0) return; + + try setStatusFlags(file, flags & ~direct_flag); + } else { + return DirectIoError.UnsupportedPlatform; + } +} + +fn switchToDirectIO(file: std.Io.File) DirectIoError!void { + if (builtin.os.tag == .linux and canUseDirectIO()) { + const flags = try getStatusFlags(file); + const direct_flag: c_int = @bitCast(std.posix.O{ .DIRECT = true }); + try setStatusFlags(file, flags | direct_flag); + } else { + return DirectIoError.UnsupportedPlatform; + } +} + +pub const File = struct { + pub const Config = struct { + direct_io: bool = true, + direct_io_alignment: std.mem.Alignment = .fromByteUnits(4 * 1024), + }; + + const Handle = struct { + inner_handle: std.Io.File.Handle, + inner_flags: std.Io.File.Flags, + direct_io_handle: ?std.Io.File.Handle, + + pub fn innerFile(self: *Handle) std.Io.File { + return .{ + .handle = self.inner_handle, + .flags = self.inner_flags, + }; + } + + pub fn directIoFile(self: *Handle) std.Io.File { + return .{ + .handle = self.direct_io_handle.?, + .flags = self.inner_flags, + }; + } + }; + + allocator: std.mem.Allocator, + mutex: std.Io.Mutex = .init, + handles: stdx.SegmentedList(Handle, 0) = .{}, + closed_handles: std.ArrayList(u32) = .empty, + config: Config, + base: VFSBase, + + pub fn init(allocator: std.mem.Allocator, inner: std.Io, config: Config) File { + return .{ + .allocator = allocator, + .config = config, + .base = .init(inner), + }; + } + + pub fn deinit(self: *File) void { + self.handles.deinit(self.allocator); + self.closed_handles.deinit(self.allocator); + } + + pub fn io(self: *File) std.Io { + return .{ + .userdata = &self.base, + .vtable = &comptime VFSBase.vtable(.{ + .operate = operate, + .dirOpenDir = dirOpenDir, + .dirStat = dirStat, + .dirStatFile = dirStatFile, + .dirAccess = dirAccess, + .dirCreateFile = dirCreateFile, + .dirOpenFile = dirOpenFile, + .dirClose = dirClose, + .dirRead = dirRead, + .dirRealPath = dirRealPath, + .dirRealPathFile = dirRealPathFile, + .fileStat = fileStat, + .fileLength = fileLength, + .fileClose = fileClose, + .fileWritePositional = fileWritePositional, + .fileWriteFileStreaming = fileWriteFileStreaming, + .fileWriteFilePositional = fileWriteFilePositional, + .fileReadPositional = fileReadPositional, + .fileSeekBy = fileSeekBy, + .fileSeekTo = fileSeekTo, + .fileRealPath = fileRealPath, + }), + }; + } + + fn openHandle(self: *File) !struct { u32, *Handle } { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + + if (self.closed_handles.pop()) |idx| { + return .{ idx, self.handles.at(idx) }; + } + return .{ @intCast(self.handles.len), try self.handles.addOne(self.allocator) }; + } + + fn closeHandle(self: *File, idx: u32) !void { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + + try self.closed_handles.append(self.allocator, idx); + } + + fn getFileHandle(self: *File, file: std.Io.File) *Handle { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + + return self.handles.at(@intCast(file.handle)); + } + + fn canOpenWithDirectIO(self: *File, file: std.Io.File, flags: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!bool { + const file_stat = self.base.inner.vtable.fileStat(self.base.inner.userdata, file) catch |err| switch (err) { + else => return std.Io.File.OpenError.Unexpected, + }; + + return self.config.direct_io and flags.mode == .read_only and file_stat.size >= self.config.direct_io_alignment.toByteUnits(); + } + + fn innerFile(self: *File, file: std.Io.File, data: []const []u8, position: u64) std.Io.File { + const handle = self.getFileHandle(file); + + if (handle.direct_io_handle == null) { + return handle.innerFile(); + } + + const alignment_bytes: usize = self.config.direct_io_alignment.toByteUnits(); + + if (!std.mem.isAligned(@as(usize, position), alignment_bytes)) { + return handle.innerFile(); + } + + var total_size: usize = 0; + for (data) |buf| { + if (!std.mem.isAligned(@intFromPtr(buf.ptr), alignment_bytes)) { + return handle.innerFile(); + } + total_size += buf.len; + } + + if (!std.mem.isAligned(total_size, alignment_bytes)) { + return handle.innerFile(); + } + + return handle.directIoFile(); + } + + fn operate(userdata: ?*anyopaque, operation: std.Io.Operation) std.Io.Cancelable!std.Io.Operation.Result { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + switch (operation) { + .file_read_streaming => |o| { + const handle = self.getFileHandle(o.file); + const position: u64 = @intCast(std.posix.system.lseek(handle.inner_handle, 0, std.posix.SEEK.CUR)); + const inner_file = self.innerFile(o.file, o.data, position); + return self.base.inner.vtable.operate(self.base.inner.userdata, .{ + .file_read_streaming = .{ + .file = inner_file, + .data = o.data, + }, + }); + }, + .file_write_streaming => |o| { + const handle = self.getFileHandle(o.file); + return self.base.inner.vtable.operate(self.base.inner.userdata, .{ + .file_write_streaming = .{ + .file = handle.innerFile(), + .header = o.header, + .data = o.data, + .splat = o.splat, + }, + }); + }, + .device_io_control, .net_receive, .net_read, .net_send, .net_write => { + return self.base.inner.vtable.operate(self.base.inner.userdata, operation); + }, + } + } + + fn dirOpenDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.OpenOptions) std.Io.Dir.OpenError!std.Io.Dir { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + return self.base.inner.vtable.dirOpenDir(self.base.inner.userdata, dir, sub_path, options); + } + + fn dirStat(userdata: ?*anyopaque, dir: std.Io.Dir) std.Io.Dir.StatError!std.Io.Dir.Stat { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + return self.base.inner.vtable.dirStat(self.base.inner.userdata, dir); + } + + fn dirStatFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.StatFileOptions) std.Io.Dir.StatFileError!std.Io.File.Stat { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + return self.base.inner.vtable.dirStatFile(self.base.inner.userdata, dir, sub_path, options); + } + + fn dirAccess(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + return self.base.inner.vtable.dirAccess(self.base.inner.userdata, dir, sub_path, options); + } + + fn dirCreateFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.Dir.CreateFileOptions) std.Io.File.OpenError!std.Io.File { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + const inner_file = try self.base.inner.vtable.dirCreateFile(self.base.inner.userdata, dir, sub_path, flags); + errdefer inner_file.close(self.base.inner); + + const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; + handle.* = .{ + .inner_handle = inner_file.handle, + .inner_flags = inner_file.flags, + .direct_io_handle = null, + }; + return .{ .handle = @intCast(idx), .flags = inner_file.flags }; + } + + fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!std.Io.File { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + const inner_file = try self.base.inner.vtable.dirOpenFile(self.base.inner.userdata, dir, sub_path, flags); + errdefer inner_file.close(self.base.inner); + + const use_direct_io = try self.canOpenWithDirectIO(inner_file, flags); + + var direct_io_handle: ?std.Io.File.Handle = null; + if (use_direct_io) { + const direct_io_file = try self.base.inner.vtable.dirOpenFile(self.base.inner.userdata, dir, sub_path, flags); + defer if (direct_io_handle == null) direct_io_file.close(self.base.inner); + + switchToDirectIO(direct_io_file) catch |err| switch (err) { + DirectIoError.UnsupportedPlatform => { + log.debug("Direct I/O is not supported on this platform", .{}); + }, + else => { + log.err("Failed to switch to Direct I/O mode: {any}", .{err}); + return std.Io.File.OpenError.Unexpected; + }, + }; + + if (useDirectIO(direct_io_file) catch false) { + direct_io_handle = direct_io_file.handle; + } + } + + const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; + handle.* = .{ + .inner_handle = inner_file.handle, + .inner_flags = inner_file.flags, + .direct_io_handle = direct_io_handle, + }; + + return .{ .handle = @intCast(idx), .flags = inner_file.flags }; + } + + fn dirClose(userdata: ?*anyopaque, dirs: []const std.Io.Dir) void { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + return self.base.inner.vtable.dirClose(self.base.inner.userdata, dirs); + } + + fn dirRead(userdata: ?*anyopaque, reader: *std.Io.Dir.Reader, entries: []std.Io.Dir.Entry) std.Io.Dir.Reader.Error!usize { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + return self.base.inner.vtable.dirRead(self.base.inner.userdata, reader, entries); + } + + fn dirRealPath(userdata: ?*anyopaque, dir: std.Io.Dir, out_buffer: []u8) std.Io.Dir.RealPathError!usize { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + return self.base.inner.vtable.dirRealPath(self.base.inner.userdata, dir, out_buffer); + } + + fn dirRealPathFile(userdata: ?*anyopaque, dir: std.Io.Dir, path_name: []const u8, out_buffer: []u8) std.Io.Dir.RealPathFileError!usize { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + return self.base.inner.vtable.dirRealPathFile(self.base.inner.userdata, dir, path_name, out_buffer); + } + + fn fileStat(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.StatError!std.Io.File.Stat { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + return self.base.inner.vtable.fileStat(self.base.inner.userdata, handle.innerFile()); + } + + fn fileLength(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.LengthError!u64 { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + return self.base.inner.vtable.fileLength(self.base.inner.userdata, handle.innerFile()); + } + + fn fileClose(userdata: ?*anyopaque, files: []const std.Io.File) void { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + for (files) |file| { + const handle = self.getFileHandle(file); + + if (handle.direct_io_handle) |_| { + self.base.inner.vtable.fileClose(self.base.inner.userdata, &.{handle.directIoFile()}); + } + + self.base.inner.vtable.fileClose(self.base.inner.userdata, &.{handle.innerFile()}); + + self.closeHandle(@intCast(file.handle)) catch unreachable; + } + } + + fn fileWritePositional(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, data: []const []const u8, splat: usize, offset: u64) std.Io.File.WritePositionalError!usize { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + return self.base.inner.vtable.fileWritePositional(self.base.inner.userdata, handle.innerFile(), header, data, splat, offset); + } + + fn fileWriteFileStreaming(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, reader: *std.Io.File.Reader, limit: std.Io.Limit) std.Io.File.Writer.WriteFileError!usize { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + const own_io = self.io(); + if (reader.io.userdata != own_io.userdata or reader.io.vtable != own_io.vtable) return error.Unimplemented; + + const destination = self.getFileHandle(file); + const source = self.getFileHandle(reader.file); + + const original_source_io = reader.io; + const original_source_file = reader.file; + reader.io = self.base.inner; + reader.file = source.innerFile(); + defer { + reader.io = original_source_io; + reader.file = original_source_file; + } + + return self.base.inner.vtable.fileWriteFileStreaming(self.base.inner.userdata, destination.innerFile(), header, reader, limit); + } + + fn fileWriteFilePositional(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, reader: *std.Io.File.Reader, limit: std.Io.Limit, offset: u64) std.Io.File.WriteFilePositionalError!usize { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + const own_io = self.io(); + if (reader.io.userdata != own_io.userdata or reader.io.vtable != own_io.vtable) return error.Unimplemented; + + const destination = self.getFileHandle(file); + const source = self.getFileHandle(reader.file); + + const original_source_io = reader.io; + const original_source_file = reader.file; + reader.io = self.base.inner; + reader.file = source.innerFile(); + defer { + reader.io = original_source_io; + reader.file = original_source_file; + } + + return self.base.inner.vtable.fileWriteFilePositional(self.base.inner.userdata, destination.innerFile(), header, reader, limit, offset); + } + + fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + const inner_file = self.innerFile(file, data, offset); + return self.base.inner.vtable.fileReadPositional(self.base.inner.userdata, inner_file, data, offset); + } + + fn fileSeekBy(userdata: ?*anyopaque, file: std.Io.File, relative_offset: i64) std.Io.File.SeekError!void { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + return self.base.inner.vtable.fileSeekBy(self.base.inner.userdata, handle.innerFile(), relative_offset); + } + + fn fileSeekTo(userdata: ?*anyopaque, file: std.Io.File, absolute_offset: u64) std.Io.File.SeekError!void { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + return self.base.inner.vtable.fileSeekTo(self.base.inner.userdata, handle.innerFile(), absolute_offset); + } + + fn fileRealPath(userdata: ?*anyopaque, file: std.Io.File, out_buffer: []u8) std.Io.File.RealPathError!usize { + const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + return self.base.inner.vtable.fileRealPath(self.base.inner.userdata, handle.innerFile(), out_buffer); + } +}; diff --git a/vfs/gcs.zig b/vfs/gcs.zig new file mode 100644 index 00000000..acc7f634 --- /dev/null +++ b/vfs/gcs.zig @@ -0,0 +1,1138 @@ +const std = @import("std"); +const builtin = @import("builtin"); + +const stdx = @import("stdx"); + +const parallel_read = @import("parallel_read.zig"); +const VFSBase = @import("base.zig").VFSBase; + +const log = std.log.scoped(.@"zml/vfs/gcs"); + +const EndpointUrl = "https://storage.googleapis.com"; +const MetadataUrl = "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token"; + +/// Check if running on Google Compute Engine / Cloud Run host infrastructure, +/// because metadata server probing should only happen on GCP. +/// Mirrors the TPU platform detection approach. +fn isOnGCP(io: std.Io) !bool { + if (builtin.os.tag != .linux) return false; + + const GoogleComputeEngine = "Google Compute Engine"; + var buffer: [GoogleComputeEngine.len]u8 = undefined; + return std.mem.eql( + u8, + GoogleComputeEngine, + try std.Io.Dir.readFile( + .cwd(), + io, + "/sys/devices/virtual/dmi/id/product_name", + &buffer, + ), + ); +} + +const OAuthToken = struct { + // https://developers.google.com/identity/protocols/oauth2#size + const Size = 2048; + + access_token: []const u8, + expires_in: u64, +}; + +const Credentials = union(enum) { + const AuthorizedUser = struct { + type: []const u8, + client_id: []const u8, + client_secret: []const u8, + refresh_token: []const u8, + quota_project_id: ?[]const u8 = null, + account: []const u8, + universe_domain: []const u8, + }; + + const ServiceAccount = struct { + pub const JwtAssertion = struct { + iss: []const u8, + scope: []const u8 = "https://www.googleapis.com/auth/devstorage.read_only", + aud: []const u8 = "https://oauth2.googleapis.com/token", + exp: i64, + iat: i64, + sub: ?[]const u8 = null, + }; + + type: []const u8, + client_email: []const u8, + private_key: []const u8, + token_uri: []const u8, + quota_project_id: ?[]const u8 = null, + + pub fn makeAssertion(self: *const ServiceAccount, io_: std.Io) JwtAssertion { + const now: std.Io.Timestamp = .now(io_, .real); + return .{ + .iss = self.client_email, + .iat = now.toSeconds(), + .exp = now.addDuration(.fromSeconds(3600)).toSeconds(), + }; + } + }; + + authorized_user: AuthorizedUser, + service_account: ServiceAccount, + metadata_server: void, + + pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) std.json.ParseError(@TypeOf(source.*))!Credentials { + return jsonParseFromValue( + allocator, + try std.json.innerParse(std.json.Value, allocator, source, options), + options, + ); + } + + pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) std.json.ParseFromValueError!Credentials { + return switch (source) { + .object => |obj| blk: { + const obj_type = obj.get("type") orelse return std.json.ParseFromValueError.UnexpectedToken; + if (std.ascii.eqlIgnoreCase(obj_type.string, "authorized_user")) { + break :blk .{ + .authorized_user = try std.json.parseFromValueLeaky( + AuthorizedUser, + allocator, + source, + options, + ), + }; + } else if (std.ascii.eqlIgnoreCase(obj_type.string, "service_account")) { + break :blk .{ + .service_account = try std.json.parseFromValueLeaky( + ServiceAccount, + allocator, + source, + options, + ), + }; + } + log.warn("Unknown credentials type: {s}", .{obj_type.string}); + return std.json.ParseFromValueError.InvalidEnumTag; + }, + else => std.json.ParseFromValueError.UnexpectedToken, + }; + } +}; + +const ReadState = struct { index: usize, objects: [][]const u8 }; + +pub const GCS = struct { + const ParallelRead = struct { + const Pool = parallel_read.Pool(Job); + + const Batch = struct { + state: parallel_read.BatchState, + backend: *GCS, + uri: std.Uri, + path: []const u8, + authorization: std.http.Client.Request.Headers.Value, + }; + + const Job = struct { + data: []const []u8, + file_offset: u64, + chunk_offset: usize, + chunk_len: usize, + batch: *Batch, + + pub fn perform(job: Job, client: *std.http.Client) anyerror!parallel_read.Status { + var range_buf: [64]u8 = undefined; + const range_header = std.fmt.bufPrint( + &range_buf, + "bytes={d}-{d}", + .{ job.file_offset, job.file_offset + @as(u64, @intCast(job.chunk_len - 1)) }, + ) catch unreachable; + + var req = client.request( + .GET, + job.batch.uri, + .{ + .headers = .{ + .accept_encoding = .{ .override = "identity" }, + .authorization = job.batch.authorization, + }, + .extra_headers = &.{.{ .name = "Range", .value = range_header }}, + }, + ) catch |err| { + switch (err) { + // transient connect failures + error.ConnectionRefused, + error.ConnectionResetByPeer, + error.HostUnreachable, + error.NetworkUnreachable, + error.NetworkDown, + error.Timeout, + error.NameServerFailure, + => { + log.warn("Failed to connect: {}", .{err}); + return .retry(); + }, + else => { + log.err("Failed to connect: {}", .{err}); + return err; + }, + } + }; + defer req.deinit(); + + req.sendBodiless() catch |err| switch (err) { + error.WriteFailed => { + log.warn("Failed to send headers: {}", .{err}); + return .retry(); + }, + }; + + var redirect_buffer: [8 * 1024]u8 = undefined; + var res = req.receiveHead(&redirect_buffer) catch |err| { + switch (err) { + // stale keep-alive / peer closed while waiting for response head + error.HttpConnectionClosing, + error.HttpRequestTruncated, + + // transport read/write failure; retry on a fresh connection + error.ReadFailed, + error.WriteFailed, + + // transient connect failures + error.ConnectionRefused, + error.ConnectionResetByPeer, + error.HostUnreachable, + error.NetworkUnreachable, + error.NetworkDown, + error.Timeout, + error.NameServerFailure, + => { + log.warn("Failed to receive headers: {}", .{err}); + return .retry(); + }, + + else => { + log.err("Failed to receive headers: {}", .{err}); + return err; + }, + } + }; + + if (res.head.status != .partial_content and res.head.status != .ok) { + const status: parallel_read.Status = switch (res.head.status) { + .request_timeout, .too_many_requests => .retry(), + else => if (res.head.status.class() == .server_error) .retry() else { + log.err("Failed to read {s}: {s}", .{ job.batch.path, res.head.bytes }); + return error.RequestFailed; + }, + }; + log.warn("Failed to read {s}: {s}", .{ job.batch.path, res.head.bytes }); + return status; + } + + const content_range = blk: { + var it = res.head.iterateHeaders(); + while (it.next()) |header| { + if (std.ascii.eqlIgnoreCase(header.name, "Content-Range")) { + break :blk parallel_read.parseContentRange(header.value); + } + } + break :blk null; + }; + + const reader = res.reader(&.{}); + parallel_read.readChunk(reader, content_range, job.file_offset, job.data, job.chunk_offset, job.chunk_len) catch |err| { + switch (err) { + error.EndOfStream, + error.ReadFailed, + => { + log.warn("Failed to read from response: {}", .{err}); + return .retry(); + }, + else => { + log.err("Failed to read from response: {}", .{err}); + return err; + }, + } + }; + + return .success; + } + }; + }; + + const Token = struct { + header: []u8, + expires_at: std.Io.Timestamp, + + fn expired(self: *const Token, io_: std.Io) bool { + const now: std.Io.Timestamp = .now(io_, .real); + return now.toSeconds() >= self.expires_at.toSeconds(); + } + }; + + pub const Config = struct { + credentials: ?Credentials = null, + endpoint_uri: std.Uri, + region: []const u8 = "auto", + }; + + const Handle = struct { + pub const Type = enum { file, directory }; + + type: Type, + uri: []const u8, + pos: u64, + size: u64, + + pub fn init(allocator: std.mem.Allocator, type_: Type, path: []const u8, size: u64) !Handle { + return .{ + .type = type_, + .uri = try allocator.dupe(u8, path), + .pos = 0, + .size = size, + }; + } + + pub fn deinit(self: *Handle, allocator: std.mem.Allocator) void { + allocator.free(self.uri); + } + }; + + allocator: std.mem.Allocator, + arena: std.heap.ArenaAllocator, + mutex: std.Io.Mutex = .init, + client: *std.http.Client, + config: Config, + token: Token, + read_pool: *ParallelRead.Pool, + handles: stdx.SegmentedList(Handle, 0) = .{}, + closed_handles: std.ArrayList(u32) = .empty, + dir_read_states: std.AutoHashMapUnmanaged(*std.Io.Dir.Reader, ReadState) = .{}, + base: VFSBase, + + pub const InitOpts = struct { + credentials: ?union(enum) { + json: *std.Io.Reader, + metadata_server: void, + } = null, + endpoint_url: []const u8 = "https://storage.googleapis.com", + region: []const u8 = "auto", + read_pool: parallel_read.InitOpts = .{ + .chunk_size = 16 << 20, + .num_workers = 32, + .queue_capacity = 128, + .max_retries = 5, + .retry_initial_delay = .fromMilliseconds(500), + .retry_max_delay = .fromSeconds(30), + }, + }; + + pub const InitError = error{ + InvalidCredentialJson, + Unexpected, + } || std.mem.Allocator.Error || std.Io.ConcurrentError; + + pub fn init(allocator: std.mem.Allocator, inner: std.Io, http_client: *std.http.Client, opts: InitOpts) InitError!GCS { + var arena: std.heap.ArenaAllocator = .init(allocator); + errdefer arena.deinit(); + + const read_pool = try allocator.create(ParallelRead.Pool); + errdefer allocator.destroy(read_pool); + + try read_pool.init(allocator, inner, http_client, opts.read_pool); + errdefer read_pool.deinit(allocator, inner); + + const config: Config = .{ + .credentials = if (opts.credentials) |creds| switch (creds) { + .json => |reader| blk: { + var json_reader: std.json.Reader = .init(allocator, reader); + defer json_reader.deinit(); + + break :blk std.json.parseFromTokenSourceLeaky( + Credentials, + arena.allocator(), + &json_reader, + .{ .allocate = .alloc_always }, + ) catch return InitError.InvalidCredentialJson; + }, + .metadata_server => .{ .metadata_server = {} }, + } else null, + .endpoint_uri = std.Uri.parse(try arena.allocator().dupe(u8, opts.endpoint_url)) catch return InitError.Unexpected, + .region = try arena.allocator().dupe(u8, opts.region), + }; + + const token: Token = .{ + .header = try arena.allocator().alloc(u8, "Bearer ".len + OAuthToken.Size), + .expires_at = .zero, + }; + + return .{ + .allocator = allocator, + .arena = arena, + .base = .init(inner), + .client = http_client, + .config = config, + .token = token, + .read_pool = read_pool, + }; + } + + pub fn auto(allocator: std.mem.Allocator, inner_io: std.Io, http_client: *std.http.Client, environ_map: *std.process.Environ.Map) !GCS { + var jsonBuffer: [1024]u8 = undefined; + + const credentials_file: ?std.Io.File = if (environ_map.get("GOOGLE_APPLICATION_CREDENTIALS")) |json_path| blk: { + break :blk std.Io.Dir.openFile(.cwd(), inner_io, json_path, .{}) catch |err| { + log.err("Unable to open GCS credentials file {s}, skipping: {t}", .{ json_path, err }); + break :blk null; + }; + } else if (applicationDefaultCredentials(inner_io, environ_map)) |f| + f + else + null; + defer if (credentials_file) |f| f.close(inner_io); + + if (credentials_file) |f| blk: { + var reader = f.reader(inner_io, &jsonBuffer); + return GCS.init(allocator, inner_io, http_client, .{ .credentials = .{ .json = &reader.interface } }) catch |err| switch (err) { + InitError.InvalidCredentialJson => { + log.err("Invalid GCS credentials JSON", .{}); + break :blk; + }, + else => return err, + }; + } + + if (isOnGCP(inner_io) catch false) { + return .init(allocator, inner_io, http_client, .{ .credentials = .{ .metadata_server = {} } }); + } + + return .init(allocator, inner_io, http_client, .{}); + } + + fn applicationDefaultCredentials(io_: std.Io, environ_map: *std.process.Environ.Map) ?std.Io.File { + var buffer: [1024]u8 = undefined; + const path = stdx.Io.Dir.path.bufJoin(&buffer, switch (builtin.os.tag) { + .windows => &.{ environ_map.get("APPDATA") orelse return null, "gcloud", "application_default_credentials.json" }, + else => &.{ environ_map.get("HOME") orelse return null, ".config", "gcloud", "application_default_credentials.json" }, + }) catch unreachable; + return std.Io.Dir.openFile(.cwd(), io_, path, .{}) catch null; + } + + fn refreshMetadataServerToken(client: *std.http.Client, buffer: []u8) !?[]const u8 { + var response_writer: std.Io.Writer = .fixed(buffer); + const result = try client.fetch(.{ + .location = .{ .url = MetadataUrl }, + .method = .GET, + .headers = .{ + .accept_encoding = .{ .override = "identity" }, + }, + .extra_headers = &.{ + .{ .name = "Metadata-Flavor", .value = "Google" }, + }, + .response_writer = &response_writer, + }); + + if (result.status != .ok) { + return null; + } + + return response_writer.buffered(); + } + + fn refreshAuthorizedUserToken(client: *std.http.Client, authorized_user: Credentials.AuthorizedUser, buffer: []u8) ![]const u8 { + var response_writer: std.Io.Writer = .fixed(buffer); + const result = try client.fetch(.{ + .location = .{ .url = "https://oauth2.googleapis.com/token" }, + .method = .POST, + .payload = try std.fmt.bufPrint(buffer, "grant_type=refresh_token&client_id={f}&client_secret={f}&refresh_token={f}", .{ + std.fmt.alt(std.Uri.Component{ .raw = authorized_user.client_id }, .formatQuery), + std.fmt.alt(std.Uri.Component{ .raw = authorized_user.client_secret }, .formatQuery), + std.fmt.alt(std.Uri.Component{ .raw = authorized_user.refresh_token }, .formatQuery), + }), + .headers = .{ + .accept_encoding = .{ .override = "identity" }, + .content_type = .{ .override = "application/x-www-form-urlencoded" }, + }, + .response_writer = &response_writer, + }); + if (result.status != .ok) { + log.err("Failed to refresh ADC token: {s}", .{response_writer.buffered()}); + return error.RequestFailed; + } + return response_writer.buffered(); + } + + fn refreshServiceAccountToken(io_: std.Io, client: *std.http.Client, service_account: Credentials.ServiceAccount, buffer: []u8) ![]const u8 { + _ = io_; + _ = client; + _ = service_account; + _ = buffer; + return error.Unimplemented; + + // const assertion = service_account.makeAssertion(io_); + + // var tmp_buffer: [8 * 1024]u8 = undefined; + + // var buffer_writer: std.Io.Writer = .fixed(buffer); + // var signing_input_buffer: [2 * 1024]u8 = undefined; + // var signing_input_writer: std.Io.Writer = .fixed(signing_input_buffer); + + // const encoded_rs256_header = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9"; + // try signing_input_writer.print("{s}.", .{encoded_rs256_header}); + // std.base64.url_safe_no_pad.Encoder.encodeWriter( + // signing_input_writer, + // try std.fmt.bufPrint(tmp_buffer, "{f}", .{ + // std.json.fmt(assertion, .{}), + // }), + // ); + // const signing_input = signing_input_writer.buffered(); + // _ = signing_input; + + // const result = try client.fetch(.{ + // .location = .{ .url = "https://oauth2.googleapis.com/token" }, + // .method = .POST, + // .payload = try std.fmt.bufPrint(buffer, "grant_type=refresh_token&client_id={f}&client_secret={f}&refresh_token={f}", .{ + // std.fmt.alt(std.Uri.Component{ .raw = service_account.client_id }, .formatQuery), + // std.fmt.alt(std.Uri.Component{ .raw = authorized_user.client_secret }, .formatQuery), + // std.fmt.alt(std.Uri.Component{ .raw = authorized_user.refresh_token }, .formatQuery), + // }), + // .headers = .{ + // .accept_encoding = .{ .override = "identity" }, + // .content_type = .{ .override = "application/x-www-form-urlencoded" }, + // }, + // .response_writer = &response_writer, + // }); + // if (result.status != .ok) { + // log.err("Failed to refresh ADC token: {s}", .{response_writer.buffered()}); + // return error.RequestFailed; + // } + } + + fn refreshToken(self: *GCS) !void { + var buffer: [4 * 1024]u8 = undefined; + const payload = switch (self.config.credentials.?) { + .authorized_user => |authorized_user| try refreshAuthorizedUserToken(self.client, authorized_user, &buffer), + .service_account => |service_account| try refreshServiceAccountToken(self.base.inner, self.client, service_account, &buffer), + .metadata_server => try refreshMetadataServerToken(self.client, &buffer) orelse return error.RequestFailed, + }; + const oauth_token = try std.json.parseFromSlice(OAuthToken, self.allocator, payload, .{ + .allocate = .alloc_if_needed, + .ignore_unknown_fields = true, + }); + defer oauth_token.deinit(); + self.token = .{ + .header = try std.fmt.bufPrint(self.token.header, "Bearer {s}", .{oauth_token.value.access_token}), + .expires_at = std.Io.Clock.now(.real, self.base.inner).addDuration(.fromSeconds(@intCast(oauth_token.value.expires_in))), + }; + } + + fn getOrRefreshToken(self: *GCS) !std.http.Client.Request.Headers.Value { + if (self.config.credentials == null) { + return .omit; + } + + if (self.token.expired(self.base.inner)) { + try self.refreshToken(); + } + return .{ .override = self.token.header }; + } + + pub fn deinit(self: *GCS) void { + self.read_pool.deinit(self.allocator, self.base.inner); + self.allocator.destroy(self.read_pool); + + var idx: usize = 0; + while (idx < self.handles.len) : (idx += 1) { + const is_closed = for (self.closed_handles.items) |closed_idx| { + if (closed_idx == idx) break true; + } else false; + + if (!is_closed) { + self.handles.at(idx).deinit(self.allocator); + } + } + self.handles.deinit(self.allocator); + self.closed_handles.deinit(self.allocator); + + var it = self.dir_read_states.iterator(); + while (it.next()) |entry| { + for (entry.value_ptr.objects) |obj| { + self.allocator.free(obj); + } + self.allocator.free(entry.value_ptr.objects); + } + self.dir_read_states.deinit(self.allocator); + self.arena.deinit(); + } + + pub fn io(self: *GCS) std.Io { + return .{ + .userdata = &self.base, + .vtable = &comptime VFSBase.vtable(.{ + .operate = operate, + .dirOpenDir = dirOpenDir, + .dirStat = dirStat, + .dirStatFile = dirStatFile, + .dirAccess = dirAccess, + .dirOpenFile = dirOpenFile, + .dirClose = dirClose, + .dirRead = dirRead, + .dirRealPath = dirRealPath, + .dirRealPathFile = dirRealPathFile, + .fileStat = fileStat, + .fileLength = fileLength, + .fileClose = fileClose, + .fileReadPositional = fileReadPositional, + .fileSeekBy = fileSeekBy, + .fileSeekTo = fileSeekTo, + .fileRealPath = fileRealPath, + }), + }; + } + + fn openHandle(self: *GCS) !struct { u32, *Handle } { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + + if (self.closed_handles.pop()) |idx| return .{ idx, self.handles.at(idx) }; + + return .{ @intCast(self.handles.len), try self.handles.addOne(self.allocator) }; + } + + fn closeHandle(self: *GCS, idx: u32) !void { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + self.handles.at(idx).deinit(self.allocator); + try self.closed_handles.append(self.allocator, idx); + } + + fn getFileHandle(self: *GCS, file: std.Io.File) *Handle { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + return self.handles.at(@intCast(file.handle)); + } + + fn getDirHandle(self: *GCS, dir: std.Io.Dir) *Handle { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + return self.handles.at(@intCast(dir.handle)); + } + + fn resolvePath(self: *GCS, dir: std.Io.Dir, sub_path: []const u8, out_buffer: []u8) ![]u8 { + if (std.meta.eql(dir, std.Io.Dir.cwd())) { + return try std.fmt.bufPrint(out_buffer, "{s}", .{sub_path}); + } + + const handle = self.getDirHandle(dir); + + const trimmed_uri = std.mem.trimEnd(u8, handle.uri, "/"); + const trimmed_sub_path = std.mem.trimStart(u8, sub_path, "/"); + + if (trimmed_uri.len == 0) return try std.fmt.bufPrint(out_buffer, "{s}", .{trimmed_sub_path}); + if (trimmed_sub_path.len == 0) return try std.fmt.bufPrint(out_buffer, "{s}", .{trimmed_uri}); + + return try std.fmt.bufPrint(out_buffer, "{s}/{s}", .{ trimmed_uri, trimmed_sub_path }); + } + + fn operate(userdata: ?*anyopaque, operation: std.Io.Operation) std.Io.Cancelable!std.Io.Operation.Result { + const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); + switch (operation) { + .file_read_streaming => |o| { + const handle = self.getFileHandle(o.file); + const total = self.performRead(handle, o.data, handle.pos) catch |err| { + log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, handle.pos, err }); + return .{ .file_read_streaming = error.EndOfStream }; + }; + + if (total == 0) { + return .{ .file_read_streaming = error.EndOfStream }; + } + + handle.pos += @intCast(total); + return .{ .file_read_streaming = total }; + }, + .file_write_streaming, .device_io_control, .net_receive, .net_read, .net_send, .net_write => { + return self.base.inner.vtable.operate(self.base.inner.userdata, operation); + }, + } + } + + fn dirOpenDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.OpenOptions) std.Io.Dir.OpenError!std.Io.Dir { + const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); + var path_buffer: [8 * 1024]u8 = undefined; + const path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.Dir.OpenError.SystemResources; + + const idx, const handle = self.openHandle() catch return std.Io.Dir.OpenError.Unexpected; + handle.* = Handle.init(self.allocator, .directory, path, 0) catch return std.Io.Dir.OpenError.Unexpected; + + return .{ .handle = @intCast(idx) }; + } + + fn dirStat(userdata: ?*anyopaque, dir: std.Io.Dir) std.Io.Dir.StatError!std.Io.Dir.Stat { + const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); + const handle = self.getDirHandle(dir); + + return .{ + .inode = @intCast(@intFromPtr(handle)), + .nlink = 0, + .size = handle.size, + .permissions = .fromMode(0o444), + .kind = .directory, + .atime = null, + .mtime = std.Io.Timestamp.zero, + .ctime = std.Io.Timestamp.zero, + .block_size = 1, + }; + } + + fn dirStatFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.StatFileOptions) std.Io.Dir.StatFileError!std.Io.File.Stat { + const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); + const size = self.fetchSize(dir, sub_path) catch |err| switch (err) { + error.FileNotFound => return std.Io.File.OpenError.FileNotFound, + error.PermissionDenied => return std.Io.File.OpenError.PermissionDenied, + else => return std.Io.File.OpenError.Unexpected, + }; + + return .{ + .inode = 0, + .nlink = 0, + .size = size, + .permissions = .fromMode(0o444), + .kind = .file, + .atime = null, + .mtime = std.Io.Timestamp.zero, + .ctime = std.Io.Timestamp.zero, + .block_size = 1, + }; + } + + fn dirAccess(_: ?*anyopaque, _: std.Io.Dir, _: []const u8, _: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void {} + + fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!std.Io.File { + const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); + const size = self.fetchSize(dir, sub_path) catch |err| switch (err) { + error.FileNotFound => return std.Io.File.OpenError.FileNotFound, + error.PermissionDenied => return std.Io.File.OpenError.PermissionDenied, + else => return std.Io.File.OpenError.Unexpected, + }; + + var path_buffer: [8 * 1024]u8 = undefined; + const path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.File.OpenError.SystemResources; + const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; + handle.* = Handle.init(self.allocator, .file, path, size) catch return std.Io.File.OpenError.Unexpected; + + return .{ .handle = @intCast(idx), .flags = .{ .nonblocking = false } }; + } + + fn dirClose(userdata: ?*anyopaque, dirs: []const std.Io.Dir) void { + const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); + for (dirs) |dir| { + self.closeHandle(@intCast(dir.handle)) catch unreachable; + } + } + + fn dirRead(userdata: ?*anyopaque, reader: *std.Io.Dir.Reader, entries: []std.Io.Dir.Entry) std.Io.Dir.Reader.Error!usize { + const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); + + if (reader.state == .finished) return 0; + + if (reader.state == .reset) { + if (self.dir_read_states.fetchRemove(reader)) |kv| { + for (kv.value.objects) |obj| self.allocator.free(obj); + self.allocator.free(kv.value.objects); + } + + const handle = self.getDirHandle(reader.dir); + const objects = self.listObjects(handle.uri) catch return std.Io.Dir.Reader.Error.Unexpected; + + self.dir_read_states.put(self.allocator, reader, .{ + .index = 0, + .objects = objects, + }) catch return std.Io.Dir.Reader.Error.Unexpected; + + reader.state = if (objects.len > 0) .reading else .finished; + + if (objects.len == 0) return 0; + } + + const state = self.dir_read_states.getPtr(reader) orelse return std.Io.Dir.Reader.Error.Unexpected; + + var count: usize = 0; + while (count < entries.len and state.index < state.objects.len) { + const obj_key = state.objects[state.index]; + const kind: std.Io.File.Kind = if (std.mem.endsWith(u8, obj_key, "/")) .directory else .file; + + const name = if (std.mem.lastIndexOfScalar(u8, std.mem.trimEnd(u8, obj_key, "/"), '/')) |idx| + obj_key[idx + 1 ..] + else + obj_key; + + entries[count] = .{ + .name = std.mem.trimEnd(u8, name, "/"), + .kind = kind, + .inode = state.index, + }; + count += 1; + state.index += 1; + } + + if (state.index >= state.objects.len) { + reader.state = .finished; + } + + return count; + } + + fn dirRealPath(userdata: ?*anyopaque, dir: std.Io.Dir, out_buffer: []u8) std.Io.Dir.RealPathError!usize { + const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); + const handle = self.getDirHandle(dir); + const path = std.fmt.bufPrint(out_buffer, "{s}", .{handle.uri}) catch return std.Io.Dir.RealPathError.SystemResources; + return path.len; + } + + fn dirRealPathFile(userdata: ?*anyopaque, dir: std.Io.Dir, path_name: []const u8, out_buffer: []u8) std.Io.Dir.RealPathFileError!usize { + const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); + const real_path = self.resolvePath(dir, path_name, out_buffer) catch return std.Io.Dir.RealPathFileError.NameTooLong; + return real_path.len; + } + + fn fileStat(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.StatError!std.Io.File.Stat { + const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); + const handle = self.getFileHandle(file); + + return .{ + .inode = @intCast(file.handle), + .nlink = 0, + .size = handle.size, + .permissions = .fromMode(0o444), + .kind = .file, + .atime = null, + .mtime = std.Io.Timestamp.zero, + .ctime = std.Io.Timestamp.zero, + .block_size = 1, + }; + } + + fn fileLength(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.LengthError!u64 { + const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); + return self.getFileHandle(file).size; + } + + fn fileClose(userdata: ?*anyopaque, files: []const std.Io.File) void { + const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); + for (files) |file| { + self.closeHandle(@intCast(file.handle)) catch unreachable; + } + } + + fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { + const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); + const handle = self.getFileHandle(file); + return self.performRead(handle, data, offset) catch |err| { + log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, offset, err }); + return std.Io.File.ReadPositionalError.Unexpected; + }; + } + + fn fileSeekBy(userdata: ?*anyopaque, file: std.Io.File, relative_offset: i64) std.Io.File.SeekError!void { + const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); + const handle = self.getFileHandle(file); + + if (relative_offset >= 0) { + handle.pos = handle.pos +| @as(u64, @intCast(relative_offset)); + } else { + const abs_offset: u64 = @intCast(-relative_offset); + if (abs_offset > handle.pos) { + handle.pos = 0; + } else { + handle.pos -= abs_offset; + } + } + } + + fn fileSeekTo(userdata: ?*anyopaque, file: std.Io.File, absolute_offset: u64) std.Io.File.SeekError!void { + const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); + const handle = self.getFileHandle(file); + handle.pos = absolute_offset; + } + + fn fileRealPath(userdata: ?*anyopaque, file: std.Io.File, out_buffer: []u8) std.Io.File.RealPathError!usize { + const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); + const handle = self.getFileHandle(file); + const path = std.fmt.bufPrint(out_buffer, "{s}", .{handle.uri}) catch return std.Io.File.RealPathError.SystemResources; + return path.len; + } + + fn getTimestamp(self: *GCS, buf: []u8) ![]const u8 { + const now_ts = std.Io.Clock.now(.real, self.base.inner); + + const now: u64 = @intCast(@divFloor(now_ts.nanoseconds, std.time.ns_per_s)); + const epoch_secs: std.time.epoch.EpochSeconds = .{ .secs = now }; + const year_day = epoch_secs.getEpochDay().calculateYearDay(); + const month_day = year_day.calculateMonthDay(); + const day_seconds = epoch_secs.getDaySeconds(); + + return try std.fmt.bufPrint(buf, "{d:0>4}{d:0>2}{d:0>2}T{d:0>2}{d:0>2}{d:0>2}Z", .{ + year_day.year, + @backingInt(month_day.month), + month_day.day_index + 1, + day_seconds.getHoursIntoDay(), + day_seconds.getMinutesIntoHour(), + day_seconds.getSecondsIntoMinute(), + }); + } + + fn pathComponents(_: *GCS, path_: []const u8) struct { []const u8, []const u8 } { + const path = std.mem.trim(u8, path_, "/"); + + if (std.mem.indexOfScalar(u8, path, '/')) |idx| { + return .{ path[0..idx], if (idx + 1 < path.len) path[idx + 1 ..] else "" }; + } else { + return .{ path, "" }; + } + } + + fn gcsUri(self: *GCS, path: []const u8) std.Uri { + var uri = self.config.endpoint_uri; + uri.path = .{ .raw = path }; + uri.query = .{ .percent_encoded = "alt=media" }; + return uri; + } + + // fn authHeaders(self: *GCS, _: std.http.Method, _: std.Uri, _: []const u8, authorization_buffer: []u8) !struct { + // authorization: ?[]const u8, + // extra_headers: [3]?std.http.Header, + // extra_len: usize, + // } { + // if (self.config.oauth_access_token == null) { + // if (self.config.googleCredentialsJsonStr) |json| { + // const oauth = try refreshAuthorizedUserTokenFromJson(self.allocator, self.base.inner, json); + // if (self.config.oauth_access_token) |old_token| self.allocator.free(old_token); + // if (self.config.quota_project_id) |old_project| self.allocator.free(old_project); + // self.config.oauth_access_token = oauth.access_token; + // self.config.quota_project_id = oauth.quota_project_id; + // } + // } + + // const token = self.config.oauth_access_token orelse return error.MissingCredentials; + // const authorization = try std.fmt.bufPrint(authorization_buffer, "Bearer {s}", .{token}); + // var extra_headers: [3]?std.http.Header = .{ null, null, null }; + // var extra_len: usize = 0; + // if (self.config.quota_project_id) |project| { + // extra_headers[0] = .{ .name = "x-goog-user-project", .value = project }; + // extra_len = 1; + // } + // return .{ + // .authorization = authorization, + // .extra_headers = extra_headers, + // .extra_len = extra_len, + // }; + // } + + fn listObjectsBody(self: *GCS, bucket: []const u8, key_prefix: []const u8) ![]u8 { + var query_buf: [4096]u8 = undefined; + var query_writer = std.Io.Writer.fixed(&query_buf); + + try query_writer.writeAll("delimiter="); + try std.Uri.Component.percentEncode(&query_writer, "/", gcsEncodeIsValid); + try query_writer.writeAll("&max-keys=1000"); + + if (key_prefix.len > 0) { + try query_writer.writeAll("&prefix="); + try std.Uri.Component.percentEncode(&query_writer, key_prefix, gcsEncodeIsValid); + if (!std.mem.endsWith(u8, key_prefix, "/")) { + try std.Uri.Component.percentEncode(&query_writer, "/", gcsEncodeIsValid); + } + } + + var path_buf: [256]u8 = undefined; + var uri = self.config.endpoint_uri; + uri.path = .{ .percent_encoded = try std.fmt.bufPrint(&path_buf, "/{s}", .{bucket}) }; + uri.query = .{ .percent_encoded = query_writer.buffered() }; + + var req = try self.client.request(.GET, uri, .{ + .redirect_behavior = .not_allowed, + .headers = .{ + .accept_encoding = .{ .override = "identity" }, + .authorization = try self.getOrRefreshToken(), + }, + .extra_headers = if (self.quotaProjectId()) |project| + &.{.{ .name = "x-goog-user-project", .value = project }} + else + &.{}, + }); + defer req.deinit(); + + try req.sendBodiless(); + + var redirect_buffer: [2 * 1024]u8 = undefined; + var res = try req.receiveHead(&redirect_buffer); + + if (res.head.status != .ok) { + log.err("Failed to list object {f}", .{uri}); + log.err("{s}", .{res.head.bytes}); + return error.RequestFailed; + } + + return if (res.head.content_length) |content_len| + try res.reader(&.{}).readAlloc(self.allocator, content_len) + else + try res.reader(&.{}).allocRemaining(self.allocator, .limited(1024 * 1024)); + } + + fn listObjects(self: *GCS, prefix: []const u8) ![][]const u8 { + const bucket, const key_prefix = self.pathComponents(prefix); + const body = try self.listObjectsBody(bucket, key_prefix); + defer self.allocator.free(body); + + return try self.parseListObjectsResponse(body, key_prefix); + } + + fn gcsEncodeIsValid(c: u8) bool { + return switch (c) { + 'A'...'Z', 'a'...'z', '0'...'9', '-', '.', '_', '~' => true, + else => false, + }; + } + + fn quotaProjectId(self: *const GCS) ?[]const u8 { + const credentials = self.config.credentials orelse return null; + return switch (credentials) { + .authorized_user => |authorized_user| authorized_user.quota_project_id, + .service_account => |service_account| service_account.quota_project_id, + .metadata_server => null, + }; + } + + fn parseListObjectsResponse(self: *GCS, xml: []const u8, prefix: []const u8) ![][]const u8 { + var results: std.ArrayListUnmanaged([]const u8) = .empty; + errdefer { + for (results.items) |item| self.allocator.free(item); + results.deinit(self.allocator); + } + + inline for (.{ "Key", "Prefix" }) |tag| { + var pos: usize = 0; + while (std.mem.indexOfPos(u8, xml, pos, "<" ++ tag ++ ">")) |start| { + const v_start = start + tag.len + 2; + const end = std.mem.indexOfPos(u8, xml, v_start, "") orelse break; + const val = xml[v_start..end]; + + const is_self = std.mem.eql(u8, std.mem.trimEnd(u8, val, "/"), std.mem.trimEnd(u8, prefix, "/")); + if (val.len > 0 and !is_self) { + try results.append(self.allocator, try self.allocator.dupe(u8, val)); + } + pos = end + tag.len + 3; + } + } + + return try results.toOwnedSlice(self.allocator); + } + + fn fetchSize(self: *GCS, dir: std.Io.Dir, sub_path: []const u8) !u64 { + var path_buffer: [8 * 1024]u8 = undefined; + const path = try self.resolvePath(dir, sub_path, &path_buffer); + const uri = self.gcsUri(path); + var req = try self.client.request(.HEAD, uri, .{ + .redirect_behavior = .not_allowed, + .headers = .{ + .accept_encoding = .{ .override = "identity" }, + .authorization = try self.getOrRefreshToken(), + }, + .extra_headers = &.{}, + }); + defer req.deinit(); + + try req.sendBodiless(); + + var redirect_buffer: [8 * 1024]u8 = undefined; + const res = try req.receiveHead(&redirect_buffer); + + const size = switch (res.head.status.class()) { + .success => res.head.content_length.?, + else => switch (res.head.status) { + .not_found => return error.FileNotFound, + .unauthorized, .forbidden => return error.PermissionDenied, + else => blk: { + log.err("Failed to fetch size for {f}: {s}", .{ uri, res.head.bytes }); + break :blk error.ServerError; + }, + }, + }; + return size; + } + + fn performRead(self: *GCS, handle: *Handle, data: []const []u8, offset: u64) !usize { + const read_size = parallel_read.readSize(handle.size, offset, data); + if (read_size == 0) return 0; + + const uri = self.gcsUri(handle.uri); + const job_count = std.math.divCeil(usize, read_size, self.read_pool.chunk_size) catch unreachable; + const jobs = try self.allocator.alloc(ParallelRead.Job, job_count); + defer self.allocator.free(jobs); + const pending: u32 = @intCast(job_count); + + var batch: ParallelRead.Batch = .{ + .state = .{ .pending = .init(pending) }, + .backend = self, + .uri = uri, + .path = handle.uri, + .authorization = try self.getOrRefreshToken(), + }; + + for (jobs, 0..) |*job, i| { + const chunk_offset = i * self.read_pool.chunk_size; + job.* = .{ + .data = data, + .file_offset = offset + @as(u64, @intCast(chunk_offset)), + .chunk_offset = chunk_offset, + .chunk_len = @min(self.read_pool.chunk_size, read_size - chunk_offset), + .batch = &batch, + }; + } + + try self.read_pool.job_queue.putAll(self.base.inner, jobs); + batch.state.waitUncancelable(self.base.inner); + if (batch.state.anyError()) |err| return err; + + return read_size; + } +}; + +test "GCS parses XML bucket listing objects and common prefixes" { + const xml = + "" ++ + "" ++ + "bucket" ++ + "LC08/01/001/002/" ++ + "LC08/01/001/002/scene_$folder$6" ++ + "LC08/01/001/002/scene/" ++ + "LC08/01/001/002/" ++ + ""; + + var backend: GCS = undefined; + backend.allocator = std.testing.allocator; + + const objects = try backend.parseListObjectsResponse(xml, "LC08/01/001/002/"); + defer { + for (objects) |object| std.testing.allocator.free(object); + std.testing.allocator.free(objects); + } + + try std.testing.expectEqual(@as(usize, 2), objects.len); + try std.testing.expectEqualStrings("LC08/01/001/002/scene_$folder$", objects[0]); + try std.testing.expectEqualStrings("LC08/01/001/002/scene/", objects[1]); +} + +test "GCS splits bucket and object prefix" { + var backend: GCS = undefined; + + const root_bucket, const root_prefix = backend.pathComponents("gcp-public-data-landsat/"); + try std.testing.expectEqualStrings("gcp-public-data-landsat", root_bucket); + try std.testing.expectEqualStrings("", root_prefix); + + const bucket, const prefix = backend.pathComponents("gcp-public-data-landsat/LC08/01"); + try std.testing.expectEqualStrings("gcp-public-data-landsat", bucket); + try std.testing.expectEqualStrings("LC08/01", prefix); +} diff --git a/vfs/hf.zig b/vfs/hf.zig new file mode 100644 index 00000000..8770fe18 --- /dev/null +++ b/vfs/hf.zig @@ -0,0 +1,920 @@ +const std = @import("std"); + +const stdx = @import("stdx"); + +const parallel_read = @import("parallel_read.zig"); +const VFSBase = @import("base.zig").VFSBase; + +const log = std.log.scoped(.@"zml/vfs/hf"); + +const ParallelRead = struct { + const Pool = parallel_read.Pool(Job); + + const Batch = struct { + state: parallel_read.BatchState, + uri: std.Uri, + url: []const u8, + authorization: std.http.Client.Request.Headers.Value, + }; + + pub const Job = struct { + data: []const []u8, + file_offset: u64, + chunk_offset: usize, + chunk_len: usize, + batch: *Batch, + + pub fn perform(job: Job, client: *std.http.Client) anyerror!parallel_read.Status { + var range_buf: [64]u8 = undefined; + const range_header = std.fmt.bufPrint( + &range_buf, + "bytes={d}-{d}", + .{ job.file_offset, job.file_offset + @as(u64, @intCast(job.chunk_len - 1)) }, + ) catch unreachable; + + var req = client.request(.GET, job.batch.uri, .{ + .headers = .{ + .accept_encoding = .{ .override = "identity" }, + .authorization = job.batch.authorization, + }, + .extra_headers = &.{.{ .name = "Range", .value = range_header }}, + }) catch |err| { + switch (err) { + // transient connect failures + error.ConnectionRefused, + error.ConnectionResetByPeer, + error.HostUnreachable, + error.NetworkUnreachable, + error.NetworkDown, + error.Timeout, + error.NameServerFailure, + => { + log.warn("Failed to connect: {}", .{err}); + return .retry(); + }, + else => { + log.err("Failed to connect: {}", .{err}); + return err; + }, + } + }; + defer req.deinit(); + + req.sendBodiless() catch |err| switch (err) { + error.WriteFailed => { + log.warn("Failed to send headers: {}", .{err}); + return .retry(); + }, + }; + + var redirect_buffer: [8 * 1024]u8 = undefined; + var res = req.receiveHead(&redirect_buffer) catch |err| { + switch (err) { + // stale keep-alive / peer closed while waiting for response head + error.HttpConnectionClosing, + error.HttpRequestTruncated, + + // transport read/write failure; retry on a fresh connection + error.ReadFailed, + error.WriteFailed, + + // transient connect failures + error.ConnectionRefused, + error.ConnectionResetByPeer, + error.HostUnreachable, + error.NetworkUnreachable, + error.NetworkDown, + error.Timeout, + error.NameServerFailure, + => { + log.warn("Failed to receive headers: {}", .{err}); + return .retry(); + }, + else => { + log.err("Failed to receive headers: {}", .{err}); + return err; + }, + } + }; + + if (res.head.status != .partial_content and res.head.status != .ok) { + const status: parallel_read.Status = switch (res.head.status) { + .request_timeout => .retry(), + .too_many_requests => b: { + var it = res.head.iterateHeaders(); + while (it.next()) |header| { + if (std.ascii.eqlIgnoreCase(header.name, "RateLimit")) { + var parts = std.mem.splitScalar(u8, header.value, ';'); + while (parts.next()) |part| { + const p = std.mem.trim(u8, part, " \t"); + if (std.mem.startsWith(u8, p, "t=")) { + const seconds = std.fmt.parseInt(u32, p[2..], 10) catch continue; + break :b .{ .retry_after = .fromSeconds(seconds) }; + } + } + } + } + break :b .retry(); + }, + else => if (res.head.status.class() == .server_error) .retry() else { + log.err("Failed to perform read for {s}\n{s}", .{ job.batch.url, res.head.bytes }); + return error.RequestFailed; + }, + }; + log.warn("Failed to perform read for {s}\n{s}", .{ job.batch.url, res.head.bytes }); + return status; + } + + const content_range = blk: { + var it = res.head.iterateHeaders(); + while (it.next()) |header| { + if (std.ascii.eqlIgnoreCase(header.name, "Content-Range")) { + break :blk parallel_read.parseContentRange(header.value); + } + } + break :blk null; + }; + + const reader = res.reader(&.{}); + parallel_read.readChunk(reader, content_range, job.file_offset, job.data, job.chunk_offset, job.chunk_len) catch |err| { + switch (err) { + error.EndOfStream, + error.ReadFailed, + => { + log.warn("Failed to read from response: {}", .{err}); + return .retry(); + }, + else => { + log.err("Failed to read from response: {}", .{err}); + return err; + }, + } + }; + + return .success; + } + }; +}; + +pub const API = struct { + const TREE_URL_TEMPLATE = "https://huggingface.co/api/models/{[repo]s}/{[model]s}/tree/{[rev]s}/{[path]s}?expand=false&recursive=true&limit=1000"; + const LFS_FILE_URL_TEMPLATE = "https://huggingface.co/{[repo]s}/{[model]s}/resolve/{[rev]s}/{[path]s}"; + + pub const Tree = struct { + oid: []const u8, + path: []const u8, + type: []const u8, + size: u64, + }; +}; + +const ReadState = struct { children: []const TreeNode, index: usize }; + +pub const TreeNode = struct { + name: []const u8, + kind: std.Io.File.Kind, + size: u64, + children: ?std.ArrayList(TreeNode), + + fn deinit(self: *TreeNode, allocator: std.mem.Allocator) void { + allocator.free(self.name); + if (self.children) |*children| { + for (children.items) |*child| child.deinit(allocator); + children.deinit(allocator); + } + } +}; + +const RepoKey = struct { + repo: []const u8, + model: []const u8, + rev: []const u8, + + fn format(self: RepoKey, buffer: []u8) ![]u8 { + return std.fmt.bufPrint(buffer, "{s}/{s}@{s}", .{ self.repo, self.model, self.rev }); + } + + fn allocFormat(self: RepoKey, allocator: std.mem.Allocator) ![]u8 { + return std.fmt.allocPrint(allocator, "{s}/{s}@{s}", .{ self.repo, self.model, self.rev }); + } +}; + +pub const HF = struct { + pub const InitOpts = struct { + read_pool: parallel_read.InitOpts = .{ + // Chunk size needs to be big enough to avoid hitting the rate limits of HF. + .chunk_size = 32 << 20, + .num_workers = 32, + .queue_capacity = 128, + .max_retries = 5, + .retry_initial_delay = .fromMilliseconds(500), + .retry_max_delay = .fromSeconds(30), + }, + }; + + pub const Repo = struct { + repo: []const u8, + model: []const u8, + rev: []const u8, + path: []const u8, + + pub fn parse(uri: []const u8) !Repo { + var parts = std.mem.splitScalar(u8, uri, '/'); + var repo: Repo = .{ + .repo = parts.first(), + .model = parts.next() orelse return error.InvalidURI, + .rev = "main", + .path = std.mem.trimEnd(u8, parts.rest(), "/"), + }; + if (std.mem.findScalar(u8, repo.model, '@')) |at_index| { + repo.rev = repo.model[at_index + 1 ..]; + repo.model = repo.model[0..at_index]; + } + return repo; + } + + fn toKey(self: Repo) RepoKey { + return .{ .repo = self.repo, .model = self.model, .rev = self.rev }; + } + }; + + const Handle = struct { + pub const Type = enum { file, directory }; + + type: Type, + uri: []const u8, + pos: u64, + size: u64, + + pub fn init(allocator: std.mem.Allocator, handle_type: Type, uri: []const u8, size: u64) !Handle { + return .{ + .type = handle_type, + .uri = try allocator.dupe(u8, uri), + .pos = 0, + .size = size, + }; + } + + pub fn deinit(self: *Handle, allocator: std.mem.Allocator) void { + allocator.free(self.uri); + } + }; + + allocator: std.mem.Allocator, + mutex: std.Io.Mutex = .init, + client: *std.http.Client, + authorization: std.http.Client.Request.Headers.Value, + read_pool: *ParallelRead.Pool, + handles: stdx.SegmentedList(Handle, 0) = .{}, + closed_handles: std.ArrayList(u32) = .empty, + base: VFSBase, + trees: std.StringHashMapUnmanaged(std.ArrayList(TreeNode)) = .{}, + dir_read_states: std.AutoHashMapUnmanaged(*std.Io.Dir.Reader, ReadState) = .{}, + + pub fn init(allocator: std.mem.Allocator, inner: std.Io, http_client: *std.http.Client, hf_token: ?[]const u8, opts: InitOpts) !HF { + const read_pool = try allocator.create(ParallelRead.Pool); + errdefer allocator.destroy(read_pool); + + try read_pool.init(allocator, inner, http_client, opts.read_pool); + errdefer read_pool.deinit(allocator, inner); + + var self: HF = .{ + .allocator = allocator, + .base = .init(inner), + .client = http_client, + .authorization = if (hf_token) |token| blk: { + break :blk .{ + .override = try std.fmt.allocPrint(allocator, "Bearer {s}", .{std.mem.trim(u8, token, " \t\n\r")}), + }; + } else blk: { + break :blk .default; + }, + .read_pool = read_pool, + }; + errdefer switch (self.authorization) { + .default, .omit => {}, + .override => |t| self.allocator.free(t), + }; + + return self; + } + + pub fn auto(allocator: std.mem.Allocator, inner: std.Io, http_client: *std.http.Client, environ_map: *std.process.Environ.Map) !HF { + const hf_token = if (environ_map.get("HF_TOKEN")) |token| blk: { + break :blk try allocator.dupe(u8, token); + } else blk: { + const home_path = environ_map.get("HOME") orelse break :blk null; + + var path_buf: [256]u8 = undefined; + const token_path = std.fmt.bufPrint(&path_buf, "{s}/.cache/huggingface/token", .{home_path}) catch break :blk null; + + var file = std.Io.Dir.openFileAbsolute(inner, token_path, .{ .mode = .read_only }) catch break :blk null; + defer file.close(inner); + + const size = file.stat(inner) catch break :blk null; + var reader = file.reader(inner, &.{}); + const token = reader.interface.readAlloc(allocator, size.size) catch break :blk null; + + break :blk token; + }; + defer if (hf_token) |tk| allocator.free(tk); + + if (hf_token == null) log.warn("No Hugging Face authentication token found in environment or home config; proceeding without authentication.", .{}); + + return init(allocator, inner, http_client, hf_token, .{}); + } + + pub fn deinit(self: *HF) void { + self.read_pool.deinit(self.allocator, self.base.inner); + self.allocator.destroy(self.read_pool); + + var idx: usize = 0; + while (idx < self.handles.len) : (idx += 1) { + const is_closed = for (self.closed_handles.items) |closed_idx| { + if (closed_idx == idx) break true; + } else false; + + if (!is_closed) { + self.handles.at(idx).deinit(self.allocator); + } + } + self.handles.deinit(self.allocator); + self.closed_handles.deinit(self.allocator); + + var it = self.trees.iterator(); + while (it.next()) |entry| { + for (entry.value_ptr.items) |*node| { + node.deinit(self.allocator); + } + entry.value_ptr.deinit(self.allocator); + self.allocator.free(entry.key_ptr.*); + } + self.trees.deinit(self.allocator); + self.dir_read_states.deinit(self.allocator); + + switch (self.authorization) { + .default, .omit => {}, + .override => |t| self.allocator.free(t), + } + } + + pub fn io(self: *HF) std.Io { + return .{ + .userdata = &self.base, + .vtable = &comptime VFSBase.vtable(.{ + .operate = operate, + .dirOpenDir = dirOpenDir, + .dirStat = dirStat, + .dirStatFile = dirStatFile, + .dirAccess = dirAccess, + .dirOpenFile = dirOpenFile, + .dirClose = dirClose, + .dirRead = dirRead, + .dirRealPath = dirRealPath, + .dirRealPathFile = dirRealPathFile, + .fileStat = fileStat, + .fileLength = fileLength, + .fileClose = fileClose, + .fileReadPositional = fileReadPositional, + .fileSeekBy = fileSeekBy, + .fileSeekTo = fileSeekTo, + .fileRealPath = fileRealPath, + }), + }; + } + + fn openHandle(self: *HF) !struct { u32, *Handle } { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + + if (self.closed_handles.pop()) |idx| { + return .{ idx, self.handles.at(idx) }; + } + const len: u32 = @intCast(self.handles.len); + return .{ len, try self.handles.addOne(self.allocator) }; + } + + fn closeHandle(self: *HF, idx: u32) !void { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + + self.handles.at(idx).deinit(self.allocator); + try self.closed_handles.append(self.allocator, idx); + } + + fn getFileHandle(self: *HF, file: std.Io.File) *Handle { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + return self.handles.at(@intCast(file.handle)); + } + + fn getDirHandle(self: *HF, dir: std.Io.Dir) *Handle { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + return self.handles.at(@intCast(dir.handle)); + } + + fn resolvePath(self: *HF, dir: std.Io.Dir, sub_path: []const u8, out_buffer: []u8) ![]u8 { + if (std.meta.eql(dir, std.Io.Dir.cwd())) { + return try std.fmt.bufPrint(out_buffer, "{s}", .{sub_path}); + } + + const handle = self.getDirHandle(dir); + + const trimmed_uri = std.mem.trimEnd(u8, handle.uri, "/"); + const trimmed_sub_path = std.mem.trimStart(u8, sub_path, "/"); + + if (trimmed_uri.len == 0) return try std.fmt.bufPrint(out_buffer, "{s}", .{trimmed_sub_path}); + if (trimmed_sub_path.len == 0) return try std.fmt.bufPrint(out_buffer, "{s}", .{trimmed_uri}); + return try std.fmt.bufPrint(out_buffer, "{s}/{s}", .{ trimmed_uri, trimmed_sub_path }); + } + + fn getOrFetchTree(self: *HF, repo: Repo) !*std.ArrayList(TreeNode) { + const key = repo.toKey(); + + var cache_key_buffer: [512]u8 = undefined; + + const cache_key_lookup = try key.format(&cache_key_buffer); + + if (self.trees.getPtr(cache_key_lookup)) |tree_ptr| return tree_ptr; + + const cache_key = try key.allocFormat(self.allocator); + errdefer self.allocator.free(cache_key); + + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + + const gop = try self.trees.getOrPut(self.allocator, cache_key); + errdefer _ = self.trees.remove(cache_key); + + if (gop.found_existing) { + self.allocator.free(cache_key); + return gop.value_ptr; + } + + const tree_root = try self.fetchTreeFromAPI(repo); + gop.value_ptr.* = tree_root; + + return gop.value_ptr; + } + + fn fetchTreeFromAPI(self: *HF, repo: Repo) !std.ArrayList(TreeNode) { + var url_buffer: [8 * 1024]u8 = undefined; + var redirect_buffer: [8 * 1024]u8 = undefined; + + const url = try std.fmt.bufPrint(&url_buffer, API.TREE_URL_TEMPLATE, .{ + .repo = repo.repo, + .model = repo.model, + .rev = repo.rev, + .path = "", + }); + const uri = try std.Uri.parse(url); + log.info("Fetching from HF: {s}", .{url}); + + var req = try self.client.request(.GET, uri, .{ + .headers = .{ + .accept_encoding = .{ .override = "identity" }, + .authorization = self.authorization, + }, + }); + defer req.deinit(); + + try req.sendBodiless(); + + var res = try req.receiveHead(&redirect_buffer); + + if (res.head.status != .ok) { + log.err("Failed to fetch tree: status={}", .{res.head.status}); + return error.RequestFailed; + } + + const body = try res.reader(&.{}).readAlloc(self.allocator, res.head.content_length.?); + defer self.allocator.free(body); + + const parsed = try std.json.parseFromSlice( + []API.Tree, + self.allocator, + body, + .{ .ignore_unknown_fields = true }, + ); + defer parsed.deinit(); + + var tree_root: std.ArrayList(TreeNode) = .empty; + for (parsed.value) |item| { + try insertTreeNode(self.allocator, &tree_root, item); + } + + return tree_root; + } + + fn findDirChildren(tree_items: []const TreeNode, dir_path: []const u8) ?[]const TreeNode { + if (dir_path.len == 0) { + return tree_items; + } + + var parts = std.mem.tokenizeScalar(u8, dir_path, '/'); + var current = tree_items; + + while (parts.next()) |part| { + var found: ?*const TreeNode = null; + for (current) |*node| { + if (std.mem.eql(u8, node.name, part)) { + found = node; + break; + } + } + + if (found) |node| { + if (node.children) |children| { + current = children.items; + } else { + return null; + } + } else { + return null; + } + } + + return current; + } + + fn findNode(tree_items: []const TreeNode, path: []const u8) ?*const TreeNode { + if (path.len == 0) return null; + + var parts = std.mem.tokenizeScalar(u8, path, '/'); + var current = tree_items; + + while (parts.next()) |part| { + const is_last = parts.peek() == null; + + var found: ?*const TreeNode = null; + for (current) |*node| { + if (std.mem.eql(u8, node.name, part)) { + found = node; + break; + } + } + + if (found) |node| { + if (is_last) return node; + if (node.children) |children| { + current = children.items; + } else { + return null; + } + } else { + return null; + } + } + + return null; + } + + fn getSizeFromTree(tree: *std.ArrayList(TreeNode), path: []const u8) ?u64 { + if (path.len == 0) { + var total: u64 = 0; + for (tree.items) |*node| { + if (node.kind == .file) total += node.size; + } + return total; + } + + const node = findNode(tree.items, path) orelse return null; + return node.size; + } + + fn insertTreeNode(allocator: std.mem.Allocator, root: *std.ArrayList(TreeNode), item: API.Tree) !void { + var parts = std.mem.tokenizeScalar(u8, item.path, '/'); + var current_list = root; + + var parents: std.ArrayList(*TreeNode) = .empty; + defer parents.deinit(allocator); + + while (true) { + const part = parts.next() orelse break; + const is_last = parts.peek() == null; + + var found: ?*TreeNode = null; + for (current_list.items) |*node| { + if (std.mem.eql(u8, node.name, part)) { + found = node; + break; + } + } + + if (found) |f| { + if (!is_last) { + try parents.append(allocator, f); + if (f.children == null) { + f.children = .empty; + } + current_list = &f.children.?; + } else if (std.mem.eql(u8, item.type, "file")) { + f.size = item.size; + } + } else { + const is_file = is_last and std.mem.eql(u8, item.type, "file"); + const new_node: TreeNode = .{ + .name = try allocator.dupe(u8, part), + .kind = if (is_file) .file else .directory, + .size = if (is_last) item.size else 0, + .children = if (is_file) null else .empty, + }; + try current_list.append(allocator, new_node); + + if (!is_last) { + const new_node_ptr = ¤t_list.items[current_list.items.len - 1]; + try parents.append(allocator, new_node_ptr); + current_list = &new_node_ptr.children.?; + } + } + } + + for (parents.items) |parent| { + parent.size += item.size; + } + } + + fn operate(userdata: ?*anyopaque, operation: std.Io.Operation) std.Io.Cancelable!std.Io.Operation.Result { + const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); + switch (operation) { + .file_read_streaming => |o| { + const handle = self.getFileHandle(o.file); + const total = self.performRead(handle, o.data, handle.pos) catch |err| { + log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, handle.pos, err }); + return .{ .file_read_streaming = error.EndOfStream }; + }; + + if (total == 0) { + return .{ .file_read_streaming = error.EndOfStream }; + } + + handle.pos += @intCast(total); + return .{ .file_read_streaming = total }; + }, + .file_write_streaming, .device_io_control, .net_receive, .net_read, .net_send, .net_write => { + return self.base.inner.vtable.operate(self.base.inner.userdata, operation); + }, + } + } + + fn dirOpenDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.OpenOptions) std.Io.Dir.OpenError!std.Io.Dir { + const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); + + var path_buffer: [8 * 1024]u8 = undefined; + const full_path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.Dir.OpenError.SystemResources; + + const repo = Repo.parse(full_path) catch return std.Io.Dir.OpenError.BadPathName; + const tree = self.getOrFetchTree(repo) catch return std.Io.Dir.OpenError.Unexpected; + const size = getSizeFromTree(tree, repo.path) orelse return std.Io.Dir.OpenError.FileNotFound; + + const idx, const handle = self.openHandle() catch return std.Io.Dir.OpenError.Unexpected; + handle.* = Handle.init(self.allocator, .directory, full_path, size) catch return std.Io.Dir.OpenError.Unexpected; + + return .{ .handle = @intCast(idx) }; + } + + fn dirStat(userdata: ?*anyopaque, dir: std.Io.Dir) std.Io.Dir.StatError!std.Io.Dir.Stat { + const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getDirHandle(dir); + + return .{ + .inode = @intCast(dir.handle), + .nlink = 0, + .size = handle.size, + .permissions = .fromMode(0o444), + .kind = .directory, + .atime = null, + .mtime = std.Io.Timestamp.zero, + .ctime = std.Io.Timestamp.zero, + .block_size = 1, + }; + } + + fn dirStatFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.StatFileOptions) std.Io.Dir.StatFileError!std.Io.File.Stat { + const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); + + var path_buffer: [8 * 1024]u8 = undefined; + const full_path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.Dir.StatFileError.SystemResources; + + const repo = Repo.parse(full_path) catch return std.Io.Dir.StatFileError.Unexpected; + const tree = self.getOrFetchTree(repo) catch return std.Io.Dir.StatFileError.Unexpected; + const node = findNode(tree.items, repo.path) orelse return std.Io.Dir.StatFileError.IsDir; + + if (node.kind == .directory) return std.Io.Dir.StatFileError.IsDir; + const size = getSizeFromTree(tree, repo.path) orelse return std.Io.Dir.StatFileError.FileNotFound; + + return .{ + .inode = @intCast(0), + .nlink = 0, + .size = size, + .permissions = .fromMode(0o444), + .kind = .file, + .atime = null, + .mtime = std.Io.Timestamp.zero, + .ctime = std.Io.Timestamp.zero, + .block_size = 1, + }; + } + + fn dirAccess(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void { + const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); + + var path_buffer: [8 * 1024]u8 = undefined; + const full_path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.Dir.AccessError.SystemResources; + + const repo = Repo.parse(full_path) catch return std.Io.Dir.AccessError.FileNotFound; + const tree = self.getOrFetchTree(repo) catch return std.Io.Dir.AccessError.Unexpected; + + if (repo.path.len == 0) return; + + const node = findNode(tree.items, repo.path); + if (node == null) return std.Io.Dir.AccessError.FileNotFound; + } + + fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!std.Io.File { + const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); + + var path_buffer: [8 * 1024]u8 = undefined; + const full_path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.File.OpenError.SystemResources; + + const repo = Repo.parse(full_path) catch return std.Io.File.OpenError.BadPathName; + const tree = self.getOrFetchTree(repo) catch return std.Io.File.OpenError.Unexpected; + const size = getSizeFromTree(tree, repo.path) orelse return std.Io.File.OpenError.FileNotFound; + + const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; + handle.* = Handle.init(self.allocator, .file, full_path, size) catch return std.Io.File.OpenError.Unexpected; + + return .{ .handle = @intCast(idx), .flags = .{ .nonblocking = false } }; + } + + fn dirClose(userdata: ?*anyopaque, dirs: []const std.Io.Dir) void { + const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); + for (dirs) |d| { + self.closeHandle(@intCast(d.handle)) catch {}; + } + } + + fn dirRead(userdata: ?*anyopaque, reader: *std.Io.Dir.Reader, entries: []std.Io.Dir.Entry) std.Io.Dir.Reader.Error!usize { + const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); + + if (reader.state == .finished) return 0; + + if (reader.state == .reset) { + _ = self.dir_read_states.remove(reader); + + const handle = self.getDirHandle(reader.dir); + const repo = Repo.parse(handle.uri) catch return std.Io.Dir.Reader.Error.Unexpected; + const tree = self.getOrFetchTree(repo) catch return std.Io.Dir.Reader.Error.Unexpected; + + const children = findDirChildren(tree.items, repo.path) orelse &.{}; + + self.dir_read_states.put(self.allocator, reader, .{ + .children = children, + .index = 0, + }) catch return std.Io.Dir.Reader.Error.Unexpected; + + reader.state = if (children.len > 0) .reading else .finished; + if (children.len == 0) return 0; + } + + const state = self.dir_read_states.getPtr(reader) orelse return std.Io.Dir.Reader.Error.Unexpected; + + var count: usize = 0; + while (count < entries.len and state.index < state.children.len) { + const node = &state.children[state.index]; + entries[count] = .{ + .name = node.name, + .kind = node.kind, + .inode = state.index, + }; + count += 1; + state.index += 1; + } + + if (state.index >= state.children.len) { + reader.state = .finished; + _ = self.dir_read_states.remove(reader); + } + + return count; + } + + fn dirRealPath(userdata: ?*anyopaque, dir: std.Io.Dir, out_buffer: []u8) std.Io.Dir.RealPathError!usize { + const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getDirHandle(dir); + const path = std.fmt.bufPrint(out_buffer, "{s}", .{handle.uri}) catch return std.Io.Dir.RealPathError.SystemResources; + return path.len; + } + + fn dirRealPathFile(userdata: ?*anyopaque, dir: std.Io.Dir, path_name: []const u8, out_buffer: []u8) std.Io.Dir.RealPathFileError!usize { + const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); + const real_path = self.resolvePath(dir, path_name, out_buffer) catch return std.Io.Dir.RealPathFileError.NameTooLong; + return real_path.len; + } + + fn fileStat(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.StatError!std.Io.File.Stat { + const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + + return .{ + .inode = @intCast(file.handle), + .nlink = 0, + .size = handle.size, + .permissions = .fromMode(0o444), + .kind = .file, + .atime = null, + .mtime = std.Io.Timestamp.zero, + .ctime = std.Io.Timestamp.zero, + .block_size = 1, + }; + } + + fn fileLength(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.LengthError!u64 { + const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); + return self.getFileHandle(file).size; + } + + fn fileClose(userdata: ?*anyopaque, files: []const std.Io.File) void { + const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); + for (files) |file| { + self.closeHandle(@intCast(file.handle)) catch unreachable; + } + } + + fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { + const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + return self.performRead(handle, data, offset) catch |err| { + log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, offset, err }); + return std.Io.File.ReadPositionalError.Unexpected; + }; + } + + fn fileSeekBy(userdata: ?*anyopaque, file: std.Io.File, relative_offset: i64) std.Io.File.SeekError!void { + const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + + handle.pos = if (relative_offset >= 0) + handle.pos + @as(u64, @intCast(relative_offset)) + else + handle.pos - @as(u64, @intCast(-relative_offset)); + } + + fn fileSeekTo(userdata: ?*anyopaque, file: std.Io.File, absolute_offset: u64) std.Io.File.SeekError!void { + const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + handle.pos = absolute_offset; + } + + fn fileRealPath(userdata: ?*anyopaque, file: std.Io.File, out_buffer: []u8) std.Io.File.RealPathError!usize { + const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + const path = std.fmt.bufPrint(out_buffer, "{s}", .{handle.uri}) catch return std.Io.File.RealPathError.SystemResources; + return path.len; + } + + fn performRead(self: *HF, handle: *Handle, data: []const []u8, offset: u64) !usize { + if (offset >= handle.size) return 0; + + var read_size: usize = 0; + for (data) |buf| read_size += buf.len; + read_size = @intCast(@min(handle.size - offset, read_size)); + if (read_size == 0) return 0; + + const job_count = std.math.divCeil(usize, read_size, self.read_pool.chunk_size) catch unreachable; + + const jobs = try self.allocator.alloc(ParallelRead.Job, job_count); + defer self.allocator.free(jobs); + const pending: u32 = @intCast(job_count); + + const repo: Repo = try .parse(handle.uri); + var url_buffer: [8 * 1024]u8 = undefined; + const url = try std.fmt.bufPrint(&url_buffer, API.LFS_FILE_URL_TEMPLATE, repo); + const uri: std.Uri = try .parse(url); + + var batch: ParallelRead.Batch = .{ + .state = .{ .pending = .init(pending) }, + .uri = uri, + .url = url, + .authorization = self.authorization, + }; + + for (jobs, 0..) |*job, i| { + const chunk_offset = i * self.read_pool.chunk_size; + job.* = .{ + .data = data, + .file_offset = offset + @as(u64, @intCast(chunk_offset)), + .chunk_offset = chunk_offset, + .chunk_len = @min(self.read_pool.chunk_size, read_size - chunk_offset), + .batch = &batch, + }; + } + + try self.read_pool.job_queue.putAll(self.base.inner, jobs); + batch.state.waitUncancelable(self.base.inner); + if (batch.state.anyError()) |err| return err; + + return read_size; + } +}; diff --git a/vfs/http.zig b/vfs/http.zig new file mode 100644 index 00000000..9054983e --- /dev/null +++ b/vfs/http.zig @@ -0,0 +1,427 @@ +const std = @import("std"); + +const stdx = @import("stdx"); + +const VFSBase = @import("base.zig").VFSBase; + +const log = std.log.scoped(.@"zml/vfs/http"); + +pub const HTTP = struct { + const Handle = struct { + pub const Type = enum { + file, + directory, + }; + + type: Type, + uri: []const u8, + pos: u64, + size: u64, + + pub fn init(allocator: std.mem.Allocator, type_: Type, path: []const u8, size: u64) !Handle { + const uri = try allocator.dupe(u8, path); + errdefer allocator.free(uri); + + return .{ + .type = type_, + .uri = uri, + .pos = 0, + .size = size, + }; + } + + pub fn deinit(self: *Handle, allocator: std.mem.Allocator) void { + allocator.free(self.uri); + } + }; + + const Protocol = enum { http, https }; + + allocator: std.mem.Allocator, + mutex: std.Io.Mutex = .init, + client: *std.http.Client, + protocol: Protocol, + handles: stdx.SegmentedList(Handle, 0) = .{}, + closed_handles: std.ArrayList(u32) = .empty, + base: VFSBase, + + pub fn init(allocator: std.mem.Allocator, inner: std.Io, http_client: *std.http.Client, protocol: Protocol) !HTTP { + return .{ + .allocator = allocator, + .base = .init(inner), + .client = http_client, + .protocol = protocol, + }; + } + + pub fn deinit(self: *HTTP) void { + var idx: usize = 0; + while (idx < self.handles.len) : (idx += 1) { + const is_closed = for (self.closed_handles.items) |closed_idx| { + if (closed_idx == idx) break true; + } else false; + + if (!is_closed) { + self.handles.at(idx).deinit(self.allocator); + } + } + self.handles.deinit(self.allocator); + self.closed_handles.deinit(self.allocator); + } + + pub fn io(self: *HTTP) std.Io { + return .{ + .userdata = &self.base, + .vtable = &comptime VFSBase.vtable(.{ + .operate = operate, + .dirOpenDir = dirOpenDir, + .dirStat = dirStat, + .dirStatFile = dirStatFile, + .dirAccess = dirAccess, + .dirOpenFile = dirOpenFile, + .dirClose = dirClose, + .dirRead = dirRead, + .dirRealPath = dirRealPath, + .dirRealPathFile = dirRealPathFile, + .fileStat = fileStat, + .fileLength = fileLength, + .fileClose = fileClose, + .fileReadPositional = fileReadPositional, + .fileSeekBy = fileSeekBy, + .fileSeekTo = fileSeekTo, + .fileRealPath = fileRealPath, + }), + }; + } + + fn openHandle(self: *HTTP) !struct { u32, *Handle } { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + + if (self.closed_handles.pop()) |idx| { + return .{ idx, self.handles.at(idx) }; + } + return .{ @intCast(self.handles.len), try self.handles.addOne(self.allocator) }; + } + + fn closeHandle(self: *HTTP, idx: u32) !void { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + + self.handles.at(idx).deinit(self.allocator); + try self.closed_handles.append(self.allocator, idx); + } + + fn getFileHandle(self: *HTTP, file: std.Io.File) *Handle { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + return self.handles.at(@intCast(file.handle)); + } + + fn getDirHandle(self: *HTTP, dir: std.Io.Dir) *Handle { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + return self.handles.at(@intCast(dir.handle)); + } + + fn resolvePath(self: *HTTP, dir: std.Io.Dir, sub_path: []const u8, out_buffer: []u8) ![]u8 { + if (std.meta.eql(dir, std.Io.Dir.cwd())) { + return try std.fmt.bufPrint(out_buffer, "{s}", .{sub_path}); + } + + const handle = self.getDirHandle(dir); + + const trimmed_uri = std.mem.trimEnd(u8, handle.uri, "/"); + const trimmed_sub_path = std.mem.trimStart(u8, sub_path, "/"); + + if (trimmed_uri.len == 0) return try std.fmt.bufPrint(out_buffer, "{s}", .{trimmed_sub_path}); + if (trimmed_sub_path.len == 0) return try std.fmt.bufPrint(out_buffer, "{s}", .{trimmed_uri}); + return try std.fmt.bufPrint(out_buffer, "{s}/{s}", .{ trimmed_uri, trimmed_sub_path }); + } + + fn operate(userdata: ?*anyopaque, operation: std.Io.Operation) std.Io.Cancelable!std.Io.Operation.Result { + const self: *HTTP = @fieldParentPtr("base", VFSBase.as(userdata)); + switch (operation) { + .file_read_streaming => |o| { + const handle = self.getFileHandle(o.file); + const total = self.performRead(handle, o.data, handle.pos) catch |err| { + log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, handle.pos, err }); + return .{ .file_read_streaming = error.EndOfStream }; + }; + + if (total == 0) { + return .{ .file_read_streaming = error.EndOfStream }; + } + + handle.pos += @intCast(total); + return .{ .file_read_streaming = total }; + }, + .file_write_streaming, .device_io_control, .net_receive, .net_read, .net_send, .net_write => { + return self.base.inner.vtable.operate(self.base.inner.userdata, operation); + }, + } + } + + fn dirOpenDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.OpenOptions) std.Io.Dir.OpenError!std.Io.Dir { + const self: *HTTP = @fieldParentPtr("base", VFSBase.as(userdata)); + + var path_buffer: [8 * 1024]u8 = undefined; + const path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.Dir.OpenError.SystemResources; + + const idx, const handle = self.openHandle() catch return std.Io.Dir.OpenError.Unexpected; + handle.* = Handle.init(self.allocator, .directory, path, 0) catch return std.Io.Dir.OpenError.Unexpected; + + return .{ .handle = @intCast(idx) }; + } + + fn dirStat(userdata: ?*anyopaque, dir: std.Io.Dir) std.Io.Dir.StatError!std.Io.Dir.Stat { + const self: *HTTP = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getDirHandle(dir); + + return .{ + .inode = @intCast(0), + .nlink = 0, + .size = handle.size, + .permissions = .fromMode(0o444), + .kind = .directory, + .atime = null, + .mtime = std.Io.Timestamp.zero, + .ctime = std.Io.Timestamp.zero, + .block_size = 0, + }; + } + + fn dirStatFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.StatFileOptions) std.Io.Dir.StatFileError!std.Io.File.Stat { + const self: *HTTP = @fieldParentPtr("base", VFSBase.as(userdata)); + const size = self.fetchSize(dir, sub_path) catch return std.Io.Dir.StatFileError.Unexpected; + + return .{ + .inode = @intCast(0), + .nlink = 0, + .size = size, + .permissions = .fromMode(0o444), + .kind = .file, + .atime = null, + .mtime = std.Io.Timestamp.zero, + .ctime = std.Io.Timestamp.zero, + .block_size = 1, + }; + } + + fn dirAccess(_: ?*anyopaque, _: std.Io.Dir, _: []const u8, _: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void {} + + fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!std.Io.File { + const self: *HTTP = @fieldParentPtr("base", VFSBase.as(userdata)); + + const size = self.fetchSize(dir, sub_path) catch return std.Io.File.OpenError.Unexpected; + + var path_buffer: [8 * 1024]u8 = undefined; + const path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.File.OpenError.SystemResources; + + const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; + handle.* = Handle.init(self.allocator, .file, path, size) catch return std.Io.File.OpenError.Unexpected; + + return .{ .handle = @intCast(idx), .flags = .{ .nonblocking = false } }; + } + + fn dirClose(userdata: ?*anyopaque, dirs: []const std.Io.Dir) void { + const self: *HTTP = @fieldParentPtr("base", VFSBase.as(userdata)); + for (dirs) |dir| { + self.closeHandle(@intCast(dir.handle)) catch unreachable; + } + } + + fn dirRead(_: ?*anyopaque, _: *std.Io.Dir.Reader, _: []std.Io.Dir.Entry) std.Io.Dir.Reader.Error!usize { + log.err("dirRead unsupported", .{}); + return std.Io.Dir.Reader.Error.Unexpected; + } + + fn dirRealPath(userdata: ?*anyopaque, dir: std.Io.Dir, out_buffer: []u8) std.Io.Dir.RealPathError!usize { + const self: *HTTP = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getDirHandle(dir); + const path = std.fmt.bufPrint(out_buffer, "{s}", .{handle.uri}) catch return std.Io.Dir.RealPathError.SystemResources; + return path.len; + } + + fn dirRealPathFile(userdata: ?*anyopaque, dir: std.Io.Dir, path_name: []const u8, out_buffer: []u8) std.Io.Dir.RealPathFileError!usize { + const self: *HTTP = @fieldParentPtr("base", VFSBase.as(userdata)); + const real_path = self.resolvePath(dir, path_name, out_buffer) catch return std.Io.Dir.RealPathFileError.NameTooLong; + return real_path.len; + } + + fn fileStat(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.StatError!std.Io.File.Stat { + const self: *HTTP = @fieldParentPtr("base", VFSBase.as(userdata)); + + const handle = self.getFileHandle(file); + + return .{ + .inode = @intCast(file.handle), + .nlink = 0, + .size = handle.size, + .permissions = .fromMode(0o444), + .kind = .file, + .atime = null, + .mtime = std.Io.Timestamp.zero, + .ctime = std.Io.Timestamp.zero, + .block_size = 1, + }; + } + + fn fileLength(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.LengthError!u64 { + const self: *HTTP = @fieldParentPtr("base", VFSBase.as(userdata)); + return self.getFileHandle(file).size; + } + + fn fileClose(userdata: ?*anyopaque, files: []const std.Io.File) void { + const self: *HTTP = @fieldParentPtr("base", VFSBase.as(userdata)); + for (files) |file| { + self.closeHandle(@intCast(file.handle)) catch unreachable; + } + } + + fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { + const self: *HTTP = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + return self.performRead(handle, data, offset) catch |err| { + log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, offset, err }); + return std.Io.File.ReadPositionalError.Unexpected; + }; + } + + fn fileSeekBy(userdata: ?*anyopaque, file: std.Io.File, relative_offset: i64) std.Io.File.SeekError!void { + const self: *HTTP = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + + handle.pos = if (relative_offset >= 0) + handle.pos + @as(u64, @intCast(relative_offset)) + else + handle.pos - @as(u64, @intCast(-relative_offset)); + } + + fn fileSeekTo(userdata: ?*anyopaque, file: std.Io.File, absolute_offset: u64) std.Io.File.SeekError!void { + const self: *HTTP = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + handle.pos = absolute_offset; + } + + fn fileRealPath(userdata: ?*anyopaque, file: std.Io.File, out_buffer: []u8) std.Io.File.RealPathError!usize { + const self: *HTTP = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + const path = std.fmt.bufPrint(out_buffer, "{s}", .{handle.uri}) catch return std.Io.File.RealPathError.SystemResources; + return path.len; + } + + fn fetchSize(self: *HTTP, dir: std.Io.Dir, sub_path: []const u8) !u64 { + var path_buffer: [8 * 1024]u8 = undefined; + var redirect_buffer: [8 * 1024]u8 = undefined; + var aux_buffer: []u8 = &redirect_buffer; + + const scheme = @tagName(self.protocol); + const url = try self.resolvePath(dir, sub_path, &path_buffer); + const full_url = try std.fmt.bufPrint(aux_buffer, "{s}://{s}", .{ scheme, url }); + + var uri = std.Uri.parse(full_url) catch return std.Io.File.OpenError.BadPathName; + while (true) { + var req = try self.client.request(.HEAD, uri, .{ + .redirect_behavior = .not_allowed, + .headers = .{ .accept_encoding = .{ .override = "identity" } }, + }); + defer req.deinit(); + + try req.sendBodiless(); + + var res = try req.receiveHead(&redirect_buffer); + + switch (res.head.status.class()) { + .server_error, .client_error => { + log.err("Failed to fetch tree size for {s}", .{url}); + log.err("{s}", .{res.head.bytes}); + return error.ServerError; + }, + .informational => return error.UnexpectedStatus, + .success => return res.head.content_length.?, + .redirect => { + const location = res.head.location.?; + @memcpy(aux_buffer[0..location.len], location); + uri = uri.resolveInPlace(location.len, &aux_buffer) catch unreachable; + continue; + }, + } + } + } + + fn performRead(self: *HTTP, handle: *Handle, data: []const []u8, offset: u64) !usize { + if (offset >= handle.size) return 0; + + var range_buf: [64]u8 = undefined; + const range_header = blk: { + var total_bytes: u64 = 0; + for (data) |buf| { + total_bytes += @as(u64, buf.len); + } + const remaining = handle.size - offset; + const take = @min(remaining, total_bytes); + const end = offset + take - 1; + break :blk std.fmt.bufPrint(&range_buf, "bytes={d}-{d}", .{ offset, end }) catch unreachable; + }; + + var url_buffer: [8 * 1024]u8 = undefined; + const url = try std.fmt.bufPrint(&url_buffer, "{s}://{s}", .{ @tagName(self.protocol), handle.uri }); + const uri: std.Uri = try .parse(url); + + var req = try self.client.request(.GET, uri, .{ + .headers = .{ .accept_encoding = .{ .override = "identity" } }, + .extra_headers = &.{.{ .name = "Range", .value = range_header }}, + }); + defer req.deinit(); + + try req.sendBodiless(); + + var redirect_buffer: [8 * 1024]u8 = undefined; + var res = try req.receiveHead(&redirect_buffer); + + if (res.head.status != .partial_content and res.head.status != .ok) { + log.err("Failed to perform read for {s}", .{handle.uri}); + log.err("{s}", .{res.head.bytes}); + return error.RequestFailed; + } + + const content_range = blk: { + var it = res.head.iterateHeaders(); + while (it.next()) |header| { + if (std.ascii.eqlIgnoreCase(header.name, "Content-Range")) { + break :blk parseContentRange(header.value); + } + } + break :blk null; + }; + + const reader = res.reader(&.{}); + + if (content_range) |cr| { + if (cr.start < offset) { + try reader.discardAll(offset - cr.start); + } + } + + return try reader.readSliceShort(data[0]); + } + + const ContentRange = struct { + start: u64, + end: u64, + total: u64, + }; + + fn parseContentRange(value: []const u8) ?ContentRange { + const space = std.mem.indexOfScalar(u8, value, ' ') orelse return null; + const dash = std.mem.indexOfScalar(u8, value, '-') orelse return null; + const slash = std.mem.indexOfScalar(u8, value, '/') orelse return null; + + return .{ + .start = std.fmt.parseInt(u64, value[space + 1 .. dash], 10) catch return null, + .end = std.fmt.parseInt(u64, value[dash + 1 .. slash], 10) catch return null, + .total = std.fmt.parseInt(u64, value[slash + 1 ..], 10) catch return null, + }; + } +}; diff --git a/vfs/parallel_read.zig b/vfs/parallel_read.zig new file mode 100644 index 00000000..70c9d89d --- /dev/null +++ b/vfs/parallel_read.zig @@ -0,0 +1,211 @@ +const std = @import("std"); + +const stdx = @import("stdx"); + +pub const InitOpts = struct { + chunk_size: usize, + num_workers: usize, + queue_capacity: usize, + max_retries: usize, + retry_initial_delay: std.Io.Duration, + retry_max_delay: std.Io.Duration, +}; + +pub const Status = union(enum) { + success, + retry_after: ?std.Io.Duration, + + pub fn retry() Status { + return .{ .retry_after = null }; + } +}; + +pub const BatchState = struct { + pending: std.atomic.Value(u32), + err: std.atomic.Value(u16) = .init(0), + + pub fn failed(batch: *BatchState) bool { + return batch.err.load(.acquire) != 0; + } + + pub fn fail(batch: *BatchState, err: anyerror) void { + _ = batch.err.cmpxchgStrong(0, @intFromError(err), .release, .monotonic); + } + + pub fn anyError(batch: *BatchState) ?anyerror { + const err = batch.err.load(.acquire); + return if (err == 0) null else @errorFromInt(err); + } + + pub fn completeOne(batch: *BatchState, io: std.Io) void { + const previous = batch.pending.fetchSub(1, .release); + std.debug.assert(previous > 0); + if (previous == 1) io.futexWake(u32, &batch.pending.raw, 1); + } + + pub fn waitUncancelable(batch: *BatchState, io: std.Io) void { + while (true) { + const pending = batch.pending.load(.acquire); + if (pending == 0) return; + io.futexWaitUncancelable(u32, &batch.pending.raw, pending); + } + } +}; + +pub fn Pool(comptime Job: type) type { + return struct { + const Self = @This(); + + group: std.Io.Group = .init, + job_queue: std.Io.Queue(Job), + queue_buf: []Job, + client: *std.http.Client, + prng: std.Random.DefaultPrng, + chunk_size: usize, + max_retries: usize, + retry_initial_delay: std.Io.Duration, + retry_max_delay: std.Io.Duration, + + pub fn init(self: *Self, allocator: std.mem.Allocator, io: std.Io, client: *std.http.Client, opts: InitOpts) !void { + stdx.debug.assert(opts.num_workers > 0, "Pool must have at least one worker", .{}); + stdx.debug.assert(opts.chunk_size > 0, "Pool must have a positive download chunk size", .{}); + stdx.debug.assert(opts.queue_capacity > 0, "Pool must have a positive queue capacity", .{}); + stdx.debug.assert(opts.retry_initial_delay.nanoseconds >= 0, "Pool retry initial delay must not be negative", .{}); + stdx.debug.assert(opts.retry_max_delay.nanoseconds >= 0, "Pool retry max delay must not be negative", .{}); + stdx.debug.assert(opts.retry_max_delay.nanoseconds >= opts.retry_initial_delay.nanoseconds, "Pool retry max delay must be at least the initial delay", .{}); + + const queue_buf = try allocator.alloc(Job, opts.queue_capacity); + errdefer allocator.free(queue_buf); + + self.* = .{ + .client = client, + .chunk_size = opts.chunk_size, + .max_retries = opts.max_retries, + .retry_initial_delay = opts.retry_initial_delay, + .retry_max_delay = opts.retry_max_delay, + .prng = .init(blk: { + var seed: u64 = undefined; + io.random(@ptrCast(&seed)); + break :blk seed; + }), + .job_queue = .init(queue_buf), + .queue_buf = queue_buf, + }; + errdefer self.group.cancel(io); + + for (0..opts.num_workers) |_| { + try self.group.concurrent(io, worker, .{ self, io }); + } + } + + pub fn deinit(self: *Self, allocator: std.mem.Allocator, io: std.Io) void { + self.job_queue.close(io); + self.group.cancel(io); + + allocator.free(self.queue_buf); + } + + fn worker(pool: *Self, io: std.Io) std.Io.Cancelable!void { + while (true) { + const job = pool.job_queue.getOne(io) catch break; + + var attempt: usize = 0; + while (true) { + if (job.batch.state.failed()) break; + const status = job.perform(pool.client) catch |err| { + job.batch.state.fail(err); + break; + }; + + switch (status) { + .success => break, + .retry_after => |duration| { + if (attempt >= pool.max_retries) { + job.batch.state.fail(error.RetriesExhausted); + break; + } + + io.sleep(duration orelse pool.backoffDuration(attempt), .awake) catch { + job.batch.state.fail(error.RetriesExhausted); + break; + }; + + attempt += 1; + }, + } + continue; + } + + job.batch.state.completeOne(io); + } + } + + fn backoffDuration(pool: *Self, attempt: usize) std.Io.Duration { + // FullJitter algorithm + const max_delay_ns: i96 = @min( + pool.retry_max_delay.toNanoseconds(), + pool.retry_initial_delay.toNanoseconds() *| (@as(i96, 1) << @as(u7, @intCast(@min(attempt, std.math.maxInt(u7))))), + ); + const delay_ns = pool.prng.random().intRangeAtMost(i96, 0, max_delay_ns); + return .fromNanoseconds(delay_ns); + } + }; +} + +pub const ContentRange = struct { + start: u64, + end: u64, + total: u64, +}; + +pub fn parseContentRange(value: []const u8) ?ContentRange { + const space = std.mem.indexOfScalar(u8, value, ' ') orelse return null; + const dash = std.mem.indexOfScalar(u8, value, '-') orelse return null; + const slash = std.mem.indexOfScalar(u8, value, '/') orelse return null; + + return .{ + .start = std.fmt.parseInt(u64, value[space + 1 .. dash], 10) catch return null, + .end = std.fmt.parseInt(u64, value[dash + 1 .. slash], 10) catch return null, + .total = std.fmt.parseInt(u64, value[slash + 1 ..], 10) catch return null, + }; +} + +pub fn readSize(file_size: u64, offset: u64, data: []const []u8) usize { + if (offset >= file_size) return 0; + + var requested: usize = 0; + for (data) |buf| requested += buf.len; + return @intCast(@min(file_size - offset, requested)); +} + +pub fn readChunk( + reader: *std.Io.Reader, + content_range: ?ContentRange, + file_offset: u64, + data: []const []u8, + chunk_offset: usize, + chunk_len: usize, +) !void { + if (content_range) |cr| { + if (cr.start > file_offset) return error.InvalidContentRange; + if (cr.start < file_offset) { + try reader.discardAll(file_offset - cr.start); + } + } + + var remaining = chunk_len; + var skip = chunk_offset; + + for (data) |buf| { + if (skip >= buf.len) { + skip -= buf.len; + continue; + } + + const destination = buf[skip..][0..@min(remaining, buf.len - skip)]; + try reader.readSliceAll(destination); + remaining -= destination.len; + if (remaining == 0) break; + skip = 0; + } +} diff --git a/vfs/s3.zig b/vfs/s3.zig new file mode 100644 index 00000000..eb849e21 --- /dev/null +++ b/vfs/s3.zig @@ -0,0 +1,997 @@ +const std = @import("std"); + +const stdx = @import("stdx"); + +const parallel_read = @import("parallel_read.zig"); +const VFSBase = @import("base.zig").VFSBase; + +const log = std.log.scoped(.@"zml/vfs/s3"); + +pub const AwsSigV4 = struct { + access_key: ?[]const u8, + secret_key: ?[]const u8, + region: []const u8, + service: []const u8, + + const ALGORITHM = "AWS4-HMAC-SHA256"; + pub const UNSIGNED_PAYLOAD = "UNSIGNED-PAYLOAD"; + + pub fn generateAuthHeader( + self: AwsSigV4, + output_buffer: []u8, + method: std.http.Method, + uri: std.Uri, + timestamp: []const u8, + extra_headers: []const std.http.Header, + ) !?[]const u8 { + const access_key = self.access_key orelse return null; + _ = self.secret_key orelse return null; + + const date = timestamp[0..8]; + + var header_buf: [32]std.http.Header = undefined; + var header_count: usize = 0; + + const host = uri.host orelse return error.MissingHost; + const host_header: std.http.Header = blk: { + const host_ = switch (host) { + .raw, .percent_encoded => |h| h, + }; + if (uri.port) |port| { + var host_buf: [128]u8 = undefined; + break :blk .{ .name = "host", .value = try std.fmt.bufPrint(&host_buf, "{s}:{d}", .{ host_, port }) }; + } else { + break :blk .{ .name = "host", .value = host_ }; + } + }; + + header_buf[0] = host_header; + header_buf[1] = .{ .name = "x-amz-content-sha256", .value = UNSIGNED_PAYLOAD }; + header_buf[2] = .{ .name = "x-amz-date", .value = timestamp }; + header_count = 3; + + for (extra_headers) |h| { + if (header_count >= header_buf.len) return error.TooManyHeaders; + header_buf[header_count] = h; + header_count += 1; + } + + const active_headers = header_buf[0..header_count]; + + std.mem.sort(std.http.Header, active_headers, {}, struct { + fn less(_: void, a: std.http.Header, b: std.http.Header) bool { + return std.ascii.lessThanIgnoreCase(a.name, b.name); + } + }.less); + + var sha = std.crypto.hash.sha2.Sha256.init(.{}); + var sha_writer = stdx.crypto.hmacWriter(&sha); + const sw = &sha_writer.interface; + + try sw.print("{s}\n", .{@tagName(method)}); + if (uri.path.isEmpty()) { + try sw.writeAll("/\n"); + } else { + try uri.path.formatPath(sw); + try sw.writeByte('\n'); + } + + if (uri.query) |query| { + try query.formatEscaped(sw); + try sw.writeByte('\n'); + } else { + try sw.writeAll("\n"); + } + + var signed_headers_buf: [512]u8 = undefined; + var sh_writer = std.Io.Writer.fixed(&signed_headers_buf); + for (active_headers, 0..) |h, i| { + for (h.name) |char| try sw.writeByte(std.ascii.toLower(char)); + try sw.print(":{s}\n", .{h.value}); + if (i > 0) _ = try sh_writer.write(";"); + for (h.name) |char| try sh_writer.writeByte(std.ascii.toLower(char)); + } + const signed_headers = sh_writer.buffered(); + + try sw.print("\n{s}\n{s}", .{ signed_headers, UNSIGNED_PAYLOAD }); + + var canonical_request_hash: [32]u8 = undefined; + sha.final(&canonical_request_hash); + + var scope_buf: [128]u8 = undefined; + const scope = try std.fmt.bufPrint(&scope_buf, "{s}/{s}/{s}/aws4_request", .{ date, self.region, self.service }); + + const signing_key = try self.deriveSigningKey(date); + + var h_state = std.crypto.auth.hmac.sha2.HmacSha256.init(&signing_key); + var h_writer = stdx.crypto.hmacWriter(&h_state); + const hw = &h_writer.interface; + + try hw.print("{s}\n{s}\n{s}\n{x}", .{ ALGORITHM, timestamp, scope, &canonical_request_hash }); + + var signature: [32]u8 = undefined; + h_state.final(&signature); + + return try std.fmt.bufPrint(output_buffer, "{s} Credential={s}/{s}, SignedHeaders={s}, Signature={x}", .{ + ALGORITHM, + access_key, + scope, + signed_headers, + &signature, + }); + } + + fn deriveSigningKey(self: AwsSigV4, date: []const u8) ![32]u8 { + var key_buf: [128]u8 = undefined; + const k_init = try std.fmt.bufPrint(&key_buf, "AWS4{s}", .{self.secret_key.?}); + + const k_date = hmacStatic(k_init, date); + const k_region = hmacStatic(&k_date, self.region); + const k_service = hmacStatic(&k_region, self.service); + return hmacStatic(&k_service, "aws4_request"); + } + + fn hmacStatic(key: []const u8, data: []const u8) [32]u8 { + var out: [32]u8 = undefined; + std.crypto.auth.hmac.sha2.HmacSha256.create(&out, data, key); + return out; + } +}; + +const ReadState = struct { index: usize, objects: [][]const u8 }; + +pub const S3 = struct { + const ParallelRead = struct { + const Pool = parallel_read.Pool(Job); + + const Batch = struct { + state: parallel_read.BatchState, + backend: *S3, + uri: std.Uri, + url: []const u8, + }; + + const Job = struct { + data: []const []u8, + file_offset: u64, + chunk_offset: usize, + chunk_len: usize, + batch: *Batch, + + pub fn perform(job: Job, client: *std.http.Client) anyerror!parallel_read.Status { + var range_buf: [64]u8 = undefined; + const range_header = std.fmt.bufPrint( + &range_buf, + "bytes={d}-{d}", + .{ job.file_offset, job.file_offset + @as(u64, @intCast(job.chunk_len - 1)) }, + ) catch unreachable; + + var timestamp_buf: [16]u8 = undefined; + const timestamp = job.batch.backend.getTimestamp(×tamp_buf) catch unreachable; + + const signer: AwsSigV4 = .{ + .access_key = job.batch.backend.config.access_key, + .secret_key = job.batch.backend.config.secret_key, + .region = job.batch.backend.config.region, + .service = job.batch.backend.config.auth_service, + }; + + var authorization_buffer: [512]u8 = undefined; + const authorization = signer.generateAuthHeader( + &authorization_buffer, + .GET, + job.batch.uri, + timestamp, + &.{.{ .name = "Range", .value = range_header }}, + ) catch |err| { + log.err("Failed to generate auth header: {}", .{err}); + return err; + }; + + var req = client.request(.GET, job.batch.uri, .{ + .headers = .{ + .accept_encoding = .{ .override = "identity" }, + .authorization = if (authorization) |auth| .{ .override = auth } else .omit, + }, + .extra_headers = &.{ + .{ .name = "x-amz-date", .value = timestamp }, + .{ .name = "x-amz-content-sha256", .value = AwsSigV4.UNSIGNED_PAYLOAD }, + .{ .name = "Range", .value = range_header }, + }, + }) catch |err| { + switch (err) { + // transient connect failures + error.ConnectionRefused, + error.ConnectionResetByPeer, + error.HostUnreachable, + error.NetworkUnreachable, + error.NetworkDown, + error.Timeout, + error.NameServerFailure, + => { + log.warn("Failed to connect: {}", .{err}); + return .retry(); + }, + else => { + log.err("Failed to connect: {}", .{err}); + return err; + }, + } + }; + defer req.deinit(); + + req.sendBodiless() catch |err| switch (err) { + error.WriteFailed => { + log.warn("Failed to send headers: {}", .{err}); + return .retry(); + }, + }; + + var redirect_buffer: [8 * 1024]u8 = undefined; + var res = req.receiveHead(&redirect_buffer) catch |err| { + switch (err) { + // stale keep-alive / peer closed while waiting for response head + error.HttpConnectionClosing, + error.HttpRequestTruncated, + + // transport read/write failure; retry on a fresh connection + error.ReadFailed, + error.WriteFailed, + + // transient connect failures + error.ConnectionRefused, + error.ConnectionResetByPeer, + error.HostUnreachable, + error.NetworkUnreachable, + error.NetworkDown, + error.Timeout, + error.NameServerFailure, + => { + log.warn("Failed to receive headers: {}", .{err}); + return .retry(); + }, + + else => { + log.err("Failed to receive headers: {}", .{err}); + return err; + }, + } + }; + + if (res.head.status != .partial_content and res.head.status != .ok) { + const status: parallel_read.Status = switch (res.head.status) { + .request_timeout, .too_many_requests => .retry(), + else => if (res.head.status.class() == .server_error) .retry() else { + log.err("Failed to read {s}: {s}", .{ job.batch.url, res.head.bytes }); + return error.RequestFailed; + }, + }; + log.warn("Failed to read {s}: {s}", .{ job.batch.url, res.head.bytes }); + return status; + } + + const content_range = blk: { + var it = res.head.iterateHeaders(); + while (it.next()) |header| { + if (std.ascii.eqlIgnoreCase(header.name, "Content-Range")) { + break :blk parallel_read.parseContentRange(header.value); + } + } + break :blk null; + }; + + const reader = res.reader(&.{}); + parallel_read.readChunk(reader, content_range, job.file_offset, job.data, job.chunk_offset, job.chunk_len) catch |err| { + switch (err) { + error.EndOfStream, + error.ReadFailed, + => { + log.warn("Failed to read from response: {}", .{err}); + return .retry(); + }, + else => { + log.err("Failed to read from response: {}", .{err}); + return err; + }, + } + }; + + return .success; + } + }; + }; + + pub const InitOpts = struct { + read_pool: parallel_read.InitOpts = .{ + .chunk_size = 16 << 20, + .num_workers = 32, + .queue_capacity = 128, + .max_retries = 5, + .retry_initial_delay = .fromMilliseconds(500), + .retry_max_delay = .fromSeconds(30), + }, + }; + + pub const Config = struct { + access_key: ?[]const u8 = null, + secret_key: ?[]const u8 = null, + endpoint_url: []const u8, + region: []const u8, + auth_service: []const u8 = "s3", + }; + + const Handle = struct { + pub const Type = enum { file, directory }; + + type: Type, + uri: []const u8, + pos: u64, + size: u64, + + pub fn init(allocator: std.mem.Allocator, type_: Type, path: []const u8, size: u64) !Handle { + return .{ + .type = type_, + .uri = try allocator.dupe(u8, path), + .pos = 0, + .size = size, + }; + } + + pub fn deinit(self: *Handle, allocator: std.mem.Allocator) void { + allocator.free(self.uri); + } + }; + + allocator: std.mem.Allocator, + mutex: std.Io.Mutex = .init, + client: *std.http.Client, + config: Config, + read_pool: *ParallelRead.Pool, + handles: stdx.SegmentedList(Handle, 0) = .{}, + closed_handles: std.ArrayList(u32) = .empty, + dir_read_states: std.AutoHashMapUnmanaged(*std.Io.Dir.Reader, ReadState) = .{}, + base: VFSBase, + + pub fn init( + allocator: std.mem.Allocator, + inner: std.Io, + http_client: *std.http.Client, + config: Config, + opts: InitOpts, + ) !S3 { + const read_pool = try allocator.create(ParallelRead.Pool); + errdefer allocator.destroy(read_pool); + + try read_pool.init(allocator, inner, http_client, opts.read_pool); + errdefer read_pool.deinit(allocator, inner); + + return .{ + .allocator = allocator, + .base = .init(inner), + .client = http_client, + .read_pool = read_pool, + .config = .{ + .access_key = if (config.access_key) |k| try allocator.dupe(u8, k) else null, + .secret_key = if (config.secret_key) |k| try allocator.dupe(u8, k) else null, + .endpoint_url = try allocator.dupe(u8, config.endpoint_url), + .region = try allocator.dupe(u8, config.region), + .auth_service = try allocator.dupe(u8, config.auth_service), + }, + }; + } + + pub fn auto(allocator: std.mem.Allocator, inner: std.Io, http_client: *std.http.Client, environ_map: *std.process.Environ.Map) !S3 { + const access_key = environ_map.get("AWS_ACCESS_KEY_ID"); + const secret_key = environ_map.get("AWS_SECRET_ACCESS_KEY"); + + var endpoint_is_default = false; + const endpoint = environ_map.get("AWS_ENDPOINT_URL") orelse environ_map.get("AWS_ENDPOINT_URL_S3") orelse blk: { + endpoint_is_default = true; + break :blk "https://s3.amazonaws.com"; + }; + + var region_is_default = false; + const region = environ_map.get("AWS_REGION") orelse environ_map.get("AWS_DEFAULT_REGION") orelse blk: { + region_is_default = true; + break :blk "us-east-1"; + }; + + if (access_key == null or secret_key == null or endpoint_is_default or region_is_default) { + var buf: [512]u8 = undefined; + var writer = std.Io.Writer.fixed(&buf); + + if (access_key == null or secret_key == null) { + try writer.writeAll("S3 initialized without full credentials (AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY). Access will be unauthenticated."); + } else { + try writer.writeAll("S3 initialized with credentials."); + } + + if (endpoint_is_default) { + try writer.print(" Using default endpoint: {s} (set AWS_ENDPOINT_URL or AWS_ENDPOINT_URL_S3).", .{endpoint}); + } + + if (region_is_default) { + try writer.print(" Using default region: {s} (set AWS_REGION or AWS_DEFAULT_REGION).", .{region}); + } + + log.warn("{s}", .{writer.buffered()}); + } + + return init(allocator, inner, http_client, .{ + .access_key = access_key, + .secret_key = secret_key, + .endpoint_url = endpoint, + .region = region, + }, .{}); + } + + pub fn deinit(self: *S3) void { + self.read_pool.deinit(self.allocator, self.base.inner); + self.allocator.destroy(self.read_pool); + + var idx: usize = 0; + while (idx < self.handles.len) : (idx += 1) { + const is_closed = for (self.closed_handles.items) |closed_idx| { + if (closed_idx == idx) break true; + } else false; + + if (!is_closed) { + self.handles.at(idx).deinit(self.allocator); + } + } + self.handles.deinit(self.allocator); + self.closed_handles.deinit(self.allocator); + + var it = self.dir_read_states.iterator(); + while (it.next()) |entry| { + for (entry.value_ptr.objects) |obj| { + self.allocator.free(obj); + } + self.allocator.free(entry.value_ptr.objects); + } + self.dir_read_states.deinit(self.allocator); + + if (self.config.access_key) |access_key| self.allocator.free(access_key); + if (self.config.secret_key) |secret_key| self.allocator.free(secret_key); + self.allocator.free(self.config.endpoint_url); + self.allocator.free(self.config.region); + self.allocator.free(self.config.auth_service); + } + + pub fn io(self: *S3) std.Io { + return .{ + .userdata = &self.base, + .vtable = &comptime VFSBase.vtable(.{ + .operate = operate, + .dirOpenDir = dirOpenDir, + .dirStat = dirStat, + .dirStatFile = dirStatFile, + .dirAccess = dirAccess, + .dirOpenFile = dirOpenFile, + .dirClose = dirClose, + .dirRead = dirRead, + .dirRealPath = dirRealPath, + .dirRealPathFile = dirRealPathFile, + .fileStat = fileStat, + .fileLength = fileLength, + .fileClose = fileClose, + .fileReadPositional = fileReadPositional, + .fileSeekBy = fileSeekBy, + .fileSeekTo = fileSeekTo, + .fileRealPath = fileRealPath, + }), + }; + } + + fn openHandle(self: *S3) !struct { u32, *Handle } { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + + if (self.closed_handles.pop()) |idx| return .{ idx, self.handles.at(idx) }; + + return .{ @intCast(self.handles.len), try self.handles.addOne(self.allocator) }; + } + + fn closeHandle(self: *S3, idx: u32) !void { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + self.handles.at(idx).deinit(self.allocator); + try self.closed_handles.append(self.allocator, idx); + } + + fn getFileHandle(self: *S3, file: std.Io.File) *Handle { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + return self.handles.at(@intCast(file.handle)); + } + + fn getDirHandle(self: *S3, dir: std.Io.Dir) *Handle { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + return self.handles.at(@intCast(dir.handle)); + } + + fn resolvePath(self: *S3, dir: std.Io.Dir, sub_path: []const u8, out_buffer: []u8) ![]u8 { + if (std.meta.eql(dir, std.Io.Dir.cwd())) { + return try std.fmt.bufPrint(out_buffer, "{s}", .{sub_path}); + } + + const handle = self.getDirHandle(dir); + + const trimmed_uri = std.mem.trimEnd(u8, handle.uri, "/"); + const trimmed_sub_path = std.mem.trimStart(u8, sub_path, "/"); + + if (trimmed_uri.len == 0) return try std.fmt.bufPrint(out_buffer, "{s}", .{trimmed_sub_path}); + if (trimmed_sub_path.len == 0) return try std.fmt.bufPrint(out_buffer, "{s}", .{trimmed_uri}); + + return try std.fmt.bufPrint(out_buffer, "{s}/{s}", .{ trimmed_uri, trimmed_sub_path }); + } + + fn operate(userdata: ?*anyopaque, operation: std.Io.Operation) std.Io.Cancelable!std.Io.Operation.Result { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + switch (operation) { + .file_read_streaming => |o| { + const handle = self.getFileHandle(o.file); + const total = self.performRead(handle, o.data, handle.pos) catch |err| { + log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, handle.pos, err }); + return .{ .file_read_streaming = error.EndOfStream }; + }; + + if (total == 0) { + return .{ .file_read_streaming = error.EndOfStream }; + } + + handle.pos += @intCast(total); + return .{ .file_read_streaming = total }; + }, + .file_write_streaming, .device_io_control, .net_receive, .net_read, .net_send, .net_write => { + return self.base.inner.vtable.operate(self.base.inner.userdata, operation); + }, + } + } + + fn dirOpenDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.OpenOptions) std.Io.Dir.OpenError!std.Io.Dir { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + var path_buffer: [8 * 1024]u8 = undefined; + const path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.Dir.OpenError.SystemResources; + + const idx, const handle = self.openHandle() catch return std.Io.Dir.OpenError.Unexpected; + handle.* = Handle.init(self.allocator, .directory, path, 0) catch return std.Io.Dir.OpenError.Unexpected; + + return .{ .handle = @intCast(idx) }; + } + + fn dirStat(userdata: ?*anyopaque, dir: std.Io.Dir) std.Io.Dir.StatError!std.Io.Dir.Stat { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getDirHandle(dir); + + return .{ + .inode = @intCast(@intFromPtr(handle)), + .nlink = 0, + .size = handle.size, + .permissions = .fromMode(0o444), + .kind = .directory, + .atime = null, + .mtime = std.Io.Timestamp.zero, + .ctime = std.Io.Timestamp.zero, + .block_size = 1, + }; + } + + fn dirStatFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.StatFileOptions) std.Io.Dir.StatFileError!std.Io.File.Stat { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + const size = self.fetchSize(dir, sub_path) catch |err| switch (err) { + error.FileNotFound => return std.Io.File.OpenError.FileNotFound, + error.BadPathName => return std.Io.File.OpenError.BadPathName, + else => return std.Io.File.OpenError.Unexpected, + }; + + return .{ + .inode = 0, + .nlink = 0, + .size = size, + .permissions = .fromMode(0o444), + .kind = .file, + .atime = null, + .mtime = std.Io.Timestamp.zero, + .ctime = std.Io.Timestamp.zero, + .block_size = 1, + }; + } + + fn dirAccess(_: ?*anyopaque, _: std.Io.Dir, _: []const u8, _: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void {} + + fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!std.Io.File { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + const size = self.fetchSize(dir, sub_path) catch |err| switch (err) { + error.FileNotFound => return std.Io.File.OpenError.FileNotFound, + error.BadPathName => return std.Io.File.OpenError.BadPathName, + else => return std.Io.File.OpenError.Unexpected, + }; + + var path_buffer: [8 * 1024]u8 = undefined; + const path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.File.OpenError.SystemResources; + const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; + handle.* = Handle.init(self.allocator, .file, path, size) catch return std.Io.File.OpenError.Unexpected; + + return .{ .handle = @intCast(idx), .flags = .{ .nonblocking = false } }; + } + + fn dirClose(userdata: ?*anyopaque, dirs: []const std.Io.Dir) void { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + for (dirs) |dir| { + self.closeHandle(@intCast(dir.handle)) catch unreachable; + } + } + + fn dirRead(userdata: ?*anyopaque, reader: *std.Io.Dir.Reader, entries: []std.Io.Dir.Entry) std.Io.Dir.Reader.Error!usize { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + + if (reader.state == .finished) return 0; + + if (reader.state == .reset) { + if (self.dir_read_states.fetchRemove(reader)) |kv| { + for (kv.value.objects) |obj| self.allocator.free(obj); + self.allocator.free(kv.value.objects); + } + + const handle = self.getDirHandle(reader.dir); + const objects = self.listObjects(handle.uri) catch return std.Io.Dir.Reader.Error.Unexpected; + + self.dir_read_states.put(self.allocator, reader, .{ + .index = 0, + .objects = objects, + }) catch return std.Io.Dir.Reader.Error.Unexpected; + + reader.state = if (objects.len > 0) .reading else .finished; + + if (objects.len == 0) return 0; + } + + const state = self.dir_read_states.getPtr(reader) orelse return std.Io.Dir.Reader.Error.Unexpected; + + var count: usize = 0; + while (count < entries.len and state.index < state.objects.len) { + const obj_key = state.objects[state.index]; + const kind: std.Io.File.Kind = if (std.mem.endsWith(u8, obj_key, "/")) .directory else .file; + + const name = if (std.mem.lastIndexOfScalar(u8, std.mem.trimEnd(u8, obj_key, "/"), '/')) |idx| + obj_key[idx + 1 ..] + else + obj_key; + + entries[count] = .{ + .name = std.mem.trimEnd(u8, name, "/"), + .kind = kind, + .inode = state.index, + }; + count += 1; + state.index += 1; + } + + if (state.index >= state.objects.len) { + reader.state = .finished; + } + + return count; + } + + fn dirRealPath(userdata: ?*anyopaque, dir: std.Io.Dir, out_buffer: []u8) std.Io.Dir.RealPathError!usize { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getDirHandle(dir); + const path = std.fmt.bufPrint(out_buffer, "{s}", .{handle.uri}) catch return std.Io.Dir.RealPathError.SystemResources; + return path.len; + } + + fn dirRealPathFile(userdata: ?*anyopaque, dir: std.Io.Dir, path_name: []const u8, out_buffer: []u8) std.Io.Dir.RealPathFileError!usize { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + const real_path = self.resolvePath(dir, path_name, out_buffer) catch return std.Io.Dir.RealPathFileError.NameTooLong; + return real_path.len; + } + + fn fileStat(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.StatError!std.Io.File.Stat { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + + return .{ + .inode = @intCast(file.handle), + .nlink = 0, + .size = handle.size, + .permissions = .fromMode(0o444), + .kind = .file, + .atime = null, + .mtime = std.Io.Timestamp.zero, + .ctime = std.Io.Timestamp.zero, + .block_size = 1, + }; + } + + fn fileLength(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.LengthError!u64 { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + return self.getFileHandle(file).size; + } + + fn fileClose(userdata: ?*anyopaque, files: []const std.Io.File) void { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + for (files) |file| { + self.closeHandle(@intCast(file.handle)) catch unreachable; + } + } + + fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + return self.performRead(handle, data, offset) catch |err| { + log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, offset, err }); + return std.Io.File.ReadPositionalError.Unexpected; + }; + } + + fn fileSeekBy(userdata: ?*anyopaque, file: std.Io.File, relative_offset: i64) std.Io.File.SeekError!void { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + + if (relative_offset >= 0) { + handle.pos = handle.pos +| @as(u64, @intCast(relative_offset)); + } else { + const abs_offset: u64 = @intCast(-relative_offset); + if (abs_offset > handle.pos) { + handle.pos = 0; + } else { + handle.pos -= abs_offset; + } + } + } + + fn fileSeekTo(userdata: ?*anyopaque, file: std.Io.File, absolute_offset: u64) std.Io.File.SeekError!void { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + handle.pos = absolute_offset; + } + + fn fileRealPath(userdata: ?*anyopaque, file: std.Io.File, out_buffer: []u8) std.Io.File.RealPathError!usize { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + const path = std.fmt.bufPrint(out_buffer, "{s}", .{handle.uri}) catch return std.Io.File.RealPathError.SystemResources; + return path.len; + } + + fn getTimestamp(self: *S3, buf: []u8) ![]const u8 { + const now_ts = std.Io.Clock.now(.real, self.base.inner); + + const now: u64 = @intCast(@divFloor(now_ts.nanoseconds, std.time.ns_per_s)); + const epoch_secs: std.time.epoch.EpochSeconds = .{ .secs = now }; + const year_day = epoch_secs.getEpochDay().calculateYearDay(); + const month_day = year_day.calculateMonthDay(); + const day_seconds = epoch_secs.getDaySeconds(); + + return try std.fmt.bufPrint(buf, "{d:0>4}{d:0>2}{d:0>2}T{d:0>2}{d:0>2}{d:0>2}Z", .{ + year_day.year, + @backingInt(month_day.month), + month_day.day_index + 1, + day_seconds.getHoursIntoDay(), + day_seconds.getMinutesIntoHour(), + day_seconds.getSecondsIntoMinute(), + }); + } + + fn pathComponents(self: *S3, path_: []const u8) struct { []const u8, []const u8, []const u8 } { + const endpoint = std.mem.trimEnd(u8, self.config.endpoint_url, "/"); + const path = std.mem.trim(u8, path_, "/"); + + if (std.mem.findScalar(u8, path, '/')) |idx| { + return .{ endpoint, path[0..idx], if (idx + 1 < path.len) path[idx + 1 ..] else "" }; + } else { + return .{ endpoint, path, "" }; + } + } + + fn s3Url(self: *S3, path: []const u8, buf: []u8) ![]const u8 { + const endpoint, const bucket, const key = self.pathComponents(path); + return try std.fmt.bufPrint(buf, "{s}/{s}/{s}", .{ endpoint, bucket, key }); + } + + fn listObjectsBody(self: *S3, endpoint: []const u8, bucket: []const u8, key_prefix: []const u8) ![]u8 { + var query_buf: [4096]u8 = undefined; + var query_writer = std.Io.Writer.fixed(&query_buf); + + try query_writer.writeAll("delimiter="); + try std.Uri.Component.percentEncode(&query_writer, "/", s3EncodeIsValid); + try query_writer.writeAll("&list-type=2&max-keys=1000"); + + if (key_prefix.len > 0) { + try query_writer.writeAll("&prefix="); + try std.Uri.Component.percentEncode(&query_writer, key_prefix, s3EncodeIsValid); + if (!std.mem.endsWith(u8, key_prefix, "/")) { + try std.Uri.Component.percentEncode(&query_writer, "/", s3EncodeIsValid); + } + } + + const endpoint_uri = try std.Uri.parse(endpoint); + + var path_buf: [256]u8 = undefined; + + const uri: std.Uri = .{ + .scheme = endpoint_uri.scheme, + .host = endpoint_uri.host, + .port = endpoint_uri.port, + .path = .{ .percent_encoded = try std.fmt.bufPrint(&path_buf, "/{s}", .{bucket}) }, + .query = .{ .percent_encoded = query_writer.buffered() }, + .fragment = null, + }; + + var timestamp_buf: [16]u8 = undefined; + const timestamp = try self.getTimestamp(×tamp_buf); + + const signer: AwsSigV4 = .{ + .access_key = self.config.access_key, + .secret_key = self.config.secret_key, + .region = self.config.region, + .service = self.config.auth_service, + }; + + var authorization_buffer: [512]u8 = undefined; + const authorization = try signer.generateAuthHeader(&authorization_buffer, .GET, uri, timestamp, &.{}); + + var req = try self.client.request(.GET, uri, .{ + .redirect_behavior = .not_allowed, + .headers = .{ + .accept_encoding = .{ .override = "identity" }, + .authorization = if (authorization) |auth| .{ .override = auth } else .omit, + }, + .extra_headers = &.{ + .{ .name = "x-amz-date", .value = timestamp }, + .{ .name = "x-amz-content-sha256", .value = AwsSigV4.UNSIGNED_PAYLOAD }, + }, + }); + defer req.deinit(); + + try req.sendBodiless(); + + var redirect_buffer: [2 * 1024]u8 = undefined; + var res = try req.receiveHead(&redirect_buffer); + + if (res.head.status != .ok) { + log.err("Failed to list object {f}", .{uri}); + log.err("{s}", .{res.head.bytes}); + return error.RequestFailed; + } + + return if (res.head.content_length) |content_len| + try res.reader(&.{}).readAlloc(self.allocator, content_len) + else + // When we don't have a content length, put a reasonable limit + try res.reader(&.{}).allocRemaining(self.allocator, .limited(1024 * 1024)); + } + + fn listObjects(self: *S3, prefix: []const u8) ![][]const u8 { + const endpoint, const bucket, const key_prefix = self.pathComponents(prefix); + const body = try self.listObjectsBody(endpoint, bucket, key_prefix); + defer self.allocator.free(body); + + return try self.parseListObjectsResponse(body, key_prefix); + } + + fn s3EncodeIsValid(c: u8) bool { + return switch (c) { + 'A'...'Z', 'a'...'z', '0'...'9', '-', '.', '_', '~' => true, + else => false, + }; + } + + fn parseListObjectsResponse(self: *S3, xml: []const u8, prefix: []const u8) ![][]const u8 { + var results: std.ArrayListUnmanaged([]const u8) = .empty; + errdefer { + for (results.items) |item| self.allocator.free(item); + results.deinit(self.allocator); + } + + inline for (.{ "Key", "Prefix" }) |tag| { + var pos: usize = 0; + while (std.mem.indexOfPos(u8, xml, pos, "<" ++ tag ++ ">")) |start| { + const v_start = start + tag.len + 2; + const end = std.mem.indexOfPos(u8, xml, v_start, "") orelse break; + const val = xml[v_start..end]; + + // Filter out the directory itself (S3 often includes the prefix as a 0-byte key) + const is_self = std.mem.eql(u8, std.mem.trimEnd(u8, val, "/"), std.mem.trimEnd(u8, prefix, "/")); + if (val.len > 0 and !is_self) { + try results.append(self.allocator, try self.allocator.dupe(u8, val)); + } + pos = end + tag.len + 3; + } + } + + return try results.toOwnedSlice(self.allocator); + } + + fn fetchSize(self: *S3, dir: std.Io.Dir, sub_path: []const u8) !u64 { + var path_buffer: [8 * 1024]u8 = undefined; + var url_buf: [8 * 1024]u8 = undefined; + + const path = try self.resolvePath(dir, sub_path, &path_buffer); + const url = try self.s3Url(path, &url_buf); + + const uri = std.Uri.parse(url) catch return error.BadPathName; + + var timestamp_buf: [16]u8 = undefined; + const timestamp = try self.getTimestamp(×tamp_buf); + + const signer: AwsSigV4 = .{ + .access_key = self.config.access_key, + .secret_key = self.config.secret_key, + .region = self.config.region, + .service = self.config.auth_service, + }; + + var authorization_buffer: [512]u8 = undefined; + const authorization = try signer.generateAuthHeader(&authorization_buffer, .HEAD, uri, timestamp, &.{}); + + var req = try self.client.request(.HEAD, uri, .{ + .redirect_behavior = .not_allowed, + .headers = .{ + .accept_encoding = .{ .override = "identity" }, + .authorization = if (authorization) |auth| .{ .override = auth } else .omit, + }, + .extra_headers = &.{ + .{ .name = "x-amz-date", .value = timestamp }, + .{ .name = "x-amz-content-sha256", .value = AwsSigV4.UNSIGNED_PAYLOAD }, + }, + }); + defer req.deinit(); + + try req.sendBodiless(); + + var redirect_buffer: [8 * 1024]u8 = undefined; + const res = try req.receiveHead(&redirect_buffer); + + return switch (res.head.status.class()) { + .success => res.head.content_length.?, + else => switch (res.head.status) { + .not_found => return error.FileNotFound, + else => blk: { + log.err("Failed to fetch size for {s}: {s}", .{ url, res.head.bytes }); + break :blk error.ServerError; + }, + }, + }; + } + + fn performRead(self: *S3, handle: *Handle, data: []const []u8, offset: u64) !usize { + const read_size = parallel_read.readSize(handle.size, offset, data); + if (read_size == 0) return 0; + + var url_buf: [8 * 1024]u8 = undefined; + const url = try self.s3Url(handle.uri, &url_buf); + const uri = std.Uri.parse(url) catch return error.BadPathName; + + const job_count = std.math.divCeil(usize, read_size, self.read_pool.chunk_size) catch unreachable; + const jobs = try self.allocator.alloc(ParallelRead.Job, job_count); + defer self.allocator.free(jobs); + const pending: u32 = @intCast(job_count); + + var batch: ParallelRead.Batch = .{ + .state = .{ .pending = .init(pending) }, + .backend = self, + .uri = uri, + .url = url, + }; + + for (jobs, 0..) |*job, i| { + const chunk_offset = i * self.read_pool.chunk_size; + job.* = .{ + .data = data, + .file_offset = offset + @as(u64, @intCast(chunk_offset)), + .chunk_offset = chunk_offset, + .chunk_len = @min(self.read_pool.chunk_size, read_size - chunk_offset), + .batch = &batch, + }; + } + + try self.read_pool.job_queue.putAll(self.base.inner, jobs); + batch.state.waitUncancelable(self.base.inner); + if (batch.state.anyError()) |err| return err; + + return read_size; + } +}; diff --git a/vfs/vfs.zig b/vfs/vfs.zig new file mode 100644 index 00000000..fe626ff9 --- /dev/null +++ b/vfs/vfs.zig @@ -0,0 +1,483 @@ +const std = @import("std"); + +const stdx = @import("stdx"); + +pub const File = @import("file.zig").File; +pub const GCS = @import("gcs.zig").GCS; +pub const HF = @import("hf.zig").HF; +pub const HTTP = @import("http.zig").HTTP; +pub const S3 = @import("s3.zig").S3; +pub const VFSBase = @import("base.zig").VFSBase; + +const log = std.log.scoped(.@"zml/vfs"); + +const CWD_HANDLE: u32 = 0; + +const VFS = @This(); +const Handle = struct { handle: u32, backend_idx: ?usize, flags: std.Io.File.Flags = .{ .nonblocking = false } }; + +allocator: std.mem.Allocator, +mutex: std.Io.Mutex = .init, + +backends: std.StringArrayHashMapUnmanaged(std.Io) = .empty, +handles: stdx.SegmentedList(Handle, 128) = .{}, +closed_handles: std.ArrayList(u32) = .empty, + +base: VFSBase, + +pub fn init(allocator: std.mem.Allocator, base_io: std.Io) !VFS { + const base = VFSBase.init(base_io); + + var handles: @FieldType(@This(), "handles") = .{}; + try handles.append(allocator, .{ .handle = CWD_HANDLE, .backend_idx = null }); + try handles.append(allocator, .{ .handle = std.posix.STDIN_FILENO, .backend_idx = null }); + try handles.append(allocator, .{ .handle = std.posix.STDOUT_FILENO, .backend_idx = null }); + try handles.append(allocator, .{ .handle = std.posix.STDERR_FILENO, .backend_idx = null }); + + return .{ + .allocator = allocator, + .handles = handles, + .base = base, + }; +} + +pub fn deinit(self: *VFS) void { + self.handles.deinit(self.allocator); + self.closed_handles.deinit(self.allocator); + self.backends.deinit(self.allocator); +} + +pub fn register(self: *VFS, scheme: []const u8, backend: std.Io) std.mem.Allocator.Error!void { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + + try self.backends.put(self.allocator, scheme, backend); +} + +pub fn unregister(self: *VFS, scheme: []const u8) bool { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + + return self.backends.remove(scheme); +} + +pub fn io(self: *VFS) std.Io { + return .{ + .userdata = &self.base, + .vtable = &comptime VFSBase.vtable(.{ + .operate = operate, + .dirOpenDir = dirOpenDir, + .dirStat = dirStat, + .dirStatFile = dirStatFile, + .dirAccess = dirAccess, + .dirCreateFile = dirCreateFile, + .dirOpenFile = dirOpenFile, + .dirClose = dirClose, + .dirRead = dirRead, + .dirRealPath = dirRealPath, + .dirRealPathFile = dirRealPathFile, + .fileStat = fileStat, + .fileLength = fileLength, + .fileClose = fileClose, + .fileWritePositional = fileWritePositional, + .fileWriteFileStreaming = fileWriteFileStreaming, + .fileWriteFilePositional = fileWriteFilePositional, + .fileReadPositional = fileReadPositional, + .fileSeekBy = fileSeekBy, + .fileSeekTo = fileSeekTo, + .fileRealPath = fileRealPath, + }), + }; +} + +fn openHandle(self: *VFS) !struct { u32, *Handle } { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + + if (self.closed_handles.pop()) |idx| { + return .{ idx, self.handles.at(idx) }; + } + return .{ @intCast(self.handles.len), try self.handles.addOne(self.allocator) }; +} + +fn closeHandle(self: *VFS, idx: u32) !void { + if (idx == CWD_HANDLE) return; + + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + + try self.closed_handles.append(self.allocator, idx); +} + +fn getFileHandle(self: *VFS, file: std.Io.File) struct { *Handle, std.Io } { + self.mutex.lockUncancelable(self.base.inner); + const handle = self.handles.at(@intCast(file.handle)); + self.mutex.unlock(self.base.inner); + + return .{ handle, self.getBackend(handle.backend_idx) }; +} + +fn getDirHandle(self: *VFS, dir: std.Io.Dir) *Handle { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + + if (std.meta.eql(dir, std.Io.Dir.cwd())) return self.handles.at(CWD_HANDLE); + return self.handles.at(@intCast(dir.handle)); +} + +fn getScheme(self: *VFS, backend_idx: ?usize) ?[]const u8 { + if (backend_idx) |idx| return self.backends.entries.items(.key)[idx] else return null; +} + +fn getBackend(self: *VFS, backend_idx: ?usize) std.Io { + if (backend_idx) |idx| return self.backends.entries.items(.value)[idx] else return self.base.inner; +} + +fn lookupDir(self: *VFS, dir: std.Io.Dir, sub_path: ?[]const u8) !struct { ?usize, std.Io.Dir, std.Io } { + // A scheme-qualified path (e.g. "hf://owner/model/file") is absolute: it + // is resolved from the scheme's root regardless of `dir`. Without this, + // opening such a path relative to an already-open dir of the same + // backend double-prefixes the dir's path. + if (sub_path) |sp| { + if (std.mem.indexOf(u8, sp, "://") != null) { + const uri = std.Uri.parse(sp) catch return error.VFSNotRegistered; + const backend_idx: usize = for (self.backends.entries.items(.key), 0..) |s, idx| { + if (std.mem.eql(u8, uri.scheme, s)) break idx; + } else return error.VFSNotRegistered; + return .{ backend_idx, std.Io.Dir.cwd(), self.getBackend(backend_idx) }; + } + } + + if (std.meta.eql(dir, std.Io.Dir.cwd())) { + if (sub_path == null) return .{ null, dir, self.base.inner }; + if (std.fs.path.isAbsolutePosix(sub_path.?)) return .{ null, dir, self.base.inner }; + + const uri = std.Uri.parse(sub_path.?) catch null; + if (uri) |u| { + const backend_idx: ?usize = for (self.backends.entries.items(.key), 0..) |s, idx| { + if (std.mem.eql(u8, u.scheme, s)) break idx; + } else null; + if (backend_idx == null) return error.VFSNotRegistered; + return .{ backend_idx, std.Io.Dir.cwd(), self.getBackend(backend_idx) }; + } else { + return .{ null, std.Io.Dir.cwd(), self.base.inner }; + } + } else { + const handle = self.getDirHandle(dir); + if (handle.backend_idx) |backend_idx| { + return .{ backend_idx, .{ .handle = @intCast(handle.handle) }, self.getBackend(backend_idx) }; + } else { + return .{ null, .{ .handle = @intCast(handle.handle) }, self.base.inner }; + } + } +} + +fn stripScheme(path: []const u8) []const u8 { + const uri = std.Uri.parse(path) catch return path; + return path[uri.scheme.len + 3 ..]; +} + +fn operate(userdata: ?*anyopaque, operation: std.Io.Operation) std.Io.Cancelable!std.Io.Operation.Result { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + switch (operation) { + .file_read_streaming => |o| { + const handle, const backend = self.getFileHandle(o.file); + return backend.vtable.operate(backend.userdata, .{ .file_read_streaming = .{ + .file = .{ .handle = @intCast(handle.handle), .flags = handle.flags }, + .data = o.data, + } }); + }, + .file_write_streaming => |o| { + const handle, const backend = self.getFileHandle(o.file); + return backend.vtable.operate(backend.userdata, .{ .file_write_streaming = .{ + .file = .{ .handle = @intCast(handle.handle), .flags = handle.flags }, + .header = o.header, + .data = o.data, + .splat = o.splat, + } }); + }, + .device_io_control, .net_receive, .net_read, .net_send, .net_write => { + return self.base.inner.vtable.operate(self.base.inner.userdata, operation); + }, + } +} + +fn dirOpenDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.OpenOptions) std.Io.Dir.OpenError!std.Io.Dir { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const backend_idx, const dir_, const backend = self.lookupDir(dir, sub_path) catch |err| { + log.err("Failed to lookup backend for opening dir '{s}' : {any}", .{ sub_path, err }); + return std.Io.Dir.OpenError.Unexpected; + }; + const fs_dir = try backend.vtable.dirOpenDir(backend.userdata, dir_, stripScheme(sub_path), options); + const idx, const handle = self.openHandle() catch return std.Io.Dir.OpenError.Unexpected; + handle.* = .{ + .handle = @intCast(fs_dir.handle), + .backend_idx = backend_idx, + }; + return .{ .handle = @intCast(idx) }; +} + +fn dirStat(userdata: ?*anyopaque, dir: std.Io.Dir) std.Io.Dir.StatError!std.Io.Dir.Stat { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + _, const dir_, const backend = self.lookupDir(dir, null) catch |err| { + log.err("Failed to lookup backend for dir stat : {any}", .{err}); + return std.Io.Dir.StatError.Unexpected; + }; + return backend.vtable.dirStat(backend.userdata, dir_); +} + +pub fn dirStatFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.StatFileOptions) std.Io.Dir.StatFileError!std.Io.File.Stat { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + _, const dir_, const backend = self.lookupDir(dir, sub_path) catch |err| { + log.err("Failed to lookup backend for dir stat file '{s}' : {any}", .{ sub_path, err }); + return std.Io.Dir.StatFileError.Unexpected; + }; + return backend.vtable.dirStatFile(backend.userdata, dir_, stripScheme(sub_path), options); +} + +pub fn dirAccess(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + _, const dir_, const backend = self.lookupDir(dir, sub_path) catch |err| { + log.err("Failed to lookup backend for dir access '{s}' : {any}", .{ sub_path, err }); + return std.Io.Dir.AccessError.Unexpected; + }; + return backend.vtable.dirAccess(backend.userdata, dir_, stripScheme(sub_path), options); +} + +pub fn dirCreateFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.Dir.CreateFileOptions) std.Io.File.OpenError!std.Io.File { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const backend_idx, const dir_, const backend = self.lookupDir(dir, sub_path) catch |err| { + log.err("Failed to lookup backend for dir create file '{s}' : {any}", .{ sub_path, err }); + return std.Io.File.OpenError.Unexpected; + }; + + const file = try backend.vtable.dirCreateFile(backend.userdata, dir_, stripScheme(sub_path), flags); + errdefer file.close(backend); + const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; + handle.* = .{ + .handle = @intCast(file.handle), + .backend_idx = backend_idx, + .flags = file.flags, + }; + return .{ .handle = @intCast(idx), .flags = file.flags }; +} + +fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!std.Io.File { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const backend_idx, const dir_, const backend = self.lookupDir(dir, sub_path) catch |err| { + log.err("Failed to lookup backend for opening file '{s}' : {any}", .{ sub_path, err }); + return std.Io.File.OpenError.Unexpected; + }; + const file = try backend.vtable.dirOpenFile(backend.userdata, dir_, stripScheme(sub_path), flags); + errdefer file.close(backend); + const idx, const handle = self.openHandle() catch return std.Io.Dir.OpenError.Unexpected; + handle.* = .{ + .handle = @intCast(file.handle), + .backend_idx = backend_idx, + .flags = file.flags, + }; + return .{ .handle = @intCast(idx), .flags = handle.flags }; +} + +fn dirClose(userdata: ?*anyopaque, dirs: []const std.Io.Dir) void { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + for (dirs) |dir| { + _, const dir_, const backend = self.lookupDir(dir, null) catch |err| { + log.err("Failed to lookup backend for closing dir : {any}", .{err}); + continue; + }; + backend.vtable.dirClose(backend.userdata, &.{dir_}); + self.closeHandle(@intCast(dir.handle)) catch unreachable; + } +} + +fn dirRead(userdata: ?*anyopaque, reader: *std.Io.Dir.Reader, entries: []std.Io.Dir.Entry) std.Io.Dir.Reader.Error!usize { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + _, const dir_, const backend = self.lookupDir(reader.dir, null) catch |err| { + log.err("Failed to lookup backend for dir real path : {any}", .{err}); + return std.Io.Dir.Reader.Error.Unexpected; + }; + + const original_dir = reader.dir; + reader.dir = dir_; + defer reader.dir = original_dir; + + return backend.vtable.dirRead(backend.userdata, reader, entries); +} + +fn dirRealPath(userdata: ?*anyopaque, dir: std.Io.Dir, out_buffer: []u8) std.Io.Dir.RealPathError!usize { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const backend_idx, const dir_, const backend = self.lookupDir(dir, null) catch |err| { + log.err("Failed to lookup backend for dir real path : {any}", .{err}); + return std.Io.Dir.RealPathError.Unexpected; + }; + + if (self.getScheme(backend_idx)) |s| { + const prefix = try std.fmt.bufPrint(out_buffer, "{s}://", .{s}); + const path_len = try backend.vtable.dirRealPath(backend.userdata, dir_, out_buffer[prefix.len..]); + return prefix.len + path_len; + } else { + return try backend.vtable.dirRealPath(backend.userdata, dir_, out_buffer); + } +} + +fn dirRealPathFile(userdata: ?*anyopaque, dir: std.Io.Dir, path_name: []const u8, out_buffer: []u8) std.Io.Dir.RealPathFileError!usize { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const backend_idx, const dir_, const backend = self.lookupDir(dir, path_name) catch |err| { + log.err("Failed to lookup backend for dir real path file '{s}' : {any}", .{ path_name, err }); + return std.Io.Dir.RealPathFileError.Unexpected; + }; + + if (self.getScheme(backend_idx)) |s| { + const prefix = try std.fmt.bufPrint(out_buffer, "{s}://", .{s}); + const path_len = try backend.vtable.dirRealPathFile(backend.userdata, dir_, path_name, out_buffer[prefix.len..]); + return prefix.len + path_len; + } else { + return try backend.vtable.dirRealPathFile(backend.userdata, dir_, path_name, out_buffer); + } +} + +fn fileStat(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.StatError!std.Io.File.Stat { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle, const backend = self.getFileHandle(file); + return backend.vtable.fileStat(backend.userdata, .{ .handle = @intCast(handle.handle), .flags = handle.flags }); +} + +fn fileLength(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.LengthError!u64 { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle, const backend = self.getFileHandle(file); + return backend.vtable.fileLength(backend.userdata, .{ .handle = @intCast(handle.handle), .flags = handle.flags }); +} + +fn fileClose(userdata: ?*anyopaque, files: []const std.Io.File) void { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + for (files) |file| { + const handle, const backend = self.getFileHandle(file); + backend.vtable.fileClose(backend.userdata, &.{.{ .handle = @intCast(handle.handle), .flags = handle.flags }}); + self.closeHandle(@intCast(file.handle)) catch unreachable; + } +} + +fn fileWritePositional(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, data: []const []const u8, splat: usize, offset: u64) std.Io.File.WritePositionalError!usize { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle, const backend = self.getFileHandle(file); + return backend.vtable.fileWritePositional(backend.userdata, .{ .handle = @intCast(handle.handle), .flags = handle.flags }, header, data, splat, offset); +} + +fn fileWriteFileStreaming(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, reader: *std.Io.File.Reader, limit: std.Io.Limit) std.Io.File.Writer.WriteFileError!usize { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const own_io = self.io(); + if (reader.io.userdata != own_io.userdata or reader.io.vtable != own_io.vtable) return error.Unimplemented; + + const dst_handle, const dst_backend = self.getFileHandle(file); + const src_handle, const src_backend = self.getFileHandle(reader.file); + if (src_backend.userdata != dst_backend.userdata or src_backend.vtable != dst_backend.vtable) return error.Unimplemented; + + const original_src_io = reader.io; + const original_src_handle = reader.file; + reader.io = src_backend; + reader.file = .{ .handle = @intCast(src_handle.handle), .flags = src_handle.flags }; + defer { + reader.io = original_src_io; + reader.file = original_src_handle; + } + + return dst_backend.vtable.fileWriteFileStreaming(dst_backend.userdata, .{ .handle = @intCast(dst_handle.handle), .flags = dst_handle.flags }, header, reader, limit); +} + +fn fileWriteFilePositional(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, reader: *std.Io.File.Reader, limit: std.Io.Limit, offset: u64) std.Io.File.WriteFilePositionalError!usize { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const own_io = self.io(); + if (reader.io.userdata != own_io.userdata or reader.io.vtable != own_io.vtable) return error.Unimplemented; + + const dst_handle, const dst_backend = self.getFileHandle(file); + const src_handle, const src_backend = self.getFileHandle(reader.file); + if (src_backend.userdata != dst_backend.userdata or src_backend.vtable != dst_backend.vtable) return error.Unimplemented; + + const original_src_io = reader.io; + const original_src_handle = reader.file; + reader.io = src_backend; + reader.file = .{ .handle = @intCast(src_handle.handle), .flags = src_handle.flags }; + defer { + reader.io = original_src_io; + reader.file = original_src_handle; + } + + return dst_backend.vtable.fileWriteFilePositional(dst_backend.userdata, .{ .handle = @intCast(dst_handle.handle), .flags = dst_handle.flags }, header, reader, limit, offset); +} + +fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle, const backend = self.getFileHandle(file); + return backend.vtable.fileReadPositional(backend.userdata, .{ .handle = @intCast(handle.handle), .flags = handle.flags }, data, offset); +} + +fn fileSeekBy(userdata: ?*anyopaque, file: std.Io.File, relative_offset: i64) std.Io.File.SeekError!void { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle, const backend = self.getFileHandle(file); + return backend.vtable.fileSeekBy(backend.userdata, .{ .handle = @intCast(handle.handle), .flags = handle.flags }, relative_offset); +} + +fn fileSeekTo(userdata: ?*anyopaque, file: std.Io.File, absolute_offset: u64) std.Io.File.SeekError!void { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle, const backend = self.getFileHandle(file); + return backend.vtable.fileSeekTo(backend.userdata, .{ .handle = @intCast(handle.handle), .flags = handle.flags }, absolute_offset); +} + +fn fileRealPath(userdata: ?*anyopaque, file: std.Io.File, out_buffer: []u8) std.Io.File.RealPathError!usize { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle, const backend = self.getFileHandle(file); + + if (self.getScheme(handle.backend_idx)) |s| { + const prefix = try std.fmt.bufPrint(out_buffer, "{s}://", .{s}); + const path_len = try backend.vtable.fileRealPath(backend.userdata, .{ .handle = @intCast(handle.handle), .flags = handle.flags }, out_buffer[prefix.len..]); + return prefix.len + path_len; + } else { + return try backend.vtable.fileRealPath(backend.userdata, .{ .handle = @intCast(handle.handle), .flags = handle.flags }, out_buffer); + } +} + +test "copy a local file through the VFS" { + const allocator = std.testing.allocator; + const process_io = std.testing.io; + + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + + const source_data: [8192]u8 = @splat('x'); + try tmp.dir.writeFile(process_io, .{ + .sub_path = "source", + .data = &source_data, + }); + + var path_buffer: [std.Io.Dir.max_path_bytes]u8 = undefined; + const path_len = try tmp.dir.realPath(process_io, &path_buffer); + const source_uri = try std.fmt.allocPrint(allocator, "file://{s}/source", .{path_buffer[0..path_len]}); + defer allocator.free(source_uri); + const destination_uri = try std.fmt.allocPrint(allocator, "file://{s}/destination", .{path_buffer[0..path_len]}); + defer allocator.free(destination_uri); + + var file_backend: File = .init(allocator, process_io, .{}); + defer file_backend.deinit(); + var vfs: VFS = try .init(allocator, process_io); + defer vfs.deinit(); + try vfs.register("file", file_backend.io()); + + const vfs_io = vfs.io(); + { + var source = try std.Io.Dir.openFile(.cwd(), vfs_io, source_uri, .{}); + defer source.close(vfs_io); + var destination = try std.Io.Dir.createFile(.cwd(), vfs_io, destination_uri, .{}); + defer destination.close(vfs_io); + + var reader: std.Io.File.Reader = .initStreaming(source, vfs_io, &.{}); + var copy_buffer: [4096]u8 align(4096) = undefined; + var writer = destination.writer(vfs_io, ©_buffer); + try std.testing.expectEqual(source_data.len, try reader.interface.streamRemaining(&writer.interface)); + try writer.interface.flush(); + } + + const copied = try tmp.dir.readFileAlloc(process_io, "destination", allocator, .limited(source_data.len + 1)); + defer allocator.free(copied); + try std.testing.expectEqualSlices(u8, &source_data, copied); +} diff --git a/zls.build.json b/zls.build.json deleted file mode 120000 index f5c47938..00000000 --- a/zls.build.json +++ /dev/null @@ -1 +0,0 @@ -examples/zls.build.json \ No newline at end of file diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index 8d5bdded..f52ecdd5 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -1,94 +1,199 @@ -load("@aspect_bazel_lib//lib:tar.bzl", "mtree_spec", "tar") -load("@rules_zig//zig:defs.bzl", "zig_library", "zls_completion") -load("//bazel:zig.bzl", "zig_cc_test") -load("//bazel:zig_proto_library.bzl", "zig_proto_library") +load("@com_google_protobuf//bazel:upb_c_proto_library.bzl", "upb_c_proto_library") +load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +upb_c_proto_library( + name = "xla_data_upb", + tags = ["manual"], + deps = ["@xla//xla:xla_data_proto"], +) + +upb_c_proto_library( + name = "xla_compile_options_upb", + tags = ["manual"], + deps = ["@xla//xla/pjrt/proto:compile_options_proto"], +) + +upb_c_proto_library( + name = "profiler_options", + tags = ["manual"], + deps = ["@tsl//tsl/profiler/protobuf:profiler_options_proto"], +) cc_library( name = "posix", hdrs = ["posix.h"], + tags = ["manual"], +) + +cc_library( + name = "profiling_traceme_lib", + srcs = ["profiling/traceme_lib.cc"], + hdrs = ["profiling/traceme.h"], + defines = select({ + "//platforms:cuda.enabled": ["ZML_RUNTIME_CUDA"], + "//conditions:default": [], + }) + select({ + "//platforms:rocm.enabled": ["ZML_RUNTIME_ROCM"], + "//conditions:default": [], + }), + linkopts = select({ + "//platforms:rocm.enabled": ["-ldl"], + "//conditions:default": [], + }), + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//ffi:cc", + "@tsl//tsl/profiler/lib:traceme", + "@tsl//tsl/profiler/protobuf:xplane_proto_cc", + "@xla//xla/tsl/profiler/backends/cpu:host_tracer_utils", + "@xla//xla/tsl/profiler/backends/cpu:traceme_recorder", + "@xla//xla/tsl/profiler/convert:post_process_single_host_xplane", + "@xla//xla/tsl/profiler/utils:time_utils", + "@xla//xla/tsl/profiler/utils:xplane_utils", + ] + select({ + "//platforms:cuda.enabled": [ + "//platforms/cuda:cuda_nvtx_headers", + ], + "//conditions:default": [], + }) + select({ + "//platforms:rocm.enabled": [ + "@amdrocm-profiler-base//:roctx_headers", + ], + "//conditions:default": [], + }), +) + +zig_library( + name = "logo", + srcs = ["logo.zig"], + import_name = "zml/logo", + main = "logo.zig", + visibility = ["//visibility:public"], ) zig_library( name = "zml", - srcs = glob([ - "*.zig", - "aio/**/*.zig", - "nn/**/*.zig", - "ops/**/*.zig", - # TODO: test_runner.zig should not be here. - # It's here for now because it seems that test_runner property in zig_test is misbehaving. - # See https://github.com/zml/rules_zig/issues/2 - "test_runner.zig", - ]), - copts = ["-lc"], + srcs = [ + "Compiler.zig", + "Sharding.zig", + "attention/Options.zig", + "attention/attention.zig", + "buffer.zig", + "buffer/HostAccessible.zig", + "constants.zig", + "dtype.zig", + "exe.zig", + "floats.zig", + "io.zig", + "kernels/flash_attention/flash_attention.zig", + "kernels/flash_attention/tests.zig", + "kernels/moe/api.zig", + "kernels/moe/moe.zig", + "kernels/moe/tests.zig", + "kernels/moe/weights.zig", + "kernels/paged_attention/options.zig", + "kernels/paged_attention/paged_attention.zig", + "logo.zig", + "mem.zig", + "meta.zig", + "mlirx.zig", + "nn.zig", + "ops.zig", + "ops/manual_computation.zig", + "pjrtx.zig", + "platform.zig", + "profiling/profiler.zig", + "profiling/tracer.zig", + "quantization.zig", + "safetensors.zig", + "shape.zig", + "slice.zig", + "tensor.zig", + "testing.zig", + "zml.zig", + "//zml/kernel", + "//zml/kernels/flash_attention/cute", + "//zml/kernels/flash_attention/metal", + "//zml/kernels/flash_attention/nki", + "//zml/kernels/moe/cute", + "//zml/kernels/moe/flashinfer_cutlass", + "//zml/kernels/moe/fly", + "//zml/kernels/moe/metal", + "//zml/kernels/moe/mosaic_tpu", + "//zml/kernels/moe/stablehlo", + "//zml/kernels/moe/tcl", + "//zml/kernels/moe/triton", + "//zml/kernels/paged_attention/cute", + "//zml/kernels/paged_attention/metal", + "//zml/kernels/paged_attention/mosaic_tpu", + "//zml/kernels/paged_attention/stablehlo", + "//zml/kernels/paged_attention/tcl", + "//zml/kernels/paged_attention/triton", + ], + data = select({ + "//platforms:neuron.enabled": [ + "//zml/kernels/flash_attention/nki:attention_kernel", + ], + "//conditions:default": [], + }), main = "zml.zig", + tags = ["manual"], visibility = ["//visibility:public"], + zigopts = [ + "-lc", + "-freference-trace=20", + ], deps = [ ":posix", - ":sentencepiece_model_proto", - ":xla_proto", - "//async", + ":profiler_options", + ":profiling_traceme_lib", + ":xla_compile_options_upb", + ":xla_data_upb", + "//bazel", + "//dsl/cuda_tile:builder", + "//dsl/cute:builder", + "//dsl/fly:builder", + "//dsl/metal:builder", + "//dsl/mosaic_tpu:builder", + "//dsl/tcl:builder", + "//dsl/triton:builder", + "//ffi:zig", "//mlir", - "//mlir/dialects", + "//mlir/dialects/func", + "//mlir/dialects/llvm", + "//mlir/dialects/mosaic_tpu", + "//mlir/dialects/nvvm", + "//mlir/dialects/shardy", + "//mlir/dialects/stablehlo", + "//mlir/dialects/ttir", + "//mlir/dialects/vector", "//pjrt", - "//runtimes", - "//zml/tools", - "@rules_zig//zig/lib:libc", + "//platforms", + "//platforms/tpu:ragged_paged", + "//stdx", + "//tools/xspace_to_perfetto:xspace_to_perfetto_lib", + "//upb", + "//vfs", + "//zml/tokenizer", "@rules_zig//zig/runfiles", - "@zig-yaml//:zig-yaml", - ], + "@zio", + ] + select({ + "//platforms:neuron.enabled": [ + "//platforms/neuron:nki_kernel", + "//platforms/neuron:topology", + ], + "//conditions:default": [], + }) + select({ + "//platforms:cuda.enabled": [ + "//platforms/cuda/flashinfer_cutlass_moe", + ], + "//conditions:default": [], + }), ) -zls_completion( - name = "completion", - deps = [":zml"], -) - -zig_proto_library( - name = "xla_proto", - import_name = "//xla:xla_proto", - deps = ["@xla//xla/pjrt:compile_options_proto"], -) - -zig_proto_library( - name = "sentencepiece_model_proto", - import_name = "//sentencepiece:model_proto", - deps = ["@sentencepiece//:sentencepiece_model_proto"], -) - -# All ZML Tests - -zig_cc_test( +zig_test( name = "test", - data = [ - "aio/torch/simple.pt", - "aio/torch/simple_test.pickle", - ], - test_runner = ":test_runner", - deps = [":zml"], -) - -filegroup( - name = "test_runner", - srcs = ["test_runner.zig"], - visibility = ["//visibility:public"], -) - -filegroup( - name = "srcs", - srcs = [":test_test_lib"], - output_group = "srcs", -) - -mtree_spec( - name = "mtree", - srcs = [":srcs"], -) - -tar( - name = "sources", - srcs = [":srcs"], - args = [ - ], - mtree = ":mtree", + deps = ["//zml"], ) diff --git a/zml/Compiler.zig b/zml/Compiler.zig new file mode 100644 index 00000000..93eafa8d --- /dev/null +++ b/zml/Compiler.zig @@ -0,0 +1,1096 @@ +const std = @import("std"); + +const c = @import("c"); +const func = @import("mlir/dialects/func"); +const mlir = @import("mlir"); +const pjrt = @import("pjrt"); +const shardy = @import("mlir/dialects/shardy"); +const stablehlo = @import("mlir/dialects/stablehlo"); +const stdx = @import("stdx"); +const upb = @import("upb"); +const zio = @import("zio"); + +const Buffer = @import("buffer.zig").Buffer; +const DataType = @import("dtype.zig").DataType; +const Exe = @import("exe.zig").Exe; +const Memory = @import("platform.zig").Memory; +const meta = @import("meta.zig"); +const mlirx = @import("mlirx.zig"); +const ops = @import("ops.zig"); +const pjrtx = @import("pjrtx.zig"); +const Platform = @import("platform.zig").Platform; +const tracer = @import("profiling/tracer.zig"); +const Shape = @import("shape.zig").Shape; +const Sharding = @import("Sharding.zig"); +const Tensor = @import("tensor.zig").Tensor; +const TensorStore = @import("io.zig").TensorStore; + +const Compiler = @This(); +const log = std.log.scoped(.@"zml/Compiler"); + +allocator: std.mem.Allocator, +io: std.Io, +arena: std.heap.ArenaAllocator, + +options: Options, +mlir_registry: *mlir.DialectRegistry, +mlir_ctx: *mlir.Context, +mlir_pass_manager: *mlir.PassManager, +module: *mlir.Module, +platform: *const Platform, +meshes: []const *const Sharding.Mesh, +partitioner: Sharding.Partitioner, + +mlir_known_types: std.enums.EnumArray(DataType, *const mlir.Type), + +scopes: stdx.BoundedArray(Scope, 16) = .empty, +manual_computation_depth: usize = 0, +unknown_location: *const mlir.Location, +location: *const mlir.Location, + +channel_id: i64 = 0, +composite_id: i64 = 0, + +threadlocal var _current: ?*Compiler = null; + +var mlir_global_init_mutex: std.Io.Mutex = .init; +var mlir_global_registry: ?*mlir.DialectRegistry = null; + +fn mlirRegistry(io: std.Io) *mlir.DialectRegistry { + mlir_global_init_mutex.lockUncancelable(io); + defer mlir_global_init_mutex.unlock(io); + + if (mlir_global_registry == null) { + mlir.registerPasses("Transforms"); + + const mlir_registry = mlir.DialectRegistry.init() catch unreachable; + func.registerDialect(mlir_registry); + stablehlo.registerDialect(mlir_registry); + shardy.registerDialect(mlir_registry); + func.registerExtensions(mlir_registry); + + mlir_global_registry = mlir_registry; + } + + return mlir_global_registry.?; +} + +pub const Options = struct { + /// Incremented once after successful compilation. + progress: std.Progress.Node = .none, + meshes: []const *const Sharding.Mesh = &.{}, + store: ?*const TensorStore = null, + // If null, will be initialized from the target + partitioner: ?Sharding.Partitioner = null, + /// HACK: Extra bytes added to the device memory size reported to XLA. + /// Some buffers (e.g. engram tables) live in host-pinned memory but are declared to XLA as + /// device memory, because XLA only supports a few GPU operations reading host-pinned memory. + /// XLA then counts them against device memory, leaves no budget for temporary buffers and + /// rematerializes aggressively. Callers set this to the per-device size of such buffers to + /// compensate. Remove once XLA properly supports host-pinned inputs. + extra_device_memory_bytes: u64 = 0, + // Debugging options + program_name: []const u8 = "zml", + xla_dump_to: ?[]const u8 = null, + xla_dump_fusion_visualization: bool = false, + xla_dump_hlo_pass_re: ?[]const u8 = null, + xla_dump_emitter_re: ?[]const u8 = null, +}; + +/// Errors surfaced while lowering and compiling a ZML program. +pub const Error = std.mem.Allocator.Error || + std.Io.Writer.Error || + mlir.Error || + upb.SerializeError || + pjrtx.Client.CompileError || + error{ MissingDeviceInTile, OpenShardingAtBoundaries }; + +pub const Donation = union(enum) { + implicit: u32, + explicit: u32, +}; + +pub const Scope = struct { + compiler: *Compiler, + block: *mlir.Block, + id_to_argument: std.AutoArrayHashMapUnmanaged(Tensor.Id, *const mlir.Value), + id_to_donation: std.AutoArrayHashMapUnmanaged(Tensor.Id, Donation), + id_to_memory: std.AutoArrayHashMapUnmanaged(Tensor.Id, Memory.Kind), + arena: std.heap.ArenaAllocator, + + pub fn initFromBlock(compiler: *Compiler, block: *mlir.Block) Scope { + const arena: std.heap.ArenaAllocator = .init(compiler.allocator); + return .{ + .compiler = compiler, + .block = block, + .id_to_argument = .empty, + .id_to_donation = .empty, + .id_to_memory = .empty, + .arena = arena, + }; + } + + pub fn pop(self: *Scope) void { + const compiler = self.compiler; + std.debug.assert(self == compiler.currentScope()); + var arena = self.arena; + _ = compiler.scopes.pop(); + arena.deinit(); + } + + pub fn registerTensorAsBlockArgument(scope: *Scope, id: Tensor.Id, arg_id: usize) void { + scope.id_to_argument.put(scope.arena.allocator(), id, scope.block.argument(arg_id)) catch @panic("OOM"); + } + + pub fn value(scope: *Scope, x: Tensor) *const mlir.Value { + return scope.id_to_argument.get(x.id) orelse + x._value orelse + @panic("Something went really wrong, tensor is not an argument nor has an mlir.Value"); + } +}; + +pub fn init(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, opts: Options) Compiler { + var arena = std.heap.ArenaAllocator.init(allocator); + const mlir_registry = mlirRegistry(io); + var mlir_ctx = mlir.Context.init(.{ .registry = mlir_registry, .threading = false }) catch unreachable; + mlir_ctx.loadAllAvailableDialects(); + + const unknown_location: *const mlir.Location = .unknown(mlir_ctx); + const module = mlir.Module.init(unknown_location); + module.operation().setAttributeByName("sym_name", .string(mlir_ctx, opts.program_name)); + + const pass_manager = mlir.PassManager.init(mlir_ctx); + { + var opm = pass_manager.asOpPassManager(); + const passes: []const []const u8 = &.{ + "canonicalize", + "cse", + "canonicalize", + }; + for (passes) |pass| { + opm.addPipeline(pass) catch unreachable; + } + } + + var mlir_known_types: std.enums.EnumArray(DataType, *const mlir.Type) = .initUndefined(); + { + for (0.., &mlir_known_types.values) |i, *mlir_type| { + mlir_type.* = mlirx.Type.fromDType(mlir_ctx, @fromBackingInt(@intCast(i))); + } + } + + var mesh_list: std.ArrayList(*const Sharding.Mesh) = .empty; + mesh_list.appendSlice(arena.allocator(), opts.meshes) catch @panic("OOM"); + if (std.mem.indexOfScalar(*const Sharding.Mesh, mesh_list.items, platform.replicated_mesh) == null) { + mesh_list.append(arena.allocator(), platform.replicated_mesh) catch @panic("OOM"); + } + const meshes = mesh_list.items; + validateMeshes(meshes) catch |err| stdx.debug.panic("Incompatible meshes: {t}", .{err}); + + return .{ + .allocator = allocator, + .io = io, + .arena = arena, + .mlir_registry = mlir_registry, + .mlir_ctx = mlir_ctx, + .mlir_pass_manager = pass_manager, + .mlir_known_types = mlir_known_types, + .module = module, + .platform = platform, + .options = opts, + .partitioner = opts.partitioner orelse .fromTarget(platform.target), + .meshes = meshes, + .location = unknown_location, + .unknown_location = unknown_location, + }; +} + +pub fn deinit(self: *Compiler) void { + if (_current == self) _current = null; + std.debug.assert(self.scopes.len == 0); + self.mlir_pass_manager.deinit(); + self.module.deinit(); + self.mlir_ctx.deinit(); + self.arena.deinit(); +} + +pub fn activate(self: *Compiler) void { + std.debug.assert(_current == null); + _current = self; +} + +pub fn deactivate(self: *Compiler) void { + _ = self; + _current = null; +} + +pub fn current() *Compiler { + return currentOrNull().?; +} + +pub fn currentOrNull() ?*Compiler { + return _current; +} + +pub fn currentScope(self: *Compiler) *Scope { + return &self.scopes.slice()[self.scopes.len - 1]; +} + +pub fn pushBlock(self: *Compiler, block: *mlir.Block) *Scope { + const scope = Scope.initFromBlock(self, block); + self.scopes.appendAssumeCapacity(scope); + return self.currentScope(); +} + +/// Change `Compiler.location` by appending a frame to the current stack of locations +/// Must be followed by a `defer compiler.popLocation` +pub fn pushLocation(self: *Compiler, location: std.builtin.SourceLocation, name: []const u8) void { + var callee: *const mlir.Location = .fromSrc(self.mlir_ctx, location); + if (name.len > 0) callee = callee.named(self.mlir_ctx, name); + self.location = if (self.location == self.unknown_location) callee else .callSite(callee, self.location); +} + +/// see `zml.Compiler.pushLocation` +pub fn pushLocationFmt(self: *Compiler, location: std.builtin.SourceLocation, comptime fmt: []const u8, args: anytype) void { + const name = std.fmt.allocPrint(self.allocator, fmt, args) catch { + return self.pushLocation(location, ""); + }; + defer self.allocator.free(name); + + return self.pushLocation(location, name); +} + +/// Remove last call frame pushed by `zml.Compiler.pushLocation` +pub fn popLocation(self: *Compiler) void { + switch (self.location.inspect()) { + .named => { + // We only pushed a named, when there was no parent location + self.location = self.unknown_location; + return; + }, + .callsite => |callsite| { + self.location = callsite.caller(); + return; + }, + .unknown, .file_line_col, .fused => @panic("unexpected location metadata"), + } +} + +test pushLocation { + const mlir_registry = mlirRegistry(std.testing.io); + var mlir_ctx = mlir.Context.init(.{ .registry = mlir_registry, .threading = false }) catch unreachable; + mlir_ctx.loadAllAvailableDialects(); + + const unknown_location: *const mlir.Location = .unknown(mlir_ctx); + + var compiler: Compiler = .{ + .allocator = std.testing.allocator, + .io = std.testing.io, + .arena = undefined, + .mlir_registry = mlir_registry, + .mlir_ctx = mlir_ctx, + .mlir_pass_manager = undefined, + .mlir_known_types = undefined, + .module = undefined, + .options = .{}, + .platform = undefined, + .partitioner = undefined, + .meshes = undefined, + .location = unknown_location, + .unknown_location = unknown_location, + }; + + compiler.pushLocation(@src(), "loc1"); + const loc1 = compiler.location; + + compiler.pushLocation(@src(), "loc2"); + compiler.popLocation(); + + try std.testing.expectEqual(loc1, compiler.location); + compiler.popLocation(); + + try std.testing.expectEqual(unknown_location, compiler.location); +} + +pub fn nextChannelId(self: *Compiler) i64 { + self.channel_id += 1; + return self.channel_id; +} + +pub fn nextCompositeId(self: *Compiler) i64 { + self.composite_id += 1; + return self.composite_id; +} + +pub fn mlirType(self: *const Compiler, dt: DataType) *const mlir.Type { + return self.mlir_known_types.get(dt); +} + +pub fn dtype(self: *const Compiler, mlir_type: *const mlir.Type) DataType { + @setRuntimeSafety(false); + for (0.., &self.mlir_known_types.values) |i, known_type| { + if (known_type == mlir_type) return @fromBackingInt(@intCast(i)); + } + std.debug.panic("Can't convert unknown mlir type to dtype: {f}", .{mlir_type}); +} + +pub fn abortOOM(self: *Compiler) noreturn { + _ = self; + @panic("OOM"); +} + +pub fn alloc(self: *Compiler, T: type, n: usize) []T { + return self.arena.allocator().alloc(T, n) catch self.abortOOM(); +} + +pub fn allocPrint(self: *Compiler, comptime fmt: []const u8, args: anytype) []u8 { + return std.fmt.allocPrint(self.arena.allocator(), fmt, args) catch self.abortOOM(); +} + +/// Lookup a mesh by it's name +pub fn mesh(compiler: *const Compiler, name: @EnumLiteral()) *const Sharding.Mesh { + const name_slice = @tagName(name); + for (compiler.meshes) |m| { + if (std.mem.eql(u8, name_slice, m.name)) { + return m; + } + } + if (name == .replicated) return compiler.platform.replicated_mesh; + std.debug.panic( + \\Found no meshes named {s}. + \\Try passing more meshes to `zml.compile`. + \\Known meshes: {f} + , .{ name_slice, stdx.fmt.slice(compiler.meshes) }); +} + +/// Find a mesh that covers all the given axes +pub fn resolveMesh(compiler: *const Compiler, logical_axes: anytype) *const Sharding.Mesh { + if (@TypeOf(logical_axes) != []const Shape.Tag) { + return compiler.resolveMesh(comptime Shape.parseTags(logical_axes)); + } + + var ok_meshe: ?*const Sharding.Mesh = null; + for (compiler.meshes) |shd| { + var covers_all: bool = true; + for (logical_axes) |input_axis| { + var covers_this: bool = false; + for (shd.logical.axes.slice()) |existing_axis| { + if (std.mem.eql(u8, std.mem.span(existing_axis), std.mem.span(input_axis))) { + covers_this = true; + break; + } + } + covers_all = covers_all and covers_this; + } + if (covers_all) { + if (ok_meshe) |first_match| { + std.debug.panic( + \\Found two meshes covering axes: {any}, expected exacty one. + \\- First match: {f} + \\- Second match: {f} + , .{ logical_axes, first_match, shd }); + } + ok_meshe = shd; + } + } + + return ok_meshe orelse std.debug.panic( + \\Found no meshes covering axes: {any}, expected exacty one. + \\Try passing more meshes to `zml.compile`. + \\Known meshes: {f} + , .{ logical_axes, stdx.fmt.slice(compiler.meshes) }); +} + +pub fn Typed(comptime func_: anytype) type { + return struct { + pub fn compile( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + opts: Options, + args: std.meta.ArgsTuple(@TypeOf(func_)), + ) Error!Exe { + return Compiler.compile(allocator, io, platform, func_, args, opts); + } + }; +} + +pub fn compile( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + comptime func_: anytype, + args: std.meta.ArgsTuple(@TypeOf(func_)), + opts: Options, +) Error!Exe { + return switch (platform.io_impl) { + .threaded => compileInternal(allocator, io, platform, func_, args, opts), + .zio => try zio.blockInPlace(struct { + fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: *const Platform, args_: @TypeOf(args), opts_: Options) Error!Exe { + return try compileInternal(allocator_, io_, platform_, func_, args_, opts_); + } + }.call, .{ allocator, io, platform, args, opts }), + }; +} + +pub fn compileInternal( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + comptime func_: anytype, + args: std.meta.ArgsTuple(@TypeOf(func_)), + opts: Options, +) Error!Exe { + // TODO: Here we have somewhat of a requirement + // Emitting MLIR requires to have the compiler context available at all times using `Compiler.current()`. + // If in the future, we inject an Io that is not thread-based, we might have some surprises. + // + // I think the correct implementation would be to dispatch `emitMlir` to a thread pool, then wait for the result + // asynchronously using the provided Io. For now, we'll simply make that blocking as it's not a big deal but keep + // in mind we might want to revisit that later. + // + // This is NOT compatible with std.Io.Evented. + // It is compatible with zio, hence the blockInPlace above. + + const span_name = try tracer.formatSpanName(allocator, "zml.module.compile", .{ + .program_name = opts.program_name, + .arg_count = args.len, + }); + + defer allocator.free(span_name); + var span = tracer.Span.start(span_name); + defer span.end(); + + var compiler: Compiler = .init(allocator, io, platform, opts); + defer compiler.deinit(); + + var result = try compiler.emitMlir(func_, args); + defer result.output_info.deinit(compiler.allocator); + defer result.input_info.deinit(compiler.allocator); + + try addPartitionerOperations(&compiler); + + _ = result.func.appendTo(compiler.module.body()); + + // Mesh replication is encoded in shardings within one SPMD executable. + const num_partitions = compiler.meshes[0].numDevices(); + const num_replicas = 1; + const num_devices = num_partitions * num_replicas; + + compiler.module.operation().setAttributeByName( + "mhlo.num_partitions", + .int(compiler.mlir_ctx, .i32, num_partitions), + ); + compiler.module.operation().setAttributeByName( + "mhlo.num_replicas", + .int(compiler.mlir_ctx, .i32, num_replicas), + ); + + compiler.mlir_pass_manager.runOnOp(compiler.module.operation()) catch |err| switch (err) { + error.MlirUnexpected => { + compiler.handleCompilationError(io, compiler.arena.allocator(), opts, err, "Failed to canonicalize invalid MLIR"); + @panic("ZML generated invalid mlir. Please open a bug report"); + }, + }; + + const loaded_executable = compiler.compileModuleToPjrtExecutable(opts) catch |err| { + compiler.handleCompilationError(io, compiler.arena.allocator(), opts, err, "Pjrt failed to compile the following MLIR"); + return err; + }; + + log.debug("\n******** ZML generated MLIR ********\n{f}", .{compiler.module.operation()}); + + const exe = try Exe.init( + allocator, + platform, + loaded_executable, + num_devices, + num_partitions, + // This will get copied into exe + result.input_info.items(.shape), + result.output_info.items(.shape), + result.input_info.items(.aliasing_output), + ); + errdefer exe.deinit(); + + opts.progress.completeOne(); + return exe; +} + +/// Declare all logical meshes used by this program +fn addPartitionerOperations(ctx: *Compiler) !void { + const mlir_ctx = ctx.mlir_ctx; + const module = ctx.module; + + switch (ctx.partitioner) { + .gspmd => { + // GSPMD doesn't use meshes, but explicit partitioning operations. + }, + .shardy => { + for (ctx.meshes) |shd| { + _ = shardy.mesh(mlir_ctx, shd.name, shd.sdyMeshAttr(mlir_ctx), .unknown(mlir_ctx)).appendTo(module.body()); + } + }, + } +} + +const EmitMlirResult = struct { + func: *mlir.Operation, + input_info: std.MultiArrayList(TensorInfo), + output_info: std.MultiArrayList(TensorInfo), +}; + +pub const TensorInfo = struct { + id: Tensor.Id, + shape: Shape, + value: *const mlir.Value, + + // Only used for input tensors, stores which output tensor ends up with their buffer + aliasing_output: ?u32 = null, + + pub fn attributes(info: TensorInfo, arena: std.mem.Allocator, compiler: *const Compiler, scope: *const Scope) error{OutOfMemory}!AttributeList { + var attrs: AttributeList = .empty; + + const mlir_ctx = compiler.mlir_ctx; + const mesh_attr = try compiler.tensorShardingAttr(arena, info.shape); + attrs.appendAssumeCapacity(switch (compiler.partitioner) { + .gspmd => .named(mlir_ctx, "mhlo.sharding", mesh_attr), + .shardy => shardy.shardingNamedAttr(mlir_ctx, mesh_attr.isA(shardy.TensorShardingAttr).?), + }); + + const memory = scope.id_to_memory.get(info.id) orelse .device; + if (memory != .device) { + attrs.appendAssumeCapacity(.named( + mlir_ctx, + "mhlo.memory_kind", + .string(mlir_ctx, compiler.platform.memoryKind(memory)), + )); + } + + if (info.aliasing_output) |output_index| { + attrs.appendAssumeCapacity( + .named(mlir_ctx, "tf.aliasing_output", .int(mlir_ctx, .i32, output_index)), + ); + } + + return attrs; + } +}; + +const AttributeList = stdx.BoundedArray(mlir.NamedAttribute, 3); + +fn emitMlir(compiler: *Compiler, comptime func_: anytype, args: std.meta.ArgsTuple(@TypeOf(func_))) !EmitMlirResult { + const module = mlir.Module.init(.unknown(compiler.mlir_ctx)); + errdefer module.deinit(); + + const block = mlir.Block.init(&.{}, &.{}); + errdefer block.deinit(); + + const fn_scope: *Scope = compiler.pushBlock(block); + defer fn_scope.pop(); + + var input_info = try createBlockArguments(compiler, fn_scope, &args); + errdefer input_info.deinit(compiler.allocator); + + var output_info = blk: { + compiler.activate(); + defer compiler.deactivate(); + + var result = @call(.auto, func_, args); + // Output collection can emit replication constraints, so it is part of tracing. + break :blk try collectOutputInfo(compiler, fn_scope, &result); + }; + errdefer output_info.deinit(compiler.allocator); + + // Finalize in a separate function that doesn't depend on `func` type. + return try finalizeMlirFunc(compiler, fn_scope, input_info, output_info); +} + +fn createBlockArguments(compiler: *Compiler, scope: *Scope, v: anytype) error{ OutOfMemory, OpenShardingAtBoundaries }!std.MultiArrayList(TensorInfo) { + const CreateBlockArgumentsCb = struct { + compiler: *Compiler, + scope: *Scope, + + current_argument_id: u32 = 0, + infos: std.MultiArrayList(TensorInfo) = .empty, + + fn cb(ctx: *@This(), tensor: *const Tensor) error{ OutOfMemory, OpenShardingAtBoundaries }!void { + // Declare the program argument as a packed u8 + const og_shape = tensor._shape; + if (og_shape.containsPartitionSpec(.open)) { + log.err("Executable input {d} has open sharding: {f}. Specify a logical mesh axis or .replicated for every dimension.", .{ ctx.current_argument_id, og_shape }); + return error.OpenShardingAtBoundaries; + } + const packed_shape = og_shape.packedShape(); + const mlir_ctx = ctx.compiler.mlir_ctx; + const mlir_type = mlirx.Type.rankedTensor(mlir_ctx, packed_shape); + var location: *const mlir.Location = ctx.compiler.unknown_location; + if (ctx.compiler.options.store) |store| { + if (store.id_to_sources.get(tensor.id)) |sources| { + location = location.named(mlir_ctx, sources.tensors[0].name); + } + } + var value = ctx.scope.block.addArgument(mlir_type, location); + + // But immediately create the MLIR value corresponding to the fp4 data. + // That way the user code only sees a proper fp4 tensor. + if (tensor.dtype().bitSizeOf() < 8) { + value = unpack(ctx.compiler, ctx.scope, og_shape, value); + } + const gop = ctx.scope.id_to_argument.getOrPutValue(ctx.scope.arena.allocator(), tensor.id, value) catch @panic("OOM"); + if (gop.found_existing) std.debug.panic("Tensor with id {} has already been used once as an argument", .{tensor.id}); + + // Associate each input argument with their own buffer + ctx.scope.id_to_donation.putNoClobber(ctx.scope.arena.allocator(), tensor.id, .{ .implicit = ctx.current_argument_id }) catch @panic("OOM"); + + defer ctx.current_argument_id += 1; + + try ctx.infos.append(ctx.compiler.allocator, .{ + .id = tensor.id, + .shape = og_shape, + .value = value, + }); + } + }; + + var context: CreateBlockArgumentsCb = .{ .compiler = compiler, .scope = scope }; + errdefer context.infos.deinit(compiler.allocator); + + try meta.visit(CreateBlockArgumentsCb.cb, &context, v); + + return context.infos; +} + +fn collectOutputInfo(compiler: *Compiler, scope: *Scope, v: anytype) error{ OutOfMemory, OpenShardingAtBoundaries }!std.MultiArrayList(TensorInfo) { + const CollectOutputInfoCb = struct { + compiler: *Compiler, + scope: *Scope, + + infos: std.MultiArrayList(TensorInfo) = .empty, + + fn cb(ctx: *@This(), tensor: *const Tensor) !void { + // Outputs with no declared sharding default to replication. Constrain the + // returned value so the partitioner produces a replicated output buffer. + const output = if (tensor.shape()._sharding.mesh != null or tensor.shape().isFullyReplicated()) tensor.* else tensor.replicate(); + const og_shape = output.shape(); + if (og_shape.containsPartitionSpec(.open)) { + log.err("Executable output {d} has open sharding: {f}. Specify a logical mesh axis or .replicated for every dimension.", .{ ctx.infos.len, og_shape }); + return error.OpenShardingAtBoundaries; + } + var value = ctx.scope.id_to_argument.get(output.id) orelse + output._value orelse + @panic("no value found for output tensor"); + + if (tensor.dtype().bitSizeOf() < 8) { + value = repack(ctx.compiler, ctx.scope, og_shape, value); + } + + try ctx.infos.append(ctx.compiler.allocator, .{ + .id = tensor.id, + // const packed_shape = og_shape.packedShape(); + // TODO: clarify why this og_shape and not packedShape() + .shape = og_shape, + .value = value, + }); + } + }; + + var context: CollectOutputInfoCb = .{ .compiler = compiler, .scope = scope }; + errdefer context.infos.deinit(compiler.allocator); + + try meta.visit(CollectOutputInfoCb.cb, &context, v); + return context.infos; +} + +fn finalizeMlirFunc(compiler: *Compiler, fn_scope: *Scope, input_info: std.MultiArrayList(TensorInfo), output_info: std.MultiArrayList(TensorInfo)) error{OutOfMemory}!EmitMlirResult { + var arena_state = std.heap.ArenaAllocator.init(compiler.allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + const mlir_ctx = compiler.mlir_ctx; + + const fn_return = func.returns(mlir_ctx, output_info.items(.value), .unknown(mlir_ctx)); + _ = fn_return.appendTo(fn_scope.block); + + // Resolve donations + for (0.., output_info.items(.id)) |output_index, output_id| { + if (fn_scope.id_to_donation.get(output_id)) |donation| { + const donated_input = switch (donation) { + // don't emit implicit donation since they modify the way the function is called + .implicit => continue, + .explicit => |input| input, + }; + const aliasing_output: *?u32 = &input_info.items(.aliasing_output)[donated_input]; + if (aliasing_output.*) |previous_aliased_output| { + std.debug.panic("Input {d} buffer {} was reused twice with `reuseBuffer` for output {} and output {}. Expected `reuseBuffer` to be called at most once", .{ donated_input, input_info.items(.shape)[donated_input], previous_aliased_output, output_index }); + } + aliasing_output.* = @intCast(output_index); + } + } + + // Input mesh/memory/aliasing attributes + const input_attributes = try arena.alloc(*const mlir.Attribute, input_info.len); + for (0.., input_attributes) |i, *input_attrs| { + const attrs_list = try input_info.get(i).attributes(arena, compiler, fn_scope); + input_attrs.* = .dict(mlir_ctx, attrs_list.constSlice()); + } + + // Output mesh/memory attributes + const output_attributes = try arena.alloc(*const mlir.Attribute, output_info.len); + for (0.., output_attributes) |i, *output_attrs| { + const attrs_list = try output_info.get(i).attributes(arena, compiler, fn_scope); + output_attrs.* = .dict(mlir_ctx, attrs_list.constSlice()); + } + + const input_types = try arena.alloc(*const mlir.Type, input_info.len); + for (input_types, 0..) |*input_type, i| input_type.* = fn_scope.block.argument(i).type_(); + + const output_types = try arena.alloc(*const mlir.Type, output_info.len); + for (output_types, output_info.items(.value)) |*output_type, value| output_type.* = value.type_(); + + const mlir_func = func.func(mlir_ctx, .{ + .name = "main", + .args = input_types, + .results = output_types, + .block = fn_scope.block, + .location = .unknown(mlir_ctx), + .args_attributes = input_attributes, + .results_attributes = output_attributes, + .verify = false, + }); + + return .{ + .func = mlir_func, + .input_info = input_info, + .output_info = output_info, + }; +} + +fn unpack(compiler: *Compiler, scope: *Scope, og_shape: Shape, tensor_value: *const mlir.Value) *const mlir.Value { + const true_type = mlirx.Type.rankedTensor(compiler.mlir_ctx, og_shape); + const elem_per_bytes = @divExact(8, og_shape.dtype().bitSizeOf()); + const intermediary_type = mlirx.Type.rankedTensor(compiler.mlir_ctx, og_shape.splitAxis(-1, .{ -1, elem_per_bytes })); + + const bit_cast_op = stablehlo.bitcast_convert( + compiler.mlir_ctx, + tensor_value, + intermediary_type, + .unknown(compiler.mlir_ctx), + ).appendTo(scope.block); + + const reshape_op = stablehlo.reshape( + compiler.mlir_ctx, + bit_cast_op.result(0), + true_type, + .unknown(compiler.mlir_ctx), + ).appendTo(scope.block); + + return reshape_op.result(0); +} + +fn repack(compiler: *Compiler, scope: *Scope, og_shape: Shape, value: *const mlir.Value) *const mlir.Value { + const packed_type = mlirx.Type.rankedTensor(compiler.mlir_ctx, og_shape.packedShape()); + const elem_per_bytes = @divExact(8, og_shape.dtype().bitSizeOf()); + const intermediary_type = mlirx.Type.rankedTensor(compiler.mlir_ctx, og_shape.splitAxis(-1, .{ -1, elem_per_bytes })); + + const reshape_op = stablehlo.reshape( + compiler.mlir_ctx, + value, + intermediary_type, + .unknown(compiler.mlir_ctx), + ).appendTo(scope.block); + + const bit_cast_op = stablehlo.bitcast_convert( + compiler.mlir_ctx, + reshape_op.result(0), + packed_type, + .unknown(compiler.mlir_ctx), + ).appendTo(scope.block); + + return bit_cast_op.result(0); +} + +fn validateMeshes(meshes: []const *const Sharding.Mesh) !void { + stdx.debug.assert(meshes.len >= 1, "Waiting at leat 1 mesh strategy to be implemented", .{}); + + for (meshes) |m| { + const num_replicas = m.numReplicas(); + if (num_replicas > 1 and !std.mem.eql(u8, "replicated", m.name)) { + // This seems like a performance footgun, rather that something anyone would want to use. + // It's generally better to have meshes that cover all devices, + // even if later it means that some mesh axis is never used. + // Otherwise a "sharded" tensor can in fact be replicated because the underlying mesh has replicas + // TODO(#914): decide if we should forbid this altogether + // currently DirectMemoryWriter tests have replicated meshes depending on the platform they target + log.warn("Mesh({s}) has numReplicas() == {d}. This imply that all tensor put on this mesh will be implicitly replicated {d} times.\n{f}", .{ m.name, num_replicas, num_replicas, m }); + } + log.debug("Using {f}", .{m}); + } + + const first = meshes[0]; + const devices = first.numDevices(); + + for (meshes[1..]) |m| { + if (m.numDevices() != devices) { + // todo: deviceAssignments should also be checked for consistency here, but for simplicity we just check the cardinality numbers + return error.InconsistentMesheCardinality; + } + } +} + +pub fn tensorShardingAttr( + compiler: *const Compiler, + allocator: std.mem.Allocator, + shape: Shape, +) error{OutOfMemory}!*const mlir.Attribute { + const mesh_ = shape._sharding.mesh orelse compiler.platform.replicated_mesh; + return switch (compiler.partitioner) { + .shardy => (try mesh_.sdyShardingAttrForShape(allocator, compiler.mlir_ctx, shape)).attribute(), + .gspmd => mesh_.gspmdShardingAttrForShape(allocator, compiler.mlir_ctx, shape) catch |err| switch (err) { + error.WriteFailed => error.OutOfMemory, // We're writing to memory + error.OutOfMemory => error.OutOfMemory, + error.IncompatibleSharding => @panic("Shape dimensions must be divisible by their mesh partitions"), + }, + }; +} + +fn setXlaOverrideFlag(map: *c.upb_Map, flag: []const u8, value: anytype, upb_arena: *c.upb_Arena) !void { + const result = c.upb_Map_Set( + map, + .{ .str_val = upb.stringView(flag) }, + .{ .msg_val = blk: { + const field = try upb.new(c.xla_OptionOverrideProto, upb_arena); + switch (@typeInfo(@TypeOf(value))) { + .bool => c.xla_OptionOverrideProto_set_bool_field(field, value), + .comptime_int, .int => c.xla_OptionOverrideProto_set_int_field(field, @intCast(value)), + .comptime_float, .float => c.xla_OptionOverrideProto_set_double_field(field, @floatCast(value)), + else => c.xla_OptionOverrideProto_set_string_field(field, upb.stringView(value)), + } + break :blk @ptrCast(field); + } }, + upb_arena, + ); + + if (result == false) { + return std.mem.Allocator.Error.OutOfMemory; + } +} + +fn compileModuleToPjrtExecutable(compiler: *Compiler, opts: Options) !*pjrt.LoadedExecutable { + const use_shardy_partitioner = switch (compiler.partitioner) { + .shardy => true, + .gspmd => false, + }; + + const platform = compiler.platform; + const main_mesh = compiler.meshes[0]; + // Mesh replicas are part of the SPMD device set, not XLA program replicas. + const num_partitions: i32 = @intCast(main_mesh.numDevices()); + const num_replicas = 1; + + const device_assignment = try main_mesh.deviceAssignment(compiler.allocator); + defer compiler.allocator.free(device_assignment); + + var arena = std.heap.ArenaAllocator.init(compiler.allocator); + defer arena.deinit(); + var upb_alloc: upb.Allocator = .init(arena.allocator()); + const upb_arena = c.upb_Arena_Init(null, 0, upb_alloc.inner()); + defer c.upb_Arena_Free(upb_arena); + const options = blk: { + const options = try upb.new(c.xla_CompileOptionsProto, upb_arena); + c.xla_CompileOptionsProto_set_executable_build_options(options, executable_build_options_blk: { + const exec_build_options = try upb.new(c.xla_ExecutableBuildOptionsProto, upb_arena); + c.xla_ExecutableBuildOptionsProto_set_device_ordinal(exec_build_options, -1); + c.xla_ExecutableBuildOptionsProto_set_num_replicas(exec_build_options, num_replicas); + c.xla_ExecutableBuildOptionsProto_set_num_partitions(exec_build_options, num_partitions); + c.xla_ExecutableBuildOptionsProto_set_use_spmd_partitioning(exec_build_options, true); + c.xla_ExecutableBuildOptionsProto_set_use_shardy_partitioner(exec_build_options, use_shardy_partitioner); + + // Tell XLA how much device memory PJRT actually made available. + // Without this, the GPU scheduler falls back to 80% of physical + // memory. Programs whose donated inputs and outputs exceed that + // artificial limit get a zero-byte temporary-memory budget and + // can trigger excessive rematerialization. + var device_memory_size: ?u64 = null; + for (platform.devices) |device| { + const bytes_limit = device.memoryStats().bytes_limit orelse { + device_memory_size = null; + break; + }; + device_memory_size = @min(device_memory_size orelse bytes_limit, bytes_limit); + } + if (device_memory_size) |bytes_limit| { + // HACK: See `Options.extra_device_memory_bytes`. + c.xla_ExecutableBuildOptionsProto_set_device_memory_size(exec_build_options, @intCast(bytes_limit + opts.extra_device_memory_bytes)); + } + + c.xla_ExecutableBuildOptionsProto_set_device_assignment(exec_build_options, device_assignment_blk: { + const device_assignment_proto = try upb.new(c.xla_DeviceAssignmentProto, upb_arena); + + c.xla_DeviceAssignmentProto_set_replica_count(device_assignment_proto, num_replicas); + c.xla_DeviceAssignmentProto_set_computation_count(device_assignment_proto, num_partitions); + + const computation_devices = c.xla_DeviceAssignmentProto_resize_computation_devices( + device_assignment_proto, + @intCast(num_partitions), + upb_arena, + ); + + for (computation_devices[0..@intCast(num_partitions)], 0..) |*computation_device, i| { + computation_device.* = try upb.new(c.xla_DeviceAssignmentProto_ComputationDevice, upb_arena); + _ = c.xla_DeviceAssignmentProto_ComputationDevice_add_replica_device_ids( + computation_device.*, + @intCast(device_assignment[@intCast(i)]), + upb_arena, + ); + } + + break :device_assignment_blk device_assignment_proto; + }); + + break :executable_build_options_blk exec_build_options; + }); + + const overrides_map = c._xla_CompileOptionsProto_env_option_overrides_mutable_upb_map(options, upb_arena); + switch (platform.target) { + .cuda => { + // NVIDIA recommends these settings + // https://github.com/NVIDIA/JAX-Toolbox?tab=readme-ov-file#environment-variables + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_latency_hiding_scheduler", true, upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_experimental_scaled_dot_with_tile_ir", true, upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_experimental_enable_subchannel_dequantisation_fusion", true, upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_unsupported_enable_triton_multi_output_fusion", true, upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_command_buffer_scheduling_mode", "CONCURRENT", upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_command_buffer_update_mode", "SKIP_TEMP", upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_experimental_use_collective_kernels", "COLLECTIVE_KERNEL_ALL_REDUCE", upb_arena); + // Add collectives to the default list + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_command_buffer", "COLLECTIVES_KERNEL,COLLECTIVES,CUBLAS,CUBLASLT,CUDNN,CUSTOM_CALL,DYNAMIC_SLICE_FUSION,FUSION", upb_arena); + // With SKIP_TEMP, captured NCCL collectives can retain stale physical mappings + // if VMM reclaims their backing under memory pressure, causing silent corruption. + // Enable user buffers to assign color 1 and receive the reclaim exemption: + // https://github.com/openxla/xla/pull/46029 + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_nccl_user_buffers", true, upb_arena); + // Enable for both Blackwell+ and Ampere+ + // try setXlaOverrideFlag(overrides_map, "xla_gpu_cudnn_gemm_fusion_level", 2, upb_arena); + }, + .rocm => { + try setXlaOverrideFlag(overrides_map, "xla_gpu_command_buffer_scheduling_mode", "CONCURRENT", upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_command_buffer_update_mode", "SKIP_TEMP", upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_experimental_use_collective_kernels", "COLLECTIVE_KERNEL_ALL_REDUCE", upb_arena); + // Only capture all-reduce and all-gather collectives in HIP graphs; other collective types execute outside command buffers since they are unsupported in HIP graphs. + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_collectives_command_buffer_filter", "ALLREDUCE,ALLGATHER", upb_arena); + // Add collectives to the default list + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_command_buffer", "COLLECTIVES_KERNEL,COLLECTIVES,CUBLAS,CUBLASLT,CUDNN,CUSTOM_CALL,DYNAMIC_SLICE_FUSION,FUSION", upb_arena); + // With SKIP_TEMP, captured NCCL collectives can retain stale physical mappings + // if VMM reclaims their backing under memory pressure, causing silent corruption. + // Enable user buffers to assign color 1 and receive the reclaim exemption: + // https://github.com/openxla/xla/pull/46029 + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_nccl_user_buffers", true, upb_arena); + // Enables fusion to produce output for multiple separate output. + try setXlaOverrideFlag(overrides_map, "xla_gpu_unsupported_enable_triton_multi_output_fusion", true, upb_arena); + // Rearranges operations so that communication between GPUs happens in the background while the compute units are busy doing computation. + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_latency_hiding_scheduler", true, upb_arena); + }, + .metal => { + try setXlaOverrideFlag(overrides_map, "xla_gpu_ftz", true, upb_arena); + }, + .oneapi => { + // More efficient for the allgather/broadcast implementation of the collective permute. + try setXlaOverrideFlag(overrides_map, "xla_gpu_collective_permute_connected_components", true, upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_autotune_level", 0, upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_command_buffer", "CUBLASLT,CUSTOM_CALL,FUSION", upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_command_buffer_scheduling_mode", "CONCURRENT", upb_arena); + + // Grouped GEMM still needs a oneDNN lowering. + try setXlaOverrideFlag(overrides_map, "xla_gpu_experimental_use_ragged_dot_grouped_gemm", false, upb_arena); + }, + else => {}, + } + + if (opts.xla_dump_to) |xla_dump_to| { + try setXlaOverrideFlag(overrides_map, "xla_dump_to", xla_dump_to, upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_dump_hlo_as_proto", true, upb_arena); + if (opts.xla_dump_fusion_visualization) { + try setXlaOverrideFlag(overrides_map, "xla_dump_fusion_visualization", true, upb_arena); + } + if (opts.xla_dump_hlo_pass_re) |re| { + try setXlaOverrideFlag(overrides_map, "xla_dump_hlo_pass_re", re, upb_arena); + } + if (opts.xla_dump_emitter_re) |re| { + try setXlaOverrideFlag(overrides_map, "xla_dump_emitter_re", re, upb_arena); + } + } + + switch (platform.target) { + .rocm, .cuda, .oneapi => if (std.c.getenv("ZML_AUTOTUNE_CACHE_DIR")) |path| { + try setXlaOverrideFlag(overrides_map, "xla_gpu_per_fusion_autotune_cache_dir", std.mem.span(path), upb_arena); + }, + else => {}, + } + + break :blk options; + }; + + const loaded_executable = try pjrtx.Client.compile( + platform.pjrt_client, + platform.pjrt_api, + compiler.allocator, + compiler.io, + compiler.module, + try upb.serialize(options, upb_arena), + ); + errdefer loaded_executable.deinit(); + + return loaded_executable; +} + +fn handleCompilationError(compiler: *Compiler, io: std.Io, allocator: std.mem.Allocator, opts: Options, err: anyerror, msg: []const u8) void { + const module_op = compiler.module.operation().fmt(.{ .debug_info = true, .debug_info_pretty_form = true }); + const truncated_msg = "..."; + var too_big: bool = true; + + const buffer: []u8 = allocator.alloc(u8, 8192) catch { + log.err("{s} ({}):\n{f}", .{ msg, err, module_op }); + return; + }; + defer allocator.free(buffer); + + var buffer_writer: std.Io.Writer = .fixed(buffer); + const module_mlir: []const u8 = module_mlir: { + module_op.format(&buffer_writer) catch { + @memcpy(buffer[8192 - truncated_msg.len ..], truncated_msg); + too_big = true; + break :module_mlir buffer; + }; + too_big = false; + break :module_mlir buffer_writer.buffered(); + }; + + log.err("{s} ({}):\n{s}", .{ msg, err, module_mlir }); + if (!too_big) return; + + if (opts.xla_dump_to == null) { + log.warn("To see the full .mlir, set `zml.Compiler.Options.xla_dump_to`", .{}); + return; + } + const xla_dir = opts.xla_dump_to.?; + const dir = std.Io.Dir.openDirAbsolute(io, xla_dir, .{}) catch dir: { + std.Io.Dir.createDirAbsolute(io, xla_dir, .default_dir) catch |e| { + log.err("failed to dump mlir to {s}: {}", .{ xla_dir, e }); + return; + }; + break :dir std.Io.Dir.openDirAbsolute(io, xla_dir, .{}) catch unreachable; + }; + const filename = std.fmt.allocPrint(allocator, "{s}.mlir", .{opts.program_name}) catch return; + const mlir_file = dir.createFile(io, filename, .{ .truncate = true }) catch |e| { + log.err("failed to dump mlir to {s}/{s}: {}", .{ xla_dir, filename, e }); + return; + }; + + // Reuse the previously allocated buffer + var mlir_file_writer = mlir_file.writer(io, buffer); + module_op.format(&mlir_file_writer.interface) catch |e| { + log.err("Partial mlir dump at: {s}/{s} ({})", .{ xla_dir, filename, e }); + return; + }; + log.warn("Full .mlir at: {s}/{s}", .{ xla_dir, filename }); +} diff --git a/zml/Sharding.zig b/zml/Sharding.zig new file mode 100644 index 00000000..040216bc --- /dev/null +++ b/zml/Sharding.zig @@ -0,0 +1,3290 @@ +//! Explain how to split a buffer across different devices. +const std = @import("std"); +const builtin = @import("builtin"); + +const func = @import("mlir/dialects/func"); +const mlir = @import("mlir"); +const pjrt = @import("pjrt"); +const platforms = @import("platforms"); +const shardy = @import("mlir/dialects/shardy"); +const stdx = @import("stdx"); + +const Platform = @import("platform.zig").Platform; +const PlatformDevice = @import("platform.zig").Device; +const Shape = @import("shape.zig").Shape; +const Slice = @import("slice.zig").Slice; +const Target = @import("platform.zig").Target; + +const Sharding = @This(); + +mesh: ?*const Mesh, +partition: Partitioning, + +pub const MAX_MESH_RANK = 4; + +/// Full replication across participating devices. +/// For buffers, each device holds a complete copy. Applied as a graph constraint, +/// this closes every dimension to partitioning. `Shape.init` uses this sharding. +/// Unconstrained graph results use `open` instead. +pub const replicated: Sharding = .{ .mesh = null, .partition = .replicated(Shape.MAX_RANK) }; + +/// Unspecified sharding for graph results. During compilation, the partitioner +/// may infer sharding from surrounding constraints or leave the value replicated. +/// Executable inputs require closed sharding. An executable output with this +/// unspecified sharding defaults to replication. An explicitly declared output +/// sharding containing open dimensions is rejected. +pub const open: Sharding = .{ .mesh = null, .partition = .open(Shape.MAX_RANK) }; + +pub fn get(sharding: Sharding, ax: usize) PartitionSpec { + return sharding.partition.get(ax); +} + +pub fn set(self: Sharding, ax: usize, spec: PartitionSpec) Sharding { + var res = self; + res.partition = res.partition.set(ax, spec); + return res; +} + +pub fn insert(self: Sharding, ax: usize, spec: PartitionSpec) Sharding { + var res = self; + res.partition = res.partition.insert(ax, spec); + return res; +} + +pub fn orderedRemove(self: Sharding, ax: usize) Sharding { + var res = self; + res.partition = res.partition.orderedRemove(ax); + return res; +} + +pub fn toArray(self: Sharding) [Shape.MAX_RANK]PartitionSpec { + var result: [Shape.MAX_RANK]PartitionSpec = undefined; + for (&result, 0..) |*spec, ax| spec.* = self.get(ax); + return result; +} + +pub fn isFullyReplicated(self: Sharding) bool { + for (self.partition.toArray()) |spec| { + if (spec != .replicated and spec != .out_of_bound) return false; + } + return true; +} + +/// An unspecified mesh is compatible with any sharding. On the same mesh, +/// open axes accept any in-bounds partition, while closed axes must match. +pub fn isCompatible(self: Sharding, other: Sharding) bool { + if (self.mesh == null or other.mesh == null) return true; + if (self.mesh != other.mesh) return false; + for (self.partition.toArray(), other.partition.toArray()) |lhs, rhs| { + if (lhs == rhs) continue; + if (lhs == .out_of_bound or rhs == .out_of_bound) return false; + if (lhs != .open and rhs != .open) return false; + } + return true; +} + +test isCompatible { + const tp: Mesh = .initTest(.tp, .{.model}); + const dp: Mesh = .initTest(.dp, .{ .data, .model }); + + const sharded: Sharding = .{ .mesh = &tp, .partition = .init(&.{ .mesh_axis_0, .replicated }) }; + const cases = [_]struct { lhs: Sharding, rhs: Sharding, compatible: bool }{ + .{ .lhs = .replicated, .rhs = .replicated, .compatible = true }, + .{ .lhs = .replicated, .rhs = sharded, .compatible = true }, + .{ .lhs = sharded, .rhs = sharded, .compatible = true }, + .{ .lhs = sharded, .rhs = .{ .mesh = &tp, .partition = .open(2) }, .compatible = true }, + .{ .lhs = sharded.set(1, .open), .rhs = sharded.set(0, .open), .compatible = true }, + .{ .lhs = sharded, .rhs = sharded.set(0, .mesh_axis_1), .compatible = false }, + .{ .lhs = sharded, .rhs = sharded.set(0, .replicated), .compatible = false }, + .{ .lhs = sharded, .rhs = .{ .mesh = &dp, .partition = sharded.partition }, .compatible = false }, + .{ .lhs = sharded, .rhs = .{ .mesh = &dp, .partition = .open(2) }, .compatible = false }, + .{ .lhs = sharded, .rhs = sharded.set(1, .out_of_bound), .compatible = false }, + .{ .lhs = sharded, .rhs = sharded.set(2, .open), .compatible = false }, + }; + for (cases) |case| { + errdefer std.log.err("{f}.isCompatible({f}) expected to be {}, got {}", .{ case.lhs, case.rhs, case.compatible, !case.compatible }); + + try std.testing.expectEqual(case.compatible, case.lhs.isCompatible(case.rhs)); + try std.testing.expectEqual(case.compatible, case.rhs.isCompatible(case.lhs)); + } +} + +/// Formats the declared sharding, not the partitioner's inferred sharding. +/// Without a mesh, prints `{replicated}` for full replication or `{open}` for +/// unspecified graph sharding. +/// +/// With a mesh, entries follow tensor dimension order, e.g. `{data,replicated,?}@dp`: +/// - `data`: split this dimension over the logical mesh axis named `data`. +/// - `replicated`: keep this dimension whole on each shard. +/// - `?`: leave this dimension to partitioner inference during compilation. +/// - `@dp`: the mesh name. +/// Trailing `.out_of_bound` entries are omitted. Any remaining ones print as `!`. +pub fn format(self: Sharding, writer: *std.Io.Writer) !void { + const mesh = self.mesh orelse return writer.writeAll(if (self.isFullyReplicated()) "{replicated}" else "{open}"); + const specs = self.partition.toArray(); + var rank = specs.len; + while (rank > 0 and specs[rank - 1] == .out_of_bound) rank -= 1; + + try writer.writeByte('{'); + for (specs[0..rank], 0..) |spec, i| { + if (i > 0) try writer.writeByte(','); + switch (spec) { + .replicated => try writer.writeAll("replicated"), + .open => try writer.writeByte('?'), + .out_of_bound => try writer.writeByte('!'), + else => try writer.print("{s}", .{mesh.logical.axes.get(spec.meshAxis().?)}), + } + } + try writer.print("}}@{s}", .{mesh.name}); +} + +/// Describes how one tensor dimension is distributed across a logical mesh. +/// A mesh axis splits the dimension. `.replicated` keeps it whole on each shard. +/// In a graph constraint, `.open` lets the partitioner choose either behavior. +/// Executable buffers require closed sharding specs: a mesh axis or `.replicated`. +pub const PartitionSpec = enum(u4) { + mesh_axis_0 = 0, + mesh_axis_1 = 1, + mesh_axis_2 = 2, + mesh_axis_3 = 3, + mesh_axis_4 = 4, + mesh_axis_5 = 5, + mesh_axis_6 = 6, + mesh_axis_7 = 7, + // an 8D mesh seems already a lot, the max we know about is 3D. + replicated = 8, + + // Partitioning don't encode the rank of corresponding tensor, so this is used to mark invalid positions. + out_of_bound = 0xa, + open = 15, + + pub fn sharded(mesh_axis: u3) PartitionSpec { + return @fromBackingInt(@intCast(mesh_axis)); + } + + /// Extract the mesh axis along which we are sharded. Null if not sharded. + pub fn meshAxis(self: PartitionSpec) ?u3 { + const ax = @backingInt(self); + return if (ax < @backingInt(PartitionSpec.replicated)) @intCast(ax) else null; + } + + pub fn isSharded(self: PartitionSpec) bool { + return @backingInt(self) < @backingInt(PartitionSpec.replicated); + } + + pub fn isClosed(self: PartitionSpec) bool { + return @backingInt(self) <= @backingInt(PartitionSpec.replicated); + } + + test isClosed { + try std.testing.expect(PartitionSpec.mesh_axis_2.isClosed()); + try std.testing.expect(!PartitionSpec.open.isClosed()); + + try std.testing.expect(PartitionSpec.replicated.isClosed()); + + try std.testing.expect(!PartitionSpec.out_of_bound.isClosed()); + } +}; + +pub const Partitioner = union(enum) { + shardy, + gspmd, + + pub fn fromTarget(target: Target) Partitioner { + return switch (target) { + .cpu, .cuda, .rocm, .tpu, .oneapi, .neuron, .metal, .furiosa => .shardy, + }; + } +}; + +fn shapeHasAxisPartition(shape: Shape) bool { + for (0..shape.rank()) |ax| { + if (shape.partition(ax).meshAxis()) |_| return true; + } + return false; +} + +pub const DimSharding = union(enum) { + sharded: struct { + dim: i64, + factor: u32, + }, + replicated, +}; + +pub fn sdyPerValueShardingAttr( + allocator: std.mem.Allocator, + ctx: *mlir.Context, + shapes: []const Shape, + mesh: *const Mesh, +) error{OutOfMemory}!*const shardy.TensorShardingPerValueAttr { + const meshe_attrs = try allocator.alloc(*const shardy.TensorShardingAttr, shapes.len); + for (meshe_attrs, shapes) |*attr, shape| { + attr.* = try mesh.sdyShardingAttrForShape(allocator, ctx, shape); + } + return .get(ctx, meshe_attrs); +} + +pub fn sdyManualAxesAttr( + allocator: std.mem.Allocator, + ctx: *mlir.Context, + in_shapes: []const Shape, + out_shapes: []const Shape, + mesh: *const Mesh, +) error{OutOfMemory}!*const shardy.ManualAxesAttr { + var axis_names = std.ArrayList([]const u8).empty; + defer axis_names.deinit(allocator); + + const Collect = struct { + fn appendUnique(list: *std.ArrayList([]const u8), allocator_: std.mem.Allocator, axis_name: []const u8) void { + for (list.items) |existing| { + if (std.mem.eql(u8, existing, axis_name)) return; + } + list.append(allocator_, axis_name) catch unreachable; + } + }; + + for (in_shapes) |shape| { + const attr = try mesh.sdyShardingAttrForShape(allocator, ctx, shape); + for (0..attr.numReplicatedAxes()) |i| { + Collect.appendUnique(&axis_names, allocator, attr.replicatedAxis(i).name()); + } + for (0..attr.numDimensions()) |i| { + const dim = attr.dimension(i); + for (0..dim.numAxes()) |j| { + Collect.appendUnique(&axis_names, allocator, dim.axis(j).name()); + } + } + } + for (out_shapes) |shape| { + const attr = try mesh.sdyShardingAttrForShape(allocator, ctx, shape); + for (0..attr.numReplicatedAxes()) |i| { + Collect.appendUnique(&axis_names, allocator, attr.replicatedAxis(i).name()); + } + for (0..attr.numDimensions()) |i| { + const dim = attr.dimension(i); + for (0..dim.numAxes()) |j| { + Collect.appendUnique(&axis_names, allocator, dim.axis(j).name()); + } + } + } + + const axes = try allocator.alloc(*const mlir.StringAttribute, axis_names.items.len); + for (axis_names.items, 0..) |axis_name, i| { + axes[i] = mlir.StringAttribute.init(ctx, axis_name); + } + + return .get(ctx, axes); +} + +pub const Partitioning = packed struct { + _0: PartitionSpec, + _1: PartitionSpec, + _2: PartitionSpec, + _3: PartitionSpec, + _4: PartitionSpec, + _5: PartitionSpec, + _6: PartitionSpec, + _7: PartitionSpec, + + pub const out_of_bound: Partitioning = splat(.out_of_bound); + + const MAX_RANK = Shape.MAX_RANK; + const Vec = @Vector(MAX_RANK, u4); + + pub fn init(specs: []const PartitionSpec) Partitioning { + var res: Partitioning = out_of_bound; + for (0.., specs) |i, spec| { + res = res.set(i, spec); + } + return res; + } + + pub fn replicated(rank_: usize) Partitioning { + return repeat(.replicated, rank_); + } + + pub fn open(rank_: usize) Partitioning { + return repeat(.open, rank_); + } + + pub fn splat(spec: PartitionSpec) Partitioning { + const vec: Vec = @splat(@backingInt(spec)); + return @bitCast(vec); + } + + pub fn repeat(spec: PartitionSpec, rank_: usize) Partitioning { + std.debug.assert(rank_ <= MAX_RANK); + const splatted: Vec = @splat(@backingInt(spec)); + const mask = std.simd.iota(u4, 8) < @as(Vec, @splat(@truncate(rank_))); + return @bitCast(@select(u4, mask, splatted, @as(Vec, @bitCast(out_of_bound)))); + } + + pub fn get(p: Partitioning, ax: usize) PartitionSpec { + std.debug.assert(ax < MAX_RANK); + const pack: u32 = @bitCast(p); + const shift: u5 = @intCast(4 * ax); + return @fromBackingInt(@truncate(pack >> shift)); + } + + pub fn set(p: Partitioning, ax: usize, spec: PartitionSpec) Partitioning { + std.debug.assert(ax < MAX_RANK); + const pack: u32 = @bitCast(p); + const shift: u5 = @intCast(4 * ax); + const mask = @as(u32, 0xf) << shift; + return @bitCast((pack & ~mask) | (@as(u32, @backingInt(spec)) << shift)); + } + + pub fn rank(p: Partitioning) u8 { + for (0..MAX_RANK) |i_usize| { + const i: u8 = @truncate(i_usize); + if (p.get(i) == .out_of_bound) return i; + } + return MAX_RANK; + } + + /// Inserts a spec, shifting subsequent slots right and discarding the last slot. + pub fn insert(p: Partitioning, ax: usize, spec: PartitionSpec) Partitioning { + std.debug.assert(ax < MAX_RANK); + const pack: u32 = @bitCast(p); + const shift: u5 = @intCast(4 * ax); + const lower_mask = (@as(u32, 1) << shift) - 1; + return @bitCast((pack & lower_mask) | ((pack & ~lower_mask) << 4) | (@as(u32, @backingInt(spec)) << shift)); + } + + /// Removes a spec, shifting subsequent slots left and filling the last with unknown. + pub fn orderedRemove(p: Partitioning, ax: usize) Partitioning { + std.debug.assert(ax < MAX_RANK); + const pack: u32 = @bitCast(p); + const shift: u5 = @intCast(4 * ax); + const lower_mask = (@as(u32, 1) << shift) - 1; + return @bitCast((pack & lower_mask) | ((pack >> 4) & ~lower_mask) | (@as(u32, @backingInt(PartitionSpec.out_of_bound)) << 28)); + } + + pub fn toArray(p: Partitioning) [MAX_RANK]PartitionSpec { + var res: [MAX_RANK]PartitionSpec = undefined; + for (&res, 0..) |*spec, ax| spec.* = p.get(ax); + return res; + } + + pub fn hasUniqueAxes(p: Partitioning) bool { + var used_mesh_axes: std.StaticBitSet(MAX_RANK) = .empty; + for (0..MAX_RANK) |shape_ax| { + const spec = p.get(shape_ax); + if (spec.meshAxis()) |mesh_axis| { + if (used_mesh_axes.isSet(mesh_axis)) return false; + used_mesh_axes.set(mesh_axis); + } + } + return true; + } + + test hasUniqueAxes { + try std.testing.expect(hasUniqueAxes(.init(&.{ .mesh_axis_0, .mesh_axis_1, .replicated, .mesh_axis_2 }))); + try std.testing.expect(!hasUniqueAxes(.init(&.{ .mesh_axis_0, .mesh_axis_1, .replicated, .mesh_axis_0 }))); + } + + test "packed access and updates" { + try std.testing.expectEqual(32, @bitSizeOf(Partitioning)); + try std.testing.expectEqual(4, @sizeOf(Partitioning)); + inline for (std.meta.tags(PartitionSpec)) |spec| { + const filled = Partitioning.splat(spec); + for (0..MAX_RANK) |ax| { + try std.testing.expectEqual(spec, filled.get(ax)); + const updated = Partitioning.out_of_bound.set(ax, spec); + for (0..MAX_RANK) |i| { + try std.testing.expectEqual(if (i == ax) spec else .out_of_bound, updated.get(i)); + } + } + } + for (0..MAX_RANK + 1) |rank_| { + const parts = Partitioning.replicated(rank_); + for (0..MAX_RANK) |ax| { + try std.testing.expectEqual(if (ax < rank_) PartitionSpec.replicated else .out_of_bound, parts.get(ax)); + } + } + } + + test "insertion and removal at every slot" { + var parts: Partitioning = .out_of_bound; + for (0..MAX_RANK) |ax| parts = parts.set(ax, @fromBackingInt(@intCast(ax))); + const original = parts.toArray(); + for (0..MAX_RANK) |ax| { + var inserted = original; + std.mem.copyBackwards(PartitionSpec, inserted[ax + 1 ..], original[ax .. MAX_RANK - 1]); + inserted[ax] = .open; + try std.testing.expectEqualSlices(PartitionSpec, &inserted, &parts.insert(ax, .open).toArray()); + + var removed = original; + std.mem.copyForwards(PartitionSpec, removed[ax .. MAX_RANK - 1], original[ax + 1 ..]); + removed[MAX_RANK - 1] = .out_of_bound; + try std.testing.expectEqualSlices(PartitionSpec, &removed, &parts.orderedRemove(ax).toArray()); + try std.testing.expectEqual(parts, parts.orderedRemove(ax).insert(ax, parts.get(ax))); + } + try std.testing.expectEqual(Partitioning.out_of_bound, Partitioning.out_of_bound.orderedRemove(0)); + const comptime_parts = comptime Partitioning.out_of_bound.insert(7, .open).orderedRemove(0).set(0, .replicated); + try std.testing.expectEqual(PartitionSpec.replicated, comptime_parts.get(0)); + try std.testing.expectEqual(PartitionSpec.open, comptime_parts.get(6)); + try std.testing.expectEqual(PartitionSpec.out_of_bound, comptime_parts.get(7)); + } + + /// Parse a struct literal into a partitioning object. + /// The struct passed to partitioning must only have fields names present in `tags`. + /// The ordering of tags matter, the return struct will map names to axis as specified by `tags`. + /// The default value is important. + /// In Tensor code unspecified axis will be inferred as `.open` while in shape code, it's `.replicated`. + pub fn parse(tags: []const [*:0]const u8, mesh: *const Mesh, default_value: PartitionSpec, partitioning: anytype) Partitioning { + const logical = if (@TypeOf(partitioning) == Logical) + partitioning + else + // capture all the comptime logic in Logical. + comptime Logical.parse(partitioning); + + return logical.applyUseAll(tags, mesh, default_value); + } + + pub const Logical = union(enum) { + /// Replicates every dimension, including when used as a graph constraint. + replicated, + mapping: struct { + src: Shape.TagsArray, + tgt: Shape.TagsArray, + }, + + const replicated_tag: Shape.Tag = Shape.toTag(.replicated); + const open_tag: Shape.Tag = Shape.toTag(.open); + + pub fn parse(v: anytype) Logical { + const V = @TypeOf(v); + + var src_: Shape.TagsArray = .empty; + var tgt_: Shape.TagsArray = .empty; + + if (comptime stdx.meta.isStruct(V)) { + const field_names = comptime std.meta.fieldNames(V); + stdx.debug.assertComptime(field_names.len <= @TypeOf(src_).max_len, "Too many tags in struct {} ({d})", .{ V, field_names.len }); + inline for (field_names) |field_name| { + const fv = @field(v, field_name); + tgt_.appendAssumeCapacity(@tagName(fv)); + + if (!comptime stdx.meta.isTuple(V)) { + src_.appendAssumeCapacity(field_name); + } + } + return .{ .mapping = .{ .src = src_, .tgt = tgt_ } }; + } + + stdx.debug.compileError("Sharding.Partitioning.parse expects a struct of enum literals, eg {{ .b = .data, .d = .model }}, got: {any}", .{V}); + } + + test "Logical.parse" { + const logical = comptime Logical.parse(.{ .a = .x, .b = .y }); + + try std.testing.expectEqualSlices(Shape.Tag, &.{ "a".ptr, "b".ptr }, logical.mapping.src.constSlice()); + try std.testing.expectEqualSlices(Shape.Tag, &.{ "x".ptr, "y".ptr }, logical.mapping.tgt.constSlice()); + } + + test "Logical.replicated" { + const mesh: Mesh = .initTest(.logical, .{.model}); + const tags: []const Shape.Tag = &.{ "a", "b" }; + const logical: Logical = .replicated; + for ([_]PartitionSpec{ .open, .replicated }) |default| { + try std.testing.expectEqual(Partitioning.replicated(tags.len), logical.applyUseAll(tags, &mesh, default)); + } + const unspecified = Logical.parse(.{}); + try std.testing.expectEqual(Partitioning.open(tags.len), unspecified.applyUseAll(tags, &mesh, .open)); + } + + pub fn apply(logical: *const Logical, tags: []const [*:0]const u8, mesh: *const Mesh, default_value: PartitionSpec) struct { Partitioning, ?u8 } { + const mapping = switch (logical.*) { + .replicated => return .{ .replicated(tags.len), null }, + .mapping => |*mapping_| mapping_, + }; + + var partition_: Partitioning = .repeat(default_value, tags.len); + var missing_src_axis: ?u8 = null; + + for (0.., mapping.src.slice(), mapping.tgt.slice()) |i, shape_tag, mesh_tag| { + const shape_ax = Shape.axisFromTagMaybe(tags, shape_tag) orelse { + missing_src_axis = @intCast(i); + continue; + }; + + const spec: PartitionSpec = if (mesh_tag == Logical.replicated_tag) + .replicated + else if (mesh_tag == Logical.open_tag) + .open + else spec: { + const mesh_ax = mesh.resolveLogicalAxis(mesh_tag) orelse { + stdx.debug.panic("{f} has no logical axis '{s}'", .{ mesh, mesh_tag }); + }; + break :spec .sharded(@intCast(mesh_ax)); + }; + partition_ = partition_.set(shape_ax, spec); + } + return .{ partition_, missing_src_axis }; + } + + pub fn applyUseAll(logical: *const Logical, tags: []const [*:0]const u8, mesh: *const Mesh, default_value: PartitionSpec) Partitioning { + const p, const missing_shape_axis = logical.apply(tags, mesh, default_value); + + if (missing_shape_axis) |ax| { + std.debug.panic("{f} doesn't have an axis {s} to be partitioned on.", .{ stdx.fmt.stringsZ(tags), logical.mapping.src.get(ax) }); + } + return p; + } + }; +}; + +/// Device as part of a PhysicalMesh. +pub const Device = struct { + /// Unique device identifier in the mesh + id: u32, + + /// Coordinates in the physical mesh + coords: Coords, + + const Coords = [MAX_MESH_RANK]u8; + // Placeholder coords, that would most likely trigger panic if used like that. + // Those are used when creating a mesh, before calling assignCoords + // Private so it's kept as implementation detail + const undefined_coords: Coords = @splat(0xff); + + fn hasValidCoords(device: Device) bool { + return @as(u32, @bitCast(device.coords)) != @as(u32, @bitCast(undefined_coords)); + } +}; + +/// Physical axis tags represent hardware interconnect tiers. +pub const PhysicalAxisTag = enum { + link, // Intra-device / island-local 1D interconnect + link_x, // Torus/Mesh Dim X (TPU, Trainium) + link_y, // Torus/Mesh Dim Y (TPU, Trainium) + link_z, // Torus/Mesh Dim Z (TPU v3/v4/v5) + bus, // PCIe / NUMA (Potential isolation boundary) +}; + +/// Ring geometry style for 1D neighbor-only interconnects. +pub const RingKind = enum { + linear, // Neighbors only, open ends + closed_ring, // Neighbors only, circular (wrap-around) +}; + +/// Mesh geometry style for 2D/3D neighbor-only interconnects. +pub const MeshKind = enum { + grid, // 2D/3D Neighbors only, open ends + torus, // 2D/3D Neighbors only, circular wrap-around +}; + +/// AxisGeometry describes the communication pattern for an axis. +pub const AxisGeometry = union(enum) { + point_to_point: void, // Full all-to-all (NVLink Switch, TPU v5p) + ring: RingKind, // Neighbor-only 1D + mesh: MeshKind, // Neighbor-only 2D/3D + tree: void, // Hierarchical (Standard PCIe) + isolated: void, // No direct communication (AWS Inf2 islands) + + pub fn format(self: AxisGeometry, writer: *std.Io.Writer) !void { + switch (self) { + .point_to_point => try writer.writeAll("P2P"), + .ring => |k| try writer.print("Ring({s})", .{@tagName(k)}), + .mesh => |k| try writer.print("Mesh({s})", .{@tagName(k)}), + .tree => try writer.writeAll("Tree"), + .isolated => try writer.writeAll("Isolated"), + } + } +}; + +/// Represents the hierarchical physical topology of the hardware devices. +pub const PhysicalNode = union(enum) { + /// A branch represents a physical axis + branch: struct { + tag: PhysicalAxisTag, + geometry: AxisGeometry, + children: []PhysicalNode, + }, + /// A leaf represents the actual hardware device + leaf: Device, + + pub fn axis(tag: PhysicalAxisTag, geometry_: AxisGeometry, children: []const PhysicalNode) PhysicalNode { + return .{ + .branch = .{ + .tag = tag, + .geometry = geometry_, + .children = @constCast(children), + }, + }; + } + + pub fn device(device_: PlatformDevice) PhysicalNode { + return .{ + .leaf = .{ + .id = @intCast(device_.id()), + .coords = Device.undefined_coords, + }, + }; + } + + pub fn countDevices(self: PhysicalNode) usize { + return switch (self) { + .leaf => 1, + .branch => |b| { + var sum: usize = 0; + for (b.children) |child| sum += child.countDevices(); + return sum; + }, + }; + } +}; + +/// PhysicalMesh models hardware as a tree of axes. +/// Each branch = one physical axis (link_x/link_y/bus/etc), +/// each leaf = one device. +/// +/// Diagram: +/// link_x +/// ├─ link_y +/// │ └─ device +/// └─ link_y +/// └─ device +/// +/// The DFS axis order defines canonical coordinate ordering. +pub const PhysicalMesh = struct { + /// Explicit topology tree API (canonical form) + pub const Tree = PhysicalNode; + + /// AxisTraversal captures canonical axis order and depth mapping. + pub const AxisTraversal = struct { + pub const Order = stdx.BoundedArray(PhysicalAxisTag, MAX_MESH_RANK); + pub const DepthByTag = std.EnumArray(PhysicalAxisTag, ?u8); + + /// Order of axes (DFS, first-child descent). + order: Order, + /// Depth (index) of each axis tag in the order. + depth_by_tag: DepthByTag, + + /// Build axis order and depth mapping from a topology tree. + pub fn init(root: PhysicalNode) AxisTraversal { + var order: Order = .empty; + axisOrderNode(root, &order); + + var depth_by_tag: DepthByTag = .initFill(null); + for (order.slice(), 0..) |t, i| { + depth_by_tag.set(t, @intCast(i)); + } + + return .{ + .order = order, + .depth_by_tag = depth_by_tag, + }; + } + + pub fn depth(self: AxisTraversal, tag: PhysicalAxisTag) ?u8 { + return self.depth_by_tag.get(tag); + } + + /// Canonical DFS axis order: + /// + /// Tree: + /// link_x + /// └─ link_y + /// └─ leaf + /// + /// Order = [link_x, link_y] + /// + /// We descend only the first child to keep a stable axis order that + /// depends on structure (axes), not on device count. + fn axisOrderNode(node: PhysicalNode, out: *Order) void { + switch (node) { + .leaf => {}, + .branch => |b| { + out.appendAssumeCapacity(b.tag); + if (b.children.len > 0) axisOrderNode(b.children[0], out); + }, + } + } + }; + + /// AxisInfo exposes the size and geometry for a tag. + pub const AxisInfo = struct { + tag: PhysicalAxisTag, + size: i64, + geometry: AxisGeometry, + }; + + target: Target, + root: PhysicalNode, + axis_traversal: AxisTraversal, + devices_in_canonical_order: []Device, + + /// Build a PhysicalMesh from a tree: + /// - clone the tree into `allocator` + /// - validate geometry invariants + /// - assign coordinates to leaves + /// - compute axis traversal and canonical device order + pub fn fromTree(allocator: std.mem.Allocator, target: Target, root: PhysicalNode) !PhysicalMesh { + const cloned = try cloneNode(allocator, root); + errdefer freeNode(allocator, cloned); + + return try fromOwnedTree(allocator, target, cloned); + } + + fn fromOwnedTree(allocator: std.mem.Allocator, target: Target, root: PhysicalNode) !PhysicalMesh { + try validateGeometry(root); + + var owned_root = root; + var path: Device.Coords = @splat(0); + assignCoords(&owned_root, &path, 0); + + var mesh: PhysicalMesh = .{ + .target = target, + .root = owned_root, + .axis_traversal = .init(owned_root), + .devices_in_canonical_order = undefined, + }; + try mesh.populateDevicesInCanonicalOrder(allocator); + for (mesh.devices_in_canonical_order) |device| { + std.debug.assert(device.hasValidCoords()); + } + return mesh; + } + + pub fn torus1d( + allocator: std.mem.Allocator, + target: Target, + devices: []const PlatformDevice, + ) !Sharding.PhysicalMesh { + const nodes = try allocator.alloc(PhysicalNode, devices.len); + for (nodes, devices) |*node, device| { + node.* = .device(device); + } + + return .fromOwnedTree(allocator, target, .axis(.link_x, .{ .mesh = .torus }, nodes)); + } + + pub fn torus2d( + allocator: std.mem.Allocator, + target: Target, + devices: []const PlatformDevice, + ) !Sharding.PhysicalMesh { + // Find the best split: 4 -> 2x2, 6 -> 2x3, 8 -> 2x4 + var link_x: usize = 1; + var link_y: usize = devices.len; + if (devices.len < 2) return error.NotEnoughDevices; + + for (2..devices.len) |i| { + if (@mod(devices.len, i) != 0) continue; + const j = @divExact(devices.len, i); + if (i <= j) { + link_x = i; + link_y = j; + } else break; + } + + // This only happen if number of devices is 1 or a prime number + if (link_x == 1) return error.NotEnoughDevices; + + const x_axes = try allocator.alloc(PhysicalNode, link_x); + var device_queue = devices; + for (x_axes) |*x_ax| { + const y_devices = try allocator.alloc(PhysicalNode, link_y); + for (y_devices) |*y_dev| { + y_dev.* = .device(device_queue[0]); + device_queue = device_queue[1..]; + } + + x_ax.* = .axis(.link_y, .{ .mesh = .torus }, y_devices); + } + + return .fromOwnedTree(allocator, target, .axis(.link_x, .{ .mesh = .torus }, x_axes)); + } + + pub fn torus3d( + allocator: std.mem.Allocator, + target: Target, + devices: []const PlatformDevice, + ) !Sharding.PhysicalMesh { + if (devices.len < 8) return error.NotEnoughDevices; + if (devices.len > 8) @panic("TODO: adapt logic from torus2D"); + const topology: Sharding.PhysicalMesh.Tree = .axis(.link_x, .{ .mesh = .torus }, &.{ + .axis(.link_y, .{ .mesh = .torus }, &.{ + .axis(.link_z, .{ .mesh = .torus }, &.{ + .device(devices[0]), + .device(devices[1]), + }), + .axis(.link_z, .{ .mesh = .torus }, &.{ + .device(devices[2]), + .device(devices[3]), + }), + }), + .axis(.link_y, .{ .mesh = .torus }, &.{ + .axis(.link_z, .{ .mesh = .torus }, &.{ + .device(devices[4]), + .device(devices[5]), + }), + .axis(.link_z, .{ .mesh = .torus }, &.{ + .device(devices[6]), + .device(devices[7]), + }), + }), + }); + + return .fromTree(allocator, target, topology); + } + + fn populateDevicesInCanonicalOrder(self: *PhysicalMesh, allocator: std.mem.Allocator) !void { + var devices_: std.ArrayList(Device) = .empty; + errdefer devices_.deinit(allocator); + try devicesNodeInto(allocator, self.root, &devices_); + self.devices_in_canonical_order = try devices_.toOwnedSlice(allocator); + + const Order = struct { + mesh: *const PhysicalMesh, + fn lessThan(ctx: @This(), a: Device, b: Device) bool { + const ia = ctx.mesh.linearIndexFromCoords(a.coords); + const ib = ctx.mesh.linearIndexFromCoords(b.coords); + return ia < ib; + } + }; + + std.mem.sort(Device, self.devices_in_canonical_order, Order{ .mesh = self }, Order.lessThan); + } + + fn cloneNode(allocator: std.mem.Allocator, node: PhysicalNode) !PhysicalNode { + return switch (node) { + .leaf => |d| .{ + .leaf = .{ + .id = d.id, + .coords = d.coords, + }, + }, + .branch => |b| blk: { + const children = try allocator.alloc(PhysicalNode, b.children.len); + errdefer allocator.free(children); + for (b.children, 0..) |child, i| { + children[i] = try cloneNode(allocator, child); + } + break :blk .{ + .branch = .{ + .tag = b.tag, + .geometry = b.geometry, + .children = children, + }, + }; + }, + }; + } + + fn freeNode(allocator: std.mem.Allocator, node: PhysicalNode) void { + switch (node) { + .leaf => {}, + .branch => |b| { + for (b.children) |child| freeNode(allocator, child); + allocator.free(b.children); + }, + } + } + + pub fn deinit(self: *PhysicalMesh, allocator: std.mem.Allocator) void { + allocator.free(self.devices_in_canonical_order); + freeNode(allocator, self.root); + self.* = undefined; + } + + fn validateGeometry(node: PhysicalNode) !void { + switch (node) { + .leaf => return, + .branch => |b| { + if (b.children.len == 0) return error.InvalidPhysicalMesh; + + switch (b.geometry) { + .mesh => { + var has_leaf = false; + var has_branch = false; + var expected_tag: ?PhysicalAxisTag = null; + + for (b.children) |child| { + switch (child) { + .leaf => has_leaf = true, + .branch => |cb| { + has_branch = true; + if (expected_tag == null) { + expected_tag = cb.tag; + } else if (cb.tag != expected_tag.?) { + return error.IncompatibleGeometry; + } + }, + } + } + + if (has_leaf and has_branch) return error.IncompatibleGeometry; + }, + .ring => { + if (b.children.len < 2) return error.IncompatibleGeometry; + }, + .point_to_point, .tree, .isolated => {}, + } + + for (b.children) |child| try validateGeometry(child); + }, + } + } + + /// Assign coords to each leaf using DFS path indices. + /// + /// Example (2x2): + /// link_x + /// └─ link_y + /// + /// coords: + /// (0,0) (0,1) + /// (1,0) (1,1) + fn assignCoords(node: *PhysicalNode, path: *Device.Coords, depth: usize) void { + switch (node.*) { + .leaf => |*d| { + // Coords can already been set by caller + if (d.hasValidCoords()) return; + + d.coords = path.*; + }, + .branch => |*b| { + for (b.children, 0..) |*child, i| { + path[depth] = @intCast(i); + assignCoords(child, path, depth + 1); + } + }, + } + } + + pub fn countDevices(self: PhysicalMesh) usize { + return self.root.countDevices(); + } + + fn devicesNodeInto(allocator: std.mem.Allocator, node: PhysicalNode, out: *std.ArrayList(Device)) !void { + switch (node) { + .leaf => |d| try out.append(allocator, d), + .branch => |b| { + for (b.children) |child| { + try devicesNodeInto(allocator, child, out); + } + }, + } + } + + /// Canonical DFS linearization for coordinate consensus. + /// + /// sizes: S0..Sk, coords: C0..Ck + /// linear = (((C0 * S1) + C1) * S2 + C2) ... + pub fn linearIndexFromCoords(self: PhysicalMesh, coords: Device.Coords) usize { + const order = self.axisOrder(); + var idx: usize = 0; + for (order.slice()) |tag| { + const depth = self.axis_traversal.depth(tag); + if (depth) |d| { + const coord = coords[@intCast(d)]; + const size = self.axis(tag); + idx = idx * @as(usize, @intCast(size)) + coord; + } + } + return idx; + } + + pub fn axisInfo(self: PhysicalMesh, tag: PhysicalAxisTag) ?AxisInfo { + return axisInfoNode(self.root, tag); + } + + fn axisInfoNode(node: PhysicalNode, tag: PhysicalAxisTag) ?AxisInfo { + return switch (node) { + .leaf => null, + .branch => |b| { + if (b.tag == tag) { + return .{ + .tag = b.tag, + .size = @intCast(b.children.len), + .geometry = b.geometry, + }; + } + + for (b.children) |child| { + if (axisInfoNode(child, tag)) |info| return info; + } + + return null; + }, + }; + } + + pub fn hasAxis(self: PhysicalMesh, tag: PhysicalAxisTag) bool { + return self.axisInfo(tag) != null; + } + + pub fn axis(self: PhysicalMesh, tag: PhysicalAxisTag) i64 { + if (self.axisInfo(tag)) |info| return info.size; + + return 1; + } + + pub fn axisOrder(self: PhysicalMesh) AxisTraversal.Order { + return self.axis_traversal.order; + } + + pub fn geometry(self: PhysicalMesh, tag: PhysicalAxisTag) ?AxisGeometry { + if (self.axisInfo(tag)) |info| return info.geometry; + + return null; + } + + /// Preferred axis ordering by speed: fastest -> slowest. + /// This is used by Strategy.suggest to match intent to bandwidth. + pub fn shardableAxes(self: PhysicalMesh) []const PhysicalAxisTag { + return switch (self.target) { + .tpu => &.{ .link_x, .link_y, .link_z }, + .neuron => &.{ .link, .link_x, .link_y, .link_z }, + .cuda, .rocm => &.{.link}, + .cpu, .metal, .furiosa => &.{.bus}, + .oneapi => &.{ .link, .bus }, + }; + } + + pub fn isShardable(self: PhysicalMesh, tag: PhysicalAxisTag) bool { + for (self.shardableAxes()) |t| { + if (t == tag) return true; + } + + return false; + } + + pub fn format(self: PhysicalMesh, writer: *std.Io.Writer) !void { + try writer.print("PhysicalMesh(platform={s} shardable_axes={any} num_devices={d})\n", .{ @tagName(self.target), self.shardableAxes(), self.countDevices() }); + try formatNode(self.root, writer, "", 0, true); + } + + fn formatNode(node: PhysicalNode, writer: *std.Io.Writer, prefix: []const u8, depth: u8, is_last: bool) !void { + try writer.print("{s}{s}", .{ prefix, if (is_last) "└── " else "├── " }); + switch (node) { + .leaf => |device| { + try writer.print("Device(id={d}, coords={any})\n", .{ device.id, device.coords[0..depth] }); + }, + .branch => |b| { + try writer.print("[{s}] 0..{d} ({s})\n", .{ @tagName(b.tag), b.children.len, @tagName(b.geometry) }); + + var buf: [256]u8 = undefined; + const new_prefix = std.fmt.bufPrint(&buf, "{s}{s}", .{ prefix, if (is_last) " " else "│ " }) catch unreachable; + + for (b.children, 0..) |child, i| { + const child_is_last = (i == b.children.len - 1); + try formatNode(child, writer, new_prefix, depth +| 1, child_is_last); + } + }, + } + } + + /// CoordsTopology builds a mesh from PJRT `coords` attributes. + const CoordsTopology = struct { + const CoordLayout = struct { + rank: usize, + axis_sizes: [MAX_MESH_RANK]usize, + }; + + const IndexedCoord = struct { + placement: Device, + linear: usize, + }; + + fn parseDevice(device: PlatformDevice) !Device { + const api = device.platform.pjrt_api; + const coords_attr = device.pjrt_desc.attribute(api, "coords") orelse return error.MissingDeviceCoords; + const coords: Device.Coords = coords: switch (coords_attr) { + .int64list => |values| { + var coords: Device.Coords = @splat(0); + if (values.len == 0 or values.len > MAX_MESH_RANK) return error.InvalidDeviceCoords; + for (coords[0..values.len], values) |*coord, value| { + if (value < 0) return error.InvalidDeviceCoords; + coord.* = @intCast(value); + } + break :coords coords; + }, + else => return error.InvalidDeviceCoords, + }; + + return .{ .id = device.id(), .coords = coords }; + } + + pub fn collect(allocator: std.mem.Allocator, platform_devices: []const PlatformDevice) ![]Device { + const placements = try allocator.alloc(Device, platform_devices.len); + for (platform_devices, 0..) |device, i| { + placements[i] = try parseDevice(device); + } + return placements; + } + + pub fn layout(placements: []const Device, max_rank: usize) !CoordLayout { + if (placements.len == 0) return error.InvalidPhysicalMesh; + + const rank = placements[0].coords.len; + if (rank == 0) return error.InvalidDeviceCoords; + if (rank > max_rank) return error.UnsupportedDeviceCoordsRank; + + for (placements[1..]) |coord_placement| { + if (coord_placement.coords.len != rank) return error.InvalidDeviceCoordsRank; + } + + var axis_sizes: [MAX_MESH_RANK]usize = @splat(1); + for (0..rank) |ax_i| { + var max_coord: usize = 0; + for (placements) |coord_placement| { + max_coord = @max(max_coord, coord_placement.coords[ax_i]); + } + axis_sizes[ax_i] = max_coord + 1; + } + + return .{ + .rank = rank, + .axis_sizes = axis_sizes, + }; + } + + fn linearIndex(coords: Device.Coords, rank: usize, axis_sizes: []const usize) usize { + var linear: usize = 0; + var stride: usize = 1; + var ax = rank; + while (ax > 0) { + ax -= 1; + linear += coords[ax] * stride; + stride *= axis_sizes[ax]; + } + return linear; + } + + pub fn sorted(allocator: std.mem.Allocator, placements: []const Device, rank: usize, axis_sizes: []const usize) ![]IndexedCoord { + const indexed = try allocator.alloc(IndexedCoord, placements.len); + for (placements, indexed) |coord_placement, *out| { + out.* = .{ + .placement = coord_placement, + .linear = linearIndex(coord_placement.coords, rank, axis_sizes), + }; + } + + const SortCtx = struct { + fn lessThan(_: @This(), a: IndexedCoord, b: IndexedCoord) bool { + return a.linear < b.linear; + } + }; + std.mem.sort(IndexedCoord, indexed, SortCtx{}, SortCtx.lessThan); + + for (indexed[1..], 1..) |entry, i| { + if (entry.linear == indexed[i - 1].linear) return error.InvalidDeviceTopology; + } + + return indexed; + } + + pub fn axisStrides(axis_sizes: []const usize) ![MAX_MESH_RANK]usize { + var strides: [MAX_MESH_RANK]usize = @splat(0); + var stride: usize = 1; + + var i = axis_sizes.len; + while (i > 0) { + i -= 1; + strides[i] = stride; + stride = std.math.mul(usize, stride, axis_sizes[i]) catch return error.InvalidDeviceTopology; + } + + return strides; + } + + pub fn buildMeshNode( + allocator: std.mem.Allocator, + indexed: []const IndexedCoord, + axis_sizes: []const usize, + axis_tags: []const PhysicalAxisTag, + axis_strides: []const usize, + depth: usize, + base_offset: usize, + ) !PhysicalNode { + if (depth == axis_sizes.len) { + const device = &indexed[base_offset].placement; + return .{ .leaf = device.* }; + } + + const children = try allocator.alloc(PhysicalNode, axis_sizes[depth]); + var built: usize = 0; + errdefer { + for (children[0..built]) |child| freeNode(allocator, child); + allocator.free(children); + } + + for (children, 0..) |*child, i| { + child.* = try buildMeshNode( + allocator, + indexed, + axis_sizes, + axis_tags, + axis_strides, + depth + 1, + base_offset + i * axis_strides[depth], + ); + built += 1; + } + + return .{ + .branch = .{ + .tag = axis_tags[depth], + .geometry = .{ .mesh = .torus }, + .children = children, + }, + }; + } + }; + + pub fn auto(allocator: std.mem.Allocator, target: Target, platform_devices: []const PlatformDevice) !PhysicalMesh { + const root = try switch (target) { + .cpu => cpu(allocator, platform_devices), + .cuda, .rocm => gpu(allocator, platform_devices), + .tpu => tpu(allocator, platform_devices), + .neuron => return neuron(allocator, platform_devices), + .oneapi, .metal, .furiosa => cpu(allocator, platform_devices), + }; + errdefer freeNode(allocator, root); + + // We check a posteriori that the device id is a valid offset into platform.devices. + // This is generally true because platform.devices is sorted by id and ids are 0-N. + // For platforms where this isn't true like Neuron, + // the platform specific builder need to correctly update the ids so that it's true. + const mesh = try fromOwnedTree(allocator, target, root); + for (mesh.devices_in_canonical_order) |pl| { + const device = platform_devices[pl.id]; + std.debug.assert(pl.hasValidCoords()); + std.debug.assert(device.id() == pl.id); + } + + return mesh; + } + + pub fn cpu(allocator: std.mem.Allocator, platform_devices: []const PlatformDevice) !Tree { + const nodes = try allocator.alloc(PhysicalNode, platform_devices.len); + + for (nodes, platform_devices) |*n, d| n.* = .device(d); + + return .{ + .branch = .{ + .tag = .bus, + .geometry = .tree, + .children = nodes, + }, + }; + } + + pub fn gpu(allocator: std.mem.Allocator, platform_devices: []const PlatformDevice) !Tree { + const placements = try CoordsTopology.collect(allocator, platform_devices); + defer allocator.free(placements); + + const layout = try CoordsTopology.layout(placements, MAX_MESH_RANK); + const indexed = try CoordsTopology.sorted(allocator, placements, layout.rank, layout.axis_sizes[0..layout.rank]); + defer allocator.free(indexed); + + const nodes = try allocator.alloc(PhysicalNode, platform_devices.len); + for (nodes, indexed, 0..) |*n, entry, i| { + const linear_coord: Device.Coords = .{ @intCast(i), 0, 0, 0 }; + n.* = .{ .leaf = .{ .id = entry.placement.id, .coords = linear_coord } }; + } + + return .{ + .branch = .{ + .tag = .link, + .geometry = .point_to_point, + .children = nodes, + }, + }; + } + + pub fn tpu(allocator: std.mem.Allocator, platform_devices: []const PlatformDevice) !Tree { + if (platform_devices.len == 0) return error.InvalidPhysicalMesh; + + const placements = try CoordsTopology.collect(allocator, platform_devices); + defer allocator.free(placements); + + const layout = try CoordsTopology.layout(placements, 4); + var rank = layout.rank; + var axis_sizes = layout.axis_sizes; + + while (rank > 1 and axis_sizes[rank - 1] == 1) : (rank -= 1) {} + const indexed = try CoordsTopology.sorted(allocator, placements, rank, axis_sizes[0..rank]); + defer allocator.free(indexed); + + var total_devices: usize = 1; + for (axis_sizes[0..rank]) |axis_size| { + total_devices = std.math.mul(usize, total_devices, axis_size) catch return error.InvalidDeviceTopology; + } + if (total_devices != indexed.len) return error.InvalidDeviceTopology; + for (indexed, 0..) |entry, i| { + if (entry.linear != i) return error.InvalidDeviceTopology; + } + + const axis_tags: [3]PhysicalAxisTag = .{ .link_x, .link_y, .link_z }; + const axis_strides = try CoordsTopology.axisStrides(axis_sizes[0..rank]); + + return CoordsTopology.buildMeshNode(allocator, indexed, axis_sizes[0..rank], axis_tags[0..rank], axis_strides[0..rank], 0, 0); + } + + pub fn neuron(allocator: std.mem.Allocator, platform_devices: []const PlatformDevice) !PhysicalMesh { + if (comptime !platforms.isEnabled(.neuron)) { + return error.UnsupportedPlatform; + } + + const neuron_topology = @import("platforms/neuron/topology"); + + const nc_ids = try allocator.alloc(usize, platform_devices.len); + defer allocator.free(nc_ids); + for (platform_devices, nc_ids) |device, *nc_id| { + nc_id.* = @intCast(device.localHardwareId()); + } + + var topology = try neuron_topology.visibleMeshFromNcIds(allocator, nc_ids); + defer topology.deinit(allocator); + + var axis_tags: stdx.BoundedArray(PhysicalAxisTag, MAX_MESH_RANK) = .empty; + var axis_sizes: stdx.BoundedArray(usize, MAX_MESH_RANK) = .empty; + var axis_geometries: stdx.BoundedArray(AxisGeometry, MAX_MESH_RANK) = .empty; + + for (topology.axes()) |mesh_axis| { + axis_tags.appendAssumeCapacity(switch (mesh_axis.fabric) { + .core => .link, + .chip_x => .link_x, + .chip_y => .link_y, + .chip_z => .link_z, + }); + axis_sizes.appendAssumeCapacity(mesh_axis.size); + axis_geometries.appendAssumeCapacity(switch (mesh_axis.geometry) { + .linear_ring => .{ .ring = .linear }, + .closed_ring => .{ .ring = .closed_ring }, + .torus => .{ .mesh = .torus }, + .point_to_point => .point_to_point, + }); + } + + var coords_buf: [MAX_MESH_RANK]usize = @splat(0); + var next_device: usize = 0; + + const Node = struct { + fn build( + allocator_: std.mem.Allocator, + platform_devices_: []const PlatformDevice, + placements: []const neuron_topology.Placement, + axis_tags_: []const PhysicalAxisTag, + axis_sizes_: []const usize, + axis_geometries_: []const AxisGeometry, + depth: usize, + coords_buf_: *[MAX_MESH_RANK]usize, + next_device_: *usize, + ) !PhysicalNode { + if (depth == axis_tags_.len) { + const placement_ = placements[next_device_.*]; + next_device_.* += 1; + + var coords: Device.Coords = @splat(0); + for (coords_buf_[0..axis_tags_.len], 0..) |coord, i| coords[i] = @intCast(coord); + + return .{ .leaf = .{ .id = @intCast(placement_.nc_id), .coords = coords } }; + } + + const children = try allocator_.alloc(PhysicalNode, axis_sizes_[depth]); + var built: usize = 0; + errdefer { + for (children[0..built]) |child| freeNode(allocator_, child); + allocator_.free(children); + } + + for (children, 0..) |*child, i| { + coords_buf_[depth] = i; + child.* = try @This().build( + allocator_, + platform_devices_, + placements, + axis_tags_, + axis_sizes_, + axis_geometries_, + depth + 1, + coords_buf_, + next_device_, + ); + built += 1; + } + + return .{ + .branch = .{ + .tag = axis_tags_[depth], + .geometry = axis_geometries_[depth], + .children = children, + }, + }; + } + }; + + const root = try Node.build( + allocator, + platform_devices, + topology.placements, + axis_tags.slice(), + axis_sizes.slice(), + axis_geometries.slice(), + 0, + &coords_buf, + &next_device, + ); + const mesh: PhysicalMesh = try fromOwnedTree(allocator, .neuron, root); + for (mesh.devices_in_canonical_order) |*dev| { + // In Node.build we use nc_id as the Device id. + // But now we want to use the id to get the offset into platform.devices. + const dev_nc_id = dev.id; + for (0.., topology.placements) |order, placement_| { + if (placement_.nc_id == dev_nc_id) { + dev.id = @intCast(order); + std.debug.assert(platform_devices[order].localHardwareId() == dev_nc_id); + break; + } + } + } + return mesh; + } +}; + +pub const LogicalAxisIntent = enum { + /// Needs the fastest possible path (e.g. Weight mesh / All-Reduce). + /// Requires high-bandwidth, low-latency interconnects (Cores/NVLink). + high_bandwidth, + + /// Needs reliable throughput but can handle some latency (e.g. Shuffling / All-to-All). + /// Suitable for chip-to-chip interconnects (NeuronLink/ICI). + balanced, + + /// Can tolerate lower bandwidth and higher latency (e.g. Batch mesh). + /// Suitable for system bus or network (PCIe/Ethernet). + low_bandwidth, +}; + +/// Semantic axes, their communication intents, and optional device-count constraints. +pub const LogicalMesh = struct { + pub const Axes = stdx.BoundedArray(Shape.Tag, MAX_MESH_RANK); + pub const Intents = stdx.BoundedArray(LogicalAxisIntent, MAX_MESH_RANK); + + axes: Axes, + intents: Intents, + dimensions: [MAX_MESH_RANK]i32, + + /// Create a logical mesh. + /// * names of the logical axes: .{ .a, .b }, + /// * intents: intent for each axis + /// * dimensions: dimension for each axis, -1 infers the size from underlying topology + pub fn init(names: anytype, intents: []const LogicalAxisIntent, dimensions: []const i32) !LogicalMesh { + if (@TypeOf(names) != []const Shape.Tag) { + const parsed_names: []const Shape.Tag = comptime Shape.parseTags(names); + return .init(parsed_names, intents, dimensions); + } + + if (names.len == 0 or names.len > MAX_MESH_RANK or names.len != intents.len or names.len != dimensions.len) @panic("invalid logical mesh"); + + var result: LogicalMesh = .{ .axes = .init(names), .intents = .init(intents), .dimensions = @splat(-1) }; + for (names, dimensions, 0..) |name, dimension, i| { + if (dimension != -1 and dimension <= 0) return error.InvalidLogicalDimension; + if (std.mem.findScalar(Shape.Tag, names[0..i], name)) |_| { + @panic("invalid logical mesh"); + } + + result.dimensions[i] = dimension; + } + return result; + } + + pub fn mesh(axes_: anytype) LogicalMesh { + const T = @TypeOf(axes_); + + var axes: Axes = .empty; + var intents: Intents = .empty; + + inline for (comptime std.meta.fieldNames(T)) |field_name| { + const value = @field(axes_, field_name); + axes.appendAssumeCapacity(Shape.toTag(field_name)); + intents.appendAssumeCapacity(intentFromValue(value)); + } + + if (axes.len == 0) { + stdx.debug.panic("LogicalMesh must have at least one axis defined", .{}); + } + + return .{ .axes = axes, .intents = intents, .dimensions = @splat(-1) }; + } + + pub fn hasAxis(m: *const LogicalMesh, logical_ax: Shape.Tag) ?u3 { + return @truncate(std.mem.findScalar(Shape.Tag, m.axes.slice(), logical_ax) orelse return null); + } + + pub fn intent(self: LogicalMesh, tag: Shape.Tag) ?LogicalAxisIntent { + const target = std.mem.span(tag); + for (self.axes.slice(), self.intents.slice()) |t, i| { + if (std.mem.eql(u8, std.mem.span(t), target)) return i; + } + + return null; + } + + fn intentFromValue(v: anytype) LogicalAxisIntent { + const V = @TypeOf(v); + if (V == LogicalAxisIntent) return v; + + if (V == @EnumLiteral()) { + return @field(LogicalAxisIntent, @tagName(v)); + } + + stdx.debug.compileError("LogicalMesh intent must be LogicalAxisIntent, got {}", .{V}); + } + + pub fn format(self: LogicalMesh, writer: *std.Io.Writer) !void { + try writer.writeAll("LogicalMesh("); + for (self.axes.slice(), self.intents.slice(), 0..) |axis, int, i| { + try writer.print(" {s}={s}", .{ axis, @tagName(int) }); + if (self.dimensions[i] != -1) try writer.print("[{d}]", .{self.dimensions[i]}); + } + try writer.writeAll(")"); + } +}; + +pub const PhysicalList = stdx.BoundedArray(PhysicalAxisTag, MAX_MESH_RANK); + +pub const Mesh = struct { + pub const zml_no_meta_visit: void = {}; + + name: []const u8, + physical: *const PhysicalMesh, + logical: LogicalMesh, + + grid: DeviceGrid, + + /// Compact binding table: logical axis -> physical axes + bindings: stdx.BoundedArray(PhysicalList, MAX_MESH_RANK), + + pub fn shardableDim(mesh: *const Mesh, dim: i64, logical_axis: anytype, must_divide: i64) DimSharding { + const partitions = mesh.numPartitionsForLogicalAxis(logical_axis); + + if (@mod(dim, partitions) == 0) { + return if (@mod(must_divide, dim) == 0) .{ .sharded = .{ .dim = dim, .factor = 1 } } else .replicated; + } + + const gcd: i64 = @intCast(std.math.gcd(@as(u64, @intCast(dim)), @as(u64, @intCast(partitions)))); + const repeat_factor: u32 = @intCast(@divExact(partitions, gcd)); + const materialized_dim = std.math.mul(i64, dim, @as(i64, repeat_factor)) catch return .replicated; + if (@mod(must_divide, materialized_dim) == 0) { + return .{ .sharded = .{ .dim = materialized_dim, .factor = repeat_factor } }; + } else { + return .replicated; + } + } + + pub fn binding(self: *const Mesh, mesh_axis: usize) []const PhysicalAxisTag { + return self.bindings.constSlice()[mesh_axis].constSlice(); + } + + pub fn resolveLogicalAxis(self: *const Mesh, tag: Shape.Tag) ?u8 { + if (@inComptime()) { + for (0.., self.logical.axes.slice()) |i, logical_axis| { + if (std.mem.eql(u8, std.mem.span(logical_axis), std.mem.span(tag))) return @truncate(i); + } + } else { + for (0.., self.logical.axes.slice()) |i, logical_axis| { + if (logical_axis == tag) return @truncate(i); + } + } + return null; + } + + pub fn init( + name_: []const u8, + physical: *const PhysicalMesh, + logical: LogicalMesh, + strategy: Strategy, + ) !Mesh { + const num_devices = physical.countDevices(); + std.debug.assert(num_devices == physical.devices_in_canonical_order.len); + const axis_order = physical.axisOrder().slice(); + if (axis_order.len == 0) return error.InvalidPhysicalMesh; + if (strategy.bindings.len == 0) return error.InvalidStrategy; + + // Create one binding per logical axis + var bindings: @FieldType(Mesh, "bindings") = .empty; + for (logical.axes.slice()) |_| { + bindings.appendAssumeCapacity(.empty); + } + + for (strategy.bindings.slice()) |strat_binding| { + for (logical.axes.slice(), bindings.slice()) |logical_axis, *binding_| { + if (std.mem.eql(u8, std.mem.span(logical_axis), std.mem.span(strat_binding.logical))) { + for (strat_binding.physical.constSlice()) |part| binding_.appendAssumeCapacity(part.axis); + break; + } + } else { + std.debug.panic("{f} expects strategy to only list known logical axes, got unknown {s}", .{ logical, strat_binding.logical }); + } + } + + var mesh: Mesh = .{ + .name = name_, + .logical = logical, + .grid = undefined, + .physical = physical, + .bindings = bindings, + }; + mesh.grid = try DeviceGrid.init(&mesh, &strategy); + std.debug.assert(num_devices == physical.devices_in_canonical_order.len); + std.debug.assert(@mod(mesh.numDevices(), mesh.numPartitions()) == 0); + return mesh; + } + + pub fn initTest(name_: @EnumLiteral(), tagz: anytype) Mesh { + if (!builtin.is_test) @compileError("Cannot use zml.Sharding.Mesh.initTest outside of a test block"); + const axes_tags = Shape.parseTags(tagz); + return .{ + .name = @tagName(name_), + .logical = .{ + .axes = .init(axes_tags), + .intents = .{ .buffer = @splat(.high_bandwidth), .len = axes_tags.len }, + .dimensions = @splat(-1), + }, + .physical = undefined, + .grid = undefined, + .bindings = .empty, + }; + } + + pub fn devicesInCanonicalOrder(mesh: *const Mesh) []const Device { + return mesh.physical.devices_in_canonical_order; + } + + pub fn logicalIndexFromCoords(self: *const Mesh, coords: []const u8) usize { + return self.physical.linearIndexFromCoords(coords); + } + + pub fn numPartitions(self: *const Mesh) u32 { + var partitions: u32 = 1; + for (self.grid.axis_sizes[0..self.grid.rank]) |size| partitions *= size; + return @intCast(partitions); + } + + pub fn numPartitionsForLogicalAxis(self: *const Mesh, logical_axis: anytype) u32 { + const logical_tag = Shape.toTag(logical_axis); + + const logical_ax = self.resolveLogicalAxis(logical_tag) orelse std.debug.panic("mesh {f} has no axis {s}", .{ self, logical_tag }); + return self.grid.axis_sizes[logical_ax]; + } + + pub fn numDevices(self: *const Mesh) u32 { + return @intCast(self.physical.devices_in_canonical_order.len); + } + + pub fn numReplicas(self: *const Mesh) u32 { + return @divExact(self.numDevices(), self.numPartitions()); + } + + pub fn axis(mesh: *const Mesh, ax: @EnumLiteral()) u3 { + return mesh.logical.hasAxis(Shape.toTag(ax)) orelse std.debug.panic("Mesh(name={s}) doesn't have axis {}: {f}", .{ mesh.name, ax, mesh.logical }); + } + + pub fn sdyMeshAttr(self: *const Mesh, ctx: *mlir.Context) *const shardy.MeshAttr { + var axes: stdx.BoundedArray(*const shardy.MeshAxisAttr, MAX_MESH_RANK + 1) = .empty; + var tile_count: usize = 1; + for (self.logical.axes.slice(), 0..) |ax, i| { + const size = self.grid.axis_sizes[i]; + tile_count *= size; + axes.appendAssumeCapacity(.get(ctx, .{ .name = std.mem.span(ax), .size = @intCast(size) })); + } + const replicas = @divExact(self.grid.device_count, tile_count); + if (replicas > 1) { + axes.appendAssumeCapacity(.get(ctx, .{ .name = self.sdyReplicaAxis(), .size = @intCast(replicas) })); + } + var replica_indices: [Platform.MAX_NUM_DEVICES]usize = @splat(0); + var ids: [Platform.MAX_NUM_DEVICES]i64 = undefined; + for (0..self.grid.device_count) |partition_index| { + const coords: [MAX_MESH_RANK]u8 = self.grid.logical_coords[partition_index]; + var tile_index: usize = 0; + for (0..self.logical.axes.len) |ax| { + tile_index = tile_index * self.grid.axis_sizes[ax] + coords[ax]; + } + // Device IDs here are executable partition indices, not PJRT device IDs. + ids[tile_index * replicas + replica_indices[tile_index]] = @intCast(partition_index); + replica_indices[tile_index] += 1; + } + return .get(ctx, .{ .axes = axes.constSlice(), .device_ids = ids[0..self.grid.device_count] }); + } + + fn sdyReplicaAxis(self: *const Mesh) []const u8 { + // There are at most MAX_MESH_RANK logical names, so one candidate is free. + inline for (0..MAX_MESH_RANK + 1) |i| { + const name = std.fmt.comptimePrint("__replica_{d}", .{i}); + const taken = for (self.logical.axes.slice()) |ax| { + if (std.mem.eql(u8, std.mem.span(ax), name)) break true; + } else false; + if (!taken) return name; + } + unreachable; + } + + /// Converts `shape`'s declared partitioning into a `#sdy.sharding` attribute + /// referencing this mesh. Entries in `[...]` follow tensor dimension order. + /// Mesh axis indices in the specs resolve to this mesh's logical axis names. + /// The tensor rank and logical mesh rank are independent. + /// + /// Each dimension spec is encoded as follows: + /// - `PartitionSpec.sharded(i)` -> `{"axis_name"}`: split the dimension over + /// logical mesh axis `i`. The dimension is closed to further partitioning. + /// - `.replicated` -> `{}`: keep the dimension whole on each shard. It is + /// closed, so the partitioner cannot assign a mesh axis to it. + /// - `.open` -> `{?}`: let the partitioner infer sharding from the graph. + /// It may acquire available mesh axes or remain replicated. + /// Closed therefore means no additional mesh axes, not necessarily replicated. + /// Every dimension within `shape.rank()` must have a spec other than `.out_of_bound`. + /// + /// Dimension replication (`{}`) applies to one tensor dimension. The optional + /// tensor-wide `replicated={...}` instead names mesh axes: each listed axis + /// must carry identical tensor data across its coordinates and cannot be used + /// to split any dimension, including an open one. + /// + /// For example, take a mesh `@dp_mp` with `data=2`, `model=2`, no extra replicas, + /// and a shape whose dimensions are ordered `batch`, `hidden`: + /// + /// ``` + /// Partitioning: .{ .batch = .data, .hidden = .open } + /// #sdy.sharding<@dp_mp, [{"data"}, {?}]> + /// ``` + /// `batch` is split over `data`. The partitioner may assign `model` to `hidden`. + /// The unused `model` axis stays available because an open dimension exists. + /// + /// ``` + /// Partitioning: .{ .batch = .data, .hidden = .replicated } + /// #sdy.sharding<@dp_mp, [{"data"}, {}], replicated={"model"}> + /// ``` + /// Both dimensions are closed. Unused logical axes are explicitly replicated, + /// so devices differing only in their `model` coordinate hold the same shard. + /// + /// ``` + /// Partitioning: .{ .batch = .open, .hidden = .replicated } + /// #sdy.sharding<@dp_mp, [{?}, {}]> + /// ``` + /// Only `batch` may acquire axes, and both `data` and `model` are available. + /// Adding `replicated={"data", "model"}` would prevent either axis from being + /// inferred for `batch`, despite its `{?}` entry. + pub fn sdyShardingAttrForShape( + data: *const Mesh, + parent_allocator: std.mem.Allocator, + ctx: *mlir.Context, + shape: Shape, + ) error{OutOfMemory}!*const shardy.TensorShardingAttr { + var arena = try stdx.arenaWithCapacity(parent_allocator, 1024); + defer arena.deinit(); + const allocator = arena.allocator(); + const dimensions = try allocator.alloc(*const shardy.DimensionShardingAttr, shape.rank()); + var used: [MAX_MESH_RANK]bool = @splat(false); + const has_open_dimension = shape.containsPartitionSpec(.open); + for (0.., dimensions) |i, *d| { + d.* = switch (shape.partition(i)) { + .out_of_bound => @panic("undefined"), + .replicated => .replicated(ctx), + .open => .open(ctx, &.{}), + else => d: { + const ax = shape.partition(i).meshAxis().?; + used[ax] = true; + break :d .closed(ctx, &.{.named(ctx, std.mem.span(data.logical.axes.get(ax)))}); + }, + }; + } + + var replicated_axes: stdx.BoundedArray(*const shardy.AxisRefAttr, MAX_MESH_RANK + 1) = .empty; + var tile_count: usize = 1; + for (data.logical.axes.slice(), 0..) |ax, i| { + tile_count *= data.grid.axis_sizes[i]; + // Open dimensions can acquire unused logical axes during inference. + // Only reserve those axes for replication when every dimension is closed. + if (!has_open_dimension and !used[i]) replicated_axes.appendAssumeCapacity(.named(ctx, std.mem.span(ax))); + } + if (tile_count < data.grid.device_count) { + replicated_axes.appendAssumeCapacity(.named(ctx, data.sdyReplicaAxis())); + } + + return .get(ctx, .{ .mesh = data.name, .dimensions = dimensions, .replicated_axes = replicated_axes.slice() }); + } + + pub fn gspmdShardingAttrForShape(self: *const Mesh, parent_allocator: std.mem.Allocator, ctx: *mlir.Context, shape: Shape) !*const mlir.Attribute { + var arena = try stdx.arenaWithCapacity(parent_allocator, 1024); + defer arena.deinit(); + const allocator = arena.allocator(); + + var tile_shape: stdx.BoundedArray(i64, Shape.MAX_RANK + 1) = .empty; + var tile_count: usize = 1; + for (0..shape.rank()) |i| { + const size = if (shape.partition(i).meshAxis()) |ax| self.grid.axis_sizes[ax] else 1; + tile_shape.appendAssumeCapacity(size); + tile_count *= size; + } + if (tile_count == 1) return .string(ctx, "{replicated}"); + + // All devices at the same logical tile coordinate are replicas, including + // unused logical axes and physical axes that have no logical binding. + const replicas = @divExact(self.grid.device_count, tile_count); + const has_replication = replicas > 1; + if (has_replication) tile_shape.appendAssumeCapacity(@intCast(replicas)); + + var out: std.Io.Writer.Allocating = .init(allocator); + errdefer out.deinit(); + + const ids = try self.gspmdTileAssignment(allocator, shape); + defer allocator.free(ids); + + try out.writer.writeAll("{devices=["); + for (tile_shape.slice(), 0..) |s, i| { + if (i > 0) try out.writer.writeAll(","); + try out.writer.print("{d}", .{s}); + } + try out.writer.writeAll("]"); + + for (ids, 0..) |id, i| { + if (i == 0) { + try out.writer.print("{d}", .{id}); + } else { + try out.writer.print(",{d}", .{id}); + } + } + + if (has_replication) { + try out.writer.writeAll(" last_tile_dim_replicate"); + } + + try out.writer.writeAll("}"); + + return .string(ctx, try out.toOwnedSlice()); + } + + fn gspmdTileAssignment(self: *const Mesh, allocator: std.mem.Allocator, shape: Shape) ![]u32 { + _ = try Sharding.Placement.init(&shape); + const devices = self.devicesInCanonicalOrder(); + var tile_axes: stdx.BoundedArray(u8, MAX_MESH_RANK) = .empty; + var used_axes: u8 = 0; + var tile_count: usize = 1; + for (0..shape.rank()) |ax| { + const logical_axis = shape.partition(ax).meshAxis() orelse continue; + const bit = @as(u8, 1) << @intCast(logical_axis); + if (used_axes & bit != 0) continue; + used_axes |= bit; + tile_axes.appendAssumeCapacity(@intCast(logical_axis)); + tile_count *= self.grid.axis_sizes[logical_axis]; + } + const replicas_per_tile = @divExact(devices.len, tile_count); + const replica_indices = try allocator.alloc(usize, tile_count); + defer allocator.free(replica_indices); + @memset(replica_indices, 0); + const ids = try allocator.alloc(u32, devices.len); + + for (0..devices.len) |partition_index| { + const coords: [MAX_MESH_RANK]u8 = self.grid.logical_coords[partition_index]; + var tile_index: usize = 0; + for (tile_axes.constSlice()) |logical_axis| { + tile_index = tile_index * self.grid.axis_sizes[logical_axis] + + coords[logical_axis]; + } + // HLO shardings refer to logical partitions; executable device assignment + // separately maps those partitions to physical PJRT device IDs. + ids[tile_index * replicas_per_tile + replica_indices[tile_index]] = @intCast(partition_index); + replica_indices[tile_index] += 1; + } + return ids; + } + + pub fn deviceAssignment(self: *const Mesh, allocator: std.mem.Allocator) ![]u32 { + const count = self.physical.devices_in_canonical_order.len; + + var ids = try allocator.alloc(u32, count); + @memset(ids, std.math.maxInt(u32)); + + for (self.physical.devices_in_canonical_order) |d| { + const coords = d.coords; + const idx = self.physical.linearIndexFromCoords(coords); + ids[idx] = d.id; + } + + for (ids) |id| { + if (id == std.math.maxInt(u32)) return error.MissingDeviceInTile; + } + + return ids; + } + + pub fn format(self: Mesh, writer: *std.Io.Writer) !void { + try writer.print("Mesh(name={s}, partitions={d}, replicas={d})\n", .{ self.name, self.numPartitions(), self.numReplicas() }); + + try writer.writeAll("┌ Mappings (logical coordinates from physical coordinates):\n"); + const num_axes = self.logical.axes.len; + for (self.logical.axes.slice(), self.logical.intents.slice(), 0..) |tag, intent, logical_axis| { + try writer.print("{s}── {s} ({s}) -> ", .{ if (logical_axis == num_axes - 1) "└" else "├", tag, @tagName(intent) }); + try self.formatAxisMapping(writer, logical_axis); + try writer.writeAll("\n"); + } + try writer.writeAll("\n"); + try writer.print("┌ {f}\n", .{self.physical}); + + try writer.writeAll("┌ Logical coordinates:\n"); + var coords: [MAX_MESH_RANK]usize = @splat(0); + try self.formatLogicalCoordinates(writer, &coords, 0); + } + + /// Reverse engineer the mapping logical -> physical from the device grid. + fn formatAxisMapping(self: Mesh, writer: *std.Io.Writer, logical_axis: usize) !void { + const Part = struct { + tag: PhysicalAxisTag, + physical_size: usize, + physical_stride: usize, + size: usize, + logical_stride: usize, + }; + // A u8 coordinate has at most eight nontrivial mixed-radix digits. + var parts: stdx.BoundedArray(Part, @bitSizeOf(u8)) = .empty; + const logical_size = self.grid.axis_sizes[logical_axis]; + var logical_stride: usize = 1; + while (logical_stride < logical_size) { + const remaining = @divExact(logical_size, logical_stride); + var size = remaining; + const part: Part = found: { + // Prefer the largest matching portion, so whole axes stay readable. + while (size > 1) : (size -= 1) { + if (remaining % size != 0) continue; + for (self.physical.axisOrder().constSlice()) |tag| { + const physical_size: usize = @intCast(self.physical.axis(tag)); + const depth = self.physical.axis_traversal.depth(tag).?; + var physical_stride: usize = 1; + while (physical_stride * size <= physical_size) : (physical_stride += 1) { + if (physical_size % (physical_stride * size) != 0) continue; + for (self.devicesInCanonicalOrder(), 0..) |device, i| { + const coords: [MAX_MESH_RANK]u8 = self.grid.logical_coords[i]; + if ((coords[logical_axis] / logical_stride) % size != + (device.coords[depth] / physical_stride) % size) break; + } else { + break :found .{ + .tag = tag, + .physical_size = physical_size, + .physical_stride = physical_stride, + .size = size, + .logical_stride = logical_stride, + }; + } + } + } + } + try writer.writeAll("device-dependent (see logical coordinates)"); + return; + }; + parts.appendAssumeCapacity(part); + logical_stride *= part.size; + } + if (parts.len == 0) { + try writer.writeAll("0 (replicated)"); + return; + } + var i: usize = parts.len; + while (i > 0) { + i -= 1; + const part = parts.constSlice()[i]; + if (i + 1 < parts.len) try writer.writeAll(" + "); + const partial = part.size != part.physical_size; + if (partial) try writer.writeAll("("); + try writer.writeAll(@tagName(part.tag)); + if (part.physical_stride > 1) try writer.print(" / {d}", .{part.physical_stride}); + if (part.physical_stride * part.size < part.physical_size) try writer.print(" % {d}", .{part.size}); + if (partial) try writer.writeAll(")"); + if (part.logical_stride > 1) try writer.print(" * {d}", .{part.logical_stride}); + } + } + + fn formatLogicalCoordinates(self: Mesh, writer: *std.Io.Writer, coords: *[MAX_MESH_RANK]usize, depth: usize) std.Io.Writer.Error!void { + const axes = self.logical.axes.constSlice(); + if (axes.len == 0) return; + const last = axes.len - 1; + if (depth < last) { + const size: usize = @intCast(self.numPartitionsForLogicalAxis(axes[depth])); + for (0..size) |coord| { + coords[depth] = coord; + try writer.splatByteAll(' ', 4 * depth); + try writer.print("└── {s}={d}\n", .{ axes[depth], coord }); + try self.formatLogicalCoordinates(writer, coords, depth + 1); + } + return; + } + + try writer.splatByteAll(' ', 4 * depth); + try writer.print("└── {s}=", .{axes[last]}); + const size: usize = @intCast(self.numPartitionsForLogicalAxis(axes[last])); + for (0..size) |coord| { + if (coord > 0) try writer.writeAll(","); + try writer.print("{d}", .{coord}); + } + try writer.writeAll(" -> devices="); + for (0..size) |coord| { + coords[last] = coord; + if (coord > 0) try writer.writeAll(","); + var count: usize = 0; + for (0..self.grid.device_count) |device_index| { + if (self.matchesLogicalCoordinates(device_index, coords[0..axes.len])) count += 1; + } + // Keep replicas grouped so each entry corresponds to one last-axis coordinate. + if (count != 1) try writer.writeAll("["); + var written: usize = 0; + for (self.devicesInCanonicalOrder(), 0..) |device, device_index| { + if (!self.matchesLogicalCoordinates(device_index, coords[0..axes.len])) continue; + if (written > 0) try writer.writeAll(","); + try writer.print("{d}", .{device.id}); + written += 1; + } + if (count != 1) try writer.writeAll("]"); + } + try writer.writeAll("\n"); + } + + fn matchesLogicalCoordinates(self: Mesh, device_index: usize, coords: []const usize) bool { + const actual: [MAX_MESH_RANK]u8 = self.grid.logical_coords[device_index]; + for (coords, 0..) |expected, logical_axis| { + if (actual[logical_axis] != expected) return false; + } + return true; + } +}; + +/// Resolved logical coordinates in canonical physical-device order. +/// Unused logical axes have coordinate zero. +pub const DeviceGrid = struct { + pub const Coordinates = [MAX_MESH_RANK]u8; + + comptime { + std.debug.assert(Platform.MAX_NUM_DEVICES <= 256); + } + + rank: u8, + device_count: usize, + axis_sizes: [MAX_MESH_RANK]u32, + logical_coords: [Platform.MAX_NUM_DEVICES]Coordinates align(16), + + fn init(mesh_: *const Mesh, strategy: *const Strategy) !DeviceGrid { + for (strategy.folding.constSlice()) |entry| { + for (entry.sources.constSlice()) |source| { + if (!mesh_.physical.hasAxis(source)) return error.InvalidPhysicalAxis; + } + } + const devices = mesh_.devicesInCanonicalOrder(); + if (devices.len > Platform.MAX_NUM_DEVICES) return error.InvalidPhysicalMesh; + var grid: DeviceGrid = .{ + .rank = @intCast(mesh_.logical.axes.len), + .device_count = devices.len, + .axis_sizes = @splat(1), + .logical_coords = @splat(@splat(0)), + }; + for (mesh_.logical.axes.constSlice(), 0..) |logical_tag, logical_axis| { + var used: std.EnumSet(PhysicalAxisTag) = .empty; + for (strategy.bindings.constSlice()) |binding_| { + if (!std.mem.eql(u8, std.mem.span(binding_.logical), std.mem.span(logical_tag))) continue; + for (binding_.physical.constSlice()) |part| { + if (part.size) |size| { + const info = mesh_.physical.axisInfo(part.axis) orelse return error.InvalidPhysicalAxis; + const physical_size: u32 = @intCast(info.size); + if (size == 0 or part.stride == 0 or size > physical_size or part.stride > physical_size / size) return error.InvalidPartialBinding; + if (physical_size % (part.stride * size) != 0) return error.InvalidPartialBinding; + try grid.appendAxis(mesh_, logical_axis, part.axis, part.stride, size); + } else { + for (strategy.foldSources(part.axis) orelse &.{part.axis}) |source| { + if (used.contains(source)) continue; + used.insert(source); + const info = mesh_.physical.axisInfo(source) orelse return error.InvalidPhysicalAxis; + try grid.appendAxis(mesh_, logical_axis, source, 1, @intCast(info.size)); + } + } + } + } + const dimension = mesh_.logical.dimensions[logical_axis]; + if (dimension != -1) { + if (dimension <= 0) return error.InvalidLogicalDimension; + const size: u32 = @intCast(dimension); + if (grid.axis_sizes[logical_axis] % size != 0) return error.IncompatibleLogicalDimension; + grid.axis_sizes[logical_axis] = size; + // Keep the least-significant portion of the ordered binding; the rest replicates. + for (grid.logical_coords[0..devices.len]) |*row| { + var coords: [MAX_MESH_RANK]u8 = row.*; + coords[logical_axis] = @intCast(coords[logical_axis] % size); + row.* = coords; + } + } + } + // Shared physical axes are valid only when their portions produce a complete + // Cartesian logical grid with the same number of replicas at every coordinate. + var partitions: usize = 1; + for (grid.axis_sizes[0..grid.rank]) |size| { + if (size > devices.len / partitions) return error.OverlappingLogicalAxes; + partitions *= size; + } + if (devices.len % partitions != 0) return error.OverlappingLogicalAxes; + var counts: [Platform.MAX_NUM_DEVICES]usize = @splat(0); + for (grid.logical_coords[0..devices.len]) |row| { + const coords: [MAX_MESH_RANK]u8 = row; + var index: usize = 0; + for (coords[0..grid.rank], grid.axis_sizes[0..grid.rank]) |coord, size| index = index * size + coord; + counts[index] += 1; + } + for (counts[0..partitions]) |count| { + if (count != devices.len / partitions) return error.OverlappingLogicalAxes; + } + return grid; + } + + fn appendAxis(self: *DeviceGrid, mesh_: *const Mesh, logical_axis: usize, tag: PhysicalAxisTag, stride: u32, size: u32) !void { + if (size > Platform.MAX_NUM_DEVICES / self.axis_sizes[logical_axis]) return error.InvalidPartialBinding; + const depth = mesh_.physical.axis_traversal.depth(tag).?; + self.axis_sizes[logical_axis] *= size; + for (mesh_.devicesInCanonicalOrder(), 0..) |device, i| { + var coords: [MAX_MESH_RANK]u8 = self.logical_coords[i]; + coords[logical_axis] = @intCast(@as(u32, coords[logical_axis]) * size + (device.coords[depth] / stride) % size); + self.logical_coords[i] = coords; + } + } +}; + +pub const Strategy = struct { + pub const Fold = struct { + target: PhysicalAxisTag, + sources: PhysicalList, + }; + pub const PhysicalBinding = struct { + axis: PhysicalAxisTag, + stride: u32 = 1, + /// null binds the whole axis (including any explicit fold). + size: ?u32 = null, + }; + pub const Binding = struct { + logical: Shape.Tag, + physical: stdx.BoundedArray(PhysicalBinding, MAX_MESH_RANK), + }; + + pub const Bindings = stdx.BoundedArray(Binding, MAX_MESH_RANK); + pub const Folding = stdx.BoundedArray(Strategy.Fold, MAX_MESH_RANK); + + bindings: Bindings, + folding: Folding, + + // TODO: remove .init, promote .parseBindings to .init + // It's currently an error to call .init and never calling `addBinding` + pub const init: Strategy = .{ .bindings = .empty, .folding = .empty }; + + pub fn parseBindings(bindings: anytype) Strategy { + const err_msg = "Strategy.parseBindings excepts fields to be PhysicalAxisTag or tuple of PhysicalAxisTag, got {}"; + var res: Strategy = .{ .bindings = .empty, .folding = .empty }; + const info = @typeInfo(@TypeOf(bindings)).@"struct"; + if (info.field_names.len == 0) @compileError("Strategy.parseBindings requires at least one binding"); + inline for (info.field_names, info.field_types) |field_name, Field| { + switch (@typeInfo(Field)) { + .enum_literal, .@"enum" => res.addBinding(field_name, @field(bindings, field_name)), + .@"struct" => |struct_info| { + stdx.debug.assertComptime(struct_info.is_tuple, err_msg, .{@TypeOf(bindings)}); + inline for (@field(bindings, field_name)) |axis_tag| { + res.addBinding(field_name, axis_tag); + } + }, + else => stdx.debug.compileError(err_msg, .{@TypeOf(bindings)}), + } + } + return res; + } + + pub fn addBinding(self: *Strategy, logical: anytype, physical: PhysicalAxisTag) void { + self.appendBinding(Shape.toTag(logical), .{ .axis = physical }); + } + + /// Append the mixed-radix portion `(coordinate / stride) % size` of a physical axis. + /// `stride * size` must divide the physical axis size. Partial bindings do not expand folds. + /// Portions are concatenated in insertion order, most significant first. + pub fn addPartialBinding(self: *Strategy, logical: anytype, physical: PhysicalAxisTag, stride: u32, size: u32) void { + self.appendBinding(Shape.toTag(logical), .{ .axis = physical, .stride = stride, .size = size }); + } + + fn appendBinding(self: *Strategy, logical: Shape.Tag, physical: PhysicalBinding) void { + for (self.bindings.slice()) |*binding_| { + if (std.mem.eql(u8, std.mem.span(binding_.logical), std.mem.span(logical))) { + binding_.physical.appendAssumeCapacity(physical); + return; + } + } + self.bindings.appendAssumeCapacity(.{ .logical = logical, .physical = .init(&.{physical}) }); + } + + fn foldSources(self: *const Strategy, tag: PhysicalAxisTag) ?[]const PhysicalAxisTag { + for (self.folding.constSlice()) |*fold| { + if (fold.target == tag) return fold.sources.constSlice(); + } + return null; + } + + /// Explicitly fold axes: `target` is the kept axis, `sources` define order. + /// If `target` is missing from `sources`, it is prepended. + pub fn addFold(self: *Strategy, target: PhysicalAxisTag, sources: []const PhysicalAxisTag) void { + var list: PhysicalList = .empty; + + var has_target = false; + for (sources) |s| { + if (s == target) { + has_target = true; + break; + } + } + + if (!has_target) list.appendAssumeCapacity(target); + for (sources) |s| list.appendAssumeCapacity(s); + + for (self.folding.slice()) |*f| { + if (f.target == target) { + f.sources = list; + return; + } + } + + self.folding.appendAssumeCapacity(.{ .target = target, .sources = list }); + } + + /// suggest builds a Strategy from logical intents. + pub fn suggest(logical: LogicalMesh, physical: *const PhysicalMesh) Strategy { + var strategy: Strategy = .{ .bindings = .empty, .folding = .empty }; + + const base_order = physical.shardableAxes(); + var available: stdx.BoundedArray(PhysicalAxisTag, MAX_MESH_RANK) = .empty; + for (base_order) |tag| { + if (physical.hasAxis(tag)) available.appendAssumeCapacity(tag); + } + + if (available.len == 0) { + for (physical.axisOrder().slice()) |tag| available.appendAssumeCapacity(tag); + } + + const avail = available.constSlice(); + if (avail.len == 0) return strategy; + var remaining: [MAX_MESH_RANK]u32 = @splat(1); + for (avail, 0..) |tag, i| remaining[i] = @intCast(physical.axis(tag)); + + // Reserve fixed dimensions before automatic axes can consume their capacity. + // Carve fixed portions from the most-significant end, leaving a contiguous + // low-order portion for automatic axes (e.g. model parallelism on one link). + for ([_]bool{ true, false }) |fixed| { + var counters: std.EnumArray(LogicalAxisIntent, u32) = .initFill(0); + inline for (std.meta.tags(LogicalAxisIntent)) |target_intent| { + for (logical.axes.constSlice(), logical.intents.constSlice(), 0..) |tag, intent, logical_axis| { + if (intent != target_intent) continue; + const i = counters.get(target_intent); + counters.set(target_intent, @intCast(i + 1)); + const dimension = logical.dimensions[logical_axis]; + if ((dimension != -1) != fixed) continue; + const preferred = switch (target_intent) { + .high_bandwidth => i % avail.len, + .balanced => (avail.len / 2 + i) % avail.len, + .low_bandwidth => avail.len - 1 - (i % avail.len), + }; + if (fixed) { + if (dimension <= 1) { + strategy.addPartialBinding(tag, avail[preferred], 1, 1); + continue; + } + var needed: u32 = @intCast(dimension); + for (0..avail.len) |offset| { + const axis = if (target_intent == .low_bandwidth) + (preferred + avail.len - offset) % avail.len + else + (preferred + offset) % avail.len; + const size = std.math.gcd(needed, remaining[axis]); + if (size == 1) continue; + remaining[axis] = @divExact(remaining[axis], size); + strategy.addPartialBinding(tag, avail[axis], remaining[axis], size); + needed = @divExact(needed, size); + if (needed == 1) break; + } + // Mesh.init reports an incompatible dimension if capacity was insufficient. + if (needed == @as(u32, @intCast(dimension))) strategy.addPartialBinding(tag, avail[preferred], 1, 1); + } else if (remaining[preferred] > 1) { + strategy.addPartialBinding(tag, avail[preferred], 1, remaining[preferred]); + remaining[preferred] = 1; + } + } + } + } + if (strategy.bindings.len == 0) strategy.addPartialBinding(logical.axes.constSlice()[0], avail[0], 1, 1); + + return strategy; + } +}; + +/// For a given shape, compute the shape of the slice each shard will receive. +pub fn shardedShape(shape: Shape) error{IncompatibleSharding}!Shape { + if (shape._sharding.mesh != null) { + const pl: Placement = try Sharding.Placement.init(&shape); + var local_shape = pl.localShape(shape); + local_shape._sharding = .replicated; + return local_shape; + } else { + // fully replicated + return shape; + } +} + +/// Owns per-device addressing metadata for a shape. +pub const Placement = struct { + pub const Slices = stdx.BoundedArray(Slice1d, Shape.MAX_RANK); + pub const Slice1d = struct { + start: i64, + size: i64, + }; + + local_dims: stdx.BoundedArray(i64, Shape.MAX_RANK), + mesh_ptr: ?*const Mesh, + byte_steps: [MAX_MESH_RANK]u64, + + pub fn init(shape: *const Shape) error{IncompatibleSharding}!Placement { + const byte_strides = shape.computeByteStrides(); + return initWithByteStrides(shape, byte_strides.constSlice()); + } + + /// Reuse the caller's contiguous byte strides without recomputing them. + pub fn initWithByteStrides(shape: *const Shape, byte_strides: []const i64) error{IncompatibleSharding}!Placement { + std.debug.assert(byte_strides.len == shape.rank()); + if (builtin.mode == .debug) std.debug.assert(shape._sharding.partition.hasUniqueAxes()); + var result: Placement = .{ + .local_dims = .{ .buffer = undefined, .len = shape.rank() }, + .mesh_ptr = shape._sharding.mesh, + .byte_steps = @splat(0), + }; + for (shape.dims(), result.local_dims.slice(), 0..) |dim, *result_local_dim, ax| { + result_local_dim.* = dim; + const mesh_ = result.mesh_ptr orelse continue; + const logical_axis = shape.partition(ax).meshAxis() orelse continue; + const count = mesh_.grid.axis_sizes[logical_axis]; + if (@rem(dim, count) != 0) return error.IncompatibleSharding; + const local_dim = @divExact(dim, count); + result_local_dim.* = local_dim; + result.byte_steps[logical_axis] = @intCast(local_dim * byte_strides[ax]); + } + return result; + } + + pub fn mesh(pl: *const Placement) ?*const Mesh { + return pl.mesh_ptr; + } + + /// Reconstruct metadata only for consumers that need a full shard shape. + pub fn localShape(pl: *const Placement, source: Shape) Shape { + std.debug.assert(source.rank() == pl.local_dims.len); + var result = source; + @memcpy(result._dims.slice(), pl.local_dims.constSlice()); + result._sharding.mesh = pl.mesh_ptr; + return result; + } + + pub fn slices(pl: *const Placement, device_index: usize, source: Shape) Slices { + std.debug.assert(source.rank() == pl.local_dims.len); + var res: Slices = .{ .buffer = undefined, .len = pl.local_dims.len }; + if (pl.mesh_ptr == null) { + for (res.slice(), pl.local_dims.constSlice()) |*slice, dim| slice.* = .{ .start = 0, .size = dim }; + return res; + } + const grid = &pl.mesh_ptr.?.grid; + std.debug.assert(device_index < grid.device_count); + const coords: [MAX_MESH_RANK]u8 = grid.logical_coords[device_index]; + var used_axes: u8 = 0; + for (res.slice(), pl.local_dims.constSlice(), 0..) |*r, size, ax| { + var coordinate: i64 = 0; + if (source.partition(ax).meshAxis()) |logical_axis| { + const bit = @as(u8, 1) << @intCast(logical_axis); + if (used_axes & bit == 0) coordinate = coords[logical_axis]; + used_axes |= bit; + } + r.* = .{ .start = coordinate * size, .size = size }; + } + return res; + } + + /// Validate the contiguous source slice once before iterating over devices. + /// The returned pointer includes the slice's byte offset. + pub fn basePtr(pl: *const Placement, slice: Slice) [*]const u8 { + if (builtin.mode == .debug) { + std.debug.assert(pl.local_dims.len == slice.shape.rank()); + } + return slice.constData().ptr; + } + + /// device_index is a canonical ordinal, never a PJRT device ID. + /// base_ptr must reference validated contiguous data for the global shape. + pub fn shardPtr(pl: *const Placement, device_index: usize, base_ptr: [*]const u8) [*]const u8 { + return base_ptr + pl.byteOffset(device_index); + } + + fn byteOffset(pl: *const Placement, device_index: usize) usize { + const mesh_ = pl.mesh() orelse return 0; + std.debug.assert(device_index < mesh_.grid.device_count); + const coords: [MAX_MESH_RANK]u8 = mesh_.grid.logical_coords[device_index]; + const steps = pl.byte_steps; + // Placement coefficients and grid coordinates describe offsets within the + // validated source slice. Keep arithmetic checks in debug only. + return offset: { + @setRuntimeSafety(builtin.mode == .debug); + var offset: u64 = 0; + inline for (0..MAX_MESH_RANK) |axis| { + offset += @as(u64, coords[axis]) * steps[axis]; + } + break :offset @intCast(offset); + }; + } + + /// Given a Slice, returns the subslice corresponding to a specific device. + /// Depending on the layout, the sub-slice may or may not be contiguous. + pub fn shardSlice(pl: *const Placement, device_index: usize, slice: Slice) Slice { + const shard_ptr = pl.shardPtr(device_index, pl.basePtr(slice)); + const offset_bytes = @intFromPtr(shard_ptr) - @intFromPtr(slice.bytes.ptr); + return .{ + .bytes = slice.bytes, + .mutable = slice.mutable, + .shape = pl.localShape(slice.shape), + .offset_bytes = offset_bytes, + .byte_strides = slice.byte_strides, + }; + } + + pub fn format(pl: *const Placement, writer: *std.Io.Writer) std.Io.Writer.Error!void { + try writer.print( + "Placement(local_dims={any}, mesh={*}, byte_steps={any})\n", + .{ pl.local_dims.constSlice(), pl.mesh_ptr, pl.byte_steps }, + ); + } +}; + +const ShardingTest = struct { + pub const ExpectedShard = struct { + // TODO: remove device_id from here, we always have ids from 0 to N + device_id: usize, + slices: []const [2]i64, + }; + + pub const Scenario = struct { + mesh: *const Mesh, + shape: Shape, + + expected_num_partitions: u32, + expected_num_replicas: u32, + expect_error: ?anyerror = null, + expected_sdy: ?[]const u8 = null, + expected_shards: []const ExpectedShard = &.{}, + + pub fn format(scenario: Scenario, writer: *std.Io.Writer) std.Io.Writer.Error!void { + try writer.print("Scenario: partitions={d}, replicas={d}\n", .{ scenario.expected_num_partitions, scenario.expected_num_replicas }); + if (scenario.expect_error) |err| try writer.print("- error {}\n", .{err}); + if (scenario.expected_sdy) |sdy| try writer.print("- shardy: {s}\n", .{sdy}); + if (scenario.expected_shards.len > 0) try writer.print("- shards:\n", .{}); + for (scenario.expected_shards) |shard| { + try writer.print("\t- device {d}: slices={any}\n", .{ shard.device_id, shard.slices }); + } + } + }; + + allocator: std.mem.Allocator, + + pub fn init(allocator: std.mem.Allocator) ShardingTest { + return .{ .allocator = allocator }; + } + + /// Creates a 1D-3D PhysicalMesh from simple dimensions. + pub fn physical(self: ShardingTest, dims: anytype, geometry: AxisGeometry) !PhysicalMesh { + const info = @typeInfo(@TypeOf(dims)).@"struct"; + const N = info.field_names.len; + var sizes: [N]usize = undefined; + inline for (info.field_names, sizes[0..]) |field_name, *s| { + s.* = @intCast(@field(dims, field_name)); + } + + const tags: [3]PhysicalAxisTag = .{ .link_x, .link_y, .link_z }; + var next_id: u32 = 0; + const root = try self.buildNode(tags[0..N], sizes[0..N], 0, &next_id, geometry); + return try .fromTree(self.allocator, .tpu, root); + } + + fn buildNode(self: ShardingTest, tags: []const PhysicalAxisTag, sizes: []const usize, depth: usize, next_id: *u32, geometry: AxisGeometry) !PhysicalNode { + if (depth == tags.len) { + const id = next_id.*; + next_id.* += 1; + return .{ .leaf = .{ .id = id, .coords = Device.undefined_coords } }; + } + const count = sizes[depth]; + const children = try self.allocator.alloc(PhysicalNode, count); + for (children) |*child| { + child.* = try self.buildNode(tags, sizes, depth + 1, next_id, geometry); + } + return .{ .branch = .{ .tag = tags[depth], .geometry = geometry, .children = children } }; + } + + pub fn run(self: ShardingTest, s: Scenario) !void { + const mesh = s.mesh; + errdefer std.log.warn("Failed mesh {f}", .{s}); + + try std.testing.expectEqual(s.expected_num_partitions, mesh.numPartitions()); + try std.testing.expectEqual(s.expected_num_replicas, mesh.numReplicas()); + try std.testing.expectEqual(mesh.grid.device_count, @as(usize, @intCast(mesh.numDevices()))); + + // Verify MLIR String + if (s.expected_sdy) |expected_attr| { + errdefer std.log.warn("Expected shardy annotation failed", .{}); + const registry: *mlir.DialectRegistry = try .init(); + defer registry.deinit(); + shardy.registerDialect(registry); + func.registerExtensions(registry); + + var ctx: *mlir.Context = try .init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + + const actual_attr = try mesh.sdyShardingAttrForShape(self.allocator, ctx, s.shape); + var writer: std.Io.Writer.Allocating = .init(self.allocator); + defer writer.deinit(); + try actual_attr.attribute().format(&writer.writer); + try std.testing.expectEqualStrings(expected_attr, writer.written()); + } + + // Verify Placement logic / Error + const ordered_devices = mesh.devicesInCanonicalOrder(); + if (s.expect_error) |err| { + errdefer std.log.warn("Expected error failed", .{}); + try std.testing.expect(ordered_devices.len > 0); + try std.testing.expectError(err, Sharding.Placement.init(&s.shape)); + return; + } + + // Verify Shard Slices (Math) + if (s.expected_shards.len > 0) { + const pl: Placement = try Sharding.Placement.init(&s.shape); + errdefer std.log.warn("Expected shards failed. Got {f}", .{pl}); + try std.testing.expectEqual(s.expected_shards.len, ordered_devices.len); + for (s.expected_shards, ordered_devices, 0..) |expected, device, device_index| { + { + try std.testing.expectEqual(expected.device_id, device.id); + + // Check pl.slices(device) + const actual_slices = pl.slices(device_index, s.shape); + + // Consider rewriting test cases to use Slice1D. This requires passing the axis + const actual_start_len = try self.allocator.alloc([2]i64, actual_slices.len); + defer self.allocator.free(actual_start_len); + for (actual_start_len, actual_slices.slice()) |*s_l, slice| { + s_l.* = .{ slice.start, slice.size }; + } + + try std.testing.expectEqualSlices([2]i64, expected.slices, actual_start_len); + } + { + // Check pl.shape + const expected_sizes = try self.allocator.alloc(i64, expected.slices.len); + defer self.allocator.free(expected_sizes); + for (expected_sizes, expected.slices) |*expected_size, slice| { + expected_size.* = slice[1]; + } + + try std.testing.expectEqualSlices(i64, expected_sizes, pl.local_dims.constSlice()); + } + } + } + } +}; + +test "mesh: unknown partitioning implies replication" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const runner: ShardingTest = .init(arena.allocator()); + + const physical: PhysicalMesh = try runner.physical(.{ 2, 2 }, .{ .mesh = .torus }); + const logical: LogicalMesh = .mesh(.{ .batch = .low_bandwidth }); + + const strategy: Strategy = .parseBindings(.{ .batch = .link_x }); + const mesh: Sharding.Mesh = try .init("folded_mesh", &physical, logical, strategy); + + try runner.run(.{ + .expected_num_partitions = 2, + .expected_num_replicas = 2, + .mesh = &mesh, + // No partitioning on shape + .shape = Shape.init(.{ .batch = 8 }, .f32), + .expected_sdy = "#sdy.sharding<@folded_mesh, [{}], replicated={\"batch\", \"__replica_0\"}>", + .expected_shards = &.{ + .{ .device_id = 0, .slices = &.{.{ 0, 8 }} }, + .{ .device_id = 1, .slices = &.{.{ 0, 8 }} }, + .{ .device_id = 2, .slices = &.{.{ 0, 8 }} }, + .{ .device_id = 3, .slices = &.{.{ 0, 8 }} }, + }, + }); +} + +test "suggest splits a one-dimensional topology for fixed data and automatic model axes" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const allocator = arena.allocator(); + const runner: ShardingTest = .init(allocator); + const logical = try LogicalMesh.init(.{ .data, .model }, &.{ .low_bandwidth, .high_bandwidth }, &.{ 2, -1 }); + for ([_]usize{ 2, 4, 8 }) |count| { + const physical = try runner.physical(.{count}, .{ .mesh = .torus }); + const mesh_ = try Mesh.init("DP-MP", &physical, logical, .suggest(logical, &physical)); + try std.testing.expectEqual(2, mesh_.numPartitionsForLogicalAxis(.data)); + try std.testing.expectEqual(@as(u32, @intCast(count / 2)), mesh_.numPartitionsForLogicalAxis(.model)); + try std.testing.expectEqual(1, mesh_.numReplicas()); + for (0..count) |i| { + const coords: [MAX_MESH_RANK]u8 = mesh_.grid.logical_coords[i]; + try std.testing.expectEqual(@as(u8, @intCast(i / (count / 2))), coords[0]); + try std.testing.expectEqual(@as(u8, @intCast(i % (count / 2))), coords[1]); + } + const shape = Shape.init(.{ .b = 8, .m = 8 }, .f32).withPartitioning(&mesh_, .{ .b = .data, .m = .model }); + try runner.run(.{ .mesh = &mesh_, .shape = shape, .expected_num_partitions = @intCast(count), .expected_num_replicas = 1 }); + } +} + +test "suggest reserves fixed dimensions across physical axes before automatic allocation" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const runner: ShardingTest = .init(arena.allocator()); + const physical = try runner.physical(.{ 4, 3 }, .{ .mesh = .torus }); + const logical = try LogicalMesh.init(.{ .model, .data }, &.{ .high_bandwidth, .low_bandwidth }, &.{ -1, 6 }); + const mesh_ = try Mesh.init("split", &physical, logical, .suggest(logical, &physical)); + try std.testing.expectEqual(2, mesh_.numPartitionsForLogicalAxis(.model)); + try std.testing.expectEqual(6, mesh_.numPartitionsForLogicalAxis(.data)); + try std.testing.expectEqual(1, mesh_.numReplicas()); + + const fixed = try LogicalMesh.init(.{ .data, .model }, &.{ .low_bandwidth, .high_bandwidth }, &.{ 3, 4 }); + const fixed_mesh = try Mesh.init("fixed", &physical, fixed, .suggest(fixed, &physical)); + try std.testing.expectEqual(12, fixed_mesh.numPartitions()); + const impossible = try LogicalMesh.init(.{ .data, .model }, &.{ .low_bandwidth, .high_bandwidth }, &.{ 5, -1 }); + try std.testing.expectError(error.IncompatibleLogicalDimension, Mesh.init("impossible", &physical, impossible, .suggest(impossible, &physical))); + const replica_logical = try LogicalMesh.init(.{ .data, .model }, &.{ .low_bandwidth, .high_bandwidth }, &.{ 1, 1 }); + const replica_mesh = try Mesh.init("replicated", &physical, replica_logical, .suggest(replica_logical, &physical)); + try std.testing.expectEqual(12, replica_mesh.numReplicas()); +} + +test "mesh: suggest strategy realization" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const runner: ShardingTest = .init(arena.allocator()); + + const physical = try runner.physical(.{ 2, 2, 2 }, .{ .mesh = .torus }); + + const logical: LogicalMesh = .mesh(.{ + .batch = .low_bandwidth, + .model = .high_bandwidth, + }); + + const strategy: Strategy = .suggest(logical, &physical); + const mesh: Mesh = try .init("suggested_mesh", &physical, logical, strategy); + + try runner.run(.{ + .expected_num_partitions = 4, + .expected_num_replicas = 2, + .mesh = &mesh, + .shape = Shape.init(.{ .batch = 4, .model = 4 }, .f32) + .withPartitioning(&mesh, .{ .batch = .batch, .model = .model }), + .expected_sdy = "#sdy.sharding<@suggested_mesh, [{\"batch\"}, {\"model\"}], replicated={\"__replica_0\"}>", + .expected_shards = &.{ + .{ .device_id = 0, .slices = &.{ .{ 0, 2 }, .{ 0, 2 } } }, // x0, y0, z0 + .{ .device_id = 1, .slices = &.{ .{ 2, 2 }, .{ 0, 2 } } }, // x0, y0, z1 (batch split) + .{ .device_id = 2, .slices = &.{ .{ 0, 2 }, .{ 0, 2 } } }, // x0, y1, z0 (replicated y) + .{ .device_id = 3, .slices = &.{ .{ 2, 2 }, .{ 0, 2 } } }, // x0, y1, z1 + .{ .device_id = 4, .slices = &.{ .{ 0, 2 }, .{ 2, 2 } } }, // x1, y0, z0 (model split) + .{ .device_id = 5, .slices = &.{ .{ 2, 2 }, .{ 2, 2 } } }, + .{ .device_id = 6, .slices = &.{ .{ 0, 2 }, .{ 2, 2 } } }, + .{ .device_id = 7, .slices = &.{ .{ 2, 2 }, .{ 2, 2 } } }, + }, + }); +} + +test "mesh: suggest leaves conflicting logical axes replicated" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const runner: ShardingTest = .init(arena.allocator()); + + const physical = try runner.physical(.{4}, .point_to_point); + + const logical: LogicalMesh = .mesh(.{ + .model = .high_bandwidth, + .experts = .high_bandwidth, + }); + + const strategy: Strategy = .suggest(logical, &physical); + + const mesh: Mesh = try .init("fold_mesh", &physical, logical, strategy); + try runner.run(.{ + .expected_num_partitions = 4, + .expected_num_replicas = 1, + .mesh = &mesh, + .shape = Shape.init(.{ .model = 4, .experts = 4 }, .f32) + .withPartitioning(&mesh, .{ .model = .model, .experts = .experts }), + .expected_sdy = "#sdy.sharding<@fold_mesh, [{\"model\"}, {\"experts\"}]>", + .expected_shards = &.{ + .{ .device_id = 0, .slices = &.{ .{ 0, 1 }, .{ 0, 4 } } }, + .{ .device_id = 1, .slices = &.{ .{ 1, 1 }, .{ 0, 4 } } }, + .{ .device_id = 2, .slices = &.{ .{ 2, 1 }, .{ 0, 4 } } }, + .{ .device_id = 3, .slices = &.{ .{ 3, 1 }, .{ 0, 4 } } }, + }, + }); + // Swapping tensor dimensions must not change which logical axis owns the devices. + try runner.run(.{ + .expected_num_partitions = 4, + .expected_num_replicas = 1, + .mesh = &mesh, + .shape = Shape.init(.{ .experts = 4, .model = 4 }, .f32) + .withPartitioning(&mesh, .{ .experts = .experts, .model = .model }), + .expected_shards = &.{ + .{ .device_id = 0, .slices = &.{ .{ 0, 4 }, .{ 0, 1 } } }, + .{ .device_id = 1, .slices = &.{ .{ 0, 4 }, .{ 1, 1 } } }, + .{ .device_id = 2, .slices = &.{ .{ 0, 4 }, .{ 2, 1 } } }, + .{ .device_id = 3, .slices = &.{ .{ 0, 4 }, .{ 3, 1 } } }, + }, + }); +} + +test "mesh: multiple physical axes on one logical dimension (folding)" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const runner: ShardingTest = .init(arena.allocator()); + + const physical: PhysicalMesh = try runner.physical(.{ 2, 2 }, .{ .mesh = .torus }); + const logical: LogicalMesh = .mesh(.{ .model = .high_bandwidth }); + + { + const strategy: Strategy = .parseBindings(.{ .model = .{ .link_x, .link_y } }); + const mesh: Mesh = try .init("folded_mesh", &physical, logical, strategy); + try runner.run(.{ + .expected_num_partitions = 4, + .expected_num_replicas = 1, + .mesh = &mesh, + .shape = Shape.init(.{ .model = 16 }, .f32).withPartitioning(&mesh, .{ .model = .model }), + .expected_sdy = "#sdy.sharding<@folded_mesh, [{\"model\"}]>", + .expected_shards = &.{ + .{ .device_id = 0, .slices = &.{.{ 0, 4 }} }, + .{ .device_id = 1, .slices = &.{.{ 4, 4 }} }, + .{ .device_id = 2, .slices = &.{.{ 8, 4 }} }, + .{ .device_id = 3, .slices = &.{.{ 12, 4 }} }, + }, + }); + } + + { + // Swap link_x and link_y order: + // -> the roles of devices 1 (0,1) and 2 (1,0) are swapped + const strategy: Strategy = .parseBindings(.{ .model = .{ .link_y, .link_x } }); + const mesh: Mesh = try .init("folded_mesh", &physical, logical, strategy); + try runner.run(.{ + .expected_num_partitions = 4, + .expected_num_replicas = 1, + .mesh = &mesh, + .shape = Shape.init(.{ .model = 16 }, .f32).withPartitioning(&mesh, .{ .model = .model }), + .expected_sdy = "#sdy.sharding<@folded_mesh, [{\"model\"}]>", + .expected_shards = &.{ + .{ .device_id = 0, .slices = &.{.{ 0, 4 }} }, + .{ .device_id = 1, .slices = &.{.{ 8, 4 }} }, + .{ .device_id = 2, .slices = &.{.{ 4, 4 }} }, + .{ .device_id = 3, .slices = &.{.{ 12, 4 }} }, + }, + }); + } +} + +test "mesh: explicit strategy folding" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const runner: ShardingTest = .init(arena.allocator()); + + const physical: PhysicalMesh = try runner.physical(.{ 2, 2, 2 }, .{ .mesh = .torus }); + const logical: LogicalMesh = .mesh(.{ .model = .high_bandwidth }); + + var strategy: Strategy = .parseBindings(.{ .model = .link_x }); + strategy.addFold(.link_x, &.{ .link_x, .link_z }); + + const mesh: Mesh = try .init("strategy_fold", &physical, logical, strategy); + try runner.run(.{ + .expected_num_partitions = 4, + .expected_num_replicas = 2, + .mesh = &mesh, + .shape = Shape.init(.{ .model = 16 }, .f32).withPartitioning(&mesh, .{ .model = .model }), + .expected_sdy = "#sdy.sharding<@strategy_fold, [{\"model\"}], replicated={\"__replica_0\"}>", + .expected_shards = &.{ + .{ .device_id = 0, .slices = &.{.{ 0, 4 }} }, + .{ .device_id = 1, .slices = &.{.{ 4, 4 }} }, + .{ .device_id = 2, .slices = &.{.{ 0, 4 }} }, + .{ .device_id = 3, .slices = &.{.{ 4, 4 }} }, + .{ .device_id = 4, .slices = &.{.{ 8, 4 }} }, + .{ .device_id = 5, .slices = &.{.{ 12, 4 }} }, + .{ .device_id = 6, .slices = &.{.{ 8, 4 }} }, + .{ .device_id = 7, .slices = &.{.{ 12, 4 }} }, + }, + }); +} + +test "mesh: open and replicated dimension mix" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const runner: ShardingTest = .init(arena.allocator()); + + const physical: PhysicalMesh = try runner.physical(.{ 2, 2 }, .{ .mesh = .torus }); + const logical: LogicalMesh = .mesh(.{ .batch = .low_bandwidth, .model = .high_bandwidth }); + + const strategy: Strategy = .parseBindings(.{ .batch = .link_x, .model = .link_y }); + + const mesh: Mesh = try .init("mix_mesh", &physical, logical, strategy); + try runner.run(.{ + .expected_num_partitions = 4, + .expected_num_replicas = 1, + .mesh = &mesh, + .shape = Shape.init(.{ .batch = 8, .model = 8 }, .f32).withPartitioning(&mesh, .{ .batch = .open, .model = .replicated }), + .expected_sdy = "#sdy.sharding<@mix_mesh, [{?}, {}]>", + .expected_shards = &.{ + .{ .device_id = 0, .slices = &.{ .{ 0, 8 }, .{ 0, 8 } } }, + .{ .device_id = 1, .slices = &.{ .{ 0, 8 }, .{ 0, 8 } } }, + .{ .device_id = 2, .slices = &.{ .{ 0, 8 }, .{ 0, 8 } } }, + .{ .device_id = 3, .slices = &.{ .{ 0, 8 }, .{ 0, 8 } } }, + }, + }); +} + +test "mesh: full 3D cluster mesh" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const runner: ShardingTest = .init(arena.allocator()); + + const physical: PhysicalMesh = try runner.physical(.{ 2, 2, 2 }, .{ .mesh = .torus }); + const logical: LogicalMesh = .mesh(.{ .batch = .low_bandwidth, .model = .high_bandwidth, .context = .balanced }); + + const strategy: Strategy = .parseBindings(.{ + .batch = .link_x, + .model = .link_y, + .context = .link_z, + }); + + const mesh: Mesh = try .init("3d_mesh", &physical, logical, strategy); + try runner.run(.{ + .expected_num_partitions = 8, + .expected_num_replicas = 1, + .mesh = &mesh, + // Note: .context and .model are swapped compared to .link_x and .link_z + .shape = Shape.init(.{ .batch = 4, .context = 4, .model = 4 }, .f32) + .withPartitioning(&mesh, .{ .batch = .batch, .model = .model, .context = .context }), + .expected_sdy = "#sdy.sharding<@\"3d_mesh\", [{\"batch\"}, {\"context\"}, {\"model\"}]>", + .expected_shards = &.{ + .{ .device_id = 0, .slices = &.{ .{ 0, 2 }, .{ 0, 2 }, .{ 0, 2 } } }, + .{ .device_id = 1, .slices = &.{ .{ 0, 2 }, .{ 2, 2 }, .{ 0, 2 } } }, + .{ .device_id = 2, .slices = &.{ .{ 0, 2 }, .{ 0, 2 }, .{ 2, 2 } } }, + .{ .device_id = 3, .slices = &.{ .{ 0, 2 }, .{ 2, 2 }, .{ 2, 2 } } }, + .{ .device_id = 4, .slices = &.{ .{ 2, 2 }, .{ 0, 2 }, .{ 0, 2 } } }, + .{ .device_id = 5, .slices = &.{ .{ 2, 2 }, .{ 2, 2 }, .{ 0, 2 } } }, + .{ .device_id = 6, .slices = &.{ .{ 2, 2 }, .{ 0, 2 }, .{ 2, 2 } } }, + .{ .device_id = 7, .slices = &.{ .{ 2, 2 }, .{ 2, 2 }, .{ 2, 2 } } }, + }, + }); +} + +test "mesh: num partitions for logical axis" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const runner: ShardingTest = .init(arena.allocator()); + + const physical: PhysicalMesh = try runner.physical(.{ 2, 2, 2 }, .{ .mesh = .torus }); + const logical: LogicalMesh = .mesh(.{ + .model = .high_bandwidth, + .batch = .low_bandwidth, + }); + + var strategy: Strategy = .parseBindings(.{ .model = .link_x }); + strategy.addFold(.link_x, &.{ .link_x, .link_z }); + + const mesh: Mesh = try .init("axis_parts_mesh", &physical, logical, strategy); + + try std.testing.expectEqual(4, mesh.numPartitionsForLogicalAxis(.model)); + try std.testing.expectEqual(1, mesh.numPartitionsForLogicalAxis(.batch)); +} + +test "mesh: size-one and unbound logical axes leave all devices replicated" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const runner: ShardingTest = .init(arena.allocator()); + const physical = try runner.physical(.{ 1, 6 }, .{ .mesh = .torus }); + const mesh_ = try Mesh.init("replicas", &physical, .mesh(.{ .data = .balanced, .model = .balanced }), .parseBindings(.{ .data = .link_x })); + try runner.run(.{ + .mesh = &mesh_, + .shape = Shape.init(.{ .batch = 12 }, .f32).withPartitioning(&mesh_, .{ .batch = .data }), + .expected_num_partitions = 1, + .expected_num_replicas = 6, + }); +} + +test "mesh: logical partitions and replicas can have different sizes" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const runner: ShardingTest = .init(arena.allocator()); + const physical = try runner.physical(.{ 2, 3, 2 }, .{ .mesh = .torus }); + var strategy: Strategy = .parseBindings(.{ .model = .link_x }); + strategy.addFold(.link_x, &.{ .link_z, .link_x }); + const mesh_ = try Mesh.init("folded", &physical, .mesh(.{ .model = .balanced }), strategy); + try runner.run(.{ + .mesh = &mesh_, + .shape = Shape.init(.{ .model = 8 }, .f32).withPartitioning(&mesh_, .{ .model = .model }), + .expected_num_partitions = 4, + .expected_num_replicas = 3, + }); +} + +test "localized shapes do not retain global sharding" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const runner: ShardingTest = .init(arena.allocator()); + const physical = try runner.physical(.{2}, .{ .mesh = .torus }); + const mesh_ = try Mesh.init("local", &physical, .mesh(.{ .batch = .balanced }), .parseBindings(.{ .batch = .link_x })); + try std.testing.expectEqual(mesh_.bindings.constSlice()[0].constSlice().ptr, mesh_.binding(0).ptr); + const global = Shape.init(.{ .batch = 8 }, .f32).withPartitioning(&mesh_, .{ .batch = .batch }); + const local = try shardedShape(global); + try std.testing.expectEqual(4, local.dim(.batch)); + try std.testing.expectEqual(null, local._sharding.mesh); + try std.testing.expectEqual(8, global.dim(.batch)); + try std.testing.expectEqual(&mesh_, global._sharding.mesh); +} + +test "GSPMD tiles use tensor axis order and logical partition indices" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const runner: ShardingTest = .init(arena.allocator()); + const physical = try runner.physical(.{ 2, 2, 2 }, .{ .mesh = .torus }); + for (physical.devices_in_canonical_order, 0..) |*device, i| device.id = @intCast(7 - i); + const mesh_ = try Mesh.init("tiles", &physical, .mesh(.{ .batch = .balanced, .model = .balanced }), .parseBindings(.{ .batch = .link_z, .model = .link_x })); + const shape = Shape.init(.{ .batch = 8, .model = 8 }, .f32).withPartitioning(&mesh_, .{ .batch = .batch, .model = .model }); + const ids = try mesh_.gspmdTileAssignment(arena.allocator(), shape); + try std.testing.expectEqualSlices(u32, &.{ 0, 2, 4, 6, 1, 3, 5, 7 }, ids); +} + +test "DeviceGrid stores aligned logical coordinates in canonical order" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const runner: ShardingTest = .init(arena.allocator()); + const physical = try runner.physical(.{ 2, 2 }, .{ .mesh = .torus }); + for (physical.devices_in_canonical_order, 0..) |*device, i| device.id = @intCast(3 - i); + const mesh_ = try Mesh.init("grid", &physical, .mesh(.{ .model = .balanced, .data = .balanced }), .parseBindings(.{ .model = .link_y, .data = .link_x })); + const expected = [_][4]u8{ .{ 0, 0, 0, 0 }, .{ 1, 0, 0, 0 }, .{ 0, 1, 0, 0 }, .{ 1, 1, 0, 0 } }; + try std.testing.expectEqual(@as(usize, 0), @intFromPtr(&mesh_.grid.logical_coords) % 16); + for (mesh_.grid.logical_coords[0..mesh_.grid.device_count], expected, 0..) |coords, want, i| { + try std.testing.expectEqual(@intFromPtr(&mesh_.grid.logical_coords) + 4 * i, @intFromPtr(&mesh_.grid.logical_coords[i])); + const actual: [4]u8 = coords; + try std.testing.expectEqualSlices(u8, &want, &actual); + } + try std.testing.expectEqual(@as(usize, 4), @sizeOf(DeviceGrid.Coordinates)); + const shape = Shape.init(.{ .m = 16, .n = 24 }, .f32).withPartitioning(&mesh_, .{ .m = .model, .n = .data }); + const placement = try Sharding.Placement.init(&shape); + const offsets = [_]usize{ 0, 768, 48, 816 }; + var data: [16 * 24 * 4 + 16]u8 = undefined; + var slice = Slice.init(shape, data[16..]); + slice.bytes = &data; + slice.offset_bytes = 16; + const base_ptr = placement.basePtr(slice); + try std.testing.expectEqual(@intFromPtr(&data) + 16, @intFromPtr(base_ptr)); + for (offsets, 0..) |offset, i| { + try std.testing.expectEqual(offset, placement.byteOffset(i)); + try std.testing.expectEqual(@intFromPtr(&data) + 16 + offset, @intFromPtr(placement.shardPtr(i, base_ptr))); + } +} + +test "partial bindings share physical axes and combine portions in order" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const allocator = arena.allocator(); + const runner: ShardingTest = .init(allocator); + const physical = try runner.physical(.{ 4, 6 }, .{ .mesh = .torus }); + const logical = try LogicalMesh.init(.{ .model, .data }, &.{ .balanced, .low_bandwidth }, &.{ -1, -1 }); + var strategy: Strategy = .init; + strategy.addPartialBinding(.model, .link_y, 2, 3); + strategy.addPartialBinding(.model, .link_x, 1, 2); + strategy.addPartialBinding(.data, .link_x, 2, 2); + strategy.addPartialBinding(.data, .link_y, 1, 2); + const mesh_ = try Mesh.init("partial", &physical, logical, strategy); + try std.testing.expectEqual(6, mesh_.numPartitionsForLogicalAxis(.model)); + try std.testing.expectEqual(4, mesh_.numPartitionsForLogicalAxis(.data)); + try std.testing.expectEqual(1, mesh_.numReplicas()); + for (physical.devices_in_canonical_order, 0..) |device, i| { + const coords: [MAX_MESH_RANK]u8 = mesh_.grid.logical_coords[i]; + try std.testing.expectEqual((device.coords[1] / 2) * 2 + device.coords[0] % 2, coords[0]); + try std.testing.expectEqual((device.coords[0] / 2) * 2 + device.coords[1] % 2, coords[1]); + } + const formatted = try std.fmt.allocPrint(allocator, "{f}", .{mesh_}); + try std.testing.expect(std.mem.indexOf(u8, formatted, "model (balanced) -> (link_y / 2) * 2 + (link_x % 2)") != null); + const shape = Shape.init(.{ .m = 12, .d = 8 }, .f32).withPartitioning(&mesh_, .{ .m = .model, .d = .data }); + try runner.run(.{ .mesh = &mesh_, .shape = shape, .expected_num_partitions = 24, .expected_num_replicas = 1 }); +} + +test "fixed logical dimensions constrain whole partial and folded bindings" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const runner: ShardingTest = .init(arena.allocator()); + const physical = try runner.physical(.{ 4, 6 }, .{ .mesh = .torus }); + const logical = try LogicalMesh.init(.{ .model, .data, .unused }, &.{ .balanced, .low_bandwidth, .balanced }, &.{ 2, -1, 1 }); + var strategy: Strategy = .parseBindings(.{ .model = .link_x, .data = .link_y }); + const mesh_ = try Mesh.init("constrained", &physical, logical, strategy); + try std.testing.expectEqual(2, mesh_.numPartitionsForLogicalAxis(.model)); + try std.testing.expectEqual(6, mesh_.numPartitionsForLogicalAxis(.data)); + try std.testing.expectEqual(1, mesh_.numPartitionsForLogicalAxis(.unused)); + try std.testing.expectEqual(2, mesh_.numReplicas()); + for (physical.devices_in_canonical_order, 0..) |device, i| { + try std.testing.expectEqual(device.coords[0] % 2, mesh_.grid.logical_coords[i][0]); + } + + const fixed = try LogicalMesh.init(.{.model}, &.{.balanced}, &.{2}); + strategy = .init; + strategy.addPartialBinding(.model, .link_x, 1, 4); + const partial = try Mesh.init("partial", &physical, fixed, strategy); + try std.testing.expectEqual(2, partial.numPartitions()); + try std.testing.expectEqual(12, partial.numReplicas()); + + strategy = .parseBindings(.{ .model = .link_x }); + strategy.addFold(.link_x, &.{ .link_y, .link_x }); + const folded = try Mesh.init("folded", &physical, try LogicalMesh.init(.{.model}, &.{.balanced}, &.{8}), strategy); + try std.testing.expectEqual(8, folded.numPartitions()); + try std.testing.expectEqual(3, folded.numReplicas()); + for (physical.devices_in_canonical_order, 0..) |device, i| { + try std.testing.expectEqual((device.coords[1] * 4 + device.coords[0]) % 8, folded.grid.logical_coords[i][0]); + } + for ([_]i32{ 5, 32 }) |dimension| { + try std.testing.expectError(error.IncompatibleLogicalDimension, Mesh.init("invalid", &physical, try LogicalMesh.init(.{.model}, &.{.balanced}, &.{dimension}), strategy)); + } +} + +test "partial bindings reject invalid portions and correlated logical axes" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const runner: ShardingTest = .init(arena.allocator()); + const physical = try runner.physical(.{8}, .{ .mesh = .torus }); + for ([_][2]u32{ .{ 0, 2 }, .{ 1, 0 }, .{ 3, 2 }, .{ 1, 3 }, .{ 4, 4 }, .{ 1, std.math.maxInt(u32) } }) |portion| { + var strategy: Strategy = .init; + strategy.addPartialBinding(.model, .link_x, portion[0], portion[1]); + try std.testing.expectError(error.InvalidPartialBinding, Mesh.init("invalid", &physical, .mesh(.{ .model = .balanced }), strategy)); + } + var strategy: Strategy = .init; + strategy.addPartialBinding(.model, .link_y, 1, 2); + try std.testing.expectError(error.InvalidPhysicalAxis, Mesh.init("invalid", &physical, .mesh(.{ .model = .balanced }), strategy)); + strategy = .init; + strategy.addPartialBinding(.model, .link_x, 1, 2); + strategy.addPartialBinding(.data, .link_x, 1, 2); + // Four claimed partitions fit on eight devices, but only two coordinates occur. + try std.testing.expectError(error.OverlappingLogicalAxes, Mesh.init("overlap", &physical, .mesh(.{ .model = .balanced, .data = .balanced }), strategy)); +} + +test "DeviceGrid resolves folds once in strategy order" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const allocator = arena.allocator(); + const runner: ShardingTest = .init(allocator); + const physical = try runner.physical(.{ 2, 3, 2 }, .{ .mesh = .torus }); + for (physical.devices_in_canonical_order, 0..) |*device, i| device.id = @intCast(11 - i); + var strategy: Strategy = .parseBindings(.{ .model = .{ .link_x, .link_z } }); + strategy.addFold(.link_x, &.{ .link_z, .link_x }); + const mesh_ = try Mesh.init("folded", &physical, .mesh(.{ .model = .balanced }), strategy); + strategy.folding = .empty; + + try std.testing.expectEqual(4, mesh_.numPartitions()); + try std.testing.expectEqual(3, mesh_.numReplicas()); + const expected = [_]u8{ 0, 2, 0, 2, 0, 2, 1, 3, 1, 3, 1, 3 }; + for (expected, 0..) |coord, i| { + try std.testing.expectEqual(coord, mesh_.grid.logical_coords[i][0]); + } + const formatted = try std.fmt.allocPrint(allocator, "{f}", .{mesh_}); + try std.testing.expect(std.mem.indexOf(u8, formatted, "model (balanced) -> link_z * 2 + link_x\n") != null); + try std.testing.expect(std.mem.indexOf(u8, formatted, "model=0,1,2,3 -> devices=[11,9,7],[5,3,1],[10,8,6],[4,2,0]") != null); +} + +test "DeviceGrid validates fold sources and overlapping folded axes" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const runner: ShardingTest = .init(arena.allocator()); + const physical = try runner.physical(.{ 2, 2 }, .{ .mesh = .torus }); + var strategy: Strategy = .parseBindings(.{ .model = .link_x }); + strategy.addFold(.link_y, &.{ .link_y, .link_z }); + try std.testing.expectError(error.InvalidPhysicalAxis, Mesh.init("invalid", &physical, .mesh(.{ .model = .balanced }), strategy)); + + strategy = .parseBindings(.{ .model = .link_x, .data = .link_y }); + strategy.addFold(.link_x, &.{ .link_x, .link_y }); + try std.testing.expectError(error.OverlappingLogicalAxes, Mesh.init("overlap", &physical, .mesh(.{ .model = .balanced, .data = .balanced }), strategy)); +} + +test "Placement owns dimensions and byte steps across copies" { + const Helper = struct { + fn create() !Placement { + const shape = Shape.init(.{ .m = 3, .n = 5 }, .f32); + return Placement.init(&shape); + } + }; + var original = try Helper.create(); + original.byte_steps[0] = 60; + const copied = original; + original.local_dims.buffer[0] = 99; + original.byte_steps[0] = 100; + try std.testing.expectEqualSlices(i64, &.{ 3, 5 }, copied.local_dims.constSlice()); + try std.testing.expectEqualSlices(u64, &.{ 60, 0, 0, 0 }, &copied.byte_steps); + const source = Shape.init(.{ .m = 3, .n = 5 }, .f32); + try std.testing.expect(copied.localShape(source).eqlWithTags(source)); + var data: [3 * 5 * 4]u8 = undefined; + try std.testing.expectEqual(@intFromPtr(&data), @intFromPtr(copied.basePtr(.init(source, &data)))); +} + +test "Placement byte steps and dot product exceed 32 bits" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const runner: ShardingTest = .init(arena.allocator()); + const physical = try runner.physical(.{Platform.MAX_NUM_DEVICES}, .{ .mesh = .torus }); + const mesh_ = try Mesh.init("wide", &physical, .mesh(.{ .model = .balanced }), .parseBindings(.{ .model = .link_x })); + const shape = Shape.init(.{ .m = 2 * Platform.MAX_NUM_DEVICES, .n = 1 << 30 }, .f32).withPartitioning(&mesh_, .{ .m = .model }); + const placement = try Sharding.Placement.init(&shape); + try std.testing.expectEqual(@as(usize, 1) << 33, placement.byteOffset(1)); + const last_device = Platform.MAX_NUM_DEVICES - 1; + const coords: [MAX_MESH_RANK]u8 = mesh_.grid.logical_coords[last_device]; + try std.testing.expectEqual(@as(u8, @intCast(last_device)), coords[0]); + try std.testing.expectEqual(@as(usize, last_device) << 33, placement.byteOffset(last_device)); +} + +test "DeviceGrid rejects overlapping logical axes" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const runner: ShardingTest = .init(arena.allocator()); + const physical = try runner.physical(.{ 2, 2, 2 }, .{ .mesh = .torus }); + try std.testing.expectError(error.OverlappingLogicalAxes, Mesh.init("overlap", &physical, .mesh(.{ .model = .balanced, .data = .balanced }), .parseBindings(.{ .model = .{ .link_x, .link_y }, .data = .{ .link_y, .link_z } }))); +} + +test "Shardy meshes preserve logical order and unused physical replicas" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const allocator = arena.allocator(); + const runner: ShardingTest = .init(allocator); + const physical = try runner.physical(.{ 2, 2, 2 }, .{ .mesh = .torus }); + const mesh_ = try Mesh.init("logical", &physical, .mesh(.{ .batch = .balanced, .model = .balanced }), .parseBindings(.{ .batch = .link_z, .model = .link_x })); + const registry: *mlir.DialectRegistry = try .init(); + defer registry.deinit(); + shardy.registerDialect(registry); + const ctx: *mlir.Context = try .init(.{ .registry = registry, .threading = false }); + defer ctx.deinit(); + ctx.loadAllAvailableDialects(); + var text: std.Io.Writer.Allocating = .init(allocator); + defer text.deinit(); + try text.writer.print("{f}", .{mesh_.sdyMeshAttr(ctx)}); + try std.testing.expectEqualStrings( + "#sdy.mesh<[\"batch\"=2, \"model\"=2, \"__replica_0\"=2], device_ids=[0, 2, 4, 6, 1, 3, 5, 7]>", + text.written(), + ); +} + +test "Shardy replica axis avoids logical name collisions" { + const mesh_ = Mesh.initTest(.collision, .{ .__replica_0, .__replica_1 }); + try std.testing.expectEqualStrings("__replica_2", mesh_.sdyReplicaAxis()); +} + +test "GSPMD attributes use logical tile sizes and replica groups" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const allocator = arena.allocator(); + const ctx: *mlir.Context = try .init(.{ .threading = false }); + defer ctx.deinit(); + const runner: ShardingTest = .init(allocator); + const physical = try runner.physical(.{ 2, 2, 2 }, .{ .mesh = .torus }); + for (physical.devices_in_canonical_order, 0..) |*device, i| device.id = @intCast(7 - i); + const mesh_ = try Mesh.init("tiles", &physical, .mesh(.{ .batch = .balanced, .model = .balanced, .unused = .balanced }), .parseBindings(.{ .batch = .link_z, .model = .link_x })); + + const Case = struct { shape: Shape, expected: []const u8 }; + const cases = [_]Case{ + .{ + .shape = Shape.init(.{ .batch = 8, .model = 8 }, .f32).withPartitioning(&mesh_, .{ .batch = .batch, .model = .model }), + .expected = "{devices=[2,2,2]0,2,4,6,1,3,5,7 last_tile_dim_replicate}", + }, + .{ + .shape = Shape.init(.{ .model = 8, .batch = 8 }, .f32).withPartitioning(&mesh_, .{ .batch = .batch, .model = .model }), + .expected = "{devices=[2,2,2]0,2,1,3,4,6,5,7 last_tile_dim_replicate}", + }, + .{ + .shape = Shape.init(.{ .batch = 8, .model = 8 }, .f32).withPartitioning(&mesh_, .{ .batch = .batch }), + .expected = "{devices=[2,1,4]0,2,4,6,1,3,5,7 last_tile_dim_replicate}", + }, + .{ + .shape = Shape.init(.{ .batch = 8 }, .f32).withPartitioning(&mesh_, .{ .batch = .unused }), + .expected = "{replicated}", + }, + .{ + .shape = Shape.init(.{ .batch = 8 }, .f32), + .expected = "{replicated}", + }, + .{ + .shape = Shape.init(.{}, .f32), + .expected = "{replicated}", + }, + }; + for (cases) |case| { + const actual = try mesh_.gspmdShardingAttrForShape(allocator, ctx, case.shape); + try std.testing.expect(actual.eql(.string(ctx, case.expected))); + } +} + +test "GSPMD attributes preserve folded logical axis order" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const allocator = arena.allocator(); + const ctx: *mlir.Context = try .init(.{ .threading = false }); + defer ctx.deinit(); + const runner: ShardingTest = .init(allocator); + const physical = try runner.physical(.{ 2, 2, 2 }, .{ .mesh = .torus }); + var strategy: Strategy = .parseBindings(.{ .model = .link_x, .batch = .link_y }); + strategy.addFold(.link_x, &.{ .link_z, .link_x }); + const mesh_ = try Mesh.init("folded", &physical, .mesh(.{ .model = .balanced, .batch = .balanced }), strategy); + const sharded = Shape.init(.{ .model = 8, .batch = 8 }, .f32).withPartitioning(&mesh_, .{ .model = .model, .batch = .batch }); + const actual = try mesh_.gspmdShardingAttrForShape(allocator, ctx, sharded); + try std.testing.expect(actual.eql(.string(ctx, "{devices=[4,2]0,2,4,6,1,3,5,7}"))); + + const partial = Shape.init(.{ .model = 8 }, .f32).withPartitioning(&mesh_, .{ .model = .model }); + const partial_attr = try mesh_.gspmdShardingAttrForShape(allocator, ctx, partial); + try std.testing.expect(partial_attr.eql(.string(ctx, "{devices=[4,2]0,2,4,6,1,3,5,7 last_tile_dim_replicate}"))); +} + +test "GSPMD executes a reduction across a folded logical axis" { + if (comptime !Target.cpu.isEnabled()) return error.SkipZigTest; + + const zml = @import("zml.zig"); + const io = std.testing.io; + const allocator = std.testing.allocator; + + const platform = try Platform.init(allocator, io, .cpu, .{ + .cpu = .{ .device_count = 8 }, + .physical_mesh = .{ .custom = PhysicalMesh.torus3d }, + }); + defer platform.deinit(allocator, io); + var strategy: Strategy = .parseBindings(.{ .model = .link_x, .data = .link_y }); + strategy.addFold(.link_x, &.{ .link_z, .link_x }); + const mesh_ = try platform.registerMeshWithStrategy("logical", .mesh(.{ .data = .balanced, .model = .balanced }), strategy); + + const Functions = struct { + fn forward(input: zml.Tensor) zml.Tensor { + return input.sum(.m).squeeze(.m).withPartitioning(input.shape()._sharding.mesh.?, .{ .n = .data }); + } + }; + const input = zml.Tensor.init(.{ .m = 8, .n = 4 }, .i32).withPartitioning(mesh_, .{ .m = .model, .n = .data }); + const exe = try platform.compileFn(allocator, io, Functions.forward, .{input}, .{ + .meshes = &.{mesh_}, + .partitioner = .gspmd, + }); + defer exe.deinit(); + var values: [8][4]i32 = undefined; + var expected: [4]i32 = @splat(0); + for (&values, 0..) |*row, m| { + for (row, 0..) |*value, n| { + value.* = @intCast(10 * m + n); + expected[n] += value.*; + } + } + + var input_buffer = try zml.Buffer.fromBytes(io, platform, input.shape(), std.mem.asBytes(&values)); + defer input_buffer.deinit(); + var output = try exe.eval(allocator, io, .{input_buffer}); + defer output.deinit(); + try std.testing.expectEqual(expected, try output.getValue([4]i32, io)); +} diff --git a/zml/aio.zig b/zml/aio.zig deleted file mode 100644 index 3a5ca4dc..00000000 --- a/zml/aio.zig +++ /dev/null @@ -1,492 +0,0 @@ -const builtin = @import("builtin"); -const asynk = @import("async"); -const std = @import("std"); -const zml = @import("zml.zig"); -const c = @import("c"); -const posix = @import("posix.zig"); - -pub const gguf = @import("aio/gguf.zig"); -pub const nemo = @import("aio/nemo.zig"); -pub const safetensors = @import("aio/safetensors.zig"); -pub const sentencepiece = @import("aio/sentencepiece.zig"); -pub const tinyllama = @import("aio/tinyllama.zig"); -pub const torch = @import("aio/torch.zig"); -pub const yaml = @import("aio/yaml.zig"); - -pub const log = std.log.scoped(.zml_aio); -pub const Value = @import("aio/value.zig").Value; -const HostBuffer = @import("hostbuffer.zig").HostBuffer; - -test { - std.testing.refAllDecls(@This()); - std.testing.refAllDecls(gguf); - std.testing.refAllDecls(nemo); - std.testing.refAllDecls(safetensors); - std.testing.refAllDecls(sentencepiece); - std.testing.refAllDecls(tinyllama); - std.testing.refAllDecls(torch); - std.testing.refAllDecls(yaml); -} - -/// Detects the format of the model file (base on filename) and open it. -pub fn detectFormatAndOpen(allocator: std.mem.Allocator, model_path: []const u8) !BufferStore { - return if (std.mem.endsWith(u8, model_path, ".safetensors")) - try safetensors.open(allocator, model_path) - else if (std.mem.endsWith(u8, model_path, ".safetensors.index.json")) - try safetensors.open(allocator, model_path) - else if (std.mem.endsWith(u8, model_path, ".gguf")) - try gguf.open(allocator, model_path) - else if (std.mem.endsWith(u8, model_path, ".pt")) - try torch.open(allocator, model_path) - else if (std.mem.endsWith(u8, model_path, ".tinyllama")) - try tinyllama.open(allocator, model_path) - else { - std.debug.panic("File extension not recognized: {s}", .{model_path}); - }; -} - -pub fn detectFormatAndLoadTokenizer(allocator: std.mem.Allocator, tokenizer_path: []const u8) !zml.tokenizer.Tokenizer { - return if (std.mem.endsWith(u8, tokenizer_path, ".json")) - try zml.tokenizer.fromHfJson(allocator, tokenizer_path) - else if (std.mem.endsWith(u8, tokenizer_path, ".gguf")) { - const store = try gguf.open(allocator, tokenizer_path); - return gguf.getGgufTokenizer(store, allocator); - } else if (std.mem.endsWith(u8, tokenizer_path, ".pb") or std.mem.endsWith(u8, tokenizer_path, ".model")) - try sentencepiece.loadTokenizerFromPath(allocator, tokenizer_path) - else if (std.mem.endsWith(u8, tokenizer_path, ".tinyllama")) - try zml.aio.tinyllama.loadTokenizer(allocator, tokenizer_path, 32000) - else { - zml.log.err("Failed to recognized tokenizer format of: {s}", .{tokenizer_path}); - return error.FormatNotRecognized; - }; -} - -/// Creates a Model struct with tensor shapes read from the given BufferStore. -/// The result can be used to pass to `compileModel`. -/// -/// * The `Tensor` field `Model.a.b` will be populated with a `Tensor` -/// whose shape is read from the "a.b" tensor. -/// * If `Model` contains a list of layers, then the field: -/// `Model.layers[2].a.b` will be populated from the "layers.2.a.b" tensor. -pub fn populateModel(comptime Model: type, allocator: std.mem.Allocator, buffer_store: BufferStore) !Model { - return populateModelWithPrefix(Model, allocator, buffer_store, ""); -} - -/// Creates a Model struct with tensor shapes read from the given TensorStore, -/// using a given prefix. -/// The result can be used to pass to `compileWithModel`. -/// -/// * The `Tensor` field `Model.a.b` will be populated with a `Tensor` -/// whose shape is read from the "prefix.a.b" tensor. -/// * If `Model` contains a list of layers, then the field: -/// `Model.layers[2].a.b` will be populated from the "prefix.layers.2.a.b" tensor. -pub fn populateModelWithPrefix(comptime Model: type, allocator: std.mem.Allocator, store: BufferStore, prefix: []const u8) !Model { - var model: Model = undefined; - - var prefix_builder: PrefixBuilder = .{}; - try prefix_builder.push(allocator, prefix); - defer prefix_builder.deinit(allocator); - - var unique_id = zml.Tensor.reserveIdRange(@intCast(store.buffers.count())); - const ok = _populateStruct(allocator, &prefix_builder, &unique_id, store, &model, true) catch |err| { - std.debug.panic("Can't populate model of type {s}: {s}", .{ @typeName(type), @errorName(err) }); - }; - if (!ok) return error.TensorNotFound; - return model; -} - -/// A struct containing all the buffers and metadata found in a model file. -pub const BufferStore = struct { - pub const Buffers = std.StringArrayHashMapUnmanaged(HostBuffer); - pub const Metadata = std.StringArrayHashMapUnmanaged(Value); - - arena: std.heap.ArenaAllocator, - files: []MemoryMappedFile = &.{}, - buffers: Buffers = .{}, - _metadata: Metadata = .{}, - - pub fn deinit(self: BufferStore) void { - for (self.files) |*file| file.deinit(); - self.arena.deinit(); - } - - pub fn get(self: BufferStore, key: []const u8) ?HostBuffer { - return self.buffers.get(key); - } - - /// Count layers starting with the given prefix. - pub fn countLayers(self: BufferStore, prefix: []const u8) usize { - // Note: This is kinda inefficient - const digit_start_index = prefix.len + 1; - var it = self.buffers.iterator(); - var maybe_max_index: ?usize = null; - while (it.next()) |entry| { - if (!std.mem.startsWith(u8, entry.key_ptr.*, prefix)) continue; - - const next_dot_index = std.mem.indexOfScalarPos(u8, entry.key_ptr.*, digit_start_index, '.') orelse entry.key_ptr.len; - const index = std.fmt.parseInt(usize, entry.key_ptr.*[digit_start_index..next_dot_index], 10) catch continue; - if (maybe_max_index) |*max_index| { - max_index.* = @max(max_index.*, index); - } else { - maybe_max_index = index; - } - } - - return if (maybe_max_index) |index| index + 1 else 0; - } - - pub fn metadata(self: BufferStore, key: []const u8, comptime tag: std.meta.FieldEnum(Value)) ?std.meta.FieldType(Value, tag) { - const wrapped_value = self._metadata.get(key) orelse return null; - - if (wrapped_value != tag) { - zml.log.err("Tried to interpret metadata '{s}' as {}, but was of type {}", .{ key, tag, wrapped_value }); - @panic("invalid metadata type"); - } - return @field(wrapped_value, @tagName(tag)); - } - - pub fn metadataSlice(self: BufferStore, key: []const u8, comptime tag: Value.Slice.ItemType) ?[]const Value.Slice.toZigType(tag) { - const wrapped_value = self._metadata.get(key) orelse return null; - - if (wrapped_value != .array or wrapped_value.array.item_type != tag) { - return null; - } - const T = Value.Slice.toZigType(tag); - return @alignCast(std.mem.bytesAsSlice(T, wrapped_value.array.data)); - } -}; - -/// A file containing contiguous/non-contiguous buffers, that can be read with mmap -/// (assumes contiguous if `strides` is `null`). -/// This struct is meant to be wrapped into a format specific struct, like io.gguf.File. -pub const MemoryMappedFile = struct { - /// underlying file handle - file: asynk.File, - data: []align(std.mem.page_size) const u8, - data_offset: u64 = 0, - - pub fn init(file: asynk.File) !MemoryMappedFile { - const data_len: usize = (try file.stat()).size; - const data_ = try asynk.call(std.posix.mmap, .{ - null, - data_len, - std.posix.PROT.READ, - .{ .TYPE = .PRIVATE }, - file.inner.file.fd, - 0, - }); - - try asynk.call(posix.madvise, .{ data_.ptr, @intCast(data_.len), @intCast(c.MADV_SEQUENTIAL) }); - - return .{ - .file = file, - .data = data_, - }; - } - - pub fn mappedSlice(self: *MemoryMappedFile, start: usize, len: usize) []const u8 { - return self.data[self.data_offset + start ..][0..len]; - } - - pub fn deinit(self: *MemoryMappedFile) void { - std.posix.munmap(self.data); - self.file.close() catch @panic("failed to close file"); - self.* = undefined; - } -}; - -/// Helper handling prefix building. -/// -/// This allows to easily push/pop prefixes and handles the generation of the string with the correct format. -const PrefixBuilder = struct { - /// Stores the computed prefix. - data: std.ArrayListUnmanaged(u8) = .{}, - /// Stack storing the size of the intermediary prefix. - subprefixes: std.ArrayListUnmanaged(u32) = .{}, - - pub fn deinit(self: *PrefixBuilder, allocator: std.mem.Allocator) void { - self.data.deinit(allocator); - self.subprefixes.deinit(allocator); - } - - pub fn push(self: *PrefixBuilder, allocator: std.mem.Allocator, prefix: []const u8) !void { - const old_len: u32 = @intCast(self.data.items.len); - try self.subprefixes.append(allocator, old_len); - errdefer _ = self.subprefixes.pop(); - - if (old_len == 0) { - try self.data.appendSlice(allocator, prefix); - } else { - try self.data.ensureUnusedCapacity(allocator, prefix.len + 1); - self.data.appendAssumeCapacity('.'); - self.data.appendSliceAssumeCapacity(prefix); - } - } - - pub fn pushDigit(self: *PrefixBuilder, allocator: std.mem.Allocator, idx: usize) !void { - const old_len: u32 = @intCast(self.data.items.len); - try self.subprefixes.append(allocator, old_len); - errdefer _ = self.subprefixes.pop(); - - try self.data.ensureUnusedCapacity(allocator, 16); - if (old_len > 0) { - self.data.appendAssumeCapacity('.'); - } - try self.data.writer(allocator).print("{d}", .{idx}); - } - - pub fn pop(self: *PrefixBuilder) void { - const last_prefix_len = self.subprefixes.popOrNull() orelse unreachable; - self.data.shrinkRetainingCapacity(last_prefix_len); - } -}; - -fn _populateStruct( - allocator: std.mem.Allocator, - prefix_builder: *PrefixBuilder, - unique_id: *u64, - buffer_store: BufferStore, - obj: anytype, - required: bool, -) !bool { - const err_msg = "_populateStruct must be called with a pointer to type. Received "; - const type_info, const T = switch (@typeInfo(@TypeOf(obj))) { - .Pointer => |ptr_info| switch (ptr_info.size) { - .One => .{ @typeInfo(ptr_info.child), ptr_info.child }, - else => @compileError(err_msg ++ @typeName(@TypeOf(obj))), - }, - else => @compileError(err_msg ++ @typeName(@TypeOf(obj))), - }; - - const prefix = prefix_builder.data.items; - if (T == zml.Tensor) { - return if (buffer_store.get(prefix)) |buffer| { - obj.* = zml.Tensor{ - ._shape = buffer.shape(), - ._id = .{ .buffer_id = unique_id.* }, - ._donation = .input_buffer, - }; - unique_id.* += 1; - return true; - } else { - if (required) { - std.log.err("Tensor not found: {s} ({d})", .{ prefix, buffer_store.buffers.count() }); - } - return false; - }; - } - - switch (type_info) { - .Pointer => |ptr_info| { - if (ptr_info.size == .Slice) { - obj.* = &.{}; - - const len = buffer_store.countLayers(prefix); - if (len > 0) { - obj.* = try allocator.alloc(ptr_info.child, len); - - for (obj.*, 0..) |*value, i| { - try prefix_builder.pushDigit(allocator, i); - defer prefix_builder.pop(); - - const found = try _populateStruct(allocator, prefix_builder, unique_id, buffer_store, value, required); - if (!found) { - std.log.err("Not able to load {s} as {s}", .{ prefix, @typeName(ptr_info.child) }); - return false; - } - } - } else if (required) { - log.warn("No layer found at {s}", .{prefix}); - } - return true; - } else { - std.log.err("{s} - {s}: {s} type not supported", .{ @src().fn_name, prefix, @typeName(T) }); - return false; - } - }, - .Struct => |struct_info| { - var partial_struct = false; - inline for (struct_info.fields) |field| { - try prefix_builder.push(allocator, field.name); - defer prefix_builder.pop(); - - var has_default = false; - if (field.default_value) |_| has_default = true; - - const field_found = try _populateStruct(allocator, prefix_builder, unique_id, buffer_store, &@field(obj, field.name), required and !has_default); - partial_struct = partial_struct or field_found; - if (!field_found) { - if (field.default_value) |v| { - @field(obj, field.name) = @as(*const field.type, @alignCast(@ptrCast(v))).*; - } else { - if (partial_struct) { - log.warn("Incomplete metadata '{0s}': {1s}. Missing field: '{2s}'. '{0s}' will be ignored.", .{ prefix, @typeName(T), field.name }); - obj.* = undefined; - return false; - } - - return false; - } - } - } - return true; - }, - .Optional => |opt_info| { - obj.* = @as(opt_info.child, undefined); - const found = try _populateStruct(allocator, prefix_builder, unique_id, buffer_store, &(obj.*.?), false); - if (!found) obj.* = null; - return true; - }, - .Int => { - obj.* = undefined; - return true; - }, - .Float => { - obj.* = undefined; - return true; - }, - else => if (required) { - std.log.err("{s}: {s} type not supported", .{ prefix, @typeName(T) }); - return error.UnsupportedMetadataType; - } else return false, - } -} - -/// Creates a bufferized version of a Model from the given BufferStore. For details about -/// bufferization, see the documentation of Bufferized(T). -/// -/// This will represent the weights of the model, loaded on a specific platform. -/// It can be used with a `module.Exe` (a compiled version of the same Model), to make a -/// `module.ExeWithWeights` ready to be called. -/// -/// The `init_args` are used to initialize the non Buffer fields, using `Model.init` function. -pub fn loadBuffers( - comptime Model: type, - init_args: anytype, - buffer_store: BufferStore, - allocator: std.mem.Allocator, - platform: zml.Platform, -) !zml.Bufferized(Model) { - var arena_state = std.heap.ArenaAllocator.init(allocator); - defer arena_state.deinit(); - const arena = arena_state.allocator(); - var model: Model = try zml.aio.populateModel(Model, arena, buffer_store); - - // If the Model has a "init" function, call it with the given parameters. - if (@hasDecl(Model, "init")) { - @call(.auto, Model.init, .{&model} ++ init_args); - } else { - zml.meta.assertComptime(@TypeOf(init_args) == void or @TypeOf(init_args) == @TypeOf(.{}), "Model of type {} has no init function, so `loadBuffers` should be call with init_args set to {{}} (void)", .{Model}); - } - - return loadModelBuffers(Model, model, buffer_store, allocator, platform); -} - -/// Creates a bufferized version of a Model from the given BufferStore. For details about -/// bufferization, see the documentation of Bufferized(T). -/// -/// This will represent the weights of the model, loaded on a specific platform. -/// It can be used with a `module.Exe` (a compiled version of the same Model), to make a -/// `module.ExeWithWeights` ready to be called. -pub fn loadModelBuffers( - comptime Model: type, - model: Model, - buffer_store: BufferStore, - allocator: std.mem.Allocator, - platform: zml.Platform, -) !zml.Bufferized(Model) { - return try loadModelBuffersWithPrefix(Model, model, buffer_store, allocator, platform, ""); -} -/// Creates a bufferized version of a Model from the given BufferStore and the given prefix. -/// For details about bufferization, see the documentation of Bufferized(T). -/// -/// This will represent the weights of the model, loaded on a specific platform. -/// It can be used with a `module.Exe` (a compiled version of the same Model), to make a -/// `module.ExeWithWeights` ready to be called. -pub fn loadModelBuffersWithPrefix( - comptime Model: type, - model: Model, - buffer_store: BufferStore, - allocator: std.mem.Allocator, - platform: zml.Platform, - prefix: []const u8, -) !zml.Bufferized(Model) { - // Allocate the bufferized version. - // We copy the shape, and let visitStructAndLoadBuffer write the other fields. - // to write them just afterward. - var res: zml.Bufferized(Model) = undefined; - try zml.meta.mapAlloc(struct { - pub fn initBuffer(_: void, tensor: zml.Tensor) zml.Buffer { - return .{ ._shape = tensor.shape(), ._api = undefined, ._shards = undefined }; - } - }.initBuffer, allocator, {}, model, &res); - - var prefix_builder: PrefixBuilder = .{}; - try prefix_builder.push(allocator, prefix); - defer prefix_builder.deinit(allocator); - - try visitStructAndLoadBuffer(allocator, &prefix_builder, buffer_store, &res, platform); - return res; -} - -/// Takes a bufferized version of a `model`, ie a mirror struct of the `model`, and deinit all the -/// Buffer found. -pub fn unloadBuffers(model: anytype) void { - zml.meta.visit((struct { - fn cb(_: void, buffer: *zml.Buffer) void { - buffer.deinit(); - } - }).cb, {}, model); -} - -fn visitStructAndLoadBuffer(allocator: std.mem.Allocator, prefix_builder: *PrefixBuilder, buffer_store: BufferStore, obj: anytype, platform: zml.Platform) !void { - const err_msg = "visitStructAndLoadBuffer must be called with a pointer to type. Received "; - const type_info, const T = switch (@typeInfo(@TypeOf(obj))) { - .Pointer => |ptr_info| switch (ptr_info.size) { - .One => .{ @typeInfo(ptr_info.child), ptr_info.child }, - else => @compileError(err_msg ++ @typeName(@TypeOf(obj))), - }, - else => @compileError(err_msg ++ @typeName(@TypeOf(obj))), - }; - - const prefix = prefix_builder.data.items; - if (T == zml.Buffer) { - return if (buffer_store.get(prefix)) |host_buffer| { - // obj._shape has been set inside `loadModelBuffersWithPrefix`, before calling us. - var buf_with_metadata = host_buffer; - log.warn("loading {s} ({})", .{ prefix, obj._shape }); - zml.meta.assert(host_buffer.shape().eql(obj._shape), "loadModelBuffers expects to find the same shapes in the model and in the buffer store, got {} and {} for tensor {s}", .{ obj._shape, host_buffer, prefix }); - buf_with_metadata._shape = obj._shape; - obj.* = try zml.Buffer.from(platform, buf_with_metadata); - } else { - return error.BufferNotFound; - }; - } - - switch (type_info) { - .Pointer => |ptr_info| { - if (ptr_info.size == .Slice) { - for (obj.*, 0..) |*value, i| { - try prefix_builder.pushDigit(allocator, i); - defer prefix_builder.pop(); - - try visitStructAndLoadBuffer(allocator, prefix_builder, buffer_store, value, platform); - } - } else return error.TypeNotSupported; - }, - .Struct => |struct_info| { - inline for (struct_info.fields) |field| { - try prefix_builder.push(allocator, field.name); - defer prefix_builder.pop(); - - try visitStructAndLoadBuffer(allocator, prefix_builder, buffer_store, &@field(obj, field.name), platform); - } - }, - .Optional => { - if (obj.*) |*obj_val| { - try visitStructAndLoadBuffer(allocator, prefix_builder, buffer_store, obj_val, platform); - } - }, - else => {}, - } -} diff --git a/zml/aio/gguf.zig b/zml/aio/gguf.zig deleted file mode 100644 index 05a6c876..00000000 --- a/zml/aio/gguf.zig +++ /dev/null @@ -1,141 +0,0 @@ -const asynk = @import("async"); -const core = @import("gguf/core.zig"); -const std = @import("std"); -const zml = @import("../zml.zig"); - -const HostBuffer = @import("../hostbuffer.zig").HostBuffer; - -const Allocator = std.mem.Allocator; -const assert = std.debug.assert; - -const log = std.log.scoped(.zml_io); - -pub fn open(allocator: Allocator, path: []const u8) !zml.aio.BufferStore { - var file = try core.GgufFile.open(path); - errdefer file.close(); - - var res: zml.aio.BufferStore = .{ - .arena = std.heap.ArenaAllocator.init(allocator), - }; - errdefer res.arena.deinit(); - const arena = res.arena.allocator(); - - res.files = try arena.dupe(zml.aio.MemoryMappedFile, &.{file.file}); - - // metadata must be read in order to read tensors - try loadMetadata(arena, &res, &file); - try loadBuffers(arena, &res, &file); - if (res.buffers.count() != file.header.tensor_count) { - log.warn("Expected to find {d} tensors in {s}, only found {d}", .{ file.header.tensor_count, path, res.buffers.count() }); - } - return res; -} - -pub fn getGgufTokenizer(self: zml.aio.BufferStore, allocator: std.mem.Allocator) !zml.tokenizer.Tokenizer { - const tokens = self.metadataSlice("tokenizer.ggml.tokens", .string) orelse { - log.err("GGUF File: Tokens not found", .{}); - return error.TokensNotFound; - }; - const scores = self.metadataSlice("tokenizer.ggml.scores", .float32) orelse { - log.err("GGUF File: Scores not found", .{}); - return error.ScoresNotFound; - }; - assert(tokens.len == scores.len); - const tokenizer_type = self.metadata("tokenizer.ggml.model", .string) orelse "llama"; - const tokenizer_impl: zml.tokenizer.KnownImplementation = if (std.mem.eql(u8, tokenizer_type, "gpt2")) .gpt2 else .sentencepiece; - const bos = self.metadata("tokenizer.ggml.bos_token_id", .uint32); - const eos = self.metadata("tokenizer.ggml.eos_token_id", .uint32); - const unk = self.metadata("tokenizer.ggml.unknown_token_id", .uint32); - const pad = self.metadata("tokenizer.ggml.padding_token_id", .uint32); - - const NOT_FOUND = std.math.maxInt(u32); - const special_tokens: zml.tokenizer.Tokenizer.SpecialTokens = .{ - .bos = bos.?, - .eos = eos.?, - .unk = unk orelse NOT_FOUND, - .pad = pad orelse NOT_FOUND, - }; - - const gguf_normalizer = if (tokenizer_impl == .gpt2) - zml.tokenizer.Normalizer.wellKnown(.gpt2) - else - zml.tokenizer.Normalizer.wellKnown(.sentencepiece); - - const extra_tokens: u8 = if (tokenizer_impl == .gpt2) 1 else 0; - const n_tokens: u32 = @intCast(tokens.len + extra_tokens); - - var tokenizer = try zml.tokenizer.Tokenizer.init( - allocator, - n_tokens, - 32, - gguf_normalizer, - special_tokens, - true, - ); - - var gpt2_unicode = if (tokenizer_impl == .gpt2) - try zml.tokenizer.Gpt2TextDecoder.init(allocator) - else - null; - defer if (gpt2_unicode) |*gpt2| gpt2.deinit(); - var decoded = std.ArrayList(u8).init(allocator); - defer decoded.deinit(); - - // copy the tokens to the tokenizer arena. - for (tokens, 0..tokens.len) |t, i| { - if (tokenizer_impl == .gpt2) { - decoded.clearRetainingCapacity(); - try tokenizer.addToken(scores[i], try gpt2_unicode.?.decode(&decoded, t)); - // log.debug("token: {s} -> {s}", .{t, decoded.items}); - } else { - try tokenizer.addToken(scores[i], t); - } - } - - // Gpt2 tokenizer always splits on spaces. - if (tokenizer_impl == .gpt2) { - tokenizer.special_tokens.hard_space = tokenizer.next_token_id; - tokenizer.addOwnedToken(0, " "); - } - - return tokenizer; -} - -fn loadMetadata(allocator: Allocator, store: *zml.aio.BufferStore, file: *core.GgufFile) !void { - try store._metadata.ensureTotalCapacity(allocator, @intCast(file.header.metadata_kv_count)); - - while (file.readMetadata(allocator)) |entry| { - log.info("Loading MetaData: {s}", .{entry.name}); - const res = store._metadata.getOrPutAssumeCapacity(entry.name); - if (res.found_existing) { - // This file seems invalid. Since most metadatas aren't required, continue ahead. - log.warn("Found duplicated metadata key: {s}", .{entry.name}); - continue; - } - res.value_ptr.* = entry.val.asLoaderValue(); - } else |err| switch (err) { - error.EndOfMetadata => {}, - else => return err, - } -} - -fn loadBuffers(allocator: Allocator, store: *zml.aio.BufferStore, file: *core.GgufFile) !void { - try store.buffers.ensureTotalCapacity(allocator, @intCast(file.header.tensor_count)); - while (file.readTensorInfo(allocator)) |info| { - const res = store.buffers.getOrPutAssumeCapacity(info.name); - if (res.found_existing) { - // This file seems invalid. Try to continue anyway. - log.warn("Found duplicated tensor: {s}", .{info.name}); - continue; - } - - // TODO: handle quantized types - const dtype: zml.DataType = info.t.toDtype() orelse return error.UnsupportedGgufType; - const buffer = HostBuffer.fromBytes(zml.Shape.init(info.shape(), dtype), file.file.mappedSlice(info.start, info.byte_len)); - res.value_ptr.* = buffer; - // store the info index. - } else |err| switch (err) { - error.EndOfMetadata => {}, - else => return err, - } -} diff --git a/zml/aio/gguf/core.zig b/zml/aio/gguf/core.zig deleted file mode 100644 index 61b51370..00000000 --- a/zml/aio/gguf/core.zig +++ /dev/null @@ -1,530 +0,0 @@ -const asynk = @import("async"); -const std = @import("std"); -const utils = @import("../utils.zig"); -const zml = @import("../../zml.zig"); - -const assert = std.debug.assert; -const log = std.log.scoped(.zml_io); - -pub const GgufErrors = error{ - ValueTypeMismatch, - InvalidGguf, - UnsupportedGgufType, - EndOfMetadata, - OutOfMemory, -}; - -// Enums and structures -pub const TensorType = enum(u32) { - f32 = 0, - f16 = 1, - q4_0 = 2, - q4_1 = 3, - deprecated_q4_2 = 4, - deprecated_q4_3 = 5, - q5_0 = 6, - q5_1 = 7, - q8_0 = 8, - q8_1 = 9, - // k-quantizations - q2_k = 10, - q3_k = 11, - q4_k = 12, - q5_k = 13, - q6_k = 14, - q8_k = 15, - i8 = 16, - i16 = 17, - i32 = 18, - - const MAX_KNOWN_ENUM = 18; - - pub fn canConvertQuant(self: TensorType) bool { - return switch (self) { - .q8_0, .q4_k, .q6_k, .q2_k, .q4_0, .q4_1 => true, - else => false, - }; - } - - pub fn toDtype(self: TensorType) ?zml.DataType { - return switch (self) { - .f32 => .f32, - .f16 => .f16, - .i8 => .i8, - .i16 => .i16, - .i32 => .i32, - else => null, - }; - } - - pub fn sizeOf(self: TensorType) usize { - return self.toDtype().?.sizeOf(); - } - - /// Return the tensor type features - pub fn getFeatures(t: TensorType) TensorTypeFeatures { - return switch (t) { - inline else => |val| @field(TENSOR_TYPE_FEATURES, @tagName(val)), - }; - } -}; - -/// GGUF tensor type to features lookup table. -pub const TensorTypeFeatures = struct { - items_per_block: u29, - bytes_per_block: u29, - - pub fn alignment(features: TensorTypeFeatures) u8 { - return std.math.log2_int(u29, features.bytes_per_block); - } -}; - -pub const TENSOR_TYPE_FEATURES: std.enums.EnumFieldStruct(TensorType, TensorTypeFeatures, null) = .{ - .f32 = .{ .items_per_block = 1, .bytes_per_block = @sizeOf(f32) }, - .f16 = .{ .items_per_block = 1, .bytes_per_block = @sizeOf(f16) }, - .q4_0 = .{ .items_per_block = 32, .bytes_per_block = 18 }, - .q4_1 = .{ .items_per_block = 32, .bytes_per_block = 20 }, - .deprecated_q4_2 = .{ .items_per_block = 0, .bytes_per_block = 0 }, - .deprecated_q4_3 = .{ .items_per_block = 0, .bytes_per_block = 0 }, - .q5_0 = .{ .items_per_block = 32, .bytes_per_block = 22 }, - .q5_1 = .{ .items_per_block = 32, .bytes_per_block = 24 }, - .q8_0 = .{ .items_per_block = 32, .bytes_per_block = 34 }, - .q8_1 = .{ .items_per_block = 32, .bytes_per_block = 40 }, - .q2_k = .{ .items_per_block = 256, .bytes_per_block = 82 }, - .q3_k = .{ .items_per_block = 256, .bytes_per_block = 110 }, - .q4_k = .{ .items_per_block = 256, .bytes_per_block = 144 }, - .q5_k = .{ .items_per_block = 256, .bytes_per_block = 176 }, - .q6_k = .{ .items_per_block = 256, .bytes_per_block = 210 }, - .q8_k = .{ .items_per_block = 256, .bytes_per_block = 292 }, - .i8 = .{ .items_per_block = 1, .bytes_per_block = @sizeOf(i8) }, - .i16 = .{ .items_per_block = 1, .bytes_per_block = @sizeOf(i16) }, - .i32 = .{ .items_per_block = 1, .bytes_per_block = @sizeOf(i32) }, -}; - -pub const GgufValueType = enum(u32) { - // The value is a 8-bit unsigned integer. - uint8 = 0, - // The value is a 8-bit signed integer. - int8 = 1, - // The value is a 16-bit unsigned little-endian integer. - uint16 = 2, - // The value is a 16-bit signed little-endian integer. - int16 = 3, - // The value is a 32-bit unsigned little-endian integer. - uint32 = 4, - // The value is a 32-bit signed little-endian integer. - int32 = 5, - // The value is a 32-bit IEEE754 floating point number. - float32 = 6, - // The value is a boolean. - // 1-byte value where 0 is false and 1 is true. - // Anything else is invalid, and should be treated as either the model - // being invalid or the reader being buggy. - bool = 7, - // The value is a UTF-8 non-null-terminated string, with length prepended. - string = 8, - // The value is an array of other values, with the length and type - // prepended. Arrays can be nested, and the length of the array is the - // number of elements in the array, not the number of bytes. - array = 9, - // The value is a 64-bit unsigned little-endian integer. - uint64 = 10, - // The value is a 64-bit signed little-endian integer. - int64 = 11, - // The value is a 64-bit IEEE754 floating point number. - float64 = 12, - // Special values used by the callbacks of gguf_do_with_value(). - array_start = 100, - array_end = 101, - - // Allow other values in case GGUF add more types without us noticing - _, - - pub fn sizeOf(self: GgufValueType) usize { - return switch (self) { - .uint8 => @sizeOf(u8), - .int8 => @sizeOf(i8), - .uint16 => @sizeOf(u16), - .int16 => @sizeOf(i16), - .uint32 => @sizeOf(u32), - .int32 => @sizeOf(i32), - .float32 => @sizeOf(f32), - .bool => @sizeOf(bool), - .uint64 => @sizeOf(u64), - .int64 => @sizeOf(i64), - .float64 => @sizeOf(f64), - .string => @sizeOf([]u8), - else => unreachable, - }; - } - - pub fn arrayTypeCheck(self: GgufValueType, comptime T: type) !void { - switch (self) { - .string => if (T != []u8 and T != []const u8) return error.ValueTypeMismatch, - .uint8 => if (T != u8) return error.ValueTypeMismatch, - .int8 => if (T != i8) return error.ValueTypeMismatch, - .uint16 => if (T != u16) return error.ValueTypeMismatch, - .int16 => if (T != i16) return error.ValueTypeMismatch, - .uint32 => if (T != u32) return error.ValueTypeMismatch, - .int32 => if (T != i32) return error.ValueTypeMismatch, - .float32 => if (T != f32) return error.ValueTypeMismatch, - .bool => if (T != bool) return error.ValueTypeMismatch, - .uint64 => if (T != u64) return error.ValueTypeMismatch, - .int64 => if (T != i64) return error.ValueTypeMismatch, - .float64 => if (T != f64) return error.ValueTypeMismatch, - else => {}, - } - } -}; - -pub const ValueType = enum { - uint8, - int8, - uint16, - int16, - uint32, - int32, - float32, - uint64, - int64, - float64, - boolval, - string, - array, -}; - -// Union of possible values. -pub const GgufValue = union(ValueType) { - uint8: u8, - int8: i8, - uint16: u16, - int16: i16, - uint32: u32, - int32: i32, - float32: f32, - uint64: u64, - int64: i64, - float64: f64, - boolval: bool, - string: []const u8, - array: Array, - - pub const Array = struct { - // Any value type is valid, including arrays. - child: GgufValueType, - // Number of elements, not bytes - len: usize, - data: []u8, - }; - - pub fn asLoaderValue(self: GgufValue) zml.aio.Value { - return switch (self) { - .array => |v| .{ - .array = .{ - .item_type = switch (v.child) { - .bool => .boolval, - .uint8 => .uint8, - .int8 => .int8, - .uint16 => .uint16, - .int16 => .int16, - .uint32 => .uint32, - .int32 => .int32, - .float32 => .float32, - .uint64 => .uint64, - .int64 => .int64, - .float64 => .float64, - .string => .string, - // TODO: .array => .array, - else => unreachable, - }, - .data = v.data, - }, - }, - inline else => |v, tag| @unionInit(zml.aio.Value, @tagName(tag), v), - }; - } -}; - -// Header -const GgufHeader = extern struct { - // Magic number to announce that this is a GGUF file. Must be `GUFF`. - magic: [4]u8, - // The version of the format implemented. - // Must be `3` for version described in this spec. - version: u32, - // The number of tensors in the file. - // This is explicit, instead of being included in the metadata, to ensure - // it is always present for loading the tensors. - tensor_count: usize, - // The number of metadata key-value pairs. - metadata_kv_count: usize, - - pub fn validate(self: GgufHeader) !void { - if (!std.mem.eql(u8, &self.magic, "GGUF")) { - log.err("Invalid GGUF file: wrong header {s}", .{self.magic}); - return error.InvalidGguf; - } - } -}; - -// Key representation in this library API. -pub const GgufMetadataKv = struct { - name: []const u8, - type_: GgufValueType, - val: GgufValue, -}; - -// Tensor representation in this library API. -const GGUF_TENSOR_MAX_DIM: usize = 8; // Future-proof: actual limit is 4. -pub const GgufTensorInfo = struct { - name: []const u8, - t: TensorType, // Tensor type (enum TensorType). - rank: usize, // Number of dimensions of the tensor. - dims: [GGUF_TENSOR_MAX_DIM]i64, // Dimensions (Eg. [512, 1024, 1, 1]). - start: usize, // Offset from start of data section. - byte_len: usize, // Total size in bytes. - num_weights: usize, // Total number of parameters. - - pub inline fn shape(info: GgufTensorInfo) []const i64 { - return info.dims[0..info.rank]; - } -}; - -// Return the value type name given the type ID. -fn getValueTypeName(t: u32) []const u8 { - if (@as(usize, @intCast(t)) >= GGUF_VALUE_NAME.len) return "unknown"; - return GGUF_VALUE_NAME[@intCast(t)]; -} - -const GGUF_VALUE_NAME = [_][]const u8{ - "uint8", "int8", "uint16", "int16", "uint32", "int32", - "float32", "bool", "string", "array", "uint64", "int64", - "float64", -}; - -/// GGUF file API -/// A memory-mapped view of a .gguf file. -/// Format used by GGML models: https://github.com/ggerganov/ggml/ -pub const GgufFile = struct { - header: GgufHeader, // GUFF file header info. - size: usize, // Total file size. - file: zml.aio.MemoryMappedFile, - left_kv: usize, // Number of key-value pairs yet to read. - left_tensors: usize, // Number of tensors yet to read. - off: usize, // Offset of the next item to parse. - alignment: usize = 32, // File data alignment. Default: 32 bytes. - - /// Open and memmap the given file. - pub fn open(path: []const u8) !GgufFile { - const file = try asynk.File.open(path, .{}); - const header = try file.reader().readStruct(GgufHeader); - try header.validate(); - return .{ - .header = header, - .size = (try file.stat()).size, - .file = try zml.aio.MemoryMappedFile.init(file), - .off = @sizeOf(GgufHeader), - .left_kv = header.metadata_kv_count, - .left_tensors = header.tensor_count, - }; - } - - /// Unmap the file memory and close the file handle. - pub fn close(self: *GgufFile) void { - self.file.deinit(); - } - - /// Set the context to read the first key-value entry in the GGUF - /// file and then all the rest. Is used when creating a new context - /// and also when you want to restart scanning the key-value - /// items in the file. - fn rewind(ctx: *GgufFile) void { - ctx.off = @sizeOf(GgufHeader); - ctx.left_kv = ctx.header.metadata_kv_count; - ctx.left_tensors = ctx.header.tensor_count; - } - - pub fn seek(self: *GgufFile, pos: usize) void { - assert(pos < self.size); - self.off = pos; - } - - fn readInt(self: *GgufFile, comptime T: type) !T { - if (self.off + @sizeOf(T) >= self.size) return error.InvalidGguf; - const res = self.file.file.reader().readInt(T, .little); - self.off += @sizeOf(T); - return res; - } - - fn readTensorType(self: *GgufFile) !TensorType { - const raw = try self.readInt(u32); - if (raw > TensorType.MAX_KNOWN_ENUM) { - log.err("Unsupported GGUF tensor type: {d}", .{raw}); - return error.UnsupportedGgufType; - } - return @enumFromInt(raw); - } - - fn readValueType(self: *GgufFile) !GgufValueType { - const raw = try self.readInt(u32); - const t: GgufValueType = @enumFromInt(raw); - switch (t) { - .uint8, .int8, .uint16, .int16, .uint32, .int32, .float32, .bool, .string, .array, .uint64, .int64, .float64, .array_start, .array_end => {}, - else => { - log.err("Unsupported GGUF value type: {s}", .{@tagName(t)}); - return error.UnsupportedGgufType; - }, - } - return t; - } - - pub fn readAlloc(self: *GgufFile, allocator: std.mem.Allocator, len: usize) ![]u8 { - const data = try allocator.alloc(u8, len); - const read = try self.file.file.reader().readAll(data); - if (read != data.len) return error.InvalidGguf; - self.off += len; - return data; - } - - pub fn skipBytes(self: *GgufFile, len: usize) !void { - try self.file.file.seekBy(@intCast(len)); - self.off += len; - } - - /// Read the len then the actual bytes. - pub fn readString(self: *GgufFile, allocator: std.mem.Allocator) ![]u8 { - const len: usize = try self.readInt(u64); - return self.readAlloc(allocator, len); - } - - pub fn skipString(self: *GgufFile) !void { - const len: usize = try self.readInt(u64); - return self.skipBytes(len); - } - - fn readArrayHeader(self: *GgufFile, allocator: std.mem.Allocator) !GgufValue.Array { - const child = try self.readValueType(); - const len: usize = try self.readInt(u64); - const data = switch (child) { - // Since strings have variable lenghts, we need to read them one by one - .string => str: { - var data = try allocator.alloc([]u8, len); - for (0..len) |i| data[i] = try self.readString(allocator); - break :str std.mem.sliceAsBytes(data); - }, - else => try self.readAlloc(allocator, len * child.sizeOf()), - }; - return .{ - .child = child, - .len = len, - .data = data, - }; - } - - fn readTypedValue(self: *GgufFile, allocator: std.mem.Allocator, t: GgufValueType) !GgufValue { - return switch (t) { - .uint8 => .{ .uint8 = try self.readInt(u8) }, - .int8 => .{ .int8 = try self.readInt(i8) }, - .uint16 => .{ .uint16 = try self.readInt(u16) }, - .int16 => .{ .int16 = try self.readInt(i16) }, - .uint32 => .{ .uint32 = try self.readInt(u32) }, - .int32 => .{ .int32 = try self.readInt(i32) }, - .float32 => .{ .float32 = @bitCast(try self.readInt(u32)) }, - .bool => .{ .boolval = try self.readInt(u8) != 0 }, - .string => .{ .string = try self.readString(allocator) }, - .array => .{ .array = try self.readArrayHeader(allocator) }, - .uint64 => .{ .uint64 = try self.readInt(u64) }, - .int64 => .{ .int64 = try self.readInt(i64) }, - .float64 => .{ .float64 = @bitCast(try self.readInt(u64)) }, - else => error.UnsupportedGgufType, - }; - } - - /// Parses the next metadata entry. - /// Returns error.EndOfMetadata if there are no longer metadata to process in this GGUF file. - pub fn readMetadata(self: *GgufFile, allocator: std.mem.Allocator) !GgufMetadataKv { - if (self.left_kv == 0) return error.EndOfMetadata; - self.left_kv -= 1; - const name = try self.readString(allocator); - const type_ = try self.readValueType(); - const val: GgufValue = try self.readTypedValue(allocator, type_); - return .{ .name = name, .type_ = type_, .val = val }; - } - - // Set the data section offset. This function must be called exactly when - // all the key-values are consumed, in the context of the first call of - // ctx.getTensor(): this way we will be able to return tensor offsets - // as absolute positions and pointers to the mmapped file. - fn setDataOffset(self: *GgufFile) !void { - const base_off = self.off; - - assert(self.left_kv == 0 and self.left_tensors == self.header.tensor_count); - - for (0..self.left_tensors) |_| try self.skipTensor(); - const padding: usize = getAlignmentPadding(self.alignment, self.off); - self.file.data_offset = self.off + padding; - - try self.file.file.seekTo(base_off); - self.off = base_off; - } - - pub fn skipTensor(self: *GgufFile) !void { - try self.skipString(); // Skip name - const num_dim: u32 = try self.readInt(u32); - // dimensions, type, and offset. - try self.skipBytes(8 * num_dim + 4 + 8); - } - - /// Parses the next tensor entry. - /// Returns error.EndOfMetadata if there are no longer tensor metadata to process in this GGUF file. - pub fn readTensorInfo(self: *GgufFile, allocator: std.mem.Allocator) !GgufTensorInfo { - if (self.left_tensors == 0 or self.left_kv != 0) { - return error.EndOfMetadata; - } - - // We want to return tensor data with offsets relative to the start - // of the file, so that the user of the API is able to access tensors - // as it iterates over them. To do so, we need to perform a full - // scan if this is the first tensor info we are reading. - // TODO: explicitly set the data offset in - if (self.file.data_offset == 0) try self.setDataOffset(); - self.left_tensors -= 1; - const name = try self.readString(allocator); - const num_dim = try self.readInt(u32); - assert(@as(usize, @intCast(num_dim)) <= GGUF_TENSOR_MAX_DIM); - // Read the dimentions; unused dimensions are left `undefined`. - // Note: we reverse the order of the dimensions to match zml convention. - var dims: [GGUF_TENSOR_MAX_DIM]i64 = undefined; - var num_weights: usize = 1; - for (0..num_dim) |j| { - const d = try self.readInt(u64); - dims[num_dim - 1 - j] = @intCast(d); - num_weights *= d; - } - const t: TensorType = try self.readTensorType(); - const start = try self.readInt(u64); - // To accurately calculate the bytes used by this tensor on the GGUF - // file, we need to take into account that quantization methods store - // tensors as block of N weights. So first of all we need to understand - // the number of padding weights (since the last block may have just - // fewer weights stored inside, but still requires to be stored to its full - // length). Then we can do the math to see how many blocks we need, and - // multiply by the block size to obtain the final total size. - const tf = t.getFeatures(); - const byte_len: usize = (std.math.divCeil(usize, num_weights, tf.items_per_block) catch unreachable) * tf.bytes_per_block; - return .{ - .name = name, - .t = t, - .rank = num_dim, - .dims = dims, - .start = start, - .byte_len = byte_len, - .num_weights = num_weights, - }; - } -}; - -/// Given an offset or a length, returns the padding needed to align it to alignment. -fn getAlignmentPadding(alignment: usize, offset: usize) usize { - return @rem((alignment - @rem(offset, alignment)), alignment); -} diff --git a/zml/aio/json.zig b/zml/aio/json.zig deleted file mode 100644 index c8ba494c..00000000 --- a/zml/aio/json.zig +++ /dev/null @@ -1,112 +0,0 @@ -const asynk = @import("async"); -const std = @import("std"); -const utils = @import("utils.zig"); -const zml = @import("../zml.zig"); - -const StringBuilder = std.ArrayListUnmanaged(u8); -const Allocator = std.mem.Allocator; - -pub fn open(allocator: std.mem.Allocator, path: []const u8) !zml.aio.BufferStore { - const file = try std.fs.cwd().openFile(path, .{}); - defer file.close(); - var res: zml.aio.BufferStore = .{ - .arena = std.heap.ArenaAllocator.init(allocator), - }; - errdefer res.arena.deinit(); - const arena = res.arena.allocator(); - - const json_data = try file.reader().readAllAlloc(arena, (try file.metadata()).size()); - const metadata = try std.json.parseFromSliceLeaky(std.json.Value, allocator, json_data, .{ .allocate = .alloc_if_needed }); - - var it = metadata.object.iterator(); - while (it.next()) |entry| { - var prefix_buf: [1024]u8 = undefined; - try parseMetadata(allocator, &res, StringBuilder.initBuffer(&prefix_buf), entry.value_ptr.*); - } - - return res; -} - -pub fn parseMetadata(allocator: Allocator, store: *zml.aio.BufferStore, prefix: StringBuilder, val: std.json.Value) !void { - const metadata = &store._metadata; - const key = prefix.items; - return switch (val) { - .null => try metadata.put(allocator, try allocator.dupe(u8, key), .{ .null = {} }), - .bool => |v| try metadata.put(allocator, try allocator.dupe(u8, key), .{ .boolval = v }), - .integer => |v| try metadata.put(allocator, try allocator.dupe(u8, key), .{ .int64 = v }), - .float => |v| try metadata.put(allocator, try allocator.dupe(u8, key), .{ .float64 = v }), - .number_string, .string => |v| try metadata.put(allocator, try allocator.dupe(u8, key), .{ .string = try allocator.dupe(u8, v) }), - .array => |v| { - if (v.items.len == 0) return; - return if (validSlice(v)) |item_type| { - const data, const dtype: zml.aio.Value.Slice.ItemType = switch (item_type) { - .bool => blk: { - const values = try allocator.alloc(bool, v.items.len); - for (v.items, 0..) |item, i| values[i] = item.bool; - break :blk .{ std.mem.sliceAsBytes(values), .boolval }; - }, - .integer => blk: { - const values = try allocator.alloc(i64, v.items.len); - for (v.items, 0..) |item, i| values[i] = item.integer; - break :blk .{ std.mem.sliceAsBytes(values), .int64 }; - }, - .float => blk: { - const values = try allocator.alloc(f64, v.items.len); - for (v.items, 0..) |item, i| values[i] = item.float; - break :blk .{ std.mem.sliceAsBytes(values), .float64 }; - }, - inline .string, .number_string => |tag| blk: { - const values = try allocator.alloc([]const u8, v.items.len); - for (v.items, 0..) |item, i| { - values[i] = @field(item, @tagName(tag)); - } - break :blk .{ std.mem.sliceAsBytes(values), .string }; - }, - .null, .array, .object => unreachable, - }; - try metadata.put( - allocator, - try allocator.dupe(u8, key), - .{ .array = .{ .item_type = dtype, .data = data } }, - ); - } else { - for (v.items, 0..) |item, i| { - var new_prefix = prefix; - if (prefix.items.len > 0) - new_prefix.appendAssumeCapacity('.'); - new_prefix.items.len += std.fmt.formatIntBuf(new_prefix.unusedCapacitySlice(), i, 10, .lower, .{}); - try parseMetadata(allocator, store, new_prefix, item); - } - }; - }, - .object => |v| { - var obj_iter = v.iterator(); - while (obj_iter.next()) |entry| { - var new_prefix = prefix; - if (prefix.items.len > 0) - new_prefix.appendAssumeCapacity('.'); - new_prefix.appendSliceAssumeCapacity(entry.key_ptr.*); - try parseMetadata(allocator, store, new_prefix, entry.value_ptr.*); - } - }, - }; -} - -/// We can only create a Zig slice out of json array, if all values -/// in the array have the same type. -fn validSlice(v: std.json.Array) ?std.meta.Tag(std.json.Value) { - if (v.items.len == 0) return null; - - const item_type: std.meta.Tag(std.json.Value) = v.items[0]; - switch (item_type) { - .null, .array, .object => return null, - else => {}, - } - - for (v.items[1..]) |item| { - if (item != item_type) - return null; - } - - return item_type; -} diff --git a/zml/aio/nemo.zig b/zml/aio/nemo.zig deleted file mode 100644 index d52854e4..00000000 --- a/zml/aio/nemo.zig +++ /dev/null @@ -1,56 +0,0 @@ -const asynk = @import("async"); -const eval = @import("torch/eval.zig"); -const std = @import("std"); -const yaml = @import("zig-yaml"); -const zml = @import("../zml.zig"); - -const Decoder = @import("torch/parser.zig").Decoder; - -const StringBuilder = std.ArrayListUnmanaged(u8); - -pub fn open(allocator: std.mem.Allocator, path: []const u8) !zml.aio.BufferStore { - var res: zml.aio.BufferStore = .{ - .arena = std.heap.ArenaAllocator.init(allocator), - }; - errdefer res.arena.deinit(); - - const arena = res.arena.allocator(); - - var mapped_file = try zml.aio.MemoryMappedFile.init(try asynk.File.open(path, .{})); - errdefer mapped_file.deinit(); - - var file_name_buffer: [std.fs.max_path_bytes]u8 = undefined; - var link_name_buffer: [std.fs.max_path_bytes]u8 = undefined; - var tar_iter = std.tar.iterator( - mapped_file.file.reader(), - .{ - .file_name_buffer = &file_name_buffer, - .link_name_buffer = &link_name_buffer, - }, - ); - while (try tar_iter.next()) |file| { - if (std.mem.endsWith(u8, file.name, ".yaml")) { - const yaml_data = try file.reader().readAllAlloc(arena, file.size); - const parsed = try yaml.Yaml.load(arena, yaml_data); - - var prefix_buf: [1024]u8 = undefined; - try zml.aio.yaml.parseMetadata(arena, &res, StringBuilder.initBuffer(&prefix_buf), parsed.docs.items[0]); - } else if (std.mem.endsWith(u8, file.name, ".ckpt") or std.mem.endsWith(u8, file.name, ".pt")) { - const start = try mapped_file.file.getPos(); - var tmp: zml.aio.torch.PickleData = .{ - .data = try Decoder.fromTarFile(arena, mapped_file, file), - .memo = undefined, - .stack = undefined, - }; - tmp.stack, tmp.memo = try eval.evaluate(arena, tmp.data.ops, true); - try tmp.parseModel(arena, &res); - // Since we directly manipulate the file handle pointer, - // reset to the end of file so iterator does not error - // and avoid `skipBytes`. - try mapped_file.file.seekTo(start + file.size); - file.unread_bytes.* = 0; - } else if (std.mem.eql(u8, file.name, "./model_weights/")) @panic(".NeMo sharded weights are not yet supported") else continue; - } - - return res; -} diff --git a/zml/aio/safetensors.zig b/zml/aio/safetensors.zig deleted file mode 100644 index 519be290..00000000 --- a/zml/aio/safetensors.zig +++ /dev/null @@ -1,144 +0,0 @@ -const asynk = @import("async"); -const std = @import("std"); -const zml = @import("../zml.zig"); -const helpers = @import("../helpers.zig"); -const utils = @import("utils.zig"); -const json = @import("json.zig"); -const HostBuffer = @import("../hostbuffer.zig").HostBuffer; -const MemoryMappedFile = @import("../aio.zig").MemoryMappedFile; - -const StringBuilder = std.ArrayListUnmanaged(u8); -const Allocator = std.mem.Allocator; -const log = std.log.scoped(.zml_io); - -pub fn open(allocator: std.mem.Allocator, path: []const u8) !zml.aio.BufferStore { - var res: zml.aio.BufferStore = .{ - .arena = std.heap.ArenaAllocator.init(allocator), - }; - errdefer res.arena.deinit(); - const arena = res.arena.allocator(); - - var files = std.ArrayList(MemoryMappedFile).init(arena); - errdefer files.deinit(); - - if (std.mem.endsWith(u8, path, ".safetensors.index.json")) { - try loadFromIndex(arena, &res, &files, path); - } else { - try loadFile(arena, &res, &files, path); - } - res.files = try files.toOwnedSlice(); - return res; -} - -fn loadFromIndex(allocator: Allocator, store: *zml.aio.BufferStore, files: *std.ArrayList(MemoryMappedFile), path: []const u8) !void { - const file = asynk.File.open(path, .{}) catch |err| { - log.err("Failed to open {s}: {}", .{ path, err }); - return err; - }; - errdefer file.close() catch unreachable; - var r = file.reader(); - - const json_data = try allocator.alloc(u8, (try file.stat()).size); - _ = try r.readAtLeast(json_data, json_data.len); - const metadata = try std.json.parseFromSliceLeaky(std.json.Value, allocator, json_data, .{ .allocate = .alloc_if_needed }); - var loaded_files = std.StringHashMap(void).init(allocator); - - const weight_map = metadata.object.get("weight_map").?.object; - var it = weight_map.iterator(); - while (it.next()) |entry| { - const filename = entry.value_ptr.string; - if (loaded_files.contains(filename)) { - continue; - } - - log.debug("Loading shard: {s}", .{filename}); - try loaded_files.put(filename, {}); - - const full_filename = try std.fs.path.join(allocator, &.{ std.fs.path.dirname(path).?, filename }); - try loadFile(allocator, store, files, full_filename); - } -} - -fn loadFile(allocator: Allocator, store: *zml.aio.BufferStore, files: *std.ArrayList(MemoryMappedFile), path: []const u8) !void { - const file = asynk.File.open(path, .{}) catch |err| { - log.err("Failed to open {s}: {}", .{ path, err }); - return err; - }; - errdefer file.close() catch unreachable; - var r = file.reader(); - - const json_header_length: usize = @intCast(try r.readInt(u64, std.builtin.Endian.little)); - const json_data = try allocator.alloc(u8, json_header_length); - const n = try r.readAll(json_data); - if (n != json_header_length) { - log.err("Failed to read the full {} bytes of json header from file {s}", .{ n, path }); - return error.CorruptedFile; - } - - const metadata = try std.json.parseFromSliceLeaky(std.json.Value, allocator, json_data[0..n], .{}); - var buffer_file = try MemoryMappedFile.init(file); - errdefer buffer_file.deinit(); - buffer_file.data_offset = 8 + json_header_length; - - try files.append(buffer_file); - errdefer _ = files.popOrNull(); - - var it = metadata.object.iterator(); - while (it.next()) |entry| { - const key = entry.key_ptr.*; - if (std.mem.eql(u8, key, "__metadata__")) { - var prefix_buf: [1024]u8 = undefined; - try json.parseMetadata(allocator, store, StringBuilder.initBuffer(&prefix_buf), entry.value_ptr.*); - continue; - } - const val = entry.value_ptr.*; - const shape_field = val.object.get("shape").?.array; - if (shape_field.items.len > zml.Shape.MAX_RANK) { - // Not an error until someone tries to read the tensor itself. - log.warn("Can't load tensor {s}, too many dims: {}", .{ key, shape_field.items.len }); - continue; - } - const offset_field = val.object.get("data_offsets").?; - const start: usize = @intCast(offset_field.array.items[0].integer); - const end: usize = @intCast(offset_field.array.items[1].integer); - const dtype = try stringToDtype(val.object.get("dtype").?.string); - var dims: std.BoundedArray(i64, zml.Shape.MAX_RANK) = .{}; - for (shape_field.items) |d| { - dims.appendAssumeCapacity(d.integer); - } - - const out_shape = zml.Shape.init(dims.constSlice(), dtype); - // We aren't storing 'end', so check we can infer it from the tensor shape. - // This is fine cause safetensor only allow storing contiguous tensors. - // https://github.com/huggingface/safetensors/blob/main/README.md#format - // > The byte buffer needs to be entirely indexed, and cannot contain holes. This prevents the creation of polyglot files. - std.debug.assert(end - start == out_shape.byteSize()); - - const buf = HostBuffer.fromBytes(out_shape, buffer_file.mappedSlice(start, out_shape.byteSize())); - try store.buffers.put(allocator, try allocator.dupe(u8, key), buf); - } -} - -fn stringToDtype(safetensor_type: []const u8) !zml.DataType { - const map = std.StaticStringMap(zml.DataType).initComptime(.{ - .{ "F64", .f64 }, - .{ "F32", .f32 }, - .{ "F16", .f16 }, - .{ "BF16", .bf16 }, - .{ "F8_E4M3", .f8e4m3fn }, - .{ "I64", .i64 }, - .{ "I32", .i32 }, - .{ "I16", .i16 }, - .{ "I8", .i8 }, - .{ "U64", .u64 }, - .{ "U32", .u32 }, - .{ "U16", .u16 }, - .{ "U8", .u8 }, - .{ "BOOL", .bool }, - }); - - return map.get(safetensor_type) orelse { - log.err("Unsupported safetensor data type: {s}", .{safetensor_type}); - return error.UnsupportedDataType; - }; -} diff --git a/zml/aio/sentencepiece.zig b/zml/aio/sentencepiece.zig deleted file mode 100644 index 90ee128b..00000000 --- a/zml/aio/sentencepiece.zig +++ /dev/null @@ -1,89 +0,0 @@ -const std = @import("std"); -const asynk = @import("async"); -const zml = @import("../zml.zig"); - -const sentencepiece_proto = @import("//sentencepiece:model_proto"); -const Normalizer = zml.tokenizer.Normalizer; -const Tokenizer = zml.tokenizer.Tokenizer; - -pub fn loadTokenizerFromPath(allocator: std.mem.Allocator, path: []const u8) !Tokenizer { - const file = try asynk.File.open(path, .{}); - defer file.close() catch unreachable; - - return loadTokenizerFromFile(allocator, file); -} - -pub fn loadTokenizerFromFile(allocator: std.mem.Allocator, file: asynk.File) !Tokenizer { - const reader = file.reader(); - const input = try reader.readAllAlloc(allocator, 16 * 1024 * 1024); - defer allocator.free(input); - - var proto_arena = std.heap.ArenaAllocator.init(allocator); - defer proto_arena.deinit(); - - const model = try sentencepiece_proto.ModelProto.decode(input, proto_arena.allocator()); - // no deinit, memory will be freed by the proto_arena - - return loadTokenizerFromModelProto(allocator, model); -} - -pub fn loadTokenizerFromModelProto(allocator: std.mem.Allocator, model: sentencepiece_proto.ModelProto) !Tokenizer { - std.debug.assert(model.trainer_spec.?.model_type.? == .BPE); - const special_tokens: Tokenizer.SpecialTokens = .{ - .unk = @intCast(model.trainer_spec.?.unk_id.?), - .bos = @intCast(model.trainer_spec.?.bos_id.?), - .eos = @intCast(model.trainer_spec.?.eos_id.?), - .pad = parseTokenId(model.trainer_spec.?.pad_id), - }; - - var tokenizer = try Tokenizer.init( - allocator, - @intCast(model.pieces.items.len), - @intCast(model.trainer_spec.?.max_sentencepiece_length.?), - normalizerFromSpec(model.normalizer_spec.?), - special_tokens, - true, - ); - errdefer tokenizer.deinit(); - - for (model.pieces.items) |*piece| { - try tokenizer.addToken(piece.score.?, piece.piece.?.getSlice()); - } - const byte_fallback = model.trainer_spec.?.byte_fallback orelse false; - if (byte_fallback) { - try tokenizer.rewriteByteFallbackTokens(); - } - - return tokenizer; -} - -fn parseTokenId(id: ?i32) u32 { - if (id) |idx| { - if (idx > 0) return @intCast(idx); - } - - return std.math.maxInt(u32); -} - -pub fn normalizerFromSpec(spec: sentencepiece_proto.NormalizerSpec) Normalizer { - std.log.info("NormalizerSpec: {}", .{spec}); - if (spec.normalization_rule_tsv) |rule_tsv| { - if (!rule_tsv.isEmpty()) { - std.debug.panic("SentencePiece model with normalization rules not supported: model.normalizer_spec.normalization_rule_tsv: {s}", .{spec.normalization_rule_tsv.?.getSlice()}); - } - } - if (!std.mem.eql(u8, spec.name.?.getSlice(), "identity")) std.debug.panic("Normalizer only supports NormalizerSpec with name \"identity\", got \"{s}\"", .{spec.name.?.getSlice()}); - if (!spec.escape_whitespaces.?) std.debug.panic("Normalizer only supports NormalizerSpec with \"escape_whitespaces\" flag set", .{}); - if (spec.remove_extra_whitespaces) |_| {} else std.debug.panic("Normalizer only supports NormalizerSpec with \"remove_extra_whitespaces\" flag set", .{}); - - return Normalizer.init( - .{ - .remove_extra_whitespaces = spec.remove_extra_whitespaces orelse false, - .add_dummy_prefix = spec.add_dummy_prefix orelse false, - .add_dummy_suffix = false, - .lower_case_ascii = false, - .split_on_punct_ascii = false, - }, - if (spec.escape_whitespaces orelse false) Normalizer.sentencepiece_space else null, - ); -} diff --git a/zml/aio/tinyllama.zig b/zml/aio/tinyllama.zig deleted file mode 100644 index 3f65149c..00000000 --- a/zml/aio/tinyllama.zig +++ /dev/null @@ -1,162 +0,0 @@ -/// Tools to load models from https://huggingface.co/karpathy/tinyllamas/ -/// Originally made to be run with https://github.com/karpathy/llama2.c -const std = @import("std"); - -const asynk = @import("async"); - -const zml = @import("../zml.zig"); - -const TinyLlamaConfig = extern struct { - dim: u32, - hidden_dim: u32, - n_layers: u32, - n_heads: u32, - n_kv_heads: u32, - // llama2.c uses the sign bit of the vocab size to store if we need - // a second embedding layer. - vocab: packed struct(u32) { - size: u31, - has_lm_head: bool, - }, - seq_len: u32, -}; - -/// Open a tinyllama file. -/// For convenience we use the same layer names -/// than the one used by the Llama-3.1 models. -pub fn open(allocator: std.mem.Allocator, model_path: []const u8) !zml.aio.BufferStore { - var res: zml.aio.BufferStore = .{ - .arena = std.heap.ArenaAllocator.init(allocator), - }; - errdefer res.arena.deinit(); - const arena = res.arena.allocator(); - - const file = try asynk.File.open(model_path, .{}); - res.files = try arena.alloc(zml.aio.MemoryMappedFile, 1); - res.files[0] = try zml.aio.MemoryMappedFile.init(file); - - const c = try file.reader().readStructEndian(TinyLlamaConfig, .little); - zml.log.info("Read TinyLlamaConfig: {}", .{c}); - - const dim = c.dim; - const hidden_dim = c.hidden_dim; - const n_layers = c.n_layers; - const n_heads = c.n_heads; - const n_kv_heads = c.n_kv_heads; - const seq_len = c.seq_len; - const head_size = c.dim / c.n_heads; - - var off: usize = @sizeOf(TinyLlamaConfig); - try res.buffers.ensureUnusedCapacity(arena, 8 + 9 * n_layers); - - // Map tinyllama naming to Meta llama3 naming. - // token_embedding_table - off = newBuff(&res, "model.embed_tokens.weight", .{ .voc = c.vocab.size, .d = dim }, off); - // rms_att_weight - off = try splitBuff(&res, "model.layers.{d}.input_layernorm.weight", .{dim}, n_layers, off); - // wq - off = try splitBuff(&res, "model.layers.{d}.self_attn.q_proj.weight", .{ dim, n_heads * head_size }, n_layers, off); - // wk - off = try splitBuff(&res, "model.layers.{d}.self_attn.k_proj.weight", .{ dim, n_kv_heads * head_size }, n_layers, off); - // wv - off = try splitBuff(&res, "model.layers.{d}.self_attn.v_proj.weight", .{ dim, n_kv_heads * head_size }, n_layers, off); - // wo - off = try splitBuff(&res, "model.layers.{d}.self_attn.o_proj.weight", .{ n_heads * head_size, dim }, n_layers, off); - // rms_ffn_weight - off = try splitBuff(&res, "model.layers.{d}.post_attention_layernorm.weight", .{dim}, n_layers, off); - // w1 - off = try splitBuff(&res, "model.layers.{d}.mlp.gate_proj.weight", .{ hidden_dim, dim }, n_layers, off); - // w2 - off = try splitBuff(&res, "model.layers.{d}.mlp.down_proj.weight", .{ dim, hidden_dim }, n_layers, off); - // w3 - off = try splitBuff(&res, "model.layers.{d}.mlp.up_proj.weight", .{ hidden_dim, dim }, n_layers, off); - // rms_final_weight - off = newBuff(&res, "model.norm.weight", .{dim}, off); - // freq_cis_real (not used) - off = newBuff(&res, "freq_cis_real", .{ seq_len, head_size / 2 }, off); - off = newBuff(&res, "freq_cis_imag", .{ seq_len, head_size / 2 }, off); - - // wcls - if (c.vocab.has_lm_head) { - off = newBuff(&res, "lm_head.weight", .{ c.vocab.size, c.dim }, off); - } else { - res.buffers.putAssumeCapacityNoClobber("lm_head.weight", res.buffers.get("model.embed_tokens.weight").?); - } - - const weights_size = off; - std.log.info("Loaded a tinyllama file of {} bytes.\nThis is the parsed configuration of this llama model: {}", .{ weights_size, c }); - if (file.stat() catch null) |stat| { - zml.meta.assert(weights_size == stat.size, "Expected to have a tinyllama file of {} bytes but file only got {} !\nThis is the parsed configuration of this llama model: {}", .{ weights_size, stat.size, c }); - } - - { - try res._metadata.ensureUnusedCapacity(arena, 11); - res._metadata.putAssumeCapacityNoClobber("dim", .{ .int64 = c.dim }); - res._metadata.putAssumeCapacityNoClobber("hidden_dim", .{ .int64 = c.hidden_dim }); - res._metadata.putAssumeCapacityNoClobber("n_layers", .{ .int64 = c.n_layers }); - res._metadata.putAssumeCapacityNoClobber("num_heads", .{ .int64 = c.n_heads }); - res._metadata.putAssumeCapacityNoClobber("num_kv_heads", .{ .int64 = c.n_kv_heads }); - res._metadata.putAssumeCapacityNoClobber("vocab_size", .{ .int64 = c.vocab.size }); - res._metadata.putAssumeCapacityNoClobber("has_lm_head", .{ .boolval = c.vocab.has_lm_head }); - res._metadata.putAssumeCapacityNoClobber("max_seq_len", .{ .int64 = c.seq_len }); - res._metadata.putAssumeCapacityNoClobber("rope_impl", .{ .string = "interleaved" }); - res._metadata.putAssumeCapacityNoClobber("rope_freq_base", .{ .float64 = 10_000 }); - res._metadata.putAssumeCapacityNoClobber("rms_norm_eps", .{ .float64 = 1e-6 }); - } - - return res; -} - -fn newBuff(store: *zml.aio.BufferStore, name: []const u8, sh: anytype, offset: usize) usize { - var shape = zml.Shape.init(sh, .f32); - const n = shape.byteSize(); - const buff = zml.HostBuffer.fromBytes(shape, store.files[0].data[offset..][0..n]); - store.buffers.putAssumeCapacityNoClobber(name, buff); - zml.log.info("Found {s}: {}", .{ name, shape }); - return offset + n; -} - -fn splitBuff(store: *zml.aio.BufferStore, comptime fmt: []const u8, sh: anytype, layers: usize, offset: usize) !usize { - var shape = zml.Shape.init(sh, .f32); - const n = shape.byteSize(); - var off = offset; - for (0..layers) |i| { - const name = try std.fmt.allocPrint(store.arena.allocator(), fmt, .{i}); - const buff = zml.HostBuffer.fromBytes(shape, store.files[0].data[off..][0..n]); - store.buffers.putAssumeCapacityNoClobber(name, buff); - off += n; - if (i == 0) zml.log.info("Found {s}: {}", .{ name, shape }); - } - return off; -} - -pub fn loadTokenizer(allocator: std.mem.Allocator, tokenizer_path: []const u8, vocab_size: u32) !zml.tokenizer.Tokenizer { - const tokenizer_file = try std.fs.cwd().openFile(tokenizer_path, .{}); - defer tokenizer_file.close(); - var tok_reader = std.io.bufferedReader(tokenizer_file.reader()); - const r = tok_reader.reader(); - - const max_token_len = try r.readInt(u32, .little); - const special_tokens: zml.tokenizer.Tokenizer.SpecialTokens = .{ - .unk = 0, - .bos = 1, - .eos = 2, - }; - var tokenizer = try zml.tokenizer.Tokenizer.init(allocator, vocab_size, max_token_len, null, special_tokens, true); - var i: u32 = 0; - while (readToken(&tokenizer, &r)) : (i += 1) { - // Pass - } else |_| { - if (i < vocab_size) { - zml.log.info("Read {d} words out of {?d}", .{ i, vocab_size }); - } - tokenizer.vocab_size = i; - } - return tokenizer; -} - -fn readToken(tokenizer: *zml.tokenizer.Tokenizer, tok_reader: anytype) !void { - const score: f32 = @bitCast(try tok_reader.readInt(u32, .little)); - const len: usize = @intCast(try tok_reader.readInt(u32, .little)); - try tokenizer.readTokenInto(score, len, tok_reader); -} diff --git a/zml/aio/torch.zig b/zml/aio/torch.zig deleted file mode 100644 index 5620a693..00000000 --- a/zml/aio/torch.zig +++ /dev/null @@ -1,422 +0,0 @@ -const asynk = @import("async"); -const std = @import("std"); -const zml = @import("../zml.zig"); - -const HostBuffer = @import("../hostbuffer.zig").HostBuffer; - -const toVoidSlice = @import("utils.zig").toVoidSlice; -const eval = @import("torch/eval.zig"); -const utils = @import("torch/utils.zig"); -const value = @import("torch/value.zig"); -const Decoder = @import("torch/parser.zig").Decoder; -const PersId = value.PersId; -const PickleMemo = eval.PickleMemo; -const PickleStack = eval.PickleStack; -const Sequence = value.Sequence; -const Value = value.Value; -const ValueType = value.ValueType; - -const StringBuilder = std.ArrayListUnmanaged(u8); -const Allocator = std.mem.Allocator; -const log = std.log.scoped(.zml_io); - -/// Opens and loads a BufferStore from the torch file at the given path. -pub fn open(allocator: Allocator, path: []const u8) !zml.aio.BufferStore { - const file = asynk.File.open(path, .{}) catch |err| { - log.err("Failed to open {s}: {}", .{ path, err }); - return err; - }; - errdefer file.close() catch unreachable; - - var res: zml.aio.BufferStore = .{ - .arena = std.heap.ArenaAllocator.init(allocator), - }; - - const arena = res.arena.allocator(); - - var tmp: PickleData = .{ - .data = try Decoder.init(arena, file), - .memo = undefined, - .stack = undefined, - }; - tmp.stack, tmp.memo = try eval.evaluate(arena, tmp.data.ops, true); - res.files = try arena.dupe(zml.aio.MemoryMappedFile, &.{tmp.data.buffer_file}); - try tmp.parseModel(arena, &res); - return res; -} - -/// Convert from a torch.Storage to a `zml.DataType`. -/// TODO: make this future proof, storage type are going to get replaced with torch.UntypedStorage -/// See https://pytorch.org/docs/stable/storage.html -fn storageToDtype(storage_type: []const u8) !zml.DataType { - const torch_type = storage_type[0 .. storage_type.len - "Storage".len]; - const map = std.StaticStringMap(zml.DataType).initComptime(.{ - .{ "Double", .f64 }, - .{ "Float", .f32 }, - .{ "Half", .f16 }, - .{ "Long", .i64 }, - .{ "Int", .i32 }, - .{ "Short", .i16 }, - .{ "Char", .i8 }, - .{ "Byte", .u8 }, - .{ "Bool", .bool }, - .{ "BFloat16", .bf16 }, - .{ "ComplexDouble", .c128 }, - .{ "ComplexFloat", .c64 }, - // QUInt8Storage - // QInt8Storage - // QInt32Storage - // QUInt4x2Storage - // QUInt2x4Storage - }); - - return map.get(torch_type) orelse { - log.err("Unsupported torch storage type: {s}", .{storage_type}); - return error.UnsupportedDataType; - }; -} - -pub const PickleData = struct { - stack: PickleStack, - memo: PickleMemo, - data: Decoder, - - fn basicTypeCheck(v: Value, ns: []const u8, name: []const u8) bool { - return switch (v) { - .global => |object| switch (object.member) { - .raw => |raw| { - if (std.mem.eql(u8, ns, raw.global.module) and std.mem.eql(u8, name, raw.global.class) and object.args[0] == .seq) { - return true; - } else return false; - }, - else => false, - }, - else => false, - }; - } - - fn isTensor(v: Value) bool { - if (basicTypeCheck(v, "torch._utils", "_rebuild_tensor_v2")) { - const args = v.global.args[0].seq.values; - if (args.len >= 5 and - args[0] == .pers_id and - args[1] == .int64 and - args[2] == .seq and args[2].seq.type == .tuple and - args[3] == .seq and args[3].seq.type == .tuple) - { - return true; - } else @panic("Unexpected value in call to torch._utils._rebuild_tensor_v2"); - } - return false; - } - - fn dimsFromValues(values: []Value) [zml.Tensor.MAX_RANK]i64 { - std.debug.assert(values.len <= zml.Tensor.MAX_RANK); - var result: [zml.Tensor.MAX_RANK]i64 = undefined; - for (values, result[0..values.len]) |val, *elem| { - switch (val) { - .int64 => |int| elem.* = int, - else => @panic("Bad value for shape item"), - } - } - return result; - } - - pub fn parseModel(self: *PickleData, allocator: Allocator, store: *zml.aio.BufferStore) !void { - for (self.stack.stack) |item| { - var prefix_buf: [1024]u8 = undefined; - try self.parseValue(allocator, store, StringBuilder.initBuffer(&prefix_buf), item); - } - } - - fn tensorOffset(self: *PickleData, seekable_stream: anytype, sfile: []const u8) !u64 { - if (self.data.file_map.get(sfile)) |entry| { - const local_header = blk: { - try seekable_stream.seekTo(entry.file_offset); - break :blk try seekable_stream.context.reader().readStructEndian(std.zip.LocalFileHeader, .little); - }; - if (!std.mem.eql(u8, &local_header.signature, &std.zip.local_file_header_sig)) - return error.ZipBadFileOffset; - if (local_header.version_needed_to_extract != entry.version_needed_to_extract) - return error.ZipMismatchVersionNeeded; - if (local_header.last_modification_time != entry.last_modification_time) - return error.ZipMismatchModTime; - if (local_header.last_modification_date != entry.last_modification_date) - return error.ZipMismatchModDate; - - if (@as(u16, @bitCast(local_header.flags)) != entry.flags) - return error.ZipMismatchFlags; - if (local_header.crc32 != 0 and local_header.crc32 != entry.crc32) - return error.ZipMismatchCrc32; - if (local_header.compressed_size != 0 and - local_header.compressed_size != entry.compressed_size) - return error.ZipMismatchCompLen; - if (local_header.uncompressed_size != 0 and - local_header.uncompressed_size != entry.uncompressed_size) - return error.ZipMismatchUncompLen; - if (local_header.filename_len != entry.filename_len) - return error.ZipMismatchFilenameLen; - - return (try seekable_stream.context.getPos()) + - @as(u64, local_header.filename_len) + - @as(u64, local_header.extra_len); - } - - std.log.err("Could not find file ending in `{s}` in archive", .{sfile}); - return error.TensorNotFound; - } - - fn parseTorchGlobal(self: *PickleData, allocator: Allocator, store: *zml.aio.BufferStore, prefix: StringBuilder, v: Value) !bool { - return switch (v) { - .global => |object| { - if (isTensor(v)) { - const args = object.args[0].seq.values; - const pidval: *PersId, var offs: u64, const raw_shape: Sequence, const raw_strides: Sequence = .{ args[0].pers_id, @intCast(args[1].int64), args[2].seq, args[3].seq }; - const rank = raw_shape.values.len; - const shape = dimsFromValues(raw_shape.values); - var strides = dimsFromValues(raw_strides.values); - const storage_type, const sfile = switch (pidval.ref) { - .seq => |seq| blk: { - const sargs = seq.values; - if (seq.type == .tuple and - sargs.len >= 5 and - sargs[0] == .string and std.mem.eql(u8, sargs[0].string, "storage") and - sargs[1] == .raw and sargs[1].raw == .global and - sargs[2] == .string and - sargs[3] == .string) - { - const op = sargs[1].raw.global; - const sfile = sargs[2].string; - // const sdev = sargs[3].string; - if (std.mem.eql(u8, "torch", op.module) and std.mem.endsWith(u8, op.class, "Storage")) { - break :blk .{ op.class, sfile }; - } else @panic("Unexpected storage type part of persistant ID"); - } else @panic("Unexpected value for persistant ID"); - }, - else => @panic("Unexpected value for persistant ID"), - }; - - const data_type = try storageToDtype(storage_type); - for (strides[0..rank]) |*s| s.* *= data_type.sizeOf(); - - var sfile_buf = std.ArrayList(u8).init(allocator); - defer sfile_buf.deinit(); - try sfile_buf.writer().print("{s}data/{s}", .{ self.data.zip_prefix, sfile }); - - // find offsets for tensor zip file - const absolute_offset = blk: { - if (self.data.tar_file) |t| { - break :blk try self.tensorOffset(t.seekableStream(), sfile_buf.items); - } else { - break :blk try self.tensorOffset(self.data.buffer_file.file.seekableStream(), sfile_buf.items); - } - }; - offs = offs * data_type.sizeOf(); - const key = try allocator.dupe(u8, prefix.items); - const entry = try store.buffers.getOrPut(allocator, key); - if (entry.found_existing) { - log.warn("Duplicate key: {s}", .{prefix.items}); - allocator.free(key); - } - const out_shape = zml.Shape.init(shape[0..rank], data_type); - entry.value_ptr.* = HostBuffer.fromStridedSlice( - out_shape, - self.data.buffer_file.mappedSlice((if (self.data.tar_file) |t| t.start else 0) + absolute_offset + offs, out_shape.byteSize()), - strides[0..rank], - ); - return true; - } else if (basicTypeCheck(v, "torch", "Size")) { - const size = object.args[0].seq.values[0].seq.values; - const key = try allocator.dupe(u8, prefix.items); - const entry = try store._metadata.getOrPut(allocator, key); - if (entry.found_existing) { - log.warn("Duplicate key: {s}", .{prefix.items}); - allocator.free(key); - } - const d = try allocator.alloc(i64, size.len); - for (d, 0..) |*di, i| di.* = size[i].int64; - entry.value_ptr.* = .{ .array = .{ .item_type = .int64, .data = std.mem.sliceAsBytes(d) } }; - return true; - } else if (basicTypeCheck(v, "fractions", "Fraction")) { - const fraction_str = object.args[0].seq.values[0].string; - if (std.mem.indexOfScalar(u8, fraction_str, '/')) |split_idx| { - { - var new_prefix = prefix; - new_prefix.appendSliceAssumeCapacity(".numerator"); - try store._metadata.put(allocator, try allocator.dupe(u8, new_prefix.items), .{ .int64 = try std.fmt.parseInt(i64, fraction_str[0..split_idx], 10) }); - } - { - var new_prefix = prefix; - new_prefix.appendSliceAssumeCapacity(".denominator"); - try store._metadata.put(allocator, try allocator.dupe(u8, new_prefix.items), .{ .int64 = try std.fmt.parseInt(i64, fraction_str[split_idx + 1 ..], 10) }); - } - return true; - } - } - return false; - }, - else => false, - }; - } - - pub fn parseValue(self: *PickleData, allocator: Allocator, store: *zml.aio.BufferStore, prefix: StringBuilder, v: Value) !void { - switch (v) { - .app, .object, .global => |object| { - if (!(try self.parseTorchGlobal(allocator, store, prefix, v))) { - try self.parseValue(allocator, store, prefix, object.member); - for (object.args) |item| { - // if possible, coerce to `kv_tuple` (only if key val doesn't match root of prefix) - if (item == .seq and item.seq.type == .tuple and item.seq.values.len == 2 and item.seq.values[0] == .string) { - try self.parseValue(allocator, store, prefix, .{ .seq = .{ .type = .kv_tuple, .values = item.seq.values } }); - } else try self.parseValue(allocator, store, prefix, item); - } - } - }, - .build => |build| { - // `build` contains info about python struct being constructed - switch (build.member) { - .object => |obj| switch (obj.member) { - .raw => |raw| switch (raw) { - .global => |global| { - // in this case, we can capture the name of the python type - // which can be used for codegen (e.g. `torch.nn.modules.conv.Conv2d`) - var new_prefix = prefix; - if (prefix.items.len > 0) { - new_prefix.appendAssumeCapacity('.'); - } - new_prefix.appendSliceAssumeCapacity("_gen_type_helper"); - const key = try allocator.dupe(u8, new_prefix.items); - const d = try store._metadata.getOrPut(allocator, key); - if (d.found_existing) { - log.err("Duplicate key: {s}", .{new_prefix.items}); - allocator.free(key); - } else { - const val = try std.mem.join(allocator, ".", &.{ global.module, global.class }); - d.value_ptr.* = .{ .string = val }; - } - }, - else => try self.parseValue(allocator, store, prefix, build.member), // parse normally - }, - else => try self.parseValue(allocator, store, prefix, build.member), // parse normally - }, - else => try self.parseValue(allocator, store, prefix, build.member), // parse normally - } - try self.parseValue(allocator, store, prefix, build.args); - }, - .pers_id => |pers_id| try self.parseValue(allocator, store, prefix, pers_id.ref), - .seq => |seq| { - switch (seq.type) { - .list, .tuple, .set, .frozen_set => { - if (seq.values.len == 0) return; - var valid_slice = true; - switch (seq.values[0]) { - inline .int64, .float64, .boolval => |val0, tag| { - const ItemType = switch (tag) { - .int64 => i64, - .float64 => f64, - .boolval => bool, - else => unreachable, - }; - var values: std.ArrayListUnmanaged(ItemType) = .{}; - try values.append(allocator, val0); - for (seq.values[1..], 1..) |val, i| { - if (std.meta.activeTag(val) != tag) valid_slice = false; - if (valid_slice) { - try values.append(allocator, @field(val, @tagName(tag))); - } else { - var new_prefix = prefix; - if (prefix.items.len > 0) { - new_prefix.appendAssumeCapacity('.'); - } - new_prefix.items.len += std.fmt.formatIntBuf(new_prefix.unusedCapacitySlice(), i, 10, .lower, .{}); - try self.parseValue(allocator, store, new_prefix, val); - } - } - - if (valid_slice) { - try store._metadata.put( - allocator, - try allocator.dupe(u8, prefix.items), - .{ .array = .{ .item_type = std.meta.stringToEnum(zml.aio.Value.Slice.ItemType, @tagName(tag)).?, .data = std.mem.sliceAsBytes(try values.toOwnedSlice(allocator)) } }, - ); - } else { - for (values.items, 0..) |val, i| { - var new_prefix = prefix; - if (prefix.items.len > 0) { - new_prefix.appendAssumeCapacity('.'); - } - new_prefix.items.len += std.fmt.formatIntBuf(new_prefix.unusedCapacitySlice(), i, 10, .lower, .{}); - try store._metadata.put(allocator, try allocator.dupe(u8, new_prefix.items), @unionInit(zml.aio.Value, @tagName(tag), val)); - } - } - }, - else => { - for (seq.values, 0..) |item, i| { - var new_prefix = prefix; - if (v.isPrimitive()) { - if (prefix.items.len > 0) { - new_prefix.appendAssumeCapacity('.'); - } - new_prefix.items.len += std.fmt.formatIntBuf(new_prefix.unusedCapacitySlice(), i, 10, .lower, .{}); - } - try self.parseValue(allocator, store, new_prefix, item); - } - }, - } - }, - .dict => for (seq.values) |item| { - try self.parseValue(allocator, store, prefix, item); - }, - .kv_tuple => { - const key, const val = seq.values[0..2].*; - switch (key) { - .string => |s| { - // Handle Pytorch specific fields - if (std.mem.eql(u8, s, "_modules") or std.mem.eql(u8, s, "_parameters") or std.mem.eql(u8, s, "_buffers")) { - try self.parseValue(allocator, store, prefix, val); - } else { - var new_prefix = prefix; - if (prefix.items.len > 0) { - new_prefix.appendAssumeCapacity('.'); - } - new_prefix.appendSliceAssumeCapacity(s); - try self.parseValue(allocator, store, new_prefix, val); - } - }, - .int64 => |int| { - var new_prefix = prefix; - if (prefix.items.len > 0) { - new_prefix.appendAssumeCapacity('.'); - } - new_prefix.items.len += std.fmt.formatIntBuf(new_prefix.unusedCapacitySlice(), int, 10, .lower, .{}); - try self.parseValue(allocator, store, new_prefix, val); - }, - inline else => |_, tag| std.debug.panic("Unexpected key type: {s}", .{@tagName(tag)}), - } - }, - } - }, - .bytes => |val| { - const key = try allocator.dupe(u8, prefix.items); - const d = try store._metadata.getOrPut(allocator, key); - if (d.found_existing) { - log.warn("Duplicate key: {s}", .{prefix.items}); - allocator.free(key); - } else d.value_ptr.* = .{ .array = .{ .item_type = .uint8, .data = @constCast(val) } }; - }, - inline .float64, .int64, .boolval, .bigint, .string => |val, tag| { - const key = try allocator.dupe(u8, prefix.items); - const d = try store._metadata.getOrPut(allocator, key); - if (d.found_existing) { - log.warn("Duplicate key: {s}", .{prefix.items}); - allocator.free(key); - } else d.value_ptr.* = @unionInit(zml.aio.Value, @tagName(tag), val); - }, - else => {}, - } - } -}; - -test { - std.testing.refAllDecls(@This()); -} diff --git a/zml/aio/torch/b_tree_map.zig b/zml/aio/torch/b_tree_map.zig deleted file mode 100644 index 19cc8bf6..00000000 --- a/zml/aio/torch/b_tree_map.zig +++ /dev/null @@ -1,652 +0,0 @@ -const std = @import("std"); - -/// BTreeMap Node implementation. -pub fn NodeType(comptime K: type, comptime V: type, comptime B: u32) type { - return struct { - const Self = @This(); - keys: [2 * B - 1]K = [_]K{undefined} ** (2 * B - 1), - values: [2 * B - 1]V = [_]V{undefined} ** (2 * B - 1), - len: usize = 0, - edges: [2 * B]?*Self = [_]?*Self{null} ** (2 * B), - - pub const KV = struct { key: K, value: V }; - const KVE = struct { key: K, value: V, edge: ?*Self }; - const Entry = struct { key_ptr: *K, value_ptr: *V }; - - /// Initializes an empty Node. - pub fn initEmpty(allocator: std.mem.Allocator) !*Self { - const res: *Self = try allocator.create(Self); - res.* = .{}; - return res; - } - - /// Initializes a Node with a single Entry. - pub fn initKeyValue(allocator: std.mem.Allocator, entry: struct { K, V }) !*Self { - const key, const value = entry; - var res = try Self.initEmpty(allocator); - res.keys[0] = key; - res.values[0] = value; - res.len = 1; - return res; - } - - fn initFromSplit(allocator: std.mem.Allocator, keys: []K, values: []V, edges: []?*Self) !*Self { - var out = try Self.initEmpty(allocator); - std.mem.copyBackwards(K, out.keys[0..], keys); - std.mem.copyBackwards(V, out.values[0..], values); - std.mem.copyBackwards(?*Self, out.edges[0..], edges); - out.len = keys.len; - return out; - } - - pub fn count(self: Self) usize { - var len: usize = self.len; - for (0..self.len + 1) |i| { - if (!self.isLeaf()) { - len += self.edges[i].?.count(); - } - } - return len; - } - - // Searches the Node for a key. - pub fn search(self: Self, key: K) std.meta.Tuple(&.{ bool, usize }) { - var i: usize = 0; - while (i < self.len) : (i += 1) { - if (eql(key, self.keys[i])) { - return .{ true, i }; - } else if (lt(key, self.keys[i])) { - return .{ false, i }; - } - } - return .{ false, self.len }; - } - - pub fn insertOrSplit( - self: *Self, - allocator: std.mem.Allocator, - index: usize, - key: K, - value: V, - edge: ?*Self, - ) !?KVE { - if (self.isFull()) { - var split_result = try self.split(allocator); - switch (index < B) { - true => self.insert(index, key, value, edge), - false => split_result.edge.?.insert(index - B, key, value, edge), - } - return split_result; - } - self.insert(index, key, value, edge); - return null; - } - - pub fn swapValue(self: *Self, index: usize, value: V) V { - const out = self.values[index]; - self.values[index] = value; - return out; - } - - pub fn swapKeyValue(self: *Self, index: usize, key: K, value: V) KV { - const out = .{ .key = self.keys[index], .value = self.values[index] }; - self.values[index] = value; - self.keys[index] = key; - return out; - } - - pub fn orderedRemove(self: *Self, index: usize) KVE { - const out: KVE = .{ - .key = self.keys[index], - .value = self.values[index], - .edge = self.edges[index + 1], - }; - std.mem.copyForwards(K, self.keys[index..], self.keys[index + 1 .. self.len]); - std.mem.copyForwards(V, self.values[index..], self.values[index + 1 .. self.len]); - self.keys[self.len - 1] = undefined; - self.values[self.len - 1] = undefined; - if (!self.isLeaf()) { - std.mem.copyForwards(?*Self, self.edges[index + 1 ..], self.edges[index + 2 .. self.len + 1]); - self.edges[self.len] = null; - } - self.len -= 1; - return out; - } - - fn pop(self: *Self) KVE { - return self.orderedRemove(self.len - 1); - } - - fn shift(self: *Self) KVE { - const out: KVE = .{ - .key = self.keys[0], - .value = self.values[0], - .edge = self.edges[0], - }; - std.mem.copyForwards(K, self.keys[0..], self.keys[1..self.len]); - std.mem.copyForwards(V, self.values[0..], self.values[1..self.len]); - self.keys[self.len - 1] = undefined; - self.values[self.len - 1] = undefined; - if (!self.isLeaf()) { - std.mem.copyForwards( - ?*Self, - self.edges[0..], - self.edges[1 .. self.len + 1], - ); - self.edges[self.len] = null; - } - self.len -= 1; - return out; - } - - fn insert(self: *Self, index: usize, key: K, value: V, edge: ?*Self) void { - std.mem.copyBackwards( - K, - self.keys[index + 1 .. self.len + 1], - self.keys[index..self.len], - ); - self.keys[index] = key; - std.mem.copyBackwards(V, self.values[index + 1 .. self.len + 1], self.values[index..self.len]); - self.values[index] = value; - if (!self.isLeaf()) { - std.mem.copyBackwards(?*Self, self.edges[index + 2 .. self.len + 2], self.edges[index + 1 .. self.len + 1]); - self.edges[index + 1] = edge; - } - self.len += 1; - } - - fn append(self: *Self, key: K, value: V, edge: ?*Self) void { - self.keys[self.len] = key; - self.values[self.len] = value; - self.edges[self.len + 1] = edge; - self.len += 1; - } - - fn unshift(self: *Self, key: K, value: V, edge: ?*Self) void { - std.mem.copyBackwards(K, self.keys[1 .. self.len + 1], self.keys[0..self.len]); - self.keys[0] = key; - std.mem.copyBackwards(V, self.values[1 .. self.len + 1], self.values[0..self.len]); - self.values[0] = value; - if (!self.isLeaf()) { - std.mem.copyBackwards(?*Self, self.edges[1 .. self.len + 2], self.edges[0 .. self.len + 1]); - self.edges[0] = edge; - } - self.len += 1; - } - - pub fn borrowRight(self: *Self, index: usize) bool { - if (index == self.len) return false; - var from = self.edges[index + 1].?; - if (from.len > B - 1) { - var to = self.edges[index].?; - const borrowed = from.shift(); - to.append(self.keys[index], self.values[index], borrowed.edge); - _ = self.swapKeyValue(index, borrowed.key, borrowed.value); - return true; - } - return false; - } - - pub fn borrowLeft(self: *Self, index: usize) bool { - if (index == 0) return false; - var from = self.edges[index - 1].?; - if (from.len > B - 1) { - var to = self.edges[index].?; - const borrowed = from.pop(); - to.unshift(self.keys[index - 1], self.values[index - 1], borrowed.edge); - _ = self.swapKeyValue(index - 1, borrowed.key, borrowed.value); - return true; - } - return false; - } - - pub fn mergeEdges(self: *Self, allocator: std.mem.Allocator, left_edge_index: usize) void { - var left = self.edges[left_edge_index].?; - const removed = self.orderedRemove(left_edge_index); - left.append(removed.key, removed.value, null); - std.mem.copyBackwards(K, left.keys[left.len..], removed.edge.?.keys[0..removed.edge.?.len]); - std.mem.copyBackwards(V, left.values[left.len..], removed.edge.?.values[0..removed.edge.?.len]); - std.mem.copyBackwards(?*Self, left.edges[left.len..], removed.edge.?.edges[0 .. removed.edge.?.len + 1]); - left.len += removed.edge.?.len; - allocator.destroy(removed.edge.?); - } - - fn split(self: *Self, allocator: std.mem.Allocator) !KVE { - const median = B - 1; - const new_key = self.keys[median]; - const new_value = self.values[median]; - const new_node = try Self.initFromSplit( - allocator, - self.keys[median + 1 .. self.len], - self.values[median + 1 .. self.len], - self.edges[median + 1 .. self.len + 1], - ); - @memset(self.keys[median..], undefined); - @memset(self.values[median..], undefined); - @memset(self.edges[median + 1 ..], null); - self.len = median; - return .{ .key = new_key, .value = new_value, .edge = new_node }; - } - - pub fn isLeaf(self: Self) bool { - return self.edges[0] == null; - } - - pub fn isFull(self: Self) bool { - return self.len == 2 * B - 1; - } - - pub fn isLacking(self: Self) bool { - return self.len < B - 1; - } - }; -} - -pub fn BTreeMap(comptime K: type, comptime V: type) type { - return struct { - const Self = @This(); - - const B = 6; - const Node = NodeType(K, V, B); - const KV = Node.KV; - const SearchResult = std.meta.Tuple(&.{ bool, usize }); - const StackEntry = struct { node: *Node, index: usize }; - - allocator: std.mem.Allocator, - root: ?*Node = null, - - pub fn init(allocator: std.mem.Allocator) Self { - return .{ .allocator = allocator }; - } - - pub fn deinit(self: Self) !void { - if (self.root == null) return; - var stack = std.ArrayList(*Node).init(self.allocator); - defer stack.deinit(); - if (self.root) |root| { - try stack.append(root); - } - while (stack.popOrNull()) |node| { - if (!node.isLeaf()) { - for (0..node.len + 1) |i| { - try stack.append(node.edges[i].?); - } - } - self.allocator.destroy(node); - } - } - - pub fn count(self: Self) usize { - if (self.root == null) return 0; - var len: usize = 0; - if (self.root) |node| { - len += node.count(); - } - return len; - } - - pub fn isEmpty(self: *const Self) bool { - if (self.root == null) return true; - return self.root.?.len == 0; - } - - pub fn get(self: Self, key: K) ?V { - var current = self.root; - while (current) |node| { - const found, const index = node.search(key); - switch (found) { - true => return node.values[index], - false => current = node.edges[index], - } - } - return null; - } - - pub fn getPtr(self: Self, key: K) ?*V { - var current = self.root; - while (current) |node| { - const found, const index = node.search(key); - switch (found) { - true => return &node.values[index], - false => current = node.edges[index], - } - } - return null; - } - - pub fn fetchPut(self: *Self, key: K, value: V) !?KV { - if (self.root == null) { - self.root = try Node.initKeyValue(self.allocator, .{ key, value }); - return null; - } - var stack = std.ArrayList(StackEntry).init(self.allocator); - defer stack.deinit(); - var current = self.root; - var search_result: SearchResult = undefined; - while (current) |node| { - search_result = node.search(key); - if (search_result[0]) { - return .{ .key = key, .value = node.swapValue(search_result[1], value) }; - } - current = node.edges[search_result[1]]; - try stack.append(.{ .node = node, .index = search_result[1] }); - } - var stack_next: ?StackEntry = stack.pop(); - var split_result = try stack_next.?.node.insertOrSplit( - self.allocator, - stack_next.?.index, - key, - value, - null, - ); - if (split_result == null) { - return null; - } - stack_next = stack.popOrNull(); - while (split_result) |split_result_unwrapped| { - if (stack_next) |stack_next_unwrapped| { - split_result = try stack_next_unwrapped.node.insertOrSplit( - self.allocator, - stack_next_unwrapped.index, - split_result_unwrapped.key, - split_result_unwrapped.value, - split_result_unwrapped.edge, - ); - stack_next = stack.popOrNull(); - } else { - var new_root = try Node.initKeyValue( - self.allocator, - .{ split_result_unwrapped.key, split_result_unwrapped.value }, - ); - new_root.edges[0] = self.root; - new_root.edges[1] = split_result_unwrapped.edge; - self.root = new_root; - return null; - } - } else return null; - } - - pub fn fetchRemove(self: *Self, key: K) !?KV { - var stack = std.ArrayList(StackEntry).init(self.allocator); - defer stack.deinit(); - var current = self.root; - var search_result: SearchResult = undefined; - var found_key_ptr: ?*K = null; - var found_value_ptr: ?*V = null; - while (current) |node| { - search_result = node.search(key); - if (search_result[0]) { - found_key_ptr = &node.keys[search_result[1]]; - found_value_ptr = &node.values[search_result[1]]; - if (!node.isLeaf()) search_result[1] += 1; - } - try stack.append(.{ - .node = node, - .index = search_result[1], - }); - current = node.edges[search_result[1]]; - if (search_result[0]) break; - } else return null; - while (current) |node| { - try stack.append(.{ .node = node, .index = 0 }); - current = node.edges[0]; - } - var current_stack = stack.pop(); - const out: KV = .{ .key = found_key_ptr.?.*, .value = found_value_ptr.?.* }; - found_key_ptr.?.* = current_stack.node.keys[current_stack.index]; - found_value_ptr.?.* = current_stack.node.values[current_stack.index]; - _ = current_stack.node.orderedRemove(current_stack.index); - if (current_stack.node == self.root) return out; - while (current_stack.node.isLacking()) { - current_stack = stack.pop(); - if (current_stack.node.borrowRight(current_stack.index)) return out; - if (current_stack.node.borrowLeft(current_stack.index)) return out; - if (current_stack.index == current_stack.node.len) { - current_stack.node.mergeEdges(self.allocator, current_stack.index - 1); - } else { - current_stack.node.mergeEdges(self.allocator, current_stack.index); - } - if (current_stack.node == self.root) { - if (self.root.?.len == 0) { - const new_root = current_stack.node.edges[0].?; - self.allocator.destroy(self.root.?); - self.root.? = new_root; - } - break; - } - } - return out; - } - - const Iterator = struct { - stack: std.ArrayList(StackEntry), - backwards: bool, - - pub fn deinit(it: Iterator) void { - it.stack.deinit(); - } - - pub fn next(it: *Iterator) ?Node.Entry { - while (it.topStackItem()) |item| { - if (!item.node.isLeaf() and !it.backwards) { - const child = item.node.edges[item.index].?; - it.stack.append(StackEntry{ .node = child, .index = 0 }) catch unreachable; - } else { - if (item.index < item.node.len) { - const out: Node.Entry = .{ .key_ptr = &item.node.keys[item.index], .value_ptr = &item.node.values[item.index] }; - item.index += 1; - it.backwards = false; - return out; - } else { - _ = it.stack.popOrNull(); - it.backwards = true; - } - } - } else return null; - } - - fn topStackItem(it: *Iterator) ?*StackEntry { - return switch (it.stack.items.len) { - 0 => null, - else => &it.stack.items[it.stack.items.len - 1], - }; - } - }; - - pub fn iterator(self: *const Self) Iterator { - var new_stack = std.ArrayList(StackEntry).init(self.allocator); - if (self.root) |root| { - new_stack.append(.{ .node = root, .index = 0 }) catch unreachable; - } - return Iterator{ - .stack = new_stack, - .backwards = false, - }; - } - }; -} - -/// Compares two of any type for equality. Containers are compared on a field-by-field basis, -/// where possible. Pointers are followed if the addresses are not equal. -fn eql(a: anytype, b: @TypeOf(a)) bool { - const T = @TypeOf(a); - switch (@typeInfo(T)) { - .Struct => |info| { - inline for (info.fields) |field_info| { - if (!eql(@field(a, field_info.name), @field(b, field_info.name))) return false; - } - return true; - }, - .ErrorUnion => { - if (a) |a_p| { - if (b) |b_p| return eql(a_p, b_p) else |_| return false; - } else |a_e| { - if (b) |_| return false else |b_e| return a_e == b_e; - } - }, - .Union => |info| { - if (info.tag_type) |UnionTag| { - const tag_a = std.meta.activeTag(a); - const tag_b = std.meta.activeTag(b); - if (tag_a != tag_b) return false; - - inline for (info.fields) |field_info| { - if (@field(UnionTag, field_info.name) == tag_a) { - return eql(@field(a, field_info.name), @field(b, field_info.name)); - } - } - return false; - } - - @compileError("Cannot compare untagged union type " ++ @typeName(T)); - }, - .Array => { - if (a.len != b.len) return false; - for (a, 0..) |e, i| - if (!eql(e, b[i])) return false; - return true; - }, - .Vector => |info| { - var i: usize = 0; - while (i < info.len) : (i += 1) { - if (!eql(a[i], b[i])) return false; - } - return true; - }, - .Pointer => |info| { - return switch (info.size) { - .One => if (a == b) true else eql(a.*, b.*), - .Many => if (a == b) true else { - if (info.sentinel) { - if (std.mem.len(a) != std.mem.len(b)) return false; - var i: usize = 0; - while (i < std.mem.len(a)) : (i += 1) - if (!eql(a[i], b[i])) return false; - return true; - } - @compileError("Cannot compare many-item Pointers without sentinel value"); - }, - .C => if (a == b) true else @compileError("Cannot compare C pointers"), - .Slice => if (a.ptr == b.ptr and a.len == b.len) true else { - if (a.len != b.len) return false; - for (a, 0..) |_, i| - if (!eql(a[i], b[i])) return false; - return true; - }, - }; - }, - .Optional => { - if (a == null and b == null) return true; - if (a == null or b == null) return false; - return eql(a.?, b.?); - }, - else => return a == b, - } -} - -fn lt(a: anytype, b: @TypeOf(a)) bool { - const T = @TypeOf(a); - - switch (@typeInfo(T)) { - .Int, .ComptimeInt, .Float, .ComptimeFloat => { - return a < b; - }, - .Struct => { - if (!@hasDecl(T, "lt")) { - @compileError("Type `" ++ @typeName(T) ++ "` must implement a `lt` comparison method."); - } - return T.lt(a, b); - }, - .Union => |info| { - if (info.tag_type) |UnionTag| { - const tag_a = std.meta.activeTag(a); - const tag_b = std.meta.activeTag(b); - // if tags are not equal, perform comparison based on tag - if (tag_a != tag_b) { - return std.ascii.lessThanIgnoreCase(@tagName(tag_a), @tagName(tag_b)); - } - // if tags are equal, compare based on the active field - inline for (info.fields) |field_info| { - if (@field(UnionTag, field_info.name) == tag_a) { - return lt(@field(a, field_info.name), @field(b, field_info.name)); - } - } - return false; - } - - @compileError("Cannot perform `lt` check on untagged union type " ++ @typeName(T)); - }, - .Array => { - for (a, 0..) |_, i| { - if (lt(a[i], b[i])) { - return true; - } else if (eql(a[i], b[i])) { - continue; - } else { - return false; - } - } - return false; - }, - .Vector => |info| { - var i: usize = 0; - while (i < info.len) : (i += 1) { - if (lt(a[i], b[i])) { - return true; - } else if (eql(a[i], b[i])) { - continue; - } else { - return false; - } - } - return false; - }, - .Pointer => |info| { - switch (info.size) { - .One => return lt(a.*, b.*), - .Slice => { - const n = @min(a.len, b.len); - for (a[0..n], 0..) |_, i| { - if (lt(a[i], b[i])) { - return true; - } else if (eql(a[i], b[i])) { - continue; - } else { - return false; - } - } - return lt(a.len, b.len); - }, - .Many => { - if (info.sentinel) { - const n = @min(std.mem.len(a), std.mem.len(b)); - var i: usize = 0; - while (i < n) : (i += 1) { - if (lt(a[i], b[i])) { - return true; - } else if (eql(a[i], b[i])) { - continue; - } else { - return false; - } - } - return lt(std.mem.len(a), std.mem.len(b)); - } - @compileError("Cannot compare many-item pointer to unknown number of items without sentinel value"); - }, - .C => @compileError("Cannot compare C pointers"), - } - }, - .Optional => { - if (a == null or b == null) return false; - return lt(a.?, b.?); - }, - else => { - @compileError("Cannot compare type '" ++ @typeName(T) ++ "'"); - }, - } -} - -pub fn gt(a: anytype, b: @TypeOf(a)) bool { - return !lt(a, b) and !eql(a, b); -} diff --git a/zml/aio/torch/eval.zig b/zml/aio/torch/eval.zig deleted file mode 100644 index d54f02da..00000000 --- a/zml/aio/torch/eval.zig +++ /dev/null @@ -1,485 +0,0 @@ -const std = @import("std"); -const zml = @import("../../zml.zig"); -const meta = zml.meta; - -const value = @import("value.zig"); -const BTreeMap = @import("b_tree_map.zig").BTreeMap; -const PickleOp = @import("ops.zig").PickleOp; - -const Build = value.Build; -const Object = value.Object; -const PersId = value.PersId; -const Sequence = value.Sequence; -const SequenceType = value.SequenceType; -const Value = value.Value; - -const MAX_DEPTH: usize = 250; -const MAX_PROTOCOL: u8 = 5; - -pub const PickleMemo = struct { - allocator: std.mem.Allocator, - map: BTreeMap(u32, Value), - - pub fn init(allocator: std.mem.Allocator) PickleMemo { - return .{ - .allocator = allocator, - .map = BTreeMap(u32, Value).init(allocator), - }; - } - - pub fn deinit(self: *PickleMemo) void { - var iterator = self.map.iterator(); - defer iterator.deinit(); - while (iterator.next()) |entry| { - entry.value_ptr.deinit(self.allocator); - } - self.map.deinit() catch unreachable; - self.* = undefined; - } - - pub fn resolve(self: *PickleMemo, allocator: std.mem.Allocator, op: Value, recursive: bool) !Value { - var used_op = op; - while (used_op == .ref) { - var count: usize = 0; - const val = self.map.get(op.ref) orelse { - return error.BadMemoRef; - }; - if (!recursive) { - return val.clone(allocator); - } - - count += 1; - if (count >= MAX_DEPTH or val != .ref) { - used_op = try val.clone(allocator); - break; - } - used_op = val; - } - if (used_op.containsRef()) { - switch (used_op) { - .app, .object, .global => |v| { - if (v.member.containsRef()) { - v.member = try self.resolve(allocator, v.member, recursive); - } - for (v.args) |*item| { - if (item.containsRef()) { - item.* = try self.resolve(allocator, item.*, recursive); - } - } - }, - .build => |v| { - if (v.member.containsRef()) { - v.member = try self.resolve(allocator, v.member, recursive); - } - if (v.args.containsRef()) { - v.args = try self.resolve(allocator, v.args, recursive); - } - }, - .pers_id => |v| { - if (v.ref.containsRef()) { - v.ref = try self.resolve(allocator, v.ref, recursive); - } - }, - .seq => |*v| { - for (v.values) |*item| { - if (item.containsRef()) { - item.* = try self.resolve(allocator, item.*, recursive); - } - } - }, - else => {}, - } - } - return used_op; - } - - pub fn insert(self: *PickleMemo, mid: u32, val: Value) !void { - _ = try self.map.fetchPut(mid, val); - } - - pub fn resolveMut(self: *PickleMemo, op: *Value, recursive: bool) !*Value { - if (op.* != .ref) return op; - var lastmid = op.ref; - var count: usize = 0; - var val = self.map.get(lastmid) orelse { - return error.BadMemoRef; - }; - while (val == .ref) { - lastmid = val.ref; - if (!recursive) { - break; - } - count += 1; - if (count >= MAX_DEPTH) { - break; - } - val = self.map.get(lastmid) orelse { - return error.BadMemoRef; - }; - } - return (self.map.getPtr(lastmid) orelse { - return error.BadMemoRef; - }); - } - - const MemoError = std.math.big.int.Managed.ConvertError || std.mem.Allocator.Error || error{BadMemoRef}; - - pub fn resolveAllRefsIter(self: *PickleMemo, allocator: std.mem.Allocator, depth: usize, vals: []Value, fix_values: bool) MemoError![]Value { - if (depth >= MAX_DEPTH) { - return vals; - } - const res = try allocator.alloc(Value, vals.len); - for (vals, 0..) |v, i| { - res[i] = try self.resolveAllRefs(allocator, depth + 1, v, fix_values); - } - return res; - } - - pub fn resolveAllRefs(self: *PickleMemo, allocator: std.mem.Allocator, depth: usize, val: Value, fix_values: bool) !Value { - var output: Value = switch (val) { - .ref => try self.resolve(allocator, val, true), - inline .app, .object, .global => |v, tag| @unionInit(Value, @tagName(tag), try Object.init( - allocator, - try self.resolveAllRefs(allocator, depth + 1, v.member, fix_values), - try self.resolveAllRefsIter(allocator, depth + 1, v.args, fix_values), - )), - .build => |v| .{ .build = try Build.init( - allocator, - try self.resolveAllRefs(allocator, depth + 1, v.member, fix_values), - try self.resolveAllRefs(allocator, depth + 1, v.args, fix_values), - ) }, - .seq => |v| .{ .seq = .{ .type = v.type, .values = try self.resolveAllRefsIter(allocator, depth + 1, v.values, fix_values) } }, - .pers_id => |v| .{ .pers_id = try PersId.init(allocator, try self.resolveAllRefs(allocator, depth + 1, v.ref, fix_values)) }, - else => try val.clone(allocator), - }; - if (fix_values) { - output = try output.coerceFromRaw(allocator); - } - return output; - } -}; - -pub const InternalStack = struct { - allocator: std.mem.Allocator, - values: std.ArrayList(Value), - - pub fn init(allocator: std.mem.Allocator) InternalStack { - return .{ - .allocator = allocator, - .values = std.ArrayList(Value).init(allocator), - }; - } - - pub fn deinit(self: *InternalStack) void { - for (0..self.values.items.len) |i| self.values.items[i].deinit(self.allocator); - self.values.deinit(); - self.* = undefined; - } - - pub fn pop(self: *InternalStack) !Value { - if (self.values.items.len == 0) { - return error.StackUnderrun; - } - return self.values.pop(); - } - - pub fn popMark(self: *InternalStack, allocator: ?std.mem.Allocator) ![]Value { - const markidx = try self.findMark(); - var postmark: []Value = &[_]Value{}; - if (allocator) |a| { - postmark = try a.alloc(Value, self.values.items.len - (markidx + 1)); - @memcpy(postmark, self.values.items[markidx + 1 ..]); - } - self.values.shrinkAndFree(markidx); - return postmark; - } - - pub fn lastMut(self: *InternalStack) !*Value { - if (self.values.items.len == 0) { - return error.UnexpectedEmptyStack; - } - return &self.values.items[self.values.items.len - 1]; - } - - pub fn findMark(self: *InternalStack) !usize { - const len = self.values.items.len; - for (0..len) |i| { - const idx = (len - 1) - i; - const val = self.values.items[idx]; - if (val == .raw and val.raw == .mark) { - return idx; - } - } - zml.log.warn("pytorch loader: missing mark", .{}); - return 0; - } - - pub fn toPickleStack(self: *InternalStack) !PickleStack { - return .{ .stack = try self.values.toOwnedSlice(), .allocator = self.allocator }; - } -}; - -pub const PickleStack = struct { - stack: []Value, - allocator: std.mem.Allocator, - - pub fn init(allocator: std.mem.Allocator, values: []Value) PickleStack { - return .{ .allocator = allocator, .stack = values }; - } - - pub fn deinit(self: *PickleStack) void { - for (self.stack) |*v| v.deinit(self.allocator); - self.allocator.free(self.stack); - } -}; - -pub fn evaluate(allocator: std.mem.Allocator, x: []const PickleOp, resolve_refs: bool) !struct { PickleStack, PickleMemo } { - var stack = InternalStack.init(allocator); - defer stack.deinit(); - var memo = PickleMemo.init(allocator); - errdefer memo.deinit(); - - const makeKVList = (struct { - pub fn call(alloc: std.mem.Allocator, items: []const Value) ![]Value { - meta.assert(items.len & 1 == 0, "Bad value for setitems", .{}); - var kv_items = try std.ArrayList(Value).initCapacity(alloc, items.len); - errdefer kv_items.deinit(); - var idx: usize = 0; - while (idx < items.len) : (idx += 2) { - if (idx + 1 >= items.len) { - return error.MissingValueItem; - } - const kv = try alloc.alloc(Value, 2); - kv[0] = items[idx]; - kv[1] = items[idx + 1]; - kv_items.appendAssumeCapacity(.{ .seq = .{ .type = .kv_tuple, .values = kv } }); - } - return kv_items.toOwnedSlice(); - } - }).call; - - outer: for (x) |op| { - switch (op) { - .mark => try stack.values.append(.{ .raw = op }), - .stop => break :outer, - .pop => _ = try stack.pop(), - .pop_mark => _ = try stack.popMark(allocator), - .dup => { - if (stack.values.getLastOrNull()) |item| { - try stack.values.append(try item.clone(allocator)); - } else { - return error.CannotDupEmptyStack; - } - }, - .persid => |v| try stack.values.append(.{ .pers_id = try PersId.init(allocator, .{ .string = try allocator.dupe(u8, v) }) }), - .binpersid => try stack.values.append(.{ .pers_id = try PersId.init(allocator, try stack.pop()) }), - .reduce => try stack.values.append(.{ .global = blk: { - const values = try allocator.alloc(Value, 1); - values[0] = try memo.resolve(allocator, try stack.pop(), true); - break :blk try Object.init(allocator, try memo.resolve(allocator, try stack.pop(), true), values); - } }), - .build => try stack.values.append(blk: { - const args = try memo.resolve(allocator, try stack.pop(), true); - const member = try memo.resolve(allocator, try stack.pop(), true); - break :blk .{ .build = try Build.init(allocator, member, args) }; - }), - .empty_dict => try stack.values.append(.{ .seq = .{ .type = .dict, .values = &[_]Value{} } }), - .get => |v| try stack.values.append(.{ .ref = try std.fmt.parseInt(u32, v, 10) }), - inline .binget, .long_binget => |v| try stack.values.append(.{ .ref = v }), - .empty_list => try stack.values.append(.{ .seq = .{ .type = .list, .values = &[_]Value{} } }), - .binput, .long_binput => |v| { - try memo.insert(v, try stack.pop()); - try stack.values.append(.{ .ref = v }); - }, - .tuple => try stack.values.append(blk: { - const popped = try stack.popMark(allocator); - break :blk .{ .seq = .{ .type = .tuple, .values = popped } }; - }), - .empty_tuple => try stack.values.append(.{ .seq = .{ .type = .tuple, .values = &[_]Value{} } }), - .setitem => { - const v, const k = .{ try stack.pop(), try stack.pop() }; - const top = try stack.lastMut(); - const rtop = try memo.resolveMut(top, true); - switch (rtop.*) { - .global => |obj| { - obj.args = try assuredResize(Value, allocator, obj.args, obj.args.len + 1); - obj.args[obj.args.len - 1] = .{ .seq = .{ .type = .tuple, .values = try allocator.dupe(Value, &.{ k, v }) } }; - }, - .seq => |*tup| { - tup.values = try assuredResize(Value, allocator, tup.values, tup.values.len + 1); - tup.values[tup.values.len - 1] = .{ .seq = .{ .type = .tuple, .values = try allocator.dupe(Value, &.{ k, v }) } }; - }, - else => { - return error.BadStackTopForSetItem; - }, - } - }, - .setitems => { - const popped = try stack.popMark(allocator); - defer allocator.free(popped); - const kv_items = try makeKVList(allocator, popped); - const top = try stack.lastMut(); - const rtop = try memo.resolveMut(top, true); - switch (rtop.*) { - .global => |obj| { - obj.args = try assuredResize(Value, allocator, obj.args, obj.args.len + 1); - obj.args[obj.args.len - 1] = .{ .seq = .{ .type = .tuple, .values = kv_items } }; - }, - .seq => |*tup| { - tup.values = try assuredResize(Value, allocator, tup.values, tup.values.len + 1); - tup.values[tup.values.len - 1] = .{ .seq = .{ .type = .tuple, .values = kv_items } }; - }, - else => { - defer allocator.free(kv_items); - return error.BadStackTopForSetItems; - }, - } - }, - .proto => |proto| meta.assert(proto <= MAX_PROTOCOL, "Unsupported protocol {d}", .{proto}), - .tuple1 => try stack.values.append(blk: { - const tup_values = try allocator.alloc(Value, 1); - tup_values[0] = try stack.pop(); - break :blk .{ .seq = .{ .type = .tuple, .values = tup_values } }; - }), - .tuple2 => try stack.values.append(blk: { - const tup_values = try allocator.alloc(Value, 2); - inline for (0..2) |i| tup_values[(tup_values.len - 1) - i] = try stack.pop(); - break :blk .{ .seq = .{ .type = .tuple, .values = tup_values } }; - }), - .tuple3 => try stack.values.append(blk: { - const tup_values = try allocator.alloc(Value, 3); - inline for (0..3) |i| tup_values[(tup_values.len - 1) - i] = try stack.pop(); - break :blk .{ .seq = .{ .type = .tuple, .values = tup_values } }; - }), - .append => { - const v = try stack.pop(); - const top = try stack.lastMut(); - const rtop = try memo.resolveMut(top, true); - switch (rtop.*) { - .global => |obj| { - obj.args = try assuredResize(Value, allocator, obj.args, obj.args.len + 1); - obj.args[obj.args.len - 1] = v; - }, - .seq => |*tup| { - tup.values = try assuredResize(Value, allocator, tup.values, tup.values.len + 1); - tup.values[tup.values.len - 1] = v; - }, - else => { - return error.BadStackTopForAppend; - }, - } - }, - .appends => { - const postmark = try stack.popMark(allocator); - defer allocator.free(postmark); - const top = try stack.lastMut(); - const rtop = try memo.resolveMut(top, true); - switch (rtop.*) { - .global => |obj| { - const obj_len = obj.args.len; - obj.args = try assuredResize(Value, allocator, obj.args, obj_len + postmark.len); - @memcpy(obj.args[obj_len..], postmark); - }, - .seq => |*tup| { - const tup_len = tup.values.len; - tup.values = try assuredResize(Value, allocator, tup.values, tup_len + postmark.len); - @memcpy(tup.values[tup_len..], postmark); - }, - else => { - return error.BadStackTopForAppends; - }, - } - }, - .dict => try stack.values.append(blk: { - const popped = try stack.popMark(allocator); - defer allocator.free(popped); - const kv_items = try makeKVList(allocator, popped); - break :blk .{ .seq = .{ .type = .dict, .values = kv_items } }; - }), - .list => try stack.values.append(.{ .seq = .{ .type = .list, .values = try stack.popMark(allocator) } }), - .inst => |v| try stack.values.append(blk: { - const tup_items = try allocator.dupe(Value, &.{ .{ .string = v.module }, .{ .string = v.class } }); - break :blk .{ .object = try Object.init(allocator, .{ .seq = .{ .type = .tuple, .values = tup_items } }, try stack.popMark(allocator)) }; - }), - .obj => try stack.values.append(blk: { - const markidx = try stack.findMark(); - const args = try allocator.alloc(Value, stack.values.items.len - (markidx + 2)); - @memcpy(args, stack.values.items[markidx + 2 ..]); - const member = stack.values.items[markidx + 1]; - break :blk .{ .object = try Object.init(allocator, member, args) }; - }), - .put => |v| { - const mid = try std.fmt.parseInt(u32, v, 10); - try memo.insert(mid, try stack.pop()); - try stack.values.append(.{ .ref = mid }); - }, - .newobj => try stack.values.append(blk: { - const args = try allocator.alloc(Value, 1); - args[0] = try stack.pop(); - break :blk .{ .object = try Object.init(allocator, try stack.pop(), args) }; - }), - .empty_set => try stack.values.append(.{ .seq = .{ .type = .set, .values = &[_]Value{} } }), - .additems => { - const postmark = try stack.popMark(allocator); - defer allocator.free(postmark); - const top = try stack.lastMut(); - const rtop = try memo.resolveMut(top, true); - switch (rtop.*) { - .global => |obj| { - const obj_len = obj.args.len; - obj.args = try assuredResize(Value, allocator, obj.args, obj_len + postmark.len); - @memcpy(obj.args[obj_len..], postmark); - }, - .seq => |*tup| { - const tup_len = tup.values.len; - tup.values = try assuredResize(Value, allocator, tup.values, tup_len + postmark.len); - @memcpy(tup.values[tup_len..], postmark); - }, - else => { - return error.BadStackTopForSetItem; - }, - } - }, - .frozenset => try stack.values.append(.{ .seq = .{ .type = .frozen_set, .values = try stack.popMark(allocator) } }), - .newobj_ex => try stack.values.append(blk: { - const kwargs, const args, const cls = .{ try stack.pop(), try stack.pop(), try stack.pop() }; - const new_seq: Sequence = .{ .type = .tuple, .values = try allocator.dupe(Value, &.{ args, kwargs }) }; - break :blk .{ .object = try Object.init(allocator, cls, try allocator.dupe(Value, &.{.{ .seq = new_seq }})) }; - }), - .stack_global => try stack.values.append(blk: { - const gn, const mn = .{ - try memo.resolve(allocator, try stack.pop(), true), - try memo.resolve(allocator, try stack.pop(), true), - }; - const new_seq: Sequence = .{ .type = .tuple, .values = try allocator.dupe(Value, &.{ gn, mn }) }; - break :blk .{ .object = try Object.init(allocator, .{ .seq = new_seq }, &[_]Value{}) }; - }), - .memoize => { - const item = stack.values.getLastOrNull() orelse { - return error.StackUnderrun; - }; - try memo.insert(@intCast(memo.map.count()), try item.clone(allocator)); - }, - else => try stack.values.append(.{ .raw = try op.clone(allocator) }), - } - } - if (!resolve_refs) { - return .{ try stack.toPickleStack(), memo }; - } - return .{ - PickleStack.init(allocator, try memo.resolveAllRefsIter(allocator, 0, stack.values.items, true)), - memo, - }; -} - -// TODO: this is a unmanaged array list, minus the optimisation. We should use that instead -fn assuredResize(comptime T: type, allocator: std.mem.Allocator, old: []T, new_length: usize) ![]T { - if (allocator.resize(old, new_length)) { - return old; - } else { - defer allocator.free(old); - const new = try allocator.alloc(T, new_length); - @memcpy(new[0..old.len], old); - return new; - } -} diff --git a/zml/aio/torch/ops.zig b/zml/aio/torch/ops.zig deleted file mode 100644 index 1a00f5d1..00000000 --- a/zml/aio/torch/ops.zig +++ /dev/null @@ -1,225 +0,0 @@ -const std = @import("std"); - -/// A decoded Pickle operation in its natural state. -pub const PickleOp = union(RawPickleOp) { - mark, - stop, - pop, - pop_mark, - dup, - float: []const u8, - int: []const u8, - binint: i32, - binint1: u8, - long: []const u8, - binint2: u16, - none, - persid: []const u8, - binpersid, - reduce, - string: []const u8, - binstring: []const u8, - short_binstring: []const u8, - unicode: []const u8, - binunicode: []const u8, - append, - build, - global: PyType, - dict, - empty_dict, - appends, - get: []const u8, - binget: u8, - inst: PyType, - long_binget: u32, - list, - empty_list, - obj, - put: []const u8, - binput: u8, - long_binput: u32, - setitem, - tuple, - empty_tuple, - setitems, - binfloat: f64, - proto: u8, - newobj, - ext1: u8, - ext2: i16, - ext4: i32, - tuple1, - tuple2, - tuple3, - newtrue, - newfalse, - long1: []const u8, - long4: []const u8, - binbytes: []const u8, - short_binbytes: []const u8, - short_binunicode: []const u8, - binunicode8: []const u8, - binbytes8: []const u8, - empty_set, - additems, - frozenset, - newobj_ex, - stack_global, - memoize, - frame: u64, - bytearray8: []const u8, - next_buffer, - readonly_buffer, - - pub const PyType = struct { module: []const u8, class: []const u8 }; - - pub fn deinit(self: PickleOp, allocator: std.mem.Allocator) void { - switch (self) { - .float, - .int, - .long, - .persid, - .string, - .binstring, - .short_binstring, - .unicode, - .binunicode, - .get, - .put, - .long1, - .long4, - .binbytes, - .short_binbytes, - .short_binunicode, - .binunicode8, - .binbytes8, - .bytearray8, - => |v| allocator.free(v), - .global, .inst => |py_type| { - allocator.free(py_type.module); - allocator.free(py_type.class); - }, - else => {}, - } - } - - pub fn clone(self: PickleOp, allocator: std.mem.Allocator) !PickleOp { - var res = self; - return switch (self) { - inline .float, - .int, - .long, - .persid, - .string, - .binstring, - .short_binstring, - .unicode, - .binunicode, - .get, - .put, - .long1, - .long4, - .binbytes, - .short_binbytes, - .short_binunicode, - .binunicode8, - .binbytes8, - .bytearray8, - => |v, tag| { - const cloned = try allocator.alloc(u8, v.len); - @memcpy(cloned, v); - @field(res, @tagName(tag)) = cloned; - return res; - }, - inline .global, .inst => |v, tag| { - @field(res, @tagName(tag)) = PyType{ - .module = try allocator.dupe(u8, v.module), - .class = try allocator.dupe(u8, v.class), - }; - return res; - }, - else => self, - }; - } -}; - -/// The values for the possible opcodes are in this enum. -pub const RawPickleOp = enum(u8) { - mark = '(', // push special markobject on stack - stop = '.', // every pickle ends with stop - pop = '0', // discard topmost stack item - pop_mark = '1', // discard stack top through topmost markobject - dup = '2', // duplicate top stack item - float = 'F', // push float object; decimal string argument - int = 'I', // push integer or bool; decimal string argument - binint = 'J', // push four-byte signed int - binint1 = 'K', // push 1-byte unsigned int - long = 'L', // push long; decimal string argument - binint2 = 'M', // push 2-byte unsigned int - none = 'N', // push None - persid = 'P', // push persistent object; id is taken from string arg - binpersid = 'Q', // " " " ; " " " " stack - reduce = 'R', // apply callable to argtuple, both on stack - string = 'S', // push string; NL-terminated string argument - binstring = 'T', // push string; counted binary string argument - short_binstring = 'U', // " " ; " " " " < 256 bytes - unicode = 'V', // push Unicode string; raw-unicode-escaped'd argument - binunicode = 'X', // " " " ; counted UTF-8 string argument - append = 'a', // append stack top to list below it - build = 'b', // call __setstate__ or __dict__.update() - global = 'c', // push self.find_class(modname, name); 2 string args - dict = 'd', // build a dict from stack items - empty_dict = '}', // push empty dict - appends = 'e', // extend list on stack by topmost stack slice - get = 'g', // push item from memo on stack; index is string arg - binget = 'h', // " " " " " " ; " " 1-byte arg - inst = 'i', // build & push class instance - long_binget = 'j', // push item from memo on stack; index is 4-byte arg - list = 'l', // build list from topmost stack items - empty_list = ']', // push empty list - obj = 'o', // build & push class instance - put = 'p', // store stack top in memo; index is string arg - binput = 'q', // " " " " " ; " " 1-byte arg - long_binput = 'r', // " " " " " ; " " 4-byte arg - setitem = 's', // add key+value pair to dict - tuple = 't', // build tuple from topmost stack items - empty_tuple = ')', // push empty tuple - setitems = 'u', // modify dict by adding topmost key+value pairs - binfloat = 'G', // push float; arg is 8-byte float encoding - - // Protocol 2 - proto = '\x80', // identify pickle protocol - newobj = '\x81', // build object by applying cls.__new__ to argtuple - ext1 = '\x82', // push object from extension registry; 1-byte index - ext2 = '\x83', // ditto, but 2-byte index - ext4 = '\x84', // ditto, but 4-byte index - tuple1 = '\x85', // build 1-tuple from stack top - tuple2 = '\x86', // build 2-tuple from two topmost stack items - tuple3 = '\x87', // build 3-tuple from three topmost stack items - newtrue = '\x88', // push True - newfalse = '\x89', // push False - long1 = '\x8a', // push long from < 256 bytes - long4 = '\x8b', // push really big long - - // Protocol 3 - binbytes = 'B', // push bytes; counted binary string argument - short_binbytes = 'C', // " " ; " " " " < 256 bytes - - // Protocol 4 - short_binunicode = '\x8c', // push short string; UTF-8 length < 256 bytes - binunicode8 = '\x8d', // push very long string - binbytes8 = '\x8e', // push very long bytes string - empty_set = '\x8f', // push empty set on the stack - additems = '\x90', // modify set by adding topmost stack items - frozenset = '\x91', // build frozenset from topmost stack items - newobj_ex = '\x92', // like newobj but work with keyword only arguments - stack_global = '\x93', // same as GLOBAL but using names on the stacks - memoize = '\x94', // store top of the stack in memo - frame = '\x95', // indicate the beginning of a new frame - - // Protocol 5 - bytearray8 = '\x96', // push bytearray - next_buffer = '\x97', // push next out-of-band buffer - readonly_buffer = '\x98', // make top of stack readonly - _, -}; diff --git a/zml/aio/torch/parser.zig b/zml/aio/torch/parser.zig deleted file mode 100644 index 754b055e..00000000 --- a/zml/aio/torch/parser.zig +++ /dev/null @@ -1,475 +0,0 @@ -const asynk = @import("async"); -const std = @import("std"); -const zml = @import("../../zml.zig"); - -const utils = @import("utils.zig"); -const PickleOp = @import("ops.zig").PickleOp; -const RawPickleOp = @import("ops.zig").RawPickleOp; - -const Allocator = std.mem.Allocator; -const testing = std.testing; - -test { - std.testing.refAllDecls(@This()); -} - -pub const Decoder = struct { - buffer_file: zml.aio.MemoryMappedFile, - file_map: std.StringArrayHashMapUnmanaged(FileEntry) = .{}, - tar_file: ?TarStream = null, - ops: []PickleOp, - is_zip_file: bool, - zip_prefix: []const u8 = &[_]u8{}, - - pub const FileEntry = struct { - version_needed_to_extract: u16, - flags: u16, - compression_method: std.zip.CompressionMethod, - last_modification_time: u16, - last_modification_date: u16, - header_zip_offset: u64, - crc32: u32, - filename_len: u32, - compressed_size: u64, - uncompressed_size: u64, - file_offset: u64, - - pub fn init(entry: anytype) FileEntry { - return .{ - .version_needed_to_extract = entry.version_needed_to_extract, - .flags = @as(u16, @bitCast(entry.flags)), - .compression_method = entry.compression_method, - .last_modification_time = entry.last_modification_time, - .last_modification_date = entry.last_modification_date, - .header_zip_offset = entry.header_zip_offset, - .crc32 = entry.crc32, - .filename_len = entry.filename_len, - .compressed_size = entry.compressed_size, - .uncompressed_size = entry.uncompressed_size, - .file_offset = entry.file_offset, - }; - } - }; - - const magic = "PK\x03\x04"; - - pub fn fromTarFile(allocator: Allocator, mapped: zml.aio.MemoryMappedFile, file: std.tar.Iterator(asynk.File.Reader).File) !Decoder { - const tar_stream = try TarStream.init(file); - const file_magic = try tar_stream.reader().readBytesNoEof(magic.len); - try tar_stream.seekTo(0); - var self: Decoder = .{ - .buffer_file = mapped, - .tar_file = tar_stream, - .ops = undefined, - .is_zip_file = std.mem.eql(u8, &file_magic, magic), - }; - if (!self.is_zip_file) { - const reader = tar_stream.reader(); - self.ops = try parse(allocator, reader, try tar_stream.getEndPos()); - } else { - self.ops = try self.parseOps(allocator, self.tar_file.?.seekableStream()); - } - return self; - } - - pub fn init(allocator: Allocator, file: asynk.File) !Decoder { - const file_magic = try file.reader().readBytesNoEof(magic.len); - try file.seekTo(0); - var self: Decoder = .{ - .buffer_file = try zml.aio.MemoryMappedFile.init(file), - .is_zip_file = std.mem.eql(u8, &file_magic, magic), - .ops = undefined, - }; - if (!self.is_zip_file) { - const reader = self.buffer_file.file.reader(); - self.ops = try parse(allocator, reader, try reader.context.getEndPos()); - } else { - self.ops = try self.parseOps(allocator, self.buffer_file.file.seekableStream()); - } - return self; - } - - pub fn deinit(self: *Decoder) void { - self.buffer_file.deinit(); - self.* = undefined; - } - - fn parseOps(self: *Decoder, allocator: Allocator, seekable_stream: anytype) ![]PickleOp { - var iter = try std.zip.Iterator(@TypeOf(seekable_stream)).init(seekable_stream); - var filename_buf: [std.fs.max_path_bytes]u8 = undefined; - while (try iter.next()) |entry| { - const filename = filename_buf[0..entry.filename_len]; - try seekable_stream.seekTo(entry.header_zip_offset + @sizeOf(std.zip.CentralDirectoryFileHeader)); - const len = try seekable_stream.context.reader().readAll(filename); - if (len != filename.len) return error.ZipBadFileOffset; - if (isBadFilename(filename)) return error.ZipBadFilename; - std.mem.replaceScalar(u8, filename, '\\', '/'); // normalize path separators - try self.file_map.put(allocator, try allocator.dupe(u8, filename), FileEntry.init(entry)); - } - - var file_iter = self.file_map.iterator(); - while (file_iter.next()) |e| { - const entry = e.value_ptr.*; - const filename = e.key_ptr.*; - if (std.mem.indexOf(u8, filename, "data.pkl")) |idx| { - self.zip_prefix = filename[0..idx]; - const local_data_header_offset: u64 = local_data_header_offset: { - const local_header = blk: { - try seekable_stream.seekTo(entry.file_offset); - break :blk try seekable_stream.context.reader().readStructEndian(std.zip.LocalFileHeader, .little); - }; - if (!std.mem.eql(u8, &local_header.signature, &std.zip.local_file_header_sig)) - return error.ZipBadFileOffset; - if (local_header.version_needed_to_extract != entry.version_needed_to_extract) - return error.ZipMismatchVersionNeeded; - if (local_header.last_modification_time != entry.last_modification_time) - return error.ZipMismatchModTime; - if (local_header.last_modification_date != entry.last_modification_date) - return error.ZipMismatchModDate; - - if (@as(u16, @bitCast(local_header.flags)) != entry.flags) - return error.ZipMismatchFlags; - if (local_header.crc32 != 0 and local_header.crc32 != entry.crc32) - return error.ZipMismatchCrc32; - if (local_header.compressed_size != 0 and - local_header.compressed_size != entry.compressed_size) - return error.ZipMismatchCompLen; - if (local_header.uncompressed_size != 0 and - local_header.uncompressed_size != entry.uncompressed_size) - return error.ZipMismatchUncompLen; - if (local_header.filename_len != entry.filename_len) - return error.ZipMismatchFilenameLen; - - break :local_data_header_offset @as(u64, local_header.filename_len) + - @as(u64, local_header.extra_len); - }; - - const local_data_file_offset: u64 = - @as(u64, entry.file_offset) + - @as(u64, @sizeOf(std.zip.LocalFileHeader)) + - local_data_header_offset; - try seekable_stream.seekTo(local_data_file_offset); - - switch (entry.compression_method) { - .store => { - return parse(allocator, seekable_stream.context.reader(), entry.uncompressed_size); - }, - .deflate => { - // TODO(cryptodeal): handle decompress - @panic("TODO support use of `deflate`"); - }, - else => @panic("TODO support other modes of compression"), - } - } - } - - std.log.err("Could not find file ending in `data.pkl` in archive", .{}); - return error.PickleNotFound; - } - - fn parse(allocator: Allocator, reader: anytype, len: usize) ![]PickleOp { - var results = std.ArrayList(PickleOp).init(allocator); - errdefer results.deinit(); - outer: while (true) { - const b = try reader.readByte(); - switch (@as(RawPickleOp, @enumFromInt(b))) { - .mark => try results.append(.{ .mark = {} }), - .stop => { - try results.append(.{ .stop = {} }); - break :outer; - }, - .pop => try results.append(.{ .pop = {} }), - .pop_mark => try results.append(.{ .pop_mark = {} }), - .dup => try results.append(.{ .dup = {} }), - .float => { - const buf = try reader.readUntilDelimiterAlloc(allocator, '\n', len); - errdefer allocator.free(buf); - try results.append(.{ .float = buf }); - }, - .int => { - const buf = try reader.readUntilDelimiterAlloc(allocator, '\n', len); - errdefer allocator.free(buf); - try results.append(.{ .int = buf }); - }, - .binint => try results.append(.{ .binint = try reader.readInt(i32, .little) }), - .binint1 => try results.append(.{ .binint1 = try reader.readByte() }), - .long => { - const buf = try reader.readUntilDelimiterAlloc(allocator, '\n', len); - errdefer allocator.free(buf); - try results.append(.{ .long = buf }); - }, - .binint2 => try results.append(.{ .binint2 = try reader.readInt(u16, .little) }), - .none => try results.append(.{ .none = {} }), - .persid => { - const buf = try reader.readUntilDelimiterAlloc(allocator, '\n', len); - errdefer allocator.free(buf); - try results.append(.{ .persid = buf }); - }, - .binpersid => try results.append(.{ .binpersid = {} }), - .reduce => try results.append(.{ .reduce = {} }), - .string => { - const buf = try reader.readUntilDelimiterAlloc(allocator, '\n', len); - errdefer allocator.free(buf); - try results.append(.{ .string = buf }); - }, - .binstring => { - const str_len = try reader.readInt(u32, .little); - const buf = try allocator.alloc(u8, str_len); - errdefer allocator.free(buf); - _ = try reader.read(buf); - try results.append(.{ .binstring = buf }); - }, - .short_binstring => { - const str_len = try reader.readByte(); - const buf = try allocator.alloc(u8, str_len); - errdefer allocator.free(buf); - _ = try reader.read(buf); - try results.append(.{ .short_binstring = buf }); - }, - .unicode => { - const buf = try reader.readUntilDelimiterAlloc(allocator, '\n', len); - errdefer allocator.free(buf); - try results.append(.{ .unicode = buf }); - }, - .binunicode => { - const str_len = try reader.readInt(u32, .little); - const buf = try allocator.alloc(u8, str_len); - errdefer allocator.free(buf); - _ = try reader.read(buf); - try results.append(.{ .binunicode = buf }); - }, - .append => try results.append(.{ .append = {} }), - .build => try results.append(.{ .build = {} }), - .global, .inst => { - const module = try reader.readUntilDelimiterAlloc(allocator, '\n', len); - errdefer allocator.free(module); - const class = try reader.readUntilDelimiterAlloc(allocator, '\n', len); - errdefer allocator.free(class); - try results.append(.{ .global = .{ .module = module, .class = class } }); - }, - .dict => try results.append(.{ .dict = {} }), - .empty_dict => try results.append(.{ .empty_dict = {} }), - .appends => try results.append(.{ .appends = {} }), - .get => { - const buf = try reader.readUntilDelimiterAlloc(allocator, '\n', len); - errdefer allocator.free(buf); - try results.append(.{ .get = buf }); - }, - .binget => try results.append(.{ .binget = try reader.readByte() }), - .long_binget => try results.append(.{ .long_binget = try reader.readInt(u32, .little) }), - .list => try results.append(.{ .list = {} }), - .empty_list => try results.append(.{ .empty_list = {} }), - .obj => try results.append(.{ .obj = {} }), - .put => { - const buf = try reader.readUntilDelimiterAlloc(allocator, '\n', len); - errdefer allocator.free(buf); - try results.append(.{ .put = buf }); - }, - .binput => { - try results.append(.{ .binput = try reader.readByte() }); - }, - .long_binput => { - try results.append(.{ .long_binput = try reader.readInt(u32, .little) }); - }, - .setitem => try results.append(.{ .setitem = {} }), - .tuple => try results.append(.{ .tuple = {} }), - .empty_tuple => try results.append(.{ .empty_tuple = {} }), - .setitems => try results.append(.{ .setitems = {} }), - .binfloat => try results.append(.{ .binfloat = @bitCast(try reader.readInt(u64, .big)) }), - .proto => try results.append(.{ .proto = try reader.readByte() }), - .newobj => try results.append(.{ .newobj = {} }), - .ext1 => try results.append(.{ .ext1 = try reader.readByte() }), - .ext2 => try results.append(.{ .ext2 = try reader.readInt(i16, .little) }), - .ext4 => try results.append(.{ .ext4 = try reader.readInt(i32, .little) }), - .tuple1 => try results.append(.{ .tuple1 = {} }), - .tuple2 => try results.append(.{ .tuple2 = {} }), - .tuple3 => try results.append(.{ .tuple3 = {} }), - .newtrue => try results.append(.{ .newtrue = {} }), - .newfalse => try results.append(.{ .newfalse = {} }), - .long1 => { - const str_len = try reader.readByte(); - const buf = try allocator.alloc(u8, str_len); - errdefer allocator.free(buf); - _ = try reader.read(buf); - try results.append(.{ .long1 = buf }); - }, - .long4 => { - const str_len = try reader.readInt(u32, .little); - const buf = try allocator.alloc(u8, str_len); - errdefer allocator.free(buf); - _ = try reader.read(buf); - try results.append(.{ .long4 = buf }); - }, - .binbytes => { - const str_len = try reader.readInt(u32, .little); - const buf = try allocator.alloc(u8, str_len); - errdefer allocator.free(buf); - _ = try reader.read(buf); - try results.append(.{ .binbytes = buf }); - }, - .binbytes8 => { - const str_len = try reader.readInt(u64, .little); - const buf = try allocator.alloc(u8, str_len); - errdefer allocator.free(buf); - _ = try reader.read(buf); - try results.append(.{ .binbytes8 = buf }); - }, - .short_binbytes => { - const str_len = try reader.readByte(); - const buf = try allocator.alloc(u8, str_len); - errdefer allocator.free(buf); - _ = try reader.read(buf); - try results.append(.{ .short_binbytes = buf }); - }, - .binunicode8 => { - const str_len = try reader.readInt(u64, .little); - const buf = try allocator.alloc(u8, str_len); - errdefer allocator.free(buf); - _ = try reader.read(buf); - try results.append(.{ .binunicode8 = buf }); - }, - .short_binunicode => { - const str_len = try reader.readByte(); - const buf = try allocator.alloc(u8, str_len); - errdefer allocator.free(buf); - _ = try reader.read(buf); - try results.append(.{ .binunicode8 = buf }); - }, - .empty_set => try results.append(.{ .empty_set = {} }), - .additems => try results.append(.{ .additems = {} }), - .frozenset => try results.append(.{ .frozenset = {} }), - .newobj_ex => try results.append(.{ .newobj_ex = {} }), - .stack_global => try results.append(.{ .stack_global = {} }), - .memoize => try results.append(.{ .memoize = {} }), - .frame => try results.append(.{ .frame = try reader.readInt(u64, .little) }), - .bytearray8 => { - const str_len = try reader.readInt(u64, .little); - const buf = try allocator.alloc(u8, str_len); - errdefer allocator.free(buf); - _ = try reader.read(buf); - try results.append(.{ .bytearray8 = buf }); - }, - .next_buffer => try results.append(.{ .next_buffer = {} }), - .readonly_buffer => try results.append(.{ .readonly_buffer = {} }), - else => {}, - } - } - return results.toOwnedSlice(); - } -}; - -const TarStream = struct { - pub const SeekableStream = std.io.SeekableStream( - TarStream, - asynk.File.SeekError, - asynk.File.GetSeekPosError, - TarStream.seekTo, - TarStream.seekBy, - TarStream.getPos, - TarStream.getEndPos, - ); - - file: std.tar.Iterator(asynk.File.Reader).File, - start: usize, - - pub fn init(file: std.tar.Iterator(asynk.File.Reader).File) !TarStream { - return .{ - .file = file, - .start = try file.parent_reader.context.getPos(), - }; - } - - pub fn reader(file: TarStream) std.tar.Iterator(asynk.File.Reader).File.Reader { - return file.file.reader(); - } - - pub fn seekTo(self: TarStream, offset: u64) !void { - return self.file.parent_reader.context.seekTo(self.start + offset); - } - - pub fn seekBy(self: TarStream, offset: i64) !void { - return self.file.parent_reader.context.seekBy(offset); - } - - pub fn getPos(self: TarStream) !u64 { - return try self.file.parent_reader.context.getPos() - self.start; - } - - pub fn getEndPos(self: TarStream) !u64 { - return self.file.size; - } - - pub fn seekableStream(self: TarStream) TarStream.SeekableStream { - return .{ .context = self }; - } -}; - -test "Read pickle (simple)" { - const Value = @import("value.zig").Value; - var arena = std.heap.ArenaAllocator.init(testing.allocator); - defer arena.deinit(); - const allocator = arena.allocator(); - const eval = @import("eval.zig"); - const file = try asynk.File.open("zml/aio/torch/simple_test.pickle", .{ .mode = .read_only }); - var data = try Decoder.init(allocator, file); - defer data.deinit(); - var vals, var memo = try eval.evaluate(allocator, data.ops, true); - defer vals.deinit(); - defer memo.deinit(); - - try testing.expect(vals.stack.len == 2); - // skip first value (frame) - try testing.expect(vals.stack[1] == .seq); - try testing.expect(vals.stack[1].seq.type == .dict); - const entries = vals.stack[1].seq.values[0].seq.values; - try testing.expect(entries.len == 5); - const expected: []const Value = &.{ - .{ .seq = .{ .type = .kv_tuple, .values = @constCast(@as([]const Value, &.{ .{ .string = "hello" }, .{ .string = "world" } })) } }, - .{ .seq = .{ .type = .kv_tuple, .values = @constCast(@as([]const Value, &.{ .{ .string = "int" }, .{ .int64 = 1 } })) } }, - .{ .seq = .{ .type = .kv_tuple, .values = @constCast(@as([]const Value, &.{ .{ .string = "float" }, .{ .float64 = 3.141592 } })) } }, - .{ .seq = .{ .type = .kv_tuple, .values = @constCast(@as([]const Value, &.{ - .{ .string = "list" }, - .{ .seq = .{ .type = .list, .values = @constCast(@as([]const Value, &.{ - .{ .int64 = 0 }, - .{ .int64 = 1 }, - .{ .int64 = 2 }, - .{ .int64 = 3 }, - .{ .int64 = 4 }, - })) } }, - })) } }, - .{ .seq = .{ .type = .kv_tuple, .values = @constCast(@as([]const Value, &.{ - .{ .string = "tuple" }, - .{ .seq = .{ - .type = .tuple, - .values = @constCast(@as([]const Value, &.{ - .{ .string = "a" }, - .{ .int64 = 10 }, - })), - } }, - })) } }, - }; - - try std.testing.expectEqualDeep(expected, entries); -} - -test "Read pickle (zipped)" { - var arena = std.heap.ArenaAllocator.init(testing.allocator); - defer arena.deinit(); - const allocator = arena.allocator(); - const file = try asynk.File.open("zml/aio/torch/simple.pt", .{ .mode = .read_only }); - var data = try Decoder.init(allocator, file); - defer data.deinit(); -} - -pub fn isBadFilename(filename: []const u8) bool { - if (filename.len == 0 or filename[0] == '/') - return true; - - var it = std.mem.splitScalar(u8, filename, '/'); - while (it.next()) |part| { - if (std.mem.eql(u8, part, "..")) - return true; - } - - return false; -} diff --git a/zml/aio/torch/simple.pt b/zml/aio/torch/simple.pt deleted file mode 100644 index beeb39db..00000000 Binary files a/zml/aio/torch/simple.pt and /dev/null differ diff --git a/zml/aio/torch/simple_test.pickle b/zml/aio/torch/simple_test.pickle deleted file mode 100644 index 19d72601..00000000 Binary files a/zml/aio/torch/simple_test.pickle and /dev/null differ diff --git a/zml/aio/torch/utils.zig b/zml/aio/torch/utils.zig deleted file mode 100644 index 2b573dd7..00000000 --- a/zml/aio/torch/utils.zig +++ /dev/null @@ -1,23 +0,0 @@ -const std = @import("std"); - -const Value = @import("value.zig").Value; - -pub fn allTrue(values: []const Value, func: fn (v: Value) bool) bool { - for (values) |v| { - if (!func(v)) return false; - } - return true; -} - -pub fn isBadFilename(filename: []const u8) bool { - if (filename.len == 0 or filename[0] == '/') - return true; - - var it = std.mem.splitScalar(u8, filename, '/'); - while (it.next()) |part| { - if (std.mem.eql(u8, part, "..")) - return true; - } - - return false; -} diff --git a/zml/aio/torch/value.zig b/zml/aio/torch/value.zig deleted file mode 100644 index 9c6acc4c..00000000 --- a/zml/aio/torch/value.zig +++ /dev/null @@ -1,404 +0,0 @@ -const std = @import("std"); -const utils = @import("utils.zig"); - -const PickleOp = @import("ops.zig").PickleOp; - -const big_int = std.math.big.int; - -/// The types of sequences that exist. -pub const SequenceType = enum { - list, - dict, - kv_tuple, - tuple, - set, - frozen_set, -}; - -pub const Object = struct { - allocator: std.mem.Allocator, - member: Value, - args: []Value, - - pub fn init(allocator: std.mem.Allocator, member: Value, args: []Value) !*Object { - const self = try allocator.create(Object); - self.* = .{ .allocator = allocator, .member = member, .args = args }; - return self; - } - - pub fn clone(self: *Object, allocator: std.mem.Allocator) std.mem.Allocator.Error!*Object { - const res = try allocator.create(Object); - res.* = .{ .allocator = allocator, .member = try self.member.clone(allocator), .args = try allocator.alloc(Value, self.args.len) }; - for (self.args, 0..) |v, i| res.args[i] = try v.clone(allocator); - return res; - } - - pub fn deinit(self: *Object) void { - self.member.deinit(self.allocator); - for (self.args) |*v| v.deinit(self.allocator); - self.allocator.free(self.args); - self.allocator.destroy(self); - } -}; - -pub const Build = struct { - allocator: std.mem.Allocator, - member: Value, - args: Value, - - pub fn init(allocator: std.mem.Allocator, member: Value, args: Value) !*Build { - const self = try allocator.create(Build); - self.* = .{ .allocator = allocator, .member = member, .args = args }; - return self; - } - - pub fn clone(self: *Build, allocator: std.mem.Allocator) std.mem.Allocator.Error!*Build { - const res = try allocator.create(Build); - res.* = .{ .allocator = allocator, .member = try self.member.clone(allocator), .args = try self.args.clone(allocator) }; - return res; - } - - pub fn deinit(self: *Build) void { - self.member.deinit(self.allocator); - self.args.deinit(self.allocator); - self.allocator.destroy(self); - } -}; - -pub const Sequence = struct { - type: SequenceType, - values: []Value, -}; - -pub const PersId = struct { - allocator: std.mem.Allocator, - ref: Value, - - pub fn init(allocator: std.mem.Allocator, ref: Value) !*PersId { - const self = try allocator.create(PersId); - self.* = .{ .allocator = allocator, .ref = ref }; - return self; - } - - pub fn clone(self: *PersId, allocator: std.mem.Allocator) std.mem.Allocator.Error!*PersId { - const res = try allocator.create(PersId); - res.* = .{ .allocator = allocator, .ref = try self.ref.clone(allocator) }; - return res; - } - - pub fn deinit(self: *PersId) void { - self.ref.deinit(self.allocator); - self.allocator.destroy(self); - } -}; - -pub const ValueType = enum { - raw, - ref, - app, - object, - build, - pers_id, - global, - seq, - string, - bytes, - int64, - bigint, - float64, - raw_num, - boolval, - none, -}; - -/// A processed value. -pub const Value = union(ValueType) { - /// Types that we can't handle or just had to give up on processing. - raw: PickleOp, - - /// A reference. You might be able to look it up in the memo map - /// unless there's something weird going on like recursive references. - /// You generally shouldn't see this in the result unless bad things - /// are going on... - ref: u32, - - /// The result of applying a thing to another thing. We're not - /// Python so we don't really know what a "thing" is. - app: *Object, - - /// An object or something. The first tuple member is the - /// thing, the second one is the arguments it got applied to. - object: *Object, - - /// Something we tried to build. The first tuple member is the - /// thing, the second one is the arguments it got applied to. - build: *Build, - - /// References to persistant storage. They basically could be anything. - /// You kind of have to know what the thing you're trying to - /// interface wants to use as keys for persistant storage. - /// Good luck. - pers_id: *PersId, - - /// A global value of some kind. The first tuple member is - /// the thing, the second one is the arguments it got applied to. - global: *Object, - - /// A sequence. We don't really distinguish between them - /// much. The one exception is when the SequenceType is - /// Dict we try to split the flat list of `[k, v, k, v, k, v]` - /// into a list of tuples with the key and value. - seq: Sequence, - - /// A string, but not the crazy strings that have to be - /// unescaped as if they were Python strings. If you - /// need one of those, look for it inside a `Value.raw`. - string: []const u8, - - /// Some bytes. It might be a byte array or a binary - /// string that couldn't get UTF8 decoded. We do the best - /// we can. - bytes: []const u8, - - /// An integer, but not the crazy kind that comes as a string - /// that has to be parsed. You can look in `Value.raw_num` for - /// those. - int64: i64, - - /// An integer that can't fit in i64. - bigint: big_int.Managed, - - /// An float, but not the crazy kind that comes as a string - /// that has to be parsed. You can look in `Value.raw_num` for - /// those. - float64: f64, - - /// Some kind of weird number we can't handle. - raw_num: PickleOp, - - /// A boolean value. - boolval: bool, - - /// Python `None`. - none: void, - - pub fn deinit(self: *Value, allocator: std.mem.Allocator) void { - switch (self.*) { - .raw, .raw_num => |v| v.deinit(allocator), - inline .app, .object, .global, .build, .pers_id => |v| v.deinit(), - .seq => |v| { - for (v.values) |*val| val.deinit(allocator); - allocator.free(v.values); - }, - .string, .bytes => |v| allocator.free(v), - .bigint => self.bigint.deinit(), - else => {}, - } - self.* = undefined; - } - - inline fn writeIndents(indents: usize, writer: anytype) !void { - try writer.writeBytesNTimes(" ", indents); // resolve tab = 2 spaces - // try writer.writeByteNTimes('\t'); - } - - fn internalFormat(value: Value, indents: usize, writer: anytype) !void { - try writeIndents(indents, writer); - try writer.writeAll(".{\n"); - try writeIndents(indents + 1, writer); - try writer.print(".{s} = ", .{@tagName(std.meta.activeTag(value))}); - switch (value) { - inline .ref, .int64, .float64 => |v| try writer.print("{d} ", .{v}), - .app, .object, .global => |v| { - try writer.writeAll(".{\n"); - try internalFormat(v.member, indents + 2, writer); - try writer.writeAll(",\n"); - try writeIndents(indents + 2, writer); - if (v.args.len > 0) { - try writer.writeAll(".{\n"); - for (v.args, 0..) |arg, i| { - try internalFormat(arg, indents + 3, writer); - if (i < v.args.len - 1) try writer.writeAll(","); - try writer.writeByte('\n'); - } - try writeIndents(indents + 2, writer); - try writer.writeAll("}\n"); - } else { - try writer.writeAll(".{}\n"); - } - try writeIndents(indents + 1, writer); - try writer.writeAll("}"); - }, - .build => |v| { - try writer.writeAll(".{\n"); - try internalFormat(v.member, indents + 2, writer); - try writer.writeAll(",\n"); - try internalFormat(v.args, indents + 2, writer); - try writer.writeAll(",\n"); - try writeIndents(indents + 1, writer); - try writer.writeAll("}"); - }, - inline .pers_id => |v| { - try writer.writeByte('\n'); - try internalFormat(v.ref, indents + 2, writer); - }, - .seq => |v| { - try writer.writeAll(".{\n"); - try writeIndents(indents + 2, writer); - try writer.print(".{s},\n", .{@tagName(v.type)}); - try writeIndents(indents + 2, writer); - if (v.values.len > 0) { - try writer.writeAll(".{\n"); - for (v.values, 0..) |arg, i| { - try internalFormat(arg, indents + 3, writer); - if (i < v.values.len - 1) try writer.writeAll(","); - try writer.writeByte('\n'); - } - try writeIndents(indents + 2, writer); - try writer.writeAll("}\n"); - } else { - try writer.writeAll(".{}\n"); - } - - try writeIndents(indents + 1, writer); - try writer.writeAll("}"); - }, - .string => |v| try writer.print("\"{s}\"", .{v}), - .raw => |v| switch (v) { - .global => |py_type| try writer.print("\"{s}\", \"{s}\"", .{ py_type.module, py_type.class }), - else => try writer.print("{any}", .{v}), - }, - inline else => |v| { - try writer.print("{any}", .{v}); - }, - } - try writer.writeByte('\n'); - try writeIndents(indents, writer); - try writer.writeByte('}'); - } - - pub fn format(self: Value, comptime _: []const u8, _: std.fmt.FormatOptions, writer: anytype) !void { - return internalFormat(self, 0, writer); - } - - pub fn clone(self: Value, allocator: std.mem.Allocator) !Value { - return switch (self) { - inline .raw, .raw_num => |v, tag| @unionInit(Value, @tagName(tag), try v.clone(allocator)), - inline .app, .object, .global, .build, .pers_id => |v, tag| @unionInit(Value, @tagName(tag), try v.clone(allocator)), - .seq => |seq| blk: { - const new_val: Sequence = .{ .type = seq.type, .values = try allocator.alloc(Value, seq.values.len) }; - for (seq.values, 0..) |v, i| new_val.values[i] = try v.clone(allocator); - break :blk .{ .seq = new_val }; - }, - inline .string, .bytes => |v, tag| @unionInit(Value, @tagName(tag), try allocator.dupe(u8, v)), - .bigint => |v| .{ .bigint = try v.clone() }, - else => self, - }; - } - - pub fn isPrimitive(self: Value) bool { - return switch (self) { - .int64, .bigint, .float64, .string, .bytes, .boolval, .none => true, - .seq => |seq| utils.allTrue(seq.values, Value.isPrimitive), - else => false, - }; - } - - pub fn containsRef(self: Value) bool { - switch (self) { - .ref => return true, - .app, .object, .global => |v| { - if (v.member.containsRef()) return true; - for (v.args) |arg| if (arg.containsRef()) return true; - return false; - }, - .build => |v| { - if (v.member.containsRef()) return true; - if (v.args.containsRef()) return true; - return false; - }, - .pers_id => |v| return v.ref.containsRef(), - .seq => |v| { - for (v.values) |val| if (val.containsRef()) return true; - return false; - }, - else => return false, - } - } - - const BI64MIN = big_int.Const{ - .limbs = &.{@intCast(@abs(std.math.minInt(i64)))}, - .positive = false, - }; - - const BI64MAX = big_int.Const{ - .limbs = &.{@intCast(std.math.maxInt(i64))}, - .positive = true, - }; - - pub fn coerceFromRaw(self: Value, allocator: std.mem.Allocator) !Value { - return switch (self) { - .raw => |raw_val| switch (raw_val) { - .binint, .binint1, .binint2 => |val| .{ .int64 = val }, - .long1, .long4 => |b| if (b.len != 0) { - var bint = try big_int.Managed.initCapacity(allocator, std.math.big.int.calcTwosCompLimbCount(b.len)); - var mutable = bint.toMutable(); - mutable.readTwosComplement(b, b.len, .little, .signed); - const min_comp = bint.toConst().order(BI64MIN); - const max_comp = bint.toConst().order(BI64MAX); - if ((min_comp == .gt or min_comp == .eq) and (max_comp == .lt or max_comp == .eq)) { - defer bint.deinit(); - return .{ .int64 = try bint.to(i64) }; - } else return .{ .bigint = bint }; - } else .{ .raw_num = raw_val }, - .binfloat => |val| .{ .float64 = val }, - .binunicode, .binunicode8, .short_binunicode => |s| .{ .string = s }, - .binbytes, .binbytes8, .short_binbytes, .bytearray8 => |b| .{ .bytes = b }, - // This isn't how Pickle actually works but we just try to UTF8 decode the - // string and if it fails, we make it a bytes value instead. If anyone - // actually cares they can just fix values themselves or recover the raw bytes - // from the UTF8 string (it's guaranteed to be reversible, as far as I know). - .binstring, .short_binstring => |b| if (std.unicode.utf8ValidateSlice(b)) .{ .string = b } else .{ .bytes = b }, - .newtrue => .{ .boolval = true }, - .newfalse => .{ .boolval = false }, - .none => .{ .none = {} }, - inline .int, - .float, - .long, - => |v, tag| { - if (tag == .int and std.mem.eql(u8, v, "01")) { - return .{ .boolval = true }; - } else if (tag == .int and std.mem.eql(u8, v, "00")) { - return .{ .boolval = false }; - } else { - return .{ .raw_num = raw_val }; - } - }, - else => self, - }, - .app, .object, .global => |v| blk: { - v.member = try v.member.coerceFromRaw(allocator); - for (v.args) |*arg| { - arg.* = try arg.coerceFromRaw(allocator); - } - break :blk self; - }, - .build => |v| blk: { - v.member = try v.member.coerceFromRaw(allocator); - v.args = try v.args.coerceFromRaw(allocator); - break :blk self; - }, - .pers_id => |v| blk: { - v.ref = try v.ref.coerceFromRaw(allocator); - break :blk self; - }, - .seq => |v| blk: { - for (v.values) |*val| { - val.* = try val.coerceFromRaw(allocator); - } - break :blk self; - }, - else => self, - }; - } -}; diff --git a/zml/aio/utils.zig b/zml/aio/utils.zig deleted file mode 100644 index 0bf1043c..00000000 --- a/zml/aio/utils.zig +++ /dev/null @@ -1,7 +0,0 @@ -pub fn toVoidSlice(data: anytype) []void { - const info = @typeInfo(@TypeOf(data)); - if (info != .Pointer or info.Pointer.size != .Slice) { - @compileError("toVoidSlice expects a slice"); - } - return @as([*]void, @ptrCast(@alignCast(data.ptr)))[0..data.len]; -} diff --git a/zml/aio/value.zig b/zml/aio/value.zig deleted file mode 100644 index 7cc4d2f3..00000000 --- a/zml/aio/value.zig +++ /dev/null @@ -1,262 +0,0 @@ -const std = @import("std"); - -pub const Value = union(enum) { - pub const Slice = struct { - pub const ItemType = enum { - uint8, - int8, - uint16, - int16, - uint32, - int32, - uint64, - int64, - float16, - float32, - float64, - boolval, - string, - // TODO (cryptodeal): gguf/torch/json (safetensors) in theory support nested arrays; - // we should support for the sake of completeness, but we have not yet encountered - // a model containing these types. - // TODO (cryptodeal): array, - }; - - item_type: ItemType, - data: []u8, - - fn isNestedSlice(comptime T: type) bool { - const info = @typeInfo(T); - if (info != .Pointer or info.Pointer.size != .Slice) { - return false; - } - - var child_info = @typeInfo(info.Pointer.child); - while (child_info == .Pointer and child_info.Pointer.size == .Slice) : (child_info = @typeInfo(child_info.Pointer.child)) {} - return switch (@TypeOf(child_info)) {}; - } - - fn fromZigType(comptime T: type) ItemType { - return switch (T) { - u8 => .uint8, - i8 => .int8, - u16 => .uint16, - i16 => .int16, - u32 => .uint32, - i32 => .int32, - u64 => .uint64, - i64 => .int64, - f16 => .float16, - f32 => .float32, - f64 => .float64, - bool => .boolval, - []const u8 => .string, - else => @panic("Unsupported type for LoaderValue.Slice: " ++ @typeName(T)), - }; - } - - pub fn toZigType(comptime kind: ItemType) type { - return switch (kind) { - .uint8 => u8, - .int8 => i8, - .uint16 => u16, - .int16 => i16, - .uint32 => u32, - .int32 => i32, - .uint64 => u64, - .int64 => i64, - .float16 => f16, - .float32 => f32, - .float64 => f64, - .boolval => bool, - .string => []const u8, - }; - } - - pub fn cast(self: *Slice, comptime T: type) []T { - if (fromZigType(T) != self.item_type) { - @panic("Type mismatch in LoaderValue.Slice cast"); - } - return @as([*]T, @ptrCast(@alignCast(self.data.ptr)))[0..self.data.len]; - } - }; - - // TODO: this is overkill we don't need that many different types - // to represent metadata. bool, i64, f64, string are enough (like Json). - null, - uint8: u8, - int8: i8, - uint16: u16, - int16: i16, - uint32: u32, - int32: i32, - uint64: u64, - int64: i64, - float16: f16, - float32: f32, - float64: f64, - bigint: std.math.big.int.Managed, - boolval: bool, - array: Slice, - string: []const u8, - - pub fn wrap(x: anytype) Value { - const tag = switch (@TypeOf(x)) { - u8 => .uint8, - i8 => .int8, - u16 => .uint16, - i16 => .int16, - u32 => .uint32, - i32 => .int32, - u64 => .uint64, - i64 => .int64, - f16 => .float16, - f32 => .float32, - f64 => .float64, - bool => .boolval, - []const u8 => .string, - else => @panic("Unsupported type for zml.aio.Value: " ++ @typeName(@TypeOf(x))), - }; - return @unionInit(Value, @tagName(tag), x); - } - - pub fn @"null"(self: Value) error{IncorrectType}!void { - switch (self) { - .null => {}, - inline else => |v| { - std.log.err("Expected `null`, actual value is {s}\n", .{@typeName(@TypeOf(v))}); - return error.IncorrectType; - }, - } - } - - pub fn @"u8"(self: Value) error{IncorrectType}!u8 { - switch (self) { - .uint8 => |v| return v, - inline else => |v| { - std.log.err("Expected `u8`, actual value is {s}\n", .{@typeName(@TypeOf(v))}); - return error.IncorrectType; - }, - } - } - - pub fn @"i8"(self: Value) error{IncorrectType}!i8 { - switch (self) { - .int8 => |v| return v, - inline else => |v| { - std.log.err("Expected `i8`, actual value is {s}\n", .{@typeName(@TypeOf(v))}); - return error.IncorrectType; - }, - } - } - - pub fn @"u16"(self: Value) error{IncorrectType}!u16 { - switch (self) { - .uint16 => |v| return v, - inline else => |v| { - std.log.err("Expected `u16`, actual value is {s}\n", .{@typeName(@TypeOf(v))}); - return error.IncorrectType; - }, - } - } - - pub fn @"i16"(self: Value) error{IncorrectType}!i16 { - switch (self) { - .int16 => |v| return v, - inline else => |v| { - std.log.err("Expected `i16`, actual value is {s}\n", .{@typeName(@TypeOf(v))}); - return error.IncorrectType; - }, - } - } - - pub fn @"u32"(self: Value) error{IncorrectType}!u32 { - switch (self) { - .uint32 => |v| return v, - inline else => |v| { - std.log.err("Expected `u32`, actual value is {s}\n", .{@typeName(@TypeOf(v))}); - return error.IncorrectType; - }, - } - } - - pub fn @"i32"(self: Value) error{IncorrectType}!i32 { - switch (self) { - .int32 => |v| return v, - inline else => |v| { - std.log.err("Expected `i32`, actual value is {s}\n", .{@typeName(@TypeOf(v))}); - return error.IncorrectType; - }, - } - } - - pub fn @"u64"(self: Value) error{IncorrectType}!u64 { - switch (self) { - .uint64 => |v| return v, - inline else => |v| { - std.log.err("Expected `u64`, actual value is {s}\n", .{@typeName(@TypeOf(v))}); - return error.IncorrectType; - }, - } - } - - pub fn @"i64"(self: Value) error{IncorrectType}!i64 { - switch (self) { - .int64 => |v| return v, - inline else => |v| { - std.log.err("Expected `i64`, actual value is {s}\n", .{@typeName(@TypeOf(v))}); - return error.IncorrectType; - }, - } - } - - pub fn @"f16"(self: Value) error{IncorrectType}!f16 { - switch (self) { - .float16 => |v| return v, - inline else => |v| { - std.log.err("Expected `f16`, actual value is {s}\n", .{@typeName(@TypeOf(v))}); - return error.IncorrectType; - }, - } - } - - pub fn @"f32"(self: Value) error{IncorrectType}!f32 { - switch (self) { - .float32 => |v| return v, - inline else => |v| { - std.log.err("Expected `f32`, actual value is {s}\n", .{@typeName(@TypeOf(v))}); - return error.IncorrectType; - }, - } - } - - pub fn @"f64"(self: Value) error{IncorrectType}!f64 { - switch (self) { - .float64 => |v| return v, - inline else => |v| { - std.log.err("Expected `f64`, actual value is {s}\n", .{@typeName(@TypeOf(v))}); - return error.IncorrectType; - }, - } - } - - pub fn @"bool"(self: Value) error{IncorrectType}!bool { - switch (self) { - .boolval => |v| return v, - inline else => |v| { - std.log.err("Expected `bool`, actual value is {s}\n", .{@typeName(@TypeOf(v))}); - return error.IncorrectType; - }, - } - } - - pub fn string(self: Value) error{IncorrectType}![]const u8 { - switch (self) { - .string => |v| return v, - inline else => |v| { - std.log.err("Expected string (`[]const u8`), actual value is {s}\n", .{@typeName(@TypeOf(v))}); - return error.IncorrectType; - }, - } - } -}; diff --git a/zml/aio/yaml.zig b/zml/aio/yaml.zig deleted file mode 100644 index 8ceffe04..00000000 --- a/zml/aio/yaml.zig +++ /dev/null @@ -1,94 +0,0 @@ -const std = @import("std"); -const utils = @import("utils.zig"); -const yaml = @import("zig-yaml"); -const zml = @import("../zml.zig"); - -const Allocator = std.mem.Allocator; - -const StringBuilder = std.ArrayListUnmanaged(u8); - -pub fn open(allocator: Allocator, path: []const u8) !zml.aio.BufferStore { - const file = try std.fs.cwd().openFile(path, .{}); - defer file.close(); - var res: zml.aio.BufferStore = .{ - .arena = std.heap.ArenaAllocator.init(allocator), - }; - errdefer res.arena.deinit(); - const arena = res.arena.allocator(); - - const yaml_data = try file.reader().readAllAlloc(arena, (try file.metadata()).size()); - const parsed = try yaml.Yaml.load(arena, yaml_data); - - var prefix_buf: [1024]u8 = undefined; - try parseMetadata(arena, &res, StringBuilder.initBuffer(&prefix_buf), parsed.docs.items[0]); - return res; -} - -pub fn parseMetadata(allocator: Allocator, store: *zml.aio.BufferStore, key: StringBuilder, val: yaml.Value) !void { - switch (val) { - .int => |v| try store._metadata.put(allocator, try allocator.dupe(u8, key.items), .{ .int64 = v }), - .float => |v| try store._metadata.put(allocator, try allocator.dupe(u8, key.items), .{ .float64 = v }), - .string => |v| try store._metadata.put(allocator, try allocator.dupe(u8, key.items), .{ .string = v }), - .list => |v| switch (validSlice(v)) { - true => { - if (v.len == 0) return; - switch (v[0]) { - .int => { - const values = try allocator.alloc(i64, v.len); - errdefer allocator.free(values); - for (v, 0..) |item, i| values[i] = item.int; - try store._metadata.put(allocator, try allocator.dupe(u8, key.items), .{ .array = .{ .item_type = .int64, .data = std.mem.sliceAsBytes(values) } }); - }, - .float => { - const values = try allocator.alloc(f64, v.len); - errdefer allocator.free(values); - for (v, 0..) |item, i| values[i] = item.float; - try store._metadata.put(allocator, try allocator.dupe(u8, key.items), .{ .array = .{ .item_type = .float64, .data = std.mem.sliceAsBytes(values) } }); - }, - .string => { - const values = try allocator.alloc([]const u8, v.len); - errdefer allocator.free(values); - for (v, 0..) |item, i| { - values[i] = try allocator.dupe(u8, item.string); - } - try store._metadata.put(allocator, try allocator.dupe(u8, key.items), .{ .array = .{ .item_type = .string, .data = std.mem.sliceAsBytes(values) } }); - }, - .list => unreachable, - else => {}, - } - }, - false => for (v, 0..) |item, i| { - var new_key = key; - if (key.items.len > 0) - new_key.appendAssumeCapacity('.'); - new_key.items.len += std.fmt.formatIntBuf(new_key.unusedCapacitySlice(), i, 10, .lower, .{}); - try parseMetadata(allocator, store, new_key, item); - }, - }, - .map => { - var map_iter = val.map.iterator(); - while (map_iter.next()) |entry| { - var new_prefix = key; - if (key.items.len > 0) - new_prefix.appendAssumeCapacity('.'); - new_prefix.appendSliceAssumeCapacity(entry.key_ptr.*); - try parseMetadata(allocator, store, new_prefix, entry.value_ptr.*); - } - }, - else => {}, - } -} - -fn validSlice(v: []yaml.Value) bool { - if (v.len == 0) return false; - const item_type = std.meta.activeTag(v[0]); - switch (item_type) { - .empty, .list, .map => return false, - else => {}, - } - - for (v[1..]) |item| - if (item_type != std.meta.activeTag(item)) return false; - - return true; -} diff --git a/zml/attention/Options.zig b/zml/attention/Options.zig new file mode 100644 index 00000000..539f753f --- /dev/null +++ b/zml/attention/Options.zig @@ -0,0 +1,3 @@ +/// Query j sees the keys up to token_index + j, as in a decoder. Otherwise every query sees every +/// key, as in an encoder or a diffusion transformer, and token_index is ignored. +causal: bool = true, diff --git a/zml/attention/attention.zig b/zml/attention/attention.zig new file mode 100644 index 00000000..edd9b4ca --- /dev/null +++ b/zml/attention/attention.zig @@ -0,0 +1,172 @@ +const std = @import("std"); + +const zml = @import("../zml.zig"); +const flash_attention = @import("../kernels/flash_attention/flash_attention.zig"); + +test { + std.testing.refAllDecls(@This()); +} + +pub const BackendType = enum { + stablehlo, + flash_attention, +}; + +pub const Backend = union(BackendType) { + stablehlo: void, + flash_attention: flash_attention.Backend, + + pub fn auto(platform: *const zml.Platform) Backend { + return if (flash_attention.isAvailable(platform)) + .{ .flash_attention = flash_attention.Backend.auto(platform) } + else + .{ .stablehlo = {} }; + } + + pub fn isAvailable(backend: Backend, platform: *const zml.Platform) bool { + return switch (backend) { + .stablehlo => true, + .flash_attention => |b| b.isAvailable(platform), + }; + } +}; + +pub const Parameters = union(BackendType) { + stablehlo: void, + flash_attention: flash_attention.Parameters, + + pub const InitOptions = union(BackendType) { + stablehlo: void, + flash_attention: flash_attention.Parameters.InitOptions, + + pub fn fromBackend(backend: Backend) InitOptions { + return switch (backend) { + .stablehlo => .{ .stablehlo = {} }, + .flash_attention => |b| .{ .flash_attention = .fromBackend(b) }, + }; + } + }; + + pub fn init(opts: InitOptions) Parameters { + return switch (opts) { + .stablehlo => .{ .stablehlo = {} }, + .flash_attention => |v| .{ .flash_attention = .init(v) }, + }; + } +}; + +pub const Metadata = union(BackendType) { + stablehlo: void, + flash_attention: flash_attention.Metadata, + + pub const InitOptions = union(BackendType) { + stablehlo: void, + flash_attention: flash_attention.Metadata.InitOptions, + + pub fn fromBackend(backend: Backend) InitOptions { + return switch (backend) { + .stablehlo => .{ .stablehlo = {} }, + .flash_attention => |b| .{ .flash_attention = .fromBackend(b) }, + }; + } + }; + + pub fn init(opts: InitOptions) Metadata { + return switch (opts) { + .stablehlo => .{ .stablehlo = {} }, + .flash_attention => |v| .{ .flash_attention = .init(v) }, + }; + } + + pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(Metadata) { + return switch (self) { + .stablehlo => .{ .stablehlo = {} }, + .flash_attention => |v| .{ .flash_attention = try v.initBuffer(io, platform) }, + }; + } + + pub fn deinitBuffer(self: *zml.Bufferized(Metadata)) void { + switch (self.*) { + .stablehlo => {}, + .flash_attention => |*v| flash_attention.Metadata.deinitBuffer(v), + } + } +}; + +pub const Options = @import("Options.zig"); + +pub const Partitioning = struct { + const Logical = zml.Sharding.Partitioning.Logical; + + in_specs: struct { + q: Logical, + k: Logical, + v: Logical, + token_index: Logical = .replicated, + /// Omit to replicate backend metadata tensors. + metadata: ?zml.ops.PartitionSpecs(Metadata) = null, + }, + out_specs: Logical, + + /// Shards the `.h` dimension of Q, K, V and the output over `mesh_axis`. + /// Other dimensions, token indices and backend metadata are replicated. + pub fn head(comptime mesh_axis: @EnumLiteral()) Partitioning { + const heads: Logical = .parse(.{ .h = mesh_axis }); + return .{ .in_specs = .{ .q = heads, .k = heads, .v = heads }, .out_specs = heads }; + } +}; + +/// Causal attention as used in transformers. +/// +/// **Shapes**: +/// - q, result: .{ .q, .h, .hd } +/// - k, v: .{ .k, .h, .hd } +/// +/// Where: +/// - .h is the number of head +/// - .q is the number of queries +/// - .k is the number of keys +/// - .hd is the head dimension +pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters, mesh: *const zml.Mesh, partitioning: Partitioning) zml.Tensor { + return attentionOpts(q, k, v, token_index, metadata, parameters, .{}, mesh, partitioning); +} + +/// `attention`, causal or not depending on `opts`. +pub fn attentionOpts(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters, opts: Options, mesh: *const zml.Mesh, partitioning: Partitioning) zml.Tensor { + const args: Attention = .{ .q = q, .k = k, .v = v, .token_index = token_index, .metadata = metadata, .parameters = parameters, .opts = opts }; + return zml.ops.manualComputation(Attention.forward, args, q.shape(), mesh, .{ + .in_specs = .{ + .q = partitioning.in_specs.q, + .k = partitioning.in_specs.k, + .v = partitioning.in_specs.v, + .token_index = partitioning.in_specs.token_index, + .metadata = partitioning.in_specs.metadata orelse zml.ops.partitionSpecs(metadata, .replicated), + }, + .out_specs = partitioning.out_specs, + }); +} + +const Attention = struct { + q: zml.Tensor, + k: zml.Tensor, + v: zml.Tensor, + token_index: zml.Tensor, + metadata: Metadata, + parameters: Parameters, + opts: Options, + + fn forward(args: @This(), _: zml.Shape) zml.Tensor { + return switch (args.parameters) { + .stablehlo => b: { + const seq_len = args.k.dim(.k); + const mask: ?zml.Tensor = if (args.opts.causal) + zml.nn.causalAttnMask(.{ .q = seq_len, .k = seq_len }, args.q.dtype(), null) + .gatherSlices(zml.Shape.init(.{ .q = args.q.dim(.q) }, args.q.dtype()), args.token_index.appendAxes(.{.coord}), .{}) + else + null; + break :b zml.nn.sdpa(args.q, args.k, args.v, .{ .attn_mask = mask }); + }, + .flash_attention => flash_attention.attention(args.q, args.k, args.v, args.token_index, args.metadata.flash_attention, args.parameters.flash_attention, args.opts), + }; + } +}; diff --git a/zml/buffer.zig b/zml/buffer.zig index d093d65b..8714d80f 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -1,217 +1,113 @@ const std = @import("std"); -const testing = std.testing; const pjrt = @import("pjrt"); +const platforms = @import("platforms"); +const stdx = @import("stdx"); -const meta = @import("meta.zig"); -const Context = @import("context.zig").Context; -const Data = @import("dtype.zig").Data; +const constants = @import("constants.zig"); const DataType = @import("dtype.zig").DataType; -const HostBuffer = @import("hostbuffer.zig").HostBuffer; +const mem = @import("mem.zig"); +const Memory = @import("platform.zig").Memory; +const meta = @import("meta.zig"); +const pjrtx = @import("pjrtx.zig"); const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; - -test { - std.testing.refAllDecls(@This()); - std.testing.refAllDecls(Buffer); -} +const Sharding = @import("Sharding.zig"); +const Slice = @import("slice.zig").Slice; +const Target = @import("platform.zig").Target; +const testing = @import("testing.zig"); const log = std.log.scoped(.zml); +test { + std.testing.refAllDecls(Buffer); + std.testing.refAllDecls(Buffer.HostAccessible); +} + /// Buffer is a multi-dimension array, whose memory is allocated on an accelerator. /// /// * contains a handle that the ZML runtime can use to convert into a physical address, but there is no guarantee this address is visible from the CPU. /// * loading weights from disk directly to the `device zml.aio.loadBuffers` /// * can be created by calling `HostBuffer.toDevice(platform)`. pub const Buffer = struct { + _platform: *const Platform, _shape: Shape, - _api: *const pjrt.Api, _shards: Shards, - pub const MAX_NUM_SHARDS: u8 = Platform.MAX_NUM_DEVICES; - pub const Shards = std.BoundedArray(*pjrt.Buffer, MAX_NUM_SHARDS); + /// Whether executions may donate a buffer, writing their outputs into it. + /// `HostAccessible.init` uses it to decide whether devices share allocations. + pub const Donation = enum { + /// Executions only read the buffer. Unlike XLA's undonatable buffers, the + /// host may still rewrite a HostAccessible one between executions. + undonatable, + /// Executions may donate the buffer, replacing its handles with their outputs. + donatable, + }; - /// Copies the content of the given buffer from host memory to the accelerator memory. - pub fn from(platform: Platform, host_buffer: HostBuffer) !Buffer { - var res: Buffer = .{ - ._api = platform.pjrt_api, - ._shape = host_buffer.shape(), - ._shards = .{}, - }; + pub const HostAccessible = @import("buffer/HostAccessible.zig"); - // We shard only on the first axis so that the chunks are still contiguous. - // TODO: support more advanced sharding specs - meta.assert(platform.sharding().num_replicas == 1, "ZML doesn't support num_replicas > 1 for now, got: {}", .{platform.sharding()}); - const sharding_ax: ?u3 = std.simd.firstTrue(host_buffer.shape()._sharding_info); - const n_partitions = platform.sharding().num_partitions; - const chunk_size = if (sharding_ax) |ax| cs: { - // This kind of sharding error should be detected earlier on. - meta.assert(@rem(host_buffer.dim(ax), n_partitions) == 0, "Buffer.from({}) expects the sharding axis {} to have a dimension divisble by the number of devices ({}).", .{ host_buffer, ax, n_partitions }); - break :cs @divExact(host_buffer.dim(ax), n_partitions); - } else 0; + pub const MAX_NUM_SHARDS: u16 = Platform.MAX_NUM_DEVICES; + pub const Shards = stdx.BoundedArray(*pjrt.Buffer, MAX_NUM_SHARDS); - const buffer_type = bufferTypeFromDtype(host_buffer.shape().dtype()); - const byte_strides = host_buffer.strides() orelse host_buffer.shape().computeStrides().constSlice(); + pub const Shard = struct { + _platform: *const Platform, + _pjrt_buffer: *pjrt.Buffer, - var events: std.BoundedArray(*pjrt.Event, MAX_NUM_SHARDS) = .{}; - const devices = platform.getDevices(); - for (0..n_partitions) |i| { - // If no sharding if found, the given buffer is replicated on all devices. - const buf = if (sharding_ax) |ax| buf: { - const start: i64 = @as(i64, @intCast(i)) * chunk_size; - break :buf host_buffer.slice1d(ax, .{ .start = start, .end = start + chunk_size }); - } else host_buffer; + pub fn devicePtr(self: *const Shard) *anyopaque { + return self._pjrt_buffer.opaqueDeviceMemoryDataPointer(self._platform.pjrt_api) catch unreachable; + } + }; - const pjrt_buffer, const event = try platform.pjrt_client.bufferFromHostBuffer(platform.pjrt_api, .{ - .data = buf.data, - .buffer_type = buffer_type, - .dims = buf.shape().dims(), - .byte_strides = byte_strides, - .device = devices[i], - .host_buffer_semantics = .ImmutableUntilTransferCompletes, - }); + pub const ShardIterator = struct { + _platform: *const Platform, + _shards: []const *pjrt.Buffer, + _index: usize = 0, - events.appendAssumeCapacity(event); - res._shards.appendAssumeCapacity(pjrt_buffer); + pub fn remaining(self: *ShardIterator) usize { + return self._shards.len -| self._index; } - for (events.constSlice()) |event| { - try platform.awaitEvent(event); + pub fn next(self: *ShardIterator) ?Shard { + defer self._index += 1; + if (self._index >= self._shards.len) return null; + + return .{ + ._pjrt_buffer = self._shards[self._index], + ._platform = self._platform, + }; } - return res; - } + }; - /// Wraps pre-exisiting `pjrt.Buffer` shards into one `zml.Buffer`. - pub fn fromPjrtBuffers(platform: Platform, shape_: Shape, pjrt_buffers: []const *pjrt.Buffer) Buffer { - meta.assert(pjrt_buffers.len <= MAX_NUM_SHARDS, "ZML doesn't support having more than {} shards. Received {} shards for one buffer.", .{ MAX_NUM_SHARDS, pjrt_buffers.len }); - meta.assert(pjrt_buffers.len > 0, "fromPjrtBuffers expects at least one buffer, got 0.", .{}); - var shards: Shards = .{}; - shards.appendSliceAssumeCapacity(pjrt_buffers); - return .{ - ._api = platform.pjrt_api, - ._shape = shape_, - ._shards = shards, - }; - } - - /// Copies the given Zig slice to the accelerator memory and - /// return a Buffer with the given dimensions. - pub fn fromSlice(platform: Platform, dimz: anytype, s: anytype) !Buffer { - const sh = Shape.init(dimz, DataType.fromSliceElementType(s)); - return from(platform, HostBuffer.fromBytes(sh, std.mem.sliceAsBytes(s))); - } - - /// Copies the given Zig array to the accelerator memory and - /// return a Buffer using the array shape. - pub fn fromArray(platform: Platform, arr: anytype) !Buffer { - const host_buffer = HostBuffer.fromArray(&arr); - return try from(platform, host_buffer); - } - - /// Creates a Buffer with a single element. - pub fn scalar(platform: Platform, val: anytype, dtype_: DataType) !Buffer { - const x = dtype_.constant(val); - const host_buffer = HostBuffer.fromBytes(Shape.init(.{}, dtype_), x.constSlice()); - return try from(platform, host_buffer); - } - - /// Creates a Buffer with a single element repeated manytime. - pub fn constant(platform: Platform, shape_: Shape, val: anytype) !Buffer { - const x = shape_.dtype().constant(val); - const host_buffer: HostBuffer = .{ - ._shape = shape_, - ._strides = [1]i64{0} ** Shape.MAX_RANK, - .data = x.constSlice(), - }; - return try from(platform, host_buffer); - } - - test constant { - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - - const x = try constant(platform, Shape.init(.{ 4, 3, 2 }, .u16), 42); - const y = try x.getValue([4 * 3 * 2]u16); - try std.testing.expectEqual([_]u16{42} ** (4 * 3 * 2), y); - } - - /// Creates a Buffer as a view of memory visible from the device, - /// thus avoiding a copy. - /// - /// On CUDA, it also allows you to specify a host allocated slice as they seem to be accessible. - /// Be careful though, as it requires a specific alignment. - /// Also note that it might not work on all platforms, - /// could lead to crashes and is considerably slower. - pub fn asViewOf(platform: Platform, buf: HostBuffer) !Buffer { - const minor_to_major: [Shape.MAX_RANK]i64 = comptime blk: { - var res: [Shape.MAX_RANK]i64 = undefined; - for (0..Shape.MAX_RANK) |i| { - res[i] = @intCast(Shape.MAX_RANK - i - 1); - } - break :blk res; - }; - - const pjrt_buffer = try platform.pjrt_client.createViewOfDeviceBuffer(platform.pjrt_api, .{ - .data = buf.data, - .element_type = bufferTypeFromDtype(buf.shape().dtype()), - .dims = buf.shape().dims(), - // TODO: split in shards - .device = platform.getDevices()[0], - .layout = .{ - .Tiled = .{ - .minor_to_major = minor_to_major[Shape.MAX_RANK - buf.shape().rank() ..], - .tile_dims = &.{}, - .tile_dims_sizes = &.{}, - }, - }, - }); - - var shards: Shards = .{}; - shards.appendAssumeCapacity(pjrt_buffer); - return .{ - ._api = platform.pjrt_api, - ._shape = buf.shape(), - ._shards = shards, - }; - } - - /// Fetches the content of the given buffer into a stack variable of the given type. - pub fn getValue(self: Buffer, T: type) !T { - meta.assert(self._shape.byteSize() == @sizeOf(T), "Buffer {} has {d} bytes of data, can't load it to a {s} with {d} bytes", .{ self, self._shape.byteSize(), @typeName(T), @sizeOf(T) }); - var res: T = undefined; - meta.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); - const event = try self._shards.get(0).toHostBuffer(self._api, std.mem.asBytes(&res)); - try event.await_(self._api); - return res; - } - - /// Copies the content of the Buffer back to host, in the given buffer, - /// and return a new `HostBuffer` object with the same shape. - /// The returned `HostBuffer` doesn't own the memory. - pub fn toHost(self: Buffer, output: []u8) !HostBuffer { - meta.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); - const event = try self._shards.get(0).toHostBuffer(self._api, output); - try event.await_(self._api); - return HostBuffer.fromBytes(self.shape(), output); - } - - /// Copies the content of the Buffer to the host. - /// The returned `HostBuffer` does own the memory. - pub fn toHostAlloc(self: Buffer, allocator: std.mem.Allocator) !HostBuffer { - const output = try HostBuffer.empty(allocator, self.shape()); - meta.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); - const event = try self._shards.get(0).toHostBuffer(self._api, @constCast(output.data)); - try event.await_(self._api); - return output; - } + pub const FromOptions = struct { wait: bool = true, memory: Memory.Kind = .default }; /// Frees the accelerator memory. /// Depending on the platform, the memory is typically not released to the OS /// but just marked as available in the memory pool. - pub fn deinit(self: *const Buffer) void { + pub fn deinit(self: *Buffer) void { for (self._shards.constSlice()) |buffer| { - buffer.deinit(self._api); + buffer.deinit(self._platform.pjrt_api); } + self._shards = .empty; + } + + /// Given a flat struct (static size, no slices) containing `zml.Buffer`, `deinit` each one of them. + pub fn deinitAll(T: type, buffers: *mem.Bufferized(T)) void { + meta.visitFlatStruct(struct { + fn deinit(_: void, x: *Buffer) void { + x.deinit(); + } + }.deinit, {}, buffers); + } + + /// Given an arbitrary struct `deinit` all `zml.Buffer` containing. + /// If the struct contains slices of `zml.Buffer` the memory of the slices will NOT be freed, + /// This only impacts device memory. + pub fn freeDeviceMemoryButKeepHostMetadataMemory(T: type, buffers: *mem.Bufferized(T)) void { + meta.visit(struct { + fn deinit(_: void, x: *Buffer) void { + x.deinit(); + } + }.deinit, {}, buffers); } /// This Buffer shape. @@ -219,111 +115,342 @@ pub const Buffer = struct { return self._shape; } - /// This Buffer shape as a slice of dims. - pub fn dims(self: *const Buffer) []const i64 { - return self._shape.dims(); + pub fn numShards(self: Buffer) u32 { + return @intCast(self._shards.len); } - /// This Buffer element type. - pub fn dtype(self: Buffer) DataType { - return self._shape.dtype(); + pub fn shards(self: *const Buffer) ShardIterator { + return .{ + ._platform = self._platform, + ._shards = self._shards.constSlice(), + }; } - /// This Buffer rank. - pub fn rank(self: Buffer) u4 { - return self._shape.rank(); + pub fn format(self: Buffer, writer: *std.Io.Writer) !void { + const pl = Sharding.Placement.init(&self._shape) catch |err| { + return try writer.print("Buffer({f}) with invalid sharding {} ({})", .{ self._shape, self._shape._sharding, err }); + }; + try writer.print("{f}", .{pl}); } - /// Test helper: returns a new Buffer with the given tags. - /// Allows to call `zml.testing.compileAndCall` when the tested - /// functions requires tagged tensors. - pub fn withTags(self: Buffer, tags_: anytype) Buffer { - var res = self; - res._shape = self._shape.withTags(tags_); + /// Copies the content of the given buffer from host memory to the accelerator memory. + pub fn from( + io: std.Io, + platform: *const Platform, + shape_: Shape, + data_: []const u8, + opts: FromOptions, + ) !Buffer { + var pjrt_shards: Shards = .empty; + var packed_shape = shape_.packedShape(); + if (packed_shape._sharding.mesh == null) packed_shape._sharding.mesh = platform.replicated_mesh; + const byte_strides = packed_shape.computeByteStrides(); + const placement = Sharding.Placement.initWithByteStrides(&packed_shape, byte_strides.constSlice()) catch |err| { + @branchHint(.cold); + std.debug.panic("Buffer shape {f} is incompatible with its mesh: {t}", .{ packed_shape, err }); + }; + const ty = pjrtx.bufferTypeFromDtype(packed_shape.dtype()); + const layout = platform.defaultMemoryLayout(placement.local_dims.constSlice(), packed_shape.dtype()); + + errdefer for (pjrt_shards.slice()) |shard| { + shard.deinit(platform.pjrt_api); + }; + + stdx.debug.assert(platform.devices[0].memory(opts.memory) != null, "Device doesn't have {} memory", .{opts.memory}); + const byte_size: usize = if (packed_shape.rank() == 0) + packed_shape.dtype().sizeOf() + else + @intCast(packed_shape.dim(0) * byte_strides.get(0)); + stdx.debug.assert(data_.len == byte_size, "Expected data length {}, got {}", .{ byte_size, data_.len }); + const base_ptr = data_.ptr; + + for (platform.physical_mesh.devices_in_canonical_order, 0..) |device, device_index| { + const memory = platform.devices[device.id].memory(opts.memory).?; + const args: pjrt.Client.BufferFromHostBufferArgs = .{ + // Change for each device + .data = placement.shardPtr(device_index, base_ptr), + .dst = .{ .memory = memory.pjrt_memory }, + // Constant across devices + .layout = layout, + .dims = placement.local_dims.constSlice(), + .buffer_type = ty, + .byte_strides = byte_strides.constSlice(), + .host_buffer_semantics = .ImmutableUntilTransferCompletes, + }; + + const pjrt_buffer, const event = try platform.pjrt_client.bufferFromHostBuffer(platform.pjrt_api, args); + if (event) |ev| ev.deinit(platform.pjrt_api); + + pjrt_shards.appendAssumeCapacity(pjrt_buffer); + } + + if (opts.wait) { + try awaitShards(pjrt_shards.constSlice(), platform, io); + } + + return .{ ._platform = platform, ._shape = shape_, ._shards = pjrt_shards }; + } + + /// Copies the given Zig bytes to the accelerator memory and + /// return a Buffer with the given dimensions. + pub fn fromBytes(io: std.Io, platform: *const Platform, sh: Shape, data: []const u8) !Buffer { + return fromBytesOpts(io, platform, sh, data, .{}); + } + + pub fn fromBytesOpts(io: std.Io, platform: *const Platform, sh: Shape, data: []const u8, opts: FromOptions) !Buffer { + return from(io, platform, sh, data, opts); + } + + /// Copies the given zml.Slice to the accelerator memory and + /// return a Buffer. + pub fn fromSlice(io: std.Io, platform: *const Platform, slice: Slice) !Buffer { + return fromSliceOpts(io, platform, slice, .{}); + } + + pub fn fromSliceOpts(io: std.Io, platform: *const Platform, slice: Slice, opts: FromOptions) !Buffer { + return from(io, platform, slice.shape, std.mem.sliceAsBytes(slice.constData()), opts); + } + + /// Creates a Buffer with a single element. + pub fn scalar(io: std.Io, platform: *const Platform, val: anytype, dtype_: DataType) !Buffer { + const x = dtype_.constant(val); + return fromBytes(io, platform, .scalar(dtype_), x.asBytes()); + } + + pub fn await(self: Buffer, io: std.Io) !void { + try awaitShards(self._shards.constSlice(), self._platform, io); + } + + fn awaitShards(pjrt_shards: []const *pjrt.Buffer, platform: *const Platform, io: std.Io) !void { + for (pjrt_shards) |shard| { + const event = shard.readyEvent(platform.pjrt_api); + defer event.deinit(platform.pjrt_api); + try event.await(platform.pjrt_api, io); + } + } + + pub const UnitializedOptions = struct { memory: Memory.Kind = .default }; + + pub fn uninitialized( + _: std.Io, + platform: *const Platform, + shape_: Shape, + opts: UnitializedOptions, + ) !Buffer { + std.log.debug("uninitialized {f}", .{shape_}); + var buffer: Buffer = .{ + ._platform = platform, + ._shape = shape_, + ._shards = .empty, + }; + const metadata: PjrtBufferMetadata = .init(platform, shape_); + errdefer for (buffer._shards.slice()) |shard| { + shard.deinit(platform.pjrt_api); + }; + + stdx.debug.assert(platform.devices[0].memory(opts.memory) != null, "Device doesn't have {} memory", .{opts.memory}); + + for (platform.physical_mesh.devices_in_canonical_order) |device| { + const memory = platform.devices[device.id].memory(opts.memory).?; + const args: pjrt.Client.CreateUninitializedBufferArgs = .{ + // Change for each device + .dst = .{ .memory = memory.pjrt_memory }, + // Constant across devices + .layout = metadata.layout, + .dims = metadata.placement.local_dims.constSlice(), + .element_type = metadata.ty, + }; + + const shard_buffer = try platform.pjrt_client.createUninitializedBuffer(platform.pjrt_api, args); + buffer._shards.appendAssumeCapacity(shard_buffer); + } + + return buffer; + } + + /// Wraps pre-exisiting `pjrt.Buffer` shards into one `zml.Buffer`. + pub fn fromPjrtBuffers(platform: *const Platform, sh: Shape, pjrt_buffers: []const *pjrt.Buffer) Buffer { + stdx.debug.assert(pjrt_buffers.len <= MAX_NUM_SHARDS, "ZML doesn't support having more than {} shards. Received {} shards for one buffer.", .{ MAX_NUM_SHARDS, pjrt_buffers.len }); + stdx.debug.assert(pjrt_buffers.len > 0, "fromPjrtBuffers expects at least one buffer, got 0.", .{}); + + return .{ + ._platform = platform, + ._shape = sh, + ._shards = Shards.fromSlice(pjrt_buffers) catch unreachable, + }; + } + + /// Fetches the content of the given buffer into a stack variable of the given type. + pub fn getValue(self: Buffer, T: type, io: std.Io) !T { + stdx.debug.assert(self._shape.byteSize() == @sizeOf(T), "Buffer {f} has {d} bytes of data, can't load it to a {s} with {d} bytes", .{ self, self._shape.byteSize(), @typeName(T), @sizeOf(T) }); + var res: T = undefined; + + try self.toSlice(io, .init(self.shape(), std.mem.asBytes(&res))); + return res; } - pub fn format( - self: Buffer, - comptime fmt: []const u8, - options: std.fmt.FormatOptions, - writer: anytype, - ) !void { - _ = fmt; - _ = options; - try writer.print("Buffer({_})", .{self._shape}); + /// Copies the content of the Buffer to the provided slice. + pub fn toSlice(self: Buffer, io: std.Io, slice: Slice) !void { + stdx.debug.assert(self._shape.eql(slice.shape), "Buffer shape {f} doesn't match destination slice {f}", .{ self._shape, slice.shape }); + + const placement = placementOrPanic(self._platform, self._shape); + for (0..self._platform.physical_mesh.devices_in_canonical_order.len) |shard_index| { + // TODO: handle replicated information, we shouldn't iterate over all the devices unless needed + const sub_slice = placement.shardSlice(shard_index, slice); + if (!sub_slice.isContiguous()) return error.NonContiguousShardRead; + + const size_bytes = placement.localShape(self._shape).byteSize(); + const destination = sub_slice.data()[0..size_bytes]; + const maybe_event = try self._shards.get(shard_index).toHostBuffer(self._platform.pjrt_api, destination); + + if (maybe_event) |event| { + defer event.deinit(self._platform.pjrt_api); + try event.await(self._platform.pjrt_api, io); + } + } } - fn hasShardedAxis(self: Buffer) bool { - if (self._shards.len == 1) return false; - return @reduce(.Or, self._shape._sharding_info); + /// Copies the content of the Buffer to the provided slice. + /// The returned slice owns the memory. + pub fn toSliceAlloc(self: Buffer, allocator: std.mem.Allocator, io: std.Io) !Slice { + const slice = try Slice.alloc(allocator, self.shape()); + errdefer slice.free(allocator); + + const placement = placementOrPanic(self._platform, self._shape); + + var shard_slice = try Slice.alloc(allocator, placement.localShape(self._shape)); + defer shard_slice.free(allocator); + + for (0..self._platform.physical_mesh.devices_in_canonical_order.len) |shard_index| { + const sub_slice = placement.shardSlice(shard_index, slice); + const maybe_event = try self._shards.get(shard_index).toHostBuffer(self._platform.pjrt_api, shard_slice.data()); + if (maybe_event) |event| { + defer event.deinit(self._platform.pjrt_api); + try event.await(self._platform.pjrt_api, io); + } + + // TODO: why is this using Slice.copy while `toSlice` errors out ? + // TODO: why is this writing in a copy rather than directly in place. + sub_slice.copy(shard_slice.constData()); + } + + return slice; + } + + /// The memory used by this Buffer across all devices + /// ie: `num_devices * shard_byte_size` + /// `shard_byte_size` can be up to `self.shape().byteSize()` when the buffer is fully replicated. + pub fn byteSize(self: Buffer) usize { + const placement = placementOrPanic(self._platform, self._shape); + return placement.localShape(self._shape).byteSize() * self._shards.len; + } + + pub fn opaqueDevicePtr(self: Buffer, device_id: usize) *anyopaque { + return self._shards.get(device_id).opaqueDeviceMemoryDataPointer(self._platform.pjrt_api) catch unreachable; + } + + /// Creates a view of the given buffer + pub fn createView(self: Buffer) !Buffer { + const platform = self._platform; + + var view_shards_buffer: [MAX_NUM_SHARDS]*pjrt.Buffer = undefined; + var view_shard_list: std.ArrayList(*pjrt.Buffer) = .initBuffer(&view_shards_buffer); + errdefer for (view_shard_list.items) |shard| { + shard.deinit(platform.pjrt_api); + }; + + for (self._shards.constSlice()) |shard| { + const view_shard = try platform.pjrt_client.createViewOfDeviceBuffer(platform.pjrt_api, .{ + .device_buffer_ptr = try shard.opaqueDeviceMemoryDataPointer(platform.pjrt_api), + .dims = shard.dimensions(platform.pjrt_api), + .element_type = shard.elementType(platform.pjrt_api), + .layout = try shard.memoryLayout(platform.pjrt_api), + .device = try shard.device(platform.pjrt_api), + }); + view_shard_list.appendAssumeCapacity(view_shard); + } + + return fromPjrtBuffers(platform, self._shape, view_shard_list.items); } }; -pub fn bufferTypeFromDtype(dt: DataType) pjrt.BufferType { - return switch (dt) { - .bool => .PRED, - .f8e4m3b11fnuz => .F8E4M3B11FNUZ, - .f8e4m3fn => .F8E4M3FN, - .f8e4m3fnuz => .F8E4M3FNUZ, - .f8e5m2 => .F8E5M2, - .f8e5m2fnuz => .F8E5M2FNUZ, - .bf16 => .BF16, - .f16 => .F16, - .f32 => .F32, - .f64 => .F64, - .i8 => .S8, - .i4 => .S4, - .i16 => .S16, - .i32 => .S32, - .i64 => .S64, - .u4 => .U4, - .u8 => .U8, - .u16 => .U16, - .u32 => .U32, - .u64 => .U64, - .c64 => .C64, - .c128 => .C128, - }; -} +test "device round-trip" { + const zml = @import("zml.zig"); + const io = std.testing.io; + const allocator = std.testing.allocator; + const platform = zml.testing.env(); + const tp = zml.testing.tpMesh(); -pub fn dtypeFromBufferType(pjrt_type: pjrt.BufferType) DataType { - return switch (pjrt_type) { - .PRED => .bool, - .F8E4M3B11FNUZ => .f8e4m3b11fnuz, - .F8E4M3FN => .f8e4m3fn, - .F8E4M3FNUZ => .f8e4m3fnuz, - .F8E5M2 => .f8e5m2, - .F8E5M2FNUZ => .f8e5m2fnuz, - .BF16 => .bf16, - .F16 => .f16, - .F32 => .f32, - .F64 => .f64, - .S8 => .i8, - .S4 => .i4, - .S16 => .i16, - .S32 => .i32, - .S64 => .i64, - .U4 => .u4, - .U8 => .u8, - .U16 => .u16, - .U32 => .u32, - .U64 => .u64, - .C64 => .c64, - .C128 => .c128, - .INVALID => @panic("Found an invalid pjrt buffer"), + const x: [8][8]u32 = .{ + .{ 0, 1, 2, 3, 4, 5, 6, 7 }, + .{ 8, 9, 10, 11, 12, 13, 14, 15 }, + .{ 16, 17, 18, 19, 20, 21, 22, 23 }, + .{ 24, 25, 26, 27, 28, 29, 30, 31 }, + .{ 32, 33, 34, 35, 36, 37, 38, 39 }, + .{ 40, 41, 42, 43, 44, 45, 46, 47 }, + .{ 48, 49, 50, 51, 52, 53, 54, 55 }, + .{ 56, 57, 58, 59, 60, 61, 62, 63 }, }; -} -test bufferTypeFromDtype { - inline for (@typeInfo(DataType).Enum.fields) |field| { - const dt: DataType = @enumFromInt(field.value); - try std.testing.expectEqual(dt, dtypeFromBufferType(bufferTypeFromDtype(dt))); + const x_h: zml.Slice = .init(.withPartitioning( + .init(.{ .b = 8, .d = 8 }, .u32), + tp, + .{ .b = .model }, + ), std.mem.asBytes(&x)); + // no free: x_h is stack allocated + const x_d: zml.Buffer = try .fromSlice(io, platform, x_h); + try std.testing.expectEqual(platform.devices.len, x_d.numShards()); + + { + const x_h_reborn: zml.Slice = try x_d.toSliceAlloc(allocator, io); + defer x_h_reborn.free(allocator); + + errdefer std.log.err(" - reference: {d}\n- actual: {d}", .{ x_h, x_h_reborn }); + try zml.testing.expectClose(io, x_h, x_h_reborn, .exact_match); } - inline for (@typeInfo(pjrt.BufferType).Enum.fields) |field| { - const dt: pjrt.BufferType = @enumFromInt(field.value); - if (dt == .INVALID) continue; - try std.testing.expectEqual(dt, bufferTypeFromDtype(dtypeFromBufferType(dt))); + { + var x_2: @TypeOf(x) = undefined; + const x_h_reborn: zml.Slice = .init(x_h.shape, std.mem.asBytes(&x_2)); + // no free: x_h_reborn is stack allocated + try x_d.toSlice(io, x_h_reborn); + + errdefer std.log.err(" - reference: {d}\n- actual: {d}", .{ x_h, x_h_reborn }); + try zml.testing.expectClose(io, x_h, x_h_reborn, .exact_match); } } + +pub const PjrtBufferMetadata = struct { + shape: Shape, + ty: pjrt.BufferType, + placement: Sharding.Placement, + layout: pjrt.MemoryLayout, + + /// Inline because the layout may point to stack memory, as with + /// `Platform.defaultMemoryLayout`. Outside `Buffer` so that + /// `Buffer.HostAccessible` can use it without it being part of the API: + /// zml.zig only exports `Buffer` from this file. + pub inline fn init(platform: *const Platform, shape_: Shape) PjrtBufferMetadata { + + // Shards use the PJRT shape, which packs sub-byte elements. + const packed_shape = shape_.packedShape(); + const placement = placementOrPanic(platform, packed_shape); + + return .{ + .shape = packed_shape, + .ty = pjrtx.bufferTypeFromDtype(packed_shape.dtype()), + .placement = placement, + .layout = platform.defaultMemoryLayout(placement.local_dims.constSlice(), packed_shape.dtype()), + }; + } +}; + +fn placementOrPanic(platform: *const Platform, shape: Shape) Sharding.Placement { + var sh = shape; + if (sh._sharding.mesh == null) sh._sharding.mesh = platform.replicated_mesh; + return Sharding.Placement.init(&sh) catch |err| { + @branchHint(.cold); + std.debug.panic("Buffer shape {f} is incompatible with its mesh: {t}", .{ shape, err }); + }; +} diff --git a/zml/buffer/HostAccessible.zig b/zml/buffer/HostAccessible.zig new file mode 100644 index 00000000..b3e82a9c --- /dev/null +++ b/zml/buffer/HostAccessible.zig @@ -0,0 +1,2202 @@ +//! A `Buffer` in `.host_pinned` memory that the host reads and writes in place +//! through views, which a plain `Buffer` placed there doesn't offer. Views +//! address elements as PJRT stores them, with sub-byte elements packed in bytes. +//! The host may only access the buffer between executions. Host pointers are +//! read at creation, and by `await` when an execution replaced the handles. +//! Executions must preserve shape, sharding and layout. +//! +//! Like `Buffer`, it is a value. Pass `buffer` to an execution to read it as +//! an input, or `&buffer` to donate it as an output, then `await` the +//! HostAccessible whose `buffer` the execution replaced. Copies share the +//! mapping, and a copy whose handles the execution did not replace panics on +//! `view` rather than write to an allocation the execution may have released. +const std = @import("std"); +const builtin = @import("builtin"); + +const pjrt = @import("pjrt"); +const stdx = @import("stdx"); + +const Buffer = @import("../buffer.zig").Buffer; +const PjrtBufferMetadata = @import("../buffer.zig").PjrtBufferMetadata; +const Platform = @import("../platform.zig").Platform; +const Shape = @import("../shape.zig").Shape; +const Sharding = @import("../Sharding.zig"); +const testing = @import("../testing.zig"); + +const HostAccessible = @This(); + +/// The device side, with one PJRT buffer per device, for executions to read +/// or donate. +buffer: Buffer, +/// What views read, heap allocated so views can borrow it while the +/// HostAccessible moves. +mapping: *Mapping, + +/// The host side of the buffer. Views and blocks copy nothing from it: the +/// layout and shard placement are prepared once at creation and take several +/// KB, and copying them into every view, slice and block made memcpy most of +/// llmd's decode thread. +const Mapping = struct { + /// The PJRT shape, which packs sub-byte elements in bytes, unlike + /// `buffer._shape`. Views address its elements. + packed_shape: Shape, + shard_shape: Shape, + layout_metadata: LayoutMetadata, + allocations: HostPinnedAllocations, + /// Every shard holds the whole array densely, so element `i` sits at + /// `i * element_size` in each of them, as in replicated buffers on every + /// platform but TPU. `get`, `set` and blocks then skip locating elements + /// through the origins and the layout. + contiguous: bool, + + /// Takes ownership of the allocations, on success only. + fn create( + allocator: std.mem.Allocator, + shape_: Shape, + shard_shape: Shape, + layout: pjrt.MemoryLayout, + allocations: HostPinnedAllocations, + ) !*Mapping { + const mapping = try allocator.create(Mapping); + errdefer allocator.destroy(mapping); + + const layout_metadata: LayoutMetadata = .init(shape_, shard_shape, layout); + + mapping.* = .{ + .packed_shape = shape_, + .shard_shape = shard_shape, + .layout_metadata = layout_metadata, + .allocations = allocations, + .contiguous = layout_metadata.addressing == .row_major and + allocations.regions.len == 1 and + shard_shape.count() == shape_.count(), + }; + + return mapping; + } +}; + +/// The host allocations behind the buffer's shards. Devices that share an +/// allocation map to one entry, and entries holding the same region of the +/// array are its replicas. Entries stay in region order, so a region's +/// replicas are a range of every field. +const HostPinnedAllocations = struct { + entries: std.MultiArrayList(Allocation) = .empty, + /// The regions the entries hold, in order of first appearance. + regions: stdx.BoundedArray(Region, Platform.MAX_NUM_DEVICES) = .empty, + + const empty: HostPinnedAllocations = .{}; + + const Allocation = struct { + /// Global coordinates of the allocation's first element. + origin: Coordinates, + /// Read from `handle`. Blocks borrow their region's range. + ptr: [*]u8, + /// The shard owning the allocation, which must destroy it: an index + /// into `buffer._shards`, whose handles execution may replace. + shard_index: usize, + /// The handle `ptr` was read from. Execution replaces the handles of + /// donated buffers with its outputs, which may not reuse the allocation. + handle: *pjrt.Buffer, + /// Whether `handle` holds an external reference, which keeps the + /// allocation alive for the device views of other devices. + shared: bool = false, + }; + + /// `entries[start..][0..len]` hold the region's replicas, in the order they + /// were added. Reads use the first. + const Region = struct { start: u8, len: u8 }; + + fn deinit(self: *HostPinnedAllocations, allocator: std.mem.Allocator) void { + self.entries.deinit(allocator); + } + + /// Add an allocation to the replicas of its region, within the + /// reserved capacity. + fn add(self: *HostPinnedAllocations, shard_shape: Shape, allocation: Allocation) void { + const region_index = self.findRegion(allocation.origin, shard_shape) orelse { + self.regions.appendAssumeCapacity(.{ .start = @intCast(self.entries.len), .len = 1 }); + self.entries.appendAssumeCapacity(allocation); + return; + }; + const region = &self.regions.slice()[region_index]; + // Keep each region's entries together by shifting the regions after it. + self.entries.insertAssumeCapacity(region.start + region.len, allocation); + region.len += 1; + for (self.regions.slice()[region_index + 1 ..]) |*next| next.start += 1; + } + + /// The index of the region holding the shards that start at `origin`. + fn findRegion(self: *const HostPinnedAllocations, origin: Coordinates, shard_shape: Shape) ?usize { + const origins = self.entries.items(.origin); + for (self.regions.constSlice(), 0..) |region, i| { + if (origin.eqlWithin(origins[region.start], shard_shape)) return i; + } + return null; + } + + /// The host pointers of a region's replicas. + fn replicas(self: *const HostPinnedAllocations, region: Region) []const [*]u8 { + return self.entries.items(.ptr)[region.start..][0..region.len]; + } + + /// Read the host pointers of the allocations from their current handles. + /// Each read takes and releases a hold on the PJRT buffer and allocates an + /// external reference, about 0.9 us: llmd would make thousands per step if + /// views read them, so they are read at creation and by `await` only. + fn refreshPointers(self: *HostPinnedAllocations, api: *const pjrt.Api, handles: []const *pjrt.Buffer) !void { + const shard_indices = self.entries.items(.shard_index); + // Read every pointer before replacing any, so a failure keeps the previous ones. + var ptrs: [Platform.MAX_NUM_DEVICES][*]u8 = undefined; + for (shard_indices, ptrs[0..shard_indices.len]) |shard_index, *ptr| { + ptr.* = @ptrCast(try handles[shard_index].opaqueDeviceMemoryDataPointer(api)); + } + @memcpy(self.entries.items(.ptr), ptrs[0..shard_indices.len]); + for (self.entries.items(.handle), shard_indices) |*handle, shard_index| handle.* = handles[shard_index]; + } + + /// Whether the host pointers were read from the current handles, which + /// execution replaces for donated buffers. + fn pointersAreCurrent(self: *const HostPinnedAllocations, handles: []const *pjrt.Buffer) bool { + // Host memory in tests has no handles: its pointers never change. + if (handles.len == 0) return true; + for (self.entries.items(.handle), self.entries.items(.shard_index)) |handle, shard_index| { + if (handle != handles[shard_index]) return false; + } + return true; + } + + /// Destroy every handle, the device views before the allocations they borrow. + fn deinitHandles(self: *const HostPinnedAllocations, api: *const pjrt.Api, handles: []const *pjrt.Buffer) void { + const shard_indices = self.entries.items(.shard_index); + for (handles, 0..) |handle, index| { + if (std.mem.indexOfScalar(usize, shard_indices, index) == null) handle.deinit(api); + } + for (shard_indices, self.entries.items(.shared)) |shard_index, shared| { + const handle = handles[shard_index]; + // `shared` is set by the one successful increase, and only undonatable + // buffers share allocations, whose handles execution never replaces. + if (shared) handle.decreaseExternalReferenceCount(api) catch |err| { + std.debug.panic("shared HostAccessible allocation has no reference to release, was it donated? {}", .{err}); + }; + handle.deinit(api); + } + } +}; + +/// Host memory standing in for PJRT buffers, to test layouts and shardings +/// that no available platform produces: every platform but TPU uses dense +/// row-major layouts, and CI has few devices. `origins` holds the row-major +/// index in `shape_` of each allocation's first element, and `ptrs` one pointer +/// per origin. Copies the layout, the origins and the pointers, and borrows the +/// storage. +pub fn initForTests( + allocator: std.mem.Allocator, + shape_: Shape, + shard_shape: Shape, + layout: pjrt.MemoryLayout, + origins: []const usize, + ptrs: []const [*]u8, +) !HostAccessible { + if (!builtin.is_test) @compileError("HostAccessible.initForTests is only available in tests"); + + var allocations: HostPinnedAllocations = .empty; + errdefer allocations.deinit(allocator); + + try allocations.entries.ensureTotalCapacity(allocator, origins.len); + // Without PJRT buffers, there are no handles to read pointers from or destroy. + for (origins, ptrs) |origin, ptr| { + allocations.add( + shard_shape, + .{ + // An empty array has no element to locate the origin of. + .origin = if (shape_.count() == 0) .zero else .unflatten(shape_, origin), + .ptr = ptr, + .shard_index = undefined, + .handle = undefined, + }, + ); + } + + return .{ + .buffer = .{ ._platform = undefined, ._shape = shape_, ._shards = .empty }, + .mapping = try .create(allocator, shape_, shard_shape, layout, allocations), + }; +} + +/// With `.undonatable`, devices holding the same shard share one allocation on +/// CPU, CUDA and ROCm: the first allocates it, and the others read it through +/// device views. Elsewhere each device owns an allocation, and writes reach +/// every one of them. Execution never replaces the handles of an undonatable +/// buffer, so `await` never reads its host pointers again. +/// With `.donatable`, each device owns an allocation: writing a shared one +/// would expose the output of a device to the reads of another. +pub fn init( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + shape_: Shape, + donation: Buffer.Donation, +) !HostAccessible { + const can_share_allocations = switch (platform.target) { + .cpu, .cuda, .rocm => donation == .undonatable, + else => false, + }; + + var buffer: Buffer = .{ + ._platform = platform, + ._shape = shape_, + ._shards = .empty, + }; + const metadata: PjrtBufferMetadata = .init(platform, shape_); + const devices = platform.physical_mesh.devices_in_canonical_order; + const api = platform.pjrt_api; + + var allocations: HostPinnedAllocations = .empty; + errdefer allocations.deinit(allocator); + + try allocations.entries.ensureTotalCapacity(allocator, devices.len); + errdefer allocations.deinitHandles(api, buffer._shards.constSlice()); + + for (devices, 0..) |device, shard_index| { + const memory = platform.devices[device.id].memory(.host_pinned).?.pjrt_memory; + const origin: Coordinates = .ofShard(&metadata.placement, shard_index, metadata.shape); + const shared_region = if (can_share_allocations) allocations.findRegion(origin, metadata.placement.localShape(metadata.shape)) else null; + + if (shared_region) |region_index| { + // Devices sharing allocations add no replicas: the region has a single entry. + const entry = allocations.regions.get(region_index).start; + const existing = allocations.entries.get(entry); + if (!existing.shared) { + try existing.handle.increaseExternalReferenceCount(api); + allocations.entries.items(.shared)[entry] = true; + } + buffer._shards.appendAssumeCapacity(try platform.pjrt_client.createViewOfDeviceBuffer(api, .{ + .device_buffer_ptr = try existing.handle.opaqueDeviceMemoryDataPointer(api), + .dims = metadata.placement.local_dims.constSlice(), + .element_type = metadata.ty, + .layout = try existing.handle.memoryLayout(api), + // The backing host allocation is accessible across devices + // (portable pinned memory on CUDA/ROCm). This memory space + // associates the view with the current device rather than + // the device that originally allocated the memory. + .memory = memory, + })); + } else { + const handle = try platform.pjrt_client.createUninitializedBuffer(api, .{ + .dims = metadata.placement.local_dims.constSlice(), + .element_type = metadata.ty, + .layout = metadata.layout, + .dst = .{ .memory = memory }, + }); + buffer._shards.appendAssumeCapacity(handle); + // The pointer is read once the creation completes, after `await` below. + allocations.add(metadata.placement.localShape(metadata.shape), .{ + .origin = origin, + .ptr = undefined, + .shard_index = shard_index, + .handle = handle, + }); + } + } + + try buffer.await(io); + try allocations.refreshPointers(api, buffer._shards.constSlice()); + + return .{ + .buffer = buffer, + .mapping = try .create(allocator, metadata.shape, metadata.placement.localShape(metadata.shape), metadata.layout, allocations), + }; +} + +/// Execution must be done with the buffer. Copies must not be used after. +pub fn deinit(self: *const HostAccessible, allocator: std.mem.Allocator) void { + // Host memory in tests has neither handles nor a platform. + if (self.buffer._shards.len != 0) self.mapping.allocations.deinitHandles(self.buffer._platform.pjrt_api, self.buffer._shards.constSlice()); + self.mapping.allocations.deinit(allocator); + allocator.destroy(self.mapping); +} + +/// Wait for execution to finish, and read the host pointers of handles that +/// execution replaced. Take new views to access the output. +pub fn await(self: *HostAccessible, io: std.Io) !void { + try self.buffer.await(io); + const handles = self.buffer._shards.constSlice(); + const allocations = &self.mapping.allocations; + if (!allocations.pointersAreCurrent(handles)) try allocations.refreshPointers(self.buffer._platform.pjrt_api, handles); +} + +/// Borrows the mapping. Executions using the buffer must have completed, and +/// `await` must follow any that replaced the handles: `view` panics before +/// then rather than write to an allocation the execution may have released. +/// The view and its blocks are valid until the next execution starts or the +/// HostAccessible is deinitialized. +pub fn view(self: *const HostAccessible, comptime T: type) View(T) { + const mapping = self.mapping; + std.debug.assert(@sizeOf(T) == mapping.shard_shape.dtype().sizeOf()); + stdx.debug.assert(mapping.allocations.pointersAreCurrent(self.buffer._shards.constSlice()), "await the HostAccessible after execution replaces its handles", .{}); + return .{ .mapping = mapping, .len = mapping.packed_shape.count() }; +} + +/// Prints every element, read through a view: the same rules as `view` apply. +pub fn format(self: *const HostAccessible, writer: *std.Io.Writer) std.Io.Writer.Error!void { + try writer.print("HostAccessible({f}) {{", .{self.mapping.packed_shape}); + switch (self.mapping.shard_shape.dtype()) { + inline else => |dt| { + if (comptime dt.bitSizeOf() < 8) unreachable; + const values = self.view(dt.toZigType()); + var blocks = values.blocks(); + while (blocks.next()) |block| { + for (block.firstReplica()) |value| try writer.print(" {any}", .{value}); + } + }, + } + try writer.writeAll(" }"); +} + +/// A range of the array's elements in row-major order, borrowed from a +/// HostAccessible. Reads use the first replica, and writes reach every replica. +pub fn View(comptime T: type) type { + return struct { + mapping: *const Mapping, + start: usize = 0, + len: usize, + + const Self = @This(); + + /// A view without elements, which never accesses the buffer. + pub const empty: Self = .{ + .mapping = undefined, + .len = 0, + }; + + /// Select `len_` elements starting at `start`, or the rest of the view for null. + pub fn slice(self: Self, start: usize, len_: ?usize) Self { + std.debug.assert(start <= self.len); + const len = len_ orelse self.len - start; + std.debug.assert(len <= self.len - start); + var result = self; + result.start += start; + result.len = len; + return result; + } + + // Single elements are addressed directly, without slicing and iterating. + pub fn get(self: Self, index: usize) T { + const element = self.locateElement(index); + return itemAt(self.mapping.allocations.replicas(element.region)[0], element.byte_offset).*; + } + + pub fn set(self: Self, index: usize, value: T) void { + const element = self.locateElement(index); + for (self.mapping.allocations.replicas(element.region)) |ptr| itemAt(ptr, element.byte_offset).* = value; + } + + fn locateElement(self: *const Self, index: usize) struct { region: HostPinnedAllocations.Region, byte_offset: usize } { + std.debug.assert(index < self.len); + if (self.mapping.contiguous) { + return .{ .region = self.mapping.allocations.regions.get(0), .byte_offset = (self.start + index) * @sizeOf(T) }; + } + const location = self.locate(.unflatten(self.mapping.packed_shape, self.start + index)); + return .{ .region = location.region, .byte_offset = self.mapping.layout_metadata.byteOffset(location.offset, location.local) }; + } + + pub fn fill(self: Self, value: T) void { + var iterator = self.blocks(); + while (iterator.next()) |block| { + for (0..block.replicas.len) |i| @memset(block.replica(i), value); + } + } + + /// Fill with `first + i * step`, where `i` is relative to this view. + /// `T` must be an integer type holding every `i` and value; safe builds check it. + pub fn fillIota(self: Self, first: T, step: T) void { + var iterator = self.blocks(); + while (iterator.next()) |block| { + const items = block.firstReplica(); + // TODO: SIMD + for (items, block.offset..) |*value, i| value.* = first + @as(T, @intCast(i)) * step; + for (1..block.replicas.len) |i| @memcpy(block.replica(i), items); + } + } + + pub fn copyFrom(self: Self, values: []const T) void { + std.debug.assert(values.len == self.len); + var iterator = self.blocks(); + while (iterator.next()) |block| { + for (0..block.replicas.len) |i| @memcpy(block.replica(i), values[block.offset..][0..block.len]); + } + } + + pub fn copyTo(self: Self, values: []T) void { + std.debug.assert(values.len == self.len); + var iterator = self.blocks(); + while (iterator.next()) |block| { + @memcpy(values[block.offset..][0..block.len], block.firstReplica()); + } + } + + /// The index of the first element equal to `value`, searching each + /// block with `std.mem.indexOfScalar`. + pub fn indexOfScalar(self: Self, value: T) ?usize { + var iterator = self.blocks(); + while (iterator.next()) |block| { + if (std.mem.indexOfScalar(T, block.firstReplica(), value)) |index| return block.offset + index; + } + return null; + } + + /// The index of the first element equal to any of `values`, searching + /// each block with `std.mem.indexOfAny`. + pub fn indexOfAny(self: Self, values: []const T) ?usize { + var iterator = self.blocks(); + while (iterator.next()) |block| { + if (std.mem.indexOfAny(T, block.firstReplica(), values)) |index| return block.offset + index; + } + return null; + } + + /// Source and destination must not overlap. + pub fn copyFromView(self: Self, src: Self) void { + std.debug.assert(src.len == self.len); + var iterator = self.blocks(); + while (iterator.next()) |block| { + const items = block.firstReplica(); + src.slice(block.offset, block.len).copyTo(items); + for (1..block.replicas.len) |i| @memcpy(block.replica(i), items); + } + } + + /// The elements as one read-only slice, for reads that index or zip + /// several views. Contiguous buffers, such as replicated ones on every + /// platform but TPU, return their first replica in place; others + /// copy into `scratch`, which must hold `len` elements on every + /// platform. The slice is valid until the next execution, and + /// writes must go through the view. + pub fn constSlice(self: Self, scratch: []T) []const T { + std.debug.assert(scratch.len >= self.len); + if (self.inPlace()) |block| return block.firstReplica(); + self.copyTo(scratch[0..self.len]); + return scratch[0..self.len]; + } + + /// A slice to write the elements into, then `commit`, for writes + /// that compute elements in place. Contiguous buffers hand out their + /// first replica; others hand out `scratch`, which must hold `len` + /// elements on every platform. + pub fn edit(self: Self, scratch: []T) Edit { + std.debug.assert(scratch.len >= self.len); + return .{ + .view = self, + .items = if (self.inPlace()) |block| block.firstReplica() else scratch[0..self.len], + }; + } + + pub const Edit = struct { + view: Self, + /// Holds the previous values or garbage: write every element. + items: []T, + + /// Make the writes reach every replica: copy the first replica to + /// the others, or `items` into the view. + pub fn commit(self: Edit) void { + const block = self.view.inPlace() orelse return self.view.copyFrom(self.items); + for (1..block.replicas.len) |i| @memcpy(block.replica(i), self.items); + } + }; + + /// The view as a single block, when the buffer is contiguous. + fn inPlace(self: Self) ?Block { + if (self.len == 0 or !self.mapping.contiguous) return null; + var iterator = self.blocks(); + return iterator.next().?; + } + + /// Visit runs of consecutive logical elements in global row-major + /// order, without padding, grouping replicas of each logical range. + /// Shards must cover the view with disjoint regions or exact replicas. + pub fn blocks(self: Self) BlockIterator { + return .init(self); + } + + /// Visit the same blocks in reverse order. Elements within each + /// replica slice remain in ascending logical order. + pub fn reverseBlocks(self: Self) ReverseBlockIterator { + return .init(self); + } + + /// A run of elements that are consecutive in the view's row-major order and + /// contiguous in memory, so each replica holds them as a plain slice. + /// Writes must reach every replica; reads use the first. Borrows the buffer's + /// host pointers, like the view it comes from. + pub const Block = struct { + /// Logical position of the first element, relative to this view. + offset: usize, + len: usize, + /// Byte offset of the first element in the allocation of every replica. + byte_offset: usize, + /// Host pointer of each replica's allocation. + replicas: []const [*]u8, + + /// The block's elements in its `i`-th replica. + pub fn replica(self: Block, i: usize) []T { + const ptr: [*]T = @ptrCast(@alignCast(self.replicas[i] + self.byte_offset)); + return ptr[0..self.len]; + } + + /// The elements in the first replica, which reads use. + pub fn firstReplica(self: Block) []T { + return self.replica(0); + } + }; + + pub const BlockIterator = OrderedBlockIterator(false); + pub const ReverseBlockIterator = OrderedBlockIterator(true); + + fn OrderedBlockIterator(comptime reverse: bool) type { + return struct { + view: Self, + /// Next logical offset, or one past it for reverse traversal. + offset: usize, + /// Coordinates of the next block's first (or reverse last) element. + global: Coordinates, + + const Iterator = @This(); + + fn init(v: Self) Iterator { + return .{ + .view = v, + .offset = if (reverse) v.len else 0, + .global = if (v.len == 0 or v.mapping.contiguous) .zero else .unflatten( + v.mapping.packed_shape, + v.start + (if (reverse) v.len - 1 else 0), + ), + }; + } + + pub fn next(self: *Iterator) ?Block { + const v = &self.view; + if (if (reverse) self.offset == 0 else self.offset == v.len) return null; + if (v.mapping.contiguous) { + // The whole view is one run in every replica. + self.offset = if (reverse) 0 else v.len; + return v.blockAt(v.mapping.allocations.regions.get(0), 0, v.len, v.start * @sizeOf(T)); + } + const location = v.locate(self.global); + const bounds = v.blockBounds(location.offset); + const len = if (reverse) + @min(location.offset + 1 - bounds.start, self.offset) + else + @min(bounds.end - location.offset, v.len - self.offset); + // A block is physically contiguous, so its last element's + // address also gives its start without decoding coordinates again. + const byte_offset = v.mapping.layout_metadata.byteOffset(location.offset, location.local) - + (if (reverse) (len - 1) * v.mapping.layout_metadata.element_size else 0); + const offset = if (reverse) self.offset - len else self.offset; + self.offset = if (reverse) offset else offset + len; + if (if (reverse) self.offset != 0 else self.offset != v.len) { + self.global.move(reverse, v.mapping.packed_shape, len); + } + return v.blockAt(location.region, offset, len, byte_offset); + } + }; + } + + const Bounds = struct { start: usize, end: usize }; + + /// The block containing a shard-local element: elements contiguous in both + /// physical memory and global row-major order. Bounds are shard-local + /// row-major indices, not physical memory offsets. + /// + /// Example: an unsharded [2,5] array with row-major 2x2 tiles. + /// + /// 1. Logical indices, with tile boundaries. `_` denotes padding: + /// ```text + /// tile 0 tile 1 tile 2 + /// +-----+ +-----+ +-----+ + /// | 0 1 | | 2 3 | | 4 _ | + /// | 5 6 | | 7 8 | | 9 _ | + /// +-----+ +-----+ +-----+ + /// ``` + /// 2. Tiles are stored consecutively, each in row-major order: + /// ```text + /// [0 1 5 6] [2 3 7 8] [4 _ 9 _] + /// ``` + /// Thus 0 and 1 are adjacent in memory, but 1 and 2 are not. + /// `block_size = 2` is the maximum contiguous logical run. + /// + /// 3. Align runs within each logical row, clipping before padding: + /// ```text + /// last_dim_size = 5 + /// alignment_size = max(5, 2) = 5 + /// + /// alignment_start blocks of logical elements row end + /// 0 [0 1] [2 3] [4] 5 + /// 5 [5 6] [7 8] [9] 10 + /// actual block sizes: 2 2 1 + /// + /// local alignment_start start end returned bounds + /// 4 0 4 5 [4,5) + /// 5 5 5 7 [5,7) + /// 6 5 5 7 [5,7) + /// 9 5 9 10 [9,10) + /// ``` + /// For local=6, start = 6 - (6-5)%2 = 5, end = min(5+2, 5+5) = 7. + /// For local=4, end = min(4+2, 0+5) = 5: the final run is shortened. + /// + /// Without tiles, the same dense row-major array has block_size=10 + /// and alignment_size=10: one block [0,10) spanning both rows. + fn blockBounds(self: *const Self, local: usize) Bounds { + const block_size = self.mapping.layout_metadata.block_size; + // Runs shorter than a row align within that row. Larger runs + // span a whole number of rows, so alignment uses the run itself. + const alignment_size = self.mapping.layout_metadata.alignment_size; + const alignment_start = local - local % alignment_size; + // Round the position relative to alignment_start down to a run boundary. + const start = local - (local - alignment_start) % block_size; + // A row's final run may be shorter; stop at the alignment interval's end. + return .{ .start = start, .end = @min(start + block_size, alignment_start + alignment_size) }; + } + + fn blockAt(self: *const Self, region: HostPinnedAllocations.Region, offset: usize, len: usize, byte_offset: usize) Block { + return .{ + .offset = offset, + .len = len, + .byte_offset = byte_offset, + .replicas = self.mapping.allocations.replicas(region), + }; + } + + fn itemAt(ptr: [*]u8, byte_offset: usize) *T { + return @ptrCast(@alignCast(ptr + byte_offset)); + } + + const Location = struct { + region: HostPinnedAllocations.Region, + /// Coordinates of the element within its shard. + local: Coordinates, + /// Row-major index of the element within its shard. + offset: usize, + }; + + /// Locate the region holding an element, retaining its shard-local + /// coordinates for physical addressing. + fn locate(self: *const Self, global: Coordinates) Location { + const origins = self.mapping.allocations.entries.items(.origin); + for (self.mapping.allocations.regions.constSlice()) |region| { + const origin = origins[region.start]; + var local: Coordinates = .zero; + var offset: usize = 0; + for (self.mapping.shard_shape.dims(), 0..) |dim_, axis| { + const dim: usize = @intCast(dim_); + if (global.axes[axis] < origin.axes[axis] or global.axes[axis] - origin.axes[axis] >= dim) break; + local.axes[axis] = global.axes[axis] - origin.axes[axis]; + offset = offset * dim + local.axes[axis]; + } else { + return .{ .region = region, .local = local, .offset = offset }; + } + } + @panic("The union of all regions should cover the global shape."); + } + }; +} + +/// A position in an array, one coordinate per axis. Axes past the rank stay zero. +const Coordinates = struct { + axes: [Shape.MAX_RANK]usize, + + const zero: Coordinates = .{ .axes = @splat(0) }; + + /// The global coordinates of the first element of a device's shard. + fn ofShard(placement: *const Sharding.Placement, device_index: usize, shape: Shape) Coordinates { + // TODO: derive shard coordinates directly from DeviceGrid instead of reconstructing slices. + var result: Coordinates = .zero; + for (placement.slices(device_index, shape).constSlice(), 0..) |slice, axis| result.axes[axis] = @intCast(slice.start); + return result; + } + + /// The coordinates of the element at `offset` of `shape`, in row-major order. + fn unflatten(shape_: Shape, offset: usize) Coordinates { + var result: Coordinates = .zero; + var remaining = offset; + var axis = shape_.rank(); + while (axis > 0) { + axis -= 1; + const dim: usize = @intCast(shape_.dim(axis)); + result.axes[axis] = remaining % dim; + remaining /= dim; + } + return result; + } + + /// Whether the coordinates match on the axes of `shape`. + fn eqlWithin(self: Coordinates, other: Coordinates, shape_: Shape) bool { + return std.mem.eql(usize, self.axes[0..shape_.rank()], other.axes[0..shape_.rank()]); + } + + /// Advance in row-major order, decoding only axes that carry or borrow. + /// The resulting coordinates must still be within the shape. + fn move(self: *Coordinates, comptime reverse: bool, shape_: Shape, count: usize) void { + const axes = &self.axes; + var carry = count; + var axis = shape_.rank(); + while (carry != 0 and axis > 0) { + axis -= 1; + const dim: usize = @intCast(shape_.dim(axis)); + if (reverse) { + if (carry <= axes[axis]) { + axes[axis] -= carry; + return; + } + carry -= axes[axis] + 1; + axes[axis] = dim - 1 - carry % dim; + } else { + const available = dim - axes[axis]; + if (carry < available) { + axes[axis] += carry; + return; + } + carry -= available; + axes[axis] = carry % dim; + } + carry = carry / dim + 1; + } + std.debug.assert(carry == 0); + } +}; + +/// Physical addressing and logical block boundaries prepared once per HostAccessible. +/// All metadata is stored by value; the source layout is needed only during init. +const LayoutMetadata = struct { + rank: u4, + element_size: usize, + block_size: usize, + alignment_size: usize, + addressing: Addressing, + + const Addressing = union(enum) { + row_major, + strided: ByteStrides, + tiled: Tiled, + + const ByteStrides = [Shape.MAX_RANK]usize; + + const Tiled = struct { + /// Unit and padding dimensions always hold index 0 and are left out. + digits: stdx.BoundedArray(Digit, Shape.MAX_RANK + pjrt.DefaultMemoryLayout.MAX_TILE_DIMS) = .empty, + + /// One physical dimension of a tiled layout, as a digit of one coordinate. + const Digit = struct { + byte_stride: usize, + /// Divides the coordinate, unless `div_shift` replaces it for a power of two. + divisor: usize, + /// A mask when `is_mask` is set; otherwise a general modulus. + modulus: usize, + axis: u3, + div_shift: ?std.math.Log2Int(usize), + is_mask: bool, + + fn init(axis: u3, dim: TiledDims.Dim, byte_stride: usize) Digit { + // Unbounded digits keep every bit of the quotient. + const modulus = if (dim.modulus == 0) std.math.maxInt(usize) else dim.modulus; + const is_mask = dim.modulus == 0 or std.math.isPowerOfTwo(modulus); + return .{ + .byte_stride = byte_stride, + .divisor = dim.divisor, + .modulus = if (is_mask and dim.modulus != 0) modulus - 1 else modulus, + .axis = axis, + .div_shift = if (std.math.isPowerOfTwo(dim.divisor)) @intCast(@ctz(dim.divisor)) else null, + .is_mask = is_mask, + }; + } + + fn value(self: Digit, coordinate: usize) usize { + const quotient = if (self.div_shift) |shift| coordinate >> shift else coordinate / self.divisor; + return if (self.is_mask) quotient & self.modulus else quotient % self.modulus; + } + }; + }; + }; + + fn init(shape: Shape, shard_shape: Shape, layout: pjrt.MemoryLayout) LayoutMetadata { + const last_dim_size: usize = if (shard_shape.rank() == 0) 1 else @intCast(shard_shape.dim(-1)); + const element_size: usize = shard_shape.dtype().sizeOf(); + + // Longest stretch where stepping to the next local index also steps to the next element in memory. + // Suppose a [2,5] shard tiled by (2,2): tiles are stored one after another, each row-major. + // With local indices the memory holds: + // `0 1 5 6 | 2 3 7 8 | 4 _ 9 _` + // 0 and 1 are adjacent but 1 and 2 aren't, so the run is 2. A row-major layout is a single run. + const addressing: Addressing, const physical_run: usize = addr: switch (layout) { + .strides => |strides| { + std.debug.assert(strides.byte_strides.len == shard_shape.rank()); + + const row_major = shard_shape.computeByteStrides(); + if (std.mem.eql(i64, strides.byte_strides, row_major.constSlice())) break :addr .{ .row_major, shard_shape.count() }; + + var byte_strides: Addressing.ByteStrides = @splat(0); + for (strides.byte_strides, 0..) |stride, axis| { + stdx.debug.assert(stride >= 0, "negative destination byte strides are unsupported, got {}", .{stride}); + byte_strides[axis] = @intCast(stride); + } + + const physical_run = + // Rows padded to an alignment keep their elements adjacent: a run is a row. + if (byte_strides[shard_shape.rank() - 1] == element_size) last_dim_size + // Permuted axes, e.g. a column-major [3,4] f32 with strides [4,12], put the + // innermost axis elsewhere in memory, so no two elements of a row are adjacent. + else 1; + break :addr .{ .{ .strided = byte_strides }, physical_run }; + }, + .tiled => |tiled| { + stdx.debug.assert(tiled.minor_to_major.len == shard_shape.rank(), "layout rank {} doesn't match shape rank {}", .{ tiled.minor_to_major.len, shard_shape.rank() }); + + if (tiled.tile_dims.len == 0 and isMinorToMajorRowMajor(tiled.minor_to_major, shard_shape.rank())) break :addr .{ .row_major, shard_shape.count() }; + + // The expanded dimensions are laid out row-major. + const expanded: TiledDims = .init(shard_shape, tiled); + var tiled_addressing: Addressing.Tiled = .{}; + var byte_stride = element_size; + // From the most minor physical dimension, the run grows while each + // dimension is the next digit of the innermost axis. + const last_axis = shard_shape.rank() -| 1; + var physical_run: usize = 1; + var run_grows = true; + var i = expanded.dims.len; + while (i > 0) { + i -= 1; + const dim = expanded.dims.get(i); + if (dim.size > 1) { + if (dim.axis) |axis| tiled_addressing.digits.appendAssumeCapacity(.init(axis, dim, byte_stride)); + if (run_grows) { + const next_digit = if (dim.axis) |axis| axis == last_axis and dim.divisor == physical_run else false; + if (!next_digit) { + run_grows = false; + } else if (dim.modulus == 0) { + // An unbounded digit holds the rest of the axis. + physical_run = last_dim_size; + run_grows = false; + } else { + physical_run *= dim.modulus; + // Padding after the digit's last value ends the run. + run_grows = dim.size == dim.modulus; + } + } + } + byte_stride *= dim.size; + } + physical_run = @min(physical_run, last_dim_size); + + break :addr .{ .{ .tiled = tiled_addressing }, physical_run }; + }, + }; + + // Longest stretch where stepping to the next local index also steps to the next global index. + // Suppose a global [2,6] split into two [2,3] shards, each holding half the columns. + // With global indices the shards hold: + // - shard 1: `0 1 2 | 6 7 8` + // - shard 2: `3 4 5 | 9 10 11` + // 2 and 6 are adjacent in shard 1 but not globally, so the run is 3. + const shard_run = run: { + var run: usize = 1; + var axis = shard_shape.rank(); + while (axis > 0) { + axis -= 1; + run *= @intCast(shard_shape.dim(axis)); + if (shape.dim(axis) != shard_shape.dim(axis)) break; + } + break :run run; + }; + const block_size = if (shard_shape.count() == 0) 0 else @min(physical_run, shard_run); + + return .{ + .rank = shard_shape.rank(), + .element_size = element_size, + .block_size = block_size, + // See `View.blockBounds`. + .alignment_size = @max(last_dim_size, block_size), + .addressing = addressing, + }; + } + + /// The byte offset in its shard of the element at shard-local row-major index + /// `local` and coordinates `coordinates`: row-major layouts use the index, + /// the others the coordinates. + fn byteOffset(self: *const LayoutMetadata, local: usize, coordinates: Coordinates) usize { + switch (self.addressing) { + .row_major => return local * self.element_size, + .strided => |*byte_strides| { + var offset: usize = 0; + for (coordinates.axes[0..self.rank], byte_strides[0..self.rank]) |coord, byte_stride| offset += coord * byte_stride; + return offset; + }, + .tiled => |*tiled| { + var offset: usize = 0; + for (tiled.digits.constSlice()) |digit| offset += digit.value(coordinates.axes[digit.axis]) * digit.byte_stride; + return offset; + }, + } + } +}; + +/// Whether a PJRT minor-to-major order lays the axes out row-major, like ZML shapes. +fn isMinorToMajorRowMajor(minor_to_major: []const i64, rank: usize) bool { + if (minor_to_major.len != rank) return false; + for (minor_to_major, 0..) |axis, i| { + if (axis != @as(i64, @intCast(rank - i - 1))) return false; + } + return true; +} + +/// The physical dimensions of a tiled layout, major to minor. XLA lays a tiled array +/// out row-major over these dimensions; see https://openxla.org/xla/tiled_layout. +const TiledDims = struct { + dims: stdx.BoundedArray(Dim, Shape.MAX_RANK + 2 * pjrt.DefaultMemoryLayout.MAX_TILE_DIMS) = .empty, + + /// Every dimension holds the digit `(coordinate[axis] / divisor) % modulus`. + const Dim = struct { + /// Null for padding, which only holds index 0. + axis: ?u3, + size: usize, + divisor: usize = 1, + /// Zero for a digit without an upper bound, which holds `coordinate / divisor`. + modulus: usize = 0, + + /// The index of the tile along this dimension. + fn tileCount(self: Dim, tile: usize) Dim { + const size = std.math.divCeil(usize, self.size, tile) catch unreachable; + const axis = self.axis orelse return .{ .axis = null, .size = size }; + if (self.modulus == 0) return .{ .axis = axis, .size = size, .divisor = self.divisor * tile }; + if (self.modulus % tile == 0) return .{ .axis = axis, .size = size, .divisor = self.divisor * tile, .modulus = self.modulus / tile }; + // One tile holds every value of the digit. + return .{ .axis = null, .size = size }; + } + + /// The index within the tile along this dimension. + fn tileIndex(self: Dim, tile: usize) Dim { + const axis = self.axis orelse return .{ .axis = null, .size = tile }; + // XLA assumes nested tiles add no padding. + stdx.debug.assert( + self.modulus == 0 or self.modulus % tile == 0 or tile % self.modulus == 0, + "nested tile {} must divide, or be a multiple of, the {} values it tiles", + .{ tile, self.modulus }, + ); + if (self.modulus != 0 and self.modulus < tile) return .{ .axis = axis, .size = tile, .divisor = self.divisor, .modulus = self.modulus }; + return .{ .axis = axis, .size = tile, .divisor = self.divisor, .modulus = tile }; + } + }; + + /// Follows XLA's LayoutUtil::LinearIndexForNestedTiling. The physical dimensions + /// start in minor_to_major order. Each tile then splits the most minor ones, + /// padded with leading unit dimensions if the tile has more, into tile counts + /// followed by indices within the tile. Tile sizes are listed major to minor, so + /// a later tile splits the previous tile's indices and possibly its counts. + /// + /// Two consequences are easy to get wrong: + /// - `tile_dims[j]` of a k-dimensional tile belongs to `minor_to_major[k-1-j]`, + /// not to `minor_to_major[j]` nor to logical axis j. TPU's default + /// `{2,1,0:T(8,128)}` for rank-3 32-bit arrays tiles rows by 8 and columns + /// by 128. + /// - A later tile splits the previous tile's dimensions, not the grid of + /// tiles: `(8,128)(2,1)`, the bf16 packing, interleaves pairs of rows + /// within each tile. TPU tiles bf16 and 8-bit arrays this way. + fn init(shape: Shape, layout: pjrt.MemoryLayout.Tiled) TiledDims { + const rank = shape.rank(); + stdx.debug.assert(layout.minor_to_major.len == rank, "layout rank {} doesn't match shape rank {}", .{ layout.minor_to_major.len, rank }); + var result: TiledDims = .{}; + var i = rank; + while (i > 0) { + i -= 1; + const axis: u3 = @intCast(layout.minor_to_major[i]); + result.dims.appendAssumeCapacity(.{ .axis = axis, .size = @intCast(shape.dim(axis)) }); + } + var cursor: usize = 0; + for (layout.tile_dims_sizes) |tile_rank| { + const tile = layout.tile_dims[cursor..][0..tile_rank]; + cursor += tile_rank; + while (result.dims.len < tile_rank) result.dims.insert(0, .{ .axis = null, .size = 1 }) catch unreachable; + const start = result.dims.len - tile_rank; + for (tile, start..) |size_, d| { + // XLA marks dimensions combined with the next one by a negative size. + stdx.debug.assert(size_ > 0, "unsupported tile dimension {}", .{size_}); + const size: usize = @intCast(size_); + result.dims.appendAssumeCapacity(result.dims.get(d).tileIndex(size)); + result.dims.set(d, result.dims.get(d).tileCount(size)); + } + } + stdx.debug.assert(cursor == layout.tile_dims.len, "layout tile metadata is inconsistent", .{}); + return result; + } +}; + +// +// ============= +// === TESTS === +// ============= +// + +/// The byte offset of an element, computed from the layout directly to check `LayoutMetadata`. +fn referenceByteOffset(shape_: Shape, layout: pjrt.MemoryLayout, index: []const usize) usize { + if (!builtin.is_test) @compileError("referenceByteOffset is only available in tests"); + + return switch (layout) { + .tiled => |tiled| tiledElementOffset(shape_, tiled, index) * shape_.dtype().sizeOf(), + .strides => |strides| b: { + var byte_offset: i64 = 0; + for (index, strides.byte_strides) |coord, stride| { + stdx.debug.assert(stride >= 0, "negative destination byte strides are unsupported, got {}", .{stride}); + byte_offset += @as(i64, @intCast(coord)) * stride; + } + stdx.debug.assert(byte_offset >= 0, "destination byte offset must be non-negative, got {}", .{byte_offset}); + break :b @as(usize, @intCast(byte_offset)); + }, + }; +} + +/// Computes the element offset for one logical index in XLA/PJRT tiled layout order, +/// as XLA's LayoutUtil::LinearIndexForNestedTiling does. A direct port of that +/// function, kept independent of `TiledDims`: a reference sharing the prepared +/// model of tiling once let tests miss offsets that disagreed with XLA. +fn tiledElementOffset(shape_: Shape, layout: pjrt.MemoryLayout.Tiled, index: []const usize) usize { + if (!builtin.is_test) @compileError("tiledElementOffset is only available in tests"); + + const rank = shape_.rank(); + stdx.debug.assert(layout.minor_to_major.len == rank, "layout rank {} doesn't match shape rank {}", .{ layout.minor_to_major.len, rank }); + + // Physical dimensions and indices, major to minor. + const Expanded = stdx.BoundedArray(usize, Shape.MAX_RANK + 2 * pjrt.DefaultMemoryLayout.MAX_TILE_DIMS); + var dims: Expanded = .empty; + var indices: Expanded = .empty; + var i = rank; + while (i > 0) { + i -= 1; + const axis: usize = @intCast(layout.minor_to_major[i]); + dims.appendAssumeCapacity(@intCast(shape_.dim(axis))); + indices.appendAssumeCapacity(index[axis]); + } + + var cursor: usize = 0; + for (layout.tile_dims_sizes) |tile_rank| { + const tile = layout.tile_dims[cursor..][0..tile_rank]; + cursor += tile_rank; + while (dims.len < tile_rank) { + dims.insert(0, 1) catch unreachable; + indices.insert(0, 0) catch unreachable; + } + // Tiled dimensions become tile counts, followed by the indices within the tile. + const start = dims.len - tile_rank; + for (tile, start..) |tile_dim_, d| { + const tile_dim: usize = @intCast(tile_dim_); + dims.appendAssumeCapacity(tile_dim); + indices.appendAssumeCapacity(indices.get(d) % tile_dim); + dims.set(d, std.math.divCeil(usize, dims.get(d), tile_dim) catch unreachable); + indices.set(d, indices.get(d) / tile_dim); + } + } + stdx.debug.assert(cursor == layout.tile_dims.len, "layout tile metadata is inconsistent", .{}); + + var offset: usize = 0; + for (dims.constSlice(), indices.constSlice()) |dim, coord| offset = offset * dim + coord; + return offset; +} + +test "LayoutMetadata byte offsets match physical layouts" { + const unit_tiles: [pjrt.DefaultMemoryLayout.MAX_TILE_DIMS]i64 = @splat(1); + const tile_ranks: [pjrt.DefaultMemoryLayout.MAX_NUM_TILES]usize = @splat(Shape.MAX_RANK); + const Case = struct { shape: Shape, layout: pjrt.MemoryLayout, block_size: usize }; + const cases = [_]Case{ + .{ .shape = .init(.{}, .i32), .layout = .{ .strides = .{ .byte_strides = &.{} } }, .block_size = 1 }, + .{ .shape = .init(.{ 3, 5 }, .i32), .layout = .{ .strides = .{ .byte_strides = &.{ 20, 4 } } }, .block_size = 15 }, + .{ .shape = .init(.{ 3, 5 }, .f16), .layout = .{ .strides = .{ .byte_strides = &.{ 16, 2 } } }, .block_size = 5 }, + .{ .shape = .init(.{ 3, 5 }, .i32), .layout = .{ .strides = .{ .byte_strides = &.{ 4, 16 } } }, .block_size = 1 }, + .{ + .shape = .init(.{ 3, 5 }, .i32), + .layout = .{ .tiled = .{ .minor_to_major = &.{ 1, 0 }, .tile_dims = &.{}, .tile_dims_sizes = &.{} } }, + .block_size = 15, + }, + .{ + .shape = .init(.{ 3, 5 }, .i32), + .layout = .{ .tiled = .{ .minor_to_major = &.{ 0, 1 }, .tile_dims = &.{}, .tile_dims_sizes = &.{} } }, + .block_size = 1, + }, + .{ + .shape = .init(.{ 3, 5 }, .i32), + .layout = .{ .tiled = .{ .minor_to_major = &.{ 1, 0 }, .tile_dims = &.{ 2, 2 }, .tile_dims_sizes = &.{2} } }, + .block_size = 2, + }, + .{ + .shape = .init(.{ 3, 5 }, .i32), + .layout = .{ .tiled = .{ .minor_to_major = &.{ 0, 1 }, .tile_dims = &.{ 2, 2 }, .tile_dims_sizes = &.{2} } }, + .block_size = 1, + }, + .{ + .shape = .init(.{ 3, 5 }, .f16), + .layout = .{ .tiled = .{ .minor_to_major = &.{ 1, 0 }, .tile_dims = &.{ 2, 2, 2, 1 }, .tile_dims_sizes = &.{ 2, 2 } } }, + .block_size = 1, + }, + .{ + .shape = .init(.{ 4, 8 }, .f16), + .layout = .{ .tiled = .{ .minor_to_major = &.{ 1, 0 }, .tile_dims = &.{ 2, 4, 2, 1 }, .tile_dims_sizes = &.{ 2, 2 } } }, + .block_size = 1, + }, + // The nested tile is larger than the values it tiles. + .{ + .shape = .init(.{ 3, 5 }, .i32), + .layout = .{ .tiled = .{ .minor_to_major = &.{ 1, 0 }, .tile_dims = &.{ 2, 2, 4, 1 }, .tile_dims_sizes = &.{ 2, 2 } } }, + .block_size = 1, + }, + // The nested tile also splits the tile counts. + .{ + .shape = .init(.{ 4, 8 }, .i32), + .layout = .{ .tiled = .{ .minor_to_major = &.{ 1, 0 }, .tile_dims = &.{ 1, 4, 2, 1, 1 }, .tile_dims_sizes = &.{ 2, 3 } } }, + .block_size = 1, + }, + .{ + .shape = .init(.{ 2, 3, 5 }, .i32), + .layout = .{ .tiled = .{ .minor_to_major = &.{ 2, 1, 0 }, .tile_dims = &.{ 4, 2 }, .tile_dims_sizes = &.{2} } }, + .block_size = 2, + }, + .{ + .shape = .init(.{ 2, 3, 5 }, .i32), + .layout = .{ .tiled = .{ .minor_to_major = &.{ 2, 1, 0 }, .tile_dims = &.{ 3, 2 }, .tile_dims_sizes = &.{2} } }, + .block_size = 2, + }, + .{ + .shape = .init(.{ 2, 3, 5 }, .i32), + .layout = .{ .tiled = .{ .minor_to_major = &.{ 2, 1, 0 }, .tile_dims = &.{ 3, 2, 3, 1 }, .tile_dims_sizes = &.{ 2, 2 } } }, + .block_size = 1, + }, + // The nested tile moves pairs of the innermost axis to the most minor dimension. + .{ + .shape = .init(.{ 2, 3, 5 }, .i32), + .layout = .{ .tiled = .{ .minor_to_major = &.{ 0, 2, 1 }, .tile_dims = &.{ 2, 4, 2, 2, 1 }, .tile_dims_sizes = &.{ 3, 2 } } }, + .block_size = 2, + }, + // The tile has more dimensions than the shape. + .{ + .shape = .init(.{5}, .i32), + .layout = .{ .tiled = .{ .minor_to_major = &.{0}, .tile_dims = &.{ 2, 4 }, .tile_dims_sizes = &.{2} } }, + .block_size = 4, + }, + .{ + .shape = .init(.{ 1, 1, 1, 1, 1, 1, 2, 3 }, .i32), + .layout = .{ .tiled = .{ .minor_to_major = &.{ 7, 6, 5, 4, 3, 2, 1, 0 }, .tile_dims = &unit_tiles, .tile_dims_sizes = &tile_ranks } }, + .block_size = 3, + }, + }; + for (cases) |case| { + const metadata: LayoutMetadata = .init(case.shape, case.shape, case.layout); + try std.testing.expectEqual(case.block_size, metadata.block_size); + for (0..case.shape.count()) |local| { + const index = Coordinates.unflatten(case.shape, local); + const expected = referenceByteOffset(case.shape, case.layout, index.axes[0..case.shape.rank()]); + const actual = metadata.byteOffset(local, index); + try std.testing.expectEqual(expected, actual); + // Elements after the first of a block follow the previous one in memory. + if ((local % metadata.alignment_size) % metadata.block_size != 0) { + try std.testing.expectEqual(metadata.byteOffset(local - 1, Coordinates.unflatten(case.shape, local - 1)) + metadata.element_size, actual); + } + } + } +} + +test "tiled layout offsets follow XLA" { + const Tiled = pjrt.MemoryLayout.Tiled; + // Figure 1 of https://openxla.org/xla/tiled_layout: F32[3,5]{1,0:T(2,2)}. + const tiles: Tiled = .{ .minor_to_major = &.{ 1, 0 }, .tile_dims = &.{ 2, 2 }, .tile_dims_sizes = &.{2} }; + try std.testing.expectEqual(17, tiledElementOffset(.init(.{ 3, 5 }, .f32), tiles, &.{ 2, 3 })); + const shape_ = Shape.init(.{ 3, 4 }, .i32); + try std.testing.expectEqual(0, tiledElementOffset(shape_, tiles, &.{ 0, 0 })); + try std.testing.expectEqual(1, tiledElementOffset(shape_, tiles, &.{ 0, 1 })); + try std.testing.expectEqual(2, tiledElementOffset(shape_, tiles, &.{ 1, 0 })); + try std.testing.expectEqual(3, tiledElementOffset(shape_, tiles, &.{ 1, 1 })); + try std.testing.expectEqual(4, tiledElementOffset(shape_, tiles, &.{ 0, 2 })); + try std.testing.expectEqual(8, tiledElementOffset(shape_, tiles, &.{ 2, 0 })); + + // Figure 2: 4x8 tiled by (2,4)(2,1). The nested tile pairs rows within each + // tile, as the (8,128)(2,1) tiling of bf16 on TPU does. + const nested: Tiled = .{ .minor_to_major = &.{ 1, 0 }, .tile_dims = &.{ 2, 4, 2, 1 }, .tile_dims_sizes = &.{ 2, 2 } }; + const figure2 = [4][8]usize{ + .{ 0, 2, 4, 6, 8, 10, 12, 14 }, + .{ 1, 3, 5, 7, 9, 11, 13, 15 }, + .{ 16, 18, 20, 22, 24, 26, 28, 30 }, + .{ 17, 19, 21, 23, 25, 27, 29, 31 }, + }; + for (figure2, 0..) |offsets, row| { + for (offsets, 0..) |offset, col| try std.testing.expectEqual(offset, tiledElementOffset(.init(.{ 4, 8 }, .bf16), nested, &.{ row, col })); + } + + // A tile lists sizes for the most minor physical dimensions, major to minor. + // Like {2,1,0:T(8,128)} on TPU, this tiles rows by 2 and columns by 4: + // offset = a*32 + (b/2)*16 + (c/4)*8 + (b%2)*4 + c%4. + const rank3: Shape = .init(.{ 2, 3, 5 }, .f32); + const partial: Tiled = .{ .minor_to_major = &.{ 2, 1, 0 }, .tile_dims = &.{ 2, 4 }, .tile_dims_sizes = &.{2} }; + try std.testing.expectEqual(4, tiledElementOffset(rank3, partial, &.{ 0, 1, 0 })); + try std.testing.expectEqual(8, tiledElementOffset(rank3, partial, &.{ 0, 0, 4 })); + try std.testing.expectEqual(12, tiledElementOffset(rank3, partial, &.{ 0, 1, 4 })); + try std.testing.expectEqual(16, tiledElementOffset(rank3, partial, &.{ 0, 2, 0 })); + try std.testing.expectEqual(51, tiledElementOffset(rank3, partial, &.{ 1, 2, 3 })); + + // Tiles follow the physical order: offset = (c/2)*8 + (r/2)*4 + (c%2)*2 + r%2. + const transposed: Tiled = .{ .minor_to_major = &.{ 0, 1 }, .tile_dims = &.{ 2, 2 }, .tile_dims_sizes = &.{2} }; + try std.testing.expectEqual(1, tiledElementOffset(.init(.{ 3, 5 }, .f32), transposed, &.{ 1, 0 })); + try std.testing.expectEqual(2, tiledElementOffset(.init(.{ 3, 5 }, .f32), transposed, &.{ 0, 1 })); + try std.testing.expectEqual(4, tiledElementOffset(.init(.{ 3, 5 }, .f32), transposed, &.{ 2, 0 })); + try std.testing.expectEqual(8, tiledElementOffset(.init(.{ 3, 5 }, .f32), transposed, &.{ 0, 2 })); + try std.testing.expectEqual(14, tiledElementOffset(.init(.{ 3, 5 }, .f32), transposed, &.{ 2, 3 })); + + // A tile with more dimensions than the shape pads it with leading unit + // dimensions: offset = (c/4)*8 + c%4. + const wide: Tiled = .{ .minor_to_major = &.{0}, .tile_dims = &.{ 2, 4 }, .tile_dims_sizes = &.{2} }; + try std.testing.expectEqual(3, tiledElementOffset(.init(.{5}, .f32), wide, &.{3})); + try std.testing.expectEqual(8, tiledElementOffset(.init(.{5}, .f32), wide, &.{4})); + + // A later tile can also split the previous tile counts: + // offset = r*8 + (c%4)*2 + (c/4)%2. + const cross_tile: Tiled = .{ .minor_to_major = &.{ 1, 0 }, .tile_dims = &.{ 1, 4, 2, 1, 1 }, .tile_dims_sizes = &.{ 2, 3 } }; + try std.testing.expectEqual(1, tiledElementOffset(.init(.{ 4, 8 }, .f32), cross_tile, &.{ 0, 4 })); + try std.testing.expectEqual(2, tiledElementOffset(.init(.{ 4, 8 }, .f32), cross_tile, &.{ 0, 1 })); + try std.testing.expectEqual(11, tiledElementOffset(.init(.{ 4, 8 }, .f32), cross_tile, &.{ 1, 5 })); + try std.testing.expectEqual(31, tiledElementOffset(.init(.{ 4, 8 }, .f32), cross_tile, &.{ 3, 7 })); +} + +test "HostAccessible.HostPinnedAllocations keeps each region's replicas together as allocations are added" { + var allocations: HostAccessible.HostPinnedAllocations = .empty; + defer allocations.deinit(std.testing.allocator); + try allocations.entries.ensureTotalCapacity(std.testing.allocator, 5); + var storage: [5]u8 = undefined; + const base: [*]u8 = &storage; + // A vector of 12 split into shards of 4, holding regions A, B, A, C, B: + // replicas arrive interleaved with other regions. + const shard_shape: Shape = .init(.{4}, .u8); + for ([_]usize{ 0, 4, 0, 8, 4 }, 0..) |row, i| { + allocations.add(shard_shape, .{ .origin = .{ .axes = .{row} ++ @as([Shape.MAX_RANK - 1]usize, @splat(0)) }, .ptr = base + i, .shard_index = undefined, .handle = undefined }); + } + + try std.testing.expectEqualSlices([*]u8, &.{ base, base + 2, base + 1, base + 4, base + 3 }, allocations.entries.items(.ptr)); + for (allocations.regions.constSlice(), [_][2]u8{ .{ 0, 2 }, .{ 2, 2 }, .{ 4, 1 } }) |region, expected| { + try std.testing.expectEqual(expected, [2]u8{ region.start, region.len }); + } +} + +test "HostAccessible.format prints the elements in row-major order" { + const shape_ = Shape.init(.{ 2, 2 }, .i32); + const column_major: pjrt.MemoryLayout = .{ .tiled = .{ .minor_to_major = &.{ 0, 1 }, .tile_dims = &.{}, .tile_dims_sizes = &.{} } }; + var storage = [_]i32{ 0, 2, 1, 3 }; + const pinned = try HostAccessible.initForTests(std.testing.allocator, shape_, shape_, column_major, &.{0}, &.{@ptrCast(&storage)}); + defer pinned.deinit(std.testing.allocator); + var buffer: [128]u8 = undefined; + const printed = try std.fmt.bufPrint(&buffer, "{f}", .{pinned}); + try std.testing.expectStringEndsWith(printed, ") { 0 1 2 3 }"); +} + +test "HostAccessible.View borrows the buffer's pointers through slicing" { + const shape_ = Shape.init(.{4}, .i32); + const layout: pjrt.MemoryLayout = .{ .strides = .{ .byte_strides = &.{4} } }; + var storage = [_]i32{ 0, 1, 2, 3 }; + var other = [_]i32{ 10, 11, 12, 13 }; + const pinned = try HostAccessible.initForTests(std.testing.allocator, shape_, shape_, layout, &.{0}, &.{@ptrCast(&storage)}); + defer pinned.deinit(std.testing.allocator); + const nested = pinned.view(i32).slice(1, null).slice(0, 2); + + // Slices read the pointers in place, which lets `await` refresh them + // without copying them into every view. + try std.testing.expectEqual(@as(*const HostAccessible.Mapping, pinned.mapping), nested.mapping); + pinned.mapping.allocations.entries.items(.ptr)[0] = @ptrCast(&other); + nested.fill(99); + try std.testing.expectEqualSlices(i32, &.{ 0, 1, 2, 3 }, &storage); + try std.testing.expectEqualSlices(i32, &.{ 10, 99, 99, 13 }, &other); +} + +test "HostAccessible.View blocks skip tile padding" { + const shape_ = Shape.init(.{ 3, 5 }, .i32); + const layout: pjrt.MemoryLayout = .{ .tiled = .{ + .minor_to_major = &.{ 1, 0 }, + .tile_dims = &.{ 2, 2 }, + .tile_dims_sizes = &.{2}, + } }; + var storage: [24]i32 = @splat(-1); + const ptrs = [_][*]u8{@ptrCast(&storage)}; + const pinned = try HostAccessible.initForTests(std.testing.allocator, shape_, shape_, layout, &.{0}, &ptrs); + defer pinned.deinit(std.testing.allocator); + const values = pinned.view(i32); + try std.testing.expectEqual(2, pinned.mapping.layout_metadata.block_size); + try std.testing.expectEqual(HostAccessible.View(i32).Bounds{ .start = 0, .end = 2 }, values.blockBounds(1)); + try std.testing.expectEqual(HostAccessible.View(i32).Bounds{ .start = 4, .end = 5 }, values.blockBounds(4)); + var iterator = values.blocks(); + while (iterator.next()) |block| { + for (0..block.replicas.len) |i| { + const items = block.replica(i); + for (items, block.offset..) |*value, index| value.* = @intCast(index + 1); + } + } + try std.testing.expectEqualSlices(i32, &.{ + 1, 2, 6, 7, 3, 4, 8, 9, 5, -1, 10, -1, + 11, 12, -1, -1, 13, 14, -1, -1, 15, -1, -1, -1, + }, &storage); + values.slice(4, 3).fill(90); + try std.testing.expectEqualSlices(i32, &.{ + 1, 2, 90, 90, 3, 4, 8, 9, 90, -1, 10, -1, + 11, 12, -1, -1, 13, 14, -1, -1, 15, -1, -1, -1, + }, &storage); +} + +test "HostAccessible.View nested tiles" { + const shape_ = Shape.init(.{ 3, 5 }, .i32); + const layout: pjrt.MemoryLayout = .{ .tiled = .{ + .minor_to_major = &.{ 1, 0 }, + .tile_dims = &.{ 2, 2, 2, 1 }, + .tile_dims_sizes = &.{ 2, 2 }, + } }; + var storage: [24]i32 = @splat(-1); + const ptrs = [_][*]u8{@ptrCast(&storage)}; + const pinned = try HostAccessible.initForTests(std.testing.allocator, shape_, shape_, layout, &.{0}, &ptrs); + defer pinned.deinit(std.testing.allocator); + const values = pinned.view(i32); + // Pairs of rows are interleaved within each tile, so every block is one element. + try std.testing.expectEqual(1, pinned.mapping.layout_metadata.block_size); + values.copyFrom(&.{ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }); + try std.testing.expectEqualSlices(i32, &.{ + 1, 6, 2, 7, 3, 8, 4, 9, 5, 10, -1, -1, + 11, -1, 12, -1, 13, -1, 14, -1, 15, -1, -1, -1, + }, &storage); +} + +test "HostAccessible.View dense, transposed, strided and partial-rank tiled layouts" { + const shape_ = Shape.init(.{ 2, 3 }, .i32); + const Case = struct { layout: pjrt.MemoryLayout, block_size: usize, expected: []const i32 }; + const cases = [_]Case{ + .{ + .layout = .{ .tiled = .{ .minor_to_major = &.{ 1, 0 }, .tile_dims = &.{}, .tile_dims_sizes = &.{} } }, + .block_size = 6, + .expected = &.{ 1, 2, 3, 4, 5, 6, -1, -1 }, + }, + .{ + .layout = .{ .tiled = .{ .minor_to_major = &.{ 0, 1 }, .tile_dims = &.{}, .tile_dims_sizes = &.{} } }, + .block_size = 1, + .expected = &.{ 1, 4, 2, 5, 3, 6, -1, -1 }, + }, + .{ + .layout = .{ .strides = .{ .byte_strides = &.{ 12, 4 } } }, + .block_size = 6, + .expected = &.{ 1, 2, 3, 4, 5, 6, -1, -1 }, + }, + .{ + .layout = .{ .strides = .{ .byte_strides = &.{ 20, 4 } } }, + .block_size = 3, + .expected = &.{ 1, 2, 3, -1, -1, 4, 5, 6 }, + }, + .{ + .layout = .{ .strides = .{ .byte_strides = &.{ 4, 12 } } }, + .block_size = 1, + .expected = &.{ 1, 4, -1, 2, 5, -1, 3, 6 }, + }, + .{ + .layout = .{ .tiled = .{ .minor_to_major = &.{ 1, 0 }, .tile_dims = &.{2}, .tile_dims_sizes = &.{1} } }, + .block_size = 3, + .expected = &.{ 1, 2, 3, -1, 4, 5, 6, -1 }, + }, + }; + for (cases) |case| { + var storage: [8]i32 = @splat(-1); + const ptrs = [_][*]u8{@ptrCast(&storage)}; + const pinned = try HostAccessible.initForTests(std.testing.allocator, shape_, shape_, case.layout, &.{0}, &ptrs); + defer pinned.deinit(std.testing.allocator); + const values = pinned.view(i32); + try std.testing.expectEqual(case.block_size, pinned.mapping.layout_metadata.block_size); + values.copyFrom(&.{ 1, 2, 3, 4, 5, 6 }); + try std.testing.expectEqualSlices(i32, case.expected, &storage); + } +} + +test "HostAccessible.View row-major layouts return one block across rows and replicas" { + const shape_: Shape = .init(.{ 3, 5 }, .i32); + const layouts = [_]pjrt.MemoryLayout{ + .{ .strides = .{ .byte_strides = &.{ 20, 4 } } }, + .{ .tiled = .{ .minor_to_major = &.{ 1, 0 }, .tile_dims = &.{}, .tile_dims_sizes = &.{} } }, + }; + for (layouts) |layout| { + var storage: [2][15]i32 = @splat(@splat(0)); + const ptrs = [_][*]u8{ @ptrCast(&storage[0]), @ptrCast(&storage[1]) }; + const pinned = try HostAccessible.initForTests(std.testing.allocator, shape_, shape_, layout, &.{ 0, 0 }, &ptrs); + defer pinned.deinit(std.testing.allocator); + const values = pinned.view(i32); + for ([_]HostAccessible.View(i32){ values, values.slice(1, null).slice(1, 10) }) |selected| { + var forward = selected.blocks(); + var reverse = selected.reverseBlocks(); + for ([_]HostAccessible.View(i32).Block{ forward.next().?, reverse.next().? }) |block| { + try std.testing.expectEqual(0, block.offset); + try std.testing.expectEqual(selected.len, block.len); + try std.testing.expectEqual(2, block.replicas.len); + for (0..block.replicas.len, &storage) |i, *replica| { + const items = block.replica(i); + try std.testing.expectEqual(replica[selected.start..].ptr, items.ptr); + try std.testing.expectEqual(selected.len, items.len); + } + } + try std.testing.expectEqual(null, forward.next()); + try std.testing.expectEqual(null, reverse.next()); + } + } +} + +test "HostAccessible.View dense shards stop at gaps in global row-major order" { + const Case = struct { + shape: Shape, + shard_shape: Shape, + block_size: usize, + origins: [2]usize, + expected: [2][]const i32, + }; + const cases = [_]Case{ + .{ + .shape = .init(.{ 2, 6 }, .i32), + .shard_shape = .init(.{ 2, 3 }, .i32), + .block_size = 3, + .origins = .{ 0, 3 }, + .expected = .{ &.{ 0, 1, 2, 6, 7, 8 }, &.{ 3, 4, 5, 9, 10, 11 } }, + }, + .{ + .shape = .init(.{ 2, 6, 2 }, .i32), + .shard_shape = .init(.{ 2, 3, 2 }, .i32), + .block_size = 6, + .origins = .{ 0, 6 }, + .expected = .{ + &.{ 0, 1, 2, 3, 4, 5, 12, 13, 14, 15, 16, 17 }, + &.{ 6, 7, 8, 9, 10, 11, 18, 19, 20, 21, 22, 23 }, + }, + }, + }; + for (cases) |case| { + const strides = case.shard_shape.computeByteStrides(); + var storage: [2][12]i32 = @splat(@splat(-1)); + var ptrs: [2][*]u8 = undefined; + for (&ptrs, &storage) |*ptr, *data| ptr.* = @ptrCast(data); + const pinned = try HostAccessible.initForTests(std.testing.allocator, case.shape, case.shard_shape, .{ .strides = .{ .byte_strides = strides.constSlice() } }, &case.origins, &ptrs); + defer pinned.deinit(std.testing.allocator); + const elements = pinned.view(i32); + try std.testing.expectEqual(case.block_size, pinned.mapping.layout_metadata.block_size); + try std.testing.expectEqual(case.block_size, elements.blockBounds(0).end); + var tail = elements.slice(case.block_size - 1, null).blocks(); + try std.testing.expectEqual(1, tail.next().?.len); + + elements.fillIota(0, 1); + for (storage, case.expected) |data, expected| { + try std.testing.expectEqualSlices(i32, expected, data[0..expected.len]); + } + var actual: [24]i32 = undefined; + elements.copyTo(actual[0..case.shape.count()]); + for (actual[0..case.shape.count()], 0..) |value, i| try std.testing.expectEqual(@as(i32, @intCast(i)), value); + + // A partial range crosses both a global gap and a physical shard boundary. + const start = case.block_size - 1; + const values = [_]i32{ 90, 91, 92, 93 }; + elements.slice(start, values.len).copyFrom(&values); + for (storage, case.expected) |data, expected| { + for (data[0..expected.len], expected) |value, global| { + const offset: usize = @intCast(global); + try std.testing.expectEqual(if (offset >= start and offset < start + values.len) values[offset - start] else global, value); + } + } + var copied: [4]i32 = undefined; + elements.slice(start, copied.len).copyTo(&copied); + try std.testing.expectEqualSlices(i32, &values, &copied); + } +} + +/// A 4x6 array split into 2x3 shards. Shards 4 to 7 are separate copies of shards 0 to 3. +const TestShards = struct { + storage: [8][8]i32 = @splat(@splat(-1)), + ptrs: [8][*]u8 = undefined, + origins: [8]usize = @splat(0), + pinned: ?HostAccessible = null, + + const shape_ = Shape.init(.{ 4, 6 }, .i32); + const shard_shape = Shape.init(.{ 2, 3 }, .i32); + const tiled: pjrt.MemoryLayout = .{ .tiled = .{ + .minor_to_major = &.{ 1, 0 }, + .tile_dims = &.{ 2, 2 }, + .tile_dims_sizes = &.{2}, + } }; + + fn deinit(self: *TestShards) void { + if (self.pinned) |pinned| pinned.deinit(std.testing.allocator); + } + + /// The view borrows `pinned`, which borrows the storage: the storage must + /// stay in place, and the next call replaces `pinned`. + fn view(self: *TestShards, layout: pjrt.MemoryLayout) !HostAccessible.View(i32) { + for (&self.ptrs, &self.origins, &self.storage, 0..) |*ptr, *origin, *data, i| { + ptr.* = @ptrCast(data); + const row = ((i % 4) / 2) * 2; + const column = (i % 2) * 3; + origin.* = row * 6 + column; + } + self.deinit(); + self.pinned = null; + const pinned = try HostAccessible.initForTests(std.testing.allocator, shape_, shard_shape, layout, &self.origins, &self.ptrs); + self.pinned = pinned; + return pinned.view(i32); + } + + /// With the tiled layout, every copy of each element holds `expected[global]` + /// and the tile padding is untouched. + fn expectValues(self: *const TestShards, expected: [24]i32) !void { + // Explicit offsets for the two rows of a 2x3 shard tiled by 2x2. + const physical_offsets = [_]usize{ 0, 1, 4, 2, 3, 6 }; + for (self.storage, self.origins) |data, origin| { + for (physical_offsets, 0..) |physical, local| { + const global = origin + (local / 3) * 6 + local % 3; + try std.testing.expectEqual(expected[global], data[physical]); + } + try std.testing.expectEqual(-1, data[5]); + try std.testing.expectEqual(-1, data[7]); + } + } +}; + +test "HostAccessible.View blocks visit partial global ranges in every shard" { + var fixture: TestShards = .{}; + defer fixture.deinit(); + const values = try fixture.view(TestShards.tiled); + var visits: [24]u8 = @splat(0); + // Nested slicing must keep block offsets relative to the selected range. + var iterator = values.slice(1, null).slice(1, 18).blocks(); + while (iterator.next()) |block| { + try std.testing.expect(block.offset + block.len <= 18); + try std.testing.expectEqual(2, block.replicas.len); + for (0..block.replicas.len) |i| { + const items = block.replica(i); + for (items, block.offset + 2..) |*value, global| { + visits[global] += 1; + value.* = @intCast(global); + } + } + } + var expected: [24]i32 = @splat(-1); + for (visits, &expected, 0..) |count, *value, global| { + const selected = global >= 2 and global < 20; + try std.testing.expectEqual(@as(u8, if (selected) 2 else 0), count); + if (selected) value.* = @intCast(global); + } + try fixture.expectValues(expected); + // Starting after the first row partition must skip those shards entirely. + values.set(17, 99); + try std.testing.expectEqual(99, fixture.storage[3][4]); + try std.testing.expectEqual(99, fixture.storage[7][4]); + + const Case = struct { step: i32, expected: [18]i32 }; + const cases = [_]Case{ + .{ .step = 3, .expected = .{ 7, 10, 13, 16, 19, 22, 25, 28, 31, 34, 37, 40, 43, 46, 49, 52, 55, 58 } }, + .{ .step = -2, .expected = .{ 7, 5, 3, 1, -1, -3, -5, -7, -9, -11, -13, -15, -17, -19, -21, -23, -25, -27 } }, + .{ .step = 0, .expected = @splat(7) }, + }; + for (cases) |case| { + values.slice(1, null).slice(1, 18).fillIota(7, case.step); + expected[2..20].* = case.expected; + try fixture.expectValues(expected); + } +} + +test "HostAccessible.View blocks merge shuffled shards and group replicas" { + var fixture: TestShards = .{}; + defer fixture.deinit(); + _ = try fixture.view(TestShards.tiled); + const order = [_]usize{ 3, 4, 1, 6, 2, 0, 7, 5 }; + var ptrs: [8][*]u8 = undefined; + var origins: [8]usize = undefined; + for (order, &ptrs, &origins) |index, *ptr, *origin| { + ptr.* = fixture.ptrs[index]; + origin.* = fixture.origins[index]; + } + const pinned = try HostAccessible.initForTests(std.testing.allocator, TestShards.shape_, TestShards.shard_shape, TestShards.tiled, &origins, &ptrs); + defer pinned.deinit(std.testing.allocator); + const values = pinned.view(i32); + const selected = values.slice(1, null).slice(1, 18); + const offsets = [_]usize{ 0, 1, 3, 4, 6, 7, 9, 10, 12, 13, 15, 16 }; + const lengths = [_]usize{ 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2 }; + const replicas = [_][2]usize{ .{ 4, 0 }, .{ 1, 5 }, .{ 6, 2 }, .{ 3, 7 } }; + const physical_offsets = [_]usize{ 0, 1, 4, 2, 3, 6 }; + var iterator = selected.blocks(); + for (offsets, lengths) |offset, len| { + const block = iterator.next().?; + try std.testing.expectEqual(offset, block.offset); + try std.testing.expectEqual(len, block.len); + try std.testing.expectEqual(2, block.replicas.len); + const block_start = offset + 2; + const region = block_start / 12 * 2 + block_start % 6 / 3; + const physical = physical_offsets[block_start / 6 % 2 * 3 + block_start % 3]; + for (0..block.replicas.len, replicas[region]) |i, storage_index| { + const items = block.replica(i); + try std.testing.expectEqual(fixture.storage[storage_index][physical..].ptr, items.ptr); + } + for (0..block.replicas.len) |i| { + const items = block.replica(i); + for (items, offset + 2..) |*value, global| value.* = @intCast(global); + } + } + try std.testing.expectEqual(null, iterator.next()); + var expected: [24]i32 = @splat(-1); + for (expected[2..20], 2..) |*value, global| value.* = @intCast(global); + try fixture.expectValues(expected); + try expectReverseBlocksMirrorForward(selected); +} + +test "HostAccessible.View blocks group the maximum number of replicas" { + var storage: [Platform.MAX_NUM_DEVICES][4]i32 = @splat(@splat(-1)); + var ptrs: [Platform.MAX_NUM_DEVICES][*]u8 = undefined; + const origins: [Platform.MAX_NUM_DEVICES]usize = @splat(0); + for (&ptrs, &storage) |*ptr, *items| ptr.* = @ptrCast(items); + const shape_: Shape = .init(.{4}, .i32); + const pinned = try HostAccessible.initForTests(std.testing.allocator, shape_, shape_, .{ .strides = .{ .byte_strides = &.{4} } }, &origins, &ptrs); + defer pinned.deinit(std.testing.allocator); + const values = pinned.view(i32); + var iterator = values.blocks(); + const block = iterator.next().?; + try std.testing.expectEqual(0, block.offset); + try std.testing.expectEqual(4, block.len); + try std.testing.expectEqual(Platform.MAX_NUM_DEVICES, block.replicas.len); + try std.testing.expectEqual(null, iterator.next()); + // The descriptors remain valid after advancing the iterator. + for (0..block.replicas.len, &storage) |i, *replica| { + const items = block.replica(i); + try std.testing.expectEqual(replica[0..].ptr, items.ptr); + @memset(items, 42); + try std.testing.expectEqualSlices(i32, &.{ 42, 42, 42, 42 }, replica); + } + try expectReverseBlocksMirrorForward(values); +} + +test "HostAccessible.View visits the maximum number of distinct shuffled regions" { + var storage: [Platform.MAX_NUM_DEVICES]i32 = @splat(-1); + var ptrs: [Platform.MAX_NUM_DEVICES][*]u8 = undefined; + var origins: [Platform.MAX_NUM_DEVICES]usize = undefined; + for (&ptrs, &origins, 0..) |*ptr, *origin, shard| { + const index = storage.len - shard - 1; + ptr.* = @ptrCast(&storage[index]); + origin.* = index; + } + const shape_: Shape = .init(.{Platform.MAX_NUM_DEVICES}, .i32); + const shard_shape: Shape = .init(.{1}, .i32); + const pinned = try HostAccessible.initForTests(std.testing.allocator, shape_, shard_shape, .{ .strides = .{ .byte_strides = &.{4} } }, &origins, &ptrs); + defer pinned.deinit(std.testing.allocator); + const values = pinned.view(i32); + values.fillIota(0, 1); + var forward = values.blocks(); + var reverse = values.reverseBlocks(); + for (storage, 0..) |value, index| { + try std.testing.expectEqual(@as(i32, @intCast(index)), value); + const block = forward.next().?; + try std.testing.expectEqual(index, block.offset); + try std.testing.expectEqual(1, block.len); + try std.testing.expectEqual(1, block.replicas.len); + try std.testing.expectEqual(storage[index..].ptr, block.firstReplica().ptr); + const reverse_block = reverse.next().?; + const reverse_index = storage.len - index - 1; + try std.testing.expectEqual(reverse_index, reverse_block.offset); + try std.testing.expectEqual(storage[reverse_index..].ptr, reverse_block.firstReplica().ptr); + } + try std.testing.expectEqual(null, forward.next()); + try std.testing.expectEqual(null, reverse.next()); +} + +test "HostAccessible.View multidimensional slices preserve coordinates across carries" { + const shape_: Shape = .init(.{ 2, 3, 5 }, .i32); + const layouts = [_]pjrt.MemoryLayout{ + .{ .strides = .{ .byte_strides = &.{ 60, 20, 4 } } }, + .{ .tiled = .{ .minor_to_major = &.{ 2, 1, 0 }, .tile_dims = &.{ 3, 2 }, .tile_dims_sizes = &.{2} } }, + }; + for (layouts) |layout| { + var storage: [48]i32 = @splat(-1); + const pinned = try HostAccessible.initForTests(std.testing.allocator, shape_, shape_, layout, &.{0}, &.{@ptrCast(&storage)}); + defer pinned.deinit(std.testing.allocator); + const values = pinned.view(i32); + values.fillIota(0, 1); + var expected: [48]i32 = @splat(-1); + for (0..shape_.count()) |local| { + const index = Coordinates.unflatten(shape_, local); + const physical = referenceByteOffset(shape_, layout, index.axes[0..shape_.rank()]) / @sizeOf(i32); + expected[physical] = @intCast(local); + try std.testing.expectEqual(@as(i32, @intCast(local)), values.get(local)); + } + try std.testing.expectEqualSlices(i32, &expected, &storage); + for (0..shape_.count() + 1) |start| { + for (0..shape_.count() - start + 1) |len| { + try expectReverseBlocksMirrorForward(values.slice(start, len)); + } + } + } +} + +test "HostAccessible.View reverse blocks mirror forward blocks" { + var fixture: TestShards = .{}; + defer fixture.deinit(); + const layouts = [_]pjrt.MemoryLayout{ + TestShards.tiled, + .{ .strides = .{ .byte_strides = &.{ 12, 4 } } }, + .{ .strides = .{ .byte_strides = &.{ 4, 8 } } }, + }; + for (layouts) |layout| { + const values = try fixture.view(layout); + for (0..values.len + 1) |start| { + for (0..values.len - start + 1) |len| { + try expectReverseBlocksMirrorForward(values.slice(start, len)); + } + } + } + // Dense blocks can span multiple rows, and scalar blocks have no row axis. + const dense_pinned = try HostAccessible.initForTests(std.testing.allocator, TestShards.shard_shape, TestShards.shard_shape, layouts[1], fixture.origins[0..1], fixture.ptrs[0..1]); + defer dense_pinned.deinit(std.testing.allocator); + const dense = dense_pinned.view(i32); + try expectReverseBlocksMirrorForward(dense.slice(1, 4)); + const scalar_shape: Shape = .init(.{}, .i32); + const scalar_pinned = try HostAccessible.initForTests(std.testing.allocator, scalar_shape, scalar_shape, .{ .strides = .{ .byte_strides = &.{} } }, fixture.origins[0..1], fixture.ptrs[0..1]); + defer scalar_pinned.deinit(std.testing.allocator); + const scalar = scalar_pinned.view(i32); + try expectReverseBlocksMirrorForward(scalar); +} + +fn expectReverseBlocksMirrorForward(values: HostAccessible.View(i32)) !void { + var expected: [48]HostAccessible.View(i32).Block = undefined; + var len: usize = 0; + var offset: usize = 0; + var forward = values.blocks(); + while (forward.next()) |block| : (len += 1) { + try std.testing.expectEqual(offset, block.offset); + expected[len] = block; + offset += block.len; + } + try std.testing.expectEqual(values.len, offset); + var reverse = values.reverseBlocks(); + while (len > 0) { + len -= 1; + const block = reverse.next().?; + try std.testing.expectEqual(expected[len].offset, block.offset); + try std.testing.expectEqual(expected[len].len, block.len); + try std.testing.expectEqual(expected[len].byte_offset, block.byte_offset); + try std.testing.expectEqualSlices([*]u8, expected[len].replicas, block.replicas); + } + try std.testing.expectEqual(null, reverse.next()); +} + +test "HostAccessible.View scalar, empty ranges and rank-one padding" { + var scalar: i32 = 0; + const scalar_ptrs = [_][*]u8{@ptrCast(&scalar)}; + const shape_ = Shape.scalar(.i32); + const scalar_layout: pjrt.MemoryLayout = .{ .tiled = .{ .minor_to_major = &.{}, .tile_dims = &.{}, .tile_dims_sizes = &.{} } }; + const scalar_pinned = try HostAccessible.initForTests(std.testing.allocator, shape_, shape_, scalar_layout, &.{0}, &scalar_ptrs); + defer scalar_pinned.deinit(std.testing.allocator); + const scalar_view = scalar_pinned.view(i32); + scalar_view.copyFrom(&.{42}); + try std.testing.expectEqual(42, scalar); + var past_end = scalar_view.slice(1, null).blocks(); + try std.testing.expectEqual(null, past_end.next()); + + var storage: [8]bool = @splat(false); + const vector_ptrs = [_][*]u8{@ptrCast(&storage)}; + const vector_shape = Shape.init(.{5}, .bool); + const vector_layout: pjrt.MemoryLayout = .{ .tiled = .{ .minor_to_major = &.{0}, .tile_dims = &.{4}, .tile_dims_sizes = &.{1} } }; + const vector_pinned = try HostAccessible.initForTests(std.testing.allocator, vector_shape, vector_shape, vector_layout, &.{0}, &vector_ptrs); + defer vector_pinned.deinit(std.testing.allocator); + const vector = vector_pinned.view(bool); + vector.fill(true); + try std.testing.expectEqualSlices(bool, &.{ true, true, true, true, true, false, false, false }, &storage); + + const empty_shape = Shape.init(.{0}, .bool); + const empty_pinned = try HostAccessible.initForTests(std.testing.allocator, empty_shape, empty_shape, vector_layout, &.{0}, &vector_ptrs); + defer empty_pinned.deinit(std.testing.allocator); + const empty_view = empty_pinned.view(bool); + var empty_blocks = empty_view.blocks(); + try std.testing.expectEqual(null, empty_blocks.next()); + + // The empty view has no storage or metadata to read. + const none: HostAccessible.View(u32) = .empty; + none.fill(0); + none.copyTo(&.{}); + var none_blocks = none.reverseBlocks(); + try std.testing.expectEqual(null, none_blocks.next()); +} + +test "HostAccessible devices holding a shard read one allocation" { + const platform = testing.env(); + const tp = testing.tpMesh(); + + const shape_ = Shape.init(.{ .b = @as(i64, @intCast(platform.devices.len)), .d = 5 }, .i32); + try expectDevicesReadShards(platform, shape_); + try expectDevicesReadShards(platform, shape_.withPartitioning(tp, .{ .b = .model })); +} + +test "HostAccessible devices holding a shard read one allocation in data-parallel groups" { + const allocator = std.testing.allocator; + const io = std.testing.io; + // Two data-parallel groups of two tensor-parallel devices. + const platform = Platform.auto(allocator, io, .{ + .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus2d }, + .cpu = .{ .device_count = 4 }, + }) catch return error.SkipZigTest; + defer platform.deinit(allocator, io); + const mesh: Sharding.Mesh = try .init( + "data_parallel", + &platform.physical_mesh, + .mesh(.{ .batch = .low_bandwidth, .model = .high_bandwidth }), + .parseBindings(.{ .batch = .link_x, .model = .link_y }), + ); + try expectDevicesReadShards(platform, Shape.init(.{ .b = 4, .d = 5 }, .i32).withPartitioning(&mesh, .{ .b = .batch })); +} + +/// Rows of `.d` elements written once through a view reach every device +/// holding them, and devices holding the same shard share its allocation +/// where the platform allows it. +fn expectDevicesReadShards(platform: *const Platform, shape_: Shape) !void { + if (!builtin.is_test) @compileError("expectDevicesReadShards is only available in tests"); + + const allocator = std.testing.allocator; + const io = std.testing.io; + const api = platform.pjrt_api; + const pinned = try HostAccessible.init(allocator, io, platform, shape_, .undonatable); + defer pinned.deinit(allocator); + const placement: Sharding.Placement = try Sharding.Placement.init(&shape_); + const expected_allocations = switch (platform.target) { + .cpu, .cuda, .rocm => shape_.count() / placement.localShape(shape_).count(), + else => platform.devices.len, + }; + try std.testing.expectEqual(expected_allocations, pinned.mapping.allocations.entries.len); + + pinned.view(i32).fillIota(0, 1); + const row_size: usize = @intCast(shape_.dim(.d)); + const values = try allocator.alloc(i32, placement.localShape(shape_).count()); + defer allocator.free(values); + for (pinned.buffer._shards.constSlice(), 0..) |shard, device_index| { + @memset(values, -1); + if (try shard.toHostBuffer(api, std.mem.sliceAsBytes(values))) |event| { + defer event.deinit(api); + try event.await(api, io); + } + const first_row = Coordinates.ofShard(&placement, device_index, shape_).axes[0]; + for (values, 0..) |value, i| try std.testing.expectEqual(@as(i32, @intCast(first_row * row_size + i)), value); + } +} + +test "HostAccessible executions read shared inputs on every device" { + const zml = @import("../zml.zig"); + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = testing.env(); + const api = platform.pjrt_api; + const shape_ = Shape.init(.{ 3, 5 }, .i32); + const Test = struct { + fn increment(input: zml.Tensor) zml.Tensor { + return input.onMemory(.host_pinned).toMemory(.device).addConstant(1); + } + }; + var exe = try platform.compileFn(allocator, io, Test.increment, .{zml.Tensor.fromShape(shape_)}, .{}); + defer exe.deinit(); + var runner = try exe.runner(allocator); + defer runner.deinit(allocator); + + const pinned = try HostAccessible.init(allocator, io, platform, shape_, .undonatable); + defer pinned.deinit(allocator); + for (0..2) |iteration| { + pinned.view(i32).fillIota(@intCast(iteration * 100), 1); + var output: Buffer = undefined; + runner.run(io, .{pinned.buffer}, .{&output}, .{}); + defer output.deinit(); + try output.await(io); + for (output._shards.constSlice()) |shard| { + var values: [15]i32 = @splat(-1); + if (try shard.toHostBuffer(api, std.mem.asBytes(&values))) |event| { + defer event.deinit(api); + try event.await(api, io); + } + for (values, 0..) |value, i| try std.testing.expectEqual(@as(i32, @intCast(iteration * 100 + i + 1)), value); + } + } +} + +test "HostAccessible.View.constSlice borrows contiguous buffers and copies the others" { + const shape_ = Shape.init(.{ 2, 3 }, .i32); + var scratch: [6]i32 = @splat(-1); + + var dense: [2][6]i32 = .{ .{ 0, 1, 2, 3, 4, 5 }, @splat(-1) }; + const dense_ptrs = [_][*]u8{ @ptrCast(&dense[0]), @ptrCast(&dense[1]) }; + const dense_pinned = try HostAccessible.initForTests(std.testing.allocator, shape_, shape_, .{ .strides = .{ .byte_strides = &.{ 12, 4 } } }, &.{ 0, 0 }, &dense_ptrs); + defer dense_pinned.deinit(std.testing.allocator); + const borrowed = dense_pinned.view(i32).slice(1, 4).constSlice(&scratch); + try std.testing.expectEqual(dense[0][1..].ptr, borrowed.ptr); + try std.testing.expectEqualSlices(i32, &.{ 1, 2, 3, 4 }, borrowed); + + var column_major = [_]i32{ 0, 3, 1, 4, 2, 5 }; + const transposed: pjrt.MemoryLayout = .{ .tiled = .{ .minor_to_major = &.{ 0, 1 }, .tile_dims = &.{}, .tile_dims_sizes = &.{} } }; + const transposed_pinned = try HostAccessible.initForTests(std.testing.allocator, shape_, shape_, transposed, &.{0}, &.{@ptrCast(&column_major)}); + defer transposed_pinned.deinit(std.testing.allocator); + const copied = transposed_pinned.view(i32).slice(1, 4).constSlice(&scratch); + try std.testing.expectEqual(@as([*]const i32, &scratch), copied.ptr); + try std.testing.expectEqualSlices(i32, &.{ 1, 2, 3, 4 }, copied); + + const none: HostAccessible.View(i32) = .empty; + try std.testing.expectEqual(0, none.constSlice(&.{}).len); +} + +test "HostAccessible.View.edit writes contiguous buffers in place and commits every replica" { + const shape_ = Shape.init(.{ 2, 3 }, .i32); + var scratch: [6]i32 = @splat(-1); + + // In place: the first replica takes the writes, and `commit` copies it to the others. + var dense: [2][6]i32 = @splat(@splat(0)); + const dense_ptrs = [_][*]u8{ @ptrCast(&dense[0]), @ptrCast(&dense[1]) }; + const dense_pinned = try HostAccessible.initForTests(std.testing.allocator, shape_, shape_, .{ .strides = .{ .byte_strides = &.{ 12, 4 } } }, &.{ 0, 0 }, &dense_ptrs); + defer dense_pinned.deinit(std.testing.allocator); + const dense_edit = dense_pinned.view(i32).slice(1, 4).edit(&scratch); + try std.testing.expectEqual(dense[0][1..].ptr, dense_edit.items.ptr); + @memcpy(dense_edit.items, &[_]i32{ 1, 2, 3, 4 }); + dense_edit.commit(); + for (dense) |replica| try std.testing.expectEqualSlices(i32, &.{ 0, 1, 2, 3, 4, 0 }, &replica); + + // Through the scratch: the buffer only changes on `commit`. + var column_major: [6]i32 = @splat(0); + const transposed: pjrt.MemoryLayout = .{ .tiled = .{ .minor_to_major = &.{ 0, 1 }, .tile_dims = &.{}, .tile_dims_sizes = &.{} } }; + const transposed_pinned = try HostAccessible.initForTests(std.testing.allocator, shape_, shape_, transposed, &.{0}, &.{@ptrCast(&column_major)}); + defer transposed_pinned.deinit(std.testing.allocator); + const copied_edit = transposed_pinned.view(i32).slice(1, 4).edit(&scratch); + try std.testing.expectEqual(@as([*]i32, &scratch), copied_edit.items.ptr); + @memcpy(copied_edit.items, &[_]i32{ 1, 2, 3, 4 }); + try std.testing.expectEqualSlices(i32, &.{ 0, 0, 0, 0, 0, 0 }, &column_major); + copied_edit.commit(); + try std.testing.expectEqualSlices(i32, &.{ 0, 3, 1, 4, 2, 0 }, &column_major); + + const none: HostAccessible.View(i32) = .empty; + none.edit(&.{}).commit(); +} + +test "HostAccessible.View.indexOfScalar and indexOfAny search blocks in row-major order" { + var fixture: TestShards = .{}; + defer fixture.deinit(); + const values = try fixture.view(TestShards.tiled); + values.fillIota(0, 1); + values.set(20, 7); + // Tiles split the view into blocks, and shards into regions. + try std.testing.expectEqual(7, values.indexOfScalar(7)); + try std.testing.expectEqual(12, values.slice(8, null).indexOfScalar(7)); + try std.testing.expectEqual(null, values.slice(8, 12).indexOfScalar(7)); + try std.testing.expectEqual(null, values.indexOfScalar(-1)); + try std.testing.expectEqual(null, HostAccessible.View(i32).empty.indexOfScalar(0)); + + try std.testing.expectEqual(5, values.indexOfAny(&.{ 7, 5 })); + try std.testing.expectEqual(12, values.slice(8, null).indexOfAny(&.{ -1, 7 })); + try std.testing.expectEqual(null, values.slice(8, 12).indexOfAny(&.{ 7, -1 })); + try std.testing.expectEqual(null, values.indexOfAny(&.{})); +} + +test "HostAccessible.View indexes replicas of a dense array directly" { + const shape_ = Shape.init(.{ 2, 3 }, .i32); + var storage: [2][6]i32 = @splat(@splat(-1)); + const ptrs = [_][*]u8{ @ptrCast(&storage[0]), @ptrCast(&storage[1]) }; + const pinned = try HostAccessible.initForTests(std.testing.allocator, shape_, shape_, .{ .strides = .{ .byte_strides = &.{ 12, 4 } } }, &.{ 0, 0 }, &ptrs); + defer pinned.deinit(std.testing.allocator); + try std.testing.expect(pinned.mapping.contiguous); + const values = pinned.view(i32); + for (0..6) |i| values.set(i, @intCast(10 + i)); + values.slice(4, null).set(1, 99); + for (storage) |replica| try std.testing.expectEqualSlices(i32, &.{ 10, 11, 12, 13, 14, 99 }, &replica); + try std.testing.expectEqual(99, values.get(5)); + try std.testing.expectEqual(13, values.slice(2, 3).get(1)); + + inline for (.{ false, true }) |reverse| { + const nested = values.slice(1, 4); + var blocks = if (reverse) nested.reverseBlocks() else nested.blocks(); + const block = blocks.next().?; + try std.testing.expectEqual(0, block.offset); + try std.testing.expectEqual(2, block.replicas.len); + for (0..block.replicas.len, &storage) |i, *replica| { + const items = block.replica(i); + try std.testing.expectEqualSlices(i32, &.{ 11, 12, 13, 14 }, items); + try std.testing.expectEqual(replica[1..].ptr, items.ptr); + } + try std.testing.expectEqual(null, blocks.next()); + } +} + +test "HostAccessible.View set and get address single elements in every replica" { + var fixture: TestShards = .{}; + defer fixture.deinit(); + const values = try fixture.view(TestShards.tiled); + var expected: [24]i32 = undefined; + var i: usize = expected.len; + while (i > 0) { + i -= 1; + expected[i] = @intCast(100 + i); + values.set(i, expected[i]); + } + try fixture.expectValues(expected); + for (expected, 0..) |value, index| try std.testing.expectEqual(value, values.get(index)); + + // Indices of a nested view are relative to its start. + const nested = values.slice(7, 10); + nested.set(3, -5); + expected[10] = -5; + try fixture.expectValues(expected); + try std.testing.expectEqual(-5, nested.get(3)); +} + +test "HostAccessible.View copies across layouts, shards and replicas" { + var fixture: TestShards = .{}; + defer fixture.deinit(); + const values = try fixture.view(TestShards.tiled); + values.fill(0); + values.slice(2, 20).fillIota(10, 1); + values.set(17, 99); + const expected = [_]i32{ 0, 0, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 99, 26, 27, 28, 29, 0, 0 }; + var actual: [24]i32 = undefined; + values.copyTo(&actual); + try std.testing.expectEqualSlices(i32, &expected, &actual); + for (expected, 0..) |value, i| try std.testing.expectEqual(value, values.get(i)); + try fixture.expectValues(expected); + + // Blocks expose every replica, respect physical boundaries, and stop at + // the end of a nested view even when the physical run continues. + const nested = values.slice(1, null).slice(2, 4); + var nested_blocks = nested.blocks(); + const first = nested_blocks.next().?; + try std.testing.expectEqual(0, first.offset); + try std.testing.expectEqual(2, first.replicas.len); + for (0..first.replicas.len, [_]usize{ 1, 5 }) |i, storage_index| { + const items = first.replica(i); + try std.testing.expectEqualSlices(i32, expected[3..5], items); + try std.testing.expectEqual(fixture.storage[storage_index][0..].ptr, items.ptr); + } + const middle = nested_blocks.next().?; + try std.testing.expectEqual(2, middle.offset); + for (0..middle.replicas.len) |i| try std.testing.expectEqualSlices(i32, expected[5..6], middle.replica(i)); + const last = nested_blocks.next().?; + try std.testing.expectEqual(3, last.offset); + for (0..last.replicas.len, [_]usize{ 0, 4 }) |i, storage_index| { + const items = last.replica(i); + try std.testing.expectEqualSlices(i32, expected[6..7], items); + try std.testing.expectEqual(fixture.storage[storage_index][2..].ptr, items.ptr); + } + try std.testing.expectEqual(null, nested_blocks.next()); + + // Copy between differently shaped/tiled ranges, including a nonzero source + // and destination start, and verify every destination replica physically. + const dst_shape = Shape.init(.{ 3, 7 }, .i32); + const dst_layout: pjrt.MemoryLayout = .{ .tiled = .{ + .minor_to_major = &.{ 0, 1 }, + .tile_dims = &.{}, + .tile_dims_sizes = &.{}, + } }; + var dst_storage: [2][21]i32 = @splat(@splat(-1)); + const dst_ptrs = [_][*]u8{ @ptrCast(&dst_storage[0]), @ptrCast(&dst_storage[1]) }; + const dst_pinned = try HostAccessible.initForTests(std.testing.allocator, dst_shape, dst_shape, dst_layout, &.{ 0, 0 }, &dst_ptrs); + defer dst_pinned.deinit(std.testing.allocator); + const dst = dst_pinned.view(i32); + dst.slice(1, 19).copyFromView(values.slice(1, 22).slice(1, 19)); + for (dst_storage) |data| { + for (0..21) |i| { + const physical = (i % 7) * 3 + i / 7; + try std.testing.expectEqual(if (i > 0 and i < 20) expected[i + 1] else -1, data[physical]); + } + } + var empty: [0]i32 = .{}; + values.slice(values.len, 0).copyTo(&empty); + values.slice(0, 0).copyFromView(dst.slice(0, 0)); + values.slice(values.len, null).fill(-1); + values.slice(values.len, null).fillIota(-1, 1); + values.slice(values.len, null).copyFrom(&empty); + values.copyTo(&actual); + try std.testing.expectEqualSlices(i32, &expected, &actual); +} + +test "HostAccessible await after executable donation permits the next host update" { + const zml = @import("../zml.zig"); + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = testing.env(); + const shape_ = Shape.init(.{ 3, 5 }, .i32); + const Test = struct { + fn increment(input: zml.Tensor) zml.Tensor { + const host_input = input.onMemory(.host_pinned); + return host_input.toMemory(.device).addConstant(1).toMemory(.host_pinned).reuseBuffer(host_input); + } + }; + var exe = try platform.compileFn(allocator, io, Test.increment, .{zml.Tensor.fromShape(shape_)}, .{}); + defer exe.deinit(); + var runner = try exe.runner(allocator); + defer runner.deinit(allocator); + + var pinned = try HostAccessible.init(allocator, io, platform, shape_, .donatable); + defer pinned.deinit(allocator); + try std.testing.expectEqual(pinned.buffer._shards.len, pinned.mapping.allocations.entries.len); + const first_ptr = pinned.mapping.allocations.entries.items(.ptr)[0]; + pinned.view(i32).fillIota(0, 1); + + // Reuse the same runner and output destination as the model execution paths. + // Only await waits for the asynchronous executable before host access. + for (0..3) |iteration| { + runner.run(io, .{pinned.buffer}, .{&pinned.buffer}, .{}); + try pinned.await(io); + + // Reacquire from the output handles, even when donation reuses memory. + const values = pinned.view(i32); + var actual: [15]i32 = undefined; + values.copyTo(&actual); + try std.testing.expectEqual(first_ptr, pinned.mapping.allocations.entries.items(.ptr)[0]); + for (actual, 0..) |value, i| { + const expected: i32 = if (iteration == 0 or i < 3 or i >= 12) + @intCast(i + iteration + 1) + else + @intCast(iteration * 100 + i - 3 + 1); + try std.testing.expectEqual(expected, value); + } + + // Also check the executable's logical output through PJRT, independently + // of the host-visible layout used above. + var device_values: [15]i32 = undefined; + try pinned.buffer.toSlice(io, .init(shape_, std.mem.asBytes(&device_values))); + try std.testing.expectEqualSlices(i32, &actual, &device_values); + + values.slice(3, 9).fillIota(@intCast((iteration + 1) * 100), 1); + } +} + +test "HostAccessible await follows an output that doesn't reuse the pinned allocation" { + const zml = @import("../zml.zig"); + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = testing.env(); + const shape_ = Shape.init(.{ 3, 5 }, .i32); + const Test = struct { + fn increment(input: zml.Tensor) zml.Tensor { + // Without reuseBuffer, execution writes a new host allocation. + return input.onMemory(.host_pinned).toMemory(.device).addConstant(1).toMemory(.host_pinned); + } + }; + var exe = try platform.compileFn(allocator, io, Test.increment, .{zml.Tensor.fromShape(shape_)}, .{}); + defer exe.deinit(); + var runner = try exe.runner(allocator); + defer runner.deinit(allocator); + + var pinned = try HostAccessible.init(allocator, io, platform, shape_, .donatable); + defer pinned.deinit(allocator); + pinned.view(i32).fillIota(0, 1); + + for (0..2) |iteration| { + // The input isn't donated, so release it once execution replaced it. + var input = pinned.buffer; + defer input.deinit(); + const previous = pinned.mapping.allocations.entries.items(.ptr)[0]; + runner.run(io, .{input}, .{&pinned.buffer}, .{}); + try pinned.await(io); + const values = pinned.view(i32); + try std.testing.expect(pinned.mapping.allocations.entries.items(.ptr)[0] != previous); + + var actual: [15]i32 = undefined; + values.copyTo(&actual); + for (actual, 0..) |value, i| try std.testing.expectEqual(@as(i32, @intCast(iteration * 100 + i + 1)), value); + var device_values: [15]i32 = undefined; + try pinned.buffer.toSlice(io, .init(shape_, std.mem.asBytes(&device_values))); + try std.testing.expectEqualSlices(i32, &actual, &device_values); + + // Host writes now land in the output, which the next execution reads. + values.fillIota(@intCast((iteration + 1) * 100), 1); + } +} diff --git a/zml/constants.zig b/zml/constants.zig new file mode 100644 index 00000000..39586fa4 --- /dev/null +++ b/zml/constants.zig @@ -0,0 +1,18 @@ +pub const MAX_RANK: u8 = 8; +pub const AXES_IOTA: [MAX_RANK]i64 = b: { + var values: [MAX_RANK]i64 = undefined; + for (0..MAX_RANK) |i| values[i] = i; + break :b values; +}; +pub const MINOR_TO_MAJOR: [MAX_RANK]i64 = b: { + var values: [MAX_RANK]i64 = undefined; + for (0..MAX_RANK) |i| values[i] = MAX_RANK - i - 1; + break :b values; +}; + +pub fn minorToMajor(rank: u8) []const i64 { + return MINOR_TO_MAJOR[MINOR_TO_MAJOR.len - rank ..]; +} + +pub const ones_i64: [MAX_RANK]i64 = @splat(1); +pub const zeroes_i64: [MAX_RANK]i64 = @splat(0); diff --git a/zml/context.zig b/zml/context.zig deleted file mode 100644 index 66665894..00000000 --- a/zml/context.zig +++ /dev/null @@ -1,138 +0,0 @@ -const builtin = @import("builtin"); -const std = @import("std"); - -const asynk = @import("async"); -const mlir = @import("mlir"); -const pjrt = @import("pjrt"); - -const platform = @import("platform.zig"); -const Target = @import("platform.zig").Target; -const Platform = @import("platform.zig").Platform; - -const log = std.log.scoped(.zml); - -const PjrtApiMap = std.EnumArray(Target, ?*const pjrt.Api); -const PlatformsMap = std.EnumArray(Target, ?Platform); - -/// Every program using ZML must start with a `zml.Context.init(.{});` -/// The ZML context contains global state to interact with the different -/// devices available on your system. -/// Note that the runtimes available depends on how the program was compiled. -/// For example you need to compile your program with `--//runtimes:cuda=true` -/// to have the CUDA runtime available. -pub const Context = struct { - var apis = PjrtApiMap.initFill(null); - var apis_once = std.once(struct { - fn call() void { - inline for (platform.available_targets) |t| { - if (canLoad(t)) { - if (pjrt.Api.loadFrom(platformToLibrary(t))) |api| { - Context.apis.set(t, api); - } else |_| {} - } - } - } - }.call); - - var mlir_once = std.once(struct { - fn call() void { - mlir.registerPasses("Transforms"); - } - }.call); - - platforms: PlatformsMap, - - /// Creates a ZML Context and returns it. - pub fn init() !Context { - Context.apis_once.call(); - Context.mlir_once.call(); - - var platforms = PlatformsMap.initFill(null); - var num_platforms: u8 = 0; - var it = Context.apis.iterator(); - while (it.next()) |entry| { - if (entry.value.*) |api| { - const target = entry.key; - const p = Platform.init(target, api) catch |err| { - log.err("Failed to load platform .{s}: {}", .{ @tagName(target), err }); - continue; - }; - if (p.getDevices().len == 0) { - log.err("No device found for platform {} !", .{target}); - continue; - } - platforms.set(target, p); - num_platforms += 1; - } - } - if (num_platforms == 0) return error.NotFound; - return .{ - .platforms = platforms, - }; - } - - fn platformToLibrary(comptime target: Target) []const u8 { - const ext = switch (builtin.os.tag) { - .windows => ".dll", - .macos, .ios, .watchos => ".dylib", - else => ".so", - }; - return switch (target) { - inline else => "libpjrt_" ++ @tagName(target) ++ ext, - }; - } - - fn canLoad(t: Target) bool { - return switch (t) { - .tpu => isRunningOnGCP() catch false, - else => true, - }; - } - - /// Check if running on Google Compute Engine, because TPUs will poll the - /// metadata server, hanging the process. So only do it on GCP. - /// Do it using the official method at: - /// https://cloud.google.com/compute/docs/instances/detect-compute-engine?hl=en#use_operating_system_tools_to_detect_if_a_vm_is_running_in - fn isRunningOnGCP() !bool { - // TODO: abstract that in the client and fail init - const GoogleComputeEngine = "Google Compute Engine"; - - var f = try asynk.File.open("/sys/devices/virtual/dmi/id/product_name", .{ .mode = .read_only }); - defer f.close() catch {}; - - var buf = [_]u8{0} ** GoogleComputeEngine.len; - _ = try f.reader().readAll(&buf); - - return std.mem.eql(u8, &buf, GoogleComputeEngine); - } - - pub fn pjrtApi(target: Target) *const pjrt.Api { - return Context.apis.get(target).?; - } - - pub fn deinit(self: *Context) void { - var iterator = self.platforms.iterator(); - while (iterator.next()) |entry| { - if (entry.value.*) |*p| { - p.deinit(); - } - } - self.* = undefined; - } - - /// Automatically selects the best Platform loaded in the current Context. - /// - /// For example, if supported, this will select a platform corresponding to an accelerator (GPU, TPU, ...). - pub fn autoPlatform(self: *Context) Platform { - // the last platform is the one that with the high enum number, so considered - // to be the "best" one - var platform_: ?Platform = null; - var iterator = self.platforms.iterator(); - while (iterator.next()) |entry| { - if (entry.value.*) |p| { - platform_ = p; - } - } - return platform_ orelse @panic("No platform found !"); - } -}; diff --git a/zml/dtype.zig b/zml/dtype.zig index cfc13cf7..f5cd9da8 100644 --- a/zml/dtype.zig +++ b/zml/dtype.zig @@ -1,29 +1,35 @@ const std = @import("std"); + +const stdx = @import("stdx"); + const floats = @import("floats.zig"); const C64 = std.math.Complex(f32); const C128 = std.math.Complex(f64); -test { - std.testing.refAllDecls(@This()); -} - pub const DataType = enum(u8) { bool, + f4e2m1, + // Note: the support of the float8 is a bit spotty, f8e4m3b11fnuz seems to be the most supported one on Cuda. + f8e3m4, + f8e4m3, f8e4m3b11fnuz, f8e4m3fn, f8e4m3fnuz, f8e5m2, f8e5m2fnuz, + f8e8m0, bf16, f16, f32, f64, + i2, i4, i8, i16, i32, i64, + u2, u4, u8, u16, @@ -48,8 +54,21 @@ pub const DataType = enum(u8) { pub fn class(self: DataType) Class { return switch (self) { .bool => .bool, - .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz, .bf16, .f16, .f32, .f64 => .float, - .i4, .i8, .i16, .i32, .i64, .u4, .u8, .u16, .u32, .u64 => .integer, + .f4e2m1, + .f8e3m4, + .f8e4m3, + .f8e4m3b11fnuz, + .f8e4m3fn, + .f8e4m3fnuz, + .f8e5m2, + .f8e5m2fnuz, + .f8e8m0, + .bf16, + .f16, + .f32, + .f64, + => .float, + .i2, .i4, .i8, .i16, .i32, .i64, .u2, .u4, .u8, .u16, .u32, .u64 => .integer, .c64, .c128 => .complex, }; } @@ -68,21 +87,27 @@ pub const DataType = enum(u8) { pub fn fromZigType(comptime T: type) DataType { return switch (T) { + floats.Float4E2M1 => .f4e2m1, + floats.Float8E3M4 => .f8e3m4, + floats.Float8E4M3 => .f8e4m3, floats.Float8E4M3B11FNUZ => .f8e4m3b11fnuz, floats.Float8E4M3FN => .f8e4m3fn, floats.Float8E4M3FNUZ => .f8e4m3fnuz, floats.Float8E5M2 => .f8e5m2, floats.Float8E5M2FNUZ => .f8e5m2fnuz, + floats.Float8E8M0 => .f8e8m0, floats.BFloat16 => .bf16, f16 => .f16, f32 => .f32, f64 => .f64, bool => .bool, + i2 => .i2, i4 => .i4, i8 => .i8, i16 => .i16, i32 => .i32, i64 => .i64, + u2 => .u2, u4 => .u4, u8 => .u8, u16 => .u16, @@ -96,47 +121,62 @@ pub const DataType = enum(u8) { pub fn fromSliceElementType(slice: anytype) DataType { const type_info = @typeInfo(@TypeOf(slice)); - if (type_info != .Pointer) { + if (type_info != .pointer) { @compileError("`initFromSlice` expects a slice, got " ++ @tagName(type_info)); } - return switch (type_info.Pointer.size) { - .Slice, .C, .Many => DataType.fromZigType(type_info.Pointer.child), - .One => b: { - const child_type_info = @typeInfo(type_info.Pointer.child); - break :b DataType.fromZigType(child_type_info.Array.child); + return switch (type_info.pointer.size) { + .slice, .c, .many => DataType.fromZigType(type_info.pointer.child), + .one => b: { + const child_type_info = @typeInfo(type_info.pointer.child); + break :b DataType.fromZigType(child_type_info.array.child); }, }; } pub fn toZigType(comptime dtype: DataType) type { + return @FieldType(Value, @tagName(dtype)); + } + + pub fn toPackedZigType(comptime dtype: DataType) type { return switch (dtype) { - inline else => |tag| std.meta.TagPayload(Data, tag), + .bool => bool, + .i2, .i4, .u2, .u4 => |dt| stdx.Packed(@FieldType(Value, @tagName(dt))), + .f4e2m1 => floats.Float4E2M1.Packed, + inline .c64, .c128 => |dt| @FieldType(Value, @tagName(dt)), + else => { + const T = @FieldType(Value, @tagName(dtype)); + if (@bitSizeOf(T) < 8) @compileLog("Forgot type !", T); + return T; + }, }; } pub fn isSignedInt(dtype: DataType) bool { return switch (dtype) { - .i4, .i8, .i16, .i32, .i64 => true, + .i2, .i4, .i8, .i16, .i32, .i64 => true, else => false, }; } pub fn sizeOf(self: DataType) u16 { return switch (self) { - inline else => |tag| @sizeOf(std.meta.TagPayload(Data, tag)), + inline else => |tag| @sizeOf(tag.toZigType()), }; } pub fn bitSizeOf(self: DataType) u16 { return switch (self) { - inline else => |tag| @bitSizeOf(std.meta.TagPayload(Data, tag)), + .bool => 8, // In PJRT/stablehlo bool always occupy a full byte + .c64 => 64, + .c128 => 128, + inline else => |tag| @bitSizeOf(tag.toZigType()), }; } pub fn alignOf(self: DataType) u29 { return switch (self) { - inline else => |tag| @alignOf(std.meta.TagPayload(Data, tag)), + inline else => |tag| @alignOf(tag.toZigType()), }; } @@ -170,165 +210,178 @@ pub const DataType = enum(u8) { try std.testing.expectEqual(DataType.c64.resolvePeerType(.f32), null); } - pub fn zero(dtype: DataType) Data { - return Data.init(dtype, 0); + pub fn zero(dtype: DataType) Value { + return .init(dtype, 0); } - pub fn one(dtype: DataType) Data { - return Data.init(dtype, 1); + pub fn one(dtype: DataType) Value { + return .init(dtype, 1); } - pub fn minValue(dtype: DataType) Data { + pub fn minValue(dtype: DataType) Value { return switch (dtype) { .bool => .{ .bool = false }, - inline .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2fnuz => |tag| @unionInit(Data, @tagName(tag), std.meta.FieldType(Data, tag).zero()), - inline .f8e5m2, .bf16 => |tag| @unionInit(Data, @tagName(tag), std.meta.FieldType(Data, tag).minusInf()), - inline .f16, .f32, .f64 => |tag| @unionInit(Data, @tagName(tag), -std.math.inf(std.meta.FieldType(Data, tag))), - inline .i4, .i8, .i16, .i32, .i64, .u4, .u8, .u16, .u32, .u64 => |tag| @unionInit(Data, @tagName(tag), std.math.minInt(std.meta.FieldType(Data, tag))), - inline else => |tag| @panic("Unsupported type: " ++ @tagName(tag)), + inline .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2fnuz, .f8e8m0, .f4e2m1 => |tag| @unionInit(Value, @tagName(tag), @FieldType(Value, @tagName(tag)).min), + inline .f8e5m2, .f8e3m4, .f8e4m3, .bf16 => |tag| @unionInit(Value, @tagName(tag), @FieldType(Value, @tagName(tag)).minus_inf), + inline .f16, .f32, .f64 => |tag| @unionInit(Value, @tagName(tag), -std.math.inf(@FieldType(Value, @tagName(tag)))), + inline .i2, .i4, .i8, .i16, .i32, .i64, .u2, .u4, .u8, .u16, .u32, .u64 => |tag| @unionInit(Value, @tagName(tag), std.math.minInt(@FieldType(Value, @tagName(tag)))), + inline .c64, .c128 => |tag| @panic("DataType doesn't have a min value: " ++ @tagName(tag)), }; } - pub fn maxValue(dtype: DataType) Data { + pub fn maxValue(dtype: DataType) Value { return switch (dtype) { .bool => .{ .bool = true }, - inline .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2fnuz => |tag| @panic("DataType doesn't have a max value: " ++ @tagName(tag)), - inline .f8e5m2, .bf16 => |tag| @unionInit(Data, @tagName(tag), std.meta.FieldType(Data, tag).inf()), - inline .f16, .f32, .f64 => |tag| @unionInit(Data, @tagName(tag), std.math.inf(std.meta.FieldType(Data, tag))), - inline .i4, .i8, .i16, .i32, .i64, .u4, .u8, .u16, .u32, .u64 => |tag| @unionInit(Data, @tagName(tag), std.math.maxInt(std.meta.FieldType(Data, tag))), + inline .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2fnuz, .f8e8m0, .f4e2m1 => |tag| @unionInit(Value, @tagName(tag), @FieldType(Value, @tagName(tag)).max), + inline .f8e5m2, .f8e3m4, .f8e4m3, .bf16 => |tag| @unionInit(Value, @tagName(tag), @FieldType(Value, @tagName(tag)).inf), + inline .f16, .f32, .f64 => |tag| @unionInit(Value, @tagName(tag), std.math.inf(@FieldType(Value, @tagName(tag)))), + inline .i2, .i4, .i8, .i16, .i32, .i64, .u2, .u4, .u8, .u16, .u32, .u64 => |tag| @unionInit(Value, @tagName(tag), std.math.maxInt(@FieldType(Value, @tagName(tag)))), inline .c64, .c128 => |tag| @panic("DataType doesn't have a max value: " ++ @tagName(tag)), }; } - pub fn constant(dtype: DataType, value: anytype) Data { - return Data.init(dtype, value); + pub fn constant(dtype: DataType, value: anytype) Value { + return .init(dtype, value); } -}; -pub const Data = union(DataType) { - bool: bool, - f8e4m3b11fnuz: floats.Float8E4M3B11FNUZ, - f8e4m3fn: floats.Float8E4M3FN, - f8e4m3fnuz: floats.Float8E4M3FNUZ, - f8e5m2: floats.Float8E5M2, - f8e5m2fnuz: floats.Float8E5M2FNUZ, - bf16: floats.BFloat16, - f16: f16, - f32: f32, - f64: f64, - i4: i4, - i8: i8, - i16: i16, - i32: i32, - i64: i64, - u4: u4, - u8: u8, - u16: u16, - u32: u32, - u64: u64, - c64: C64, - c128: C128, + pub const Value = union(DataType) { + bool: bool, + f4e2m1: floats.Float4E2M1, + f8e3m4: floats.Float8E3M4, + f8e4m3: floats.Float8E4M3, + f8e4m3b11fnuz: floats.Float8E4M3B11FNUZ, + f8e4m3fn: floats.Float8E4M3FN, + f8e4m3fnuz: floats.Float8E4M3FNUZ, + f8e5m2: floats.Float8E5M2, + f8e5m2fnuz: floats.Float8E5M2FNUZ, + f8e8m0: floats.Float8E8M0, + bf16: floats.BFloat16, + f16: f16, + f32: f32, + f64: f64, + i2: i2, + i4: i4, + i8: i8, + i16: i16, + i32: i32, + i64: i64, + u2: u2, + u4: u4, + u8: u8, + u16: u16, + u32: u32, + u64: u64, + c64: C64, + c128: C128, - /// Creates `Data` from a `value`. - /// - /// If the `dtype` and `@TypeOf(value)` are incompatible - /// or a cast from `value` to `FieldType(dtype)` would - /// be lossy, a panic occurs. - pub fn init(dtype_: DataType, value: anytype) Data { - const T = @TypeOf(value); - const Ti = @typeInfo(T); + /// Creates `Data` from a `value`. + /// + /// If the `dtype` and `@TypeOf(value)` are incompatible + /// or a cast from `value` to `FieldType(dtype)` would + /// be lossy, a panic occurs. + pub fn init(dtype_: DataType, value: anytype) Value { + const T = @TypeOf(value); + const Ti = @typeInfo(T); - return switch (dtype_) { - .bool => switch (Ti) { - .Bool => .{ .bool = value }, - .ComptimeInt, .Int, .ComptimeFloat, .Float => .{ .bool = value != 0 }, - else => @panic("Could not create Data of type bool from value of type " ++ @typeName(T)), - }, - inline .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz, .bf16 => |tag| switch (Ti) { - .ComptimeInt, .Int => @unionInit(Data, @tagName(tag), std.meta.FieldType(Data, tag).fromF32(@floatFromInt(value))), - .ComptimeFloat, .Float => @unionInit(Data, @tagName(tag), std.meta.FieldType(Data, tag).fromF32(@floatCast(value))), - else => @panic("Could not create Data of type bf16 from value of type " ++ @typeName(T)), - }, - inline .f16, .f32, .f64 => |tag| switch (Ti) { - .ComptimeInt, .Int => @unionInit(Data, @tagName(tag), @floatFromInt(value)), - .ComptimeFloat, .Float => @unionInit(Data, @tagName(tag), @floatCast(value)), - else => @panic("Could not create Data of type " ++ @tagName(tag) ++ " from value of type " ++ @typeName(T)), - }, - inline .i4, .i8, .i16, .i32, .i64, .u4, .u8, .u16, .u32, .u64 => |tag| switch (Ti) { - .ComptimeInt => blk: { - const OutT = std.meta.FieldType(Data, tag); - if (value >= std.math.minInt(OutT) and value <= std.math.maxInt(OutT)) { - break :blk @unionInit(Data, @tagName(tag), @intCast(value)); - } else { - @panic("Could not create Data of type " ++ @tagName(tag) ++ " from value of type " ++ @typeName(T)); - } + return switch (dtype_) { + .bool => switch (Ti) { + .bool => .{ .bool = value }, + .comptime_int, .int, .comptime_float, .float => .{ .bool = value != 0 }, + else => @panic("Could not create Value of type bool from value of type " ++ @typeName(T)), }, - .Int => @unionInit(Data, @tagName(tag), @intCast(value)), - else => @panic("Could not create Data of type " ++ @tagName(tag) ++ " from value of type " ++ @typeName(T)), - }, - .c64 => switch (T) { - C64 => .{ .c64 = value }, - C128 => .{ .c64 = .{ .re = @floatCast(value.re), .im = @floatCast(value.im) } }, - else => @panic("Could not create Data of type c64 from value of type " ++ @typeName(T)), - }, - .c128 => switch (T) { - C64 => .{ .c128 = .{ .re = @floatCast(value.re), .im = @floatCast(value.im) } }, - C128 => .{ .c128 = value }, - else => @panic("Could not create Data of type c128 from value of type " ++ @typeName(T)), - }, - }; - } - - test init { - try std.testing.expectEqual(20.0, Data.init(.f16, 20).f16); - try std.testing.expectEqual(20.5, Data.init(.f16, 20.5).f16); - try std.testing.expectEqual(20, Data.init(.f16, @as(u8, 20)).f16); - try std.testing.expectEqual(-20, Data.init(.f16, @as(i8, -20)).f16); - try std.testing.expectEqual(2000.5, Data.init(.f16, @as(f32, 2000.5)).f16); - - try std.testing.expectEqual(true, Data.init(.bool, true).bool); - - try std.testing.expectEqual(10, Data.init(.u8, 10).u8); - try std.testing.expectEqual(10, Data.init(.u8, @as(u16, 10)).u8); - - try std.testing.expectEqual(10, Data.init(.i8, 10).i8); - try std.testing.expectEqual(10, Data.init(.i8, @as(u16, 10)).i8); - try std.testing.expectEqual(-10, Data.init(.i8, -10).i8); - try std.testing.expectEqual(-10, Data.init(.i8, @as(i16, -10)).i8); - - try std.testing.expectEqual(C64.init(1, 2), Data.init(.c64, C64.init(1, 2)).c64); - try std.testing.expectEqual(C64.init(1, 2), Data.init(.c64, C128.init(1, 2)).c64); - try std.testing.expectEqual(C128.init(1, 2), Data.init(.c128, C128.init(1, 2)).c128); - try std.testing.expectEqual(C128.init(1, 2), Data.init(.c128, C64.init(1, 2)).c128); - } - - pub fn dtype(self: Data) DataType { - return std.meta.activeTag(self); - } - - pub fn constSlice(data: *const Data) []const u8 { - return switch (data.*) { - inline else => |*value| std.mem.asBytes(value), - }; - } - - pub fn as(self: Data, comptime T: type) T { - // TODO allow more lossless conversions - switch (@typeInfo(T)) { - .Bool => return self.bool, - .Float => switch (self) { - inline .f16, .f32, .f64 => |v| return @floatCast(v), - inline .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz, .bf16 => |v| return @floatCast(v.toF32()), - else => {}, - }, - .Int => switch (self) { - inline .i4, .i8, .i16, .i32, .i64, .u4, .u8, .u16, .u32, .u64 => |v| return @intCast(v), - else => {}, - }, - else => {}, + inline .f4e2m1, .f8e3m4, .f8e4m3, .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz, .f8e8m0, .bf16 => |tag| switch (Ti) { + .comptime_int, .int => @unionInit(Value, @tagName(tag), @FieldType(Value, @tagName(tag)).fromF32(@floatFromInt(value))), + .comptime_float, .float => @unionInit(Value, @tagName(tag), @FieldType(Value, @tagName(tag)).fromF32(@floatCast(value))), + else => @panic("Could not create Value of type bf16 from value of type " ++ @typeName(T)), + }, + inline .f16, .f32, .f64 => |tag| switch (Ti) { + .comptime_int, .int => @unionInit(Value, @tagName(tag), @floatFromInt(value)), + .comptime_float, .float => @unionInit(Value, @tagName(tag), @floatCast(value)), + else => @panic("Could not create Value of type " ++ @tagName(tag) ++ " from value of type " ++ @typeName(T)), + }, + inline .i2, .i4, .i8, .i16, .i32, .i64, .u2, .u4, .u8, .u16, .u32, .u64 => |tag| switch (Ti) { + .comptime_int => blk: { + const OutT = @FieldType(Value, @tagName(tag)); + if (value >= std.math.minInt(OutT) and value <= std.math.maxInt(OutT)) { + break :blk @unionInit(Value, @tagName(tag), @intCast(value)); + } else { + @panic("Could not create Value of type " ++ @tagName(tag) ++ " from value of type " ++ @typeName(T)); + } + }, + .int => @unionInit(Value, @tagName(tag), @intCast(value)), + else => @panic("Could not create Value of type " ++ @tagName(tag) ++ " from value of type " ++ @typeName(T)), + }, + .c64 => switch (T) { + C64 => .{ .c64 = value }, + C128 => .{ .c64 = .{ .re = @floatCast(value.re), .im = @floatCast(value.im) } }, + else => @panic("Could not create Value of type c64 from value of type " ++ @typeName(T)), + }, + .c128 => switch (T) { + C64 => .{ .c128 = .{ .re = @floatCast(value.re), .im = @floatCast(value.im) } }, + C128 => .{ .c128 = value }, + else => @panic("Could not create Value of type c128 from value of type " ++ @typeName(T)), + }, + }; } - std.debug.panic("Unsupported conversion {} -> {s}", .{ self.dtype(), @typeName(T) }); - } + + test init { + try std.testing.expectEqual(20.0, Value.init(.f16, 20).f16); + try std.testing.expectEqual(20.5, Value.init(.f16, 20.5).f16); + try std.testing.expectEqual(20, Value.init(.f16, @as(u8, 20)).f16); + try std.testing.expectEqual(-20, Value.init(.f16, @as(i8, -20)).f16); + try std.testing.expectEqual(2000.5, Value.init(.f16, @as(f32, 2000.5)).f16); + + try std.testing.expectEqual(true, Value.init(.bool, true).bool); + + try std.testing.expectEqual(10, Value.init(.u8, 10).u8); + try std.testing.expectEqual(10, Value.init(.u8, @as(u16, 10)).u8); + + try std.testing.expectEqual(10, Value.init(.i8, 10).i8); + try std.testing.expectEqual(10, Value.init(.i8, @as(u16, 10)).i8); + try std.testing.expectEqual(-10, Value.init(.i8, -10).i8); + try std.testing.expectEqual(-10, Value.init(.i8, @as(i16, -10)).i8); + + try std.testing.expectEqual(C64.init(1, 2), Value.init(.c64, C64.init(1, 2)).c64); + try std.testing.expectEqual(C64.init(1, 2), Value.init(.c64, C128.init(1, 2)).c64); + try std.testing.expectEqual(C128.init(1, 2), Value.init(.c128, C128.init(1, 2)).c128); + try std.testing.expectEqual(C128.init(1, 2), Value.init(.c128, C64.init(1, 2)).c128); + } + + pub fn dtype(self: Value) DataType { + return std.meta.activeTag(self); + } + + pub fn asBytes(data: *const Value) []const u8 { + return switch (data.*) { + inline else => |*value| std.mem.asBytes(value), + }; + } + + pub fn as(self: Value, comptime T: type) T { + // TODO allow more lossless conversions + switch (@typeInfo(T)) { + .bool => return self.bool, + .float => switch (self) { + inline .f16, .f32, .f64 => |v| return @floatCast(v), + inline .f4e2m1, .f8e3m4, .f8e4m3, .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz, .f8e8m0, .bf16 => |v| return @floatCast(v.toF32()), + else => {}, + }, + .int => switch (self) { + inline .i4, .i8, .i16, .i32, .i64, .u4, .u8, .u16, .u32, .u64 => |v| return @intCast(v), + else => {}, + }, + else => {}, + } + std.debug.panic("Unsupported conversion {} -> {s}", .{ self.dtype(), @typeName(T) }); + } + + test as { + try std.testing.expectEqual(@as(f32, 1.0), Value.init(.f4e2m1, 1).as(f32)); + try std.testing.expectEqual(@as(f32, 1.0), Value.init(.f8e3m4, 1).as(f32)); + try std.testing.expectEqual(@as(f32, 1.0), Value.init(.f8e4m3, 1).as(f32)); + try std.testing.expectEqual(@as(f32, 1.0), Value.init(.f8e8m0, 1).as(f32)); + } + }; }; pub fn mantissaSize(dtype: DataType) usize { @@ -342,3 +395,5 @@ pub fn mantissaSize(dtype: DataType) usize { else => @panic("Can't get mantissa size for a non-float dtype"), }; } + +pub const Data = DataType.Value; diff --git a/zml/exe.zig b/zml/exe.zig new file mode 100644 index 00000000..32c0ac48 --- /dev/null +++ b/zml/exe.zig @@ -0,0 +1,704 @@ +const std = @import("std"); +const builtin = @import("builtin"); + +const pjrt = @import("pjrt"); +const stdx = @import("stdx"); + +const Buffer = @import("buffer.zig").Buffer; +const Compiler = @import("Compiler.zig"); +const mem = @import("mem.zig"); +const meta = @import("meta.zig"); +const Platform = @import("platform.zig").Platform; +const tracer = @import("profiling/tracer.zig"); +const Shape = @import("shape.zig").Shape; +const Sharding = @import("Sharding.zig"); +const Tensor = @import("tensor.zig").Tensor; + +pub const Exe = struct { + platform: *const Platform, + exe: *pjrt.LoadedExecutable, + + context: ?*pjrt.ExecuteContext = null, + + input_shapes: []const Shape, + output_shapes: []const Shape, + + /// Inputs whose buffers are donated to an output (`reuseBuffer`). + /// Runners destroy their PJRT handles after the raw call returns. + donated_input_indices: []const u32, + + num_devices: u32, + num_partitions: u32, + + arena: std.heap.ArenaAllocator, + + pub fn init( + allocator: std.mem.Allocator, + platform: *const Platform, + exe: *pjrt.LoadedExecutable, + num_devices: u32, + num_partitions: u32, + input_shapes: []const Shape, + output_shapes: []const Shape, + input_aliasing: []const ?u32, + ) !Exe { + var arena = std.heap.ArenaAllocator.init(allocator); + errdefer arena.deinit(); + + const input_shapes_copy = try arena.allocator().dupe(Shape, input_shapes); + const output_shapes_copy = try arena.allocator().dupe(Shape, output_shapes); + + var donated_count: usize = 0; + for (input_aliasing) |aliasing| donated_count += @intFromBool(aliasing != null); + + const donated_input_indices = try arena.allocator().alloc(u32, donated_count); + + var donated_index: usize = 0; + for (input_aliasing, 0..) |aliasing, input_index| { + if (aliasing == null) continue; + donated_input_indices[donated_index] = @intCast(input_index); + donated_index += 1; + } + + return .{ + .platform = platform, + .exe = exe, + .input_shapes = input_shapes_copy, + .output_shapes = output_shapes_copy, + .donated_input_indices = donated_input_indices, + .num_devices = num_devices, + .num_partitions = num_partitions, + .arena = arena, + }; + } + + pub fn deinit(self: *const Exe) void { + if (self.context) |context| context.deinit(self.platform.pjrt_api); + self.exe.deinit(self.platform.pjrt_api); + self.arena.deinit(); + } + + pub fn args(self: *const Exe, allocator: std.mem.Allocator) !Arguments { + return Arguments.init(allocator, self.input_shapes, self.num_devices); + } + + pub fn results(self: *const Exe, allocator: std.mem.Allocator) !Results { + return Results.init(allocator, self.output_shapes, self.platform, self.num_devices); + } + + pub const FlatBuffers = struct { + buffers: []const [*]*pjrt.Buffer, + raw_buffers: []const *pjrt.Buffer, + + num_devices: usize, + + pub fn init(allocator: std.mem.Allocator, count: usize, num_devices: usize) !FlatBuffers { + const raw_buffers = try allocator.alloc(*pjrt.Buffer, num_devices * count); + errdefer allocator.free(raw_buffers); + + const buffers = try allocator.alloc([*]*pjrt.Buffer, num_devices); + errdefer allocator.free(buffers); + + for (0..num_devices) |i| { + buffers[i] = raw_buffers[i * count ..].ptr; + } + + return .{ + .buffers = buffers, + .raw_buffers = raw_buffers, + .num_devices = num_devices, + }; + } + + pub fn deinit(self: *const FlatBuffers, allocator: std.mem.Allocator) void { + allocator.free(self.buffers); + allocator.free(self.raw_buffers); + } + }; + + pub const Arguments = struct { + flat_buffers: FlatBuffers, + expected_shapes: []const Shape, + baked_count: usize = 0, + + pub fn init(allocator: std.mem.Allocator, shapes: []const Shape, num_devices: usize) error{OutOfMemory}!Arguments { + const flat_buffers = try FlatBuffers.init(allocator, shapes.len, num_devices); + errdefer flat_buffers.deinit(allocator); + + const expected_shapes = try allocator.dupe(Shape, shapes); + errdefer allocator.free(expected_shapes); + + return .{ + .flat_buffers = flat_buffers, + .expected_shapes = expected_shapes, + }; + } + + pub fn deinit(self: *const Arguments, allocator: std.mem.Allocator) void { + allocator.free(self.expected_shapes); + self.flat_buffers.deinit(allocator); + } + + pub fn set(self: *Arguments, v: anytype) void { + return self.setPartial(v, 0); + } + + pub fn setPartial(self: *Arguments, v: anytype, offset: usize) void { + const LocalContext = struct { + self: *Arguments, + current_index: usize = 0, + }; + var context: LocalContext = .{ .self = self, .current_index = offset + self.baked_count }; + meta.visit(struct { + fn cb(context_: *LocalContext, buffer: *const Buffer) void { + stdx.debug.assert( + context_.self.expected_shapes[context_.current_index].eql(buffer.shape()), + "Expected argument {} to have shape {f}, got {f}", + .{ context_.current_index, context_.self.expected_shapes[context_.current_index], buffer.shape() }, + ); + + const expected = context_.self.flat_buffers.num_devices; + const shard_count = buffer._shards.len; + + stdx.debug.assert( + shard_count == expected, + "Argument {} has {d} shards but executable expects {d}", + .{ context_.current_index, shard_count, expected }, + ); + + for (0..expected) |device_index| { + context_.self.flat_buffers.buffers[device_index][context_.current_index] = + buffer._shards.get(device_index); + } + + context_.current_index += 1; + } + }.cb, &context, &v); + } + + pub fn bake(self: *Arguments, v: anytype) void { + const LocalContext = struct { + self: *Arguments, + + current_index: usize = 0, + }; + + var context: LocalContext = .{ .self = self, .current_index = self.baked_count }; + + meta.visit(struct { + fn cb(context_: *LocalContext, buffer: *const Buffer) void { + stdx.debug.assert(context_.self.expected_shapes[context_.current_index].eql(buffer.shape()), "Expected argument {} to have shape {f}, got {f}", .{ context_.current_index, context_.self.expected_shapes[context_.current_index], buffer.shape() }); + + for (0..context_.self.flat_buffers.num_devices) |device_index| { + context_.self.flat_buffers.buffers[device_index][context_.current_index] = buffer._shards.get(device_index); + } + + context_.current_index += 1; + } + }.cb, &context, &v); + + self.baked_count = context.current_index; + } + }; + + pub const Results = struct { + platform: *const Platform, + flat_buffers: FlatBuffers, + + expected_shapes: []const Shape, + + pub fn init(allocator: std.mem.Allocator, shapes: []const Shape, platform: *const Platform, num_devices: usize) !Results { + const flat_buffers = try FlatBuffers.init(allocator, shapes.len, num_devices); + errdefer flat_buffers.deinit(allocator); + + const expected_shapes = try allocator.dupe(Shape, shapes); + errdefer allocator.free(expected_shapes); + + return .{ + .platform = platform, + .flat_buffers = flat_buffers, + .expected_shapes = expected_shapes, + }; + } + + pub fn deinit(self: *const Results, allocator: std.mem.Allocator) void { + allocator.free(self.expected_shapes); + self.flat_buffers.deinit(allocator); + } + + pub fn get(self: *Results, comptime T: type) T { + var result: T = undefined; + const LocalContext = struct { + self: *Results, + current_index: usize = 0, + }; + var context: LocalContext = .{ .self = self, .current_index = 0 }; + meta.visit(struct { + fn cb(context_: *LocalContext, buffer: *Buffer) void { + var shards: Buffer.Shards = .empty; + for (0..context_.self.flat_buffers.num_devices) |device_index| { + shards.appendAssumeCapacity(context_.self.flat_buffers.buffers[device_index][context_.current_index]); + } + buffer.* = Buffer.fromPjrtBuffers(context_.self.platform, context_.self.expected_shapes[context_.current_index], shards.constSlice()); + context_.current_index += 1; + } + }.cb, &context, &result); + return result; + } + + pub fn fill(self: *Results, v: anytype) void { + const LocalContext = struct { + results: *Results, + current_index: usize = 0, + }; + var context: LocalContext = .{ .results = self, .current_index = 0 }; + meta.visit(struct { + fn cb(ctx: *LocalContext, buffer: *Buffer) void { + //stdx.debug.assert(ctx.results.expected_shapes[ctx.current_index].eql(buffer.shape()), "Expected result {} to have shape {f}, got {f}", .{ ctx.current_index, ctx.results.expected_shapes[ctx.current_index], buffer.shape() }); + var shards: Buffer.Shards = .empty; + for (0..ctx.results.flat_buffers.num_devices) |device_index| { + shards.appendAssumeCapacity(ctx.results.flat_buffers.buffers[device_index][ctx.current_index]); + } + buffer.* = Buffer.fromPjrtBuffers(ctx.results.platform, ctx.results.expected_shapes[ctx.current_index], shards.constSlice()); + ctx.current_index += 1; + } + }.cb, &context, &v); + } + }; + + /// Executes a single-output executable and waits for its result. + /// The caller owns the returned buffer. Donated inputs are consumed. + pub fn eval(self: *const Exe, allocator: std.mem.Allocator, io: std.Io, input_values: anytype) !Buffer { + stdx.debug.assert(self.output_shapes.len == 1, "eval expects one output, got {d}; use a runner for multiple outputs", .{self.output_shapes.len}); + var result: Buffer = undefined; + var r = try self.runner(allocator); + defer r.deinit(allocator); + r.run(io, input_values, .{&result}, .{ .wait = true }); + return result; + } + + pub fn runner(self: *const Exe, allocator: std.mem.Allocator) !Runner { + return .init(self, allocator); + } + + /// Reusable argument and result storage for a borrowed executable. + /// The executable must outlive the runner, and the runner must not be used concurrently. + pub const Runner = struct { + exe: *const Exe, + args: Arguments, + results: Results, + + pub fn init(exe: *const Exe, allocator: std.mem.Allocator) !Runner { + var arguments = try exe.args(allocator); + errdefer arguments.deinit(allocator); + var results_ = try exe.results(allocator); + errdefer results_.deinit(allocator); + return .{ + .exe = exe, + .args = arguments, + .results = results_, + }; + } + + pub fn deinit(self: *Runner, allocator: std.mem.Allocator) void { + self.results.deinit(allocator); + self.args.deinit(allocator); + } + + /// Destroys donated input handles and fills the output destinations. + /// Use the donated buffer's address as its output destination to replace it safely. + /// Non-donated inputs remain owned by the caller. + pub fn run(self: *Runner, io: std.Io, input_values: anytype, output_refs: anytype, opts: CallOpts) void { + self.args.set(input_values); + self.exe.call(io, self.args, &self.results, opts); + self.exe.deinitDonatedInputs(self.args); + self.results.fill(output_refs); + } + }; + + pub const CallOpts = struct { + /// Waits for execution to complete before returning. + wait: bool = false, + }; + + fn deinitDonatedInputs(self: *const Exe, arguments: Arguments) void { + // Execution deletes donated inputs, but leaves their PJRT handles allocated. + // Arguments retain the old handles even when outputs replace the caller's buffers. + for (self.donated_input_indices) |input_index| { + for (arguments.flat_buffers.buffers[0..self.num_devices]) |device_buffers| { + device_buffers[input_index].deinit(self.platform.pjrt_api); + } + } + } + + /// Executes with optional waiting. The caller owns all input handles, including donated inputs. + /// Donated input contents cannot be used after the call; output buffers have their own handles. + /// Use a runner to destroy donated handles automatically and replace buffers through output destinations. + pub fn call(self: *const Exe, io: std.Io, arguments: Arguments, results_: *Results, opts: CallOpts) void { + var span = tracer.span("zml.exe.call", .{ + .wait = opts.wait, + .arg_count = arguments.expected_shapes.len, + .result_count = results_.expected_shapes.len, + }); + defer span.end(); + + var events: [Platform.MAX_NUM_DEVICES]?*pjrt.Event = @splat(null); + + const partition_events = events[0..@intCast(self.num_partitions)]; + const events_slice: ?[]?*pjrt.Event = switch (self.platform.target) { + .neuron => partition_events, + .cpu, .cuda, .rocm, .tpu, .oneapi, .metal, .furiosa => if (opts.wait) partition_events else null, + }; + + self.exe.execute(self.platform.pjrt_api, .{ + .arguments = arguments.flat_buffers.buffers, + .num_args = arguments.expected_shapes.len, + .results = results_.flat_buffers.buffers, + .events = events_slice, + // TODO: Set this flag for undonatable buffers. + .non_donatable_input_indices = &.{}, + .context = self.context, + }) catch |err| { + std.debug.panic("PJRT_LoadedExecutable_Execute failed with: {}", .{err}); + }; + + switch (self.platform.target) { + .neuron => { + for (events_slice.?) |e| { + if (e) |ev| { + if (opts.wait) { + ev.await(self.platform.pjrt_api, io) catch |err| { + std.debug.panic("PJRT execution failed with: {}", .{err}); + }; + } + ev.deinit(self.platform.pjrt_api); + } + } + }, + .cpu, .cuda, .rocm, .tpu, .oneapi, .metal, .furiosa => if (opts.wait) { + for (events_slice.?) |e| { + if (e) |ev| { + ev.await(self.platform.pjrt_api, io) catch |err| { + std.debug.panic("PJRT execution failed with: {}", .{err}); + }; + ev.deinit(self.platform.pjrt_api); + } + } + }, + } + } + + pub fn memoryStats(self: *const Exe) pjrt.ApiError!pjrt.CompiledMemoryStats { + const exe = try self.exe.executable(self.platform.pjrt_api); + return try exe.getCompiledMemoryStats(self.platform.pjrt_api); + } +}; + +/// A typed executable whose input and output buffer structures are derived from +/// the function used to compile it. +pub fn FnExe(comptime function_: anytype) type { + const function_info = switch (@typeInfo(@TypeOf(function_))) { + .@"fn" => |info| info, + else => @compileError("FnExe expects a function, got " ++ @typeName(@TypeOf(function_))), + }; + if (function_info.attrs.varargs or function_info.param_types.len != 1) { + @compileError("FnExe function must accept exactly one input struct"); + } + + const FunctionInput = typedFunctionInput(function_); + const FunctionOutput = typedFunctionOutput(function_); + + return struct { + raw: Exe, + + pub const Input = mem.Bufferized(FunctionInput); + pub const Output = typedOutputDestinations(mem.Bufferized(FunctionOutput)); + + const Self = @This(); + + pub fn init(raw: Exe) Self { + return .{ .raw = raw }; + } + + pub fn compile( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + opts: Compiler.Options, + args: std.meta.ArgsTuple(@TypeOf(function_)), + ) Compiler.Error!Self { + return .{ .raw = try Compiler.Typed(function_).compile(allocator, io, platform, opts, args) }; + } + + pub fn deinit(self: *const Self) void { + self.raw.deinit(); + } + + pub fn runner(self: *const Self, allocator: std.mem.Allocator) !Runner(.{}) { + return .init(self, allocator, .{}); + } + + /// Reusable argument and result storage for a borrowed executable. + /// The executable must outlive the runner, and the runner must not be used concurrently. + pub fn Runner(comptime baked_fields: anytype) type { + const count = countBackedFields(Input, baked_fields); + + return struct { + exe: *const Exe, + args: Exe.Arguments, + results: Exe.Results, + + const RunnerSelf = @This(); + + pub const BakedInput = structFieldRange(Input, 0, count); + pub const NonBakedInput = structFieldRange(Input, count, @typeInfo(Input).@"struct".field_names.len); + + pub fn init(exe: *const Self, allocator: std.mem.Allocator, baked: BakedInput) !RunnerSelf { + var arguments = try exe.raw.args(allocator); + errdefer arguments.deinit(allocator); + var results = try exe.raw.results(allocator); + errdefer results.deinit(allocator); + arguments.bake(baked); + return .{ .exe = &exe.raw, .args = arguments, .results = results }; + } + + pub fn deinit(self: *RunnerSelf, allocator: std.mem.Allocator) void { + self.results.deinit(allocator); + self.args.deinit(allocator); + } + + pub const Call = struct { + inputs: NonBakedInput, + outputs: Output, + }; + + /// Destroys donated input handles before filling outputs; non-donated inputs remain caller-owned. + pub fn run(self: *RunnerSelf, io: std.Io, call: Call, opts: Exe.CallOpts) void { + self.args.set(call.inputs); + self.exe.call(io, self.args, &self.results, opts); + self.exe.deinitDonatedInputs(self.args); + self.results.fill(call.outputs); + } + }; + } + }; +} + +fn countBackedFields(comptime Input: type, comptime baked_fields: anytype) usize { + const baked_info = switch (@typeInfo(@TypeOf(baked_fields))) { + .@"struct" => |info| info, + else => @compileError("FnExe baked fields must be a tuple of enum literals"), + }; + if (!baked_info.is_tuple) { + @compileError("FnExe baked fields must be a tuple of enum literals"); + } + + const input_info = @typeInfo(Input).@"struct"; + if (baked_info.field_names.len > input_info.field_names.len) { + @compileError("FnExe baked fields must be a prefix of its bufferized input fields"); + } + + inline for (baked_info.field_names, 0..) |tuple_field_name, i| { + const baked_field = @field(baked_fields, tuple_field_name); + if (@TypeOf(baked_field) != @EnumLiteral()) { + @compileError("FnExe baked fields must be enum literals"); + } + if (!std.mem.eql(u8, @tagName(baked_field), input_info.field_names[i])) { + @compileError("FnExe baked fields must be an ordered prefix of its bufferized input fields"); + } + } + return baked_info.field_names.len; +} + +fn structFieldRange(comptime Struct: type, comptime start: usize, comptime end: usize) type { + const info = @typeInfo(Struct).@"struct"; + return @Struct(.auto, null, info.field_names[start..end], info.field_types[start..end], info.field_attrs[start..end]); +} + +fn typedFunctionInput(comptime function_: anytype) type { + const Input = @typeInfo(@TypeOf(function_)).@"fn".param_types[0] orelse + @compileError("FnExe function must have a concrete input type"); + validateFnExeStruct("input", Input); + return Input; +} + +fn typedFunctionOutput(comptime function_: anytype) type { + const Output = @typeInfo(@TypeOf(function_)).@"fn".return_type orelse @compileError("FnExe function must return an output struct"); + validateFnExeStruct("output", Output); + return Output; +} + +fn validateFnExeStruct(comptime role: []const u8, comptime T: type) void { + const info = switch (@typeInfo(T)) { + .@"struct" => |info| info, + else => @compileError("FnExe " ++ role ++ " must be a struct, got " ++ @typeName(T)), + }; + if (info.is_tuple) { + @compileError("FnExe " ++ role ++ " must use named fields"); + } +} + +fn typedOutputDestinations(comptime BufferizedOutput: type) type { + const output_info = @typeInfo(BufferizedOutput).@"struct"; + var field_types: [output_info.field_names.len]type = undefined; + var field_attrs: [output_info.field_names.len]std.builtin.Type.Struct.FieldAttributes = undefined; + for (&field_types, &field_attrs, output_info.field_types) |*T, *attrs, Field| { + T.* = *Field; + attrs.* = .{ .@"align" = @alignOf(*Field) }; + } + return @Struct(.auto, null, output_info.field_names, &field_types, &field_attrs); +} + +test "FnExe derives baked and runtime calls from a function" { + const Inputs = struct { + weights: struct { + tensor: Tensor, + scale: f32, + }, + bias: Tensor, + tokens: Tensor, + token_count: usize, + }; + const Outputs = struct { + hidden: Tensor, + cache: struct { + key: Tensor, + length: usize, + }, + metadata: usize, + }; + const Functions = struct { + fn forward(inputs: Inputs) Outputs { + _ = inputs; + return undefined; + } + }; + + const Model = FnExe(Functions.forward); + try std.testing.expect(@hasField(Model.Input, "weights")); + try std.testing.expect(@hasField(Model.Input, "bias")); + try std.testing.expect(@hasField(Model.Input, "tokens")); + try std.testing.expect(!@hasField(Model.Input, "token_count")); + try std.testing.expect(!@hasField(@FieldType(Model.Input, "weights"), "scale")); + + const Baked = Model.Runner(.{ .weights, .bias }).BakedInput; + try std.testing.expect(@hasField(Baked, "weights")); + try std.testing.expect(@hasField(Baked, "bias")); + try std.testing.expect(!@hasField(Baked, "tokens")); + + const Runtime = Model.Runner(.{ .weights, .bias }).NonBakedInput; + try std.testing.expect(!@hasField(Runtime, "weights")); + try std.testing.expect(!@hasField(Runtime, "bias")); + try std.testing.expectEqual(Buffer, @FieldType(Runtime, "tokens")); + + try std.testing.expectEqual(*Buffer, @FieldType(Model.Output, "hidden")); + try std.testing.expectEqual( + *mem.Bufferized(@FieldType(Outputs, "cache")), + @FieldType(Model.Output, "cache"), + ); + try std.testing.expect(!@hasField(Model.Output, "metadata")); +} + +test "Exe raw calls leave donated input handle cleanup to the caller" { + const platform = @import("testing.zig").env(); + const allocator = std.testing.allocator; + const io = std.testing.io; + const Functions = struct { + fn forward(value: Tensor, weight: Tensor) Tensor { + return value.add(weight).reuseBuffer(value); + } + }; + const value: Tensor = .init(.{2}, .i32); + const weight: Tensor = .init(.{2}, .i32); + const exe = try platform.compileFn(allocator, io, Functions.forward, .{ value, weight }, .{}); + defer exe.deinit(); + try std.testing.expectEqualSlices(u32, &.{0}, exe.donated_input_indices); + + var weight_buffer = try Buffer.fromBytes(io, platform, weight.shape(), std.mem.asBytes(&[2]i32{ 10, 20 })); + defer weight_buffer.deinit(); + var args = try exe.args(allocator); + defer args.deinit(allocator); + var results = try exe.results(allocator); + defer results.deinit(allocator); + + for ([_]bool{ false, true }) |wait| { + var value_buffer = try Buffer.fromBytes(io, platform, value.shape(), std.mem.asBytes(&[2]i32{ 1, 2 })); + defer value_buffer.deinit(); + args.set(.{ value_buffer, weight_buffer }); + exe.call(io, args, &results, .{ .wait = wait }); + var output = results.get(Buffer); + defer output.deinit(); + + // The donated handle is still valid for cleanup even though its contents are deleted. + for (value_buffer._shards.constSlice()) |shard| { + try std.testing.expect(shard.isDeleted(platform.pjrt_api)); + } + value_buffer.deinit(); + try std.testing.expectEqual([2]i32{ 11, 22 }, try output.getValue([2]i32, io)); + try std.testing.expectEqual([2]i32{ 10, 20 }, try weight_buffer.getValue([2]i32, io)); + } +} + +test "FnExe runner replaces donated buffers with and without waiting" { + const platform = @import("testing.zig").env(); + const allocator = std.testing.allocator; + const io = std.testing.io; + const Functions = struct { + fn forward(inputs: struct { weight: Tensor, value: Tensor }) struct { value: Tensor } { + return .{ .value = inputs.value.add(inputs.weight).reuseBuffer(inputs.value) }; + } + }; + const Model = FnExe(Functions.forward); + const weight: Tensor = .init(.{2}, .i32); + const value: Tensor = .init(.{2}, .i32); + const exe = try Model.compile(allocator, io, platform, .{}, .{.{ .weight = weight, .value = value }}); + defer exe.deinit(); + + var weight_buffer = try Buffer.fromBytes(io, platform, weight.shape(), std.mem.asBytes(&[2]i32{ 10, 20 })); + defer weight_buffer.deinit(); + var value_buffer = try Buffer.fromBytes(io, platform, value.shape(), std.mem.asBytes(&[2]i32{ 1, 2 })); + defer value_buffer.deinit(); + var runner = try Model.Runner(.{.weight}).init(&exe, allocator, .{ .weight = weight_buffer }); + defer runner.deinit(allocator); + + runner.run(io, .{ + .inputs = .{ .value = value_buffer }, + .outputs = .{ .value = &value_buffer }, + }, .{}); + try std.testing.expectEqual([2]i32{ 11, 22 }, try value_buffer.getValue([2]i32, io)); + + runner.run(io, .{ + .inputs = .{ .value = value_buffer }, + .outputs = .{ .value = &value_buffer }, + }, .{ .wait = true }); + try std.testing.expectEqual([2]i32{ 21, 42 }, try value_buffer.getValue([2]i32, io)); +} + +test "FnExe runner replaces unchanged donated buffers" { + const platform = @import("testing.zig").env(); + const allocator = std.testing.allocator; + const io = std.testing.io; + const Functions = struct { + fn forward(inputs: struct { value: Tensor }) struct { value: Tensor } { + return .{ .value = inputs.value.reuseBuffer(inputs.value) }; + } + }; + const Model = FnExe(Functions.forward); + const value: Tensor = .init(.{2}, .i32); + const exe = try Model.compile(allocator, io, platform, .{}, .{.{ .value = value }}); + defer exe.deinit(); + try std.testing.expectEqualSlices(u32, &.{0}, exe.raw.donated_input_indices); + + var value_buffer = try Buffer.fromBytes(io, platform, value.shape(), std.mem.asBytes(&[2]i32{ 1, 2 })); + defer value_buffer.deinit(); + var runner = try exe.runner(allocator); + defer runner.deinit(allocator); + + for ([_]bool{ false, true }) |wait| { + runner.run(io, .{ + .inputs = .{ .value = value_buffer }, + .outputs = .{ .value = &value_buffer }, + }, .{ .wait = wait }); + try std.testing.expectEqual([2]i32{ 1, 2 }, try value_buffer.getValue([2]i32, io)); + } +} diff --git a/zml/floats.zig b/zml/floats.zig index 5adb5226..689c22a8 100644 --- a/zml/floats.zig +++ b/zml/floats.zig @@ -1,236 +1,572 @@ +///! Conversion utilities between different Floating point formats. const std = @import("std"); -test { - std.testing.refAllDecls(@This()); -} - -fn allBitsOne(v: anytype) bool { - return v == std.math.maxInt(@TypeOf(v)); -} - -fn FloatType(sign_bits: u1, exponent_bits: u8, mantissa_bits: u8, innerT: type) type { - return packed struct(std.meta.Int(.unsigned, @intCast(sign_bits + exponent_bits + mantissa_bits))) { - const Self = @This(); - - mantissa: std.meta.Int(.unsigned, @intCast(mantissa_bits)), - exponent: std.meta.Int(.unsigned, @intCast(exponent_bits)), - sign: std.meta.Int(.unsigned, @intCast(sign_bits)), - - pub fn zero() Self { - return .{ - .sign = 0, - .exponent = 0, - .mantissa = 0, - }; +fn FloatConversionHelpers(Float: type) type { + const info = @typeInfo(Float); + const err_msg = "FloatHelpers expect a packed struct { mantissa: uXX, exponent: uXX, sign: u1}"; + if (info != .@"struct" or info.@"struct".backing_integer == null) { + @compileError(err_msg); + } + comptime { + for (info.@"struct".field_names, &.{ "mantissa", "exponent", "sign" }) |field_name, expected_name| { + if (!std.mem.eql(u8, field_name, expected_name)) + @compileError(err_msg); } + } - pub fn neg(self: Self) Self { - return .{ - .sign = self.sign ^ 1, - .exponent = self.exponent, - .mantissa = self.mantissa, - }; - } + const zero = FloatHelpers(Float).zero; + + return struct { + const mantissa_bits: u8 = @typeInfo(@FieldType(Float, "mantissa")).int.bits; + const exponent_bits: u8 = @typeInfo(@FieldType(Float, "exponent")).int.bits; + const f32_mantissa_bits: u8 = @typeInfo(@FieldType(Float32, "mantissa")).int.bits; + const exp_bias: i16 = std.math.maxInt(@Int(.unsigned, exponent_bits - 1)); + const exp_off: u8 = FloatConversionHelpers(Float32).exp_bias - exp_bias; /// Lossy conversion from f32, similar to @floatCast - pub fn fromF32(f: f32) Self { + pub fn fromF32(f: f32) Float { + @setRuntimeSafety(false); + const vf32: Float32 = @bitCast(f); - const precision_loss = @bitSizeOf(@TypeOf(vf32.mantissa)) - mantissa_bits; - return .{ - .sign = vf32.sign, - .exponent = @intCast(vf32.exponent), - .mantissa = shr(vf32.mantissa, precision_loss), - }; + const exponent: i16 = @as(i16, vf32.exponent) - exp_off; + const overflow = std.math.isInf(f) or exponent > std.math.maxInt(@FieldType(Float, "exponent")); + if (overflow) { + @branchHint(.unlikely); + return if (@hasDecl(Float, "inf")) + if (vf32.sign == 0) Float.inf else Float.minus_inf + else if (@hasDecl(Float, "nan")) + Float.nan + else + std.debug.panic("{} doesn't have infinite representations", .{Float}); + } + + return if (exponent <= 0) + .{ + .sign = vf32.sign, + .exponent = 0, + .mantissa = shiftMantissa(vf32.mantissa, @intCast(-exponent)), + } + else + .{ + .sign = vf32.sign, + .exponent = @intCast(exponent), + .mantissa = truncMantissa(vf32.mantissa), + }; } /// Lossless conversion to f32. - pub fn toF32(self: Self) f32 { - var vf32: Float32 = undefined; - const precision_loss = @bitSizeOf(@TypeOf(vf32.mantissa)) - mantissa_bits; - vf32 = .{ - .sign = self.sign, - .exponent = self.exponent, - .mantissa = @shlExact(@as(@TypeOf(vf32.mantissa), self.mantissa), precision_loss), - }; + pub fn toF32(x: Float) f32 { + @setRuntimeSafety(false); + + if (x == zero) return 0.0; + if (isInf(x)) { + @branchHint(.unlikely); + return if (x.sign == 0) std.math.inf(f32) else -std.math.inf(f32); + } + + const vf32: Float32 = if (x.exponent > 0) + .{ + .sign = x.sign, + .exponent = @as(u8, x.exponent) + exp_off, + .mantissa = f32Mantissa(x), + } + else + .{ + .sign = x.sign, + .exponent = exp_off - @clz(x.mantissa), + .mantissa = @as(u23, x.mantissa) << @clz(x.mantissa), + }; return @bitCast(vf32); } - fn truncMantissa(T: type, x: anytype) T { - const off = @bitSizeOf(@TypeOf(x)) - @bitSizeOf(T); - return @intCast(x >> off); + fn truncMantissa(f32_mantissa: u32) @FieldType(Float, "mantissa") { + const rounding_val: u32 = @as(u32, 1) << (f32_mantissa_bits - mantissa_bits - 1); + return @truncate((f32_mantissa + rounding_val) >> (f32_mantissa_bits - mantissa_bits)); } - fn shr(x: anytype, comptime off: u8) std.meta.Int(.unsigned, @bitSizeOf(@TypeOf(x)) - off) { - // @setRuntimeSafety(false); - return @intCast(x >> off); + fn shiftMantissa(f32_mantissa: u32, underflow: u8) @FieldType(Float, "mantissa") { + const upper_bit: u32 = @as(u32, 1) << f32_mantissa_bits; + const full_mant32: u32 = f32_mantissa | upper_bit; + // divide the mantissa proportionally to the exponent underflow + const shifted_mant: u32 = full_mant32 >> @truncate(underflow + 1); + return truncMantissa(shifted_mant); } - pub fn format( - self: @This(), - comptime fmt: []const u8, - options: std.fmt.FormatOptions, - writer: anytype, - ) !void { - _ = options; - try writer.print("{" ++ fmt ++ "}", .{self.toF32()}); + fn f32Mantissa(x: Float) @FieldType(Float32, "mantissa") { + const T = @FieldType(Float32, "mantissa"); + return @as(T, x.mantissa) << f32_mantissa_bits - mantissa_bits; } - - pub usingnamespace innerT; }; } -const Float32 = FloatType(1, 8, 23, struct {}); -const Float64 = FloatType(1, 11, 52, struct {}); - -pub const Float8E4M3B11FNUZ = FloatType(1, 4, 3, struct { - pub fn nan() Float8E4M3B11FNUZ { - return .{ - .sign = 1, - .exponent = 0, - .mantissa = 0, - }; +fn FloatHelpers(Float: type) type { + const info = @typeInfo(Float); + const err_msg = "FloatHelpers expect a packed struct { mantissa: uXX, exponent: uXX, sign: u1}"; + if (info != .@"struct" or info.@"struct".backing_integer == null) { + @compileError(err_msg); } + comptime { + for (info.@"struct".field_names, &.{ "mantissa", "exponent", "sign" }) |field_name, expected_name| { + if (!std.mem.eql(u8, field_name, expected_name)) + @compileError(err_msg); + } + } + + return struct { + pub const zero: Float = .{ .sign = 0, .exponent = 0, .mantissa = 0 }; + + pub fn neg(x: Float) Float { + return .{ + .sign = x.sign ^ 1, + .exponent = x.exponent, + .mantissa = x.mantissa, + }; + } + + pub fn formatNumber(x: Float, writer: *std.Io.Writer, n: std.fmt.Number) std.Io.Writer.Error!void { + switch (n.mode) { + .binary, .octal, .hex => try writer.print("{{ .sign={}, .exp={}, .mantissa={} }}", .{ x.sign, x.exponent, x.mantissa }), + else => try writer.printFloat(x.toF32(), n), + } + } + }; +} + +pub const Float32 = packed struct(u32) { + mantissa: u23, + exponent: u8, + sign: u1, + + pub const nan: Float32 = .{ .sign = 0, .exponent = std.math.maxInt(u8), .mantissa = 1 }; + pub const max: Float32 = .{ .sign = 0, .exponent = std.math.maxInt(u8) - 1, .mantissa = std.math.maxInt(u23) }; + pub const min: Float32 = neg(max); + pub const inf: Float32 = .{ .sign = 0, .exponent = std.math.maxInt(u8), .mantissa = 0 }; + pub const minus_inf = neg(inf); + + pub fn isNan(self: Float32) bool { + return self.exponent == nan.exponent and self.mantissa != 0; + } + + const Helpers = FloatHelpers(@This()); + pub const zero = Helpers.zero; + pub const neg = Helpers.neg; + + pub fn fromF32(f: f32) Float32 { + return @bitCast(f); + } + + pub fn toF32(x: Float32) f32 { + return @bitCast(x); + } + + pub const formatNumber = Helpers.formatNumber; +}; + +const f32_exp_bias = FloatHelpers(Float32).expBias(); + +pub const Float64 = packed struct(u64) { + mantissa: u52, + exponent: u11, + sign: u1, + + pub const nan: Float64 = .{ .sign = 0, .exponent = std.math.maxInt(u11), .mantissa = 1 }; + pub const max: Float64 = .{ .sign = 0, .exponent = std.math.maxInt(u11) - 1, .mantissa = std.math.maxInt(u32) }; + pub const min: Float64 = neg(max); + pub const inf: Float64 = .{ .sign = 0, .exponent = std.math.maxInt(u11), .mantissa = 0 }; + pub const minus_inf = neg(inf); + + pub fn isNan(self: Float64) bool { + return self.exponent == nan.exponent and self.mantissa != 0; + } + + const Helpers = FloatHelpers(@This()); + pub const zero = Helpers.zero; + pub const neg = Helpers.neg; + pub fn fromF32(f: f32) Float64 { + return @bitCast(@as(f64, @floatCast(f))); + } + + pub fn toF32(x: Float64) f32 { + return @floatCast(@as(f64, @bitCast(x))); + } + + pub const formatNumber = Helpers.formatNumber; +}; + +pub const Float8E4M3B11FNUZ = packed struct(u8) { + mantissa: u3, + exponent: u4, + sign: u1, + + pub const nan: Float8E4M3B11FNUZ = .{ .sign = 1, .exponent = 0, .mantissa = 0 }; + pub const max: Float8E4M3B11FNUZ = .{ .sign = 0, .exponent = std.math.maxInt(u4), .mantissa = std.math.maxInt(u3) }; + pub const min: Float8E4M3B11FNUZ = neg(max); pub fn isNan(self: Float8E4M3B11FNUZ) bool { - return self.sign == 1 and self.exponent == 0 and self.mantissa == 0; + return self == nan; } -}); -pub const Float8E4M3FN = FloatType(1, 4, 3, struct { - pub fn nan() Float8E4M3FN { - return .{ - .sign = 0, - .exponent = std.math.maxInt(u4), - .mantissa = std.math.maxInt(u3), - }; - } + const Helpers = FloatHelpers(@This()); + pub const zero = Helpers.zero; + pub const neg = Helpers.neg; + pub const fromF32 = FloatConversionHelpers(@This()).fromF32; + pub const toF32 = FloatConversionHelpers(@This()).toF32; + pub const formatNumber = Helpers.formatNumber; +}; + +pub const Float8E4M3FN = packed struct(u8) { + mantissa: u3, + exponent: u4, + sign: u1, + + pub const nan: Float8E4M3FN = .{ .sign = 0, .exponent = std.math.maxInt(u4), .mantissa = std.math.maxInt(u3) }; + pub const max: Float8E4M3FN = .{ .sign = 0, .exponent = std.math.maxInt(u4), .mantissa = 0b110 }; + pub const min: Float8E4M3FN = neg(max); pub fn isNan(self: Float8E4M3FN) bool { - return allBitsOne(self.exponent) and allBitsOne(self.mantissa); + return self.exponent == nan.exponent and self.mantissa == nan.mantissa; } -}); -pub const Float8E4M3FNUZ = FloatType(1, 4, 3, struct { - pub fn nan() Float8E4M3FNUZ { - return .{ - .sign = 1, - .exponent = 0, - .mantissa = 0, - }; - } + const Helpers = FloatHelpers(@This()); + pub const zero = Helpers.zero; + pub const neg = Helpers.neg; + pub const fromF32 = FloatConversionHelpers(@This()).fromF32; + pub const toF32 = FloatConversionHelpers(@This()).toF32; + pub const formatNumber = Helpers.formatNumber; +}; + +pub const Float8E4M3FNUZ = packed struct(u8) { + mantissa: u3, + exponent: u4, + sign: u1, + + pub const nan: Float8E4M3FNUZ = .{ .sign = 1, .exponent = 0, .mantissa = 0 }; + pub const max: Float8E4M3FNUZ = .{ .sign = 0, .exponent = std.math.maxInt(u4), .mantissa = std.math.maxInt(u3) }; + pub const min: Float8E4M3FNUZ = neg(max); pub fn isNan(self: Float8E4M3FNUZ) bool { - return self.sign == 1 and self.exponent == 0 and self.mantissa == 0; + return self == nan; } -}); -pub const Float8E5M2 = FloatType(1, 5, 2, struct { - pub fn nan() Float8E5M2 { - return .{ - .sign = 0, - .exponent = std.math.maxInt(u5), - .mantissa = 1, - }; + const Helpers = FloatHelpers(@This()); + pub const zero = Helpers.zero; + pub const neg = Helpers.neg; + pub const fromF32 = FloatConversionHelpers(@This()).fromF32; + pub const toF32 = FloatConversionHelpers(@This()).toF32; + pub const formatNumber = Helpers.formatNumber; +}; + +test "Float8E4" { + // With 4 bits of exponents power of two can be represented exactly up to 64. + const test_case_e4: TestCase = .{ + .lossless = &[_]f32{ 0, 1.0, -2, 1.0 / 64.0, -128, -1.125 / 64.0 }, + .lossy = &[_]f32{ 3.02344107628, 1.0 / 128.0, 1.0 / 512.0 }, + }; + + inline for (.{ + Float8E4M3B11FNUZ, + Float8E4M3FN, + Float8E4M3FNUZ, + }) |Float8T| { + try testCustomFloat(Float8T, test_case_e4); + try std.testing.expectEqual(0.0, Float8T.fromF32(1.0 / 2048.0).toF32()); + + if (@hasDecl(Float8T, "inf")) @compileError("Float8E4 don't have ±∞ representations"); } +} + +pub const Float8E5M2 = packed struct(u8) { + mantissa: u2, + exponent: u5, + sign: u1, + + pub const nan: Float8E5M2 = .{ .sign = 0, .exponent = std.math.maxInt(u5), .mantissa = 0b01 }; + pub const max: Float8E5M2 = .{ .sign = 0, .exponent = 0b11110, .mantissa = std.math.maxInt(u2) }; + pub const min: Float8E5M2 = neg(max); + pub const inf: Float8E5M2 = .{ .sign = 0, .exponent = std.math.maxInt(u5), .mantissa = 0 }; + pub const minus_inf: Float8E5M2 = .neg(inf); pub fn isNan(self: Float8E5M2) bool { - return allBitsOne(self.exponent) and self.mantissa != 0; + return self.exponent == nan.exponent and self.mantissa != 0; } - pub fn minusInf() Float8E5M2 { - return .{ - .sign = 1, - .exponent = std.math.maxInt(u5), - .mantissa = 0, - }; - } + const Helpers = FloatHelpers(@This()); + pub const zero = Helpers.zero; + pub const neg = Helpers.neg; + pub const fromF32 = FloatConversionHelpers(@This()).fromF32; + pub const toF32 = FloatConversionHelpers(@This()).toF32; + pub const formatNumber = Helpers.formatNumber; +}; - pub fn inf() Float8E5M2 { - return .{ - .sign = 0, - .exponent = std.math.maxInt(u5), - .mantissa = 0, - }; - } +pub const Float8E5M2FNUZ = packed struct(u8) { + mantissa: u2, + exponent: u5, + sign: u1, - pub fn isInf(self: Float8E5M2) bool { - return allBitsOne(self.exponent) and self.mantissa == 0; - } -}); - -pub const Float8E5M2FNUZ = FloatType(1, 5, 2, struct { - pub fn nan() Float8E5M2FNUZ { - return .{ - .sign = 1, - .exponent = 0, - .mantissa = 0, - }; - } + pub const nan: Float8E5M2FNUZ = .{ .sign = 1, .exponent = 0, .mantissa = 0 }; + pub const max: Float8E5M2FNUZ = .{ .sign = 0, .exponent = std.math.maxInt(u5), .mantissa = std.math.maxInt(u2) }; + pub const min: Float8E5M2FNUZ = neg(max); pub fn isNan(self: Float8E5M2FNUZ) bool { - return self.sign == 1 and self.exponent == 0 and self.mantissa == 0; + return self == nan; } -}); -pub const BFloat16 = FloatType(1, 8, 7, struct { - pub fn nan() BFloat16 { - return .{ - .sign = 0, - .exponent = std.math.maxInt(u8), - .mantissa = 1, - }; + const Helpers = FloatHelpers(@This()); + pub const zero = Helpers.zero; + pub const neg = Helpers.neg; + pub const fromF32 = FloatConversionHelpers(@This()).fromF32; + pub const toF32 = FloatConversionHelpers(@This()).toF32; + pub const formatNumber = Helpers.formatNumber; +}; + +test "Float8E5" { + const test_case_e5: TestCase = .{ + .lossless = &[_]f32{ 0, 1.0, -2, 1.0 / 128.0, -128 }, + .lossy = &[_]f32{3.02344107628}, + }; + inline for (.{ Float8E5M2, Float8E5M2FNUZ }) |Float8T| { + try testCustomFloat(Float8T, test_case_e5); } +} + +pub const BFloat16 = packed struct(u16) { + mantissa: u7, + exponent: u8, + sign: u1, + + pub const nan: BFloat16 = .{ .sign = 0, .exponent = std.math.maxInt(u8), .mantissa = 1 }; + pub const max: BFloat16 = .{ .sign = 0, .exponent = 0b11111110, .mantissa = std.math.maxInt(u7) }; + pub const min: BFloat16 = neg(max); + pub const inf: BFloat16 = .{ .sign = 0, .exponent = std.math.maxInt(u8), .mantissa = 0 }; + pub const minus_inf: BFloat16 = .neg(inf); pub fn isNan(self: BFloat16) bool { - return allBitsOne(self.exponent) and self.mantissa != 0; + return self.exponent == nan.exponent and self.mantissa != 0; } - pub fn minusInf() BFloat16 { - return .{ - .sign = 1, - .exponent = std.math.maxInt(u8), - .mantissa = 0, - }; + // Specialized versions of to/from F32. Since BFloat16 has the same exponent range than F32, + // no overflow/underflow can happen, simplifiying conversion logic. + pub fn toF32(self: BFloat16) f32 { + // Pad the BF16 with zeros 0 + return @bitCast([2]u16{ 0, @bitCast(self) }); } - pub fn inf() BFloat16 { - return .{ - .sign = 0, - .exponent = std.math.maxInt(u8), - .mantissa = 0, - }; + pub fn fromF32(float32: f32) BFloat16 { + var int: u32 = @bitCast(float32); + // Round up if needed. + int += 0x8000; + const parts: [2]u16 = @bitCast(int); + return @bitCast(parts[1]); } - pub fn isInf(self: BFloat16) bool { - return allBitsOne(self.exponent) and self.mantissa == 0; - } -}); + const Helpers = FloatHelpers(@This()); + pub const zero = Helpers.zero; + pub const neg = Helpers.neg; + pub const formatNumber = Helpers.formatNumber; +}; test BFloat16 { // From https://en.wikipedia.org/wiki/Bfloat16_floating-point_format#Examples try std.testing.expectEqual(BFloat16.fromF32(0), BFloat16{ .sign = 0, .exponent = 0, .mantissa = 0 }); try std.testing.expectEqual(BFloat16.fromF32(-2), BFloat16{ .sign = 1, .exponent = 127 + 1, .mantissa = 0 }); - try std.testing.expectEqual(BFloat16.fromF32(3.02344107628), BFloat16{ .sign = 0, .exponent = 127 + 1, .mantissa = 65 }); + try std.testing.expectEqual(BFloat16.fromF32(3.02344107628), BFloat16{ .sign = 0, .exponent = 127 + 1, .mantissa = 66 }); try std.testing.expectEqual(BFloat16.fromF32(1.0 / 128.0), BFloat16{ .sign = 0, .exponent = 127 - 7, .mantissa = 0 }); - try std.testing.expectEqual(std.mem.toBytes(BFloat16.inf().neg()), [_]u8{ 0x80, 0xff }); + try std.testing.expectEqual(std.mem.toBytes(BFloat16.inf.neg()), [_]u8{ 0x80, 0xff }); + try std.testing.expectEqual(BFloat16.inf, BFloat16.fromF32(std.math.inf(f32))); - const lossless = [_]f32{ 0, -2, 1.0 / 128.0, -1e64, std.math.inf(f32) }; - for (&lossless) |v| { - try std.testing.expectEqual(v, BFloat16.fromF32(v).toF32()); - } - const lossy = [_]f32{3.02344107628}; - for (&lossy) |x| { - const y = BFloat16.fromF32(x).toF32(); - if (!std.math.approxEqRel(f32, x, y, 1e-2)) { - std.log.err("expected ~{d}, got {d}", .{ x, y }); - return error.TestUnexpectedResult; - } - } + try testCustomFloat(BFloat16, .{ + .lossless = &[_]f32{ 0, -2, 1.0 / 128.0, -1e64, std.math.inf(f32) }, + .lossy = &[_]f32{3.02344107628}, + }); } +pub const Float8E4M3 = packed struct(u8) { + mantissa: u3, + exponent: u4, + sign: u1, + + pub const nan: Float8E4M3 = .{ .sign = 0, .exponent = std.math.maxInt(u4), .mantissa = 1 }; + pub const max: Float8E4M3 = .{ .sign = 0, .exponent = 0b1110, .mantissa = std.math.maxInt(u3) }; + pub const min: Float8E4M3 = neg(max); + pub const inf: Float8E4M3 = .{ .sign = 0, .exponent = std.math.maxInt(u4), .mantissa = 0 }; + pub const minus_inf = neg(inf); + + pub fn isNan(self: Float8E4M3) bool { + return self.exponent == nan.exponent and self.mantissa != 0; + } + + const Helpers = FloatHelpers(@This()); + pub const zero = Helpers.zero; + pub const neg = Helpers.neg; + pub const fromF32 = FloatConversionHelpers(@This()).fromF32; + pub const toF32 = FloatConversionHelpers(@This()).toF32; + pub const formatNumber = Helpers.formatNumber; +}; + +pub const Float8E3M4 = packed struct(u8) { + mantissa: u4, + exponent: u3, + sign: u1, + + pub const nan: Float8E3M4 = .{ .sign = 0, .exponent = std.math.maxInt(u3), .mantissa = 1 }; + pub const max: Float8E3M4 = .{ .sign = 0, .exponent = 0b110, .mantissa = std.math.maxInt(u4) }; + pub const min: Float8E3M4 = neg(max); + pub const inf: Float8E3M4 = .{ .sign = 0, .exponent = std.math.maxInt(u3), .mantissa = 0 }; + pub const minus_inf = neg(inf); + + pub fn isNan(self: Float8E3M4) bool { + return self.exponent == nan.exponent and self.mantissa != 0; + } + + const Helpers = FloatHelpers(@This()); + pub const zero = Helpers.zero; + pub const neg = Helpers.neg; + pub const fromF32 = FloatConversionHelpers(@This()).fromF32; + pub const toF32 = FloatConversionHelpers(@This()).toF32; + pub const formatNumber = Helpers.formatNumber; +}; + +pub const Float8E8M0 = packed struct(u8) { + mantissa: u0 = 0, + exponent: u8, + sign: u0 = 0, + + pub const max: Float8E8M0 = .{ .exponent = std.math.maxInt(u8) }; + pub const min: Float8E8M0 = .{ .exponent = 0 }; + pub const min_scale: f32 = @bitCast(Float32{ .sign = 0, .exponent = 0, .mantissa = 0b1 << 22 }); + + /// Lossy conversion from f32, similar to @floatCast + pub fn fromF32(f: f32) Float8E8M0 { + const vf32: Float32 = @bitCast(f); + return .{ .exponent = @intCast(vf32.exponent) }; + } + + /// Lossless conversion to f32. + pub fn toF32(x: Float8E8M0) f32 { + if (x.exponent == 0) return min_scale; + const vf32: Float32 = .{ + .sign = 0, + .exponent = x.exponent, + .mantissa = 0, + }; + return @bitCast(vf32); + } + + const Helpers = FloatHelpers(@This()); + pub const formatNumber = Helpers.formatNumber; +}; + +test Float8E8M0 { + try std.testing.expectEqual(Float8E8M0{ .exponent = 127 }, Float8E8M0.fromF32(1.0)); + // try std.testing.expectEqual(5.877472e-39, Float8E8M0.toF32(.{ .exponent = 0})); + + try testCustomFloat(Float8E8M0, .{ + .lossless = &[_]f32{ Float8E8M0.min_scale, 1.0, 64.0, 1.0 / 128.0, std.math.pow(f32, 2.0, 127) }, + .lossy = &[_]f32{1.00001}, + }); +} + +pub const Float4E2M1 = packed struct(u4) { + mantissa: u1, + exponent: u2, + sign: u1, + + pub const max: Float4E2M1 = .{ .sign = 0, .exponent = std.math.maxInt(u2), .mantissa = std.math.maxInt(u1) }; + pub const min: Float4E2M1 = neg(max); + const Helpers = FloatHelpers(@This()); + pub const zero = Helpers.zero; + pub const neg = Helpers.neg; + pub const fromF32 = FloatConversionHelpers(@This()).fromF32; + pub const formatNumber = Helpers.formatNumber; + + pub const values = [_]f32{ 0.0, 0.5, 1, 1.5, 2, 3, 4, 6, -0.0, -0.5, -1, -1.5, -2, -3, -4, -6 }; + + pub fn toF32(x: Float4E2M1) f32 { + // faster implementation + return values[@as(u4, @bitCast(x))]; + } + + test toF32 { + var to_f32_res: [16]f32 = undefined; + for (&to_f32_res, 0..) |*r, i| { + const x_f4: Float4E2M1 = @bitCast(@as(u4, @intCast(i))); + r.* = x_f4.toF32(); + } + try std.testing.expectEqualSlices(f32, &Float4E2M1.values, &to_f32_res); + } + + test fromF32 { + var from_f32_res: [16]Float4E2M1 = undefined; + for (&from_f32_res, 0..) |*r, i| { + r.* = .fromF32(Float4E2M1.values[i]); + } + try std.testing.expectEqualSlices(u4, &.{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }, @ptrCast(&from_f32_res)); + } + + pub const Packed = packed struct(@Int(.unsigned, @bitSizeOf(Float4E2M1) * 2)) { + x: Float4E2M1, + y: Float4E2M1, + + pub fn fromF32(x: f32, y: f32) Packed { + return .{ .x = .fromF32(x), .y = .fromF32(y) }; + } + + pub fn toF32(xy: Packed) [2]f32 { + return .{ xy.x.toF32(), xy.y.toF32() }; + } + + pub fn formatNumber(xy: Packed, w: *std.Io.Writer, n: std.fmt.Number) std.Io.Writer.Error!void { + try xy.x.formatNumber(w, n); + try w.writeByte(','); + try xy.y.formatNumber(w, n); + } + }; +}; + pub fn floatCast(T: type, x: anytype) T { - return switch (@TypeOf(x)) { - f64, f32, f16 => @floatCast(x), - else => @floatCast(x.toF32()), + return switch (T) { + f64, f32, f16 => switch (@TypeOf(x)) { + f64, f32, f16 => @floatCast(x), + else => @floatCast(x.toF32()), + }, + else => switch (@TypeOf(x)) { + f64, f32, f16 => .fromF32(x), + else => .fromF32(x.toF32()), + }, }; } + +pub fn isInf(x: anytype) bool { + const Float = @TypeOf(x); + switch (Float) { + f64, f32, f16 => return std.math.isInf(x), + else => {}, + } + if (!@hasDecl(Float, "inf")) return false; + + const FBits = @Int(.unsigned, @bitSizeOf(Float)); + const remove_sign = ~@as(FBits, 0) >> 1; + return @as(FBits, @bitCast(x)) & remove_sign == @as(FBits, @bitCast(Float.inf)); +} + +const TestCase = struct { + lossless: []const f32, + lossy: []const f32, + tolerance: f32 = 1e-2, +}; + +fn testCustomFloat(FloatT: type, test_case: TestCase) !void { + for (test_case.lossless) |x| { + try std.testing.expectEqual(x, FloatT.fromF32(x).toF32()); + } + for (test_case.lossy) |x| { + try expectApproxEqRel(f32, x, FloatT.fromF32(x).toF32(), test_case.tolerance); + } +} + +fn expectApproxEqRel(FloatT: type, x: FloatT, y: FloatT, tolerance: FloatT) !void { + if (!std.math.approxEqRel(f32, x, y, tolerance)) { + std.log.err("expected ~{d}, got {d}", .{ x, y }); + return error.TestUnexpectedResult; + } +} diff --git a/zml/helpers.zig b/zml/helpers.zig deleted file mode 100644 index 1c01bd9b..00000000 --- a/zml/helpers.zig +++ /dev/null @@ -1,172 +0,0 @@ -//! Public helpers to manipulate tensors or shapes. -const std = @import("std"); - -const meta = @import("meta.zig"); -const Shape = @import("shape.zig").Shape; -const Tensor = @import("tensor.zig").Tensor; - -const EnumLiteral = @TypeOf(.enum_literal); -const log = std.log.scoped(.zml_tensor); - -test { - std.testing.refAllDecls(@This()); -} - -const ShapeError = error{ DimMismatch, NotFound }; -const NOT_SET: i64 = 0; -const DIM_MISMATCH: i64 = -1; - -/// Collect the given dimensions inside a struct containing tagged tensors. -pub fn collectDims( - comptime dims: anytype, - v: anytype, - comptime mode: enum { strict, allow_extra_dims, ignore_errors }, -) ShapeError!ShapeStruct(dims) { - const LocalContext = struct { - res: ShapeStruct(dims), - mode: @TypeOf(mode), - }; - var context = LocalContext{ - .res = std.mem.zeroes(ShapeStruct(dims)), - .mode = mode, - }; - - meta.visit((struct { - fn cb(ctx: *LocalContext, shape: *const Shape) void { - inline for (dims) |a| { - if (shape.hasTag(a)) |axis| { - const dim = shape.dim(axis); - - const expected_dim = &@field(ctx.res, @tagName(a)); - if (expected_dim.* == NOT_SET) { - expected_dim.* = dim; - } else if (expected_dim.* == DIM_MISMATCH) { - // this axis has already been reported as invalid. - } else if (dim != expected_dim.*) { - if (mode != .ignore_errors) { - log.warn("Dim mismatch ! Axis {0s}={1d} but received a new tensor where {0s}={2d}", .{ @tagName(a), expected_dim.*, dim }); - } - expected_dim.* = DIM_MISMATCH; - } - } - } - } - }).cb, &context, v); - - if (context.mode != .ignore_errors) { - inline for (shapeToDims(context.res), dims) |dim, dim_tag| { - if (dim == NOT_SET) { - log.warn("Axis not found: {s}", .{@tagName(dim_tag)}); - return error.NotFound; - } - if (dim == DIM_MISMATCH) return error.DimMismatch; - } - } - return context.res; -} - -fn shapeToDims(shape: anytype) [@divExact(@sizeOf(@TypeOf(shape)), @sizeOf(i64))]i64 { - return @bitCast(shape); -} - -test collectDims { - const zml = @import("zml.zig"); - - const Model = struct { - x: Shape, - y: Shape, - bias: Shape, - }; - - { - var model: Model = .{ - .x = Shape.init(.{ 2, 5 }, .f32).withTags(.{ .b, .d }), - .y = Shape.init(.{ 2, 5 }, .f32).withTags(.{ .b, .d }), - .bias = Shape.init(.{5}, .f32).withTags(.{.d}), - }; - try zml.testing.expectEqual(collectDims(.{ .b, .d }, &model, .strict), .{ .b = 2, .d = 5 }); - } - { - var model: Model = .{ - .x = Shape.init(.{ 2, 5 }, .f32).withTags(.{ .b, .d }), - .y = Shape.init(.{ 3, 5 }, .f32).withTags(.{ .b, .d }), - .bias = Shape.init(.{5}, .f32).withTags(.{.d}), - }; - try std.testing.expectEqual( - collectDims(.{ .b, .d }, &model, .strict), - error.DimMismatch, - ); - try zml.testing.expectEqual(collectDims(.{ .b, .d }, &model, .ignore_errors), .{ .b = -1, .d = 5 }); - } - { - var model: Model = .{ - .x = Shape.init(.{ 2, 5 }, .f32).withTags(.{ .b, .d }), - .y = Shape.init(.{ 2, 5 }, .f32).withTags(.{ .b, .d }), - .bias = Shape.init(.{5}, .f32).withTags(.{.d}), - }; - try std.testing.expectEqual(collectDims(.{ .b, .d, .c }, &model, .strict), error.NotFound); - try zml.testing.expectEqual(collectDims(.{ .b, .d, .c }, &model, .ignore_errors), .{ .b = 2, .d = 5, .c = 0 }); - } - { - var model: Model = .{ - .x = Shape.init(.{ 2, 5 }, .f32).withTags(.{ .b, .d }), - .y = Shape.init(.{ 2, 5 }, .f32).withTags(.{ .b, .d }), - .bias = Shape.init(.{7}, .f32).withTags(.{.d}), - }; - try std.testing.expectEqual(collectDims(.{ .b, .d }, &model, .strict), error.DimMismatch); - try zml.testing.expectEqual(collectDims(.{ .b, .d }, &model, .ignore_errors), .{ .b = 2, .d = -1 }); - } -} - -fn ShapeStruct(comptime dims: anytype) type { - const rank = dims.len; - @setEvalBranchQuota(rank + 5); - var struct_fields: [rank]std.builtin.Type.StructField = undefined; - const default: i64 = NOT_SET; - for (&struct_fields, dims) |*struct_field, axis| { - struct_field.* = .{ - .name = @tagName(axis), - .type = i64, - .default_value = &default, - .is_comptime = false, - .alignment = @alignOf(i64), - }; - } - return @Type(.{ .Struct = .{ - .layout = .@"extern", - .fields = &struct_fields, - .decls = &.{}, - .is_tuple = false, - } }); -} - -/// Return a new struct with all tensors replaced by the output of the given function. -pub fn mapTensors(func: anytype, v: anytype, args: anytype) @TypeOf(v) { - const T = @TypeOf(v); - const type_info = @typeInfo(T); - if (T == Tensor) return @call(.auto, func, .{v} ++ args); - - return switch (type_info) { - .Pointer => @compileError("mapTensors only accept by value arguments. Received: " ++ @typeName(T)), - .Struct => |struct_info| { - var copy: T = v; - inline for (struct_info.fields) |feeld| { - if (feeld.is_comptime) continue; - if (@typeInfo(feeld.type) == .Pointer) { - @compileError("mapTensors doesn't follow pointers and don't accept struct containing them. Received: " ++ @typeName(T)); - } - @field(copy, feeld.name) = mapTensors(func, @field(v, feeld.name), args); - } - return copy; - }, - .Array => { - var res: T = undefined; - for (v, &res) |item, *r| { - r.* = mapTensors(func, item, args); - } - return res; - }, - .Union, .Optional => @compileError("mapTensors doesn't yet support " ++ @typeName(T)), - else => v, - }; -} diff --git a/zml/hostbuffer.zig b/zml/hostbuffer.zig deleted file mode 100644 index 863a7c67..00000000 --- a/zml/hostbuffer.zig +++ /dev/null @@ -1,258 +0,0 @@ -const std = @import("std"); - -const meta = @import("meta.zig"); -const Buffer = @import("buffer.zig").Buffer; -const Data = @import("dtype.zig").Data; -const DataType = @import("dtype.zig").DataType; -const Platform = @import("platform.zig").Platform; -const Shape = @import("shape.zig").Shape; - -test { - std.testing.refAllDecls(HostBuffer); -} - -/// Represents a tensor with associated data allocated by user code. -/// If the memory is `.managed` it needs to be freed with `x.deinit(allocator)` -/// If the memory is `.unmanaged` it doesn't need to be freed (eg memory mapped, or tracked elsewhere). -pub const HostBuffer = struct { - _shape: Shape, - _strides: ?[Shape.MAX_RANK]i64 = null, - data: []const u8, - _memory: union(enum) { - managed: u5, - unmanaged, - } = .unmanaged, - - /// Allocates a HostBuffer with the given shape. - /// The memory is left undefined. - /// The caller owns the memory, and need to call `deinit()`. - pub fn empty(allocator: std.mem.Allocator, sh: Shape) !HostBuffer { - return .{ - ._shape = sh, - .data = try allocator.alignedAlloc(u8, std.atomic.cache_line, sh.byteSize()), - ._memory = .{ .managed = std.math.log2_int(u16, std.atomic.cache_line) }, - }; - } - - /// Wraps an exisiting slice of bytes into a HostBuffer. - /// The returned HostBuffer doesn't take ownership of the slice - /// that will still need to be deallocated. - pub fn fromBytes(shape_: Shape, data_: []const u8) HostBuffer { - std.debug.assert(shape_.byteSize() == data_.len); - return .{ - ._shape = shape_, - .data = data_, - ._memory = .unmanaged, - }; - } - - /// Frees the underlying memory if we owned it, ie if we've been created with `HostBuffer.empty`. - pub fn deinit(self: *const HostBuffer, allocator: std.mem.Allocator) void { - // This means we don't own the data. - if (self._memory == .unmanaged) return; - const log2_align = self._memory.managed; - allocator.rawFree(@constCast(self.data), log2_align, @returnAddress()); - } - - /// Wraps an exisiting slice into a HostBuffer. - /// The element type is inferred from the slice type. - /// The returned HostBuffer doesn't take ownership of the slice - /// that will still need to be deallocated. - pub fn fromSlice(sh: anytype, s: anytype) HostBuffer { - const shape_ = Shape.init(sh, DataType.fromSliceElementType(s)); - std.debug.assert(shape_.count() == s.len); - return .{ - ._shape = shape_, - .data = @alignCast(std.mem.sliceAsBytes(s)), - ._memory = .unmanaged, - }; - } - - /// Wraps an exisiting slice into a HostBuffer. - /// The element type is inferred from the slice type. - /// The values in the slice doesn't need to be contiguous, - /// strides can be specified. - /// The returned HostBuffer doesn't take ownership of the slice. - pub fn fromStridedSlice(sh: Shape, s: anytype, strides_: []const i64) HostBuffer { - // std.debug.assert(sh.count() == s.len); - var tmp: [Shape.MAX_RANK]i64 = undefined; - @memcpy(tmp[0..strides_.len], strides_); - return .{ - ._shape = sh, - .data = @alignCast(std.mem.sliceAsBytes(s)), - ._strides = tmp, - ._memory = .unmanaged, - }; - } - - /// Creates a tensor from a **pointer** to a "multi dimension" array. - /// Note this doesn't copy, the pointee array need to survive the `HostBuffer` object. - pub fn fromArray(arr_ptr: anytype) HostBuffer { - const T = @TypeOf(arr_ptr.*); - const sh = parseArrayInfo(T); - return .{ - ._shape = sh, - .data = @alignCast(std.mem.sliceAsBytes(arr_ptr)), - // Array are typically stack allocated and don't need to be freed. - ._memory = .unmanaged, - }; - } - - pub const ArangeArgs = struct { - start: i64 = 0, - end: i64, - step: i64 = 1, - }; - - /// Allocates a HostBuffer with the given shape. - /// The memory is initialized with increasing numbers. - /// The caller owns the memory, and need to call `deinit()`. - pub fn arange(allocator: std.mem.Allocator, args: ArangeArgs, dt: DataType) !HostBuffer { - meta.assert(args.start < args.end, "arange expects 'args.start' to be less than 'args.end', got {} and {}", .{ args.start, args.end }); - meta.assert(args.step > 0, "arange expects 'args.step' to be positive, got {}", .{args.step}); - - const n_steps = std.math.divCeil(i64, args.end - args.start, args.step) catch unreachable; - const b = dt.sizeOf(); - const res = try empty(allocator, Shape.init(.{n_steps}, dt)); - meta.assert(dt.class() == .integer, "arange expects type to be integer, got {} instead.", .{dt}); - var data_ = @constCast(res.data); - switch (dt) { - inline else => { - var j: i64 = args.start; - for (0..@intCast(n_steps)) |i| { - var v = Data.init(dt, j); - @memcpy(data_[i * b .. (i + 1) * b], v.constSlice()); - j +%= args.step; - } - }, - } - return res; - } - - test arange { - { - var x = try arange(std.testing.allocator, .{ .end = 8 }, .i32); - defer x.deinit(std.testing.allocator); - try std.testing.expectEqualSlices(i32, &.{ 0, 1, 2, 3, 4, 5, 6, 7 }, x.items(i32)); - } - { - var x = try arange(std.testing.allocator, .{ .start = -3, .end = 12, .step = 2 }, .i32); - defer x.deinit(std.testing.allocator); - try std.testing.expectEqualSlices(i32, &.{ -3, -1, 1, 3, 5, 7, 9, 11 }, x.items(i32)); - } - } - - /// Copies this HostBuffer to the given accelerator. - pub fn toDevice(self: HostBuffer, platform_: Platform) !Buffer { - return try Buffer.from(platform_, self); - } - - /// Interpret the underlying data as a contiguous slice. - /// WARNING: It's only valid if the buffer is contiguous. - /// Strided buffers can't use this method. - pub fn items(self: HostBuffer, comptime T: type) []const T { - if (DataType.fromZigType(T) != self.dtype()) { - std.debug.panic("Can't reinterpret {} as {s}", .{ self, @typeName(T) }); - } - if (!self.isContiguous()) { - std.debug.panic("{} isn't contiguous", .{self}); - } - const ptr: [*]const T = @alignCast(@ptrCast(self.data.ptr)); - return ptr[0..self._shape.count()]; - } - - pub fn shape(self: HostBuffer) Shape { - return self._shape; - } - - pub fn dtype(self: HostBuffer) DataType { - return self._shape.dtype(); - } - - pub fn strides(self: *const HostBuffer) ?[]const i64 { - // Pass strides per pointer otherwise we return a pointer to this stack frame. - return if (self._strides) |*strd| strd[0..self.rank()] else null; - } - - pub fn data(self: HostBuffer) []const u8 { - return self.data; - } - - pub inline fn rank(self: HostBuffer) u4 { - return self._shape.rank(); - } - - pub inline fn count(self: HostBuffer) usize { - return self._shape.count(); - } - - pub fn dim(self: HostBuffer, axis_: anytype) i64 { - return self._shape.dim(axis_); - } - - pub fn axis(self: HostBuffer, axis_: anytype) u3 { - return self._shape.axis(axis_); - } - - pub fn isContiguous(self: HostBuffer) bool { - const strd = self._strides orelse return true; - const cont_strides = self._shape.computeStrides(); - return std.mem.eql(i64, strd[0..self.rank()], cont_strides.constSlice()); - } - - pub fn reshape(self: HostBuffer, shape_: anytype) HostBuffer { - meta.assert(self.isContiguous(), "reshape expects a contiguous tensor, got: {}", .{self}); - var res = self; - res._shape = self._shape.reshape(shape_); - return res; - } - - pub const Slice = struct { - start: i64 = 0, - end: ?i64 = null, - }; - - /// Slices the input Tensor over the given axis_ using the given parameters. - pub fn slice1d(self: HostBuffer, axis_: anytype, s: Slice) HostBuffer { - const ax = self._shape.axis(axis_); - const d = self.dim(ax); - const start: i64 = if (s.start < 0) s.start + d else s.start; - var end = s.end orelse d; - if (end < 0) end += d; - meta.assert(start >= 0 and start < d, "slice1d({}, {}) expects the slice start to be between 0 and {} got: {}", .{ self, ax, d, start }); - meta.assert(end >= 1 and end <= d, "slice1d({}, {}) expects the slice end to be between 1 and {} got: {}", .{ self, ax, d, end }); - meta.assert(start < end, "slice1d({}, {}) expects the slice start ({}) to be smaller than the end ({})", .{ self, ax, start, end }); - - // If strides weren't set it means original buffer is contiguous. - // But it won't be anymore after slicing. The strides don't change though. - const _strides = self._strides orelse self._shape.computeStrides().buffer; - const offset: usize = @intCast(start * _strides[ax]); - return .{ - ._shape = self.shape().set(ax, end - start), - .data = self.data[offset..], - ._strides = _strides, - ._memory = .unmanaged, - }; - } - - pub fn format( - self: HostBuffer, - comptime fmt: []const u8, - options: std.fmt.FormatOptions, - writer: anytype, - ) !void { - _ = fmt; - _ = options; - try writer.print("HostBuffer(.{_})", .{self._shape}); - } -}; - -fn parseArrayInfo(T: type) Shape { - return switch (@typeInfo(T)) { - .Array => |arr| { - const s = parseArrayInfo(arr.child); - return s.insert(0, .{arr.len}); - }, - else => .{ ._dtype = DataType.fromZigType(T) }, - }; -} diff --git a/zml/io.zig b/zml/io.zig new file mode 100644 index 00000000..ab8d8e6e --- /dev/null +++ b/zml/io.zig @@ -0,0 +1,1743 @@ +const std = @import("std"); + +const pjrt = @import("pjrt"); +const stdx = @import("stdx"); +pub const VFS = @import("vfs"); + +const Buffer = @import("buffer.zig").Buffer; +const Bufferized = @import("zml.zig").Bufferized; +const CreateOptions = @import("platform.zig").CreateOptions; +const Exe = @import("exe.zig").Exe; +const mem = @import("mem.zig"); +const Memory = @import("platform.zig").Memory; +const meta = @import("meta.zig"); +const pjrtx = @import("pjrtx.zig"); +const Platform = @import("platform.zig").Platform; +const tracer = @import("profiling/tracer.zig"); +const safetensors = @import("safetensors.zig"); +const Shape = @import("shape.zig").Shape; +const Sharding = @import("Sharding.zig"); +const Placement = Sharding.Placement; +const Slice = @import("slice.zig").Slice; +const Tensor = @import("tensor.zig").Tensor; + +const log = std.log.scoped(.@"zml/io"); + +pub const TensorStore = struct { + pub const Binding = struct { + tensors: []*safetensors.Tensor, + transformed: bool, + memory: Memory.Kind = .default, + }; + + registry: *safetensors.TensorRegistry, + meshes: []const *const Sharding.Mesh, + id_to_sources: std.AutoHashMapUnmanaged(Tensor.Id, Binding), + allocator: std.mem.Allocator, + arena: std.heap.ArenaAllocator, + + pub fn fromRegistry(allocator: std.mem.Allocator, registry: *safetensors.TensorRegistry, meshes: []const *const Sharding.Mesh) TensorStore { + var arena: std.heap.ArenaAllocator = .init(allocator); + return .{ + .registry = registry, + .meshes = arena.allocator().dupe(*const Sharding.Mesh, meshes) catch @panic("OOM"), + .id_to_sources = .empty, + .allocator = allocator, + .arena = arena, + }; + } + + pub fn deinit(self: *TensorStore) void { + self.id_to_sources.deinit(self.allocator); + self.arena.deinit(); + } + + fn putSourcesNoClobber(self: *TensorStore, id: Tensor.Id, sources: Binding) std.mem.Allocator.Error!void { + const gop = try self.id_to_sources.getOrPut(self.allocator, id); + if (gop.found_existing) { + stdx.debug.panic("Id {} already has associated sources", .{id}); + } + errdefer self.id_to_sources.removeByPtr(gop.key_ptr); + + gop.value_ptr.* = sources; + } + + fn getPtrFromKey(self: *const TensorStore, key: []const u8) ?*safetensors.Tensor { + const tensor_desc_ptr = self.registry.tensors.getPtr(key) orelse return null; + return tensor_desc_ptr; + } + + fn dupeSource(self: *TensorStore, key: []const u8) ?*safetensors.Tensor { + const entry = self.getPtrFromKey(key) orelse return null; + + const copy = self.arena.allocator().create(safetensors.Tensor) catch @panic("OOM"); + copy.* = entry.*; + + return copy; + } + + fn getPtrFromId(self: *const TensorStore, id: usize) ?*safetensors.Tensor { + const sources = self.id_to_sources.get(id) orelse return null; + stdx.debug.assert(sources.tensors.len == 1, "Expect tensor with id {} to have only one source, got {}", .{ id, sources.tensors.len }); + return sources.tensors[0]; + } + + pub fn getReader(self: *const TensorStore, key: []const u8, io: std.Io, buffer: []u8) !safetensors.TensorReader { + return self.registry.reader(io, key, buffer); + } + + pub fn getReaderById(self: *const TensorStore, id: usize, io: std.Io, buffer: []u8) !safetensors.TensorReader { + const sources = self.id_to_sources.get(id) orelse return error.NotFound; + stdx.debug.assert(sources.tensors.len == 1, "Expect tensor with id {} to have only one source, got {}", .{ id, sources.tensors.len }); + + return sources.tensors[0].reader(io, buffer, .{}); + } + + pub fn getSourcesById(self: *const TensorStore, id: Tensor.Id) ?Binding { + return self.id_to_sources.get(id); + } + + pub fn getShape(self: *const TensorStore, key: []const u8) ?Shape { + const entry_ptr = self.getPtrFromKey(key) orelse return null; + return entry_ptr.shape; + } + + pub fn view(self: *TensorStore) View { + return .{ .store = self }; + } + + pub fn mesh(store: *const TensorStore, name: @EnumLiteral()) *const Sharding.Mesh { + const name_slice = @tagName(name); + for (store.meshes) |m| { + if (std.mem.eql(u8, name_slice, m.name)) { + return m; + } + } + std.debug.panic( + \\Found no meshes named {s} in TensorStore. + \\Try passing more meshes to `zml.TensorStore.fromRegistry`. + \\Known meshes: {f} + , .{ name_slice, stdx.fmt.slice(store.meshes) }); + } + + pub const View = struct { + store: *TensorStore, + + prefix_buffer: [256]u8 = undefined, + prefix_length: usize = 0, + + pub fn root(self: *const View) View { + return .{ + .store = self.store, + }; + } + + pub fn parent(self: *const View) View { + const slice = self.prefix() orelse unreachable; + const index = std.mem.lastIndexOfScalar(u8, slice[0 .. slice.len - 1], '.') orelse return self.root(); + var buffer: [256]u8 = undefined; + @memcpy(buffer[0 .. index + 1], slice[0 .. index + 1]); + return .{ + .store = self.store, + .prefix_buffer = buffer, + .prefix_length = index + 1, + }; + } + + pub fn withPrefix(self: *const View, prefix_: []const u8) View { + var buffer: [256]u8 = undefined; + const new_prefix = makeKey(&buffer, "{s}{s}.", .{ self.prefix() orelse "", prefix_ }); + + return .{ + .store = self.store, + .prefix_buffer = buffer, + .prefix_length = new_prefix.len, + }; + } + + pub fn withLayer(self: *const View, index: usize) View { + var buffer: [256]u8 = undefined; + const new_prefix = makeKey(&buffer, "{s}{d}.", .{ self.prefix() orelse "", index }); + + return .{ + .store = self.store, + .prefix_buffer = buffer, + .prefix_length = new_prefix.len, + }; + } + + pub fn prefix(self: *const View) ?[]const u8 { + return if (self.prefix_length == 0) null else self.prefix_buffer[0..self.prefix_length]; + } + + pub fn hasKey(self: *const View, subkey: []const u8) bool { + var buffer: [256]u8 = undefined; + const key = makeKey(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }); + return for (self.store.registry.tensors.keys()) |k| { + if (std.mem.startsWith(u8, k, key)) break true; + } else false; + } + + /// Creates a zml.Tensor from a specific entry in the store. + pub fn createTensor(self: View, subkey: []const u8, tags: anytype, mesh_: @EnumLiteral(), partitioning: anytype) Tensor { + return self.maybeCreateTensor(subkey, tags, mesh_, partitioning) orelse + stdx.debug.panic("Checkpoint has no tensor named {s}{s}", .{ self.prefix() orelse "", subkey }); + } + + pub fn maybeCreateTensor(self: View, subkey: []const u8, tags: anytype, mesh_: @EnumLiteral(), partitioning: anytype) ?Tensor { + const parsed_tags: ?[]const Shape.Tag = comptime parseTags(tags); + if (mesh_ == .replicated) { + return self.maybeCreateTensorInternal(subkey, parsed_tags, .replicated); + } + + const resolved_mesh = self.store.mesh(mesh_); + if (@TypeOf(partitioning) == Sharding.Partitioning) { + return self.maybeCreateTensorInternal(subkey, parsed_tags, .{ .mesh = resolved_mesh, .partition = partitioning }); + } + // Unspecified tensor axes are treated as replicated + const partition: Sharding.Partitioning = if (parsed_tags) |p_tags| + .parse(p_tags, resolved_mesh, .replicated, partitioning) + else + partitioning; + return self.maybeCreateTensorInternal(subkey, parsed_tags, .{ .mesh = resolved_mesh, .partition = partition }); + } + + pub fn createHostPinnedTensor(self: View, subkey: []const u8, tags: anytype, mesh_: @EnumLiteral(), partitioning: anytype) Tensor { + return self.maybeCreateHostPinnedTensor(subkey, tags, mesh_, partitioning) orelse + stdx.debug.panic("Checkpoint has no tensor named {s}{s}", .{ self.prefix() orelse "", subkey }); + } + + pub fn maybeCreateHostPinnedTensor(self: View, subkey: []const u8, tags: anytype, mesh_: @EnumLiteral(), comptime partitioning: anytype) ?Tensor { + const tensor = self.maybeCreateTensor(subkey, comptime parseTags(tags), mesh_, partitioning); + if (tensor) |t| { + const storage = self.store.id_to_sources.getPtr(t.id); + storage.?.memory = .host_pinned; + } + return tensor; + } + + pub fn createReplicatedTensor(self: View, subkey: []const u8, tags: anytype) Tensor { + return self.maybeCreateTensorInternal(subkey, comptime parseTags(tags), .replicated) orelse + stdx.debug.panic("Checkpoint has no tensor named {s}{s}", .{ self.prefix() orelse "", subkey }); + } + + fn maybeCreateTensorInternal(self: View, subkey: []const u8, tags: ?[]const Shape.Tag, sharding: Sharding) ?Tensor { + var buffer: [256]u8 = undefined; + const key = makeKey(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }); + const source = self.store.dupeSource(key) orelse return null; + + const sources = self.store.arena.allocator().alloc(*safetensors.Tensor, 1) catch |e| std.debug.panic("Not handling {} errors", .{e}); + errdefer self.store.arena.allocator().free(sources); + sources[0] = source; + + var shape = source.shape; + if (tags) |user_tags| { + stdx.debug.assert(user_tags.len == shape.rank(), "tensor {s} from store has shape {f}, but `createTensor` got only {d} tags: {f}", .{ key, source.shape, user_tags.len, stdx.fmt.stringsZ(user_tags) }); + @memcpy(shape._tags.slice(), user_tags); + } + shape._sharding = sharding; + if (sharding.mesh) |mesh_| shape = shape.withPartitioning(mesh_, sharding.partition); + + const tensor: Tensor = .fromShape(shape); + self.store.putSourcesNoClobber(tensor.id, .{ .tensors = sources, .transformed = false, .memory = .default }) catch |e| std.debug.panic("Not handling {} errors", .{e}); + + return tensor; + } + + /// Accepts "null" as tags + fn parseTags(tags: anytype) ?[]const Shape.Tag { + return if (@TypeOf(tags) == @TypeOf(null)) null else Shape.parseTags(tags); + } + + pub fn maybeCreateBinding(self: View, sources: []const []const u8, shape: Shape) ?Tensor { + const arena = self.store.arena.allocator(); + + var tensor_list = std.ArrayList(*safetensors.Tensor).initCapacity(arena, sources.len) catch |e| std.debug.panic("Not handling {} errors", .{e}); + defer tensor_list.deinit(arena); + + var buffer: [256]u8 = undefined; + for (sources) |subkey| { + const key = makeKey(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }); + const tensor = self.store.dupeSource(key) orelse return null; + tensor_list.appendAssumeCapacity(tensor); + } + + const tensors = tensor_list.toOwnedSlice(arena) catch unreachable; + errdefer arena.free(tensors); + + const tensor: Tensor = .fromShape(shape); + self.store.putSourcesNoClobber(tensor.id, .{ .tensors = tensors, .transformed = true }) catch |e| std.debug.panic("Not handling {} errors", .{e}); + + return tensor; + } + + pub fn getShape(self: View, subkey: []const u8) ?Shape { + var buffer: [256]u8 = undefined; + const key = makeKey(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }); + return self.store.getShape(key); + } + + pub fn getShapeOpts(self: View, subkey: []const u8, opts: struct { no_prefix: bool = false }) ?Shape { + var buffer: [256]u8 = undefined; + const key = if (opts.no_prefix) + subkey + else b: { + break :b makeKey(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }); + }; + return self.store.getShape(key); + } + + pub fn getReader(self: View, subkey: []const u8, io: std.Io, buffer: []u8) !safetensors.TensorReader { + var key_buffer: [256]u8 = undefined; + const key = makeKey(&key_buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }); + return self.store.getReader(key, io, buffer); + } + + pub fn count(self: View) usize { + var count_: usize = 0; + const prefix_ = self.prefix() orelse ""; + var it = self.store.registry.tensors.iterator(); + while (it.next()) |item| { + const key = item.key_ptr.*; + if (std.mem.startsWith(u8, key, prefix_)) { + count_ += 1; + } + } + return count_; + } + + fn makeKey(buffer: []u8, comptime fmt: []const u8, args: anytype) []const u8 { + const key = std.fmt.bufPrint(buffer, fmt, args) catch + std.debug.panic("Expected key to be less than {} characters", .{buffer.len}); + return key; + } + }; +}; + +pub const Loader = struct { + allocator: std.mem.Allocator, + platform: *const Platform, + dma_allocators: []const mem.DmaAllocator, + dma_chunk_size: usize, + pinned_buffer_pools: []mem.DynamicBufferPool, + group: stdx.Io.LimitedGroup, + bytes_loaded: std.atomic.Value(usize) = .init(0), + delivered: std.AutoHashMapUnmanaged(Tensor.Id, void) = .empty, + + pub const Opts = struct { + pub const default: Opts = .{ + .parallelism = 1, + .dma_chunks = 2, + .dma_chunk_size = 4096, + }; + parallelism: usize, + dma_chunks: usize, + dma_chunk_size: usize, + }; + + pub fn init(allocator: std.mem.Allocator, platform: *const Platform, opts: Opts) error{OutOfMemory}!Loader { + const pool_count = platform.devices.len; + const dma_allocators = try allocator.alloc(mem.DmaAllocator, pool_count); + errdefer allocator.free(dma_allocators); + for (platform.devices, 0..) |*device, i| { + dma_allocators[i] = .init(allocator, device); + } + + const buffer_pools = try allocator.alloc(mem.DynamicBufferPool, pool_count); + errdefer allocator.free(buffer_pools); + for (buffer_pools) |*pool_| { + pool_.* = .init(opts.dma_chunks, opts.dma_chunk_size); + } + errdefer for (buffer_pools, 0..) |*pool_, i| { + pool_.deinit(dma_allocators[i].allocator()); + }; + + return .{ + .allocator = allocator, + .platform = platform, + .dma_allocators = dma_allocators, + .dma_chunk_size = opts.dma_chunk_size, + .pinned_buffer_pools = buffer_pools, + .group = .init(opts.parallelism), + }; + } + + pub fn deinit(self: *Loader) void { + self.delivered.deinit(self.allocator); + for (self.pinned_buffer_pools, 0..) |*pool, i| pool.deinit(self.dma_allocators[i].allocator()); + self.allocator.free(self.pinned_buffer_pools); + self.allocator.free(self.dma_allocators); + } + + pub fn await(self: *Loader, io: std.Io) std.Io.Cancelable!void { + return self.group.await(io); + } + + pub const LoadOpts = struct { + progress: ?*std.Progress.Node = null, + }; + + pub const LoadError = error{TransformedTensorNotDelivered}; + + pub fn load(self: *Loader, io: std.Io, comptime T: type, model: *const T, buffers: *Bufferized(T), store: *const TensorStore, opts: LoadOpts) LoadError!void { + const tensor_count = meta.count(Tensor, model); + + var arena: std.heap.ArenaAllocator = .init(self.allocator); + defer arena.deinit(); + + const flattened_buffers = arena.allocator().alloc(*Buffer, tensor_count) catch @panic("Errors can't be handled in `loadInner`"); + meta.forEachVisit(buffers, *Buffer, struct { + fn call(i: usize, buffer: *Buffer, flattened_buffers_: []*Buffer) void { + flattened_buffers_[i] = buffer; + } + }.call, .{flattened_buffers}); + + const flattened_tensors = arena.allocator().alloc(*const Tensor, tensor_count) catch @panic("Errors can't be handled in `loadInner`"); + meta.forEachVisit(model, *const Tensor, struct { + fn call(i: usize, tensor: *const Tensor, flattened_tensors_: []*const Tensor) void { + flattened_tensors_[i] = tensor; + } + }.call, .{flattened_tensors}); + + // Refuse before scheduling anything, so a failure leaves no task in flight. + var missing: usize = 0; + for (flattened_tensors) |tensor| { + const sources = store.getSourcesById(tensor.id) orelse continue; + if (!sources.transformed or self.delivered.contains(tensor.id)) { + continue; + } + + missing += 1; + logNotDelivered(arena.allocator(), tensor, sources.tensors); + } + + if (missing != 0) { + return error.TransformedTensorNotDelivered; + } + + for (flattened_tensors, flattened_buffers) |tensor, buffer| { + if (store.getSourcesById(tensor.id)) |sources| { + if (sources.transformed) continue; + } + + self.group.async(io, defaultCallback, .{ self, io, tensor, buffer, store, opts }); + } + } + + fn logNotDelivered(arena: std.mem.Allocator, tensor: *const Tensor, sources: []const *safetensors.Tensor) void { + const max_names = 8; + var names: std.Io.Writer.Allocating = .init(arena); + for (sources[0..@min(sources.len, max_names)], 0..) |source, i| { + names.writer.print("{s}{s}", .{ if (i != 0) ", " else "", source.name }) catch break; + } + + if (sources.len > max_names) { + names.writer.print(" and {} more", .{sources.len - max_names}) catch {}; + } + + log.err("Transformed tensor {} {f} was not delivered by loadExecute before load; sources: {s}", .{ tensor.id, tensor.shape(), names.written() }); + } + + pub fn loadBuffer(loader: *Loader, io: std.Io, buffer: *Buffer, tensor: *const Tensor, store: *const TensorStore, opts: LoadOpts) void { + loader.group.async(io, defaultCallback, .{ loader, io, tensor, buffer, store, opts }); + } + + fn defaultCallback(self: *Loader, io: std.Io, tensor: *const Tensor, buffer: *Buffer, store: *const TensorStore, opts: LoadOpts) void { + const sources = store.getSourcesById(tensor.id) orelse { + std.log.debug("Failed to get sources for tensor with id: {}", .{tensor.id}); + return; + }; + stdx.debug.assert(!sources.transformed and sources.tensors.len == 1, "Tensor {} is transformed or has {} sources; `load` only streams single-source tensors", .{ tensor.id, sources.tensors.len }); + + self.loadSingleInner(io, sources.tensors[0], tensor.shape(), buffer, sources.memory, opts) catch |e| { + log.err("Errors are not handled in `defaultCallback`, got {}", .{e}); + unreachable; + }; + } + + fn loadSingle(self: *Loader, io: std.Io, source: *safetensors.Tensor, shape: Shape, buffer: *Buffer, loaded: *bool, opts: LoadOpts) void { + self.loadSingleInner(io, source, shape, buffer, .device, opts) catch |e| { + log.err("Failed to load tensor {s}: {}", .{ source.name, e }); + loaded.* = false; + return; + }; + loaded.* = true; + } + + fn loadSingleInner( + self: *Loader, + io: std.Io, + source: *safetensors.Tensor, + shape: Shape, + buffer: *Buffer, + memory: Memory.Kind, + opts: LoadOpts, + ) !void { + var reader = try source.reader(io, &.{}, .{}); + defer reader.deinit(); + + var writer = try MemoryWriter.init( + self.allocator, + io, + self.platform, + self.pinned_buffer_pools, + self.dma_allocators, + self.dma_chunk_size, + shape, + buffer, + memory, + ); + defer writer.deinit(self.allocator); + + const scale = 1024; + + if (opts.progress) |progress| { + var node = progress.start(reader.tensor.name, reader.tensor.shape.byteSize() / scale); + defer node.end(); + writer.setProgress(&node); + defer writer.setProgress(null); + var progress_writer: ProgressWriter = .init(writer.interface(), &node, .{ .scale = scale }); + const total = try reader.interface.streamRemaining(&progress_writer.interface); + try progress_writer.interface.flush(); + _ = self.bytes_loaded.fetchAdd(total, .monotonic); + } else { + const total = try reader.interface.streamRemaining(writer.interface()); + try writer.interface().flush(); + _ = self.bytes_loaded.fetchAdd(total, .monotonic); + } + } + + pub fn loadExecute( + self: *Loader, + arena: std.mem.Allocator, + io: std.Io, + tensor: Tensor, + buffer: *Buffer, + store: *const TensorStore, + exe: *const Exe, + opts: LoadOpts, + ) error{ NotFound, OutOfMemory, LoadFailed, Canceled }!void { + const sources = (store.getSourcesById(tensor.id) orelse return error.NotFound).tensors; + const buffers = try arena.alloc(Buffer, sources.len); + const loaded = try arena.alloc(bool, sources.len); + @memset(loaded, false); + defer for (buffers, loaded) |*b, l| if (l) b.deinit(); + + var node = if (opts.progress) |progress| p: { + var writer = std.Io.Writer.Allocating.init(arena); + print: { + writer.writer.writeAll("Running executable on ") catch break :print; + for (sources, 0..) |source, i| { + writer.writer.print("{s}{s}", .{ if (i != 0) ", " else "", source.name }) catch break :print; + } + } + break :p progress.start(writer.written(), 1); + } else null; + defer if (node) |*n| n.end(); + + for (sources, 0..) |source, i| { + self.group.async(io, loadSingle, .{ self, io, source, source.shape, &buffers[i], &loaded[i], .{} }); + } + self.group.await(io) catch |err| switch (err) { + error.Canceled => return error.Canceled, + }; + + if (std.mem.findScalar(bool, loaded, false)) |_| { + return error.LoadFailed; + } + + var args = try exe.args(arena); + var results = try exe.results(arena); + + args.set(.{buffers}); + exe.call(io, args, &results, .{ .wait = true }); + + buffer.* = results.get(Buffer); + try self.delivered.put(self.allocator, tensor.id, {}); + } +}; + +test "Loader.loadExecute cleans up donated and non-donated inputs" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = @import("testing.zig").env(); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + try tmp.dir.writeFile(io, .{ .sub_path = "inputs", .data = std.mem.asBytes(&[4]i32{ 1, 2, 10, 20 }) }); + const path = try tmp.dir.realPathFileAlloc(io, "inputs", allocator); + defer allocator.free(path); + + const shape: Shape = .init(.{2}, .i32); + var registry: safetensors.TensorRegistry = .init(allocator); + defer registry.deinit(); + try registry.registerTensor(.{ .name = "value", .file_uri = path, .shape = shape, .offset = 0 }); + try registry.registerTensor(.{ .name = "weight", .file_uri = path, .shape = shape, .offset = 2 * @sizeOf(i32) }); + var store: TensorStore = .fromRegistry(allocator, ®istry, &.{platform.replicated_mesh}); + defer store.deinit(); + var loader = try Loader.init(allocator, platform, .{ .parallelism = 2, .dma_chunks = 2, .dma_chunk_size = 4096 }); + defer loader.deinit(); + + inline for (.{ false, true }) |donate| { + const Functions = struct { + fn forward(value: Tensor, weight: Tensor) Tensor { + const sum = value.add(weight); + return if (donate) sum.reuseBuffer(value) else sum; + } + }; + var arena: std.heap.ArenaAllocator = .init(allocator); + defer arena.deinit(); + const tensor = store.view().maybeCreateBinding(&.{ "value", "weight" }, shape) orelse unreachable; + const exe = try platform.compileFn(allocator, io, Functions.forward, .{ Tensor.fromShape(shape), Tensor.fromShape(shape) }, .{}); + defer exe.deinit(); + try std.testing.expectEqual(@as(usize, @intFromBool(donate)), exe.donated_input_indices.len); + + var output: Buffer = undefined; + try loader.loadExecute(arena.allocator(), io, tensor, &output, &store, &exe, .{}); + defer output.deinit(); + try std.testing.expectEqual([2]i32{ 11, 22 }, try output.getValue([2]i32, io)); + try std.testing.expect(loader.delivered.contains(tensor.id)); + } +} + +pub const ProgressWriter = struct { + inner: *std.Io.Writer, + progress: *std.Progress.Node, + interface: std.Io.Writer, + total: usize = 0, + scale: usize, + + pub const InitOpts = struct { + scale: usize = 1, + }; + + pub fn init(inner_: *std.Io.Writer, progress_: *std.Progress.Node, opts: InitOpts) ProgressWriter { + return .{ + .inner = inner_, + .progress = progress_, + .scale = opts.scale, + .interface = .{ + .buffer = inner_.buffer, + .end = inner_.end, + .vtable = &.{ + .drain = drain, + .flush = flush, + .rebase = rebase, + .sendFile = sendFile, + }, + }, + }; + } + + pub fn pre(self: *ProgressWriter) usize { + self.inner.buffer = self.interface.buffer; + self.inner.end = self.interface.end; + return self.inner.end; + } + + pub fn post(self: *ProgressWriter, len_pre: usize, total: usize) void { + self.interface.buffer = self.inner.buffer; + self.interface.end = self.inner.end; + const drained_pre = len_pre -| self.interface.end; + self.total += drained_pre + total; + self.progress.setCompletedItems(self.total / self.scale); + } + + pub fn drain(w: *std.Io.Writer, data: []const []const u8, splat: usize) std.Io.Writer.Error!usize { + const self: *ProgressWriter = @alignCast(@fieldParentPtr("interface", w)); + const len_pre = self.pre(); + errdefer self.post(len_pre, 0); + const total = try self.inner.vtable.drain(self.inner, data, splat); + self.post(len_pre, total); + return total; + } + + pub fn sendFile(w: *std.Io.Writer, file_reader: *std.Io.File.Reader, limit: std.Io.Limit) std.Io.Writer.FileError!usize { + const self: *ProgressWriter = @alignCast(@fieldParentPtr("interface", w)); + const len_pre = self.pre(); + errdefer self.post(len_pre, 0); + const total = try self.inner.vtable.sendFile(self.inner, file_reader, limit); + self.post(len_pre, total); + return total; + } + + pub fn flush(w: *std.Io.Writer) std.Io.Writer.Error!void { + const self: *ProgressWriter = @alignCast(@fieldParentPtr("interface", w)); + const len_pre = self.pre(); + defer self.post(len_pre, 0); + try self.inner.vtable.flush(self.inner); + } + + pub fn rebase(w: *std.Io.Writer, preserve: usize, capacity: usize) std.Io.Writer.Error!void { + const self: *ProgressWriter = @alignCast(@fieldParentPtr("interface", w)); + const len_pre = self.pre(); + defer self.post(len_pre, 0); + try self.inner.vtable.rebase(self.inner, preserve, capacity); + } +}; + +pub const MemoryWriter = union(enum) { + direct: DirectMemoryWriter, + buffered: BufferedMemoryWriter, + + pub fn init( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + pools: []mem.DynamicBufferPool, + dma_allocators: []const mem.DmaAllocator, + dma_chunk_size: usize, + shape: Shape, + buffer: *Buffer, + memory: Memory.Kind, + ) !MemoryWriter { + return switch (platform.target) { + .cuda, .rocm, .oneapi, .furiosa => .{ + .direct = try .init(allocator, io, platform, pools, dma_allocators, dma_chunk_size, shape, buffer, memory), + }, + .tpu, .neuron, .cpu, .metal => if (memory == .host_pinned) + std.debug.panic("Host pinned memory is not supported on {}", .{platform.target}) + else + .{ .buffered = try .init(allocator, io, platform, shape, buffer) }, + }; + } + + pub fn interface(self: *MemoryWriter) *std.Io.Writer { + return switch (self.*) { + .direct => &self.direct.interface, + .buffered => &self.buffered.interface, + }; + } + + pub fn deinit(self: *MemoryWriter, allocator: std.mem.Allocator) void { + switch (self.*) { + .direct => self.direct.deinit(), + .buffered => self.buffered.deinit(allocator), + } + } + + pub fn setProgress(self: *MemoryWriter, progress: ?*std.Progress.Node) void { + switch (self.*) { + .direct => self.direct.setProgress(progress), + .buffered => {}, + } + } +}; + +pub const BufferedMemoryWriter = struct { + io: std.Io, + platform: *const Platform, + shape: Shape, + buffer: *Buffer, + interface: std.Io.Writer, + + pub fn init(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, shape: Shape, buffer: *Buffer) !BufferedMemoryWriter { + return .{ + .io = io, + .platform = platform, + .shape = shape, + .buffer = buffer, + .interface = .{ + .buffer = try allocator.alloc(u8, shape.byteSize()), + .vtable = &.{ + .drain = std.Io.Writer.fixedDrain, + .flush = flush, + .rebase = std.Io.Writer.failingRebase, + }, + }, + }; + } + + pub fn deinit(self: *BufferedMemoryWriter, allocator: std.mem.Allocator) void { + if (self.interface.buffer.len > 0) { + allocator.free(self.interface.buffer); + } + } + + pub fn flush(w: *std.Io.Writer) std.Io.Writer.Error!void { + const self: *BufferedMemoryWriter = @alignCast(@fieldParentPtr("interface", w)); + + self.buffer.* = Buffer.from( + self.io, + self.platform, + self.shape, + @ptrCast(self.interface.buffer), + .{ .wait = true }, + ) catch return std.Io.Writer.Error.WriteFailed; + } +}; + +const DirectShardWriter = struct { + const EventContext = struct { + self: *DirectShardWriter, + err: ?*pjrt.Error = null, + pjrt_event: *pjrt.Event, + event: std.Io.Event = .unset, + buffer: []u8, + }; + + allocator: std.mem.Allocator, + io: std.Io, + memory: *const Memory, + pool: *mem.DynamicBufferPool, + total: usize, + pjrt_buffer: *pjrt.Buffer, + + transfer_manager: *pjrt.AsyncHostToDeviceTransferManager, + offset: usize = 0, + + interface: std.Io.Writer, + flip_flop: u1 = 0, + events_contexts: [2]?EventContext = @splat(null), + + pub fn init(allocator: std.mem.Allocator, io: std.Io, memory: *const Memory, pool: *mem.DynamicBufferPool, shape: Shape) !DirectShardWriter { + const shape_spec: pjrt.ShapeSpec = .init(shape.dims(), pjrtx.bufferTypeFromDtype(shape.dtype())); + const transfer_manager = try memory.platform.pjrt_client.createBuffersForAsyncHostToDevice( + memory.platform.pjrt_api, + .{ + .shape_specs = &.{shape_spec}, + .memory = memory.pjrt_memory, + }, + ); + errdefer transfer_manager.deinit(memory.platform.pjrt_api); + + const pjrt_buffer = try transfer_manager.retrieveBuffer(memory.platform.pjrt_api, 0); + errdefer pjrt_buffer.deinit(memory.platform.pjrt_api); + + const buf = try pool.get(allocator, io); + + return .{ + .allocator = allocator, + .io = io, + .memory = memory, + .pool = pool, + .total = shape.byteSize(), + .pjrt_buffer = pjrt_buffer, + .transfer_manager = transfer_manager, + .interface = .{ + .buffer = buf[0..@min(buf.len, shape.byteSize())], + .vtable = &.{ + .drain = drain, + .flush = flush, + .rebase = rebase, + }, + }, + }; + } + + pub fn deinit(self: *DirectShardWriter) void { + self.transfer_manager.deinit(self.memory.platform.pjrt_api); + } + + fn drain(w: *std.Io.Writer, data: []const []const u8, _: usize) std.Io.Writer.Error!usize { + const self: *DirectShardWriter = @alignCast(@fieldParentPtr("interface", w)); + + const chunk = data[0]; + if (chunk.len > self.interface.buffer.len) return std.Io.Writer.Error.WriteFailed; + if (chunk.len > self.total - (self.offset + self.interface.end)) return std.Io.Writer.Error.WriteFailed; + + const needs_fresh_buffer = chunk.len > self.interface.buffer.len - self.interface.end; + if (needs_fresh_buffer) { + try self.interface.flush(); + } + + @memcpy(self.interface.buffer[self.interface.end..][0..chunk.len], chunk); + self.interface.end += chunk.len; + + const buffer_full = self.interface.end == self.interface.buffer.len; + if (buffer_full) { + try self.interface.flush(); + } + + return chunk.len; + } + + fn flush(w: *std.Io.Writer) std.Io.Writer.Error!void { + const self: *DirectShardWriter = @alignCast(@fieldParentPtr("interface", w)); + try self.flushBuffered(); + } + + fn rebase(w: *std.Io.Writer, preserve: usize, capacity: usize) std.Io.Writer.Error!void { + const self: *DirectShardWriter = @alignCast(@fieldParentPtr("interface", w)); + if (self.interface.buffer.len - self.interface.end >= capacity) return; + if (preserve != 0) return std.Io.Writer.Error.WriteFailed; + try self.interface.flush(); + } + + fn flushBuffered(self: *DirectShardWriter) std.Io.Writer.Error!void { + if (self.offset >= self.total) return; + + const pjrt_api = self.memory.platform.pjrt_api; + + const current_buffer = self.interface.buffer; + const buffered = self.interface.buffered(); + + const slice = buffered[0..@min(buffered.len, self.total - self.offset)]; + if (slice.len == 0) return; + const is_last = (self.offset + slice.len) >= self.total; + + const transfer_event = self.transfer_manager.transferData(pjrt_api, 0, slice, @intCast(self.offset), is_last) catch |err| { + log.err("error when transferring data to device: {any}", .{err}); + return std.Io.Writer.Error.WriteFailed; + }; + + const ctx = &self.events_contexts[@intCast(self.flip_flop)]; + ctx.* = .{ + .self = self, + .buffer = current_buffer, + .pjrt_event = transfer_event, + }; + + transfer_event.onReady(pjrt_api, EventContext, struct { + fn call(err: ?*pjrt.Error, ctx_: *EventContext) void { + ctx_.self.pool.put(ctx_.self.io, ctx_.buffer); + ctx_.err = err; + ctx_.event.set(ctx_.self.io); + } + }.call, &(ctx.*.?)) catch |err| { + log.err("error when setting up transfer completion callback: {any}", .{err}); + return std.Io.Writer.Error.WriteFailed; + }; + + if (self.events_contexts[@intCast(self.flip_flop ^ 1)]) |*ctx_previous| { + defer self.events_contexts[@intCast(self.flip_flop ^ 1)] = null; + ctx_previous.event.waitUncancelable(self.io); + defer ctx_previous.pjrt_event.deinit(pjrt_api); + if (ctx_previous.err) |e| { + defer e.deinit(pjrt_api); + log.err("error while awaiting: {s}: {s}", .{ + @tagName(e.getCode(pjrt_api)), + e.getMessage(pjrt_api), + }); + return std.Io.Writer.Error.WriteFailed; + } + } + + if (is_last) { + defer ctx.* = null; + defer self.interface = .failing; + const ctx_ = &ctx.*.?; + ctx_.event.waitUncancelable(self.io); + defer ctx_.pjrt_event.deinit(pjrt_api); + if (ctx_.err) |e| { + defer e.deinit(pjrt_api); + log.err("error while awaiting: {s}: {s}", .{ + @tagName(e.getCode(pjrt_api)), + e.getMessage(pjrt_api), + }); + return std.Io.Writer.Error.WriteFailed; + } + } else { + self.interface.end = 0; + const buf = self.pool.get(self.allocator, self.io) catch |err| { + log.err("unable to get a new buffer from the pool: {any}", .{err}); + return std.Io.Writer.Error.WriteFailed; + }; + self.interface.buffer = buf[0..@min(buf.len, self.total - (self.offset + slice.len))]; + } + self.flip_flop ^= 1; + self.offset += slice.len; + } +}; + +const ShardProgress = struct { + const scale: usize = 1024; + + node: std.Progress.Node, + label: [32]u8 = undefined, + completed: usize = 0, + + fn set(self: *ShardProgress, completed: usize) void { + self.completed = completed; + self.node.setCompletedItems(std.math.divCeil(usize, self.completed, scale) catch unreachable); + } +}; + +// Dispatch planning bridges two different orders: +// +// 1. Placement traversal emits byte ranges per shard writer, in device order. +// That order is convenient for asking "what bytes belong to this shard?", +// but it does not match how a tensor file is read. +// +// 2. Readers stream tensor bytes in global row-major order. DirectMemoryWriter +// must therefore consume dispatch spans in increasing global byte offset. +// +// We first collect placement spans, sort them by global byte range, then fold +// identical ranges into one primary dispatch span plus mirror writers. Identical +// ranges represent replicated shard data: the reader provides those bytes once, +// the primary writer receives them zero-copy, and mirrors receive a copy. +const DispatchSpans = struct { + const DispatchSpan = struct { + start: usize, + end: usize, + primary_writer: usize, + mirror_writer_start: usize, + mirror_writer_len: usize, + }; + + const PlacementSpan = struct { + writer_index: usize, + start: usize, + len: usize, + order: usize, + }; + + spans: []DispatchSpan, + mirror_writers: []usize, + + fn init(allocator: std.mem.Allocator, shape: Shape) !DispatchSpans { + const placement = try Sharding.Placement.init(&shape); + const ordered_devices = placement.mesh().?.devicesInCanonicalOrder(); + + var placement_span_count: usize = 0; + for (0..ordered_devices.len) |device_index| { + placement_span_count += placementSpanCount(shape, placement.slices(device_index, shape).constSlice()); + } + + var placement_spans: std.ArrayList(PlacementSpan) = try .initCapacity(allocator, placement_span_count); + defer placement_spans.deinit(allocator); + + const byte_strides = shape.computeByteStrides(); + + for (0..ordered_devices.len) |writer_index| { + appendShardPlacementSpans(&placement_spans, shape, placement.slices(writer_index, shape).constSlice(), byte_strides.constSlice(), writer_index); + } + + std.debug.assert(placement_spans.items.len == placement_span_count); + + var spans: std.ArrayList(DispatchSpan) = try .initCapacity(allocator, placement_spans.items.len); + errdefer spans.deinit(allocator); + + var mirror_writers: std.ArrayList(usize) = try .initCapacity(allocator, placement_spans.items.len); + errdefer mirror_writers.deinit(allocator); + + try deduplicateByRange(allocator, placement_spans.items, shape.byteSize(), &spans, &mirror_writers); + + const spans_ = try spans.toOwnedSlice(allocator); + errdefer allocator.free(spans_); + + const mirror_writers_ = try mirror_writers.toOwnedSlice(allocator); + errdefer allocator.free(mirror_writers_); + + return .{ + .spans = spans_, + .mirror_writers = mirror_writers_, + }; + } + + fn deinit(self: DispatchSpans, allocator: std.mem.Allocator) void { + allocator.free(self.spans); + allocator.free(self.mirror_writers); + } + + fn deduplicateByRange( + allocator: std.mem.Allocator, + placement_spans: []PlacementSpan, + total_bytes: usize, + spans: *std.ArrayList(DispatchSpan), + mirror_writers: *std.ArrayList(usize), + ) !void { + const SortContext = struct { + fn lessThan(_: void, lhs: PlacementSpan, rhs: PlacementSpan) bool { + if (lhs.start != rhs.start) return lhs.start < rhs.start; + if (lhs.len != rhs.len) return lhs.len < rhs.len; + return lhs.order < rhs.order; + } + }; + + std.mem.sort(PlacementSpan, placement_spans, {}, SortContext.lessThan); + + var i: usize = 0; + var cursor: usize = 0; + while (i < placement_spans.len) { + const span = placement_spans[i]; + if (span.start != cursor) return error.NonContiguousShardPlacement; + + const mirror_writer_start = mirror_writers.items.len; + var j = i + 1; + while (j < placement_spans.len) : (j += 1) { + const mirror = placement_spans[j]; + if (mirror.start != span.start or mirror.len != span.len) break; + try mirror_writers.append(allocator, mirror.writer_index); + } + + try spans.append(allocator, .{ + .start = span.start, + .end = span.start + span.len, + .primary_writer = span.writer_index, + .mirror_writer_start = mirror_writer_start, + .mirror_writer_len = j - i - 1, + }); + cursor += span.len; + i = j; + } + + if (cursor != total_bytes) return error.NonContiguousShardPlacement; + } + + fn appendPlacementSpan(placement_spans: *std.ArrayList(PlacementSpan), writer_index: usize, start: usize, len: usize) void { + placement_spans.appendAssumeCapacity(.{ + .writer_index = writer_index, + .start = start, + .len = len, + .order = placement_spans.items.len, + }); + } + + fn appendShardPlacementSpans( + placement_spans: *std.ArrayList(PlacementSpan), + shape: Shape, + slices: []const Placement.Slice1d, + byte_strides: []const i64, + writer_index: usize, + ) void { + if (shape.rank() == 0) { + appendPlacementSpan(placement_spans, writer_index, 0, shape.byteSize()); + return; + } + + appendShardAxisPlacementSpans(placement_spans, slices, byte_strides, writer_index, 0, contiguousSliceAxis(shape, slices), 0); + } + + fn appendShardAxisPlacementSpans( + placement_spans: *std.ArrayList(PlacementSpan), + slices: []const Placement.Slice1d, + byte_strides: []const i64, + writer_index: usize, + axis: usize, + contiguous_axis: usize, + base_start: i64, + ) void { + const slice = slices[axis]; + if (slice.size == 0) return; + + if (axis == contiguous_axis) { + const span_start: usize = @intCast(base_start + slice.start * byte_strides[axis]); + const span_len: usize = @intCast(slice.size * byte_strides[axis]); + appendPlacementSpan(placement_spans, writer_index, span_start, span_len); + return; + } + + var i: i64 = 0; + while (i < slice.size) : (i += 1) { + const child_start = base_start + (slice.start + i) * byte_strides[axis]; + appendShardAxisPlacementSpans(placement_spans, slices, byte_strides, writer_index, axis + 1, contiguous_axis, child_start); + } + } + + fn placementSpanCount(shape: Shape, slices: []const Placement.Slice1d) usize { + if (shape.rank() == 0) return 1; + + const contiguous_axis = contiguousSliceAxis(shape, slices); + var count: usize = 1; + for (slices[0..contiguous_axis]) |slice| { + count *= @intCast(slice.size); + } + return count; + } + + fn contiguousSliceAxis(shape: Shape, slices: []const Placement.Slice1d) usize { + var axis = shape.rank() - 1; + while (axis > 0) { + const slice = slices[axis]; + if (slice.start != 0 or slice.size != shape.dim(axis)) break; + axis -= 1; + } + return axis; + } +}; + +// Direct load writer state machine. +// +// std.Io.Reader +// | +// v +// DirectMemoryWriter.interface.buffer +// | aliases active primary DirectShardWriter DMA buffer +// v +// commitWindow() +// |-- primary: advance DirectShardWriter.interface.end +// |-- mirrors: copy committed bytes with writeAll() +// |-- boundaries: flush full shard buffers / chunk fence +// v +// DirectShardWriter.flushBuffered() +// | +// v +// PJRT AsyncHostToDeviceTransferManager +// +// The public writer never owns a host staging buffer: it exposes a visible +// prefix of a real shard DMA buffer. The visible prefix may extend beyond the +// current dispatch span to coalesce reader requests, but `commitWindow` must +// scatter those bytes before submitting or rotating the active buffer. The +// first active-primary segment is zero-copy; crossed spans and mirrors treat +// the active buffer as scratch and copy into their own shard writers. +// +// `byte_cursor` is the global tensor position. Shard writer `interface.end` is +// local to the current shard DMA buffer. +pub const DirectMemoryWriter = struct { + allocator: std.mem.Allocator, + shard_writers: []DirectShardWriter, + // Global stream-order spans produced from placement; this is the routing table for committed bytes. + dispatch_spans: DispatchSpans, + // Current entry in `dispatch_spans.spans`; advances whenever `byte_cursor` reaches that span end. + span_index: usize = 0, + // Global tensor byte offset already scattered into shard writers. + byte_cursor: usize = 0, + // Shard writer whose DMA buffer is currently exposed as `interface.buffer`. + active_writer_index: usize, + // Start offset of new reader bytes inside the active shard writer buffer. + window_start: usize, + // Logical maximum public alias window before forcing a cross-shard flush fence. + dma_chunk_size: usize, + shard_progress: ?[]ShardProgress = null, + interface: std.Io.Writer, + + pub fn init( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const Platform, + pools: []mem.DynamicBufferPool, + dma_allocators: []const mem.DmaAllocator, + dma_chunk_size: usize, + shape: Shape, + buffer: *Buffer, + memory: Memory.Kind, + ) !DirectMemoryWriter { + var placed_shape = shape; + if (placed_shape._sharding.mesh == null) placed_shape._sharding.mesh = platform.replicated_mesh; + const placement = try Sharding.Placement.init(&placed_shape); + const mesh = placement.mesh().?; + const ordered_devices = mesh.devicesInCanonicalOrder(); + var shard_writers = try allocator.alloc(DirectShardWriter, ordered_devices.len); + errdefer allocator.free(shard_writers); + + var initialized: usize = 0; + errdefer for (shard_writers[0..initialized]) |*writer| { + writer.deinit(); + }; + + var pjrt_buffers: Buffer.Shards = .empty; + for (ordered_devices, 0..) |device, i| { + const pool = &pools[device.id]; + const shard_dma_allocator = dma_allocators[device.id].allocator(); + const pjrt_mem = platform.devices[device.id].memory(memory).?; + + shard_writers[i] = try .init(shard_dma_allocator, io, pjrt_mem, pool, placement.localShape(placed_shape)); + initialized += 1; + + pjrt_buffers.appendAssumeCapacity(shard_writers[i].pjrt_buffer); + } + + buffer.* = .fromPjrtBuffers(platform, shape, pjrt_buffers.constSlice()); + + const dispatch_spans: DispatchSpans = try .init(allocator, placed_shape); + errdefer dispatch_spans.deinit(allocator); + + const first_span = dispatch_spans.spans[0]; + const first_writer = &shard_writers[first_span.primary_writer]; + const first_window = @min(dma_chunk_size, first_writer.interface.buffer.len); + + return .{ + .allocator = allocator, + .shard_writers = shard_writers, + .dispatch_spans = dispatch_spans, + .active_writer_index = first_span.primary_writer, + .window_start = first_writer.interface.end, + .dma_chunk_size = dma_chunk_size, + .interface = .{ + .buffer = first_writer.interface.buffer[0..first_window], + .end = 0, + .vtable = &.{ + .drain = drain, + .flush = flush, + .rebase = rebase, + }, + }, + }; + } + + pub fn deinit(self: *DirectMemoryWriter) void { + self.setProgress(null); + for (self.shard_writers) |*writer| { + writer.deinit(); + } + self.allocator.free(self.shard_writers); + self.dispatch_spans.deinit(self.allocator); + } + + pub fn setProgress(self: *DirectMemoryWriter, progress: ?*std.Progress.Node) void { + const parent = progress orelse { + const states = self.shard_progress orelse return; + for (states) |*s| { + s.node.end(); + } + self.allocator.free(states); + self.shard_progress = null; + return; + }; + + std.debug.assert(self.shard_progress == null); + const states = self.allocator.alloc(ShardProgress, self.shard_writers.len) catch return; + for (states, self.shard_writers, 0..) |*state, writer, i| { + state.completed = 0; + const label = std.fmt.bufPrint(&state.label, "shard[{d}]", .{i}) catch unreachable; + const total_items = std.math.divCeil(usize, writer.total, ShardProgress.scale) catch unreachable; + state.node = parent.start(label, total_items); + state.node.setCompletedItems(0); + } + self.shard_progress = states; + } + + fn drain(w: *std.Io.Writer, data: []const []const u8, splat: usize) std.Io.Writer.Error!usize { + const self: *DirectMemoryWriter = @alignCast(@fieldParentPtr("interface", w)); + + try self.commitWindow(); + if (self.interface.end == self.interface.buffer.len) return std.Io.Writer.Error.WriteFailed; + + const consumed = consumed: { + var n: usize = 0; + for (data[0 .. data.len - 1]) |bytes| { + const copied = self.writeWindow(bytes); + n += copied; + + if (copied < bytes.len) break :consumed n; + } + + const pattern = data[data.len - 1]; + switch (pattern.len) { + 0 => {}, + 1 => { + const dest = self.interface.buffer[self.interface.end..]; + const count = @min(dest.len, splat); + @memset(dest[0..count], pattern[0]); + + self.interface.end += count; + n += count; + }, + else => for (0..splat) |_| { + const copied = self.writeWindow(pattern); + n += copied; + + if (copied < pattern.len) break; + }, + } + + break :consumed n; + }; + + try self.commitWindow(); + return consumed; + } + + fn writeWindow(self: *DirectMemoryWriter, bytes: []const u8) usize { + const dest = self.interface.buffer[self.interface.end..]; + const n = @min(dest.len, bytes.len); + @memcpy(dest[0..n], bytes[0..n]); + self.interface.end += n; + return n; + } + + fn flush(w: *std.Io.Writer) std.Io.Writer.Error!void { + const self: *DirectMemoryWriter = @alignCast(@fieldParentPtr("interface", w)); + + // Commit the active public window if one remains + if (self.span_index < self.dispatch_spans.spans.len) { + try self.commitWindow(); + } + + for (self.shard_writers, 0..) |*shard_writer, i| { + try shard_writer.interface.flush(); + if (self.shard_progress) |states| { + states[i].set(shard_writer.total); + } + } + + self.interface = .failing; + } + + fn rebase(w: *std.Io.Writer, preserve: usize, capacity: usize) std.Io.Writer.Error!void { + const self: *DirectMemoryWriter = @alignCast(@fieldParentPtr("interface", w)); + + if (self.interface.buffer.len - self.interface.end >= capacity) return; + if (preserve != 0) return std.Io.Writer.Error.WriteFailed; + + try self.commitWindow(); + if (self.interface.buffer.len - self.interface.end < capacity) return std.Io.Writer.Error.WriteFailed; + } + + // `interface.buffer` is a clipped alias of the active primary shard DMA + // buffer. Reader writes advance only `DirectMemoryWriter.interface.end`; + // this function scatters that visible range through dispatch spans. + // + // The leading segment for `active_writer_index` is already in the right + // DMA buffer and is committed by advancing that shard writer. Later spans + // use the same bytes as scratch and are copied before any full buffers are + // submitted or the public alias is rotated. + fn commitWindow(self: *DirectMemoryWriter) std.Io.Writer.Error!void { + // All dispatch spans are consumed; no shard DMA buffer remains valid to + // expose as the public alias until flush marks the writer as failing. + if (self.span_index >= self.dispatch_spans.spans.len) { + self.interface.buffer = &.{}; + self.interface.end = 0; + return; + } + + // Only bytes after `window_start` are new reader writes. Bytes before + // it may already be pending in the active shard writer. + const source = self.interface.buffer[self.window_start..self.interface.end]; + if (source.len == 0) return; + + // The visible alias window may cross many dispatch spans. Walk the + // global cursor and route each subrange to its primary and mirrors. + var consumed: usize = 0; + while (consumed < source.len) { + if (self.span_index >= self.dispatch_spans.spans.len) return std.Io.Writer.Error.WriteFailed; + + const span = self.dispatch_spans.spans[self.span_index]; + if (self.byte_cursor < span.start or self.byte_cursor >= span.end) return std.Io.Writer.Error.WriteFailed; + + const n = @min(source.len - consumed, span.end - self.byte_cursor); + const chunk = source[consumed..][0..n]; + + // The first span in the active alias is already in the primary DMA + // buffer. Later ranges use the active buffer as scratch and must be + // copied into their real destination writer. + if (span.primary_writer == self.active_writer_index and consumed == 0) { + self.shard_writers[span.primary_writer].interface.end += n; + } else { + const primary_writer = &self.shard_writers[span.primary_writer]; + if (span.primary_writer == self.active_writer_index) { + @memmove(primary_writer.interface.buffer[primary_writer.interface.end..][0..n], chunk); + primary_writer.interface.end += n; + } else { + try primary_writer.interface.writeAll(chunk); + } + } + if (self.shard_progress) |states| { + states[span.primary_writer].set(states[span.primary_writer].completed + n); + } + + const mirror_writer_end = span.mirror_writer_start + span.mirror_writer_len; + for (self.dispatch_spans.mirror_writers[span.mirror_writer_start..mirror_writer_end]) |mirror_writer_index| { + const mirror_writer = &self.shard_writers[mirror_writer_index]; + if (mirror_writer_index == self.active_writer_index) { + @memmove(mirror_writer.interface.buffer[mirror_writer.interface.end..][0..n], chunk); + mirror_writer.interface.end += n; + } else { + try mirror_writer.interface.writeAll(chunk); + } + if (self.shard_progress) |states| { + states[mirror_writer_index].set(states[mirror_writer_index].completed + n); + } + } + + self.byte_cursor += n; + consumed += n; + if (self.byte_cursor == span.end) { + self.span_index += 1; + } + } + + // All scratch copies are complete now, so full DMA buffers may be + // submitted without invalidating bytes still needed for scatter. + for (self.shard_writers) |*writer| { + if (writer.interface.end == writer.interface.buffer.len) { + try writer.interface.flush(); + } + } + + if (@mod(self.byte_cursor, self.dma_chunk_size) == 0) { + for (self.shard_writers) |*writer| { + try writer.interface.flush(); + } + } + + // The last committed window may finish the tensor exactly. Leave an + // empty public window so any further writes fail through std.Io. + if (self.span_index >= self.dispatch_spans.spans.len) { + self.interface.buffer = &.{}; + self.interface.end = 0; + return; + } + + // Rotate the public alias to the primary writer for the next stream + // span. `window_start` preserves that writer's already-buffered prefix + // so only newly advanced bytes are scattered on the next commit. + const next_span = self.dispatch_spans.spans[self.span_index]; + const next_writer = &self.shard_writers[next_span.primary_writer]; + self.active_writer_index = next_span.primary_writer; + self.window_start = next_writer.interface.end; + + // Publish the next public alias. Its backing memory is the next primary + // shard buffer, but the visible length is allowed to coalesce across + // upcoming spans until a buffer or tensor boundary. + const total = self.dispatch_spans.spans[self.dispatch_spans.spans.len - 1].end; + const buffer_remaining = next_writer.interface.buffer.len - self.window_start; + const chunk_offset = @mod(self.byte_cursor, self.dma_chunk_size); + const chunk_remaining = if (chunk_offset == 0) self.dma_chunk_size else self.dma_chunk_size - chunk_offset; + const tensor_remaining = total - self.byte_cursor; + const visible_remaining = @min(buffer_remaining, @min(chunk_remaining, tensor_remaining)); + if (visible_remaining == 0) return std.Io.Writer.Error.WriteFailed; + + self.interface.buffer = next_writer.interface.buffer[0 .. self.window_start + visible_remaining]; + self.interface.end = self.window_start; + } +}; + +const DirectMemoryWriterDeviceTest = struct { + const WriteMode = enum { + stream_remaining, + writable_slice_greedy, + }; + + pub const Scenario = struct { + name: []const u8, + create_options: CreateOptions, + shape: Shape, + logical_mesh: Sharding.LogicalMesh, + strategy: Sharding.Strategy, + write_mode: WriteMode = .stream_remaining, + writable_slice_min_len: usize = 128, + pool_chunks: usize = 4, + pool_chunk_size: usize = 1 << 20, + memory: Memory.Kind = .default, + }; + + allocator: std.mem.Allocator, + io: std.Io, + + fn run(self: DirectMemoryWriterDeviceTest, scenario: Scenario, partitioning: anytype) !void { + var platform = Platform.auto(self.allocator, self.io, scenario.create_options) catch return error.SkipZigTest; + defer platform.deinit(self.allocator, self.io); + + const mesh: Sharding.Mesh = try .init(scenario.name, &platform.physical_mesh, scenario.logical_mesh, scenario.strategy); + std.log.warn("mesh: {f}", .{mesh}); + try self.runDirectMemoryWriter( + platform, + scenario.shape.withPartitioning(&mesh, partitioning), + scenario.write_mode, + scenario.writable_slice_min_len, + scenario.pool_chunks, + scenario.pool_chunk_size, + scenario.memory, + ); + } + + fn runDirectMemoryWriter( + self: DirectMemoryWriterDeviceTest, + platform: *const Platform, + shape: Shape, + write_mode: WriteMode, + writable_slice_min_len: usize, + pool_chunks: usize, + pool_chunk_size: usize, + memory: Memory.Kind, + ) !void { + const slice = try Slice.alloc(self.allocator, shape); + defer slice.free(self.allocator); + + for (slice.items(f32), 0..) |*e, i| { + e.* = @as(f32, @floatFromInt(i)); + } + + const pool_count = platform.devices.len; + const dma_allocators = try self.allocator.alloc(mem.DmaAllocator, pool_count); + defer self.allocator.free(dma_allocators); + for (platform.devices, 0..) |*device, i| { + dma_allocators[i] = .init(self.allocator, device); + } + + const pools = try self.allocator.alloc(mem.DynamicBufferPool, pool_count); + defer self.allocator.free(pools); + for (pools) |*pool| { + pool.* = .init(pool_chunks, pool_chunk_size); + } + defer for (pools, 0..) |*pool, i| { + pool.deinit(dma_allocators[i].allocator()); + }; + + var written_buffer: Buffer = undefined; + var writer: DirectMemoryWriter = try .init( + self.allocator, + self.io, + platform, + pools, + dma_allocators, + pool_chunk_size, + shape, + &written_buffer, + memory, + ); + defer writer.deinit(); + defer written_buffer.deinit(); + + switch (write_mode) { + .stream_remaining => { + var reader: std.Io.Reader = .fixed(slice.constData()); + const streamed = try reader.streamRemaining(&writer.interface); + try std.testing.expectEqual(slice.constData().len, streamed); + }, + .writable_slice_greedy => { + var offset: usize = 0; + while (offset < slice.constData().len) { + const min_len = @max(@as(usize, 1), writable_slice_min_len); + const dest = try writer.interface.writableSliceGreedy(min_len); + const to_write = @min(dest.len, slice.constData().len - offset); + if (to_write == 0) return std.Io.Writer.Error.WriteFailed; + @memcpy(dest[0..to_write], slice.constData()[offset..][0..to_write]); + writer.interface.advance(to_write); + offset += to_write; + } + }, + } + + try writer.interface.flush(); + try written_buffer.await(self.io); + + var written_slice = try written_buffer.toSliceAlloc(self.allocator, self.io); + defer written_slice.free(self.allocator); + try std.testing.expectEqualSlices(u8, slice.constData(), written_slice.constData()); + } +}; + +test "DirectMemoryWriter: replicated tensor" { + const case: DirectMemoryWriterDeviceTest = .{ + .allocator = std.testing.allocator, + .io = std.testing.io, + }; + + try case.run(.{ + .name = "replicated_auto", + .create_options = .{ + .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus2d }, + .cpu = .{ .device_count = 4 }, + }, + .shape = Shape.init(.{ .rows = 8, .cols = 128 }, .f32), + .logical_mesh = .mesh(.{ .x = .high_bandwidth }), + .strategy = .parseBindings(.{ .x = .link_x }), + }, .{ .rows = .replicated, .cols = .replicated }); +} + +test "DirectMemoryWriter: 1D model split with 2x2 physical mesh" { + const case: DirectMemoryWriterDeviceTest = .{ + .allocator = std.testing.allocator, + .io = std.testing.io, + }; + + try case.run(.{ + .name = "model_auto", + .create_options = .{ + .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus2d }, + .cpu = .{ .device_count = 4 }, + }, + .shape = Shape.init(.{ .rows = 8, .cols = 1024 }, .f32), + .logical_mesh = .mesh(.{ .model = .high_bandwidth }), + .strategy = .parseBindings(.{ .model = .link_x }), + }, .{ .rows = .replicated, .cols = .model }); +} + +test "DirectMemoryWriter: 2D batch/model split with 2x2 physical mesh" { + const case: DirectMemoryWriterDeviceTest = .{ + .allocator = std.testing.allocator, + .io = std.testing.io, + }; + + try case.run(.{ + .name = "batch_model_2d_torus", + .create_options = .{ + .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus2d }, + .cpu = .{ .device_count = 4 }, + }, + .shape = Shape.init(.{ .batch = 8, .model = 1024 }, .f32), + .logical_mesh = .mesh(.{ + .batch = .low_bandwidth, + .model = .high_bandwidth, + }), + .strategy = .parseBindings(.{ .batch = .link_x, .model = .link_y }), + }, .{ .batch = .batch, .model = .model }); +} + +test "DirectMemoryWriter: folded model mesh with 2x2 physical mesh" { + const case: DirectMemoryWriterDeviceTest = .{ + .allocator = std.testing.allocator, + .io = std.testing.io, + }; + + try case.run(.{ + .name = "model_folded_2d_torus", + .create_options = .{ + .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus2d }, + .cpu = .{ .device_count = 4 }, + }, + .shape = Shape.init(.{ .model = 4096 }, .f32), + .logical_mesh = .mesh(.{ .model = .high_bandwidth }), + .strategy = blk: { + var strategy: Sharding.Strategy = .parseBindings(.{ .model = .link_x }); + strategy.addFold(.link_x, &.{ .link_x, .link_y }); + break :blk strategy; + }, + }, .{ .model = .model }); +} + +test "DirectMemoryWriter: writableSliceGreedy with mirrored shards" { + const case: DirectMemoryWriterDeviceTest = .{ + .allocator = std.testing.allocator, + .io = std.testing.io, + }; + + try case.run(.{ + .name = "model_auto_writable_slice", + .create_options = .{ + .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus2d }, + .cpu = .{ .device_count = 4 }, + }, + .shape = Shape.init(.{ .rows = 8, .cols = 1024 }, .f32), + .logical_mesh = .mesh(.{ .model = .high_bandwidth }), + .strategy = .parseBindings(.{ .model = .link_x }), + .write_mode = .writable_slice_greedy, + .writable_slice_min_len = 64, + .pool_chunk_size = 1024, + }, .{ .rows = .replicated, .cols = .model }); +} + +test "DirectMemoryWriter: 3D topology folded model + replicated batch" { + const case: DirectMemoryWriterDeviceTest = .{ + .allocator = std.testing.allocator, + .io = std.testing.io, + }; + + try case.run(.{ + .name = "topology_3d_folded_model", + .create_options = .{ + .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus3d }, + .cpu = .{ .device_count = 8 }, + }, + .shape = Shape.init(.{ .batch = 16, .model = 4096 }, .f32), + .logical_mesh = .mesh(.{ + .batch = .low_bandwidth, + .model = .high_bandwidth, + }), + .strategy = blk: { + var strategy: Sharding.Strategy = .parseBindings(.{ .model = .link_x }); + strategy.addFold(.link_x, &.{ .link_x, .link_z }); + break :blk strategy; + }, + }, .{ .batch = .replicated, .model = .model }); +} + +test "MemoryWriter can produce a host pinned buffer" { + const case: DirectMemoryWriterDeviceTest = .{ + .allocator = std.testing.allocator, + .io = std.testing.io, + }; + + try case.run(.{ + .name = "host_pinned", + .create_options = .{ + .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus1d }, + }, + .shape = Shape.init(.{ .batch = 16, .model = 4096 }, .f32), + .logical_mesh = .mesh(.{ .model = .high_bandwidth }), + .strategy = .parseBindings(.{ .model = .link_x }), + .memory = .host_pinned, + }, .{ .batch = .replicated, .model = .model }); +} diff --git a/zml/kernel/BUILD.bazel b/zml/kernel/BUILD.bazel new file mode 100644 index 00000000..61ab44f8 --- /dev/null +++ b/zml/kernel/BUILD.bazel @@ -0,0 +1,15 @@ +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "kernel", + srcs = [ + "kernel.zig", + "//zml/kernel/cuda_tile", + "//zml/kernel/cute", + "//zml/kernel/fly", + "//zml/kernel/metal", + "//zml/kernel/mosaic_tpu", + "//zml/kernel/tcl", + "//zml/kernel/triton", + ], +) diff --git a/zml/kernel/cuda_tile/BUILD.bazel b/zml/kernel/cuda_tile/BUILD.bazel new file mode 100644 index 00000000..89b7977e --- /dev/null +++ b/zml/kernel/cuda_tile/BUILD.bazel @@ -0,0 +1,9 @@ +package(default_visibility = ["//zml/kernel:__pkg__"]) + +filegroup( + name = "cuda_tile", + srcs = [ + "cuda_tile.zig", + "test.zig", + ], +) diff --git a/zml/kernel/cuda_tile/cuda_tile.zig b/zml/kernel/cuda_tile/cuda_tile.zig new file mode 100644 index 00000000..3f6d8734 --- /dev/null +++ b/zml/kernel/cuda_tile/cuda_tile.zig @@ -0,0 +1,163 @@ +const std = @import("std"); + +const cuda_tile_builder = @import("dsl/cuda_tile/builder"); +pub const Builder = cuda_tile_builder.Builder; +pub const Value = cuda_tile_builder.Value; +pub const DType = cuda_tile_builder.DType; +pub const FinishError = cuda_tile_builder.FinishError; +pub const EntryHint = cuda_tile_builder.EntryHint; +pub const Arch = cuda_tile_builder.Arch; +const func = @import("mlir/dialects/func"); +const mlir = @import("mlir"); +const stablehlo = @import("mlir/dialects/stablehlo"); +const stdx = @import("stdx"); + +const Compiler = @import("../../Compiler.zig"); +const DataType = @import("../../dtype.zig").DataType; +const mlirx = @import("../../mlirx.zig"); +const ops = @import("../../ops.zig"); +const platform_ = @import("../../platform.zig"); +const Shape = @import("../../shape.zig").Shape; +const Tensor = @import("../../tensor.zig").Tensor; + +test { + _ = @import("test.zig"); +} + +/// Build a struct type with one field per name, all of type `FieldT`. +fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { + var name_slices: [names.len][]const u8 = undefined; + for (names, 0..) |n, i| name_slices[i] = n; + return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); +} + +fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { + mlir.registerPasses("Transforms"); + + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + + registerDialects(registry); + + func.registerExtensions(registry); + + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + + return ctx; +} + +fn resolveOutputOperandAliases( + aliases_opt: anytype, + operand_offset: i64, +) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { + var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; + + if (aliases_opt) |a| { + inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { + if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ + .output_index = @intCast(output_index), + .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, + }); + } + } + + return out; +} + +pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(cuda_tile_builder.registerDialects); +} + +pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .i1, + .i4, .u4 => .i4, + .i8, .u8 => .i8, + .i16, .u16 => .i16, + .i32, .u32 => .i32, + .i64, .u64 => .i64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f8e4m3fn => .f8e4m3fn, + .f8e5m2 => .f8e5m2, + .f8e8m0 => .f8e8m0fnu, + .f4e2m1 => .f4e2m1fn, + else => std.debug.panic("zml.kernel.cuda_tile.from: dtype {s} has no CUDA Tile IR equivalent", .{@tagName(dt)}), + }; +} + +fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; +} + +pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), +) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + /// No `num_warps`/`num_stages`: warp and CTA tuning is + /// `optimization_hints` inside the IR (`Opts.hints`). + pub const CallOpts = struct { + cfg: ConfigT, + grid: [3]i32, + /// Outputs XLA zeroes before the launch, by position in + /// `spec.outputs`, ascending. + zeroed_outputs: []const i32 = &.{}, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try cuda_tile_builder.Builder.open(allocator, ctx, name); + defer b.deinit(); + + try spec.run(&b, cfg); + + return b.finish(&.{}); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + + const ir = emit(cur.allocator, opts.cfg) catch |err| + std.debug.panic("zml.kernel.cuda_tile.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); + + const tensor_results = ops.cudaTile(inputs_arr, outputs_arr, .{ + .name = name, + .ir = ir, + .grid = opts.grid, + .zeroed_outputs = opts.zeroed_outputs, + .output_operand_aliases = aliases.constSlice(), + }); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; + return results; + } + }; +} diff --git a/zml/kernel/cuda_tile/test.zig b/zml/kernel/cuda_tile/test.zig new file mode 100644 index 00000000..c929b0b9 --- /dev/null +++ b/zml/kernel/cuda_tile/test.zig @@ -0,0 +1,28 @@ +const std = @import("std"); + +const cuda_tile = @import("cuda_tile.zig"); + +test "cuda_tile kernel emits a module XLA can parse" { + const Cfg = struct { n: i64 }; + const AddOne = cuda_tile.Kernel(Cfg, .{ + .name = "add_one", + .inputs = &.{"x"}, + .outputs = &.{"out"}, + .run = struct { + fn run(b: *cuda_tile.Builder, cfg: Cfg) cuda_tile.FinishError!void { + const a = try b.declareArgs(.{ .x = .{ .ptr = .f32 }, .out = .{ .ptr = .f32 } }); + const vx = b.partitionView(b.tensorView(a.x, &.{cfg.n}, &.{1}), &.{128}, .{}); + const vo = b.partitionView(b.tensorView(a.out, &.{cfg.n}, &.{1}), &.{128}, .{}); + const bid = b.tileBlockId(); + _ = b.store(b.load(vx, &.{bid.x}).add(1.0), vo, &.{bid.x}); + } + }.run, + }); + + const ir = try AddOne.emit(std.testing.allocator, .{ .n = 1024 }); + defer std.testing.allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "cuda_tile.module @add_one") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "entry @add_one(") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "load_view_tko") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "store_view_tko") != null); +} diff --git a/zml/kernel/cute/BUILD.bazel b/zml/kernel/cute/BUILD.bazel new file mode 100644 index 00000000..7a613072 --- /dev/null +++ b/zml/kernel/cute/BUILD.bazel @@ -0,0 +1,9 @@ +package(default_visibility = ["//zml/kernel:__pkg__"]) + +filegroup( + name = "cute", + srcs = [ + "cute.zig", + "test.zig", + ], +) diff --git a/zml/kernel/cute/cute.zig b/zml/kernel/cute/cute.zig new file mode 100644 index 00000000..2e8c919f --- /dev/null +++ b/zml/kernel/cute/cute.zig @@ -0,0 +1,264 @@ +const std = @import("std"); + +const cute_builder = @import("dsl/cute/builder"); +pub const AlgebraExpr = cute_builder.AlgebraExpr; +pub const Builder = cute_builder.Builder; +pub const ArgSpec = cute_builder.ArgSpec; +pub const Value = cute_builder.Value; +pub const View = cute_builder.View; +pub const Atom = cute_builder.Atom; +pub const DType = cute_builder.DType; +pub const AlgebraToken = cute_builder.AlgebraToken; +pub const LayoutSpec = cute_builder.LayoutSpec; +pub const ConstrainedDynamic = cute_builder.ConstrainedDynamic; +pub const FinishError = cute_builder.FinishError; +pub const TmaConfig = cute_builder.TmaConfig; +pub const TmaFormat = cute_builder.TmaFormat; +pub const BlockScaledMmaConfig = cute_builder.BlockScaledMmaConfig; +pub const CtaGroup = cute_builder.CtaGroup; +pub const ctaCount = cute_builder.ctaCount; +pub const MBarrierScope = cute_builder.MBarrierScope; +pub const OperandMajorMode = cute_builder.OperandMajorMode; +pub const Tcgen05SmemSwizzle = cute_builder.Tcgen05SmemSwizzle; +pub const Tcgen05SmemDesc = cute_builder.Tcgen05SmemDesc; +pub const Tcgen05InstrDesc = cute_builder.Tcgen05InstrDesc; +pub const CopyBulkTensorTileG2SOp = cute_builder.CopyBulkTensorTileG2SOp; +pub const CopyBulkTensor2DGather4G2SOp = cute_builder.CopyBulkTensor2DGather4G2SOp; +pub const CopyBulkTensorTileS2GOp = cute_builder.CopyBulkTensorTileS2GOp; +const func = @import("mlir/dialects/func"); +const mlir = @import("mlir"); +const stablehlo = @import("mlir/dialects/stablehlo"); +const stdx = @import("stdx"); + +const Compiler = @import("../../Compiler.zig"); +const DataType = @import("../../dtype.zig").DataType; +const mlirx = @import("../../mlirx.zig"); +const ops = @import("../../ops.zig"); +const platform_ = @import("../../platform.zig"); +const Shape = @import("../../shape.zig").Shape; +const Tensor = @import("../../tensor.zig").Tensor; + +test { + _ = @import("test.zig"); +} + +/// Build a struct type with one field per name, all of type `FieldT`. +fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { + var name_slices: [names.len][]const u8 = undefined; + for (names, 0..) |n, i| name_slices[i] = n; + return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); +} + +fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { + mlir.registerPasses("Transforms"); + + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + + registerDialects(registry); + + func.registerExtensions(registry); + + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + + return ctx; +} + +fn resolveOutputOperandAliases( + aliases_opt: anytype, + operand_offset: i64, +) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { + var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; + + if (aliases_opt) |a| { + inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { + if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ + .output_index = @intCast(output_index), + .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, + }); + } + } + + return out; +} +pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(cute_builder.registerDialects); +} + +pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .i1, + .i8, .u8 => .i8, + .i16, .u16 => .i16, + .i32, .u32 => .i32, + .i64, .u64 => .i64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f4e2m1 => .f4e2m1fn, + .f8e4m3fn => .f8e4m3fn, + .f8e5m2 => .f8e5m2, + .f8e8m0 => .f8e8m0fnu, + else => std.debug.panic("zml.kernel.cute.from: dtype {s} has no CuTe equivalent", .{@tagName(dt)}), + }; +} + +fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; +} + +pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), +) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + /// `kernel.launch(grid=, block=)` in the Python DSL; dynamic shared + /// memory is what the kernel declares. + pub const CallOpts = struct { + cfg: ConfigT, + grid: [3]i32, + block: [3]i32, + zeroed_outputs: []const i32 = &.{}, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT, block: [3]i32) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try cute_builder.Builder.open(allocator, ctx, name); + defer b.deinit(); + + try spec.run(&b, cfg); + + // XLA passes one pointer per input then per output; it does + // not check the count against the kernel. + const expected = spec.inputs.len + spec.outputs.len; + if (b.args.len != expected) { + std.debug.panic("zml.kernel.cute.Kernel({s}): declareArgs declared {d} arguments, inputs + outputs are {d}", .{ name, b.args.len, expected }); + } + + return b.finish(block); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + + const ir = emit(cur.allocator, opts.cfg, opts.block) catch |err| + std.debug.panic("zml.kernel.cute.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); + + const tensor_results = ops.cute(inputs_arr, outputs_arr, .{ + .name = name, + .ir = ir, + .grid = opts.grid, + .block = opts.block, + .zeroed_outputs = opts.zeroed_outputs, + .output_operand_aliases = aliases.constSlice(), + }); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; + return results; + } + }; +} + +fn ProgramSpec(comptime Config: type) type { + return struct { + /// Public host entry selected by XLA. The callback may give the + /// nested CUDA kernel a different (private) symbol. + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; +} + +/// A complete CuTe program: one or more `cuda.kernel` operations inside +/// a `gpu.module`, followed by a public `func.func` which constructs the +/// launch-time CuTe objects and calls `cuda.launch_ex`. +/// +/// This is the source-level equivalent of invoking a Python `@cute.jit` +/// object whose `__call__` builds TMA descriptors and launches an +/// `@cute.kernel`. XLA executes the generated host function on its CUDA +/// stream, so the custom call deliberately carries no external grid or +/// block dimensions. +pub fn Program( + comptime ConfigT: type, + comptime spec: ProgramSpec(ConfigT), +) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + pub const CallOpts = struct { + cfg: ConfigT, + zeroed_outputs: []const i32 = &.{}, + scalars: []const i64 = &.{}, + symmetric_workspace: ?ops.CuteOps.SymmetricWorkspace = null, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try cute_builder.Builder.openProgram(allocator, ctx, name); + defer b.deinit(); + try spec.run(&b, cfg); + return b.finishProgram(); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + const ir = emit(cur.allocator, opts.cfg) catch |err| + std.debug.panic("zml.kernel.cute.Program({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); + const tensor_results = ops.cute(inputs_arr, outputs_arr, .{ + .name = name, + .ir = ir, + .zeroed_outputs = opts.zeroed_outputs, + .scalars = opts.scalars, + .symmetric_workspace = opts.symmetric_workspace, + .output_operand_aliases = aliases.constSlice(), + }); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; + return results; + } + }; +} diff --git a/zml/kernel/cute/test.zig b/zml/kernel/cute/test.zig new file mode 100644 index 00000000..f4d457a3 --- /dev/null +++ b/zml/kernel/cute/test.zig @@ -0,0 +1,34 @@ +const std = @import("std"); + +const cute = @import("cute.zig"); + +test "cute kernel emits the module the CuTe compiler takes" { + const Cfg = struct { n: i64 }; + const AddOne = cute.Kernel(Cfg, .{ + .name = "add_one", + .inputs = &.{"x"}, + .outputs = &.{"out"}, + .run = struct { + fn run(b: *cute.Builder, cfg: Cfg) cute.FinishError!void { + const a = try b.declareArgs(.{ + .x = .{ .tensor = .{ .dtype = .f32, .shape = &.{cfg.n} } }, + .out = .{ .tensor = .{ .dtype = .f32, .shape = &.{cfg.n} } }, + }); + const i = b.blockIdx().x.mul(b.blockDim().x).add(b.threadIdx().x); + var guard = b.openIf(i.lt(cfg.n)); + { + a.out.set(.{i}, a.x.get(.{i}).add(1.0)); + guard.close(); + } + } + }.run, + }); + + const ir = try AddOne.emit(std.testing.allocator, .{ .n = 1000 }, .{ 128, 1, 1 }); + defer std.testing.allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "\"func.func\"() <{function_type = (") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "sym_name = \"add_one\"") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "nvvm.reqntid = array") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "cute.memref.load") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "gpu.module") == null); +} diff --git a/zml/kernel/fly/BUILD.bazel b/zml/kernel/fly/BUILD.bazel new file mode 100644 index 00000000..57203521 --- /dev/null +++ b/zml/kernel/fly/BUILD.bazel @@ -0,0 +1,9 @@ +package(default_visibility = ["//zml/kernel:__pkg__"]) + +filegroup( + name = "fly", + srcs = [ + "fly.zig", + "test.zig", + ], +) diff --git a/zml/kernel/fly/fly.zig b/zml/kernel/fly/fly.zig new file mode 100644 index 00000000..b7d6c943 --- /dev/null +++ b/zml/kernel/fly/fly.zig @@ -0,0 +1,248 @@ +const std = @import("std"); + +const fly_builder = @import("dsl/fly/builder"); +pub const Builder = fly_builder.Builder; +pub const Value = fly_builder.Value; +pub const DType = fly_builder.DType; +pub const FinishError = fly_builder.FinishError; +pub const TiledCopy = fly_builder.TiledCopy; +pub const TiledMma = fly_builder.TiledMma; +pub const Arch = fly_builder.Arch; +pub const MmaFlavor = fly_builder.MmaFlavor; +pub const fly_rocdl = fly_builder.fly_rocdl; +pub const layout = fly_builder.layout; +pub const Layout = fly_builder.Layout; +pub const Tile = fly_builder.Tile; +pub const IntTuple = fly_builder.IntTuple; +pub const L = fly_builder.L; +pub const rowMajor = fly_builder.rowMajor; +pub const colMajor = fly_builder.colMajor; +pub const ordered = fly_builder.ordered; +pub const tile = fly_builder.tile; +pub const it = fly_builder.it; +const func = @import("mlir/dialects/func"); +const mlir = @import("mlir"); +const stablehlo = @import("mlir/dialects/stablehlo"); +const stdx = @import("stdx"); + +const Compiler = @import("../../Compiler.zig"); +const DataType = @import("../../dtype.zig").DataType; +const mlirx = @import("../../mlirx.zig"); +const ops = @import("../../ops.zig"); +const platform_ = @import("../../platform.zig"); +const Shape = @import("../../shape.zig").Shape; +const Tensor = @import("../../tensor.zig").Tensor; + +test { + _ = @import("test.zig"); +} + +/// Build a struct type with one field per name, all of type `FieldT`. +fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { + var name_slices: [names.len][]const u8 = undefined; + for (names, 0..) |n, i| name_slices[i] = n; + return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); +} + +fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { + mlir.registerPasses("Transforms"); + + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + + registerDialects(registry); + + func.registerExtensions(registry); + + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + + return ctx; +} + +fn resolveOutputOperandAliases( + aliases_opt: anytype, + operand_offset: i64, +) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { + var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; + + if (aliases_opt) |a| { + inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { + if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ + .output_index = @intCast(output_index), + .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, + }); + } + } + + return out; +} + +/// Threads per wavefront: 64 on CDNA, 32 on RDNA. +pub fn waveSize(p: *const platform_.Platform) i32 { + const cc = platform_.rocm.computeCapability(p) orelse return 64; + return switch (cc.architecture()) { + .cdna1, .cdna2, .cdna3, .cdna4 => 64, + .rdna2, .rdna3, .rdna3_5, .rdna4 => 32, + }; +} + +/// The plugin sizes a block in wavefronts, so a thread count has to divide +/// evenly: a kernel's layouts are built for an exact number of threads and +/// spare ones would partition out of range. +fn warpsFor(p: *const platform_.Platform, threads: i32) i32 { + const wave = waveSize(p); + if (@rem(threads, wave) != 0) { + std.debug.panic("zml.kernel.fly: {d} threads is not a multiple of the {d}-wide wavefront", .{ threads, wave }); + } + return @divExact(threads, wave); +} + +/// The matrix atom this device provides, or null when it has none: WMMA +/// arrived with RDNA3, so gfx1030 has no matrix instruction at all. +pub fn mmaFlavor(p: *const platform_.Platform) ?MmaFlavor { + const cc = platform_.rocm.computeCapability(p) orelse return .cdna3_mfma; + return switch (cc.architecture()) { + .cdna1, .cdna2, .cdna3, .cdna4 => .cdna3_mfma, + .rdna3, .rdna3_5 => .gfx11_wmma, + .rdna4 => .gfx120x_wmma, + .rdna2 => null, + }; +} +pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(fly_builder.registerDialects); +} + +pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .i1, + .i8, .u8 => .i8, + .i16, .u16 => .i16, + .i32, .u32 => .i32, + .i64, .u64 => .i64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f8e4m3fn => .f8e4m3fn, + .f8e4m3fnuz => .f8e4m3fnuz, + .f8e5m2 => .f8e5m2, + .f8e5m2fnuz => .f8e5m2fnuz, + else => std.debug.panic("zml.kernel.fly.from: dtype {s} has no Fly equivalent", .{@tagName(dt)}), + }; +} + +fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; +} + +pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), +) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + pub const arg_names = spec.inputs ++ spec.outputs; + /// By name, as `!fly.memref` views. + pub const Args = StructOf(arg_names, Value); + + pub const CallOpts = struct { + cfg: ConfigT, + grid: [3]i32, + /// Threads per block. Converted to the wavefronts the plugin + /// wants using the device's wave size, which is 64 on CDNA and + /// 32 on RDNA, so a kernel states the thread count its layouts + /// were built for and runs on both. + threads: i32, + /// > 0 sets `amdgpu-waves-per-eu = "N, N"`: a hard occupancy clamp. + waves_per_eu: i32 = 0, + /// Required when the body uses `fly.get_dyn_shared`. + shared_mem_bytes: i64 = 0, + /// Inputs first, then outputs; see `zeroedOutput`. A no-op + /// inside command buffers. + zeroed_args: []const i32 = &.{}, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + }; + + /// The arguments `run` was declared with. + pub fn args(b: *Builder) Args { + var out: Args = undefined; + inline for (arg_names, 0..) |fname, i| @field(out, fname) = b.arg(i); + return out; + } + + /// `zeroed_args` index of an output. + pub fn zeroedOutput(comptime f: std.meta.FieldEnum(Outputs)) i32 { + return @intCast(spec.inputs.len + @backingInt(f)); + } + + /// `shapes` is inputs, then outputs. + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT, shapes: []const Shape) ![:0]const u8 { + std.debug.assert(shapes.len == arg_names.len); + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try fly_builder.Builder.open(allocator, ctx, name); + defer b.deinit(); + + var specs: [arg_names.len]Builder.ArgSpec = undefined; + inline for (arg_names, 0..) |fname, i| { + const shape = shapes[i]; + specs[i] = .{ + .name = fname, + .dtype = from(shape.dtype()), + .dims = if (shape.rank() > 0) shape.dims() else null, + }; + } + try b.declareArgsLow(&specs); + + try spec.run(&b, cfg); + + return b.finish(); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + var shapes: [arg_names.len]Shape = undefined; + inline for (0..spec.inputs.len) |i| shapes[i] = inputs_arr[i].shape(); + inline for (0..spec.outputs.len) |i| shapes[spec.inputs.len + i] = outputs_arr[i]; + + const ir = emit(cur.allocator, opts.cfg, &shapes) catch |err| + std.debug.panic("zml.kernel.fly.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); + + const tensor_results = ops.fly(inputs_arr, outputs_arr, .{ + .name = name, + .ir = ir, + .grid = opts.grid, + .num_warps = warpsFor(cur.platform, opts.threads), + .waves_per_eu = opts.waves_per_eu, + .shared_mem_bytes = opts.shared_mem_bytes, + .zeroed_args = opts.zeroed_args, + .output_operand_aliases = aliases.constSlice(), + }); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; + return results; + } + }; +} diff --git a/zml/kernel/fly/test.zig b/zml/kernel/fly/test.zig new file mode 100644 index 00000000..ae788d96 --- /dev/null +++ b/zml/kernel/fly/test.zig @@ -0,0 +1,149 @@ +const std = @import("std"); + +const fly_builder = @import("dsl/fly/builder"); + +const fly = @import("fly.zig"); +const zml = @import("../../zml.zig"); + +/// C = A + B over (M, N) f32, 128-bit vectorized and predicated on the +/// ragged edge. +const FlyVectorAdd = struct { + const Cfg = struct { + thr: fly.Layout = fly.ordered(.{ 8, 16 }, .{ 1, 0 }), + val: fly.Layout = fly.ordered(.{ 1, 4 }, .{ 0, 1 }), + }; + + const K = fly.Kernel(Cfg, .{ + .name = "vector_add", + .inputs = &.{ "a", "b" }, + .outputs = &.{"c"}, + .run = run, + }); + + fn run(b: *fly.Builder, cfg: Cfg) fly.FinishError!void { + const a = K.args(b); + fly_builder.emitVectorAdd(b, cfg.thr, cfg.val, a.a, a.b, a.c); + } +}; + +/// C = A @ B^T with one MFMA-tiled block. +const FlyTiledMma = struct { + const Cfg = struct { flavor: fly.MmaFlavor }; + const K = fly.Kernel(Cfg, .{ + .name = "tiled_mma", + .inputs = &.{ "a", "b" }, + .outputs = &.{"c"}, + .run = run, + }); + + fn run(b: *fly.Builder, cfg: Cfg) fly.FinishError!void { + const a = K.args(b); + fly_builder.emitTiledMma(b, cfg.flavor, a.a, a.b, a.c); + } +}; + +test "fly kernel emits a module XLA can parse" { + const shapes = [_]zml.Shape{ .init(.{ 100, 1000 }, .f32), .init(.{ 100, 1000 }, .f32), .init(.{ 100, 1000 }, .f32) }; + const ir = try FlyVectorAdd.K.emit(std.testing.allocator, .{}, &shapes); + defer std.testing.allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "gpu.func @vector_add(%arg0: !fly.ptr, %arg1: !fly.ptr, %arg2: !fly.ptr) kernel") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "fly.static : !fly.tile<[8|64]>") != null); +} + +test "fly kernels run on rocm" { + const platform = zml.testing.env(); + if (platform.target != .rocm) return error.SkipZigTest; + + const allocator = std.testing.allocator; + + // Compile `forward`, upload both inputs, run, and bring the result back. + const call = struct { + fn f(comptime forward: anytype, p: *const zml.Platform, ta: zml.Tensor, ha: anytype, tb: zml.Tensor, hb: anytype) !zml.Slice { + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ ta, tb }, p, .{}); + defer exe.deinit(); + var ba: zml.Buffer = try .fromBytes(std.testing.io, p, ta.shape(), std.mem.sliceAsBytes(ha)); + defer ba.deinit(); + var bb: zml.Buffer = try .fromBytes(std.testing.io, p, tb.shape(), std.mem.sliceAsBytes(hb)); + defer bb.deinit(); + var out = try exe.eval(std.testing.allocator, std.testing.io, .{ ba, bb }); + defer out.deinit(); + return out.toSliceAlloc(std.testing.allocator, std.testing.io); + } + }.f; + + // vector add: elementwise over a ragged (100, 1000), predicated at the edge. + { + const M, const N = .{ 100, 1000 }; + const Mod = struct { + pub fn forward(a: zml.Tensor, b: zml.Tensor) zml.Tensor { + return FlyVectorAdd.K.call(.{ .a = a, .b = b }, .{ .c = a.shape() }, .{ + .cfg = .{}, + .grid = .{ (M + 8 - 1) / 8, (N + 64 - 1) / 64, 1 }, + .threads = 8 * 16, + }).c; + } + }; + + const ha = try allocator.alloc(f32, M * N); + defer allocator.free(ha); + const hb = try allocator.alloc(f32, M * N); + defer allocator.free(hb); + for (ha, hb, 0..) |*va, *vb, i| { + va.* = @floatFromInt(i % 977); + vb.* = @floatFromInt(1000 + (i % 313)); + } + + var host = try call(Mod.forward, platform, .init(.{ M, N }, .f32), ha, .init(.{ M, N }, .f32), hb); + defer host.free(allocator); + for (host.items(f32), 0..) |v, i| try std.testing.expectEqual(ha[i] + hb[i], v); + } + + // tiled mma: C = A @ B^T in one block of 256 threads, over whatever matrix + // atom this device has. CDNA multiplies f32 through MFMA; RDNA3+ has WMMA, + // whose verifier rejects f32 operands, so the operand type and K come from + // the flavour rather than being written into the test. + if (fly.mmaFlavor(platform)) |flavor| switch (flavor) { + inline else => |f| { + const Elem = switch (comptime f.operandDType()) { + .f32 => f32, + .f16 => f16, + else => @compileError("unhandled MMA operand type"), + }; + const M, const N = .{ 64, 64 }; + const Kd = comptime f.blockK(); + const operand_dtype: zml.DataType = switch (comptime f.operandDType()) { + .f32 => .f32, + .f16 => .f16, + else => unreachable, + }; + + const Mod = struct { + pub fn forward(a: zml.Tensor, b: zml.Tensor) zml.Tensor { + return FlyTiledMma.K.call(.{ .a = a, .b = b }, .{ .c = zml.Shape.init(.{ M, N }, .f32) }, .{ + .cfg = .{ .flavor = f }, + .grid = .{ 1, 1, 1 }, + .threads = comptime f.blockThreads(), + }).c; + } + }; + + var ha: [M * Kd]Elem = undefined; + var hb: [N * Kd]Elem = undefined; + for (&ha, 0..) |*v, i| v.* = @as(Elem, @floatFromInt((i * 7) % 13)) - 6; + for (&hb, 0..) |*v, i| v.* = @as(Elem, @floatFromInt((i * 5) % 11)) - 5; + + var host = try call(Mod.forward, platform, .init(.{ M, Kd }, operand_dtype), &ha, .init(.{ N, Kd }, operand_dtype), &hb); + defer host.free(allocator); + + // f16 operands accumulate in f32, so the error is set by the input + // rounding rather than the sum. + const tol: f32 = if (Elem == f16) 1e-2 else 1e-3; + const out = host.items(f32); + for (0..M) |m| for (0..N) |n| { + var acc: f32 = 0; + for (0..Kd) |k| acc += @as(f32, ha[m * Kd + k]) * @as(f32, hb[n * Kd + k]); + try std.testing.expectApproxEqAbs(acc, out[m * N + n], tol); + }; + }, + }; +} diff --git a/zml/kernel/kernel.zig b/zml/kernel/kernel.zig new file mode 100644 index 00000000..204c6204 --- /dev/null +++ b/zml/kernel/kernel.zig @@ -0,0 +1,12 @@ +pub const cuda_tile = @import("cuda_tile/cuda_tile.zig"); +pub const cute = @import("cute/cute.zig"); +pub const fly = @import("fly/fly.zig"); +pub const metal = @import("metal/metal.zig"); +pub const mosaic_tpu = @import("mosaic_tpu/mosaic_tpu.zig"); +pub const tcl = @import("tcl/tcl.zig"); +pub const triton = @import("triton/triton.zig"); + +test { + const std = @import("std"); + std.testing.refAllDecls(@This()); +} diff --git a/zml/kernel/metal/BUILD.bazel b/zml/kernel/metal/BUILD.bazel new file mode 100644 index 00000000..4007beb4 --- /dev/null +++ b/zml/kernel/metal/BUILD.bazel @@ -0,0 +1,9 @@ +package(default_visibility = ["//zml/kernel:__pkg__"]) + +filegroup( + name = "metal", + srcs = [ + "metal.zig", + "test.zig", + ], +) diff --git a/zml/kernel/metal/metal.zig b/zml/kernel/metal/metal.zig new file mode 100644 index 00000000..9626a1de --- /dev/null +++ b/zml/kernel/metal/metal.zig @@ -0,0 +1,237 @@ +const std = @import("std"); + +const func = @import("mlir/dialects/func"); +const metal_builder = @import("dsl/metal/builder"); +pub const Builder = metal_builder.Builder; +pub const Value = metal_builder.Value; +pub const Type = metal_builder.Type; +pub const Struct = metal_builder.Struct; +pub const ArgSpec = metal_builder.ArgSpec; +pub const LoopOpts = metal_builder.LoopOpts; +pub const FinishError = metal_builder.FinishError; +const mlir = @import("mlir"); +const stablehlo = @import("mlir/dialects/stablehlo"); +const stdx = @import("stdx"); + +const Compiler = @import("../../Compiler.zig"); +const DataType = @import("../../dtype.zig").DataType; +const mlirx = @import("../../mlirx.zig"); +const ops = @import("../../ops.zig"); +const platform_ = @import("../../platform.zig"); +const Shape = @import("../../shape.zig").Shape; +const Tensor = @import("../../tensor.zig").Tensor; + +test { + _ = @import("test.zig"); +} + +/// Build a struct type with one field per name, all of type `FieldT`. +fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { + var name_slices: [names.len][]const u8 = undefined; + for (names, 0..) |n, i| name_slices[i] = n; + return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); +} + +fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { + mlir.registerPasses("Transforms"); + + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + + registerDialects(registry); + + func.registerExtensions(registry); + + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + + return ctx; +} + +fn resolveOutputOperandAliases( + aliases_opt: anytype, + operand_offset: i64, +) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { + var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; + + if (aliases_opt) |a| { + inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { + if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ + .output_index = @intCast(output_index), + .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, + }); + } + } + + return out; +} + +pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(metal_builder.registerDialects); +} + +pub fn from(dt: DataType) Type { + return switch (dt) { + .bool => .bool, + .i8 => .char, + .i16 => .short, + .i32 => .int, + .i64 => .long, + .u8 => .uchar, + .u16 => .ushort, + .u32 => .uint, + .u64 => .ulong, + .f16 => .half, + .bf16 => .bfloat, + .f32 => .float, + else => std.debug.panic("zml.kernel.metal.from: dtype {s} has no MSL equivalent", .{@tagName(dt)}), + }; +} + +/// The bytes of a `constant T &` argument: `.of(@as(i32, n))`, or a host +/// `extern struct` laid out like a `constant S &` argument. +pub const Constant = struct { + buffer: [256]u8 = undefined, + len: usize = 0, + + pub fn of(value: anytype) Constant { + const raw = std.mem.asBytes(&value); + var c: Constant = .{ .len = raw.len }; + @memcpy(c.buffer[0..raw.len], raw); + return c; + } + + pub fn bytes(self: *const Constant) []const u8 { + return self.buffer[0..self.len]; + } +}; + +fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + /// The arguments bound to input tensors. + inputs: []const [:0]const u8, + /// The arguments bound to output tensors; one named like an + /// input updates it in place. + outputs: []const [:0]const u8, + /// The `constant` arguments given by value at the call. + constants: []const [:0]const u8 = &.{}, + run: *const fn (*Builder, Config) FinishError!void, + fast_math: bool = true, + }; +} + +/// A Metal kernel: `run` declares its arguments with `declareArgs`, by +/// name, and builds its body. +pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), +) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + pub const Constants = StructOf(spec.constants, Constant); + + pub const CallOpts = struct { + cfg: ConfigT, + /// Threadgroups per grid. + grid: [3]u32, + /// Threads per threadgroup. + threadgroup: [3]u32, + /// Bytes of `[[threadgroup(0)]]` memory. + threadgroup_memory: u64 = 0, + constants: Constants = std.mem.zeroes(Constants), + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + var arguments: std.ArrayList(metal_builder.Argument) = .empty; + defer arguments.deinit(allocator); + return build(allocator, cfg, &arguments); + } + + fn build(allocator: std.mem.Allocator, cfg: ConfigT, arguments: *std.ArrayList(metal_builder.Argument)) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + var b = Builder.open(allocator, ctx, name, .{ .fast_math = spec.fast_math }); + defer b.deinit(); + try spec.run(&b, cfg); + const ir = try b.finish(); + errdefer allocator.free(ir); + try arguments.appendSlice(allocator, b.arguments.items); + return ir; + } + + fn indexOf(names: []const [:0]const u8, arg: []const u8) ?usize { + for (names, 0..) |n, i| if (std.mem.eql(u8, n, arg)) return i; + return null; + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + const allocator = cur.allocator; + + var arguments: std.ArrayList(metal_builder.Argument) = .empty; + defer arguments.deinit(allocator); + const ir = build(allocator, opts.cfg, &arguments) catch |err| + std.debug.panic("zml.kernel.metal.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer allocator.free(ir); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |field, i| inputs_arr[i] = @field(inputs, field); + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |field, i| outputs_arr[i] = @field(outputs, field); + var constants: [spec.constants.len][]const u8 = undefined; + inline for (spec.constants, 0..) |field, i| constants[i] = @field(opts.constants, field).bytes(); + + var bound: [spec.outputs.len]bool = @splat(false); + var bound_inputs: [spec.inputs.len]bool = @splat(false); + var bound_constants: [spec.constants.len]bool = @splat(false); + var aliases: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, spec.outputs.len) = .empty; + var bindings: std.ArrayList(i32) = .empty; + defer bindings.deinit(allocator); + for (arguments.items) |arg| { + if (arg.kind == .threadgroup) { + // `threadgroup_memory` sizes `[[threadgroup(0)]]` alone. + if (arg.index != 0) std.debug.panic("zml.kernel.metal.Kernel({s}): threadgroup argument {s} is [[threadgroup({d})]], and only [[threadgroup(0)]] gets memory", .{ name, arg.name, arg.index }); + continue; + } + std.debug.assert(arg.index == bindings.items.len); + const input = indexOf(spec.inputs, arg.name); + const output = indexOf(spec.outputs, arg.name); + if (input) |i| bound_inputs[i] = true; + if (output) |o| bound[o] = true; + const binding: i32 = if (input) |i| blk: { + if (output) |o| aliases.appendAssumeCapacity(.{ .output_index = @intCast(o), .operand_index = @intCast(i) }); + break :blk @intCast(i); + } else if (output) |o| + @intCast(spec.inputs.len + o) + else if (indexOf(spec.constants, arg.name)) |k| blk: { + if (constants[k].len != arg.size) std.debug.panic("zml.kernel.metal.Kernel({s}): constant {s} has {d} bytes, the argument {d}", .{ name, arg.name, constants[k].len, arg.size }); + bound_constants[k] = true; + break :blk -1 - @as(i32, @intCast(k)); + } else std.debug.panic("zml.kernel.metal.Kernel({s}): argument {s} is no input, output or constant", .{ name, arg.name }); + bindings.append(allocator, binding) catch @panic("OOM"); + } + for (bound, spec.outputs) |b, output| if (!b) std.debug.panic("zml.kernel.metal.Kernel({s}): output {s} is no argument", .{ name, output }); + for (bound_inputs, spec.inputs) |b, input| if (!b) std.debug.panic("zml.kernel.metal.Kernel({s}): input {s} is no argument", .{ name, input }); + for (bound_constants, spec.constants) |b, constant| if (!b) std.debug.panic("zml.kernel.metal.Kernel({s}): constant {s} is no argument", .{ name, constant }); + + const tensor_results = ops.metal(inputs_arr, outputs_arr, .{ + .name = name, + .ir = ir, + .grid = opts.grid, + .threadgroup = opts.threadgroup, + .threadgroup_memory = opts.threadgroup_memory, + .arguments = bindings.items, + .constants = &constants, + .output_operand_aliases = aliases.constSlice(), + }); + var results: Results = undefined; + inline for (spec.outputs, 0..) |field, i| @field(results, field) = tensor_results[i]; + return results; + } + }; +} diff --git a/zml/kernel/metal/test.zig b/zml/kernel/metal/test.zig new file mode 100644 index 00000000..0a342af0 --- /dev/null +++ b/zml/kernel/metal/test.zig @@ -0,0 +1,71 @@ +const std = @import("std"); + +const metal = @import("metal.zig"); +const zml = @import("../../zml.zig"); + +/// `out[i] = a[i] * scale + b[i]` over `n` floats. +const MetalAxpy = struct { + const Cfg = struct {}; + const K = metal.Kernel(Cfg, .{ + .name = "axpy", + .inputs = &.{ "a", "b" }, + .outputs = &.{"out"}, + .constants = &.{ "scale", "n" }, + .run = run, + }); + + fn run(b: *metal.Builder, _: Cfg) metal.FinishError!void { + const a = try b.declareArgs(.{ + .a = .{ .buffer = .{ .type = .float, .const_ = true } }, + .b = .{ .buffer = .{ .type = .float, .const_ = true } }, + .out = .{ .buffer = .{ .type = .float } }, + .scale = .{ .constant = .float }, + .n = .{ .constant = .uint }, + .tid = .{ .thread_position_in_grid = .uint }, + }); + b.returnIf(a.tid.ge(a.n)); + a.out.store(a.tid, b.fma(a.a.load(a.tid), a.scale, a.b.load(a.tid))); + } +}; + +test "metal kernels run on metal" { + const allocator = std.testing.allocator; + + const ir = try MetalAxpy.K.emit(allocator, .{}); + defer allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "air.fma") != null); + + const platform = zml.testing.env(); + if (platform.target != .metal) return error.SkipZigTest; + + const n = 1000; + const Mod = struct { + pub fn forward(x: zml.Tensor, y: zml.Tensor) zml.Tensor { + return MetalAxpy.K.call(.{ .a = x, .b = y }, .{ .out = x.shape() }, .{ + .cfg = .{}, + .grid = .{ (n + 255) / 256, 1, 1 }, + .threadgroup = .{ 256, 1, 1 }, + .constants = .{ .scale = .of(@as(f32, 0.5)), .n = .of(@as(u32, n)) }, + }).out; + } + }; + const x: zml.Tensor = .init(.{n}, .f32); + var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{ x, zml.Tensor.init(.{n}, .f32) }, platform, .{}); + defer exe.deinit(); + + var hx: [n]f32 = undefined; + var hy: [n]f32 = undefined; + for (&hx, &hy, 0..) |*a, *b, i| { + a.* = @floatFromInt(i); + b.* = 1; + } + var bx: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&hx)); + defer bx.deinit(); + var by: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&hy)); + defer by.deinit(); + var result = try exe.eval(allocator, std.testing.io, .{ bx, by }); + defer result.deinit(); + var host = try result.toSliceAlloc(allocator, std.testing.io); + defer host.free(allocator); + for (host.items(f32), hx, hy) |out, a, b| try std.testing.expectEqual(a * 0.5 + b, out); +} diff --git a/zml/kernel/mosaic_tpu/BUILD.bazel b/zml/kernel/mosaic_tpu/BUILD.bazel new file mode 100644 index 00000000..ba3b6b66 --- /dev/null +++ b/zml/kernel/mosaic_tpu/BUILD.bazel @@ -0,0 +1,6 @@ +package(default_visibility = ["//zml/kernel:__pkg__"]) + +filegroup( + name = "mosaic_tpu", + srcs = ["mosaic_tpu.zig"], +) diff --git a/zml/kernel/mosaic_tpu/mosaic_tpu.zig b/zml/kernel/mosaic_tpu/mosaic_tpu.zig new file mode 100644 index 00000000..a09e8368 --- /dev/null +++ b/zml/kernel/mosaic_tpu/mosaic_tpu.zig @@ -0,0 +1,284 @@ +const std = @import("std"); + +const func = @import("mlir/dialects/func"); +const mlir = @import("mlir"); +const mosaic_tpu_builder = @import("dsl/mosaic_tpu/builder"); +const Builder = mosaic_tpu_builder.Builder; +const DType = mosaic_tpu_builder.DType; +const FinishError = mosaic_tpu_builder.FinishError; +const mosaic_tpu_dialect = @import("mlir/dialects/mosaic_tpu"); +const stablehlo = @import("mlir/dialects/stablehlo"); +const stdx = @import("stdx"); + +const Compiler = @import("../../Compiler.zig"); +const DataType = @import("../../dtype.zig").DataType; +const mlirx = @import("../../mlirx.zig"); +const ops = @import("../../ops.zig"); +const platform_ = @import("../../platform.zig"); +const Shape = @import("../../shape.zig").Shape; +const Tensor = @import("../../tensor.zig").Tensor; + +/// Build a struct type with one field per name, all of type `FieldT`. +fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { + var name_slices: [names.len][]const u8 = undefined; + for (names, 0..) |n, i| name_slices[i] = n; + return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); +} + +fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { + mlir.registerPasses("Transforms"); + + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + + registerDialects(registry); + + func.registerExtensions(registry); + + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + + return ctx; +} + +fn resolveOutputOperandAliases( + aliases_opt: anytype, + operand_offset: i64, +) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { + var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; + + if (aliases_opt) |a| { + inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { + if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ + .output_index = @intCast(output_index), + .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, + }); + } + } + + return out; +} + +pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(mosaic_tpu_builder.registerDialects); +} + +pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .i1, + .i8, .u8 => .i8, + .i16, .u16 => .i16, + .i32, .u32 => .i32, + .i64, .u64 => .i64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f8e4m3fn => .f8e4m3fn, + .f8e5m2 => .f8e5m2, + else => std.debug.panic("zml.kernel.mosaic_tpu.from: dtype {s} has no Mosaic-TPU equivalent", .{@tagName(dt)}), + }; +} + +fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; +} + +pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), +) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + pub const CallExtras = struct { + vmem_limit_bytes: ?i64 = null, + disable_bounds_checks: ?bool = null, + disable_semaphore_checks: ?bool = null, + has_communication: ?bool = null, + additional_attributes: []const mlir.NamedAttribute = &.{}, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + dynamic_grid_bounds: []const Tensor = &.{}, + }; + + pub const CallOpts = struct { + cfg: ConfigT, + extras: CallExtras = .{}, + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try mosaic_tpu_builder.Builder.open(allocator, ctx, name); + defer b.deinit(); + + try spec.run(&b, cfg); + + return b.finishOpts(&.{}, .{ .canonicalize = true }); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + + const ir = emit(cur.allocator, opts.cfg) catch |err| + std.debug.panic("zml.kernel.mosaic_tpu.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + const backend_config = buildBackendConfig(cur.allocator, ir, .{ + .vmem_limit_bytes = opts.extras.vmem_limit_bytes, + .disable_bounds_checks = opts.extras.disable_bounds_checks, + .disable_semaphore_checks = opts.extras.disable_semaphore_checks, + .has_communication = opts.extras.has_communication, + }) catch |err| + std.debug.panic("zml.kernel.mosaic_tpu.Kernel({s}).call: backend_config build failed: {}", .{ name, err }); + defer cur.allocator.free(backend_config); + + var values: [spec.inputs.len]*const mlir.Value = undefined; + inline for (spec.inputs, 0..) |fname, i| values[i] = @field(inputs, fname).value(); + + var res_types: [spec.outputs.len]*const mlir.Type = undefined; + inline for (&res_types, spec.outputs) |*r, fname| { + r.* = mlirx.Type.rankedTensor(cur.mlir_ctx, @field(outputs, fname)); + } + + const aliases = resolveOutputOperandAliases(opts.extras.output_operand_aliases, @intCast(opts.extras.dynamic_grid_bounds.len)); + + const op = callTpuCustomCall(.{ + .inputs = &values, + .result_types = &res_types, + .backend_config = backend_config, + .aliases = aliases.constSlice(), + .dynamic_grid_bounds = opts.extras.dynamic_grid_bounds, + .additional_attributes = opts.extras.additional_attributes, + }); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| { + const out = @field(outputs, fname); + @field(results, fname) = Tensor._result(out, op.result(i)); + } + return results; + } + }; +} + +const CustomCallConfig = struct { + body: []const u8, + has_communication: ?bool = null, + serialization_format: i64 = 1, + needs_layout_passes: bool = true, + shape_invariant_numerics: bool = false, + disable_bounds_checks: ?bool = null, + disable_semaphore_checks: ?bool = null, +}; + +const ScopedMemoryConfig = struct { + memory_space: i64 = 1, + offset: i64 = 0, + size: i64, +}; + +const BackendConfig = struct { + custom_call_config: CustomCallConfig, + scoped_memory_configs: ?[]const ScopedMemoryConfig = null, +}; + +const BackendConfigOpts = struct { + vmem_limit_bytes: ?i64 = null, + disable_bounds_checks: ?bool = null, + disable_semaphore_checks: ?bool = null, + has_communication: ?bool = null, +}; + +fn buildBackendConfig( + allocator: std.mem.Allocator, + ir: [:0]const u8, + opts: BackendConfigOpts, +) ![]u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + const module = try mlir.Module.parse(ctx, ir); + defer module.deinit(); + + mosaic_tpu_dialect.registerMosaicSerdePass(); + const prev_allow_unregistered = ctx.allowUnregisteredDialects(); + ctx.setAllowUnregisteredDialects(true); + defer ctx.setAllowUnregisteredDialects(prev_allow_unregistered); + + const pm = mlir.PassManager.init(ctx); + defer pm.deinit(); + try pm.asOpPassManager().addPipeline("mosaic-serde{serialize=true}"); + try pm.runOnOp(module.operation()); + + const Encoder = std.base64.standard.Encoder; + const b64 = b64: { + var bytecode: std.Io.Writer.Allocating = .init(allocator); + defer bytecode.deinit(); + try module.operation().writeBytecode(.{ .desired_emit_version = 0 }, &bytecode.writer); + const buf = try allocator.alloc(u8, Encoder.calcSize(bytecode.written().len)); + break :b64 Encoder.encode(buf, bytecode.written()); + }; + defer allocator.free(b64); + + const config = BackendConfig{ + .custom_call_config = .{ + .body = b64, + .has_communication = opts.has_communication, + .disable_bounds_checks = opts.disable_bounds_checks, + .disable_semaphore_checks = opts.disable_semaphore_checks, + }, + .scoped_memory_configs = if (opts.vmem_limit_bytes) |vmem| + &.{.{ .size = vmem }} + else + null, + }; + + var json: std.Io.Writer.Allocating = .init(allocator); + errdefer json.deinit(); + + try json.writer.print("{f}", .{std.json.fmt(config, .{ .emit_null_optional_fields = false })}); + return try json.toOwnedSlice(); +} + +const TpuCustomCallArgs = struct { + inputs: []const *const mlir.Value, + result_types: []const *const mlir.Type, + backend_config: []const u8, + aliases: []const stablehlo.CustomCallOpts.OutputOperandAlias, + dynamic_grid_bounds: []const Tensor, + additional_attributes: []const mlir.NamedAttribute, +}; + +fn callTpuCustomCall(args: TpuCustomCallArgs) *mlir.Operation { + const cur = Compiler.current(); + + var all_inputs: stdx.BoundedArray(*const mlir.Value, stablehlo.CustomCallOpts.MAX_OPERANDS) = .empty; + for (args.dynamic_grid_bounds) |t| all_inputs.appendAssumeCapacity(t.value()); + all_inputs.appendSliceAssumeCapacity(args.inputs); + + return stablehlo.custom_call( + cur.mlir_ctx, + all_inputs.constSlice(), + args.result_types, + .{ + .call_target_name = "tpu_custom_call", + .backend_config = .{ .original = args.backend_config }, + .has_side_effect = false, + .api_version = .original, + .additional_attributes = args.additional_attributes, + .output_operand_aliases = args.aliases, + }, + cur.location, + ).appendTo(cur.currentScope().block); +} diff --git a/zml/kernel/tcl/BUILD.bazel b/zml/kernel/tcl/BUILD.bazel new file mode 100644 index 00000000..bc7a0aa2 --- /dev/null +++ b/zml/kernel/tcl/BUILD.bazel @@ -0,0 +1,9 @@ +package(default_visibility = ["//zml/kernel:__pkg__"]) + +filegroup( + name = "tcl", + srcs = [ + "tcl.zig", + "test.zig", + ], +) diff --git a/zml/kernel/tcl/tcl.zig b/zml/kernel/tcl/tcl.zig new file mode 100644 index 00000000..023e862f --- /dev/null +++ b/zml/kernel/tcl/tcl.zig @@ -0,0 +1,206 @@ +const std = @import("std"); + +const func = @import("mlir/dialects/func"); +const metal_builder = @import("dsl/metal/builder"); +const mlir = @import("mlir"); +const stablehlo = @import("mlir/dialects/stablehlo"); +const stdx = @import("stdx"); +const tcl_builder = @import("dsl/tcl/builder"); +pub const Builder = tcl_builder.Builder; +pub const TensorOperation = tcl_builder.TensorOperation; +pub const Value = tcl_builder.Value; +pub const Axis = tcl_builder.Axis; +pub const Expr = tcl_builder.Expr; +pub const Dram = tcl_builder.Dram; +pub const Context = tcl_builder.Context; +pub const DType = tcl_builder.DType; +pub const VeOp = tcl_builder.VeOp; +pub const ReduceMode = tcl_builder.ReduceMode; +pub const Predicate = tcl_builder.Predicate; +pub const Tactic = tcl_builder.Tactic; +pub const FinishError = tcl_builder.FinishError; + +const Compiler = @import("../../Compiler.zig"); +const DataType = @import("../../dtype.zig").DataType; +const mlirx = @import("../../mlirx.zig"); +const ops = @import("../../ops.zig"); +const platform_ = @import("../../platform.zig"); +const Shape = @import("../../shape.zig").Shape; +const Tensor = @import("../../tensor.zig").Tensor; + +test { + _ = @import("test.zig"); +} + +/// Build a struct type with one field per name, all of type `FieldT`. +fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { + var name_slices: [names.len][]const u8 = undefined; + for (names, 0..) |n, i| name_slices[i] = n; + return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); +} + +fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { + mlir.registerPasses("Transforms"); + + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + + registerDialects(registry); + + func.registerExtensions(registry); + + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + + return ctx; +} + +fn resolveOutputOperandAliases( + aliases_opt: anytype, + operand_offset: i64, +) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { + var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; + + if (aliases_opt) |a| { + inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { + if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ + .output_index = @intCast(output_index), + .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, + }); + } + } + + return out; +} + +pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(tcl_builder.registerDialects); +} + +pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .bool, + .i4 => .i4, + .i8 => .i8, + .i16 => .i16, + .i32 => .i32, + .i64 => .i64, + .u8 => .u8, + .u16 => .u16, + .u32 => .u32, + .u64 => .u64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f4e2m1 => .f4_e2, + .f8e4m3fn => .f8_e4, + .f8e5m2 => .f8_e5, + else => std.debug.panic("zml.kernel.tcl.from: dtype {s} has no TCL equivalent", .{@tagName(dt)}), + }; +} + +fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; +} + +/// A TCL function compiled by the Furiosa plugin as one native unit: +/// `declareArgs` declares the inputs in order, `ret` the outputs. +pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), +) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + pub const CallOpts = struct { + cfg: ConfigT, + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + return build(allocator, cfg, null, null, null); + } + + /// `updated`, when given, receives the input each output updates + /// in place (`Builder.update`), or null. + fn build(allocator: std.mem.Allocator, cfg: ConfigT, inputs: ?[]const Shape, outputs: ?[]const Shape, updated: ?*[spec.outputs.len]?usize) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try Builder.open(allocator, ctx, name); + defer b.deinit(); + try spec.run(&b, cfg); + + const results = b.results orelse std.debug.panic("zml.kernel.tcl.Kernel({s}): run did not call ret", .{name}); + if (b.args.len != spec.inputs.len or results.len != spec.outputs.len) { + std.debug.panic("zml.kernel.tcl.Kernel({s}): {d} arguments and {d} results for {d} inputs and {d} outputs", .{ name, b.args.len, results.len, spec.inputs.len, spec.outputs.len }); + } + + if (inputs) |shapes| for (b.args, shapes, spec.inputs) |t, s, n| check(t, s, n); + if (outputs) |shapes| for (results, shapes, spec.outputs) |t, s, n| check(t, s, n); + if (updated) |u| for (u, results) |*dst, r| { + dst.* = b.updatedArgument(r); + }; + + return b.finish(); + } + + fn check(t: tcl_builder.Tensor, s: Shape, field: []const u8) void { + var same = from(s.dtype()) == t.dtype; + if (t.shards > 1) { + // A partition holds one chip's share of a chip-split DRAM tensor. + var n: i64 = 1; + for (t.axes) |a| n *= a.size; + same = same and n == @as(i64, @intCast(s.count())) * t.shards; + } else { + same = same and s.rank() == t.axes.len; + if (same) for (t.axes, 0..) |a, i| { + same = same and a.size == s.dim(i); + }; + } + + if (!same) std.debug.panic("zml.kernel.tcl.Kernel({s}): {s} is {f}, the TCL function has {s} over {d} axes", .{ name, field, s, @tagName(t.dtype), t.axes.len }); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + var input_shapes: [spec.inputs.len]Shape = undefined; + inline for (spec.inputs, 0..) |fname, i| { + inputs_arr[i] = @field(inputs, fname); + input_shapes[i] = inputs_arr[i].shape(); + } + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + var updated: [spec.outputs.len]?usize = undefined; + const ir = build(cur.allocator, opts.cfg, &input_shapes, &outputs_arr, &updated) catch |err| + std.debug.panic("zml.kernel.tcl.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ir); + + var aliases: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, spec.outputs.len) = .empty; + for (updated, 0..) |input, output| if (input) |i| aliases.appendAssumeCapacity(.{ + .output_index = @intCast(output), + .operand_index = @intCast(i), + }); + const tensor_results = ops.tcl(inputs_arr, outputs_arr, .{ .ir = ir, .output_operand_aliases = aliases.constSlice() }); + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| { + @field(results, fname) = tensor_results[i]; + } + + return results; + } + }; +} diff --git a/zml/kernel/tcl/test.zig b/zml/kernel/tcl/test.zig new file mode 100644 index 00000000..b9ae1875 --- /dev/null +++ b/zml/kernel/tcl/test.zig @@ -0,0 +1,123 @@ +const std = @import("std"); + +const zml = @import("../../zml.zig"); +const tcl = @import("tcl.zig"); + +/// `furiosa.tcl.examples.softmax` over f32. +const TclSoftmax = struct { + const Cfg = struct { b: i64, s: i64, d: i64 }; + const K = tcl.Kernel(Cfg, .{ + .name = "softmax_kernel", + .inputs = &.{"x"}, + .outputs = &.{"y"}, + .run = run, + }); + + fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { + const B = b.axis("B", cfg.b); + const S = b.axis("S", cfg.s); + const D = b.axis("D", cfg.d); + const t = try b.declareArgs(.{ .x = .{ .dtype = .f32, .axes = &.{ B, S, D } } }); + + const op = b.tensorOperation(.{}); + const x = op.fetch(t.x, .{}); + const e = x.subf(x.reduce(&.{D}, .maxf)).exp(); + b.ret(&.{try op.commit(e.divf(e.reduce(&.{D}, .addf)), .{})}); + } +}; + +test "tcl kernels run on furiosa" { + const allocator = std.testing.allocator; + const cfg: TclSoftmax.Cfg = .{ .b = 2, .s = 16, .d = 256 }; + + const ir = try TclSoftmax.K.emit(allocator, cfg); + defer allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "func.func @softmax_kernel(") != null); + + const platform = zml.testing.env(); + if (platform.target != .furiosa) return error.SkipZigTest; + + const Mod = struct { + pub fn forward(x: zml.Tensor) zml.Tensor { + return TclSoftmax.K.call(.{ .x = x }, .{ .y = x.shape() }, .{ .cfg = cfg }).y; + } + }; + const x: zml.Tensor = .init(.{ cfg.b, cfg.s, cfg.d }, .f32); + var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{x}, platform, .{}); + defer exe.deinit(); + + var input: [cfg.b * cfg.s * cfg.d]f32 = undefined; + for (&input, 0..) |*v, i| v.* = @as(f32, @floatFromInt(i % 37)) * 0.25 - 4; + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&input)); + defer x_buffer.deinit(); + var result = try exe.eval(allocator, std.testing.io, .{x_buffer}); + defer result.deinit(); + var host = try result.toSliceAlloc(allocator, std.testing.io); + defer host.free(allocator); + + const out = host.items(f32); + var row: usize = 0; + while (row < cfg.b * cfg.s) : (row += 1) { + const in = input[row * cfg.d ..][0..cfg.d]; + var max: f32 = in[0]; + for (in) |v| max = @max(max, v); + var sum: f32 = 0; + for (in) |v| sum += @exp(v - max); + for (in, out[row * cfg.d ..][0..cfg.d]) |v, y| try std.testing.expectApproxEqAbs(@exp(v - max) / sum, y, 1e-4); + } +} + +/// exp over a tensor split across two chips in one launch, returning the +/// split result and its all_gather. +const TclSplitExp = struct { + const Cfg = struct {}; + const K = tcl.Kernel(Cfg, .{ .name = "split_exp", .inputs = &.{"x"}, .outputs = &.{ "y", "all" }, .run = run }); + fn run(b: *tcl.Builder, _: Cfg) tcl.FinishError!void { + const C = b.axis("C", 2); + const A = b.axis("A", 1024); + const t = try b.declareArgs(.{ .x = .{ .dtype = .f32, .axes = &.{ C, A }, .dram = tcl.Dram{ .chip = &.{.{ .axis = C }} } } }); + const op = b.tensorOperation(.{ .context = .{ .layout = .{ .chip = .{ .axis = C } } } }); + const local = try op.commit(op.fetch(t.x, .{}).exp(), .{}); + const all = try b.allGather(local, C, .{}); + b.ret(&.{ try b.asDram(local, .{ .chip = &.{.{ .axis = C }} }), try b.asDram(all, .{}) }); + } + fn body(ctx: struct { x: zml.Tensor }, shapes: [2]zml.Shape) [2]zml.Tensor { + const r = K.call(.{ .x = ctx.x }, .{ .y = shapes[0], .all = shapes[1] }, .{ .cfg = .{} }); + return .{ r.y, r.all }; + } +}; + +// Needs FURIOSA_VISIBLE_DEVICES=0,1. +test "tcl multi-chip kernels run on two furiosa chips" { + const allocator = std.testing.allocator; + const platform = zml.testing.env(); + if (platform.target != .furiosa or platform.devices.len != 2) return error.SkipZigTest; + const tp = zml.testing.tpMesh(); + const x_shape = zml.Shape.init(.{ .c = 2, .a = 1024 }, .f32).withPartitioning(tp, .{ .c = .model }); + const Mod = struct { + pub fn forward(x: zml.Tensor, mesh: *const zml.Sharding.Mesh) [2]zml.Tensor { + const r = zml.ops.manualComputation(TclSplitExp.body, .{ .x = x }, .{ x.shape(), zml.Shape.init(.{ .c = 2, .a = 1024 }, .f32) }, mesh, .{ + .in_specs = .{ .x = .parse(.{ .c = .model }) }, + .out_specs = .{ .parse(.{ .c = .model }), .replicated }, + }); + return .{ r[0], r[1] }; + } + }; + const x: zml.Tensor = .fromShape(x_shape); + var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{ x, tp }, platform, .{ .meshes = &.{tp} }); + defer exe.deinit(); + var hx: [2 * 1024]f32 = undefined; + for (&hx, 0..) |*v, i| v.* = @as(f32, @floatFromInt(i % 13)) * 0.125 - 0.5; + var bx: zml.Buffer = try .fromBytes(std.testing.io, platform, x_shape, std.mem.sliceAsBytes(&hx)); + defer bx.deinit(); + var runner = try exe.runner(allocator); + defer runner.deinit(allocator); + var res: [2]zml.Buffer = undefined; + runner.run(std.testing.io, .{bx}, .{ &res[0], &res[1] }, .{ .wait = true }); + defer for (&res) |*r| r.deinit(); + for (res) |r| { + var host = try r.toSliceAlloc(allocator, std.testing.io); + defer host.free(allocator); + for (host.items(f32), 0..) |v, i| try std.testing.expectApproxEqRel(@exp(hx[i]), v, 1e-4); + } +} diff --git a/zml/kernel/triton/BUILD.bazel b/zml/kernel/triton/BUILD.bazel new file mode 100644 index 00000000..1e992352 --- /dev/null +++ b/zml/kernel/triton/BUILD.bazel @@ -0,0 +1,6 @@ +package(default_visibility = ["//zml/kernel:__pkg__"]) + +filegroup( + name = "triton", + srcs = ["triton.zig"], +) diff --git a/zml/kernel/triton/triton.zig b/zml/kernel/triton/triton.zig new file mode 100644 index 00000000..6a91a03f --- /dev/null +++ b/zml/kernel/triton/triton.zig @@ -0,0 +1,165 @@ +const std = @import("std"); + +const func = @import("mlir/dialects/func"); +const mlir = @import("mlir"); +const stablehlo = @import("mlir/dialects/stablehlo"); +const stdx = @import("stdx"); +const triton_builder = @import("dsl/triton/builder"); +pub const Builder = triton_builder.Builder; +pub const Value = triton_builder.Value; +pub const DType = triton_builder.DType; +pub const FinishError = triton_builder.FinishError; + +const Compiler = @import("../../Compiler.zig"); +const DataType = @import("../../dtype.zig").DataType; +const ops = @import("../../ops.zig"); +const Shape = @import("../../shape.zig").Shape; +const Tensor = @import("../../tensor.zig").Tensor; + +/// Build a struct type with one field per name, all of type `FieldT`. +fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { + var name_slices: [names.len][]const u8 = undefined; + for (names, 0..) |n, i| name_slices[i] = n; + return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); +} + +fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { + mlir.registerPasses("Transforms"); + + const registry = try mlir.DialectRegistry.init(); + defer registry.deinit(); + + registerDialects(registry); + + func.registerExtensions(registry); + + const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); + ctx.loadAllAvailableDialects(); + + return ctx; +} + +fn resolveOutputOperandAliases( + aliases_opt: anytype, + operand_offset: i64, +) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { + var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; + + if (aliases_opt) |a| { + inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { + if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ + .output_index = @intCast(output_index), + .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, + }); + } + } + + return out; +} + +pub fn newContext() std.mem.Allocator.Error!*mlir.Context { + return makeKernelContext(triton_builder.registerDialects); +} + +pub fn from(dt: DataType) DType { + return switch (dt) { + .bool => .i1, + .i8, .u8 => .i8, + .i16, .u16 => .i16, + .i32, .u32 => .i32, + .i64, .u64 => .i64, + .f16 => .f16, + .bf16 => .bf16, + .f32 => .f32, + .f64 => .f64, + .f8e4m3fn => .f8e4m3fn, + .f8e4m3fnuz => .f8e4m3fnuz, + .f8e5m2 => .f8e5m2, + else => std.debug.panic("zml.kernel.triton.from: dtype {s} has no Triton equivalent", .{@tagName(dt)}), + }; +} + +fn Spec(comptime Config: type) type { + return struct { + name: [:0]const u8, + inputs: []const [:0]const u8, + outputs: []const [:0]const u8, + run: *const fn (*Builder, Config) FinishError!void, + }; +} + +pub fn Kernel( + comptime ConfigT: type, + comptime spec: Spec(ConfigT), +) type { + return struct { + pub const name: [:0]const u8 = spec.name; + pub const Config = ConfigT; + pub const Inputs = StructOf(spec.inputs, Tensor); + pub const Outputs = StructOf(spec.outputs, Shape); + pub const Results = StructOf(spec.outputs, Tensor); + + pub const CallOpts = struct { + cfg: ConfigT, + grid: [3]i32, + num_stages: i32, + num_warps: i32, + output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, + debug: bool = false, + /// Bytes for Triton's implicit global scratch argument, across all CTAs. + global_scratch_memory_size: i32 = 0, + }; + + pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { + const ctx = try newContext(); + defer ctx.deinit(); + + var b = try triton_builder.Builder.open(allocator, ctx, name); + defer b.deinit(); + + try spec.run(&b, cfg); + + return b.finish(); + } + + pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { + const cur = Compiler.current(); + + const ttir = emit(cur.allocator, opts.cfg) catch |err| + std.debug.panic("zml.kernel.triton.Kernel({s}).call: emit failed: {}", .{ name, err }); + defer cur.allocator.free(ttir); + + var inputs_arr: [spec.inputs.len]Tensor = undefined; + inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); + + var outputs_arr: [spec.outputs.len]Shape = undefined; + inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); + + const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); + + const call_opts: ops.TritonOps = .{ + .debug = opts.debug, + .name = name, + .ir = ttir, + .grid = opts.grid, + .num_stages = opts.num_stages, + .num_warps = opts.num_warps, + .output_operand_aliases = aliases.constSlice(), + .is_tma_allowed = opts.global_scratch_memory_size > 0, + .global_scratch_memory_size = opts.global_scratch_memory_size, + }; + + const triton_results = if (opts.global_scratch_memory_size > 0) blk: { + // XLA maps the final custom-call result to Triton's implicit + // scratch pointer; it must not appear in the TTIR signature. + const extended = outputs_arr ++ [_]Shape{.init(.{opts.global_scratch_memory_size}, .u8)}; + const res = ops.triton(inputs_arr, extended, call_opts); + break :blk res[0..spec.outputs.len].*; + } else ops.triton(inputs_arr, outputs_arr, call_opts); + + var results: Results = undefined; + inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = triton_results[i]; + return results; + } + }; +} diff --git a/zml/kernels/flash_attention/README.md b/zml/kernels/flash_attention/README.md new file mode 100644 index 00000000..81a35cf4 --- /dev/null +++ b/zml/kernels/flash_attention/README.md @@ -0,0 +1,287 @@ +# Flash attention semantics + +This directory exposes causal scaled dot-product attention through +[`zml.flash_attention.attention`](flash_attention.zig). This document describes +the attention operation, the FlashAttention algorithm, and ZML's current semantic +contract. Shapes below are logical tensor shapes. Sharding describes ownership +of those tensors across devices; platform-specific execution layouts are outside +the scope of this document. + +## The attention operation + +For one sequence and one head, let `Q` contain `Lq` query vectors, and let `K` and +`V` contain `Lk` key/value vectors. Each query is compared with each permitted key; +the resulting probabilities weight the corresponding values: + +```text +S = scale * Q @ transpose(K) + M +P = softmax(S, axis=keys) +O = P @ V +``` + +The usual scale is `1 / sqrt(D)`, where `D` is the query/key head dimension. +An additive mask `M` is zero for allowed pairs and negative infinity for forbidden +pairs. Softmax normalizes each query row independently over keys, so a masked key +has zero probability. Heads and batch elements are independent. ZML's +[`sdpa`](../../nn.zig) implements this reference computation. + +Attention in general permits a value dimension `Dv` different from the query/key +dimension `D`; the output then has dimension `Dv`. This module uses the same +`.hd = D` for queries, keys, values, and output. + +FlashAttention can evaluate either unrestricted attention or masked attention. +Causality is a choice of mask, rather than an inherent property of the algorithm. +Other attention APIs may offer local windows, additive score biases, a custom +scale, and dropout on the probabilities during training. These options change +the operation and must be matched when comparing results. The +[upstream FlashAttention API](https://github.com/Dao-AILab/flash-attention#how-to-use-flashattention) +documents such options; ZML's interface here exposes a narrower operation. + +## What makes attention "flash" + +FlashAttention computes the same mathematical attention result while avoiding +storage of the complete `Lq × Lk` score and probability matrices. It processes +query/key/value tiles and combines partial results using an online softmax. +This reduces intermediate memory and memory traffic. Dense attention still +requires `O(Lq * Lk * D)` arithmetic per head; for self-attention with `Lq = Lk`, +that remains quadratic in sequence length. "Exact" means the attention formula +is preserved, with ordinary floating-point rounding differences. See the +[FlashAttention paper](https://arxiv.org/abs/2205.14135). + +For each query row, online softmax maintains a maximum `m`, an exponential sum +`l`, and an unnormalized weighted-value vector `u`. Given scores `s_j` for a new +tile of permitted keys, update: + +```text +m_new = max(m, max_j(s_j)) +a = exp(m - m_new) +p_j = exp(s_j - m_new) +l_new = a * l + sum_j(p_j) +u_new = a * u + sum_j(p_j * V_j) +O = u / l # after all tiles +``` + +Initialize with `m = -infinity`, `l = 0`, and `u = 0`; skip tiles with no permitted +keys. Rescaling the previous accumulators preserves a single normalization across +all tiles. Independently normalizing each tile and adding its output would give +a different result. The [FlashAttention-2 paper, sections 2–3](https://arxiv.org/html/2307.08691v1) +describes this recurrence and how training can recompute probability tiles during +the backward pass instead of retaining the full matrix. This module exposes only +the attention output, with no backward entry point or returned probabilities. + +## Heads and head dimensions + +`D` counts features in **one head**. It is distinct from the number of heads and +the sequence length. In a conventional multi-head projection, +`D = hidden_size / Hq`; models with an explicit head dimension need not follow +that relation. The query projection has width `Hq * D`, while each key/value +projection has width `Hkv * D`. + +Typical dimensions include: + +| Head dimension | Context | +| --- | --- | +| `64` | A common transformer head size; also the size used by this module's shared tests. | +| `128` | Another common size; Qwen2.5-7B uses `3584 / 28 = 128`. | +| `256` | Used by models such as Gemma 2 2B and 9B. | +| `512` | Used by Gemma 4 global attention layers, for example in Gemma 4 31B. | + +The [FlashAttention-2 paper](https://arxiv.org/html/2307.08691v1) describes typical +head sizes around 64–128. The model examples follow the +[Qwen2.5-7B configuration](https://huggingface.co/Qwen/Qwen2.5-7B-Instruct/raw/main/config.json) +and [Gemma configurations](https://github.com/google/gemma_pytorch/blob/main/gemma/config.py). +Gemma can also use different head dimensions within one model: the +[Gemma 4 31B configuration](https://huggingface.co/google/gemma-4-31B/blob/main/config.json) +specifies `head_dim = 256` for local attention and `global_head_dim = 512` for +global attention. Use the dimension configured for the specific layer. +These are model dimensions, not a list of supported kernel sizes or a mathematical +requirement that `D` be a power of two. + +The number of query heads `Hq` may differ from the number of key/value heads +`Hkv`. ZML uses contiguous groups of query heads: + +```text +G = Hq / Hkv +kv_head(h) = floor(h / G) +``` + +`Hq` must be divisible by `Hkv`, and keys and values must have the same head count. +This covers multi-head attention (MHA: `Hq = Hkv`), grouped-query attention +(GQA: `1 < Hkv < Hq`), and multi-query attention (MQA: `Hkv = 1`). For example, +with `Hq = 64` and `Hkv = 8`, query heads 0–7 use KV head 0, heads 8–15 use KV +head 1, and so on. Sharing keys and values does not merge query heads: each query +head still has its own scores, normalization, and output. + +## ZML inputs and output + +The entry point is: + +```zig +attention(q, k, v, token_index, metadata, parameters) +``` + +The reference contract uses these tagged shapes, with an optional `.b` batch axis: + +| Argument | Unbatched shape | Batched shape | Meaning | +| --- | --- | --- | --- | +| `q` | `{ .q = Lq, .h = Hq, .hd = D }` | `{ .b = B, .q = Lq, .h = Hq, .hd = D }` | Queries to evaluate. | +| `k`, `v` | `{ .k = Lk, .h = Hkv, .hd = D }` | `{ .b = B, .k = Lk, .h = Hkv, .hd = D }` | Keys and values in sequence order. | +| `token_index` | Scalar | `{ .b = B }` or scalar | Zero-based position of the first query in the key sequence; a scalar applies to the whole batch. | +| Result | Same as `q` | Same as `q` | Weighted values for every query head. | + +Use matching floating-point dtypes for `q`, `k`, and `v`; the output retains the +query dtype. The shared tests use `bf16` tensors and `u32` positions. `metadata` +and `parameters` provide execution configuration rather than additional masks, +score biases, or model projections. + +### Causality and query positions + +Let `t = token_index` for a sequence. Query row `i` represents position `t + i`, +and key row `j` represents position `j`. The inclusive causal rule is: + +```text +allowed(i, j) = (j <= t + i) +M[i, j] = 0 if allowed(i, j), otherwise -infinity +``` + +The reference path constructs a square causal mask over `Lk` positions, gathers +`Lq` consecutive rows starting at `t`, and passes that mask to `sdpa`. Valid inputs +require `Lq > 0`, `t >= 0`, and `t + Lq <= Lk`. The test helper checks the upper +bound; callers must satisfy it rather than rely on out-of-range gather behavior. +Each valid query can attend to at least its own key, so fully masked rows are +outside this contract. + +| Use case | Query positions | Keys visible to query row `i` | +| --- | --- | --- | +| Initial prefill | `t = 0`, usually `Lq = Lk` | `0` through `i`. | +| Chunked prefill | `Lq > 1`, starting at `t` | `0` through `t + i`. | +| Single-token decode | `Lq = 1`, current token at `t` | `0` through `t`, including the current token. | + +For `Lq = 2`, `Lk = 5`, and `t = 2`, the mask is: + +```text + key position + 0 1 2 3 4 +query at 2 1 1 1 0 0 +query at 3 1 1 1 1 0 +``` + +Here `1` means permitted. `Lk` may be the capacity of a preallocated KV buffer; +it does not determine the current query position. All permitted KV entries must +already be populated. Positions after the last query are masked, and the call +does not append to or update a KV cache. + +For unequal query and key lengths, causal alignment must be specified explicitly. +The [upstream FlashAttention API](https://github.com/Dao-AILab/flash-attention#21-change-behavior-of-causal-flag) +uses bottom-right alignment: `j <= i + Lk - Lq`. ZML's reference rule agrees with +that convention when `t = Lk - Lq`. An explicit `token_index` also describes +queries earlier in a larger buffer, as in the example above. + +### Sharding and local attention + +Sharding changes where attention is evaluated, not its mathematical result. +The global output still has the query's tagged shape and dtype. Each local call +must receive the matching queries, keys, values, and positions for the batch +elements and heads it owns. + +| Axis or input | Sharding semantics | +| --- | --- | +| Batch `.b` | Sequences are independent. Q, K, V, and the output must assign the same batch elements to each shard. | +| Heads `.h` | Each shard must have the KV heads needed by its query heads, preserving `kv_head(h) = floor(h / G)`. K and V must have matching head ownership. | +| Query sequence `.q` | Query rows are independent once the required KV entries are available. Their causal positions remain positions in the original sequence. | +| Key sequence `.k` | Softmax normalizes over all permitted keys for a query, including keys on other shards if the sequence is distributed. Separate local softmax outputs cannot simply be added or averaged. | +| Features `.hd` | Every score uses the complete head dimension and the global scale `1 / sqrt(D)`. Splitting features requires combining partial dot products before softmax. | +| `token_index` | A scalar applies to every shard. Per-sequence positions follow the batch partition and are replicated across head shards of the same sequence. Head or batch sharding does not change their values. | + +For head sharding without KV replication, a partition count `P` must divide +`Hkv`, and therefore also `Hq`. Each shard owns complete, contiguous GQA groups: +`Hq_local = Hq / P`, `Hkv_local = Hkv / P`, and the local ratio remains `G`. +For example, with `Hq = 32`, `Hkv = 8`, and two shards, the first shard owns query +heads 0–15 and KV heads 0–3; the second owns query heads 16–31 and KV heads 4–7. +Each query still uses exactly the same KV head as in the unsharded computation. +MQA has only one KV head, so distributing its query heads over multiple devices +requires replicating that KV head rather than dividing it into independent heads. + +The local-kernel execution contract keeps `.q`, `.k`, and `.hd` complete on each +shard, with parallelism across batches and/or complete head groups. Thus each +local call has the full sequence coordinates and feature vectors needed by the +causal mask, dot products, and normalization. The mathematical possibilities for +sequence and feature sharding above do not imply that these local kernels +implement distributed sequence or feature reductions. + +An **open** partition means placement is unresolved; it does not mean the tensor +must be replicated. Operations such as cache slicing and RoPE can reopen +partition annotations even when their inputs were sharded. Before entering a +local kernel, resolve compatible partitions on a common mesh and align Q, K, V, +and positions. The KV layout anchors this alignment: an explicit KV head +partition is retained, while an open head partition can use a compatible mesh +axis whose partition count divides `Hkv`. Preserve explicit batch ownership and +do not assign the same device partition independently to both batch and heads. +An explicit replicated KV head axis remains replicated; open heads can also +remain replicated when no compatible head partition is available. Automatic +partition selection is an execution choice, not an additional attention argument. + +The output follows the **resolved** query partition. Its partition annotations +need not match the incoming query's unresolved annotations, and aligning operands +may require communication. In particular, unnecessarily resolving open KV heads +to replicated can gather the full cache on every attention call even though the +head groups could be evaluated independently. This changes cost, not attention +semantics, and does not modify the caller's KV cache. + +Splitting KV work into internal tiles or partitions is distinct from sharding the +input tensors across devices. Either way, combining partial attention states +must preserve the single softmax normalization. For disjoint key subsets with +states `(m_r, l_r, u_r)` as defined above: + +```text +m = max_r(m_r) +l = sum_r(exp(m_r - m) * l_r) +u = sum_r(exp(m_r - m) * u_r) +O = u / l +``` + +An empty subset contributes zero (`m_r = -infinity`, `l_r = 0`, `u_r = 0`). +Every valid query has at least one permitted key overall, so its merged +normalization is nonempty. + +### Scope and current conformance + +The public call always requests causal attention with scale `1 / sqrt(D)` and no +dropout. It has no arguments for an arbitrary mask, padding lengths, packed +sequence boundaries, a local window, ALiBi, score soft-capping, a softmax sink, or +a custom scale. The lower-level `sdpa` and `causalAttnMask` helpers expose some +additional options, but this wrapper does not pass them through. Separate batch +elements can have different `token_index` values; concatenating independent +sequences along `.k` does not isolate them from each other. + +Q/K/V projections, positional transformations such as RoPE, and any Q/K +normalization must happen before this call. Head concatenation and the output +projection happen afterward. A model's use of a familiar head dimension alone +does not establish compatibility with this operation. + +The reference contract above is broader than current kernel coverage. In +particular, nonzero-offset multi-query prefill and the optional batch dimension +are not uniformly supported by the current kernels; an existing prefill path +requires equal query/key lengths and assumes positions begin at zero. Treat +these as current conformance gaps, not alternative definitions of causality. + +The tests in [tests.zig](tests.zig) compare results +against `sdpa` with the gathered causal mask. The original shared cases cover +`D = 64`, MHA and GQA, +single-token decode at full and partial context lengths, batched decode with +different positions, and an eight-query chunk beginning at position 56 in a +64-position KV buffer. Additional conformance cases exercise head dimensions +through `512`, MQA, `bf16`/`f16`/`f32`, initial and offset prefill, partial tiles, +transposed tagged layouts, scalar batch positions, head/batch sharding, open +query/KV partitions, and split decode with empty partitions and cache tails. These +cases are enabled for kernels that support them; their presence does not imply +uniform coverage across every kernel. FP32 results are also checked against an +independent host calculation using FP64 arithmetic. + +Numerical agreement is tolerance-based. The reference evaluates softmax in +`f32`, then converts probabilities to the query dtype before multiplying by +values. Different accumulation orders and rounding points can produce small +differences even when the mathematical operation is identical. The shared tests +use absolute tolerance `5e-3`, relative tolerance `1e-2`, relative epsilon `1e-3`, +and a minimum close fraction of `0.99`; these are test criteria rather than a +universal error bound. diff --git a/zml/kernels/flash_attention/cute/BUILD.bazel b/zml/kernels/flash_attention/cute/BUILD.bazel new file mode 100644 index 00000000..b83a642e --- /dev/null +++ b/zml/kernels/flash_attention/cute/BUILD.bazel @@ -0,0 +1,9 @@ +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "cute", + srcs = [ + "attention.zig", + "cute.zig", + ], +) diff --git a/zml/kernels/flash_attention/cute/README.md b/zml/kernels/flash_attention/cute/README.md new file mode 100644 index 00000000..8c55d2c8 --- /dev/null +++ b/zml/kernels/flash_attention/cute/README.md @@ -0,0 +1,145 @@ +# CuTe flash attention scheduling + +The [parent README](../README.md) defines attention and sharding semantics. +This document describes decode scheduling in this implementation and the +FlashInfer heuristics that inform it. Scheduling changes parallelism and +intermediate storage, not the attention formula or public arguments. + +## What FlashInfer does + +These references are pinned to FlashInfer commit +`2a2cceab07c7bfca0525065fbfdc5fb2cff0c33a`; FlashInfer has multiple execution paths, +so these rules describe the cited CUDA decode paths rather than every backend. + +[Single-sequence decode](https://github.com/flashinfer-ai/flashinfer/blob/2a2cceab07c7bfca0525065fbfdc5fb2cff0c33a/include/flashinfer/attention/decode.cuh#L705-L744) +uses one attention launch when the live sequence has at most 256 tokens or no +scratch buffer is supplied. Otherwise it queries the compiled kernel's resident +blocks per SM and the device's SM count, then computes: + +```text +grid_budget = resident_blocks_per_SM * SM_count +split_budget = floor(grid_budget / KV_heads) +chunk_tokens = max(ceil(live_tokens / split_budget), 256) +splits = ceil(live_tokens / chunk_tokens) +``` + +It launches partial attention over these chunks followed by `MergeStates`. +The merge preserves a single softmax normalization across the chunks. + +[Batched paged decode](https://github.com/flashinfer-ai/flashinfer/blob/2a2cceab07c7bfca0525065fbfdc5fb2cff0c33a/include/flashinfer/attention/scheduler.cuh#L75-L98) +uses the per-request lengths to search for a chunk size whose total block count +fits the occupancy budget. Its +[work estimator](https://github.com/flashinfer-ai/flashinfer/blob/2a2cceab07c7bfca0525065fbfdc5fb2cff0c33a/include/flashinfer/attention/scheduler.cuh#L174-L205) +disables splitting if `batch_size * KV_heads` already fills that budget. The +minimum chunk is `max(floor(128 / page_size), 1)` pages. Without CUDA graphs it +also avoids splitting when partitioning adds no requests. Graph execution can +retain a padded split schedule to keep launch and allocation sizes stable. + +## Adaptation to this kernel + +[`decodeSplits`](attention.zig) runs after sharding has established each device's +local dimensions. A block owns at most four query heads belonging to one KV +head, so our unsplit decode block count is: + +```text +G = local_query_heads / local_KV_heads +base_blocks = local_batch * local_KV_heads * ceil(G / 4) +``` + +Counting query-head groups matters: a large GQA group already launches multiple +blocks per KV head in this kernel. Using global batch or head counts would +underestimate the need for splitting after partitioning across devices. + +XLA compiles and owns the CUDA function after this planner runs. We cannot call +CUDA's occupancy calculator on that function here. Instead, the planner uses +PJRT's `core_count` and `shared_memory_per_block_optin` attributes and a +static limit of twelve 128-thread blocks per SM: + +```text +padded_D = next_power_of_two(max(D, 32)) +shared_bytes_per_block = 2 * 16 * padded_D * sizeof(dtype) # K and V tiles +blocks_per_SM = max(1, min(12, floor(shared_budget / shared_bytes_per_block))) +grid_budget = 4 * SM_count * blocks_per_SM +``` + +The per-block shared-memory limit serves as a conservative shared-memory budget; +it is not a measurement of SM capacity or compiled-kernel occupancy. The +twelve-block limit is a scheduling estimate, not a measured occupancy or a +hardware constant. It provides enough resident work to hide latency in the +128-thread kernel; register usage and other hardware limits can reduce actual +occupancy. Four scheduling waves provide additional work for the serial +KV-tile loop; this is a measured adaptation, whereas the cited FlashInfer CUDA +planner targets one resident grid. Larger heads and FP32 reduce the budget through their shared-memory requirements. For +multiple visible devices, the smallest reported SM count and shared-memory +budget are used. Missing or invalid attributes disable splitting. + +The selection rules are: + +1. Prefill (`Lq > 1`) remains unsplit. +2. A cache capacity of at most 64 tokens remains unsplit. +3. If `base_blocks >= grid_budget`, keep one partition per block's head group. +4. Otherwise set `split_budget = ceil(grid_budget / base_blocks)`, + `minimum_chunk = 16` for capacities through 256, otherwise `32`, + `chunk = max(minimum_chunk, ceil(cache_capacity / split_budget))`, and + `splits = ceil(cache_capacity / chunk)`. +5. If the result is one split, launch attention directly into the output. Other + cases allocate FP32 partial states and launch attention followed by a merge. + +The chunk floors are one or two of this kernel's 16-key tiles. They adapt +FlashInfer's minimum-work-per-partition heuristic to our serial per-tile score +loop. Applying its literal 128/256-token floors and a smaller block budget +regressed several measured cases. The workload-based budget and split/merge +structure follow FlashInfer, but these constants are specific to this kernel. +The split budget rounds up a partial wave: rounding 32 partitions down to 31, +for example, can add a whole tile to every block. The grid budget is therefore +a target, and the launched grid may exceed it by less than one split's worth +of `base_blocks`. + +## Capacity and live lengths + +Our graph receives `token_index` on device, potentially with a different value +for each batch element. Planning uses static cache capacity to bound the grid +and scratch allocation without reading positions back to the host. At runtime, +each sequence distributes its **live** prefix of `token_index + 1` keys evenly +across the planned splits. Therefore the actual chunk can be smaller than the +planning floor. Empty partitions contribute zero to the merge. + +This is not FlashInfer's host-side ragged-length search. A short live sequence +in a large cache can still use the split path, and the merge launch remains in +the graph even when only one partition has keys. The schedule does not change +from token to token, so it can be reused during generation and graph capture. + +Partial states contain the weighted-value accumulator, maximum, and exponential +sum in FP32. Their storage is +`4 * local_batch * splits * local_query_heads * (D + 2)` bytes. Reducing splits +as the batch grows reduces both scratch traffic and merge work. Scores are +scaled by `log2(e) / sqrt(D)` so attention and merging use base-2 exponentials. + +## Validation and timing + +The existing attention harness covers GQA/MQA, different positions within a +batch, partial tiles, empty partitions, sharding, three dtypes, and head +dimensions through 512. Planner tests cover the short-cache boundary, decreasing +split counts as batch size grows, resource limits, and extra query-head groups. +Runtime checks include batches through 2048 at the 64/65-token scheduling +boundary; they establish correctness, not throughput at that batch size. + +An opt-in benchmark in [cute.zig](cute.zig) compares the previous 32-split policy +with the planner at batches 1, 4, 16, and 64, and full contexts of 128, 2048, and +8192 tokens. It uses BF16, 16 query heads, four KV heads, and `D = 128` per device. +Sixteen dependent attention calls amortize host overhead; reported microseconds +per attention call are the median of 16 samples after three warmups. This is a +repeated-cache kernel benchmark, not end-to-end model throughput. + +```sh +bazel test //zml:test --//platforms:cuda=true --//platforms:cpu=false \ + --@rules_zig//zig/settings:zigopt=--test-filter \ + --@rules_zig//zig/settings:zigopt=zml.test.zml \ + --@rules_zig//zig/settings:zigopt=--test-filter \ + '--@rules_zig//zig/settings:zigopt=CuTe decode' \ + --test_env=CUDA_VISIBLE_DEVICES=0 \ + --test_env=ZML_CUTE_ATTENTION_BENCHMARK=1 --test_output=all +``` + +`CUTE_DECODE_BENCH` rows contain batch, context length, policy, split count, and +microseconds per call. Timing is not a correctness assertion. diff --git a/zml/kernels/flash_attention/cute/attention.zig b/zml/kernels/flash_attention/cute/attention.zig new file mode 100644 index 00000000..ec0fce16 --- /dev/null +++ b/zml/kernels/flash_attention/cute/attention.zig @@ -0,0 +1,818 @@ +const std = @import("std"); +const mlir = @import("mlir"); +const llvm = @import("mlir/dialects/llvm"); +const vector = @import("mlir/dialects/vector"); + +const zml = @import("../../../zml.zig"); +const cute = zml.kernel.cute; +const B = cute.Builder; +const Value = cute.Value; + +pub const Config = struct { + batch: i64, + query_length: i64, + key_length: i64, + query_heads: i64, + kv_heads: i64, + head_dim: i64, + dtype: cute.DType, + splits: i64 = 1, + /// Query j sees the keys up to positions + j. Otherwise every query sees every key and positions is ignored. + causal: bool = true, +}; + +/// Planning uses local (post-sharding) dimensions. XLA owns the compiled CUDA +/// function, so this is an estimated resource budget, not an occupancy query. +pub const DecodeResources = struct { + multiprocessors: i64, + shared_bytes: i64, +}; + +pub fn decodeSplits(q: zml.Shape, k: zml.Shape, resources: DecodeResources) i64 { + const query_length = q.dim(.q); + const key_length = k.dim(.k); + const num_heads = q.dim(.h); + const num_kv_heads = k.dim(.h); + const head_dim = q.dim(.hd); + const batch = if (q.hasTag(.b) != null) q.dim(.b) else 1; + const dtype = q.dtype(); + + if (query_length != 1 or key_length <= 64) return 1; + if (resources.multiprocessors <= 0 or resources.shared_bytes <= 0) return 1; + const group = @divExact(num_heads, num_kv_heads); + const blocks_per_sequence = num_kv_heads * @divTrunc(group + 3, 4); + const base_blocks = batch * blocks_per_sequence; + var padded_dim: i64 = 32; + while (padded_dim < head_dim) padded_dim *= 2; + const bytes_per_element: i64 = if (dtype == .f32) 4 else 2; + const shared_per_block = 2 * tile_keys * padded_dim * bytes_per_element; + const blocks_per_sm = @max(1, @min(12, @divTrunc(resources.shared_bytes, shared_per_block))); + // Multiple scheduling waves keep the serial tile loop supplied with work. + // The measured tradeoff and distinction from CUDA occupancy are in README. + const grid_budget = 4 * resources.multiprocessors * blocks_per_sm; + if (base_blocks >= grid_budget) return 1; + // Round up a partial wave: flooring can leave SMs idle and add a whole + // extra KV tile to every block (e.g. 31 partitions versus 32). + const split_budget = @divTrunc(grid_budget + base_blocks - 1, base_blocks); + // One tile for short caches, two otherwise. FlashInfer's larger chunk + // floor is tuned for a different per-block KV processing rate. + // With static dense shapes, capacity bounds the scratch space and grid; + // each invocation balances its live prefix over these partitions on device. + const min_chunk: i64 = if (key_length <= 256) tile_keys else 2 * tile_keys; + const chunk = @max(min_chunk, @divTrunc(key_length + split_budget - 1, split_budget)); + return @divTrunc(key_length + chunk - 1, chunk); +} + +test "CuTe decode planner respects workload and resource limits" { + const resources: DecodeResources = .{ .multiprocessors = 170, .shared_bytes = 101376 }; + var q = zml.Shape.init(.{ .b = 1, .q = 1, .h = 16, .hd = 128 }, .bf16); + var k = zml.Shape.init(.{ .b = 1, .k = 2048, .h = 4, .hd = 128 }, .bf16); + try std.testing.expectEqual(64, decodeSplits(q, k, resources)); + k = k.setDim(.k, 64); + try std.testing.expectEqual(1, decodeSplits(q, k, resources)); + k = k.setDim(.k, 65); + try std.testing.expectEqual(5, decodeSplits(q, k, resources)); + k = k.setDim(.k, 2048); + var previous: i64 = 64; + for ([_]i64{ 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048 }) |batch| { + q = q.setDim(.b, batch); + k = k.setDim(.b, batch); + const splits = decodeSplits(q, k, resources); + try std.testing.expect(splits <= previous); + try std.testing.expect(splits >= 1); + try std.testing.expect(splits == 1 or (splits - 1) * batch * 4 < 8160); + previous = splits; + } + try std.testing.expectEqual(1, previous); + q = q.setDim(.b, 16); + k = k.setDim(.b, 16); + q = q.setDim(.hd, 512); + k = k.setDim(.hd, 512); + q = q.withDtype(.f32); + k = k.withDtype(.f32); + try std.testing.expectEqual(11, decodeSplits(q, k, resources)); + q = q.setDim(.q, 7); + try std.testing.expectEqual(1, decodeSplits(q, k, resources)); + q = q.setDim(.q, 1); + try std.testing.expectEqual(1, decodeSplits(q, k, .{ .multiprocessors = 0, .shared_bytes = 0 })); + // More query-head groups already create more independent CTAs. + q = q.setDim(.b, 1); + k = k.setDim(.b, 1); + q = q.setDim(.hd, 128); + k = k.setDim(.hd, 128); + q = q.withDtype(.bf16); + k = k.withDtype(.bf16); + q = q.setDim(.h, 1024); + try std.testing.expectEqual(32, decodeSplits(q, k, resources)); +} + +pub const block: [3]i32 = .{ 128, 1, 1 }; +const tile_keys = 16; + +pub const Kernel = cute.Kernel(Config, .{ + .name = "flash_attention_cute", + .inputs = &.{ "q", "k", "v", "positions" }, + .outputs = &.{"out"}, + .run = run, +}); + +fn run(b: *B, cfg: Config) cute.FinishError!void { + std.debug.assert(cfg.head_dim > 0 and cfg.head_dim <= 512); + inline for (.{ 1, 2, 4, 8, 16 }) |fragments| { + if (cfg.head_dim <= fragments * 32) return runSized(b, cfg, fragments); + } + unreachable; +} + +/// Four warps own four (query position, query head) rows sharing one KV head. +/// Only a KV tile lives in shared memory; maxima, sums, and outputs stay in +/// registers. All lanes participate in shuffles, including padded rows/features. +fn runSized(b: *B, cfg: Config, comptime fragments: usize) cute.FinishError!void { + const padded_dim = fragments * 32; + const split = cfg.splits > 1; + const out_shape = if (split) &[_]i64{ cfg.batch, cfg.splits, cfg.query_length, cfg.query_heads, cfg.head_dim + 2 } else &[_]i64{ cfg.batch, cfg.query_length, cfg.query_heads, cfg.head_dim }; + const a = try b.declareArgs(.{ + .q = .{ .tensor = .{ .dtype = cfg.dtype, .shape = &.{ cfg.batch, cfg.query_length, cfg.query_heads, cfg.head_dim } } }, + .k = .{ .tensor = .{ .dtype = cfg.dtype, .shape = &.{ cfg.batch, cfg.key_length, cfg.kv_heads, cfg.head_dim } } }, + .v = .{ .tensor = .{ .dtype = cfg.dtype, .shape = &.{ cfg.batch, cfg.key_length, cfg.kv_heads, cfg.head_dim } } }, + .positions = .{ .tensor = .{ .dtype = .i32, .shape = &.{cfg.batch} } }, + .out = .{ .tensor = .{ .dtype = if (split) .f32 else cfg.dtype, .shape = out_shape } }, + }); + const tid = b.threadIdx().x; + const lane = tid.rem(32); + const group = @divExact(cfg.query_heads, cfg.kv_heads); + const batch = b.blockIdx().y.div(cfg.kv_heads); + const kv_head = b.blockIdx().y.rem(cfg.kv_heads); + const first_row = b.blockIdx().x.mul(4); + const row = first_row.add(tid.div(32)); + // Padded warps compute a valid dummy row and suppress their final stores. + const query = row.div(group).minimum(cfg.query_length - 1); + const head = kv_head.mul(group).add(row.rem(group)); + const start = a.positions.get(.{batch}); + const position = if (cfg.causal) start.add(query) else b.constant(.i32, cfg.key_length - 1); + const live_end = if (cfg.causal) + start.add(first_row.add(3).div(group).minimum(cfg.query_length - 1)).add(1) + else + b.constant(.i32, cfg.key_length); + // Partition the live cache, rather than its capacity, so decode uses all + // blocks even when only a prefix of the cache is populated. + const chunk = live_end.add(cfg.splits - 1).div(cfg.splits); + const begin = if (split) b.blockIdx().z.mul(chunk) else b.constant(.i32, 0); + const end = if (split) begin.add(chunk).minimum(live_end) else live_end; + const zero = b.constant(.f32, 0); + const neg_inf = b.constant(.f32, -std.math.inf(f32)); + + var queries: [fragments]Value = undefined; + inline for (0..fragments) |f| { + const d = lane.add(f * 32); + var valid = b.openIfElse(d.lt(cfg.head_dim), .{.f32}); + { + valid.yieldThen(.{a.q.get(.{ batch, query, head, d }).to(.f32)}); + } + { + valid.yieldElse(.{zero}); + } + queries[f] = valid.results[0]; + } + + const layout = b.makeLayout(&.{ tile_keys, padded_dim }, .{ .stride = &.{ padded_dim, 1 } }); + const keys = b.makeTensor(b.allocSmem(cfg.dtype, tile_keys * padded_dim, 16), layout); + const values = b.makeTensor(b.allocSmem(cfg.dtype, tile_keys * padded_dim, 16), layout); + var initial: [fragments + 2]Value = @splat(zero); + initial[0] = neg_inf; + var tiles = b.openFor(begin, end, tile_keys, initial); + { + + // Zero-fill partial tiles and feature padding before the CTA-wide barrier. + var copy = b.openFor(tid, tile_keys * padded_dim, 128, .{}); + { + const tile_key = copy.iv.div(padded_dim); + const d = copy.iv.rem(padded_dim); + const key = tiles.iv.add(tile_key); + var valid_load = b.openIfElse(key.lt(end).bitAnd(d.lt(cfg.head_dim)), .{ cfg.dtype, cfg.dtype }); + { + valid_load.yieldThen(.{ a.k.get(.{ batch, key, kv_head, d }), a.v.get(.{ batch, key, kv_head, d }) }); + } + { + valid_load.yieldElse(.{ zero.to(cfg.dtype), zero.to(cfg.dtype) }); + } + keys.set(.{ tile_key, d }, valid_load.results[0]); + values.set(.{ tile_key, d }, valid_load.results[1]); + copy.close(); + } + b.syncThreads(); + + // Each lane keeps one score; dot products reduce across the feature lanes. + var scores = b.openFor(0, tile_keys, 1, .{neg_inf}); + { + var dot = zero; + inline for (0..fragments) |f| { + dot = dot.add(queries[f].mul(keys.get(.{ scores.iv, lane.add(f * 32) }).to(.f32))); + } + inline for (.{ 16, 8, 4, 2, 1 }) |offset| dot = dot.add(b.shuffleXor(dot, offset)); + const scale = std.math.log2e / std.math.sqrt(@as(f32, @floatFromInt(cfg.head_dim))); + const score = b.select(tiles.iv.add(scores.iv).le(position).bitAnd(tiles.iv.add(scores.iv).lt(end)), dot.mul(scale), neg_inf); + scores.yield(.{b.select(lane.eq(scores.iv), score, scores.carried[0])}); + } + var maximum = scores.results[0]; + inline for (.{ 16, 8, 4, 2, 1 }) |offset| maximum = maximum.maximum(b.shuffleXor(maximum, offset)); + maximum = maximum.maximum(tiles.carried[0]); + const rescale = b.exp2Approx(tiles.carried[0].sub(maximum)); + const probability = b.exp2Approx(scores.results[0].sub(maximum)); + var sum = probability; + inline for (.{ 16, 8, 4, 2, 1 }) |offset| sum = sum.add(b.shuffleXor(sum, offset)); + sum = sum.add(tiles.carried[1].mul(rescale)); + + var accumulators: [fragments]Value = undefined; + inline for (0..fragments) |f| accumulators[f] = tiles.carried[f + 2].mul(rescale); + var weighted = b.openFor(0, tile_keys, 1, accumulators); + { + const weight = b.shuffleIdx(probability, weighted.iv); + inline for (0..fragments) |f| { + accumulators[f] = weighted.carried[f].add(weight.mul(values.get(.{ weighted.iv, lane.add(f * 32) }).to(.f32))); + } + weighted.yield(accumulators); + } + // No warp may overwrite the tile before every warp has consumed it. + b.syncThreads(); + var next: [fragments + 2]Value = undefined; + next[0] = maximum; + next[1] = sum; + inline for (0..fragments) |f| next[f + 2] = weighted.results[f]; + tiles.yield(next); + } + + inline for (0..fragments) |f| { + const col = lane.add(f * 32); + var valid_store = b.openIf(row.lt(cfg.query_length * group).bitAnd(col.lt(cfg.head_dim))); + { + if (split) { + a.out.set(.{ batch, b.blockIdx().z, query, head, col }, tiles.results[f + 2]); + } else { + a.out.set(.{ batch, query, head, col }, tiles.results[f + 2].div(tiles.results[1])); + } + valid_store.close(); + } + } + if (split) { + var stats = b.openIf(row.lt(cfg.query_length * group).bitAnd(lane.eq(0))); + { + a.out.set(.{ batch, b.blockIdx().z, query, head, cfg.head_dim }, tiles.results[0]); + a.out.set(.{ batch, b.blockIdx().z, query, head, cfg.head_dim + 1 }, tiles.results[1]); + stats.close(); + } + } +} + +/// Merge unnormalized partial attention states in FP32. Empty partitions have +/// m=-inf, l=0, u=0 and contribute zero, including at the first decode token. +pub const Merge = cute.Kernel(Config, .{ + .name = "flash_attention_cute_merge", + .inputs = &.{"partials"}, + .outputs = &.{"out"}, + .run = merge, +}); + +fn merge(b: *B, cfg: Config) cute.FinishError!void { + const a = try b.declareArgs(.{ + .partials = .{ .tensor = .{ .dtype = .f32, .shape = &.{ cfg.batch, cfg.splits, cfg.query_heads, cfg.head_dim + 2 } } }, + .out = .{ .tensor = .{ .dtype = cfg.dtype, .shape = &.{ cfg.batch, cfg.query_heads, cfg.head_dim } } }, + }); + const batch = b.blockIdx().y; + const head = b.blockIdx().x; + const lane = b.threadIdx().x.rem(32); + const zero = b.constant(.f32, 0); + const neg_inf = b.constant(.f32, -std.math.inf(f32)); + // Every warp reduces the partition statistics, then independently handles + // a contiguous range of output features. + var maxima = b.openFor(lane, cfg.splits, 32, .{neg_inf}); + { + maxima.yield(.{maxima.carried[0].maximum(a.partials.get(.{ batch, maxima.iv, head, cfg.head_dim }))}); + } + var maximum = maxima.results[0]; + inline for (.{ 16, 8, 4, 2, 1 }) |offset| maximum = maximum.maximum(b.shuffleXor(maximum, offset)); + const weights = b.makeTensor(b.allocSmem(.f32, cfg.splits, 16), b.makeLayout(&.{cfg.splits}, .{})); + var stats = b.openFor(b.threadIdx().x, cfg.splits, 128, .{}); + { + weights.set(.{stats.iv}, b.exp2Approx(a.partials.get(.{ batch, stats.iv, head, cfg.head_dim }).sub(maximum))); + stats.close(); + } + b.syncThreads(); + var features = b.openFor(b.threadIdx().x, cfg.head_dim, 128, .{}); + { + var sum = b.openFor(0, cfg.splits, 1, .{ zero, zero }); + { + const weight = weights.get(.{sum.iv}); + sum.yield(.{ + sum.carried[0].add(weight.mul(a.partials.get(.{ batch, sum.iv, head, features.iv }))), + sum.carried[1].add(weight.mul(a.partials.get(.{ batch, sum.iv, head, cfg.head_dim + 1 }))), + }); + } + a.out.set(.{ batch, head, features.iv }, sum.results[0].div(sum.results[1])); + features.close(); + } +} + +// ==================== Prefill ==================== +// +// FlashAttention-2 forward for `query_length > 1`, with the tiling FA2's sm80 kernel uses on sm_120 +// (`Flash_fwd_kernel_traits<128, 128, 64, 4>`): a 128-thread block owns `prefillRows` queries of one +// head and every warp owns whole rows (all the keys of a tile, the whole head dimension). Scores and +// output stay in `mma.sync` accumulators, so P is reused from registers as the A operand of P @ V, +// without cross-warp reductions. Q, K and V tiles go through swizzled shared memory: V_j is copied +// while S_j = Q K_j^T is computed, K_{j+1} while O += P_j V_j is. +// +// It is a program, not a bare kernel: its host function launches the kernel with the dynamic shared +// memory the kernel allocates (up to 96 KiB). A bare kernel's shared memory is static, which ptxas +// caps at 48 KiB before sm_120. + +pub const Prefill = cute.Program(Config, .{ + .name = "flash_attention_cute_prefill", + .inputs = &.{ "q", "k", "v", "positions" }, + .outputs = &.{"out"}, + .run = prefillProgram, +}); + +const prefill_kernel_name = "flash_attention_cute_prefill_kernel"; + +/// Keys per tile (FA2's kBlockN). +const prefill_keys = 64; + +/// Whether `Prefill` handles `cfg`; the other configurations use `Kernel`. +/// Same as FA2 we only support 16-bit inputs +pub fn prefillSupported(cfg: Config) bool { + const dtype_ok = cfg.dtype == .bf16 or cfg.dtype == .f16; + const head_dim_ok = cfg.head_dim == 32 or cfg.head_dim == 64 or cfg.head_dim == 128 or cfg.head_dim == 256; + return cfg.query_length > 1 and cfg.splits == 1 and dtype_ok and head_dim_ok; +} + +/// m16 row tiles per warp: two up to head dimension 128, one above so the output accumulators fit in registers. +fn prefillRowTiles(head_dim: i64) i64 { + return if (head_dim <= 128) 2 else 1; +} + +/// Queries per block (FA2's kBlockM). +fn prefillRows(cfg: Config) i64 { + return 4 * 16 * prefillRowTiles(cfg.head_dim); +} + +/// Query tiles on x, so the blocks resident at the same time share the K/V of one head in L2. +fn prefillGrid(cfg: Config) [3]i32 { + return .{ @intCast(@divTrunc(cfg.query_length + prefillRows(cfg) - 1, prefillRows(cfg))), @intCast(cfg.query_heads), @intCast(cfg.batch) }; +} + +fn prefillProgram(b: *B, cfg: Config) cute.FinishError!void { + std.debug.assert(prefillSupported(cfg)); + switch (cfg.head_dim) { + inline 32, 64, 128, 256 => |d| switch (cfg.dtype) { + inline .bf16, .f16 => |dt| { + b.beginFunction(prefill_kernel_name, .cuda_kernel); + try prefillSized(b, cfg, d, dt); + b.endFunction(block); + + b.beginFunction(Prefill.name, .host); + const host = try b.declareArgs(.{ + .q = .{ .ptr = dt }, + .k = .{ .ptr = dt }, + .v = .{ .ptr = dt }, + .positions = .{ .ptr = cute.DType.i32 }, + .out = .{ .ptr = dt }, + }); + const grid = prefillGrid(cfg); + const config = b.makeLaunchConfig(.{ + .grid = .{ b.constant(.i32, grid[0]), b.constant(.i32, grid[1]), b.constant(.i32, grid[2]) }, + .block = .{ b.constant(.i32, block[0]), b.constant(.i32, block[1]), b.constant(.i32, block[2]) }, + .dynamic_smem = b.kernelSmemSize(prefill_kernel_name), + .stream = b.cudaStream(), + }); + const launch = b.launchEx(prefill_kernel_name, config, .{ host.q, host.k, host.v, host.positions, host.out }); + b.returnHostStatus(b.cudaResultStatus(launch)); + b.endFunction(null); + }, + else => unreachable, + }, + else => unreachable, + } +} + +/// PTX `max.f32` (FMNMX). `Value.maximum` and `Value.maxnum` both lower to NaN-handling compare and +/// select sequences (three instructions); prefill scores are never NaN, masked ones are -inf. +fn fmax(b: *B, x: Value, y: Value) Value { + return b.callIntrinsic("llvm.nvvm.fmax.f", &.{ x, y }, .float(b.ctx, .f32)).?; +} + +fn unpackStruct(b: *B, comptime n: usize, value: Value, dtype: cute.DType) [n]Value { + var result: [n]Value = undefined; + for (&result, 0..) |*r, i| { + r.* = b.emit(llvm.extractvalue(b.ctx, value.inner, dtype.toMlir(b.ctx), &.{@intCast(i)}, b.loc())); + } + return result; +} + +/// `ldmatrix.x4`: lanes 8i..8i+7 address the rows of matrix i, the result holds the four 8x8 fragments. +fn matrixLoad(b: *B, comptime transpose: bool, address: Value) [4]Value { + const assembly = "ldmatrix.sync.aligned.m8n8.x4" ++ (if (transpose) ".trans" else "") ++ ".shared.b16 {$0, $1, $2, $3}, [$4];"; + const result = b.emit(llvm.inline_asm(b.ctx, assembly, &.{address.inner}, .{ + .result_type = llvm.structType(b.ctx, &@as([4]*const mlir.Type, @splat(.int(b.ctx, .i32)))), + .constraints = "=r,=r,=r,=r,r", + .has_side_effects = true, + }, b.loc())); + return unpackStruct(b, 4, result, .i32); +} + +/// `mma.sync.m16n8k16` with f32 accumulators: acc += a (16x16, row) @ b (16x8, col). +fn mma(b: *B, comptime dtype: cute.DType, a: [4]Value, rhs: [2]Value, acc: [4]Value) [4]Value { + const assembly = "mma.sync.aligned.m16n8k16.row.col.f32." ++ @tagName(dtype) ++ "." ++ @tagName(dtype) ++ ".f32 " ++ + "{$0, $1, $2, $3}, {$4, $5, $6, $7}, {$8, $9}, {$10, $11, $12, $13};"; + const result = b.emit(llvm.inline_asm(b.ctx, assembly, &.{ a[0].inner, a[1].inner, a[2].inner, a[3].inner, rhs[0].inner, rhs[1].inner, acc[0].inner, acc[1].inner, acc[2].inner, acc[3].inner }, .{ + .result_type = llvm.structType(b.ctx, &@as([4]*const mlir.Type, @splat(.float(b.ctx, .f32)))), + .constraints = "=f,=f,=f,=f,r,r,r,r,r,r,f,f,f,f", + }, b.loc())); + return unpackStruct(b, 4, result, .f32); +} + +/// The four f32 accumulators of one mma.sync, carried across loop iterations as one value (the loop +/// carries at most 64 values). +fn packAcc(b: *B, values: [4]Value) Value { + return b.emit(vector.from_elements(b.ctx, &.{ values[0].inner, values[1].inner, values[2].inner, values[3].inner }, .vector(&.{4}, .float(b.ctx, .f32)), b.loc())); +} + +fn unpackAcc(b: *B, value: Value) [4]Value { + var result: [4]Value = undefined; + for (&result, 0..) |*r, i| { + r.* = b.vectorExtract(value, @intCast(i), .f32); + } + return result; +} + +/// Comptime layout of one prefill block, shared by the setup and the per-tile steps. +/// +/// Tiles are row major with the 16-byte vector index of every row XORed with row % swizzle_rows, so +/// 16-byte cp.async stores and 8-row ldmatrix reads are free of bank conflicts. Shared memory addresses +/// are folded the way CuTe's static layouts let nvcc fold them: the XORed row bits of every address a +/// thread uses are a per-thread constant, and the k-step part of a column offset is bit-disjoint from +/// the per-thread part, so each ldmatrix / cp.async address is a per-thread base plus a constant +/// (XORed into the 16-byte vector index). +fn PrefillTile(comptime d: i64, comptime dt: cute.DType) type { + return struct { + const row_tiles = prefillRowTiles(d); + const rows = 4 * 16 * row_tiles; + const n = prefill_keys; + const score_tiles = n / 8; + const out_tiles = d / 8; + const row_bytes = d * 2; + /// The XORed vector index must stay in its row: rows of d=32 only have 4 vectors. + const swizzle_rows = @min(8, d / 8); + const q_bytes = rows * row_bytes; + const kv_bytes = n * row_bytes; + /// Rows covered by one 16-byte cp.async per thread of the block. + const copy_rows = @divExact(128, d / 8); + /// Running max and per-thread partial sum of the two rows of every row tile. + const stats = 2 * row_tiles; + /// Carried by the key loop: maxima, sums, then the packed output accumulators of every + /// (row tile, 8 output features). + const carried = 2 * stats + row_tiles * out_tiles; + const scale: f32 = std.math.log2e / std.math.sqrt(@as(f32, @floatFromInt(d))); + + comptime { + // copy_rows rows starting at a multiple of copy_rows never carry into the XORed bits. + std.debug.assert(std.math.isPowerOfTwo(copy_rows)); + std.debug.assert(std.math.isPowerOfTwo(swizzle_rows)); + } + + b: *B, + cfg: Config, + lane: Value, + warp: Value, + first_query: Value, + start: Value, + q_smem: Value, + k_smem: Value, + v_smem: Value, + k_ptr: Value, + v_ptr: Value, + kv_base: Value, + tiles: Value, + /// cp.async: this thread copies the 16-byte vector `copy_col` of rows `copy_row + i * copy_rows`. + copy_row: Value, + copy_col: Value, + /// Byte offset in a tile of this thread's first cp.async destination, without the XORed part. + copy_row_offset: Value, + copy_vector_offset: Value, + /// ldmatrix rows of this thread, as byte offsets, and their XORed 16-byte vector offsets. + q_frag_row: Value, + k_frag_row: Value, + v_frag_row: Value, + qv_frag_vector: Value, + k_frag_vector: Value, + + const Self = @This(); + + fn init(b: *B, cfg: Config, tid: Value, first_query: Value, start: Value, shared: Value, k_ptr: Value, v_ptr: Value, kv_base: Value, tiles: Value) Self { + const lane = tid.rem(32); + const warp = tid.div(32); + const copy_row = tid.div(d / 8); + const copy_col = tid.rem(d / 8); + const lane8 = lane.rem(8); + // The fragment rows a thread reads are lane % 8 plus multiples of 8. + const lane_xor = lane.rem(swizzle_rows); + return .{ + .b = b, + .cfg = cfg, + .lane = lane, + .warp = warp, + .first_query = first_query, + .start = start, + .q_smem = shared, + .k_smem = shared.add(q_bytes), + .v_smem = shared.add(q_bytes + kv_bytes), + .k_ptr = k_ptr, + .v_ptr = v_ptr, + .kv_base = kv_base, + .tiles = tiles, + .copy_row = copy_row, + .copy_col = copy_col, + .copy_row_offset = copy_row.mul(row_bytes), + .copy_vector_offset = copy_col.bitXor(copy_row.rem(swizzle_rows)).mul(16), + // Q (A operand): rows lane % 16, vectors lane / 16 (+ 2 per k-step). + .q_frag_row = shared.add(warp.mul(16 * row_tiles).add(lane.rem(16)).mul(row_bytes)), + // K (B operand): keys lane % 8 + 8 * (lane / 16), vectors (lane / 8) % 2 (+ 2 per k-step). + .k_frag_row = shared.add(q_bytes).add(lane8.add(lane.div(16).mul(8)).mul(row_bytes)), + // V (B operand, transposed): keys lane % 8 + 8 * ((lane / 8) % 2), vectors lane / 16 (+ 2 per 16 features). + .v_frag_row = shared.add(q_bytes + kv_bytes).add(lane8.add(lane.div(8).rem(2).mul(8)).mul(row_bytes)), + .qv_frag_vector = lane.div(16).bitXor(lane_xor).mul(16), + .k_frag_vector = lane.div(8).rem(2).bitXor(lane_xor).mul(16), + }; + } + + /// Destination of the i-th cp.async of this thread in a tile starting at `base`. + fn copyDst(t: Self, base: Value, comptime i: usize) Value { + const xor = ((copy_rows * i) % swizzle_rows) * 16; + const vector_offset = if (xor == 0) t.copy_vector_offset else t.copy_vector_offset.bitXor(xor); + return base.add(t.copy_row_offset).add(vector_offset).add(copy_rows * i * row_bytes); + } + + /// Copy keys `key_base..key_base + n` of `src` into `dst`; keys past the end are zero-filled. + fn loadTile(t: Self, src: Value, dst: Value, key_base: Value) void { + const b = t.b; + const stride = t.cfg.kv_heads * d; + const first_key = key_base.add(t.copy_row); + const safe = src.add(t.kv_base.mul(2)); + const first = safe.add(first_key.to(.i64).mul(stride).add(t.copy_col.mul(8).to(.i64)).mul(2)); + inline for (0..@divExact(n, copy_rows)) |i| { + const valid = first_key.add(copy_rows * i).lt(t.cfg.key_length); + const source = b.select(valid, first.add(copy_rows * i * stride * 2), safe); + b.cpAsync16(t.copyDst(dst, i), source, valid); + } + } + + fn queryOf(t: Self, mi: usize, h: usize) Value { + return t.first_query.add(t.warp.mul(16 * row_tiles)).add(mi * 16).add(t.lane.div(4)).add(h * 8); + } + + fn accIndex(comptime mi: usize, comptime di: usize) usize { + return @intCast(2 * stats + @as(i64, mi) * out_tiles + @as(i64, di)); + } + + /// One key tile. `masked` tiles apply the causal and key length masks; the other ones are + /// fully visible to every query of the block. + fn step(t: Self, comptime masked: bool, tile: Value, carried_in: [carried]Value) [carried]Value { + const b = t.b; + const lane = t.lane; + const zero = b.constant(.f32, 0); + const neg_inf = b.constant(.f32, -std.math.inf(f32)); + const key_base = tile.mul(n); + b.cpAsyncWaitAll(); + b.syncThreads(); + t.loadTile(t.v_ptr, t.v_smem, key_base); + + // S = Q K^T, k-steps of 16 features. + var s: [row_tiles][score_tiles][4]Value = @splat(@splat(@splat(zero))); + inline for (0..d / 16) |ks| { + var qa: [row_tiles][4]Value = undefined; + inline for (0..row_tiles) |mi| { + qa[mi] = matrixLoad(b, false, t.q_frag_row.add(mi * 16 * row_bytes).add(t.qv_frag_vector.bitXor(ks * 32))); + } + const k_vector = t.k_frag_vector.bitXor(ks * 32); + inline for (0..score_tiles / 2) |np| { + const kb = matrixLoad(b, false, t.k_frag_row.add(np * 16 * row_bytes).add(k_vector)); + inline for (0..row_tiles) |mi| { + s[mi][2 * np] = mma(b, dt, qa[mi], .{ kb[0], kb[1] }, s[mi][2 * np]); + s[mi][2 * np + 1] = mma(b, dt, qa[mi], .{ kb[2], kb[3] }, s[mi][2 * np + 1]); + } + } + } + + // Online softmax in base 2 on the raw scores, the scale is folded into the exponent: + // p = exp2(s * scale - m * scale). Thread rows: lane / 4 (accumulators 0, 1) and lane / 4 + 8 (2, 3). + var next: [carried]Value = undefined; + var alpha: [row_tiles][2]Value = undefined; + inline for (0..row_tiles) |mi| { + if (masked) { + inline for (0..score_tiles) |ni| { + inline for (0..4) |j| { + const key = key_base.add(ni * 8).add(lane.rem(4).mul(2)).add(j % 2); + const in_range = key.lt(t.cfg.key_length); + const visible = if (t.cfg.causal) in_range.bitAnd(key.le(t.start.add(t.queryOf(mi, j / 2)))) else in_range; + s[mi][ni][j] = b.select(visible, s[mi][ni][j], neg_inf); + } + } + } + inline for (0..2) |h| { + var row_max = s[mi][0][h * 2]; + inline for (0..score_tiles) |ni| { + inline for (0..2) |j| { + row_max = fmax(b, row_max, s[mi][ni][h * 2 + j]); + } + } + inline for (.{ 2, 1 }) |offset| { + row_max = fmax(b, row_max, b.shuffleXor(row_max, offset)); + } + const old_max = carried_in[mi * 2 + h]; + const new_max = fmax(b, old_max, row_max); + // A row without any visible key so far keeps m = -inf: exponentiate against a finite value. + const neg_scaled_max = fmax(b, new_max, b.constant(.f32, -std.math.floatMax(f32))).mul(-scale); + alpha[mi][h] = b.exp2Approx(old_max.mul(scale).add(neg_scaled_max)); + next[mi * 2 + h] = new_max; + var sum = carried_in[stats + mi * 2 + h].mul(alpha[mi][h]); + inline for (0..score_tiles) |ni| { + inline for (0..2) |j| { + s[mi][ni][h * 2 + j] = b.exp2Approx(s[mi][ni][h * 2 + j].mul(scale).add(neg_scaled_max)); + sum = sum.add(s[mi][ni][h * 2 + j]); + } + } + next[stats + mi * 2 + h] = sum; + } + } + + // All warps are done with K_j once V_j has landed: start copying K_{j+1}. + b.cpAsyncWaitAll(); + b.syncThreads(); + var prefetch = b.openIf(tile.add(1).lt(t.tiles)); + { + t.loadTile(t.k_ptr, t.k_smem, key_base.add(n)); + prefetch.close(); + } + + // O = O * alpha + P V. The accumulators of two adjacent score tiles are the A operand of one k-step. + var o: [row_tiles][out_tiles][4]Value = undefined; + inline for (0..row_tiles) |mi| { + inline for (0..out_tiles) |di| { + const prev = unpackAcc(b, carried_in[accIndex(mi, di)]); + inline for (0..4) |j| { + o[mi][di][j] = prev[j].mul(alpha[mi][j / 2]); + } + } + } + inline for (0..n / 16) |ks| { + var pa: [row_tiles][4]Value = undefined; + inline for (0..row_tiles) |mi| { + pa[mi] = .{ + b.packHalf2(s[mi][2 * ks][0], s[mi][2 * ks][1], dt), + b.packHalf2(s[mi][2 * ks][2], s[mi][2 * ks][3], dt), + b.packHalf2(s[mi][2 * ks + 1][0], s[mi][2 * ks + 1][1], dt), + b.packHalf2(s[mi][2 * ks + 1][2], s[mi][2 * ks + 1][3], dt), + }; + } + const v_row = t.v_frag_row.add(ks * 16 * row_bytes); + inline for (0..out_tiles / 2) |dp| { + const vb = matrixLoad(b, true, v_row.add(t.qv_frag_vector.bitXor(dp * 32))); + inline for (0..row_tiles) |mi| { + o[mi][2 * dp] = mma(b, dt, pa[mi], .{ vb[0], vb[1] }, o[mi][2 * dp]); + o[mi][2 * dp + 1] = mma(b, dt, pa[mi], .{ vb[2], vb[3] }, o[mi][2 * dp + 1]); + } + } + } + inline for (0..row_tiles) |mi| { + inline for (0..out_tiles) |di| { + next[accIndex(mi, di)] = packAcc(b, o[mi][di]); + } + } + return next; + } + }; +} + +fn prefillSized(b: *B, cfg: Config, comptime d: i64, comptime dt: cute.DType) cute.FinishError!void { + const T = PrefillTile(d, dt); + const a = try b.declareArgs(.{ + .q = .{ .ptr = dt }, + .k = .{ .ptr = dt }, + .v = .{ .ptr = dt }, + .positions = .{ .ptr = cute.DType.i32 }, + .out = .{ .ptr = dt }, + }); + b.setFunctionAttribute(.named(b.ctx, "cu_attrs", .dict(b.ctx, &.{.named(b.ctx, "max_dynamic_shared_size_bytes", b.devMaxSharedMemoryOptin())}))); + + const tid = b.threadIdx().x; + const first_query = b.blockIdx().x.mul(T.rows); + const head = b.blockIdx().y; + const batch = b.blockIdx().z; + const kv_head = head.div(@divExact(cfg.query_heads, cfg.kv_heads)); + // One shared storage object (like a Python CuTe `SharedStorage`): the host compiler sizes the + // launch's dynamic shared memory from it, which `kernelSmemSize` returns. `allocSmem` and + // `allocSmemView` both become static shared memory instead. + b.setFunctionAttribute(.named(b.ctx, "smem.partition_num", .int(b.ctx, .i32, 1))); + const fields = [_][]const u8{ + std.fmt.comptimePrint("sQ:{d}:0", .{T.q_bytes}), + std.fmt.comptimePrint("sK:{d}:{d}", .{ T.kv_bytes, T.q_bytes }), + std.fmt.comptimePrint("sV:{d}:{d}", .{ T.kv_bytes, T.q_bytes + T.kv_bytes }), + }; + const smem = b.allocSmemStorage(.i8, 128, b.layoutType(b.layoutSpec(.{T.q_bytes + 2 * T.kv_bytes}, .{1})), 0, &fields); + const shared = b.ptrToInt(b.getIter(smem, b.ptrTy(.i8, .smem, 128) catch @panic("bad shared memory pointer")).value(), .i32); + const q_ptr = b.ptrToInt(a.q, .i64); + const out_ptr = b.ptrToInt(a.out, .i64); + const start = b.loadWords(1, b.ptrToInt(a.positions, .i64).add(batch.to(.i64).mul(4)), .global, .{})[0]; + + // Element offsets of this (batch, head) in q/out and (batch, kv head) in k/v, and the q/out row stride. + const q_stride = cfg.query_heads * d; + const q_base = batch.to(.i64).mul(cfg.query_length * q_stride).add(head.to(.i64).mul(d)); + const kv_base = batch.to(.i64).mul(cfg.key_length * cfg.kv_heads * d).add(kv_head.to(.i64).mul(d)); + + // Causal: the last query of the block sees keys up to start + last_query, and every query of the + // block sees the keys up to start + first_query. Otherwise only the last tile, past the end of the + // keys, is masked. + const last_query = first_query.add(T.rows).minimum(cfg.query_length).sub(1); + const tiles = if (cfg.causal) + start.add(last_query).add(1).minimum(cfg.key_length).add(T.n - 1).div(T.n) + else + b.constant(.i32, @divTrunc(cfg.key_length + T.n - 1, T.n)); + const full_tiles = if (cfg.causal) + start.add(first_query).add(1).minimum(cfg.key_length).div(T.n).minimum(tiles) + else + b.constant(.i32, @divTrunc(cfg.key_length, T.n)); + + const t: T = .init(b, cfg, tid, first_query, start, shared, b.ptrToInt(a.k, .i64), b.ptrToInt(a.v, .i64), kv_base, tiles); + + // Out of range rows are zero-filled and never read from global memory. + { + const first_row = first_query.add(t.copy_row); + const safe = q_ptr.add(q_base.mul(2)); + const first = safe.add(first_row.to(.i64).mul(q_stride).add(t.copy_col.mul(8).to(.i64)).mul(2)); + inline for (0..@divExact(T.rows, T.copy_rows)) |i| { + const valid = first_row.add(T.copy_rows * i).lt(cfg.query_length); + b.cpAsync16(t.copyDst(t.q_smem, i), b.select(valid, first.add(T.copy_rows * i * q_stride * 2), safe), valid); + } + } + t.loadTile(t.k_ptr, t.k_smem, b.constant(.i32, 0)); + + var initial: [T.carried]Value = undefined; + const zero = b.constant(.f32, 0); + for (initial[0..T.stats]) |*v| { + v.* = b.constant(.f32, -std.math.inf(f32)); + } + for (initial[T.stats .. 2 * T.stats]) |*v| { + v.* = zero; + } + for (initial[2 * T.stats ..]) |*v| { + v.* = packAcc(b, .{ zero, zero, zero, zero }); + } + + var full = b.openFor(0, full_tiles, 1, initial); + { + full.yield(t.step(false, full.iv, full.carried[0..T.carried].*)); + } + var diagonal = b.openFor(full_tiles, tiles, 1, full.results[0..T.carried].*); + { + diagonal.yield(t.step(true, diagonal.iv, diagonal.carried[0..T.carried].*)); + } + const results = diagonal.results[0..T.carried].*; + + // Normalize by the row sums (reduced over the quad) and store pairs of features as one 32-bit word. + inline for (0..T.row_tiles) |mi| { + inline for (0..2) |h| { + var sum = results[T.stats + mi * 2 + h]; + inline for (.{ 2, 1 }) |offset| { + sum = sum.add(b.shuffleXor(sum, offset)); + } + const norm = b.select(sum.gt(0), b.rcpApprox(sum), zero); + const query = t.queryOf(mi, h); + var valid = b.openIf(query.lt(cfg.query_length)); + { + const row_offset = q_base.add(query.to(.i64).mul(q_stride)).add(t.lane.rem(4).mul(2).to(.i64)); + inline for (0..T.out_tiles) |di| { + const acc = unpackAcc(b, results[T.accIndex(mi, di)]); + const word = b.packHalf2(acc[h * 2].mul(norm), acc[h * 2 + 1].mul(norm), dt); + b.storeWords(1, out_ptr.add(row_offset.add(di * 8).mul(2)), .global, .{word}); + } + valid.close(); + } + } + } +} + +test "CuTe attention emits tiled online softmax through head dimension 512" { + for ([_]i64{ 64, 80, 128, 256, 512 }) |head_dim| { + const ir = try Kernel.emit(std.testing.allocator, .{ + .batch = 2, + .query_length = 7, + .key_length = 37, + .query_heads = 6, + .kv_heads = 2, + .head_dim = head_dim, + .dtype = .bf16, + }, block); + defer std.testing.allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "scf.for") != null); + } +} diff --git a/zml/kernels/flash_attention/cute/cute.zig b/zml/kernels/flash_attention/cute/cute.zig new file mode 100644 index 00000000..36e0953b --- /dev/null +++ b/zml/kernels/flash_attention/cute/cute.zig @@ -0,0 +1,96 @@ +const std = @import("std"); + +const stdx = @import("stdx"); + +const zml = @import("../../../zml.zig"); +const kernel = @import("attention.zig"); + +test { + _ = kernel; +} + +pub fn isAvailable(platform: *const zml.Platform) bool { + if (platform.target != .cuda) return false; + const cc = zml.platform.cuda.computeCapability(platform) orelse return false; + return cc.major >= 8; +} + +/// `causal`: query j sees the keys up to token_index + j. Otherwise every key, and token_index is ignored. +pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, causal: bool) zml.Tensor { + stdx.debug.assert(q.shape().hasTags(.{ .q, .h, .hd }) and k.shape().hasTags(.{ .k, .h, .hd }) and v.shape().hasTags(.{ .k, .h, .hd }), "CuTe attention requires q/h/hd and k/h/hd tags", .{}); + const batched = q.shape().hasTag(.b) != null; + const rank: usize = if (batched) 4 else 3; + stdx.debug.assert(q.rank() == rank and k.rank() == rank and v.rank() == rank, "CuTe attention expects rank-3 tensors or rank-4 tensors with a batch axis", .{}); + stdx.debug.assert(q.dtype() == k.dtype() and q.dtype() == v.dtype(), "CuTe attention requires matching Q/K/V dtypes", .{}); + stdx.debug.assert(q.dtype() == .bf16 or q.dtype() == .f16 or q.dtype() == .f32, "CuTe attention supports bf16, f16, and f32", .{}); + stdx.debug.assert(q.dim(.hd) > 0 and q.dim(.hd) <= 512 and q.dim(.hd) == k.dim(.hd) and k.dim(.hd) == v.dim(.hd), "CuTe attention requires matching head dimensions in [1, 512]", .{}); + stdx.debug.assert(k.dim(.h) > 0 and q.dim(.h) > 0 and @rem(q.dim(.h), k.dim(.h)) == 0 and k.dim(.h) == v.dim(.h), "CuTe attention requires Hq divisible by Hkv and matching K/V heads", .{}); + stdx.debug.assert(q.dim(.q) > 0 and (!causal or q.dim(.q) <= k.dim(.k)) and k.dim(.k) == v.dim(.k), "CuTe attention requires 0 < Lq, Lq <= Lk when causal, and matching K/V lengths", .{}); + stdx.debug.assert(token_index.dtype().isInteger(), "CuTe attention requires integer token positions", .{}); + if (batched) { + stdx.debug.assert(k.shape().hasTag(.b) != null and v.shape().hasTag(.b) != null and q.dim(.b) == k.dim(.b) and q.dim(.b) == v.dim(.b), "CuTe attention requires matching batch dimensions", .{}); + stdx.debug.assert(token_index.rank() == 0 or (token_index.rank() == 1 and token_index.shape().hasTag(.b) != null and token_index.dim(.b) == q.dim(.b)), "CuTe attention requires a scalar position or one position per batch element", .{}); + } else { + stdx.debug.assert(token_index.rank() == 0, "Unbatched CuTe attention requires a scalar token position", .{}); + } + + const positions = if (batched) + token_index.convert(.i32).broad(zml.Shape.init(.{ .b = q.dim(.b) }, .i32)) + else + token_index.convert(.i32); + const q_ = if (batched) q.transpose(.{ .b, .q, .h, .hd }) else q.transpose(.{ .q, .h, .hd }); + const k_ = if (batched) k.transpose(.{ .b, .k, .h, .hd }) else k.transpose(.{ .k, .h, .hd }); + const v_ = if (batched) v.transpose(.{ .b, .k, .h, .hd }) else v.transpose(.{ .k, .h, .hd }); + + const splits = kernel.decodeSplits(q_.shape(), k_.shape(), decodeResources(zml.Compiler.current().platform)); + + const cfg: kernel.Config = .{ + .batch = if (batched) q_.dim(.b) else 1, + .query_length = q_.dim(.q), + .key_length = k_.dim(.k), + .query_heads = q_.dim(.h), + .kv_heads = k_.dim(.h), + .head_dim = q_.dim(.hd), + .dtype = zml.kernel.cute.from(q_.dtype()), + .splits = splits, + .causal = causal, + }; + + const out = if (kernel.prefillSupported(cfg)) + kernel.Prefill.call(.{ .q = q_, .k = k_, .v = v_, .positions = positions }, .{ .out = q_.shape() }, .{ .cfg = cfg }).out + else if (cfg.splits > 1) b: { + const partials = kernel.Kernel.call(.{ .q = q_, .k = k_, .v = v_, .positions = positions }, .{ + .out = zml.Shape.init(.{ cfg.batch, cfg.splits, cfg.query_length, cfg.query_heads, cfg.head_dim + 2 }, .f32), + }, .{ + .cfg = cfg, + .grid = .{ @intCast(@divTrunc(@divExact(cfg.query_heads, cfg.kv_heads) + 3, 4)), @intCast(cfg.kv_heads * cfg.batch), @intCast(cfg.splits) }, + .block = kernel.block, + }).out; + break :b kernel.Merge.call(.{ .partials = partials }, .{ .out = q_.shape() }, .{ + .cfg = cfg, + .grid = .{ @intCast(cfg.query_heads), @intCast(cfg.batch), 1 }, + .block = kernel.block, + }).out; + } else kernel.Kernel.call(.{ .q = q_, .k = k_, .v = v_, .positions = positions }, .{ .out = q_.shape() }, .{ + .cfg = cfg, + .grid = .{ @intCast(@divTrunc(cfg.query_length * @divExact(cfg.query_heads, cfg.kv_heads) + 3, 4)), @intCast(cfg.kv_heads * cfg.batch), 1 }, + .block = kernel.block, + }).out; + + return out.transpose(q.shape()); +} + +/// Returns the minimum number of multiprocessors and shared memory per block across all devices in the platform. +/// If there are no devices, returns 0 for both values. +/// If any device does not have the required attributes, returns 0 for both values. +fn decodeResources(platform: *const zml.Platform) kernel.DecodeResources { + var result: kernel.DecodeResources = .{ .multiprocessors = std.math.maxInt(i64), .shared_bytes = std.math.maxInt(i64) }; + if (platform.devices.len == 0) return .{ .multiprocessors = 0, .shared_bytes = 0 }; + for (platform.devices) |device| { + const sms = device.pjrt_desc.attribute(platform.pjrt_api, "core_count") orelse return .{ .multiprocessors = 0, .shared_bytes = 0 }; + const shared = device.pjrt_desc.attribute(platform.pjrt_api, "shared_memory_per_block_optin") orelse return .{ .multiprocessors = 0, .shared_bytes = 0 }; + result.multiprocessors = @min(result.multiprocessors, sms.int64); + result.shared_bytes = @min(result.shared_bytes, shared.int64); + } + return result; +} diff --git a/zml/kernels/flash_attention/flash_attention.zig b/zml/kernels/flash_attention/flash_attention.zig new file mode 100644 index 00000000..3fb898db --- /dev/null +++ b/zml/kernels/flash_attention/flash_attention.zig @@ -0,0 +1,139 @@ +const std = @import("std"); + +const stdx = @import("stdx"); + +pub const cute = @import("cute/cute.zig"); +pub const metal = @import("metal/metal.zig"); +pub const nki = @import("nki/nki.zig"); +const zml = @import("../../zml.zig"); +const Options = @import("../../attention/Options.zig"); + +test { + std.testing.refAllDecls(@This()); + _ = @import("tests.zig"); +} + +pub fn isAvailable(platform: *const zml.Platform) bool { + return switch (platform.target) { + .neuron => true, + .metal => true, + .cuda => if (cute.isAvailable(platform)) true else false, + .cpu, .rocm, .tpu, .oneapi, .furiosa => false, + }; +} + +pub const Backend = enum { + cute, + nki, + metal, + + pub fn auto(platform: *const zml.Platform) Backend { + return switch (platform.target) { + .neuron => .nki, + .metal => .metal, + .cuda => .cute, + .cpu, .rocm, .tpu, .oneapi, .furiosa => stdx.debug.panic("Flash attention is not available on the {} platform", .{platform.target}), + }; + } + + pub fn isAvailable(backend: Backend, platform: *const zml.Platform) bool { + return switch (backend) { + .cute => cute.isAvailable(platform), + .nki => platform.target == .neuron, + .metal => platform.target == .metal, + }; + } +}; + +pub const Parameters = union(Backend) { + cute: void, + nki: nki.Parameters, + metal: void, + + pub const InitOptions = union(Backend) { + cute: void, + nki: nki.Parameters, + metal: void, + + pub fn fromBackend(backend: Backend) InitOptions { + return switch (backend) { + .cute => .{ .cute = {} }, + .nki => .{ .nki = .init() }, + .metal => .{ .metal = {} }, + }; + } + }; + + pub fn init(opts: InitOptions) Parameters { + return switch (opts) { + .cute => .{ .cute = {} }, + .nki => |v| .{ .nki = v }, + .metal => .{ .metal = {} }, + }; + } +}; + +pub const Metadata = union(Backend) { + cute: void, + nki: void, + metal: metal.Metadata, + + pub const InitOptions = union(Backend) { + cute: void, + nki: void, + metal: void, + + pub fn fromBackend(backend: Backend) InitOptions { + return switch (backend) { + .cute => .{ .cute = {} }, + .nki => .{ .nki = {} }, + .metal => .{ .metal = {} }, + }; + } + }; + + pub fn init(opts: InitOptions) Metadata { + return switch (opts) { + .cute => .{ .cute = {} }, + .nki => .{ .nki = {} }, + .metal => .{ .metal = .init() }, + }; + } + + pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(Metadata) { + return switch (self) { + .cute => .{ .cute = {} }, + .nki => .{ .nki = {} }, + inline else => |v, tag| @unionInit(zml.Bufferized(Metadata), @tagName(tag), try v.initBuffer(io, platform)), + }; + } + + pub fn deinitBuffer(self: *zml.Bufferized(Metadata)) void { + switch (self.*) { + .cute => {}, + .nki => {}, + .metal => |*v| metal.Metadata.deinitBuffer(v), + } + } +}; + +/// Causal attention as used in transformers. +/// +/// **Shapes**: +/// - q, result: .{ .q, .h, .hd } +/// - k, v: .{ .k, .h, .hd } +/// +/// Where: +/// - .h is the number of head +/// - .q is the number of queries +/// - .k is the number of keys +/// - .hd is the head dimension +pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters, opts: Options) zml.Tensor { + zml.stdx.debug.assert(zml.Compiler.current().manual_computation_depth > 0, "Attention kernels require an explicit manualComputation boundary", .{}); + return switch (parameters) { + .cute => cute.attention(q, k, v, token_index, opts.causal), + // The NKI and Metal kernels are causal only. + .nki => if (opts.causal) nki.attention(q, k, v, token_index, parameters.nki) else zml.nn.sdpa(q, k, v, .{}), + .metal => if (opts.causal) metal.attention(q, k, v, token_index, metadata.metal) else zml.nn.sdpa(q, k, v, .{}), + }; +} diff --git a/zml/kernels/flash_attention/metal/BUILD.bazel b/zml/kernels/flash_attention/metal/BUILD.bazel new file mode 100644 index 00000000..f82ade45 --- /dev/null +++ b/zml/kernels/flash_attention/metal/BUILD.bazel @@ -0,0 +1,9 @@ +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "metal", + srcs = [ + "flash_attn.zig", + "metal.zig", + ], +) diff --git a/zml/kernels/flash_attention/metal/flash_attn.zig b/zml/kernels/flash_attention/metal/flash_attn.zig new file mode 100644 index 00000000..de4b68da --- /dev/null +++ b/zml/kernels/flash_attention/metal/flash_attn.zig @@ -0,0 +1,732 @@ +//! Metal flash attention over contiguous `[batch, keys, kv_heads, head_dim]` +//! K/V, causal from `token_index`: fa_vec and fa_vec_reduce for decode, +//! fa_tiled otherwise. + +const std = @import("std"); + +const metal = @import("../../../kernel/kernel.zig").metal; + +const ArgSpec = metal.ArgSpec; +const Builder = metal.Builder; +const Type = metal.Type; +const Value = metal.Value; + +const flt_max = std.math.floatMax(f32); +/// A masked score: below any real one yet finite, since fast math assumes +/// no infinities. Halved so subtracting a running maximum cannot overflow. +const masked_score = -flt_max / 2.0; +const max_half = 65504.0; +const simd_width = 32; + +/// fa_vec: bf16 decode, one query per batch row. Simdgroup s takes query head +/// s % heads of the threadgroup's KV head group and position split s / heads. +pub const FaVec = struct { + pub const Cfg = struct { + head_dim: i32, + heads: i32, + splits: i32, + }; + + pub const Args: metal.Struct = .{ .name = "fa_vec_args", .fields = &.{ + .{ .name = "num_heads", .type = .int }, + .{ .name = "n_kv", .type = .int }, + .{ .name = "k_len", .type = .int }, + .{ .name = "scale", .type = .float }, + } }; + + pub const HostArgs = extern struct { + num_heads: i32, + n_kv: i32, + k_len: i32, + scale: f32, + }; + + pub const Kernel = metal.Kernel(Cfg, .{ + .name = "fa_vec", + .inputs = &.{ "q", "k", "v", "token_index" }, + .outputs = &.{"dst"}, + .constants = &.{"args"}, + .run = run, + }); + + fn run(b: *Builder, cfg: Cfg) metal.FinishError!void { + switch (cfg.head_dim) { + 32, 64, 128, 256, 512 => {}, + else => std.debug.panic("fa_vec: head_dim {d} unsupported (32, 64, 128, 256 or 512)", .{cfg.head_dim}), + } + const char_in: ArgSpec = .{ .buffer = .{ .type = .char, .const_ = true } }; + const a = try b.declareArgs(.{ + .args = .{ .constant_struct = Args }, + .q = char_in, + .k = char_in, + .v = char_in, + .token_index = .{ .buffer = .{ .type = .int, .const_ = true } }, + .dst = .{ .buffer = .{ .type = .char } }, + .tgpig = .{ .threadgroup_position_in_grid = .uint3 }, + .tgpg = .{ .threadgroups_per_grid = .uint3 }, + .tiisg = .{ .thread_index_in_simdgroup = .ushort }, + .sgitg = .{ .simdgroup_index_in_threadgroup = .ushort }, + .nsg = .{ .simdgroups_per_threadgroup = .ushort }, + }); + impl(b, cfg, a); + } + + fn impl(b: *Builder, cfg: Cfg, a: anytype) void { + const D = cfg.head_dim; + const D4 = @divExact(D, 4); + // A K/V row spans NL lanes of NI float4s; a simdgroup reads R rows per step. + const NL = @min(D4, simd_width); + const R = @divExact(simd_width, NL); + const NI = @divExact(D4, NL); + const C = simd_width; + const HG = cfg.heads; + const NWG = cfg.splits; + const full: metal.LoopOpts = .{ .unroll = .full }; + + const num_heads = a.args.field("num_heads"); + const n_kv = a.args.field("n_kv"); + const group = num_heads.div(n_kv); + const groups = group.div(HG); + const r = a.tgpig.at(2).to(.int); + const tg_head = a.tgpig.at(1).to(.int); + const ikv = tg_head.div(groups); + const iq2 = ikv.mul(group).add(tg_head.rem(groups).mul(HG)).add(a.sgitg.rem(HG).to(.int)); + const iwg = a.tgpig.at(0).to(.int).mul(a.nsg.div(HG).to(.int)).add(a.sgitg.div(HG).to(.int)); + const rid = r.to(.long).mul(num_heads.to(.uint)).add(iq2); + const nrows = a.tgpg.at(2).to(.long).mul(num_heads.to(.uint)); + + const kv_len = a.token_index.load(r).add(1); + const live = if (NWG > 1) liveSplits(b, kv_len, NWG) else b.lift(@as(i32, 1)); + + if (NWG > 1) { + var idle = b.openIf(iwg.ge(live)); + { + b.return_(); + idle.close(); + } + } + + const position_bytes = n_kv.to(.ulong).mul(D * 2); + const first = r.to(.ulong).mul(a.args.field("k_len").to(.ulong)).mul(position_bytes).add(ikv.to(.ulong).mul(D * 2)); + const k0 = a.k.add(first); + const v0 = a.v.add(first); + + const lane = if (R == 1) a.tiisg else a.tiisg.rem(NL); + const sub = a.tiisg.div(NL).to(.int); + const q4 = a.q.add(rid.mul(D).mul(2)).to(.bfloat4).add(lane); + const qr = b.array(.float4, @intCast(NI)); + const o = b.array(.float4, @intCast(NI)); + var init = b.openForOpts(b.constant(.short, 0), NI, 1, full); + { + const ii = init.iv; + qr.store(ii, q4.load(ii.mul(NL)).to(.float4)); + o.store(ii, b.constant(.float4, 0)); + init.close(); + } + + const S = b.local(.float, 0.0); + const M = b.local(.float, -flt_max / 2.0); + + const ic0 = b.local(.int, iwg); + var blocks = b.openLoop(.{}); + { + const ic = ic0.mul(C); + var past = b.openIf(ic.ge(kv_len)); + { + blocks.breakLoop(); + past.close(); + } + + const s = b.local(.float, 0.0); + var qk = b.openForOpts(b.constant(.short, 0), @divExact(C, R), 1, full); + { + const cc = qk.iv; + const p = b.min(if (R == 1) ic.add(cc) else ic.add(cc.to(.int).mul(R)).add(sub), kv_len.sub(1)); + const pk4 = k0.add(p.to(.ulong).mul(position_bytes)).to(.bfloat4).add(lane); + const d = b.local(.float, 0.0); + var dots = b.openForOpts(b.constant(.short, 0), NI, 1, full); + { + const ii = dots.iv; + d.assign(d.add(b.dot(pk4.load(ii.mul(NL)).to(.float4), qr.load(ii)))); + dots.close(); + } + if (R == 1) { + s.assign(b.ternary(a.tiisg.to(.short).eq(cc), b.simdSum(d.rvalue()), s.rvalue())); + } else { + var x = d.rvalue(); + var mask: i32 = 1; + while (mask < NL) : (mask *= 2) x = x.add(b.simdShuffleXor(x, b.constant(.ushort, mask))); + const t = b.simdShuffle(x, a.tiisg.rem(R).mul(NL)); + s.assign(b.ternary(a.tiisg.div(R).to(.short).eq(cc), t, s.rvalue())); + } + qk.close(); + } + + const pos = ic.add(a.tiisg.to(.int)); + const score = b.ternary(pos.lt(kv_len), s.mul(a.args.field("scale")), b.constant(.float, -max_half)); + + const m = M.rvalue(); + M.assign(b.simdMax(b.max(M.rvalue(), score))); + const ms = b.exp(m.sub(M)); + const vs = b.exp(score.sub(M)); + S.assign(S.mul(ms).add(b.simdSum(vs))); + + var rescale = b.openForOpts(b.constant(.short, 0), NI, 1, full); + { + o.store(rescale.iv, o.load(rescale.iv).mul(ms)); + rescale.close(); + } + + var pv = b.openForOpts(b.constant(.short, 0), @divExact(C, R), 1, full); + { + const cc = pv.iv; + const row = cc.to(.int).mul(R).add(sub); + const p = b.min(if (R == 1) ic.add(cc) else ic.add(row), kv_len.sub(1)); + const pv4 = v0.add(p.to(.ulong).mul(position_bytes)).to(.bfloat4).add(lane); + const w = b.vec(.float4, .{if (R == 1) b.simdBroadcast(vs, cc.to(.ushort)) else b.simdShuffle(vs, row)}); + var dims = b.openForOpts(b.constant(.short, 0), NI, 1, full); + { + const ii = dims.iv; + o.store(ii, o.load(ii).add(pv4.load(ii.mul(NL)).to(.float4).mul(w))); + dims.close(); + } + pv.close(); + } + + blocks.next(); + ic0.assign(ic0.add(live)); + blocks.close(); + } + + var mask: i32 = NL; + while (mask < simd_width) : (mask *= 2) { + var sum = b.openForOpts(b.constant(.short, 0), NI, 1, full); + { + o.store(sum.iv, o.load(sum.iv).add(b.simdShuffleXor(o.load(sum.iv), b.constant(.ushort, mask)))); + sum.close(); + } + } + var first_row = if (R > 1) b.openIf(sub.eq(0)) else null; + if (NWG == 1) { + const inv_sum = b.ternary(S.eq(0.0), 0.0, b.lift(1.0).div(S)); + const dst4 = a.dst.to(.bfloat4).add(rid.mul(D4)).add(lane); + var out = b.openForOpts(b.constant(.short, 0), NI, 1, full); + { + dst4.store(out.iv.mul(NL), o.load(out.iv).mul(inv_sum).to(.bfloat4)); + out.close(); + } + } else { + // [rows][D4][splits] float4 partials, then [rows][splits] (S, M). + const dst4 = a.dst.to(.float4).add(rid.mul(D4).mul(NWG)); + var out = b.openForOpts(b.constant(.short, 0), NI, 1, full); + { + const i = lane.to(.int).add(out.iv.to(.int).mul(NL)); + dst4.store(i.mul(NWG).add(iwg), o.load(out.iv)); + out.close(); + } + var lane0 = b.openIf(a.tiisg.eq(0)); + { + const dst1 = a.dst.to(.float).add(nrows.mul(D).mul(NWG)).add(rid.mul(2 * NWG)); + dst1.store(b.lift(2).mul(iwg), S); + dst1.store(b.lift(2).mul(iwg).add(1), M); + lane0.close(); + } + } + if (first_row) |*scope| scope.close(); + } +}; + +fn liveSplits(b: *Builder, kv_len: Value, splits: i32) Value { + return b.min(splits, kv_len.add(simd_width - 1).div(simd_width)); +} + +/// fa_vec_reduce: merges the fa_vec split partials. +pub const FaVecReduce = struct { + pub const Cfg = struct { + splits: i32, + head_dim: i32, + }; + + pub const simdgroups = 8; + + pub const Kernel = metal.Kernel(Cfg, .{ + .name = "fa_vec_reduce", + .inputs = &.{ "htmp", "token_index" }, + .outputs = &.{"dst"}, + .constants = &.{"args"}, + .run = run, + }); + + fn run(b: *Builder, cfg: Cfg) metal.FinishError!void { + const a = try b.declareArgs(.{ + .args = .{ .constant_struct = FaVec.Args }, + .htmp = .{ .buffer = .{ .type = .char, .const_ = true } }, + .token_index = .{ .buffer = .{ .type = .int, .const_ = true } }, + .dst = .{ .buffer = .{ .type = .char } }, + .tgpig = .{ .threadgroup_position_in_grid = .uint2 }, + .tgpg = .{ .threadgroups_per_grid = .uint2 }, + .tiisg = .{ .thread_index_in_simdgroup = .ushort }, + .sgitg = .{ .simdgroup_index_in_threadgroup = .ushort }, + }); + const NWG = cfg.splits; + if (NWG > 4 * simd_width) std.debug.panic("fa_vec_reduce: {d} splits, at most {d}", .{ NWG, 4 * simd_width }); + const NP = @divFloor(NWG + simd_width - 1, simd_width); + const DV = cfg.head_dim; + const DV4 = @divExact(DV, 4); + const full: metal.LoopOpts = .{ .unroll = .full }; + + const rid = a.tgpig.at(0).to(.ulong); + const r = a.tgpig.at(0).div(a.args.field("num_heads").to(.uint)).to(.int); + const i = a.tgpig.at(1).to(.int).mul(simdgroups).add(a.sgitg.to(.int)); + const dst4 = a.dst.to(.bfloat4).add(rid.mul(DV4)); + const live = liveSplits(b, a.token_index.load(r).add(1), NWG); + + const ss = a.htmp.to(.float).add(a.tgpg.at(0).to(.ulong).mul(DV).mul(NWG)).add(rid.mul(2 * NWG)); + const Mj = b.array(.float, @intCast(NP)); + const Sj = b.array(.float, @intCast(NP)); + const mx = b.local(.float, -flt_max / 2.0); + var load = b.openForOpts(b.constant(.short, 0), NP, 1, full); + { + const j = load.iv; + const idx = a.tiisg.to(.int).add(j.to(.int).mul(simd_width)); + const ok = idx.lt(live); + Mj.store(j, b.ternary(ok, ss.load(idx.mul(2).add(1)), b.constant(.float, -flt_max / 2.0))); + Sj.store(j, b.ternary(ok, ss.load(idx.mul(2)), b.constant(.float, 0))); + mx.assign(b.max(mx.rvalue(), Mj.load(j))); + load.close(); + } + const m = b.simdMax(mx.rvalue()); + const sum = b.local(.float, 0.0); + var weigh = b.openForOpts(b.constant(.short, 0), NP, 1, full); + { + const j = weigh.iv; + Mj.store(j, b.exp(Mj.load(j).sub(m))); + sum.assign(sum.add(Sj.load(j).mul(Mj.load(j)))); + weigh.close(); + } + const S = b.simdSum(sum.rvalue()); + const inv_sum = b.ternary(S.eq(0.0), 0.0, b.lift(1.0).div(S)); + + const htmp4 = a.htmp.to(.float4).add(rid.mul(DV4).mul(NWG)); + const acc = b.local(.float4, b.constant(.float4, 0)); + var parts = b.openForOpts(b.constant(.short, 0), NP, 1, full); + { + const j = parts.iv; + const idx = a.tiisg.to(.int).add(j.to(.int).mul(simd_width)); + var ok = b.openIf(idx.lt(live)); + { + acc.assign(acc.add(htmp4.load(i.mul(NWG).add(idx)).mul(Mj.load(j)))); + ok.close(); + } + parts.close(); + } + const v = b.simdSum(acc.rvalue()); + var lane0 = b.openIf(a.tiisg.eq(0)); + { + dst4.store(i, v.mul(inv_sum).to(.bfloat4)); + lane0.close(); + } + } +}; + +/// fa_tiled: causal FlashAttention-2 over simdgroup matrices. Queries past +/// `live` output zeros. +pub const FaTiled = struct { + pub const Cfg = struct { + dtype: Type, + head_size: i32, + bq: i32 = 32, + tile_kv: i32 = 32, + num_threads: i32 = 128, + }; + + pub const Kernel = metal.Kernel(Cfg, .{ + .name = "fa_tiled", + .inputs = &.{ "q", "k", "v", "token_index", "live" }, + .outputs = &.{"out"}, + .constants = &.{ "num_kv_heads", "scale", "q_len", "k_len" }, + .run = run, + }); + + fn supported(cfg: Cfg) bool { + const tile = tileConfig(cfg.head_size) orelse return false; + return (cfg.dtype.eql(.half) or cfg.dtype.eql(.bfloat)) and + cfg.bq == tile[0] and cfg.tile_kv == tile[1] and cfg.num_threads == tile[2]; + } + + pub fn tileConfig(head_size: i64) ?[3]i32 { + return switch (head_size) { + 32, 64, 96, 128 => .{ 32, 32, 128 }, + 256 => .{ 16, 16, 64 }, + 512 => .{ 8, 8, 32 }, + else => null, + }; + } + + pub fn threadgroupMemory(cfg: Cfg) u64 { + const ld = cfg.head_size + @divExact(16, @as(i32, @intCast(cfg.dtype.size()))); + return @intCast((cfg.bq + 2 * cfg.tile_kv) * ld * @as(i32, @intCast(cfg.dtype.size()))); + } + + /// Lane `lane_id`'s (column, row) in an 8x8 simdgroup_matrix fragment. + fn fragCoord(b: *Builder, lane_id: Value) Value { + const qid = lane_id.to(.short).div(4).to(.short); + const fm = qid.bitAnd(4).add(lane_id.to(.short).div(2).rem(4)).to(.short); + const fn_ = qid.bitAnd(2).mul(2).add(lane_id.to(.short).rem(2).mul(2)).to(.short); + return b.vec(.short2, .{ fn_, fm }); + } + + fn fragRowReduce(b: *Builder, comptime op: enum { max, sum }, v: Value) Value { + var x = v.rvalue(); + inline for (.{ 1, 8 }) |mask| { + const y = b.simdShuffleXor(x, b.constant(.ushort, mask)); + x = switch (op) { + .max => b.max(x, y), + .sum => x.add(y), + }; + } + return x; + } + + fn setLane(p: Value, i: Value, lane: anytype, x: Value) void { + p.store(i, p.load(i).insert(lane, x)); + } + + fn run(b: *Builder, cfg: Cfg) metal.FinishError!void { + if (!supported(cfg)) std.debug.panic("fa_tiled: {s} at head size {d} and tiles {d}, {d}, {d} unsupported", .{ cfg.dtype.name(), cfg.head_size, cfg.bq, cfg.tile_kv, cfg.num_threads }); + const T = cfg.dtype; + const HEAD_SIZE = cfg.head_size; + const BQ = cfg.bq; + const TILE_KV = cfg.tile_kv; + const NUM_THREADS = cfg.num_threads; + const int_arg: ArgSpec = .{ .constant = .int }; + const int_in: ArgSpec = .{ .buffer = .{ .type = .int, .const_ = true } }; + const a = try b.declareArgs(.{ + .out = .{ .buffer = .{ .type = T } }, + .q = .{ .buffer = .{ .type = T, .const_ = true } }, + .k = .{ .buffer = .{ .type = T, .const_ = true } }, + .v = .{ .buffer = .{ .type = T, .const_ = true } }, + .token_index = int_in, + .live = int_in, + .num_kv_heads = int_arg, + .scale = .{ .constant = .float }, + .q_len = int_arg, + .k_len = int_arg, + .shared_mem = .{ .threadgroup = .char }, + .tgp = .{ .threadgroup_position_in_grid = .uint3 }, + .tgpg = .{ .threadgroups_per_grid = .uint3 }, + .tpt = .{ .thread_position_in_threadgroup = .uint3 }, + .sg_idx = .{ .simdgroup_index_in_threadgroup = .uint }, + .lane = .{ .thread_index_in_simdgroup = .uint }, + }); + + const NUM_SIMD_LANES = 32; + const NUM_SG = @divExact(NUM_THREADS, NUM_SIMD_LANES); + const TD = @divExact(HEAD_SIZE, 8); + const TK = @divExact(TILE_KV, 8); + if (@divExact(BQ, NUM_SG) != 8) @panic("fa_tiled: ROWS_PER_SG must be 8"); + const unroll: metal.LoopOpts = .{ .unroll = .enable }; + + const thread_idx = a.tpt.x().to(.int); + const head_idx = a.tgp.x().to(.int); + const q_pos_start = a.tgp.y().to(.int).mul(BQ); + const batch = a.tgp.z().to(.int); + const num_heads = a.tgpg.x().to(.int); + const kv_head_idx = head_idx.div(num_heads.div(a.num_kv_heads)); + const q_stride = num_heads.mul(HEAD_SIZE); + const kv_token_stride = a.num_kv_heads.mul(HEAD_SIZE); + + const block_rows = b.min(BQ, a.q_len.sub(q_pos_start)); + const live = a.live.load(0); + const valid_q = b.min(block_rows, b.max(0, live.sub(q_pos_start))); + const out_base = a.out.add(batch.mul(a.q_len).add(q_pos_start).to(.long).mul(q_stride).add(head_idx.mul(HEAD_SIZE))); + + var dead = b.openIf(valid_q.le(0)); + { + var zero = b.openFor(thread_idx, block_rows.mul(HEAD_SIZE), NUM_THREADS); + { + const i = zero.iv; + out_base.store(i.div(HEAD_SIZE).mul(q_stride).add(i.rem(HEAD_SIZE)), b.constant(T, 0)); + zero.close(); + } + b.return_(); + dead.close(); + } + + // Query i of the batch row is at position context_len + i. + const context_len = a.token_index.load(batch); + const seq_len = context_len.add(live); + const kv_base = batch.to(.long).mul(a.k_len).mul(kv_token_stride).add(kv_head_idx.mul(HEAD_SIZE)); + + const SMEM_PAD = @divExact(16, @as(i32, @intCast(T.size()))); + const LD = HEAD_SIZE + SMEM_PAD; + const Q_ELEMS = BQ * LD; + const KV_ELEMS = TILE_KV * LD; + + const Q_smem = a.shared_mem.to(T); + const K_smem = Q_smem.add(Q_ELEMS); + const V_smem = K_smem.add(KV_ELEMS); + + const log2e: f32 = std.math.log2e; + const scale_log2 = a.scale.mul(log2e); + + const q_base = a.q.add(batch.mul(a.q_len).add(q_pos_start).to(.long).mul(q_stride).add(head_idx.mul(HEAD_SIZE))); + var load_q = b.openFor(thread_idx, BQ * HEAD_SIZE, NUM_THREADS); + { + const i = load_q.iv; + const r = i.div(HEAD_SIZE); + const d = i.rem(HEAD_SIZE); + var value = b.openIfElse(r.lt(valid_q), .{T}); + { + value.yieldThen(.{q_base.load(r.mul(q_stride).add(d))}); + } + { + value.yieldElse(.{b.constant(T, 0)}); + } + Q_smem.store(r.mul(LD).add(d), value.results[0]); + load_q.close(); + } + + b.threadgroupBarrier(.mem_threadgroup); + + const T2: Type = .vec(T.scalar, 2); + const q_reg = b.array(T2, @intCast(TD)); + var load_q_reg = b.openForOpts(0, TD, 1, unroll); + { + const d = load_q_reg.iv; + const tmp = b.simdgroupLoad(Q_smem.add(a.sg_idx.mul(8).mul(LD)).add(d.mul(8)), LD, .{}); + q_reg.store(d, tmp.threadElements()); + load_q_reg.close(); + } + + const fc = fragCoord(b, a.lane.to(.ushort)); + const fm = fc.y(); + const fn_ = fc.x(); + + const max_score = b.local(.float, masked_score); + const sum_score = b.local(.float, 0.0); + + const o_reg = b.array(.float2, @intCast(TD)); + var zero_o = b.openForOpts(0, TD, 1, unroll); + { + o_reg.store(zero_o.iv, b.constant(.float2, 0)); + zero_o.close(); + } + + const num_kv_tiles = seq_len.add(TILE_KV).sub(1).div(TILE_KV); + const last_q_pos = context_len.add(q_pos_start).add(valid_q).sub(1); + + var tiles = b.openFor(0, num_kv_tiles, 1); + { + const tile_idx = tiles.iv; + const tile_start = tile_idx.mul(TILE_KV); + + var past = b.openIf(tile_start.gt(last_q_pos)); + { + tiles.breakLoop(); + past.close(); + } + + const VEC_BYTES: i32 = if (HEAD_SIZE >= 256) 16 else 8; + const VEC = @divExact(VEC_BYTES, @as(i32, @intCast(T.size()))); + const LoadUnit: Type = if (VEC_BYTES == 16) .uint4 else .uint2; + std.debug.assert(@rem(LD, VEC) == 0); + var rows = b.openForOpts(0, @divExact(TILE_KV, NUM_SG), 1, unroll); + { + const t = a.sg_idx.mul(@divExact(TILE_KV, NUM_SG)).add(rows.iv).to(.int); + const kv_pos = tile_start.add(t); + var present = b.openIfElse(kv_pos.lt(seq_len), .{}); + { + const off = kv_base.add(kv_pos.to(.long).mul(kv_token_stride)); + const k_ptr = a.k.add(off); + const v_ptr = a.v.add(off); + var loop = b.openForOpts(a.lane.mul(VEC).to(.int), HEAD_SIZE, NUM_SIMD_LANES * VEC, unroll); + { + const d = loop.iv; + K_smem.add(t.mul(LD).add(d)).to(LoadUnit).store(0, k_ptr.add(d).to(LoadUnit).load(0)); + V_smem.add(t.mul(LD).add(d)).to(LoadUnit).store(0, v_ptr.add(d).to(LoadUnit).load(0)); + loop.close(); + } + present.elseBranch(); + } + { + const zero = b.constant(LoadUnit, 0); + var loop = b.openForOpts(a.lane.mul(VEC).to(.int), HEAD_SIZE, NUM_SIMD_LANES * VEC, unroll); + { + const d = loop.iv; + K_smem.add(t.mul(LD).add(d)).to(LoadUnit).store(0, zero); + V_smem.add(t.mul(LD).add(d)).to(LoadUnit).store(0, zero); + loop.close(); + } + present.close(); + } + rows.close(); + } + + b.threadgroupBarrier(.mem_threadgroup); + + const s_reg = b.array(.float2, @intCast(TK)); + var zero_s = b.openForOpts(0, TK, 1, unroll); + { + s_reg.store(zero_s.iv, b.constant(.float2, 0)); + zero_s.close(); + } + + var qk_dims = b.openForOpts(0, TD, 1, unroll); + { + const d = qk_dims.iv; + const q_frag = b.matrix(T.scalar, q_reg.load(d)); + var qk_keys = b.openForOpts(0, TK, 1, unroll); + { + const k = qk_keys.iv; + const k_frag = b.simdgroupLoad(K_smem.add(k.mul(8).mul(LD)).add(d.mul(8)), LD, .{ .transpose = true }); + const s_frag = b.matrix(.float, s_reg.load(k)); + s_reg.store(k, b.simdgroupMultiplyAccumulate(q_frag, k_frag, s_frag).threadElements()); + qk_keys.close(); + } + qk_dims.close(); + } + + const q_abs_pos = context_len.add(q_pos_start).add(a.sg_idx.mul(8)).add(fm).to(.int); + const row_masked = a.sg_idx.mul(8).add(fm).ge(valid_q); + const tile_no_mask = tile_start.add(TILE_KV).le(seq_len).bitAnd(tile_start.add(TILE_KV).sub(1).lt(context_len.add(q_pos_start))); + + var masking = b.openIfElse(tile_no_mask.bitAnd(row_masked.eq(false)), .{}); + { + var scale = b.openForOpts(0, TK, 1, unroll); + { + const k = scale.iv; + setLane(s_reg, k, 0, s_reg.load(k).at(0).mul(scale_log2)); + setLane(s_reg, k, 1, s_reg.load(k).at(1).mul(scale_log2)); + scale.close(); + } + masking.elseBranch(); + } + { + var keys = b.openForOpts(0, TK, 1, unroll); + { + const k = keys.iv; + var pair = b.openForOpts(0, 2, 1, unroll); + { + const jj = pair.iv; + const s = s_reg.load(k).at(jj).mul(scale_log2); + const kv_pos = tile_start.add(k.mul(8)).add(fn_).add(jj); + const masked = row_masked.bitOr(kv_pos.ge(seq_len)).bitOr(kv_pos.gt(q_abs_pos)); + setLane(s_reg, k, jj, b.ternary(masked, masked_score, s)); + pair.close(); + } + keys.close(); + } + masking.close(); + } + + const local_max = b.local(.float, masked_score); + var max_keys = b.openForOpts(0, TK, 1, unroll); + { + const k = max_keys.iv; + local_max.assign(b.max(local_max.rvalue(), b.max(s_reg.load(k).at(0), s_reg.load(k).at(1)))); + max_keys.close(); + } + const row_max = fragRowReduce(b, .max, local_max); + + const prev_max = max_score.rvalue(); + const grown_max = b.max(prev_max, row_max); + const grew = grown_max.gt(prev_max); + const was_empty = prev_max.eq(masked_score); + const factor = b.ternary(grew, b.ternary(was_empty, 0.0, b.fast().exp2(prev_max.sub(grown_max))), 1.0); + const new_max = b.ternary(grew.eq(false).bitAnd(was_empty), 0.0, grown_max); + // A tile masking a whole row leaves the maximum masked: zero would + // pin it, and later scores far below zero would underflow. + max_score.assign(grown_max); + + const local_sum = b.local(.float, 0.0); + var p_keys = b.openForOpts(0, TK, 1, unroll); + { + const k = p_keys.iv; + var pair = b.openForOpts(0, 2, 1, unroll); + { + const jj = pair.iv; + const s = s_reg.load(k).at(jj); + const p = b.ternary(s.eq(masked_score), 0.0, b.fast().exp2(s.sub(new_max))); + setLane(s_reg, k, jj, p); + local_sum.assign(local_sum.add(p)); + pair.close(); + } + p_keys.close(); + } + const row_sum = fragRowReduce(b, .sum, local_sum); + + sum_score.assign(sum_score.mul(factor).add(row_sum)); + + var rescale_o = b.openForOpts(0, TD, 1, unroll); + { + o_reg.store(rescale_o.iv, o_reg.load(rescale_o.iv).mul(factor)); + rescale_o.close(); + } + + var pv_keys = b.openForOpts(0, TK, 1, unroll); + { + const k = pv_keys.iv; + const p_frag = b.matrix(.float, s_reg.load(k)); + var pv_dims = b.openForOpts(0, TD, 1, unroll); + { + const d = pv_dims.iv; + const v_frag = b.simdgroupLoad(V_smem.add(k.mul(8).mul(LD)).add(d.mul(8)), LD, .{}); + const o_frag = b.matrix(.float, o_reg.load(d)); + o_reg.store(d, b.simdgroupMultiplyAccumulate(p_frag, v_frag, o_frag).threadElements()); + pv_dims.close(); + } + pv_keys.close(); + } + + b.threadgroupBarrier(.mem_threadgroup); + tiles.close(); + } + + const inv_sum = b.lift(1.0).div(sum_score.add(@as(f32, 1e-6))); + var normalize = b.openForOpts(0, TD, 1, unroll); + { + o_reg.store(normalize.iv, o_reg.load(normalize.iv).mul(inv_sum)); + normalize.close(); + } + + const O_smem = Q_smem.to(.float); + var store_o = b.openForOpts(0, TD, 1, unroll); + { + const d = store_o.iv; + const o_frag = b.matrix(.float, o_reg.load(d)); + b.simdgroupStore(o_frag, O_smem.add(a.sg_idx.mul(8).mul(LD)).add(d.mul(8)), LD, .{}); + store_o.close(); + } + + b.threadgroupBarrier(.mem_threadgroup); + + var write_out = b.openFor(thread_idx, block_rows.mul(HEAD_SIZE), NUM_THREADS); + { + const i = write_out.iv; + const r = i.div(HEAD_SIZE); + const d = i.rem(HEAD_SIZE); + const value = b.ternary(r.lt(valid_q), O_smem.load(r.mul(LD).add(d)).to(T), b.constant(T, 0)); + out_base.store(r.mul(q_stride).add(d), value); + write_out.close(); + } + } +}; + +test "the flash attention kernels emit verified modules" { + const allocator = std.testing.allocator; + for ([_]i32{ 32, 64, 128, 256, 512 }) |head_dim| { + for ([_][2]i32{ .{ 1, 1 }, .{ 4, 1 }, .{ 2, 8 }, .{ 4, 128 } }) |launch| { + allocator.free(try FaVec.Kernel.emit(allocator, .{ .head_dim = head_dim, .heads = launch[0], .splits = launch[1] })); + } + for ([_]i32{ 8, 128 }) |splits| { + allocator.free(try FaVecReduce.Kernel.emit(allocator, .{ .splits = splits, .head_dim = head_dim })); + } + } + for ([_]Type{ .half, .bfloat }) |dtype| { + for ([_]i32{ 32, 64, 96, 128, 256, 512 }) |head_size| { + const tile = FaTiled.tileConfig(head_size).?; + allocator.free(try FaTiled.Kernel.emit(allocator, .{ .dtype = dtype, .head_size = head_size, .bq = tile[0], .tile_kv = tile[1], .num_threads = tile[2] })); + } + } +} diff --git a/zml/kernels/flash_attention/metal/metal.zig b/zml/kernels/flash_attention/metal/metal.zig new file mode 100644 index 00000000..bdfb00c0 --- /dev/null +++ b/zml/kernels/flash_attention/metal/metal.zig @@ -0,0 +1,143 @@ +const std = @import("std"); + +const zml = @import("../../../zml.zig"); +const flash_attn = @import("flash_attn.zig"); + +pub const Metadata = struct { + /// Live queries of a single-sequence prefill; 0 means all. + num_tokens: zml.Tensor, + + pub fn init() Metadata { + return .{ .num_tokens = .fromShape(.scalar(.u32)) }; + } + + pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(Metadata) { + _ = self; + return .{ .num_tokens = try zml.Buffer.scalar(io, platform, 0, .u32) }; + } + + pub fn deinitBuffer(self: *zml.Bufferized(Metadata)) void { + self.num_tokens.deinit(); + } +}; + +/// Causal attention: fa_vec for bf16 decode, fa_tiled otherwise, sdpa for +/// unsupported dtypes and head sizes. +pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata) zml.Tensor { + const dtype = q.dtype(); + const head_dim = q.dim(.hd); + if (!((dtype == .bf16 or dtype == .f16) and k.dtype() == dtype and v.dtype() == dtype and flash_attn.FaTiled.tileConfig(head_dim) != null)) { + const seq_len = k.dim(.k); + const mask = zml.nn.causalAttnMask(.{ .q = seq_len, .k = seq_len }, dtype, null) + .gatherSlices(zml.Shape.init(.{ .q = q.dim(.q) }, dtype), token_index.appendAxes(.{.coord}), .{}); + return zml.nn.sdpa(q, k, v, .{ .attn_mask = mask }); + } + + const batched = q.shape().hasTag(.b) != null; + const qb = if (batched) q.transpose(.{ .b, .q, .h, .hd }) else q.transpose(.{ .q, .h, .hd }).insertAxes(0, .{.b}); + const kb = if (batched) k.transpose(.{ .b, .k, .h, .hd }) else k.transpose(.{ .k, .h, .hd }).insertAxes(0, .{.b}); + const vb = if (batched) v.transpose(.{ .b, .k, .h, .hd }) else v.transpose(.{ .k, .h, .hd }).insertAxes(0, .{.b}); + const positions = token_index.convert(.i32).broad(.init(.{ .b = qb.dim(.b) }, .i32)); + const scale = 1.0 / @sqrt(@as(f32, @floatFromInt(head_dim))); + + const vec_head_dim = head_dim == 32 or head_dim == 64 or head_dim == 128 or head_dim == 256 or head_dim == 512; + const out = if (qb.dim(.q) == 1 and dtype == .bf16 and vec_head_dim) + decode(qb, kb, vb, positions, scale) + else + prefill(qb, kb, vb, positions, liveQueries(qb, metadata), scale); + return (if (batched) out else out.squeeze(.b)).transpose(q.shape()); +} + +fn liveQueries(q: zml.Tensor, metadata: Metadata) zml.Tensor { + const all: zml.Tensor = .scalar(q.dim(.q), .i32); + if (q.dim(.b) != 1 or q.dim(.q) == 1) return all; + const num_tokens = metadata.num_tokens.convert(.i32); + return num_tokens.cmp(.GE, .scalar(1, .i32)).select(num_tokens.minimum(all), all); +} + +/// `q` is `[b, 1, h, hd]`, `k` and `v` `[b, k, kv_h, hd]`. +fn decode(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, positions: zml.Tensor, scale: f32) zml.Tensor { + const batch: i32 = @intCast(q.dim(.b)); + const num_heads: i32 = @intCast(q.dim(.h)); + const head_dim: i32 = @intCast(q.dim(.hd)); + const num_kv_heads: i32 = @intCast(k.dim(.h)); + const keys: i32 = @intCast(k.dim(.k)); + const nrows = batch * num_heads; + const launch = vecLaunch(nrows, @divExact(num_heads, num_kv_heads), keys, coreCount()); + const args: flash_attn.FaVec.HostArgs = .{ .num_heads = num_heads, .n_kv = num_kv_heads, .k_len = keys, .scale = scale }; + + const dst = if (launch.splits == 1) q.shape() else zml.Shape.init(.{nrows * (head_dim * launch.splits + 2 * launch.splits)}, .f32); + const out = flash_attn.FaVec.Kernel.call(.{ .q = q, .k = k, .v = v, .token_index = positions }, .{ .dst = dst }, .{ + .cfg = .{ .head_dim = head_dim, .heads = launch.heads, .splits = launch.splits }, + .grid = .{ @intCast(@divExact(launch.splits, launch.splits_per_threadgroup)), @intCast(@divExact(num_heads, launch.heads)), @intCast(batch) }, + .threadgroup = .{ 32, @intCast(launch.heads * launch.splits_per_threadgroup), 1 }, + .constants = .{ .args = .of(args) }, + }).dst; + if (launch.splits == 1) return out; + + return flash_attn.FaVecReduce.Kernel.call(.{ .htmp = out, .token_index = positions }, .{ .dst = q.shape() }, .{ + .cfg = .{ .splits = launch.splits, .head_dim = head_dim }, + .grid = .{ @intCast(nrows), @intCast(@divExact(head_dim, 4 * flash_attn.FaVecReduce.simdgroups)), 1 }, + .threadgroup = .{ 32, flash_attn.FaVecReduce.simdgroups, 1 }, + .constants = .{ .args = .of(args) }, + }).dst; +} + +/// `q` is `[b, q, h, hd]`, `k` and `v` `[b, k, kv_h, hd]`. +fn prefill(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, positions: zml.Tensor, live: zml.Tensor, scale: f32) zml.Tensor { + const queries: i32 = @intCast(q.dim(.q)); + const tile = flash_attn.FaTiled.tileConfig(q.dim(.hd)).?; + const cfg: flash_attn.FaTiled.Cfg = .{ + .dtype = zml.kernel.metal.from(q.dtype()), + .head_size = @intCast(q.dim(.hd)), + .bq = tile[0], + .tile_kv = tile[1], + .num_threads = tile[2], + }; + return flash_attn.FaTiled.Kernel.call(.{ .q = q, .k = k, .v = v, .token_index = positions, .live = live }, .{ .out = q.shape() }, .{ + .cfg = cfg, + .grid = .{ @intCast(q.dim(.h)), @intCast(@divFloor(queries + tile[0] - 1, tile[0])), @intCast(q.dim(.b)) }, + .threadgroup = .{ @intCast(tile[2]), 1, 1 }, + .threadgroup_memory = flash_attn.FaTiled.threadgroupMemory(cfg), + .constants = .{ + .num_kv_heads = .of(@as(i32, @intCast(k.dim(.h)))), + .scale = .of(scale), + .q_len = .of(queries), + .k_len = .of(@as(i32, @intCast(k.dim(.k)))), + }, + }).out; +} + +const VecLaunch = struct { + heads: i32, + splits: i32, + splits_per_threadgroup: i32, +}; + +/// At most 256 threads, which every Apple GPU pipeline accepts. Splits past a +/// row's length exit early, so `splits` follows the core and row counts. +fn vecLaunch(rows: i32, group: i32, span: i32, cores: i32) VecLaunch { + var heads: i32 = @min(group, 4); + while (@rem(group, heads) != 0) heads -= 1; + const want: i32 = if (rows <= 2 * cores) 128 else 64; + const chunks = @divFloor(span + 31, 32); + var splits: i32 = 1; + while (splits * 2 <= @min(want, chunks)) splits *= 2; + return .{ .heads = heads, .splits = splits, .splits_per_threadgroup = @min(splits, @divFloor(8, heads)) }; +} + +fn coreCount() i32 { + const platform = zml.Compiler.current().platform; + const attribute = platform.devices[0].pjrt_desc.attribute(platform.pjrt_api, "core_count") orelse + std.debug.panic("Metal device has no core_count attribute", .{}); + return @intCast(attribute.int64); +} + +test vecLaunch { + try std.testing.expectEqual(VecLaunch{ .heads = 4, .splits = 128, .splits_per_threadgroup = 2 }, vecLaunch(16, 4, 128000, 40)); + try std.testing.expectEqual(VecLaunch{ .heads = 1, .splits = 1, .splits_per_threadgroup = 1 }, vecLaunch(8, 1, 20, 40)); +} + +test { + std.testing.refAllDecls(flash_attn); +} diff --git a/zml/kernels/flash_attention/nki/BUILD.bazel b/zml/kernels/flash_attention/nki/BUILD.bazel new file mode 100644 index 00000000..f8e4f470 --- /dev/null +++ b/zml/kernels/flash_attention/nki/BUILD.bazel @@ -0,0 +1,30 @@ +load("@rules_python//python:py_binary.bzl", "py_binary") + +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "attention_kernel", + srcs = ["attention.py"], +) + +filegroup( + name = "nki", + srcs = ["nki.zig"], +) + +py_binary( + name = "attention_harness", + srcs = ["attention_harness.py"], + data = [ + ":attention.py", + "@libzml_neuron//:sandbox", + ], + visibility = ["//visibility:public"], + deps = [ + "@neuron_py_deps//ml_dtypes:pkg", + "@neuron_py_deps//neuronx_cc:pkg", + "@neuron_py_deps//nki:pkg", + "@neuron_py_deps//numpy:pkg", + "@rules_python//python/runfiles", + ], +) diff --git a/zml/kernels/flash_attention/nki/attention.py b/zml/kernels/flash_attention/nki/attention.py new file mode 100644 index 00000000..cf76d9ae --- /dev/null +++ b/zml/kernels/flash_attention/nki/attention.py @@ -0,0 +1,473 @@ +import math + +import nki.isa as nisa +import nki.language as nl + +NEG_INF = -3.3895313892515355e38 + + +def decode(q, k, v, token_index): + q_len = q.shape[0] + num_q_heads = q.shape[1] + d_head = q.shape[2] + seq_len = k.shape[0] + num_kv_heads = k.shape[1] + heads_per_kv = num_q_heads // num_kv_heads + scale = 1.0 / (d_head**0.5) + + assert q_len == 1 + assert k.shape == v.shape + assert q.shape[2] == k.shape[2] + assert num_q_heads % num_kv_heads == 0 + + VALUE_TILE_TOKENS = nl.tile_size.pmax + assert seq_len % VALUE_TILE_TOKENS == 0, ( + "seq_len " + f"({seq_len}) must be a multiple of VALUE_TILE_TOKENS ({VALUE_TILE_TOKENS})" + ) + # Largest tensor-engine K tile that exactly partitions this static sequence. + BLOCK_K = math.gcd(seq_len, nl.tile_size.gemm_moving_fmax) + assert seq_len % BLOCK_K == 0, ( + f"seq_len ({seq_len}) must be a multiple of BLOCK_K ({BLOCK_K})" + ) + assert BLOCK_K % VALUE_TILE_TOKENS == 0, ( + "BLOCK_K " + f"({BLOCK_K}) must be a multiple of VALUE_TILE_TOKENS ({VALUE_TILE_TOKENS})" + ) + num_kv_blocks = seq_len // BLOCK_K + value_tiles_per_k_tile = BLOCK_K // VALUE_TILE_TOKENS + + out = nl.ndarray((q_len, num_q_heads, d_head), dtype=q.dtype, buffer=nl.shared_hbm) + token = nl.ndarray((1, 1), dtype=token_index.dtype, buffer=nl.sbuf) + nisa.dma_copy(dst=token, src=token_index[0:1, 0:1]) + + positions = nl.ndarray((q_len, BLOCK_K), dtype=token_index.dtype, buffer=nl.sbuf) + qk = nl.ndarray((heads_per_kv, BLOCK_K), dtype=nl.float32, buffer=nl.psum) + qk_sbuf = nl.ndarray((heads_per_kv, BLOCK_K), dtype=nl.float32, buffer=nl.sbuf) + masked_qk = nl.ndarray((heads_per_kv, BLOCK_K), dtype=nl.float32, buffer=nl.sbuf) + neg_tile = nl.full( + (heads_per_kv, BLOCK_K), NEG_INF, dtype=nl.float32, buffer=nl.sbuf + ) + tile_max = nl.ndarray((heads_per_kv, 1), dtype=nl.float32, buffer=nl.sbuf) + new_row_max = nl.ndarray((heads_per_kv, 1), dtype=nl.float32, buffer=nl.sbuf) + old_max_delta = nl.ndarray((heads_per_kv, 1), dtype=nl.float32, buffer=nl.sbuf) + old_max_scale = nl.ndarray((heads_per_kv, 1), dtype=nl.float32, buffer=nl.sbuf) + row_max = nl.ndarray((heads_per_kv, 1), dtype=nl.float32, buffer=nl.sbuf) + neg_row_max = nl.ndarray((heads_per_kv, 1), dtype=nl.float32, buffer=nl.sbuf) + exp_tile = nl.ndarray((heads_per_kv, BLOCK_K), dtype=nl.float32, buffer=nl.sbuf) + tile_sum = nl.ndarray((heads_per_kv, 1), dtype=nl.float32, buffer=nl.sbuf) + sum_row = nl.ndarray((heads_per_kv, 1), dtype=nl.float32, buffer=nl.sbuf) + scaled_sum_row = nl.ndarray((heads_per_kv, 1), dtype=nl.float32, buffer=nl.sbuf) + new_sum_row = nl.ndarray((heads_per_kv, 1), dtype=nl.float32, buffer=nl.sbuf) + inverse_sum_row = nl.ndarray((heads_per_kv, 1), dtype=nl.float32, buffer=nl.sbuf) + running_out = nl.ndarray((heads_per_kv, d_head), dtype=nl.float32, buffer=nl.sbuf) + scaled_running_out = nl.ndarray( + (heads_per_kv, d_head), dtype=nl.float32, buffer=nl.sbuf + ) + attn_out_psum = nl.ndarray((heads_per_kv, d_head), dtype=nl.float32, buffer=nl.psum) + attn_out_sbuf = nl.ndarray((heads_per_kv, d_head), dtype=q.dtype, buffer=nl.sbuf) + + prob_t_psum = nl.ndarray( + (VALUE_TILE_TOKENS, heads_per_kv), dtype=nl.float32, buffer=nl.psum + ) + prob_t = nl.ndarray( + (VALUE_TILE_TOKENS, heads_per_kv), dtype=v.dtype, buffer=nl.sbuf + ) + v_sbuf = nl.ndarray((VALUE_TILE_TOKENS, d_head), dtype=v.dtype, buffer=nl.sbuf) + + for kv_head in nl.affine_range(num_kv_heads): + q_head = kv_head * heads_per_kv + + q_sbuf = nl.ndarray((d_head, heads_per_kv), dtype=q.dtype, buffer=nl.sbuf) + nisa.dma_transpose( + dst=q_sbuf, + src=q[0, q_head : q_head + heads_per_kv, :], + ) + nisa.activation(dst=q_sbuf, op=nl.copy, data=q_sbuf, scale=scale) + + nisa.iota(positions, [[1, BLOCK_K]]) + + k_sbuf = nl.ndarray((d_head, BLOCK_K), dtype=k.dtype, buffer=nl.sbuf) + nisa.dma_transpose(dst=k_sbuf, src=k[nl.ds(0, BLOCK_K), kv_head, :]) + nisa.memset(dst=qk, value=0.0) + nisa.nc_matmul(dst=qk, stationary=q_sbuf, moving=k_sbuf) + nisa.tensor_copy(dst=qk_sbuf, src=qk) + + valid = nl.less_equal(positions, nl.broadcast_to(token, positions.shape)) + masked_qk = nl.where( + nl.broadcast_to(valid, masked_qk.shape), + qk_sbuf, + neg_tile, + dtype=nl.float32, + ) + nisa.tensor_reduce(dst=row_max, op=nl.maximum, data=masked_qk, axis=(1,)) + nisa.tensor_scalar( + dst=neg_row_max, + data=row_max, + op0=nl.multiply, + operand0=-1.0, + ) + nisa.activation( + dst=exp_tile, + op=nl.exp, + data=masked_qk, + bias=neg_row_max, + reduce_op=nl.add, + reduce_res=sum_row, + reduce_cmd=nisa.reduce_cmd.reset_reduce, + ) + + nisa.memset(dst=attn_out_psum, value=0.0) + for value_tile_idx in nl.affine_range(value_tiles_per_k_tile): + value_start = value_tile_idx * VALUE_TILE_TOKENS + nisa.nc_transpose( + dst=prob_t_psum, + data=exp_tile[:, nl.ds(value_start, VALUE_TILE_TOKENS)], + ) + nisa.tensor_copy(dst=prob_t, src=prob_t_psum) + + nisa.dma_copy( + dst=v_sbuf, + src=v[nl.ds(value_start, VALUE_TILE_TOKENS), kv_head, :], + ) + nisa.nc_matmul(dst=attn_out_psum, stationary=prob_t, moving=v_sbuf) + nisa.tensor_copy(dst=running_out, src=attn_out_psum) + + for k_tail_idx in nl.affine_range(num_kv_blocks - 1): + k_block_idx = k_tail_idx + 1 + k_start = k_block_idx * BLOCK_K + nisa.iota(positions, [[1, BLOCK_K]], offset=k_start) + + k_sbuf = nl.ndarray((d_head, BLOCK_K), dtype=k.dtype, buffer=nl.sbuf) + nisa.dma_transpose(dst=k_sbuf, src=k[nl.ds(k_start, BLOCK_K), kv_head, :]) + nisa.memset(dst=qk, value=0.0) + nisa.nc_matmul(dst=qk, stationary=q_sbuf, moving=k_sbuf) + nisa.tensor_copy(dst=qk_sbuf, src=qk) + + valid = nl.less_equal(positions, nl.broadcast_to(token, positions.shape)) + masked_qk = nl.where( + nl.broadcast_to(valid, masked_qk.shape), + qk_sbuf, + neg_tile, + dtype=nl.float32, + ) + nisa.tensor_reduce(dst=tile_max, op=nl.maximum, data=masked_qk, axis=(1,)) + nisa.tensor_tensor( + dst=new_row_max, + data1=row_max, + data2=tile_max, + op=nl.maximum, + ) + nisa.tensor_scalar( + dst=old_max_delta, + data=row_max, + op0=nl.subtract, + operand0=new_row_max, + ) + nisa.activation(dst=old_max_scale, op=nl.exp, data=old_max_delta) + nisa.tensor_scalar( + dst=neg_row_max, + data=new_row_max, + op0=nl.multiply, + operand0=-1.0, + ) + nisa.activation( + dst=exp_tile, + op=nl.exp, + data=masked_qk, + bias=neg_row_max, + reduce_op=nl.add, + reduce_res=tile_sum, + reduce_cmd=nisa.reduce_cmd.reset_reduce, + ) + nisa.tensor_scalar( + dst=scaled_sum_row, + data=sum_row, + op0=nl.multiply, + operand0=old_max_scale, + ) + nisa.tensor_tensor( + dst=new_sum_row, + data1=scaled_sum_row, + data2=tile_sum, + op=nl.add, + ) + nisa.tensor_scalar( + dst=scaled_running_out, + data=running_out, + op0=nl.multiply, + operand0=old_max_scale, + ) + + nisa.memset(dst=attn_out_psum, value=0.0) + for value_tile_idx in nl.affine_range(value_tiles_per_k_tile): + value_start = value_tile_idx * VALUE_TILE_TOKENS + nisa.nc_transpose( + dst=prob_t_psum, + data=exp_tile[:, nl.ds(value_start, VALUE_TILE_TOKENS)], + ) + nisa.tensor_copy(dst=prob_t, src=prob_t_psum) + + nisa.dma_copy( + dst=v_sbuf, + src=v[nl.ds(k_start + value_start, VALUE_TILE_TOKENS), kv_head, :], + ) + nisa.nc_matmul(dst=attn_out_psum, stationary=prob_t, moving=v_sbuf) + + nisa.tensor_tensor( + dst=running_out, + data1=scaled_running_out, + data2=attn_out_psum, + op=nl.add, + ) + nisa.tensor_copy(dst=row_max, src=new_row_max) + nisa.tensor_copy(dst=sum_row, src=new_sum_row) + + nisa.reciprocal(dst=inverse_sum_row, data=sum_row) + nisa.tensor_scalar( + dst=running_out, + data=running_out, + op0=nl.multiply, + operand0=inverse_sum_row, + ) + nisa.tensor_copy(dst=attn_out_sbuf, src=running_out) + + nisa.dma_copy( + dst=out[0, q_head : q_head + heads_per_kv, :], + src=attn_out_sbuf, + ) + + return out + + +def prefill(q, k, v): + q_len = q.shape[0] + num_q_heads = q.shape[1] + d_head = q.shape[2] + seq_len = k.shape[0] + num_kv_heads = k.shape[1] + heads_per_kv = num_q_heads // num_kv_heads + scale = 1.0 / (d_head**0.5) + + assert q.shape[2] == k.shape[2] + assert k.shape == v.shape + assert num_q_heads % num_kv_heads == 0 + assert q_len == seq_len + + VALUE_TILE_TOKENS = nl.tile_size.pmax + BLOCK_Q = nl.tile_size.pmax + # Largest tensor-engine K tile that exactly partitions this static sequence. + BLOCK_K = math.gcd(seq_len, nl.tile_size.gemm_moving_fmax) + + assert q_len % BLOCK_Q == 0, ( + f"q_len ({q_len}) must be a multiple of BLOCK_Q ({BLOCK_Q})" + ) + assert seq_len % BLOCK_K == 0, ( + f"seq_len ({seq_len}) must be a multiple of BLOCK_K ({BLOCK_K})" + ) + assert BLOCK_K % BLOCK_Q == 0, ( + f"BLOCK_K ({BLOCK_K}) must be a multiple of BLOCK_Q ({BLOCK_Q})" + ) + assert BLOCK_K % VALUE_TILE_TOKENS == 0, ( + "BLOCK_K " + f"({BLOCK_K}) must be a multiple of VALUE_TILE_TOKENS ({VALUE_TILE_TOKENS})" + ) + + num_q_blocks = q_len // BLOCK_Q + q_blocks_per_k_tile = BLOCK_K // BLOCK_Q + value_tiles_per_k_tile = BLOCK_K // VALUE_TILE_TOKENS + + out = nl.ndarray((q_len, num_q_heads, d_head), dtype=q.dtype, buffer=nl.shared_hbm) + + for kv_head in nl.affine_range(num_kv_heads): + for q_idx in nl.affine_range(heads_per_kv): + q_head = kv_head * heads_per_kv + q_idx + + for q_block_idx in nl.affine_range(num_q_blocks): + q_start = q_block_idx * BLOCK_Q + + q_sbuf = nl.ndarray((d_head, BLOCK_Q), dtype=q.dtype, buffer=nl.sbuf) + nisa.dma_transpose( + dst=q_sbuf, + src=q[nl.ds(q_start, BLOCK_Q), q_head, :], + ) + nisa.activation(dst=q_sbuf, op=nl.copy, data=q_sbuf, scale=scale) + + row_max = nl.ndarray((BLOCK_Q, 1), dtype=nl.float32, buffer=nl.sbuf) + sum_row = nl.ndarray((BLOCK_Q, 1), dtype=nl.float32, buffer=nl.sbuf) + running_out = nl.ndarray( + (BLOCK_Q, d_head), dtype=nl.float32, buffer=nl.sbuf + ) + + masked_qk = nl.ndarray( + (BLOCK_Q, BLOCK_K), dtype=nl.float32, buffer=nl.sbuf + ) + qk_sbuf = nl.ndarray( + (BLOCK_Q, BLOCK_K), dtype=nl.float32, buffer=nl.sbuf + ) + tile_max = nl.ndarray((BLOCK_Q, 1), dtype=nl.float32, buffer=nl.sbuf) + new_row_max = nl.ndarray((BLOCK_Q, 1), dtype=nl.float32, buffer=nl.sbuf) + old_max_delta = nl.ndarray( + (BLOCK_Q, 1), dtype=nl.float32, buffer=nl.sbuf + ) + old_max_scale = nl.ndarray( + (BLOCK_Q, 1), dtype=nl.float32, buffer=nl.sbuf + ) + neg_row_max = nl.ndarray((BLOCK_Q, 1), dtype=nl.float32, buffer=nl.sbuf) + exp_tile = nl.ndarray( + (BLOCK_Q, BLOCK_K), dtype=nl.float32, buffer=nl.sbuf + ) + tile_sum = nl.ndarray((BLOCK_Q, 1), dtype=nl.float32, buffer=nl.sbuf) + scaled_sum_row = nl.ndarray( + (BLOCK_Q, 1), dtype=nl.float32, buffer=nl.sbuf + ) + new_sum_row = nl.ndarray((BLOCK_Q, 1), dtype=nl.float32, buffer=nl.sbuf) + scaled_running_out = nl.ndarray( + (BLOCK_Q, d_head), dtype=nl.float32, buffer=nl.sbuf + ) + attn_out_psum = nl.ndarray( + (BLOCK_Q, d_head), dtype=nl.float32, buffer=nl.psum + ) + prob_t_psum = nl.ndarray( + (VALUE_TILE_TOKENS, BLOCK_Q), dtype=nl.float32, buffer=nl.psum + ) + prob_t = nl.ndarray( + (VALUE_TILE_TOKENS, BLOCK_Q), dtype=v.dtype, buffer=nl.sbuf + ) + qk_psum = nl.ndarray( + (BLOCK_Q, BLOCK_K), dtype=nl.float32, buffer=nl.psum + ) + v_sbuf = nl.ndarray( + (VALUE_TILE_TOKENS, d_head), dtype=v.dtype, buffer=nl.sbuf + ) + + nisa.memset(dst=row_max, value=NEG_INF) + nisa.memset(dst=sum_row, value=0.0) + nisa.memset(dst=running_out, value=0.0) + active_k_tiles = q_block_idx // q_blocks_per_k_tile + 1 + + for k_block_idx in nl.affine_range(active_k_tiles): + k_start = k_block_idx * BLOCK_K + k_sbuf = nl.ndarray( + (d_head, BLOCK_K), dtype=k.dtype, buffer=nl.sbuf + ) + nisa.dma_transpose( + dst=k_sbuf, + src=k[nl.ds(k_start, BLOCK_K), kv_head, :], + ) + nisa.memset(dst=qk_psum, value=0.0) + nisa.nc_matmul(dst=qk_psum, stationary=q_sbuf, moving=k_sbuf) + nisa.tensor_copy(dst=qk_sbuf, src=qk_psum) + nisa.affine_select( + dst=masked_qk, + pattern=[[-1, BLOCK_K]], + offset=q_start - k_start, + channel_multiplier=1, + on_true_tile=qk_sbuf, + on_false_value=NEG_INF, + cmp_op=nl.greater_equal, + ) + nisa.tensor_reduce( + dst=tile_max, + op=nl.maximum, + data=masked_qk, + axis=(1,), + ) + nisa.tensor_tensor( + dst=new_row_max, + data1=row_max, + data2=tile_max, + op=nl.maximum, + ) + nisa.tensor_scalar( + dst=old_max_delta, + data=row_max, + op0=nl.subtract, + operand0=new_row_max, + ) + nisa.activation(dst=old_max_scale, op=nl.exp, data=old_max_delta) + nisa.tensor_scalar( + dst=neg_row_max, + data=new_row_max, + op0=nl.multiply, + operand0=-1.0, + ) + nisa.activation( + dst=exp_tile, + op=nl.exp, + data=masked_qk, + bias=neg_row_max, + reduce_op=nl.add, + reduce_res=tile_sum, + reduce_cmd=nisa.reduce_cmd.reset_reduce, + ) + nisa.tensor_scalar( + dst=scaled_sum_row, + data=sum_row, + op0=nl.multiply, + operand0=old_max_scale, + ) + nisa.tensor_tensor( + dst=new_sum_row, + data1=scaled_sum_row, + data2=tile_sum, + op=nl.add, + ) + nisa.tensor_scalar( + dst=scaled_running_out, + data=running_out, + op0=nl.multiply, + operand0=old_max_scale, + ) + + nisa.memset(dst=attn_out_psum, value=0.0) + for value_tile_idx in nl.affine_range(value_tiles_per_k_tile): + value_start = value_tile_idx * VALUE_TILE_TOKENS + nisa.nc_transpose( + dst=prob_t_psum, + data=exp_tile[:, nl.ds(value_start, VALUE_TILE_TOKENS)], + ) + nisa.tensor_copy(dst=prob_t, src=prob_t_psum) + nisa.dma_copy( + dst=v_sbuf, + src=v[ + nl.ds(k_start + value_start, VALUE_TILE_TOKENS), + kv_head, + :, + ], + ) + nisa.nc_matmul( + dst=attn_out_psum, + stationary=prob_t, + moving=v_sbuf, + ) + + nisa.tensor_tensor( + dst=running_out, + data1=scaled_running_out, + data2=attn_out_psum, + op=nl.add, + ) + nisa.tensor_copy(dst=row_max, src=new_row_max) + nisa.tensor_copy(dst=sum_row, src=new_sum_row) + + inv_sum_row = nl.ndarray((BLOCK_Q, 1), dtype=nl.float32, buffer=nl.sbuf) + normalized_out = nl.ndarray( + (BLOCK_Q, d_head), dtype=nl.float32, buffer=nl.sbuf + ) + attn_out = nl.ndarray((BLOCK_Q, d_head), dtype=q.dtype, buffer=nl.sbuf) + nisa.reciprocal(dst=inv_sum_row, data=sum_row) + nisa.tensor_scalar( + dst=normalized_out, + data=running_out, + op0=nl.multiply, + operand0=inv_sum_row, + ) + nisa.tensor_copy(dst=attn_out, src=normalized_out) + + nisa.dma_copy( + dst=out[nl.ds(q_start, BLOCK_Q), q_head, :], + src=attn_out, + ) + + return out diff --git a/zml/kernels/flash_attention/nki/attention_harness.py b/zml/kernels/flash_attention/nki/attention_harness.py new file mode 100644 index 00000000..34ce5717 --- /dev/null +++ b/zml/kernels/flash_attention/nki/attention_harness.py @@ -0,0 +1,331 @@ +import argparse +import dataclasses +import importlib.util +import os +import sys +import time +from pathlib import Path + +import ml_dtypes +import nki +import numpy as np +from python.runfiles import Runfiles + + +DEFAULT_SEQ_LEN = 128 +ATOL = 1e-2 +RTOL = 1e-2 +MIN_CLOSE_FRACTION = 0.99 +INPUT_CASES = ("baseline", "signed") + + +@dataclasses.dataclass(frozen=True) +class ModelSpec: + name: str + num_heads: int + num_kv_heads: int + head_dim: int + + @property + def heads_per_kv(self): + return self.num_heads // self.num_kv_heads + + +MODEL_PRESETS = { + "llama-3.1-8b": ModelSpec( + name="llama-3.1-8b", + num_heads=32, + num_kv_heads=8, + head_dim=128, + ), +} + + +def parse_args(): + parser = argparse.ArgumentParser( + description="Run the classic Neuron NKI attention correctness contract." + ) + parser.add_argument( + "--model", + choices=(*MODEL_PRESETS.keys(), "custom"), + default="llama-3.1-8b", + ) + parser.add_argument("--seq-len", type=int, default=DEFAULT_SEQ_LEN) + parser.add_argument("--dtype", choices=("bf16", "f16", "f32"), default="bf16") + parser.add_argument("--run", choices=("both", "prefill", "decode"), default="both") + parser.add_argument("--mode", choices=("verify", "benchmark"), default="verify") + parser.add_argument("--warmups", type=int, default=1) + parser.add_argument("--iterations", type=int, default=5) + parser.add_argument("--num-heads", type=int, default=None) + parser.add_argument("--num-kv-heads", type=int, default=None) + parser.add_argument("--head-dim", type=int, default=None) + return parser.parse_args() + + +def model_spec(args): + custom_values = (args.num_heads, args.num_kv_heads, args.head_dim) + if args.model != "custom": + if any(value is not None for value in custom_values): + raise ValueError("shape flags are only accepted with --model=custom") + return MODEL_PRESETS[args.model] + + if any(value is None for value in custom_values): + raise ValueError("--model=custom requires --num-heads, --num-kv-heads, and --head-dim") + spec = ModelSpec( + name="custom", + num_heads=args.num_heads, + num_kv_heads=args.num_kv_heads, + head_dim=args.head_dim, + ) + validate_spec(spec) + return spec + + +def validate_spec(spec): + if spec.num_heads <= 0 or spec.num_kv_heads <= 0 or spec.head_dim <= 0: + raise ValueError("model dimensions must be positive") + if spec.num_heads % spec.num_kv_heads != 0: + raise ValueError("num_heads must be divisible by num_kv_heads") + + +def validate_seq_len(seq_len): + if seq_len <= 0 or seq_len % 128 != 0: + raise ValueError("--seq-len must be a positive multiple of 128") + + +def validate_execution_args(args): + if args.mode == "verify": + return + if args.warmups < 0: + raise ValueError("--warmups must be non-negative") + if args.iterations <= 0: + raise ValueError("--iterations must be positive") + + +def decode_positions(seq_len): + return tuple(dict.fromkeys((0, 1, seq_len // 2, seq_len - 1))) + + +def configure_runtime_environment(mode): + if mode == "benchmark": + os.environ.pop("NEURON_RT_LOG_LEVEL", None) + + +def load_nki_attention_module(): + path = Path(__file__).with_name("attention.py") + spec = importlib.util.spec_from_file_location("zml_nki_attention", path) + if spec is None or spec.loader is None: + raise RuntimeError(f"could not load {path}") + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def add_neuron_tools_to_path(): + runfiles = Runfiles.Create() + python_neuronx_cc = Path( + runfiles.Rlocation("+neuron_packages+libzml_neuron/sandbox/bin/python-shims/neuronx-cc") + ) + neuronx_cc = Path( + runfiles.Rlocation("+neuron_packages+libzml_neuron/sandbox/bin/neuronx-cc") + ) + os.environ["PATH"] = os.pathsep.join( + (str(python_neuronx_cc.parent), str(neuronx_cc.parent), os.environ["PATH"]) + ) + + +def numpy_dtype(name): + if name == "bf16": + return ml_dtypes.bfloat16 + if name == "f16": + return np.float16 + if name == "f32": + return np.float32 + raise ValueError(f"unsupported dtype: {name}") + + +def fill_input(shape, offset, input_case, dtype): + values = np.linspace(0.0, 1.0, num=int(np.prod(shape)), endpoint=False, dtype=np.float32) + if input_case == "signed": + data = offset + (values - 0.5) * 0.25 + else: + data = offset + values * 0.03 + return data.reshape(shape).astype(dtype) + + +def qkv_inputs(spec, seq_len, input_case, decode, dtype): + q_len = 1 if decode else seq_len + q = fill_input((q_len, spec.num_heads, spec.head_dim), 0.01, input_case, dtype) + k = fill_input((seq_len, spec.num_kv_heads, spec.head_dim), 0.02, input_case, dtype) + v = fill_input((seq_len, spec.num_kv_heads, spec.head_dim), 0.03, input_case, dtype) + return q, k, v + + +def reference_attention(q, k, v, token_index=None): + q_len, num_heads, head_dim = q.shape + seq_len, num_kv_heads, _ = k.shape + heads_per_kv = num_heads // num_kv_heads + scale = 1.0 / np.sqrt(head_dim) + q_f32 = q.astype(np.float32) + k_f32 = k.astype(np.float32) + v_f32 = v.astype(np.float32) + out = np.empty((q_len, num_heads, head_dim), dtype=np.float32) + + if token_index is None: + valid = np.arange(seq_len)[None, :] <= np.arange(q_len)[:, None] + else: + if q_len != 1: + raise ValueError("decode reference expects q_len=1") + valid = (np.arange(seq_len) <= token_index)[None, :] + + for kv_head in range(num_kv_heads): + keys = k_f32[:, kv_head, :] + values = v_f32[:, kv_head, :] + for q_idx in range(heads_per_kv): + q_head = kv_head * heads_per_kv + q_idx + scores = (q_f32[:, q_head, :] @ keys.T) * scale + scores = np.where(valid, scores, -np.inf) + scores -= np.max(scores, axis=1, keepdims=True) + probs = np.exp(scores) + probs /= np.sum(probs, axis=1, keepdims=True) + out[:, q_head, :] = probs @ values + + return out.astype(q.dtype) + + +def assert_attention_close(label, expected, actual): + expected_f32 = expected.astype(np.float32) + actual_f32 = actual.astype(np.float32) + finite = np.isfinite(actual_f32) + close = np.isclose(expected_f32, actual_f32, atol=ATOL, rtol=RTOL) + close_fraction = float(close.mean()) + max_abs = float(np.nanmax(np.abs(expected_f32 - actual_f32))) + print( + f"{label} finite={int(finite.sum())}/{actual_f32.size} " + f"close_fraction={close_fraction:.6f} max_abs={max_abs:.6g}" + ) + if not finite.all(): + raise AssertionError(f"{label}: output contains NaN or Inf") + if close_fraction < MIN_CLOSE_FRACTION: + raise AssertionError(f"{label}: close_fraction={close_fraction:.6f}") + + +def timed_run(kernel, inputs, mode, warmups, iterations): + if mode == "verify": + start = time.perf_counter() + output = kernel(*inputs) + elapsed = time.perf_counter() - start + return output, elapsed, 1 + + output = kernel(*inputs) + for _ in range(warmups): + output = kernel(*inputs) + + total = 0.0 + for _ in range(iterations): + start = time.perf_counter() + output = kernel(*inputs) + total += time.perf_counter() - start + return output, total, iterations + + +def print_timing(label, elapsed, iterations, output): + if iterations == 1: + print(f"{label} elapsed={elapsed * 1000.0:.3f}ms output={output.shape}") + return + + average = elapsed / iterations + print( + f"{label} total={elapsed * 1000.0:.3f}ms " + f"avg={average * 1000.0:.3f}ms iterations={iterations} output={output.shape}" + ) + + +def run_prefill(prefill_kernel, spec, seq_len, input_case, dtype, mode, warmups, iterations): + q, k, v = qkv_inputs(spec, seq_len, input_case, decode=False, dtype=dtype) + output, elapsed, timed_iterations = timed_run( + prefill_kernel, + (q, k, v), + mode, + warmups, + iterations, + ) + label = f"prefill input_case={input_case}" + if mode == "verify": + expected = reference_attention(q, k, v) + assert_attention_close(label, expected, output) + print_timing(label, elapsed, timed_iterations, output) + + +def run_decode(decode_kernel, spec, seq_len, input_case, token_index, dtype, mode, warmups, iterations): + q, k, v = qkv_inputs(spec, seq_len, input_case, decode=True, dtype=dtype) + token = np.array([[token_index]], dtype=np.uint32) + output, elapsed, timed_iterations = timed_run( + decode_kernel, + (q, k, v, token), + mode, + warmups, + iterations, + ) + label = f"decode input_case={input_case} token_index={token_index}" + if mode == "verify": + expected = reference_attention(q, k, v, token_index) + assert_attention_close(label, expected, output) + print_timing(label, elapsed, timed_iterations, output) + + +def main(): + args = parse_args() + spec = model_spec(args) + validate_spec(spec) + validate_seq_len(args.seq_len) + validate_execution_args(args) + positions = decode_positions(args.seq_len) + dtype = numpy_dtype(args.dtype) + + configure_runtime_environment(args.mode) + add_neuron_tools_to_path() + module = load_nki_attention_module() + prefill_kernel = nki.jit(module.prefill) + decode_kernel = nki.jit(module.decode) + + header = ( + "classic_attention_contract " + f"model={spec.name} seq_len={args.seq_len} num_heads={spec.num_heads} " + f"num_kv_heads={spec.num_kv_heads} heads_per_kv={spec.heads_per_kv} " + f"head_dim={spec.head_dim} dtype={args.dtype} run={args.run} " + f"mode={args.mode}" + ) + if args.mode == "benchmark": + header += f" warmups={args.warmups} iterations={args.iterations}" + print(f"{header} decode_positions={positions}") + for input_case in INPUT_CASES: + if args.run in ("both", "prefill"): + run_prefill( + prefill_kernel, + spec, + args.seq_len, + input_case, + dtype, + args.mode, + args.warmups, + args.iterations, + ) + if args.run in ("both", "decode"): + for token_index in positions: + run_decode( + decode_kernel, + spec, + args.seq_len, + input_case, + token_index, + dtype, + args.mode, + args.warmups, + args.iterations, + ) + + +if __name__ == "__main__": + main() diff --git a/zml/kernels/flash_attention/nki/nki.zig b/zml/kernels/flash_attention/nki/nki.zig new file mode 100644 index 00000000..f98a545d --- /dev/null +++ b/zml/kernels/flash_attention/nki/nki.zig @@ -0,0 +1,59 @@ +const platforms = @import("platforms"); + +const zml = @import("../../../zml.zig"); +const stdx = zml.stdx; + +const KernelSpec = struct { + name: []const u8, + entrypoint: []const u8, + source_path: []const u8, +}; + +pub const Parameters = struct { + compiler_target: []const u8, + decode_kernel: KernelSpec, + prefill_kernel: KernelSpec, + + pub fn init() Parameters { + if (comptime platforms.isEnabled(.neuron)) { + const nki_kernel = @import("platforms/neuron/nki_kernel"); + const compiler_target = nki_kernel.compilerTargetFromInstance(); + return .{ + .compiler_target = @tagName(compiler_target), + .decode_kernel = switch (compiler_target) { + .trn1, .inf2, .trn1n, .trn2, .trn2n, .trn3 => .{ + .name = "nki_attention_trn1_decode", + .entrypoint = "decode", + .source_path = "zml/zml/attention/nki/attention.py", + }, + }, + .prefill_kernel = switch (compiler_target) { + .trn1, .inf2, .trn1n, .trn2, .trn2n, .trn3 => .{ + .name = "nki_attention_trn1_prefill", + .entrypoint = "prefill", + .source_path = "zml/zml/attention/nki/attention.py", + }, + }, + }; + } + + stdx.debug.panic("NKI attention requires the Neuron platform", .{}); + } +}; + +/// Attention for Neuron. Both prefill and decode lower to NKI custom calls. +pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, parameters: Parameters) zml.Tensor { + const decode = q.dim(.q) == 1; + const kernel = if (decode) parameters.decode_kernel else parameters.prefill_kernel; + const options: zml.ops.NeuronNkiOps = .{ + .name = kernel.name, + .entrypoint = kernel.entrypoint, + .source_path = kernel.source_path, + .compiler_target = parameters.compiler_target, + }; + const output = if (decode) + zml.ops.neuronNki(.{ q, k, v, token_index.broad(zml.Shape.init(.{ .row = 1, .col = 1 }, token_index.dtype())) }, .{q.shape()}, options)[0] + else + zml.ops.neuronNki(.{ q, k, v }, .{q.shape()}, options)[0]; + return output.convert(q.dtype()); +} diff --git a/zml/kernels/flash_attention/tests.zig b/zml/kernels/flash_attention/tests.zig new file mode 100644 index 00000000..c3c3dca2 --- /dev/null +++ b/zml/kernels/flash_attention/tests.zig @@ -0,0 +1,346 @@ +const std = @import("std"); + +const zml = @import("../../zml.zig"); +const flash_attention = @import("flash_attention.zig"); + +test "attention: q=1,qh=64,kh=8" { + try testAttention( + .init(.{ .q = 1, .h = 64, .hd = 64 }, .bf16), + .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{63}, + ); + + try testAttention( + .init(.{ .q = 1, .h = 64, .hd = 64 }, .bf16), + .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{36}, + ); +} + +test "attention: b=4,q=1,qh=64,kh=8" { + // Full attention + try testAttention( + .init(.{ .b = 4, .q = 1, .h = 64, .hd = 64 }, .bf16), + .init(.{ .b = 4, .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{ 63, 63, 63, 63 }, + ); + + // Partial attention + try testAttention( + .init(.{ .b = 4, .q = 1, .h = 64, .hd = 64 }, .bf16), + .init(.{ .b = 4, .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{ 61, 57, 23, 63 }, + ); +} + +test "attention: q=1,qh=8,kh=8" { + try testAttention( + .init(.{ .q = 1, .h = 8, .hd = 64 }, .bf16), + .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{62}, + ); + try testAttention( + .init(.{ .q = 1, .h = 8, .hd = 64 }, .bf16), + .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{63}, + ); +} + +test "attention: q=1,k=4096,qh=32,kh=8,hd=128" { + try testAttention( + .init(.{ .q = 1, .h = 32, .hd = 128 }, .bf16), + .init(.{ .k = 4096, .h = 8, .hd = 128 }, .bf16), + &.{3000}, + ); +} + +test "attention: q=1,k=1000,qh=8,kh=2,hd=32" { + try testAttention( + .init(.{ .q = 1, .h = 8, .hd = 32 }, .bf16), + .init(.{ .k = 1000, .h = 2, .hd = 32 }, .bf16), + &.{900}, + ); +} + +test "attention: q=8,qh=64,kh=8" { + try testAttention( + .init(.{ .q = 8, .h = 64, .hd = 64 }, .bf16), + .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), + &.{56}, + ); +} + +test "attention: prefill, q=k=300,qh=8,kh=4" { + // Neither length is a multiple of the query (128) or key (64) tiles. + try testAttention( + .init(.{ .q = 300, .h = 8, .hd = 128 }, .bf16), + .init(.{ .k = 300, .h = 4, .hd = 128 }, .bf16), + &.{0}, + ); +} + +test "attention: prefill, q=200,k=1000,qh=4,kh=4,hd=64" { + // Queries start at 800: every query tile sees full key tiles before the diagonal. + try testAttention( + .init(.{ .q = 200, .h = 4, .hd = 64 }, .bf16), + .init(.{ .k = 1000, .h = 4, .hd = 64 }, .bf16), + &.{800}, + ); +} + +test "attention: prefill, q=k=200,qh=4,kh=2,hd=32" { + // Rows of 32 features only hold 4 16-byte vectors: the shared memory swizzle must stay inside them. + try testAttention( + .init(.{ .q = 200, .h = 4, .hd = 32 }, .bf16), + .init(.{ .k = 200, .h = 2, .hd = 32 }, .bf16), + &.{0}, + ); +} + +test "attention: prefill, q=k=130,qh=4,kh=2,hd=256" { + try testAttention( + .init(.{ .q = 130, .h = 4, .hd = 256 }, .bf16), + .init(.{ .k = 130, .h = 2, .hd = 256 }, .bf16), + &.{0}, + ); +} + +test "attention: prefill, b=2,q=150,k=300,qh=4,kh=2" { + try testAttention( + .init(.{ .b = 2, .q = 150, .h = 4, .hd = 128 }, .bf16), + .init(.{ .b = 2, .k = 300, .h = 2, .hd = 128 }, .bf16), + &.{ 150, 100 }, + ); +} + +test "attention: non-causal prefill, q=k=300,qh=8,kh=4" { + try testAttentionOpts( + .init(.{ .q = 300, .h = 8, .hd = 128 }, .bf16), + .init(.{ .k = 300, .h = 4, .hd = 128 }, .bf16), + &.{0}, + .{ .causal = false }, + ); +} + +test "attention: non-causal prefill, b=2,q=200,k=1000,qh=4,kh=4,hd=64" { + // Queries are a suffix of the keys, as for the image tokens of a joint text/image attention. + try testAttentionOpts( + .init(.{ .b = 2, .q = 200, .h = 4, .hd = 64 }, .bf16), + .init(.{ .b = 2, .k = 1000, .h = 4, .hd = 64 }, .bf16), + &.{ 0, 0 }, + .{ .causal = false }, + ); +} + +test "attention: non-causal prefill, more queries than keys, q=300,k=100,qh=4,kh=2" { + try testAttentionOpts( + .init(.{ .q = 300, .h = 4, .hd = 128 }, .bf16), + .init(.{ .k = 100, .h = 2, .hd = 128 }, .bf16), + &.{0}, + .{ .causal = false }, + ); +} + +test "attention: non-causal decode ignores token_index, q=1,k=1000,qh=64,kh=8" { + try testAttentionOpts( + .init(.{ .q = 1, .h = 64, .hd = 64 }, .bf16), + .init(.{ .k = 1000, .h = 8, .hd = 64 }, .bf16), + &.{5}, + .{ .causal = false }, + ); +} + +pub fn testAttention(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []const u32) !void { + return testAttentionOpts(q_shape, k_shape, token_index_h, .{}); +} + +fn testAttentionOpts(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []const u32, opts: zml.attention.Options) !void { + return testAttentionBackends(q_shape, k_shape, token_index_h, opts, std.enums.values(flash_attention.Backend)); +} + +fn testAttentionBackends(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []const u32, opts: zml.attention.Options, backends: []const flash_attention.Backend) !void { + const io = std.testing.io; + const allocator = std.testing.allocator; + + const platform = zml.testing.env(); + + var arena_state: std.heap.ArenaAllocator = .init(allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + + const token_index_shape: zml.Shape = if (token_index_h.len > 1) b: { + std.debug.assert(token_index_h.len == q_shape.dim(.b)); + break :b .init(.{ .b = q_shape.dim(.b) }, .u32); + } else .init(.{}, .u32); + + const max_k_idx: i64 = k_shape.dim(.k) - 1; + const max_q_idx: i64 = q_shape.dim(.q) - 1; + if (opts.causal) { + // Check for out of bound reads, token_index is ignored by non-causal attention. + for (token_index_h) |index| { + std.debug.assert(index + max_q_idx <= max_k_idx); + } + } + + const tensors: struct { q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor } = .{ + .q = .fromShape(q_shape), + .k = .fromShape(k_shape), + .v = .fromShape(k_shape), + .token_index = .fromShape(token_index_shape), + }; + + var q = try attentionTestInput(q_shape, 17); + defer q.deinit(); + var k = try attentionTestInput(k_shape, 29); + defer k.deinit(); + var v = try attentionTestInput(k_shape, 43); + defer v.deinit(); + var token_index = try zml.Buffer.fromBytes(io, platform, token_index_shape, @ptrCast(token_index_h)); + defer token_index.deinit(); + + const meshes = platform.meshes.values(); + const mesh = zml.testing.tpMesh(); + const partitioning: zml.attention.Partitioning = if (@rem(k_shape.dim(.h), mesh.numPartitionsForLogicalAxis(.model)) == 0) .head(.model) else .{ + .in_specs = .{ .q = .replicated, .k = .replicated, .v = .replicated }, + .out_specs = .replicated, + }; + const AttentionCall = struct { + fn forward(q_: zml.Tensor, k_: zml.Tensor, v_: zml.Tensor, position: zml.Tensor, metadata: zml.attention.Metadata, parameters: zml.attention.Parameters, opts_: zml.attention.Options, mesh_: *const zml.Mesh, partitioning_: zml.attention.Partitioning) zml.Tensor { + return zml.attention.attentionOpts(q_, k_, v_, position, metadata, parameters, opts_, mesh_, partitioning_); + } + }; + const reference_metadata: zml.attention.Metadata = .{ .stablehlo = {} }; + const reference_parameters: zml.attention.Parameters = .{ .stablehlo = {} }; + const reference_exe = try platform.compileFn(allocator, io, AttentionCall.forward, .{ tensors.q, tensors.k, tensors.v, tensors.token_index, reference_metadata, reference_parameters, opts, mesh, partitioning }, .{ + .program_name = "attention_stablehlo", + .meshes = meshes, + }); + defer reference_exe.deinit(); + + var reference_d = try reference_exe.eval(allocator, io, .{ q, k, v, token_index, reference_metadata }); + defer reference_d.deinit(); + try reference_d.await(io); + try std.testing.expectEqualDeep(q_shape.withPartitioning(mesh, partitioning.out_specs)._sharding, reference_d.shape()._sharding); + const reference_h: zml.Slice = try reference_d.toSliceAlloc(allocator, io); + defer reference_h.free(allocator); + + for (backends) |backend| { + if (!backend.isAvailable(platform)) continue; + + const metadata: flash_attention.Metadata = .init(.fromBackend(backend)); + const parameters: flash_attention.Parameters = .init(.fromBackend(backend)); + const exe = try platform.compileFn( + allocator, + io, + AttentionCall.forward, + .{ tensors.q, tensors.k, tensors.v, tensors.token_index, .{ .flash_attention = metadata }, .{ .flash_attention = parameters }, opts, mesh, partitioning }, + .{ + .program_name = try std.fmt.allocPrint(arena, "attention_{t}", .{backend}), + .meshes = meshes, + }, + ); + defer exe.deinit(); + + var metadata_d = try metadata.initBuffer(io, platform); + defer flash_attention.Metadata.deinitBuffer(&metadata_d); + + var output_d = try exe.eval(allocator, io, .{ q, k, v, token_index, metadata_d }); + defer output_d.deinit(); + try output_d.await(io); + const output_h = try output_d.toSliceAlloc(allocator, io); + defer output_h.free(allocator); + + errdefer std.log.err( + \\ Attention test failed, {0t} output doesn't match reference. + \\ - reference: {1d} + \\ - {0t}: {2d} + , .{ backend, reference_h, output_h }); + try zml.testing.expectClose(io, reference_h, output_h, .{ + .absolute_tolerance = 5e-3, + .relative_tolerance = 1e-2, + .epsilon_relative = 1e-3, + .minimum_close_fraction = 0.99, + }); + } +} + +fn attentionTestInput(shape: zml.Shape, seed: u64) !zml.Buffer { + const host = try zml.Slice.alloc(std.testing.allocator, shape); + defer host.free(std.testing.allocator); + var prng: std.Random.DefaultPrng = .init(seed); + const random = prng.random(); + switch (shape.dtype()) { + inline .bf16, .f16, .f32 => |dtype| { + for (host.items(dtype.toZigType())) |*value| { + const x = random.float(f32) * 4 - 2; + value.* = if (dtype == .bf16) .fromF32(x) else @floatCast(x); + } + }, + else => unreachable, + } + return zml.Buffer.fromSlice(std.testing.io, zml.testing.env(), host); +} + +/// Independent f64 oracle: the SDPA reference uses fast dot precision, so its +/// f32 result alone cannot establish the accuracy of the kernel's f32 arithmetic. +fn checkAttentionF32(q: zml.Slice, k: zml.Slice, v: zml.Slice, positions: []const u32, output: zml.Slice) !void { + const Layout = struct { + b: usize, + s: usize, + h: usize, + d: usize, + + fn stride(shape: zml.Shape, tag: anytype) usize { + const axis = shape.hasTag(tag) orelse return 0; + var result: usize = 1; + for (shape.dims()[axis + 1 ..]) |dim| result *= @intCast(dim); + return result; + } + + fn init(shape: zml.Shape) @This() { + return .{ .b = stride(shape, .b), .s = if (shape.hasTag(.q) != null) stride(shape, .q) else stride(shape, .k), .h = stride(shape, .h), .d = stride(shape, .hd) }; + } + + fn offset(self: @This(), b: usize, s: usize, h: usize, d: usize) usize { + return b * self.b + s * self.s + h * self.h + d * self.d; + } + }; + const ql = Layout.init(q.shape); + const kl = Layout.init(k.shape); + const vl = Layout.init(v.shape); + const batch: usize = if (q.shape.hasTag(.b) != null) @intCast(q.shape.dim(.b)) else 1; + const queries: usize = @intCast(q.shape.dim(.q)); + const heads: usize = @intCast(q.shape.dim(.h)); + const dim: usize = @intCast(q.shape.dim(.hd)); + const group: usize = @intCast(@divExact(q.shape.dim(.h), k.shape.dim(.h))); + const scores = try std.testing.allocator.alloc(f64, @intCast(k.shape.dim(.k))); + defer std.testing.allocator.free(scores); + const scale = 1.0 / @sqrt(@as(f64, @floatFromInt(dim))); + for (0..batch) |b| { + const start: usize = positions[if (positions.len == 1) 0 else b]; + for (0..queries) |i| { + for (0..heads) |h| { + var maximum: f64 = -std.math.inf(f64); + const active = scores[0 .. start + i + 1]; + for (active, 0..) |*score, j| { + var dot: f64 = 0; + for (0..dim) |d| dot += @as(f64, q.items(f32)[ql.offset(b, i, h, d)]) * @as(f64, k.items(f32)[kl.offset(b, j, h / group, d)]); + score.* = dot * scale; + maximum = @max(maximum, score.*); + } + var sum: f64 = 0; + for (active) |*score| { + score.* = @exp(score.* - maximum); + sum += score.*; + } + for (0..dim) |d| { + var expected: f64 = 0; + for (active, 0..) |weight, j| expected += weight * @as(f64, v.items(f32)[vl.offset(b, j, h / group, d)]); + expected /= sum; + const actual = output.items(f32)[ql.offset(b, i, h, d)]; + try std.testing.expectApproxEqAbs(expected, @as(f64, actual), 2e-5 + 2e-5 * @abs(expected)); + } + } + } + } +} diff --git a/zml/kernels/moe/api.zig b/zml/kernels/moe/api.zig new file mode 100644 index 00000000..b66593c1 --- /dev/null +++ b/zml/kernels/moe/api.zig @@ -0,0 +1,164 @@ +const std = @import("std"); + +const stdx = @import("stdx"); + +const Tensor = @import("../../tensor.zig").Tensor; + +const log = std.log.scoped(.@"zml/moe"); + +pub const Options = struct { + activation: Activation, + /// Quantize activations for Triton FP8 GEMMs; false keeps BF16 activations. + quantize_input: bool, + /// Where routing weights are applied; FlashInfer, Mosaic, Metal and TCL require after_down. + routing_weight_placement: RoutingWeightPlacement, +}; + +pub const ActivationKind = enum { + /// Gelu activation function + gelu, + /// ReLU activation function + relu, + /// SiLU activation function + silu, + /// SwiGLU activation function (including clamped/clipped/scaled/biased variants) + swiglu, + /// SwiGLU activation function with step function + swiglu_step, + /// GeGlu activation function + geglu, + /// GeGlu activation function with tanh + geglu_tanh, +}; + +pub const Activation = union(ActivationKind) { + gelu: void, + relu: void, + silu: void, + swiglu: struct { + limit: ?f32 = null, + scale: ?f32 = null, + bias: ?f32 = null, + }, + swiglu_step: struct { + limit: ?f32 = null, + }, + geglu: void, + geglu_tanh: void, + + pub fn applyActivation(self: Activation, input: Tensor, layout: ProjectionLayout) Tensor { + const x = input.convert(.f32); + return switch (self) { + .gelu => x.gelu(), + .relu => x.relu(), + .silu => x.silu(), + .swiglu, .swiglu_step, .geglu, .geglu_tanh => b: { + const mid = @divFloor(x.dim(.out), 2); + var gate, var up = switch (layout) { + .concatenated => .{ x.slice(.out, .{ .end = mid }), x.slice(.out, .{ .start = mid }) }, + .row_interleaved_1 => .{ x.slice(.out, .{ .start = 0, .step = 2 }), x.slice(.out, .{ .start = 1, .step = 2 }) }, + .row_interleaved_8 => blk: { + const grouped = x.splitAxis(.out, .{ .group = @divExact(mid, 8), .pair = 2, .row = 8 }); + break :blk .{ + grouped.slice(.pair, .{ .end = 1 }).merge(.{ .out = .{ .group, .pair, .row } }), + grouped.slice(.pair, .{ .start = 1 }).merge(.{ .out = .{ .group, .pair, .row } }), + }; + }, + }; + + break :b switch (self) { + .swiglu => |parameters| { + stdx.debug.assert(parameters.scale == null, "triton and fly moe backend don't support swiglu scale", .{}); + + const limit: ?Tensor = if (parameters.limit) |limit| .scalar(limit, x.dtype()) else null; + + // Apply limit on gate and clamp up + gate = if (limit) |l| gate.minimum(l) else gate; + up = if (limit) |l| up.clamp(l.negate(), l) else up; + + // Apply bias + up = if (parameters.bias) |bias| up.addConstant(bias) else up; + + break :b gate.silu().mul(up); + }, + .swiglu_step => |parameters| { + gate = gate.silu(); + + const limit: ?Tensor = if (parameters.limit) |limit| .scalar(limit, x.dtype()) else null; + gate = if (limit) |l| gate.minimum(l) else gate; + up = if (limit) |l| up.clamp(l.negate(), l) else up; + break :b gate.mul(up); + }, + .geglu_tanh => gate.gelu().mul(up), + .geglu => { + log.warn("The geglu activation function was requested but we only support the tanh approximation", .{}); + break :b gate.gelu().mul(up); + }, + else => unreachable, // already treated by the top-level switch + }; + }, + }; + } +}; + +pub const RoutingWeightPlacement = enum { before_down, after_down }; + +pub const ProjectionLayout = enum { + concatenated, // G0 G1 … Gn U0 U1 … Un + row_interleaved_1, // G0 U0 G1 U1 … + row_interleaved_8, // G0…G7 U0…U7 G8…G15 U8…U15 … +}; + +/// How a backend expects the expert weights to be stored +/// The packer writes this layout and forwardMoe reads it +pub const ExpertsLayout = struct { + /// Gate/up ordering before applying the backend-specific packing. + gate_up: ProjectionLayout, + /// Backend specific transformation of the weights and scales. + packing: Packing, + + pub const Packing = enum { + /// Experts stacked as stored in the checkpoint. + plain, + /// Block scales swizzled to the 128x4 tensor-core layout (rows tiled by 4x32, columns by 4). + swizzled_scales, + /// Weights, block scales and global scales in the FlashInfer CUTLASS NVFP4 layout. + flashinfer_nvfp4, + /// MXFP4 gate/up rows interleaved by 8. E8M0 scales use 512-byte + /// 128x4 atoms in MN-major order `[expert, k/128, rows/128, 512]`, + /// represented as `[expert, k/128, rows]` i32 words of four scales. + mxfp4_interleaved8, + }; +}; + +/// Routing for a local token batch and a contiguous range of experts. +/// IDs retain the model's global expert numbering, while the projection weights +/// contain only the local experts. Kernels compute those experts contributions and +/// `forwardMoe` sums them along the expert axis. +pub const LocalRouting = struct { + /// Global expert IDs for local tokens, with shape {.t, .topk} + ids: Tensor, + weights: Tensor, + global_num_experts: u32, + local_num_experts: u32, + /// Scalar i32 containing the first local expert's global ID. + /// Null when all experts are local and global IDs already index the weights. + expert_start: ?Tensor = null, + + /// Maps routing IDs to local weight indices, using -1 for nonlocal experts. + /// For a shard owning experts 4–7, routing IDs [1, 6] become [-1, 2]. + pub fn localIds(self: LocalRouting) Tensor { + const start = self.expert_start orelse return self.ids; + const local = self.ids.cmp(.GE, start).logical(.AND, self.ids.cmp(.LT, start.addConstant(self.local_num_experts))); + return local.select(self.ids.sub(start), .scalar(-1, .i32)); + } + + /// Returns a lookup table from every global expert ID to its local weight + /// index or -1 if nonlocal. Returns null when every expert is local. + pub fn expertMap(self: LocalRouting) ?Tensor { + const start = self.expert_start orelse return null; + const ids = Tensor.arange(.{ .end = self.global_num_experts }, .i32).withTags(.{.expert}); + const local = ids.cmp(.GE, start).logical(.AND, ids.cmp(.LT, start.addConstant(self.local_num_experts))); + return local.select(ids.sub(start), .scalar(-1, .i32)); + } +}; diff --git a/zml/kernels/moe/cute/BUILD.bazel b/zml/kernels/moe/cute/BUILD.bazel new file mode 100644 index 00000000..ca87f37a --- /dev/null +++ b/zml/kernels/moe/cute/BUILD.bazel @@ -0,0 +1,9 @@ +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "cute", + srcs = [ + "cute.zig", + "//zml/kernels/moe/cute/sm10x", + ], +) diff --git a/zml/kernels/moe/cute/cute.zig b/zml/kernels/moe/cute/cute.zig new file mode 100644 index 00000000..8bc03c9c --- /dev/null +++ b/zml/kernels/moe/cute/cute.zig @@ -0,0 +1,139 @@ +//! Blackwell MXFP4 and NVFP4 MoE backend with persistent CuTe GEMMs. + +const stdx = @import("stdx"); + +const zml = @import("../../../zml.zig"); +const Options = @import("../api.zig").Options; +const RoutingWeightPlacement = @import("../api.zig").RoutingWeightPlacement; +const LocalRouting = @import("../api.zig").LocalRouting; +pub const kernels = @import("sm10x/moe.zig"); + +// `moe.zig`'s `refAllDecls` reaches this file but not its imports, so the +// kernel emit tests are only collected if `kernels` is referenced here. +test { + _ = kernels; +} + +pub fn isAvailable(platform: *const zml.Platform) bool { + if (platform.target != .cuda) return false; + const cc = zml.platform.cuda.computeCapability(platform) orelse return false; + return cc.major == 10; +} + +/// The NVFP4 kernels are only validated on SM103. +pub fn isNvfp4Supported(platform: *const zml.Platform) bool { + if (platform.target != .cuda) return false; + const cc = zml.platform.cuda.computeCapability(platform) orelse return false; + return cc.major == 10 and cc.minor == 3; +} + +pub fn validateOptions(opts: Options) void { + stdx.debug.assert(opts.activation == .swiglu, "cute backend only accepts swiglu activation, got {}", .{opts.activation}); + stdx.debug.assert(opts.activation.swiglu.bias == null, "cute backend requires swiglu bias to be null", .{}); + stdx.debug.assert(opts.activation.swiglu.scale == null, "cute backend requires swiglu scale to be null", .{}); +} + +/// MXFP4 weights are `u8` (or `i8`) packed FP4 with E8M0 block scales. NVFP4 +/// weights are packed `u8` with E4M3 block scales; their weight global scale +/// is the per-expert multiplier dequantizing each GEMM output (it folds in +/// the inverse of the input's global scale), and their input scale is the +/// multiplier applied to the GEMM's input before NVFP4 quantization. Both +/// expect block scales in the 128x4 tensor-core layout. MXFP4 gate/up rows +/// interleave by 8 and its scale atoms are MN-major packed i32 words. +pub fn fusedExperts( + input: zml.Tensor, + routing: LocalRouting, + gate_up: zml.nn.Linear, + down: zml.nn.Linear, + options: Options, +) zml.Tensor { + validateOptions(options); + stdx.debug.assert(input.dtype() == .bf16, "cute backend only supports bf16 inputs, got {}", .{input.dtype()}); + stdx.debug.assert(gate_up.bias == null and down.bias == null, "cute backend expects gate_up bias and down bias to be null", .{}); + const gq = gate_up.quantization orelse @panic("cute backend requires gate_up quantization to be set"); + const dq = down.quantization orelse @panic("cute backend requires down quantization to be set"); + stdx.debug.assert(gq.scheme == dq.scheme, "cute expects gate_up and down quantization scheme to match, got {} and {}", .{ gq.scheme, dq.scheme }); + + const format: Format = switch (gq.scheme) { + .mxfp4 => blk: { + // Weight storage for mxfp4 in HF is expressed as u8 or i8 + stdx.debug.assert(gate_up.weight.dtype() == .u8 or gate_up.weight.dtype() == .i8, "cute expects gate_up weight dtype to be u8 or i8, got {}", .{gate_up.weight.dtype()}); + stdx.debug.assert(down.weight.dtype() == .u8 or down.weight.dtype() == .i8, "cute expects down weight dtype to be u8 or i8, got {}", .{down.weight.dtype()}); + // Check the boundary quantizer and CuTe GEMM tile constraints. + kernels.validateMxfp4Shapes(down.weight.dim(1), down.weight.dim(2) * 2); + // The SwiGLU clamp is a kernel parameter, but the routing weight is + // multiplied in the up epilogue, before the down projection: applying it + // after would have to move into the down epilogue or the reduction. + stdx.debug.assert(options.routing_weight_placement == .before_down, "cute MXFP4 backend only supports routing_weight_placement = .before_down, got {}", .{options.routing_weight_placement}); + break :blk .mxfp4; + }, + .nvfp4 => blk: { + stdx.debug.assert(!options.quantize_input, "cute NVFP4 backend always quantizes its inputs to NVFP4", .{}); + stdx.debug.assert(gq.swizzled_scales and dq.swizzled_scales, "cute NVFP4 backend expects swizzled block scales", .{}); + stdx.debug.assert(gate_up.weight.dtype() == .u8 and down.weight.dtype() == .u8, "cute NVFP4 backend expects packed u8 weights, got {} and {}", .{ gate_up.weight.dtype(), down.weight.dtype() }); + const scale = struct { + fn f(global: ?zml.Quantization.GlobalScale) zml.Tensor { + return (global orelse @panic("cute NVFP4 backend requires global and input scales")).asMultiplier(); + } + }.f; + break :blk .{ .nvfp4 = .{ + .alpha1 = scale(gq.global_scale), + .alpha2 = scale(dq.global_scale), + .act1 = scale(gq.input_scale), + .act2 = scale(dq.input_scale), + } }; + }, + else => stdx.debug.panic("cute backend only supports mxfp4 and nvfp4 quantization, got {}", .{gq.scheme}), + }; + + const hidden = down.weight.dim(1); + const intermediate = down.weight.dim(2) * 2; + const topk = routing.ids.dim(.topk); + const experts = gate_up.weight.dim(.expert); + const ids = routing.localIds().convert(.i32).reshape(.{ .token = .auto, .topk = topk }); + const tokens = ids.dim(.token); + const routing_weights = routing.weights.convert(.f32).reshape(.{ tokens, topk }); + + return switch (format) { + .mxfp4 => kernels.forwardMxfp4(tokens, hidden, intermediate, experts, topk, options.activation.swiglu.limit, .{ + .x = input.reshape(.{ tokens, hidden }), + .routing_weights = routing_weights, + .w1 = gate_up.weight, + .s1 = gq.scales, + .w2 = down.weight, + .s2 = dq.scales, + .ids = ids, + }, null), + .nvfp4 => |f| kernels.forwardNvfp4(tokens, hidden, intermediate, experts, topk, .{ + .x = input.reshape(.{ tokens, hidden }), + .routing_weights = routing_weights, + .ids = ids, + .w1 = gate_up.weight, + .s1 = gq.scales, + .w2 = down.weight, + .s2 = dq.scales, + // `alpha` is per expert; `act` is one scale shared by every + // expert, which the kernels read per expert. + .alpha1 = f.alpha1.reshape(.{experts}), + .alpha2 = f.alpha2.reshape(.{experts}), + .act1 = f.act1.reshape(.{}).broad(.init(.{experts}, .f32)), + .act2 = f.act2.reshape(.{}).broad(.init(.{experts}, .f32)), + }, .{ + .swiglu_limit = options.activation.swiglu.limit, + .routing_weight_placement = options.routing_weight_placement, + }), + }.reshape(input.shape().dims()).withTags(input.shape()); +} + +/// The format-specific tensors: NVFP4's per-expert global scales. They stay +/// as given until the sharded body, so they split across ranks like the +/// weights. +const Format = union(enum) { + mxfp4, + nvfp4: struct { + alpha1: zml.Tensor, + alpha2: zml.Tensor, + act1: zml.Tensor, + act2: zml.Tensor, + }, +}; diff --git a/zml/kernels/moe/cute/sm10x/BUILD.bazel b/zml/kernels/moe/cute/sm10x/BUILD.bazel new file mode 100644 index 00000000..2b23bfbb --- /dev/null +++ b/zml/kernels/moe/cute/sm10x/BUILD.bazel @@ -0,0 +1,14 @@ +package(default_visibility = ["//zml/kernels/moe/cute:__pkg__"]) + +filegroup( + name = "sm10x", + srcs = [ + "common.zig", + "moe.zig", + "mxfp4.zig", + "nvfp4.zig", + "persistent_mxfp4.zig", + "persistent_nvfp4.zig", + "reduce.zig", + ], +) diff --git a/zml/kernels/moe/cute/sm10x/common.zig b/zml/kernels/moe/cute/sm10x/common.zig new file mode 100644 index 00000000..da639c75 --- /dev/null +++ b/zml/kernels/moe/cute/sm10x/common.zig @@ -0,0 +1,227 @@ +//! Machinery shared by the MXFP4 and NVFP4 CuTe MoE kernels: pipeline +//! barriers and state, the static persistent tile scheduler, direct routing, +//! shared-memory pointers, boundary kernel launches and small IR helpers. +const std = @import("std"); +const mlir = @import("mlir"); + +const zml = @import("../../../../zml.zig"); +const cute = zml.kernel.cute; +const B = cute.Builder; +const V = cute.Value; + +pub fn typed(type_: *const mlir.Type) struct { mlir_type: cute.ArgSpec.MlirTypeSpec } { + return .{ .mlir_type = .{ .type_ = type_ } }; +} + +pub fn gridTyped(type_: *const mlir.Type) struct { mlir_type: cute.ArgSpec.MlirTypeSpec } { + return .{ .mlir_type = .{ .type_ = type_, .grid_constant = true } }; +} + +pub fn barrierAt(b: *B, base: V, stage: V) V { + return b.addOffset( + base, + b.makeIntTuple(.{stage}), + b.ptrTy(.i64, .smem, 8) catch @panic("bad pipeline barrier pointer"), + ).value(); +} + +pub const PipelinePosition = struct { + index: V, + phase: V, +}; + +/// Scalar form of CUTLASS's StaticPersistentTileScheduler state for the +/// one-CTA DeepSeek specialization. Python carries the decoded M/N/L +/// coordinate, validity predicate, linear work index, and executed-tile +/// count through every role's `scf.while`. Keeping the same state shape is +/// important to the CuTe software-pipeline passes, even though N is always +/// zero for one tile per group. +pub const PersistentWork = struct { + m: V, + n: V, + group: V, + valid: V, + linear: V, + executed: V, +}; + +/// Direct routing (one token) runs one single-row group per route. Routes +/// owned by other expert-parallel ranks carry expert -1: the persistent tiles +/// cover the valid routes only, in route order, so they do not occupy a wave. +pub const DirectRoutes = struct { + expert_ids: cute.View, + routes: usize, + + pub fn count(self: DirectRoutes, b: *B) V { + var valid = b.constant(.i32, 0); + for (0..self.routes) |r| valid = valid.add(self.expertAt(b, r).ge(0).to(.i32)); + return valid; + } + + /// Route of the `k`-th valid route. + fn nth(self: DirectRoutes, b: *B, k: V) V { + var seen = b.constant(.i32, 0); + var route = b.constant(.i32, 0); + for (0..self.routes) |r| { + const valid = self.expertAt(b, r).ge(0); + route = b.select(valid.bitAnd(seen.eq(k)), b.constant(.i32, r), route); + seen = seen.add(valid.to(.i32)); + } + return route; + } + + fn expertAt(self: DirectRoutes, b: *B, r: usize) V { + return self.expert_ids.get(.{b.constant(.i32, r)}, .i32); + } +}; + +pub fn persistentWorkAt( + b: *B, + linear: V, + executed: V, + active: V, + tiles_m: V, + div_m: V, + div_n: V, + direct: ?DirectRoutes, +) PersistentWork { + // StaticPersistentTileScheduler uses two first-class FastDivmod objects + // to decode the column-major (M, N=1, L) problem layout. Preserve those + // operations even though ordinary rem/div is mathematically equivalent; + // CuTe's scheduler reconstruction and loop canonicalization key off these + // values and carry them through every persistent loop. + const group_and_m = b.fastDivmod(linear, div_m); + const group_and_n = b.fastDivmod(group_and_m[0], div_n); + const group = if (direct) |routes| routes.nth(b, group_and_n[0]) else group_and_n[0]; + return .{ + .m = b.makeWarpUniform(group_and_m[1]), + .n = b.makeWarpUniform(group_and_n[1]), + .group = b.makeWarpUniform(group), + .valid = b.makeWarpUniform(linear.lt(active.mul(tiles_m))), + .linear = linear, + .executed = executed, + }; +} + +pub fn advancePersistentWork( + b: *B, + linear: V, + executed: V, + persistent_clusters: V, + active: V, + tiles_m: V, + div_m: V, + div_n: V, + direct: ?DirectRoutes, +) PersistentWork { + return persistentWorkAt(b, linear.add(persistent_clusters), executed.add(1), active, tiles_m, div_m, div_n, direct); +} + +/// Equivalent to CUTLASS PipelineState.advance(). Keep the index and phase +/// as explicit loop-carried SSA values instead of recovering them with +/// division/remainder; the generated CuTe program then has the same circular +/// state transition as the Python DSL. +pub fn advancePipeline(b: *B, index: V, phase: V, comptime stages: usize) PipelinePosition { + const advanced = index.add(1); + const wraps = advanced.eq(stages); + const next_index = b.select(wraps, 0, advanced); + var next_phase = b.openIfElse(wraps, .{phase}); + { + next_phase.yieldThen(.{phase.bitXor(1)}); + } + { + next_phase.yieldElse(.{phase}); + } + return .{ .index = next_index, .phase = next_phase.results[0] }; +} + +pub fn peekBarrierIfRemaining(b: *B, base: V, count: V, limit: usize, state: PipelinePosition) V { + var peek = b.openIfElse(count.lt(limit), .{.i1}); + { + peek.yieldThen(.{b.mbarrierTryWait(barrierAt(b, base, state.index), state.phase, .{ .scope = .cta })}); + } + { + peek.yieldElse(.{b.constant(.i1, true)}); + } + return peek.results[0]; +} + +/// `cute.add_offset` infers the alignment a pointer keeps as the gcd of the +/// base alignment and the offset, and `cute.recast_iter` may not change it, so +/// the field's own alignment has to be what the offset leaves. +pub fn sharedPointerAt( + b: *B, + base: V, + comptime byte_offset: i64, + comptime dtype: cute.DType, + comptime alignment: u64, +) V { + const moved = comptime movedAlignment(byte_offset); + comptime std.debug.assert(moved >= alignment); + const byte_ptr = b.addOffset( + base, + b.makeIntTuple(.{byte_offset}), + b.ptrTy(.i8, .smem, moved) catch @panic("bad shared byte pointer"), + ); + return b.recastPointer(byte_ptr, dtype, .smem, moved); +} + +/// Alignment of `shared_base + byte_offset`, which the shared storage is +/// allocated with. +fn movedAlignment(comptime byte_offset: i64) u64 { + return offsetAlignment(1024, byte_offset, 1); +} + +/// `cute.add_offset` keeps the gcd of the operand alignment and the offset, +/// and the op's result type has to say so. +pub fn offsetAlignment(base_bytes: u64, offset_elements: i64, element_bytes: u64) u64 { + if (offset_elements == 0) return base_bytes; + const moved = @as(u64, @intCast(@abs(offset_elements))) * element_bytes; + return @max(std.math.gcd(base_bytes, moved), element_bytes); +} + +pub fn Tuple(comptime T: type, comptime len: usize) type { + return @Tuple(&@as([len]type, @splat(T))); +} + +pub fn arity(comptime T: type) usize { + return @typeInfo(T).@"struct".field_names.len; +} + +/// The MLIR types of a tuple of values, as `openWhile` expects them. +pub fn typesOf(values: anytype) Tuple(*const mlir.Type, arity(@TypeOf(values))) { + const len = comptime arity(@TypeOf(values)); + var out: Tuple(*const mlir.Type, len) = undefined; + inline for (0..len) |i| out[i] = values[i].type_(); + return out; +} + +/// Forward a whole loop-carried array, as `yieldBefore` expects a tuple. +pub fn tupleOf(array: anytype) Tuple(@TypeOf(array[0]), array.len) { + var out: Tuple(@TypeOf(array[0]), array.len) = undefined; + inline for (0..array.len) |i| out[i] = array[i]; + return out; +} + +pub fn fmt(b: *B, comptime format: []const u8, args: anytype) []const u8 { + return std.fmt.allocPrint(b.arena.allocator(), format, args) catch @panic("OOM"); +} + +pub fn view(b: *B, value: V) cute.View { + return .{ .inner = value.inner, .kernel = b }; +} + +/// Host launch of a boundary kernel: forward the buffer pointers, with PDL. +pub fn launchBoundary(b: *B, kernel_name: [:0]const u8, grid: [3]i32, args: anytype) cute.FinishError!void { + const one = b.constant(.i32, 1); + const config = b.makeLaunchConfig(.{ + .grid = .{ b.constant(.i32, grid[0]), b.constant(.i32, grid[1]), b.constant(.i32, grid[2]) }, + .block = .{ b.constant(.i32, 128), one, one }, + .dynamic_smem = b.kernelSmemSize(kernel_name), + .stream = b.cudaStream(), + .cluster = .{ one, one, one }, + .use_pdl = true, + }); + const launch = b.launchEx(kernel_name, config, args); + b.returnHostStatus(b.cudaResultStatus(launch)); +} diff --git a/zml/kernels/moe/cute/sm10x/moe.zig b/zml/kernels/moe/cute/sm10x/moe.zig new file mode 100644 index 00000000..ff6edec2 --- /dev/null +++ b/zml/kernels/moe/cute/sm10x/moe.zig @@ -0,0 +1,348 @@ +//! SM100 CuTe MoE, MXFP8 x MXFP4 (the Zig port of the CuTe DSL `NativeMoE`) +//! and NVFP4 x NVFP4: routing, routed input quantization, persistent up GEMM +//! with a fused SwiGLU + quantization epilogue, persistent down GEMM storing +//! rows in route order, and the top-k reduction. MXFP4 applies routing weights +//! before the down projection, with the reference model's BF16 rounding. +const std = @import("std"); + +const stdx = @import("stdx"); + +const zml = @import("../../../../zml.zig"); +const RoutingWeightPlacement = @import("../../api.zig").RoutingWeightPlacement; +const triton_kernels = @import("../../triton/moe.zig"); +pub const mxfp4 = @import("mxfp4.zig"); +pub const mxfp4_persistent = @import("persistent_mxfp4.zig"); +pub const nvfp4 = @import("nvfp4.zig"); +pub const nvfp4_persistent = @import("persistent_nvfp4.zig"); +const reduce = @import("reduce.zig"); + +pub const Mxfp4Inputs = struct { + /// `[tokens, hidden]` BF16. + x: zml.Tensor, + /// `[tokens, topk]` FP32. + routing_weights: zml.Tensor, + /// `[experts, 2 * intermediate, hidden / 2]` packed FP4, gate/up rows interleaved by 8. + w1: zml.Tensor, + /// `[experts, hidden / 128, 2 * intermediate]` i32 words of four E8M0 scales, + /// in MN-major 128x4 atoms, with rows interleaved like `w1`. + s1: zml.Tensor, + /// `[experts, hidden, intermediate / 2]` packed FP4. + w2: zml.Tensor, + /// `[experts, intermediate / 128, hidden]` i32 words in MN-major 128x4 atoms. + s2: zml.Tensor, + /// `[tokens, topk]` local expert ids. + ids: zml.Tensor, +}; + +/// The boundary quantizer handles four 128-column tiles per CTA. The GEMMs +/// support 128-column tiles and a fixed set of reduction tile counts. +pub fn validateMxfp4Shapes(hidden: i64, intermediate: i64) void { + stdx.debug.assert(hidden > 0 and @rem(hidden, 4 * 128) == 0, "cute backend requires hidden dimension to be a positive multiple of 512, got {}", .{hidden}); + stdx.debug.assert(intermediate > 0 and @rem(intermediate, 128) == 0, "cute backend requires intermediate dimension to be a positive multiple of 128, got {}", .{intermediate}); + stdx.debug.assert(supportedReductionTiles(@divTrunc(hidden, 128)), "cute backend does not support hidden reduction tile count {}, got hidden dimension {}", .{ @divTrunc(hidden, 128), hidden }); + stdx.debug.assert(supportedReductionTiles(@divTrunc(intermediate, 128)), "cute backend does not support intermediate reduction tile count {}, got intermediate dimension {}", .{ @divTrunc(intermediate, 128), intermediate }); +} + +fn supportedReductionTiles(tiles: i64) bool { + return switch (tiles) { + 16, 18, 32, 40 => true, + else => false, + }; +} + +/// Route each token/top-k pair to its own one-row group instead of grouping +/// routes by expert. A token's top-k experts are distinct, so for a single +/// token grouping cannot share any weight tile and only adds the routing +/// kernels. Python `grouped=False`. +fn isDirect(tokens: i64) bool { + return tokens == 1; +} + +/// GEMM tile N, which is also the routed expert-group size. Wider tiles +/// reuse each weight tile across more routes but pad more rows per expert, +/// so they only pay off once experts receive tens of routes. +fn tileN(tokens: i64, experts: i64, topk: i64) i64 { + // Direct routing keeps one row per group: the narrowest MMA wastes the + // least of each B tile and leaves room for a 12-stage A/B ring. + if (isDirect(tokens)) return 8; + if (tokens < 512) return 16; + // Prefill: each group re-reads the whole expert weight matrix, so widen + // the tile once experts hold enough routes to fill it. Half the routes + // are remote under expert parallelism, hence the conservative estimate. + // The thresholds are measured on the DeepSeek V4.1 expert geometry. + const routes_per_expert = @divTrunc(tokens * topk, 2 * experts); + if (routes_per_expert >= 128) return 128; + if (routes_per_expert >= 32) return 64; + return 32; +} + +/// One persistent CTA per SM. Each CTA needs the full 227 KiB of shared +/// memory, so launching more CTAs than SMs (e.g. 160 on a 152-SM GB300) +/// runs the surplus as a second wave and nearly doubles the GEMM time. +fn persistentCtas() i64 { + const platform = zml.Compiler.current().platform; + if (platform.devices.len == 0) return 148; + const attribute = platform.devices[0].pjrt_desc.attribute(platform.pjrt_api, "core_count") orelse return 148; + return if (attribute.int64 > 0) attribute.int64 else 148; +} + +// TODO(Corentin): settle on groups/capacity. + +/// Upper bound on the number of `group`-row expert groups for all routes. +fn computeCapacity(tokens: i64, topk: i64, experts: i64, group: i64) i64 { + return @divTrunc(tokens * topk + group - 1, group) + experts - 1; +} + +/// Where each route lives in the grouped GEMM operands. +const Routing = struct { + /// Expert groups, i.e. the GEMMs' L extent. + groups: i64, + /// Route -> physical routed row. Null for direct routing, where route + /// `r` is row `r`. + route_map: ?zml.Tensor, + /// Physical routed row -> route. Direct routing does not read it. + route_inverse: zml.Tensor, + /// `[group experts | group sizes | active group count]` for grouped + /// routing, `[route experts]` for direct routing. + schedule: zml.Tensor, +}; + +const atomic_route_threshold = 6 * 2048; + +fn route(tokens: i64, experts: i64, topk: i64, n: i64, direct: bool, ids: zml.Tensor) Routing { + const routes = tokens * topk; + const flat_ids = ids.convert(.i32).reshape(.{routes}); + if (direct) return .{ .groups = routes, .route_map = null, .route_inverse = flat_ids, .schedule = flat_ids }; + const capacity = computeCapacity(tokens, topk, experts, n); + + std.debug.assert(capacity > 0); + // `route` rescans every route once per expert: cheapest for decode, but + // quadratic. Past a few thousand tokens, count with atomics instead. + const r: struct { counts: zml.Tensor, pos: zml.Tensor } = if (routes >= atomic_route_threshold) blk: { + const counted = triton_kernels.RouteAtomic.Kernel.call( + .{ .ids = ids, .zeros = zml.Tensor.zeroes(.init(.{experts}, .i32)) }, + .{ .counts = .init(.{experts}, .i32), .pos = .init(.{routes}, .i32) }, + .{ + .cfg = .{ + .tokens = tokens, + .topk = topk, + .expert_offset = 0, + .experts = experts, + }, + .grid = .{ @intCast(@divTrunc(routes + triton_kernels.RouteAtomic.route_block - 1, triton_kernels.RouteAtomic.route_block)), 1, 1 }, + .num_warps = 4, + .num_stages = 1, + .output_operand_aliases = .{ .counts = .zeros }, + }, + ); + break :blk .{ .counts = counted.counts, .pos = counted.pos }; + } else blk: { + const scanned = triton_kernels.Route.Kernel.call( + .{ .ids = ids }, + .{ .counts = .init(.{experts}, .i32), .pos = .init(.{routes}, .i32) }, + .{ + .cfg = .{ .tokens = tokens, .topk = topk, .expert_offset = 0 }, + .grid = .{ @intCast(experts), 1, 1 }, + .num_warps = 4, + .num_stages = 1, + }, + ); + break :blk .{ .counts = scanned.counts, .pos = scanned.pos }; + }; + + const s = triton_kernels.Schedule.Kernel.call( + .{ .ids = ids, .counts = r.counts, .pos = r.pos }, + .{ + .map = .init(.{capacity * n}, .i32), + .perm = .init(.{tokens * topk}, .i32), + .sched = .init(.{2 * capacity + 1}, .i32), + }, + .{ + .cfg = .{ + .tokens = tokens, + .topk = topk, + .expert_offset = 0, + .experts = experts, + .capacity = capacity, + .group = n, + }, + .grid = .{ @intCast(@divTrunc(tokens * topk + 127, 128)), 1, 1 }, + .num_warps = 4, + .num_stages = 1, + }, + ); + + return .{ .groups = capacity, .route_map = s.perm, .route_inverse = s.map, .schedule = s.sched }; +} + +/// `[tokens, hidden]` BF16 output of the MXFP4 MoE layer. `tile_n` is the +/// GEMM tile N, for tuning; null selects it from the batch size. +pub fn forwardMxfp4(tokens: i64, hidden: i64, intermediate: i64, experts: i64, topk: i64, swiglu_limit: ?f32, a: Mxfp4Inputs, tile_n: ?i64) zml.Tensor { + const n = tile_n orelse tileN(tokens, experts, topk); + const direct = isDirect(tokens); + const routing = route(tokens, experts, topk, n, direct, a.ids); + const routes = tokens * topk; + + const up_cfg: mxfp4_persistent.Config = .{ + .experts = experts, + .m = 2 * intermediate, + .n = n, + .k = hidden, + .groups = routing.groups, + .persistent_ctas = persistentCtas(), + .epilogue = .swiglu_mxfp8, + .routes = routes, + .direct = direct, + .swiglu_limit = swiglu_limit, + }; + var down_cfg = up_cfg; + down_cfg.m = hidden; + down_cfg.k = intermediate; + down_cfg.epilogue = .route_rows; + + const q1 = mxfp4.groupedQuantize(a.x, routing.route_map orelse routing.schedule, .{ + .tokens = tokens, + .hidden = hidden, + .topk = topk, + .group_size = up_cfg.groupRows(), + .capacity = routing.groups, + .direct = direct, + }); + const q2 = mxfp4_persistent.upQuantized(up_cfg, .{ + .weight = a.w1, + .schedule = routing.schedule, + .input_quant = q1.q, + .weight_scale = a.s1, + .input_scale = q1.s, + }, routing.route_inverse, a.routing_weights.reshape(.{routes})); + const down = mxfp4_persistent.downRows(down_cfg, .{ + .weight = a.w2, + .schedule = routing.schedule, + .input_quant = q2.q, + .weight_scale = a.s2, + .input_scale = q2.s, + }, routing.route_inverse); + return reduce.topk(down, routing.route_map orelse routing.schedule, .{ + .tokens = tokens, + .hidden = hidden, + .topk = topk, + .columns_per_cta = reduce.columnsPerCta(tokens, hidden), + }); +} + +pub const Nvfp4Inputs = struct { + /// `[tokens, hidden]` BF16. + x: zml.Tensor, + /// `[tokens, topk]` FP32. + routing_weights: zml.Tensor, + /// `[tokens, topk]` local expert ids, -1 for routes of other ranks. + ids: zml.Tensor, + /// `[experts, 2 * intermediate, hidden / 2]` packed FP4, gate/up rows interleaved. + w1: zml.Tensor, + /// `w1` E4M3 scales `[experts, 2 * intermediate, hidden / 16]` in the 128x4 layout. + s1: zml.Tensor, + /// `[experts, hidden, intermediate / 2]` packed FP4. + w2: zml.Tensor, + /// `w2` E4M3 scales `[experts, hidden, intermediate / 16]` in the 128x4 layout. + s2: zml.Tensor, + /// `[experts]` FP32 dequantization scale of each GEMM output, which folds + /// in the inverse of the GEMM input's global scale. + alpha1: zml.Tensor, + alpha2: zml.Tensor, + /// `[experts]` FP32 global scale each GEMM's input is multiplied by + /// before NVFP4 quantization. + act1: zml.Tensor, + act2: zml.Tensor, +}; + +pub const Nvfp4Options = struct { + swiglu_limit: ?f32 = null, + routing_weight_placement: RoutingWeightPlacement = .after_down, + /// GEMM tile N; null selects it from the batch size. + tile_n: ?i64 = null, +}; + +pub fn validateNvfp4Shapes(hidden: i64, intermediate: i64) void { + const k_tile = nvfp4_persistent.k_tile; + stdx.debug.assert(@rem(hidden, k_tile) == 0, "cute NVFP4 backend expects hidden to be a multiple of {}, got {}", .{ k_tile, hidden }); + stdx.debug.assert(@rem(intermediate, k_tile) == 0, "cute NVFP4 backend expects intermediate to be a multiple of {}, got {}", .{ k_tile, intermediate }); +} + +/// NVFP4 GEMM tile N: the expected routes of one expert, so that each +/// expert's weight tiles are read once per group instead of once per few rows. +fn nvfp4TileN(tokens: i64, experts: i64, topk: i64) i64 { + if (isDirect(tokens)) return 8; + const per_expert = std.math.divCeil(i64, tokens * topk, experts) catch unreachable; + return @intCast(std.math.clamp(std.math.ceilPowerOfTwoAssert(u64, @intCast(per_expert)), 16, 128)); +} + +/// `[tokens, hidden]` BF16 output of the NVFP4 MoE layer. +pub fn forwardNvfp4(tokens: i64, hidden: i64, intermediate: i64, experts: i64, topk: i64, a: Nvfp4Inputs, opts: Nvfp4Options) zml.Tensor { + validateNvfp4Shapes(hidden, intermediate); + const n = opts.tile_n orelse nvfp4TileN(tokens, experts, topk); + const direct = isDirect(tokens); + const routes = tokens * topk; + const flat_ids = a.ids.convert(.i32).reshape(.{routes}); + const routing = route(tokens, experts, topk, n, direct, a.ids); + const route_map = routing.route_map orelse flat_ids; + const weights = a.routing_weights.convert(.f32).reshape(.{routes}); + + const up_cfg: nvfp4_persistent.Config = .{ + .experts = experts, + .m = 2 * intermediate, + .n = n, + .k = hidden, + .groups = routing.groups, + .persistent_ctas = persistentCtas(), + .epilogue = .swiglu_nvfp4, + .routes = routes, + .direct = direct, + .swiglu_limit = opts.swiglu_limit, + .routing_weights = opts.routing_weight_placement == .before_down, + }; + var down_cfg = up_cfg; + down_cfg.m = hidden; + down_cfg.k = intermediate; + down_cfg.epilogue = .route_rows; + down_cfg.routing_weights = opts.routing_weight_placement == .after_down; + + const q1 = nvfp4.groupedQuantize(a.x, route_map, flat_ids, a.act1, .{ + .tokens = tokens, + .hidden = hidden, + .topk = topk, + .experts = experts, + .group_size = up_cfg.groupRows(), + .capacity = routing.groups, + .direct = direct, + }); + const q2 = nvfp4_persistent.upQuantized(up_cfg, .{ + .weight = a.w1, + .schedule = routing.schedule, + .input_quant = q1.q, + .weight_scale = a.s1.bitCast(.u8), + .input_scale = q1.s, + .alpha = a.alpha1, + }, routing.route_inverse, weights, a.act2); + const down = nvfp4_persistent.downRows(down_cfg, .{ + .weight = a.w2, + .schedule = routing.schedule, + .input_quant = q2.q, + .weight_scale = a.s2.bitCast(.u8), + .input_scale = q2.s, + .alpha = a.alpha2, + }, routing.route_inverse, weights); + return reduce.topk(down, route_map, .{ + .tokens = tokens, + .hidden = hidden, + .topk = topk, + .columns_per_cta = reduce.columnsPerCta(tokens, hidden), + }); +} + +test { + _ = mxfp4; + _ = mxfp4_persistent; + _ = nvfp4; + _ = nvfp4_persistent; + _ = reduce; +} diff --git a/zml/kernels/moe/cute/sm10x/mxfp4.zig b/zml/kernels/moe/cute/sm10x/mxfp4.zig new file mode 100644 index 00000000..29e68977 --- /dev/null +++ b/zml/kernels/moe/cute/sm10x/mxfp4.zig @@ -0,0 +1,139 @@ +//! CuTe input quantization kernel in front of the persistent MXFP4 GEMMs of +//! `persistent_mxfp4.zig`: routed MXFP8 input quantization. +const std = @import("std"); + +const zml = @import("../../../../zml.zig"); +const cute = zml.kernel.cute; +const B = cute.Builder; +const launchBoundary = @import("common.zig").launchBoundary; + +pub const GroupedQuantConfig = struct { + tokens: i64, + hidden: i64, + topk: i64, + /// Rows stored per expert group. + group_size: i64, + capacity: i64, + /// Route `r` goes to row `r * group_size`; `route_map` is not read. + direct: bool = false, +}; + +// A program rather than a bare kernel so its launch carries the +// programmatic-dependent-launch attribute: it then starts while the previous +// kernel drains, and waits on `griddepcontrol` before its first read. A bare +// kernel is launched by XLA without that attribute. +const GroupedQuantize = cute.Program(GroupedQuantConfig, .{ + .name = "mxfp4_cute_grouped_quantize", + .inputs = &.{ "x", "route_map" }, + .outputs = &.{ "q", "s" }, + .run = groupedQuantizeProgram, +}); + +const quantize_kernel_name = "mxfp4_cute_grouped_quantize_kernel"; + +/// Quantize each token once, then scatter it to the physical rows assigned by +/// the routing. Scale bytes use the 128x4 layout consumed by the SM100 +/// block-scaled MMA scale-factor tensor map. +pub fn groupedQuantize(x: zml.Tensor, route_map: zml.Tensor, cfg: GroupedQuantConfig) struct { q: zml.Tensor, s: zml.Tensor } { + std.debug.assert(cfg.group_size == 8 or cfg.group_size == 16 or cfg.group_size == 32 or cfg.group_size == 64 or cfg.group_size == 128); + const result = GroupedQuantize.call( + .{ .x = x, .route_map = route_map }, + .{ + .q = .init(.{ cfg.capacity * cfg.group_size, cfg.hidden }, .f8e4m3fn), + .s = .init(.{ cfg.capacity, @divExact(cfg.hidden, 32) * 128 }, .u8), + }, + .{ .cfg = cfg }, + ); + return .{ .q = result.q, .s = result.s }; +} + +/// Python `prepare_input_vector`: each thread quantizes four consecutive +/// values and stores them as one packed word, so eight lanes cover one +/// 32-value MX block. +fn groupedQuantizeProgram(b: *B, cfg: GroupedQuantConfig) cute.FinishError!void { + b.beginFunction(quantize_kernel_name, .cuda_kernel); + try groupedQuantizeKernel(b, cfg); + b.endFunction(.{ 128, 1, 1 }); + + b.beginFunction(GroupedQuantize.name, .host); + // Host parameters carry the custom call's element types; the kernel reads + // four FP8 values per 32-bit word. + const host = try b.declareArgs(.{ + .x = .{ .ptr = cute.DType.bf16 }, + .route_map = .{ .ptr = cute.DType.i32 }, + .q = .{ .ptr = cute.DType.f8e4m3fn }, + .s = .{ .ptr = cute.DType.i8 }, + }); + try launchBoundary(b, quantize_kernel_name, .{ @intCast(@divExact(cfg.hidden, 4 * 128)), @intCast(cfg.tokens), 1 }, .{ + host.x, host.route_map, b.recastPointer(host.q, .i32, .gmem, 16), host.s, + }); + b.endFunction(null); +} + +fn groupedQuantizeKernel(b: *B, cfg: GroupedQuantConfig) cute.FinishError!void { + std.debug.assert(@rem(cfg.hidden, 4 * 128) == 0); + const a = try b.declareArgs(.{ + .x = .{ .tensor = .{ .dtype = .bf16, .shape = &.{ cfg.tokens, cfg.hidden } } }, + .route_map = .{ .tensor = .{ .dtype = .i32, .shape = &.{cfg.tokens * cfg.topk} } }, + // Four FP8 values per word. + .q = .{ .tensor = .{ .dtype = .i32, .shape = &.{ cfg.capacity * cfg.group_size, @divExact(cfg.hidden, 4) } } }, + .s = .{ .tensor = .{ .dtype = .i8, .shape = &.{ cfg.capacity, @divExact(cfg.hidden, 32) * 128 } } }, + }); + + const tid = b.threadIdx().x; + // This grid may have started before its producer finished: the routing and + // the previous layer write what it reads below. + b.waitForDependency(); + const token = b.blockIdx().y; + const word = b.blockIdx().x.mul(128).add(tid); + var x: [4]cute.Value = undefined; + for (&x, 0..) |*value, i| value.* = a.x.get(.{ token, word.mul(4).add(@as(i32, @intCast(i))) }).to(.f32); + + var maximum = x[0].abs().maximum(x[1].abs()).maximum(x[2].abs().maximum(x[3].abs())); + inline for (.{ 1, 2, 4 }) |offset| { + maximum = maximum.maximum(b.shuffleXor(maximum, offset)); + } + const raw_bits = maximum.maximum(1e-4).mul(1.0 / 448.0).bitCast(.i32); + const exponent = raw_bits.shrLogical(23).add(raw_bits.bitAnd(0x7fffff).ne(0).to(.i32)); + const inverse_scale = b.constant(.i32, 254).sub(exponent).shl(23).bitCast(.f32); + const packed_word = b.packFp8x4(x[0].mul(inverse_scale), x[1].mul(inverse_scale), x[2].mul(inverse_scale), x[3].mul(inverse_scale)); + + const scale_col = word.div(8); + for (0..@intCast(cfg.topk)) |slot| { + const route = token.mul(cfg.topk).add(slot); + const row = if (cfg.direct) route.mul(cfg.group_size) else a.route_map.get(.{route}); + // Routes of other expert-parallel ranks are not scheduled. + var scheduled = if (cfg.direct) null else b.openIf(row.ge(0)); + a.q.set(.{ row, word }, packed_word); + + var leader = b.openIf(tid.rem(8).eq(0)); + const scale_row = row.div(cfg.group_size); + const row_in_group = row.rem(cfg.group_size); + const swizzled_col = scale_col.div(4).mul(512) + .add(row_in_group.rem(32).mul(16)) + .add(row_in_group.div(32).mul(4)) + .add(scale_col.rem(4)); + a.s.set(.{ scale_row, swizzled_col }, exponent.to(.i8)); + leader.close(); + if (scheduled) |*s| s.close(); + } + // Only now are the rows this CTA owns complete: release the persistent + // GEMM, which waits on this signal before its first TMA read. + b.launchDependents(); +} + +test "MXFP4 CuTe input quantization emits for decode and prefill batches" { + inline for (.{ @as(i64, 1), 6, 96, 16_384 }) |tokens| { + const quant_ir = try GroupedQuantize.emit(std.testing.allocator, .{ + .tokens = tokens, + .hidden = 5120, + .topk = 6, + .group_size = 16, + .capacity = @divTrunc(tokens * 6 + 15, 16) + 191, + }); + defer std.testing.allocator.free(quant_ir); + try std.testing.expect(std.mem.indexOf(u8, quant_ir, "griddepcontrol.launch_dependents;") != null); + try std.testing.expect(std.mem.indexOf(u8, quant_ir, "cuda.launch_ex") != null); + try std.testing.expect(std.mem.indexOf(u8, quant_ir, "cvt.rn.satfinite.e4m3x2.f32") != null); + } +} diff --git a/zml/kernels/moe/cute/sm10x/nvfp4.zig b/zml/kernels/moe/cute/sm10x/nvfp4.zig new file mode 100644 index 00000000..5afee090 --- /dev/null +++ b/zml/kernels/moe/cute/sm10x/nvfp4.zig @@ -0,0 +1,176 @@ +//! CuTe input quantization kernel in front of the persistent NVFP4 GEMMs of +//! `persistent_nvfp4.zig`: routed NVFP4 input quantization. +const std = @import("std"); + +const zml = @import("../../../../zml.zig"); +const cute = zml.kernel.cute; +const B = cute.Builder; +const V = cute.Value; + +pub const GroupedQuantConfig = struct { + tokens: i64, + hidden: i64, + topk: i64, + experts: i64, + /// Rows stored per expert group. + group_size: i64, + capacity: i64, + /// Route `r` goes to row `r * group_size`; `route_map` is not read. + direct: bool = false, +}; + +const GroupedQuantize = cute.Program(GroupedQuantConfig, .{ + .name = "nvfp4_cute_grouped_quantize", + .inputs = &.{ "x", "route_map", "ids", "global_scale" }, + .outputs = &.{ "q", "s" }, + .run = groupedQuantizeProgram, +}); + +const quantize_kernel_name = "nvfp4_cute_grouped_quantize_kernel"; +/// Packed FP4 words (eight values) a CTA quantizes per token. +const words_per_cta = 128; + +/// NVFP4 E4M3 block scale of a block maximum: `scale = e4m3(global * +/// maximum / 6)` and the quantization multiplier `global / scale`, zero for an +/// all-zero block. +pub fn blockScale(b: *B, maximum: V, global: V) struct { bits: V, multiplier: V } { + const raw = global.mul(maximum.mul(1.0 / 6.0)).minimum(448.0); + const zero = b.constant(.f32, 0); + const bits = b.packFp8x4(raw, zero, zero, zero).bitAnd(0xff); + const scale = b.e4m3ToF32(bits); + const inverse = b.unaryF32("rcp.approx.ftz.f32", scale.mul(b.unaryF32("rcp.approx.ftz.f32", global))); + return .{ .bits = bits, .multiplier = b.select(scale.ne(0.0), inverse, zero) }; +} + +/// Offset of the scale of row `row` (within its 128-row group) and 16-wide +/// block `block` in the 128x4 tensor-core layout. +pub fn swizzledScaleOffset(row: V, block: V) V { + return block.div(4).mul(512) + .add(row.rem(32).mul(16)) + .add(row.div(32).mul(4)) + .add(block.rem(4)); +} + +/// Quantize each token once per routed expert and scatter it to the physical +/// rows assigned by the routing: packed FP4 `[capacity * group, hidden / 2]` +/// and E4M3 scales `[capacity, hidden / 16 * 128]` in the 128x4 layout. +/// `ids` are the `[tokens * topk]` local experts and `global_scale` the +/// `[experts]` FP32 activation global scales. +pub fn groupedQuantize(x: zml.Tensor, route_map: zml.Tensor, ids: zml.Tensor, global_scale: zml.Tensor, cfg: GroupedQuantConfig) struct { q: zml.Tensor, s: zml.Tensor } { + std.debug.assert(@rem(cfg.hidden, 64) == 0); + const result = GroupedQuantize.call( + .{ .x = x, .route_map = route_map, .ids = ids, .global_scale = global_scale }, + .{ + .q = .init(.{ cfg.capacity * cfg.group_size, @divExact(cfg.hidden, 2) }, .u8), + .s = .init(.{ cfg.capacity, @divExact(cfg.hidden, 16) * 128 }, .u8), + }, + .{ .cfg = cfg }, + ); + return .{ .q = result.q, .s = result.s }; +} + +fn groupedQuantizeProgram(b: *B, cfg: GroupedQuantConfig) cute.FinishError!void { + b.beginFunction(quantize_kernel_name, .cuda_kernel); + try groupedQuantizeKernel(b, cfg); + b.endFunction(.{ words_per_cta, 1, 1 }); + + b.beginFunction(GroupedQuantize.name, .host); + const host = try b.declareArgs(.{ + .x = .{ .ptr = cute.DType.bf16 }, + .route_map = .{ .ptr = cute.DType.i32 }, + .ids = .{ .ptr = cute.DType.i32 }, + .global_scale = .{ .ptr = cute.DType.f32 }, + .q = .{ .ptr = cute.DType.i8 }, + .s = .{ .ptr = cute.DType.i8 }, + }); + const one = b.constant(.i32, 1); + const words = @divExact(cfg.hidden, 8); + const config = b.makeLaunchConfig(.{ + .grid = .{ b.constant(.i32, @as(i32, @intCast(@divTrunc(words + words_per_cta - 1, words_per_cta)))), b.constant(.i32, @as(i32, @intCast(cfg.tokens))), one }, + .block = .{ b.constant(.i32, words_per_cta), one, one }, + .dynamic_smem = b.kernelSmemSize(quantize_kernel_name), + .stream = b.cudaStream(), + .cluster = .{ one, one, one }, + .use_pdl = true, + }); + const launch = b.launchEx(quantize_kernel_name, config, .{ + b.recastPointer(host.x, .i32, .gmem, 16), host.route_map, host.ids, host.global_scale, b.recastPointer(host.q, .i32, .gmem, 16), host.s, + }); + b.returnHostStatus(b.cudaResultStatus(launch)); + b.endFunction(null); +} + +/// Each thread quantizes eight consecutive values into one packed word, so +/// two lanes cover one 16-value NVFP4 block. +fn groupedQuantizeKernel(b: *B, cfg: GroupedQuantConfig) cute.FinishError!void { + const words = @divExact(cfg.hidden, 8); + const routes = cfg.tokens * cfg.topk; + const a = try b.declareArgs(.{ + // Two BF16 values per word. + .x = .{ .tensor = .{ .dtype = .i32, .shape = &.{ cfg.tokens, @divExact(cfg.hidden, 2) } } }, + .route_map = .{ .tensor = .{ .dtype = .i32, .shape = &.{routes} } }, + .ids = .{ .tensor = .{ .dtype = .i32, .shape = &.{routes} } }, + .global_scale = .{ .tensor = .{ .dtype = .f32, .shape = &.{cfg.experts} } }, + // Eight FP4 values per word. + .q = .{ .tensor = .{ .dtype = .i32, .shape = &.{ cfg.capacity * cfg.group_size, words } } }, + .s = .{ .tensor = .{ .dtype = .i8, .shape = &.{ cfg.capacity, @divExact(cfg.hidden, 16) * 128 } } }, + }); + + const tid = b.threadIdx().x; + // The routing and the previous layer write what this grid reads. + b.waitForDependency(); + const token = b.blockIdx().y; + const word = b.blockIdx().x.mul(words_per_cta).add(tid); + // Out-of-range lanes still take part in the pair reduction. + const in_range = word.lt(@as(i32, @intCast(words))); + const safe_word = b.select(in_range, word, b.constant(.i32, 0)); + var x: [8]V = undefined; + for (0..4) |i| { + const pair = a.x.get(.{ token, safe_word.mul(4).add(@as(i32, @intCast(i))) }); + x[2 * i] = pair.shl(16).bitCast(.f32); + x[2 * i + 1] = pair.bitAnd(@as(i32, @bitCast(@as(u32, 0xffff0000)))).bitCast(.f32); + } + var maximum = x[0].abs(); + for (x[1..]) |value| maximum = maximum.maximum(value.abs()); + maximum = maximum.maximum(b.shuffleXor(maximum, 1)); + + const block = safe_word.div(2); + var in_range_scope = b.openIf(in_range); + for (0..@intCast(cfg.topk)) |slot| { + const route = token.mul(cfg.topk).add(slot); + const row = if (cfg.direct) route.mul(cfg.group_size) else a.route_map.get(.{route}); + // Routes of other expert-parallel ranks are not scheduled. + const expert = a.ids.get(.{route}); + var scheduled = b.openIf(row.ge(0).bitAnd(expert.ge(0))); + const scale = blockScale(b, maximum, a.global_scale.get(.{expert})); + var quantized: [8]V = undefined; + for (&quantized, x) |*q, value| q.* = value.mul(scale.multiplier); + a.q.set(.{ row, safe_word }, b.packFp4x8(quantized)); + + var leader = b.openIf(tid.rem(2).eq(0)); + a.s.set(.{ row.div(cfg.group_size), swizzledScaleOffset(row.rem(cfg.group_size), block) }, scale.bits.to(.i8)); + leader.close(); + scheduled.close(); + } + in_range_scope.close(); + // The rows this CTA owns are complete: release the persistent GEMM. + b.launchDependents(); +} + +test "NVFP4 CuTe quantization emits for decode and prefill batches" { + inline for (.{ @as(i64, 1), 6, 96, 16_384 }) |tokens| { + const ir = try GroupedQuantize.emit(std.testing.allocator, .{ + .tokens = tokens, + .hidden = 5120, + .topk = 6, + .experts = 192, + .group_size = 16, + .capacity = @divTrunc(tokens * 6 + 15, 16) + 191, + .direct = tokens == 1, + }); + defer std.testing.allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "griddepcontrol.launch_dependents;") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "cuda.launch_ex") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "cvt.rn.satfinite.e2m1x2.f32") != null); + } +} diff --git a/zml/kernels/moe/cute/sm10x/persistent_mxfp4.zig b/zml/kernels/moe/cute/sm10x/persistent_mxfp4.zig new file mode 100644 index 00000000..30062a3a --- /dev/null +++ b/zml/kernels/moe/cute/sm10x/persistent_mxfp4.zig @@ -0,0 +1,1923 @@ +//! Source-level SM100 MXFP4 grouped GEMM. +//! +//! This file is the Zig CuTe-DSL counterpart of the CuTe DSL +//! `Sm100BlockScaledPersistentDenseGemmKernel` with its fused MoE epilogues. +//! It emits a complete CuTe program: the nested `cuda.kernel` consumes the +//! objects constructed by the public host `func.func`, and the host function +//! creates the block-scaled MMA objects, four TMA tensor maps, persistent +//! scheduler parameters, and the PDL-enabled launch. +//! +//! One CTA computes a 128xN output tile of one expert group: warp 5 drives +//! the TMA ring, warp 4 issues the S2T scale copies and UMMA, and warps 0-3 +//! drain the accumulator from tensor memory into one of the two epilogues. +const std = @import("std"); +const mlir = @import("mlir"); +const nvvm = @import("mlir/dialects/nvvm"); + +const zml = @import("../../../../zml.zig"); +const cute = zml.kernel.cute; +const B = cute.Builder; +const V = cute.Value; +const common = @import("common.zig"); +const typed = common.typed; +const gridTyped = common.gridTyped; +const barrierAt = common.barrierAt; +const PipelinePosition = common.PipelinePosition; +const PersistentWork = common.PersistentWork; +const DirectRoutes = common.DirectRoutes; +const persistentWorkAt = common.persistentWorkAt; +const advancePersistentWork = common.advancePersistentWork; +const advancePipeline = common.advancePipeline; +const peekBarrierIfRemaining = common.peekBarrierIfRemaining; +const sharedPointerAt = common.sharedPointerAt; +const offsetAlignment = common.offsetAlignment; +const typesOf = common.typesOf; +const tupleOf = common.tupleOf; +const fmt = common.fmt; +const view = common.view; + +const accumulator_stages = 2; + +/// Shared-memory plan of the Python kernel for one tile width, copied from +/// its generated `SharedStorage` (`smem.struct_fields`). The A/B ring depth +/// is whatever fits next to the two C epilogue stages; the fused epilogues +/// only use the start of `sC` as scratch. +pub const Tile = struct { + n: i64, + stages: usize, + smem_bytes: i64, + tmem_columns: u16, + empty_barriers: i64, + accumulator_full: i64, + accumulator_empty: i64, + tmem_dealloc: i64, + tmem_holding: i64, + c: i64, + a: i64, + b: i64, + sfa: i64, + sfb: i64, + /// Bytes reserved at `c`. Only the epilogue scratch is used; the wide + /// prefill tiles keep just that instead of two full C stages. + c_bytes: ?i64 = null, + + pub fn of(comptime n: i64) Tile { + return switch (n) { + 8 => .{ + .n = 8, + .stages = 12, + .smem_bytes = 230400, + .tmem_columns = 64, + .empty_barriers = 96, + .accumulator_full = 192, + .accumulator_empty = 208, + .tmem_dealloc = 224, + .tmem_holding = 232, + .c = 1024, + .a = 9216, + .b = 205824, + .sfa = 218112, + .sfb = 224256, + }, + 16 => .{ + .n = 16, + .stages = 11, + .smem_bytes = 232448, + .tmem_columns = 64, + .empty_barriers = 88, + .accumulator_full = 176, + .accumulator_empty = 192, + .tmem_dealloc = 208, + .tmem_holding = 216, + .c = 1024, + .a = 17408, + .b = 197632, + .sfa = 220160, + .sfb = 226304, + }, + 32 => .{ + .n = 32, + .stages = 9, + .smem_bytes = 228352, + .tmem_columns = 128, + .empty_barriers = 72, + .accumulator_full = 144, + .accumulator_empty = 160, + .tmem_dealloc = 176, + .tmem_holding = 184, + .c = 1024, + .a = 33792, + .b = 181248, + .sfa = 218112, + .sfb = 223232, + }, + // Prefill tiles. One expert weight tile feeds 64 or 128 routed + // rows instead of 32, which is what keeps large batches from + // re-reading the expert weights once per group. They hold only + // the epilogue scratch at `c`, so the A/B ring stays deep. + 64 => .{ + .n = 64, + .stages = 8, + .smem_bytes = 206848, + .tmem_columns = 256, + .empty_barriers = 64, + .accumulator_full = 128, + .accumulator_empty = 144, + .tmem_dealloc = 160, + .tmem_holding = 168, + .c = 1024, + .c_bytes = 512, + .a = 2048, + .sfa = 133120, + .sfb = 137216, + .b = 141312, + }, + 128 => .{ + .n = 128, + .stages = 6, + .smem_bytes = 204800, + .tmem_columns = 512, + .empty_barriers = 48, + .accumulator_full = 96, + .accumulator_empty = 112, + .tmem_dealloc = 128, + .tmem_holding = 136, + .c = 1024, + .c_bytes = 512, + .a = 2048, + .b = 100352, + .sfa = 198656, + .sfb = 201728, + }, + else => @compileError("unsupported MXFP4 tile N"), + }; + } + + /// Routed rows an epilogue pass handles: one per warp lane. + pub fn chunkRows(self: Tile) i64 { + return @min(self.n, 32); + } + + /// Epilogue passes over the tile width. + pub fn chunks(self: Tile) usize { + return @intCast(@divExact(self.n, self.chunkRows())); + } + + const a_stage_bytes = 128 * 128; + const sf_stage_bytes = 512; + + fn bStageBytes(self: Tile) i64 { + return self.n * 128; + } + + fn cStageElements(self: Tile) i64 { + return 128 * self.n; + } + + /// TMA bytes landing in one A/B stage: unpacked FP4 A is counted as + /// 128x128 nibbles, then the FP8 B tile and both 512-byte scale atoms. + fn transactionBytes(self: Tile) u32 { + return @intCast(8192 + self.bStageBytes() + 2 * sf_stage_bytes); + } + + fn fields(self: Tile, b: *B) []const []const u8 { + const arena = b.arena.allocator(); + const s: i64 = @intCast(self.stages); + const out = arena.alloc([]const u8, 11) catch @panic("OOM"); + const Field = struct { []const u8, i64, i64 }; + const list = [_]Field{ + .{ "ab_full_mbar_ptr", 8 * s, 0 }, + .{ "ab_empty_mbar_ptr", 8 * s, self.empty_barriers }, + .{ "acc_full_mbar_ptr", 16, self.accumulator_full }, + .{ "acc_empty_mbar_ptr", 16, self.accumulator_empty }, + .{ "tmem_dealloc_mbar", 8, self.tmem_dealloc }, + .{ "tmem_holding_buf", 4, self.tmem_holding }, + .{ "sC", self.c_bytes orelse accumulator_stages * self.cStageElements() * 4, self.c }, + .{ "sA", s * a_stage_bytes, self.a }, + .{ "sB", s * self.bStageBytes(), self.b }, + .{ "sSFA", s * sf_stage_bytes, self.sfa }, + .{ "sSFB", s * sf_stage_bytes, self.sfb }, + }; + for (list, out) |field, *text| { + text.* = std.fmt.allocPrint(arena, "{s}:{d}:{d}", .{ field[0], field[1], field[2] }) catch @panic("OOM"); + } + return out; + } +}; + +const StageStorage = struct { + a: V, + b: V, + sfa: V, + sfb: V, + c: V, + load_barriers: V, + mma_barriers: V, + accumulator_full_barriers: V, + accumulator_empty_barriers: V, +}; + +const StageBuffers = struct { + a_destination: cute.View, + b_destination: cute.View, + sfa_destination: cute.View, + sfb_destination: cute.View, + load_barrier: V, + mma_barrier: V, +}; + +const StagedOperands = struct { + a: cute.View, + b: cute.View, + sfa: cute.View, + sfb: cute.View, +}; + +/// What the epilogue warps do with each finished 128xN accumulator tile. +pub const Epilogue = enum { + /// Up projection: clamped SwiGLU of the interleaved gate/up rows, BF16 + /// rounding and the routing weight as in the reference model, then MXFP8 + /// quantization straight into the down GEMM's input and 128x4 scale + /// layouts. Python `fused=1`. + swiglu_mxfp8, + /// Down projection: BF16 rows stored in route order `[routes, m]`, + /// skipping padding rows. Python `fused=2`. BF16 halves the traffic of + /// this store and of the top-k reduction that reads it back; the routed + /// partial sums are rounded exactly like the reference finalize. + route_rows, +}; + +pub const Config = struct { + experts: i64, + /// GEMM output rows (4608 for gate/up, 5120 for down). + m: i64, + /// Tile N: rows in one routed expert group. + n: i64 = 16, + /// Reduction dimension (5120 for gate/up, 2304 for down). + k: i64, + groups: i64, + /// Resident CTAs of the static persistent tile scheduler: one per SM. + persistent_ctas: i64 = 148, + epilogue: Epilogue, + /// Token/top-k routes; sizes the route-indexed epilogue operands. + routes: i64, + /// Clamp applied to both halves of the fused SwiGLU. Emitted as an IR + /// constant, so a threaded value costs exactly what a literal did. + swiglu_limit: ?f32 = 10.0, + /// Ungrouped routing: group `g` is route `g` and holds one live row. + /// `schedule` is then just the `[routes]` expert ids; its size and + /// active-count sections and `route_inverse` are not read. + direct: bool = false, + + /// Rows stored per group in the routed operands: one whole N tile, of + /// which direct routing fills only the first. A tensor map whose row mode + /// had extent one would lose that mode altogether — the TMA atom infers a + /// zero stride for it — and every group would then read the same row. + pub fn groupRows(self: Config) i64 { + return self.n; + } + + pub fn routedRows(self: Config) i64 { + return self.groups * self.groupRows(); + } +}; + +/// `schedule` is the routing schedule `[group experts | group sizes | active +/// group count]`; the kernel reads the first and last sections in place. +const inputs = [_][:0]const u8{ + "weight", + "schedule", + "input_quant", + "weight_scale", + "input_scale", +}; + +/// `route_inverse` maps each physical routed row to its token/top-k route; +/// entries of padding rows are never read. +const UpQuantized = cute.Program(Config, .{ + .name = "zml_mxfp4_up_swiglu_zig", + .inputs = &(inputs ++ [_][:0]const u8{ "route_inverse", "routing_weights" }), + .outputs = &.{ "q", "s" }, + .run = buildProgram, +}); + +const DownRows = cute.Program(Config, .{ + .name = "zml_mxfp4_down_rows_zig", + .inputs = &(inputs ++ [_][:0]const u8{"route_inverse"}), + .outputs = &.{"output"}, + .run = buildProgram, +}); + +pub const Inputs = struct { + weight: zml.Tensor, + schedule: zml.Tensor, + input_quant: zml.Tensor, + weight_scale: zml.Tensor, + input_scale: zml.Tensor, +}; + +/// Up projection with the SwiGLU + MXFP8 epilogue: the routed FP8 rows +/// `[groups * n, m / 2]` and their 128x4 scales `[groups, m / 64 * 128]`, +/// ready for the down GEMM. `routing_weights` is `[routes]` FP32. +pub fn upQuantized(cfg: Config, a: Inputs, route_inverse: zml.Tensor, routing_weights: zml.Tensor) struct { q: zml.Tensor, s: zml.Tensor } { + std.debug.assert(cfg.epilogue == .swiglu_mxfp8); + const activations = @divExact(cfg.m, 2); + const result = UpQuantized.call(.{ + .weight = a.weight, + .schedule = a.schedule, + .input_quant = a.input_quant, + .weight_scale = a.weight_scale, + .input_scale = a.input_scale, + .route_inverse = route_inverse, + .routing_weights = routing_weights, + }, .{ + .q = .init(.{ cfg.routedRows(), activations }, .f8e4m3fn), + .s = .init(.{ cfg.groups, @divExact(activations, 32) * 128 }, .u8), + }, .{ .cfg = cfg, .scalars = &.{ cfg.m, cfg.n, cfg.k, cfg.groups, cfg.experts } }); + return .{ .q = result.q, .s = result.s }; +} + +/// Down projection storing FP32 `[routes, m]` in token/top-k order. +pub fn downRows(cfg: Config, a: Inputs, route_inverse: zml.Tensor) zml.Tensor { + std.debug.assert(cfg.epilogue == .route_rows); + return DownRows.call(.{ + .weight = a.weight, + .schedule = a.schedule, + .input_quant = a.input_quant, + .weight_scale = a.weight_scale, + .input_scale = a.input_scale, + .route_inverse = route_inverse, + }, .{ .output = .init(.{ cfg.routes, cfg.m }, .bf16) }, .{ .cfg = cfg, .scalars = &.{ cfg.m, cfg.n, cfg.k, cfg.groups, cfg.experts } }).output; +} + +/// The two projections share this kernel but get distinct symbols, so a +/// profile shows them as separate rows instead of one merged entry. +fn deviceName(comptime epilogue: Epilogue) [:0]const u8 { + return switch (epilogue) { + .swiglu_mxfp8 => "mxfp4_sm100_persistent_grouped_gemm_up_swiglu", + .route_rows => "mxfp4_sm100_persistent_grouped_gemm_down_rows", + }; +} + +fn buildProgram(b: *B, cfg: Config) cute.FinishError!void { + return switch (cfg.n) { + inline 8, 16, 32, 64, 128 => |n| switch (cfg.epilogue) { + inline else => |epilogue| buildTiledProgram(b, cfg, comptime Tile.of(n), epilogue), + }, + else => std.debug.panic("unsupported MXFP4 tile N {d}", .{cfg.n}), + }; +} + +/// Types and layouts shared by the device ABI and the host launch. +fn Layouts(comptime tile: Tile) type { + return struct { + const n = tile.n; + const stages = tile.stages; + const ab_swizzle: B.Swizzle = .{ .bits = 3, .base = 4, .shift = 3 }; + + fn aCoord(b: *B, cfg: Config) cute.LayoutSpec { + return b.layoutSpec(.{ cfg.m, cfg.k, cfg.experts }, .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}), b.basis(1, 1, .{2}) }); + } + fn bCoord(b: *B, cfg: Config) cute.LayoutSpec { + return b.layoutSpec(.{ cfg.groupRows(), cfg.k, cfg.groups }, .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}), b.basis(1, 1, .{2}) }); + } + fn sfCoord(b: *B) cute.LayoutSpec { + return b.layoutSpec( + .{ .{ .{ 32, 4 }, cute.AlgebraToken.dynamic }, .{ .{ 32, 4 }, cute.AlgebraToken.dynamic }, .{ 1, cute.AlgebraToken.dynamic } }, + .{ + .{ .{ b.basis(8, 1, .{0}), b.basis(2, 1, .{0}) }, b.basis(1, 1, .{1}) }, + .{ .{ 0, b.basis(1, 2, .{0}) }, b.basis(1, 1, .{2}) }, + .{ 0, b.basis(1, 1, .{3}) }, + }, + ); + } + fn aBasis(b: *B) cute.LayoutSpec { + return b.layoutSpec(.{ 128, 128, 1 }, .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}), b.basis(1, 1, .{2}) }); + } + fn bBasis(b: *B) cute.LayoutSpec { + return b.layoutSpec(.{ 128, n, 1 }, .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}), b.basis(1, 1, .{2}) }); + } + fn sfBasis(b: *B) cute.LayoutSpec { + return b.layoutSpec( + .{ .{ 2, 4, 32 }, 1, 1, 1 }, + .{ .{ b.basis(1, 1, .{ 1, 0, 1 }), b.basis(1, 1, .{ 0, 0, 1 }), b.basis(1, 1, .{ 0, 0, 0 }) }, b.basis(1, 1, .{ 0, 1 }), b.basis(1, 1, .{ 1, 1 }), b.basis(1, 1, .{ 2, 1 }) }, + ); + } + fn tmaA(b: *B, cfg: Config) cute.TmaConfig { + return .{ .dtype = .f4e2m1fn, .copy_bits = 65536, .global_basis = aBasis(b), .coordinate_layout = aCoord(b, cfg), .coordinate_rank = 3, .format = .u4_unpack_u8 }; + } + fn tmaB(b: *B, cfg: Config) cute.TmaConfig { + return .{ .dtype = .f8e4m3fn, .copy_bits = @intCast(tile.bStageBytes() * 8), .global_basis = bBasis(b), .coordinate_layout = bCoord(b, cfg), .coordinate_rank = 3 }; + } + fn tmaSf(b: *B) cute.TmaConfig { + return .{ .dtype = .f8e8m0fnu, .copy_bits = 4096, .global_basis = sfBasis(b), .coordinate_layout = sfCoord(b), .coordinate_rank = 4, .format = .u16 }; + } + // Shared-memory layouts, in the same element units as Python. + fn aSmem(b: *B, comptime staged: bool) cute.LayoutSpec { + return if (staged) + b.layoutSpec(.{ .{ 128, 32 }, 1, 4, stages }, .{ .{ 128, 1 }, 0, 32, Tile.a_stage_bytes }) + else + b.layoutSpec(.{ .{ 128, 32 }, 1, 4 }, .{ .{ 128, 1 }, 0, 32 }); + } + fn bSmem(b: *B, comptime staged: bool) cute.LayoutSpec { + return if (staged) + b.layoutSpec(.{ .{ n, 32 }, 1, 4, stages }, .{ .{ 128, 1 }, 0, 32, tile.bStageBytes() }) + else + b.layoutSpec(.{ .{ n, 32 }, 1, 4 }, .{ .{ 128, 1 }, 0, 32 }); + } + fn sfSmem(b: *B, comptime staged: bool) cute.LayoutSpec { + return if (staged) + b.layoutSpec(.{ .{ .{ .{ 32, 4 }, 1 }, .{ 32, 1 } }, 1, 4, stages }, .{ .{ .{ .{ 16, 4 }, 0 }, .{ 0, 0 } }, 0, 1, Tile.sf_stage_bytes }) + else + b.layoutSpec(.{ .{ .{ .{ 32, 4 }, 1 }, .{ 32, 1 } }, 1, 4 }, .{ .{ .{ .{ 16, 4 }, 0 }, .{ 0, 0 } }, 0, 1 }); + } + }; +} + +/// Device memrefs of the route-indexed epilogue operands. +const EpilogueTypes = struct { + sizes: *const mlir.Type, + inverse: *const mlir.Type, + weights: *const mlir.Type, + q: *const mlir.Type, + s: *const mlir.Type, + rows: *const mlir.Type, + + fn of(b: *B, cfg: Config) EpilogueTypes { + const activations = @divExact(cfg.m, 2); + return .{ + // Group sizes follow the group experts inside `schedule`. + .sizes = b.memrefType(.i32, .gmem, 4, b.layoutType(b.layoutSpec(cfg.groups, 1))), + .inverse = b.memrefType(.i32, .gmem, 16, b.layoutType(b.layoutSpec(cfg.routedRows(), 1))), + .weights = b.memrefType(.f32, .gmem, 16, b.layoutType(b.layoutSpec(@max(cfg.routes, 1), 1))), + .q = b.memrefType(.f8e4m3fn, .gmem, 16, b.layoutType(b.layoutSpec(.{ cfg.routedRows(), activations }, .{ activations, 1 }))), + .s = b.memrefType(.i8, .gmem, 16, b.layoutType(b.layoutSpec(.{ cfg.groups, @divExact(activations, 32) * 128 }, .{ @divExact(activations, 32) * 128, 1 }))), + .rows = b.memrefType(.bf16, .gmem, 16, b.layoutType(b.layoutSpec(.{ @max(cfg.routes, 1), cfg.m }, .{ cfg.m, 1 }))), + }; + } +}; + +fn buildTiledProgram(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: Epilogue) cute.FinishError!void { + @setEvalBranchQuota(20_000); + std.debug.assert(@rem(cfg.m, 128) == 0); + std.debug.assert(@rem(cfg.k, 128) == 0); + const L = Layouts(tile); + const public_name = b.name; + + const ids_memref = b.memrefType(.i32, .gmem, 16, b.layoutType(b.layoutSpec(cfg.groups, 1))); + // The active group count is the last schedule entry, 4-byte aligned. + const active_memref = b.memrefType(.i32, .gmem, 4, b.layoutType(b.layoutSpec(1, 1))); + const et = EpilogueTypes.of(b, cfg); + const i32_type = cute.DType.i32.toMlir(b.ctx); + const mma_type = b.blockScaledMmaType(.{ .n = @intCast(tile.n) }); + const mma_sfb_type = b.blockScaledMmaType(.{ .n = 128 }); + + // Device ABI. The host function below constructs every non-buffer value. + b.beginFunction(deviceName(epilogue), .cuda_kernel); + const d = try b.declareArgs(switch (epilogue) { + .swiglu_mxfp8 => .{ + .expert_ids = typed(ids_memref), + .active_count = typed(active_memref), + .mma = typed(mma_type), + .mma_sfb = typed(mma_sfb_type), + .tma_a = gridTyped(b.tmaLoadAtomType(L.tmaA(b, cfg))), + .gA = typed(b.coordTensorType(3, L.aCoord(b, cfg))), + .tma_b = gridTyped(b.tmaLoadAtomType(L.tmaB(b, cfg))), + .gB = typed(b.coordTensorType(3, L.bCoord(b, cfg))), + .tma_sfa = gridTyped(b.tmaLoadAtomType(L.tmaSf(b))), + .gSFA = typed(b.coordTensorType(4, L.sfCoord(b))), + .tma_sfb = gridTyped(b.tmaLoadAtomType(L.tmaSf(b))), + .gSFB = typed(b.coordTensorType(4, L.sfCoord(b))), + .group_sizes = typed(et.sizes), + .route_inverse = typed(et.inverse), + .routing_weights = typed(et.weights), + .q = typed(et.q), + .s = typed(et.s), + .tiles_m = typed(i32_type), + }, + .route_rows => .{ + .expert_ids = typed(ids_memref), + .active_count = typed(active_memref), + .mma = typed(mma_type), + .mma_sfb = typed(mma_sfb_type), + .tma_a = gridTyped(b.tmaLoadAtomType(L.tmaA(b, cfg))), + .gA = typed(b.coordTensorType(3, L.aCoord(b, cfg))), + .tma_b = gridTyped(b.tmaLoadAtomType(L.tmaB(b, cfg))), + .gB = typed(b.coordTensorType(3, L.bCoord(b, cfg))), + .tma_sfa = gridTyped(b.tmaLoadAtomType(L.tmaSf(b))), + .gSFA = typed(b.coordTensorType(4, L.sfCoord(b))), + .tma_sfb = gridTyped(b.tmaLoadAtomType(L.tmaSf(b))), + .gSFB = typed(b.coordTensorType(4, L.sfCoord(b))), + .group_sizes = typed(et.sizes), + .route_inverse = typed(et.inverse), + .rows = typed(et.rows), + .tiles_m = typed(i32_type), + }, + }); + b.setFunctionAttribute(nvvm.minctasm(b.ctx, 1)); + b.setFunctionAttribute(.named(b.ctx, "smem.partition_num", .int(b.ctx, .i32, 2))); + b.setFunctionAttribute(.named(b.ctx, "smem.hint_smem_base_uniform", .unit(b.ctx))); + b.setFunctionAttribute(.named(b.ctx, "cu_attrs", .dict(b.ctx, &.{ + .named(b.ctx, "max_dynamic_shared_size_bytes", b.devMaxSharedMemoryOptin()), + .named(b.ctx, "non_portable_cluster_size_allowed", .int(b.ctx, .i32, 1)), + }))); + + // Prefetch every tensor-map descriptor before the specialized warps begin + // the persistent pipeline. This is the work PDL buys us: it runs while the + // producing grid is still writing the operands. + const warp = b.makeWarpUniform(b.makeWarpUniform(b.threadIdx().x.div(32))); + var tma_warp = b.openIf(warp.eq(5)); + { + inline for (.{ d.tma_a, d.tma_b, d.tma_sfa, d.tma_sfb }) |value| { + const descriptor: cute.Atom = .{ .inner = value.inner, .kernel = b }; + b.prefetchTmaDesc(descriptor); + } + tma_warp.close(); + } + // Everything below reads the schedule and the quantized activations the + // preceding grid produces, so the whole block waits here. + b.waitForDependency(); + buildUmmaDevice(b, cfg, tile, epilogue, d); + // The epilogue has written every row this CTA owns. + b.launchDependents(); + b.endFunction(.{ 192, 1, 1 }); + + // Public host ABI: the XLA operands, the outputs, then the problem shape. + // XLA owns packed storage as signless bytes. Recast only inside the host + // program when constructing typed tensor maps. + b.beginFunction(public_name, .host); + const operands = .{ + .weight = .{ .ptr = cute.DType.i8 }, + .schedule = .{ .ptr = cute.DType.i32 }, + .input_quant = .{ .ptr = cute.DType.f8e4m3fn }, + .weight_scale = .{ .ptr = cute.DType.i32 }, + .input_scale = .{ .ptr = cute.DType.i8 }, + }; + const shape = .{ + .problem_m = typed(i32_type), + .problem_n = typed(i32_type), + .problem_k = typed(i32_type), + .problem_l = typed(i32_type), + .expert_count = typed(i32_type), + }; + _ = try b.declareArgs(switch (epilogue) { + .swiglu_mxfp8 => .{ + .weight = operands.weight, + .schedule = operands.schedule, + .input_quant = operands.input_quant, + .weight_scale = operands.weight_scale, + .input_scale = operands.input_scale, + .route_inverse = .{ .ptr = cute.DType.i32 }, + .routing_weights = .{ .ptr = cute.DType.f32 }, + .q = .{ .ptr = cute.DType.f8e4m3fn }, + .s = .{ .ptr = cute.DType.i8 }, + .problem_m = shape.problem_m, + .problem_n = shape.problem_n, + .problem_k = shape.problem_k, + .problem_l = shape.problem_l, + .expert_count = shape.expert_count, + }, + .route_rows => .{ + .weight = operands.weight, + .schedule = operands.schedule, + .input_quant = operands.input_quant, + .weight_scale = operands.weight_scale, + .input_scale = operands.input_scale, + .route_inverse = .{ .ptr = cute.DType.i32 }, + .output = .{ .ptr = cute.DType.bf16 }, + .problem_m = shape.problem_m, + .problem_n = shape.problem_n, + .problem_k = shape.problem_k, + .problem_l = shape.problem_l, + .expert_count = shape.expert_count, + }, + }); + try buildHostLaunch(b, cfg, tile, epilogue); + b.endFunction(null); +} + +/// Host argument positions: the five mainloop operands, the epilogue's +/// extra inputs and outputs, then the five problem-shape scalars. +fn HostArgs(comptime epilogue: Epilogue) type { + return struct { + const weight = 0; + const schedule = 1; + const input_quant = 2; + const weight_scale = 3; + const input_scale = 4; + const route_inverse = 5; + const routing_weights = 6; + const output = switch (epilogue) { + .swiglu_mxfp8 => 7, + .route_rows => 6, + }; + const scales = 8; + const problem_m = switch (epilogue) { + .swiglu_mxfp8 => 9, + .route_rows => 7, + }; + const problem_n = problem_m + 1; + const problem_k = problem_m + 2; + const problem_l = problem_m + 3; + const expert_count = problem_m + 4; + }; +} + +/// Complete staged shared-memory tensors, as Python's `PipelineTmaUmma` +/// consumers see them. Per-stage views are slices of these. +fn stagedOperands(b: *B, comptime tile: Tile, storage: StageStorage) StagedOperands { + const L = Layouts(tile); + const a_full = b.makeView( + b.recastSwizzledPointer(storage.a, .i8, .smem, 1024, L.ab_swizzle), + b.staticLayout(L.aSmem(b, true)), + b.memrefTypeFromPointer(b.swizzledPtrTy(.i8, .smem, 1024, L.ab_swizzle) catch @panic("bad A shared pointer"), b.layoutType(L.aSmem(b, true))), + ); + const b_full = b.makeView( + b.recastSwizzledPointer(storage.b, .f8e4m3fn, .smem, 1024, L.ab_swizzle), + b.staticLayout(L.bSmem(b, true)), + b.memrefTypeFromPointer(b.swizzledPtrTy(.f8e4m3fn, .smem, 1024, L.ab_swizzle) catch @panic("bad B shared pointer"), b.layoutType(L.bSmem(b, true))), + ); + const sfa_full = b.makeView(storage.sfa, b.staticLayout(L.sfSmem(b, true)), b.memrefType(.f8e8m0fnu, .smem, 1024, b.layoutType(L.sfSmem(b, true)))); + const sfb_full = b.makeView(storage.sfb, b.staticLayout(L.sfSmem(b, true)), b.memrefType(.f8e8m0fnu, .smem, 1024, b.layoutType(L.sfSmem(b, true)))); + return .{ .a = a_full, .b = b_full, .sfa = sfa_full, .sfb = sfb_full }; +} + +fn stageBuffersAt(b: *B, comptime tile: Tile, storage: StageStorage, stage_index: V) StageBuffers { + const L = Layouts(tile); + // Keep the shared operands as staged tensors and select the current stage + // with CuTe slicing. This is the representation used by the Python + // PipelineTmaUmma implementation; rebuilding a memref from a runtime + // pointer loses the staged descriptor semantics at ring wrap. + const full = stagedOperands(b, tile, storage); + const all = cute.AlgebraToken.all; + const a = b.slice(full.a, .{ all, all, all, stage_index }, b.memrefTypeFromPointer(b.swizzledPtrTy(.i8, .smem, 1024, L.ab_swizzle) catch @panic("bad A stage pointer"), b.layoutType(L.aSmem(b, false)))); + const rhs = b.slice(full.b, .{ all, all, all, stage_index }, b.memrefTypeFromPointer(b.swizzledPtrTy(.f8e4m3fn, .smem, 1024, L.ab_swizzle) catch @panic("bad B stage pointer"), b.layoutType(L.bSmem(b, false)))); + // One scale stage is 512 bytes, so slicing a stage out of the ring keeps + // that alignment, not the array's. + const sf_stage_align = Tile.sf_stage_bytes; + const sfa = b.slice(full.sfa, .{ all, all, all, stage_index }, b.memrefType(.f8e8m0fnu, .smem, sf_stage_align, b.layoutType(L.sfSmem(b, false)))); + const sfb = b.slice(full.sfb, .{ all, all, all, stage_index }, b.memrefType(.f8e8m0fnu, .smem, sf_stage_align, b.layoutType(L.sfSmem(b, false)))); + + // `tma_partition` preserves a singleton CTA mode. Keep it in the copy + // views: it is part of the executable TMA atom's expected algebra even + // for a one-CTA cluster. + const a_partition_layout = b.staticLayout(b.layoutSpec(.{ .{ Tile.a_stage_bytes, 1 }, .{1} }, .{ .{ 1, 0 }, .{0} })); + const b_partition_layout = b.staticLayout(b.layoutSpec(.{ .{ tile.bStageBytes(), 1 }, .{1} }, .{ .{ 1, 0 }, .{0} })); + const sf_partition_layout = b.staticLayout(b.layoutSpec(.{ .{ Tile.sf_stage_bytes, 1 }, .{1} }, .{ .{ 1, 0 }, .{0} })); + + return .{ + .a_destination = b.withLayout(a, a_partition_layout), + .b_destination = b.withLayout(rhs, b_partition_layout), + .sfa_destination = b.withLayout(sfa, sf_partition_layout), + .sfb_destination = b.withLayout(sfb, sf_partition_layout), + .load_barrier = barrierAt(b, storage.load_barriers, stage_index), + .mma_barrier = barrierAt(b, storage.mma_barriers, stage_index), + }; +} + +/// Issue the four transactions that fill one mainloop stage. The caller is +/// already restricted to the TMA warp; one elected lane owns the mbarrier +/// arrival. +fn issueTmaStage( + b: *B, + comptime tile: Tile, + comptime reduction_tiles: usize, + partitioned_a_target: cute.View, + partitioned_b_target: cute.View, + stage: StageBuffers, + k_tile: V, + tile_m: V, + group: V, + expert: V, + exec_a: cute.Atom, + exec_b: cute.Atom, + exec_sfa: cute.Atom, + exec_sfb: cute.Atom, +) void { + const n = tile.n; + const all = cute.AlgebraToken.all; + const dynamic = cute.AlgebraToken.dynamic; + const div_128: cute.ConstrainedDynamic = .{ .divisible_by = 128 }; + const div_n: cute.ConstrainedDynamic = .{ .divisible_by = @intCast(n) }; + const a_copy_layout = b.staticLayout(b.layoutSpec( + .{ .{ .{ 128, 128 }, 1 }, .{1} }, + .{ .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0 }, .{0} }, + )); + const b_copy_layout = b.staticLayout(b.layoutSpec( + .{ .{ .{ 128, n }, 1 }, .{1} }, + .{ .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0 }, .{0} }, + )); + const sf_copy_spec = b.layoutSpec( + .{ .{ 512, 1 }, .{1} }, + .{ .{ b.basis(1, 2, .{0}), 0 }, .{0} }, + ); + // Follow the same coordinate algebra as the Python kernel. TMA + // partitioning produces a tensor over (tile, M, K, expert/group); first + // select the output tile and expert/group, then select the K stage. The + // remaining iterator carries the tensor-map coordinates needed by copy. + const a_tile_k = b.slice( + partitioned_a_target, + .{ all, tile_m, all, expert }, + b.coordTensorTypeWithTuple(.{ 0, div_128, dynamic }, b.layoutSpec( + .{ .{ .{ 128, 128 }, 1 }, reduction_tiles }, + .{ .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0 }, b.basis(128, 1, .{0}) }, + )), + ); + const a_tile = b.slice( + a_tile_k, + .{ all, k_tile }, + b.coordTensorTypeWithTuple(.{ div_128, div_128, dynamic }, b.layoutSpec( + .{.{ .{ 128, 128 }, 1 }}, + .{.{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0 }}, + )), + ); + const a_source = b.rebuildCoordTensorIterator(3, a_tile, .{ div_128, div_128, dynamic }, .{ 128, 128, 1 }, a_copy_layout); + // B's tile spans all of a group's rows, so its N-tile index is zero. Pass + // it as a value rather than a literal: a statically zero leaf collapses out + // of the coordinate tuple, and the TMA lowering then shifts the remaining + // tensor-map coordinates, silently reading group zero for every group. + const b_tile_n = b.constant(.i32, 0); + const b_tile_k = b.slice( + partitioned_b_target, + .{ all, b_tile_n, all, group }, + b.coordTensorTypeWithTuple(.{ 0, div_n, dynamic }, b.layoutSpec( + .{ .{ .{ 128, n }, 1 }, reduction_tiles }, + .{ .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0 }, b.basis(128, 1, .{0}) }, + )), + ); + const b_tile = b.slice( + b_tile_k, + .{ all, k_tile }, + b.coordTensorTypeWithTuple(.{ div_128, div_n, dynamic }, b.layoutSpec( + .{.{ .{ 128, n }, 1 }}, + .{.{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0 }}, + )), + ); + const b_source = b.rebuildCoordTensorIterator(3, b_tile, .{ div_128, div_n, dynamic }, .{ 128, n, 1 }, b_copy_layout); + const sfa_source = b.makeCoordTensor(.{ 0, tile_m, k_tile, expert }, b.staticLayout(sf_copy_spec)); + const sfb_source = b.makeCoordTensor(.{ 0, b.constant(.i32, 0), k_tile, group }, b.staticLayout(sf_copy_spec)); + + var elected = b.openIf(b.electSync()); + { + b.mbarrierArriveExpectTx(stage.load_barrier, tile.transactionBytes()); + elected.close(); + } + b.copy(b.setTmaBarrier(exec_a, stage.load_barrier), .{a_source}, .{stage.a_destination}, .{}); + b.copy(b.setTmaBarrier(exec_b, stage.load_barrier), .{b_source}, .{stage.b_destination}, .{}); + b.copy(b.setTmaBarrier(exec_sfa, stage.load_barrier), .{sfa_source}, .{stage.sfa_destination}, .{}); + b.copy(b.setTmaBarrier(exec_sfb, stage.load_barrier), .{sfb_source}, .{stage.sfb_destination}, .{}); +} + +fn accumulatorTmemAt(b: *B, comptime tile: Tile, tmem: V, stage: V) V { + // The stage offset is a whole tile width, which `cute.add_offset` needs to + // know: it derives the result alignment from what it knows of the offset. + return b.addOffset( + tmem, + b.makeIntTupleDivBy(stage.mul(tile.n), @intCast(tile.n)), + b.ptrTy(.f32, .tmem, 16) catch @panic("bad accumulator TMEM pointer"), + ).value(); +} + +/// Warp 5 runs independently over every persistent output tile. Keep the +/// producer index and phase as loop-carried values across tiles, exactly like +/// CUTLASS PipelineState. Only the per-tile count is reset to zero. +fn runTmaProducer( + b: *B, + comptime tile: Tile, + comptime k_tiles: usize, + storage: StageStorage, + active: V, + tiles_m: V, + expert_ids: cute.View, + direct: ?DirectRoutes, + partitioned_a_target: cute.View, + partitioned_b_target: cute.View, + exec_a: cute.Atom, + exec_b: cute.Atom, + exec_sfa: cute.Atom, + exec_sfb: cute.Atom, +) void { + const initial_index = b.constant(.i32, 0); + const initial_phase = b.constant(.i32, 1); + const initial_count = b.constant(.i32, 0); + const persistent_clusters = b.gridDim().z; + const cta_m = b.blockIdx().x; + const cta_n = b.blockIdx().y; + const cta_l = b.constant(.i32, 0); + const div_m = b.fastDivmodCreate(tiles_m); + const div_n = b.fastDivmodCreate(1); + const initial_work = persistentWorkAt(b, b.blockIdx().z, b.constant(.i32, 0), active, tiles_m, div_m, div_n, direct); + const carried = .{ + initial_work.m, initial_work.n, initial_work.group, initial_work.valid, + initial_count, initial_index, initial_phase, persistent_clusters, + initial_work.linear, cta_m, cta_n, cta_l, + initial_work.executed, tiles_m, active, div_m, + div_n, + }; + var tiles = b.openWhile(carried, carried); + { + tiles.yieldBefore(tiles.before_carried[3], tiles.before_carried); + } + { + const tile_m = tiles.after_carried[0]; + const group = tiles.after_carried[2]; + const expert = expert_ids.get(.{group}, .i32); + const first_state: PipelinePosition = .{ + .index = tiles.after_carried[5], + .phase = tiles.after_carried[6], + }; + const first_ready = b.mbarrierTryWait(barrierAt(b, storage.mma_barriers, first_state.index), first_state.phase, .{ .scope = .cta }); + var stages = b.openFor( + 0, + k_tiles, + 1, + .{ first_ready, b.constant(.i32, 0), first_state.index, first_state.phase }, + ); + { + const k = stages.carried[1]; + const state: PipelinePosition = .{ + .index = stages.carried[2], + .phase = stages.carried[3], + }; + const buffers = stageBuffersAt(b, tile, storage, state.index); + var pending = b.openIf(stages.carried[0].eq(false)); + { + b.mbarrierWait(buffers.mma_barrier, state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); + pending.close(); + } + issueTmaStage(b, tile, k_tiles, partitioned_a_target, partitioned_b_target, buffers, k, tile_m, group, expert, exec_a, exec_b, exec_sfa, exec_sfb); + const next_count = stages.carried[1].add(1); + const next_state = advancePipeline(b, state.index, state.phase, tile.stages); + const next_ready = peekBarrierIfRemaining(b, storage.mma_barriers, next_count, k_tiles, next_state); + stages.yield(.{ next_ready, next_count, next_state.index, next_state.phase }); + } + const next_work = advancePersistentWork( + b, + tiles.after_carried[8], + tiles.after_carried[12], + tiles.after_carried[7], + tiles.after_carried[14], + tiles.after_carried[13], + tiles.after_carried[15], + tiles.after_carried[16], + direct, + ); + tiles.yieldAfter(.{ + next_work.m, + next_work.n, + next_work.group, + next_work.valid, + stages.results[1], + stages.results[2], + stages.results[3], + tiles.after_carried[7], + next_work.linear, + tiles.after_carried[9], + tiles.after_carried[10], + tiles.after_carried[11], + next_work.executed, + tiles.after_carried[13], + tiles.after_carried[14], + tiles.after_carried[15], + tiles.after_carried[16], + }); + } + + // PipelineTmaUmma.producer_tail: drain every empty barrier before the TMA + // warp exits so no outstanding tcgen05 arrival outlives the CTA. + var tail_state: PipelinePosition = .{ + .index = tiles.results[5], + .phase = tiles.results[6], + }; + inline for (0..tile.stages) |stage| { + b.mbarrierWait(barrierAt(b, storage.mma_barriers, tail_state.index), tail_state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); + if (stage + 1 < tile.stages) { + tail_state = advancePipeline(b, tail_state.index, tail_state.phase, tile.stages); + } + } +} + +/// SM100 MXFP4 mainloop objects of one pipeline stage, produced by +/// `StagedMxfp4Mainloop.atStage`. The dependent CuTe result types are +/// centralized here; callers only provide tensors and the allocated +/// tensor-memory pointer. +const Mxfp4Mainloop = struct { + a: V, + b: V, + accumulator: V, + sfa: V, + sfb: V, + sfa_copy: cute.Atom, + sfb_copy: cute.Atom, + sfa_source: V, + sfb_source: V, + sfa_target: V, + sfb_target: V, + + pub fn copyScales(self: Mxfp4Mainloop, builder: *B) void { + builder.copy(self.sfa_copy, .{self.sfa_source}, .{self.sfa_target}, .{}); + builder.copy(self.sfb_copy, .{self.sfb_source}, .{self.sfb_target}, .{}); + } +}; + +/// Pipeline-wide SM100 MXFP4 mainloop objects. CUTLASS derives the UMMA and +/// S2T descriptors from the complete staged tensors once, then slices those +/// descriptors at the current PipelineState index. Keeping the stage mode in +/// the descriptor is required when the circular buffer wraps back to stage 0. +const StagedMxfp4Mainloop = struct { + a: cute.View, + b: cute.View, + sfa_source: cute.View, + sfb_source: cute.View, + sfa: V, + sfb: V, + sfa_copy: cute.Atom, + sfb_copy: cute.Atom, + sfa_target: V, + sfb_target: V, + /// MMA tile N: accumulator columns per stage. + n: i64, + + pub fn atStage(self: StagedMxfp4Mainloop, b: *B, index: V, accumulator_tmem: V) Mxfp4Mainloop { + const all = cute.AlgebraToken.all; + const a = b.slice( + self.a, + .{ all, all, all, index }, + b.smemDescViewType(b.layoutSpec(.{ 1, 1, 4 }, .{ 0, 0, 2 })), + ); + const rhs = b.slice( + self.b, + .{ all, all, all, index }, + b.smemDescViewType(b.layoutSpec(.{ 1, 1, 4 }, .{ 0, 0, 2 })), + ); + const sfa_stage = b.slice( + self.sfa_source, + .{ all, all, all, all, index }, + b.smemDescViewType(b.layoutSpec(.{ .{ .{ 32, 1, 1, 4 }, 1 }, 1, 1, 1 }, .{ .{ .{ 1, 1, 1, 0 }, 0 }, 0, 0, 0 })), + ); + const sfb_stage = b.slice( + self.sfb_source, + .{ all, all, all, all, index }, + b.smemDescViewType(b.layoutSpec(.{ .{ .{ 32, 1, 1, 4 }, 1 }, 1, 1, 1 }, .{ .{ .{ 1, 1, 1, 0 }, 0 }, 0, 0, 0 })), + ); + const source_grouped_spec = b.layoutSpec(.{ .{ .{ 32, 1, 1, 4 }, 1 }, .{ 1, 1, 1 } }, .{ .{ .{ 1, 1, 1, 0 }, 0 }, .{ 0, 0, 0 } }); + const source_grouped_layout = b.staticLayout(source_grouped_spec); + const source_grouped_ty = b.smemDescViewType(source_grouped_spec); + const sfa_source = b.makeView( + b.getIter(sfa_stage, b.smemDescType()), + source_grouped_layout, + source_grouped_ty, + ); + const sfb_source = b.makeView( + b.getIter(sfb_stage, b.smemDescType()), + source_grouped_layout, + source_grouped_ty, + ); + const acc_spec = b.layoutSpec(.{ .{ 128, self.n }, 1, 1 }, .{ .{ 65536, 1 }, 0, 0 }); + const acc_layout = b.staticLayout(acc_spec); + const accumulator = b.makeView( + accumulator_tmem, + acc_layout, + b.memrefType(.f32, .tmem, 16, b.layoutType(acc_spec)), + ); + return .{ + .a = a.value(), + .b = rhs.value(), + .accumulator = accumulator.value(), + .sfa = self.sfa, + .sfb = self.sfb, + .sfa_copy = self.sfa_copy, + .sfb_copy = self.sfb_copy, + .sfa_source = sfa_source.value(), + .sfb_source = sfb_source.value(), + .sfa_target = self.sfa_target, + .sfb_target = self.sfb_target, + }; + } +}; + +/// Build the same pipeline-wide descriptors as Python's +/// `Sm100BlockScaledPersistentDenseGemmKernel`: `make_fragment_A/B` and the +/// S2T partitions see the complete staged tensors. `atStage` performs the +/// per-iteration slicing afterwards. +fn makeMxfp4StagedMainloop( + b: *B, + tiled_mma: cute.Atom, + comptime n: i64, + comptime stages: usize, + smem_a: V, + smem_b: V, + smem_sfa: V, + smem_sfb: V, + scale_tmem: V, +) StagedMxfp4Mainloop { + // Descriptor strides are in 16-byte units: a 128x128 unpacked-FP4 A + // stage is 16 KiB and an Nx128 FP8 B stage is N*128 bytes. + const frag_a = b.mmaMakeFragment( + tiled_mma, + smem_a, + 0, + b.smemDescViewType(b.layoutSpec(.{ 1, 1, 4, stages }, .{ 0, 0, 2, 1024 })), + ); + const frag_b = b.mmaMakeFragment( + tiled_mma, + smem_b, + 1, + b.smemDescViewType(b.layoutSpec(.{ 1, 1, 4, stages }, .{ 0, 0, 2, n * 8 })), + ); + + // TMEM columns: two N-wide accumulator stages, then 16 SFA and 16 SFB + // columns, matching num_accumulator_tmem_cols/num_sfa_tmem_cols in Python. + const sf_spec = b.layoutSpec(.{ .{ .{ .{ 32, 4 }, 4 }, .{ 32, 1 } }, 1, 4 }, .{ .{ .{ .{ 262144, 4 }, 8388608 }, .{ 0, 0 } }, 0, 1 }); + const sf_layout = b.staticLayout(sf_spec); + const tmem_f32_ptr = b.ptrTy(.f32, .tmem, 16) catch @panic("bad scale-factor TMEM pointer"); + const tmem_sf_ptr = b.ptrTy(.f8e8m0fnu, .tmem, 16) catch @panic("bad scale-factor TMEM pointer"); + const sfa_col = b.addOffset(scale_tmem, b.staticIntTuple(2 * n), tmem_f32_ptr); + const sfb_col = b.addOffset(scale_tmem, b.staticIntTuple(2 * n + 16), tmem_f32_ptr); + const sfa_ptr = b.recastIter(sfa_col, tmem_sf_ptr); + const sfb_ptr = b.recastIter(sfb_col, tmem_sf_ptr); + const sf_memref = b.memrefType(.f8e8m0fnu, .tmem, 16, b.layoutType(sf_spec)); + const sfa = b.makeView(sfa_ptr, sf_layout, sf_memref); + const sfb = b.makeView(sfb_ptr, sf_layout, sf_memref); + + const smem_filtered_ty = b.memrefType(.f8e8m0fnu, .smem, 1024, b.layoutType(b.layoutSpec( + .{ .{ .{ .{ 32, 4 }, 1 }, .{ 1, 1 } }, 1, 4, stages }, + .{ .{ .{ .{ 16, 4 }, 0 }, .{ 0, 0 } }, 0, 1, 512 }, + ))); + const tmem_filtered_ty = b.memrefType(.f8e8m0fnu, .tmem, 16, b.layoutType(b.layoutSpec( + .{ .{ .{ .{ 32, 4 }, 4 }, .{ 1, 1 } }, 1, 4 }, + .{ .{ .{ .{ 262144, 4 }, 8388608 }, .{ 0, 0 } }, 0, 1 }, + ))); + const sfa_smem_filtered = b.filterZeros(smem_sfa, null, smem_filtered_ty); + const sfb_smem_filtered = b.filterZeros(smem_sfb, null, smem_filtered_ty); + const sfa_tmem_filtered = b.filterZeros(sfa, null, tmem_filtered_ty); + const sfb_tmem_filtered = b.filterZeros(sfb, null, tmem_filtered_ty); + + const copy_atom_ty = b.s2tCopyAtomType(.f8e8m0fnu, .{ .num_dp = 32, .num_bit = 128, .broadcast = .x4 }); + const copy_atom = b.makeAtom(copy_atom_ty, .{}); + const copy_ty = b.tiledCopyType( + copy_atom_ty, + b.layoutSpec(.{ 1, .{ 32, .{ 4, 4 }, 4 } }, .{ 0, .{ 1, .{ 512, 32 }, 128 } }), + &.{ b.layoutSpec(512, 1), b.layoutSpec(1, 0), b.layoutSpec(4, 1) }, + ); + const sfa_copy = b.makeS2tCopy(copy_atom, sfa_tmem_filtered, copy_ty); + const sfb_copy = b.makeS2tCopy(copy_atom, sfb_tmem_filtered, copy_ty); + + const source_partition_ty = b.memrefType(.f8e8m0fnu, .smem, 1024, b.layoutType(b.layoutSpec( + .{ .{ .{ 32, 4, 4, 4 }, 1 }, 1, 1, 1, stages }, + .{ .{ .{ 16, 1, 4, 0 }, 0 }, 0, 0, 0, 512 }, + ))); + const source_desc_ty = b.smemDescViewType(b.layoutSpec( + .{ .{ .{ 32, 1, 1, 4 }, 1 }, 1, 1, 1, stages }, + .{ .{ .{ 1, 1, 1, 0 }, 0 }, 0, 0, 0, 32 }, + )); + const target_partition_ty = b.memrefType(.f8e8m0fnu, .tmem, 16, b.layoutType(b.layoutSpec( + .{ .{ .{ 32, 4, .{ 4, 4 } }, 1 }, 1, 1, 1 }, + .{ .{ .{ 262144, 1, .{ 4, 8388608 } }, 0 }, 0, 0, 0 }, + ))); + const target_grouped_spec = b.layoutSpec( + .{ .{ .{ 32, 4, .{ 4, 4 } }, 1 }, .{ 1, 1, 1 } }, + .{ .{ .{ 262144, 1, .{ 4, 8388608 } }, 0 }, .{ 0, 0, 0 } }, + ); + const target_grouped_ty = b.memrefType(.f8e8m0fnu, .tmem, 16, b.layoutType(target_grouped_spec)); + const sfa_src_partition = b.tiledCopyPartition(sfa_copy, sfa_smem_filtered, .{0}, false, source_partition_ty); + const sfb_src_partition = b.tiledCopyPartition(sfb_copy, sfb_smem_filtered, .{0}, false, source_partition_ty); + const sfa_desc = b.s2tSmemDescriptor(sfa_copy, sfa_src_partition, source_desc_ty); + const sfb_desc = b.s2tSmemDescriptor(sfb_copy, sfb_src_partition, source_desc_ty); + const sfa_dst_partition = b.tiledCopyPartition(sfa_copy, sfa_tmem_filtered, .{0}, true, target_partition_ty); + const sfb_dst_partition = b.tiledCopyPartition(sfb_copy, sfb_tmem_filtered, .{0}, true, target_partition_ty); + const target_grouped_layout = b.staticLayout(target_grouped_spec); + const sfa_dst = b.makeView(b.getIter(sfa_dst_partition, tmem_sf_ptr), target_grouped_layout, target_grouped_ty); + const sfb_dst = b.makeView(b.getIter(sfb_dst_partition, tmem_sf_ptr), target_grouped_layout, target_grouped_ty); + + return .{ + .a = frag_a, + .b = frag_b, + .sfa_source = sfa_desc, + .sfb_source = sfb_desc, + .sfa = sfa.value(), + .sfb = sfb.value(), + .sfa_copy = sfa_copy, + .sfb_copy = sfb_copy, + .sfa_target = sfa_dst.value(), + .sfb_target = sfb_dst.value(), + .n = n, + }; +} + +/// Warp 4 drains the A/B ring into one of two independent accumulator stages. +fn runMmaConsumer( + b: *B, + comptime tile: Tile, + comptime k_tiles: usize, + storage: StageStorage, + tiled_mma: cute.Atom, + tmem_holding: V, + active: V, + tiles_m: V, +) void { + const tmem = b.retrieveTmemPtr(tmem_holding, .f32, 16); + // Match Python's construction order: derive A/B UMMA fragments and S2T + // scale descriptors from the complete staged tensors once. The K loop + // slices these compiler-derived descriptors by PipelineState.index. + const staged = stagedOperands(b, tile, storage); + const staged_mainloop = makeMxfp4StagedMainloop( + b, + tiled_mma, + tile.n, + tile.stages, + staged.a.value(), + staged.b.value(), + staged.sfa.value(), + staged.sfb.value(), + tmem, + ); + const persistent_clusters = b.gridDim().z; + const div_m = b.fastDivmodCreate(tiles_m); + const div_n = b.fastDivmodCreate(1); + // The MMA warp never reads the group, only the tile count. + const initial_work = persistentWorkAt(b, b.blockIdx().z, b.constant(.i32, 0), active, tiles_m, div_m, div_n, null); + const carried = .{ + initial_work.m, initial_work.n, initial_work.group, initial_work.valid, + b.constant(.i32, 0), b.constant(.i32, 0), b.constant(.i32, 0), tiled_mma.value(), + b.constant(.i32, 0), b.constant(.i32, 0), b.constant(.i32, 1), persistent_clusters, + initial_work.linear, b.blockIdx().x, b.blockIdx().y, b.constant(.i32, 0), + initial_work.executed, tiles_m, active, div_m, + div_n, + }; + var tiles = b.openWhile(carried, carried); + { + tiles.yieldBefore(tiles.before_carried[3], tiles.before_carried); + } + { + // Python peeks the first AB-full token before it blocks on an available + // accumulator stage. Preserve that ordering so the two independent + // pipelines enter the K loop with the same outstanding operations. + const first_state: PipelinePosition = .{ + .index = tiles.after_carried[5], + .phase = tiles.after_carried[6], + }; + const first_ready = b.mbarrierTryWait(barrierAt(b, storage.load_barriers, first_state.index), first_state.phase, .{ .scope = .cta }); + const accumulator_state: PipelinePosition = .{ + .index = tiles.after_carried[9], + .phase = tiles.after_carried[10], + }; + const accumulator_stage = accumulator_state.index; + const accumulator_empty = barrierAt(b, storage.accumulator_empty_barriers, accumulator_stage); + b.mbarrierWait(accumulator_empty, accumulator_state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); + + const persistent_mma: cute.Atom = .{ .inner = tiles.after_carried[7].inner, .kernel = b }; + const initial_mma = b.setMmaAccumulate(persistent_mma, b.constant(.i1, false)); + var stages = b.openFor( + 0, + k_tiles, + 1, + // Match PipelineTmaUmma's Python loop-carried ABI exactly. The + // software-pipeline pass recognizes the leading try-wait token and + // keeps the MMA atom immediately after it. + .{ first_ready, initial_mma.value(), b.constant(.i32, 0), first_state.index, first_state.phase }, + ); + { + const k = stages.carried[2]; + const state: PipelinePosition = .{ .index = stages.carried[3], .phase = stages.carried[4] }; + var pending = b.openIf(stages.carried[0].eq(false)); + { + b.mbarrierWait(barrierAt(b, storage.load_barriers, state.index), state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); + pending.close(); + } + const accumulator_tmem = accumulatorTmemAt(b, tile, tmem, accumulator_stage); + const mainloop = staged_mainloop.atStage(b, state.index, accumulator_tmem); + mainloop.copyScales(b); + const loop_mma: cute.Atom = .{ .inner = stages.carried[1].inner, .kernel = b }; + const mma = b.setMmaAccumulate(loop_mma, k.ne(0)); + b.gemm( + mma, + mainloop.accumulator, + .{ mainloop.a, mainloop.sfa }, + .{ mainloop.b, mainloop.sfb }, + mainloop.accumulator, + ); + var commit_lane = b.openIf(b.electSync()); + { + b.tcgen05Commit(barrierAt(b, storage.mma_barriers, state.index), .{}); + commit_lane.close(); + } + const next_count = stages.carried[2].add(1); + const next_state = advancePipeline(b, state.index, state.phase, tile.stages); + const next_ready = peekBarrierIfRemaining(b, storage.load_barriers, next_count, k_tiles, next_state); + stages.yield(.{ next_ready, mma.value(), next_count, next_state.index, next_state.phase }); + } + + var full_lane = b.openIf(b.electSync()); + { + b.tcgen05Commit(barrierAt(b, storage.accumulator_full_barriers, accumulator_stage), .{}); + full_lane.close(); + } + const next_accumulator_state = advancePipeline(b, accumulator_state.index, accumulator_state.phase, accumulator_stages); + const next_acc_count = tiles.after_carried[8].add(1); + const next_work = advancePersistentWork( + b, + tiles.after_carried[12], + tiles.after_carried[16], + tiles.after_carried[11], + tiles.after_carried[18], + tiles.after_carried[17], + tiles.after_carried[19], + tiles.after_carried[20], + null, + ); + tiles.yieldAfter(.{ + next_work.m, + next_work.n, + next_work.group, + next_work.valid, + stages.results[2], + stages.results[3], + stages.results[4], + stages.results[1], + next_acc_count, + next_accumulator_state.index, + next_accumulator_state.phase, + tiles.after_carried[11], + next_work.linear, + tiles.after_carried[13], + tiles.after_carried[14], + tiles.after_carried[15], + next_work.executed, + tiles.after_carried[17], + tiles.after_carried[18], + tiles.after_carried[19], + tiles.after_carried[20], + }); + } + + // Match PipelineUmmaAsync.producer_tail. The state points at the next + // accumulator slot; advance to the last slot used by this producer and + // wait until the epilogue has released it before the MMA warp exits. + var accumulator_tail: PipelinePosition = .{ + .index = tiles.results[9], + .phase = tiles.results[10], + }; + inline for (0..accumulator_stages - 1) |_| { + accumulator_tail = advancePipeline(b, accumulator_tail.index, accumulator_tail.phase, accumulator_stages); + } + b.mbarrierWait(barrierAt(b, storage.accumulator_empty_barriers, accumulator_tail.index), accumulator_tail.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); +} + +/// Epilogue operands; which fields are set depends on the `Epilogue`. +const EpilogueOperands = struct { + group_sizes: cute.View, + route_inverse: cute.View, + routing_weights: ?cute.View = null, + q: ?cute.View = null, + s: ?cute.View = null, + rows: ?cute.View = null, + direct: bool, + /// `Config.groupRows`. + group_rows: i64 = 0, + /// `Config.swiglu_limit`. + swiglu_limit: ?f32 = 10.0, +}; + +/// Warps 0-3 drain each TMEM accumulator stage into registers, release it to +/// the MMA warp, then write the tile out according to `epilogue`. +fn runEpilogue( + b: *B, + comptime tile: Tile, + comptime epilogue: Epilogue, + storage: StageStorage, + tmem_holding: V, + active: V, + tiles_m: V, + direct: ?DirectRoutes, + operands: EpilogueOperands, +) void { + const n = tile.n; + const tid = b.threadIdx().x; + const warp = b.makeWarpUniform(b.makeWarpUniform(tid.div(32))); + const tmem = b.retrieveTmemPtr(tmem_holding, .f32, 16); + + const div_m = b.fastDivmodCreate(tiles_m); + const div_n = b.fastDivmodCreate(1); + const initial_work = persistentWorkAt(b, b.blockIdx().z, b.constant(.i32, 0), active, tiles_m, div_m, div_n, direct); + const carried = .{ + initial_work.m, initial_work.n, initial_work.group, initial_work.valid, + b.constant(.i32, 0), b.constant(.i32, 0), b.constant(.i32, 0), initial_work.linear, + initial_work.executed, + }; + var tiles = b.openWhile(carried, .{ + initial_work.m, + initial_work.n, + initial_work.group, + .i32, + .i32, + .i32, + initial_work.linear, + initial_work.executed, + }); + { + tiles.yieldBefore(tiles.before_carried[3], .{ + tiles.before_carried[0], + tiles.before_carried[1], + tiles.before_carried[2], + tiles.before_carried[4], + tiles.before_carried[5], + tiles.before_carried[6], + tiles.before_carried[7], + tiles.before_carried[8], + }); + } + { + const tile_m = tiles.after_carried[0]; + const group = tiles.after_carried[2]; + const accumulator_state: PipelinePosition = .{ + .index = tiles.after_carried[4], + .phase = tiles.after_carried[5], + }; + const stage = accumulator_state.index; + b.mbarrierWait(barrierAt(b, storage.accumulator_full_barriers, stage), accumulator_state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); + + const accumulator_tmem = accumulatorTmemAt(b, tile, tmem, stage); + const warp_tmem = b.addOffset( + accumulator_tmem, + b.makeIntTupleDivBy(warp.mul(32 * 65536), 4), + b.ptrTy(.f32, .tmem, 16) catch @panic("bad epilogue TMEM pointer"), + ).value(); + // A warp lane holds one routed row, so tiles wider than a warp are drained + // 32 rows at a time. The accumulator stage is released once its last chunk + // is in registers. + const chunk_rows: usize = comptime @intCast(tile.chunkRows()); + const tile_chunks = comptime tile.chunks(); + inline for (0..tile_chunks) |chunk| { + const chunk_tmem = if (chunk == 0) warp_tmem else b.addOffset( + warp_tmem, + b.makeIntTuple(.{@as(i32, @intCast(chunk * chunk_rows))}), + b.ptrTy(.f32, .tmem, offsetAlignment(16, @intCast(chunk * chunk_rows), 4)) catch @panic("bad epilogue TMEM pointer"), + ).value(); + // Route metadata does not depend on the accumulator. + const routed = routedLanes(b, tile, operands, group, chunk); + const values = b.tmemLoadVector(chunk_tmem, @intCast(chunk_rows), .{ .num_dp = 32, .num_b = 32, .num_rep = @intCast(chunk_rows) }); + b.fenceTmemLoad(); + if (chunk + 1 == tile_chunks) { + // Registers now own the accumulator values, so all 128 epilogue + // threads release this TMEM stage before the stores. + _ = b.mbarrierArrive(barrierAt(b, storage.accumulator_empty_barriers, stage), .{}); + } + var columns: [@intCast(@min(n, 32))]V = undefined; + inline for (&columns, 0..) |*column, i| column.* = b.vectorExtract(values, i, .i32).bitCast(.f32); + switch (epilogue) { + .swiglu_mxfp8 => storeTileSwiglu(b, tile, storage, operands, routed, tile_m, group, &columns, chunk), + .route_rows => storeTileRows(b, tile, operands, routed, tile_m, &columns, chunk), + } + } + const next_accumulator_state = advancePipeline(b, accumulator_state.index, accumulator_state.phase, accumulator_stages); + const next_work = advancePersistentWork(b, tiles.after_carried[6], tiles.after_carried[7], b.gridDim().z, active, tiles_m, div_m, div_n, direct); + tiles.yieldAfter(.{ + next_work.m, + next_work.n, + next_work.group, + next_work.valid, + tiles.after_carried[3].add(1), + next_accumulator_state.index, + next_accumulator_state.phase, + next_work.linear, + next_work.executed, + }); + } + + var release_warp = b.openIf(warp.eq(0)); + { + // Permit release and deallocation are warp collective as well. + b.relinquishTmemAllocPermit(.{}); + release_warp.close(); + } + b.barrierCtaSyncAligned(1, 128); + var dealloc_warp = b.openIf(warp.eq(0)); + { + b.deallocTmem(tmem, tile.tmem_columns, .{}); + dealloc_warp.close(); + } +} + +/// E8M0 exponent of the power-of-two MX scale for a block maximum, rounded +/// up so that `maximum / scale <= 448`, and the matching inverse scale. +fn mxScale(b: *B, maximum: V) struct { exponent: V, inverse: V } { + const raw_bits = maximum.maximum(1e-4).mul(1.0 / 448.0).bitCast(.i32); + const exponent = raw_bits.shrLogical(23).add(raw_bits.bitAnd(0x7fffff).ne(0).to(.i32)); + return .{ .exponent = exponent, .inverse = b.constant(.i32, 254).sub(exponent).shl(23).bitCast(.f32) }; +} + +/// Per-row routing data of one expert group, spread across a warp: lane `l` +/// holds routed row `l` of the group. Columns index it with `shuffleIdx`. +const RoutedLanes = struct { + count: V, + /// Token/top-k route of the lane's row; 0 for padding rows. + route: V, + /// Routing weight of the lane's row (up projection only). + weight: ?V, +}; + +fn routedLanes(b: *B, comptime tile: Tile, operands: EpilogueOperands, group: V, comptime chunk: usize) RoutedLanes { + // Only the up projection applies routing weights. + const weights = operands.routing_weights; + if (operands.direct) return .{ + .count = b.constant(.i32, 1), + .route = group, + .weight = if (weights) |w| w.get(.{group}, .f32) else null, + }; + const lane = b.threadIdx().x.rem(32); + const count = operands.group_sizes.get(.{group}, .i32); + // Lanes past the tile width alias row 0, which is always live. Wide tiles + // run one 32-row chunk at a time, so lane `l` holds row `32 * chunk + l`. + const chunk_rows = comptime tile.chunkRows(); + const in_chunk = b.select(lane.lt(chunk_rows), lane, b.constant(.i32, 0)); + const row_in_group = if (chunk == 0) in_chunk else in_chunk.add(@as(i64, @intCast(chunk)) * chunk_rows); + const live = row_in_group.lt(count); + const inverse = operands.route_inverse.get(.{group.mul(tile.n).add(row_in_group)}, .i32); + const route = b.select(live, inverse, b.constant(.i32, 0)); + const weight = if (weights) |w| w.get(.{route}, .f32) else null; + return .{ .count = count, .route = route, .weight = weight }; +} + +/// Thread `t` owns output row `128 * tile_m + t` of the interleave-8 +/// projection. Gate and up lanes are paired by XOR 8. Each gate thread +/// computes one SwiGLU activation per routed row; 32 consecutive activations +/// span two warps, whose partial maxima meet in shared memory. +/// +/// At decode sizes most rows of a group are padding, whose outputs are never +/// read. `count` is uniform across the CTA, so whole 8-row chunks of padding +/// are skipped; per-row branches would serialize the shuffle chains of +/// independent rows. Do not duplicate this body under a full-group branch: +/// ptxas then loses track of convergence at the tile loop's warp-uniform +/// shuffles, and lanes still outside the gate-lane stores read garbage. +fn storeTileSwiglu(b: *B, comptime tile: Tile, storage: StageStorage, operands: EpilogueOperands, routed: RoutedLanes, tile_m: V, group: V, columns: []const V, comptime chunk: usize) void { + // Direct routing has exactly one live row per group. + if (operands.direct) + swigluRows(b, tile, storage, operands, routed, tile_m, group, columns, 1, false, chunk) + else + swigluRows(b, tile, storage, operands, routed, tile_m, group, columns, comptime @intCast(tile.chunkRows()), true, chunk); +} + +const swiglu_chunk = 8; + +/// The first `live` rows of `storeTileSwiglu`; with `skip_padding`, chunks +/// of rows past `count` are skipped. +fn swigluRows(b: *B, comptime tile: Tile, storage: StageStorage, operands: EpilogueOperands, routed: RoutedLanes, tile_m: V, group: V, columns: []const V, comptime live: usize, comptime skip_padding: bool, comptime chunk: usize) void { + const n = comptime tile.chunkRows(); + // First routed row of this epilogue pass. + const chunk_first: i32 = comptime @intCast(chunk * @as(usize, @intCast(tile.chunkRows()))); + const tid = b.threadIdx().x; + const lane = tid.rem(32); + const warp = b.makeWarpUniform(b.makeWarpUniform(tid.div(32))); + const is_gate = tid.rem(16).lt(8); + const first_row = group.mul(operands.group_rows); + const f32_type = cute.DType.f32.toMlir(b.ctx); + // The TMA C staging buffer is unused by this epilogue. + const scratch_layout = b.layoutSpec(.{ 4, n }, .{ n, 1 }); + const scratch = b.makeView(storage.c, b.staticLayout(scratch_layout), b.memrefType(.f32, .smem, 1024, b.layoutType(scratch_layout))); + const row_chunks = std.math.divCeil(usize, live, swiglu_chunk) catch unreachable; + + var activations: [@intCast(n)]V = undefined; + // Lane `c` collects the warp maximum of column `c`. + var lane_maximum = b.constant(.f32, 0); + for (0..row_chunks) |row_chunk| { + const first = row_chunk * swiglu_chunk; + const last = @min(first + swiglu_chunk, live); + // Row 0 of an active group is always live. + const guarded = skip_padding and (row_chunk > 0 or chunk > 0); + const types: [swiglu_chunk + 1]*const mlir.Type = @splat(f32_type); + // `@TypeOf` does not evaluate its operand. + var guard: ?@TypeOf(b.openIfElse(routed.count, types)) = if (guarded) b.openIfElse(routed.count.gt(chunk_first + @as(i32, @intCast(first))), types) else null; + var chunk_maximum = lane_maximum; + for (columns[first..last], activations[first..last], first..) |value, *activation, column| { + const c: i32 = @intCast(column); + // The reference model rounds the projection to BF16. + var gate = value.to(.bf16).to(.f32); + if (operands.swiglu_limit) |limit| { + gate = gate.minimum(limit); + } + var up_value = b.shuffleXor(value, 8).to(.bf16).to(.f32); + if (operands.swiglu_limit) |limit| { + up_value = up_value.maximum(-limit).minimum(limit); + } + // up * SiLU(gate), with the hardware approximations Python's + // fastmath selects: exp(-g) = 2^(-g * log2 e). + const exp_neg = b.unaryF32("ex2.approx.ftz.f32", gate.mul(-std.math.log2e)); + var result = up_value.mul(gate).mul(b.unaryF32("rcp.approx.ftz.f32", exp_neg.add(1.0))); + result = result.mul(b.shuffleIdx(routed.weight.?, c)).to(.bf16).to(.f32); + // Up lanes hold no activation and must not raise the maxima. + activation.* = b.select(is_gate, result, b.constant(.f32, 0)); + const maximum = b.warpMaxNonNegative(activation.*.abs()); + chunk_maximum = b.select(lane.eq(c), maximum, chunk_maximum); + } + if (guard) |*g| { + var then_values: [swiglu_chunk + 1]V = @splat(chunk_maximum); + for (activations[first..last], then_values[0 .. last - first]) |activation, *t| t.* = activation; + g.yieldThen(then_values); + // Padding chunk: zero activations, maxima unchanged. The zero is + // emitted in the else block, where it is used. + var else_values: [swiglu_chunk + 1]V = @splat(b.constant(.f32, 0)); + else_values[swiglu_chunk] = lane_maximum; + g.yieldElse(else_values); + for (activations[first..last], g.results[0 .. last - first]) |*activation, result| activation.* = result; + lane_maximum = g.results[swiglu_chunk]; + } else { + lane_maximum = chunk_maximum; + } + } + var owner = b.openIf(lane.lt(n)); + { + scratch.set(.{ warp, lane }, lane_maximum); + owner.close(); + } + b.barrierCtaSyncAligned(1, 128); + + // Lane `c` turns the two half-block maxima of column `c` into its scale. + const pair = warp.bitAnd(~@as(i32, 1)); + const lane_column = b.select(lane.lt(n), lane, b.constant(.i32, 0)); + const scale = mxScale(b, scratch.get(.{ pair, lane_column }, .f32).maximum(scratch.get(.{ pair.add(1), lane_column }, .f32))); + // The next tile reuses the scratch maxima. + b.barrierCtaSyncAligned(1, 128); + + const activation_row = tid.div(16).mul(8).add(tid.rem(8)); + const activation_index = tile_m.mul(64).add(activation_row); + for (0..row_chunks) |row_chunk| { + const first = row_chunk * swiglu_chunk; + const last = @min(first + swiglu_chunk, live); + var guard: ?@TypeOf(b.openIf(routed.count)) = if (skip_padding and (chunk > 0 or first > 0)) b.openIf(routed.count.gt(chunk_first + @as(i32, @intCast(first)))) else null; + var inverses: [swiglu_chunk]V = undefined; + for (inverses[0 .. last - first], first..) |*inverse, column| inverse.* = b.shuffleIdx(scale.inverse, @as(i32, @intCast(column))); + var gate_lane = b.openIf(is_gate); + { + for (activations[first..last], inverses[0 .. last - first], first..) |activation, inverse, column| { + const quantized = activation.mul(inverse).maximum(-448.0).minimum(448.0).to(.f8e4m3fn); + operands.q.?.set(.{ first_row.add(chunk_first + @as(i32, @intCast(column))), activation_index }, quantized); + } + gate_lane.close(); + } + if (guard) |*g| g.close(); + } + + // Warps 0 and 2 each own one 32-activation block of this tile. + var scale_lane = b.openIf(warp.rem(2).eq(0).bitAnd(lane.add(chunk_first).lt(routed.count))); + { + const block = tile_m.mul(2).add(warp.div(2)); + const swizzled = block.div(4).mul(512).add(lane.mul(16)).add(chunk * 4).add(block.rem(4)); + operands.s.?.set(.{ group, swizzled }, scale.exponent.to(.i8)); + scale_lane.close(); + } +} + +/// Python `down_epilogue`: store each live routed row at its token/top-k +/// route, so the final reduction reads rows in route order. +fn storeTileRows(b: *B, comptime tile: Tile, operands: EpilogueOperands, routed: RoutedLanes, tile_m: V, columns: []const V, comptime chunk: usize) void { + const chunk_first: i32 = comptime @intCast(chunk * @as(usize, @intCast(tile.chunkRows()))); + const output_column = tile_m.mul(128).add(b.threadIdx().x); + if (operands.direct) { + operands.rows.?.set(.{ routed.route, output_column }, columns[0].to(.bf16)); + return; + } + var routes: [32]V = undefined; + for (routes[0..columns.len], 0..) |*route, column| route.* = b.shuffleIdx(routed.route, @as(i32, @intCast(column))); + for (columns, routes[0..columns.len], 0..) |value, route, column| { + var live: ?@TypeOf(b.openIf(routed.count)) = if (column == 0 and chunk == 0) null else b.openIf(routed.count.gt(chunk_first + @as(i32, @intCast(column)))); + operands.rows.?.set(.{ route, output_column }, value.to(.bf16)); + if (live) |*l| l.close(); + } +} + +fn buildUmmaDevice(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: Epilogue, d: anytype) void { + const L = Layouts(tile); + const n = tile.n; + const stages = tile.stages; + // Allocate the complete Python SharedStorage object as one compiler-visible + // memref. `cute.kernel_smem_size` only accounts for memref allocations; + // treating every field as an architecture pointer left the launch at zero + // dynamic shared bytes and made the first TMA instruction fault. + const shared = b.allocSmemStorage( + .i8, + 1024, + b.layoutType(b.layoutSpec(.{tile.smem_bytes}, .{1})), + 1, + tile.fields(b), + ); + const shared_base = b.getIter( + shared, + b.ptrTy(.i8, .smem, 1024) catch @panic("bad shared base pointer"), + ).value(); + + const tmem_holding = sharedPointerAt(b, shared_base, tile.tmem_holding, .i32, 8); + const storage: StageStorage = .{ + .c = sharedPointerAt(b, shared_base, tile.c, .f32, 1024), + .a = sharedPointerAt(b, shared_base, tile.a, .i8, 1024), + .b = sharedPointerAt(b, shared_base, tile.b, .f8e4m3fn, 1024), + .sfa = sharedPointerAt(b, shared_base, tile.sfa, .f8e8m0fnu, 1024), + .sfb = sharedPointerAt(b, shared_base, tile.sfb, .f8e8m0fnu, 1024), + .load_barriers = sharedPointerAt(b, shared_base, 0, .i64, 1024), + .mma_barriers = sharedPointerAt(b, shared_base, tile.empty_barriers, .i64, 8), + .accumulator_full_barriers = sharedPointerAt(b, shared_base, tile.accumulator_full, .i64, 16), + .accumulator_empty_barriers = sharedPointerAt(b, shared_base, tile.accumulator_empty, .i64, 16), + }; + + const tid = b.threadIdx().x; + const warp = b.makeWarpUniform(b.makeWarpUniform(tid.div(32))); + const tiled_mma: cute.Atom = .{ .inner = d.mma.inner, .kernel = b }; + + // Reproduce the Python CuTe path for the two matrix operands: + // coordinate tensor -> local_tile -> tiled_mma.partition -> group modes + // -> tma_partition. All dependent result types come from Zig layout + // objects; this kernel does not embed textual MLIR. + const m_tiles = @divExact(cfg.m, 128); + const k_tiles = @divExact(cfg.k, 128); + const a_local_spec = b.layoutSpec( + .{ 128, 128, m_tiles, k_tiles, cfg.experts }, + .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}), b.basis(128, 1, .{1}), b.basis(128, 1, .{0}), b.basis(1, 1, .{2}) }, + ); + // The group tile covers all of B's rows, so the N-tile mode has extent one + // and its stride is inferred as zero. Direct routing is no different: the + // tensor map still spans a whole tile of rows, zero-filling all but one. + const b_local_spec = b.layoutSpec( + .{ n, 128, 1, k_tiles, cfg.groups }, + .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}), 0, b.basis(128, 1, .{0}), b.basis(1, 1, .{2}) }, + ); + const all_tiles = .{ cute.AlgebraToken.all, cute.AlgebraToken.all, cute.AlgebraToken.all }; + const tiled_a = b.localTile(d.gA, b.makeTile(&.{ 128, 128 }), all_tiles, b.coordTensorType(3, a_local_spec), null); + const tiled_b = b.localTile(d.gB, b.makeTile(&.{ n, 128 }), all_tiles, b.coordTensorType(3, b_local_spec), null); + + const a_mma_spec = b.layoutSpec( + .{ .{ 128, 32 }, 1, 4, m_tiles, k_tiles, cfg.experts }, + .{ .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}) }, 0, b.basis(32, 1, .{0}), b.basis(128, 1, .{1}), b.basis(128, 1, .{0}), b.basis(1, 1, .{2}) }, + ); + const b_mma_spec = b.layoutSpec( + .{ .{ n, 32 }, 1, 4, 1, k_tiles, cfg.groups }, + .{ .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}) }, 0, b.basis(32, 1, .{0}), 0, b.basis(128, 1, .{0}), b.basis(1, 1, .{2}) }, + ); + const mma_a = b.tiledMmaPartition(tiled_mma, tiled_a, .{0}, 0, b.coordTensorType(3, a_mma_spec)); + const mma_b = b.tiledMmaPartition(tiled_mma, tiled_b, .{0}, 1, b.coordTensorType(3, b_mma_spec)); + + const a_grouped_layout = b.staticLayout(b.layoutSpec( + .{ .{ .{ 128, 32 }, 1, 4 }, m_tiles, k_tiles, cfg.experts }, + .{ .{ .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}) }, 0, b.basis(32, 1, .{0}) }, b.basis(128, 1, .{1}), b.basis(128, 1, .{0}), b.basis(1, 1, .{2}) }, + )); + const b_grouped_layout = b.staticLayout(b.layoutSpec( + .{ .{ .{ n, 32 }, 1, 4 }, 1, k_tiles, cfg.groups }, + .{ .{ .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}) }, 0, b.basis(32, 1, .{0}) }, b.basis(n, 1, .{1}), b.basis(128, 1, .{0}), b.basis(1, 1, .{2}) }, + )); + const staged = stagedOperands(b, tile, storage); + const smem_a_grouped = b.withLayout(staged.a, b.staticLayout(b.layoutSpec( + .{ .{ .{ 128, 32 }, 1, 4 }, stages }, + .{ .{ .{ 128, 1 }, 0, 32 }, Tile.a_stage_bytes }, + ))); + const smem_b_grouped = b.withLayout(staged.b, b.staticLayout(b.layoutSpec( + .{ .{ .{ n, 32 }, 1, 4 }, stages }, + .{ .{ .{ 128, 1 }, 0, 32 }, tile.bStageBytes() }, + ))); + const one_cta = b.staticLayout(b.layoutSpec(.{1}, .{0})); + const a_partition_layout = b.layoutSpec(.{ .{ Tile.a_stage_bytes, 1 }, stages }, .{ .{ 1, 0 }, Tile.a_stage_bytes }); + const b_partition_layout = b.layoutSpec(.{ .{ tile.bStageBytes(), 1 }, stages }, .{ .{ 1, 0 }, tile.bStageBytes() }); + const a_target_layout = b.layoutSpec( + .{ .{ .{ 128, 128 }, 1 }, m_tiles, k_tiles, cfg.experts }, + .{ .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0 }, b.basis(128, 1, .{1}), b.basis(128, 1, .{0}), b.basis(1, 1, .{2}) }, + ); + const b_target_layout = b.layoutSpec( + .{ .{ .{ 128, n }, 1 }, 1, k_tiles, cfg.groups }, + .{ .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0 }, b.basis(n, 1, .{1}), b.basis(128, 1, .{0}), b.basis(1, 1, .{2}) }, + ); + const partitioned_a = b.tmaPartition( + .{ .inner = d.tma_a.inner, .kernel = b }, + .{0}, + one_cta, + smem_a_grouped, + .{b.withLayout(mma_a, a_grouped_layout)}, + b.memrefTypeFromPointer(b.swizzledPtrTy(.i8, .smem, 1024, L.ab_swizzle) catch @panic("bad smem pointer"), b.layoutType(a_partition_layout)), + &.{b.coordTensorType(3, a_target_layout)}, + ); + const partitioned_b = b.tmaPartition( + .{ .inner = d.tma_b.inner, .kernel = b }, + .{0}, + one_cta, + smem_b_grouped, + .{b.withLayout(mma_b, b_grouped_layout)}, + b.memrefTypeFromPointer(b.swizzledPtrTy(.f8e4m3fn, .smem, 1024, L.ab_swizzle) catch @panic("bad smem pointer"), b.layoutType(b_partition_layout)), + &.{b.coordTensorType(3, b_target_layout)}, + ); + + const exec_a = b.makeExecTma(.{ .inner = d.tma_a.inner, .kernel = b }, b.tmaLoadExecAtomType(L.tmaA(b, cfg))); + const exec_b = b.makeExecTma(.{ .inner = d.tma_b.inner, .kernel = b }, b.tmaLoadExecAtomType(L.tmaB(b, cfg))); + const exec_sfa = b.makeExecTma(.{ .inner = d.tma_sfa.inner, .kernel = b }, b.tmaLoadExecAtomType(L.tmaSf(b))); + const exec_sfb = b.makeExecTma(.{ .inner = d.tma_sfb.inner, .kernel = b }, b.tmaLoadExecAtomType(L.tmaSf(b))); + + const expert_ids: cute.View = .{ .inner = d.expert_ids.inner, .kernel = b }; + const active_count: cute.View = .{ .inner = d.active_count.inner, .kernel = b }; + const direct: ?DirectRoutes = if (cfg.direct) .{ .expert_ids = expert_ids, .routes = @intCast(cfg.groups) } else null; + const active = if (direct) |routes| routes.count(b) else active_count.get(.{0}, .i32); + + // Match CUTLASS PipelineTmaUmma/PipelineUmmaAsync initialization. Warp 0 + // owns initialization, and one elected lane initializes each complete + // barrier array. mbarrier.init is not warp collective: issuing it from + // every lane races on the same shared state and breaks the first phase + // transition when the A/B ring wraps. + var init_warp = b.openIf(warp.eq(0)); + { + const barrier_arrays = .{ + .{ storage.load_barriers, stages, 1 }, + .{ storage.mma_barriers, stages, 1 }, + .{ storage.accumulator_full_barriers, accumulator_stages, 1 }, + .{ storage.accumulator_empty_barriers, accumulator_stages, 128 }, + }; + inline for (barrier_arrays) |array| { + var lane = b.openIf(b.electSync()); + { + inline for (0..array[1]) |stage| { + b.mbarrierInit(barrierAt(b, array[0], b.constant(.i32, stage)), array[2]); + } + lane.close(); + } + } + b.fenceMbarrierInit(); + init_warp.close(); + } + b.syncThreads(); + + // Each role walks the same persistent tile sequence independently. The + // AB and accumulator mbarriers carry all cross-warp dependencies. + var tma_warp = b.openIf(warp.eq(5)); + { + switch (k_tiles) { + inline 16, 18, 32, 40 => |kt| runTmaProducer(b, tile, kt, storage, active, d.tiles_m, expert_ids, direct, partitioned_a.targets[0], partitioned_b.targets[0], exec_a, exec_b, exec_sfa, exec_sfb), + else => std.debug.panic("unsupported MXFP4 reduction tile count {d}", .{k_tiles}), + } + tma_warp.close(); + } + + var mma_warp = b.openIf(warp.eq(4)); + { + // Python's MMA role reaches its own TmemAllocator.wait_for_alloc site. + b.barrierCtaSyncAligned(2, 160); + switch (k_tiles) { + inline 16, 18, 32, 40 => |kt| runMmaConsumer(b, tile, kt, storage, tiled_mma, tmem_holding, active, d.tiles_m), + else => unreachable, + } + mma_warp.close(); + } + + var epilogue_warps = b.openIf(warp.lt(4)); + { + // TmemAllocator.allocate is issued collectively by warp 0. The four + // epilogue warps then meet the MMA warp at named barrier 2. + var alloc_warp = b.openIf(warp.eq(0)); + { + b.allocTmem(tile.tmem_columns, tmem_holding, .{}); + alloc_warp.close(); + } + b.barrierCtaSyncAligned(2, 160); + switch (epilogue) { + .swiglu_mxfp8 => runEpilogue(b, tile, epilogue, storage, tmem_holding, active, d.tiles_m, direct, .{ + .group_sizes = view(b, d.group_sizes), + .route_inverse = view(b, d.route_inverse), + .routing_weights = view(b, d.routing_weights), + .q = view(b, d.q), + .s = view(b, d.s), + .direct = cfg.direct, + .group_rows = cfg.groupRows(), + .swiglu_limit = cfg.swiglu_limit, + }), + .route_rows => runEpilogue(b, tile, epilogue, storage, tmem_holding, active, d.tiles_m, direct, .{ + .group_sizes = view(b, d.group_sizes), + .route_inverse = view(b, d.route_inverse), + .rows = view(b, d.rows), + .direct = cfg.direct, + }), + } + epilogue_warps.close(); + } +} + +fn buildHostLaunch(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: Epilogue) cute.FinishError!void { + const L = Layouts(tile); + const host_arg = HostArgs(epilogue); + const n = tile.n; + const a_layout = b.layoutSpec(.{ cfg.m, cfg.k, cfg.experts }, .{ cfg.k, 1, cfg.m * cfg.k }); + const b_layout = b.layoutSpec(.{ cfg.groupRows(), cfg.k, cfg.groups }, .{ cfg.k, 1, cfg.k * cfg.groupRows() }); + // `cute.tile_to_shape` derives that the dynamic strides step whole + // 512-byte scale atoms, and the result type has to carry it. + const sf_stride: cute.ConstrainedDynamic = .{ .divisible_by = Tile.sf_stage_bytes }; + const dynamic_sf_layout = b.layoutSpec( + .{ .{ .{ 32, 4 }, cute.AlgebraToken.dynamic }, .{ .{ 32, 4 }, cute.AlgebraToken.dynamic }, .{ 1, cute.AlgebraToken.dynamic } }, + .{ .{ .{ 16, 4 }, sf_stride }, .{ .{ 0, 1 }, 512 }, .{ 0, sf_stride } }, + ); + + const gA = b.makeTensorView(b.recastPointer(b.arg(host_arg.weight), .f4e2m1fn, .gmem, 16), b.staticLayout(a_layout)); + // `schedule` is `[group experts | group sizes | active count]`. + const schedule = b.arg(host_arg.schedule); + const ids = b.makeTensorView(schedule, b.staticLayout(b.layoutSpec(cfg.groups, 1))); + const active_ptr = b.addOffset( + schedule, + b.makeIntTuple(.{2 * cfg.groups}), + b.ptrTy(.i32, .gmem, offsetAlignment(16, 2 * cfg.groups, 4)) catch @panic("bad active count pointer"), + ); + const active = b.makeTensorView(active_ptr, b.staticLayout(b.layoutSpec(1, 1))); + const gB = b.makeTensorView(b.arg(host_arg.input_quant), b.staticLayout(b_layout)); + // Match Python's host construction instead of materializing these two + // layouts with `cute.static`. In particular, SFB's M extent is one; CuTe + // must still retain that mode while encoding the tensor map. + const sf_atom_layout = b.staticLayout(b.layoutSpec( + .{ .{ 32, 4 }, .{ 32, 4 } }, + .{ .{ 16, 4 }, .{ 0, 1 } }, + )); + // The Zig dialect binding exposes the zero-based form of Python CuTe's + // `(2, 1, 3)` tile-to-shape order. + const sf_order = b.makeIntTuple(.{ 1, 0, 2 }); + const sfa_shape = b.makeShapeValues(.{ b.arg(host_arg.problem_m), b.arg(host_arg.problem_k), b.arg(host_arg.expert_count) }, b.shapeType(.{ cute.AlgebraToken.dynamic, cute.AlgebraToken.dynamic, cute.AlgebraToken.dynamic })); + const sfb_shape = b.makeShapeValues(.{ b.arg(host_arg.problem_n), b.arg(host_arg.problem_k), b.arg(host_arg.problem_l) }, b.shapeType(.{ cute.AlgebraToken.dynamic, cute.AlgebraToken.dynamic, cute.AlgebraToken.dynamic })); + // Weight scales step M atoms first; activation scales retain K-major order. + const sfa_order = b.makeIntTuple(.{ 0, 1, 2 }); + const sfa_layout = b.layoutSpec( + .{ .{ .{ 32, 4 }, cute.AlgebraToken.dynamic }, .{ .{ 32, 4 }, cute.AlgebraToken.dynamic }, .{ 1, cute.AlgebraToken.dynamic } }, + .{ .{ .{ 16, 4 }, 512 }, .{ .{ 0, 1 }, sf_stride }, .{ 0, sf_stride } }, + ); + const dynamic_sfa_layout = b.tileToShape(sf_atom_layout, sfa_shape, sfa_order, b.layoutType(sfa_layout)); + const dynamic_sfb_layout = b.tileToShape(sf_atom_layout, sfb_shape, sf_order, b.layoutType(dynamic_sf_layout)); + const gSFA = b.makeTensorView(b.recastPointer(b.arg(host_arg.weight_scale), .f8e8m0fnu, .gmem, 16), dynamic_sfa_layout); + const gSFB = b.makeTensorView(b.recastPointer(b.arg(host_arg.input_scale), .f8e8m0fnu, .gmem, 16), dynamic_sfb_layout); + + const zero_tmem = b.intToPtr(0, .f8e8m0fnu, .tmem, 1); + const mma = b.makeBlockScaledMma(.{ .n = @intCast(n) }, zero_tmem); + const mma_sfb = b.makeBlockScaledMma(.{ .n = 128 }, zero_tmem); + + const a_smem = b.makeComposedLayout(L.ab_swizzle, 0, L.aSmem(b, false)); + const b_smem = b.makeComposedLayout(L.ab_swizzle, 0, L.bSmem(b, false)); + const sf_smem = b.staticLayout(L.sfSmem(b, false)); + const a_map = b.staticLayout(b.layoutSpec(.{ .{ 128, 32 }, 1, 4 }, .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0, b.basis(32, 1, .{1}) })); + const b_map = b.staticLayout(b.layoutSpec(.{ .{ n, 32 }, 1, 4 }, .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0, b.basis(32, 1, .{1}) })); + const sf_map = b.staticLayout(b.layoutSpec( + .{ .{ .{ 32, 4 }, 32 }, 1, 4 }, + .{ .{ .{ b.basis(1, 1, .{ 0, 0, 0 }), b.basis(1, 1, .{ 0, 0, 1 }) }, b.basis(1, 1, .{ 1, 0, 0 }) }, 0, b.basis(1, 1, .{ 1, 0, 1 }) }, + )); + + const tma_a = b.makeTiledTmaLoadAtom(gA.value(), a_smem.value(), a_map.value(), L.tmaA(b, cfg)); + const tma_b = b.makeTiledTmaLoadAtom(gB.value(), b_smem.value(), b_map.value(), L.tmaB(b, cfg)); + const tma_sfa = b.makeTiledTmaLoadAtom(gSFA.value(), sf_smem.value(), sf_map.value(), L.tmaSf(b)); + const tma_sfb = b.makeTiledTmaLoadAtom(gSFB.value(), sf_smem.value(), sf_map.value(), L.tmaSf(b)); + + const one = b.constant(.i32, 1); + const tiles_m = b.constant(.i32, @divExact(cfg.m, 128)); + const grid_z = b.constant(.i32, @min(@divExact(cfg.m, 128) * cfg.groups, cfg.persistent_ctas)); + const config = b.makeLaunchConfig(.{ + .grid = .{ one, one, grid_z }, + .block = .{ b.constant(.i32, 192), one, one }, + .dynamic_smem = b.kernelSmemSize(deviceName(epilogue)), + .stream = b.cudaStream(), + .cluster = .{ one, one, one }, + .use_pdl = true, + }); + const mainloop = .{ + ids, active, mma.tiled, mma_sfb.tiled, tma_a.atom, tma_a.tensor, tma_b.atom, tma_b.tensor, + tma_sfa.atom, tma_sfa.tensor, tma_sfb.atom, tma_sfb.tensor, + }; + const activations = @divExact(cfg.m, 2); + const group_sizes = b.makeTensorView( + b.addOffset(schedule, b.makeIntTuple(.{cfg.groups}), b.ptrTy(.i32, .gmem, offsetAlignment(16, cfg.groups, 4)) catch @panic("bad group size pointer")), + b.staticLayout(b.layoutSpec(cfg.groups, 1)), + ); + const launch = switch (epilogue) { + .swiglu_mxfp8 => b.launchEx(deviceName(epilogue), config, mainloop ++ .{ + group_sizes, + b.makeTensorView(b.arg(host_arg.route_inverse), b.staticLayout(b.layoutSpec(cfg.routedRows(), 1))), + b.makeTensorView(b.arg(host_arg.routing_weights), b.staticLayout(b.layoutSpec(@max(cfg.routes, 1), 1))), + b.makeTensorView(b.arg(host_arg.output), b.staticLayout(b.layoutSpec(.{ cfg.routedRows(), activations }, .{ activations, 1 }))), + b.makeTensorView(b.arg(host_arg.scales), b.staticLayout(b.layoutSpec(.{ cfg.groups, @divExact(activations, 32) * 128 }, .{ @divExact(activations, 32) * 128, 1 }))), + tiles_m, + }), + .route_rows => b.launchEx(deviceName(epilogue), config, mainloop ++ .{ + group_sizes, + b.makeTensorView(b.arg(host_arg.route_inverse), b.staticLayout(b.layoutSpec(cfg.routedRows(), 1))), + b.makeTensorView(b.arg(host_arg.output), b.staticLayout(b.layoutSpec(.{ @max(cfg.routes, 1), cfg.m }, .{ cfg.m, 1 }))), + tiles_m, + }), + }; + b.returnHostStatus(b.cudaResultStatus(launch)); +} + +test "MXFP4 GEMM programs emit their host launch for every tile width" { + inline for (.{ 8, 16, 32 }) |n| { + inline for (.{ .{ UpQuantized, Epilogue.swiglu_mxfp8, 4608, 5120 }, .{ DownRows, Epilogue.route_rows, 5120, 2304 } }) |program| { + const cfg: Config = .{ + .experts = 192, + .m = program[2], + .n = n, + .k = program[3], + .groups = 227, + .epilogue = program[1], + .routes = 576, + .direct = n == 8, + }; + const ir = try program[0].emit(std.testing.allocator, cfg); + defer std.testing.allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, deviceName(program[1])) != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "cuda.launch_ex") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "make_non_exec_tiled_tma_load") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, std.fmt.comptimePrint("sm100.mma_bs<128x{d}x32", .{n})) != null); + } + } +} diff --git a/zml/kernels/moe/cute/sm10x/persistent_nvfp4.zig b/zml/kernels/moe/cute/sm10x/persistent_nvfp4.zig new file mode 100644 index 00000000..0de45007 --- /dev/null +++ b/zml/kernels/moe/cute/sm10x/persistent_nvfp4.zig @@ -0,0 +1,1679 @@ +//! Source-level SM100 NVFP4 grouped GEMM. +//! +//! The NVFP4 counterpart of `persistent_mxfp4.zig`, following the CuTe DSL +//! `Sm100BlockScaledPersistentDenseGemmKernel` with FP4 x FP4 operands, E4M3 +//! scale factors every 16 elements and the `kind::mxf4nvf4.block16` MMA. Both +//! operands are packed FP4 in global and shared memory (TMA format `U4`); one +//! pipeline stage holds a 256-deep K slice, consumed by four K = 64 MMAs. +//! +//! One CTA computes a 128xN output tile of one expert group: warp 5 drives +//! the TMA ring, warp 4 issues the S2T scale copies and UMMA, and warps 0-3 +//! drain the accumulator from tensor memory into one of the two epilogues. +//! The pipeline, scheduler and warp roles are shared with the MXFP4 kernel. +const std = @import("std"); +const mlir = @import("mlir"); +const nvvm = @import("mlir/dialects/nvvm"); + +const zml = @import("../../../../zml.zig"); +const cute = zml.kernel.cute; +const B = cute.Builder; +const V = cute.Value; +const common = @import("common.zig"); +const nvfp4 = @import("nvfp4.zig"); + +const accumulator_stages = 2; +/// K elements of one pipeline stage: four K = 64 FP4 MMAs. +pub const k_tile = 256; +/// Largest dynamic shared-memory allocation of an SM100 CTA. +const smem_capacity = 232448; +const mma_kind: cute.BlockScaledMmaConfig.Kind = .nvfp4_nvfp4; + +/// Shared-memory plan for one tile width, laid out like the Python +/// `SharedStorage`: barriers, epilogue scratch, then the A, B, SFA and SFB +/// rings. The ring is as deep as the capacity allows. +pub const Tile = struct { + n: i64, + stages: usize, + smem_bytes: i64, + tmem_columns: u16, + empty_barriers: i64, + accumulator_full: i64, + accumulator_empty: i64, + tmem_dealloc: i64, + tmem_holding: i64, + c: i64, + a: i64, + b: i64, + sfa: i64, + sfb: i64, + + /// 128x256 packed FP4. + const a_stage_bytes = 128 * k_tile / 2; + /// One 128x16 E4M3 scale tile: four 512-byte 128x4 atoms. + const sf_stage_bytes = 128 * k_tile / 16; + const c_bytes = 512; + const header_bytes = 1024; + + pub fn of(comptime n: i64) Tile { + comptime { + std.debug.assert(n >= 8 and n <= 128 and @rem(n, 8) == 0); + const per_stage = a_stage_bytes + @divExact(n * k_tile, 2) + 2 * sf_stage_bytes; + const stages: usize = @intCast(@divTrunc(smem_capacity - header_bytes - c_bytes, per_stage)); + const s: i64 = @intCast(stages); + const a = header_bytes + 1024; + const b = a + s * a_stage_bytes; + const sfa = b + @divExact(s * n * k_tile, 2); + const sfb = sfa + s * sf_stage_bytes; + // Accumulators, then 16 SFA and 16 SFB columns. + const columns = std.math.ceilPowerOfTwo(u16, @max(32, accumulator_stages * n + 32)) catch unreachable; + return .{ + .n = n, + .stages = stages, + .smem_bytes = sfb + s * sf_stage_bytes, + .tmem_columns = columns, + .empty_barriers = 8 * s, + .accumulator_full = 16 * s, + .accumulator_empty = 16 * s + 16, + .tmem_dealloc = 16 * s + 32, + .tmem_holding = 16 * s + 40, + .c = header_bytes, + .a = a, + .b = b, + .sfa = sfa, + .sfb = sfb, + }; + } + } + + /// Routed rows an epilogue pass handles: one per warp lane. + pub fn chunkRows(self: Tile) i64 { + return @min(self.n, 32); + } + + /// Epilogue passes over the tile width. + pub fn chunks(self: Tile) usize { + return @intCast(@divExact(self.n, self.chunkRows())); + } + + fn bStageBytes(self: Tile) i64 { + return @divExact(self.n * k_tile, 2); + } + + /// TMA bytes landing in one stage: packed A and B, then both scale tiles. + fn transactionBytes(self: Tile) u32 { + return @intCast(a_stage_bytes + self.bStageBytes() + 2 * sf_stage_bytes); + } + + fn fields(self: Tile, b: *B) []const []const u8 { + const arena = b.arena.allocator(); + const s: i64 = @intCast(self.stages); + const out = arena.alloc([]const u8, 11) catch @panic("OOM"); + const Field = struct { []const u8, i64, i64 }; + const list = [_]Field{ + .{ "ab_full_mbar_ptr", 8 * s, 0 }, + .{ "ab_empty_mbar_ptr", 8 * s, self.empty_barriers }, + .{ "acc_full_mbar_ptr", 16, self.accumulator_full }, + .{ "acc_empty_mbar_ptr", 16, self.accumulator_empty }, + .{ "tmem_dealloc_mbar", 8, self.tmem_dealloc }, + .{ "tmem_holding_buf", 4, self.tmem_holding }, + .{ "sC", c_bytes, self.c }, + .{ "sA", s * a_stage_bytes, self.a }, + .{ "sB", s * self.bStageBytes(), self.b }, + .{ "sSFA", s * sf_stage_bytes, self.sfa }, + .{ "sSFB", s * sf_stage_bytes, self.sfb }, + }; + for (list, out) |field, *text| { + text.* = std.fmt.allocPrint(arena, "{s}:{d}:{d}", .{ field[0], field[1], field[2] }) catch @panic("OOM"); + } + return out; + } +}; + +const StageStorage = struct { + a: V, + b: V, + sfa: V, + sfb: V, + c: V, + load_barriers: V, + mma_barriers: V, + accumulator_full_barriers: V, + accumulator_empty_barriers: V, +}; + +const StageBuffers = struct { + a_destination: cute.View, + b_destination: cute.View, + sfa_destination: cute.View, + sfb_destination: cute.View, + load_barrier: V, + mma_barrier: V, +}; + +const StagedOperands = struct { + a: cute.View, + b: cute.View, + sfa: cute.View, + sfb: cute.View, +}; + +/// What the epilogue warps do with each finished 128xN accumulator tile. +pub const Epilogue = enum { + /// Up projection: per-expert dequantization scale, BF16 rounding, + /// SwiGLU of the interleaved gate/up rows (optionally clamped), the + /// routing weight when it precedes the down projection, BF16 rounding, + /// then NVFP4 quantization (E4M3 scale per 16 activations, scaled by the + /// per-expert activation global scale) straight into the down GEMM's + /// packed input and 128x4 scale layouts. + swiglu_nvfp4, + /// Down projection: per-expert dequantization scale and, when it follows + /// the down projection, the routing weight, then BF16 rows stored in + /// route order `[routes, m]`, skipping padding rows. + route_rows, +}; + +pub const Config = struct { + experts: i64, + /// GEMM output rows (2 x intermediate for gate/up, hidden for down). + m: i64, + /// Tile N: rows in one routed expert group. + n: i64 = 16, + /// Reduction dimension, a multiple of `k_tile`. + k: i64, + groups: i64, + /// Resident CTAs of the static persistent tile scheduler: one per SM. + persistent_ctas: i64 = 148, + epilogue: Epilogue, + /// Token/top-k routes; sizes the route-indexed epilogue operands. + routes: i64, + /// Clamp of both halves of the fused SwiGLU, if any. + swiglu_limit: ?f32 = null, + /// Multiply rows by their routing weight in this projection's epilogue. + routing_weights: bool = false, + /// Ungrouped routing: group `g` is route `g` and holds one live row. + direct: bool = false, + + /// Rows stored per group in the routed operands: one whole N tile. + pub fn groupRows(self: Config) i64 { + return self.n; + } + + pub fn routedRows(self: Config) i64 { + return self.groups * self.groupRows(); + } +}; + +/// `schedule` is the routing schedule `[group experts | group sizes | active +/// group count]`; `alpha` the per-expert FP32 dequantization scale of the +/// GEMM output. +const inputs = [_][:0]const u8{ + "weight", + "schedule", + "input_quant", + "weight_scale", + "input_scale", + "alpha", +}; + +/// `route_inverse` maps each physical routed row to its token/top-k route; +/// `act_scale` is the per-expert global scale of the down GEMM's NVFP4 input. +const UpQuantized = cute.Program(Config, .{ + .name = "zml_nvfp4_up_swiglu_zig", + .inputs = &(inputs ++ [_][:0]const u8{ "route_inverse", "routing_weights", "act_scale" }), + .outputs = &.{ "q", "s" }, + .run = buildProgram, +}); + +const DownRows = cute.Program(Config, .{ + .name = "zml_nvfp4_down_rows_zig", + .inputs = &(inputs ++ [_][:0]const u8{ "route_inverse", "routing_weights" }), + .outputs = &.{"output"}, + .run = buildProgram, +}); + +pub const Inputs = struct { + /// `[experts, m, k / 2]` packed FP4. + weight: zml.Tensor, + schedule: zml.Tensor, + /// `[groups * n, k / 2]` packed FP4. + input_quant: zml.Tensor, + /// `[experts, m, k / 16]` E4M3 bytes in the 128x4 layout. + weight_scale: zml.Tensor, + /// `[groups, k / 16 * 128]` E4M3 bytes in the 128x4 layout. + input_scale: zml.Tensor, + /// `[experts]` FP32. + alpha: zml.Tensor, +}; + +/// Up projection with the SwiGLU + NVFP4 epilogue: the routed packed FP4 +/// rows `[groups * n, m / 4]` and their 128x4 E4M3 scales +/// `[groups, m / 32 * 128]`, ready for the down GEMM. `routing_weights` is +/// `[routes]` FP32 and `act_scale` `[experts]` FP32. +pub fn upQuantized(cfg: Config, a: Inputs, route_inverse: zml.Tensor, routing_weights: zml.Tensor, act_scale: zml.Tensor) struct { q: zml.Tensor, s: zml.Tensor } { + std.debug.assert(cfg.epilogue == .swiglu_nvfp4); + const activations = @divExact(cfg.m, 2); + const result = UpQuantized.call(.{ + .weight = a.weight, + .schedule = a.schedule, + .input_quant = a.input_quant, + .weight_scale = a.weight_scale, + .input_scale = a.input_scale, + .alpha = a.alpha, + .route_inverse = route_inverse, + .routing_weights = routing_weights, + .act_scale = act_scale, + }, .{ + .q = .init(.{ cfg.routedRows(), @divExact(activations, 2) }, .u8), + .s = .init(.{ cfg.groups, @divExact(activations, 16) * 128 }, .u8), + }, .{ .cfg = cfg, .scalars = &.{ cfg.m, cfg.n, cfg.k, cfg.groups, cfg.experts } }); + return .{ .q = result.q, .s = result.s }; +} + +/// Down projection storing BF16 `[routes, m]` in token/top-k order. +pub fn downRows(cfg: Config, a: Inputs, route_inverse: zml.Tensor, routing_weights: zml.Tensor) zml.Tensor { + std.debug.assert(cfg.epilogue == .route_rows); + return DownRows.call(.{ + .weight = a.weight, + .schedule = a.schedule, + .input_quant = a.input_quant, + .weight_scale = a.weight_scale, + .input_scale = a.input_scale, + .alpha = a.alpha, + .route_inverse = route_inverse, + .routing_weights = routing_weights, + }, .{ .output = .init(.{ @max(cfg.routes, 1), cfg.m }, .bf16) }, .{ .cfg = cfg, .scalars = &.{ cfg.m, cfg.n, cfg.k, cfg.groups, cfg.experts } }).output; +} + +const typed = common.typed; +const gridTyped = common.gridTyped; +const fmt = common.fmt; +const view = common.view; +const barrierAt = common.barrierAt; +const PipelinePosition = common.PipelinePosition; +const DirectRoutes = common.DirectRoutes; +const persistentWorkAt = common.persistentWorkAt; +const advancePersistentWork = common.advancePersistentWork; +const advancePipeline = common.advancePipeline; +const peekBarrierIfRemaining = common.peekBarrierIfRemaining; +const sharedPointerAt = common.sharedPointerAt; +const offsetAlignment = common.offsetAlignment; +const typesOf = common.typesOf; +const tupleOf = common.tupleOf; + +fn deviceName(comptime epilogue: Epilogue) [:0]const u8 { + return switch (epilogue) { + .swiglu_nvfp4 => "nvfp4_sm100_persistent_grouped_gemm_up_swiglu", + .route_rows => "nvfp4_sm100_persistent_grouped_gemm_down_rows", + }; +} + +pub const tile_widths = [_]i64{ 8, 16, 32, 64, 128 }; + +fn buildProgram(b: *B, cfg: Config) cute.FinishError!void { + return switch (cfg.n) { + inline 8, 16, 32, 64, 128 => |n| switch (cfg.epilogue) { + inline else => |epilogue| buildTiledProgram(b, cfg, comptime Tile.of(n), epilogue), + }, + else => std.debug.panic("unsupported NVFP4 tile N {d}", .{cfg.n}), + }; +} + +/// Types and layouts shared by the device ABI and the host launch, as the +/// CuTe DSL derives them for FP4 x FP4 with 16-element E4M3 scale vectors. +fn Layouts(comptime tile: Tile) type { + return struct { + const n = tile.n; + const stages = tile.stages; + const ab_swizzle: B.Swizzle = .{ .bits = 3, .base = 4, .shift = 3 }; + + fn aCoord(b: *B, cfg: Config) cute.LayoutSpec { + return b.layoutSpec(.{ cfg.m, cfg.k, cfg.experts }, .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}), b.basis(1, 1, .{2}) }); + } + fn bCoord(b: *B, cfg: Config) cute.LayoutSpec { + return b.layoutSpec(.{ cfg.groupRows(), cfg.k, cfg.groups }, .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}), b.basis(1, 1, .{2}) }); + } + /// TMA coordinates of the 128x4 scale layout: byte in the atom, K + /// atom, M block, L. + fn sfCoord(b: *B) cute.LayoutSpec { + return b.layoutSpec( + .{ .{ .{ 32, 4 }, cute.AlgebraToken.dynamic }, .{ .{ 16, 4 }, cute.AlgebraToken.dynamic }, .{ 1, cute.AlgebraToken.dynamic } }, + .{ + .{ .{ b.basis(8, 1, .{0}), b.basis(2, 1, .{0}) }, b.basis(1, 1, .{2}) }, + .{ .{ 0, b.basis(1, 2, .{0}) }, b.basis(1, 1, .{1}) }, + .{ 0, b.basis(1, 1, .{3}) }, + }, + ); + } + fn aBasis(b: *B) cute.LayoutSpec { + return b.layoutSpec(.{ k_tile, 128, 1 }, .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}), b.basis(1, 1, .{2}) }); + } + fn bBasis(b: *B) cute.LayoutSpec { + return b.layoutSpec(.{ k_tile, n, 1 }, .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}), b.basis(1, 1, .{2}) }); + } + fn sfBasis(b: *B) cute.LayoutSpec { + return b.layoutSpec( + .{ .{ 2, 4, 32 }, 4, 1, 1 }, + .{ .{ b.basis(1, 1, .{ 1, 0, 1 }), b.basis(1, 1, .{ 0, 0, 1 }), b.basis(1, 1, .{ 0, 0, 0 }) }, b.basis(1, 1, .{ 1, 1 }), b.basis(1, 1, .{ 0, 1 }), b.basis(1, 1, .{ 2, 1 }) }, + ); + } + fn tmaA(b: *B, cfg: Config) cute.TmaConfig { + return .{ .dtype = .f4e2m1fn, .copy_bits = 128 * k_tile * 4, .global_basis = aBasis(b), .coordinate_layout = aCoord(b, cfg), .coordinate_rank = 3, .format = .u4 }; + } + fn tmaB(b: *B, cfg: Config) cute.TmaConfig { + return .{ .dtype = .f4e2m1fn, .copy_bits = @intCast(n * k_tile * 4), .global_basis = bBasis(b), .coordinate_layout = bCoord(b, cfg), .coordinate_rank = 3, .format = .u4 }; + } + fn tmaSf(b: *B) cute.TmaConfig { + return .{ .dtype = .f8e4m3fn, .copy_bits = Tile.sf_stage_bytes * 8, .global_basis = sfBasis(b), .coordinate_layout = sfCoord(b), .coordinate_rank = 4, .format = .u16 }; + } + // Shared-memory layouts in FP4 elements for A/B, bytes for scales. + fn aSmem(b: *B, comptime staged: bool) cute.LayoutSpec { + return if (staged) + b.layoutSpec(.{ .{ 128, 64 }, 1, 4, stages }, .{ .{ k_tile, 1 }, 0, 64, 2 * Tile.a_stage_bytes }) + else + b.layoutSpec(.{ .{ 128, 64 }, 1, 4 }, .{ .{ k_tile, 1 }, 0, 64 }); + } + fn bSmem(b: *B, comptime staged: bool) cute.LayoutSpec { + return if (staged) + b.layoutSpec(.{ .{ n, 64 }, 1, 4, stages }, .{ .{ k_tile, 1 }, 0, 64, 2 * tile.bStageBytes() }) + else + b.layoutSpec(.{ .{ n, 64 }, 1, 4 }, .{ .{ k_tile, 1 }, 0, 64 }); + } + fn sfSmem(b: *B, comptime staged: bool) cute.LayoutSpec { + return if (staged) + b.layoutSpec(.{ .{ .{ .{ 32, 4 }, 1 }, .{ 16, 4 } }, 1, 4, stages }, .{ .{ .{ .{ 16, 4 }, 0 }, .{ 0, 1 } }, 0, 512, Tile.sf_stage_bytes }) + else + b.layoutSpec(.{ .{ .{ .{ 32, 4 }, 1 }, .{ 16, 4 } }, 1, 4 }, .{ .{ .{ .{ 16, 4 }, 0 }, .{ 0, 1 } }, 0, 512 }); + } + fn fp4SmemPtrTy(b: *B) *const mlir.Type { + return b.swizzledPtrTy(.f4e2m1fn, .smem, 1024, ab_swizzle) catch @panic("bad FP4 shared pointer"); + } + }; +} + +/// Device memrefs of the epilogue operands. +const EpilogueTypes = struct { + sizes: *const mlir.Type, + inverse: *const mlir.Type, + weights: *const mlir.Type, + per_expert: *const mlir.Type, + q: *const mlir.Type, + s: *const mlir.Type, + rows: *const mlir.Type, + + fn of(b: *B, cfg: Config) EpilogueTypes { + const activations = @divExact(cfg.m, 2); + return .{ + .sizes = b.memrefType(.i32, .gmem, 4, b.layoutType(b.layoutSpec(cfg.groups, 1))), + .inverse = b.memrefType(.i32, .gmem, 16, b.layoutType(b.layoutSpec(cfg.routedRows(), 1))), + .weights = b.memrefType(.f32, .gmem, 16, b.layoutType(b.layoutSpec(@max(cfg.routes, 1), 1))), + .per_expert = b.memrefType(.f32, .gmem, 16, b.layoutType(b.layoutSpec(cfg.experts, 1))), + // Eight packed FP4 activations per word. + .q = b.memrefType(.i32, .gmem, 16, b.layoutType(b.layoutSpec(.{ cfg.routedRows(), @divExact(activations, 8) }, .{ @divExact(activations, 8), 1 }))), + .s = b.memrefType(.i8, .gmem, 16, b.layoutType(b.layoutSpec(.{ cfg.groups, @divExact(activations, 16) * 128 }, .{ @divExact(activations, 16) * 128, 1 }))), + .rows = b.memrefType(.bf16, .gmem, 16, b.layoutType(b.layoutSpec(.{ @max(cfg.routes, 1), cfg.m }, .{ cfg.m, 1 }))), + }; + } +}; + +fn buildTiledProgram(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: Epilogue) cute.FinishError!void { + @setEvalBranchQuota(20_000); + std.debug.assert(@rem(cfg.m, 128) == 0); + std.debug.assert(@rem(cfg.k, k_tile) == 0); + const L = Layouts(tile); + const public_name = b.name; + + const ids_memref = b.memrefType(.i32, .gmem, 16, b.layoutType(b.layoutSpec(cfg.groups, 1))); + const active_memref = b.memrefType(.i32, .gmem, 4, b.layoutType(b.layoutSpec(1, 1))); + const et = EpilogueTypes.of(b, cfg); + const i32_type = cute.DType.i32.toMlir(b.ctx); + const mma_type = b.blockScaledMmaType(.{ .kind = mma_kind, .n = @intCast(tile.n) }); + const mma_sfb_type = b.blockScaledMmaType(.{ .kind = mma_kind, .n = 128 }); + + b.beginFunction(deviceName(epilogue), .cuda_kernel); + const d = try b.declareArgs(switch (epilogue) { + .swiglu_nvfp4 => .{ + .expert_ids = typed(ids_memref), + .active_count = typed(active_memref), + .mma = typed(mma_type), + .mma_sfb = typed(mma_sfb_type), + .tma_a = gridTyped(b.tmaLoadAtomType(L.tmaA(b, cfg))), + .gA = typed(b.coordTensorType(3, L.aCoord(b, cfg))), + .tma_b = gridTyped(b.tmaLoadAtomType(L.tmaB(b, cfg))), + .gB = typed(b.coordTensorType(3, L.bCoord(b, cfg))), + .tma_sfa = gridTyped(b.tmaLoadAtomType(L.tmaSf(b))), + .gSFA = typed(b.coordTensorType(4, L.sfCoord(b))), + .tma_sfb = gridTyped(b.tmaLoadAtomType(L.tmaSf(b))), + .gSFB = typed(b.coordTensorType(4, L.sfCoord(b))), + .alpha = typed(et.per_expert), + .group_sizes = typed(et.sizes), + .route_inverse = typed(et.inverse), + .routing_weights = typed(et.weights), + .act_scale = typed(et.per_expert), + .q = typed(et.q), + .s = typed(et.s), + .tiles_m = typed(i32_type), + }, + .route_rows => .{ + .expert_ids = typed(ids_memref), + .active_count = typed(active_memref), + .mma = typed(mma_type), + .mma_sfb = typed(mma_sfb_type), + .tma_a = gridTyped(b.tmaLoadAtomType(L.tmaA(b, cfg))), + .gA = typed(b.coordTensorType(3, L.aCoord(b, cfg))), + .tma_b = gridTyped(b.tmaLoadAtomType(L.tmaB(b, cfg))), + .gB = typed(b.coordTensorType(3, L.bCoord(b, cfg))), + .tma_sfa = gridTyped(b.tmaLoadAtomType(L.tmaSf(b))), + .gSFA = typed(b.coordTensorType(4, L.sfCoord(b))), + .tma_sfb = gridTyped(b.tmaLoadAtomType(L.tmaSf(b))), + .gSFB = typed(b.coordTensorType(4, L.sfCoord(b))), + .alpha = typed(et.per_expert), + .group_sizes = typed(et.sizes), + .route_inverse = typed(et.inverse), + .routing_weights = typed(et.weights), + .rows = typed(et.rows), + .tiles_m = typed(i32_type), + }, + }); + b.setFunctionAttribute(nvvm.minctasm(b.ctx, 1)); + b.setFunctionAttribute(.named(b.ctx, "smem.partition_num", .int(b.ctx, .i32, 2))); + b.setFunctionAttribute(.named(b.ctx, "smem.hint_smem_base_uniform", .unit(b.ctx))); + b.setFunctionAttribute(.named(b.ctx, "cu_attrs", .dict(b.ctx, &.{ + .named(b.ctx, "max_dynamic_shared_size_bytes", b.devMaxSharedMemoryOptin()), + .named(b.ctx, "non_portable_cluster_size_allowed", .int(b.ctx, .i32, 1)), + }))); + + // Prefetch every tensor-map descriptor while the producing grid drains. + const warp = b.makeWarpUniform(b.makeWarpUniform(b.threadIdx().x.div(32))); + var tma_warp = b.openIf(warp.eq(5)); + inline for (.{ d.tma_a, d.tma_b, d.tma_sfa, d.tma_sfb }) |value| { + const descriptor: cute.Atom = .{ .inner = value.inner, .kernel = b }; + b.prefetchTmaDesc(descriptor); + } + tma_warp.close(); + b.waitForDependency(); + buildUmmaDevice(b, cfg, tile, epilogue, d); + b.launchDependents(); + b.endFunction(.{ 192, 1, 1 }); + + // Public host ABI: the XLA operands, the outputs, then the problem shape. + b.beginFunction(public_name, .host); + const operands = .{ + .weight = .{ .ptr = cute.DType.i8 }, + .schedule = .{ .ptr = cute.DType.i32 }, + .input_quant = .{ .ptr = cute.DType.i8 }, + .weight_scale = .{ .ptr = cute.DType.i8 }, + .input_scale = .{ .ptr = cute.DType.i8 }, + .alpha = .{ .ptr = cute.DType.f32 }, + .route_inverse = .{ .ptr = cute.DType.i32 }, + .routing_weights = .{ .ptr = cute.DType.f32 }, + }; + const shape = .{ + .problem_m = typed(i32_type), + .problem_n = typed(i32_type), + .problem_k = typed(i32_type), + .problem_l = typed(i32_type), + .expert_count = typed(i32_type), + }; + _ = try b.declareArgs(switch (epilogue) { + .swiglu_nvfp4 => .{ + .weight = operands.weight, + .schedule = operands.schedule, + .input_quant = operands.input_quant, + .weight_scale = operands.weight_scale, + .input_scale = operands.input_scale, + .alpha = operands.alpha, + .route_inverse = operands.route_inverse, + .routing_weights = operands.routing_weights, + .act_scale = .{ .ptr = cute.DType.f32 }, + .q = .{ .ptr = cute.DType.i8 }, + .s = .{ .ptr = cute.DType.i8 }, + .problem_m = shape.problem_m, + .problem_n = shape.problem_n, + .problem_k = shape.problem_k, + .problem_l = shape.problem_l, + .expert_count = shape.expert_count, + }, + .route_rows => .{ + .weight = operands.weight, + .schedule = operands.schedule, + .input_quant = operands.input_quant, + .weight_scale = operands.weight_scale, + .input_scale = operands.input_scale, + .alpha = operands.alpha, + .route_inverse = operands.route_inverse, + .routing_weights = operands.routing_weights, + .output = .{ .ptr = cute.DType.bf16 }, + .problem_m = shape.problem_m, + .problem_n = shape.problem_n, + .problem_k = shape.problem_k, + .problem_l = shape.problem_l, + .expert_count = shape.expert_count, + }, + }); + try buildHostLaunch(b, cfg, tile, epilogue); + b.endFunction(null); +} + +/// Host argument positions: the six mainloop operands, the route operands, +/// the epilogue's extra inputs and outputs, then the five shape scalars. +fn HostArgs(comptime epilogue: Epilogue) type { + return struct { + const weight = 0; + const schedule = 1; + const input_quant = 2; + const weight_scale = 3; + const input_scale = 4; + const alpha = 5; + const route_inverse = 6; + const routing_weights = 7; + const act_scale = 8; + const output = switch (epilogue) { + .swiglu_nvfp4 => 9, + .route_rows => 8, + }; + const scales = 10; + const problem_m = switch (epilogue) { + .swiglu_nvfp4 => 11, + .route_rows => 9, + }; + const problem_n = problem_m + 1; + const problem_k = problem_m + 2; + const problem_l = problem_m + 3; + const expert_count = problem_m + 4; + }; +} + +/// Complete staged shared-memory tensors. Per-stage views are slices. +fn stagedOperands(b: *B, comptime tile: Tile, storage: StageStorage) StagedOperands { + const L = Layouts(tile); + const a_full = b.makeView( + b.recastSwizzledPointer(storage.a, .f4e2m1fn, .smem, 1024, L.ab_swizzle), + b.staticLayout(L.aSmem(b, true)), + b.memrefTypeFromPointer(L.fp4SmemPtrTy(b), b.layoutType(L.aSmem(b, true))), + ); + const b_full = b.makeView( + b.recastSwizzledPointer(storage.b, .f4e2m1fn, .smem, 1024, L.ab_swizzle), + b.staticLayout(L.bSmem(b, true)), + b.memrefTypeFromPointer(L.fp4SmemPtrTy(b), b.layoutType(L.bSmem(b, true))), + ); + const sfa_full = b.makeView(storage.sfa, b.staticLayout(L.sfSmem(b, true)), b.memrefType(.f8e4m3fn, .smem, 1024, b.layoutType(L.sfSmem(b, true)))); + const sfb_full = b.makeView(storage.sfb, b.staticLayout(L.sfSmem(b, true)), b.memrefType(.f8e4m3fn, .smem, 1024, b.layoutType(L.sfSmem(b, true)))); + return .{ .a = a_full, .b = b_full, .sfa = sfa_full, .sfb = sfb_full }; +} + +fn stageBuffersAt(b: *B, comptime tile: Tile, storage: StageStorage, stage_index: V) StageBuffers { + const L = Layouts(tile); + const full = stagedOperands(b, tile, storage); + const all = cute.AlgebraToken.all; + const a = b.slice(full.a, .{ all, all, all, stage_index }, b.memrefTypeFromPointer(L.fp4SmemPtrTy(b), b.layoutType(L.aSmem(b, false)))); + const rhs = b.slice(full.b, .{ all, all, all, stage_index }, b.memrefTypeFromPointer(L.fp4SmemPtrTy(b), b.layoutType(L.bSmem(b, false)))); + const sf_stage_align = 1024; + const sfa = b.slice(full.sfa, .{ all, all, all, stage_index }, b.memrefType(.f8e4m3fn, .smem, sf_stage_align, b.layoutType(L.sfSmem(b, false)))); + const sfb = b.slice(full.sfb, .{ all, all, all, stage_index }, b.memrefType(.f8e4m3fn, .smem, sf_stage_align, b.layoutType(L.sfSmem(b, false)))); + + // `tma_partition` keeps a singleton CTA mode; the copy views keep it too. + const a_partition_layout = b.staticLayout(b.layoutSpec(.{ .{ 2 * Tile.a_stage_bytes, 1 }, .{1} }, .{ .{ 1, 0 }, .{0} })); + const b_partition_layout = b.staticLayout(b.layoutSpec(.{ .{ 2 * tile.bStageBytes(), 1 }, .{1} }, .{ .{ 1, 0 }, .{0} })); + const sf_partition_layout = b.staticLayout(b.layoutSpec(.{ .{ Tile.sf_stage_bytes, 1 }, .{1} }, .{ .{ 1, 0 }, .{0} })); + + return .{ + .a_destination = b.withLayout(a, a_partition_layout), + .b_destination = b.withLayout(rhs, b_partition_layout), + .sfa_destination = b.withLayout(sfa, sf_partition_layout), + .sfb_destination = b.withLayout(sfb, sf_partition_layout), + .load_barrier = barrierAt(b, storage.load_barriers, stage_index), + .mma_barrier = barrierAt(b, storage.mma_barriers, stage_index), + }; +} + +/// The TMA coordinates of one stage: slice the TMA-partitioned operand at +/// its output tile and L index, then at the K stage, and rebuild the +/// iterator as the Python front end does. +fn tmaSource(b: *B, partitioned: cute.View, tile_coord: anytype, k: V, comptime rows: i64, reduction_tiles: usize, copy_layout: cute.View) cute.View { + const all = cute.AlgebraToken.all; + const tile_k = b.slice( + partitioned, + tile_coord, + b.coordTensorTypeWithTuple(.{ 0, cute.ConstrainedDynamic{ .divisible_by = rows }, cute.AlgebraToken.dynamic }, b.layoutSpec( + .{ .{ .{ k_tile, rows }, 1 }, reduction_tiles }, + .{ .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0 }, b.basis(k_tile, 1, .{0}) }, + )), + ); + const div_k = cute.ConstrainedDynamic{ .divisible_by = k_tile }; + const div_rows = cute.ConstrainedDynamic{ .divisible_by = rows }; + const dynamic = cute.AlgebraToken.dynamic; + const stage = b.slice( + tile_k, + .{ all, k }, + b.coordTensorTypeWithTuple(.{ div_k, div_rows, dynamic }, b.layoutSpec( + .{.{ .{ k_tile, rows }, 1 }}, + .{.{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0 }}, + )), + ); + return b.rebuildCoordTensorIterator(3, stage, .{ div_k, div_rows, dynamic }, .{ k_tile, rows, 1 }, copy_layout); +} + +/// Issue the four transactions that fill one mainloop stage. +fn issueTmaStage( + b: *B, + comptime tile: Tile, + reduction_tiles: usize, + partitioned_a_target: cute.View, + partitioned_b_target: cute.View, + stage: StageBuffers, + k: V, + tile_m: V, + group: V, + expert: V, + exec_a: cute.Atom, + exec_b: cute.Atom, + exec_sfa: cute.Atom, + exec_sfb: cute.Atom, +) void { + const n = tile.n; + const all = cute.AlgebraToken.all; + const a_copy_layout = b.staticLayout(b.layoutSpec( + .{ .{ .{ k_tile, 128 }, 1 }, .{1} }, + .{ .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0 }, .{0} }, + )); + const b_copy_layout = b.staticLayout(b.layoutSpec( + .{ .{ .{ k_tile, n }, 1 }, .{1} }, + .{ .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0 }, .{0} }, + )); + // One stage moves four 512-byte atoms, one per K = 64 slice. + const sf_copy_spec = b.layoutSpec( + .{ .{ .{ 512, 4 }, 1 }, .{1} }, + .{ .{ .{ b.basis(1, 2, .{0}), b.basis(1, 1, .{1}) }, 0 }, .{0} }, + ); + // B's tile spans all of a group's rows, so its N-tile index is zero. Pass + // it as a value: a static zero leaf collapses out of the coordinate tuple + // and shifts the remaining tensor-map coordinates. + const b_tile_n = b.constant(.i32, 0); + const a_source = tmaSource(b, partitioned_a_target, .{ all, tile_m, all, expert }, k, 128, reduction_tiles, a_copy_layout); + const b_source = tmaSource(b, partitioned_b_target, .{ all, b_tile_n, all, group }, k, n, reduction_tiles, b_copy_layout); + // Scale coordinates: (byte, K atom, M block, L); a stage covers 4 K atoms. + const k_atom = k.mul(4); + const sfa_source = b.makeCoordTensor(.{ 0, k_atom, tile_m, expert }, b.staticLayout(sf_copy_spec)); + const sfb_source = b.makeCoordTensor(.{ 0, k_atom, b.constant(.i32, 0), group }, b.staticLayout(sf_copy_spec)); + + var elected = b.openIf(b.electSync()); + b.mbarrierArriveExpectTx(stage.load_barrier, tile.transactionBytes()); + elected.close(); + b.copy(b.setTmaBarrier(exec_a, stage.load_barrier), .{a_source}, .{stage.a_destination}, .{}); + b.copy(b.setTmaBarrier(exec_b, stage.load_barrier), .{b_source}, .{stage.b_destination}, .{}); + b.copy(b.setTmaBarrier(exec_sfa, stage.load_barrier), .{sfa_source}, .{stage.sfa_destination}, .{}); + b.copy(b.setTmaBarrier(exec_sfb, stage.load_barrier), .{sfb_source}, .{stage.sfb_destination}, .{}); +} + +fn accumulatorTmemAt(b: *B, comptime tile: Tile, tmem: V, stage: V) V { + return b.addOffset( + tmem, + b.makeIntTupleDivBy(stage.mul(tile.n), @intCast(tile.n)), + b.ptrTy(.f32, .tmem, 16) catch @panic("bad accumulator TMEM pointer"), + ).value(); +} + +/// Warp 5 runs independently over every persistent output tile. +fn runTmaProducer( + b: *B, + comptime tile: Tile, + k_tiles: usize, + storage: StageStorage, + active: V, + tiles_m: V, + expert_ids: cute.View, + direct: ?DirectRoutes, + partitioned_a_target: cute.View, + partitioned_b_target: cute.View, + exec_a: cute.Atom, + exec_b: cute.Atom, + exec_sfa: cute.Atom, + exec_sfb: cute.Atom, +) void { + const initial_index = b.constant(.i32, 0); + const initial_phase = b.constant(.i32, 1); + const initial_count = b.constant(.i32, 0); + const persistent_clusters = b.gridDim().z; + const cta_m = b.blockIdx().x; + const cta_n = b.blockIdx().y; + const cta_l = b.constant(.i32, 0); + const div_m = b.fastDivmodCreate(tiles_m); + const div_n = b.fastDivmodCreate(1); + const initial_work = persistentWorkAt(b, b.blockIdx().z, b.constant(.i32, 0), active, tiles_m, div_m, div_n, direct); + const carried = .{ + initial_work.m, initial_work.n, initial_work.group, initial_work.valid, + initial_count, initial_index, initial_phase, persistent_clusters, + initial_work.linear, cta_m, cta_n, cta_l, + initial_work.executed, tiles_m, active, div_m, + div_n, + }; + var tiles = b.openWhile(carried, typesOf(carried)); + tiles.yieldBefore(tiles.before_carried[3], tupleOf(tiles.before_carried)); + const tile_m = tiles.after_carried[0]; + const group = tiles.after_carried[2]; + const expert = expert_ids.get(.{group}, .i32); + const first_state: PipelinePosition = .{ + .index = tiles.after_carried[5], + .phase = tiles.after_carried[6], + }; + const first_ready = b.mbarrierTryWait(barrierAt(b, storage.mma_barriers, first_state.index), first_state.phase, .{ .scope = .cta }); + var stages = b.openFor( + 0, + @as(i32, @intCast(k_tiles)), + 1, + .{ first_ready, b.constant(.i32, 0), first_state.index, first_state.phase }, + ); + const k = stages.carried[1]; + const state: PipelinePosition = .{ + .index = stages.carried[2], + .phase = stages.carried[3], + }; + const buffers = stageBuffersAt(b, tile, storage, state.index); + var pending = b.openIf(stages.carried[0].eq(false)); + b.mbarrierWait(buffers.mma_barrier, state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); + pending.close(); + issueTmaStage(b, tile, k_tiles, partitioned_a_target, partitioned_b_target, buffers, k, tile_m, group, expert, exec_a, exec_b, exec_sfa, exec_sfb); + const next_count = stages.carried[1].add(1); + const next_state = advancePipeline(b, state.index, state.phase, tile.stages); + const next_ready = peekBarrierIfRemaining(b, storage.mma_barriers, next_count, k_tiles, next_state); + stages.yield(.{ next_ready, next_count, next_state.index, next_state.phase }); + const next_work = advancePersistentWork( + b, + tiles.after_carried[8], + tiles.after_carried[12], + tiles.after_carried[7], + tiles.after_carried[14], + tiles.after_carried[13], + tiles.after_carried[15], + tiles.after_carried[16], + direct, + ); + tiles.yieldAfter(.{ + next_work.m, + next_work.n, + next_work.group, + next_work.valid, + stages.results[1], + stages.results[2], + stages.results[3], + tiles.after_carried[7], + next_work.linear, + tiles.after_carried[9], + tiles.after_carried[10], + tiles.after_carried[11], + next_work.executed, + tiles.after_carried[13], + tiles.after_carried[14], + tiles.after_carried[15], + tiles.after_carried[16], + }); + + // PipelineTmaUmma.producer_tail: drain every empty barrier before exit. + var tail_state: PipelinePosition = .{ + .index = tiles.results[5], + .phase = tiles.results[6], + }; + inline for (0..tile.stages) |stage| { + b.mbarrierWait(barrierAt(b, storage.mma_barriers, tail_state.index), tail_state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); + if (stage + 1 < tile.stages) { + tail_state = advancePipeline(b, tail_state.index, tail_state.phase, tile.stages); + } + } +} + +/// NVFP4 mainloop objects of one pipeline stage. +const Mainloop = struct { + a: V, + b: V, + accumulator: V, + sfa: V, + sfb: V, + sfa_copy: cute.Atom, + sfb_copy: cute.Atom, + sfa_source: V, + sfb_source: V, + sfa_target: V, + sfb_target: V, + + pub fn copyScales(self: Mainloop, builder: *B) void { + builder.copy(self.sfa_copy, .{self.sfa_source}, .{self.sfa_target}, .{}); + builder.copy(self.sfb_copy, .{self.sfb_source}, .{self.sfb_target}, .{}); + } +}; + +/// Pipeline-wide UMMA and S2T descriptors derived from the complete staged +/// tensors once; `atStage` slices them at the current stage index. +const StagedMainloop = struct { + a: cute.View, + b: cute.View, + sfa_source: cute.View, + sfb_source: cute.View, + sfa: V, + sfb: V, + sfa_copy: cute.Atom, + sfb_copy: cute.Atom, + sfa_target: V, + sfb_target: V, + n: i64, + + pub fn atStage(self: StagedMainloop, b: *B, index: V, accumulator_tmem: V) Mainloop { + const all = cute.AlgebraToken.all; + const a = b.slice(self.a, .{ all, all, all, index }, b.smemDescViewType(b.layoutSpec(.{ 1, 1, 4 }, .{ 0, 0, 2 }))); + const rhs = b.slice(self.b, .{ all, all, all, index }, b.smemDescViewType(b.layoutSpec(.{ 1, 1, 4 }, .{ 0, 0, 2 }))); + const sfa_stage = b.slice(self.sfa_source, .{ all, all, all, all, index }, b.smemDescViewType(b.layoutSpec(.{ .{ .{ .{ 32, 1, 1 }, 4 }, 1 }, 1, 1, 4 }, .{ .{ .{ .{ 1, 1, 1 }, 0 }, 0 }, 0, 0, 32 }))); + const sfb_stage = b.slice(self.sfb_source, .{ all, all, all, all, index }, b.smemDescViewType(b.layoutSpec(.{ .{ .{ .{ 32, 1, 1 }, 4 }, 1 }, 1, 1, 4 }, .{ .{ .{ .{ 1, 1, 1 }, 0 }, 0 }, 0, 0, 32 }))); + const source_grouped_spec = b.layoutSpec(.{ .{ .{ .{ 32, 1, 1 }, 4 }, 1 }, .{ 1, 1, 4 } }, .{ .{ .{ .{ 1, 1, 1 }, 0 }, 0 }, .{ 0, 0, 32 } }); + const source_grouped_layout = b.staticLayout(source_grouped_spec); + const source_grouped_ty = b.smemDescViewType(source_grouped_spec); + const sfa_source = b.makeView(b.getIter(sfa_stage, b.smemDescType()), source_grouped_layout, source_grouped_ty); + const sfb_source = b.makeView(b.getIter(sfb_stage, b.smemDescType()), source_grouped_layout, source_grouped_ty); + const acc_spec = b.layoutSpec(.{ .{ 128, self.n }, 1, 1 }, .{ .{ 65536, 1 }, 0, 0 }); + const acc_layout = b.staticLayout(acc_spec); + const accumulator = b.makeView( + accumulator_tmem, + acc_layout, + b.memrefType(.f32, .tmem, 16, b.layoutType(acc_spec)), + ); + return .{ + .a = a.value(), + .b = rhs.value(), + .accumulator = accumulator.value(), + .sfa = self.sfa, + .sfb = self.sfb, + .sfa_copy = self.sfa_copy, + .sfb_copy = self.sfb_copy, + .sfa_source = sfa_source.value(), + .sfb_source = sfb_source.value(), + .sfa_target = self.sfa_target, + .sfb_target = self.sfb_target, + }; + } +}; + +fn makeStagedMainloop( + b: *B, + tiled_mma: cute.Atom, + comptime n: i64, + comptime stages: usize, + smem_a: V, + smem_b: V, + smem_sfa: V, + smem_sfb: V, + scale_tmem: V, +) StagedMainloop { + // Descriptor strides are in 16-byte units: one 64-deep K slice of packed + // FP4 is 32 bytes, a 128x256 A stage 16 KiB and an Nx256 B stage N*128 bytes. + const frag_a = b.mmaMakeFragment( + tiled_mma, + smem_a, + 0, + b.smemDescViewType(b.layoutSpec(.{ 1, 1, 4, stages }, .{ 0, 0, 2, Tile.a_stage_bytes / 16 })), + ); + const frag_b = b.mmaMakeFragment( + tiled_mma, + smem_b, + 1, + b.smemDescViewType(b.layoutSpec(.{ 1, 1, 4, stages }, .{ 0, 0, 2, n * 8 })), + ); + + // TMEM columns: two N-wide accumulator stages, then 16 SFA and 16 SFB + // columns: four K = 64 slices of four E4M3 scales per lane each. + const sf_spec = b.layoutSpec(.{ .{ .{ .{ 32, 4 }, 4 }, .{ 16, 4 } }, 1, 4 }, .{ .{ .{ .{ 262144, 4 }, 8388608 }, .{ 0, 1 } }, 0, 16 }); + const sf_layout = b.staticLayout(sf_spec); + const sfa_col = b.addOffset(scale_tmem, b.staticIntTuple(2 * n), b.ptrTy(.f32, .tmem, 16) catch @panic("bad TMEM pointer")); + const sfb_col = b.addOffset(scale_tmem, b.staticIntTuple(2 * n + 16), b.ptrTy(.f32, .tmem, 16) catch @panic("bad TMEM pointer")); + const sfa_ptr = b.recastIter(sfa_col, b.ptrTy(.f8e4m3fn, .tmem, 16) catch @panic("bad scale-factor TMEM pointer")); + const sfb_ptr = b.recastIter(sfb_col, b.ptrTy(.f8e4m3fn, .tmem, 16) catch @panic("bad scale-factor TMEM pointer")); + const sf_memref = b.memrefType(.f8e4m3fn, .tmem, 16, b.layoutType(sf_spec)); + const sfa = b.makeView(sfa_ptr, sf_layout, sf_memref); + const sfb = b.makeView(sfb_ptr, sf_layout, sf_memref); + + const smem_filtered_ty = b.memrefType(.f8e4m3fn, .smem, 1024, b.layoutType(b.layoutSpec(.{ .{ .{ .{ 32, 4 }, 1 }, .{ 1, 4 } }, 1, 4, stages }, .{ .{ .{ .{ 16, 4 }, 0 }, .{ 0, 1 } }, 0, 512, 2048 }))); + const tmem_filtered_ty = b.memrefType(.f8e4m3fn, .tmem, 16, b.layoutType(b.layoutSpec(.{ .{ .{ .{ 32, 4 }, 4 }, .{ 1, 4 } }, 1, 4 }, .{ .{ .{ .{ 262144, 4 }, 8388608 }, .{ 0, 1 } }, 0, 16 }))); + const sfa_smem_filtered = b.filterZeros(smem_sfa, null, smem_filtered_ty); + const sfb_smem_filtered = b.filterZeros(smem_sfb, null, smem_filtered_ty); + const sfa_tmem_filtered = b.filterZeros(sfa, null, tmem_filtered_ty); + const sfb_tmem_filtered = b.filterZeros(sfb, null, tmem_filtered_ty); + + const copy_atom_ty = b.s2tCopyAtomType(.f8e4m3fn, .{ .num_dp = 32, .num_bit = 128, .broadcast = .x4 }); + const copy_atom = b.makeAtom(copy_atom_ty, .{}); + const copy_ty = b.tiledCopyType(copy_atom_ty, b.layoutSpec(.{ 1, .{ 32, .{ 4, 4 }, 4 } }, .{ 0, .{ 1, .{ 32, 128 }, 512 } }), &.{ b.layoutSpec(.{ 32, 4, 16 }, .{ 1, 512, 32 }), b.layoutSpec(1, 0), b.layoutSpec(1, 0) }); + const sfa_copy = b.makeS2tCopy(copy_atom, sfa_tmem_filtered, copy_ty); + const sfb_copy = b.makeS2tCopy(copy_atom, sfb_tmem_filtered, copy_ty); + + const source_partition_ty = b.memrefType(.f8e4m3fn, .smem, 1024, b.layoutType(b.layoutSpec(.{ .{ .{ .{ 32, 4, 4 }, 4 }, 1 }, 1, 1, 4, stages }, .{ .{ .{ .{ 16, 4, 4 }, 0 }, 0 }, 0, 0, 512, 2048 }))); + const source_desc_ty = b.smemDescViewType(b.layoutSpec(.{ .{ .{ .{ 32, 1, 1 }, 4 }, 1 }, 1, 1, 4, stages }, .{ .{ .{ .{ 1, 1, 1 }, 0 }, 0 }, 0, 0, 32, 128 })); + const target_partition_ty = b.memrefType(.f8e4m3fn, .tmem, 16, b.layoutType(b.layoutSpec(.{ .{ .{ 32, 16, 4 }, 1 }, 1, 1, 4 }, .{ .{ .{ 262144, 1, 8388608 }, 0 }, 0, 0, 16 }))); + const target_grouped_spec = b.layoutSpec(.{ .{ .{ 32, 16, 4 }, 1 }, .{ 1, 1, 4 } }, .{ .{ .{ 262144, 1, 8388608 }, 0 }, .{ 0, 0, 16 } }); + const target_grouped_ty = b.memrefType(.f8e4m3fn, .tmem, 16, b.layoutType(target_grouped_spec)); + const sfa_src_partition = b.tiledCopyPartition(sfa_copy, sfa_smem_filtered, .{0}, false, source_partition_ty); + const sfb_src_partition = b.tiledCopyPartition(sfb_copy, sfb_smem_filtered, .{0}, false, source_partition_ty); + const sfa_desc = b.s2tSmemDescriptor(sfa_copy, sfa_src_partition, source_desc_ty); + const sfb_desc = b.s2tSmemDescriptor(sfb_copy, sfb_src_partition, source_desc_ty); + const sfa_dst_partition = b.tiledCopyPartition(sfa_copy, sfa_tmem_filtered, .{0}, true, target_partition_ty); + const sfb_dst_partition = b.tiledCopyPartition(sfb_copy, sfb_tmem_filtered, .{0}, true, target_partition_ty); + const target_grouped_layout = b.staticLayout(target_grouped_spec); + const sfa_dst = b.makeView(b.getIter(sfa_dst_partition, b.ptrTy(.f8e4m3fn, .tmem, 16) catch @panic("bad scale-factor TMEM pointer")), target_grouped_layout, target_grouped_ty); + const sfb_dst = b.makeView(b.getIter(sfb_dst_partition, b.ptrTy(.f8e4m3fn, .tmem, 16) catch @panic("bad scale-factor TMEM pointer")), target_grouped_layout, target_grouped_ty); + + return .{ + .a = frag_a, + .b = frag_b, + .sfa_source = sfa_desc, + .sfb_source = sfb_desc, + .sfa = sfa.value(), + .sfb = sfb.value(), + .sfa_copy = sfa_copy, + .sfb_copy = sfb_copy, + .sfa_target = sfa_dst.value(), + .sfb_target = sfb_dst.value(), + .n = n, + }; +} + +/// Warp 4 drains the A/B ring into one of two independent accumulator stages. +fn runMmaConsumer( + b: *B, + comptime tile: Tile, + k_tiles: usize, + storage: StageStorage, + tiled_mma: cute.Atom, + tmem_holding: V, + active: V, + tiles_m: V, +) void { + const tmem = b.retrieveTmemPtr(tmem_holding, .f32, 16); + const staged = stagedOperands(b, tile, storage); + const staged_mainloop = makeStagedMainloop( + b, + tiled_mma, + tile.n, + tile.stages, + staged.a.value(), + staged.b.value(), + staged.sfa.value(), + staged.sfb.value(), + tmem, + ); + const persistent_clusters = b.gridDim().z; + const div_m = b.fastDivmodCreate(tiles_m); + const div_n = b.fastDivmodCreate(1); + const initial_work = persistentWorkAt(b, b.blockIdx().z, b.constant(.i32, 0), active, tiles_m, div_m, div_n, null); + const carried = .{ + initial_work.m, initial_work.n, initial_work.group, initial_work.valid, + b.constant(.i32, 0), b.constant(.i32, 0), b.constant(.i32, 0), tiled_mma.value(), + b.constant(.i32, 0), b.constant(.i32, 0), b.constant(.i32, 1), persistent_clusters, + initial_work.linear, b.blockIdx().x, b.blockIdx().y, b.constant(.i32, 0), + initial_work.executed, tiles_m, active, div_m, + div_n, + }; + var tiles = b.openWhile(carried, typesOf(carried)); + tiles.yieldBefore(tiles.before_carried[3], tupleOf(tiles.before_carried)); + const first_state: PipelinePosition = .{ + .index = tiles.after_carried[5], + .phase = tiles.after_carried[6], + }; + const first_ready = b.mbarrierTryWait(barrierAt(b, storage.load_barriers, first_state.index), first_state.phase, .{ .scope = .cta }); + const accumulator_state: PipelinePosition = .{ + .index = tiles.after_carried[9], + .phase = tiles.after_carried[10], + }; + const accumulator_stage = accumulator_state.index; + const accumulator_empty = barrierAt(b, storage.accumulator_empty_barriers, accumulator_stage); + b.mbarrierWait(accumulator_empty, accumulator_state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); + + const persistent_mma: cute.Atom = .{ .inner = tiles.after_carried[7].inner, .kernel = b }; + const initial_mma = b.setMmaAccumulate(persistent_mma, b.constant(.i1, false)); + var stages = b.openFor( + 0, + @as(i32, @intCast(k_tiles)), + 1, + .{ first_ready, initial_mma.value(), b.constant(.i32, 0), first_state.index, first_state.phase }, + ); + const k = stages.carried[2]; + const state: PipelinePosition = .{ .index = stages.carried[3], .phase = stages.carried[4] }; + var pending = b.openIf(stages.carried[0].eq(false)); + b.mbarrierWait(barrierAt(b, storage.load_barriers, state.index), state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); + pending.close(); + const accumulator_tmem = accumulatorTmemAt(b, tile, tmem, accumulator_stage); + const mainloop = staged_mainloop.atStage(b, state.index, accumulator_tmem); + mainloop.copyScales(b); + const loop_mma: cute.Atom = .{ .inner = stages.carried[1].inner, .kernel = b }; + const mma = b.setMmaAccumulate(loop_mma, k.ne(0)); + b.gemm( + mma, + mainloop.accumulator, + .{ mainloop.a, mainloop.sfa }, + .{ mainloop.b, mainloop.sfb }, + mainloop.accumulator, + ); + var commit_lane = b.openIf(b.electSync()); + b.tcgen05Commit(barrierAt(b, storage.mma_barriers, state.index), .{}); + commit_lane.close(); + const next_count = stages.carried[2].add(1); + const next_state = advancePipeline(b, state.index, state.phase, tile.stages); + const next_ready = peekBarrierIfRemaining(b, storage.load_barriers, next_count, k_tiles, next_state); + stages.yield(.{ next_ready, mma.value(), next_count, next_state.index, next_state.phase }); + + var full_lane = b.openIf(b.electSync()); + b.tcgen05Commit(barrierAt(b, storage.accumulator_full_barriers, accumulator_stage), .{}); + full_lane.close(); + const next_accumulator_state = advancePipeline(b, accumulator_state.index, accumulator_state.phase, accumulator_stages); + const next_acc_count = tiles.after_carried[8].add(1); + const next_work = advancePersistentWork( + b, + tiles.after_carried[12], + tiles.after_carried[16], + tiles.after_carried[11], + tiles.after_carried[18], + tiles.after_carried[17], + tiles.after_carried[19], + tiles.after_carried[20], + null, + ); + tiles.yieldAfter(.{ + next_work.m, + next_work.n, + next_work.group, + next_work.valid, + stages.results[2], + stages.results[3], + stages.results[4], + stages.results[1], + next_acc_count, + next_accumulator_state.index, + next_accumulator_state.phase, + tiles.after_carried[11], + next_work.linear, + tiles.after_carried[13], + tiles.after_carried[14], + tiles.after_carried[15], + next_work.executed, + tiles.after_carried[17], + tiles.after_carried[18], + tiles.after_carried[19], + tiles.after_carried[20], + }); + + // PipelineUmmaAsync.producer_tail. + var accumulator_tail: PipelinePosition = .{ + .index = tiles.results[9], + .phase = tiles.results[10], + }; + inline for (0..accumulator_stages - 1) |_| { + accumulator_tail = advancePipeline(b, accumulator_tail.index, accumulator_tail.phase, accumulator_stages); + } + b.mbarrierWait(barrierAt(b, storage.accumulator_empty_barriers, accumulator_tail.index), accumulator_tail.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); +} + +/// Epilogue operands; which fields are set depends on the `Epilogue`. +const EpilogueOperands = struct { + expert_ids: cute.View, + alpha: cute.View, + group_sizes: cute.View, + route_inverse: cute.View, + routing_weights: ?cute.View = null, + act_scale: ?cute.View = null, + q: ?cute.View = null, + s: ?cute.View = null, + rows: ?cute.View = null, + direct: bool, + group_rows: i64 = 0, + swiglu_limit: ?f32 = null, +}; + +/// Warps 0-3 drain each TMEM accumulator stage into registers, release it to +/// the MMA warp, then write the tile out according to `epilogue`. +fn runEpilogue( + b: *B, + comptime tile: Tile, + comptime epilogue: Epilogue, + storage: StageStorage, + tmem_holding: V, + active: V, + tiles_m: V, + direct: ?DirectRoutes, + operands: EpilogueOperands, +) void { + const n = tile.n; + const tid = b.threadIdx().x; + const warp = b.makeWarpUniform(b.makeWarpUniform(tid.div(32))); + const tmem = b.retrieveTmemPtr(tmem_holding, .f32, 16); + + const div_m = b.fastDivmodCreate(tiles_m); + const div_n = b.fastDivmodCreate(1); + const initial_work = persistentWorkAt(b, b.blockIdx().z, b.constant(.i32, 0), active, tiles_m, div_m, div_n, direct); + const carried = .{ + initial_work.m, initial_work.n, initial_work.group, initial_work.valid, + b.constant(.i32, 0), b.constant(.i32, 0), b.constant(.i32, 0), initial_work.linear, + initial_work.executed, + }; + var tiles = b.openWhile(carried, .{ + initial_work.m.type_(), + initial_work.n.type_(), + initial_work.group.type_(), + cute.DType.i32.toMlir(b.ctx), + cute.DType.i32.toMlir(b.ctx), + cute.DType.i32.toMlir(b.ctx), + initial_work.linear.type_(), + initial_work.executed.type_(), + }); + tiles.yieldBefore(tiles.before_carried[3], .{ + tiles.before_carried[0], + tiles.before_carried[1], + tiles.before_carried[2], + tiles.before_carried[4], + tiles.before_carried[5], + tiles.before_carried[6], + tiles.before_carried[7], + tiles.before_carried[8], + }); + const tile_m = tiles.after_carried[0]; + const group = tiles.after_carried[2]; + const accumulator_state: PipelinePosition = .{ + .index = tiles.after_carried[4], + .phase = tiles.after_carried[5], + }; + const stage = accumulator_state.index; + // Per-expert scalars do not depend on the accumulator. + const expert = operands.expert_ids.get(.{group}, .i32); + const alpha = operands.alpha.get(.{expert}, .f32); + const act_scale = if (operands.act_scale) |scales| scales.get(.{expert}, .f32) else null; + b.mbarrierWait(barrierAt(b, storage.accumulator_full_barriers, stage), accumulator_state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); + + const accumulator_tmem = accumulatorTmemAt(b, tile, tmem, stage); + const warp_tmem = b.addOffset( + accumulator_tmem, + b.makeIntTupleDivBy(warp.mul(32 * 65536), 4), + b.ptrTy(.f32, .tmem, 16) catch @panic("bad epilogue TMEM pointer"), + ).value(); + const chunk_rows: usize = comptime @intCast(tile.chunkRows()); + const tile_chunks = comptime tile.chunks(); + inline for (0..tile_chunks) |chunk| { + const chunk_tmem = if (chunk == 0) warp_tmem else b.addOffset( + warp_tmem, + b.makeIntTuple(.{@as(i32, @intCast(chunk * chunk_rows))}), + b.ptrTy(.f32, .tmem, offsetAlignment(16, @intCast(chunk * chunk_rows), 4)) catch @panic("bad epilogue TMEM pointer"), + ).value(); + const routed = routedLanes(b, tile, operands, group, chunk); + const values = b.tmemLoadVector(chunk_tmem, @intCast(chunk_rows), .{ .num_dp = 32, .num_b = 32, .num_rep = @intCast(chunk_rows) }); + b.fenceTmemLoad(); + if (chunk + 1 == tile_chunks) { + _ = b.mbarrierArrive(barrierAt(b, storage.accumulator_empty_barriers, stage), .{}); + } + var columns: [@intCast(@min(n, 32))]V = undefined; + inline for (&columns, 0..) |*column, i| column.* = b.vectorExtract(values, i, .i32).bitCast(.f32).mul(alpha); + switch (epilogue) { + .swiglu_nvfp4 => storeTileSwiglu(b, tile, operands, routed, tile_m, group, act_scale.?, &columns, chunk), + .route_rows => storeTileRows(b, tile, operands, routed, tile_m, &columns, chunk), + } + } + const next_accumulator_state = advancePipeline(b, accumulator_state.index, accumulator_state.phase, accumulator_stages); + const next_work = advancePersistentWork(b, tiles.after_carried[6], tiles.after_carried[7], b.gridDim().z, active, tiles_m, div_m, div_n, direct); + tiles.yieldAfter(.{ + next_work.m, + next_work.n, + next_work.group, + next_work.valid, + tiles.after_carried[3].add(1), + next_accumulator_state.index, + next_accumulator_state.phase, + next_work.linear, + next_work.executed, + }); + + var release_warp = b.openIf(warp.eq(0)); + b.relinquishTmemAllocPermit(.{}); + release_warp.close(); + b.barrierCtaSyncAligned(1, 128); + var dealloc_warp = b.openIf(warp.eq(0)); + b.deallocTmem(tmem, tile.tmem_columns, .{}); + dealloc_warp.close(); +} + +/// Per-row routing data of one expert group, spread across a warp: lane `l` +/// holds routed row `l` of the current chunk. +const RoutedLanes = struct { + count: V, + /// Token/top-k route of the lane's row; 0 for padding rows. + route: V, + /// Routing weight of the lane's row, when this projection applies it. + weight: ?V, +}; + +fn routedLanes(b: *B, comptime tile: Tile, operands: EpilogueOperands, group: V, comptime chunk: usize) RoutedLanes { + const weights = operands.routing_weights; + if (operands.direct) return .{ + .count = b.constant(.i32, 1), + .route = group, + .weight = if (weights) |w| w.get(.{group}, .f32) else null, + }; + const lane = b.threadIdx().x.rem(32); + const count = operands.group_sizes.get(.{group}, .i32); + // Lanes past the tile width alias row 0, which is always live. + const chunk_rows = comptime tile.chunkRows(); + const in_chunk = b.select(lane.lt(@as(i32, @intCast(chunk_rows))), lane, b.constant(.i32, 0)); + const row_in_group = if (chunk == 0) in_chunk else in_chunk.add(@as(i32, @intCast(@as(i64, @intCast(chunk)) * chunk_rows))); + const live = row_in_group.lt(count); + const inverse = operands.route_inverse.get(.{group.mul(tile.n).add(row_in_group)}, .i32); + const route = b.select(live, inverse, b.constant(.i32, 0)); + const weight = if (weights) |w| w.get(.{route}, .f32) else null; + return .{ .count = count, .route = route, .weight = weight }; +} + +const swiglu_chunk = 8; + +/// Thread `t` owns output row `128 * tile_m + t` of the interleaved +/// projection: even threads hold a gate and their odd neighbour the matching +/// up value, so each even thread computes one SwiGLU activation per routed +/// row. A warp's 16 activations are exactly one NVFP4 block, whose maximum is +/// a single warp reduction. Activation `a` of row `r` lands in nibble `a` of +/// row `r` of `q`, and its scale at the 128x4 position of `(r, a / 16)`. +fn storeTileSwiglu(b: *B, comptime tile: Tile, operands: EpilogueOperands, routed: RoutedLanes, tile_m: V, group: V, act_scale: V, columns: []const V, comptime chunk: usize) void { + if (operands.direct) + swigluRows(b, tile, operands, routed, tile_m, group, act_scale, columns, 1, false, chunk) + else + swigluRows(b, tile, operands, routed, tile_m, group, act_scale, columns, comptime @intCast(tile.chunkRows()), true, chunk); +} + +fn swigluRows(b: *B, comptime tile: Tile, operands: EpilogueOperands, routed: RoutedLanes, tile_m: V, group: V, act_scale: V, columns: []const V, comptime live: usize, comptime skip_padding: bool, comptime chunk: usize) void { + const n = comptime tile.chunkRows(); + const chunk_first: i32 = comptime @intCast(chunk * @as(usize, @intCast(tile.chunkRows()))); + const tid = b.threadIdx().x; + const lane = tid.rem(32); + const warp = b.makeWarpUniform(b.makeWarpUniform(tid.div(32))); + const is_gate = tid.rem(2).eq(0); + const first_row = group.mul(operands.group_rows); + const f32_type = cute.DType.f32.toMlir(b.ctx); + const row_chunks = std.math.divCeil(usize, live, swiglu_chunk) catch unreachable; + + var activations: [@intCast(n)]V = undefined; + // Lane `c` collects the block maximum of column `c`. + var lane_maximum = b.constant(.f32, 0); + for (0..row_chunks) |row_chunk| { + const first = row_chunk * swiglu_chunk; + const last = @min(first + swiglu_chunk, live); + const guarded = skip_padding and (row_chunk > 0 or chunk > 0); + const types: [swiglu_chunk + 1]*const mlir.Type = @splat(f32_type); + var guard: ?@TypeOf(b.openIfElse(routed.count, tupleOf(types))) = if (guarded) b.openIfElse(routed.count.gt(chunk_first + @as(i32, @intCast(first))), tupleOf(types)) else null; + var chunk_maximum = lane_maximum; + for (columns[first..last], activations[first..last], first..) |value, *activation, column| { + const c: i32 = @intCast(column); + // The reference rounds the projection to BF16. + var gate = value.to(.bf16).to(.f32); + var up_value = b.shuffleXor(value, 1).to(.bf16).to(.f32); + if (operands.swiglu_limit) |limit| { + gate = gate.minimum(limit); + up_value = up_value.maximum(-limit).minimum(limit); + } + // up * SiLU(gate), exp(-g) = 2^(-g * log2 e). + const exp_neg = b.unaryF32("ex2.approx.ftz.f32", gate.mul(-std.math.log2e)); + var result = up_value.mul(gate).mul(b.unaryF32("rcp.approx.ftz.f32", exp_neg.add(1.0))); + if (routed.weight) |weight| result = result.mul(b.shuffleIdx(weight, c)); + // The NVFP4 quantizer reads BF16 activations. + result = result.to(.bf16).to(.f32); + // Odd threads hold no activation and must not raise the maxima. + activation.* = b.select(is_gate, result, b.constant(.f32, 0)); + const maximum = b.warpMaxNonNegative(activation.*.abs()); + chunk_maximum = b.select(lane.eq(c), maximum, chunk_maximum); + } + if (guard) |*g| { + var then_values: [swiglu_chunk + 1]V = @splat(chunk_maximum); + for (activations[first..last], then_values[0 .. last - first]) |activation, *t| t.* = activation; + g.yieldThen(tupleOf(then_values)); + var else_values: [swiglu_chunk + 1]V = @splat(b.constant(.f32, 0)); + else_values[swiglu_chunk] = lane_maximum; + g.yieldElse(tupleOf(else_values)); + for (activations[first..last], g.results[0 .. last - first]) |*activation, result| activation.* = result; + lane_maximum = g.results[swiglu_chunk]; + } else { + lane_maximum = chunk_maximum; + } + } + const scale = nvfp4.blockScale(b, lane_maximum, act_scale); + + // Lane `16 h` writes the word of activations `16 * warp + 8 h ..`. + const word = tile_m.mul(8).add(warp.mul(2)).add(lane.div(16)); + const word_lane = lane.rem(16).eq(0); + for (0..row_chunks) |row_chunk| { + const first = row_chunk * swiglu_chunk; + const last = @min(first + swiglu_chunk, live); + var guard: ?@TypeOf(b.openIf(routed.count)) = if (skip_padding and (chunk > 0 or first > 0)) b.openIf(routed.count.gt(chunk_first + @as(i32, @intCast(first)))) else null; + for (activations[first..last], first..) |activation, column| { + const c: i32 = @intCast(column); + const quantized = activation.mul(b.shuffleIdx(scale.multiplier, c)); + // Gate lane `4i` pairs with gate lane `4i + 2`, then pairs of + // bytes and pairs of halves meet at lanes `8i` and `16i`. + const pair = b.packFp4x2(quantized, b.shuffleXor(quantized, 2)); + const half = pair.bitOr(b.shuffleXor(pair, 4).shl(8)); + const packed_word = half.bitOr(b.shuffleXor(half, 8).shl(16)); + var writer = b.openIf(word_lane); + operands.q.?.set(.{ first_row.add(chunk_first + c), word }, packed_word); + writer.close(); + } + if (guard) |*g| g.close(); + } + + // Lane `c` stores the scale of row `c`, block `4 * tile_m + warp`. + var scale_lane = b.openIf(lane.lt(@as(i32, @intCast(n))).bitAnd(lane.add(chunk_first).lt(routed.count))); + const swizzled = tile_m.mul(512).add(lane.mul(16)).add(chunk * 4).add(warp); + operands.s.?.set(.{ group, swizzled }, scale.bits.to(.i8)); + scale_lane.close(); +} + +/// Store each live routed row at its token/top-k route, so the final +/// reduction reads rows in route order. +fn storeTileRows(b: *B, comptime tile: Tile, operands: EpilogueOperands, routed: RoutedLanes, tile_m: V, columns: []const V, comptime chunk: usize) void { + const chunk_first: i32 = comptime @intCast(chunk * @as(usize, @intCast(tile.chunkRows()))); + const output_column = tile_m.mul(128).add(b.threadIdx().x); + if (operands.direct) { + const value = if (routed.weight) |weight| columns[0].mul(weight) else columns[0]; + operands.rows.?.set(.{ routed.route, output_column }, value.to(.bf16)); + return; + } + var routes: [32]V = undefined; + var weights: [32]?V = @splat(null); + for (routes[0..columns.len], weights[0..columns.len], 0..) |*route, *weight, column| { + route.* = b.shuffleIdx(routed.route, @as(i32, @intCast(column))); + if (routed.weight) |w| weight.* = b.shuffleIdx(w, @as(i32, @intCast(column))); + } + for (columns, routes[0..columns.len], weights[0..columns.len], 0..) |value, route, weight, column| { + var live: ?@TypeOf(b.openIf(routed.count)) = if (column == 0 and chunk == 0) null else b.openIf(routed.count.gt(chunk_first + @as(i32, @intCast(column)))); + const scaled = if (weight) |w| value.mul(w) else value; + operands.rows.?.set(.{ route, output_column }, scaled.to(.bf16)); + if (live) |*l| l.close(); + } +} + +fn buildUmmaDevice(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: Epilogue, d: anytype) void { + const L = Layouts(tile); + const n = tile.n; + const stages = tile.stages; + const shared = b.allocSmemStorage( + .i8, + 1024, + b.layoutType(b.layoutSpec(.{tile.smem_bytes}, .{1})), + 1, + tile.fields(b), + ); + const shared_base = b.getIter( + shared, + b.ptrTy(.i8, .smem, 1024) catch @panic("bad shared base pointer"), + ).value(); + + const tmem_holding = sharedPointerAt(b, shared_base, tile.tmem_holding, .i32, 8); + const storage: StageStorage = .{ + .c = sharedPointerAt(b, shared_base, tile.c, .f32, 1024), + .a = sharedPointerAt(b, shared_base, tile.a, .i8, 1024), + .b = sharedPointerAt(b, shared_base, tile.b, .i8, 1024), + .sfa = sharedPointerAt(b, shared_base, tile.sfa, .f8e4m3fn, 1024), + .sfb = sharedPointerAt(b, shared_base, tile.sfb, .f8e4m3fn, 1024), + .load_barriers = sharedPointerAt(b, shared_base, 0, .i64, 1024), + .mma_barriers = sharedPointerAt(b, shared_base, tile.empty_barriers, .i64, 8), + .accumulator_full_barriers = sharedPointerAt(b, shared_base, tile.accumulator_full, .i64, 16), + .accumulator_empty_barriers = sharedPointerAt(b, shared_base, tile.accumulator_empty, .i64, 16), + }; + + const tid = b.threadIdx().x; + const warp = b.makeWarpUniform(b.makeWarpUniform(tid.div(32))); + const tiled_mma: cute.Atom = .{ .inner = d.mma.inner, .kernel = b }; + + // coordinate tensor -> local_tile -> tiled_mma.partition -> group modes + // -> tma_partition, with the result types of the CuTe DSL. + const m_tiles = @divExact(cfg.m, 128); + const k_tiles = @divExact(cfg.k, k_tile); + const m_step = b.basis(128, 1, .{1}); + const k_step = b.basis(k_tile, 1, .{0}); + const a_local_spec = b.layoutSpec( + .{ 128, k_tile, m_tiles, k_tiles, cfg.experts }, + .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}), m_step, k_step, b.basis(1, 1, .{2}) }, + ); + const b_local_spec = b.layoutSpec( + .{ n, k_tile, 1, k_tiles, cfg.groups }, + .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}), 0, k_step, b.basis(1, 1, .{2}) }, + ); + const all_tiles = .{ cute.AlgebraToken.all, cute.AlgebraToken.all, cute.AlgebraToken.all }; + const tiled_a = b.localTile(d.gA, b.makeTile(&.{ 128, k_tile }), all_tiles, b.coordTensorType(3, a_local_spec), null); + const tiled_b = b.localTile(d.gB, b.makeTile(&.{ n, k_tile }), all_tiles, b.coordTensorType(3, b_local_spec), null); + + const a_mma_spec = b.layoutSpec( + .{ .{ 128, 64 }, 1, 4, m_tiles, k_tiles, cfg.experts }, + .{ .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}) }, 0, b.basis(64, 1, .{0}), m_step, k_step, b.basis(1, 1, .{2}) }, + ); + const b_mma_spec = b.layoutSpec( + .{ .{ n, 64 }, 1, 4, 1, k_tiles, cfg.groups }, + .{ .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}) }, 0, b.basis(64, 1, .{0}), 0, k_step, b.basis(1, 1, .{2}) }, + ); + const mma_a = b.tiledMmaPartition(tiled_mma, tiled_a, .{0}, 0, b.coordTensorType(3, a_mma_spec)); + const mma_b = b.tiledMmaPartition(tiled_mma, tiled_b, .{0}, 1, b.coordTensorType(3, b_mma_spec)); + + const a_grouped_layout = b.staticLayout(b.layoutSpec( + .{ .{ .{ 128, 64 }, 1, 4 }, m_tiles, k_tiles, cfg.experts }, + .{ .{ .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}) }, 0, b.basis(64, 1, .{0}) }, m_step, k_step, b.basis(1, 1, .{2}) }, + )); + const b_grouped_layout = b.staticLayout(b.layoutSpec( + .{ .{ .{ n, 64 }, 1, 4 }, 1, k_tiles, cfg.groups }, + .{ .{ .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}) }, 0, b.basis(64, 1, .{0}) }, b.basis(n, 1, .{1}), k_step, b.basis(1, 1, .{2}) }, + )); + const staged = stagedOperands(b, tile, storage); + const smem_a_grouped = b.withLayout(staged.a, b.staticLayout(b.layoutSpec( + .{ .{ .{ 128, 64 }, 1, 4 }, stages }, + .{ .{ .{ k_tile, 1 }, 0, 64 }, 2 * Tile.a_stage_bytes }, + ))); + const smem_b_grouped = b.withLayout(staged.b, b.staticLayout(b.layoutSpec( + .{ .{ .{ n, 64 }, 1, 4 }, stages }, + .{ .{ .{ k_tile, 1 }, 0, 64 }, 2 * tile.bStageBytes() }, + ))); + const one_cta = b.staticLayout(b.layoutSpec(.{1}, .{0})); + const a_partition_layout = b.layoutSpec(.{ .{ 2 * Tile.a_stage_bytes, 1 }, stages }, .{ .{ 1, 0 }, 2 * Tile.a_stage_bytes }); + const b_partition_layout = b.layoutSpec(.{ .{ 2 * tile.bStageBytes(), 1 }, stages }, .{ .{ 1, 0 }, 2 * tile.bStageBytes() }); + const a_target_layout = b.layoutSpec( + .{ .{ .{ k_tile, 128 }, 1 }, m_tiles, k_tiles, cfg.experts }, + .{ .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0 }, m_step, k_step, b.basis(1, 1, .{2}) }, + ); + const b_target_layout = b.layoutSpec( + .{ .{ .{ k_tile, n }, 1 }, 1, k_tiles, cfg.groups }, + .{ .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0 }, b.basis(n, 1, .{1}), k_step, b.basis(1, 1, .{2}) }, + ); + const partitioned_a = b.tmaPartition( + .{ .inner = d.tma_a.inner, .kernel = b }, + .{0}, + one_cta, + smem_a_grouped, + .{b.withLayout(mma_a, a_grouped_layout)}, + b.memrefTypeFromPointer(L.fp4SmemPtrTy(b), b.layoutType(a_partition_layout)), + &.{b.coordTensorType(3, a_target_layout)}, + ); + const partitioned_b = b.tmaPartition( + .{ .inner = d.tma_b.inner, .kernel = b }, + .{0}, + one_cta, + smem_b_grouped, + .{b.withLayout(mma_b, b_grouped_layout)}, + b.memrefTypeFromPointer(L.fp4SmemPtrTy(b), b.layoutType(b_partition_layout)), + &.{b.coordTensorType(3, b_target_layout)}, + ); + + const exec_a = b.makeExecTma(.{ .inner = d.tma_a.inner, .kernel = b }, b.tmaLoadExecAtomType(L.tmaA(b, cfg))); + const exec_b = b.makeExecTma(.{ .inner = d.tma_b.inner, .kernel = b }, b.tmaLoadExecAtomType(L.tmaB(b, cfg))); + const exec_sfa = b.makeExecTma(.{ .inner = d.tma_sfa.inner, .kernel = b }, b.tmaLoadExecAtomType(L.tmaSf(b))); + const exec_sfb = b.makeExecTma(.{ .inner = d.tma_sfb.inner, .kernel = b }, b.tmaLoadExecAtomType(L.tmaSf(b))); + + const expert_ids: cute.View = .{ .inner = d.expert_ids.inner, .kernel = b }; + const active_count: cute.View = .{ .inner = d.active_count.inner, .kernel = b }; + const direct: ?DirectRoutes = if (cfg.direct) .{ .expert_ids = expert_ids, .routes = @intCast(cfg.groups) } else null; + const active = if (direct) |routes| routes.count(b) else active_count.get(.{0}, .i32); + + // Warp 0 initializes the barriers; one elected lane per barrier array. + var init_warp = b.openIf(warp.eq(0)); + const barrier_arrays = .{ + .{ storage.load_barriers, stages, 1 }, + .{ storage.mma_barriers, stages, 1 }, + .{ storage.accumulator_full_barriers, accumulator_stages, 1 }, + .{ storage.accumulator_empty_barriers, accumulator_stages, 128 }, + }; + inline for (barrier_arrays) |array| { + var lane = b.openIf(b.electSync()); + inline for (0..array[1]) |stage| { + b.mbarrierInit(barrierAt(b, array[0], b.constant(.i32, stage)), array[2]); + } + lane.close(); + } + b.fenceMbarrierInit(); + init_warp.close(); + b.syncThreads(); + + const reduction_tiles: usize = @intCast(k_tiles); + var tma_warp = b.openIf(warp.eq(5)); + runTmaProducer(b, tile, reduction_tiles, storage, active, d.tiles_m, expert_ids, direct, partitioned_a.targets[0], partitioned_b.targets[0], exec_a, exec_b, exec_sfa, exec_sfb); + tma_warp.close(); + + var mma_warp = b.openIf(warp.eq(4)); + b.barrierCtaSyncAligned(2, 160); + runMmaConsumer(b, tile, reduction_tiles, storage, tiled_mma, tmem_holding, active, d.tiles_m); + mma_warp.close(); + + var epilogue_warps = b.openIf(warp.lt(4)); + var alloc_warp = b.openIf(warp.eq(0)); + b.allocTmem(tile.tmem_columns, tmem_holding, .{}); + alloc_warp.close(); + b.barrierCtaSyncAligned(2, 160); + const shared_operands: EpilogueOperands = .{ + .expert_ids = expert_ids, + .alpha = view(b, d.alpha), + .group_sizes = view(b, d.group_sizes), + .route_inverse = view(b, d.route_inverse), + .routing_weights = if (cfg.routing_weights) view(b, d.routing_weights) else null, + .direct = cfg.direct, + .group_rows = cfg.groupRows(), + .swiglu_limit = cfg.swiglu_limit, + }; + var operands = shared_operands; + switch (epilogue) { + .swiglu_nvfp4 => { + operands.act_scale = view(b, d.act_scale); + operands.q = view(b, d.q); + operands.s = view(b, d.s); + }, + .route_rows => operands.rows = view(b, d.rows), + } + runEpilogue(b, tile, epilogue, storage, tmem_holding, active, d.tiles_m, direct, operands); + epilogue_warps.close(); +} + +fn buildHostLaunch(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: Epilogue) cute.FinishError!void { + const L = Layouts(tile); + const host_arg = HostArgs(epilogue); + const n = tile.n; + const a_layout = b.layoutSpec(.{ cfg.m, cfg.k, cfg.experts }, .{ cfg.k, 1, cfg.m * cfg.k }); + const b_layout = b.layoutSpec(.{ cfg.groupRows(), cfg.k, cfg.groups }, .{ cfg.k, 1, cfg.k * cfg.groupRows() }); + // `cute.tile_to_shape` derives that the dynamic strides step whole + // 512-byte scale atoms, and the result type has to carry it. + const sf_stride: cute.ConstrainedDynamic = .{ .divisible_by = 512 }; + const dynamic_sf_layout = b.layoutSpec( + .{ .{ .{ 32, 4 }, cute.AlgebraToken.dynamic }, .{ .{ 16, 4 }, cute.AlgebraToken.dynamic }, .{ 1, cute.AlgebraToken.dynamic } }, + .{ .{ .{ 16, 4 }, sf_stride }, .{ .{ 0, 1 }, 512 }, .{ 0, sf_stride } }, + ); + + // XLA owns packed storage as signless bytes; recast to FP4 here. + const gA = b.makeTensorView(b.recastPointer(b.arg(host_arg.weight), .f4e2m1fn, .gmem, 16), b.staticLayout(a_layout)); + const schedule = b.arg(host_arg.schedule); + const ids = b.makeTensorView(schedule, b.staticLayout(b.layoutSpec(cfg.groups, 1))); + const active_ptr = b.addOffset( + schedule, + b.makeIntTuple(.{2 * cfg.groups}), + b.ptrTy(.i32, .gmem, offsetAlignment(16, 2 * cfg.groups, 4)) catch @panic("bad active count pointer"), + ); + const active = b.makeTensorView(active_ptr, b.staticLayout(b.layoutSpec(1, 1))); + const gB = b.makeTensorView(b.recastPointer(b.arg(host_arg.input_quant), .f4e2m1fn, .gmem, 16), b.staticLayout(b_layout)); + const sf_atom_layout = b.staticLayout(b.layoutSpec( + .{ .{ 32, 4 }, .{ 16, 4 } }, + .{ .{ 16, 4 }, .{ 0, 1 } }, + )); + // Zero-based form of Python CuTe's `(2, 1, 3)` tile-to-shape order. + const sf_order = b.makeIntTuple(.{ 1, 0, 2 }); + const sfa_shape = b.makeShapeValues(.{ b.arg(host_arg.problem_m), b.arg(host_arg.problem_k), b.arg(host_arg.expert_count) }, b.shapeType(.{ cute.AlgebraToken.dynamic, cute.AlgebraToken.dynamic, cute.AlgebraToken.dynamic })); + const sfb_shape = b.makeShapeValues(.{ b.arg(host_arg.problem_n), b.arg(host_arg.problem_k), b.arg(host_arg.problem_l) }, b.shapeType(.{ cute.AlgebraToken.dynamic, cute.AlgebraToken.dynamic, cute.AlgebraToken.dynamic })); + const dynamic_sfa_layout = b.tileToShape(sf_atom_layout, sfa_shape, sf_order, b.layoutType(dynamic_sf_layout)); + const dynamic_sfb_layout = b.tileToShape(sf_atom_layout, sfb_shape, sf_order, b.layoutType(dynamic_sf_layout)); + const gSFA = b.makeTensorView(b.recastPointer(b.arg(host_arg.weight_scale), .f8e4m3fn, .gmem, 16), dynamic_sfa_layout); + const gSFB = b.makeTensorView(b.recastPointer(b.arg(host_arg.input_scale), .f8e4m3fn, .gmem, 16), dynamic_sfb_layout); + + const zero_tmem = b.intToPtr(0, .f8e4m3fn, .tmem, 1); + const mma = b.makeBlockScaledMma(.{ .kind = mma_kind, .n = @intCast(n) }, zero_tmem); + const mma_sfb = b.makeBlockScaledMma(.{ .kind = mma_kind, .n = 128 }, zero_tmem); + + const a_smem = b.makeComposedLayout(L.ab_swizzle, 0, L.aSmem(b, false)); + const b_smem = b.makeComposedLayout(L.ab_swizzle, 0, L.bSmem(b, false)); + const sf_smem = b.staticLayout(L.sfSmem(b, false)); + const a_map = b.staticLayout(b.layoutSpec(.{ .{ 128, 64 }, 1, 4 }, .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0, b.basis(64, 1, .{1}) })); + const b_map = b.staticLayout(b.layoutSpec(.{ .{ n, 64 }, 1, 4 }, .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0, b.basis(64, 1, .{1}) })); + const sf_map = b.staticLayout(b.layoutSpec( + .{ .{ .{ 32, 4 }, .{ 16, 4 } }, 1, 4 }, + .{ .{ .{ b.basis(1, 1, .{ 0, 0, 0 }), b.basis(1, 1, .{ 0, 0, 1 }) }, .{ b.basis(1, 1, .{ 1, 0, 0 }), b.basis(1, 1, .{ 1, 0, 1 }) } }, 0, b.basis(1, 1, .{ 1, 1 }) }, + )); + + const tma_a = b.makeTiledTmaLoadAtom(gA.value(), a_smem.value(), a_map.value(), L.tmaA(b, cfg)); + const tma_b = b.makeTiledTmaLoadAtom(gB.value(), b_smem.value(), b_map.value(), L.tmaB(b, cfg)); + const tma_sfa = b.makeTiledTmaLoadAtom(gSFA.value(), sf_smem.value(), sf_map.value(), L.tmaSf(b)); + const tma_sfb = b.makeTiledTmaLoadAtom(gSFB.value(), sf_smem.value(), sf_map.value(), L.tmaSf(b)); + + const one = b.constant(.i32, 1); + const tiles_m = b.constant(.i32, @divExact(cfg.m, 128)); + const grid_z = b.constant(.i32, @min(@divExact(cfg.m, 128) * cfg.groups, cfg.persistent_ctas)); + const config = b.makeLaunchConfig(.{ + .grid = .{ one, one, grid_z }, + .block = .{ b.constant(.i32, 192), one, one }, + .dynamic_smem = b.kernelSmemSize(deviceName(epilogue)), + .stream = b.cudaStream(), + .cluster = .{ one, one, one }, + .use_pdl = true, + }); + const group_sizes = b.makeTensorView( + b.addOffset(schedule, b.makeIntTuple(.{cfg.groups}), b.ptrTy(.i32, .gmem, offsetAlignment(16, cfg.groups, 4)) catch @panic("bad group size pointer")), + b.staticLayout(b.layoutSpec(cfg.groups, 1)), + ); + const per_expert = b.staticLayout(b.layoutSpec(cfg.experts, 1)); + const mainloop = .{ + ids, active, mma.tiled, mma_sfb.tiled, tma_a.atom, tma_a.tensor, tma_b.atom, tma_b.tensor, + tma_sfa.atom, tma_sfa.tensor, tma_sfb.atom, tma_sfb.tensor, b.makeTensorView(b.arg(host_arg.alpha), per_expert), group_sizes, b.makeTensorView(b.arg(host_arg.route_inverse), b.staticLayout(b.layoutSpec(cfg.routedRows(), 1))), b.makeTensorView(b.arg(host_arg.routing_weights), b.staticLayout(b.layoutSpec(@max(cfg.routes, 1), 1))), + }; + const activations = @divExact(cfg.m, 2); + const launch = switch (epilogue) { + .swiglu_nvfp4 => b.launchEx(deviceName(epilogue), config, mainloop ++ .{ + b.makeTensorView(b.arg(host_arg.act_scale), per_expert), + b.makeTensorView(b.recastPointer(b.arg(host_arg.output), .i32, .gmem, 16), b.staticLayout(b.layoutSpec(.{ cfg.routedRows(), @divExact(activations, 8) }, .{ @divExact(activations, 8), 1 }))), + b.makeTensorView(b.arg(host_arg.scales), b.staticLayout(b.layoutSpec(.{ cfg.groups, @divExact(activations, 16) * 128 }, .{ @divExact(activations, 16) * 128, 1 }))), + tiles_m, + }), + .route_rows => b.launchEx(deviceName(epilogue), config, mainloop ++ .{ + b.makeTensorView(b.arg(host_arg.output), b.staticLayout(b.layoutSpec(.{ @max(cfg.routes, 1), cfg.m }, .{ cfg.m, 1 }))), + tiles_m, + }), + }; + b.returnHostStatus(b.cudaResultStatus(launch)); +} + +test "NVFP4 GEMM programs emit their host launch for every tile width" { + inline for (tile_widths) |n| { + inline for (.{ .{ UpQuantized, Epilogue.swiglu_nvfp4, 4608, 5120 }, .{ DownRows, Epilogue.route_rows, 5120, 2304 } }) |program| { + const cfg: Config = .{ + .experts = 192, + .m = program[2], + .n = n, + .k = program[3], + .groups = 227, + .epilogue = program[1], + .routes = 576, + .direct = n == 8, + .routing_weights = program[1] == .route_rows, + }; + const ir = try program[0].emit(std.testing.allocator, cfg); + defer std.testing.allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, deviceName(program[1])) != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "cuda.launch_ex") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "tma_format = U4") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, std.fmt.comptimePrint("sm100.mma_bs<128x{d}x64", .{n})) != null); + } + } +} diff --git a/zml/kernels/moe/cute/sm10x/reduce.zig b/zml/kernels/moe/cute/sm10x/reduce.zig new file mode 100644 index 00000000..63244ac4 --- /dev/null +++ b/zml/kernels/moe/cute/sm10x/reduce.zig @@ -0,0 +1,124 @@ +//! CuTe top-k reduction behind the persistent MoE GEMMs: sums each token's +//! BF16 route rows. +const std = @import("std"); + +const zml = @import("../../../../zml.zig"); +const cute = zml.kernel.cute; +const B = cute.Builder; +const launchBoundary = @import("common.zig").launchBoundary; + +pub const Config = struct { + tokens: i64, + hidden: i64, + topk: i64, + columns_per_cta: i64, +}; + +// A program rather than a bare kernel so its launch carries the +// programmatic-dependent-launch attribute (see `mxfp4.zig`). +const Topk = cute.Program(Config, .{ + .name = "moe_cute_reduce_topk", + .inputs = &.{ "d", "routes" }, + .outputs = &.{"y"}, + .run = topkProgram, +}); + +const topk_kernel_name = "moe_cute_reduce_topk_kernel"; + +/// Amortize launch overhead for large prefills while retaining enough CTAs +/// to saturate the GPU for decode and small batches. +pub fn columnsPerCta(tokens: i64, hidden: i64) i64 { + const elements = tokens * hidden; + if (tokens > 1 and elements >= 512 * 2048) return 2048; + if (tokens > 1 and elements >= 512 * 1024) return 1024; + // Two columns per thread is the smallest tile: 128 threads, one 32-bit + // word each. + return 256; +} + +/// Sum the `[tokens * topk, hidden]` BF16 route rows of each token into BF16. +/// Routing weights were already applied before the down projection. +/// `routes` is negative for routes this rank does not compute (expert +/// parallelism): their rows were never written. +pub fn topk(d: zml.Tensor, routes: zml.Tensor, cfg: Config) zml.Tensor { + return Topk.call( + .{ .d = d, .routes = routes }, + .{ .y = .init(.{ cfg.tokens, cfg.hidden }, .bf16) }, + .{ .cfg = cfg }, + ).y; +} + +/// Each thread owns two adjacent BF16 columns, read and written as one +/// 32-bit word: a warp then moves a full 128-byte line per access. +fn topkProgram(b: *B, cfg: Config) cute.FinishError!void { + b.beginFunction(topk_kernel_name, .cuda_kernel); + try topkKernel(b, cfg); + b.endFunction(.{ 128, 1, 1 }); + + b.beginFunction(Topk.name, .host); + // The kernel reads and writes two BF16 columns per 32-bit word. + const host = try b.declareArgs(.{ + .d = .{ .ptr = cute.DType.bf16 }, + .routes = .{ .ptr = cute.DType.i32 }, + .y = .{ .ptr = cute.DType.bf16 }, + }); + try launchBoundary(b, topk_kernel_name, .{ + @intCast(@divTrunc(cfg.hidden + cfg.columns_per_cta - 1, cfg.columns_per_cta)), + @intCast(cfg.tokens), + 1, + }, .{ b.recastPointer(host.d, .i32, .gmem, 16), host.routes, b.recastPointer(host.y, .i32, .gmem, 16) }); + b.endFunction(null); +} + +fn topkKernel(b: *B, cfg: Config) cute.FinishError!void { + const pairs = @divExact(cfg.hidden, 2); + const a = try b.declareArgs(.{ + .d = .{ .tensor = .{ .dtype = .i32, .shape = &.{ cfg.tokens * cfg.topk, pairs } } }, + .routes = .{ .tensor = .{ .dtype = .i32, .shape = &.{cfg.tokens * cfg.topk} } }, + .y = .{ .tensor = .{ .dtype = .i32, .shape = &.{ cfg.tokens, pairs } } }, + }); + + // The down projection's rows arrive from a grid this one may overlap. + b.waitForDependency(); + const token = b.blockIdx().y; + const pairs_per_cta = @divExact(cfg.columns_per_cta, 2); + const first_pair = b.blockIdx().x.mul(pairs_per_cta).add(b.threadIdx().x); + for (0..@intCast(@divExact(pairs_per_cta, 128))) |chunk| { + const pair = first_pair.add(chunk * 128); + var in_bounds = b.openIf(pair.lt(pairs)); + var low = b.constant(.f32, 0); + var high = b.constant(.f32, 0); + for (0..@intCast(cfg.topk)) |slot| { + const route = token.mul(cfg.topk).add(slot); + // Routes of other expert-parallel ranks were never written. + var computed = b.openIfElse(a.routes.get(.{route}).ge(0), .{ cute.DType.f32.toMlir(b.ctx), cute.DType.f32.toMlir(b.ctx) }); + const word = a.d.get(.{ route, pair }); + computed.yieldThen(.{ + low.add(word.shl(16).bitCast(.f32)), + high.add(word.bitAnd(@as(i32, @bitCast(@as(u32, 0xffff0000)))).bitCast(.f32)), + }); + computed.yieldElse(.{ low, high }); + low = computed.results[0]; + high = computed.results[1]; + } + const low_bits = low.to(.bf16).to(.f32).bitCast(.i32).shrLogical(16); + const high_bits = high.to(.bf16).to(.f32).bitCast(.i32).bitAnd(@as(i32, @bitCast(@as(u32, 0xffff0000)))); + a.y.set(.{ token, pair }, low_bits.bitOr(high_bits)); + in_bounds.close(); + } + b.launchDependents(); +} + +test "CuTe top-k reduction emits for decode and prefill batches" { + inline for (.{ @as(i64, 1), 6, 96, 16_384 }) |tokens| { + const ir = try Topk.emit(std.testing.allocator, .{ + .tokens = tokens, + .hidden = 5120, + .topk = 6, + .columns_per_cta = columnsPerCta(tokens, 5120), + }); + defer std.testing.allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "cuda.launch_ex") != null); + try std.testing.expect(std.mem.indexOf(u8, ir, "scf.if") != null); + } +} diff --git a/zml/kernels/moe/flashinfer_cutlass/BUILD.bazel b/zml/kernels/moe/flashinfer_cutlass/BUILD.bazel new file mode 100644 index 00000000..c9c5d9a6 --- /dev/null +++ b/zml/kernels/moe/flashinfer_cutlass/BUILD.bazel @@ -0,0 +1,10 @@ +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "flashinfer_cutlass", + srcs = [ + "contract.zig", + "flashinfer_cutlass.zig", + "runtime.zig", + ], +) diff --git a/zml/kernels/moe/flashinfer_cutlass/contract.zig b/zml/kernels/moe/flashinfer_cutlass/contract.zig new file mode 100644 index 00000000..b4b77eaa --- /dev/null +++ b/zml/kernels/moe/flashinfer_cutlass/contract.zig @@ -0,0 +1,40 @@ +pub fn Inputs(comptime T: type) type { + return struct { + hidden_states: T, + fc1_weights: T, + fc2_weights: T, + topk_weights: T, + topk_ids: T, + fc1_act_global: T, + fc1_weight_block: T, + fc1_global: T, + fc2_act_global: T, + fc2_weight_block: T, + fc2_global: T, + }; +} + +pub fn Outputs(comptime T: type) type { + return struct { + output: T, + workspace: T, + }; +} + +pub const Attributes = struct { + runners: u64 = 0, + num_tokens: i64, + hidden_size: i64, + intermediate_size: i64, + num_experts: i32, + top_k: i32, + activation: i32, + enable_pdl: bool, + fc1_act_per_expert: bool, + fc2_act_per_expert: bool, + gemm1_tactic: i32, + gemm2_tactic: i32, +}; + +pub const compute_on_host: bool = false; +pub const name = "flashinfer_cutlass_nvfp4_routed_moe"; diff --git a/zml/kernels/moe/flashinfer_cutlass/flashinfer_cutlass.zig b/zml/kernels/moe/flashinfer_cutlass/flashinfer_cutlass.zig new file mode 100644 index 00000000..92317c7c --- /dev/null +++ b/zml/kernels/moe/flashinfer_cutlass/flashinfer_cutlass.zig @@ -0,0 +1,355 @@ +const std = @import("std"); + +const fi_cutlass_moe = @import("platforms/cuda/flashinfer_cutlass_moe"); +const platforms = @import("platforms"); +const stdx = @import("stdx"); + +const zml = @import("../../../zml.zig"); +const GlobalOptions = @import("../api.zig").Options; +const Activation = @import("../api.zig").Activation; +const LocalRouting = @import("../api.zig").LocalRouting; +const unpackedWeight = @import("../weights.zig").unpackedWeight; +const contract = @import("contract.zig"); +const Attributes = contract.Attributes; +const runtime = @import("runtime.zig"); + +const log = std.log.scoped(.@"zml/moe/flashinfer_cutlass"); + +pub const auto_tactic: i32 = -1; + +pub const Options = struct { + /// Device used at graph-construction time to select the architecture + /// library and size the XLA-owned scratch buffer. + workspace_query_device: i32, + activation: Activation, + enable_pdl: bool, + gemm1_tactic: i32, + /// GEMM2 uses FlashInfer's absolute tactic index. Query tacticCounts() to + /// obtain the first valid GEMM2 index. + gemm2_tactic: i32, + + pub fn fromGlobalOptions(opts: GlobalOptions) Options { + return .{ + .workspace_query_device = 0, + .activation = opts.activation, + .enable_pdl = false, + .gemm1_tactic = auto_tactic, + .gemm2_tactic = auto_tactic, + }; + } +}; + +pub fn validateOptions(opts: GlobalOptions) void { + stdx.debug.assert(!opts.quantize_input, "Optional FP8 input quantization requires the Triton MoE backend", .{}); + stdx.debug.assert(opts.routing_weight_placement == .after_down, "Non-Triton MoE backends require routing weights after the down projection", .{}); + stdx.debug.assert(opts.activation == .swiglu, "flashinfer_cutlass backend only accepts swiglu activation, got {}", .{opts.activation}); + stdx.debug.assert(opts.activation.swiglu.limit == null, "Activation thresholds require the Triton MoE backend", .{}); + stdx.debug.assert(opts.activation.swiglu.bias == null, "flashinfer_cutlass backend requires swiglu bias to be null", .{}); + stdx.debug.assert(opts.activation.swiglu.scale == null, "flashinfer_cutlass backend requires swiglu scale to be null", .{}); +} + +const Inputs = contract.Inputs(zml.Tensor); + +const Outputs = contract.Outputs(zml.Shape); + +fn currentRunners() !*runtime.Runners { + const platform = zml.Compiler.current().platform; + const runners = platform.state.cuda.flashinfer_cutlass_moe_runners; + return runners orelse error.RunnersNotLoaded; +} + +fn cutlassActivation(activation: Activation) i32 { + return switch (activation) { + .gelu => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_GELU, + .relu => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_RELU, + .silu => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_SILU, + .swiglu => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_SWIGLU, + .swiglu_step => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_SWIGLU_STEP, + .geglu => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_GEGLU, + .geglu_tanh => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_GEGLU_TANH, + }; +} + +pub fn call(input_tensors: Inputs, output_shapes: Outputs, attributes: Attributes) zml.ops.ShapeToTensor(Outputs) { + const opts: zml.ops.CustomCallOptions = .{ + .has_side_effect = false, + .output_operand_aliases = comptime zml.ops.customCallOutputOperandAliases(Inputs, Outputs, null), + .compute_on_host = contract.compute_on_host, + }; + + return zml.ops.typedCustomCall( + contract.name, + opts, + input_tensors, + output_shapes, + attributes, + ); +} + +fn computeCapability(platform: *const zml.Platform) !u16 { + const cc = zml.platform.cuda.computeCapability(platform) orelse return error.UnsupportedPlatform; + return switch (cc.sm()) { + 100, 103, 120 => |sm| sm, + else => error.UnsupportedArchitecture, + }; +} + +pub fn isAvailable(platform: *const zml.Platform) bool { + if (platform.target != .cuda) return false; + if (platform.state.cuda.flashinfer_cutlass_moe_runners == null) return false; + _ = computeCapability(platform) catch return false; + return true; +} + +pub fn isNvfp4Supported(platform: *const zml.Platform) bool { + if (platform.state.cuda.flashinfer_cutlass_moe_runners == null) return false; + const compute_capability = computeCapability(platform) catch return false; + return switch (compute_capability) { + 100, 103, 120 => true, + else => false, + }; +} + +fn roundUp(value: i64, alignment: i64) i64 { + return @divTrunc(value + alignment - 1, alignment) * alignment; +} + +pub fn fc1BlockScaleShape( + num_experts: i64, + hidden_size: i64, + intermediate_size: i64, + activation: Activation, +) zml.Shape { + const rows = switch (activation) { + .silu, .gelu, .relu => intermediate_size, + .swiglu, .geglu, .geglu_tanh, .swiglu_step => 2 * intermediate_size, + }; + return .init( + .{ num_experts, roundUp(rows, 128), roundUp(@divExact(hidden_size, 16), 4) }, + .f8e4m3fn, + ); +} + +pub fn fc2BlockScaleShape( + num_experts: i64, + hidden_size: i64, + intermediate_size: i64, +) zml.Shape { + return .init( + .{ num_experts, roundUp(hidden_size, 128), roundUp(@divExact(intermediate_size, 16), 4) }, + .f8e4m3fn, + ); +} + +fn isGlobalOrPerExpertScale(tensor: zml.Tensor, num_experts: i64) bool { + return tensor.dtype() == .f32 and + (tensor.rank() == 0 or (tensor.rank() == 1 and tensor.dim(0) == num_experts)); +} + +fn validateInputs( + hidden_states: zml.Tensor, + fc1_weights: zml.Tensor, + fc2_weights: zml.Tensor, + topk_weights: zml.Tensor, + topk_ids: zml.Tensor, + fc1_act_global: zml.Tensor, + fc1_weight_block: zml.Tensor, + fc1_global: zml.Tensor, + fc2_act_global: zml.Tensor, + fc2_weight_block: zml.Tensor, + fc2_global: zml.Tensor, + options: Options, +) !Attributes { + if (hidden_states.dtype() != .bf16 or + fc1_weights.dtype() != .f4e2m1 or + fc2_weights.dtype() != .f4e2m1 or + topk_weights.dtype() != .f32 or + topk_ids.dtype() != .i32 or + fc1_weight_block.dtype() != .f8e4m3fn or + fc2_weight_block.dtype() != .f8e4m3fn or + fc1_global.dtype() != .f32 or + fc2_global.dtype() != .f32) + { + return error.UnsupportedType; + } + if (hidden_states.rank() != 2 or + fc1_weights.rank() != 3 or + fc2_weights.rank() != 3 or + topk_weights.rank() != 2 or + topk_ids.rank() != 2) + { + return error.InvalidShape; + } + + const tokens = hidden_states.dim(.t); + const hidden_size = hidden_states.dim(.d); + const num_experts = fc1_weights.dim(0); + const fc1_rows = fc1_weights.dim(1); + const intermediate_size = switch (options.activation) { + .silu, .gelu, .relu => fc1_rows, + .swiglu, .geglu, .geglu_tanh, .swiglu_step => @divExact(fc1_rows, 2), + }; + const top_k = topk_ids.dim(.topk); + + if (tokens <= 0 or hidden_size <= 0 or + num_experts <= 0 or intermediate_size <= 0 or + @mod(hidden_size, 16) != 0 or @mod(intermediate_size, 16) != 0 or + top_k <= 0 or top_k > num_experts) + { + return error.InvalidShape; + } + if (fc1_weights.dim(2) != hidden_size or + fc2_weights.dim(0) != num_experts or + fc2_weights.dim(1) != hidden_size or + fc2_weights.dim(2) != intermediate_size or + topk_weights.dim(.t) != tokens or + topk_weights.dim(.topk) != top_k or + topk_ids.dim(.t) != tokens or + !fc1_weight_block.shape().eql(fc1BlockScaleShape( + num_experts, + hidden_size, + intermediate_size, + options.activation, + )) or + !fc2_weight_block.shape().eql(fc2BlockScaleShape( + num_experts, + hidden_size, + intermediate_size, + )) or + !fc1_global.shape().eql(.init(.{num_experts}, .f32)) or + !fc2_global.shape().eql(.init(.{num_experts}, .f32)) or + !isGlobalOrPerExpertScale(fc1_act_global, num_experts) or + !isGlobalOrPerExpertScale(fc2_act_global, num_experts)) + { + return error.InvalidShape; + } + + return .{ + .num_tokens = tokens, + .hidden_size = hidden_size, + .intermediate_size = intermediate_size, + .num_experts = @intCast(num_experts), + .top_k = @intCast(top_k), + .activation = cutlassActivation(options.activation), + .enable_pdl = options.enable_pdl, + .fc1_act_per_expert = fc1_act_global.rank() == 1, + .fc2_act_per_expert = fc2_act_global.rank() == 1, + .gemm1_tactic = options.gemm1_tactic, + .gemm2_tactic = options.gemm2_tactic, + }; +} + +/// Routed NVFP4 CUTLASS MoE. Weights are logical E2M1 tensors in [expert, output, input] order. +/// Block scales must use FlashInfer nvfp4_block_scale_interleave layout +/// BF16 activations are dynamically quantized to NVFP4 by the fused runner before each expert GEMM. +pub fn fusedExpertsNvfp4( + hidden_states: zml.Tensor, + fc1_weights: zml.Tensor, + fc2_weights: zml.Tensor, + topk_weights: zml.Tensor, + topk_ids: zml.Tensor, + fc1_act_global: zml.Tensor, + fc1_weight_block: zml.Tensor, + fc1_global: zml.Tensor, + fc2_act_global: zml.Tensor, + fc2_weight_block: zml.Tensor, + fc2_global: zml.Tensor, + options: Options, +) zml.Tensor { + const runners = currentRunners() catch |e| stdx.debug.panic("Failed to get runners: {}", .{e}); + var attributes = validateInputs( + hidden_states, + fc1_weights, + fc2_weights, + topk_weights, + topk_ids, + fc1_act_global, + fc1_weight_block, + fc1_global, + fc2_act_global, + fc2_weight_block, + fc2_global, + options, + ) catch |e| stdx.debug.panic("Invalid inputs: {}", .{e}); + + attributes.runners = @intFromPtr(runners); + const deviceRunner = runners.ensureRunner(options.workspace_query_device) catch |e| stdx.debug.panic("Failed to ensure runner: {}", .{e}); + const context = runtime.makeContext(attributes); + var requirements = std.mem.zeroes(fi_cutlass_moe.WorkspaceRequirements); + requirements.struct_size = @sizeOf(fi_cutlass_moe.WorkspaceRequirements); + runtime.checkStatus( + deviceRunner.api, + deviceRunner.api.getWorkspaceRequirements( + deviceRunner.runner, + &context, + &requirements, + ), + ) catch |e| stdx.debug.panic("Failed to get workspace requirements: {}", .{e}); + + const result = call( + .{ + .hidden_states = hidden_states, + .fc1_weights = fc1_weights, + .fc2_weights = fc2_weights, + .topk_weights = topk_weights, + .topk_ids = topk_ids, + .fc1_act_global = fc1_act_global, + .fc1_weight_block = fc1_weight_block, + .fc1_global = fc1_global, + .fc2_act_global = fc2_act_global, + .fc2_weight_block = fc2_weight_block, + .fc2_global = fc2_global, + }, + .{ + .output = hidden_states.shape(), + .workspace = .init(.{@as(i64, @intCast(requirements.total_bytes))}, .u8), + }, + attributes, + ); + return result.output; +} + +pub fn fusedExperts( + input: zml.Tensor, + routing: LocalRouting, + gate_up: zml.nn.Linear, + down: zml.nn.Linear, + opts: GlobalOptions, +) zml.Tensor { + if (comptime !platforms.isEnabled(.cuda)) { + @panic("FlashInfer CUTLASS MoE is only supported on CUDA platforms"); + } + + if (gate_up.bias != null or down.bias != null) { + @panic("FlashInfer CUTLASS MoE does not support bias in gate_up or down linear layers"); + } + + const topk_ids = routing.localIds(); + // The runner ignores routes whose local expert id is -1. + const topk_weights = topk_ids.cmp(.GE, .scalar(0, .i32)).select(routing.weights, .scalar(-1, routing.weights.dtype())); + + const runner_options: Options = .fromGlobalOptions(opts); + + const quant_scheme: ?zml.Quantization.Scheme = if (gate_up.quantization) |q| q.scheme else null; + stdx.debug.assert(quant_scheme == .nvfp4, "FlashInfer CUTLASS MoE only supports NVFP4, got {?}", .{quant_scheme}); + + const gate_up_weight_unpacked = unpackedWeight(gate_up); + const down_weight_unpacked = unpackedWeight(down); + + // TODO(Corentin): Do error checking on nvfp4 + // Also, maybe pass `zml.nn.Linear` directly + return fusedExpertsNvfp4( + input, + gate_up_weight_unpacked, + down_weight_unpacked, + topk_weights, + topk_ids, + gate_up.quantization.?.input_scale.?.asMultiplier(), + gate_up.quantization.?.scales, + gate_up.quantization.?.global_scale.?.asMultiplier(), + down.quantization.?.input_scale.?.asMultiplier(), + down.quantization.?.scales, + down.quantization.?.global_scale.?.asMultiplier(), + runner_options, + ); +} diff --git a/zml/kernels/moe/flashinfer_cutlass/runtime.zig b/zml/kernels/moe/flashinfer_cutlass/runtime.zig new file mode 100644 index 00000000..028ef892 --- /dev/null +++ b/zml/kernels/moe/flashinfer_cutlass/runtime.zig @@ -0,0 +1,223 @@ +const std = @import("std"); + +const fi_cutlass_moe = @import("platforms/cuda/flashinfer_cutlass_moe"); +const pjrt = @import("pjrt"); + +const DataType = @import("../../../dtype.zig").DataType; +const contract = @import("contract.zig"); + +const log = std.log.scoped(.@"zml/kernels/moe/flashinfer_cutlass/runtime"); + +// TODO(Corentin): This was coming from Platform but we are now downstream. +// Find a way to solve that. +// Maybe the max_num_devices could be its own module because it only depends on Target. +const max_num_devices = 64; + +// TODO(Corentin): replace that with Shape when its dependencies are untangled and +// it doesn't depend on the platform anymore. +fn count(buffer: *const pjrt.ffi.Buffer) usize { + if (buffer.rank == 0) return @intCast(buffer.dims()[0]); + + var c: usize = 1; + for (buffer.dims()) |dim| c = c * @as(usize, @intCast(dim)); + + return c; +} + +fn byteSize(buffer: *const pjrt.ffi.Buffer) usize { + const dt: DataType = switch (buffer.dtype) { + .invalid => @panic("Found an invalid pjrt buffer"), + .token => @panic("Found an token pjrt buffer"), + inline else => |tag| @field(DataType, @tagName(tag)), + }; + return std.math.divCeil(usize, count(buffer) * dt.bitSizeOf(), 8) catch unreachable; +} + +pub fn handler(call_frame: *pjrt.ffi.CallFrame) callconv(.c) ?*pjrt.ffi.Error { + if (call_frame.registeringHook()) return null; + + // TODO(Corentin): Find a place to abstract that + var input_buffers: contract.Inputs(*const pjrt.ffi.Buffer) = undefined; + inline for (@typeInfo(contract.Inputs(*const pjrt.ffi.Buffer)).@"struct".field_names, 0..) |field_name, i| { + const buf = call_frame.args.buffers()[i]; + @field(input_buffers, field_name) = buf; + } + + var output_buffers: contract.Outputs(*const pjrt.ffi.Buffer) = undefined; + inline for (@typeInfo(contract.Outputs(*const pjrt.ffi.Buffer)).@"struct".field_names, 0..) |field_name, i| { + const buf = call_frame.results.buffers()[i]; + @field(output_buffers, field_name) = buf; + } + + const attributes: contract.Attributes = switch (@typeInfo(contract.Attributes)) { + .void => {}, + .@"struct" => blk: { + var attrs: contract.Attributes = undefined; + inline for (@typeInfo(contract.Attributes).@"struct".field_names, @typeInfo(contract.Attributes).@"struct".field_types) |field_name, Field| { + // TODO: Use getByIndex instead? + const attribute = call_frame.attrs.getByName(.scalar, field_name) orelse + @panic("Attribute not found: " ++ field_name); + @field(attrs, field_name) = attribute.get(Field); + } + break :blk attrs; + }, + else => @compileError("Unsupported attributes type: " ++ @typeName(contract.Attributes)), + }; + + return handlerInner(call_frame, input_buffers, output_buffers, attributes) catch |err| + pjrt.ffi.Error.create(call_frame.api, .unknown, @errorName(err)); +} + +fn handlerInner( + call_frame: *pjrt.ffi.CallFrame, + inputs: contract.Inputs(*const pjrt.ffi.Buffer), + outputs: contract.Outputs(*const pjrt.ffi.Buffer), + attributes: contract.Attributes, +) !?*pjrt.ffi.Error { + const device = try call_frame.ctx.getDeviceOrdinal(call_frame.api); + const runners = try runnersFromAttributes(attributes); + const deviceRunner = try runners.ensureRunner(device); + const context = makeContext(attributes); + + var io = std.mem.zeroes(fi_cutlass_moe.Io); + io.struct_size = @sizeOf(fi_cutlass_moe.Io); + io.input = inputs.hidden_states.data; + io.token_selected_experts = @ptrCast(@alignCast(inputs.topk_ids.data)); + io.token_final_scales = @ptrCast(@alignCast(inputs.topk_weights.data)); + io.fc1_expert_weights = inputs.fc1_weights.data; + io.fc2_expert_weights = inputs.fc2_weights.data; + io.quant_scales[0] = inputs.fc1_act_global.data; + io.quant_scales[1] = inputs.fc1_weight_block.data; + io.quant_scales[2] = inputs.fc1_global.data; + io.quant_scales[3] = inputs.fc2_act_global.data; + io.quant_scales[4] = inputs.fc2_weight_block.data; + io.quant_scales[5] = inputs.fc2_global.data; + io.quant_scale_count = 6; + io.quant_scale_per_expert_mask = + (@as(u32, @intFromBool(attributes.fc1_act_per_expert)) << 0) | + (@as(u32, @intFromBool(attributes.fc2_act_per_expert)) << 3); + io.output = outputs.output.data; + + var workspace = std.mem.zeroes(fi_cutlass_moe.Workspace); + workspace.struct_size = @sizeOf(fi_cutlass_moe.Workspace); + workspace.data = outputs.workspace.data; + workspace.data_bytes = byteSize(outputs.workspace); + + try checkStatus( + deviceRunner.api, + deviceRunner.api.run( + deviceRunner.runner, + &context, + &io, + &workspace, + @ptrCast(call_frame.api.stream(call_frame.ctx)), + attributes.gemm1_tactic, + attributes.gemm2_tactic, + ), + ); + return null; +} + +pub fn makeContext(attributes: contract.Attributes) fi_cutlass_moe.Context { + var context = std.mem.zeroes(fi_cutlass_moe.Context); + context.struct_size = @sizeOf(fi_cutlass_moe.Context); + context.num_tokens = attributes.num_tokens; + context.hidden_size = attributes.hidden_size; + context.intermediate_size = attributes.intermediate_size; + context.num_experts = attributes.num_experts; + context.num_experts_on_rank = attributes.num_experts; + context.top_k = attributes.top_k; + context.tp_size = 1; + context.ep_size = 1; + context.activation = @intCast(attributes.activation); + context.enable_pdl = @intFromBool(attributes.enable_pdl); + context.swizzled_input_sf = 1; + return context; +} + +pub fn checkStatus(api: *const fi_cutlass_moe.Api, status: fi_cutlass_moe.Status) !void { + if (status == fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_STATUS_SUCCESS) return; + if (api.lastError()) |message| { + log.err("FlashInfer CUTLASS MoE failed (status {d}): {s}", .{ + status, + std.mem.span(message), + }); + } else { + log.err("FlashInfer CUTLASS MoE failed with status {d}", .{status}); + } + return switch (status) { + fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_STATUS_INVALID_ARGUMENT => error.InvalidArgument, + fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_STATUS_UNSUPPORTED => error.UnsupportedArchitecture, + fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_STATUS_CUDA_ERROR => error.Cuda, + else => error.FlashinferCutlassMoe, + }; +} + +fn runnersFromAttributes(attributes: contract.Attributes) !*Runners { + if (attributes.runners == 0) return error.RunnersNotLoaded; + return @ptrFromInt(attributes.runners); +} + +pub const Runners = struct { + runners: [max_num_devices]?DeviceRunner = @splat(null), + + pub fn deinit(self: *Runners) void { + for (self.runners) |device_runner| { + if (device_runner) |runner| { + _ = runner.api.runnerDestroy(runner.runner); + } + } + } + + pub fn ensureRunner(self: *Runners, device: i32) !DeviceRunner { + if (device < 0 or device >= max_num_devices) return error.UnsupportedDevice; + + const index: usize = @intCast(device); + if (self.runners[index]) |runner| return runner; + + const api = try fi_cutlass_moe.apiForDevice(device); + const options = runnerOptions(device); + var runner: ?*fi_cutlass_moe.Runner = null; + try checkStatus(api, api.runnerCreate(&options, &runner)); + const result: DeviceRunner = .{ + .api = api, + .runner = runner orelse return error.RunnerInitializationFailed, + }; + self.runners[index] = result; + return result; + } +}; + +const DeviceRunner = struct { + api: *fi_cutlass_moe.Api, + runner: *fi_cutlass_moe.Runner, +}; + +fn runnerOptions(device: i32) fi_cutlass_moe.RunnerOptions { + var options = std.mem.zeroes(fi_cutlass_moe.RunnerOptions); + options.struct_size = @sizeOf(fi_cutlass_moe.RunnerOptions); + options.activation_dtype = fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_DTYPE_BF16; + options.weight_dtype = fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_DTYPE_PACKED_FP4; + options.output_dtype = fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_DTYPE_BF16; + options.device = device; + options.use_fused_finalize = 1; + return options; +} + +pub fn tacticCounts( + runners: *const Runners, + device: i32, +) !struct { gemm1: i32, gemm2: i32 } { + const deviceRunner = try runners.ensureRunner(device); + var gemm1: i32 = 0; + var gemm2: i32 = 0; + try checkStatus( + deviceRunner.api, + deviceRunner.api.getTacticCounts(deviceRunner.runner, &gemm1, &gemm2), + ); + return .{ .gemm1 = gemm1, .gemm2 = gemm2 }; +} + +pub fn load(allocator: std.mem.Allocator, io: std.Io, sm: u16) !void { + try fi_cutlass_moe.load(allocator, io, sm); +} diff --git a/zml/kernels/moe/fly/BUILD.bazel b/zml/kernels/moe/fly/BUILD.bazel new file mode 100644 index 00000000..cb3b2ff4 --- /dev/null +++ b/zml/kernels/moe/fly/BUILD.bazel @@ -0,0 +1,9 @@ +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "fly", + srcs = [ + "fly.zig", + "moe.zig", + ], +) diff --git a/zml/kernels/moe/fly/fly.zig b/zml/kernels/moe/fly/fly.zig new file mode 100644 index 00000000..ba3d3c00 --- /dev/null +++ b/zml/kernels/moe/fly/fly.zig @@ -0,0 +1,294 @@ +const std = @import("std"); + +const stdx = @import("stdx"); + +const zml = @import("../../../zml.zig"); +const Tensor = zml.Tensor; +const Shape = zml.Shape; +const DataType = zml.DataType; +const ExpertsLayout = @import("../api.zig").ExpertsLayout; +const ProjectionLayout = @import("../api.zig").ProjectionLayout; +const RoutingWeightPlacement = @import("../api.zig").RoutingWeightPlacement; +const Activation = @import("../api.zig").Activation; +const Options = @import("../api.zig").Options; +const LocalRouting = @import("../api.zig").LocalRouting; +const fly = @import("moe.zig"); + +const log = std.log.scoped(.@"zml/moe/fused_experts"); + +test { + std.testing.refAllDecls(@This()); +} + +pub fn fusedExperts( + hidden_states: Tensor, + routing_: LocalRouting, + gate_up_: zml.nn.Linear, + down_: zml.nn.Linear, + opts: Options, +) Tensor { + const expert_map_ = routing_.expertMap(); + const layout: ProjectionLayout = if (gate_up_.quantizationScheme() == .mxfp4) .row_interleaved_1 else .concatenated; + const num_tokens = hidden_states.dim(.t); + var launch_config = launchConfigForTokens(num_tokens); + + const gate_up_scheme = gate_up_.quantizationScheme(); + const down_scheme = down_.quantizationScheme(); + if (gate_up_scheme) |scheme| switch (scheme) { + .nvfp4 => stdx.debug.panic("fly backend doesn't support gate_up nvfp4 quantization scheme", .{}), + .fp8_block128 => launch_config.block_size_k = 128, + .fp8_block32 => launch_config.block_size_k = 32, + .mxfp4, .mxfp8, .fp8_per_channel, .fp8_per_tensor => {}, + }; + + var down_launch_config = launchConfigForTokens(num_tokens); + if (down_scheme) |scheme| switch (scheme) { + .nvfp4 => stdx.debug.panic("fly backend doesn't support down nvfp4 quantization scheme", .{}), + .fp8_block128 => down_launch_config.block_size_k = 128, + .fp8_block32 => down_launch_config.block_size_k = 32, + .mxfp4, .mxfp8, .fp8_per_channel, .fp8_per_tensor => {}, + }; + + const hidden = hidden_states.reshape(.{ .token = num_tokens, .in = hidden_states.dim(.d) }).withTags(.{ .token, .in }); + const gate_up = gate_up_.weight.withTags(.{ .expert, .out, .in }); + const down = down_.weight.withTags(.{ .expert, .out, .mid }); + const routing_weights = routing_.weights.reshape(.{ .token = num_tokens, .in = routing_.weights.dim(.topk) }).withTags(.{ .token, .topk }); + const ids = routing_.ids.reshape(.{ .token = num_tokens, .in = routing_.ids.dim(.topk) }).withTags(.{ .token, .topk }); + + stdx.debug.assert(hidden.dtype() == .bf16, "fly backend expected BF16 hidden states, got {}", .{hidden.dtype()}); + if (gate_up_scheme == .mxfp4) { + stdx.debug.assert(gate_up.dtype() == .u8 or gate_up.dtype() == .i8 or gate_up.dtype() == .f4e2m1, "fly backend expected u8, i8 or f4e2m1 gate_up data type when quantization scheme is mxfp4, got {}", .{gate_up.dtype()}); + } else { + stdx.debug.assert(gate_up.dtype() == .bf16 or gate_up.dtype() == .f8e4m3fn or gate_up.dtype() == .f8e4m3fnuz, "fly backend expected gate_up data type to be bf16, fp8e4m3fn or f8e4m3fnuz, got {}", .{gate_up.dtype()}); + } + if (down_scheme == .mxfp4) { + stdx.debug.assert(down.dtype() == .u8 or down.dtype() == .i8 or down.dtype() == .f4e2m1, "fly backend expected u8, i8 or f4e2m1 down data type when quantization scheme is mxfp4, got {}", .{down.dtype()}); + } else { + stdx.debug.assert(down.dtype() == .bf16 or down.dtype() == .f8e4m3fn or down.dtype() == .f8e4m3fnuz, "fly backend expected down data type to be bf16, fp8e4m3fn or f8e4m3fnuz, got {}", .{down.dtype()}); + } + + stdx.debug.assert(routing_weights.dtype() == .f32 or routing_weights.dtype() == .bf16, "fly backend expected f32 or bf16 routing weights, got {}", .{routing_weights.dtype()}); + stdx.debug.assert(ids.dtype() == .i32, "fly backend expected i32 expert ids, got {}", .{ids.dtype()}); + const gate_up_k = gate_up.dim(.in) * @as(i64, if (gate_up_scheme == .mxfp4 and gate_up.dtype() != .f4e2m1) 2 else 1); + const down_k = down.dim(.mid) * @as(i64, if (down_scheme == .mxfp4 and down.dtype() != .f4e2m1) 2 else 1); + stdx.debug.assert(hidden.dim(.in) == gate_up_k, "fly backend expected hidden width to match gate/up input width, got {} and {}", .{ hidden.dim(.in), gate_up_k }); + const activation_reduction: i64 = switch (opts.activation) { + .gelu, .relu, .silu => 1, + .swiglu, .swiglu_step, .geglu, .geglu_tanh => 2, + }; + stdx.debug.assert(@rem(gate_up.dim(.out), activation_reduction) == 0, "fly backend expected gate/up output width to be divisible by {}, got {}", .{ activation_reduction, gate_up.dim(.out) }); + stdx.debug.assert(down_k == @divFloor(gate_up.dim(.out), activation_reduction), "fly backend expected down input width to match activated width, got {} and {}", .{ down_k, @divFloor(gate_up.dim(.out), activation_reduction) }); + stdx.debug.assert(ids.dim(.token) == hidden.dim(.token) and routing_weights.dim(.token) == hidden.dim(.token), "fly backend expected routing ids and weights to match hidden token count, got {}, {} and {}", .{ ids.dim(.token), routing_weights.dim(.token), hidden.dim(.token) }); + stdx.debug.assert(ids.dim(.topk) == routing_weights.dim(.topk), "fly backend expected routing ids and weights to have matching top-k dimensions, got {} and {}", .{ ids.dim(.topk), routing_weights.dim(.topk) }); + stdx.debug.assert(gate_up.dim(.expert) == down.dim(.expert), "fly backend expected gate/up and down expert counts to match, got {} and {}", .{ gate_up.dim(.expert), down.dim(.expert) }); + if (expert_map_) |expert_map| + stdx.debug.assert(expert_map.dtype() == .i32 and expert_map.rank() == 1, "fly backend expected a rank-1 i32 expert map, got rank {} and dtype {}", .{ expert_map.rank(), expert_map.dtype() }); + + const num_experts = if (expert_map_) |expert_map| expert_map.dim(.expert) else gate_up.dim(.expert); + const routing = prepareRouting(ids, num_experts, @intCast(launch_config.block_size_m)); + + const expert_ids = if (expert_map_) |expert_map| + expert_map.gather(.{ .expert = routing.expert_ids }, .{}).withTags(.{.g}) + else + routing.expert_ids; + + const hidden_quant, const input_scale = prepareInput(hidden, gate_up_scheme, gate_up.dtype(), opts.quantize_input); + + const gate_up_out = gemm(.{ + .input = hidden_quant, + .weight = gate_up, + .bias = gate_up_.bias, + .input_scale = input_scale, + .weight_scale = gate_up_.quantizationScales(), + .routing = routing, + .expert_ids = expert_ids, + .launch_config = launch_config, + .quant_scheme = gate_up_scheme, + .top_k = @intCast(ids.dim(.topk)), + .output_shape = Shape.init(.{ .token = routing.num_assignments, .out = gate_up.dim(.out) }, .bf16), + }); + + var activated = opts.activation.applyActivation(gate_up_out, layout); + if (opts.routing_weight_placement == .before_down) { + const weights = routing_weights.reshape(.{ .token = routing.num_assignments }).convert(.f32); + activated = activated.mul(weights.broad(activated.shape())); + } + const activated_quant, const down_input_scale = prepareInput(activated, down_scheme, down.dtype(), opts.quantize_input); + + const down_out = gemm(.{ + .input = activated_quant, + .weight = down, + .bias = down_.bias, + .input_scale = down_input_scale, + .weight_scale = down_.quantizationScales(), + .routing_weights = if (opts.routing_weight_placement == .after_down) routing_weights else null, + .routing = routing, + .expert_ids = expert_ids, + .launch_config = down_launch_config, + .quant_scheme = down_scheme, + .top_k = 1, + .output_shape = Shape.init(.{ .token = num_tokens, .topk = ids.dim(.topk), .out = down.dim(.out) }, .bf16), + }); + + const output = down_out.convert(.f32).sum(.topk).squeeze(.topk).convert(.bf16); + + return output.reshape(.{ .t = num_tokens, .d = down.dim(.out) }); +} + +// TODO(Corentin): unused for now +pub fn supports(hidden_states: Tensor, routing: LocalRouting, gate_up_: zml.nn.Linear, down_: zml.nn.Linear, opts: Options) bool { + if (hidden_states.dtype() != .bf16 or gate_up_.quantizationScheme() != .mxfp4 or + down_.quantizationScheme() != .mxfp4 or gate_up_.bias != null or down_.bias != null or + opts.routing_weight_placement != .before_down) return false; + const gate_up = gate_up_.weight; + const down = down_.weight; + if (gate_up.rank() != 3 or down.rank() != 3) return false; + const gate_up_scale = gate_up_.quantizationScales() orelse return false; + const down_scale = down_.quantizationScales() orelse return false; + const experts = routing.global_num_experts; + if (experts < 1 or experts > 512) return false; + var max_width: i64 = hidden_states.dim(.d); + inline for (.{ .{ gate_up, gate_up_scale }, .{ down, down_scale } }) |projection| { + const weight, const scale = projection; + const packing_factor: i64 = switch (weight.dtype()) { + .u8, .i8 => 2, + .f4e2m1 => 1, + else => return false, + }; + const n = weight.dim(1); + const k = weight.dim(2) * packing_factor; + if (!fly.Mfma.supportsShape(n, k)) return false; + max_width = @max(max_width, n); + if (weight.dim(0) < 1 or weight.dim(0) > 512 or + scale.rank() != 3 or scale.dim(0) != weight.dim(0) or scale.dim(1) != n or scale.dim(2) != @divExact(k, 32)) return false; + switch (scale.dtype()) { + .u8, .i8, .f8e8m0 => {}, + else => return false, + } + } + const routes = hidden_states.dim(.t) * routing.ids.dim(.topk); + return routes > 0 and routes <= @divTrunc(std.math.maxInt(i32), max_width); +} + +pub fn prepareInput(x: Tensor, scheme: ?zml.Quantization.Scheme, _: zml.DataType, _: bool) struct { Tensor, ?Tensor } { + std.debug.assert(scheme == .mxfp4); + return .{ x.convert(.bf16), null }; +} + +pub const GemmOptions = struct { + input: Tensor, + weight: Tensor, + bias: ?Tensor = null, + input_scale: ?Tensor = null, + weight_scale: ?Tensor = null, + routing_weights: ?Tensor = null, + routing: Routing, + expert_ids: Tensor, + launch_config: LaunchConfig, + quant_scheme: ?zml.Quantization.Scheme = null, + top_k: usize, + output_shape: Shape, +}; + +pub fn gemm(opts: GemmOptions) zml.Tensor { + zml.stdx.debug.assert(opts.quant_scheme == .mxfp4 and opts.bias == null and opts.routing_weights == null and opts.input_scale == null, "Fly MoE requires MXFP4 weights, BF16 activations and routing weights before the down projection", .{}); + const weight = if (opts.weight.dtype() == .f4e2m1) + opts.weight.reshape(opts.weight.shape().setDim(2, @divExact(opts.weight.dim(2), 2)).append(.{ .nibble = 2 })).bitCast(.u8) + else + opts.weight; + return fly.Mfma.call(opts.input, weight, opts.weight_scale.?.bitCast(.u8), opts.routing.sorted_token_ids, opts.expert_ids, opts.routing.num_tokens_post_padded, opts.output_shape, opts.top_k, opts.routing.naive_block_assignment, opts.launch_config.block_size_m); +} + +pub const Routing = struct { + sorted_token_ids: Tensor, + expert_ids: Tensor, + num_tokens_post_padded: Tensor, + max_num_tokens_padded: i64, + num_assignments: i64, + naive_block_assignment: bool, +}; + +pub fn prepareRouting(topk_ids: Tensor, num_experts: i64, block_size_m: i64) Routing { + std.debug.assert(num_experts > 0 and num_experts <= 512); + std.debug.assert(block_size_m > 0 and block_size_m <= 128 and @mod(block_size_m, 16) == 0); + const ids = topk_ids.withTags(.{ .token, .topk }).convert(.i32); + const num_assignments = ids.dim(.token) * ids.dim(.topk); + // GEMV handles up to six routes; MFMA shares weights across sorted rows. + const naive_block_assignment = num_assignments <= 6; + const max_num_tokens_padded = if (naive_block_assignment or num_assignments < num_experts) + num_assignments * block_size_m + else + (std.math.divCeil(i64, num_assignments + num_experts * (block_size_m - 1), block_size_m) catch unreachable) * block_size_m; + const sorted_token_ids, const expert_ids, const num_tokens_post_padded = if (naive_block_assignment) blk: { + break :blk .{ + Tensor.zeroes(Shape.init(.{ .g = 1 }, .i32)), + ids.reshape(.{ .g = num_assignments }), + Tensor.constant(.{ .i32 = @as(i32, @intCast(max_num_tokens_padded)) }).reshape(.{1}), + }; + } else blk: { + const flat_experts = ids.reshape(.{ .g = num_assignments }); + const aligned = fly.Align.call( + flat_experts, + Tensor.zeroes(Shape.init(.{ .g = max_num_tokens_padded }, .i32)), + Tensor.zeroes(Shape.init(.{ .g = @divExact(max_num_tokens_padded, block_size_m) }, .i32)), + Tensor.zeroes(Shape.init(.{ .g = 1 }, .i32)), + Tensor.zeroes(Shape.init(.{ .g = num_experts + 1 }, .i32)), + block_size_m, + ); + const sorted = fly.Sort.call(flat_experts, aligned.sorted_token_ids, aligned.cumsum); + break :blk .{ sorted.sorted_token_ids, aligned.expert_ids, aligned.num_tokens_post_pad }; + }; + return .{ + .sorted_token_ids = sorted_token_ids, + .expert_ids = expert_ids, + .num_tokens_post_padded = num_tokens_post_padded, + .max_num_tokens_padded = max_num_tokens_padded, + .num_assignments = num_assignments, + .naive_block_assignment = naive_block_assignment, + }; +} + +// ============================================================================= +// Config / validation helpers +// ============================================================================= + +const LaunchConfig = struct { + block_size_m: usize, + block_size_n: usize, + block_size_k: usize, + group_size_m: usize, + num_warps: i32, + num_stages: i32, +}; + +const launch_configs = [_]struct { tokens: i64, config: LaunchConfig }{ + .{ .tokens = 1, .config = .{ .block_size_m = 16, .block_size_n = 32, .block_size_k = 64, .group_size_m = 1, .num_warps = 4, .num_stages = 4 } }, + .{ .tokens = 2, .config = .{ .block_size_m = 16, .block_size_n = 32, .block_size_k = 64, .group_size_m = 1, .num_warps = 4, .num_stages = 4 } }, + .{ .tokens = 4, .config = .{ .block_size_m = 16, .block_size_n = 32, .block_size_k = 64, .group_size_m = 1, .num_warps = 4, .num_stages = 3 } }, + .{ .tokens = 8, .config = .{ .block_size_m = 16, .block_size_n = 128, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 3 } }, + .{ .tokens = 16, .config = .{ .block_size_m = 16, .block_size_n = 64, .block_size_k = 64, .group_size_m = 64, .num_warps = 4, .num_stages = 5 } }, + .{ .tokens = 24, .config = .{ .block_size_m = 16, .block_size_n = 64, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 2 } }, + .{ .tokens = 32, .config = .{ .block_size_m = 16, .block_size_n = 32, .block_size_k = 128, .group_size_m = 1, .num_warps = 4, .num_stages = 2 } }, + .{ .tokens = 48, .config = .{ .block_size_m = 16, .block_size_n = 32, .block_size_k = 128, .group_size_m = 64, .num_warps = 4, .num_stages = 2 } }, + .{ .tokens = 64, .config = .{ .block_size_m = 16, .block_size_n = 64, .block_size_k = 128, .group_size_m = 1, .num_warps = 4, .num_stages = 2 } }, + .{ .tokens = 96, .config = .{ .block_size_m = 16, .block_size_n = 128, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 3 } }, + .{ .tokens = 128, .config = .{ .block_size_m = 16, .block_size_n = 256, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 2 } }, + .{ .tokens = 256, .config = .{ .block_size_m = 16, .block_size_n = 256, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 2 } }, + .{ .tokens = 512, .config = .{ .block_size_m = 32, .block_size_n = 128, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 3 } }, + .{ .tokens = 1024, .config = .{ .block_size_m = 64, .block_size_n = 128, .block_size_k = 64, .group_size_m = 1, .num_warps = 4, .num_stages = 3 } }, + .{ .tokens = 1536, .config = .{ .block_size_m = 64, .block_size_n = 128, .block_size_k = 64, .group_size_m = 1, .num_warps = 4, .num_stages = 3 } }, + .{ .tokens = 2048, .config = .{ .block_size_m = 128, .block_size_n = 128, .block_size_k = 64, .group_size_m = 16, .num_warps = 8, .num_stages = 3 } }, + .{ .tokens = 3072, .config = .{ .block_size_m = 128, .block_size_n = 256, .block_size_k = 64, .group_size_m = 1, .num_warps = 8, .num_stages = 4 } }, + .{ .tokens = 4096, .config = .{ .block_size_m = 128, .block_size_n = 256, .block_size_k = 64, .group_size_m = 16, .num_warps = 8, .num_stages = 4 } }, +}; + +fn launchConfigForTokens(num_tokens: i64) LaunchConfig { + var best = launch_configs[0]; + for (launch_configs[1..]) |candidate| { + // The table is ascending, so ties retain the smaller token bucket. + if (@abs(candidate.tokens - num_tokens) < @abs(best.tokens - num_tokens)) { + best = candidate; + } + } + return best.config; +} diff --git a/zml/kernels/moe/fly/moe.zig b/zml/kernels/moe/fly/moe.zig new file mode 100644 index 00000000..3cb82b96 --- /dev/null +++ b/zml/kernels/moe/fly/moe.zig @@ -0,0 +1,747 @@ +const std = @import("std"); +const zml = @import("../../../zml.zig"); +const Tensor = zml.Tensor; +const Shape = zml.Shape; +const fly = zml.kernel.fly; +const Builder = fly.Builder; +const Value = fly.Value; + +pub const Align = struct { + const Result = struct { + sorted_token_ids: Tensor, + expert_ids: Tensor, + num_tokens_post_pad: Tensor, + cumsum: Tensor, + }; + + fn supports(ids: Tensor, sorted: Tensor, experts: Tensor, total: Tensor, cumsum: Tensor, block_size: i64) bool { + for ([_]Tensor{ ids, sorted, experts, total, cumsum }) |tensor| { + if (tensor.rank() != 1 or tensor.dtype() != .i32) return false; + } + const num_experts = cumsum.dim(0) - 1; + if (num_experts < 1 or num_experts > 512 or total.dim(0) != 1 or ids.dim(0) > std.math.maxInt(i32) or + block_size <= 0 or block_size > 128 or @mod(block_size, 16) != 0) return false; + const capacity_bound = if (ids.dim(0) < num_experts) + ids.dim(0) * block_size + else + @divTrunc(ids.dim(0) + num_experts * (block_size - 1) + block_size - 1, block_size) * block_size; + return capacity_bound <= std.math.maxInt(i32) and sorted.dim(0) >= capacity_bound and + sorted.dim(0) <= std.math.maxInt(i32) and @mod(sorted.dim(0), block_size) == 0 and + experts.dim(0) == @divTrunc(sorted.dim(0), block_size); + } + + /// All four outputs alias their corresponding input buffers. Invalid expert IDs + /// are ignored, and all unused expert blocks are marked -1. Prefix ends in LDS + /// let each thread find its output block's expert without scanning every expert. + pub fn call(ids: Tensor, sorted: Tensor, experts: Tensor, total: Tensor, cumsum: Tensor, block_size: i64) Result { + std.debug.assert(Align.supports(ids, sorted, experts, total, cumsum, block_size)); + const cfg: Config = .{ + .routes = ids.dim(0), + .experts = cumsum.dim(0) - 1, + .capacity = sorted.dim(0), + .blocks = experts.dim(0), + .block_size = block_size, + .padding = block_size - 1, + }; + const result = Kernel.call(.{ .ids = ids, .sorted = sorted, .experts = experts, .total = total, .cumsum_in = cumsum }, .{ .sorted_token_ids = sorted.shape(), .expert_ids = experts.shape(), .num_tokens_post_pad = total.shape(), .cumsum = cumsum.shape() }, .{ + .cfg = cfg, + .threads = 512, + .grid = .{ 2, 1, 1 }, + .output_operand_aliases = .{ .sorted_token_ids = .sorted, .expert_ids = .experts, .num_tokens_post_pad = .total, .cumsum = .cumsum_in }, + }); + return .{ .sorted_token_ids = result.sorted_token_ids, .expert_ids = result.expert_ids, .num_tokens_post_pad = result.num_tokens_post_pad, .cumsum = result.cumsum }; + } + + const Config = struct { + routes: i64, + experts: i64, + capacity: i64, + blocks: i64, + block_size: i64, + padding: i64, + }; + + const Kernel = zml.kernel.fly.Kernel(Config, .{ + .name = "fly_moe_align", + .inputs = &.{ "ids", "sorted", "experts", "total", "cumsum_in" }, + .outputs = &.{ "sorted_token_ids", "expert_ids", "num_tokens_post_pad", "cumsum" }, + .run = run, + }); + + fn run(b: *Builder, cfg: Config) fly.FinishError!void { + const args = Kernel.args(b); + const tid = b.threadId(.x); + const tid64 = tid.to(.i64); + const zero = b.constant(.i32, 0); + const one = b.constant(.i32, 1); + const experts = b.constant(.i32, cfg.experts); + // Distinct arrays match the histogram, inclusive ends and wave totals. + const counts = b.sharedArray(.i32, 512, 16); + const ends = b.sharedArray(.i32, 512, 16); + const waves = b.sharedArray(.i32, 8, 16); + var fill = b.openIfElse(b.blockId(.x).ne(0), .{}); + { + var positions = b.openFor(tid64, cfg.capacity, 512, .{}); + { + args.sorted_token_ids.set(positions.iv, b.constant(.i32, cfg.routes)); + positions.close(); + } + fill.elseBranch(); + } + { + counts.set(tid, zero); + b.barrier(); + var routes = b.openFor(tid64, cfg.routes, 512, .{}); + { + const id = args.ids.at(routes.iv); + var valid_id = b.openIf(id.ge(0).bitAnd(id.lt(cfg.experts))); + { + _ = b.ptrAtomicAdd(counts.emitIter().addOffset(id), one, .workgroup); + valid_id.close(); + } + routes.close(); + } + b.barrier(); + + const padded = counts.at(tid).add(cfg.padding).divUnsigned(cfg.block_size).mul(cfg.block_size); + const lane = tid.remUnsigned(64); + const wave = tid.divUnsigned(64); + var prefix = padded; + inline for (.{ 1, 2, 4, 8, 16, 32 }) |shift| { + const peer = prefix.shuffle(.up, shift, 64); + prefix = prefix.add(lane.cmpUnsigned(.ge, shift).select(peer, zero)); + } + var last_lane = b.openIf(lane.eq(63)); + { + waves.set(wave, prefix); + last_lane.close(); + } + b.barrier(); + + var sums = b.openFor(zero, 8, 1, .{ zero, zero }); + { + const value = waves.at(sums.iv); + const preceding = sums.carried[0].add(sums.iv.cmpUnsigned(.lt, wave).select(value, zero)); + const total = sums.carried[1].add(value); + sums.yield(.{ preceding, total }); + } + const end = sums.results[0].add(prefix); + const start = end.sub(padded); + ends.set(tid, end); + var has_expert = b.openIf(tid.cmpUnsigned(.lt, cfg.experts)); + { + args.cumsum.set(tid64, start); + has_expert.close(); + } + var first = b.openIf(tid.eq(0)); + { + args.cumsum.set(b.constant(.i64, cfg.experts), sums.results[1]); + args.num_tokens_post_pad.set(b.constant(.i64, 0), sums.results[1]); + first.close(); + } + b.barrier(); + + var blocks = b.openFor(tid64, cfg.blocks, 512, .{}); + { + const offset = blocks.iv.to(.i32).mul(cfg.block_size); + var block_expert = b.openIfElse(offset.cmpUnsigned(.lt, sums.results[1]), .{zero}); + { + // Upper bound of inclusive expert ends handles empty experts without a scan. + var bounds = b.openWhile(.{ zero, experts }, .{ zero, zero }); + { + bounds.yieldBefore(bounds.before_carried[0].cmpUnsigned(.lt, bounds.before_carried[1]), .{ bounds.before_carried[0], bounds.before_carried[1] }); + } + { + const lo = bounds.after_carried[0]; + const hi = bounds.after_carried[1]; + const mid = lo.add(hi).shrU(1); + const advance = ends.at(mid).cmpUnsigned(.le, offset); + bounds.yieldAfter(.{ advance.select(mid.add(1), lo), advance.select(hi, mid) }); + } + block_expert.yieldThen(.{bounds.results[0]}); + } + { + block_expert.yieldElse(.{b.constant(.i32, -1)}); + } + args.expert_ids.set(blocks.iv, block_expert.results[0]); + blocks.close(); + } + fill.close(); + } + } +}; + +pub const Sort = struct { + const Result = struct { + sorted_token_ids: Tensor, + cumsum: Tensor, + }; + + fn supports(ids: Tensor, sorted_token_ids: Tensor, cumsum: Tensor) bool { + return ids.rank() == 1 and sorted_token_ids.rank() == 1 and cumsum.rank() == 1 and + ids.dtype() == .i32 and sorted_token_ids.dtype() == .i32 and cumsum.dtype() == .i32 and + cumsum.dim(0) >= 2 and cumsum.dim(0) <= std.math.maxInt(i32) and + ids.dim(0) <= std.math.maxInt(i32) and sorted_token_ids.dim(0) >= ids.dim(0); + } + + /// cumsum contains one starting offset per expert followed by the total padded + /// capacity. Each valid route atomically advances its expert's cursor; ordering + /// within an expert is unspecified. Invalid expert IDs leave both buffers alone. + pub fn call(ids: Tensor, sorted_token_ids: Tensor, cumsum: Tensor) Result { + std.debug.assert(Sort.supports(ids, sorted_token_ids, cumsum)); + if (ids.dim(0) == 0) return .{ .sorted_token_ids = sorted_token_ids, .cumsum = cumsum }; + const routes = ids.dim(0); + const grid = @min(@divTrunc(routes + 255, 256), 65535); + const cfg: Config = .{ .routes = routes, .experts = cumsum.dim(0) - 1, .step = grid * 256 }; + const result = Kernel.call(.{ .ids = ids, .sorted = sorted_token_ids, .cumsum_in = cumsum }, .{ .sorted_token_ids = sorted_token_ids.shape(), .cumsum = cumsum.shape() }, .{ + .cfg = cfg, + .threads = 256, + .grid = .{ @intCast(grid), 1, 1 }, + .output_operand_aliases = .{ .sorted_token_ids = .sorted, .cumsum = .cumsum_in }, + }); + return .{ .sorted_token_ids = result.sorted_token_ids, .cumsum = result.cumsum }; + } + + const Config = struct { + routes: i64, + experts: i64, + step: i64, + }; + + const Kernel = zml.kernel.fly.Kernel(Config, .{ + .name = "fly_count_and_sort_expert_tokens", + .inputs = &.{ "ids", "sorted", "cumsum_in" }, + .outputs = &.{ "sorted_token_ids", "cumsum" }, + .run = run, + }); + + fn run(b: *Builder, cfg: Config) fly.FinishError!void { + const args = Kernel.args(b); + const start = b.blockId(.x).to(.i64).mul(256).add(b.threadId(.x).to(.i64)); + var routes = b.openFor(start, cfg.routes, cfg.step, .{}); + { + const id = args.ids.at(routes.iv); + var valid = b.openIf(id.ge(0).bitAnd(id.lt(cfg.experts))); + { + const cursor = args.cumsum.emitIter().addOffset(id.to(.i64)); + const rank = b.ptrAtomicAdd(cursor, b.constant(.i32, 1), .agent); + args.sorted_token_ids.set(rank.to(.i64), routes.iv.to(.i32)); + valid.close(); + } + routes.close(); + } + } +}; + +pub const Mfma = struct { + /// MFMA loads full 64-column by 128-reduction tiles without tail masks. + pub fn supportsShape(n: i64, k: i64) bool { + return n > 0 and k > 0 and @mod(n, 64) == 0 and @mod(k, 128) == 0; + } + + /// The caller owns route localization, activation and reduction. + /// `expert_ids` contains local bank indices, including -1 for remote experts. + /// Sorted routes must contain a real prefix followed by padding per expert. + /// `block_m` is their alignment; native tiles contain 16 or 32 rows. + pub fn call( + input: Tensor, + weight: Tensor, + scales: Tensor, + sorted_ids: Tensor, + expert_ids: Tensor, + padded_count: Tensor, + output: zml.Shape, + topk: usize, + naive: bool, + block_m: usize, + ) Tensor { + std.debug.assert(input.rank() == 2 and input.dtype() == .bf16 and weight.rank() == 3 and + (weight.dtype() == .u8 or weight.dtype() == .i8) and scales.rank() == 3 and scales.dtype() == .u8 and + sorted_ids.rank() == 1 and sorted_ids.dtype() == .i32 and expert_ids.rank() == 1 and expert_ids.dtype() == .i32 and + padded_count.count() == 1 and padded_count.dtype() == .i32 and output.rank() >= 2 and output.dtype() == .bf16 and + topk > 0 and block_m > 0 and block_m % 16 == 0); + const n = weight.dim(1); + const k = input.dim(1); + std.debug.assert(supportsShape(n, k)); + std.debug.assert(input.dim(0) > 0 and weight.dim(0) > 0 and weight.dim(2) == @divExact(k, 2) and + scales.dim(0) == weight.dim(0) and scales.dim(1) == n and scales.dim(2) == @divExact(k, 32) and + output.dim(-1) == n); + // Keep packed-word counts and route/output indices within signed i32. + std.debug.assert(weight.count() / 4 <= std.math.maxInt(i32) and output.count() <= std.math.maxInt(i32) and + sorted_ids.count() <= std.math.maxInt(i32)); + const routes: i64 = @intCast(output.count() / @as(usize, @intCast(n))); + std.debug.assert(topk <= routes and @mod(@as(usize, @intCast(routes)), topk) == 0 and + input.dim(0) == @divTrunc(routes, @as(i64, @intCast(topk)))); + const capacity: i64 = @intCast(sorted_ids.count()); + const blocks: i64 = @intCast(expert_ids.count()); + if (naive) { + std.debug.assert(blocks == routes); + } else { + std.debug.assert(capacity > 0 and block_m <= capacity and @mod(capacity, @as(i64, @intCast(block_m))) == 0 and + blocks == @divExact(capacity, @as(i64, @intCast(block_m)))); + } + if (naive and routes <= 6) return Gemv.call(input, weight, scales, expert_ids, output, topk); + return callMfma(input, weight, scales, sorted_ids, expert_ids, padded_count, output, topk, naive, block_m); + } + + fn callMfma(input: Tensor, weight: Tensor, scales: Tensor, sorted_ids: Tensor, expert_ids: Tensor, padded_count: Tensor, output: zml.Shape, topk: usize, naive: bool, block_m: usize) Tensor { + const n = weight.dim(1); + const k = input.dim(1); + const routes = input.dim(0) * @as(i64, @intCast(topk)); + const native_m: i64 = if (!naive and block_m % 32 == 0) 32 else 16; + const cfg: Config = .{ + .tokens = input.dim(0), + .experts = weight.dim(0), + .n = n, + .k = k, + .ka = @divExact(k, 4), + .kb = @divExact(k, 8), + .ks = @divExact(k, 32), + .topk = @as(i64, @intCast(topk)), + .routes = routes, + .capacity = @as(i64, @intCast(sorted_ids.count())), + .blocks = @as(i64, @intCast(expert_ids.count())), + .subtiles = @divExact(@as(i64, @intCast(block_m)), native_m), + .native_m = native_m, + .wide = @as(i64, @intFromBool(native_m == 32)), + .stage_elements = native_m * 128, + .shared_elements = native_m * 256, + .naive = @as(i64, @intFromBool(naive)), + }; + const a = input.reshape(.{ input.dim(0), @divExact(k, 4), 4 }).bitCast(.i64); + const b = weight.reshape(.{ weight.dim(0), n, @divExact(k, 8), 4 }).bitCast(.i32); + return Kernel.call(.{ .a = a, .b = b, .scales = scales, .sorted = sorted_ids, .experts = expert_ids, .padded = padded_count.reshape(.{1}) }, .{ + .output = zml.Shape.init(.{ routes, n }, .bf16), + }, .{ + .cfg = cfg, + .threads = 256, + .grid = .{ @intCast(@divExact(n, 64)), @intCast(if (naive) routes else @divExact(cfg.capacity, native_m)), 1 }, + }).output.reshape(output); + } + + const Config = struct { + tokens: i64, + experts: i64, + n: i64, + k: i64, + ka: i64, + kb: i64, + ks: i64, + topk: i64, + routes: i64, + capacity: i64, + blocks: i64, + subtiles: i64, + native_m: i64, + wide: i64, + stage_elements: i64, + shared_elements: i64, + naive: i64, + }; + + const Kernel = zml.kernel.fly.Kernel(Config, .{ + .name = "mxfp4_raw_mfma", + .inputs = &.{ "a", "b", "scales", "sorted", "experts", "padded" }, + .outputs = &.{"output"}, + .run = run, + }); + + fn run(b: *Builder, cfg: Config) fly.FinishError!void { + const args = Kernel.args(b); + const zero = b.constant(.i32, 0); + const fzero = b.constant(.f32, 0.0); + const zeros = fzero.splat(4); + const block_n = b.blockId(.x).to(.i64); + const block_m = b.blockId(.y).to(.i64); + const route_start = block_m.mul(cfg.native_m); + const has_rows = if (cfg.naive != 0) b.constant(.i1, 1) else blk: { + const padded = args.padded.at(b.constant(.i64, 0)).to(.i64); + var has_block = b.openIfElse(route_start.cmpUnsigned(.lt, padded), .{.i1}); + { + has_block.yieldThen(.{args.sorted.at(route_start).ne(cfg.routes)}); + } + { + has_block.yieldElse(.{b.constant(.i1, 0)}); + } + break :blk has_block.results[0]; + }; + var block = b.openIf(has_rows); + { + const expert_block = if (cfg.naive != 0) block_m else block_m.divUnsigned(cfg.subtiles); + const eid = args.experts.at(expert_block); + const valid_expert = eid.ge(0).bitAnd(eid.lt(cfg.experts)); + const thread = b.threadId(.x).to(.i64); + const lane = thread.remUnsigned(64); + const wave = thread.divUnsigned(64); + const mn_lane = lane.remUnsigned(16); + const k_lane = lane.divUnsigned(16); + const out_n = block_n.mul(64).add(wave.mul(16)).add(mn_lane); + // Each expert has a contiguous real prefix before route-count padding. + const second_active = if (cfg.wide != 0) blk: { + const first = args.sorted.at(route_start.add(16)); + break :blk first.ge(0).bitAnd(first.lt(cfg.routes)); + } else b.constant(.i1, 0); + + var sums = b.openIfElse(valid_expert, .{ zeros, zeros }); + { + const expert = eid.to(.i64); + const load_row = thread.divUnsigned(16); + const load_chunk = thread.remUnsigned(16); + const a_route = routeForRow(b, args.sorted, block_m, route_start, load_row, cfg.naive != 0); + const a_valid = a_route.ge(0).bitAnd(a_route.lt(cfg.routes)); + const a_base = a_route.to(.i64).divUnsigned(cfg.topk).mul(cfg.ka); + var second_route = b.openIfElse(second_active, .{zero}); + { + second_route.yieldThen(.{args.sorted.at(route_start.add(load_row).add(16))}); + } + { + second_route.yieldElse(.{b.constant(.i32, -1)}); + } + const a_route_g1 = second_route.results[0]; + const a_valid_g1 = a_route_g1.ge(0).bitAnd(a_route_g1.lt(cfg.routes)); + const a_base_g1 = a_route_g1.to(.i64).divUnsigned(cfg.topk).mul(cfg.ka); + + // One allocation, two disjoint ping-pong regions; never two identical leaves. + const shared = b.sharedArray(.i16, cfg.shared_elements, 16); + const load_a = b.copyAtom(.{ .universal = 128 }, .i64); + const load_b = b.copyAtom(.{ .universal = 128 }, .i32); + const copy_shared = b.copyAtom(.{ .universal = 128 }, .i16); + const loader: TileLoader = .{ + .a = args.a, + .weight = args.b, + .scales = args.scales, + .shared = shared, + .load_a = load_a, + .load_b = load_b, + .copy_shared = copy_shared, + .a_base = a_base, + .a_base_g1 = a_base_g1, + .a_valid = a_valid, + .a_valid_g1 = a_valid_g1, + .second_active = second_active, + .a_col = load_chunk.mul(2), + .a_store = load_row.mul(128).add(load_chunk.bitXor(load_row).mul(8)), + .b_base = expert.mul(cfg.n * cfg.kb).add(out_n.mul(cfg.kb)), + .expert = expert, + .out_n = out_n, + .k_lane = k_lane, + }; + const initial = loader.load(b, b.constant(.i64, 0), b.constant(.i64, 0)); + b.barrier(); + const atom = b.mmaAtom((fly.fly_rocdl.MmaOpCDNA3MFMAType.get(b.ctx, .{ + .m = 16, + .n = 16, + .k = 16, + .elemTyA = fly.DType.bf16.toMlir(b.ctx), + .elemTyB = fly.DType.bf16.toMlir(b.ctx), + .elemTyAcc = fly.DType.f32.toMlir(b.ctx), + }) catch return error.InvalidMlir).type_()); + var k = b.openFor(b.constant(.i64, 0), cfg.k, 128, .{ zeros, zeros, initial.words, initial.scale }); + { + const read_base = k.iv.divUnsigned(128).bitAnd(1).mul(cfg.stage_elements); + const write_base = k.iv.divUnsigned(128).bitAnd(1).bitXor(1).mul(cfg.stage_elements); + var a_fragments: [4]Value = undefined; + var a_fragments_g1: [4]Value = undefined; + inline for (0..4) |u| { + const offset = read_base.add(mn_lane.mul(128)).add(k_lane.mul(4).add(u).bitXor(mn_lane).mul(8)); + a_fragments[u] = b.copyAtomLoad(copy_shared, vectorView(shared, offset, 8)); + } + inline for (0..4) |u| { + const offset = read_base.add(mn_lane.mul(128)).add(k_lane.mul(4).add(u).bitXor(mn_lane).mul(8)).add(2048); + const azero = b.constant(.i16, 0).splat(8); + var second = b.openIfElse(second_active, .{azero}); + { + second.yieldThen(.{b.copyAtomLoad(copy_shared, vectorView(shared, offset, 8))}); + } + { + second.yieldElse(.{azero}); + } + a_fragments_g1[u] = second.results[0]; + } + // Prefetch the other LDS region only after all current A fragments are read. + const next_k = k.iv.add(128); + var next = b.openIfElse(next_k.cmpUnsigned(.lt, cfg.k), .{ initial.words, initial.scale }); + { + const next_tile = loader.load(b, next_k, write_base); + next.yieldThen(.{ next_tile.words, next_tile.scale }); + } + { + next.yieldElse(.{ b.constant(.i32, 0).splat(4), fzero }); + } + var acc = k.carried[0]; + var acc_g1 = k.carried[1]; + inline for (0..4) |u| { + const decoded = decodeWeight(b, k.carried[2].extract(u), k.carried[3]); + const a0 = a_fragments[u].shuffleVector(a_fragments[u], &.{ 0, 1, 2, 3 }).bitCast(.bf16); + const a1 = a_fragments[u].shuffleVector(a_fragments[u], &.{ 4, 5, 6, 7 }).bitCast(.bf16); + const w0 = decoded.shuffleVector(decoded, &.{ 0, 1, 2, 3 }); + const w1 = decoded.shuffleVector(decoded, &.{ 4, 5, 6, 7 }); + acc = b.mmaAtomCall(atom, a0, w0, acc); + acc = b.mmaAtomCall(atom, a1, w1, acc); + var second = b.openIfElse(second_active, .{zeros}); + { + const a0_g1 = a_fragments_g1[u].shuffleVector(a_fragments_g1[u], &.{ 0, 1, 2, 3 }).bitCast(.bf16); + const a1_g1 = a_fragments_g1[u].shuffleVector(a_fragments_g1[u], &.{ 4, 5, 6, 7 }).bitCast(.bf16); + const first_acc = b.mmaAtomCall(atom, a0_g1, w0, acc_g1); + second.yieldThen(.{b.mmaAtomCall(atom, a1_g1, w1, first_acc)}); + } + { + second.yieldElse(.{acc_g1}); + } + acc_g1 = second.results[0]; + } + // All current reads finish before a later iteration reuses this LDS region. + b.barrier(); + k.yield(.{ acc, acc_g1, next.results[0], next.results[1] }); + } + sums.yieldThen(.{ k.results[0], k.results[1] }); + } + { + sums.yieldElse(.{ zeros, zeros }); + } + + inline for (0..4) |i| { + storeRow(b, args.output, args.sorted, block_m, route_start, k_lane.mul(4).add(i), out_n, sums.results[0].extract(i), cfg); + } + var store_second = b.openIf(second_active); + { + inline for (0..4) |i| { + storeRow(b, args.output, args.sorted, block_m, route_start, k_lane.mul(4).add(i + 16), out_n, sums.results[1].extract(i), cfg); + } + store_second.close(); + } + block.close(); + } + } + + fn routeForRow(b: *Builder, sorted: Value, block: Value, start: Value, row: Value, naive: bool) Value { + return if (naive) row.eq(0).select(block.to(.i32), b.constant(.i32, -1)) else sorted.at(start.add(row)); + } + + fn storeRow(b: *Builder, output: Value, sorted: Value, block: Value, start: Value, row: Value, col: Value, value: Value, cfg: Config) void { + const route = routeForRow(b, sorted, block, start, row, cfg.naive != 0); + var valid = b.openIf(route.ge(0).bitAnd(route.lt(cfg.routes))); + { + output.set(.{ route.to(.i64), col }, value.to(.bf16)); + valid.close(); + } + } + + fn vectorView(tensor: Value, offset: Value, comptime n: i64) Value { + return tensor.emitIter().addOffset(offset).view(fly.L(n, 1)); + } + + const TileLoader = struct { + a: Value, + weight: Value, + scales: Value, + shared: Value, + load_a: Value, + load_b: Value, + copy_shared: Value, + a_base: Value, + a_base_g1: Value, + a_valid: Value, + a_valid_g1: Value, + second_active: Value, + a_col: Value, + a_store: Value, + b_base: Value, + expert: Value, + out_n: Value, + k_lane: Value, + + fn load(self: TileLoader, b: *Builder, k: Value, write_base: Value) struct { words: Value, scale: Value } { + self.storeActivation(b, self.a_base, self.a_valid, k, self.a_store.add(write_base)); + var second = b.openIf(self.second_active); + { + self.storeActivation(b, self.a_base_g1, self.a_valid_g1, k, self.a_store.add(write_base).add(2048)); + second.close(); + } + const offset = self.b_base.add(k.divUnsigned(8)).add(self.k_lane.mul(4)); + const words = b.copyAtomLoad(self.load_b, vectorView(self.weight, offset, 4)); + const byte = self.scales.at(.{ self.expert, self.out_n, k.divUnsigned(32).add(self.k_lane) }); + const scale = byte.toUnsigned(.i32); + const finite = scale.eq(0).select(b.constant(.i32, 0x00400000), scale.shl(23)); + return .{ .words = words, .scale = scale.eq(255).select(b.constant(.i32, 0x7fc00000), finite).bitCast(.f32) }; + } + + fn storeActivation(self: TileLoader, b: *Builder, base: Value, valid: Value, k: Value, destination: Value) void { + const zeros = b.constant(.i64, 0).splat(2); + var guarded = b.openIfElse(valid, .{zeros}); + { + const offset = base.add(k.divUnsigned(4)).add(self.a_col); + guarded.yieldThen(.{b.copyAtomLoad(self.load_a, vectorView(self.a, offset, 2))}); + } + { + guarded.yieldElse(.{zeros}); + } + b.copyAtomStore(self.copy_shared, guarded.results[0].bitCast(.i16), vectorView(self.shared, destination, 8)); + } + }; + + fn decodeWeight(b: *Builder, word: Value, scale: Value) Value { + const lo = b.constant(.i32, 0x44403800); + const hi = b.constant(.i32, 0x54504c48); + const even = b.perm(hi, lo, word.bitAnd(0x07070707)); + const odd = b.perm(hi, lo, word.shrU(4).bitAnd(0x07070707)); + const ev0 = b.cvtPkF32Fp8(even, false); + const od0 = b.cvtPkF32Fp8(odd, false); + const ev1 = b.cvtPkF32Fp8(even, true); + const od1 = b.cvtPkF32Fp8(odd, true); + const first = ev0.shuffleVector(od0, &.{ 0, 2, 1, 3 }); + const last = ev1.shuffleVector(od1, &.{ 0, 2, 1, 3 }); + const magnitude = first.shuffleVector(last, &.{ 0, 1, 2, 3, 4, 5, 6, 7 }); + var shifts = b.constant(.i32, 0).splat(8); + inline for (0..8) |i| shifts = shifts.insert(i, b.constant(.i32, i * 4)); + const signs = word.splat(8).shrU(shifts).bitAnd(8).shl(28); + // FNUZ has no -0: restore FP4's sign in F32 before scaling and BF16 rounding. + return magnitude.bitCast(.i32).bitOr(signs).bitCast(.f32).mul(scale).to(.bf16); + } +}; + +const Gemv = struct { + // Packed-word GEMV derived from the supplied moe-mi300x/fly_word.py algorithm. + // Several rows share a CTA, and the shorter down projection uses subwaves to + // expose independent outputs without paying for a full wave reduction per row. + const Config = struct { rows: usize, lanes: usize, unroll: usize }; + + /// One local expert ID per original route; negative/out-of-bank IDs write zero. + fn call(x: Tensor, weight: Tensor, scales: Tensor, ids: Tensor, output: zml.Shape, topk: usize) Tensor { + const cfg: Gemv.Config = if (x.dim(1) >= 4096) + .{ .rows = 4, .lanes = 64, .unroll = 1 } + else + .{ .rows = 4, .lanes = 32, .unroll = 5 }; + const rows = cfg.rows; + const lanes = cfg.lanes; + const unroll = cfg.unroll; + const n = weight.dim(1); + const k = x.dim(1); + const routes: i64 = @intCast(ids.count()); + std.debug.assert(x.dtype() == .bf16 and (weight.dtype() == .u8 or weight.dtype() == .i8)); + std.debug.assert(@mod(n, 8) == 0 and @mod(k, 32) == 0); + std.debug.assert(topk > 0 and routes == x.dim(0) * @as(i64, @intCast(topk))); + std.debug.assert(weight.dim(2) * 2 == k); + std.debug.assert(scales.dim(0) == weight.dim(0) and scales.dim(1) == n and scales.dim(2) == @divExact(k, 32)); + const tile_rows = rows * (64 / lanes); + std.debug.assert(@mod(n, @as(i64, @intCast(tile_rows * 8))) == 0); + const fields: EmitConfig = .{ .unroll = unroll, .rows = tile_rows, .lanes = lanes, .stride = unroll * lanes, .ntiles = @divExact(n, @as(i64, @intCast(tile_rows))), .tokens = x.dim(0), .experts = weight.dim(0), .n = n, .k_pairs = @divExact(k, 2), .k_words = @divExact(k, 8), .k_scales = @divExact(k, 32), .routes = routes, .grid = @divExact(n, @as(i64, @intCast(tile_rows))) * routes, .topk = @as(i64, @intCast(topk)) }; + const a = x.reshape(.{ .token = x.dim(0), .in = @divExact(k, 2), .pair = 2 }).bitCast(.i32); + const b = weight.reshape(.{ .expert = weight.dim(0), .out = n, .in = @divExact(weight.dim(2), 4), .word = 4 }).bitCast(.i32); + const matrix_output = zml.Shape.init(.{ .token = routes, .out = n }, output.dtype()); + return Gemv.Kernel.call(.{ .a = a, .b = b, .scales = scales.bitCast(.u8), .ids = ids.reshape(.{routes}) }, .{ + .output = matrix_output, + }, .{ + .cfg = fields, + .threads = @intCast(rows * 64), + .grid = .{ @intCast(fields.grid), 1, 1 }, + }).output.reshape(output); + } + + const EmitConfig = struct { + unroll: usize, + rows: usize, + lanes: usize, + stride: usize, + ntiles: i64, + tokens: i64, + experts: i64, + n: i64, + k_pairs: i64, + k_words: i64, + k_scales: i64, + routes: i64, + grid: i64, + topk: i64, + }; + + const Kernel = zml.kernel.fly.Kernel(EmitConfig, .{ + .name = "mxfp4_gemv", + .inputs = &.{ "a", "b", "scales", "ids" }, + .outputs = &.{"output"}, + .run = Gemv.run, + }); + + fn run(b: *Builder, cfg: EmitConfig) fly.FinishError!void { + const args = Gemv.Kernel.args(b); + const block = b.blockId(.x).to(.i64); + const thread = b.threadId(.x).to(.i64); + const lane = thread.remUnsigned(cfg.lanes); + const rowgroup = thread.divUnsigned(cfg.lanes); + // Preserve the eight-XCD workgroup permutation and original subwave layout. + const pid = block.remUnsigned(8).mul(@divExact(cfg.grid, 8)).add(block.divUnsigned(8)); + const row = pid.remUnsigned(cfg.ntiles).mul(cfg.rows).add(rowgroup); + const route = pid.divUnsigned(cfg.ntiles); + const expert32 = args.ids.at(route); + const expert = expert32.to(.i64); + const token = route.divUnsigned(cfg.topk); + const local = expert32.ge(0).bitAnd(expert32.lt(cfg.experts)); + const inrow = row.cmpUnsigned(.lt, cfg.n); + const fzero = b.constant(.f32, 0.0); + var sum = b.openIfElse(local.bitAnd(inrow), .{fzero}); + { + var k = b.openFor(lane, cfg.k_words, cfg.stride, .{fzero}); + { + var carried = k.carried[0]; + for (0..cfg.unroll) |u| { + const pos = k.iv.add(u * cfg.lanes); + var inbounds = b.openIfElse(pos.cmpUnsigned(.lt, cfg.k_words), .{fzero}); + { + const word = args.b.at(.{ expert, row, pos }); + const scale8 = args.scales.at(.{ expert, row, pos.divUnsigned(4) }); + const scale = decodeScale(b, scale8); + const low = word.bitAnd(0x07070707); + const high = word.shrU(4).bitAnd(0x07070707); + const even_bits = b.perm(b.constant(.i32, 0xd4d0ccc8), b.constant(.i32, 0xc4c0b800), low); + const odd_bits = b.perm(b.constant(.i32, 0xd4d0ccc8), b.constant(.i32, 0xc4c0b800), high); + const even = even_bits.bitAnd(word.shl(4).bitOr(0x7f7f7f7f)); + const odd = odd_bits.bitAnd(word.bitOr(0x7f7f7f7f)); + // Retain the scalar decoder and its FP32 scale -> BF16 -> FP32 boundary. + const weights = [_]Value{ + b.cvtPkF32Fp8(even, false).mul(scale).to(.bf16).to(.f32), + b.cvtPkF32Fp8(odd, false).mul(scale).to(.bf16).to(.f32), + b.cvtPkF32Fp8(even, true).mul(scale).to(.bf16).to(.f32), + b.cvtPkF32Fp8(odd, true).mul(scale).to(.bf16).to(.f32), + }; + var partial = fzero; + inline for (0..4) |pair| { + const x = args.a.at(.{ token, pos.mul(4).add(pair) }); + const xlo = x.to(.i16).bitCast(.bf16).to(.f32); + const xhi = x.shrU(16).to(.i16).bitCast(.bf16).to(.f32); + const wlo = weights[(pair / 2) * 2].extract(pair % 2); + const whi = weights[(pair / 2) * 2 + 1].extract(pair % 2); + // Odd then even FMA, followed by one add to the K-loop accumulator. + partial = b.fma(xlo, wlo, b.fma(xhi, whi, partial)); + } + inbounds.yieldThen(.{partial.add(carried)}); + } + { + inbounds.yieldElse(.{carried}); + } + carried = inbounds.results[0]; + } + k.yield(.{carried}); + } + var reduced = k.results[0]; + var shift: i32 = 1; + while (shift < @as(i32, @intCast(cfg.lanes))) : (shift *= 2) { + reduced = reduced.add(reduced.shuffleXor(shift, 64)); + } + sum.yieldThen(.{reduced}); + } + { + sum.yieldElse(.{fzero}); + } + var store = b.openIf(lane.eq(0).bitAnd(inrow)); + { + args.output.set(.{ route, row }, sum.results[0].to(.bf16)); + store.close(); + } + } + + fn decodeScale(b: *Builder, byte: Value) Value { + const scale = byte.toUnsigned(.i32); + const finite = scale.eq(0).select(b.constant(.i32, 0x00400000), scale.shl(23)); + return scale.eq(255).select(b.constant(.i32, 0x7fc00000), finite).bitCast(.f32); + } +}; diff --git a/zml/kernels/moe/metal/BUILD.bazel b/zml/kernels/moe/metal/BUILD.bazel new file mode 100644 index 00000000..b4240b00 --- /dev/null +++ b/zml/kernels/moe/metal/BUILD.bazel @@ -0,0 +1,8 @@ +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "metal", + srcs = [ + "metal.zig", + ], +) diff --git a/zml/kernels/moe/metal/metal.zig b/zml/kernels/moe/metal/metal.zig new file mode 100644 index 00000000..f835c889 --- /dev/null +++ b/zml/kernels/moe/metal/metal.zig @@ -0,0 +1,164 @@ +const std = @import("std"); + +const stdx = @import("stdx"); + +const zml = @import("../../../zml.zig"); +const Tensor = zml.Tensor; +const Activation = @import("../api.zig").Activation; +const GlobalOptions = @import("../api.zig").Options; +const LocalRouting = @import("../api.zig").LocalRouting; +const unpackedWeight = @import("../weights.zig").unpackedWeight; + +const log = std.log.scoped(.@"zml/moe/metal"); + +fn validateOptions(opts: GlobalOptions) void { + stdx.debug.assert(!opts.quantize_input, "Optional FP8 input quantization requires the Triton MoE backend", .{}); + stdx.debug.assert(opts.routing_weight_placement == .after_down, "Non-Triton MoE backends require routing weights after the down projection", .{}); +} + +const QuantMode = enum { none, fp8, nvfp4 }; + +fn quantMode(dtype: zml.DataType) QuantMode { + return switch (dtype) { + .bf16, .f16, .f32 => .none, + .f8e4m3fn => .fp8, + .u8, .f4e2m1 => .nvfp4, + else => stdx.debug.panic("unsupported data type: {}", .{dtype}), + }; +} + +fn moeGemm( + x_rows: Tensor, + w: Tensor, + scale: ?Tensor, + expert_ids: Tensor, + out_shape: zml.Shape, + mode: QuantMode, +) Tensor { + const side: zml.ops.CustomCallOptions = .{ .has_side_effect = false }; + return switch (mode) { + .none => zml.ops.customCall("__metal$moe_gemm", .{ x_rows, w, expert_ids }, .{out_shape}, .{}, side), + .fp8 => zml.ops.customCall("__metal$moe_gemm$f8", .{ x_rows, w, scale.?, expert_ids }, .{out_shape}, .{}, side), + .nvfp4 => zml.ops.customCall("__metal$moe_gemm$f4", .{ x_rows, w, scale.?, expert_ids }, .{out_shape}, .{}, side), + }; +} + +fn applyGateUpGlobalScale(output: Tensor, global_scale: ?Tensor, expert_ids: Tensor) Tensor { + const scale = global_scale orelse return output; + stdx.debug.assert(scale.dtype() == .f32, "metal backend expected gate_up scale data type to be f32, got {}", .{scale.dtype()}); + stdx.debug.assert(scale.rank() == 2 and scale.dim(.proj) == 2, "metal backend expected gate_up scale to be a 2D tensor with a .proj dim equal to 2, got {}", .{scale.shape()}); + + const mid = @divExact(output.dim(.out), 2); + const selected = scale.gather(.{ .expert = expert_ids }, .{}); + const expanded = selected.appendAxes(.{.mid}) + .broad(zml.Shape.init(.{ .r = output.dim(.r), .proj = 2, .mid = mid }, .f32)) + .merge(.{ .out = .{ .proj, .mid } }); + return output.convert(.f32).mul(expanded).convert(output.dtype()); +} + +fn applyDownGlobalScale(output: Tensor, global_scale: ?Tensor, expert_ids: Tensor) Tensor { + const scale = global_scale orelse return output; + stdx.debug.assert(scale.dtype() == .f32, "metal backend expected down scale data type to be f32, got {}", .{scale.dtype()}); + stdx.debug.assert(scale.rank() == 1, "metal backend expected down scale to be a 1D tensor, got {}", .{scale.shape()}); + + const selected = scale.gather(.{ .expert = expert_ids }, .{}) + .appendAxes(.{.d}) + .broad(output.shape().withDtype(.f32)); + return output.convert(.f32).mul(selected).convert(output.dtype()); +} + +pub fn fusedExperts( + hidden_states: zml.Tensor, + routing: LocalRouting, + gate_up_: zml.nn.Linear, + down_: zml.nn.Linear, + opts: GlobalOptions, +) zml.Tensor { + validateOptions(opts); + stdx.debug.assert(gate_up_.bias == null, "metal backend doesn't support gate_up bias", .{}); + stdx.debug.assert(down_.bias == null, "metal backend doesn't support down bias", .{}); + + const gate_up_scales: ?zml.Tensor = if (gate_up_.quantization) |q| q.scales else null; + const gate_up_global_scale: ?zml.Tensor = if (gate_up_.quantization) |q| (if (q.global_scale) |scale| scale.asMultiplier() else null) else null; + + const down_scales: ?zml.Tensor = if (down_.quantization) |q| q.scales else null; + const down_global_scale: ?zml.Tensor = if (down_.quantization) |q| (if (q.global_scale) |scale| scale.asMultiplier() else null) else null; + + const gate_up = unpackedWeight(gate_up_); + const down = unpackedWeight(down_); + stdx.debug.assert(gate_up.dtype() == down.dtype(), "metal backend expected gate_up and down data type to match, got {} and {}", .{ gate_up.dtype(), down.dtype() }); + const mode = quantMode(gate_up.dtype()); + switch (mode) { + .none => { + stdx.debug.assert(gate_up_scales == null and down_scales == null, "metal backend expected gate_up and down scale to be null when the weights are not quantized", .{}); + stdx.debug.assert(gate_up_global_scale == null and down_global_scale == null, "metal backend expected gate_up and down global scale to be null when the weights are not quantized", .{}); + const is_hidden_states_valid_dtype = switch (hidden_states.dtype()) { + .bf16, .f16, .f32 => true, + else => false, + }; + stdx.debug.assert(is_hidden_states_valid_dtype, "metal backend expected hidden_states data type to be bf16, f16 or f32, got {}", .{hidden_states.dtype()}); + }, + .fp8 => { + stdx.debug.assert(gate_up_scales != null and down_scales != null, "metal backend expected gate_up and down scale to be non null", .{}); + stdx.debug.assert(gate_up_scales.?.dtype() == .bf16 and down_scales.?.dtype() == .bf16, "metal backend expected gate_up and down scale data type to be bf16, got {} and {}", .{ gate_up_scales.?.dtype(), down_scales.?.dtype() }); + stdx.debug.assert(gate_up_global_scale == null and down_global_scale == null, "metal backend expected gate_up and down global scale to be null", .{}); + stdx.debug.assert(hidden_states.dtype() == .bf16, "metal backend expected hidden_states data type to be bf16, got {}", .{hidden_states.dtype()}); + }, + .nvfp4 => { + stdx.debug.assert(gate_up_scales != null and down_scales != null, "metal backend expected gate_up and down scale to be non null", .{}); + stdx.debug.assert(gate_up_scales.?.dtype() == .f8e4m3fn and down_scales.?.dtype() == .f8e4m3fn, "metal backend expected gate_up and down scale data type to be f8e4m3fn, got {} and {}", .{ gate_up_scales.?.dtype(), down_scales.?.dtype() }); + stdx.debug.assert(hidden_states.dtype() == .bf16, "metal backend expected hidden_states data type to be bf16, got {}", .{hidden_states.dtype()}); + }, + } + + const is_topk_weights_valid_dtype = switch (routing.weights.dtype()) { + .bf16, .f16, .f32 => true, + else => false, + }; + stdx.debug.assert(is_topk_weights_valid_dtype, "metal backend expected topk_weights data type to be bf16, f16 or f32, got {}", .{routing.weights.dtype()}); + stdx.debug.assert(routing.ids.dtype() == .i32, "metal backend expected topk_ids data type to be i32, got {}", .{routing.ids.dtype()}); + + const d = hidden_states.dim(.d); + const num_tokens = hidden_states.dim(.t); + const topk = routing.ids.dim(.topk); + const num_routes = num_tokens * topk; + + const act_dtype: zml.DataType = switch (mode) { + .none => hidden_states.dtype(), + .fp8, .nvfp4 => .bf16, + }; + + const hidden = hidden_states.reshape(.{ .token = num_tokens, .d = d }).withTags(.{ .token, .d }); + const x_rows = hidden.insertAxes(.d, .{.topk}) + .broad(zml.Shape.init(.{ .token = num_tokens, .topk = topk, .d = d }, act_dtype)) + .merge(.{ .r = .{ .token, .topk } }); + const expert_ids = routing.ids.reshape(.{ .token = num_tokens, .topk = topk }).withTags(.{ .token, .topk }) + .merge(.{ .r = .{ .token, .topk } }).convert(.i32); + + const gate_up_out = applyGateUpGlobalScale(moeGemm( + x_rows, + gate_up, + gate_up_scales, + expert_ids, + zml.Shape.init(.{ .r = num_routes, .out = gate_up.dim(.dout) }, act_dtype), + mode, + ), gate_up_global_scale, expert_ids); + const activated = opts.activation.applyActivation(gate_up_out, .concatenated).convert(act_dtype); + + const down_out = applyDownGlobalScale(moeGemm( + activated, + down, + down_scales, + expert_ids, + zml.Shape.init(.{ .r = num_routes, .d = down.dim(.d) }, act_dtype), + mode, + ), down_global_scale, expert_ids); + + const weights = routing.weights.reshape(.{ .token = num_tokens, .topk = topk }).withTags(.{ .token, .topk }) + .merge(.{ .r = .{ .token, .topk } }); + const weighted = down_out.mul(weights.convert(down_out.dtype()).broad(down_out.shape())); + const combined = weighted.splitAxis(.r, .{ .token = num_tokens, .topk = topk }) + .sum(.topk).squeeze(.topk); + + return combined.reshape(.{ .t = num_tokens, .d = down.dim(.d) }); +} diff --git a/zml/kernels/moe/moe.zig b/zml/kernels/moe/moe.zig new file mode 100644 index 00000000..eb697f36 --- /dev/null +++ b/zml/kernels/moe/moe.zig @@ -0,0 +1,298 @@ +const std = @import("std"); + +const platforms = @import("platforms"); + +const zml = @import("../../zml.zig"); +const stdx = zml.stdx; +pub const Activation = @import("api.zig").Activation; +pub const ActivationKind = @import("api.zig").ActivationKind; +pub const cute = @import("cute/cute.zig"); +pub const ExpertsLayout = @import("api.zig").ExpertsLayout; +pub const flashinfer_cutlass = @import("flashinfer_cutlass/flashinfer_cutlass.zig"); +pub const fly = @import("fly/fly.zig"); +pub const LocalRouting = @import("api.zig").LocalRouting; +pub const metal = @import("metal/metal.zig"); +pub const mosaic_tpu = @import("mosaic_tpu/mosaic_tpu.zig"); +pub const Options = @import("api.zig").Options; +pub const ProjectionLayout = @import("api.zig").ProjectionLayout; +pub const RoutingWeightPlacement = @import("api.zig").RoutingWeightPlacement; +pub const stablehlo = @import("stablehlo/stablehlo.zig"); +pub const tcl = @import("tcl/tcl.zig"); +pub const triton = @import("triton/triton.zig"); + +const log = std.log.scoped(.@"zml/moe"); + +test { + std.testing.refAllDecls(@This()); + _ = @import("tests.zig"); +} + +pub const Backend = enum { + stablehlo, + cute, + flashinfer_cutlass, + triton, + fly, + mosaic_tpu, + metal, + tcl, + + pub fn auto(platform: *const zml.Platform, scheme: ?zml.Quantization.Scheme, dtype: zml.DataType) !Backend { + return switch (platform.target) { + .cuda => b: { + const s = scheme orelse break :b .triton; + break :b switch (s) { + .mxfp4 => if (cute.isAvailable(platform)) + .cute + else + .triton, + .nvfp4 => if (cute.isNvfp4Supported(platform)) + .cute + else if (flashinfer_cutlass.isNvfp4Supported(platform)) + .flashinfer_cutlass + else + error.UnsupportedQuantization, + .mxfp8, .fp8_per_channel, .fp8_per_tensor, .fp8_block128, .fp8_block32 => .triton, + }; + }, + .rocm => b: { + const s = scheme orelse break :b .triton; + break :b switch (s) { + .mxfp4 => if (zml.platform.rocm.computeCapability(platform) == .gfx942) .fly else .triton, + .mxfp8, .fp8_per_channel, .fp8_per_tensor, .fp8_block128, .fp8_block32 => .triton, + .nvfp4 => error.UnsupportedQuantization, + }; + }, + .oneapi => b: { + const s = scheme orelse break :b .triton; + break :b switch (s) { + .mxfp4 => .triton, + .nvfp4, .mxfp8, .fp8_per_channel, .fp8_per_tensor, .fp8_block128, .fp8_block32 => error.UnsupportedQuantization, + }; + }, + .tpu => if (scheme == null) .mosaic_tpu else error.UnsupportedQuantization, + .furiosa => b: { + const s = scheme orelse break :b .tcl; + break :b switch (s) { + .fp8_block128 => if (dtype == .f8e4m3fn) .tcl else error.UnsupportedQuantization, + .mxfp4, .nvfp4, .mxfp8, .fp8_per_channel, .fp8_per_tensor, .fp8_block32 => error.UnsupportedQuantization, + }; + }, + .metal => b: { + const s = scheme orelse break :b .metal; + break :b switch (s) { + .nvfp4, .mxfp8, .fp8_per_channel, .fp8_per_tensor, .fp8_block128, .fp8_block32 => .metal, + .mxfp4 => error.UnsupportedQuantization, + }; + }, + else => error.UnimplementedMoEBackend, + }; + } + + pub fn isAvailable(backend: Backend, platform: *const zml.Platform) bool { + return switch (backend) { + .stablehlo => true, + .cute => cute.isAvailable(platform), + .flashinfer_cutlass => flashinfer_cutlass.isAvailable(platform), + .fly => switch (platform.target) { + .rocm => zml.platform.rocm.computeCapability(platform) == .gfx942, + else => false, + }, + .triton => switch (platform.target) { + .cuda, .rocm, .oneapi => true, + else => false, + }, + .mosaic_tpu => platform.target == .tpu, + .metal => platform.target == .metal, + .tcl => platform.target == .furiosa, + }; + } + + /// The layout contract the expert weights must have for a given backend with scheme + pub fn expertsLayout(backend: Backend, scheme: ?zml.Quantization.Scheme) !ExpertsLayout { + return switch (backend) { + .stablehlo => .{ .gate_up = .concatenated, .packing = .plain }, + .cute => if (scheme) |s| switch (s) { + .mxfp4 => .{ .gate_up = .row_interleaved_8, .packing = .mxfp4_interleaved8 }, + .nvfp4 => .{ .gate_up = .row_interleaved_1, .packing = .swizzled_scales }, + else => error.UnsupportedQuantization, + } else error.UnsupportedQuantization, + .flashinfer_cutlass => if (scheme) |s| switch (s) { + .nvfp4 => .{ .gate_up = .concatenated, .packing = .flashinfer_nvfp4 }, + else => error.UnsupportedQuantization, + } else error.UnsupportedQuantization, + .mosaic_tpu, .metal, .tcl => .{ .gate_up = .concatenated, .packing = .plain }, + .triton, .fly => if (scheme == .mxfp4) .{ .gate_up = .row_interleaved_1, .packing = .plain } else .{ .gate_up = .concatenated, .packing = .plain }, + }; + } + + pub fn register(backend: Backend, platform: *zml.Platform) !void { + return switch (backend) { + .stablehlo => {}, + .cute => {}, + .flashinfer_cutlass => flashinfer_cutlass.register(platform), + .triton, .fly => {}, + .mosaic_tpu => {}, + .metal => {}, + .tcl => {}, + }; + } +}; + +/// Check the shared token-wise contract for activations and routing tensors. +pub fn validateRoutingInputs(input: zml.Tensor, ids: zml.Tensor, weights: zml.Tensor) void { + stdx.debug.assert(input.rank() == 2 and ids.rank() == 2 and weights.rank() == 2, "MoE expects input {{.t, .d}} and routing {{.t, .topk}}, got {f}, {f}, {f}", .{ input.shape(), ids.shape(), weights.shape() }); + stdx.debug.assert(input.dim(.t) == ids.dim(.t) and input.dim(.t) == weights.dim(.t), "MoE input and routing token counts must match", .{}); + stdx.debug.assert(ids.dim(.topk) == weights.dim(.topk), "MoE routing ids and weights must have matching top-k dimensions", .{}); + stdx.debug.assert(input.shape().hasTag(.t) == 0 and input.shape().hasTag(.d) == 1 and + ids.shape().hasTag(.t) == 0 and ids.shape().hasTag(.topk) == 1 and + weights.shape().hasTag(.t) == 0 and weights.shape().hasTag(.topk) == 1, "MoE expects ordered tags {{.t, .d}} for input and {{.t, .topk}} for routing", .{}); +} + +pub const Partitioning = struct { + const Logical = zml.Sharding.Partitioning.Logical; + + in_specs: struct { + input: Logical, + ids: Logical, + weights: Logical, + gate_up: Logical, + down: Logical, + }, + out_specs: Logical, + + /// Shards the `.expert` dimension of both projections over `mesh_axis`. + /// Other dimensions, input tokens, routing IDs/weights and output are replicated. + pub fn expert(comptime mesh_axis: @EnumLiteral()) Partitioning { + const experts: Logical = .parse(.{ .expert = mesh_axis }); + return .{ + .in_specs = .{ + .input = .replicated, + .ids = .replicated, + .weights = .replicated, + .gate_up = experts, + .down = experts, + }, + .out_specs = .replicated, + }; + } +}; + +/// Input has shape {.t, .d}; routing IDs and weights have shape {.t, .topk}. +/// Returns {.t, .d}. Flatten batched input with merge(.{ .t = .{ .b, .s } }). +pub fn forwardMoe( + input: zml.Tensor, + topk_ids: zml.Tensor, + topk_weights: zml.Tensor, + gate_up: zml.nn.Linear, + down: zml.nn.Linear, + backend: Backend, + mesh: *const zml.Mesh, + partitioning: Partitioning, + opts: Options, +) zml.Tensor { + validateRoutingInputs(input, topk_ids, topk_weights); + + const shapes = .{ + .input = input.shape().withPartitioning(mesh, partitioning.in_specs.input), + .ids = topk_ids.shape().withPartitioning(mesh, partitioning.in_specs.ids), + .weights = topk_weights.shape().withPartitioning(mesh, partitioning.in_specs.weights), + .gate_up = gate_up.weight.shape().withPartitioning(mesh, partitioning.in_specs.gate_up), + .down = down.weight.shape().withPartitioning(mesh, partitioning.in_specs.down), + }; + const output_shape = input.shape().withPartitioning(mesh, partitioning.out_specs); + + const tokens = shapes.input.partition(.t); + const experts = shapes.gate_up.partition(.expert); + // Local kernels require complete feature vectors and top-k routing lists. + stdx.debug.assert(shapes.input.partition(.d) == .replicated and shapes.ids.partition(.topk) == .replicated and shapes.weights.partition(.topk) == .replicated, "MoE feature and top-k dimensions must be replicated", .{}); + // The expert-axis reduction must add contributions for the same token rows. + // Sharding tokens along that axis would mix contributions for different tokens. + stdx.debug.assert(shapes.ids.partition(.t) == tokens and shapes.weights.partition(.t) == tokens and output_shape.partition(.t) == tokens and output_shape.partition(.d) == .replicated, "MoE input, routing and output must have matching token partitioning", .{}); + stdx.debug.assert(shapes.down.partition(.expert) == experts and (!experts.isSharded() or experts != tokens), "MoE projections must agree on expert partitioning; tokens must be replicated along the expert axis", .{}); + for ([_]zml.Shape{ shapes.gate_up, shapes.down }) |shape| { + stdx.debug.assert(!shape.containsPartitionSpec(.open), "MoE kernel weight partitioning must be closed", .{}); + for (0..shape.rank()) |axis| { + stdx.debug.assert(axis == shape.axis(.expert) or shape.partition(axis) == .replicated, "MoE kernel weights can only be partitioned along experts", .{}); + } + } + const expert_axis = experts.meshAxis(); + const parallel = if (expert_axis) |axis| mesh.grid.axis_sizes[axis] > 1 else false; + if (backend == .metal) stdx.debug.assert(!parallel, "This MoE backend does not support expert parallelism", .{}); + + const Call = struct { + input: zml.Tensor, + ids: zml.Tensor, + weights: zml.Tensor, + gate_up: zml.nn.Linear, + down: zml.nn.Linear, + backend: Backend, + opts: Options, + mesh: *const zml.Mesh, + expert_axis: ?u3, + global_experts: u32, + + fn forward(args: @This(), _: zml.Shape) zml.Tensor { + const routing: LocalRouting = .{ + .ids = args.ids, + .weights = args.weights, + .global_num_experts = args.global_experts, + .local_num_experts = @intCast(args.gate_up.weight.dim(.expert)), + .expert_start = if (args.expert_axis) |axis| zml.ops.axisIndex(args.mesh, args.mesh.logical.axes.get(axis)).convert(.i32).scale(args.gate_up.weight.dim(.expert)) else null, + }; + const output = switch (args.backend) { + .stablehlo => stablehlo.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), + .cute => cute.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), + .flashinfer_cutlass => flashinfer_cutlass.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), + .triton => triton.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), + .fly => fly.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), + .mosaic_tpu => mosaic_tpu.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), + .metal => metal.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), + .tcl => tcl.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), + }; + return if (args.expert_axis) |axis| zml.ops.allReduceOnAxis(output, zml.Tensor.add, args.mesh, args.mesh.logical.axes.get(axis)) else output; + } + }; + const args: Call = .{ + .input = input, + .ids = topk_ids, + .weights = topk_weights, + .gate_up = gate_up, + .down = down, + .backend = backend, + .opts = opts, + .mesh = mesh, + .expert_axis = if (parallel) expert_axis else null, + .global_experts = @intCast(gate_up.weight.dim(.expert)), + }; + const Logical = zml.Sharding.Partitioning.Logical; + var projection_specs: [2]zml.ops.PartitionSpecs(zml.nn.Linear) = undefined; + for ([_]zml.nn.Linear{ gate_up, down }, &projection_specs) |linear, *specs| { + zml.meta.mapAlloc(struct { + fn spec(ctx: struct { experts: i64, axis: ?zml.Shape.Tag }, tensor: zml.Tensor) Logical { + const axis = ctx.axis orelse return .replicated; + if (tensor.count() == 1) return .replicated; + // Packed scales may be untagged; experts are their leading dimension. + const dim = tensor.shape().hasTag(.expert) orelse 0; + stdx.debug.assert(tensor.dim(dim) == ctx.experts, "MoE tensor must contain {} experts, got {f}", .{ ctx.experts, tensor.shape() }); + return .{ .mapping = .{ + .src = .init(&.{if (tensor.shape().hasTag(.expert) != null) zml.Shape.toTag(.expert) else zml.Shape.toTag(._0)}), + .tgt = .init(&.{axis}), + } }; + } + }.spec, zml.Compiler.current().arena.allocator(), .{ + .experts = linear.weight.dim(.expert), + .axis = if (expert_axis) |axis| mesh.logical.axes.get(axis) else null, + }, linear, specs) catch @panic("OOM"); + } + return zml.ops.manualComputation(Call.forward, args, output_shape, mesh, .{ + .in_specs = .{ + .input = partitioning.in_specs.input, + .ids = partitioning.in_specs.ids, + .weights = partitioning.in_specs.weights, + .gate_up = projection_specs[0], + .down = projection_specs[1], + }, + .out_specs = partitioning.out_specs, + }); +} diff --git a/zml/kernels/moe/mosaic_tpu/BUILD.bazel b/zml/kernels/moe/mosaic_tpu/BUILD.bazel new file mode 100644 index 00000000..ed4dfcd9 --- /dev/null +++ b/zml/kernels/moe/mosaic_tpu/BUILD.bazel @@ -0,0 +1,9 @@ +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "mosaic_tpu", + srcs = [ + "gmm_ep.zig", + "mosaic_tpu.zig", + ], +) diff --git a/zml/kernels/moe/mosaic_tpu/gmm_ep.zig b/zml/kernels/moe/mosaic_tpu/gmm_ep.zig new file mode 100644 index 00000000..c30d805b --- /dev/null +++ b/zml/kernels/moe/mosaic_tpu/gmm_ep.zig @@ -0,0 +1,573 @@ +const std = @import("std"); + +const zml = @import("../../../zml.zig"); +const mtt = @import("dsl/mosaic_tpu/builder"); +const pipeline = mtt.pipeline; +const mtt_kernel = zml.kernel.mosaic_tpu; + +fn cdiv(x: i64, y: i64) i64 { + return @divFloor(x + y - 1, y); +} + +fn alignTo(x: i64, y: i64) i64 { + return cdiv(x, y) * y; +} + +fn floorModI32(b: *mtt.Builder, x: mtt.Value, y: mtt.Value) mtt.Value { + const c0 = b.constant(.i32, 0); + const r = b.remsi(x, y); + const r_nz = b.cmpi(.ne, r, c0); + const r_neg = b.cmpi(.slt, r, c0); + return b.select(b.andi(r_neg, r_nz), b.addi(r, y), r); +} + +fn floorDivPosI32(b: *mtt.Builder, x: mtt.Value, y: mtt.Value) mtt.Value { + const c0 = b.constant(.i32, 0); + const c1 = b.constant(.i32, 1); + const q = b.divsi(x, y); + const sign = b.subi( + b.extui(b.cmpi(.sgt, x, c0), .i32), + b.extui(b.cmpi(.slt, x, c0), .i32), + ); + const sign_not_pos = b.cmpi(.ne, sign, c1); + const rem_nz = b.cmpi(.ne, b.remsi(x, y), c0); + return b.select(b.andi(sign_not_pos, rem_nz), b.subi(q, c1), q); +} + +fn ceilDivPosI32(b: *mtt.Builder, x: mtt.Value, y: mtt.Value) mtt.Value { + return b.divsi(b.addi(x, b.subi(y, b.constant(.i32, 1))), y); +} + +pub const Cfg = struct { + pub const FuseAct = enum { + none, + silu, + gelu, + }; + + size_m: i64, + size_k: i64, + size_n: i64, + size_group: i64, + size_lhs_group: i64, + tile_m: i64, + tile_k: i64, + tile_n: i64, + lhs_dtype: mtt.DType = .bf16, + rhs_dtype: mtt.DType = .bf16, + out_dtype: mtt.DType = .bf16, + acc_dtype: mtt.DType = .f32, + size_lhs_sublane: i64 = 8, + fuse_act: FuseAct = .none, + + pub fn outSizeN(self: Cfg) i64 { + return switch (self.fuse_act) { + .none => self.size_n, + .silu, .gelu => @divExact(self.size_n, 2), + }; + } + + pub fn alignedN(self: Cfg) i64 { + return switch (self.fuse_act) { + .none => alignTo(self.outSizeN(), 128), + .silu, .gelu => self.outSizeN(), + }; + } + + pub fn rhsTileN(self: Cfg) i64 { + return switch (self.fuse_act) { + .none => self.tile_n, + .silu, .gelu => self.tile_n * 2, + }; + } + + pub fn maxNumGm(self: Cfg) i64 { + return self.size_group + cdiv(self.size_m, self.tile_m) - 1; + } +}; + +fn validateConfig(cfg: Cfg) void { + std.debug.assert(cfg.size_group <= cfg.size_lhs_group); + std.debug.assert(cfg.size_m > 0); + std.debug.assert(cfg.size_k > 0); + std.debug.assert(cfg.size_n > 0); + std.debug.assert(cfg.tile_m > 0); + std.debug.assert(cfg.tile_k > 0); + std.debug.assert(cfg.tile_n > 0); + std.debug.assert(cfg.size_lhs_sublane > 0); + std.debug.assert(@mod(cfg.size_m, cfg.size_lhs_sublane) == 0); + std.debug.assert(cdiv(cfg.size_k, cfg.tile_k) == 1); + std.debug.assert(@mod(cfg.tile_k, 128) == 0); + std.debug.assert(@mod(cfg.rhsTileN(), 128) == 0); + if (cfg.fuse_act != .none) { + std.debug.assert(@mod(cfg.size_n, 2) == 0); + std.debug.assert(cdiv(cfg.alignedN(), cfg.tile_n) == 1); + } else { + std.debug.assert(@mod(cfg.alignedN(), cfg.tile_n) == 0); + } + std.debug.assert(cfg.acc_dtype == .f32); +} + +const Meta = struct { + group_id: mtt.Value, + m_offset: mtt.Value, +}; + +const PipeCtx = struct { + cfg: Cfg, + meta: Meta, + num_gm: mtt.Value, +}; + +fn winRef(b: *mtt.Builder, rw: pipeline.RefWindow, shape: []const i64) mtt.Value { + const buf = rw.buf.?; + const slot = rw.slot orelse return buf; + const a = b.arena.allocator(); + const rank = shape.len + 1; + const base = a.alloc(mtt.Value, rank) catch @panic("gmm_ep winRef OOM"); + const result_shape = a.alloc(i64, rank) catch @panic("gmm_ep winRef OOM"); + base[0] = slot; + result_shape[0] = 1; + for (shape, 0..) |d, i| { + base[i + 1] = b.constant(.i32, 0); + result_shape[i + 1] = d; + } + const sliced = b.memRefSlice(buf, base, result_shape, &.{}); + return b.memRefSqueeze(sliced, shape); +} + +fn metaLoad(b: *mtt.Builder, ref: mtt.Value, idx: mtt.Value) mtt.Value { + return b.scalarLoad(ref, &.{b.toIndex(idx)}); +} + +fn lhsIndexMap(b: *mtt.Builder, indices: []const mtt.Value, ctx_opaque: ?*anyopaque) []const pipeline.BlockIndex { + const ctx: *PipeCtx = @ptrCast(@alignCast(ctx_opaque.?)); + const cfg = ctx.cfg; + const gm_id = indices[1]; + const k_id = indices[2]; + + const m_start = metaLoad(b, ctx.meta.m_offset, gm_id); + const m_end = metaLoad(b, ctx.meta.m_offset, b.addi(gm_id, b.constant(.i32, 1))); + const sublane = b.constant(.i32, cfg.size_lhs_sublane); + const row_start = floorDivPosI32(b, m_start, sublane); + const row_end = ceilDivPosI32(b, m_end, sublane); + const row_size = b.subi(row_end, row_start); + + const out = b.arena.allocator().alloc(pipeline.BlockIndex, 3) catch @panic("gmm_ep lhsIndexMap OOM"); + out[0] = .{ .slice = .{ .start = row_start, .size = row_size } }; + out[1] = .{ .scalar = b.constant(.i32, 0) }; + out[2] = .{ .scalar = k_id }; + return out; +} + +fn rhsIndexMap(b: *mtt.Builder, indices: []const mtt.Value, ctx_opaque: ?*anyopaque) []const pipeline.BlockIndex { + const ctx: *PipeCtx = @ptrCast(@alignCast(ctx_opaque.?)); + const group_id = metaLoad(b, ctx.meta.group_id, indices[1]); + const out = b.arena.allocator().alloc(pipeline.BlockIndex, 3) catch @panic("gmm_ep rhsIndexMap OOM"); + out[0] = .{ .scalar = group_id }; + out[1] = .{ .scalar = indices[2] }; + out[2] = .{ .scalar = indices[0] }; + return out; +} + +fn outIndexMap(b: *mtt.Builder, indices: []const mtt.Value, ctx_opaque: ?*anyopaque) []const pipeline.BlockIndex { + const ctx: *PipeCtx = @ptrCast(@alignCast(ctx_opaque.?)); + const cfg = ctx.cfg; + const gm_id = indices[1]; + const n_id = indices[0]; + + const m_start = metaLoad(b, ctx.meta.m_offset, gm_id); + const m_end = metaLoad(b, ctx.meta.m_offset, b.addi(gm_id, b.constant(.i32, 1))); + const sublane = b.constant(.i32, cfg.size_lhs_sublane); + const row_start = floorDivPosI32(b, m_start, sublane); + const capped_row_end = floorDivPosI32(b, m_end, sublane); + const last_row_end = ceilDivPosI32(b, m_end, sublane); + const row_end = b.select( + b.cmpi(.eq, gm_id, b.subi(ctx.num_gm, b.constant(.i32, 1))), + last_row_end, + capped_row_end, + ); + const row_size = b.subi(row_end, row_start); + + const out = b.arena.allocator().alloc(pipeline.BlockIndex, 3) catch @panic("gmm_ep outIndexMap OOM"); + out[0] = .{ .slice = .{ .start = row_start, .size = row_size } }; + out[1] = .{ .scalar = b.constant(.i32, 0) }; + out[2] = .{ .scalar = n_id }; + return out; +} + +fn indices1DTo3D(b: *mtt.Builder, indices: []const mtt.Value) []const mtt.Value { + const out = b.arena.allocator().alloc(mtt.Value, 3) catch @panic("gmm_ep indices1DTo3D OOM"); + out[0] = b.constant(.i32, 0); + out[1] = indices[0]; + out[2] = b.constant(.i32, 0); + return out; +} + +fn lhsIndexMap1D(b: *mtt.Builder, indices: []const mtt.Value, ctx_opaque: ?*anyopaque) []const pipeline.BlockIndex { + return lhsIndexMap(b, indices1DTo3D(b, indices), ctx_opaque); +} + +fn rhsIndexMap1D(b: *mtt.Builder, indices: []const mtt.Value, ctx_opaque: ?*anyopaque) []const pipeline.BlockIndex { + return rhsIndexMap(b, indices1DTo3D(b, indices), ctx_opaque); +} + +fn outIndexMap1D(b: *mtt.Builder, indices: []const mtt.Value, ctx_opaque: ?*anyopaque) []const pipeline.BlockIndex { + return outIndexMap(b, indices1DTo3D(b, indices), ctx_opaque); +} + +fn fillMetadata(b: *mtt.Builder, cfg: Cfg, a: anytype) mtt.Value { + const c0 = b.constant(.i32, 0); + const c1 = b.constant(.i32, 1); + const group_offset = b.scalarLoad(a.group_offset, &.{b.constIndex(0)}); + const max_num_group = b.addi(group_offset, b.constant(.i32, cfg.size_group)); + b.scalarStore(a.metadata_m_offset, c0, &.{b.constIndex(0)}); + + var outer = b.openFor(c0, max_num_group, c1, .{ c0, c0 }); + { + b.traceStart(10, "outer_group_loop"); + const lhs_group_id = outer.iv; + const num_gm = outer.carried[0]; + const start_m_offset = outer.carried[1]; + const group_id = b.subi(lhs_group_id, group_offset); + const group_size = b.scalarLoad(a.group_sizes, &.{b.toIndex(lhs_group_id)}); + const end_m_offset = b.addi(start_m_offset, group_size); + const sublane = b.constant(.i32, cfg.size_lhs_sublane); + const local_offset = floorModI32(b, start_m_offset, sublane); + const aligned_group_size = b.addi(group_size, local_offset); + const tile_m = b.constant(.i32, cfg.tile_m); + var curr_num_gm = b.divsi(b.addi(aligned_group_size, b.subi(tile_m, c1)), tile_m); + const should_process = b.andi(b.cmpi(.sgt, group_size, c0), b.cmpi(.sge, group_id, c0)); + curr_num_gm = b.select(should_process, curr_num_gm, c0); + const next_num_gm = b.addi(num_gm, curr_num_gm); + + var inner = b.openFor(num_gm, next_num_gm, c1, .{start_m_offset}); + { + b.traceStart(10, "inner_tm_loop"); + const tm_id = inner.iv; + const curr_m_offset = inner.carried[0]; + const local = floorModI32(b, curr_m_offset, sublane); + const tm_room = b.subi(tile_m, local); + const remaining = b.subi(end_m_offset, curr_m_offset); + const tm_size = b.minsi(tm_room, remaining); + const next_m_offset = b.addi(curr_m_offset, tm_size); + b.scalarStore(a.metadata_group_id, group_id, &.{b.toIndex(tm_id)}); + b.scalarStore(a.metadata_m_offset, curr_m_offset, &.{b.toIndex(tm_id)}); + b.scalarStore(a.metadata_m_offset, next_m_offset, &.{b.toIndex(b.addi(tm_id, c1))}); + b.traceStop(); + inner.yield(.{next_m_offset}); + } + + b.traceStop(); + outer.yield(.{ next_num_gm, end_m_offset }); + } + return outer.results[0]; +} + +fn matmulByMxuColumns(b: *mtt.Builder, cfg: Cfg, lhs: mtt.Value, rhs: mtt.Value, cols: i64) mtt.Value { + if (cfg.tile_k > cfg.size_k) { + std.debug.assert(cdiv(cfg.size_k, cfg.tile_k) == 1); + std.debug.assert(@mod(cols, 128) == 0); + + const chunks = @divExact(cols, 128); + const parts = b.arena.allocator().alloc(mtt.Value, @intCast(chunks)) catch @panic("gmm_ep matmul parts OOM"); + const tail_k = cfg.tile_k - cfg.size_k; + const lhs_valid = b.vectorExtractStridedSlice(lhs, &.{ 0, 0 }, &.{ cfg.tile_m, cfg.size_k }, &.{ 1, 1 }); + const lhs_tail = b.vectorExtractStridedSlice(lhs, &.{ 0, cfg.size_k }, &.{ cfg.tile_m, tail_k }, &.{ 1, 1 }); + + var rev: i64 = 0; + while (rev < chunks) : (rev += 1) { + const chunk = chunks - 1 - rev; + const start_n = chunk * 128; + const rhs_valid = b.vectorExtractStridedSlice(rhs, &.{ 0, start_n }, &.{ cfg.size_k, 128 }, &.{ 1, 1 }); + const rhs_tail = b.vectorExtractStridedSlice(rhs, &.{ cfg.size_k, start_n }, &.{ tail_k, 128 }, &.{ 1, 1 }); + const zero = b.zeros(&.{ cfg.tile_m, 128 }, cfg.acc_dtype); + const tail_acc = b.matmulOpts(lhs_tail, rhs_tail, zero, .{ + .dimension_numbers = b.dotDimensionNumbers(&.{1}, &.{0}, &.{0}, &.{1}, &.{ 0, 0, 1, 1 }, &.{}, &.{}), + }); + parts[@intCast(chunk)] = b.matmulOpts(lhs_valid, rhs_valid, tail_acc, .{ + .dimension_numbers = b.dotDimensionNumbers(&.{1}, &.{0}, &.{0}, &.{1}, &.{ 0, 0, 1, 1 }, &.{}, &.{}), + }); + } + + if (chunks == 1) return parts[0]; + + const max_concat_inputs = 16; + if (chunks <= max_concat_inputs) return b.concatenate(parts, 1); + + const groups = cdiv(chunks, max_concat_inputs); + const group_parts = b.arena.allocator().alloc(mtt.Value, @intCast(groups)) catch @panic("gmm_ep matmul group parts OOM"); + var group: i64 = 0; + while (group < groups) : (group += 1) { + const start = group * max_concat_inputs; + const end = @min(start + max_concat_inputs, chunks); + const start_usize: usize = @intCast(start); + const end_usize: usize = @intCast(end); + const group_slice = parts[start_usize..end_usize]; + group_parts[@intCast(group)] = if (group_slice.len == 1) group_slice[0] else b.concatenate(group_slice, 1); + } + return b.concatenate(group_parts, 1); + } + + if (cols <= 128) { + return b.matmulOpts(lhs, rhs, b.zeros(&.{ cfg.tile_m, cols }, cfg.acc_dtype), .{ + .dimension_numbers = b.dotDimensionNumbers(&.{1}, &.{0}, &.{0}, &.{1}, &.{ 0, 0, 1, 1 }, &.{}, &.{}), + }); + } + + std.debug.assert(@mod(cols, 128) == 0); + const chunks = @divExact(cols, 128); + const parts = b.arena.allocator().alloc(mtt.Value, @intCast(chunks)) catch @panic("gmm_ep matmul parts OOM"); + var rev: i64 = 0; + while (rev < chunks) : (rev += 1) { + const chunk = chunks - 1 - rev; + const start_n = chunk * 128; + const rhs_chunk = b.vectorExtractStridedSlice(rhs, &.{ 0, start_n }, &.{ cfg.tile_k, 128 }, &.{ 1, 1 }); + const zero = b.zeros(&.{ cfg.tile_m, 128 }, cfg.acc_dtype); + parts[@intCast(chunk)] = b.matmulOpts(lhs, rhs_chunk, zero, .{ + .dimension_numbers = b.dotDimensionNumbers(&.{1}, &.{0}, &.{0}, &.{1}, &.{ 0, 0, 1, 1 }, &.{}, &.{}), + }); + } + + if (chunks == 1) return parts[0]; + + const max_concat_inputs = 16; + if (chunks <= max_concat_inputs) return b.concatenate(parts, 1); + + const groups = cdiv(chunks, max_concat_inputs); + const group_parts = b.arena.allocator().alloc(mtt.Value, @intCast(groups)) catch @panic("gmm_ep matmul group parts OOM"); + var group: i64 = 0; + while (group < groups) : (group += 1) { + const start = group * max_concat_inputs; + const end = @min(start + max_concat_inputs, chunks); + const start_usize: usize = @intCast(start); + const end_usize: usize = @intCast(end); + const group_slice = parts[start_usize..end_usize]; + group_parts[@intCast(group)] = if (group_slice.len == 1) group_slice[0] else b.concatenate(group_slice, 1); + } + return b.concatenate(group_parts, 1); +} + +fn geluApprox(b: *mtt.Builder, x: mtt.Value, shape: []const i64) mtt.Value { + const x2 = b.mulf(x, x); + const x3 = b.mulf(x2, x); + const cubic = b.mulf(x3, b.splat(@as(f32, 0.044715), shape, cfgAccDtype())); + const inner = b.mulf(b.addf(x, cubic), b.splat(@as(f32, 0.7978845608028654), shape, cfgAccDtype())); + return b.mulf( + b.mulf(x, b.splat(@as(f32, 0.5), shape, cfgAccDtype())), + b.addf(b.tanh(inner), b.splat(@as(f32, 1.0), shape, cfgAccDtype())), + ); +} + +fn cfgAccDtype() mtt.DType { + return .f32; +} + +fn silu(b: *mtt.Builder, x: mtt.Value, shape: []const i64) mtt.Value { + const one = b.splat(@as(f32, 1.0), shape, cfgAccDtype()); + const denom = b.addf(one, b.exp(b.negf(x))); + return b.divf(x, denom); +} + +fn applyFuseAct(b: *mtt.Builder, cfg: Cfg, acc: mtt.Value) mtt.Value { + return switch (cfg.fuse_act) { + .none => acc, + .silu, .gelu => { + const shape = &.{ cfg.tile_m, cfg.tile_n }; + const gate = b.vectorExtractStridedSlice(acc, &.{ 0, 0 }, shape, &.{ 1, 1 }); + const up = b.vectorExtractStridedSlice(acc, &.{ 0, cfg.outSizeN() }, shape, &.{ 1, 1 }); + const activated = switch (cfg.fuse_act) { + .silu => silu(b, gate, shape), + .gelu => geluApprox(b, gate, shape), + .none => unreachable, + }; + return b.mulf(activated, up); + }, + }; +} + +fn innerBody( + b: *mtt.Builder, + indices: []const mtt.Value, + refs: []const pipeline.RefWindow, + scratches: []const mtt.Value, + ctx_opaque: ?*anyopaque, +) mtt.FinishError!void { + const ctx: *PipeCtx = @ptrCast(@alignCast(ctx_opaque.?)); + const cfg = ctx.cfg; + const gm_id = indices[1]; + + const lhs_shape = &.{ cdiv(cfg.tile_m, cfg.size_lhs_sublane), cfg.size_lhs_sublane, cfg.tile_k }; + const out_shape = &.{ cdiv(cfg.tile_m, cfg.size_lhs_sublane), cfg.size_lhs_sublane, cfg.tile_n }; + const rhs_tile_n = cfg.rhsTileN(); + + b.traceStart(10, "matmul_first_last"); + const lhs_ref = winRef(b, refs[0], lhs_shape); + const lhs_ref_2d = b.memRefReshape(lhs_ref, &.{ cfg.tile_m, cfg.tile_k }); + const lhs = b.refLoad(lhs_ref_2d); + const rhs_slot = b.toIndex(refs[1].slot.?); + var rhs = b.shapeCast( + b.vectorLoadShape(refs[1].buf.?, &.{ rhs_slot, b.constIndex(0), b.constIndex(0) }, &.{ 1, cfg.tile_k, rhs_tile_n }), + &.{ cfg.tile_k, rhs_tile_n }, + ); + const valid_k = @mod(cfg.size_k, cfg.tile_k); + if (valid_k != 0) { + const k_iota = b.iota(&.{ cfg.tile_k, rhs_tile_n }, .i32, &.{0}); + const valid = b.splat(@as(i32, @intCast(valid_k)), &.{ cfg.tile_k, rhs_tile_n }, .i32); + rhs = b.select(b.cmpi(.slt, k_iota, valid), rhs, b.zeros(&.{ cfg.tile_k, rhs_tile_n }, cfg.rhs_dtype)); + } + var acc = matmulByMxuColumns(b, cfg, lhs, rhs, rhs_tile_n); + acc = applyFuseAct(b, cfg, acc); + + const m_start = metaLoad(b, ctx.meta.m_offset, gm_id); + const m_end = metaLoad(b, ctx.meta.m_offset, b.addi(gm_id, b.constant(.i32, 1))); + const sublane = b.constant(.i32, cfg.size_lhs_sublane); + const m_offset = b.subi(m_start, floorModI32(b, m_start, sublane)); + const m_start_local = b.subi(m_start, m_offset); + const m_end_local = b.subi(m_end, m_offset); + const row_iota = b.iota(&.{ cfg.tile_m, cfg.tile_n }, .i32, &.{0}); + const row_mask = b.andi( + b.cmpi(.sle, b.broadcastTo(m_start_local, &.{ cfg.tile_m, cfg.tile_n }), row_iota), + b.cmpi(.slt, row_iota, b.broadcastTo(m_end_local, &.{ cfg.tile_m, cfg.tile_n })), + ); + acc = b.select(row_mask, acc, b.zeros(&.{ cfg.tile_m, cfg.tile_n }, cfg.acc_dtype)); + + const out_slot_i32 = refs[2].slot.?; + const out_slot = b.toIndex(out_slot_i32); + b.vectorStoreAt( + refs[2].buf.?, + b.shapeCast(b.shapeCast(acc, out_shape).to(cfg.out_dtype), &.{ 1, cdiv(cfg.tile_m, cfg.size_lhs_sublane), cfg.size_lhs_sublane, cfg.tile_n }), + &.{ out_slot, b.constIndex(0), b.constIndex(0), b.constIndex(0) }, + ); + + const out_ref = winRef(b, refs[2], out_shape); + if (cfg.tile_m == cfg.size_lhs_sublane) { + const out_value = b.shapeCast(b.refLoad(out_ref), &.{ cfg.tile_m, cfg.tile_n }); + const partial_zero = b.zeros(&.{ cfg.tile_m, cfg.tile_n }, cfg.out_dtype); + const partial_prev = b.select( + b.cmpi(.eq, gm_id, b.constant(.i32, 0)), + partial_zero, + b.refLoad(scratches[0]), + ); + const out_accum = b.addf(out_value, partial_prev); + const out_ref_for_store = winRef(b, refs[2], out_shape); + b.refStore(out_ref_for_store, b.shapeCast(out_accum, out_shape)); + } else { + const first_row = b.shapeCast( + b.vectorLoadShape(out_ref, &.{ b.constIndex(0), b.constIndex(0), b.constIndex(0) }, &.{ 1, cfg.size_lhs_sublane, cfg.tile_n }), + &.{ cfg.size_lhs_sublane, cfg.tile_n }, + ); + const partial_zero = b.zeros(&.{ cfg.size_lhs_sublane, cfg.tile_n }, cfg.out_dtype); + const partial_prev = b.select( + b.cmpi(.eq, gm_id, b.constant(.i32, 0)), + partial_zero, + b.refLoad(scratches[0]), + ); + const out_accum = b.addf(first_row, partial_prev); + const out_ref_for_store = winRef(b, refs[2], out_shape); + b.vectorStoreAt( + out_ref_for_store, + b.shapeCast(out_accum, &.{ 1, cfg.size_lhs_sublane, cfg.tile_n }), + &.{ b.constIndex(0), b.constIndex(0), b.constIndex(0) }, + ); + } + + const last_row = floorDivPosI32(b, m_end_local, sublane); + const m_end_mod = floorModI32(b, m_end_local, sublane); + const partial_complete = b.cmpi(.eq, m_end_mod, b.constant(.i32, 0)); + const out_ref_for_partial = winRef(b, refs[2], out_shape); + const partial_shape = if (cfg.tile_m == cfg.size_lhs_sublane) + &.{ cfg.tile_m, cfg.tile_n } + else + &.{ cfg.size_lhs_sublane, cfg.tile_n }; + const partial_zero = b.zeros(partial_shape, cfg.out_dtype); + const partial_row = b.shapeCast( + b.vectorLoadShape(out_ref_for_partial, &.{ b.toIndex(last_row), b.constIndex(0), b.constIndex(0) }, &.{ 1, cfg.size_lhs_sublane, cfg.tile_n }), + partial_shape, + ); + b.refStore(scratches[0], b.select(partial_complete, partial_zero, partial_row)); + b.traceStop(); +} + +fn innerBody1D( + b: *mtt.Builder, + indices: []const mtt.Value, + refs: []const pipeline.RefWindow, + scratches: []const mtt.Value, + ctx_opaque: ?*anyopaque, +) mtt.FinishError!void { + try innerBody(b, indices1DTo3D(b, indices), refs, scratches, ctx_opaque); +} + +pub fn run(b: *mtt.Builder, cfg: Cfg) mtt.FinishError!void { + validateConfig(cfg); + + const a = try b.declareArgsOpts(.{ + .group_sizes = .{ .ref = .{ .shape = &.{cfg.size_lhs_group}, .dtype = .i32, .memory_space = .smem, .role = .scalar_prefetch } }, + .group_offset = .{ .ref = .{ .shape = &.{1}, .dtype = .i32, .memory_space = .smem, .role = .scalar_prefetch } }, + .lhs = .{ .ref = .{ .shape = &.{ cfg.size_m, cfg.size_k }, .dtype = cfg.lhs_dtype, .memory_space = .hbm } }, + .rhs = .{ .ref = .{ .shape = &.{ cfg.size_group, cfg.size_k, cfg.size_n }, .dtype = cfg.rhs_dtype, .memory_space = .hbm } }, + .out = .{ .ref = .{ .shape = &.{ cfg.size_m, cfg.alignedN() }, .dtype = cfg.out_dtype, .memory_space = .hbm, .role = .output } }, + .partial_out = .{ .ref = .{ .shape = &.{ cfg.size_lhs_sublane, cfg.tile_n }, .dtype = cfg.out_dtype, .role = .scratch } }, + .acc = .{ .ref = .{ .shape = &.{ cfg.tile_m, cfg.tile_n }, .dtype = cfg.acc_dtype, .role = .scratch } }, + .metadata_group_id = .{ .ref = .{ .shape = &.{cfg.maxNumGm()}, .dtype = .i32, .memory_space = .smem, .role = .scratch } }, + .metadata_m_offset = .{ .ref = .{ .shape = &.{cfg.maxNumGm() + 1}, .dtype = .i32, .memory_space = .smem, .role = .scratch } }, + }, &.{}, .{ + .dimension_semantics = &.{}, + .scalar_prefetch = 2, + .scratch_operands = 4, + }); + + const num_k = b.constant(.i32, cdiv(cfg.size_k, cfg.tile_k)); + const num_n = b.constant(.i32, cdiv(cfg.alignedN(), cfg.tile_n)); + const num_gm = fillMetadata(b, cfg, a); + + const ctx = b.arena.allocator().create(PipeCtx) catch @panic("gmm_ep PipeCtx OOM"); + ctx.* = .{ + .cfg = cfg, + .meta = .{ .group_id = a.metadata_group_id, .m_offset = a.metadata_m_offset }, + .num_gm = num_gm, + }; + + const lhs_spec = pipeline.PipeSpec{ + .full_shape = &.{ @divExact(cfg.size_m, cfg.size_lhs_sublane), cfg.size_lhs_sublane, cfg.size_k }, + .dtype = cfg.lhs_dtype, + .source_reshape_shape = &.{ @divExact(cfg.size_m, cfg.size_lhs_sublane), cfg.size_lhs_sublane, cfg.size_k }, + .block_shape = &.{ .{ .bounded_slice = cdiv(cfg.tile_m, cfg.size_lhs_sublane) }, .{ .blocked = cfg.size_lhs_sublane }, .{ .blocked = cfg.tile_k } }, + .index_map = if (cdiv(cfg.alignedN(), cfg.tile_n) == 1 and cdiv(cfg.size_k, cfg.tile_k) == 1 and cfg.size_m > cfg.tile_m) lhsIndexMap1D else lhsIndexMap, + .index_map_ctx = ctx, + .bounded_slice_tiling = 1, + }; + const rhs_spec = pipeline.PipeSpec{ + .full_shape = &.{ cfg.size_group, cfg.size_k, cfg.size_n }, + .dtype = cfg.rhs_dtype, + .block_shape = &.{ .squeezed, .{ .blocked = cfg.tile_k }, .{ .blocked = cfg.rhsTileN() } }, + .index_map = if (cdiv(cfg.alignedN(), cfg.tile_n) == 1 and cdiv(cfg.size_k, cfg.tile_k) == 1 and cfg.size_m > cfg.tile_m) rhsIndexMap1D else rhsIndexMap, + .index_map_ctx = ctx, + .buffer_count = 3, + }; + const out_spec = pipeline.PipeSpec{ + .full_shape = &.{ @divExact(cfg.size_m, cfg.size_lhs_sublane), cfg.size_lhs_sublane, cfg.alignedN() }, + .dtype = cfg.out_dtype, + .source_reshape_shape = &.{ @divExact(cfg.size_m, cfg.size_lhs_sublane), cfg.size_lhs_sublane, cfg.alignedN() }, + .block_shape = &.{ .{ .bounded_slice = cdiv(cfg.tile_m, cfg.size_lhs_sublane) }, .{ .blocked = cfg.size_lhs_sublane }, .{ .blocked = cfg.tile_n } }, + .index_map = if (cdiv(cfg.alignedN(), cfg.tile_n) == 1 and cdiv(cfg.size_k, cfg.tile_k) == 1 and cfg.size_m > cfg.tile_m) outIndexMap1D else outIndexMap, + .index_map_ctx = ctx, + .bounded_slice_tiling = 1, + }; + const use_1d_pipeline = cdiv(cfg.alignedN(), cfg.tile_n) == 1 and cdiv(cfg.size_k, cfg.tile_k) == 1 and cfg.size_m > cfg.tile_m; + try pipeline.emitPipeline(b, &.{ a.lhs, a.rhs, a.out }, &.{ a.partial_out, a.acc, a.metadata_group_id, a.metadata_m_offset }, .{ + .body = if (use_1d_pipeline) innerBody1D else innerBody, + .body_ctx = ctx, + .grid = if (use_1d_pipeline) &.{num_gm} else &.{ num_n, num_gm, num_k }, + .in_specs = &.{ lhs_spec, rhs_spec }, + .out_specs = &.{out_spec}, + }); +} + +pub const TpuKernel = mtt_kernel.Kernel(Cfg, .{ + .name = "gmm_ep_kernel", + .inputs = &.{ "group_sizes", "group_offset", "lhs", "rhs" }, + .outputs = &.{"out"}, + .run = run, +}); diff --git a/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig b/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig new file mode 100644 index 00000000..bb60594b --- /dev/null +++ b/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig @@ -0,0 +1,301 @@ +const std = @import("std"); + +const mtt = @import("dsl/mosaic_tpu/builder"); +const stdx = @import("stdx"); + +const zml = @import("../../../zml.zig"); +const Tensor = zml.Tensor; +const Activation = @import("../api.zig").Activation; +const GlobalOptions = @import("../api.zig").Options; +const LocalRouting = @import("../api.zig").LocalRouting; +pub const gmm_ep = @import("gmm_ep.zig"); + +const log = std.log.scoped(.@"zml/moe/mosaic_tpu"); + +var gate_up_transpose: std.atomic.Value(bool) = .init(false); +var down_transpose: std.atomic.Value(bool) = .init(false); + +fn validateOptions(opts: GlobalOptions) void { + stdx.debug.assert(!opts.quantize_input, "Optional FP8 input quantization requires the Triton MoE backend", .{}); + stdx.debug.assert(opts.routing_weight_placement == .after_down, "Non-Triton MoE backends require routing weights after the down projection", .{}); +} + +fn validateInputs(hidden: Tensor, gate_up: Tensor, down: Tensor, weights: Tensor, ids: Tensor) !void { + switch (hidden.dtype()) { + .bf16, .f16, .f32 => {}, + else => return error.UnsupportedType, + } + switch (gate_up.dtype()) { + .bf16, .f16, .f32 => {}, + else => return error.UnsupportedType, + } + switch (down.dtype()) { + .bf16, .f16, .f32 => {}, + else => return error.UnsupportedType, + } + switch (weights.dtype()) { + .bf16, .f16, .f32 => {}, + else => return error.UnsupportedType, + } + if (ids.dtype() != .i32) return error.UnsupportedType; + if (hidden.dim(.in) != gate_up.dim(.in)) return error.InvalidShape; + if (@rem(gate_up.dim(.out), 2) != 0) return error.InvalidShape; + if (down.dim(.mid) != @divFloor(gate_up.dim(.out), 2)) return error.InvalidShape; + if (ids.dim(.token) != hidden.dim(.token) or weights.dim(.token) != hidden.dim(.token)) return error.InvalidShape; + if (ids.dim(.topk) != weights.dim(.topk)) return error.InvalidShape; + if (gate_up.dim(.expert) != down.dim(.expert)) return error.InvalidShape; +} + +fn warningTranspose(flag: *std.atomic.Value(bool), projection_name: []const u8, expected_shape: []const u8) void { + if (!flag.swap(true, .monotonic)) { + log.warn("{s} MoE weights are transposed at runtime; weights must be loaded with the correct shape {s} to avoid the transpose at runtime.", .{ projection_name, expected_shape }); + } +} + +fn canonicalizeGateUp(w1: Tensor, hidden_size: i64) !Tensor { + if (w1.rank() != 3) return error.InvalidShape; + + if (w1.dim(1) == hidden_size) return w1.withTags(.{ .expert, .in, .out }); + if (w1.dim(2) == hidden_size) { + warningTranspose(&gate_up_transpose, "gate_up_proj", "[expert, in, out]"); + return w1.withTags(.{ .expert, .out, .in }).transpose(.{ .expert, .in, .out }); + } + return error.InvalidShape; +} + +fn canonicalizeDown(w2: Tensor, hidden_size: i64, gate_up_out: i64) !Tensor { + if (w2.rank() != 3) return error.InvalidShape; + + const mid_size = @divFloor(gate_up_out, 2); + if (w2.dim(1) == mid_size and w2.dim(2) == hidden_size) return w2.withTags(.{ .expert, .mid, .out }); + if (w2.dim(1) == hidden_size and w2.dim(2) == mid_size) { + warningTranspose(&down_transpose, "down_proj", "[expert, mid, out]"); + return w2.withTags(.{ .expert, .out, .mid }).transpose(.{ .expert, .mid, .out }); + } + return error.InvalidShape; +} + +fn histogramCounts(ids: Tensor, num_bins: i64) Tensor { + const zeros = Tensor.zeroes(.init(.{ .expert = num_bins }, .i32)); + const ones = Tensor.constant(.{ .i32 = 1 }).broad(ids.shape().withDtype(.i32)); + return zeros.scatterSlices( + .{ .expert = ids }, + ones, + .{ + .update_fn = Tensor.ScatterOpts.increment, + .indices_are_unique = false, + }, + ); +} + +fn alignSortedRowsByGroup(rows: Tensor, expert_ids_sorted: Tensor, group_sizes: Tensor, tile_m: i64) struct { + rows: Tensor, + group_sizes: Tensor, + positions: Tensor, +} { + const num_groups = group_sizes.dim(.expert); + const max_padded_rows = rows.dim(.token) + num_groups * (tile_m - 1); + + const group_ends = group_sizes.cumulativeSum(.expert); + const group_starts = Tensor.concatenate(&.{ + Tensor.zeroes(.init(.{ .expert = 1 }, .i32)), + group_ends.slice(.expert, .{ .end = num_groups - 1 }), + }, .expert); + + const padded_group_sizes = group_sizes.addConstant(tile_m - 1).divByConst(tile_m).mul(Tensor.scalar(tile_m, .i32)); + const padded_group_ends = padded_group_sizes.cumulativeSum(.expert); + const padded_group_starts = Tensor.concatenate(&.{ + Tensor.zeroes(.init(.{ .expert = 1 }, .i32)), + padded_group_ends.slice(.expert, .{ .end = num_groups - 1 }), + }, .expert); + + const logical_positions = Tensor.arange(.{ .end = rows.dim(.token) }, .i32).withTags(.{.token}); + const group_start_per_token = group_starts.gather(.{ .expert = expert_ids_sorted }, .{}); + const padded_group_start_per_token = padded_group_starts.gather(.{ .expert = expert_ids_sorted }, .{}); + const padded_positions = logical_positions.sub(group_start_per_token).add(padded_group_start_per_token).withTags(.{.token}); + + const padded = Tensor.zeroes(rows.shape().setDim(.token, max_padded_rows)) + .scatterSlices( + .{ .token = padded_positions }, + rows, + .{ .indices_are_unique = true, .update_fn = Tensor.ScatterOpts.override }, + ); + + return .{ + .rows = padded, + .group_sizes = padded_group_sizes, + .positions = padded_positions, + }; +} + +fn permuteSmallBatch(hidden: Tensor, sorted_route: Tensor, topk: i64) Tensor { + const num_tokens = hidden.dim(.token); + const num_routes = num_tokens * topk; + + const token_indices = Tensor.arange(.{ .end = num_tokens }, .i32) + .withTags(.{.token}) + .repeat1d(.token, @intCast(topk)) + .rename(.{ .token = .route }); + const token_indices_sorted = token_indices.gather(.{ .route = sorted_route }, .{}).rename(.{ .perm = .route }); + + const onehot_shape = zml.Shape.init(.{ .route = num_routes, .token_src = num_tokens }, .i32); + const token_indices_2d = token_indices_sorted.insertAxes(.last, .{.token_src}).broad(onehot_shape); + const source_tokens_2d = Tensor.arange(.{ .end = num_tokens }, .i32) + .withTags(.{.token_src}) + .insertAxes(0, .{.route}) + .broad(onehot_shape); + const onehot = token_indices_2d.cmp(.EQ, source_tokens_2d).convert(hidden.dtype()); + + return onehot.dot(hidden.rename(.{ .token = .token_src }), .token_src).rename(.{ .route = .token }); +} + +fn gmmDType(dtype: zml.DataType) mtt.DType { + return switch (zml.kernel.mosaic_tpu.from(dtype)) { + .bf16 => .bf16, + .f16 => .f16, + .f32 => .f32, + else => stdx.debug.panic("mosaic_tpu MoE expects bf16/f16/f32 tensors, got {}", .{dtype}), + }; +} + +pub fn callGmmEp( + lhs: Tensor, + rhs_k_n: Tensor, + group_sizes: Tensor, + out_dtype: zml.DataType, + fuse_act: gmm_ep.Cfg.FuseAct, +) Tensor { + const k = lhs.dim(1); + const n = rhs_k_n.dim(.out); + const out_n = switch (fuse_act) { + .none => n, + .silu, .gelu => @divExact(n, 2), + }; + const aligned_k = @divFloor(k + 127, 128) * 128; + const aligned_n = if (fuse_act == .none) @divFloor(out_n + 127, 128) * 128 else out_n; + + const cfg: gmm_ep.Cfg = .{ + .size_m = lhs.dim(.token), + .size_k = k, + .size_n = n, + .size_group = rhs_k_n.dim(.expert), + .size_lhs_group = group_sizes.dim(.expert), + .tile_m = 128, + .tile_k = aligned_k, + .tile_n = if (fuse_act == .none) aligned_n else out_n, + .lhs_dtype = gmmDType(lhs.dtype()), + .rhs_dtype = gmmDType(rhs_k_n.dtype()), + .out_dtype = gmmDType(out_dtype), + .acc_dtype = .f32, + .size_lhs_sublane = 8, + .fuse_act = fuse_act, + }; + + const out = gmm_ep.TpuKernel.call( + .{ + .group_sizes = group_sizes.withTags(.{.expert}), + .group_offset = Tensor.zeroes(.init(.{1}, .i32)), + .lhs = lhs, + .rhs = rhs_k_n, + }, + .{ + .out = zml.Shape.init(.{ .token = lhs.dim(.token), .out = aligned_n }, out_dtype), + }, + .{ + .cfg = cfg, + .extras = .{ + // todo leverage tpu api like this https://github.com/openxla/tokamax/blob/f65135e85acd7571531be91d4d3241e1929421f0/tokamax/_src/ops/ragged_dot/pallas_mosaic_tpu_v2_tgmm_kernel.py#L692 + .vmem_limit_bytes = 32 * 1024 * 1024, + }, + }, + ).out; + + return out.slice(.out, .{ .end = out_n }); +} + +pub fn fusedExperts( + hidden_states: zml.Tensor, + routing: LocalRouting, + gate_up_: zml.nn.Linear, + down_: zml.nn.Linear, + opts: GlobalOptions, +) zml.Tensor { + validateOptions(opts); + const expert_map_ = routing.expertMap(); + + const tokens = hidden_states.dim(.t); + const hidden = hidden_states.reshape(.{ .token = tokens, .in = hidden_states.dim(.d) }); + // TODO(Corentin): Better error message + const gate_up = canonicalizeGateUp(gate_up_.weight, hidden.dim(.in)) catch |e| stdx.debug.panic("Invalid gate_up weights for fusedExperts: {}", .{e}); + const down = canonicalizeDown(down_.weight, hidden.dim(.in), gate_up.dim(.out)) catch |e| stdx.debug.panic("Invalid down weights for fusedExperts: {}", .{e}); + const weights = routing.weights.reshape(.{ .token = tokens, .topk = routing.weights.dim(.topk) }); + const ids = routing.ids.reshape(.{ .token = tokens, .topk = routing.ids.dim(.topk) }); + + const num_experts = routing.global_num_experts; + if (expert_map_) |expert_map| { + if (expert_map.dtype() != .i32) stdx.debug.panic("expert_map must be of dtype i32, got {}", .{expert_map.dtype()}); + if (expert_map.rank() != 1 or expert_map.dim(.expert) != num_experts) stdx.debug.panic("expert_map must be a 1D tensor of length num_experts, got shape {}", .{expert_map.shape()}); + } + + const flat_ids_global = ids.transpose(.{ .topk, .token }).flatten().withTags(.{.route}); + const flat_weights = weights.transpose(.{ .topk, .token }).flatten().withTags(.{.route}); + + const flat_ids_local, const local_route_mask = if (expert_map_) |expert_map| blk: { + const mapped = expert_map.gather(.{ .expert = flat_ids_global }, .{}).withTags(.{.route}); + const route_mask = mapped.cmp(.GE, Tensor.scalar(0, .i32).broad(mapped.shape())); + const safe_ids = route_mask.select(mapped, Tensor.zeroes(mapped.shape())); + break :blk .{ safe_ids, route_mask }; + } else .{ + flat_ids_global, + Tensor.scalar(true, .bool).broad(flat_ids_global.shape()), + }; + + const sorted_route = flat_ids_local.argsort(.route, .{ .descending = false }).rename(.{ .route = .perm }); + const hidden_sorted = if (expert_map_ != null) + permuteSmallBatch(hidden, sorted_route, ids.dim(.topk)) + else blk: { + // If not sharded, we can save the sort + gather by just repeating the hidden states for each expert and gathering with the sorted route + const hidden_repeated = hidden.repeat1d(.token, @intCast(ids.dim(.topk))).rename(.{ .token = .route }); + break :blk hidden_repeated.gather(.{ .route = sorted_route }, .{}).rename(.{ .perm = .token }); + }; + const expert_ids_sorted = flat_ids_local.gather(.{ .route = sorted_route }, .{}).rename(.{ .perm = .token }); + const local_route_mask_sorted = local_route_mask.gather(.{ .route = sorted_route }, .{}).rename(.{ .perm = .token }); + const sorted_weights = flat_weights.gather(.{ .route = sorted_route }, .{}).rename(.{ .perm = .token }); + const router_weights_sorted = local_route_mask_sorted.select( + sorted_weights, + Tensor.zeroes(sorted_weights.shape()), + ); + + const group_sizes = histogramCounts(flat_ids_local, gate_up.dim(.expert)); + const tile_m: i64 = 128; + const aligned_hidden = alignSortedRowsByGroup(hidden_sorted, expert_ids_sorted, group_sizes, tile_m); + + var gate_up_out = callGmmEp(aligned_hidden.rows, gate_up, aligned_hidden.group_sizes, hidden.dtype(), .none) + .gather(.{ .token = aligned_hidden.positions.rename(.{ .token = .route }) }, .{}) + .rename(.{ .route = .token }); + + if (gate_up_.bias) |bias| { + const bias_per_token = bias.withTags(.{ .expert, .out }).gather(.{ .expert = expert_ids_sorted }, .{}); + gate_up_out = gate_up_out.add(bias_per_token); + } + + const activated = opts.activation.applyActivation(gate_up_out, .concatenated); + + const aligned_activated = alignSortedRowsByGroup(activated, expert_ids_sorted, group_sizes, tile_m); + var down_out = callGmmEp(aligned_activated.rows, down, aligned_activated.group_sizes, hidden.dtype(), .none) + .gather(.{ .token = aligned_activated.positions.rename(.{ .token = .route }) }, .{}) + .rename(.{ .route = .token }); + + if (down_.bias) |bias| { + const bias_per_token = bias.withTags(.{ .expert, .out }).gather(.{ .expert = expert_ids_sorted }, .{}); + down_out = down_out.add(bias_per_token); + } + + const weighted = down_out.mul(router_weights_sorted.convert(down_out.dtype()).broad(down_out.shape())); + + const restore_indices = sorted_route.argsort(.perm, .{ .descending = false }).rename(.{ .perm = .restore }); + const unsorted = weighted.gather(.{ .token = restore_indices }, .{}).rename(.{ .restore = .route }); + const restored = unsorted.reshape(.{ .topk = ids.dim(.topk), .token = hidden.dim(.token), .out = down.dim(.out) }); + const combined = restored.sum(.topk).squeeze(.topk); + return combined.reshape(.{ .t = tokens, .d = down.dim(.out) }); +} diff --git a/zml/kernels/moe/stablehlo/BUILD.bazel b/zml/kernels/moe/stablehlo/BUILD.bazel new file mode 100644 index 00000000..c31b27f0 --- /dev/null +++ b/zml/kernels/moe/stablehlo/BUILD.bazel @@ -0,0 +1,8 @@ +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "stablehlo", + srcs = [ + "stablehlo.zig", + ], +) diff --git a/zml/kernels/moe/stablehlo/stablehlo.zig b/zml/kernels/moe/stablehlo/stablehlo.zig new file mode 100644 index 00000000..ba723234 --- /dev/null +++ b/zml/kernels/moe/stablehlo/stablehlo.zig @@ -0,0 +1,225 @@ +//! Deliberately simple MoE reference: no routed GEMMs, packing, or custom calls. +//! Weights use [expert, output, input] order with concatenated gate/up columns. +const std = @import("std"); + +const stdx = @import("stdx"); + +const zml = @import("../../../zml.zig"); +const Tensor = zml.Tensor; +const Shape = zml.Shape; +const GlobalOptions = @import("../api.zig").Options; +const Activation = @import("../api.zig").Activation; +const RoutingWeightPlacement = @import("../api.zig").RoutingWeightPlacement; +const LocalRouting = @import("../api.zig").LocalRouting; + +pub fn activate(x: Tensor, activation: Activation) Tensor { + return switch (activation) { + .silu => x.silu(), + .gelu => x.gelu(), + .relu => x.relu(), + .swiglu, .swiglu_step, .geglu, .geglu_tanh => blk: { + const mid = @divExact(x.dim(.out), 2); + var gate = x.slice(.out, .{ .end = mid }); + var up = x.slice(.out, .{ .start = mid }); + + switch (activation) { + .swiglu_step => { + gate = gate.div(gate.negate().exp().addConstant(1)); + if (activation.swiglu_step.limit) |limit| { + gate = gate.minimum(Tensor.scalar(limit, x.dtype())); + up = up.clamp(Tensor.scalar(-limit, x.dtype()), Tensor.scalar(limit, x.dtype())); + } + break :blk gate.mul(up); + }, + .swiglu => { + if (activation.swiglu.limit) |limit| { + gate = gate.minimum(Tensor.scalar(limit, x.dtype())); + up = up.clamp(Tensor.scalar(-limit, x.dtype()), Tensor.scalar(limit, x.dtype())); + } + if (activation.swiglu.bias) |bias| up = up.addConstant(bias); + break :blk gate.div(gate.negate().exp().addConstant(1)).mul(up); + }, + .geglu => break :blk gelu(gate).mul(up), + .geglu_tanh => break :blk gate.gelu().mul(up), + else => unreachable, // already handled above + } + }, + }; +} + +/// More precise GELU approximation than the default Zig implementation, which is +/// based on tanh. This is the same approximation used in JAX. +pub fn gelu(x: Tensor) Tensor { + const xf = x.convert(.f32); + const cdf = xf.scale(-1.0 / @sqrt(@as(f64, 2))) + .erfc() + .scale(0.5); + return xf.mul(cdf).convert(x.dtype()); +} + +fn expandScales(scales: Tensor, shape: Shape) Tensor { + if (scales.count() == 1) return scales.asScalar().convert(.f32).broad(shape.withDtype(.f32)); + stdx.debug.assert(scales.rank() == shape.rank(), "reference scale rank must match weights", .{}); + var result = scales.convert(.f32); + for (0..shape.rank()) |axis| { + const factor = std.math.divCeil(i64, shape.dim(axis), result.dim(axis)) catch unreachable; + const expanded = result.shape().insert(axis + 1, .{factor}); + result = result.broadcast(expanded, Shape.range(expanded.rank(), .i64).remove(axis + 1).dims()) + .reshape(result.shape().setDim(axis, result.dim(axis) * factor)) + .slice(axis, .{ .end = shape.dim(axis) }); + } + return result.withTags(shape.tags()); +} + +pub fn dequantize(linear: zml.nn.Linear) Tensor { + var weight = linear.weight; + const q = linear.quantization orelse return weight.convert(.f32); + stdx.debug.assert(!q.swizzled_scales, "StableHLO reference requires plain block scales", .{}); + if (zml.nn.isPackedFp4(q.scheme, weight.dtype())) { + // StableHLO bitcast expands the low nibble first along the input axis. + weight = weight.bitCast(.f4e2m1).merge(.{ .unpacked = .{ linear.tag, .bitcast } }).renameTag(.unpacked, linear.tag); + } + const scales = if (q.scheme.isMx() and q.scales.dtype() == .u8) q.scales.bitCast(.f8e8m0) else q.scales; + var result = weight.convert(.f32).mul(expandScales(scales, weight.shape())); + if (q.global_scale) |scale| result = result.mul(scale.asMultiplier().broad(result.shape())); + return result; +} + +// E2M1 round-to-nearest, ties-to-even expressed independently of native +// float-to-FP4 lowering. Even encoding indices win midpoint ties. +fn roundFp4(x: Tensor) Tensor { + const magnitude = x.abs(); + var rounded = Tensor.scalar(6, .f32).broad(x.shape()); + const thresholds = [_]f32{ 5, 3.5, 2.5, 1.75, 1.25, 0.75, 0.25 }; + const values = [_]f32{ 4, 3, 2, 1.5, 1, 0.5, 0 }; + inline for (thresholds, values, 0..) |threshold, value, i| { + rounded = magnitude.cmp(if (i % 2 == 0) .LE else .LT, Tensor.scalar(threshold, .f32)) + .select(Tensor.scalar(value, .f32).broad(x.shape()), rounded); + } + return x.cmp(.LT, Tensor.scalar(0, .f32)).select(rounded.negate(), rounded); +} + +// Quantize and dequantize activation groups using tensor math, independently of +// Triton's quantizer. The return value is the represented value in FP32. +fn roundedInput(x: Tensor, linear: zml.nn.Linear, opts: Options, dtype: zml.DataType) Tensor { + if (opts.input_quantization == .none) return x.convert(dtype).convert(.f32); + const q = linear.quantization orelse return x.convert(dtype).convert(.f32); + const fixed_mxfp8 = opts.input_quantization == .mxfp8 or opts.input_quantization == .mxfp8_bf16; + const scheme = if (fixed_mxfp8) .fp8_block32 else q.scheme; + if (scheme == .nvfp4) { + const global = if (q.input_scale) |scale| scale.asMultiplier() else Tensor.scalar(1, .f32); + stdx.debug.assert(global.count() == 1, "reference NVFP4 input scale must be global", .{}); + // Keep both stored rounding boundaries when XLA fuses the quantizer. + const grouped = x.convert(.bf16).optimizationBarrier().convert(.f32).div(global.asScalar()) + .splitAxis(.in, .{ .group = @divExact(x.dim(.in), 16), .element = 16 }); + const scale = grouped.abs().max(.element).scale(1.0 / 6.0) + .minimum(Tensor.scalar(448, .f32)) + .convert(.f8e4m3fn).optimizationBarrier().convert(.f32).broad(grouped.shape()); + const divisor = scale.cmp(.NE, Tensor.scalar(0, .f32)).select(scale, Tensor.scalar(1, .f32).broad(scale.shape())); + return roundFp4(grouped.mul(Tensor.scalar(1, .f32).div(divisor))).mul(scale) + .reshape(x.shape().withDtype(.f32)).mul(global.asScalar()); + } + if (opts.input_quantization == .automatic and (!opts.quantize_input or scheme == .mxfp4)) return x.convert(dtype).convert(.f32); + const output_dtype = if (fixed_mxfp8) .f8e4m3fn else linear.weight.dtype(); + const group: i64 = switch (scheme) { + .mxfp8, .fp8_block32 => 32, + .fp8_block128 => 128, + .fp8_per_channel, .fp8_per_tensor => x.dim(.in), + else => unreachable, + }; + const source = if (opts.input_quantization == .mxfp8_bf16) x.convert(.bf16) else x; + const grouped = source.convert(.f32).splitAxis(.in, .{ .group = @divExact(x.dim(.in), group), .element = group }); + const epsilon: f32 = if (scheme == .fp8_block32) 1e-4 else 1e-10; + const max_value: f32 = if (output_dtype == .f8e4m3fnuz) 224 else 448; + var scale = grouped.abs().max(.element).maximum(Tensor.scalar(epsilon, .f32)).scale(1 / max_value); + if (scheme == .mxfp8 or scheme == .fp8_block32) scale = scale.log().scale(1 / @log(@as(f32, 2))).ceil().scale(@log(@as(f32, 2))).exp(); + const broad = scale.broad(grouped.shape()); + return grouped.div(broad).clamp(Tensor.scalar(-max_value, .f32), Tensor.scalar(max_value, .f32)) + .convert(output_dtype).optimizationBarrier().convert(.f32).mul(broad).reshape(x.shape().withDtype(.f32)); +} + +fn project(x: Tensor, linear: zml.nn.Linear, ids: Tensor, opts: Options, dtype: zml.DataType) Tensor { + const weight = dequantize(linear).withTags(.{ .expert, .out, .in }); + const selected = weight.gather(.{ .expert = ids }, .{}); + var result = roundedInput(x, linear, opts, dtype).dot(selected, .in); + if (linear.bias) |bias| { + const selected_bias = if (bias.rank() == 2) + bias.withTags(.{ .expert, .out }).gather(.{ .expert = ids }, .{}) + else + bias.withTags(.{.out}); + result = result.add(selected_bias.convert(.f32).broad(result.shape())); + } + return result; +} + +pub const Options = struct { + activation: Activation, + quantize_input: bool = false, + /// Model fixed backend contracts independently of optional input quantization: + /// Metal uses weight-only quantization; specialized MXFP4 GEMMs always use MXFP8 inputs. + /// CuTe additionally rounds the intermediate activation to BF16 before quantizing. + input_quantization: enum { automatic, none, mxfp8, mxfp8_bf16 } = .automatic, + routing_weight_placement: RoutingWeightPlacement = .after_down, +}; + +/// Negative expert IDs denote nonlocal routes and contribute zero, including bias. +pub fn reference(input: Tensor, topk_ids: Tensor, topk_weights: Tensor, gate_up: zml.nn.Linear, down: zml.nn.Linear, opts: Options) Tensor { + stdx.debug.assert(topk_ids.dim(.topk) > 0, "reference top-k must be positive", .{}); + stdx.debug.assert(topk_ids.dtype() == .i32, "reference expects i32 expert ids", .{}); + stdx.debug.assert(gate_up.weight.rank() == 3 and down.weight.rank() == 3, "reference expects rank-three expert weights", .{}); + stdx.debug.assert(gate_up.weight.dim(.expert) == down.weight.dim(.expert), "reference expert counts must match", .{}); + const tokens = input.dim(.t); + const k = topk_ids.dim(.topk); + // Nonlocal routes use expert zero for safe gathers; discard their full + // contribution after the down projection, including any projection bias. + const local_routes = topk_ids.cmp(.GE, Tensor.scalar(0, .i32)); + const safe_ids = topk_ids.maximum(Tensor.scalar(0, .i32)); + const ids = safe_ids.reshape(.{ .route = tokens * k }); + const weights = topk_weights.reshape(.{ .route = tokens * k }).convert(.f32); + const hidden = input.reshape(.{ .token = tokens, .in = input.dim(.d) }) + .insertAxes(.in, .{.topk}).broad(Shape.init(.{ .token = tokens, .topk = k, .in = input.dim(.d) }, input.dtype())) + .merge(.{ .route = .{ .token, .topk } }); + // Keep the projection's rounding boundary when XLA fuses activation/routing + // arithmetic; otherwise the reference can disagree with explicit BF16 GEMMs. + const projected = project(hidden, gate_up, ids, opts, input.dtype()).convert(input.dtype()).optimizationBarrier().convert(.f32); + var activated = activate(projected, opts.activation); + if (opts.routing_weight_placement == .before_down) activated = activated.mul(weights.broad(activated.shape())); + var output = project(activated.rename(.{ .out = .in }), down, ids, opts, input.dtype()); + if (opts.routing_weight_placement == .after_down) output = output.mul(weights.broad(output.shape())); + output = local_routes.reshape(.{ .route = tokens * k }).broad(output.shape()).select(output, Tensor.scalar(0, output.dtype())); + return output.convert(input.dtype()).convert(.f32) + .reshape(.{ .t = tokens, .topk = k, .d = output.dim(.out) }) + .sum(.topk).squeeze(.topk).convert(input.dtype()); +} + +pub fn fusedExperts(input: Tensor, routing: LocalRouting, gate_up: zml.nn.Linear, down: zml.nn.Linear, opts: GlobalOptions) Tensor { + stdx.debug.assert(routing.ids.dim(.topk) > 0 and routing.ids.dim(.topk) <= routing.global_num_experts, "MoE top-k must fit the global expert count", .{}); + return reference(input, routing.localIds(), routing.weights, gate_up, down, .{ + .activation = opts.activation, + .quantize_input = opts.quantize_input, + .routing_weight_placement = opts.routing_weight_placement, + }); +} + +test "MoE compliance: FP4 midpoint rounding" { + const Local = struct { + fn call(x: Tensor) Tensor { + return roundFp4(x); + } + }; + const values = [_]f32{ -7, -5, -3.5, -2.5, -1.75, -1.25, -0.75, -0.25, 0, 0.25, 0.75, 1.25, 1.75, 2.5, 3.5, 5, 7 }; + const expected = [_]f32{ -6, -4, -4, -2, -2, -1, -1, 0, 0, 0, 1, 1, 2, 2, 4, 4, 6 }; + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = zml.testing.env(); + const x: Tensor = .init(.{values.len}, .f32); + var exe = try platform.compileFn(allocator, io, Local.call, .{x}, .{}); + defer exe.deinit(); + var input = try zml.Buffer.fromBytes(io, platform, x.shape(), std.mem.asBytes(&values)); + defer input.deinit(); + var output = try exe.eval(allocator, io, .{input}); + defer output.deinit(); + const actual = try output.toSliceAlloc(allocator, io); + defer actual.free(allocator); + try std.testing.expectEqualSlices(f32, &expected, actual.constItems(f32)); +} diff --git a/zml/kernels/moe/tcl/BUILD.bazel b/zml/kernels/moe/tcl/BUILD.bazel new file mode 100644 index 00000000..44b56e08 --- /dev/null +++ b/zml/kernels/moe/tcl/BUILD.bazel @@ -0,0 +1,9 @@ +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "tcl", + srcs = [ + "moe.zig", + "tcl.zig", + ], +) diff --git a/zml/kernels/moe/tcl/moe.zig b/zml/kernels/moe/tcl/moe.zig new file mode 100644 index 00000000..23fb2274 --- /dev/null +++ b/zml/kernels/moe/tcl/moe.zig @@ -0,0 +1,457 @@ +//! TCL kernel schedules for Furiosa RNGD fused experts. +const std = @import("std"); + +const builder = @import("dsl/tcl/builder"); + +const zml = @import("../../../zml.zig"); +const tcl = zml.kernel.tcl; +const ActivationKind = @import("../api.zig").ActivationKind; + +pub const Format = enum { bf16, fp8_block128 }; + +/// Dispatch in `tcl.zig` uses the first matching schedule. +/// +/// - `.small_batch` for FP8 up to 64 tokens before padding. Run each selected +/// expert on the whole batch to reuse its weights, with zero routing weight +/// for unrouted pairs. +/// - `.per_route` when `num_tokens * top_k <= local_num_experts`. Gather +/// weights for each route to avoid the overhead of grouping routes. +/// - `.blockwise` otherwise. Pack routed tokens into blocks by expert to +/// reuse weights within each block, then gather results back by route. +pub const Schedule = enum { per_route, blockwise, small_batch }; + +pub const Cfg = struct { + num_tokens: i64, + top_k: i64, + local_num_experts: i64, + hidden_size: i64, + expert_intermediate_size: i64, + weight_scale_dtype: tcl.DType = .bf16, + activation: ActivationKind = .swiglu, +}; + +const base_inputs: []const [:0]const u8 = &.{ "x", "ids", "w", "gate_up", "down" }; +const scale_inputs: []const [:0]const u8 = &.{ "gate_up_scale", "down_scale" }; + +pub fn Kernel(comptime format: Format, comptime schedule: Schedule) type { + if (schedule == .small_batch and format != .fp8_block128) { + @compileError("TCL MoE FP8 small-batch schedule requires block-128 FP8"); + } + return tcl.Kernel(Cfg, .{ + .name = "moe_" ++ @tagName(schedule) ++ switch (format) { + .bf16 => "", + .fp8_block128 => "_fp8", + }, + .inputs = switch (format) { + .bf16 => base_inputs, + .fp8_block128 => base_inputs ++ scale_inputs, + }, + .outputs = &.{"y"}, + .run = struct { + fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { + configure(b, schedule); + const a: Axes = .init(b, cfg); + const in = try args(b, a, cfg, format); + switch (schedule) { + .per_route => try perRoute(b, a, in, cfg.activation, format), + .blockwise => try blockwise(b, a, in, cfg, format), + .small_batch => try fp8SmallBatch(b, a, in, cfg), + } + } + }.run, + }); +} + +const Axes = struct { + T: tcl.Axis, + K: tcl.Axis, + E: tcl.Axis, + D: tcl.Axis, + I: tcl.Axis, + I2: tcl.Axis, + S2: tcl.Axis, + S1: tcl.Axis, + + fn init(b: *tcl.Builder, cfg: Cfg) Axes { + return .{ + .T = b.axis("T", cfg.num_tokens), + .K = b.axis("K", cfg.top_k), + .E = b.axis("E", cfg.local_num_experts), + .D = b.axis("D", cfg.hidden_size), + .I = b.axis("I", cfg.expert_intermediate_size), + .I2 = b.axis("I2", 2 * cfg.expert_intermediate_size), + .S2 = b.axis("S2", 2), + .S1 = b.axis("S1", 1), + }; + } +}; + +fn args(b: *tcl.Builder, a: Axes, cfg: Cfg, comptime format: Format) tcl.FinishError!Args { + const weight_dtype: tcl.DType = switch (format) { + .bf16 => .bf16, + .fp8_block128 => .f8_e4, + }; + const spec = .{ + .x = .{ .dtype = .bf16, .axes = &.{ a.T, a.D } }, + .ids = .{ .dtype = .i32, .axes = &.{ a.T, a.K } }, + .w = .{ .dtype = .f32, .axes = &.{ a.T, a.K } }, + .gate_up = .{ .dtype = weight_dtype, .axes = &.{ a.E, a.I2, a.D } }, + .down = .{ .dtype = weight_dtype, .axes = &.{ a.E, a.D, a.I } }, + }; + if (format == .bf16) { + const t = try b.declareArgs(spec); + return .{ .x = t.x, .ids = t.ids, .w = t.w, .gate_up = t.gate_up, .down = t.down }; + } + const IG = b.axis("IG", @divExact(cfg.expert_intermediate_size, 128)); + const DG = b.axis("DG", @divExact(cfg.hidden_size, 128)); + const I2G = b.axis("I2G", 2 * @divExact(cfg.expert_intermediate_size, 128)); + const t = try b.declareArgs(.{ + .x = spec.x, + .ids = spec.ids, + .w = spec.w, + .gate_up = spec.gate_up, + .down = spec.down, + .gate_up_scale = .{ .dtype = cfg.weight_scale_dtype, .axes = &.{ a.E, I2G, DG } }, + .down_scale = .{ .dtype = cfg.weight_scale_dtype, .axes = &.{ a.E, DG, IG } }, + }); + return .{ .x = t.x, .ids = t.ids, .w = t.w, .gate_up = t.gate_up, .down = t.down, .gate_up_scale = t.gate_up_scale, .down_scale = t.down_scale }; +} + +const Args = struct { + x: builder.Tensor, + ids: builder.Tensor, + w: builder.Tensor, + gate_up: builder.Tensor, + down: builder.Tensor, + gate_up_scale: ?builder.Tensor = null, + down_scale: ?builder.Tensor = null, +}; + +const GatheredScales = struct { gate_up: ?builder.Tensor = null, down: ?builder.Tensor = null }; + +fn gatherScales(b: *tcl.Builder, a: Axes, in: Args, ids: builder.Tensor, batch: []const tcl.Axis, comptime format: Format) tcl.FinishError!GatheredScales { + if (format == .bf16) return .{}; + const IG = b.axis("IG", @divExact(a.I.size, 128)); + const DG = b.axis("DG", @divExact(a.D.size, 128)); + const gate_up = try b.reshape(in.gate_up_scale.?, &.{ a.E, a.S2, IG, DG }, .{}); + var gu_axes: [8]tcl.Axis = undefined; + var dn_axes: [8]tcl.Axis = undefined; + @memcpy(gu_axes[0..batch.len], batch); + @memcpy(dn_axes[0..batch.len], batch); + @memcpy(gu_axes[batch.len..][0..3], &[_]tcl.Axis{ a.S2, IG, DG }); + @memcpy(dn_axes[batch.len..][0..2], &[_]tcl.Axis{ DG, IG }); + return .{ + .gate_up = try b.gather(gate_up, ids, a.E, gu_axes[0 .. batch.len + 3], .{}), + .down = try b.gather(in.down_scale.?, ids, a.E, dn_axes[0 .. batch.len + 2], .{}), + }; +} + +fn blockedAxes(axes: []const tcl.Axis, axis: tcl.Axis, group: tcl.Axis, lane: tcl.Axis, storage: *[16]tcl.Axis) []const tcl.Axis { + var n: usize = 0; + for (axes) |a| { + if (std.mem.eql(u8, a.name, axis.name)) { + storage[n] = group; + n += 1; + storage[n] = lane; + } else storage[n] = a; + n += 1; + } + return storage[0..n]; +} + +/// Block scales multiply each K-block partial before K-block reduction. +/// Only the requested experts are gathered; their bank remains FP8 in HBM. +fn project(b: *tcl.Builder, x: builder.Tensor, weight: builder.Tensor, scale: ?builder.Tensor, contracting: tcl.Axis, output: tcl.Axis, to: []const tcl.Axis, dtype: ?tcl.DType, comptime format: Format) tcl.FinishError!builder.Tensor { + if (format == .bf16) { + const op = b.tensorOperation(.{}); + return op.commit(op.contract(x, weight, to), .{ .dtype = dtype }); + } + const CG = b.axis(if (std.mem.eql(u8, contracting.name, "D")) "DG" else "IG", @divExact(contracting.size, 128)); + const CL = b.axis(if (std.mem.eql(u8, contracting.name, "D")) "DB" else "IB", 128); + const OG = b.axis(if (std.mem.eql(u8, output.name, "D")) "DG" else "IG", @divExact(output.size, 128)); + const OL = b.axis(if (std.mem.eql(u8, output.name, "D")) "DB" else "IB", 128); + var x_axes: [16]tcl.Axis = undefined; + var w_axes: [16]tcl.Axis = undefined; + var w2_axes: [16]tcl.Axis = undefined; + var out_axes: [16]tcl.Axis = undefined; + var partial_axes: [16]tcl.Axis = undefined; + const xb = try b.reshape(x, blockedAxes(x.axes, contracting, CG, CL, &x_axes), .{}); + const wb = try b.reshape(weight, blockedAxes(blockedAxes(weight.axes, output, OG, OL, &w_axes), contracting, CG, CL, &w2_axes), .{}); + const out = blockedAxes(to, output, OG, OL, &out_axes); + @memcpy(partial_axes[0..out.len], out); + partial_axes[out.len] = CG; + + // Furiosa's published FP8 recipe uses an additive epsilon, including zero blocks. + var op = b.tensorOperation(.{}); + const qscale = try op.commit(op.fetch(xb, .{}).unary(.absf).reduce(&.{CL}, .maxf).divf(448.0).addf(1e-12), .{ .dtype = .f32 }); + op = b.tensorOperation(.{}); + // Native F32 reciprocal accuracy is qualified separately (within one ULP). + const reciprocal = op.ve(.divf, .{ 1.0, op.fetch(qscale, .{}) }); + const quantized = try op.commit(reciprocal.mulf(op.fetch(xb, .{})), .{ .dtype = .f8_e4 }); + op = b.tensorOperation(.{}); + const scaled = op.contract(quantized, wb, partial_axes[0 .. out.len + 1]).mulf(op.fetch(qscale, .{})).mulf(op.fetch(scale.?, .{})).reduce(&.{CG}, .addf); + return b.reshape(try op.commit(scaled, .{ .dtype = dtype, .axes = out }), to, .{}); +} + +/// `silu(gate) * up` or `gelu_tanh(gate) * up` of `h = [batch..., S2, I]` +/// (gate first), times `weights` (`[batch...]`) unless null. +fn gated(b: *tcl.Builder, a: Axes, h: builder.Tensor, batch: []const tcl.Axis, weights: ?builder.Tensor, activation: ActivationKind, comptime format: Format) tcl.FinishError!builder.Tensor { + var sliced: [4]tcl.Axis = undefined; + var flat: [3]tcl.Axis = undefined; + @memcpy(sliced[0..batch.len], batch); + @memcpy(flat[0..batch.len], batch); + sliced[batch.len] = a.S1; + sliced[batch.len + 1] = a.I; + flat[batch.len] = a.I; + const s = sliced[0 .. batch.len + 2]; + const f = flat[0 .. batch.len + 1]; + const gate = try b.reshape(try b.slice(h, a.S2, 0, s, .{}), f, .{}); + const up = try b.reshape(try b.slice(h, a.S2, 1, s, .{}), f, .{}); + // The builder can infer the TCL `Interleaving` tactic for two inputs with + // matching axes, such as `gate` and `up`. That tactic allows exactly one + // binary instruction, so force `.elementwise` for this activation chain. + var op = b.tensorOperation(.{ .tactic = .elementwise }); + const g = op.fetch(gate, .{}); + const activated = if (format == .fp8_block128) blk: { + if (activation != .swiglu) std.debug.panic("TCL MoE block-128 FP8 requires SwiGLU, got {s}", .{@tagName(activation)}); + + // WORKAROUND (TCC 2026.5.0b0): native fusion elides an internal + // BF16 SiLU write, even through as_dram/as_logical. Make its RNE + // rounding observable as integer bits before multiplying by up. + // Remove once the native reproducer preserves the BF16 boundary. + const bits = try op.commit(g.sigmoid().mulf(g).unary(.reinterpret_as_int), .{ .dtype = .i32 }); + op = b.tensorOperation(.{ .tactic = .elementwise }); + const v = op.fetch(bits, .{}); + const rounded = v.binary(.shr_logical, 16).binary(.bit_andi, 1).addi(32767).addi(v).binary(.bit_andi, -65536).unary(.reinterpret_as_float); + const silu = try op.commit(rounded, .{ .dtype = .bf16 }); + op = b.tensorOperation(.{ .tactic = .elementwise }); + break :blk op.fetch(silu, .{}); + } else switch (activation) { + .swiglu => g.sigmoid().mulf(g), + .geglu_tanh => geluTanh(g), + else => std.debug.panic("TCL MoE does not support activation {s}", .{@tagName(activation)}), + }; + var v = activated.mulf(op.fetch(up, .{})); + if (weights) |w| v = v.mulf(op.fetch(w, .{})); + return op.commit(v, .{ .dtype = .bf16 }); +} + +fn geluTanh(g: builder.Value) builder.Value { + const cube = g.mulf(g).mulf(g); + const inner = g.addf(cube.mulf(0.044715)).mulf(@sqrt(2.0 / std.math.pi)); + return inner.tanh().addf(1.0).mulf(g).mulf(0.5); +} + +/// Furiosa's compiler configuration for MoE kernels (`qwen3_moe/config.py`): +/// Use IO-bound tactics for per-route and FP8 small-batch schedules and +/// compute-bound tactics for blockwise. +/// Keep the compiler's default lowering mode and omit the adaptive-einsum and +/// half-VRF settings removed in TCC 2026.5. +fn configure(b: *tcl.Builder, comptime schedule: Schedule) void { + const config = comptime blk: { + const common = .{ + .enable_einsum_fusion = true, + .padding_policy = .{ .Small = 1.1 }, + .instruction_mem_budget = 0xB0000, + .reshape_einsum_mode = .{ .Reshape = .{ .permute = true } }, + .tensor_unit_bridge_threshold_in_page = 12, + .allow_external_operators = false, + .enable_tactic_pruning = false, + .scheduler_beam_search = true, + .allow_reduce_by_ve_cluster_chip_reduce = false, + .allow_reduce_by_ve_cluster_chip_reduce_base_population = false, + .propagate_sparse_axis_from_op = "None", + }; + const tactics = switch (schedule) { + .blockwise => .{ + .tactic_hint = .{ .ForLlmModelComputeBound = 200 }, + .dma_preference = 1.2, + }, + .per_route, .small_batch => .{ + .tactic_hint = .{ .ForLlmModelIOBound = 200 }, + .num_transaction_simulation_per_pe = 1024, + .dma_preference = 0.8, + }, + }; + // The transaction-simulation setting is only emitted for IO-bound schedules. + const common_info = @typeInfo(@TypeOf(common)).@"struct"; + const tactics_info = @typeInfo(@TypeOf(tactics)).@"struct"; + const Config = @Struct( + .auto, + null, + common_info.field_names ++ tactics_info.field_names, + common_info.field_types ++ tactics_info.field_types, + common_info.field_attrs ++ tactics_info.field_attrs, + ); + var merged: Config = undefined; + for (common_info.field_names) |field| @field(merged, field) = @field(common, field); + for (tactics_info.field_names) |field| @field(merged, field) = @field(tactics, field); + break :blk merged; + }; + b.compilerConfig(config); +} + +/// Gather the selected experts for every token. +fn perRoute(b: *tcl.Builder, a: Axes, in: Args, activation: ActivationKind, comptime format: Format) tcl.FinishError!void { + const gate_up = try b.reshape(in.gate_up, &.{ a.E, a.S2, a.I, a.D }, .{}); + const gu = try b.gather(gate_up, in.ids, a.E, &.{ a.T, a.K, a.S2, a.I, a.D }, .{}); + const dn = try b.gather(in.down, in.ids, a.E, &.{ a.T, a.K, a.D, a.I }, .{}); + + const scales = try gatherScales(b, a, in, in.ids, &.{ a.T, a.K }, format); + const h = try project(b, in.x, gu, scales.gate_up, a.D, a.I, &.{ a.T, a.K, a.S2, a.I }, if (format == .fp8_block128) .bf16 else null, format); + const act = try gated(b, a, h, &.{ a.T, a.K }, if (format == .fp8_block128) null else in.w, activation, format); + if (format == .fp8_block128) { + const y = try project(b, act, dn, scales.down, a.I, a.D, &.{ a.T, a.K, a.D }, .f32, format); + const op = b.tensorOperation(.{}); + b.ret(&.{try op.commit(op.fetch(y, .{}).mulf(op.fetch(in.w, .{})).reduce(&.{a.K}, .addf), .{ .dtype = .bf16 })}); + } else { + // The BF16 recipe weights activations and contracts experts together. + const op = b.tensorOperation(.{}); + b.ret(&.{try op.commit(op.contract(act, dn, &.{ a.T, a.D }), .{ .dtype = .bf16 })}); + } +} + +/// Rows per expert block: the next power of two of the routes per expert. +fn blockRows(cfg: Cfg) i64 { + const per_expert = std.math.divCeil(i64, cfg.num_tokens * cfg.top_k, cfg.local_num_experts) catch unreachable; + return @intCast(std.math.ceilPowerOfTwo(u64, @intCast(@max(per_expert, 4))) catch unreachable); +} + +const blocks_per_chunk = 32; + +/// Group routes into expert-major blocks, then run both projections in a +/// device loop over chunks of blocks. +fn blockwise(b: *tcl.Builder, a: Axes, in: Args, cfg: Cfg, comptime format: Format) tcl.FinishError!void { + const bb = blockRows(cfg); + // One block per expert at least, then enough for every route. + const max_blocks = std.math.divCeil(i64, @max(cfg.num_tokens * cfg.top_k - cfg.local_num_experts, 0), bb) catch unreachable; + const chunks = std.math.divCeil(i64, max_blocks + cfg.local_num_experts, blocks_per_chunk) catch unreachable; + const BB = b.axis("BB", bb); + const BGi = b.axis("BGi", blocks_per_chunk); + const Go = b.axis("Go", chunks); + const BG = b.axis("BG", chunks * blocks_per_chunk); + const GB = b.axis("GB", chunks * blocks_per_chunk * bb); + const T, const K, const E, const D, const I = .{ a.T, a.K, a.E, a.D, a.I }; + + // Routing: route (t, k) goes to row `slot` of block `start[e] + rank`. + const experts = try b.arange(cfg.local_num_experts, .i32, &.{E}, .{}); + var op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const dist = try op.commit(op.fetch(in.ids, .{}).subi(op.fetch(experts, .{})), .{}); + op = b.tensorOperation(.{}); + const onehot = try op.commit(op.where(dist, .eq, 0, 1, 0), .{}); + op = b.tensorOperation(.{}); + const rank = try op.commit(op.fetch(onehot, .{}).cumsum(&.{ T, K }), .{}); + op = b.tensorOperation(.{}); + const log2_bb: i64 = std.math.log2_int(u64, @intCast(bb)); + const blocks = try op.commit(op.fetch(onehot, .{}).reduce(&.{ T, K }, .addi).addi(bb - 1).binary(.shr_arith, log2_bb), .{}); + op = b.tensorOperation(.{}); + const block_end = try op.commit(op.fetch(blocks, .{}).cumsum(&.{E}), .{}); + op = b.tensorOperation(.{}); + const start_rows = try op.commit(op.fetch(block_end, .{}).subi(op.fetch(blocks, .{})).muli(bb), .{}); + op = b.tensorOperation(.{}); + const slot = try op.commit(op.fetch(rank, .{}).addi(op.fetch(start_rows, .{})).muli(op.fetch(onehot, .{})).reduce(&.{E}, .addi).subi(1), .{}); + // The expert of block g: how many experts end at or before g. + const block_ids = try b.arange(chunks * blocks_per_chunk, .i32, &.{BG}, .{}); + op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const ends = try op.commit(op.fetch(block_end, .{}).subi(op.fetch(block_ids, .{})), .{}); + op = b.tensorOperation(.{}); + const ended = try op.commit(op.where(ends, .le, 0, 1, 0), .{}); + op = b.tensorOperation(.{}); + // Padding blocks past the last expert reuse the last one. + const block_expert = try op.commit(op.fetch(ended, .{}).reduce(&.{E}, .addi).binary(.mini, cfg.local_num_experts - 1), .{}); + const used = try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(block_end), blocks_per_chunk - 1), blocks_per_chunk); + + // Rows of unused slots are never read back. + const rows = try b.reshape(try b.scatter(in.x, slot, GB, &.{ GB, D }, .{}), &.{ Go, BGi, BB, D }, .{}); + const chunk_experts = try b.reshape(block_expert, &.{ Go, BGi }, .{}); + const gate_up = try b.reshape(in.gate_up, &.{ E, a.S2, I, D }, .{}); + + var loop = b.openFor(used, .{try b.scratchpad(if (format == .fp8_block128) .f32 else .bf16, &.{ Go, BGi, BB, D })}); + { + const chunk = try b.indexRead(rows, loop.iv); + const ids = try b.indexRead(chunk_experts, loop.iv); + const gu = try b.gather(gate_up, ids, E, &.{ BGi, a.S2, I, D }, .{}); + const dn = try b.gather(in.down, ids, E, &.{ BGi, D, I }, .{}); + const scales = try gatherScales(b, a, in, ids, &.{BGi}, format); + const h = try project(b, chunk, gu, scales.gate_up, D, I, &.{ BGi, BB, a.S2, I }, if (format == .fp8_block128) .bf16 else null, format); + const act = try gated(b, a, h, &.{ BGi, BB }, null, cfg.activation, format); + const y = try project(b, act, dn, scales.down, I, D, &.{ BGi, BB, D }, if (format == .fp8_block128) .f32 else .bf16, format); + try loop.yield(.{try b.indexWrite(loop.carried[0], loop.iv, y)}); + } + const out = loop.results[0]; + + const per_route = try b.gather(try b.reshape(out, &.{ GB, D }, .{}), slot, GB, &.{ T, K, D }, .{}); + op = b.tensorOperation(.{}); + b.ret(&.{try op.commit(op.fetch(per_route, .{ .typecast_to = .f32 }).mulf(op.fetch(in.w, .{})).reduce(&.{K}, .addf), .{ .dtype = .bf16 })}); +} + +/// Number of experts processed per device-loop iteration, based on the +/// batch's route count (`cfg.num_tokens * cfg.top_k`). +fn expertsPerChunk(cfg: Cfg) i64 { + const routes: u64 = @intCast(std.math.divCeil(i64, cfg.num_tokens * cfg.top_k, 16) catch unreachable); + return @intCast(std.math.clamp(std.math.ceilPowerOfTwo(u64, routes) catch unreachable, 2, 16)); +} + +/// Process the FP8 experts selected by the batch in groups sized by `expertsPerChunk`. +/// Load each selected expert once and run it for every token. Routing weights +/// scale the F32 down-projection results before summing the experts and steps. +fn fp8SmallBatch(b: *tcl.Builder, a: Axes, in: Args, cfg: Cfg) tcl.FinishError!void { + const chunk = expertsPerChunk(cfg); + const steps = std.math.divCeil(i64, @min(cfg.local_num_experts, cfg.num_tokens * cfg.top_k), chunk) catch unreachable; + const C = b.axis("C", chunk); + const Go = b.axis("Go", steps); + const J = b.axis("J", steps * chunk); + const T, const K, const E, const D, const I = .{ a.T, a.K, a.E, a.D, a.I }; + + // Each token's routing weight per expert, and the used experts. + const experts = try b.arange(cfg.local_num_experts, .i32, &.{E}, .{}); + var op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const dist = try op.commit(op.fetch(in.ids, .{}).subi(op.fetch(experts, .{})), .{}); + op = b.tensorOperation(.{}); + const gate = try op.commit(op.where(op.fetch(dist, .{}), .eq, 0, op.fetch(in.w, .{ .broadcast_to = &.{ T, K, E } }), 0.0).reduce(&.{K}, .addf), .{}); + op = b.tensorOperation(.{}); + const onehot = try op.commit(op.where(dist, .eq, 0, 1, 0), .{}); + op = b.tensorOperation(.{}); + const counts = try op.commit(op.fetch(onehot, .{}).reduce(&.{ T, K }, .addi), .{}); + op = b.tensorOperation(.{}); + const used = try op.commit(op.where(counts, .gt, 0, 1, 0), .{}); + op = b.tensorOperation(.{}); + const position = try op.commit(op.fetch(used, .{}).cumsum(&.{E}), .{}); + // A used expert's rank among them, -1 for the others. + op = b.tensorOperation(.{}); + const rank = try op.commit(op.fetch(position, .{}).muli(op.fetch(used, .{})).subi(1), .{}); + const trips = try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(position), chunk - 1), chunk); + + // Slot j holds the used expert of rank j and its tokens' weights; + // slots past the last hold expert 0 with no weight. + const ranks = try b.arange(steps * chunk, .i32, &.{J}, .{}); + op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const offset = try op.commit(op.fetch(rank, .{}).subi(op.fetch(ranks, .{})), .{}); + op = b.tensorOperation(.{}); + const slot_experts = try op.commit(op.where(op.fetch(offset, .{}), .eq, 0, op.fetch(experts, .{ .broadcast_to = &.{ E, J } }), 0).reduce(&.{E}, .addi), .{ .dtype = .i32 }); + // A reduction's condition is read plainly, so broadcast it first. + op = b.tensorOperation(.{}); + const token_offset = try op.commit(op.fetch(offset, .{ .broadcast_to = &.{ T, E, J } }), .{}); + op = b.tensorOperation(.{}); + const slot_gate = try op.commit(op.where(op.fetch(token_offset, .{}), .eq, 0, op.fetch(gate, .{ .broadcast_to = &.{ T, E, J } }), 0.0).reduce(&.{E}, .addf), .{ .dtype = .f32, .axes = &.{ J, T } }); + const step_experts = try b.reshape(slot_experts, &.{ Go, C }, .{}); + const step_gate = try b.reshape(slot_gate, &.{ Go, C, T }, .{}); + const gate_up = try b.reshape(in.gate_up, &.{ E, a.S2, I, D }, .{}); + + var loop = b.openFor(trips, .{try b.full(0, .f32, &.{ Go, T, D })}); + { + const ids = try b.indexRead(step_experts, loop.iv); + const weights = try b.indexRead(step_gate, loop.iv); + const gu = try b.gather(gate_up, ids, E, &.{ C, a.S2, I, D }, .{}); + const dn = try b.gather(in.down, ids, E, &.{ C, D, I }, .{}); + const scales = try gatherScales(b, a, in, ids, &.{C}, .fp8_block128); + const h = try project(b, in.x, gu, scales.gate_up, D, I, &.{ C, T, a.S2, I }, .bf16, .fp8_block128); + const act = try gated(b, a, h, &.{ C, T }, null, cfg.activation, .fp8_block128); + const per_expert = try project(b, act, dn, scales.down, I, D, &.{ C, T, D }, .f32, .fp8_block128); + op = b.tensorOperation(.{}); + const y = try op.commit(op.fetch(per_expert, .{}).mulf(op.fetch(weights, .{})).reduce(&.{C}, .addf), .{ .dtype = .f32 }); + try loop.yield(.{try b.indexWrite(loop.carried[0], loop.iv, y)}); + } + op = b.tensorOperation(.{}); + b.ret(&.{try op.commit(op.fetch(loop.results[0], .{}).reduce(&.{Go}, .addf), .{ .dtype = .bf16 })}); +} diff --git a/zml/kernels/moe/tcl/tcl.zig b/zml/kernels/moe/tcl/tcl.zig new file mode 100644 index 00000000..f05727ce --- /dev/null +++ b/zml/kernels/moe/tcl/tcl.zig @@ -0,0 +1,250 @@ +//! Furiosa RNGD fused experts in TCL, after Furiosa's production kernels. +const std = @import("std"); + +const stdx = @import("stdx"); + +const zml = @import("../../../zml.zig"); +const Tensor = zml.Tensor; +const tcl = zml.kernel.tcl; +const kernels = @import("moe.zig"); +const api = @import("../api.zig"); + +/// `input` is `{.t, .d}`, routing IDs and weights are `{.t, .topk}`, `gate_up` +/// is `{.expert, .dout = 2 * inter (gate then up), .d}`, and `down` is +/// `{.expert, .d, .dout = inter}`. Experts are BF16 or block-128 FP8 with +/// BF16/F32 scales. Supports plain SwiGLU and BF16 GEGLU with tanh GELU. +pub fn fusedExperts(input: Tensor, routing: api.LocalRouting, gate_up_linear: zml.nn.Linear, down_linear: zml.nn.Linear, opts: api.Options) Tensor { + const topk_ids = routing.localIds(); + stdx.debug.assert(!opts.quantize_input and opts.routing_weight_placement == .after_down, "TCL MoE requires quantize_input=false and routing_weight_placement=after_down", .{}); + stdx.debug.assert(gate_up_linear.bias == null and down_linear.bias == null, "TCL MoE takes experts without bias", .{}); + stdx.debug.assert(input.dtype() == .bf16 and topk_ids.dtype() == .i32, "TCL MoE takes BF16 activations and i32 ids", .{}); + const format = expertFormat(gate_up_linear, down_linear); + const swiglu = opts.activation == .swiglu and std.meta.eql(opts.activation.swiglu, .{}); + stdx.debug.assert(swiglu or (format == .bf16 and opts.activation == .geglu_tanh), "TCL MoE supports plain SwiGLU or GEGLU-tanh with BF16 experts, got {s}", .{@tagName(opts.activation)}); + + const gate_up = gate_up_linear.weight; + const down = down_linear.weight; + const cfg: kernels.Cfg = .{ + .num_tokens = input.dim(.t), + .top_k = topk_ids.dim(.topk), + .local_num_experts = gate_up.dim(.expert), + .hidden_size = input.dim(.d), + .expert_intermediate_size = down.dim(.dout), + .weight_scale_dtype = switch (format) { + .bf16 => .bf16, + .fp8_block128 => |s| tcl.from(s.gate_up.dtype()), + }, + .activation = std.meta.activeTag(opts.activation), + }; + stdx.debug.assert(gate_up.dim(.dout) == 2 * cfg.expert_intermediate_size and down.dim(.d) == cfg.hidden_size and gate_up.dim(.d) == cfg.hidden_size, "TCL MoE: mismatched expert shapes", .{}); + + const weights = routing.weights.convert(.f32); + const local_weights = switch (format) { + .bf16 => weights, + .fp8_block128 => topk_ids.cmp(.GE, .scalar(0, .i32)).select(weights, .scalar(0, .f32)), + }; + return switch (format) { + .bf16 => selectScheduleAndRun(.bf16, cfg, .{ .x = input, .ids = topk_ids, .w = local_weights, .gate_up = gate_up, .down = down }), + .fp8_block128 => |s| selectScheduleAndRun(.fp8_block128, cfg, .{ + .x = input, + .ids = topk_ids, + .w = local_weights, + .gate_up = gate_up, + .down = down, + .gate_up_scale = s.gate_up, + .down_scale = s.down, + }), + }; +} + +const Scales = struct { gate_up: Tensor, down: Tensor }; + +const Format = union(kernels.Format) { + bf16: void, + fp8_block128: Scales, +}; + +fn expertFormat(gate_up: zml.nn.Linear, down: zml.nn.Linear) Format { + const scheme = gate_up.quantizationScheme(); + stdx.debug.assert(scheme == down.quantizationScheme(), "TCL MoE projections must have matching quantization schemes", .{}); + return switch (scheme orelse { + stdx.debug.assert(gate_up.weight.dtype() == .bf16 and down.weight.dtype() == .bf16, "TCL MoE unquantized expert weights must be BF16", .{}); + return .bf16; + }) { + .fp8_block128 => blk: { + const scales: Scales = .{ .gate_up = fp8Scales(gate_up), .down = fp8Scales(down) }; + stdx.debug.assert(scales.gate_up.dtype() == scales.down.dtype(), "TCL MoE projection scales must have matching types", .{}); + break :blk .{ .fp8_block128 = scales }; + }, + else => |unsupported| stdx.debug.panic("TCL MoE does not support quantization scheme {s}", .{@tagName(unsupported)}), + }; +} + +fn fp8Scales(linear: zml.nn.Linear) Tensor { + const q = linear.quantization.?; + stdx.debug.assert(linear.weight.dtype() == .f8e4m3fn and !q.swizzled_scales and q.global_scale == null and q.input_scale == null, "TCL MoE takes plain E4M3FN block-128 weights", .{}); + stdx.debug.assert(q.scales.dtype() == .bf16 or q.scales.dtype() == .f32, "TCL MoE FP8 scales must be BF16 or F32", .{}); + const w = linear.weight; + const scale = q.scales; + stdx.debug.assert(@mod(w.dim(.d), 128) == 0 and @mod(w.dim(.dout), 128) == 0 and scale.dim(.expert) == w.dim(.expert) and scale.dim(.d) * 128 == w.dim(.d) and scale.dim(.dout) * 128 == w.dim(.dout), "TCL MoE FP8 scales must cover 128x128 weight blocks", .{}); + return scale; +} + +// Maximum total input tokens per MoE call before padding for the FP8 +// small-batch schedule. It loads each used expert's weights once and reuses +// them across the batch. +// Larger batches fall back to per-route or blockwise dispatch. +const fp8_small_batch_max_tokens = 64; + +fn selectScheduleAndRun(comptime format: kernels.Format, cfg: kernels.Cfg, inputs: kernels.Kernel(format, .per_route).Inputs) Tensor { + if (format == .fp8_block128 and cfg.num_tokens <= fp8_small_batch_max_tokens) { + return run(format, .small_batch, cfg, inputs); + } + if (cfg.num_tokens * cfg.top_k <= cfg.local_num_experts) { + return run(format, .per_route, cfg, inputs); + } + return run(format, .blockwise, cfg, inputs); +} + +fn run(comptime format: kernels.Format, comptime schedule: kernels.Schedule, cfg: kernels.Cfg, inputs: kernels.Kernel(format, .per_route).Inputs) Tensor { + const call_cfg, const prepared = if (schedule == .small_batch) + prepareFp8SmallBatch(cfg, inputs) + else + .{ cfg, prepareRoutedInputs(cfg, inputs) }; + + const shape = prepared.x.shape(); + var y = kernels.Kernel(format, schedule).call(prepared, .{ .y = shape }, .{ .cfg = call_cfg }).y; + + if (schedule == .small_batch) y = y.slice(0, .{ .end = cfg.num_tokens }); + return y.reshape(inputs.x.shape()); +} + +fn prepareRoutedInputs(cfg: kernels.Cfg, inputs: anytype) @TypeOf(inputs) { + var prepared = inputs; + prepared.x = inputs.x.reshape(.{ cfg.num_tokens, cfg.hidden_size }); + const ids = inputs.ids.reshape(.{ cfg.num_tokens, cfg.top_k }); + const w = inputs.w.reshape(.{ cfg.num_tokens, cfg.top_k }); + // These kernels gather every route, and blockwise also scatters each route + // into an expert slot. Give nonlocal routes a valid slot and zero weight. + const valid = ids.cmp(.GE, Tensor.scalar(0, .i32)).logical(.AND, ids.cmp(.LT, Tensor.scalar(cfg.local_num_experts, .i32))); + prepared.ids = valid.select(ids, Tensor.scalar(0, .i32)); + prepared.w = valid.select(w, Tensor.scalar(0, .f32)); + return prepared; +} + +fn prepareFp8SmallBatch(cfg: kernels.Cfg, inputs: kernels.Kernel(.fp8_block128, .small_batch).Inputs) struct { kernels.Cfg, @TypeOf(inputs) } { + // WORKAROUND (TCC 2026.5.0b0): a token count other than a power of two + // panics its adder tree ("Fail to cover valid range", 5 tokens padded + // to 8): pad with unrouted tokens. + var padded = cfg; + padded.num_tokens = @intCast(std.math.ceilPowerOfTwo(u64, @intCast(cfg.num_tokens)) catch unreachable); + const pad = padded.num_tokens - cfg.num_tokens; + var prepared = inputs; + prepared.x = inputs.x.pad(0, .{ .t = Tensor.Pad{ .high = pad } }).reshape(.{ padded.num_tokens, cfg.hidden_size }); + prepared.ids = inputs.ids.pad(-1, .{ .t = Tensor.Pad{ .high = pad } }).reshape(.{ padded.num_tokens, cfg.top_k }); + prepared.w = inputs.w.pad(0, .{ .t = Tensor.Pad{ .high = pad } }).reshape(.{ padded.num_tokens, cfg.top_k }); + return .{ padded, prepared }; +} + +test "tcl fused experts run on furiosa" { + try testFusedExperts(.{ .swiglu = .{} }); + try testFusedExperts(.geglu_tanh); +} + +fn testFusedExperts(activation: zml.moe.Activation) !void { + const platform = zml.testing.env(); + if (platform.target != .furiosa) return error.SkipZigTest; + const allocator = std.testing.allocator; + const io = std.testing.io; + const bf16 = zml.floats.BFloat16; + + // With two routes per token and eight experts, four tokens use per-route + // dispatch and sixteen use blockwise dispatch. + for ([_]i64{ 4, 16 }) |t| { + const k = 2; + const e = 8; + const d = 256; + const i = 64; + const x: Tensor = .init(.{ .t = t, .d = d }, .bf16); + const ids: Tensor = .init(.{ .t = t, .topk = k }, .i32); + const w: Tensor = .init(.{ .t = t, .topk = k }, .f32); + const gu: Tensor = .init(.{ .expert = e, .dout = 2 * i, .d = d }, .bf16); + const dn: Tensor = .init(.{ .expert = e, .d = d, .dout = i }, .bf16); + const Mod = struct { + pub fn forward(x_: Tensor, ids_: Tensor, w_: Tensor, gu_: Tensor, dn_: Tensor, act: zml.moe.Activation) Tensor { + return zml.moe.forwardMoe(x_, ids_, w_, .init(gu_, null, .dout), .init(dn_, null, .dout), .tcl, zml.Compiler.current().platform.replicated_mesh, .expert(.replicated), .{ + .activation = act, + .quantize_input = false, + .routing_weight_placement = .after_down, + }); + } + }; + var exe = try platform.compileFn(allocator, io, Mod.forward, .{ x, ids, w, gu, dn, activation }, .{}); + defer exe.deinit(); + + var prng: std.Random.DefaultPrng = .init(@intCast(t)); + const r = prng.random(); + const hx = try allocator.alloc(bf16, @intCast(t * d)); + defer allocator.free(hx); + const hgu = try allocator.alloc(bf16, e * 2 * i * d); + defer allocator.free(hgu); + const hdn = try allocator.alloc(bf16, e * d * i); + defer allocator.free(hdn); + for (hx) |*v| v.* = .fromF32(r.floatNorm(f32) * 0.5); + for (hgu) |*v| v.* = .fromF32(r.floatNorm(f32) / 16); + for (hdn) |*v| v.* = .fromF32(r.floatNorm(f32) / 8); + const hids = try allocator.alloc(i32, @intCast(t * k)); + defer allocator.free(hids); + const hw = try allocator.alloc(f32, @intCast(t * k)); + defer allocator.free(hw); + for (0..@intCast(t)) |tok| for (0..k) |j| { + hids[tok * k + j] = @intCast((tok * 3 + j * 5) % e); + hw[tok * k + j] = if (j == 0) 0.7 else 0.3; + }; + + var bx: zml.Buffer = try .fromBytes(io, platform, x.shape(), std.mem.sliceAsBytes(hx)); + defer bx.deinit(); + var bids: zml.Buffer = try .fromBytes(io, platform, ids.shape(), std.mem.sliceAsBytes(hids)); + defer bids.deinit(); + var bw: zml.Buffer = try .fromBytes(io, platform, w.shape(), std.mem.sliceAsBytes(hw)); + defer bw.deinit(); + var bgu: zml.Buffer = try .fromBytes(io, platform, gu.shape(), std.mem.sliceAsBytes(hgu)); + defer bgu.deinit(); + var bdn: zml.Buffer = try .fromBytes(io, platform, dn.shape(), std.mem.sliceAsBytes(hdn)); + defer bdn.deinit(); + var result = try exe.eval(allocator, io, .{ bx, bids, bw, bgu, bdn }); + defer result.deinit(); + var host = try result.toSliceAlloc(allocator, io); + defer host.free(allocator); + const out = host.items(bf16); + + for (0..@intCast(t)) |tok| { + var expected: [d]f32 = @splat(0); + for (0..k) |j| { + const ex: usize = @intCast(hids[tok * k + j]); + var act: [i]f32 = undefined; + for (0..i) |n| { + var g: f32 = 0; + var u: f32 = 0; + for (0..d) |m| { + const xv = hx[tok * d + m].toF32(); + g += xv * hgu[(ex * 2 * i + n) * d + m].toF32(); + u += xv * hgu[(ex * 2 * i + i + n) * d + m].toF32(); + } + const gate = switch (activation) { + .swiglu => g / (1 + @exp(-g)), + .geglu_tanh => 0.5 * g * (1 + std.math.tanh(@sqrt(2.0 / std.math.pi) * (g + 0.044715 * g * g * g))), + else => unreachable, + }; + act[n] = gate * u; + } + for (0..d) |m| { + var y: f32 = 0; + for (0..i) |n| y += act[n] * hdn[(ex * d + m) * i + n].toF32(); + expected[m] += hw[tok * k + j] * y; + } + } + for (expected, out[tok * d ..][0..d]) |want, got| try std.testing.expectApproxEqAbs(want, got.toF32(), 2e-2 + 2e-2 * @abs(want)); + } + } +} diff --git a/zml/kernels/moe/tests.zig b/zml/kernels/moe/tests.zig new file mode 100644 index 00000000..c74406c3 --- /dev/null +++ b/zml/kernels/moe/tests.zig @@ -0,0 +1,752 @@ +//! Shared public-entry-point tests. Hardware absence is the only runtime skip: +//! supported cases are explicit, and errors in those cases must fail the test. +const std = @import("std"); + +const zml = @import("../../zml.zig"); +const Tensor = zml.Tensor; +const Shape = zml.Shape; +const Activation = @import("api.zig").Activation; +const ExpertsLayout = @import("api.zig").ExpertsLayout; +const GlobalOptions = @import("api.zig").Options; +const Placement = @import("api.zig").RoutingWeightPlacement; +const reference = @import("stablehlo/stablehlo.zig"); + +const Case = struct { + backend: zml.moe.Backend, + activation: Activation = .{ .swiglu = .{} }, + scheme: ?zml.Quantization.Scheme = null, + activation_dtype: zml.DataType = .bf16, + quantize_input: bool = false, + placement: Placement = .after_down, + bias: bool = false, + non_unit_scales: bool = false, + tokens: i64 = 1, + topk: i64 = 1, + width: i64 = 128, + intermediate: i64 = 128, + experts: i64 = 8, + expert_mesh: *const zml.Mesh, + + fn inputShape(c: Case) Shape { + const shape = Shape.init(.{ .t = c.tokens, .d = c.width }, c.activation_dtype); + const spec: zml.Sharding.Partitioning.Logical = if (c.expert_mesh.resolveLogicalAxis("data") != null) .parse(.{ .t = .data }) else .replicated; + return shape.withPartitioning(c.expert_mesh, spec); + } + + pub fn compile(c: Case, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform) !zml.Exe { + std.debug.assert(c.backend.isAvailable(platform)); + const x: Tensor = .fromShape(c.inputShape()); + return platform.compileFn(allocator, io, forward, .{ x, c }, .{ .meshes = &.{c.expert_mesh} }); + } + + pub fn check(c: Case, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, exe: *const zml.Exe) !void { + const x: Tensor = .fromShape(c.inputShape()); + + var host = try zml.Slice.alloc(allocator, x.shape()); + defer host.free(allocator); + switch (c.activation_dtype) { + inline .bf16, .f16, .f32 => |dtype| { + for (host.items(dtype.toZigType()), 0..) |*value, i| { + const f = @as(f32, @floatFromInt(@as(i32, @intCast((i * 13 + i / 128) % 23)) - 11)) / 32; + value.* = if (dtype == .bf16) .fromF32(f) else @floatCast(f); + } + }, + else => unreachable, + } + + var input = try zml.Buffer.fromSlice(io, platform, host); + defer input.deinit(); + + var runner = try exe.runner(allocator); + defer runner.deinit(allocator); + var output: zml.Bufferized(Outputs) = undefined; + runner.run(io, .{input}, .{&output}, .{ .wait = true }); + defer zml.Buffer.deinitAll(Outputs, &output); + try std.testing.expectEqualDeep(c.inputShape()._sharding, output.actual.shape()._sharding); + + // BF16 intermediate rounding and different GEMM reduction orders. Every + // element must pass; near-zero values use the absolute bound. + try zml.testing.expectClose(io, output.expected, output.actual, .{ + // Share the NVFP4 comparison budget across CUDA backends. + .absolute_tolerance = if (c.scheme == .nvfp4) 0.078125 else 0.015625, + .relative_tolerance = 0.02, + .minimum_close_fraction = 1, + }); + } +}; + +/// Generate a reproducible Tensor with a simple pattern. +/// +/// Generate as many elements as the shape requires, with this formula: +/// value[i] = (((i * multiplier + 3) % 17) - 8) / divisor +/// +/// Then reshape to the requested shape and convert to f32. +fn generateDeterministicTensor(shape: Shape, multiplier: i64, divisor: f64) Tensor { + return Tensor.arange(.{ .end = @intCast(shape.count()) }, .i32) + .mul(Tensor.scalar(multiplier, .i32)).addConstant(3) + .remainder(Tensor.scalar(17, .i32)).addConstant(-8) + .convert(.f32).scale(1 / divisor).reshape(shape.withDtype(.f32)); +} + +/// Select exactly representable FP4 values with distinct even/odd sequences. +fn generateFp4Tensor(shape: Shape, multiplier: i64) Tensor { + const values = Tensor.constantTensor(.init(.{ .value = 16 }, .f32), std.mem.sliceAsBytes(&[_]f32{ + 0, 0.5, 1, 1.5, 2, 3, 4, 6, -0.0, -0.5, -1, -1.5, -2, -3, -4, -6, + })); + const index = Tensor.arange(.{ .end = @intCast(shape.count()) }, .i32); + const pair = index.div(Tensor.scalar(2, .i32)); + // Prime periods distinguish experts, output rows, and gate/up halves. + const even = pair.mul(Tensor.scalar(multiplier, .i32)).addConstant(3) + .remainder(Tensor.scalar(17, .i32)).remainder(Tensor.scalar(16, .i32)); + const odd = pair.mul(Tensor.scalar(3, .i32)).addConstant(7) + .remainder(Tensor.scalar(19, .i32)).remainder(Tensor.scalar(16, .i32)); + const indices = index.remainder(Tensor.scalar(2, .i32)).cmp(.EQ, Tensor.scalar(0, .i32)).select(even, odd); + return values.gather(.{ .value = indices }, .{}).convert(.f4e2m1).reshape(shape.withDtype(.f4e2m1)); +} + +/// Create a Linear fixture with a reproducible pattern, optionally quantized and biased. +fn linear(shape: Shape, scheme: ?zml.Quantization.Scheme, bias: bool, multiplier: i64) zml.nn.Linear { + const fixture_bias = if (bias) generateDeterministicTensor(shape.remove(2), 3, 32).convert(shape.dtype()) else null; + + const s = scheme orelse return .{ + .weight = generateDeterministicTensor(shape, multiplier, 16).convert(shape.dtype()), + .tag = shape.tag(2), + .bias = fixture_bias, + }; + + const weight = switch (s) { + .mxfp4, .nvfp4 => generateFp4Tensor(shape, multiplier), + else => generateDeterministicTensor(shape, multiplier, 16).convert(.f8e4m3fn), + }; + + const scale_shape: Shape = switch (s) { + .mxfp4, .mxfp8 => .init(.{ shape.dim(0), shape.dim(1), @divExact(shape.dim(2), 32) }, .f8e8m0), + .nvfp4 => .init(.{ shape.dim(0), shape.dim(1), @divExact(shape.dim(2), 16) }, .f8e4m3fn), + .fp8_per_tensor => .init(.{ shape.dim(0), 1, 1 }, .f32), + .fp8_per_channel => .init(.{ shape.dim(0), shape.dim(1), 1 }, .f32), + .fp8_block128 => .init(.{ shape.dim(0), @divExact(shape.dim(1), 128), @divExact(shape.dim(2), 128) }, .f32), + .fp8_block32 => .init(.{ shape.dim(0), @divExact(shape.dim(1), 32), @divExact(shape.dim(2), 32) }, .f32), + }; + + const scales = Tensor.arange(.{ .end = @intCast(scale_shape.count()) }, .i32) + .remainder(Tensor.scalar(2, .i32)).convert(.f32).addConstant(1).scale(1.0 / 16.0) + .reshape(scale_shape.withDtype(.f32)).convert(scale_shape.dtype()); + + const input_scale: ?zml.Quantization.GlobalScale = if (s == .nvfp4) .{ .value = Tensor.scalar(1, .f32), .operation = .multiply } else null; + const global_scale: ?zml.Quantization.GlobalScale = if (s == .nvfp4) .{ .value = Tensor.scalar(1, .f32).broad(.init(.{ .expert = shape.dim(0) }, .f32)), .operation = .multiply } else null; + + return .{ + .weight = weight, + .tag = shape.tag(2), + .bias = fixture_bias, + .quantization = .{ + .scheme = s, + .scales = scales, + .input_scale = input_scale, + .global_scale = global_scale, + }, + }; +} + +const ExpertFixture = struct { + gate_up: zml.nn.Linear, + down: zml.nn.Linear, +}; + +/// Canonical weights use logical dimensions, concatenated gate/up, and plain scales. +fn makeExpertFixture(c: Case) ExpertFixture { + const projection_width: i64 = switch (c.activation) { + .silu, .gelu, .relu => c.intermediate, + else => 2 * c.intermediate, + }; + var fixture: ExpertFixture = .{ + .gate_up = linear(.init(.{ .expert = c.experts, .dout = projection_width, .d = c.width }, c.activation_dtype), c.scheme, c.bias, 7), + .down = linear(.init(.{ .expert = c.experts, .d = c.width, .dout = c.intermediate }, c.activation_dtype), c.scheme, c.bias, 11), + }; + if (c.non_unit_scales) { + for ([_]*zml.nn.Linear{ &fixture.gate_up, &fixture.down }, 0..) |projection, i| { + const q = &projection.quantization.?; + std.debug.assert(q.scheme == .nvfp4); + // Exercise both scale polarity and distinct per-expert multipliers. + q.input_scale = .{ .value = Tensor.scalar(@as(f32, if (i == 0) 2 else 0.5), .f32), .operation = .divide }; + q.global_scale = .{ + .value = Tensor.arange(.{ .end = @intCast(c.experts) }, .i32) + .remainder(Tensor.scalar(2, .i32)).addConstant(1).convert(.f32).withTags(.{.expert}), + .operation = .divide, + }; + } + } + return fixture; +} + +fn interleave(tensor: Tensor, rows: i64) Tensor { + return tensor.reshape(.{ .expert = tensor.dim(0), .projection = 2, .row_block = @divExact(tensor.dim(1), 2 * rows), .row = rows, .column = tensor.dim(2) }) + .transpose(.{ .expert, .row_block, .projection, .row, .column }).reshape(tensor.shape()); +} + +/// Blackwell block scales: [E, N/128, 4, 32, K/4, 4] -> +/// [E, N/128, K/4, 32, 4, 4], retaining the declared [E, N, K] shape. +fn swizzleScales(scales: Tensor) Tensor { + return scales.reshape(.{ .expert = scales.dim(0), .row_block = @divExact(scales.dim(1), 128), .row_quad = 4, .row_lane = 32, .scale_word = @divExact(scales.dim(2), 4), .scale_byte = 4 }) + .transpose(.{ .expert, .row_block, .scale_word, .row_lane, .row_quad, .scale_byte }).reshape(scales.shape()); +} + +fn packFp4Bytes(weight: Tensor) Tensor { + std.debug.assert(weight.dtype() == .f4e2m1); + return weight.reshape(weight.shape().setDim(2, @divExact(weight.dim(2), 2)).append(.{ .nibble = 2 })).bitCast(.u8); +} + +fn prepareGateUp(tensor: Tensor, layout: ExpertsLayout) Tensor { + return switch (layout.gate_up) { + .concatenated => tensor, + .row_interleaved_1 => interleave(tensor, 1), + .row_interleaved_8 => interleave(tensor, 8), + }; +} + +/// Adapt a copy of both projections to the backend's storage and layout contract. +fn prepareForBackend(canonical: ExpertFixture, backend: zml.moe.Backend) ExpertFixture { + var prepared = canonical; + const scheme = canonical.gate_up.quantizationScheme(); + const layout = backend.expertsLayout(scheme) catch |err| zml.stdx.debug.panic("fixture layout: {}", .{err}); + // Plain activations have one projection, so gate/up ordering does not apply. + const gated = canonical.gate_up.weight.dim(1) == 2 * canonical.down.weight.dim(2); + if (gated) { + prepared.gate_up.weight = prepareGateUp(canonical.gate_up.weight, layout); + if (canonical.gate_up.bias) |bias| prepared.gate_up.bias = prepareGateUp(bias.appendAxes(.{.dummy}), layout).squeeze(.dummy); + if (prepared.gate_up.quantization) |*q| q.scales = prepareGateUp(q.scales, layout); + } + + for ([_]*zml.nn.Linear{ &prepared.gate_up, &prepared.down }) |projection| { + switch (layout.packing) { + .swizzled_scales, .flashinfer_nvfp4 => { + const q = &projection.quantization.?; + q.scales = swizzleScales(q.scales); + q.swizzled_scales = true; + }, + .mxfp4_interleaved8 => { + const q = &projection.quantization.?; + const sf = q.scales; + q.scales = sf.bitCast(.u8) + .reshape(.{ .expert = sf.dim(0), .row_block = @divExact(sf.dim(1), 128), .row_quad = 4, .row_lane = 32, .scale_word = @divExact(sf.dim(2), 4), .scale_byte = 4 }) + .transpose(.{ .expert, .scale_word, .row_block, .row_lane, .row_quad, .scale_byte }) + .reshape(.{ .expert = sf.dim(0), .scale_word = @divExact(sf.dim(2), 4), .row = sf.dim(1), .scale_byte = 4 }).bitCast(.i32) + .withTags(sf.shape().transpose(.{ 0, 2, 1 })); + }, + .plain => {}, + } + switch (backend) { + .cute => { + projection.weight = packFp4Bytes(projection.weight); + const q = &projection.quantization.?; + if (scheme == .nvfp4) { + q.scales = q.scales.bitCast(.u8); + // CuTe takes the input quantization multiplier and a GEMM + // output multiplier that folds in its inverse. + const input_scale = q.input_scale.?.asMultiplier(); + q.global_scale = .{ .value = q.global_scale.?.asMultiplier().mul(input_scale), .operation = .multiply }; + q.input_scale = .{ .value = Tensor.scalar(1, .f32).div(input_scale), .operation = .multiply }; + } + }, + .triton => if (scheme == .mxfp4) { + projection.weight = packFp4Bytes(projection.weight); + }, + .mosaic_tpu => { + projection.weight = projection.weight.transpose(.{ 0, 2, 1 }); + }, + .metal => if (projection.quantization) |*q| { + if (q.scheme != .nvfp4) q.scales = q.scales.convert(.bf16); + }, + else => {}, + } + } + + if (backend == .metal and scheme == .nvfp4) { + const global = &prepared.gate_up.quantization.?.global_scale.?; + global.value = global.value.appendAxes(.{.proj}).repeat1d(.proj, 2); + } + return prepared; +} + +const Outputs = struct { actual: Tensor, expected: Tensor }; + +/// Forward a fixture through the MoE implementation under test, and also through the StableHLO +/// reference implementation. Return both outputs for comparison. +fn forward(input: Tensor, c: Case) Outputs { + const canonical = makeExpertFixture(c); + const prepared = prepareForBackend(canonical, c.backend); + + const route_shape = Shape.init(.{ .t = c.tokens, .topk = c.topk }, .i32); + const route = Tensor.arange(.{ .end = @intCast(route_shape.count()) }, .i32).reshape(route_shape); + + // Only six of eight experts receive routes, with nonuniform counts. + const ids = route.mul(Tensor.scalar(5, .i32)).addConstant(2).remainder(Tensor.scalar(@min(c.experts, 6), .i32)); + + const weights = route.remainder(Tensor.scalar(3, .i32)).convert(.f32).addConstant(1).scale(0.25); + const options: GlobalOptions = .{ + .activation = c.activation, + .quantize_input = c.quantize_input, + .routing_weight_placement = c.placement, + }; + + const data_spec: zml.Sharding.Partitioning.Logical = if (c.expert_mesh.resolveLogicalAxis("data") != null) .parse(.{ .t = .data }) else .replicated; + const expert_spec: zml.Sharding.Partitioning.Logical = if (c.expert_mesh.resolveLogicalAxis("experts") != null) .parse(.{ .expert = .experts }) else .replicated; + const actual = zml.moe.forwardMoe(input, ids, weights, prepared.gate_up, prepared.down, c.backend, c.expert_mesh, .{ + .in_specs = .{ + .input = data_spec, + .ids = data_spec, + .weights = data_spec, + .gate_up = expert_spec, + .down = expert_spec, + }, + .out_specs = data_spec, + }, options); + // Keep the oracle input replicated so token sharding does not change its GEMM rounding. + const expected = reference.reference(input.replicate(), ids, weights, canonical.gate_up, canonical.down, .{ + .activation = c.activation, + .quantize_input = c.quantize_input, + .routing_weight_placement = c.placement, + .input_quantization = switch (c.backend) { + .metal => .none, + .cute => if (c.scheme == .nvfp4) .automatic else .mxfp8_bf16, + .triton => if (c.scheme == .mxfp4 and zml.Compiler.current().platform.target == .cuda) .mxfp8 else .automatic, + else => .automatic, + }, + }); + + return .{ .actual = actual, .expected = expected }; +} + +const gated_activations = [_]Activation{ + .{ .swiglu = .{} }, + .{ .swiglu = .{ .limit = 0.25 } }, + .{ .swiglu = .{ .bias = 1 } }, + .{ .swiglu = .{ .limit = 0.25, .bias = 1 } }, + .{ .swiglu_step = .{} }, + .{ .swiglu_step = .{ .limit = 0.25 } }, + .geglu, + .geglu_tanh, +}; +const all_activations = gated_activations ++ [_]Activation{ .relu, .silu, .gelu }; +const clipped_swiglu = [_]Activation{ + .{ .swiglu = .{ .limit = 0.25 } }, + .{ .swiglu = .{ .limit = 7 } }, +}; + +const FixtureShape = struct { + tokens: i64 = 1, + topk: i64 = 1, + width: i64 = 128, + intermediate: i64 = 128, + experts: i64 = 8, +}; + +// Every option combination runs with 1, 17, and 34 flattened tokens. +// Shapes and floating-point parameter values are representative finite sets. +const fixture_shapes = [_]FixtureShape{ + .{}, + .{ .tokens = 17, .topk = 2 }, + .{ .tokens = 34, .topk = 3 }, +}; +const specialized_shapes = [_]FixtureShape{ + .{ .width = 5120, .intermediate = 2304, .experts = 2 }, + .{ .width = 5120, .intermediate = 2304, .experts = 2, .tokens = 17, .topk = 2 }, + .{ .width = 5120, .intermediate = 2304, .experts = 2, .tokens = 34, .topk = 2 }, +}; +// Exercise direct decode routing with multiple experts; a single route is unsupported. +const specialized_shapes_cute = [_]FixtureShape{ + .{ .width = 5120, .intermediate = 2304, .experts = 2, .tokens = 1, .topk = 2 }, +} ++ specialized_shapes[1..].*; + +const Format = struct { + non_unit_scales: bool = false, + scheme: ?zml.Quantization.Scheme = null, + activation_dtype: zml.DataType = .bf16, +}; + +const Matrix = struct { + formats: []const Format = &.{.{}}, + activations: []const Activation, + quantize_inputs: []const bool = &.{false}, + placements: []const Placement = &.{.after_down}, + biases: []const bool = &.{false}, + shapes: []const FixtureShape = &fixture_shapes, + + pub fn count(self: Matrix) usize { + return self.formats.len * self.activations.len * self.quantize_inputs.len * + self.placements.len * self.biases.len * self.shapes.len; + } + + /// Shapes vary fastest, matching the nested loops in the backend tests. + pub fn caseAt(self: Matrix, backend: zml.moe.Backend, expert_mesh: *const zml.Mesh, index: usize) Case { + std.debug.assert(index < self.count()); + var remaining = index; + const shape = self.shapes[remaining % self.shapes.len]; + remaining /= self.shapes.len; + const bias = self.biases[remaining % self.biases.len]; + remaining /= self.biases.len; + const placement = self.placements[remaining % self.placements.len]; + remaining /= self.placements.len; + const quantize_input = self.quantize_inputs[remaining % self.quantize_inputs.len]; + remaining /= self.quantize_inputs.len; + const activation = self.activations[remaining % self.activations.len]; + remaining /= self.activations.len; + const format = self.formats[remaining]; + return .{ + .backend = backend, + .activation = activation, + .scheme = format.scheme, + .activation_dtype = format.activation_dtype, + .non_unit_scales = format.non_unit_scales, + .quantize_input = quantize_input, + .placement = placement, + .bias = bias, + .tokens = shape.tokens, + .topk = shape.topk, + .width = shape.width, + .intermediate = shape.intermediate, + .experts = shape.experts, + .expert_mesh = expert_mesh, + }; + } +}; + +const Options = struct { + producers: usize = 16, + consumers: usize = 1, + case_queue_capacity: usize = 16, + executable_queue_capacity: usize = 16, +}; + +const CaseJob = struct { + index: usize, + case: Case, +}; + +const ExecutableJob = struct { + job: CaseJob, + exe: zml.Exe, +}; + +const Result = struct { + case: Case, + outcome: union(enum) { + passed, + skipped, + compilation_failed: anyerror, + check_failed: anyerror, + }, +}; + +const Pipeline = struct { + platform: *const zml.Platform, + + fn generateCases(self: *const Pipeline, io: std.Io, backend: zml.moe.Backend, matrix: Matrix, case_queue: *std.Io.Queue(CaseJob), results: []?Result) std.Io.Cancelable!void { + defer case_queue.close(io); + const available = backend.isAvailable(self.platform); + + var index: usize = 0; + for (0..matrix.count()) |local_index| { + const c = matrix.caseAt(backend, self.platform.replicated_mesh, local_index); + if (available) { + case_queue.putOne(io, .{ .index = index, .case = c }) catch |err| return switch (err) { + error.Closed => {}, + error.Canceled => error.Canceled, + }; + } else { + results[index] = .{ .case = c, .outcome = .skipped }; + } + index += 1; + } + } + + fn compileExecutablesWorker(self: *const Pipeline, io: std.Io, allocator: std.mem.Allocator, case_queue: *std.Io.Queue(CaseJob), executable_queue: *std.Io.Queue(ExecutableJob), results: []?Result) std.Io.Cancelable!void { + while (true) { + const job = case_queue.getOne(io) catch |err| return switch (err) { + error.Closed => {}, + error.Canceled => error.Canceled, + }; + var exe = job.case.compile(allocator, io, self.platform) catch |err| { + if (err == error.Canceled) return error.Canceled; + results[job.index] = .{ .case = job.case, .outcome = .{ .compilation_failed = err } }; + continue; + }; + // Ownership transfers only on a successful enqueue. + executable_queue.putOne(io, .{ .job = job, .exe = exe }) catch |err| { + exe.deinit(); + return switch (err) { + error.Closed => {}, + error.Canceled => error.Canceled, + }; + }; + } + } + + fn runTestCaseWorker(self: *const Pipeline, io: std.Io, allocator: std.mem.Allocator, executable_queue: *std.Io.Queue(ExecutableJob), results: []?Result) std.Io.Cancelable!void { + while (true) { + var compiled = executable_queue.getOne(io) catch |err| return switch (err) { + error.Closed => {}, + error.Canceled => error.Canceled, + }; + defer compiled.exe.deinit(); + const job = compiled.job; + job.case.check(allocator, io, self.platform, &compiled.exe) catch |err| { + if (err == error.Canceled) return error.Canceled; + results[job.index] = .{ .case = job.case, .outcome = .{ .check_failed = err } }; + continue; + }; + results[job.index] = .{ .case = job.case, .outcome = .passed }; + } + } + + fn run(self: *const Pipeline, io: std.Io, allocator: std.mem.Allocator, backend: zml.moe.Backend, matrix: Matrix, options: Options) !void { + var arena: std.heap.ArenaAllocator = .init(allocator); + defer arena.deinit(); + + const cases_buffer = try arena.allocator().alloc(CaseJob, options.case_queue_capacity); + const executables_buffer = try arena.allocator().alloc(ExecutableJob, options.executable_queue_capacity); + const results = try arena.allocator().alloc(?Result, matrix.count()); + @memset(results, null); + + var case_queue: std.Io.Queue(CaseJob) = .init(cases_buffer); + var executable_queue: std.Io.Queue(ExecutableJob) = .init(executables_buffer); + + var enumerator: std.Io.Group = .init; + var producers: std.Io.Group = .init; + var consumers: std.Io.Group = .init; + defer { + case_queue.close(io); + executable_queue.close(io); + enumerator.cancel(io); + producers.cancel(io); + consumers.cancel(io); + // On partial startup or cancellation, queued executables still own resources. + while (executable_queue.getOneUncancelable(io)) |compiled| { + compiled.exe.deinit(); + } else |_| {} + } + + for (0..options.consumers) |_| try consumers.concurrent(io, runTestCaseWorker, .{ self, io, allocator, &executable_queue, results }); + for (0..options.producers) |_| try producers.concurrent(io, compileExecutablesWorker, .{ self, io, allocator, &case_queue, &executable_queue, results }); + try enumerator.concurrent(io, generateCases, .{ self, io, backend, matrix, &case_queue, results }); + + try enumerator.await(io); + try producers.await(io); + // Closed queues drain before getOne returns error.Closed. + executable_queue.close(io); + try consumers.await(io); + + try report(results); + } + + fn report(results: []const ?Result) !void { + var failed: usize = 0; + for (results, 0..) |result, index| { + const r = result orelse return error.MissingCaseResult; + switch (r.outcome) { + .passed, .skipped => {}, + inline .compilation_failed, .check_failed => |err, stage| { + failed += 1; + std.debug.print("MoE case {}/{} {s}: {any}\nError: {}\n", .{ + index + 1, results.len, @tagName(stage), r.case, err, + }); + }, + } + } + if (failed != 0) return error.MoeComplianceFailed; + } +}; + +test "Triton compatibility matrix" { + const platform = zml.testing.env(); + + const pipeline: Pipeline = .{ .platform = platform }; + + try pipeline.run(std.testing.io, std.testing.allocator, .triton, .{ + .formats = &.{ + .{}, + .{ .scheme = .mxfp8 }, + .{ .scheme = .fp8_per_tensor }, + .{ .scheme = .fp8_per_channel }, + .{ .scheme = .fp8_block128 }, + .{ .scheme = .fp8_block32 }, + }, + .activations = &all_activations, + // BF16 ignores this option; exercise both accepted settings. + .quantize_inputs = &.{ false, true }, + .placements = &.{ .before_down, .after_down }, + .biases = &.{ false, true }, + }, .{}); + + if (platform.target != .cuda) { + // Other platforms retain the general MXFP4 path and its full contract. + try pipeline.run(std.testing.io, std.testing.allocator, .triton, .{ + .formats = &.{.{ .scheme = .mxfp4 }}, + .activations = &all_activations, + .quantize_inputs = &.{ false, true }, + .placements = &.{ .before_down, .after_down }, + .biases = &.{ false, true }, + }, .{}); + return; + } + + // Preserve the 12 supported cases from the former general MXFP4 matrix. + // Its other 252 cases require activations or projection bias unsupported + // by the simplified path. The former specialized matrix remains below. + try pipeline.run(std.testing.io, std.testing.allocator, .triton, .{ + .formats = &.{.{ .scheme = .mxfp4 }}, + .activations = &.{gated_activations[1]}, + .quantize_inputs = &.{ false, true }, + .placements = &.{ .before_down, .after_down }, + }, .{}); +} + +// Keep triton mxfp4 separate because it only supports a subset for now +test "Triton MXFP4 compatibility matrix" { + const platform = zml.testing.env(); + if (platform.target != .cuda) return error.SkipZigTest; + const pipeline: Pipeline = .{ .platform = platform }; + + // The merged backend uses the former specialized MXFP4 path: clipped + // SwiGLU without projection bias, with unconditional MXFP8 activations. + try pipeline.run(std.testing.io, std.testing.allocator, .triton, .{ + .formats = &.{.{ .scheme = .mxfp4 }}, + .activations = &clipped_swiglu, + .quantize_inputs = &.{ false, true }, + .placements = &.{ .before_down, .after_down }, + }, .{}); +} + +test "Triton MoE prefill launch buckets fit shared memory" { + const platform = zml.testing.env(); + if (!zml.moe.Backend.triton.isAvailable(platform) or platform.target != .rocm) return error.SkipZigTest; + + for ([_]i64{ 8, 32, 96, 128, 256, 512, 1024, 2048, 3072, 4096 }) |tokens| { + const case: Case = .{ + .backend = .triton, + .expert_mesh = platform.replicated_mesh, + .tokens = tokens, + .width = 256, + .intermediate = 128, + .topk = 2, + }; + var exe = try case.compile(std.testing.allocator, std.testing.io, platform); + defer exe.deinit(); + try case.check(std.testing.allocator, std.testing.io, platform, &exe); + } +} + +test "FlashInfer compatibility matrix" { + const platform = zml.testing.env(); + + const pipeline: Pipeline = .{ .platform = platform }; + + // Only default SwiGLU, no linear bias, and routing after the down projection. + try pipeline.run(std.testing.io, std.testing.allocator, .flashinfer_cutlass, .{ + .formats = &.{.{ .scheme = .nvfp4 }}, + .activations = &.{.{ .swiglu = .{} }}, + }, .{}); +} + +test "specialized CuTe MXFP4 compatibility matrix" { + const platform = zml.testing.env(); + + const pipeline: Pipeline = .{ .platform = platform }; + + try pipeline.run(std.testing.io, std.testing.allocator, .cute, .{ + .formats = &.{.{ .scheme = .mxfp4 }}, + .activations = &clipped_swiglu, + .quantize_inputs = &.{ false, true }, + .placements = &.{.before_down}, + .shapes = &specialized_shapes_cute, + }, .{}); +} + +test "CuTe NVFP4 compatibility matrix" { + const platform = zml.testing.env(); + if (!zml.moe.cute.isNvfp4Supported(platform)) return error.SkipZigTest; + + const pipeline: Pipeline = .{ .platform = platform }; + try pipeline.run(std.testing.io, std.testing.allocator, .cute, .{ + .formats = &.{ + .{ .scheme = .nvfp4 }, + .{ .scheme = .nvfp4, .non_unit_scales = true }, + }, + .activations = &.{ .{ .swiglu = .{} }, .{ .swiglu = .{ .limit = 0.25 } } }, + .placements = &.{ .before_down, .after_down }, + .shapes = &.{ + .{ .width = 5120, .intermediate = 2304, .tokens = 1, .topk = 2 }, + .{ .width = 5120, .intermediate = 2304, .tokens = 17, .topk = 2 }, + .{ .width = 5120, .intermediate = 2304, .tokens = 34, .topk = 3 }, + }, + }, .{}); +} + +test "Fly compatibility matrix" { + const platform = zml.testing.env(); + + const pipeline: Pipeline = .{ .platform = platform }; + + // These constraints keep execution on Fly rather than its Triton fallback. + try pipeline.run(std.testing.io, std.testing.allocator, .fly, .{ + .formats = &.{.{ .scheme = .mxfp4 }}, + .activations = &gated_activations, + .quantize_inputs = &.{ false, true }, + .placements = &.{.before_down}, + .shapes = &specialized_shapes, + }, .{}); +} + +test "Fly general projection shapes" { + const pipeline: Pipeline = .{ .platform = zml.testing.env() }; + + // Cover GEMV, sorted MFMA with 16- and 32-row tiles, and wider projections. + try pipeline.run(std.testing.io, std.testing.allocator, .fly, .{ + .formats = &.{.{ .scheme = .mxfp4 }}, + .activations = &.{.{ .swiglu = .{} }}, + .placements = &.{.before_down}, + .shapes = &.{ + .{ .width = 384, .intermediate = 128, .experts = 2 }, + .{ .width = 384, .intermediate = 128, .experts = 2, .tokens = 7 }, + .{ .width = 384, .intermediate = 128, .experts = 2, .tokens = 1024 }, + .{ .width = 7168, .intermediate = 2048, .experts = 2 }, + .{ .width = 7168, .intermediate = 2048, .experts = 2, .tokens = 7 }, + }, + }, .{}); +} + +test "Metal compatibility matrix" { + const platform = zml.testing.env(); + + const pipeline: Pipeline = .{ .platform = platform }; + + // Quantized Metal kernels require BF16 activations and do not support bias. + try pipeline.run(std.testing.io, std.testing.allocator, .metal, .{ + .formats = &.{ + .{}, + .{ .activation_dtype = .f16 }, + .{ .activation_dtype = .f32 }, + .{ .scheme = .nvfp4 }, + .{ .scheme = .mxfp8 }, + .{ .scheme = .fp8_per_tensor }, + .{ .scheme = .fp8_per_channel }, + .{ .scheme = .fp8_block128 }, + .{ .scheme = .fp8_block32 }, + }, + .activations = &all_activations, + }, .{}); +} + +test "Mosaic TPU compatibility matrix" { + const platform = zml.testing.env(); + + const pipeline: Pipeline = .{ .platform = platform }; + + // canonicalizeDown currently requires a gated projection; quantized weights + // are not supported by gmmDType. Plain activations need a backend fix first. + try pipeline.run(std.testing.io, std.testing.allocator, .mosaic_tpu, .{ + .formats = &.{ .{}, .{ .activation_dtype = .f16 }, .{ .activation_dtype = .f32 } }, + .activations = &gated_activations, + .biases = &.{ false, true }, + }, .{}); +} diff --git a/zml/kernels/moe/triton/BUILD.bazel b/zml/kernels/moe/triton/BUILD.bazel new file mode 100644 index 00000000..8dda7f88 --- /dev/null +++ b/zml/kernels/moe/triton/BUILD.bazel @@ -0,0 +1,9 @@ +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "triton", + srcs = [ + "moe.zig", + "triton.zig", + ], +) diff --git a/zml/kernels/moe/triton/moe.zig b/zml/kernels/moe/triton/moe.zig new file mode 100644 index 00000000..6e4006fa --- /dev/null +++ b/zml/kernels/moe/triton/moe.zig @@ -0,0 +1,1178 @@ +//! Triton MoE kernel declarations — direct ports of the `@triton.jit` +//! functions in `triton_kernels/moe.py`. Each kernel here is a +//! `tri.Kernel(...)` declaration with its own inline config struct (no +//! defaults — every field is required at call time). Production callers +//! invoke these via `K.call(inputs, outputs, opts)`; offline tooling reaches the TTIR +//! string via `K.emit(allocator, ctx, cfg)`. + +const std = @import("std"); + +const zml = @import("../../../zml.zig"); +const tri = zml.kernel.triton; +const Builder = tri.Builder; +const Value = tri.Value; +const DType = tri.DType; +const RoutingWeightPlacement = @import("../api.zig").RoutingWeightPlacement; + +const log = std.log.scoped(.@"zml/moe/triton_kernels"); + +/// Floor to a multiple of 16 — matches the Python `(v // 16) * 16` guards +/// that keep dynamic strides aligned for tt.load/store. +fn blockFloor16(v: Value) Value { + return v.div(16).mul(16); +} + +// ============================================================================= +// per_token_group_quant_fp8 +// ============================================================================= + +/// Direct port of Python `per_token_group_quant_fp8` — per-token-group +/// float8 quantization. +pub const PerTokenGroupQuantFp8 = struct { + pub const Cfg = struct { + input_dtype: DType, + output_dtype: DType, + scale_dtype: DType, + block: usize, + quant_scheme: zml.Quantization.Scheme, + }; + pub const Kernel = tri.Kernel(Cfg, .{ + .name = "per_token_group_quant_fp8", + .inputs = &.{ "y_ptr", "group_size_ptr", "y_num_columns_ptr", "y_row_stride_ptr", "eps_ptr" }, + .outputs = &.{ "y_q", "y_s" }, + .run = run, + }); + fn run(b: *Builder, cfg: Cfg) tri.FinishError!void { + const a = try b.declareArgs(.{ + .y_ptr = .{ .ptr = cfg.input_dtype }, + .group_size_ptr = .{ .ptr = .i64 }, + .y_num_columns_ptr = .{ .ptr = .i64 }, + .y_row_stride_ptr = .{ .ptr = .i64 }, + .eps_ptr = .{ .ptr = .f32 }, + .y_q_ptr = .{ .ptr = cfg.output_dtype }, + .y_s_ptr = .{ .ptr = cfg.scale_dtype }, + }); + + const block: i64 = @intCast(cfg.block); + const out_dt = cfg.output_dtype; + const fp8_max: f32 = switch (out_dt) { + .f8e4m3fn => 448.0, + .f8e4m3fnuz => 224.0, + else => return error.InvalidMlir, + }; + const scale_dt = cfg.scale_dtype; + + const group_size = b.load(a.group_size_ptr); + const y_num_columns = b.load(a.y_num_columns_ptr); + const y_row_stride = b.load(a.y_row_stride_ptr); + const eps = b.load(a.eps_ptr); + + const groups_per_row = y_num_columns.div(group_size); + // g_id is i32 from `tl.program_id`; the i64 promotion only happens + // inside the `(row|row_g_id).to(tl.int64)` casts in the Python source. + // y_s_ptr_shifted reuses the raw i32 `g_id` directly (no extsi). + const pid_i32 = b.programId(.x); + const g_id = pid_i32.to(.i64); + + // y_ptr += (g_id // groups_per_row) * y_row_stride + (g_id % groups_per_row) * group_size + // Python emit order: divsi, remsi, muli (row * stride), muli (row_g_id * group_size), addi. + const row = g_id.div(groups_per_row); + const row_g_id = g_id.rem(groups_per_row); + const row_off = row.mul(y_row_stride); + const grp_off = row_g_id.mul(group_size); + const y_ptr_shifted = a.y_ptr.addPtr(row_off.add(grp_off)); + const y_q_ptr_shifted = a.y_q_ptr.addPtr(g_id.mul(group_size)); + // y_s_ptr += g_id (raw i32, no cast — matches `y_s_ptr += g_id` in Python). + const y_s_ptr_shifted = a.y_s_ptr.addPtr(pid_i32); + + // cols = arange(0, BLOCK) — i64 for both pointer offsets and mask + // comparison against the i64 `group_size` (matches Python's + // `cols < group_size` after Triton's auto-promotion). + const cols_i32 = b.arange(0, block, .i32); + const cols_i64 = cols_i32.to(.i64); + const mask = cols_i64.lt(group_size); + + // Python: `tl.load(y_ptr + cols, mask=mask, other=0.0)` — explicit + // 3-operand form. Use `.other` to materialize the const-zero tensor + // (without it, `tt.load` is 2-operand and Python's emit doesn't match). + const y_load_ptrs = y_ptr_shifted.addPtr(cols_i64); + const y_other = b.zeros(&.{block}, cfg.input_dtype); + const y = b.loadOpts(y_load_ptrs, .{ .mask = mask, .other = y_other }).to(.f32); + + // _absmax = max(tl.max(tl.abs(y)), eps) — `b.max(...).maximum(eps)` + // resolves to `arith.maxnumf` (matches Python's `tl.maximum`). + const absmax = b.max(b.absf(y)).maximum(eps); + + // scale_raw = _absmax * (1.0 / fp8_max) + const scale_raw = absmax.mul(1.0 / fp8_max); + const y_s: Value, const stored_scale: Value = switch (cfg.quant_scheme) { + .mxfp8 => blk: { + const exponent = b.ceil(b.log2(scale_raw)); + // E8M0 stores the biased exponent as a byte. + break :blk .{ b.exp2(exponent), exponent.add(127).to(.i32).to(.i8) }; + }, + .fp8_per_channel, .fp8_per_tensor, .fp8_block128 => .{ scale_raw, scale_raw.to(scale_dt) }, + .fp8_block32 => blk: { + // Match quantizeBlockFp8(..., .f8e8m0), but store the scales as FP32 here. + // It is unclear whether power-of-two scaling is best for this kernel, + // but it keeps activation quantization consistent with that function. + // Round up from the FP32 bits, including values just above a power of two. + const bits = scale_raw.bitCast(.i32); + const exponent = bits.div(0x800000); + const fractional = bits.bitAnd(0x7fffff).ne(0).to(.i32); + const scale = exponent.add(fractional).mul(0x800000).bitCast(.f32); + break :blk .{ scale, scale.to(scale_dt) }; + }, + .mxfp4, .nvfp4 => unreachable, + }; + + // y_q = clamp(y / y_s, fp8_min, fp8_max).to(output_dtype) + const y_div = y.div(y_s); + const clamped = b.clampf( + y_div, + b.splat(-fp8_max, &.{block}), + b.splat(fp8_max, &.{block}), + ); + const y_q = clamped.to(out_dt); + + const y_q_ptrs = y_q_ptr_shifted.addPtr(cols_i64); + b.storeOpts(y_q_ptrs, y_q, .{ .mask = mask }); + b.store(y_s_ptr_shifted, stored_scale); + } +}; + +// ============================================================================= +// write_zeros_to_output — helper called by FusedMoe +// ============================================================================= + +/// Direct port of Python `write_zeros_to_output`. Stores a masked zero-tile +/// for blocks whose expert is `-1` (no expert assigned). `stride_cn` is +/// hardcoded to 1 (contiguous last dim), so the N-axis offset is just +/// `offs_cn` itself. +fn writeZerosToOutput( + b: *Builder, + c_ptr: Value, + stride_cm: Value, + pid_n: Value, + n: Value, + offs_token: Value, + token_mask: Value, + block_size_m: i64, + block_size_n: i64, + compute_type: DType, +) void { + const accumulator = b.zeros(&.{ block_size_m, block_size_n }, compute_type); + const offs_cn = pid_n.mul(block_size_n).add(b.arange(0, block_size_n, .i64)); + + // c_ptrs in two addptrs to match Python's `c_ptr + cm[:, None] + cn[None, :]` + // left-to-right evaluation. `stride_cm * offs_token[:, None]` puts the + // splatted stride on the LHS of arith.muli (Python source order). + const offs_token_col = b.expandDims(offs_token, 1); + const cm_col = stride_cm.mul(offs_token_col); + const c_ptrs_col = b.splat(c_ptr, &.{ block_size_m, 1 }).addPtr(cm_col); + const offs_cn_row = b.expandDims(offs_cn, 0); + const c_ptrs_2d = b.broadcastTo(c_ptrs_col, &.{ block_size_m, block_size_n }); + const cn_off_2d = b.broadcastTo(offs_cn_row, &.{ block_size_m, block_size_n }); + const c_ptrs = c_ptrs_2d.addPtr(cn_off_2d); + + // c_mask: Python emit order — expand_dims, splat, cmpi, broadcast, broadcast, andi. + const token_mask_col = b.expandDims(token_mask, 1); + const n_splat = b.splat(n, &.{ 1, block_size_n }); + const offs_cn_lt_2d = offs_cn_row.lt(n_splat); + const token_mask_full = b.broadcastTo(token_mask_col, &.{ block_size_m, block_size_n }); + const offs_cn_lt_full = b.broadcastTo(offs_cn_lt_2d, &.{ block_size_m, block_size_n }); + b.storeOpts(c_ptrs, accumulator, .{ .mask = token_mask_full.bitAnd(offs_cn_lt_full) }); +} + +// ============================================================================= +// fused_moe_kernel +// ============================================================================= + +/// Routed GEMM supporting BF16, MXFP4, MXFP8, and per-tensor, per-channel, or +/// 128x128 block-scaled FP8 weights. +pub const FusedMoe = struct { + pub const Cfg = struct { + a_dtype: DType, + b_dtype: DType, + c_dtype: DType, + a_scale_dtype: ?DType, + b_scale_dtype: ?DType, + b_bias_dtype: ?DType, + routing_weights_dtype: ?DType, + block_size_m: usize, + block_size_n: usize, + block_size_k: usize, + group_size_m: usize, + top_k: usize, + naive_block_assignment: bool, + compute_type: DType, + quant_scheme: ?zml.Quantization.Scheme, + }; + pub const Kernel = tri.Kernel(Cfg, .{ + .name = "fused_moe_kernel", + .inputs = &.{ + "a_ptr", "b_ptr", "b_bias_ptr", "a_scale_ptr", "b_scale_ptr", + "routing_weights_ptr", "sorted_token_ids_ptr", "expert_ids_ptr", "num_tokens_post_padded_ptr", "N_ptr", + "K_ptr", "EM_ptr", "num_valid_tokens_ptr", "stride_am_ptr", "stride_be_ptr", + "stride_bn_ptr", "stride_cm_ptr", "stride_asm_ptr", "stride_ask_ptr", "stride_bse_ptr", + "stride_bsk_ptr", "stride_bsn_ptr", "stride_bbe_ptr", "stride_bbn_ptr", + }, + .outputs = &.{"c"}, + .run = run, + }); + fn run(b: *Builder, cfg: Cfg) tri.FinishError!void { + const fp8_block_size: usize = if (cfg.quant_scheme == .fp8_block32) 32 else 128; + if ((cfg.quant_scheme == .fp8_block128 or cfg.quant_scheme == .fp8_block32) and cfg.block_size_k != fp8_block_size) { + log.err("fused_moe_kernel: block-scaled FP8 requires BLOCK_SIZE_K={}", .{fp8_block_size}); + return error.InvalidMlir; + } + + if ((cfg.quant_scheme == .mxfp4 or cfg.quant_scheme == .mxfp8) and cfg.block_size_k % 32 != 0) { + log.err("fused_moe_kernel: MXFP4/MXFP8 require BLOCK_SIZE_K divisible by 32", .{}); + return error.InvalidMlir; + } + + const a = try b.declareArgs(.{ + .a_ptr = .{ .ptr = cfg.a_dtype }, + .b_ptr = .{ .ptr = cfg.b_dtype }, + // The fixed signature requires concrete pointer types even for absent, + // unused inputs. FP32 matches the caller's placeholder tensors. + .b_bias_ptr = .{ .ptr = cfg.b_bias_dtype orelse .f32 }, + .a_scale_ptr = .{ .ptr = cfg.a_scale_dtype orelse .f32 }, + .b_scale_ptr = .{ .ptr = cfg.b_scale_dtype orelse .f32 }, + .routing_weights_ptr = .{ .ptr = cfg.routing_weights_dtype orelse .f32 }, + .sorted_token_ids_ptr = .{ .ptr = .i32 }, + .expert_ids_ptr = .{ .ptr = .i32 }, + .num_tokens_post_padded_ptr = .{ .ptr = .i32 }, + .N_ptr = .{ .ptr = .i64 }, + .K_ptr = .{ .ptr = .i64 }, + .EM_ptr = .{ .ptr = .i64 }, + .num_valid_tokens_ptr = .{ .ptr = .i64 }, + .stride_am_ptr = .{ .ptr = .i64 }, + .stride_be_ptr = .{ .ptr = .i64 }, + .stride_bn_ptr = .{ .ptr = .i64 }, + .stride_cm_ptr = .{ .ptr = .i64 }, + .stride_asm_ptr = .{ .ptr = .i64 }, + .stride_ask_ptr = .{ .ptr = .i64 }, + .stride_bse_ptr = .{ .ptr = .i64 }, + .stride_bsk_ptr = .{ .ptr = .i64 }, + .stride_bsn_ptr = .{ .ptr = .i64 }, + .stride_bbe_ptr = .{ .ptr = .i64 }, + .stride_bbn_ptr = .{ .ptr = .i64 }, + .c_ptr = .{ .ptr = cfg.c_dtype }, + }); + + const block_size_m: i64 = @intCast(cfg.block_size_m); + const block_size_n: i64 = @intCast(cfg.block_size_n); + const block_size_k: i64 = @intCast(cfg.block_size_k); + const weight_packing: i64 = if (cfg.quant_scheme == .mxfp4) 2 else 1; + const block_size_bk = @divExact(block_size_k, weight_packing); + const group_size_m: i64 = @intCast(cfg.group_size_m); + const top_k: i64 = @intCast(cfg.top_k); + const compute_type = cfg.compute_type; + + // Matrix dimensions, strides, and padded routing capacity are checked + // for 16-element alignment by the caller. + const n_raw = b.load(a.N_ptr); + const k_raw = b.load(a.K_ptr); + const em_raw = b.load(a.EM_ptr); + const num_valid_tokens = b.load(a.num_valid_tokens_ptr); + const stride_am_raw = b.load(a.stride_am_ptr); + const stride_be_raw = b.load(a.stride_be_ptr); + const stride_bn_raw = b.load(a.stride_bn_ptr); + const stride_cm_raw = b.load(a.stride_cm_ptr); + const n_block = blockFloor16(n_raw); + const k_block = blockFloor16(k_raw); + const em_block = blockFloor16(em_raw); + const stride_am_block = blockFloor16(stride_am_raw); + const stride_be_block = blockFloor16(stride_be_raw); + const stride_bn_block = blockFloor16(stride_bn_raw); + const stride_cm_block = blockFloor16(stride_cm_raw); + // stride_ak / stride_bk / stride_cn are all 1 in this port — the A/B/C + // inner dimension is contiguous, so those mul-by-1 terms are elided. + + // pid grouping. Python keeps `pid` as i32 and lets the divsi auto- + // promote at the use site — emit the cdivs first, then extsi at the + // first i64 use. Manual order keeps `extsi pid` after the cdivs in TTIR. + const pid_i32 = b.programId(.x); + const num_pid_m = em_block.cdiv(block_size_m); + const num_pid_n = n_block.cdiv(block_size_n); + const num_pid_in_group = num_pid_n.mul(group_size_m); + const pid = pid_i32.to(.i64); + const group_id = pid.div(num_pid_in_group); + const first_pid_m = group_id.mul(group_size_m); + const gsm_actual = num_pid_m.sub(first_pid_m).minimum(group_size_m); + const pid_mod_in_group = pid.rem(num_pid_in_group); + const pid_m = first_pid_m.add(pid_mod_in_group.rem(gsm_actual)); + const pid_n = pid_mod_in_group.div(gsm_actual); + + // Python source order: arange first, then load + early-return. + // Match emit order so the make_range op lands before the cf.cond_br. + const offs = b.arange(0, block_size_m, .i64); + // Order: load (i32), muli, extsi (Python emits the muli before the + // i32→i64 extsi at the cmpi). + const num_tokens_post_padded_i32 = b.load(a.num_tokens_post_padded_ptr); + const pid_m_block = pid_m.mul(block_size_m); + const num_tokens_post_padded = num_tokens_post_padded_i32.to(.i64); + const out_of_range = pid_m_block.ge(num_tokens_post_padded); + + b.returnIfOpts(out_of_range, .{ .inline_return = true }); + + const offs_token = if (cfg.naive_block_assignment) + b.select( + offs.eq(0), + pid_m.splatTo(&.{block_size_m}), + num_valid_tokens.splatTo(&.{block_size_m}), + ) + else off: { + const ids_ptrs = a.sorted_token_ids_ptr + .addPtr(pid_m.mul(block_size_m).add(offs)); + break :off b.load(ids_ptrs).to(.i64); + }; + const token_mask = offs_token.lt(num_valid_tokens); + + // off_experts = expert_ids[pid_m].to(i64). pid_m is i64; Python's + // `expert_ids_ptr + pid_m` keeps the offset as i64. + const off_experts = b.load(a.expert_ids_ptr.addPtr(pid_m)).to(.i64); + const is_dead = off_experts.eq(-1); + + // Python: if off_experts == -1: write_zeros_to_output(...); return + var dead_ret = b.openReturnIfOpts(is_dead, .{ .inline_return = true }); + { + writeZerosToOutput( + b, + a.c_ptr, + stride_cm_block, + pid_n, + n_block, + offs_token, + token_mask, + block_size_m, + block_size_n, + compute_type, + ); + dead_ret.close(); + } + + // CSE `pid_n * BLOCK_SIZE_N` so it reuses across offs_bn / offs_cn. + const pid_n_block_n = pid_n.mul(block_size_n); + const offs_bn = pid_n_block_n.add(b.arange(0, block_size_n, .i64)).rem(n_block); + + // offs_k stays i32 through expand_dims; extsi to i64 happens on the 2D form. + const offs_k_i32 = b.arange(0, block_size_k, .i32); + + // a_ptrs: `offs_token[:, None] // top_k * stride_am + offs_k[None, :]`. + // (stride_ak == 1 here, so `* stride_ak` folds away.) Build with explicit + // broadcastTo to match Python's emit shape. + const offs_token_col = b.expandDims(offs_token, 1); + const am_col = offs_token_col.div(top_k).mul(stride_am_block); + const offs_k_row = b.expandDims(offs_k_i32, 0).to(.i64); + const am_term = b.broadcastTo(am_col, &.{ block_size_m, block_size_k }); + const ak_term = b.broadcastTo(offs_k_row, &.{ block_size_m, block_size_k }); + const a_ptrs_init = a.a_ptr.addPtr(am_term.add(ak_term)); + + // b_ptrs: `b_ptr + off_experts * stride_be` (scalar addptr) THEN + // `+ (offs_k[:, None] * stride_bk + offs_bn[None, :] * stride_bn)` (2D + // addptr) — matches Python's left-to-right evaluation. Defer extsi + // for offs_k to after offs_bn's expand_dims/mul (Python emit order). + const b_ptr_shifted = a.b_ptr.addPtr(off_experts.mul(stride_be_block)); + // MXFP4 packs two K values per byte; activation K remains unpacked. + const offs_bk_i32 = b.arange(0, block_size_bk, .i32); + const offs_k_col_i32 = b.expandDims(offs_bk_i32, 1); + const offs_bn_row = b.expandDims(offs_bn, 0); + const bn_row = offs_bn_row.mul(stride_bn_block); + const offs_k_col = offs_k_col_i32.to(.i64); + const bk_term = b.broadcastTo(offs_k_col, &.{ block_size_bk, block_size_n }); + const bn_term = b.broadcastTo(bn_row, &.{ block_size_bk, block_size_n }); + const b_ptrs_init = b_ptr_shifted.addPtr(bk_term.add(bn_term)); + + const acc_init = b.zeros(&.{ block_size_m, block_size_n }, .f32); + + // Loop bounds stay i64 because `k_block` is i64. iter_args order + // matches Python's TTIR for the matrix and accumulator state. + const num_k_iters = k_block.cdiv(block_size_k); + var loop = b.openFor(0, num_k_iters, 1, .{ a_ptrs_init, b_ptrs_init, acc_init }); + { + const k_iter = loop.iv; + const a_ptrs = loop.carried[0]; + const b_ptrs = loop.carried[1]; + const acc = loop.carried[2]; + + const k_remaining = k_block.sub(k_iter.mul(block_size_k)); + + // mask_a = token_mask[:, None] & (offs_k[None, :] < k_remaining). + const token_mask_col_a = b.expandDims(token_mask, 1); + const offs_k_row_a = b.expandDims(offs_k_i32, 0).to(.i64); + const offs_k_lt_a = offs_k_row_a.lt(b.splat(k_remaining, &.{ 1, block_size_k })); + const token_mask_full_a = b.broadcastTo(token_mask_col_a, &.{ block_size_m, block_size_k }); + const offs_k_lt_a_full = b.broadcastTo(offs_k_lt_a, &.{ block_size_m, block_size_k }); + const mask_a = token_mask_full_a.bitAnd(offs_k_lt_a_full); + + const a_val = b.loadOpts(a_ptrs, .{ + .mask = mask_a, + .other = b.zeros(&.{ block_size_m, block_size_k }, cfg.a_dtype), + }); + + const offs_k_col_b = b.expandDims(offs_bk_i32, 1).to(.i64); + const offs_k_lt_b = offs_k_col_b.lt(b.splat(k_remaining.cdiv(weight_packing), &.{ block_size_bk, 1 })); + const mask_b = b.broadcastTo(offs_k_lt_b, &.{ block_size_bk, block_size_n }); + + const b_val = b.loadOpts(b_ptrs, .{ + .mask = mask_b, + .other = b.zeros(&.{ block_size_bk, block_size_n }, cfg.b_dtype), + }); + + const new_acc = if (cfg.quant_scheme) |scheme| switch (scheme) { + .mxfp4, .mxfp8 => scaled: { + const scale_k = @divExact(block_size_k, 32); + const groups = k_iter.mul(scale_k).add(b.arange(0, scale_k, .i64)); + const stride_bse = b.load(a.stride_bse_ptr); + const stride_bsk = b.load(a.stride_bsk_ptr); + const stride_bsn = b.load(a.stride_bsn_ptr); + // dot_scaled expects RHS scales in [N, K/32] order. + const rows = off_experts.mul(stride_bse).add(offs_bn.mul(stride_bsn)).expandDims(1); + const cols = groups.mul(stride_bsk).expandDims(0); + const offsets = b.broadcastTo(rows, &.{ block_size_n, scale_k }) + .add(b.broadcastTo(cols, &.{ block_size_n, scale_k })); + const n_mask = b.broadcastTo(offs_bn.lt(n_block).expandDims(1), &.{ block_size_n, scale_k }); + const k_mask = b.broadcastTo(groups.lt(k_block.cdiv(32)).expandDims(0), &.{ block_size_n, scale_k }); + const scales = b.loadOpts(a.b_scale_ptr.addPtr(offsets), .{ + .mask = n_mask.bitAnd(k_mask), + .other = b.full(&.{ block_size_n, scale_k }, 127, .i8), + }); + const a_scales: ?Value = if (cfg.a_scale_dtype != null) input_scales: { + const a_rows = offs_token.div(top_k).mul(b.load(a.stride_asm_ptr)).expandDims(1); + const a_cols = groups.mul(b.load(a.stride_ask_ptr)).expandDims(0); + const a_offsets = b.broadcastTo(a_rows, &.{ block_size_m, scale_k }) + .add(b.broadcastTo(a_cols, &.{ block_size_m, scale_k })); + const a_mask = b.broadcastTo(token_mask.expandDims(1), &.{ block_size_m, scale_k }) + .bitAnd(b.broadcastTo(groups.lt(k_block.cdiv(32)).expandDims(0), &.{ block_size_m, scale_k })); + break :input_scales b.loadOpts(a.a_scale_ptr.addPtr(a_offsets), .{ + .mask = a_mask, + .other = b.full(&.{ block_size_m, scale_k }, 127, .i8), + }); + } else null; + break :scaled b.dotScaledOpts(a_val, b_val, acc, a_scales, scales, if (a_scales != null) .e4m3 else .bf16, if (scheme == .mxfp4) .e2m1 else .e4m3, .{}); + }, + .fp8_per_channel, .fp8_per_tensor => b.dotOpts(a_val, if (cfg.a_scale_dtype != null) b_val else b_val.to(.bf16), acc, .{ + .input_precision = .tf32, + .max_num_imprecise_acc = 0, + }), + .fp8_block32, .fp8_block128 => scaled: { + // Activation scales are [token, K/block] and weight scales are + // [expert, N/block, K/block]. Compute their addresses from the + // loop index; activation scales are optional. + const stride_bse = b.load(a.stride_bse_ptr); + const stride_bsk = b.load(a.stride_bsk_ptr); + const stride_bsn = b.load(a.stride_bsn_ptr); + const b_scale_ptrs = a.b_scale_ptr.addPtr( + off_experts.mul(stride_bse) + .add(offs_bn.div(fp8_block_size).mul(stride_bsn)) + .add(k_iter.mul(stride_bsk)), + ); + const dot = b.dotOpts(a_val, if (cfg.a_scale_dtype != null) b_val else b_val.to(.bf16), b.zeros(&.{ block_size_m, block_size_n }, .f32), .{ + .input_precision = .tf32, + .max_num_imprecise_acc = 0, + }); + const b_s = b.loadOpts(b_scale_ptrs, .{ + .mask = offs_bn.lt(n_block), + .other = b.zeros(&.{block_size_n}, cfg.b_scale_dtype orelse .f32), + }).to(.f32); + var scaled_dot = dot.mul(b_s.expandDims(0)); + if (cfg.a_scale_dtype) |dtype| { + const a_scale_ptrs = a.a_scale_ptr.addPtr( + offs_token.div(top_k).mul(b.load(a.stride_asm_ptr)) + .add(k_iter.mul(b.load(a.stride_ask_ptr))), + ); + const a_s = b.loadOpts(a_scale_ptrs, .{ + .mask = token_mask, + .other = b.zeros(&.{block_size_m}, dtype), + }).to(.f32); + scaled_dot = scaled_dot.mul(a_s.expandDims(1)); + } + break :scaled acc.add(scaled_dot); + }, + .nvfp4 => unreachable, + } else b.dotOpts(a_val, b_val, acc, .{ + .input_precision = .tf32, + .max_num_imprecise_acc = 0, + }); + + // BLOCK_SIZE_K is `tl.constexpr` → i32 dense splat (stride_ak == 1). + const bsk_i32: i32 = @intCast(cfg.block_size_k); + const new_a_ptrs = a_ptrs.addPtr(b.splat(bsk_i32, &.{ block_size_m, block_size_k })); + const new_b_ptrs = b_ptrs.addPtr(b.splat(@as(i32, @intCast(block_size_bk)), &.{ block_size_bk, block_size_n })); + + loop.yield(.{ + new_a_ptrs, + new_b_ptrs, + new_acc, + }); + } + var accumulator = loop.results[2]; + + if (cfg.quant_scheme == .fp8_per_channel or cfg.quant_scheme == .fp8_per_tensor) { + // These scales are constant along K, so apply them after accumulation. + const expert_offset = off_experts.mul(b.load(a.stride_bse_ptr)); + const offsets = if (cfg.quant_scheme == .fp8_per_channel) + expert_offset.add(offs_bn.mul(b.load(a.stride_bsn_ptr))) + else + b.splat(expert_offset, &.{block_size_n}); + const scales = b.loadOpts(a.b_scale_ptr.addPtr(offsets), .{ + .mask = offs_bn.lt(n_block), + .other = b.zeros(&.{block_size_n}, cfg.b_scale_dtype.?), + }).to(.f32); + accumulator = accumulator.mul(scales.expandDims(0)); + if (cfg.a_scale_dtype) |dtype| { + const offsets_a = offs_token.div(top_k).mul(b.load(a.stride_asm_ptr)); + const scales_a = b.loadOpts(a.a_scale_ptr.addPtr(offsets_a), .{ + .mask = token_mask, + .other = b.zeros(&.{block_size_m}, dtype), + }).to(.f32); + accumulator = accumulator.mul(scales_a.expandDims(1)); + } + } + + if (cfg.b_bias_dtype) |dtype| { + const offsets = off_experts.mul(b.load(a.stride_bbe_ptr)) + .add(offs_bn.mul(b.load(a.stride_bbn_ptr))); + const bias = b.loadOpts(a.b_bias_ptr.addPtr(offsets), .{ + .mask = offs_bn.lt(n_block), + .other = b.zeros(&.{block_size_n}, dtype), + }).to(.f32); + // Bias is in output units and must receive the routing weight too. + accumulator = accumulator.add(bias.expandDims(0)); + } + + if (cfg.routing_weights_dtype) |dtype| { + const other = b.zeros(&.{block_size_m}, dtype); + const routing_weights = b.loadOpts(a.routing_weights_ptr.addPtr(offs_token), .{ .mask = token_mask, .other = other }); + accumulator = accumulator.mul(routing_weights.to(.f32).expandDims(1)); + } + + accumulator = accumulator.to(compute_type); + + // c_ptrs: two addptrs to match Python's `c_ptr + cm[:, None] + cn[None, :]` + // left-to-right eval. `stride_cm * offs_token[:, None]` puts the splatted + // stride on the LHS of arith.muli (Python source order). + const offs_cn = pid_n_block_n.add(b.arange(0, block_size_n, .i64)); + const offs_token_col_c = b.expandDims(offs_token, 1); + const cm_col = stride_cm_block.mul(offs_token_col_c); + const c_ptrs_col = b.splat(a.c_ptr, &.{ block_size_m, 1 }).addPtr(cm_col); + const offs_cn_row = b.expandDims(offs_cn, 0); + const c_ptrs_2d = b.broadcastTo(c_ptrs_col, &.{ block_size_m, block_size_n }); + const c_ptrs = c_ptrs_2d.addPtr(b.broadcastTo(offs_cn_row, &.{ block_size_m, block_size_n })); + + // c_mask = token_mask[:, None] & (offs_cn[None, :] < n_block). + const token_mask_col_c = b.expandDims(token_mask, 1); + const offs_cn_lt_2d = offs_cn_row.lt(b.splat(n_block, &.{ 1, block_size_n })); + const token_mask_full_c = b.broadcastTo(token_mask_col_c, &.{ block_size_m, block_size_n }); + const offs_cn_lt_full = b.broadcastTo(offs_cn_lt_2d, &.{ block_size_m, block_size_n }); + b.storeOpts(c_ptrs, accumulator, .{ .mask = token_mask_full_c.bitAnd(offs_cn_lt_full) }); + } +}; + +test "FP8 activation quantization emits float and E8M0 scales" { + for ([_]zml.Quantization.Scheme{ .mxfp8, .fp8_per_channel, .fp8_per_tensor, .fp8_block128, .fp8_block32 }) |scheme| { + for ([_]DType{ .bf16, .f32 }) |input_dtype| { + const ir = try PerTokenGroupQuantFp8.Kernel.emit(std.testing.allocator, .{ + .input_dtype = input_dtype, + .output_dtype = .f8e4m3fn, + .scale_dtype = if (scheme == .mxfp8) .i8 else .f32, + .block = if (scheme == .mxfp8 or scheme == .fp8_block32) 32 else 256, + .quant_scheme = scheme, + }); + defer std.testing.allocator.free(ir); + if (scheme == .mxfp8) { + try std.testing.expect(std.mem.indexOf(u8, ir, "arith.trunci") != null); + } + } + } +} + +test "FusedMoe emits each FP8 and MXFP4 scaling path" { + const allocator = std.testing.allocator; + const schemes = [_]?zml.Quantization.Scheme{ null, .mxfp4, .mxfp8, .fp8_per_tensor, .fp8_per_channel, .fp8_block128, .fp8_block32 }; + for (schemes) |scheme| { + for ([_]DType{ .f8e4m3fn, .f8e4m3fnuz }) |fp8_dtype| { + if ((scheme == .mxfp4 or scheme == .mxfp8) and fp8_dtype == .f8e4m3fnuz) continue; + for ([_]bool{ false, true }) |quantize_input| { + if ((scheme == null or scheme == .mxfp4) and quantize_input) continue; + for ([_]?DType{ null, .bf16, .f32 }) |bias_dtype| { + const ir = try FusedMoe.Kernel.emit(allocator, .{ + .a_dtype = if (quantize_input) fp8_dtype else .bf16, + .b_dtype = if (scheme == .mxfp4) .i8 else if (scheme != null) fp8_dtype else .bf16, + .c_dtype = .bf16, + .a_scale_dtype = if (quantize_input) (if (scheme == .mxfp8) .i8 else .f32) else null, + .b_scale_dtype = if (scheme == .mxfp4 or scheme == .mxfp8) .i8 else if (scheme != null) .f32 else null, + .b_bias_dtype = bias_dtype, + .routing_weights_dtype = .bf16, + .block_size_m = 16, + .block_size_n = 32, + .block_size_k = if (scheme == .fp8_block32) 32 else 128, + .group_size_m = 1, + .top_k = 2, + .naive_block_assignment = false, + .compute_type = .bf16, + .quant_scheme = scheme, + }); + defer allocator.free(ir); + if (scheme == .mxfp4 or scheme == .mxfp8) { + try std.testing.expect(std.mem.indexOf(u8, ir, "tt.dot_scaled") != null); + } else { + try std.testing.expect(std.mem.indexOf(u8, ir, "tt.dot_scaled") == null); + try std.testing.expect(std.mem.indexOf(u8, ir, "tt.dot") != null); + if (scheme != null and !quantize_input) { + try std.testing.expect(std.mem.indexOf(u8, ir, "tt.fp_to_fp") != null); + } + } + } + } + } + } +} + +// ============================================================================= +// moe_align_block_size_kernel +// ============================================================================= + +/// Direct port of Python `moe_align_block_size_kernel`. pid==0 runs the +/// histogram + cumsum + block-to-expert assignment pass; pid==1 fills +/// sorted_token_ids with NUMEL. +/// +/// Block assignment are set up to `num_tokens_post_padded` and the remaining blocks of +/// `expert_ids` are not set to a sentinel value. +pub const MoeAlignBlockSize = struct { + pub const Cfg = struct { + numel: usize, + num_experts: usize, + padded_num_experts: usize, + max_num_tokens_padded: usize, + max_num_m_blocks: usize, + block_size_m: usize, + hist_block: usize, + }; + pub const Kernel = tri.Kernel(Cfg, .{ + .name = "moe_align_block_size_kernel", + .inputs = &.{ "topk_ids_ptr", "sorted_token_ids_ptr", "expert_ids_ptr", "num_tokens_post_pad_ptr", "cumsum_ptr" }, + .outputs = &.{ "sorted_token_ids", "expert_ids", "num_tokens_post_pad", "cumsum" }, + .run = run, + }); + fn run(b: *Builder, cfg: Cfg) tri.FinishError!void { + const a = try b.declareArgs(.{ + .topk_ids_ptr = .{ .ptr = .i32 }, + .sorted_token_ids_ptr = .{ .ptr = .i32 }, + .expert_ids_ptr = .{ .ptr = .i32 }, + .num_tokens_post_pad_ptr = .{ .ptr = .i32 }, + .cumsum_ptr = .{ .ptr = .i32 }, + .out0_ptr = .{ .ptr = .i32 }, + .out1_ptr = .{ .ptr = .i32 }, + .out2_ptr = .{ .ptr = .i32 }, + .out3_ptr = .{ .ptr = .i32 }, + }); + + const block_size_m: i32 = @intCast(cfg.block_size_m); + const numel: i32 = @intCast(cfg.numel); + const num_experts: i32 = @intCast(cfg.num_experts); + const padded_num_experts: i64 = @intCast(cfg.padded_num_experts); + const max_num_tokens_padded: i32 = @intCast(cfg.max_num_tokens_padded); + const hist_block: i64 = @intCast(cfg.hist_block); + + const pid = b.programId(.x); + const fill_offs = b.arange(0, hist_block, .i32); + + // if pid == 1: fill sorted_token_ids with NUMEL; return + var fill_branch = b.openReturnIfOpts(pid.eq(1), .{ .inline_return = true }); + { + var fill_loop = b.openFor(0, max_num_tokens_padded, hist_block, .{}); + { + const offs = fill_loop.iv.add(fill_offs); + const mask = offs.lt(max_num_tokens_padded); + b.storeOpts( + a.sorted_token_ids_ptr.addPtr(offs), + b.splat(numel, &.{hist_block}), + .{ .mask = mask }, + ); + fill_loop.close(); + } + fill_branch.close(); + } + + // pid != 1: histogram + cumsum + block assignment. + const expert_offs = b.arange(0, padded_num_experts, .i32); + const token_offs = b.arange(0, hist_block, .i32); + const expert_mask = expert_offs.lt(num_experts); + const counts_init = b.zeros(&.{padded_num_experts}, .i32); + + var hist_loop = b.openFor(0, numel, hist_block, .{counts_init}); + { + const offs = hist_loop.iv.add(token_offs); + const mask = offs.lt(numel); + const expert_vals = b.loadOpts(a.topk_ids_ptr.addPtr(offs), .{ + .mask = mask, + .other = b.splat(num_experts, &.{hist_block}), + }); + const valid = mask.bitAnd(expert_vals.lt(num_experts)); + const h = b.histogramOpts(expert_vals, padded_num_experts, .{ .mask = valid }); + hist_loop.yield(.{hist_loop.carried[0].add(h)}); + } + const counts = hist_loop.results[0]; + + // padded_counts = where(expert_mask, cdiv(counts, BLOCK_SIZE_M) * BLOCK_SIZE_M, 0) + const padded_counts = b.select( + expert_mask, + counts.cdiv(block_size_m).mul(block_size_m), + b.zeros(&.{padded_num_experts}, .i32), + ); + const padded_cumsum = b.cumsumOpts(padded_counts, .{ .axis = 0 }); + const starts = padded_cumsum.sub(padded_counts); + const total_tokens_post_pad = b.sumOpts(padded_counts, .{ .axis = 0 }); + + b.storeOpts(a.cumsum_ptr.addPtr(expert_offs), starts, .{ .mask = expert_mask }); + b.store(a.cumsum_ptr.addPtr(num_experts), total_tokens_post_pad); + b.store(a.num_tokens_post_pad_ptr, total_tokens_post_pad); + + // Each expert owns a disjoint padded range, including empty ranges. + // Scatter from register-resident prefixes; no cross-lane global reads. + const block_counts = padded_counts.div(block_size_m); + const max_blocks = b.maxOpts(block_counts, .{ .axis = 0 }); + var scatter = b.openFor(0, max_blocks, 1, .{}); + { + const block_ids = starts.div(block_size_m).add(scatter.iv); + const mask = expert_mask.bitAnd(block_counts.gt(scatter.iv)); + b.storeOpts(a.expert_ids_ptr.addPtr(block_ids), expert_offs, .{ .mask = mask }); + scatter.close(); + } + } +}; + +// ============================================================================= +// count_and_sort_expert_tokens_kernel +// ============================================================================= + +/// Direct port of Python `count_and_sort_expert_tokens_kernel`. Each program +/// atomically bumps `cumsum[expert]`, writing its block's token offsets into +/// `sorted_token_ids` at the returned rank. +pub const CountAndSortExpertTokens = struct { + pub const Cfg = struct { + numel: usize, + num_experts: usize, + sort_block_size: usize, + }; + pub const Kernel = tri.Kernel(Cfg, .{ + .name = "count_and_sort_expert_tokens_kernel", + .inputs = &.{ "topk_ids_ptr", "sorted_token_ids_ptr", "cumsum_ptr" }, + .outputs = &.{ "sorted_token_ids", "cumsum" }, + .run = run, + }); + fn run(b: *Builder, cfg: Cfg) tri.FinishError!void { + const a = try b.declareArgs(.{ + .topk_ids_ptr = .{ .ptr = .i32 }, + .sorted_token_ids_ptr = .{ .ptr = .i32 }, + .cumsum_ptr = .{ .ptr = .i32 }, + .out0_ptr = .{ .ptr = .i32 }, + .out1_ptr = .{ .ptr = .i32 }, + }); + + const block: i64 = @intCast(cfg.sort_block_size); + const numel: i32 = @intCast(cfg.numel); + const num_experts: i32 = @intCast(cfg.num_experts); + const block_i32: i32 = @intCast(block); + + const pid = b.programId(.x); + const num_progs = b.numPrograms(.x); + const token_offs = b.makeRange(0, block); + + const token_start_init = pid.mul(block_i32); + + // while token_start < NUMEL: + var w = b.openWhile(.{token_start_init}, .{.i32}); + { + const ts = w.before_carried[0]; + w.yieldBefore(ts.lt(numel), .{ts}); + } + { + const ts = w.after_carried[0]; + + // offs = ts + token_offs; mask = offs < NUMEL + const offs = ts.add(token_offs); + const mask = offs.lt(numel); + + // expert_vals = load(topk_ids + offs, mask=mask, other=NUM_EXPERTS) + const topk_ptrs = a.topk_ids_ptr.addPtr(offs); + const expert_vals = b.loadOpts(topk_ptrs, .{ + .mask = mask, + .other = b.splat(num_experts, &.{block}), + }); + const valid = mask.bitAnd(expert_vals.lt(num_experts)); + + // rank = atomic_add(cumsum + expert_vals, 1, mask=valid, sem="relaxed") + const cumsum_ptrs = a.cumsum_ptr.addPtr(expert_vals); + const rank = b.atomicRmwOpts(.add, cumsum_ptrs, b.ones(&.{block}, .i32), .{ + .mask = valid, + .sem = .relaxed, + .scope = .gpu, + }); + + // store(sorted_token_ids + rank, offs, mask=valid) + const sorted_ptrs = a.sorted_token_ids_ptr.addPtr(rank); + b.storeOpts(sorted_ptrs, offs, .{ .mask = valid }); + + // token_start += num_programs * BLOCK_SIZE + // Compute step INSIDE the loop body (Python's source position). + // It's loop-invariant, so XLA's LICM hoists it out — the hoisted + // op-order then matches Python's IR (splats before step). + const step = num_progs.mul(block_i32); + w.yieldAfter(.{ts.add(step)}); + } + } +}; + +/// Returns the next power of two. +fn pow2(x: i64) i64 { + return @intCast(std.math.ceilPowerOfTwo(u64, @intCast(x)) catch unreachable); +} + +pub fn quantRows(tokens: i64) i64 { + return if (tokens >= 16) 16 else 1; +} + +fn ar(b: *Builder, n: i64) Value { + return b.arange(0, n, .i32); +} + +fn ld(b: *Builder, p: Value, mask: Value, dt: tri.DType, other: i64) Value { + return b.loadOpts(p, .{ .mask = mask, .other = if (p.isTensor()) b.full(p.shape().constSlice(), other, dt) else b.constant(dt, other) }); +} + +fn ci(b: *Builder, n: i64) Value { + return b.constant(.i32, n); +} + +fn quant32(b: *Builder, x: Value) struct { q: Value, s: Value } { + const rows = x.shape().constSlice()[0]; + const cols = x.shape().constSlice()[1]; + const groups = @divExact(cols, 32); + const blocks = b.reshape(x, &.{ rows * groups, 32 }); + const sf = b.exp2(b.ceil(b.log2(b.maxOpts(b.absf(blocks), .{ .axis = 1 }).maximum(1e-4).mul(1.0 / 448.0)))); + const scaled = blocks.div(sf.expandDims(1)).maximum(-448.0).minimum(448.0); + // TODO(Corentin): We should gate that behind a platform check so that we don't do useless stuff on non-SM89 platforms. + // Find a way to add that to the builder, maybe a `b.platform()` or something like that. + const q = b: { + // SM89 lowers FP32 -> FP8 through FP16 with truncation. Round to odd at + // FP16 precision first so that this intermediate cannot turn a value just + // above an FP8 midpoint into an exact tie. Direct FP8 conversion is unchanged. + const scaled_bits = scaled.bitCast(.i32); + const sticky = b.select(scaled_bits.bitAnd(0x1fff).ne(0), b.full(scaled.shape().constSlice(), 0x2000, .i32), b.zeros(scaled.shape().constSlice(), .i32)); + const prepared = scaled_bits.bitAnd(-0x2000).bitOr(sticky).bitCast(.f32); + break :b prepared.to(.f8e4m3fn); + }; + const bits = b.shrui(sf.bitCast(.i32), b.full(sf.shape().constSlice(), 23, .i32)).bitAnd(255).to(.i8); + return .{ .q = b.reshape(q, &.{ rows, cols }), .s = b.reshape(bits, &.{ rows, groups }) }; +} + +pub fn combineBlock(tokens: i64, hidden: i64) i64 { + // Keep at least 512 CTAs before increasing per-CTA reduction work. + const elements = tokens * hidden; + if (tokens > 1 and elements >= 512 * 2048) return 2048; + if (tokens > 1 and elements >= 512 * 1024) return 1024; + return 256; +} + +/// Computes an upper bound on the number of expert tiles needed. +/// When there is only one token, reserve a single tile per selected expert. +pub fn tiles(tokens: i64, topk: i64, experts: i64, bn: i64) i64 { + if (tokens == 1) return topk; + return @min(tokens * topk, @divTrunc(tokens * topk + experts * (bn - 1) + bn - 1, bn)); +} + +// ============================================================================= +// mxfp4_triton_quant +// ============================================================================= + +pub const Quant = struct { + pub const Cfg = struct { + tokens: i64, + hidden: i64, + }; + + pub const Kernel = tri.Kernel(Cfg, .{ .name = "mxfp4_triton_quant", .inputs = &.{"x"}, .outputs = &.{ "q", "s" }, .run = run }); + + fn run(b: *Builder, c: Cfg) tri.FinishError!void { + const a = try b.declareArgs(.{ .x = .{ .ptr = .bf16 }, .q = .{ .ptr = .f8e4m3fn }, .s = .{ .ptr = .i8 } }); + const rows = b.programId(.x).mul(quantRows(c.tokens)).add(ar(b, quantRows(c.tokens))); + const block = b.programId(.y); + const cols = block.mul(128).add(ar(b, 128)); + const offsets = rows.expandDims(1).mul(c.hidden).add(cols.expandDims(0)); + const mask = rows.lt(c.tokens).expandDims(1); + const x = ld(b, a.x.addPtr(offsets), mask, .bf16, 0).to(.f32); + const q = quant32(b, x); + b.storeOpts(a.q.addPtr(offsets), q.q, .{ .mask = mask }); + const scale_offsets = rows.expandDims(1).mul(@divTrunc(c.hidden, 32)).add(block.mul(4).add(ar(b, 4)).expandDims(0)); + b.storeOpts(a.s.addPtr(scale_offsets), q.s, .{ .mask = mask }); + } +}; + +// ============================================================================= +// mxfp4_triton_route +// ============================================================================= + +pub const Route = struct { + pub const Cfg = struct { + tokens: i64, + topk: i64, + expert_offset: i64, + }; + + pub const Kernel = tri.Kernel(Cfg, .{ .name = "mxfp4_triton_route", .inputs = &.{"ids"}, .outputs = &.{ "counts", "pos" }, .run = run }); + + fn run(b: *Builder, c: Cfg) tri.FinishError!void { + const a = try b.declareArgs(.{ .ids = .{ .ptr = .i32 }, .counts = .{ .ptr = .i32 }, .pos = .{ .ptr = .i32 } }); + const e = b.programId(.x); + const p = ar(b, pow2(c.tokens * c.topk)); + const ids = ld(b, a.ids.addPtr(p), p.lt(c.tokens * c.topk), .i32, -1); + const valid = p.lt(c.tokens * c.topk).bitAnd(ids.eq(e.add(c.expert_offset))); + b.storeOpts(a.pos.addPtr(p), valid.to(.i32).cumsum().sub(1), .{ .mask = valid }); + b.store(a.counts.addPtr(e), valid.to(.i32).sum()); + } +}; + +// ============================================================================= +// mxfp4_triton_route_atomic +// ============================================================================= + +pub const RouteAtomic = struct { + pub const route_block = 1024; + + pub const Cfg = struct { + tokens: i64, + topk: i64, + expert_offset: i64, + experts: i64, + }; + + pub const Kernel = tri.Kernel(Cfg, .{ .name = "mxfp4_triton_route_atomic", .inputs = &.{ "ids", "zeros" }, .outputs = &.{ "counts", "pos" }, .run = run }); + + /// Routing for large batches, used by `scheduleForCute`. + /// Each route takes the next free slot of its expert with an atomic + /// increment of `counts` (zero-initialized through `zeros`). Slots within an + /// expert are therefore unordered, which the grouped GEMMs do not care + /// about: every routed row is computed independently. + fn run(b: *Builder, c: Cfg) tri.FinishError!void { + const a = try b.declareArgs(.{ .ids = .{ .ptr = .i32 }, .zeros = .{ .ptr = .i32 }, .counts = .{ .ptr = .i32 }, .pos = .{ .ptr = .i32 } }); + const p = b.programId(.x).mul(route_block).add(ar(b, route_block)); + const in_range = p.lt(c.tokens * c.topk); + const e = ld(b, a.ids.addPtr(p), in_range, .i32, -1).sub(ci(b, c.expert_offset)); + const valid = in_range.bitAnd(e.ge(0)).bitAnd(e.lt(c.experts)); + const slot = b.atomicRmwOpts(.add, a.counts.addPtr(e.maximum(0)), b.full(&.{route_block}, 1, .i32), .{ .mask = valid, .sem = .relaxed }); + b.storeOpts(a.pos.addPtr(p), slot, .{ .mask = valid }); + } +}; + +// ============================================================================= +// mxfp4_triton_schedule +// ============================================================================= + +pub const Schedule = struct { + pub const Cfg = struct { + tokens: i64, + topk: i64, + expert_offset: i64, + experts: i64, + /// Expert groups the schedule may fill. + capacity: i64, + /// Routed rows per expert group, i.e. the consuming GEMM's tile N. + group: i64, + }; + + pub const Kernel = tri.Kernel(Cfg, .{ .name = "mxfp4_triton_schedule", .inputs = &.{ "ids", "counts", "pos" }, .outputs = &.{ "map", "perm", "sched" }, .run = run }); + + fn run(b: *Builder, cfg: Cfg) tri.FinishError!void { + const nt = cfg.capacity; + // Route positions are i32 tensors; keep the group size in that type. + const group: i32 = @intCast(cfg.group); + const a = try b.declareArgs(.{ .ids = .{ .ptr = .i32 }, .counts = .{ .ptr = .i32 }, .pos = .{ .ptr = .i32 }, .map = .{ .ptr = .i32 }, .perm = .{ .ptr = .i32 }, .sched = .{ .ptr = .i32 } }); + const p = b.programId(.x).mul(128).add(ar(b, 128)); + const e = ld(b, a.ids.addPtr(p), p.lt(cfg.tokens * cfg.topk), .i32, -1).sub(ci(b, cfg.expert_offset)); + const es = ar(b, pow2(cfg.experts)); + const counts = ld(b, a.counts.addPtr(es), es.lt(cfg.experts), .i32, 0); + const ts = counts.cdiv(group); + const starts = ts.cumsum().sub(ts); + var first = b.openIf(b.programId(.x).eq(0)); + { + b.store(a.sched.addPtr(2 * nt), ts.sum()); + first.close(); + } + const live = p.lt(cfg.tokens * cfg.topk).bitAnd(e.ge(0)).bitAnd(e.lt(cfg.experts)); + const safe = e.maximum(0).minimum(pow2(cfg.experts) - 1); + const pos = ld(b, a.pos.addPtr(p), live, .i32, 0); + const start = starts.gather(safe, 0); + const tile = start.add(pos.div(group)); + const row = pos.rem(group); + b.storeOpts(a.perm.addPtr(p), b.select(live, tile.mul(group).add(row), b.full(&.{128}, -1, .i32)), .{ .mask = p.lt(cfg.tokens * cfg.topk) }); + b.storeOpts(a.map.addPtr(tile.mul(group).add(row)), p, .{ .mask = live }); + b.storeOpts(a.sched.addPtr(tile), e, .{ .mask = live.bitAnd(row.eq(0)) }); + b.storeOpts(a.sched.addPtr(nt).addPtr(tile), counts.gather(safe, 0).sub(pos).minimum(group), .{ .mask = live.bitAnd(row.eq(0)) }); + } +}; + +// ============================================================================= +// mxfp4_triton_combine +// ============================================================================= + +pub const Combine = struct { + pub const Cfg = struct { + tokens: i64, + hidden: i64, + topk: i64, + expert_offset: i64, + experts: i64, + routing_weight_placement: RoutingWeightPlacement = .before_down, + bn: i64, + }; + pub const Kernel = tri.Kernel(Cfg, .{ .name = "mxfp4_triton_combine", .inputs = &.{ "d", "ids", "perm", "rw" }, .outputs = &.{"y"}, .run = run }); + + fn run(b: *Builder, c: Cfg) tri.FinishError!void { + return combineWithBn(b, c); + } + + fn combineWithBn(b: *Builder, c: Cfg) tri.FinishError!void { + const a = try b.declareArgs(.{ .d = .{ .ptr = .bf16 }, .ids = .{ .ptr = .i32 }, .perm = .{ .ptr = .i32 }, .rw = .{ .ptr = .f32 }, .y = .{ .ptr = .bf16 } }); + const t = b.programId(.x); + const block = combineBlock(c.tokens, c.hidden); + const h = b.programId(.y).mul(block).add(ar(b, block)); + var acc = b.zeros(&.{block}, .f32); + for (0..@intCast(c.topk)) |s| { + const pair = t.mul(c.topk).add(@as(i64, @intCast(s))); + const row = if (c.tokens == 1) pair.mul(c.bn) else b.load(a.perm.addPtr(pair)); + const valid = if (c.tokens == 1) v: { + const e = b.load(a.ids.addPtr(pair)).sub(ci(b, c.expert_offset)); + break :v e.ge(0).bitAnd(e.lt(c.experts)); + } else row.ge(0); + var value = ld(b, a.d.addPtr(row.mul(c.hidden).add(h)), valid.bitAnd(h.lt(c.hidden)), .bf16, 0).to(.f32); + if (c.routing_weight_placement == .after_down) { + value = value.mul(ld(b, a.rw.addPtr(pair), valid, .f32, 0)); + } + acc = acc.add(value); + } + b.storeOpts(a.y.addPtr(t.mul(c.hidden).add(h)), acc.to(.bf16), .{ .mask = h.lt(c.hidden) }); + } +}; + +// ============================================================================= +// mxfp4 gemm +// ============================================================================= + +pub const GateUp = struct { + pub const Cfg = GemmConfig; + + pub const Kernel = tri.Kernel(Cfg, .{ .name = "mxfp4_triton_gate_up", .inputs = &.{ "q", "s", "w", "ws", "ids", "rw", "map", "sched" }, .outputs = &.{ "mid", "ms" }, .run = run }); + + fn run(b: *Builder, c: Cfg) tri.FinishError!void { + return gemm(b, c, .gate_up); + } +}; + +pub const Down = struct { + pub const Cfg = GemmConfig; + + pub const Kernel = tri.Kernel(Cfg, .{ .name = "mxfp4_triton_down", .inputs = &.{ "mid", "ms", "w", "ws", "ids", "sched" }, .outputs = &.{"d"}, .run = run }); + + fn run(b: *Builder, c: Cfg) tri.FinishError!void { + return gemm(b, c, .down); + } +}; + +const GemmConfig = struct { + tokens: i64, + hidden: i64, + intermediate: i64, + experts: i64, + topk: i64, + expert_offset: i64 = 0, + // Swiglu clamping. Set to 0 to disable. + swiglu_limit: f32, + scale_stride: i64, + routing_weight_placement: RoutingWeightPlacement = .before_down, + bn: i64, + bk: i64, +}; + +const Projection = enum { gate_up, down }; + +fn gemm(b: *Builder, c: GemmConfig, comptime projection: Projection) tri.FinishError!void { + const is_up = projection == .gate_up; + const a = if (is_up) try b.declareArgs(.{ .q = .{ .ptr = .f8e4m3fn }, .s = .{ .ptr = .i8 }, .w = .{ .ptr = .i8 }, .ws = .{ .ptr = .i8 }, .ids = .{ .ptr = .i32 }, .rw = .{ .ptr = .f32 }, .map = .{ .ptr = .i32 }, .sched = .{ .ptr = .i32 }, .mid = .{ .ptr = .f8e4m3fn }, .ms = .{ .ptr = .i8 } }) else try b.declareArgs(.{ .q = .{ .ptr = .f8e4m3fn }, .s = .{ .ptr = .i8 }, .w = .{ .ptr = .i8 }, .ws = .{ .ptr = .i8 }, .ids = .{ .ptr = .i32 }, .sched = .{ .ptr = .i32 }, .d = .{ .ptr = .bf16 } }); + const block = b.programId(.x); + const tile = b.programId(.y); + const n = ar(b, c.bn); + const nt = tiles(c.tokens, c.topk, c.experts, c.bn); + const expert = if (c.tokens == 1) b.load(a.ids.addPtr(tile)).sub(ci(b, c.expert_offset)) else ld(b, a.sched.addPtr(tile), tile.lt(b.load(a.sched.addPtr(2 * nt))), .i32, 0); + const valid = if (c.tokens == 1) expert.ge(0).bitAnd(expert.lt(c.experts)) else tile.lt(b.load(a.sched.addPtr(2 * nt))); + var live = b.openIf(valid); + { + const count = if (c.tokens == 1) ci(b, 1) else b.load(a.sched.addPtr(nt).addPtr(tile)); + const pair = if (c.tokens == 1) b.full(&.{c.bn}, 0, .i32).add(tile) else if (is_up) ld(b, a.map.addPtr(tile.mul(c.bn).add(n)), n.lt(count), .i32, 0) else n; + const width: i64 = if (is_up and c.tokens > 8) 256 else 128; + const ksize = if (is_up) c.hidden else c.intermediate; + const rows = if (is_up) 2 * c.intermediate else c.hidden; + const m = block.mul(width).add(ar(b, width)); + const k = ar(b, c.bk); + const ks = ar(b, @divExact(c.bk, 32)); + const row = if (is_up) pair.div(c.topk) else tile.mul(c.bn).add(n); + const wd = b.makeTensorDescriptor(a.w.addPtr(expert.to(.i64).mul(@divTrunc(rows * ksize, 2))), &.{ ci(b, rows), ci(b, @divTrunc(ksize, 2)) }, &.{ b.constant(.i64, @divTrunc(ksize, 2)), b.constant(.i64, 1) }, .pad_zero, &.{ width, @divExact(c.bk, 2) }, .i8); + var loop = b.openFor(0, @divTrunc(ksize + c.bk - 1, c.bk), 1, .{b.zeros(&.{ width, c.bn }, .f32)}); + { + const sk = loop.iv.mul(@divExact(c.bk, 32)).add(ks); + const kk = loop.iv.mul(c.bk).add(k); + const w = b.descriptorLoad(wd, &.{ block.mul(ci(b, width)), loop.iv.mul(ci(b, @divExact(c.bk, 2))) }, &.{ width, @divExact(c.bk, 2) }, .i8); + const scale_offset = m.expandDims(1).mul(c.scale_stride).add(sk.expandDims(0)); + const ws = ld(b, a.ws.addPtr(expert.to(.i64).mul(rows * c.scale_stride)).addPtr(scale_offset), sk.expandDims(0).lt(@divTrunc(ksize, 32)), .i8, 127); + var mask = kk.expandDims(0).lt(ksize); + var smask = sk.expandDims(0).lt(@divTrunc(ksize, 32)); + // The small-batch epilogue leaves padded rows unwritten. + if (is_up or c.tokens <= 8) { + mask = mask.bitAnd(n.lt(count).expandDims(1)); + smask = smask.bitAnd(n.lt(count).expandDims(1)); + } + const x = ld(b, a.q.addPtr(row.expandDims(1).mul(ksize).add(kk.expandDims(0))), mask, .f8e4m3fn, 0); + const xs = ld(b, a.s.addPtr(row.expandDims(1).mul(@divTrunc(ksize, 32)).add(sk.expandDims(0))), smask, .i8, 127); + loop.yield(.{b.dotScaled(w, b.permute(x, &.{ 1, 0 }), loop.carried[0], ws, xs, .e2m1, .e4m3)}); + } + const epilogue_n: i64 = if (c.tokens == 1) 1 else c.bn; + const out_n = ar(b, epilogue_n); + const acc = if (c.tokens == 1) live_row: { + // The remaining MMA columns are padding during single-token decode. + // Extract column zero before activation/reduction and output stores. + var v = loop.results[0]; + var cols: i64 = c.bn; + while (cols > 1) : (cols = @divTrunc(cols, 2)) { + v = b.split(b.reshape(v, &.{ width, @divTrunc(cols, 2), 2 }))[0]; + } + break :live_row b.reshape(v, &.{ 1, width }).to(.bf16); + } else b.permute(loop.results[0], &.{ 1, 0 }).to(.bf16); + if (is_up) { + const columns = @divExact(width, 2); + const halves = b.split(b.reshape(acc.to(.f32), &.{ epilogue_n, columns, 2 })); + var g = halves[0]; + var u = halves[1]; + if (c.swiglu_limit > 0) { + g = g.minimum(c.swiglu_limit); + u = u.minimum(c.swiglu_limit).maximum(-c.swiglu_limit); + } + const ex = b.externElementwise(&.{g.mul(-1.0)}, &.{ epilogue_n, columns }, .f32, "libdevice", "", "__nv_expf"); + var mid = u.mul(b.divRn(g, ex.add(1.0))); + if (c.routing_weight_placement == .before_down) { + const rw = if (c.tokens == 1) b.load(a.rw.addPtr(tile)) else ld(b, a.rw.addPtr(pair), n.lt(count), .f32, 0).expandDims(1); + mid = mid.mul(rw); + } + // Compute scales and quantize directly from the FP32 SwiGLU result. + const q = quant32(b, mid); + const col = block.mul(columns).add(ar(b, columns)); + b.store(a.mid.addPtr(tile.mul(c.bn).add(out_n).expandDims(1).mul(c.intermediate).add(col.expandDims(0))), q.q); + const scale_cols = block.mul(@divExact(columns, 32)).add(ar(b, @divExact(columns, 32))); + b.store(a.ms.addPtr(tile.mul(c.bn).add(out_n).expandDims(1).mul(@divTrunc(c.intermediate, 32)).add(scale_cols.expandDims(0))), q.s); + } else b.store(a.d.addPtr(tile.mul(c.bn).add(out_n).expandDims(1).mul(c.hidden).add(m.expandDims(0))), acc); + live.close(); + } +} diff --git a/zml/kernels/moe/triton/triton.zig b/zml/kernels/moe/triton/triton.zig new file mode 100644 index 00000000..8ba799a5 --- /dev/null +++ b/zml/kernels/moe/triton/triton.zig @@ -0,0 +1,1072 @@ +const std = @import("std"); + +const stdx = @import("stdx"); + +const zml = @import("../../../zml.zig"); +const Tensor = zml.Tensor; +const Shape = zml.Shape; +const DataType = zml.DataType; +const toDType = zml.kernel.triton.from; +const ExpertsLayout = @import("../api.zig").ExpertsLayout; +const ProjectionLayout = @import("../api.zig").ProjectionLayout; +const RoutingWeightPlacement = @import("../api.zig").RoutingWeightPlacement; +const Activation = @import("../api.zig").Activation; +const Options = @import("../api.zig").Options; +const LocalRouting = @import("../api.zig").LocalRouting; +const kernels = @import("moe.zig"); + +const log = std.log.scoped(.@"zml/moe/triton"); + +fn numStages(config: LaunchConfig) i32 { + const platform = zml.Compiler.current().platform; + if (platform.target != .rocm) return config.num_stages; + + // ROCm buffers num_stages - 1 BF16 operand pairs in LDS. A single stage + // reuses scratch space between operand layout conversions. + const bytes_per_stage: i64 = @intCast(2 * config.block_size_k * (config.block_size_m + config.block_size_n)); + var stages = config.num_stages; + for (platform.devices) |device| { + const attribute = device.pjrt_desc.attribute(platform.pjrt_api, "shared_memory_per_block_optin"); + const limit = if (attribute) |value| value.int64 else 64 * 1024; + stages = @intCast(@min(stages, @divTrunc(limit, bytes_per_stage) + 1)); + } + return stages; +} + +test { + std.testing.refAllDecls(@This()); +} + +pub fn fusedExperts( + input: zml.Tensor, + routing: LocalRouting, + gate_up: zml.nn.Linear, + down: zml.nn.Linear, + options: Options, +) zml.Tensor { + // HACK(Corentin): only rely on the scheme of gate_up for now. + + if (gate_up.quantization) |q| switch (q.scheme) { + .mxfp4 => if (zml.Compiler.current().platform.target == .cuda) { + return fusedExpertsMxfp4(input, routing, gate_up, down, options); + }, + else => {}, + }; + + const layout: ProjectionLayout = if (gate_up.quantizationScheme() == .mxfp4) .row_interleaved_1 else .concatenated; + return fusedExpertsBase(input, routing, gate_up, down, layout, options); +} + +pub fn fusedExpertsBase( + hidden_states: Tensor, + routing_: LocalRouting, + gate_up_: zml.nn.Linear, + down_: zml.nn.Linear, + layout: ProjectionLayout, + opts: Options, +) Tensor { + const expert_map_ = routing_.expertMap(); + const num_tokens = hidden_states.dim(.t); + var launch_config = launchConfigForTokens(num_tokens); + + const gate_up_scheme = gate_up_.quantizationScheme(); + const down_scheme = down_.quantizationScheme(); + if (gate_up_scheme) |scheme| switch (scheme) { + .nvfp4 => stdx.debug.panic("triton backend doesn't support gate_up nvfp4 quantization scheme", .{}), + .fp8_block128 => launch_config.block_size_k = 128, + .fp8_block32 => launch_config.block_size_k = 32, + .mxfp4, .mxfp8, .fp8_per_channel, .fp8_per_tensor => {}, + }; + + var down_launch_config = launchConfigForTokens(num_tokens); + if (down_scheme) |scheme| switch (scheme) { + .nvfp4 => stdx.debug.panic("triton backend doesn't support down nvfp4 quantization scheme", .{}), + .fp8_block128 => down_launch_config.block_size_k = 128, + .fp8_block32 => down_launch_config.block_size_k = 32, + .mxfp4, .mxfp8, .fp8_per_channel, .fp8_per_tensor => {}, + }; + + const hidden = hidden_states.reshape(.{ .token = num_tokens, .in = hidden_states.dim(.d) }).withTags(.{ .token, .in }); + const gate_up = gate_up_.weight.withTags(.{ .expert, .out, .in }); + const down = down_.weight.withTags(.{ .expert, .out, .mid }); + const routing_weights = routing_.weights.reshape(.{ .token = num_tokens, .in = routing_.weights.dim(.topk) }).withTags(.{ .token, .topk }); + const ids = routing_.ids.reshape(.{ .token = num_tokens, .in = routing_.ids.dim(.topk) }).withTags(.{ .token, .topk }); + + stdx.debug.assert(hidden.dtype() == .bf16, "triton backend expected BF16 hidden states, got {}", .{hidden.dtype()}); + if (gate_up_scheme == .mxfp4) { + stdx.debug.assert(gate_up.dtype() == .u8 or gate_up.dtype() == .i8 or gate_up.dtype() == .f4e2m1, "triton backend expected u8, i8 or f4e2m1 gate_up data type when quantization scheme is mxfp4, got {}", .{gate_up.dtype()}); + } else { + stdx.debug.assert(gate_up.dtype() == .bf16 or gate_up.dtype() == .f8e4m3fn or gate_up.dtype() == .f8e4m3fnuz, "triton backend expected gate_up data type to be bf16, fp8e4m3fn or f8e4m3fnuz, got {}", .{gate_up.dtype()}); + } + if (down_scheme == .mxfp4) { + stdx.debug.assert(down.dtype() == .u8 or down.dtype() == .i8 or down.dtype() == .f4e2m1, "triton backend expected u8, i8 or f4e2m1 down data type when quantization scheme is mxfp4, got {}", .{down.dtype()}); + } else { + stdx.debug.assert(down.dtype() == .bf16 or down.dtype() == .f8e4m3fn or down.dtype() == .f8e4m3fnuz, "triton backend expected down data type to be bf16, fp8e4m3fn or f8e4m3fnuz, got {}", .{down.dtype()}); + } + + stdx.debug.assert(routing_weights.dtype() == .f32 or routing_weights.dtype() == .bf16, "triton backend expected f32 or bf16 routing weights, got {}", .{routing_weights.dtype()}); + stdx.debug.assert(ids.dtype() == .i32, "triton backend expected i32 expert ids, got {}", .{ids.dtype()}); + const gate_up_k = gate_up.dim(.in) * @as(i64, if (gate_up_scheme == .mxfp4 and gate_up.dtype() != .f4e2m1) 2 else 1); + const down_k = down.dim(.mid) * @as(i64, if (down_scheme == .mxfp4 and down.dtype() != .f4e2m1) 2 else 1); + stdx.debug.assert(hidden.dim(.in) == gate_up_k, "triton backend expected hidden width to match gate/up input width, got {} and {}", .{ hidden.dim(.in), gate_up_k }); + const activation_reduction: i64 = switch (opts.activation) { + .gelu, .relu, .silu => 1, + .swiglu, .swiglu_step, .geglu, .geglu_tanh => 2, + }; + stdx.debug.assert(@rem(gate_up.dim(.out), activation_reduction) == 0, "triton backend expected gate/up output width to be divisible by {}, got {}", .{ activation_reduction, gate_up.dim(.out) }); + stdx.debug.assert(down_k == @divFloor(gate_up.dim(.out), activation_reduction), "triton backend expected down input width to match activated width, got {} and {}", .{ down_k, @divFloor(gate_up.dim(.out), activation_reduction) }); + stdx.debug.assert(ids.dim(.token) == hidden.dim(.token) and routing_weights.dim(.token) == hidden.dim(.token), "triton backend expected routing ids and weights to match hidden token count, got {}, {} and {}", .{ ids.dim(.token), routing_weights.dim(.token), hidden.dim(.token) }); + stdx.debug.assert(ids.dim(.topk) == routing_weights.dim(.topk), "triton backend expected routing ids and weights to have matching top-k dimensions, got {} and {}", .{ ids.dim(.topk), routing_weights.dim(.topk) }); + stdx.debug.assert(gate_up.dim(.expert) == down.dim(.expert), "triton backend expected gate/up and down expert counts to match, got {} and {}", .{ gate_up.dim(.expert), down.dim(.expert) }); + if (expert_map_) |expert_map| + stdx.debug.assert(expert_map.dtype() == .i32 and expert_map.rank() == 1, "triton backend expected a rank-1 i32 expert map, got rank {} and dtype {}", .{ expert_map.rank(), expert_map.dtype() }); + + const num_experts = if (expert_map_) |expert_map| expert_map.dim(.expert) else gate_up.dim(.expert); + const routing = prepareRouting(ids, num_experts, @intCast(launch_config.block_size_m)); + + const expert_ids = if (expert_map_) |expert_map| + expert_map.gather(.{ .expert = routing.expert_ids }, .{}).withTags(.{.g}) + else + routing.expert_ids; + + const hidden_quant, const input_scale = prepareInput(hidden, gate_up_scheme, gate_up.dtype(), opts.quantize_input); + + const gate_up_out = gemm(.{ + .input = hidden_quant, + .weight = gate_up, + .bias = gate_up_.bias, + .input_scale = input_scale, + .weight_scale = gate_up_.quantizationScales(), + .routing = routing, + .expert_ids = expert_ids, + .launch_config = launch_config, + .quant_scheme = gate_up_scheme, + .top_k = @intCast(ids.dim(.topk)), + .output_shape = Shape.init(.{ .token = routing.num_assignments, .out = gate_up.dim(.out) }, .bf16), + }); + + var activated = opts.activation.applyActivation(gate_up_out, layout); + if (opts.routing_weight_placement == .before_down) { + const weights = routing_weights.reshape(.{ .token = routing.num_assignments }).convert(.f32); + activated = activated.mul(weights.broad(activated.shape())); + } + const activated_quant, const down_input_scale = prepareInput(activated, down_scheme, down.dtype(), opts.quantize_input); + + const down_out = gemm(.{ + .input = activated_quant, + .weight = down, + .bias = down_.bias, + .input_scale = down_input_scale, + .weight_scale = down_.quantizationScales(), + .routing_weights = if (opts.routing_weight_placement == .after_down) routing_weights else null, + .routing = routing, + .expert_ids = expert_ids, + .launch_config = down_launch_config, + .quant_scheme = down_scheme, + .top_k = 1, + .output_shape = Shape.init(.{ .token = num_tokens, .topk = ids.dim(.topk), .out = down.dim(.out) }, .bf16), + }); + + const output = down_out.convert(.f32).sum(.topk).squeeze(.topk).convert(.bf16); + + return output.reshape(.{ .t = num_tokens, .d = down.dim(.out) }); +} + +pub const GemmOptions = struct { + input: Tensor, + weight: Tensor, + bias: ?Tensor = null, + input_scale: ?Tensor = null, + weight_scale: ?Tensor = null, + routing_weights: ?Tensor = null, + routing: Routing, + expert_ids: Tensor, + launch_config: LaunchConfig, + quant_scheme: ?zml.Quantization.Scheme = null, + top_k: usize, + output_shape: Shape, +}; + +pub fn gemm(opts: GemmOptions) Tensor { + // Native FP4 tensors expose logical K; the Triton operand is byte-packed. + const weight = if (opts.quant_scheme == .mxfp4 and opts.weight.dtype() == .f4e2m1) + opts.weight.reshape(opts.weight.shape().setDim(2, @divExact(opts.weight.dim(2), 2)).append(.{ .nibble = 2 })).bitCast(.u8) + else + opts.weight; + const weight_k = weight.dim(2) * @as(i64, if (opts.quant_scheme == .mxfp4) 2 else 1); + stdx.debug.assert(opts.input.dim(1) == weight_k, "input width {} must match logical weight K {}", .{ opts.input.dim(1), weight_k }); + if (opts.quant_scheme == .mxfp4) { + stdx.debug.assert(@mod(weight_k, 32) == 0 and opts.weight_scale != null, "MXFP4 requires K divisible by 32 and weight scales", .{}); + stdx.debug.assert(opts.input.dtype() == .bf16 and opts.input_scale == null, "MXFP4 requires BF16 activations without input scales", .{}); + } + + if (opts.quant_scheme == .fp8_block32) { + const scales = opts.weight_scale orelse stdx.debug.panic("block32 FP8 requires weight scales", .{}); + stdx.debug.assert(weight.dtype() == .f8e4m3fn and @mod(weight.dim(1), 32) == 0 and @mod(weight_k, 32) == 0, "block32 FP8 requires E4M3FN weights with N and K divisible by 32", .{}); + stdx.debug.assert(scales.rank() == 3 and scales.dim(0) == weight.dim(0) and scales.dim(1) == @divExact(weight.dim(1), 32) and scales.dim(2) == @divExact(weight_k, 32), "block32 FP8 requires scales shaped [expert, N/32, K/32], got {f}", .{scales.shape()}); + stdx.debug.assert(scales.dtype() == .f8e8m0 or scales.dtype() == .bf16 or scales.dtype() == .f32, "unsupported block32 FP8 scale dtype {}", .{scales.dtype()}); + } + + stdx.debug.assert(opts.quant_scheme != null or (opts.input_scale == null and opts.weight_scale == null), "scales require a quantization scheme", .{}); + for ([_]i64{ weight.dim(1), weight.dim(2), opts.input.dim(1), weight.dim(1) * weight.dim(2), opts.output_shape.dim(-1) }) |dim_or_stride| { + stdx.debug.assert(dim_or_stride > 0 and @mod(dim_or_stride, 16) == 0, "FusedMoe dimensions and matrix strides must be positive multiples of 16, got {}", .{dim_or_stride}); + } + const block_size_m: i64 = @intCast(opts.launch_config.block_size_m); + const block_size_n: i64 = @intCast(opts.launch_config.block_size_n); + const m_tokens = opts.input.dim(0); + const em_effective = if (m_tokens < block_size_m) + @min(opts.routing.max_num_tokens_padded, opts.routing.num_assignments * block_size_m) + else + opts.routing.max_num_tokens_padded; + stdx.debug.assert(@mod(em_effective, block_size_m) == 0, "routing capacity {} must be a multiple of block size {}", .{ em_effective, block_size_m }); + const grid_x = + @divExact(em_effective, block_size_m) * + (std.math.divCeil(i64, weight.dim(1), block_size_n) catch unreachable); + + const weight_scale: ?Tensor = if (opts.weight_scale) |scale| blk: { + const scheme = opts.quant_scheme orelse break :blk scale; + break :blk switch (scheme) { + // dot_scaled takes E8M0 encodings as bytes, including native E8M0 tensors. + .mxfp4, .mxfp8 => scale.bitCast(.u8), + .fp8_per_channel => scale.reshape(.{ weight.dim(0), weight.dim(1), 1 }), + .fp8_per_tensor => if (scale.count() == 1) scale else scale.reshape(.{ weight.dim(0), 1, 1 }), + // Tiled FP8 uses ordinary dot products with FP32 scale arithmetic. + .fp8_block32 => scale.convert(.f32), + .fp8_block128, .nvfp4 => scale, + }; + } else null; + + const stride_asm: i64 = if (opts.input_scale) |scale| (if (scale.rank() == 2) scale.dim(1) else 0) else 0; + const stride_ask: i64 = if (opts.input_scale) |scale| (if (scale.rank() == 2) 1 else 0) else 0; + const has_weight_scale_expert_stride = if (weight_scale) |scale| + !(opts.quant_scheme == .fp8_per_tensor and scale.count() == 1) + else + false; + const stride_bse: i64 = if (weight_scale) |scale| (if (scale.rank() == 3 and has_weight_scale_expert_stride) scale.dim(1) * scale.dim(2) else 0) else 0; + const stride_bsk: i64 = if (weight_scale) |scale| (if (scale.rank() == 3) 1 else 0) else 0; + const stride_bsn: i64 = if (weight_scale) |scale| (if (scale.rank() == 3) scale.dim(2) else 0) else 0; + + return kernels.FusedMoe.Kernel.call( + .{ + .a_ptr = opts.input, + .b_ptr = weight, + .b_bias_ptr = opts.bias orelse Tensor.scalar(0, .f32), + .a_scale_ptr = opts.input_scale orelse Tensor.scalar(1.0, .f32), + .b_scale_ptr = weight_scale orelse Tensor.scalar(1.0, .f32), + .routing_weights_ptr = opts.routing_weights orelse Tensor.scalar(1.0, .f32), + .sorted_token_ids_ptr = opts.routing.sorted_token_ids, + .expert_ids_ptr = opts.expert_ids, + .num_tokens_post_padded_ptr = opts.routing.num_tokens_post_padded, + .N_ptr = Tensor.constant(.{ .i64 = weight.dim(1) }).reshape(.{1}), + .K_ptr = Tensor.constant(.{ .i64 = weight_k }).reshape(.{1}), + .EM_ptr = Tensor.constant(.{ .i64 = em_effective }).reshape(.{1}), + .num_valid_tokens_ptr = Tensor.constant(.{ .i64 = opts.routing.num_assignments }).reshape(.{1}), + .stride_am_ptr = Tensor.constant(.{ .i64 = opts.input.dim(1) }).reshape(.{1}), + .stride_be_ptr = Tensor.constant(.{ .i64 = weight.dim(1) * weight.dim(2) }).reshape(.{1}), + .stride_bn_ptr = Tensor.constant(.{ .i64 = weight.dim(2) }).reshape(.{1}), + .stride_cm_ptr = Tensor.constant(.{ .i64 = weight.dim(.out) }).reshape(.{1}), + .stride_asm_ptr = Tensor.constant(.{ .i64 = stride_asm }).reshape(.{1}), + .stride_ask_ptr = Tensor.constant(.{ .i64 = stride_ask }).reshape(.{1}), + .stride_bse_ptr = Tensor.constant(.{ .i64 = stride_bse }).reshape(.{1}), + .stride_bsk_ptr = Tensor.constant(.{ .i64 = stride_bsk }).reshape(.{1}), + .stride_bsn_ptr = Tensor.constant(.{ .i64 = stride_bsn }).reshape(.{1}), + .stride_bbe_ptr = Tensor.constant(.{ .i64 = if (opts.bias) |bias| (if (bias.rank() == 2 and bias.dim(0) > 1) bias.dim(1) else 0) else 0 }).reshape(.{1}), + .stride_bbn_ptr = Tensor.constant(.{ .i64 = if (opts.bias) |bias| (if (bias.rank() > 0 and bias.dim(-1) > 1) 1 else 0) else 0 }).reshape(.{1}), + }, + .{ .c = opts.output_shape }, + .{ + .cfg = .{ + .a_dtype = toDType(opts.input.dtype()), + .b_dtype = toDType(weight.dtype()), + .c_dtype = toDType(opts.output_shape.dtype()), + .a_scale_dtype = if (opts.input_scale) |scale| toDType(scale.dtype()) else null, + .b_scale_dtype = if (weight_scale) |scale| toDType(scale.dtype()) else null, + .b_bias_dtype = if (opts.bias) |bias| toDType(bias.dtype()) else null, + .routing_weights_dtype = if (opts.routing_weights) |weights| toDType(weights.dtype()) else null, + .block_size_m = opts.launch_config.block_size_m, + .block_size_n = opts.launch_config.block_size_n, + .block_size_k = opts.launch_config.block_size_k, + .group_size_m = opts.launch_config.group_size_m, + .top_k = opts.top_k, + .naive_block_assignment = opts.routing.naive_block_assignment, + .compute_type = .bf16, + .quant_scheme = opts.quant_scheme, + }, + .grid = .{ @intCast(grid_x), 1, 1 }, + .num_warps = opts.launch_config.num_warps, + .num_stages = numStages(opts.launch_config), + }, + ).c; +} + +pub const Routing = struct { + sorted_token_ids: Tensor, + expert_ids: Tensor, + num_tokens_post_padded: Tensor, + max_num_tokens_padded: i64, + num_assignments: i64, + naive_block_assignment: bool, +}; + +pub fn prepareRouting(topk_ids: Tensor, num_experts: i64, block_size_m: i64) Routing { + stdx.debug.assert(block_size_m > 0 and @mod(block_size_m, 16) == 0, "routing block size must be a positive multiple of 16, got {}", .{block_size_m}); + const ids = topk_ids.withTags(.{ .token, .topk }).convert(.i32); + const num_assignments = ids.dim(.token) * ids.dim(.topk); + const sparsity_factor: i64 = 4; + const naive_block_assignment = num_assignments * sparsity_factor <= num_experts; + const max_num_tokens_padded = if (naive_block_assignment) + num_assignments * block_size_m + else if (num_assignments < num_experts) + num_assignments * block_size_m + else + (std.math.divCeil(i64, num_assignments + num_experts * (block_size_m - 1), block_size_m) catch unreachable) * block_size_m; + + const sorted_token_ids, const expert_ids, const num_tokens_post_padded = if (naive_block_assignment) blk: { + log.debug("Using naive block assignment for MoE kernels. Num assignments: {d}, Num experts: {d}", .{ num_assignments, num_experts }); + break :blk .{ + Tensor.zeroes(Shape.init(.{ .g = 1 }, .i32)), + ids.reshape(.{ .g = num_assignments }), + Tensor.constant(.{ .i32 = @as(i32, @intCast(max_num_tokens_padded)) }).reshape(.{1}), + }; + } else alignBlockSize(ids, num_experts, block_size_m); + + return .{ + .sorted_token_ids = sorted_token_ids, + .expert_ids = expert_ids, + .num_tokens_post_padded = num_tokens_post_padded, + .max_num_tokens_padded = max_num_tokens_padded, + .num_assignments = num_assignments, + .naive_block_assignment = naive_block_assignment, + }; +} + +fn alignBlockSize(topk_ids: Tensor, num_experts: i64, block_size_m: i64) struct { Tensor, Tensor, Tensor } { + const topk_ids_ = topk_ids.withTags(.{ .token, .topk }).convert(.i32); + const num_tokens = topk_ids_.dim(.token); + const topk = topk_ids_.dim(.topk); + const num_assignments = num_tokens * topk; + const max_num_tokens_padded = if (num_assignments < num_experts) + num_assignments * block_size_m + else + (std.math.divCeil(i64, num_assignments + num_experts * (block_size_m - 1), block_size_m) catch unreachable) * block_size_m; + const max_num_m_blocks = @divExact(max_num_tokens_padded, block_size_m); + const warp_size: i64 = 32; + const padded_num_experts: i64 = @intCast(std.math.ceilPowerOfTwoAssert(u64, @intCast(@max(num_experts, warp_size)))); + const sort_block_size: i64 = 256; + const sort_grid_x: i64 = @min(std.math.divCeil(i64, num_assignments, sort_block_size) catch unreachable, 65535); + + const flat_experts = topk_ids_.reshape(.{ .g = num_assignments }); + var cumsums = Tensor.zeroes(Shape.init(.{ .g = num_experts + 1 }, .i32)); + var expert_ids = Tensor.zeroes(Shape.init(.{ .g = max_num_m_blocks }, .i32)); + var sorted_token_ids = Tensor.zeroes(Shape.init(.{ .g = max_num_tokens_padded }, .i32)); + var num_tokens_post_padded = Tensor.zeroes(Shape.init(.{ .g = 1 }, .i32)); + + { + const align_outs = kernels.MoeAlignBlockSize.Kernel.call( + .{ + .topk_ids_ptr = flat_experts, + .sorted_token_ids_ptr = sorted_token_ids, + .expert_ids_ptr = expert_ids, + .num_tokens_post_pad_ptr = num_tokens_post_padded, + .cumsum_ptr = cumsums, + }, + .{ + .sorted_token_ids = sorted_token_ids.shape(), + .expert_ids = expert_ids.shape(), + .num_tokens_post_pad = num_tokens_post_padded.shape(), + .cumsum = cumsums.shape(), + }, + .{ + .cfg = .{ + .numel = @intCast(num_assignments), + .num_experts = @intCast(num_experts), + .padded_num_experts = @intCast(padded_num_experts), + .max_num_tokens_padded = @intCast(max_num_tokens_padded), + .max_num_m_blocks = @intCast(max_num_m_blocks), + .block_size_m = @intCast(block_size_m), + .hist_block = 256, + }, + .grid = .{ 2, 1, 1 }, + .num_stages = 1, + .num_warps = 8, + .output_operand_aliases = .{ + .sorted_token_ids = .sorted_token_ids_ptr, + .expert_ids = .expert_ids_ptr, + .num_tokens_post_pad = .num_tokens_post_pad_ptr, + .cumsum = .cumsum_ptr, + }, + }, + ); + sorted_token_ids = align_outs.sorted_token_ids; + expert_ids = align_outs.expert_ids; + num_tokens_post_padded = align_outs.num_tokens_post_pad; + cumsums = align_outs.cumsum; + } + + { + const sort_outs = kernels.CountAndSortExpertTokens.Kernel.call( + .{ + .topk_ids_ptr = flat_experts, + .sorted_token_ids_ptr = sorted_token_ids, + .cumsum_ptr = cumsums, + }, + .{ + .sorted_token_ids = sorted_token_ids.shape(), + .cumsum = cumsums.shape(), + }, + .{ + .cfg = .{ + .numel = @intCast(num_assignments), + .num_experts = @intCast(num_experts), + .sort_block_size = @intCast(sort_block_size), + }, + .grid = .{ @intCast(sort_grid_x), 1, 1 }, + .num_stages = 1, + .num_warps = 4, + .output_operand_aliases = .{ + .sorted_token_ids = .sorted_token_ids_ptr, + .cumsum = .cumsum_ptr, + }, + }, + ); + sorted_token_ids = sort_outs.sorted_token_ids; + cumsums = sort_outs.cumsum; + } + + return .{ sorted_token_ids, expert_ids, num_tokens_post_padded }; +} + +pub fn prepareInput(x: Tensor, scheme: ?zml.Quantization.Scheme, output_dtype: zml.DataType, quantize: bool) struct { Tensor, ?Tensor } { + if (quantize) { + if (scheme) |scheme_| { + if (scheme_ != .mxfp4) { + const input, const scale = quantizeFp8Input(x, scheme_, output_dtype); + return .{ input, scale }; + } + } + } + return .{ x.convert(.bf16), null }; +} + +fn quantizeFp8Input(x: Tensor, scheme: zml.Quantization.Scheme, output_dtype: DataType) struct { Tensor, Tensor } { + const group_size: i64, const scale_dtype: DataType, const absmax_epsilon: f32 = switch (scheme) { + .mxfp8 => .{ 32, .u8, 1e-10 }, + .fp8_per_channel, .fp8_per_tensor => .{ x.dim(1), .f32, 1e-10 }, + .fp8_block128 => .{ 128, .f32, 1e-10 }, + .fp8_block32 => .{ 32, .f32, 1e-4 }, + .mxfp4, .nvfp4 => unreachable, + }; + stdx.debug.assert(x.rank() == 2, "expected a rank-2 activation matrix, got {f}", .{x.shape()}); + stdx.debug.assert(@mod(x.dim(1), group_size) == 0, "activation width must be divisible by group size {d}, got {d}", .{ group_size, x.dim(1) }); + + const groups_per_row = @divExact(x.dim(1), group_size); + const quantized = Tensor.zeroes(Shape.init(.{ .token = x.dim(0), .feature = x.dim(1) }, output_dtype)); + const scales = Tensor.zeroes(Shape.init(.{ .token = x.dim(0), .group = groups_per_row }, scale_dtype)); + + const outs = kernels.PerTokenGroupQuantFp8.Kernel.call( + .{ + .y_ptr = x, + .group_size_ptr = Tensor.constant(.{ .i64 = group_size }).reshape(.{1}), + .y_num_columns_ptr = Tensor.constant(.{ .i64 = x.dim(1) }).reshape(.{1}), + .y_row_stride_ptr = Tensor.constant(.{ .i64 = x.dim(1) }).reshape(.{1}), + .eps_ptr = Tensor.scalar(absmax_epsilon, .f32), + }, + .{ .y_q = quantized.shape(), .y_s = scales.shape() }, + .{ + .cfg = .{ + .input_dtype = toDType(x.dtype()), + .output_dtype = toDType(output_dtype), + .scale_dtype = toDType(scale_dtype), + .block = std.math.ceilPowerOfTwoAssert(usize, @intCast(group_size)), + .quant_scheme = scheme, + }, + .grid = .{ @intCast(x.dim(0) * groups_per_row), 1, 1 }, + .num_stages = 1, + .num_warps = 1, + }, + ); + + return .{ outs.y_q, outs.y_s }; +} + +test "SwiGLU uses FP32 math for concatenated and interleaved BF16 inputs" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = zml.testing.env(); + const Local = struct { + fn forward(x: Tensor, layout: ProjectionLayout, activation: Activation) Tensor { + return activation.applyActivation(x, layout); + } + }; + const x: Tensor = .init(.{ .token = 1, .out = 6 }, .bf16); + const gates = [_]f32{ 0.75, -1.25, 3.5 }; + const ups = [_]f32{ 0.875, -2.25, 4.5 }; + for ([_]ProjectionLayout{ .concatenated, .row_interleaved_1 }) |layout| { + var values: [6]zml.floats.BFloat16 = undefined; + for (gates, ups, 0..) |gate, up, i| { + values[if (layout == .concatenated) i else 2 * i] = .fromF32(gate); + values[if (layout == .concatenated) i + 3 else 2 * i + 1] = .fromF32(up); + } + var input = try zml.Buffer.fromBytes(io, platform, x.shape(), std.mem.asBytes(&values)); + defer input.deinit(); + for ([_]?f32{ null, 2 }) |threshold| { + const activation: Activation = .{ .swiglu = .{ .limit = threshold } }; + var exe = try platform.compileFn(allocator, io, Local.forward, .{ x, layout, activation }, .{}); + defer exe.deinit(); + try zml.testing.expectEqualShapes(Shape.init(.{ .token = 1, .out = 3 }, .f32), exe.output_shapes[0]); + var output = try exe.eval(allocator, io, .{input}); + defer output.deinit(); + var actual = try output.toSliceAlloc(allocator, io); + defer actual.free(allocator); + for (gates, ups, actual.constItems(f32)) |gate, up, value| { + const g = if (threshold) |limit| @min(gate, limit) else gate; + const u = if (threshold) |limit| std.math.clamp(up, -limit, limit) else up; + try std.testing.expectApproxEqAbs(g / (1 + @exp(-g)) * u, value, 1e-6); + } + } + } +} + +test "FP8 routed GEMM with bias matches dequantized weights" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = zml.testing.env(); + if (platform.target != .cuda) return error.SkipZigTest; + + const Local = struct { + const Outputs = struct { actual: Tensor, expected: Tensor }; + + fn expandScales(scales: Tensor, shape: Shape) Tensor { + var expanded = scales.convert(.f32); + for (0..shape.rank()) |axis| { + const factor = @divExact(shape.dim(axis), expanded.dim(axis)); + const broad_shape = expanded.shape().insert(axis + 1, .{factor}); + const axes = Shape.range(broad_shape.rank(), .i64).remove(axis + 1); + expanded = expanded.broadcast(broad_shape, axes.dims()) + .reshape(expanded.shape().setDim(axis, shape.dim(axis))); + } + return expanded.withTags(shape.tags()); + } + + fn forward(x: Tensor, w: Tensor, scales: Tensor, ids: Tensor, weights: Tensor, bias: Tensor, scheme: zml.Quantization.Scheme, mx_native: bool, quantize_input: bool) Outputs { + const weight = w.convert(.f8e4m3fn); + var input = x; + var input_scale: ?Tensor = null; + var reference_input = x.convert(.f32); + if (quantize_input) { + input, input_scale = prepareInput(x, scheme, .f8e4m3fn, true); + const scale = if (scheme == .mxfp8) input_scale.?.bitCast(.f8e8m0) else input_scale.?; + reference_input = input.convert(.f32).withTags(x.shape().tags()) + .mul(expandScales(scale.convert(.f32), x.shape())); + } + const weight_scale = if (scheme == .mxfp8) blk: { + const native = scales.convert(.f8e8m0); + break :blk if (mx_native) native else native.bitCast(.u8); + } else scales; + const config = launchConfigForTokens(8); + const routing = prepareRouting(ids, w.dim(.expert), @intCast(config.block_size_m)); + const actual = gemm(.{ + .input = input, + .weight = weight, + .bias = bias, + .input_scale = input_scale, + .weight_scale = weight_scale, + .routing_weights = weights, + .routing = routing, + .expert_ids = routing.expert_ids, + .launch_config = config, + .quant_scheme = scheme, + .top_k = 1, + .output_shape = Shape.init(.{ .token = x.dim(.token), .topk = 1, .out = w.dim(.out) }, .bf16), + }).squeeze(.topk); + const expanded_weight = weight.convert(.f32).mul(expandScales(scales, weight.shape())); + const selected_weight = expanded_weight.gather(.{ .expert = ids.squeeze(.topk) }, .{}); + const dot = reference_input.dot(selected_weight, .in); + const selected_bias = if (bias.rank() == 2) + bias.gather(.{ .expert = ids.squeeze(.topk) }, .{}) + else + bias; + const unweighted = dot.add(selected_bias.convert(.f32).broad(dot.shape())); + const expected = unweighted.mul(weights.squeeze(.topk).convert(.f32).broad(unweighted.shape())).convert(.bf16); + return .{ .actual = actual, .expected = expected }; + } + }; + + for ([_]zml.Quantization.Scheme{ .mxfp8, .fp8_per_tensor, .fp8_per_channel, .fp8_block128 }) |scheme| { + for ([_]bool{ false, true }) |quantize_input| { + for ([_]i64{ 1, 17 }) |tokens| { + const n: i64 = if (scheme == .fp8_block128) 256 else 48; + const scale_dims: [3]i64 = switch (scheme) { + .mxfp8 => .{ 8, n, 8 }, + .fp8_per_tensor => if (tokens == 1) .{ 1, 1, 1 } else .{ 8, 1, 1 }, + .fp8_per_channel => .{ 8, n, 1 }, + .fp8_block128 => .{ 8, 2, 2 }, + else => unreachable, + }; + const x: Tensor = .init(.{ .token = tokens, .in = 256 }, .bf16); + const w: Tensor = .init(.{ .expert = 8, .out = n, .in = 256 }, .bf16); + const scales: Tensor = .init(.{ scale_dims[0], scale_dims[1], scale_dims[2] }, .f32); + const ids: Tensor = .init(.{ .token = tokens, .topk = 1 }, .i32); + const weights: Tensor = .init(.{ .token = tokens, .topk = 1 }, .bf16); + const bias: Tensor = if (tokens == 1) + .init(.{ .out = n }, .bf16) + else + .init(.{ .expert = 8, .out = n }, .f32); + var exe = try platform.compileFn(allocator, io, Local.forward, .{ x, w, scales, ids, weights, bias, scheme, tokens == 1, quantize_input }, .{}); + defer exe.deinit(); + var buffers: [6]zml.Buffer = undefined; + var initialized: usize = 0; + defer for (buffers[0..initialized]) |*buffer| buffer.deinit(); + for ([_]Shape{ x.shape(), w.shape(), scales.shape(), ids.shape(), weights.shape(), bias.shape() }, 0..) |shape, i| { + const slice = try zml.Slice.alloc(allocator, shape); + defer slice.free(allocator); + for (0..shape.count()) |j| { + switch (i) { + 2 => slice.items(f32)[j] = @as(f32, @floatFromInt(@as(u32, 1) << @as(u5, @intCast(j % 4)))) / 16, + 3 => slice.items(i32)[j] = @intCast((j * 3 + 5) % 8), + 5 => { + const value = @as(f32, @floatFromInt(@as(i32, @intCast(j % 17)) - 8)) / 2; + if (shape.dtype() == .f32) { + slice.items(f32)[j] = value; + } else { + slice.items(zml.floats.BFloat16)[j] = .fromF32(value); + } + }, + else => slice.items(zml.floats.BFloat16)[j] = .fromF32(if (i == 4) + @as(f32, @floatFromInt(1 + j % 3)) / 4 + else + @as(f32, @floatFromInt(@as(i32, @intCast((j * 7 + j / 256) % 13)) - 6)) / 4), + } + } + buffers[i] = try zml.Buffer.fromSlice(io, platform, slice); + initialized += 1; + } + var output: zml.Bufferized(Local.Outputs) = undefined; + var runner = try exe.runner(allocator); + defer runner.deinit(allocator); + runner.run(io, .{ buffers[0], buffers[1], buffers[2], buffers[3], buffers[4], buffers[5] }, .{&output}, .{ .wait = true }); + defer zml.Buffer.deinitAll(Local.Outputs, &output); + zml.testing.expectClose(io, output.expected, output.actual, .{ .absolute_tolerance = 0.03125, .relative_tolerance = 0.01 }) catch |err| { + log.warn("FP8 routed GEMM failed for scheme={s}, quantize_input={}, tokens={}", .{ @tagName(scheme), quantize_input, tokens }); + return err; + }; + } + } + } +} + +test "fused experts support BF16 and MXFP4 layouts, bias, and routing weights" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = zml.testing.env(); + if (platform.target != .cuda) return error.SkipZigTest; + const Local = struct { + fn forward(x: Tensor, layout: ProjectionLayout, placement: RoutingWeightPlacement, storage_dtype: DataType) Tensor { + const fp4 = storage_dtype != .bf16; + const columns = Tensor.arange(.{ .end = 256 }, .i32).withTags(.{.dout}); + const is_gate = switch (layout) { + .concatenated => columns.cmp(.LT, Tensor.scalar(128, .i32)), + .row_interleaved_1 => columns.remainder(Tensor.scalar(2, .i32)).cmp(.EQ, Tensor.scalar(0, .i32)), + .row_interleaved_8 => columns.divByConst(8).remainder(Tensor.scalar(2, .i32)).cmp(.EQ, Tensor.scalar(0, .i32)), + }; + const dtype: DataType = if (fp4) .u8 else .bf16; + // E2M1 codes 1, 2, 3 encode 0.5, 1, 1.5; each byte holds two values. + const gate_value = if (fp4) Tensor.scalar(0x11, .u8) else Tensor.scalar(0.5, .bf16); + const up_value = if (fp4) Tensor.scalar(0x33, .u8) else Tensor.scalar(1.5, .bf16); + const gate_up_values = is_gate.select(gate_value, up_value); + var gate_up: zml.nn.Linear = .{ + .weight = gate_up_values.broad(Shape.init(.{ .expert = 8, .dout = 256, .d = @as(i64, if (fp4) 64 else 128) }, dtype)), + .tag = Shape.toTag(.d), + .quantization = if (fp4) .{ + .scheme = .mxfp4, + .scales = Tensor.scalar(127, .u8).broad(Shape.init(.{ .expert = 8, .dout = 256, .d = 4 }, .u8)), + } else null, + }; + var down: zml.nn.Linear = .{ + .weight = (if (fp4) Tensor.scalar(0x22, .u8) else Tensor.scalar(1, .bf16)).broad(Shape.init(.{ .expert = 8, .d = 128, .dout = @as(i64, if (fp4) 64 else 128) }, dtype)), + .bias = Tensor.scalar(2, .bf16).broad(Shape.init(.{ .expert = 8, .d = 128 }, .bf16)), + .tag = Shape.toTag(.dout), + .quantization = if (fp4) .{ + .scheme = .mxfp4, + .scales = Tensor.scalar(127, .u8).broad(Shape.init(.{ .expert = 8, .d = 128, .dout = 4 }, .u8)), + } else null, + }; + if (storage_dtype == .f4e2m1) { + gate_up.weight = gate_up.weight.bitCast(.f4e2m1).reshape(.{ .expert = 8, .dout = 256, .d = 128 }); + down.weight = down.weight.bitCast(.f4e2m1).reshape(.{ .expert = 8, .d = 128, .dout = 128 }); + gate_up.quantization.?.scales = gate_up.quantization.?.scales.bitCast(.f8e8m0); + down.quantization.?.scales = down.quantization.?.scales.bitCast(.f8e8m0); + } + const route = Tensor.arange(.{ .end = 2 }, .i32).reshape(.{ .t = 1, .topk = 2 }) + .broad(Shape.init(.{ .t = x.dim(.t), .topk = 2 }, .i32)); + return fusedExpertsBase(x, .{ + .ids = route.addConstant(2), + .weights = route.convert(.f32).addConstant(1).scale(0.25), + .global_num_experts = 8, + .local_num_experts = 8, + }, gate_up, down, layout, .{ + .routing_weight_placement = placement, + .quantize_input = false, + .activation = .{ .swiglu = .{} }, + }); + } + }; + for ([_]DataType{ .bf16, .u8, .f4e2m1 }) |storage_dtype| { + const tokens: i64 = if (storage_dtype == .u8) 17 else 1; + const x: Tensor = .init(.{ .t = tokens, .d = 128 }, .bf16); + const host = try zml.Slice.alloc(allocator, x.shape()); + defer host.free(allocator); + @memset(host.items(zml.floats.BFloat16), .fromF32(1.0 / 128.0)); + var input = try zml.Buffer.fromSlice(io, platform, host); + defer input.deinit(); + for ([_]ProjectionLayout{ .concatenated, .row_interleaved_1 }) |layout| { + for ([_]RoutingWeightPlacement{ .before_down, .after_down }) |placement| { + var exe = try platform.compileFn(allocator, io, Local.forward, .{ x, layout, placement, storage_dtype }, .{}); + defer exe.deinit(); + var output = try exe.eval(allocator, io, .{input}); + defer output.deinit(); + var actual = try output.toSliceAlloc(allocator, io); + defer actual.free(allocator); + const activation: f32 = 0.5 / (1 + @exp(@as(f32, -0.5))) * 1.5; + // The two routes have weights 0.25 and 0.5. Bias distinguishes placement. + const bias: f32 = if (placement == .before_down) 4 else 1.5; + const expected = 128 * activation * 0.75 + bias; + for (actual.constItems(zml.floats.BFloat16)) |value| { + try std.testing.expectApproxEqAbs(expected, value.toF32(), 0.5); + } + } + } + } +} + +// ============================================================================= +// Config / validation helpers +// ============================================================================= + +const LaunchConfig = struct { + block_size_m: usize, + block_size_n: usize, + block_size_k: usize, + group_size_m: usize, + num_warps: i32, + num_stages: i32, +}; + +const launch_configs = [_]struct { tokens: i64, config: LaunchConfig }{ + .{ .tokens = 1, .config = .{ .block_size_m = 16, .block_size_n = 32, .block_size_k = 64, .group_size_m = 1, .num_warps = 4, .num_stages = 4 } }, + .{ .tokens = 2, .config = .{ .block_size_m = 16, .block_size_n = 32, .block_size_k = 64, .group_size_m = 1, .num_warps = 4, .num_stages = 4 } }, + .{ .tokens = 4, .config = .{ .block_size_m = 16, .block_size_n = 32, .block_size_k = 64, .group_size_m = 1, .num_warps = 4, .num_stages = 3 } }, + .{ .tokens = 8, .config = .{ .block_size_m = 16, .block_size_n = 128, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 3 } }, + .{ .tokens = 16, .config = .{ .block_size_m = 16, .block_size_n = 64, .block_size_k = 64, .group_size_m = 64, .num_warps = 4, .num_stages = 5 } }, + .{ .tokens = 24, .config = .{ .block_size_m = 16, .block_size_n = 64, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 2 } }, + .{ .tokens = 32, .config = .{ .block_size_m = 16, .block_size_n = 32, .block_size_k = 128, .group_size_m = 1, .num_warps = 4, .num_stages = 2 } }, + .{ .tokens = 48, .config = .{ .block_size_m = 16, .block_size_n = 32, .block_size_k = 128, .group_size_m = 64, .num_warps = 4, .num_stages = 2 } }, + .{ .tokens = 64, .config = .{ .block_size_m = 16, .block_size_n = 64, .block_size_k = 128, .group_size_m = 1, .num_warps = 4, .num_stages = 2 } }, + .{ .tokens = 96, .config = .{ .block_size_m = 16, .block_size_n = 128, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 3 } }, + .{ .tokens = 128, .config = .{ .block_size_m = 16, .block_size_n = 256, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 2 } }, + .{ .tokens = 256, .config = .{ .block_size_m = 16, .block_size_n = 256, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 2 } }, + .{ .tokens = 512, .config = .{ .block_size_m = 32, .block_size_n = 128, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 3 } }, + .{ .tokens = 1024, .config = .{ .block_size_m = 64, .block_size_n = 128, .block_size_k = 64, .group_size_m = 1, .num_warps = 4, .num_stages = 3 } }, + .{ .tokens = 1536, .config = .{ .block_size_m = 64, .block_size_n = 128, .block_size_k = 64, .group_size_m = 1, .num_warps = 4, .num_stages = 3 } }, + .{ .tokens = 2048, .config = .{ .block_size_m = 128, .block_size_n = 128, .block_size_k = 64, .group_size_m = 16, .num_warps = 8, .num_stages = 3 } }, + .{ .tokens = 3072, .config = .{ .block_size_m = 128, .block_size_n = 256, .block_size_k = 64, .group_size_m = 1, .num_warps = 8, .num_stages = 4 } }, + .{ .tokens = 4096, .config = .{ .block_size_m = 128, .block_size_n = 256, .block_size_k = 64, .group_size_m = 16, .num_warps = 8, .num_stages = 4 } }, +}; + +fn launchConfigForTokens(num_tokens: i64) LaunchConfig { + var best = launch_configs[0]; + for (launch_configs[1..]) |candidate| { + // The table is ascending, so ties retain the smaller token bucket. + if (@abs(candidate.tokens - num_tokens) < @abs(best.tokens - num_tokens)) { + best = candidate; + } + } + return best.config; +} + +test "MoE launch config selects the nearest token bucket" { + try std.testing.expectEqualDeep(launchConfigForTokens(4), launchConfigForTokens(6)); + try std.testing.expectEqualDeep(launchConfigForTokens(8), launchConfigForTokens(7)); + try std.testing.expectEqualDeep(launchConfigForTokens(1), launchConfigForTokens(0)); + try std.testing.expectEqualDeep(launchConfigForTokens(4096), launchConfigForTokens(8192)); + try std.testing.expectEqual(@as(usize, 64), launchConfigForTokens(16).group_size_m); + try std.testing.expectEqual(@as(i32, 5), launchConfigForTokens(16).num_stages); +} + +pub fn isAvailable(platform: *const zml.Platform) bool { + if (platform.target != .cuda) return false; + const cc = zml.platform.cuda.computeCapability(platform) orelse return false; + return cc.major == 10; +} + +pub fn validateOptions(opts: Options) void { + stdx.debug.assert(opts.activation == .swiglu, "triton backend only accepts swiglu activation, got {}", .{opts.activation}); + stdx.debug.assert(opts.activation.swiglu.limit != null, "triton backend requires swiglu limit to be set", .{}); + stdx.debug.assert(opts.activation.swiglu.bias == null, "triton backend requires swiglu bias to be null", .{}); + stdx.debug.assert(opts.activation.swiglu.scale == null, "triton backend requires swiglu scale to be null", .{}); +} + +pub const Config = struct { + tokens: i64, + hidden: i64, + intermediate: i64, + experts: i64, + global_experts: i64, + topk: i64, + expert_offset: i64 = 0, + // Swiglu clamping. Set to 0 to disable. + swiglu_limit: f32, + /// Scale SwiGLU before FP8 quantization, or scale the down result before reduction. + routing_weight_placement: RoutingWeightPlacement = .before_down, + + pub fn validate(c: Config) !void { + if (c.tokens <= 0 or + c.experts <= 0 or + c.topk <= 0 or c.topk > c.global_experts or + c.expert_offset < 0 or c.expert_offset + c.experts > c.global_experts) + { + return error.InvalidRoutingShape; + } + + if (c.hidden < 128 or @mod(c.hidden, 128) != 0 or + c.intermediate < 128 or @mod(c.intermediate, 128) != 0) + { + return error.InvalidHiddenShape; + } + + if (!std.math.isFinite(c.swiglu_limit) or c.swiglu_limit < 0) { + return error.InvalidScale; + } + } +}; + +fn validateInputs( + c: Config, + input: zml.Tensor, + ids: zml.Tensor, + expert_weights: zml.Tensor, + gate_up: zml.Tensor, + gate_up_scales: zml.Tensor, + down: zml.Tensor, + down_scales: zml.Tensor, +) void { + stdx.debug.assert(gate_up_scales.rank() == 3 and down_scales.rank() == 3, "Invalid MXFP4 tensor layout, expected gate_up_scales and down_scales to be rank 3, got {f} and {f}", .{ gate_up_scales, down_scales }); + + const shapes = .{ + .input = zml.Shape.init(.{ c.tokens, c.hidden }, .bf16), + .ids = zml.Shape.init(.{ c.tokens, c.topk }, .i32), + .expert_weights = zml.Shape.init(.{ c.tokens, c.topk }, .f32), + .gate_up = zml.Shape.init(.{ c.experts, 2 * c.intermediate, @divExact(c.hidden, 2) }, .i8), + .gate_up_scales = zml.Shape.init(.{ c.experts, 2 * c.intermediate, gate_up_scales.dim(2) }, .f8e8m0), + .down = zml.Shape.init(.{ c.experts, c.hidden, @divExact(c.intermediate, 2) }, .i8), + .down_scales = zml.Shape.init(.{ c.experts, c.hidden, down_scales.dim(2) }, .f8e8m0), + }; + + stdx.debug.assert(input.shape().eql(shapes.input) and + ids.shape().eql(shapes.ids) and + expert_weights.shape().eql(shapes.expert_weights) and + // Note: we accept both i8 and u8 for gate_up and down + // TODO: we should instead use f4e2m1 + (gate_up.shape().eql(shapes.gate_up) or gate_up.shape().eql(shapes.gate_up.withDtype(.u8))) and + gate_up_scales.shape().eql(shapes.gate_up_scales) and + (down.shape().eql(shapes.down) or down.shape().eql(shapes.down.withDtype(.u8))) and + down_scales.shape().eql(shapes.down_scales), + \\Invalid MXFP4 tensor layout: + \\ - expected input={f}, got {f} + \\ - expected ids={f}, got {f} + \\ - expected expert_weights={f}, got {f} + \\ - expected gate_up={f}, got {f} + \\ - expected gate_up_scales={f}, got {f} + \\ - expected down={f}, got {f} + \\ - expected down_scales={f}, got {f} + , .{ + shapes.input, input, // + shapes.ids, ids, // + shapes.expert_weights, expert_weights, // + shapes.gate_up, gate_up, // + shapes.gate_up_scales, gate_up_scales, // + shapes.down, down, // + shapes.down_scales, down_scales, // + }); + + stdx.debug.assert( + gate_up_scales.dim(2) >= @divTrunc(c.hidden, 32) and + down_scales.dim(2) >= @divTrunc(c.intermediate, 32), + "Invalid MXFP4 tensor layout, not enough scales, expected a ratio of at most 32 for config {}, got {f} and {f}", + .{ c, gate_up_scales, down_scales }, + ); +} + +/// Row-major MXFP4 E2M1 weights and linear E8M0 block32 scales. +/// BF16 activations are quantized on the GPU to FP8 with per-32 E8M0 scales. +pub fn fusedExpertsMxfp4( + input: zml.Tensor, + routing: LocalRouting, + gate_up: zml.nn.Linear, + down: zml.nn.Linear, + options: Options, +) zml.Tensor { + validateOptions(options); + stdx.debug.assert(input.dtype() == .bf16, "triton backend only supports bf16 inputs, got {}", .{input.dtype()}); + stdx.debug.assert(gate_up.bias == null and down.bias == null, "triton backend expects gate_up bias and down bias to be null", .{}); + + const gq = gate_up.quantization orelse @panic("triton backend requires gate_up quantization to be set"); + const dq = down.quantization orelse @panic("triton backend requires down quantization to be set"); + stdx.debug.assert(gq.scheme == .mxfp4 and dq.scheme == .mxfp4, "triton expects gate_up and down quantization scheme to be mxfp4, got {} and {}", .{ gq.scheme, dq.scheme }); + + // Weight storage for mxfp4 in HF is expressed as u8 or i8 + stdx.debug.assert(gate_up.weight.dtype() == .u8 or gate_up.weight.dtype() == .i8, "triton expects gate_up weight dtype to be u8 or i8, got {}", .{gate_up.weight.dtype()}); + stdx.debug.assert(down.weight.dtype() == .u8 or down.weight.dtype() == .i8, "triton expects down weight dtype to be u8 or i8, got {}", .{gate_up.weight.dtype()}); + + const topk = routing.ids.dim(.topk); + const experts = gate_up.weight.dim(.expert); + const hidden = down.weight.dim(1); + const intermediate = down.weight.dim(2) * 2; + const ids = routing.localIds().convert(.i32).reshape(.{ .token = .auto, .topk = topk }); + const tokens = ids.dim(.token); + + const x = input.reshape(.{ tokens, hidden }); + const scales = routing.weights.convert(.f32).reshape(.{ tokens, topk }); + + const cfg: Config = .{ + .tokens = tokens, + .hidden = hidden, + .intermediate = intermediate, + .experts = experts, + .global_experts = routing.global_num_experts, + .topk = topk, + .swiglu_limit = options.activation.swiglu.limit.?, + .routing_weight_placement = options.routing_weight_placement, + }; + + cfg.validate() catch @panic("Invalid MXFP4 MoE configuration"); + validateInputs(cfg, x, ids, scales, gate_up.weight, gq.scales, down.weight, dq.scales); + + const bn = 16; + const bk = 512; + + const tile_count = kernels.tiles(cfg.tokens, cfg.topk, cfg.experts, bn); + + // Quantize activations + const q = kernels.Quant.Kernel.call( + .{ .x = x }, + .{ + .q = .init(.{ cfg.tokens, cfg.hidden }, .f8e4m3fn), + .s = .init(.{ cfg.tokens, @divTrunc(cfg.hidden, 32) }, .u8), + }, + .{ + .cfg = .{ .tokens = cfg.tokens, .hidden = cfg.hidden }, + .grid = .{ @intCast(@divTrunc(cfg.tokens + kernels.quantRows(cfg.tokens) - 1, kernels.quantRows(cfg.tokens))), @intCast(@divTrunc(cfg.hidden, 128)), 1 }, + .num_warps = 4, + .num_stages = 1, + }, + ); + + const dummy = zml.Tensor.scalar(0, .i32); + var map = dummy; + var perm = dummy; + var sched = dummy; + + if (cfg.tokens != 1) { + const r = kernels.Route.Kernel.call( + .{ .ids = ids }, + .{ .counts = .init(.{cfg.experts}, .i32), .pos = .init(.{cfg.tokens * cfg.topk}, .i32) }, + .{ .cfg = .{ + .tokens = cfg.tokens, + .topk = cfg.topk, + .expert_offset = cfg.expert_offset, + }, .grid = .{ @intCast(cfg.experts), 1, 1 }, .num_warps = 4, .num_stages = 1 }, + ); + + const s = kernels.Schedule.Kernel.call( + .{ .ids = ids, .counts = r.counts, .pos = r.pos }, + .{ .map = .init(.{tile_count * bn}, .i32), .perm = .init(.{cfg.tokens * cfg.topk}, .i32), .sched = .init(.{2 * tile_count + 1}, .i32) }, + .{ + .cfg = .{ + .tokens = cfg.tokens, + .topk = cfg.topk, + .expert_offset = cfg.expert_offset, + .experts = cfg.experts, + .capacity = tile_count, + .group = bn, + }, + .grid = .{ @intCast(@divTrunc(cfg.tokens * cfg.topk + 127, 128)), 1, 1 }, + .num_warps = 4, + .num_stages = 1, + }, + ); + + map = s.map; + perm = s.perm; + sched = s.sched; + } + + // Each narrow tile produces 64 activations: two complete scale groups. + const up_columns: i64 = if (cfg.tokens <= 8) 64 else 128; + const m = kernels.GateUp.Kernel.call( + .{ .q = q.q, .s = q.s, .w = gate_up.weight.bitCast(.u8), .ws = gq.scales.bitCast(.u8), .ids = ids, .rw = scales, .map = map, .sched = sched }, + .{ .mid = .init(.{ tile_count * bn, cfg.intermediate }, .f8e4m3fn), .ms = .init(.{ tile_count * bn, @divTrunc(cfg.intermediate, 32) }, .u8) }, + .{ + .cfg = .{ + .tokens = cfg.tokens, + .hidden = cfg.hidden, + .intermediate = cfg.intermediate, + .experts = cfg.experts, + .topk = cfg.topk, + .expert_offset = cfg.expert_offset, + .swiglu_limit = cfg.swiglu_limit, + .scale_stride = gq.scales.dim(2), + .routing_weight_placement = cfg.routing_weight_placement, + .bn = bn, + .bk = bk, + }, + .grid = .{ @intCast(@divTrunc(cfg.intermediate, up_columns)), @intCast(tile_count), 1 }, + .num_warps = if (cfg.tokens <= 8) 4 else 8, + .num_stages = 2, + .global_scratch_memory_size = @intCast(128 * @divTrunc(cfg.intermediate, up_columns) * tile_count), + }, + ); + + const d = kernels.Down.Kernel.call( + .{ .mid = m.mid, .ms = m.ms, .w = down.weight.bitCast(.u8), .ws = dq.scales.bitCast(.u8), .ids = ids, .sched = sched }, + .{ .d = .init(.{ tile_count * bn, cfg.hidden }, .bf16) }, + .{ + .cfg = .{ + .tokens = cfg.tokens, + .hidden = cfg.hidden, + .intermediate = cfg.intermediate, + .experts = cfg.experts, + .topk = cfg.topk, + .expert_offset = cfg.expert_offset, + .swiglu_limit = cfg.swiglu_limit, + .scale_stride = dq.scales.dim(2), + .routing_weight_placement = cfg.routing_weight_placement, + .bn = bn, + .bk = bk, + }, + .grid = .{ @intCast(@divTrunc(cfg.hidden, 128)), @intCast(tile_count), 1 }, + .num_warps = if (cfg.tokens <= 8) 4 else 8, + .num_stages = 2, + .global_scratch_memory_size = @intCast(128 * @divTrunc(cfg.hidden, 128) * tile_count), + }, + ); + + return kernels.Combine.Kernel.call( + .{ .d = d.d, .ids = ids, .perm = perm, .rw = scales }, + .{ .y = x.shape() }, + .{ + .cfg = .{ + .tokens = cfg.tokens, + .hidden = cfg.hidden, + .topk = cfg.topk, + .expert_offset = cfg.expert_offset, + .experts = cfg.experts, + .routing_weight_placement = cfg.routing_weight_placement, + .bn = bn, + }, + .grid = .{ @intCast(cfg.tokens), @intCast(@divTrunc(cfg.hidden + kernels.combineBlock(cfg.tokens, cfg.hidden) - 1, kernels.combineBlock(cfg.tokens, cfg.hidden))), 1 }, + .num_warps = 4, + .num_stages = 1, + }, + ).y.reshape(input.shape().dims()).withTags(input.shape()); +} diff --git a/zml/kernels/moe/weights.zig b/zml/kernels/moe/weights.zig new file mode 100644 index 00000000..0e288856 --- /dev/null +++ b/zml/kernels/moe/weights.zig @@ -0,0 +1,10 @@ +const nn = @import("../../nn.zig"); +const Tensor = @import("../../tensor.zig").Tensor; + +pub fn unpackedWeight(linear: nn.Linear) Tensor { + const quantization = linear.quantization orelse return linear.weight; + return if (nn.isPackedFp4(quantization.scheme, linear.weight.dtype())) + nn.unpackFp4(linear.weight, linear.tag, linear.tag) + else + linear.weight; +} diff --git a/zml/kernels/paged_attention/README.md b/zml/kernels/paged_attention/README.md new file mode 100644 index 00000000..20ba9d7d --- /dev/null +++ b/zml/kernels/paged_attention/README.md @@ -0,0 +1,184 @@ +# Paged attention semantics + +[paged_attention.zig](paged_attention.zig) evaluates attention over a KV cache +whose logical sequences are stored in fixed-size physical pages. Paging changes +how keys and values are addressed; softmax still normalizes over all permitted +keys in one sequence, independently for each query and query head. + +The public entry point is: + +```zig +pagedAttention(parameters, q, k, v, kv_cache, opts) +``` + +The result has the query's shape and dtype. The `k` and `v` arguments are currently +ignored: attention reads exclusively from `kv_cache`. Populate the cache, +including the current queries' keys and values, **before** calling attention. +The attention call does not update the cache or allocate pages. Projections, +RoPE, and any Q/K normalization happen before this operation; the output +projection happens afterward. + +## Tensor layouts and heads + +Let `T` be the packed query token count, `B` the sequence capacity, `N` the +physical page count, `P` the tokens per page, `M` the page-table capacity per +sequence, `Hkv` the KV head count, `G` the query heads per KV head, and `D` the +features per head. + +| Input | Tagged shape | Meaning | +| --- | --- | --- | +| `q` | `{ .b = T, .hkv = Hkv, .hg = G, .hd = D }` | Queries packed in sequence order. Here `.b` counts tokens, not sequences. | +| Split cache `k`, `v` | `{ .page = N, .k_chunk = P, .hkv = Hkv, .hd = D }` | Physical key/value storage. | +| `parameters.block_table` | `{ .b = B, .p = M }`, `i32` | Logical page to physical page mapping for each sequence. | +| `parameters.seq_lens` | `{ .b = B }`, `i32` | Valid KV token counts, including the current queries. | +| `parameters.query_start_len` | `{ .b = B + 1 }`, `i32` | Cumulative offsets into packed queries. | + +The total query head count is `Hq = Hkv * G`. Flattening the two head axes gives +`h = hkv * G + hg`, so query head `h` reads KV head `floor(h / G)`. Each query +head has its own scores and softmax. This covers MHA (`G = 1`), GQA, and MQA +(`Hkv = 1`). `D` is the feature count of one head, not the number of heads or +the sequence length. Use matching floating-point query and cache dtypes. + +The TPU backend instead requires `KvCache.dense`, a fused tensor with layout +`{ .page = N, .k_chunk = P, .hkv = 2 * Hkv, .hd = Dpad }`. The combined head +axis interleaves `K0, V0, K1, V1, ...`. `Dpad` must equal `D` rounded up to a +multiple of 128. Pad unused cache features with zeros. The wrapper pads queries, +uses the logical `D` for the default scale, and trims the output back to `D`. +Its accepted query/cache dtypes are `bf16`, `f16`, and `f32`. + +## Packed sequences and positions + +For sequence `s`, define: + +```text +start = query_start_len[s] +end = query_start_len[s + 1] +Lq = end - start +Lk = seq_lens[s] +context_len = Lk - Lq +position(i) = context_len + i # 0 <= i < Lq +``` + +Queries `q[start:end]` represent the final `Lq` tokens of that sequence. The +kernel infers their positions from lengths; there is no separate position input. +An initial prefill has `Lq = Lk`, a chunked prefill has `Lq > 1` after a cached +prefix, and a decode normally has `Lq = 1` at position `Lk - 1`. A batch may mix +prefill and decode sequences. Set `is_prefill = true` for such mixed batches; +`false` selects the all-decode execution configuration on Triton. It does not +change the attention equation. `max_seqlen_q` is a configuration bound for the +largest query chunk, not a runtime sequence length. + +Offsets must begin at zero and be nondecreasing. Active sequences require +`0 < Lq <= Lk <= M * P`. Place inactive sequences after the active sequences, +with repeated offsets and normally `seq_lens = 0`. This ordering matters: +StableHLO stops at the first sequence with no queries, and TPU passes a count of +active sequences to its kernel. The final offset is the active packed token +count; any extra query capacity is padding whose output should not be consumed. + +## Page addressing and cache updates + +A logical key position `j` in sequence `s` is addressed as: + +```text +logical_page = floor(j / P) +physical_page = block_table[s, logical_page] +offset = j % P +K(s, j) = k_cache[physical_page, offset] +V(s, j) = v_cache[physical_page, offset] +``` + +Physical pages need not be contiguous or ordered. Different sequences may map +to the same physical pages to share a prefix. Every page covering a valid token +must have a valid physical ID in `[0, N)` and populated KV data. Entries beyond +`ceil(Lk / P)` are unused; tests use `-1` there. A negative ID is not a portable +way to mask a valid logical page. `seq_lens`, rather than allocated page count, +determines which tokens in the last page are valid. + +`KvCache.update(new_k, new_v, slot_mapping, chunk_size, backend)` writes split +cache entries using physical flattened slots: + +```text +chunk_size = P +physical_page = slot_mapping[t] / P +offset = slot_mapping[t] % P +``` + +`new_k` and `new_v` have packed token/head/feature axes matching the split +cache. `slot_mapping` is a physical address, whereas `block_table` translates +logical sequence positions. The caller maintains both mappings and lengths; +`update` does not derive or update sequence metadata. Writes override existing +entries and request buffer reuse. Use valid, distinct slots for writes requiring +deterministic results; conflicting scatter writes have no ordering guarantee. +The generic update method currently panics for `.dense`. + +## Attention equation and masks + +For local query row `i`, KV head `a`, and query group head `g`: + +```text +p = Lk - Lq + i +score_j = scale * dot(q[start + i, a, g], K(s, j, a)) +weight = softmax(score over permitted j) +out = sum_j(weight_j * V(s, j, a)) +``` + +`opts.scale` defaults to `1 / sqrt(D)`. There is no dropout or returned +probability matrix. Internal paging, tiling, or splitting of the key range must +preserve one softmax normalization; summing separately normalized page outputs +would compute a different result. Accumulation order and dtype conversions can +cause numerical differences between backends. + +The default mask is `.standard = .{ .causal = true }`: + +| Mask | Permitted logical key positions | +| --- | --- | +| Default causal | `0 <= j < Lk` and `j <= p`. Includes the query's own key. | +| `AttentionMask.none` | Every `0 <= j < Lk`. | +| Causal with `sliding_window = W` | `0 <= j < Lk` and `p - W < j <= p`. | +| Non-causal with `sliding_window = W` | `0 <= j < Lk` and `p - W < j`. Includes all later valid keys. | + +Use a positive `W`; it counts the current token as part of the left window. +Use `null` for no window. A non-causal window is a left bound, not a symmetric +window around the query. + +Triton also accepts `.indices`, an `i32` tensor shaped +`{ .b = T, .topk = K }` with `K > 0`. Each packed query row supplies logical +key positions in its own sequence, shared across its heads. Positions outside +`[0, Lk)` are ignored. This replaces the standard mask, so future valid keys +are permitted. Duplicate positions are processed repeatedly and therefore +change their weight; provide unique positions to represent a set of keys. + +On Triton, `opts.sink` supplies one softmax logit per flattened query head. +It adds `exp(sink[h])` to the denominator and contributes zero to the weighted +value numerator. It is not a cache token and does not consume a page or sequence +position. Other wrappers currently ignore `sink`. For portable use, ensure each +active query has at least one permitted key; fully masked rows are not given a +shared cross-backend output contract. + +## Example + +With `P = 4`, consider a two-query chunk after a four-token prefix and a decode +after a two-token prefix, followed by one inactive sequence: + +```text +query_start_len = [0, 2, 3, 3] +seq_lens = [6, 3, 0] +block_table = [[5, 2], [8, -1], [-1, -1]] +``` + +Packed queries 0 and 1 belong to sequence 0 at positions 4 and 5. Their current +KV entries occupy physical page 2, offsets 0 and 1 (slots 8 and 9). Packed query +2 belongs to sequence 1 at position 2, physical page 8, offset 2 (slot 34). +The default causal mask permits keys 0–4, 0–5, and 0–2 respectively, always +within the query's own sequence. With a window of 3, the permitted positions +become 2–4, 3–5, and 0–2. + +## Backend implementations + +See each backend's README for execution details, supported options, and current +limitations: + +- [Triton](triton/README.md) — CUDA, ROCm, and oneAPI. +- [Mosaic TPU](mosaic_tpu/README.md) — TPU. +- [Metal](metal/README.md) — Metal. +- [StableHLO](stablehlo/README.md) — CPU and Furiosa fallback. diff --git a/zml/kernels/paged_attention/cute/BUILD.bazel b/zml/kernels/paged_attention/cute/BUILD.bazel new file mode 100644 index 00000000..6329ae12 --- /dev/null +++ b/zml/kernels/paged_attention/cute/BUILD.bazel @@ -0,0 +1,47 @@ +load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_test") + +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "cute", + srcs = [ + "attention.zig", + "cute.zig", + "//zml/kernels/paged_attention/cute/sm10x", + "//zml/kernels/paged_attention/cute/sm120", + ], +) + +zig_test( + name = "correctness_test", + srcs = [ + "cases.zig", + "fixtures.zig", + "reference.zig", + ], + main = "correctness_test.zig", + tags = [ + "exclusive", + "requires-gpu", + ], + target_compatible_with = select({ + "//platforms:cuda.enabled": [], + "//conditions:default": ["@platforms//:incompatible"], + }), + deps = ["//zml"], +) + +zig_binary( + name = "benchmark", + srcs = [ + "cases.zig", + "fixtures.zig", + ], + main = "benchmark.zig", + tags = ["manual"], + target_compatible_with = select({ + "//platforms:cuda.enabled": [], + "//conditions:default": ["@platforms//:incompatible"], + }), + deps = ["//zml"], +) diff --git a/zml/kernels/paged_attention/cute/attention.zig b/zml/kernels/paged_attention/cute/attention.zig new file mode 100644 index 00000000..50eb2531 --- /dev/null +++ b/zml/kernels/paged_attention/cute/attention.zig @@ -0,0 +1,117 @@ +const std = @import("std"); + +const zml = @import("../../../zml.zig"); +const cute = zml.kernel.cute; +const V = cute.Value; + +pub const Config = struct { + head_dim: i64, + tokens: i64, + batch: i64, + kv_heads: i64, + group: i64, + pages: i64, + pages_per_seq: i64, + page_size: i64, + dtype: cute.DType, + causal: bool, + window: i64, + scale: f32, + sink: bool, + indexed: bool, + topk: i64, +}; + +pub const Kernel = cute.Kernel(Config, .{ + .name = "paged_attention_cute_generic", + .inputs = &.{ "q", "k", "v", "table", "lengths", "starts", "sink", "indices" }, + .outputs = &.{"out"}, + .run = run, +}); + +fn run(b: *cute.Builder, c: Config) cute.FinishError!void { + switch (c.head_dim) { + 32 => try runDim(b, c, 1), + 64 => try runDim(b, c, 2), + 128 => try runDim(b, c, 4), + 256 => try runDim(b, c, 8), + 512 => try runDim(b, c, 16), + else => unreachable, + } +} + +fn runDim(b: *cute.Builder, c: Config, comptime chunks: usize) cute.FinishError!void { + const a = try b.declareArgs(.{ + .q = .{ .tensor = .{ .dtype = c.dtype, .shape = &.{ c.tokens, c.kv_heads * c.group, c.head_dim } } }, + .k = .{ .tensor = .{ .dtype = c.dtype, .shape = &.{ c.pages, c.page_size, c.kv_heads, c.head_dim } } }, + .v = .{ .tensor = .{ .dtype = c.dtype, .shape = &.{ c.pages, c.page_size, c.kv_heads, c.head_dim } } }, + .table = .{ .tensor = .{ .dtype = .i32, .shape = &.{ c.batch, c.pages_per_seq } } }, + .lengths = .{ .tensor = .{ .dtype = .i32, .shape = &.{c.batch} } }, + .starts = .{ .tensor = .{ .dtype = .i32, .shape = &.{c.batch + 1} } }, + .sink = .{ .tensor = .{ .dtype = .f32, .shape = &.{c.kv_heads * c.group} } }, + .indices = .{ .tensor = .{ .dtype = .i32, .shape = &.{ c.tokens, c.topk } } }, + .out = .{ .tensor = .{ .dtype = c.dtype, .shape = &.{ c.tokens, c.kv_heads * c.group, c.head_dim } } }, + }); + const token = b.blockIdx().x; + const head = b.blockIdx().y; + const lane = b.threadIdx().x; + const zero = b.constant(.f32, 0); + // One full warp owns a query/head; all control flow is warp-uniform. + // Also define output for unused capacity at the end of the packed queries. + for (0..chunks) |i| a.out.set(.{ token, head, lane.add(i * 32) }, zero.to(c.dtype)); + var active = b.openIf(token.lt(a.starts.get(.{c.batch}))); + { + var find_seq = b.openFor(0, c.batch, 1, .{b.constant(.i32, 0)}); + { + const owns = token.ge(a.starts.get(.{find_seq.iv})).bitAnd(token.lt(a.starts.get(.{find_seq.iv.add(1)}))); + find_seq.yield(.{b.select(owns, find_seq.iv, find_seq.carried[0])}); + } + const seq = find_seq.results[0]; + const start = a.starts.get(.{seq}); + const qlen = a.starts.get(.{seq.add(1)}).sub(start); + const len = a.lengths.get(.{seq}); + const position = len.sub(qlen).add(token.sub(start)); + const begin = if (c.window > 0) position.sub(c.window - 1).maximum(0) else b.constant(.i32, 0); + const end = if (c.causal) position.add(1).minimum(len) else len; + var query: [chunks]V = undefined; + for (&query, 0..) |*q, i| q.* = a.q.get(.{ token, head, lane.add(i * 32) }).to(.f32); + + // State is running max, normalizer, then each lane's output features. + // A sink is an initial softmax entry with zero value. + var initial: [chunks + 2]V = @splat(zero); + initial[0] = if (c.sink) a.sink.get(.{head}).mul(std.math.log2e) else b.constant(.f32, -std.math.inf(f32)); + initial[1] = if (c.sink) b.constant(.f32, 1) else zero; + var keys = b.openFor(if (c.indexed) b.constant(.i32, 0) else begin, if (c.indexed) b.constant(.i32, c.topk) else end, 1, initial); + { + const key = if (c.indexed) a.indices.get(.{ token, keys.iv }) else keys.iv; + const valid = key.ge(0).bitAnd(key.lt(len)); + // Invalid indexed entries must not load the page table, K, or V. Duplicates + // deliberately contribute once per occurrence, as independent softmax entries. + var entry = b.openIfElse(valid, @as([chunks + 2]V, @splat(zero))); + { + const page = a.table.get(.{ seq, key.div(c.page_size) }); + const offset = key.rem(c.page_size); + const kv_head = head.div(c.group); + var score = zero; + for (query, 0..) |q, i| score = score.add(q.mul(a.k.get(.{ page, offset, kv_head, lane.add(i * 32) }).to(.f32))); + inline for (.{ 16, 8, 4, 2, 1 }) |step| score = score.add(b.shuffleXor(score, step)); + score = score.mul(c.scale * std.math.log2e); + const maximum = keys.carried[0].maximum(score); + const alpha = b.exp2Approx(keys.carried[0].sub(maximum)); + const weight = b.exp2Approx(score.sub(maximum)); + var next: [chunks + 2]V = undefined; + next[0] = maximum; + next[1] = keys.carried[1].mul(alpha).add(weight); + for (0..chunks) |i| next[i + 2] = keys.carried[i + 2].mul(alpha).add(weight.mul(a.v.get(.{ page, offset, kv_head, lane.add(i * 32) }).to(.f32))); + entry.yieldThen(next); + } + { + entry.yieldElse(keys.carried); + } + keys.yield(entry.results); + } + const norm = b.select(keys.results[1].gt(0), b.rcpApprox(keys.results[1]), zero); + for (0..chunks) |i| a.out.set(.{ token, head, lane.add(i * 32) }, keys.results[i + 2].mul(norm).to(c.dtype)); + active.close(); + } +} diff --git a/zml/kernels/paged_attention/cute/benchmark.zig b/zml/kernels/paged_attention/cute/benchmark.zig new file mode 100644 index 00000000..587af69f --- /dev/null +++ b/zml/kernels/paged_attention/cute/benchmark.zig @@ -0,0 +1,145 @@ +const std = @import("std"); +const zml = @import("zml"); +const cases = @import("cases.zig"); +const fixtures = @import("fixtures.zig"); +const Inputs = fixtures.Inputs; + +pub fn main(init: std.process.Init) !void { + const platform = try zml.Platform.auto(init.gpa, init.io, .{}); + defer platform.deinit(init.gpa, init.io); + + if (platform.target != .cuda) return error.CudaRequired; + + const mesh = try platform.registerMesh("tp", .mesh(.{ .model = .high_bandwidth })); + + const cc = zml.platform.cuda.computeCapability(platform) orelse return error.UnknownComputeCapability; + std.debug.print("PAGED_ATTENTION dtype=bf16 page_size=16 seed=42 cc={d}.{d}\n{f}\n", .{ + cc.major, + cc.minor, + platform.fmtVerbose(), + }); + + std.debug.print("timing=synchronized_invocation_us devices={d} hkv=per_device (KV heads sharded across all visible GPUs; includes runtime, output allocation, attention, merge and wait; excludes compile, upload and output destruction)\n", .{platform.devices.len}); + var failures: usize = 0; + for (cases.benchmarks) |c| { + std.debug.print("CASE name={s} d={d} b={d} hkv={d} g={d} prefill={d} chunk={d} kv={d} mask={s} backend=cute\n", .{ + c.name, + c.head_dim, + c.batch, + c.kv_heads, + c.group, + c.prefill, + if (c.prefill > 0) c.chunk else 1, + c.kv_len, + @tagName(c.mask), + }); + + runBenchmark(init.gpa, init.io, platform, mesh, c) catch |err| { + failures += 1; + std.debug.print("FAIL error={s}\n", .{@errorName(err)}); + }; + } + + if (failures != 0) return error.BenchmarkFailed; +} + +fn runBenchmark(a: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mesh: *const zml.Mesh, c: cases.Case) !void { + // Keep number of head per GPU constant. + var global_case = c; + global_case.kv_heads *= platform.devices.len; + var inputs = Inputs.init(global_case); + inputs.q = inputs.q.withPartitioning(mesh, .{ .hkv = .model }); + inputs.k = inputs.k.withPartitioning(mesh, .{ .hkv = .model }); + inputs.v = inputs.v.withPartitioning(mesh, .{ .hkv = .model }); + inputs.sinks = inputs.sinks.withPartitioning(mesh, .{ ._0 = .model }); + + const compile_start: std.Io.Timestamp = .now(io, .awake); + const exe = try platform.compileFn(a, io, forward, .{ inputs, global_case, mesh }, .{ + .program_name = "paged_attention_harness", + .meshes = &.{mesh}, + }); + defer exe.deinit(); + const compile_us = elapsedUs(compile_start, io); + + var runner = try exe.runner(a); + defer runner.deinit(a); + + var arena = std.heap.ArenaAllocator.init(a); + defer arena.deinit(); + + const fixture = try fixtures.Fixture.init(arena.allocator(), global_case); + + var buffers = try fixtures.upload(io, platform, inputs, fixture); + defer zml.Buffer.deinitAll(Inputs, &buffers); + + var output: zml.Buffer = undefined; + + // Warmup + for (0..5) |_| { + runner.run(io, .{buffers}, .{&output}, .{ .wait = true }); + output.deinit(); + } + + var samples: [101]f64 = undefined; + var sample_count: usize = 0; + var accumulated_us: f64 = 0; + while (sample_count < samples.len and (sample_count < 21 or accumulated_us < 100_000)) { + const call_start: std.Io.Timestamp = .now(io, .awake); + runner.run(io, .{buffers}, .{&output}, .{ .wait = true }); + samples[sample_count] = elapsedUs(call_start, io); + accumulated_us += samples[sample_count]; + sample_count += 1; + output.deinit(); + } + + std.mem.sort(f64, samples[0..sample_count], {}, std.sort.asc(f64)); + const median = if (sample_count % 2 == 0) + (samples[sample_count / 2 - 1] + samples[sample_count / 2]) / 2 + else + samples[sample_count / 2]; + + std.debug.print("RESULT name={s} d={d} b={d} hkv={d} g={d} prefill={d} chunk={d} kv={d} backend=cute compile_ms={d:.3} n={d} median_us={d:.3} p10_us={d:.3} p90_us={d:.3}\n", .{ + c.name, + c.head_dim, + c.batch, + c.kv_heads, + c.group, + c.prefill, + if (c.prefill > 0) c.chunk else 1, + c.kv_len, + compile_us / 1000, + sample_count, + median, + samples[sample_count / 10], + samples[(sample_count - 1) * 9 / 10], + }); +} + +fn forward(inputs: Inputs, c: cases.Case, mesh: *const zml.Mesh) zml.Tensor { + const Call = struct { + inputs: Inputs, + case: cases.Case, + + fn local(self: @This(), _: zml.Shape) zml.Tensor { + return fixtures.forward(self.inputs, self.case); + } + }; + const heads: zml.Sharding.Partitioning.Logical = .parse(.{ .hkv = .model }); + return zml.ops.manualComputation(Call.local, .{ .inputs = inputs, .case = c }, inputs.q.shape(), mesh, .{ + .in_specs = .{ .inputs = .{ + .q = heads, + .k = heads, + .v = heads, + .table = .replicated, + .lengths = .replicated, + .starts = .replicated, + .sinks = .parse(.{ ._0 = .model }), + .indices = .replicated, + } }, + .out_specs = heads, + }); +} + +fn elapsedUs(start: std.Io.Timestamp, io: std.Io) f64 { + return @as(f64, @floatFromInt(start.untilNow(io, .awake).toNanoseconds())) / 1000; +} diff --git a/zml/kernels/paged_attention/cute/cases.zig b/zml/kernels/paged_attention/cute/cases.zig new file mode 100644 index 00000000..e8dedeaa --- /dev/null +++ b/zml/kernels/paged_attention/cute/cases.zig @@ -0,0 +1,103 @@ +const std = @import("std"); + +pub const page_size = 16; +pub const topk = 8; +pub const Mask = enum { causal, non_causal, window, non_causal_window, indices }; + +pub const Case = struct { + name: []const u8 = "standard", + head_dim: usize = 128, + batch: usize = 2, + kv_heads: usize = 2, + group: usize = 4, + prefill: usize = 0, + chunk: usize = 17, + kv_len: usize = 65, + kv_capacity: ?usize = null, + inactive: usize = 0, + shuffle_pages: bool = false, + skew: bool = false, + mask: Mask = .causal, + empty_indices: bool = false, + window: u32 = 17, + sink: bool = false, + scale: ?f32 = null, + + pub fn tokens(c: Case) usize { + return c.prefill * c.chunk + c.batch - c.prefill - c.inactive; + } + pub fn pages(c: Case) usize { + return std.math.divCeil(usize, c.kv_capacity orelse c.kv_len, page_size) catch unreachable; + } + pub fn heads(c: Case) usize { + return c.kv_heads * c.group; + } + pub fn queryLen(c: Case, seq: usize) usize { + return if (seq >= c.batch - c.inactive) 0 else if (seq < c.prefill) c.chunk else 1; + } + pub fn kvLen(c: Case, seq: usize) usize { + const qlen = c.queryLen(seq); + if (qlen == 0) return 0; + const reduction = if (c.skew) (seq * 17) % c.kv_len else 0; + return @max(qlen, c.kv_len - @min(reduction, c.kv_len - 1)); + } +}; + +// Eight geometry cases plus orthogonal semantics/boundary cases, not a Cartesian product. +pub const correctness = blk: { + var result: [34]Case = undefined; + var i: usize = 0; + for ([_]usize{ 64, 128, 256, 512 }) |dim| { + for ([_]bool{ false, true }) |mixed| { + result[i] = .{ .head_dim = dim, .prefill = if (mixed) 1 else 0 }; + i += 1; + } + } + result[8] = .{ .name = "mha_page_tail", .group = 1, .kv_len = 15 }; + result[9] = .{ .name = "mqa_page_exact", .kv_heads = 1, .group = 8, .kv_len = 16 }; + result[10] = .{ .name = "non_causal", .head_dim = 256, .prefill = 1, .mask = .non_causal }; + result[11] = .{ .name = "window_sink", .head_dim = 256, .prefill = 1, .mask = .window, .sink = true, .scale = 0.125 }; + result[12] = .{ .name = "indices", .head_dim = 256, .prefill = 1, .mask = .indices, .sink = true }; + result[13] = .{ .name = "long_decode", .head_dim = 512, .kv_heads = 1, .group = 2, .batch = 1, .kv_len = 2049 }; + result[14] = .{ .name = "skew", .head_dim = 256, .batch = 4, .prefill = 2, .skew = true, .kv_len = 129 }; + result[15] = .{ .name = "mixed_short_chunks", .batch = 4, .prefill = 2, .chunk = 16, .skew = true, .kv_len = 64 }; + result[16] = .{ .name = "empty_indices", .prefill = 1, .mask = .indices, .empty_indices = true }; + result[17] = .{ .name = "d32", .head_dim = 32, .group = 1, .kv_len = 33 }; + result[18] = .{ .name = "odd_group", .group = 5, .prefill = 1, .chunk = 9, .kv_len = 79 }; + result[19] = .{ .name = "group_tile_tail", .head_dim = 256, .batch = 1, .group = 19, .kv_len = 257 }; + result[20] = .{ .name = "shuffled_mixed", .head_dim = 512, .batch = 4, .prefill = 2, .group = 3, .chunk = 11, .kv_len = 129, .shuffle_pages = true }; + result[21] = .{ .name = "split_sink", .head_dim = 512, .batch = 1, .kv_len = 4097, .sink = true, .shuffle_pages = true }; + result[22] = .{ .name = "split_window_sink", .head_dim = 256, .batch = 1, .kv_len = 4097, .mask = .window, .window = 257, .sink = true }; + result[23] = .{ .name = "inactive_sequence", .batch = 4, .prefill = 1, .inactive = 1, .kv_len = 129 }; + result[24] = .{ .name = "non_causal_window", .head_dim = 64, .prefill = 1, .kv_len = 65, .mask = .non_causal_window, .window = 7 }; + result[25] = .{ .name = "empty_splits", .batch = 1, .kv_len = 1, .kv_capacity = 4096, .sink = true }; + result[26] = .{ .name = "d32_mixed", .head_dim = 32, .group = 3, .prefill = 1, .chunk = 9, .kv_len = 257, .shuffle_pages = true }; + result[27] = .{ .name = "d64_group_tail", .head_dim = 64, .group = 19, .kv_len = 257 }; + result[28] = .{ .name = "split_mixed_sink", .head_dim = 128, .batch = 1, .prefill = 1, .chunk = 3, .kv_len = 4097, .sink = true, .shuffle_pages = true }; + result[29] = .{ .name = "d512_prefill", .head_dim = 512, .batch = 2, .prefill = 2, .chunk = 65, .kv_len = 257, .sink = true, .shuffle_pages = true }; + result[30] = .{ .name = "negative_scale", .head_dim = 128, .prefill = 1, .kv_len = 257, .mask = .non_causal, .scale = -0.125 }; + result[31] = .{ .name = "zero_scale", .head_dim = 256, .prefill = 1, .kv_len = 257, .sink = true, .scale = 0 }; + result[32] = .{ .name = "d512_window_prefill", .head_dim = 512, .batch = 1, .prefill = 1, .chunk = 33, .kv_len = 513, .mask = .window, .window = 129, .sink = true }; + result[33] = .{ .name = "prefill_inactive", .head_dim = 64, .batch = 4, .prefill = 2, .inactive = 2, .kv_len = 257, .group = 5 }; + break :blk result; +}; + +pub const benchmarks = blk: { + var result: [24]Case = undefined; + var i: usize = 0; + for ([_]usize{ 64, 128, 256, 512 }) |dim| { + for ([_]usize{ 2, 8, 32 }) |batch| { + for ([_]bool{ false, true }) |mixed| { + result[i] = .{ + .head_dim = dim, + .batch = batch, + .prefill = if (mixed) @max(1, batch / 4) else 0, + .chunk = 64, + .kv_len = 2048, + }; + i += 1; + } + } + } + break :blk result; +}; diff --git a/zml/kernels/paged_attention/cute/correctness_test.zig b/zml/kernels/paged_attention/cute/correctness_test.zig new file mode 100644 index 00000000..436b9d3a --- /dev/null +++ b/zml/kernels/paged_attention/cute/correctness_test.zig @@ -0,0 +1,93 @@ +const reference = @import("reference.zig"); +const std = @import("std"); +const zml = @import("zml"); +const cases = @import("cases.zig"); +const fixtures = @import("fixtures.zig"); +const Inputs = fixtures.Inputs; + +test "CPU reference self checks" { + _ = @import("reference.zig"); +} + +test "paged attention correctness" { + const platform = zml.testing.env(); + if (platform.target != .cuda) return error.CudaRequired; + + const a = std.testing.allocator; + const io = std.testing.io; + + const cc = zml.platform.cuda.computeCapability(platform) orelse return error.UnknownComputeCapability; + std.debug.print("PAGED_ATTENTION dtype=bf16 page_size=16 seed=42 cc={d}.{d}\n{f}\n", .{ + cc.major, cc.minor, platform.fmtVerbose(), + }); + + var failures: usize = 0; + for (cases.correctness) |c| { + std.debug.print("CASE name={s} d={d} b={d} hkv={d} g={d} prefill={d} chunk={d} kv={d} mask={s} backend=cute\n", .{ + c.name, c.head_dim, c.batch, c.kv_heads, c.group, c.prefill, if (c.prefill > 0) c.chunk else 1, c.kv_len, @tagName(c.mask), + }); + + runCorrectness(a, io, platform, c, false) catch |err| { + failures += 1; + std.debug.print("FAIL error={s}\n", .{@errorName(err)}); + }; + } + + try std.testing.expectEqual(@as(usize, 0), failures); +} + +test "FP16 paged attention" { + const platform = zml.testing.env(); + if (platform.target != .cuda) return error.CudaRequired; + for ([_]cases.Case{ + .{ .name = "fp16_decode", .head_dim = 64, .kv_len = 257 }, + .{ .name = "fp16_mixed", .prefill = 1, .mask = .non_causal }, + .{ .name = "fp16_window_sink", .head_dim = 256, .prefill = 1, .mask = .window, .sink = true }, + .{ .name = "fp16_split_sink", .head_dim = 512, .batch = 1, .kv_len = 2049, .sink = true, .shuffle_pages = true }, + }) |c| try runCorrectness(std.testing.allocator, std.testing.io, platform, c, true); +} + +fn forwardF16(i: Inputs, c: cases.Case) zml.Tensor { + var inputs = i; + inputs.q = i.q.convert(.f16); + inputs.k = i.k.convert(.f16); + inputs.v = i.v.convert(.f16); + return fixtures.forward(inputs, c).convert(.bf16); +} + +fn runCorrectness(a: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, c: cases.Case, comptime compute_f16: bool) !void { + const exe = try platform.compileFn(a, io, if (compute_f16) forwardF16 else fixtures.forward, .{ Inputs.init(c), c }, .{ .program_name = "paged_attention_harness" }); + defer exe.deinit(); + + var runner = try exe.runner(a); + defer runner.deinit(a); + + var arena = std.heap.ArenaAllocator.init(a); + defer arena.deinit(); + + const fixture = try fixtures.Fixture.init(arena.allocator(), c); + if (compute_f16) { + // Exactly representable in both formats, so the existing CPU reference + // receives precisely the values used by the FP16 tensor-core path. + for ([_][]zml.floats.BFloat16{ fixture.q, fixture.k, fixture.v }) |values| { + for (values) |*v| if (std.math.isFinite(v.toF32())) { + v.* = .fromF32(@round(v.toF32() * 64) / 64); + }; + } + } + + var buffers = try fixtures.upload(io, platform, Inputs.init(c), fixture); + defer zml.Buffer.deinitAll(Inputs, &buffers); + + var output: zml.Buffer = undefined; + runner.run(io, .{buffers}, .{&output}, .{ .wait = true }); + defer output.deinit(); + + var host = try output.toSliceAlloc(a, io); + defer host.free(a); + + const max_error = try reference.check(a, c, fixture, host.items(zml.floats.BFloat16)); + std.debug.print("RESULT name={s} d={d} b={d} hkv={d} g={d} prefill={d} chunk={d} kv={d} backend=cute compute_dtype={s} check=all_outputs max_abs={d:.6}\n", .{ + c.name, c.head_dim, c.batch, c.kv_heads, c.group, c.prefill, if (c.prefill > 0) c.chunk else 1, c.kv_len, if (compute_f16) "f16" else "bf16", max_error, + }); +} diff --git a/zml/kernels/paged_attention/cute/cute.zig b/zml/kernels/paged_attention/cute/cute.zig new file mode 100644 index 00000000..ef78cc49 --- /dev/null +++ b/zml/kernels/paged_attention/cute/cute.zig @@ -0,0 +1,115 @@ +const stdx = @import("stdx"); + +const zml = @import("../../../zml.zig"); +pub const AttentionOptions = @import("../options.zig").AttentionOptions; +const kernel = @import("attention.zig"); +const sm10x = @import("sm10x/sm10x.zig"); +const sm120 = @import("sm120/sm120.zig"); + +pub const Options = struct { + batch_size: usize, + max_num_pages: usize, + max_seqlen_q: usize, + is_prefill: bool, + + pub fn isPrefill(self: Options) bool { + return self.is_prefill; + } + + pub fn maxNumPages(self: Options) usize { + return self.max_num_pages; + } +}; + +pub const Parameters = struct { + block_table: zml.Tensor, + seq_lens: zml.Tensor, + query_start_len: zml.Tensor, + options_: Options, + + pub fn init(options_: Options) Parameters { + return .{ + .block_table = .init(.{ .b = options_.batch_size, .p = options_.max_num_pages }, .i32), + .seq_lens = .init(.{ .b = options_.batch_size }, .i32), + .query_start_len = .init(.{ .b = options_.batch_size + 1 }, .i32), + .options_ = options_, + }; + } + + pub fn allocationSize(self: Parameters) usize { + return self.block_table.byteSize() + self.seq_lens.byteSize() + self.query_start_len.byteSize(); + } + + pub fn options(self: Parameters) Options { + return self.options_; + } +}; + +// TODO(Corentin): This should be moved in the platform probably +pub fn multiprocessorCount(platform: *const zml.Platform) usize { + const attribute = platform.devices[0].pjrt_desc.attribute(platform.pjrt_api, "core_count") orelse return 1; + if (attribute.int64 <= 0) return 1; + return @intCast(attribute.int64); +} + +test { + _ = kernel; + _ = sm10x; + _ = sm120; +} + +pub fn isAvailable(platform: *const zml.Platform) bool { + if (platform.target != .cuda) return false; + const cc = zml.platform.cuda.computeCapability(platform) orelse return false; + return cc.major > 8 or (cc.major == 8 and cc.minor >= 9); +} + +pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: AttentionOptions) zml.Tensor { + stdx.debug.assert(isAvailable(zml.Compiler.current().platform), "CuTe paged attention requires SM89 or newer", .{}); + stdx.debug.assert(q.dtype() == k.dtype() and q.dtype() == v.dtype() and (q.dtype() == .bf16 or q.dtype() == .f16), "CuTe paged attention requires matching BF16 or FP16 Q/K/V", .{}); + stdx.debug.assert(q.dim(.hd) == 32 or q.dim(.hd) == 64 or q.dim(.hd) == 128 or q.dim(.hd) == 256 or q.dim(.hd) == 512, "CuTe paged attention supports head dimensions 32, 64, 128, 256 and 512", .{}); + if (sm10x.isAvailable(zml.Compiler.current().platform) and opts.mask != .indices) + return sm10x.pagedAttention(parameters, q, k, v, opts); + if (sm120.isAvailable(zml.Compiler.current().platform) and opts.mask != .indices) + return sm120.pagedAttention(parameters, q, k, v, opts); + const q_ = q.transpose(.{ .b, .hkv, .hg, .hd }); + const k_ = k.transpose(.{ .page, .k_chunk, .hkv, .hd }); + const v_ = v.transpose(.{ .page, .k_chunk, .hkv, .hd }); + stdx.debug.assert(q_.dim(.hkv) == k_.dim(.hkv) and k_.shape().eql(v_.shape()) and k_.dim(.hd) == q_.dim(.hd) and q_.dim(.hg) > 0, "CuTe paged attention requires aligned local Q/KV heads and matching caches", .{}); + const indices = switch (opts.mask) { + .indices => |indices| indices.transpose(.{ .b, .topk }), + else => zml.Tensor.zeroes(.init(.{ q_.dim(.b), 1 }, .i32)), + }; + const cfg: kernel.Config = .{ + .head_dim = q_.dim(.hd), + .tokens = q_.dim(.b), + .batch = parameters.block_table.dim(.b), + .kv_heads = k_.dim(.hkv), + .group = q_.dim(.hg), + .pages = k_.dim(.page), + .pages_per_seq = parameters.block_table.dim(.p), + .page_size = k_.dim(.k_chunk), + .dtype = zml.kernel.cute.from(q_.dtype()), + .causal = opts.mask.isCausal(), + .window = opts.mask.slidingWindow() orelse 0, + .scale = opts.scale orelse (1 / @sqrt(@as(f32, @floatFromInt(q_.dim(.hd))))), + .sink = opts.sink != null, + .indexed = opts.mask == .indices, + .topk = indices.dim(-1), + }; + const sink = if (opts.sink) |s| s.convert(.f32).reshape(.{cfg.kv_heads * cfg.group}) else zml.Tensor.zeroes(.init(.{cfg.kv_heads * cfg.group}, .f32)); + return kernel.Kernel.call(.{ + .q = q_, + .k = k_, + .v = v_, + .table = parameters.block_table, + .lengths = parameters.seq_lens, + .starts = parameters.query_start_len, + .sink = sink, + .indices = indices, + }, .{ .out = q_.shape() }, .{ + .cfg = cfg, + .grid = .{ @intCast(cfg.tokens), @intCast(cfg.kv_heads * cfg.group), 1 }, + .block = .{ 32, 1, 1 }, + }).out.transpose(q.shape()); +} diff --git a/zml/kernels/paged_attention/cute/fixtures.zig b/zml/kernels/paged_attention/cute/fixtures.zig new file mode 100644 index 00000000..c0815d1d --- /dev/null +++ b/zml/kernels/paged_attention/cute/fixtures.zig @@ -0,0 +1,141 @@ +const std = @import("std"); + +const zml = @import("zml"); + +const cases = @import("cases.zig"); + +pub const Fixture = struct { + q: []zml.floats.BFloat16, + k: []zml.floats.BFloat16, + v: []zml.floats.BFloat16, + table: []i32, + lengths: []i32, + starts: []i32, + sinks: []f32, + indices: []i32, + + // The caller owns an arena; all arrays live until the case is complete. + pub fn init(a: std.mem.Allocator, c: cases.Case) !Fixture { + const f: Fixture = .{ + .q = try a.alloc(zml.floats.BFloat16, c.tokens() * c.heads() * c.head_dim), + .k = try a.alloc(zml.floats.BFloat16, c.batch * c.pages() * cases.page_size * c.kv_heads * c.head_dim), + .v = try a.alloc(zml.floats.BFloat16, c.batch * c.pages() * cases.page_size * c.kv_heads * c.head_dim), + .table = try a.alloc(i32, c.batch * c.pages()), + .lengths = try a.alloc(i32, c.batch), + .starts = try a.alloc(i32, c.batch + 1), + .sinks = try a.alloc(f32, c.heads()), + .indices = try a.alloc(i32, c.tokens() * cases.topk), + }; + var rng = std.Random.DefaultPrng.init(42); + const random = rng.random(); + for (f.q) |*v| v.* = .fromF32(random.float(f32) * 2 - 1); + for (f.k) |*v| v.* = .fromF32(random.float(f32) * 2 - 1); + for (f.v) |*v| v.* = .fromF32(random.float(f32) * 2 - 1); + for (f.sinks, 0..) |*v, h| v.* = @as(f32, @floatFromInt(h % 5)) - 2; + @memset(f.indices, -1); + f.starts[0] = 0; + for (0..c.batch) |s| { + const len = c.kvLen(s); + f.lengths[s] = @intCast(len); + f.starts[s + 1] = f.starts[s] + @as(i32, @intCast(c.queryLen(s))); + for (0..c.pages()) |p| { + f.table[s * c.pages() + p] = if (p * cases.page_size < len) @intCast(s * c.pages() + p) else -1; + } + const live_pages = std.math.divCeil(usize, len, cases.page_size) catch unreachable; + if (c.shuffle_pages) random.shuffle(i32, f.table[s * c.pages() ..][0..live_pages]); + // Unused physical slots must not contribute to PV, even via zero * NaN. + for (len..c.pages() * cases.page_size) |j| { + const page = if (j / cases.page_size < live_pages) + @as(usize, @intCast(f.table[s * c.pages() + j / cases.page_size])) + else + s * c.pages() + j / cases.page_size; + const offset = (page * cases.page_size + j % cases.page_size) * c.kv_heads * c.head_dim; + @memset(f.k[offset..][0 .. c.kv_heads * c.head_dim], .fromF32(std.math.nan(f32))); + @memset(f.v[offset..][0 .. c.kv_heads * c.head_dim], .fromF32(std.math.nan(f32))); + } + for (@intCast(f.starts[s])..@intCast(f.starts[s + 1])) |t| { + const l: i32 = @intCast(len); + @memcpy(f.indices[t * cases.topk ..][0..cases.topk], &[_]i32{ 0, l - 1, @divTrunc(l, 2), -1, l, 0, 1, l - 2 }); + } + } + if (c.empty_indices) { + for (f.indices, 0..) |*index, i| index.* = if (i % 2 == 0) -1 else @intCast(c.kv_len); + @memset(f.k, .fromF32(std.math.nan(f32))); + @memset(f.v, .fromF32(std.math.nan(f32))); + } + return f; + } + + pub fn kvOffset(f: Fixture, c: cases.Case, seq: usize, key: usize, head: usize) usize { + const page: usize = @intCast(f.table[seq * c.pages() + key / cases.page_size]); + return ((page * cases.page_size + key % cases.page_size) * c.kv_heads + head) * c.head_dim; + } +}; + +pub const Inputs = struct { + q: zml.Tensor, + k: zml.Tensor, + v: zml.Tensor, + table: zml.Tensor, + lengths: zml.Tensor, + starts: zml.Tensor, + sinks: zml.Tensor, + indices: zml.Tensor, + + pub fn init(c: cases.Case) Inputs { + return .{ + .q = .init(.{ .b = c.tokens(), .hkv = c.kv_heads, .hg = c.group, .hd = c.head_dim }, .bf16), + .k = .init(.{ .page = c.batch * c.pages(), .k_chunk = cases.page_size, .hkv = c.kv_heads, .hd = c.head_dim }, .bf16), + .v = .init(.{ .page = c.batch * c.pages(), .k_chunk = cases.page_size, .hkv = c.kv_heads, .hd = c.head_dim }, .bf16), + .table = .init(.{ .b = c.batch, .p = c.pages() }, .i32), + .lengths = .init(.{ .b = c.batch }, .i32), + .starts = .init(.{ .b = c.batch + 1 }, .i32), + .sinks = .init(.{c.heads()}, .f32), + .indices = .init(.{ .b = c.tokens(), .topk = cases.topk }, .i32), + }; + } + + fn options(i: Inputs, c: cases.Case) zml.paged_attention.AttentionOptions { + return .{ + .mask = switch (c.mask) { + .causal => .{ .standard = .{} }, + .non_causal => .none, + .window => .{ .standard = .{ .sliding_window = c.window } }, + .non_causal_window => .{ .standard = .{ .causal = false, .sliding_window = c.window } }, + .indices => .{ .indices = i.indices }, + }, + .sink = if (c.sink) i.sinks else null, + .scale = c.scale, + }; + } +}; + +pub fn forward(i: Inputs, c: cases.Case) zml.Tensor { + const parameters: zml.paged_attention.Parameters = .{ + .cute = .{ + .block_table = i.table, + .seq_lens = i.lengths, + .query_start_len = i.starts, + .options_ = .{ + .batch_size = c.batch, + .max_num_pages = c.pages(), + .max_seqlen_q = if (c.prefill > 0) c.chunk else 1, + .is_prefill = c.prefill > 0, + }, + }, + }; + return zml.paged_attention.cute.pagedAttention(parameters.cute, i.q, i.k, i.v, i.options(c)); +} + +pub fn upload(io: std.Io, platform: *const zml.Platform, inputs: Inputs, f: Fixture) !zml.Bufferized(Inputs) { + var result: zml.Bufferized(Inputs) = undefined; + var initialized: usize = 0; + errdefer inline for (@typeInfo(Inputs).@"struct".field_names, 0..) |name, index| { + if (index < initialized) @field(result, name).deinit(); + }; + inline for (@typeInfo(Inputs).@"struct".field_names) |name| { + @field(result, name) = try .fromBytes(io, platform, @field(inputs, name).shape(), std.mem.sliceAsBytes(@field(f, name))); + initialized += 1; + } + return result; +} diff --git a/zml/kernels/paged_attention/cute/reference.zig b/zml/kernels/paged_attention/cute/reference.zig new file mode 100644 index 00000000..1a030646 --- /dev/null +++ b/zml/kernels/paged_attention/cute/reference.zig @@ -0,0 +1,127 @@ +const std = @import("std"); +const zml = @import("zml"); +const cases = @import("cases.zig"); +const fixtures = @import("fixtures.zig"); + +/// One query/head, independent of GPU tensor operations. Duplicate indices are +/// separate softmax entries; invalid indices are excluded. +pub fn row(c: cases.Case, f: fixtures.Fixture, seq: usize, token: usize, head: usize, scores: []f32, out: []f32) void { + const len: usize = @intCast(f.lengths[seq]); + const start: usize = @intCast(f.starts[seq]); + const qlen: usize = @intCast(f.starts[seq + 1] - f.starts[seq]); + const position = len - qlen + token - start; + const count = if (c.mask == .indices) cases.topk else len; + const scale = c.scale orelse 1 / @sqrt(@as(f32, @floatFromInt(c.head_dim))); + const q = f.q[(token * c.heads() + head) * c.head_dim ..][0..c.head_dim]; + var maximum: f32 = if (c.sink) f.sinks[head] else -std.math.inf(f32); + for (0..count) |j| { + const key = keyAt(c, f, token, j); + const valid = key >= 0 and key < len and switch (c.mask) { + .indices, .non_causal => true, + .causal => key <= position, + .window => key <= position and @as(i64, @intCast(key)) > @as(i64, @intCast(position)) - c.window, + .non_causal_window => @as(i64, @intCast(key)) > @as(i64, @intCast(position)) - c.window, + }; + scores[j] = -std.math.inf(f32); + if (!valid) continue; + const offset = f.kvOffset(c, seq, @intCast(key), head / c.group); + var dot: f32 = 0; + for (q, f.k[offset..][0..c.head_dim]) |qv, kv| dot += qv.toF32() * kv.toF32(); + scores[j] = dot * scale; + maximum = @max(maximum, scores[j]); + } + @memset(out, 0); + if (maximum == -std.math.inf(f32)) return; + var denominator: f32 = if (c.sink) @exp(f.sinks[head] - maximum) else 0; + for (0..count) |j| { + if (scores[j] == -std.math.inf(f32)) continue; + const weight = @exp(scores[j] - maximum); + denominator += weight; + const offset = f.kvOffset(c, seq, @intCast(keyAt(c, f, token, j)), head / c.group); + for (out, f.v[offset..][0..c.head_dim]) |*o, v| o.* += weight * v.toF32(); + } + for (out) |*v| v.* /= denominator; +} + +fn keyAt(c: cases.Case, f: fixtures.Fixture, token: usize, j: usize) i64 { + return if (c.mask == .indices) f.indices[token * cases.topk + j] else @intCast(j); +} + +pub fn check(a: std.mem.Allocator, c: cases.Case, f: fixtures.Fixture, actual: []const zml.floats.BFloat16) !f32 { + const active_tokens: usize = @intCast(f.starts[c.batch]); + for (actual[0 .. active_tokens * c.heads() * c.head_dim]) |v| { + if (!std.math.isFinite(v.toF32())) return error.NonFiniteOutput; + } + const scores = try a.alloc(f32, @max(c.kv_len, cases.topk)); + defer a.free(scores); + const expected = try a.alloc(f32, c.head_dim); + defer a.free(expected); + var max_error: f32 = 0; + var failures: usize = 0; + for (0..c.batch) |s| { + for (@intCast(f.starts[s])..@intCast(f.starts[s + 1])) |t| { + for (0..c.heads()) |h| { + row(c, f, s, t, h, scores, expected); + const base = (t * c.heads() + h) * c.head_dim; + for (expected, actual[base..][0..c.head_dim], 0..) |want, got_bf16, d| { + const got = got_bf16.toF32(); + const err = @abs(want - got); + max_error = @max(max_error, err); + // Fixed BF16 error budget; never adapted to observed failures. + if (!std.math.isFinite(want) or err > 0.003 + 0.02 * @abs(want)) { + if (failures == 0) std.debug.print("Mismatch seq={d} token={d} head={d} dim={d}: expected={d} actual={d}\n", .{ s, t, h, d, want, got }); + failures += 1; + } + } + } + } + } + if (failures > 0) return error.ReferenceMismatch; + return max_error; +} + +test "reference single key, uniform causal scores, and sink" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + var c: cases.Case = .{ .head_dim = 64, .batch = 1, .kv_heads = 1, .group = 1, .prefill = 1, .chunk = 2, .kv_len = 2 }; + const f = try fixtures.Fixture.init(arena.allocator(), c); + @memset(f.q, .fromF32(0)); + @memset(f.v[0..64], .fromF32(2)); + @memset(f.v[64..128], .fromF32(4)); + var scores: [2]f32 = undefined; + var out: [64]f32 = undefined; + row(c, f, 0, 0, 0, &scores, &out); + for (out) |v| try std.testing.expectEqual(@as(f32, 2), v); + row(c, f, 0, 1, 0, &scores, &out); + for (out) |v| try std.testing.expectEqual(@as(f32, 3), v); + c.sink = true; + f.sinks[0] = 0; + row(c, f, 0, 1, 0, &scores, &out); + for (out) |v| try std.testing.expectEqual(@as(f32, 2), v); +} + +test "reference windows, noncausal, duplicate and invalid indices" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + var c: cases.Case = .{ .head_dim = 64, .batch = 1, .kv_heads = 1, .group = 1, .prefill = 1, .chunk = 2, .kv_len = 2 }; + const f = try fixtures.Fixture.init(arena.allocator(), c); + @memset(f.q, .fromF32(0)); + @memset(f.v[0..64], .fromF32(2)); + @memset(f.v[64..128], .fromF32(4)); + var scores: [cases.topk]f32 = undefined; + var out: [64]f32 = undefined; + c.mask = .non_causal; + row(c, f, 0, 0, 0, &scores, &out); + for (out) |v| try std.testing.expectEqual(@as(f32, 3), v); + c.mask = .window; + c.window = 1; + row(c, f, 0, 1, 0, &scores, &out); + for (out) |v| try std.testing.expectEqual(@as(f32, 4), v); + c.mask = .indices; + @memset(f.indices, -1); + @memcpy(f.indices[0..4], &[_]i32{ 0, 0, 1, 2 }); + row(c, f, 0, 0, 0, &scores, &out); + for (out) |v| try std.testing.expectApproxEqAbs(@as(f32, 8.0 / 3.0), v, 1e-6); + row(c, f, 0, 1, 0, &scores, &out); + for (out) |v| try std.testing.expectEqual(@as(f32, 0), v); +} diff --git a/zml/kernels/paged_attention/cute/sm10x/BUILD.bazel b/zml/kernels/paged_attention/cute/sm10x/BUILD.bazel new file mode 100644 index 00000000..05cd7edc --- /dev/null +++ b/zml/kernels/paged_attention/cute/sm10x/BUILD.bazel @@ -0,0 +1,10 @@ +package(default_visibility = ["//zml/kernels/paged_attention/cute:__pkg__"]) + +filegroup( + name = "sm10x", + srcs = [ + "attention.zig", + "sm103.zig", + "sm10x.zig", + ], +) diff --git a/zml/kernels/paged_attention/cute/sm10x/attention.zig b/zml/kernels/paged_attention/cute/sm10x/attention.zig new file mode 100644 index 00000000..12d6273c --- /dev/null +++ b/zml/kernels/paged_attention/cute/sm10x/attention.zig @@ -0,0 +1,667 @@ +const std = @import("std"); + +const zml = @import("../../../../zml.zig"); +const cute = zml.kernel.cute; +const B = cute.Builder; +const V = cute.Value; +const sm103 = @import("sm103.zig"); + +pub const Architecture = enum { sm100, sm103 }; + +pub const Config = struct { + architecture: Architecture, + head_dim: i64, + tokens: i64, + batch: i64, + kv_heads: i64, + group: i64, + pages_per_seq: i64, + page_size: i64, + max_query: i64, + dtype: cute.DType, + decode: bool, + causal: bool, + window: i64, + scale: f32, + sink: bool, + splits: i64 = 1, + + pub fn rows(c: Config) i64 { + return if (c.decode) (if (c.group <= 8) 8 else 16) else 64; + } + pub fn keys(c: Config) i64 { + return if (c.decode) 128 else 64; + } + pub fn queryTiles(c: Config) i64 { + return @divTrunc(c.max_query * c.group + c.rows() - 1, c.rows()); + } + fn paddedDim(c: Config) i64 { + return @max(128, c.head_dim); + } + fn features(c: Config) i64 { + return @divExact(c.paddedDim(), 128); + } + fn buffers(c: Config) i64 { + return if (c.decode) 2 else 3; + } + fn kvBytes(c: Config) i64 { + return c.keys() * 128 * 2; + } + fn qBase(c: Config) i64 { + return c.buffers() * c.kvBytes(); + } + fn pBase(c: Config) i64 { + return c.qBase() + c.rows() * c.paddedDim() * 2; + } + fn barriers(c: Config) i64 { + return c.pBase() + c.rows() * c.keys() * 2; + } + fn completion(c: Config) i64 { + return c.barriers() + c.buffers() * 16; + } + fn holder(c: Config) i64 { + return c.completion() + 16; + } + fn reductions(c: Config) i64 { + return c.holder() + 16; + } + fn sharedBytes(c: Config) i64 { + return c.reductions() + 4 * 16 * 4; + } + fn scoreColumn(c: Config) i64 { + return if (c.decode) c.features() * c.rows() else @min(c.paddedDim(), 448); + } + fn columns(c: Config) u32 { + return std.math.ceilPowerOfTwoAssert(u32, @intCast(@max(32, c.scoreColumn() + (if (c.decode) c.rows() else c.keys())))); + } +}; + +pub const block: [3]i32 = .{ 256, 1, 1 }; +pub const Attention = cute.Kernel(Config, .{ + .name = "paged_attention_cute", + .inputs = &.{ "q", "k", "v", "table", "lengths", "starts", "sink" }, + .outputs = &.{"out"}, + .run = attention, +}); + +fn loadInt(b: *B, pointer: V, index: V) V { + return b.loadWords(1, pointer.add(index.to(.i64).mul(4)), .global, .{})[0]; +} + +// Each 64-feature plane uses a 128-byte XOR swizzle. The descriptor and +// vector copies use this same layout for Q, K, V and probabilities. +fn swizzle(row: V, col: V, rows: i64) V { + return col.div(64).mul(rows * 128).add(row.mul(128)).add(col.rem(64).div(8).bitXor(row.rem(8)).mul(16)).add(col.rem(8).mul(2)); +} + +fn descriptor(b: *B, address: V, leading_bytes: i64) V { + const bits: u64 = (64 << 32) | (1 << 46) | (@as(u64, 2) << 61) | (@as(u64, @intCast(@divExact(leading_bytes, 16))) << 16); + return address.to(.i64).shrLogical(4).bitAnd(0x3fff).bitOr(b.constant(.i64, @as(i64, @bitCast(bits)))); +} + +fn instruction(c: Config, qk: bool) u32 { + const format: u32 = if (c.dtype == .bf16) 1 else 0; + const m: u32 = if (c.decode) 128 else 64; + const n: u32 = @intCast(if (c.decode) c.rows() else if (qk) c.keys() else 128); + return (1 << 4) | (format << 7) | (format << 10) | + (@as(u32, @intFromBool(c.decode and !qk)) << 15) | + (@as(u32, @intFromBool(!c.decode and !qk)) << 16) | ((n >> 3) << 17) | ((m >> 4) << 24); +} + +const Range = struct { first: V, count: V }; + +// Used by both attention and merge: every output row must merge exactly the +// partitions produced for its query tile, including causal/window boundaries. +fn keyRange(b: *B, c: Config, first_row: V, qlen: V, klen: V) Range { + const first_pos = klen.sub(qlen).add(first_row.div(c.group)); + const last_pos = klen.sub(qlen).add(first_row.add(c.rows() - 1).div(c.group).minimum(qlen.sub(1))); + const begin = if (c.window > 0) first_pos.sub(c.window - 1).maximum(0) else b.constant(.i32, 0); + const end = if (c.causal) last_pos.add(1).minimum(klen) else klen; + const first = begin.div(c.keys()); + return .{ .first = first, .count = end.add(c.keys() - 1).div(c.keys()).sub(first).maximum(0) }; +} + +fn liveSplits(c: Config, tiles: V) V { + return tiles.add(1).div(2).maximum(1).minimum(c.splits); +} + +const Tile = struct { + b: *B, + c: Config, + q: V, + k: V, + v: V, + table: V, + sink: V, + out: V, + shared: V, + tmem: V, + tid: V, + lane: V, + warp: V, + seq: V, + head: V, + first_row: V, + start: V, + qlen: V, + klen: V, + first_key_tile: V, + tiles: V, + split: V, + + fn ready(t: Tile, id: V) V { + return t.shared.add(t.c.barriers()).add(id.rem(t.c.buffers()).mul(8)); + } + fn released(t: Tile, id: V) V { + return t.ready(id).add(t.c.buffers() * 8); + } + fn phase(t: Tile, id: V) V { + return id.div(t.c.buffers()).rem(2); + } + fn valid(t: Tile, row: V, key: V) V { + const pos = t.klen.sub(t.qlen).add(row.div(t.c.group)); + var result = row.lt(t.qlen.mul(t.c.group)).bitAnd(key.lt(t.klen)); + if (t.c.causal) result = result.bitAnd(key.le(pos)); + if (t.c.window > 0) result = result.bitAnd(key.gt(pos.sub(t.c.window))); + return result; + } + fn putProbability(t: Tile, row: V, key: V, value: V) void { + t.b.storeHalf(t.shared.add(t.c.pBase()).add(swizzle(row, key, t.c.rows())), .shared, value.to(t.c.dtype).bitCast(.i16).to(.i32)); + } + fn put(t: Tile, row: V, dim: V, value: V) void { + const token = t.start.add(row.div(t.c.group)); + var index = token.to(.i64).mul(t.c.kv_heads).add(t.head.to(.i64)).mul(t.c.group).add(row.rem(t.c.group).to(.i64)); + if (t.c.splits > 1) { + index = t.split.to(.i64).mul(t.c.tokens * t.c.kv_heads * t.c.group).add(index).mul(t.c.head_dim + 2).add(dim.to(.i64)); + t.b.storeWords(1, t.out.add(index.mul(4)), .global, .{value.bitCast(.i32)}); + } else { + index = index.mul(t.c.head_dim).add(dim.to(.i64)); + t.b.storeHalf(t.out.add(index.mul(2)), .global, value.to(t.c.dtype).bitCast(.i16).to(.i32)); + } + } + fn norm(t: Tile, row: V, maximum: V, denominator: V) V { + const b = t.b; + if (t.c.splits > 1) return b.constant(.f32, 1); + if (!t.c.sink) return b.select(denominator.gt(0), b.rcpApprox(denominator), b.constant(.f32, 0)); + const sink = loadInt(b, t.sink, t.head.mul(t.c.group).add(row.rem(t.c.group))).bitCast(.f32).mul(std.math.log2e); + const reference = maximum.maximum(sink).maximum(-std.math.floatMax(f32)); + const weight = b.exp2Approx(maximum.sub(reference)); + return weight.div(denominator.mul(weight).add(b.exp2Approx(sink.sub(reference)))); + } +}; + +fn attention(b: *B, c: Config) cute.FinishError!void { + const a = try b.declareArgs(.{ + .q = .{ .ptr = c.dtype }, + .k = .{ .ptr = c.dtype }, + .v = .{ .ptr = c.dtype }, + .table = .{ .ptr = cute.DType.i32 }, + .lengths = .{ .ptr = cute.DType.i32 }, + .starts = .{ .ptr = cute.DType.i32 }, + .sink = .{ .ptr = cute.DType.f32 }, + .out = .{ .ptr = if (c.splits > 1) cute.DType.f32 else c.dtype }, + }); + std.debug.assert(c.sharedBytes() <= 227 * 1024); + b.setFunctionAttribute(.named(b.ctx, "cu_attrs", .dict(b.ctx, &.{.named(b.ctx, "max_dynamic_shared_size_bytes", b.devMaxSharedMemoryOptin())}))); + const seq = b.blockIdx().x.div(c.queryTiles()); + const first_row = b.blockIdx().x.rem(c.queryTiles()).mul(c.rows()); + const starts = b.ptrToInt(a.starts, .i64); + const start = loadInt(b, starts, seq); + const qlen = loadInt(b, starts, seq.add(1)).sub(start); + const klen = loadInt(b, b.ptrToInt(a.lengths, .i64), seq); + const range = keyRange(b, c, first_row, qlen, klen); + const splits = liveSplits(c, range.count); + const split = b.blockIdx().z; + var active = b.openIf(first_row.lt(qlen.mul(c.group)).bitAnd(split.lt(splits))); + { + const tid = b.threadIdx().x; + const lo = range.count.mul(split).div(splits); + const hi = range.count.mul(split.add(1)).div(splits); + const shared = b.ptrToInt(b.allocSmem(.i8, @intCast(c.sharedBytes()), 1024), .i32); + var t: Tile = .{ + .b = b, + .c = c, + .q = b.ptrToInt(a.q, .i64), + .k = b.ptrToInt(a.k, .i64), + .v = b.ptrToInt(a.v, .i64), + .table = b.ptrToInt(a.table, .i64), + .sink = b.ptrToInt(a.sink, .i64), + .out = b.ptrToInt(a.out, .i64), + .shared = shared, + .tmem = undefined, + .tid = tid, + .lane = tid.rem(32), + .warp = tid.div(32), + .seq = seq, + .head = b.blockIdx().y, + .first_row = first_row, + .start = start, + .qlen = qlen, + .klen = klen, + .first_key_tile = range.first.add(lo), + .tiles = hi.sub(lo), + .split = split, + }; + var leader = b.openIf(tid.eq(0)); + { + for (0..@as(usize, @intCast(c.buffers()))) |i| { + b.mbarrierInit(shared.add(c.barriers() + @as(i64, @intCast(i * 8))), 128); + b.mbarrierInit(shared.add(c.barriers() + c.buffers() * 8 + @as(i64, @intCast(i * 8))), 1); + } + b.mbarrierInit(shared.add(c.completion()), 1); + b.fenceMbarrierInit(); + leader.close(); + } + var alloc = b.openIf(tid.lt(32)); + { + b.tmemAllocAt(shared.add(c.holder()), c.columns()); + b.relinquishTmemAllocPermit(.{}); + alloc.close(); + } + var queries = b.openFor(tid, c.rows() * @divExact(c.paddedDim(), 8), 256, .{}); + { + const row = queries.iv.div(@divExact(c.paddedDim(), 8)); + const col = queries.iv.rem(@divExact(c.paddedDim(), 8)).mul(8); + const packed_row = first_row.add(row); + const valid = packed_row.lt(qlen.mul(c.group)).bitAnd(col.lt(c.head_dim)); + const token = start.add(b.select(valid, packed_row.div(c.group), b.constant(.i32, 0))); + const offset = token.to(.i64).mul(c.kv_heads).add(t.head.to(.i64)).mul(c.group).add(packed_row.rem(c.group).to(.i64)).mul(c.head_dim).add(col.to(.i64)); + b.cpAsync16(shared.add(c.qBase()).add(swizzle(row, col, c.rows())), t.q.add(offset.mul(2)), valid); + queries.close(); + } + b.cpAsyncWaitAll(); + b.syncThreads(); + t.tmem = b.loadWords(1, shared.add(c.holder()), .shared, .{})[0]; + b.tcgen05Fence(.after); + var producer = b.openIf(tid.ge(128)); + { + produce(t); + producer.close(); + } + var consumer = b.openIf(tid.lt(128)); + { + if (c.decode) { + if (c.group <= 2) decode(t, 2) else if (c.group <= 4) decode(t, 4) else if (c.group <= 8) decode(t, 8) else decode(t, 16); + } else prefill(t); + consumer.close(); + } + b.tcgen05Fence(.before); + b.syncThreads(); + var free = b.openIf(tid.lt(32)); + { + b.tmemDeallocAt(t.tmem, c.columns()); + free.close(); + } + active.close(); + } +} + +fn produce(t: Tile) void { + const b = t.b; + const c = t.c; + const row0 = t.tid.sub(128).div(16); + const col = t.tid.rem(16).mul(8); + var tiles = b.openFor(0, t.tiles, 1, .{}); + { + // Paged offsets are shared by K/V and every 128-feature stage. + var offsets: [16]V = undefined; + var valid: [16]V = undefined; + for (0..@as(usize, @intCast(@divExact(c.keys(), 8)))) |i| { + const key = t.first_key_tile.add(tiles.iv).mul(c.keys()).add(row0).add(i * 8); + valid[i] = key.lt(t.klen).bitAnd(col.lt(c.head_dim)); + const safe = b.select(key.lt(t.klen), key, b.constant(.i32, 0)); + const page = loadInt(b, t.table, t.seq.mul(c.pages_per_seq).add(safe.div(c.page_size))); + offsets[i] = page.to(.i64).mul(c.page_size).add(safe.rem(c.page_size).to(.i64)).mul(c.kv_heads).add(t.head.to(.i64)).mul(c.head_dim).add(col.to(.i64)).mul(2); + } + for (0..@as(usize, @intCast(c.features() * 2))) |stage| { + const id = tiles.iv.mul(c.features() * 2).add(stage); + var reuse = b.openIf(id.ge(c.buffers())); + { + b.mbarrierWait(t.released(id), t.phase(id).bitXor(1), .{ .scope = .cta }); + reuse.close(); + } + const source = if (stage < c.features()) t.k else t.v; + const feature_offset: i64 = @mod(@as(i64, @intCast(stage)), c.features()) * 256; + for (0..@as(usize, @intCast(@divExact(c.keys(), 8)))) |i| { + const row = row0.add(i * 8); + b.cpAsync16(t.shared.add(id.rem(c.buffers()).mul(c.kvBytes())).add(swizzle(row, col, c.keys())), source.add(offsets[i]).add(feature_offset), valid[i]); + } + b.cpAsyncMbarrierArrive(t.ready(id), true); + } + tiles.close(); + } +} + +fn multiply(t: Tile, tile: V, comptime qk: bool) void { + const b = t.b; + const c = t.c; + var leader = b.openIf(t.tid.eq(0)); + { + for (0..@as(usize, @intCast(c.features()))) |part| { + const id = tile.mul(c.features() * 2).add(@as(i64, @intCast(part)) + if (qk) @as(i64, 0) else c.features()); + b.mbarrierWait(t.ready(id), t.phase(id), .{ .scope = .cta }); + b.tcgen05Fence(.after); + const kv = descriptor(b, t.shared.add(id.rem(c.buffers()).mul(c.kvBytes())), if (qk) 16 else c.keys() * 128); + const qp = descriptor(b, t.shared.add(if (qk) c.qBase() else c.pBase()), 16); + const steps: usize = @intCast(if (qk) 8 else @divExact(c.keys(), 16)); + for (0..steps) |step| { + const kv_offset: i64 = if (qk) @as(i64, @intCast(step / 4)) * c.keys() * 128 + @as(i64, @intCast(step % 4)) * 32 else @as(i64, @intCast(step)) * 2048; + const qp_offset: i64 = (if (qk) @as(i64, @intCast(part * 2 + step / 4)) else @as(i64, @intCast(step / 4))) * c.rows() * 128 + @as(i64, @intCast(step % 4)) * 32; + const left = if (c.decode) kv.add(@divExact(kv_offset, 16)) else qp.add(@divExact(qp_offset, 16)); + const right = if (c.decode) qp.add(@divExact(qp_offset, 16)) else kv.add(@divExact(kv_offset, 16)); + const column: i64 = if (qk) c.scoreColumn() else @as(i64, @intCast(part)) * (if (c.decode) c.rows() else 128); + const accumulate = if (qk) b.constant(.i32, @intFromBool(part > 0 or step > 0)).ne(0) else tile.gt(0).bitOr(b.constant(.i32, @intFromBool(step > 0)).ne(0)); + b.tcgen05MmaF16(t.tmem.add(column), left, right, b.constant(.i32, @as(i32, @bitCast(instruction(c, qk)))), accumulate); + } + b.tcgen05Commit(t.released(id), .{}); + } + b.tcgen05Commit(t.shared.add(c.completion()), .{}); + leader.close(); + } + b.mbarrierWait(t.shared.add(c.completion()), b.constant(.i32, if (qk) 0 else 1), .{ .scope = .cta }); + b.tcgen05Fence(.after); + // A fast warp must not wrap completion parity before every consumer has + // observed the preceding PV completion. + if (!qk) b.barrierCtaSyncAligned(1, 128); +} + +fn publishProbabilities(t: Tile) void { + t.b.fenceProxyAsyncShared(); + t.b.tcgen05Fence(.before); + t.b.barrierCtaSyncAligned(1, 128); + t.b.tcgen05Fence(.after); +} + +fn reduceRow(comptime n: usize, values: [n]V, comptime maximum: bool) V { + var tree = values; + var width: usize = n; + while (width > 1) { + width /= 2; + for (0..width) |i| tree[i] = if (maximum) tree[i * 2].maximum(tree[i * 2 + 1]) else tree[i * 2].add(tree[i * 2 + 1]); + } + return tree[0]; +} + +fn reduceHeads(t: Tile, comptime heads: usize, values: [heads]V, comptime maximum: bool) [heads]V { + const b = t.b; + for (values, 0..) |value, h| { + var reduced = value; + inline for (.{ 16, 8, 4, 2, 1 }) |step| reduced = if (maximum) reduced.maximum(b.shuffleXor(reduced, step)) else reduced.add(b.shuffleXor(reduced, step)); + var leader = b.openIf(t.lane.eq(0)); + { + b.storeWords(1, t.shared.add(t.c.reductions()).add(t.warp.mul(64)).add(h * 4), .shared, .{reduced.bitCast(.i32)}); + leader.close(); + } + } + b.barrierCtaSyncAligned(1, 128); + var result: [heads]V = undefined; + for (0..heads) |h| { + var value = b.loadWords(1, t.shared.add(t.c.reductions() + @as(i64, @intCast(h * 4))), .shared, .{})[0].bitCast(.f32); + for (1..4) |w| { + const next = b.loadWords(1, t.shared.add(t.c.reductions() + @as(i64, @intCast(w * 64 + h * 4))), .shared, .{})[0].bitCast(.f32); + value = if (maximum) value.maximum(next) else value.add(next); + } + result[h] = value; + } + b.barrierCtaSyncAligned(1, 128); + return result; +} + +fn decode(t: Tile, comptime heads: usize) void { + const b = t.b; + const c = t.c; + const address = t.tmem.add(t.warp.mul(32).shl(16)); + const key_row = t.warp.mul(32).add(t.lane); + const zero = b.constant(.f32, 0); + const neg_inf = b.constant(.f32, -std.math.inf(f32)); + var initial: [2 * heads]V = undefined; + for (0..heads) |h| { + initial[h] = neg_inf; + initial[heads + h] = zero; + } + var tiles = b.openFor(0, t.tiles, 1, initial); + { + multiply(t, tiles.iv, true); + const raw = b.tmemLoad32x32b(heads, address.add(c.scoreColumn())); + b.fenceTmemLoad(); + const key = t.first_key_tile.add(tiles.iv).mul(c.keys()).add(key_row); + var scores: [heads]V = undefined; + for (0..heads) |h| scores[h] = b.select(t.valid(t.first_row.add(h), key), raw[h].bitCast(.f32).mul(c.scale * std.math.log2e), neg_inf); + const maxima = reduceHeads(t, heads, scores, true); + var next: [2 * heads]V = undefined; + var alpha: [heads]V = undefined; + var probabilities: [heads]V = undefined; + for (0..heads) |h| { + next[h] = tiles.carried[h].maximum(maxima[h]); + const reference = next[h].maximum(-std.math.floatMax(f32)); + alpha[h] = b.exp2Approx(tiles.carried[h].sub(reference)); + probabilities[h] = b.exp2Approx(scores[h].sub(reference)); + } + const sums = reduceHeads(t, heads, probabilities, false); + for (0..heads) |h| { + next[heads + h] = tiles.carried[heads + h].mul(alpha[h]).add(sums[h]); + t.putProbability(b.constant(.i32, h), key_row, probabilities[h]); + } + for (heads..@as(usize, @intCast(c.rows()))) |h| t.putProbability(b.constant(.i32, h), key_row, zero); + var rescale = b.openIf(tiles.iv.gt(0)); + { + for (0..@as(usize, @intCast(c.features()))) |part| { + const addr = address.add(@as(i64, @intCast(part)) * c.rows()); + var out = b.tmemLoad32x32b(heads, addr); + b.fenceTmemLoad(); + for (0..heads) |h| out[h] = out[h].bitCast(.f32).mul(alpha[h]).bitCast(.i32); + b.tmemStore32x32b(heads, addr, out); + } + b.fenceTmemStore(); + rescale.close(); + } + publishProbabilities(t); + multiply(t, tiles.iv, false); + tiles.yield(next); + } + for (0..@as(usize, @intCast(c.features()))) |part| { + const out = b.tmemLoad32x32b(heads, address.add(@as(i64, @intCast(part)) * c.rows())); + b.fenceTmemLoad(); + const dim = key_row.add(part * 128); + for (0..heads) |h| { + const row = t.first_row.add(h); + var valid = b.openIf(row.lt(t.qlen.mul(c.group))); + { + var feature = b.openIf(dim.lt(c.head_dim)); + { + const value = b.select(tiles.results[heads + h].gt(0), out[h].bitCast(.f32).mul(t.norm(row, tiles.results[h], tiles.results[heads + h])), zero); + t.put(row, dim, value); + feature.close(); + } + if (c.splits > 1 and part == 0) { + var stats = b.openIf(t.tid.eq(0)); + { + t.put(row, b.constant(.i32, c.head_dim), tiles.results[h]); + t.put(row, b.constant(.i32, c.head_dim + 1), tiles.results[heads + h]); + stats.close(); + } + } + valid.close(); + } + } + } +} + +fn prefill(t: Tile) void { + const b = t.b; + const c = t.c; + const address = t.tmem.add(t.warp.mul(32).shl(16)); + const local_row = t.warp.mul(16).add(t.lane); + const row = t.first_row.add(local_row); + const owns = t.lane.lt(16); + const zero = b.constant(.f32, 0); + const neg_inf = b.constant(.f32, -std.math.inf(f32)); + var tiles = b.openFor(0, t.tiles, 1, .{ neg_inf, zero }); + { + // D512 fills TMEM with O. Preserve its last 64 columns in registers + // while QK reuses them for scores, avoiding duplicate QK/softmax CTAs. + var saved: [64]V = undefined; + if (c.head_dim == 512) { + var preserve = b.openIfElse(tiles.iv.gt(0), @as([64]V, @splat(zero))); + { + const tail = b.tmemLoad32x32b(64, address.add(c.scoreColumn())); + b.fenceTmemLoad(); + var words: [64]V = undefined; + for (tail, 0..) |value, i| words[i] = value.bitCast(.f32); + preserve.yieldThen(words); + } + { + preserve.yieldElse(@as([64]V, @splat(zero))); + } + @memcpy(&saved, &preserve.results); + b.barrierCtaSyncAligned(1, 128); + } + multiply(t, tiles.iv, true); + var scores: [64]V = undefined; + var hardware_max = neg_inf; + if (c.architecture == .sm103) { + const loaded = sm103.loadScores(b, address.add(c.scoreColumn())); + scores = loaded.values; + hardware_max = loaded.maximum; + } else { + const loaded = b.tmemLoad32x32b(64, address.add(c.scoreColumn())); + b.fenceTmemLoad(); + for (loaded, 0..) |value, i| scores[i] = value.bitCast(.f32); + } + const key0 = t.first_key_tile.add(tiles.iv).mul(c.keys()); + // Hardware maxima describe raw scores. Only fully visible tiles may + // use them; page tails and mask boundaries reduce post-mask values. + const full = owns.bitAnd(t.valid(row, key0)).bitAnd(t.valid(row, key0.add(63))); + for (0..64) |i| { + scores[i] = b.select(owns.bitAnd(t.valid(row, key0.add(i))), scores[i].mul(c.scale * std.math.log2e), neg_inf); + } + const tile_max = if (c.architecture == .sm103 and c.scale > 0) blk: { + var choose = b.openIfElse(full, .{zero}); + { + choose.yieldThen(.{hardware_max.mul(c.scale * std.math.log2e)}); + } + { + choose.yieldElse(.{reduceRow(64, scores, true)}); + } + break :blk choose.results[0]; + } else reduceRow(64, scores, true); + const maximum = tiles.carried[0].maximum(tile_max); + const reference = maximum.maximum(-std.math.floatMax(f32)); + const alpha = b.exp2Approx(tiles.carried[0].sub(reference)); + var probabilities: [64]V = undefined; + for (0..64) |i| { + probabilities[i] = b.exp2Approx(scores[i].sub(reference)); + } + const denominator = tiles.carried[1].mul(alpha).add(reduceRow(64, probabilities, false)); + var write = b.openIf(owns); + { + for (0..64) |i| t.putProbability(local_row, b.constant(.i32, i), probabilities[i]); + write.close(); + } + var rescale = b.openIf(tiles.iv.gt(0)); + { + for (0..@as(usize, @intCast(@divExact(if (c.head_dim == 512) c.scoreColumn() else c.paddedDim(), 32)))) |chunk| { + const addr = address.add(chunk * 32); + var out = b.tmemLoad32x32b(32, addr); + b.fenceTmemLoad(); + for (&out) |*value| value.* = value.bitCast(.f32).mul(alpha).bitCast(.i32); + b.tmemStore32x32b(32, addr, out); + } + if (c.head_dim == 512) { + for (&saved) |*value| value.* = value.mul(alpha).bitCast(.i32); + b.tmemStore32x32b(64, address.add(c.scoreColumn()), saved); + } + b.fenceTmemStore(); + rescale.close(); + } + publishProbabilities(t); + multiply(t, tiles.iv, false); + tiles.yield(.{ maximum, denominator }); + } + const norm = t.norm(row, tiles.results[0], tiles.results[1]); + for (0..@as(usize, @intCast(@divExact(c.paddedDim(), 32)))) |chunk| { + const out = b.tmemLoad32x32b(32, address.add(chunk * 32)); + b.fenceTmemLoad(); + var valid = b.openIf(owns.bitAnd(row.lt(t.qlen.mul(c.group)))); + { + for (0..32) |i| { + const dim = chunk * 32 + i; + if (dim < c.head_dim) t.put(row, b.constant(.i32, dim), b.select(tiles.results[1].gt(0), out[i].bitCast(.f32).mul(norm), zero)); + } + if (c.splits > 1 and chunk == 0) { + t.put(row, b.constant(.i32, c.head_dim), tiles.results[0]); + t.put(row, b.constant(.i32, c.head_dim + 1), tiles.results[1]); + } + valid.close(); + } + } +} + +pub const Merge = cute.Kernel(Config, .{ + .name = "paged_attention_cute_merge", + .inputs = &.{ "partials", "lengths", "starts", "sink" }, + .outputs = &.{"out"}, + .run = merge, +}); + +fn merge(b: *B, c: Config) cute.FinishError!void { + const a = try b.declareArgs(.{ + .partials = .{ .tensor = .{ .dtype = .f32, .shape = &.{ c.splits, c.tokens, c.kv_heads * c.group, c.head_dim + 2 } } }, + .lengths = .{ .tensor = .{ .dtype = .i32, .shape = &.{c.batch} } }, + .starts = .{ .tensor = .{ .dtype = .i32, .shape = &.{c.batch + 1} } }, + .sink = .{ .tensor = .{ .dtype = .f32, .shape = &.{if (c.sink) c.kv_heads * c.group else 1} } }, + .out = .{ .tensor = .{ .dtype = c.dtype, .shape = &.{ c.tokens, c.kv_heads * c.group, c.head_dim } } }, + }); + const token = b.blockIdx().x; + const head = b.blockIdx().y; + var active = b.openIf(token.lt(a.starts.get(.{c.batch}))); + { + var find = b.openFor(0, c.batch, 1, .{b.constant(.i32, 0)}); + { + const owns = token.ge(a.starts.get(.{find.iv})).bitAnd(token.lt(a.starts.get(.{find.iv.add(1)}))); + find.yield(.{b.select(owns, find.iv, find.carried[0])}); + } + const seq = find.results[0]; + const start = a.starts.get(.{seq}); + const qlen = a.starts.get(.{seq.add(1)}).sub(start); + const row = token.sub(start).mul(c.group).add(head.rem(c.group)); + const range = keyRange(b, c, row.div(c.rows()).mul(c.rows()), qlen, a.lengths.get(.{seq})); + const splits = liveSplits(c, range.count); + const zero = b.constant(.f32, 0); + const sink = if (c.sink) a.sink.get(.{head}).mul(std.math.log2e) else b.constant(.f32, -std.math.inf(f32)); + var maximum = b.openFor(0, splits, 1, .{sink}); + { + maximum.yield(.{maximum.carried[0].maximum(a.partials.get(.{ maximum.iv, token, head, c.head_dim }))}); + } + const reference = maximum.results[0].maximum(-std.math.floatMax(f32)); + var dims = b.openFor(b.threadIdx().x, c.head_dim, 128, .{}); + { + var sum = b.openFor(0, splits, 1, .{ zero, if (c.sink) b.exp2Approx(sink.sub(reference)) else zero }); + { + const weight = b.exp2Approx(a.partials.get(.{ sum.iv, token, head, c.head_dim }).sub(reference)); + sum.yield(.{ + sum.carried[0].add(weight.mul(a.partials.get(.{ sum.iv, token, head, dims.iv }))), + sum.carried[1].add(weight.mul(a.partials.get(.{ sum.iv, token, head, c.head_dim + 1 }))), + }); + } + const norm = b.select(sum.results[1].gt(0), b.rcpApprox(sum.results[1]), zero); + a.out.set(.{ token, head, dims.iv }, sum.results[0].mul(norm).to(c.dtype)); + dims.close(); + } + active.close(); + } +} + +test "SM10x layouts and instruction feature gates" { + for ([_]i64{ 32, 64, 128, 256, 512 }) |dim| for ([_]bool{ true, false }) |decode_only| for ([_]Architecture{ .sm100, .sm103 }) |architecture| { + const c: Config = .{ .architecture = architecture, .head_dim = dim, .tokens = 80, .batch = 4, .kv_heads = 2, .group = 5, .pages_per_seq = 256, .page_size = 16, .max_query = if (decode_only) 1 else 17, .dtype = .bf16, .decode = decode_only, .causal = true, .window = 0, .scale = 0.125, .sink = true, .splits = 4 }; + try std.testing.expect(c.columns() >= 32 and c.columns() <= 512); + try std.testing.expect(c.sharedBytes() <= 227 * 1024); + const ir = try Attention.emit(std.testing.allocator, c, block); + defer std.testing.allocator.free(ir); + try std.testing.expect(std.mem.indexOf(u8, ir, "tcgen05.mma") != null); + try std.testing.expectEqual(architecture == .sm103 and !decode_only, std.mem.indexOf(u8, ir, "tcgen05.ld.red") != null); + const merge_ir = try Merge.emit(std.testing.allocator, c, .{ 128, 1, 1 }); + defer std.testing.allocator.free(merge_ir); + }; +} diff --git a/zml/kernels/paged_attention/cute/sm10x/sm103.zig b/zml/kernels/paged_attention/cute/sm10x/sm103.zig new file mode 100644 index 00000000..fe96916d --- /dev/null +++ b/zml/kernels/paged_attention/cute/sm10x/sm103.zig @@ -0,0 +1,36 @@ +const std = @import("std"); +const mlir = @import("mlir"); +const llvm = @import("mlir/dialects/llvm"); + +const cute = @import("../../../../zml.zig").kernel.cute; + +pub const Scores = struct { + values: [64]cute.Value, + maximum: cute.Value, +}; + +// SM103 adds a per-TMEM-lane reduction across the loaded columns. This is +// useful for query-major scores; swapped decode reduces across lanes instead. +pub fn loadScores(b: *cute.Builder, address: cute.Value) Scores { + const asm_parts = comptime blk: { + @setEvalBranchQuota(20000); + var registers: []const u8 = ""; + var constraints: []const u8 = ""; + for (0..65) |i| { + if (i < 64) registers = registers ++ (if (i == 0) "" else ", ") ++ std.fmt.comptimePrint("${d}", .{i}); + constraints = constraints ++ (if (i == 0) "" else ",") ++ "=f"; + } + break :blk .{ + "tcgen05.ld.red.sync.aligned.32x32b.x64.max.f32 {" ++ registers ++ "}, $64, [$65];", + constraints ++ ",r", + }; + }; + const result = b.emit(llvm.inline_asm(b.ctx, asm_parts[0], &.{address.inner}, .{ .result_type = llvm.structType(b.ctx, &@as([65]*const mlir.Type, @splat(.float(b.ctx, .f32)))), .constraints = asm_parts[1], .has_side_effects = true }, b.loc())); + var scores: Scores = undefined; + for (0..65) |i| { + const value = b.emit(llvm.extractvalue(b.ctx, result.inner, cute.DType.f32.toMlir(b.ctx), &.{@intCast(i)}, b.loc())); + if (i < 64) scores.values[i] = value else scores.maximum = value; + } + b.fenceTmemLoad(); + return scores; +} diff --git a/zml/kernels/paged_attention/cute/sm10x/sm10x.zig b/zml/kernels/paged_attention/cute/sm10x/sm10x.zig new file mode 100644 index 00000000..afc4298e --- /dev/null +++ b/zml/kernels/paged_attention/cute/sm10x/sm10x.zig @@ -0,0 +1,151 @@ +const std = @import("std"); +const stdx = @import("stdx"); + +const zml = @import("../../../../zml.zig"); +const cute = @import("../cute.zig"); +const kernel = @import("attention.zig"); + +pub const Options = cute.Options; +pub const Parameters = cute.Parameters; + +test { + _ = kernel; +} + +pub fn isAvailable(platform: *const zml.Platform) bool { + if (platform.target != .cuda) return false; + const cc = zml.platform.cuda.computeCapability(platform) orelse return false; + return cc.major == 10; +} + +pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: cute.AttentionOptions) zml.Tensor { + const q_ = q.transpose(.{ .b, .hkv, .hg, .hd }); + const k_ = k.transpose(.{ .page, .k_chunk, .hkv, .hd }); + const v_ = v.transpose(.{ .page, .k_chunk, .hkv, .hd }); + stdx.debug.assert(q_.dim(.hkv) == k_.dim(.hkv) and k_.shape().eql(v_.shape()) and k_.dim(.hd) == q_.dim(.hd) and q_.dim(.hg) > 0, "CuTe paged attention requires aligned local Q/KV heads and matching caches", .{}); + const decode = !parameters.options_.is_prefill; + const cc = zml.platform.cuda.computeCapability(zml.Compiler.current().platform).?; + var c: kernel.Config = .{ + .architecture = if (cc.minor >= 3) .sm103 else .sm100, + .head_dim = q_.dim(.hd), + .tokens = q_.dim(.b), + .batch = parameters.block_table.dim(.b), + .kv_heads = k_.dim(.hkv), + .group = q_.dim(.hg), + .pages_per_seq = parameters.block_table.dim(.p), + .page_size = k_.dim(.k_chunk), + .max_query = if (decode) 1 else @min(q_.dim(.b), @as(i64, @intCast(parameters.options_.max_seqlen_q))), + .dtype = zml.kernel.cute.from(q_.dtype()), + .decode = decode, + .causal = opts.mask.isCausal(), + .window = opts.mask.slidingWindow() orelse 0, + .scale = opts.scale orelse 1 / @sqrt(@as(f32, @floatFromInt(q_.dim(.hd)))), + .sink = opts.sink != null, + }; + const query_tiles = if (decode) c.batch * c.queryTiles() else c.batch + @divTrunc(c.tokens * c.group + c.rows() - 1, c.rows()); + const work = query_tiles * c.kv_heads; + const bound = if (c.window > 0 and c.causal) @min(c.window + @divTrunc(c.rows() + c.group - 1, c.group), c.pages_per_seq * c.page_size) else c.pages_per_seq * c.page_size; + c.splits = @max(1, @min(32, @min(@divTrunc(@as(i64, @intCast(cute.multiprocessorCount(zml.Compiler.current().platform))) * 2 + work - 1, work), @divTrunc(bound + 2 * c.keys() - 1, 2 * c.keys())))); + // An unused scalar stays a constant buffer; broadcasting a dummy for + // every head would add a helper kernel to each attention invocation. + const sink = if (opts.sink) |s| s.convert(.f32).reshape(.{c.kv_heads * c.group}) else zml.Tensor.scalar(0, .f32).reshape(.{1}); + const out = kernel.Attention.call(.{ .q = q_, .k = k_, .v = v_, .table = parameters.block_table, .lengths = parameters.seq_lens, .starts = parameters.query_start_len, .sink = sink }, .{ + .out = if (c.splits > 1) zml.Shape.init(.{ c.splits, c.tokens, c.kv_heads * c.group, c.head_dim + 2 }, .f32) else q_.shape(), + }, .{ .cfg = c, .grid = .{ @intCast(c.batch * c.queryTiles()), @intCast(c.kv_heads), @intCast(c.splits) }, .block = kernel.block }).out; + const result = if (c.splits > 1) kernel.Merge.call(.{ .partials = out, .lengths = parameters.seq_lens, .starts = parameters.query_start_len, .sink = sink }, .{ .out = q_.shape() }, .{ + .cfg = c, + .grid = .{ @intCast(c.tokens), @intCast(c.kv_heads * c.group), 1 }, + .block = .{ 128, 1, 1 }, + }).out else out; + return result.transpose(q.shape()); +} + +const ArchitectureCheck = struct { + fn forward(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, c: kernel.Config) zml.Tensor { + const table_values: [9]i32 = .{ 4, 2, 8, 1, 6, 0, 7, 3, 5 }; + const table = zml.Tensor.constantTensor(.init(.{ 1, 9 }, .i32), std.mem.asBytes(&table_values)); + const lengths = zml.Tensor.constantTensor(.init(.{1}, .i32), std.mem.asBytes(&[_]i32{129})); + const starts = zml.Tensor.constantTensor(.init(.{2}, .i32), std.mem.asBytes(&[_]i32{ 0, 3 })); + const sink = zml.Tensor.zeroes(.init(.{3}, .f32)); + const partials = kernel.Attention.call(.{ .q = q, .k = k, .v = v, .table = table, .lengths = lengths, .starts = starts, .sink = sink }, .{ + .out = .init(.{ 2, 3, 3, c.head_dim + 2 }, .f32), + }, .{ .cfg = c, .grid = .{ 1, 1, 2 }, .block = kernel.block }).out; + return kernel.Merge.call(.{ .partials = partials, .lengths = lengths, .starts = starts, .sink = sink }, .{ .out = q.shape() }, .{ + .cfg = c, + .grid = .{ 3, 3, 1 }, + .block = .{ 128, 1, 1 }, + }).out; + } +}; + +test "SM10x GB200 and GB300 instruction paths match CPU reference" { + const platform = zml.testing.env(); + if (!isAvailable(platform)) return error.SkipZigTest; + const cc = zml.platform.cuda.computeCapability(platform).?; + const a = std.testing.allocator; + const io = std.testing.io; + const BF16 = zml.floats.BFloat16; + const table: [9]usize = .{ 4, 2, 8, 1, 6, 0, 7, 3, 5 }; + for ([_]i64{ 128, 512 }) |head_dim| { + const d: usize = @intCast(head_dim); + const q_host = try a.alloc(BF16, 9 * d); + defer a.free(q_host); + const k_host = try a.alloc(BF16, 144 * d); + defer a.free(k_host); + const v_host = try a.alloc(BF16, 144 * d); + defer a.free(v_host); + for (q_host, 0..) |*value, i| value.* = .fromF32(@as(f32, @floatFromInt(i % 13)) / 16 - 0.375); + @memset(k_host, .fromF32(std.math.nan(f32))); + @memset(v_host, .fromF32(std.math.nan(f32))); + for (0..129) |key| for (0..d) |feature| { + const offset = (table[key / 16] * 16 + key % 16) * d + feature; + k_host[offset] = .fromF32(@as(f32, @floatFromInt((key * 7 + feature) % 17)) / 16 - 0.5); + v_host[offset] = .fromF32(@as(f32, @floatFromInt((key + feature * 3) % 23)) / 16 - 0.625); + }; + const q: zml.Tensor = .init(.{ .b = 3, .hkv = 1, .hg = 3, .hd = d }, .bf16); + const k: zml.Tensor = .init(.{ .page = 9, .k_chunk = 16, .hkv = 1, .hd = d }, .bf16); + const v: zml.Tensor = .fromShape(k.shape()); + var qb = try zml.Buffer.fromBytes(io, platform, q.shape(), std.mem.sliceAsBytes(q_host)); + defer qb.deinit(); + var kb = try zml.Buffer.fromBytes(io, platform, k.shape(), std.mem.sliceAsBytes(k_host)); + defer kb.deinit(); + var vb = try zml.Buffer.fromBytes(io, platform, k.shape(), std.mem.sliceAsBytes(v_host)); + defer vb.deinit(); + for ([_]kernel.Architecture{ .sm100, .sm103 }) |architecture| { + if (architecture == .sm103 and cc.minor < 3) continue; + const c: kernel.Config = .{ .architecture = architecture, .head_dim = head_dim, .tokens = 3, .batch = 1, .kv_heads = 1, .group = 3, .pages_per_seq = 9, .page_size = 16, .max_query = 3, .dtype = .bf16, .decode = false, .causal = true, .window = 0, .scale = 0.125, .sink = true, .splits = 2 }; + const exe = try platform.compileFn(a, io, ArchitectureCheck.forward, .{ q, k, v, c }, .{}); + defer exe.deinit(); + var output = try exe.eval(a, io, .{ qb, kb, vb }); + defer output.deinit(); + var host = try output.toSliceAlloc(a, io); + defer host.free(a); + const actual = host.items(BF16); + for (0..9) |row| { + const keys = 127 + row / 3; + var scores: [129]f32 = undefined; + var maximum: f32 = 0; // Sink score, with zero value. + for (0..keys) |key| { + const offset = (table[key / 16] * 16 + key % 16) * d; + var score: f32 = 0; + for (0..d) |feature| score += q_host[row * d + feature].toF32() * k_host[offset + feature].toF32(); + scores[key] = score * c.scale; + maximum = @max(maximum, scores[key]); + } + var denominator = @exp(-maximum); + for (0..keys) |key| { + scores[key] = @exp(scores[key] - maximum); + denominator += scores[key]; + } + for (0..d) |feature| { + var expected: f32 = 0; + for (0..keys) |key| expected += scores[key] * v_host[(table[key / 16] * 16 + key % 16) * d + feature].toF32(); + expected /= denominator; + const got = actual[row * d + feature].toF32(); + try std.testing.expect(std.math.isFinite(got)); + try std.testing.expect(@abs(got - expected) <= 0.003 + 0.02 * @abs(expected)); + } + } + } + } +} diff --git a/zml/kernels/paged_attention/cute/sm120/BUILD.bazel b/zml/kernels/paged_attention/cute/sm120/BUILD.bazel new file mode 100644 index 00000000..f93bd5e0 --- /dev/null +++ b/zml/kernels/paged_attention/cute/sm120/BUILD.bazel @@ -0,0 +1,9 @@ +package(default_visibility = ["//zml/kernels/paged_attention/cute:__pkg__"]) + +filegroup( + name = "sm120", + srcs = [ + "attention.zig", + "sm120.zig", + ], +) diff --git a/zml/kernels/paged_attention/cute/sm120/attention.zig b/zml/kernels/paged_attention/cute/sm120/attention.zig new file mode 100644 index 00000000..e9675252 --- /dev/null +++ b/zml/kernels/paged_attention/cute/sm120/attention.zig @@ -0,0 +1,446 @@ +const std = @import("std"); +const mlir = @import("mlir"); +const llvm = @import("mlir/dialects/llvm"); +const vector = @import("mlir/dialects/vector"); + +const zml = @import("../../../../zml.zig"); +const cute = zml.kernel.cute; +const B = cute.Builder; +const V = cute.Value; + +pub const Config = struct { + head_dim: i64, + tokens: i64, + batch: i64, + kv_heads: i64, + group: i64, + pages_per_seq: i64, + page_size: i64, + max_query: i64, + dtype: cute.DType, + warps_per_query: i64, + kv_tile: i64, + causal: bool, + window: i64, + scale: f32, + sink: bool, + splits: i64 = 1, + + pub fn rows(c: Config) i64 { + return @divExact(64, c.warps_per_query); + } + pub fn queryTiles(c: Config) i64 { + return @divTrunc(c.max_query * c.group + c.rows() - 1, c.rows()); + } + fn kvBase(c: Config) i64 { + return c.rows() * c.head_dim * 2; + } + fn pBase(c: Config) i64 { + return c.kvBase() + c.kv_tile * c.head_dim * 2; + } + fn redBase(c: Config) i64 { + return c.pBase() + c.rows() * c.kv_tile * 2; + } + fn sharedBytes(c: Config) i64 { + return c.redBase() + 2 * 64 * 4; + } +}; + +pub const Kernel = cute.Kernel(Config, .{ + .name = "paged_attention_cute_sm120", + .inputs = &.{ "q", "k", "v", "table", "lengths", "starts", "sink" }, + .outputs = &.{"out"}, + .run = run, +}); + +fn flatten(comptime n: usize, values: [n][4]V) [n * 4]V { + var result: [n * 4]V = undefined; + for (0..n) |i| @memcpy(result[i * 4 ..][0..4], &values[i]); + return result; +} + +fn packAcc(b: *B, values: [4]V) V { + return b.emit(vector.from_elements(b.ctx, &.{ values[0].inner, values[1].inner, values[2].inner, values[3].inner }, .vector(&.{4}, .float(b.ctx, .f32)), b.loc())); +} + +fn loadI32(b: *B, ptr: V, index: V) V { + return b.loadWords(1, ptr.add(index.to(.i64).mul(4)), .global, .{})[0]; +} + +// XOR whole 16-byte vectors so cp.async and ldmatrix agree on bank layout. +fn swizzle(row: V, col: V, width: i64) V { + return row.mul(width).add(col.bitXor(row.rem(@min(8, @divExact(width, 8))).mul(8))).mul(2); +} + +fn unpackStruct(b: *B, comptime n: usize, value: V, dtype: cute.DType) [n]V { + var result: [n]V = undefined; + for (&result, 0..) |*r, i| r.* = b.emit(llvm.extractvalue(b.ctx, value.inner, dtype.toMlir(b.ctx), &.{@intCast(i)}, b.loc())); + return result; +} + +fn matrixLoad(b: *B, comptime n: usize, comptime transpose: bool, address: V) [n]V { + const assembly = std.fmt.comptimePrint("ldmatrix.sync.aligned.m8n8.x{d}{s}.shared.b16 {{", .{ n, if (transpose) ".trans" else "" }) ++ + (if (n == 4) "$0, $1, $2, $3}, [$4];" else "$0, $1}, [$2];"); + const result = b.emit(llvm.inline_asm(b.ctx, assembly, &.{address.inner}, .{ .result_type = llvm.structType(b.ctx, &@as([n]*const mlir.Type, @splat(.int(b.ctx, .i32)))), .constraints = if (n == 4) "=r,=r,=r,=r,r" else "=r,=r,r", .has_side_effects = true }, b.loc())); + return unpackStruct(b, n, result, .i32); +} + +fn mma(b: *B, comptime dtype: cute.DType, a: [4]V, k: [2]V, acc: [4]V) [4]V { + const assembly = "mma.sync.aligned.m16n8k16.row.col.f32." ++ @tagName(dtype) ++ "." ++ @tagName(dtype) ++ ".f32 " ++ + "{$0, $1, $2, $3}, {$4, $5, $6, $7}, {$8, $9}, {$10, $11, $12, $13};"; + const result = b.emit(llvm.inline_asm(b.ctx, assembly, &.{ a[0].inner, a[1].inner, a[2].inner, a[3].inner, k[0].inner, k[1].inner, acc[0].inner, acc[1].inner, acc[2].inner, acc[3].inner }, .{ .result_type = llvm.structType(b.ctx, &@as([4]*const mlir.Type, @splat(.float(b.ctx, .f32)))), .constraints = "=f,=f,=f,=f,r,r,r,r,r,r,f,f,f,f" }, b.loc())); + return unpackStruct(b, 4, result, .f32); +} + +fn run(b: *B, c: Config) cute.FinishError!void { + switch (c.head_dim) { + inline 32, 64, 128, 256, 512 => |d| switch (c.warps_per_query) { + inline 2, 4 => |w| switch (c.kv_tile) { + inline 32, 64 => |n| switch (c.dtype) { + inline .bf16, .f16 => |dt| try runTile(b, c, d, w, n, dt), + else => unreachable, + }, + else => unreachable, + }, + else => unreachable, + }, + else => unreachable, + } +} + +const TileContext = struct { + b: *B, + c: Config, + q: V, + k: V, + v: V, + table: V, + sink: V, + out: V, + shared: V, + tid: V, + lane: V, + warp: V, + seq: V, + head: V, + first_row: V, + start: V, + qlen: V, + klen: V, + split: V, + + // Every load resolves its logical page on device. Tail copies are zeroed + // before either MMA, including V tails whose backing allocation holds NaN. + fn loadKV(t: TileContext, source: V, key_base: V) void { + const b = t.b; + const c = t.c; + if (c.page_size == 16 and c.head_dim >= 64) { + // A page ID is shared by all feature vectors in its 16 rows. + // Resolve it once, then issue independent vector copies instead + // of serializing a page-table load before every cp.async. + var pages = b.openFor(0, c.kv_tile, 16, .{}); + { + const first_key = key_base.add(pages.iv); + const page_valid = first_key.lt(t.klen); + const logical_page = b.select(page_valid, first_key.div(16), b.constant(.i32, 0)); + const page = loadI32(b, t.table, t.seq.mul(c.pages_per_seq).add(logical_page)); + const base = page.to(.i64).mul(16 * c.kv_heads * c.head_dim).add(t.head.to(.i64).mul(c.head_dim)); + for (0..@as(usize, @intCast(@divExact(c.head_dim, 64)))) |i| { + const vector_index = t.tid.add(i * 128); + const row = vector_index.div(@divExact(c.head_dim, 8)); + const col = vector_index.rem(@divExact(c.head_dim, 8)).mul(8); + const offset = base.add(row.to(.i64).mul(c.kv_heads * c.head_dim)).add(col.to(.i64)); + b.cpAsync16(t.shared.add(c.kvBase()).add(swizzle(pages.iv.add(row), col, c.head_dim)), source.add(offset.mul(2)), first_key.add(row).lt(t.klen)); + } + pages.close(); + } + return; + } + var copy = b.openFor(t.tid, c.kv_tile * @divExact(c.head_dim, 8), 128, .{}); + { + const row = copy.iv.div(@divExact(c.head_dim, 8)); + const col = copy.iv.rem(@divExact(c.head_dim, 8)).mul(8); + const key = key_base.add(row); + const valid = key.lt(t.klen); + const safe_key = b.select(valid, key, b.constant(.i32, 0)); + const page = loadI32(b, t.table, t.seq.mul(c.pages_per_seq).add(safe_key.div(c.page_size))); + const offset = page.to(.i64).mul(c.page_size).add(safe_key.rem(c.page_size).to(.i64)).mul(c.kv_heads).add(t.head.to(.i64)).mul(c.head_dim).add(col.to(.i64)); + b.cpAsync16(t.shared.add(c.kvBase()).add(swizzle(row, col, c.head_dim)), source.add(offset.mul(2)), valid); + copy.close(); + } + } + + fn publishReduction(t: TileContext, row: V, value: V, comptime maximum: bool) void { + const b = t.b; + var v = value; + inline for (.{ 2, 1 }) |step| v = if (maximum) v.maximum(b.shuffleXor(v, step)) else v.add(b.shuffleXor(v, step)); + const address = t.shared.add(t.c.redBase() + (if (maximum) @as(i64, 0) else 256)).add(t.warp.mul(16).add(row).mul(4)); + var leader = b.openIf(t.lane.rem(4).eq(0)); + { + b.storeWords(1, address, .shared, .{v.bitCast(.i32)}); + leader.close(); + } + // Callers synchronize once both row halves have been published. + } + + fn readReduction(t: TileContext, comptime w: usize, row: V, comptime maximum: bool) V { + const b = t.b; + const base = t.shared.add(t.c.redBase() + (if (maximum) @as(i64, 0) else 256)).add(t.warp.div(w).mul(w * 16).add(row).mul(4)); + var result = b.loadWords(1, base, .shared, .{})[0].bitCast(.f32); + for (1..w) |i| { + const v = b.loadWords(1, base.add(i * 16 * 4), .shared, .{})[0].bitCast(.f32); + result = if (maximum) result.maximum(v) else result.add(v); + } + return result; + } + + fn store(t: TileContext, row: V, dim: V, value: V) void { + const c = t.c; + const token = t.start.add(row.div(c.group)); + var offset = token.to(.i64).mul(c.kv_heads).add(t.head.to(.i64)).mul(c.group).add(row.rem(c.group).to(.i64)); + if (c.splits > 1) { + offset = t.split.to(.i64).mul(c.tokens * c.kv_heads * c.group).add(offset).mul(c.head_dim + 2).add(dim.to(.i64)); + t.b.storeWords(1, t.out.add(offset.mul(4)), .global, .{value.bitCast(.i32)}); + } else { + offset = offset.mul(c.head_dim).add(dim.to(.i64)); + t.b.storeHalf(t.out.add(offset.mul(2)), .global, value.to(c.dtype).bitCast(.i16).to(.i32)); + } + } +}; + +fn runTile(b: *B, c: Config, comptime d: usize, comptime w: usize, comptime n: usize, comptime dt: cute.DType) cute.FinishError!void { + std.debug.assert(c.sharedBytes() <= 99 * 1024); + const a = try b.declareArgs(.{ + .q = .{ .ptr = dt }, + .k = .{ .ptr = dt }, + .v = .{ .ptr = dt }, + .table = .{ .ptr = cute.DType.i32 }, + .lengths = .{ .ptr = cute.DType.i32 }, + .starts = .{ .ptr = cute.DType.i32 }, + .sink = .{ .ptr = cute.DType.f32 }, + .out = .{ .ptr = if (c.splits > 1) cute.DType.f32 else dt }, + }); + b.setFunctionAttribute(.named(b.ctx, "cu_attrs", .dict(b.ctx, &.{.named(b.ctx, "max_dynamic_shared_size_bytes", b.devMaxSharedMemoryOptin())}))); + const seq = b.blockIdx().x.div(c.queryTiles()); + const first_row = b.blockIdx().x.rem(c.queryTiles()).mul(c.rows()); + const starts = b.ptrToInt(a.starts, .i64); + const start = loadI32(b, starts, seq); + const qlen = loadI32(b, starts, seq.add(1)).sub(start); + var active = b.openIf(first_row.lt(qlen.mul(c.group))); + { + const tid = b.threadIdx().x; + const t: TileContext = .{ + .b = b, + .c = c, + .q = b.ptrToInt(a.q, .i64), + .k = b.ptrToInt(a.k, .i64), + .v = b.ptrToInt(a.v, .i64), + .table = b.ptrToInt(a.table, .i64), + .sink = b.ptrToInt(a.sink, .i64), + .out = b.ptrToInt(a.out, .i64), + .shared = b.ptrToInt(b.allocSmem(.i8, c.sharedBytes(), 128), .i32), + .tid = tid, + .lane = tid.rem(32), + .warp = tid.div(32), + .seq = seq, + .head = b.blockIdx().y, + .first_row = first_row, + .start = start, + .qlen = qlen, + .klen = loadI32(b, b.ptrToInt(a.lengths, .i64), seq), + .split = b.blockIdx().z, + }; + var copy_q = b.openFor(tid, c.rows() * (d / 8), 128, .{}); + { + const row = copy_q.iv.div(d / 8); + const col = copy_q.iv.rem(d / 8).mul(8); + const packed_row = first_row.add(row); + const valid = packed_row.lt(qlen.mul(c.group)); + const token = start.add(b.select(valid, packed_row.div(c.group), b.constant(.i32, 0))); + const offset = token.to(.i64).mul(c.kv_heads).add(t.head.to(.i64)).mul(c.group).add(packed_row.rem(c.group).to(.i64)).mul(d).add(col.to(.i64)); + b.cpAsync16(t.shared.add(swizzle(row, col, d)), t.q.add(offset.mul(2)), valid); + copy_q.close(); + } + const first_pos = t.klen.sub(qlen).add(first_row.div(c.group)); + const last_pos = t.klen.sub(qlen).add(first_row.add(c.rows() - 1).div(c.group).minimum(qlen.sub(1))); + const begin = if (c.window > 0) first_pos.sub(c.window - 1).maximum(0) else b.constant(.i32, 0); + const end = if (c.causal) last_pos.add(1).minimum(t.klen) else t.klen; + const first_tile = begin.div(n); + const tiles = end.add(n - 1).div(n).sub(first_tile).maximum(0); + const lo = first_tile.add(tiles.mul(t.split).div(c.splits)); + const hi = first_tile.add(tiles.mul(t.split.add(1)).div(c.splits)); + const row_base = t.warp.div(w).mul(16); + const key_part = t.warp.rem(w).mul(n / w); + const dim_part = t.warp.rem(w).mul(d / w); + const row0 = t.lane.div(4); + const col0 = t.lane.rem(4).mul(2); + const zero = b.constant(.f32, 0); + const neg_inf = b.constant(.f32, -std.math.inf(f32)); + const nf = n / w / 8; + const of = d / w / 8; + const count = 4 + of; + var initial: [count]V = @splat(zero); + initial[0] = neg_inf; + initial[1] = neg_inf; + for (initial[4..]) |*v| v.* = packAcc(b, .{ zero, zero, zero, zero }); + b.cpAsyncWaitAll(); + b.syncThreads(); + var loop = b.openFor(lo, hi, 1, initial); + { + const key_base = loop.iv.mul(n); + t.loadKV(t.k, key_base); + b.cpAsyncWaitAll(); + b.syncThreads(); + var scores: [nf][4]V = @splat(@splat(zero)); + // Keep only MMA operands live in registers; Q remains in shared + // memory so D512 does not consume the entire per-thread register file. + var qk = b.openFor(0, d, 16, @as([nf * 4]V, @splat(zero))); + { + const qa = matrixLoad(b, 4, false, t.shared.add(swizzle(row_base.add(t.lane.rem(16)), qk.iv.add(t.lane.div(16).mul(8)), d))); + for (0..nf) |i| { + const kb = matrixLoad(b, 2, false, t.shared.add(c.kvBase()).add(swizzle(key_part.add(i * 8).add(t.lane.rem(8)), qk.iv.add(t.lane.div(8).rem(2).mul(8)), d))); + scores[i] = mma(b, dt, qa, kb, qk.carried[i * 4 ..][0..4].*); + } + qk.yield(flatten(nf, scores)); + } + for (0..nf) |i| scores[i] = qk.results[i * 4 ..][0..4].*; + var local_max: [2]V = @splat(neg_inf); + for (0..nf) |i| { + for (0..4) |j| { + const row = first_row.add(row_base).add(row0).add((j / 2) * 8); + const position = t.klen.sub(qlen).add(row.div(c.group)); + const key = key_base.add(key_part).add(i * 8).add(col0).add(j % 2); + var valid = row.lt(qlen.mul(c.group)).bitAnd(key.lt(t.klen)); + if (c.causal) valid = valid.bitAnd(key.le(position)); + if (c.window > 0) valid = valid.bitAnd(key.gt(position.sub(c.window))); + scores[i][j] = b.select(valid, scores[i][j].mul(c.scale * std.math.log2e), neg_inf); + local_max[j / 2] = local_max[j / 2].maximum(scores[i][j]); + } + } + for (0..2) |h| t.publishReduction(row0.add(h * 8), local_max[h], true); + b.syncThreads(); + // All K reads have completed. Reuse its allocation for V, and + // overlap these global loads with softmax and probability stores. + t.loadKV(t.v, key_base); + var m: [2]V = undefined; + var alpha: [2]V = undefined; + var local_sum: [2]V = @splat(zero); + for (0..2) |h| { + m[h] = loop.carried[h].maximum(t.readReduction(w, row0.add(h * 8), true)); + // Empty/fully masked row tiles have m=-inf and denominator 0. + alpha[h] = b.exp2Approx(loop.carried[h].sub(m[h].maximum(-std.math.floatMax(f32)))); + } + for (0..nf) |i| { + for (0..4) |j| { + scores[i][j] = b.exp2Approx(scores[i][j].sub(m[j / 2].maximum(-std.math.floatMax(f32)))); + local_sum[j / 2] = local_sum[j / 2].add(scores[i][j]); + } + for (0..2) |h| { + const row = row_base.add(row0).add(h * 8); + const col = key_part.add(i * 8).add(col0); + b.storeWords(1, t.shared.add(c.pBase()).add(swizzle(row, col, n)), .shared, .{b.packHalf2(scores[i][h * 2], scores[i][h * 2 + 1], dt)}); + } + } + for (0..2) |h| t.publishReduction(row0.add(h * 8), local_sum[h], false); + b.cpAsyncWaitAll(); + b.syncThreads(); + var next: [count]V = undefined; + for (0..2) |h| { + next[h] = m[h]; + next[h + 2] = loop.carried[h + 2].mul(alpha[h]).add(t.readReduction(w, row0.add(h * 8), false)); + } + var acc: [of][4]V = undefined; + for (0..of) |i| for (0..4) |j| { + acc[i][j] = b.vectorExtract(loop.carried[4 + i], @intCast(j), .f32).mul(alpha[j / 2]); + }; + var packed_acc: [of]V = undefined; + for (0..of) |i| packed_acc[i] = packAcc(b, acc[i]); + var pv = b.openFor(0, n, 16, packed_acc); + { + const pa = matrixLoad(b, 4, false, t.shared.add(c.pBase()).add(swizzle(row_base.add(t.lane.rem(16)), pv.iv.add(t.lane.div(16).mul(8)), n))); + for (0..of) |i| { + const vb = matrixLoad(b, 2, true, t.shared.add(c.kvBase()).add(swizzle(pv.iv.add(t.lane.rem(16)), dim_part.add(i * 8), d))); + var prev: [4]V = undefined; + for (0..4) |j| prev[j] = b.vectorExtract(pv.carried[i], @intCast(j), .f32); + acc[i] = mma(b, dt, pa, vb, prev); + packed_acc[i] = packAcc(b, acc[i]); + } + pv.yield(packed_acc); + } + @memcpy(next[4..], &pv.results); + b.syncThreads(); + loop.yield(next); + } + for (0..2) |h| { + const row = first_row.add(row_base).add(row0).add(h * 8); + var valid = b.openIf(row.lt(qlen.mul(c.group))); + { + const m = loop.results[h]; + const l = loop.results[h + 2]; + var norm = b.select(l.gt(0), b.rcpApprox(l), zero); + if (c.splits == 1 and c.sink) { + const sink = loadI32(b, t.sink, t.head.mul(c.group).add(row.rem(c.group))).bitCast(.f32).mul(std.math.log2e); + const maximum = m.maximum(sink); + const weight = b.exp2Approx(m.sub(maximum)); + norm = weight.div(l.mul(weight).add(b.exp2Approx(sink.sub(maximum)))); + } + for (0..of) |i| for (0..2) |j| { + const value = b.vectorExtract(loop.results[4 + i], @intCast(h * 2 + j), .f32); + t.store(row, dim_part.add(i * 8).add(col0).add(j), if (c.splits > 1) value else value.mul(norm)); + }; + if (c.splits > 1) { + var leader = b.openIf(t.warp.rem(w).eq(0).bitAnd(t.lane.rem(4).eq(0))); + { + t.store(row, b.constant(.i32, d), m); + t.store(row, b.constant(.i32, d + 1), l); + leader.close(); + } + } + valid.close(); + } + } + active.close(); + } +} + +pub const Merge = cute.Kernel(Config, .{ + .name = "paged_attention_cute_sm120_merge", + .inputs = &.{ "partials", "starts", "sink" }, + .outputs = &.{"out"}, + .run = merge, +}); + +fn merge(b: *B, c: Config) cute.FinishError!void { + const a = try b.declareArgs(.{ + .partials = .{ .tensor = .{ .dtype = .f32, .shape = &.{ c.splits, c.tokens, c.kv_heads * c.group, c.head_dim + 2 } } }, + .starts = .{ .tensor = .{ .dtype = .i32, .shape = &.{c.batch + 1} } }, + .sink = .{ .tensor = .{ .dtype = .f32, .shape = &.{c.kv_heads * c.group} } }, + .out = .{ .tensor = .{ .dtype = c.dtype, .shape = &.{ c.tokens, c.kv_heads * c.group, c.head_dim } } }, + }); + const token = b.blockIdx().x; + const head = b.blockIdx().y; + var valid = b.openIf(token.lt(a.starts.get(.{c.batch}))); + { + const zero = b.constant(.f32, 0); + const sink = if (c.sink) a.sink.get(.{head}).mul(std.math.log2e) else b.constant(.f32, -std.math.inf(f32)); + var maximum = b.openFor(0, c.splits, 1, .{sink}); + { + maximum.yield(.{maximum.carried[0].maximum(a.partials.get(.{ maximum.iv, token, head, c.head_dim }))}); + } + const m = maximum.results[0].maximum(-std.math.floatMax(f32)); + var dims = b.openFor(b.threadIdx().x, c.head_dim, 128, .{}); + { + var sum = b.openFor(0, c.splits, 1, .{ zero, if (c.sink) b.exp2Approx(sink.sub(m)) else zero }); + { + const weight = b.exp2Approx(a.partials.get(.{ sum.iv, token, head, c.head_dim }).sub(m)); + sum.yield(.{ + sum.carried[0].add(weight.mul(a.partials.get(.{ sum.iv, token, head, dims.iv }))), + sum.carried[1].add(weight.mul(a.partials.get(.{ sum.iv, token, head, c.head_dim + 1 }))), + }); + } + a.out.set(.{ token, head, dims.iv }, sum.results[0].mul(b.select(sum.results[1].gt(0), b.rcpApprox(sum.results[1]), zero)).to(c.dtype)); + dims.close(); + } + valid.close(); + } +} diff --git a/zml/kernels/paged_attention/cute/sm120/sm120.zig b/zml/kernels/paged_attention/cute/sm120/sm120.zig new file mode 100644 index 00000000..6645abf1 --- /dev/null +++ b/zml/kernels/paged_attention/cute/sm120/sm120.zig @@ -0,0 +1,69 @@ +const stdx = @import("stdx"); + +const zml = @import("../../../../zml.zig"); +const cute = @import("../cute.zig"); +const kernel = @import("attention.zig"); + +pub const Options = cute.Options; +pub const Parameters = cute.Parameters; + +test { + _ = kernel; +} + +pub fn isAvailable(platform: *const zml.Platform) bool { + if (platform.target != .cuda) return false; + const cc = zml.platform.cuda.computeCapability(platform) orelse return false; + return cc.major == 12; +} + +pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: cute.AttentionOptions) zml.Tensor { + const q_ = q.transpose(.{ .b, .hkv, .hg, .hd }); + const k_ = k.transpose(.{ .page, .k_chunk, .hkv, .hd }); + const v_ = v.transpose(.{ .page, .k_chunk, .hkv, .hd }); + stdx.debug.assert(q_.dim(.hkv) == k_.dim(.hkv) and k_.shape().eql(v_.shape()) and k_.dim(.hd) == q_.dim(.hd) and q_.dim(.hg) > 0, "CuTe paged attention requires aligned local Q/KV heads and matching caches", .{}); + const decode = !parameters.options_.is_prefill; + var c: kernel.Config = .{ + .head_dim = q_.dim(.hd), + .tokens = q_.dim(.b), + .batch = parameters.block_table.dim(.b), + .kv_heads = k_.dim(.hkv), + .group = q_.dim(.hg), + .pages_per_seq = parameters.block_table.dim(.p), + .page_size = k_.dim(.k_chunk), + .max_query = if (decode) 1 else @min(q_.dim(.b), @as(i64, @intCast(parameters.options_.max_seqlen_q))), + .dtype = zml.kernel.cute.from(q_.dtype()), + .warps_per_query = if (decode) 4 else 2, + .kv_tile = if (q_.dim(.hd) == 512) 32 else 64, + .causal = opts.mask.isCausal(), + .window = opts.mask.slidingWindow() orelse 0, + .scale = opts.scale orelse 1 / @sqrt(@as(f32, @floatFromInt(q_.dim(.hd)))), + .sink = opts.sink != null, + }; + // Mixed batches also contain single-token rows. Partition their KV + // range so one long decode cannot hold up all the prefill CTAs. + const query_tiles = if (decode) c.batch * c.queryTiles() else c.batch + @divTrunc(c.tokens * c.group + c.rows() - 1, c.rows()); + const work = query_tiles * c.kv_heads; + const bound = if (c.window > 0) @min(c.window, c.pages_per_seq * c.page_size) else c.pages_per_seq * c.page_size; + c.splits = @max(1, @min(32, @min(@divTrunc(@as(i64, @intCast(cute.multiprocessorCount(zml.Compiler.current().platform))) * 2 + work - 1, work), @divTrunc(bound + 127, 128)))); + const sink = if (opts.sink) |s| s.convert(.f32).reshape(.{c.kv_heads * c.group}) else zml.Tensor.zeroes(.init(.{c.kv_heads * c.group}, .f32)); + const out = kernel.Kernel.call(.{ + .q = q_, + .k = k_, + .v = v_, + .table = parameters.block_table, + .lengths = parameters.seq_lens, + .starts = parameters.query_start_len, + .sink = sink, + }, .{ .out = if (c.splits > 1) zml.Shape.init(.{ c.splits, c.tokens, c.kv_heads * c.group, c.head_dim + 2 }, .f32) else q_.shape() }, .{ + .cfg = c, + .grid = .{ @intCast(c.batch * c.queryTiles()), @intCast(c.kv_heads), @intCast(c.splits) }, + .block = .{ 128, 1, 1 }, + }).out; + const result = if (c.splits > 1) kernel.Merge.call(.{ .partials = out, .starts = parameters.query_start_len, .sink = sink }, .{ .out = q_.shape() }, .{ + .cfg = c, + .grid = .{ @intCast(c.tokens), @intCast(c.kv_heads * c.group), 1 }, + .block = .{ 128, 1, 1 }, + }).out else out; + return result.transpose(q.shape()); +} diff --git a/zml/kernels/paged_attention/metal/BUILD.bazel b/zml/kernels/paged_attention/metal/BUILD.bazel new file mode 100644 index 00000000..39d92259 --- /dev/null +++ b/zml/kernels/paged_attention/metal/BUILD.bazel @@ -0,0 +1,9 @@ +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "metal", + srcs = [ + "metal.zig", + "paged_attn.zig", + ], +) diff --git a/zml/kernels/paged_attention/metal/README.md b/zml/kernels/paged_attention/metal/README.md new file mode 100644 index 00000000..8ccc6fd9 --- /dev/null +++ b/zml/kernels/paged_attention/metal/README.md @@ -0,0 +1,28 @@ +# Metal paged attention + +This backend is automatically selected for Metal. See the +[parent README](../README.md) for the attention equation, packed sequence +metadata, and page addressing contract. + +[The wrapper](metal.zig) requires split KV caches. It merges query head groups, +transposes queries and caches into the kernels' layouts, launches the +[Metal DSL kernels](paged_attn.zig), and restores the query's tagged layout for +the output. + +Queries and caches are `bf16` or `f16`; other dtypes panic. Causal `bf16` +decode with head dimension 32, 64, 128, 256 or 512 runs `fa_vec_paged`, which +is `bf16` only: a threadgroup holds up to four query heads of a KV head, times +splits of the KV positions, at most 256 threads. Rows split over the positions +are merged by `fa_vec_paged_reduce`; the split count follows the GPU core count +and the number of rows. Everything else, `f16` decode included, runs +`paged_attention_tiled` (head dimension 32, 64, 96, 128, 256 or 512, pages of +8, 16 or 32 positions), then `zero_padding_rows`. + +Causal and non-causal standard masks, left sliding windows, and a custom scale +are passed to the kernels. Explicit key indices are rejected, and +`opts.sink` is currently ignored. Fused caches are rejected by +ordinary paged attention. + +When sharding heads, keep complete GQA groups together with matching query and +cache head ownership. Each local computation needs the full required key range, +features, and matching sequence metadata. diff --git a/zml/kernels/paged_attention/metal/metal.zig b/zml/kernels/paged_attention/metal/metal.zig new file mode 100644 index 00000000..2cec68f5 --- /dev/null +++ b/zml/kernels/paged_attention/metal/metal.zig @@ -0,0 +1,240 @@ +const std = @import("std"); + +const stdx = @import("stdx"); + +const zml = @import("../../../zml.zig"); +const AttentionOptions = @import("../options.zig").AttentionOptions; +const paged_attn = @import("paged_attn.zig"); + +const log = std.log.scoped(.@"zml/attention/metal"); + +test { + _ = paged_attn; +} + +pub const Options = struct { + batch_size: usize, + max_num_pages: usize, + max_seqlen_q: usize, + is_prefill: bool, + + pub fn isPrefill(self: Options) bool { + return self.is_prefill; + } + + pub fn maxNumPages(self: Options) usize { + return self.max_num_pages; + } +}; + +pub const Parameters = struct { + block_table: zml.Tensor, + seq_lens: zml.Tensor, + query_start_len: zml.Tensor, + options_: Options, + + pub fn init(options_: Options) Parameters { + return .{ + .block_table = .init(.{ .b = options_.batch_size, .p = options_.max_num_pages }, .i32), + .seq_lens = .init(.{ .b = options_.batch_size }, .i32), + .query_start_len = .init(.{ .b = options_.batch_size + 1 }, .i32), + .options_ = options_, + }; + } + + pub fn allocationSize(self: Parameters) usize { + var allocation_size: usize = 0; + + allocation_size += self.block_table.byteSize(); + allocation_size += self.seq_lens.byteSize(); + allocation_size += self.query_start_len.byteSize(); + + return allocation_size; + } + + pub fn options(self: Parameters) Options { + return self.options_; + } +}; + +pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions) zml.Tensor { + if (opts.mask == .indices) @panic("Sparse Attention is not implemented on Metal"); + + const num_kv_heads = q.dim(.hkv); + + const qh = q.merge(.{ .h = .{ .hkv, .hg } }).transpose(.{ .b, .h, .hd }); + const kc = k_cache.transpose(.{ .page, .k_chunk, .hkv, .hd }); + const vc = v_cache.transpose(.{ .page, .k_chunk, .hkv, .hd }); + + const sliding_window: i32 = if (opts.mask.slidingWindow()) |window| @intCast(window) else -1; + + const out = dispatch(qh, kc, vc, parameters.block_table, parameters.seq_lens, parameters.query_start_len, .{ + .is_causal = opts.mask.isCausal(), + .sliding_window = sliding_window, + .scale = opts.scale orelse 1.0 / @sqrt(@as(f32, @floatFromInt(q.dim(.hd)))), + }); + + // [total_q_tokens, num_heads, head_dim] -> {.b, .hkv, .hg, .hd} (q's order) + return out.splitAxis(.h, .{ .hkv = num_kv_heads, .hg = .auto }).transpose(q.shape()); +} + +const DispatchOptions = struct { + is_causal: bool, + /// Below 0 for global attention. + sliding_window: i32, + scale: f32, +}; + +/// Launches fa_vec_paged for causal bf16 decode and paged_attention_tiled +/// otherwise. +/// `q` and the result are `[tokens, heads, head_dim]`, the caches +/// `[pages, block_size, kv_heads, head_dim]`, `block_table` +/// `[seqs, max_blocks]`, `seq_lens` `[seqs]` and `query_start_len` +/// `[seqs + 1]`, all i32. +fn dispatch(q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, block_table: zml.Tensor, seq_lens: zml.Tensor, query_start_len: zml.Tensor, opts: DispatchOptions) zml.Tensor { + const dtype = q.dtype(); + if (dtype != .bf16 and dtype != .f16) std.debug.panic("paged attention: Metal takes bf16 or f16 queries, not {t}", .{dtype}); + if (k_cache.dtype() != dtype or v_cache.dtype() != dtype) std.debug.panic("paged attention: Metal takes caches of the queries' {t}, not {t} and {t}", .{ dtype, k_cache.dtype(), v_cache.dtype() }); + + const total_q_tokens: i32 = @intCast(q.dim(0)); + const num_heads: i32 = @intCast(q.dim(1)); + const head_dim: i32 = @intCast(q.dim(2)); + const block_size: i32 = @intCast(k_cache.dim(1)); + const num_kv_heads: i32 = @intCast(k_cache.dim(2)); + const num_seqs: i32 = @intCast(block_table.dim(0)); + const max_blocks: i32 = @intCast(block_table.dim(1)); + + const vec_decode = opts.is_causal and total_q_tokens == num_seqs and + (head_dim == 32 or head_dim == 64 or head_dim == 128 or head_dim == 256 or head_dim == 512) and + dtype == .bf16; + + if (vec_decode) { + const args: paged_attn.FaVecPaged.HostArgs = .{ + .num_heads = num_heads, + .n_kv = num_kv_heads, + .max_blocks = max_blocks, + .num_seqs = num_seqs, + .scale = opts.scale, + .sliding_window = opts.sliding_window, + }; + + const inputs: paged_attn.FaVecPaged.Kernel.Inputs = .{ + .q = q, + .k_cache = k_cache, + .v_cache = v_cache, + .block_tables = block_table, + .seq_lens = seq_lens, + .cu_seqlens_q = query_start_len, + }; + + const kv_capacity = max_blocks * block_size; + const span = if (opts.sliding_window < 0) kv_capacity else @min(kv_capacity, opts.sliding_window + 32); + const nrows = total_q_tokens * num_heads; + const launch = vecLaunch(nrows, @divExact(num_heads, num_kv_heads), span, coreCount()); + + const cfg: paged_attn.FaVecPaged.Cfg = .{ + .head_dim = head_dim, + .block_size = block_size, + .position_stride = num_kv_heads * head_dim, + .heads = launch.heads, + .splits = launch.splits, + }; + const dst = if (launch.splits == 1) q.shape() else zml.Shape.init(.{nrows * (head_dim * launch.splits + 2 * launch.splits)}, .f32); + const out = paged_attn.FaVecPaged.Kernel.call(inputs, .{ .dst = dst }, .{ + .cfg = cfg, + .grid = .{ @intCast(@divExact(launch.splits, launch.splits_per_threadgroup)), @intCast(@divExact(num_heads, launch.heads)), @intCast(total_q_tokens) }, + .threadgroup = .{ 32, @intCast(launch.heads * launch.splits_per_threadgroup), 1 }, + .constants = .{ .args = .of(args) }, + }).dst; + if (launch.splits == 1) return out; + + return paged_attn.FaVecPagedReduce.Kernel.call(.{ .htmp = out, .seq_lens = seq_lens, .cu_seqlens_q = query_start_len }, .{ .dst = q.shape() }, .{ + .cfg = .{ .splits = launch.splits, .head_dim = head_dim }, + .grid = .{ @intCast(nrows), @intCast(@divExact(head_dim, 4 * paged_attn.FaVecPagedReduce.simdgroups)), 1 }, + .threadgroup = .{ 32, paged_attn.FaVecPagedReduce.simdgroups, 1 }, + .constants = .{ .args = .of(args) }, + }).dst; + } + + const tile = paged_attn.PagedAttentionTiled.tileConfig(head_dim) orelse std.debug.panic("paged attention: head_dim {d} unsupported", .{head_dim}); + + const cfg: paged_attn.PagedAttentionTiled.Cfg = .{ + .dtype = zml.kernel.metal.from(dtype), + .head_size = head_dim, + .block_size = block_size, + .bq = tile[0], + .tile_kv = tile[1], + .num_threads = tile[2], + .causal = opts.is_causal, + }; + + const total_q_blocks = @divTrunc(total_q_tokens, tile[0]) + num_seqs; + + const out = paged_attn.PagedAttentionTiled.Kernel.call(.{ + .q = q, + .k_cache = k_cache, + .v_cache = v_cache, + .block_tables = block_table, + .context_lens = seq_lens, + .cu_seqlens_q = query_start_len, + }, .{ .out = q.shape() }, .{ + .cfg = cfg, + .grid = .{ @intCast(num_heads), @intCast(total_q_blocks), 1 }, + .threadgroup = .{ @intCast(tile[2]), 1, 1 }, + .threadgroup_memory = paged_attn.PagedAttentionTiled.threadgroupMemory(cfg), + .constants = .{ + .num_kv_heads = .of(num_kv_heads), + .scale = .of(opts.scale), + .softcapping = .of(@as(f32, 0)), + .max_num_blocks_per_seq = .of(max_blocks), + .q_stride = .of(num_heads * head_dim), + .kv_block_stride = .of(block_size * num_kv_heads * head_dim), + .kv_head_stride = .of(head_dim), + .num_seqs = .of(num_seqs), + .sliding_window = .of(opts.sliding_window), + }, + }).out; + + // The tiled kernel leaves the padding rows untouched, so they are zeroed in + // its output. Zeroing a fresh buffer first, from cu_seqlens_q alone, would + // make XLA share that buffer across every call and copy it for each. + return paged_attn.ZeroPaddingRows.Kernel.call(.{ .cu_seqlens_q = query_start_len, .out = out }, .{ .out = q.shape() }, .{ + .cfg = .{ .dtype = cfg.dtype }, + .grid = .{ @intCast(total_q_tokens), 1, 1 }, + .threadgroup = .{ 256, 1, 1 }, + .constants = .{ .num_seqs = .of(num_seqs), .row_stride = .of(num_heads * head_dim) }, + }).out; +} + +const VecLaunch = struct { + heads: i32, + splits: i32, + splits_per_threadgroup: i32, +}; + +/// At most 256 threads, which every Apple GPU pipeline accepts. Splits past a +/// row's length exit early, so `splits` follows the core and row counts. +fn vecLaunch(rows: i32, group: i32, span: i32, cores: i32) VecLaunch { + var heads: i32 = @min(group, 4); + while (@rem(group, heads) != 0) heads -= 1; + const want: i32 = if (rows <= 2 * cores) 128 else 64; + const chunks = @divFloor(span + 31, 32); + var splits: i32 = 1; + while (splits * 2 <= @min(want, chunks)) splits *= 2; + return .{ .heads = heads, .splits = splits, .splits_per_threadgroup = @min(splits, @divFloor(8, heads)) }; +} + +fn coreCount() i32 { + const platform = zml.Compiler.current().platform; + const attribute = platform.devices[0].pjrt_desc.attribute(platform.pjrt_api, "core_count") orelse + std.debug.panic("Metal device has no core_count attribute", .{}); + return @intCast(attribute.int64); +} + +test vecLaunch { + try std.testing.expectEqual(VecLaunch{ .heads = 4, .splits = 128, .splits_per_threadgroup = 2 }, vecLaunch(16, 4, 128000, 40)); + try std.testing.expectEqual(VecLaunch{ .heads = 2, .splits = 64, .splits_per_threadgroup = 4 }, vecLaunch(256, 2, 128000, 40)); + try std.testing.expectEqual(VecLaunch{ .heads = 4, .splits = 64, .splits_per_threadgroup = 2 }, vecLaunch(512, 4, 128000, 40)); + try std.testing.expectEqual(VecLaunch{ .heads = 3, .splits = 8, .splits_per_threadgroup = 2 }, vecLaunch(48, 6, 288, 40)); + try std.testing.expectEqual(VecLaunch{ .heads = 1, .splits = 1, .splits_per_threadgroup = 1 }, vecLaunch(8, 1, 20, 40)); +} diff --git a/zml/kernels/paged_attention/metal/paged_attn.zig b/zml/kernels/paged_attention/metal/paged_attn.zig new file mode 100644 index 00000000..04f4c1ac --- /dev/null +++ b/zml/kernels/paged_attention/metal/paged_attn.zig @@ -0,0 +1,908 @@ +//! Metal paged attention kernels: fa_vec_paged and its split-K reduce for +//! decode, and paged_attention_tiled with zero_padding_rows otherwise. + +const std = @import("std"); + +const metal = @import("../../../kernel/kernel.zig").metal; + +const ArgSpec = metal.ArgSpec; +const Builder = metal.Builder; +const Type = metal.Type; +const Value = metal.Value; + +const flt_max = std.math.floatMax(f32); +/// A masked score: below any real one yet finite, since fast math assumes +/// no infinities. Halved so subtracting a running maximum cannot overflow. +const masked_score = -flt_max / 2.0; +const max_half = 65504.0; +const simd_width = 32; + +/// fa_vec_paged: bf16 decode over paged caches. Simdgroup s takes query head +/// s % heads of the threadgroup's KV head group and position split s / heads. +pub const FaVecPaged = struct { + pub const Cfg = struct { + /// The key and value head dimension. + head_dim: i32, + /// The page size. + block_size: i32, + /// The in-page position stride, n_kv * head_dim elements. + position_stride: i32, + heads: i32, + splits: i32, + }; + + pub const Args: metal.Struct = .{ .name = "fa_vec_paged_args", .fields = &.{ + .{ .name = "num_heads", .type = .int }, + .{ .name = "n_kv", .type = .int }, + .{ .name = "max_blocks", .type = .int }, + .{ .name = "num_seqs", .type = .int }, + .{ .name = "scale", .type = .float }, + .{ .name = "sliding_window", .type = .int }, + } }; + + pub const HostArgs = extern struct { + num_heads: i32, + n_kv: i32, + max_blocks: i32, + num_seqs: i32, + scale: f32, + /// Below 0 for global attention. + sliding_window: i32, + }; + + pub const Kernel = metal.Kernel(Cfg, .{ + .name = "fa_vec_paged", + .inputs = &.{ "q", "k_cache", "v_cache", "block_tables", "seq_lens", "cu_seqlens_q" }, + .outputs = &.{"dst"}, + .constants = &.{"args"}, + .run = run, + }); + + fn run(b: *Builder, cfg: Cfg) metal.FinishError!void { + switch (cfg.head_dim) { + 32, 64, 128, 256, 512 => {}, + else => std.debug.panic("fa_vec_paged: head_dim {d} unsupported (32, 64, 128, 256 or 512)", .{cfg.head_dim}), + } + const char_in: ArgSpec = .{ .buffer = .{ .type = .char, .const_ = true } }; + const int_in: ArgSpec = .{ .buffer = .{ .type = .int, .const_ = true } }; + const a = try b.declareArgs(.{ + .args = .{ .constant_struct = Args }, + .q = char_in, + .k_cache = char_in, + .v_cache = char_in, + .block_tables = int_in, + .seq_lens = int_in, + .cu_seqlens_q = int_in, + .dst = .{ .buffer = .{ .type = .char } }, + .tgpig = .{ .threadgroup_position_in_grid = .uint3 }, + .tgpg = .{ .threadgroups_per_grid = .uint3 }, + .tiisg = .{ .thread_index_in_simdgroup = .ushort }, + .sgitg = .{ .simdgroup_index_in_threadgroup = .ushort }, + .nsg = .{ .simdgroups_per_threadgroup = .ushort }, + }); + impl(b, cfg, a); + } + + fn impl(b: *Builder, cfg: Cfg, a: anytype) void { + const D = cfg.head_dim; + const D4 = @divExact(D, 4); + // A K/V row spans NL lanes of NI float4s; a simdgroup reads R rows per step. + const NL = @min(D4, simd_width); + const R = @divExact(simd_width, NL); + const NI = @divExact(D4, NL); + const C = simd_width; + const BS = cfg.block_size; + const PS = cfg.position_stride; + const HG = cfg.heads; + const NWG = cfg.splits; + const full: metal.LoopOpts = .{ .unroll = .full }; + + const num_heads = a.args.field("num_heads"); + const group = num_heads.div(a.args.field("n_kv")); + const groups = group.div(HG); + const r = a.tgpig.at(2).to(.int); + const tg_head = a.tgpig.at(1).to(.int); + const ikv = tg_head.div(groups); + const iq2 = ikv.mul(group).add(tg_head.rem(groups).mul(HG)).add(a.sgitg.rem(HG).to(.int)); + const iwg = a.tgpig.at(0).to(.int).mul(a.nsg.div(HG).to(.int)).add(a.sgitg.div(HG).to(.int)); + const rid = r.to(.long).mul(num_heads.to(.uint)).add(iq2); + const nrows = a.tgpg.at(2).to(.long).mul(num_heads.to(.uint)); + + // Padding rows: FaVecPagedReduce zeroes them when there are splits. + var padding = b.openIf(r.ge(a.cu_seqlens_q.load(a.args.field("num_seqs")))); + { + if (NWG == 1) { + const dst4 = a.dst.to(.bfloat4).add(rid.mul(D4)); + var loop = b.openFor(a.tiisg.to(.short), D4, simd_width); + { + dst4.store(loop.iv, b.constant(.float4, 0).to(.bfloat4)); + loop.close(); + } + } + b.return_(); + padding.close(); + } + + const seq = sequenceOf(b, a, r); + const kv_len = seq.kv_len; + const kv_lo = seq.kv_lo; + const ic0_lo = kv_lo.div(C); + const live = if (NWG > 1) liveSplits(b, seq, NWG) else b.lift(@as(i32, 1)); + + if (NWG > 1) { + var idle = b.openIf(iwg.ge(live)); + { + b.return_(); + idle.close(); + } + } + + const pt = a.block_tables.add(seq.idx.to(.ulong).mul(a.args.field("max_blocks").to(.uint))); + const k_cache = a.k_cache.add(ikv.to(.ulong).mul(D).mul(2)); + const v_cache = a.v_cache.add(ikv.to(.ulong).mul(D).mul(2)); + + const lane = if (R == 1) a.tiisg else a.tiisg.rem(NL); + const sub = a.tiisg.div(NL).to(.int); + const q4 = a.q.add(rid.mul(D).mul(2)).to(.bfloat4).add(lane); + const qr = b.array(.float4, @intCast(NI)); + const o = b.array(.float4, @intCast(NI)); + var init = b.openForOpts(b.constant(.short, 0), NI, 1, full); + { + const ii = init.iv; + qr.store(ii, q4.load(ii.mul(NL)).to(.float4)); + o.store(ii, b.constant(.float4, 0)); + init.close(); + } + + const S = b.local(.float, 0.0); + const M = b.local(.float, -flt_max / 2.0); + + const ic0 = b.local(.int, ic0_lo.add(iwg)); + var blocks = b.openLoop(.{}); + { + const ic = ic0.mul(C); + var past = b.openIf(ic.ge(kv_len)); + { + blocks.breakLoop(); + past.close(); + } + + const s = b.local(.float, 0.0); + var qk = b.openForOpts(b.constant(.short, 0), @divExact(C, R), 1, full); + { + const cc = qk.iv; + const p = b.min(if (R == 1) ic.add(cc) else ic.add(cc.to(.int).mul(R)).add(sub), kv_len.sub(1)); + const pk4 = pagedRow(b, k_cache, pt, p, BS, PS).to(.bfloat4).add(lane); + const d = b.local(.float, 0.0); + var dots = b.openForOpts(b.constant(.short, 0), NI, 1, full); + { + const ii = dots.iv; + d.assign(d.add(b.dot(pk4.load(ii.mul(NL)).to(.float4), qr.load(ii)))); + dots.close(); + } + if (R == 1) { + s.assign(b.ternary(a.tiisg.to(.short).eq(cc), b.simdSum(d.rvalue()), s.rvalue())); + } else { + var x = d.rvalue(); + var mask: i32 = 1; + while (mask < NL) : (mask *= 2) x = x.add(b.simdShuffleXor(x, b.constant(.ushort, mask))); + const t = b.simdShuffle(x, a.tiisg.rem(R).mul(NL)); + s.assign(b.ternary(a.tiisg.div(R).to(.short).eq(cc), t, s.rvalue())); + } + qk.close(); + } + + const pos = ic.add(a.tiisg.to(.int)); + const in_kv = pos.lt(kv_len).bitAnd(pos.ge(kv_lo)); + const score = b.ternary(in_kv, s.mul(a.args.field("scale")), b.constant(.float, -max_half)); + + const m = M.rvalue(); + M.assign(b.simdMax(b.max(M.rvalue(), score))); + const ms = b.exp(m.sub(M)); + const vs = b.exp(score.sub(M)); + S.assign(S.mul(ms).add(b.simdSum(vs))); + + var rescale = b.openForOpts(b.constant(.short, 0), NI, 1, full); + { + o.store(rescale.iv, o.load(rescale.iv).mul(ms)); + rescale.close(); + } + + var pv = b.openForOpts(b.constant(.short, 0), @divExact(C, R), 1, full); + { + const cc = pv.iv; + const row = cc.to(.int).mul(R).add(sub); + const p = b.min(if (R == 1) ic.add(cc) else ic.add(row), kv_len.sub(1)); + const pv4 = pagedRow(b, v_cache, pt, p, BS, PS).to(.bfloat4).add(lane); + const w = b.vec(.float4, .{if (R == 1) b.simdBroadcast(vs, cc.to(.ushort)) else b.simdShuffle(vs, row)}); + var dims = b.openForOpts(b.constant(.short, 0), NI, 1, full); + { + const ii = dims.iv; + o.store(ii, o.load(ii).add(pv4.load(ii.mul(NL)).to(.float4).mul(w))); + dims.close(); + } + pv.close(); + } + + blocks.next(); + ic0.assign(ic0.add(live)); + blocks.close(); + } + + var mask: i32 = NL; + while (mask < simd_width) : (mask *= 2) { + var sum = b.openForOpts(b.constant(.short, 0), NI, 1, full); + { + o.store(sum.iv, o.load(sum.iv).add(b.simdShuffleXor(o.load(sum.iv), b.constant(.ushort, mask)))); + sum.close(); + } + } + var first = if (R > 1) b.openIf(sub.eq(0)) else null; + if (NWG == 1) { + const inv_sum = b.ternary(S.eq(0.0), 0.0, b.lift(1.0).div(S)); + const dst4 = a.dst.to(.bfloat4).add(rid.mul(D4)).add(lane); + var out = b.openForOpts(b.constant(.short, 0), NI, 1, full); + { + dst4.store(out.iv.mul(NL), o.load(out.iv).mul(inv_sum).to(.bfloat4)); + out.close(); + } + } else { + // [rows][D4][splits] float4 partials, then [rows][splits] (S, M). + const dst4 = a.dst.to(.float4).add(rid.mul(D4).mul(NWG)); + var out = b.openForOpts(b.constant(.short, 0), NI, 1, full); + { + const i = lane.to(.int).add(out.iv.to(.int).mul(NL)); + dst4.store(i.mul(NWG).add(iwg), o.load(out.iv)); + out.close(); + } + var lane0 = b.openIf(a.tiisg.eq(0)); + { + const dst1 = a.dst.to(.float).add(nrows.mul(D).mul(NWG)).add(rid.mul(2 * NWG)); + dst1.store(b.lift(2).mul(iwg), S); + dst1.store(b.lift(2).mul(iwg).add(1), M); + lane0.close(); + } + } + if (first) |*scope| scope.close(); + } + + /// The address of position `p`'s bf16 row in `cache`, through the page + /// table `pt`. + fn pagedRow(b: *Builder, cache: Value, pt: Value, p: Value, BS: i32, PS: i32) Value { + const page = pt.load(p.div(BS)).to(.uint).to(.ulong).mul(b.constant(.uint, BS * PS)); + const row = p.rem(BS).to(.uint).mul(b.constant(.uint, PS)); + return cache.add(page.add(row).mul(2)); + } +}; + +/// The sequence of query row `r` and its causal KV range [kv_lo, kv_len). +fn sequenceOf(b: *Builder, a: anytype, r: Value) struct { idx: Value, kv_len: Value, kv_lo: Value } { + const first_seq = b.local(.int, 0); + const last_seq = b.local(.int, a.args.field("num_seqs").sub(1)); + var search = b.openLoop(.{}); + { + search.body(first_seq.lt(last_seq)); + } + { + const mid = first_seq.add(last_seq).add(1).div(2); + var half = b.openIfElse(a.cu_seqlens_q.load(mid).le(r), .{}); + { + first_seq.assign(mid); + half.elseBranch(); + } + { + last_seq.assign(mid.sub(1)); + half.close(); + } + search.close(); + } + const idx = first_seq.rvalue(); + const q_pos = r.sub(a.cu_seqlens_q.load(idx)); + const query_len = a.cu_seqlens_q.load(idx.add(1)).sub(a.cu_seqlens_q.load(idx)); + const kv_len = a.seq_lens.load(idx).sub(query_len).add(q_pos).add(1); + const kv_lo = b.ternary(a.args.field("sliding_window").lt(0), 0, b.max(0, kv_len.sub(a.args.field("sliding_window")))); + return .{ .idx = idx, .kv_len = kv_len, .kv_lo = kv_lo }; +} + +fn liveSplits(b: *Builder, seq: anytype, splits: i32) Value { + const chunks = seq.kv_len.add(simd_width - 1).div(simd_width).sub(seq.kv_lo.div(simd_width)); + return b.min(splits, chunks); +} + +/// fa_vec_paged_reduce: merges the fa_vec_paged split partials. +pub const FaVecPagedReduce = struct { + pub const Cfg = struct { + splits: i32, + /// The head dimension. + head_dim: i32, + }; + + pub const simdgroups = 8; + + pub const Kernel = metal.Kernel(Cfg, .{ + .name = "fa_vec_paged_reduce", + .inputs = &.{ "htmp", "seq_lens", "cu_seqlens_q" }, + .outputs = &.{"dst"}, + .constants = &.{"args"}, + .run = run, + }); + + fn run(b: *Builder, cfg: Cfg) metal.FinishError!void { + const int_in: ArgSpec = .{ .buffer = .{ .type = .int, .const_ = true } }; + const a = try b.declareArgs(.{ + .args = .{ .constant_struct = FaVecPaged.Args }, + .htmp = .{ .buffer = .{ .type = .char, .const_ = true } }, + .seq_lens = int_in, + .cu_seqlens_q = int_in, + .dst = .{ .buffer = .{ .type = .char } }, + .tgpig = .{ .threadgroup_position_in_grid = .uint2 }, + .tgpg = .{ .threadgroups_per_grid = .uint2 }, + .tiisg = .{ .thread_index_in_simdgroup = .ushort }, + .sgitg = .{ .simdgroup_index_in_threadgroup = .ushort }, + }); + const NWG = cfg.splits; + if (NWG > 4 * simd_width) std.debug.panic("fa_vec_paged_reduce: {d} splits, at most {d}", .{ NWG, 4 * simd_width }); + const NP = @divFloor(NWG + simd_width - 1, simd_width); + const DV = cfg.head_dim; + const DV4 = @divExact(DV, 4); + const full: metal.LoopOpts = .{ .unroll = .full }; + + const rid = a.tgpig.at(0).to(.ulong); + const r = a.tgpig.at(0).div(a.args.field("num_heads").to(.uint)).to(.int); + const i = a.tgpig.at(1).to(.int).mul(simdgroups).add(a.sgitg.to(.int)); + const dst4 = a.dst.to(.bfloat4).add(rid.mul(DV4)); + + const live = b.local(.int, 0); + var real = b.openIf(r.lt(a.cu_seqlens_q.load(a.args.field("num_seqs")))); + { + live.assign(liveSplits(b, sequenceOf(b, a, r), NWG)); + real.close(); + } + var empty = b.openIf(live.le(0)); + { + var lane0 = b.openIf(a.tiisg.eq(0)); + { + dst4.store(i, b.constant(.float4, 0).to(.bfloat4)); + lane0.close(); + } + b.return_(); + empty.close(); + } + + const ss = a.htmp.to(.float).add(a.tgpg.at(0).to(.ulong).mul(DV).mul(NWG)).add(rid.mul(2 * NWG)); + const Mj = b.array(.float, @intCast(NP)); + const Sj = b.array(.float, @intCast(NP)); + const mx = b.local(.float, -flt_max / 2.0); + var load = b.openForOpts(b.constant(.short, 0), NP, 1, full); + { + const j = load.iv; + const idx = a.tiisg.to(.int).add(j.to(.int).mul(simd_width)); + const ok = idx.lt(live); + Mj.store(j, b.ternary(ok, ss.load(idx.mul(2).add(1)), b.constant(.float, -flt_max / 2.0))); + Sj.store(j, b.ternary(ok, ss.load(idx.mul(2)), b.constant(.float, 0))); + mx.assign(b.max(mx.rvalue(), Mj.load(j))); + load.close(); + } + const m = b.simdMax(mx.rvalue()); + const sum = b.local(.float, 0.0); + var weigh = b.openForOpts(b.constant(.short, 0), NP, 1, full); + { + const j = weigh.iv; + Mj.store(j, b.exp(Mj.load(j).sub(m))); + sum.assign(sum.add(Sj.load(j).mul(Mj.load(j)))); + weigh.close(); + } + const S = b.simdSum(sum.rvalue()); + const inv_sum = b.ternary(S.eq(0.0), 0.0, b.lift(1.0).div(S)); + + const htmp4 = a.htmp.to(.float4).add(rid.mul(DV4).mul(NWG)); + const acc = b.local(.float4, b.constant(.float4, 0)); + var parts = b.openForOpts(b.constant(.short, 0), NP, 1, full); + { + const j = parts.iv; + const idx = a.tiisg.to(.int).add(j.to(.int).mul(simd_width)); + var ok = b.openIf(idx.lt(live)); + { + acc.assign(acc.add(htmp4.load(i.mul(NWG).add(idx)).mul(Mj.load(j)))); + ok.close(); + } + parts.close(); + } + const v = b.simdSum(acc.rvalue()); + var lane0 = b.openIf(a.tiisg.eq(0)); + { + dst4.store(i, v.mul(inv_sum).to(.bfloat4)); + lane0.close(); + } + } +}; + +/// zero_padding_rows: zeroes the output rows past the last real token. +pub const ZeroPaddingRows = struct { + pub const Cfg = struct { dtype: Type }; + + pub const Kernel = metal.Kernel(Cfg, .{ + .name = "zero_padding_rows", + .inputs = &.{ "cu_seqlens_q", "out" }, + .outputs = &.{"out"}, + .constants = &.{ "num_seqs", "row_stride" }, + .run = run, + }); + + fn run(b: *Builder, cfg: Cfg) metal.FinishError!void { + switch (cfg.dtype.scalar) { + .bfloat, .half => {}, + else => std.debug.panic("zero_padding_rows: {s} unsupported (half or bfloat)", .{cfg.dtype.name()}), + } + const a = try b.declareArgs(.{ + .out = .{ .buffer = .{ .type = cfg.dtype } }, + .cu_seqlens_q = .{ .buffer = .{ .type = .int, .const_ = true } }, + .num_seqs = .{ .constant = .int }, + .row_stride = .{ .constant = .int }, + .tgp = .{ .threadgroup_position_in_grid = .uint3 }, + .tpt = .{ .thread_position_in_threadgroup = .uint3 }, + .ntg = .{ .threads_per_threadgroup = .uint3 }, + }); + const row = a.tgp.x().to(.int); + b.returnIf(row.lt(a.cu_seqlens_q.load(a.num_seqs))); + const dst = a.out.add(row.to(.ulong).mul(a.row_stride.to(.ulong))); + var loop = b.openFor(a.tpt.x().to(.int), a.row_stride, a.ntg.x().to(.int)); + { + dst.store(loop.iv, b.constant(cfg.dtype, 0)); + loop.close(); + } + } +}; + +/// paged_attention_tiled: FlashAttention-2 over 8x8 simdgroup matrices, causal +/// or bidirectional, one threadgroup per (query head, query block). +pub const PagedAttentionTiled = struct { + pub const Cfg = struct { + /// half or bfloat. + dtype: Type, + head_size: i32, + block_size: i32, + bq: i32 = 32, + tile_kv: i32 = 32, + num_threads: i32 = 128, + /// Bidirectional when false. + causal: bool = true, + }; + + pub const Kernel = metal.Kernel(Cfg, .{ + .name = "paged_attention_tiled", + .inputs = &.{ "q", "k_cache", "v_cache", "block_tables", "context_lens", "cu_seqlens_q" }, + .outputs = &.{"out"}, + .constants = &.{ "num_kv_heads", "scale", "softcapping", "max_num_blocks_per_seq", "q_stride", "kv_block_stride", "kv_head_stride", "num_seqs", "sliding_window" }, + .run = run, + }); + + /// What the MSL instantiates: half or bfloat, a head size of tileConfig + /// with its tiles, and pages of 8, 16 or 32 positions. + fn supported(cfg: Cfg) bool { + const tile = tileConfig(cfg.head_size) orelse return false; + return (cfg.dtype.eql(.half) or cfg.dtype.eql(.bfloat)) and + (cfg.block_size == 8 or cfg.block_size == 16 or cfg.block_size == 32) and + cfg.bq == tile[0] and cfg.tile_kv == tile[1] and cfg.num_threads == tile[2]; + } + + /// The (bq, tile_kv, num_threads) of a head size, null when unsupported. + pub fn tileConfig(head_size: i64) ?[3]i32 { + return switch (head_size) { + 32, 64, 96, 128 => .{ 32, 32, 128 }, + 256 => .{ 16, 16, 64 }, + 512 => .{ 8, 8, 32 }, + else => null, + }; + } + + pub fn threadgroupMemory(cfg: Cfg) u64 { + const ld = cfg.head_size + @divExact(16, @as(i32, @intCast(cfg.dtype.size()))); + return @intCast((cfg.bq + 2 * cfg.tile_kv) * ld * @as(i32, @intCast(cfg.dtype.size()))); + } + + /// Lane `lane_id`'s (column, row) in an 8x8 simdgroup_matrix fragment. + fn fragCoord(b: *Builder, lane_id: Value) Value { + const qid = lane_id.to(.short).div(4).to(.short); + const fm = qid.bitAnd(4).add(lane_id.to(.short).div(2).rem(4)).to(.short); + const fn_ = qid.bitAnd(2).mul(2).add(lane_id.to(.short).rem(2).mul(2)).to(.short); + return b.vec(.short2, .{ fn_, fm }); + } + + /// Reduces `v` with `op` across the lanes that hold one row of an 8x8 + /// simdgroup_matrix fragment. + fn fragRowReduce(b: *Builder, comptime op: enum { max, sum }, v: Value) Value { + var x = v.rvalue(); + inline for (.{ 1, 8 }) |mask| { + const y = b.simdShuffleXor(x, b.constant(.ushort, mask)); + x = switch (op) { + .max => b.max(x, y), + .sum => x.add(y), + }; + } + return x; + } + + /// Sets lane `lane` of the vector `p[i]` to `x`. + fn setLane(p: Value, i: Value, lane: anytype, x: Value) void { + p.store(i, p.load(i).insert(lane, x)); + } + + fn run(b: *Builder, cfg: Cfg) metal.FinishError!void { + if (!supported(cfg)) std.debug.panic("paged_attention_tiled: {s} at head size {d}, block size {d} (8, 16 or 32) and tiles {d}, {d}, {d} unsupported", .{ cfg.dtype.name(), cfg.head_size, cfg.block_size, cfg.bq, cfg.tile_kv, cfg.num_threads }); + const T = cfg.dtype; + const HEAD_SIZE = cfg.head_size; + const BLOCK_SIZE = cfg.block_size; + const BQ = cfg.bq; + const TILE_KV = cfg.tile_kv; + const NUM_THREADS = cfg.num_threads; + const int_arg: ArgSpec = .{ .constant = .int }; + const float_arg: ArgSpec = .{ .constant = .float }; + const a = try b.declareArgs(.{ + .out = .{ .buffer = .{ .type = T } }, + .q = .{ .buffer = .{ .type = T, .const_ = true } }, + .k_cache = .{ .buffer = .{ .type = T, .const_ = true } }, + .v_cache = .{ .buffer = .{ .type = T, .const_ = true } }, + .num_kv_heads = int_arg, + .scale = float_arg, + .softcapping = float_arg, + .block_tables = .{ .buffer = .{ .type = .uint, .const_ = true } }, + .context_lens = .{ .buffer = .{ .type = .uint, .const_ = true } }, + .max_num_blocks_per_seq = int_arg, + .q_stride = int_arg, + .kv_block_stride = int_arg, + .kv_head_stride = int_arg, + .cu_seqlens_q = .{ .buffer = .{ .type = .int, .const_ = true } }, + .num_seqs = int_arg, + .sliding_window = int_arg, + .shared_mem = .{ .threadgroup = .char }, + .tgp = .{ .threadgroup_position_in_grid = .uint3 }, + .tgpg = .{ .threadgroups_per_grid = .uint3 }, + .tpt = .{ .thread_position_in_threadgroup = .uint3 }, + .sg_idx = .{ .simdgroup_index_in_threadgroup = .uint }, + .lane = .{ .thread_index_in_simdgroup = .uint }, + }); + + const NUM_SIMD_LANES = 32; + const NUM_SG = @divExact(NUM_THREADS, NUM_SIMD_LANES); + const TD = @divExact(HEAD_SIZE, 8); + const TK = @divExact(TILE_KV, 8); + if (@divExact(BQ, NUM_SG) != 8) @panic("paged_attention_tiled: ROWS_PER_SG must be 8"); + const unroll: metal.LoopOpts = .{ .unroll = .enable }; + + const thread_idx = a.tpt.x().to(.int); + const head_idx = a.tgp.x().to(.int); + const q_block_global_idx = a.tgp.y().to(.int); + const num_heads = a.tgpg.x().to(.int); + const num_queries_per_kv = num_heads.div(a.num_kv_heads); + const kv_head_idx = head_idx.div(num_queries_per_kv); + const kv_token_stride = a.num_kv_heads.mul(a.kv_head_stride); + + const first_seq = b.local(.int, 0); + const last_seq = b.local(.int, a.num_seqs); + var search = b.openLoop(.{}); + { + search.body(first_seq.lt(last_seq)); + } + { + const mid = first_seq.add(last_seq).add(1).div(2); + var half = b.openIfElse(a.cu_seqlens_q.load(mid).div(BQ).add(mid).le(q_block_global_idx), .{}); + { + first_seq.assign(mid); + half.elseBranch(); + } + { + last_seq.assign(mid.sub(1)); + half.close(); + } + search.close(); + } + // The grid is an upper bound on the query blocks: the extra ones land past + // the last sequence and clamp to it. + const seq_idx = b.min(first_seq.rvalue(), a.num_seqs.sub(1)); + + const q_seq_start = a.cu_seqlens_q.load(seq_idx); + const cur_batch_query_len = a.cu_seqlens_q.load(seq_idx.add(1)).sub(q_seq_start); + const q_block_start = q_seq_start.div(BQ).add(seq_idx); + const q_block_local = q_block_global_idx.sub(q_block_start); + const q_pos_start = q_block_local.mul(BQ); + + b.returnIf(q_pos_start.ge(cur_batch_query_len)); + + const seq_len = a.context_lens.load(seq_idx).to(.int); + const context_len = seq_len.sub(cur_batch_query_len); + const valid_q = b.min(BQ, cur_batch_query_len.sub(q_pos_start)); + + const SMEM_PAD = @divExact(16, @as(i32, @intCast(T.size()))); + const LD = HEAD_SIZE + SMEM_PAD; + const Q_ELEMS = BQ * LD; + const KV_ELEMS = TILE_KV * LD; + + const Q_smem = a.shared_mem.to(T); + const K_smem = Q_smem.add(Q_ELEMS); + const V_smem = K_smem.add(KV_ELEMS); + + const log2e: f32 = std.math.log2e; + const scale_log2 = a.scale.mul(log2e); + + const q_base = a.q.add(q_seq_start.add(q_pos_start).mul(a.q_stride)).add(head_idx.mul(HEAD_SIZE)); + var load_q = b.openFor(thread_idx, BQ * HEAD_SIZE, NUM_THREADS); + { + const i = load_q.iv; + const r = i.div(HEAD_SIZE); + const d = i.rem(HEAD_SIZE); + var value = b.openIfElse(r.lt(valid_q), .{T}); + { + value.yieldThen(.{q_base.load(r.mul(a.q_stride).add(d))}); + } + { + value.yieldElse(.{b.constant(T, 0)}); + } + Q_smem.store(r.mul(LD).add(d), value.results[0]); + load_q.close(); + } + + b.threadgroupBarrier(.mem_threadgroup); + + const T2: Type = .vec(T.scalar, 2); + const q_reg = b.array(T2, @intCast(TD)); + var load_q_reg = b.openForOpts(0, TD, 1, unroll); + { + const d = load_q_reg.iv; + const tmp = b.simdgroupLoad(Q_smem.add(a.sg_idx.mul(8).mul(LD)).add(d.mul(8)), LD, .{}); + q_reg.store(d, tmp.threadElements()); + load_q_reg.close(); + } + + const fc = fragCoord(b, a.lane.to(.ushort)); + const fm = fc.y(); + const fn_ = fc.x(); + + const max_score = b.local(.float, masked_score); + const sum_score = b.local(.float, 0.0); + + const o_reg = b.array(.float2, @intCast(TD)); + var zero_o = b.openForOpts(0, TD, 1, unroll); + { + o_reg.store(zero_o.iv, b.constant(.float2, 0)); + zero_o.close(); + } + + const block_table = a.block_tables.add(seq_idx.mul(a.max_num_blocks_per_seq)); + const num_kv_tiles = seq_len.add(TILE_KV).sub(1).div(TILE_KV); + + var tiles = b.openFor(0, num_kv_tiles, 1); + { + const tile_idx = tiles.iv; + const tile_start = tile_idx.mul(TILE_KV); + + if (cfg.causal) { + var past = b.openIf(tile_start.gt(context_len.add(q_pos_start).add(valid_q).sub(1))); + { + tiles.breakLoop(); + past.close(); + } + } + + const VEC_BYTES: i32 = if (HEAD_SIZE >= 256) 16 else 8; + const VEC = @divExact(VEC_BYTES, @as(i32, @intCast(T.size()))); + const LoadUnit: Type = if (VEC_BYTES == 16) .uint4 else .uint2; + std.debug.assert(@rem(LD, VEC) == 0); + var rows = b.openForOpts(0, @divExact(TILE_KV, NUM_SG), 1, unroll); + { + const t = a.sg_idx.mul(@divExact(TILE_KV, NUM_SG)).add(rows.iv).to(.int); + const kv_pos = tile_start.add(t); + var present = b.openIfElse(kv_pos.lt(seq_len), .{}); + { + const pb = block_table.load(kv_pos.div(BLOCK_SIZE)).to(.long); + const off = pb.mul(a.kv_block_stride).add(kv_pos.rem(BLOCK_SIZE).mul(kv_token_stride)).add(kv_head_idx.mul(a.kv_head_stride)); + const k_ptr = a.k_cache.add(off); + const v_ptr = a.v_cache.add(off); + var loop = b.openForOpts(a.lane.mul(VEC).to(.int), HEAD_SIZE, NUM_SIMD_LANES * VEC, unroll); + { + const d = loop.iv; + const k = k_ptr.add(d).to(LoadUnit).load(0); + K_smem.add(t.mul(LD).add(d)).to(LoadUnit).store(0, k); + const v = v_ptr.add(d).to(LoadUnit).load(0); + V_smem.add(t.mul(LD).add(d)).to(LoadUnit).store(0, v); + loop.close(); + } + present.elseBranch(); + } + { + const zero = b.constant(LoadUnit, 0); + var loop = b.openForOpts(a.lane.mul(VEC).to(.int), HEAD_SIZE, NUM_SIMD_LANES * VEC, unroll); + { + const d = loop.iv; + K_smem.add(t.mul(LD).add(d)).to(LoadUnit).store(0, zero); + V_smem.add(t.mul(LD).add(d)).to(LoadUnit).store(0, zero); + loop.close(); + } + present.close(); + } + rows.close(); + } + + b.threadgroupBarrier(.mem_threadgroup); + + const s_reg = b.array(.float2, @intCast(TK)); + var zero_s = b.openForOpts(0, TK, 1, unroll); + { + s_reg.store(zero_s.iv, b.constant(.float2, 0)); + zero_s.close(); + } + + var qk_dims = b.openForOpts(0, TD, 1, unroll); + { + const d = qk_dims.iv; + const q_frag = b.matrix(T.scalar, q_reg.load(d)); + var qk_keys = b.openForOpts(0, TK, 1, unroll); + { + const k = qk_keys.iv; + const k_frag = b.simdgroupLoad(K_smem.add(k.mul(8).mul(LD)).add(d.mul(8)), LD, .{ .transpose = true }); + const s_frag = b.matrix(.float, s_reg.load(k)); + s_reg.store(k, b.simdgroupMultiplyAccumulate(q_frag, k_frag, s_frag).threadElements()); + qk_keys.close(); + } + qk_dims.close(); + } + + const q_abs_pos = context_len.add(q_pos_start).add(a.sg_idx.mul(8)).add(fm).to(.int); + const row_masked = a.sg_idx.mul(8).add(fm).ge(valid_q); + const min_q_abs_pos = context_len.add(q_pos_start); + var tile_no_mask = tile_start.add(TILE_KV).le(seq_len).bitAnd(a.softcapping.le(0.0)).bitAnd(a.sliding_window.lt(0)); + if (cfg.causal) tile_no_mask = tile_no_mask.bitAnd(tile_start.add(TILE_KV).sub(1).lt(min_q_abs_pos)); + + var masking = b.openIfElse(tile_no_mask.bitAnd(row_masked.eq(false)), .{}); + { + var scale = b.openForOpts(0, TK, 1, unroll); + { + const k = scale.iv; + setLane(s_reg, k, 0, s_reg.load(k).at(0).mul(scale_log2)); + setLane(s_reg, k, 1, s_reg.load(k).at(1).mul(scale_log2)); + scale.close(); + } + masking.elseBranch(); + } + { + var keys = b.openForOpts(0, TK, 1, unroll); + { + const k = keys.iv; + var pair = b.openForOpts(0, 2, 1, unroll); + { + const jj = pair.iv; + const s = b.local(.float, s_reg.load(k).at(jj).mul(scale_log2)); + var capped = b.openIf(a.softcapping.gt(0.0)); + { + const s_orig = s.div(log2e); + s.assign(a.softcapping.mul(b.precise().tanh(s_orig.div(a.softcapping))).mul(log2e)); + capped.close(); + } + const kv_pos = tile_start.add(k.mul(8)).add(fn_).add(jj); + var masked = row_masked.bitOr(kv_pos.ge(seq_len)); + if (cfg.causal) masked = masked.bitOr(kv_pos.gt(q_abs_pos)); + const before_window = a.sliding_window.ge(0).bitAnd(kv_pos.lt(q_abs_pos.add(1).sub(a.sliding_window))); + masked = masked.bitOr(before_window); + setLane(s_reg, k, jj, b.ternary(masked, masked_score, s)); + pair.close(); + } + keys.close(); + } + masking.close(); + } + + const local_max = b.local(.float, masked_score); + var max_keys = b.openForOpts(0, TK, 1, unroll); + { + const k = max_keys.iv; + local_max.assign(b.max(local_max.rvalue(), b.max(s_reg.load(k).at(0), s_reg.load(k).at(1)))); + max_keys.close(); + } + const row_max = fragRowReduce(b, .max, local_max); + + const prev_max = max_score.rvalue(); + const grown_max = b.max(prev_max, row_max); + const grew = grown_max.gt(prev_max); + const was_empty = prev_max.eq(masked_score); + const factor = b.ternary(grew, b.ternary(was_empty, 0.0, b.fast().exp2(prev_max.sub(grown_max))), 1.0); + const new_max = b.ternary(grew.eq(false).bitAnd(was_empty), 0.0, grown_max); + // A tile masking a whole row leaves the maximum masked: zero would + // pin it, and later scores far below zero would underflow. + max_score.assign(grown_max); + + const local_sum = b.local(.float, 0.0); + var p_keys = b.openForOpts(0, TK, 1, unroll); + { + const k = p_keys.iv; + var pair = b.openForOpts(0, 2, 1, unroll); + { + const jj = pair.iv; + const s = s_reg.load(k).at(jj); + const p = b.ternary(s.eq(masked_score), 0.0, b.fast().exp2(s.sub(new_max))); + setLane(s_reg, k, jj, p); + local_sum.assign(local_sum.add(p)); + pair.close(); + } + p_keys.close(); + } + const row_sum = fragRowReduce(b, .sum, local_sum); + + sum_score.assign(sum_score.mul(factor).add(row_sum)); + + var rescale_o = b.openForOpts(0, TD, 1, unroll); + { + o_reg.store(rescale_o.iv, o_reg.load(rescale_o.iv).mul(factor)); + rescale_o.close(); + } + + var pv_keys = b.openForOpts(0, TK, 1, unroll); + { + const k = pv_keys.iv; + const p_frag = b.matrix(.float, s_reg.load(k)); + var pv_dims = b.openForOpts(0, TD, 1, unroll); + { + const d = pv_dims.iv; + const v_frag = b.simdgroupLoad(V_smem.add(k.mul(8).mul(LD)).add(d.mul(8)), LD, .{}); + const o_frag = b.matrix(.float, o_reg.load(d)); + o_reg.store(d, b.simdgroupMultiplyAccumulate(p_frag, v_frag, o_frag).threadElements()); + pv_dims.close(); + } + pv_keys.close(); + } + + b.threadgroupBarrier(.mem_threadgroup); + tiles.close(); + } + + const inv_sum = b.lift(1.0).div(sum_score.add(@as(f32, 1e-6))); + var normalize = b.openForOpts(0, TD, 1, unroll); + { + o_reg.store(normalize.iv, o_reg.load(normalize.iv).mul(inv_sum)); + normalize.close(); + } + + const O_smem = Q_smem.to(.float); + var store_o = b.openForOpts(0, TD, 1, unroll); + { + const d = store_o.iv; + const o_frag = b.matrix(.float, o_reg.load(d)); + b.simdgroupStore(o_frag, O_smem.add(a.sg_idx.mul(8).mul(LD)).add(d.mul(8)), LD, .{}); + store_o.close(); + } + + b.threadgroupBarrier(.mem_threadgroup); + + const out_base = a.out.add(q_seq_start.add(q_pos_start).mul(a.q_stride)).add(head_idx.mul(HEAD_SIZE)); + const total = valid_q.mul(HEAD_SIZE); + var write_out = b.openFor(thread_idx, total, NUM_THREADS); + { + const i = write_out.iv; + const r = i.div(HEAD_SIZE); + const d = i.rem(HEAD_SIZE); + const value = O_smem.load(r.mul(LD).add(d)).to(T); + out_base.store(r.mul(a.q_stride).add(d), value); + write_out.close(); + } + } +}; + +test "the paged attention kernels emit verified modules" { + const allocator = std.testing.allocator; + for ([_]i32{ 32, 64, 128, 256, 512 }) |head_dim| { + for ([_][2]i32{ .{ 1, 1 }, .{ 4, 1 }, .{ 2, 8 }, .{ 4, 128 } }) |launch| { + allocator.free(try FaVecPaged.Kernel.emit(allocator, .{ .head_dim = head_dim, .block_size = 16, .position_stride = 2 * head_dim, .heads = launch[0], .splits = launch[1] })); + } + for ([_]i32{ 8, 128 }) |splits| { + allocator.free(try FaVecPagedReduce.Kernel.emit(allocator, .{ .splits = splits, .head_dim = head_dim })); + } + } + for ([_]Type{ .half, .bfloat }) |dtype| { + allocator.free(try ZeroPaddingRows.Kernel.emit(allocator, .{ .dtype = dtype })); + for ([_]i32{ 32, 64, 96, 128, 256, 512 }) |head_size| { + const tile = PagedAttentionTiled.tileConfig(head_size).?; + for ([_]bool{ true, false }) |causal| { + allocator.free(try PagedAttentionTiled.Kernel.emit(allocator, .{ .dtype = dtype, .head_size = head_size, .block_size = 16, .bq = tile[0], .tile_kv = tile[1], .num_threads = tile[2], .causal = causal })); + } + } + } +} diff --git a/zml/kernels/paged_attention/mosaic_tpu/BUILD.bazel b/zml/kernels/paged_attention/mosaic_tpu/BUILD.bazel new file mode 100644 index 00000000..6a0e4e87 --- /dev/null +++ b/zml/kernels/paged_attention/mosaic_tpu/BUILD.bazel @@ -0,0 +1,9 @@ +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "mosaic_tpu", + srcs = [ + "mosaic_tpu.zig", + "ragged_attention.zig", + ], +) diff --git a/zml/kernels/paged_attention/mosaic_tpu/README.md b/zml/kernels/paged_attention/mosaic_tpu/README.md new file mode 100644 index 00000000..6fff585c --- /dev/null +++ b/zml/kernels/paged_attention/mosaic_tpu/README.md @@ -0,0 +1,28 @@ +# Mosaic TPU paged attention + +This backend is automatically selected for TPU. See the +[parent README](../README.md) for the attention equation and packed sequence +metadata contract. + +[The wrapper](mosaic_tpu.zig) requires fused `KvCache.dense` storage with shape +`{ .page = N, .k_chunk = P, .hkv = 2 * Hkv, .hd = Dpad }`. The combined head +axis interleaves `K0, V0, K1, V1, ...`. `Dpad` must equal the logical query head +dimension rounded up to a multiple of 128, with unused cache features zeroed. +Queries are padded before execution and outputs trimmed afterward. The default +scale uses the logical head dimension. Accepted query/cache dtypes are `bf16`, +`f16`, and `f32`. + +Causal and non-causal standard masks and left sliding windows are supported. +Explicit key indices are rejected. `opts.sink` is currently ignored. Split +caches are rejected by ordinary paged attention, and the generic +`KvCache.update` method does not implement fused-cache writes. + +The wrapper partitions heads on the model mesh and replicates sequence metadata. +It counts sequences with nonempty query intervals and passes that count to the +ragged kernel, so active sequences must precede trailing inactive entries. +Each local head computation must have the full required key range and features. + +The shared split-cache comparison test does not exercise this backend. Wrapper +unit tests cover head-dimension alignment and configuration, including scale +and standard mask options; they do not establish runtime support for every +shape or dtype. diff --git a/zml/kernels/paged_attention/mosaic_tpu/mosaic_tpu.zig b/zml/kernels/paged_attention/mosaic_tpu/mosaic_tpu.zig new file mode 100644 index 00000000..84716379 --- /dev/null +++ b/zml/kernels/paged_attention/mosaic_tpu/mosaic_tpu.zig @@ -0,0 +1,275 @@ +const std = @import("std"); + +const mlir = @import("mlir"); +const ragged_paged = @import("platforms/tpu/ragged_paged"); +const stdx = @import("stdx"); + +const zml = @import("../../../zml.zig"); +const AttentionOptions = @import("../options.zig").AttentionOptions; +const ragged_attention = @import("ragged_attention.zig"); + +const log = std.log.scoped(.tpu_attention); + +test { + std.testing.refAllDecls(@This()); + std.testing.refAllDecls(ragged_attention); +} + +pub const mosaic_tpu = struct { + const PreparedInputs = struct { + q: zml.Tensor, + kv_pages: zml.Tensor, + seq_lens: zml.Tensor, + block_table: zml.Tensor, + query_start_len: zml.Tensor, + num_seqs: zml.Tensor, + }; + + pub fn alignedKernelHeadDim(head_dim: usize) usize { + stdx.debug.assert(head_dim > 0, "mosaic_tpu ragged paged attention head_dim must be non-zero", .{}); + const alignment: usize = 128; + return (std.math.divCeil(usize, head_dim, alignment) catch unreachable) * alignment; + } + + pub const Options = struct { + is_prefill: bool, + batch_size: usize, + max_num_pages: usize, + max_seqlen_k: usize, + max_token_count: usize, + num_heads: usize, + num_kv_heads: usize, + head_dim: usize, + pages_per_compute_block: usize = 1, + + pub fn isPrefill(self: Options) bool { + return self.is_prefill; + } + + pub fn maxNumPages(self: Options) usize { + return self.max_num_pages; + } + }; + + pub const Parameters = struct { + opts: Options, + block_table: zml.Tensor, + seq_lens: zml.Tensor, + query_start_len: zml.Tensor, + + pub fn init(opts: Options) Parameters { + return .{ + .opts = opts, + .block_table = zml.Tensor.init(.{ .b = opts.batch_size, .p = opts.max_num_pages }, .i32), + .seq_lens = zml.Tensor.init(.{ .b = opts.batch_size }, .i32), + .query_start_len = zml.Tensor.init(.{ .b = opts.batch_size + 1 }, .i32), + }; + } + + pub fn options(self: Parameters) Options { + return self.opts; + } + + pub fn allocationSize(self: Parameters) usize { + var allocation_size: usize = 0; + allocation_size += self.block_table.byteSize(); + allocation_size += self.seq_lens.byteSize(); + allocation_size += self.query_start_len.byteSize(); + return allocation_size; + } + }; + + fn raggedPagedKernelCall( + q: zml.Tensor, + kv_pages: zml.Tensor, + seq_lens: zml.Tensor, + block_table: zml.Tensor, + query_start_len: zml.Tensor, + num_seqs: zml.Tensor, + cfg: ragged_paged.Cfg, + ) zml.Tensor { + const mlir_ctx = zml.Compiler.current().mlir_ctx; + const out = ragged_attention.Kernel.call( + .{ + .kv_lens = seq_lens, + .page_indices = block_table, + .cu_q_lens = query_start_len, + .seq_buf_idx = .zeroes(.init(.{2}, .i32)), + .num_seqs = num_seqs, + .q = q, + .kv_pages = kv_pages, + }, + .{ .o = q.shape() }, + .{ + .cfg = cfg, + .extras = .{ + .vmem_limit_bytes = cfg.vmem_limit_bytes, + .additional_attributes = &.{ + .named(mlir_ctx, "kernel_name", .string(mlir_ctx, "ragged_paged_attention_kernel")), + .named(mlir_ctx, "mhlo.frontend_attributes", .dict(mlir_ctx, &.{ + .named(mlir_ctx, "kernel_metadata", .string(mlir_ctx, "{}")), + })), + }, + }, + }, + ); + return out.o; + } + + fn activeSequenceCount(query_start_len: zml.Tensor) zml.Tensor { + const start = query_start_len.slice(.b, .{ .end = query_start_len.dim(.b) - 1 }); + const end = query_start_len.slice(.b, .{ .start = 1 }); + const query_lens = end.sub(start); + return query_lens + .cmp(.GT, .zeroes(query_lens.shape())) + .convert(.i32) + .sum(.b) + .reshape(.{1}); + } + + fn cfgDtype(dtype: zml.DataType) @FieldType(ragged_paged.Cfg, "q_dtype") { + const cast = zml.kernel.mosaic_tpu.from(dtype); + return switch (cast) { + .bf16, .f16, .f32 => cast, + else => stdx.debug.panic("mosaic_tpu ragged paged attention expects q/kv dtype bf16/f16/f32, got {}", .{dtype}), + }; + } + + inline fn restoreQueryHeads(q_template: zml.Tensor, q_out: zml.Tensor) zml.Tensor { + const restored = q_out.splitAxis(.h, .{ .hkv = q_template.dim(.hkv), .hg = q_template.dim(.hg) }); + if (restored.dim(.hd) == q_template.dim(.hd)) return restored; + stdx.debug.assert( + restored.dim(.hd) > q_template.dim(.hd), + "mosaic_tpu ragged paged attention cannot restore output from head_dim {} to {}", + .{ restored.dim(.hd), q_template.dim(.hd) }, + ); + return restored.slice(.hd, .{ .end = q_template.dim(.hd) }); + } + + inline fn alignQueryHeadDimForKernel(q: zml.Tensor, target_head_dim: i64) zml.Tensor { + if (q.dim(.hd) == target_head_dim) return q; + stdx.debug.assert( + q.dim(.hd) < target_head_dim, + "mosaic_tpu ragged paged attention q head_dim {} exceeds kernel/cache head_dim {}", + .{ q.dim(.hd), target_head_dim }, + ); + return q.pad(0, .{ .hd = zml.Tensor.Pad{ .high = target_head_dim - q.dim(.hd) } }); + } + + inline fn prepareInputs(parameters: Parameters, q: zml.Tensor, kv_pages: zml.Tensor) PreparedInputs { + const q_merged = q.merge(.{ .h = .{ .hkv, .hg } }); + const logical_head_dim: usize = @intCast(q_merged.dim(.hd)); + const cache_head_dim: usize = @intCast(kv_pages.dim(.hd)); + stdx.debug.assert( + cache_head_dim == alignedKernelHeadDim(logical_head_dim), + "mosaic_tpu ragged paged attention expects KV cache head_dim {} to match aligned logical head_dim {}", + .{ cache_head_dim, logical_head_dim }, + ); + const q_ragged = alignQueryHeadDimForKernel(q_merged, kv_pages.dim(.hd)); + const query_start_len = parameters.query_start_len; + + return .{ + .q = q_ragged, + .kv_pages = kv_pages, + .seq_lens = parameters.seq_lens, + .block_table = parameters.block_table, + .query_start_len = query_start_len, + .num_seqs = activeSequenceCount(query_start_len), + }; + } + + fn buildCfg(prepared_inputs: PreparedInputs, parameters: Parameters, opts: AttentionOptions) ragged_paged.Cfg { + const q_shape = prepared_inputs.q.shape(); + const kv_pages_shape = prepared_inputs.kv_pages.shape(); + const block_table_shape = prepared_inputs.block_table.shape(); + + const q_token_count = if (q_shape.hasTag(.q) != null) q_shape.dim(.q) else q_shape.dim(.b); + const max_num_seqs = block_table_shape.dim(.b); + const num_q_heads = q_shape.dim(.h); + stdx.debug.assert( + q_shape.dim(.hd) == kv_pages_shape.dim(.hd), + "mosaic_tpu ragged paged attention expects q and KV cache head_dim to match, got {} and {}", + .{ q_shape.dim(.hd), kv_pages_shape.dim(.hd) }, + ); + stdx.debug.assert(@mod(kv_pages_shape.dim(.hkv), 2) == 0, "Expected fused KV pages .hkv dimension to be even, got {}", .{kv_pages_shape.dim(.hkv)}); + const num_kv_heads = std.math.divExact(i64, kv_pages_shape.dim(.hkv), 2) catch + stdx.debug.panic("Expected fused KV pages .hkv dimension to be even, got {}", .{kv_pages_shape.dim(.hkv)}); + const num_queries_per_block = @max(@as(i64, 1), @min(q_token_count, max_num_seqs)); + const logical_head_dim: f64 = @floatFromInt(parameters.opts.head_dim); + + const sliding_window: ?i64 = if (opts.mask.slidingWindow()) |window| @intCast(window) else null; + + return .{ + .num_q_tokens = q_token_count, + .num_q_heads = num_q_heads, + .num_kv_heads = num_kv_heads, + .head_dim = q_shape.dim(.hd), + .total_num_pages = kv_pages_shape.dim(.page), + .page_size = kv_pages_shape.dim(.k_chunk), + .max_num_seqs = max_num_seqs, + .pages_per_seq = @intCast(parameters.opts.max_num_pages), + .num_kv_pages_per_block = @intCast(parameters.opts.pages_per_compute_block), + .num_queries_per_block = num_queries_per_block, + .q_dtype = cfgDtype(q_shape.dtype()), + .kv_dtype = cfgDtype(kv_pages_shape.dtype()), + .sm_scale = opts.scale orelse @floatCast(1.0 / @sqrt(logical_head_dim)), + .is_causal = opts.mask.isCausal(), + .sliding_window = sliding_window, + }; + } + + pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, kv_cache: zml.Tensor, opts: AttentionOptions) zml.Tensor { + if (opts.mask == .indices) @panic("Sparse Attention is not supported on TPU"); + const prepared = prepareInputs(parameters, q, kv_cache); + const cfg = buildCfg(prepared, parameters, opts); + const q_out = raggedPagedKernelCall(prepared.q, prepared.kv_pages, prepared.seq_lens, prepared.block_table, prepared.query_start_len, prepared.num_seqs, cfg); + const restored = restoreQueryHeads(q, q_out); + stdx.debug.assert(restored.shape().eql(q.shape()), "mosaic_tpu ragged paged attention output shape mismatch, got {f}, expected {f}", .{ restored.shape(), q.shape() }); + return restored; + } +}; + +test "mosaic_tpu aligns kernel head dimensions to 128" { + try std.testing.expectEqual(@as(usize, 128), mosaic_tpu.alignedKernelHeadDim(64)); + try std.testing.expectEqual(@as(usize, 128), mosaic_tpu.alignedKernelHeadDim(128)); + try std.testing.expectEqual(@as(usize, 256), mosaic_tpu.alignedKernelHeadDim(180)); + try std.testing.expectEqual(@as(usize, 256), mosaic_tpu.alignedKernelHeadDim(256)); +} + +test "mosaic_tpu cfg uses kernel head dim and logical scale" { + const parameters = mosaic_tpu.Parameters.init(.{ + .is_prefill = false, + .batch_size = 1, + .max_num_pages = 2, + .max_seqlen_k = 32, + .max_token_count = 1, + .num_heads = 8, + .num_kv_heads = 2, + .head_dim = 64, + }); + const prepared_inputs: mosaic_tpu.PreparedInputs = .{ + .q = zml.Tensor.init(.{ .b = 1, .h = 8, .hd = 128 }, .bf16), + .kv_pages = zml.Tensor.init(.{ .page = 2, .k_chunk = 16, .hkv = 4, .hd = 128 }, .bf16), + .seq_lens = parameters.seq_lens, + .block_table = parameters.block_table, + .query_start_len = parameters.query_start_len, + .num_seqs = zml.Tensor.init(.{1}, .i32), + }; + + const cfg = mosaic_tpu.buildCfg(prepared_inputs, parameters, .{}); + try std.testing.expectEqual(@as(i64, 128), cfg.head_dim); + try std.testing.expect(cfg.is_causal); + try std.testing.expectApproxEqAbs( + @as(f32, 1.0 / @sqrt(@as(f32, 64.0))), + cfg.sm_scale, + 0.000001, + ); + + const non_causal_cfg = mosaic_tpu.buildCfg(prepared_inputs, parameters, .{ .mask = .{ .standard = .{ .causal = false } } }); + try std.testing.expect(!non_causal_cfg.is_causal); + try std.testing.expectEqual(@as(?i64, null), non_causal_cfg.sliding_window); + + const window_cfg = mosaic_tpu.buildCfg(prepared_inputs, parameters, .{ .mask = .{ .standard = .{ .sliding_window = 16 } } }); + try std.testing.expect(window_cfg.is_causal); + try std.testing.expectEqual(@as(?i64, 16), window_cfg.sliding_window); +} diff --git a/zml/kernels/paged_attention/mosaic_tpu/ragged_attention.zig b/zml/kernels/paged_attention/mosaic_tpu/ragged_attention.zig new file mode 100644 index 00000000..30536306 --- /dev/null +++ b/zml/kernels/paged_attention/mosaic_tpu/ragged_attention.zig @@ -0,0 +1,14 @@ +const ragged_paged = @import("platforms/tpu/ragged_paged"); + +const zml = @import("../../../zml.zig"); + +test { + _ = Kernel; +} + +pub const Kernel = zml.kernel.mosaic_tpu.Kernel(ragged_paged.Cfg, .{ + .name = "ragged_paged_attention_kernel", + .inputs = &.{ "kv_lens", "page_indices", "cu_q_lens", "seq_buf_idx", "num_seqs", "q", "kv_pages" }, + .outputs = &.{"o"}, + .run = ragged_paged.run, +}); diff --git a/zml/kernels/paged_attention/options.zig b/zml/kernels/paged_attention/options.zig new file mode 100644 index 00000000..35ad4049 --- /dev/null +++ b/zml/kernels/paged_attention/options.zig @@ -0,0 +1,36 @@ +const Tensor = @import("../../tensor.zig").Tensor; + +pub const AttentionMask = union(enum) { + standard: struct { + /// Restrict attention to keys at or before the query position. + causal: bool = true, + /// Optional left window size, including the current token. + /// Non-causal attention also includes every later valid key. + sliding_window: ?u32 = null, + }, + /// Attend to keys at specific positions. + indices: Tensor, + + /// Attend to every valid key in the sequence. + pub const none: AttentionMask = .{ .standard = .{ .causal = false } }; + + pub fn isCausal(mask: AttentionMask) bool { + return switch (mask) { + .standard => |options| options.causal, + .indices => false, + }; + } + + pub fn slidingWindow(mask: AttentionMask) ?u32 { + return switch (mask) { + .standard => |options| options.sliding_window, + .indices => null, + }; + } +}; + +pub const AttentionOptions = struct { + mask: AttentionMask = .{ .standard = .{} }, + scale: ?f32 = null, + sink: ?Tensor = null, +}; diff --git a/zml/kernels/paged_attention/paged_attention.zig b/zml/kernels/paged_attention/paged_attention.zig new file mode 100644 index 00000000..4a97a38e --- /dev/null +++ b/zml/kernels/paged_attention/paged_attention.zig @@ -0,0 +1,502 @@ +const std = @import("std"); + +const stdx = @import("stdx"); + +const zml = @import("../../zml.zig"); +pub const AttentionMask = @import("options.zig").AttentionMask; +pub const AttentionOptions = @import("options.zig").AttentionOptions; +pub const cute = @import("cute/cute.zig"); +const metal = @import("metal/metal.zig"); +const tpu = @import("mosaic_tpu/mosaic_tpu.zig"); +const stablehlo = @import("stablehlo/stablehlo.zig"); +pub const tcl = @import("tcl/tcl.zig"); +const triton = @import("triton/triton.zig"); + +const PagedAttention = @This(); + +test { + _ = tcl; + std.testing.refAllDecls(Backend); + std.testing.refAllDecls(Options); + std.testing.refAllDecls(Parameters); + std.testing.refAllDecls(KvCache); +} + +pub const Backend = enum { + cute, + triton, + mosaic_tpu, + metal, + stablehlo, + tcl, + + pub fn auto(platform: *const zml.Platform) Backend { + return switch (platform.target) { + .cuda => if (cute.isAvailable(platform)) .cute else .triton, + .rocm => .triton, + .oneapi => .triton, + .tpu => .mosaic_tpu, + .metal => .metal, + .cpu => .stablehlo, + .furiosa => .tcl, + .neuron => stdx.debug.panic("Paged attention is not supported on {s} yet", .{@tagName(platform.target)}), + }; + } + + pub fn isAvailable(backend: Backend, platform: *const zml.Platform) bool { + return switch (backend) { + .cute => cute.isAvailable(platform), + .stablehlo => true, + .tcl => platform.target == .furiosa, + .triton => switch (platform.target) { + .cuda, .rocm, .oneapi => true, + .cpu, .tpu, .neuron, .metal, .furiosa => false, + }, + .metal => platform.target == .metal, + .mosaic_tpu => platform.target == .tpu, + }; + } +}; + +pub const Options = union(Backend) { + cute: cute.Options, + triton: triton.paged.Options, + mosaic_tpu: tpu.mosaic_tpu.Options, + metal: metal.Options, + stablehlo: triton.paged.Options, + tcl: tcl.paged.Options, + + const Args = struct { + backend: Backend, + is_prefill: bool, + batch_size: u32, + batch_size_prefill: ?u32 = null, + batch_size_decode: ?u32 = null, + seq_len: u32, + max_num_pages: u32, + max_token_count: u32, + num_heads: u32, + num_kv_heads: u32, + head_dim: u32, + max_seqlen_q: u32, + }; + + pub fn fromBackend(args: Args) Options { + return switch (args.backend) { + .mosaic_tpu => .{ + .mosaic_tpu = .{ + .is_prefill = args.is_prefill, + .batch_size = args.batch_size, + .max_num_pages = args.max_num_pages, + .max_seqlen_k = args.seq_len, + .max_token_count = args.max_token_count, + .num_heads = args.num_heads, + .num_kv_heads = args.num_kv_heads, + .head_dim = args.head_dim, + }, + }, + inline .cute, .triton, .metal, .stablehlo, .tcl => |t| @unionInit(Options, @tagName(t), .{ + .batch_size = args.batch_size, + .max_num_pages = args.max_num_pages, + .max_seqlen_q = args.max_seqlen_q, + .is_prefill = args.is_prefill, + }), + }; + } + + pub fn isPrefill(self: Options) bool { + return switch (self) { + inline else => |v| v.isPrefill(), + }; + } + + pub fn maxNumPages(self: Options) usize { + return switch (self) { + inline else => |v| v.maxNumPages(), + }; + } +}; + +pub const Parameters = union(Backend) { + cute: cute.Parameters, + triton: triton.paged.Parameters, + mosaic_tpu: tpu.mosaic_tpu.Parameters, + metal: metal.Parameters, + stablehlo: triton.paged.Parameters, + tcl: tcl.paged.Parameters, + + pub fn init(options_: Options) Parameters { + return switch (options_) { + inline else => |opts, backend| @unionInit(Parameters, @tagName(backend), .init(opts)), + }; + } + + pub fn options(self: Parameters) Options { + return switch (self) { + inline else => |v, backend| @unionInit(Options, @tagName(backend), v.options()), + }; + } + + pub fn allocationSize(self: Parameters) usize { + return switch (self) { + inline else => |v| v.allocationSize(), + }; + } + + pub fn onMemory(self: Parameters, memory: zml.platform.Memory.Kind) Parameters { + zml.Tensor.onMemoryAll(&self, memory); + return self; + } + + pub fn toMemory(self: Parameters, memory: zml.platform.Memory.Kind) Parameters { + return zml.Tensor.toMemoryAll(self, memory); + } +}; + +pub const KvCache = union(enum) { + split: struct { + k: zml.Tensor, + v: zml.Tensor, + }, + // TODO: rename `dense` to `fused`. + dense: zml.Tensor, + + pub fn update( + self: KvCache, + new_k: zml.Tensor, + new_v: zml.Tensor, + slot_mapping: zml.Tensor, + chunk_size: u32, + backend: Backend, + ) KvCache { + const page_index, const offset = getPageAndOffsetFromSlotMapping(slot_mapping, chunk_size); + + const kv: KvCache = switch (self) { + .split => |split| switch (backend) { + .cute, .triton, .mosaic_tpu, .metal, .stablehlo, .tcl => .{ + .split = .{ + .k = split.k.scatterSlices( + .{ .page = page_index, .k_chunk = offset }, + new_k, + .{ .update_fn = zml.Tensor.ScatterOpts.override, .indices_are_unique = false, .indices_are_sorted = false }, + ).reuseBuffer(self.split.k), + .v = split.v.scatterSlices( + .{ .page = page_index, .k_chunk = offset }, + new_v, + .{ .update_fn = zml.Tensor.ScatterOpts.override, .indices_are_unique = false, .indices_are_sorted = false }, + ).reuseBuffer(self.split.v), + }, + }, + }, + .dense => @panic("TODO"), + }; + + return kv; + } + + pub fn getPageAndOffsetFromSlotMapping(slot_mapping: zml.Tensor, page_size: u32) [2]zml.Tensor { + const page_index = slot_mapping.divByConst(page_size); + const offset = slot_mapping.remainder(.scalar(page_size, slot_mapping.dtype())); + return .{ page_index, offset }; + } +}; + +pub const Partitioning = zml.ops.ManualPartitioning(Attention.forward); + +pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, kv_cache: KvCache, opts: AttentionOptions, mesh: *const zml.Mesh, partitioning: Partitioning) zml.Tensor { + _ = k; + _ = v; + const args: Attention = .{ .parameters = parameters, .q = q, .kv_cache = kv_cache, .opts = opts }; + return zml.ops.manualComputation(Attention.forward, args, q.shape(), mesh, partitioning); +} + +const Attention = struct { + parameters: Parameters, + q: zml.Tensor, + kv_cache: KvCache, + opts: AttentionOptions, + + fn forward(args: @This(), _: zml.Shape) zml.Tensor { + return switch (args.parameters) { + .cute => |p| switch (args.kv_cache) { + .split => |split| cute.pagedAttention(p, args.q, split.k, split.v, args.opts), + .dense => @panic("CuTe paged attention requires split KV pages"), + }, + .triton => |triton_parameters| switch (args.kv_cache) { + .split => |split| triton.paged.pagedAttention(triton_parameters, args.q, split.k, split.v, args.opts), + .dense => std.debug.panic("fused KV pages are only supported with the mosaic_tpu backend", .{}), + }, + .mosaic_tpu => |mosaic_tpu_parameters| switch (args.kv_cache) { + .split => std.debug.panic("split KV pages is not supported with the mosaic_tpu backend", .{}), + .dense => tpu.mosaic_tpu.pagedAttention(mosaic_tpu_parameters, args.q, args.kv_cache.dense, args.opts), + }, + .metal => |metal_parameters| switch (args.kv_cache) { + .split => |split| metal.pagedAttention(metal_parameters, args.q, split.k, split.v, args.opts), + .dense => std.debug.panic("fused KV pages are only supported with the mosaic_tpu backend", .{}), + }, + .stablehlo => |params| switch (args.kv_cache) { + .split => |split| stablehlo.pagedAttention(params, args.q, split.k, split.v, args.opts), + .dense => std.debug.panic("fused KV pages are only supported with the mosaic_tpu backend", .{}), + }, + .tcl => |params| switch (args.kv_cache) { + .split => |split| tcl.paged.pagedAttention(params, args.q, split.k, split.v, args.opts), + .dense => std.debug.panic("TCL paged attention takes separate K and V pages", .{}), + }, + }; + } +}; + +test "Backend.auto selects mosaic_tpu on TPU" { + const platform: zml.Platform = .{ + .arena = undefined, + .target = .tpu, + .pjrt_api = undefined, + .pjrt_client = undefined, + .state = zml.platform.State.init(.tpu), + .devices = &.{}, + .memories = &.{}, + .physical_mesh = undefined, + .replicated_mesh = undefined, + .meshes = .empty, + .io_impl = .threaded, + }; + + try std.testing.expectEqual(Backend.mosaic_tpu, Backend.auto(&platform)); +} + +test "Backend.auto selects triton on oneAPI" { + const platform: zml.Platform = .{ + .arena = undefined, + .target = .oneapi, + .pjrt_api = undefined, + .pjrt_client = undefined, + .state = zml.platform.State.init(.oneapi), + .devices = &.{}, + .memories = &.{}, + .physical_mesh = undefined, + .replicated_mesh = undefined, + .meshes = .empty, + .io_impl = .threaded, + }; + + try std.testing.expectEqual(Backend.triton, Backend.auto(&platform)); +} + +test pagedAttention { + const platform = zml.testing.env(); + const reference_backend: Backend = if (Backend.triton.isAvailable(platform)) + .triton + else if (Backend.metal.isAvailable(platform)) + .stablehlo + else + return error.SkipZigTest; + const io = std.testing.io; + const allocator = std.testing.allocator; + var arena_state: std.heap.ArenaAllocator = .init(allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + + const num_pages = 64; + const page_size = 16; + const max_num_pages = 16; + const prefill_token_count = 32; + const num_prefill = 1; + const num_decode = 6; + const active_batch_size = num_prefill + num_decode; + const batch_size = active_batch_size + 1; + const query_token_count = prefill_token_count + num_decode; + const dt: zml.DataType = .bf16; + const tp = zml.testing.tpMesh(); + const num_kv_heads = tp.numPartitionsForLogicalAxis(.model); + const partition = .{ .hkv = .model }; + const tensors: struct { q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, kv_cache: KvCache } = .{ + .q = zml.Tensor.fromShape(.init(.{ .b = query_token_count, .hkv = num_kv_heads, .hg = 4, .hd = 32 }, dt)).withPartitioning(tp, partition), + .k = zml.Tensor.fromShape(.init(.{ .b = query_token_count, .hkv = num_kv_heads, .hd = 32 }, dt)).withPartitioning(tp, partition), + .v = zml.Tensor.fromShape(.init(.{ .b = query_token_count, .hkv = num_kv_heads, .hd = 32 }, dt)).withPartitioning(tp, partition), + .kv_cache = .{ + .split = .{ + .k = zml.Tensor.fromShape(.init(.{ .page = num_pages, .k_chunk = page_size, .hkv = num_kv_heads, .hd = 32 }, dt)).withPartitioning(tp, partition), + .v = zml.Tensor.fromShape(.init(.{ .page = num_pages, .k_chunk = page_size, .hkv = num_kv_heads, .hd = 32 }, dt)).withPartitioning(tp, partition), + }, + }, + }; + + const meshes: []const *const zml.Mesh = &.{ platform.replicated_mesh, tp }; + const rng_q = try platform.compileFn(allocator, io, zml.Tensor.Rng.normal, .{ tensors.q.shape(), .{} }, .{ .meshes = meshes }); + defer rng_q.deinit(); + const rng_k = try platform.compileFn(allocator, io, zml.Tensor.Rng.normal, .{ tensors.k.shape(), .{} }, .{ .meshes = meshes }); + defer rng_k.deinit(); + const rng_kv_cache = try platform.compileFn(allocator, io, zml.Tensor.Rng.normal, .{ tensors.kv_cache.split.k.shape(), .{} }, .{ .meshes = meshes }); + defer rng_kv_cache.deinit(); + + const triton_options_args: Options.Args = .{ + .backend = .triton, + .is_prefill = true, + .batch_size = batch_size, + .batch_size_prefill = num_prefill, + .batch_size_decode = num_decode, + .seq_len = max_num_pages * page_size, + .max_num_pages = max_num_pages, + .max_token_count = 16 * page_size, + .num_heads = @intCast(tensors.q.dim(.hg) * tensors.q.dim(.hkv)), + .num_kv_heads = @intCast(tensors.q.dim(.hkv)), + .head_dim = @intCast(tensors.q.dim(.hd)), + .max_seqlen_q = 16 * 2, + }; + const triton_parameters: Parameters = .init(.fromBackend(triton_options_args)); + var q = try rng_q.eval(allocator, io, {}); + defer q.deinit(); + var new_k = try rng_k.eval(allocator, io, {}); + defer new_k.deinit(); + var new_v = try rng_k.eval(allocator, io, {}); + defer new_v.deinit(); + + var kv_cache_d: zml.Bufferized(KvCache) = .{ .split = undefined }; + kv_cache_d.split.k = try rng_kv_cache.eval(allocator, io, .{}); + defer kv_cache_d.split.k.deinit(); + kv_cache_d.split.v = try rng_kv_cache.eval(allocator, io, .{}); + defer kv_cache_d.split.v.deinit(); + + const block_table: [batch_size][max_num_pages]i32 = .{ + // prefilling pages 9-10 + .{ 0, 1, 9, 10, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 }, + // generation + .{ 0, 1, 2, 3, 4, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 }, + .{ 0, 1, 2, 3, 4, 6, 7, 8, -1, -1, -1, -1, -1, -1, -1, -1 }, + .{ 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63 }, + .{ 0, 1, 2, 3, 4, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 }, + .{ 0, 1, 2, 3, 4, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 }, + .{ 0, 1, 2, 3, 4, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 }, + .{ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 }, + }; + + const seq_lens: [batch_size]i32 = .{ + 4 * page_size, + 4 * page_size + 3, + 7 * page_size + 11, + 15 * page_size + 9, + 4 * page_size + 3, + 4 * page_size + 3, + 4 * page_size + 3, + 0, + }; + + const query_start_len: [batch_size + 1]i32 = .{ 0, 32, 33, 34, 35, 36, 37, 38, 38 }; + var triton_parameters_d: zml.Bufferized(Parameters) = .{ .triton = .{ + .block_table = try .fromBytes(io, platform, triton_parameters.triton.block_table.shape(), @ptrCast(&block_table)), + .seq_lens = try .fromBytes(io, platform, triton_parameters.triton.seq_lens.shape(), @ptrCast(&seq_lens)), + .query_start_len = try .fromBytes(io, platform, triton_parameters.triton.query_start_len.shape(), @ptrCast(&query_start_len)), + } }; + defer zml.Buffer.deinitAll(Parameters, &triton_parameters_d); + + const all_backends = std.enums.values(Backend); + // stablehlo ignores these options, mosaic_tpu requires a dense + // KV cache instead of the split cache used by this test. + const option_sensitive_backends = [_]Backend{ .triton, .cute, .metal, .tcl }; + const test_cases = [_]struct { + name: []const u8, + attention_options: AttentionOptions, + backends: []const Backend, + }{ + .{ + .name = "unbounded", + .attention_options = .{ .mask = .{ .standard = .{} } }, + .backends = all_backends, + }, + .{ + .name = "non_causal", + .attention_options = .{ .mask = .{ .standard = .{ .causal = false } } }, + .backends = &option_sensitive_backends, + }, + .{ + .name = "sliding_window", + .attention_options = .{ .mask = .{ .standard = .{ .sliding_window = page_size } } }, + .backends = &option_sensitive_backends, + }, + }; + + for (test_cases) |test_case| { + var results_per_backend: std.enums.EnumArray(Backend, ?zml.Slice) = .initFill(null); + defer { + for (std.enums.values(Backend)) |backend| { + if (results_per_backend.get(backend)) |slice| slice.free(allocator); + } + } + + for (test_case.backends) |backend| { + if (!backend.isAvailable(platform)) { + std.log.warn("paged_attention backend {t} not available", .{backend}); + continue; + } + std.log.warn("Testing paged_attention {s} with backend {t}", .{ test_case.name, backend }); + + var backend_options_args = triton_options_args; + backend_options_args.backend = backend; + const parameters: Parameters = .init(.fromBackend(backend_options_args)); + + const head_spec: zml.Sharding.Partitioning.Logical = .parse(.{ .hkv = .model }); + const partitioning: Partitioning = .{ + .in_specs = .{ + .parameters = try zml.ops.partitionSpecsAlloc(arena, parameters, .replicated), + .q = head_spec, + .kv_cache = .{ .split = .{ .k = head_spec, .v = head_spec } }, + .opts = .{ .mask = .{ .standard = {} }, .sink = null }, + }, + .out_specs = head_spec, + }; + + const exe = try platform.compileFn( + allocator, + io, + pagedAttention, + .{ parameters, tensors.q, tensors.k, tensors.v, tensors.kv_cache, test_case.attention_options, tp, partitioning }, + .{ .program_name = try std.fmt.allocPrint(arena, "paged_attention_{s}_{t}", .{ test_case.name, backend }), .meshes = meshes }, + ); + defer exe.deinit(); + + const parameters_d: zml.Bufferized(Parameters) = switch (parameters) { + .triton => triton_parameters_d, + inline .cute, .metal, .mosaic_tpu, .stablehlo, .tcl => |_, t| @unionInit(zml.Bufferized(Parameters), @tagName(t), .{ + .block_table = triton_parameters_d.triton.block_table, + .seq_lens = triton_parameters_d.triton.seq_lens, + .query_start_len = triton_parameters_d.triton.query_start_len, + }), + }; + + var output_d = try exe.eval(allocator, io, .{ parameters_d, q, new_k, new_v, kv_cache_d, .{ .mask = .{ .standard = {} }, .sink = null } }); + defer output_d.deinit(); + results_per_backend.set(backend, try output_d.toSliceAlloc(allocator, io)); + } + + const reference = results_per_backend.get(reference_backend) orelse { + std.log.warn("test pagedAttention {s} has no {t} reference", .{ test_case.name, reference_backend }); + continue; + }; + + var num_failed: u32 = 0; + for (std.enums.values(Backend)) |backend| { + if (backend == reference_backend) continue; + const output_h = results_per_backend.get(backend) orelse continue; + + const tolerance: zml.testing.CompareOpts = .{ + .absolute_tolerance = 1e-2, + .relative_tolerance = 1e-2, + .epsilon_relative = 1e-6, + }; + + zml.testing.expectClose(io, reference, output_h, tolerance) catch |err| switch (err) { + error.TestUnexpectedResult => { + num_failed += 1; + std.log.err("test pagedAttention {s} failed on backend {t} against reference {t}", .{ + test_case.name, + backend, + reference_backend, + }); + }, + else => |e| return e, + }; + } + + if (num_failed > 0) return error.TestUnexpectedResult; + } +} diff --git a/zml/kernels/paged_attention/stablehlo/BUILD.bazel b/zml/kernels/paged_attention/stablehlo/BUILD.bazel new file mode 100644 index 00000000..f60464f4 --- /dev/null +++ b/zml/kernels/paged_attention/stablehlo/BUILD.bazel @@ -0,0 +1,6 @@ +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "stablehlo", + srcs = ["stablehlo.zig"], +) diff --git a/zml/kernels/paged_attention/stablehlo/README.md b/zml/kernels/paged_attention/stablehlo/README.md new file mode 100644 index 00000000..34144345 --- /dev/null +++ b/zml/kernels/paged_attention/stablehlo/README.md @@ -0,0 +1,30 @@ +# StableHLO paged attention + +This fallback is automatically selected for CPU and Furiosa. See the +[parent README](../README.md) for the general attention equation, packed +sequence metadata, and page addressing contract. + +[The implementation](stablehlo.zig) iterates over sequences, query chunks, and +KV pages using StableHLO control flow. It merges partial softmax states with +`f32` maxima, exponential sums, and weighted-value accumulators, then converts +the final output to the query dtype. Query-key dot products use the input dtype. + +## Current limitations + +- Only split KV caches work. Although public dispatch forwards `.dense`, this + implementation accesses `.split`. Latent caches are rejected by dispatch. +- Attention is always causal: `causal = false` is currently ignored. Sliding + windows and explicit key indices are rejected. `opts.sink` is ignored; custom + scale is supported. +- Multi-query chunks are processed a full page at a time, and the packed output + cursor advances by the page size. Use page-multiple query chunks; arbitrary + partial query chunks are not covered by this implementation. Single-query + decode uses a separate one-row path. +- The sequence loop stops at the first empty query interval. Active sequences + must precede trailing inactive entries. + +The shared tests in [paged_attention.zig](../paged_attention.zig) use this path +as the causal comparison reference with `bf16`, page size 16, mixed +prefill/decode, shared prefix pages, partial final KV pages, and trailing +inactive sequences. Those cases do not establish support for every shape or +dtype, or conformance for the unsupported options above. diff --git a/zml/kernels/paged_attention/stablehlo/stablehlo.zig b/zml/kernels/paged_attention/stablehlo/stablehlo.zig new file mode 100644 index 00000000..55ee01d5 --- /dev/null +++ b/zml/kernels/paged_attention/stablehlo/stablehlo.zig @@ -0,0 +1,262 @@ +const zml = @import("../../../zml.zig"); +const AttentionOptions = @import("../options.zig").AttentionOptions; +const triton = @import("../triton/triton.zig"); + +pub fn pagedAttention( + parameters: triton.paged.Parameters, + q: zml.Tensor, + k: zml.Tensor, + v: zml.Tensor, + opts: AttentionOptions, +) zml.Tensor { + switch (opts.mask) { + .standard => |options| if (options.sliding_window != null) @panic("Sliding window attention mask is not supported on StableHLO"), + .indices => @panic("Sparse attention mask is not supported on StableHLO"), + } + + const page_size = k.dim(.k_chunk); + const padded_q = q.pad(0, .{ .b = zml.Tensor.Pad{ .high = page_size } }); + + const final_state = zml.ops.@"while"( + AttentionLoop, + .{ .q = padded_q, .k = k, .v = v, .parameters = parameters, .opts = opts }, + .{ + // i32 to match triton conventions + .slot_id = .scalar(0, .i32), + .seq_id = .scalar(0, .i32), + .q_page_idx = .scalar(0, .i32), + .k_page_idx = .scalar(0, .i32), + // Note: we do all partial softmax in f32 + .partial_softmax_prefill = .zeroes(q.shape().setDim(.b, page_size).withDtype(.f32)), + .partial_softmax_decode = .zeroes(q.shape().setDim(.b, 1).withDtype(.f32)), + .out = .zeroes(padded_q.shape()), + }, + ); + + return final_state.out.slice(.b, .{ .end = q.dim(.b) }); +} + +const AttentionLoop = struct { + q: zml.Tensor, + k: zml.Tensor, + v: zml.Tensor, + parameters: triton.paged.Parameters, + opts: AttentionOptions, + + const While = @This(); + + pub const State = struct { + slot_id: zml.Tensor, + seq_id: zml.Tensor, + q_page_idx: zml.Tensor, + k_page_idx: zml.Tensor, + partial_softmax_prefill: PartialSoftmax, + partial_softmax_decode: PartialSoftmax, + out: zml.Tensor, + }; + + pub fn cond(self: While, state: State) zml.Tensor { + const has_more_sequences = state.seq_id.cmp(.LT, .scalar(self.parameters.seq_lens.count(), .i32)); + const query_start = self.parameters.query_start_len.slice(0, .dynSingle(state.seq_id)); + const query_end = self.parameters.query_start_len.slice(0, .dynSingle(state.seq_id.addConstant(1))); + const has_queries = query_start.cmp(.LT, query_end).asScalar(); + return has_more_sequences.logical(.AND, has_queries); + } + + pub fn body(self: While, state_: State) State { + const query_start = self.parameters.query_start_len.slice(0, .dynSingle(state_.seq_id)); + const query_end = self.parameters.query_start_len.slice(0, .dynSingle(state_.seq_id.addConstant(1))); + const seq_num_queries_: zml.Tensor = .asScalar(.sub(query_end, query_start)); + const page_size_: u32 = @intCast(self.k.dim(.k_chunk)); + + return zml.ops.@"if"( + struct { + const IfPrefill = @This(); + while_body: While, + state: State, + seq_num_queries: zml.Tensor, + page_size: u32, + + /// Prefill version: num_slots == page_size, we multiply a full page of queries with a page of keys. + pub fn onTrue(if_ctx: IfPrefill) State { + const while_body = if_ctx.while_body; + const state = if_ctx.state; + const page_size = if_ctx.page_size; + const num_slots: u32 = page_size; + + // seq_lens includes the current queries; subtracting their count gives the context length. + const seq_len = while_body.parameters.seq_lens.slice(0, .dynSingle(state.seq_id)); + const q_offset = seq_len.sub(if_ctx.seq_num_queries).add(state.q_page_idx.scale(page_size)); + const next_k_offset = state.k_page_idx.addConstant(1).scale(page_size); + + // The current K page is processed in this iteration. Continue if the next page contains + // any key visible to the current query chunk, whose exclusive end is q_offset + num_slots. + const query_has_more_keys: zml.Tensor = .cmp(next_k_offset, .LT, q_offset.addConstant(num_slots)); + const sequence_has_more_queries: zml.Tensor = .cmp(state.q_page_idx.addConstant(1).scale(num_slots), .LT, if_ctx.seq_num_queries); + + const partial_softmax = while_body.attentionOnePage(state, q_offset, num_slots, page_size); + return updateState(state, num_slots, query_has_more_keys, sequence_has_more_queries, partial_softmax, while_body.nextQueryStart(state)); + } + + /// Decode version: num_slots == 1, we multiply one query with a page of keys. + pub fn onFalse(if_ctx: IfPrefill) State { + const while_body = if_ctx.while_body; + const state = if_ctx.state; + const page_size = if_ctx.page_size; + const num_slots: u32 = 1; + + // A decode query is the final token in seq_len, so its zero-based offset is seq_len - 1. + const seq_len = while_body.parameters.seq_lens.slice(0, .dynSingle(state.seq_id)).asScalar(); + const q_offset = seq_len.sub(if_ctx.seq_num_queries); + const next_k_offset = state.k_page_idx.addConstant(1).scale(page_size); + + const query_has_more_keys: zml.Tensor = .cmp(next_k_offset, .LT, q_offset.addConstant(num_slots)); + const sequence_has_more_queries: zml.Tensor = .scalar(false, .bool); + + const partial_softmax = while_body.attentionOnePage(state, q_offset, num_slots, page_size); + return updateState(state, num_slots, query_has_more_keys, sequence_has_more_queries, partial_softmax, while_body.nextQueryStart(state)); + } + }, + .{ + .while_body = self, + .state = state_, + .seq_num_queries = seq_num_queries_, + .page_size = page_size_, + }, + .cmp(seq_num_queries_, .GT, .scalar(1, .i32)), + ); + } + + pub fn attentionOnePage(self: While, state: State, q_offset: zml.Tensor, q_chunk: u32, page_size: u32) PartialSoftmax { + const k_offset = state.k_page_idx.scale(page_size); + const active_q = self.q.slice(self.q.axis(.b), .dyn(state.slot_id, q_chunk)); + + const active_k_page_id = self.parameters.block_table.slices( + .{ .b, .p }, + &.{ .dynSingle(state.seq_id), .dynSingle(state.k_page_idx) }, + ); + + const page_axis = self.k.axis(.page); + const active_k = self.k.slice(page_axis, .dynSingle(active_k_page_id)); + const active_v = self.v.slice(page_axis, .dynSingle(active_k_page_id)); + + const dtype = self.q.dtype(); + const attn_mask: zml.Tensor = attn_mask: { + const mask_shape: zml.Shape = .init(.{ .b = q_chunk, .k_chunk = page_size }, dtype); + + const q_idx = zml.Tensor.iota(mask_shape, .b).add(q_offset); + const k_idx = zml.Tensor.iota(mask_shape, .k_chunk).add(k_offset); + + const mask = zml.Tensor.cmp(k_idx, .LE, q_idx); + const minus_inf = zml.Tensor.constant(dtype.minValue()); + break :attn_mask .select(mask, .scalar(0, dtype), minus_inf); + }; + + const scale: f32 = self.opts.scale orelse @floatCast(1.0 / @sqrt(@as(f64, @floatFromInt(active_q.dim(.hd))))); + var attn_weights = active_q.dot(active_k, .hd).scale(scale); + attn_weights = attn_weights.add(attn_mask.broad(attn_weights.shape())); + + const og_softmax = if (q_chunk == 1) state.partial_softmax_decode else state.partial_softmax_prefill; + // Note: we compute q * k using q and k dtype, but the partial softmax is in f32. + const precise_dt = og_softmax.values.dtype(); + var partial = partialSoftmax(attn_weights.convert(precise_dt), .k_chunk); + partial.values = partial.values.dot(active_v.convert(precise_dt), .k_chunk).transpose(og_softmax.values.shape()); + partial.max_value = partial.max_value.transpose(og_softmax.max_value.shape()); + partial.exp_sum = partial.exp_sum.transpose(og_softmax.exp_sum.shape()); + + return og_softmax.merge(partial); + } + + fn nextQueryStart(self: While, state: State) zml.Tensor { + return self.parameters.query_start_len.slice(0, .dynSingle(state.seq_id.addConstant(1))).asScalar(); + } + + pub fn updateState(state: State, num_slots: u32, query_has_more_keys: zml.Tensor, sequence_has_more_queries: zml.Tensor, softmax: PartialSoftmax, next_query_start: zml.Tensor) State { + return zml.ops.if2( + query_has_more_keys, + // If the query has more pages: update partial softmax with attention on current page, delay updating output. + State{ + .slot_id = state.slot_id, + .seq_id = state.seq_id, + .q_page_idx = state.q_page_idx, + .k_page_idx = state.k_page_idx.addConstant(1), + .partial_softmax_prefill = if (num_slots == 1) state.partial_softmax_prefill else softmax, + .partial_softmax_decode = if (num_slots == 1) softmax else state.partial_softmax_decode, + .out = state.out, + }, + // Query has no more pages, we commit softmax to output and increment slot + State{ + .k_page_idx = .scalar(0, .i32), + .partial_softmax_prefill = if (num_slots == 1) state.partial_softmax_prefill else .reset(softmax), + .partial_softmax_decode = if (num_slots == 1) .reset(softmax) else state.partial_softmax_decode, + .out = state.out.dynamicUpdateSlice(.{ .b = state.slot_id }, softmax.finalize().convert(state.out.dtype())), + .slot_id = .select(sequence_has_more_queries, state.slot_id.addConstant(num_slots), next_query_start), + // If the sequence has more query chunks: increment q_page_idx, otherwise start over at 0. + .q_page_idx = .select(sequence_has_more_queries, state.q_page_idx.addConstant(1), .scalar(0, .i32)), + // If the sequence has more query chunks: keep seq_id, otherwise increment seq_id. + .seq_id = .select(sequence_has_more_queries, state.seq_id, state.seq_id.addConstant(1)), + }, + ); + } +}; + +pub const PartialSoftmax = struct { + values: zml.Tensor, + exp_sum: zml.Tensor, + max_value: zml.Tensor, + + pub fn zeroes(q_shape: zml.Shape) PartialSoftmax { + return .{ + .values = .zeroes(q_shape), + .exp_sum = .zeroes(q_shape.drop(.hd)), + .max_value = .broad(.constant(q_shape.dtype().minValue()), q_shape.drop(.hd)), + }; + } + + pub fn reset(self: PartialSoftmax) PartialSoftmax { + return .zeroes(self.values.shape()); + } + + pub fn merge(self: PartialSoftmax, other: PartialSoftmax) PartialSoftmax { + // Rescale self and other using the new global_max. + const global_max = self.max_value.maximum(other.max_value.convert(self.max_value.dtype())); + const new_self = self.rescale(global_max); + const new_other = other.rescale(global_max); + + // Now that self and other are using the same scale, we can just add them: + return .{ + .max_value = global_max, + .values = new_self.values.add(new_other.values), + .exp_sum = new_self.exp_sum.add(new_other.exp_sum), + }; + } + + /// Update max_value and rescale attn and exp_sum accordingly. + pub fn rescale(self: PartialSoftmax, max_value: zml.Tensor) PartialSoftmax { + const max_diff_exp: zml.Tensor = .exp(self.max_value.sub(max_value)); + const sum_dtype = self.exp_sum.dtype(); + return .{ + .max_value = max_value, + .values = self.values.mul(max_diff_exp.broad(self.values.shape())), + .exp_sum = self.exp_sum.mul(max_diff_exp.convert(sum_dtype)), + }; + } + + /// Divides the intermediary results by the exp_sum to get the proper attention values. + pub fn finalize(self: PartialSoftmax) zml.Tensor { + return self.values.div(self.exp_sum.broad(self.values.shape()).convert(self.values.dtype())); + } +}; + +/// Compute softmax over a chunk. +/// Returns intermediary results to allow aggregating later. +pub fn partialSoftmax(self: zml.Tensor, axis: anytype) PartialSoftmax { + const a = self.axis(axis); + const max_val = self.max(a).squeeze(a).maximum(.scalar(-1e16, self.dtype())); + const out = self.sub(max_val.broad(self.shape())).exp(); + return .{ + .values = out, + .exp_sum = out.convert(.f32).sum(a).squeeze(a), + .max_value = max_val, + }; +} diff --git a/zml/kernels/paged_attention/tcl/BUILD.bazel b/zml/kernels/paged_attention/tcl/BUILD.bazel new file mode 100644 index 00000000..4057eea3 --- /dev/null +++ b/zml/kernels/paged_attention/tcl/BUILD.bazel @@ -0,0 +1,6 @@ +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "tcl", + srcs = ["tcl.zig"], +) diff --git a/zml/kernels/paged_attention/tcl/tcl.zig b/zml/kernels/paged_attention/tcl/tcl.zig new file mode 100644 index 00000000..d04b684e --- /dev/null +++ b/zml/kernels/paged_attention/tcl/tcl.zig @@ -0,0 +1,1781 @@ +//! Furiosa RNGD paged attention in TCL: f32 softmax and accumulation, BF16 contractions. +//! `defaultPlan` picks fusible BF16 decode kernels, else one-shot attention or a device loop over key chunks. +//! `Repro:` files are in the reproducer repository, platform/furiosa. +const std = @import("std"); + +const builder = @import("dsl/tcl/builder"); +const stdx = @import("stdx"); + +const zml = @import("../../../zml.zig"); +const Tensor = zml.Tensor; +const tcl = zml.kernel.tcl; +const AttentionOptions = @import("../options.zig").AttentionOptions; +const triton = @import("../triton/triton.zig"); + +/// WORKAROUND (TCC 2026.5.0b0): flash decode with 8192-key chunks fails to +/// lower, so decode chunks hold 4096 keys. Repro: repro_flash_decode_8k_chunks.py. +/// Remove once 8192-key chunks compile. +const chunk_keys = 4096; +const flash_chunk_pairs = 512 * 2048; +const prefill_chunk_keys = 2048; +const prefill_block = 32; + +/// WORKAROUND (TCC 2026.5.0b0): block counts that are not a power of two slow TCC's layout +/// search or panic its allocator. Repro: repro_flash_allocator_block_group_panic.py. +/// Remove once any count compiles. +fn prefillBlocks(tokens: i64, sequences: i64, block: i64) i64 { + const blocks = @divTrunc(tokens + sequences * (block - 1), block); + return @intCast(std.math.ceilPowerOfTwo(u64, @intCast(blocks)) catch unreachable); +} + +fn prefillBlock(tokens: i64, sequences: i64) i64 { + const short = prefill_block / 2; + return if (short * prefillBlocks(tokens, sequences, short) < prefill_block * prefillBlocks(tokens, sequences, prefill_block)) short else prefill_block; +} + +/// A straight-line head over each token's first `head` keys, then a device loop over +/// `chunk`-key chunks (none when zero), with tokens in groups of `group`. +pub const Plan = struct { + head: i64, + chunk: i64, + group: i64, +}; + +pub var calibration_plan: ?Plan = null; + +pub const paged = struct { + pub const Options = triton.paged.Options; + pub const Parameters = triton.paged.Parameters; + + /// `q` is `{.b = tokens, .hkv, .hg, .hd}`, the caches `{.page, .k_chunk, .hkv, .hd}`. + /// Sinks are one logit per query head. + pub fn pagedAttention(parameters: Parameters, q: Tensor, k_cache: Tensor, v_cache: Tensor, opts: AttentionOptions) Tensor { + if (opts.mask == .indices) @panic("Sparse attention masks are not supported by TCL paged attention"); + stdx.debug.assert(q.rank() == 4 and k_cache.rank() == 4 and v_cache.rank() == 4 and k_cache.shape().eqlDims(v_cache.shape()), "TCL attention requires matching rank-four split K/V caches", .{}); + stdx.debug.assert(q.dtype().isFloat() and k_cache.dtype().isFloat() and v_cache.dtype().isFloat(), "TCL attention requires floating-point queries and caches", .{}); + stdx.debug.assert(q.dim(.hkv) == k_cache.dim(.hkv) and q.dim(.hd) == k_cache.dim(.hd) and q.dim(.hg) > 0, "TCL attention query and cache head dimensions must match", .{}); + stdx.debug.assert(parameters.block_table.dtype() == .i32 and parameters.seq_lens.dtype() == .i32 and parameters.query_start_len.dtype() == .i32, "TCL attention metadata must be I32", .{}); + stdx.debug.assert(parameters.block_table.dim(.b) == parameters.seq_lens.dim(.b) and parameters.query_start_len.dim(.b) == parameters.seq_lens.dim(.b) + 1, "TCL attention metadata batch dimensions must match", .{}); + stdx.debug.assert(parameters.seq_lens.dim(.b) > 0 and parameters.block_table.dim(.p) > 0 and k_cache.dim(.page) > 0 and k_cache.dim(.k_chunk) > 0, "TCL attention requires nonempty sequence and cache capacities", .{}); + stdx.debug.assert(k_cache.dim(.page) * k_cache.dim(.k_chunk) <= std.math.maxInt(i32) and q.dim(.b) <= std.math.maxInt(i32), "TCL attention indices must fit in I32", .{}); + const compiler = zml.Compiler.current(); + stdx.debug.assert(compiler.platform.target == .furiosa, "TCL attention requires Furiosa chips", .{}); + const attributes = compiler.platform.devices[0].pjrt_device.attributes(compiler.platform.pjrt_api) catch + std.debug.panic("TCL attention could not verify the Furiosa device's PE range", .{}); + defer attributes.deinit(); + const first_pe = attributes.get("pe_start"); + const last_pe = attributes.get("pe_end"); + stdx.debug.assert(first_pe != null and first_pe.? == .int64 and first_pe.?.int64 == 0 and + last_pe != null and last_pe.? == .int64 and last_pe.?.int64 == 7, "TCL attention requires a complete 8-PE RNGD chip", .{}); + return attend(parameters, q, k_cache, v_cache, opts); + } + + fn attend(parameters: Parameters, q: Tensor, k_cache: Tensor, v_cache: Tensor, opts: AttentionOptions) Tensor { + if (q.dim(.b) == 0) return Tensor.zeroes(q.shape()); + const page = k_cache.dim(.k_chunk); + // WORKAROUND (TCC 2026.5.0b0): no DPE tactic for 32-key attention, so pad to + // at least 64 keys with invalid pages. Repro: repro_decode_32_keys_no_tactic.py. + // Remove once 16- to 48-key decode compiles. + const keys = @max(parameters.block_table.dim(.p), std.math.divCeil(i64, 64, page) catch unreachable) * page; + // WORKAROUND (TCC 2026.5.0b0): TCC fails to allocate a 4080-key chunk (85 + // pages of 48), so chunks hold whole pages and a multiple of 64 keys. Repro: + // repro_flash_allocator_block_group_panic.py. Remove once such chunks compile. + const key_alignment = @divExact(page, @as(i64, @intCast(std.math.gcd(@as(u64, @intCast(page)), 64)))) * 64; + const alignment = if (key_alignment <= chunk_keys) key_alignment else page; + const decode = !parameters.options_.is_prefill and q.dim(.b) == parameters.seq_lens.dim(.b); + const block = if (decode) 1 else prefillBlock(q.dim(.b), parameters.seq_lens.dim(.b)); + const block_count = if (decode) q.dim(.b) else prefillBlocks(q.dim(.b), parameters.seq_lens.dim(.b), block); + const rows = block_count * block; + // A flash chunk's query-key pairs are bounded: the largest chunks do not finish compiling. + const chunk_limit = @max(1, @divTrunc(@min(@as(i64, if (decode) chunk_keys else prefill_chunk_keys), @divTrunc(flash_chunk_pairs, rows)), alignment)) * alignment; + const chunk = @min(keys, chunk_limit); + var cfg: Cfg = .{ + .tokens = q.dim(.b), + .sequences = parameters.seq_lens.dim(.b), + .b = block_count, + .i = block, + .a = (std.math.divCeil(i64, keys, chunk) catch unreachable) * chunk, + .chunk = chunk, + .hkv = q.dim(.hkv), + .g = q.dim(.hg), + .d = q.dim(.hd), + .pages = k_cache.dim(.page), + .pg = page, + .scale = opts.scale orelse @floatCast(1.0 / @sqrt(@as(f64, @floatFromInt(q.dim(.hd))))), + .decode = decode, + .use_sink = opts.sink != null, + .is_causal = opts.mask.isCausal(), + .window = @intCast(opts.mask.slidingWindow() orelse 0), + }; + const plain_decode = decode and !cfg.use_sink and cfg.window == 0; + // WORKAROUND (TCC 2026.5.0b0): an in-unit cache conversion and several tokens + // over a non-power-of-two page count fail to compile, so straight decode reads + // BF16 caches and pads the table. Repro: repro_sparse_propagation_transmute.py, + // repro_metadata_page_count_cover_valid_range.py. Remove once both compile. + const straight = decode and k_cache.dtype() == .bf16 and v_cache.dtype() == .bf16; + const straight_keys = if (straight and cfg.tokens > 1) @as(i64, @intCast(std.math.ceilPowerOfTwo(u64, @intCast(@divExact(keys, page))) catch unreachable)) * page else keys; + const plan = calibration_plan orelse defaultPlan(cfg, straight_keys, parameters.block_table.dim(.p), plain_decode and straight, straight, chunk); + const sink = if (opts.sink) |s| s.convert(.f32).reshape(.{ .hkv = cfg.hkv, .hg = cfg.g }) else Tensor.zeroes(.init(.{ .hkv = cfg.hkv, .hg = cfg.g }, .f32)); + if (plan.head > 0 and plan.chunk > 0) { + stdx.debug.assert(plain_decode and cfg.tokens == 1, "TCL hybrid attention takes one plain decode token", .{}); + return hybridDecodeAttention(parameters, q, k_cache, v_cache, cfg.scale, cfg.is_causal, plan).convert(q.dtype()); + } + if (plan.chunk == 0 and straight) { + return decodeAttention(parameters, q, k_cache, v_cache, sink, .{ + .t = cfg.tokens, + .a = straight_keys, + .group = plan.group, + .hkv = cfg.hkv, + .g = cfg.g, + .d = cfg.d, + .pages = cfg.pages, + .pg = page, + .scale = cfg.scale, + .is_causal = cfg.is_causal, + .window = cfg.window, + .use_sink = cfg.use_sink, + }).convert(q.dtype()); + } + cfg.chunk = if (plan.chunk > 0) plan.chunk else keys; + cfg.a = (std.math.divCeil(i64, keys, cfg.chunk) catch unreachable) * cfg.chunk; + const table = parameters.block_table.pad(-1, .{ .p = Tensor.Pad{ .high = @divExact(cfg.a, page) - parameters.block_table.dim(.p) } }); + // WORKAROUND (TCC 2026.5.0b0): attention over query groups of a size + // other than a power of two fails ("Inconsistent tensor group + // offsets"): pad the groups with zero queries, whose outputs are + // dropped. Repro: + // repro_flash_prefill_group6_inconsistent_offsets.py (zml/reproducer, + // platform/furiosa). Remove once any group size compiles. + const groups: i64 = @intCast(std.math.ceilPowerOfTwo(u64, @intCast(cfg.g)) catch unreachable); + const extra = groups - cfg.g; + cfg.g = groups; + const queries = q.convert(.bf16).pad(0, .{ .hg = Tensor.Pad{ .high = extra } }); + const inputs: OneShot.Inputs = .{ + .q = queries, + .pages = table, + .seq_lens = parameters.seq_lens, + .query_offsets = parameters.query_start_len, + .k_cache = k_cache.convert(.bf16), + .v_cache = v_cache.convert(.bf16), + .sink = sink.pad(0, .{ .hg = Tensor.Pad{ .high = extra } }), + }; + const outputs: OneShot.Outputs = .{ .out = queries.shape() }; + const out = if (cfg.a == cfg.chunk) + OneShot.call(inputs, outputs, .{ .cfg = cfg }).out + else + Flash.call(inputs, outputs, .{ .cfg = cfg }).out; + return out.slice(.hg, .{ .end = groups - extra }).convert(q.dtype()); + } +}; + +fn defaultPlan(cfg: Cfg, keys: i64, columns: i64, plain_decode: bool, straight: bool, chunk: i64) Plan { + // WORKAROUND (TCC 2026.5.0b0): the hybrid kernel's tail does not compile in + // reasonable time for 2 or 4 KV heads wider than 128, which run flash instead. + // Repro: repro_hybrid_decode_slow_compile.py. Remove once that tail compiles. + if (plain_decode and cfg.tokens == 1 and columns > std.math.divCeil(i64, straight_decode_keys, cfg.pg) catch unreachable) { + if (cfg.d <= 128 or cfg.hkv == 1 or cfg.hkv >= 8) return .{ .head = head_keys, .chunk = tail_chunk_keys, .group = 1 }; + return .{ .head = 0, .chunk = chunk, .group = cfg.b * cfg.i }; + } + if (straight and cfg.window > 0) { + const window_chunk = @max(window_chunk_keys, @as(i64, @intCast(std.math.ceilPowerOfTwo(u64, @intCast(cfg.window)) catch unreachable))); + const chunks: i64 = if (cfg.tokens < chunked_decode_tokens) window_flash_chunks else batch_window_flash_chunks; + if (window_chunk <= chunk and @rem(window_chunk, cfg.pg) == 0 and keys >= chunks * window_chunk) + return .{ .head = 0, .chunk = window_chunk, .group = cfg.tokens }; + } + if (straight and keys <= decode_keys and (cfg.tokens < chunked_decode_tokens or keys <= straight_batch_decode_keys)) + return .{ .head = keys, .chunk = 0, .group = @divExact(cfg.tokens, decodeGroups(cfg.tokens, keys)) }; + if (chunk == keys) return .{ .head = keys, .chunk = 0, .group = cfg.tokens }; + return .{ .head = 0, .chunk = chunk, .group = cfg.b * cfg.i }; +} + +const window_chunk_keys = 1024; +const window_flash_chunks = 16; +const batch_window_flash_chunks = 4; + +/// Widest block table the straight-line decode kernel takes for one token. +/// WORKAROUND (TCC 2026.5.0b0): several tokens do not use it, as TCC cannot +/// propagate gather bounds outside a loop over a token axis. Repro: +/// repro_sparse_propagation_loop.py. Remove once such kernels propagate. +const straight_decode_keys = 32768; + +/// Keys a straight-line decode kernel attends to in one pass. WORKAROUND (TCC +/// 2026.5.0b0): TCC fails on the decode metadata for 131072 keys in pages of 16. +/// Repro: repro_metadata_einsum_by_ve_no_tactic.py. Remove once that compiles. +const decode_keys = 65536; + +const chunked_decode_tokens = 16; +const straight_batch_decode_keys = 8192; + +/// Decode contracts the gathered cache unmasked: the select gives unused entries (NaN included) zero +/// weight. WORKAROUND (TCC 2026.5.0b0): TCC panics ("slice out of bounds") on score contractions of more +/// tokens times keys than this. Repro: repro_gather_einsum_slice_panic.py. Remove once larger products compile. +const unmasked_decode_keys = 1 << 17; + +const masked_decode_keys = 4096; + +const DecodeCfg = struct { + t: i64, + a: i64, + group: i64, + hkv: i64, + g: i64, + d: i64, + pages: i64, + pg: i64, + scale: f32, + is_causal: bool, + window: i64, + use_sink: bool, +}; + +/// The token metadata depends only on parameters every layer shares, so one kernel computes it per forward pass. +fn decodeAttention(parameters: paged.Parameters, q: Tensor, k_cache: Tensor, v_cache: Tensor, sink: Tensor, cfg: DecodeCfg) Tensor { + const columns = @divExact(cfg.a, cfg.pg); + const table = parameters.block_table.pad(0, .{ .p = Tensor.Pad{ .high = columns - parameters.block_table.dim(.p) } }); + const meta = DecodeMeta.call(.{ .seq_lens = parameters.seq_lens, .query_offsets = parameters.query_start_len, .table = table }, .{ + .pages = zml.Shape.init(.{ .t = cfg.t, .p = columns }, .i32), + .bounds = zml.Shape.init(.{ .t = cfg.t }, .i32), + .mask = zml.Shape.init(.{ .t = cfg.t, .a = cfg.a }, .i32), + .keyed = zml.Shape.init(.{ .t = cfg.t }, .f32), + }, .{ .cfg = .{ .t = cfg.t, .s = parameters.seq_lens.dim(.b), .ap = columns, .pg = cfg.pg, .pages = cfg.pages, .is_causal = cfg.is_causal, .window = cfg.window } }); + return Decode.call(.{ + .q = q.convert(.bf16), + .pages = meta.pages, + .bounds = meta.bounds, + .mask = meta.mask, + .keyed = meta.keyed, + .k_cache = k_cache.convert(.bf16), + .v_cache = v_cache.convert(.bf16), + .sink = sink, + }, .{ .out = q.shape().withDtype(.bf16) }, .{ .cfg = cfg }).out; +} + +const MetaCfg = struct { t: i64, s: i64, ap: i64, pg: i64, pages: i64, is_causal: bool, window: i64 }; + +/// WORKAROUND (TCC 2026.5.0b0): one-hot row selection and `attentionMask`'s clamp arithmetic, as TCC +/// fails on small integer gathers and the plugin's comparisons. Repro: repro_scalar_gather_no_axis_to_pad.py, +/// repro_row_broadcast_past_4096.py. Remove once both compile. +const DecodeMeta = tcl.Kernel(MetaCfg, .{ + .name = "paged_decode_metadata", + .inputs = &.{ "seq_lens", "query_offsets", "table" }, + .outputs = &.{ "pages", "bounds", "mask", "keyed" }, + .run = struct { + fn run(b: *tcl.Builder, cfg: MetaCfg) tcl.FinishError!void { + const T = b.axis("T", cfg.t); + const S = b.axis("S", cfg.s); + const O = b.axis("O", cfg.s + 1); + const AP = b.axis("AP", cfg.ap); + const PG = b.axis("PG", cfg.pg); + const A = b.axis("A", cfg.ap * cfg.pg); + const t = try b.declareArgs(.{ + .seq_lens = .{ .dtype = .i32, .axes = &.{S} }, + .query_offsets = .{ .dtype = .i32, .axes = &.{O} }, + .table = .{ .dtype = .i32, .axes = &.{ S, AP } }, + }); + const ends = try selectMetadata(b, t.query_offsets, try selector(b, try b.arange(cfg.s + 1, .i32, &.{S}, .{ .start = 1 }), O, cfg.s + 1), O); + const tokens = try b.arange(cfg.t, .i32, &.{T}, .{}); + const seq = try countBefore(b, ends, tokens, S); + const own = try selector(b, seq, S, cfg.s); + var op = b.tensorOperation(.{}); + const active = try op.commit(op.fetch(seq, .{}).muli(-1).addi(cfg.s).binary(.maxi, 0).binary(.mini, 1), .{}); + var visible = try selectMetadata(b, t.seq_lens, own, S); + if (cfg.is_causal) { + const end = try selectMetadata(b, ends, own, S); + op = b.tensorOperation(.{}); + visible = try op.commit(op.fetch(visible, .{}).subi(op.fetch(end, .{})).addi(op.fetch(tokens, .{})).addi(1), .{}); + } + op = b.tensorOperation(.{}); + const limit = try op.commit(op.fetch(visible, .{}).binary(.maxi, 0).binary(.mini, cfg.ap * cfg.pg).muli(op.fetch(active, .{})), .{}); + const entries = try selectMetadata(b, t.table, own, S); + const pages = try validIndices(b, entries, cfg.pages); + const firsts = try b.arange(cfg.ap * cfg.pg, .i32, &.{AP}, .{ .step = cfg.pg }); + op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const used = try op.commit(op.fetch(limit, .{}).subi(op.fetch(firsts, .{})).binary(.maxi, 0).binary(.mini, 1), .{}); + op = b.tensorOperation(.{}); + const bounds = try op.commit(op.fetch(used, .{}).reduce(&.{AP}, .addi), .{}); + op = b.tensorOperation(.{}); + const spread = try op.commit(op.fetch(try b.arange(cfg.ap, .i32, &.{AP}, .{}), .{}).binary(.mini, cfg.pages - 1), .{}); + op = b.tensorOperation(.{}); + const read = try op.commit(op.where(op.fetch(used, .{}), .ne, 0, op.fetch(pages.safe, .{}), op.fetch(spread, .{ .broadcast_to = &.{ T, AP } })), .{}); + op = b.tensorOperation(.{}); + const keyed = try op.commit(op.fetch(used, .{}).muli(op.fetch(pages.valid, .{})).reduce(&.{AP}, .maxi).toFloat(), .{}); + op = b.tensorOperation(.{}); + const valid = try op.commit(op.fetch(pages.valid, .{ .broadcast_to = &.{ T, AP, PG } }), .{}); + op = b.tensorOperation(.{}); + const position = try op.commit(op.fetch(try b.arange(cfg.ap * cfg.pg, .i32, &.{A}, .{}), .{ .broadcast_to = &.{ T, A } }), .{}); + op = b.tensorOperation(.{ .tactic = .elementwise }); + var mask = try op.commit(op.fetch(position, .{}).muli(-1).addi(op.fetch(limit, .{})).binary(.maxi, 0).binary(.mini, 1).muli(op.fetch(try b.reshape(valid, &.{ T, A }, .{}), .{})), .{}); + if (cfg.window > 0) { + op = b.tensorOperation(.{ .tactic = .elementwise }); + mask = try op.commit(op.fetch(position, .{}).subi(op.fetch(limit, .{})).addi(cfg.window + 1).binary(.maxi, 0).binary(.mini, 1).muli(op.fetch(mask, .{})), .{}); + } + b.ret(&.{ read, bounds, mask, keyed }); + } + }.run, +}); + +const Decode = tcl.Kernel(DecodeCfg, .{ + .name = "paged_decode_attention", + .inputs = &.{ "q", "pages", "bounds", "mask", "keyed", "k_cache", "v_cache", "sink" }, + .outputs = &.{"out"}, + .run = struct { + fn run(b: *tcl.Builder, cfg: DecodeCfg) tcl.FinishError!void { + b.fuse(); + configure(b, .{ .tokens = cfg.t, .sequences = cfg.t, .b = cfg.t, .i = 1, .a = cfg.a, .chunk = cfg.a, .hkv = cfg.hkv, .g = cfg.g, .d = cfg.d, .pages = cfg.pages, .pg = cfg.pg, .scale = cfg.scale, .decode = true }); + const T = b.axis("T", cfg.t); + const A = b.axis("A", cfg.a); + const AP = b.axis("AP", @divExact(cfg.a, cfg.pg)); + const Hkv = b.axis("Hkv", cfg.hkv); + const G = b.axis("G", cfg.g); + const D = b.axis("D", cfg.d); + const P = b.axis("P", cfg.pages); + const PG = b.axis("PG", cfg.pg); + const t = try b.declareArgs(.{ + .q = .{ .dtype = .bf16, .axes = &.{ T, Hkv, G, D } }, + .pages = .{ .dtype = .i32, .axes = &.{ T, AP } }, + .bounds = .{ .dtype = .i32, .axes = &.{T} }, + .mask = .{ .dtype = .i32, .axes = &.{ T, A } }, + .keyed = .{ .dtype = .f32, .axes = &.{T} }, + .k_cache = .{ .dtype = .bf16, .axes = &.{ P, PG, Hkv, D } }, + .v_cache = .{ .dtype = .bf16, .axes = &.{ P, PG, Hkv, D } }, + .sink = .{ .dtype = .f32, .axes = &.{ Hkv, G } }, + }); + const sink = if (cfg.use_sink) t.sink else null; + const groups = @divExact(cfg.t, cfg.group); + if (groups == 1) return b.ret(&.{try decodeTokens(b, cfg.scale, t.q, t.pages, t.bounds, t.mask, t.keyed, t.k_cache, t.v_cache, sink)}); + const NG = b.axis("NG", groups); + const TG = b.axis("TG", @divExact(cfg.t, groups)); + var loop = b.openFor(groups, .{try b.scratchpad(.bf16, &.{ NG, TG, Hkv, G, D })}); + { + const q = try b.indexRead(try b.reshape(t.q, &.{ NG, TG, Hkv, G, D }, .{}), loop.iv); + const pages = try b.indexRead(try b.reshape(t.pages, &.{ NG, TG, AP }, .{}), loop.iv); + const bounds = try b.indexRead(try b.reshape(t.bounds, &.{ NG, TG }, .{}), loop.iv); + const mask = try b.indexRead(try b.reshape(t.mask, &.{ NG, TG, A }, .{}), loop.iv); + const keyed = try b.indexRead(try b.reshape(t.keyed, &.{ NG, TG }, .{}), loop.iv); + const out = try decodeTokens(b, cfg.scale, q, pages, bounds, mask, keyed, t.k_cache, t.v_cache, sink); + try loop.yield(.{try b.indexWrite(loop.carried[0], loop.iv, out)}); + } + b.ret(&.{try b.reshape(loop.results[0], &.{ T, Hkv, G, D }, .{})}); + } + }.run, +}); + +const unmasked_group_tokens = 4; + +/// Groups of at most `unmasked_group_tokens` tokens whose tokens times keys fit `unmasked_decode_keys`. +fn decodeGroups(tokens: i64, keys: i64) i64 { + var groups: i64 = 1; + while ((@divExact(tokens, groups) > unmasked_group_tokens or @divExact(tokens, groups) * keys > unmasked_decode_keys) and groups < tokens) { + groups += 1; + while (@rem(tokens, groups) != 0) groups += 1; + } + return groups; +} + +/// `keyed` scales the softmax: zero for tokens without keys. +fn decodeTokens(b: *tcl.Builder, scale: f32, q: builder.Tensor, pages: builder.Tensor, bounds: builder.Tensor, mask: builder.Tensor, keyed: builder.Tensor, k_cache: builder.Tensor, v_cache: builder.Tensor, sink: ?builder.Tensor) tcl.FinishError!builder.Tensor { + const T = q.axes[0]; + const Hkv = q.axes[1]; + const G = q.axes[2]; + const D = q.axes[3]; + const AP = pages.axes[1]; + const P = k_cache.axes[0]; + const PG = k_cache.axes[1]; + const keys: []const tcl.Axis = &.{ AP, PG }; + const key_mask = try b.reshape(mask, &.{ T, AP, PG }, .{}); + var k = try b.gather(k_cache, pages, P, &.{ T, AP, PG, Hkv, D }, .{ .valid_length = bounds }); + var v = try b.gather(v_cache, pages, P, &.{ T, AP, PG, Hkv, D }, .{ .valid_length = bounds }); + if (T.size == 1 and AP.size * PG.size <= masked_decode_keys) { + k = try maskCacheValues(b, k, key_mask); + v = try maskCacheValues(b, v, key_mask); + } + var op = b.tensorOperation(.{}); + const scores = try op.commit(op.contract(q, k, &.{ Hkv, G, T, AP, PG }).mulf(scale), .{}); + op = b.tensorOperation(.{}); + const s = op.fetch(scores, .{}); + const masked = try op.commit(op.where(op.fetch(key_mask, .{}), .ne, 0, s, score_min), .{}); + op = b.tensorOperation(.{}); + const p = if (sink) |logits| blk: { + const maximum = try op.commit(op.fetch(masked, .{}).reduce(keys, .maxf).maxf(op.fetch(logits, .{ .broadcast_to = &.{ Hkv, G, T } })), .{}); + op = b.tensorOperation(.{}); + const e = try op.commit(op.fetch(masked, .{}).subf(op.fetch(maximum, .{})).exp(), .{}); + op = b.tensorOperation(.{}); + const sink_exp = try op.commit(op.fetch(maximum, .{}).mulf(-1).addf(op.fetch(logits, .{})).exp(), .{}); + op = b.tensorOperation(.{}); + const denominator = try op.commit(op.fetch(e, .{}).reduce(keys, .addf).addf(op.fetch(sink_exp, .{})), .{}); + op = b.tensorOperation(.{}); + break :blk try op.commit(op.fetch(e, .{}).divf(op.fetch(denominator, .{})).mulf(op.fetch(keyed, .{})), .{ .dtype = .bf16 }); + } else blk: { + const x = op.fetch(masked, .{}); + const e = x.subf(x.reduce(keys, .maxf)).exp(); + break :blk try op.commit(e.divf(e.reduce(keys, .addf)).mulf(op.fetch(keyed, .{})), .{ .dtype = .bf16 }); + }; + op = b.tensorOperation(.{}); + return op.commit(op.contract(v, p, &.{ T, Hkv, G, D }), .{ .dtype = .bf16 }); +} + +const head_keys = 4096; +const tail_chunk_keys = 8192; + +const HybridCfg = struct { + t: i64, + s: i64, + /// Head pages, tail chunks and pages per tail chunk. + hp: i64, + nc: i64, + cp: i64, + pg: i64, + hkv: i64, + g: i64, + d: i64, + pages: i64, + scale: f32, + is_causal: bool, +}; + +fn hybridDecodeAttention(parameters: paged.Parameters, q: Tensor, k_cache: Tensor, v_cache: Tensor, scale: f32, is_causal: bool, plan: Plan) Tensor { + const pg = k_cache.dim(.k_chunk); + const t = q.dim(.b); + const s = parameters.seq_lens.dim(.b); + const hp = std.math.divCeil(i64, plan.head, pg) catch unreachable; + const cp = std.math.divCeil(i64, plan.chunk, pg) catch unreachable; + const nc = std.math.divCeil(i64, parameters.block_table.dim(.p) - hp, cp) catch unreachable; + const cfg: HybridCfg = .{ .t = t, .s = s, .hp = hp, .nc = nc, .cp = cp, .pg = pg, .hkv = q.dim(.hkv), .g = q.dim(.hg), .d = q.dim(.hd), .pages = k_cache.dim(.page), .scale = scale, .is_causal = is_causal }; + const table = parameters.block_table.pad(-1, .{ .p = Tensor.Pad{ .high = hp + nc * cp - parameters.block_table.dim(.p) } }); + const meta = HybridMeta.call(.{ .seq_lens = parameters.seq_lens, .query_offsets = parameters.query_start_len, .table = table }, .{ + .hpages = zml.Shape.init(.{ .t = t, .p = hp }, .i32), + .hmask = zml.Shape.init(.{ .t = t, .a = hp * pg }, .i32), + .hbounds = zml.Shape.init(.{ .t = t }, .i32), + .tpages = zml.Shape.init(.{ .c = nc, .t = t, .p = cp }, .i32), + .tmask = zml.Shape.init(.{ .c = nc, .t = t, .a = cp * pg }, .i32), + .tbounds = zml.Shape.init(.{ .c = nc, .t = t }, .i32), + .limits = zml.Shape.init(.{ .t = t }, .i32), + .keyed = zml.Shape.init(.{ .t = t }, .f32), + }, .{ .cfg = cfg }); + return HybridDecode.call(.{ + .q = q.convert(.bf16), + .hpages = meta.hpages, + .hmask = meta.hmask, + .hbounds = meta.hbounds, + .tpages = meta.tpages, + .tmask = meta.tmask, + .tbounds = meta.tbounds, + .limits = meta.limits, + .keyed = meta.keyed, + .k_cache = k_cache.convert(.bf16), + .v_cache = v_cache.convert(.bf16), + }, .{ .out = q.shape().withDtype(.bf16) }, .{ .cfg = cfg }).out; +} + +/// Per token, once per forward pass. Tail chunks are chunk-major, only those holding keys of the +/// longest token; `limits` is the visible key count. +const HybridMeta = tcl.Kernel(HybridCfg, .{ + .name = "paged_hybrid_metadata", + .inputs = &.{ "seq_lens", "query_offsets", "table" }, + .outputs = &.{ "hpages", "hmask", "hbounds", "tpages", "tmask", "tbounds", "limits", "keyed" }, + .run = struct { + fn run(b: *tcl.Builder, cfg: HybridCfg) tcl.FinishError!void { + const total = (cfg.hp + cfg.nc * cfg.cp) * cfg.pg; + const T = b.axis("T", cfg.t); + const S = b.axis("S", cfg.s); + const O = b.axis("O", cfg.s + 1); + const AP = b.axis("AP", cfg.hp + cfg.nc * cfg.cp); + const HP = b.axis("HP", cfg.hp); + const RP = b.axis("RP", cfg.nc * cfg.cp); + const NC = b.axis("NC", cfg.nc); + const CP = b.axis("CP", cfg.cp); + const HA = b.axis("HA", cfg.hp * cfg.pg); + const CK = b.axis("CK", cfg.cp * cfg.pg); + const t = try b.declareArgs(.{ + .seq_lens = .{ .dtype = .i32, .axes = &.{S} }, + .query_offsets = .{ .dtype = .i32, .axes = &.{O} }, + .table = .{ .dtype = .i32, .axes = &.{ S, AP } }, + }); + const ends = try selectMetadata(b, t.query_offsets, try selector(b, try b.arange(cfg.s + 1, .i32, &.{S}, .{ .start = 1 }), O, cfg.s + 1), O); + const tokens = try b.arange(cfg.t, .i32, &.{T}, .{}); + const seq = try countBefore(b, ends, tokens, S); + const own = try selector(b, seq, S, cfg.s); + var op = b.tensorOperation(.{}); + const active = try op.commit(op.fetch(seq, .{}).muli(-1).addi(cfg.s).binary(.maxi, 0).binary(.mini, 1), .{}); + var visible = try selectMetadata(b, t.seq_lens, own, S); + if (cfg.is_causal) { + const end = try selectMetadata(b, ends, own, S); + op = b.tensorOperation(.{}); + visible = try op.commit(op.fetch(visible, .{}).subi(op.fetch(end, .{})).addi(op.fetch(tokens, .{})).addi(1), .{}); + } + op = b.tensorOperation(.{}); + const limit = try op.commit(op.fetch(visible, .{}).binary(.maxi, 0).binary(.mini, total).muli(op.fetch(active, .{})), .{}); + + // WORKAROUND (TCC 2026.5.0b0): a 3-D one-hot product computes garbage when the one-hot is read + // first, so the selection is 2-D per chunk. Repro: repro_onehot_einsum_by_ve_wrong_values_run.py. + // Remove once it is exact on RNGD. + const rows1 = try b.reshape(try b.slice(t.table, AP, cfg.hp, &.{ S, RP }, .{}), &.{ S, NC, CP }, .{}); + op = b.tensorOperation(.{}); + const rows = try op.commit(op.fetch(rows1, .{}), .{ .axes = &.{ NC, S, CP } }); + // Trips never exceed nc: `limit` is at most the table's keys. + const trips = try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(limit), cfg.cp * cfg.pg - 1 - cfg.hp * cfg.pg), cfg.cp * cfg.pg); + var loop = b.openFor(trips, .{ try b.scratchpad(.i32, &.{ NC, T, CP }), try b.scratchpad(.i32, &.{ NC, T, CK }), try b.scratchpad(.i32, &.{ NC, T }), try b.full(0, .i32, &.{T}) }); + { + const reads, const masks, const counts, const seen = loop.carried; + const first = try b.symExpr(.add, try b.symExpr(.mul, loop.iv, cfg.cp * cfg.pg), cfg.hp * cfg.pg); + const chunk = try hybridColumns(b, cfg, own, try b.indexRead(rows, loop.iv), limit, first, CP, CK); + op = b.tensorOperation(.{}); + const any = try op.commit(op.fetch(seen, .{}).binary(.maxi, op.fetch(chunk.any, .{})), .{}); + try loop.yield(.{ + try b.indexWrite(reads, loop.iv, chunk.pages), + try b.indexWrite(masks, loop.iv, chunk.mask), + try b.indexWrite(counts, loop.iv, chunk.bounds), + any, + }); + } + const head = try hybridColumns(b, cfg, own, try b.slice(t.table, AP, 0, &.{ S, HP }, .{}), limit, 0, HP, HA); + op = b.tensorOperation(.{}); + const keyed = try op.commit(op.fetch(head.any, .{}).binary(.maxi, op.fetch(loop.results[3], .{})).toFloat(), .{}); + b.ret(&.{ head.pages, head.mask, head.bounds, loop.results[0], loop.results[1], loop.results[2], limit, keyed }); + } + }.run, +}); + +/// Columns start at key `first`; sparse gathers never read past a token's `bounds`. +fn hybridColumns(b: *tcl.Builder, cfg: HybridCfg, own: builder.Tensor, table: builder.Tensor, limit: builder.Tensor, first: anytype, cols: tcl.Axis, keys: tcl.Axis) tcl.FinishError!struct { pages: builder.Tensor, mask: builder.Tensor, bounds: builder.Tensor, any: builder.Tensor } { + const T = own.axes[0]; + const S = own.axes[1]; + const PG = b.axis("PG", cfg.pg); + const pages = try validIndices(b, try selectMetadata(b, table, own, S), cfg.pages); + var op = b.tensorOperation(.{}); + const firsts = try op.commit(op.fetch(try b.arange(cols.size * cfg.pg, .i32, &.{cols}, .{ .step = cfg.pg }), .{}).addi(if (@TypeOf(first) == builder.Tensor) op.fetch(first, .{}) else first), .{}); + op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const used = try op.commit(op.fetch(limit, .{}).subi(op.fetch(firsts, .{})).binary(.maxi, 0).binary(.mini, 1), .{}); + op = b.tensorOperation(.{}); + const bounds = try op.commit(op.fetch(used, .{}).reduce(&.{cols}, .addi), .{}); + op = b.tensorOperation(.{}); + const any = try op.commit(op.fetch(used, .{}).muli(op.fetch(pages.valid, .{})).reduce(&.{cols}, .maxi), .{}); + op = b.tensorOperation(.{}); + const key_ids = try op.commit(op.fetch(try b.arange(keys.size, .i32, &.{keys}, .{}), .{}).addi(if (@TypeOf(first) == builder.Tensor) op.fetch(first, .{}) else first), .{}); + const to: []const tcl.Axis = &.{ T, cols, PG }; + op = b.tensorOperation(.{}); + const position = try op.commit(op.fetch(try b.reshape(key_ids, &.{ cols, PG }, .{}), .{ .broadcast_to = to }), .{}); + op = b.tensorOperation(.{}); + const valid = try op.commit(op.fetch(pages.valid, .{ .broadcast_to = to }), .{}); + op = b.tensorOperation(.{ .tactic = .elementwise }); + const mask = try op.commit(op.fetch(position, .{}).muli(-1).addi(op.fetch(limit, .{})).binary(.maxi, 0).binary(.mini, 1).muli(op.fetch(valid, .{})), .{}); + return .{ .pages = pages.safe, .mask = try b.reshape(mask, &.{ T, keys }, .{}), .bounds = bounds, .any = any }; +} + +const SoftmaxState = struct { max: builder.Tensor, sum: builder.Tensor, out: builder.Tensor }; + +const HybridDecode = tcl.Kernel(HybridCfg, .{ + .name = "paged_hybrid_decode_attention", + .inputs = &.{ "q", "hpages", "hmask", "hbounds", "tpages", "tmask", "tbounds", "limits", "keyed", "k_cache", "v_cache" }, + .outputs = &.{"out"}, + .run = struct { + fn run(b: *tcl.Builder, cfg: HybridCfg) tcl.FinishError!void { + b.fuse(); + const a = hybridAxes(b, cfg); + const t = try b.declareArgs(.{ + .q = .{ .dtype = .bf16, .axes = &.{ a.T, a.Hkv, a.G, a.D } }, + .hpages = .{ .dtype = .i32, .axes = &.{ a.T, a.HP } }, + .hmask = .{ .dtype = .i32, .axes = &.{ a.T, a.HA } }, + .hbounds = .{ .dtype = .i32, .axes = &.{a.T} }, + .tpages = .{ .dtype = .i32, .axes = &.{ a.NC, a.T, a.CP } }, + .tmask = .{ .dtype = .i32, .axes = &.{ a.NC, a.T, a.CK } }, + .tbounds = .{ .dtype = .i32, .axes = &.{ a.NC, a.T } }, + .limits = .{ .dtype = .i32, .axes = &.{a.T} }, + .keyed = .{ .dtype = .f32, .axes = &.{a.T} }, + .k_cache = .{ .dtype = .bf16, .axes = &.{ a.P, a.PG, a.Hkv, a.D } }, + .v_cache = .{ .dtype = .bf16, .axes = &.{ a.P, a.PG, a.Hkv, a.D } }, + }); + const tail = try hybridTail(b, cfg, a, t.q, t.tpages, t.tmask, t.tbounds, t.limits, t.k_cache, t.v_cache); + b.ret(&.{try hybridHead(b, cfg, a, t.q, t.hpages, t.hmask, t.hbounds, t.keyed, t.k_cache, t.v_cache, tail)}); + } + }.run, +}); + +const HybridAxes = struct { T: tcl.Axis, HP: tcl.Axis, HA: tcl.Axis, NC: tcl.Axis, CP: tcl.Axis, CK: tcl.Axis, Hkv: tcl.Axis, G: tcl.Axis, D: tcl.Axis, P: tcl.Axis, PG: tcl.Axis }; + +fn hybridAxes(b: *tcl.Builder, cfg: HybridCfg) HybridAxes { + return .{ + .T = b.axis("T", cfg.t), + .HP = b.axis("HP", cfg.hp), + .HA = b.axis("HA", cfg.hp * cfg.pg), + .NC = b.axis("NC", cfg.nc), + .CP = b.axis("CP", cfg.cp), + .CK = b.axis("CK", cfg.cp * cfg.pg), + .Hkv = b.axis("Hkv", cfg.hkv), + .G = b.axis("G", cfg.g), + .D = b.axis("D", cfg.d), + .P = b.axis("P", cfg.pages), + .PG = b.axis("PG", cfg.pg), + }; +} + +fn hybridTail(b: *tcl.Builder, cfg: HybridCfg, a: HybridAxes, q: builder.Tensor, tpages: builder.Tensor, tmask: builder.Tensor, tbounds: builder.Tensor, limits: builder.Tensor, k_cache: builder.Tensor, v_cache: builder.Tensor) tcl.FinishError!SoftmaxState { + const stats: []const tcl.Axis = &.{ a.Hkv, a.G, a.T }; + const trips = try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(limits), cfg.cp * cfg.pg - 1 - cfg.hp * cfg.pg), cfg.cp * cfg.pg); + var loop = b.openFor(trips, .{ try b.full(score_min, .f32, stats), try b.full(0, .f32, stats), try b.full(0, .f32, &.{ a.T, a.Hkv, a.G, a.D }) }); + { + const max, const sum, const out = loop.carried; + const mask = try b.reshape(try b.indexRead(tmask, loop.iv), &.{ a.T, a.CP, a.PG }, .{}); + const bounds = try b.indexRead(tbounds, loop.iv); + const pages = try b.indexRead(tpages, loop.iv); + const k = try b.gather(k_cache, pages, a.P, &.{ a.T, a.CP, a.PG, a.Hkv, a.D }, .{ .valid_length = bounds }); + const v = try b.gather(v_cache, pages, a.P, &.{ a.T, a.CP, a.PG, a.Hkv, a.D }, .{ .valid_length = bounds }); + const merged = try mergeSoftmax(b, .{ .max = max, .sum = sum, .out = out }, try chunkSoftmax(b, cfg, q, k, v, mask)); + try loop.yield(.{ merged.max, merged.sum, merged.out }); + } + return .{ .max = loop.results[0], .sum = loop.results[1], .out = loop.results[2] }; +} + +fn hybridHead(b: *tcl.Builder, cfg: HybridCfg, a: HybridAxes, q: builder.Tensor, hpages: builder.Tensor, hmask: builder.Tensor, hbounds: builder.Tensor, keyed: builder.Tensor, k_cache: builder.Tensor, v_cache: builder.Tensor, tail: SoftmaxState) tcl.FinishError!builder.Tensor { + const head_mask = try b.reshape(hmask, &.{ a.T, a.HP, a.PG }, .{}); + const head_k = try b.gather(k_cache, hpages, a.P, &.{ a.T, a.HP, a.PG, a.Hkv, a.D }, .{ .valid_length = hbounds }); + const head_v = try b.gather(v_cache, hpages, a.P, &.{ a.T, a.HP, a.PG, a.Hkv, a.D }, .{ .valid_length = hbounds }); + const keys: []const tcl.Axis = &.{ a.HP, a.PG }; + var op = b.tensorOperation(.{}); + const scores = try op.commit(op.contract(q, head_k, &.{ a.Hkv, a.G, a.T, a.HP, a.PG }).mulf(cfg.scale), .{}); + op = b.tensorOperation(.{}); + const sv = op.fetch(scores, .{}); + const masked = try op.commit(op.where(op.fetch(head_mask, .{}), .ne, 0, sv, score_min), .{}); + op = b.tensorOperation(.{}); + const max = try op.commit(op.fetch(masked, .{}).reduce(keys, .maxf).maxf(op.fetch(tail.max, .{})), .{}); + const alpha = try expDifference(b, tail.max, max); + op = b.tensorOperation(.{}); + const carried = try op.commit(op.fetch(tail.sum, .{}).mulf(op.fetch(alpha, .{})), .{}); + op = b.tensorOperation(.{}); + const denominator = try op.commit(op.fetch(masked, .{}).subf(op.fetch(max, .{})).exp().reduce(keys, .addf).addf(op.fetch(carried, .{})), .{}); + op = b.tensorOperation(.{}); + const p = try op.commit(op.fetch(masked, .{}).subf(op.fetch(max, .{})).exp().divf(op.fetch(denominator, .{})).mulf(op.fetch(keyed, .{})), .{ .dtype = .bf16 }); + op = b.tensorOperation(.{}); + const head = try op.commit(op.contract(head_v, p, &.{ a.T, a.Hkv, a.G, a.D }), .{}); + op = b.tensorOperation(.{}); + const factor = try op.commit(op.fetch(alpha, .{}).divf(op.fetch(denominator, .{})), .{}); + op = b.tensorOperation(.{}); + return op.commit(op.fetch(tail.out, .{}).mulf(op.fetch(factor, .{})).addf(op.fetch(head, .{})).mulf(op.fetch(keyed, .{})), .{ .dtype = .bf16 }); +} + +fn chunkSoftmax(b: *tcl.Builder, cfg: HybridCfg, q: builder.Tensor, k: builder.Tensor, v: builder.Tensor, mask: builder.Tensor) tcl.FinishError!SoftmaxState { + const T = q.axes[0]; + const Hkv = q.axes[1]; + const G = q.axes[2]; + const D = q.axes[3]; + const keys = mask.axes[1..]; + var op = b.tensorOperation(.{}); + const scores = try op.commit(op.contract(q, k, &.{ Hkv, G, T, keys[0], keys[1] }).mulf(cfg.scale), .{}); + op = b.tensorOperation(.{}); + const sv = op.fetch(scores, .{}); + const masked = try op.commit(op.where(op.fetch(mask, .{}), .ne, 0, sv, score_min), .{}); + op = b.tensorOperation(.{}); + const max = try op.commit(op.fetch(masked, .{}).reduce(keys, .maxf), .{}); + const exps = try maskedExp(b, masked, max, mask); + op = b.tensorOperation(.{}); + const sum = try op.commit(op.fetch(exps, .{}).reduce(keys, .addf), .{}); + op = b.tensorOperation(.{}); + const p = try op.commit(op.fetch(exps, .{}).mulf(1.0), .{ .dtype = .bf16 }); + op = b.tensorOperation(.{}); + const out = try op.commit(op.contract(v, p, &.{ T, Hkv, G, D }), .{}); + return .{ .max = max, .sum = sum, .out = out }; +} + +fn mergeSoftmax(b: *tcl.Builder, x: SoftmaxState, y: SoftmaxState) tcl.FinishError!SoftmaxState { + var op = b.tensorOperation(.{ .tactic = .elementwise }); + const max = try op.commit(op.fetch(x.max, .{}).maxf(op.fetch(y.max, .{})), .{}); + const alpha = try expDifference(b, x.max, max); + const beta = try expDifference(b, y.max, max); + op = b.tensorOperation(.{}); + const xs = try op.commit(op.fetch(x.sum, .{}).mulf(op.fetch(alpha, .{})), .{}); + op = b.tensorOperation(.{}); + const sum = try op.commit(op.fetch(y.sum, .{}).mulf(op.fetch(beta, .{})).addf(op.fetch(xs, .{})), .{}); + op = b.tensorOperation(.{}); + const xo = try op.commit(op.fetch(x.out, .{}).mulf(op.fetch(alpha, .{})), .{}); + op = b.tensorOperation(.{}); + const out = try op.commit(op.fetch(y.out, .{}).mulf(op.fetch(beta, .{})).addf(op.fetch(xo, .{})), .{}); + return .{ .max = max, .sum = sum, .out = out }; +} + +const Cfg = struct { + tokens: i64, + sequences: i64, + /// Query blocks and query rows per block. + b: i64, + i: i64, + /// Keys per block (padded to whole chunks) and keys per chunk. + a: i64, + chunk: i64, + hkv: i64, + g: i64, + d: i64, + pages: i64, + pg: i64, + scale: f32, + decode: bool, + use_sink: bool = false, + is_causal: bool = true, + window: i32 = -1, +}; + +/// Prefill bounds the tactic shapes searched (fewer for narrow heads): the compute-bound hint compiles too slowly. +fn configure(b: *tcl.Builder, cfg: Cfg) void { + const shapes: i64 = if (cfg.hkv * cfg.g * cfg.d >= 1024) 16 else 8; + if (cfg.decode) b.compilerConfig(.{ + .enable_einsum_fusion = true, + .padding_policy = .{ .Small = 1.1 }, + .instruction_mem_budget = 0xB0000, + .reshape_einsum_mode = .{ .Reshape = .{ .permute = false } }, + .tensor_unit_bridge_threshold_in_page = 12, + .allow_external_operators = false, + .enable_tactic_pruning = false, + .scheduler_beam_search = true, + .allow_reduce_by_ve_cluster_chip_reduce = false, + .allow_reduce_by_ve_cluster_chip_reduce_base_population = false, + .propagate_sparse_axis_from_op = "Gather", + .tactic_hint = .{ .ForLlmModelIOBound = 200 }, + .num_transaction_simulation_per_pe = 1024, + .dma_preference = 0.8, + }) else b.compilerConfig(.{ + .enable_einsum_fusion = true, + .padding_policy = .{ .Small = 1.1 }, + .instruction_mem_budget = 0xB0000, + .reshape_einsum_mode = .{ .Reshape = .{ .permute = false } }, + .tensor_unit_bridge_threshold_in_page = 12, + .allow_external_operators = false, + .scheduler_beam_search = true, + .allow_reduce_by_ve_cluster_chip_reduce = false, + .allow_reduce_by_ve_cluster_chip_reduce_base_population = false, + .tactic_hint = .{ .DefaultWithMaxNumUniqueShapes = shapes }, + .dma_preference = 1.2, + }); +} + +const Axes = struct { + T: tcl.Axis, + S: tcl.Axis, + O: tcl.Axis, + AP: tcl.Axis, + B: tcl.Axis, + I: tcl.Axis, + Hkv: tcl.Axis, + G: tcl.Axis, + D: tcl.Axis, + P: tcl.Axis, + PG: tcl.Axis, + + fn init(b: *tcl.Builder, cfg: Cfg) Axes { + return .{ + .T = b.axis("T", cfg.tokens), + .S = b.axis("S", cfg.sequences), + .O = b.axis("O", cfg.sequences + 1), + .AP = b.axis("AP", @divExact(cfg.a, cfg.pg)), + .B = b.axis("B", cfg.b), + .I = b.axis("I", cfg.i), + .Hkv = b.axis("Hkv", cfg.hkv), + .G = b.axis("G", cfg.g), + .D = b.axis("D", cfg.d), + .P = b.axis("P", cfg.pages), + .PG = b.axis("PG", cfg.pg), + }; + } +}; + +const NativeInputs = struct { + q: builder.Tensor, + pages: builder.Tensor, + seq_lens: builder.Tensor, + query_offsets: builder.Tensor, + k_cache: builder.Tensor, + v_cache: builder.Tensor, + sink: builder.Tensor, +}; + +fn declareInputs(b: *tcl.Builder, a: Axes) tcl.FinishError!NativeInputs { + const t = try b.declareArgs(.{ + .q = .{ .dtype = .bf16, .axes = &.{ a.T, a.Hkv, a.G, a.D } }, + .pages = .{ .dtype = .i32, .axes = &.{ a.S, a.AP } }, + .seq_lens = .{ .dtype = .i32, .axes = &.{a.S} }, + .query_offsets = .{ .dtype = .i32, .axes = &.{a.O} }, + .k_cache = .{ .dtype = .bf16, .axes = &.{ a.P, a.PG, a.Hkv, a.D } }, + .v_cache = .{ .dtype = .bf16, .axes = &.{ a.P, a.PG, a.Hkv, a.D } }, + .sink = .{ .dtype = .f32, .axes = &.{ a.Hkv, a.G } }, + }); + return .{ .q = t.q, .pages = t.pages, .seq_lens = t.seq_lens, .query_offsets = t.query_offsets, .k_cache = t.k_cache, .v_cache = t.v_cache, .sink = t.sink }; +} + +const Prepared = struct { + q: builder.Tensor, + /// Page table rows: one per block for decode, one per sequence for prefill (blocks select theirs with `pick`). + pages: builder.Tensor, + pick: ?builder.Tensor, + qpos: builder.Tensor, + kvlen: builder.Tensor, + row: builder.Tensor, + active: builder.Tensor, + /// Per page row: the first query position and the keys. + first: builder.Tensor, + keys: builder.Tensor, + + /// WORKAROUND (TCC 2026.5.0b0): TCC has no tactic for selecting whole rows of 8192 pages, so flash + /// attention selects per chunk. Repro: repro_metadata_einsum_by_ve_no_tactic.py. Remove once such + /// products compile. + fn blockPages(self: Prepared, b: *tcl.Builder, a: Axes, pages: builder.Tensor) tcl.FinishError!builder.Tensor { + const pick = self.pick orelse return pages; + return selectMetadata(b, pages, pick, a.S); + } +}; + +fn countBefore(b: *tcl.Builder, ends: builder.Tensor, positions: builder.Tensor, axis: tcl.Axis) tcl.FinishError!builder.Tensor { + var op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const distance = try op.commit(op.fetch(ends, .{}).subi(op.fetch(positions, .{})), .{}); + op = b.tensorOperation(.{}); + const passed = try op.commit(op.where(distance, .le, 0, 1, 0), .{}); + op = b.tensorOperation(.{}); + return op.commit(op.fetch(passed, .{}).reduce(&.{axis}, .addi), .{}); +} + +fn selector(b: *tcl.Builder, indices: builder.Tensor, axis: tcl.Axis, count: i64) tcl.FinishError!builder.Tensor { + const source = try b.arange(count, .i32, &.{axis}, .{}); + var op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const distance = try op.commit(op.fetch(indices, .{}).subi(op.fetch(source, .{})), .{}); + op = b.tensorOperation(.{}); + return op.commit(op.where(distance, .eq, 0, 1, 0), .{}); +} + +// Keep metadata exact in I32. WORKAROUND (TCC 2026.5.0b0): no native tactics for +// scalar gathers or four-byte-offset slices. Repro: repro_scalar_gather_no_axis_to_pad.py, +// repro_i32_slice_odd_offset.py. Remove once both compile. +fn selectMetadata(b: *tcl.Builder, values: builder.Tensor, selected: builder.Tensor, axis: tcl.Axis) tcl.FinishError!builder.Tensor { + const op = b.tensorOperation(.{ .tactic = if (values.axes.len > 1) .einsum_by_ve else .reduce_by_ve }); + return op.commit(op.fetch(selected, .{}).muli(op.fetch(values, .{})).reduce(&.{axis}, .addi), .{}); +} + +/// Reads adjacent I32 offsets inside the native unit: a separate `offsets[1:]` argument would +/// need alignment staging at every invocation. +fn prepareQueries(b: *tcl.Builder, cfg: Cfg, a: Axes, t: NativeInputs) tcl.FinishError!Prepared { + const start_indices = try b.arange(cfg.sequences, .i32, &.{a.S}, .{}); + const end_indices = try b.arange(cfg.sequences + 1, .i32, &.{a.S}, .{ .start = 1 }); + const starts = try selectMetadata(b, t.query_offsets, try selector(b, start_indices, a.O, cfg.sequences + 1), a.O); + const ends = try selectMetadata(b, t.query_offsets, try selector(b, end_indices, a.O, cfg.sequences + 1), a.O); + var op = b.tensorOperation(.{}); + const qlen = try op.commit(op.fetch(ends, .{}).subi(op.fetch(starts, .{})), .{}); + const tokens = try b.arange(cfg.tokens, .i32, &.{a.T}, .{}); + const seq_of_token = try countBefore(b, ends, tokens, a.S); + // Padding tokens follow all sequence ends; this vector avoids a symbolic scalar reduction in a vector operation. + op = b.tensorOperation(.{}); + const token_tail = try op.commit(op.fetch(seq_of_token, .{}).subi(cfg.sequences), .{}); + op = b.tensorOperation(.{}); + const active = try op.commit(op.where(token_tail, .lt, 0, 1, 0), .{}); + op = b.tensorOperation(.{}); + const token_seq = try op.commit(op.fetch(seq_of_token, .{}).binary(.mini, cfg.sequences - 1), .{}); + if (cfg.decode) { + op = b.tensorOperation(.{}); + const live = try op.commit(op.where(qlen, .gt, 0, 1, 0), .{}); + op = b.tensorOperation(.{}); + const lengths = try op.commit(op.fetch(t.seq_lens, .{}).muli(op.fetch(live, .{})), .{}); + op = b.tensorOperation(.{}); + const qpos = try op.commit(op.fetch(lengths, .{}).subi(1), .{}); + op = b.tensorOperation(.{}); + const safe_start = try op.commit(op.fetch(starts, .{}).binary(.mini, cfg.tokens - 1), .{}); + const q = if (cfg.tokens == 1) t.q else try b.gather(t.q, safe_start, a.T, &.{ a.S, a.Hkv, a.G, a.D }, .{}); + return .{ + .q = try b.reshape(q, &.{ a.B, a.I, a.Hkv, a.G, a.D }, .{}), + .pages = try b.reshape(t.pages, &.{ a.B, a.AP }, .{}), + .pick = null, + .qpos = try b.reshape(qpos, &.{ a.B, a.I }, .{}), + .kvlen = try b.reshape(lengths, &.{a.B}, .{}), + .row = token_seq, + .active = active, + .first = try b.reshape(qpos, &.{a.B}, .{}), + .keys = try b.reshape(lengths, &.{a.B}, .{}), + }; + } + + const shift = std.math.log2_int(u64, @intCast(cfg.i)); + op = b.tensorOperation(.{}); + const blocks = try op.commit(op.fetch(qlen, .{}).addi(cfg.i - 1).binary(.shr_arith, shift), .{}); + op = b.tensorOperation(.{}); + const block_end = try op.commit(op.fetch(blocks, .{}).cumsum(&.{a.S}), .{}); + op = b.tensorOperation(.{}); + const block_start = try op.commit(op.fetch(block_end, .{}).subi(op.fetch(blocks, .{})), .{}); + const block_ids = try b.arange(cfg.b, .i32, &.{a.B}, .{}); + const seq_of_block = try countBefore(b, block_end, block_ids, a.S); + op = b.tensorOperation(.{}); + const beyond = try op.commit(op.fetch(seq_of_block, .{}).subi(cfg.sequences), .{}); + op = b.tensorOperation(.{}); + const live_block = try op.commit(op.where(beyond, .lt, 0, 1, 0), .{}); + op = b.tensorOperation(.{}); + const seq = try op.commit(op.fetch(seq_of_block, .{}).binary(.mini, cfg.sequences - 1), .{}); + const pick = try selector(b, seq, a.S, cfg.sequences); + const block_begin = try selectMetadata(b, block_start, pick, a.S); + const begin = try selectMetadata(b, starts, pick, a.S); + const query_length = try selectMetadata(b, qlen, pick, a.S); + const key_length = try selectMetadata(b, t.seq_lens, pick, a.S); + const local_rows = try b.arange(cfg.i, .i32, &.{a.I}, .{}); + op = b.tensorOperation(.{}); + const first_row = try op.commit(op.fetch(block_ids, .{}).subi(op.fetch(block_begin, .{})).muli(cfg.i), .{}); + op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const qi = try op.commit(op.fetch(first_row, .{}).addi(op.fetch(local_rows, .{})), .{}); + op = b.tensorOperation(.{}); + const remaining = try op.commit(op.fetch(qi, .{}).subi(op.fetch(query_length, .{})), .{}); + op = b.tensorOperation(.{}); + const inside = try op.commit(op.where(remaining, .lt, 0, 1, 0), .{}); + op = b.tensorOperation(.{}); + const live = try op.commit(op.fetch(inside, .{}).muli(op.fetch(live_block, .{})), .{}); + op = b.tensorOperation(.{}); + const qidx = try op.commit(op.fetch(qi, .{}).addi(op.fetch(begin, .{})).muli(op.fetch(live, .{})), .{}); + const q = try b.gather(t.q, qidx, a.T, &.{ a.B, a.I, a.Hkv, a.G, a.D }, .{}); + op = b.tensorOperation(.{}); + const pos = try op.commit(op.fetch(qi, .{}).addi(op.fetch(key_length, .{})).subi(op.fetch(query_length, .{})), .{}); + op = b.tensorOperation(.{}); + const qpos = try op.commit(op.where(live, .ne, 0, op.fetch(pos, .{}), -1), .{}); + op = b.tensorOperation(.{}); + const kvlen = try op.commit(op.fetch(key_length, .{}).muli(op.fetch(live_block, .{})), .{}); + + const token_pick = try selector(b, token_seq, a.S, cfg.sequences); + const token_begin = try selectMetadata(b, starts, token_pick, a.S); + const token_block = try selectMetadata(b, block_start, token_pick, a.S); + op = b.tensorOperation(.{}); + const row = try op.commit(op.fetch(token_block, .{}).muli(cfg.i).addi(op.fetch(tokens, .{})).subi(op.fetch(token_begin, .{})).binary(.maxi, 0).binary(.mini, cfg.b * cfg.i - 1), .{}); + op = b.tensorOperation(.{}); + const first = try op.commit(op.fetch(t.seq_lens, .{}).subi(op.fetch(qlen, .{})), .{}); + return .{ .q = q, .pages = t.pages, .pick = pick, .qpos = qpos, .kvlen = kvlen, .row = row, .active = active, .first = first, .keys = t.seq_lens }; +} + +fn restoreQueries(b: *tcl.Builder, cfg: Cfg, a: Axes, prepared: Prepared, output: builder.Tensor) tcl.FinishError!builder.Tensor { + if (cfg.decode and cfg.tokens == 1) return b.reshape(output, &.{ a.T, a.Hkv, a.G, a.D }, .{}); + const R = b.axis("R", cfg.b * cfg.i); + const rows = try b.reshape(output, &.{ R, a.Hkv, a.G, a.D }, .{}); + const ordered = try b.gather(rows, prepared.row, R, &.{ a.T, a.Hkv, a.G, a.D }, .{}); + const op = b.tensorOperation(.{ .tactic = .elementwise }); + // Establish the full data layout before broadcasting the I32 row mask. + const values = op.fetch(ordered, .{}); + return op.commit(op.where(op.fetch(prepared.active, .{ .broadcast_to = ordered.axes }), .ne, 0, values, 0.0), .{ .dtype = .bf16 }); +} + +const score_min = -std.math.floatMax(f32); + +fn validIndices(b: *tcl.Builder, indices: builder.Tensor, count: i64) tcl.FinishError!struct { safe: builder.Tensor, valid: builder.Tensor } { + var op = b.tensorOperation(.{}); + const safe = try op.commit(op.fetch(indices, .{}).binary(.maxi, 0).binary(.mini, count - 1), .{}); + op = b.tensorOperation(.{}); + const nonnegative = try op.commit(op.fetch(indices, .{}).addi(1).binary(.maxi, 0).binary(.mini, 1), .{}); + op = b.tensorOperation(.{ .tactic = .elementwise }); + const valid = try op.commit(op.fetch(indices, .{}).muli(-1).addi(count).binary(.maxi, 0).binary(.mini, 1).muli(op.fetch(nonnegative, .{})), .{}); + return .{ .safe = safe, .valid = valid }; +} + +/// Validity is per sequence, not per query, so K/V are not replicated per row of a prefill block. +fn cacheMask(b: *tcl.Builder, valid_pages: builder.Tensor, kvlen: builder.Tensor, position: builder.Tensor) tcl.FinishError!builder.Tensor { + var op = b.tensorOperation(.{}); + const pos = try op.commit(op.fetch(position, .{ .broadcast_to = valid_pages.axes }), .{}); + op = b.tensorOperation(.{ .tactic = .elementwise }); + return op.commit(op.fetch(pos, .{}).muli(-1).addi(op.fetch(kvlen, .{})).binary(.maxi, 0).binary(.mini, 1).muli(op.fetch(valid_pages, .{})), .{}); +} + +/// A select, not a multiply by zero: NaN/Inf in unwritten slots must not reach the contraction. +fn maskCacheValues(b: *tcl.Builder, values: builder.Tensor, valid: builder.Tensor) tcl.FinishError!builder.Tensor { + const op = b.tensorOperation(.{ .tactic = .elementwise }); + const data = op.fetch(values, .{}); + return op.commit(op.where(op.fetch(valid, .{ .broadcast_to = values.axes }), .ne, 0, data, 0.0), .{ .dtype = .bf16 }); +} + +fn attentionMask(b: *tcl.Builder, cfg: Cfg, qpos: builder.Tensor, kvlen: builder.Tensor, position: builder.Tensor, valid_keys: builder.Tensor) tcl.FinishError!builder.Tensor { + const axes = &.{ qpos.axes[0], qpos.axes[1], position.axes[position.axes.len - 1] }; + const read: builder.TensorOperation.FetchOptions = .{ .broadcast_to = axes }; + // Elementwise operations require their first read to cover the output + // axes, and each VE instruction must consume the current accumulator. + var op = b.tensorOperation(.{}); + const pos = try op.commit(op.fetch(position, read), .{}); + op = b.tensorOperation(.{}); + const query = try op.commit(op.fetch(qpos, read), .{}); + op = b.tensorOperation(.{}); + var mask = try op.commit(op.fetch(valid_keys, read), .{}); + op = b.tensorOperation(.{ .tactic = .elementwise }); + mask = try op.commit(op.fetch(pos, .{}).muli(-1).addi(op.fetch(kvlen, .{})).binary(.maxi, 0).binary(.mini, 1).muli(op.fetch(mask, .{})), .{}); + op = b.tensorOperation(.{ .tactic = .elementwise }); + mask = try op.commit(op.fetch(query, .{}).addi(1).binary(.maxi, 0).binary(.mini, 1).muli(op.fetch(mask, .{})), .{}); + if (cfg.is_causal) { + op = b.tensorOperation(.{ .tactic = .elementwise }); + mask = try op.commit(op.fetch(query, .{}).subi(op.fetch(pos, .{})).addi(1).binary(.maxi, 0).binary(.mini, 1).muli(op.fetch(mask, .{})), .{}); + } + if (cfg.window > 0) { + op = b.tensorOperation(.{ .tactic = .elementwise }); + mask = try op.commit(op.fetch(pos, .{}).subi(op.fetch(query, .{})).addi(cfg.window).binary(.maxi, 0).binary(.mini, 1).muli(op.fetch(mask, .{})), .{}); + } + return mask; +} + +fn maskedScores(b: *tcl.Builder, a: Axes, cfg: Cfg, q: builder.Tensor, k: builder.Tensor, mask: builder.Tensor, keys: tcl.Axis) tcl.FinishError!builder.Tensor { + const scores_axes = &.{ a.Hkv, a.G, a.B, a.I, keys }; + var op = b.tensorOperation(.{}); + const scores = try op.commit(op.contract(q, k, scores_axes).mulf(cfg.scale), .{}); + op = b.tensorOperation(.{}); + const s = op.fetch(scores, .{}); + return op.commit(op.where(op.fetch(mask, .{}), .ne, 0, s, score_min), .{}); +} + +fn maskedExp(b: *tcl.Builder, scores: builder.Tensor, maximum: builder.Tensor, mask: builder.Tensor) tcl.FinishError!builder.Tensor { + const op = b.tensorOperation(.{}); + const e = op.fetch(scores, .{}).subf(op.fetch(maximum, .{})).exp(); + // A sentinel minus itself gives exp(0), so masking only the logits fails for empty rows and fully masked chunks. + return op.commit(op.where(op.fetch(mask, .{}), .ne, 0, e, 0.0), .{}); +} + +fn expDifference(b: *tcl.Builder, x: builder.Tensor, maximum: builder.Tensor) tcl.FinishError!builder.Tensor { + const op = b.tensorOperation(.{ .tactic = .elementwise }); + const m = op.fetch(maximum, .{}); + return op.commit(op.fetch(x, .{}).subf(m).exp(), .{}); +} + +fn safeDenominator(b: *tcl.Builder, sum: builder.Tensor) tcl.FinishError!builder.Tensor { + const op = b.tensorOperation(.{}); + const s = op.fetch(sum, .{}); + return op.commit(op.where(s, .gt, 0, s, 1.0), .{}); +} + +const OneShot = tcl.Kernel(Cfg, .{ + .name = "attention", + .inputs = &.{ "q", "pages", "seq_lens", "query_offsets", "k_cache", "v_cache", "sink" }, + .outputs = &.{"out"}, + .run = struct { + fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { + configure(b, cfg); + const a: Axes = .init(b, cfg); + const A = b.axis("A", cfg.a); + const Nb = b.axis("Nb", cfg.pages * cfg.pg); + const t = try declareInputs(b, a); + const prepared = try prepareQueries(b, cfg, a, t); + const kc = try b.reshape(t.k_cache, &.{ Nb, a.Hkv, a.D }, .{}); + const vc = try b.reshape(t.v_cache, &.{ Nb, a.Hkv, a.D }, .{}); + const page_indices = try validIndices(b, try prepared.blockPages(b, a, prepared.pages), cfg.pages); + const within_page = try b.arange(cfg.pg, .i32, &.{a.PG}, .{}); + var slot_op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const flat_slots = try slot_op.commit(slot_op.fetch(page_indices.safe, .{}).muli(cfg.pg).addi(slot_op.fetch(within_page, .{})), .{}); + const expanded_slots = try b.reshape(flat_slots, &.{ a.B, A }, .{}); + slot_op = b.tensorOperation(.{}); + const valid = try slot_op.commit(slot_op.fetch(page_indices.valid, .{ .broadcast_to = &.{ a.B, a.AP, a.PG } }), .{}); + const valid_keys = try b.reshape(valid, &.{ a.B, A }, .{}); + slot_op = b.tensorOperation(.{}); + const slots = try slot_op.commit(slot_op.where(valid_keys, .ne, 0, slot_op.fetch(expanded_slots, .{}), 0), .{}); + const valid_length = if (cfg.decode) prepared.kvlen else null; + const position = try b.arange(cfg.a, .i32, &.{A}, .{}); + const valid_cache = try cacheMask(b, valid_keys, prepared.kvlen, position); + const k = try maskCacheValues(b, try b.gather(kc, slots, Nb, &.{ a.B, A, a.Hkv, a.D }, .{ .valid_length = valid_length }), valid_cache); + const v = try maskCacheValues(b, try b.gather(vc, slots, Nb, &.{ a.B, A, a.Hkv, a.D }, .{ .valid_length = valid_length }), valid_cache); + const mask = try attentionMask(b, cfg, prepared.qpos, prepared.kvlen, position, valid_keys); + const masked = try maskedScores(b, a, cfg, prepared.q, k, mask, A); + var op = b.tensorOperation(.{}); + const p = if (cfg.decode) compact: { + const exp_sink = if (cfg.use_sink) sink: { + const maximum = try op.commit(op.fetch(masked, .{}).reduce(&.{A}, .maxf).maxf(op.fetch(t.sink, .{})), .{}); + break :sink try expDifference(b, t.sink, maximum); + } else null; + op = b.tensorOperation(.{}); + const scores = op.fetch(masked, .{}); + var maximum = scores.reduce(&.{A}, .maxf); + if (cfg.use_sink) maximum = maximum.maxf(op.fetch(t.sink, .{})); + const e = scores.subf(maximum).exp(); + var sum = e.reduce(&.{A}, .addf); + if (exp_sink) |s| sum = sum.addf(op.fetch(s, .{})); + const probabilities = try op.commit(e.divf(sum), .{ .dtype = .bf16 }); + // Empty rows have uniform probabilities from finite sentinels: clear masked ones before the value contraction. + op = b.tensorOperation(.{}); + break :compact try op.commit(op.where(op.fetch(mask, .{}), .ne, 0, op.fetch(probabilities, .{}), 0.0), .{ .dtype = .bf16 }); + } else tiled: { + // Separate reductions schedule better for multi-query blocks with the current compiler. + var m = op.fetch(masked, .{}).reduce(&.{A}, .maxf); + if (cfg.use_sink) m = m.maxf(op.fetch(t.sink, .{})); + const maximum = try op.commit(m, .{}); + const e = try maskedExp(b, masked, maximum, mask); + const exp_sink = if (cfg.use_sink) try expDifference(b, t.sink, maximum) else null; + op = b.tensorOperation(.{}); + var sum = op.fetch(e, .{}).reduce(&.{A}, .addf); + if (exp_sink) |s| sum = sum.addf(op.fetch(s, .{})); + const denominator = try safeDenominator(b, try op.commit(sum, .{})); + op = b.tensorOperation(.{}); + break :tiled try op.commit(op.fetch(e, .{}).divf(op.fetch(denominator, .{})), .{ .dtype = .bf16 }); + }; + op = b.tensorOperation(.{}); + const output = try op.commit(op.contract(v, p, &.{ a.B, a.I, a.Hkv, a.G, a.D }), .{ .dtype = .bf16 }); + b.ret(&.{try restoreQueries(b, cfg, a, prepared, output)}); + } + }.run, +}); + +const Flash = tcl.Kernel(Cfg, .{ + .name = "flash_attention", + .inputs = &.{ "q", "pages", "seq_lens", "query_offsets", "k_cache", "v_cache", "sink" }, + .outputs = &.{"out"}, + .run = struct { + fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { + configure(b, cfg); + const a: Axes = .init(b, cfg); + const C = b.axis("C", @divExact(cfg.a, cfg.chunk)); + const Ck = b.axis("Ck", cfg.chunk); + const Cp = b.axis("Cp", @divExact(cfg.chunk, cfg.pg)); + const t = try declareInputs(b, a); + const prepared = try prepareQueries(b, cfg, a, t); + const rows = prepared.pages.axes[0]; + const by_row = try b.reshape(prepared.pages, &.{ rows, C, Cp }, .{}); + const page_op = b.tensorOperation(.{}); + const pages = try page_op.commit(page_op.fetch(by_row, .{}), .{ .axes = &.{ C, rows, Cp } }); + const stats = &.{ a.Hkv, a.G, a.B, a.I }; + const out_axes = &.{ a.B, a.I, a.Hkv, a.G, a.D }; + var op = b.tensorOperation(.{}); + const shift = std.math.log2_int(u64, @intCast(cfg.chunk)); + // With a window, each page row's loop starts at the chunk holding its first visible key. TCL divides + // by shifts, so chunks that are not a power of two run every chunk and the mask applies the window. + const window = if (cfg.window > 0 and std.math.isPowerOfTwo(cfg.chunk)) blk: { + op = b.tensorOperation(.{}); + const start = try op.commit(op.fetch(prepared.first, .{}).subi(cfg.window - 1).binary(.maxi, 0).binary(.shr_arith, shift), .{}); + op = b.tensorOperation(.{}); + const count = try op.commit(op.fetch(prepared.keys, .{}).addi(cfg.chunk - 1).binary(.shr_arith, shift).subi(op.fetch(start, .{})), .{}); + // WORKAROUND (TCC 2026.5.0b0): TCC reads the loop index only as an index, so each trip's chunk per + // row is built before the loop. Repro: repro_loop_index_as_data.py. Remove once the loop index + // compiles as data. + const trip = try b.arange(C.size, .i32, &.{C}, .{}); + op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const chunks = try op.commit(op.fetch(trip, .{}).addi(op.fetch(start, .{})), .{ .axes = &.{ C, rows } }); + break :blk .{ .chunks = chunks, .trips = try b.reduceMaxI32(count) }; + } else null; + const n = if (window) |w| w.trips else try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(prepared.kvlen), cfg.chunk - 1), cfg.chunk); + var loop = b.openFor(n, .{ try b.full(0, .f32, stats), try b.full(0, .f32, out_axes), try b.full(score_min, .f32, stats) }); + { + const sum_acc, const qkv_acc, const max_acc = loop.carried; + const local_pos = try b.arange(cfg.chunk, .i32, &.{Ck}, .{}); + const local_pages = try b.arange(@divExact(cfg.chunk, cfg.pg), .i32, &.{Cp}, .{}); + var row_pages: builder.Tensor = undefined; + var position: builder.Tensor = undefined; + var first_keys: builder.Tensor = undefined; + if (window) |w| { + const chunk_index = try b.indexRead(w.chunks, loop.iv); + op = b.tensorOperation(.{}); + const read_index = try op.commit(op.fetch(chunk_index, .{}).binary(.mini, @divExact(cfg.a, cfg.chunk) - 1), .{}); + // WORKAROUND (TCC 2026.5.0b0): a batched gather of the chunk does not lower + // for some shapes. Repro: repro_batched_gather_all_tactics_failed.py. Remove + // once batched gathers of several rows compile. + row_pages = try selectMetadata(b, by_row, try selector(b, read_index, C, C.size), C); + op = b.tensorOperation(.{}); + const row_offset = try op.commit(op.fetch(chunk_index, .{}).muli(cfg.chunk), .{}); + const offset = if (prepared.pick) |pick| try selectMetadata(b, row_offset, pick, a.S) else row_offset; + op = b.tensorOperation(.{}); + position = try op.commit(op.fetch(local_pos, .{ .broadcast_to = &.{ a.B, Ck } }).addi(op.fetch(offset, .{})), .{}); + op = b.tensorOperation(.{}); + first_keys = try op.commit(op.fetch(local_pages, .{ .broadcast_to = &.{ a.B, Cp } }).muli(cfg.pg).addi(op.fetch(offset, .{})), .{}); + } else { + row_pages = try b.indexRead(pages, loop.iv); + const offset = try b.symExpr(.mul, loop.iv, cfg.chunk); + op = b.tensorOperation(.{}); + position = try op.commit(op.fetch(local_pos, .{}).addi(op.fetch(offset, .{})), .{}); + op = b.tensorOperation(.{}); + first_keys = try op.commit(op.fetch(local_pages, .{}).muli(cfg.pg).addi(op.fetch(offset, .{})), .{}); + } + const indices = try validIndices(b, try prepared.blockPages(b, a, row_pages), cfg.pages); + const idx = indices.safe; + op = b.tensorOperation(.{}); + const valid_pages = try op.commit(op.fetch(indices.valid, .{ .broadcast_to = &.{ a.B, Cp, a.PG } }), .{}); + const valid_keys = try b.reshape(valid_pages, &.{ a.B, Ck }, .{}); + op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); + const remaining = try op.commit(op.fetch(prepared.kvlen, .{}).subi(op.fetch(first_keys, .{})), .{}); + op = b.tensorOperation(.{}); + const live_pages = try op.commit(op.where(remaining, .gt, 0, 1, 0), .{}); + op = b.tensorOperation(.{}); + const cvl = try op.commit(op.fetch(live_pages, .{}).reduce(&.{Cp}, .addi), .{}); + const mask = try attentionMask(b, cfg, prepared.qpos, prepared.kvlen, position, valid_keys); + const k = try b.reshape(try b.gather(t.k_cache, idx, a.P, &.{ a.B, Cp, a.PG, a.Hkv, a.D }, .{ .valid_length = cvl }), &.{ a.B, Ck, a.Hkv, a.D }, .{}); + const v = try b.reshape(try b.gather(t.v_cache, idx, a.P, &.{ a.B, Cp, a.PG, a.Hkv, a.D }, .{ .valid_length = cvl }), &.{ a.B, Ck, a.Hkv, a.D }, .{}); + const masked = try maskedScores(b, a, cfg, prepared.q, k, mask, Ck); + op = b.tensorOperation(.{}); + const max_cur = try op.commit(op.fetch(masked, .{}).reduce(&.{Ck}, .maxf), .{}); + op = b.tensorOperation(.{ .tactic = .elementwise }); + const max_new = try op.commit(op.fetch(max_cur, .{}).maxf(op.fetch(max_acc, .{})), .{}); + const rescale = try expDifference(b, max_acc, max_new); + const e = try maskedExp(b, masked, max_new, mask); + op = b.tensorOperation(.{}); + const sum_cur = try op.commit(op.fetch(e, .{}).reduce(&.{Ck}, .addf), .{}); + op = b.tensorOperation(.{}); + const sum_new = try op.commit(op.fetch(sum_acc, .{}).mulf(op.fetch(rescale, .{})).addf(op.fetch(sum_cur, .{})), .{}); + op = b.tensorOperation(.{}); + // Materialize the narrowing conversion through the vector engine. + const p = try op.commit(op.fetch(e, .{}).mulf(1.0), .{ .dtype = .bf16 }); + op = b.tensorOperation(.{}); + const qkv_cur = try op.commit(op.contract(p, v, out_axes), .{}); + op = b.tensorOperation(.{}); + const qkv_new = try op.commit(op.fetch(qkv_acc, .{}).mulf(op.fetch(rescale, .{})).addf(op.fetch(qkv_cur, .{})), .{}); + try loop.yield(.{ sum_new, qkv_new, max_new }); + } + + var sum = loop.results[0]; + var numerator = loop.results[1]; + if (cfg.use_sink) { + op = b.tensorOperation(.{}); + const maximum = try op.commit(op.fetch(loop.results[2], .{}).maxf(op.fetch(t.sink, .{})), .{}); + const alpha = try expDifference(b, loop.results[2], maximum); + const exp_sink = try expDifference(b, t.sink, maximum); + op = b.tensorOperation(.{}); + sum = try op.commit(op.fetch(sum, .{}).mulf(op.fetch(alpha, .{})).addf(op.fetch(exp_sink, .{})), .{}); + op = b.tensorOperation(.{}); + numerator = try op.commit(op.fetch(numerator, .{}).mulf(op.fetch(alpha, .{})), .{}); + } + const denominator = try safeDenominator(b, sum); + op = b.tensorOperation(.{}); + const output = try op.commit(op.fetch(numerator, .{}).divf(op.fetch(denominator, .{})), .{ .dtype = .bf16 }); + b.ret(&.{try restoreQueries(b, cfg, a, prepared, output)}); + } + }.run, +}); + +test { + std.testing.refAllDecls(attention_test); +} + +const attention_test = struct { + pub const Options = struct { + is_causal: bool = true, + sliding_window: i32 = -1, + scale: ?f32 = null, + sink: ?[]const f32 = null, + }; + + pub fn reference(allocator: std.mem.Allocator, q: zml.Slice, k: zml.Slice, v: zml.Slice, table: []const i32, lengths: []const i32, starts: []const i32, opts: Options) !zml.Slice { + const bf16 = zml.floats.BFloat16; + const hkv: usize = @intCast(q.shape.dim(.hkv)); + const groups: usize = @intCast(q.shape.dim(.hg)); + const d: usize = @intCast(q.shape.dim(.hd)); + const page: usize = @intCast(k.shape.dim(.k_chunk)); + const pages: usize = @intCast(k.shape.dim(.page)); + const max_pages = table.len / lengths.len; + const scale = opts.scale orelse 1.0 / @sqrt(@as(f32, @floatFromInt(d))); + const queries = q.constItems(bf16); + const keys = k.constItems(bf16); + const values = v.constItems(bf16); + const result = try zml.Slice.alloc(allocator, q.shape.withDtype(.f32)); + errdefer result.free(allocator); + const out = result.items(f32); + @memset(out, 0); + const scores = try allocator.alloc(f32, max_pages * page); + defer allocator.free(scores); + + for (lengths, 0..) |length, seq| { + const qlen = starts[seq + 1] - starts[seq]; + for (@intCast(starts[seq])..@intCast(starts[seq + 1])) |token| { + const qpos = length - qlen + @as(i32, @intCast(token)) - starts[seq]; + if (qpos < 0) continue; + for (0..hkv) |h| for (0..groups) |g| { + const offset = ((token * hkv + h) * groups + g) * d; + const sink = if (opts.sink) |s| s[h * groups + g] else -std.math.inf(f32); + var maximum = sink; + @memset(scores, -std.math.inf(f32)); + for (0..@intCast(length)) |key| { + const pos: i32 = @intCast(key); + if (opts.is_causal and pos > qpos) continue; + if (opts.sliding_window > 0 and qpos - pos >= opts.sliding_window) continue; + const physical = table[seq * max_pages + key / page]; + if (physical < 0 or physical >= pages) continue; + const kv_offset = ((@as(usize, @intCast(physical)) * page + key % page) * hkv + h) * d; + var dot: f32 = 0; + for (0..d) |i| dot += queries[offset + i].toF32() * keys[kv_offset + i].toF32(); + scores[key] = dot * scale; + maximum = @max(maximum, scores[key]); + } + if (maximum == -std.math.inf(f32)) continue; + var denominator = @exp(sink - maximum); + for (scores[0..@intCast(length)]) |score| denominator += @exp(score - maximum); + for (0..@intCast(length)) |key| { + if (scores[key] == -std.math.inf(f32)) continue; + const physical: usize = @intCast(table[seq * max_pages + key / page]); + const kv_offset = ((physical * page + key % page) * hkv + h) * d; + const probability = @exp(scores[key] - maximum) / denominator; + for (0..d) |i| out[offset + i] += probability * values[kv_offset + i].toF32(); + } + }; + } + } + return result; + } + + test "paged attention host reference scale and sink normalization" { + const bf16 = zml.floats.BFloat16; + const queries: [3]bf16 = @splat(.fromF32(1)); + const keys = [_]bf16{ .fromF32(0), .fromF32(1) }; + const values = [_]bf16{ .fromF32(2), .fromF32(4) }; + const q = zml.Slice.initConst(.init(.{ .b = 3, .hkv = 1, .hg = 1, .hd = 1 }, .bf16), std.mem.sliceAsBytes(&queries)); + const k = zml.Slice.initConst(.init(.{ .page = 1, .k_chunk = 2, .hkv = 1, .hd = 1 }, .bf16), std.mem.sliceAsBytes(&keys)); + const v = zml.Slice.initConst(k.shape, std.mem.sliceAsBytes(&values)); + const cases = [_]struct { opts: Options, expected: f32 }{ + .{ .opts = .{ .is_causal = false, .scale = 0 }, .expected = 3 }, + .{ .opts = .{ .is_causal = false, .scale = @log(@as(f32, 3)) }, .expected = 3.5 }, + .{ .opts = .{ .is_causal = false, .scale = @log(@as(f32, 3)), .sink = &.{0} }, .expected = 2.8 }, + }; + for (cases) |case| { + const result = try reference(std.testing.allocator, q, k, v, &.{0}, &.{2}, &.{ 0, 2 }, case.opts); + defer result.free(std.testing.allocator); + for (result.constItems(f32)[0..2]) |value| try std.testing.expectApproxEqAbs(case.expected, value, 1e-6); + try std.testing.expectEqual(@as(f32, 0), result.constItems(f32)[2]); + } + } + + pub fn expectClose(expected: zml.Slice, actual: zml.Slice) !void { + try std.testing.expectEqualSlices(i64, expected.shape.dims(), actual.shape.dims()); + for (expected.constItems(f32), 0..) |want, i| { + const got = floatAt(actual, i); + try std.testing.expect(std.math.isFinite(got)); + std.testing.expectApproxEqAbs(want, got, 3e-2 + 3e-2 * @abs(want)) catch |err| { + std.log.err("paged attention mismatch at element {d}: expected {d}, got {d}", .{ i, want, got }); + return err; + }; + } + } + + fn floatAt(slice: zml.Slice, index: usize) f32 { + return switch (slice.shape.dtype()) { + .bf16 => slice.constItems(zml.floats.BFloat16)[index].toF32(), + .f16 => @floatCast(slice.constItems(f16)[index]), + .f32 => slice.constItems(f32)[index], + else => unreachable, + }; + } + + test "paged attention host reference windows sinks and padding" { + const bf16 = zml.floats.BFloat16; + const queries: [3]bf16 = @splat(.fromF32(0)); + const keys: [2]bf16 = @splat(.fromF32(0)); + const values = [_]bf16{ .fromF32(2), .fromF32(4) }; + const q = zml.Slice.initConst(.init(.{ .b = 3, .hkv = 1, .hg = 1, .hd = 1 }, .bf16), std.mem.sliceAsBytes(&queries)); + const k = zml.Slice.initConst(.init(.{ .page = 1, .k_chunk = 2, .hkv = 1, .hd = 1 }, .bf16), std.mem.sliceAsBytes(&keys)); + const v = zml.Slice.initConst(k.shape, std.mem.sliceAsBytes(&values)); + const cases = [_]struct { opts: Options, expected: [3]f32 }{ + .{ .opts = .{}, .expected = .{ 2, 3, 0 } }, + .{ .opts = .{ .sliding_window = 1 }, .expected = .{ 2, 4, 0 } }, + .{ .opts = .{ .is_causal = false, .sliding_window = 1 }, .expected = .{ 3, 4, 0 } }, + .{ .opts = .{ .sink = &.{0} }, .expected = .{ 1, 2, 0 } }, + .{ .opts = .{ .sink = &.{-std.math.inf(f32)} }, .expected = .{ 2, 3, 0 } }, + }; + for (cases) |case| { + const result = try reference(std.testing.allocator, q, k, v, &.{0}, &.{2}, &.{ 0, 2 }, case.opts); + defer result.free(std.testing.allocator); + try std.testing.expectEqualSlices(f32, &case.expected, result.constItems(f32)); + } + } +}; + +const TestCase = struct { + prefill: bool, + seqs: []const [2]i32, + padding: i64 = 0, + opts: attention_test.Options = .{}, + mask_all: bool = false, + padding_page: i32 = -1, + poison: enum { none, k, v, both } = .none, + poison_value: f32 = std.math.nan(f32), +}; + +const TestDims = struct { + batch: usize = 4, + hkv: usize = 2, + g: usize = 4, + d: usize = 64, + page: usize = 16, + uniform: bool = false, + uniform_scores: bool = false, + spikes: []const usize = &.{}, + dtype: zml.DataType = .bf16, +}; + +test "tcl paged attention unit scale and dtype boundaries on furiosa" { + if (zml.testing.env().target != .furiosa) return error.SkipZigTest; + try runCases(16, 32, &.{.{ .prefill = true, .seqs = &@as([4][2]i32, @splat(.{ 32, 256 })), .opts = .{ .scale = 1.0 } }}, .{}); + inline for (.{ .f16, .f32 }) |dtype| { + try runCases(2, 8, &.{.{ .prefill = false, .seqs = &.{ .{ 1, 17 }, .{ 1, 1 }, .{ 1, 3 }, .{ 0, 0 } }, .padding = 1, .opts = .{ .sink = &.{ 0, -std.math.inf(f32) } } }}, .{ .hkv = 1, .g = 2, .d = 32, .dtype = dtype }); + } +} + +test "tcl paged attention single decode row needs no rearrangement" { + if (zml.testing.env().target != .furiosa) return error.SkipZigTest; + try runCases(16, 32, &.{ + .{ .prefill = false, .seqs = &.{ .{ 1, 251 }, .{ 0, 0 }, .{ 0, 0 }, .{ 0, 0 } } }, + .{ .prefill = false, .seqs = &@as([4][2]i32, @splat(.{ 0, 0 })), .padding = 1 }, + }, .{ .batch = 1 }); + inline for (.{ 257, 320 }) |max_pages| { + try runCases(max_pages, 2 * max_pages, &.{.{ + .prefill = false, + .seqs = &.{ .{ 1, @intCast(max_pages * 16 - 1) }, .{ 0, 0 }, .{ 0, 0 }, .{ 0, 0 } }, + }}, .{ .batch = 1, .hkv = 8, .g = 4, .d = 128 }); + } + inline for (.{ 5, 1025 }) |max_pages| { + try runCases(max_pages, 2 * max_pages, &.{.{ + .prefill = false, + .seqs = &.{ .{ 1, @intCast(max_pages * 48 - 1) }, .{ 0, 0 }, .{ 0, 0 }, .{ 0, 0 } }, + }}, .{ .batch = 1, .hkv = 1, .g = 1, .d = 32, .page = 48 }); + } +} + +test "tcl paged attention packs inactive leading sequences and odd batches" { + if (zml.testing.env().target != .furiosa) return error.SkipZigTest; + try runCases(16, 32, &.{ + .{ .prefill = false, .seqs = &.{ .{ 0, 17 }, .{ 1, 63 }, .{ 0, 23 }, .{ 1, 65 } }, .padding = 2 }, + .{ .prefill = true, .seqs = &.{ .{ 0, 17 }, .{ 33, 64 }, .{ 0, 23 }, .{ 2, 3 } }, .padding = 3 }, + }, .{}); + try runCases(2, 8, &.{ + .{ .prefill = false, .seqs = &.{ .{ 0, 9 }, .{ 1, 17 }, .{ 1, 1 }, .{ 0, 0 } }, .padding = 1 }, + }, .{ .batch = 3, .hkv = 1, .g = 2, .d = 32 }); +} + +test "tcl paged attention masks inactive decode rows across all heads" { + if (zml.testing.env().target != .furiosa) return error.SkipZigTest; + try runCases(32, 64, &.{.{ + .prefill = false, + .seqs = &.{ .{ 1, 511 }, .{ 1, 504 }, .{ 1, 497 }, .{ 1, 490 }, .{ 0, 483 }, .{ 1, 476 }, .{ 1, 469 }, .{ 1, 462 } }, + .padding = 1, + }}, .{ .batch = 8 }); +} + +test "tcl paged attention invalid pages cannot wrap into slot zero" { + if (zml.testing.env().target != .furiosa) return error.SkipZigTest; + const cases = [_]TestCase{ + .{ .prefill = false, .seqs = &@as([4][2]i32, @splat(.{ 1, 17 })), .mask_all = true, .padding_page = 1 << 28 }, + .{ .prefill = false, .seqs = &@as([4][2]i32, @splat(.{ 1, 17 })), .mask_all = true, .padding_page = std.math.maxInt(i32) }, + }; + try runCases(2, 8, &cases, .{ .hkv = 1, .g = 1, .d = 32 }); +} + +test "tcl paged attention emits decode and prefill kernels with and without sinks" { + inline for (.{ false, true }) |decode| { + inline for (.{ false, true }) |use_sink| { + const cfg: Cfg = .{ .tokens = if (decode) 2 else 8, .sequences = 2, .b = if (decode) 2 else 3, .i = if (decode) 1 else prefill_block, .a = 32, .chunk = 16, .hkv = 2, .g = 4, .d = 32, .pages = 8, .pg = 16, .scale = 0.125, .decode = decode, .use_sink = use_sink }; + const one_shot = try OneShot.emit(std.testing.allocator, cfg); + defer std.testing.allocator.free(one_shot); + const flash = try Flash.emit(std.testing.allocator, cfg); + defer std.testing.allocator.free(flash); + for ([_][]const u8{ one_shot, flash }) |ir| { + try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, ir, "\"tcl.dpe\"")); + try std.testing.expect(std.mem.indexOf(u8, ir, "!tcl.logical") == null); + try std.testing.expect(std.mem.indexOf(u8, ir, "#tcl.symbol<\"I\">") == null); + + const decode = try Decode.emit(std.testing.allocator, .{ .t = 2, .a = 64, .group = 2, .hkv = 2, .g = 4, .d = 32, .pages = 8, .pg = 16, .scale = 0.125, .is_causal = true, .window = 0, .use_sink = false }); + defer std.testing.allocator.free(decode); + try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, decode, "\"tcl.dpe\"")); + try std.testing.expect(std.mem.indexOf(u8, decode, "tcl.fusible") != null); + try std.testing.expect(std.mem.indexOf(u8, decode, "propagate_sparse_axis_from_op = \"Gather\"") != null); +} + +test "tcl paged attention excludes poisoned unused cache entries on furiosa" { + if (zml.testing.env().target != .furiosa) return error.SkipZigTest; + const seqs = [_][2]i32{ .{ 1, 17 }, .{ 1, 31 }, .{ 0, 0 }, .{ 0, 23 } }; + try runCases(8, 32, &.{ + .{ .prefill = false, .seqs = &seqs, .padding = 2, .poison = .k }, + .{ .prefill = false, .seqs = &seqs, .padding = 2, .poison = .v }, + .{ .prefill = false, .seqs = &seqs, .padding = 2, .poison = .both }, + .{ .prefill = false, .seqs = &seqs, .padding = 2, .poison = .both, .poison_value = std.math.inf(f32) }, + .{ .prefill = false, .seqs = &seqs, .padding = 2, .poison = .both, .poison_value = -std.math.inf(f32) }, + .{ .prefill = false, .seqs = &seqs, .padding = 2, .poison = .both, .poison_value = 1e30 }, + // A sink selects OneShot's compact decode softmax instead of Decode. + .{ .prefill = false, .seqs = &seqs, .padding = 2, .poison = .both, .opts = .{ .scale = 0.2, .sink = &.{ 0, -std.math.inf(f32) } } }, + .{ .prefill = false, .seqs = &seqs, .padding = 2, .poison = .both, .mask_all = true, .padding_page = std.math.maxInt(i32) }, + .{ .prefill = false, .seqs = &.{ .{ 0, 0 }, .{ 0, 0 }, .{ 0, 0 }, .{ 0, 0 } }, .padding = 4, .poison = .both }, + }, .{ .hkv = 1, .g = 2, .d = 32 }); + + const mixed = [_][2]i32{ .{ 37, 65 }, .{ 0, 19 }, .{ 1, 17 }, .{ 4, 2 } }; + try runCases(8, 32, &.{ + .{ .prefill = true, .seqs = &mixed, .padding = 3, .poison = .k }, + .{ .prefill = true, .seqs = &mixed, .padding = 3, .poison = .v }, + .{ .prefill = true, .seqs = &mixed, .padding = 3, .poison = .both, .opts = .{ .is_causal = false, .sliding_window = 17, .scale = 0.2, .sink = &.{ 0, -std.math.inf(f32) } } }, + }, .{ .hkv = 1, .g = 2, .d = 32 }); + + const boundary = [_][2]i32{ .{ 37, 4097 }, .{ 0, 0 }, .{ 1, 4096 }, .{ 2, 17 } }; + try runCases(257, 1028, &.{ + .{ .prefill = true, .seqs = &boundary, .padding = 3, .poison = .both }, + .{ .prefill = true, .seqs = &boundary, .padding = 3, .poison = .both, .poison_value = std.math.inf(f32), .opts = .{ .is_causal = false, .sliding_window = 33, .scale = 0.2, .sink = &.{ 0, -std.math.inf(f32) } } }, + .{ .prefill = false, .seqs = &.{ .{ 1, 4095 }, .{ 1, 4096 }, .{ 1, 4097 }, .{ 0, 0 } }, .padding = 1, .poison = .both, .opts = .{ .sliding_window = 17, .sink = &.{ 0, -std.math.inf(f32) } } }, + }, .{ .hkv = 1, .g = 2, .d = 32 }); + try runCases(86, 344, &.{.{ + .prefill = true, + .seqs = &.{ .{ 3, 4033 }, .{ 1, 4032 }, .{ 0, 0 }, .{ 1, 4031 } }, + .padding = 1, + .poison = .both, + .poison_value = -std.math.inf(f32), + .opts = .{ .sliding_window = 49 }, + }}, .{ .hkv = 1, .g = 1, .d = 32, .page = 48 }); +} + +test "tcl paged attention runs on furiosa" { + if (zml.testing.env().target != .furiosa) return error.SkipZigTest; + inline for (.{ 16, 320 }) |max_pages| { + const cases = if (max_pages == 16) [_]TestCase{ + .{ .prefill = true, .seqs = &.{ .{ 37, 200 }, .{ 1, 77 }, .{ 20, 20 }, .{ 0, 0 } } }, + .{ .prefill = false, .seqs = &.{ .{ 1, 250 }, .{ 1, 3 }, .{ 1, 64 }, .{ 1, 129 } } }, + } else [_]TestCase{ + .{ .prefill = true, .seqs = &.{ .{ 37, 4500 }, .{ 1, 77 }, .{ 70, 70 }, .{ 0, 0 } } }, + .{ .prefill = false, .seqs = &.{ .{ 1, 5000 }, .{ 1, 3 }, .{ 1, 4097 }, .{ 1, 129 } } }, + }; + try runCases(max_pages, 2 * max_pages, &cases, .{}); + } +} + +test "tcl paged attention options and inactive queries on furiosa" { + if (zml.testing.env().target != .furiosa) return error.SkipZigTest; + const sinks = [_]f32{ -std.math.inf(f32), -5, 0, 2, 8, -1, 1, 20 }; + const disabled: [8]f32 = @splat(-std.math.inf(f32)); + const mixed = [4][2]i32{ .{ 37, 200 }, .{ 0, 0 }, .{ 1, 77 }, .{ 20, 20 } }; + const cases = [_]TestCase{ + .{ .prefill = true, .seqs = &mixed, .padding = 5, .opts = .{ .sliding_window = 17, .scale = 0.2, .sink = &sinks } }, + .{ .prefill = true, .seqs = &mixed, .padding = 5, .opts = .{ .is_causal = false, .sliding_window = 17, .scale = 0.2, .sink = &sinks } }, + .{ .prefill = true, .seqs = &mixed, .padding = 5, .opts = .{ .is_causal = false, .sliding_window = 0 } }, + .{ .prefill = true, .seqs = &mixed, .padding = 5, .opts = .{ .sliding_window = 1, .sink = &disabled } }, + .{ .prefill = false, .seqs = &.{ .{ 1, 251 }, .{ 0, 73 }, .{ 1, 1 }, .{ 0, 17 } }, .padding = 2, .opts = .{ .sink = &sinks } }, + .{ .prefill = false, .seqs = &@as([4][2]i32, @splat(.{ 0, 0 })), .padding = 4 }, + .{ .prefill = false, .seqs = &@as([4][2]i32, @splat(.{ 1, 17 })), .mask_all = true }, + .{ .prefill = false, .seqs = &@as([4][2]i32, @splat(.{ 1, 17 })), .mask_all = true, .opts = .{ .sink = &sinks } }, + .{ .prefill = false, .seqs = &.{ .{ 1, 251 }, .{ 0, 73 }, .{ 1, 20 }, .{ 0, 17 } }, .padding = 2, .opts = .{ .is_causal = false, .sliding_window = 17, .scale = 0.2, .sink = &sinks } }, + .{ .prefill = false, .seqs = &.{ .{ 1, 251 }, .{ 0, 73 }, .{ 1, 20 }, .{ 0, 17 } }, .padding = 2, .opts = .{ .sliding_window = 1, .sink = &disabled } }, + }; + try runCases(16, 32, &cases, .{}); +} + +test "tcl paged attention chunk boundaries on furiosa" { + if (zml.testing.env().target != .furiosa) return error.SkipZigTest; + const cases = [_]TestCase{ + .{ .prefill = true, .seqs = &.{ .{ 37, 4097 }, .{ 0, 0 }, .{ 1, 4096 }, .{ 2, 4095 } }, .padding = 7, .opts = .{ .is_causal = false, .sliding_window = 33, .scale = 0.2, .sink = &.{ 0, 10 } } }, + .{ .prefill = false, .seqs = &.{ .{ 1, 4095 }, .{ 1, 4096 }, .{ 1, 4097 }, .{ 0, 0 } }, .padding = 1, .opts = .{ .sliding_window = 17, .sink = &.{ -std.math.inf(f32), 0 } } }, + .{ .prefill = false, .seqs = &@as([4][2]i32, @splat(.{ 0, 0 })), .padding = 4 }, + .{ .prefill = false, .seqs = &@as([4][2]i32, @splat(.{ 1, 4097 })), .mask_all = true, .opts = .{ .sink = &.{ -std.math.inf(f32), 0 } } }, + }; + try runCases(257, 514, &cases, .{ .hkv = 1, .g = 2, .d = 32 }); + try runCases(256, 512, &.{.{ .prefill = false, .seqs = &.{ .{ 1, 4095 }, .{ 1, 4096 }, .{ 0, 0 }, .{ 0, 0 } }, .padding = 2 }}, .{ .hkv = 1, .g = 1, .d = 32 }); + try runCases(86, 172, &.{.{ .prefill = false, .seqs = &.{ .{ 1, 4079 }, .{ 1, 4080 }, .{ 1, 4081 }, .{ 0, 0 } }, .padding = 1, .opts = .{ .sliding_window = 49 } }}, .{ .hkv = 1, .g = 1, .d = 32, .page = 48 }); +} + +test "tcl paged attention preserves constant values across chunks" { + if (zml.testing.env().target != .furiosa) return error.SkipZigTest; + const cases = [_]TestCase{ + .{ .prefill = false, .seqs = &.{ .{ 1, 64 }, .{ 1, 65 }, .{ 1, 66 }, .{ 0, 0 } }, .padding = 1 }, + .{ .prefill = false, .seqs = &.{ .{ 1, 4095 }, .{ 1, 4096 }, .{ 1, 4097 }, .{ 0, 0 } }, .padding = 1 }, + .{ .prefill = false, .seqs = &.{ .{ 1, 4095 }, .{ 1, 4096 }, .{ 1, 4097 }, .{ 0, 0 } }, .padding = 1, .opts = .{ .sliding_window = 1, .sink = &.{0} } }, + .{ .prefill = true, .seqs = &.{ .{ 37, 4097 }, .{ 0, 0 }, .{ 1, 77 }, .{ 2, 2 } }, .padding = 3 }, + }; + try runCases(257, 514, &cases, .{ .hkv = 1, .g = 1, .d = 32, .uniform = true }); +} + +test "tcl paged attention chunked decode on furiosa" { + if (zml.testing.env().target != .furiosa) return error.SkipZigTest; + const cases = [_]TestCase{ + .{ .prefill = false, .seqs = &.{ .{ 1, 64 }, .{ 1, 65 }, .{ 1, 66 }, .{ 0, 0 } }, .padding = 1 }, + .{ .prefill = false, .seqs = &.{ .{ 1, 4095 }, .{ 1, 4096 }, .{ 1, 4097 }, .{ 0, 0 } }, .padding = 1 }, + }; + try runCases(257, 514, &cases, .{ .hkv = 1, .g = 1, .d = 32 }); +} + +test "tcl paged attention hybrid decode on furiosa" { + if (zml.testing.env().target != .furiosa) return error.SkipZigTest; + try runCases(2049, 2101, &.{ + .{ .prefill = false, .seqs = &.{.{ 1, 300 }}, .poison = .both }, + .{ .prefill = false, .seqs = &.{.{ 1, 4097 }}, .poison = .both }, + .{ .prefill = false, .seqs = &.{.{ 1, 32784 }}, .poison = .both, .poison_value = std.math.inf(f32) }, + .{ .prefill = false, .seqs = &.{.{ 0, 0 }}, .padding = 1, .poison = .both }, + .{ .prefill = false, .seqs = &.{.{ 1, 17 }}, .mask_all = true, .poison = .both }, + }, .{ .batch = 1, .hkv = 1, .g = 2, .d = 32 }); + try runCases(2049, 2101, &.{.{ .prefill = false, .seqs = &.{.{ 1, 32784 }}, .poison = .both }}, .{ .batch = 1, .hkv = 1, .g = 2, .d = 32, .uniform_scores = true, .spikes = &.{ 4095, 4096, 12287, 12288, 32783 } }); +} + +test "tcl paged attention flash decode of many tokens on furiosa" { + if (zml.testing.env().target != .furiosa) return error.SkipZigTest; + const seqs = [_][2]i32{ .{ 1, 32784 }, .{ 1, 4097 }, .{ 1, 1 }, .{ 0, 0 } } ++ @as([12][2]i32, @splat(.{ 1, 300 })); + try runCases(2049, 2600, &.{ + .{ .prefill = false, .seqs = &seqs, .padding = 1, .poison = .both }, + .{ .prefill = false, .seqs = &seqs, .padding = 1, .poison = .both, .opts = .{ .sliding_window = 1024 } }, + .{ .prefill = false, .seqs = &seqs, .padding = 1, .poison = .both, .opts = .{ .sliding_window = 5000, .sink = &.{ 0, -std.math.inf(f32) } } }, + }, .{ .batch = 16, .hkv = 1, .g = 2, .d = 32 }); + try runCases(2049, 2700, &.{ + .{ .prefill = true, .seqs = &.{ .{ 37, 32784 }, .{ 5, 9000 } }, .padding = 2, .poison = .both, .opts = .{ .sliding_window = 1024 } }, + .{ .prefill = true, .seqs = &.{ .{ 70, 32784 }, .{ 0, 0 } }, .padding = 1, .poison = .both, .opts = .{ .sliding_window = 3000 } }, + }, .{ .batch = 2, .hkv = 1, .g = 2, .d = 32 }); +} + +test "tcl paged attention uniform probabilities on furiosa" { + if (zml.testing.env().target != .furiosa) return error.SkipZigTest; + try runCases(257, 514, &.{.{ .prefill = false, .seqs = &.{ .{ 1, 64 }, .{ 1, 65 }, .{ 1, 66 }, .{ 0, 0 } }, .padding = 1 }}, .{ .hkv = 1, .g = 1, .d = 32, .uniform_scores = true }); +} + +fn runCases(comptime max_pages: usize, comptime num_pages: usize, cases: []const TestCase, comptime dims: TestDims) !void { + const platform = zml.testing.env(); + const allocator = std.testing.allocator; + const io = std.testing.io; + const bf16 = zml.floats.BFloat16; + const hkv = dims.hkv; + const g = dims.g; + const d = dims.d; + const page = dims.page; + const batch = dims.batch; + var prng: std.Random.DefaultPrng = .init(7); + const r = prng.random(); + const kc = try allocator.alloc(bf16, num_pages * page * hkv * d); + defer allocator.free(kc); + const vc = try allocator.alloc(bf16, num_pages * page * hkv * d); + defer allocator.free(vc); + const k_t: Tensor = .init(.{ .page = num_pages, .k_chunk = page, .hkv = hkv, .hd = d }, dims.dtype); + + for (cases) |case| { + var tokens: i64 = case.padding; + for (case.seqs[0..batch]) |s| tokens += s[0]; + const params: paged.Parameters = .init(.{ .batch_size = batch, .max_num_pages = max_pages, .max_seqlen_q = 128, .is_prefill = case.prefill }); + const q_t: Tensor = .init(.{ .b = tokens, .hkv = hkv, .hg = g, .hd = d }, dims.dtype); + const Mod = struct { + pub fn forward(p: paged.Parameters, q: Tensor, k: Tensor, v: Tensor, opts: AttentionOptions) Tensor { + return paged.pagedAttention(p, q, k, v, opts); + } + }; + const opts: AttentionOptions = .{ + .mask = .{ .standard = .{ + .causal = case.opts.is_causal, + .sliding_window = if (case.opts.sliding_window > 0) @intCast(case.opts.sliding_window) else null, + } }, + .scale = case.opts.scale, + .sink = if (case.opts.sink != null) Tensor.init(.{ .h = hkv * g }, .f32) else null, + }; + const v_t: Tensor = .init(k_t.shape(), dims.dtype); + var exe = try zml.module.compile(allocator, io, Mod.forward, .{ params, q_t, k_t, v_t, opts }, platform, .{}); + defer exe.deinit(); + + const q = try allocator.alloc(bf16, @intCast(tokens * hkv * g * d)); + defer allocator.free(q); + for (q) |*v| v.* = .fromF32(if (dims.uniform or dims.uniform_scores) 0 else r.floatNorm(f32)); + var table: [batch][max_pages]i32 = @splat(@splat(case.padding_page)); + var seq_lens: [batch]i32 = undefined; + var starts: [batch + 1]i32 = undefined; + starts[0] = 0; + var next_page: i32 = 0; + for (case.seqs[0..batch], 0..) |s, seq| { + seq_lens[seq] = s[1]; + starts[seq + 1] = starts[seq] + s[0]; + if (!case.mask_all) { + for (0..@intCast(std.math.divCeil(i32, s[1], page) catch unreachable)) |p| { + table[seq][p] = @mod(next_page * 7, @as(i32, num_pages)); + next_page += 1; + } + } + } + + for (kc) |*v| v.* = .fromF32(if (dims.uniform or dims.uniform_scores) 0 else r.floatNorm(f32)); + for (vc, 0..) |*v, i| v.* = .fromF32(if (dims.uniform) @as(f32, @floatFromInt(i % d)) / d else r.floatNorm(f32)); + if (dims.spikes.len > 0) { + for (case.seqs[0..batch], 0..) |seq, row| { + for (0..@intCast(seq[1])) |key| { + const slot: usize = @intCast(table[row][key / page] * @as(i32, page) + @as(i32, @intCast(key % page))); + const spike = std.mem.indexOfScalar(usize, dims.spikes, key) != null; + @memset(vc[slot * hkv * d ..][0 .. hkv * d], bf16.fromF32(if (spike) 32768 else 0)); + } + } + } + if (case.poison != .none) { + const live = try allocator.alloc(bool, num_pages * page); + defer allocator.free(live); + @memset(live, false); + for (case.seqs[0..batch], 0..) |s, seq| { + if (s[0] == 0) continue; + for (0..@intCast(s[1])) |key| { + const physical = table[seq][key / page]; + if (physical < 0 or physical >= num_pages) continue; + live[@as(usize, @intCast(physical)) * page + key % page] = true; + } + } + const poison = bf16.fromF32(case.poison_value); + for (live, 0..) |valid, slot| { + if (valid) continue; + const start = slot * hkv * d; + if (case.poison == .k or case.poison == .both) @memset(kc[start..][0 .. hkv * d], poison); + if (case.poison == .v or case.poison == .both) @memset(vc[start..][0 .. hkv * d], poison); + } + } + var bk = try testBuffer(platform, k_t.shape(), kc); + defer bk.deinit(); + var bv = try testBuffer(platform, k_t.shape(), vc); + defer bv.deinit(); + var bq = try testBuffer(platform, q_t.shape(), q); + defer bq.deinit(); + var bp: zml.Bufferized(paged.Parameters) = .{ + .block_table = try .fromBytes(io, platform, params.block_table.shape(), std.mem.sliceAsBytes(&table)), + .seq_lens = try .fromBytes(io, platform, params.seq_lens.shape(), std.mem.sliceAsBytes(&seq_lens)), + .query_start_len = try .fromBytes(io, platform, params.query_start_len.shape(), std.mem.sliceAsBytes(&starts)), + }; + defer zml.Buffer.deinitAll(paged.Parameters, &bp); + var opts_d: zml.Bufferized(AttentionOptions) = .{ + .mask = .{ .standard = {} }, + .sink = if (case.opts.sink) |s| try .fromBytes(io, platform, opts.sink.?.shape(), std.mem.sliceAsBytes(s)) else null, + }; + defer zml.Buffer.deinitAll(AttentionOptions, &opts_d); + var result = try exe.eval(allocator, io, .{ bp, bq, bk, bv, opts_d }); + defer result.deinit(); + const host = try result.toSliceAlloc(allocator, io); + defer host.free(allocator); + const expected = try attention_test.reference( + allocator, + .initConst(q_t.shape().withDtype(.bf16), std.mem.sliceAsBytes(q)), + .initConst(k_t.shape().withDtype(.bf16), std.mem.sliceAsBytes(kc)), + .initConst(k_t.shape().withDtype(.bf16), std.mem.sliceAsBytes(vc)), + @as([*]const i32, @ptrCast(&table))[0 .. batch * max_pages], + &seq_lens, + &starts, + case.opts, + ); + defer expected.free(allocator); + try std.testing.expectEqual(dims.dtype, host.shape.dtype()); + attention_test.expectClose(expected, host) catch |err| { + std.log.err("TCL attention fixture: {any}", .{case}); + return err; + }; + const active_elements = if (case.mask_all) 0 else @as(usize, @intCast(starts[batch])) * hkv * g * d; + for (active_elements..@intCast(tokens * hkv * g * d)) |index| try std.testing.expectEqual(@as(f32, 0), attention_test.floatAt(host, index)); + for (case.seqs[0..batch], 0..) |seq, index| { + const empty_queries: usize = @intCast(@max(0, seq[0] - seq[1])); + const begin: usize = @as(usize, @intCast(starts[index])) * hkv * g * d; + for (begin..begin + empty_queries * hkv * g * d) |element| try std.testing.expectEqual(@as(f32, 0), attention_test.floatAt(host, element)); + } + } +} + +fn testBuffer(platform: *const zml.Platform, shape: zml.Shape, values: []const zml.floats.BFloat16) !zml.Buffer { + const slice = try zml.Slice.alloc(std.testing.allocator, shape); + defer slice.free(std.testing.allocator); + switch (shape.dtype()) { + .bf16 => @memcpy(slice.items(zml.floats.BFloat16), values), + .f16 => for (slice.items(f16), values) |*out, value| { + out.* = @floatCast(value.toF32()); + }, + .f32 => for (slice.items(f32), values) |*out, value| { + out.* = value.toF32(); + }, + else => unreachable, + } + return .fromBytes(std.testing.io, platform, shape, slice.bytes); +} diff --git a/zml/kernels/paged_attention/triton/BUILD.bazel b/zml/kernels/paged_attention/triton/BUILD.bazel new file mode 100644 index 00000000..eff7d096 --- /dev/null +++ b/zml/kernels/paged_attention/triton/BUILD.bazel @@ -0,0 +1,11 @@ +package(default_visibility = ["//zml:__pkg__"]) + +filegroup( + name = "triton", + srcs = [ + "sparse_unified_attention.zig", + "triton.zig", + "unified_attention.zig", + "unified_attention_oneapi.zig", + ], +) diff --git a/zml/kernels/paged_attention/triton/README.md b/zml/kernels/paged_attention/triton/README.md new file mode 100644 index 00000000..37aba88b --- /dev/null +++ b/zml/kernels/paged_attention/triton/README.md @@ -0,0 +1,57 @@ +# Triton paged attention + +This backend is automatically selected for CUDA, ROCm, and oneAPI. See the +[parent README](../README.md) for the attention equation, packed sequence +metadata, and page addressing contract. + +[Triton's wrapper](triton.zig) uses split KV caches and supports causal and +non-causal attention, a left sliding window, explicit key indices, and softmax +sinks. Query heads are grouped by KV head. Keep complete GQA groups together +when sharding `.hkv`, with matching query and cache ownership and complete +`.hg` and `.hd` axes. Each local computation needs its full key range and the +matching sequence metadata. + +For standard masks, the wrapper chooses between a 2D kernel and a 3D split-key +kernel followed by segment reduction. Prefill uses the 2D path; all-decode +batches select a path based on batch size and KV head count. Explicit indices +use the dedicated sparse path described below. oneAPI forces split-key execution for +all-decode batches and has a specialized path for standard masks. Segment +reduction combines softmax maxima, exponential sums, and weighted values to +preserve one normalization over the key range. + +Set `is_prefill = true` for mixed prefill/decode batches. `max_seqlen_q` guides +kernel configuration. These options do not change logical query positions. + +The shared tests in [paged_attention.zig](../paged_attention.zig) exercise +`bf16`, mixed prefill/decode, shared prefix pages, partial final KV pages, +trailing inactive sequences, and option-specific comparisons. Kernel selection +and numerical behavior depend on the target; these cases do not establish +support for every shape or dtype. + +## Sparse attention + +`.indices` mask dispatches to a dedicated sparse kernel in +`sparse_unified_attention.zig`. +The `i32` tensor has shape `{ .b = T, .topk = K }`, with one list per packed +query shared across heads. Each entry is a logical position within that query's +sequence, including its cached prefix. The kernel translates each selected +position through the existing page table; it does not compute top-k selection. +Negative and out-of-sequence entries are ignored, including holes before valid +tail entries. Duplicate entries count repeatedly. Indices replace the standard +mask, so they do not implicitly enforce causality. Empty selections return zero; +optional attention sinks contribute only to the softmax denominator. + +Prefill processes one query and one KV-head group per program and writes the +normalized result directly. Decode uses 16- or 32-position gather tiles and an +occupancy-based split count bounded by selection length. Single-token decode +addresses sequence slots directly; other query layouts use packed-query lookup. +Inactive sequence slots and unused query capacity are skipped. Only segments +covering selection slots launch. Small empty segments skip Q and KV loading, +and larger segments skip wholly empty tiles. Known strides and scale are +specialized, and sparse gathers use the default cache policy. + +Split-K decode writes FP32 partials and contiguous-head statistics. Its reducer +handles at most 64 features per program, ignores unlaunched and zero-mass +segments, and never reads their unwritten partial outputs. The direct and split +paths use the same online softmax and support non-power-of-two GQA groups and +padded head dimensions. Dense attention retains its existing dispatch and kernels. diff --git a/zml/kernels/paged_attention/triton/sparse_unified_attention.zig b/zml/kernels/paged_attention/triton/sparse_unified_attention.zig new file mode 100644 index 00000000..79113f1e --- /dev/null +++ b/zml/kernels/paged_attention/triton/sparse_unified_attention.zig @@ -0,0 +1,326 @@ +const std = @import("std"); +const zml = @import("../../../zml.zig"); +const tri = zml.kernel.triton; +const Builder = tri.Builder; +const Value = tri.Value; +const DType = tri.DType; + +/// `log_2(e)` — pre-multiply with this to turn `exp(x)` into `exp2(x * RCP_LN2)`. +const RCP_LN2: f32 = 1.4426950408889634; + +/// Partition selection slots, not logical KV positions. Scratch capacity may be +/// larger than the number of segments that have any selection slots to visit. +pub const SparseSegments = struct { + tiles_per_segment: i64, + span: i64, + launched: i64, + + pub fn init(count: i64, capacity: i64, tile: i64) SparseSegments { + std.debug.assert(count > 0 and capacity > 0 and tile > 0); + const tiles = std.math.divCeil(i64, count, capacity * tile) catch unreachable; + return .{ + .tiles_per_segment = tiles, + .span = tiles * tile, + .launched = std.math.divCeil(i64, count, tiles * tile) catch unreachable, + }; + } +}; + +pub const SparseAttentionConfig = struct { + dtype: DType, + num_seqs: i32, + num_query_heads: i64, + num_queries_per_kv: i64, + head_size: i64, + head_size_padded: i64, + block_m: i64, + block_size: i64, + tile_size: i64, + indices_count: i64, + num_segments: i64 = 1, + decode_sequence_grid: bool = false, + use_sinks: bool = false, + scale: f32, + block_table_stride: i64, + query_strides: [3]i64, + key_strides: [4]i64, + value_strides: [4]i64, + indices_strides: [2]i64, +}; + +fn findSeqIdx( + k: *Builder, + query_start_len_ptr: Value, + target_idx: Value, + num_seqs: Value, + block_q: i64, + use_q_block_mode: bool, +) Value { + const left_init = k.constant(.i32, 0); + + var w = k.openWhile(.{ left_init, num_seqs }, .{ .i32, .i32 }); + { + const left = w.before_carried[0]; + const right = w.before_carried[1]; + w.yieldBefore(left.lt(right), .{ left, right }); + } + { + const left = w.after_carried[0]; + const right = w.after_carried[1]; + const mid = left.add(right).div(2); + const val = k.load(query_start_len_ptr.addPtr(mid)); + const mid_val = if (use_q_block_mode) + val.div(@as(i32, @intCast(block_q))).add(mid) + else + val; + + var i = k.openIfElse(mid_val.le(target_idx), .{ .i32, .i32 }); + { + i.yieldThen(.{ mid.add(1), right }); + } + { + i.yieldElse(.{ left, mid }); + } + w.yieldAfter(.{ i.results[0], i.results[1] }); + } + + return w.results[0].sub(1); +} + +/// One program processes all selected keys for one query/KV-head group and write the normalized output. +pub const KernelSparseAttention2dPtr = KernelSparseAttentionPtr(false); + +/// Also known as split-K in the litterature, this 3d variant uses multiple programs process different +/// portions of that same selection list. They write partial results, then a reduction kernels combines +/// them into one normalized output. +/// +/// Mostly useful during decode when there are few queries and a long top-k list. +pub const KernelSparseAttention3dPtr = KernelSparseAttentionPtr(true); + +/// One query and one GQA group per program. The direct specialization writes +/// normalized output; split decode writes FP32 partials for SparseReducePtr. +fn KernelSparseAttentionPtr(comptime split_k: bool) type { + return struct { + pub const Config = SparseAttentionConfig; + pub const Kernel = tri.Kernel(Config, .{ + .name = if (split_k) "kernel_sparse_attention_3d_ptr" else "kernel_sparse_attention_2d_ptr", + .inputs = &.{ + "query_ptr", "key_ptr", "value_ptr", "indices_ptr", + "block_tables_ptr", "seq_lens_ptr", "query_start_len_ptr", "sink_ptr", + }, + .outputs = if (split_k) &.{ "output", "segm_max", "segm_expsum" } else &.{"output"}, + .run = run, + }); + + fn run(k: *Builder, cfg: Config) tri.FinishError!void { + const a = if (split_k) try k.declareArgs(.{ + .query_ptr = .{ .ptr = cfg.dtype }, + .key_ptr = .{ .ptr = cfg.dtype }, + .value_ptr = .{ .ptr = cfg.dtype }, + .indices_ptr = .{ .ptr = .i32 }, + .block_tables_ptr = .{ .ptr = .i32 }, + .seq_lens_ptr = .{ .ptr = .i32 }, + .query_start_len_ptr = .{ .ptr = .i32 }, + .sink_ptr = .{ .ptr = .f32 }, + .output_ptr = .{ .ptr = .f32 }, + .segm_max_ptr = .{ .ptr = .f32 }, + .segm_expsum_ptr = .{ .ptr = .f32 }, + }) else try k.declareArgs(.{ + .query_ptr = .{ .ptr = cfg.dtype }, + .key_ptr = .{ .ptr = cfg.dtype }, + .value_ptr = .{ .ptr = cfg.dtype }, + .indices_ptr = .{ .ptr = .i32 }, + .block_tables_ptr = .{ .ptr = .i32 }, + .seq_lens_ptr = .{ .ptr = .i32 }, + .query_start_len_ptr = .{ .ptr = .i32 }, + .sink_ptr = .{ .ptr = .f32 }, + .output_ptr = .{ .ptr = cfg.dtype }, + }); + + std.debug.assert(cfg.block_m >= cfg.num_queries_per_kv); + std.debug.assert(split_k or cfg.num_segments == 1); + + const partition = SparseSegments.init(cfg.indices_count, cfg.num_segments, cfg.tile_size); + const program = k.programId(.x); + const num_seqs = k.constant(.i32, cfg.num_seqs); + if (!cfg.decode_sequence_grid) { + const active_queries = k.load(a.query_start_len_ptr.addPtr(num_seqs)); + k.returnIf(program.ge(active_queries)); + } + + const seq = if (cfg.decode_sequence_grid) program else findSeqIdx(k, a.query_start_len_ptr, program, num_seqs, 1, false); + const seq_len = k.load(a.seq_lens_ptr.addPtr(seq)); + const query_start = k.load(a.query_start_len_ptr.addPtr(seq)); + const query_end = k.load(a.query_start_len_ptr.addPtr(seq).addPtr(1)); + if (cfg.decode_sequence_grid) k.returnIf(query_start.eq(query_end)); + const query = if (cfg.decode_sequence_grid) query_start else program; + const kv_head = if (cfg.num_query_heads == cfg.num_queries_per_kv) k.constant(.i32, 0) else k.programId(.y); + + // Launch the partial tail first, without launching unused capacity. + const slot = if (split_k) k.programId(.z) else k.constant(.i32, 0); + const segment = if (split_k) k.where(slot.eq(0), k.constant(.i32, partition.launched - 1), slot.sub(1)) else slot; + const segment_start = segment.mul(@as(i32, @intCast(partition.span))); + const heads = k.arange(0, cfg.block_m, .i32); + const head = kv_head.mul(@as(i32, @intCast(cfg.num_queries_per_kv))).add(heads); + const head_mask = heads.lt(@as(i32, @intCast(cfg.num_queries_per_kv))); + const dims = k.arange(0, cfg.head_size_padded, .i32); + const dim_mask = dims.lt(@as(i32, @intCast(cfg.head_size))); + const indices_row = a.indices_ptr.addPtr(query.to(.i64).mul(cfg.indices_strides[0])); + + const sink = if (cfg.use_sinks) k.loadOpts(a.sink_ptr.addPtr(head), .{ + .mask = head_mask.bitAnd(segment.eq(0)), + .other = k.full(&.{cfg.block_m}, -std.math.inf(f32), .f32), + }).mul(RCP_LN2) else k.full(&.{cfg.block_m}, -std.math.inf(f32), .f32); + const has_sink = if (cfg.use_sinks) segment.eq(0).bitAnd(head_mask) else k.full(&.{cfg.block_m}, 0, .i1); + const initial_max = k.where(has_sink, sink, k.full(&.{cfg.block_m}, -std.math.inf(f32), .f32)); + const initial_sum = has_sink.to(.f32); + const initial_acc = k.zeros(&.{ cfg.block_m, cfg.head_size_padded }, .f32); + + if (partition.span <= 256) { + const scan_size: i64 = @intCast(std.math.ceilPowerOfTwoAssert(u64, @intCast(partition.span))); + const selected = segment_start.add(k.arange(0, scan_size, .i32)); + const scan_mask = selected.lt(cfg.indices_count).bitAnd(selected.lt(segment_start.add(@as(i32, @intCast(partition.span))))); + const positions = k.loadOpts(indices_row.addPtr(selected.to(.i64).mul(cfg.indices_strides[1])), .{ + .mask = scan_mask, + .other = k.full(&.{scan_size}, -1, .i32), + }); + const empty = k.sum(scan_mask.bitAnd(positions.ge(0)).bitAnd(positions.lt(seq_len)).to(.i32)).eq(0); + var branch = k.openIfElse(empty, .{}); + { + storeResult(k, cfg, a, query, head, head_mask, segment, dims, dim_mask, initial_max, initial_sum, initial_acc); + branch.elseBranch(); + } + { + branch.close(); + } + k.returnIf(empty); + } + + const q_offset = query.to(.i64).mul(cfg.query_strides[0]) + .add(head.to(.i64).mul(cfg.query_strides[1]).expandDims(1)) + .add(dims.to(.i64).mul(cfg.query_strides[2]).expandDims(0)); + const Q = k.loadOpts(a.query_ptr.addPtr(q_offset), .{ + .mask = k.mask2d(head_mask, dim_mask, cfg.block_m, cfg.head_size_padded), + .other = initial_acc.to(cfg.dtype), + }); + const table = a.block_tables_ptr.addPtr(seq.to(.i64).mul(cfg.block_table_stride)); + const offsets = k.arange(0, cfg.tile_size, .i32); + var loop = k.openFor(0, partition.tiles_per_segment, 1, .{ initial_max, initial_sum, initial_acc }); + { + const selected = segment_start.add(loop.iv.mul(@as(i32, @intCast(cfg.tile_size)))).add(offsets); + const position = k.loadOpts(indices_row.addPtr(selected.to(.i64).mul(cfg.indices_strides[1])), .{ + .mask = selected.lt(cfg.indices_count), + .other = k.full(&.{cfg.tile_size}, -1, .i32), + }); + const valid = selected.lt(cfg.indices_count).bitAnd(position.ge(0)).bitAnd(position.lt(seq_len)); + // A bounded single-tile segment was already checked above. + const nonempty = if (partition.tiles_per_segment == 1 and partition.span <= 256) k.constant(.i1, 1) else k.sum(valid.to(.i32)).gt(0); + var branch = k.openIfElse(nonempty, .{ k.tensorTy(&.{cfg.block_m}, .f32), k.tensorTy(&.{cfg.block_m}, .f32), k.tensorTy(&.{ cfg.block_m, cfg.head_size_padded }, .f32) }); + { + const safe_position = k.where(valid, position, k.zeros(&.{cfg.tile_size}, .i32)); + const page = k.loadOpts(table.addPtr(safe_position.div(@as(i32, @intCast(cfg.block_size)))), .{ + .mask = valid, + .other = k.zeros(&.{cfg.tile_size}, .i32), + }).to(.i64); + const in_page = safe_position.rem(@as(i32, @intCast(cfg.block_size))).to(.i64); + const key_offset = page.mul(cfg.key_strides[0]).add(in_page.mul(cfg.key_strides[1])).add(kv_head.to(.i64).mul(cfg.key_strides[2])).expandDims(0) + .add(dims.to(.i64).mul(cfg.key_strides[3]).expandDims(1)); + const value_offset = page.mul(cfg.value_strides[0]).add(in_page.mul(cfg.value_strides[1])).add(kv_head.to(.i64).mul(cfg.value_strides[2])).expandDims(1) + .add(dims.to(.i64).mul(cfg.value_strides[3]).expandDims(0)); + const K = k.loadOpts(a.key_ptr.addPtr(key_offset), .{ + .mask = k.mask2d(dim_mask, valid, cfg.head_size_padded, cfg.tile_size), + .other = k.zeros(&.{ cfg.head_size_padded, cfg.tile_size }, cfg.dtype), + }); + const V = k.loadOpts(a.value_ptr.addPtr(value_offset), .{ + .mask = k.mask2d(valid, dim_mask, cfg.tile_size, cfg.head_size_padded), + .other = k.zeros(&.{ cfg.tile_size, cfg.head_size_padded }, cfg.dtype), + }); + const scores = k.dot(Q, K, k.zeros(&.{ cfg.block_m, cfg.tile_size }, .f32)).mul(cfg.scale * RCP_LN2); + const S = k.where(k.mask2d(head_mask, valid, cfg.block_m, cfg.tile_size), scores, k.full(&.{ cfg.block_m, cfg.tile_size }, -std.math.inf(f32), .f32)); + const max_score = loop.carried[0].maximum(k.maxOpts(S, .{ .axis = 1 })); + const next_max = k.where(max_score.gt(-std.math.inf(f32)), max_score, k.zeros(&.{cfg.block_m}, .f32)); + const alpha = k.exp2(loop.carried[0].sub(next_max)); + const P = k.exp2(S.sub(next_max.expandDims(1))); + const sum = loop.carried[1].mul(alpha).add(k.sumOpts(P, .{ .axis = 1 })); + const acc = k.dot(P.to(cfg.dtype), V, loop.carried[2].mul(alpha.expandDims(1))); + branch.yieldThen(.{ next_max, sum, acc }); + } + { + branch.yieldElse(.{ loop.carried[0], loop.carried[1], loop.carried[2] }); + } + loop.yield(.{ branch.results[0], branch.results[1], branch.results[2] }); + } + + storeResult(k, cfg, a, query, head, head_mask, segment, dims, dim_mask, loop.results[0], loop.results[1], loop.results[2]); + } + + fn storeResult(k: *Builder, cfg: Config, a: anytype, query: Value, head: Value, head_mask: Value, segment: Value, dims: Value, dim_mask: Value, max_score: Value, sum: Value, acc: Value) void { + if (split_k) { + // Adjacent heads are written together by the producer. + const stats_offset = query.to(.i64).mul(cfg.num_segments).add(segment.to(.i64)).mul(cfg.num_query_heads).add(head.to(.i64)); + k.storeOpts(a.segm_max_ptr.addPtr(stats_offset), max_score, .{ .mask = head_mask }); + k.storeOpts(a.segm_expsum_ptr.addPtr(stats_offset), sum, .{ .mask = head_mask }); + const offset = query.to(.i64).mul(cfg.num_query_heads).add(head.to(.i64)).mul(cfg.num_segments).add(segment.to(.i64)).mul(cfg.head_size_padded).expandDims(1).add(dims.to(.i64).expandDims(0)); + // Zero-mass segments need no partial output; the reducer masks them. + k.storeOpts(a.output_ptr.addPtr(offset), acc, .{ .mask = k.mask2d(head_mask.bitAnd(sum.gt(0.0)), dim_mask, cfg.block_m, cfg.head_size_padded) }); + } else { + const denominator = k.where(sum.gt(0.0), sum, k.full(&.{cfg.block_m}, 1.0, .f32)); + const offset = query.to(.i64).mul(cfg.query_strides[0]).add(head.to(.i64).mul(cfg.query_strides[1]).expandDims(1)).add(dims.to(.i64).mul(cfg.query_strides[2]).expandDims(0)); + k.storeOpts(a.output_ptr.addPtr(offset), acc.div(denominator.expandDims(1)).to(cfg.dtype), .{ .mask = k.mask2d(head_mask, dim_mask, cfg.block_m, cfg.head_size_padded) }); + } + } + }; +} + +pub const SparseReducePtr = struct { + pub const Config = struct { + dtype: DType, + num_seqs: i32, + num_query_heads: i64, + head_size: i64, + head_size_padded: i64, + head_tile: i64, + num_segments: i64, + launched_segments: i64, + output_strides: [3]i64, + }; + pub const Kernel = tri.Kernel(Config, .{ + .name = "reduce_sparse_attention_ptr", + .inputs = &.{ "segm_output_ptr", "segm_max_ptr", "segm_expsum_ptr", "query_start_len_ptr" }, + .outputs = &.{"output"}, + .run = run, + }); + fn run(k: *Builder, cfg: Config) tri.FinishError!void { + const a = try k.declareArgs(.{ + .segm_output_ptr = .{ .ptr = .f32 }, + .segm_max_ptr = .{ .ptr = .f32 }, + .segm_expsum_ptr = .{ .ptr = .f32 }, + .query_start_len_ptr = .{ .ptr = .i32 }, + .output_ptr = .{ .ptr = cfg.dtype }, + }); + const query = k.programId(.x); + k.returnIf(query.ge(k.load(a.query_start_len_ptr.addPtr(cfg.num_seqs)))); + const head = k.programId(.y); + const segments = k.arange(0, cfg.num_segments, .i32); + const segment_mask = segments.lt(cfg.launched_segments); + const dims = k.programId(.z).mul(@as(i32, @intCast(cfg.head_tile))).add(k.arange(0, cfg.head_tile, .i32)); + const dim_mask = dims.lt(cfg.head_size); + const stats = query.to(.i64).mul(cfg.num_segments).add(segments.to(.i64)).mul(cfg.num_query_heads).add(head.to(.i64)); + const maxima = k.loadOpts(a.segm_max_ptr.addPtr(stats), .{ .mask = segment_mask, .other = k.full(&.{cfg.num_segments}, -std.math.inf(f32), .f32) }); + const sums = k.loadOpts(a.segm_expsum_ptr.addPtr(stats), .{ .mask = segment_mask, .other = k.zeros(&.{cfg.num_segments}, .f32) }); + const nonempty = segment_mask.bitAnd(sums.gt(0.0)); + const maximum = k.max(k.where(nonempty, maxima, k.full(&.{cfg.num_segments}, -std.math.inf(f32), .f32))); + const safe_maximum = k.where(maximum.gt(-std.math.inf(f32)), maximum, k.constant(.f32, 0.0)); + const rescale = k.exp2(maxima.sub(safe_maximum)); + const denominator = k.sum(sums.mul(rescale)); + const offset = query.to(.i64).mul(cfg.num_query_heads).add(head.to(.i64)).mul(cfg.num_segments).add(segments.to(.i64)).mul(cfg.head_size_padded).expandDims(1).add(dims.to(.i64).expandDims(0)); + const partial = k.loadOpts(a.segm_output_ptr.addPtr(offset), .{ + .mask = k.mask2d(nonempty, dim_mask, cfg.num_segments, cfg.head_tile), + .other = k.zeros(&.{ cfg.num_segments, cfg.head_tile }, .f32), + }); + const numerator = k.sumOpts(partial.mul(rescale.expandDims(1)), .{ .axis = 0 }); + const output = numerator.div(k.where(denominator.gt(0.0), denominator, k.constant(.f32, 1.0))); + const output_offset = query.to(.i64).mul(cfg.output_strides[0]).add(head.to(.i64).mul(cfg.output_strides[1])).add(dims.to(.i64).mul(cfg.output_strides[2])); + k.storeOpts(a.output_ptr.addPtr(output_offset), output.to(cfg.dtype), .{ .mask = dim_mask }); + } +}; diff --git a/zml/kernels/paged_attention/triton/triton.zig b/zml/kernels/paged_attention/triton/triton.zig new file mode 100644 index 00000000..2a12e012 --- /dev/null +++ b/zml/kernels/paged_attention/triton/triton.zig @@ -0,0 +1,861 @@ +const std = @import("std"); + +const stdx = @import("stdx"); + +const zml = @import("../../../zml.zig"); +const triton = zml.kernel.triton; +const AttentionOptions = @import("../options.zig").AttentionOptions; +const kernels = @import("unified_attention.zig"); +const kernels_oneapi = @import("unified_attention_oneapi.zig"); +const kernels_sparse = @import("sparse_unified_attention.zig"); + +const log = std.log.scoped(.@"zml/attention/triton"); + +fn isOneapiTarget() bool { + return zml.Compiler.current().platform.target == .oneapi; +} + +fn use2dKernel(all_decode: bool, batch_size: usize, num_kv_heads: usize) bool { + // Intel decode spills the 2D whole-sequence kernel; force the 3D split-K path. + if (all_decode and isOneapiTarget()) return false; + // prefill uses 2D; decode uses 3D until the batch is large enough to + // provide at least 128 2D launch programs across KV heads. + if (all_decode) { + const seq_threshold_3d = @divFloor(128, num_kv_heads); + return batch_size > seq_threshold_3d; + } + + return true; +} + +pub const Config2D = struct { + block_m: usize, + block_q: usize, + tile_size: usize, + num_warps: usize, + num_stages: usize, + total_q_blocks: usize, +}; + +fn isCudaComputeCapability(expected: zml.platform.cuda.ComputeCapability) bool { + const platform = zml.module.CompilationContext.current().platform; + return if (zml.platform.cuda.computeCapability(platform)) |cc| cc.eql(expected) else false; +} + +/// Largest `BLOCK_M * HEAD_SIZE_PADDED` the 2D kernel survives. That product is +/// the shape of both the query tile and the f32 accumulator carried through the +/// tile loop; above it the kernel faults with CUDA_ERROR_MISALIGNED_ADDRESS. +/// +/// Measured on a GB300 with Gemma-4-31B, whose global-attention layers are +/// head_dim 512 (its sliding ones are 256): +/// +/// 64x256, 32x512, 16x512 ok 64x512, 128x512 fault +/// +/// The bound is on this product alone. Two things that do *not* move it, so do +/// not reach for them if a new head size faults: TILE_SIZE, which at 64 pushes +/// total shared memory to ~160 KB and still passes at BLOCK_M 32 while 64x512 +/// faults at 98 KB; and num_warps, where 8 halves the per-thread register share +/// and still faults -- which is also why the obvious "the accumulator outgrows +/// the register file" story cannot be the whole explanation. The underlying +/// Triton codegen bug is not diagnosed; this only keeps us out of it. +const max_query_tile_elements: usize = 16 * 1024; + +fn select2dConfig(options: paged.PagedAttentionOptions) Config2D { + const max_num_stages_2d: usize = if (options.head_dim <= 128) 4 else 2; + + // Until we test on other platforms, gate the fix to GB300 + const is_gb300 = isCudaComputeCapability(.{ .major = 10, .minor = 3 }); + + var num_stages_2d: usize, var num_warps: usize, var tile_size: usize = if (!options.all_decode) .{ 1, 2, 64 } else .{ 3, 2, options.block_size }; + + var block_m = options.block_m; + var block_q = options.block_q; + if (options.max_seqlen_q >= 256) { + if (options.head_dim >= 256) { + block_m = 64; + tile_size = 16; + } else { + block_m = 128; + } + + if (is_gb300) { + // Halving BLOCK_M for every doubling of the head keeps that product + // constant: 128 at head_dim 128, 64 at 256, 32 at 512. The first two + // are the values this branch has always picked. + const head_dim_padded = std.math.ceilPowerOfTwoAssert(usize, options.head_dim); + block_m = @min(128, max_query_tile_elements / head_dim_padded); + } + num_stages_2d = 1; + num_warps = 4; + } + block_q = block_m / options.numQueriesPerKv(); + const total_q_blocks = options.num_tokens / block_q + options.batch_size; + + return .{ + .block_m = block_m, + .block_q = block_q, + .tile_size = tile_size, + .num_warps = num_warps, + .num_stages = @min(max_num_stages_2d, num_stages_2d), + .total_q_blocks = total_q_blocks, + }; +} + +pub const Config3D = struct { + const AttentionConfig = struct { + tile_size: usize, + num_segments_per_seq: usize, + num_warps: usize, + num_stages: usize, + block_q: usize, + block_m: usize, + total_q_blocks: usize, + }; + const ReduceConfig = struct { + tile_size: usize, + num_segments_per_seq: usize, + num_warps: usize, + num_stages: usize, + block_q: usize, + block_m: usize, + }; + attention: AttentionConfig, + reduce: ReduceConfig, +}; + +fn select3dConfig(options: paged.PagedAttentionOptions) Config3D { + var reduce_num_warps: usize = 2; + // Intel decode needs more warps to spread the work and avoid register spill. + const attn_warps: usize = if (options.all_decode and isOneapiTarget()) 8 else 2; + const tile_size = options.block_size; + + //const MAX_SEGMENTS: usize = @min(128, std.math.divCeil(usize, max_seqlen_k, tile_size)); + var num_segments = std.math.divCeil(usize, options.target_num_prgms, options.num_2d_prgms) catch unreachable; + num_segments = std.math.ceilPowerOfTwoAssert(usize, num_segments); + num_segments = @min(num_segments, 128); + if (options.all_decode and !isOneapiTarget()) { + // Keep the number of segments small to then limit reduce cost + // Didn't change the computation of Intel decode at the momment + // Need to be tested + num_segments = @min(num_segments, 16); + } + const min_segments: usize = if (tile_size <= 16) 16 else 8; + num_segments = @max(num_segments, min_segments); + if (num_segments == min_segments) { + reduce_num_warps = 1; + } + + return .{ + .attention = .{ + .tile_size = tile_size, + .num_segments_per_seq = num_segments, + .num_warps = attn_warps, + .num_stages = 1, + .block_m = options.block_m, + .block_q = options.block_q, + .total_q_blocks = options.total_q_blocks, + }, + .reduce = .{ + .tile_size = tile_size, + .num_segments_per_seq = num_segments, + .num_warps = reduce_num_warps, + .num_stages = 1, + .block_m = options.block_m, + .block_q = options.block_q, + }, + }; +} + +pub fn getCuCount() usize { + const platform = zml.Compiler.current().platform; + if (platform.devices.len == 0) return 1; + const attribute = platform.devices[0].pjrt_desc.attribute(platform.pjrt_api, "core_count") orelse return 1; + if (attribute.int64 <= 0) return 1; + return @intCast(attribute.int64); +} + +const SparseConfig = struct { + tile_size: usize, + num_segments: usize, +}; + +fn selectSparseConfig(count: usize, batch_size: usize, num_kv_heads: usize, cu_count: usize, all_decode: bool) SparseConfig { + std.debug.assert(count > 0 and batch_size > 0 and num_kv_heads > 0); + if (!all_decode) return .{ .tile_size = 32, .num_segments = 1 }; + + const target_num_prgms = cu_count * 4; + const occupancy_splits = std.math.ceilPowerOfTwoAssert(usize, std.math.divCeil(usize, target_num_prgms * 4, batch_size * num_kv_heads) catch unreachable); + const tile: usize = if (occupancy_splits > 128) 16 else 32; + const tile_splits = std.math.ceilPowerOfTwoAssert(usize, std.math.divCeil(usize, count, tile) catch unreachable); + + return .{ + .tile_size = tile, + .num_segments = @min(tile_splits, @min(if (tile == 16) @as(usize, 256) else 128, @max(8, occupancy_splits))), + }; +} + +pub const paged = struct { + pub const Options = struct { + batch_size: usize, + max_num_pages: usize, + max_seqlen_q: usize, + is_prefill: bool, + + pub fn isPrefill(self: Options) bool { + return self.is_prefill; + } + + pub fn maxNumPages(self: Options) usize { + return self.max_num_pages; + } + }; + + pub const Parameters = struct { + block_table: zml.Tensor, + seq_lens: zml.Tensor, + query_start_len: zml.Tensor, + options_: Options, + + pub fn init(options_: Options) Parameters { + return .{ + .block_table = .init(.{ .b = options_.batch_size, .p = options_.max_num_pages }, .i32), + .seq_lens = .init(.{ .b = options_.batch_size }, .i32), + .query_start_len = .init(.{ .b = options_.batch_size + 1 }, .i32), + .options_ = options_, + }; + } + + pub fn allocationSize(self: Parameters) usize { + var allocation_size: usize = 0; + + allocation_size += self.block_table.byteSize(); + allocation_size += self.seq_lens.byteSize(); + allocation_size += self.query_start_len.byteSize(); + + return allocation_size; + } + + pub fn options(self: Parameters) Options { + return self.options_; + } + }; + + pub const PagedAttentionOptions = struct { + cu_count: usize, + all_decode: bool, + num_tokens: usize, + num_heads: usize, + num_kv_heads: usize, + head_dim: usize, + batch_size: usize, + block_size: usize, + num_blocks: usize, + max_num_block_per_seq: usize, + sliding_window: usize, + block_m: usize, + block_q: usize, + total_q_blocks: usize, + target_num_prgms: usize, + num_2d_prgms: usize, + max_seqlen_q: usize, + scale: ?f32, + + pub fn numQueriesPerKv(self: PagedAttentionOptions) usize { + return self.num_heads / self.num_kv_heads; + } + + pub fn maxSeqLenK(self: PagedAttentionOptions) usize { + return self.max_num_block_per_seq * self.block_size; + } + }; + + pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions) zml.Tensor { + if (opts.mask == .indices) { + return pagedSparseAttention(parameters, q, k_cache, v_cache, opts); + } + + const cu_count = getCuCount(); + const num_heads: usize = @intCast(q.dim(.hkv) * q.dim(.hg)); + const num_kv_heads: usize = @intCast(k_cache.dim(.hkv)); + const num_queries_per_kv: usize = num_heads / num_kv_heads; + // Intel decode: pack exactly one GQA group per tile (block_q == 1) so the + // single decode query token doesn't carry masked-out fp32 acc lanes. + // oneAPI decode keeps one GQA group per tile, padded to a power of two so tt.make_range emits legal Triton IR. + const block_m: usize = if (!parameters.options_.is_prefill and isOneapiTarget()) + std.math.ceilPowerOfTwoAssert(usize, num_queries_per_kv) + else if (num_queries_per_kv <= 16) 16 else std.math.ceilPowerOfTwoAssert(usize, num_queries_per_kv); + const block_q: usize = block_m / num_queries_per_kv; + const num_tokens: usize = @intCast(q.dim(.b)); + const num_seqs: usize = @intCast(parameters.block_table.dim(.b)); + const total_q_blocks: usize = num_tokens / block_q + num_seqs; + const target_num_prgms: usize = cu_count * 4; + const num_2d_prgms: usize = total_q_blocks * num_kv_heads; + + const sliding_window: usize = opts.mask.slidingWindow() orelse 0; + + const paged_attention_opts: PagedAttentionOptions = .{ + .cu_count = getCuCount(), + .all_decode = !parameters.options_.is_prefill, + .num_tokens = num_tokens, + .num_heads = num_heads, + .num_kv_heads = num_kv_heads, + .head_dim = @intCast(q.dim(.hd)), + .batch_size = @intCast(parameters.block_table.dim(.b)), + .block_size = @intCast(k_cache.dim(.k_chunk)), + .num_blocks = @intCast(k_cache.dim(.page)), + .max_num_block_per_seq = @intCast(parameters.block_table.dim(.p)), + .sliding_window = sliding_window, + .block_m = block_m, + .block_q = block_q, + .total_q_blocks = total_q_blocks, + .target_num_prgms = target_num_prgms, + .num_2d_prgms = num_2d_prgms, + .max_seqlen_q = parameters.options_.max_seqlen_q, + .scale = opts.scale, + }; + + const use_2d_kernel = use2dKernel( + paged_attention_opts.all_decode, + paged_attention_opts.batch_size, + paged_attention_opts.num_kv_heads, + ); + const output = if (use_2d_kernel) + pagedAttention2d(parameters, q, k_cache, v_cache, opts, paged_attention_opts) + else if (isOneapiTarget()) + pagedAttention3dOneapi(parameters, q, k_cache, v_cache, opts, paged_attention_opts) + else + pagedAttention3d(parameters, q, k_cache, v_cache, opts, paged_attention_opts); + + return output; + } + + pub fn pagedSparseAttention(parameters: Parameters, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions) zml.Tensor { + stdx.debug.assert(opts.mask == .indices, "Expected indices mask but got {}", .{opts.mask}); + const indices = opts.mask.indices; + stdx.debug.assert(indices.rank() == 2 and indices.dtype() == .i32, "Sparse indices must be an i32 [query, topk] tensor, got {f}", .{indices.shape()}); + stdx.debug.assert(indices.dim(.b) == q.dim(.b) and indices.dim(.topk) > 0, "Sparse indices {f} must match queries {f} with a nonempty selection capacity", .{ indices.shape(), q.shape() }); + stdx.debug.assert(indices.shape().hasTags(.{ .b, .topk }), "Sparse indices expect to have .b, .topk tags", .{}); + stdx.debug.assert(q.dtype() == k_cache.dtype() and q.dtype() == v_cache.dtype(), "Sparse attention requires matching query and KV dtypes", .{}); + + const config = selectSparseConfig( + @intCast(opts.mask.indices.dim(.topk)), + @intCast(parameters.block_table.dim(.b)), + @intCast(q.dim(.hkv)), + getCuCount(), + !parameters.options_.is_prefill, + ); + + return if (config.num_segments == 1) + pagedSparseAttention2d(parameters, q, k_cache, v_cache, opts, config) + else + pagedSparseAttention3d(parameters, q, k_cache, v_cache, opts, config); + } + + pub fn pagedSparseAttention2d(parameters: Parameters, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions, tuning: SparseConfig) zml.Tensor { + std.debug.assert(tuning.num_segments == 1); + + const indices = opts.mask.indices; + + const group = q.dim(.hg); + const num_kv_heads = q.dim(.hkv); + std.debug.assert(num_kv_heads == k_cache.dim(.hkv) and num_kv_heads == v_cache.dim(.hkv)); + std.debug.assert(q.dim(.hd) == k_cache.dim(.hd) and q.dim(.hd) == v_cache.dim(.hd)); + + const all_decode = !parameters.options_.is_prefill; + + const q_shape = q.shape().mergeAxes(.{ .h = .{ .hkv, .hg } }); + + const qs = q_shape.computeElementStrides().constSlice(); + const ks = k_cache.shape().computeElementStrides().constSlice(); + const vs = v_cache.shape().computeElementStrides().constSlice(); + const is = indices.shape().computeElementStrides().constSlice(); + const ts = parameters.block_table.shape().computeElementStrides().constSlice(); + + const head_size_padded: i64 = @intCast(std.math.ceilPowerOfTwoAssert(u64, @intCast(@max(16, q.dim(.hd))))); + + const config: kernels_sparse.SparseAttentionConfig = .{ + .dtype = triton.from(q.dtype()), + .num_seqs = @intCast(parameters.block_table.dim(.b)), + .num_query_heads = num_kv_heads * group, + .num_queries_per_kv = group, + .head_size = q.dim(.hd), + .head_size_padded = head_size_padded, + .block_m = @intCast(@max(16, std.math.ceilPowerOfTwoAssert(u64, @intCast(group)))), + .block_size = k_cache.dim(.k_chunk), + .tile_size = @intCast(tuning.tile_size), + .indices_count = indices.dim(.topk), + .num_segments = @intCast(tuning.num_segments), + .decode_sequence_grid = all_decode and parameters.options_.max_seqlen_q == 1, + .use_sinks = opts.sink != null, + .scale = opts.scale orelse @floatCast(1.0 / @sqrt(@as(f64, @floatFromInt(q.dim(.hd))))), + .block_table_stride = ts[parameters.block_table.axis(.b)], + .query_strides = .{ qs[q_shape.axis(.b)], qs[q_shape.axis(.h)], qs[q_shape.axis(.hd)] }, + .key_strides = .{ ks[k_cache.axis(.page)], ks[k_cache.axis(.k_chunk)], ks[k_cache.axis(.hkv)], ks[k_cache.axis(.hd)] }, + .value_strides = .{ vs[v_cache.axis(.page)], vs[v_cache.axis(.k_chunk)], vs[v_cache.axis(.hkv)], vs[v_cache.axis(.hd)] }, + .indices_strides = .{ is[indices.axis(.b)], is[indices.axis(.topk)] }, + }; + + const inputs: kernels_sparse.KernelSparseAttention2dPtr.Kernel.Inputs = .{ + .query_ptr = q, + .key_ptr = k_cache, + .value_ptr = v_cache, + .indices_ptr = indices, + .block_tables_ptr = parameters.block_table, + .seq_lens_ptr = parameters.seq_lens, + .query_start_len_ptr = parameters.query_start_len, + .sink_ptr = if (opts.sink) |sink| sink.convert(.f32) else .scalar(0, .f32), + }; + + const grid: [3]i32 = .{ + if (config.decode_sequence_grid) config.num_seqs else @intCast(q.dim(.b)), + @intCast(num_kv_heads), + 1, + }; + + const warps: i32 = if (all_decode and isOneapiTarget()) 8 else 4; + + const output = kernels_sparse.KernelSparseAttention2dPtr.Kernel.call(inputs, .{ + .output = q.shape(), + }, .{ + .cfg = config, + .grid = grid, + .num_stages = 1, + .num_warps = warps, + }); + + return output.output; + } + + pub fn pagedSparseAttention3d(parameters: Parameters, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions, tuning: SparseConfig) zml.Tensor { + std.debug.assert(tuning.num_segments > 1); + + const indices = opts.mask.indices; + const group = q.dim(.hg); + const num_kv_heads = q.dim(.hkv); + std.debug.assert(num_kv_heads == k_cache.dim(.hkv) and num_kv_heads == v_cache.dim(.hkv)); + std.debug.assert(q.dim(.hd) == k_cache.dim(.hd) and q.dim(.hd) == v_cache.dim(.hd)); + + const all_decode = !parameters.options_.is_prefill; + + const q_shape = q.shape().mergeAxes(.{ .h = .{ .hkv, .hg } }); + + const qs = q_shape.computeElementStrides().constSlice(); + const ks = k_cache.shape().computeElementStrides().constSlice(); + const vs = v_cache.shape().computeElementStrides().constSlice(); + const is = indices.shape().computeElementStrides().constSlice(); + const ts = parameters.block_table.shape().computeElementStrides().constSlice(); + + const head_size_padded: i64 = @intCast(std.math.ceilPowerOfTwoAssert(u64, @intCast(@max(16, q.dim(.hd))))); + + const config: kernels_sparse.SparseAttentionConfig = .{ + .dtype = triton.from(q.dtype()), + .num_seqs = @intCast(parameters.block_table.dim(.b)), + .num_query_heads = num_kv_heads * group, + .num_queries_per_kv = group, + .head_size = q.dim(.hd), + .head_size_padded = head_size_padded, + .block_m = @intCast(@max(16, std.math.ceilPowerOfTwoAssert(u64, @intCast(group)))), + .block_size = k_cache.dim(.k_chunk), + .tile_size = @intCast(tuning.tile_size), + .indices_count = indices.dim(.topk), + .num_segments = @intCast(tuning.num_segments), + .decode_sequence_grid = all_decode and parameters.options_.max_seqlen_q == 1, + .use_sinks = opts.sink != null, + .scale = opts.scale orelse @floatCast(1.0 / @sqrt(@as(f64, @floatFromInt(q.dim(.hd))))), + .block_table_stride = ts[parameters.block_table.axis(.b)], + .query_strides = .{ qs[q_shape.axis(.b)], qs[q_shape.axis(.h)], qs[q_shape.axis(.hd)] }, + .key_strides = .{ ks[k_cache.axis(.page)], ks[k_cache.axis(.k_chunk)], ks[k_cache.axis(.hkv)], ks[k_cache.axis(.hd)] }, + .value_strides = .{ vs[v_cache.axis(.page)], vs[v_cache.axis(.k_chunk)], vs[v_cache.axis(.hkv)], vs[v_cache.axis(.hd)] }, + .indices_strides = .{ is[indices.axis(.b)], is[indices.axis(.topk)] }, + }; + + const inputs: kernels_sparse.KernelSparseAttention3dPtr.Kernel.Inputs = .{ + .query_ptr = q, + .key_ptr = k_cache, + .value_ptr = v_cache, + .indices_ptr = indices, + .block_tables_ptr = parameters.block_table, + .seq_lens_ptr = parameters.seq_lens, + .query_start_len_ptr = parameters.query_start_len, + .sink_ptr = if (opts.sink) |sink| sink.convert(.f32) else .scalar(0, .f32), + }; + + const partition = kernels_sparse.SparseSegments.init(config.indices_count, config.num_segments, config.tile_size); + const grid: [3]i32 = .{ + if (config.decode_sequence_grid) config.num_seqs else @intCast(q.dim(.b)), + @intCast(num_kv_heads), + @intCast(partition.launched), + }; + + const warps: i32 = if (all_decode) 8 else 4; + + const stats_shape = zml.Shape.init(.{ q.dim(.b), config.num_segments, config.num_query_heads }, .f32); + + const partials = kernels_sparse.KernelSparseAttention3dPtr.Kernel.call(inputs, .{ + .output = .init(.{ q.dim(.b), config.num_query_heads, config.num_segments, head_size_padded }, .f32), + .segm_max = stats_shape, + .segm_expsum = stats_shape, + }, .{ + .cfg = config, + .grid = grid, + .num_stages = 1, + .num_warps = warps, + }); + + const head_tile = @min(64, head_size_padded); + + const output = kernels_sparse.SparseReducePtr.Kernel.call(.{ + .segm_output_ptr = partials.output, + .segm_max_ptr = partials.segm_max, + .segm_expsum_ptr = partials.segm_expsum, + .query_start_len_ptr = parameters.query_start_len, + }, .{ .output = q.shape() }, .{ + .cfg = .{ + .dtype = config.dtype, + .num_seqs = config.num_seqs, + .num_query_heads = config.num_query_heads, + .head_size = config.head_size, + .head_size_padded = head_size_padded, + .head_tile = head_tile, + .num_segments = config.num_segments, + .launched_segments = partition.launched, + .output_strides = config.query_strides, + }, + .grid = .{ @intCast(q.dim(.b)), @intCast(config.num_query_heads), @intCast(@divExact(head_size_padded, head_tile)) }, + .num_stages = 1, + .num_warps = 4, + }).output; + + return output; + } + + pub fn pagedAttention2d(parameters: Parameters, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions, paged_attention_opts: PagedAttentionOptions) zml.Tensor { + const config = select2dConfig(paged_attention_opts); + const k_strides = k_cache.shape().computeElementStrides().constSlice(); + const v_strides = v_cache.shape().computeElementStrides().constSlice(); + + const kernel_config: kernels.KernelUnifiedAttention2dPtr.Config = .{ + .q_dtype = triton.from(q.dtype()), + .kv_dtype = triton.from(k_cache.dtype()), + .k_strides = .{ k_strides[0], k_strides[1], k_strides[2] }, + .v_strides = .{ v_strides[0], v_strides[1], v_strides[2] }, + .o_dtype = triton.from(q.dtype()), + .num_query_heads = @intCast(paged_attention_opts.num_heads), + .num_queries_per_kv = @intCast(paged_attention_opts.numQueriesPerKv()), + .block_size = @intCast(paged_attention_opts.block_size), + .tile_size = @intCast(config.tile_size), + .head_size = @intCast(paged_attention_opts.head_dim), + .head_size_padded = @intCast(std.math.ceilPowerOfTwoAssert(usize, paged_attention_opts.head_dim)), + .use_alibi_slopes = false, + .use_qq_bias = false, + .use_softcap = false, + .use_sinks = (opts.sink != null), + .sliding_window = @intCast(paged_attention_opts.sliding_window), + .block_q = @intCast(config.block_q), + .block_m = @intCast(config.block_m), + .use_fp8 = false, + .all_decode = paged_attention_opts.all_decode, + .is_causal = opts.mask.isCausal(), + }; + log.debug("pagedAttention2d config: {any}", .{kernel_config}); + + const dummy: zml.Tensor = .scalar(0, .i8); + const block_table_strides = parameters.block_table.shape().computeElementStrides().constSlice(); + + const q_shape = q.shape().mergeAxes(.{ .h = .{ .hkv, .hg } }); + const q_strides = q_shape.computeElementStrides().constSlice(); + + const scale: f32 = paged_attention_opts.scale orelse @floatCast(1.0 / @sqrt(@as(f64, @floatFromInt(q.dim(.hd))))); + const num_seqs = parameters.block_table.dim(0); + + const sink = if (opts.sink) |sink| sink.convert(.f32) else dummy; + + const output = kernels.KernelUnifiedAttention2dPtr.Kernel.call( + .{ + .query_ptr = q, + .key_cache_ptr = k_cache, + .value_cache_ptr = v_cache, + .sink_ptr = sink, + .block_tables_ptr = parameters.block_table, + .seq_lens_ptr = parameters.seq_lens, + .alibi_slopes_ptr = dummy, + .qq_bias_ptr = dummy, + .scale_ptr = .scalar(scale, .f32), + .k_scale_ptr = dummy, + .v_scale_ptr = dummy, + .out_scale_ptr = dummy, + .softcap_ptr = dummy, + .block_table_stride_ptr = .scalar(block_table_strides[0], .i64), + .query_stride_0_ptr = .scalar(q_strides[0], .i64), + .query_stride_1_ptr = .scalar(q_strides[1], .i64), + .output_stride_0_ptr = .scalar(q_strides[0], .i64), + .output_stride_1_ptr = .scalar(q_strides[1], .i64), + .qq_bias_stride_0_ptr = dummy, + .query_start_len_ptr = parameters.query_start_len, + .num_seqs_ptr = .scalar(num_seqs, .i32), + }, + .{ .output = q.shape() }, + .{ + .cfg = kernel_config, + .grid = .{ @intCast(paged_attention_opts.num_kv_heads), @intCast(config.total_q_blocks), 1 }, + .num_stages = @intCast(config.num_stages), + .num_warps = @intCast(config.num_warps), + }, + ); + return output.output; + } + + pub fn pagedAttention3d(parameters: Parameters, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions, paged_attention_opts: PagedAttentionOptions) zml.Tensor { + const config = select3dConfig(paged_attention_opts); + const k_strides = k_cache.shape().computeElementStrides().constSlice(); + const v_strides = v_cache.shape().computeElementStrides().constSlice(); + + const head_size_padded: i64 = @intCast(std.math.ceilPowerOfTwoAssert(usize, paged_attention_opts.head_dim)); + const attn_kernel_config: kernels.KernelUnifiedAttention3dPtr.Config = .{ + .q_dtype = triton.from(q.dtype()), + .kv_dtype = triton.from(k_cache.dtype()), + .k_strides = .{ k_strides[0], k_strides[1], k_strides[2] }, + .v_strides = .{ v_strides[0], v_strides[1], v_strides[2] }, + .num_query_heads = @intCast(paged_attention_opts.num_heads), + .num_queries_per_kv = @intCast(paged_attention_opts.numQueriesPerKv()), + .block_size = @intCast(paged_attention_opts.block_size), + .tile_size = @intCast(config.attention.tile_size), + .head_size = @intCast(paged_attention_opts.head_dim), + .head_size_padded = head_size_padded, + .use_alibi_slopes = false, + .use_qq_bias = false, + .use_softcap = false, + .use_sinks = (opts.sink != null), + .sliding_window = @intCast(paged_attention_opts.sliding_window), + .block_q = @intCast(config.attention.block_q), + .block_m = @intCast(config.attention.block_m), + .num_segments_per_seq = @intCast(config.attention.num_segments_per_seq), + .all_decode = paged_attention_opts.all_decode, + .is_causal = opts.mask.isCausal(), + }; + log.debug("pagedAttention3d attention config: {any}", .{attn_kernel_config}); + + const reduce_kernel_config: kernels.ReduceSegmentsPtr.Config = .{ + .o_dtype = triton.from(q.dtype()), + .num_query_heads = @intCast(paged_attention_opts.num_heads), + .tile_size = @intCast(config.reduce.tile_size), + .head_size = @intCast(paged_attention_opts.head_dim), + .head_size_padded = head_size_padded, + .block_q = @intCast(config.reduce.block_q), + .num_segments_per_seq = @intCast(config.reduce.num_segments_per_seq), + .use_fp8 = false, + }; + log.debug("pagedAttention3d reduce config: {any}", .{reduce_kernel_config}); + + const dummy: zml.Tensor = .scalar(0, .i8); + const block_table_strides = parameters.block_table.shape().computeElementStrides().constSlice(); + + const q_shape = q.shape().mergeAxes(.{ .h = .{ .hkv, .hg } }); + const q_strides = q_shape.computeElementStrides().constSlice(); + + const scale: f32 = paged_attention_opts.scale orelse @floatCast(1.0 / @sqrt(@as(f64, @floatFromInt(q.dim(.hd))))); + const num_seqs = parameters.block_table.dim(0); + + const attn_grid: [3]i32 = .{ + @intCast(config.attention.total_q_blocks), + @intCast(paged_attention_opts.num_kv_heads), + @intCast(config.attention.num_segments_per_seq), + }; + + const sink = if (opts.sink) |sink| sink.convert(.f32) else dummy; + + const attn_output = kernels.KernelUnifiedAttention3dPtr.Kernel.call( + .{ + .query_ptr = q, + .key_cache_ptr = k_cache, + .value_cache_ptr = v_cache, + .sink_ptr = sink, + .block_tables_ptr = parameters.block_table, + .seq_lens_ptr = parameters.seq_lens, + .alibi_slopes_ptr = dummy, + .qq_bias_ptr = dummy, + .scale_ptr = .scalar(scale, .f32), + .k_scale_ptr = dummy, + .v_scale_ptr = dummy, + .softcap_ptr = dummy, + .block_table_stride_ptr = .scalar(block_table_strides[0], .i64), + .query_stride_0_ptr = .scalar(q_strides[0], .i64), + .query_stride_1_ptr = .scalar(q_strides[1], .i64), + .qq_bias_stride_0_ptr = dummy, + .query_start_len_ptr = parameters.query_start_len, + .num_seqs_ptr = .scalar(num_seqs, .i32), + }, + .{ + .segm_output = zml.Shape.init(.{ paged_attention_opts.num_tokens, paged_attention_opts.num_heads, config.attention.num_segments_per_seq, std.math.ceilPowerOfTwoAssert(usize, paged_attention_opts.head_dim) }, .f32), + .segm_max = zml.Shape.init(.{ paged_attention_opts.num_tokens, paged_attention_opts.num_heads, config.attention.num_segments_per_seq }, .f32), + .segm_expsum = zml.Shape.init(.{ paged_attention_opts.num_tokens, paged_attention_opts.num_heads, config.attention.num_segments_per_seq }, .f32), + }, + .{ + .cfg = attn_kernel_config, + .grid = attn_grid, + .num_stages = @intCast(config.attention.num_stages), + .num_warps = @intCast(config.attention.num_warps), + }, + ); + + const output = kernels.ReduceSegmentsPtr.Kernel.call( + .{ + .segm_output_ptr = attn_output.segm_output, + .segm_max_ptr = attn_output.segm_max, + .segm_expsum_ptr = attn_output.segm_expsum, + .seq_lens_ptr = parameters.seq_lens, + .num_seqs_ptr = .scalar(num_seqs, .i32), + .out_scale_inv_ptr = dummy, + .output_stride_0_ptr = .scalar(q_strides[0], .i64), + .output_stride_1_ptr = .scalar(q_strides[1], .i64), + .block_table_stride_ptr = .scalar(block_table_strides[0], .i64), + .query_start_len_ptr = parameters.query_start_len, + }, + .{ .output = q.shape() }, + .{ + .cfg = reduce_kernel_config, + .grid = .{ + @intCast(paged_attention_opts.num_tokens), + @intCast(paged_attention_opts.num_heads), + 1, + }, + .num_stages = @intCast(config.reduce.num_stages), + .num_warps = @intCast(config.reduce.num_warps), + }, + ); + + return output.output; + } + + /// oneAPI/Intel specialization of pagedAttention3d: routes the attention pass to + /// the SIMD16-tuned kernel (compile-time KV strides, cached loads, same-page fast + /// path) while reusing the shared segment-reduce kernel. See unified_attention_oneapi.zig. + pub fn pagedAttention3dOneapi(parameters: Parameters, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions, paged_attention_opts: PagedAttentionOptions) zml.Tensor { + const config = select3dConfig(paged_attention_opts); + + const head_size_padded: i64 = @intCast(std.math.ceilPowerOfTwoAssert(usize, paged_attention_opts.head_dim)); + + const k_strides = k_cache.shape().computeElementStrides().constSlice(); + const v_strides = v_cache.shape().computeElementStrides().constSlice(); + + const attn_kernel_config: kernels_oneapi.KernelUnifiedAttention3dPtr.Config = .{ + .q_dtype = triton.from(q.dtype()), + .kv_dtype = triton.from(k_cache.dtype()), + .num_query_heads = @intCast(paged_attention_opts.num_heads), + .num_queries_per_kv = @intCast(paged_attention_opts.numQueriesPerKv()), + .block_size = @intCast(paged_attention_opts.block_size), + .tile_size = @intCast(config.attention.tile_size), + .head_size = @intCast(paged_attention_opts.head_dim), + .head_size_padded = head_size_padded, + .use_alibi_slopes = false, + .use_qq_bias = false, + .use_softcap = false, + .use_sinks = (opts.sink != null), + .sliding_window = @intCast(paged_attention_opts.sliding_window), + .is_causal = opts.mask.isCausal(), + .block_q = @intCast(config.attention.block_q), + .block_m = @intCast(config.attention.block_m), + .num_segments_per_seq = @intCast(config.attention.num_segments_per_seq), + .all_decode = paged_attention_opts.all_decode, + .stride_k_cache_0 = k_strides[k_cache.axis(.page)], + .stride_k_cache_1 = k_strides[k_cache.axis(.k_chunk)], + .stride_k_cache_2 = k_strides[k_cache.axis(.hkv)], + .stride_v_cache_0 = v_strides[v_cache.axis(.page)], + .stride_v_cache_1 = v_strides[v_cache.axis(.k_chunk)], + .stride_v_cache_2 = v_strides[v_cache.axis(.hkv)], + // Intel: keep KV loads cached (.none); the streaming .cg hint lowers to + // fully-uncached here and defeats the L2 prefetcher. + .kv_cache_modifier = .none, + }; + log.debug("pagedAttention3dOneapi attention config: {any}", .{attn_kernel_config}); + + const reduce_kernel_config: kernels.ReduceSegmentsPtr.Config = .{ + .o_dtype = triton.from(q.dtype()), + .num_query_heads = @intCast(paged_attention_opts.num_heads), + .tile_size = @intCast(config.reduce.tile_size), + .head_size = @intCast(paged_attention_opts.head_dim), + .head_size_padded = head_size_padded, + .block_q = @intCast(config.reduce.block_q), + .num_segments_per_seq = @intCast(config.reduce.num_segments_per_seq), + .use_fp8 = false, + }; + log.debug("pagedAttention3d reduce config: {any}", .{reduce_kernel_config}); + + const dummy: zml.Tensor = .scalar(0, .i8); + const block_table_strides = parameters.block_table.shape().computeElementStrides().constSlice(); + + const q_shape = q.shape().mergeAxes(.{ .h = .{ .hkv, .hg } }); + const q_strides = q_shape.computeElementStrides().constSlice(); + + const scale: f32 = paged_attention_opts.scale orelse @floatCast(1.0 / @sqrt(@as(f64, @floatFromInt(q.dim(.hd))))); + const num_seqs = parameters.block_table.dim(0); + + const attn_grid: [3]i32 = .{ @intCast(config.attention.total_q_blocks), @intCast(paged_attention_opts.num_kv_heads), @intCast(config.attention.num_segments_per_seq) }; + + const sink = if (opts.sink) |sink| sink.convert(.f32) else dummy; + + const attn_output = kernels_oneapi.KernelUnifiedAttention3dPtr.Kernel.call( + .{ + .query_ptr = q, + .key_cache_ptr = k_cache, + .value_cache_ptr = v_cache, + .sink_ptr = sink, + .block_tables_ptr = parameters.block_table, + .seq_lens_ptr = parameters.seq_lens, + .alibi_slopes_ptr = dummy, + .qq_bias_ptr = dummy, + .scale_ptr = .scalar(scale, .f32), + .k_scale_ptr = dummy, + .v_scale_ptr = dummy, + .softcap_ptr = dummy, + .block_table_stride_ptr = .scalar(block_table_strides[0], .i64), + .query_stride_0_ptr = .scalar(q_strides[0], .i64), + .query_stride_1_ptr = .scalar(q_strides[1], .i64), + .qq_bias_stride_0_ptr = dummy, + .stride_k_cache_0_ptr = .scalar(k_strides[k_cache.axis(.page)], .i64), + .stride_k_cache_1_ptr = .scalar(k_strides[k_cache.axis(.k_chunk)], .i64), + .stride_k_cache_2_ptr = .scalar(k_strides[k_cache.axis(.hkv)], .i64), + .stride_v_cache_0_ptr = .scalar(v_strides[v_cache.axis(.page)], .i64), + .stride_v_cache_1_ptr = .scalar(v_strides[v_cache.axis(.k_chunk)], .i64), + .stride_v_cache_2_ptr = .scalar(v_strides[v_cache.axis(.hkv)], .i64), + .query_start_len_ptr = parameters.query_start_len, + .num_seqs_ptr = .scalar(num_seqs, .i32), + }, + .{ + .segm_output = zml.Shape.init(.{ paged_attention_opts.num_tokens, paged_attention_opts.num_heads, config.attention.num_segments_per_seq, std.math.ceilPowerOfTwoAssert(usize, paged_attention_opts.head_dim) }, .f32), + .segm_max = zml.Shape.init(.{ paged_attention_opts.num_tokens, paged_attention_opts.num_heads, config.attention.num_segments_per_seq }, .f32), + .segm_expsum = zml.Shape.init(.{ paged_attention_opts.num_tokens, paged_attention_opts.num_heads, config.attention.num_segments_per_seq }, .f32), + }, + .{ + .cfg = attn_kernel_config, + .grid = attn_grid, + .num_stages = @intCast(config.attention.num_stages), + .num_warps = @intCast(config.attention.num_warps), + }, + ); + + const output = kernels.ReduceSegmentsPtr.Kernel.call( + .{ + .segm_output_ptr = attn_output.segm_output, + .segm_max_ptr = attn_output.segm_max, + .segm_expsum_ptr = attn_output.segm_expsum, + .seq_lens_ptr = parameters.seq_lens, + .num_seqs_ptr = .scalar(num_seqs, .i32), + .out_scale_inv_ptr = dummy, + .output_stride_0_ptr = .scalar(q_strides[0], .i64), + .output_stride_1_ptr = .scalar(q_strides[1], .i64), + .block_table_stride_ptr = .scalar(block_table_strides[0], .i64), + .query_start_len_ptr = parameters.query_start_len, + }, + .{ .output = q.shape() }, + .{ + .cfg = reduce_kernel_config, + .grid = .{ @intCast(paged_attention_opts.num_tokens), @intCast(paged_attention_opts.num_heads), 1 }, + .num_stages = @intCast(config.reduce.num_stages), + .num_warps = @intCast(config.reduce.num_warps), + }, + ); + + return output.output; + } +}; diff --git a/zml/kernels/paged_attention/triton/unified_attention.zig b/zml/kernels/paged_attention/triton/unified_attention.zig new file mode 100644 index 00000000..7b2b9428 --- /dev/null +++ b/zml/kernels/paged_attention/triton/unified_attention.zig @@ -0,0 +1,1121 @@ +const std = @import("std"); + +const mlir = @import("mlir"); +const ttir = @import("mlir/dialects/ttir"); + +const zml = @import("../../../zml.zig"); +const tri = zml.kernel.triton; +const Builder = tri.Builder; +const Value = tri.Value; +const DType = tri.DType; + +const log = std.log.scoped(.@"zml/attention/triton"); + +pub const FP8_E4M3_MIN: f32 = -448.0; +pub const FP8_E4M3_MAX: f32 = 448.0; + +/// `log_2(e)` — pre-multiply with this to turn `exp(x)` into `exp2(x * RCP_LN2)`. +const RCP_LN2: f32 = 1.4426950408889634; + +pub const KernelUnifiedAttention2dPtr = struct { + pub const Config = struct { + q_dtype: DType, + kv_dtype: DType, + k_strides: [3]i64, + v_strides: [3]i64, + o_dtype: DType, + scale_dtype: DType = .f32, + + num_query_heads: i64, + num_queries_per_kv: i64, + + block_size: i64, + tile_size: i64, + head_size: i64, + head_size_padded: i64, + + use_alibi_slopes: bool, + use_qq_bias: bool, + use_softcap: bool, + use_sinks: bool, + sliding_window: i64, + + stride_k_cache_3: i64 = 1, + stride_v_cache_3: i64 = 1, + + block_q: i64, + block_m: i64, + use_fp8: bool, + fp8_min: f32 = FP8_E4M3_MIN, + fp8_max: f32 = FP8_E4M3_MAX, + all_decode: bool = false, + is_causal: bool = true, + }; + + pub const Kernel = tri.Kernel(Config, .{ + .name = "kernel_unified_attention_2d_ptr", + .inputs = &.{ + "query_ptr", "key_cache_ptr", "value_cache_ptr", "sink_ptr", + "block_tables_ptr", "seq_lens_ptr", "alibi_slopes_ptr", "qq_bias_ptr", + "scale_ptr", "k_scale_ptr", "v_scale_ptr", "out_scale_ptr", + "softcap_ptr", "block_table_stride_ptr", "query_stride_0_ptr", "query_stride_1_ptr", + "output_stride_0_ptr", "output_stride_1_ptr", "qq_bias_stride_0_ptr", "query_start_len_ptr", + "num_seqs_ptr", + }, + .outputs = &.{"output"}, + .run = run, + }); + fn run(b: *Builder, cfg: Config) tri.FinishError!void { + const a = try b.declareArgs(.{ + .query_ptr = .{ .ptr = cfg.q_dtype }, + .key_cache_ptr = .{ .ptr = cfg.kv_dtype }, + .value_cache_ptr = .{ .ptr = cfg.kv_dtype }, + .sink_ptr = .{ .ptr = cfg.scale_dtype }, + .block_tables_ptr = .{ .ptr = .i32 }, + .seq_lens_ptr = .{ .ptr = .i32 }, + .alibi_slopes_ptr = .{ .ptr = cfg.scale_dtype }, + .qq_bias_ptr = .{ .ptr = cfg.scale_dtype }, + .scale_ptr = .{ .ptr = .f32 }, + .k_scale_ptr = .{ .ptr = .f32 }, + .v_scale_ptr = .{ .ptr = .f32 }, + .out_scale_ptr = .{ .ptr = .f32 }, + .softcap_ptr = .{ .ptr = .f32 }, + .block_table_stride_ptr = .{ .ptr = .i64 }, + .query_stride_0_ptr = .{ .ptr = .i64 }, + .query_stride_1_ptr = .{ .ptr = .i64 }, + .output_stride_0_ptr = .{ .ptr = .i64 }, + .output_stride_1_ptr = .{ .ptr = .i64 }, + .qq_bias_stride_0_ptr = .{ .ptr = .i64 }, + .query_start_len_ptr = .{ .ptr = .i32 }, + .num_seqs_ptr = .{ .ptr = .i32 }, + .output_ptr = .{ .ptr = cfg.o_dtype }, + }); + + const scale = b.load(a.scale_ptr); + const k_scale = b.load(a.k_scale_ptr); + const v_scale = b.load(a.v_scale_ptr); + const out_scale = b.load(a.out_scale_ptr); + const softcap = b.load(a.softcap_ptr); + const block_table_stride = b.load(a.block_table_stride_ptr); + const query_stride_0 = b.load(a.query_stride_0_ptr); + const query_stride_1 = b.load(a.query_stride_1_ptr); + const output_stride_0 = b.load(a.output_stride_0_ptr); + const output_stride_1 = b.load(a.output_stride_1_ptr); + const qq_bias_stride_0 = b.load(a.qq_bias_stride_0_ptr); + const num_seqs = b.load(a.num_seqs_ptr); + + kernelUnifiedAttention2d( + b, + a.output_ptr, + a.query_ptr, + a.key_cache_ptr, + a.value_cache_ptr, + a.sink_ptr, + a.block_tables_ptr, + a.seq_lens_ptr, + a.alibi_slopes_ptr, + a.qq_bias_ptr, + scale, + k_scale, + v_scale, + out_scale, + softcap, + block_table_stride, + query_stride_0, + query_stride_1, + output_stride_0, + output_stride_1, + qq_bias_stride_0, + a.query_start_len_ptr, + num_seqs, + cfg, + ); + } +}; + +/// Direct port of Python `kernel_unified_attention_3d_ptr`. +pub const KernelUnifiedAttention3dPtr = struct { + pub const Config = struct { + q_dtype: DType, + kv_dtype: DType, + k_strides: [3]i64, + v_strides: [3]i64, + scale_dtype: DType = .f32, + + num_query_heads: i64, + num_queries_per_kv: i64, + + block_size: i64, + tile_size: i64, + head_size: i64, + head_size_padded: i64, + + use_alibi_slopes: bool, + use_qq_bias: bool, + use_softcap: bool, + use_sinks: bool, + sliding_window: i64, + + stride_k_cache_3: i64 = 1, + stride_v_cache_3: i64 = 1, + + block_q: i64, + block_m: i64, + num_segments_per_seq: i64, + all_decode: bool = false, + is_causal: bool = true, + }; + + pub const Kernel = tri.Kernel(Config, .{ + .name = "kernel_unified_attention_3d_ptr", + .inputs = &.{ + "query_ptr", "key_cache_ptr", "value_cache_ptr", "sink_ptr", + "block_tables_ptr", "seq_lens_ptr", "alibi_slopes_ptr", "qq_bias_ptr", + "scale_ptr", "k_scale_ptr", "v_scale_ptr", "softcap_ptr", + "block_table_stride_ptr", "query_stride_0_ptr", "query_stride_1_ptr", "qq_bias_stride_0_ptr", + "query_start_len_ptr", "num_seqs_ptr", + }, + .outputs = &.{ "segm_output", "segm_max", "segm_expsum" }, + .run = run, + }); + fn run(b: *Builder, cfg: Config) tri.FinishError!void { + const a = try b.declareArgs(.{ + .query_ptr = .{ .ptr = cfg.q_dtype }, + .key_cache_ptr = .{ .ptr = cfg.kv_dtype }, + .value_cache_ptr = .{ .ptr = cfg.kv_dtype }, + .sink_ptr = .{ .ptr = cfg.scale_dtype }, + .block_tables_ptr = .{ .ptr = .i32 }, + .seq_lens_ptr = .{ .ptr = .i32 }, + .alibi_slopes_ptr = .{ .ptr = cfg.scale_dtype }, + .qq_bias_ptr = .{ .ptr = cfg.scale_dtype }, + .scale_ptr = .{ .ptr = .f32 }, + .k_scale_ptr = .{ .ptr = .f32 }, + .v_scale_ptr = .{ .ptr = .f32 }, + .softcap_ptr = .{ .ptr = .f32 }, + .block_table_stride_ptr = .{ .ptr = .i64 }, + .query_stride_0_ptr = .{ .ptr = .i64 }, + .query_stride_1_ptr = .{ .ptr = .i64 }, + .qq_bias_stride_0_ptr = .{ .ptr = .i64 }, + .query_start_len_ptr = .{ .ptr = .i32 }, + .num_seqs_ptr = .{ .ptr = .i32 }, + .segm_output_ptr = .{ .ptr = .f32 }, + .segm_max_ptr = .{ .ptr = .f32 }, + .segm_expsum_ptr = .{ .ptr = .f32 }, + }); + + const scale = b.load(a.scale_ptr); + const k_scale = b.load(a.k_scale_ptr); + const v_scale = b.load(a.v_scale_ptr); + const softcap = b.load(a.softcap_ptr); + const block_table_stride = b.load(a.block_table_stride_ptr); + const query_stride_0 = b.load(a.query_stride_0_ptr); + const query_stride_1 = b.load(a.query_stride_1_ptr); + const qq_bias_stride_0 = b.load(a.qq_bias_stride_0_ptr); + const num_seqs = b.load(a.num_seqs_ptr); + + kernelUnifiedAttention3d( + b, + a.segm_output_ptr, + a.segm_max_ptr, + a.segm_expsum_ptr, + a.query_ptr, + a.key_cache_ptr, + a.value_cache_ptr, + a.sink_ptr, + a.block_tables_ptr, + a.seq_lens_ptr, + a.alibi_slopes_ptr, + a.qq_bias_ptr, + scale, + k_scale, + v_scale, + softcap, + block_table_stride, + query_stride_0, + query_stride_1, + qq_bias_stride_0, + a.query_start_len_ptr, + num_seqs, + cfg, + ); + } +}; + +/// Direct port of Python `reduce_segments_ptr`. +pub const ReduceSegmentsPtr = struct { + pub const Config = struct { + o_dtype: DType, + scale_dtype: DType = .f32, + + num_query_heads: i64, + tile_size: i64, + head_size: i64, + head_size_padded: i64, + + block_q: i64, + num_segments_per_seq: i64, + + use_fp8: bool, + fp8_min: f32 = FP8_E4M3_MIN, + fp8_max: f32 = FP8_E4M3_MAX, + }; + + pub const Kernel = tri.Kernel(Config, .{ + .name = "reduce_segments_ptr", + .inputs = &.{ + "segm_output_ptr", "segm_max_ptr", "segm_expsum_ptr", "seq_lens_ptr", + "num_seqs_ptr", "out_scale_inv_ptr", "output_stride_0_ptr", "output_stride_1_ptr", + "block_table_stride_ptr", "query_start_len_ptr", + }, + .outputs = &.{"output"}, + .run = run, + }); + fn run(b: *Builder, cfg: Config) tri.FinishError!void { + const a = try b.declareArgs(.{ + .segm_output_ptr = .{ .ptr = .f32 }, + .segm_max_ptr = .{ .ptr = .f32 }, + .segm_expsum_ptr = .{ .ptr = .f32 }, + .seq_lens_ptr = .{ .ptr = .i32 }, + .num_seqs_ptr = .{ .ptr = .i32 }, + .out_scale_inv_ptr = .{ .ptr = .f32 }, + .output_stride_0_ptr = .{ .ptr = .i64 }, + .output_stride_1_ptr = .{ .ptr = .i64 }, + .block_table_stride_ptr = .{ .ptr = .i64 }, + .query_start_len_ptr = .{ .ptr = .i32 }, + .output_ptr = .{ .ptr = cfg.o_dtype }, + }); + + const num_seqs = b.load(a.num_seqs_ptr); + const out_scale_inv = b.load(a.out_scale_inv_ptr); + const output_stride_0 = b.load(a.output_stride_0_ptr); + const output_stride_1 = b.load(a.output_stride_1_ptr); + const block_table_stride = b.load(a.block_table_stride_ptr); + + reduceSegments( + b, + a.output_ptr, + a.segm_output_ptr, + a.segm_max_ptr, + a.segm_expsum_ptr, + a.seq_lens_ptr, + num_seqs, + out_scale_inv, + output_stride_0, + output_stride_1, + block_table_stride, + a.query_start_len_ptr, + cfg, + ); + } +}; + +// ============================================================================ +// Helpers +// ============================================================================ + +fn ctx() *mlir.Context { + return zml.Compiler.current().mlir_ctx; +} + +fn isFp8(dt: DType) bool { + return dt == .f8e4m3fn or dt == .f8e5m2; +} + +fn fastExp(k: *Builder, x: Value) Value { + return k.exp2(x.mul(RCP_LN2)); +} + +fn cdivFn(x: Value, y: anytype) Value { + return switch (@typeInfo(@TypeOf(y))) { + .comptime_int, .comptime_float => x.add(y - 1).div(y), + else => x.add(y).sub(1).div(y), + }; +} + +fn applySoftcap(k: *Builder, s_val: Value, x: Value) Value { + const sdiv = s_val.div(x); + const p1 = k.exp2(sdiv); + const p2 = k.exp2(k.negf(sdiv)); + return x.mul(p1.sub(p2)).div(p1.add(p2)); +} + +fn findSeqIdx( + k: *Builder, + query_start_len_ptr: Value, + target_idx: Value, + num_seqs: Value, + block_q: i64, + use_q_block_mode: bool, +) Value { + const left_init = k.constant(.i32, 0); + + var w = k.openWhile(.{ left_init, num_seqs }, .{ .i32, .i32 }); + { + const left = w.before_carried[0]; + const right = w.before_carried[1]; + w.yieldBefore(left.lt(right), .{ left, right }); + } + { + const left = w.after_carried[0]; + const right = w.after_carried[1]; + const mid = left.add(right).div(2); + const val = k.load(query_start_len_ptr.addPtr(mid)); + const mid_val = if (use_q_block_mode) + val.div(@as(i32, @intCast(block_q))).add(mid) + else + val; + + var i = k.openIfElse(mid_val.le(target_idx), .{ .i32, .i32 }); + { + i.yieldThen(.{ mid.add(1), right }); + } + { + i.yieldElse(.{ left, mid }); + } + w.yieldAfter(.{ i.results[0], i.results[1] }); + } + + return w.results[0].sub(1); +} + +fn dequantKv(loaded: Value, scale: Value, kv_is_fp8: bool, q_dtype: DType) Value { + if (!kv_is_fp8 or isFp8(q_dtype)) return loaded; + return loaded.to(.f32).mul(scale).to(q_dtype); +} + +fn kernelUnifiedAttention2d( + k: *Builder, + output_ptr: Value, + query_ptr: Value, + key_cache_ptr: Value, + value_cache_ptr: Value, + sink_ptr: Value, + block_tables_ptr: Value, + seq_lens_ptr: Value, + alibi_slopes_ptr: Value, + qq_bias_ptr: Value, + scale: Value, + k_scale: Value, + v_scale: Value, + out_scale: Value, + softcap: Value, + block_table_stride: Value, + query_stride_0: Value, + query_stride_1: Value, + output_stride_0: Value, + output_stride_1: Value, + qq_bias_stride_0: Value, + query_start_len_ptr: Value, + num_seqs: Value, + config: KernelUnifiedAttention2dPtr.Config, +) void { + const BLOCK_M: i64 = config.block_m; + const BLOCK_Q: i64 = config.block_q; + const BLOCK_SIZE: i64 = config.block_size; + const TILE_SIZE: i64 = config.tile_size; + const HEAD_SIZE: i64 = config.head_size; + const HEAD_SIZE_PADDED: i64 = config.head_size_padded; + const NUM_QUERIES_PER_KV: i64 = config.num_queries_per_kv; + const NUM_QUERY_HEADS: i64 = config.num_query_heads; + const SLIDING_WINDOW: i64 = config.sliding_window; + const stride_k_cache_0 = k.constant(.i64, config.k_strides[0]); + const stride_k_cache_1 = k.constant(.i64, config.k_strides[1]); + const stride_k_cache_2 = k.constant(.i64, config.k_strides[2]); + const stride_v_cache_0 = k.constant(.i64, config.v_strides[0]); + const stride_v_cache_1 = k.constant(.i64, config.v_strides[1]); + const stride_v_cache_2 = k.constant(.i64, config.v_strides[2]); + + const kv_head_idx = k.programId(.x); + const q_block_global_idx = k.programId(.y); + + const qk_scale = scale.mul(RCP_LN2); + + const seq_idx = findSeqIdx(k, query_start_len_ptr, q_block_global_idx, num_seqs, BLOCK_Q, true); + + const q_start_raw = k.load(query_start_len_ptr.addPtr(seq_idx)); + const q_block_start_idx = q_start_raw.div(@as(i32, @intCast(BLOCK_Q))).add(seq_idx); + + const q_block_local_idx = q_block_global_idx.sub(q_block_start_idx); + + const cur_batch_in_all_start_index = q_start_raw; + // Match Python's left-to-right `query_start_len_ptr + seq_idx + 1`: two `tt.addptr`s. + const cur_batch_in_all_stop_index = k.load(query_start_len_ptr.addPtr(seq_idx).addPtr(1)); + const cur_batch_query_len = cur_batch_in_all_stop_index.sub(cur_batch_in_all_start_index); + + // Python: if q_block_local_idx * BLOCK_Q >= cur_batch_query_len: return + const out_of_range = q_block_local_idx.mul(@as(i32, @intCast(BLOCK_Q))).ge(cur_batch_query_len); + k.returnIf(out_of_range); + + const offs_m = k.arange(0, BLOCK_M, .i32); + const offs_d = k.arange(0, HEAD_SIZE_PADDED, .i32); + const offs_t = k.arange(0, TILE_SIZE, .i32); + + // query_pos = q_block_local_idx * BLOCK_Q + offs_m // NUM_QUERIES_PER_KV + const q_local_bq = q_block_local_idx.mul(@as(i32, @intCast(BLOCK_Q))); + const query_pos = q_local_bq.add(offs_m.div(@as(i32, @intCast(NUM_QUERIES_PER_KV)))); + + const query_offset_0 = cur_batch_in_all_start_index.add(query_pos); + const query_offset_1 = kv_head_idx.mul(@as(i32, @intCast(NUM_QUERIES_PER_KV))) + .add(offs_m.rem(@as(i32, @intCast(NUM_QUERIES_PER_KV)))); + + const qo0_2d = k.expandDims(query_offset_0, 1).mul(query_stride_0); + const qo1_2d = k.expandDims(query_offset_1, 1).mul(query_stride_1); + const query_offset = qo0_2d.add(qo1_2d).add(k.expandDims(offs_d, 0)); + + const dim_mask: Value = if (HEAD_SIZE_PADDED != HEAD_SIZE) + offs_d.lt(@as(i32, @intCast(HEAD_SIZE))) + else + k.full(&.{HEAD_SIZE_PADDED}, 1, .i1); + const query_mask_0 = query_pos.lt(cur_batch_query_len); + const query_mask_1 = query_offset_1.lt(@as(i32, @intCast(NUM_QUERY_HEADS))); + + // Q : (BLOCK_M, HEAD_SIZE_PADDED) + const q_mask_ab = k.mask2d(query_mask_0, dim_mask, BLOCK_M, HEAD_SIZE_PADDED); + const q_mask_c = query_mask_1.expandDims(1); + const q_mask = q_mask_ab.bitAnd(q_mask_c); + const q_cache_mod: ttir.CacheModifier = + if (config.all_decode or BLOCK_M >= NUM_QUERY_HEADS) .cg else .none; + const Q = k.loadOpts(query_ptr.addPtr(query_offset), .{ + .mask = q_mask, + .other = k.zeros(&.{ BLOCK_M, HEAD_SIZE_PADDED }, config.q_dtype), + .cache_modifier = q_cache_mod, + }); + + const block_table_offset = seq_idx.to(.i64).mul(block_table_stride); + + // M init — sinks or -inf. + const m_init: Value = if (config.use_sinks) mb: { + const loaded = k.loadOpts(sink_ptr.addPtr(query_offset_1), .{ + .mask = query_mask_1, + .other = k.full(&.{BLOCK_M}, -std.math.inf(f32), config.scale_dtype), + }); + break :mb loaded.to(.f32).mul(RCP_LN2); + } else k.full(&.{BLOCK_M}, -std.math.inf(f32), .f32); + + const l_init = k.full(&.{BLOCK_M}, 1.0, .f32); + const acc_init = k.zeros(&.{ BLOCK_M, HEAD_SIZE_PADDED }, .f32); + + const seq_len = k.load(seq_lens_ptr.addPtr(seq_idx)); + const context_len = seq_len.sub(cur_batch_query_len); + + const alibi_slope: Value = if (config.use_alibi_slopes) + k.loadOpts(alibi_slopes_ptr.addPtr(query_offset_1), .{ + .mask = query_mask_1, + .other = k.full(&.{BLOCK_M}, 0.0, config.scale_dtype), + }) + else + k.zeros(&.{BLOCK_M}, config.scale_dtype); + + // In the `use_qq_bias = true` branch, the tensor offset auto-splats the + // scalar ptr via addPtr. In the else branch there is no offset, so we + // need an explicit splatTo to give expandDims a tensor to work on. + const qq_bias_row_ptrs: Value = if (config.use_qq_bias) + qq_bias_ptr.addPtr(query_pos.to(.i64).mul(qq_bias_stride_0)) + else + qq_bias_ptr.splatTo(&.{BLOCK_M}); + + // In causal mode, each query block only scans keys up to the last query + // represented in this block. In non-causal mode, scan the whole sequence + // so draft rows can attend to all rows written for the same sequence. + const pad_term: i32 = @intCast(@divTrunc(BLOCK_M - 1, NUM_QUERIES_PER_KV) + 1); + const max_seq_prefix_len = if (config.is_causal) b: { + const max_prefix_raw = context_len.add(q_local_bq).add(pad_term); + break :b max_prefix_raw.minimum(seq_len); + } else seq_len; + + const num_tiles = cdivFn(max_seq_prefix_len, @as(i32, @intCast(TILE_SIZE))); + + // Sliding-window tile pruning — comptime branch. + var tile_start: Value = k.constant(.i32, 0); + var tile_end: Value = num_tiles; + if (SLIDING_WINDOW > 0) { + const qpos_lo = q_local_bq; + if (config.is_causal) { + const qpos_hi_raw = qpos_lo.add(pad_term - 1); + const qpos_hi = qpos_hi_raw.minimum(cur_batch_query_len.sub(1)); + const first_allowed_key = context_len.add(qpos_lo).sub(@as(i32, @intCast(SLIDING_WINDOW - 1))); + const last_allowed_key = context_len.add(qpos_hi); + tile_start = first_allowed_key.div(@as(i32, @intCast(TILE_SIZE))).maximum(0); + tile_end = last_allowed_key.div(@as(i32, @intCast(TILE_SIZE))).add(1).minimum(num_tiles); + } else { + const first_allowed_key = context_len.add(qpos_lo).sub(@as(i32, @intCast(SLIDING_WINDOW - 1))); + tile_start = first_allowed_key.div(@as(i32, @intCast(TILE_SIZE))).maximum(0); + } + } + + const kv_cache_mod: ttir.CacheModifier = + if (config.all_decode) .cg else .none; + + const block_tables_ptr_shifted = block_tables_ptr.addPtr(block_table_offset); + + // Iterate through tiles (loop carries M, L, acc). + var loop = k.openFor(tile_start, tile_end, 1, .{ m_init, l_init, acc_init }); + { + const j = loop.iv; + const M = loop.carried[0]; + const L = loop.carried[1]; + const acc = loop.carried[2]; + + const seq_offset = j.mul(@as(i32, @intCast(TILE_SIZE))).add(offs_t); + + const tile_mask: Value = seq_offset.lt(max_seq_prefix_len); + + const physical_block_idx = k.load( + block_tables_ptr_shifted.addPtr(seq_offset.div(@as(i32, @intCast(BLOCK_SIZE)))), + ).to(.i64); + + const v_offset = k.expandDims(physical_block_idx, 1).mul(stride_v_cache_0) + .add(kv_head_idx.to(.i64).mul(stride_v_cache_2)) + .add(k.expandDims(offs_d, 0).mul(config.stride_v_cache_3)) + .add(k.expandDims(seq_offset.rem(@as(i32, @intCast(BLOCK_SIZE))), 1).mul(stride_v_cache_1)); + + // k_offset : (HEAD_SIZE_PADDED, TILE_SIZE) + const k_offset = k.expandDims(physical_block_idx, 0).mul(stride_k_cache_0) + .add(kv_head_idx.to(.i64).mul(stride_k_cache_2)) + .add(k.expandDims(offs_d, 1).mul(config.stride_k_cache_3)) + .add(k.expandDims(seq_offset.rem(@as(i32, @intCast(BLOCK_SIZE))), 0).mul(stride_k_cache_1)); + + // K : (HEAD_SIZE_PADDED, TILE_SIZE) + const k_mask = k.mask2d(dim_mask, tile_mask, HEAD_SIZE_PADDED, TILE_SIZE); + const K_load = k.loadOpts(key_cache_ptr.addPtr(k_offset), .{ + .mask = k_mask, + .other = k.zeros(&.{ HEAD_SIZE_PADDED, TILE_SIZE }, config.kv_dtype), + .cache_modifier = kv_cache_mod, + }); + const K = dequantKv(K_load, k_scale, isFp8(config.kv_dtype), config.q_dtype); + + // V : (TILE_SIZE, HEAD_SIZE_PADDED) + const v_mask = k.mask2d(tile_mask, dim_mask, TILE_SIZE, HEAD_SIZE_PADDED); + const V_load = k.loadOpts(value_cache_ptr.addPtr(v_offset), .{ + .mask = v_mask, + .other = k.zeros(&.{ TILE_SIZE, HEAD_SIZE_PADDED }, config.kv_dtype), + .cache_modifier = kv_cache_mod, + }); + const V = dequantKv(V_load, v_scale, isFp8(config.kv_dtype), config.q_dtype); + + // S : (BLOCK_M, TILE_SIZE) + const acc_zero = k.zeros(&.{ BLOCK_M, TILE_SIZE }, .f32); + const qk = k.dot(Q, K, acc_zero); + var S = qk_scale.mul(qk); + + if (config.use_softcap) { + S = applySoftcap(k, S, softcap).mul(RCP_LN2); + } + + const ql_2d_i32 = query_pos.expandDims(1); + const so_2d = seq_offset.expandDims(0); + const seq_mask = if (config.is_causal) b: { + // seq_offset[None, :] < context_len + query_pos[:, None] + 1 + const rhs = ql_2d_i32.add(context_len).add(1); + break :b so_2d.lt(rhs); + } else k.broadcastTo(tile_mask.expandDims(0), &.{ BLOCK_M, TILE_SIZE }); + + // S = tl.where(qmask_1 & qmask_0 & seq_mask, S, -inf) + const qm0_2d = query_mask_0.expandDims(1); + const qm1_2d = query_mask_1.expandDims(1); + const keep_mask = qm1_2d.bitAnd(qm0_2d).bitAnd(seq_mask); + S = k.where(keep_mask, S, k.full(&.{ BLOCK_M, TILE_SIZE }, -std.math.inf(f32), .f32)); + + if (SLIDING_WINDOW > 0) { + const diff = ql_2d_i32.add(context_len).sub(so_2d); + const in_win = diff.lt(@as(i32, @intCast(SLIDING_WINDOW))); + S = k.where(in_win, S, k.full(&.{ BLOCK_M, TILE_SIZE }, -std.math.inf(f32), .f32)); + } + + if (config.use_alibi_slopes) { + const alibi_2d = alibi_slope.expandDims(1); + const pos_diff = seq_offset.sub(context_len).to(config.scale_dtype).expandDims(0); + S = S.add(alibi_2d.mul(pos_diff).to(.f32).mul(RCP_LN2)); + } + + if (config.use_qq_bias) { + const key_rel_pos = seq_offset.sub(context_len); + const is_query_key = key_rel_pos.ge(0) + .bitAnd(key_rel_pos.to(.i64).lt(qq_bias_stride_0)); + const qq_ptrs = qq_bias_row_ptrs.expandDims(1) + .addPtr(key_rel_pos.to(.i64).expandDims(0)); + const qq_bias = k.loadOpts(qq_ptrs, .{ + .mask = is_query_key.expandDims(0), + .other = k.zeros(&.{ BLOCK_M, TILE_SIZE }, config.scale_dtype), + }); + S = S.add(qq_bias.to(.f32).mul(RCP_LN2)); + } + + // m_j = max(M, max(S, axis=1)) + var m_j = M.maximum(k.maxOpts(S, .{ .axis = 1 })); + m_j = k.where(m_j.gt(-std.math.inf(f32)), m_j, k.full(&.{BLOCK_M}, 0.0, .f32)); + + // P = exp2(S - m_j[:, None]); l_j = sum(P, axis=1); alpha = exp2(M - m_j) + const mj_2d = m_j.expandDims(1); + const P = k.exp2(S.sub(mj_2d)); + const l_j = k.sumOpts(P, .{ .axis = 1 }); + const alpha = k.exp2(M.sub(m_j)); + + const alpha_2d = alpha.expandDims(1); + const acc_scaled = acc.mul(alpha_2d); + const new_L = L.mul(alpha).add(l_j); + const new_M = m_j; + + const P_cast = P.to(config.q_dtype); + const new_acc = k.dot(P_cast, V, acc_scaled); + + loop.yield(.{ new_M, new_L, new_acc }); + } + const L_final = loop.results[1]; + var acc_final = loop.results[2]; + + const L_2d = k.expandDims(L_final, 1); + const one_over_L = k.full(&.{ BLOCK_M, 1 }, 1.0, .f32).div(L_2d); + acc_final = acc_final.mul(k.broadcastTo(one_over_L, &.{ BLOCK_M, HEAD_SIZE_PADDED })); + + if (config.use_fp8) { + acc_final = acc_final.mul(out_scale); + acc_final = k.clampf( + acc_final, + k.full(&.{ BLOCK_M, HEAD_SIZE_PADDED }, config.fp8_min, .f32), + k.full(&.{ BLOCK_M, HEAD_SIZE_PADDED }, config.fp8_max, .f32), + ); + } + + const oo0_2d = query_offset_0.expandDims(1).mul(output_stride_0); + const oo1_2d = query_offset_1.expandDims(1).mul(output_stride_1); + const output_offset = oo0_2d.add(oo1_2d).add(k.expandDims(offs_d, 0)); + + const store_mask = k.mask2d(query_mask_0, dim_mask, BLOCK_M, HEAD_SIZE_PADDED) + .bitAnd(query_mask_1.expandDims(1)); + k.storeOpts( + output_ptr.addPtr(output_offset), + acc_final.to(config.o_dtype), + .{ .mask = store_mask }, + ); +} + +fn kernelUnifiedAttention3d( + k: *Builder, + segm_output_ptr: Value, + segm_max_ptr: Value, + segm_expsum_ptr: Value, + query_ptr: Value, + key_cache_ptr: Value, + value_cache_ptr: Value, + sink_ptr: Value, + block_tables_ptr: Value, + seq_lens_ptr: Value, + alibi_slopes_ptr: Value, + qq_bias_ptr: Value, + scale: Value, + k_scale: Value, + v_scale: Value, + softcap: Value, + block_table_stride: Value, + query_stride_0: Value, + query_stride_1: Value, + qq_bias_stride_0: Value, + query_start_len_ptr: Value, + num_seqs: Value, + config: KernelUnifiedAttention3dPtr.Config, +) void { + const BLOCK_M: i64 = config.block_m; + const BLOCK_Q: i64 = config.block_q; + const BLOCK_SIZE: i64 = config.block_size; + const TILE_SIZE: i64 = config.tile_size; + const HEAD_SIZE: i64 = config.head_size; + const HEAD_SIZE_PADDED: i64 = config.head_size_padded; + const NUM_QUERIES_PER_KV: i64 = config.num_queries_per_kv; + const NUM_QUERY_HEADS: i64 = config.num_query_heads; + const NUM_SEGMENTS_PER_SEQ: i64 = config.num_segments_per_seq; + const SLIDING_WINDOW: i64 = config.sliding_window; + const stride_k_cache_0 = k.constant(.i64, config.k_strides[0]); + const stride_k_cache_1 = k.constant(.i64, config.k_strides[1]); + const stride_k_cache_2 = k.constant(.i64, config.k_strides[2]); + const stride_v_cache_0 = k.constant(.i64, config.v_strides[0]); + const stride_v_cache_1 = k.constant(.i64, config.v_strides[1]); + const stride_v_cache_2 = k.constant(.i64, config.v_strides[2]); + + const q_block_global_idx = k.programId(.x); + const kv_head_idx = k.programId(.y); + const segm_idx = k.programId(.z); + + const qk_scale = scale.mul(RCP_LN2); + + const seq_idx = findSeqIdx(k, query_start_len_ptr, q_block_global_idx, num_seqs, BLOCK_Q, true); + + const q_start_raw = k.load(query_start_len_ptr.addPtr(seq_idx)); + const q_block_start_idx = q_start_raw.div(@as(i32, @intCast(BLOCK_Q))).add(seq_idx); + const q_block_local_idx = q_block_global_idx.sub(q_block_start_idx); + + const cur_batch_in_all_start_index = q_start_raw; + // Match Python's left-to-right `query_start_len_ptr + seq_idx + 1`: two `tt.addptr`s. + const cur_batch_in_all_stop_index = k.load(query_start_len_ptr.addPtr(seq_idx).addPtr(1)); + const cur_batch_query_len = cur_batch_in_all_stop_index.sub(cur_batch_in_all_start_index); + + const q_out_of_range = q_block_local_idx.mul(@as(i32, @intCast(BLOCK_Q))).ge(cur_batch_query_len); + k.returnIf(q_out_of_range); + + const seq_len = k.load(seq_lens_ptr.addPtr(seq_idx)); + const tiles_per_segment = cdivFn(seq_len, @as(i32, @intCast(NUM_SEGMENTS_PER_SEQ * TILE_SIZE))); + + const segm_lo = segm_idx.mul(tiles_per_segment).mul(@as(i32, @intCast(TILE_SIZE))); + const segm_out_of_range = segm_lo.ge(seq_len); + k.returnIf(segm_out_of_range); + + const offs_m = k.arange(0, BLOCK_M, .i32); + const offs_d = k.arange(0, HEAD_SIZE_PADDED, .i32); + const offs_t = k.arange(0, TILE_SIZE, .i32); + + const q_local_bq = q_block_local_idx.mul(@as(i32, @intCast(BLOCK_Q))); + const query_pos = q_local_bq.add(offs_m.div(@as(i32, @intCast(NUM_QUERIES_PER_KV)))); + + const query_offset_0 = cur_batch_in_all_start_index.add(query_pos); + const query_offset_1 = kv_head_idx.mul(@as(i32, @intCast(NUM_QUERIES_PER_KV))) + .add(offs_m.rem(@as(i32, @intCast(NUM_QUERIES_PER_KV)))); + + // query_offset = qo_0[:, None]*qstride_0 + qo_1[:, None]*qstride_1 + offs_d[None, :] + // Inline `expandDims(offs_d, 0)` *inside* the chain so it's emitted after + // the qo0+qo1 addi + const qo0_2d = k.expandDims(query_offset_0, 1).mul(query_stride_0); + const qo1_2d = k.expandDims(query_offset_1, 1).mul(query_stride_1); + const query_offset = qo0_2d.add(qo1_2d).add(k.expandDims(offs_d, 0)); + + const dim_mask: Value = if (HEAD_SIZE_PADDED != HEAD_SIZE) + offs_d.lt(@as(i32, @intCast(HEAD_SIZE))) + else + k.full(&.{HEAD_SIZE_PADDED}, 1, .i1); + const query_mask_0 = query_pos.lt(cur_batch_query_len); + const query_mask_1 = query_offset_1.lt(@as(i32, @intCast(NUM_QUERY_HEADS))); + + const q_mask_pre = k.expandDims(query_mask_0, 1).bitAnd(k.expandDims(query_mask_1, 1)); + const q_mask_partial: Value = if (HEAD_SIZE_PADDED != HEAD_SIZE) + q_mask_pre.bitAnd(k.expandDims(dim_mask, 0)) + else + q_mask_pre; + const q_ptrs = query_ptr.addPtr(query_offset); + const Q = k.loadOpts(q_ptrs, .{ + .mask = k.broadcastTo(q_mask_partial, &.{ BLOCK_M, HEAD_SIZE_PADDED }), + .other = k.zeros(&.{ BLOCK_M, HEAD_SIZE_PADDED }, config.q_dtype), + }); + + const block_table_offset = seq_idx.to(.i64).mul(block_table_stride); + + // M init: USE_SINKS + segm_idx==0 → loaded sinks; else -inf. + const segm_is_zero = segm_idx.eq(0); + const m_init: Value = if (config.use_sinks) mb: { + var mi = k.openIfElse(segm_is_zero, .{k.tensorTy(&.{BLOCK_M}, .f32)}); + { + const loaded = k.loadOpts(sink_ptr.addPtr(query_offset_1), .{ + .mask = query_mask_1, + .other = k.full(&.{BLOCK_M}, -std.math.inf(f32), config.scale_dtype), + }); + mi.yieldThen(.{loaded.to(.f32).mul(RCP_LN2)}); + } + { + mi.yieldElse(.{k.full(&.{BLOCK_M}, -std.math.inf(f32), .f32)}); + } + break :mb mi.results[0]; + } else k.full(&.{BLOCK_M}, -std.math.inf(f32), .f32); + + const l_init = k.full(&.{BLOCK_M}, 1.0, .f32); + const acc_init = k.zeros(&.{ BLOCK_M, HEAD_SIZE_PADDED }, .f32); + + const context_len = seq_len.sub(cur_batch_query_len); + + const alibi_slope: Value = if (config.use_alibi_slopes) + k.loadOpts(alibi_slopes_ptr.addPtr(query_offset_1), .{ + .mask = query_mask_1, + .other = k.full(&.{BLOCK_M}, 0.0, config.scale_dtype), + }) + else + k.zeros(&.{BLOCK_M}, config.scale_dtype); + + // See 2d-body note on qq_bias_row_ptrs: else branch needs an explicit splatTo + // (no sibling offset to auto-broadcast against). + const qq_bias_row_ptrs: Value = if (config.use_qq_bias) + qq_bias_ptr.addPtr(query_pos.to(.i64).mul(qq_bias_stride_0)) + else + qq_bias_ptr.splatTo(&.{BLOCK_M}); + + const pad_term: i32 = @intCast(@divTrunc(BLOCK_M - 1, NUM_QUERIES_PER_KV) + 1); + const max_seq_prefix_len = if (config.is_causal) b: { + const max_prefix_raw = context_len.add(q_local_bq).add(pad_term); + break :b max_prefix_raw.minimum(seq_len); + } else seq_len; + const num_tiles = cdivFn(max_seq_prefix_len, @as(i32, @intCast(TILE_SIZE))); + + const segm_tile_lo = segm_idx.mul(tiles_per_segment); + const segm_tile_hi = segm_idx.add(1).mul(tiles_per_segment).minimum(num_tiles); + + const kv_cache_mod: ttir.CacheModifier = + if (config.all_decode) .cg else .none; + + var loop = k.openFor(segm_tile_lo, segm_tile_hi, 1, .{ m_init, l_init, acc_init }); + { + const j = loop.iv; + const M = loop.carried[0]; + const L = loop.carried[1]; + const acc = loop.carried[2]; + + const seq_offset = j.mul(@as(i32, @intCast(TILE_SIZE))).add(offs_t); + const tile_mask: Value = seq_offset.lt(max_seq_prefix_len); + + const physical_block_idx = k.load( + block_tables_ptr.addPtr(block_table_offset).addPtr(seq_offset.div(@as(i32, @intCast(BLOCK_SIZE)))), + ).to(.i64); + const v_offset = k.expandDims(physical_block_idx, 1).mul(stride_v_cache_0) + .add(kv_head_idx.to(.i64).mul(stride_v_cache_2)) + .add(k.expandDims(offs_d, 0).mul(config.stride_v_cache_3)) + .add(k.expandDims(seq_offset.rem(@as(i32, @intCast(BLOCK_SIZE))), 1).mul(stride_v_cache_1)); + + const k_offset = k.expandDims(physical_block_idx, 0).mul(stride_k_cache_0) + .add(kv_head_idx.to(.i64).mul(stride_k_cache_2)) + .add(k.expandDims(offs_d, 1).mul(config.stride_k_cache_3)) + .add(k.expandDims(seq_offset.rem(@as(i32, @intCast(BLOCK_SIZE))), 0).mul(stride_k_cache_1)); + + const k_mask_partial: Value = if (HEAD_SIZE_PADDED != HEAD_SIZE) + k.expandDims(dim_mask, 1).bitAnd(k.expandDims(tile_mask, 0)) + else + k.expandDims(tile_mask, 0); + const K_load = k.loadOpts(key_cache_ptr.addPtr(k_offset), .{ + .mask = k.broadcastTo(k_mask_partial, &.{ HEAD_SIZE_PADDED, TILE_SIZE }), + .other = k.zeros(&.{ HEAD_SIZE_PADDED, TILE_SIZE }, config.kv_dtype), + .cache_modifier = kv_cache_mod, + }); + const K = dequantKv(K_load, k_scale, isFp8(config.kv_dtype), config.q_dtype); + + const v_mask_partial: Value = if (HEAD_SIZE_PADDED != HEAD_SIZE) + k.expandDims(tile_mask, 1).bitAnd(k.expandDims(dim_mask, 0)) + else + k.expandDims(tile_mask, 1); + const V_load = k.loadOpts(value_cache_ptr.addPtr(v_offset), .{ + .mask = k.broadcastTo(v_mask_partial, &.{ TILE_SIZE, HEAD_SIZE_PADDED }), + .other = k.zeros(&.{ TILE_SIZE, HEAD_SIZE_PADDED }, config.kv_dtype), + .cache_modifier = kv_cache_mod, + }); + const V = dequantKv(V_load, v_scale, isFp8(config.kv_dtype), config.q_dtype); + + const ql_2d_i32 = query_pos.expandDims(1); + const so_2d = seq_offset.expandDims(0); + const seq_mask = if (config.is_causal) b: { + // seq_offset[None, :] < context_len + query_pos[:, None] + 1 + const rhs = context_len.add(ql_2d_i32).add(1); + break :b so_2d.lt(rhs); + } else k.broadcastTo(tile_mask.expandDims(0), &.{ BLOCK_M, TILE_SIZE }); + + const acc_zero = k.zeros(&.{ BLOCK_M, TILE_SIZE }, .f32); + const qk = k.dot(Q, K, acc_zero); + var S = qk_scale.mul(qk); + + if (config.use_softcap) { + S = applySoftcap(k, S, softcap).mul(RCP_LN2); + } + + const qm0_2d = query_mask_0.expandDims(1); + const qm1_2d = query_mask_1.expandDims(1); + const keep_mask = qm1_2d.bitAnd(qm0_2d).bitAnd(seq_mask); + S = k.where(keep_mask, S, k.full(&.{ BLOCK_M, TILE_SIZE }, -std.math.inf(f32), .f32)); + + if (SLIDING_WINDOW > 0) { + const diff = ql_2d_i32.add(context_len).sub(so_2d); + const in_win = diff.lt(@as(i32, @intCast(SLIDING_WINDOW))); + S = k.where(in_win, S, k.full(&.{ BLOCK_M, TILE_SIZE }, -std.math.inf(f32), .f32)); + } + + if (config.use_alibi_slopes) { + const alibi_2d = alibi_slope.expandDims(1); + const pos_diff = seq_offset.sub(context_len).to(config.scale_dtype).expandDims(0); + S = S.add(alibi_2d.mul(pos_diff).to(.f32).mul(RCP_LN2)); + } + + if (config.use_qq_bias) { + const key_rel_pos = seq_offset.sub(context_len); + const is_query_key = key_rel_pos.ge(0) + .bitAnd(key_rel_pos.to(.i64).lt(qq_bias_stride_0)); + const qq_ptrs = qq_bias_row_ptrs.expandDims(1) + .addPtr(key_rel_pos.to(.i64).expandDims(0)); + const qq_bias = k.loadOpts(qq_ptrs, .{ + .mask = is_query_key.expandDims(0), + .other = k.zeros(&.{ BLOCK_M, TILE_SIZE }, config.scale_dtype), + }); + S = S.add(qq_bias.to(.f32).mul(RCP_LN2)); + } + + var m_j = M.maximum(k.maxOpts(S, .{ .axis = 1 })); + m_j = k.where(m_j.gt(-std.math.inf(f32)), m_j, k.full(&.{BLOCK_M}, 0.0, .f32)); + + const mj_2d = m_j.expandDims(1); + const P = k.exp2(S.sub(mj_2d)); + const l_j = k.sumOpts(P, .{ .axis = 1 }); + const alpha = k.exp2(M.sub(m_j)); + + const alpha_2d = alpha.expandDims(1); + const acc_scaled = acc.mul(alpha_2d); + const new_L = L.mul(alpha).add(l_j); + + const P_cast = P.to(config.q_dtype); + const new_acc = k.dot(P_cast, V, acc_scaled); + + loop.yield(.{ m_j, new_L, new_acc }); + } + const M_final = loop.results[0]; + const L_final = loop.results[1]; + const acc_final = loop.results[2]; + + // segm_output_offset = qo_0[:, None].to(i64) * (NUM_QUERY_HEADS * NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED) + // + qo_1[:, None] * (NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED) + // + segm_idx * HEAD_SIZE_PADDED + offs_d[None, :] + const segm_out_head_stride_i32: i32 = @intCast(NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED); + const segm_out_token_stride_i64: i64 = NUM_QUERY_HEADS * NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED; + + // Inline `segm_idx * HEAD_SIZE_PADDED` and `offs_d[None, :]` inside the chain + // so each term's ops emit in source order + const oo0_2d = k.expandDims(query_offset_0, 1).to(.i64).mul(segm_out_token_stride_i64); + const oo1_2d = k.expandDims(query_offset_1, 1).mul(segm_out_head_stride_i32); + const segm_output_offset = oo0_2d.add(oo1_2d) + .add(segm_idx.mul(@as(i32, @intCast(HEAD_SIZE_PADDED)))) + .add(k.expandDims(offs_d, 0)); + + const store_mask_2d = k.mask2d(query_mask_0, dim_mask, BLOCK_M, HEAD_SIZE_PADDED) + .bitAnd(query_mask_1.expandDims(1)); + k.storeOpts( + segm_output_ptr.addPtr(segm_output_offset), + acc_final, + .{ .mask = store_mask_2d }, + ); + + // segm_offset = qo_0.to(i64) * (NUM_QUERY_HEADS * NUM_SEGMENTS_PER_SEQ) + // + qo_1 * NUM_SEGMENTS_PER_SEQ + segm_idx + const segm_head_stride_i32: i32 = @intCast(NUM_SEGMENTS_PER_SEQ); + const segm_token_stride_i64: i64 = NUM_QUERY_HEADS * NUM_SEGMENTS_PER_SEQ; + const segm_offset = query_offset_0.to(.i64).mul(segm_token_stride_i64) + .add(query_offset_1.mul(segm_head_stride_i32)) + .add(segm_idx); + const store_mask_1d = query_mask_0.bitAnd(query_mask_1); + k.storeOpts(segm_max_ptr.addPtr(segm_offset), M_final, .{ .mask = store_mask_1d }); + k.storeOpts(segm_expsum_ptr.addPtr(segm_offset), L_final, .{ .mask = store_mask_1d }); +} + +fn reduceSegments( + k: *Builder, + output_ptr: Value, + segm_output_ptr: Value, + segm_max_ptr: Value, + segm_expsum_ptr: Value, + seq_lens_ptr: Value, + num_seqs: Value, + out_scale_inv: Value, + output_stride_0: Value, + output_stride_1: Value, + block_table_stride: Value, + query_start_len_ptr: Value, + config: ReduceSegmentsPtr.Config, +) void { + _ = block_table_stride; + + const TILE_SIZE: i64 = config.tile_size; + const HEAD_SIZE: i64 = config.head_size; + const HEAD_SIZE_PADDED: i64 = config.head_size_padded; + const NUM_QUERY_HEADS: i64 = config.num_query_heads; + const NUM_SEGMENTS_PER_SEQ: i64 = config.num_segments_per_seq; + const BLOCK_Q: i64 = config.block_q; + + const query_token_idx = k.programId(.x); + const query_head_idx = k.programId(.y); + + const offs_d = k.arange(0, HEAD_SIZE_PADDED, .i32); + const dim_mask: Value = if (HEAD_SIZE_PADDED != HEAD_SIZE) + offs_d.lt(@as(i32, @intCast(HEAD_SIZE))) + else + k.full(&.{HEAD_SIZE_PADDED}, 1, .i1); + const output_offset = query_token_idx.to(.i64).mul(output_stride_0) + .add(query_head_idx.to(.i64).mul(output_stride_1)) + .add(offs_d.to(.i64)); + + // The launch covers the padded query capacity, but the attention pass only + // writes scratch for active tokens. Zero the tail before looking up a + // sequence or dividing by its segment size (zero for inactive sequences). + const active_tokens = k.load(query_start_len_ptr.addPtr(num_seqs)); + const inactive = query_token_idx.ge(active_tokens); + k.storeOpts(output_ptr.addPtr(output_offset), k.zeros(&.{HEAD_SIZE_PADDED}, config.o_dtype), .{ + .mask = dim_mask.bitAnd(inactive), + }); + k.returnIf(inactive); + + const seq_idx = findSeqIdx(k, query_start_len_ptr, query_token_idx, num_seqs, BLOCK_Q, false); + + const seq_len = k.load(seq_lens_ptr.addPtr(seq_idx)); + const tiles_per_segment = cdivFn(seq_len, @as(i32, @intCast(NUM_SEGMENTS_PER_SEQ * TILE_SIZE))); + const act_num_segments = cdivFn(seq_len, tiles_per_segment.mul(@as(i32, @intCast(TILE_SIZE)))); + + const seg_range = k.arange(0, NUM_SEGMENTS_PER_SEQ, .i32); + const segm_mask = seg_range.lt(act_num_segments); + + // segm_offset = query_token_idx*NQH*NSPS + query_head_idx*NSPS + arange(NSPS) + const tok_stride: i64 = NUM_QUERY_HEADS * NUM_SEGMENTS_PER_SEQ; + const head_stride: i32 = @intCast(NUM_SEGMENTS_PER_SEQ); + const segm_offset = query_token_idx.to(.i64).mul(tok_stride) + .add(query_head_idx.mul(head_stride).to(.i64)) + .add(seg_range.to(.i64)); + + const segm_max = k.loadOpts(segm_max_ptr.addPtr(segm_offset), .{ + .mask = segm_mask, + .other = k.full(&.{NUM_SEGMENTS_PER_SEQ}, -std.math.inf(f32), .f32), + }); + const overall_max = k.max(segm_max); + + var segm_expsum = k.loadOpts(segm_expsum_ptr.addPtr(segm_offset), .{ + .mask = segm_mask, + .other = k.zeros(&.{NUM_SEGMENTS_PER_SEQ}, .f32), + }); + segm_expsum = segm_expsum.mul(k.exp2(segm_max.sub(overall_max))); + const overall_expsum = k.sum(segm_expsum); + + // segm_output_offset : (NSPS, HSP) + const out_tok_stride: i64 = NUM_QUERY_HEADS * NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED; + const out_head_stride: i32 = @intCast(NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED); + const head_size_padded_i32: i32 = @intCast(HEAD_SIZE_PADDED); + const base = query_token_idx.to(.i64).mul(out_tok_stride) + .add(query_head_idx.mul(out_head_stride).to(.i64)); + const seg_off_2d = b: { + const seg_col_i32 = k.expandDims(seg_range, 1); // (NSPS, 1) i32 + const seg_col_scaled = seg_col_i32.mul(head_size_padded_i32); // (NSPS, 1) i32 + break :b seg_col_scaled.to(.i64); // extsi → (NSPS, 1) i64 + }; + const dim_off_2d = b: { + const dim_row_i32 = k.expandDims(offs_d, 0); // (1, HSP) i32 + break :b dim_row_i32.to(.i64); // extsi → (1, HSP) i64 + }; + const seg_offset_2d_full = k.broadcastTo(base.add(seg_off_2d), &.{ NUM_SEGMENTS_PER_SEQ, HEAD_SIZE_PADDED }); + const dim_offset_2d_full = k.broadcastTo(dim_off_2d, &.{ NUM_SEGMENTS_PER_SEQ, HEAD_SIZE_PADDED }); + const segm_output_offset = seg_offset_2d_full.add(dim_offset_2d_full); + + var segm_output = k.loadOpts( + segm_output_ptr.addPtr(segm_output_offset), + .{ + .mask = k.mask2d(segm_mask, dim_mask, NUM_SEGMENTS_PER_SEQ, HEAD_SIZE_PADDED), + .other = k.zeros(&.{ NUM_SEGMENTS_PER_SEQ, HEAD_SIZE_PADDED }, .f32), + }, + ); + const rescale = k.exp2(segm_max.sub(overall_max)); + segm_output = segm_output.mul(rescale.expandDims(1)); + const acc_sum = k.sumOpts(segm_output, .{ .axis = 0 }); + + var acc = k.where( + overall_expsum.eq(0.0), + k.full(&.{HEAD_SIZE_PADDED}, 0.0, .f32), + acc_sum.div(overall_expsum), + ); + + if (config.use_fp8) { + acc = acc.mul(out_scale_inv); + acc = k.clampf( + acc, + k.full(&.{HEAD_SIZE_PADDED}, config.fp8_min, .f32), + k.full(&.{HEAD_SIZE_PADDED}, config.fp8_max, .f32), + ); + } + + k.storeOpts( + output_ptr.addPtr(output_offset), + acc.to(config.o_dtype), + .{ .mask = dim_mask }, + ); +} diff --git a/zml/kernels/paged_attention/triton/unified_attention_oneapi.zig b/zml/kernels/paged_attention/triton/unified_attention_oneapi.zig new file mode 100644 index 00000000..b7c724f9 --- /dev/null +++ b/zml/kernels/paged_attention/triton/unified_attention_oneapi.zig @@ -0,0 +1,536 @@ +const std = @import("std"); + +const mlir = @import("mlir"); +const ttir = @import("mlir/dialects/ttir"); + +const zml = @import("../../../zml.zig"); +const tri = zml.kernel.triton; +const Builder = tri.Builder; +const Value = tri.Value; +const DType = tri.DType; + +const log = std.log.scoped(.@"zml/attention/triton"); + +// oneAPI/Intel (SIMD16) specialization of the 3D split-K paged-attention kernel. +// Duplicated from unified_attention.zig so the shared kernel stays backend-neutral; +// only the Intel decode path routes here (see triton_attention.zig pagedAttention3dOneapi). + +/// `log_2(e)` — pre-multiply with this to turn `exp(x)` into `exp2(x * RCP_LN2)`. +const RCP_LN2: f32 = 1.4426950408889634; + +fn isFp8(dt: DType) bool { + return dt == .f8e4m3fn or dt == .f8e5m2; +} + +fn cdivFn(x: Value, y: anytype) Value { + return switch (@typeInfo(@TypeOf(y))) { + .comptime_int, .comptime_float => x.add(y - 1).div(y), + else => x.add(y).sub(1).div(y), + }; +} + +fn applySoftcap(k: *Builder, s_val: Value, x: Value) Value { + const sdiv = s_val.div(x); + const p1 = k.exp2(sdiv); + const p2 = k.exp2(k.negf(sdiv)); + return x.mul(p1.sub(p2)).div(p1.add(p2)); +} + +fn findSeqIdx( + k: *Builder, + query_start_len_ptr: Value, + target_idx: Value, + num_seqs: Value, + block_q: i64, + use_q_block_mode: bool, +) Value { + const left_init = k.constant(.i32, 0); + + var w = k.openWhile(.{ left_init, num_seqs }, .{ .i32, .i32 }); + { + const left = w.before_carried[0]; + const right = w.before_carried[1]; + w.yieldBefore(left.lt(right), .{ left, right }); + } + { + const left = w.after_carried[0]; + const right = w.after_carried[1]; + const mid = left.add(right).div(2); + const val = k.load(query_start_len_ptr.addPtr(mid)); + const mid_val = if (use_q_block_mode) + val.div(@as(i32, @intCast(block_q))).add(mid) + else + val; + + var i = k.openIfElse(mid_val.le(target_idx), .{ .i32, .i32 }); + { + i.yieldThen(.{ mid.add(1), right }); + } + { + i.yieldElse(.{ left, mid }); + } + w.yieldAfter(.{ i.results[0], i.results[1] }); + } + + return w.results[0].sub(1); +} + +fn dequantKv(loaded: Value, scale: Value, kv_is_fp8: bool, q_dtype: DType) Value { + if (!kv_is_fp8 or isFp8(q_dtype)) return loaded; + return loaded.to(.f32).mul(scale).to(q_dtype); +} + +pub const KernelUnifiedAttention3dPtr = struct { + pub const Config = struct { + q_dtype: DType, + kv_dtype: DType, + scale_dtype: DType = .f32, + + num_query_heads: i64, + num_queries_per_kv: i64, + + block_size: i64, + tile_size: i64, + head_size: i64, + head_size_padded: i64, + + use_alibi_slopes: bool, + use_qq_bias: bool, + use_softcap: bool, + use_sinks: bool, + sliding_window: i64, + is_causal: bool = true, + + stride_k_cache_0: i64 = 0, + stride_k_cache_1: i64 = 0, + stride_k_cache_2: i64 = 0, + stride_k_cache_3: i64 = 1, + stride_v_cache_0: i64 = 0, + stride_v_cache_1: i64 = 0, + stride_v_cache_2: i64 = 0, + stride_v_cache_3: i64 = 1, + + block_q: i64, + block_m: i64, + num_segments_per_seq: i64, + all_decode: bool = false, + kv_cache_modifier: ttir.CacheModifier = .none, + }; + + pub const Kernel = tri.Kernel(Config, .{ + .name = "kernel_unified_attention_3d_ptr_oneapi", + .inputs = &.{ + "query_ptr", "key_cache_ptr", "value_cache_ptr", "sink_ptr", + "block_tables_ptr", "seq_lens_ptr", "alibi_slopes_ptr", "qq_bias_ptr", + "scale_ptr", "k_scale_ptr", "v_scale_ptr", "softcap_ptr", + "block_table_stride_ptr", "query_stride_0_ptr", "query_stride_1_ptr", "qq_bias_stride_0_ptr", + "stride_k_cache_0_ptr", "stride_k_cache_1_ptr", "stride_k_cache_2_ptr", "stride_v_cache_0_ptr", + "stride_v_cache_1_ptr", "stride_v_cache_2_ptr", "query_start_len_ptr", "num_seqs_ptr", + }, + .outputs = &.{ "segm_output", "segm_max", "segm_expsum" }, + .run = run, + }); + fn run(b: *Builder, cfg: Config) tri.FinishError!void { + const a = try b.declareArgs(.{ + .query_ptr = .{ .ptr = cfg.q_dtype }, + .key_cache_ptr = .{ .ptr = cfg.kv_dtype }, + .value_cache_ptr = .{ .ptr = cfg.kv_dtype }, + .sink_ptr = .{ .ptr = cfg.scale_dtype }, + .block_tables_ptr = .{ .ptr = .i32 }, + .seq_lens_ptr = .{ .ptr = .i32 }, + .alibi_slopes_ptr = .{ .ptr = cfg.scale_dtype }, + .qq_bias_ptr = .{ .ptr = cfg.scale_dtype }, + .scale_ptr = .{ .ptr = .f32 }, + .k_scale_ptr = .{ .ptr = .f32 }, + .v_scale_ptr = .{ .ptr = .f32 }, + .softcap_ptr = .{ .ptr = .f32 }, + .block_table_stride_ptr = .{ .ptr = .i64 }, + .query_stride_0_ptr = .{ .ptr = .i64 }, + .query_stride_1_ptr = .{ .ptr = .i64 }, + .qq_bias_stride_0_ptr = .{ .ptr = .i64 }, + .stride_k_cache_0_ptr = .{ .ptr = .i64 }, + .stride_k_cache_1_ptr = .{ .ptr = .i64 }, + .stride_k_cache_2_ptr = .{ .ptr = .i64 }, + .stride_v_cache_0_ptr = .{ .ptr = .i64 }, + .stride_v_cache_1_ptr = .{ .ptr = .i64 }, + .stride_v_cache_2_ptr = .{ .ptr = .i64 }, + .query_start_len_ptr = .{ .ptr = .i32 }, + .num_seqs_ptr = .{ .ptr = .i32 }, + .segm_output_ptr = .{ .ptr = .f32 }, + .segm_max_ptr = .{ .ptr = .f32 }, + .segm_expsum_ptr = .{ .ptr = .f32 }, + }); + + const scale = b.load(a.scale_ptr); + const k_scale = b.load(a.k_scale_ptr); + const v_scale = b.load(a.v_scale_ptr); + const softcap = b.load(a.softcap_ptr); + const block_table_stride = b.load(a.block_table_stride_ptr); + const query_stride_0 = b.load(a.query_stride_0_ptr); + const query_stride_1 = b.load(a.query_stride_1_ptr); + const qq_bias_stride_0 = b.load(a.qq_bias_stride_0_ptr); + const stride_k_cache_0 = b.constant(.i64, cfg.stride_k_cache_0); + const stride_k_cache_1 = b.constant(.i64, cfg.stride_k_cache_1); + const stride_k_cache_2 = b.constant(.i64, cfg.stride_k_cache_2); + const stride_v_cache_0 = b.constant(.i64, cfg.stride_v_cache_0); + const stride_v_cache_1 = b.constant(.i64, cfg.stride_v_cache_1); + const stride_v_cache_2 = b.constant(.i64, cfg.stride_v_cache_2); + const num_seqs = b.load(a.num_seqs_ptr); + + kernelUnifiedAttention3d( + b, + a.segm_output_ptr, + a.segm_max_ptr, + a.segm_expsum_ptr, + a.query_ptr, + a.key_cache_ptr, + a.value_cache_ptr, + a.sink_ptr, + a.block_tables_ptr, + a.seq_lens_ptr, + a.alibi_slopes_ptr, + a.qq_bias_ptr, + scale, + k_scale, + v_scale, + softcap, + block_table_stride, + query_stride_0, + query_stride_1, + qq_bias_stride_0, + stride_k_cache_0, + stride_k_cache_1, + stride_k_cache_2, + stride_v_cache_0, + stride_v_cache_1, + stride_v_cache_2, + a.query_start_len_ptr, + num_seqs, + cfg, + ); + } +}; + +fn kernelUnifiedAttention3d( + k: *Builder, + segm_output_ptr: Value, + segm_max_ptr: Value, + segm_expsum_ptr: Value, + query_ptr: Value, + key_cache_ptr: Value, + value_cache_ptr: Value, + sink_ptr: Value, + block_tables_ptr: Value, + seq_lens_ptr: Value, + alibi_slopes_ptr: Value, + qq_bias_ptr: Value, + scale: Value, + k_scale: Value, + v_scale: Value, + softcap: Value, + block_table_stride: Value, + query_stride_0: Value, + query_stride_1: Value, + qq_bias_stride_0: Value, + stride_k_cache_0: Value, + stride_k_cache_1: Value, + stride_k_cache_2: Value, + stride_v_cache_0: Value, + stride_v_cache_1: Value, + stride_v_cache_2: Value, + query_start_len_ptr: Value, + num_seqs: Value, + config: KernelUnifiedAttention3dPtr.Config, +) void { + const BLOCK_M: i64 = config.block_m; + const BLOCK_Q: i64 = config.block_q; + const BLOCK_SIZE: i64 = config.block_size; + const TILE_SIZE: i64 = config.tile_size; + const HEAD_SIZE: i64 = config.head_size; + const HEAD_SIZE_PADDED: i64 = config.head_size_padded; + const NUM_QUERIES_PER_KV: i64 = config.num_queries_per_kv; + const NUM_QUERY_HEADS: i64 = config.num_query_heads; + const NUM_SEGMENTS_PER_SEQ: i64 = config.num_segments_per_seq; + const SLIDING_WINDOW: i64 = config.sliding_window; + + const q_block_global_idx = k.programId(.x); + const kv_head_idx = k.programId(.y); + const segm_idx = k.programId(.z); + + const qk_scale = scale.mul(RCP_LN2); + + const seq_idx = findSeqIdx(k, query_start_len_ptr, q_block_global_idx, num_seqs, BLOCK_Q, true); + + const q_start_raw = k.load(query_start_len_ptr.addPtr(seq_idx)); + const q_block_start_idx = q_start_raw.div(@as(i32, @intCast(BLOCK_Q))).add(seq_idx); + const q_block_local_idx = q_block_global_idx.sub(q_block_start_idx); + + const cur_batch_in_all_start_index = q_start_raw; + // Match Python's left-to-right `query_start_len_ptr + seq_idx + 1`: two `tt.addptr`s. + const cur_batch_in_all_stop_index = k.load(query_start_len_ptr.addPtr(seq_idx).addPtr(1)); + const cur_batch_query_len = cur_batch_in_all_stop_index.sub(cur_batch_in_all_start_index); + + const q_out_of_range = q_block_local_idx.mul(@as(i32, @intCast(BLOCK_Q))).ge(cur_batch_query_len); + k.returnIf(q_out_of_range); + + const seq_len = k.load(seq_lens_ptr.addPtr(seq_idx)); + const tiles_per_segment = cdivFn(seq_len, @as(i32, @intCast(NUM_SEGMENTS_PER_SEQ * TILE_SIZE))); + + const segm_lo = segm_idx.mul(tiles_per_segment).mul(@as(i32, @intCast(TILE_SIZE))); + const segm_out_of_range = segm_lo.ge(seq_len); + k.returnIf(segm_out_of_range); + + const offs_m = k.arange(0, BLOCK_M, .i32); + const offs_d = k.arange(0, HEAD_SIZE_PADDED, .i32); + const offs_t = k.arange(0, TILE_SIZE, .i32); + + const q_local_bq = q_block_local_idx.mul(@as(i32, @intCast(BLOCK_Q))); + const query_pos = q_local_bq.add(offs_m.div(@as(i32, @intCast(NUM_QUERIES_PER_KV)))); + + const query_offset_0 = cur_batch_in_all_start_index.add(query_pos); + const query_offset_1 = kv_head_idx.mul(@as(i32, @intCast(NUM_QUERIES_PER_KV))) + .add(offs_m.rem(@as(i32, @intCast(NUM_QUERIES_PER_KV)))); + + // query_offset = qo_0[:, None]*qstride_0 + qo_1[:, None]*qstride_1 + offs_d[None, :] + // Inline `expandDims(offs_d, 0)` *inside* the chain so it's emitted after + // the qo0+qo1 addi + const qo0_2d = k.expandDims(query_offset_0, 1).mul(query_stride_0); + const qo1_2d = k.expandDims(query_offset_1, 1).mul(query_stride_1); + const query_offset = qo0_2d.add(qo1_2d).add(k.expandDims(offs_d, 0)); + + const dim_mask: Value = if (HEAD_SIZE_PADDED != HEAD_SIZE) + offs_d.lt(@as(i32, @intCast(HEAD_SIZE))) + else + k.full(&.{HEAD_SIZE_PADDED}, 1, .i1); + const query_mask_0 = query_pos.lt(cur_batch_query_len); + const query_mask_1 = query_offset_1.lt(@as(i32, @intCast(NUM_QUERY_HEADS))); + + const q_mask_pre = k.expandDims(query_mask_0, 1).bitAnd(k.expandDims(query_mask_1, 1)); + const q_mask_partial: Value = if (HEAD_SIZE_PADDED != HEAD_SIZE) + q_mask_pre.bitAnd(k.expandDims(dim_mask, 0)) + else + q_mask_pre; + const q_ptrs = query_ptr.addPtr(query_offset); + const Q = k.loadOpts(q_ptrs, .{ + .mask = k.broadcastTo(q_mask_partial, &.{ BLOCK_M, HEAD_SIZE_PADDED }), + .other = k.zeros(&.{ BLOCK_M, HEAD_SIZE_PADDED }, config.q_dtype), + }); + + const block_table_offset = seq_idx.to(.i64).mul(block_table_stride); + + // M init: USE_SINKS + segm_idx==0 → loaded sinks; else -inf. + const segm_is_zero = segm_idx.eq(0); + const m_init: Value = if (config.use_sinks) mb: { + var mi = k.openIfElse(segm_is_zero, .{k.tensorTy(&.{BLOCK_M}, .f32)}); + { + const loaded = k.loadOpts(sink_ptr.addPtr(query_offset_1), .{ + .mask = query_mask_1, + .other = k.full(&.{BLOCK_M}, -std.math.inf(f32), config.scale_dtype), + }); + mi.yieldThen(.{loaded.to(.f32).mul(RCP_LN2)}); + } + { + mi.yieldElse(.{k.full(&.{BLOCK_M}, -std.math.inf(f32), .f32)}); + } + break :mb mi.results[0]; + } else k.full(&.{BLOCK_M}, -std.math.inf(f32), .f32); + + const l_init = k.full(&.{BLOCK_M}, 1.0, .f32); + const acc_init = k.zeros(&.{ BLOCK_M, HEAD_SIZE_PADDED }, .f32); + + const context_len = seq_len.sub(cur_batch_query_len); + + const alibi_slope: Value = if (config.use_alibi_slopes) + k.loadOpts(alibi_slopes_ptr.addPtr(query_offset_1), .{ + .mask = query_mask_1, + .other = k.full(&.{BLOCK_M}, 0.0, config.scale_dtype), + }) + else + k.zeros(&.{BLOCK_M}, config.scale_dtype); + + // See 2d-body note on qq_bias_row_ptrs: else branch needs an explicit splatTo + // (no sibling offset to auto-broadcast against). + const qq_bias_row_ptrs: Value = if (config.use_qq_bias) + qq_bias_ptr.addPtr(query_pos.to(.i64).mul(qq_bias_stride_0)) + else + qq_bias_ptr.splatTo(&.{BLOCK_M}); + + const pad_term: i32 = @intCast(@divTrunc(BLOCK_M - 1, NUM_QUERIES_PER_KV) + 1); + const max_prefix_raw = context_len.add(q_local_bq).add(pad_term); + const max_seq_prefix_len = if (config.is_causal) max_prefix_raw.minimum(seq_len) else seq_len; + const num_tiles = cdivFn(max_seq_prefix_len, @as(i32, @intCast(TILE_SIZE))); + + const segm_tile_lo = segm_idx.mul(tiles_per_segment); + const segm_tile_hi = segm_idx.add(1).mul(tiles_per_segment).minimum(num_tiles); + + const kv_cache_mod: ttir.CacheModifier = config.kv_cache_modifier; + + var loop = k.openFor(segm_tile_lo, segm_tile_hi, 1, .{ m_init, l_init, acc_init }); + { + const j = loop.iv; + const M = loop.carried[0]; + const L = loop.carried[1]; + const acc = loop.carried[2]; + + const seq_offset = j.mul(@as(i32, @intCast(TILE_SIZE))).add(offs_t); + const tile_mask: Value = seq_offset.lt(max_seq_prefix_len); + + const same_block = TILE_SIZE == BLOCK_SIZE; + const in_block_offs: Value = if (same_block) offs_t else seq_offset.rem(@as(i32, @intCast(BLOCK_SIZE))); + const physical_block_idx: Value = if (same_block) + k.load(block_tables_ptr.addPtr(block_table_offset).addPtr(j)).to(.i64).splatTo(&.{TILE_SIZE}) + else + k.load( + block_tables_ptr.addPtr(block_table_offset).addPtr(seq_offset.div(@as(i32, @intCast(BLOCK_SIZE)))), + ).to(.i64); + + const v_offset = k.expandDims(physical_block_idx, 1).mul(stride_v_cache_0) + .add(kv_head_idx.to(.i64).mul(stride_v_cache_2)) + .add(k.expandDims(offs_d, 0).mul(config.stride_v_cache_3)) + .add(k.expandDims(in_block_offs, 1).mul(stride_v_cache_1)); + + const k_offset = k.expandDims(physical_block_idx, 0).mul(stride_k_cache_0) + .add(kv_head_idx.to(.i64).mul(stride_k_cache_2)) + .add(k.expandDims(offs_d, 1).mul(config.stride_k_cache_3)) + .add(k.expandDims(in_block_offs, 0).mul(stride_k_cache_1)); + + const block_io_loads = same_block and HEAD_SIZE_PADDED == HEAD_SIZE; + + const K_load = if (block_io_loads) + k.loadOpts(key_cache_ptr.addPtr(k_offset), .{ .cache_modifier = kv_cache_mod }) + else kb: { + const k_mask_partial: Value = if (HEAD_SIZE_PADDED != HEAD_SIZE) + k.expandDims(dim_mask, 1).bitAnd(k.expandDims(tile_mask, 0)) + else + k.expandDims(tile_mask, 0); + break :kb k.loadOpts(key_cache_ptr.addPtr(k_offset), .{ + .mask = k.broadcastTo(k_mask_partial, &.{ HEAD_SIZE_PADDED, TILE_SIZE }), + .other = k.zeros(&.{ HEAD_SIZE_PADDED, TILE_SIZE }, config.kv_dtype), + .cache_modifier = kv_cache_mod, + }); + }; + const K = dequantKv(K_load, k_scale, isFp8(config.kv_dtype), config.q_dtype); + + const V_raw = if (block_io_loads) + k.loadOpts(value_cache_ptr.addPtr(v_offset), .{ .cache_modifier = kv_cache_mod }) + else vb: { + const v_mask_partial: Value = if (HEAD_SIZE_PADDED != HEAD_SIZE) + k.expandDims(tile_mask, 1).bitAnd(k.expandDims(dim_mask, 0)) + else + k.expandDims(tile_mask, 1); + break :vb k.loadOpts(value_cache_ptr.addPtr(v_offset), .{ + .mask = k.broadcastTo(v_mask_partial, &.{ TILE_SIZE, HEAD_SIZE_PADDED }), + .other = k.zeros(&.{ TILE_SIZE, HEAD_SIZE_PADDED }, config.kv_dtype), + .cache_modifier = kv_cache_mod, + }); + }; + const V_load = if (block_io_loads) + k.where( + k.broadcastTo(k.expandDims(tile_mask, 1), &.{ TILE_SIZE, HEAD_SIZE_PADDED }), + V_raw, + k.zeros(&.{ TILE_SIZE, HEAD_SIZE_PADDED }, config.kv_dtype), + ) + else + V_raw; + const V = dequantKv(V_load, v_scale, isFp8(config.kv_dtype), config.q_dtype); + + // seq_mask = seq_offset[None, :] < context_len + query_pos[:, None] + 1 + const ql_2d_i32 = query_pos.expandDims(1); + const so_2d = seq_offset.expandDims(0); + const seq_mask = if (config.is_causal) b: { + const rhs = context_len.add(ql_2d_i32).add(1); + break :b so_2d.lt(rhs); + } else k.broadcastTo(tile_mask.expandDims(0), &.{ BLOCK_M, TILE_SIZE }); + + const acc_zero = k.zeros(&.{ BLOCK_M, TILE_SIZE }, .f32); + const qk = k.dot(Q, K, acc_zero); + var S = qk_scale.mul(qk); + + if (config.use_softcap) { + S = applySoftcap(k, S, softcap).mul(RCP_LN2); + } + + const qm0_2d = query_mask_0.expandDims(1); + const qm1_2d = query_mask_1.expandDims(1); + const keep_mask = qm1_2d.bitAnd(qm0_2d).bitAnd(seq_mask); + S = k.where(keep_mask, S, k.full(&.{ BLOCK_M, TILE_SIZE }, -std.math.inf(f32), .f32)); + + if (SLIDING_WINDOW > 0) { + const diff = ql_2d_i32.add(context_len).sub(so_2d); + const in_win = diff.lt(@as(i32, @intCast(SLIDING_WINDOW))); + S = k.where(in_win, S, k.full(&.{ BLOCK_M, TILE_SIZE }, -std.math.inf(f32), .f32)); + } + + if (config.use_alibi_slopes) { + const alibi_2d = alibi_slope.expandDims(1); + const pos_diff = seq_offset.sub(context_len).to(config.scale_dtype).expandDims(0); + S = S.add(alibi_2d.mul(pos_diff).to(.f32).mul(RCP_LN2)); + } + + if (config.use_qq_bias) { + const key_rel_pos = seq_offset.sub(context_len); + const is_query_key = key_rel_pos.ge(0) + .bitAnd(key_rel_pos.to(.i64).lt(qq_bias_stride_0)); + const qq_ptrs = qq_bias_row_ptrs.expandDims(1) + .addPtr(key_rel_pos.to(.i64).expandDims(0)); + const qq_bias = k.loadOpts(qq_ptrs, .{ + .mask = is_query_key.expandDims(0), + .other = k.zeros(&.{ BLOCK_M, TILE_SIZE }, config.scale_dtype), + }); + S = S.add(qq_bias.to(.f32).mul(RCP_LN2)); + } + + var m_j = M.maximum(k.maxOpts(S, .{ .axis = 1 })); + m_j = k.where(m_j.gt(-std.math.inf(f32)), m_j, k.full(&.{BLOCK_M}, 0.0, .f32)); + + const mj_2d = m_j.expandDims(1); + const P = k.exp2(S.sub(mj_2d)); + const l_j = k.sumOpts(P, .{ .axis = 1 }); + const alpha = k.exp2(M.sub(m_j)); + + const alpha_2d = alpha.expandDims(1); + const acc_scaled = acc.mul(alpha_2d); + const new_L = L.mul(alpha).add(l_j); + + const P_cast = P.to(config.q_dtype); + const new_acc = k.dot(P_cast, V, acc_scaled); + + loop.yield(.{ m_j, new_L, new_acc }); + } + const M_final = loop.results[0]; + const L_final = loop.results[1]; + const acc_final = loop.results[2]; + + // segm_output_offset = qo_0[:, None].to(i64) * (NUM_QUERY_HEADS * NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED) + // + qo_1[:, None] * (NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED) + // + segm_idx * HEAD_SIZE_PADDED + offs_d[None, :] + const segm_out_head_stride_i32: i32 = @intCast(NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED); + const segm_out_token_stride_i64: i64 = NUM_QUERY_HEADS * NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED; + + // Inline `segm_idx * HEAD_SIZE_PADDED` and `offs_d[None, :]` inside the chain + // so each term's ops emit in source order + const oo0_2d = k.expandDims(query_offset_0, 1).to(.i64).mul(segm_out_token_stride_i64); + const oo1_2d = k.expandDims(query_offset_1, 1).mul(segm_out_head_stride_i32); + const segm_output_offset = oo0_2d.add(oo1_2d) + .add(segm_idx.mul(@as(i32, @intCast(HEAD_SIZE_PADDED)))) + .add(k.expandDims(offs_d, 0)); + + const store_mask_2d = k.mask2d(query_mask_0, dim_mask, BLOCK_M, HEAD_SIZE_PADDED) + .bitAnd(query_mask_1.expandDims(1)); + k.storeOpts( + segm_output_ptr.addPtr(segm_output_offset), + acc_final, + .{ .mask = store_mask_2d }, + ); + + // segm_offset = qo_0.to(i64) * (NUM_QUERY_HEADS * NUM_SEGMENTS_PER_SEQ) + // + qo_1 * NUM_SEGMENTS_PER_SEQ + segm_idx + const segm_head_stride_i32: i32 = @intCast(NUM_SEGMENTS_PER_SEQ); + const segm_token_stride_i64: i64 = NUM_QUERY_HEADS * NUM_SEGMENTS_PER_SEQ; + const segm_offset = query_offset_0.to(.i64).mul(segm_token_stride_i64) + .add(query_offset_1.mul(segm_head_stride_i32)) + .add(segm_idx); + const store_mask_1d = query_mask_0.bitAnd(query_mask_1); + k.storeOpts(segm_max_ptr.addPtr(segm_offset), M_final, .{ .mask = store_mask_1d }); + k.storeOpts(segm_expsum_ptr.addPtr(segm_offset), L_final, .{ .mask = store_mask_1d }); +} diff --git a/zml/logo.zig b/zml/logo.zig new file mode 100644 index 00000000..235bc43d --- /dev/null +++ b/zml/logo.zig @@ -0,0 +1,211 @@ +const std = @import("std"); + +pub const Color = enum { + reset, + cyan, + sky, + blue, + violet, + pink, + shine_cyan, + shine_sky, + shine_blue, + shine_violet, + shine_pink, + + pub fn escape(self: Color, support: ColorSupport) []const u8 { + return switch (support) { + .true_color => self.trueColor(), + ._256 => self.xterm256(), + else => "", + }; + } + + fn trueColor(self: Color) []const u8 { + return switch (self) { + .reset => "\x1b[0m", + .cyan => "\x1b[38;2;0;255;255m", + .sky => "\x1b[38;2;0;221;255m", + .blue => "\x1b[38;2;135;143;255m", + .violet => "\x1b[38;2;215;90;219m", + .pink => "\x1b[38;2;242;48;174m", + .shine_cyan => "\x1b[1;38;2;140;255;255m", + .shine_sky => "\x1b[1;38;2;128;244;255m", + .shine_blue => "\x1b[1;38;2;205;210;255m", + .shine_violet => "\x1b[1;38;2;255;170;255m", + .shine_pink => "\x1b[1;38;2;255;150;220m", + }; + } + + fn xterm256(self: Color) []const u8 { + return switch (self) { + .reset => "\x1b[0m", + .cyan => "\x1b[38;5;51m", + .sky => "\x1b[38;5;45m", + .blue => "\x1b[38;5;105m", + .violet => "\x1b[38;5;170m", + .pink => "\x1b[38;5;205m", + .shine_cyan => "\x1b[1;38;5;123m", + .shine_sky => "\x1b[1;38;5;123m", + .shine_blue => "\x1b[1;38;5;189m", + .shine_violet => "\x1b[1;38;5;219m", + .shine_pink => "\x1b[1;38;5;212m", + }; + } +}; + +pub const Row = struct { + text: []const u8, + color: Color = .reset, + shine: Color = .reset, +}; + +pub const Block = struct { + rows: []const Row, +}; + +pub const ColorSupport = enum { + no_color, + ansi, + _256, + true_color, +}; + +pub const Options = struct { + color_support: ColorSupport, + is_shiny: *const fn ([]const u8) bool = isShineGlyph, +}; + +pub const zml_art_blocks = [_]Block{ + .{ .rows = &.{ + .{ .text = " ", .color = .cyan, .shine = .shine_cyan }, + .{ .text = " ", .color = .cyan, .shine = .shine_cyan }, + .{ .text = " ███████╗", .color = .cyan, .shine = .shine_cyan }, + .{ .text = " ╚══███╔╝", .color = .sky, .shine = .shine_sky }, + .{ .text = " ███╔╝ ", .color = .blue, .shine = .shine_blue }, + .{ .text = " ███╔╝ ", .color = .violet, .shine = .shine_violet }, + .{ .text = " ███████╗", .color = .pink, .shine = .shine_pink }, + .{ .text = " ╚══════╝", .color = .pink, .shine = .shine_pink }, + } }, + .{ .rows = &.{ + .{ .text = " ", .color = .cyan, .shine = .shine_cyan }, + .{ .text = "███╗ ███╗", .color = .cyan, .shine = .shine_cyan }, + .{ .text = "████╗ ████║", .color = .sky, .shine = .shine_sky }, + .{ .text = "██╔████╔██║", .color = .blue, .shine = .shine_blue }, + .{ .text = "██║╚██╔╝██║", .color = .violet, .shine = .shine_violet }, + .{ .text = "██║ ╚═╝ ██║", .color = .pink, .shine = .shine_pink }, + .{ .text = "╚═╝ ╚═╝", .color = .pink, .shine = .shine_pink }, + .{ .text = " ", .color = .pink, .shine = .shine_pink }, + } }, + .{ .rows = &.{ + .{ .text = "██╗ ", .color = .cyan, .shine = .shine_cyan }, + .{ .text = "██║ ", .color = .sky, .shine = .shine_sky }, + .{ .text = "██║ ", .color = .blue, .shine = .shine_blue }, + .{ .text = "██║ .ai", .color = .violet, .shine = .shine_violet }, + .{ .text = "███████╗", .color = .pink, .shine = .shine_pink }, + .{ .text = "╚══════╝", .color = .pink, .shine = .shine_pink }, + .{ .text = " ", .color = .pink, .shine = .shine_pink }, + .{ .text = " ", .color = .pink, .shine = .shine_pink }, + } }, +}; + +pub fn detectColorSupport(io: std.Io, file: std.Io.File, env: *const std.process.Environ.Map) ColorSupport { + var no_color: bool = false; + var clicolor_force: bool = false; + no_color = if (env.get("NO_COLOR")) |v| v.len != 0 else false; + clicolor_force = if (env.get("CLICOLOR_FORCE")) |v| v.len != 0 else false; + + const mode = std.Io.Terminal.Mode.detect(io, file, no_color, clicolor_force) catch return .no_color; + if (mode != .escape_codes) return .no_color; + + if (env.get("COLORTERM")) |value| { + if (std.ascii.eqlIgnoreCase(value, "truecolor") or + std.ascii.eqlIgnoreCase(value, "24bit")) + { + return .true_color; + } + } + + if (env.get("TERM")) |value| { + if (std.mem.endsWith(u8, value, "-direct")) return .true_color; + if (std.mem.indexOf(u8, value, "truecolor") != null) return .true_color; + + if (std.mem.indexOf(u8, value, "256color") != null) return ._256; + } + + return .ansi; +} + +pub fn writeZmlArt(writer: *std.Io.Writer, options: Options) !void { + try write(writer, zml_art_blocks[0..], options); +} + +pub fn write(writer: *std.Io.Writer, blocks: []const Block, options: Options) !void { + if (blocks.len == 0) return; + + const row_count = blocks[0].rows.len; + for (blocks[1..]) |block| { + if (block.rows.len != row_count) return error.InvalidLogoBlocks; + } + + for (0..row_count) |row_index| { + const last_block = findLastVisibleBlockIndex(blocks, row_index) orelse { + try writer.writeAll("\n"); + continue; + }; + + for (blocks[0 .. last_block + 1], 0..) |block, block_index| { + const row = block.rows[row_index]; + try writeRow(writer, row, options, block_index == last_block); + } + try writer.writeAll(Color.reset.escape(options.color_support)); + try writer.writeAll("\n"); + } +} + +pub fn writeRow(writer: *std.Io.Writer, row: Row, options: Options, rtrim_whitespace: bool) !void { + const text = if (rtrim_whitespace) trimRightWhitespace(row.text) else row.text; + if (text.len == 0) return; + + const color = row.color.escape(options.color_support); + const shine = row.shine.escape(options.color_support); + + var view: std.unicode.Utf8View = .initUnchecked(text); + var iter = view.iterator(); + var is_shiny = false; + + try writer.writeAll(color); + while (iter.nextCodepointSlice()) |codepoint| { + const should_shine = options.is_shiny(codepoint); + if (should_shine != is_shiny) { + try writer.writeAll(if (should_shine) shine else color); + is_shiny = should_shine; + } + try writer.writeAll(codepoint); + } +} + +fn findLastVisibleBlockIndex(blocks: []const Block, row_index: usize) ?usize { + var index = blocks.len; + while (index > 0) { + index -= 1; + if (trimRightWhitespace(blocks[index].rows[row_index].text).len != 0) return index; + } + return null; +} + +fn trimRightWhitespace(text: []const u8) []const u8 { + return std.mem.trimEnd(u8, text, " \t\r\n"); +} + +pub fn isShineGlyph(codepoint: []const u8) bool { + return std.mem.eql(u8, codepoint, "╔") or + std.mem.eql(u8, codepoint, "╗") or + std.mem.eql(u8, codepoint, "╚") or + std.mem.eql(u8, codepoint, "╝") or + std.mem.eql(u8, codepoint, "═") or + std.mem.eql(u8, codepoint, "║") or + std.mem.eql(u8, codepoint, ".") or + std.mem.eql(u8, codepoint, "a") or + std.mem.eql(u8, codepoint, "i"); +} diff --git a/zml/mem.zig b/zml/mem.zig new file mode 100644 index 00000000..a64ae968 --- /dev/null +++ b/zml/mem.zig @@ -0,0 +1,386 @@ +const std = @import("std"); +const Allocator = std.mem.Allocator; +const Alignment = std.mem.Alignment; +const assert = std.debug.assert; + +const pjrt = @import("pjrt"); + +const Buffer = @import("buffer.zig").Buffer; +const Device = @import("platform.zig").Device; +const Memory = @import("platform.zig").Memory; +const meta = @import("meta.zig"); +const Platform = @import("platform.zig").Platform; +const Tensor = @import("tensor.zig").Tensor; + +const log = std.log.scoped(.@"zml/mem"); + +pub const DmaAllocator = union(enum) { + passthrough: std.mem.Allocator, + uib: UninitializedBufferAllocator, + dmam: DmaMapAllocator, + + pub fn init(parent: std.mem.Allocator, device: *const Device) DmaAllocator { + return switch (device.platform.target) { + .cuda => .{ .dmam = .init(parent, device.platform) }, + .rocm, .oneapi, .tpu, .furiosa => .{ .uib = .init(device.memory(.host_pinned).?) }, + .cpu, .neuron, .metal => .{ .passthrough = parent }, + }; + } + + pub fn allocator(self: *const DmaAllocator) std.mem.Allocator { + return switch (self.*) { + .passthrough => |a| a, + inline else => |*a| a.allocator(), + }; + } +}; + +pub const UninitializedBufferAllocator = struct { + memory: *const Memory, + + const Header = struct { + buffer: *pjrt.Buffer, + }; + + pub fn init(memory: *const Memory) UninitializedBufferAllocator { + return .{ + .memory = memory, + }; + } + + pub fn allocator(self: *const UninitializedBufferAllocator) std.mem.Allocator { + return .{ + .ptr = @constCast(self), + .vtable = &.{ + .alloc = alloc, + .resize = resize, + .remap = remap, + .free = free, + }, + }; + } + + fn alloc(ctx: *anyopaque, len: usize, alignment: std.mem.Alignment, _: usize) ?[*]u8 { + const self: *UninitializedBufferAllocator = @ptrCast(@alignCast(ctx)); + const pjrt_api = self.memory.platform.pjrt_api; + const pjrt_client = self.memory.platform.pjrt_client; + + const total_len = std.mem.alignForward(usize, @sizeOf(Header) + len, alignment.toByteUnits()); + + const pjrt_buffer = pjrt_client.createUninitializedBuffer(pjrt_api, .{ + .dims = &.{@intCast(total_len)}, + .element_type = .u8, + .layout = .{ + .tiled = .{ + .minor_to_major = &.{0}, + .tile_dims = &.{}, + .tile_dims_sizes = &.{}, + }, + }, + .dst = .{ .memory = self.memory.pjrt_memory }, + }) catch return null; + + const opaque_ptr: [*]u8 = @ptrCast(pjrt_buffer.opaqueDeviceMemoryDataPointer(pjrt_api) catch unreachable); + const data_with_header: []u8 = opaque_ptr[0..total_len]; + + const header = std.mem.bytesAsValue(Header, opaque_ptr); + header.* = .{ + .buffer = pjrt_buffer, + }; + const offset = std.mem.alignForward(usize, @sizeOf(Header), alignment.toByteUnits()); + return @ptrCast(data_with_header[offset..]); + } + + fn free(ctx: *anyopaque, buf: []u8, alignment: Alignment, ret_addr: usize) void { + _ = ret_addr; + const self: *UninitializedBufferAllocator = @ptrCast(@alignCast(ctx)); + const pjrt_api = self.memory.platform.pjrt_api; + const header: *Header = @ptrFromInt(std.mem.alignBackward(usize, @intFromPtr(buf.ptr) - @sizeOf(Header), alignment.toByteUnits())); + header.buffer.deinit(pjrt_api); + } + + fn resize(ctx: *anyopaque, buf: []u8, alignment: Alignment, new_len: usize, ret_addr: usize) bool { + _ = ctx; + _ = buf; + _ = alignment; + _ = new_len; + _ = ret_addr; + return false; + } + + fn remap(ctx: *anyopaque, buf: []u8, alignment: Alignment, new_len: usize, ret_addr: usize) ?[*]u8 { + _ = ctx; + _ = buf; + _ = alignment; + _ = new_len; + _ = ret_addr; + return null; + } +}; + +pub const DmaMapAllocator = struct { + parent: std.mem.Allocator, + platform: *const Platform, + + pub fn init(parent: std.mem.Allocator, platform: *const Platform) DmaMapAllocator { + return .{ + .parent = parent, + .platform = platform, + }; + } + + pub fn allocator(self: *const DmaMapAllocator) Allocator { + return .{ + .ptr = @constCast(self), + .vtable = &.{ + .alloc = alloc, + .resize = resize, + .remap = remap, + .free = free, + }, + }; + } + + fn alloc(ctx: *anyopaque, len: usize, alignment: std.mem.Alignment, ret_addr: usize) ?[*]u8 { + const self: *const DmaMapAllocator = @ptrCast(@alignCast(ctx)); + const allocation = self.parent.rawAlloc(len, alignment, ret_addr); + if (allocation) |loc| { + const data = loc[0..len]; + self.platform.pjrt_client.dmaMap(self.platform.pjrt_api, @ptrCast(data)) catch { + self.parent.rawFree(data, alignment, ret_addr); + return null; + }; + } + return allocation; + } + + fn resize(ctx: *anyopaque, buf: []u8, alignment: std.mem.Alignment, new_len: usize, ret_addr: usize) bool { + _ = ctx; + _ = buf; + _ = alignment; + _ = new_len; + _ = ret_addr; + return false; + } + + fn remap(ctx: *anyopaque, buf: []u8, alignment: std.mem.Alignment, new_len: usize, ret_addr: usize) ?[*]u8 { + _ = ctx; + _ = buf; + _ = alignment; + _ = new_len; + _ = ret_addr; + return null; + } + + fn free(ctx: *anyopaque, buf: []u8, alignment: std.mem.Alignment, ret_addr: usize) void { + const self: *const DmaMapAllocator = @ptrCast(@alignCast(ctx)); + self.platform.pjrt_client.dmaUnmap(self.platform.pjrt_api, @ptrCast(buf[0..buf.len])) catch unreachable; + self.parent.rawFree(buf, alignment, ret_addr); + } +}; + +pub const FixedBufferPool = struct { + buffer: []u8, + block_size: usize, + q_buf: []const u16, + q: std.Io.Queue(u16), + + pub fn init(allocator: std.mem.Allocator, buffer_: []u8, blocks_: u16) !FixedBufferPool { + const block_size = @divExact(buffer_.len, blocks_); + const q_buf = try allocator.alloc(u16, blocks_); + for (q_buf, 0..) |*idx, i| { + idx.* = @intCast(i); + } + var q: std.Io.Queue(u16) = .init(q_buf); + q.type_erased.len = q.type_erased.buffer.len; // make the queue full + return .{ + .buffer = buffer_, + .block_size = block_size, + .q = q, + .q_buf = q_buf, + }; + } + + pub fn deinit(self: *FixedBufferPool, allocator: std.mem.Allocator) void { + allocator.free(self.q_buf); + } + + pub fn get(self: *FixedBufferPool, io: std.Io) ![]u8 { + const idx = try self.q.getOneUncancelable(io); + return self.buffer[idx * self.block_size ..][0..self.block_size]; + } + + fn inRange(sub_buffer: []const u8, buffer: []const u8) bool { + return @intFromPtr(sub_buffer.ptr) >= @intFromPtr(buffer.ptr) and + @intFromPtr(sub_buffer[sub_buffer.len - 1 ..].ptr) <= @intFromPtr(buffer[buffer.len - 1 ..].ptr); + } + + pub fn put(self: *FixedBufferPool, io: std.Io, buf: []u8) void { + // is the pointer in range ? + std.debug.assert(inRange(buf, self.buffer)); + const idx = @divExact(@intFromPtr(buf.ptr) - @intFromPtr(self.buffer.ptr), self.block_size); + self.q.putOneUncancelable(io, @intCast(idx)) catch unreachable; + } +}; + +pub const DynamicBufferPool = struct { + const Node = struct { next: ?*Node }; + const alignment: std.mem.Alignment = .of(Node); + + block_size: usize, + max_blocks: usize, + + arena_state: std.heap.ArenaAllocator.State = .{}, + free_stack: std.atomic.Value(?*Node) = .init(null), + in_flight: std.atomic.Value(usize) = .init(0), + mutex: std.Io.Mutex = .init, + condition: std.Io.Condition = .init, + + pub fn init(max_blocks: usize, block_size: usize) DynamicBufferPool { + return .{ + .block_size = block_size, + .max_blocks = max_blocks, + }; + } + + pub fn deinit(self: *DynamicBufferPool, allocator: std.mem.Allocator) void { + self.arena_state.promote(allocator).deinit(); + } + + pub fn get(self: *DynamicBufferPool, allocator: std.mem.Allocator, io: std.Io) ![]u8 { + while (true) { + var in_flight = self.in_flight.load(.acquire); + while (in_flight < self.max_blocks) { + if (self.in_flight.cmpxchgWeak(in_flight, in_flight + 1, .release, .acquire)) |actual| { + in_flight = actual; + continue; + } + + if (self.pop()) |node| { + return self.nodeToSlice(node); + } + + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + + var arena = self.arena_state.promote(allocator); + defer self.arena_state = arena.state; + + errdefer _ = self.in_flight.fetchSub(1, .release); + const buffer = try arena.allocator().alignedAlloc(u8, alignment, self.block_size); + return @alignCast(buffer); + } + + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + while (self.in_flight.load(.acquire) >= self.max_blocks) { + try self.condition.wait(io, &self.mutex); + } + } + } + + pub fn put(self: *DynamicBufferPool, io: std.Io, buf: []u8) void { + const node: *Node = @ptrCast(@alignCast(buf.ptr)); + var head = self.free_stack.load(.acquire); + while (true) { + node.next = head; + if (self.free_stack.cmpxchgWeak(head, node, .release, .acquire)) |actual_head| { + head = actual_head; + } else { + break; + } + } + + self.mutex.lockUncancelable(io); + defer self.mutex.unlock(io); + _ = self.in_flight.fetchSub(1, .release); + self.condition.signal(io); + } + + fn pop(self: *DynamicBufferPool) ?*Node { + var head = self.free_stack.load(.acquire); + while (true) { + if (self.free_stack.cmpxchgWeak(head, if (head) |h| h.next else null, .acquire, .acquire)) |actual| { + head = actual; + continue; + } + + return head; + } + } + + fn nodeToSlice(self: DynamicBufferPool, node: *Node) []u8 { + const ptr: [*]u8 = @ptrCast(node); + return ptr[0..self.block_size]; + } +}; + +/// Return a clone of a type with Tensors replaced by Buffer. +/// Non-Tensor metadata is stripped out of the resulting struct. +/// Recursively descends into the type. +pub fn Bufferized(comptime T: type) type { + @setEvalBranchQuota(10_000); + return meta.MapRestrict(Tensor, Buffer).map(T); +} + +fn bufferizeInner(allocator: std.mem.Allocator, model: anytype, bufferized_: *Bufferized(@TypeOf(model))) !void { + @setEvalBranchQuota(10_000); + const Model = @TypeOf(model); + const ModelBufferized = Bufferized(Model); + + if (ModelBufferized == Buffer) { + bufferized_.* = .{ + // I'm not sure I like that. I'd rather set all fields but _shards than leaving most of the m undefined. + ._shape = model._shape, + ._shards = .empty, + ._platform = undefined, + }; + return; + } + + const type_info = @typeInfo(ModelBufferized); + switch (type_info) { + .@"struct" => |struct_type_info| { + inline for (struct_type_info.field_names) |field_name| { + try bufferizeInner(allocator, @field(model, field_name), &@field(bufferized_, field_name)); + } + }, + .@"union" => { + switch (model) { + inline else => |v, tag| { + bufferized_.* = @unionInit(ModelBufferized, @tagName(tag), undefined); + try bufferizeInner(allocator, v, &@field(bufferized_, @tagName(tag))); + }, + } + }, + .optional => |optional_type_info| { + if (model == null) { + bufferized_.* = null; + } else { + bufferized_.* = @as(optional_type_info.child, undefined); + try bufferizeInner(allocator, model.?, &bufferized_.*.?); + } + }, + .pointer => |p| { + switch (p.size) { + .slice => { + bufferized_.* = try allocator.alignedAlloc(p.child, .fromByteUnits(p.attrs.@"align" orelse @alignOf(p.child)), model.len); + for (model, bufferized_.*) |src, *dst| { + try bufferizeInner(allocator, src, dst); + } + }, + else => unreachable, + } + }, + .void, .int, .@"enum", .bool, .enum_literal, .float, .vector => {}, + else => unreachable, + } +} + +/// Convert a model to its bufferized form by replacing Tensor fields with Buffer +/// and allocating any required slices using the provided allocator. +pub inline fn bufferize(allocator: std.mem.Allocator, comptime ModelType: type, model: *const ModelType) !Bufferized(ModelType) { + var bufferized: Bufferized(ModelType) = undefined; + try bufferizeInner(allocator, model.*, &bufferized); + return bufferized; +} diff --git a/zml/meta.zig b/zml/meta.zig index 532afc73..84fb9d36 100644 --- a/zml/meta.zig +++ b/zml/meta.zig @@ -1,218 +1,11 @@ const std = @import("std"); - const testing = std.testing; -test { - std.testing.refAllDecls(@This()); -} - -/// Computes floating point value division between two integers. -pub fn divFloat(T: type, numerator: anytype, denominator: anytype) T { - return toFloat(T, numerator) / toFloat(T, denominator); -} - -fn toFloat(T: type, x: anytype) T { - return switch (@typeInfo(@TypeOf(x))) { - .Float => @floatCast(x), - else => @floatFromInt(x), - }; -} - -pub fn guard(check: bool, src: std.builtin.SourceLocation) void { - assert(check, "Invalid inputs {s}@{s}:{d}", .{ src.file, src.fn_name, src.line }); -} - -pub inline fn internalAssert(check: bool, comptime msg: []const u8, args: anytype) void { - assert(check, "ZML internal error: " ++ msg, args); -} - -pub fn assert(check: bool, comptime msg: []const u8, args: anytype) void { - if (!check) panic(msg, args); -} - -pub fn panic(comptime msg: []const u8, args: anytype) noreturn { - std.log.err(msg, args); - @panic(msg); -} - -pub fn compileLog(comptime msg: []const u8, comptime args: anytype) void { - @compileLog(std.fmt.comptimePrint(msg, args)); -} - -pub fn compileError(comptime msg: []const u8, comptime args: anytype) noreturn { - @compileError(std.fmt.comptimePrint(msg, args)); -} - -pub fn assertComptime(comptime check: bool, comptime msg: []const u8, comptime args: anytype) void { - if (check == false) { - compileError(msg, args); - } -} - -pub fn isStruct(comptime T: type) bool { - return switch (@typeInfo(T)) { - .Struct => true, - else => false, - }; -} - -pub fn isTuple(comptime T: type) bool { - return switch (@typeInfo(T)) { - .Struct => |info| info.is_tuple, - else => false, - }; -} - -pub fn isStructOf(comptime T: type, comptime Elem: type) bool { - return switch (@typeInfo(T)) { - .Struct => |info| blk: { - inline for (info.fields) |field| { - if (field.type != Elem) { - break :blk false; - } - } - break :blk true; - }, - else => false, - }; -} - -pub fn isStructOfAny(comptime T: type, comptime f: fn (comptime type) bool) bool { - return switch (@typeInfo(T)) { - .Struct => |info| blk: { - inline for (info.fields) |field| { - if (f(field.type) == false) { - break :blk false; - } - } - break :blk true; - }, - else => false, - }; -} - -pub fn isTupleOf(comptime T: type, comptime Elem: type) bool { - return isTuple(T) and isStructOf(T, Elem); -} - -pub fn isTupleOfAny(comptime T: type, comptime f: fn (comptime type) bool) bool { - return isTuple(T) and isStructOfAny(T, f); -} - -pub fn isSliceOf(comptime T: type, comptime Elem: type) bool { - return switch (@typeInfo(T)) { - .Pointer => |info| switch (info.size) { - .Slice => info.child == Elem, - .One => switch (@typeInfo(info.child)) { - // As Zig, convert pointer to Array as a slice. - .Array => |arr_info| arr_info.child == Elem, - else => false, - }, - else => false, - }, - else => false, - }; -} - -pub fn asSlice(comptime T: type) type { - const err_msg = "Type " ++ @typeName(T) ++ " can't be interpreted as a slice"; - return switch (@typeInfo(T)) { - .Pointer => |info| switch (info.size) { - .Slice => info.child, - .One => switch (@typeInfo(info.child)) { - // As Zig, convert pointer to Array as a slice. - .Array => |arr_info| arr_info.child, - else => @compileError(err_msg), - }, - else => @compileError(err_msg), - }, - else => @compileError(err_msg), - }; -} - -pub fn isInteger(comptime T: type) bool { - return switch (@typeInfo(T)) { - .Int, .ComptimeInt => true, - else => false, - }; -} - -pub fn isSliceOfAny(comptime T: type, comptime f: fn (comptime type) bool) bool { - return switch (@typeInfo(T)) { - .Pointer => |info| info.size == .Slice and f(info.child), - else => false, - }; -} - -pub fn DeclEnum(comptime T: type) type { - return std.meta.DeclEnum(UnwrapPtr(T)); -} - -pub fn UnwrapPtr(comptime T: type) type { - return switch (@typeInfo(T)) { - .Pointer => |info| switch (info.size) { - .One => info.child, - else => T, - }, - else => T, - }; -} - -pub fn FnParam(func: anytype, n: comptime_int) type { - return @typeInfo(@TypeOf(func)).Fn.params[n].type orelse @compileError("anytype not supported in callbacks"); -} - -pub fn FnParams(func: anytype) type { - return std.meta.ArgsTuple(@TypeOf(func)); -} - -pub fn FnResult(func: anytype) type { - return @typeInfo(@TypeOf(func)).Fn.return_type.?; -} - -pub fn FnResultPayload(func: anytype) type { - const return_type = @typeInfo(@TypeOf(func)).Fn.return_type.?; - const payload_type = switch (@typeInfo(return_type)) { - .ErrorUnion => |u| u.payload, - else => return_type, - }; - return payload_type; -} - -pub fn FnResultErrorSet(func: anytype) ?type { - const return_type = @typeInfo(@TypeOf(func)).Fn.return_type.?; - const error_set = switch (@typeInfo(return_type)) { - .ErrorUnion => |u| u.error_set, - else => null, - }; - return error_set; -} - -pub fn Signature(comptime func: anytype, comptime argsT: ?type) type { - return struct { - pub const FuncT = if (@TypeOf(func) == type) func else @TypeOf(func); - pub const ArgsT = blk: { - if (@typeInfo(FuncT).Fn.params.len == 0) { - break :blk @TypeOf(.{}); - } - break :blk argsT orelse std.meta.ArgsTuple(FuncT); - }; - pub const ReturnT = @TypeOf(@call(.auto, func, @as(ArgsT, undefined))); - pub const ReturnPayloadT = blk: { - break :blk switch (@typeInfo(ReturnT)) { - .ErrorUnion => |u| u.payload, - else => ReturnT, - }; - }; - pub const ReturnErrorSet: ?type = blk: { - break :blk switch (@typeInfo(ReturnT)) { - .ErrorUnion => |u| u.error_set, - else => null, - }; - }; - }; -} +const stdx = @import("stdx"); +const FnParam = stdx.meta.FnParam; +const asSlice = stdx.meta.asSlice; +/// Visit a given type `T` and replace all fields containing `From` by fields containing `To`. pub fn MapType(From: type, To: type) type { return struct { pub fn map(T: type) type { @@ -226,88 +19,155 @@ pub fn MapType(From: type, To: type) type { } return switch (@typeInfo(T)) { - .Struct => |struct_infos| { - const fields = struct_infos.fields; + .@"struct" => |struct_infos| { + const field_names = struct_infos.field_names; var same: bool = true; - var struct_fields: [fields.len]std.builtin.Type.StructField = undefined; - for (struct_fields[0..], fields) |*struct_field, field| { - if (!field.is_comptime) { - const R = map(field.type); - if (R == field.type) { - struct_field.* = field; + var struct_field_names: [field_names.len][]const u8 = undefined; + var struct_field_types: [field_names.len]type = undefined; + var struct_field_attrs: [field_names.len]std.builtin.Type.Struct.FieldAttributes = undefined; + for ( + struct_field_names[0..], + struct_field_types[0..], + struct_field_attrs[0..], + struct_infos.field_names, + struct_infos.field_types, + struct_infos.field_attrs, + ) |*field_name, *field_type, *field_attr, source_name, SourceField, source_attrs| { + if (!source_attrs.@"comptime") { + const R = map(SourceField); + if (R == SourceField) { + field_name.* = source_name; + field_type.* = SourceField; + field_attr.* = source_attrs; } else { - struct_field.* = .{ - .name = field.name, - .type = R, - .default_value = null, - .is_comptime = field.is_comptime, - .alignment = @alignOf(R), + field_name.* = source_name; + field_type.* = R; + field_attr.* = .{ + .@"comptime" = source_attrs.@"comptime", + .@"align" = @alignOf(R), }; same = false; // Handle the case `field: ?Tensor = null` // Generic handling of default value is complicated, // it would require to call the callback at comptime. if (R == ?To) { - struct_field.default_value = &@as(R, null); + field_attr.default_value_ptr = &@as(R, null); } } } else { - struct_field.* = field; + field_name.* = source_name; + field_type.* = SourceField; + field_attr.* = source_attrs; } } if (same) return T; - return @Type(.{ .Struct = .{ - .layout = .auto, - .fields = struct_fields[0..], - .decls = &.{}, - .is_tuple = struct_infos.is_tuple, - } }); + return if (struct_infos.is_tuple) @Tuple(&struct_field_types) else @Struct(.auto, null, &struct_field_names, &struct_field_types, &struct_field_attrs); }, - .Array => |arr_info| [arr_info.len]map(arr_info.child), - .Pointer => |ptr_info| switch (ptr_info.size) { - .Slice => if (ptr_info.is_const) + .@"union" => |union_info| { + const field_names = union_info.field_names; + var same: bool = true; + var union_field_names: [field_names.len][]const u8 = undefined; + var union_field_types: [field_names.len]type = undefined; + var union_field_attrs: [field_names.len]std.builtin.Type.Union.FieldAttributes = undefined; + for ( + union_field_names[0..], + union_field_types[0..], + union_field_attrs[0..], + union_info.field_names, + union_info.field_types, + union_info.field_attrs, + ) |*field_name, *field_type, *field_attr, source_name, SourceField, source_attrs| { + const R = map(SourceField); + if (R == SourceField) { + field_name.* = source_name; + field_type.* = SourceField; + field_attr.* = source_attrs; + } else { + field_name.* = source_name; + field_type.* = R; + field_attr.* = .{ + .@"align" = @alignOf(R), + }; + same = false; + } + } + if (same) return T; + return @Union(.auto, union_info.tag_type, &union_field_names, &union_field_types, &union_field_attrs); + }, + .array => |arr_info| [arr_info.len]map(arr_info.child), + .pointer => |ptr_info| switch (ptr_info.size) { + .slice => if (ptr_info.attrs.@"const") []const map(ptr_info.child) else []map(ptr_info.child), - .One => if (ptr_info.is_const) + .one => if (ptr_info.attrs.@"const") *const map(ptr_info.child) else *map(ptr_info.child), else => T, }, - .Optional => |opt_info| ?map(opt_info.child), + .optional => |opt_info| ?map(opt_info.child), else => T, }; } }; } +test MapType { + const A = struct { a: u32 }; + const B = struct { b: u32 }; + + const A2B = MapType(A, B); + + const StructA = struct { some: []const A, one: A, maybe: ?A, other: u32 }; + const struct_b = A2B.map(StructA){ + .some = &[2]B{ .{ .b = 0 }, .{ .b = 1 } }, + .maybe = null, + .one = .{ .b = 2 }, + .other = 43, + }; + _ = struct_b; + + const UnionA = union { some: []const A, one: A, maybe: ?A, other: u32 }; + const union_b = [_]A2B.map(UnionA){ + .{ .some = &[2]B{ .{ .b = 0 }, .{ .b = 1 } } }, + .{ .one = .{ .b = 2 } }, + .{ .maybe = null }, + .{ .other = 43 }, + }; + _ = union_b; +} + /// Given a callback: `fn(Ctx, From) To`, recursively visits the given `from` struct /// and calls the callback when it finds a `From` element, and writes it to the `to` struct. /// The `to` parameter must be passed with mutable pointer, and tensor data need to be mutable if callback needs it. /// `mapAlloc` tries as much as possible to respect the conversions made by Zig itself. /// For example it can convert from a comptime array to a runtime slice. -/// `mapAlloc` can allocate new slices to write the result if the result struct requires it. +/// `mapAlloc` allocates slices and single-item pointers needed to hold mapped values. /// The caller is owning said allocations, using an `ArenaAllocator` might help tracking them. -// TODO: handle tuple to slice conversion +/// +/// Note: to avoid infinite loop, mapAlloc doesn't look for `From` fields inside `To` struct. +/// Any `To` struct inside `from` will be copied over to the target. pub fn mapAlloc(comptime cb: anytype, allocator: std.mem.Allocator, ctx: FnParam(cb, 0), from: anytype, to: anytype) !void { - // const Ctx = FnParam(cb, 0); + // TODO: handle tuple to slice conversion + // TODO: handle error bubbling up const From = FnParam(cb, 1); + const To = stdx.meta.FnReturn(cb); const FromStruct = @TypeOf(from); - const type_info_to_ptr = @typeInfo(@TypeOf(to)); - if (type_info_to_ptr != .Pointer) { - @compileError("convertType is expecting a mutable `to` argument but received: " ++ @typeName(@TypeOf(to))); + if (type_info_to_ptr != .pointer) { + stdx.debug.compileError("convertType is expecting a mutable `to` argument but received: {}", .{@TypeOf(to)}); } - const ToStruct = type_info_to_ptr.Pointer.child; + const ToStruct = type_info_to_ptr.pointer.child; const type_info_to = @typeInfo(ToStruct); if (FromStruct == From) { - // Special case for converting from shape to tensor: - // If the target type is Shape, skip tensor conversion. - // A general `to.* = from` assignment causes a Zig error in this scenario. - // (see below) - if (ToStruct == @import("shape.zig").Shape and FromStruct == ToStruct) { // FromStruct) { + // We have an issues with `Tensor` -> `Shape` -> `Tensor` conversion when compiling ZML functions where one argument is a Shape itself. + // Normally we should call `cb` on all `Shape`. + // But the "ShapeOf" struct will have more Shape than need on the output. + // So here we take a hint from the receiving object. + // If the target is indeed a Tensor, use the callback, but if the target is `Shape` just copy it over. + if (ToStruct != To and FromStruct == ToStruct) { to.* = from; } else { to.* = @call(.auto, cb, .{ ctx, from }); @@ -315,73 +175,89 @@ pub fn mapAlloc(comptime cb: anytype, allocator: std.mem.Allocator, ctx: FnParam return; } - // This is generally due to a user error, but let this fn compile, - // and the user will have a Zig error. - if (FromStruct == ToStruct) { + if (FromStruct == To) { to.* = from; return; } - // Don't go into Shape objects because of the weird tag. - // TODO: we could not error on pointers to basic types like u8 - if (FromStruct == @import("shape.zig").Shape) { + if (FromStruct == ToStruct and !comptime Contains(FromStruct, From)) { to.* = from; return; } + + if (@sizeOf(ToStruct) == 0) return; + switch (type_info_to) { - .Struct => |info| inline for (info.fields) |field| { - // if (field.is_comptime) continue; - const field_type_info = @typeInfo(field.type); + .@"struct" => |info| inline for (info.field_names, info.field_types, info.field_attrs) |field_name, Field, field_attrs| { + if (field_attrs.@"comptime" or @sizeOf(Field) == 0) continue; + const field_type_info = @typeInfo(Field); // If the field is already a pointer, we recurse with it directly, otherwise, we recurse with a pointer to the field. switch (field_type_info) { - // .Pointer => try convertType(From, To, allocator, @field(from, field.name), @field(to, field.name), Ctx, ctx, cb), - .Array, .Optional, .Union, .Struct, .Pointer => if (@hasField(FromStruct, field.name)) { + // .pointer => try convertType(From, To, allocator, @field(from, field.name), @field(to, field.name), Ctx, ctx, cb), + .array, .optional, .@"union", .@"struct", .pointer => if (@hasField(FromStruct, field_name)) { try mapAlloc( cb, allocator, ctx, - @field(from, field.name), - &@field(to, field.name), + @field(from, field_name), + &@field(to, field_name), ); - } else if (field.default_value) |_| { - @field(to, field.name) = null; + } else if (field_attrs.default_value_ptr) |_| { + @field(to, field_name) = null; } else { - compileError("Mapping {} to {} failed. Missing field {s}", .{ FromStruct, ToStruct, field.name }); + stdx.debug.compileError("Mapping {} -> {} inside {} failed. Missing field {s} in {}", .{ From, To, FromStruct, field_name, ToStruct }); }, - else => @field(to, field.name) = @field(from, field.name), + else => @field(to, field_name) = @field(from, field_name), } }, - .Array => for (from, to) |f, *t| { + .@"union" => { + switch (from) { + inline else => |_, tag| { + to.* = @unionInit(ToStruct, @tagName(tag), undefined); + try mapAlloc(cb, allocator, ctx, @field(from, @tagName(tag)), &@field(to, @tagName(tag))); + }, + } + }, + .array => for (from, to) |f, *t| { try mapAlloc(cb, allocator, ctx, f, t); }, - .Pointer => |ptr_info| switch (ptr_info.size) { - .One => switch (type_info_to_ptr.Pointer.size) { + .pointer => |ptr_info| switch (ptr_info.size) { + .one => switch (type_info_to_ptr.pointer.size) { // pointer to array -> slice promotion - .Slice => { - to.* = try allocator.alloc(type_info_to_ptr.Pointer.child, from.len); - for (from, to.*) |f, *t| { + .slice => { + const items = try allocator.alloc(type_info_to_ptr.pointer.child, from.len); + for (from, items) |f, *t| { try mapAlloc(cb, allocator, ctx, f, t); } + to.* = items; + }, + else => { + const item = try allocator.create(ptr_info.child); + try mapAlloc(cb, allocator, ctx, from.*, item); + to.* = item; }, - else => try mapAlloc(cb, allocator, ctx, from.*, to.*), }, - .Slice => { - const items = try allocator.alloc(@typeInfo(ToStruct).Pointer.child, from.len); + .slice => { + const items = try allocator.alloc(@typeInfo(ToStruct).pointer.child, from.len); for (from, items) |f, *t| { try mapAlloc(cb, allocator, ctx, f, t); } to.* = items; }, - else => @compileError("zml.meta.mapAlloc doesn't support: " ++ @typeName(FromStruct)), + .many => if (FromStruct == [*:0]const u8 and ToStruct == FromStruct and From != u8) { + // Tensor axis tags are borrowed immutable strings. + to.* = from; + } else stdx.debug.compileError("zml.meta.mapAlloc doesn't support: {}", .{FromStruct}), + else => stdx.debug.compileError("zml.meta.mapAlloc doesn't support: {}", .{FromStruct}), }, - .Optional => if (from) |f| { - to.* = @as(@typeInfo(type_info_to_ptr.Pointer.child).Optional.child, undefined); + .optional => if (from) |f| { + to.* = @as(@typeInfo(type_info_to_ptr.pointer.child).optional.child, undefined); try mapAlloc(cb, allocator, ctx, f, &(to.*.?)); } else { to.* = null; }, - .Int, .Float => to.* = from, - else => @compileError("zml.meta.mapAlloc doesn't support: " ++ @typeName(FromStruct)), + .int, .float, .@"enum" => to.* = from, + else => stdx.debug.compileError("zml.meta.mapAlloc doesn't support: {}", .{FromStruct}), } } @@ -394,124 +270,288 @@ test mapAlloc { } }; + const Empty = struct {}; + const AA = struct { field: A, array: [2]A, slice: []const A, + pointer: *const A, other: u8, + tag: [*:0]const u8, // We want to allow conversion from comptime to runtime, because Zig type inference works like this. comptime static_val: u8 = 8, comptime static_slice: [2]A = .{ .{ .a = 11 }, .{ .a = 12 } }, + field_with_empty: struct { A, Empty }, }; const BB = struct { field: B, array: [2]B, slice: []const B, + pointer: *const B, other: u8, + tag: [*:0]const u8, static_val: u8, static_slice: []B, + field_with_empty: struct { B, Empty }, }; const aa: AA = .{ .field = .{ .a = 4 }, .array = .{ .{ .a = 5 }, .{ .a = 6 } }, .other = 7, + .tag = "feature", .slice = &.{ .{ .a = 9 }, .{ .a = 10 } }, + .pointer = &.{ .a = 13 }, + .field_with_empty = .{ .{ .a = 9 }, .{} }, }; var bb: BB = undefined; try mapAlloc(A.convert, testing.allocator, {}, aa, &bb); defer testing.allocator.free(bb.slice); defer testing.allocator.free(bb.static_slice); + defer testing.allocator.destroy(bb.pointer); try testing.expectEqual(4, bb.field.b); try testing.expectEqual(5, bb.array[0].b); try testing.expectEqual(6, bb.array[1].b); try testing.expectEqual(7, bb.other); + try testing.expectEqual(aa.tag, bb.tag); try testing.expectEqual(8, bb.static_val); try testing.expectEqual(9, bb.slice[0].b); try testing.expectEqual(10, bb.slice[1].b); try testing.expectEqual(11, bb.static_slice[0].b); try testing.expectEqual(12, bb.static_slice[1].b); + try testing.expectEqual(13, bb.pointer.b); +} + +/// Visit a given type `T` and: +/// * replace all fields containing `From` by fields containing `To` +/// * drop all fields not containing any `From`. +/// The returned type will contains only `To` making it easy for the compiler to produce compact layout. +/// Used by `zml.Bufferized` to strip compile time arguments from a model struct. +pub fn MapRestrict(From: type, To: type) type { + @setEvalBranchQuota(100000); + return struct { + pub fn map(T: type) type { + switch (T) { + From => return To, + ?From => return ?To, + *From => return *To, + *const From => return *const To, + []From => return []To, + []const From => return []const To, + else => {}, + } + + if (!Contains(T, From)) return void; + + return switch (@typeInfo(T)) { + .@"struct" => |struct_infos| { + // We know that at least one of the struct field contains a From. + // We map each field individually. Fields without From and comptime fields are removed. + const field_names = struct_infos.field_names; + var num_fields: usize = 0; + + var struct_field_names: [field_names.len][]const u8 = undefined; + var struct_field_types: [field_names.len]type = undefined; + var struct_field_attrs: [field_names.len]std.builtin.Type.Struct.FieldAttributes = undefined; + for (struct_infos.field_names, struct_infos.field_types, struct_infos.field_attrs) |field_name, Field, field_attrs| { + if (!field_attrs.@"comptime" and Contains(Field, From)) { + const R = map(Field); + if (R == Field) { + struct_field_names[num_fields] = field_name; + struct_field_types[num_fields] = Field; + struct_field_attrs[num_fields] = field_attrs; + } else { + const name = if (struct_infos.is_tuple) struct_infos.field_names[num_fields] else field_name; + struct_field_names[num_fields] = name; + struct_field_types[num_fields] = R; + struct_field_attrs[num_fields] = .{ + .@"align" = @alignOf(R), + }; + // Handle the case `field: ?Tensor = null` + // Generic handling of default value is not possible. + if (R == ?To) { + struct_field_attrs[num_fields].default_value_ptr = &@as(R, null); + } + } + num_fields += 1; + } + } + if (num_fields == 0) return void; + return if (struct_infos.is_tuple) @Tuple(struct_field_types[0..num_fields]) else @Struct( + .auto, + null, + struct_field_names[0..num_fields], + struct_field_types[0..num_fields], + struct_field_attrs[0..num_fields], + ); + }, + .@"union" => |union_info| { + // We know that at least one of the union field contains a From. + // We map each field individually. Fields without From, are replaced by "void". + const field_names = union_info.field_names; + var union_field_names: [field_names.len][]const u8 = undefined; + var union_field_types: [field_names.len]type = undefined; + var union_field_attrs: [field_names.len]std.builtin.Type.Union.FieldAttributes = undefined; + for (0.., union_info.field_names, union_info.field_types) |i, field_name, Field| { + const FT = map(Field); + union_field_names[i] = field_name; + union_field_types[i] = FT; + union_field_attrs[i] = .{ + .@"align" = @alignOf(FT), + }; + } + return @Union(.auto, union_info.tag_type, &union_field_names, &union_field_types, &union_field_attrs); + }, + .array => |arr_info| [arr_info.len]map(arr_info.child), + .pointer => |ptr_info| switch (ptr_info.size) { + .slice => if (ptr_info.attrs.@"const") + []const map(ptr_info.child) + else + []map(ptr_info.child), + .one => if (ptr_info.attrs.@"const") + *const map(ptr_info.child) + else + *map(ptr_info.child), + .many => if (ptr_info.attrs.@"const") + [*]const map(ptr_info.child) + else + [*]map(ptr_info.child), + .c => if (ptr_info.attrs.@"const") + [*c]map(ptr_info.child) + else + [*c]map(ptr_info.child), + }, + .optional => |opt_info| ?map(opt_info.child), + else => T, + }; + } + }; +} + +test MapRestrict { + const A = struct { a: u32 }; + const B = struct { b: u32 }; + + const A2B = MapRestrict(A, B); + + const StructA = struct { some: []const A, one: A, maybe: ?A, other: u32 }; + const struct_b = A2B.map(StructA){ + .some = &[2]B{ .{ .b = 0 }, .{ .b = 1 } }, + .maybe = null, + .one = .{ .b = 2 }, + // Note how struct_b doesn't even have a .other field now. + }; + _ = struct_b; + + const UnionA = union { some: []const A, one: A, maybe: ?A, other: u32 }; + const union_b = [_]A2B.map(UnionA){ + .{ .some = &[2]B{ .{ .b = 0 }, .{ .b = 1 } } }, + .{ .one = .{ .b = 2 } }, + .{ .maybe = null }, + // Note how union_b.other is void now. + .{ .other = {} }, + }; + _ = union_b; } /// Recursively visit the given struct and calls the callback for each K found. /// The `v` parameter must me a pointer, and tensor data need to be mutable if callbacks needs it. -pub fn visit(comptime cb: anytype, ctx: FnParam(cb, 0), v: anytype) void { - const T = @TypeOf(v); - const type_info_v = @typeInfo(T); - const K = switch (@typeInfo(FnParam(cb, 1))) { - .Pointer => |info| info.child, - else => @compileError("zml.meta.visit is expecting a pointer value as second parameter in callback to use but found " ++ @typeName(FnParam(cb, 1))), +pub fn VisitReturn(comptime cb: anytype) type { + return if (stdx.meta.FnReturnErrorSet(cb)) |error_set| + error_set!void + else + void; +} + +const Visitor = struct { + pub const Action = enum { + callback, + recurse, + exit, }; - if (type_info_v != .Pointer) { - const Callback = @TypeOf(cb); - @compileError("zml.meta.visit is expecting a pointer input to go with following callback signature: " ++ @typeName(Callback) ++ " but received: " ++ @typeName(T)); - } - const ptr_info = type_info_v.Pointer; - if (@typeInfo(ptr_info.child) == .Fn) return; - if (ptr_info.child == anyopaque) return; - // This is important, because with trivial types like void, - // Zig sometimes decide to call `visit` at comptime, but can't do - // the pointer wrangling logic at comptime. - // So we detect early this case and return. - if (@sizeOf(ptr_info.child) == 0) return; + pub fn determineAction(comptime callback: anytype, comptime PtrTypeOfV: type) Action { + const ptr_info = switch (@typeInfo(PtrTypeOfV)) { + .pointer => |info| info, + else => stdx.debug.compileError("zml.meta.visit({any}) is expecting a pointer/slice input, but received: {any}", .{ @TypeOf(callback), PtrTypeOfV }), + }; - switch (ptr_info.size) { - // If we have a single pointer, two cases: - // * It's a pointer to K, in which case we call the callback. - // * It's a pointer to something else, in which case, we explore and recurse if needed. - .One => if (ptr_info.child == K) { - cb(ctx, v); - } else if (ptr_info.child == ?K) { - if (v.*) |*val| cb(ctx, val); - } else switch (@typeInfo(ptr_info.child)) { - .Struct => |s| inline for (s.fields) |field_info| { - if (field_info.is_comptime) continue; - const field_type_info = @typeInfo(field_info.type); - // If the field is already a pointer, we recurse with it directly, otherwise, we recurse with a pointer to the field. - switch (field_type_info) { - .Pointer => visit(cb, ctx, @field(v, field_info.name)), - .Array, .Optional, .Union, .Struct => visit(cb, ctx, &@field(v, field_info.name)), - else => {}, - } - }, - .Array => |_| for (v) |*elem| visit(cb, ctx, elem), - .Optional => if (v.* != null) visit(cb, ctx, &v.*.?), - .Union => switch (v.*) { - inline else => |*v_field| visit(cb, ctx, v_field), + const TargetType = switch (@typeInfo(FnParam(callback, 1))) { + .pointer => |info| info.child, + else => stdx.debug.compileError("zml.meta.visit({any}) is expecting a callback with a pointer as second argument but found {any}", .{ @TypeOf(callback), FnParam(callback, 1) }), + }; + + if (ptr_info.size == .one and ptr_info.child == TargetType) { + return .callback; + } + + switch (@typeInfo(ptr_info.child)) { + .@"struct", .@"union" => { + if (@hasDecl(ptr_info.child, "zml_no_meta_visit")) return .exit; }, else => {}, - }, - // If we have a slice, two cases also: - // * It's a slice of K, in which case we call the callback for each element of the slice. - // * It's a slice to something else, in which case, for each element we explore and recurse if needed. - .Slice => { - for (v) |*v_elem| { - if (ptr_info.child == K) { - cb(ctx, v_elem); - } else switch (@typeInfo(ptr_info.child)) { - .Struct => |s| inline for (s.fields) |field_info| { - const field_type_info = @typeInfo(field_info.type); - // If the field is already a pointer, we recurse with it directly, otherwise, we recurse with a pointer to the field. - if (field_type_info == .Pointer) { - visit(cb, ctx, @field(v_elem, field_info.name)); + } + + return .recurse; + } +}; + +pub fn visit(comptime callback: anytype, ctx: FnParam(callback, 0), v: anytype) VisitReturn(callback) { + const action = comptime Visitor.determineAction(callback, @TypeOf(v)); + + const PtrTypeOfV = @TypeOf(v); + const ptr_info_v = @typeInfo(PtrTypeOfV).pointer; + const ChildTypeV = ptr_info_v.child; + + const can_error = stdx.meta.FnReturnErrorSet(callback) != null; + + return switch (action) { + .exit => {}, + .callback => callback(ctx, v), + .recurse => { + const TargetType, const mutating_cb = switch (@typeInfo(FnParam(callback, 1))) { + .pointer => |info| .{ info.child, !info.attrs.@"const" }, + else => stdx.debug.compileError("zml.meta.visit({any}) is expecting a callback with a pointer as second argument but found {any}", .{ @TypeOf(callback), FnParam(callback, 1) }), + }; + + if (comptime !Contains(PtrTypeOfV, TargetType)) return; + + if (@typeInfo(ChildTypeV) == .@"struct" and @hasDecl(ChildTypeV, "constSlice") and @hasDecl(ChildTypeV, "slice")) { + return visit(callback, ctx, if (mutating_cb) v.slice() else v.constSlice()); + } + + switch (ptr_info_v.size) { + .one => switch (@typeInfo(ChildTypeV)) { + .@"struct" => |s| inline for (s.field_names, s.field_types, s.field_attrs) |field_name, Field, field_attrs| { + if (field_attrs.@"comptime" or comptime !Contains(Field, TargetType)) continue; + if (@typeInfo(Field) == .pointer) { + if (can_error) try visit(callback, ctx, @field(v, field_name)) else visit(callback, ctx, @field(v, field_name)); } else { - visit(cb, ctx, &@field(v_elem, field_info.name)); + if (can_error) try visit(callback, ctx, &@field(v, field_name)) else visit(callback, ctx, &@field(v, field_name)); } }, - .Array => |_| for (v) |*elem| visit(cb, ctx, elem), - .Optional => if (v.* != null) visit(cb, ctx, &v.*.?), - .Union => switch (v_elem.*) { - inline else => |*v_field| visit(cb, ctx, v_field), + .array => for (v[0..]) |*elem| { + if (can_error) try visit(callback, ctx, elem) else visit(callback, ctx, elem); }, - else => {}, - } + .optional => return if (v.*) |*opt| visit(callback, ctx, opt), + .@"union" => return switch (v.*) { + inline else => |*v_field| visit(callback, ctx, v_field), + }, + .pointer => return visit(callback, ctx, v.*), + else => stdx.debug.compileError("zml.meta.visit({any}) doesn't support visiting pointer type {any}", .{ @TypeOf(callback), ChildTypeV }), + }, + .slice => { + for (v) |*v_elem| { + if (can_error) try visit(callback, ctx, v_elem) else visit(callback, ctx, v_elem); + } + }, + .many, .c => stdx.debug.compileError("zml.meta.visit({any}) doesn't support [*] style pointers got {any}", .{ @TypeOf(callback), ChildTypeV }), } }, - else => @compileError("Only single pointer and slice are supported. Received " ++ @typeName(T)), - } + }; } test visit { @@ -520,22 +560,26 @@ test visit { const NestedAttr = struct { nested: Attr }; const NestedAttrOptional = struct { nested: ?Attr }; const SimpleStruct = struct { prop: Attr }; - const MultipleTypesStruct = struct { prop1: Attr, prop2: OtherAttr, prop3: ?Attr }; - const NestedTypesStruct = struct { prop1: Attr, prop2: OtherAttr, prop3: NestedAttr, prop4: NestedAttrOptional }; + const MultipleTypesStruct = struct { prop1: Attr, prop2: OtherAttr, prop3: ?Attr, prop4: []Attr }; + const NestedTypesStruct = struct { prop1: Attr, prop2: OtherAttr, prop3: NestedAttr, prop4: NestedAttrOptional, prop5: stdx.BoundedArray(Attr, 8) }; const LocalContext = struct { result: usize, + + fn cb(ctx: *@This(), attr: *const Attr) void { + ctx.result += attr.data; + } + + fn mutCb(ctx: *@This(), attr: *Attr) void { + ctx.result += attr.data; + } }; { var context: LocalContext = .{ .result = 0 }; const container: SimpleStruct = .{ .prop = .{ .data = 1 } }; - visit((struct { - fn cb(ctx: *LocalContext, attr: *const Attr) void { - ctx.result += attr.data; - } - }).cb, &context, &container); + visit(LocalContext.cb, &context, &container); try std.testing.expectEqual(1, context.result); } @@ -543,11 +587,143 @@ test visit { var context: LocalContext = .{ .result = 0 }; var container: SimpleStruct = .{ .prop = .{ .data = 1 } }; - visit((struct { - fn cb(ctx: *LocalContext, attr: *Attr) void { - ctx.result += attr.data; + visit(LocalContext.mutCb, &context, &container); + + try std.testing.expectEqual(1, context.result); + } + { + var context: LocalContext = .{ .result = 0 }; + var more_props: [3]Attr = .{ .{ .data = 4 }, .{ .data = 5 }, .{ .data = 6 } }; + + var container: MultipleTypesStruct = .{ .prop1 = .{ .data = 1 }, .prop2 = .{ .other = "hello" }, .prop3 = null, .prop4 = &more_props }; + + visit(LocalContext.mutCb, &context, &container); + + try std.testing.expectEqual(1 + 4 + 5 + 6, context.result); + } + { + var context: LocalContext = .{ .result = 0 }; + var more_props: [3]Attr = .{ .{ .data = 4 }, .{ .data = 5 }, .{ .data = 6 } }; + const container: MultipleTypesStruct = .{ .prop1 = .{ .data = 1 }, .prop2 = .{ .other = "hello" }, .prop3 = .{ .data = 2 }, .prop4 = &more_props }; + + visit(LocalContext.cb, &context, &container); + + try std.testing.expectEqual(1 + 2 + 4 + 5 + 6, context.result); + } + { + var context: LocalContext = .{ .result = 0 }; + const prop5: stdx.BoundedArray(Attr, 8) = .{ + .buffer = @splat(.{ .data = 4 }), + .len = 2, + }; + const container: NestedTypesStruct = .{ + .prop1 = .{ .data = 1 }, + .prop2 = .{ .other = "hello" }, + .prop3 = .{ .nested = .{ .data = 2 } }, + .prop4 = .{ .nested = .{ .data = 3 } }, + .prop5 = prop5, // 4 will be counted twice. + }; + + visit(LocalContext.cb, &context, &container); + + try std.testing.expectEqual(14, context.result); + } +} + +/// Similar to `visit` but doesn't follow pointers or slices. +/// This guarantees that the given type is a flat struct. +/// +/// see: `zml.meta.visit` +pub fn visitFlatStruct(comptime callback: anytype, ctx: FnParam(callback, 0), v: anytype) VisitReturn(callback) { + const action = comptime Visitor.determineAction(callback, @TypeOf(v)); + + const TargetType = switch (@typeInfo(FnParam(callback, 1))) { + .pointer => |info| info.child, + else => stdx.debug.compileError("zml.meta.visitStruct({any}) is expecting a callback with a pointer as second argument but found {any}", .{ @TypeOf(callback), FnParam(callback, 1) }), + }; + + const PtrTypeOfV = @TypeOf(v); + const ptr_info_v = @typeInfo(PtrTypeOfV).pointer; + const ChildTypeV = ptr_info_v.child; + + const can_error = stdx.meta.FnReturnErrorSet(callback) != null; + const err_msg = "zml.meta.visitStruct only handles flat struct, found pointer in {}"; + const err_args = .{ChildTypeV}; + + switch (action) { + .exit => {}, + .callback => return callback(ctx, v), + .recurse => { + if (comptime !Contains(ChildTypeV, TargetType)) return; + + // bounded array support (like visit) + if (@typeInfo(ChildTypeV) == .@"struct" and @hasDecl(ChildTypeV, "constSlice") and @hasDecl(ChildTypeV, "slice")) { + return for (v.slice()) |*elem| { + if (can_error) + try visitFlatStruct(callback, ctx, elem) + else + visitFlatStruct(callback, ctx, elem); + }; } - }).cb, &context, &container); + + return switch (ptr_info_v.size) { + .one => switch (@typeInfo(ChildTypeV)) { + .@"struct" => |s| inline for (s.field_names, s.field_types, s.field_attrs) |field_name, Field, field_attrs| { + if (field_attrs.@"comptime" or comptime !Contains(Field, TargetType)) continue; + stdx.debug.assertComptime(@typeInfo(Field) != .pointer, err_msg, err_args); + + if (can_error) try visitFlatStruct(callback, ctx, &@field(v, field_name)) else visitFlatStruct(callback, ctx, &@field(v, field_name)); + }, + .array => for (v[0..]) |*elem| { + if (can_error) try visitFlatStruct(callback, ctx, elem) else visitFlatStruct(callback, ctx, elem); + }, + .optional => if (v.*) |*opt| visitFlatStruct(callback, ctx, opt), + .@"union" => switch (v.*) { + inline else => |*v_field| visitFlatStruct(callback, ctx, v_field), + }, + else => stdx.debug.compileError(err_msg, err_args), + }, + else => stdx.debug.compileError(err_msg, err_args), + }; + }, + } +} + +test visitFlatStruct { + const Attr = struct { data: usize }; + const OtherAttr = struct { other: []const u8 }; + const NestedAttr = struct { nested: Attr }; + const NestedAttrOptional = struct { nested: ?Attr }; + const SimpleStruct = struct { prop: Attr }; + // Note: no slice anymore, no stdx.BoundedArray + const MultipleTypesStruct = struct { prop1: Attr, prop2: OtherAttr, prop3: ?Attr }; + const NestedTypesStruct = struct { prop1: Attr, prop2: OtherAttr, prop3: NestedAttr, prop4: NestedAttrOptional, prop5: stdx.BoundedArray(Attr, 8) }; + + const LocalContext = struct { + result: usize, + + fn cb(ctx: *@This(), attr: *const Attr) void { + ctx.result += attr.data; + } + + fn mutCb(ctx: *@This(), attr: *Attr) void { + ctx.result += attr.data; + } + }; + + { + var context: LocalContext = .{ .result = 0 }; + const container: SimpleStruct = .{ .prop = .{ .data = 1 } }; + + visitFlatStruct(LocalContext.cb, &context, &container); + + try std.testing.expectEqual(1, context.result); + } + { + var context: LocalContext = .{ .result = 0 }; + var container: SimpleStruct = .{ .prop = .{ .data = 1 } }; + + visitFlatStruct(LocalContext.mutCb, &context, &container); try std.testing.expectEqual(1, context.result); } @@ -555,11 +731,7 @@ test visit { var context: LocalContext = .{ .result = 0 }; var container: MultipleTypesStruct = .{ .prop1 = .{ .data = 1 }, .prop2 = .{ .other = "hello" }, .prop3 = null }; - visit((struct { - fn cb(ctx: *LocalContext, attr: *Attr) void { - ctx.result += attr.data; - } - }).cb, &context, &container); + visitFlatStruct(LocalContext.mutCb, &context, &container); try std.testing.expectEqual(1, context.result); } @@ -567,43 +739,57 @@ test visit { var context: LocalContext = .{ .result = 0 }; const container: MultipleTypesStruct = .{ .prop1 = .{ .data = 1 }, .prop2 = .{ .other = "hello" }, .prop3 = .{ .data = 2 } }; - visit((struct { - fn cb(ctx: *LocalContext, attr: *const Attr) void { - ctx.result += attr.data; - } - }).cb, &context, &container); + visitFlatStruct(LocalContext.cb, &context, &container); - try std.testing.expectEqual(3, context.result); + try std.testing.expectEqual(1 + 2, context.result); } { var context: LocalContext = .{ .result = 0 }; + const prop5: stdx.BoundedArray(Attr, 8) = .{ + .buffer = @splat(.{ .data = 4 }), + .len = 2, + }; const container: NestedTypesStruct = .{ .prop1 = .{ .data = 1 }, .prop2 = .{ .other = "hello" }, .prop3 = .{ .nested = .{ .data = 2 } }, .prop4 = .{ .nested = .{ .data = 3 } }, + .prop5 = prop5, // 4 will be counted twice. }; - visit((struct { - fn cb(ctx: *LocalContext, attr: *const Attr) void { - ctx.result += attr.data; - } - }).cb, &context, &container); + visitFlatStruct(LocalContext.cb, &context, &container); - try std.testing.expectEqual(6, context.result); + try std.testing.expectEqual(1 + 2 + 3 + 4 + 4, context.result); } } +pub fn count(T: type, value: anytype) u32 { + var counter: u32 = 0; + visit(struct { + pub fn cb(res: *u32, _: *const T) void { + res.* += 1; + } + }.cb, &counter, value); + return counter; +} + +pub fn first(T: type, value: anytype) T { + var res: ?T = null; + visit(struct { + pub fn cb(res_ptr: *?T, x: *const T) void { + if (res_ptr.* == null) res_ptr.* = x.*; + } + }.cb, &res, &value); + return res.?; +} /// Given a `fn([]const T, Args) T` and a slice of values, /// will combine all values in one value. /// Only T elements of values will be looked at. /// This only works for simple types, in particular `zip` doesn't follow pointers. /// Which means that zip only allocate temp memory, and nothing need to be freed after the call. -pub fn zip(func: anytype, allocator: std.mem.Allocator, values: anytype, args: anytype) error{OutOfMemory}!asSlice(@TypeOf(values)) { +pub fn zip(comptime func: anytype, allocator: std.mem.Allocator, values: anytype, args: anytype) error{OutOfMemory}!asSlice(@TypeOf(values)) { const sliceT = @typeInfo(FnParam(func, 0)); - assertComptime(sliceT == .Pointer and sliceT.Pointer.size == .Slice and sliceT.Pointer.child == FnResult(func), "zip requires a `fn([]const T, Args) T`, received: {}", .{@TypeOf(func)}); - - const T = sliceT.Pointer.child; + const T = sliceT.pointer.child; const V = asSlice(@TypeOf(values)); if (V == T) { return @call(.auto, func, .{values} ++ args); @@ -611,26 +797,26 @@ pub fn zip(func: anytype, allocator: std.mem.Allocator, values: anytype, args: a // const fn_args return switch (@typeInfo(V)) { - .Pointer => @compileError("zip only accept by value arguments. Received: " ++ @typeName(V)), - .Struct => |struct_info| { + .pointer => stdx.debug.compileError("zip only accept by value arguments. Received: {}", .{V}), + .@"struct" => |struct_info| { var out: V = values[0]; - inline for (struct_info.fields) |f| { - if (f.is_comptime) continue; - if (@typeInfo(f.type) == .Pointer) { - @compileError("zip doesn't follow pointers and don't accept struct containing them. Received: " ++ @typeName(V)); + inline for (struct_info.field_names, struct_info.field_types, struct_info.field_attrs) |field_name, Field, field_attrs| { + if (field_attrs.@"comptime") continue; + if (@typeInfo(Field) == .pointer) { + stdx.debug.compileError("zip doesn't follow pointers and don't accept struct containing them. Received: {}", .{V}); } - var fields = try allocator.alloc(f.type, values.len); + var fields = try allocator.alloc(Field, values.len); defer allocator.free(fields); for (values, 0..) |val, i| { - fields[i] = @field(val, f.name); + fields[i] = @field(val, field_name); } - @field(out, f.name) = try zip(func, allocator, fields, args); + @field(out, field_name) = try zip(func, allocator, fields, args); } return out; }, - .Array => |arr_info| { - if (@typeInfo(arr_info.child) == .Pointer) { - @compileError("zip doesn't follow pointers and don't accept struct containing them. Received: " ++ @typeName(V)); + .array => |arr_info| { + if (@typeInfo(arr_info.child) == .pointer) { + stdx.debug.compileError("zip doesn't follow pointers and don't accept struct containing them. Received: {}", .{V}); } var out: V = undefined; var slice = try allocator.alloc(arr_info.child, values.len); @@ -643,7 +829,7 @@ pub fn zip(func: anytype, allocator: std.mem.Allocator, values: anytype, args: a } return out; }, - .Union, .Optional => @compileError("zip doesn't yet support " ++ @typeName(V)), + .@"union", .optional => stdx.debug.compileError("zip doesn't yet support {}", .{V}), else => values[0], }; } @@ -666,33 +852,182 @@ test zip { /// Given a func(X) -> Y or a func(Ctx, X) -> Y, /// finds all X in the given object, and write the result of func(X) into an arraylist. -pub fn collect(func: anytype, func_ctx: _CollectCtx(func), out: *std.ArrayList(FnResult(func)), obj: anytype) error{OutOfMemory}!void { - assertComptime(@typeInfo(@TypeOf(func)).Fn.params.len <= 2, "zml.meta.collect expects a func with two arguments, got: {}", .{@TypeOf(func)}); +pub fn collectAlloc( + func: anytype, + func_ctx: _CollectCtx(func), + allocator: std.mem.Allocator, + obj: anytype, +) std.mem.Allocator.Error![]stdx.meta.FnReturn(func) { + stdx.debug.assertComptime(@typeInfo(@TypeOf(func)).@"fn".param_types.len <= 2, "zml.meta.collect expects a func with two arguments, got: {}", .{@TypeOf(func)}); + + const CollectAllocCtx = struct { + func_ctx: _CollectCtx(func), + allocator: std.mem.Allocator, + out: std.ArrayList(stdx.meta.FnReturn(func)) = .empty, + oom: bool = false, + + fn cb(ctx: *@This(), val: *const _CollectArg(func)) void { + if (ctx.oom) return; + const res = if (_CollectCtx(func) == void) func(val.*) else func(ctx.func_ctx, val.*); + ctx.out.append(ctx.allocator, res) catch { + ctx.oom = true; + }; + } + }; + var context = CollectAllocCtx{ .func_ctx = func_ctx, .allocator = allocator }; + visit(CollectAllocCtx.cb, &context, obj); + if (context.oom) return error.OutOfMemory; + + return context.out.toOwnedSlice(allocator); +} + +/// Given a func(X) -> Y or a func(Ctx, X) -> Y, +/// finds all X in the given object, and write the result of func(X) into a given slice. +/// Asserts that the number of X found is equal to the slice len. +pub fn collectBuf(func: anytype, func_ctx: _CollectCtx(func), obj: anytype, out: []stdx.meta.FnReturn(func)) void { + stdx.debug.assertComptime(@typeInfo(@TypeOf(func)).@"fn".param_types.len <= 2, "zml.meta.collectBuf expects a func with one or two arguments, got: {}", .{@TypeOf(func)}); const LocalContext = struct { func_ctx: _CollectCtx(func), - out: *std.ArrayList(FnResult(func)), - oom: bool = false, + out: @TypeOf(out), + idx: usize = 0, }; var context = LocalContext{ .func_ctx = func_ctx, .out = out }; visit((struct { fn cb(ctx: *LocalContext, val: *const _CollectArg(func)) void { - if (ctx.oom) return; + if (ctx.idx >= ctx.out.len) return; + const res = if (_CollectCtx(func) == void) func(val.*) else func(ctx.func_ctx, val.*); - ctx.out.append(res) catch { + ctx.out[ctx.idx] = res; + ctx.idx += 1; + } + }).cb, &context, obj); + std.debug.assert(context.idx == context.out.len); +} + +/// Finds all X in the given object, and write their pointers into an arraylist. +pub fn collectPtrs( + T: type, + allocator: std.mem.Allocator, + obj: anytype, +) std.mem.Allocator.Error![]*const T { + const CollectPtrCtx = struct { + allocator: std.mem.Allocator, + out: std.ArrayList(*const T) = .empty, + oom: bool = false, + + fn cb(ctx: *@This(), val: *const T) void { + if (ctx.oom) return; + ctx.out.append(ctx.allocator, val) catch { ctx.oom = true; }; } - }).cb, &context, obj); + }; + var context = CollectPtrCtx{ .allocator = allocator }; + visit(CollectPtrCtx.cb, &context, obj); if (context.oom) return error.OutOfMemory; + + return context.out.toOwnedSlice(allocator); +} + +/// Finds all X in the given object, and write their mutable pointers into an arraylist. +pub fn collectMutPtrs( + T: type, + allocator: std.mem.Allocator, + obj: anytype, +) std.mem.Allocator.Error![]*T { + const ok = switch (@typeInfo(@TypeOf(obj))) { + .pointer => |ptr_info| !ptr_info.attrs.@"const", + else => false, + }; + stdx.debug.assertComptime(ok, "zml.meta.collectMutPtrs requires a mutable pointer input, got: {}", .{@TypeOf(obj)}); + return @ptrCast(try collectPtrs(T, allocator, obj)); +} + +/// Finds all X in the given object, and copy them by value into an arraylist. +pub fn collectValues( + T: type, + allocator: std.mem.Allocator, + obj: anytype, +) std.mem.Allocator.Error![]T { + const CollectValuesCtx = struct { + allocator: std.mem.Allocator, + out: std.ArrayList(T) = .empty, + oom: bool = false, + + fn cb(ctx: *@This(), val: *const T) void { + if (ctx.oom) return; + ctx.out.append(ctx.allocator, val.*) catch { + ctx.oom = true; + }; + } + }; + var context = CollectValuesCtx{ .allocator = allocator }; + visit(CollectValuesCtx.cb, &context, obj); + if (context.oom) return error.OutOfMemory; + + return context.out.toOwnedSlice(allocator); } fn _CollectCtx(func: anytype) type { - const params = @typeInfo(@TypeOf(func)).Fn.params; + const params = @typeInfo(@TypeOf(func)).@"fn".param_types; if (params.len == 1) return void; - return params[0].type orelse @compileError("anytype not supported in collect"); + return params[0] orelse @compileError("anytype not supported in collect"); } fn _CollectArg(func: anytype) type { - const params = @typeInfo(@TypeOf(func)).Fn.params; - return params[params.len - 1].type orelse @compileError("anytype not supported in collect"); + const params = @typeInfo(@TypeOf(func)).@"fn".param_types; + return params[params.len - 1] orelse @compileError("anytype not supported in collect"); +} + +pub fn Contains(Haystack: type, T: type) bool { + switch (Haystack) { + T, ?T => return true, + *T, ?*T => return true, + *const T, ?*const T => return true, + []const T, ?[]const T => return true, + anyopaque => return false, + else => {}, + } + + return switch (@typeInfo(Haystack)) { + .@"struct" => |info| { + if (@hasDecl(Haystack, "zml_no_meta_visit")) return false; + inline for (info.field_types, info.field_attrs) |Field, field_attrs| { + if (!field_attrs.@"comptime" and Contains(Field, T)) { + return true; + } + } + return false; + }, + .@"union" => |info| { + if (@hasDecl(Haystack, "zml_no_meta_visit")) return false; + inline for (info.field_types) |Field| { + if (Contains(Field, T)) + return true; + } + return false; + }, + .array => |info| Contains(info.child, T), + .pointer => |info| Contains(info.child, T), + .optional => |info| Contains(info.child, T), + .vector => |info| Contains(info.child, T), + else => false, + }; +} + +pub fn forEachVisit(v: anytype, comptime T: type, f: anytype, args: stdx.meta.TupleRange(std.meta.ArgsTuple(@TypeOf(f)), 2, null)) void { + const Ctx = struct { + i: usize, + args_: @TypeOf(args), + }; + var ctx: Ctx = .{ + .i = 0, + .args_ = args, + }; + visit(struct { + fn cb(ctx_: *Ctx, elem: T) void { + @call(.auto, f, .{ ctx_.i, elem } ++ ctx_.args_); + ctx_.i += 1; + } + }.cb, &ctx, v); } diff --git a/zml/mlir.zig b/zml/mlir.zig deleted file mode 100644 index b90763de..00000000 --- a/zml/mlir.zig +++ /dev/null @@ -1,176 +0,0 @@ -const mlir = @This(); - -const builtin = @import("builtin"); -const std = @import("std"); - -const dtype = @import("dtype.zig"); -const meta = @import("meta.zig"); - -const Shape = @import("shape.zig").Shape; -const Tensor = @import("tensor.zig").Tensor; - -const log = std.log.scoped(.zml_mlir); - -pub usingnamespace @import("mlir"); - -pub const ext = struct { - pub fn mlirType(ctx: mlir.Context, sh: Shape) mlir.Type { - return mlir.RankedTensorType.init(sh.dims(), mlir.ext.Type.fromDType(ctx, sh.dtype())).as(mlir.Type).?; - } - - pub fn denseElementAttrType(dt: dtype.DataType) mlir.DenseElementsAttributeTypes { - return switch (dt) { - .bool => .bool, - .i8 => .i8, - .i16 => .i16, - .i32 => .i32, - .i64 => .i64, - .u8 => .u8, - .u16 => .u16, - .u32 => .u32, - .u64 => .u64, - .bf16 => .bf16, - .f16 => .f16, - .f32 => .f32, - .f64 => .f64, - inline else => |tag| @panic("Unsupported data type: " ++ @tagName(tag)), - }; - } - - pub fn denseElementsAttr(dt: dtype.DataType, _: usize, bytes: []const u8, ranked_type: mlir.RankedTensorType) mlir.Attribute { - const ranked_type_ = ranked_type.as(mlir.Type).?; - return switch (dt) { - .bool => mlir.DenseIntOrFPElementsAttribute(.bool).init(ranked_type_, bytes).as(mlir.Attribute).?, - .i8 => mlir.DenseIntOrFPElementsAttribute(.i8).init(ranked_type_, bytes).as(mlir.Attribute).?, - .i16 => mlir.DenseIntOrFPElementsAttribute(.i16).init(ranked_type_, bytes).as(mlir.Attribute).?, - .i32 => mlir.DenseIntOrFPElementsAttribute(.i32).init(ranked_type_, bytes).as(mlir.Attribute).?, - .i64 => mlir.DenseIntOrFPElementsAttribute(.i64).init(ranked_type_, bytes).as(mlir.Attribute).?, - .u8 => mlir.DenseIntOrFPElementsAttribute(.u8).init(ranked_type_, bytes).as(mlir.Attribute).?, - .u16 => mlir.DenseIntOrFPElementsAttribute(.u16).init(ranked_type_, bytes).as(mlir.Attribute).?, - .u32 => mlir.DenseIntOrFPElementsAttribute(.u32).init(ranked_type_, bytes).as(mlir.Attribute).?, - .u64 => mlir.DenseIntOrFPElementsAttribute(.u64).init(ranked_type_, bytes).as(mlir.Attribute).?, - .bf16 => mlir.DenseIntOrFPElementsAttribute(.bf16).init(ranked_type_, bytes).as(mlir.Attribute).?, - .f16 => mlir.DenseIntOrFPElementsAttribute(.f16).init(ranked_type_, bytes).as(mlir.Attribute).?, - .f32 => mlir.DenseIntOrFPElementsAttribute(.f32).init(ranked_type_, bytes).as(mlir.Attribute).?, - .f64 => mlir.DenseIntOrFPElementsAttribute(.f64).init(ranked_type_, bytes).as(mlir.Attribute).?, - inline else => |tag| @panic("Unsupported data type: " ++ @tagName(tag)), - }; - } - - pub const RankedTensorType = struct { - pub fn fromShape(ctx: mlir.Context, sh: Shape) mlir.RankedTensorType { - return mlir.RankedTensorType.init(sh.dims(), mlir.ext.Type.fromDType(ctx, sh.dtype())); - } - }; - - pub const Type = struct { - pub fn fromDType(ctx: mlir.Context, dt: dtype.DataType) mlir.Type { - return switch (dt) { - .bool => mlir.IntegerType(.i1).init(ctx).as(mlir.Type).?, - .f8e4m3b11fnuz => mlir.FloatType(.f8e4m3b11fnuz).init(ctx).as(mlir.Type).?, - .f8e4m3fn => mlir.FloatType(.f8e4m3fn).init(ctx).as(mlir.Type).?, - .f8e4m3fnuz => mlir.FloatType(.f8e4m3fnuz).init(ctx).as(mlir.Type).?, - .f8e5m2 => mlir.FloatType(.f8e5m2).init(ctx).as(mlir.Type).?, - .f8e5m2fnuz => mlir.FloatType(.f8e5m2fnuz).init(ctx).as(mlir.Type).?, - .bf16 => mlir.FloatType(.bf16).init(ctx).as(mlir.Type).?, - .f16 => mlir.FloatType(.f16).init(ctx).as(mlir.Type).?, - .f32 => mlir.FloatType(.f32).init(ctx).as(mlir.Type).?, - .f64 => mlir.FloatType(.f64).init(ctx).as(mlir.Type).?, - .i4 => mlir.IntegerType(.i4).init(ctx).as(mlir.Type).?, - .i8 => mlir.IntegerType(.i8).init(ctx).as(mlir.Type).?, - .i16 => mlir.IntegerType(.i16).init(ctx).as(mlir.Type).?, - .i32 => mlir.IntegerType(.i32).init(ctx).as(mlir.Type).?, - .i64 => mlir.IntegerType(.i64).init(ctx).as(mlir.Type).?, - .u4 => mlir.IntegerType(.u4).init(ctx).as(mlir.Type).?, - .u8 => mlir.IntegerType(.u8).init(ctx).as(mlir.Type).?, - .u16 => mlir.IntegerType(.u16).init(ctx).as(mlir.Type).?, - .u32 => mlir.IntegerType(.u32).init(ctx).as(mlir.Type).?, - .u64 => mlir.IntegerType(.u64).init(ctx).as(mlir.Type).?, - .c64 => mlir.ComplexType(.c64).init(ctx).as(mlir.Type).?, - .c128 => mlir.ComplexType(.c128).init(ctx).as(mlir.Type).?, - }; - } - - pub fn toDType(mlir_type: mlir.Type) dtype.DataType { - const mapping = .{ - .{ .bool, mlir.IntegerType(.i1) }, - - .{ .f8e4m3b11fnuz, mlir.FloatType(.f8e4m3b11fnuz) }, - .{ .f8e4m3fn, mlir.FloatType(.f8e4m3fn) }, - .{ .f8e4m3fnuz, mlir.FloatType(.f8e4m3fnuz) }, - .{ .f8e5m2, mlir.FloatType(.f8e5m2) }, - .{ .f8e5m2fnuz, mlir.FloatType(.f8e5m2fnuz) }, - .{ .bf16, mlir.FloatType(.bf16) }, - .{ .f16, mlir.FloatType(.f16) }, - .{ .f32, mlir.FloatType(.f32) }, - .{ .f64, mlir.FloatType(.f64) }, - - .{ .i4, mlir.IntegerType(.i4) }, - .{ .i8, mlir.IntegerType(.i8) }, - .{ .i16, mlir.IntegerType(.i16) }, - .{ .i32, mlir.IntegerType(.i32) }, - .{ .i64, mlir.IntegerType(.i64) }, - - .{ .u4, mlir.IntegerType(.u4) }, - .{ .u8, mlir.IntegerType(.u8) }, - .{ .u16, mlir.IntegerType(.u16) }, - .{ .u32, mlir.IntegerType(.u32) }, - .{ .u64, mlir.IntegerType(.u64) }, - - .{ .c64, mlir.ComplexType(.c64) }, - .{ .c128, mlir.ComplexType(.c128) }, - }; - - inline for (mapping) |entry| { - const dt, const mlirT = entry; - if (mlir_type.as(mlirT)) |_| { - return dt; - } - } - - meta.panic("Could not convert mlir.Type to DataType: {}", .{mlir_type}); - } - }; - - pub const Attribute = struct { - pub fn fromData(data: dtype.Data, ctx: mlir.Context) mlir.Attribute { - switch (data) { - .bool => |val| { - return mlir.IntegerAttribute(.i1).init(ctx, @intFromBool(val)).as(mlir.Attribute).?; - }, - inline .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz => |val, tag| { - const float_type = @field(mlir.FloatTypes, @tagName(tag)); - const float_attr = mlir.FloatAttribute(float_type).init(ctx, val.toF32()); - return float_attr.as(mlir.Attribute).?; - }, - inline .i4, .i8, .i16, .i32, .i64, .u4, .u8, .u16, .u32, .u64 => |val, tag| { - const int_type = @field(mlir.IntegerTypes, @tagName(tag)); - const int_attr = mlir.IntegerAttribute(int_type).init(ctx, @intCast(val)); - return int_attr.as(mlir.Attribute).?; - }, - inline else => |_, tag| meta.panic("Unsupported data type: {any}", .{tag}), - } - } - }; - - pub const DenseIntOrFPElementsAttribute = struct { - pub fn fromData(data: dtype.Data, result_type: mlir.Type) mlir.Attribute { - return switch (data.dtype()) { - .bool => mlir.DenseIntOrFPElementsAttribute(.bool).init(result_type, data.constSlice()).as(mlir.Attribute).?, - .i8 => mlir.DenseIntOrFPElementsAttribute(.i8).init(result_type, data.constSlice()).as(mlir.Attribute).?, - .i16 => mlir.DenseIntOrFPElementsAttribute(.i16).init(result_type, data.constSlice()).as(mlir.Attribute).?, - .i32 => mlir.DenseIntOrFPElementsAttribute(.i32).init(result_type, data.constSlice()).as(mlir.Attribute).?, - .i64 => mlir.DenseIntOrFPElementsAttribute(.i64).init(result_type, data.constSlice()).as(mlir.Attribute).?, - .u8 => mlir.DenseIntOrFPElementsAttribute(.u8).init(result_type, data.constSlice()).as(mlir.Attribute).?, - .u16 => mlir.DenseIntOrFPElementsAttribute(.u16).init(result_type, data.constSlice()).as(mlir.Attribute).?, - .u32 => mlir.DenseIntOrFPElementsAttribute(.u32).init(result_type, data.constSlice()).as(mlir.Attribute).?, - .u64 => mlir.DenseIntOrFPElementsAttribute(.u64).init(result_type, data.constSlice()).as(mlir.Attribute).?, - .bf16 => mlir.DenseIntOrFPElementsAttribute(.bf16).init(result_type, data.constSlice()).as(mlir.Attribute).?, - .f16 => mlir.DenseIntOrFPElementsAttribute(.f16).init(result_type, data.constSlice()).as(mlir.Attribute).?, - .f32 => mlir.DenseIntOrFPElementsAttribute(.f32).init(result_type, data.constSlice()).as(mlir.Attribute).?, - .f64 => mlir.DenseIntOrFPElementsAttribute(.f64).init(result_type, data.constSlice()).as(mlir.Attribute).?, - inline else => |tag| meta.panic("Unsupported data type: {any}", .{tag}), - }; - } - }; -}; diff --git a/zml/mlirx.zig b/zml/mlirx.zig new file mode 100644 index 00000000..bc92eee0 --- /dev/null +++ b/zml/mlirx.zig @@ -0,0 +1,47 @@ +const std = @import("std"); + +const mlir = @import("mlir"); + +const dtype = @import("dtype.zig"); +const Shape = @import("shape.zig").Shape; + +const mlirx = @This(); + +pub const Type = struct { + pub fn rankedTensor(ctx: *mlir.Context, shape: Shape) *const mlir.Type { + return .rankedTensor(shape.dims(), mlirx.Type.fromDType(ctx, shape.dtype())); + } + + pub fn fromDType(ctx: *mlir.Context, dt: dtype.DataType) *const mlir.Type { + return switch (dt) { + .bool => .int(ctx, .i1), + .f4e2m1 => .float(ctx, .f4e2m1fn), + .f8e3m4 => .float(ctx, .f8e3m4), + .f8e4m3 => .float(ctx, .f8e4m3), + .f8e4m3fn => .float(ctx, .f8e4m3fn), + .f8e4m3fnuz => .float(ctx, .f8e4m3fnuz), + .f8e4m3b11fnuz => .float(ctx, .f8e4m3b11fnuz), + .f8e5m2 => .float(ctx, .f8e5m2), + .f8e5m2fnuz => .float(ctx, .f8e5m2fnuz), + .f8e8m0 => .float(ctx, .f8e8m0fnu), + .bf16 => .float(ctx, .bf16), + .f16 => .float(ctx, .f16), + .f32 => .float(ctx, .f32), + .f64 => .float(ctx, .f64), + .i2 => .int(ctx, .i2), + .i4 => .int(ctx, .i4), + .i8 => .int(ctx, .i8), + .i16 => .int(ctx, .i16), + .i32 => .int(ctx, .i32), + .i64 => .int(ctx, .i64), + .u2 => .int(ctx, .u2), + .u4 => .int(ctx, .u4), + .u8 => .int(ctx, .u8), + .u16 => .int(ctx, .u16), + .u32 => .int(ctx, .u32), + .u64 => .int(ctx, .u64), + .c64 => .complex(ctx, .c64), + .c128 => .complex(ctx, .c128), + }; + } +}; diff --git a/zml/module.zig b/zml/module.zig deleted file mode 100644 index f87ad8d6..00000000 --- a/zml/module.zig +++ /dev/null @@ -1,1555 +0,0 @@ -const builtin = @import("builtin"); -const std = @import("std"); -const pjrt = @import("pjrt"); - -const runfiles = @import("runfiles"); - -const xla_pb = @import("//xla:xla_proto"); -const meta = @import("meta.zig"); -const mlir = @import("mlir.zig"); -const ops = @import("ops.zig"); -const protobuf = @import("io/protobuf"); -const asynk = @import("async"); -const aio = @import("aio.zig"); - -const dialect = @import("mlir/dialects"); - -const assert = std.debug.assert; -const Context = @import("context.zig").Context; -const Location = mlir.Location; -const Platform = @import("platform.zig").Platform; -const Target = @import("platform.zig").Target; -const Tensor = @import("tensor.zig").Tensor; -const ShapeOf = @import("tensor.zig").ShapeOf; -const Shape = @import("shape.zig").Shape; -const Buffer = @import("buffer.zig").Buffer; -const Bufferized = @import("tensor.zig").Bufferized; -const Tracer = @import("tools/tracer.zig").Tracer; - -const log = std.log.scoped(.zml_module); - -test { - std.testing.refAllDecls(@This()); -} - -pub const CompilationContext = struct { - _platform: Platform, - - _mlir_ctx: mlir.Context, - _mlir_registry: mlir.Registry, - _mlir_canonicalizer: mlir.PassManager, - - _module: mlir.Module, - - _blocks: std.BoundedArray(mlir.Block, 64), - _fn_cache: FnCache, - _allocator: std.mem.Allocator, - - _buffer_to_arg: TensorToBlockArg = .{}, - _unique_id: u64 = 10000, - _tracer: Tracer, - - threadlocal var _current: ?*CompilationContext = null; - - const TensorToBlockArg = std.AutoHashMapUnmanaged(Tensor._Id, struct { mlir.Value, Tensor._Donation }); - const AttributeList = std.BoundedArray(mlir.NamedAttribute, 3); - - pub fn init(allocator: std.mem.Allocator, name: []const u8, platform: Platform) !CompilationContext { - const mlir_registry = mlir.Registry.init() catch unreachable; - inline for (.{ "func", "stablehlo" }) |d| { - mlir.DialectHandle.fromString(d).insertDialect(mlir_registry); - } - var mlir_ctx = mlir.Context.initWithRegistry(mlir_registry, false) catch unreachable; - mlir_ctx.loadAllAvailableDialects(); - - const loc = mlir_ctx.location(@src()).named(mlir_ctx, "main"); - const module = mlir.Module.init(loc); - module.op().setAttributeByName("sym_name", mlir.StringAttribute.init(mlir_ctx, name).as(mlir.Attribute).?); - - var canonicalizer = try mlir.PassManager.init(mlir_ctx); - { - var opm = canonicalizer.asOpPassManager(); - try opm.addPipeline("canonicalize"); - try opm.addPipeline("cse"); - try opm.addPipeline("canonicalize"); - } - - return .{ - ._platform = platform, - ._mlir_ctx = mlir_ctx, - ._mlir_registry = mlir_registry, - ._mlir_canonicalizer = canonicalizer, - ._module = module, - ._blocks = .{}, - ._fn_cache = FnCache.init(allocator), - ._allocator = allocator, - ._tracer = Tracer.init("ai.zml.compilation"), - }; - } - - pub fn deinit(self: *CompilationContext) void { - self._fn_cache.deinit(); - self._mlir_ctx.deinit(); - self._mlir_registry.deinit(); - self._buffer_to_arg.deinit(self._allocator); - } - - pub fn activate(self: *CompilationContext) void { - _current = self; - } - - pub fn deactivate(self: *CompilationContext) void { - std.debug.assert(_current != null and _current.? == self); - _current = null; - } - - pub fn current() *CompilationContext { - return _current.?; - } - - pub fn target(self: *const CompilationContext) Target { - return self._platform.target; - } - - pub fn targetIs(self: *const CompilationContext, value: Target) bool { - return self.target() == value; - } - - pub fn mlirCtx(self: *const CompilationContext) mlir.Context { - return self._mlir_ctx; - } - - pub fn currentBlock(self: *const CompilationContext) ?mlir.Block { - return if (self._blocks.len > 0) self._blocks.get(self._blocks.len - 1) else null; - } - - pub fn openBlock(self: *CompilationContext, args: []const mlir.Type, locs: []const mlir.Location) !mlir.Block { - const block = try mlir.Block.init(args, locs); - self.pushBlock(block); - return block; - } - - pub fn closeBlock(self: *CompilationContext, block: *mlir.Block) void { - const popped = self._blocks.pop(); - std.debug.assert(block.equals(popped)); - } - - fn pushBlock(self: *CompilationContext, block: mlir.Block) void { - self._blocks.appendAssumeCapacity(block); - } - - /// Transform a Tensor -> Tensor function into an Mlir block. - /// `blkctx` represents values from outside the block that can be accessed inside the block. - pub fn makeBlock( - self: *CompilationContext, - comptime S: ops.BlockSignature, - func: *const S.Fn, - blkctx: S.BlkCtx, - args: S.Args, - ) mlir.Block { - const N = S.nIn; - const locations = .{mlir.Location.unknown(self.mlirCtx())} ** N; - var input_types: [N]mlir.Type = undefined; - fillMlirTypes(&args, self.mlirCtx(), &input_types); - - var block = self.openBlock(&input_types, &locations) catch unreachable; - defer self.closeBlock(&block); - - // Here we want to create the block with the correct mlir types. - // but we don't want to use the values themselves. - // So we create a copy of the arguments, and replace values - // by the block arguments. - var blk_args = args; - assert(assignBlockArguments(&blk_args, block, 0) == N); - - const loc = self.mlirCtx().location(@src()); - const block_res = @call(.auto, func, S.blkArgs(blkctx, blk_args)); - - var block_res_values: [S.nOut]mlir.Value = undefined; - self.extractValues(&block_res, &block_res_values); - const block_ret = dialect.stablehlo.returns_(self.mlirCtx(), &block_res_values, loc); - block.addOperationsRecursive(block_ret); - - return block; - } - - /// Generate an MLIR function from a ZML function. - /// The caller is responsible to have properly created the input - /// tensors with unique tensor ids. - pub fn generateBytecode( - self: *CompilationContext, - allocator: std.mem.Allocator, - fn_name: [:0]const u8, - comptime func: anytype, - model: *const ModuleSignature(func).ModelT, - args: *const ModuleSignature(func).ArgsT, - ) error{OutOfMemory}!MlirFn { - const frame = self._tracer.frameStart("generateBytecode.emit"); - errdefer self._tracer.frameEnd(frame, "generateBytecode.emit"); - - // Note: only temp allocations are done in the arena, - // the other allocations are managed by the caller. - var arena_state = std.heap.ArenaAllocator.init(allocator); - defer arena_state.deinit(); - const arena = arena_state.allocator(); - - const model_tensor_count = countTensors(model); - const args_tensor_count = countTensors(args); - - const tensor_count = model_tensor_count + args_tensor_count; - - const mlir_ctx = self.mlirCtx(); - const loc = mlir_ctx.location(@src()); - - const locations = try arena.alloc(mlir.Location, tensor_count); - @memset(locations, mlir.Location.unknown(mlir_ctx)); - - var input_shapes = try std.ArrayList(Shape).initCapacity(arena, tensor_count); - meta.collect(Tensor.shape, {}, &input_shapes, model) catch unreachable; - meta.internalAssert(input_shapes.items.len == model_tensor_count, "model has changed ?", .{}); - meta.collect(Tensor.shape, {}, &input_shapes, args) catch unreachable; - meta.internalAssert(input_shapes.items.len == tensor_count, "args have changed ?", .{}); - - const input_types = try arena.alloc(mlir.Type, tensor_count); - for (input_types, input_shapes.items) |*t, sh| t.* = mlir.ext.mlirType(mlir_ctx, sh); - - // Note: this isn't stricly necessary. We call `countTensor` on `fn_res`. - // But it forces user to have simpler function. - const out_tensor_count = comptime ops.staticCountTensors(ModuleSignature(func).ReturnT) orelse @compileError("Can't use " ++ @typeName(ModuleSignature(func).ReturnT) ++ " in an MLIR function, because it has a variable number of tensors"); - // Those are returned to caller so we don't put them in the arena. - const fn_res_types = try allocator.alloc(mlir.Type, out_tensor_count); - const fn_res_shapes = try allocator.alloc(Shape, out_tensor_count); - const fn_res_donations = try allocator.alloc(Tensor._Donation, out_tensor_count); - var fn_body = self.openBlock(input_types, locations) catch unreachable; - { - defer self.closeBlock(&fn_body); - // Note: we could shrink self._buffer_to_arg once we called `func`. - // But for now we are only compiling one function per CompilationContext. - // So we don't need to do this since we won't reuse self._buffer_to_arg anyway. - // const n = self._buffer_to_arg.count(); - // defer self._buffer_to_arg.shrinkRetainingCapacity(n); - - try self._buffer_to_arg.ensureUnusedCapacity(self._allocator, @intCast(tensor_count)); - const assigned_model_count = self.mapBlockArguments(model, fn_body, 0); - const assigned_args_count = self.mapBlockArguments(args, fn_body, assigned_model_count); - assert(assigned_model_count == model_tensor_count); - assert(assigned_args_count == tensor_count); - - const fn_res = forward: { - self.activate(); - defer self.deactivate(); - break :forward @call(.auto, func, .{model.*} ++ args.*); - }; - - var fn_res_values: [out_tensor_count]mlir.Value = undefined; - self.extractValuesAndTypes(&fn_res, &fn_res_values, fn_res_types, fn_res_shapes, fn_res_donations); - - const fn_ret = dialect.func.return_(mlir_ctx, &fn_res_values, loc); - fn_body.addOperationsRecursive(fn_ret); - } - - const arg_attrs = try arena.alloc(AttributeList, tensor_count); - @memset(arg_attrs, .{}); - - const res_attrs = try arena.alloc(AttributeList, out_tensor_count); - @memset(res_attrs, .{}); - - // Donations attributes only make sense on the main function. - self.addDonationsAttributes(arg_attrs, fn_res_donations); - - if (self._platform.sharding().num_partitions > 1) { - self.addShardingAttributes(arg_attrs, res_attrs, input_shapes.items, fn_res_shapes); - } - const mlir_fn = dialect.func.func(self.mlirCtx(), .{ - .sym_name = fn_name, - .args = input_types, - .arg_attrs = try finalizeAttributeList(arena, mlir_ctx, arg_attrs), - .results = fn_res_types, - .res_attrs = try finalizeAttributeList(arena, mlir_ctx, res_attrs), - .block = fn_body, - .location = loc, - }); - - self._tracer.frameEnd(frame, "generateBytecode.emit"); - const canonicalize_frame = self._tracer.frameStart("generateBytecode.canonicalize"); - defer self._tracer.frameEnd(canonicalize_frame, "generateBytecode.canonicalize"); - self._mlir_canonicalizer.runOnOp(mlir_fn) catch |err| switch (err) { - error.InvalidMlir => { - log.err("Failed to canonicalize invalid mlir: {}", .{mlir_fn.mlirFormatter(.{})}); - // user errors should have triggered a panic before we reach this. - @panic("ZML generated invalid mlir. Please open a bug report"); - }, - }; - - return .{ - .mlir_fn = mlir_fn, - .name = fn_name, - .n_model = @intCast(model_tensor_count), - .n_args = @intCast(args_tensor_count), - .res_types = fn_res_types, - .res_shapes = fn_res_shapes, - .res_donations = fn_res_donations, - }; - } - - /// Given a list of donations mapping output buffers to input buffers, - /// generate donation attribute for each `n_args` input argument. - fn addDonationsAttributes(self: CompilationContext, attributes: []AttributeList, donations: []const Tensor._Donation) void { - var n_donations: usize = 0; - for (donations, 0..) |donation, index| { - switch (donation) { - .no_buffer => {}, - // This is an input buffer that has been returned, - // but without explicitly calling `reuseBuffer`. - // So we assume the intent was to return a new buffer. - .input_buffer => {}, - .arg => |a| { - n_donations += 1; - // This will break the day we writer another attribute before donation. - // When the time come, do a more fancy lookup here to check if an argument - // is donated twice. - meta.assert(attributes[a].len == 0, "Donation error ! Argument {} has been donated twice ! To {} and to {}", .{ a, index, attributes[a].buffer[0] }); - attributes[a].appendAssumeCapacity( - mlir.NamedAttribute.init( - mlir.Identifier.get(self.mlirCtx(), "tf.aliasing_output"), - mlir.IntegerAttribute(.i32).init(self.mlirCtx(), @intCast(index)).as(mlir.Attribute).?, - ), - ); - // log.debug("attribute: {}", .{attributes[a].constSlice()}); - }, - } - } - } - - test addDonationsAttributes { - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - var arena = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena.deinit(); - const allocator = arena.allocator(); - - const s = Shape.init(.{8}, .f16); - - const Local = struct { - bias: Tensor, - - pub fn forward(self: @This(), x: Tensor) Tensor { - const y = x.add(self.bias); - return y.reuseBuffer(x); - } - }; - - const model: Local = .{ - .bias = zml.Tensor{ ._shape = s, ._id = .{ .buffer_id = 0 } }, - }; - - var comp = try zml.module.CompilationContext.init(allocator, "test", platform); - defer comp.deinit(); - var tensor_args = .{Tensor{ ._shape = s, ._id = .{ .arg_id = 1234 } }}; - const f = try comp.generateBytecode(allocator, "test.generateBytecode.Local.forward", Local.forward, &model, &tensor_args); - - var mlir_bytecode: std.ArrayListUnmanaged(u8) = .{}; - try mlir_bytecode.writer(allocator).print("{}", .{f.mlir_fn.mlirFormatter(.{})}); - - // Check that the `x` input argument gives its buffer to the result tensor. - // `%arg0` is the bias of the model, `%arg1` is `x`. - try std.testing.expectEqual(1, f.n_model); - try std.testing.expectEqual(1, f.n_args); - std.testing.expect(std.mem.indexOf(u8, mlir_bytecode.items, "tf.aliasing_output = 0 : i32") != null) catch |err| { - log.warn("Didn't produced the expected IR:\n{s}", .{mlir_bytecode.items}); - return err; - }; - } - - pub fn getShardingAttr(self: CompilationContext, shape: Shape) mlir.StringAttribute { - const mlir_ctx = self.mlirCtx(); - - const num_partitions = self._platform.sharding().num_partitions; - var sharding_str: std.BoundedArray(u8, 128) = .{}; - - writeShardingRepresentation(shape, num_partitions, sharding_str.writer()) catch unreachable; - return mlir.StringAttribute.init(mlir_ctx, sharding_str.constSlice()); - } - - fn addShardingAttributes(self: CompilationContext, arg_attrs: []AttributeList, res_attrs: []AttributeList, input_shapes: []const Shape, output_shapes: []const Shape) void { - const mlir_ctx = self.mlirCtx(); - if (!self._platform.compilation_options.sharding_enabled) return; - - const mhlo_default_layout = mlir.NamedAttribute.init( - mlir.Identifier.get(mlir_ctx, "mhlo.layout_mode"), - mlir.StringAttribute.init(mlir_ctx, "default").asAttr(), - ); - for (arg_attrs, input_shapes) |*attr, shape| { - attr.appendAssumeCapacity(mhlo_default_layout); - - const sharding_attr = self.getShardingAttr(shape); - attr.appendAssumeCapacity(mlir.NamedAttribute.init( - mlir.Identifier.get(mlir_ctx, "mhlo.sharding"), - sharding_attr.asAttr(), - )); - } - - for (res_attrs, output_shapes) |*attr, shape| { - attr.appendAssumeCapacity(mhlo_default_layout); - - const sharding_attr = self.getShardingAttr(shape); - - attr.appendAssumeCapacity(mlir.NamedAttribute.init( - mlir.Identifier.get(mlir_ctx, "mhlo.sharding"), - sharding_attr.asAttr(), - )); - } - } - - fn writeShardingRepresentation(shape: Shape, num_partitions: u8, writer: anytype) @TypeOf(writer).Error!void { - const n_sharded: u8 = @popCount(@as(u8, @bitCast(shape._sharding_info))); - if (n_sharded == 0 or num_partitions == 1) { - try writer.writeAll("{replicated}"); - return; - } - try writer.writeAll("{devices=["); - for (0..shape.rank()) |i| { - try writer.print("{d}", .{if (shape._sharding_info[i]) num_partitions else 1}); - if (i < shape.rank() - 1) try writer.writeByte(','); - } - try writer.print("]<=[{d}]}}", .{num_partitions}); - } - - test writeShardingRepresentation { - var rule: [64]u8 = undefined; - const x = Shape.init(.{ 16, 8 }, .f32); - - // By default tensors are replicated. - { - var fbs = std.io.fixedBufferStream(&rule); - try writeShardingRepresentation(x, 4, fbs.writer()); - try std.testing.expectEqualStrings("{replicated}", fbs.getWritten()); - } - // Shard along first axis. - { - var fbs = std.io.fixedBufferStream(&rule); - try writeShardingRepresentation(x.withSharding(.{0}), 4, fbs.writer()); - try std.testing.expectEqualStrings("{devices=[4,1]<=[4]}", fbs.getWritten()); - } - // Also shard along second axis. - { - var fbs = std.io.fixedBufferStream(&rule); - try writeShardingRepresentation(x.withSharding(.{ 0, 1 }), 2, fbs.writer()); - try std.testing.expectEqualStrings("{devices=[2,2]<=[2]}", fbs.getWritten()); - } - } - - fn finalizeAttributeList(allocator: std.mem.Allocator, mlir_ctx: mlir.Context, attributes: []AttributeList) ![]mlir.Attribute { - const res = try allocator.alloc(mlir.Attribute, attributes.len); - for (res, attributes) |*r, attr| { - r.* = mlir.DictionaryAttribute.init(mlir_ctx, attr.constSlice()).asAttr(); - } - return res; - } - - /// Generates an MLIR `func.call` of the given function. - /// If the function has not been seen yet, we generate MLIR for it, - /// in a independent function. - /// The main benefit of this is to generate MLIR that maps more closely - /// to the Zig code, but compilation speed stays similar. - pub fn callFunc( - self: *CompilationContext, - func_name: [:0]const u8, - comptime func: anytype, - model: *const ModuleSignature(func).ModelT, - args: *ModuleSignature(func).ArgsT, - ) ModuleSignature(func).ReturnT { - var arena_state = std.heap.ArenaAllocator.init(self._allocator); - defer arena_state.deinit(); - const arena = arena_state.allocator(); - - // first, do the "compile" and check the bytecode - // the result of this will also have the correct tags of the result shapes - const dummy_result = self.generateMlirBytecodeForFunction( - arena, - func_name, - func, - model, - args, - ) catch unreachable; // TODO: do we like unreachable? - const bytecode_hash = hashArgs(dummy_result.bytecode_tmp); - - const key: FnCache.Key = .{ .fn_ptr = &func, .input_hash = bytecode_hash }; - const function = self._fn_cache.getEntry(key) orelse b: { - const full_name: [:0]const u8 = if (std.mem.eql(u8, "main", func_name)) - arena.dupeZ(u8, func_name) catch unreachable - else - std.fmt.allocPrintZ(arena, "{s}.{s}_{x}", .{ @typeName(ModuleSignature(func).ModelT), func_name, key.input_hash }) catch unreachable; - - log.info("addFuncToModule {any} {s}", .{ key, full_name }); - - const value = self.addFuncToModule( - arena, - full_name, - func, - model, - args, - ) catch unreachable; - - break :b self._fn_cache.addEntry(key, value) catch unreachable; - }; - - // Note: we won't increase the size of the cache until next `call` so - // we can use the memory there without worrying about fragmentation. - - const loc = self.mlirCtx().location(@src()); - - const values = arena.alloc(mlir.Value, function.n_model + function.n_args) catch unreachable; - extractValues(model, values[0..function.n_model]); - extractValues(args, values[function.n_model..]); - - const op = dialect.func.call(self.mlirCtx(), function.name, values, function.res_types, loc); - var res: meta.FnResult(func) = undefined; - assignResults(&res, function.res_shapes, op); - return res; - } - - /// Visit the given struct and recursively associate the `block` arguments with the `value` field of each encountered Tensor. - /// - /// This is done so that we have a mapping between the arguments of the kernel associated with a module and the actual Tensors - /// stored in the Module. - /// Caller need to allocate required memory in self._buffer_to_arg. - pub fn mapBlockArguments(self: *CompilationContext, v: anytype, block: mlir.Block, start: usize) usize { - const LocalContext = struct { - index: usize, - block: mlir.Block, - self: *CompilationContext, - }; - var context = LocalContext{ .self = self, .block = block, .index = start }; - meta.visit((struct { - fn cb(ctx: *LocalContext, tensor: *const Tensor) void { - const arg_value = ctx.block.argument(ctx.index); - // log.debug("mapping {} to arg {}", .{ tensor._id, ctx.index }); - - const res = ctx.self._buffer_to_arg.getOrPutAssumeCapacity(tensor._id); - if (res.found_existing) { - std.debug.panic("Failed compilation because received two tensors arguments with the same ID: {} and {}({}).", .{ res.key_ptr.*, tensor, tensor._id }); - } else { - res.value_ptr.* = .{ arg_value, .{ .arg = @intCast(ctx.index) } }; - } - ctx.index += 1; - } - }).cb, &context, v); - return context.index; - } - - /// Create tensor from the given shapes. - /// Each created tensor will receive a unique id, local to this CompilationContext. - pub fn tensorFromShapes(self: *CompilationContext, ArgsT: type, allocator: std.mem.Allocator, args_shapes: anytype) ArgsT { - const Local = struct { - fn tensorFromShape(arg_id: *u64, shape: Shape) Tensor { - defer arg_id.* += 1; - return Tensor{ - ._shape = shape, - ._id = .{ .arg_id = arg_id.* }, - ._donation = .input_buffer, - }; - } - }; - var tensor_args: ArgsT = undefined; - try meta.mapAlloc(Local.tensorFromShape, allocator, &self._unique_id, args_shapes, &tensor_args); - return tensor_args; - } - - /// Visit the given struct and extract the mlir.Value and mlir.Type associated with each tensor found. - pub fn extractValuesAndTypes(self: *const CompilationContext, v: anytype, values: []mlir.Value, types: []mlir.Type, shapes: []Shape, donations: []Tensor._Donation) void { - assert(values.len == types.len); - const LocalContext = struct { - self: *const CompilationContext, - index: usize = 0, - values: []mlir.Value, - types: []mlir.Type, - shapes: []Shape, - donations: []Tensor._Donation, - }; - var context = LocalContext{ .self = self, .values = values, .types = types, .shapes = shapes, .donations = donations }; - meta.visit((struct { - fn cb(ctx: *LocalContext, tensor: *const Tensor) void { - const value, const donation = ctx.self.getValueAndDonation(tensor.*); - ctx.values[ctx.index] = value; - ctx.types[ctx.index] = value.getType(); - ctx.shapes[ctx.index] = tensor._shape; - ctx.donations[ctx.index] = donation; - ctx.index += 1; - } - }).cb, &context, v); - assert(context.index == values.len); - } - - pub fn getValueAndDonation(self: *const CompilationContext, tensor: Tensor) struct { mlir.Value, Tensor._Donation } { - return switch (tensor._id) { - .buffer_id, .arg_id => if (self._buffer_to_arg.get(tensor._id)) |res| - .{ res[0], res[1] } - else { - log.err("Found unknown tensor id {}({})", .{ tensor, tensor._id }); - @panic("Found unknown tensor id"); - }, - .mlir => |v| .{ v, tensor._donation }, - }; - } - - /// Visit the given struct and copies the mlir.Value associated with each tensor found. - pub fn extractValues(self: *const CompilationContext, v: anytype, values: []mlir.Value) void { - const LocalContext = struct { - self: *const CompilationContext, - index: usize = 0, - values: []mlir.Value, - }; - var context = LocalContext{ .self = self, .values = values }; - meta.visit((struct { - fn cb(ctx: *LocalContext, tensor: *const Tensor) void { - const value, const donation = ctx.self.getValueAndDonation(tensor.*); - _ = donation; - - ctx.values[ctx.index] = value; - ctx.index += 1; - } - }).cb, &context, v); - assert(context.index == values.len); - } -}; - -/// Visit the given struct and recursively counts the number of tensors found. -pub fn countTensors(v: anytype) usize { - const LocalContext = struct { - count: usize = 0, - }; - var context = LocalContext{}; - meta.visit((struct { - fn cb(inner_context: *LocalContext, _: *const Tensor) void { - inner_context.count += 1; - } - }).cb, &context, v); - return context.count; -} - -/// Visit the given struct and recursively fill the `types` slice with the mlir.Type associated with encountered Tensor. -pub fn fillMlirTypes(v: anytype, mlir_ctx: mlir.Context, types: []mlir.Type) void { - const LocalContext = struct { - index: usize = 0, - mlir_ctx: mlir.Context, - types: []mlir.Type, - }; - var context = LocalContext{ .mlir_ctx = mlir_ctx, .types = types }; - meta.visit((struct { - fn cb(inner_context: *LocalContext, tensor: *const Tensor) void { - inner_context.types[inner_context.index] = mlir.ext.mlirType(inner_context.mlir_ctx, tensor.shape()); - inner_context.index += 1; - } - }).cb, &context, v); - assert(context.index == types.len); -} - -/// Visit the given struct and recursively associate the `block` arguments with the `value` field of each encountered Tensor. -/// -/// This is done so that we have a mapping between the arguments of the kernel associated with a module and the actual Tensors -/// stored in the Module. -fn assignBlockArguments(v: anytype, block: mlir.Block, start: usize) usize { - const LocalContext = struct { index: usize, block: mlir.Block }; - var context = LocalContext{ .block = block, .index = start }; - meta.visit((struct { - fn cb(ctx: *LocalContext, tensor: *Tensor) void { - tensor._id = .{ .mlir = ctx.block.argument(ctx.index) }; - tensor._donation = .{ .arg = @intCast(ctx.index) }; - ctx.index += 1; - } - }).cb, &context, v); - return context.index; -} - -/// Visit the given struct and fill the `buffers` slice with the buffer associated with encountered Tensor. -fn fillBuffers(v: anytype, buffers: []const [*]*pjrt.Buffer, start: u32, len: u32) void { - const LocalContext = struct { - index: u32, - buffers: []const [*]*pjrt.Buffer, - }; - var context: LocalContext = .{ - .index = start, - .buffers = buffers, - }; - meta.visit((struct { - fn cb(ctx: *LocalContext, buffer: *const Buffer) void { - // meta.assert(!buffer._data.isDeleted(), "Can't use {} (argument buffer {}) because its pjrt buffer has been donated", .{ buffer, ctx.index }); - const model_sharding = ctx.buffers.len; - meta.assert(buffer._shards.len == model_sharding, "Can't feed a {}-sharded tensor into a {}-sharded model", .{ buffer._shards.len, ctx.buffers.len }); - for (buffer._shards.constSlice(), 0..) |shard, d| { - ctx.buffers[d][ctx.index] = shard; - } - ctx.index += 1; - } - }).cb, &context, v); - assert(context.index == start + len); -} - -/// Visit the given struct and override tensors by creating a new one using the provided PJRT buffers. -pub fn assignRawBuffers(v: anytype, platform: Platform, buffers: []const [*]*pjrt.Buffer, buffer_shapes: []Shape, expected_count: u32) void { - const LocalContext = struct { - index: u32, - platform: Platform, - buffers: []const [*]*pjrt.Buffer, - expected_count: u32, - buffer_shapes: []Shape, - }; - var local_ctx: LocalContext = .{ - .index = 0, - .platform = platform, - .buffers = buffers, - .expected_count = expected_count, - .buffer_shapes = buffer_shapes, - }; - meta.visit((struct { - fn cb(ctx: *LocalContext, buffer: *Buffer) void { - const i = ctx.index; - ctx.index += 1; - if (i >= ctx.expected_count) return; - - var shards: Buffer.Shards = .{}; - for (ctx.buffers) |buff| { - shards.appendAssumeCapacity(buff[i]); - } - buffer.* = Buffer.fromPjrtBuffers(ctx.platform, ctx.buffer_shapes[i], shards.constSlice()); - } - }).cb, &local_ctx, v); - meta.internalAssert(local_ctx.index == expected_count, "Pjrt call returned {} tensors, but the return type {s}, contains {} Buffers. Note that modules need to have a comptime know number of returned tensors.", .{ buffers.len, @typeName(@TypeOf(v)), local_ctx.index }); -} - -/// Visit the given struct and assign op results to each tensor found. -pub fn assignResults(v: anytype, shapes: ?[]Shape, op: mlir.Operation) void { - const LocalContext = struct { - index: usize, - op: mlir.Operation, - shapes: ?[]Shape, - }; - var context = LocalContext{ .index = 0, .op = op, .shapes = shapes }; - meta.visit((struct { - fn cb(inner_ctx: *LocalContext, tensor: *Tensor) void { - tensor.* = Tensor.fromMlirValue(inner_ctx.op.result(inner_ctx.index)); - if (inner_ctx.shapes) |sh| { - tensor._shape = sh[inner_ctx.index]; - } - inner_ctx.index += 1; - } - }).cb, &context, v); - assert(context.index == op.numResults()); -} - -/// Represents an MLIR module compiled into a PJRT executable. -/// The BaseExe is a plain old struct and doesn't have information -/// about Zig types. -const BaseExe = struct { - /// The platform for which this module was compiled. - platform: Platform, - /// The PJRT executable representing the compiled module. - exe: *pjrt.LoadedExecutable, - /// Number of buffers in the model. - model_buffer_count: u32, - /// Number of buffers in the arguments. - args_buffer_count: u32, - /// Number of buffers in result. - result_buffer_count: u32, - /// Shapes of buffers in result. - result_buffer_shapes: []Shape, - /// Num devices used (>1 for sharded executable) - num_devices: u8, - /// Allocator backing result_buffer_shapes and deinit by ExeWithWeights - _allocator: std.heap.ArenaAllocator, -}; - -/// Represents a ZML model, compiled into a PJRT executable. -/// -/// It's not directly callable, as it doesn't have associated model weights. -/// use `prepare` to assign weights and pre allocate memory needed to call. -pub fn Exe(comptime func: anytype) type { - const Signature = ModuleSignature(func); - return struct { - const Self = @This(); - - /// The raw untyped compiled module. - inner: BaseExe, - - /// Packages the given model weight with an `Exe` to produce an `ExeWithWeights` that can be called. - pub fn prepare(self: Self, allocator: std.mem.Allocator, model: Bufferized(Signature.ModelT)) !ExeWithWeights(func) { - return ExeWithWeights(func).initFromModel(allocator, self.inner, model); - } - }; -} - -/// Represents a ZML model, compiled into a PJRT executable, and ready to call. -/// The buffers for the model weights are saved inside the struct and will be used in `call`. -/// You only need to pass the remaining arguments. -pub fn ExeWithWeights(comptime func: anytype) type { - const Signature = ModuleSignature(func); - return struct { - const Self = @This(); - - /// The raw untyped compiled module. - inner: BaseExe, - - /// Pre-allocated slice of buffers to use as inputs when the module is called. - input_per_device: []const [*]*pjrt.Buffer, - - /// Pre-allocated slice of buffers to use as outputs when the module is called. - output_per_device: []const [*]*pjrt.Buffer, - - /// Internal memory slice used. - _all_buffers: []*pjrt.Buffer, - _all_per_device: [][*]*pjrt.Buffer, - - /// And the allocator backing _data_buffer. - _allocator: std.mem.Allocator, - - pub fn initFromModel(allocator: std.mem.Allocator, inner: BaseExe, model: Bufferized(Signature.ModelT)) !Self { - const n_input_buffers = inner.model_buffer_count + inner.args_buffer_count; - const n_output_buffers = inner.result_buffer_count; - const n_devices = inner.num_devices; - - // Allocate once for all the *pjrt.Buffer we need to store ... - const all_buffers = try allocator.alloc(*pjrt.Buffer, (n_input_buffers + n_output_buffers) * n_devices); - errdefer allocator.free(all_buffers); - const all_input_buffers, const all_output_buffers = splitBuffer(*pjrt.Buffer, all_buffers, .{ n_input_buffers * n_devices, n_output_buffers * n_devices }); - - // ... and once for all the [*]*pjrt.Buffer. - const all_per_device = try allocator.alloc([*]*pjrt.Buffer, 2 * n_devices); - errdefer allocator.free(all_per_device); - const input_per_device, const output_per_device = splitBuffer([*]*pjrt.Buffer, all_per_device, .{ n_devices, n_devices }); - - for (0..n_devices) |i| { - input_per_device[i] = all_input_buffers[i * n_input_buffers ..].ptr; - output_per_device[i] = all_output_buffers[i * n_output_buffers ..].ptr; - } - - fillBuffers(&model, input_per_device, 0, inner.model_buffer_count); - // Note: all_output_buffers is left undefined, it will be written to in `call`. - - return .{ - .inner = inner, - .input_per_device = input_per_device, - .output_per_device = output_per_device, - ._all_buffers = all_buffers, - ._all_per_device = all_per_device, - ._allocator = allocator, - }; - } - - pub fn deinit(self: Self) void { - // Free in reverse order of allocation. - self._allocator.free(self._all_per_device); - self._allocator.free(self._all_buffers); - self.inner._allocator.deinit(); - } - - pub fn platform(self: Self) Platform { - return self.inner.platform; - } - - pub fn call(self: Self, args: Bufferized(Signature.ArgsT)) Bufferized(Signature.ReturnT) { - fillBuffers(&args, self.input_per_device, self.inner.model_buffer_count, self.inner.args_buffer_count); - var events: [Platform.MAX_NUM_DEVICES]*pjrt.Event = undefined; - const sharding = self.platform().sharding(); - - self.inner.exe.execute(self.inner.platform.pjrt_api, .{ - .arguments = self.input_per_device, - .num_args = self.inner.args_buffer_count + self.inner.model_buffer_count, - .results = self.output_per_device, - .events = events[0..sharding.num_partitions], - // TODO: this allows to tell a specific buffer shouldn't be donated, - // even if it has been marked as "can be donated" during compilation. - .non_donatable_input_indices = &.{}, - }) catch unreachable; - - for (events[0..sharding.num_partitions]) |e| { - e.await_(self.inner.platform.pjrt_api) catch unreachable; - } - - var result: Bufferized(Signature.ReturnT) = undefined; - assignRawBuffers(&result, self.inner.platform, self.output_per_device, self.inner.result_buffer_shapes, self.inner.result_buffer_count); - return result; - } - }; -} - -/// Compiles the given module with the given arguments. -/// The `model` (first fn argument), is treated differently from the other args. -/// This helps to have two separate lifetimes for the model buffers, -/// and for the arguments buffer. -fn compileInternal( - allocator: std.mem.Allocator, - context: *CompilationContext, - comptime func: anytype, - model: ModuleSignature(func).ModelT, - args: ShapeOf(ModuleSignature(func).ArgsT), -) !BaseExe { - var arena_state = std.heap.ArenaAllocator.init(allocator); - defer arena_state.deinit(); - const arena = arena_state.allocator(); - - var timer = std.time.Timer.start() catch null; - const tensor_args = context.tensorFromShapes(ModuleSignature(func).ArgsT, arena, args); - // Run in a dedicated thread because compilation relies on `threadlocal`. - const f = try asynk.callGeneric(CompilationContext.generateBytecode, .{ context, arena, "main", func, &model, &tensor_args }); - context._module.getBody().appendOperation(f.mlir_fn); - - const sharding = context._platform.sharding(); - const mlir_ctx = context._mlir_ctx; - context._module.op().setAttributeByName("mhlo.num_replicas", mlir.IntegerAttribute(.i32).init(mlir_ctx, sharding.num_replicas).asAttr()); - context._module.op().setAttributeByName("mhlo.num_partitions", mlir.IntegerAttribute(.i32).init(mlir_ctx, sharding.num_partitions).asAttr()); - - const loaded_executable = loadOrCompilePjrtExecutable(arena, context._platform, context._module) catch |err| { - log.err( - "pjrt-{s} failed to compile following valid MLIR:\n{}\n{}", - .{ @tagName(context._platform.target), context._module.op().mlirFormatter(.{}), err }, - ); - return err; - }; - - log.debug("******** ZML generated MLIR ********", .{}); - log.debug("{}", .{context._module.op().mlirFormatter(.{})}); - - if (timer) |*t| { - const time_ms = @divFloor(t.lap(), std.time.ns_per_ms); - if (time_ms > 1000) log.info("Compilation took {d:.3}s", .{meta.divFloat(f32, time_ms, 1000)}); - } - - var arena_state_exe = std.heap.ArenaAllocator.init(allocator); - const arena_exe = arena_state_exe.allocator(); - - return .{ - .platform = context._platform, - .exe = loaded_executable, - .model_buffer_count = f.n_model, - .args_buffer_count = f.n_args, - .result_buffer_count = @intCast(f.res_types.len), - .result_buffer_shapes = arena_exe.dupe(Shape, f.res_shapes) catch unreachable, - .num_devices = sharding.num_replicas * sharding.num_partitions, - ._allocator = arena_state_exe, - }; -} - -/// Compiles a Model struct with the given configuration and shapes, for the given platform. -/// The steps are: -/// * lookup at tensors available in the store and create a `model: Model` struct with them -/// * call `model.init(init_args)` to fields of the model that aren't Tensor, ie hyperparemeters/config -/// * generate MLIR by calling `model.forward` with tensor of the given shapes and other arguments -pub fn compile( - allocator: std.mem.Allocator, - comptime Model: type, - init_args: anytype, - comptime func: @TypeOf(.literal), - args_shapes: ShapeOf(ModuleSignature(@field(Model, @tagName(func))).ArgsT), - buffer_store: aio.BufferStore, - platform: Platform, -) !Exe(@field(Model, @tagName(func))) { - var arena_state = std.heap.ArenaAllocator.init(allocator); - defer arena_state.deinit(); - const arena = arena_state.allocator(); - var model = try aio.populateModel(Model, arena, buffer_store); - - // If the Model has a "init" function, call it with the given parameters. - if (@hasDecl(Model, "init")) { - // TODO(Corentin,@Improvement): Add a warning/error if there is no init function but init_args is non-void. - @call(.auto, Model.init, .{@as(*Model, &model)} ++ init_args); - } - - return compileModel(allocator, model, func, args_shapes, platform); -} - -/// Compiles a Model struct with the given configuration and shapes, for the given platform. -/// Generate MLIR by calling `model.forward` with tensor of the given shapes and other arguments -pub fn compileModel( - allocator: std.mem.Allocator, - model: anytype, - comptime func: @TypeOf(.literal), - args_shapes: ShapeOf(ModuleSignature(@field(@TypeOf(model), @tagName(func))).ArgsT), - platform: Platform, -) !Exe(@field(@TypeOf(model), @tagName(func))) { - const Model = @TypeOf(model); - const name = @typeName(Model) ++ "." ++ @tagName(func); - log.info("Compiling {s} with {}", .{ name, args_shapes }); - - var context = try CompilationContext.init(allocator, name, platform); - defer context.deinit(); - - const raw_module = try compileInternal(allocator, &context, @field(Model, @tagName(func)), model, args_shapes); - - return Exe(@field(Model, @tagName(func))){ .inner = raw_module }; -} - -/// Compiles a function with the given configuration and shapes, for the given platform. -/// Generate MLIR by calling the given function with tensor of the given shapes. -pub fn compileFn( - allocator: std.mem.Allocator, - func: anytype, - args: ShapeOf(meta.FnParams(func)), - platform: Platform, -) !ExeWithWeights(FnWithVoidArg(func)) { - const name = @typeName(@TypeOf(func)); - var context = try CompilationContext.init(allocator, name, platform); - defer context.deinit(); - - const Local = struct { - // This is the function we will actually compile. - pub fn forward(_: void, inner_args: meta.FnParams(func)) meta.FnResult(func) { - return @call(.auto, func, inner_args); - } - }; - - const void_model: void = {}; - const raw_module = try compileInternal(allocator, &context, Local.forward, void_model, .{args}); - // But we set the signature so that you can call the module as you would call the function. - return try ExeWithWeights(FnWithVoidArg(func)).initFromModel(allocator, raw_module, void_model); -} - -fn FnWithVoidArg(func: anytype) type { - const fn_info = @typeInfo(@TypeOf(func)).Fn; - const void_param = std.builtin.Type.Fn.Param{ .is_generic = false, .is_noalias = false, .type = void }; - meta.assertComptime(!fn_info.is_generic, "Can't do reflection on generic function: {}", .{@TypeOf(func)}); - return @Type(.{ .Fn = .{ - .calling_convention = fn_info.calling_convention, - .is_generic = false, - .is_var_args = fn_info.is_var_args, - .return_type = fn_info.return_type, - .params = [1]std.builtin.Type.Fn.Param{void_param} ++ fn_info.params, - } }); -} - -fn computeModuleHash(platform: Platform, module: mlir.Module) u64 { - var hasher = std.hash.XxHash64.init(0); - var hasher_writer = xxHash64Writer(&hasher); - const writer = hasher_writer.writer(); - - // Hash the canonicalized IR, without debug information that can change across builds. - module.op().writeBytecode(writer); - //module.op().print(writer, .{ .debug_info = false }); - // Writes can't fail because we are writing to a hasher. - writer.writeAll(platform.pjrt_client.getPlatformName(platform.pjrt_api)) catch unreachable; - const api_version = platform.pjrt_api.version(); - writer.writeInt(i64, api_version.major, .little) catch unreachable; - writer.writeInt(i64, api_version.minor, .little) catch unreachable; - - return hasher.final(); -} - -const max_pjrt_executable_size = 20 * 1024 * 1024; - -fn loadPjrtExecutable(arena: std.mem.Allocator, platform: Platform, module_hash: u64, compilation_cache_location: []const u8) !*pjrt.LoadedExecutable { - const resolved_path = try std.fs.cwd().realpathAlloc(arena, compilation_cache_location); - const compilation_cache_dir = try std.fs.openDirAbsolute(resolved_path, .{}); - var buf: [16]u8 = undefined; - const filename = try std.fmt.bufPrint(&buf, "{x}", .{module_hash}); - const loaded_executable_file = try compilation_cache_dir.openFile(filename, .{}); - defer loaded_executable_file.close(); - - const bytes = try loaded_executable_file.readToEndAlloc(arena, max_pjrt_executable_size); - - return platform.pjrt_client.deserializeAndLoad(platform.pjrt_api, bytes); -} - -fn storePjrtExecutable(arena: std.mem.Allocator, platform: Platform, loaded_executable: *pjrt.LoadedExecutable, module_hash: u64, compilation_cache_location: []const u8) !void { - const resolved_path = try std.fs.cwd().realpathAlloc(arena, compilation_cache_location); - const compilation_cache_dir = try std.fs.openDirAbsolute(resolved_path, .{}); - - const loaded_executable_file = try compilation_cache_dir.createFile(try std.fmt.allocPrint(arena, "{x}", .{module_hash}), .{}); - defer loaded_executable_file.close(); - - var executable = try loaded_executable.getExecutable(platform.pjrt_api); - defer executable.deinit(platform.pjrt_api); - - var serialize_result = try executable.serialize(platform.pjrt_api); - defer serialize_result.deinit(); - - try loaded_executable_file.writeAll(serialize_result.bytes); -} - -fn loadOrCompilePjrtExecutable( - arena: std.mem.Allocator, - platform: Platform, - module: mlir.Module, -) !*pjrt.LoadedExecutable { - const tracer = Tracer.init("ai.zml.compilation"); - const compile_frame = tracer.frameStart("pjrt cached compilation"); - defer tracer.frameEnd(compile_frame, "pjrt cached compilation"); - const module_hash = computeModuleHash(platform, module); - - if (platform.compilation_options.cache_location) |compilation_cache_location| { - log.debug("Loading module from {s}", .{compilation_cache_location}); - return loadPjrtExecutable(arena, platform, module_hash, compilation_cache_location) catch |err| { - log.debug("Failed to load module: {}", .{err}); - return compileModuleToPjrtExecutable(arena, platform, module, module_hash); - }; - } else { - return compileModuleToPjrtExecutable(arena, platform, module, module_hash); - } -} - -fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, platform: Platform, module: mlir.Module, module_hash: u64) !*pjrt.LoadedExecutable { - const sharding = platform.sharding(); - var options: xla_pb.CompileOptionsProto = .{ - .executable_build_options = .{ - .num_replicas = sharding.num_replicas, - .num_partitions = sharding.num_partitions, - .use_spmd_partitioning = sharding.num_partitions > 1 or sharding.num_replicas > 1, - }, - }; - // Let the arena deinit, zig-protobuf deinit is very slow. - if (platform.compilation_options.xla_dump_to) |xla_dump_to| { - try options.env_option_overrides.append(arena, .{ - .key = .{ .Const = "xla_dump_to" }, - .value = .{ .value = .{ .string_field = .{ .Const = xla_dump_to } } }, - }); - if (platform.compilation_options.xla_dump_fusion_visualization) { - try options.env_option_overrides.append(arena, .{ - .key = .{ .Const = "xla_dump_hlo_as_html" }, - .value = .{ .value = .{ .bool_field = true } }, - }); - try options.env_option_overrides.append(arena, .{ - .key = .{ .Const = "xla_dump_hlo_as_dot" }, - .value = .{ .value = .{ .bool_field = true } }, - }); - try options.env_option_overrides.append(arena, .{ - .key = .{ .Const = "xla_dump_fusion_visualization" }, - .value = .{ .value = .{ .bool_field = true } }, - }); - } - } - switch (platform.target) { - .cuda => cuda_dir: { - // NVIDIA recommends to disable Triton GEMM on JAX: - // https://github.com/NVIDIA/JAX-Toolbox?tab=readme-ov-file#environment-variables - try options.env_option_overrides.append(arena, .{ - .key = .{ .Const = "xla_gpu_enable_triton_gemm" }, - .value = .{ .value = .{ .bool_field = false } }, - }); - var r_ = try runfiles.Runfiles.create(.{ .allocator = arena }) orelse { - log.warn("Bazel runfile not found !", .{}); - break :cuda_dir; - }; - defer r_.deinit(arena); - const source_repo = @import("bazel_builtin").current_repository; - const r = r_.withSourceRepo(source_repo); - const cuda_data_dir = (try r.rlocationAlloc(arena, "libpjrt_cuda/sandbox")).?; - log.info("xla_gpu_cuda_data_dir: {s}", .{cuda_data_dir}); - try options.env_option_overrides.append(arena, .{ - .key = .{ .Const = "xla_gpu_cuda_data_dir" }, - .value = .{ - .value = .{ - .string_field = .{ .Const = cuda_data_dir }, - }, - }, - }); - }, - .rocm => { - // Disable Triton GEMM on ROCM. For some reason it's much, much slower when - // enabled on CDNA and it's used on RDNA. Disable it altogether. - try options.env_option_overrides.append(arena, .{ - .key = .{ .Const = "xla_gpu_enable_triton_gemm" }, - .value = .{ .value = .{ .bool_field = false } }, - }); - }, - else => {}, - } - - const options_bytes = try options.encode(arena); - - var mlir_bytecode = std.ArrayList(u8).init(arena); - defer mlir_bytecode.deinit(); - // Note: we may need to restore IR downgrade if we need to support old pjrt plugins. - module.op().writeBytecode(mlir_bytecode.writer()); - - const loaded_executable = try asynk.call(pjrt.Client.compile, .{ - platform.pjrt_client, platform.pjrt_api, .{ - .bytecode = mlir_bytecode.items, - .bytecode_format = .mlir, - .compile_options_pb = options_bytes, - }, - }); - - errdefer loaded_executable.deinit(); - - if (platform.compilation_options.cache_location) |compilation_cache_location| { - log.debug("Storing module to {s}", .{compilation_cache_location}); - storePjrtExecutable(arena, platform, loaded_executable, module_hash, compilation_cache_location) catch |err| { - log.debug("Failed to store module: {}", .{err}); - }; - } - - return loaded_executable; -} - -pub const XxHash64Writer = struct { - hasher: *std.hash.XxHash64, - - pub const Error = error{}; - pub const Writer = std.io.Writer(*XxHash64Writer, Error, write); - - pub fn writer(self: *XxHash64Writer) Writer { - return .{ .context = self }; - } - - pub fn write(self: *XxHash64Writer, bytes: []const u8) Error!usize { - self.hasher.update(bytes); - return bytes.len; - } -}; - -pub fn xxHash64Writer(hasher: *std.hash.XxHash64) XxHash64Writer { - return .{ .hasher = hasher }; -} - -pub fn hasTensors(comptime T: type) bool { - if (T == Tensor) return true; - - return switch (@typeInfo(T)) { - inline .Array, .Pointer, .Optional => |info| hasTensors(info.child), - inline .Struct, .Union => |info| { - inline for (info.fields) |field| { - if (hasTensors(field.type)) return true; - } - return false; - }, - else => false, - }; -} - -test "hasTensors" { - comptime { - try std.testing.expect(hasTensors(?Tensor)); - try std.testing.expect(hasTensors(struct { u8, ?Tensor })); - try std.testing.expect(!hasTensors(struct { u8, usize })); - } -} - -pub fn hasConstTensors(comptime T: type, comptime self_const: bool) bool { - if (T == Tensor) return self_const; - - return switch (@typeInfo(T)) { - inline .Array, .Optional => |info| hasTensors(info.child) and self_const, - .Pointer => |ptr_info| hasConstTensors(ptr_info.child, ptr_info.is_const), - inline .Struct, .Union => |info| { - inline for (info.fields) |field| { - if (hasConstTensors(field.type, self_const)) return true; - } - return false; - }, - else => false, - }; -} - -test "hasConstTensors" { - try std.testing.expect(!hasConstTensors(?Tensor, false)); - try std.testing.expect(hasConstTensors(struct { u8, ?Tensor }, true)); - try std.testing.expect(!hasConstTensors(struct { u8, *Tensor }, true)); - try std.testing.expect(hasConstTensors(struct { u8, *const Tensor }, false)); - try std.testing.expect(!hasConstTensors(struct { *Tensor }, false)); - try std.testing.expect(!hasConstTensors(std.meta.Tuple(&[_]type{*Tensor}), false)); - try std.testing.expect(!hasConstTensors(struct { u8, usize }, false)); - try std.testing.expect(hasConstTensors(struct { [5]Tensor, usize }, true)); - try std.testing.expect(!hasConstTensors(struct { [5]Tensor, usize }, false)); -} - -// making this a struct force all fields to be evaluted on creation, -// which gives a better error stacktrace -// than delaying the error to when the object fields are read. -const Sign = struct { - FuncT: type, - ModelT: type, - ArgsT: type, - ReturnT: type, -}; - -pub fn ModuleSignature(comptime func: anytype) Sign { - const FuncT = if (@TypeOf(func) == type) func else @TypeOf(func); - return .{ - .FuncT = FuncT, - .ModelT = @typeInfo(FuncT).Fn.params[0].type orelse @compileError("cannot create,ModuleSignature for function with an 'anytype' parameter"), - .ArgsT = blk: { - const function_info = @typeInfo(FuncT); - if (function_info.Fn.params[1..].len == 0) { - break :blk @TypeOf(.{}); - } - - var argument_field_list: [function_info.Fn.params.len - 1]type = undefined; - for (function_info.Fn.params[1..], 0..) |arg, i| { - const T = arg.type orelse @compileError("cannot create ModuleSignature for function with an 'anytype' parameter"); - argument_field_list[i] = T; - } - - break :blk std.meta.Tuple(&argument_field_list); - }, - .ReturnT = @typeInfo(FuncT).Fn.return_type.?, - }; -} - -pub const MlirFn = struct { - name: [:0]const u8, - n_model: u32, - n_args: u32, - res_types: []mlir.Type, - res_shapes: []Shape, - res_donations: []Tensor._Donation, - mlir_fn: mlir.Operation, -}; - -// TODO(Corentin): Remove that -pub const FnCache = struct { - pub const Key = struct { fn_ptr: *const anyopaque, input_hash: u64 }; - - // TODO: merge arenas - cache: std.AutoHashMapUnmanaged(Key, MlirFn), - // Arena for the cache entries - cache_arena: std.heap.ArenaAllocator, - // Arena for the cache data (name, res_type) - cache_data_arena: std.heap.ArenaAllocator, - - pub fn init(allocator: std.mem.Allocator) FnCache { - return .{ - .cache = .{}, - .cache_arena = std.heap.ArenaAllocator.init(allocator), - .cache_data_arena = std.heap.ArenaAllocator.init(allocator), - }; - } - - pub fn deinit(self: FnCache) void { - self.cache_arena.deinit(); - self.cache_data_arena.deinit(); - } - - pub fn getEntry(self: *const FnCache, key: Key) ?MlirFn { - return self.cache.get(key); - } - - pub fn addEntry(self: *FnCache, key: Key, value: MlirFn) !MlirFn { - var cache_data_allocator = self.cache_data_arena.allocator(); - - const res_types_copy = try cache_data_allocator.dupe(mlir.Type, value.res_types); - errdefer cache_data_allocator.free(res_types_copy); - - const res_shapes_copy = try cache_data_allocator.dupe(Shape, value.res_shapes); - errdefer cache_data_allocator.free(res_shapes_copy); - - const res_donations_copy = try cache_data_allocator.dupe(Tensor._Donation, value.res_donations); - errdefer cache_data_allocator.free(res_donations_copy); - - const name_copy = try cache_data_allocator.dupeZ(u8, value.name); - errdefer cache_data_allocator.free(name_copy); - - const owned_value: MlirFn = .{ - .name = name_copy, - .mlir_fn = value.mlir_fn, - .n_model = value.n_model, - .n_args = value.n_args, - .res_types = res_types_copy, - .res_shapes = res_shapes_copy, - .res_donations = res_donations_copy, - }; - - try self.cache.putNoClobber(self.cache_arena.allocator(), key, owned_value); - return owned_value; - } -}; - -test FnCache { - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - - const NN = struct { - const NN_ = @This(); - layer_weights: [3]Tensor, - layer_biases: [3]Tensor, - - pub fn forward(self: NN_, x0: Tensor) Tensor { - var x = x0; - for (self.layer_weights, self.layer_biases) |w, b| { - // TODO use the `call` magic helper - // x = ops.callFunc(ctx, NN_, "reluLayer", .{ w, b, x }); - x = NN_.reluLayer(w, b, x); - } - return x; - } - - pub fn forwardRefImpl(self: NN_, x0: Tensor) Tensor { - var x = x0; - for (self.layer_weights, self.layer_biases) |w, b| { - x = NN_.reluLayer(w, b, x); - } - return x; - } - - pub fn reluLayer(w: Tensor, b: Tensor, x: Tensor) Tensor { - const wx = w.dotGeneral(x, &.{.{ -1, 0 }}, &.{}); - return wx.add(b.broadcastLeft(wx.shape())).relu(); - } - }; - - const x = try zml.Buffer.fromSlice(platform, .{2}, &[_]f16{ -1, 1 }); - const nn: zml.Bufferized(NN) = .{ - .layer_weights = .{ - try zml.Buffer.fromSlice(platform, .{ 2, 2 }, &[_]f16{ 1, -1, 0, 1 }), - try zml.Buffer.fromSlice(platform, .{ 2, 2 }, &[_]f16{ 1, 2, 1, -1 }), - // third layer is different - try zml.Buffer.fromSlice(platform, .{ 3, 2 }, &[_]f16{ 1, 2, 0, 1, -1, 0 }), - }, - .layer_biases = .{ - try zml.Buffer.fromSlice(platform, .{2}, &[_]f16{ 0, 0 }), - try zml.Buffer.fromSlice(platform, .{2}, &[_]f16{ 10, 10 }), - try zml.Buffer.fromSlice(platform, .{3}, &[_]f16{ -10, -10, -10 }), - }, - }; - const res = try zml.testing.compileAndCall(platform, NN.forward, .{ nn, x }); - const expected = try zml.testing.compileAndCall(platform, NN.forwardRefImpl, .{ nn, x }); - try zml.testing.expectClose(expected, res, 1e-4); -} - -pub fn hashArgs(mod: anytype) u64 { - var hasher = std.hash.Wyhash.init(0); - hash(&hasher, mod, .DeepRecursive); - return hasher.final(); -} - -pub fn hashTensor(hasher: *std.hash.Wyhash, tensor: Tensor) void { - // Note: if we enforced 0-init dims then we could hash dims instead. - hashArray(hasher, tensor.dims(), .Shallow); - hash(hasher, tensor.dtype(), .Shallow); -} - -const HashStrategy = std.hash.Strategy; -const tensorAwareHash = hash; // alias for when "hash" is ambiguous - -/// Provides generic hashing for any eligible type. -/// Strategy is provided to determine if pointers should be followed or not. -pub fn hash(hasher: *std.hash.Wyhash, key: anytype, comptime strat: HashStrategy) void { - const Key = @TypeOf(key); - if (Key == Tensor) return hashTensor(hasher, key); - - if (strat == .Shallow and std.meta.hasUniqueRepresentation(Key)) { - hasher.update(std.mem.asBytes(&key)); - return; - } - - switch (@typeInfo(Key)) { - .NoReturn, .Opaque, .Undefined, .Null, .ComptimeFloat, .ComptimeInt, .Type, .EnumLiteral, .Frame, .Void => return, - - // Help the optimizer see that hashing an int is easy by inlining! - // TODO Check if the situation is better after #561 is resolved. - .Int => |int| switch (int.signedness) { - .signed => hash(hasher, @as(@Type(.{ .Int = .{ - .bits = int.bits, - .signedness = .unsigned, - } }), @bitCast(key)), strat), - .unsigned => { - if (std.meta.hasUniqueRepresentation(Key)) { - hasher.update(std.mem.asBytes(&key)); - } else { - // Take only the part containing the key value, the remaining - // bytes are undefined and must not be hashed! - const byte_size = comptime std.math.divCeil(comptime_int, @bitSizeOf(Key), 8) catch unreachable; - hasher.update(std.mem.asBytes(&key)[0..byte_size]); - } - }, - }, - // Note: contrary to Zig we accept hashing floats. - // Typically the float we are going to hash here are hyperparameters, - // and not the result of an operation, so bytes should be the same everytime. - .Float => hasher.update(std.mem.asBytes(&key)), - .Bool => hash(hasher, @intFromBool(key), strat), - .Enum => hash(hasher, @intFromEnum(key), strat), - .ErrorSet => hash(hasher, @intFromError(key), strat), - .AnyFrame, .Fn => hash(hasher, @intFromPtr(key), strat), - .Pointer => |info| switch (info.size) { - .One => switch (strat) { - .Shallow => hash(hasher, @intFromPtr(key), .Shallow), - .Deep => hash(hasher, key.*, .Shallow), - .DeepRecursive => switch (@typeInfo(info.child)) { - .Opaque, .Fn => hash(hasher, @intFromPtr(key), .Shallow), - else => hash(hasher, key.*, .DeepRecursive), - }, - }, - .Slice => { - switch (strat) { - .Shallow => hash(hasher, @intFromPtr(key.ptr), .Shallow), - .Deep => hashArray(hasher, key, .Shallow), - .DeepRecursive => hashArray(hasher, key, .DeepRecursive), - } - hash(hasher, key.len, .Shallow); - }, - .Many, - .C, - => switch (strat) { - .Shallow => hash(hasher, @intFromPtr(key), .Shallow), - else => @compileError( - \\ unknown-length pointers and C pointers cannot be hashed deeply. - \\ Consider providing your own hash function. - ), - }, - }, - .Optional => if (key) |k| hash(hasher, k, strat), - - .Array => hashArray(hasher, key, strat), - - .Vector => |info| { - if (std.meta.hasUniqueRepresentation(Key)) { - hasher.update(std.mem.asBytes(&key)); - } else { - comptime var i = 0; - inline while (i < info.len) : (i += 1) { - hash(hasher, key[i], strat); - } - } - }, - - .Struct => |info| { - inline for (info.fields) |field| { - // We reuse the hash of the previous field as the seed for the - // next one so that they're dependant. - hash(hasher, @field(key, field.name), strat); - } - }, - - .Union => |info| { - if (info.tag_type) |tag_type| { - const tag = std.meta.activeTag(key); - hash(hasher, tag, strat); - inline for (info.fields) |field| { - if (@field(tag_type, field.name) == tag) { - if (field.type != void) { - hash(hasher, @field(key, field.name), strat); - } - // TODO use a labelled break when it does not crash the compiler. cf #2908 - // break :blk; - return; - } - } - unreachable; - } else @compileError("cannot hash untagged union type: " ++ @typeName(Key) ++ ", provide your own hash function"); - }, - - .ErrorUnion => blk: { - const payload = key catch |err| { - hash(hasher, err, strat); - break :blk; - }; - hash(hasher, payload, strat); - }, - } -} - -fn hashArray(hasher: anytype, key: anytype, comptime strat: HashStrategy) void { - for (key) |element| { - hash(hasher, element, strat); - } -} - -fn splitBuffer(T: type, buffer: []T, lengths: anytype) [lengths.len][]T { - var res: [lengths.len][]T = undefined; - var i: usize = 0; - inline for (&res, lengths) |*r, len| { - r.* = buffer[i .. i + len]; - i += len; - } - std.debug.assert(i == buffer.len); - return res; -} diff --git a/zml/nn.zig b/zml/nn.zig index d609f3c1..eceb316b 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -1,49 +1,370 @@ //! Common layer definition and functions for Neural Networks (NN) const std = @import("std"); -const assert = std.debug.assert; -const testing = std.testing; -const zml = @import("zml.zig"); -const meta = @import("meta.zig"); -const helpers = @import("helpers.zig"); -const ops = @import("ops.zig"); +const mlir = @import("mlir"); +const stablehlo = @import("mlir/dialects/stablehlo"); +const stdx = @import("stdx"); +const constants = @import("constants.zig"); const DataType = @import("dtype.zig").DataType; +const meta = @import("meta.zig"); +const mlirx = @import("mlirx.zig"); +const ops = @import("ops.zig"); +const quantization = @import("quantization.zig"); +pub const Quantization = quantization.Quantization; const Shape = @import("shape.zig").Shape; +const Slice = @import("slice.zig").Slice; const Tensor = @import("tensor.zig").Tensor; +const zml = @import("zml.zig"); -const log = std.log.scoped(.zml_tensor); - -const cuda = @import("nn/cuda.zig"); - -test { - _ = cuda; - std.testing.refAllDecls(@This()); -} +const log = std.log.scoped(.@"zml/nn"); pub const Linear = struct { weight: Tensor, bias: ?Tensor = null, + tag: Shape.Tag, + quantization: ?Quantization = null, - pub fn forward(self: Linear, x: Tensor) Tensor { - var y = x.dotGeneral(self.weight.convert(x.dtype()), &.{.{ -1, -1 }}, &.{}); - // If self.weight doesn't have tags, preserve tags from x. - if (y.shape().tag(-1) == Shape.TagUnknown) { - y._shape._tags.set(y.rank() - 1, x.shape().tag(-1)); + pub fn init(weight: Tensor, bias: ?Tensor, tag: anytype) Linear { + return .{ + .weight = weight, + .bias = bias, + .tag = zml.Shape.toTag(tag), + }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(Linear)) void { + zml.Buffer.deinitAll(Linear, self); + } + + pub fn quantizationScheme(self: Linear) ?Quantization.Scheme { + return if (self.quantization) |q| q.scheme else null; + } + + pub fn quantizationScales(self: Linear) ?Tensor { + return if (self.quantization) |q| q.scales else null; + } + + pub fn forward(self: Linear, x: Tensor, out_dtype: DataType) Tensor { + if (self.quantization) |q| { + if (quantization.quantizeInput(q, x, self.tag, zml.Compiler.current().platform)) |input| { + return self.forwardQuantized(input, out_dtype); + } + return self.forwardScaled(x.convert(.bf16), null, null, out_dtype); } + const y = x.dot(self.weight.convert(x.dtype()), self.tag).convert(out_dtype); + return if (self.bias) |bias| y.add(bias.convert(y.dtype()).broad(y.shape())) else y; + } - // log.debug("Linear({*}): {d} -> {d} -> {d}", .{ self, x.dims(), y.dims(), if (self.bias) |bias| y.add(bias).dims() else y.dims() }); - return if (self.bias) |bias| y.add(bias.broadcastLeft(y.shape())) else y; + /// Apply this layer to reusable quantized activation values and scales. + /// Convert to output_dtype before adding bias. + pub fn forwardQuantized(self: Linear, input: quantization.QuantizedInput, output_dtype: DataType) Tensor { + stdx.debug.assert(self.quantization != null, "forwardQuantized requires quantized weights", .{}); + return self.forwardScaled(input.values, input.scales, input.global_scale, output_dtype); + } + + fn forwardScaled(self: Linear, lhs: Tensor, lhs_scale: ?Tensor, input_global_scale: ?Tensor, output_dtype: DataType) Tensor { + const q = self.quantization.?; + const weight_global_scale: ?Tensor = if (q.global_scale) |s| s.asMultiplier() else null; + const weight = if (isPackedFp4(q.scheme, self.weight.dtype())) unpackFp4(self.weight, self.tag, self.tag) else self.weight; + const scales = if (q.scheme.isMx() and q.scales.dtype() == .u8) q.scales.bitCast(.f8e8m0) else q.scales; + const acc = scaledDot(lhs, weight, lhs_scale, scales, output_dtype, self.tag, .{ .rhs_scale_swizzled = q.swizzled_scales }); + const y = applyGlobalScale(acc, input_global_scale, weight_global_scale).convert(output_dtype); + return if (self.bias) |bias| y.add(bias.convert(y.dtype()).broad(y.shape())) else y; } }; +pub fn isPackedFp4(scheme: ?Quantization.Scheme, weight_dtype: DataType) bool { + return (scheme == .nvfp4 or scheme == .mxfp4) and (weight_dtype == .u8 or weight_dtype == .i8); +} + +/// Unpacks two f4e2m1 values per byte along `packed_tag` and names the expanded +/// axis `k_tag`. Group-size agnostic: NVFP4 (16) and MXFP4 (32) share this packing. +pub fn unpackFp4(w: Tensor, packed_tag: anytype, contracting_tag: anytype) Tensor { + stdx.debug.assert(w.dtype() == .u8 or w.dtype() == .i8, "unpackFp4 expects packed 8-bit weights, got {}", .{w.dtype()}); + const source_tag = Shape.toTag(packed_tag); + const result_tag = Shape.toTag(contracting_tag); + return w.bitCast(.f4e2m1) // bitcast inserts a tag (it respects shlo), but maybe we should simplify it + .merge(.{ .kb = .{ source_tag, .bitcast } }) + .renameTag(.kb, result_tag); +} + +test "unpackFp4 expands the requested axis" { + const platform = zml.testing.env(); + const packed_weight: Tensor = .init(.{ .out = 2, .stored = 4 }, .u8); + const Local = struct { + fn call(w: Tensor) Tensor { + return unpackFp4(w, .stored, .contracted); + } + }; + + var exe = try platform.compileFn(std.testing.allocator, std.testing.io, Local.call, .{packed_weight}, .{}); + defer exe.deinit(); + + try zml.testing.expectEqualShapes(.init(.{ .out = 2, .contracted = 8 }, .f4e2m1), exe.output_shapes[0]); +} + +/// Scaled matrix multiply (`xla.scaled_dot`): `acc = (lhs * lhs_scale) @ (rhs * rhs_scale)`. +/// +/// - **NVFP4**: values `.f4e2m1`, scales `.f8e4m3fn`, block 16 (weight-only bf16 lhs ok) +/// - **MXFP4**: values `.f4e2m1`, scales `.f8e8m0fnu`, block 32 +/// - **MXFP8**: values `.f8e4m3fn` / `.f8e5m2`, scales `.f8e8m0fnu`, block 32 +/// - **Block FP8**: E4M3FN / E4M3FNUZ / E8M0 values, BF16 or F32 128x128 scales +/// - TODO: INT4/8 and FP8 per tensor +/// +/// Backends: +/// XLA selects specialized GPU kernels (TileIR, Triton, or block-128 W8A8), +/// with dequant + Dot as the fallback. +/// +/// Axes follow `Tensor.dot`; XLA currently accepts one contracting axis and at +/// most one batch axis. Scales preserve operand axis order, and each scale +/// dimension must divide its corresponding value dimension. +/// +/// CPU has no specialized path. +/// +/// `out_dtype` is the element type of the returned tensor; only BF16 and F32 stay on +/// the fused path. It buys no accuracy: the kernels accumulate in F32 either way. +pub fn scaledDot( + lhs: Tensor, + rhs: Tensor, + lhs_scale: ?Tensor, + rhs_scale: Tensor, + out_dtype: DataType, + args: anytype, + opts: ScaledDotOpts, +) Tensor { + // A narrow output would be an unscaled cast of the F32 accumulator + stdx.debug.assert( + out_dtype.isFloat() and out_dtype.bitSizeOf() >= 16, + "scaledDot output dtype must be a 16-bit-or-wider float, got {s}", + .{@tagName(out_dtype)}, + ); + const dot_axes = lhs.dotAxes(rhs, args); + + const Axes = stdx.BoundedArray(i64, constants.MAX_RANK); + + var res_shape: Shape = .scalar(out_dtype); + var lhs_batching_axes: Axes = .empty; + var rhs_batching_axes: Axes = .empty; + for (dot_axes.batching.constSlice()) |b_axes| { + const l, const r = b_axes; + stdx.debug.assert(lhs._shape.dim(l) == rhs._shape.dim(r), "scaledDot expects batching dimensions to be equal, got {} and {} in {f} and {f}", .{ l, r, lhs, rhs }); + var t = lhs._shape.tag(l); + if (t == Shape.TagUnknown) t = rhs._shape.tag(r); + res_shape = res_shape.appendDim(lhs._shape.dim(l), t); + lhs_batching_axes.appendAssumeCapacity(lhs._shape.axis(l)); + rhs_batching_axes.appendAssumeCapacity(rhs._shape.axis(r)); + } + + var lhs_contracting_axes: Axes = .empty; + var rhs_contracting_axes: Axes = .empty; + for (dot_axes.contracting.constSlice()) |c_axes| { + const l, const r = c_axes; + stdx.debug.assert(lhs._shape.dim(l) == rhs._shape.dim(r), "scaledDot expects contracting dimensions to be equal, got {} and {} in {f} and {f}", .{ l, r, lhs, rhs }); + lhs_contracting_axes.appendAssumeCapacity(lhs._shape.axis(l)); + rhs_contracting_axes.appendAssumeCapacity(rhs._shape.axis(r)); + } + + for (0..lhs.rank()) |l| { + if (std.mem.indexOfScalar(i64, lhs_contracting_axes.constSlice(), @intCast(l))) |_| { + continue; + } + if (std.mem.indexOfScalar(i64, lhs_batching_axes.constSlice(), @intCast(l))) |_| { + continue; + } + res_shape = res_shape.appendDim(lhs._shape.dim(l), lhs._shape.tag(l)); + } + for (0..rhs.rank()) |r| { + if (std.mem.indexOfScalar(i64, rhs_contracting_axes.constSlice(), @intCast(r))) |_| { + continue; + } + if (std.mem.indexOfScalar(i64, rhs_batching_axes.constSlice(), @intCast(r))) |_| { + continue; + } + res_shape = res_shape.appendDim(rhs._shape.dim(r), rhs._shape.tag(r)); + } + + const lhs_scale_operand = lhs_scale orelse blk: { + break :blk Tensor.constantTensor(onesGrid(lhs.shape(), .bf16), DataType.bf16.one().asBytes()); + }; + + const rhs_scale_operand = if (rhs_scale.rank() != rhs.rank() and rhs_scale.shape().count() == 1) + rhs_scale.reshape(onesGrid(rhs.shape(), rhs_scale.dtype())) + else + rhs_scale; + + const mlir_ctx = zml.Compiler.current().mlir_ctx; + const dnums = mlir.Attribute.array(mlir_ctx, &.{ + .array(mlir_ctx, &.{ + .intArray(mlir_ctx, i64, lhs_contracting_axes.constSlice()), + .intArray(mlir_ctx, i64, rhs_contracting_axes.constSlice()), + }), + .array(mlir_ctx, &.{ + .intArray(mlir_ctx, i64, lhs_batching_axes.constSlice()), + .intArray(mlir_ctx, i64, rhs_batching_axes.constSlice()), + }), + }); + const attributes = [_]mlir.NamedAttribute{ + .named(mlir_ctx, "dimension_numbers", dnums), + .named(mlir_ctx, "rhs_scale_swizzled", .boolean(mlir_ctx, true)), + }; + if (opts.rhs_scale_swizzled) { + stdx.debug.assert(rhs.rank() == 2 and rhs_scale.rank() == 2 and rhs_contracting_axes.get(0) == 1, "scaledDot: a swizzled rhs scale needs a [n, k] rhs, got {f}", .{rhs}); + } + return ops.composite("xla.scaled_dot", &.{ lhs, rhs, lhs_scale_operand, rhs_scale_operand }, &.{res_shape}, scaledDotReference, res_shape, .{ + .composite_attributes = if (opts.rhs_scale_swizzled) &attributes else attributes[0..1], + })[0]; +} + +pub const ScaledDotOpts = struct { + /// The rhs scale's `[n, k / block]` grid is stored in the 128x4 blocked layout of Blackwell's block-scaled mma, + /// each device's slice on its own. + // TODO: fine for now, but we should find a better way to express how a scale is laid out. + rhs_scale_swizzled: bool = false, +}; + +test "block128 scaled dot layouts" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = zml.testing.env(); + // FIXME: Add rocm when we have a PJRT plugin with native fp8 dot support (by the end of Sept 2026) + if (platform.target != .cuda and platform.target != .furiosa) return error.SkipZigTest; + const dtype: DataType = if (platform.target == .rocm and @import("platform.zig").rocm.computeCapability(platform) == .gfx942) .f8e4m3fnuz else .f8e4m3fn; + const Local = struct { + const Outputs = struct { actual: Tensor, expected: Tensor, linear: Tensor, linear_expected: Tensor }; + + fn dequantize(values: Tensor, scales: Tensor) Tensor { + var expanded = scales.convert(.bf16); + for (0..values.rank()) |axis| { + const factor = @divExact(values.dim(axis), scales.dim(axis)); + const shape = expanded.shape().insert(axis + 1, .{factor}); + const axes = Shape.range(shape.rank(), .i64).remove(axis + 1); + expanded = expanded.broadcast(shape, axes.dims()).reshape(expanded.shape().setDim(axis, values.dim(axis))); + } + return values.convert(.bf16).mul(expanded); + } + + fn forward(x: Tensor, w: Tensor, scales: Tensor, fp8: DataType, prequantized: bool) Outputs { + const weight = w.convert(fp8); + const input = quantization.quantizeBlockFp8(x, .k, 128, fp8, .f32); + const linear: Linear = .{ .weight = weight, .tag = Shape.toTag(.k), .quantization = .{ .scheme = .fp8_block128, .scales = scales } }; + return .{ + .linear = linear.forward(x, .bf16), + .linear_expected = dequantize(input.values, input.scales).dot(dequantize(weight, scales), .k), + .actual = if (prequantized) + scaledDot(input.values, weight, input.scales.convert(scales.dtype()), scales, .bf16, .k, .{}) + else + scaledDot(x, weight, null, scales, .bf16, .k, .{}), + .expected = (if (prequantized) dequantize(input.values, input.scales.convert(scales.dtype())) else x) + .dot(dequantize(weight, scales), .k), + }; + } + }; + inline for (.{ + .{ .{ .m = 16, .k = 256 }, .{ .n = 256, .k = 256 } }, + .{ .{ .b = 2, .m = 16, .k = 256 }, .{ .b = 2, .n = 256, .k = 256 } }, + .{ .{ .k = 256, .b = 2, .m = 3 }, .{ .n = 256, .k = 256, .b = 2 } }, + .{ .{ .b = 2, .k = 256, .s = 2, .m = 3 }, .{ .n = 128, .b = 2, .k = 256 } }, + .{ .{ .b = 2, .m = 3, .k = 256 }, .{ .k = 256, .n = 128 } }, + .{ .{ .k = 128 }, .{ .n = 128, .k = 128 } }, + .{ .{ .m = 3, .k = 256 }, .{ .p = 2, .k = 256, .n = 128 } }, + .{ .{ .m = 3, .k = 128 }, .{ .n = 64, .k = 128 } }, + }) |layout| { + inline for (.{ DataType.f32, DataType.bf16 }) |scale_dtype| { + inline for (.{ false, true }) |prequantized| { + const x: Tensor = .init(layout[0], .bf16); + const w: Tensor = .init(layout[1], .bf16); + const scales: Tensor = .init(w.shape().setDim(.n, std.math.divCeil(i64, w.dim(.n), 128) catch unreachable).setDim(.k, @divExact(w.dim(.k), 128)).withDtype(scale_dtype), scale_dtype); + var exe = try platform.compileFn(allocator, io, Local.forward, .{ x, w, scales, dtype, prequantized }, .{}); + defer exe.deinit(); + try zml.testing.expectEqualShapes(exe.output_shapes[1], exe.output_shapes[0]); + var buffers: [3]zml.Buffer = undefined; + var initialized: usize = 0; + defer for (buffers[0..initialized]) |*buffer| buffer.deinit(); + for ([_]Shape{ x.shape(), w.shape(), scales.shape() }, 0..) |shape, i| { + const slice = try Slice.alloc(allocator, shape); + defer slice.free(allocator); + for (0..shape.count()) |j| { + const value: f32 = if (i == 2) @as(f32, @floatFromInt(1 + (j * 3 + j / 4) % 7)) / 16 else @as(f32, @floatFromInt(@as(i32, @intCast((j * 7 + j / 256) % 13)) - 6)) / 4; + if (shape.dtype() == .f32) { + slice.items(f32)[j] = value; + } else { + slice.items(zml.floats.BFloat16)[j] = .fromF32(value); + } + } + buffers[i] = try zml.Buffer.fromSlice(io, platform, slice); + initialized += 1; + } + var output: zml.Bufferized(Local.Outputs) = undefined; + var runner = try exe.runner(allocator); + defer runner.deinit(allocator); + runner.run(io, .{ buffers[0], buffers[1], buffers[2] }, .{&output}, .{ .wait = true }); + defer zml.Buffer.deinitAll(Local.Outputs, &output); + var expected = try output.expected.toSliceAlloc(allocator, io); + defer expected.free(allocator); + try zml.testing.expectClose(io, expected, output.actual, .{ .absolute_tolerance = 0.125, .relative_tolerance = 0.02 }); + var linear_expected = try output.linear_expected.toSliceAlloc(allocator, io); + defer linear_expected.free(allocator); + try zml.testing.expectClose(io, linear_expected, output.linear, .{ .absolute_tolerance = 0.125, .relative_tolerance = 0.02 }); + } + } + } +} + +/// `shape` with every dimension collapsed to 1 +fn onesGrid(shape: Shape, dt: DataType) Shape { + var res = shape.withDtype(dt); + for (0..res.rank()) |i| { + res = res.setDim(i, 1); + } + + return res; +} + +fn scaledDotReference(in: []const Tensor, out_shape: Shape) Tensor { + // Keep the unmatched-composite marker in the graph for backend lowering. + const ctx = zml.Compiler.current(); + const op = stablehlo.custom_call( + ctx.mlir_ctx, + &.{ in[0].value(), in[1].value(), in[2].value(), in[3].value() }, + &.{mlirx.Type.rankedTensor(ctx.mlir_ctx, out_shape)}, + .{ + .call_target_name = "zml$scaled_dot_unmatched", + .has_side_effect = false, + .backend_config = .{ .typed_ffi = .dict(ctx.mlir_ctx, &.{}) }, + }, + ctx.location, + ).appendTo(ctx.currentScope().block); + if (ctx.manual_computation_depth > 0 and ctx.partitioner == .gspmd) { + op.setAttributeByName("mhlo.sharding", .string(ctx.mlir_ctx, "{manual}")); + } + return Tensor._result(out_shape, op.result(0)); +} + +fn applyGlobalScale(acc: Tensor, igs: ?Tensor, wgs: ?Tensor) Tensor { + if (igs == null and wgs == null) { + return acc; + } + + var res = acc.convert(.f32); + if (wgs) |w| { + res = res.mul(w.convert(.f32).broad(res.shape())); + } + + if (igs) |i| { + res = res.mul(i.convert(.f32).broad(res.shape())); + } + + return res.convert(acc.dtype()); +} + pub const TokenEmbedding = struct { weight: Tensor, pub fn forward(self: TokenEmbedding, idx: Tensor) Tensor { - meta.assert(idx.dtype().isInteger(), "TokenEmbedding expects an integer input, received: {}", .{idx}); - meta.assert(self.weight.rank() == 2, "TokenEmbedding expects it's weight Tensor to be a 2D matrix, got {}", .{self.weight}); - return self.weight.gatherValues(0, idx, .{}); + stdx.debug.assert(idx.dtype().isInteger(), "TokenEmbedding expects an integer input, received: {f}", .{idx}); + stdx.debug.assert(self.weight.rank() == 2, "TokenEmbedding expects it's weight Tensor to be a 2D matrix, got {f}", .{self.weight}); + return self.weight.withTags(.{ .voc, .d }).gather(.{ .voc = idx }, .{}); } }; @@ -52,6 +373,7 @@ pub const Activation = union(enum) { tanh, relu, leakyReLU: f32, + elu: f32, silu, gelu, quick_gelu, @@ -63,24 +385,18 @@ pub const Activation = union(enum) { .relu => x.relu(), .silu => x.silu(), .gelu => x.gelu(), + .elu => |alpha| elu(x, alpha), .quick_gelu => x.quickGelu(), .leakyReLU => |slope| x.leakyReLU(slope), }; } }; -pub fn chainModules(module_list: anytype, input: Tensor) Tensor { - const T = @TypeOf(module_list); - switch (@typeInfo(T)) { - .Struct => |struct_info| { - var x = input; - inline for (struct_info.fields) |field| { - x = @field(module_list, field.name).forward(x); - } - return x; - }, - else => @compileError("chainModules only works on a struct with only containing 'module' struct."), - } +pub fn elu(x: Tensor, alpha: f32) Tensor { + return x.cmp(.GE, Tensor.scalar(0, x.dtype())).select( + x, + x.exp().addConstant(-1).scale(alpha), + ); } /// Layer Normalization @@ -91,14 +407,23 @@ pub const LayerNorm = struct { pub fn forward(self: LayerNorm, x: Tensor) Tensor { const normed = normalizeVariance(x, self.eps); - - var out = normed.mul(self.weight.broadcastLeft(x.shape())); - if (self.bias) |bias| out = out.add(bias.broadcastLeft(x.shape())); + const ax = x.axis(-1); + var out = normed.mul(self.weight.broadcast(x.shape(), &.{ax})); + if (self.bias) |bias| out = out.add(bias.broadcast(x.shape(), &.{ax})); return out; } }; +pub fn rmsNorm(x_: Tensor, axis: anytype, eps: f32) Tensor { + const ax = x_.axis(axis); + // upcast to improve precision + var x = x_.convert(.f32); + const variance = x.powByConst(2).mean(ax); + const rsqrt = Tensor.rsqrt(variance.addConstant(eps)); + return x.mul(rsqrt.broad(x.shape())).convert(x_.dtype()); +} + /// Center and scale by the variance. /// normalize(x, eps) = (x - mean(x)) / sqrt(var(x) + eps) /// Work on the last axis. @@ -108,129 +433,505 @@ pub fn normalizeVariance(x: Tensor, eps: f32) Tensor { // Upcast to improve precision const xf32 = x.convert(.f32); const mean = xf32.sum(-1).scale(1.0 / N); - const mean_dev = xf32.sub(mean.broadcastRight(xf32.shape())); - const variance = mean_dev.mul(mean_dev).sum(-1).scale(1.0 / N); + const mean_dev = xf32.sub(mean); + const variance = mean_dev.mul(mean_dev).sum(-1).divByConst(N); const rsqrt = Tensor.rsqrt(variance.addConstant(eps)); - return mean_dev.mul(rsqrt.broadcastRight(mean_dev.shape())).convert(x.dtype()); + return mean_dev.mul(rsqrt).convert(x.dtype()); } // ref: https://pytorch.org/docs/stable/generated/torch.nn.functional.normalize.html // Implementation equivalent to `nn.functional.normalize(tensor, dim=-1)` call pub fn normalizeL2(input: Tensor, eps: f32) Tensor { - const inv_norm = input.pow(Tensor.scalar(2, input.dtype())).sum(-1).addConstant(eps).rsqrt(); - return input.mul(inv_norm.broad(input.shape())); + const inv_norm = input.powByConst(2).sum(-1).addConstant(eps).rsqrt(); + return input.mul(inv_norm); } test normalizeL2 { const platform = zml.testing.env(); - const input = try zml.Buffer.fromSlice(platform, .{ 2, 2 }, &[_]f32{ -0.9686, -1.0058, -1.7808, 0.6698 }); + const input: zml.Tensor = .init(.{ 2, 2 }, .f32); - const res = try zml.testing.compileAndCall(platform, zml.nn.normalizeL2, .{ input, 1e-12 }); - const expectation = zml.HostBuffer.fromSlice(.{ 2, 2 }, &[_]f32{ -0.6937, -0.7203, -0.9360, 0.3520 }); - try zml.testing.expectClose(expectation, res, 1e-4); + var exe = try platform.compileFn(std.testing.allocator, std.testing.io, normalizeL2, .{ input, 1e-12 }, .{}); + defer exe.deinit(); + + var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), std.mem.sliceAsBytes(&[_]f32{ -0.9686, -1.0058, -1.7808, 0.6698 })); + defer input_buffer.deinit(); + + var res = try exe.eval(std.testing.allocator, std.testing.io, .{input_buffer}); + defer res.deinit(); + + const expectation: Slice = .init(input.shape(), std.mem.sliceAsBytes(&[_]f32{ -0.6937, -0.7203, -0.9360, 0.3520 })); + try zml.testing.expectClose(std.testing.io, expectation, res, .{}); } pub const RopeOpts = struct { - /// There are two implementations corresponding to how to split `x` in real/imag parts. - /// * Interleaved means that the real/imag of each scalar is contiguous. - /// * Sequential means that you first read all real values then all imag values. - pub const Implementation = enum { interleaved, sequential }; + layout: Layout = .real_im_pass, + scaling: Scaling = .{ .default = .{} }, - impl: Implementation, - freq_base: f32 = 10_000, + /// There are 3 layouts corresponding to how to split `x` in real/imag/passthrough parts. + /// The hard part is that HF models don't specify the layout they use. + /// But HF models don't use the `interleaved` format, + /// and most models don't have a Rope passthrough, so real_im_pass and real_pass_im_pass are the same. + /// + /// * real_im_pass: [x_re0, x_re1, ..., x_im0, x_im1, ..., p_re0, p_re1, ..., p_im0, p_im1, ...] + /// * real_pass_im_pass: [x_re0, x_re1, ..., p_re0, p_re1, ..., x_im0, x_im1, ..., p_im0, p_im1, ...] + /// * interleaved: : [x_re0, x_im0, x_re1, x_im1, ..., p_re0, p_im0, p_re1, p_im1, ...] + pub const Layout = enum { real_im_pass, real_pass_im_pass, interleaved }; + + /// There are several ways to init the scaling aka "inv_freq" + pub const Scaling = union(enum) { + default: Default, + llama3: Llama3, + yarn: Yarn, + linear: Scaling.Linear, + proportional: Scaling.Proportional, + + pub const Default = struct { + rope_theta: f32 = 10000, + }; + + pub const Llama3 = struct { + factor: f32, + high_freq_factor: f32, + low_freq_factor: f32, + original_max_position_embeddings: u32, + truncate: bool = true, + rope_theta: f32 = 10000, + }; + + pub const Yarn = struct { + beta_fast: f32 = 32.0, + beta_slow: f32 = 1.0, + factor: ?f32 = null, + mscale: ?f32 = null, + mscale_all_dim: ?f32 = null, + truncate: bool = true, + original_max_position_embeddings: u32, + rope_theta: f32 = 10000, + llama_4_scaling_beta: ?f32 = null, + attention_factor: ?f32 = null, + partial_rotary_factor: f32 = 1.0, + }; + + pub const Linear = struct { + factor: f32, + rope_theta: f32 = 10000, + }; + + pub const Proportional = struct { + partial_rotary_factor: f32 = 1.0, + rope_theta: f32 = 10000, + }; + + /// Read a Rope scaling config from HF config.json format. + pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) !Scaling { + return jsonParseFromValue( + allocator, + try std.json.innerParse(std.json.Value, allocator, source, options), + options, + ); + } + + pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) std.json.ParseFromValueError!Scaling { + _ = allocator; + _ = options; + if (source == .null) return .{ .default = .{} }; + + if (source != .object) return error.InvalidEnumTag; + + const obj = source.object; + const impl = obj.get("rope_type") orelse return error.MissingField; + if (impl != .string) return error.InvalidEnumTag; + if (std.mem.eql(u8, impl.string, "llama3")) { + // Note: leaky is fine here cause Llama3 struct don't need to allocate memory. + return .{ .llama3 = try std.json.parseFromValueLeaky(Llama3, stdx.noalloc, source, .{ .ignore_unknown_fields = true }) }; + } else if (std.mem.eql(u8, impl.string, "default")) { + return .{ .default = try std.json.parseFromValueLeaky(Default, stdx.noalloc, source, .{ .ignore_unknown_fields = true }) }; + } else if (std.mem.eql(u8, impl.string, "yarn")) { + return .{ .yarn = try std.json.parseFromValueLeaky(Yarn, stdx.noalloc, source, .{ .ignore_unknown_fields = true }) }; + } else if (std.mem.eql(u8, impl.string, "linear")) { + return .{ .linear = try std.json.parseFromValueLeaky(Scaling.Linear, stdx.noalloc, source, .{ .ignore_unknown_fields = true }) }; + } else if (std.mem.eql(u8, impl.string, "proportional")) { + return .{ .proportional = try std.json.parseFromValueLeaky(Scaling.Proportional, stdx.noalloc, source, .{ .ignore_unknown_fields = true }) }; + } else { + log.warn("Unsupported Rope implementation: {s}, will use the default one which will produce altered results", .{impl.string}); + return .{ .default = .{} }; + } + } + + fn getMScale(factor: f32, mscale: f32) f32 { + if (mscale < 1.0) { + return 1.0; + } + return 0.1 * mscale * @log(factor) + 1.0; + } + + pub fn attentionScaling(scaling: Scaling) f32 { + return switch (scaling) { + .yarn => |yarn| yarn_scaling: { + const attention_factor = if (yarn.attention_factor) |af| af else b: { + break :b if (yarn.mscale != null and yarn.mscale_all_dim != null) + getMScale(yarn.factor.?, yarn.mscale.?) / getMScale(yarn.factor.?, yarn.mscale_all_dim.?) + else + getMScale(yarn.factor.?, 1.0); + }; + break :yarn_scaling attention_factor; + }, + else => 1.0, + }; + } + + pub fn setRopeTheta(self: *Scaling, rope_theta: f32) void { + // NOTE(Corendos): We can use a "inline else" because `rope_theta` is mandatory in each variant. + return switch (self.*) { + inline else => |*s| s.rope_theta = rope_theta, + }; + } + + pub fn getRopeTheta(self: *const Scaling) f32 { + // NOTE(Corendos): We can use a "inline else" because `rope_theta` is mandatory in each variant. + return switch (self.*) { + inline else => |s| s.rope_theta, + }; + } + + pub fn partialRotaryDim(self: *const Scaling, head_dim: i64) u32 { + return switch (self.*) { + inline .yarn, .proportional => |s| @intFromFloat(@floor( + @as(f32, @floatFromInt(head_dim)) * std.math.clamp(s.partial_rotary_factor, 0.0, 1.0), + )), + .default, .llama3, .linear => @intCast(head_dim), + }; + } + }; }; -pub const CosSin = [2]Tensor; - /// Rotary position embedding modify queries and keys tensor before compute Q * K in self attention. /// This biases a token to look at token near him. -/// The nice thing with this solution is that you can cache the modified queries and keys directly. +/// The nice thing with rope is that you can cache the modified queries and keys directly. /// See: https://paperswithcode.com/method/rope -pub fn rope(x: Tensor, cos_sin_cache: CosSin, opts: RopeOpts) Tensor { - const cos, const sin = cos_sin_cache; - meta.assert(x.dim(-1) == 2 * cos.dim(-1), "Couldn't compute rope({}, {}, {})", .{ x, cos, sin }); - // broadcast cos / sin to .{ batch, .seq, .half_dim } - const x_real, const x_imag = splitRealImg(x, opts.impl); - const has_tags = cos.shape().tag(0) != Shape.TagUnknown; - const b_cos = if (has_tags) cos.broad(x_real.shape()) else cos.broadcastLeft(x_real.shape()); - const b_sin = if (has_tags) sin.broad(x_real.shape()) else sin.broadcastLeft(x_real.shape()); +/// +/// Expected shapes of tensor: +/// - x: .{ .s, .hd } where .s is the sequence length and .hd the head dimension +/// - pos_idx: optional tensor which indicates which positions are needed. +/// When not set `rope` return all positions from 0 to x.dim(.s) which is the max seq len. +pub fn rope(x: Tensor, pos_idx: ?Tensor, opts: RopeOpts) Tensor { + const head_dim = x.dim(.hd); + stdx.debug.assert(@mod(head_dim, 2) == 0, "rope expects a even head dim (.hd), got {f}", .{x}); - // apply rotation - const y_real = x_real.mul(b_cos).sub(x_imag.mul(b_sin)); - const y_imag = x_real.mul(b_sin).add(x_imag.mul(b_cos)); + const idx = if (pos_idx) |idx| blk: { + stdx.debug.assert(x.shape().hasTags(.{.hd}), "rope expects x argument to have .hd axes got: rope(x={f}, idx={f})", .{ x, idx }); + break :blk idx; + } else blk: { + stdx.debug.assert(x.shape().hasTags(.{ .s, .hd }), "rope expects x argument to have both .s and .hd axes got: rope(x={f})", .{x}); + break :blk Tensor.arange(.{ .end = x.dim(.s) }, .f32).withTags(.{.s}); + }; - // flatten last dimensions - const y = mergeRealImg(y_real, y_imag, opts.impl); + const rotary_dim: u32 = opts.scaling.partialRotaryDim(head_dim); + stdx.debug.assert(rotary_dim > 0 and @mod(rotary_dim, 2) == 0, "partial rope expects a even head dim (.hd), got {d}", .{rotary_dim}); - return y; -} - -pub fn ropeCosSin(sh: anytype, dtype: DataType, opts: RopeOpts) CosSin { - const shape = Shape.init(sh, dtype); - meta.assert(shape.rank() == 2, "ropeCosSin({}) shape need to exactly have 2 axes", .{shape}); - const seq_len, const head_dim = .{ shape.dim(0), shape.dim(1) }; - meta.assert(@mod(head_dim, 2) == 0, "ropeCosSin requires an even head_dim, got {}", .{head_dim}); + const x_real, const x_imag, const x_pass = zml.nn.splitRealImgPass(x, opts.layout, rotary_dim); + const inv_freq = invFreq(head_dim, opts).withTags(.{.hd}); // compute sin and cos in f32 before downcasting to x type. - const inv_freq = invFreq(head_dim, opts.freq_base, .f32); - var inv_freq_pos = Tensor.outer(Tensor.arange(.{ .end = seq_len }, .f32), inv_freq).convert(shape.dtype()); - inv_freq_pos._shape._tags = shape._tags; - const cos = inv_freq_pos.cos(); - const sin = inv_freq_pos.sin(); - return .{ cos, sin }; + const inv_freq_pos = zml.Tensor.outer(idx.convert(.f32), inv_freq); + const scaling = opts.scaling.attentionScaling(); + const cos = inv_freq_pos.cos().scale(scaling).convert(x.dtype()).broad(x_real.shape()); + const sin = inv_freq_pos.sin().scale(scaling).convert(x.dtype()).broad(x_real.shape()); + + // apply rotation + const y_real = x_real.mul(cos).sub(x_imag.mul(sin)); + const y_imag = x_real.mul(sin).add(x_imag.mul(cos)); + + return zml.nn.mergeRealImgPass(y_real, y_imag, x_pass, opts.layout); } -pub fn splitRealImg(x: Tensor, impl: RopeOpts.Implementation) [2]Tensor { +pub fn splitRealImg(x: Tensor, layout: RopeOpts.Layout) [2]Tensor { const n = x.dim(-1); - return switch (impl) { - .sequential => .{ - x.slice1d(-1, .{ .end = @divExact(n, 2) }), - x.slice1d(-1, .{ .start = @divExact(n, 2), .end = n }), + return switch (layout) { + .real_im_pass, .real_pass_im_pass => .{ + x.slice(-1, .{ .end = @divExact(n, 2) }), + x.slice(-1, .{ .start = @divExact(n, 2), .end = n }), }, .interleaved => .{ - x.slice1d(-1, .{ .start = 0, .step = 2 }), - x.slice1d(-1, .{ .start = 1, .step = 2 }), + x.slice(-1, .{ .start = 0, .step = 2 }), + x.slice(-1, .{ .start = 1, .step = 2 }), }, }; } -pub fn mergeRealImg(x_real: Tensor, x_imag: Tensor, impl: RopeOpts.Implementation) Tensor { - return switch (impl) { - .sequential => Tensor.concatenate(&.{ x_real, x_imag }, -1), - .interleaved => Tensor.concatenate(&.{ - x_real.appendAxes(.{.interleaved_real_img}), - x_imag.appendAxes(.{.interleaved_real_img}), - }, -1).flatten(-2), +const Pass = union(RopeOpts.Layout) { + real_im_pass: Tensor, + real_pass_im_pass: [2]Tensor, + interleaved: Tensor, +}; + +pub fn splitRealImgPass(x: Tensor, layout: RopeOpts.Layout, rotary_dim: u32) struct { Tensor, Tensor, ?Pass } { + const ax = x.axis(.hd); + const hd = x.dim(ax); + if (rotary_dim == hd) { + const real, const img = splitRealImg(x, layout); + return .{ real, img, null }; + } + + const half = @divExact(x.dim(ax), 2); + const half_rotary = @divExact(rotary_dim, 2); + return switch (layout) { + .real_im_pass => .{ + x.slice(ax, .{ .end = half_rotary }), + x.slice(ax, .{ .start = half_rotary, .end = rotary_dim }), + .{ .real_im_pass = x.slice(ax, .{ .start = rotary_dim }) }, + }, + .real_pass_im_pass => .{ + x.slice(ax, .{ .end = half_rotary }), + x.slice(ax, .{ .start = half, .end = half + half_rotary }), + .{ .real_pass_im_pass = .{ + x.slice(ax, .{ .start = half_rotary, .end = half }), + x.slice(ax, .{ .start = half + half_rotary }), + } }, + }, + .interleaved => .{ + x.slice(ax, .{ .start = 0, .end = rotary_dim, .step = 2 }), + x.slice(ax, .{ .start = 1, .end = rotary_dim, .step = 2 }), + .{ .interleaved = x.slice(ax, .{ .start = rotary_dim }) }, + }, + }; +} + +pub fn mergeRealImg(x_real: Tensor, x_imag: Tensor, layout: RopeOpts.Layout) Tensor { + return switch (layout) { + .real_im_pass, .real_pass_im_pass => .concatenate(&.{ x_real, x_imag }, -1), + .interleaved => Tensor.stack(&.{ x_real, x_imag }, .last, .interleaved_real_img) + .merge(.{ .hd = .{ .hd, .interleaved_real_img } }), + }; +} + +pub fn mergeRealImgPass(x_real: Tensor, x_imag: Tensor, x_pass: ?Pass, layout: RopeOpts.Layout) Tensor { + if (x_pass == null) return mergeRealImg(x_real, x_imag, layout); + + return switch (x_pass.?) { + .real_im_pass => |pass| .concatenate(&.{ x_real, x_imag, pass }, .hd), + .real_pass_im_pass => |pass| .concatenate(&.{ x_real, pass[0], x_imag, pass[1] }, .hd), + .interleaved => |pass| .concatenate( + &.{ + Tensor.stack(&.{ x_real, x_imag }, .last, .interleaved_real_img) + .merge(.{ .hd = .{ .hd, .interleaved_real_img } }), + pass, + }, + .hd, + ), }; } /// {exp( - n * ln(10_000) / N ) | n in [0..N] } -pub fn invFreq(N: i64, theta: f32, dtype: DataType) Tensor { - const freq = -@log(theta) / @as(f32, @floatFromInt(N)); - const range = Tensor.arange(.{ .start = 0, .end = N, .step = 2 }, dtype).scale(freq); - return range.exp(); +pub fn invFreq(N: i64, opts: RopeOpts) Tensor { + const allocator = zml.Compiler.current().allocator; + const N_half: u32 = @intCast(@divExact(N, 2)); + const num_freqs: u32 = opts.scaling.partialRotaryDim(N_half); + + const inv_freq = allocator.alloc(f32, num_freqs) catch @panic("OOM"); + defer allocator.free(inv_freq); + + _invFreq(opts, inv_freq, N_half); + return .constantTensor(.init(.{num_freqs}, .f32), @ptrCast(inv_freq)); +} + +fn _invFreq(opts: RopeOpts, inv_freq: []f32, half_head_dim: u32) void { + const N: u32 = @intCast(inv_freq.len); + // Default frequencies + for (0.., inv_freq) |n, *f| { + f.* = @exp(-@log(opts.scaling.getRopeTheta()) * stdx.math.divFloat(f32, n, N)); + } + + switch (opts.scaling) { + .default => {}, + .proportional => { + // proportional uses the original number of dimensions, + // instead of the actual number of rotated dims. + for (0.., inv_freq[0..]) |n, *f| { + f.* = @exp(-@log(opts.scaling.getRopeTheta()) * stdx.math.divFloat(f32, n, half_head_dim)); + } + }, + .linear => |l| { + for (inv_freq) |*f| f.* /= l.factor; + }, + .llama3 => |s| { + // https://arxiv.org/pdf/2309.16039 + // After Llama2 they observed that the rope frequencies where too sharp and hurting long distance attention. + // In Llama3 they used a higher base freq and also downscaled low frequencies. + std.debug.assert(s.low_freq_factor < s.high_freq_factor); + const M: f64 = @floatFromInt(s.original_max_position_embeddings); + const f_high = s.high_freq_factor * (2 * std.math.pi) / M; + const f_low = s.low_freq_factor * (2 * std.math.pi) / M; + const downscaling = 1.0 / s.factor; + + for (0..N, inv_freq) |n, f| { + if (f > f_high) { + // High freq match default implem + } else if (f < f_low) { + // Downscaling for low freq + inv_freq[n] *= downscaling; + } else { + // Linear interpolation for middle freq + const lerp: f64 = (inv_freq[n] - f_low) / (f_high - f_low); + inv_freq[n] *= @floatCast(lerp + (1 - lerp) * downscaling); + } + } + }, + .yarn => |s| { + const N_f: f64 = @floatFromInt(N); + const M: f64 = @floatFromInt(s.original_max_position_embeddings); + const f_high = s.beta_fast * (2 * std.math.pi) / M; + const f_low = s.beta_slow * (2 * std.math.pi) / M; + const downscaling = 1.0 / s.factor.?; + + // This isn't a typo: low n have a high frequency, high n have a low frequency. + var n_low: f64 = -@log(f_high) / @log(opts.scaling.getRopeTheta()) * N_f; + var n_high: f64 = -@log(f_low) / @log(opts.scaling.getRopeTheta()) * N_f; + if (s.truncate) { + n_high = std.math.ceil(n_high); + n_low = std.math.floor(n_low); + } + std.debug.assert(n_high > n_low); + for (0..N, inv_freq) |n, f| { + if (f > f_high) { + // High freq match default implem + } else if (f < f_low) { + // Downscaling for low freq + inv_freq[n] *= downscaling; + } else { + // Yarn use lerp too but not in the frequency space, in the time space. + const lerp: f64 = (n_high - @as(f64, @floatFromInt(n))) / (n_high - n_low); + inv_freq[n] *= @floatCast(lerp + (1 - lerp) * downscaling); + } + } + }, + } +} + +test "invFreq Llama3" { + // Llama 3.2-1B config + const llama_conf: RopeOpts = .{ + .scaling = .{ .llama3 = .{ + .factor = 32, + .high_freq_factor = 4, + .low_freq_factor = 1, + .original_max_position_embeddings = 8192, + .rope_theta = 500_000, + } }, + }; + const llama_freq = [_]f32{ 1.000000000000e+00, 6.636012792587e-01, 4.403666257858e-01, 2.922278344631e-01, 1.939227581024e-01, 1.286873817444e-01, 8.539710193872e-02, 5.666961893439e-02, 3.760603070259e-02, 2.495540864766e-02, 1.656044088304e-02, 1.098952908069e-02, 7.292665075511e-03, 4.839421249926e-03, 3.211446106434e-03, 1.290548010729e-03, 4.295567050576e-04, 9.708286233945e-05, 1.946163865796e-05, 1.291476746701e-05, 8.570255886298e-06, 5.687232260243e-06, 3.774054448513e-06, 2.504467147446e-06, 1.661967417022e-06, 1.102883629756e-06, 7.318749339902e-07, 4.856731266045e-07, 3.222932889457e-07, 2.138742303259e-07, 1.419272024350e-07, 9.418306490261e-08 }; + + var inv_freq: @TypeOf(llama_freq) = undefined; + _invFreq(llama_conf, &inv_freq, 32); + for (llama_freq, inv_freq, 0..) |expected, actual, i| { + errdefer log.err("Mismatch at position {d}.\nExpected: {any}\nActual: {any}", .{ i, llama_freq, inv_freq }); + try std.testing.expectApproxEqRel(expected, actual, 1e-5); + } +} + +test "invFreq Yarn" { + const yarn_conf: RopeOpts = .{ + .scaling = .{ .yarn = .{ + .factor = 32.0, + .beta_fast = 32.0, + .beta_slow = 1.0, + .original_max_position_embeddings = 4096, + .truncate = true, + .rope_theta = 150_000, + } }, + }; + const yarn_freq = [_]f32{ 1.000000000000e+00, 6.890442967415e-01, 4.747820496559e-01, 3.271458745003e-01, 2.254180014133e-01, 1.553229838610e-01, 1.070244237781e-01, 7.374456524849e-02, 5.081327259541e-02, 3.162075206637e-02, 1.945096626878e-02, 1.179219130427e-02, 7.015713956207e-03, 4.069554619491e-03, 2.277272054926e-03, 1.206130953506e-03, 5.809474969283e-04, 2.279478358105e-04, 3.830881178146e-05, 2.639646845637e-05, 1.818833698053e-05, 1.253256959899e-05, 8.635495760245e-06, 5.950239483354e-06, 4.099978468730e-06, 2.825066758305e-06, 1.946596285052e-06, 1.341290953860e-06, 9.242089618056e-07, 6.368209142238e-07, 4.387978549403e-07, 3.023511396805e-07 }; + + var inv_freq: @TypeOf(yarn_freq) = undefined; + _invFreq(yarn_conf, &inv_freq, 32); + for (yarn_freq, inv_freq, 0..) |expected, actual, i| { + errdefer log.err("Mismatch at position {d}.\nExpected: {d}\nActual: {d}", .{ i, stdx.fmt.slice(&yarn_freq), stdx.fmt.slice(&inv_freq) }); + try std.testing.expectApproxEqRel(expected, actual, 1e-5); + } + try std.testing.expectApproxEqRel(1.3465735902799727, yarn_conf.scaling.attentionScaling(), 1e-5); +} + +test "RopeOpts.Scaling parses proportional" { + const json = + \\{ + \\ "rope_type": "proportional", + \\ "partial_rotary_factor": 0.25, + \\ "rope_theta": 1000000 + \\} + ; + var value = try std.json.parseFromSlice(std.json.Value, std.testing.allocator, json, .{}); + defer value.deinit(); + + const scaling = try RopeOpts.Scaling.jsonParseFromValue(std.testing.allocator, value.value, .{}); + switch (scaling) { + .proportional => |p| { + try std.testing.expectApproxEqRel(0.25, p.partial_rotary_factor, 1e-6); + try std.testing.expectApproxEqRel(1_000_000, p.rope_theta, 1e-6); + }, + else => try std.testing.expect(false), + } +} + +test "RopeOpts.Scaling parses yarn partial rotary factor" { + const json = + \\{ + \\ "rope_type": "yarn", + \\ "factor": 32.0, + \\ "original_max_position_embeddings": 4096, + \\ "partial_rotary_factor": 0.5 + \\} + ; + var value = try std.json.parseFromSlice(std.json.Value, std.testing.allocator, json, .{}); + defer value.deinit(); + + const scaling = try RopeOpts.Scaling.jsonParseFromValue(std.testing.allocator, value.value, .{}); + switch (scaling) { + .yarn => |y| { + try std.testing.expectApproxEqRel(0.5, y.partial_rotary_factor, 1e-6); + }, + else => try std.testing.expect(false), + } +} + +test "RopeOpts.Scaling parses default rope type" { + const json = + \\{ + \\ "rope_type": "default", + \\ "rope_theta": 10000 + \\} + ; + var value = try std.json.parseFromSlice(std.json.Value, std.testing.allocator, json, .{}); + defer value.deinit(); + + const scaling = try RopeOpts.Scaling.jsonParseFromValue(std.testing.allocator, value.value, .{}); + switch (scaling) { + .default => |d| { + try std.testing.expectApproxEqRel(10_000, d.rope_theta, 1e-6); + }, + else => try std.testing.expect(false), + } } test "real/img" { const platform = zml.testing.env(); const Fns = struct { - fn testSplitMergeIsId(impl: RopeOpts.Implementation) Tensor { - const x = Tensor.arange(.{ .end = 20 }, .f32).reshape(.{ 5, 4 }); - const real, const imag = splitRealImg(x, impl); - const y = mergeRealImg(real, imag, impl); - const real2, const imag2 = splitRealImg(y, impl); - return real.cmp(.EQ, real2).flatten(0).convert(.i32).sum(-1).add( - imag.cmp(.EQ, imag2).flatten(0).convert(.i32).sum(-1), + fn testSplitMergeIsId(layout: RopeOpts.Layout) Tensor { + const x = Tensor.arange(.{ .end = 20 }, .f32).reshape(.{ .n = 5, .hd = 4 }); + const real, const imag = splitRealImg(x, layout); + const y = mergeRealImg(real, imag, layout); + const real2, const imag2 = splitRealImg(y, layout); + return real.cmp(.EQ, real2).flatten().convert(.i32).sum(-1).add( + imag.cmp(.EQ, imag2).flatten().convert(.i32).sum(-1), ); } fn testSplitSeqVoid(_: void) Tensor { - const x = Tensor.arange(.{ .end = 20 }, .f32).reshape(.{ 5, 4 }); - const real, const imag = splitRealImg(x, .sequential); + const x = Tensor.arange(.{ .end = 20 }, .f32).reshape(.{ .n = 5, .hd = 4 }); + const real, const imag = splitRealImg(x, .real_im_pass); const x_real = Tensor.concatenate(&.{ Tensor.arange(.{ .start = 0, .end = 20, .step = 4 }, .f32).reshape(.{ 5, 1 }), Tensor.arange(.{ .start = 1, .end = 20, .step = 4 }, .f32).reshape(.{ 5, 1 }), @@ -240,14 +941,14 @@ test "real/img" { Tensor.arange(.{ .start = 3, .end = 20, .step = 4 }, .f32).reshape(.{ 5, 1 }), }, 1); - return real.cmp(.EQ, x_real).flatten(0).convert(.i32).sum(-1).add( - imag.cmp(.EQ, x_imag).flatten(0).convert(.i32).sum(-1), + return real.cmp(.EQ, x_real).flatten().convert(.i32).sum(-1).add( + imag.cmp(.EQ, x_imag).flatten().convert(.i32).sum(-1), ); } fn testSplitSeq() Tensor { const x = Tensor.arange(.{ .end = 20 }, .f32).reshape(.{ 5, 4 }); - const real, const imag = splitRealImg(x, .sequential); + const real, const imag = splitRealImg(x, .real_im_pass); const x_real = Tensor.concatenate(&.{ Tensor.arange(.{ .start = 0, .end = 20, .step = 4 }, .f32).reshape(.{ 5, 1 }), Tensor.arange(.{ .start = 1, .end = 20, .step = 4 }, .f32).reshape(.{ 5, 1 }), @@ -257,8 +958,8 @@ test "real/img" { Tensor.arange(.{ .start = 3, .end = 20, .step = 4 }, .f32).reshape(.{ 5, 1 }), }, 1); - return real.cmp(.EQ, x_real).flatten(0).convert(.i32).sum(-1).add( - imag.cmp(.EQ, x_imag).flatten(0).convert(.i32).sum(-1), + return real.cmp(.EQ, x_real).flatten().convert(.i32).sum(-1).add( + imag.cmp(.EQ, x_imag).flatten().convert(.i32).sum(-1), ); } @@ -268,55 +969,75 @@ test "real/img" { const x_real = Tensor.arange(.{ .start = 0, .end = 20, .step = 2 }, .f32).reshape(.{ 5, 2 }); const x_imag = Tensor.arange(.{ .start = 1, .end = 20, .step = 2 }, .f32).reshape(.{ 5, 2 }); - return real.cmp(.EQ, x_real).flatten(0).convert(.i32).sum(-1).add( - imag.cmp(.EQ, x_imag).flatten(0).convert(.i32).sum(-1), + return real.cmp(.EQ, x_real).flatten().convert(.i32).sum(-1).add( + imag.cmp(.EQ, x_imag).flatten().convert(.i32).sum(-1), ); } }; + { + var exe = try platform.compileFn(std.testing.allocator, std.testing.io, Fns.testSplitMergeIsId, .{.interleaved}, .{}); + defer exe.deinit(); - const d_interleaved = try zml.testing.compileAndCall(platform, Fns.testSplitMergeIsId, .{.interleaved}); - try testing.expectEqual(20, d_interleaved.getValue(i32)); + var d_interleaved = try exe.eval(std.testing.allocator, std.testing.io, {}); + defer d_interleaved.deinit(); + try std.testing.expectEqual(20, try d_interleaved.getValue(i32, std.testing.io)); + } + { + var exe = try platform.compileFn(std.testing.allocator, std.testing.io, Fns.testSplitMergeIsId, .{.real_im_pass}, .{}); + defer exe.deinit(); - const d_sequential = try zml.testing.compileAndCall(platform, Fns.testSplitMergeIsId, .{.sequential}); - try testing.expectEqual(20, d_sequential.getValue(i32)); + var d_sequential = try exe.eval(std.testing.allocator, std.testing.io, {}); + defer d_sequential.deinit(); + try std.testing.expectEqual(20, try d_sequential.getValue(i32, std.testing.io)); + } // test the function that accepts 1 void argument - const d_split_seq_void = try zml.testing.compileAndCall(platform, Fns.testSplitSeqVoid, .{{}}); - try testing.expectEqual(20, d_split_seq_void.getValue(i32)); + { + var exe = try platform.compileFn(std.testing.allocator, std.testing.io, Fns.testSplitSeqVoid, .{{}}, .{}); + defer exe.deinit(); + + var d_split_seq_void = try exe.eval(std.testing.allocator, std.testing.io, {}); + defer d_split_seq_void.deinit(); + try std.testing.expectEqual(20, try d_split_seq_void.getValue(i32, std.testing.io)); + } // test the function that takes NO arguments - const d_split_seq = try zml.testing.compileAndCall(platform, Fns.testSplitSeq, .{}); - try testing.expectEqual(20, d_split_seq.getValue(i32)); - - // now try compiling and calling ourselves { - const mod = try zml.compileFn(std.testing.allocator, Fns.testSplitSeq, .{}, platform); - defer mod.deinit(); - const ret = mod.call(.{}); - try testing.expectEqual(20, ret.getValue(i32)); + var exe = try platform.compileFn(std.testing.allocator, std.testing.io, Fns.testSplitSeq, .{}, .{}); + defer exe.deinit(); + + var d_split_seq = try exe.eval(std.testing.allocator, std.testing.io, {}); + defer d_split_seq.deinit(); + try std.testing.expectEqual(20, try d_split_seq.getValue(i32, std.testing.io)); + } + + { + var exe = try platform.compileFn(std.testing.allocator, std.testing.io, Fns.testSplitInterleaved, .{}, .{}); + defer exe.deinit(); + + var d_split_seq = try exe.eval(std.testing.allocator, std.testing.io, {}); + defer d_split_seq.deinit(); + try std.testing.expectEqual(20, try d_split_seq.getValue(i32, std.testing.io)); } - const d_split_interleaved = try zml.testing.compileAndCall(platform, Fns.testSplitInterleaved, .{}); - try testing.expectEqual(20, d_split_interleaved.getValue(i32)); } -test "rope" { +test rope { const platform = zml.testing.env(); - const TestRope = struct { - fn forward(x: Tensor, opts: RopeOpts) Tensor { + const Local = struct { + fn _fwd(x: Tensor, opts: RopeOpts) Tensor { var input = x; { // Convert input to the requested format - const real, const imag = splitRealImg(input, .sequential); - input = mergeRealImg(real, imag, opts.impl); + const real, const imag = splitRealImg(input, .real_im_pass); + input = mergeRealImg(real, imag, opts.layout); } - const cos_sin = ropeCosSin(.{ input.dim(-2), input.dim(-1) }, input.dtype(), opts); - var res = rope(input, cos_sin, opts).squeeze(0); + var res = rope(input, null, opts).squeeze(0); { // Convert back to sequential - const real, const imag = splitRealImg(res, opts.impl); - res = mergeRealImg(real, imag, .sequential); + const real, const imag = splitRealImg(res, opts.layout); + res = mergeRealImg(real, imag, .real_im_pass); } return res; } @@ -324,20 +1045,119 @@ test "rope" { // x is made such as the interleaved and sequential reps are the same. // So the two implementations should give the same results. - const x = try zml.Buffer.fromSlice(platform, .{ 1, 5, 4 }, &[_]f32{ 1.0, 0.1, -1.0, -0.5 } ** 5); - const res1 = try zml.testing.compileAndCall(platform, TestRope.forward, .{ x, RopeOpts{ .impl = .interleaved } }); - const res2 = try zml.testing.compileAndCall(platform, TestRope.forward, .{ x, RopeOpts{ .impl = .sequential } }); + const x: zml.Tensor = .init(.{ .b = 1, .s = 5, .hd = 4 }, .f32); + var exe_interleaved = try platform.compileFn(std.testing.allocator, std.testing.io, Local._fwd, .{ x, RopeOpts{ .layout = .interleaved } }, .{}); + defer exe_interleaved.deinit(); - try zml.testing.expectClose(res1, res2, 1e-4); + var exe_sequential = try platform.compileFn(std.testing.allocator, std.testing.io, Local._fwd, .{ x, RopeOpts{ .layout = .real_im_pass } }, .{}); + defer exe_sequential.deinit(); + + const x_values: [5][4]f32 = @splat(.{ 1.0, 0.1, -1.0, -0.5 }); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&x_values)); + defer x_buffer.deinit(); + + var res1 = try exe_interleaved.eval(std.testing.allocator, std.testing.io, .{x_buffer}); + defer res1.deinit(); + var res2 = try exe_sequential.eval(std.testing.allocator, std.testing.io, .{x_buffer}); + defer res2.deinit(); + + try zml.testing.expectClose(std.testing.io, res1, res2, .{}); } -/// In neural network we generally care about the relative precision, -/// but on a given dimension, if the output is close to 0, then the precision -/// don't matter as much. -fn approxEq(comptime Float: type, l: Float, r: Float, tolerance: Float) bool { - const closeRel = std.math.approxEqRel(Float, l, r, @floatCast(tolerance)); - const closeAbs = std.math.approxEqAbs(Float, l, r, @floatCast(tolerance / 2)); - return closeRel or closeAbs; +test "rope: Proportional" { + const io = std.testing.io; + const allocator = std.testing.allocator; + const platform = zml.testing.env(); + + const x: zml.Tensor = .init(.{ .s = 5, .hd = 16 }, .f32); + var exe = try platform.compileFn( + allocator, + io, + rope, + .{ + x, + null, + .{ + .layout = .real_pass_im_pass, + .scaling = .{ .proportional = .{ .partial_rotary_factor = 0.25 } }, + }, + }, + .{}, + ); + defer exe.deinit(); + + const x_h: [5][16]f32 = @splat([_]f32{ 0.754, -0.146, 0.00443, -0.958, -0.533, 0.834, 0.555, 0.853, -0.309, 0.999, 0.983, -0.693, 0.979, 0.997, -0.359, 0.888 }); + var x_buffer: zml.Buffer = try .fromBytes(io, platform, x.shape(), @ptrCast(&x_h)); + defer x_buffer.deinit(); + + const expected_h: [5][16]f32 = .{ + .{ 0.754, -0.146, 0.00443, -0.958, -0.533, 0.834, 0.555, 0.853, -0.309, 0.999, 0.983, -0.693, 0.979, 0.997, -0.359, 0.888 }, + .{ 0.667, -0.449, 0.00443, -0.958, -0.533, 0.834, 0.555, 0.853, 0.468, 0.904, 0.983, -0.693, 0.979, 0.997, -0.359, 0.888 }, + .{ -0.0328, -0.708, 0.00443, -0.958, -0.533, 0.834, 0.555, 0.853, 0.814, 0.719, 0.983, -0.693, 0.979, 0.997, -0.359, 0.888 }, + .{ -0.703, -0.897, 0.00443, -0.958, -0.533, 0.834, 0.555, 0.853, 0.412, 0.464, 0.983, -0.693, 0.979, 0.997, -0.359, 0.888 }, + .{ -0.727, -0.997, 0.00443, -0.958, -0.533, 0.834, 0.555, 0.853, -0.369, 0.162, 0.983, -0.693, 0.979, 0.997, -0.359, 0.888 }, + }; + const expected: zml.Slice = .init(x.shape(), std.mem.sliceAsBytes(&expected_h)); + + var res = try exe.eval(allocator, io, .{ x_buffer, null }); + defer res.deinit(); + const res_h = try res.toSliceAlloc(allocator, io); + defer res_h.free(allocator); + + errdefer std.log.warn("{d:20}", .{res_h}); + try zml.testing.expectClose(io, expected, res, .{}); +} + +test "rope: Yarn with partial_rotary_factor" { + const io = std.testing.io; + const allocator = std.testing.allocator; + const platform = zml.testing.env(); + + const x: zml.Tensor = .init(.{ .s = 5, .hd = 16 }, .f32); + var exe = try platform.compileFn( + allocator, + io, + rope, + .{ + x, + null, + RopeOpts{ + .layout = .real_im_pass, + .scaling = .{ .yarn = .{ + .factor = 32.0, + .beta_fast = 32.0, + .beta_slow = 1.0, + .original_max_position_embeddings = 4096, + .truncate = true, + .rope_theta = 150_000, + .partial_rotary_factor = 0.25, + } }, + }, + }, + .{}, + ); + defer exe.deinit(); + + const x_h: [5][16]f32 = @splat([_]f32{ 0.754, -0.146, 0.00443, -0.958, -0.533, 0.834, 0.555, 0.853, -0.309, 0.999, 0.983, -0.693, 0.979, 0.997, -0.359, 0.888 }); + var x_buffer: zml.Buffer = try .fromBytes(io, platform, x.shape(), @ptrCast(&x_h)); + defer x_buffer.deinit(); + + const expected_h: [5][16]f32 = .{ + .{ 1.02, -0.197, 0.00597, -1.29, -0.533, 0.834, 0.555, 0.853, -0.309, 0.999, 0.983, -0.693, 0.979, 0.997, -0.359, 0.888 }, + .{ 0.544, -0.195, 0.858, -1.29, -0.533, 0.834, 0.555, 0.853, -0.309, 0.999, 0.983, -0.693, 0.979, 0.997, -0.359, 0.888 }, + .{ -0.428, -0.193, 0.921, -1.29, -0.533, 0.834, 0.555, 0.853, -0.309, 0.999, 0.983, -0.693, 0.979, 0.997, -0.359, 0.888 }, + .{ -1.01, -0.191, 0.137, -1.29, -0.533, 0.834, 0.555, 0.853, -0.309, 0.999, 0.983, -0.693, 0.979, 0.997, -0.359, 0.888 }, + .{ -0.659, -0.19, -0.772, -1.29, -0.533, 0.834, 0.555, 0.853, -0.309, 0.999, 0.983, -0.693, 0.979, 0.997, -0.359, 0.888 }, + }; + const expected: zml.Slice = .init(x.shape(), std.mem.sliceAsBytes(&expected_h)); + + var res = try exe.eval(allocator, io, .{ x_buffer, null }); + defer res.deinit(); + const res_h = try res.toSliceAlloc(allocator, io); + defer res_h.free(allocator); + + errdefer std.log.warn("{d:20}", .{res_h}); + try zml.testing.expectClose(io, expected, res, .{}); } pub const UpsampleMode = enum { @@ -355,8 +1175,8 @@ pub fn upsample( ) Tensor { // TODO(james): make `nearest` compatible with resizeBilinear and resizeBicubic, and wrap them here. // resize* have API which are more explicit, this assume you want to scale the N-2 last axes. - meta.assert(3 <= input.rank() and input.rank() <= 5, "upsample is only implemented for (3,4,5)-D tensors, received {}", .{input}); - meta.assert(opts.scale_factor.len == 1 or opts.scale_factor.len == input.rank() - 2, "scale factors", .{}); + stdx.debug.assert(3 <= input.rank() and input.rank() <= 5, "upsample is only implemented for (3,4,5)-D tensors, received {f}", .{input}); + stdx.debug.assert(opts.scale_factor.len == 1 or opts.scale_factor.len == input.rank() - 2, "scale factors", .{}); return switch (opts.mode) { .nearest => { var scale_factors: [3]f64 = undefined; @@ -377,21 +1197,21 @@ pub fn nearest(input: Tensor, scale_factor: []const f64) Tensor { out_shape._dims.set(i + 2, @intFromFloat(@floor(@as(f64, @floatFromInt(out_shape.dim(i + 2))) * sf))); } // TODO(james): remove this implicit two batching dims - var sd: [3]usize = undefined; + var sd: [3]u3 = undefined; var len_sd: usize = 0; for (2..input.rank()) |i| { if (input.dim(i) != out_shape.dim(i)) { - sd[len_sd] = i; + sd[len_sd] = @intCast(i); len_sd += 1; } } - const spatial_dims = sd[0..len_sd]; + const spatial_axes = sd[0..len_sd]; var res = input; - for (spatial_dims) |d| { - const n = out_shape.dim(d); - const ratio = meta.divFloat(f32, input.dim(d), n); + for (spatial_axes) |ax| { + const n = out_shape.dim(ax); + const ratio = stdx.math.divFloat(f32, input.dim(ax), n); const offsets = Tensor.arange(.{ .end = n }, .f32).addConstant(0.5).scale(ratio).floor().convert(.i32); - res = res.gatherValues(d, offsets, .{ .indices_are_sorted = true }); + res = ops.gather(res, &.{ax}, &.{offsets}, .{ .indices_are_sorted = true }); } return res; } @@ -401,21 +1221,55 @@ test nearest { // 3D Tensor (basic) { - const input_3d_basic = try zml.Buffer.fromArray(platform, [1][1][2]i32{.{.{ 1, 2 }}}); - const result = try zml.testing.compileAndCall(platform, upsample, .{ input_3d_basic, .{ .scale_factor = &.{3}, .mode = .nearest } }); - try std.testing.expectEqualSlices(i64, &.{ 1, 1, 6 }, result.dims()); - const expected: [1][1][6]i32 = .{.{.{ 1, 1, 1, 2, 2, 2 }}}; - try zml.testing.expectClose(zml.HostBuffer.fromArray(&expected), result, 0); + const input_3d_basic: zml.Tensor = .init(.{ 1, 1, 2 }, .i32); + var exe = try platform.compileFn( + std.testing.allocator, + std.testing.io, + upsample, + .{ input_3d_basic, .{ .scale_factor = &.{3}, .mode = .nearest } }, + .{}, + ); + defer exe.deinit(); + + var input_3d_basic_buffer: zml.Buffer = try .fromBytes( + std.testing.io, + platform, + input_3d_basic.shape(), + std.mem.sliceAsBytes(&[1][1][2]i32{.{.{ 1, 2 }}}), + ); + defer input_3d_basic_buffer.deinit(); + + var result = try exe.eval(std.testing.allocator, std.testing.io, .{input_3d_basic_buffer}); + defer result.deinit(); + + try std.testing.expectEqualSlices(i64, &.{ 1, 1, 6 }, result.shape().dims()); + const expected: Slice = .init(Shape.init(.{ 1, 1, 6 }, .i32), std.mem.sliceAsBytes(&[1][1][6]i32{.{.{ 1, 1, 1, 2, 2, 2 }}})); + try zml.testing.expectClose(std.testing.io, expected, result, .{}); } + // 3D Tensor (advanced) { - const input_3d_advanced = try zml.Buffer.fromArray(platform, [2][3][4]i32{ + const input_3d_advanced: zml.Tensor = .init(.{ 2, 3, 4 }, .i32); + var exe = try platform.compileFn( + std.testing.allocator, + std.testing.io, + upsample, + .{ input_3d_advanced, .{ .scale_factor = &.{2}, .mode = .nearest } }, + .{}, + ); + defer exe.deinit(); + + var input_3d_advanced_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_3d_advanced.shape(), std.mem.sliceAsBytes(&[2][3][4]i32{ .{ .{ 1, 2, 3, 4 }, .{ 5, 6, 7, 8 }, .{ 9, 10, 11, 12 } }, .{ .{ 13, 14, 15, 16 }, .{ 17, 18, 19, 20 }, .{ 21, 22, 23, 24 } }, - }); - const result = try zml.testing.compileAndCall(platform, upsample, .{ input_3d_advanced, .{ .scale_factor = &.{2}, .mode = .nearest } }); - try std.testing.expectEqualSlices(i64, &.{ 2, 3, 8 }, result.dims()); - const expected: [2][3][8]i32 = .{ + })); + defer input_3d_advanced_buffer.deinit(); + + var result = try exe.eval(std.testing.allocator, std.testing.io, .{input_3d_advanced_buffer}); + defer result.deinit(); + + try std.testing.expectEqualSlices(i64, &.{ 2, 3, 8 }, result.shape().dims()); + const expected: Slice = .init(Shape.init(.{ 2, 3, 8 }, .i32), std.mem.sliceAsBytes(&[2][3][8]i32{ .{ .{ 1, 1, 2, 2, 3, 3, 4, 4 }, .{ 5, 5, 6, 6, 7, 7, 8, 8 }, @@ -426,36 +1280,65 @@ test nearest { .{ 17, 17, 18, 18, 19, 19, 20, 20 }, .{ 21, 21, 22, 22, 23, 23, 24, 24 }, }, - }; - try zml.testing.expectClose(zml.HostBuffer.fromArray(&expected), result, 0); + })); + try zml.testing.expectClose(std.testing.io, expected, result, .{}); } + // 4D Tensor (basic) { - const input_4d_basic = try zml.Buffer.fromSlice(platform, .{ 1, 1, 2, 2 }, &[_]i32{ 1, 2, 3, 4 }); - const result = try zml.testing.compileAndCall(platform, upsample, .{ input_4d_basic, .{ .scale_factor = &.{ 3, 3 }, .mode = .nearest } }); - try std.testing.expectEqualSlices(i64, &.{ 1, 1, 6, 6 }, result.dims()); - const expected: [1][1][6][6]i32 = .{.{.{ + const input_4d_basic: zml.Tensor = .init(.{ 1, 1, 2, 2 }, .i32); + var exe = try platform.compileFn( + std.testing.allocator, + std.testing.io, + upsample, + .{ input_4d_basic, .{ .scale_factor = &.{ 3, 3 }, .mode = .nearest } }, + .{}, + ); + defer exe.deinit(); + + var input_4d_basic_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_4d_basic.shape(), std.mem.sliceAsBytes(&[_]i32{ 1, 2, 3, 4 })); + defer input_4d_basic_buffer.deinit(); + + var result = try exe.eval(std.testing.allocator, std.testing.io, .{input_4d_basic_buffer}); + defer result.deinit(); + + try std.testing.expectEqualSlices(i64, &.{ 1, 1, 6, 6 }, result.shape().dims()); + const expected: Slice = .init(Shape.init(.{ 1, 1, 6, 6 }, .i32), std.mem.sliceAsBytes(&[1][1][6][6]i32{.{.{ .{ 1, 1, 1, 2, 2, 2 }, .{ 1, 1, 1, 2, 2, 2 }, .{ 1, 1, 1, 2, 2, 2 }, .{ 3, 3, 3, 4, 4, 4 }, .{ 3, 3, 3, 4, 4, 4 }, .{ 3, 3, 3, 4, 4, 4 }, - }}}; - try std.testing.expectEqual(expected, result.getValue([1][1][6][6]i32)); + }}})); + try zml.testing.expectClose(std.testing.io, expected, result, .{}); } // 4D Tensor (advanced) { - const input_4d_advanced = try zml.Buffer.fromArray(platform, [2][2][2][2]i32{ .{ + const input_4d_advanced: zml.Tensor = .init(.{ 2, 2, 2, 2 }, .i32); + var exe = try platform.compileFn( + std.testing.allocator, + std.testing.io, + upsample, + .{ input_4d_advanced, .{ .scale_factor = &.{ 2, 2 }, .mode = .nearest } }, + .{}, + ); + defer exe.deinit(); + + var input_4d_advanced_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_4d_advanced.shape(), std.mem.sliceAsBytes(&[2][2][2][2]i32{ .{ .{ .{ 1, 2 }, .{ 3, 4 } }, .{ .{ 5, 6 }, .{ 7, 8 } }, }, .{ .{ .{ 9, 10 }, .{ 11, 12 } }, .{ .{ 13, 14 }, .{ 15, 16 } }, - } }); - const result = try zml.testing.compileAndCall(platform, upsample, .{ input_4d_advanced, .{ .scale_factor = &.{ 2, 2 }, .mode = .nearest } }); - try std.testing.expectEqualSlices(i64, &.{ 2, 2, 4, 4 }, result.dims()); - const expected: [2][2][4][4]i32 = .{ + } })); + defer input_4d_advanced_buffer.deinit(); + + var result = try exe.eval(std.testing.allocator, std.testing.io, .{input_4d_advanced_buffer}); + defer result.deinit(); + + try std.testing.expectEqualSlices(i64, &.{ 2, 2, 4, 4 }, result.shape().dims()); + const expected: Slice = .init(Shape.init(.{ 2, 2, 4, 4 }, .i32), std.mem.sliceAsBytes(&[2][2][4][4]i32{ .{ .{ .{ 1, 1, 2, 2 }, @@ -484,15 +1367,29 @@ test nearest { .{ 15, 15, 16, 16 }, }, }, - }; - try zml.testing.expectClose(zml.HostBuffer.fromArray(&expected), result, 0); + })); + try zml.testing.expectClose(std.testing.io, expected, result, .{}); } // 5D Tensor (basic) { - const input_5d = try zml.Buffer.fromSlice(platform, .{ 1, 1, 1, 2, 2 }, &[_]i32{ 1, 2, 3, 4 }); - const result = try zml.testing.compileAndCall(platform, upsample, .{ input_5d, .{ .scale_factor = &.{2}, .mode = .nearest } }); - try std.testing.expectEqualSlices(i64, &.{ 1, 1, 2, 4, 4 }, result.dims()); - const expected: [1][1][2][4][4]i32 = .{ + const input_5d: zml.Tensor = .init(.{ 1, 1, 1, 2, 2 }, .i32); + var exe = try platform.compileFn( + std.testing.allocator, + std.testing.io, + upsample, + .{ input_5d, .{ .scale_factor = &.{2}, .mode = .nearest } }, + .{}, + ); + defer exe.deinit(); + + var input_5d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_5d.shape(), std.mem.sliceAsBytes(&[_]i32{ 1, 2, 3, 4 })); + defer input_5d_buffer.deinit(); + + var result = try exe.eval(std.testing.allocator, std.testing.io, .{input_5d_buffer}); + defer result.deinit(); + + try std.testing.expectEqualSlices(i64, &.{ 1, 1, 2, 4, 4 }, result.shape().dims()); + const expected: Slice = .init(Shape.init(.{ 1, 1, 2, 4, 4 }, .i32), std.mem.sliceAsBytes(&[1][1][2][4][4]i32{ .{ .{ .{ @@ -509,8 +1406,8 @@ test nearest { }, }, }, - }; - try zml.testing.expectClose(zml.HostBuffer.fromArray(&expected), result, 0); + })); + try zml.testing.expectClose(std.testing.io, expected, result, .{}); } } @@ -533,7 +1430,7 @@ pub fn resizeBilinear(image: Tensor, resized_axes: anytype, opt: ResizeOpts) Ten for (new_size.constSlice(), tags_.constSlice()) |d, t| { const ax = image.shape().axis(t); const child_opt: ResizeOpts = .{ - .original_len = if (opt.original_len) |o| o.choose1d(0, ax) else null, + .original_len = if (opt.original_len) |o| o.slice(0, .single(ax)) else null, }; out = resizeLinear1d(out, ax, d, child_opt); } @@ -544,18 +1441,22 @@ test resizeBilinear { const platform = zml.testing.env(); // Only test shapes - var comp = try zml.module.CompilationContext.init(std.heap.page_allocator, "test", platform); + var comp = zml.Compiler.init(std.testing.allocator, std.testing.io, platform, .{}); defer comp.deinit(); comp.activate(); defer comp.deactivate(); + const block = @import("mlir").Block.init(&.{}, &.{}); + const scope = comp.pushBlock(block); + defer scope.pop(); + inline for (.{ .{ .{ .a = 10, .b = 10 }, .{ .a = 20 }, .{ .a = 20, .b = 10 } }, .{ .{ .a = 10, .b = 10 }, .{ .b = 5 }, .{ .a = 10, .b = 5 } }, .{ .{ .a = 10, .b = 10 }, .{ .a = 20, .b = 5 }, .{ .a = 20, .b = 5 } }, }) |testcase| { const x_shape, const resizing, const res_shape = testcase; - const x = Tensor.constant(x_shape, .{ .f16 = 0 }); + const x = Tensor.constant(.{ .f16 = 0 }).broad(Shape.init(x_shape, .f16)); const y = resizeBilinear(x, resizing, .{}); try zml.testing.expectEqualShapes(Shape.init(res_shape, .f16), y.shape()); try std.testing.expect(y.value().owner().verify()); @@ -563,11 +1464,12 @@ test resizeBilinear { } pub fn resizeLinear1d(image: Tensor, axis: i8, new_len: u63, opt: ResizeOpts) Tensor { - const res_shape = image.shape().set(axis, new_len); + const ax = image.axis(axis); + const res_shape = image.shape().set(ax, new_len); const dtype = opt.precision orelse if (image.dtype().class() == .integer) .f32 else image.dtype(); - const og_len = opt.original_len orelse Tensor.scalar(image.dim(axis), dtype); - const ratio = og_len.convert(dtype).scale(meta.divFloat(f32, 1, new_len)); + const og_len = opt.original_len orelse Tensor.scalar(image.dim(ax), dtype); + const ratio = og_len.convert(dtype).scale(stdx.math.divFloat(f32, 1, new_len)); const scaled = Tensor.arange(.{ .end = new_len }, dtype).mul(ratio); const left = scaled.floor(); const right = left.addConstant(1); @@ -575,11 +1477,11 @@ pub fn resizeLinear1d(image: Tensor, axis: i8, new_len: u63, opt: ResizeOpts) Te // TODO: check that two gather isn't too bad perf wise. // Normally we should use gatherSlices to collect the values 2 by 2, // but gatherSlices messes up with the order of axes. - const left_val = image.gatherValues(axis, left.convert(.i32), .{ .indices_are_sorted = true }).convert(dtype); - const right_val = image.gatherValues(axis, right.convert(.i32), .{ .indices_are_sorted = true }).convert(dtype); + const left_val = ops.gather(image, &.{ax}, &.{left.convert(.i32)}, .{ .indices_are_sorted = true }).convert(dtype); + const right_val = ops.gather(image, &.{ax}, &.{right.convert(.i32)}, .{ .indices_are_sorted = true }).convert(dtype); - const left_weight = right.sub(scaled).broadcast(res_shape, &.{axis}); - const right_weight = scaled.sub(left).broadcast(res_shape, &.{axis}); + const left_weight = right.sub(scaled).broadcast(res_shape, &.{ax}); + const right_weight = scaled.sub(left).broadcast(res_shape, &.{ax}); const res = left_val.mul(left_weight).add(right_val.mul(right_weight)); return res.convert(image.dtype()).withTags(image.shape().tags()); @@ -595,7 +1497,7 @@ pub fn resizeBicubic(image: Tensor, resized_axes: anytype, opt: ResizeOpts) Tens for (new_size.constSlice(), tags_.constSlice()) |d, t| { const ax = image.shape().axis(t); const child_opt: ResizeOpts = .{ - .original_len = if (opt.original_len) |o| o.choose1d(0, ax) else null, + .original_len = if (opt.original_len) |o| o.slice(0, .single(ax)) else null, }; out = resizeCubic1d(out, ax, d, child_opt); } @@ -606,18 +1508,22 @@ test resizeBicubic { const platform = zml.testing.env(); // Only test shapes - var comp = try zml.module.CompilationContext.init(std.heap.page_allocator, "test", platform); + var comp = zml.Compiler.init(std.testing.allocator, std.testing.io, platform, .{}); defer comp.deinit(); comp.activate(); defer comp.deactivate(); + const block = @import("mlir").Block.init(&.{}, &.{}); + const scope = comp.pushBlock(block); + defer scope.pop(); + inline for (.{ .{ .{ .a = 10, .b = 10 }, .{ .a = 20 }, .{ .a = 20, .b = 10 } }, .{ .{ .a = 10, .b = 10 }, .{ .b = 5 }, .{ .a = 10, .b = 5 } }, .{ .{ .a = 10, .b = 10 }, .{ .a = 20, .b = 5 }, .{ .a = 20, .b = 5 } }, }) |testcase| { const x_shape, const resizing, const res_shape = testcase; - const x = Tensor.constant(x_shape, .{ .f16 = 0 }); + const x = Tensor.constant(.{ .f16 = 0 }).broad(Shape.init(x_shape, .f16)); const y = resizeBicubic(x, resizing, .{}); try zml.testing.expectEqualShapes(Shape.init(res_shape, .f16), y.shape()); try std.testing.expect(y.value().owner().verify()); @@ -629,14 +1535,14 @@ pub fn resizeCubic1d(image: Tensor, axis: i8, new_len: u63, opt: ResizeOpts) Ten const dtype = opt.precision orelse if (image.dtype().class() == .integer) .f32 else image.dtype(); const og_len = opt.original_len orelse Tensor.scalar(image.dim(axis), dtype); - const ratio = og_len.convert(dtype).scale(meta.divFloat(f32, 1, new_len)); + const ratio = og_len.convert(dtype).scale(stdx.math.divFloat(f32, 1, new_len)); const scaled = Tensor.arange(.{ .end = new_len }, dtype).mul(ratio); const t = scaled.sub(scaled.floor()); const pos = Tensor.stack(&.{ - Tensor.constant(t.shape(), dtype.one()), + Tensor.constant(dtype.one()).broad(t.shape()), t, t.mul(t), - t.pow(Tensor.scalar(3, dtype)), + t.powByConst(3), }, .last, ._interpolated); std.debug.assert(pos.dim(0) == new_len); @@ -656,7 +1562,7 @@ pub fn resizeCubic1d(image: Tensor, axis: i8, new_len: u63, opt: ResizeOpts) Ten .{ 1, -2.5, 2, -0.5 }, .{ -0.5, 1.5, -1.5, 0.5 }, }; - const weights = zml.Tensor.constantTensor(zml.HostBuffer.fromArray(&weights_)).convert(dtype).withTags(.{ ._interpolated, ._neighbors }); + const weights = zml.Tensor.constantTensor(Shape.init(.{ 4, 4 }, .f32), std.mem.sliceAsBytes(&weights_)).convert(dtype).withTags(.{ ._interpolated, ._neighbors }); // actually do the interpolation. // Note: ideally this matmul should be inlined with the gather, but that's currently not the case. @@ -684,11 +1590,11 @@ pub fn causalAttnMask( attn_window_len: ?u32, ) Tensor { const attn_shape = Shape.init(attn_shape_, dtype); - meta.assert(attn_shape.rank() == 2, "causalAttnMask({}) shape need to be exactly 2 axes", .{attn_shape}); + stdx.debug.assert(attn_shape.rank() == 2, "causalAttnMask({f}) shape need to be exactly 2 axes", .{attn_shape}); const qlen = attn_shape.dim(-2); - const q_idx = Tensor.iota(attn_shape, .i32, -2); + const q_idx = Tensor.iota(attn_shape, -2); const klen = attn_shape.dim(-1); - const k_idx = Tensor.iota(attn_shape, .i32, -1); + const k_idx = Tensor.iota(attn_shape, -1); // all elements > main diagonal must be 0 // (q_idx - window_len < k_idx <= q_idx) @@ -701,10 +1607,7 @@ pub fn causalAttnMask( } if (dtype.isFloat()) { - meta.guard(dtype.isFloat(), @src()); // -inf only exists for floats - const zeros = Tensor.constant(mask.shape(), dtype.zero()); - const minus_inf = Tensor.constant(mask.shape(), dtype.minValue()); - mask = Tensor.select(mask, zeros, minus_inf); + mask = Tensor.select(mask, .scalar(0, dtype), .scalar(-std.math.inf(f32), dtype)); } else { mask = mask.convert(dtype); } @@ -715,10 +1618,7 @@ pub fn causalAttnMask( pub const SdpaOpts = struct { attn_mask: ?Tensor = null, scale: ?Tensor = null, - bias: ?Tensor = null, - allow_cudnn: bool = true, - // TODO: put a callback instead of all this field, - // so that + softmax_bias: ?Tensor = null, }; /// Scaled dot product attention. @@ -738,232 +1638,339 @@ pub const SdpaOpts = struct { pub fn sdpa(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) Tensor { var q, var k, var v = .{ q_, k_, v_ }; - const err_template = "sdpa(q: {}, k: {}, v: {}, attn: {?}) is invalid ! "; + const err_template = "sdpa(q: {f}, k: {f}, v: {f}, attn: {?f}) is invalid ! "; const err_args = .{ q, k, v, opts.attn_mask }; - meta.assert(q.shape().hasTags(.{ .h, .q, .hd }), err_template ++ "q is missing tags {{.h, .q, .hd}}", err_args); - meta.assert(k.shape().hasTags(.{ .h, .k, .hd }), err_template ++ "k is missing tags {{.h, .k, .hd}}", err_args); - meta.assert(v.shape().hasTags(.{ .h, .k, .hd }), err_template ++ "v is missing tags {{.h, .k, .hd}}", err_args); + stdx.debug.assert(q.shape().hasTags(.{ .h, .q, .hd }), err_template ++ "q is missing tags {{.h, .q, .hd}}", err_args); + stdx.debug.assert(k.shape().hasTags(.{ .h, .k, .hd }), err_template ++ "k is missing tags {{.h, .k, .hd}}", err_args); + stdx.debug.assert(v.shape().hasTags(.{ .h, .k, .hd }), err_template ++ "v is missing tags {{.h, .k, .hd}}", err_args); - if (opts.allow_cudnn and cuda.canUseCudnnSdpa(q.dim(.hd), q.dtype())) { - return cuda.sdpa(q, k, v, opts); - } - - if (q.dim(.h) != k.dim(.h)) { - meta.assert(@mod(q.dim(.h), k.dim(.h)) == 0, err_template ++ "Different number of heads for keys and queries, but can't repeat keys.", err_args); - // Note: we don't try to repeat queries. - // Repeating keys is the interesting optimisation cause it reduces KV cache memory usage. - const num_rep: u63 = @intCast(@divExact(q.dim(.h), k.dim(.h))); - k, v = .{ k.repeat1d(.h, num_rep), v.repeat1d(.h, num_rep) }; - } + // Handle different numbers of head by splitting q heads. + // This is a bit error prone in the sense that it depends of the layout of q heads. + // This is the Llama convention though. + q = q.splitAxis(.h, .{ .h = k.dim(.h), .hq = .auto }); const attn_mask = if (opts.attn_mask) |m| m else null; - const dims = helpers.collectDims(.{ .h, .q, .k, .hd }, &.{ q, k, v, attn_mask }, .strict) catch { - meta.panic(err_template ++ "Inputs have incompatible shapes.", err_args); + const dims = collectDims(.{ .h, .q, .k, .hd }, &.{ q, k, v, attn_mask }, .strict) catch { + stdx.debug.panic(err_template ++ "Inputs have incompatible shapes.", err_args); }; - const sqrtHeadDim: f16 = 1.0 / std.math.sqrt(@as(f16, @floatFromInt(dims.hd))); - const scale_logit = if (opts.scale) |s| s else Tensor.scalar(sqrtHeadDim, .f16); - k = k.mul(scale_logit.convert(k.dtype())); - - var attn_weights = q.dot(k, .{.hd}); - // log.debug("attn_weights : {}", .{attn_weights}); - // log.debug("attn_mask : {?}", .{attn_mask}); - if (attn_mask) |mask| attn_weights = attn_weights.add(mask.broadcastLeft(attn_weights.shape())); + const sqrtHeadDim: f32 = 1.0 / std.math.sqrt(@as(f32, @floatFromInt(dims.hd))); + const head_scaling = if (opts.scale) |s| s else Tensor.scalar(sqrtHeadDim, k.dtype()); + k = k.mul(head_scaling.convert(k.dtype())); + var attn_weights = q.dot(k, .hd); + if (attn_mask) |mask| attn_weights = attn_weights.add(mask.broad(attn_weights.shape())); attn_weights = attn_weights.convert(.f32); - if (opts.bias) |bias| { - attn_weights = attn_weights.add(bias); - } - attn_weights = attn_weights.softmax(.k).convert(q.dtype()); + attn_weights = if (opts.softmax_bias) |softmax_bias| attn: { + // The split is needed because we also split q ourselves. + // TODO: consider letting the user do that. + const bias = softmax_bias.splitAxis(.h, .{ .h = k.dim(.h), .hq = .auto }); + break :attn attn_weights.convert(.f32).softmaxBiased(.k, bias).convert(q.dtype()); + } else attn_weights.convert(.f32).softmax(.k).convert(q.dtype()); - var attn = attn_weights.dot(v, .{.k}); - return attn.transpose(q.shape()); + var attn = attn_weights.dot(v, .k); + return attn.transpose(q.shape()).merge(.{ .h = .{ .h, .hq } }); } -pub const MemEfficientOps = struct { - scale: ?f32 = null, - query_chunk_size: u32, - key_chunk_size: u32, - opts: SdpaOpts = .{}, -}; +pub const GatedDeltaNet = struct { + /// Query tensor .{ .s, .h, .k }. + queries: Tensor, + /// Key tensor .{ .s, .h, .k }. + keys: Tensor, + /// Value tensor .{ .s, .h, .v }. + values: Tensor, + /// Forget gate .{ .s, .h }. + alphas: Tensor, + /// Delta gate .{ .s, .h }. + betas: Tensor, + /// Number of sequence positions to process, clamped to the input length. + active_length: Tensor, -pub fn sdpaMemEfficient(q_: Tensor, k_: Tensor, v_: Tensor, opts: MemEfficientOps) Tensor { - const q = q_.withTags(.{ .b, .hq, .sq, .hd }); - const k = k_.withTags(.{ .b, .hk, .sk, .hd }); - const v = v_.withTags(.{ .b, .hk, .sk, .hd }); - var sdpa_opts = opts.opts; - if (sdpa_opts.attn_mask) |*attn_mask| attn_mask.* = attn_mask.withTags(.{ .sq, .sk }); + pub const State = struct { + /// Per-head recurrent state with shape .{ .h, .v, .k }. + s: Tensor, + /// Sequence of outputs with shape .{ .s, .h, .v }. + outputs: Tensor, - const sdpa_mem_efficient: SdpaMemEfficient = .{ .q = q, .k = k, .v = v, .opt = .{ - .query_chunk_size = @intCast(@min(q.dim(.sq), opts.query_chunk_size)), - .key_chunk_size = @intCast(@min(k.dim(.sk), opts.key_chunk_size)), - .scale = opts.scale, - .opts = sdpa_opts, - } }; - - // TODO(Corentin): Maybe `withTags` could take a Shape to copy from. - var result = sdpa_mem_efficient.forward(); - result._shape = q_.shape(); - return result; -} - -const SdpaMemEfficient = struct { - q: Tensor, - k: Tensor, - v: Tensor, - opt: MemEfficientOps, - - fn forward(self: SdpaMemEfficient) Tensor { - const n_q_chunks = @divExact(self.q.dim(.sq), self.opt.query_chunk_size); - const res = ops.for_(SdpaMemEfficient.nextQueriesChunk, self, .{ .nq = n_q_chunks }); - // TODO: should "for_" operate on an axis ? - // res: (nq, b, nh, qlen / nq, dim) -> (b, nh, qlen, dim) - return res.transpose(.{ 1, 2, 0, 3, 4 }).flatten(2); - // return res.transpose(.{ .b, .hq, .nq, .sq, .hd }).merge(.{ .nq, .sq }, .sq); - } - - fn nextQueriesChunk(self: SdpaMemEfficient, idx: Tensor) Tensor { - const offset = idx.scale(self.opt.query_chunk_size); - const q_chunk = self.q.dynamicSlice(.{ .sq = .{ .start = offset, .len = self.opt.query_chunk_size } }); - const attn_chunk = if (self.opt.opts.attn_mask) |attn_mask| attn_mask.dynamicSlice1d(0, self.opt.query_chunk_size, offset) else null; - - var chunk: SdpaMemEfficient = self; - chunk.q = q_chunk; - chunk.opt.opts.attn_mask = attn_chunk; - return chunk.scanKeyVal(); - } - - fn scanKeyVal(self: SdpaMemEfficient) Tensor { - const n_chunks = @divExact(self.k.dim(.sk), self.opt.key_chunk_size); - const res = ops.for_(SdpaMemEfficient.nextKeyValChunk, self, .{ .k_chunk = n_chunks }); - const global_max = res.max_value.max(.k_chunk).broad(res.max_value.shape()); - const max_diffs = res.max_value.sub(global_max).exp(); - const attn = res.attn.mul(max_diffs.broad(res.attn.shape())).sum(.k_chunk).squeeze(.k_chunk); - const exp_sum = res.exp_sum.mul(max_diffs.convert(.f32)).sum(.k_chunk).squeeze(.k_chunk).convert(attn.dtype()); - return attn.div(exp_sum.broad(self.q.shape())); - } - - fn nextKeyValChunk(self: SdpaMemEfficient, idx: Tensor) PartialAttn { - const offset = idx.scale(self.opt.key_chunk_size); - const k_chunk = self.k.dynamicSlice(.{ .sk = .{ .start = offset, .len = self.opt.key_chunk_size } }); - const v_chunk = self.v.dynamicSlice(.{ .sk = .{ .start = offset, .len = self.opt.key_chunk_size } }); - const attn_chunk = if (self.opt.opts.attn_mask) |mask| mask.dynamicSlice1d(1, self.opt.key_chunk_size, offset) else null; - - return sdpaChunk(self.q, k_chunk, v_chunk, .{ .attn_mask = attn_chunk }); - } -}; - -pub const PartialAttn = struct { - attn: Tensor, - exp_sum: Tensor, - max_value: Tensor, -}; - -/// Compute softmax over a chunk. -/// Returns intermediary results to allow aggregating later. -pub fn partialSoftmax(self: Tensor, axis: anytype) PartialAttn { - const a = self.axis(axis); - const max_val = self.max(a); - const out = self.sub(max_val.broad(self.shape())).exp(); - return .{ - .attn = out, - .exp_sum = out.convert(.f32).sum(a).squeeze(a), - .max_value = max_val.squeeze(a), + /// Current step, scalar + step: Tensor, }; -} -/// Compute sdpa on a chunk, and computes a partial softmax. -/// q: (B, H, Sq, H_dim) ⊙ k: (B, H, Sk, H_dim) -> qk: (B, H, Sq, Sk) -fn sdpaChunk(q: Tensor, k: Tensor, v: Tensor, opts: SdpaOpts) PartialAttn { - // const bs, const num_head, const sk, const h_dim = q.dims[0..4]; - // TODO: rewrite using modern ZML + pub const Input = struct { + s: Tensor, + }; - // If we have more query heads (hq) than key heads (hk), repeat keys. - const k_rep, const v_rep = if (q.dim(.hq) != k.dim(.hk)) blk: { - const num_rep: u63 = @intCast(@divExact(q.dim(.hq), k.dim(.hk))); - break :blk .{ k.repeat1d(0, num_rep).rename(.{ .hk = .hq }), v.repeat1d(0, num_rep).rename(.{ .hk = .hq }) }; - } else .{ k.rename(.{ .hk = .hq }), v.rename(.{ .hk = .hq }) }; + pub const Output = struct { + /// Per-head recurrent state with shape .{ .h, .v, .k }. + state: Input, + /// Sequence of outputs with shape .{ .s, .h, .v }. + outputs: Tensor, + }; - var qk = q.dot(k_rep, .{.hd}); - - const sqrtHeadDim: f32 = 1.0 / std.math.sqrt(@as(f32, @floatFromInt(q.dim(.hd)))); - qk = qk.scale(sqrtHeadDim); - - std.debug.assert(qk.rank() == q.rank()); - if (opts.attn_mask) |mask| { - qk = qk.add(mask.broad(qk.shape())); + pub fn cond(gdn: GatedDeltaNet, state: State) Tensor { + const active_length = gdn.active_length + .convert(.i32) + .maximum(.scalar(0, .i32)) + .minimum(.scalar(gdn.queries.dim(.s), .i32)); + return state.step.cmp(.LT, active_length); } - const partial = partialSoftmax(qk, -1); - const attn = partial.attn.dot(v_rep, .{.sk}); + /// Single-step recurrent update for Gated Delta Net. + pub fn body(gdn: GatedDeltaNet, state: State) State { + // q, k: .{ .h, .k } + const q = sliceStep(gdn.queries, state.step); + const k = sliceStep(gdn.keys, state.step); + // v: .{ .h, .v } + const v = sliceStep(gdn.values, state.step); + // alpha, beta: .{ .h } + const alpha = sliceStep(gdn.alphas, state.step); + const beta = sliceStep(gdn.betas, state.step); - return .{ - .attn = attn, - .exp_sum = partial.exp_sum, - .max_value = partial.max_value, - }; -} -test "sdpaMemEfficient without mask" { + const hvk = state.s.shape(); + const hv = hvk.drop(.k); + + const v_hat = state.s.dot(k, .k).mul(alpha.broad(hv)); + const delta = v.sub(v_hat).mul(beta.broad(hv)); + const delta_k = delta.broad(hvk).mul(k.broad(hvk)); + const s_new = state.s.mul(alpha.broad(hvk)).add(delta_k); + const y = s_new.dot(q, .k); + + return .{ + .s = s_new, + .outputs = state.outputs.dynamicUpdateSlice(.{ .s = state.step }, y), + .step = state.step.addConstant(1), + }; + } + + fn sliceStep(input: Tensor, step_: Tensor) Tensor { + return input.slice(.s, .dynSingle(step_)); + } + + fn validateInitialState(gdn: GatedDeltaNet, state: State) void { + const err_template = "GatedDeltaNet.step(state: {f}, q: {f}, k: {f}, v: {f}, alpha: {f}, beta: {f}) is invalid ! "; + const err_args = .{ state.s, gdn.queries, gdn.keys, gdn.values, gdn.alphas, gdn.betas }; + + stdx.debug.assert(state.s.shape().hasTags(.{ .h, .v, .k }), err_template ++ "state.s is missing tags {{.h, .v, .k}}", err_args); + stdx.debug.assert(gdn.queries.shape().hasTags(.{ .s, .h, .k }), err_template ++ "q is missing tags {{.h, .k}}", err_args); + stdx.debug.assert(gdn.keys.shape().hasTags(.{ .s, .h, .k }), err_template ++ "k is missing tags {{.h, .k}}", err_args); + stdx.debug.assert(gdn.values.shape().hasTags(.{ .s, .h, .v }), err_template ++ "v is missing tags {{.h, .v}}", err_args); + stdx.debug.assert(gdn.alphas.shape().hasTags(.{ .s, .h }), err_template ++ "alphas is missing tag {{.h}}", err_args); + stdx.debug.assert(gdn.betas.shape().hasTags(.{ .s, .h }), err_template ++ "betas is missing tag {{.h}}", err_args); + stdx.debug.assert(gdn.active_length.rank() == 0 and gdn.active_length.dtype().isInteger(), err_template ++ "active_length must be an integer scalar", err_args); + + _ = collectDims(.{.h}, &.{ state.s, gdn.queries, gdn.keys, gdn.values, gdn.alphas, gdn.betas }, .strict) catch { + stdx.debug.panic(err_template ++ "head dimensions are inconsistent.", err_args); + }; + _ = collectDims(.{.k}, &.{ state.s, gdn.queries, gdn.keys }, .strict) catch { + stdx.debug.panic(err_template ++ "key dimensions are inconsistent.", err_args); + }; + _ = collectDims(.{.v}, &.{ state.s, gdn.values }, .strict) catch { + stdx.debug.panic(err_template ++ "value dimensions are inconsistent.", err_args); + }; + } + + /// Processes a sequence with Gated Delta Net recurrence using `stablehlo.while`. + /// + /// Shapes: + /// - `queries`, `keys`: .{ .s, .h, .k } + /// - `values`, `outputs`: .{ .s, .h, .v } + /// - `alphas`, `betas`: .{ .s, .h } + /// - `initial_state.s`: .{ .h, .v, .k } + /// - `active_length`: integer scalar + pub fn forward( + queries: Tensor, + keys: Tensor, + values: Tensor, + alphas: Tensor, + betas: Tensor, + initial_state: Input, + active_length: Tensor, + ) Output { + const gdn: GatedDeltaNet = .{ + .queries = queries, + .keys = keys, + .values = values, + .alphas = alphas, + .betas = betas, + .active_length = active_length, + }; + const state: GatedDeltaNet.State = .{ + .step = .scalar(0, .i32), + .s = initial_state.s, + .outputs = .zeroes(values.shape()), + }; + validateInitialState(gdn, state); + + const final_state = if (queries.dim(.s) == 1) blk: { + // avoids a `stablehlo.while`, which costs a host sync per call on GPU. + const next = gdn.body(state); + const run = gdn.cond(state); + break :blk GatedDeltaNet.State{ + .s = run.broad(state.s.shape()).select(next.s, state.s), + .outputs = run.broad(state.outputs.shape()).select(next.outputs, state.outputs), + .step = run.select(next.step, state.step), + }; + } else ops.@"while"(GatedDeltaNet, gdn, state); + + return .{ + .outputs = final_state.outputs, + .state = .{ .s = final_state.s }, + }; + } +}; + +test "gated delta net" { const platform = zml.testing.env(); - const allocator = std.testing.allocator; - // Note we use small input vectors to have the tests run reasonably fast, - // but don't expect speed ups with this small sizes. - const rng = try zml.compileFn(allocator, Tensor.Rng.normal, .{ Shape.init(.{ 1, 10, 512, 64 }, .f32), .{ .mean = 0, .stddev = 1 } }, platform); - defer rng.deinit(); + const queries: zml.Tensor = .init(.{ .s = 2, .h = 2, .k = 2 }, .f32); + const keys: zml.Tensor = .init(.{ .s = 2, .h = 2, .k = 2 }, .f32); + const values: zml.Tensor = .init(.{ .s = 2, .h = 2, .v = 2 }, .f32); + const alphas: zml.Tensor = .init(.{ .s = 2, .h = 2 }, .f32); + const betas: zml.Tensor = .init(.{ .s = 2, .h = 2 }, .f32); + const initial_s: zml.Tensor = .init(.{ .h = 2, .v = 2, .k = 2 }, .f32); + const active_length: zml.Tensor = .init(.{}, .u32); - // Note: it's fine to pass undefined here, cause the arguments have already been baked into the executable. - const q = rng.call(undefined); - const k = rng.call(undefined); - const v = rng.call(undefined); + var exe = try platform.compileFn( + std.testing.allocator, + std.testing.io, + GatedDeltaNet.forward, + .{ queries, keys, values, alphas, betas, .{ .s = initial_s }, active_length }, + .{}, + ); + defer exe.deinit(); - const ref_res = try zml.testing.compileAndCallWithTensors(platform, sdpa, .{ - q.shape().withTags(.{ .b, .h, .q, .hd }), - k.shape().withTags(.{ .b, .h, .k, .hd }), - v.shape().withTags(.{ .b, .h, .k, .hd }), - .{ .attn_mask = null, .scale = null, .bias = null }, - }, .{ q, k, v, undefined }); - try std.testing.expectEqualSlices(i64, q.shape().dims(), ref_res.shape().dims()); - - const opts: zml.ShapeOf(MemEfficientOps) = .{ .query_chunk_size = 256, .key_chunk_size = 128, .opts = .{ .attn_mask = null, .scale = null, .bias = null } }; - const res = try zml.testing.compileAndCallWithTensors( + var queries_buffer: zml.Buffer = try .fromBytes( + std.testing.io, platform, - sdpaMemEfficient, - .{ q.shape(), k.shape(), v.shape(), opts }, - .{ q, k, v, undefined }, + queries.shape(), + std.mem.sliceAsBytes(&[2][2][2]f32{ + .{ .{ 1.0, 0.0 }, .{ 0.0, 1.0 } }, + .{ .{ 1.0, 1.0 }, .{ 1.0, -1.0 } }, + }), + ); + defer queries_buffer.deinit(); + var keys_buffer: zml.Buffer = try .fromBytes( + std.testing.io, + platform, + keys.shape(), + std.mem.sliceAsBytes(&[2][2][2]f32{ + .{ .{ 1.0, 2.0 }, .{ 0.0, 1.0 } }, + .{ .{ 2.0, 1.0 }, .{ 1.0, 0.0 } }, + }), + ); + defer keys_buffer.deinit(); + var values_buffer: zml.Buffer = try .fromBytes( + std.testing.io, + platform, + values.shape(), + std.mem.sliceAsBytes(&[2][2][2]f32{ + .{ .{ 3.0, 1.0 }, .{ 2.0, 4.0 } }, + .{ .{ 1.0, 5.0 }, .{ 3.0, 0.0 } }, + }), + ); + defer values_buffer.deinit(); + var alphas_buffer: zml.Buffer = try .fromBytes( + std.testing.io, + platform, + alphas.shape(), + std.mem.sliceAsBytes(&[2][2]f32{ + .{ 0.5, 0.25 }, + .{ 0.8, 0.6 }, + }), + ); + defer alphas_buffer.deinit(); + var betas_buffer: zml.Buffer = try .fromBytes( + std.testing.io, + platform, + betas.shape(), + std.mem.sliceAsBytes(&[2][2]f32{ + .{ 1.0, 0.5 }, + .{ 0.75, 1.0 }, + }), + ); + defer betas_buffer.deinit(); + var initial_s_buffer: zml.Buffer = try .fromBytes( + std.testing.io, + platform, + initial_s.shape(), + std.mem.sliceAsBytes(&[2][2][2]f32{ + .{ .{ 1.0, 0.0 }, .{ 0.0, 1.0 } }, + .{ .{ 2.0, 1.0 }, .{ 1.0, 0.0 } }, + }), + ); + defer initial_s_buffer.deinit(); + var oversized_length_buffer = try zml.Buffer.scalar(std.testing.io, platform, @as(u32, 3), .u32); + defer oversized_length_buffer.deinit(); + + var result: zml.Bufferized(GatedDeltaNet.Output) = undefined; + var runner = try exe.runner(std.testing.allocator); + defer runner.deinit(std.testing.allocator); + runner.run( + std.testing.io, + .{ queries_buffer, keys_buffer, values_buffer, alphas_buffer, betas_buffer, .{ .s = initial_s_buffer }, oversized_length_buffer }, + .{&result}, + .{ .wait = true }, + ); + defer result.outputs.deinit(); + defer result.state.s.deinit(); + + const expected_outputs: zml.Slice = .init( + zml.Shape.init(.{ 2, 2, 2 }, .f32), + std.mem.sliceAsBytes(&[2][2][2]f32{ + .{ .{ 3.0, 0.0 }, .{ 1.125, 2.0 } }, + .{ .{ -11.15, 10.75 }, .{ 2.325, -1.2 } }, + }), + ); + const expected_final_s: zml.Slice = .init( + zml.Shape.init(.{ 2, 2, 2 }, .f32), + std.mem.sliceAsBytes(&[2][2][2]f32{ + .{ .{ -9.3, -1.85 }, .{ 6.9, 3.85 } }, + .{ .{ 3.0, 0.675 }, .{ 0.0, 1.2 } }, + }), ); - try zml.testing.expectClose(ref_res, res, 2e-3); -} + try zml.testing.expectClose(std.testing.io, expected_outputs, result.outputs, .{ + .absolute_tolerance = 1e-4, + .relative_tolerance = 1e-4, + }); + try zml.testing.expectClose(std.testing.io, expected_final_s, result.state.s, .{ + .absolute_tolerance = 1e-4, + .relative_tolerance = 1e-4, + }); -test "sdpaMemEfficient with mask" { - const platform = zml.testing.env(); - const allocator = std.testing.allocator; + var prefix_length_buffer = try zml.Buffer.scalar(std.testing.io, platform, @as(u32, 1), .u32); + defer prefix_length_buffer.deinit(); + var prefix_result: zml.Bufferized(GatedDeltaNet.Output) = undefined; + runner.run( + std.testing.io, + .{ queries_buffer, keys_buffer, values_buffer, alphas_buffer, betas_buffer, .{ .s = initial_s_buffer }, prefix_length_buffer }, + .{&prefix_result}, + .{ .wait = true }, + ); + defer prefix_result.outputs.deinit(); + defer prefix_result.state.s.deinit(); - // Note we use small input vectors to have the tests run reasonably fast, - // but don't expect speed ups with this small sizes. - const rng = try zml.compileFn(allocator, Tensor.Rng.normal, .{ Shape.init(.{ 1, 10, 512, 64 }, .f32), .{ .mean = 0, .stddev = 1 } }, platform); - defer rng.deinit(); - - const rng_mask = try zml.compileFn(allocator, Tensor.Rng.normal, .{ Shape.init(.{ 512, 512 }, .f32), .{ .mean = 0, .stddev = 1 } }, platform); - defer rng_mask.deinit(); - - // Note: it's fine to pass undefined here, cause the arguments have already been backed into the executable. - const q = rng.call(undefined); - const k = rng.call(undefined); - const v = rng.call(undefined); - const mask = rng_mask.call(undefined); - - const ref_res = try zml.testing.compileAndCall(platform, sdpa, .{ q.withTags(.{ .b, .h, .q, .hd }), k.withTags(.{ .b, .h, .k, .hd }), v.withTags(.{ .b, .h, .k, .hd }), .{ .attn_mask = mask.withTags(.{ .q, .k }), .scale = null, .bias = null } }); - try std.testing.expectEqualSlices(i64, q.shape().dims(), ref_res.shape().dims()); - - const res = try zml.testing.compileAndCall(platform, sdpaMemEfficient, .{ q, k, v, .{ - .query_chunk_size = 256, - .key_chunk_size = 128, - .scale = null, - .opts = .{ .attn_mask = mask, .scale = null, .bias = null, .allow_cudnn = false }, - } }); - - try zml.testing.expectClose(ref_res, res, 2e-3); + const expected_prefix_outputs: zml.Slice = .init( + zml.Shape.init(.{ 2, 2, 2 }, .f32), + std.mem.sliceAsBytes(&[2][2][2]f32{ + .{ .{ 3.0, 0.0 }, .{ 1.125, 2.0 } }, + .{ .{ 0.0, 0.0 }, .{ 0.0, 0.0 } }, + }), + ); + const expected_prefix_final_s: zml.Slice = .init( + zml.Shape.init(.{ 2, 2, 2 }, .f32), + std.mem.sliceAsBytes(&[2][2][2]f32{ + .{ .{ 3.0, 5.0 }, .{ 0.0, 0.5 } }, + .{ .{ 0.5, 1.125 }, .{ 0.25, 2.0 } }, + }), + ); + try zml.testing.expectClose(std.testing.io, expected_prefix_outputs, prefix_result.outputs, .{ + .absolute_tolerance = 1e-4, + .relative_tolerance = 1e-4, + }); + try zml.testing.expectClose(std.testing.io, expected_prefix_final_s, prefix_result.state.s, .{ + .absolute_tolerance = 1e-4, + .relative_tolerance = 1e-4, + }); } /// Options controlling generation. The default values correspond to greedy decoding. @@ -977,13 +1984,13 @@ pub const SamplingStrategy = struct { /// Returns an integer tensor with a shape similar to the input, but without the .voc axis. pub fn sampleTokens(activations: Tensor, opts: SamplingStrategy, rng: Tensor.Rng) struct { Tensor, Tensor.Rng } { if (opts.topk <= 1) { - const next_tokens = activations.argMax(.voc, .i32).indices.squeeze(.voc); + const next_tokens = activations.argMax(.voc).indices.squeeze(.voc); return .{ next_tokens, rng }; } - const topk = activations.topK(opts.topk, .voc, .{}); - // After the topk, we don't have .voc values, anymore, only topk. - var x = topk.values.rename(.{ .voc = .topk }); + const topk = activations.topK(.{ .topk = .voc }, opts.topk, .{}); + // After the topk, we don't have .voc values, anymore, only .topk. + var x = topk.values; if (opts.temperature != 1.0) { x = x.scale(1 / opts.temperature); } @@ -994,11 +2001,407 @@ pub fn sampleTokens(activations: Tensor, opts: SamplingStrategy, rng: Tensor.Rng // https://en.wikipedia.org/wiki/Gumbel_distribution#Gumbel_reparametrization_tricks const next_rng, const gumbel_noise = rng.gumbel(x.shape()); x = x.add(gumbel_noise); - const topk_idx = x.argMax(.topk, .i32).indices; + const topk_idx = x.argMax(.topk).indices; // topk_idx is indices into topk.values ! so in the range [0, topk] // Convert for the original indices from the full [0, voc] range. - const next_tokens = topk.indices.gatherValues(.voc, topk_idx.squeeze(.topk), .{}); + const next_tokens = topk.indices.gather(.{ .topk = topk_idx.squeeze(.topk) }, .{}); // log.debug("sampleTokens({}) -> {} -> {} -> {}", .{ activations, topk.indices, topk_idx, next_tokens }); return .{ next_tokens, next_rng }; } + +test sampleTokens { + const platform = zml.testing.env(); + + const rng: zml.Tensor.Rng = .init(); + const activations: zml.Tensor = .init(.{ .voc = 4 }, .f32); + + var exe = try platform.compileFn(std.testing.allocator, std.testing.io, sampleTokens, .{ activations, .{ .topk = 4, .temperature = 2.0 }, rng }, .{}); + defer exe.deinit(); + + var rng_buffer = try rng.initBuffer(std.testing.io, platform, 0xdeadbeef); + defer rng_buffer._state.deinit(); + + const inf = std.math.inf(f32); + + inline for (.{ + .{ [_]f32{ inf, 3.0, 2.0, 1.0 }, 0 }, + .{ [_]f32{ -inf, 3.0, -inf, -inf }, 1 }, + .{ [_]f32{ 3.0, 2, inf, inf }, 2 }, + }) |logits_expected| { + const logits, const expected: i32 = logits_expected; + + var activations_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, activations.shape(), std.mem.sliceAsBytes(&logits)); + defer activations_buffer.deinit(); + + var sampled: zml.Buffer = undefined; + var runner = try exe.runner(std.testing.allocator); + defer runner.deinit(std.testing.allocator); + runner.run(std.testing.io, .{ activations_buffer, rng_buffer }, .{ &sampled, &rng_buffer }, .{ .wait = true }); + defer sampled.deinit(); + + try zml.testing.expectEqual(expected, try sampled.getValue(i32, std.testing.io)); + } +} + +pub const DynamicSamplingStrategy = struct { + max_top_k: u32, + top_k: Tensor, + temperature: Tensor, + top_p: Tensor, + min_p: Tensor, + + pub const Opts = struct { + top_k: u32, + temperature: f32 = 1.0, + top_p: f32 = 1.0, + min_p: f32 = 0.0, + }; + + pub fn init(dtype: DataType, max_top_k: u32) DynamicSamplingStrategy { + const scalar_float_shape = Shape.init(.{}, dtype); + const scalar_i32_shape = Shape.init(.{}, .i32); + return .{ + .max_top_k = max_top_k, + .top_k = .fromShape(scalar_i32_shape), + .temperature = .fromShape(scalar_float_shape), + .top_p = .fromShape(scalar_float_shape), + .min_p = .fromShape(scalar_float_shape), + }; + } + + pub fn makeBuffers( + io: std.Io, + platform: *const zml.Platform, + dtype: zml.DataType, + opts: Opts, + ) !zml.Bufferized(DynamicSamplingStrategy) { + return .{ + .top_k = try zml.Buffer.scalar(io, platform, opts.top_k, .i32), + .temperature = try zml.Buffer.scalar(io, platform, opts.temperature, dtype), + .top_p = try zml.Buffer.scalar(io, platform, opts.top_p, dtype), + .min_p = try zml.Buffer.scalar(io, platform, opts.min_p, dtype), + }; + } + + pub fn deinitBuffers(self: *zml.Bufferized(DynamicSamplingStrategy)) void { + self.top_k.deinit(); + self.temperature.deinit(); + self.top_p.deinit(); + self.min_p.deinit(); + } +}; + +/// Given the output of the last layer of a LM with a `.voc` axis, +/// Compute indices for the next tokens, following the given sampling strategy. +/// The dynamic sampling strategy is more expressive but top_p requires computing the softmax. +/// +/// Options are: +/// +/// * top_k: only sample among the k top scoring tokens, +/// * max_top_k: limit a compilation time what is the max possible runtime value for top_k, saving memory and compute by not having to fully sort the tokens. +/// * top_p: only sample among top scoring tokens whose probabilities sum up to top_p +/// * min_p: drop tokens whose probabilities are lower than a ratio of the most likely token +pub fn sampleTokensDynamic(logits: Tensor, opts: DynamicSamplingStrategy, rng: Tensor.Rng) struct { Tensor, Tensor.Rng } { + var x, const topk_indices = fixupLogits(logits, opts); + + // the rest is similar to sampleTokens + const next_rng, const gumbel_noise = rng.gumbel(x.shape()); + x = x.add(gumbel_noise); + + const topk_idx = x.argMax(.topk).indices; + const next_tokens = topk_indices.gather(.{ .voc = topk_idx.squeeze(.topk) }, .{}); + return .{ next_tokens, next_rng }; +} + +fn fixupLogits(logits: Tensor, opts: DynamicSamplingStrategy) [2]Tensor { + const min_inf = Tensor.constant(logits.dtype().minValue()); + + // First reduce the vocab size to a reasonable sub set of candidate. + const full_topk = if (opts.max_top_k > 0) + logits.topK(.{ .voc = .voc }, opts.max_top_k, .{ .descending = true }) + else + logits.sort(.voc, .{ .descending = true }); + + // After the topk, we don't have .voc indices, anymore, only topk. + var x = full_topk.values.rename(.{ .voc = .topk }); + // mask values above the dynamic top_k + const top_k = opts.top_k.broad(x.shape()); + x = Tensor.iota(x.shape(), .topk).cmp(.GE, top_k).select(min_inf, x); + const temperature = opts.temperature.convert(x.dtype()).broad(x.shape()); + x = x.div(temperature); + + // if there are high values in x, softmax can overflow and will create nans in full probs + // this propagate to probs_sum and probs_max. + const probs = x.softmax(.topk); + const probs_sum = probs.cumulativeSum(.topk); + const probs_max = probs.slice(.topk, .{ .start = 0, .end = 1 }); + + const top_p = opts.top_p.convert(x.dtype()).broad(x.shape()); + const min_p = opts.min_p.convert(x.dtype()).broad(probs_max.shape()).mul(probs_max).broad(x.shape()); + + // * if first candidate has very high prob, then probs_sum is always greater than top_p and candidate is full false + // * if first candidate score is even bigger, the probs become Nan because of the softmax, + // then cmp is is full false, and candidate is full false too. + const candidate = probs_sum.cmp(.LE, top_p).logical(.AND, probs.cmp(.GE, min_p)); + // * so we explicitly always accept first candidate. + const first_token = Tensor.iota(x.shape(), .topk).cmp(.EQ, Tensor.scalar(0, .i32)); + x = candidate.logical(.OR, first_token).select(x, min_inf); + + return .{ x, full_topk.indices }; +} + +test sampleTokensDynamic { + const platform = zml.testing.env(); + + const ___ = -std.math.inf(f32); + const logits_data = [_]f32{ @log(2.0), @log(1.0), @log(4.0), @log(3.0) }; + const top_k_indices = [_]i32{ 2, 3, 0, 1 }; + + const logits: zml.Tensor = .init(.{ .voc = logits_data.len }, .f32); + const dynamic_sampling_strategy = DynamicSamplingStrategy.init(.f32, 0); + + var exe = try platform.compileFn(std.testing.allocator, std.testing.io, fixupLogits, .{ logits, dynamic_sampling_strategy }, .{}); + defer exe.deinit(); + + var logits_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, logits.shape(), std.mem.sliceAsBytes(&logits_data)); + defer logits_buffer.deinit(); + + const Args = struct { DynamicSamplingStrategy.Opts, [4]f32 }; + inline for ([_]Args{ + // top_k == logits.len -> just sort the input + .{ .{ .top_k = 4 }, [_]f32{ @log(4.0), @log(3.0), @log(2.0), @log(1.0) } }, + .{ .{ .top_k = 2 }, [_]f32{ @log(4.0), @log(3.0), ___, ___ } }, + .{ .{ .top_k = 2, .temperature = 0.1 }, [_]f32{ @log(4.0) / 0.1, @log(3.0) / 0.1, ___, ___ } }, + // top_k == logits.len and small top_p -> make sure at least one is returned + .{ .{ .top_k = 4, .top_p = 0.1 }, [_]f32{ @log(4.0), ___, ___, ___ } }, + .{ .{ .top_k = 4, .top_p = 0.701 }, [_]f32{ @log(4.0), @log(3.0), ___, ___ } }, + .{ .{ .top_k = 4, .top_p = 0.901 }, [_]f32{ @log(4.0), @log(3.0), @log(2.0), ___ } }, + // Here top_p is computed on the top 3 items, so 0.701 isn't enougth anymore to allow @log(3.0) + .{ .{ .top_k = 3, .top_p = 0.701 }, [_]f32{ @log(4.0), ___, ___, ___ } }, + // Here top_p allows the first 3 results, but min_p only accepts the first two. + .{ .{ .top_k = 4, .top_p = 0.901, .min_p = 0.6 }, [_]f32{ @log(4.0), @log(3.0), ___, ___ } }, + }) |args_expected| { + const args, const expected = args_expected; + var dynamic_sampling_strategy_buffers = try DynamicSamplingStrategy.makeBuffers(std.testing.io, platform, .f32, args); + defer DynamicSamplingStrategy.deinitBuffers(&dynamic_sampling_strategy_buffers); + var new_logits: zml.Buffer = undefined; + var indices: zml.Buffer = undefined; + var runner = try exe.runner(std.testing.allocator); + defer runner.deinit(std.testing.allocator); + runner.run(std.testing.io, .{ logits_buffer, dynamic_sampling_strategy_buffers }, .{ &new_logits, &indices }, .{ .wait = true }); + defer new_logits.deinit(); + defer indices.deinit(); + try std.testing.expectEqual(top_k_indices, try indices.getValue(@TypeOf(top_k_indices), std.testing.io)); + try zml.testing.expectEqual(expected, try new_logits.getValue(@TypeOf(expected), std.testing.io)); + } + + { + // Similar but use bf16, and uses infinity to trigger nans after the softmax. + const bf16 = zml.floats.BFloat16; + + const logits_bf16: zml.Tensor = .init(.{ .voc = logits_data.len }, .bf16); + const dynamic_sampling_strategy_bf16 = DynamicSamplingStrategy.init(.bf16, 0); + var exe_bf16 = try platform.compileFn(std.testing.allocator, std.testing.io, fixupLogits, .{ logits_bf16, dynamic_sampling_strategy_bf16 }, .{}); + defer exe_bf16.deinit(); + + const boost = bf16.inf; + const nerf = bf16.minus_inf; + + var logits_bf16_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, logits_bf16.shape(), std.mem.sliceAsBytes(&[4]bf16{ boost, boost, bf16.fromF32(2), nerf })); + defer logits_bf16_buffer.deinit(); + var dynamic_sampling_strategy_bf16_buffers = try DynamicSamplingStrategy.makeBuffers(std.testing.io, platform, .bf16, .{ .top_k = 4, .top_p = 0.9, .min_p = 0.1 }); + defer DynamicSamplingStrategy.deinitBuffers(&dynamic_sampling_strategy_bf16_buffers); + + var new_logits: zml.Buffer = undefined; + var indices: zml.Buffer = undefined; + var runner = try exe_bf16.runner(std.testing.allocator); + defer runner.deinit(std.testing.allocator); + runner.run(std.testing.io, .{ logits_bf16_buffer, dynamic_sampling_strategy_bf16_buffers }, .{ &new_logits, &indices }, .{ .wait = true }); + defer new_logits.deinit(); + defer indices.deinit(); + + try std.testing.expectEqual([_]i32{ 0, 1, 2, 3 }, try indices.getValue([4]i32, std.testing.io)); + try zml.testing.expectEqual([_]bf16{ boost, nerf, nerf, nerf }, try new_logits.getValue([4]bf16, std.testing.io)); + } +} + +const ShapeError = error{ DimMismatch, NotFound }; +const NOT_SET: i64 = -2; +const DIM_MISMATCH: i64 = -1; + +/// Collect the given dimensions inside a struct containing tagged tensors. +pub fn collectDims( + comptime dims: anytype, + v: anytype, + comptime mode: enum { strict, allow_extra_dims, ignore_errors }, +) ShapeError!ShapeStruct(dims) { + const LocalContext = struct { + res: ShapeStruct(dims), + mode: @TypeOf(mode), + }; + + var context = LocalContext{ + .res = undefined, + .mode = mode, + }; + @memset(std.mem.bytesAsSlice(i64, std.mem.asBytes(&context.res)), NOT_SET); + + meta.visit((struct { + fn cb(ctx: *LocalContext, shape: *const Shape) void { + inline for (dims) |a| { + if (shape.hasTag(a)) |axis| { + const dim = shape.dim(axis); + + const expected_dim = &@field(ctx.res, @tagName(a)); + if (expected_dim.* == NOT_SET) { + expected_dim.* = dim; + } else if (expected_dim.* == DIM_MISMATCH) { + // this axis has already been reported as invalid. + } else if (dim != expected_dim.*) { + if (mode != .ignore_errors) { + log.warn("Dim mismatch ! Axis {0s}={1d} but received a new tensor where {0s}={2d}", .{ @tagName(a), expected_dim.*, dim }); + } + expected_dim.* = DIM_MISMATCH; + } + } + } + } + }).cb, &context, v); + + if (context.mode != .ignore_errors) { + inline for (shapeToDims(context.res), dims) |dim, dim_tag| { + if (dim == NOT_SET) { + log.warn("Axis not found: {s}", .{@tagName(dim_tag)}); + return error.NotFound; + } + if (dim == DIM_MISMATCH) return error.DimMismatch; + } + } + return context.res; +} + +fn shapeToDims(shape: anytype) [std.meta.fieldNames(@TypeOf(shape)).len]i64 { + const field_names = comptime std.meta.fieldNames(@TypeOf(shape)); + var result: [field_names.len]i64 = undefined; + inline for (field_names, 0..) |field_name, i| { + result[i] = @field(shape, field_name); + } + return result; +} + +test collectDims { + const Model = struct { + x: Shape, + y: Shape, + bias: Shape, + }; + + { + var model: Model = .{ + .x = Shape.init(.{ 2, 5 }, .f32).withTags(.{ .b, .d }), + .y = Shape.init(.{ 2, 5 }, .f32).withTags(.{ .b, .d }), + .bias = Shape.init(.{5}, .f32).withTags(.{.d}), + }; + try zml.testing.expectEqual(collectDims(.{ .b, .d }, &model, .strict), .{ .b = 2, .d = 5 }); + } + { + var model: Model = .{ + .x = Shape.init(.{ 2, 5 }, .f32).withTags(.{ .b, .d }), + .y = Shape.init(.{ 3, 5 }, .f32).withTags(.{ .b, .d }), + .bias = Shape.init(.{5}, .f32).withTags(.{.d}), + }; + try std.testing.expectEqual( + collectDims(.{ .b, .d }, &model, .strict), + error.DimMismatch, + ); + try zml.testing.expectEqual(collectDims(.{ .b, .d }, &model, .ignore_errors), .{ .b = DIM_MISMATCH, .d = 5 }); + } + { + var model: Model = .{ + .x = Shape.init(.{ 2, 5 }, .f32).withTags(.{ .b, .d }), + .y = Shape.init(.{ 2, 5 }, .f32).withTags(.{ .b, .d }), + .bias = Shape.init(.{5}, .f32).withTags(.{.d}), + }; + try std.testing.expectEqual(collectDims(.{ .b, .d, .c }, &model, .strict), error.NotFound); + try zml.testing.expectEqual(collectDims(.{ .b, .d, .c }, &model, .ignore_errors), .{ .b = 2, .d = 5, .c = NOT_SET }); + } + { + var model: Model = .{ + .x = Shape.init(.{ 2, 5 }, .f32).withTags(.{ .b, .d }), + .y = Shape.init(.{ 2, 5 }, .f32).withTags(.{ .b, .d }), + .bias = Shape.init(.{7}, .f32).withTags(.{.d}), + }; + try std.testing.expectEqual(collectDims(.{ .b, .d }, &model, .strict), error.DimMismatch); + try zml.testing.expectEqual(collectDims(.{ .b, .d }, &model, .ignore_errors), .{ .b = 2, .d = DIM_MISMATCH }); + } +} + +fn ShapeStruct(comptime dims: anytype) type { + const rank = dims.len; + @setEvalBranchQuota(rank + 5); + var struct_field_names: [rank][]const u8 = undefined; + var struct_field_types: [rank]type = undefined; + var struct_field_attrs: [rank]std.builtin.Type.Struct.FieldAttributes = undefined; + const default: i64 = NOT_SET; + for (&struct_field_names, &struct_field_types, &struct_field_attrs, dims) |*field_name, *field_type, *field_attr, axis| { + field_name.* = @tagName(axis); + field_type.* = i64; + field_attr.* = .{ + .@"align" = @alignOf(i64), + .default_value_ptr = &default, + }; + } + + return @Struct(.@"extern", null, &struct_field_names, &struct_field_types, &struct_field_attrs); +} + +/// Smooth approximation of ReLU function: log(1 + exp(x)) +/// For numerical stability, falls back to the linear function when `x > opts.threshold`. +pub fn softplus( + x: Tensor, + opts: struct { threshold: f32 = 20.0 }, +) zml.Tensor { + // Compute log1p(z) as z * log(u) / (u - 1), where u = 1 + z. + // Using log(1 + exp(x)) directly loses precision for x < ~-16.6, causing strongly negative logits to produce zero scores. + const z = x.exp(); + const u = z.addConstant(1); + const log1p = u.cmp(.EQ, .scalar(1, x.dtype())).select(z, z.mul(u.log()).div(u.subConstant(1))); + return x.cmp(.GT, .scalar(opts.threshold, x.dtype())).select(x, log1p); +} + +test "softplus" { + const inputs = [_]f32{ -20.0, -1.0, 0.0, 1.0, 2.0, 2.1, 3.0 }; + const expected = [_]f32{ 2.0611537e-9, 0.3132617, 0.6931472, 1.3132616, 2.126928, 2.1, 3.0 }; + + const platform = zml.testing.env(); + const input: Tensor = .init(.{ .x = inputs.len }, .f32); + const Local = struct { + fn call(x: Tensor, dtype: DataType) Tensor { + return softplus(x.convert(dtype), .{ .threshold = 2.0 }).convert(x.dtype()); + } + }; + + var input_buffer: zml.Buffer = try .fromBytes( + std.testing.io, + platform, + input.shape(), + std.mem.sliceAsBytes(&inputs), + ); + defer input_buffer.deinit(); + + for ([_]DataType{ .f32, .bf16 }) |dtype| { + var exe = try platform.compileFn(std.testing.allocator, std.testing.io, Local.call, .{ input, dtype }, .{}); + defer exe.deinit(); + + var result = try exe.eval(std.testing.allocator, std.testing.io, .{input_buffer}); + defer result.deinit(); + + try zml.testing.expectClose( + std.testing.io, + Slice.init(input.shape(), std.mem.sliceAsBytes(&expected)), + result, + .{}, + ); + } +} diff --git a/zml/nn/cuda.zig b/zml/nn/cuda.zig deleted file mode 100644 index 0bffc5f1..00000000 --- a/zml/nn/cuda.zig +++ /dev/null @@ -1,143 +0,0 @@ -const std = @import("std"); - -const Context = @import("../context.zig").Context; -const module = @import("../module.zig"); -const mlir = @import("../mlir.zig"); -const dialect = @import("mlir/dialects"); - -const Tensor = @import("../tensor.zig").Tensor; -const Shape = @import("../shape.zig").Shape; -const SdpaOpts = @import("../nn.zig").SdpaOpts; -const DataType = @import("../dtype.zig").DataType; -const Data = @import("../dtype.zig").Data; -const CompilationContext = module.CompilationContext; - -pub fn canUseCudnnSdpa(head_dim: i64, dtype: DataType) bool { - const ctx = CompilationContext.current(); - // TODO(Corendos): Check cuda version, cudnn version, device compatibility. - if (!ctx.targetIs(.cuda)) return false; - - // NOTE(Corentin): In Cudnn fused MHA head_dim is limited to 128. - if (head_dim > 128) return false; - - // NOTE(Corentin): In Cudnn fused MHA data type is limited to F16 and BF16. - if (dtype != .f16 and dtype != .bf16) return false; - - return true; -} - -fn elementTypeFromDataType(dtype: DataType) [:0]const u8 { - return switch (dtype) { - .f16 => "F16", - .bf16 => "BF16", - else => @panic("Unsupported DataType"), - }; -} - -pub fn sdpa(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) Tensor { - const ctx = CompilationContext.current(); - const q = q_.transpose(.{ .b, .h, .q, .hd }); - var k = k_.transpose(.{ .b, .h, .k, .hd }); - const v = v_.transpose(.{ .b, .h, .k, .hd }); - - const sqrtHeadDim: f32 = 1.0 / std.math.sqrt(@as(f32, @floatFromInt(q.dim(.hd)))); - const scale: f32 = if (opts.scale) |_| 1.0 else sqrtHeadDim; - - if (opts.scale) |s| k = k.mul(s); - - var buffer: [4096]u8 = undefined; - var fba = std.heap.FixedBufferAllocator.init(&buffer); - const allocator = fba.allocator(); - - const backend_config = - std.fmt.allocPrintZ( - allocator, - \\{{ - \\ "operation_queue_id":"0", - \\ "wait_on_operation_queues":[], - \\ "cudnn_fmha_backend_config": {{ - \\ "algorithm": {{ - \\ "algo_id": "0", - \\ "math_type": "TENSOR_OP_MATH", - \\ "tuning_knobs": {{"17": "1", "24": "0"}}, - \\ "is_cudnn_frontend": true, - \\ "workspace_size": "0", - \\ }}, - \\ "fmha_scale": {d}, - \\ "dropout_rate": 0, - \\ "intermediate_tensor_shape": {{ - \\ "element_type": "{s}", - \\ "dimensions": ["{}", "{}", "{}", "{}"], - \\ "tuple_shapes": [], - \\ "layout": {{ - \\ "dim_level_types": [], - \\ "dim_unique": [], - \\ "dim_ordered": [], - \\ "minor_to_major": ["3", "2", "1", "0"], - \\ "tiles": [], - \\ "element_size_in_bits": "0", - \\ "memory_space": "0", - \\ "index_primitive_type": "PRIMITIVE_TYPE_INVALID", - \\ "pointer_primitive_type": "PRIMITIVE_TYPE_INVALID", - \\ "dynamic_shape_metadata_prefix_bytes": "0", - \\ }}, - \\ "is_dynamic_dimension": [false, false, false, false], - \\ }}, - \\ "seed": 0, - \\ "is_flash_attention": true, - \\ "mask_type": "NO_MASK", - \\ "bmm1_dot_dimension_numbers": {{ - \\ "lhs_contracting_dimensions": ["3"], - \\ "rhs_contracting_dimensions": ["3"], - \\ "lhs_batch_dimensions": ["0", "1"], - \\ "rhs_batch_dimensions": ["0", "1"], - \\ }}, - \\ "bmm2_dot_dimension_numbers": {{ - \\ "lhs_contracting_dimensions": ["3"], - \\ "rhs_contracting_dimensions": ["2"], - \\ "lhs_batch_dimensions": ["0", "1"], - \\ "rhs_batch_dimensions": ["0", "1"], - \\ }} - \\ }} - \\}} - , - .{ - scale, - elementTypeFromDataType(q.dtype()), - q.dim(.b), - q.dim(.h), - q.dim(.q), - k.dim(.k), - }, - ) catch unreachable; - - var bias = Tensor.constant(Shape.init(.{ .b = q.dim(.b), .h = q.dim(.h), .q = q.dim(.q), .k = k.dim(.k) }, q.dtype()), Data.init(q.dtype(), 0)); - - if (opts.attn_mask) |attn_mask| { - const mask = attn_mask.withTags(.{ .q, .k }).broad(bias.shape()); - bias = bias.add(mask); - } - if (opts.bias) |b| { - bias = bias.add(b); - } - - const loc = ctx.mlirCtx().location(@src()); - const op = dialect.stablehlo.custom_call( - ctx.mlirCtx(), - &.{ q.value(), k.value(), v.value(), bias.value() }, - .{ - .call_target_name = "__cudnn$fmhaScaleBiasSoftmax", - .backend_config = backend_config, - .api_version = 2, - .has_side_effect = false, - .output_operand_aliases = &.{}, - }, - &.{ - mlir.ext.mlirType(ctx.mlirCtx(), q.shape()), - mlir.RankedTensorType.init(&.{0}, mlir.IntegerType(.u8).init(ctx.mlirCtx()).as(mlir.Type).?).as(mlir.Type).?, - }, - loc, - ); - const result = Tensor._result(q.shape(), op.result(0)).transpose(q_.shape()); - return result; -} diff --git a/zml/ops.zig b/zml/ops.zig index 35765cec..f8fdba7e 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -1,208 +1,401 @@ const std = @import("std"); -const mlir = @import("mlir.zig"); -const buffer = @import("buffer.zig"); +const builtin = @import("builtin"); -const helpers = @import("helpers.zig"); -const module = @import("module.zig"); -const meta = @import("meta.zig"); +const func = @import("mlir/dialects/func"); +const mlir = @import("mlir"); +const pjrt = @import("pjrt"); +const platforms = @import("platforms"); +const stablehlo = @import("mlir/dialects/stablehlo"); +const stdx = @import("stdx"); -const CompilationContext = module.CompilationContext; -const Tensor = @import("tensor.zig").Tensor; -const Shape = @import("shape.zig").Shape; -const Data = @import("dtype.zig").Data; +const Buffer = @import("buffer.zig").Buffer; +const Bufferized = @import("mem.zig").Bufferized; +const Compiler = @import("Compiler.zig"); +const constants = @import("constants.zig"); +const CustomCallBuffer = @import("pjrtx.zig").CustomCallBuffer; const DataType = @import("dtype.zig").DataType; -const Buffer = buffer.Buffer; -const EnumLiteral = @TypeOf(.enum_literal); - -const dialect = struct { - const stablehlo = @import("mlir/dialects").stablehlo; -}; - -const assert = std.debug.assert; -const log = std.log.scoped(.zml_tensor); +const meta = @import("meta.zig"); +const mlirx = @import("mlirx.zig"); +const manual_computation = @import("ops/manual_computation.zig"); +pub const manualComputation = manual_computation.manualComputation; +pub const ManualPartitioning = manual_computation.ManualPartitioning; +pub const PartitionSpecs = manual_computation.PartitionSpecs; +pub const partitionSpecs = manual_computation.partitionSpecs; +pub const partitionSpecsAlloc = manual_computation.partitionSpecsAlloc; +const Platform = @import("platform.zig").Platform; +const Shape = @import("shape.zig").Shape; +const ShapeToCustomCallBuffer = @import("pjrtx.zig").ShapeToCustomCallBuffer; +const Sharding = @import("Sharding.zig"); +const Tensor = @import("tensor.zig").Tensor; +const TensorToCustomCallBuffer = @import("pjrtx.zig").TensorToCustomCallBuffer; test { - std.testing.refAllDecls(@This()); + _ = manual_computation; } -/// Generate an MLIR call to the given member function with the given tensors. -pub fn call(self: anytype, comptime func: meta.DeclEnum(@TypeOf(self)), args: anytype) @TypeOf(@call(.auto, @field(meta.UnwrapPtr(@TypeOf(self)), @tagName(func)), .{self} ++ args)) { - // TODO: this should use `self.getContext().callFunc(self, args)` +const log = std.log.scoped(.@"zml/Compiler"); - return @call(.auto, @field(@TypeOf(self), @tagName(func)), .{self} ++ args); +/// Reduces local tensor values across all processes in the computation. +/// For each tensor index, `func_` combines the values at that index from every +/// process. The result is identical on all processes. +/// +/// Inside a manual computation, `inputs` describes the local tensors at each +/// partition. For example, summing `[1, 2]` and `[3, 4]` from two partitions gives +/// `[4, 6]` at each partition. The local tensor dimensions stay unchanged. +/// +/// * `inputs` is a `Tensor`, a nonempty tuple of tensors, or an `[N]Tensor`. +/// Each input must have matching shapes and dtypes across processes. +/// * `func_` describes how to combine two scalar contributions: +/// - For one input, it must be `fn (left: Tensor, right: Tensor) Tensor`, such +/// as `Tensor.add`. This also applies to a one-element tuple or array. +/// - For multiple inputs, it takes one `ReduceArgs` per input and returns a +/// tuple or array with one scalar tensor per input, in the same order. +/// +/// Returns the same container type as `inputs`, preserving tensor attributes. +/// With one process, returns `inputs` unchanged. +/// +/// The reduction uses one group containing all processes, including mesh replicas. +/// To reduce along one logical mesh axis, use `allReduceOnAxis`. +pub fn allReduce(inputs: anytype, comptime func_: anytype) AllReduceReturnType(@TypeOf(inputs)) { + const count = Compiler.current().meshes[0].numDevices(); + var devices: [Platform.MAX_NUM_DEVICES]i64 = undefined; + for (devices[0..count], 0..) |*device, i| device.* = @intCast(i); + return allReduceGroups(inputs, func_, devices[0..count], count); } -pub fn while_( - comptime cond_fn: anytype, - comptime body_fn: anytype, - blkctx: BlockSign(body_fn).BlkCtx, - inputs: BlockSign(body_fn).Args, -) BlockSign(body_fn).Return { - const CondS = comptime BlockSign(cond_fn); - const BodyS = comptime BlockSign(body_fn); - if (CondS.Args != BodyS.Args) { - @compileError("cond_fn and body_fn signatures don't match ! " ++ @typeName(@TypeOf(cond_fn)) ++ " and " ++ @typeName(@TypeOf(body_fn))); +/// Reduces local tensor values within process groups defined by `axis` in `mesh`. +/// A group contains processes with the same coordinates on every other logical +/// mesh axis. Separate replicas of the mesh belong to separate groups. Within +/// each group, `func_` combines corresponding elements and the result is identical +/// on all member processes. +/// +/// Use this inside a manual computation to combine partial results along `axis` +/// while preserving the independent computations along the other mesh axes. +/// +/// * `inputs` and `func_` follow `allReduce`'s tensor and reducer contracts. +/// * `mesh` is the logical mesh used by the manual computation and must be +/// included in the compilation's meshes. +/// * `axis` names an axis of that mesh, for example `.experts`. It can be an enum +/// literal or a mesh axis tag. The axis must exist in `mesh`. +/// +/// Returns the same container type with tensor attributes as +/// `inputs`. An axis of size one returns `inputs` unchanged. +/// +/// For a mesh with `.data = 2` and `.experts = 4`, the following sums four local +/// expert contributions independently for each of the two data shards: +/// +/// ```zig +/// const output = allReduceOnAxis(local_output, Tensor.add, mesh, .experts); +/// ``` +/// +/// Each expert shard now has the complete output for its data shard. To expose +/// this result through `manualComputation`, an output spec such as +/// `.parse(.{ .t = .data })` keeps tokens split across `.data` and declares the +/// result replicated along `.experts`. +pub fn allReduceOnAxis(inputs: anytype, comptime func_: anytype, mesh: *const Sharding.Mesh, axis: anytype) AllReduceReturnType(@TypeOf(inputs)) { + const logical_axis = mesh.resolveLogicalAxis(Shape.toTag(axis)) orelse @panic("Unknown logical reduction axis"); + const size = mesh.grid.axis_sizes[logical_axis]; + var groups: [Platform.MAX_NUM_DEVICES]i64 = undefined; + var replica_indices: [Platform.MAX_NUM_DEVICES]usize = @splat(0); + const replicas = mesh.numReplicas(); + for (0..mesh.numDevices()) |device| { + const coords = mesh.grid.logical_coords[device]; + var tile: usize = 0; + var group: usize = 0; + for (0..mesh.logical.axes.len) |dim| { + tile = tile * mesh.grid.axis_sizes[dim] + coords[dim]; + if (dim != logical_axis) group = group * mesh.grid.axis_sizes[dim] + coords[dim]; + } + group = group * replicas + replica_indices[tile]; + replica_indices[tile] += 1; + groups[group * size + coords[logical_axis]] = @intCast(device); } - const ctx = CompilationContext.current(); - const cond_block = ctx.makeBlock(CondS, &cond_fn, blkctx, inputs); - const body_block = ctx.makeBlock(BodyS, &body_fn, blkctx, inputs); - var input_values: [BodyS.nIn]mlir.Value = undefined; - ctx.extractValues(&inputs, &input_values); - - const loc = ctx.mlirCtx().location(@src()); - - const op = mlir.Operation.make(ctx.mlirCtx(), "stablehlo.while", .{ - .variadic_operands = &.{&input_values}, - .result_type_inference = true, - .blocks = &.{ cond_block, body_block }, - // We can't verify right away, cause the weights captured by the while haven't been added yet. - .verify = false, - .location = loc, - }); - - var res: BodyS.Args = inputs; - module.assignResults(&res, null, op); - return res; + return allReduceGroups(inputs, func_, groups[0..mesh.numDevices()], size); } -test "simple while" { - const CountInts = struct { - step: Tensor, - end: Tensor, - const CountInts = @This(); +/// Reduces corresponding tensor elements independently within each process group. +/// `inputs`, `func_` and the return value follow `allReduce`'s contract. +/// +/// `groups` identifies processes by their executable partition IDs, as returned +/// by `partitionId()`. Each consecutive `group_size` IDs defines one group. +/// +/// For example, `groups = &.{ 0, 2, 1, 3 }` and `group_size = 2` combine partitions +/// 0 and 2 together, and partitions 1 and 3 together. Both members of each group +/// receive that group's result. Groups of size one return `inputs` unchanged. +fn allReduceGroups(inputs: anytype, comptime func_: anytype, groups: []const i64, group_size: usize) AllReduceReturnType(@TypeOf(inputs)) { + const ctx = Compiler.current(); + const mlir_ctx = ctx.mlir_ctx; - pub fn hasNext(self: CountInts, i: Tensor, sum: Tensor) Tensor { - _ = sum; - return i.cmp(.LT, self.end); - } - - pub fn next(self: CountInts, i: Tensor, sum: Tensor) [2]Tensor { - const r1 = i.add(self.step); - const r2 = sum.add(i); - return .{ r1, r2 }; - } - - pub fn forward(self: CountInts, init_i: Tensor, init_sum: Tensor) [2]Tensor { - const x = init_i.scale(2); - return while_(CountInts.hasNext, CountInts.next, self, .{ x, init_sum }); - } - - pub fn zigForward(step: i64, end: i64, init_i: i64, init_sum: i64) [2]i64 { - const x = init_i * 2; - var i = x; - var sum = init_sum; - while (i < end) { - const r1 = i + step; - const r2 = sum + i; - i, sum = .{ r1, r2 }; + const InputsT = @TypeOf(inputs); + const n_inputs = switch (@typeInfo(InputsT)) { + .@"struct" => |struct_info| b: { + if (InputsT == Tensor) break :b 1; + if (!struct_info.is_tuple) { + @compileError("zml.ops.allReduce expects Tensor, tuple of Tensor, or [N]Tensor inputs"); } - return .{ i, sum }; + break :b struct_info.field_names.len; + }, + .array => |array_info| b: { + if (array_info.child != Tensor) { + @compileError("zml.ops.allReduce expects [N]Tensor inputs"); + } + break :b array_info.len; + }, + else => @compileError("zml.ops.allReduce expects Tensor, tuple of Tensor, or [N]Tensor inputs"), + }; + comptime stdx.debug.assertComptime(n_inputs > 0, "zml.ops.allReduce requires at least one input tensor", .{}); + + const input_tensors: [n_inputs]Tensor = switch (@typeInfo(InputsT)) { + .@"struct" => |struct_info| b: { + if (InputsT == Tensor) { + break :b .{inputs}; + } + if (!struct_info.is_tuple) { + @compileError("zml.ops.allReduce expects Tensor, tuple of Tensor, or [N]Tensor inputs"); + } + + var flat_inputs: [n_inputs]Tensor = undefined; + meta.collectBuf((struct { + pub fn cb(t: Tensor) Tensor { + return t; + } + }).cb, {}, &inputs, &flat_inputs); + break :b flat_inputs; + }, + .array => inputs, + else => @compileError("zml.ops.allReduce expects Tensor, tuple of Tensor, or [N]Tensor inputs"), + }; + + if (group_size <= 1) return inputs; + + const reducer_block = b: { + const ArgsTypes: [input_tensors.len]type = @splat(ReduceArgs); + var args: @Tuple(&ArgsTypes) = undefined; + var block_types: [2 * input_tensors.len]*const mlir.Type = undefined; + + inline for (0..input_tensors.len) |i| { + const scalar_shape = Shape.init(.{}, input_tensors[i].dtype()); + args[i].left = .fromShape(scalar_shape); + args[i].right = .fromShape(scalar_shape); + block_types[i] = mlirx.Type.rankedTensor(mlir_ctx, scalar_shape); + block_types[i + input_tensors.len] = mlirx.Type.rankedTensor(mlir_ctx, scalar_shape); } + + const block_locs: [2 * input_tensors.len]*const mlir.Location = @splat(ctx.unknown_location); + + const block = mlir.Block.init(&block_types, &block_locs); + errdefer block.deinit(); + + const reducer_scope = ctx.pushBlock(block); + defer reducer_scope.pop(); + + ctx.pushLocation(@src(), stdx.meta.fnName(func_)); + defer ctx.popLocation(); + + inline for (0..input_tensors.len) |i| { + reducer_scope.registerTensorAsBlockArgument(args[i].left.id, i); + reducer_scope.registerTensorAsBlockArgument(args[i].right.id, i + input_tensors.len); + } + + var reduced_values: [input_tensors.len]*const mlir.Value = undefined; + if (input_tensors.len == 1) { + const reduced = @call(.auto, func_, .{ args[0].left, args[0].right }); + reduced_values[0] = reduced.value(); + } else { + var reduced = @call(.auto, func_, args); + var reduced_tensors: [input_tensors.len]Tensor = undefined; + meta.collectBuf(stdx.meta.identity, {}, &reduced, &reduced_tensors); + + for (reduced_values, reduced_tensors) |*vi, ti| { + vi.* = ti.value(); + } + } + + _ = stablehlo.returns(mlir_ctx, &reduced_values, ctx.location).appendTo(block); + break :b block; }; - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - const init_i = try zml.Buffer.fromSlice(platform, .{}, &[_]i64{0}); - const init_sum = try zml.Buffer.fromSlice(platform, .{}, &[_]i64{0}); - const counter = .{ - .step = try zml.Buffer.fromSlice(platform, .{}, &[_]i64{1}), - .end = try zml.Buffer.fromSlice(platform, .{}, &[_]i64{10}), + const replica_groups_attr = mlir.Attribute.denseElements( + .rankedTensor(&.{ @intCast(groups.len / group_size), @intCast(group_size) }, .int(mlir_ctx, .i64)), + groups, + ); + + const handle = ctx.nextChannelId(); + const channel_handle_attr = stablehlo.channelHandleAttr( + mlir_ctx, + handle, + .collective, + ); + + var input_values: [input_tensors.len]*const mlir.Value = undefined; + var result_types: [input_tensors.len]*const mlir.Type = undefined; + inline for (0..input_tensors.len) |i| { + input_values[i] = input_tensors[i].value(); + result_types[i] = input_values[i].type_(); + } + + const op = stablehlo.allReduce( + mlir_ctx, + &input_values, + &result_types, + reducer_block, + replica_groups_attr, + channel_handle_attr, + ctx.location, + ).appendTo(ctx.currentScope().block); + + return switch (@typeInfo(@TypeOf(inputs))) { + .@"struct" => |struct_info| b: { + if (@TypeOf(inputs) == Tensor) { + break :b Tensor._result(input_tensors[0].shape(), op.result(0)); + } + + var out: @TypeOf(inputs) = undefined; + inline for (struct_info.field_names, 0..) |field_name, i| { + @field(out, field_name) = Tensor._result(input_tensors[i].shape(), op.result(i)); + } + break :b out; + }, + .array => |array_info| b: { + var out: [array_info.len]Tensor = undefined; + inline for (0..array_info.len) |i| { + out[i] = Tensor._result(input_tensors[i].shape(), op.result(i)); + } + break :b out; + }, + else => unreachable, }; - const res0, const res1 = try zml.testing.compileAndCall(platform, CountInts.forward, .{ counter, init_i, init_sum }); - const last_i = try res0.getValue(i64); - const sum = try res1.getValue(i64); - - try std.testing.expectEqual(10, last_i); - try std.testing.expectEqual(45, sum); - - try std.testing.expectEqual(.{ 10, 45 }, CountInts.zigForward(1, 10, 0, 0)); } -pub fn reduce( - comptime body_fn: anytype, - inputs: meta.FnParam(body_fn, 0), - inits: meta.FnParam(body_fn, 0), - axes: []const i64, -) BlockSignNoCtx(body_fn).Return { - // TODO: actualAxes - const BodyS = comptime BlockSignNoCtx(body_fn); - comptime { - if (BodyS.Return != @TypeOf(inputs)) @compileError("reduce body function need to have the following signature `fn (left: T, right: T) T`, got: " ++ @typeName(body_fn)); - } - const ctx = CompilationContext.current(); - const N = comptime @divExact(BodyS.nIn, 2); - var input_values: [N]mlir.Value = undefined; - ctx.extractValues(&inputs, &input_values); - var init_values: [N]mlir.Value = undefined; - ctx.extractValues(&inits, &init_values); +fn AllReduceReturnType(comptime InputsT: type) type { + if (InputsT == Tensor) return Tensor; - const body_block = ctx.makeBlock(BodyS, &body_fn, {}, .{ inits, inits }); - - const loc = ctx.mlirCtx().location(@src()); - - const op = mlir.Operation.make(ctx.mlirCtx(), "stablehlo.reduce", .{ - .variadic_operands = &.{ &input_values, &init_values }, - .result_type_inference = true, - .blocks = &.{body_block}, - .attributes = &.{ - .{ "dimensions", mlir.DenseArrayAttribute(.i64).init(ctx.mlirCtx(), axes).as(mlir.Attribute).? }, + return switch (@typeInfo(InputsT)) { + .@"struct" => |struct_info| b: { + if (!struct_info.is_tuple) { + @compileError("zml.ops.allReduce expects Tensor, tuple of Tensor or [N]Tensor inputs"); + } + break :b InputsT; }, - // We can't verify right away, cause the weights captured by the reduce haven't been added yet. - .verify = false, - .location = loc, - }); + .array => |array_info| b: { + if (array_info.child != Tensor) { + @compileError("zml.ops.allReduce expects [N]Tensor inputs"); + } + break :b InputsT; + }, + else => @compileError("zml.ops.allReduce expects Tensor, tuple of Tensor, or [N]Tensor inputs"), + }; +} + +/// Returns the current process's zero based coordinate on `axis` in `mesh` +/// as a scalar u32. +/// +/// Example: for a mesh {.data = 2, .experts = 2}, each data group has expert coordinates +/// 0 and 1. Use this coordinate inside a manual computation to identify a shard +/// along one axis while `partitionId()` identifies a partition across the whole mesh. +pub fn axisIndex(mesh: *const Sharding.Mesh, axis: anytype) Tensor { + const logical_axis = mesh.resolveLogicalAxis(Shape.toTag(axis)) orelse @panic("Unknown logical mesh axis"); + var coords: [Platform.MAX_NUM_DEVICES]u32 = undefined; + for (coords[0..mesh.numDevices()], 0..) |*coord, i| coord.* = mesh.grid.logical_coords[i][logical_axis]; + return Tensor.constantTensor(.init(.{ .device = mesh.numDevices() }, .u32), std.mem.sliceAsBytes(coords[0..mesh.numDevices()])) + .gather(.{ .device = partitionId() }, .{}); +} + +pub fn partitionId() Tensor { + const ctx = Compiler.current(); + const op = stablehlo.partition_id(ctx.mlir_ctx, ctx.location).appendTo(ctx.currentScope().block); + return Tensor._result(.init(.{}, .u32), op.result(0)); +} + +pub const ReduceArgs = struct { + left: Tensor, + right: Tensor, +}; + +pub fn reduce(inputs: anytype, inits: anytype, axes_: []const i64, comptime func_: anytype, context: anytype) stdx.meta.FnReturn(func_) { + const compiler = Compiler.current(); + const caller_scope = compiler.currentScope(); + const mlir_ctx = compiler.mlir_ctx; + + // Note: inputs and inits are infered to be tuple of ReduceArgs + const reduce_block, var result = b: { + const ArgsTypes: [inits.len]type = @splat(ReduceArgs); + const ArgsType = @Tuple(&ArgsTypes); + var args: ArgsType = undefined; + var block_types: [2 * inits.len]*const mlir.Type = undefined; + + inline for (0..inits.len) |i| { + args[i].left = .fromShape(inits[i].shape()); + args[i].right = .fromShape(inits[i].shape()); + + block_types[i] = mlirx.Type.rankedTensor(mlir_ctx, args[i].left.shape()); + block_types[i + inits.len] = mlirx.Type.rankedTensor(mlir_ctx, args[i].right.shape()); + } + + const block_locs: [2 * inits.len]*const mlir.Location = @splat(compiler.unknown_location); + const reduce_block = mlir.Block.init(&block_types, &block_locs); + errdefer reduce_block.deinit(); + + const reduce_scope = compiler.pushBlock(reduce_block); + defer reduce_scope.pop(); + + compiler.pushLocation(@src(), stdx.meta.fnName(func_)); + defer compiler.popLocation(); + + inline for (0..inits.len) |i| { + reduce_scope.registerTensorAsBlockArgument(args[i].left.id, i); + reduce_scope.registerTensorAsBlockArgument(args[i].right.id, i + inits.len); + } + + var result = @call(.auto, func_, args ++ context); + + var result_values: [inits.len]*const mlir.Value = undefined; + inline for (0..inits.len) |i| { + result_values[i] = result[i].value(); + } + + const block_result = stablehlo.returns(mlir_ctx, &result_values, compiler.location); + _ = block_result.appendTo(reduce_block); + break :b .{ reduce_block, result }; + }; + var input_values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inputs.len) |i| input_values[i] = inputs[i].value(); + + var init_values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inits.len) |i| init_values[i] = inits[i].value(); + + const reduce_op = stablehlo.reduce(mlir_ctx, &input_values, &init_values, reduce_block, axes_, compiler.location).appendTo(caller_scope.block); // `stablehlo.reduce` drops axes. We want to avoid that to propagate tags. // So we need to broadcast the output of `stablehlo.reduce` to the input shapes. - // To that order, we initialize `result` to `inputs`, then we use meta.visit, + // To that order, we initialize `result` to `inputs`, then we use stdx.meta.visit, // to find the correct mlir.Value, but we first broadcast before creating the final // Tensor struct. - var broadcasting_axes: std.BoundedArray(i64, Tensor.MAX_RANK) = .{}; - for (0..Tensor.MAX_RANK) |i| { - if (std.mem.indexOfScalar(i64, axes, @intCast(i)) == null) { + var broadcasting_axes: stdx.BoundedArray(i64, constants.MAX_RANK) = .empty; + for (0..constants.MAX_RANK) |i| { + if (std.mem.indexOfScalar(i64, axes_, @intCast(i)) == null) { broadcasting_axes.append(@intCast(i)) catch unreachable; } } - var res: BodyS.Return = inputs; - const LocalContext = struct { - axes: []const i64, - broadcasting_axes: []const i64, - n_reduced: u8, - op: mlir.Operation, - loc: mlir.Location, - index: usize = 0, - }; - var local_context = LocalContext{ - .axes = axes, - .broadcasting_axes = broadcasting_axes.constSlice(), - .n_reduced = @intCast(axes.len), - .op = op, - .loc = loc, - }; - meta.visit((struct { - fn cb(inner_ctx: *LocalContext, tensor: *Tensor) void { - const val = inner_ctx.op.result(inner_ctx.index); - // compute the target reduced shape - var reduced_shape = tensor.shape(); - for (inner_ctx.axes) |a| { - reduced_shape = reduced_shape.setDim(a, 1); - } - const mlir_ctx = CompilationContext.current().mlirCtx(); - - const broad_val = dialect.stablehlo.broadcast_in_dim( - mlir_ctx, - val, - inner_ctx.broadcasting_axes[0 .. tensor.rank() - inner_ctx.n_reduced], - mlir.ext.RankedTensorType.fromShape(mlir_ctx, reduced_shape).as(mlir.Type).?, - inner_ctx.loc, - ); - tensor.* = Tensor._result(reduced_shape, broad_val.result(0)); - inner_ctx.index += 1; + inline for (0..result.len) |i| { + var reduced_shape: Shape = inputs[i].shape(); + for (axes_) |a| { + reduced_shape = reduced_shape.setDim(a, 1); } - }).cb, &local_context, &res); - assert(local_context.index == op.numResults()); - return res; + + const broad_op = stablehlo.broadcast_in_dim( + mlir_ctx, + reduce_op.result(i), + broadcasting_axes.slice()[0 .. reduced_shape.rank() - axes_.len], + mlirx.Type.rankedTensor(mlir_ctx, reduced_shape), + compiler.location, + ).appendTo(caller_scope.block); + + result[i] = Tensor._result(reduced_shape, broad_op.result(0)); + } + + return result; } pub const ReduceWindowOpts = struct { @@ -211,209 +404,411 @@ pub const ReduceWindowOpts = struct { window_strides: []const i64, base_dilations: []const i64, window_dilations: []const i64, - padding_values: []const i64, - padding_shape: []const i64, + padding: []const [2]i64, }; -pub fn reduceWindow( - comptime body_fn: anytype, - inputs: meta.FnParam(body_fn, 0), - inits: meta.FnParam(body_fn, 0), - opts: ReduceWindowOpts, -) meta.FnResult(body_fn) { - const BodyS = comptime BlockSignNoCtx(body_fn); - comptime { - if (BodyS.Return != @TypeOf(inputs)) @compileError("reduce body function need to have the following signature `fn (left: T, right: T) T`, got: " ++ @typeName(body_fn)); - } - const ctx = CompilationContext.current(); - const body_block = ctx.makeBlock(BodyS, &body_fn, {}, .{ inits, inits }); - const N = comptime @divExact(BodyS.nIn, 2); - var input_values: [N]mlir.Value = undefined; - ctx.extractValues(&inputs, &input_values); - var init_values: [N]mlir.Value = undefined; - ctx.extractValues(&inits, &init_values); - - const loc = ctx.mlirCtx().location(@src()); - - const pad_shape = mlir.RankedTensorType.init(opts.padding_shape, mlir.ext.Type.fromDType(ctx.mlirCtx(), .i64)).as(mlir.Type).?; - const op = mlir.Operation.make(ctx.mlirCtx(), "stablehlo.reduce_window", .{ - .variadic_operands = &.{ input_values[0..], init_values[0..] }, - .result_type_inference = true, - .blocks = &.{body_block}, - .attributes = &.{ - .{ "window_dimensions", mlir.DenseArrayAttribute(.i64).init(ctx.mlirCtx(), opts.window_dimensions).as(mlir.Attribute).? }, - .{ "window_strides", mlir.DenseArrayAttribute(.i64).init(ctx.mlirCtx(), opts.window_strides).as(mlir.Attribute).? }, - .{ "base_dilations", mlir.DenseArrayAttribute(.i64).init(ctx.mlirCtx(), opts.base_dilations).as(mlir.Attribute).? }, - .{ "window_dilations", mlir.DenseArrayAttribute(.i64).init(ctx.mlirCtx(), opts.window_dilations).as(mlir.Attribute).? }, - .{ "padding", mlir.DenseIntOrFPElementsAttribute(.i64).init(pad_shape, std.mem.sliceAsBytes(opts.padding_values)).as(mlir.Attribute).? }, - }, - .location = loc, - }); - - var res: BodyS.Return = inputs; - module.assignResults(&res, null, op); - return res; +pub fn ReduceWindowFn(N: comptime_int) type { + return @Fn( + &@as([N]type, @splat(ReduceArgs)), + &@as([N]std.builtin.Type.Fn.ParamAttributes, @splat(.{})), + [N]Tensor, + .{}, + ); } -/// Runs a given function for several steps, and returns a stack of each step output. -/// The step outputs will be stacked along the first axis. -pub fn for_(comptime func: anytype, blk_ctx: BlockSign(func).BlkCtx, num_steps_: anytype) BlockSign(func).Return { - const num_steps: u32, const step_tag = blk: { - const dims, const tags = Shape.parseDimensions(num_steps_); - meta.assert(dims.len == 1, "zml.for_ only supports one num_step, Received: {any}", .{num_steps_}); - break :blk .{ @intCast(dims.get(0)), tags.get(0) }; +pub fn reduceWindow(N: comptime_int, inputs: [N]Tensor, inits: [N]Tensor, opts: ReduceWindowOpts, func_: *const ReduceWindowFn(N)) [N]Tensor { + const compiler = Compiler.current(); + const mlir_ctx = compiler.mlir_ctx; + + const reduce_block, var result = b: { + const Args = @Tuple(&@as([N]type, @splat(ReduceArgs))); + var args: Args = undefined; + var block_types: [2 * N]*const mlir.Type = undefined; + + inline for (0..N) |i| { + args[i].left = .fromShape(inits[i].shape()); + args[i].right = .fromShape(inits[i].shape()); + + block_types[i] = mlirx.Type.rankedTensor(mlir_ctx, args[i].left.shape()); + block_types[i + N] = mlirx.Type.rankedTensor(mlir_ctx, args[i].right.shape()); + } + + const block_locs: [2 * N]*const mlir.Location = @splat(compiler.unknown_location); + const reduce_block = mlir.Block.init(&block_types, &block_locs); + errdefer reduce_block.deinit(); + + const reduce_scope = compiler.pushBlock(reduce_block); + defer reduce_scope.pop(); + + inline for (0..N) |i| { + reduce_scope.registerTensorAsBlockArgument(args[i].left.id, i); + reduce_scope.registerTensorAsBlockArgument(args[i].right.id, i + N); + } + + var result = @call(.auto, func_, args); + + var result_values: [N]*const mlir.Value = undefined; + inline for (0..N) |i| { + result_values[i] = result[i].value(); + } + + _ = stablehlo.returns(mlir_ctx, &result_values, compiler.location).appendTo(reduce_block); + break :b .{ reduce_block, result }; }; - const S = comptime BlockSign(func); - - const ForBlk = struct { - blk_ctx: S.BlkCtx, - step_tag: @TypeOf(step_tag), // This is a Shape.Tag, but we rather keep it private - num_steps: u32, - const Self = @This(); - - fn next(self: Self, res: S.Return, idx: Tensor) struct { S.Return, Tensor } { - const step_res = @call(.auto, func, .{ self.blk_ctx, idx }); - var buf: [@sizeOf(S.Return) * 2]u8 = undefined; - var fba = std.heap.FixedBufferAllocator.init(&buf); - return .{ - meta.zip(updateResBuffer, fba.allocator(), &[_]S.Return{ res, step_res }, .{idx}) catch unreachable, - idx.addConstant(1), - }; - } - - fn done(self: Self, res: S.Return, idx: Tensor) Tensor { - _ = res; - return idx.cmp(.LT, Tensor.scalar(self.num_steps, idx.dtype())); - } - - fn updateResBuffer(inputs: []const Tensor, idx: Tensor) Tensor { - meta.internalAssert(inputs.len == 2, "too many tensors", .{}); - const res, const step_res = inputs[0..2].*; - return res.dynamicUpdateSlice1d(step_res.insertAxes(0, .{._}), 0, idx); - } - - /// Prepare buffer to store all results steps. - fn prep(self: Self, x: Tensor) Tensor { - var shape = x.shape(); - shape._dims.insert(0, self.num_steps) catch unreachable; - shape._tags.insert(0, self.step_tag) catch unreachable; - return Tensor.constant(shape, x.dtype().zero()); - } - - fn wrapFirstStep(x: Tensor, tag_: @TypeOf(step_tag)) Tensor { - var shape = x.shape(); - shape._dims.insert(0, 1) catch unreachable; - shape._tags.insert(0, tag_) catch unreachable; - return x.reshape(shape); - } - }; - - // This first step won't appear in the generated MLIR, - // it's only used to infer the output shapes. - const first_step = @call(.auto, func, .{ blk_ctx, Tensor.scalar(0, .i32) }); - log.debug("for_ first_step: {}", .{first_step}); - // Optimize for small num reps - if (num_steps == 1) { - // return helpers.mapTensors(ForBlk.wrapFirstStep, first_step, .{ step_tag }); - return first_step; + var input_values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inputs.len) |i| { + input_values[i] = inputs[i].value(); } - const allocator = CompilationContext.current()._allocator; - if (num_steps <= 4) { - var steps: [4]S.Return = undefined; - steps[0] = first_step; - for (1..num_steps) |i| { - steps[i] = @call(.auto, func, .{ blk_ctx, Tensor.scalar(i, .i32) }); - } + var init_values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inits.len) |i| init_values[i] = inits[i].value(); - const res = meta.zip(Tensor.stack, allocator, steps[0..num_steps], .{ 0, step_tag }) catch unreachable; - return res; + const reduce_op = stablehlo.reduce_window(mlir_ctx, &input_values, &init_values, reduce_block, opts.window_dimensions, .{ + .window_strides = opts.window_strides, + .base_dilations = opts.base_dilations, + .window_dilations = opts.window_dilations, + .padding = opts.padding, + }, compiler.location).appendTo(compiler.currentScope().block); + + inline for (0..result.len) |i| { + result[i] = Tensor.fromMlirValue(reduce_op.result(i)).withTags(inputs[i].shape()); } - const for_blk: ForBlk = .{ .blk_ctx = blk_ctx, .step_tag = step_tag, .num_steps = num_steps }; - var result_buffers: @TypeOf(first_step) = undefined; - try meta.mapAlloc(ForBlk.prep, allocator, for_blk, first_step, &result_buffers); - - return while_( - ForBlk.done, - ForBlk.next, - for_blk, - .{ - result_buffers, - Tensor.scalar(0, .i32), - }, - )[0]; + return result; } -test for_ { - const Squares = struct { - const Squares = @This(); +pub const SortArgs = struct { + left: Tensor, + right: Tensor, +}; - pub fn sq(self: Squares, i: Tensor) Tensor { - _ = self; - const f = i.convert(.f32); - return f.mul(f); +pub fn sort(inputs: anytype, axis_: i64, comptime func_: anytype, context: anytype, is_stable: bool) [inputs.len]Tensor { + const compiler = Compiler.current(); + const mlir_ctx = compiler.mlir_ctx; + + const sort_block = b: { + const ArgsTypes: [inputs.len]type = @splat(SortArgs); + const ArgsType = @Tuple(&ArgsTypes); + var args: ArgsType = undefined; + var block_types: [2 * inputs.len]*const mlir.Type = undefined; + + inline for (0..inputs.len) |i| { + args[i].left = Tensor.init(.{}, inputs[i].shape().dtype()); + args[i].right = Tensor.init(.{}, inputs[i].shape().dtype()); + + block_types[2 * i] = mlirx.Type.rankedTensor(mlir_ctx, args[i].left.shape()); + block_types[2 * i + 1] = mlirx.Type.rankedTensor(mlir_ctx, args[i].right.shape()); } - pub fn forward(num_steps: u63) Tensor { - return for_(Squares.sq, .{}, .{num_steps}); + const block_locs: [2 * inputs.len]*const mlir.Location = @splat(compiler.unknown_location); + const sort_block = mlir.Block.init(&block_types, &block_locs); + errdefer sort_block.deinit(); + + const scope = compiler.pushBlock(sort_block); + defer scope.pop(); + + inline for (0..inputs.len) |i| { + scope.registerTensorAsBlockArgument(args[i].left.id, 2 * i); + scope.registerTensorAsBlockArgument(args[i].right.id, 2 * i + 1); + } + + var result = @call(.auto, func_, args ++ context); + + _ = stablehlo.return_(mlir_ctx, result.value(), compiler.location).appendTo(sort_block); + break :b sort_block; + }; + + var input_values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inputs.len) |i| { + input_values[i] = inputs[i].value(); + } + + const sort_op = stablehlo.sort(mlir_ctx, &input_values, sort_block, .{ + .dimension = axis_, + .is_stable = is_stable, + }, compiler.location).appendTo(compiler.currentScope().block); + + var result: [inputs.len]Tensor = undefined; + inline for (0..inputs.len) |i| { + result[i] = Tensor._result(inputs[i].shape(), sort_op.result(i)); + } + + return result; +} + +/// Implement a while loop. +/// The given While struct contains all the parameters that are fixed through the while. +/// It must have: +/// * a State struct definition that captures all the loop state that will change on each iteration +/// * a `cond(While, State) Tensor` function returning a scalar boolean tensor +/// * a `body(While, State) State` function that preserves the state shapes. +pub fn @"while"( + While: type, + context: While, + initial_state: While.State, +) While.State { + const comp = Compiler.current(); + + var arena = std.heap.ArenaAllocator.init(comp.allocator); + defer arena.deinit(); + const allocator = arena.allocator(); + + const mlir_ctx = comp.mlir_ctx; + comp.pushLocation(@src(), @typeName(While)); + defer comp.popLocation(); + + // Force to materialize tensor.value() before we push a new scope. + var captured_context: While = undefined; + meta.mapAlloc(struct { + fn capture(_: void, tensor: Tensor) Tensor { + return Tensor._result(tensor.shape(), tensor.value()); + } + }.capture, allocator, {}, context, &captured_context) catch unreachable; + + const flat_operands: []*const Tensor = meta.collectPtrs(Tensor, allocator, &initial_state) catch @panic("OOM"); + + const OperandInfo = struct { + type: *const mlir.Type, + location: *const mlir.Location, + value: *const mlir.Value, + }; + + var operands_info = std.MultiArrayList(OperandInfo).initCapacity(allocator, flat_operands.len) catch @panic("OOM"); + + for (flat_operands) |input| { + operands_info.appendAssumeCapacity(.{ + .type = mlirx.Type.rankedTensor(mlir_ctx, input._shape), + .location = comp.location, + .value = input.value(), + }); + } + + const cond_block = b: { + const block = mlir.Block.init(operands_info.items(.type), operands_info.items(.location)); + errdefer block.deinit(); + + const scope = comp.pushBlock(block); + defer scope.pop(); + + comp.pushLocationFmt(@src(), "{s}.cond", .{@typeName(While)}); + defer comp.popLocation(); + + // Interpret initial_state as the block argument + for (0.., flat_operands) |i, input| { + scope.registerTensorAsBlockArgument(input.id, i); + } + + const cond: Tensor = captured_context.cond(initial_state); + stdx.debug.assert(cond.rank() == 0 and cond.dtype() == .bool, "zml.ops.while expects cond to return a scalar bool Tensor, got {f}", .{cond}); + + _ = stablehlo.return_(mlir_ctx, cond.value(), comp.location).appendTo(block); + break :b block; + }; + + const body_block, var result = b: { + const block = mlir.Block.init(operands_info.items(.type), operands_info.items(.location)); + errdefer block.deinit(); + + const scope = comp.pushBlock(block); + defer scope.pop(); + + comp.pushLocationFmt(@src(), "{s}.body", .{@typeName(While)}); + defer comp.popLocation(); + + // Interpret operands as the block argument + scope.id_to_argument.ensureUnusedCapacity(scope.arena.allocator(), flat_operands.len) catch @panic("OOM"); + for (0.., flat_operands) |i, input| { + scope.registerTensorAsBlockArgument(input.id, i); + } + + const result: While.State = captured_context.body(initial_state); + + if (builtin.mode == .debug) { + const result_tensors = meta.collectPtrs(Tensor, allocator, &result) catch @panic("OOM"); + stdx.debug.assert(flat_operands.len == result_tensors.len, "zml.ops.while({s}) expects body to return {d} Tensor values, got {d} values in ", .{ @typeName(While), flat_operands.len, result_tensors.len }); + for (0.., flat_operands, result_tensors) |i, in, out| { + stdx.debug.assert(in.shape().eql(out.shape()), "zml.ops.while({s}) expects body to preserve loop state shape, got {f} for operand {d} but expected {f}", .{ @typeName(While), in, i, out }); + } + } + + const result_values = meta.collectAlloc(Tensor.value, {}, allocator, &result) catch @panic("OOM"); + _ = stablehlo.returns(mlir_ctx, result_values, comp.location).appendTo(block); + break :b .{ block, result }; + }; + + const op = stablehlo.while_( + mlir_ctx, + operands_info.items(.value), + operands_info.items(.type), + cond_block, + body_block, + comp.location, + ).appendTo(comp.currentScope().block); + + const AssignResultCtx = struct { + op_: *mlir.Operation, + og_tensors: []*const Tensor, + idx: usize = 0, + + fn assign(ctx: *@This(), tensor: *Tensor) void { + tensor.* = Tensor._result(ctx.og_tensors[ctx.idx].shape(), ctx.op_.result(ctx.idx)); + ctx.idx += 1; } }; + var assign_result_ctx: AssignResultCtx = .{ + .op_ = op, + .og_tensors = flat_operands, + }; + meta.visit(AssignResultCtx.assign, &assign_result_ctx, &result); + + return result; +} + +pub const while_ = @"while"; + +test @"while" { const zml = @import("zml.zig"); const platform = zml.testing.env(); - // Just one baby step - { - const squares = try zml.testing.compileAndCall(platform, Squares.forward, .{1}); - try std.testing.expectEqual(0, squares.getValue(f32)); - } - // Wow 4 in rows ! - { - const squares = try zml.testing.compileAndCall(platform, Squares.forward, .{4}); - try std.testing.expectEqual([_]f32{ 0, 1, 4, 9 }, try squares.getValue([4]f32)); - } - // AGI is coming, computing 10 squares as it's nothing. - { - const squares = try zml.testing.compileAndCall(platform, Squares.forward, .{10}); - try std.testing.expectEqual( - [_]f32{ 0, 1, 4, 9, 16, 25, 36, 49, 64, 81 }, - try squares.getValue([10]f32), - ); - } + const While = struct { + // While struct can contain tensor and non-tensor fields. + limit: i64, + step: Tensor, + + pub const State = struct { + i: zml.Tensor, + sum: zml.Tensor, + }; + + pub fn cond(self: @This(), state: State) Tensor { + return state.i.cmp(.LT, Tensor.scalar(self.limit, .i64)); + } + + pub fn body(self: @This(), state: State) State { + return .{ + .i = state.i.add(self.step), + .sum = state.sum.add(self.step), + }; + } + + fn forward(i: Tensor, sum: Tensor) State { + return zml.ops.@"while"( + @This(), + .{ .limit = 10, .step = .scalar(1, .i64) }, + .{ .i = i, .sum = sum }, + ); + } + }; + + const initial_i: zml.Tensor = .init(.{}, .i64); + const initial_sum: zml.Tensor = .init(.{}, .i64); + var exe = try zml.module.compile( + std.testing.allocator, + std.testing.io, + While.forward, + .{ initial_i, initial_sum }, + platform, + .{}, + ); + defer exe.deinit(); + + var i_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, initial_i.shape(), std.mem.sliceAsBytes(&[1]i64{1})); + defer i_buffer.deinit(); + var sum_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, initial_sum.shape(), std.mem.sliceAsBytes(&[1]i64{0})); + defer sum_buffer.deinit(); + + var results: zml.Bufferized(While.State) = undefined; + var runner = try exe.runner(std.testing.allocator); + defer runner.deinit(std.testing.allocator); + runner.run(std.testing.io, .{ i_buffer, sum_buffer }, .{&results}, .{ .wait = true }); + defer results.i.deinit(); + defer results.sum.deinit(); + + var cpu_i = try results.i.toSliceAlloc(std.testing.allocator, std.testing.io); + defer cpu_i.free(std.testing.allocator); + var cpu_sum = try results.sum.toSliceAlloc(std.testing.allocator, std.testing.io); + defer cpu_sum.free(std.testing.allocator); + + try std.testing.expectEqual(@as(i64, 10), cpu_i.items(i64)[0]); + try std.testing.expectEqual(@as(i64, 9), cpu_sum.items(i64)[0]); } -pub fn if_2(pred: Tensor, comptime Closure: type, blkctx: BlockSignNoArgs(@field(Closure, "then")).BlkCtx) BlockSignNoArgs(@field(Closure, "then")).Return { - return if_(pred, @field(Closure, "then"), @field(Closure, "else_"), blkctx); -} - -pub fn if_( +/// Implement if/else branches. +/// The given If struct contains all the parameters that can be referenced in the if/else branches. +/// +/// It must have: +/// * an `onTrue(If) T` function returning some tensors +/// * an `onFalse(If) T` function returning similar shaped tensors +/// +/// If some expression in `onTrue` or `onFalse` has side effects (like printing), +/// stablehlo guarantees they will only be evaluated in its branch. +pub fn @"if"( + If: type, + if_captures: If, pred: Tensor, - comptime true_branch_fn: anytype, - comptime false_branch_fn: anytype, - blkctx: BlockSignNoArgs(true_branch_fn).BlkCtx, -) BlockSignNoArgs(true_branch_fn).Return { - const TrueBlockSignature = comptime BlockSignNoArgs(true_branch_fn); - const FalseBlockSignature = comptime BlockSignNoArgs(false_branch_fn); - if (TrueBlockSignature.Return != FalseBlockSignature.Return) { - @compileError("true_branch_fn and false_branch_fn return types don't match ! " ++ @typeName(TrueBlockSignature.Return) ++ " and " ++ @typeName(FalseBlockSignature.Return)); - } - const ctx = CompilationContext.current(); - const true_branch_block = ctx.makeBlock(TrueBlockSignature, &true_branch_fn, blkctx, {}); - const false_branch_block = ctx.makeBlock(TrueBlockSignature, &false_branch_fn, blkctx, {}); - const loc = ctx.mlirCtx().location(@src()); +) stdx.meta.FnReturn(If.onTrue) { + stdx.debug.assertComptime( + stdx.meta.FnReturn(If.onTrue) == stdx.meta.FnReturn(If.onFalse), + "zml.ops.if expects same return types for onTrue and onFalse, got {s} and {s}", + .{ @typeName(stdx.meta.FnReturn(If.onTrue)), @typeName(stdx.meta.FnReturn(If.onFalse)) }, + ); - const op = mlir.Operation.make(ctx.mlirCtx(), "stablehlo.if", .{ - .operands = &.{pred.value()}, - .result_type_inference = true, - .blocks = &.{ true_branch_block, false_branch_block }, - // We can't verify right away, cause the weights captured by the if haven't been added yet. - .verify = false, - .location = loc, - }); + stdx.debug.assert(pred.dtype() == .bool and pred.count() == 1, "zml.ops.if expects the condition to have exactly one element of dtype .bool, got {f}", .{pred}); - var res: TrueBlockSignature.Return = undefined; - module.assignResults(&res, null, op); - return res; + const compiler = Compiler.current(); + var arena = std.heap.ArenaAllocator.init(compiler.allocator); + defer arena.deinit(); + + const allocator = arena.allocator(); + var blkctx: If = undefined; + + // Explicitly capture values from the parent block. + meta.mapAlloc(struct { + fn capture(_: void, tensor: Tensor) Tensor { + return Tensor._result(tensor.shape(), tensor.value()); + } + }.capture, arena.allocator(), {}, if_captures, &blkctx) catch unreachable; + + const mlir_ctx = Compiler.current().mlir_ctx; + const loc: *const mlir.Location = compiler.location; + + compiler.pushLocation(@src(), @typeName(If)); + defer compiler.popLocation(); + + const true_branch, const true_branch_block = b: { + const block = mlir.Block.init(&.{}, &.{}); + errdefer block.deinit(); + + const scope = Compiler.current().pushBlock(block); + defer scope.pop(); + + const result = blkctx.onTrue(); + const result_values = meta.collectAlloc(Tensor.value, {}, allocator, &result) catch @panic("OOM"); + defer allocator.free(result_values); + _ = stablehlo.returns(mlir_ctx, result_values, loc).appendTo(block); + break :b .{ result, block }; + }; + + const false_branch, const false_branch_block = b: { + const block = mlir.Block.init(&.{}, &.{}); + errdefer block.deinit(); + + const scope = Compiler.current().pushBlock(block); + defer scope.pop(); + + const result = blkctx.onFalse(); + const result_values = meta.collectAlloc(Tensor.value, {}, allocator, &result) catch @panic("OOM"); + defer allocator.free(result_values); + _ = stablehlo.returns(mlir_ctx, result_values, loc).appendTo(block); + break :b .{ result, block }; + }; + // TODO check that false branch values matches right branch + _ = false_branch; + + const op = stablehlo.if_(mlir_ctx, pred.asScalar().value(), true_branch_block, false_branch_block, loc); + _ = op.appendTo(Compiler.current().currentScope().block); + + return fromMlirOperationWithTags(op, true_branch); } test "if" { @@ -422,217 +817,2240 @@ test "if" { const allocator = std.testing.allocator; const IfMod = struct { - pub fn forward(pred: Tensor, a: Tensor, b: Tensor) Tensor { - const result = if_(pred.convert(.bool), condTrue, condFalse, .{ a, b }); - return result; + a: Tensor, + b: Tensor, + + pub fn _fwd(pred: Tensor, a: Tensor, b: Tensor) Tensor { + return @"if"(@This(), .{ .a = a, .b = b }, pred.convert(.bool)); } - pub fn condTrue(a: Tensor, b: Tensor) Tensor { - return a.matmul(b); + pub fn onTrue(ctx: @This()) Tensor { + return ctx.a.dotGeneral(ctx.b, &.{.{ 1, 0 }}, &.{}); } - pub fn condFalse(a: Tensor, b: Tensor) Tensor { - return b.matmul(a); + pub fn onFalse(ctx: @This()) Tensor { + return ctx.b.dotGeneral(ctx.a, &.{.{ 1, 0 }}, &.{}); } }; { - const pred = Shape.init(.{}, .i32); - const a = Shape.init(.{ 4, 4 }, .f32); - const b = Shape.init(.{ 4, 4 }, .f32); - const mod = try zml.compileFn(allocator, IfMod.forward, .{ pred, a, b }, platform); + const pred: Tensor = .init(.{}, .i32); + const a: Tensor = .init(.{ 8, 8 }, .f32); + const b: Tensor = .init(.{ 8, 8 }, .f32); + const mod = try platform.compileFn(allocator, std.testing.io, IfMod._fwd, .{ pred, a, b }, .{}); defer mod.deinit(); } } -pub fn sort( - comptime comp_fn: anytype, - blkctx: BlockSign(comp_fn).BlkCtx, - inputs: [@divExact(BlockSign(comp_fn).nIn, 2)]Tensor, - dimension: i64, - is_stable: bool, -) [@divExact(BlockSign(comp_fn).nIn, 2)]Tensor { - const BodyS = comptime BlockSign(comp_fn); - var inits: BlockSign(comp_fn).Args = undefined; - inline for (0..@divExact(BlockSign(comp_fn).nIn, 2)) |i| { - const arg_shape = Shape.init(.{}, inputs[i].dtype()); - // Note: the id doesn't matter cause makeBlock will correctly fill it. - inits[i * 2] = Tensor{ ._shape = arg_shape, ._id = undefined, ._donation = .no_buffer }; - inits[i * 2 + 1] = Tensor{ ._shape = arg_shape, ._id = undefined, ._donation = .no_buffer }; +/// Simpler variant of `zml.ops.if` that assumes code-motion is supported by the backend. +/// The two branches are evaluated before the condition, then the condition chose which branches to keep. +pub fn if2( + pred: Tensor, + on_true: anytype, + on_false: @TypeOf(on_true), +) @TypeOf(on_true) { + stdx.debug.assert(pred.dtype() == .bool and pred.count() == 1, "zml.ops.if expects the condition to have exactly one element of dtype .bool, got {f}", .{pred}); + + var compiler = Compiler.current(); + + var arena = std.heap.ArenaAllocator.init(compiler.allocator); + defer arena.deinit(); + + const allocator = arena.allocator(); + + const mlir_ctx = compiler.mlir_ctx; + const loc: *const mlir.Location = compiler.location; + + const true_values = meta.collectAlloc(Tensor.value, {}, allocator, &on_true) catch @panic("OOM"); + defer allocator.free(true_values); + const true_branch_block = b: { + const block = mlir.Block.init(&.{}, &.{}); + errdefer block.deinit(); + + const scope = compiler.pushBlock(block); + defer scope.pop(); + _ = stablehlo.returns(mlir_ctx, true_values, loc).appendTo(block); + break :b block; + }; + + const false_values = meta.collectAlloc(Tensor.value, {}, allocator, &on_false) catch @panic("OOM"); + defer allocator.free(false_values); + const false_branch_block = b: { + const block = mlir.Block.init(&.{}, &.{}); + errdefer block.deinit(); + + const scope = compiler.pushBlock(block); + defer scope.pop(); + + _ = stablehlo.returns(mlir_ctx, false_values, loc).appendTo(block); + break :b block; + }; + + const op = stablehlo.if_(mlir_ctx, pred.asScalar().value(), true_branch_block, false_branch_block, loc); + _ = op.appendTo(compiler.currentScope().block); + + return fromMlirOperationWithTags(op, on_true); +} + +test if2 { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + const allocator = std.testing.allocator; + + const IfMod = struct { + pub fn _fwd(pred: Tensor, a: Tensor, b: Tensor) Tensor { + return if2( + pred.convert(.bool), + a.dotGeneral(b, &.{.{ 1, 0 }}, &.{}), + b.dotGeneral(a, &.{.{ 1, 0 }}, &.{}), + ); + } + }; + + { + const pred: Tensor = .init(.{}, .i32); + const a: Tensor = .init(.{ 8, 8 }, .f32); + const b: Tensor = .init(.{ 8, 8 }, .f32); + const mod = try platform.compileFn(allocator, std.testing.io, IfMod._fwd, .{ pred, a, b }, .{}); + defer mod.deinit(); } - const ctx = CompilationContext.current(); - const block = ctx.makeBlock(BodyS, &comp_fn, blkctx, inits); - var input_values: [@divExact(BodyS.nIn, 2)]mlir.Value = undefined; - ctx.extractValues(&inputs, &input_values); +} - const loc = ctx.mlirCtx().location(@src()).namedFmt(ctx.mlirCtx(), "sort(dimension={d}, is_stable={})", .{ dimension, is_stable }); +/// Create a Tensor struct similar to base, keeping base tags, +/// but using mlir value and dims from the mlir operation. +fn fromMlirOperationWithTags(op: *const mlir.Operation, base: anytype) @TypeOf(base) { + const LocalContext = struct { + index: usize, + op: *const mlir.Operation, + }; + var context = LocalContext{ .index = 0, .op = op }; + var res = base; + meta.visit((struct { + fn cb(inner_ctx: *LocalContext, tensor: *Tensor) void { + var new = Tensor.fromMlirValue(inner_ctx.op.result(inner_ctx.index)); + stdx.debug.internalAssert(new.rank() == tensor.rank(), "expected operand result to have rank {} but got {f}", .{ tensor.rank(), new }); + // Copy tags; the partitioner infers result sharding. + // some ops can change dims eg reduceWindow, so we trust mlir here. + new._shape._tags = tensor._shape._tags; + tensor.* = new; + inner_ctx.index += 1; + } + }).cb, &context, &res); + std.debug.assert(context.index == op.numResults()); + return res; +} - const op = mlir.Operation.make(ctx.mlirCtx(), "stablehlo.sort", .{ - .variadic_operands = &.{&input_values}, - .result_type_inference = true, - .blocks = &.{block}, - .attributes = &.{ - .{ "dimension", mlir.IntegerAttribute(.i64).init(ctx.mlirCtx(), dimension).as(mlir.Attribute).? }, - .{ "is_stable", mlir.BoolAttribute.init(ctx.mlirCtx(), is_stable).as(mlir.Attribute).? }, - }, - .location = loc, +pub const TritonOps = struct { + debug: bool = false, + name: []const u8, + ir: []const u8, + grid: [3]i32, + num_stages: i32, + num_warps: i32, + is_tma_allowed: bool = false, + global_scratch_memory_size: i32 = 0, + output_operand_aliases: []const stablehlo.CustomCallOpts.OutputOperandAlias = &.{}, +}; + +/// Generate an MLIR call to the given member function with the given tensors. +pub fn triton(inputs: anytype, outputs: anytype, opts: TritonOps) [outputs.len]Tensor { + const compiler = Compiler.current(); + const mlir_ctx = compiler.mlir_ctx; + var arena = std.heap.ArenaAllocator.init(compiler.allocator); + defer arena.deinit(); + + var values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inputs.len) |i| { + values[i] = inputs[i].value(); + } + + var res_types: [outputs.len]*const mlir.Type = undefined; + inline for (outputs, 0..) |output, i| { + res_types[i] = mlirx.Type.rankedTensor(mlir_ctx, output); + } + + const backend_config: *const mlir.Attribute = .dict(mlir_ctx, &.{ + .named(mlir_ctx, "name", .string(mlir_ctx, opts.name)), + .named(mlir_ctx, "ir", .string(mlir_ctx, opts.ir)), + .named(mlir_ctx, "grid_x", .int(mlir_ctx, .i32, opts.grid[0])), + .named(mlir_ctx, "grid_y", .int(mlir_ctx, .i32, opts.grid[1])), + .named(mlir_ctx, "grid_z", .int(mlir_ctx, .i32, opts.grid[2])), + .named(mlir_ctx, "num_stages", .int(mlir_ctx, .i32, opts.num_stages)), + .named(mlir_ctx, "num_warps", .int(mlir_ctx, .i32, opts.num_warps)), + .named(mlir_ctx, "is_tma_allowed", .boolean(mlir_ctx, opts.is_tma_allowed)), + .named(mlir_ctx, "global_scratch_memory_size", .int(mlir_ctx, .i32, opts.global_scratch_memory_size)), }); - var res: [@divExact(BlockSign(comp_fn).nIn, 2)]Tensor = undefined; - inline for (0..@divExact(BlockSign(comp_fn).nIn, 2)) |i| { - res[i] = Tensor._result(inputs[i].shape(), op.result(i)); + var operands_layouts: [inputs.len][]const usize = undefined; + inline for (inputs, 0..) |input, i| { + operands_layouts[i] = arena.allocator().dupe(usize, toUsize(constants.minorToMajor(input.rank())).constSlice()) catch unreachable; + } + + var results_layouts: [outputs.len][]const usize = undefined; + inline for (outputs, 0..) |output, i| { + results_layouts[i] = arena.allocator().dupe(usize, toUsize(constants.minorToMajor(output.rank())).constSlice()) catch unreachable; + } + + const op = stablehlo.custom_call( + mlir_ctx, + &values, + &res_types, + .{ + .call_target_name = "__gpu$xla.gpu.triton", + .backend_config = .{ .typed_ffi = backend_config }, + .has_side_effect = false, + .operand_layouts = &operands_layouts, + .result_layouts = &results_layouts, + .output_operand_aliases = opts.output_operand_aliases, + }, + compiler.location, + ).appendTo(compiler.currentScope().block); + + var outputs_: [outputs.len]Tensor = undefined; + inline for (outputs, 0..) |output, i| { + outputs_[i] = Tensor._result(output, op.result(i)); + } + + return outputs_; +} + +pub const FlyOps = struct { + name: []const u8, + /// The whole module text: a `gpu.func @name ... kernel` inside a + /// `gpu.module`, one `!fly.ptr` per operand then per result; or a + /// top-level `func.func @name` in the destination-passing tensor ABI. + ir: []const u8, + grid: [3]i32, + /// Threads per block = num_warps * 64 on CDNA. + num_warps: i32, + waves_per_eu: i32 = 0, + /// Required when the kernel uses `fly.get_dyn_shared`. + shared_mem_bytes: i64 = 0, + /// Indices into the full argument list, operands first, so result `i` is + /// `inputs.len + i`. A no-op inside HIP graphs: a kernel that must see + /// zeroed outputs should zero them itself. + zeroed_args: []const i32 = &.{}, + output_operand_aliases: []const stablehlo.CustomCallOpts.OutputOperandAlias = &.{}, +}; + +pub fn fly(inputs: anytype, outputs: anytype, opts: FlyOps) [outputs.len]Tensor { + const compiler = Compiler.current(); + const mlir_ctx = compiler.mlir_ctx; + var arena = std.heap.ArenaAllocator.init(compiler.allocator); + defer arena.deinit(); + + var values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inputs.len) |i| { + values[i] = inputs[i].value(); + } + + var res_types: [outputs.len]*const mlir.Type = undefined; + inline for (outputs, 0..) |output, i| { + res_types[i] = mlirx.Type.rankedTensor(mlir_ctx, output); + } + + // The plugin silently ignores unknown or wrong-typed optional keys, so a + // misspelled one here is not an error, just an override that never lands. + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 10) = .empty; + attrs.appendSliceAssumeCapacity(&.{ + .named(mlir_ctx, "name", .string(mlir_ctx, opts.name)), + .named(mlir_ctx, "ir", .string(mlir_ctx, opts.ir)), + .named(mlir_ctx, "num_warps", .int(mlir_ctx, .i32, opts.num_warps)), + .named(mlir_ctx, "grid_x", .int(mlir_ctx, .i32, opts.grid[0])), + .named(mlir_ctx, "grid_y", .int(mlir_ctx, .i32, opts.grid[1])), + .named(mlir_ctx, "grid_z", .int(mlir_ctx, .i32, opts.grid[2])), + }); + if (opts.waves_per_eu > 0) attrs.appendAssumeCapacity(.named(mlir_ctx, "waves_per_eu", .int(mlir_ctx, .i32, opts.waves_per_eu))); + if (opts.shared_mem_bytes > 0) attrs.appendAssumeCapacity(.named(mlir_ctx, "shared_mem_bytes", .int(mlir_ctx, .i64, opts.shared_mem_bytes))); + if (opts.zeroed_args.len > 0) { + const Opts = stablehlo.CustomCallOpts; + var zeroed: stdx.BoundedArray(*const mlir.Attribute, Opts.MAX_OPERANDS + Opts.MAX_RESULTS) = .empty; + for (opts.zeroed_args) |i| { + std.debug.assert(i >= 0 and i < inputs.len + outputs.len); + zeroed.appendAssumeCapacity(.int(mlir_ctx, .i32, i)); + } + attrs.appendAssumeCapacity(.named(mlir_ctx, "zeroed_outputs", .array(mlir_ctx, zeroed.constSlice()))); + } + const backend_config: *const mlir.Attribute = .dict(mlir_ctx, attrs.constSlice()); + + // The kernel sees raw pointers and assumes row-major: pin the layouts. + var operands_layouts: [inputs.len][]const usize = undefined; + inline for (inputs, 0..) |input, i| { + operands_layouts[i] = arena.allocator().dupe(usize, toUsize(constants.minorToMajor(input.rank())).constSlice()) catch unreachable; + } + var results_layouts: [outputs.len][]const usize = undefined; + inline for (outputs, 0..) |output, i| { + results_layouts[i] = arena.allocator().dupe(usize, toUsize(constants.minorToMajor(output.rank())).constSlice()) catch unreachable; + } + + const op = stablehlo.custom_call( + mlir_ctx, + &values, + &res_types, + .{ + .call_target_name = "__gpu$xla.gpu.fly", + .backend_config = .{ .typed_ffi = backend_config }, + .has_side_effect = false, + .operand_layouts = &operands_layouts, + .result_layouts = &results_layouts, + .output_operand_aliases = opts.output_operand_aliases, + }, + compiler.location, + ).appendTo(compiler.currentScope().block); + + var outputs_: [outputs.len]Tensor = undefined; + inline for (outputs, 0..) |output, i| { + outputs_[i] = Tensor._result(output, op.result(i)); + } + + return outputs_; +} + +test "fly custom call, both entry ABIs" { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + if (platform.target != .rocm) return error.SkipZigTest; + + const allocator = std.testing.allocator; + + // Compile `forward` over one f32 input and bring the result back. + const call = struct { + fn f(comptime forward: anytype, p: *const zml.Platform, t: Tensor, h: []const f32) !zml.Slice { + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{t}, p, .{}); + defer exe.deinit(); + var buf: zml.Buffer = try .fromBytes(std.testing.io, p, t.shape(), std.mem.sliceAsBytes(h)); + defer buf.deinit(); + var out = try exe.eval(std.testing.allocator, std.testing.io, .{buf}); + defer out.deinit(); + return out.toSliceAlloc(std.testing.allocator, std.testing.io); + } + }.f; + + // Native Fly ABI: a `gpu.func ... kernel` taking one !fly.ptr per operand + // then per result. This is what `zml.kernel.fly` emits. + { + const Mod = struct { + const ir = + \\module attributes {gpu.container_module} { + \\ gpu.module @zml_fly_kernels { + \\ gpu.func @add_one(%a: !fly.ptr, %c: !fly.ptr) kernel { + \\ %tid = gpu.thread_id x + \\ %step = gpu.block_dim x + \\ %n = arith.constant 128 : index + \\ scf.for %i = %tid to %n step %step { + \\ %t = arith.index_cast %i : index to i32 + \\ %off = fly.make_int_tuple(%t) : (i32) -> !fly.int_tuple + \\ %pa = fly.add_offset(%a, %off) : (!fly.ptr, !fly.int_tuple) -> !fly.ptr + \\ %pc = fly.add_offset(%c, %off) : (!fly.ptr, !fly.int_tuple) -> !fly.ptr + \\ %va = fly.ptr.load(%pa) : (!fly.ptr) -> f32 + \\ %one = arith.constant 1.0 : f32 + \\ %vc = arith.addf %va, %one : f32 + \\ fly.ptr.store(%vc, %pc) : (f32, !fly.ptr) -> () + \\ } + \\ gpu.return + \\ } + \\ } + \\} + ; + pub fn forward(a: Tensor) Tensor { + return fly(.{a}, .{a.shape()}, .{ .name = "add_one", .ir = ir, .grid = .{ 1, 1, 1 }, .num_warps = 2 })[0]; + } + }; + + var input: [128]f32 = undefined; + for (&input, 0..) |*v, i| v.* = @floatFromInt(i); + + var host = try call(Mod.forward, platform, .init(.{ .n = 128 }, .f32), &input); + defer host.free(allocator); + for (host.items(f32), 0..) |v, i| try std.testing.expectEqual(@as(f32, @floatFromInt(i)) + 1, v); + } + + { + const Mod = struct { + const ir = + \\module { + \\ func.func @add_one(%input: tensor<64xf32>, %output: tensor<64xf32>) -> tensor<64xf32> { + \\ %thread = gpu.thread_id x + \\ %last = arith.constant 63 : index + \\ %i = arith.minui %thread, %last : index + \\ %value = tensor.extract %input[%i] : tensor<64xf32> + \\ %one = arith.constant 1.0 : f32 + \\ %sum = arith.addf %value, %one : f32 + \\ %updated = tensor.insert %sum into %output[%i] : tensor<64xf32> + \\ return %updated : tensor<64xf32> + \\ } + \\} + ; + pub fn forward(a: Tensor) Tensor { + return fly(.{a}, .{a.shape()}, .{ .name = "add_one", .ir = ir, .grid = .{ 1, 1, 1 }, .num_warps = 2 })[0]; + } + }; + + var input: [64]f32 = undefined; + for (&input, 0..) |*v, i| v.* = @floatFromInt(i); + + var host = try call(Mod.forward, platform, .init(.{ .n = 64 }, .f32), &input); + defer host.free(allocator); + for (host.items(f32), 0..) |v, i| try std.testing.expectEqual(@as(f32, @floatFromInt(i + 1)), v); + } +} + +pub const NeuronNkiOps = struct { + name: []const u8, + entrypoint: []const u8, + source_path: []const u8, + compiler_target: []const u8, + has_side_effect: bool = false, + output_operand_aliases: []const stablehlo.CustomCallOpts.OutputOperandAlias = &.{}, +}; + +/// Generate an MLIR call for a Neuron NKI kernel. +/// +/// This API is Neuron-only: the source is compiled to an +/// `AwsNeuronCustomNativeKernel` backend config while emitting the graph. +pub fn neuronNki(inputs: anytype, outputs: anytype, opts: NeuronNkiOps) [outputs.len]Tensor { + const ctx = Compiler.current(); + switch (ctx.platform.target) { + .neuron => {}, + .cpu, .cuda, .rocm, .tpu, .oneapi, .metal, .furiosa => { + stdx.debug.panic("neuronNki is only available on Neuron, got {s}", .{@tagName(ctx.platform.target)}); + }, + } + if (comptime !platforms.isEnabled(.neuron)) unreachable; + const nki_kernel = @import("platforms/neuron/nki_kernel"); + + const mlir_ctx = ctx.mlir_ctx; + var arena = std.heap.ArenaAllocator.init(ctx.allocator); + defer arena.deinit(); + + const allocator = arena.allocator(); + + var values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inputs.len) |i| { + values[i] = inputs[i].value(); + } + + var res_types: [outputs.len]*const mlir.Type = undefined; + inline for (outputs, 0..) |output, i| { + res_types[i] = mlirx.Type.rankedTensor(mlir_ctx, output); + } + + var operands_layouts: [inputs.len][]const usize = undefined; + inline for (inputs, 0..) |input, i| { + operands_layouts[i] = allocator.dupe(usize, toUsize(constants.minorToMajor(input.rank())).constSlice()) catch unreachable; + } + + var results_layouts: [outputs.len][]const usize = undefined; + inline for (outputs, 0..) |output, i| { + results_layouts[i] = allocator.dupe(usize, toUsize(constants.minorToMajor(output.rank())).constSlice()) catch unreachable; + } + + var input_signatures: [inputs.len]nki_kernel.TensorSignature = undefined; + inline for (inputs, 0..) |input, i| { + input_signatures[i] = .{ + .dtype = @tagName(input.dtype()), + .dims = input.dims(), + }; + } + var output_signatures: [outputs.len]nki_kernel.TensorSignature = undefined; + inline for (outputs, 0..) |output, i| { + output_signatures[i] = .{ + .dtype = @tagName(output.dtype()), + .dims = output.dims(), + }; + } + + const compiled_backend_config = nki_kernel.compileNkiKernel(ctx.allocator, ctx.io, .{ + .name = opts.name, + .entrypoint = opts.entrypoint, + .source_path = opts.source_path, + .compiler_target = opts.compiler_target, + .inputs = &input_signatures, + .outputs = &output_signatures, + }) catch |err| { + stdx.debug.panic("failed to compile Neuron NKI kernel: {}", .{err}); + }; + defer ctx.allocator.free(compiled_backend_config); + + const op = stablehlo.custom_call( + mlir_ctx, + &values, + &res_types, + .{ + .call_target_name = "AwsNeuronCustomNativeKernel", + .has_side_effect = opts.has_side_effect, + .operand_layouts = &operands_layouts, + .result_layouts = &results_layouts, + .output_operand_aliases = opts.output_operand_aliases, + .additional_attributes = &.{ + .named(mlir_ctx, "backend_config", .string(mlir_ctx, compiled_backend_config)), + }, + }, + ctx.location, + ).appendTo(ctx.currentScope().block); + + var outputs_: [outputs.len]Tensor = undefined; + inline for (outputs, 0..) |output, i| { + outputs_[i] = Tensor._result(output, op.result(i)); + } + + return outputs_; +} + +test "triton" { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + if (platform.target != .cuda and platform.target != .rocm and platform.target != .oneapi) return error.SkipZigTest; + + const ir = + \\ module { + \\ tt.func public @add_one(%arg0: !tt.ptr {tt.divisibility = 32 : i32}, %arg1: !tt.ptr {tt.divisibility = 32 : i32}, %arg2: !tt.ptr {tt.divisibility = 32 : i32}, %arg3: !tt.ptr {tt.divisibility = 32 : i32}) { + \\ %0 = tt.get_program_id x : i32 + \\ %1 = tt.load %arg0 {cache = 1 : i32, evict = 1 : i32, isVolatile = false} : !tt.ptr + \\ %2 = tt.load %arg1 {cache = 1 : i32, evict = 1 : i32, isVolatile = false} : !tt.ptr + \\ %cst = arith.constant 1.000000e+00 : f32 + \\ %3 = arith.addf %1, %cst : f32 + \\ tt.store %arg2, %3 {cache = 1 : i32, evict = 1 : i32} : !tt.ptr + \\ tt.store %arg3, %2 {cache = 1 : i32, evict = 1 : i32} : !tt.ptr + \\ tt.return + \\ } + \\ } + ; + + const TritonMod = struct { + pub fn forward(a: Tensor, b: Tensor) [2]Tensor { + return triton(.{ a, b }, .{ a.shape(), b.shape() }, .{ + .debug = false, + .name = "add_one", + .ir = ir, + .grid = .{ 1, 1, 1 }, + .num_stages = 1, + .num_warps = 1, + }); + } + }; + + const a: zml.Tensor = .init(.{}, .f32); + const b: zml.Tensor = .init(.{}, .f32); + + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, TritonMod.forward, .{ a, b }, platform, .{}); + defer exe.deinit(); + + var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), std.mem.sliceAsBytes(&[1]f32{1})); + defer a_buffer.deinit(); + var b_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, b.shape(), std.mem.sliceAsBytes(&[1]f32{3})); + defer b_buffer.deinit(); + + var results: [2]zml.Buffer = undefined; + var runner = try exe.runner(std.testing.allocator); + defer runner.deinit(std.testing.allocator); + runner.run(std.testing.io, .{ a_buffer, b_buffer }, .{&results}, .{ .wait = true }); + defer results[0].deinit(); + defer results[1].deinit(); + + var cpu_result_0 = try results[0].toSliceAlloc(std.testing.allocator, std.testing.io); + defer cpu_result_0.free(std.testing.allocator); + var cpu_result_1 = try results[1].toSliceAlloc(std.testing.allocator, std.testing.io); + defer cpu_result_1.free(std.testing.allocator); + + const expected_result_a: f32 = 2.0; + const expected_result_b: f32 = 3.0; + + try std.testing.expectEqual(expected_result_a, cpu_result_0.items(f32)[0]); + try std.testing.expectEqual(expected_result_b, cpu_result_1.items(f32)[0]); +} + +pub const CudaTileOps = struct { + name: []const u8, + ir: []const u8, + grid: [3]i32, + /// The Tile IR bytecode version XLA serializes at, "MAJOR.MINOR"; XLA's + /// default is 13.3. + ir_version: ?[]const u8 = null, + /// Result indices XLA zeroes before the launch, ascending. For a kernel + /// that accumulates, or writes less than the whole output. + zeroed_outputs: []const i32 = &.{}, + output_operand_aliases: []const stablehlo.CustomCallOpts.OutputOperandAlias = &.{}, +}; + +/// A `__gpu$xla.gpu.cuda_tile` custom call: XLA assembles `ir` with tileiras +/// and launches `name` over `grid` tile blocks. Every operand and result +/// arrives as one raw device pointer, in order, in the default layout. +pub fn cudaTile(inputs: anytype, outputs: anytype, opts: CudaTileOps) [outputs.len]Tensor { + const compiler = Compiler.current(); + const mlir_ctx = compiler.mlir_ctx; + + var values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inputs.len) |i| { + values[i] = inputs[i].value(); + } + + var res_types: [outputs.len]*const mlir.Type = undefined; + inline for (outputs, 0..) |output, i| { + res_types[i] = mlirx.Type.rankedTensor(mlir_ctx, output); + } + + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 10) = .empty; + attrs.appendSliceAssumeCapacity(&.{ + .named(mlir_ctx, "name", .string(mlir_ctx, opts.name)), + .named(mlir_ctx, "kernel_type", .string(mlir_ctx, "cuda_tile")), + .named(mlir_ctx, "ir", .string(mlir_ctx, opts.ir)), + .named(mlir_ctx, "grid_x", .int(mlir_ctx, .i32, opts.grid[0])), + .named(mlir_ctx, "grid_y", .int(mlir_ctx, .i32, opts.grid[1])), + .named(mlir_ctx, "grid_z", .int(mlir_ctx, .i32, opts.grid[2])), + }); + if (opts.ir_version) |v| attrs.appendAssumeCapacity(.named(mlir_ctx, "ir_version", .string(mlir_ctx, v))); + if (opts.zeroed_outputs.len > 0) { + var zeroed: stdx.BoundedArray(*const mlir.Attribute, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; + for (opts.zeroed_outputs) |i| zeroed.appendAssumeCapacity(.int(mlir_ctx, .i32, i)); + attrs.appendAssumeCapacity(.named(mlir_ctx, "zeroed_outputs", .array(mlir_ctx, zeroed.constSlice()))); + } + const backend_config: *const mlir.Attribute = .dict(mlir_ctx, attrs.constSlice()); + + const op = stablehlo.custom_call( + mlir_ctx, + &values, + &res_types, + .{ + .call_target_name = "__gpu$xla.gpu.cuda_tile", + .backend_config = .{ .typed_ffi = backend_config }, + .has_side_effect = false, + .output_operand_aliases = opts.output_operand_aliases, + }, + compiler.location, + ).appendTo(compiler.currentScope().block); + + var outputs_: [outputs.len]Tensor = undefined; + inline for (outputs, 0..) |output, i| { + outputs_[i] = Tensor._result(output, op.result(i)); + } + + return outputs_; +} + +pub const TclOps = struct { + ir: []const u8, + /// Outputs the function updates in place in an input's buffer. + output_operand_aliases: []const stablehlo.CustomCallOpts.OutputOperandAlias = &.{}, +}; + +/// A TCL function the Furiosa plugin compiles as its own native unit. +pub fn tcl(inputs: anytype, outputs: anytype, opts: TclOps) [outputs.len]Tensor { + const compiler = Compiler.current(); + const mlir_ctx = compiler.mlir_ctx; + + var values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inputs.len) |i| values[i] = inputs[i].value(); + + var res_types: [outputs.len]*const mlir.Type = undefined; + inline for (outputs, 0..) |output, i| res_types[i] = mlirx.Type.rankedTensor(mlir_ctx, output); + + const op = stablehlo.custom_call(mlir_ctx, &values, &res_types, .{ + .call_target_name = "furiosa.tcl", + .backend_config = .{ .original = opts.ir }, + .has_side_effect = false, + .output_operand_aliases = opts.output_operand_aliases, + }, compiler.location).appendTo(compiler.currentScope().block); + + var outputs_: [outputs.len]Tensor = undefined; + inline for (outputs, 0..) |output, i| outputs_[i] = Tensor._result(output, op.result(i)); + return outputs_; +} + +pub const MetalOps = struct { + /// The kernel symbol. + name: []const u8, + /// The kernel's module, in the AIR dialect. + ir: []const u8, + /// Threadgroups per grid. + grid: [3]u32, + /// Threads per threadgroup. + threadgroup: [3]u32, + /// Bytes of `[[threadgroup(0)]]` memory. + threadgroup_memory: u64 = 0, + /// What each `[[buffer(n)]]` binds: `k >= 0` the XLA buffer `k` (the + /// operands, then the results), `k < 0` the constant `-1 - k`. + arguments: []const i32, + /// The bytes of the constant arguments. + constants: []const []const u8 = &.{}, + output_operand_aliases: []const stablehlo.CustomCallOpts.OutputOperandAlias = &.{}, +}; + +/// A `__gpu$xla.gpu.air` custom call: XLA lowers `ir` to AIR, compiles it to +/// a metallib and launches `name`. +pub fn metal(inputs: anytype, outputs: anytype, opts: MetalOps) [outputs.len]Tensor { + const compiler = Compiler.current(); + const mlir_ctx = compiler.mlir_ctx; + + var values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inputs.len) |i| values[i] = inputs[i].value(); + + var res_types: [outputs.len]*const mlir.Type = undefined; + inline for (outputs, 0..) |output, i| res_types[i] = mlirx.Type.rankedTensor(mlir_ctx, output); + + const allocator = compiler.allocator; + const constant_attrs = allocator.alloc(*const mlir.Attribute, opts.constants.len) catch @panic("OOM"); + defer allocator.free(constant_attrs); + for (opts.constants, constant_attrs) |bytes, *attribute| attribute.* = .denseArray(mlir_ctx, .i8, @as([]const i8, @ptrCast(bytes))); + + const sizes = [2][3]u32{ opts.grid, opts.threadgroup }; + var dims: [2][3]*const mlir.Attribute = undefined; + for (&dims, sizes) |*elems, dim| { + for (elems, dim) |*e, size| e.* = .int(mlir_ctx, .i32, size); + } + const backend_config: *const mlir.Attribute = .dict(mlir_ctx, &.{ + .named(mlir_ctx, "name", .string(mlir_ctx, opts.name)), + .named(mlir_ctx, "ir", .string(mlir_ctx, opts.ir)), + .named(mlir_ctx, "grid", .array(mlir_ctx, &dims[0])), + .named(mlir_ctx, "threadgroup", .array(mlir_ctx, &dims[1])), + .named(mlir_ctx, "threadgroup_memory", .int(mlir_ctx, .i64, @as(i64, @intCast(opts.threadgroup_memory)))), + .named(mlir_ctx, "arguments", .denseArray(mlir_ctx, .i32, opts.arguments)), + .named(mlir_ctx, "constants", .array(mlir_ctx, constant_attrs)), + }); + + const op = stablehlo.custom_call(mlir_ctx, &values, &res_types, .{ + .call_target_name = "__gpu$xla.gpu.air", + .backend_config = .{ .typed_ffi = backend_config }, + .has_side_effect = false, + .output_operand_aliases = opts.output_operand_aliases, + }, compiler.location).appendTo(compiler.currentScope().block); + + var outputs_: [outputs.len]Tensor = undefined; + inline for (outputs, 0..) |output, i| outputs_[i] = Tensor._result(output, op.result(i)); + return outputs_; +} + +pub const CuteOps = struct { + /// The kernel symbol. + name: []const u8, + /// Textual CuTe DSL module: either public `func.func` kernels, launched + /// with `grid`/`block`, or a `gpu.module` plus a host launch function + /// that owns the launch configuration. + ir: []const u8, + grid: ?[3]i32 = null, + block: ?[3]i32 = null, + zeroed_outputs: []const i32 = &.{}, + scalars: []const i64 = &.{}, + symmetric_workspace: ?SymmetricWorkspace = null, + output_operand_aliases: []const stablehlo.CustomCallOpts.OutputOperandAlias = &.{}, + + /// Named per-device workspace shared across CuTe calls and executables. + /// Every rank allocates `bytes`; XLA passes peer addresses and the rank. + pub const SymmetricWorkspace = struct { + name: []const u8, + bytes: i64, + }; +}; + +/// A `__gpu$xla.gpu.cute` custom call: XLA compiles `ir` with the CuTe compiler +/// and launches `name`. Every operand and result arrives as one raw device +/// pointer, in order, in the default layout. +pub fn cute(inputs: anytype, outputs: anytype, opts: CuteOps) [outputs.len]Tensor { + const compiler = Compiler.current(); + + const mlir_ctx = compiler.mlir_ctx; + + var values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inputs.len) |i| { + values[i] = inputs[i].value(); + } + + var res_types: [outputs.len]*const mlir.Type = undefined; + inline for (outputs, 0..) |output, i| { + res_types[i] = mlirx.Type.rankedTensor(mlir_ctx, output); + } + + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 8) = .empty; + attrs.appendSliceAssumeCapacity(&.{ + .named(mlir_ctx, "name", .string(mlir_ctx, opts.name)), + .named(mlir_ctx, "kernel_type", .string(mlir_ctx, "cute")), + .named(mlir_ctx, "ir", .string(mlir_ctx, opts.ir)), + }); + inline for (.{ "grid", "block" }) |key| { + if (@field(opts, key)) |dims| { + var elems: [3]*const mlir.Attribute = undefined; + for (&elems, dims) |*e, d| e.* = .int(mlir_ctx, .i32, d); + attrs.appendAssumeCapacity(.named(mlir_ctx, key, .array(mlir_ctx, &elems))); + } + } + if (opts.zeroed_outputs.len > 0) { + var zeroed: stdx.BoundedArray(*const mlir.Attribute, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; + for (opts.zeroed_outputs) |i| zeroed.appendAssumeCapacity(.int(mlir_ctx, .i32, i)); + attrs.appendAssumeCapacity(.named(mlir_ctx, "zeroed_outputs", .array(mlir_ctx, zeroed.constSlice()))); + } + if (opts.scalars.len > 0) { + const allocator = Compiler.current().allocator; + const scalar_attrs = allocator.alloc(*const mlir.Attribute, opts.scalars.len) catch @panic("OOM"); + defer allocator.free(scalar_attrs); + for (opts.scalars, scalar_attrs) |value, *attribute| attribute.* = .int(mlir_ctx, .i64, value); + attrs.appendAssumeCapacity(.named(mlir_ctx, "scalars", .array(mlir_ctx, scalar_attrs))); + } + if (opts.symmetric_workspace) |workspace| { + stdx.debug.assert(workspace.name.len > 0 and workspace.bytes > 0, "zml.ops.cute({s}): symmetric_workspace needs a name and a positive size", .{opts.name}); + attrs.appendAssumeCapacity(.named(mlir_ctx, "symmetric_workspace", .dict(mlir_ctx, &.{ + .named(mlir_ctx, "name", .string(mlir_ctx, workspace.name)), + .named(mlir_ctx, "bytes", .int(mlir_ctx, .i64, workspace.bytes)), + }))); + } + const backend_config: *const mlir.Attribute = .dict(mlir_ctx, attrs.constSlice()); + + const op = stablehlo.custom_call( + mlir_ctx, + &values, + &res_types, + .{ + .call_target_name = "__gpu$xla.gpu.cute", + .backend_config = .{ .typed_ffi = backend_config }, + .has_side_effect = false, + .output_operand_aliases = opts.output_operand_aliases, + }, + compiler.location, + ).appendTo(compiler.currentScope().block); + + var outputs_: [outputs.len]Tensor = undefined; + inline for (outputs, 0..) |output, i| { + outputs_[i] = Tensor._result(output, op.result(i)); + } + + return outputs_; +} + +test "cute" { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + if (platform.target != .cuda) return error.SkipZigTest; + + // out[i] = in[i] + 1 over 128 f32, in the CuTe DSL's frontend form. + const ir = + \\module attributes {gpu.container_module} { + \\ gpu.module @kernels { + \\ cuda.kernel @add_one(%arg0: !cute.ptr>, %arg1: !cute.ptr>) attributes {cute.kernel, gpu.kernel, nvvm.reqntid = array} { + \\ %i = nvvm.read.ptx.sreg.tid.x : i32 + \\ %shape = cute.make_shape() : () -> !cute.shape<"128"> + \\ %lay = cute.make_layout(%shape) : !cute.layout<"128:1"> + \\ %coord = cute.make_coord(%i) : (i32) -> !cute.coord<"?"> + \\ %in = cute.make_view(%arg0, %lay) : !cute.memref, "128:1"> + \\ %out = cute.make_view(%arg1, %lay) : !cute.memref, "128:1"> + \\ %x = cute.memref.load(%in, %coord) : (!cute.memref, "128:1">, !cute.coord<"?">) -> f32 + \\ %one = arith.constant 1.0 : f32 + \\ %y = arith.addf %x, %one : f32 + \\ cute.memref.store(%out, %coord, %y) : (!cute.memref, "128:1">, !cute.coord<"?">, f32) -> () + \\ return + \\ } + \\ } + \\ func.func @launch(%arg0: !cute.ptr>, %arg1: !cute.ptr>) -> i32 attributes {llvm.emit_c_interface} { + \\ %smem = cute.kernel_smem_size @kernels::@add_one : i64 + \\ %c0_i64 = arith.constant 0 : i64 + \\ %stream = cuda.cast %c0_i64 : i64 -> !cuda.stream + \\ %c1 = arith.constant 1 : i32 + \\ %c128 = arith.constant 128 : i32 + \\ %cfg = cuda.launch_cfg.create (blockDim = (%c128, %c1, %c1), dynamicSmemBytes = %smem, gridDim = (%c1, %c1, %c1), stream = %stream) : i32, i32, i32, i64, i32, i32, i32, !cuda.stream -> !cuda.launch_cfg + \\ %r = cuda.launch_ex @kernels::@add_one<%cfg> (%arg0, %arg1) : !cuda.launch_cfg, (!cute.ptr>, !cute.ptr>) -> !cuda.result + \\ %status = cuda.cast %r : !cuda.result -> i32 + \\ return %status : i32 + \\ } + \\} + ; + + const Mod = struct { + pub fn forward(a: Tensor) Tensor { + return cute(.{a}, .{a.shape()}, .{ .name = "add_one", .ir = ir })[0]; + } + }; + + const a: zml.Tensor = .init(.{ .n = 128 }, .f32); + + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Mod.forward, .{a}, platform, .{}); + defer exe.deinit(); + + var input: [128]f32 = undefined; + for (&input, 0..) |*v, i| v.* = @floatFromInt(i); + var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), std.mem.sliceAsBytes(&input)); + defer a_buffer.deinit(); + + var result = try exe.eval(std.testing.allocator, std.testing.io, .{a_buffer}); + defer result.deinit(); + + var host = try result.toSliceAlloc(std.testing.allocator, std.testing.io); + defer host.free(std.testing.allocator); + + for (host.items(f32), 0..) |v, i| { + try std.testing.expectEqual(@as(f32, @floatFromInt(i + 1)), v); + } +} + +test "cuda_tile" { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + if (platform.target != .cuda) return error.SkipZigTest; + + // out[i] = in[i] + 1 over 128 f32; the XLA milestone kernel. + const ir = + \\cuda_tile.module @m { + \\ entry @add_one(%in : tile>, %out : tile>) { + \\ %offsets = iota : tile<128xi32> + \\ %in_r = reshape %in : tile> -> tile<1xptr> + \\ %in_b = broadcast %in_r : tile<1xptr> -> tile<128xptr> + \\ %in_p = offset %in_b, %offsets : tile<128xptr>, tile<128xi32> -> tile<128xptr> + \\ %v, %t0 = load_ptr_tko weak %in_p : tile<128xptr> -> tile<128xf32>, token + \\ %one = constant : tile + \\ %one_r = reshape %one : tile -> tile<1xf32> + \\ %one_b = broadcast %one_r : tile<1xf32> -> tile<128xf32> + \\ %sum = addf %v, %one_b rounding : tile<128xf32> + \\ %out_r = reshape %out : tile> -> tile<1xptr> + \\ %out_b = broadcast %out_r : tile<1xptr> -> tile<128xptr> + \\ %out_p = offset %out_b, %offsets : tile<128xptr>, tile<128xi32> -> tile<128xptr> + \\ %t1 = store_ptr_tko weak %out_p, %sum : tile<128xptr>, tile<128xf32> -> token + \\ return + \\ } + \\} + ; + + const Mod = struct { + pub fn forward(a: Tensor) Tensor { + return cudaTile(.{a}, .{a.shape()}, .{ + .name = "add_one", + .ir = ir, + .grid = .{ 1, 1, 1 }, + })[0]; + } + }; + + const a: zml.Tensor = .init(.{ .n = 128 }, .f32); + + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Mod.forward, .{a}, platform, .{}); + defer exe.deinit(); + + var input: [128]f32 = undefined; + for (&input, 0..) |*v, i| v.* = @floatFromInt(i); + var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), std.mem.sliceAsBytes(&input)); + defer a_buffer.deinit(); + + var result = try exe.eval(std.testing.allocator, std.testing.io, .{a_buffer}); + defer result.deinit(); + + var host = try result.toSliceAlloc(std.testing.allocator, std.testing.io); + defer host.free(std.testing.allocator); + + for (host.items(f32), 0..) |v, i| { + try std.testing.expectEqual(@as(f32, @floatFromInt(i + 1)), v); + } +} + +test "cuda_tile grid" { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + if (platform.target != .cuda) return error.SkipZigTest; + + // Each of the 2x3x4 tile blocks writes its linear id t = x + 2y + 6z + // over out[128*t .. 128*t+128); the input is unused. + const ir = + \\cuda_tile.module @m { + \\ entry @grid(%in : tile>, %out : tile>) { + \\ %bx, %by, %bz = get_tile_block_id : tile + \\ %c2 = constant : tile + \\ %c6 = constant : tile + \\ %c128 = constant : tile + \\ %y2 = muli %by, %c2 : tile + \\ %z6 = muli %bz, %c6 : tile + \\ %xy = addi %bx, %y2 : tile + \\ %t = addi %xy, %z6 : tile + \\ %base = muli %t, %c128 : tile + \\ %offsets = iota : tile<128xi32> + \\ %base_r = reshape %base : tile -> tile<1xi32> + \\ %base_b = broadcast %base_r : tile<1xi32> -> tile<128xi32> + \\ %idx = addi %offsets, %base_b : tile<128xi32> + \\ %out_r = reshape %out : tile> -> tile<1xptr> + \\ %out_b = broadcast %out_r : tile<1xptr> -> tile<128xptr> + \\ %out_p = offset %out_b, %idx : tile<128xptr>, tile<128xi32> -> tile<128xptr> + \\ %tf = itof %t signed rounding : tile -> tile + \\ %tf_r = reshape %tf : tile -> tile<1xf32> + \\ %tf_b = broadcast %tf_r : tile<1xf32> -> tile<128xf32> + \\ %tok = store_ptr_tko weak %out_p, %tf_b : tile<128xptr>, tile<128xf32> -> token + \\ return + \\ } + \\} + ; + + const Mod = struct { + pub fn forward(a: Tensor) Tensor { + return cudaTile(.{a}, .{a.shape()}, .{ + .name = "grid", + .ir = ir, + .grid = .{ 2, 3, 4 }, + })[0]; + } + }; + + const n = 24 * 128; + const a: zml.Tensor = .init(.{ .n = n }, .f32); + + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Mod.forward, .{a}, platform, .{}); + defer exe.deinit(); + + const input: [n]f32 = @splat(0); + var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), std.mem.sliceAsBytes(&input)); + defer a_buffer.deinit(); + + var result = try exe.eval(std.testing.allocator, std.testing.io, .{a_buffer}); + defer result.deinit(); + + var host = try result.toSliceAlloc(std.testing.allocator, std.testing.io); + defer host.free(std.testing.allocator); + + for (host.items(f32), 0..) |v, i| { + try std.testing.expectEqual(@as(f32, @floatFromInt(i / 128)), v); + } +} + +test "cuda_tile zeroed_outputs" { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + if (platform.target != .cuda) return error.SkipZigTest; + + // Writes only the first 64 of 128; `zeroed_outputs` owns the rest. + const ir = + \\cuda_tile.module @m { + \\ entry @half(%in : tile>, %out : tile>) { + \\ %offsets = iota : tile<64xi32> + \\ %in_r = reshape %in : tile> -> tile<1xptr> + \\ %in_b = broadcast %in_r : tile<1xptr> -> tile<64xptr> + \\ %in_p = offset %in_b, %offsets : tile<64xptr>, tile<64xi32> -> tile<64xptr> + \\ %v, %t0 = load_ptr_tko weak %in_p : tile<64xptr> -> tile<64xf32>, token + \\ %one = constant : tile + \\ %one_r = reshape %one : tile -> tile<1xf32> + \\ %one_b = broadcast %one_r : tile<1xf32> -> tile<64xf32> + \\ %sum = addf %v, %one_b rounding : tile<64xf32> + \\ %out_r = reshape %out : tile> -> tile<1xptr> + \\ %out_b = broadcast %out_r : tile<1xptr> -> tile<64xptr> + \\ %out_p = offset %out_b, %offsets : tile<64xptr>, tile<64xi32> -> tile<64xptr> + \\ %tok = store_ptr_tko weak %out_p, %sum : tile<64xptr>, tile<64xf32> -> token + \\ return + \\ } + \\} + ; + + const Mod = struct { + pub fn forward(a: Tensor) Tensor { + return cudaTile(.{a}, .{a.shape()}, .{ + .name = "half", + .ir = ir, + .grid = .{ 1, 1, 1 }, + .zeroed_outputs = &.{0}, + })[0]; + } + }; + + const a: zml.Tensor = .init(.{ .n = 128 }, .f32); + + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Mod.forward, .{a}, platform, .{}); + defer exe.deinit(); + + var input: [128]f32 = undefined; + for (&input, 0..) |*v, i| v.* = @floatFromInt(i); + var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), std.mem.sliceAsBytes(&input)); + defer a_buffer.deinit(); + + var result = try exe.eval(std.testing.allocator, std.testing.io, .{a_buffer}); + defer result.deinit(); + + var host = try result.toSliceAlloc(std.testing.allocator, std.testing.io); + defer host.free(std.testing.allocator); + + for (host.items(f32), 0..) |v, i| { + const want: f32 = if (i < 64) @floatFromInt(i + 1) else 0; + try std.testing.expectEqual(want, v); + } +} + +pub const ScatterArgs = struct { + input: Tensor, + update: Tensor, +}; + +/// Generalized version of scatter to many inputs. +/// See `zml.Tensor.scatterSlices` for documentation on scatter. +/// +/// This allows to use the same indices to update several tensors at once, +/// and where the update function is allow to look at elements from the different tensors +/// to compute the final value. +/// +/// This sounds nice but in practice XLA doesn't support this well on GPU, +/// and will generate slow code. In practice stick with `zml.Tensor.scatterSlices`. +pub fn scatter( + inputs: anytype, + index_tensors: anytype, + updates: anytype, + comptime func_: anytype, + context: anytype, + opts: Tensor.ScatterOpts, +) stdx.meta.FnReturn(func_) { + const compiler = Compiler.current(); + var arena = std.heap.ArenaAllocator.init(compiler.allocator); + defer arena.deinit(); + + const mlir_ctx = compiler.mlir_ctx; + + const update_block, var result = b: { + const ArgsTypes: [inputs.len]type = @splat(ScatterArgs); + const ArgsType = @Tuple(&ArgsTypes); + var args: ArgsType = undefined; + var block_types: [2 * inputs.len]*const mlir.Type = undefined; + + inline for (0..inputs.len) |i| { + args[i].input = Tensor.init(.{}, inputs[i].dtype()); + args[i].update = Tensor.init(.{}, inputs[i].dtype()); + + block_types[i] = mlirx.Type.rankedTensor(mlir_ctx, args[i].input.shape()); + block_types[i + inputs.len] = mlirx.Type.rankedTensor(mlir_ctx, args[i].update.shape()); + } + + const block_locs: [2 * inputs.len]*const mlir.Location = @splat(compiler.unknown_location); + const update_block = mlir.Block.init(&block_types, &block_locs); + errdefer update_block.deinit(); + + const scope = compiler.pushBlock(update_block); + defer scope.pop(); + + inline for (0..inputs.len) |i| { + scope.registerTensorAsBlockArgument(args[i].input.id, i); + scope.registerTensorAsBlockArgument(args[i].update.id, i + inputs.len); + } + + var result = @call(.auto, func_, args ++ context); + + var result_values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inputs.len) |i| { + result_values[i] = result[i].value(); + } + + _ = stablehlo.returns(mlir_ctx, &result_values, compiler.location).appendTo(update_block); + break :b .{ update_block, result }; + }; + + // Note: I was a bit lazy here, and I only look at tags on the first tensor. + // we probably should check all of them. + const self = meta.first(Tensor, inputs); + const update = meta.first(Tensor, updates); + var indices_per_axis, var indices_axes = Shape.parseStruct(Tensor, index_tensors); + + if (indices_per_axis.len == 0) return inputs; + + // validate coord axes: all coord_axes should exist inside self + for (indices_axes.constSlice()) |t| { + stdx.debug.assert(self._shape.hasTag(t) != null, "zml.ops.scatter expects axes of indices to be axes of inputs, got input={f} and indices={any}", .{ self, indices_axes.constSlice() }); + } + + // Handle scalar indices by broadcasting them to the indices with the highest rank. + const indices_shape = blk: { + var higher_rank = indices_per_axis.get(0).shape(); + for (indices_per_axis.constSlice()[1..]) |indices| { + if (indices.rank() > higher_rank.rank()) { + higher_rank = indices.shape(); + } + } + break :blk higher_rank; + }; + for (indices_per_axis.slice()) |*idx| { + stdx.debug.assert(idx.shape().canBroadcastTo(indices_shape), "zml.ops.scatter expects all indices tensor to have the same shape, got {any}", .{indices_per_axis.slice()}); + stdx.debug.assert(idx.dtype() == indices_shape.dtype(), "zml.ops.scatter expects all indices tensor to have the same dtype, got {any}", .{indices_per_axis.slice()}); + idx.* = idx.broad(indices_shape); + } + + // rewrite simple scatters to dynamicUpdateSlice. + if (@TypeOf(inputs) == struct { Tensor } and indices_shape.rank() == 0) { + return .{self.dynamicUpdateSlice(index_tensors, update)}; + } + + var updates_values: [inputs.len]*const mlir.Value = undefined; + LoweringCompatibility.preserveScatterDropSemantics(indices_per_axis.slice(), updates, opts, &updates_values); + + // TODO: ideally we should catch all possible scatter errors and provide nice error messages. + var config = scatterConfig(self.shape(), update.shape(), indices_per_axis, indices_axes); + const indices = scatterPrepareIndices(&config, self.shape(), update.shape(), &indices_per_axis, &indices_axes); + // const n_indices_axes = update.rank() - _collectAxes(AxisKind, up_kind, .update_window).len; + // stdx.debug.assert(n_indices_axe == indices_axes.len, "scatter({f}, {any}) expects 'updates' to contain all axes from 'indices', got indices={s}, updates={f}", .{ self, index_tensors, indices_axes.constSlice(), update }); + + var input_values: [inputs.len]*const mlir.Value = undefined; + inline for (0..inputs.len) |i| { + input_values[i] = inputs[i].value(); + } + + const op = stablehlo.scatter( + mlir_ctx, + &input_values, + &.{indices.value()}, + &updates_values, + update_block, + .{ + .update_window_dims = _collectAxes(ScatterAxisKind, config.up_kind, .update_window).constSlice(), + .inserted_window_dims = _collectAxes(ScatterAxisKind, config.op_kind, .inserted_window).constSlice(), + .input_batching_dims = _collectAxes(ScatterAxisKind, config.op_kind, .batching).constSlice(), + .scatter_indices_batching_dims = config.indices_batch_axes.constSlice(), + .scatter_dims_to_operand_dims = config.scatter_to_operand_axes.constSlice(), + .index_vector_dim = indices.rank() - 1, + .indices_are_sorted = opts.indices_are_sorted, + .unique_indices = opts.indices_are_unique, + }, + compiler.location, + ).appendTo(compiler.currentScope().block); + + inline for (0..result.len) |i| { + result[i] = Tensor._result(inputs[i].shape(), op.result(i)); + } + + return result; +} + +const ScatterConfig = struct { + op_kind: stdx.BoundedArray(ScatterAxisKind, constants.MAX_RANK) = .empty, + up_kind: stdx.BoundedArray(ScatterAxisKind, constants.MAX_RANK) = .empty, + indices_batch_axes: Shape.DimsArray = .empty, + scatter_to_operand_axes: Shape.DimsArray = .empty, + updates_transpose: Shape.AxesArray = .empty, +}; + +const ScatterAxisKind = enum { batching, update_window, inserted_window, window_id }; + +fn scatterConfig( + op: Shape, + update: Shape, + indices_per_axis: stdx.BoundedArray(Tensor, constants.MAX_RANK), + indices_axes: Shape.TagsArray, +) ScatterConfig { + var op_kind: stdx.BoundedArray(ScatterAxisKind, constants.MAX_RANK) = .empty; + var up_kind: stdx.BoundedArray(ScatterAxisKind, constants.MAX_RANK) = .empty; + var indices_batch_axes: Shape.DimsArray = .empty; + var scatter_to_operand_axes: Shape.DimsArray = .empty; + var updates_transpose: Shape.AxesArray = .empty; + + const tagged_api = indices_axes.len > 0; + const indices = indices_per_axis.get(0).shape(); + + if (tagged_api) { + for (indices_axes.constSlice()) |t| { + scatter_to_operand_axes.appendAssumeCapacity(op.axis(t)); + } + for (indices.tags()) |t| { + stdx.debug.assert(update.hasTag(t) != null, "scatter expects 'updates' to have all axes of 'indices', got self={f}, updates={f} and indices={f}", .{ op, update, indices }); + updates_transpose.appendAssumeCapacity(update.axis(t)); + } + + for (op.tags()) |t| { + if (update.hasTag(t)) |up_ax| { + updates_transpose.appendAssumeCapacity(up_ax); + + if (indices.hasTag(t)) |id_ax| { + if (std.mem.indexOfScalar(Shape.Tag, indices_axes.constSlice(), t) != null) { + // tag is in indices AND in coords -> it's a batching dim that has been rewritten to a regular insertion dim + op_kind.appendAssumeCapacity(.inserted_window); + } else { + // tag is in op, indices and updates -> it's a batching dim + op_kind.appendAssumeCapacity(.batching); + indices_batch_axes.appendAssumeCapacity(@intCast(id_ax)); + } + } else { + op_kind.appendAssumeCapacity(.update_window); + } + } else { + op_kind.appendAssumeCapacity(.inserted_window); + } + } + + for (update.tags(), 0..) |t, up_ax| { + // Handle batch axes right away. + if (op.hasTag(t)) |self_ax| { + if (op_kind.get(self_ax) == .batching) { + up_kind.appendAssumeCapacity(.batching); + continue; + } + } + if (indices.hasTag(t) != null) { + up_kind.appendAssumeCapacity(.window_id); + } else if (op.hasTag(t)) |self_ax| { + stdx.debug.assert(update.dim(up_ax) <= op.dim(self_ax), "scatter expects the slices described in 'updates' to fit inside 'op', but along axis .{s} it doesn't. Got op={f}, updates={f}.", .{ t, op, update }); + up_kind.appendAssumeCapacity(.update_window); + } else { + // TODO: consider accepting untagged update here. + std.debug.panic("scatter expects 'updates' to be made of axes from op={f} and from indices={any}, got unknown tag {s} in {f}", .{ op, indices_axes.constSlice(), std.mem.sliceTo(t, 0), update }); + } + } + } else { + for (0..indices_per_axis.len) |i| { + op_kind.appendAssumeCapacity(.inserted_window); + scatter_to_operand_axes.appendAssumeCapacity(@intCast(i)); + up_kind.appendAssumeCapacity(.window_id); + } + for (indices_per_axis.len..op.rank()) |_| { + op_kind.appendAssumeCapacity(.update_window); + } + for (indices_per_axis.len..update.rank()) |_| { + up_kind.appendAssumeCapacity(.update_window); + } + for (0..update.rank()) |i| { + updates_transpose.appendAssumeCapacity(@intCast(i)); + } + } + + return .{ + .op_kind = op_kind, + .up_kind = up_kind, + .indices_batch_axes = indices_batch_axes, + .scatter_to_operand_axes = scatter_to_operand_axes, + .updates_transpose = updates_transpose, + }; +} + +test scatterConfig { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + var comp: zml.Compiler = .init(std.testing.allocator, std.testing.io, platform, .{}); + defer comp.deinit(); + comp.activate(); + defer comp.deactivate(); + + const block = mlir.Block.init(&.{}, &.{}); + const scope = comp.pushBlock(block); + defer scope.pop(); + + const Local = struct { + pub fn _idx(idx_shape: anytype) Tensor { + return Tensor.constant(.{ .i32 = 0 }).broad(Shape.init(idx_shape, .i32)); + } + }; + + const idx = Local._idx; + const op = Shape.init(.{ .a = 10, .b = 20 }, .f32); + + // Use .a as a batching axis with .a=10 x .n=8 updates of 2 elements of .b + { + const indices, const coords_tags = Shape.parseStruct(Tensor, .{ .b = idx(.{ .a = 10, .n = 8 }) }); + const update = Shape.init(.{ .a = 10, .n = 8, .b = 2 }, .f32); + + const cfg = scatterConfig(op, update, indices, coords_tags); + try std.testing.expectEqualSlices(ScatterAxisKind, &.{ .batching, .update_window }, cfg.op_kind.constSlice()); + try std.testing.expectEqualSlices(ScatterAxisKind, &.{ .batching, .window_id, .update_window }, cfg.up_kind.constSlice()); + } + + // similar, but use the normalized form where .a is no longer an explicit batching axis. + { + const indices, const coords_tags = Shape.parseStruct(Tensor, .{ .a = idx(.{ .a = 10, .n = 8 }), .b = idx(.{ .a = 10, .n = 8 }) }); + const update = Shape.init(.{ .a = 10, .n = 8, .b = 2 }, .f32); + + const cfg = scatterConfig(op, update, indices, coords_tags); + try std.testing.expectEqualSlices(ScatterAxisKind, &.{ .inserted_window, .update_window }, cfg.op_kind.constSlice()); + try std.testing.expectEqualSlices(ScatterAxisKind, &.{ .window_id, .window_id, .update_window }, cfg.up_kind.constSlice()); + } +} + +/// Concatenate all indices tensor in one tensor. +/// +/// Is allowed to reorder stuff to simplify the job of the backend, +/// and to expand the batching dims. +fn scatterPrepareIndices( + cfg: *ScatterConfig, + op: Shape, + update: Shape, + indices_per_axis: *stdx.BoundedArray(Tensor, constants.MAX_RANK), + indices_axes: *Shape.TagsArray, +) Tensor { + var old_scatter_to_op_axes = cfg.scatter_to_operand_axes; + const batching = _collectAxes(ScatterAxisKind, cfg.op_kind, .batching); + for (batching.constSlice()) |batch_ax| { + const id_shape = indices_per_axis.get(0).shape(); + // batching requires tagging, so we're sure to have a tag here. + const batch_tag = op.tag(batch_ax); + indices_axes.appendAssumeCapacity(batch_tag); + const batch_id = Tensor.iota(id_shape, batch_tag).convert(id_shape.dtype()); + indices_per_axis.appendAssumeCapacity(batch_id); + cfg.op_kind.buffer[@intCast(batch_ax)] = .inserted_window; + cfg.up_kind.buffer[update.axis(batch_tag)] = .window_id; + old_scatter_to_op_axes.appendAssumeCapacity(batch_ax); + } + cfg.indices_batch_axes = .empty; + + // Reorder the axes so that in indices_per_axis is ordered like in op if possible. + // TODO: transpose updates if needed + var indices: stdx.BoundedArray(Tensor, constants.MAX_RANK) = .empty; + var scatter_to_op_axes: Shape.DimsArray = .empty; + + while (old_scatter_to_op_axes.len > 0) { + const scatter_ax = std.sort.argMin(i64, old_scatter_to_op_axes.constSlice(), {}, std.sort.asc(i64)).?; + const op_ax = old_scatter_to_op_axes.orderedRemove(scatter_ax); + const scatter_idx = indices_per_axis.orderedRemove(scatter_ax); + + scatter_to_op_axes.appendAssumeCapacity(op_ax); + indices.appendAssumeCapacity(scatter_idx); + } + cfg.scatter_to_operand_axes = scatter_to_op_axes; + + for (scatter_to_op_axes.constSlice(), 0..) |sc_ax, i| { + if (i != sc_ax) { + //log.warn("Found a slow scatter pattern, which is going to generate a while loop: scatter({f}, {any}, {f}). Because the index axes aren't the major ones in the input tensor.", .{ op, scatter_to_op_axes.constSlice(), update }); + break; + } + } + return Tensor.stack(indices.constSlice(), .last, .coord); +} + +pub const GatherAxisKind = enum { batching, offset, collapsed, indices }; + +pub const GatherOpts = struct { indices_are_sorted: bool = false }; + +pub fn gather(self: Tensor, idx_axes: []const u3, idx_per_axis: []const Tensor, opts: GatherOpts) Tensor { + const compiler = Compiler.current(); + const mlir_ctx = compiler.mlir_ctx; + + stdx.debug.assert(idx_axes.len > 0, "gather expects 1 or more axes to operate one, received none. Example: `x.gather(.a, indices, .{{}})`", .{}); + for (idx_axes, 0..) |a, i| { + if (i > 0) { + stdx.debug.assert(a == idx_axes[i - 1] + 1, "gather expects 'idx_axes' to be sequential. But {any} aren't sequential in {f}", .{ idx_axes, self }); + } + } + var indices_per_axis, _ = Shape.parseStruct(Tensor, idx_per_axis); + + var indices_shape = indices_per_axis.get(0).shape(); + for (indices_per_axis.constSlice()[1..]) |idx| { + if (idx.rank() > indices_shape.rank()) { + indices_shape = idx.shape(); + } + } + for (indices_per_axis.slice()) |*idx| { + stdx.debug.assert(idx.shape().canBroadcastTo(indices_shape), "gather indices can't be broadcasted together {any}", .{idx_per_axis}); + idx.* = idx.broad(indices_shape); + } + LoweringCompatibility.preserveGatherFillSemantics(indices_per_axis.slice()); + + var idx_batch_axes: Shape.DimsArray = .empty; + + var self_kind: stdx.BoundedArray(GatherAxisKind, constants.MAX_RANK) = .{ .buffer = @splat(.offset), .len = self.rank() }; + + for (self._shape.tags(), 0..self.rank()) |t, self_ax| { + const is_gather_axis = std.mem.containsAtLeastScalar(u3, idx_axes, @intCast(self_ax), 1); + if (indices_shape.hasTag(t)) |id_ax| { + // tag is both in self and indices -> it's a batching dim + // Note: tags are required for batching. + self_kind.buffer[self_ax] = .batching; + idx_batch_axes.appendAssumeCapacity(id_ax); + stdx.debug.assert(!is_gather_axis, "gather expects axes to appear at most twice. Axis {s} has been found both in 'self={f}', in 'idx_axes={any}' and in 'indices={f}'", .{ t, self, idx_axes, indices_shape }); + } else if (is_gather_axis) { + // we collapsed all gathered axes + self_kind.buffer[self_ax] = .collapsed; + // idx_kind.buffer[id_ax] = .indices; + } else { + self_kind.buffer[self_ax] = .offset; + } + } + + // compute res shape + var res_shape = Shape.init(.{}, self.dtype()); + var res_kind: stdx.BoundedArray(GatherAxisKind, constants.MAX_RANK) = .empty; + for (self_kind.slice(), 0..) |kind, ax_usize| { + const ax: u3 = @intCast(ax_usize); + if (ax == idx_axes[0]) { + // The first val_ax is special cause this is the place where we insert indices axes. + for (0.., indices_shape.tags(), indices_shape.dims()) |id_axis_order, id_axis, id_inserted_dim| { + const is_batching_axis = std.mem.containsAtLeastScalar(i64, idx_batch_axes.constSlice(), @intCast(id_axis_order), 1); + // Batching axis is already in self. + if (is_batching_axis) continue; + + res_shape = res_shape.appendDim(id_inserted_dim, id_axis); + res_kind.appendAssumeCapacity(.indices); + } + } + switch (kind) { + .collapsed => continue, + else => { + res_shape = res_shape.appendDim(self.dim(ax), self._shape.tag(ax)); + res_kind.appendAssumeCapacity(kind); + }, + } + } + + // This is not a gather, but a dynamicSlice. + // Sometimes the backend recognize this pattern, but not always. + // So let us handle that. + if (indices_shape.count() == 1 and idx_axes.len == 1) { + return self + .slice(idx_axes[0], .dyn(indices_per_axis.get(0).asScalar(), 1)) + // Keep downstream remeshe after the slice. SPMD engines like Neuron otherwise + // may hoist an all-gather before the slice and materialize the full tensor. + .optimizationBarrier() + .reshape(res_shape); + } + + var slice_dims: Shape.DimsArray = .empty; + for (self_kind.slice(), self.dims()) |k, d| { + slice_dims.appendAssumeCapacity(switch (k) { + .batching, .collapsed => 1, + .offset => d, + .indices => unreachable, + }); + } + + // TODO: try changing .last by other axis and see the perf impact. + const indices = Tensor.stack(indices_per_axis.constSlice(), .last, .coord); + // scoped_log.debug("gather --> {} {any}", .{ res_shape, res_kind.constSlice() }); + const gather_op = stablehlo.gather( + mlir_ctx, + self.value(), + indices.value(), + slice_dims.constSlice(), + .{ + .offset_dims = _collectAxes(GatherAxisKind, res_kind, .offset).constSlice(), + .collapsed_slice_dims = _collectAxes(GatherAxisKind, self_kind, .collapsed).constSlice(), + .operand_batching_dims = _collectAxes(GatherAxisKind, self_kind, .batching).constSlice(), + .start_indices_batching_dims = idx_batch_axes.constSlice(), + .start_index_map = _collectAxes(GatherAxisKind, self_kind, .collapsed).constSlice(), + .index_vector_dim = indices.axis(.coord), + .indices_are_sorted = opts.indices_are_sorted, + }, + compiler.location, + ).appendTo(compiler.currentScope().block); + + const mlir_shape = Tensor.fromMlirValue(gather_op.result(0)).shape(); + stdx.debug.assert(mlir_shape.eql(res_shape), "gather expects that batching indices appear in the same order in 'self' and 'indices', got: self={f}, indices={f}. You should transpose one or the other.", .{ self, indices }); + return Tensor._result(res_shape, gather_op.result(0)); +} + +/// Backend lowering compatibility shims live here so public tensor/model code +/// can keep expressing StableHLO semantics directly. +/// Each rewrite should +/// preserve the source operation's semantics while avoiding a backend compiler +/// or runtime path that currently mishandles that legal StableHLO. +/// Fill/drop indexed ops use max-value integer sentinels for inactive lanes; +/// those lanes must not issue indirect memory accesses after lowering. +pub const LoweringCompatibility = struct { + /// Keep scalar integer broadcasts as scalar broadcasts until StableHLO broadcast emission. + /// + /// Some backend compilers fold rank-0 integer scalars into full-rank dense + /// splats too early, which can perturb later indexed-update lowering. A + /// scalar optimization barrier preserves the source-level broadcast shape. + pub fn preserveIntegerScalarBroadcast(self: Tensor, output_shape: Shape) ?Tensor { + switch (Compiler.current().platform.target) { + .neuron => {}, + .cpu, .cuda, .rocm, .tpu, .oneapi, .metal, .furiosa => return null, + } + + if (self.rank() != 0 or output_shape.rank() == 0 or !self.dtype().isInteger()) return null; + return self.optimizationBarrier().broadcast(output_shape, &.{}); + } + + /// Preserve gather fill semantics before backend indirect-memory lowering. + /// + /// Frontends represent inactive gather rows by passing an out-of-bounds + /// integer sentinel together with fill semantics. StableHLO says those rows + /// should produce the fill value, so the sentinel lane is not a real memory + /// access. Some backend lowerings still route every lane through a gather/DGE + /// path before applying fill behavior, so sanitize inactive lanes early while + /// keeping the active rows unchanged. Related upstream Neuron report: + /// https://github.com/aws-neuron/aws-neuron-sdk/issues/1335. + pub fn preserveGatherFillSemantics(indices: []Tensor) void { + switch (Compiler.current().platform.target) { + .neuron => {}, + .cpu, .cuda, .rocm, .tpu, .oneapi, .metal, .furiosa => return, + } + + const active_lanes = activeLanesForFillDropIndices(indices) orelse return; + replaceInactiveIndirectIndices(indices, active_lanes); + } + + /// Preserve scatter drop semantics before backend indirect-memory lowering. + /// Same as above but for scatter drop semantics. + pub fn preserveScatterDropSemantics(indices: []Tensor, updates: anytype, opts: Tensor.ScatterOpts, update_values: *[updates.len]*const mlir.Value) void { + const active_lanes: ?Tensor = switch (Compiler.current().platform.target) { + .neuron => if (opts.update_fn == Tensor.ScatterOpts.increment) activeLanesForFillDropIndices(indices) else null, + .cpu, .cuda, .rocm, .tpu, .oneapi, .metal, .furiosa => null, + }; + + if (active_lanes) |active| replaceInactiveIndirectIndices(indices, active); + + inline for (updates, 0..) |update, i| { + const update_tensor = if (active_lanes) |active| blk: { + const mask = active.broad(update.shape().withDtype(.bool)); + const zero = Tensor.constant(update.dtype().zero()).broad(update.shape()); + break :blk mask.select(update, zero); + } else update; + update_values[i] = update_tensor.value(); + } + } + + /// Builds the lane mask used by fill/drop compatibility rewrites. + /// + /// For normal index widths, the max representable value is the inactive + /// sentinel. For 64-bit indices, the affected lowering path narrows to the + /// i32 indirect-index domain, so only values that remain valid after that + /// narrowing are considered active. When multiple index tensors form a + /// coordinate, a lane is active only if every component is active. + fn activeLanesForFillDropIndices(indices: []const Tensor) ?Tensor { + var active: ?Tensor = null; + for (indices) |idx| { + if (!idx.dtype().isInteger()) continue; + + const idx_active = switch (idx.dtype()) { + .i64 => blk: { + const zero = Tensor.scalar(@as(i64, 0), .i64).broad(idx.shape()); + const max_i32 = Tensor.scalar(@as(i64, std.math.maxInt(i32)), .i64).broad(idx.shape()); + break :blk idx.cmp(.GE, zero).logical(.AND, idx.cmp(.LE, max_i32)); + }, + .u64 => blk: { + const max_i32 = Tensor.scalar(@as(u64, std.math.maxInt(i32)), .u64).broad(idx.shape()); + break :blk idx.cmp(.LE, max_i32); + }, + inline .i2, .i4, .i8, .i16, .i32, .u2, .u4, .u8, .u16, .u32 => blk: { + const sentinel = Tensor.constant(idx.dtype().maxValue()).broad(idx.shape()); + break :blk idx.cmp(.NE, sentinel); + }, + .bool, .f4e2m1, .f8e3m4, .f8e4m3, .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz, .f8e8m0, .bf16, .f16, .f32, .f64, .c64, .c128 => unreachable, + }; + active = if (active) |current| current.logical(.AND, idx_active) else idx_active; + } + return active; + } + + /// Replaces inactive sentinel indices with a safe index before indirect lowering. + /// + /// The replacement index is intentionally zero: it is always in-bounds for + /// non-empty operands and the corresponding scatter update is masked away by + /// the caller. This keeps the lowered indirect operation from touching the + /// sentinel address while preserving fill/drop semantics at the tensor level. + fn replaceInactiveIndirectIndices(indices: []Tensor, active_lanes: Tensor) void { + for (indices) |*idx| { + if (!idx.dtype().isInteger()) continue; + + const safe_index = Tensor.constant(idx.dtype().zero()).broad(idx.shape()); + idx.* = active_lanes.select(idx.*, safe_index); + } + } +}; + +pub fn _collectAxes(T: type, bounded_array: stdx.BoundedArray(T, constants.MAX_RANK), value: T) stdx.BoundedArray(i64, constants.MAX_RANK) { + var res: stdx.BoundedArray(i64, constants.MAX_RANK) = .empty; + for (bounded_array.constSlice(), 0..) |v, ax| { + if (v == value) { + res.appendAssumeCapacity(@intCast(ax)); + } } return res; } -pub const BlockSignature = struct { - Fn: type, - BlkCtx: type, - Args: type, - FullArgs: type, - Return: type, - nIn: usize, - nOut: usize, +fn CustomCallResultTypeFromOutputSpec(comptime OutputSpecT: type) type { + const type_info = @typeInfo(OutputSpecT); + return switch (type_info) { + .void => void, + .@"struct" => |struct_info| b: { + if (OutputSpecT == Shape) break :b Tensor; + if (!struct_info.is_tuple) @compileError("Expected tuple output shape spec"); + break :b if (struct_info.field_names.len == 1) + Tensor + else + [struct_info.field_names.len]Tensor; + }, + .array => |array_info| b: { + break :b if (array_info.len == 1) + Tensor + else + [array_info.len]Tensor; + }, + .pointer => |pointer_info| b: { + if (pointer_info.size != .slice) @compileError("Expected output shape slice"); + break :b []Tensor; + }, + else => @compileError("Unsupported customCall output shape spec type: " ++ @typeName(OutputSpecT)), + }; +} - pub inline fn blkArgs(self: BlockSignature, blk_ctx: self.BlkCtx, args: self.Args) self.FullArgs { - if (self.BlkCtx == void) return args; - if (self.Args == void) return blk_ctx; - return .{blk_ctx} ++ args; - } +pub const CustomCallOptions = struct { + has_side_effect: bool, + output_operand_aliases: ?[]const stablehlo.CustomCallOpts.OutputOperandAlias = null, + compute_on_host: bool = false, }; -const BlockType = enum { default, no_ctx, no_args }; +fn customCallAdditionalAttributes(ctx: *Compiler, opts: CustomCallOptions) []const mlir.NamedAttribute { + if (!opts.compute_on_host or ctx.platform.target == .cpu) return &.{}; -pub inline fn BlockSign(comptime func: anytype) BlockSignature { - return _BlockSign(func, .default); + const frontend_attributes = mlir.Attribute.dict(ctx.mlir_ctx, &.{ + .named(ctx.mlir_ctx, "_xla_compute_type", .string(ctx.mlir_ctx, "host")), + }); + + const additional_attributes = ctx.arena.allocator().alloc(mlir.NamedAttribute, 1) catch unreachable; + additional_attributes[0] = .named(ctx.mlir_ctx, "mhlo.frontend_attributes", frontend_attributes); + return additional_attributes; } -pub inline fn BlockSignNoCtx(comptime func: anytype) BlockSignature { - return _BlockSign(func, .no_ctx); -} +pub fn customCallOutputOperandAliases( + comptime I: type, + comptime O: type, + comptime aliases: ?CustomCallOutputOperandAliases(I, O), +) ?[]const stablehlo.CustomCallOpts.OutputOperandAlias { + const aliases_ = aliases orelse return null; -pub inline fn BlockSignNoArgs(comptime func: anytype) BlockSignature { - return _BlockSign(func, .no_args); -} - -pub fn fnInfo(comptime func: anytype) std.builtin.Type.Fn { - if (@TypeOf(func) == type) { - if (@typeInfo(func) == .Struct and @hasDecl(func, "forward")) { - return fnInfo(func.forward); - } - @compileError("Given type doesn't have a forward function: " ++ @typeName(func)); + const output_field_names = @typeInfo(O).@"struct".field_names; + comptime var num_aliases = 0; + inline for (output_field_names) |field_name| { + if (@field(aliases_, field_name) != null) num_aliases += 1; } - const type_info = @typeInfo(@TypeOf(func)); - const err_msg = "`func` must be a function and return one or more `Tensor`. Got: "; - if (type_info != .Fn or type_info.Fn.return_type == null) { - @compileError(err_msg ++ @typeName(@TypeOf(func))); - } - return type_info.Fn; -} -fn _BlockSign(comptime func: anytype, blk_type: BlockType) BlockSignature { - const fn_info = fnInfo(func); - const err_msg = "`func` must be a function and return one or more `Tensor`. Got: "; - - var full_args: [fn_info.params.len]type = undefined; - const arg_start = switch (blk_type) { - .default => 1, - .no_ctx => 0, - .no_args => fn_info.params.len, - }; - var n_tensors: usize = 0; - // var n_inner_tensors: usize = 0; - inline for (fn_info.params, 0..) |arg, i| { - const ArgType = if (arg.type) |T| T else @compileError(err_msg ++ @typeName(@TypeOf(func))); - full_args[i] = ArgType; - if (i >= arg_start) { - n_tensors += staticCountTensors(ArgType) orelse @compileError("Can't use " ++ @typeName(ArgType) ++ " in an MLIR function, because it has a variable number of tensors"); - } - } - const FullArgs = std.meta.Tuple(&full_args); - const BlkCtx = switch (blk_type) { - .default => full_args[0], - .no_ctx => void, - .no_args => FullArgs, - }; - const Args = switch (blk_type) { - .default => std.meta.Tuple(full_args[1..]), - .no_ctx => FullArgs, - .no_args => void, - }; - - return .{ - .Fn = @TypeOf(func), - .BlkCtx = BlkCtx, - .Args = Args, - .FullArgs = FullArgs, - .Return = fn_info.return_type.?, - .nIn = n_tensors, - .nOut = staticCountTensors(fn_info.return_type.?) orelse @compileError("Can't use " ++ @typeName(fn_info.return_type.?) ++ " in an MLIR function, because it has a variable number of tensors"), - }; -} - -pub fn staticIsOnlyTensors(comptime T: type) bool { - if (T == Tensor) return true; - - return switch (@typeInfo(T)) { - .Array => |array_info| staticIsOnlyTensors(array_info.child), - .Pointer => |ptr_info| ptr_info.size == .One and staticIsOnlyTensors(ptr_info.child), - .Struct => |struct_info| { - inline for (struct_info.fields) |field| { - if (!staticIsOnlyTensors(field.type)) return false; + const output_operand_aliases = comptime blk: { + var aliases_buffer: [num_aliases]stablehlo.CustomCallOpts.OutputOperandAlias = undefined; + var i = 0; + for (output_field_names, 0..) |field_name, output_index| { + if (@field(aliases_, field_name)) |operand| { + aliases_buffer[i] = .{ + .output_index = @intCast(output_index), + .operand_index = @intCast(@backingInt(operand)), + }; + i += 1; } - return true; - }, - else => false, + } + break :blk aliases_buffer; }; + + return &output_operand_aliases; } -pub fn staticCountTensors(comptime T: type) ?usize { - if (T == Tensor) return 1; +pub const CompositeOpts = struct { + version: i32 = 0, + composite_attributes: []const mlir.NamedAttribute = &.{}, +}; - return switch (@typeInfo(T)) { - .Array => |array_info| array_info.len * (staticCountTensors(array_info.child) orelse return null), - .Pointer => |ptr_info| { - const n = staticCountTensors(ptr_info.child) orelse return null; - if (ptr_info.size != .One and n > 0) return null; - return n; - }, - .Struct => |struct_info| { - var count: usize = 0; - inline for (struct_info.fields) |field| { - count += staticCountTensors(field.type) orelse return null; +pub fn composite( + name: [:0]const u8, + inputs: []const Tensor, + outputs: []const Shape, + comptime decomposition: anytype, + context: anytype, + opts: CompositeOpts, +) []Tensor { + const ctx = Compiler.current(); + const mlir_ctx = ctx.mlir_ctx; + + const decomp_name = ctx.allocPrint("{s}.impl_{d}", .{ name, ctx.nextCompositeId() }); + + { + const block_types = ctx.alloc(*const mlir.Type, inputs.len); + const block_locs = ctx.alloc(*const mlir.Location, inputs.len); + @memset(block_locs, ctx.unknown_location); + + for (inputs, 0..) |t, i| { + block_types[i] = mlirx.Type.rankedTensor(mlir_ctx, t.shape()); + } + + const block = mlir.Block.init(block_types, block_locs); + + const scope = ctx.pushBlock(block); + { + const arg_tensors = ctx.alloc(Tensor, inputs.len); + for (inputs, 0..) |t, i| { + arg_tensors[i] = Tensor._result(t.shape(), block.argument(i)); } - return count; - }, - else => 0, - }; -} -/// Produces a custom call to `name` that takes a tensor and returns it. -/// -/// For example, this can be used to extract tokens quickly if they run on a loop on the -/// GPU. -pub fn identityCustomCall(name: [:0]const u8, input: Tensor, context: *anyopaque) Tensor { - const address: [8]u8 = @bitCast(@intFromPtr(context)); - var backend_config: [8:0]u8 = undefined; - @memcpy(backend_config[0..8], address[0..8]); - const ctx = CompilationContext.current(); + var result = @call(.auto, decomposition, .{ arg_tensors, context }); + const rtensors = ctx.alloc(Tensor, outputs.len); + meta.collectBuf((struct { + pub fn func_(t: Tensor) Tensor { + return t; + } + }).func_, {}, &result, rtensors); - const loc = ctx.mlirCtx().location(@src()).namedFmt(ctx.mlirCtx(), "name={s}", .{name}); + const rvals = ctx.alloc(*const mlir.Value, outputs.len); + for (rtensors, 0..) |t, i| { + rvals[i] = t.value(); + } - const op = dialect.stablehlo.custom_call( - ctx.mlirCtx(), - &.{input.value()}, + _ = func.returns(mlir_ctx, rvals, ctx.location).appendTo(block); + } + scope.pop(); + + _ = func.func(mlir_ctx, .{ + .name = decomp_name, + .block = block, + .location = ctx.location, + .visibility = .private, + .verify = false, + }).appendTo(ctx.module.body()); + } + + const operand_values = ctx.alloc(*const mlir.Value, inputs.len); + for (inputs, 0..) |t, i| { + operand_values[i] = t.value(); + } + + const result_types = ctx.alloc(*const mlir.Type, outputs.len); + for (outputs, 0..) |s, i| { + result_types[i] = mlirx.Type.rankedTensor(mlir_ctx, s); + } + + const op = stablehlo.composite( + mlir_ctx, + operand_values, + result_types, .{ - .api_version = 1, - .has_side_effect = false, - .call_target_name = name, - .backend_config = backend_config[0..], - .output_operand_aliases = &.{0}, + .name = name, + .decomposition = decomp_name, + .composite_attributes = opts.composite_attributes, + .version = opts.version, }, - &.{input.value().getType()}, - loc, + ctx.location, + ).appendTo(ctx.currentScope().block); + + const out_tensors = ctx.alloc(Tensor, outputs.len); + for (outputs, 0..) |s, i| { + out_tensors[i] = Tensor._result(s, op.result(i)); + } + + return out_tensors; +} + +pub fn customCall(target_name: [:0]const u8, inputs: anytype, outputs: anytype, metadata: anytype, opts: CustomCallOptions) CustomCallResultTypeFromOutputSpec(@TypeOf(outputs)) { + // Transform generic inputs to flat slice. + const inputs_: []const Tensor = switch (@typeInfo(@TypeOf(inputs))) { + .@"struct" => |struct_info| b: { + if (@TypeOf(inputs) == Tensor) { + break :b &[1]Tensor{inputs}; + } + if (!struct_info.is_tuple) @compileError("Expected tuple"); + var inputs_: [struct_info.field_names.len]Tensor = undefined; + meta.collectBuf((struct { + pub fn func_(t: Tensor) Tensor { + return t; + } + }).func_, {}, &inputs, &inputs_); + break :b &inputs_; + }, + .array => &inputs, + .pointer => |pointer_info| b: { + if (pointer_info.size != .slice) @compileError("Expected slice"); + break :b inputs; + }, + else => @compileError("Unsupported type: " ++ @typeName(@TypeOf(inputs))), + }; + + // Transform generic outputs to flat slice. + const output_shapes: []const Shape = switch (@typeInfo(@TypeOf(outputs))) { + .void => &.{}, + .@"struct" => |struct_info| b: { + if (@TypeOf(outputs) == Shape) { + break :b &[1]Shape{outputs}; + } + if (!struct_info.is_tuple) @compileError("Expected tuple"); + var output_shapes: [struct_info.field_names.len]Shape = undefined; + meta.collectBuf((struct { + pub fn func_(t: Shape) Shape { + return t; + } + }).func_, {}, &outputs, &output_shapes); + break :b &output_shapes; + }, + .array => &outputs, + .pointer => |pointer_info| b: { + if (pointer_info.size != .slice) @compileError("Expected slice"); + break :b outputs; + }, + else => @compileError("Unsupported type: " ++ @typeName(@TypeOf(outputs))), + }; + + const outputs_flat = typedCustomCall( + target_name, + opts, + inputs_, + output_shapes, + metadata, ); - return Tensor._result(input.shape(), op.result(0)); + if (comptime CustomCallResultTypeFromOutputSpec(@TypeOf(outputs)) == void) { + stdx.debug.assert(outputs_flat.len == 0, "customCall expected zero outputs for void return, got {}", .{outputs_flat.len}); + return; + } + + // Transform flat slice to generic outputs. + return switch (@typeInfo(@TypeOf(outputs))) { + .@"struct" => |struct_info| b: { + if (@TypeOf(outputs) == Shape) break :b outputs_flat[0]; + if (!struct_info.is_tuple) @compileError("Expected tuple"); + if (struct_info.field_names.len == 1) break :b outputs_flat[0]; + var outputs_: [struct_info.field_names.len]Tensor = undefined; + @memcpy(&outputs_, outputs_flat); + break :b outputs_; + }, + .array => |array_info| b: { + if (array_info.len == 1) break :b outputs_flat[0]; + var outputs_: [array_info.len]Tensor = undefined; + @memcpy(&outputs_, outputs_flat); + break :b outputs_; + }, + .pointer => |pointer_info| b: { + if (pointer_info.size != .slice) @compileError("Expected slice"); + break :b outputs_flat; + }, + else => @compileError("Unsupported type: " ++ @typeName(@TypeOf(outputs))), + }; +} + +fn metadataIntegerFieldToMlirAttribute(mlir_ctx: *mlir.Context, int_field: std.builtin.Type.Int, value: anytype) *const mlir.Attribute { + return switch (int_field.signedness) { + .signed => switch (int_field.bits) { + 8 => .int(mlir_ctx, .i8, value), + 16 => .int(mlir_ctx, .i16, value), + 32 => .int(mlir_ctx, .i32, value), + 64 => .int(mlir_ctx, .i64, value), + else => @panic("Unsupported DataType"), + }, + .unsigned => switch (int_field.bits) { + 8 => .int(mlir_ctx, .u8, value), + 16 => .int(mlir_ctx, .u16, value), + 32 => .int(mlir_ctx, .u32, value), + 64 => .int(mlir_ctx, .u64, value), + else => @panic("Unsupported DataType"), + }, + }; +} + +fn metadataFloatFieldToMlirAttribute(mlir_ctx: *mlir.Context, float_field: std.builtin.Type.Float, value: anytype) *const mlir.Attribute { + return switch (float_field.bits) { + 16 => .float(mlir_ctx, .f16, @as(f64, value)), + 32 => .float(mlir_ctx, .f32, @as(f64, value)), + 64 => .float(mlir_ctx, .f64, @as(f64, value)), + else => @panic("Unsupported DataType"), + }; +} + +fn metadataFieldToMlirAttribute(mlir_ctx: *mlir.Context, comptime T: type, value: anytype) ?*const mlir.Attribute { + const type_info = @typeInfo(T); + return switch (type_info) { + .comptime_int => .int(mlir_ctx, .u64, @as(u64, value)), + .@"enum" => |enum_field| switch (@typeInfo(enum_field.tag_type)) { + .int => |int_tag| metadataIntegerFieldToMlirAttribute(mlir_ctx, int_tag, @backingInt(value)), + else => @compileError("Unsupported tag type for enum metadata: " ++ @typeName(enum_field.tag_type)), + }, + .int => |int_field| metadataIntegerFieldToMlirAttribute(mlir_ctx, int_field, value), + .float => |float_field| metadataFloatFieldToMlirAttribute(mlir_ctx, float_field, value), + .bool => .boolean(mlir_ctx, value), + .pointer => |pointer_info| switch (pointer_info.size) { + .slice => if (pointer_info.child == u8) + .string(mlir_ctx, value) + else + @panic("Unsupported pointer type in metadata"), + else => @panic("Unsupported pointer type in metadata"), + }, + .optional => |optional_info| if (value) |wrapped_value| switch (@typeInfo(optional_info.child)) { + .int => |int_field| metadataIntegerFieldToMlirAttribute(mlir_ctx, int_field, wrapped_value), + .float => |float_field| metadataFloatFieldToMlirAttribute(mlir_ctx, float_field, wrapped_value), + .bool => .boolean(mlir_ctx, wrapped_value), + else => @panic("Unsupported optional child type in metadata"), + } else null, + else => @compileError("Unsupported metadata type: " ++ @typeName(T)), + }; +} + +pub fn CustomCall( + // A, I and O are assumed to be simple struct without any nesting like: { a: zml.Tensor } + I: type, + O: type, + A: type, + // Receives the custom call buffers for the input and outputs a struct with the same fields as I and O + // and the provided attributes. So if I is `{a: zml.Tensor}`, the first argument is `{a: CustomCallBuffer}` + // (TensorToCustomCallBuffer(i), ShapeToCustomCallBuffer(O), A) -> !?*pjrt.ffi.Error + comptime func_: anytype, + comptime params: struct { + /// Name of the custom call, must be unique. + name: [:0]const u8, + /// Whether the function has any side-effect, meaning that XLA cannot re-order it/optimize it out. A typical example is print. + has_side_effect: bool, + /// Similar to reuseBuffer on tensors, tells XLA that the custom call re-uses an input buffer for its output. + /// Use the input/output field names to express the mapping: `{ . = . }` + output_operand_aliases: ?CustomCallOutputOperandAliases(I, O) = null, + /// Request XLA host-compute offload for this custom call on accelerator backends. + /// Input/Output buffers will automatically be moved d2h/h2d without the need for an explicit `.toMemory(...)`. + compute_on_host: bool = false, + }, +) type { + return struct { + pub const InputTensors = I; + pub const OutputShapes = O; + pub const Attributes = A; + + /// Register the custom call so you can use it. + pub fn register(platform: *const Platform) !void { + try platform.registerFfi(.{ + .name = params.name, + .platform_name = if (params.compute_on_host) "Host" else null, + .handler = @This().handler, + .traits = .{ + .command_buffer_compatible = true, + }, + }); + } + + pub fn call(input_tensors: I, output_shapes: O, attributes: A) ShapeToTensor(O) { + const opts: CustomCallOptions = .{ + .has_side_effect = params.has_side_effect, + .output_operand_aliases = comptime customCallOutputOperandAliases(I, O, params.output_operand_aliases), + .compute_on_host = params.compute_on_host, + }; + return typedCustomCall(params.name, opts, input_tensors, output_shapes, attributes); + } + + fn handler(call_frame: *pjrt.ffi.CallFrame) callconv(.c) ?*pjrt.ffi.Error { + if (call_frame.registeringHook()) return null; + + const input_buffers: TensorToCustomCallBuffer(I), const output_buffers: ShapeToCustomCallBuffer(O), const attributes: A = customCallArgsFromPjrtCallFrame(I, O, A, call_frame); + return func_(call_frame, input_buffers, output_buffers, attributes) catch |err| + pjrt.ffi.Error.create(call_frame.api, .unknown, @errorName(err)); + } + }; +} + +pub fn ShapeToTensor(T: type) type { + return meta.MapRestrict(Shape, Tensor).map(T); +} + +pub fn CustomCallOutputOperandAliases(I: type, O: type) type { + const field_names = @typeInfo(O).@"struct".field_names; + return @Struct( + .auto, + null, + field_names, + &@splat(?std.meta.FieldEnum(I)), + &@splat(.{ .default_value_ptr = &@as(?std.meta.FieldEnum(I), null) }), + ); +} + +pub fn typedCustomCall( + target_name: [:0]const u8, + opts: CustomCallOptions, + input: anytype, + output: anytype, + attributes: anytype, +) ShapeToTensor(@TypeOf(output)) { + const Input = @TypeOf(input); + const Output = @TypeOf(output); + const Attributes = @TypeOf(attributes); + + const ctx = Compiler.current(); + const allocator = ctx.arena.allocator(); + + stdx.debug.assert(ctx.manual_computation_depth > 0, "customCall '{s}' must be emitted inside manualComputation", .{target_name}); + + const input_tensors: []const Tensor = switch (@typeInfo(Input)) { + .@"struct" => |struct_info| b: { + var input_tensors: [struct_info.field_names.len]Tensor = undefined; + inline for (struct_info.field_names, 0..) |field_name, i| { + input_tensors[i] = @field(input, field_name); + } + break :b &input_tensors; + }, + .pointer => |pointer_info| b: { + if (pointer_info.size != .slice) @compileError("Expected input slice"); + break :b input; + }, + else => @compileError("Unsupported input type: " ++ @typeName(Input)), + }; + + const output_shapes: []const Shape = switch (@typeInfo(Output)) { + .@"struct" => |struct_info| b: { + var output_shapes: [struct_info.field_names.len]Shape = undefined; + inline for (struct_info.field_names, 0..) |field_name, i| { + output_shapes[i] = @field(output, field_name); + } + break :b &output_shapes; + }, + .pointer => |pointer_info| b: { + if (pointer_info.size != .slice) @compileError("Expected input slice"); + break :b output; + }, + else => @compileError("Unsupported output type: " ++ @typeName(Output)), + }; + + var metadata_attribute_list = switch (@typeInfo(Attributes)) { + .void => std.ArrayList(mlir.NamedAttribute).initCapacity(allocator, 2) catch unreachable, + .@"struct" => |struct_info| std.ArrayList(mlir.NamedAttribute).initCapacity(allocator, struct_info.field_names.len + 2) catch unreachable, + else => @compileError("Unsupported type: " ++ @typeName(Attributes)), + }; + if (@typeInfo(Attributes) == .@"struct") { + inline for (@typeInfo(Attributes).@"struct".field_names, @typeInfo(Attributes).@"struct".field_types) |field_name, Field| { + if (metadataFieldToMlirAttribute(ctx.mlir_ctx, Field, @field(attributes, field_name))) |attribute| { + metadata_attribute_list.appendAssumeCapacity(mlir.NamedAttribute.named(ctx.mlir_ctx, field_name, attribute)); + } + } + } + metadata_attribute_list.appendSliceAssumeCapacity(&[_]mlir.NamedAttribute{ + .named(ctx.mlir_ctx, "pjrt_api", .int(ctx.mlir_ctx, .u64, @as(u64, @bitCast(@intFromPtr(ctx.platform.pjrt_api))))), + .named(ctx.mlir_ctx, "pjrt_client", .int(ctx.mlir_ctx, .u64, @as(u64, @bitCast(@intFromPtr(ctx.platform.pjrt_client))))), + }); + + const values = allocator.alloc(*const mlir.Value, input_tensors.len) catch unreachable; + const input_shapes = allocator.alloc(Shape, input_tensors.len) catch unreachable; + for (input_tensors, 0..) |input_tensor, i| { + values[i] = input_tensor.value(); + input_shapes[i] = input_tensor.shape(); + } + + const result_types = allocator.alloc(*const mlir.Type, output_shapes.len) catch unreachable; + for (output_shapes, 0..) |shape, i| { + result_types[i] = .rankedTensor(shape.dims(), mlirx.Type.fromDType(ctx.mlir_ctx, shape.dtype())); + } + + const backend_config: *const mlir.Attribute = .dict(ctx.mlir_ctx, metadata_attribute_list.items); + + const operand_layouts = allocator.alloc([]const usize, input_shapes.len) catch unreachable; + for (input_shapes, 0..) |shape, i| { + operand_layouts[i] = allocator.dupe(usize, toUsize(constants.minorToMajor(shape.rank())).constSlice()) catch unreachable; + } + + const result_layouts = allocator.alloc([]const usize, output_shapes.len) catch unreachable; + for (output_shapes, 0..) |shape, i| { + result_layouts[i] = allocator.dupe(usize, toUsize(constants.minorToMajor(shape.rank())).constSlice()) catch unreachable; + } + + const op = stablehlo.custom_call( + ctx.mlir_ctx, + values, + result_types, + .{ + .call_target_name = target_name, + .backend_config = .{ .typed_ffi = backend_config }, + .has_side_effect = opts.has_side_effect, + .operand_layouts = operand_layouts, + .result_layouts = result_layouts, + .output_operand_aliases = opts.output_operand_aliases orelse &.{}, + .additional_attributes = customCallAdditionalAttributes(ctx, opts), + }, + ctx.location, + ).appendTo(ctx.currentScope().block); + + if (ctx.partitioner == .gspmd) { + op.setAttributeByName("mhlo.sharding", .string(ctx.mlir_ctx, "{manual}")); + } + + switch (@typeInfo(Output)) { + .@"struct" => |struct_info| { + var out: ShapeToTensor(Output) = undefined; + inline for (output_shapes, struct_info.field_names, 0..) |output_shape, field_name, i| { + @field(out, field_name) = Tensor._result(output_shape, op.result(i)); + } + return out; + }, + .pointer => |pointer_info| { + if (pointer_info.size != .slice) @compileError("Expected input slice"); + var out: []Tensor = allocator.alloc(Tensor, output_shapes.len) catch unreachable; + for (output_shapes, 0..) |output_shape, i| { + out[i] = Tensor._result(output_shape, op.result(i)); + } + return out; + }, + else => @compileError("Unsupported output type: " ++ @typeName(Output)), + } +} + +fn customCallArgsFromPjrtCallFrame(I: type, O: type, A: type, call_frame: *pjrt.ffi.CallFrame) struct { + TensorToCustomCallBuffer(I), + ShapeToCustomCallBuffer(O), + A, +} { + var input: TensorToCustomCallBuffer(I) = undefined; + inline for (@typeInfo(I).@"struct".field_names, 0..) |field_name, i| { + const buf = call_frame.args.buffers()[i]; + @field(input, field_name) = .fromPjrt(buf); + } + + var output: ShapeToCustomCallBuffer(O) = undefined; + inline for (@typeInfo(O).@"struct".field_names, 0..) |field_name, i| { + const shape_buf = call_frame.results.buffers()[i]; + @field(output, field_name) = .fromPjrt(shape_buf); + } + + const attributes: A = switch (@typeInfo(A)) { + .void => {}, + .@"struct" => blk: { + var attrs: A = undefined; + inline for (@typeInfo(A).@"struct".field_names, @typeInfo(A).@"struct".field_types) |field_name, Field| { + // TODO: Use getByIndex instead? + const attribute = call_frame.attrs.getByName(.scalar, field_name) orelse + @panic("Attribute not found: " ++ field_name); + @field(attrs, field_name) = attribute.get(Field); + } + break :blk attrs; + }, + else => @compileError("Unsupported attributes type: " ++ @typeName(A)), + }; + + return .{ input, output, attributes }; +} + +test customCall { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + const tp = zml.testing.tpMesh(); + + var comp: zml.Compiler = .init(std.testing.allocator, std.testing.io, platform, .{ .meshes = &.{tp} }); + defer comp.deinit(); + comp.activate(); + defer comp.deactivate(); + + const block = mlir.Block.init(&.{}, &.{}); + const scope = comp.pushBlock(block); + defer scope.pop(); + + const shape = zml.Shape.init(.{128}, .bf16).withPartitioning(tp, .{ ._0 = .model }); + const input = Tensor.constant(zml.DataType.bf16.constant(0)).broad(shape); + const Local = struct { + fn body(x: Tensor, output_shape: Shape) Tensor { + return customCall("my_custom_call", .{x}, .{output_shape}, .{}, .{ + .has_side_effect = false, + .output_operand_aliases = &.{.{ .output_index = 0, .operand_index = 0 }}, + }); + } + }; + const output = manualComputation(Local.body, input, shape, tp, .{ + .in_specs = .parse(.{ ._0 = .model }), + .out_specs = .parse(.{ ._0 = .model }), + }); + + try zml.testing.expectEqualShapes(input.shape(), output.shape()); +} + +test "CustomCallOutputOperandAliases maps named fields to indices" { + const Input = struct { + q: Tensor, + k: Tensor, + v: Tensor, + }; + const Output = struct { + attn: Shape, + scratch: Shape, + }; + + const aliases = comptime customCallOutputOperandAliases(Input, Output, .{ + .attn = .q, + .scratch = .v, + }).?; + + try std.testing.expectEqual(@as(usize, 2), aliases.len); + try std.testing.expectEqualDeep( + stablehlo.CustomCallOpts.OutputOperandAlias{ .output_index = 0, .operand_index = 0 }, + aliases[0], + ); + try std.testing.expectEqualDeep( + stablehlo.CustomCallOpts.OutputOperandAlias{ .output_index = 1, .operand_index = 2 }, + aliases[1], + ); +} + +fn toUsize(values: anytype) stdx.BoundedArray(usize, constants.MAX_RANK) { + var res: stdx.BoundedArray(usize, constants.MAX_RANK) = .empty; + for (values) |val| res.appendAssumeCapacity(@intCast(val)); + return res; +} + +pub fn scaled_dot(lhs: Tensor, rhs: Tensor, lhs_scale: Tensor, rhs_scale: Tensor, args: anytype) Tensor { + const real_lhs = lhs.convert(.bf16).mul(lhs_scale.convert(.bf16)); + const real_rhs = rhs.convert(.bf16).mul(rhs_scale.convert(.bf16)); + + return real_lhs.dot(real_rhs, args).convert(.bf16); } diff --git a/zml/ops/manual_computation.zig b/zml/ops/manual_computation.zig new file mode 100644 index 00000000..a0ec0037 --- /dev/null +++ b/zml/ops/manual_computation.zig @@ -0,0 +1,546 @@ +const std = @import("std"); + +const mlir = @import("mlir"); +const shardy = @import("mlir/dialects/shardy"); +const stablehlo = @import("mlir/dialects/stablehlo"); +const stdx = @import("stdx"); + +const Compiler = @import("../Compiler.zig"); +const meta = @import("../meta.zig"); +const mlirx = @import("../mlirx.zig"); +const Shape = @import("../shape.zig").Shape; +const Sharding = @import("../Sharding.zig"); +const Tensor = @import("../tensor.zig").Tensor; + +const log = std.log.scoped(.@"zml/Compiler"); + +/// Partitioning declarations for the tensors in `T`. +/// Each `Tensor` becomes a `Sharding.Partitioning.Logical`. +/// For example, `struct { x: Tensor, bias: ?Tensor, scale: f32 }` becomes +/// `struct { x: Logical, bias: ?Logical }`. +/// +/// Initialize individual fields to declare different shardings, or use +/// `partitionSpecs` to apply one spec to every tensor in a value. +pub fn PartitionSpecs(comptime T: type) type { + return meta.MapRestrict(Tensor, Sharding.Partitioning.Logical).map(T); +} + +/// Declares the same sharding for a group of tensors at a `manualComputation` +/// boundary. +/// +/// ```zig +/// .in_specs = .{ +/// .query = .parse(.{ .h = .model }), +/// .cache = partitionSpecs(cache, .parse(.{ .h = .model })), +/// .metadata = partitionSpecs(metadata, .replicated), +/// } +/// ``` +/// +/// Requires an active compiler and uses its arena for allocations. Use +/// `partitionSpecsAlloc` outside graph construction. Use named struct fields +/// when mixing tensors with non-tensor metadata. +pub fn partitionSpecs(tensors: anytype, spec: Sharding.Partitioning.Logical) PartitionSpecs(@TypeOf(tensors)) { + return partitionSpecsAlloc(Compiler.current().arena.allocator(), tensors, spec) catch @panic("OOM"); +} + +/// Builds the same declarations as `partitionSpecs` using an explicit allocator. +pub fn partitionSpecsAlloc(allocator: std.mem.Allocator, tensors: anytype, spec: Sharding.Partitioning.Logical) std.mem.Allocator.Error!PartitionSpecs(@TypeOf(tensors)) { + var specs: PartitionSpecs(@TypeOf(tensors)) = undefined; + try meta.mapAlloc(struct { + fn assign(spec_: Sharding.Partitioning.Logical, _: Tensor) Sharding.Partitioning.Logical { + return spec_; + } + }.assign, allocator, spec, tensors, &specs); + return specs; +} + +/// Describes how the inputs and outputs of `body_fn` are partitioned across a mesh. +/// Returns the type of the `partitioning` argument to `manualComputation`. +/// +/// * `in_specs` specifies the sharding of each tensor in the body's first argument. +/// It follows the argument's structure, omitting non-tensor metadata. +/// * `out_specs` specifies the sharding of each shape in the body's second argument. +/// It follows the output shapes' structure and can be omitted for a `void` argument. +/// +/// Each spec is a `Sharding.Partitioning.Logical`. A mapping such as +/// `.parse(.{ .hidden = .model })` splits the tensor's `.hidden` dimension across +/// the mesh's `.model` axis. Omitted dimensions are replicated; `.replicated` +/// replicates the entire tensor. The specs must not contain open dimensions. +/// Slice lengths, optional presence and active union tags must match the arguments. +/// +/// For a body taking `struct { x: Tensor, bias: Tensor, scale: f32 }` and a single +/// output `Shape`, the partitioning can be declared as: +/// +/// ```zig +/// const partitioning: ManualPartitioning(body) = .{ +/// .in_specs = .{ .x = .parse(.{ .hidden = .model }), .bias = .replicated }, +/// .out_specs = .parse(.{ .hidden = .model }), +/// }; +/// ``` +pub fn ManualPartitioning(comptime body_fn: anytype) type { + const Logical = Sharding.Partitioning.Logical; + + const Inputs = stdx.meta.FnParam(body_fn, 0); + const OutputShapes = stdx.meta.FnParam(body_fn, 1); + + const InputSpecs = PartitionSpecs(Inputs); + const OutputSpecs = meta.MapRestrict(Shape, Logical).map(OutputShapes); + + if (OutputSpecs == void) return struct { + in_specs: InputSpecs, + out_specs: void = {}, + }; + + return struct { + in_specs: InputSpecs, + out_specs: OutputSpecs, + }; +} + +/// Describes a computation on local shards of tensors distributed across `mesh`. +/// ZML operations normally describe work on global tensors. The partitioner +/// distributes the inputs according to `in_specs`, and `body_fn` describes the work +/// performed on each shard. The results are exposed to the surrounding graph as +/// global tensors with the sharding declared by `out_specs`. +/// +/// * `body_fn` must be `fn (inputs: I, output_shapes: O) R`, where: +/// - `I` contains the local input tensors and any metadata needed by the body. +/// - `O` is `void`, `Shape`, or a container of shapes describing the local outputs. +/// - `R` is `void`, `Tensor`, or a container of tensors matching those local shapes +/// in traversal order. +/// * `inputs` contains the global tensors and metadata. The partitioner reshards +/// the tensors as needed to satisfy `in_specs`; metadata is passed to the body unchanged. +/// * `output_shapes` contains the global output shapes. The body receives their +/// local shard shapes, computed using `out_specs` and the sizes of the mesh axes. +/// * `mesh` is the logical mesh whose axis names are referenced by the specs. +/// * `partitioning` declares the input and output shardings; see `ManualPartitioning`. +/// Each partitioned dimension must be divisible by the size of its mesh axis. +/// +/// For example, splitting a dimension of size 64 across a mesh axis of size 8 gives +/// the body a dimension of size 8. A replicated dimension keeps its global size. +/// The caller supplies the global output shapes because the body only describes +/// local results: their shapes alone do not determine the global tensor shapes. +pub fn manualComputation( + comptime body_fn: anytype, + inputs: stdx.meta.FnParam(body_fn, 0), + output_shapes: stdx.meta.FnParam(body_fn, 1), + mesh: *const Sharding.Mesh, + partitioning: ManualPartitioning(body_fn), +) stdx.meta.FnReturn(body_fn) { + const Inputs = stdx.meta.FnParam(body_fn, 0); + const OutputShapes = stdx.meta.FnParam(body_fn, 1); + const Outputs = stdx.meta.FnReturn(body_fn); + + const compiler = Compiler.current(); + var arena_state: std.heap.ArenaAllocator = .init(compiler.allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + + // The body may return containers borrowed from its inputs or output shapes. + const body_arena = compiler.arena.allocator(); + + var owned_inputs: Inputs = undefined; + meta.mapAlloc((struct { + fn copy(_: void, tensor: Tensor) Tensor { + return tensor; + } + }).copy, body_arena, {}, inputs, &owned_inputs) catch @panic("OOM"); + + const flat_input_tensors: []*Tensor = meta.collectMutPtrs(Tensor, arena, &owned_inputs) catch @panic("OOM"); + const in_specs = meta.collectValues(Sharding.Partitioning.Logical, arena, &partitioning.in_specs) catch @panic("OOM"); + stdx.debug.assert(flat_input_tensors.len == in_specs.len, "manualComputation has {} input tensors but {} input specs", .{ flat_input_tensors.len, in_specs.len }); + + for (flat_input_tensors, in_specs) |tensor, spec| { + tensor._shape = tensor.shape().withPartitioning(mesh, spec); + stdx.debug.assert(!tensor.shape().containsPartitionSpec(.open), "manualComputation input sharding must be closed: {f}", .{tensor.shape()}); + } + + var local_output_shapes: OutputShapes = undefined; + meta.mapAlloc((struct { + fn copy(_: void, shape: Shape) Shape { + return shape; + } + }).copy, body_arena, {}, output_shapes, &local_output_shapes) catch @panic("OOM"); + + const flat_local_output_shapes = meta.collectMutPtrs(Shape, arena, &local_output_shapes) catch @panic("OOM"); + const out_specs = meta.collectValues(Sharding.Partitioning.Logical, arena, &partitioning.out_specs) catch @panic("OOM"); + stdx.debug.assert(flat_local_output_shapes.len == out_specs.len, "manualComputation has {} output shapes but {} output specs", .{ flat_local_output_shapes.len, out_specs.len }); + + const flat_output_shapes = arena.alloc(Shape, flat_local_output_shapes.len) catch @panic("OOM"); + for (flat_output_shapes, flat_local_output_shapes, out_specs) |*global, local, spec| { + global.* = local.withPartitioning(mesh, spec); + stdx.debug.assert(!global.containsPartitionSpec(.open), "manualComputation output sharding must be closed: {f}", .{global}); + local.* = Sharding.shardedShape(global.*) catch std.debug.panic("can't shard output {f} for {f}", .{ global, mesh }); + } + + const WrappedInputs = struct { + inputs: *const Inputs, + output_shapes: *const OutputShapes, + + fn wrap(allocator: std.mem.Allocator, opaque_wrapper: *const anyopaque, opaque_outputs: *anyopaque) error{OutOfMemory}![]*Tensor { + const wrapped: *const @This() = @ptrCast(@alignCast(opaque_wrapper)); + const outputs_ptr: *Outputs = @ptrCast(@alignCast(opaque_outputs)); + + outputs_ptr.* = body_fn(wrapped.inputs.*, wrapped.output_shapes.*); + + return try meta.collectMutPtrs(Tensor, allocator, outputs_ptr); + } + }; + + var outputs_struct: Outputs = undefined; + var wrapped_inputs: WrappedInputs = .{ .inputs = &owned_inputs, .output_shapes = &local_output_shapes }; + + _ = manualComputationInternal( + compiler, + arena, + mesh, + @typeName(@TypeOf(body_fn)), + &wrapped_inputs, + flat_input_tensors, + @ptrCast(&outputs_struct), + flat_output_shapes, + WrappedInputs.wrap, + ) catch |err| switch (err) { + error.OutOfMemory => @panic("OOM"), + }; + + return outputs_struct; +} + +const LocalComputationFn = *const fn (std.mem.Allocator, *const anyopaque, *anyopaque) error{OutOfMemory}![]*Tensor; + +/// Reified version of manualComputation. +/// This one is compiled exactly once, user code just need to compile the `collectPtrs` +noinline fn manualComputationInternal( + compiler: *Compiler, + arena: std.mem.Allocator, + mesh: *const Sharding.Mesh, + functor_name: []const u8, + input_struct: *anyopaque, + input_tensors: []*Tensor, + output_struct: *anyopaque, + global_output_shapes: []const Shape, + local_fn: LocalComputationFn, +) error{OutOfMemory}![]*Tensor { + const global_scope = compiler.currentScope(); + const mlir_ctx = compiler.mlir_ctx; + + log.debug("manualComputation({s}) will use mesh {s}", .{ functor_name, mesh.name }); + + const global_input_shapes = try arena.alloc(Shape, input_tensors.len); + for (global_input_shapes, input_tensors) |*shape, tensor| shape.* = tensor._shape; + + const input_types = try arena.alloc(*const mlir.Type, input_tensors.len); + for (input_types, input_tensors) |*input_ty, tensor_ptr| { + const global = tensor_ptr._shape; + const local = Sharding.shardedShape(global) catch std.debug.panic("can't shard {f} for {f}", .{ global, mesh }); + log.debug("\t- {f} -> {f}", .{ global, local }); + input_ty.* = mlirx.Type.rankedTensor(mlir_ctx, local); + } + const input_locs = try arena.alloc(*const mlir.Location, input_tensors.len); + @memset(input_locs, compiler.unknown_location); + + const input_values = try arena.alloc(*const mlir.Value, input_tensors.len); + for (input_values, input_tensors) |*input_val, tensor_ptr| { + if (compiler.partitioner == .gspmd) { + tensor_ptr.* = tensor_ptr.withPartitioning(mesh, tensor_ptr.shape()._sharding.partition); + } + input_val.* = global_scope.value(tensor_ptr.*); + } + + const manual_block = switch (compiler.partitioner) { + .shardy => mlir.Block.init(input_types, input_locs), + .gspmd => global_scope.block, + }; + errdefer if (compiler.partitioner == .shardy) manual_block.deinit(); + + const local_scope = compiler.pushBlock(manual_block); + defer local_scope.pop(); + for (0.., input_tensors, global_input_shapes) |i, tensor_ptr, global| { + const local = Sharding.shardedShape(global) catch unreachable; // we already computed this above + tensor_ptr.* = switch (compiler.partitioner) { + .shardy => Tensor._result(local, manual_block.argument(i)), + .gspmd => gspmd.globalToLocal(mlir_ctx, global_scope, tensor_ptr.*, local, compiler.location), + }; + } + + compiler.manual_computation_depth += 1; + defer compiler.manual_computation_depth -= 1; + + const local_outputs = try local_fn(arena, input_struct, output_struct); + stdx.debug.assert(local_outputs.len == global_output_shapes.len, "manualComputation body returned {} values, expected {}", .{ local_outputs.len, global_output_shapes.len }); + for (0.., local_outputs, global_output_shapes) |i, local_tensor, global_shape| { + const expected_local = Sharding.shardedShape(global_shape) catch std.debug.panic("can't shard output {f} for {f}", .{ global_shape, mesh }); + stdx.debug.assert(expected_local.eql(local_tensor._shape), "manualComputation body output {d} expected {f}, got {f}", .{ i, expected_local, local_tensor._shape }); + } + + const global_output_types = try arena.alloc(*const mlir.Type, global_output_shapes.len); + for (global_output_shapes, 0..) |output_shape, i| { + global_output_types[i] = mlirx.Type.rankedTensor(mlir_ctx, output_shape); + } + + const local_output_values = try arena.alloc(*const mlir.Value, local_outputs.len); + for (local_output_values, local_outputs) |*output_val, tensor_ptr| { + output_val.* = local_scope.value(tensor_ptr.*); + } + + const op: *const mlir.Operation = switch (compiler.partitioner) { + .shardy => op: { + // Shardy creates a sdy.manual_computation block. + _ = shardy.return_(mlir_ctx, local_output_values, compiler.location).appendTo(manual_block); + + const out_mesh_attrs = try arena.alloc(*const shardy.TensorShardingAttr, global_output_shapes.len); + for (out_mesh_attrs, global_output_shapes) |*attr, shape| { + attr.* = try mesh.sdyShardingAttrForShape(arena, mlir_ctx, shape); + } + + break :op shardy.manual_computation( + mlir_ctx, + input_values, + global_output_types, + manual_block, + try Sharding.sdyPerValueShardingAttr(arena, mlir_ctx, global_input_shapes, mesh), + .get(mlir_ctx, out_mesh_attrs), + try Sharding.sdyManualAxesAttr(arena, mlir_ctx, global_input_shapes, global_output_shapes, mesh), + compiler.location, + ).appendTo(global_scope.block); + }, + .gspmd => op: { + // Gspmd calls SPMDShardToFullShape and wrap into an optimizationBarrier + // Shortcut to avoid empty optimization barrier + if (global_output_shapes.len == 0) return &.{}; + + const global_values = try arena.alloc(*const mlir.Value, global_output_shapes.len); + const global_types = try arena.alloc(*const mlir.Type, global_output_shapes.len); + for (global_output_shapes, 0..) |global_shape, i| { + global_values[i], global_types[i] = try gspmd.localToGlobal(compiler, arena, global_scope, local_output_values[i], global_shape); + } + + break :op stablehlo.optimizationBarrier( + mlir_ctx, + global_values, + global_types, + compiler.location, + ).appendTo(global_scope.block); + }, + }; + + // Returned tensors describe the operation's global output shapes and sharding. + for (0.., local_outputs, global_output_shapes) |o, output_tensor_ptr, output_shape| { + output_tensor_ptr.* = Tensor._result(output_shape, op.result(o)); + output_tensor_ptr._shape._sharding = output_shape._sharding; + } + return local_outputs; +} + +const gspmd = struct { + fn globalToLocal(mlir_ctx: *mlir.Context, scope: *Compiler.Scope, local_tensor: Tensor, local_shape: Shape, location: *const mlir.Location) Tensor { + const local_type = mlirx.Type.rankedTensor(mlir_ctx, local_shape); + const full_to_shard = stablehlo.custom_call( + mlir_ctx, + &.{scope.value(local_tensor)}, + &.{local_type}, + .{ + .call_target_name = "SPMDFullToShardShape", + .has_side_effect = false, + .backend_config = .{ .original = "" }, + .additional_attributes = &.{ + .named(mlir_ctx, "mhlo.sharding", .string(mlir_ctx, "{manual}")), + }, + }, + location, + ).appendTo(scope.block); + return Tensor._result(local_shape, full_to_shard.result(0)); + } + + fn localToGlobal(compiler: *Compiler, arena: std.mem.Allocator, scope: *Compiler.Scope, local_value: *const mlir.Value, global_shape: Shape) !struct { *const mlir.Value, *const mlir.Type } { + const gspmd_attr = try compiler.tensorShardingAttr(arena, global_shape); + const global_type = mlirx.Type.rankedTensor(compiler.mlir_ctx, global_shape); + + const shard_to_full = stablehlo.custom_call( + compiler.mlir_ctx, + &.{local_value}, + &.{global_type}, + .{ + .call_target_name = "SPMDShardToFullShape", + .has_side_effect = false, + .backend_config = .{ .original = "" }, + .additional_attributes = &.{ + .named(compiler.mlir_ctx, "mhlo.sharding", gspmd_attr), + }, + }, + compiler.location, + ).appendTo(scope.block); + return .{ shard_to_full.result(0), global_type }; + } +}; + +test manualComputation { + const zml = @import("../zml.zig"); + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = zml.testing.env(); + const mesh = zml.testing.tpMesh(); + + const LocalSum = struct { + x: Tensor, + bias: Tensor, + scale: i32, + + fn body(self: @This(), output_shape: Shape) Tensor { + return self.x.sum(.x).scale(self.scale).add(self.bias).broad(output_shape); + } + + fn forward(x: Tensor, mesh_: *const Sharding.Mesh) Tensor { + return manualComputation( + body, + .{ .x = x, .bias = .scalar(10, .i32), .scale = 2 }, + x.shape(), + mesh_, + .{ .in_specs = .{ .x = .parse(.{ .x = .model }), .bias = .replicated }, .out_specs = .parse(.{ .x = .model }) }, + ); + } + }; + + // Each shard sums two elements and broadcasts the result to its local output shape. + const shape: Shape = .init(.{ .x = 2 * mesh.numPartitionsForLogicalAxis(.model) }, .i32); + const values = try allocator.alloc(i32, shape.count()); + defer allocator.free(values); + + for (values, 0..) |*value, i| value.* = @intCast(i + 1); + + var input = try zml.Buffer.fromBytes(io, platform, shape, std.mem.sliceAsBytes(values)); + defer input.deinit(); + + for ([_]Sharding.Partitioner{ .shardy, .gspmd }) |partitioner| { + const exe = try platform.compileFn(allocator, io, LocalSum.forward, .{ .fromShape(shape), mesh }, .{ + .meshes = &.{ platform.replicated_mesh, mesh }, + .partitioner = partitioner, + }); + defer exe.deinit(); + + var output = try exe.eval(allocator, io, .{input}); + defer output.deinit(); + + const actual = try output.toSliceAlloc(allocator, io); + defer actual.free(allocator); + + for (actual.constItems(i32), 0..) |value, i| { + const pair = values[i / 2 * 2 ..][0..2]; + try std.testing.expectEqual((pair[0] + pair[1]) * 2 + 10, value); + } + + try std.testing.expect(output.shape().eqlWithTags(shape)); + try std.testing.expectEqualDeep(shape.withPartitioning(mesh, .{ .x = .model })._sharding, output.shape()._sharding); + } +} + +test "manualComputation composes across logical and physical meshes" { + const zml = @import("../zml.zig"); + if (comptime !zml.Target.cpu.isEnabled()) return error.SkipZigTest; + const allocator = std.testing.allocator; + const io = std.testing.io; + + const Topologies = enum { flat, torus }; + + const Model = struct { + fn increment(x: Tensor, _: Shape) Tensor { + return x.addConstant(1); + } + + fn forward(x: Tensor, mesh: *const Sharding.Mesh) Tensor { + const partitioning: ManualPartitioning(increment) = .{ + .in_specs = partitionSpecs(x, .parse(.{ .x = .model })), + .out_specs = .parse(.{ .x = .model }), + }; + + const first = manualComputation(increment, x, x.shape(), mesh, partitioning); + const converted = first.convert(.f32); + return manualComputation(increment, converted, converted.shape(), mesh, partitioning); + } + }; + + const shape: Shape = .init(.{ .x = 16 }, .i32); + + var values: [16]i32 = undefined; + var expected: [16]f32 = undefined; + for (&values, &expected, 0..) |*value, *result, i| { + value.* = @intCast(i); + result.* = @floatFromInt(i + 2); + } + + for ([_]Topologies{ .flat, .torus }) |topology| { + const platform = try zml.Platform.init(allocator, io, .cpu, .{ + .cpu = .{ .device_count = 8 }, + .physical_mesh = if (topology == .torus) .{ .custom = Sharding.PhysicalMesh.torus3d } else .auto, + }); + defer platform.deinit(allocator, io); + + // Fold three physical axes into one logical axis, or split one into three. + const logical: Sharding.LogicalMesh = if (topology == .torus) + try .init(.{.model}, &.{.balanced}, &.{8}) + else + try Sharding.LogicalMesh.init(.{ .data, .model, .pipeline }, &.{ .low_bandwidth, .high_bandwidth, .balanced }, &.{ 2, 2, 2 }); + + const mesh = try platform.registerMesh("manual", logical); + + var input: zml.Buffer = try .fromBytes(io, platform, shape, std.mem.asBytes(&values)); + defer input.deinit(); + + for ([_]Sharding.Partitioner{ .shardy, .gspmd }) |partitioner| { + const exe = try platform.compileFn(allocator, io, Model.forward, .{ Tensor.fromShape(shape), mesh }, .{ + .meshes = &.{ platform.replicated_mesh, mesh }, + .partitioner = partitioner, + }); + defer exe.deinit(); + + var output = try exe.eval(allocator, io, .{input}); + defer output.deinit(); + + try std.testing.expectEqual(expected, try output.getValue([16]f32, io)); + try std.testing.expect(output.shape().eqlWithTags(shape.withDtype(.f32))); + try std.testing.expectEqualDeep(shape.withPartitioning(mesh, .{ .x = .model })._sharding, output.shape()._sharding); + } + } +} + +test "partitionSpecs preserves optional tensors and active cache variants" { + const zml = @import("../zml.zig"); + const Logical = Sharding.Partitioning.Logical; + const Inputs = struct { + layers: []const Tensor, + bias: ?Tensor, + cache: union(enum) { disabled, values: [2]Tensor }, + scale: f32, + }; + + var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); + defer arena.deinit(); + + const tensor: Tensor = .fromShape(.init(.{ .h = 8, .d = 16 }, .f32)); + var inputs: Inputs = .{ + .layers = &.{ tensor, tensor }, + .bias = null, + .cache = .{ .values = .{ tensor, tensor } }, + .scale = 2, + }; + const heads: Logical = .parse(.{ .h = .model }); + const specs = try zml.ops.partitionSpecsAlloc(arena.allocator(), inputs, heads); + const expected: zml.ops.PartitionSpecs(Inputs) = .{ + .layers = &.{ heads, heads }, + .bias = null, + .cache = .{ .values = .{ heads, heads } }, + }; + try std.testing.expectEqualDeep(expected, specs); + + inputs.layers = &.{}; + inputs.bias = tensor; + inputs.cache = .disabled; + const replicated = try zml.ops.partitionSpecsAlloc(arena.allocator(), inputs, .replicated); + const expected_replicated: zml.ops.PartitionSpecs(Inputs) = .{ + .layers = &.{}, + .bias = .replicated, + .cache = .disabled, + }; + try std.testing.expectEqualDeep(expected_replicated, replicated); +} diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig new file mode 100644 index 00000000..824d2bd1 --- /dev/null +++ b/zml/pjrtx.zig @@ -0,0 +1,130 @@ +const std = @import("std"); + +const mlir = @import("mlir"); +const pjrt = @import("pjrt"); +const stablehlo = @import("mlir/dialects/stablehlo"); + +pub const DataType = @import("dtype.zig").DataType; +const meta = @import("meta.zig"); +const Shape = @import("shape.zig").Shape; +const Tensor = @import("tensor.zig").Tensor; + +const log = std.log.scoped(.@"zml/pjrtx"); + +pub const Client = opaque { + pub const CompileError = std.mem.Allocator.Error || error{InvalidMlirBytecodeVersion} || pjrt.ApiError; + + pub fn compile(client: *const pjrt.Client, api: *const pjrt.Api, allocator: std.mem.Allocator, io: std.Io, module: *const mlir.Module, compile_options_pb: []const u8) CompileError!*pjrt.LoadedExecutable { + const stablehlo_version = blk: { + if (api.stablehloCurrentVersion()) |requested_version| { + break :blk stablehlo.smallerVersion(requested_version, stablehlo.currentVersion()); + } + break :blk stablehlo.minimumVersion(); + }; + + const cloned_op = module.operation().clone(); + var cloned_module = mlir.Module.fromOperation(cloned_op); + defer cloned_module.deinit(); + + var serialized_buffer: std.Io.Writer.Allocating = try .initCapacity(allocator, 4096); + defer serialized_buffer.deinit(); + + stablehlo.serializePortableArtifact2(cloned_module, stablehlo_version, &serialized_buffer.writer) catch |err| { + log.err("failed to serialize to portable artifact: {}", .{err}); + return switch (err) { + std.Io.Writer.Error.WriteFailed => error.OutOfMemory, + else => |e| e, + }; + }; + + return @ptrCast(try client.compile(api, io, .{ + .bytecode = serialized_buffer.written(), + .bytecode_format = .mlir, + .compile_options_pb = compile_options_pb, + })); + } +}; + +pub fn TensorToCustomCallBuffer(T: type) type { + return meta.MapRestrict(Tensor, CustomCallBuffer).map(T); +} + +pub fn ShapeToCustomCallBuffer(T: type) type { + return meta.MapRestrict(Shape, CustomCallBuffer).map(T); +} + +pub const CustomCallBuffer = struct { + shape: Shape, + ptr: *anyopaque, + + pub fn format( + self: @This(), + writer: *std.Io.Writer, + ) std.Io.Writer.Error!void { + try writer.print("{f}@{*}", .{ self.shape, self.ptr }); + } + + pub fn fromPjrt(buffer: *const pjrt.ffi.Buffer) @This() { + const dt = switch (buffer.dtype) { + .invalid => @panic("Found an invalid pjrt buffer"), + .token => @panic("Found an token pjrt buffer"), + inline else => |tag| @field(DataType, @tagName(tag)), + }; + return .{ + .ptr = buffer.data, + .shape = .init(buffer.dims(), dt), + }; + } + + // XLA: + // Align to 64-bytes, to mimic tsl::Allocator::kAllocatorAlignment. + // + // Preferred XLA:CPU alignment for buffers. XLA:CPU itself aligns intermediate + // buffers to this boundary (slices inside the preallocated temp buffer). + // https://github.com/openxla/xla/blob/fa2b7cfd909e8ffd343da1b7c5c341ace5e53aaa/xla/backends/cpu/alignment.h#L4 + pub const ALIGN = 64; + + // Assumes as host-side buffer + pub fn asHostSlice(self: CustomCallBuffer) []align(ALIGN) u8 { + const bytes: [*]align(ALIGN) u8 = @ptrCast(@alignCast(self.ptr)); + return bytes[0..self.shape.byteSize()]; + } + + // Assumes as host-side buffer + pub fn asHostConstSlice(self: CustomCallBuffer) []align(ALIGN) const u8 { + const bytes: [*]align(ALIGN) const u8 = @ptrCast(@alignCast(self.ptr)); + return bytes[0..self.shape.byteSize()]; + } + + // Assumes as host-side buffer + pub fn asHostScalar(self: CustomCallBuffer, T: type) T { + std.debug.assert(self.shape.byteSize() == @sizeOf(T)); + return std.mem.bytesAsValue(T, self.asHostConstSlice()[0..@sizeOf(T)]).*; + } +}; + +pub fn bufferTypeFromDtype(dt: DataType) pjrt.BufferType { + return switch (dt) { + inline else => |tag| @field(pjrt.BufferType, @tagName(tag)), + }; +} + +pub fn dtypeFromBufferType(pjrt_type: pjrt.BufferType) DataType { + return switch (pjrt_type) { + .invalid => @panic("Found an invalid pjrt buffer"), + inline else => |tag| @field(DataType, @tagName(tag)), + }; +} + +test bufferTypeFromDtype { + inline for (@typeInfo(DataType).@"enum".field_values) |field_value| { + const dt: DataType = @fromBackingInt(@intCast(field_value)); + try std.testing.expectEqual(dt, dtypeFromBufferType(bufferTypeFromDtype(dt))); + } + + inline for (@typeInfo(pjrt.BufferType).@"enum".field_values) |field_value| { + const dt: pjrt.BufferType = @fromBackingInt(@intCast(field_value)); + if (dt == .invalid) continue; + try std.testing.expectEqual(dt, bufferTypeFromDtype(dtypeFromBufferType(dt))); + } +} diff --git a/zml/platform.zig b/zml/platform.zig index f1844e15..54fb90e4 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -1,134 +1,1097 @@ -const builtin = @import("builtin"); const std = @import("std"); +const builtin = @import("builtin"); +const c = @import("c"); const pjrt = @import("pjrt"); -const asynk = @import("async"); +const stdx = @import("stdx"); +pub const Target = @import("platforms").Platform; + +const constants = @import("constants.zig"); +const Exe = @import("exe.zig").Exe; +const flashinfer_cutlass_contract = @import("kernels/moe/flashinfer_cutlass/contract.zig"); +const flashinfer_cutlass_runtime = @import("kernels/moe/flashinfer_cutlass/runtime.zig"); +const pjrtx = @import("pjrtx.zig"); +const profiler_ = @import("profiling/profiler.zig"); +const Sharding = @import("Sharding.zig"); +const zml = @import("zml.zig"); -const meta = @import("meta.zig"); -const module = @import("module.zig"); const log = std.log.scoped(.zml); -pub const Target = enum { - cpu, - cuda, - rocm, - tpu, +var api_map: std.enums.EnumArray(Target, ?*const pjrt.Api) = .initFill(null); + +fn disableXlaLogs() void { + // https://deepreg.readthedocs.io/en/latest/docs/logging.html#tensorflow-logging + const TF_CPP_LOG_LEVEL = struct { + const DEBUG = "0"; + const INFO = "1"; + const WARNING = "2"; + const ERROR = "3"; + }; + _ = c.setenv( + "TF_CPP_MIN_LOG_LEVEL", + std.c.getenv("TF_CPP_MIN_LOG_LEVEL") orelse TF_CPP_LOG_LEVEL.ERROR, + 1, + ); +} + +fn validateDeviceCount(target: Target, num_devices: usize) !void { + if (num_devices == 0) { + log.err("The selected platform requires at least 1 device, got {}", .{num_devices}); + return error.MissingDevices; + } + switch (target) { + .cpu, .cuda, .rocm, .tpu, .neuron, .metal, .oneapi, .furiosa => { + if (!std.math.isPowerOfTwo(num_devices)) { + log.err("Platform {} requires a power-of-two device count, got {}", .{ target, num_devices }); + return error.InvalidDeviceCount; + } + }, + } +} + +fn loadOrGetApi(allocator: std.mem.Allocator, io: std.Io, target: Target) !*const pjrt.Api { + return switch (target) { + inline else => |tgt| api_map.get(tgt) orelse b: { + disableXlaLogs(); + const api = try tgt.load(allocator, io); + api_map.set(tgt, api); + break :b api; + }, + }; +} + +pub const Memory = struct { + pub const Kind = enum { + default, + host_unpinned, + host_pinned, + device, + }; + + pjrt_memory: *const pjrt.Memory, + platform: *const Platform, + addressable_by_devices: []*const Device, + + fn init(allocator: std.mem.Allocator, pjrt_memory: *const pjrt.Memory, platform: *Platform) !Memory { + const pjrt_addressable_by_devices = pjrt_memory.addressableByDevices(platform.pjrt_api); + const addressable_by_devices = try allocator.alloc(*const Device, pjrt_addressable_by_devices.len); + + return .{ + .pjrt_memory = pjrt_memory, + .platform = platform, + .addressable_by_devices = addressable_by_devices, + }; + } + + pub fn kind(self: Memory) []const u8 { + return self.pjrt_memory.kind_(self.platform.pjrt_api); + } + + pub fn isOfKind(self: Memory, kind_: Kind) bool { + switch (self.platform.target) { + .cuda, .rocm, .oneapi, .tpu, .furiosa => { + const zml_kind: Memory.Kind = switch (self.kind().len) { + "device".len => .device, + "pinned_host".len => .host_pinned, + "unpinned_host".len => .host_unpinned, + else => std.debug.panic("unknown memory {s}", .{self.kind()}), + }; + return zml_kind == kind_; + }, + .cpu, .neuron, .metal => return true, + } + } + + fn deinit(self: *Memory, allocator: std.mem.Allocator) void { + allocator.free(self.addressable_devices); + } + + fn populateAddressableByDevices(self: *Memory) void { + const pjrt_addressable_by_devices = self.pjrt_memory.addressableByDevices(self.platform.pjrt_api); + for (pjrt_addressable_by_devices, self.addressable_by_devices) |pjrt_device, *addressable_by_device| { + addressable_by_device.* = self.platform.deviceFromPjrt(pjrt_device); + } + } }; -pub const available_targets = switch (builtin.os.tag) { - .macos => [_]Target{ - .cpu, - }, - .linux => [_]Target{ - .cpu, - .cuda, - .rocm, - .tpu, - }, - else => [_]Target{}, +pub const Device = struct { + platform: *const Platform, + pjrt_device: *const pjrt.Device, + pjrt_desc: *const pjrt.DeviceDescription, + addressable_memories: []*const Memory, + memory_by_kind: std.EnumArray(Memory.Kind, ?*const Memory), + + fn init(allocator: std.mem.Allocator, pjrt_device_: *const pjrt.Device, platform: *const Platform) !Device { + const pjrt_addressable_memories = pjrt_device_.addressableMemories(platform.pjrt_api); + const addressable_memories = try allocator.alloc(*const Memory, pjrt_addressable_memories.len); + for (pjrt_addressable_memories, addressable_memories) |pjrt_memory, *addressable_memory| { + addressable_memory.* = platform.memoryFromPjrt(pjrt_memory); + } + + // Cache memory lookups since they are expensive + const default_memory: *const Memory = resolveDefaultMemory(pjrt_device_, platform, addressable_memories); + const memory_by_kind: std.EnumArray(Memory.Kind, ?*const Memory) = .init(.{ + .default = default_memory, + .device = resolveMemory(addressable_memories, .device), + .host_pinned = resolveMemory(addressable_memories, .host_pinned), + .host_unpinned = resolveMemory(addressable_memories, .host_unpinned), + }); + + return .{ + .platform = platform, + .pjrt_device = pjrt_device_, + .pjrt_desc = pjrt_device_.getDescription(platform.pjrt_api), + .addressable_memories = addressable_memories, + .memory_by_kind = memory_by_kind, + }; + } + + fn deinit(self: *Device, allocator: std.mem.Allocator) void { + allocator.free(self.addressable_memories); + } + + fn resolveDefaultMemory(pjrt_device_: *const pjrt.Device, platform: *const Platform, addressable_memories: []*const Memory) *const Memory { + const pjrt_memory = pjrt_device_.defaultMemory(platform.pjrt_api); + for (addressable_memories) |mem| { + if (mem.pjrt_memory == pjrt_memory) return mem; + } + return platform.memoryFromPjrt(pjrt_memory); + } + + fn resolveMemory(addressable_memories: []*const Memory, memory_kind: Memory.Kind) ?*const Memory { + std.debug.assert(memory_kind != .default); + + for (addressable_memories) |mem| { + if (mem.isOfKind(memory_kind)) { + return mem; + } + } + return null; + } + + pub fn id(self: Device) u32 { + return @intCast(self.pjrt_desc.id(self.platform.pjrt_api)); + } + + pub fn processIndex(self: Device) i32 { + return self.pjrt_desc.processIndex(self.platform.pjrt_api); + } + + pub fn localHardwareId(self: Device) i32 { + return @intCast(self.pjrt_device.localHardwareId(self.platform.pjrt_api)); + } + + pub fn kind(self: Device) []const u8 { + return self.pjrt_desc.kind(self.platform.pjrt_api); + } + + pub fn debugString(self: Device) []const u8 { + return self.pjrt_desc.debugString(self.platform.pjrt_api); + } + + pub fn toString(self: Device) []const u8 { + return self.pjrt_desc.toString(self.platform.pjrt_api); + } + + pub fn memoryStats(self: Device) pjrt.Device.MemoryStats { + return self.pjrt_device.memoryStats(self.platform.pjrt_api) catch .zeroes; + } + + /// NUMA node closest to the device, when the plugin reports one. + pub fn numaNode(self: Device) ?u32 { + const attributes = self.pjrt_device.attributes(self.platform.pjrt_api) catch return null; + defer attributes.deinit(); + + const value = attributes.get("numa_node") orelse return null; + + return switch (value) { + .int64 => |node| std.math.cast(u32, node), + else => null, + }; + } + + pub fn format(self: Device, writer: *std.Io.Writer) std.Io.Writer.Error!void { + try writer.print("{s} ({s})", .{ + self.pjrt_desc.kind(self.platform.pjrt_api), + self.pjrt_desc.debugString(self.platform.pjrt_api), + }); + } + + pub fn memory(self: *const Device, memory_kind: Memory.Kind) ?*const Memory { + return self.memory_by_kind.values[@backingInt(memory_kind)]; + } }; -pub const CompilationOptions = struct { - xla_dump_to: ?[]const u8 = null, - xla_dump_fusion_visualization: bool = false, - cache_location: ?[]const u8 = null, - sharding_enabled: bool = false, - sharding_axes: std.BoundedArray([*:0]const u8, 8) = .{}, +fn platformDeviceSortId(target: Target, device: Device) usize { + return switch (target) { + .neuron => @intCast(device.localHardwareId()), + .cuda, .rocm, .tpu, .cpu, .oneapi, .metal, .furiosa => device.id(), + }; +} + +fn sortDevicesById(target: Target, devices: []Device) void { + const Context = struct { + target: Target, + + fn lessThan(ctx: @This(), lhs: Device, rhs: Device) bool { + return platformDeviceSortId(ctx.target, lhs) < platformDeviceSortId(ctx.target, rhs); + } + }; + + std.mem.sort(Device, devices, Context{ .target = target }, Context.lessThan); + + if (builtin.mode == .debug) { + for (devices, 0..) |device, expected_id| { + std.debug.assert(platformDeviceSortId(target, device) == expected_id); + } + } +} + +// State union tagged on target platform to handle related resources +pub const State = union(Target) { + cpu: void, + cuda: CudaState, + rocm: void, + tpu: void, + neuron: void, + oneapi: void, + metal: void, + furiosa: void, + + pub const CudaState = struct { + flashinfer_cutlass_moe_runners: ?*flashinfer_cutlass_runtime.Runners = null, + + fn deinit(self: *CudaState) void { + if (self.flashinfer_cutlass_moe_runners) |runners| { + runners.deinit(); + self.flashinfer_cutlass_moe_runners = null; + } + } + }; + + pub fn init(target: Target) State { + return switch (target) { + .cpu => .{ .cpu = {} }, + .cuda => .{ .cuda = .{} }, + .rocm => .{ .rocm = {} }, + .tpu => .{ .tpu = {} }, + .neuron => .{ .neuron = {} }, + .oneapi => .{ .oneapi = {} }, + .metal => .{ .metal = {} }, + .furiosa => .{ .furiosa = {} }, + }; + } + + pub fn deinit(self: *State) void { + switch (self.*) { + .cuda => |*cuda_state| cuda_state.deinit(), + else => {}, + } + } }; pub const Platform = struct { + arena: std.heap.ArenaAllocator, target: Target, pjrt_api: *const pjrt.Api, pjrt_client: *pjrt.Client, - compilation_options: CompilationOptions = .{}, + state: State, + devices: []const Device, + memories: []const Memory, + physical_mesh: zml.Sharding.PhysicalMesh, + meshes: std.StringArrayHashMapUnmanaged(*const zml.Mesh), + replicated_mesh: *const zml.Mesh, + io_impl: IoImpl, - pub const MAX_NUM_DEVICES: u8 = 8; + pub const MAX_NUM_DEVICES: u16 = if (Target.tpu.isEnabled()) 64 else 32; - pub fn init(target: Target, api: *const pjrt.Api) !Platform { - const pjrt_client = try pjrt.Client.init(api, &.{}); - const true_num_devices = pjrt_client.getAddressableDevices(api).len; - if (true_num_devices > MAX_NUM_DEVICES) { - log.warn("platform {} got {} devices, but ZML only support up to {} devices. Some devices won't be used.", .{ target, true_num_devices, MAX_NUM_DEVICES }); + pub const IoImpl = enum { + threaded, + zio, + }; + + pub fn init(allocator: std.mem.Allocator, io: std.Io, target: Target, options: CreateOptions) !*Platform { + const api = try loadOrGetApi(allocator, io, target); + + var named_values_buf: [16]pjrt.NamedValue = undefined; + const pjrt_client = try pjrt.Client.init(api, options.toNamedValues(target, &named_values_buf)); + const pjrt_devices = pjrt_client.addressableDevices(api); + try validateDeviceCount(target, pjrt_devices.len); + if (pjrt_devices.len > MAX_NUM_DEVICES) { + log.warn("platform {} got {} devices, but ZML only support up to {} devices. Some devices won't be used.", .{ target, pjrt_devices.len, MAX_NUM_DEVICES }); } - return .{ - .target = target, - .pjrt_api = api, - .pjrt_client = pjrt_client, - .compilation_options = .{}, + + const pjrt_memories = pjrt_client.addressableMemories(api); + + // Note: Platform is a self-owning struct. It contains the arena that created it in the first place + // But it does mean we have to be careful to pass the arena state that contains the node + const platform: *Platform = platform: { + var initial_arena = std.heap.ArenaAllocator.init(allocator); + errdefer initial_arena.deinit(); + + var initial_allocator = initial_arena.allocator(); + // Pre-heat the arena, to avoid fragmentation. + initial_allocator.free(try initial_allocator.alloc(u8, 8 * 1024)); + + const platform = try initial_allocator.create(Platform); + platform.* = .{ + .arena = initial_arena, + .target = target, + .pjrt_api = api, + .pjrt_client = pjrt_client, + .state = State.init(target), + .meshes = .empty, + .replicated_mesh = undefined, + // set below + .devices = undefined, + .memories = undefined, + .physical_mesh = undefined, + .io_impl = options.io_impl, + }; + break :platform platform; }; - } - pub fn getDevices(self: Platform) []const *const pjrt.Device { - const all_devices = self.pjrt_client.getAddressableDevices(self.pjrt_api); - if (all_devices.len > MAX_NUM_DEVICES) { - return all_devices[0..MAX_NUM_DEVICES]; - } - return all_devices; - } + const arena = platform.arena.allocator(); + errdefer platform.arena.deinit(); + try platform.meshes.ensureTotalCapacity(arena, 8); - pub const Sharding = struct { num_replicas: u8, num_partitions: u8 }; + { + const devices = try arena.alloc(Device, pjrt_devices.len); + platform.devices = devices; + const memories = try arena.alloc(Memory, pjrt_memories.len); + platform.memories = memories; - pub fn sharding(self: Platform) Sharding { - // replicas run the same function but with different inputs, - // while partitions contribute to one evaluation over a shared input. - // Inside an inference process, we generally don't want replicas, - // as it's best to fully isolate replicas on different processes. - // For now we hardcode num_replicas = 1. - const num_devices: u8 = @intCast(self.getDevices().len); - return if (self.compilation_options.sharding_enabled) - .{ .num_replicas = 1, .num_partitions = num_devices } - else - .{ .num_replicas = 1, .num_partitions = 1 }; - } - - pub fn withCompilationOptions(self: Platform, opts: CompilationOptions) Platform { - var res = self; - res.compilation_options = opts; - return res; - } - - pub fn deinit(self: *Platform) void { - self.pjrt_client.deinit(self.pjrt_api); - } - - /// Returns the Profiler for this API. - /// Not all platform have a profiling api, for those the profiler object will do nothing. - /// Platforms with known profiler extensions: cuda, xpu - pub fn getProfiler(self: Platform, options: pjrt.Profiler.Options) pjrt.Profiler { - return self.pjrt_client.getProfiler(self.pjrt_api, options); - } - - /// Suspend the current co-routine while awaiting for a pjrt event to be over. - pub fn awaitEvent(self: Platform, event: *pjrt.Event) !void { - defer event.deinit(self.pjrt_api); - // If we aren't in a coroutine just use the normal blocking api. - if (!asynk.inCoro()) { - return try event.await_(self.pjrt_api); - } - - var ctx = struct { - err: ?*pjrt.Error = null, - notif: asynk.Notification, - ready: bool = false, - }{ - .notif = try asynk.Notification.init(), - }; - // NOTE(Corentin): Disable this to avoid crash with epoll backend - //defer ctx.notif.deinit(); - - try event.onReady(self.pjrt_api, &(struct { - fn call(err: ?*pjrt.Error, user_arg: ?*anyopaque) callconv(.C) void { - const ctx_: *@TypeOf(ctx) = @ptrCast(@alignCast(user_arg.?)); - ctx_.err = err; - @atomicStore(bool, &ctx_.ready, true, .seq_cst); - ctx_.notif.notify() catch @panic("Unable to notify"); + // TODO: part of the complication here is that we layout the data in spaghetti mode, + // where devices and memories point to each other and also point to the platform. + for (pjrt_memories, memories) |pjrt_memory, *platform_memory| { + platform_memory.* = try .init(arena, pjrt_memory, platform); } - }.call), &ctx); - // Suspend - try ctx.notif.wait(); - if (ctx.err) |e| { - defer e.deinit(self.pjrt_api); - return e.getCode(self.pjrt_api).toApiError(); + for (pjrt_devices, devices) |pjrt_device, *platform_device| { + platform_device.* = try .init(arena, pjrt_device, platform); + } + sortDevicesById(target, devices); + for (memories) |*platform_memory| { + platform_memory.populateAddressableByDevices(); + } + + platform.physical_mesh = try switch (options.physical_mesh) { + .auto => zml.Sharding.PhysicalMesh.auto(arena, target, devices), + .custom => |builder| builder(arena, target, devices), + }; + platform.replicated_mesh = try platform.registerMesh("replicated", .mesh(.{ .all = .high_bandwidth })); } + + switch (target) { + .cuda => { + cuda.initFlashinferCutlass(platform, io) catch |err| { + log.warn("Failed to initialize Flashinfer CUTLASS MoE: {}", .{err}); + }; + }, + else => {}, + } + + platform.registerFfi( + .{ + .name = "zml$print", + .handler = printCallback, + .traits = .{ .command_buffer_compatible = false }, + }, + ) catch |err| { + log.warn("Failed to register FFI custom call \"zml$print\", error: {}", .{err}); + }; + + return platform; + } + + pub fn auto(allocator: std.mem.Allocator, io: std.Io, options: CreateOptions) !*Platform { + const ordered_targets: []const Target = &.{ + .tpu, + .neuron, + .rocm, + .cuda, + .oneapi, + .metal, + .furiosa, + .cpu, + }; + return for (ordered_targets) |target| { + break init(allocator, io, target, options) catch continue; + } else error.Unavailable; + } + + pub fn formatWithAttributes(self: *const Platform, writer: *std.Io.Writer) std.Io.Writer.Error!void { + const tee = "├─ "; + const line = "│ "; + const langle = "└─ "; + + try writer.print("platform: {s}\n", .{@tagName(self.target)}); + try writer.print("version: {f}\n", .{self.pjrt_api.version()}); + + try writer.print("extensions:\n", .{}); + { + var it = self.pjrt_api.extensions(); + while (it.next()) |ext| { + try writer.print("{s}{s}\n", .{ if (it.current != null) tee else langle, @tagName(ext) }); + } + } + + try writer.print("plugin attributes:\n", .{}); + { + const attributes = self.pjrt_api.pluginAttributes(); + if (attributes.len == 0) { + try writer.print("{s}(none)\n", .{langle}); + } else { + for (attributes, 0..) |attr, i| { + const is_last_attr = i == attributes.len - 1; + try writer.print("{s}{s}", .{ if (is_last_attr) langle else tee, attr.name() }); + switch (attr.value()) { + .string => |v| try writer.print(": \"{s}\"", .{v}), + .int64 => |v| try writer.print(": {d}", .{v}), + .int64list => |v| { + for (v, 0..) |item, j| { + if (j == 0) { + try writer.print(": {d}", .{item}); + } else { + try writer.print(".{d}", .{item}); + } + } + }, + .float => |v| try writer.print(": {d}", .{v}), + .bool => |v| try writer.print(": {}", .{v}), + } + try writer.writeAll("\n"); + } + } + } + + try writer.print("devices:\n", .{}); + for (self.devices, 0..) |device, i| { + const is_last_device = i == self.devices.len - 1; + const child_indent = if (is_last_device) " " else line; + + try writer.print("{s}{f}\n", .{ if (is_last_device) langle else tee, device }); + + { + const device_attrs = device.pjrt_desc.attributes(self.pjrt_api); + if (device_attrs.len > 0) { + var last_name: ?[]const u8 = null; + var remaining: usize = device_attrs.len; + + while (remaining > 0) : (remaining -= 1) { + var next_index: ?usize = null; + for (device_attrs, 0..) |attr, j| { + const name = attr.name(); + if (last_name) |last| { + if (std.mem.order(u8, name, last) != .gt) continue; + } + if (next_index) |ni| { + if (std.mem.order(u8, name, device_attrs[ni].name()) == .lt) { + next_index = j; + } + } else { + next_index = j; + } + } + + if (next_index == null) break; + const attr = device_attrs[next_index.?]; + + try writer.print("{s}{s}{s}", .{ child_indent, tee, attr.name() }); + switch (attr.value()) { + .string => |v| try writer.print(": \"{s}\"", .{v}), + .int64 => |v| try writer.print(": {d}", .{v}), + .int64list => |v| try writer.print(": {any}", .{v}), + .float => |v| try writer.print(": {d}", .{v}), + .bool => |v| try writer.print(": {}", .{v}), + } + try writer.writeAll("\n"); + + last_name = attr.name(); + } + } + } + + try writer.print("{s}{s}memories:\n", .{ child_indent, langle }); + { + const memory_indent = " "; + if (device.addressable_memories.len == 0) { + try writer.print("{s}{s}{s}(none)\n", .{ child_indent, memory_indent, langle }); + } else { + for (device.addressable_memories, 0..) |mem, j| { + const is_last_mem = j == device.addressable_memories.len - 1; + try writer.print("{s}{s}{s}memory: {s}\n", .{ + child_indent, + memory_indent, + if (is_last_mem) langle else tee, + mem.pjrt_memory.debugString(self.pjrt_api), + }); + } + } + } + } + } + + pub fn fmtVerbose(self: *const Platform) std.fmt.Alt(*const Platform, formatWithAttributes) { + return .{ .data = self }; + } + + pub fn deinit(self: *Platform, allocator: std.mem.Allocator, io: std.Io) void { + _ = io; + _ = allocator; + if (comptime Target.cuda.isEnabled()) { + self.state.deinit(); + } + self.physical_mesh.deinit(self.arena.allocator()); + self.pjrt_client.deinit(self.pjrt_api); + self.arena.deinit(); + } + + pub fn compile( + self: *const Platform, + allocator: std.mem.Allocator, + io: std.Io, + model_: anytype, + comptime func: std.meta.DeclEnum(@TypeOf(model_)), + args: stdx.meta.Tail( + std.meta.ArgsTuple(@TypeOf(@field(@TypeOf(model_), @tagName(func)))), + ), + opts: zml.Compiler.Options, + ) !Exe { + return self.compileFn( + allocator, + io, + @field(@TypeOf(model_), @tagName(func)), + .{model_} ++ args, + opts, + ); + } + + pub fn compileModel( + self: *const Platform, + allocator: std.mem.Allocator, + io: std.Io, + comptime func: anytype, + model: stdx.meta.Head(std.meta.ArgsTuple(@TypeOf(func))), + args: stdx.meta.Tail(std.meta.ArgsTuple(@TypeOf(func))), + opts: zml.Compiler.Options, + ) !Exe { + return self.compileFn(allocator, io, func, .{model} ++ args, opts); + } + + pub fn compileFn( + self: *const Platform, + allocator: std.mem.Allocator, + io: std.Io, + comptime func: anytype, + args: std.meta.ArgsTuple(@TypeOf(func)), + opts: zml.Compiler.Options, + ) !Exe { + return zml.Compiler.compile(allocator, io, self, func, args, opts); + } + + pub fn format(self: *const Platform, writer: *std.Io.Writer) std.Io.Writer.Error!void { + try writer.print("{s} {{ ", .{@tagName(self.target)}); + for (self.devices(), 0..) |device, i| { + try writer.print("{s}(\"{s}\")", .{ device.toString(), device.kind() }); + if (i < self.devices.len - 1) try writer.writeAll(", "); + } + try writer.writeAll(" }"); + } + + pub fn memoryKind(self: *const Platform, kind: Memory.Kind) []const u8 { + for (self.memories) |mem| { + if (mem.isOfKind(kind)) { + return mem.kind(); + } + } + unreachable; + } + + pub const FfiRegistration = struct { + name: []const u8, + handler: *const pjrt.ffi.Handler, + traits: pjrt.ffi.HandlerTraits = .{ .command_buffer_compatible = false }, + platform_name: ?[]const u8 = null, + }; + + pub fn registerFfi(self: *const zml.Platform, registration: FfiRegistration) !void { + const platform_name = registration.platform_name orelse self.pjrt_client.platformName(self.pjrt_api); + if (self.pjrt_api.ffi()) |ffi| { + try ffi.register(self.pjrt_api, registration.name, platform_name, registration.handler, registration.traits); + } else { + log.warn("PJRT FFI extension not available for {s}", .{@tagName(self.target)}); + } + } + + pub const Profiler = profiler_.Profiler; + pub const ProfilerOptions = profiler_.ProfilerOptions; + + pub fn profiler(self: *const Platform, allocator: std.mem.Allocator, io: std.Io, options: ProfilerOptions) !Profiler { + return try profiler_.profiler(self.pjrt_api, allocator, io, options); + } + + /// Create a Mesh based on the given logical mesh and the default strategy. + /// Memory is owned by the platform, making it safe to copy around. + pub fn registerMesh(platform: *Platform, name: []const u8, logical: Sharding.LogicalMesh) error{OutOfMemory}!*const Sharding.Mesh { + return platform.registerMeshWithStrategy( + name, + logical, + .suggest(logical, &platform.physical_mesh), + ) catch |err| switch (err) { + error.InvalidPhysicalMesh, error.InvalidStrategy, error.InvalidPhysicalAxis, error.OverlappingLogicalAxes => { + std.debug.panic("ZML failed to create a valid mesh for logical mesh: {f}\nand physical_mesh: {f}\nPlease report this bug.", .{ logical, platform.physical_mesh }); + }, + error.InvalidPartialBinding, error.InvalidLogicalDimension, error.IncompatibleLogicalDimension => { + std.debug.panic("ZML cannot satisfy the dimensions of logical mesh {f} on physical mesh {f}", .{ logical, platform.physical_mesh }); + }, + error.OutOfMemory => |e| return e, + }; + } + + /// Create a Mesh based on the given logical mesh and a strategy. + /// Memory is owned by the platform, making it safe to copy around. + pub fn registerMeshWithStrategy(platform: *Platform, name: []const u8, logical: Sharding.LogicalMesh, strategy: Sharding.Strategy) !*const Sharding.Mesh { + const arena = platform.arena.allocator(); + const entry = try platform.meshes.getOrPut(arena, name); + if (entry.found_existing) { + std.debug.panic("Another mesh already exists with this name: {s}", .{name}); + } + + const owned_name = try arena.dupe(u8, name); + const owned_mesh = try arena.create(Sharding.Mesh); + owned_mesh.* = try .init(owned_name, &platform.physical_mesh, logical, strategy); + entry.key_ptr.* = owned_name; + entry.value_ptr.* = owned_mesh; + + return owned_mesh; + } + + fn memoryFromPjrt(self: *const Platform, pjrt_memory: *const pjrt.Memory) *const Memory { + for (self.memories) |*mem| { + if (mem.pjrt_memory == pjrt_memory) return mem; + } + unreachable; + } + + fn deviceFromPjrt(self: *const Platform, pjrt_device: *const pjrt.Device) *const Device { + for (self.devices) |*device| { + if (device.pjrt_device == pjrt_device) return device; + } + unreachable; + } + + pub inline fn defaultMemoryLayout(platform: *const Platform, dims: []const i64, dtype: zml.DataType) pjrt.MemoryLayout { + // inline cause `default` is a huge ass struct allocated on the stack, + // and toMemoryLayout returns slices into it. + // There is probably a better way of doing this, + // The queried layout can include backend-specific tiling. + return switch (platform.target) { + .tpu, .furiosa => { + if (comptime !Target.tpu.isEnabled() and !Target.furiosa.isEnabled()) unreachable; + const element_type = pjrtx.bufferTypeFromDtype(dtype); + const default = platform.pjrt_client.defaultMemoryLayout(platform.pjrt_api, element_type, dims) catch @panic("Failed to get default memory layout"); + return default.toMemoryLayout(); + }, + .cuda, .rocm, .neuron, .oneapi, .cpu, .metal => .{ + // If this is the default layout on the platform, there is no point calling PJRT + .tiled = .{ + .minor_to_major = constants.minorToMajor(@intCast(dims.len)), + .tile_dims = &.{}, + .tile_dims_sizes = &.{}, + }, + }, + }; } }; + +pub const CreateOptions = struct { + pub const CreatePhysicalMeshFn = *const fn ( + allocator: std.mem.Allocator, + target: Target, + devices: []const Device, + ) anyerror!zml.Sharding.PhysicalMesh; + + pub const PhysicalMesh = union(enum) { + auto, + custom: CreatePhysicalMeshFn, + }; + + physical_mesh: PhysicalMesh = .auto, + cpu: Cpu = .{ .device_count = 4 }, + + // bump memory fraction from XLA defaults of 75% to 90%. + // Even on a 8GB GPU it should leave enough space for the platform driver/runtime. + // https://github.com/openxla/xla/blob/3e87afa11a865cf91137522492918ad18bfe5b7c/xla/pjrt/plugin/xla_gpu/xla_gpu_allocator_config.h#L25-L60 + rocm: XlaGpu = .{ .allocator = .{ .vmm = .{ .memory_fraction = 0.90 } } }, + cuda: XlaGpu = .{ .allocator = .{ .vmm = .{ .memory_fraction = 0.90 } } }, + tpu: struct {} = .{}, + neuron: struct {} = .{}, + furiosa: Furiosa = .{}, + oneapi: XlaGpu = .{ .allocator = .{ .bfc = .{ .preallocate = true, .memory_fraction = 0.90 } } }, + metal: XlaGpu = .{ .allocator = .{ .bfc = .{ .preallocate = true, .memory_fraction = 0.90 } } }, + io_impl: Platform.IoImpl = .threaded, + + pub const Furiosa = struct { + pe_count: u8 = 8, + + fn writeNamedValues(self: Furiosa, values: *std.ArrayList(pjrt.NamedValue)) void { + values.appendAssumeCapacity(.init(.int64, "pe_count", self.pe_count)); + } + }; + + pub const Cpu = struct { + device_count: u32, + + fn writeNamedValues(self: Cpu, values: *std.ArrayList(pjrt.NamedValue)) void { + values.appendAssumeCapacity(.init(.int64, "cpu_device_count", self.device_count)); + } + }; + + pub const XlaGpu = struct { + allocator: Allocator = .{ .bfc = .{} }, + /// The PJRT C API still exposes this under legacy + /// `use_tfrt_gpu_client` name. + gpu_async_dispatch: bool = false, + // TODO support all of https://github.com/openxla/xla/blob/3d31c48c719d331d432132b3e0c2c5ce52650675/xla/pjrt/c/pjrt_c_api_gpu_internal.cc#L76-L86 + // visible_devices: []const i64 = &.{}, + // node_id + // num_nodes + // enable_mock_nccl + // mock_gpu_topology + + pub const Allocator = union(enum) { + /// "Best-Fit with Coalescing" algorithm + bfc: Options, + /// use cudaMallocAsync + async: Options, + /// use virtual memory allocator + vmm: VmmOptions, + /// use raw cuMalloc + platform, + + pub const Options = struct { + preallocate: bool = true, + memory_fraction: f32 = 0.90, + collective_memory_size_mb: i64 = 0, + }; + + pub const VmmOptions = struct { + memory_fraction: f32 = 0.90, + }; + }; + + fn writeNamedValues(self: XlaGpu, target: Target, values: *std.ArrayList(pjrt.NamedValue)) void { + switch (self.allocator) { + .platform => { + values.appendAssumeCapacity(.init(.string, "allocator", "platform")); + }, + .bfc, .async => |opt| { + values.appendAssumeCapacity(.init(.string, "allocator", switch (self.allocator) { + .bfc => "bfc", + .async => "cuda_async", + .platform, .vmm => unreachable, + })); + values.appendAssumeCapacity(.init(.bool, "preallocate", opt.preallocate)); + if (opt.memory_fraction > 0) { + values.appendAssumeCapacity(.init(.float, "memory_fraction", opt.memory_fraction)); + } + if (opt.collective_memory_size_mb > 0) { + values.appendAssumeCapacity(.init(.int64, "collective_memory_size", opt.collective_memory_size_mb * 1024 * 1024)); + } + }, + .vmm => |opt| { + values.appendAssumeCapacity(.init(.string, "allocator", "vmm")); + if (opt.memory_fraction > 0) { + values.appendAssumeCapacity(.init(.float, "memory_fraction", opt.memory_fraction)); + } + }, + } + switch (target) { + .cuda => values.appendAssumeCapacity(.init(.bool, "use_tfrt_gpu_client", self.gpu_async_dispatch)), + else => {}, + } + } + }; + + pub fn toNamedValues(self: CreateOptions, target: Target, out: []pjrt.NamedValue) []pjrt.NamedValue { + var values = std.ArrayList(pjrt.NamedValue).fromOwnedSlice(out); + values.shrinkRetainingCapacity(0); + switch (target) { + .cpu => self.cpu.writeNamedValues(&values), + .furiosa => self.furiosa.writeNamedValues(&values), + .cuda => self.cuda.writeNamedValues(target, &values), + .rocm => self.rocm.writeNamedValues(target, &values), + .oneapi => self.oneapi.writeNamedValues(target, &values), + .metal => self.metal.writeNamedValues(target, &values), + inline else => |t| { + stdx.debug.assertComptime(@hasField(CreateOptions, @tagName(t)), "zml.platform.CreateOptions doesn't list target {s}", .{@tagName(t)}); + const options = @field(self, @tagName(t)); + stdx.debug.assertComptime(@sizeOf(@TypeOf(options)) == 0, "zml.platform.CreateOptions.{s} is discarded", .{@tagName(t)}); + }, + } + return values.items; + } +}; + +// TODO(Corendos): Consider moving that in its own file if its size increase too much. +pub const cuda = struct { + pub const ComputeCapability = struct { + major: u8, + minor: u8, + + pub fn parse(text: []const u8) ?ComputeCapability { + var parts = std.mem.splitScalar(u8, text, '.'); + return .{ + .major = std.fmt.parseInt(u8, parts.first(), 10) catch return null, + .minor = std.fmt.parseInt(u8, parts.next() orelse "0", 10) catch return null, + }; + } + + pub fn eql(self: ComputeCapability, other: ComputeCapability) bool { + return self.major == other.major and self.minor == other.minor; + } + + pub fn atLeast(self: ComputeCapability, other: ComputeCapability) bool { + return self.major > other.major or (self.major == other.major and self.minor >= other.minor); + } + + pub fn sm(self: ComputeCapability) u16 { + return @as(u16, self.major) * 10 + self.minor; + } + }; + + /// SM count of an addressable CUDA device; unavailable attributes return null. + pub fn coreCount(platform: *const zml.Platform, device_index: usize) ?u32 { + const attribute = platform.devices[device_index].pjrt_desc.attribute(platform.pjrt_api, "core_count") orelse return null; + return std.math.cast(u32, attribute.int64); + } + + pub fn computeCapability(platform: *const zml.Platform) ?ComputeCapability { + if (platform.target != .cuda) return null; + const devices = platform.pjrt_client.devices(platform.pjrt_api); + if (devices.len == 0) return null; + + const attributes = devices[0].getDescription(platform.pjrt_api).attributes(platform.pjrt_api); + return for (attributes) |attr| { + if (std.mem.eql(u8, attr.name(), "compute_capability")) { + break ComputeCapability.parse(attr.value().string); + } + } else null; + } + + fn initFlashinferCutlass(platform: *Platform, io: std.Io) !void { + if (comptime !Target.cuda.isEnabled()) return error.UnsupportedPlatform; + const cc = computeCapability(platform) orelse return error.UnsupportedArchitecture; + + const allocator = platform.arena.allocator(); + const runners = try allocator.create(flashinfer_cutlass_runtime.Runners); + errdefer allocator.destroy(runners); + + try flashinfer_cutlass_runtime.load(allocator, io, cc.sm()); + + runners.* = .{}; + errdefer runners.deinit(); + + try platform.registerFfi(.{ + .name = flashinfer_cutlass_contract.name, + .handler = flashinfer_cutlass_runtime.handler, + .traits = .{ .command_buffer_compatible = true }, + .platform_name = if (flashinfer_cutlass_contract.compute_on_host) "Host" else null, + }); + + platform.state.cuda.flashinfer_cutlass_moe_runners = runners; + } +}; + +pub const rocm = struct { + pub const ComputeCapability = enum { + /// CDNA1: MI100. + gfx908, + /// CDNA2: MI200 series. + gfx90a, + /// CDNA3: MI300 series. + gfx942, + /// CDNA4: MI350 series. + gfx950, + /// RDNA2: RX 6000 series. + gfx1030, + /// RDNA3 (Navi 31): RX 7900 series. + gfx1100, + /// RDNA3 (Navi 32): RX 7800 and RX 7700 series. + gfx1101, + /// RDNA3 (Navi 33): RX 7600 series. + gfx1102, + /// RDNA3 APU: Phoenix. + gfx1103, + /// RDNA3.5: newer Ryzen AI APUs. + gfx1150, + /// RDNA4 (Navi 44): RX 9060 family. + gfx1200, + /// RDNA4 (Navi 48): RX 9070 family. + gfx1201, + + pub const Architecture = enum { + cdna1, + cdna2, + cdna3, + cdna4, + rdna2, + rdna3, + rdna3_5, + rdna4, + }; + + pub fn architecture(self: ComputeCapability) Architecture { + return switch (self) { + .gfx908 => .cdna1, + .gfx90a => .cdna2, + .gfx942 => .cdna3, + .gfx950 => .cdna4, + .gfx1030 => .rdna2, + .gfx1100, .gfx1101, .gfx1102, .gfx1103 => .rdna3, + .gfx1150 => .rdna3_5, + .gfx1200, .gfx1201 => .rdna4, + }; + } + }; + + /// Assumes homogeneous devices. + pub fn computeCapability(platform: *const zml.Platform) ?ComputeCapability { + stdx.debug.assert(platform.target == .rocm, "computeCapability expects .rocm platform, got {}", .{platform.target}); + const device = platform.pjrt_client.devices(platform.pjrt_api)[0]; + const description = device.getDescription(platform.pjrt_api); + + const attributes = description.attributes(platform.pjrt_api); + return for (attributes) |attr| { + if (std.mem.eql(u8, attr.name(), "compute_capability")) { + break std.meta.stringToEnum(ComputeCapability, std.mem.sliceTo(attr.value().string, ':')); + } + } else null; + } +}; + +fn dataTypeFromFfiDataType(ffi_dt: pjrt.ffi.DataType) zml.DataType { + return switch (ffi_dt) { + .bool => .bool, + .i8 => .i8, + .i16 => .i16, + .i32 => .i32, + .i64 => .i64, + .u8 => .u8, + .u16 => .u16, + .u32 => .u32, + .u64 => .u64, + .f16 => .f16, + .f32 => .f32, + .f64 => .f64, + .bf16 => .bf16, + .c64 => .c64, + .c128 => .c128, + .f8e5m2 => .f8e5m2, + .f8e4m3fn => .f8e4m3fn, + .f8e4m3b11fnuz => .f8e4m3b11fnuz, + .f8e5m2fnuz => .f8e5m2fnuz, + .f8e4m3fnuz => .f8e4m3fnuz, + else => unreachable, + }; +} + +fn shapeFromFfiBuffer(buffer: *const pjrt.ffi.Buffer) zml.Shape { + return .init(buffer.dims(), dataTypeFromFfiDataType(buffer.dtype)); +} + +fn getScalarAttributeAs(comptime T: type, call_frame: *pjrt.ffi.CallFrame, attribute_name: []const u8) ?T { + const attribute = call_frame.attrs.getByName(.scalar, attribute_name) orelse return null; + return attribute.get(T); +} + +fn printCallback(call_frame: *pjrt.ffi.CallFrame) callconv(.c) ?*pjrt.ffi.Error { + return printCallbackInner(call_frame) catch |e| b: { + log.err("Error in print callback: {}", .{e}); + break :b pjrt.ffi.Error.create(call_frame.api, .unknown, "Unknown"); + }; +} + +fn printCallbackInner(call_frame: *pjrt.ffi.CallFrame) !?*pjrt.ffi.Error { + if (call_frame.registeringHook()) return null; + + const buffers = call_frame.args.buffers(); + if (buffers.len <= 0) return null; + + const pjrt_api: *pjrt.Api = @ptrFromInt(getScalarAttributeAs(u64, call_frame, "pjrt_api").?); + const pjrt_client: *pjrt.Client = @ptrFromInt(getScalarAttributeAs(u64, call_frame, "pjrt_client").?); + + const device_ordinal: usize = @intCast(try call_frame.ctx.getDeviceOrdinal(call_frame.api)); + + const buffer = buffers[0]; + const shape = shapeFromFfiBuffer(buffer); + + // NOTE(Corentin): This is a hack. We take the first non device memory, hoping that it's host visible, + // and copy the buffer there to read it on the CPU and print it. + const device = pjrt_client.devices(pjrt_api)[device_ordinal]; + const addressable_memories = device.addressableMemories(pjrt_api); + const first_non_device_memory = for (addressable_memories) |memory| { + if (!std.mem.eql(u8, memory.kind_(pjrt_api), "device")) break memory; + } else return error.MemoryNotFound; + + var pjrt_buffer = try pjrt_client.createViewOfDeviceBuffer(pjrt_api, .{ + .device_buffer_ptr = buffer.data, + .dims = shape.dims(), + .element_type = pjrtx.bufferTypeFromDtype(shape.dtype()), + .device = device, + .layout = .{ + .tiled = .{ + .minor_to_major = zml.constants.minorToMajor(shape.rank()), + .tile_dims = &.{}, + .tile_dims_sizes = &.{}, + }, + }, + }); + defer pjrt_buffer.deinit(pjrt_api); + + pjrt_buffer = try pjrt_buffer.copyToMemory(pjrt_api, first_non_device_memory); + + const event = pjrt_buffer.readyEvent(pjrt_api); + defer event.deinit(pjrt_api); + try event.awaitRaw(pjrt_api); + + const host_visible_data: [*]u8 = @ptrCast(@alignCast(try pjrt_buffer.opaqueDeviceMemoryDataPointer(pjrt_api))); + + const slice: zml.Slice = .init(shape, host_visible_data[0..shape.byteSize()]); + const name = call_frame.attrs.getByName(.string, "name").?.slice(); + + std.debug.print("{s} [device={d}]: {d}\n", .{ name, device_ordinal, slice }); + + return null; +} + +test "platform defaultMemoryLayout is boring" { + const platform = zml.testing.env(); + + const shapes = [_][]const i64{ + &.{4096}, + &.{ 4096, 4096 }, + &.{ 4096, 4096, 4096 }, + }; + for (shapes) |dims| { + // Checks that the PJRT client always return the same thing than `platform.defaultMemoryLayout` + // This allows to bypass the PJRT calls and the string of pjrt_client.defaultMemoryLayout. + const default_layout = try platform.pjrt_client.defaultMemoryLayout(platform.pjrt_api, pjrtx.bufferTypeFromDtype(.f32), dims); + const mem_layout = default_layout.toMemoryLayout(); + + if (platform.target == .furiosa) { + try std.testing.expectEqualDeep(mem_layout, platform.defaultMemoryLayout(dims, .f32)); + continue; + } + + // Note: I'm not just calling platform.defaultMemoryLayout because I'm investigating + // wether TPU requires its special branch. + try std.testing.expectEqualDeep(mem_layout, pjrt.MemoryLayout{ + .tiled = .{ + .minor_to_major = constants.minorToMajor(@intCast(dims.len)), + .tile_dims = &.{}, + .tile_dims_sizes = &.{}, + }, + }); + } +} diff --git a/zml/posix.h b/zml/posix.h index 8b05b1aa..0d66200c 100644 --- a/zml/posix.h +++ b/zml/posix.h @@ -1 +1,2 @@ +#include #include diff --git a/zml/posix.zig b/zml/posix.zig deleted file mode 100644 index 9ebd48f8..00000000 --- a/zml/posix.zig +++ /dev/null @@ -1,16 +0,0 @@ -const std = @import("std"); -const c = @import("c"); - -pub fn madvise(ptr: [*]align(std.mem.page_size) u8, length: usize, advice: u32) std.posix.MadviseError!void { - switch (std.posix.errno(c.madvise(ptr, @intCast(length), @intCast(advice)))) { - .SUCCESS => return, - .ACCES => return error.AccessDenied, - .AGAIN => return error.SystemResources, - .BADF => unreachable, // The map exists, but the area maps something that isn't a file. - .INVAL => return error.InvalidSyscall, - .IO => return error.WouldExceedMaximumResidentSetSize, - .NOMEM => return error.OutOfMemory, - .NOSYS => return error.MadviseUnavailable, - else => |err| return std.posix.unexpectedErrno(err), - } -} diff --git a/zml/profiling/profiler.zig b/zml/profiling/profiler.zig new file mode 100644 index 00000000..d46a0c4c --- /dev/null +++ b/zml/profiling/profiler.zig @@ -0,0 +1,287 @@ +const std = @import("std"); +const log = std.log; + +const c = @import("c"); +const pjrt = @import("pjrt"); +const upb = @import("upb"); +const xspace_to_perfetto = @import("tools/xspace_to_perfetto"); +const zffi = @import("ffi"); + +pub fn profiler(api: *const pjrt.Api, allocator: std.mem.Allocator, io: std.Io, options: ProfilerOptions) !Profiler { + var arena = std.heap.ArenaAllocator.init(allocator); + defer arena.deinit(); + + var upb_alloc: upb.Allocator = .init(arena.allocator()); + const upb_arena = c.upb_Arena_Init(null, 0, upb_alloc.inner()); + defer c.upb_Arena_Free(upb_arena); + + const profile_options = try upb.new(c.tensorflow_ProfileOptions, upb_arena); + try options.writeProto(upb_arena, profile_options); + + const serialized_options = try upb.serialize(profile_options, upb_arena); + const inner = try api.profiler(serialized_options); + + return try .init(allocator, io, inner, options); +} + +pub const ProfilerOptions = struct { + pub const default_version: u32 = 1; + + pub const DeviceType = enum(i32) { + unspecified = c.tensorflow_ProfileOptions_UNSPECIFIED, + cpu = c.tensorflow_ProfileOptions_CPU, + gpu = c.tensorflow_ProfileOptions_GPU, + tpu = c.tensorflow_ProfileOptions_TPU, + pluggable_device = c.tensorflow_ProfileOptions_PLUGGABLE_DEVICE, + }; + + pub const TraceOptions = struct { + host_traceme_filter_mask: ?u64 = null, + + fn isEmpty(self: TraceOptions) bool { + return self.host_traceme_filter_mask == null; + } + + fn writeProto(self: TraceOptions, proto: *c.tensorflow_ProfileOptions_TraceOptions) void { + if (self.host_traceme_filter_mask) |value| { + c.tensorflow_ProfileOptions_TraceOptions_set_host_traceme_filter_mask(proto, value); + } + } + }; + + pub const AdvancedConfigValue = union(enum) { + string: []const u8, + boolean: bool, + int64: i64, + + fn writeProto(self: AdvancedConfigValue, proto: *c.tensorflow_ProfileOptions_AdvancedConfigValue) void { + switch (self) { + .string => |value| c.tensorflow_ProfileOptions_AdvancedConfigValue_set_string_value(proto, upb.stringView(value)), + .boolean => |value| c.tensorflow_ProfileOptions_AdvancedConfigValue_set_bool_value(proto, value), + .int64 => |value| c.tensorflow_ProfileOptions_AdvancedConfigValue_set_int64_value(proto, value), + } + } + }; + + pub const AdvancedConfiguration = struct { + key: []const u8, + value: AdvancedConfigValue, + }; + + pub const defaults: ProfilerOptions = .{}; + + version: u32 = default_version, + repository_path: []const u8 = "/tmp/xprof", + session_id: []const u8 = "profiling", + device_type: ?DeviceType = .unspecified, + include_dataset_ops: ?bool = true, + host_tracer_level: ?u32 = 3, + device_tracer_level: ?u32 = 3, + python_tracer_level: ?u32 = 1, + enable_hlo_proto: ?bool = true, + start_timestamp_ns: ?u64 = null, + duration_ms: ?u64 = null, + trace_options: ?TraceOptions = null, + advanced_configuration: []const AdvancedConfiguration = &.{}, + raise_error_on_start_failure: ?bool = null, + override_hostname: ?[]const u8 = null, + + pub fn writeProto(self: ProfilerOptions, upb_arena: *c.upb_Arena, proto: *c.tensorflow_ProfileOptions) !void { + c.tensorflow_ProfileOptions_set_version(proto, self.version); + c.tensorflow_ProfileOptions_set_session_id(proto, upb.stringView(self.session_id)); + c.tensorflow_ProfileOptions_set_repository_path(proto, upb.stringView(self.repository_path)); + + if (self.device_type) |value| { + c.tensorflow_ProfileOptions_set_device_type(proto, @backingInt(value)); + } + if (self.include_dataset_ops) |value| { + c.tensorflow_ProfileOptions_set_include_dataset_ops(proto, value); + } + if (self.host_tracer_level) |value| { + c.tensorflow_ProfileOptions_set_host_tracer_level(proto, value); + } + if (self.device_tracer_level) |value| { + c.tensorflow_ProfileOptions_set_device_tracer_level(proto, value); + } + if (self.python_tracer_level) |value| { + c.tensorflow_ProfileOptions_set_python_tracer_level(proto, value); + } + if (self.enable_hlo_proto) |value| { + c.tensorflow_ProfileOptions_set_enable_hlo_proto(proto, value); + } + if (self.start_timestamp_ns) |value| { + c.tensorflow_ProfileOptions_set_start_timestamp_ns(proto, value); + } + if (self.duration_ms) |value| { + c.tensorflow_ProfileOptions_set_duration_ms(proto, value); + } + if (self.trace_options) |trace_options| { + if (!trace_options.isEmpty()) { + const trace_proto = try upb.new(c.tensorflow_ProfileOptions_TraceOptions, upb_arena); + trace_options.writeProto(trace_proto); + c.tensorflow_ProfileOptions_set_trace_options(proto, trace_proto); + } + } + for (self.advanced_configuration) |entry| { + const value_proto = try upb.new(c.tensorflow_ProfileOptions_AdvancedConfigValue, upb_arena); + entry.value.writeProto(value_proto); + if (!c.tensorflow_ProfileOptions_advanced_configuration_set(proto, upb.stringView(entry.key), value_proto, upb_arena)) { + return error.OutOfMemory; + } + } + if (self.raise_error_on_start_failure) |value| { + c.tensorflow_ProfileOptions_set_raise_error_on_start_failure(proto, value); + } + if (self.override_hostname) |value| { + c.tensorflow_ProfileOptions_set_override_hostname(proto, upb.stringView(value)); + } + } +}; + +pub const Profiler = struct { + arena: std.heap.ArenaAllocator, + allocator: std.mem.Allocator, + io: std.Io, + inner: ?pjrt.Profiler, + start_timestamp_ns: ?u64, + stop_timestamp_ns: ?u64, + raise_error_on_start_failure: bool, + local_host_tracer_level: ?u32, + local_host_traceme_filter_mask: ?u64, + session_dir: []const u8, + profile: Profile, + + pub const Profile = struct { + protobuf_path: []const u8, + perfetto_path: []const u8, + }; + + pub fn init(allocator: std.mem.Allocator, io: std.Io, inner: ?pjrt.Profiler, options: ProfilerOptions) !Profiler { + var arena = std.heap.ArenaAllocator.init(allocator); + errdefer arena.deinit(); + + const session_dir = try std.Io.Dir.path.join(allocator, &.{ options.repository_path, "plugins", "profile", options.session_id }); + errdefer allocator.free(session_dir); + + try std.Io.Dir.cwd().createDirPath(io, session_dir); + + const protobuf_path = try std.Io.Dir.path.join(allocator, &.{ session_dir, "profiling.xplane.pb" }); + errdefer allocator.free(protobuf_path); + + const perfetto_path = try std.Io.Dir.path.join(allocator, &.{ session_dir, "profiling.trace.json" }); + errdefer allocator.free(perfetto_path); + + return .{ + .arena = arena, + .allocator = allocator, + .io = io, + .inner = inner, + .start_timestamp_ns = null, + .stop_timestamp_ns = null, + .raise_error_on_start_failure = options.raise_error_on_start_failure orelse false, + .local_host_tracer_level = options.host_tracer_level, + .local_host_traceme_filter_mask = if (options.trace_options) |trace_options| trace_options.host_traceme_filter_mask else null, + .session_dir = session_dir, + .profile = .{ + .protobuf_path = protobuf_path, + .perfetto_path = perfetto_path, + }, + }; + } + + pub fn deinit(self: *Profiler) void { + self.allocator.free(self.profile.protobuf_path); + self.allocator.free(self.profile.perfetto_path); + self.allocator.free(self.session_dir); + self.arena.deinit(); + } + + pub fn start(self: *Profiler) !void { + self.start_timestamp_ns = @intCast(std.Io.Timestamp.now(self.io, .real).nanoseconds); + self.stop_timestamp_ns = null; + + if (shouldSkipPjrtProfiler()) { + log.warn("Detected SKIP_PJRT_PROFILER=true; disabling nested backend profiling and keeping local host tracing only.", .{}); + self.inner = null; + } else if (self.inner) |*inner| { + inner.start() catch |err| { + if (self.raise_error_on_start_failure) { + return err; + } + log.warn("Profiler backend failed to start; continuing with local host tracing only: {}", .{err}); + self.inner = null; + }; + } + if (self.local_host_tracer_level) |level| { + c.zml_traceme_session_start( + @intCast(level), + self.local_host_traceme_filter_mask orelse 0, + self.local_host_traceme_filter_mask != null, + ); + } + } + + pub fn stop(self: *Profiler) !?Profile { + self.stop_timestamp_ns = @intCast(std.Io.Timestamp.now(self.io, .real).nanoseconds); + + const protobuf = if (self.inner) |saved_inner| blk: { + var inner = saved_inner; + self.inner = null; + try inner.stop(); + break :blk try inner.collectData(self.arena.allocator()); + } else &.{}; + const merged_protobuf = c.zml_traceme_session_merge(zffi.ZigSlice.from(protobuf)); + defer if (merged_protobuf.len != 0) { + c.zml_traceme_str_free(merged_protobuf); + }; + const final_protobuf = if (merged_protobuf.len != 0) zffi.ZigSlice.to(u8, merged_protobuf) else protobuf; + if (final_protobuf.len == 0) { + return null; + } + + const normalized_protobuf = c.zml_traceme_session_postprocess( + zffi.ZigSlice.from(final_protobuf), + self.start_timestamp_ns orelse self.stop_timestamp_ns.?, + self.stop_timestamp_ns.?, + ); + defer if (normalized_protobuf.len != 0) { + c.zml_traceme_str_free(normalized_protobuf); + }; + if (normalized_protobuf.len == 0) { + log.err("Failed to normalize profile protobuf", .{}); + return error.ProfileTraceNormalizationFailed; + } + + const final_normalized_protobuf = zffi.ZigSlice.to(u8, normalized_protobuf); + + try self.writeFile(self.profile.protobuf_path, final_normalized_protobuf); + + xspace_to_perfetto.dumpXSpaceProtoToTraceJsonFile( + self.allocator, + self.io, + final_normalized_protobuf, + self.profile.perfetto_path, + ) catch |err| { + log.err("Failed to convert profile protobuf to Perfetto trace: {}", .{err}); + return error.ProfileTraceConversionFailed; + }; + + log.info("Profile dumped: {s} and {s}", .{ + self.profile.protobuf_path, + self.profile.perfetto_path, + }); + return self.profile; + } + + fn writeFile(self: *const Profiler, path: []const u8, contents: []const u8) !void { + const file = try std.Io.Dir.createFile(.cwd(), self.io, path, .{}); + defer file.close(self.io); + + try file.writePositionalAll(self.io, contents, 0); + } +}; + +fn shouldSkipPjrtProfiler() bool { + const value = std.c.getenv("SKIP_PJRT_PROFILER") orelse return false; + return std.mem.eql(u8, std.mem.span(value), "true"); +} diff --git a/zml/profiling/traceme.h b/zml/profiling/traceme.h new file mode 100644 index 00000000..a1df3d04 --- /dev/null +++ b/zml/profiling/traceme.h @@ -0,0 +1,27 @@ +#pragma once + +#include +#include + +#include "ffi/zig_slice.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct zml_traceme zml_traceme; + +zml_traceme *zml_traceme_start(zig_slice name); +void zml_traceme_stop(zml_traceme *traceme); +bool zml_traceme_enabled(void); +void zml_traceme_session_start(int level, uint64_t filter_mask, + bool enable_filter); +zig_slice zml_traceme_session_merge(zig_slice xspace); +zig_slice zml_traceme_session_postprocess(zig_slice xspace, + uint64_t start_time_ns, + uint64_t stop_time_ns); +void zml_traceme_str_free(zig_slice text); + +#ifdef __cplusplus +} +#endif diff --git a/zml/profiling/traceme_lib.cc b/zml/profiling/traceme_lib.cc new file mode 100644 index 00000000..ca37dee7 --- /dev/null +++ b/zml/profiling/traceme_lib.cc @@ -0,0 +1,252 @@ +#include "zml/profiling/traceme.h" + +#include +#include +#include + +#include "xla/tsl/profiler/convert/post_process_single_host_xplane.h" +#include "xla/tsl/profiler/backends/cpu/host_tracer_utils.h" +#include "xla/tsl/profiler/backends/cpu/traceme_recorder.h" +#include "xla/tsl/profiler/utils/time_utils.h" +#include "xla/tsl/profiler/utils/xplane_utils.h" +#include "tsl/profiler/lib/traceme.h" +#include "tsl/profiler/protobuf/xplane.pb.h" + +#if defined(__APPLE__) +#include +#include +#endif + +#if defined(ZML_RUNTIME_CUDA) +#include "nvtx3/nvToolsExt.h" +#endif + +#if defined(ZML_RUNTIME_ROCM) +#include "rocprofiler-sdk-roctx/roctx.h" +#endif + +namespace { + +struct LocalHostTraceSession { + bool active = false; + uint64_t start_timestamp_ns = 0; +}; + +LocalHostTraceSession g_local_host_trace_session; + +tensorflow::profiler::XPlane* FindOrAddHostPlane( + tensorflow::profiler::XSpace* space) { + for (auto& plane : *space->mutable_planes()) { + if (plane.name() == "/host:CPU") { + return &plane; + } + } + tensorflow::profiler::XPlane* plane = space->add_planes(); + plane->set_name("/host:CPU"); + return plane; +} + +zig_slice CopyToOwnedSlice(const std::string& data) { + if (data.empty()) { + return {}; + } + void* ptr = std::malloc(data.size()); + if (ptr == nullptr) { + return {}; + } + std::memcpy(ptr, data.data(), data.size()); + return {ptr, data.size()}; +} + +#if defined(ZML_RUNTIME_CUDA) +struct CudaNvtxRangeBackend { + explicit CudaNvtxRangeBackend(const std::string& name) + : domain(TslNvtxDomain()), active(domain != nullptr) { + if (!active) { + return; + } + + nvtxEventAttributes_t attrs{}; + attrs.version = NVTX_VERSION; + attrs.size = NVTX_EVENT_ATTRIB_STRUCT_SIZE; + attrs.messageType = NVTX_MESSAGE_TYPE_ASCII; + attrs.message.ascii = name.c_str(); + nvtxDomainRangePushEx(domain, &attrs); + } + + ~CudaNvtxRangeBackend() { + if (active) { + nvtxDomainRangePop(domain); + } + } + + static bool IsEnabled() { return TslNvtxDomain() != nullptr; } + + private: + static nvtxDomainHandle_t TslNvtxDomain() { + static nvtxDomainHandle_t domain = nvtxDomainCreateA("TSL"); + return domain; + } + + nvtxDomainHandle_t domain = nullptr; + bool active = false; +}; +#else +struct CudaNvtxRangeBackend { + explicit CudaNvtxRangeBackend(const std::string&) {} + static bool IsEnabled() { return false; } +}; +#endif + +#if defined(ZML_RUNTIME_ROCM) +struct RocmRoctxRangeBackend { + explicit RocmRoctxRangeBackend(const std::string& name) + : active(RangePush() != nullptr && RangePop() != nullptr) { + if (!active) { + return; + } + + const int level = RangePush()(name.c_str()); + active = level >= 0; + } + + ~RocmRoctxRangeBackend() { + if (active) { + RangePop()(); + } + } + + static bool IsEnabled() { + return RangePush() != nullptr && RangePop() != nullptr; + } + + private: + using RangePushFn = decltype(&roctxRangePushA); + using RangePopFn = decltype(&roctxRangePop); + + static RangePushFn RangePush() { + return reinterpret_cast(dlsym(RTLD_DEFAULT, "roctxRangePushA")); + } + + static RangePopFn RangePop() { + return reinterpret_cast(dlsym(RTLD_DEFAULT, "roctxRangePop")); + } + + bool active = false; +}; +#else +struct RocmRoctxRangeBackend { + explicit RocmRoctxRangeBackend(const std::string&) {} + static bool IsEnabled() { return false; } +}; +#endif + +#if defined(__APPLE__) +struct MacOsSignpostRangeBackend { + explicit MacOsSignpostRangeBackend(const std::string& name) + : log(Logger()), signpost_id(os_signpost_id_generate(log)) { + os_signpost_interval_begin(log, signpost_id, "zml.scope", "%{public}s", + name.c_str()); + } + + ~MacOsSignpostRangeBackend() { + os_signpost_interval_end(log, signpost_id, "zml.scope", "%{public}s", + ""); + } + + static bool IsEnabled() { return true; } + + private: + static os_log_t Logger() { + static os_log_t log = + os_log_create("ai.zml", OS_LOG_CATEGORY_POINTS_OF_INTEREST); + return log; + } + + os_log_t log; + os_signpost_id_t signpost_id; +}; +#else +struct MacOsSignpostRangeBackend { + explicit MacOsSignpostRangeBackend(const std::string&) {} + static bool IsEnabled() { return false; } +}; +#endif + +} // namespace + +struct zml_traceme { + explicit zml_traceme(std::string encoded_name) + : name(std::move(encoded_name)), + traceme([&] { return name; }), + cuda_backend(name), + rocm_backend(name), + macos_backend(name) {} + + std::string name; + tsl::profiler::TraceMe traceme; + CudaNvtxRangeBackend cuda_backend; + RocmRoctxRangeBackend rocm_backend; + MacOsSignpostRangeBackend macos_backend; +}; + +extern "C" zml_traceme* zml_traceme_start(zig_slice name) { + return new zml_traceme( + std::string(static_cast(name.ptr), name.len)); +} + +extern "C" void zml_traceme_stop(zml_traceme* traceme) { delete traceme; } + +extern "C" bool zml_traceme_enabled(void) { + return tsl::profiler::TraceMe::Active() || + CudaNvtxRangeBackend::IsEnabled() || + RocmRoctxRangeBackend::IsEnabled() || + MacOsSignpostRangeBackend::IsEnabled(); +} + +extern "C" void zml_traceme_session_start(int level, uint64_t filter_mask, + bool enable_filter) { + g_local_host_trace_session.start_timestamp_ns = + tsl::profiler::GetCurrentTimeNanos(); + g_local_host_trace_session.active = + enable_filter ? tsl::profiler::TraceMeRecorder::Start(level, filter_mask) + : tsl::profiler::TraceMeRecorder::Start(level); +} + +extern "C" zig_slice zml_traceme_session_merge(zig_slice xspace) { + tensorflow::profiler::XSpace space; + if (xspace.len != 0 && + !space.ParseFromArray(xspace.ptr, static_cast(xspace.len))) { + return CopyToOwnedSlice( + std::string(static_cast(xspace.ptr), xspace.len)); + } + + if (g_local_host_trace_session.active) { + auto events = tsl::profiler::TraceMeRecorder::Stop(); + g_local_host_trace_session.active = false; + if (!events.empty()) { + tsl::profiler::ConvertCompleteEventsToXPlane( + g_local_host_trace_session.start_timestamp_ns, std::move(events), + FindOrAddHostPlane(&space)); + } + } + + return CopyToOwnedSlice(space.SerializeAsString()); +} + +extern "C" zig_slice zml_traceme_session_postprocess(zig_slice xspace, + uint64_t start_time_ns, + uint64_t stop_time_ns) { + tensorflow::profiler::XSpace space; + if (xspace.len != 0 && + !space.ParseFromArray(xspace.ptr, static_cast(xspace.len))) { + return {}; + } + + tsl::profiler::SetXSpacePidIfNotSet(space, getpid()); + tsl::profiler::PostProcessSingleHostXSpace(&space, start_time_ns, + stop_time_ns); + return CopyToOwnedSlice(space.SerializeAsString()); +} + +extern "C" void zml_traceme_str_free(zig_slice text) { std::free(text.ptr); } diff --git a/zml/profiling/tracer.zig b/zml/profiling/tracer.zig new file mode 100644 index 00000000..5aed01df --- /dev/null +++ b/zml/profiling/tracer.zig @@ -0,0 +1,235 @@ +const std = @import("std"); +const builtin = @import("builtin"); + +const c = @import("c"); +const platforms = @import("platforms"); +const stdx = @import("stdx"); +const zffi = @import("ffi"); + +pub const Span = struct { + inner: ?*c.zml_traceme = null, + + /// Name is copied underneath + pub fn start(name: []const u8) Span { + return .{ + .inner = c.zml_traceme_start(zffi.ZigSlice.from(name)), + }; + } + + pub fn end(self: *Span) void { + if (self.inner) |inner| { + c.zml_traceme_stop(inner); + self.inner = null; + } + } +}; + +// TraceMe is always the base host tracing path. The C tracing bridge adds NVTX +// ranges on CUDA/Linux builds, roctx ranges on ROCm/Linux builds, and +// os_signpost intervals on macOS. +pub const supportsDeviceAnnotations = switch (builtin.os.tag) { + .macos => true, + .linux => platforms.target == .cuda or platforms.target == .rocm, + else => false, +}; + +pub fn enabled() bool { + return c.zml_traceme_enabled(); +} + +/// Creates a Span with encoded metadata. The metadata encoded size must be known at compile time. +/// If it's not your case, use `formatSpanName` to generate the span name before and use void for the metadata. +pub fn span(comptime name: []const u8, metadata: anytype) Span { + switch (@typeInfo(@TypeOf(metadata))) { + .void => { + return .start(name); + }, + .@"struct" => |info| { + if (info.field_names.len == 0) { + return .start(name); + } + + var buffer: [computeEncodedNameLen(name, @TypeOf(metadata))]u8 = undefined; + var encoded: std.ArrayList(u8) = .initBuffer(&buffer); + + appendEncodedName(&encoded, .failing, name, metadata) catch unreachable; + return .start(encoded.items); + }, + else => @compileError("Unsupported metadata."), + } +} + +/// Returns an allocator-owned encoded span name for metadata that cannot be +/// bounded by `span` at comptime. The caller owns the returned slice. +pub fn formatSpanName(allocator: std.mem.Allocator, name: []const u8, metadata: anytype) ![]const u8 { + switch (@typeInfo(@TypeOf(metadata))) { + .@"struct" => { + var encoded: std.ArrayList(u8) = .empty; + errdefer encoded.deinit(allocator); + + try appendEncodedName(&encoded, allocator, name, metadata); + return try encoded.toOwnedSlice(allocator); + }, + else => @compileError("trace metadata must be a struct literal like .{ .step_num = 42 }"), + } +} + +fn appendEncodedName(buffer: *std.ArrayList(u8), allocator: std.mem.Allocator, name: []const u8, metadata: anytype) !void { + try buffer.appendSlice(allocator, name); + + var field_count: usize = 0; + try buffer.append(allocator, '#'); + inline for (comptime std.meta.fieldNames(@TypeOf(metadata))) |field_name| { + if (shouldEncodeField(field_name, @field(metadata, field_name))) { + if (field_count != 0) { + try buffer.append(allocator, ','); + } + field_count += 1; + + const key = comptime metadataKey(field_name); + validateMetadataToken(key); + try buffer.appendSlice(allocator, key); + try buffer.append(allocator, '='); + try appendMetadataValue(buffer, allocator, @field(metadata, field_name), field_name); + } + } + + // We may not have written any fields because of optional values. + if (field_count == 0) { + buffer.items.len = name.len; + } else { + try buffer.append(allocator, '#'); + } +} + +fn validateMetadataToken(token: []const u8) void { + if (std.mem.indexOfScalar(u8, token, '#') != null) { + @panic("trace metadata cannot contain '#'"); + } +} + +fn metadataKey(comptime field_name: []const u8) []const u8 { + return if (comptime std.mem.eql(u8, field_name, "root")) "_r" else field_name; +} + +fn computeEncodedNameLen(comptime name: []const u8, comptime Metadata: type) usize { + switch (@typeInfo(Metadata)) { + .@"struct" => |info| { + if (info.field_names.len == 0) return name.len; + + // name#[attrs]# + var max_len = name.len + 2; + inline for ( + info.field_names, + info.field_types, + info.field_attrs, + ) |field_name, FieldType, field_attr| { + const key = comptime metadataKey(field_name); + validateMetadataToken(key); + + const default: ?FieldType = field_attr.defaultValue(FieldType); + // [key]=[value], + max_len += key.len + 2; + max_len += switch (@typeInfo(FieldType)) { + .bool => 1, + .int => @max( + std.fmt.comptimePrint("{}", .{std.math.minInt(FieldType)}).len, + std.fmt.comptimePrint("{}", .{std.math.maxInt(FieldType)}).len, + ), + .@"enum" => |ty| b: { + var max_enum_len: usize = 0; + for (ty.field_names) |enum_field_name| { + max_enum_len = @max(max_enum_len, enum_field_name.len); + } + break :b max_enum_len; + }, + .comptime_int => std.fmt.comptimePrint("{}", .{default.?}).len, + .comptime_float => std.fmt.comptimePrint("{d}", .{default.?}).len, + .enum_literal => @tagName(default.?).len, + else => @compileError("trace metadata field '" ++ field_name ++ "' is not statically bounded; use formatspanName"), + }; + } + return max_len; + }, + else => @compileError("trace metadata must be a struct literal like .{ .step_num = 42 }"), + } +} + +fn shouldEncodeField(comptime field_name: []const u8, value: anytype) bool { + const FieldType = @TypeOf(value); + return switch (@typeInfo(FieldType)) { + .optional => value != null, + .bool => if (comptime std.mem.eql(u8, field_name, "root")) value else true, + else => true, + }; +} + +fn appendMetadataValue(buffer: *std.ArrayList(u8), allocator: std.mem.Allocator, value: anytype, comptime field_name: []const u8) !void { + const T = @TypeOf(value); + switch (@typeInfo(T)) { + .optional => { + try appendMetadataValue(buffer, allocator, value.?, field_name); + }, + .bool => { + if (comptime std.mem.eql(u8, field_name, "root")) { + try buffer.append(allocator, '1'); + } else { + try buffer.appendSlice(allocator, if (value) "1" else "0"); + } + }, + .int, .comptime_int => try buffer.print(allocator, "{}", .{value}), + .float, .comptime_float => try buffer.print(allocator, "{d}", .{value}), + .enum_literal => try buffer.appendSlice(allocator, @tagName(value)), + .@"enum" => try buffer.appendSlice(allocator, @tagName(value)), + .pointer => |ptr| { + if (ptr.size == .slice and ptr.child == u8) { + validateMetadataToken(value); + try buffer.appendSlice(allocator, value); + return; + } + + if (ptr.size == .one) switch (@typeInfo(ptr.child)) { + .array => |arr| { + if (arr.child == u8) { + const slice = value[0..]; + validateMetadataToken(slice); + try buffer.appendSlice(allocator, slice); + return; + } + }, + else => {}, + }; + + @compileError("trace metadata pointers must be []const u8 or string literals"); + }, + .array => |arr| { + if (arr.child != u8) { + @compileError("trace metadata arrays must be [N]u8"); + } + validateMetadataToken(value[0..]); + try buffer.appendSlice(allocator, value[0..]); + }, + else => @compileError("unsupported trace metadata type for field '" ++ field_name ++ "'"), + } +} + +test "formatspanName encodes metadata" { + const formatted = try formatSpanName(std.testing.allocator, "test.span", .{ + .root = true, + .count = @as(u8, 42), + .label = "batch", + }); + defer std.testing.allocator.free(formatted); + + try std.testing.expectEqualStrings("test.span#_r=1,count=42,label=batch#", formatted); +} + +test "formatspanName omits empty metadata" { + const formatted = try formatSpanName(std.testing.allocator, "test.span", .{ + .root = false, + .optional = @as(?u8, null), + }); + defer std.testing.allocator.free(formatted); + + try std.testing.expectEqualStrings("test.span", formatted); +} diff --git a/zml/quantization.zig b/zml/quantization.zig index 2a850a4d..db20619a 100644 --- a/zml/quantization.zig +++ b/zml/quantization.zig @@ -1,125 +1,519 @@ +//! Quantization metadata and scheme classification. const std = @import("std"); + +const stdx = @import("stdx"); + const zml = @import("zml.zig"); -const Allocator = std.mem.Allocator; -const module = zml.module; +const DataType = @import("dtype.zig").DataType; +const platform_mod = @import("platform.zig"); +const Platform = platform_mod.Platform; +const Shape = @import("shape.zig").Shape; +const Tensor = @import("tensor.zig").Tensor; -pub fn Q4_0(comptime dtype: zml.DataType) type { - return struct { - const Self = @This(); +pub const nvfp4_block_size = 16; +pub const mx_block_size = 32; - const QuantType = zml.io.gguf.quants.QuantType.q4_0; +pub const Quantization = struct { + scheme: Scheme, // decided when the checkpoint is read + scales: Tensor, + global_scale: ?GlobalScale = null, + input_scale: ?GlobalScale = null, + /// HACK: `scales` is stored in the 128x4 blocked layout of Blackwell's block-scaled mma. + // Something cleaner will need to be done to differentiate different kinds of swizzling + swizzled_scales: bool = false, - quant_buffer: zml.Tensor, + /// A per-tensor scale and the polarity its producer wrote it in + pub const GlobalScale = struct { + value: Tensor, + operation: Operation, - pub fn compile( - allocator: Allocator, - ctx: *zml.Context, - input: zml.Tensor, - shape: zml.Shape, - ) !module.CompiledModule(Self.forward) { - std.debug.assert(input.dtype() == .u8); - std.debug.assert(input.rank() == 1); - return module.compile( - allocator, - ctx, - Self.forward, - Self{ .quant_buffer = input }, - .{shape}, - ) catch unreachable; - } + pub const Operation = enum { multiply, divide }; - /// Each block is composed of a f16 scale and 32 4-bit values. - const block_stride = 18; + pub fn asMultiplier(self: GlobalScale) Tensor { + const s = self.value.convert(.f32); + const flat = if (s.shape().count() == 1) s.asScalar() else s; // essentially normalization - pub fn forward(self: Self, shape: zml.Shape) zml.Tensor { - const input = self.quant_buffer; - const block_count: u63 = @intCast(@divExact(input.dim(0), block_stride)); - const scales = extractScales(block_count, input); - const weights = extractWeights(block_count, input); - - return scales.reshape(.{ block_count, 1 }) - .broadcastLeft(zml.Shape.init(.{ block_count, 32 }, .f32)) - .mul(weights) - .convert(dtype) - .reshape(.{block_count * 32}) - .reshape(shape); - } - - pub fn scaleIndices(block_count: u63) zml.Tensor { - // indices1 is the offsets of the scale bytes, repeated block_count times. - const indices1 = zml.Tensor.arange(.{ .start = 0, .end = 2 }, .i32).repeat1d(0, block_count); - - // indices2 is the offsets of the blocks, repeated for each scale byte, repeated block_count times. - const indices2 = zml.Tensor.arange(.{ .start = 0, .end = block_stride * block_count, .step = block_stride }, .i32) - .reshape(.{ block_count, 1 }).broadcastLeft(zml.Shape.init(.{ block_count, 2 }, .i32)).reshape(.{2 * block_count}); - - // indices is the sum of the two, which is the offsets to all the bytes we are interested in. - return indices1.add(indices2); - } - - pub fn weightIndices(block_count: u63) zml.Tensor { - // indices1 is the offsets of the data bytes, repeated block_count times. - const indices1 = zml.Tensor.arange(.{ .start = 2, .end = 18 }, .i32).repeat1d(0, block_count); - - // indices2 is the offsets of the blocks, repeated for each data byte, repeated block_count times. - const indices2 = zml.Tensor.arange(.{ .start = 0, .end = block_stride * block_count, .step = block_stride }, .i32) - .reshape(.{ block_count, 1 }).broadcastLeft(zml.Shape.init(.{ block_count, 16 }, .i32)).reshape(.{16 * block_count}); - - // indices is the sum of the two, which is the offsets to all the bytes we are interested in. - return indices1.add(indices2); - } - - pub fn extractScales(block_count: u63, input: zml.Tensor) zml.Tensor { - // The goal here is to get the first two bytes of every 18-bytes block in the input. For that, - // we generate a list of indices that we will use to gather from the input. - - // indices1 is the offsets of the scale bytes, repeated block_count times. - const indices1 = zml.Tensor.arange(.{ .start = 0, .end = 2 }, .i32).repeat1d(0, block_count); - - // indices2 is the offsets of the blocks, repeated for each scale byte, repeated block_count times. - const indices2 = zml.Tensor.arange(.{ .start = 0, .end = block_stride * block_count, .step = block_stride }, .i32) - .reshape(.{ block_count, 1 }).broadcastLeft(zml.Shape.init(.{ block_count, 2 }, .i32)).reshape(.{2 * block_count}); - - // indices is the sum of the two, which is the offsets to all the bytes we are interested in. - const indices = indices1.add(indices2); - - // We select the values we are interested in with the indices, group them by pair and bitcast them to f16, then convert them to f32. - const scales = input.gatherValues(0, indices, .{ .indices_are_sorted = true }).reshape(.{ block_count, 2 }).bitCast(.f16).convert(.f32); - - return scales; - } - - pub fn extractWeights(block_count: u63, input: zml.Tensor) zml.Tensor { - // The goal here is to get everything but the first two bytes of every 18-bytes block in the input. For that, - // we generate a list of indices that we will use to gather from the input. - - // indices1 is the offsets of the data bytes, repeated block_count times. - const indices1 = zml.Tensor.arange(.{ .start = 2, .end = 18 }, .i32).repeat1d(0, block_count); - - // indices2 is the offsets of the blocks, repeated for each data byte, repeated block_count times. - const indices2 = zml.Tensor.arange(.{ .start = 0, .end = block_stride * block_count, .step = block_stride }, .i32) - .reshape(.{ block_count, 1 }).broadcastLeft(zml.Shape.init(.{ block_count, 16 }, .i32)).reshape(.{16 * block_count}); - - // indices is the sum of the two, which is the offsets to all the bytes we are interested in. - const indices = indices1.add(indices2); - - // NOTE(Corendos): i4 is not supported by bitcast convert, so we need the following workaround. - - // We select the values we are interested in with the indices, these are our quantized_weights. - const quantized_weights = input.gatherValues(0, indices, .{ .indices_are_sorted = true }); - const lb_weights = quantized_weights - .logical(.And, zml.Tensor.constant(.{16 * block_count}, zml.Data.init(.u8, 0xf))) - .bitCast(.i8); - const hb_weights = quantized_weights - .shiftRightLogical(zml.Tensor.constant(.{16 * block_count}, zml.Data.init(.u8, 4))).bitCast(.i8); - const weights = zml.Tensor.concatenate( - &.{ lb_weights.reshape(.{ block_count, 16 }), hb_weights.reshape(.{ block_count, 16 }) }, - 1, - ) - .sub(zml.Tensor.constant(.{ block_count, 32 }, zml.Data.init(.i8, 8))) - .convert(.f32); - return weights; + return switch (self.operation) { + .multiply => flat, + .divide => Tensor.scalar(1.0, .f32).broad(flat.shape()).div(flat), + }; } }; + + pub const Scheme = enum { + /// f4e2m1 values (`u8`-packed or native), f8e4m3fn scale per 16 contracted values. + /// Emitted by llm-compressor and by NVIDIA's ModelOpt. + nvfp4, + /// f8e4m3fn values, one e8m0 (power-of-two) scale per 32 contracted values + mxfp8, + /// f4e2m1 values (`u8`-packed or native), e8m0 scale per 32 contracted values. + // GPT-OSS like + mxfp4, + /// f8e4m3fn values, one bf16 or f32 scale per output channel, constant along the contraction. + /// Emitted by llm-compressor, including for the layers an NVFP4 recipe leaves in FP8. + fp8_per_channel, + /// f8e4m3fn, f8e4m3fnuz, or f8e8m0 values, one bf16 or f32 scale per 128x128 tile. + /// The DeepSeek-style FP8 that model vendors publish themselves, under `weight_scale_inv`. + fp8_block128, + /// E4M3FN or E4M3FNUZ weights with E8M0, BF16 or F32 scales per 32x32 tile. + fp8_block32, + /// f8e4m3fn values, one scale for the whole tensor. Spelled `[1, 1]` rather than as a + /// scalar: XLA's composite rewriter requires the scale to have the operand's rank. + fp8_per_tensor, + + pub fn accepts(self: Scheme, weight: Shape, scale: Shape) bool { + if (weight.rank() != 2) return false; + + const n = weight.dim(0); + const k = if (isPackedFp4(self, weight.dtype())) 2 * weight.dim(1) else weight.dim(1); + + return switch (self) { + .nvfp4 => (weight.dtype() == .u8 or weight.dtype() == .f4e2m1) and + scale.dtype() == .f8e4m3fn and scale.rank() == 2 and + scale.dim(0) == n and scale.dim(1) * nvfp4_block_size == k, + .mxfp8 => weight.dtype() == .f8e4m3fn and isMxScale(scale) and + scale.dim(0) == n and scale.dim(1) * mx_block_size == k, + .mxfp4 => (weight.dtype() == .u8 or weight.dtype() == .i8 or weight.dtype() == .f4e2m1) and + isMxScale(scale) and + scale.dim(0) == n and scale.dim(1) * mx_block_size == k, + .fp8_per_tensor => weight.dtype() == .f8e4m3fn and scale.count() == 1, + .fp8_per_channel => weight.dtype() == .f8e4m3fn and + (scale.dtype() == .bf16 or scale.dtype() == .f32) and + scale.count() > 1 and scale.rank() == 2 and + scale.dim(0) == n and scale.dim(1) == 1, + .fp8_block32 => (weight.dtype() == .f8e4m3fn or weight.dtype() == .f8e4m3fnuz) and + (scale.dtype() == .f8e8m0 or ((scale.dtype() == .bf16 or scale.dtype() == .f32) and scale.count() > 1)) and + scale.rank() == 2 and scale.dim(0) * 32 == n and scale.dim(1) * 32 == k, + .fp8_block128 => (weight.dtype() == .f8e4m3fn or weight.dtype() == .f8e4m3fnuz or weight.dtype() == .f8e8m0) and + (scale.dtype() == .bf16 or scale.dtype() == .f32) and + scale.count() > 1 and scale.rank() == 2 and + scale.dim(0) == (std.math.divCeil(i64, n, 128) catch unreachable) and + scale.dim(1) == (std.math.divCeil(i64, k, 128) catch unreachable), + }; + } + + pub fn classify(weight: Shape, scale: Shape) ?Scheme { + for (std.enums.values(Scheme)) |scheme| { + if (scheme.accepts(weight, scale)) return scheme; + } + + return null; + } + + pub fn isMx(self: Scheme) bool { + return self == .mxfp8 or self == .mxfp4; + } + }; +}; + +pub const QuantizedInput = struct { + values: Tensor, + scales: Tensor, + global_scale: ?Tensor, + + pub fn reuseBuffer(self: QuantizedInput, other: QuantizedInput) QuantizedInput { + return .{ + .values = self.values.reuseBuffer(other.values), + .scales = self.scales.reuseBuffer(other.scales), + .global_scale = if (self.global_scale != null and other.global_scale != null) self.global_scale.?.reuseBuffer(other.global_scale.?) else self.global_scale, + }; + } +}; + +pub fn quantizeInput(quantization: Quantization, input: Tensor, axis: Shape.Tag, platform: *const Platform) ?QuantizedInput { + const global_scale: ?Tensor = if (quantization.input_scale) |scale| scale.asMultiplier() else null; + + return switch (quantization.scheme) { + .nvfp4 => if (supportsNvfp4InputQuantization(platform)) quantizeNvfp4(input.convert(.bf16), global_scale, axis) else null, + .fp8_block32, .fp8_block128 => blk: { + const dtype: DataType = switch (platform.target) { + .cuda => .f8e4m3fn, + // RNGD contracts matching FP8 pairs in 128-wide K blocks. + .furiosa => if (quantization.scheme == .fp8_block128) .f8e4m3fn else return null, + .rocm => if (platform_mod.rocm.computeCapability(platform)) |capability| switch (capability.architecture()) { + .cdna4, .rdna4 => .f8e4m3fn, + .cdna3 => .f8e4m3fnuz, + .cdna1, .cdna2, .rdna2, .rdna3, .rdna3_5 => return null, + } else return null, + else => return null, + }; + + break :blk switch (quantization.scheme) { + .fp8_block32 => quantizeBlockFp8(input, axis, 32, dtype, .f8e8m0), + .fp8_block128 => quantizeBlockFp8(input.convert(.bf16), axis, 128, dtype, .f32), + else => unreachable, + }; + }, + .mxfp8, .mxfp4, .fp8_per_channel, .fp8_per_tensor => null, + }; +} + +pub fn quantizeMxfp4(x: Tensor, axis: anytype) QuantizedInput { + stdx.debug.assert(@mod(x.dim(axis), mx_block_size) == 0, "MXFP4 activation width must be divisible by {}, got {f}", .{ mx_block_size, x.shape() }); + + const grouped = x.convert(.f32).splitAxis(axis, .{ .mx_ks = -1, .mx_block = mx_block_size }); + const raw_scale = grouped.abs().max(.mx_block).maximum(.scalar(6 * 0x1p-126, .f32)).scale(1.0 / 6.0); + const scales = ceilPowerOfTwo(raw_scale); + + return .{ + .values = grouped.div(scales.broad(grouped.shape())) + .clamp(.scalar(-6, .f32), .scalar(6, .f32)) + .convert(.f4e2m1) + .reshape(x.shape().withDtype(.f4e2m1)), + .scales = scales.reshape(x.shape().setDim(axis, @divExact(x.dim(axis), mx_block_size)).withDtype(.f32)).convert(.f8e8m0), + .global_scale = null, + }; +} + +pub fn quantizeMxfp8(x: Tensor, axis: anytype) QuantizedInput { + const platform = zml.Compiler.current().platform; + + const dtype: ?DataType = switch (platform.target) { + .cuda, .cpu, .tpu => .f8e4m3fn, + .rocm => if (platform_mod.rocm.computeCapability(platform)) |capability| switch (capability.architecture()) { + .cdna4, .rdna4 => .f8e4m3fn, + .cdna3 => .f8e4m3fnuz, + .cdna1, .cdna2, .rdna2, .rdna3, .rdna3_5 => null, + } else null, + else => null, + }; + + if (dtype == null) std.debug.panic("MXFP8 is not supported on {}", .{platform.target}); + return quantizeBlockFp8(x, axis, mx_block_size, dtype.?, .f8e8m0); +} + +fn ceilPowerOfTwo(raw_scale: Tensor) Tensor { + const bits = raw_scale.bitCast(.u32); + return bits.addConstant(0x7fffff).logical(.AND, .scalar(0xff800000, .u32)).bitCast(.f32); +} + +/// Quantize activation blocks, preserving the input's axes. +/// E8M0 scales round upward to a power of two to avoid overflowing FP8 values. +pub fn quantizeBlockFp8(x: Tensor, axis: anytype, block_size: i64, dtype: DataType, scale_dtype: DataType) QuantizedInput { + stdx.debug.assert(block_size > 0, "block FP8 size must be positive, got {}", .{block_size}); + stdx.debug.assert(@mod(x.dim(axis), block_size) == 0, "block FP8 activation width must be divisible by {}, got {f}", .{ block_size, x.shape() }); + // Match the routed-MoE quantizer's FNUZ range. + const fp8_max: f32 = switch (dtype) { + .f8e4m3fn => 448.0, + .f8e4m3fnuz => 224.0, + else => stdx.debug.panic("expected E4M3 FP8 dtype, got {s}", .{@tagName(dtype)}), + }; + const grouped = x.convert(.f32).splitAxis(axis, .{ .fp8_ks = -1, .fp8_block = block_size }); + const scales = switch (scale_dtype) { + // Put scale arithmetic before the reduction so it can fuse into the producer. + .f32 => grouped.abs() + .maximum(.scalar(1e-10, .f32)) + .scale(1.0 / fp8_max) + .max(.fp8_block), + .f8e8m0 => blk: { + const raw_scale = grouped.abs().maximum(.scalar(1e-4, .f32)).scale(1.0 / fp8_max).max(.fp8_block); + break :blk ceilPowerOfTwo(raw_scale); + }, + else => stdx.debug.panic("expected F32 or E8M0 scale dtype, got {s}", .{@tagName(scale_dtype)}), + }; + const values = grouped.div(scales.broad(grouped.shape())) + .clamp(.scalar(-fp8_max, .f32), .scalar(fp8_max, .f32)) + .convert(dtype) + .reshape(x.shape().withDtype(dtype)); + return .{ + .values = values, + .scales = scales.reshape(x.shape().setDim(axis, @divExact(x.dim(axis), block_size)).withDtype(.f32)).convert(scale_dtype), + .global_scale = null, + }; +} + +pub fn quantizeNvfp4(x: Tensor, input_global_scale: ?Tensor, axis: anytype) QuantizedInput { + const value_max = 6.0; + const scale_min_normal = 0x1p-6; + const scale_max = DataType.f8e4m3fn.maxValue().as(f32); + + stdx.debug.assert(x.shape().hasTag(axis) != null, "quantizeNvfp4 expects x to have {any} tag, got {f}", .{ axis, x.shape() }); + stdx.debug.assert(@rem(x.dim(axis), nvfp4_block_size) == 0, "quantizeNvfp4 expects {any} to be a multiple of {}, got {f}", .{ axis, nvfp4_block_size, x.shape() }); + + const dt = x.dtype(); + const scaled = if (input_global_scale) |igs| + x.div(igs.convert(dt).broad(x.shape())) + else + x; + const grouped = scaled.splitAxis(axis, .{ .sc = -1, .blk = nvfp4_block_size }); + + const amax = grouped.abs() + .scale(1.0 / value_max) + .clamp(.scalar(scale_min_normal, dt), .scalar(scale_max, dt)) + .max(.blk); + + const scales = amax.convert(.f8e4m3fn); + + const divisor = scales.convert(dt) + .maximum(.scalar(scale_min_normal, dt)) + .broad(grouped.shape()); + + const values = grouped.div(divisor) + .convert(.f4e2m1) + .reshape(x.shape().withDtype(.f4e2m1)); + + return .{ + .values = values, + .scales = scales.squeeze(.blk), + .global_scale = input_global_scale, + }; +} + +fn supportsNvfp4InputQuantization(platform: *const Platform) bool { + return if (platform_mod.cuda.computeCapability(platform)) |cc| cc.atLeast(.{ .major = 10, .minor = 0 }) else false; +} + +fn isPackedFp4(scheme: ?Quantization.Scheme, weight_dtype: DataType) bool { + return (scheme == .nvfp4 or scheme == .mxfp4) and (weight_dtype == .u8 or weight_dtype == .i8); +} + +fn isMxScale(scale: Shape) bool { + return scale.rank() == 2 and (scale.dtype() == .f8e8m0 or scale.dtype() == .u8); +} + +// Note: this test will evolve as we support more (INT4/8 and FP8 block-128 as a +// scaled-dot are still open; see the scaledDot doc comment). +test "Quantization.Scheme.classify" { + const expect = std.testing.expectEqual; + + // unsloth/Qwen3.6-27B-NVFP4: compressed-tensors, carrying both of its schemes under + // `weight_scale` -- NVFP4 on the MLPs of layers 0-55, FP8 per-channel everywhere else. + const nvfp4_packed: Shape = .init(.{ .dout = 17408, .d = 2560 }, .u8); + try expect(@as(?Quantization.Scheme, .nvfp4), Quantization.Scheme.classify(nvfp4_packed, .init(.{ .dout = 17408, .sc = 320 }, .f8e4m3fn))); + try expect(@as(?Quantization.Scheme, .fp8_per_channel), Quantization.Scheme.classify( + .init(.{ .dout = 17408, .d = 5120 }, .f8e4m3fn), + .init(.{ .dout = 17408, .sc = 1 }, .bf16), + )); + + // nvidia/Gemma-4-31B-IT-NVFP4: ModelOpt, packed values under the plain `weight` name. + try expect(@as(?Quantization.Scheme, .nvfp4), Quantization.Scheme.classify( + .init(.{ .dout = 21504, .d = 2688 }, .u8), + .init(.{ .dout = 21504, .sc = 336 }, .f8e4m3fn), + )); + + // Native (unpacked) f4e2m1, with K no longer doubled. + try expect(@as(?Quantization.Scheme, .nvfp4), Quantization.Scheme.classify( + .init(.{ .dout = 17408, .d = 5120 }, .f4e2m1), + .init(.{ .dout = 17408, .sc = 320 }, .f8e4m3fn), + )); + + // Qwen/Qwen3.6-27B-FP8: block 128, spelled `weight_scale_inv`. + try expect(@as(?Quantization.Scheme, .fp8_block128), Quantization.Scheme.classify( + .init(.{ .dout = 17408, .d = 5120 }, .f8e4m3fn), + .init(.{ .dout = 136, .sc = 40 }, .bf16), + )); + try expect(@as(?Quantization.Scheme, .fp8_block128), Quantization.Scheme.classify( + .init(.{ .dout = 5120, .d = 6144 }, .f8e4m3fn), + .init(.{ .dout = 40, .sc = 48 }, .bf16), + )); + + inline for (.{ .f8e4m3fn, .f8e4m3fnuz }) |dtype| { + try expect(@as(?Quantization.Scheme, .fp8_block32), Quantization.Scheme.classify( + .init(.{ .dout = 1280, .d = 5120 }, dtype), + .init(.{ .dout = 40, .sc = 160 }, .f8e8m0), + )); + } + + // GLM uses f32 scales and permits a partial final 128-row tile. + try expect(@as(?Quantization.Scheme, .fp8_block128), Quantization.Scheme.classify( + .init(.{ .dout = 2048, .d = 6144 }, .f8e4m3fn), + .init(.{ .dout = 16, .sc = 48 }, .f32), + )); + try expect(@as(?Quantization.Scheme, .fp8_block128), Quantization.Scheme.classify( + .init(.{ .dout = 576, .d = 6144 }, .f8e4m3fn), + .init(.{ .dout = 5, .sc = 48 }, .f32), + )); + try expect(@as(?Quantization.Scheme, .fp8_block128), Quantization.Scheme.classify( + .init(.{ .dout = 576, .d = 6144 }, .f8e8m0), + .init(.{ .dout = 5, .sc = 48 }, .f32), + )); + + // Mistral's per-tensor FP8: one scale for the whole tensor, rank 0 or [1]. + try expect(@as(?Quantization.Scheme, .fp8_per_tensor), Quantization.Scheme.classify(.init(.{ .dout = 4096, .d = 4096 }, .f8e4m3fn), .init(.{}, .f32))); + try expect(@as(?Quantization.Scheme, .fp8_per_tensor), Quantization.Scheme.classify(.init(.{ .dout = 4096, .d = 4096 }, .f8e4m3fn), .init(.{ .g = 1 }, .f32))); + + // RadixArk/Muse-Glimmer-NVFP4: ModelOpt's MIXED_PRECISION recipe, whose MXFP8 half + // lands on down_proj and lm_head. safetensors has no e8m0 dtype, so the scale arrives + // as u8 under `weight_scale_inv` -- the shape below is verbatim from the checkpoint. + try expect(@as(?Quantization.Scheme, .mxfp8), Quantization.Scheme.classify( + .init(.{ .dout = 6656, .d = 19968 }, .f8e4m3fn), + .init(.{ .dout = 6656, .sc = 624 }, .u8), + )); + try expect(@as(?Quantization.Scheme, .mxfp8), Quantization.Scheme.classify( + .init(.{ .dout = 202048, .d = 6656 }, .f8e4m3fn), + .init(.{ .dout = 202048, .sc = 208 }, .u8), + )); + // The same tensor from a store that types the scale natively. + try expect(@as(?Quantization.Scheme, .mxfp8), Quantization.Scheme.classify( + .init(.{ .dout = 6656, .d = 19968 }, .f8e4m3fn), + .init(.{ .dout = 6656, .sc = 624 }, .f8e8m0), + )); + // MXFP8 rejects any group but 32: 19968 / 16 = 1248, / 64 = 312. + try expect(@as(?Quantization.Scheme, null), Quantization.Scheme.classify( + .init(.{ .dout = 6656, .d = 19968 }, .f8e4m3fn), + .init(.{ .dout = 6656, .sc = 1248 }, .u8), + )); + try expect(@as(?Quantization.Scheme, null), Quantization.Scheme.classify( + .init(.{ .dout = 6656, .d = 19968 }, .f8e4m3fn), + .init(.{ .dout = 6656, .sc = 312 }, .u8), + )); + + // MXFP4: the same u8 packing as NVFP4 (two nibbles per byte, K halved on disk), with + // an e8m0 scale per 32 -- so K = 2 * 2560 = 5120 and 5120 / 32 = 160. + try expect(@as(?Quantization.Scheme, .mxfp4), Quantization.Scheme.classify(nvfp4_packed, .init(.{ .dout = 17408, .sc = 160 }, .u8))); + try expect(@as(?Quantization.Scheme, .mxfp4), Quantization.Scheme.classify(nvfp4_packed, .init(.{ .dout = 17408, .sc = 160 }, .f8e8m0))); + // DeepSeek V4 stores the same packed FP4 bits in signed bytes. + try expect(@as(?Quantization.Scheme, .mxfp4), Quantization.Scheme.classify( + .init(.{ .dout = 2048, .d = 2048 }, .i8), + .init(.{ .dout = 2048, .sc = 128 }, .f8e8m0), + )); + // Native (unpacked) f4e2m1, K no longer halved. + try expect(@as(?Quantization.Scheme, .mxfp4), Quantization.Scheme.classify( + .init(.{ .dout = 17408, .d = 5120 }, .f4e2m1), + .init(.{ .dout = 17408, .sc = 160 }, .f8e8m0), + )); + // An e8m0 scale never turns a non-fp4/fp8 weight into MX. + try expect(@as(?Quantization.Scheme, null), Quantization.Scheme.classify( + .init(.{ .dout = 17408, .d = 5120 }, .bf16), + .init(.{ .dout = 17408, .sc = 160 }, .f8e8m0), + )); + + // Rejected: everything no backend here can express. + const fp8: Shape = .init(.{ .dout = 10240, .d = 5120 }, .f8e4m3fn); + // Group 32 with an e4m3 scale: neither NVFP4 (wrong group) nor MX (wrong scale type). + try expect(@as(?Quantization.Scheme, null), Quantization.Scheme.classify(nvfp4_packed, .init(.{ .dout = 17408, .sc = 160 }, .f8e4m3fn))); + // Group 16 with an e8m0 scale: the mirror image, and equally unclaimed. + try expect(@as(?Quantization.Scheme, null), Quantization.Scheme.classify(nvfp4_packed, .init(.{ .dout = 17408, .sc = 320 }, .f8e8m0))); + // Per-channel in f32: what fusing per-tensor projections produces. The Metal + // per-channel kernels have f32 entries, so this is a real scheme, not a decline. + try expect(@as(?Quantization.Scheme, .fp8_per_channel), Quantization.Scheme.classify(fp8, .init(.{ .dout = 10240, .sc = 1 }, .f32))); + // A block grid that is neither per-channel nor 128x128. + try expect(@as(?Quantization.Scheme, null), Quantization.Scheme.classify(fp8, .init(.{ .dout = 160, .sc = 80 }, .bf16))); + // Stacked MoE weights are rank 3 and go through the MoE path, not this one. + try expect(@as(?Quantization.Scheme, null), Quantization.Scheme.classify( + .init(.{ .expert = 128, .dout = 1408, .d = 1408 }, .u8), + .init(.{ .expert = 128, .dout = 1408, .sc = 176 }, .f8e4m3fn), + )); + + // `fp8_per_tensor` never checks the scale dtype, so a [1, 1] u8 scale on a [1, 32] + // weight satisfies it as well as `mxfp8`. Declaration order is what decides, and + // `mxfp8` is declared first + try expect(@as(?Quantization.Scheme, .mxfp8), Quantization.Scheme.classify( + .init(.{ .dout = 1, .d = 32 }, .f8e4m3fn), + .init(.{ .dout = 1, .sc = 1 }, .u8), + )); + + // A one-tile block grid is indistinguishable from a per-tensor scale by shape alone; the + // `count() > 1` guard on the grids is what makes per-tensor win. + try expect(@as(?Quantization.Scheme, .fp8_per_tensor), Quantization.Scheme.classify( + .init(.{ .dout = 128, .d = 128 }, .f8e4m3fn), + .init(.{ .dout = 1, .sc = 1 }, .bf16), + )); +} + +test "block FP8 quantization preserves axes and reconstructs constant blocks in FN and FNUZ" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = zml.testing.env(); + inline for (.{ DataType.f8e4m3fn, DataType.f8e4m3fnuz }) |dtype| { + const Local = struct { + const Outputs = struct { values: Tensor, scales: Tensor }; + fn forward(x: Tensor) Outputs { + const input = quantizeBlockFp8(x, .k, 128, dtype, .f32); + return .{ .values = input.values.convert(.f32), .scales = input.scales }; + } + }; + const x: Tensor = .init(.{ .b = 2, .k = 256, .s = 3 }, .bf16); + var exe = try platform.compileFn(allocator, io, Local.forward, .{x}, .{}); + defer exe.deinit(); + try zml.testing.expectEqualShapes(x.shape().withDtype(.f32), exe.output_shapes[0]); + try zml.testing.expectEqualShapes(x.shape().setDim(.k, 2).withDtype(.f32), exe.output_shapes[1]); + var host: [2 * 256 * 3]zml.floats.BFloat16 = undefined; + for (&host, 0..) |*value, i| { + const magnitude: f32 = if ((i / 3) % 256 < 128) 1.0 else 2.0; + value.* = .fromF32(switch (i % 3) { + 0 => 0.0, + 1 => magnitude, + else => -magnitude, + }); + } + var buffer = try zml.Buffer.fromBytes(io, platform, x.shape(), std.mem.asBytes(&host)); + defer buffer.deinit(); + var output: zml.Bufferized(Local.Outputs) = undefined; + var runner = try exe.runner(allocator); + defer runner.deinit(allocator); + runner.run(io, .{buffer}, .{&output}, .{ .wait = true }); + defer zml.Buffer.deinitAll(Local.Outputs, &output); + var values = try output.values.toSliceAlloc(allocator, io); + defer values.free(allocator); + var scales = try output.scales.toSliceAlloc(allocator, io); + defer scales.free(allocator); + for (scales.constItems(f32)) |scale| try std.testing.expect(std.math.isFinite(scale) and scale > 0); + for (values.constItems(f32), 0..) |value, i| { + const scale_index = (i / (256 * 3)) * 6 + ((i / 3) % 256) / 128 * 3 + i % 3; + try std.testing.expectApproxEqAbs(host[i].toF32(), value * scales.constItems(f32)[scale_index], 1e-6); + } + } +} + +test "block FP8 E8M0 scales preserve power-of-two boundaries and nonminor axes" { + const allocator = std.testing.allocator; + const io = std.testing.io; + const platform = zml.testing.env(); + inline for (.{ DataType.f8e4m3fn, DataType.f8e4m3fnuz }) |dtype| { + const Local = struct { + const Outputs = struct { values: Tensor, scales: Tensor }; + fn forward(x: Tensor) Outputs { + const input = quantizeBlockFp8(x, .k, 32, dtype, .f8e8m0); + return .{ .values = input.values.convert(.f32), .scales = input.scales.convert(.f32) }; + } + }; + const x: Tensor = .init(.{ .b = 2, .k = 256, .s = 3 }, .f32); + var exe = try platform.compileFn(allocator, io, Local.forward, .{x}, .{}); + defer exe.deinit(); + try zml.testing.expectEqualShapes(x.shape().withDtype(.f32), exe.output_shapes[0]); + try zml.testing.expectEqualShapes(x.shape().setDim(.k, 8).withDtype(.f32), exe.output_shapes[1]); + const fp8_max: f32 = if (dtype == .f8e4m3fn) 448.0 else 224.0; + var host: [2 * 256 * 3]f32 = undefined; + var maxima: [2 * 8 * 3]f32 = @splat(0); + for (&host, 0..) |*value, i| { + const block = (i / 3) % 256 / 32; + const scale_index = (i / (256 * 3)) * 24 + block * 3 + i % 3; + const boundary = fp8_max * std.math.pow(f32, 2, @as(f32, @floatFromInt(block)) - 4); + const bits: u32 = @bitCast(boundary); + const magnitude: f32 = switch (i % 3) { + 0 => @bitCast(bits - 1), + 1 => boundary, + else => @bitCast(bits + 1), + }; + value.* = if (block == 0) 0 else if (block == 1) 1e-30 else magnitude * (if (i % 2 == 0) @as(f32, 1) else -1); + maxima[scale_index] = @max(maxima[scale_index], @abs(value.*)); + } + var expected_scales: [2 * 8 * 3]f32 = undefined; + for (&expected_scales, maxima) |*scale, maximum| { + const raw = @max(maximum, 1e-4) * (1.0 / fp8_max); + const bits: u32 = @bitCast(raw); + const exponent = (bits >> 23) + @intFromBool(bits & 0x7fffff != 0); + scale.* = @bitCast(exponent << 23); + } + var buffer = try zml.Buffer.fromBytes(io, platform, x.shape(), std.mem.asBytes(&host)); + defer buffer.deinit(); + var output: zml.Bufferized(Local.Outputs) = undefined; + var runner = try exe.runner(allocator); + defer runner.deinit(allocator); + runner.run(io, .{buffer}, .{&output}, .{ .wait = true }); + defer zml.Buffer.deinitAll(Local.Outputs, &output); + var values = try output.values.toSliceAlloc(allocator, io); + defer values.free(allocator); + var scales = try output.scales.toSliceAlloc(allocator, io); + defer scales.free(allocator); + try std.testing.expectEqualSlices(f32, &expected_scales, scales.constItems(f32)); + for (values.constItems(f32), 0..) |value, i| { + const scale_index = (i / (256 * 3)) * 24 + ((i / 3) % 256) / 32 * 3 + i % 3; + const normalized = std.math.clamp(host[i] / expected_scales[scale_index], -fp8_max, fp8_max); + const expected = if (dtype == .f8e4m3fn) zml.floats.Float8E4M3FN.fromF32(normalized).toF32() else zml.floats.Float8E4M3FNUZ.fromF32(normalized).toF32(); + try std.testing.expectEqual(expected, value); + } + } } diff --git a/zml/safetensors.zig b/zml/safetensors.zig new file mode 100644 index 00000000..3827fe27 --- /dev/null +++ b/zml/safetensors.zig @@ -0,0 +1,641 @@ +const std = @import("std"); + +const DataType = @import("dtype.zig").DataType; +const Shape = @import("shape.zig").Shape; + +const log = std.log.scoped(.@"zml/safetensors"); + +// TODO: consider promoting this to the top level struct +// TODO: Feels like we are missing a mapping file -> tensors that would be helfpful for loading weights +/// Maps tensor names to their shape and positions in different sub files. +pub const TensorRegistry = struct { + arena: std.heap.ArenaAllocator, + + tensors: std.StringArrayHashMapUnmanaged(Tensor), + + /// Metadata found in `__metadata__` field of the safetensors header. + /// This is rarely use and the information there is not standardized, so we let user code handle it. + /// Here we just store the plain bytes, reach out if you need this to be already parsed. + metadata: []const u8, + + pub fn init(allocator: std.mem.Allocator) TensorRegistry { + return .{ + .arena = std.heap.ArenaAllocator.init(allocator), + .tensors = .{}, + .metadata = "", + }; + } + + pub fn deinit(self: *TensorRegistry) void { + self.arena.deinit(); + } + + pub fn fromRepo(allocator: std.mem.Allocator, io: std.Io, repo: std.Io.Dir) !TensorRegistry { + const file_type, const file = try resolveModelEntrypoint(io, repo); + defer file.close(io); + + var registry: TensorRegistry = .init(allocator); + switch (file_type) { + .safetensors => try registry.parseSafetensors(io, allocator, file), + .index => try registry.parseIndexJson(io, allocator, repo, file), + .full_index => try registry.parseFullIndexJson(io, allocator, repo, file), + .unknown => return error.InvalidPath, + } + return registry; + } + + pub fn fromPath(allocator: std.mem.Allocator, io: std.Io, path: []const u8) !TensorRegistry { + const file_type: FileType = .resolve(path); + switch (file_type) { + .safetensors => { + // open directly + const file = try std.Io.Dir.openFile(.cwd(), io, path, .{}); + var registry: TensorRegistry = .init(allocator); + try registry.parseSafetensors(io, allocator, file); + return registry; + }, + .full_index, + .index, + => { + // resolve repo as parent folder then open + const repo_path = std.fs.path.dirname(path) orelse return ModelPathResolutionError.InvalidPath; + const repo = std.Io.Dir.openDir(.cwd(), io, repo_path, .{}) catch |e| { + return switch (e) { + error.FileNotFound => ModelPathResolutionError.FileNotFound, + else => ModelPathResolutionError.InvalidPath, + }; + }; + const file = try std.Io.Dir.openFile(.cwd(), io, path, .{}); + + var registry: TensorRegistry = .init(allocator); + try switch (file_type) { + .index => registry.parseIndexJson(io, allocator, repo, file), + .full_index => registry.parseFullIndexJson(io, allocator, repo, file), + .safetensors, .unknown => unreachable, + }; + return registry; + }, + .unknown => { + // the path is possibly a folder, let's try that + const repo = std.Io.Dir.openDir(.cwd(), io, path, .{}) catch |e| { + return switch (e) { + error.FileNotFound => ModelPathResolutionError.FileNotFound, + else => ModelPathResolutionError.InvalidPath, + }; + }; + return try fromRepo(allocator, io, repo); + }, + } + } + + /// Register a tensor, duplicate tensor name and uri. + /// If you create a lot of tensors sharing the same uri, + /// considere calling `registry.tensors` directly and avoid duplicated uris. + pub fn registerTensor(self: *TensorRegistry, tensor: Tensor) !void { + const allocator = self.arena.allocator(); + + const owned_name = try allocator.dupe(u8, tensor.name); + errdefer allocator.free(owned_name); + const owned_uri = try allocator.dupe(u8, tensor.file_uri); + errdefer allocator.free(owned_uri); + + const owned_tensor: Tensor = .{ + .name = owned_name, + .file_uri = owned_uri, + .shape = tensor.shape, + .offset = tensor.offset, + }; + try self.tensors.put(allocator, owned_name, owned_tensor); + } + + pub fn reader( + self: *TensorRegistry, + io: std.Io, + tensor_name: []const u8, + buffer: []u8, + ) !TensorReader { + const tensor = self.tensors.get(tensor_name) orelse { + log.err("Tensor {s} not found in registry", .{tensor_name}); + return error.TensorNotFound; + }; + + return try .init(io, tensor, buffer, .{}); + } + + pub fn totalBytes(self: *TensorRegistry) u64 { + var total: u64 = 0; + + var it = self.tensors.iterator(); + while (it.next()) |entry| { + total += entry.value_ptr.byteSize(); + } + + return total; + } + + /// Parse a given .safetensors file and create entries for each tensor + pub fn parseSafetensors( + self: *TensorRegistry, + io: std.Io, + scratch_allocator: std.mem.Allocator, + file: std.Io.File, + ) !void { + var tmp_state = std.heap.ArenaAllocator.init(scratch_allocator); + defer tmp_state.deinit(); + const tmp_arena = tmp_state.allocator(); + + const self_arena = self.arena.allocator(); + var owned_file_name = try self_arena.alloc(u8, std.fs.max_path_bytes); + const file_name_len = try file.realPath(io, owned_file_name); + // Try to release extra memory, but we don't care if it fails + _ = self_arena.resize(owned_file_name, file_name_len); + owned_file_name = owned_file_name[0..file_name_len]; + + const header_buf = try tmp_arena.alloc(u8, 64 * 1024); + var file_reader: std.Io.File.Reader = file.reader(io, header_buf); + + const json_header_length: u64 = try file_reader.interface.takeInt(u64, .little); + const BYTES_HEADER = 8; + const data_start_offset = BYTES_HEADER + json_header_length; + const json_data = try tmp_arena.alloc(u8, @intCast(json_header_length)); + + try file_reader.interface.readSliceAll(json_data); + const headers: std.json.Value = try std.json.parseFromSliceLeaky( + std.json.Value, + tmp_arena, + json_data, + .{}, + ); + + try self.tensors.ensureUnusedCapacity(self_arena, headers.object.count()); + for (headers.object.keys(), headers.object.values()) |key, value| { + if (std.mem.eql(u8, key, "__metadata__")) { + // value belong to tmp arena, duplicate it into self arena. + var owned_metadata: std.Io.Writer.Allocating = .init(self_arena); + try std.json.fmt(value, .{ .whitespace = .minified }).format(&owned_metadata.writer); + self.metadata = owned_metadata.written(); + continue; + } + + const owned_name = try self_arena.dupe(u8, key); + const tensor: Tensor = try .parse(value.object, owned_file_name, owned_name, data_start_offset); + self.tensors.putAssumeCapacity(owned_name, tensor); + } + } + + pub fn parseIndexJson( + self: *TensorRegistry, + io: std.Io, + gpa: std.mem.Allocator, + repo: std.Io.Dir, + index_json_file: std.Io.File, + ) !void { + var tmp_state: std.heap.ArenaAllocator = .init(gpa); + defer tmp_state.deinit(); + const tmp = tmp_state.allocator(); + + const file_reader_buf = try tmp.alloc(u8, 64 * 1024); + var file_reader: std.Io.File.Reader = index_json_file.reader(io, file_reader_buf); + var json_reader: std.json.Reader = .init(tmp, &file_reader.interface); + const index = try std.json.parseFromTokenSourceLeaky( + std.json.Value, + tmp, + &json_reader, + .{ .allocate = .alloc_if_needed }, + ); + + var safetensors_files: std.StringArrayHashMapUnmanaged(void) = .empty; + try safetensors_files.ensureUnusedCapacity(tmp, 64); + if (index.object.get("weight_map")) |wm| { + for (wm.object.values()) |file_name| { + try safetensors_files.put(tmp, file_name.string, {}); + } + } else { + log.warn("No weight_map attribute found in index", .{}); + } + + for (safetensors_files.keys()) |file_name| { + const file = try repo.openFile(io, file_name, .{}); + defer file.close(io); + try self.parseSafetensors(io, tmp, file); + } + } + + pub fn parseFullIndexJson( + self: *TensorRegistry, + io: std.Io, + gpa: std.mem.Allocator, + repo: std.Io.Dir, + index_json_file: std.Io.File, + ) !void { + var tmp_state: std.heap.ArenaAllocator = .init(gpa); + defer tmp_state.deinit(); + const tmp = tmp_state.allocator(); + + const self_arena = self.arena.allocator(); + + var repo_uri_buf = try tmp.alloc(u8, 1024); + const repo_name = repo_uri_buf[0..try repo.realPath(io, repo_uri_buf)]; + + // Note: this is relatively inefficient, we read the full file, copy most of it's content, + // then reduce to TensorRegistry format. + // We could update TensorRegistry as we stream the json. + // Zig std json API are a bit awkward around streaming though. + const file_reader_buf = try tmp.alloc(u8, 64 * 1024); + var file_reader = index_json_file.reader(io, file_reader_buf); + var json_reader: std.json.Reader = .init(tmp, &file_reader.interface); + const index = try std.json.parseFromTokenSourceLeaky( + std.json.Value, + tmp, + &json_reader, + .{ .allocate = .alloc_if_needed }, + ); + + var realpaths: std.StringArrayHashMapUnmanaged([]const u8) = .empty; + try realpaths.ensureUnusedCapacity(tmp, 64); + const wm = index.object.get("weight_map") orelse return error.InvalidJson; + + try self.tensors.ensureUnusedCapacity(self_arena, wm.object.count()); + for (wm.object.keys(), wm.object.values()) |key, value| { + const tensor_name = try self_arena.dupe(u8, key); + const object = value.object; + const filename = value.object.get("filename").?.string; + const filename_entry = try realpaths.getOrPut(tmp, filename); + const file_uri = if (filename_entry.found_existing) + filename_entry.value_ptr.* + else uri: { + const uri = try std.fs.path.join(self_arena, &.{ repo_name, filename }); + filename_entry.value_ptr.* = uri; + break :uri uri; + }; + + const tensor_info: Tensor = try .parse(object, file_uri, tensor_name, 0); + self.tensors.putAssumeCapacity(tensor_name, tensor_info); + } + + if (index.object.get("metadata")) |metadata| { + // value belong to tmp arena, duplicate it into self arena. + var owned_metadata: std.Io.Writer.Allocating = .init(self_arena); + try std.json.fmt(metadata, .{ .whitespace = .minified }).format(&owned_metadata.writer); + self.metadata = owned_metadata.written(); + } + } + + pub fn saveAsFullIndexJson( + self: *const TensorRegistry, + io: std.Io, + gpa: std.mem.Allocator, + repo: std.Io.Dir, + out: *std.Io.Writer, + ) !void { + var repo_uri_buf = try gpa.alloc(u8, std.fs.max_path_bytes); + defer gpa.free(repo_uri_buf); + const repo_name = repo_uri_buf[0..try repo.realPath(io, repo_uri_buf)]; + var total_size: u64 = 0; + for (self.tensors.values()) |value| total_size += value.byteSize(); + + try out.print( + \\{{ + \\"metadata": {{"num_tensors":{d},"total_size":{d}}}, + \\"weight_map": {{ + \\ + , .{ self.tensors.count(), total_size }); + + const n = self.tensors.count(); + for (0.., self.tensors.keys(), self.tensors.values()) |i, key, value| { + std.debug.assert(std.mem.startsWith(u8, value.file_uri, repo_name)); + const local_name = value.file_uri[repo_name.len + 1 ..]; + const end_offset = value.offset + value.byteSize(); + const safetensors_dtype: []const u8 = dt: { + for (safetensors_dtype_to_zml_dtype.keys(), safetensors_dtype_to_zml_dtype.values()) |st_dt, zml_dt| { + if (zml_dt == value.shape._dtype) break :dt st_dt; + } + // Note: we could bitcast to u8, + // but reaching this panic implies the user didn't loaded this TensorRegistry + // from a safetensors file, and manually created the dict. + // In which case, they know what they are doing, + // and can easily insert the bitcast themselves. + std.debug.panic("Safetensors doesn't support data type {}", .{value.shape._dtype}); + }; + const comma = if (i + 1 == n) "" else ","; + try out.print( + \\ "{s}": {{"filename":"{s}","shape":{f},"data_offsets":[{d},{d}],"dtype":"{s}"}}{s} + \\ + , .{ key, local_name, std.json.fmt(value.shape.dims(), .{ .whitespace = .minified }), value.offset, end_offset, safetensors_dtype, comma }); + } + + try out.writeAll( + \\} + \\} + \\ + ); + try out.flush(); + } +}; + +pub const TensorReader = struct { + tensor: Tensor, + file: std.Io.File, + file_reader: std.Io.File.Reader, + alignment: ?std.mem.Alignment, + remaining: u64, + padding_remaining: u64, + + io: std.Io, + interface: std.Io.Reader, + + pub const InitOpts = struct { + alignment: ?std.mem.Alignment = null, + }; + + pub fn init( + io: std.Io, + tensor: Tensor, + buffer: []u8, + opts: InitOpts, + ) !TensorReader { + const file = try std.Io.Dir.openFile(.cwd(), io, tensor.file_uri, .{ .mode = .read_only }); + errdefer file.close(io); + + const offset, const padding_remaining = if (opts.alignment) |alignment| blk: { + const aligned_offset = std.mem.alignBackward(u64, tensor.offset, alignment.toByteUnits()); + break :blk .{ aligned_offset, tensor.offset - aligned_offset }; + } else blk: { + break :blk .{ tensor.offset, 0 }; + }; + + var file_reader = file.reader(io, buffer); + try file_reader.seekTo(offset); + + return .{ + .tensor = tensor, + .file = file, + .file_reader = file_reader, + .alignment = opts.alignment, + .remaining = tensor.byteSize(), + .padding_remaining = padding_remaining, + .io = io, + .interface = .{ + .vtable = if (opts.alignment) |_| blk: { + break :blk &.{ + .stream = streamAligned, + .discard = discardAligned, + }; + } else blk: { + break :blk &.{ + .stream = stream, + .discard = discard, + }; + }, + .buffer = &.{}, + .seek = 0, + .end = 0, + }, + }; + } + + pub fn deinit(self: *TensorReader) void { + self.file.close(self.io); + } + + // Limit single reads to 1GB to avoid issues on macOS where pread returns + // INT_MAX when the requested size is larger than 2GB + const max_single_read_bytes: u64 = 1 << 30; + + fn stream(r: *std.Io.Reader, w: *std.Io.Writer, limit: std.Io.Limit) std.Io.Reader.StreamError!usize { + const self: *TensorReader = @fieldParentPtr("interface", r); + if (self.remaining == 0) { + return error.EndOfStream; + } + + const combined_limit = limit.min(.limited64(self.remaining)).min(.limited64(max_single_read_bytes)); + const n = try self.file_reader.interface.stream(w, combined_limit); + self.remaining -= n; + return n; + } + + fn discard(r: *std.Io.Reader, limit: std.Io.Limit) std.Io.Reader.Error!usize { + const self: *TensorReader = @fieldParentPtr("interface", r); + if (self.remaining == 0) { + return error.EndOfStream; + } + + const combined_limit = limit.min(.limited64(self.remaining)); + const n = try self.file_reader.interface.discard(combined_limit); + self.remaining -= n; + return n; + } + + fn streamAligned(r: *std.Io.Reader, w: *std.Io.Writer, limit: std.Io.Limit) std.Io.Reader.StreamError!usize { + const self: *TensorReader = @fieldParentPtr("interface", r); + if (self.remaining == 0) { + return error.EndOfStream; + } + + const requested_bytes = @min(limit.minInt64(self.remaining), max_single_read_bytes); + + if (self.padding_remaining > 0) { + _ = try self.file_reader.interface.discard(.limited64(self.padding_remaining)); + self.padding_remaining = 0; + } + + const align_bytes = self.alignment.?.toByteUnits(); + const file_offset = self.file_reader.pos - std.mem.alignBackward(u64, self.file_reader.pos, align_bytes); + const writer_addr = @intFromPtr(w.buffer.ptr) + w.end; + const writer_offset = writer_addr - std.mem.alignBackward(usize, writer_addr, align_bytes); + + if (file_offset != writer_offset) { + const skip = if (file_offset >= writer_offset) blk: { + break :blk file_offset - writer_offset; + } else blk: { + break :blk align_bytes - writer_offset + file_offset; + }; + + w.advance(skip); + } + + if (w.writableSliceGreedy(align_bytes)) |dest| { + const phys_buf = dest.ptr - file_offset; + const phys_pos = self.file_reader.pos - file_offset; + const phys_size = std.mem.alignBackward(usize, @min(dest.len, requested_bytes) + file_offset, align_bytes); + + if (phys_size > file_offset) { + const phys_read = self.file.readPositional(self.io, &.{phys_buf[0..phys_size]}, phys_pos) catch unreachable; + if (phys_read > file_offset) { + const bytes_read = phys_read - file_offset; + w.advance(bytes_read); + self.file_reader.pos += bytes_read; + self.remaining -= bytes_read; + return bytes_read; + } + } + } else |_| {} + + const bytes_read = try self.file_reader.interface.stream(w, .limited64(requested_bytes)); + self.remaining -= bytes_read; + + return bytes_read; + } + + fn discardAligned(r: *std.Io.Reader, limit: std.Io.Limit) std.Io.Reader.Error!usize { + const self: *TensorReader = @fieldParentPtr("interface", r); + if (self.remaining == 0) return 0; + + if (self.padding_remaining > 0) { + _ = try self.file_reader.interface.discard(.limited64(self.padding_remaining)); + self.padding_remaining = 0; + } + + const combined_limit = limit.min(.limited64(self.remaining)); + const n = try self.file_reader.interface.discard(combined_limit); + self.remaining -= n; + return n; + } +}; + +pub const Tensor = struct { + file_uri: []const u8, + name: []const u8, + shape: Shape, + offset: u64, + + pub fn byteSize(self: Tensor) u64 { + return self.shape.byteSize(); + } + + pub fn reader(self: Tensor, io: std.Io, buffer: []u8, opts: TensorReader.InitOpts) !TensorReader { + return TensorReader.init(io, self, buffer, opts); + } + + pub fn parse(object: std.json.ObjectMap, file_uri: []const u8, name: []const u8, base_offset: u64) !Tensor { + const shape_field = object.get("shape").?.array; + if (shape_field.items.len > Shape.MAX_RANK) { + log.warn("Can't load tensor {s}, too many dims: {}", .{ name, shape_field.items.len }); + return error.TooManyDimensions; + } + + const offset_field = object.get("data_offsets").?; + const start: u64 = @intCast(offset_field.array.items[0].integer); + const dtype = try stringToDtype(object.get("dtype").?.string); + + var dims: Shape.DimsArray = .empty; + for (shape_field.items) |d| { + dims.appendAssumeCapacity(d.integer); + } + + return .{ + .file_uri = file_uri, + .name = name, + .shape = .init(dims.slice(), dtype), + .offset = base_offset + start, + }; + } + + pub fn format(self: Tensor, writer: *std.Io.Writer) !void { + try writer.print("Tensor(name={s} shape={f} size={d}, offset={d}, file_uri={s})", .{ + self.name, + self.shape, + self.byteSize(), + self.offset, + self.file_uri, + }); + } +}; + +pub const FileType = enum { + full_index, + index, + safetensors, + unknown, + + pub fn resolve(path: []const u8) FileType { + if (std.mem.endsWith(u8, path, ".safetensors.index.json")) { + return .index; + } else if (std.mem.endsWith(u8, path, ".safetensors.full_index.json")) { + return .full_index; + } else if (std.mem.endsWith(u8, path, ".safetensors")) { + return .safetensors; + } else { + return .unknown; + } + } +}; + +pub const ModelPathResolutionError = error{ + FileNotFound, + InvalidPath, +} || std.mem.Allocator.Error; + +pub fn resolveModelRepo(io: std.Io, path: []const u8) ModelPathResolutionError!std.Io.Dir { + switch (FileType.resolve(path)) { + .full_index, .index, .safetensors => { + // The path is a file, let's open it's folder + const dir_path = std.fs.path.dirname(path) orelse return ModelPathResolutionError.InvalidPath; + + const dir = std.Io.Dir.openDir(.cwd(), io, dir_path, .{}) catch |e| { + return switch (e) { + error.FileNotFound => ModelPathResolutionError.FileNotFound, + else => ModelPathResolutionError.InvalidPath, + }; + }; + + return dir; + }, + .unknown => { + // The path is probably a directory, let's try to open it as such + const dir = std.Io.Dir.openDir(.cwd(), io, path, .{}) catch |e| { + return switch (e) { + error.FileNotFound => ModelPathResolutionError.FileNotFound, + else => ModelPathResolutionError.InvalidPath, + }; + }; + return dir; + }, + } +} + +pub fn resolveModelEntrypoint(io: std.Io, repo: std.Io.Dir) ModelPathResolutionError!struct { FileType, std.Io.File } { + const default_entrypoints: []const struct { FileType, []const u8 } = &.{ + .{ .full_index, "model.safetensors.full_index.json" }, + .{ .full_index, "consolidated.safetensors.full_index.json" }, + .{ .index, "model.safetensors.index.json" }, + .{ .index, "consolidated.safetensors.index.json" }, + .{ .safetensors, "model.safetensors" }, + .{ .safetensors, "consolidated.safetensors" }, + }; + + for (default_entrypoints) |entrypoint| { + const file_type, const file_name = entrypoint; + const file = repo.openFile(io, file_name, .{ .mode = .read_only }) catch continue; + + return .{ file_type, file }; + } + + return error.FileNotFound; +} + +const safetensors_dtype_to_zml_dtype = std.StaticStringMap(DataType).initComptime(.{ + .{ "F64", .f64 }, + .{ "F32", .f32 }, + .{ "F16", .f16 }, + .{ "BF16", .bf16 }, + .{ "F8_E8M0", .f8e8m0 }, + .{ "F8_E4M3", .f8e4m3fn }, + .{ "F4_E2M1", .f4e2m1 }, + .{ "I64", .i64 }, + .{ "I32", .i32 }, + .{ "I16", .i16 }, + .{ "I8", .i8 }, + .{ "U64", .u64 }, + .{ "U32", .u32 }, + .{ "U16", .u16 }, + .{ "U8", .u8 }, + .{ "BOOL", .bool }, +}); + +fn stringToDtype(safetensor_dtype: []const u8) !DataType { + return safetensors_dtype_to_zml_dtype.get(safetensor_dtype) orelse { + log.err("Unsupported safetensor data type: {s}", .{safetensor_dtype}); + return error.UnsupportedDataType; + }; +} diff --git a/zml/shape.zig b/zml/shape.zig index 2ed40161..386b594e 100644 --- a/zml/shape.zig +++ b/zml/shape.zig @@ -1,11 +1,13 @@ -const builtin = @import("builtin"); const std = @import("std"); const testing = std.testing; +const builtin = @import("builtin"); -const meta = @import("meta.zig"); +const stdx = @import("stdx"); + +const constants = @import("constants.zig"); const DataType = @import("dtype.zig").DataType; +const Sharding = @import("Sharding.zig"); -const EnumLiteral = @TypeOf(.enum_literal); const log = std.log.scoped(.shape); test { @@ -13,24 +15,28 @@ test { } /// Represent the shape of a tensor. +/// Contains also the sharding information, +/// so it describes exactly how the corresponding tensor/buffer will be layed out +/// across all devices of the current Platform. +/// +/// By default all created Shape are replicated, meaning the corresponding tensor / buffer will be copied on each device. +/// `.withPartitioning` allows to specify how it should be split. pub const Shape = struct { - pub const MAX_RANK: u8 = 8; + _dtype: DataType, + _dims: DimsArray, + _tags: TagsArray, + _sharding: Sharding, pub const Tag = [*:0]const u8; pub const TagUnknown = "_".ptr; const TagLast = "last".ptr; + const TagReplicated = "replicated".ptr; + const TagOpen = "open".ptr; - pub const DimsArray = std.BoundedArray(i64, MAX_RANK); - pub const TagsArray = std.BoundedArray(Tag, MAX_RANK); - pub const AxesArray = std.BoundedArray(u3, MAX_RANK); - pub const ShardingInfo = @Vector(MAX_RANK, bool); - - const UnknownTags: TagsArray = .{ .len = 0, .buffer = [_]Tag{TagUnknown} ** MAX_RANK }; - - _dtype: DataType, - _dims: DimsArray = .{}, - _tags: TagsArray = UnknownTags, - _sharding_info: ShardingInfo = @splat(false), + pub const MAX_RANK: u8 = constants.MAX_RANK; + pub const DimsArray = stdx.BoundedArray(i64, constants.MAX_RANK); + pub const TagsArray = stdx.BoundedArray(Tag, constants.MAX_RANK); + pub const AxesArray = stdx.BoundedArray(u3, constants.MAX_RANK); pub fn parseDimensions(v: anytype) struct { DimsArray, TagsArray } { const T = @TypeOf(v); @@ -39,9 +45,9 @@ pub const Shape = struct { return .{ v._dims, v._tags }; } - if (comptime meta.isSliceOfAny(T, meta.isInteger)) { - var dims_ = DimsArray.init(0) catch unreachable; - var tags_ = TagsArray.init(0) catch unreachable; + if (comptime stdx.meta.isSliceOfAny(T, stdx.meta.isInteger)) { + var dims_: DimsArray = .empty; + var tags_: TagsArray = .empty; for (v) |d| { dims_.appendAssumeCapacity(@intCast(d)); tags_.appendAssumeCapacity(TagUnknown); @@ -49,29 +55,29 @@ pub const Shape = struct { return .{ dims_, tags_ }; } - if (comptime meta.isStruct(T)) { - var dims_: DimsArray = .{}; - var tags_: TagsArray = .{}; - inline for (std.meta.fields(T)) |field| { - const fv = @field(v, field.name); - if (comptime meta.isInteger(field.type)) { + if (comptime stdx.meta.isStruct(T)) { + var dims_: DimsArray = .empty; + var tags_: TagsArray = .empty; + inline for (comptime std.meta.fieldNames(T), comptime std.meta.fieldTypes(T)) |field_name, Field| { + const fv = @field(v, field_name); + if (comptime stdx.meta.isInteger(Field)) { dims_.appendAssumeCapacity(@intCast(fv)); - } else if (comptime isAutoDim(fv)) { + } else if (@TypeOf(fv) == @EnumLiteral() and comptime isAutoDim(fv)) { dims_.appendAssumeCapacity(-1); } else { - meta.compileError("Field {s} should be an integer or an auto dimension", .{field.name}); + stdx.debug.compileError("Field {s} should be an integer or an auto dimension, got {}", .{ field_name, Field }); } - if (comptime meta.isTuple(T)) { + if (comptime stdx.meta.isTuple(T)) { tags_.appendAssumeCapacity(TagUnknown); } else { - tags_.appendAssumeCapacity(toTag(field)); + tags_.appendAssumeCapacity(toTag(field_name)); } } return .{ dims_, tags_ }; } - meta.compileError("expected a dimension tuple eg '.{{ .a = 10, .b = 20}}' or '.{{ 10, 20 }}', got {}", .{T}); + stdx.debug.compileError("expected a dimension tuple eg '.{{ .a = 10, .b = 20}}' or '.{{ 10, 20 }}', got {}", .{T}); } test parseDimensions { @@ -89,10 +95,10 @@ pub const Shape = struct { return self.parseAxes(self.tags()); } - var axes_ = AxesArray.init(0) catch unreachable; - var tags_ = TagsArray.init(0) catch unreachable; + var axes_: AxesArray = .empty; + var tags_: TagsArray = .empty; - if (comptime meta.isSliceOfAny(T, isAxisConvertible)) { + if (comptime stdx.meta.isSliceOfAny(T, isAxisConvertible)) { for (v) |d| { axes_.appendAssumeCapacity(self.axis(d)); tags_.appendAssumeCapacity(self.tag(d)); @@ -100,44 +106,79 @@ pub const Shape = struct { return .{ axes_, tags_ }; } - if (comptime meta.isTupleOfAny(T, isAxisConvertible)) { - inline for (std.meta.fields(T)) |field| { - axes_.appendAssumeCapacity(self.axis(@field(v, field.name))); - tags_.appendAssumeCapacity(self.tag(@field(v, field.name))); + if (comptime stdx.meta.isTupleOfAny(T, isAxisConvertible)) { + inline for (comptime std.meta.fieldNames(T)) |field_name| { + axes_.appendAssumeCapacity(self.axis(@field(v, field_name))); + tags_.appendAssumeCapacity(self.tag(@field(v, field_name))); } return .{ axes_, tags_ }; } - meta.compileError("Wrong type, got {}. Expected .{{.a, .b}}", .{T}); + stdx.debug.compileError("Wrong type, got {}. Expected .{{.a, .b}}", .{T}); } - pub fn parseTags(v: anytype) TagsArray { + pub fn parseTags(v: anytype) []const Shape.Tag { const T = @TypeOf(v); - meta.assertComptime(meta.isTupleOf(T, EnumLiteral), "Wrong type, got {}. Expected .{{ .a, .b }}", .{T}); - var tags_ = TagsArray.init(0) catch unreachable; - inline for (v) |field| { - tags_.appendAssumeCapacity(toTag(field)); + if (T == []const Shape.Tag) return v; + comptime validateTags(T); + return comptime frozen: { + var tagz: [v.len]Shape.Tag = undefined; + for (v, 0..) |field, i| tagz[i] = @tagName(field).ptr; + // Force the creation of a comptime constant, so that we can return a pointer to it. + const frozen_tags = tagz; + break :frozen &frozen_tags; + }; + } + + fn validateTags(T: type) void { + const message = "Expected an enum-literal tuple such as .{ .b, .d }, got " ++ @typeName(T); + switch (@typeInfo(T)) { + .@"struct" => |info| { + if (!info.is_tuple) @compileError(message); + for (info.field_types) |Field| { + if (Field != @EnumLiteral()) @compileError(message); + } + if (info.field_types.len > Shape.MAX_RANK) @compileError("Too many shape tags (maximum 8)"); + }, + else => @compileError(message), } - return tags_; } /// Create a shape from a struct literal, eg: /// Shape.init(.{ .h = 1024, .w = 512, .c = 3 }); /// Shape.init(.{ 1024, 512, 3 }); + /// + /// The created shape is replicated. pub fn init(dimz: anytype, dt: DataType) Shape { - var res: Shape = .{ ._dtype = dt }; - res._dims, res._tags = parseDimensions(dimz); - return res; + const _dims, const _tags = parseDimensions(dimz); + return .{ + ._dtype = dt, + ._dims = _dims, + ._tags = _tags, + ._sharding = .replicated, + }; + } + + pub fn scalar(dt: DataType) Shape { + return .{ + ._dtype = dt, + ._dims = .empty, + ._tags = .{ .len = 0, .buffer = @splat(TagUnknown) }, + ._sharding = .replicated, + }; } /// Creates a Shape with dims set to `.{0, 1, 2, ..., rank-1}`. pub fn range(rank_: usize, dt: DataType) Shape { - var res: Shape = .{ ._dtype = dt }; + stdx.debug.assert(rank_ <= MAX_RANK, "zml.Shape.range expects a maximum rank of {d}, got {}", .{ MAX_RANK, rank_ }); + var res: Shape = .{ + ._dtype = dt, + ._dims = .empty, + ._tags = .{ .buffer = @splat(TagUnknown), .len = rank_ }, + ._sharding = .replicated, + }; for (0..rank_) |i| { - res._dims.append(@intCast(i)) catch { - meta.panic("Too many dimensions! Max: {d}, passed: {d}", .{ res._dims.capacity(), rank_ }); - }; - res._tags.append(TagUnknown) catch unreachable; + res._dims.appendAssumeCapacity(@intCast(i)); } return res; } @@ -147,49 +188,49 @@ pub const Shape = struct { } pub fn rank(self: Shape) u4 { - self.ensureDimsAndTagsAreSync(); - return self._dims.len; + self.ensureAttributesAreSync(); + return @intCast(self._dims.len); } pub fn dim(self: Shape, ax: anytype) i64 { - self.ensureDimsAndTagsAreSync(); + self.ensureAttributesAreSync(); return self._dims.get(self.axis(ax)); } pub fn dims(self: *const Shape) []const i64 { - self.ensureDimsAndTagsAreSync(); + self.ensureAttributesAreSync(); return self._dims.constSlice(); } fn isAxisConvertible(comptime T: type) bool { - return meta.isInteger(T) or isTagConvertible(T); + return stdx.meta.isInteger(T) or isTagConvertible(T); } fn isTagConvertible(comptime T: type) bool { return switch (T) { - EnumLiteral => true, - std.builtin.Type.StructField => true, - Tag => true, + @EnumLiteral(), [:0]const u8, Tag => true, else => false, }; } - fn toTag(v: anytype) Tag { + pub fn toTag(v: anytype) Tag { const T = @TypeOf(v); return switch (T) { - EnumLiteral => @tagName(v).ptr, - std.builtin.Type.StructField => v.name.ptr, + @EnumLiteral() => @tagName(v).ptr, + [:0]const u8 => v.ptr, Tag => v, - else => meta.compileError("Value should be an EnumLiteral, a Shape.Tag or a StructField, got {}", .{T}), + else => stdx.debug.compileError("Shape tag should be an @EnumLiteral(), a Shape.Tag or a field name, got {}", .{T}), }; } - inline fn ensureDimsAndTagsAreSync(self: Shape) void { - meta.assert(self._dims.len == self._tags.len, "Tags and dims have diverged! dims={d} tags={d}", .{ self._dims.len, self._tags.len }); + fn ensureAttributesAreSync(self: Shape) void { + if (builtin.mode == .debug) { + stdx.debug.assert(self._dims.len == self._tags.len, "Tags, dims and partitioning have diverged! dims={d} tags={d}", .{ self._dims.len, self._tags.len }); + } } pub fn tag(self: Shape, ax: anytype) Tag { - self.ensureDimsAndTagsAreSync(); + self.ensureAttributesAreSync(); return self._tags.get(self.axis(ax)); } @@ -209,18 +250,18 @@ pub const Shape = struct { } pub fn tags(self: *const Shape) []const Tag { - self.ensureDimsAndTagsAreSync(); + self.ensureAttributesAreSync(); return self._tags.constSlice(); } pub fn hasTag(self: Shape, tag_: anytype) ?u3 { - return self.axisFromTagMaybe(toTag(tag_)); + return axisFromTagMaybe(self.tags(), toTag(tag_)); } pub fn hasTags(self: Shape, tagz: anytype) bool { const T = @TypeOf(tagz); - if (comptime meta.isSliceOf(T, Tag) or meta.isSliceOf(T, EnumLiteral)) { + if (comptime stdx.meta.isSliceOf(T, Tag) or stdx.meta.isSliceOf(T, @EnumLiteral())) { for (tagz) |t| { if (self.hasTag(t) == null) { return false; @@ -229,7 +270,7 @@ pub const Shape = struct { return true; } - if (comptime meta.isTupleOf(T, Tag) or meta.isTupleOf(T, EnumLiteral)) { + if (comptime stdx.meta.isTupleOf(T, Tag) or stdx.meta.isTupleOf(T, @EnumLiteral())) { inline for (tagz) |t| { if (self.hasTag(t) == null) { return false; @@ -238,7 +279,7 @@ pub const Shape = struct { return true; } - meta.compileError("Expected tuple of tags, got {any}", .{T}); + stdx.debug.compileError("Expected tuple of tags, got {any}", .{T}); } pub fn isFullyTagged(self: Shape) bool { @@ -249,22 +290,28 @@ pub const Shape = struct { } pub fn axis(self: Shape, axis_: anytype) u3 { - self.ensureDimsAndTagsAreSync(); + self.ensureAttributesAreSync(); const T = @TypeOf(axis_); - if (comptime meta.isInteger(T)) { - return self.axisFromInt(@intCast(axis_)); - } - - if (comptime isTagConvertible(T)) { - return self.axisFromTag(toTag(axis_)); - } - - meta.compileError("Wrong axis type, expected .literal, or an integer, got: {any}", .{T}); + // Special handling of unsigned int, to remove runtime computations and favor inlining + return switch (@typeInfo(T)) { + .int => |info| if (info.signedness == .signed) + self.axisFromInt(axis_) + else + @intCast(axis_), + .comptime_int => if (axis_ < 0) + self.axisFromInt(axis_) + else + axis_, + else => if (comptime isTagConvertible(T)) + self.axisFromTag(toTag(axis_)) + else + stdx.debug.compileError("Wrong axis type, expected .literal, or an integer, got: {any}", .{T}), + }; } pub fn axes(self: Shape, axes_: anytype) AxesArray { - self.ensureDimsAndTagsAreSync(); + self.ensureAttributesAreSync(); const T = @TypeOf(axes_); @@ -272,60 +319,72 @@ pub const Shape = struct { return self.axes(axes_.tags()); } - var res = AxesArray.init(0) catch unreachable; + var res: AxesArray = .empty; - if (comptime meta.isSliceOfAny(T, meta.isInteger) or meta.isSliceOf(T, Tag)) { + if (comptime stdx.meta.isSliceOfAny(T, stdx.meta.isInteger) or stdx.meta.isSliceOf(T, Tag)) { for (axes_) |ax| { res.appendAssumeCapacity(self.axis(ax)); } return res; } - if (comptime meta.isStruct(T)) { - inline for (std.meta.fields(T)) |field| { - res.appendAssumeCapacity(self.axis(@field(axes_, field.name))); + if (comptime stdx.meta.isStruct(T)) { + inline for (comptime std.meta.fieldNames(T)) |field_name| { + res.appendAssumeCapacity(self.axis(@field(axes_, field_name))); } return res; } - meta.compileError("axes expects an int-tuple or a tuple of enum literal, got {}", .{T}); + stdx.debug.compileError("axes expects an int-tuple or a tuple of enum literal, got {}", .{T}); } - fn axisFromInt(self: Shape, d: isize) u3 { + pub fn axisFromInt(self: Shape, a: isize) u3 { const rk: i8 = self.rank(); - if (d < -rk or d > rk) { - meta.panic("Tensor {} doesn't have dimension: {d}", .{ self, d }); + if (a < -rk or a > rk) { + stdx.debug.panic("Tensor {f} doesn't have dimension: {d}", .{ self, a }); } - return if (d < 0) - @intCast(d + rk) + return if (a < 0) + @intCast(a + rk) else - @intCast(d); + @intCast(a); } - fn axisFromTagMaybe(self: Shape, d: Tag) ?u3 { - if (d == TagUnknown) { - return null; - } + pub fn axisFromTagMaybe(tagz: []const Tag, t: Tag) ?u3 { + if (t == TagUnknown) return null; + + if (axisFromLiteralInt(t)) |ax| return ax; + if (@inComptime()) { - for (0.., self.tags()) |tagIndex, t| { - const a: []const u8 = std.mem.span(t); - const b: []const u8 = std.mem.span(d); - if (std.mem.eql(u8, a, b)) { - return @intCast(tagIndex); + // At comptime two duplicated strings may have two different representations + const t_bytes: []const u8 = std.mem.span(t); + for (tagz, 0..) |self_tag, ax| { + if (std.mem.eql(u8, t_bytes, std.mem.span(self_tag))) { + return @truncate(ax); } } return null; } - if (std.mem.indexOfScalar(Tag, self.tags(), d)) |d_| { - return @intCast(d_); + + // But at runtime the comptime strings have been deduplicated and ptr match is enough. + if (std.mem.indexOfScalar(Tag, tagz, t)) |ax| { + return @truncate(ax); + } + return null; + } + + /// Handle .{ ._0 = x } syntax. + fn axisFromLiteralInt(t: Tag) ?u3 { + // match .{ '_', '0-9', null } + if (t[0] == '_' and t[1] >= '0' and t[1] < '8' and t[2] == 0) { + return @intCast(t[1] - '0'); } return null; } fn axisFromTag(self: Shape, d: Tag) u3 { - meta.assert(d != TagUnknown, "The unknown tag .{s} can't be used to fetch axis in {}", .{ d, self }); - return self.axisFromTagMaybe(d) orelse { - meta.panic("Tensor {} doesn't have dimension with tag: {s}", .{ self, d }); + stdx.debug.assert(d != TagUnknown, "The unknown tag .{s} can't be used to fetch axis in {f}", .{ d, self }); + return axisFromTagMaybe(self.tags(), d) orelse { + stdx.debug.panic("Tensor {f} doesn't have dimension with tag: {s}", .{ self, d }); }; } @@ -339,15 +398,15 @@ pub const Shape = struct { pub fn count(self: Shape) usize { var res: i64 = 1; for (self.dims()) |d| { - meta.assert(d >= 0, "Can't count elements in shape with negative dimension: {}", .{self}); + stdx.debug.assert(d >= 0, "Can't count elements in shape with negative dimension: {f}", .{&self}); res *= d; } return @intCast(res); } - /// Total size in bytes needed to represent this shape. + /// Total size in bytes needed to represent this shape on host pub fn byteSize(self: Shape) usize { - return self.dtype().sizeOf() * self.count(); + return std.math.divCeil(usize, self.count() * self.dtype().bitSizeOf(), 8) catch unreachable; } /// Compares the two shapes described, ignoring tagging. @@ -366,44 +425,85 @@ pub const Shape = struct { } /// Format the shape. - /// Default format: "Shape({.a=10, .b=20}, dtype=.f32)" - /// Bare format {_}: "{.a=10, .b=20}, dtype=.f32" - pub fn format( - self: Shape, - comptime fmt: []const u8, - options: std.fmt.FormatOptions, - writer: anytype, - ) !void { - _ = options; - const bare_fmt = fmt.len == 1 and fmt[0] == '_'; - _ = try writer.write(if (bare_fmt) "{" else "Shape({"); + /// "Shape({.a=10, .b=20}, dtype=.f32)" + pub fn format(self: Shape, writer: *std.Io.Writer) !void { + _ = try writer.writeByte('{'); + var need_comma = false; for (self.dims(), 0..) |d, i| { - const prefix = if (i == 0) "" else ","; + if (need_comma) try writer.writeByte(','); const t = self.tag(i); if (t != TagUnknown) { - try writer.print("{s}.{s}={d}", .{ prefix, t, d }); + try writer.print("{s}={d}", .{ t, d }); } else { - try writer.print("{s}{d}", .{ prefix, d }); + try writer.print("{d}", .{d}); } - if (self._sharding_info[i]) { - try writer.writeByte('!'); + + const partitioning = self._sharding.get(i); + switch (partitioning) { + .replicated => {}, + .open => try writer.writeAll("/?"), + // this state is invalid, but don't panic cause we may be called from a panic + .out_of_bound => try writer.writeAll("/!"), + else => { + const mesh_ax = partitioning.meshAxis().?; + try writer.print("/{s}", .{self._sharding.mesh.?.logical.axes.get(mesh_ax)}); + }, } + + need_comma = true; + } + if (need_comma) try writer.writeByte(','); + _ = try writer.write(@tagName(self.dtype())); + _ = try writer.writeByte('}'); + if (self._sharding.mesh) |m| { + try writer.writeByte('@'); + try writer.writeAll(m.name); } - _ = try writer.print("}}, dtype=.{s}", .{@tagName(self.dtype())}); - if (!bare_fmt) _ = try writer.write(")"); } + /// Broadcasts a Tensor to the given shape, extending dimensions if needed. + pub fn canBroadcastTo(self: Shape, other: Shape) bool { + // Already the right shape + if (std.mem.eql(i64, self.dims(), other.dims())) return true; + + // Non ambiguous broadcasting + // TODO: broad is error prone because of this: + // it will happily broadcast .{ .a = 10, .b = 1 } to .{ .b = 10, .a = 5 } + if (self.rank() == 0 or self.rank() == other.rank()) { + for (0..self.rank()) |i| { + if (self.dim(i) != 1 and self.dim(i) != other.dim(i)) return false; + } + return true; + } + + for (self.dims(), self.tags()) |d, t| { + // TODO this is also wrong when the axes are in different order in the two shapes. + const other_ax = other.hasTag(t) orelse return false; + if (d != 1 and d != other.dim(other_ax)) return false; + } + return true; + } + + /// Reshape a given shape but keep the same total number of elements. + /// Partitioning information is generally NOT preserved, + /// unless input was fully replicated. pub fn reshape(self: Shape, new_shape_: anytype) Shape { - var new_shape: Shape = .{ ._dtype = self.dtype() }; - new_shape._dims, new_shape._tags = parseDimensions(new_shape_); - new_shape.inferMissingAxis(self.count()); - meta.assert(self.count() == new_shape.count(), "Can't reshape {d} to {d}", .{ self.dims(), new_shape.dims() }); + const _dims, const _tags = parseDimensions(new_shape_); + var new_shape: Shape = .{ + ._dtype = self._dtype, + ._dims = _dims, + ._tags = _tags, + ._sharding = .{ .mesh = self._sharding.mesh, .partition = if (self.isFullyReplicated()) .replicated(_dims.len) else .open(_dims.len) }, + }; + new_shape.inferMissingAxis(self.count()) catch |err| { + std.debug.panic("Can't reshape {any} to {any}: {t}", .{ self.dims(), new_shape.dims(), err }); + }; return new_shape; } - fn inferMissingAxis(self: *Shape, n_: usize) void { - meta.assert(std.mem.count(i64, self.dims(), &.{-1}) < 2, "Cannot infer multiple dimensions when reshaping to: {}", .{self.*}); + fn inferMissingAxis(self: *Shape, n_: usize) error{ UnexpectedRemainder, TooBig }!void { + stdx.debug.assert(std.mem.count(i64, self.dims(), &.{-1}) < 2, "Cannot infer multiple dimensions when reshaping to: {f}", .{self.*}); const inferred_ax = std.mem.indexOfScalar(i64, self.dims(), -1) orelse return; // We can't use `self.count()` yet cause we have negative dims. @@ -414,10 +514,10 @@ pub const Shape = struct { } } const n: i64 = @intCast(n_); + if (tmp_count > n) return error.TooBig; // Abort, `reshape` will panic with more context. - if (@mod(n, tmp_count) != 0) { - return; - } + if (@mod(n, tmp_count) != 0) return error.UnexpectedRemainder; + self._dims.set(inferred_ax, @divExact(n, tmp_count)); } @@ -435,6 +535,11 @@ pub const Shape = struct { const res = x.reshape(.{-1}); try testing.expectEqualSlices(i64, &.{30}, res.dims()); } + { + var y: Shape = .init(.{ 2, 5, -1 }, .f32); + try testing.expectError(error.UnexpectedRemainder, inferMissingAxis(&y, 15)); + try testing.expectError(error.TooBig, inferMissingAxis(&y, 8)); + } } pub fn setDim(self: Shape, ax: anytype, d: i64) Shape { @@ -463,7 +568,7 @@ pub const Shape = struct { /// ``` pub fn insert(self: Shape, axis_: anytype, dimz: anytype) Shape { const dims_, const tags_ = parseDimensions(dimz); - const ax = if (@TypeOf(axis_) == EnumLiteral and axis_ == .last) + const ax = if (@TypeOf(axis_) == @EnumLiteral() and axis_ == .last) self.rank() else self.axis(axis_); @@ -471,6 +576,11 @@ pub const Shape = struct { var res = self; res._dims.insertSlice(ax, dims_.constSlice()) catch unreachable; res._tags.insertSlice(ax, tags_.constSlice()) catch unreachable; + + for (0..dims_.len) |_| { + res._sharding = res._sharding.insert(ax, .replicated); + } + return res; } @@ -481,9 +591,9 @@ pub const Shape = struct { } pub fn insertTag(self: Shape, axis_: anytype, d: i64, tag_: anytype) Shape { - meta.assert(self.rank() < MAX_RANK - 1, "Can't insert new axis in {}, it's already at max rank.", .{self}); + stdx.debug.assert(self.rank() < constants.MAX_RANK - 1, "Can't insert new axis in {f}, it's already at max rank.", .{self}); - const ax = if (@TypeOf(axis_) == EnumLiteral and axis_ == .last) + const ax = if (@TypeOf(axis_) == @EnumLiteral() and axis_ == .last) self.rank() else self.axis(axis_); @@ -491,6 +601,8 @@ pub const Shape = struct { var res = self; res._dims.insert(ax, d) catch unreachable; res._tags.insert(ax, toTag(tag_)) catch unreachable; + res._sharding = res._sharding.insert(ax, .replicated); + return res; } @@ -499,6 +611,7 @@ pub const Shape = struct { const dims_, const tags_ = parseDimensions(v); res._dims.appendSliceAssumeCapacity(dims_.constSlice()); res._tags.appendSliceAssumeCapacity(tags_.constSlice()); + for (self.rank()..res.rank()) |ax| res._sharding = res._sharding.set(ax, .replicated); return res; } @@ -520,32 +633,77 @@ pub const Shape = struct { var res = self; res._dims.appendAssumeCapacity(d); res._tags.appendAssumeCapacity(if (tag_) |t| t else TagUnknown); + res._sharding = res._sharding.set(self.rank(), .replicated); return res; } - pub fn remove(self: Shape, d: anytype) Shape { + pub fn remove(self: Shape, axis_: anytype) Shape { var res = self; - const d_ = self.axis(d); - _ = res._dims.orderedRemove(d_); - _ = res._tags.orderedRemove(d_); + const a = self.axis(axis_); + _ = res._dims.orderedRemove(a); + _ = res._tags.orderedRemove(a); + res._sharding = res._sharding.orderedRemove(a); return res; } pub const drop = remove; test remove { - try std.testing.expectEqualSlices(i64, &.{ 10, 12 }, Shape.init(.{ 10, 11, 12 }, .f32).remove(1).dims()); - try std.testing.expectEqualSlices(i64, &.{ 10, 11, 12 }, Shape.init(.{ 10, 11, 12, 13 }, .f32).remove(-1).dims()); + try testing.expectEqualSlices(i64, &.{ 10, 12 }, Shape.init(.{ 10, 11, 12 }, .f32).remove(1).dims()); + try testing.expectEqualSlices(i64, &.{ 10, 11, 12 }, Shape.init(.{ 10, 11, 12, 13 }, .f32).remove(-1).dims()); + } + + pub fn removeMany(self: Shape, axes_: anytype) Shape { + var to_remove = self.axes(axes_); + if (to_remove.len == 0) return self; + std.mem.sort(u3, to_remove.slice(), {}, std.sort.asc(u3)); + + var sh: Shape = self; + sh._sharding.partition = .out_of_bound; + const rk = self.rank(); + var res_ax: u32 = 0; + for (0..rk) |ax| { + if (std.mem.indexOfScalar(u3, to_remove.constSlice(), @intCast(ax))) |_| { + continue; + } + + sh._dims.buffer[res_ax] = self._dims.buffer[ax]; + sh._tags.buffer[res_ax] = self._tags.buffer[ax]; + sh._sharding = sh._sharding.set(res_ax, self._sharding.get(ax)); + res_ax += 1; + } + sh._dims.len = rk - to_remove.len; + sh._tags.len = rk - to_remove.len; + return sh; + } + + test removeMany { + try testing.expectEqualSlices( + i64, + &.{12}, + Shape.init(.{ 10, 11, 12 }, .f32).removeMany(.{ 0, 1 }).dims(), + ); + try testing.expectEqualSlices( + i64, + &.{ 10, 11 }, + Shape.init(.{ 10, 11, 12, 13 }, .f32).removeMany(.{ -1, -2 }).dims(), + ); } pub fn transpose(self: Shape, permutations: anytype) Shape { std.debug.assert(self.rank() == permutations.len); const permutations_ = self.axes(permutations); var res = self; + const original_dims = self._dims; + const original_tags = self._tags; + const original_parts = self._sharding; + for (permutations_.constSlice(), 0..) |permutation, i| { - res._dims.set(i, self.dim(permutation)); - res._tags.set(i, self.tag(permutation)); + res._dims.set(i, original_dims.get(permutation)); + res._tags.set(i, original_tags.get(permutation)); + res._sharding = res._sharding.set(i, original_parts.get(permutation)); } + return res; } @@ -561,6 +719,10 @@ pub const Shape = struct { Shape.init(.{ .a = 10, .b = 11, .c = 12, .d = 13 }, .f32).transpose(.{ 0, 2, 1, 3 }), ), ); + + const shape = Shape.init(.{ .a = 10, .c = 12, .b = 11 }, .f32).withPartitioning(&test_mesh, .{ .a = .batch, .c = .colors, .b = .open }).transpose(.{ 0, 2, 1 }); + try testing.expect(shape.eqlWithTags(Shape.init(.{ .a = 10, .b = 11, .c = 12 }, .f32))); + try testing.expectEqualDeep(Sharding.Partitioning.init(&.{ .mesh_axis_0, .open, .mesh_axis_2 }), shape._sharding.partition); } /// Tag each ax of this shape with tags from a tuple. @@ -573,23 +735,23 @@ pub const Shape = struct { var res = self; - if (comptime meta.isSliceOf(T, Tag) or meta.isSliceOf(T, EnumLiteral)) { - meta.assert(tagz.len == self.rank(), "Not enough tags for shape {}, got {any}", .{ self, tagz }); + if (comptime stdx.meta.isSliceOf(T, Tag) or stdx.meta.isSliceOf(T, @EnumLiteral())) { + stdx.debug.assert(tagz.len == self.rank(), "Not enough tags for shape {f}, got {any}", .{ self, tagz }); for (tagz, 0..) |tag_, i| { res._tags.set(i, toTag(tag_)); } return res; } - if (comptime meta.isTupleOf(T, Tag) or meta.isTupleOf(T, EnumLiteral)) { - meta.assert(tagz.len == self.rank(), "Not enough tags for shape {}, got {}", .{ self, tagz }); + if (comptime stdx.meta.isTupleOf(T, Tag) or stdx.meta.isTupleOf(T, @EnumLiteral())) { + stdx.debug.assert(tagz.len == self.rank(), "Not enough tags for shape {f}, got {}", .{ self, tagz }); inline for (tagz, 0..) |tag_, i| { res._tags.set(i, toTag(tag_)); } return res; } - meta.compileError("Expected a tuple of enum literals eg: .{ .a, .b, .c } got: {any}", .{@TypeOf(tagz)}); + stdx.debug.compileError("Expected a tuple of enum literals eg: .{{ .a, .b, .c }} got: {any}", .{@TypeOf(tagz)}); } test withTags { @@ -620,23 +782,23 @@ pub const Shape = struct { var res = self; - if (comptime meta.isSliceOf(T, Tag) or meta.isSliceOf(T, EnumLiteral)) { - meta.assert(tagz.len <= self.rank(), "Too many tags for shape {}, got {any}", .{ self, tagz }); + if (comptime stdx.meta.isSliceOf(T, Tag) or stdx.meta.isSliceOf(T, @EnumLiteral())) { + stdx.debug.assert(tagz.len <= self.rank(), "Too many tags for shape {f}, got {any}", .{ self, tagz }); for (tagz, self.rank() - tagz.len..) |tag_, i| { res._tags.set(i, toTag(tag_)); } return res; } - if (comptime meta.isTupleOf(T, Tag) or meta.isTupleOf(T, EnumLiteral)) { - meta.assert(tagz.len <= self.rank(), "Too many tags for shape {}, got {}", .{ self, tagz }); + if (comptime stdx.meta.isTupleOf(T, Tag) or stdx.meta.isTupleOf(T, @EnumLiteral())) { + stdx.debug.assert(tagz.len <= self.rank(), "Too many tags for shape {f}, got {}", .{ self, tagz }); inline for (tagz, self.rank() - tagz.len..) |tag_, i| { res._tags.set(i, toTag(tag_)); } return res; } - meta.compileError("Expected a tuple of enum literals eg: .{ .a, .b, .c } got: {any}", .{@TypeOf(tagz)}); + stdx.debug.compileError("Expected a tuple of enum literals eg: .{ .a, .b, .c } got: {any}", .{@TypeOf(tagz)}); } test withPartialTags { @@ -663,30 +825,231 @@ pub const Shape = struct { } } + pub fn assertTags(self: Shape, tagz: anytype) void { + const T = @TypeOf(tagz); + if (T == Shape) { + return self.assertTags(tagz.tags()); + } else if (T != []const Shape.Tag) { + return self.assertTags(comptime parseTags(tagz)); + } + + stdx.debug.assert(tagz.len == self.rank(), "Not enough tags for shape {f}, got {}", .{ self, stdx.fmt.stringsZ(tagz) }); + stdx.debug.assert(std.mem.eql(Shape.Tag, tagz, self.tags()), "Expected tags {f}, got {f}", .{ stdx.fmt.stringsZ(tagz), self }); + } + pub fn withDtype(self: Shape, dt: DataType) Shape { var res = self; res._dtype = dt; return res; } - pub fn withSharding(self: Shape, axes_: anytype) Shape { + // These tests only resolve logical names; placement does not use this fixture. + const test_mesh: Sharding.Mesh = .{ + .name = "test_mesh", + .physical = undefined, + .logical = .mesh(.{ .batch = .balanced, .feature = .balanced, .colors = .balanced }), + .bindings = .init(&.{ + .init(&.{.link_x}), + .init(&.{.link_y}), + .init(&.{.link_z}), + }), + .grid = undefined, + }; + + /// Returns shape metadata with the requested sharding on `mesh`. + /// Omitted dimensions in a mapping are replicated, including during compilation. + /// Use `Tensor.withPartitioning` to emit a constraint on a graph value. + pub fn withPartitioning(self: Shape, mesh: *const Sharding.Mesh, partitioning: anytype) Shape { + if (@TypeOf(partitioning) != Sharding.Partitioning) { + return self.withPartitioning(mesh, Sharding.Partitioning.parse(self.tags(), mesh, .replicated, partitioning)); + } var res = self; - // Reset sharding. - res._sharding_info = @splat(false); - for (self.axes(axes_).constSlice()) |ax| { - res._sharding_info[ax] = true; + res._sharding = .{ .mesh = mesh, .partition = partitioning }; + for (0..self.rank()) |ax| { + if (res._sharding.get(ax) == .out_of_bound) res._sharding = res._sharding.set(ax, .replicated); + } + for (self.rank()..MAX_RANK) |ax| res._sharding.partition = res._sharding.partition.set(ax, .out_of_bound); + + stdx.debug.assert(res._sharding.partition.hasUniqueAxes(), "{f}.withPartitioning({s}, ...) expects mesh axes to be used a most once, got {any}", .{ self, mesh.name, partitioning.toArray()[0..self.rank()] }); + + return res; + } + + test withPartitioning { + var shape = Shape.init(.{ .a = 10, .b = 20, .c = 30 }, .f32).withPartitioning(&test_mesh, .{ .c = .feature, .a = .batch }); + try testing.expectEqualSlices(Sharding.PartitionSpec, &.{ .sharded(0), .replicated, .sharded(1) }, shape._sharding.toArray()[0..shape.rank()]); + + shape = shape.withDefaultPartitioning(); + try testing.expectEqualSlices(Sharding.PartitionSpec, &.{ .replicated, .replicated, .replicated }, shape._sharding.toArray()[0..shape.rank()]); + + shape = shape.withPartitioning(&test_mesh, .{ .a = .batch, .b = .open, .c = .feature }); + try testing.expectEqualSlices(Sharding.PartitionSpec, &.{ .sharded(0), .open, .sharded(1) }, shape._sharding.toArray()[0..shape.rank()]); + } + + pub fn mapPartitioningAxes(self: Shape, mesh: *const Sharding.Mesh, mapping: anytype) Shape { + const T = @TypeOf(mapping); + stdx.debug.assertComptime(stdx.meta.isStruct(T), "Mapping must be a struct, e.g., .{ .old_axis = .new_axis }", .{}); + var res = self; + for (0..self.rank()) |ax| { + const mesh_axis = self.partition(ax).meshAxis() orelse continue; + inline for (comptime std.meta.fieldNames(T)) |field_name| { + const old_axis = mesh.resolveLogicalAxis(toTag(field_name)) orelse + stdx.debug.panic("Mesh '{s}' has no logical axis '{s}'", .{ mesh.name, field_name }); + if (mesh_axis == old_axis) { + const new_tag = toTag(@field(mapping, field_name)); + const new_axis = mesh.resolveLogicalAxis(new_tag) orelse + stdx.debug.panic("Mesh '{s}' has no logical axis '{s}'", .{ mesh.name, new_tag }); + res._sharding = res._sharding.set(ax, .sharded(@intCast(new_axis))); + break; + } + } } return res; } + test mapPartitioningAxes { + const shape = Shape.init(.{ .batch = 64, .seq = 128, .features = 512 }, .f32) + .withPartitioning(&test_mesh, .{ .batch = .batch, .seq = .replicated, .features = .feature }); + const mapped = shape.mapPartitioningAxes(&test_mesh, .{ .batch = .feature, .feature = .batch }); + try testing.expectEqual(Sharding.PartitionSpec.sharded(1), mapped.partition(.batch)); + try testing.expectEqual(Sharding.PartitionSpec.sharded(0), mapped.partition(.features)); + try testing.expectEqual(Sharding.PartitionSpec.replicated, mapped.partition(.seq)); + const partial = shape.mapPartitioningAxes(&test_mesh, .{ .batch = .colors }); + try testing.expectEqual(Sharding.PartitionSpec.sharded(2), partial.partition(.batch)); + try testing.expectEqual(shape.partition(.features), partial.partition(.features)); + } + + pub fn withDefaultPartitioning(self: Shape) Shape { + var res = self; + res._sharding.partition = .replicated(self.rank()); + return res; + } + + test withDefaultPartitioning { + var shape = Shape.init(.{ 10, 20, 30 }, .f32); + try testing.expectEqual(3, shape.rank()); + try testing.expectEqualSlices(Sharding.PartitionSpec, &.{ .replicated, .replicated, .replicated }, shape._sharding.toArray()[0..shape.rank()]); + + shape = shape.withPartitioning(&test_mesh, .{ ._1 = .batch }); + try testing.expectEqualSlices(Sharding.PartitionSpec, &.{ .replicated, .sharded(0), .replicated }, shape._sharding.toArray()[0..shape.rank()]); + + shape = shape.withDefaultPartitioning(); + try testing.expectEqual(3, shape.rank()); + try testing.expectEqualSlices(Sharding.PartitionSpec, &.{ .replicated, .replicated, .replicated }, shape._sharding.toArray()[0..shape.rank()]); + } + + pub fn withReplicatedPartitioning(self: Shape) Shape { + var res = self; + res._sharding.partition = .replicated(self._dims.len); + return res; + } + + test withReplicatedPartitioning { + var shape = Shape.init(.{ 10, 20, 30 }, .f32); + try testing.expectEqual(3, shape.rank()); + try testing.expectEqualSlices(Sharding.PartitionSpec, &.{ .replicated, .replicated, .replicated }, shape._sharding.toArray()[0..shape.rank()]); + + shape = shape.withPartitioning(&test_mesh, .{ ._1 = .batch }); + try testing.expectEqualSlices(Sharding.PartitionSpec, &.{ .replicated, .sharded(0), .replicated }, shape._sharding.toArray()[0..shape.rank()]); + + shape = shape.withReplicatedPartitioning(); + try testing.expectEqual(3, shape.rank()); + try testing.expectEqualSlices(Sharding.PartitionSpec, &.{ .replicated, .replicated, .replicated }, shape._sharding.toArray()[0..shape.rank()]); + } + + pub fn hasAtLeastOnePartitionedAxis(self: Shape) bool { + for (0..self.rank()) |ax| { + const spec = self._sharding.get(ax); + if (spec.isSharded()) return true; + } + return false; + } + + test hasAtLeastOnePartitionedAxis { + var shape = Shape.init(.{ .a = 10, .b = 20 }, .f32).withPartitioning(&test_mesh, .{ .a = .batch, .b = .replicated }); + try testing.expect(shape.hasAtLeastOnePartitionedAxis()); + + shape = shape.withPartitioning(&test_mesh, .{ .a = .replicated, .b = .replicated }); + try testing.expect(!shape.hasAtLeastOnePartitionedAxis()); + + shape = shape.withDefaultPartitioning(); + try testing.expect(!shape.hasAtLeastOnePartitionedAxis()); + } + + pub fn isFullyPartitioned(self: Shape) bool { + for (0..self.rank()) |ax| { + const spec = self._sharding.get(ax); + if (!spec.isSharded()) return false; + } + return true; + } + + test isFullyPartitioned { + var shape = Shape.init(.{ .a = 10, .b = 20 }, .f32).withPartitioning(&test_mesh, .{ .a = .batch, .b = .feature }); + try testing.expect(shape.isFullyPartitioned()); + + shape = shape.withPartitioning(&test_mesh, .{ .a = .batch, .b = .replicated }); + try testing.expect(!shape.isFullyPartitioned()); + + shape = shape.withDefaultPartitioning(); + try testing.expect(!shape.isFullyPartitioned()); + } + + pub fn isFullyReplicated(self: Shape) bool { + return self._sharding.isFullyReplicated(); + } + + test isFullyReplicated { + var shape = Shape.init(.{ .a = 10, .b = 20 }, .f32).withPartitioning(&test_mesh, .{ .a = .replicated, .b = .replicated }); + try testing.expect(shape.isFullyReplicated()); + + shape = shape.withPartitioning(&test_mesh, .{ .a = .batch, .b = .replicated }); + try testing.expect(!shape.isFullyReplicated()); + } + + pub fn partition(self: Shape, ax: anytype) Sharding.PartitionSpec { + return self._sharding.get(self.axis(ax)); + } + + test partition { + const shape = Shape.init(.{ .a = 10, .b = 20, .c = 30 }, .f32).withPartitioning(&test_mesh, .{ .a = .batch, .b = .open, .c = .feature }); + try testing.expectEqual(Sharding.PartitionSpec.sharded(0), shape.partition(.a)); + try testing.expectEqual(.open, shape.partition(.b)); + try testing.expectEqual(Sharding.PartitionSpec.sharded(1), shape.partition(.c)); + } + + pub fn containsPartitionSpec(self: Shape, part: Sharding.PartitionSpec) bool { + for (0..self.rank()) |ax| { + const dim_part = self._sharding.get(ax); + if (dim_part == part) return true; + } + return false; + } + + test containsPartitionSpec { + var shape = Shape.init(.{ .a = 10, .b = 20, .c = 30, .d = 40 }, .f32).withPartitioning(&test_mesh, .{ .a = .batch, .b = .open, .c = .feature }); + try testing.expect(shape.containsPartitionSpec(.sharded(0))); + try testing.expect(shape.containsPartitionSpec(.open)); + try testing.expect(shape.containsPartitionSpec(.replicated)); + try testing.expect(!shape.containsPartitionSpec(.out_of_bound)); + + shape = shape.withDefaultPartitioning(); + try testing.expect(!shape.containsPartitionSpec(.sharded(0))); + try testing.expect(!shape.containsPartitionSpec(.out_of_bound)); + try testing.expect(shape.containsPartitionSpec(.replicated)); + try testing.expect(!shape.containsPartitionSpec(.open)); + } + /// Renames some of the tags in this shape. /// Shape.init(.{ .a = 10, .b = 20 }).rename(.{ .b = .batch }); // .{ .a = 10, .batch = 20 }; pub fn rename(self: Shape, renames: anytype) Shape { const T = @TypeOf(renames); - meta.assertComptime(meta.isStructOfAny(T, isAxisConvertible), "Must pass a struct of enum literals. Passed: {any}", .{T}); + stdx.debug.assertComptime(stdx.meta.isStructOfAny(T, isAxisConvertible), "Must pass a struct of enum literals. Passed: {any}", .{T}); var res = self; - inline for (std.meta.fields(T)) |field| { - res._tags.set(self.axis(field), toTag(@field(renames, field.name))); + inline for (comptime std.meta.fieldNames(T)) |field_name| { + const new_field = @field(renames, field_name); + stdx.debug.assert(self.hasTag(new_field) == null, "{f}.rename({any}) failed because of duplicated axis {any}", .{ self, renames, new_field }); + res._tags.set(self.axis(field_name), toTag(new_field)); } return res; } @@ -705,16 +1068,124 @@ pub const Shape = struct { } } - pub fn computeStrides(self: Shape) std.BoundedArray(i64, MAX_RANK) { - const base_stride = self.dtype().sizeOf(); - const rk = self.rank(); - var strides: std.BoundedArray(i64, MAX_RANK) = .{ .len = @intCast(self.rank()) }; - if (rk == 0) return strides; - strides.buffer[rk - 1] = base_stride; - for (1..rk) |i| { - const j = @as(usize, rk) - 1 - i; - strides.buffer[j] = self._dims.get(j + 1) * strides.buffer[j + 1]; + /// Computes the strides of the shape in units of elements. + /// For a shape {d0, d1, d2}, the strides would be {d1*d2, d2, 1}. + pub fn computeElementStrides(self: Shape) stdx.BoundedArray(i64, constants.MAX_RANK) { + return self.computeStrides(1); + } + + test computeElementStrides { + { + const s = Shape.init(.{ 4, 1024 }, .bf16); + const strides = s.computeElementStrides(); + try std.testing.expectEqualSlices(i64, &.{ 1024, 1 }, strides.constSlice()); } + + { + const s = Shape.init(.{ 2, 3, 4 }, .f32); + const strides = s.computeElementStrides(); + try std.testing.expectEqualSlices(i64, &.{ 12, 4, 1 }, strides.constSlice()); + } + + { + const s = Shape.init(.{ 5, 6 }, .i16); + const strides = s.computeElementStrides(); + try std.testing.expectEqualSlices(i64, &.{ 6, 1 }, strides.constSlice()); + } + + { + const s = Shape.init(.{10}, .u8); + const strides = s.computeElementStrides(); + try std.testing.expectEqualSlices(i64, &.{1}, strides.constSlice()); + } + + { + const s = Shape.scalar(.f64); + const strides = s.computeElementStrides(); + try std.testing.expectEqualSlices(i64, &.{}, strides.constSlice()); + } + + { + const s = Shape.init(.{ 4, 1, 5 }, .i32); + const strides = s.computeElementStrides(); + try std.testing.expectEqualSlices(i64, &.{ 5, 5, 1 }, strides.constSlice()); + } + + { + const s = Shape.init(.{ .n = 8, .c = 3, .h = 224, .w = 224 }, .f32); + const strides = s.computeElementStrides(); + try std.testing.expectEqualSlices(i64, &.{ 3 * 224 * 224, 224 * 224, 224, 1 }, strides.constSlice()); + } + } + + /// Computes the strides of the shape in units of bytes. + /// For a shape {d0, d1, d2} of type T, the strides would be + /// {(d1*d2)*sizeof(T), d2*sizeof(T), 1*sizeof(T)}. + pub fn computeByteStrides(self: Shape) stdx.BoundedArray(i64, constants.MAX_RANK) { + return self.computeStrides(self.dtype().sizeOf()); + } + + test computeByteStrides { + { + const s = Shape.init(.{ 2, 3, 4 }, .f16); // 2 bytes + const strides = s.computeByteStrides(); + try std.testing.expectEqualSlices(i64, &.{ 12 * 2, 4 * 2, 1 * 2 }, strides.constSlice()); + } + + { + const s = Shape.init(.{ 4, 1024 }, .bf16); // 2 bytes + const strides = s.computeByteStrides(); + try std.testing.expectEqualSlices(i64, &.{ 1024 * 2, 1 * 2 }, strides.constSlice()); + } + + { + const s = Shape.init(.{ 5, 6 }, .c64); // 8 bytes (f32 + f32) + const strides = s.computeByteStrides(); + try std.testing.expectEqual(@sizeOf(std.math.Complex(f32)), 8); + try std.testing.expectEqualSlices(i64, &.{ 6 * 8, 1 * 8 }, strides.constSlice()); + } + + { + const s = Shape.init(.{ 128, 512 }, .i8); // 1 byte + const strides = s.computeByteStrides(); + try std.testing.expectEqualSlices(i64, &.{ 512, 1 }, strides.constSlice()); + } + + { + const s = Shape.init(.{ 1, 4096 }, .u32); // 4 bytes + const strides = s.computeByteStrides(); + try std.testing.expectEqualSlices(i64, &.{ 4096 * 4, 1 * 4 }, strides.constSlice()); + } + + { + const s = Shape.scalar(.f64); // 8 bytes + const strides = s.computeByteStrides(); + try std.testing.expectEqualSlices(i64, &.{}, strides.constSlice()); + } + } + + test "byte strides handle zero dimensions and maximum rank" { + const empty = Shape.init(.{ 2, 0, 5 }, .f32).computeByteStrides(); + try std.testing.expectEqualSlices(i64, &.{ 0, 20, 4 }, empty.constSlice()); + const full_rank = Shape.init(.{ 2, 3, 5, 7, 11, 13, 17, 19 }, .u8).computeByteStrides(); + try std.testing.expectEqualSlices(i64, &.{ 4849845, 1616615, 323323, 46189, 4199, 323, 19, 1 }, full_rank.constSlice()); + const large = Shape.init(.{std.math.maxInt(i64)}, .f64).computeByteStrides(); + try std.testing.expectEqualSlices(i64, &.{8}, large.constSlice()); + } + + fn computeStrides(self: Shape, element_size: i64) stdx.BoundedArray(i64, constants.MAX_RANK) { + @setRuntimeSafety(false); + const rk = self.rank(); + var strides: stdx.BoundedArray(i64, constants.MAX_RANK) = .{ .buffer = undefined, .len = rk }; + var stride = element_size; + var ax: usize = rk; + while (ax > 0) { + ax -= 1; + strides.buffer[ax] = stride; + // Axis 0 is not part of any stride; avoid computing the total size. + if (ax > 0) stride *= self._dims.buffer[ax]; + } + return strides; } @@ -722,7 +1193,7 @@ pub const Shape = struct { /// so that the given axes are contiguous. pub fn contiguousPerm(self: Shape, axes_: anytype) AxesArray { const axes__, _ = self.parseAxes(axes_); - var perms = AxesArray.init(0) catch unreachable; + var perms: AxesArray = .empty; for (0..self.rank()) |i| { if (std.mem.indexOfScalar(u3, axes__.constSlice(), @intCast(i))) |_| { continue; @@ -770,30 +1241,37 @@ pub const Shape = struct { var new_shape = self; new_shape._dims.replaceRange(ax, 1, dims_.constSlice()) catch unreachable; new_shape._tags.replaceRange(ax, 1, tags_.constSlice()) catch unreachable; - new_shape.inferMissingAxis(self.count()); + new_shape._sharding = new_shape._sharding.orderedRemove(ax); + + for (0..dims_.len) |_| { + new_shape._sharding = new_shape._sharding.insert(ax, .replicated); + } + + new_shape.inferMissingAxis(self.count()) catch |err| { + std.debug.panic("Can't split {any} along axis {d} into {any}: {t}", .{ self.dims(), ax, new_shape.dims(), err }); + }; + return new_shape; } test splitAxis { - try testing.expect( - Shape.init(.{ .a1 = 5, .a2 = 2, .b = 3 }, .f32).eql( - Shape.init(.{ .a = 10, .b = 3 }, .f32).splitAxis(.a, .{ .a1 = 5, .a2 = 2 }), - ), + try expectEqualShapes( + .init(.{ .a1 = 5, .a2 = 2, .b = 3 }, .f32), + .splitAxis(.init(.{ .a = 10, .b = 3 }, .f32), .a, .{ .a1 = 5, .a2 = 2 }), ); - try testing.expect( - Shape.init(.{ .a1 = 5, .a2 = 2, .b = 3 }, .f32).eql( - Shape.init(.{ .a = 10, .b = 3 }, .f32).splitAxis(.a, .{ .a1 = .auto, .a2 = 2 }), - ), + try expectEqualShapes( + .init(.{ .a1 = 5, .a2 = 2, .b = 3 }, .f32), + .splitAxis(.init(.{ .a = 10, .b = 3 }, .f32), .a, .{ .a1 = .auto, .a2 = 2 }), ); } pub fn splitAxes(self: Shape, axes_: anytype) Shape { const T = @TypeOf(axes_); - meta.assertComptime(meta.isStruct(T), "Must pass struct of enum literals like .{ .a = .{ .a1, .a2 } }. Passed: {any}", .{T}); + stdx.debug.assertComptime(stdx.meta.isStruct(T), "Must pass struct of enum literals like .{ .a = .{ .a1, .a2 } }. Passed: {any}", .{T}); var res = self; - inline for (std.meta.fields(T)) |field| { - res = res.splitAxis(field, @field(axes_, field.name)); + inline for (comptime std.meta.fieldNames(T)) |field_name| { + res = res.splitAxis(field_name, @field(axes_, field_name)); } return res; } @@ -814,22 +1292,38 @@ pub const Shape = struct { /// Merge the given axes into one axis. /// eg: `Shape.init(.{.a1 = 5, .a2 = 2, .b = 3}).merge(.{ .a = .{ .a1, .a2 }); -> .{ .a = 10, .b = 3 }` pub fn mergeAxis(self: Shape, axis_: anytype, axes_: anytype) Shape { - const axes__ = self.axes(axes_); + const axes_to_merge = self.axes(axes_); + stdx.debug.assert(axes_to_merge.len > 1, "Must merge at least two axes, got {any}", .{axes_}); - const first_axis = axes__.get(0); - const last_axis = axes__.get(axes__.len - 1); + // Merging partitioned axes is complex. If ANY axis being merged is partitioned, + // the resulting mesh is ambiguous. A propagation pass must resolve this. + // Marking it 'open' is the correct signal. If all are replicated/unknown, the + // result is also replicated/unknown. + // rule: .replicated wins over .out_of_bound. .open wins over everything. - var new_dim: i64 = 1; - for (axes__.constSlice(), first_axis..) |ax, counter| { - new_dim *= self.dim(ax); - meta.assert(ax == counter, "Can't merge shape {} along non-contiguous axes {any}", .{ self, axes_ }); + const first_axis_to_merge = axes_to_merge.get(0); + const num_axes_to_merge = axes_to_merge.len; + + var merged_dim: i64 = 1; + var merged_spec: Sharding.PartitionSpec = .out_of_bound; + for (axes_to_merge.constSlice()) |ax| { + merged_dim *= self.dim(ax); + const spec = self.partition(ax); + if (spec.isSharded() or spec == .open) { + merged_spec = .open; + } else if (spec == .replicated and merged_spec != .open) { + merged_spec = .replicated; + } } var new_shape = self; - new_shape._dims.set(first_axis, new_dim); - new_shape._dims.replaceRange(first_axis + 1, self.dims()[first_axis + 1 ..].len, self.dims()[last_axis + 1 ..]) catch unreachable; - new_shape._tags.set(first_axis, if (comptime isTagConvertible(@TypeOf(axis_))) toTag(axis_) else TagUnknown); - new_shape._tags.replaceRange(first_axis + 1, self.dims()[first_axis + 1 ..].len, self.tags()[last_axis + 1 ..]) catch unreachable; + for (1..num_axes_to_merge) |_| { + new_shape._sharding = new_shape._sharding.orderedRemove(first_axis_to_merge); + } + new_shape._sharding = new_shape._sharding.set(first_axis_to_merge, merged_spec); + new_shape._dims.replaceRange(first_axis_to_merge, num_axes_to_merge, &.{merged_dim}) catch @panic("mergeAxis failed on dims"); + new_shape._tags.replaceRange(first_axis_to_merge, num_axes_to_merge, &.{toTag(axis_)}) catch @panic("mergeAxis failed on tags"); + return new_shape; } @@ -861,14 +1355,36 @@ pub const Shape = struct { ); } + test "mergeAxis partitioning" { + // Merging a partitioned axis with a replicated one -> open + const shape1 = Shape.init(.{ .a = 5, .b = 2, .c = 3 }, .f32) + .withPartitioning(&test_mesh, .{ .a = .replicated, .b = .batch, .c = .replicated }); + const merged1 = shape1.mergeAxis(.ab, .{ .a, .b }); + try testing.expect(merged1.partition(.ab) == .open); + try testing.expect(merged1.dim(.ab) == 10); + try testing.expect(merged1.rank() == 2); + + // Merging two replicated axes -> replicated + const shape2 = Shape.init(.{ .a = 5, .b = 2, .c = 3 }, .f32) + .withPartitioning(&test_mesh, .{ .a = .replicated, .b = .replicated, .c = .feature }); + const merged2 = shape2.mergeAxis(.ab, .{ .a, .b }); + try testing.expect(merged2.partition(.ab) == .replicated); + try testing.expect(merged2.partition(.c) == Sharding.PartitionSpec.sharded(1)); + + // Merging two unknown axes -> unknown + const shape3 = Shape.init(.{ .a = 5, .b = 2, .c = 3 }, .f32).withDefaultPartitioning(); + const merged3 = shape3.mergeAxis(.ab, .{ .a, .b }); + try testing.expect(merged3.partition(.ab) == .replicated); + } + pub fn mergeAxes(self: Shape, axes_: anytype) Shape { const T = @TypeOf(axes_); - meta.assertComptime(meta.isStruct(T), "Must pass struct of enum literals like .{ .a = .{ .a1, .a2 } }. Passed: {any}", .{T}); + stdx.debug.assertComptime(stdx.meta.isStruct(T), "Must pass struct of enum literals like .{ .a = .{ .a1, .a2 } }. Passed: {any}", .{T}); var res = self; - inline for (std.meta.fields(T)) |field| { - meta.assertComptime(meta.isTupleOfAny(field.type, isAxisConvertible) or meta.isSliceOfAny(field.type, isAxisConvertible), "Must pass struct of axes. Passed: {any}", .{field.type}); - res = res.mergeAxis(field, @field(axes_, field.name)); + inline for (comptime std.meta.fieldNames(T), comptime std.meta.fieldTypes(T)) |field_name, Field| { + stdx.debug.assertComptime(stdx.meta.isTupleOfAny(Field, isAxisConvertible) or stdx.meta.isSliceOfAny(Field, isAxisConvertible), "Must pass struct of axes. Passed: {any}", .{Field}); + res = res.mergeAxis(field_name, @field(axes_, field_name)); } return res; } @@ -897,7 +1413,7 @@ pub const Shape = struct { } fn intersectTags(a: []const Tag, b: []const Tag) TagsArray { - var res = TagsArray.init(0) catch unreachable; + var res: TagsArray = .empty; for (a) |tag_| { if (std.mem.indexOfScalar(Tag, b, tag_)) { res.appendAssumeCapacity(tag_); @@ -906,34 +1422,34 @@ pub const Shape = struct { return res; } - pub fn parseStruct(T: type, v: anytype) struct { std.BoundedArray(T, MAX_RANK), TagsArray } { + pub fn parseStruct(T: type, v: anytype) struct { stdx.BoundedArray(T, constants.MAX_RANK), TagsArray } { const V = @TypeOf(v); - var vals_: std.BoundedArray(T, MAX_RANK) = .{}; - var tags_: TagsArray = .{}; + var vals_: stdx.BoundedArray(T, constants.MAX_RANK) = .empty; + var tags_: TagsArray = .empty; - if (comptime meta.isSliceOf(V, T)) { + if (comptime stdx.meta.isSliceOf(V, T)) { for (v) |d| { vals_.appendAssumeCapacity(d); } return .{ vals_, tags_ }; } - if (comptime meta.isStruct(V)) { - const fields = std.meta.fields(V); - meta.assertComptime(fields.len <= MAX_RANK, "Too many fields in struct {} ({d}). Max supported is {d}.", .{ V, fields.len, MAX_RANK }); - inline for (fields) |field| { - const fv = @field(v, field.name); + if (comptime stdx.meta.isStruct(V)) { + const field_names = comptime std.meta.fieldNames(V); + stdx.debug.assertComptime(field_names.len <= constants.MAX_RANK, "Too many fields in struct {} ({d}). Max supported is {d}.", .{ V, field_names.len, constants.MAX_RANK }); + inline for (field_names) |field_name| { + const fv = @field(v, field_name); vals_.appendAssumeCapacity(fv); - if (!comptime meta.isTuple(V)) { - tags_.appendAssumeCapacity(toTag(field)); + if (!comptime stdx.meta.isTuple(V)) { + tags_.appendAssumeCapacity(toTag(field_name)); } } return .{ vals_, tags_ }; } - meta.compileError("parseStruct expects struct or tuple, got {}", .{V}); + stdx.debug.compileError("parseStruct expects struct or tuple, got {}", .{V}); } test parseStruct { @@ -943,20 +1459,254 @@ pub const Shape = struct { try testing.expectEqualSlices(Tag, &.{ "a".ptr, "b".ptr }, tags_.constSlice()); } + /// Parses a struct literal into a list of options for each axes. + pub fn parseAxesOptions(self: Shape, T: type, options: anytype, default: T) stdx.BoundedArray(T, constants.MAX_RANK) { + const V = @TypeOf(options); + + var res: stdx.BoundedArray(T, constants.MAX_RANK) = .empty; + if (comptime stdx.meta.isSliceOf(V, T)) { + stdx.debug.assert(options.len == self.rank(), "expects exactly {} options in slice, for {} got {}", .{ self.rank(), self, options.len }); + for (options) |d| { + res.appendAssumeCapacity(d); + } + } + + if (comptime stdx.meta.isStruct(V)) { + for (0..self.rank()) |_| res.appendAssumeCapacity(default); + const field_names = comptime std.meta.fieldNames(V); + stdx.debug.assertComptime(field_names.len <= constants.MAX_RANK, "expects up to {} options struct literal, got {}", .{ V, constants.MAX_RANK, field_names.len }); + inline for (field_names) |field_name| { + const a = self.axis(field_name); + res.buffer[a] = @field(options, field_name); + } + return res; + } + + stdx.debug.compileError("parseStruct expects struct or tuple, got {}", .{V}); + } + + test parseAxesOptions { + const shape = Shape.init(.{ .a = 10, .b = 20, .c = 30 }, .u8); + const scaling = shape.parseAxesOptions(f32, .{ .b = 1.2, .a = 0.1 }, 1.0); + + try testing.expectEqualSlices(f32, &.{ 0.1, 1.2, 1.0 }, scaling.constSlice()); + } + test "comptimeShape" { comptime { const s = Shape.init(.{ .a = 5, .b = 6, .c = 7 }, .f32); - try std.testing.expectEqual(3, s.rank()); - try std.testing.expectEqual(4 * 5 * 6 * 7, s.byteSize()); - try std.testing.expectEqual(1, s.axis(.b)); + try testing.expectEqual(3, s.rank()); + try testing.expectEqual(4 * 5 * 6 * 7, s.byteSize()); + try testing.expectEqual(1, s.axis(.b)); } // comptime only the shape { const s = comptime Shape.init(.{ .a = 5, .b = 6, .c = 7 }, .f32); - try std.testing.expectEqual(3, s.rank()); - try std.testing.expectEqual(4 * 5 * 6 * 7, s.byteSize()); - try std.testing.expectEqual(1, s.axis(.b)); + try testing.expectEqual(3, s.rank()); + try testing.expectEqual(4 * 5 * 6 * 7, s.byteSize()); + try testing.expectEqual(1, s.axis(.b)); } } + + pub fn outer(self: Shape, other: Shape) Shape { + var res_shape = self; + var batching_axes: u8 = 0; + for (0..other.rank()) |ax| { + if (other.tag(ax) != Shape.TagUnknown) { + if (self.hasTag(other.tag(ax))) |batching_ax| { + stdx.debug.assert(batching_ax == batching_axes and batching_ax == ax, "outer expects batching dims to be the first dims in both tensors, got outer({f}, {f})", .{ self, other }); + batching_axes += 1; + } + } + + res_shape = res_shape.appendDim(other.dim(ax), other.tag(ax)); + } + return res_shape; + } + + pub fn iterator(self: Shape) MultiDimIterator { + return .{ + .shape = self, + .current_coords = @splat(0), + .flat_index = 0, + }; + } + + /// Generate the packed shape visible to PJRT + pub fn packedShape(sh: Shape) Shape { + const bit_size = sh.dtype().bitSizeOf(); + if (bit_size >= 8) return sh; + + const items_per_byte = @divExact(8, bit_size); + var res: Shape = sh.setDim(-1, @divExact(sh.dim(-1), items_per_byte)); + res._dtype = .u8; + return res; + } + + test packedShape { + const x: Shape = .init(.{ 4, 8 }, .u2); + try expectEqualShapes(.init(.{ 4, 2 }, .u8), x.packedShape()); + } + + test "shape transformations preserve partition specs and clear unused slots" { + var shape = Shape.init(.{ 2, 3, 4 }, .f32); + shape._sharding = .{ .mesh = &test_mesh, .partition = Sharding.Partitioning.out_of_bound.set(0, .mesh_axis_0).set(1, .open).set(2, .mesh_axis_1) }; + for ([_]Shape{ + shape.insert(1, .{5}), + shape.append(.{5}), + shape.remove(1), + shape.removeMany(.{ 0, 1 }), + shape.transpose(.{ 2, 0, 1 }), + shape.reshape(.{24}), + shape.splitAxis(1, .{ 1, 3 }), + shape.mergeAxis(.merged, .{ 0, 1 }), + shape.withDefaultPartitioning(), + }) |transformed| { + try testing.expectEqual(&test_mesh, transformed._sharding.mesh); + } + const parts = shape._sharding.partition; + try testing.expectEqual(parts.insert(1, .replicated), shape.insert(1, .{5})._sharding.partition); + try testing.expectEqual(parts.insert(1, .replicated), shape.insertTag(1, 5, .new)._sharding.partition); + try testing.expectEqual(parts.set(3, .replicated), shape.append(.{5})._sharding.partition); + try testing.expectEqual(parts.set(3, .replicated), shape.appendDim(5, null)._sharding.partition); + try testing.expectEqual(parts.orderedRemove(1), shape.remove(1)._sharding.partition); + const removed = shape.removeMany(.{ 0, 1 }); + try testing.expectEqual(Sharding.Partitioning.out_of_bound.set(0, .mesh_axis_1), removed._sharding.partition); + try testing.expectEqual(Sharding.PartitionSpec.replicated, removed.append(.{5}).partition(1)); + try testing.expectEqual(Sharding.Partitioning.out_of_bound.set(0, .mesh_axis_1).set(1, .mesh_axis_0).set(2, .open), shape.transpose(.{ 2, 0, 1 })._sharding.partition); + try testing.expectEqual(Sharding.Partitioning.out_of_bound.set(0, .mesh_axis_0).set(1, .replicated).set(2, .replicated).set(3, .mesh_axis_1), shape.splitAxis(1, .{ 1, 3 })._sharding.partition); + try testing.expectEqual(Sharding.Partitioning.replicated(3), shape.withDefaultPartitioning()._sharding.partition); + try testing.expectEqual(Sharding.Partitioning.replicated(3), shape.withReplicatedPartitioning()._sharding.partition); + } }; + +/// Iterates over all coordinates of a shape, yielding the multi-dimensional +/// coordinates and the corresponding flat index for each element. +pub const MultiDimIterator = struct { + shape: Shape, + current_coords: [Shape.MAX_RANK]i64, + flat_index: usize, + + pub const Item = struct { + coords: [Shape.MAX_RANK]i64, + rank: u4, + flat_index: usize, + }; + + pub fn next(self: *MultiDimIterator) ?Item { + if (self.flat_index >= self.shape.count()) { + return null; + } + + const rank = self.shape.rank(); + const item = Item{ + .coords = self.current_coords, + .rank = rank, + .flat_index = self.flat_index, + }; + + self.flat_index += 1; + + var i: usize = rank; + while (i > 0) { + i -= 1; + self.current_coords[i] += 1; + if (self.current_coords[i] < self.shape.dim(i)) { + return item; + } + self.current_coords[i] = 0; + } + + return item; + } +}; + +test "MultiDimIterator / Scalar" { + const shape = Shape.scalar(.f32); + var iter = shape.iterator(); + const item = iter.next().?; + + try std.testing.expectEqual(0, item.rank); + try std.testing.expectEqual(0, item.flat_index); + try std.testing.expect(iter.next() == null); +} + +test "MultiDimIterator / Empty" { + const shape = Shape.init(.{ .x = 5, .y = 0, .z = 2 }, .f32); + var iter = shape.iterator(); + + try std.testing.expect(iter.next() == null); +} + +test "MultiDimIterator / 1D shape" { + const shape = Shape.init(.{5}, .f32); + var iter = shape.iterator(); + var count: i64 = 0; + + while (iter.next()) |item| : (count += 1) { + try std.testing.expectEqual(count, item.coords[0]); + } + try std.testing.expectEqual(5, count); +} + +test "MultiDimIterator / 2D shape" { + const shape = Shape.init(.{ .rows = 2, .cols = 3 }, .f32); + var iter = shape.iterator(); + var items: std.ArrayList(MultiDimIterator.Item) = .empty; + defer items.deinit(std.testing.allocator); + + while (iter.next()) |item| { + try items.append(std.testing.allocator, item); + } + + try std.testing.expectEqual(6, items.items.len); + try std.testing.expectEqualSlices(i64, &.{ 0, 0 }, items.items[0].coords[0..2]); + try std.testing.expectEqualSlices(i64, &.{ 0, 1 }, items.items[1].coords[0..2]); + try std.testing.expectEqualSlices(i64, &.{ 0, 2 }, items.items[2].coords[0..2]); + try std.testing.expectEqualSlices(i64, &.{ 1, 0 }, items.items[3].coords[0..2]); + try std.testing.expectEqualSlices(i64, &.{ 1, 1 }, items.items[4].coords[0..2]); + try std.testing.expectEqualSlices(i64, &.{ 1, 2 }, items.items[5].coords[0..2]); + try std.testing.expectEqual(5, items.items[5].flat_index); +} + +test "MultiDimIterator / 3D shape" { + const shape = Shape.init(.{ .d = 2, .h = 2, .w = 2 }, .f32); + var iter = shape.iterator(); + const expected_coords = [_][3]i64{ + .{ 0, 0, 0 }, + .{ 0, 0, 1 }, + .{ 0, 1, 0 }, + .{ 0, 1, 1 }, + .{ 1, 0, 0 }, + .{ 1, 0, 1 }, + .{ 1, 1, 0 }, + .{ 1, 1, 1 }, + }; + var count: usize = 0; + + while (iter.next()) |item| : (count += 1) { + try std.testing.expectEqual(count, item.flat_index); + try std.testing.expectEqualSlices(i64, &expected_coords[count], item.coords[0..3]); + } + + try std.testing.expectEqual(8, count); +} + +test "MultiDimIterator / Shape with dimension of size 1" { + const shape = Shape.init(.{ .a = 2, .b = 1, .c = 3 }, .f32); + var iter = shape.iterator(); + var count: usize = 0; + + while (iter.next()) |item| : (count += 1) { + try std.testing.expectEqual(0, item.coords[1]); + } + try std.testing.expectEqual(6, count); +} + +fn expectEqualShapes(expected: Shape, actual: Shape) error{TestExpectedEqual}!void { + if (expected.eqlWithTags(actual)) return; + + std.debug.print("Expected {f}, got {f}", .{ expected, actual }); + return error.TestExpectedEqual; +} diff --git a/zml/slice.zig b/zml/slice.zig new file mode 100644 index 00000000..9451165b --- /dev/null +++ b/zml/slice.zig @@ -0,0 +1,541 @@ +const std = @import("std"); + +const stdx = @import("stdx"); + +const constants = @import("constants.zig"); +const DataType = @import("dtype.zig").DataType; +const floats = @import("floats.zig"); +const Shape = @import("shape.zig").Shape; +const builtin = @import("builtin"); + +pub fn isBytes(comptime T: type) bool { + const type_info = @typeInfo(T); + if (type_info != .pointer) return false; + if (type_info.pointer.child == u8) return true; + const child_type_info = @typeInfo(type_info.pointer.child); + if (child_type_info != .array) return false; + if (child_type_info.array.child != u8) return false; + return true; +} + +pub const Slice = struct { + bytes: []const u8, + mutable: bool, + shape: Shape, + offset_bytes: usize = 0, + byte_strides: stdx.BoundedArray(i64, constants.MAX_RANK), + + pub fn init(shape: Shape, bytes: anytype) Slice { + stdx.debug.assertComptime(isBytes(@TypeOf(bytes)), "Expected \"bytes\" to be a []u8 or []const u8, got {s}", .{@typeName(@TypeOf(bytes))}); + stdx.debug.assert(bytes.len == shape.byteSize(), "Expected \"bytes\" to have the same length as shape.byteSize() ({}), got {}", .{ shape.byteSize(), bytes.len }); + const type_info = @typeInfo(@TypeOf(bytes)); + const byte_strides = shape.computeByteStrides(); + return .{ + .bytes = bytes, + .mutable = !type_info.pointer.attrs.@"const", + .shape = shape, + .offset_bytes = 0, + .byte_strides = byte_strides, + }; + } + + pub fn initConst(shape: Shape, bytes: []const u8) Slice { + stdx.debug.assert(bytes.len == shape.byteSize(), "Expected \"bytes\" to have the same length as shape.byteSize() ({}), got {}", .{ shape.byteSize(), bytes.len }); + return .{ + .bytes = bytes, + .mutable = false, + .shape = shape, + .offset_bytes = 0, + .byte_strides = shape.computeByteStrides(), + }; + } + + pub fn alloc(allocator: std.mem.Allocator, shape_: Shape) !Slice { + const size = shape_.byteSize(); + const bytes: []u8 = switch (shape_.dtype().alignOf()) { + 1 => try allocator.alignedAlloc(u8, .@"1", size), + 2 => try allocator.alignedAlloc(u8, .@"2", size), + 4 => try allocator.alignedAlloc(u8, .@"4", size), + 8 => try allocator.alignedAlloc(u8, .@"8", size), + 16 => try allocator.alignedAlloc(u8, .@"16", size), + 32 => try allocator.alignedAlloc(u8, .@"32", size), + 64 => try allocator.alignedAlloc(u8, .@"64", size), + else => |v| stdx.debug.panic("Unsupported alignment: {}", .{v}), + }; + + return .{ + .bytes = bytes, + .mutable = true, + .shape = shape_, + .offset_bytes = 0, + .byte_strides = shape_.computeByteStrides(), + }; + } + + pub fn free(slice: Slice, allocator: std.mem.Allocator) void { + const d = slice.bytes; + // TODO: It would be better to store the aligment, this can be error prone in the presence of type casting + switch (slice.shape.dtype().alignOf()) { + 1 => allocator.free(@as([]align(1) const u8, @alignCast(d))), + 2 => allocator.free(@as([]align(2) const u8, @alignCast(d))), + 4 => allocator.free(@as([]align(4) const u8, @alignCast(d))), + 8 => allocator.free(@as([]align(8) const u8, @alignCast(d))), + 16 => allocator.free(@as([]align(16) const u8, @alignCast(d))), + 32 => allocator.free(@as([]align(32) const u8, @alignCast(d))), + 64 => allocator.free(@as([]align(64) const u8, @alignCast(d))), + else => |v| stdx.debug.panic("Unsupported alignment: {}", .{v}), + } + } + + pub fn dtype(slice: Slice) DataType { + return slice.shape.dtype(); + } + + pub fn isContiguous(slice: Slice) bool { + const expected = slice.shape.computeByteStrides(); + for (expected.slice(), slice.byte_strides.slice()) |exp, actual| { + if (exp != actual) return false; + } + return true; + } + + /// Returns the actual element-wise strides for this specific slice. + /// This respects transpositions and non-contiguous views. + pub fn elementStrides(self: Slice) stdx.BoundedArray(i64, constants.MAX_RANK) { + const element_size = @as(i64, @intCast(self.dtype().sizeOf())); + var res = self.byte_strides; + for (res.slice()) |*stride| { + // We divide the actual byte strides by the size of the data type + stride.* = @divExact(stride.*, element_size); + } + return res; + } + + pub fn subSlice(slice: Slice, axis: anytype, start: i64, size: i64) Slice { + const dim = slice.shape.dim(axis); + + stdx.debug.assert(start >= 0 and size >= 0 and start + size <= dim, "subSlice out of bounds: axis {d} start {d} size {d} dim {d}", .{ axis, start, size, dim }); + + const stride_bytes = slice.byte_strides.get(axis); + const new_offset = @as(i64, @intCast(slice.offset_bytes)) + start * stride_bytes; + stdx.debug.assert(new_offset >= 0, "subSlice produced negative offset: {d}", .{new_offset}); + + var res = slice; + res.offset_bytes = @intCast(new_offset); + res.shape = slice.shape.set(axis, size); + return res; + } + + pub fn squeeze(slice: Slice, axis_: anytype) Slice { + const axis = slice.shape.axis(axis_); + var res = slice; + res.shape = slice.shape.drop(axis); + _ = res.byte_strides.orderedRemove(@intCast(axis)); + return res; + } + + pub fn data(slice: Slice) []u8 { + std.debug.assert(slice.mutable); + if (builtin.mode == .debug) std.debug.assert(slice.isContiguous()); + const end = slice.offset_bytes + slice.shape.byteSize(); + return @constCast(slice.bytes[slice.offset_bytes..end]); + } + + pub fn constData(slice: Slice) []const u8 { + if (builtin.mode == .debug) std.debug.assert(slice.isContiguous()); + const end = slice.offset_bytes + slice.shape.byteSize(); + return slice.bytes[slice.offset_bytes..end]; + } + + pub fn items(slice: Slice, comptime T: type) []T { + stdx.debug.assertComptime(T == bool or @bitSizeOf(T) >= 8, "zml.Slice stores packed sub-bytes type so you need to pass a packed struct here like stdx.Packed({0}). Got: {0}", .{T}); + return @ptrCast(@alignCast(slice.data())); + } + + pub fn constItems(slice: Slice, comptime T: type) []const T { + stdx.debug.assertComptime(T == bool or @bitSizeOf(T) >= 8, "zml.Slice stores packed sub-bytes type so you need to pass a packed struct here like zml.Float4E2M1.Packed. Got: {}", .{T}); + return @ptrCast(@alignCast(slice.constData())); + } + + pub fn contiguousItemsIterator(slice: *const Slice, comptime T: type) ContiguousItemsIterator(T) { + std.debug.assert(slice.dtype() == DataType.fromZigType(T)); + return .init(slice); + } + + fn ContiguousItemsIterator(comptime T: type) type { + return struct { + slice: *const Slice, + contiguous_axis_start: u4, + contiguous_item_count: usize, + outer_index: usize, + outer_count: usize, + outer_coords: [constants.MAX_RANK]usize, + next_offset_bytes: usize, + + const Iterator = @This(); + + fn init(slice: *const Slice) Iterator { + const rank = slice.shape.rank(); + const total_count = slice.shape.count(); + var contiguous_axis_start: u4 = rank; + var contiguous_item_count: usize = 1; + var expected_stride: i64 = @intCast(@sizeOf(T)); + + var axis = rank; + while (axis > 0) { + axis -= 1; + const dim = slice.shape.dim(axis); + const stride = slice.byte_strides.get(axis); + if (dim != 1 and stride != expected_stride) break; + contiguous_axis_start = axis; + contiguous_item_count *= @intCast(dim); + expected_stride *= dim; + } + + if (total_count == 0) { + contiguous_item_count = 0; + } + + return .{ + .slice = slice, + .contiguous_axis_start = contiguous_axis_start, + .contiguous_item_count = contiguous_item_count, + .outer_index = 0, + .outer_count = if (contiguous_item_count == 0) 0 else total_count / contiguous_item_count, + .outer_coords = @splat(0), + .next_offset_bytes = slice.offset_bytes, + }; + } + + pub fn next(self: *Iterator) ?[]const T { + if (self.outer_index >= self.outer_count) return null; + + const start = self.next_offset_bytes; + self.outer_index += 1; + if (self.outer_index < self.outer_count) { + self.advanceOuterCoords(); + } + + const byte_len = self.contiguous_item_count * @sizeOf(T); + return @ptrCast(@alignCast(self.slice.bytes[start..][0..byte_len])); + } + + fn advanceOuterCoords(self: *Iterator) void { + var axis = self.contiguous_axis_start; + while (axis > 0) { + axis -= 1; + + self.outer_coords[axis] += 1; + self.next_offset_bytes = @intCast(@as(i64, @intCast(self.next_offset_bytes)) + self.slice.byte_strides.get(axis)); + + const dim: usize = @intCast(self.slice.shape.dim(axis)); + if (self.outer_coords[axis] < dim) return; + + self.outer_coords[axis] = 0; + self.next_offset_bytes = @intCast(@as(i64, @intCast(self.next_offset_bytes)) - @as(i64, @intCast(dim)) * self.slice.byte_strides.get(axis)); + } + } + }; + } + + pub fn format( + slice: @This(), + writer: *std.Io.Writer, + ) std.Io.Writer.Error!void { + return writer.print("Slice({f})@0x{x} [mut={}, off={d}, strides={any}]", .{ slice.shape, @intFromPtr(slice.bytes.ptr), slice.mutable, slice.offset_bytes, slice.byte_strides.slice() }); + } + + pub fn formatNumber(slice: Slice, writer: *std.Io.Writer, n: std.fmt.Number) std.Io.Writer.Error!void { + return slice.prettyPrintIndented(writer, 4, 0, n); + } + + pub fn prettyPrint(slice: Slice, writer: *std.Io.Writer, options: std.fmt.Number) !void { + return slice.prettyPrintIndented(writer, 4, 0, options); + } + + fn prettyPrintIndented(slice: Slice, writer: *std.Io.Writer, num_rows: u8, indent_level: u8, options: std.fmt.Number) !void { + if (slice.shape.rank() == 0) { + // Special case input tensor is a scalar + return switch (slice.dtype()) { + inline else => |dt| { + const single_ptr: *const dt.toZigType() = @ptrCast(@alignCast(slice.bytes.ptr)); + const val = single_ptr.*; + return switch (comptime dt.class()) { + // Since we have custom floats, we need to explicitly convert to float32 ourselves. + .float => stdx.fmt.formatFloat(floats.floatCast(f32, val), options, writer), + .integer => stdx.fmt.formatInt(val, options, writer), + .bool => stdx.fmt.formatBool(val, options, writer), + .complex => stdx.fmt.formatComplex(val, options, writer), + }; + }, + }; + } + + if (slice.shape.rank() == 1) { + // Print a contiguous slice of items from the buffer in one line. + // The number of items printed is controlled by the user through format syntax. + try writer.splatByteAll(' ', indent_level); + switch (slice.dtype()) { + inline else => |dt| { + const T = dt.toPackedZigType(); + const n = slice.shape.dim(0); + const elem_per_bytes: comptime_int = @max(1, 8 / comptime dt.bitSizeOf()); + const stride = @divExact(slice.byte_strides.get(0), @as(i64, @sizeOf(T))); + + // Fetch n next elements, accounting for strides and packing + const needed_len: usize = @as(usize, if (n == 0) 0 else @intCast(@abs((n - 1) * stride) + 1)) / elem_per_bytes; + + // The formatter consumes this physical tail using `stride`, so this + // rank-1 path can read raw items without requiring contiguity. + const raw_values: []const T = @ptrCast(@alignCast(slice.bytes[slice.offset_bytes..])); + const values = raw_values[0..needed_len]; + + if (comptime dt.bitSizeOf() < 8) { + return try stdx.fmt.formatSliceAny(values, options, stride, writer); + } + + switch (comptime dt.class()) { + .float => try stdx.fmt.formatFloatSlice(values, options, stride, writer), + .integer => try stdx.fmt.formatIntSlice(values, options, stride, writer), + .complex => try stdx.fmt.formatComplexSlice(values, options, stride, writer), + .bool => try stdx.fmt.formatBoolSlice(values, options, stride, writer), + } + }, + } + return; + } + // TODO: consider removing the \n if dim is 1 for this axis. + try writer.splatByteAll(' ', indent_level); + _ = try writer.write("{\n"); + + const n: u64 = @intCast(slice.shape.dim(0)); + if (n >= num_rows) { + // Write first rows + for (0..@min(num_rows, n)) |d| { + const byte_stride: usize = @intCast(slice.shape.computeByteStrides().get(0)); + const sub_slice: Slice = .init(slice.shape.drop(0), slice.constData()[d * byte_stride ..][0..byte_stride]); + try sub_slice.prettyPrintIndented(writer, num_rows, indent_level + 2, options); + try writer.writeAll(",\n"); + } + + // Skip middle rows + if (n > 2 * num_rows) { + try writer.splatByteAll(' ', indent_level + 2); + _ = try writer.write("...\n"); + } + // Write last rows + for (@max(n - num_rows, num_rows)..n) |d| { + const byte_stride: usize = @intCast(slice.shape.computeByteStrides().get(0)); + const sub_slice: Slice = .init(slice.shape.drop(0), slice.constData()[d * byte_stride ..][0..byte_stride]); + try sub_slice.prettyPrintIndented(writer, num_rows, indent_level + 2, options); + try writer.writeAll(",\n"); + } + } else { + for (0..n) |d| { + const byte_stride: usize = @intCast(slice.shape.computeByteStrides().get(0)); + const sub_slice: Slice = .init(slice.shape.drop(0), slice.constData()[d * byte_stride ..][0..byte_stride]); + try sub_slice.prettyPrintIndented(writer, num_rows, indent_level + 2, options); + try writer.writeAll(",\n"); + } + } + + try writer.splatByteAll(' ', indent_level); + _ = try writer.write("}"); + } + + pub fn copy(destination: Slice, source: []const u8) void { + const total_bytes = destination.shape.byteSize(); + stdx.debug.assert(source.len >= total_bytes, "Source buffer size ({}) smaller than destination shape ({})", .{ source.len, total_bytes }); + + if (total_bytes == 0) return; + + if (destination.isContiguous()) { + @memcpy(destination.data()[0..total_bytes], source[0..total_bytes]); + return; + } + + const rank = destination.shape.rank(); + stdx.debug.assert(rank > 0, "Non-contiguous slice with rank 0 is unexpected", .{}); + + const outer_len: usize = @intCast(destination.shape.dim(0)); + stdx.debug.assert(outer_len > 0, "Invalid dimension size: {d}", .{outer_len}); + + stdx.debug.assert(total_bytes % outer_len == 0, "Non-even split for contiguous source", .{}); + const outer_span_bytes = total_bytes / outer_len; + + for (0..outer_len) |i| { + const destination_outer = destination.subSlice(0, @intCast(i), 1).squeeze(0); + const source_outer_start = i * outer_span_bytes; + const source_outer_end = source_outer_start + outer_span_bytes; + destination_outer.copy(source[source_outer_start..source_outer_end]); + } + } +}; + +test "slice expectClose compares dense and strided slices" { + const testing = @import("testing.zig"); + + const dense_data: [2][2]f32 = .{ + .{ 1, 2 }, + .{ 5, 6 }, + }; + const strided_data: [2][4]f32 = .{ + .{ 1, 2, 99, 98 }, + .{ 5, 6, 97, 96 }, + }; + + const dense = Slice.init(.init(.{ 2, 2 }, .f32), std.mem.asBytes(&dense_data)); + const storage = Slice.init(.init(.{ 2, 4 }, .f32), std.mem.asBytes(&strided_data)); + const strided = storage.subSlice(1, 0, 2); + + try std.testing.expect(!strided.isContiguous()); + try testing.expectClose(std.testing.io, dense, strided, .exact_match); + try testing.expectClose(std.testing.io, strided, dense, .exact_match); +} + +test "slice expectClose applies close fraction across strided slices" { + const testing = @import("testing.zig"); + + const dense_data: [2][2]f32 = .{ + .{ 1, 2 }, + .{ 5, 6 }, + }; + const strided_data: [2][4]f32 = .{ + .{ 1, 3, 99, 98 }, + .{ 5, 6, 97, 96 }, + }; + + const dense = Slice.init(.init(.{ 2, 2 }, .f32), std.mem.asBytes(&dense_data)); + const storage = Slice.init(.init(.{ 2, 4 }, .f32), std.mem.asBytes(&strided_data)); + const strided = storage.subSlice(1, 0, 2); + const opts: testing.CompareOpts = .{ + .absolute_tolerance = 0, + .relative_tolerance = 0, + .minimum_close_fraction = 0.75, + }; + + try std.testing.expect(!strided.isContiguous()); + try testing.expectClose(std.testing.io, dense, strided, opts); + try testing.expectClose(std.testing.io, strided, dense, opts); +} + +test "slice pretty print rank 3" { + const data: [4][4][4]i32 = .{ + .{ + .{ 0, 1, 2, 3 }, + .{ 4, 5, 6, 7 }, + .{ 8, 9, 10, 11 }, + .{ 12, 13, 14, 15 }, + }, + .{ + .{ 16, 17, 18, 19 }, + .{ 20, 21, 22, 23 }, + .{ 24, 25, 26, 27 }, + .{ 28, 29, 30, 31 }, + }, + .{ + .{ 32, 33, 34, 35 }, + .{ 36, 37, 38, 39 }, + .{ 40, 41, 42, 43 }, + .{ 44, 45, 46, 47 }, + }, + .{ + .{ 48, 49, 50, 51 }, + .{ 52, 53, 54, 55 }, + .{ 56, 57, 58, 59 }, + .{ 60, 61, 62, 63 }, + }, + }; + const slice = Slice.init(.init(.{ 4, 4, 4 }, .i32), std.mem.asBytes(&data)); + const expected = + \\{ + \\ { + \\ {0,1,2,3}, + \\ {4,5,6,7}, + \\ {8,9,10,11}, + \\ {12,13,14,15}, + \\ }, + \\ { + \\ {16,17,18,19}, + \\ {20,21,22,23}, + \\ {24,25,26,27}, + \\ {28,29,30,31}, + \\ }, + \\ { + \\ {32,33,34,35}, + \\ {36,37,38,39}, + \\ {40,41,42,43}, + \\ {44,45,46,47}, + \\ }, + \\ { + \\ {48,49,50,51}, + \\ {52,53,54,55}, + \\ {56,57,58,59}, + \\ {60,61,62,63}, + \\ }, + \\} + ; + + try std.testing.expectFmt(expected, "{d}", .{slice}); +} + +test "slice pretty print rank 2" { + const data: [4]i32 = .{ 0, 1, 2, 3 }; + const slice = Slice.init(.init(.{ 1, 4 }, .i32), std.mem.asBytes(&data)); + const expected = + \\{ + \\ {0,1,2,3}, + \\} + ; + try std.testing.expectFmt(expected, "{d}", .{slice}); +} + +test "slice pretty print rank 1" { + const data: [4]i32 = .{ 0, 1, 2, 3 }; + const slice = Slice.init(.init(.{4}, .i32), std.mem.asBytes(&data)); + const expected = "{0,1,2,3}"; + try std.testing.expectFmt(expected, "{d}", .{slice}); +} + +test "slice pretty print rank 0" { + const data: [1]i32 = .{0}; + const slice = Slice.init(.init(.{}, .i32), std.mem.asBytes(&data)); + const expected = "0"; + try std.testing.expectFmt(expected, "{d}", .{slice}); +} + +test "slice pretty print bool" { + const data: [2]bool = .{ true, false }; + const slice = Slice.init(.init(.{2}, .bool), std.mem.asBytes(&data)); + const expected = "{1,0}"; + try std.testing.expectEqualSlices(bool, &data, slice.constItems(bool)); + try std.testing.expectFmt(expected, "{d}", .{slice}); +} + +test "slice pretty print ellipsis" { + const data: [9][1]i32 = .{ .{0}, .{1}, .{2}, .{3}, .{4}, .{5}, .{6}, .{7}, .{8} }; + const slice = Slice.init(.init(.{ 9, 1 }, .i32), std.mem.asBytes(&data)); + const expected = + \\{ + \\ {0}, + \\ {1}, + \\ {2}, + \\ {3}, + \\ ... + \\ {5}, + \\ {6}, + \\ {7}, + \\ {8}, + \\} + ; + try std.testing.expectFmt(expected, "{d}", .{slice}); +} + +test "slice pretty print f4" { + const x_f4_packed = [_]floats.Float4E2M1.Packed{ .fromF32(0, 0.5), .fromF32(1, 1.5), .fromF32(2, 3), .fromF32(4, 6) }; + const slice = Slice.initConst(.init(.{8}, .f4e2m1), @ptrCast(&x_f4_packed)); + + const expected = "{0,0.5,1,1.5,2,3,4,6}"; + + try std.testing.expectFmt(expected, "{d}", .{slice}); +} diff --git a/zml/tensor.zig b/zml/tensor.zig index 0d182fb7..ee084835 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -1,65 +1,53 @@ -const builtin = @import("builtin"); const std = @import("std"); -const assert = std.debug.assert; -const testing = std.testing; +const builtin = @import("builtin"); -const meta = @import("meta.zig"); -const mlir = @import("mlir.zig"); -const ops = @import("ops.zig"); -const module = @import("module.zig"); +const mlir = @import("mlir"); +const shardy = @import("mlir/dialects/shardy"); +const stablehlo = @import("mlir/dialects/stablehlo"); +const stdx = @import("stdx"); -const Location = mlir.Location; -const CompilationContext = module.CompilationContext; -const Shape = @import("shape.zig").Shape; -const Buffer = @import("buffer.zig").Buffer; -const HostBuffer = @import("hostbuffer.zig").HostBuffer; -const Data = @import("dtype.zig").Data; +const Compiler = @import("Compiler.zig"); +const constants = @import("constants.zig"); const DataType = @import("dtype.zig").DataType; +const mem = @import("mem.zig"); +const Memory = @import("platform.zig").Memory; +const meta = @import("meta.zig"); +const mlirx = @import("mlirx.zig"); +const ops = @import("ops.zig"); const Platform = @import("platform.zig").Platform; -const EnumLiteral = @TypeOf(.enum_literal); - -const dialect = struct { - const stablehlo = @import("mlir/dialects").stablehlo; -}; - -const scoped_log = std.log.scoped(.zml_tensor); +const Shape = @import("shape.zig").Shape; +const Sharding = @import("Sharding.zig"); test { + std.testing.refAllDecls(@This()); std.testing.refAllDecls(Tensor); } -/// Represents an abstract Tensor object, which can be the input, -/// output, weight or activations of a neural network. -/// Tensor are abstract in the sense they only represent a computation, -/// but not a specific memory buffer. -/// Tensor namespace contains most of linear algebra needed to -/// represent mathematical operations. -/// More operations are available in `zml.nn` and `zml.torch` namespaces. pub const Tensor = struct { + pub const Id = enum(u64) { _ }; + var current_id: std.atomic.Value(u64) = .init(1); + + id: Tensor.Id, + auto_broadcast: bool = false, _shape: Shape, - _id: _Id, - _donation: _Donation = .no_buffer, + _value: ?*const mlir.Value = null, - pub const _Donation = union(enum) { no_buffer, input_buffer, arg: u16 }; - pub const _Id = union(enum) { mlir: mlir.Value, buffer_id: u64, arg_id: u64 }; + const ResolvedAxis = u3; - pub const MAX_RANK = Shape.MAX_RANK; - - /// Returns the current compilation context. - pub fn getContext(self: Tensor) *CompilationContext { - _ = self; - return CompilationContext.current(); + pub fn init(shape_like: anytype, dt: DataType) Tensor { + return .fromShape(.init(shape_like, dt)); } - pub fn format( - self: Tensor, - comptime fmt: []const u8, - options: std.fmt.FormatOptions, - writer: anytype, - ) !void { - _ = fmt; - _ = options; - try writer.print("Tensor({_})", .{self._shape}); + fn nextTensorId() Id { + return @fromBackingInt(@intCast(Tensor.current_id.fetchAdd(1, .seq_cst))); + } + + pub fn fromShape(shape_: Shape) Tensor { + return .{ .id = nextTensorId(), ._shape = shape_ }; + } + + pub fn format(self: Tensor, writer: *std.Io.Writer) !void { + try writer.print("Tensor({f})", .{self._shape}); } /// Returns the shape of a Tensor. @@ -73,7 +61,7 @@ pub const Tensor = struct { } /// Returns the rank of a Tensor. - pub inline fn rank(self: Tensor) u4 { + pub fn rank(self: Tensor) u4 { return self._shape.rank(); } @@ -87,45 +75,55 @@ pub const Tensor = struct { return self._shape.byteSize(); } - /// Internal use - /// - /// Creates a tensor from a Shape and an mlir.Value. - pub fn _result(sh: Shape, val: mlir.Value) Tensor { - const res: Tensor = .{ - ._shape = sh, - ._id = .{ .mlir = val }, - }; + /// Internal: creates a graph tensor with dimensions, dtype and tags from `sh`, + /// and open sharding for the partitioner to infer from graph constraints. + pub fn _result(sh: Shape, val: *const mlir.Value) Tensor { + var res: Tensor = .{ ._shape = sh, ._value = val, .id = nextTensorId() }; + res._shape._sharding = .open; - if (builtin.mode == .Debug) { + if (builtin.mode == .debug) { // Check that the MLIR value actually have the same shape. const other = fromMlirValue(val); - meta.internalAssert(sh.eql(other._shape), "Created a {} from Mlir value but expected {}", .{ other._shape, res._shape }); + const eql_dtype = sh.dtype() == other.dtype(); + const eql_dims = std.mem.eql(i64, sh.dims(), other.dims()); + stdx.debug.internalAssert(eql_dims and eql_dtype, "Created a {f} from Mlir value but expected {f}", .{ other._shape, res._shape }); } return res; } - /// Creates a Tensor from a mlir.Value - /// - /// The shape is derived from the type of the mlir.Value. - pub fn fromMlirValue(val: mlir.Value) Tensor { - const ranked_tensor = val.getType().as(mlir.RankedTensorType).?; - const n = ranked_tensor.getRank(); + /// Internal: creates a constructor result and honors `sh`'s partitioning when + /// it specifies a mesh. + pub fn _resultPropagateSharding(sh: Shape, val: *const mlir.Value) Tensor { + const result = _result(sh, val); + const mesh = sh._sharding.mesh orelse return result; + return result.withPartitioning(mesh, sh._sharding.partition); + } - meta.assert(n <= MAX_RANK, "Can't represent MLIR tensor of rank {}, max supported rank is {}.", .{ n, MAX_RANK }); + /// Wraps an MLIR tensor value, deriving dimensions and dtype from its type. + /// Dimension tags are unknown and sharding is open for partitioner inference. + pub fn fromMlirValue(val: *const mlir.Value) Tensor { + const ctx = Compiler.current(); + const ranked_tensor = val.type_().isA(mlir.RankedTensorType).?; + const n = ranked_tensor.rank(); - var sh: Shape = .{ ._dtype = mlir.ext.Type.toDType(ranked_tensor.getElementType()) }; + stdx.debug.assert(n <= constants.MAX_RANK, "Can't represent MLIR tensor of rank {}, max supported rank is {}.", .{ n, constants.MAX_RANK }); + + var sh: Shape = .{ + ._dtype = ctx.dtype(ranked_tensor.elementType()), + ._dims = .empty, + ._tags = .{ .buffer = @splat(Shape.TagUnknown), .len = n }, + ._sharding = .open, + }; for (0..n) |i| { - sh._dims.appendAssumeCapacity(ranked_tensor.getDimension(i)); + sh._dims.appendAssumeCapacity(ranked_tensor.dimension(i)); } - sh._tags.resize(n) catch unreachable; - - return .{ ._shape = sh, ._id = .{ .mlir = val } }; + return .{ ._shape = sh, ._value = val, .id = nextTensorId() }; } /// Returns the dimension of axis 'axis_'. /// - /// 'axis_' can be a signed integer or a tag. + /// 'axis_' can be an integer or a tag. pub fn dim(self: Tensor, axis_: anytype) i64 { return self._shape.dim(axis_); } @@ -137,11 +135,18 @@ pub const Tensor = struct { /// Returns the index of axis 'axis_'. /// - /// 'axis_' can be a signed integer or a tag. - pub fn axis(self: Tensor, axis_: anytype) u3 { + /// 'axis_' can be an integer or a tag. + pub fn axis(self: Tensor, axis_: anytype) ResolvedAxis { return self._shape.axis(axis_); } + /// Returns the indices of each of the given axes. + /// + /// 'axis_' can be an integer or a tag. + pub fn axes(self: Tensor, axes_: anytype) stdx.BoundedArray(ResolvedAxis, constants.MAX_RANK) { + return self._shape.axes(axes_); + } + /// Returns a Tensor tagged with the tags in 'tagz'. pub fn withTags(self: Tensor, tagz: anytype) Tensor { var res = self; @@ -158,31 +163,244 @@ pub const Tensor = struct { return res; } - pub fn withSharding(self: Tensor, axes_: anytype) Tensor { - return switch (self._id) { - .arg_id, .mlir => { - const ctx = self.getContext(); - var res = self; - res._shape = self._shape.withSharding(axes_); + /// Declares this tensor's sharding on `mesh_`. + /// Pass a `*const Mesh`, or a mesh name such as `.tp` during compilation. + /// `partitioning` accepts a `Sharding.Partitioning` or a mapping from dimension + /// tags to logical mesh axes, `.replicated`, or `.open`. + /// + /// During compilation, emits a graph constraint. For example, + /// `x.withPartitioning(.tp, .{ .h = .model })` splits dimension `h` over `model` + /// and leaves omitted dimensions open for inference. Use `.replicated` to + /// keep a dimension whole on each shard. + /// + /// Outside compilation, updates the tensor's shape to describe an executable + /// input's sharding. Omitted dimensions are replicated and no graph constraint is + /// emitted. An explicit `Sharding.Partitioning` uses its supplied specs in both contexts. + /// + /// Executable inputs and explicitly annotated outputs must have no open + /// dimensions. An output with no declared sharding defaults to replication. + pub fn withPartitioning(self: Tensor, mesh_: anytype, partitioning: anytype) Tensor { + if (@TypeOf(mesh_) == @EnumLiteral()) { + const compiler = Compiler.currentOrNull() orelse @panic("Out side of compilation, withPartitioning expects an explicit *const zml.Mesh object as input"); + return self.withPartitioning(compiler.mesh(mesh_), partitioning); + } - const sharding = ctx.getShardingAttr(res._shape); + if (@TypeOf(partitioning) != Sharding.Partitioning) { + const default: Sharding.PartitionSpec = if (Compiler.currentOrNull() == null) .replicated else .open; + const parsed = Sharding.Partitioning.parse(self._shape.tags(), mesh_, default, partitioning); + return self.withPartitioning(mesh_, parsed); + } - const op = dialect.stablehlo.sharding( - ctx.mlirCtx(), - &.{self.value()}, - sharding, - &.{self.value().getType()}, - ctx.mlirCtx().location(@src()), - ); + const partitioned_shape = self._shape.withPartitioning(mesh_, partitioning); - return _result(res._shape, op.result(0)); + const ctx = Compiler.currentOrNull() orelse { + var res = self; + res._shape = partitioned_shape; + return res; + }; + + const attr = ctx.tensorShardingAttr(ctx.allocator, partitioned_shape) catch @panic("OOM"); + + const op_result = switch (ctx.partitioner) { + .shardy => blk: { + const op = shardy.sharding_constraint( + ctx.mlir_ctx, + self.value(), + attr.isA(shardy.TensorShardingAttr).?, + ctx.location, + ).appendTo(currentBlock()); + break :blk op.result(0); }, - .buffer_id => { - var res = self; - res._shape = self._shape.withSharding(axes_); - return res; + .gspmd => blk: { + const op = stablehlo.custom_call( + ctx.mlir_ctx, + &.{self.value()}, + &.{self.value().type_()}, + .{ + .call_target_name = "Sharding", + .has_side_effect = false, + .backend_config = .{ .original = "" }, + .additional_attributes = &.{ + .named(ctx.mlir_ctx, "mhlo.sharding", attr), + }, + }, + ctx.location, + ).appendTo(currentBlock()); + break :blk op.result(0); }, }; + + var res = _result(partitioned_shape, op_result); + res._shape = partitioned_shape; + return res; + } + + /// Emits a graph constraint requiring a complete copy on every device. + /// Requires active compilation. For a replicated input or buffer description, + /// use the default sharding from `Tensor.init` or `Shape.init`. + pub fn replicate(self: Tensor) Tensor { + const compiler = Compiler.current(); + const partitioning: Sharding.Partitioning = .repeat(.replicated, self.rank()); + return self.withPartitioning(compiler.platform.replicated_mesh, partitioning); + } + + test withPartitioning { + // Create two similar meshes, + // Then check that tensor.withPartitioning don't confuse the meshes + const dp_mp: Sharding.Mesh = .{ + .name = "dp_mp", + .physical = undefined, + .logical = .mesh(.{ .data = .low_bandwidth, .model = .high_bandwidth }), + .bindings = .init(&.{.init(&.{.link_x})}), + .grid = undefined, + }; + + const mp_dp: Sharding.Mesh = .{ + .name = "mp_dp", + .physical = undefined, + .logical = .mesh(.{ .model = .low_bandwidth, .data = .high_bandwidth }), + .bindings = .init(&.{.init(&.{.link_x})}), + .grid = undefined, + }; + + const x = Tensor.init(.{ .h = 8 }, .f32).withPartitioning(&dp_mp, .{ .h = .model }); + try std.testing.expectEqual(&dp_mp, x.shape()._sharding.mesh); + try std.testing.expectEqual(Sharding.PartitionSpec.sharded(1), x.shape().partition(.h)); + try std.testing.expectEqual(&dp_mp, x.shape().reshape(.{ 2, 4 })._sharding.mesh); + + const x2 = x.withPartitioning(&mp_dp, .{ .h = .model }); + try std.testing.expectEqual(&mp_dp, x2.shape()._sharding.mesh); + try std.testing.expectEqual(Sharding.PartitionSpec.sharded(0), x2.shape().partition(.h)); + try std.testing.expectEqual(&mp_dp, x2.shape().reshape(.{ 2, 4 })._sharding.mesh); + + const x2_replicated = x.withPartitioning(&mp_dp, .{ .h = .replicated }); + try std.testing.expectEqual(&mp_dp, x2_replicated.shape()._sharding.mesh); + try std.testing.expectEqual(Sharding.PartitionSpec.replicated, x2_replicated.shape().partition(.h)); + } + + /// Copy the given tensor to the specified memory, preserving its sharding. + pub fn toMemory(self: Tensor, kind: Memory.Kind) Tensor { + const ctx = Compiler.current(); + switch (ctx.platform.target) { + .cpu, .neuron, .metal => { + // A single memory needs no copy, but the result must still be a distinct + // tensor: `reuseBuffer` donations are keyed by tensor id. + var res = _result(self._shape, self.value()); + res._shape._sharding = self._shape._sharding; + return res; + }, + .cuda, .rocm, .tpu, .oneapi, .furiosa => {}, + } + + const frontend_attributes: *const mlir.Attribute = .dict(ctx.mlir_ctx, &.{ + .named( + ctx.mlir_ctx, + "_xla_buffer_placement", + .string(ctx.mlir_ctx, ctx.platform.memoryKind(kind)), + ), + }); + + const op = stablehlo.custom_call( + ctx.mlir_ctx, + &.{self.value()}, + &.{self.value().type_()}, + .{ + .call_target_name = "annotate_device_placement", + .has_side_effect = true, + .backend_config = .{ .original = "" }, + .additional_attributes = &.{ + .named(ctx.mlir_ctx, "mhlo.frontend_attributes", frontend_attributes), + }, + }, + ctx.location, + ).appendTo(currentBlock()); + + var res = _result(self._shape, op.result(0)); + res._shape._sharding = self._shape._sharding; + ctx.currentScope().id_to_memory.putNoClobber(ctx.currentScope().arena.allocator(), res.id, kind) catch @panic("OOM"); + return res; + } + + test toMemory { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + const io = std.testing.io; + + const inputs: [8]f32 = .{ -3.0, -2, -1, 1, 2, 3, 5, -5 }; + const x_t = Tensor.init(.{8}, .f32); + + const Local = struct { + fn memcpyH2D(x: Tensor) Tensor { + return x.onMemory(.host_pinned).toMemory(.device); + } + }; + + const exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local.memcpyH2D, .{x_t}, platform, .{}); + defer exe.deinit(); + + var x_h = try zml.Buffer.fromBytesOpts(io, platform, x_t.shape(), @ptrCast(&inputs), .{ .memory = .host_pinned }); + defer x_h.deinit(); + + const x_h_ptr: [*]f32 = @ptrCast(@alignCast(x_h.opaqueDevicePtr(0))); + try std.testing.expectEqualSlices(f32, &inputs, x_h_ptr[0..8]); + + var x_d = try exe.eval(std.testing.allocator, io, .{x_h}); + defer x_d.deinit(); + + try zml.testing.expectClose(std.testing.io, x_h, x_d, .exact_match); + } + + /// Copy all the given tensor to the specified memory. + /// The input struct is copied on the stack, so it must be a simple flat struct without pointers. + pub fn toMemoryAll(flat_tensors: anytype, kind: Memory.Kind) @TypeOf(flat_tensors) { + const ctx = Compiler.current(); + switch (ctx.platform.target) { + .cpu, .neuron, .metal => return flat_tensors, + .cuda, .rocm, .tpu, .oneapi, .furiosa => {}, + } + + var copy = flat_tensors; + meta.visitFlatStruct(struct { + fn toMemory(k: Memory.Kind, x: *Tensor) void { + x.* = x.toMemory(k); + } + }.toMemory, kind, ©); + return copy; + } + + /// Mark the given input tensor as being physically located on a specific memory. + /// Has no effect if the input tensor is not an executable input. + pub fn onMemory(self: Tensor, kind: Memory.Kind) Tensor { + const ctx = Compiler.current(); + switch (ctx.platform.target) { + .cpu, .neuron, .metal => return self, + .cuda, .rocm, .tpu, .oneapi, .furiosa => {}, + } + + if (ctx.currentScope().id_to_argument.get(self.id) == null) { + return self; + } + + ctx.currentScope().id_to_memory.put(ctx.currentScope().arena.allocator(), self.id, kind) catch unreachable; + + return self; + } + + /// Mark all the given input tensors as being physically located on a specific memory + /// see `zml.Tensor.onMemory` + pub fn onMemoryAll(tensors: anytype, kind: Memory.Kind) void { + const ctx = Compiler.current(); + switch (ctx.platform.target) { + // Only one memory kind on those platform + .cpu, .neuron, .metal => return, + .cuda, .rocm, .tpu, .oneapi, .furiosa => {}, + } + + meta.visit(struct { + fn onMemory(k: Memory.Kind, x: *const Tensor) void { + _ = x.onMemory(k); + } + }.onMemory, kind, &tensors); } /// Returns a Tensor with new tag names. @@ -192,7 +410,13 @@ pub const Tensor = struct { return res; } - pub fn renameAxis(self: Tensor, ax: i8, name: EnumLiteral) Tensor { + pub fn renameTag(self: Tensor, old: anytype, new: anytype) Tensor { + var res = self; + res._shape = self._shape.setTag(old, new); + return res; + } + + pub fn renameAxis(self: Tensor, ax: i8, name: @EnumLiteral()) Tensor { var res = self; res._shape._tags.set(self.axis(ax), @tagName(name).ptr); return res; @@ -200,44 +424,121 @@ pub const Tensor = struct { /// Returns the mlir.Value associated with the Tensor. /// - /// This will fail if used outside of a compilation context. - pub fn value(self: Tensor) mlir.Value { - return self.getContext().getValueAndDonation(self)[0]; + /// This will fail if used outside of a Compiler context. + pub fn value(self: Tensor) *const mlir.Value { + const scope = Compiler.current().currentScope(); + return scope.id_to_argument.get(self.id) orelse + self._value orelse + @panic("Something went really wrong, tensor is not an argument nor has an mlir.Value"); } + /// Tell PJRT compiler that memory should be reuse between the two tensors. /// The compiler is already aggressively reusing tensors for intermediate results, /// but this API allows to reuse buffer between input and output arguments /// of a given function. /// Note this is visible from the outside. The caller of a function with donations - /// is not allowed to reuse the donated input buffer after the call. + /// cannot use the donated input's contents after execution. Runners destroy its + /// handle automatically; raw `Exe.call` leaves it for the caller to deinit. /// For `reuseBuffer` to be effective, it needs to propagate all the way through the output. + /// + /// The sharding of the origin tensor is propagated to this one. pub fn reuseBuffer(self: Tensor, origin: Tensor) Tensor { - // Note: check donation docs, this may be too permissive. - meta.assert(self.byteSize() == origin.byteSize(), "Can't reuse buffers between tensors of different size: {} and {}", .{ self, origin }); + const compilation_context = Compiler.current(); + const scope = compilation_context.currentScope(); + if (scope.id_to_donation.get(origin.id)) |og_donation| { + // reuseBuffer is making the donation explicit + const donation: Compiler.Donation = switch (og_donation) { + .implicit => |input| .{ .explicit = input }, + .explicit => og_donation, + }; + scope.id_to_donation.put(scope.arena.allocator(), self.id, donation) catch @panic("OOM"); - // TODO: should we store all donations inside the context ? - var res = self; - res._donation = self.getContext().getValueAndDonation(origin)[1]; + // The origin is mapped to a physical buffer either because it's an input or through `reuseBuffer` + // We can now look at its sharding. + if (origin._shape.isFullyReplicated()) { + stdx.debug.assert(self.byteSize() == origin.byteSize(), ".reuseBuffer expects two tensors with same byteSize, got {f} ({B}) and {f} ({B})", .{ self, self.byteSize(), origin, origin.byteSize() }); + } else { + // Origin is sharded, we can't just use the global byte size, we actually need the same byte size per shard. + // We are stricter here and require the same shape. + stdx.debug.assert(self._shape.eql(origin._shape), ".reuseBuffer expects two sharded tensors with same shape, got {f} and {f}", .{ self, origin }); - return res; + // Normally self sharding is unspecified and we can just propagate origin sharding, + // but if the user called `.withPartitioning` explicitly it would be weird to override it silently, hence the check. + stdx.debug.assert(self._shape._sharding.isCompatible(origin._shape._sharding), ".reuseBuffer expects compatible shardings, got {f} and {f}", .{ self, origin }); + } + + var res = self; + res._shape._sharding = origin._shape._sharding; + return res; + } else { + // Origin tensor doesn't has any physical buffer associated so we can move on. + // This is allowed to allow a function like KVCache.update to call `reuseBuffer`, + // even though the caller may have constructed the inputs internally. + return self; + } } - var _global_tensor_counter: u64 = 0; + test reuseBuffer { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + const io = std.testing.io; - /// Internal use - pub fn reserveIdRange(len: u32) u64 { - return @atomicRmw(u64, &_global_tensor_counter, .Add, len, .seq_cst); - } + const inputs: [6]stdx.Packed(i4) = .{ .pack(-3.0, -2), .pack(-1, 1), .pack(2, 3), .pack(1, 2), .pack(3, 4), .pack(5, -5) }; + const x_t = Tensor.init(.{ 6, 2 }, .i4); + const y_t = Tensor.init(.{ 6, 2 }, .i4); + const z_t = Tensor.init(.{ 6, 2 }, .i4); - /// Internal use - pub fn setUniqueId(self: *Tensor) void { - self._id = .{ .buffer_id = reserveIdRange(1) }; + const Local = struct { + fn memcopy(x: Tensor, y: Tensor, z: Tensor) [2]Tensor { + // the first output reuses the given y, but the second ouput does NOT reuse z. + return .{ x.addConstant(1).reuseBuffer(y), z }; + } + }; + + const exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local.memcopy, .{ x_t, y_t, z_t }, platform, .{}); + defer exe.deinit(); + + var x_d = try zml.Buffer.fromBytes(io, platform, x_t.shape(), @ptrCast(&inputs)); + defer x_d.deinit(); + + var y_d = try zml.Buffer.uninitialized(io, platform, y_t.shape(), .{}); + defer y_d.deinit(); + + var z_d = try zml.Buffer.uninitialized(io, platform, z_t.shape(), .{}); + defer z_d.deinit(); + + var y_memory: [Platform.MAX_NUM_DEVICES]*anyopaque = undefined; + for (0..y_d.numShards()) |dev| { + y_memory[dev] = y_d.opaqueDevicePtr(dev); + } + + var runner = try exe.runner(std.testing.allocator); + defer runner.deinit(std.testing.allocator); + + for ([_]bool{ false, true }) |wait| { + var z2_d: zml.Buffer = undefined; + runner.run(io, .{ x_d, y_d, z_d }, .{ &y_d, &z2_d }, .{ .wait = wait }); + defer z2_d.deinit(); + try y_d.await(io); + + var output_memory: [Platform.MAX_NUM_DEVICES]*anyopaque = undefined; + for (0..y_d.numShards()) |dev| { + output_memory[dev] = y_d.opaqueDevicePtr(dev); + } + + // Check that the replacement y_d reuses the donated device memory. + try std.testing.expectEqualSlices(*anyopaque, y_memory[0..y_d.numShards()], output_memory[0..y_d.numShards()]); + + // Check that z_d and z2_d DO NOT POINT to the same addresses + for (0..z2_d.numShards()) |dev| { + try std.testing.expect(z2_d.opaqueDevicePtr(dev) != z_d.opaqueDevicePtr(dev)); + } + } } /// Returns a Tensor containing the absolute value of each element of the input Tensor. pub fn abs(self: Tensor) Tensor { - const loc = self.getContext().mlirCtx().location(@src()); - const op = dialect.stablehlo.abs(self.getContext().mlirCtx(), self.value(), loc); + const op = stablehlo.abs(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); const dt = switch (self.dtype()) { .c64 => .f32, .c128 => .f64, @@ -262,63 +563,49 @@ pub const Tensor = struct { break :gt res; } else lt: { // several contiguous elements of self maps to one element of the result - meta.assert(self.dim(-1) * src_bit_size == tgt_bit_size, "bitcast expects elements of the input tensor last dimension to map to one element of the target datatype, got {0} elements (bitsize of {0}x{1}={2}) and {3} (bitsize of {4})", .{ self.dim(-1), src_bit_size, self.dim(-1) * src_bit_size, dt, tgt_bit_size }); + stdx.debug.assert(self.dim(-1) * src_bit_size == tgt_bit_size, "bitcast expects elements of the input tensor last dimension to map to one element of the target datatype, got {0} elements (bitsize of {0}x{1}={2}) and {3} (bitsize of {4})", .{ self.dim(-1), src_bit_size, self.dim(-1) * src_bit_size, dt, tgt_bit_size }); break :lt self._shape.remove(-1); }; res_shape = res_shape.withDtype(dt); - const loc = self.getContext().mlirCtx().location(@src()).namedFmt(self.getContext().mlirCtx(), "bitCast({})", .{dt}); - const op = dialect.stablehlo.bitcast_convert( - self.getContext().mlirCtx(), + const op = stablehlo.bitcast_convert( + mlirCtx(), self.value(), - mlir.ext.RankedTensorType.fromShape(self.getContext().mlirCtx(), res_shape).as(mlir.Type).?, - loc, - ); + mlirx.Type.rankedTensor(mlirCtx(), res_shape), + currentLoc(), + ).appendTo(currentBlock()); return _result(res_shape, op.result(0)); } + /// Returns the given tensor as one contiguous buffer of bytes. + pub fn bytes(self: Tensor) Tensor { + return self.bitCast(.u8).flatten().withTags(.{.bytes}); + } + /// Returns a Tensor containing the element-wise number of leading 0 bits in the input Tensor. pub fn countLeadingZeros(self: Tensor) Tensor { - const loc = self.getContext().mlirCtx().location(@src()); - const op = dialect.stablehlo.count_leading_zeros(self.getContext().mlirCtx(), self.value(), loc); + const op = stablehlo.count_leading_zeros(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } /// Returns a Tensor containing booleans indicating if each element of the input Tensor is finite. pub fn isFinite(self: Tensor) Tensor { - const loc = self.getContext().mlirCtx().location(@src()); - const op = dialect.stablehlo.is_finite(self.getContext().mlirCtx(), self.value(), loc); + const op = stablehlo.is_finite(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape.withDtype(.bool), op.result(0)); } - /// Returns a Tensor containing the element-wise logistic operation on the input Tensor. - pub fn logistic(self: Tensor) Tensor { - const loc = self.getContext().mlirCtx().location(@src()); - const op = dialect.stablehlo.logistic(self.getContext().mlirCtx(), self.value(), loc); - return _result(self._shape, op.result(0)); - } - /// Returns a Tensor containing the element-wise number of bits set in the input Tensor. pub fn popcnt(self: Tensor) Tensor { - meta.assert(self.dtype().isInteger(), "popcnt expects tensor type to be an integer, got {}", .{self.dtype()}); - const loc = self.getContext().mlirCtx().location(@src()); - const op = dialect.stablehlo.popcnt(self.getContext().mlirCtx(), self.value(), loc); + stdx.debug.assert(self.dtype().isInteger(), "popcnt expects tensor type to be an integer, got {}", .{self.dtype()}); + const op = stablehlo.popcnt(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } /// Returns a Tensor containing the sign of the input Tensor element-wise. pub fn sign(self: Tensor) Tensor { - const loc = self.getContext().mlirCtx().location(@src()); - const op = dialect.stablehlo.sign(self.getContext().mlirCtx(), self.value(), loc); - return _result(self._shape, op.result(0)); - } - - /// Returns a Tensor containing the element-wise remainder of dividend 'self' and divisor 'other'. - pub fn remainder(self: Tensor, other: Tensor) Tensor { - const loc = self.getContext().mlirCtx().location(@src()); - const op = dialect.stablehlo.remainder(self.getContext().mlirCtx(), self.value(), other.value(), loc); + const op = stablehlo.sign(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } @@ -326,7 +613,7 @@ pub const Tensor = struct { /// /// See https://pytorch.org/docs/stable/generated/torch.fmod.html for more details. pub fn fmod(self: Tensor, divisor: f32) Tensor { - return self.remainder(Tensor.scalar(divisor, .f32).broadcast(self._shape, &.{})); + return self.remainder(scalar(divisor, .f32).broadcast(self._shape, &.{})); } test fmod { @@ -338,75 +625,85 @@ pub const Tensor = struct { const divisors: [2]f32 = .{ 2, -1.5 }; inline for (inputs, expectations, divisors) |i, e, d| { - const input = try zml.Buffer.fromSlice(platform, .{6}, &i); - const output = try zml.testing.compileAndCall(platform, Tensor.fmod, .{ input, d }); + const input: zml.Tensor = .init(.{6}, .f32); - try zml.testing.expectClose(zml.HostBuffer.fromSlice(.{6}, &e), output, 1e-4); + const exe = try zml.module.compile( + std.testing.allocator, + std.testing.io, + Tensor.fmod, + .{ input, d }, + platform, + .{}, + ); + defer exe.deinit(); + + var input_buffer = try zml.Buffer.fromBytes(std.testing.io, platform, input.shape(), std.mem.sliceAsBytes(&i)); + defer input_buffer.deinit(); + + var output = try exe.eval(std.testing.allocator, std.testing.io, .{input_buffer}); + defer output.deinit(); + + try zml.testing.expectClose(std.testing.io, zml.Slice.init(zml.Shape.init(.{6}, .f32), std.mem.sliceAsBytes(&e)), output, .{}); } } /// Returns a Tensor containing the element-wise left-shift operation of 'self' by 'other'. pub fn shiftLeft(self: Tensor, other: Tensor) Tensor { - return binaryOp("shiftLeft", dialect.stablehlo.shift_left)(self, other); + return binaryOp("shiftLeft", stablehlo.shift_left)(self, other); } /// Returns a Tensor containing the element-wise arithmetic right-shift operation of 'self' by 'other'. pub fn shiftRightArithmetic(self: Tensor, other: Tensor) Tensor { - return binaryOp("shiftRightArithmetic", dialect.stablehlo.shift_right_arithmetic)(self, other); + return binaryOp("shiftRightArithmetic", stablehlo.shift_right_arithmetic)(self, other); } /// Returns a Tensor containing the element-wise logical right-shift operation of 'self' by 'other'. pub fn shiftRightLogical(self: Tensor, other: Tensor) Tensor { - return binaryOp("shiftRightLogical", dialect.stablehlo.shift_right_logical)(self, other); + return binaryOp("shiftRightLogical", stablehlo.shift_right_logical)(self, other); } /// Returns the Cholesky decomposition of the input Tensor. /// - /// 'lower' controls the form of the outut Tensor. The output will be lower-triangular if 'lower' is true + /// 'lower' controls the form of the output Tensor. The output will be lower-triangular if 'lower' is true /// and upper-triangular otherwise. pub fn cholesky(self: Tensor, lower: bool) Tensor { - meta.assert(self.rank() <= 2, "cholesky expects tensor rank to be <= 2, got {}", .{self.rank()}); + stdx.debug.assert(self.rank() <= 2, "cholesky expects tensor rank to be <= 2, got {}", .{self.rank()}); - const loc = self.getContext().mlirCtx().location(@src()).namedFmt(self.getContext().mlirCtx(), "lower={}", .{lower}); - const op = dialect.stablehlo.cholesky(self.getContext().mlirCtx(), self.value(), lower, loc); + const op = stablehlo.cholesky(mlirCtx(), self.value(), lower, currentLoc()).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } /// Solves the system of linear equations formed by the input tensors. - pub fn triangularSolve(self: Tensor, other: Tensor, opts: dialect.stablehlo.TriangularSolveOpts) Tensor { - meta.assert(self.dtype() == other.dtype(), "triangularSolve expects tensors to be of the same type, got {} and {}", .{ self.dtype(), other.dtype() }); - meta.assert(self.rank() <= 2 and self.rank() == other.rank(), "triangularSolve expects tensors to have the same rank and be <= 2, got {} and {}", .{ self.rank(), other.rank() }); + pub fn triangularSolve(self: Tensor, other: Tensor, opts: stablehlo.TriangularSolveOpts) Tensor { + stdx.debug.assert(self.dtype() == other.dtype(), "triangularSolve expects tensors to be of the same type, got {} and {}", .{ self.dtype(), other.dtype() }); + stdx.debug.assert(self.rank() >= 2 and self.rank() == other.rank(), "triangularSolve expects tensors to have the same rank >= 2, got {} and {}", .{ self.rank(), other.rank() }); - const loc = self.getContext().mlirCtx().location(@src()).namedFmt(self.getContext().mlirCtx(), "opts={}", .{opts}); - const op = dialect.stablehlo.triangular_solve(self.getContext().mlirCtx(), self.value(), other.value(), loc, opts); - return _result(self._shape, op.result(0)); + const op = stablehlo.triangular_solve(mlirCtx(), self.value(), other.value(), opts, currentLoc()).appendTo(currentBlock()); + return _result(other._shape, op.result(0)); } /// Returns a Tensor containing the element-wise rounding towards the nearest integer, breaking ties away from zero, of the input Tensor. pub fn roundNearestAfz(self: Tensor) Tensor { - meta.assert(self.dtype().isFloat(), "roundNearestAfz expects tensor type to be a float, got {}", .{self.dtype()}); + stdx.debug.assert(self.dtype().isFloat(), "roundNearestAfz expects tensor type to be a float, got {}", .{self.dtype()}); - const loc = self.getContext().mlirCtx().location(@src()); - const op = dialect.stablehlo.round_nearest_afz(self.getContext().mlirCtx(), self.value(), loc); + const op = stablehlo.round_nearest_afz(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } /// Returns a Tensor containing the element-wise rounding towards the nearest integer, breaking ties towards the even integer, of the input Tensor. pub fn roundNearestEven(self: Tensor) Tensor { - meta.assert(self.dtype().isFloat(), "roundNearestEven expects tensor type to be a float, got {}", .{self.dtype()}); + stdx.debug.assert(self.dtype().isFloat(), "roundNearestEven expects tensor type to be a float, got {}", .{self.dtype()}); - const loc = self.getContext().mlirCtx().location(@src()); - const op = dialect.stablehlo.round_nearest_even(self.getContext().mlirCtx(), self.value(), loc); + const op = stablehlo.round_nearest_even(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } /// Returns a Tensor of complex number converted from a pair of real and imaginary Tensors. pub fn complex(re: Tensor, im: Tensor) Tensor { - meta.assert(re._shape.eql(im._shape), "complex expects tensor shapes to match, got {} and {}", .{ re._shape, im._shape }); - meta.assert(re.dtype() == .f32 or re.dtype() == .f64, "complex expects tensors type to be f32 or f64, got {}", .{re.dtype()}); + stdx.debug.assert(re._shape.eql(im._shape), "complex expects tensor shapes to match, got {f} and {f}", .{ re._shape, im._shape }); + stdx.debug.assert(re.dtype() == .f32 or re.dtype() == .f64, "complex expects tensors type to be f32 or f64, got {}", .{re.dtype()}); - const loc = re.getContext().mlirCtx().location(@src()); - const op = dialect.stablehlo.complex(re.getContext().mlirCtx(), re.value(), im.value(), loc); + const op = stablehlo.complex(mlirCtx(), re.value(), im.value(), currentLoc()).appendTo(currentBlock()); const dt: DataType = if (re.dtype() == .f32) .c64 else .c128; return _result(re._shape.withDtype(dt), op.result(0)); } @@ -415,7 +712,7 @@ pub const Tensor = struct { /// /// Tensor type can float or complex. pub fn real(self: Tensor) Tensor { - meta.assert(self.dtype().isComplex() or self.dtype().isFloat(), "real expects tensor type to be a float or a complex, got {}", .{self.dtype()}); + stdx.debug.assert(self.dtype().isComplex() or self.dtype().isFloat(), "real expects tensor type to be a float or a complex, got {}", .{self.dtype()}); if (self.dtype().isFloat()) { return self; @@ -426,8 +723,7 @@ pub const Tensor = struct { .c128 => .f64, else => unreachable, }; - const loc = self.getContext().mlirCtx().location(@src()); - const op = dialect.stablehlo.real(self.getContext().mlirCtx(), self.value(), loc); + const op = stablehlo.real(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape.withDtype(dt), op.result(0)); } @@ -435,11 +731,11 @@ pub const Tensor = struct { /// /// Tensor type can float or complex. pub fn imag(self: Tensor) Tensor { - meta.assert(self.dtype().isFloat() or self.dtype().isComplex(), "imag expects tensor type to be a float or a complex, got {}", .{self.dtype()}); + stdx.debug.assert(self.dtype().isFloat() or self.dtype().isComplex(), "imag expects tensor type to be a float or a complex, got {}", .{self.dtype()}); // Real tensors don't have imaginary part. if (self.dtype().isFloat()) { - return Tensor.constant(self._shape, self.dtype().zero()); + return Tensor.constant(self.dtype().zero()).broad(self._shape); } const dt: DataType = switch (self.dtype()) { @@ -448,69 +744,104 @@ pub const Tensor = struct { .c128 => .f64, else => unreachable, }; - const loc = self.getContext().mlirCtx().location(@src()); - const op = dialect.stablehlo.imag(self.getContext().mlirCtx(), self.value(), loc); + const op = stablehlo.imag(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape.withDtype(dt), op.result(0)); } /// Returns the Fast Fourier Transform (FFT) of the input Tensor. - pub fn fft(self: Tensor, opts: dialect.stablehlo.FftOpts) Tensor { + pub fn fft(self: Tensor, opts: stablehlo.FftOpts) Tensor { // TODO: support tagged API. - meta.assert(1 <= opts.length.len and opts.length.len <= 3, "fft expects 'opts.length' length to be between 1 and 3 (inclusive), got {}", .{opts.length.len}); - meta.assert(opts.length.len <= self.rank(), "fft expects 'opts.length' length to be less than tensor rank, got {} and {}", .{ opts.length.len, self.rank() }); + stdx.debug.assert(1 <= opts.length.len and opts.length.len <= 3, "fft expects 'opts.length' length to be between 1 and 3 (inclusive), got {}", .{opts.length.len}); + stdx.debug.assert(opts.length.len <= self.rank(), "fft expects 'opts.length' length to be less than tensor rank, got {} and {}", .{ opts.length.len, self.rank() }); const sh = switch (opts.kind) { .FFT, .IFFT => blk: { - meta.assert(self.dtype().isComplex(), "fft({any}) expects tensor type to be complex, got {}", .{ opts, self.dtype() }); + stdx.debug.assert(self.dtype().isComplex(), "fft({any}) expects tensor type to be complex, got {}", .{ opts, self.dtype() }); break :blk self._shape; }, .RFFT => blk: { - meta.assert(self.dtype() == .f32 or self.dtype() == .f64, "fft({}) expects tensor type to be f32 or f64, got {}", .{ opts, self.dtype() }); - meta.assert(std.mem.eql(i64, self.dims()[self.rank() - opts.length.len ..], opts.length), "fft({}) expects tensor last dimensions to match given lengths, got {} and {}", .{ opts, self.dims()[self.rank() - opts.length.len ..].len, opts.length.len }); + stdx.debug.assert(self.dtype() == .f32 or self.dtype() == .f64, "fft({}) expects tensor type to be f32 or f64, got {}", .{ opts, self.dtype() }); + stdx.debug.assert(std.mem.eql(i64, self.dims()[self.rank() - opts.length.len ..], opts.length), "fft({}) expects tensor last dimensions to match given lengths, got {} and {}", .{ opts, self.dims()[self.rank() - opts.length.len ..].len, opts.length.len }); const dt: DataType = switch (self.dtype()) { .f32 => .c64, else => .c128, }; - const shape_ = self._shape.setDim(-1, @divExact(self.dim(-1), 2) + 1); + const shape_ = self._shape.setDim(-1, rfftOutputLen(self.dim(-1))); break :blk shape_.withDtype(dt); }, .IRFFT => blk: { - meta.assert(self.dtype().isComplex(), "fft({any}) expects tensor type to be complex, got {}", .{ opts, self.dtype() }); - meta.assert(std.mem.eql(i64, self.dims()[self.rank() - opts.length.len ..], opts.length), "fft({any}) expects tensor last dimensions to match given lengths, got {} and {}", .{ opts, self.dims()[self.rank() - opts.length.len ..].len, opts.length.len }); + stdx.debug.assert(self.dtype().isComplex(), "fft({any}) expects tensor type to be complex, got {}", .{ opts, self.dtype() }); + const fft_dims = self.dims()[self.rank() - opts.length.len ..]; + const fft_last_dim_index = opts.length.len - 1; + stdx.debug.assert(std.mem.eql(i64, fft_dims[0..fft_last_dim_index], opts.length[0..fft_last_dim_index]), "fft({any}) expects non-innermost tensor FFT dimensions to match given lengths, got {any} and {any}", .{ opts, fft_dims[0..fft_last_dim_index], opts.length[0..fft_last_dim_index] }); + stdx.debug.assert(self.dim(-1) == rfftOutputLen(opts.length[fft_last_dim_index]), "fft({any}) expects IRFFT innermost dimension to match the RFFT output length, got {} for FFT length {}", .{ opts, self.dim(-1), opts.length[fft_last_dim_index] }); const dt: DataType = switch (self.dtype()) { .c64 => .f32, else => .f64, }; - const shape_ = self._shape.setDim(-1, @divExact(self.dim(-1) - 1, 2)); + const shape_ = self._shape.setDim(-1, opts.length[fft_last_dim_index]); break :blk shape_.withDtype(dt); }, }; - const loc = self.getContext().mlirCtx().location(@src()).namedFmt(self.getContext().mlirCtx(), "opts={}", .{opts}); - const op = dialect.stablehlo.fft(self.getContext().mlirCtx(), self.value(), loc, opts); + const op = stablehlo.fft(mlirCtx(), self.value(), opts, currentLoc()).appendTo(currentBlock()); return _result(sh, op.result(0)); } + fn rfftOutputLen(input_len: i64) i64 { + const rfft_len_divisor = 2; + const rfft_edge_bin_count = 1; + return @divTrunc(input_len, rfft_len_divisor) + rfft_edge_bin_count; + } + + test "Tensor.fft IRFFT restores RFFT last dimension" { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + const rfft_input_len: i64 = 10; + + const Local = struct { + pub fn _fwd(x: Tensor) Tensor { + const freq = x.fft(.{ .kind = .RFFT, .length = &.{rfft_input_len} }); + std.debug.assert(freq.dim(-1) == rfftOutputLen(rfft_input_len)); + const time = freq.fft(.{ .kind = .IRFFT, .length = &.{rfft_input_len} }); + std.debug.assert(time.dim(-1) == x.dim(-1)); + return time; + } + }; + + var exe = try zml.module.compile( + std.testing.allocator, + std.testing.io, + Local._fwd, + .{Tensor.init(.{rfft_input_len}, .f32)}, + platform, + .{}, + ); + defer exe.deinit(); + } + pub const Rng = struct { _state: Tensor, - algorithm: dialect.stablehlo.RngAlgorithm.Type = .DEFAULT, + algorithm: stablehlo.RngAlgorithmAttr.Type = .DEFAULT, - pub fn shape() ShapeOf(Rng) { + pub const Buffer = mem.Bufferized(Rng); + + pub fn init() Rng { + return .{ ._state = .init(.{2}, .u64) }; + } + + pub fn initBuffer(rng: Rng, io: std.Io, platform: *const Platform, seed: u128) !Buffer { return .{ - ._state = Shape.init(.{2}, .u64), + ._state = try .fromBytes(io, platform, rng._state._shape, std.mem.asBytes(&seed)), }; } - pub fn init(platform: Platform, seed: u128) !Bufferized(Rng) { - const bits: [2]u64 = @bitCast(seed); - return .{ - ._state = try Buffer.fromSlice(platform, Shape.init(.{2}, .u64), &bits), - .algorithm = undefined, - }; + pub fn deinitBuffer(self: *Buffer) void { + self._state.deinit(); } /// Returns a Tensor of the given shape, filled with uniform random bits, and a new Rng state. @@ -519,20 +850,18 @@ pub const Tensor = struct { /// The output is guaranteed to be deterministic function of `self` Rng state, /// but it is not guaranteed to be deterministic between implementations. pub fn bitGenerator(self: Rng, sh: Shape) struct { Rng, Tensor } { - const ctx = CompilationContext.current(); - const loc = ctx.mlirCtx().location(@src()).namedFmt(ctx.mlirCtx(), "rand.bitGen({})", .{sh}); - const op = dialect.stablehlo.rng_bit_generator( - ctx.mlirCtx(), + const op = stablehlo.rng_bit_generator( + mlirCtx(), self.algorithm, self._state.value(), - mlir.ext.mlirType(ctx.mlirCtx(), self._state._shape), - mlir.ext.mlirType(ctx.mlirCtx(), sh), - loc, - ); - return .{ self.update(op.result(0)), _result(sh, op.result(1)) }; + mlirx.Type.rankedTensor(mlirCtx(), self._state.shape()), + mlirx.Type.rankedTensor(mlirCtx(), sh), + currentLoc(), + ).appendTo(currentBlock()); + return .{ self.update(op.result(0)), _resultPropagateSharding(sh, op.result(1)) }; } - fn update(self: Rng, new_state: mlir.Value) Rng { + fn update(self: Rng, new_state: *const mlir.Value) Rng { return .{ ._state = _result(self._state._shape, new_state).reuseBuffer(self._state), .algorithm = self.algorithm, @@ -558,7 +887,7 @@ pub const Tensor = struct { 16 => .u16, 32 => .u32, 64 => .u64, - else => meta.panic("uniform don't support non-byte aligned dtype. Got: {}", .{shape_}), + else => stdx.debug.panic("uniform don't support non-byte aligned dtype. Got: {f}", .{shape_}), }; const rng, const bits = self.bitGenerator(shape_.withDtype(uint_dtype)); @@ -575,7 +904,8 @@ pub const Tensor = struct { floats = floats.mul(scalar(opts.max - opts.min, dt)).addConstant(opts.min); // Convert back to integer if needed. - return .{ rng, floats.convert(shape_.dtype()) }; + const result = floats.convert(shape_.dtype()); + return .{ rng, _resultPropagateSharding(shape_, result.value()) }; } test uniform { @@ -607,32 +937,38 @@ pub const Tensor = struct { const platform = zml.testing.env(); // Compute stats over a uniform distribution on [-2, 10]. - const rand, const stats = try zml.testing.compileAndCall( - platform, - Stats.uniformStats, - .{ - try Rng.init(platform, 1234), - Shape.init(.{1024}, .f32), - .{ .min = -2, .max = 10 }, - }, - ); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Stats.uniformStats, .{ Rng.init(), Shape.init(.{1024}, .f32), .{ .min = -2, .max = 10 } }, platform, .{}); + defer exe.deinit(); + + var rng_buffer = try Rng.init().initBuffer(std.testing.io, platform, 1234); + defer rng_buffer._state.deinit(); + + var stats: zml.Bufferized(Stats) = undefined; + var runner = try exe.runner(std.testing.allocator); + defer runner.deinit(std.testing.allocator); + runner.run(std.testing.io, .{rng_buffer}, .{ &rng_buffer, &stats }, .{ .wait = true }); + defer stats.mean.deinit(); + defer stats.variance.deinit(); + defer stats.min.deinit(); + defer stats.max.deinit(); + // Check the Rng state has been modified. - try std.testing.expect(try rand._state.getValue(i128) != 1234); + try std.testing.expect(try rng_buffer._state.getValue(u128, std.testing.io) != 1234); // Check the mean and variance are close to theoritical values. - const mean_ = try stats.mean.getValue(f32); + const mean_ = try stats.mean.getValue(f32, std.testing.io); try std.testing.expectApproxEqAbs(4, mean_, 0.03); - const variance = try stats.variance.getValue(f32); + const variance = try stats.variance.getValue(f32, std.testing.io); try std.testing.expectApproxEqAbs(12.0 * 12.0 / 12.0, variance, 0.01); // Check that no value is outside of the interval // and we have samples close to the edges. - const min_ = try stats.min.getValue(f32); + const min_ = try stats.min.getValue(f32, std.testing.io); try std.testing.expect(min_ >= -2); try std.testing.expectApproxEqAbs(-2, min_, 0.05); - const max_ = try stats.max.getValue(f32); + const max_ = try stats.max.getValue(f32, std.testing.io); try std.testing.expect(max_ < 10); try std.testing.expectApproxEqAbs(10, max_, 0.05); } @@ -642,15 +978,13 @@ pub const Tensor = struct { /// Note: this uses stablehlo.rng which is deprecated. /// https://github.com/openxla/stablehlo/blob/main/rfcs/20240503-opset-deprecations.md pub fn normal(sh: Shape, opts: struct { mean: f64 = 0, stddev: f64 = 1 }) Tensor { - meta.assert(sh.dtype().isFloat(), "normal expects tensor type to be a float, got {}", .{sh.dtype()}); + stdx.debug.assert(sh.dtype().isFloat(), "normal expects tensor type to be a float, got {}", .{sh.dtype()}); - const ctx = CompilationContext.current().mlirCtx(); - const loc = ctx.location(@src()).namedFmt(ctx, "rand.normal({}, opts={})", .{ sh, opts }); - const a = Tensor.constant(.{}, Data.init(sh.dtype(), opts.mean)); - const b = Tensor.constant(.{}, Data.init(sh.dtype(), opts.stddev)); - const res_shape = Tensor.constantTensor(HostBuffer.fromSlice(.{sh.rank()}, sh.dims())); - const op = dialect.stablehlo.rng(ctx, a.value(), b.value(), res_shape.value(), .NORMAL, loc); - return _result(sh, op.result(0)); + const a = Tensor.constant(DataType.Value.init(sh.dtype(), opts.mean)); + const b = Tensor.constant(DataType.Value.init(sh.dtype(), opts.stddev)); + const res_tensor_shape = Tensor.constantTensor(Shape.init(.{sh.rank()}, .i64), std.mem.sliceAsBytes(sh.dims())); + const op = stablehlo.rng(mlirCtx(), a.value(), b.value(), res_tensor_shape.value(), .NORMAL, currentLoc()).appendTo(currentBlock()); + return _resultPropagateSharding(sh, op.result(0)); } /// Returns a Tensor of the given shape, filled with floating point numbers sampled from a Gumbel distribution, and a new Rng state. @@ -663,11 +997,13 @@ pub const Tensor = struct { /// Note: we only implement the μ=0, β=1 version. pub fn gumbel(self: Rng, shape_: Shape) struct { Rng, Tensor } { const rand, const u = self.uniform( - shape_, + // Always use .f32 to have a big enough mantissa. + shape_.withDtype(.f32), // We don't want 0 to be sampled otherwise `log` will return -inf. - .{ .min = std.math.floatEps(f64), .max = 1 }, + .{ .min = std.math.floatEps(f32), .max = 1 }, ); - return .{ rand, u.log().scale(-1).log().scale(-1) }; + const result = u.log().scale(-1).log().scale(-1).convert(shape_.dtype()); + return .{ rand, _resultPropagateSharding(shape_, result.value()) }; } test gumbel { @@ -682,14 +1018,14 @@ pub const Tensor = struct { pub fn gumbelStats(rand: Rng, target_dist: Tensor) struct { Rng, Stats } { const s = Shape.init(.{ .n = 1024, .d = 4 }, .f32); const rng, const data = rand.gumbel(s); - const flat = data.flattenAll(); + const flat = data.flatten(); const mean_ = flat.mean(0); const variance = flat.sub(mean_.broad(flat.shape())).pow(Tensor.scalar(2, .f32)).mean(0); // Test out the gumbel reparametrization trick var x = target_dist.log().withTags(.{.d}).broad(s); x = x.add(data); - const samples = x.argMax(.d, .i32).indices.squeeze(.d); + const samples = x.argMax(.d).indices.squeeze(.d); // count 0, 1, 2 and 3 in samples: // - map 0 to 1, 1 to 2**16, 2 to 2**32, 3 to N**58 @@ -700,33 +1036,48 @@ pub const Tensor = struct { for (&powers, 0..) |*p, i| p.* = std.math.pow(u64, 2, i * 16); break :blk powers; }; - const values = Tensor.constantTensor(HostBuffer.fromArray(&powers)).withTags(.{.d}); - const counts = values.gatherValues(.d, samples, .{}).sum(.n).bitCast(.u16); + const values = Tensor.constantTensor(Shape.init(.{4}, .u64), std.mem.sliceAsBytes(&powers)).withTags(.{.d}); + const counts = values.gather(.{ .d = samples }, .{}).sum(.n).bitCast(.u16); const actual_dist = counts.reshape(target_dist.shape()).convert(target_dist.dtype()).divByConst(s.dim(.n)); return .{ rng, .{ .mean = mean_, .variance = variance, .actual_dist = actual_dist } }; } }; const platform = zml.testing.env(); - const tgt_dist = [_]f32{ 2.0, 1.0, 4.0, 3.0 }; - const rand, const stats = try zml.testing.compileAndCall(platform, Stats.gumbelStats, .{ - try Rng.init(platform, 1234), try HostBuffer.fromArray(&tgt_dist).toDevice(platform), - }); + + const tgt_dist_data = [_]f32{ 2.0, 1.0, 4.0, 3.0 }; + const tgt_dist: Tensor = .init(.{tgt_dist_data.len}, .f32); + + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Stats.gumbelStats, .{ Rng.init(), tgt_dist }, platform, .{}); + defer exe.deinit(); + + var rng_buffer = try Rng.init().initBuffer(std.testing.io, platform, 1234); + defer rng_buffer._state.deinit(); + var tgt_dist_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, tgt_dist.shape(), @ptrCast(&tgt_dist_data)); + defer tgt_dist_buffer.deinit(); + + var stats: zml.Bufferized(Stats) = undefined; + var runner = try exe.runner(std.testing.allocator); + defer runner.deinit(std.testing.allocator); + runner.run(std.testing.io, .{ rng_buffer, tgt_dist_buffer }, .{ &rng_buffer, &stats }, .{ .wait = true }); + defer stats.mean.deinit(); + defer stats.variance.deinit(); + defer stats.actual_dist.deinit(); + // Check the Rng state has been modified. - try std.testing.expect(try rand._state.getValue(i128) != 1234); + try std.testing.expect(try rng_buffer._state.getValue(i128, std.testing.io) != 1234); // Check the mean and variance are close to theoritical values. - const mean_ = try stats.mean.getValue(f32); + const mean_ = try stats.mean.getValue(f32, std.testing.io); try std.testing.expectApproxEqAbs(0.5772, mean_, 0.02); - const variance = try stats.variance.getValue(f32); + const variance = try stats.variance.getValue(f32, std.testing.io); const pi = std.math.pi; try std.testing.expectApproxEqAbs(pi * pi / 6.0, variance, 0.03); // Check the distribution obtained with the gumbel trick matches the target distribution. - const actual_dist = try stats.actual_dist.getValue([4]f32); - scoped_log.debug("tgt_dist: {d}, actual_dist: {d}", .{ tgt_dist, actual_dist }); - for (tgt_dist, actual_dist) |tgt, actual| { + const actual_dist = try stats.actual_dist.getValue([4]f32, std.testing.io); + for (tgt_dist_data, actual_dist) |tgt, actual| { // We normalize tgt_dist to make it a well formed distribution. // We didn't do it before calling gumbel, because the gumbel trick // doesn't require normalized distributions as input. @@ -737,66 +1088,65 @@ pub const Tensor = struct { /// Returns a Tensor containing the element-wise conversion to another floating point type. pub fn reducePrecision(self: Tensor, exponent_bits: i32, mantissa_bits: i32) Tensor { - meta.assert(self.dtype().isFloat(), "reducePrecision expects tensor type to be a float, got {}", .{self.dtype()}); - meta.assert(1 <= exponent_bits, "reducePrecision expects 'exponent_bits' to be >= 1, got {}", .{exponent_bits}); - meta.assert(0 <= mantissa_bits, "reducePrecision expects 'mantissa_bits' to be positive, got {}", .{mantissa_bits}); + stdx.debug.assert(self.dtype().isFloat(), "reducePrecision expects tensor type to be a float, got {}", .{self.dtype()}); + stdx.debug.assert(1 <= exponent_bits, "reducePrecision expects 'exponent_bits' to be >= 1, got {}", .{exponent_bits}); + stdx.debug.assert(0 <= mantissa_bits, "reducePrecision expects 'mantissa_bits' to be positive, got {}", .{mantissa_bits}); - const loc = self.getContext().mlirCtx().location(@src()).namedFmt(self.getContext().mlirCtx(), "reducePrecision(exponent_bits={}, mantissa_bits={})", .{ exponent_bits, mantissa_bits }); - const op = dialect.stablehlo.reduce_precision(self.getContext().mlirCtx(), self.value(), exponent_bits, mantissa_bits, loc); + const op = stablehlo.reduce_precision(mlirCtx(), self.value(), exponent_bits, mantissa_bits, currentLoc()).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } - inline fn convolution(self: Tensor, other: Tensor, opts: dialect.stablehlo.ConvolutionOpts, loc: mlir.Location) Tensor { - meta.assert(self.rank() == other.rank(), "convolution expects tensor ranks to match, got {} and {}", .{ self.rank(), other.rank() }); + fn convolution(self: Tensor, other: Tensor, opts: stablehlo.ConvolutionOpts) Tensor { + stdx.debug.assert(self.rank() == other.rank(), "convolution expects tensor ranks to match, got {} and {}", .{ self.rank(), other.rank() }); const N = self.rank(); - meta.guard(opts.window_strides.len == N - 2, @src()); - for (opts.window_strides) |s| meta.guard(0 < s, @src()); - meta.guard(opts.lhs_dilation.len == N - 2, @src()); - for (opts.lhs_dilation) |d| meta.guard(0 < d, @src()); - meta.guard(opts.rhs_dilation.len == N - 2, @src()); - for (opts.rhs_dilation) |d| meta.guard(0 < d, @src()); - meta.guard(opts.window_reversal.len == N - 2, @src()); - meta.guard(@rem(self.dim(opts.input_batch_dimension), opts.batch_group_count) == 0, @src()); - meta.guard(@rem(self.dim(opts.input_feature_dimension), opts.feature_group_count) == 0, @src()); - meta.guard(opts.input_spatial_dimensions.len == N - 2, @src()); - meta.guard(opts.input_batch_dimension != opts.input_feature_dimension, @src()); - meta.guard(0 <= opts.input_batch_dimension and opts.input_batch_dimension < N, @src()); - meta.guard(0 <= opts.input_feature_dimension and opts.input_feature_dimension < N, @src()); + stdx.debug.guard(opts.window_strides.len == N - 2, @src()); + for (opts.window_strides) |s| stdx.debug.guard(0 < s, @src()); + stdx.debug.guard(opts.lhs_dilation.len == N - 2, @src()); + for (opts.lhs_dilation) |d| stdx.debug.guard(0 < d, @src()); + stdx.debug.guard(opts.rhs_dilation.len == N - 2, @src()); + for (opts.rhs_dilation) |d| stdx.debug.guard(0 < d, @src()); + stdx.debug.guard(opts.window_reversal.len == N - 2, @src()); + stdx.debug.guard(@rem(self.dim(opts.input_batch_dimension), opts.batch_group_count) == 0, @src()); + stdx.debug.guard(@rem(self.dim(opts.input_feature_dimension), opts.feature_group_count) == 0, @src()); + stdx.debug.guard(opts.input_spatial_dimensions.len == N - 2, @src()); + stdx.debug.guard(opts.input_batch_dimension != opts.input_feature_dimension, @src()); + stdx.debug.guard(0 <= opts.input_batch_dimension and opts.input_batch_dimension < N, @src()); + stdx.debug.guard(0 <= opts.input_feature_dimension and opts.input_feature_dimension < N, @src()); for (opts.input_spatial_dimensions, 0..) |d, i| { - meta.guard(d != opts.input_batch_dimension, @src()); - meta.guard(d != opts.input_feature_dimension, @src()); - meta.guard(0 <= d and d < N, @src()); + stdx.debug.guard(d != opts.input_batch_dimension, @src()); + stdx.debug.guard(d != opts.input_feature_dimension, @src()); + stdx.debug.guard(0 <= d and d < N, @src()); if (i < opts.input_spatial_dimensions.len - 1) continue; - meta.guard(std.mem.indexOfScalar(i64, opts.input_spatial_dimensions[i + 1 ..], d) == null, @src()); + stdx.debug.guard(std.mem.indexOfScalar(i64, opts.input_spatial_dimensions[i + 1 ..], d) == null, @src()); } - meta.guard(other.dim(opts.kernel_input_feature_dimension) == @divTrunc(self.dim(opts.input_feature_dimension), opts.feature_group_count), @src()); - meta.guard(@rem(other.dim(opts.kernel_output_feature_dimension), opts.batch_group_count) == 0, @src()); - meta.guard(@rem(other.dim(opts.kernel_output_feature_dimension), opts.feature_group_count) == 0, @src()); - meta.guard(opts.kernel_spatial_dimensions.len == N - 2, @src()); - meta.guard(opts.kernel_input_feature_dimension != opts.kernel_output_feature_dimension, @src()); - meta.guard(0 <= opts.kernel_input_feature_dimension and opts.kernel_input_feature_dimension < N, @src()); - meta.guard(0 <= opts.kernel_output_feature_dimension and opts.kernel_output_feature_dimension < N, @src()); + stdx.debug.guard(other.dim(opts.kernel_input_feature_dimension) == @divTrunc(self.dim(opts.input_feature_dimension), opts.feature_group_count), @src()); + stdx.debug.guard(@rem(other.dim(opts.kernel_output_feature_dimension), opts.batch_group_count) == 0, @src()); + stdx.debug.guard(@rem(other.dim(opts.kernel_output_feature_dimension), opts.feature_group_count) == 0, @src()); + stdx.debug.guard(opts.kernel_spatial_dimensions.len == N - 2, @src()); + stdx.debug.guard(opts.kernel_input_feature_dimension != opts.kernel_output_feature_dimension, @src()); + stdx.debug.guard(0 <= opts.kernel_input_feature_dimension and opts.kernel_input_feature_dimension < N, @src()); + stdx.debug.guard(0 <= opts.kernel_output_feature_dimension and opts.kernel_output_feature_dimension < N, @src()); for (opts.kernel_spatial_dimensions, 0..) |d, i| { - meta.guard(d != opts.kernel_input_feature_dimension, @src()); - meta.guard(d != opts.kernel_output_feature_dimension, @src()); - meta.guard(0 <= d and d < N, @src()); + stdx.debug.guard(d != opts.kernel_input_feature_dimension, @src()); + stdx.debug.guard(d != opts.kernel_output_feature_dimension, @src()); + stdx.debug.guard(0 <= d and d < N, @src()); if (i < opts.kernel_spatial_dimensions.len - 1) continue; - meta.guard(std.mem.indexOfScalar(i64, opts.kernel_spatial_dimensions[i + 1 ..], d) == null, @src()); + stdx.debug.guard(std.mem.indexOfScalar(i64, opts.kernel_spatial_dimensions[i + 1 ..], d) == null, @src()); } - meta.guard(opts.output_spatial_dimensions.len == N - 2, @src()); - meta.guard(opts.output_batch_dimension != opts.output_feature_dimension, @src()); - meta.guard(0 <= opts.output_batch_dimension and opts.output_batch_dimension < N, @src()); - meta.guard(0 <= opts.output_feature_dimension and opts.output_feature_dimension < N, @src()); + stdx.debug.guard(opts.output_spatial_dimensions.len == N - 2, @src()); + stdx.debug.guard(opts.output_batch_dimension != opts.output_feature_dimension, @src()); + stdx.debug.guard(0 <= opts.output_batch_dimension and opts.output_batch_dimension < N, @src()); + stdx.debug.guard(0 <= opts.output_feature_dimension and opts.output_feature_dimension < N, @src()); for (opts.output_spatial_dimensions, 0..) |d, i| { - meta.guard(d != opts.output_batch_dimension, @src()); - meta.guard(d != opts.output_feature_dimension, @src()); - meta.guard(0 <= d and d < N, @src()); + stdx.debug.guard(d != opts.output_batch_dimension, @src()); + stdx.debug.guard(d != opts.output_feature_dimension, @src()); + stdx.debug.guard(0 <= d and d < N, @src()); if (i < opts.output_spatial_dimensions.len - 1) continue; - meta.guard(std.mem.indexOfScalar(i64, opts.output_spatial_dimensions[i + 1 ..], d) == null, @src()); + stdx.debug.guard(std.mem.indexOfScalar(i64, opts.output_spatial_dimensions[i + 1 ..], d) == null, @src()); } - meta.guard(0 < opts.feature_group_count, @src()); - meta.guard(0 < opts.batch_group_count, @src()); - meta.guard(opts.feature_group_count == 1 or opts.batch_group_count == 1, @src()); + stdx.debug.guard(0 < opts.feature_group_count, @src()); + stdx.debug.guard(0 < opts.batch_group_count, @src()); + stdx.debug.guard(opts.feature_group_count == 1 or opts.batch_group_count == 1, @src()); var used_opts = opts; used_opts.pad_shape = &.{ @intCast(N - 2), 2 }; used_opts.precision_config = &.{ .DEFAULT, .DEFAULT }; @@ -829,14 +1179,14 @@ pub const Tensor = struct { } // inferred shape '[1, 256, 1, 12008]' is incompatible with return type of operation '[1, 256, 1, 11978]' - const op = dialect.stablehlo.convolution( - self.getContext().mlirCtx(), + const op = stablehlo.convolution( + mlirCtx(), self.value(), other.value(), + mlirx.Type.rankedTensor(mlirCtx(), new_shape), used_opts, - mlir.ext.RankedTensorType.fromShape(self.getContext().mlirCtx(), new_shape).as(mlir.Type).?, - loc, - ); + currentLoc(), + ).appendTo(currentBlock()); return _result(new_shape, op.result(0)); } @@ -864,7 +1214,6 @@ pub const Tensor = struct { batch_group_count: i64 = 1, }, ) Tensor { - const loc = input.getContext().mlirCtx().location(@src()).namedFmt(input.getContext().mlirCtx(), "opts={}", .{opts}); return input.convolution(kernel, .{ .window_strides = &.{opts.window_strides}, .pad_value = opts.padding, @@ -882,7 +1231,7 @@ pub const Tensor = struct { .output_spatial_dimensions = &.{opts.output_spatial_dimensions}, .feature_group_count = opts.feature_group_count, .batch_group_count = opts.batch_group_count, - }, loc); + }); } /// Returns a Tensor containing the result of the 2D convolution of 'input' by 'kernel'. @@ -909,7 +1258,6 @@ pub const Tensor = struct { batch_group_count: i64 = 1, }, ) Tensor { - const loc = input.getContext().mlirCtx().location(@src()).namedFmt(input.getContext().mlirCtx(), "opts={}", .{opts}); return input.convolution(kernel, .{ .window_strides = opts.window_strides, .pad_value = opts.padding, @@ -927,57 +1275,77 @@ pub const Tensor = struct { .output_spatial_dimensions = opts.output_spatial_dimensions, .feature_group_count = opts.feature_group_count, .batch_group_count = opts.batch_group_count, - }, loc); + }); } /// Returns a Tensor containing the element-wise addition of the input Tensors. pub fn add(self: Tensor, other: Tensor) Tensor { - return binaryOp("add", dialect.stablehlo.add)(self, other); + return binaryOp("add", stablehlo.add)(self, other); } /// Returns a Tensor containing the element-wise subtraction of the input Tensors. pub fn sub(self: Tensor, other: Tensor) Tensor { - return binaryOp("subtract", dialect.stablehlo.subtract)(self, other); + return binaryOp("subtract", stablehlo.subtract)(self, other); } /// Returns a Tensor containing the element-wise multiplication of the input Tensors. pub fn mul(self: Tensor, other: Tensor) Tensor { - return binaryOp("mul", dialect.stablehlo.multiply)(self, other); + return binaryOp("mul", stablehlo.multiply)(self, other); } /// Returns a Tensor containing the element-wise division of the input Tensors. pub fn div(self: Tensor, other: Tensor) Tensor { - return binaryOp("div", dialect.stablehlo.divide)(self, other); + return binaryOp("div", stablehlo.divide)(self, other); } /// Returns a Tensor containing the element-wise exponentiation of the input Tensors. pub fn pow(self: Tensor, other: Tensor) Tensor { - return binaryOp("pow", dialect.stablehlo.power)(self, other); + return binaryOp("pow", stablehlo.power)(self, other); } /// Returns a Tensor containing the element-wise maximum operation of the input Tensors. pub fn maximum(self: Tensor, other: Tensor) Tensor { - return binaryOp("maximum", dialect.stablehlo.maximum)(self, other); + return binaryOp("maximum", stablehlo.maximum)(self, other); } /// Returns a Tensor containing the element-wise minimum operation of the input Tensors. pub fn minimum(self: Tensor, other: Tensor) Tensor { - return binaryOp("minimum", dialect.stablehlo.minimum)(self, other); + return binaryOp("minimum", stablehlo.minimum)(self, other); + } + + /// Returns a Tensor containing the element-wise remainder of dividend 'self' and divisor 'other'. + pub fn remainder(self: Tensor, other: Tensor) Tensor { + return binaryOp("remainder", stablehlo.remainder)(self, other); + } + + /// Returns a Tensor containing the element-wise substraction of the input Tensor with a constant. + pub fn remainderConst(self: Tensor, b: anytype) Tensor { + return self.remainder(.scalar(b, self.dtype())); } /// Returns a Tensor containing the element-wise addition of the input Tensor with a constant. pub fn addConstant(self: Tensor, b: anytype) Tensor { - return self.add(Tensor.scalar(b, self.dtype())); + return self.add(.scalar(b, self.dtype())); + } + + /// Returns a Tensor containing the element-wise substraction of the input Tensor with a constant. + pub fn subConstant(self: Tensor, b: anytype) Tensor { + return self.sub(.scalar(b, self.dtype())); } /// Returns a Tensor containing the element-wise division of the input Tensor by a constant. pub fn divByConst(self: Tensor, b: anytype) Tensor { - return self.div(Tensor.scalar(b, self.dtype())); + return self.div(.scalar(b, self.dtype())); + } + + /// Returns a Tensor containing the element-wise power of the input Tensor by a constant. + pub fn powByConst(self: Tensor, b: anytype) Tensor { + return self.pow(.scalar(b, self.dtype())); } /// Returns a Tensor containing the element-wise multiplication of the input Tensor by a constant. - pub inline fn scale(self: Tensor, val: anytype) Tensor { - return self.mul(Tensor.scalar(val, self.dtype())); + pub fn scale(self: Tensor, val: anytype) Tensor { + return self.mul(.scalar(val, self.dtype())); } pub const LogicalOp = enum { OR, XOR, AND }; @@ -985,128 +1353,269 @@ pub const Tensor = struct { /// Returns a Tensor containing the element-wise logical operation of the input Tensors. pub fn logical(self: Tensor, comptime logical_op: LogicalOp, other: Tensor) Tensor { return switch (logical_op) { - .OR => binaryOp("or", dialect.stablehlo.or_)(self, other), - .XOR => binaryOp("xor", dialect.stablehlo.xor)(self, other), - .AND => binaryOp("and", dialect.stablehlo.and_)(self, other), + .OR => binaryOp("or", stablehlo.or_)(self, other), + .XOR => binaryOp("xor", stablehlo.xor)(self, other), + .AND => binaryOp("and", stablehlo.and_)(self, other), }; } /// Returns a Tensor containing the element-wise floor operation of the input Tensor. pub fn floor(self: Tensor) Tensor { - const loc = self.getContext().mlirCtx().location(@src()); - return _result(self._shape, dialect.stablehlo.floor(self.getContext().mlirCtx(), self.value(), loc).result(0)); + return _result(self._shape, stablehlo.floor(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()).result(0)); } /// Returns a Tensor containing the element-wise ceil operation of the input Tensor. pub fn ceil(self: Tensor) Tensor { - const loc = self.getContext().mlirCtx().location(@src()); - return _result(self._shape, dialect.stablehlo.ceil(self.getContext().mlirCtx(), self.value(), loc).result(0)); + return _result(self._shape, stablehlo.ceil(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()).result(0)); } /// Returns a Tensor containing the element-wise conversion to another type. - pub fn convert(self: Tensor, dt: DataType) Tensor { - if (dt == self.dtype()) { + pub fn convert(self: Tensor, to: DataType) Tensor { + if (to == self.dtype()) { return self; } - const res_type = mlir.RankedTensorType.init(self.dims(), mlir.ext.Type.fromDType(self.getContext().mlirCtx(), dt)).as(mlir.Type).?; - const loc = self.getContext().mlirCtx().location(@src()).namedFmt(self.getContext().mlirCtx(), "dtype={}", .{dt}); + const res_type = mlirx.Type.rankedTensor(mlirCtx(), self.shape().withDtype(to)); + const op = stablehlo.convert(mlirCtx(), self.value(), res_type, currentLoc()).appendTo(currentBlock()); + return _result(self._shape.withDtype(to), op.result(0)); + } - const op = dialect.stablehlo.convert(self.getContext().mlirCtx(), self.value(), res_type, loc); - return _result(self._shape.withDtype(dt), op.result(0)); + test "convert f32 -> f4e2m1" { + const floats = @import("floats.zig"); + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + const x_f32 = [_]f32{ 0.0, 0.5, 1, 1.5, 2, 3, 4, 6, -0.0, -0.5, -1, -1.5, -2, -3, -4, -6 }; + var x_f4: [x_f32.len / 2]floats.Float4E2M1.Packed = undefined; + for (&x_f4, 0..) |*f4_pack, i| { + f4_pack.* = .fromF32(x_f32[2 * i], x_f32[2 * i + 1]); + } + + const x_d: Tensor = .init(.{x_f32.len}, .f32); + const exe = try zml.module.compile( + std.testing.allocator, + std.testing.io, + Tensor.convert, + .{ x_d, .f4e2m1 }, + platform, + .{}, + ); + defer exe.deinit(); + + var x_d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x_d.shape(), @ptrCast(&x_f32)); + defer x_d_buffer.deinit(); + + var x_f4_xla_d = try exe.eval(std.testing.allocator, std.testing.io, .{x_d_buffer}); + defer x_f4_xla_d.deinit(); + const x_f4_xla = try x_f4_xla_d.toSliceAlloc(std.testing.allocator, std.testing.io); + defer x_f4_xla.free(std.testing.allocator); + const x_f4_items = x_f4_xla.items(floats.Float4E2M1.Packed); + + errdefer std.log.warn("convert(f32 -> f4e2m1) failed !\ninput f32:\n{d}\nzml.floats computed:\n{d}\nxla computed:\n{d}", .{ stdx.fmt.slice(&x_f32), stdx.fmt.slice(&x_f4), x_f4_xla }); + try std.testing.expectEqualSlices(floats.Float4E2M1.Packed, &x_f4, x_f4_items); + } + + test "convert f32 -> f8e3m4" { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + const x = [_]f32{ 1.1 / 4.0, 1.1 / 8.0, 1.1 / 16.0, 1.1 / 32.0, 1.1 / 64.0, 1.1 / 128.0 }; + var x_f8e3: [x.len]zml.floats.Float8E3M4 = undefined; + for (&x_f8e3, &x) |*xi_f8e3, xi| xi_f8e3.* = .fromF32(xi); + + const x_d: Tensor = .init(.{x.len}, .f32); + const exe = try zml.module.compile( + std.testing.allocator, + std.testing.io, + Tensor.convert, + .{ x_d, .f8e3m4 }, + platform, + .{}, + ); + defer exe.deinit(); + + var x_d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x_d.shape(), std.mem.sliceAsBytes(&x)); + defer x_d_buffer.deinit(); + + var x_f8e3_xla_d = try exe.eval(std.testing.allocator, std.testing.io, .{x_d_buffer}); + defer x_f8e3_xla_d.deinit(); + const x_f8e3_xla = try x_f8e3_xla_d.toSliceAlloc(std.testing.allocator, std.testing.io); + defer x_f8e3_xla.free(std.testing.allocator); + + errdefer std.log.warn("convert(.f8e3m4) failed !\ninput f32:\n{d}\nzml.floats computed:\n{d}\nxla computed:\n{d}", .{ stdx.fmt.slice(&x), stdx.fmt.slice(&x_f8e3), x_f8e3_xla }); + try std.testing.expectEqualSlices(u8, @ptrCast(&x_f8e3), x_f8e3_xla.constData()); + } + + test "convert u2 -> u8" { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + const x_u2: [2]stdx.Packed(u2) = .{ .pack(0, 1, 2, 3), .pack(3, 2, 1, 0) }; + comptime std.debug.assert(@sizeOf(@TypeOf(x_u2)) == 2); + const x_u8: [8]u8 = .{ 0, 1, 2, 3, 3, 2, 1, 0 }; + + const x_u2_t: Tensor = .init(.{8}, .u2); + const exe = try zml.module.compile( + std.testing.allocator, + std.testing.io, + Tensor.convert, + .{ x_u2_t, .u8 }, + platform, + .{}, + ); + defer exe.deinit(); + + var x_d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x_u2_t.shape(), @ptrCast(&x_u2)); + defer x_d_buffer.deinit(); + + var x_u8_xla_d = try exe.eval(std.testing.allocator, std.testing.io, .{x_d_buffer}); + defer x_u8_xla_d.deinit(); + const x_u8_xla = try x_u8_xla_d.toSliceAlloc(std.testing.allocator, std.testing.io); + defer x_u8_xla.free(std.testing.allocator); + + try std.testing.expectEqualSlices(u8, &x_u8, x_u8_xla.constData()); + } + + test "convert f4e2m1 -> f32" { + const floats = @import("floats.zig"); + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + const x_f4_packed = [_]floats.Float4E2M1.Packed{ .fromF32(0, 0.5), .fromF32(1, 1.5), .fromF32(2, 3), .fromF32(4, 6) }; + + const x_t: Tensor = .init(.{x_f4_packed.len * 2}, .f4e2m1); + const exe = try zml.module.compile( + std.testing.allocator, + std.testing.io, + Tensor.convert, + .{ x_t, .f32 }, + platform, + .{}, + ); + defer exe.deinit(); + + var x_f4_d: zml.Buffer = try .fromBytes(std.testing.io, platform, x_t.shape(), @ptrCast(&x_f4_packed)); + defer x_f4_d.deinit(); + + var x_f32_d = try exe.eval(std.testing.allocator, std.testing.io, .{x_f4_d}); + defer x_f32_d.deinit(); + const x_f32_h = try x_f32_d.toSliceAlloc(std.testing.allocator, std.testing.io); + defer x_f32_h.free(std.testing.allocator); + + const x_f32_expected = [_]f32{ 0.0, 0.5, 1, 1.5, 2, 3, 4, 6 }; + errdefer std.log.warn("convert(f4e2m1x2 -> f32) failed !\ninput f4e2m1x2:\n{e}\nexpected:\n{any}\nxla computed:\n{any}", .{ stdx.fmt.slice(&x_f4_packed), x_f32_expected, x_f32_h }); + try std.testing.expectEqualSlices(f32, &x_f32_expected, x_f32_h.items(f32)); } /// Returns a Tensor containing the element-wise rounding operation of the input Tensor. pub fn round(self: Tensor) Tensor { - const loc = self.getContext().mlirCtx().location(@src()); - const sine_op = dialect.stablehlo.round_nearest_even(self.getContext().mlirCtx(), self.value(), loc); - return _result(self._shape, sine_op.result(0)); + const round_op = stablehlo.round_nearest_even(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); + return _result(self._shape, round_op.result(0)); } /// Returns a Tensor containing the element-wise clamping operation of the input Tensor. pub fn clamp(self: Tensor, min_: Tensor, max_: Tensor) Tensor { - const loc = self.getContext().mlirCtx().location(@src()); - const op = dialect.stablehlo.clamp(self.getContext().mlirCtx(), min_.value(), self.value(), max_.value(), loc); + const op = stablehlo.clamp(mlirCtx(), min_.value(), self.value(), max_.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } - /// See torch.matmul - pub fn matmul(lhs: Tensor, rhs: Tensor) Tensor { - return @import("torch.zig").matmul(lhs, rhs); - } - /// Matrix multiplication, where contracting axes are specified using their tags. - /// eg dot(.{ .a, .b, .c }, .{ .a, .c, .d }, .{ .c }) -> .{ .a, .c, .d } + /// eg dot(.{ .a, .b, .c }, .{ .a, .c, .d }, .c) -> .{ .a, .b, .d } /// Axes with the same tag on both sides, and which aren't contracting, /// are considered "batching axes". - pub fn dot(lhs: Tensor, rhs: Tensor, comptime contracting: anytype) Tensor { - var contracting_axes: [contracting.len][2]i8 = undefined; - inline for (contracting, 0..) |c, i| { - contracting_axes[i] = .{ lhs.axis(c), rhs.axis(c) }; + pub fn dot(lhs: Tensor, rhs: Tensor, args: anytype) Tensor { + const dot_axes = lhs.dotAxes(rhs, args); + return lhs.dotGeneral(rhs, dot_axes.contracting.constSlice(), dot_axes.batching.constSlice()); + } + + pub const DotAxes = struct { + contracting: stdx.BoundedArray([2]i8, constants.MAX_RANK), + batching: stdx.BoundedArray([2]i8, constants.MAX_RANK), + }; + + /// Returns contracting and batching axis pairs inferred from one or more tags. + pub fn dotAxes(lhs: Tensor, rhs: Tensor, tags: anytype) DotAxes { + const T = @TypeOf(tags); + if (comptime T == Shape.Tag or T == @EnumLiteral()) { + return lhs.dotAxes(rhs, @as([]const Shape.Tag, &.{Shape.toTag(tags)})); } - var batching_axes: [MAX_RANK][2]i8 = undefined; - var n_batching: u8 = 0; - for (lhs._shape.tags(), 0..) |l, li| { - meta.assert(l != Shape.TagUnknown, "Can't use `dot(..., {any})` on {any}, it need to be explictily tagged.", .{ contracting, lhs }); + const lhs_contracting_axes, _ = lhs.shape().parseAxes(tags); + const rhs_contracting_axes, _ = rhs.shape().parseAxes(tags); + var contracting_axes: stdx.BoundedArray([2]i8, constants.MAX_RANK) = .empty; + for (lhs_contracting_axes.constSlice(), rhs_contracting_axes.constSlice()) |l, r| { + contracting_axes.appendAssumeCapacity(.{ @intCast(l), @intCast(r) }); + } - for (rhs._shape.tags(), 0..) |r, ri| { - meta.assert(r != Shape.TagUnknown, "Can't use `dot(..., {any})` on {any}, it need to be explictily tagged.", .{ contracting, rhs }); + var batching_axes: stdx.BoundedArray([2]i8, constants.MAX_RANK) = .empty; + for (0..lhs.rank()) |lhs_tag_index| { + if (std.mem.indexOfScalar(u3, lhs_contracting_axes.constSlice(), @intCast(lhs_tag_index)) != null) { + continue; + } - if (l == r) { - for (contracting_axes) |ct| { - if (l == lhs._shape.tag(ct[0])) { - break; - } - } else { - // tag is both in lhs and rhs but not in contracting -> it's a batching dim. - batching_axes[n_batching] = .{ @intCast(li), @intCast(ri) }; - n_batching += 1; - } - } + const lhs_tag = lhs.shape().tag(lhs_tag_index); + if (rhs.shape().hasTag(lhs_tag)) |rhs_tag_index| { + batching_axes.appendAssumeCapacity(.{ @intCast(lhs_tag_index), @intCast(rhs_tag_index) }); } } - return dotGeneral(lhs, rhs, contracting_axes[0..], batching_axes[0..n_batching]); + return .{ .contracting = contracting_axes, .batching = batching_axes }; + } + + test dotAxes { + const lhs: Tensor = .init(.{ .a = 20, .b = 21, .c = 22 }, .f32); + const rhs: Tensor = .init(.{ .c = 22, .b = 21, .d = 23, .a = 20 }, .f32); + + const dot_axes = lhs.dotAxes(rhs, .{ .c, .a }); + + try std.testing.expectEqualSlices([2]i8, &.{ .{ 2, 0 }, .{ 0, 3 } }, dot_axes.contracting.constSlice()); + try std.testing.expectEqualSlices([2]i8, &.{.{ 1, 1 }}, dot_axes.batching.constSlice()); } test dot { const zml = @import("zml.zig"); const platform = zml.testing.env(); - var comp = try zml.module.CompilationContext.init(std.heap.page_allocator, "test", platform); - defer comp.deinit(); - - comp.activate(); - defer comp.deactivate(); - inline for (.{ - .{ .{ .c = 20 }, .{ .c = 20 }, .{.c}, .{} }, + .{ .{ .c = 20 }, .{ .c = 20 }, .c, .{} }, .{ .{ .a = 20, .b = 21, .c = 22 }, .{ .a = 20, .d = 23, .c = 22 }, - .{.c}, + .c, .{ .a = 20, .b = 21, .d = 23 }, }, .{ .{ .a = 20, .b = 21, .c = 22 }, .{ .c = 22, .d = 23, .e = 24 }, - .{.c}, + .c, .{ .a = 20, .b = 21, .d = 23, .e = 24 }, }, - .{ - .{ .a = 20, .b = 21, .c = 22 }, - .{ .c = 22, .d = 23, .a = 20 }, - .{ .c, .a }, - .{ .b = 21, .d = 23 }, - }, + // TODO(Corentin): Re-enable that + //.{ + // .{ .a = 20, .b = 21, .c = 22 }, + // .{ .c = 22, .d = 23, .a = 20 }, + // .{ .c, .a }, + // .{ .b = 21, .d = 23 }, + //}, }) |testcase| { - const x_shape, const y_shape, const ctr, const z_shape = testcase; - const x = Tensor.constant(x_shape, .{ .f32 = 0.0 }); - const y = Tensor.constant(y_shape, .{ .f32 = 0.0 }); - const z = x.dot(y, ctr); + const x: Tensor = .init(testcase[0], .f32); + const y: Tensor = .init(testcase[1], .f32); + const ctr = Shape.toTag(testcase[2]); + const z_shape: Shape = .init(testcase[3], .f32); + const forward = struct { + fn forward(x_: Tensor, y_: Tensor, tag: Shape.Tag) Tensor { + return x_.dot(y_, tag); + } + }.forward; - try zml.testing.expectEqualShapes(Shape.init(z_shape, .f32), z.shape()); + const exe = try zml.module.compile( + std.testing.allocator, + std.testing.io, + forward, + .{ x, y, ctr }, + platform, + .{}, + ); + defer exe.deinit(); + + try zml.testing.expectEqualShapes(Shape.init(z_shape, .f32), exe.output_shapes[0]); } } @@ -1120,17 +1629,17 @@ pub const Tensor = struct { contracting_axes: []const [2]i8, batching_axes: []const [2]i8, ) Tensor { - meta.assert(lhs.dtype() == rhs.dtype(), "dotGeneral expects tensors to be of the same type, got {} and {}", .{ lhs.dtype(), rhs.dtype() }); + stdx.debug.assert(lhs.dtype() == rhs.dtype(), "dotGeneral expects tensors to be of the same type, got {} and {}", .{ lhs.dtype(), rhs.dtype() }); - const Axes = std.BoundedArray(i64, MAX_RANK); + const Axes = stdx.BoundedArray(i64, constants.MAX_RANK); - var res_shape: Shape = .{ ._dtype = lhs.dtype() }; + var res_shape: Shape = .scalar(lhs.dtype()); // Validate batching axes - var lhs_batching_axes: Axes = .{}; - var rhs_batching_axes: Axes = .{}; + var lhs_batching_axes: Axes = .empty; + var rhs_batching_axes: Axes = .empty; for (batching_axes) |b_axes| { const l, const r = b_axes; - meta.assert(lhs._shape.dim(l) == rhs._shape.dim(r), "dotGeneral expects batching dimensions to be equal, got {} and {} in {} and {}", .{ l, r, lhs, rhs }); + stdx.debug.assert(lhs._shape.dim(l) == rhs._shape.dim(r), "dotGeneral expects batching dimensions to be equal, got {} and {} in {f} and {f}", .{ l, r, lhs, rhs }); var t = lhs._shape.tag(l); if (t == Shape.TagUnknown) t = rhs._shape.tag(r); res_shape = res_shape.appendDim(lhs._shape.dim(l), t); @@ -1139,11 +1648,11 @@ pub const Tensor = struct { } // Validate contracting axes - var lhs_contracting_axes: Axes = .{}; - var rhs_contracting_axes: Axes = .{}; + var lhs_contracting_axes: Axes = .empty; + var rhs_contracting_axes: Axes = .empty; for (contracting_axes) |c_axes| { const l, const r = c_axes; - meta.assert(lhs._shape.dim(l) == rhs._shape.dim(r), "dotGeneral expects contracting dimensions to be equal, got {} and {} in {} and {}", .{ l, r, lhs, rhs }); + stdx.debug.assert(lhs._shape.dim(l) == rhs._shape.dim(r), "dotGeneral expects contracting dimensions to be equal, got {} and {} in {f} and {f}", .{ l, r, lhs, rhs }); lhs_contracting_axes.appendAssumeCapacity(lhs._shape.axis(l)); rhs_contracting_axes.appendAssumeCapacity(rhs._shape.axis(r)); } @@ -1170,34 +1679,34 @@ pub const Tensor = struct { res_shape = res_shape.appendDim(rhs._shape.dim(r), rhs._shape.tag(r)); } - const loc = lhs.getContext().mlirCtx().location(@src()); - const op = dialect.stablehlo.dot_general( - lhs.getContext().mlirCtx(), + const op = stablehlo.dot_general( + mlirCtx(), lhs.value(), rhs.value(), - mlir.ext.mlirType(lhs.getContext().mlirCtx(), res_shape), - loc, + mlirx.Type.rankedTensor(mlirCtx(), res_shape), .{ .lhs_batching_dimensions = lhs_batching_axes.constSlice(), .rhs_batching_dimensions = rhs_batching_axes.constSlice(), .lhs_contracting_dimensions = lhs_contracting_axes.constSlice(), .rhs_contracting_dimensions = rhs_contracting_axes.constSlice(), - .precision = &.{ .DEFAULT, .DEFAULT }, + .dot_precision = .fast, }, - ); + currentLoc(), + ).appendTo(currentBlock()); return _result(res_shape, op.result(0)); } /// Returns a Tensor containing the sigmoid function applied to each element of the input Tensor. pub fn sigmoid(self: Tensor) Tensor { - // until the metal plugin supports `stablehlo.logistics`, implement in the way JAX does it - const one = Tensor.constant(&.{}, self.dtype().one()).broadcast(self._shape, &.{}); - return one.div(one.add(self.negate().exp())); + const op = stablehlo.logistic(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); + return _result(self._shape, op.result(0)); } + pub const logistic = sigmoid; + /// Returns a Tensor containing the ReLU activation function applied to each element of the input Tensor. pub fn relu(self: Tensor) Tensor { - return self.maximum(Tensor.constant(self.dims(), self.dtype().zero())); + return self.maximum(Tensor.constant(self.dtype().zero()).broad(.init(self.dims(), self.dtype()))); } /// Returns a Tensor containing the leaky-ReLU activation function applied to each element of the input Tensor. @@ -1205,7 +1714,7 @@ pub const Tensor = struct { /// LeakyReLU(x) = max(0,x) + negative_slope * min(0,x) /// ref: https://paperswithcode.com/method/leaky-relu pub fn leakyReLU(self: Tensor, negative_slope: f32) Tensor { - const below_zero = self.scale(negative_slope).minimum(Tensor.scalar(0, self.dtype())); + const below_zero = self.scale(negative_slope).minimum(.scalar(0, self.dtype())); return self.relu().add(below_zero); } @@ -1213,19 +1722,194 @@ pub const Tensor = struct { const zml = @import("zml.zig"); const platform = zml.testing.env(); - const input = try zml.Buffer.fromSlice(platform, .{2}, &[_]f32{ -0.6884, 1.6795 }); - const res = try zml.testing.compileAndCall(platform, leakyReLU, .{ input, 0.1 }); + const input: Tensor = .init(.{2}, .f32); + var exe = try zml.module.compile( + std.testing.allocator, + std.testing.io, + Tensor.leakyReLU, + .{ input, 0.1 }, + platform, + .{}, + ); + defer exe.deinit(); - const expectation = zml.HostBuffer.fromArray(&[2]f32{ -0.0688, 1.6795 }); - try zml.testing.expectClose(expectation, res, 1e-4); + var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), std.mem.sliceAsBytes(&[_]f32{ -0.6884, 1.6795 })); + defer input_buffer.deinit(); + + var res = try exe.eval(std.testing.allocator, std.testing.io, .{input_buffer}); + defer res.deinit(); + + const expectation: zml.Slice = .init(input.shape(), std.mem.sliceAsBytes(&[2]f32{ -0.0688, 1.6795 })); + try zml.testing.expectClose(std.testing.io, expectation, res, .{}); } /// Returns a Tensor containing the SwiGLU activation function applied to the input Tensor. - pub fn swiglu(self: Tensor, beta: f32, w: Tensor, b: Tensor) Tensor { - const sigmoid_tensor = self.mul(Tensor.constant(self._shape, Data.init(self.dtype(), beta))).sigmoid(); - const one_minus_sigmoid_tensor = Tensor.constant(self._shape, Data.init(self.dtype(), 1)).sub(sigmoid_tensor); + pub fn swiglu(self: Tensor, beta: f32, w: Tensor, v: Tensor, b: Tensor, c: Tensor, tag: Shape.Tag) Tensor { + var value_path = self.dot(w, tag); + value_path = value_path.add(b.broad(value_path.shape())); - return self.mul(sigmoid_tensor).add(one_minus_sigmoid_tensor.mul(w.matmul(self).add(b))); + const sigmoid_tensor = value_path.scale(beta).sigmoid(); + + var gate_path = self.dot(v, tag); + gate_path = gate_path.add(c.broad(gate_path.shape())); + + return value_path.mul(sigmoid_tensor).outer(gate_path); + } + + test swiglu { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + const input: Tensor = .init(.{ .d = 4 }, .f32); + const beta = 3; + const w: Tensor = .init(.{ .c = 3, .d = 4 }, .f32); + const v: Tensor = .init(.{ .e = 3, .d = 4 }, .f32); + const b: Tensor = .init(.{ .c = 3 }, .f32); + const c: Tensor = .init(.{ .e = 3 }, .f32); + + var exe = try zml.module.compile( + std.testing.allocator, + std.testing.io, + Tensor.swiglu, + .{ input, beta, w, v, b, c, Shape.toTag(.d) }, + platform, + .{}, + ); + defer exe.deinit(); + + var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), std.mem.sliceAsBytes(&[4]f32{ 1, 0, 1, 2 })); + defer input_buffer.deinit(); + + var w_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, w.shape(), std.mem.sliceAsBytes(&[3][4]f32{ + .{ 1, 0, 2, 0 }, + .{ 2, 1, 3, 3 }, + .{ 1, 1, 0, 0 }, + })); + defer w_buffer.deinit(); + + var v_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, v.shape(), std.mem.sliceAsBytes(&[3][4]f32{ + .{ 0, 0, 2, 1 }, + .{ 1, 2, 1, 3 }, + .{ 0, 0, 2, 0 }, + })); + defer v_buffer.deinit(); + + var b_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, b.shape(), std.mem.sliceAsBytes(&[3]f32{ 1, 1, 1 })); + defer b_buffer.deinit(); + + var c_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, c.shape(), std.mem.sliceAsBytes(&[3]f32{ 2, 2, 2 })); + defer c_buffer.deinit(); + + var res = try exe.eval(std.testing.allocator, std.testing.io, .{ input_buffer, w_buffer, v_buffer, b_buffer, c_buffer }); + defer res.deinit(); + + const output_shape = Shape.init(.{ .c = 3, .e = 3 }, .f32); + const expectation: zml.Slice = .init(output_shape, std.mem.sliceAsBytes(&[3][3]f32{ + .{ 23.999856, 39.99976, 15.999904 }, + .{ 72.000000, 120.000000, 48.000000 }, + .{ 11.970324, 19.950540, 7.980216 }, + })); + + try zml.testing.expectClose(std.testing.io, expectation, res, .{}); + } + + /// Returns an element-wise approximation of the error function, computed and returned in FP32. + /// Uses the Eigen rational approximation from CHLO's FP32 erf legalization: + /// https://github.com/openxla/stablehlo/blob/95ee4c70146864ed7d0a21dd135ccec37ece1b52/stablehlo/transforms/ChloLegalizeToStablehlo.cpp#L1038 + pub fn erf(input: Tensor) Tensor { + const x = input.convert(.f32).clamp(.scalar(-4, .f32), .scalar(4, .f32)); + const x2 = x.mul(x); + // Coefficients run from the highest degree to the constant term. + const alpha = [_]f32{ + -2.72614225801306e-10, + 2.77068142495902e-08, + -2.10102402082508e-06, + -5.69250639462346e-05, + -7.34990630326855e-04, + -2.95459980854025e-03, + -1.60960333262415e-02, + }; + const beta = [_]f32{ + -1.45660718464996e-05, + -2.13374055278905e-04, + -1.68282697438203e-03, + -7.37332916720468e-03, + -1.42647390514189e-02, + }; + var numerator = x2.scale(alpha[0]).addConstant(alpha[1]); + inline for (alpha[2..]) |coefficient| { + numerator = numerator.mul(x2).addConstant(coefficient); + } + var denominator = x2.scale(beta[0]).addConstant(beta[1]); + inline for (beta[2..]) |coefficient| { + denominator = denominator.mul(x2).addConstant(coefficient); + } + return x.mul(numerator).div(denominator).clamp(.scalar(-1, .f32), .scalar(1, .f32)); + } + + /// Returns an element-wise approximation of the complementary error function in FP32. + /// Uses CHLO's Cephes-based approximation to preserve the small positive tail: + /// https://github.com/openxla/stablehlo/blob/95ee4c70146864ed7d0a21dd135ccec37ece1b52/stablehlo/transforms/ChloLegalizeToStablehlo.cpp#L1018 + pub fn erfc(input: Tensor) Tensor { + const x = input.convert(.f32); + const abs_x = x.abs(); + const x2 = x.mul(x); + const one = Tensor.scalar(1, .f32); + + // For |x| < 1, erf(x) = x T(x^2). Coefficients run from highest degree to constant. + const t = [_]f32{ + 7.853861353153693e-5, + -8.010193625184903e-4, + 5.188327685732524e-3, + -2.685381193529856e-2, + 1.128358514861418e-1, + -3.761262582423300e-1, + 1.128379165726710, + }; + var small_poly = x2.scale(t[0]).addConstant(t[1]); + inline for (t[2..]) |coefficient| { + small_poly = small_poly.mul(x2).addConstant(coefficient); + } + const small = x.mul(small_poly).negate().addConstant(1); + + // For |x| >= 1, evaluate exp(-x^2) / |x| times P(1/x^2) or R(1/x^2). + const p = [_]f32{ + 2.326819970068386e-2, + -1.387039388740657e-1, + 3.687424674597105e-1, + -5.824733027278666e-1, + 6.210004621745983e-1, + -4.944515323274145e-1, + 3.404879937665872e-1, + -2.741127028184656e-1, + 5.638259427386472e-1, + }; + const r = [_]f32{ + -1.047766399936249e1, + 1.297719955372516e1, + -7.495518717768503, + 2.921019019210786, + -1.015265279202700, + 4.218463358204948e-1, + -2.820767439740514e-1, + 5.641895067754075e-1, + }; + const reciprocal_x2 = one.div(x2); + var middle_poly = reciprocal_x2.scale(p[0]).addConstant(p[1]); + inline for (p[2..]) |coefficient| { + middle_poly = middle_poly.mul(reciprocal_x2).addConstant(coefficient); + } + var tail_poly = reciprocal_x2.scale(r[0]).addConstant(r[1]); + inline for (r[2..]) |coefficient| { + tail_poly = tail_poly.mul(reciprocal_x2).addConstant(coefficient); + } + const poly = abs_x.cmp(.LT, .scalar(2, .f32)).select(middle_poly, tail_poly); + const z = x2.negate(); + const tail = z.exp().mul(one.div(abs_x)).mul(poly); + // Match CHLO's underflow cutoff, then use erfc(-x) = 2 - erfc(x). + const positive = z.cmp(.LT, .scalar(-88.72283905206835, .f32)).select(.scalar(0, .f32), tail); + const large = x.cmp(.LT, .scalar(0, .f32)).select(positive.negate().addConstant(2), positive); + return abs_x.cmp(.LT, one).select(small, large); } /// Returns a Tensor containing the Gaussian Error Linear Units (GeLU) activation function applied to each element of the input Tensor. @@ -1235,9 +1919,9 @@ pub const Tensor = struct { /// see: https://paperswithcode.com/method/gelu pub fn gelu(x: Tensor) Tensor { const scaled_x_cube = x.mul(x).mul(x).scale(0.044715); - const one = Tensor.constant(x._shape, x.dtype().one()); - const one_plus_tanh = Tensor.add(x, scaled_x_cube).scale(std.math.sqrt(2.0 / std.math.pi)).tanh().add(one); - return one_plus_tanh.mul(x).scale(0.5); + const beta = std.math.sqrt(2.0 / std.math.pi); + const tanh_ = x.add(scaled_x_cube).scale(beta).tanh(); + return tanh_.addConstant(1).mul(x).scale(0.5); } /// Returns a Tensor containing an approximation of the Gaussian Error Linear Units (GeLU) activation function applied to each element of the input Tensor. @@ -1250,80 +1934,175 @@ pub const Tensor = struct { /// Returns a Tensor containing the Sigmoid Linear Unit (SiLU) activation function applied to each element of the input Tensor. /// /// silu(x) = x σ(x) - /// https://paperswithcode.com/method/silu pub fn silu(x: Tensor) Tensor { - return x.mul(x.sigmoid()); + return x.mul(.sigmoid(x)); } - /// Returns a Tensor containing the softmax function applied to each element of the input Tensor. + /// Computes softmax along the given axis. + /// y[i] = exp(x[i]) / ( Σ_k exp(x[k]) + bias ) pub fn softmax(self: Tensor, axis_: anytype) Tensor { const a = self.axis(axis_); - const exp_diff_max = self.sub(self.max(a).broad(self._shape)).exp(); - return exp_diff_max.div(exp_diff_max.sum(a).broad(self._shape)); + const max_val = self.max(a); + const row_mask = max_val.cmp(.GT, .scalar(-std.math.inf(f64), self.dtype())); + + const exp_diff_max = self.sub(max_val).convert(.f32).exp(); + const res = exp_diff_max.div(exp_diff_max.sum(a)).convert(self.dtype()); + + // If a row is full -inf return full 0 instead of full nan, + // this fix attention when mask hides a full row. + return row_mask.broad(self.shape()).select(res, .scalar(0, self.dtype())); + } + + /// Computes softmax, but adds a bias to the sum of exponentiel. + /// y[i] = exp(x[i]) / ( Σ_k exp(x[k]) + bias ) + pub fn softmaxBiased(self: Tensor, axis_: anytype, bias: ?Tensor) Tensor { + const a = self.axis(axis_); + + if (bias == null) return self.softmax(axis_); + const b = bias.?.convert(self.dtype()).broad(self.shape().setDim(a, 1)); + const max_val: Tensor = maximum(self.max(a), b); + const exp_diff_max = self.sub(max_val).exp(); + const bias_diff_max = b.sub(max_val).exp(); + const res = exp_diff_max.div(exp_diff_max.sum(a).add(bias_diff_max)); + + // The bias means that denominator won't be 0, we don't need to handle that case. + return res; } /// Returns a Tensor containing the log of the sum of exponential over the given axis. pub fn logSumExp(self: Tensor, axis_: anytype) Tensor { const a = self.axis(axis_); // stabilization: shift `self` by it's max value before passing to exponent. - const M = self.max(a); - const log_sum_exp = log(sum(exp(self.sub(M.broad(self._shape))), a)); + const max_ = self.max(a); + const log_sum_exp = log(sum(exp(self.sub(max_.broad(self._shape))), a)); // restore the shift again - return M.add(log_sum_exp); + return max_.add(log_sum_exp); } /// Returns a Tensor containing the sum of elements over the given axis. + /// Output shape is the input shape with the axis_ dim set to 1. pub fn sum(self: Tensor, axis_: anytype) Tensor { const a = self.axis(axis_); - return ops.reduce( - struct { - pub fn acc(x: Tensor, res: Tensor) Tensor { - return res.add(x.convert(res.dtype())); - } - }.acc, - self, - Tensor.scalar(0, self.dtype()), - &.{a}, - ); + return ops.reduce(.{self}, .{Tensor.constant(self.dtype().zero())}, &.{a}, struct { + pub fn acc(args: ops.ReduceArgs) struct { Tensor } { + return .{args.right.add(args.left.convert(args.right.dtype()))}; + } + }.acc, .{})[0]; } /// Returns a Tensor containing the mean of elements over the given axis. + /// Output shape is the input shape with the axis_ dim set to 1. pub fn mean(self: Tensor, axis_: anytype) Tensor { return self.sum(axis_).divByConst(self.dim(axis_)); } + /// Returns a Tensor containing the cumulative sum of elements over the given axis. + /// Output shape is the same as input shape. + /// [0, 1, 0, 1, 0, 0, 1, 1].cumulativeSum(0) -> [0, 1, 1, 2, 2, 2, 3, 4] + /// The last value contains the sum of all element in the array. + pub fn cumulativeSum(self: Tensor, axis_: anytype) Tensor { + const rk = self.rank(); + const a = self.axis(axis_); + + var window_dimensions = constants.ones_i64; + window_dimensions[a] = self.dim(a); + var padding: [constants.MAX_RANK][2]i64 = @splat(.{ 0, 0 }); + padding[a] = .{ self.dim(a) - 1, 0 }; + + const result = ops.reduceWindow( + 1, + .{self}, + .{.scalar(0, self.dtype())}, + .{ + .base_dilations = constants.ones_i64[0..rk], + .window_dilations = constants.ones_i64[0..rk], + .window_strides = constants.ones_i64[0..rk], + .window_dimensions = window_dimensions[0..rk], + .padding = padding[0..rk], + }, + struct { + fn add(values: ops.ReduceArgs) [1]Tensor { + return .{.add(values.left, values.right)}; + } + }.add, + ); + return result[0]; + } + + test cumulativeSum { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + const Local = struct { + pub fn _cumsum(input: Tensor) Tensor { + const x = input.withPartialTags(.{.n}); + const y = x.cumulativeSum(.n); + // Check that tags are propagated + std.debug.assert(y.shape().eqlWithTags(x.shape())); + return y; + } + }; + + const x: Tensor = .init(.{ 2, 5 }, .f32); + + var exe = try zml.module.compile( + std.testing.allocator, + std.testing.io, + Local._cumsum, + .{x}, + platform, + .{}, + ); + defer exe.deinit(); + + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), @ptrCast(&[2][5]f32{ .{ 0, 1, 1, 0, 1 }, .{ 3, 1, 0, 2, 1 } })); + defer x_buffer.deinit(); + + var res = try exe.eval(std.testing.allocator, std.testing.io, .{x_buffer}); + defer res.deinit(); + + try std.testing.expectEqual([2][5]f32{ .{ 0, 1, 2, 2, 3 }, .{ 3, 4, 4, 6, 7 } }, try res.getValue([2][5]f32, std.testing.io)); + } + /// Returns a transposed Tensor computed using the given axes. pub fn transpose(self: Tensor, axes_: anytype) Tensor { const axes__ = self.axes(axes_).constSlice(); - const default_perm = [MAX_RANK]i64{ 7, 6, 5, 4, 3, 2, 1, 0 }; - const no_op = [MAX_RANK]i64{ 0, 1, 2, 3, 4, 5, 6, 7 }; + const no_op = constants.AXES_IOTA; + const default_perm = b: { + var buf = constants.AXES_IOTA; + std.mem.reverse(i64, &buf); + break :b buf; + }; const permutation: []const i64 = if (axes__.len == 0) - default_perm[MAX_RANK - self.rank() ..] + default_perm[constants.MAX_RANK - self.rank() ..] else - toI64(axes__); + toI64(axes__).constSlice(); - meta.assert(permutation.len == self.rank(), "transpose expects input tensor rank and 'axes_' length to be equal, got {} and {}", .{ self.rank(), permutation.len }); + stdx.debug.assert(permutation.len == self.rank(), "transpose expects input tensor rank and 'axes_' length to be equal, got {f} and {any}", .{ self, permutation[0..@min(permutation.len, constants.MAX_RANK + 2)] }); if (std.mem.eql(i64, permutation, no_op[0..self.rank()])) { return self; } const res_shape = self._shape.transpose(permutation); - const loc = self.getContext().mlirCtx().location(@src()).namedFmt(self.getContext().mlirCtx(), "tr({any})", .{axes_}); - const op = dialect.stablehlo.transpose( - self.getContext().mlirCtx(), + if (transposeIsJustAReshape(self.shape(), permutation)) { + return self.reshape(res_shape); + } + + const op = stablehlo.transpose( + mlirCtx(), self.value(), - mlir.ext.mlirType(self.getContext().mlirCtx(), res_shape), - loc, - .{ .permutation = toI64(permutation) }, - ); + mlirx.Type.rankedTensor(mlirCtx(), res_shape), + .{ .permutation = toI64(permutation).constSlice() }, + currentLoc(), + ).appendTo(currentBlock()); return _result(res_shape, op.result(0)); } pub fn swapAxes(self: Tensor, a: anytype, b: anytype) Tensor { if (self.axis(a) == self.axis(b)) return self; - var perm: Shape.AxesArray = .{}; + var perm: Shape.AxesArray = .empty; for (0..self.rank()) |i| { perm.appendAssumeCapacity(@intCast(i)); } @@ -1336,19 +2115,19 @@ pub const Tensor = struct { /// /// unflatten((d0, d1, axis_m, d3), 2, n) -> (d0, d1, n, d2_m, d3) pub fn unflatten(self: Tensor, axis_: i8, n: i64) Tensor { - meta.assert(self.rank() < Tensor.MAX_RANK, "unflatten expects input tensor rank to be less than {}, got {}", .{ Tensor.MAX_RANK, self.rank() }); + // TODO: move to torch.zig, this equivalent to `spitAxis` + stdx.debug.assert(self.rank() < constants.MAX_RANK, "unflatten expects input tensor rank to be less than {}, got {}", .{ constants.MAX_RANK, self.rank() }); const a = if (axis_ >= 0) self.axis(axis_) else self.axis(axis_) + 1; const new_dim = std.math.divExact(i64, self.dim(a), n) catch std.debug.panic("unflatten expects chosen dimension to be divisible by 'n' but {} is not divisible by {}", .{ self.dim(a), n }); const new_shape = self._shape.set(a, n).insert(a + 1, .{ ._ = new_dim }); - const loc = self.getContext().mlirCtx().location(@src()).namedFmt(self.getContext().mlirCtx(), "axis={}, n={}", .{ axis_, n }); - const reshaped_val = dialect.stablehlo.reshape( - self.getContext().mlirCtx(), + const reshaped_val = stablehlo.reshape( + mlirCtx(), self.value(), - mlir.ext.RankedTensorType.fromShape(self.getContext().mlirCtx(), new_shape), - loc, - ); + mlirx.Type.rankedTensor(mlirCtx(), new_shape), + currentLoc(), + ).appendTo(currentBlock()); return _result(new_shape, reshaped_val.result(0)); } @@ -1359,13 +2138,12 @@ pub const Tensor = struct { pub fn splitAxis(self: Tensor, ax: anytype, split_shape: anytype) Tensor { const new_shape = self._shape.splitAxis(ax, split_shape); - const loc = self.getContext().mlirCtx().location(@src()).namedFmt(self.getContext().mlirCtx(), "splitAxis({}, {any})", .{ ax, split_shape }); - const reshaped_val = dialect.stablehlo.reshape( - self.getContext().mlirCtx(), + const reshaped_val = stablehlo.reshape( + mlirCtx(), self.value(), - mlir.ext.RankedTensorType.fromShape(self.getContext().mlirCtx(), new_shape), - loc, - ); + mlirx.Type.rankedTensor(mlirCtx(), new_shape), + currentLoc(), + ).appendTo(currentBlock()); return _result(new_shape, reshaped_val.result(0)); } @@ -1377,9 +2155,9 @@ pub const Tensor = struct { /// Merges two or more non-contiguous axes into one axis. /// Will make a transpose if needed. /// .{ .a, .b, .c }.mergeTranspose(.{ .a, .c }, .ac) -> .{ .b, .ac } - pub fn mergeTranspose(self: Tensor, axes_: anytype, merged: EnumLiteral) Tensor { + pub fn mergeTranspose(self: Tensor, axes_: anytype, merged: @EnumLiteral()) Tensor { const cont = self.contiguous(axes_); - return cont.reshape(cont._shape.mergeAxis(axes_, merged)); + return cont.reshape(cont._shape.mergeAxis(merged, axes_)); } /// Transposes the input Tensor, such has the given axes end up in contiguous position. @@ -1389,104 +2167,246 @@ pub const Tensor = struct { return self.transpose(perm.constSlice()); } - /// Flattens the given axis and the next one, into one new axis. - pub fn flatten(self: Tensor, axis_: anytype) Tensor { - const old_shape = self._shape; - const a = self.axis(axis_); - // meta.assert(a + 1 < self.rank(), "Can't flatten {} on the last axis {}.", .{ self, axis }); - const new_shape = old_shape.remove(a + 1).set(a, old_shape.dim(a) * old_shape.dim(a + 1)); - - const loc = self.getContext().mlirCtx().location(@src()).namedFmt(self.getContext().mlirCtx(), "axis={}", .{axis_}); - - const reshaped_val = dialect.stablehlo.reshape( - self.getContext().mlirCtx(), - self.value(), - mlir.ext.RankedTensorType.fromShape(self.getContext().mlirCtx(), new_shape), - loc, - ); - // log.debug("flatten({d}, {d}) -> {d}", .{ self.dims(), axis_, new_shape[0 .. self.rank() - 1] }); - return _result(new_shape, reshaped_val.result(0)); - } - - pub inline fn flattenAll(self: Tensor) Tensor { + pub fn flatten(self: Tensor) Tensor { return self.reshape(.{self.count()}); } pub const Slice = struct { - single: ?i64 = null, start: i64 = 0, - end: ?i64 = null, - step: i64 = 1, - }; + dyn_start: ?Tensor = null, + len: i64 = to_the_end, + end: i64 = to_the_end, + step: i32 = 1, + singleton: bool = false, - /// Slices the input Tensor over the given axis using the given parameters. - pub fn slice1d(self: Tensor, axis_: anytype, s: Slice) Tensor { - var slices = [_]Slice{.{}} ** MAX_RANK; - slices[self.axis(axis_)] = s; - return self.slice(slices[0..self.rank()]); - } + /// Select the full slice + pub const full: Slice = .{ .start = 0, .end = to_the_end, .step = 1 }; - /// Slices the input Tensor using the given parameters. - pub fn slice(self: Tensor, slices: []const Slice) Tensor { - var start_indices: [MAX_RANK]i64 = undefined; - var strides: [MAX_RANK]i64 = undefined; - var limit_indices: [MAX_RANK]i64 = undefined; - var res_shape: Shape = self._shape; - - for (slices, 0..) |s, a| { - meta.assert(s.step > 0, "slice expects 'step' to be positive, got {} at index {}", .{ s.step, a }); - - const args: Slice = .{ - .start = self.wrapIndex(a, s.start), - .end = if (s.end) |end| self.wrapIndex(a, end) else self.dim(a), - .step = s.step, - }; - start_indices[a] = args.start; - limit_indices[a] = args.end.?; - strides[a] = args.step; - res_shape = res_shape.setDim(a, std.math.divCeil(i64, args.end.? - args.start, args.step) catch unreachable); + /// Select a single element and drop the corresponding axis (to keep the axis use .{ .start = off, .len =1 }) + pub fn single(offset: i64) Slice { + return .{ .start = offset, .end = offset + 1, .singleton = true }; } - const mlir_ctx = self.getContext().mlirCtx(); - const loc = mlir_ctx.location(@src()).namedFmt(mlir_ctx, "slices={any}", .{slices}); - const result_type = mlir.ext.RankedTensorType.fromShape(mlir_ctx, res_shape).as(mlir.Type).?; - const slice_op = dialect.stablehlo.slice( - mlir_ctx, + /// Select a single element and drop the corresponding axis (to keep the axis use .{ .start = off, .len =1 }) + pub fn dyn(start: Tensor, len: i64) Slice { + stdx.debug.assert(start.rank() == 0, "Slice.dyn expects scalar input, got {f}", .{start}); + return .{ .dyn_start = start, .len = len }; + } + + pub fn dynSingle(start: Tensor) Slice { + stdx.debug.assert(start.rank() == 0, "Slice.dynSingle expects scalar input, got {f}", .{start}); + return .{ .dyn_start = start, .len = 1, .singleton = true }; + } + + pub fn absolute(s: Slice, d: i64) Slice { + const start = if (s.start < 0) d + s.start else s.start; + const end = if (s.singleton) + start + 1 + else if (s.end == to_the_end) + d + else if (s.end < 0) + d + s.end + else if (s.len != to_the_end) + start + s.len + else + s.end; + const res: Slice = .{ .start = start, .end = end, .step = s.step, .singleton = s.singleton }; + stdx.debug.assert(start < end, "Slice {f} is invalid for axis of dimension {d} (resolved to {f})", .{ s, d, res }); + stdx.debug.assert(end <= d, "Slice {f} is invalid for axis of dimension {d} (resolved to {f})", .{ s, d, res }); + return res; + } + + const to_the_end = std.math.maxInt(i64); + + pub fn format(s: Slice, writer: *std.Io.Writer) !void { + const end = if (s.len != to_the_end) s.start +| s.len else s.end; + if (s.dyn_start) |_| { + if (s.singleton) + try writer.writeAll("[x]") + else + try writer.print("[x..x+{}]", .{s.len}); + } else if (s.singleton) { + try writer.print("[{d}]", .{s.start}); + } else if (end == to_the_end and s.step == 1) { + try writer.print("[{d}..]", .{s.start}); + } else if (s.step == 1) { + try writer.print("[{d}..{d}]", .{ s.start, end }); + } else { + try writer.print("[{d}..{d}:{d}]", .{ s.start, end, s.step }); + } + } + }; + + /// Slice a Tensor across a specific axes. + /// + /// Due to the nature of stablehlo, the length of the slices need to be known when compiling the IR. + /// Examples: + /// ``` + /// Tensor(.{.a=20,.b=30,.c=40 }).slice(.a, .{ .start = 5, .len = 11}); + /// Tensor(.{.a=20,.b=30,.c=40 }).slice(0, .single(5)); + /// ``` + pub fn slice(self: Tensor, axis_: anytype, slice_: Slice) Tensor { + return self.slices(@as([]const ResolvedAxis, &.{self.axis(axis_)}), &.{slice_}); + } + + /// Slices a Tensor across many axes. + /// + /// ``` + /// Tensor(.{.a=20,.b=30,.c=40 }).slices(.{ .a, .b }, &.{ .start = 5, .end = 11}, .dyn(x) }); + /// Tensor(.{.a=20,.b=30,.c=40 }).slices(.{ + /// .a = .{ .start = a_off, .len = 11 }, + /// .b = .{ .start = b_off, .len = 12 }, + /// }); + /// Tensor(.{ 20,30,40}).slices(.{ 0, 2 }, .{.dyn(a_off, .len = 20), .{ .start = 8, .len = 12 }}); + /// ``` + pub fn slices(self: Tensor, axes_: anytype, slices_: []const Slice) Tensor { + // Note: with typed slices, the callsites are less akwards. + if (@TypeOf(axes_) != []const ResolvedAxis) { + const resolved_axes: []const ResolvedAxis = self._shape.parseAxes(axes_)[0].slice(); + return self.slices(resolved_axes, slices_); + } + + const rk = self.rank(); + any_dyn: { + for (slices_) |s| { + if (s.dyn_start) |_| break :any_dyn; + } + return self.staticSlices(axes_, slices_); + } + + const zero = scalar(0, .i64).value(); + var start_indices: [constants.MAX_RANK]*const mlir.Value = @splat(zero); + var new_shape: Shape = self._shape; + var to_remove: Shape.AxesArray = .empty; + + for (axes_, slices_) |a, s| { + stdx.debug.assert(s.step == 1, "Tensor.slices doesn't support mixing strided and dynamic slices, got {f}", .{stdx.fmt.slice(slices_)}); + new_shape._dims.buffer[a] = s.len; + start_indices[a] = if (s.dyn_start) |dyn| + dyn.convert(.i64).value() + else + scalar(s.start, .i64).value(); + if (s.singleton) to_remove.appendAssumeCapacity(@intCast(a)); + } + + const op = stablehlo.dynamic_slice( + mlirCtx(), + self.value(), + new_shape.dims(), + start_indices[0..rk], + currentLoc(), + ).appendTo(currentBlock()); + + const res = _result(new_shape, op.result(0)); + return res.reshape(new_shape.removeMany(to_remove.constSlice())); + } + + fn staticSlices(self: Tensor, axes_: []const ResolvedAxis, slices_: []const Slice) Tensor { + var start_indices: [constants.MAX_RANK]i64 = @splat(0); + var strides: [constants.MAX_RANK]i64 = @splat(1); + var limit_indices: [constants.MAX_RANK]i64 = self._shape._dims.buffer; + var res_shape: Shape = self._shape; + + for (axes_, slices_) |a, s| { + stdx.debug.assert(s.step > 0, "slice expects 'step' to be positive, got {} on axis {}", .{ s.step, a }); + stdx.debug.assert(s.step > 0, "slice expects 'step' to be positive, got {} on axis {}", .{ s.step, a }); + + const args: Slice = s.absolute(self.dim(a)); + start_indices[a] = args.start; + limit_indices[a] = args.end; + strides[a] = args.step; + res_shape = res_shape.setDim(a, std.math.divCeil(i64, args.end - args.start, args.step) catch unreachable); + } + + const ctx = Compiler.current(); + const slice_op = stablehlo.slice( + ctx.mlir_ctx, self.value(), start_indices[0..self.rank()], limit_indices[0..self.rank()], strides[0..self.rank()], - result_type, - loc, - ); + mlirx.Type.rankedTensor(ctx.mlir_ctx, res_shape), + ctx.location, + ).appendTo(currentBlock()); - return _result(res_shape, slice_op.result(0)); + var res = _result(res_shape, slice_op.result(0)); + var to_remove: Shape.AxesArray = .empty; + for (axes_, slices_) |a, s| { + if (s.singleton) to_remove.appendAssumeCapacity(a); + } + return res.reshape(res_shape.removeMany(to_remove.constSlice())); } test slice { const zml = @import("zml.zig"); const platform = zml.testing.env(); - const x = try zml.Buffer.fromSlice(platform, .{ 2, 5 }, &[_]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }); + const x: Tensor = .init(.{ 2, 5 }, .f32); - // Wrap slice1d to hide the anytype in the signature. + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 })); + defer x_buffer.deinit(); + + // Wrap slice to hide the anytype in the signature. + // TODO(wzk): stdx.meta.specifyFn (waiting 0.17 for SoA meta programming) const Local = struct { - pub fn slice1dAxis(input: Tensor, ax: i8, slice_: Tensor.Slice) Tensor { - return input.slice1d(ax, slice_); + fn _slice1dAxis(input: Tensor, ax: i8, slice_: Tensor.Slice) Tensor { + return input.slice(ax, slice_); } }; - { - const res = try zml.testing.compileAndCallWithTensors(platform, Local.slice1dAxis, .{ x.shape(), 0, .{ .end = 1 } }, .{ x, 0, .{ .end = 1 } }); - try testing.expectEqual([5]f32{ 0, 1, 2, 3, 4 }, try res.getValue([5]f32)); + inline for (.{ + .{ 0, Tensor.Slice{ .end = 1 }, [5]f32{ 0, 1, 2, 3, 4 } }, + .{ 1, Tensor.Slice{ .start = 1, .step = 2 }, [4]f32{ 1, 3, 6, 8 } }, + .{ -1, Tensor.Slice{ .start = -2 }, [4]f32{ 3, 4, 8, 9 } }, + }) |testcase| { + const ax, const slice_, const expectation = testcase; + var exe = try zml.module.compile( + std.testing.allocator, + std.testing.io, + Local._slice1dAxis, + .{ x, ax, slice_ }, + platform, + .{}, + ); + defer exe.deinit(); + + var res = try exe.eval(std.testing.allocator, std.testing.io, .{ x_buffer, .{} }); + defer res.deinit(); + + try std.testing.expectEqual(expectation, try res.getValue(@TypeOf(expectation), std.testing.io)); } - { - const res = try zml.testing.compileAndCallWithTensors(platform, Local.slice1dAxis, .{ x.shape(), 1, .{ .start = 1, .step = 2 } }, .{ x, 0, .{ .start = 1, .step = 2 } }); - try testing.expectEqual([4]f32{ 1, 3, 6, 8 }, try res.getValue([4]f32)); - } - { - const res = try zml.testing.compileAndCallWithTensors(platform, Local.slice1dAxis, .{ x.shape(), -1, .{ .start = -2 } }, .{ x, 0, .{ .start = -2 } }); - try testing.expectEqual([4]f32{ 3, 4, 8, 9 }, try res.getValue([4]f32)); + } + + test "dynamic slices" { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + const Local = struct { + fn dynSliceResolved(x: Tensor, ax: ResolvedAxis, dyn_slice: Slice) Tensor { + return x.slice(ax, dyn_slice); + } + }; + + inline for (.{ + .{ [10]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }, Shape.init(.{10}, .f32), [2]f32{ 4, 5 }, 4, 0 }, + .{ [2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } }, Shape.init(.{ 2, 5 }, .f32), [4]f32{ 3, 4, 8, 9 }, 3, 1 }, + }) |testcase| { + const x_data, const x_shape, const expectation, const z_value: i32, const ax = testcase; + const x: Tensor = .fromShape(x_shape); + const z: Tensor = .init(.{}, .i32); + + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local.dynSliceResolved, .{ x, ax, .dyn(z, 2) }, platform, .{}); + defer exe.deinit(); + + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&x_data)); + defer x_buffer.deinit(); + var z_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, z.shape(), std.mem.asBytes(&z_value)); + defer z_buffer.deinit(); + + var res = try exe.eval(std.testing.allocator, std.testing.io, .{ x_buffer, .{ .dyn_start = z_buffer } }); + defer res.deinit(); + + try std.testing.expectEqual(expectation, try res.getValue(@TypeOf(expectation), std.testing.io)); } } @@ -1494,15 +2414,11 @@ pub const Tensor = struct { return if (idx < 0) self.dim(axis_) + idx else idx; } - pub fn choose1d(self: Tensor, axis_: i64, i: i64) Tensor { - return self.slice1d(axis_, .{ .start = i, .end = i + 1 }).squeeze(axis_); - } - /// Concatenates the input Tensors along the given axis. - pub fn concatenate(tensors: []const Tensor, axis_: i64) Tensor { - meta.assert(tensors.len <= 32, "concatenate only supports up to 32 tensors, got {}", .{tensors.len}); - // TODO taggedVal - var buffer: [32]mlir.Value = undefined; + pub fn concatenate(tensors: []const Tensor, axis_: anytype) Tensor { + if (tensors.len == 1) return tensors[0]; + const ctx = Compiler.current(); + var buffer = ctx.alloc(*const mlir.Value, tensors.len); std.debug.assert(tensors.len <= buffer.len); std.debug.assert(tensors.len > 0); const a = tensors[0].axis(axis_); @@ -1515,8 +2431,7 @@ pub const Tensor = struct { } const res_shape = tensors[0]._shape.set(a, concatenated_dim); - const loc = tensors[0].getContext().mlirCtx().location(@src()).namedFmt(tensors[0].getContext().mlirCtx(), "axis={}", .{axis_}); - const op = dialect.stablehlo.concatenate(tensors[0].getContext().mlirCtx(), buffer[0..tensors.len], a, loc); + const op = stablehlo.concatenate(mlirCtx(), buffer, a, currentLoc()).appendTo(currentBlock()); // log.debug("concatenate({}, {}, {d}) -> {d}", .{ tensors[0], tensors[1], a, res_shape }); return _result(res_shape, op.result(0)); } @@ -1527,34 +2442,32 @@ pub const Tensor = struct { /// - Tensor.stack(&.{x, y, z}, 1, .layers) -> .{ .a, .layers, .b, .c } /// - Tensor.stack(&.{x, y, z}, .last, .layers) -> .{ .a, .b, .c, .layers } pub fn stack(tensors: []const Tensor, axis_: anytype, tag: anytype) Tensor { - // Note: we could ask the compilation context for some memory instead of stack allocating - meta.assert(tensors.len <= 32, "stack only supports up to 32 tensors, got {}", .{tensors.len}); - const shape0 = tensors[0]._shape; const res_shape = shape0.insertTag(axis_, 1, tag); for (tensors[1..]) |tensor| { - meta.assert(shape0.eqlWithTags(tensor._shape), "stack expects tensor shapes to match, got {} and {}", .{ tensor._shape, shape0 }); + stdx.debug.assert(shape0.eqlWithTags(tensor._shape), "stack expects tensor shapes to match, got {f} and {f}", .{ shape0, tensor._shape }); } - var reshaped: [32]Tensor = undefined; + const ctx = Compiler.current(); + var reshaped = ctx.alloc(Tensor, tensors.len); for (tensors, 0..) |tensor, i| { reshaped[i] = tensor.reshape(res_shape); } // Be careful here: we need to resolve ax before calling concatenate, // because we added an axis, so all - const ax = if (@TypeOf(axis_) == EnumLiteral and axis_ == .last) + const ax = if (@TypeOf(axis_) == @EnumLiteral() and axis_ == .last) shape0.rank() else shape0.axis(axis_); - return Tensor.concatenate(reshaped[0..tensors.len], ax); + return Tensor.concatenate(reshaped, ax); } /// Repeats a Tensor several times along the given axis. /// - /// * repeat1d(x, concat(&.{x, x, x, x}, axis); + /// * repeat1d(x, axis, 4) = concat(&.{x, x, x, x}, axis); /// * repeat1d([0, 1, 2, 3], 0, 2) = [0, 1, 2, 3, 0, 1, 2, 3] pub fn repeat1d(self: Tensor, axis_: anytype, n_rep: u63) Tensor { if (n_rep == 1) { @@ -1562,20 +2475,18 @@ pub const Tensor = struct { } const a = self.axis(axis_); - const broadshape = self._shape.insert(a + 1, .{n_rep}); - const repeat_dims = Shape.range(self.rank() + 1, self.dtype()).remove(a + 1); + const res_shape = self._shape.setDim(a, self.dim(a) * n_rep); - var res = self.broadcast(broadshape, repeat_dims.dims()).flatten(a); - // Restor the tag that has been lost by flatten. - res._shape._tags.set(a, self._shape.tag(a)); + const broadshape = self._shape.insert(a, .{n_rep}); + const repeat_dims = Shape.range(self.rank() + 1, self.dtype()).remove(a); - return res; + return self.broadcast(broadshape, repeat_dims.dims()).reshape(res_shape); } /// Repeats a Tensor several times along the given axes. pub fn repeat(self: Tensor, n_reps: []const u63) Tensor { // TODO: this should support the tagged syntax: x.repeat(.{ .a = 3, .b = 2}); - meta.assert(n_reps.len == self.rank(), "repeat expects tensor rank and 'n_reps' length to be equal, got {} and {}", .{ self.rank(), n_reps.len }); + stdx.debug.assert(n_reps.len == self.rank(), "repeat expects tensor rank and 'n_reps' length to be equal, got {} and {}", .{ self.rank(), n_reps.len }); var res = self; for (n_reps, 0..) |n_rep, a| { @@ -1586,20 +2497,60 @@ pub const Tensor = struct { return res; } + test repeat1d { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + const Local = struct { + fn repeat1d(x: Tensor, axis_: u3, n_reps: u32) Tensor { + return x.repeat1d(axis_, n_reps); + } + }; + + inline for (.{ + .{ [3]u8{ 1, 2, 3 }, Shape.init(.{3}, .u8), [6]u8{ 1, 2, 3, 1, 2, 3 }, 0, 2 }, + .{ [2][3]u8{ .{ 1, 2, 3 }, .{ 4, 5, 6 } }, Shape.init(.{ 2, 3 }, .u8), [2][6]u8{ .{ 1, 2, 3, 1, 2, 3 }, .{ 4, 5, 6, 4, 5, 6 } }, 1, 2 }, + }) |testcase| { + const input_data, const shape_, const expectation, const ax, const reps = testcase; + const input: Tensor = .fromShape(shape_); + + var exe = try zml.module.compile( + std.testing.allocator, + std.testing.io, + Local.repeat1d, + .{ input, ax, reps }, + platform, + .{}, + ); + defer exe.deinit(); + + var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), std.mem.sliceAsBytes(&input_data)); + defer input_buffer.deinit(); + + var output = try exe.eval(std.testing.allocator, std.testing.io, .{input_buffer}); + defer output.deinit(); + try std.testing.expectEqual(expectation, try output.getValue(@TypeOf(expectation), std.testing.io)); + } + } + /// Repeats in line each value along the given axis. /// - /// * stutter1d([0, 1, 2, 3], 0, 2) = [0, 0, 1, 1, 2, 2, 3, 3] - pub fn stutter1d(self: Tensor, axis_: i64, n_rep: u63) Tensor { + /// * stutter1d([0, 1, 2, 3], -1, 2) = [0, 0, 1, 1, 2, 2, 3, 3] + /// This is equivalent to repeat(ax+1) unless ax is the last axis. + pub fn stutter1d(self: Tensor, axis_: i8, n_rep: u32) Tensor { + stdx.debug.assert(n_rep > 0, "stutter1d expects a positive repeat count, got {}", .{n_rep}); const a = self.axis(axis_); const broadshape = self._shape.insert(a + 1, .{n_rep}); - const stutter_dims = Shape.range(self.rank() + 1, self.dtype()).remove(a + 1); + const res_shape = self._shape.setDim(a, self.dim(a) * @as(i64, n_rep)); - return self.broadcast(broadshape, stutter_dims.dims()).flatten(a); + const stutter_dims = Shape.range(self.rank() + 1, self.dtype()).remove(a + 1); + return self.broadcast(broadshape, stutter_dims.dims()).reshape(res_shape); } /// Repeats in line each value along the given axes. - pub fn stutter(self: Tensor, n_reps: []const u63) Tensor { - meta.assert(n_reps.len == self.rank(), "stutter expects tensor rank and 'n_reps' length to be equal, got {} and {}", .{ self.rank(), n_reps.len }); + pub fn stutter(self: Tensor, n_reps: []const u32) Tensor { + // TODO: this should support the tagged syntax: x.repeat(.{ .a = 3, .b = 2}); + stdx.debug.assert(n_reps.len == self.rank(), "stutter expects tensor rank and 'n_reps' length to be equal, got {} and {}", .{ self.rank(), n_reps.len }); var res = self; for (n_reps, 0..) |n_rep, a| { @@ -1609,82 +2560,116 @@ pub const Tensor = struct { return res; } + test stutter1d { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + const Local = struct { + fn stutter1d(x: Tensor, axis_: u3, n_reps: u32) Tensor { + return x.stutter1d(axis_, n_reps); + } + }; + + inline for (.{ + .{ [3]u8{ 1, 2, 3 }, Shape.init(.{3}, .u8), [6]u8{ 1, 1, 2, 2, 3, 3 }, 0, 2 }, + .{ [2][3]u8{ .{ 1, 2, 3 }, .{ 4, 5, 6 } }, Shape.init(.{ 2, 3 }, .u8), [2][6]u8{ .{ 1, 1, 2, 2, 3, 3 }, .{ 4, 4, 5, 5, 6, 6 } }, 1, 2 }, + .{ [2][3]u8{ .{ 1, 2, 3 }, .{ 4, 5, 6 } }, Shape.init(.{ 2, 3 }, .u8), [2][6]u8{ .{ 1, 2, 3, 1, 2, 3 }, .{ 4, 5, 6, 4, 5, 6 } }, 0, 2 }, + }) |testcase| { + const input_data, const shape_, const expectation, const ax, const reps = testcase; + const input: Tensor = .fromShape(shape_); + + var exe = try zml.module.compile( + std.testing.allocator, + std.testing.io, + Local.stutter1d, + .{ input, ax, reps }, + platform, + .{}, + ); + defer exe.deinit(); + + var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), std.mem.sliceAsBytes(&input_data)); + defer input_buffer.deinit(); + + var output = try exe.eval(std.testing.allocator, std.testing.io, .{input_buffer}); + defer output.deinit(); + try std.testing.expectEqual(expectation, try output.getValue(@TypeOf(expectation), std.testing.io)); + } + } + /// Returns a Tensor containing the element-wise negation of the input Tensor. pub fn negate(self: Tensor) Tensor { - const loc = self.getContext().mlirCtx().location(@src()); - const negate_op = dialect.stablehlo.negate(self.getContext().mlirCtx(), self.value(), loc); + const negate_op = stablehlo.negate(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, negate_op.result(0)); } /// Returns a Tensor containing the element-wise cosine of the input Tensor. pub fn cos(self: Tensor) Tensor { - const loc = self.getContext().mlirCtx().location(@src()); - const cosine_op = dialect.stablehlo.cosine(self.getContext().mlirCtx(), self.value(), loc); + const cosine_op = stablehlo.cosine(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, cosine_op.result(0)); } /// Returns a Tensor containing the element-wise sine of the input Tensor. pub fn sin(self: Tensor) Tensor { - const loc = self.getContext().mlirCtx().location(@src()); - const sine_op = dialect.stablehlo.sine(self.getContext().mlirCtx(), self.value(), loc); + const sine_op = stablehlo.sine(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, sine_op.result(0)); } /// Returns a Tensor containing the element-wise exponential operation of the input Tensor. pub fn exp(self: Tensor) Tensor { - const loc = self.getContext().mlirCtx().location(@src()); - const op = dialect.stablehlo.exponential(self.getContext().mlirCtx(), self.value(), loc); + const op = stablehlo.exponential(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } /// Returns a Tensor containing the element-wise logarithm operation of the input Tensor. pub fn log(self: Tensor) Tensor { - const loc = self.getContext().mlirCtx().location(@src()); - const op = dialect.stablehlo.log(self.getContext().mlirCtx(), self.value(), loc); + const op = stablehlo.log(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } /// Returns a Tensor containing the element-wise square-root of the input Tensor. pub fn sqrt(self: Tensor) Tensor { - const loc = self.getContext().mlirCtx().location(@src()); - const sqrt_op = dialect.stablehlo.sqrt(self.getContext().mlirCtx(), self.value(), loc); + const sqrt_op = stablehlo.sqrt(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, sqrt_op.result(0)); } /// Returns a Tensor containing the element-wise reverse square-root of the input Tensor. pub fn rsqrt(self: Tensor) Tensor { - const loc = self.getContext().mlirCtx().location(@src()); - const rsqrt_op = dialect.stablehlo.rsqrt(self.getContext().mlirCtx(), self.value(), loc); + const rsqrt_op = stablehlo.rsqrt(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, rsqrt_op.result(0)); } /// Returns a Tensor containing the element-wise hyperbolic tangent of the input Tensor. pub fn tanh(self: Tensor) Tensor { - const loc = self.getContext().mlirCtx().location(@src()); - const tanh_op = dialect.stablehlo.tanh(self.getContext().mlirCtx(), self.value(), loc); + const tanh_op = stablehlo.tanh(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, tanh_op.result(0)); } /// Returns a Tensor containing the element-wise exponential minus one operation of the input Tensor. pub fn exponentialMinusOne(self: Tensor) Tensor { - const loc = self.getContext().mlirCtx().location(@src()); - const expm1_op = dialect.stablehlo.exponential_minus_one(self.getContext().mlirCtx(), self.value(), loc); + const expm1_op = stablehlo.exponential_minus_one(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, expm1_op.result(0)); } - pub const ArangeArgs = HostBuffer.ArangeArgs; + pub const ArangeArgs = struct { + start: i64 = 0, + end: i64, + step: i64 = 1, + }; /// Returns a Tensor containing evenly spaced values within a given interval. pub fn arange(args: ArangeArgs, dt: DataType) Tensor { - meta.assert(args.start < args.end, "arange expects 'args.start' to be less than 'args.end', got {} and {}", .{ args.start, args.end }); - meta.assert(args.step > 0, "arange expects 'args.step' to be positive, got {}", .{args.step}); - - const ctx = CompilationContext.current(); - const loc = ctx.mlirCtx().location(@src()).namedFmt(ctx.mlirCtx(), "{}, dtype={}", .{ args, dt }); + stdx.debug.assert(args.start <= args.end, "arange expects 'args.start' to be less than 'args.end', got {} and {}", .{ args.start, args.end }); + stdx.debug.assert(args.step > 0, "arange expects 'args.step' to be positive, got {}", .{args.step}); const n_steps = std.math.divCeil(i64, args.end - args.start, args.step) catch unreachable; const sh = Shape.init(.{n_steps}, dt); - var op = dialect.stablehlo.iota(ctx.mlirCtx(), 0, mlir.ext.mlirType(ctx.mlirCtx(), sh), loc); + var op = stablehlo.iota( + mlirCtx(), + 0, + mlirx.Type.rankedTensor(mlirCtx(), sh), + currentLoc(), + ).appendTo(currentBlock()); var res = _result(sh, op.result(0)); if (args.step != 1) { @@ -1698,15 +2683,48 @@ pub const Tensor = struct { return res; } - /// Returns a Tensor containing values in increasing order starting from 0 along the given axis. - pub fn iota(sh: Shape, dt: DataType, axis_: anytype) Tensor { - const ctx = CompilationContext.current(); - const loc = ctx.mlirCtx().location(@src()).namedFmt(ctx.mlirCtx(), "iota({}, {})", .{ sh, axis_ }); + test arange { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + const exe = try zml.module.compile( + std.testing.allocator, + std.testing.io, + Tensor.arange, + .{ .{ .start = 2, .end = 26, .step = 4 }, .u32 }, + platform, + .{}, + ); + defer exe.deinit(); + + var output = try exe.eval(std.testing.allocator, std.testing.io, {}); + defer output.deinit(); + + try zml.testing.expectClose( + std.testing.io, + zml.Slice.initConst(.init(.{6}, .u32), @ptrCast(&[_]u32{ 2, 6, 10, 14, 18, 22 })), + output, + .exact_match, + ); + } + + /// Returns a Tensor containing values in increasing order starting from 0 along the given axis. + /// + /// The output dtype will be `.i32`, unless the given axis has a too big dimension, in that case we use `.i64`. + /// In most program this shouldn't matter, because typically this will be used in a comparison, + /// or explicitly converted by the user to do floating point arithmetic. + pub fn iota(sh: Shape, axis_: anytype) Tensor { const a = sh.axis(axis_); + const dt: DataType = if (sh.dim(a) <= std.math.maxInt(i32)) .i32 else .i64; const res_shape = sh.withDtype(dt); - var op = dialect.stablehlo.iota(ctx.mlirCtx(), @intCast(a), mlir.ext.RankedTensorType.fromShape(ctx.mlirCtx(), res_shape).as(mlir.Type).?, loc); - return _result(res_shape, op.result(0)); + + var op = stablehlo.iota( + mlirCtx(), + a, + mlirx.Type.rankedTensor(mlirCtx(), res_shape), + currentLoc(), + ).appendTo(currentBlock()); + return _resultPropagateSharding(res_shape, op.result(0)); } pub const LinspaceArgs = struct { @@ -1717,15 +2735,17 @@ pub const Tensor = struct { /// Returns a Tensor containing 'args.steps' values evenly spaced from 'args.start' to 'args.end', inclusive. pub fn linspace(args: LinspaceArgs, dt: DataType) Tensor { - meta.assert(args.start < args.end, "linspace expects 'args.start' to be less than 'args.end', got {} and {}", .{ args.start, args.end }); - meta.assert(args.steps > 0, "linspace expects 'args.steps' to be positive, got {}", .{args.steps}); - meta.assert(dt.isFloat(), "linspace expects type to be a float, got {} (hint: use arange instead)", .{dt}); - - const ctx = CompilationContext.current(); - const loc = ctx.mlirCtx().location(@src()).namedFmt(ctx.mlirCtx(), "linspace({}, dtype={})", .{ args, dt }); + stdx.debug.assert(args.start < args.end, "linspace expects 'args.start' to be less than 'args.end', got {} and {}", .{ args.start, args.end }); + stdx.debug.assert(args.steps > 0, "linspace expects 'args.steps' to be positive, got {}", .{args.steps}); + stdx.debug.assert(dt.isFloat(), "linspace expects type to be a float, got {} (hint: use arange instead)", .{dt}); const sh = Shape.init(.{args.steps}, dt); - var iota_op = dialect.stablehlo.iota(ctx.mlirCtx(), 0, mlir.ext.mlirType(ctx.mlirCtx(), sh), loc); + var iota_op = stablehlo.iota( + mlirCtx(), + 0, + mlirx.Type.rankedTensor(mlirCtx(), sh), + currentLoc(), + ).appendTo(currentBlock()); var res = _result(sh, iota_op.result(0)); if (args.steps != 1) { @@ -1739,69 +2759,81 @@ pub const Tensor = struct { return res; } - /// Returns a 0d Tensor with the given value. + /// Returns a 0-rank Tensor with the given value. pub fn scalar(val: anytype, dt: DataType) Tensor { - return Tensor.constant(.{}, Data.init(dt, val)); + const data = DataType.Value.init(dt, val); + switch (dt.class()) { + .float => stdx.debug.assert(!std.math.isNan(val), "scalar(NaN) is probably due to compiling a model with an uninitialized field", .{}), + else => {}, + } + return Tensor.constant(data); + } + + test scalar { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + const Local = struct { + pub fn _fwd() [6]Tensor { + var res: [6]Tensor = undefined; + const dtypes = .{ .bool, .u8, .i32, .f32, .bf16, .u64 }; + inline for (0..6) |i| res[i] = scalar(0, dtypes[i]); + return res; + } + }; + + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._fwd, .{}, platform, .{}); + defer exe.deinit(); } /// Returns a constant Tensor with the given value. - pub fn constant(dimz: anytype, val: Data) Tensor { - const sh = Shape.init(dimz, val.dtype()); - const singleton_sh = Shape.init(.{}, val.dtype()); - const ctx = CompilationContext.current().mlirCtx(); - const loc = ctx.location(@src()).namedFmt(ctx, "dims={d}, value={}", .{ sh, val }); - const result_type = mlir.ext.RankedTensorType.fromShape(ctx, singleton_sh); - const elem_type = mlir.ext.denseElementAttrType(val.dtype()); - var constant_op = dialect.stablehlo.constant(ctx, result_type, elem_type, val.constSlice(), loc); - if (sh.rank() > 0) { - constant_op = dialect.stablehlo.broadcast_in_dim(ctx, constant_op.result(0), &.{}, mlir.ext.RankedTensorType.fromShape(ctx, sh).as(mlir.Type).?, loc); - } - return _result(sh, constant_op.result(0)); + pub fn constant(val: DataType.Value) Tensor { + const op = stablehlo.constant( + mlirCtx(), + &.{}, + mlirx.Type.fromDType(mlirCtx(), val.dtype()), + val.asBytes(), + currentLoc(), + ).appendTo(currentBlock()); + return _result(.init(&.{}, val.dtype()), op.result(0)); + } + + pub fn zeroes(sh: Shape) Tensor { + const result = Tensor.constant(sh.dtype().zero()).broad(sh); + return _resultPropagateSharding(sh, result.value()); + } + + pub fn uninitialized(sh: Shape) Tensor { + const ctx = Compiler.current(); + const buffer_type = mlir.Type.memRef( + mlirx.Type.fromDType(ctx.mlir_ctx, sh.dtype()), + sh.dims(), + null, + null, + ); + const buffer = stablehlo.createBuffer(ctx.mlir_ctx, buffer_type, ctx.location) + .appendTo(currentBlock()); + const tensor = stablehlo.unpin(ctx.mlir_ctx, buffer.result(0), ctx.location) + .appendTo(currentBlock()); + return _resultPropagateSharding(sh, tensor.result(0)); } /// Embeds a buffer with concrete values into an Mlir program. - pub fn constantTensor(val: HostBuffer) Tensor { - const ctx = CompilationContext.current().mlirCtx(); - const result_type = mlir.ext.RankedTensorType.fromShape(ctx, val.shape()); - const loc = ctx.location(@src()); - const elem_type = mlir.ext.denseElementAttrType(val.dtype()); - const constant_op = dialect.stablehlo.constant(ctx, result_type, elem_type, val.data, loc); - return _result(val.shape(), constant_op.result(0)); + pub fn constantTensor(sh: Shape, bytes_: []const u8) Tensor { + const elem_type = mlirx.Type.fromDType(mlirCtx(), sh.dtype()); + //const elem_type = mlirx.denseElementAttrType(val.dtype()) orelse std.debug.panic("constantTensor expects a dtype that can be serialized to MLIR, like f32 or i32, got {f}", .{val.shape()}); + const constant_op = stablehlo.constant(mlirCtx(), sh.dims(), elem_type, bytes_, currentLoc()).appendTo(currentBlock()); + return _resultPropagateSharding(sh, constant_op.result(0)); } /// Returns a Tensor containing the result of the outer product between the input Tensors. pub fn outer(self: Tensor, other: Tensor) Tensor { - meta.assert(self.rank() < 2 and other.rank() < 2 and self.rank() + other.rank() != 0, "outer expects tensor ranks to be at most 1, got {} and {}", .{ self.rank(), other.rank() }); - if (self.rank() + other.rank() == 1) { return self.mul(other); } - const dimz = .{ self.dim(0), other.dim(0) }; - const left = self.broadcast(Shape.init(dimz, self.dtype()), &.{0}); - const right = other.broadcast(Shape.init(dimz, other.dtype()), &.{1}); - return left.mul(right); - } - - /// Creates a 2D Tensor with its diagonal set to the input vector. - pub fn diag(self: Tensor) Tensor { - meta.assert(self.rank() == 1, "diag only supports tensor of rank 1, got {}", .{self.rank()}); // TODO: support 2D tensors - // - const indices = Tensor.arange(.{ .end = self.dim(0) }, .i64); - const sh = Shape.init(.{ self.dim(0), self.dim(0) }, self.dtype()); - const indices_1 = indices.broadcast(sh, &.{1}); - const indices_0 = indices.broadcast(sh, &.{0}); - - const loc = self.getContext().mlirCtx().location(@src()); - // TODO: handle more complex cases (2D, specifying `diagonal` index). - const op = dialect.stablehlo.select( - self.getContext().mlirCtx(), - indices_1.cmp(.EQ, indices_0).value(), - self.broadcast(sh, &.{1}).value(), - Tensor.constant(self.dims(), self.dtype().zero()).value(), - loc, - ); - return _result(sh, op.result(0)); + const res_shape = self.shape().outer(other.shape()); + return self.broad(res_shape).mul(other.broad(res_shape)); } /// Given a tensor and a shape of the same rank, @@ -1813,18 +2845,26 @@ pub const Tensor = struct { /// you will lose the tags. /// To avoid use favorise `.broad(shape)` when working with tagged tensors. pub fn broadcast(self: Tensor, output_shape: Shape, axes_: []const i64) Tensor { + stdx.debug.assert(axes_.len == self.rank(), "broadcast expects axes_ to map all axes from self to axes of the output shape, got broadcast({f}, {f}, {any})", .{ self, output_shape, axes_ }); + for (0.., axes_) |self_ax, other_ax| { + const d = self.dim(self_ax); + stdx.debug.assert(d == 1 or d == output_shape.dim(other_ax), "broadcast expects shape axes to either be 1-sized or to match the target size. got broadcast({f}, {f}, {any}), error on self axis {d} mapping to other axis {d}", .{ self, output_shape, axes_, self_ax, other_ax }); + } + const res_shape = output_shape.withDtype(self.dtype()); - - const result_type = mlir.ext.RankedTensorType.fromShape(self.getContext().mlirCtx(), res_shape).as(mlir.Type).?; - const loc = self.getContext().mlirCtx().location(@src()).namedFmt(self.getContext().mlirCtx(), "broadcast({any}, axes={d})", .{ res_shape, axes_ }); - const broadcast_op = dialect.stablehlo.broadcast_in_dim(self.getContext().mlirCtx(), self.value(), axes_, result_type, loc); - + if (std.mem.eql(i64, self.dims(), output_shape.dims())) { + // No broadcast needed. We don't emit a new stablehlo value + // but we propagate output_shape tags. + return _result(res_shape, self.value()); + } + const result_type = mlirx.Type.rankedTensor(mlirCtx(), res_shape); + const broadcast_op = stablehlo.broadcast_in_dim(mlirCtx(), self.value(), axes_, result_type, currentLoc()).appendTo(currentBlock()); return _result(res_shape, broadcast_op.result(0)); } /// Broadcasts a Tensor to the given shape, adding axes at the beginning. pub fn broadcastLeft(self: Tensor, output_shape: Shape) Tensor { - meta.assert(self.rank() <= output_shape.rank(), "broadcastLeft expects tensor rank to be less than output tensor rank, got {} and {}", .{ self.rank(), output_shape.rank() }); + stdx.debug.assert(self.rank() <= output_shape.rank(), "broadcastLeft expects tensor rank to be less than output tensor rank, got {d} and {d}", .{ self.rank(), output_shape.rank() }); const a = output_shape.rank() - self.rank(); if (self.rank() == output_shape.rank() and std.mem.eql(i64, self.dims(), output_shape.dims())) { @@ -1836,7 +2876,7 @@ pub const Tensor = struct { /// Broadcasts a Tensor to the given shape, adding axes at the end. pub fn broadcastRight(self: Tensor, output_shape: Shape) Tensor { - meta.assert(self.rank() <= output_shape.rank(), "broadcastRight expects tensor rank to be less than output tensor rank, got {} and {}", .{ self.rank(), output_shape.rank() }); + stdx.debug.assert(self.rank() <= output_shape.rank(), "broadcastRight expects tensor rank to be less than output tensor rank, got {d} and {d}", .{ self.rank(), output_shape.rank() }); if (self.rank() == output_shape.rank() and self._shape.eql(output_shape)) { return self; @@ -1847,97 +2887,116 @@ pub const Tensor = struct { /// Broadcasts a Tensor to the given shape, extending dimensions if needed. pub fn broad(self: Tensor, other: Shape) Tensor { + // TODO: broad is too restrictive because sometime you only want to specify one specific axis + // Note: if you code below, make sure to update Shape.canBroadcastTo. + stdx.debug.assert(self._shape.canBroadcastTo(other), "Can't broadcast {f} to {f}", .{ self, other }); + + // Already the right shape + if (std.mem.eql(i64, self.dims(), other.dims())) return self; + + if (ops.LoweringCompatibility.preserveIntegerScalarBroadcast(self, other)) |rewritten| return rewritten; + // Non ambiguous broadcasting + // TODO: broad is error prone because of this: + // it will happily broadcast .{ .a = 10, .b = 1 } to .{ .b = 10, .a = 5 } if (self._shape.rank() == 0 or self._shape.rank() == other.rank()) { - return self.broadcast(other, Shape.range(self._shape.rank(), self.dtype()).dims()); + return self.broadcast(other, constants.AXES_IOTA[0..self.rank()]); } // check that each axis of self maps to an axis of other - var axes_: std.BoundedArray(i64, MAX_RANK) = .{}; + var axes_: stdx.BoundedArray(i64, constants.MAX_RANK) = .empty; for (self._shape.tags()) |t| { - if (t != Shape.TagUnknown) { - if (other.hasTag(t)) |ax| { - axes_.appendAssumeCapacity(@intCast(other.axis(ax))); - } else { - std.debug.panic("Can't broadcast {} to {}", .{ self, other }); - } - } + axes_.appendAssumeCapacity(@intCast(other.axis(t))); } return self.broadcast(other, axes_.constSlice()); } + pub fn optimizationBarrier(self: Tensor) Tensor { + const ctx = Compiler.current(); + + const op = stablehlo.optimizationBarrier( + ctx.mlir_ctx, + &.{self.value()}, + &.{self.value().type_()}, + ctx.location, + ).appendTo(ctx.currentScope().block); + + return _result(self.shape(), op.result(0)); + } + /// Reshapes the input Tensor with the given shape. pub fn reshape(self: Tensor, output_shape_: anytype) Tensor { const output_shape = self._shape.reshape(output_shape_); - const tensor_type = mlir.ext.RankedTensorType.fromShape(self.getContext().mlirCtx(), output_shape); - const loc = self.getContext().mlirCtx().location(@src()).namedFmt(self.getContext().mlirCtx(), "reshape({any})", .{output_shape}); - const reshape_value = dialect.stablehlo.reshape(self.getContext().mlirCtx(), self.value(), tensor_type, loc); + const tensor_type = mlirx.Type.rankedTensor(mlirCtx(), output_shape); + const reshape_value = stablehlo.reshape(mlirCtx(), self.value(), tensor_type, currentLoc()).appendTo(currentBlock()); return _result(output_shape, reshape_value.result(0)); } - pub const Pad1dOpts = struct { low: i64, high: i64, interior: i64 = 0 }; - - /// Pads the input Tensor with the given value over the given axis. - pub fn pad1d(self: Tensor, axis_: i8, pad_value: anytype, opts: Pad1dOpts) Tensor { - const ZEROS = [_]i64{0} ** MAX_RANK; - var padding_low = ZEROS; - var padding_high = ZEROS; - var padding_interior = ZEROS; - const a = self.axis(axis_); - padding_low[a] = opts.low; - padding_high[a] = opts.high; - padding_interior[a] = opts.interior; - - const rk = self.rank(); - return self.pad( - pad_value, - .{ .low = padding_low[0..rk], .high = padding_high[0..rk], .interior = padding_interior[0..rk] }, - ); + /// Converts the given 1 element Tensor into a 0-rank Tensor. + pub fn asScalar(self: Tensor) Tensor { + stdx.debug.assert(self.count() == 1, "Tensor.asScalar expects an input with exactly 1-element got {f}", .{self}); + return if (self.rank() == 0) self else self.reshape(.{}); } + pub const Pad = struct { + low: i64 = 0, + high: i64 = 0, + interior: i64 = 0, + }; + /// Pads the input Tensor with the given values. - pub fn pad(self: Tensor, pad_value: anytype, opts: dialect.stablehlo.PadOpts) Tensor { - meta.assert(opts.low.len == self.rank(), "pad expects tensor rank and 'opts.low' length to be equal, got {} and {}", .{ self.rank(), opts.low.len }); - meta.assert(opts.high.len == self.rank(), "pad expects tensor rank and 'opts.high' length to be equal, got {} and {}", .{ self.rank(), opts.high.len }); - const ZEROS = [_]i64{0} ** MAX_RANK; - const interior = opts.interior orelse ZEROS[0..self.rank()]; + /// Usage: x.pad(0, .{ .a = .{ .low = 1, .high = 1 }}); + pub fn pad(self: Tensor, padding_value: anytype, paddings: anytype) Tensor { + const _paddings = self.shape().parseAxesOptions(Pad, paddings, .{}); - var new_shape = self._shape; + const ZEROS: [constants.MAX_RANK]i64 = @splat(0); + var low = ZEROS; + var high = ZEROS; + var interior = ZEROS; - for (0..self.rank()) |i| { - const d = self.dim(i) + opts.low[i] + (@max(self.dim(i) - 1, 0) * interior[i]) + opts.high[i]; - new_shape = new_shape.set(i, d); + var res_shape = self._shape; + for (_paddings.constSlice(), 0..) |padding, i| { + low[i] = padding.low; + high[i] = padding.high; + interior[i] = padding.interior; + + var d: i64 = self.dim(i); + d += low[i] + (@max(d - 1, 0) * interior[i]) + high[i]; + res_shape._dims.set(i, d); } - const loc = self.getContext().mlirCtx().location(@src()).namedFmt(self.getContext().mlirCtx(), "pad(value={}, opts={})", .{ pad_value, opts }); - var full_opts = opts; - full_opts.interior = opts.interior orelse ZEROS[0..self.rank()]; - const pad_value_tensor = Tensor.scalar(pad_value, self.dtype()); - const pad_op = dialect.stablehlo.pad(self.getContext().mlirCtx(), self.value(), pad_value_tensor.value(), full_opts, loc); + const rk = self.rank(); + const pad_op = stablehlo.pad( + mlirCtx(), + self.value(), + Tensor.scalar(padding_value, self.dtype()).value(), + .{ .low = low[0..rk], .high = high[0..rk], .interior = interior[0..rk] }, + currentLoc(), + ).appendTo(currentBlock()); - return _result(new_shape, pad_op.result(0)); + return _result(res_shape, pad_op.result(0)); } /// Inserts 1-dim axes at the given position, with the given tags. /// `.{.a = 5, .b = 4}.insert(.b, .{ .c, .d }) -> .{ .a = 5, .c = 1, .d = 1, .b = 4 }` pub fn insertAxes(self: Tensor, axis_: anytype, tags: anytype) Tensor { const tags_ = Shape.parseTags(tags); - const ax = if (@TypeOf(axis_) == EnumLiteral and axis_ == .last) + const ax = if (@TypeOf(axis_) == @EnumLiteral() and axis_ == .last) self.rank() else self.axis(axis_); var res_shape = self._shape; - const ones = [_]i64{1} ** MAX_RANK; + const ones: [constants.MAX_RANK]i64 = @splat(1); res_shape._dims.insertSlice(ax, ones[0..tags_.len]) catch unreachable; - res_shape._tags.insertSlice(ax, tags_.constSlice()) catch unreachable; + res_shape._tags.insertSlice(ax, tags_) catch unreachable; return self.reshape(res_shape); } /// Appends a 1-dim axis, with the given tag. pub fn appendAxes(self: Tensor, t: anytype) Tensor { - meta.assert(self.rank() < Tensor.MAX_RANK - t.len, "appendAxis expects tensor rank to be small enough in order to extend it, got {} and {} (max is {})", .{ self.rank(), t.len, Tensor.MAX_RANK }); + // stdx.debug.assert(self.rank() < Tensor.MAX_RANK - t.len, "appendAxis expects tensor rank to be small enough in order to extend it, got {} and {} (max is {})", .{ self.rank(), t.len, Tensor.MAX_RANK }); return self.insertAxes(.last, t); } @@ -1945,7 +3004,7 @@ pub const Tensor = struct { /// Drops a 1-dim axis at the given index pub fn squeeze(self: Tensor, axis_: anytype) Tensor { const a = self.axis(axis_); - meta.assert(self.dim(a) == 1, "squeeze expects axis to be squeezed to have a dimension of 1, got {}", .{self.dim(a)}); + stdx.debug.assert(self.dim(a) == 1, "squeeze expects axis to be squeezed to have a dimension of 1, got {}", .{self.dim(a)}); const new_shape = self._shape.remove(a); // log.debug("squeeze({}, {d}={d}) -> ({})", .{ self, axis, a, new_shape }); @@ -1957,185 +3016,108 @@ pub const Tensor = struct { pub fn reverse(self: Tensor, axes_: anytype) Tensor { const actual_axes = self._shape.axes(axes_); - const loc = self.getContext().mlirCtx().location(@src()).namedFmt(self.getContext().mlirCtx(), "reverse({any})", .{axes_}); - const reverse_op = dialect.stablehlo.reverse(self.getContext().mlirCtx(), self.value(), toI64(actual_axes.constSlice()), loc); + const reverse_op = stablehlo.reverse(mlirCtx(), self.value(), toI64(actual_axes.constSlice()).constSlice(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, reverse_op.result(0)); } - pub const GatherOpts = struct { indices_are_sorted: bool = false }; + pub const GatherOpts = ops.GatherOpts; - /// For each coordinate in `indices`, - /// `gatherValues` extracts a single value of the given tensor. + /// `gather` extracts slices from the given tensor at the specified offsets. + /// example: `values.gather(.{ .a = idx }, .{})` + /// + /// * indices is a named list of integer tensors: eg `.{ .a = idx }`. + /// Each names specify a gathering axis, it must refer to an axis of self, + /// and the corresponding idx Tensor must contains valid indices into axis .a. + /// All indices must have the same shape or broadcast to the same shape. /// - /// * axes_ is a single axis, or a tuple of axis: .b, or .{ .b, .c } - /// * indices is an integer tensor /// * result is a tensor whose shape is similar to the input shape /// where the gathered axes have been replaced by axes from 'indices'. /// /// Some example input for the base case where we work on one axis: - /// - gatherValues(f:[a]->float, .a, ind:[n]->int)[n] == f[ind[n]] - /// - gatherValues(f:[a, b], .a, ind:[n])[n, b] == f[ind[n], b] - /// - gatherValues(f: [a,b,c], .{.b}, ind: [n,m])[a, n, m, c] == f[a, ind[n, m], c] + /// - gather(f:[a], .{ .a = idx:[n]})[n] == f[idx[n]] + /// - gather(f:[a, b], .a, idx:[n])[n, b] == f[idx[n], b] + /// - gather(f:[a,b,c], .{.b = idx:[n,m]})[a, n, m, c] == f[a, idx[n, m], c] /// /// If an axis in common between `self` and `indices`, /// it is treated as a "batching" axis, meaning that semantically - /// the operator is doing a gatherValues one time per dimension of this axis: - /// - gatherValues(f: [a,b,c], .{.b}, ind: [a,n])[a, n] == f[a, ind[a, n]] + /// the operator is doing a gather one time per dimension of this axis: + /// - gather(f: [a,b,c], .{.b=idx: [a,n]})[a, n] == f[a, idx[a, n]] /// - /// It is an error to have an axis present in `self`, `axes_` and `indices`. + /// It's possible to pass several indices: + /// - gather(f: [a,b,c], .{.b=idx_b[n], .c=idx_c[n]})[a, n] == f[a, idx_b[n], idx_c[n]] + /// - gather(f: [a,b,c,d], .{.b=idx_b[a,n], .c=idx_c[a, n]})[a, n, d] == f[a, idx_b[a, n], idx_c[a, n], d] /// - /// If several axes are passed, then the last axis of indices is treated as coordinates: - /// - gatherValues(f: [a,b,c], .{.b, .c}, ind: [n,2])[a, n] == f[a, ind[n][0], ind[n][1]] - /// - gatherValues(f: [a,b,c,d], .{.b, .c}, ind: [a, n,2])[a, n, d] == f[a, ind[a, n][0], ind[a, n][1], d] + /// If `self` isn't tagged, you can use `gather_` to specify gathered axis by their position but batching won't be available. /// - /// It is possible to use gatherValues without tags, but batching won't be available. - pub fn gatherValues(self: Tensor, coord_axes: anytype, indices: Tensor, opts: GatherOpts) Tensor { - // scoped_log.debug("gatherValues({}, {any}, {})", .{ self, coord_axes, indices }); - const single_coord, const coord_axes_ = _parseGatherCoord(self, coord_axes); - - meta.assert(coord_axes_.len > 0, "gatherValues expects 1 or more axes to operate one, received none. Example: `x.gatherValues(.a, indices, .{{}})`", .{}); - for (coord_axes_.constSlice(), 0..) |a, i| { - if (i > 0) { - meta.assert(a == coord_axes_.get(i - 1) + 1, "gatherValues expects 'coord_axes' to be sequential. But {any} aren't sequential in {}", .{ coord_axes, self }); - } + /// For performance it's better to have batching and gathering axes of `self` be the first one, + /// so that gather can + pub fn gather(self: Tensor, _indices: anytype, opts: GatherOpts) Tensor { + const idx_per_axis, const idx_tags = Shape.parseStruct(Tensor, _indices); + var idx_axes: Shape.AxesArray = .empty; + for (idx_tags.slice()) |t| { + idx_axes.appendAssumeCapacity(self.axis(t)); } - const AxisKind = enum { batching, offset, collapsed, indices }; - var self_kind: std.BoundedArray(AxisKind, MAX_RANK) = .{}; - var indices_batch_axes: Shape.DimsArray = .{}; - for (self._shape.tags(), 0..self.rank()) |t, self_ax| { - const maybe_coord_ax = std.mem.indexOfScalar(u3, coord_axes_.constSlice(), @intCast(self_ax)); - if (indices._shape.hasTag(t)) |id_ax| { - // tag is both in self and indices -> it's a batching dim - // Note: tags are required for batching. - self_kind.appendAssumeCapacity(.batching); - indices_batch_axes.appendAssumeCapacity(id_ax); - meta.assert(maybe_coord_ax == null, "gatherValues expects axes to appear at most twice. Axis {s} has been found both in 'self={any}', in 'coord_axes_={any}' and in 'indices={}'", .{ self._shape._tags.get(self_ax), self, coord_axes, indices }); - } else if (maybe_coord_ax) |_| { - // for gatherValues we collapsed all gathered axes - // (contrary to gatherSlices where we collapse none) - self_kind.appendAssumeCapacity(.collapsed); - } else { - self_kind.appendAssumeCapacity(.offset); - } - } - - // When we receive several coord_axes we need an extra dimension to store - // one index per axis, which makes the coordinates of one value. - // Otherwi se stablehlo uses the "indices.rank()" default value. - const index_coord_axis = if (single_coord) - indices.rank() - else blk: { - const ax = indices._shape.hasTag(.coord) orelse indices._shape.axis(-1); - meta.assert(indices.dim(ax) == coord_axes_.len, "gatherValues with axes={any}, expects indices to be of shape [..., {}], got: {}", .{ coord_axes, coord_axes_.len, indices }); - break :blk ax; - }; - - // compute res shape - var res_shape = Shape.init(.{}, self.dtype()); - var res_kind: std.BoundedArray(AxisKind, MAX_RANK) = .{}; - for (self_kind.constSlice(), 0..) |kind, ax_usize| { - const ax: u3 = @intCast(ax_usize); - if (ax == coord_axes_.get(0)) { - // The first val_ax is special cause this is the place where we insert indices axes. - for (indices._shape.tags(), 0..indices.rank()) |t, id_ax| { - if (id_ax == index_coord_axis) continue; - if (std.mem.indexOfScalar(i64, indices_batch_axes.constSlice(), @intCast(id_ax))) |_| { - // batching dim are already in res - continue; - } - - res_shape = res_shape.appendDim(indices.dim(id_ax), t); - res_kind.appendAssumeCapacity(.indices); - } - } - switch (kind) { - .collapsed => continue, - else => { - res_shape = res_shape.appendDim(self.dim(ax), self._shape.tag(ax)); - res_kind.appendAssumeCapacity(kind); - }, - } - } - - // This is not a gather, but a dynamicSlice. - // Sometimes the backend recognize this pattern, but not always. - // So let us handle that. - if (indices.count() == 1) { - return self.dynamicSlice1d(coord_axes_.get(0), 1, indices.flattenAll().squeeze(0)).reshape(res_shape); - } - - var slice_dims: Shape.DimsArray = .{}; - for (self_kind.constSlice(), self.dims()) |k, d| { - slice_dims.appendAssumeCapacity(switch (k) { - .batching, .collapsed => 1, - .offset => d, - .indices => unreachable, - }); - } - - // scoped_log.debug("gatherValues --> {} {any}", .{ res_shape, res_kind.constSlice() }); - const loc = self.getContext().mlirCtx().location(@src()); - const gather_op = dialect.stablehlo.gather( - self.getContext().mlirCtx(), - self.value(), - indices.value(), - slice_dims.constSlice(), - loc, - .{ - .offset_dims = _collectAxes(AxisKind, res_kind, .offset).constSlice(), - .collapsed_slice_dims = _collectAxes(AxisKind, self_kind, .collapsed).constSlice(), - .operand_batching_dims = _collectAxes(AxisKind, self_kind, .batching).constSlice(), - .start_indices_batching_dims = indices_batch_axes.constSlice(), - .start_index_map = _collectAxes(AxisKind, self_kind, .collapsed).constSlice(), - .index_vector_dim = index_coord_axis, - .indices_are_sorted = opts.indices_are_sorted, - }, - ); - - const mlir_shape = fromMlirValue(gather_op.result(0)).shape(); - meta.assert(mlir_shape.eql(res_shape), "gatherValues expects that batching indices appear in the same order in 'self' and 'indices', got: self={}, indices={}. You should transpose one or the other.", .{ self, indices }); - return _result(res_shape, gather_op.result(0)); + // TODO: sort indices following self.shape instead of asking the user to do it. + return ops.gather(self, idx_axes.slice(), idx_per_axis.slice(), opts); } - test gatherValues { + test gather { const zml = @import("zml.zig"); const platform = zml.testing.env(); + const Local = struct { + pub fn _idx(idx_shape: anytype) Tensor { + return Tensor.constant(.{ .i32 = 0 }).broad(Shape.init(idx_shape, .i64)); + } + }; + + const idx = Local._idx; + { // Only test shapes - var comp = try zml.module.CompilationContext.init(std.heap.page_allocator, "test", platform); + var comp: zml.Compiler = .init(std.testing.allocator, std.testing.io, platform, .{}); defer comp.deinit(); comp.activate(); defer comp.deactivate(); + const block = mlir.Block.init(&.{}, &.{}); + const scope = comp.pushBlock(block); + defer scope.pop(); + inline for (.{ - .{ .{ .a = 10 }, .a, .{}, .{} }, - .{ .{ .a = 10 }, .a, .{ .n = 8 }, .{ .n = 8 } }, - .{ .{ .a = 10, .b = 20 }, .a, .{}, .{ .b = 20 } }, - .{ .{ .a = 10, .b = 20 }, .a, .{ .n = 8 }, .{ .n = 8, .b = 20 } }, - .{ .{ .a = 10, .b = 20 }, 0, .{ .n = 8 }, .{ .n = 8, .b = 20 } }, + .{ .{ .a = 10 }, .{ .a = idx(.{}) }, .{} }, + .{ .{ .a = 10 }, .{ .a = idx(.{ .n = 8 }) }, .{ .n = 8 } }, + .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{}) }, .{ .b = 20 } }, + .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{ .n = 8 }) }, .{ .n = 8, .b = 20 } }, + // .{ .{ .a = 10, .b = 20 }, 0, idx(.{ .n = 8 }), .{ .n = 8, .b = 20 } }, // Favor val shape, instead of indices shape. - .{ .{ .a = 10, .b = 20 }, .b, .{ .n = 8 }, .{ .a = 10, .n = 8 } }, - .{ .{ .a = 10, .b = 20, .c = 30 }, .b, .{ .n = 8 }, .{ .a = 10, .n = 8, .c = 30 } }, + .{ .{ .a = 10, .b = 20 }, .{ .b = idx(.{ .n = 8 }) }, .{ .a = 10, .n = 8 } }, + .{ .{ .a = 10, .b = 20, .c = 30 }, .{ .b = idx(.{ .n = 8 }) }, .{ .a = 10, .n = 8, .c = 30 } }, // batching axes are implicits. - .{ .{ .a = 10, .b = 20 }, .b, .{ .a = 10 }, .{ .a = 10 } }, - .{ .{ .a = 10, .b = 20 }, .a, .{ .b = 20 }, .{ .b = 20 } }, - .{ .{ .a = 10, .b = 20 }, .b, .{ .a = 10, .n = 8 }, .{ .a = 10, .n = 8 } }, + .{ .{ .a = 10, .b = 20 }, .{ .b = idx(.{ .a = 10 }) }, .{ .a = 10 } }, + .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{ .b = 20 }) }, .{ .b = 20 } }, + .{ .{ .a = 10, .b = 20 }, .{ .b = idx(.{ .a = 10, .n = 8 }) }, .{ .a = 10, .n = 8 } }, // stablehlo.gather is biased toward indices shape (like gatherSlice). // This make it awkward to use when you have both batching dimension and new indices dimensions. // For now we reject those, and let user explicitly transpose self or indices if needed. - // .{ .{ .a = 10, .b = 20 }, .b, .{ .n = 8, .a = 10 }, .{ .a = 10, .n = 8 } }, + // .{ .{ .a = 10, .b = 20 }, .{.b = idx(.{ .n = 8, .a = 10 })}, .{ .a = 10, .n = 8 } }, // Also handle tuples - .{ .{ .a = 10, .b = 20 }, .{ .a, .b }, .{ .n = 8, ._ = 2 }, .{ .n = 8 } }, - .{ .{ 10, 20 }, .{ -2, -1 }, .{ 8, 2 }, .{8} }, - // and 1-tuple - .{ .{ .a = 10, .b = 20 }, .{.b}, .{ .n = 8, ._ = 1 }, .{ .a = 10, .n = 8 } }, + .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{ .n = 8 }), .b = idx(.{ .n = 8 }) }, .{ .n = 8 } }, }) |testcase| { - const x_shape, const tag, const idx_shape, const res_shape = testcase; - const x = Tensor.constant(x_shape, .{ .f16 = 0 }); - const idx = Tensor.constant(idx_shape, .{ .i32 = 0 }); - const y = gatherValues(x, tag, idx, .{}); + const x_shape, const indices, const res_shape = testcase; + const x = Tensor.constant(.{ .f16 = 0 }).broad(Shape.init(x_shape, .i64)); + const y = gather(x, indices, .{}); + try zml.testing.expectEqualShapes(Shape.init(res_shape, .f16), y.shape()); + try std.testing.expect(y.value().owner().verify()); + } + + inline for (.{ + .{ .{ 10, 20 }, &[_]u3{ 0, 1 }, &[_]Tensor{ idx(.{8}), idx(.{8}) }, .{8} }, + }) |testcase| { + const x_shape, const idx_axes, const idx_per_axis, const res_shape = testcase; + const x = Tensor.constant(.{ .f16 = 0 }).broad(Shape.init(x_shape, .i64)); + const y = ops.gather(x, idx_axes, idx_per_axis, .{}); try zml.testing.expectEqualShapes(Shape.init(res_shape, .f16), y.shape()); try std.testing.expect(y.value().owner().verify()); } @@ -2164,30 +3146,31 @@ pub const Tensor = struct { /// while gatherValues, always copy values one by one, and as such don't have the same issues. /// In our example the contiguous dimension .d is not sliced /// and gatherSlices can copy data by group of C'*D elements. - pub fn gatherSlices(self: Tensor, slice_shape: Shape, indices: Tensor, opts: GatherOpts) Tensor { - // scoped_log.debug("gatherSlice({}, {_}, {})", .{ self, slice_shape, indices }); + pub fn gatherSlices(self: Tensor, slice_shape_: anytype, indices: Tensor, opts: GatherOpts) Tensor { + const slice_shape = if (@TypeOf(slice_shape_) == Shape) slice_shape_ else Shape.init(slice_shape_, .i32); + // scoped_log.debug("gatherSlice({}, {f}, {})", .{ self, slice_shape, indices }); const tagged_api = slice_shape.isFullyTagged(); if (tagged_api) { for (slice_shape.tags()) |t| { - meta.assert(self._shape.hasTag(t) != null, "gatherSlices expects `slices_shape` to only use tags from `self`. But {s} wasn't found in {}", .{ t, self }); + stdx.debug.assert(self._shape.hasTag(t) != null, "gatherSlices expects `slices_shape` to only use tags from `self`. But {s} wasn't found in {f}", .{ t, self }); } } else { // For untagged api, we require all slices to be specified. // Note: we could relax this and right align the slice. - meta.assert(slice_shape.rank() == self.rank(), "gatherSlices expects `slice_shape.rank()` to match `self.rank()`. Got: gatherSlices({}, slice={_}). To avoid specifying all axes in `slice_shape`, you can use tags.", .{ self, slice_shape }); + stdx.debug.assert(slice_shape.rank() == self.rank(), "gatherSlices expects `slice_shape.rank()` to match `self.rank()`. Got: gatherSlices({f}, slice={f}). To avoid specifying all axes in `slice_shape`, you can use tags.", .{ self, slice_shape }); } const index_coord_axis = indices._shape.hasTag(.coord) orelse indices._shape.axis(-1); - meta.assert(indices.dim(index_coord_axis) == slice_shape.rank(), "gatherSlices({}, slice={_}, indices) expects 'indices' to be a tensor [..., {}], got {}", .{ self, slice_shape, slice_shape.rank(), indices }); + stdx.debug.assert(indices.dim(index_coord_axis) == slice_shape.rank(), "gatherSlices({f}, slice={f}, indices) expects 'indices' to be a tensor [..., {}], got {f}", .{ self, slice_shape, slice_shape.rank(), indices }); // Compute result shape var res_shape = indices._shape.remove(index_coord_axis).withDtype(self.dtype()); var slice_dims = self._shape._dims; - var self_batch_axes: std.BoundedArray(i64, MAX_RANK) = .{}; - var indices_batch_axes: std.BoundedArray(i64, MAX_RANK) = .{}; - var start_index_map: std.BoundedArray(i64, MAX_RANK) = .{}; - var self_offset_axes: std.BoundedArray(i64, MAX_RANK) = .{}; + var self_batch_axes: stdx.BoundedArray(i64, constants.MAX_RANK) = .empty; + var indices_batch_axes: stdx.BoundedArray(i64, constants.MAX_RANK) = .empty; + var start_index_map: stdx.BoundedArray(i64, constants.MAX_RANK) = .empty; + var self_offset_axes: stdx.BoundedArray(i64, constants.MAX_RANK) = .empty; for (self._shape.tags(), 0..self.rank()) |t, self_ax| { const maybe_slice_ax: ?u3 = if (tagged_api) slice_shape.hasTag(t) else @intCast(self_ax); @@ -2197,12 +3180,12 @@ pub const Tensor = struct { self_batch_axes.appendAssumeCapacity(@intCast(self_ax)); indices_batch_axes.appendAssumeCapacity(indices._shape.axis(t)); slice_dims.set(self_ax, 1); - meta.assert(slice_shape.hasTag(t) == null, "gatherSlices expect axes to be either batches or slices axes. Axis {s} has been found both in `slices={_}` and `indices={}`", .{ t, slice_shape, indices }); + stdx.debug.assert(slice_shape.hasTag(t) == null, "gatherSlices expect axes to be either batches or slices axes. Axis {s} has been found both in `slices={f}` and `indices={f}`", .{ t, slice_shape, indices }); } else if (maybe_slice_ax) |slice_ax| { // Specified axes contains the start offset of the slices, // and are collected in `start_index_map`. const slice_dim = slice_shape.dim(slice_ax); - meta.assert(slice_dim <= self._shape.dim(self_ax), "gatherSlices expects `slice_shape` to be smaller than `self.shape()`. On axis {s}, got {} > {}.", .{ t, slice_shape, self._shape }); + stdx.debug.assert(slice_dim <= self._shape.dim(self_ax), "gatherSlices expects `slice_shape` to be smaller than `self.shape()`. On axis {s}, got {f} > {f}.", .{ t, slice_shape, self._shape }); slice_dims.set(self_ax, slice_dim); res_shape = res_shape.appendDim(slice_dim, t); start_index_map.appendAssumeCapacity(@intCast(self_ax)); @@ -2214,13 +3197,11 @@ pub const Tensor = struct { } } - const loc = self.getContext().mlirCtx().location(@src()); - const gather_op = dialect.stablehlo.gather( - self.getContext().mlirCtx(), + const gather_op = stablehlo.gather( + mlirCtx(), self.value(), indices.value(), slice_dims.constSlice(), - loc, .{ .offset_dims = self_offset_axes.constSlice(), .collapsed_slice_dims = &.{}, @@ -2230,7 +3211,8 @@ pub const Tensor = struct { .index_vector_dim = index_coord_axis, .indices_are_sorted = opts.indices_are_sorted, }, - ); + currentLoc(), + ).appendTo(currentBlock()); return _result(res_shape, gather_op.result(0)); } @@ -2238,13 +3220,23 @@ pub const Tensor = struct { const zml = @import("zml.zig"); const platform = zml.testing.env(); + const Local = struct { + pub fn _gatherSlices(self: Tensor, slice_shape: Shape, indices: Tensor, opts: GatherOpts) Tensor { + return self.gatherSlices(slice_shape, indices, opts); + } + }; + { // Only test shapes - var comp = try zml.module.CompilationContext.init(std.heap.page_allocator, "test", platform); + var comp: zml.Compiler = .init(std.testing.allocator, std.testing.io, platform, .{}); defer comp.deinit(); comp.activate(); defer comp.deactivate(); + const block = mlir.Block.init(&.{}, &.{}); + const scope = comp.pushBlock(block); + defer scope.pop(); + inline for (.{ .{ .{ .a = 10 }, .{}, .{ ._ = 0 }, .{ .a = 10 } }, .{ .{ .a = 10 }, .{ .a = 7 }, .{ ._ = 1 }, .{ .a = 7 } }, @@ -2265,34 +3257,49 @@ pub const Tensor = struct { .{ .{ .b = 200, .a = 100, .c = 300 }, .{ .c = 300 }, .{ .a = 100, .b = 200, ._ = 1 }, .{ .a = 100, .b = 200, .c = 300 } }, }) |testcase| { const x_shape, const slice_dims, const idx_shape, const res_shape = testcase; - const x = Tensor.constant(x_shape, .{ .f16 = 0 }); + const x = Tensor.constant(.{ .f16 = 0 }).broad(Shape.init(x_shape, .f16)); const slice_shape = Shape.init(slice_dims, .u16); - const idx = Tensor.constant(idx_shape, .{ .i32 = 0 }); + const idx = Tensor.constant(.{ .i32 = 0 }).broad(Shape.init(idx_shape, .i32)); const y = gatherSlices(x, slice_shape, idx, .{}); try zml.testing.expectEqualShapes(Shape.init(res_shape, .f16), y.shape()); try std.testing.expect(y.value().owner().verify()); - const mod = try zml.compileFn( - std.testing.allocator, - gatherSlices, - .{ x.shape(), slice_shape, idx.shape(), .{ .indices_are_sorted = true } }, - platform, - ); - defer mod.deinit(); + _ = Local._gatherSlices(x, slice_shape, idx, .{ .indices_are_sorted = true }); } } // Test with actual values. - const range = try zml.HostBuffer.arange(std.testing.allocator, .{ .end = 2 * 4 * 6 }, .u16); - defer range.deinit(std.testing.allocator); - const operand = try range.reshape(.{ .a = 2, .b = 4, .c = 6 }).toDevice(platform); - defer operand.deinit(); - const start_indices = (try zml.Buffer.fromArray(platform, [2][2]i32{ .{ 2, 1 }, .{ 0, 3 } })).withTags(.{ .n, ._ }); - defer start_indices.deinit(); + const operand: Tensor = .init(.{ .a = 2, .b = 4, .c = 6 }, .u16); + const indices: Tensor = .init(.{ .n = 2, ._ = 2 }, .i32); + var exe = try zml.module.compile( + std.testing.allocator, + std.testing.io, + Local._gatherSlices, + .{ operand, Shape.init(.{ .b = 2, .c = 3 }, .u16), indices, .{} }, + platform, + .{}, + ); + defer exe.deinit(); - const result = try zml.testing.compileAndCall(platform, gatherSlices, .{ operand, Shape.init(.{ .b = 2, .c = 3 }, .u16), start_indices, .{} }); + var indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, indices.shape(), std.mem.sliceAsBytes(&[2][2]i32{ .{ 2, 1 }, .{ 0, 3 } })); + defer indices_buffer.deinit(); + var operand_buffer: zml.Buffer = b: { + const temp_func = struct { + fn forward() Tensor { + return Tensor.arange(.{ .end = 2 * 4 * 6 }, .u16).reshape(.{ .a = 2, .b = 4, .c = 6 }); + } + }.forward; + var temp_exe = try zml.module.compile(std.testing.allocator, std.testing.io, temp_func, .{}, platform, .{}); + defer temp_exe.deinit(); - const expected = zml.HostBuffer.fromArray(&[2][2][2][3]u16{ + const buffer = try temp_exe.eval(std.testing.allocator, std.testing.io, {}); + break :b buffer; + }; + defer operand_buffer.deinit(); + var result = try exe.eval(std.testing.allocator, std.testing.io, .{ operand_buffer, indices_buffer }); + defer result.deinit(); + + const expected: zml.Slice = .init(Shape.init(.{ 2, 2, 2, 3 }, .u16), std.mem.sliceAsBytes(&[2][2][2][3]u16{ .{ .{ .{ 13, 14, 15 }, .{ 19, 20, 21 } }, .{ .{ 37, 38, 39 }, .{ 43, 44, 45 } }, @@ -2301,17 +3308,18 @@ pub const Tensor = struct { .{ .{ 3, 4, 5 }, .{ 9, 10, 11 } }, .{ .{ 27, 28, 29 }, .{ 33, 34, 35 } }, }, - }); - try zml.testing.expectClose(expected, result, 0); + })); + try zml.testing.expectClose(std.testing.io, expected, result, .{}); } pub const ScatterOpts = struct { - /// Promise scatter that all coordinates in `indices` are sorted, wrt to the final in memory offset. + /// Promise scatter that all coordinates in `indices` are sorted, wrt to the final offset in `self` /// Result is undefined if the promise is violated. indices_are_sorted: bool = false, /// Promise scatter that slices don't overlap. /// Result is undefined if the promise is violated. + /// This allows for better code generation, because it means that updates can be applied in parallel. indices_are_unique: bool = false, /// Function used to update previous value in `self` with values from `updates`. @@ -2319,132 +3327,102 @@ pub const Tensor = struct { /// then you should make sure the slices don't overlap, /// otherwise the result will depend on the runtime scheduling /// of the operator which is backend specific. - update_fn: *const fn (*const anyopaque, Tensor, Tensor) Tensor = increment, + update_fn: *const fn (ops.ScatterArgs) struct { Tensor } = increment, - /// Extra data that may be needed for a custom update function. - /// `override` and `increment` don't need it, leaving it to undefined works. - update_fn_ctx: *const anyopaque = undefined, - - pub fn increment(_: *const anyopaque, old_value: Tensor, new_value: Tensor) Tensor { - return old_value.add(new_value); + pub fn increment(values: ops.ScatterArgs) struct { Tensor } { + return .{values.input.add(values.update)}; } - pub fn override(_: *const anyopaque, old_value: Tensor, new_value: Tensor) Tensor { - _ = old_value; - return new_value; + pub fn override(values: ops.ScatterArgs) struct { Tensor } { + return .{values.update}; } }; - /// Update the given tensors, by copying `values` into self slices. + /// Update the given tensor, by copying `values` into slice by slice into `self`. /// The slices are chosen at runtime by interpreting indices as coordinates into `self`. - /// * `indices` represents a set of coordinates into `self`. - /// For the sake of simplifying the creation of `indices` tensor, - /// it's allowed to not mention a specific axis if the coordinate for this axis is always `0`. - /// Similarly to `gatherValues`, the coordinates are read from the `.coord` axis, or last axis if `.coord` is not found. - /// The coordinates represent the "top-left" corner of the slice to extract. - /// `indices.dim(.coord)` must match `coord_axes.len`. - /// Other axes identify one "slice" and they must be found inside `updates`. + /// This is a generalized version of `dynamicUpdateSlice` where more than one offset can be specified at a time. /// - /// * the output tensor starts with axes from `indices`. - /// * if the input tensor has tagged axes, matching `indices` axes, - /// they will be considered "batching" axes. + /// ### Arguments /// - /// Sample input/output shapes: - /// * scatterSlices([A, B, C, D], .{b, c}, [N, 2], [N, B', C']) -> [A, B, C, D] - /// * scatterSlices(x(a,b,c,d), g(n,m), y[n,b,c]) [A,B,C,D] { - /// var z = x; - /// for (0..N) |n| { z[a,g[n,0]+b',g[n,1]+c',d] = y[n,a,b',c',d]; } + /// - Return a tensor with same shape than `self`, with updated content. + /// - `indices` is a set of Tensor (typically rank 1), representing coordinates into `self`. + /// all indices must have the same shape, but scalars are accepted. + /// - each `indices` entry contains offset along an axes into `self`. + /// Typically axes are identified by their tags, but in the absence of tags on `indices`, + /// The entry in indices will be assigned to axes of `self` from major to minor axis. + /// It is recommended to have indices referencing only major axes of `self` for better performance. + /// - `values` shape is obtained by concatenating the shape of `indices` with the shape of the slices to be extracted. + /// - `opts`: `zml.Tensor.ScatterOpts` des + /// + /// ### Sample input/output shapes with corresponding pseudo-code. + /// + /// Basic `scatterSlices` with the first two axes (.a, .b) being indexed, and full (.c, .d) slice copies: + /// + /// ``` + /// fn scatterSlices(x[A, B, C, D], .{.a=off_a[N], .b=off_b[N]}, y[N, C, D]) [A, B, C, D] { + /// var z = x; + /// for (0..N) |n| { + /// for (0..C) |c| for (0..D) |d| {{ + /// z[off_a[n],off_b[n],c,d] += y[n, c, d]; + /// }} + /// } + /// return z; /// } + /// ``` /// - /// **Warning**: if `opts.update_fn` is not associative not all calls to `scatterSlices` are sound. + /// `scatterSlices` with the first three axes (.a, .b, .c) being indexed, and a partial copy of (.c, .d). + /// Note that .c axis is present both in the indices and updates, and `updates.dim(.c) < self.dim(.c)`. + /// + /// ``` + /// fn scatterSlices(x[A, B, C, D], .{.a=off_a[N], .b=off_b[N], .c=off_c[N]}, y[N, C', D]) [A, B, C, D] { + /// var z = x; + /// for (0..N) |n| { + /// for (0..C') |c| for (0..D) |d| {{ + /// z[off_a[n],off_b[n],off_c[n]+c,d] += y[n, c, d]; + /// }} + /// } + /// return z; + /// } + /// ``` + /// + /// `scatterSlices` with the first axis .a being indexed, and where .b is used as a batching axis. + /// Note that here .b axis is present in `self`, `off_a`, and `updates`, + /// and is not mentionned in the axes of indices. + /// + /// ``` + /// fn scatterSlices(x[A, B, C, D], .{.a=off_a[B,N]}, y[N, B, C, D]) [A, B, C, D] { + /// var z = x; + /// for (0..B) |b| { + /// for (0..N) |n| { + /// for (0..C) |c| for (0..D) |d| {{ + /// z[off_a[b,n],b,c,d] += y[n, b, c, d]; + /// }} + /// } + /// } + /// return z; + /// } + /// ``` + /// + /// ### Warnings + /// + /// - if `opts.update_fn` is not associative not all calls to `scatterSlices` are sound. /// In particular if you scatter overlapping slices, with `zml.Tensor.ScatterOpts.override`, /// then the result will depend on the execution order that you don't control. - pub fn scatterSlices(self: Tensor, coord_axes: anytype, indices: Tensor, updates: Tensor, opts: ScatterOpts) Tensor { - const loc = @src(); - // scoped_log.debug("scatterSlices({}, {any}, {}, {})", .{ self, coord_axes, indices, updates }); + /// - `scatterSlices` is a very expressive operator, and can lead to complicated code generation + /// that requires host<->device synchronization. + /// ZML tries to generate the easiest to optimize IR, and will warn you if it generates known problematic IR. + pub fn scatterSlices(self: Tensor, indices: anytype, updates: Tensor, opts: ScatterOpts) Tensor { + //scoped_log.debug("scatterSlices({}, {any}, {})", .{ self, indices, updates }); - meta.assert(self.dtype() == updates.dtype(), "scatterSlices expects input and 'updates' tensors to be of the same type, got {} and {}", .{ self.dtype(), updates.dtype() }); + const UpdateType = @TypeOf(ScatterOpts.increment); - const single_coord, const coord_axes_ = _parseGatherCoord(self, coord_axes); - const AxisKind = enum { batching, update_window, inserted_window, window_id }; - var self_kind: std.BoundedArray(AxisKind, MAX_RANK) = .{}; - var indices_batch_axes: Shape.DimsArray = .{}; - for (self._shape.tags()) |t| { - if (updates._shape.hasTag(t)) |_| { - if (indices._shape.hasTag(t)) |id_ax| { - // tag is in self, indices and updates -> it's a batching dim - self_kind.appendAssumeCapacity(.batching); - indices_batch_axes.appendAssumeCapacity(id_ax); - } else { - self_kind.appendAssumeCapacity(.update_window); - } - } else { - self_kind.appendAssumeCapacity(.inserted_window); + const Custom = struct { + pub fn inc(values: ops.ScatterArgs, custom: *const UpdateType) struct { Tensor } { + return @call(.auto, custom, .{values}); } - } - // scoped_log.warn(" self_kind -> {any}", .{self_kind.constSlice()}); - - const index_coord_axis = if (single_coord) - indices.rank() - else blk: { - const ax = indices._shape.hasTag(.coord) orelse indices._shape.axis(-1); - meta.assert(indices.dim(ax) == coord_axes_.len, "scatterSlices({}, coord_axes={any}, indices, updates) expects 'indices' to be a tensor [..., {}], got {}", .{ self, coord_axes, coord_axes_.len, indices }); - - break :blk ax; }; - if (indices.count() == 1) { - return self.dynamicUpdateSlice1d(updates, coord_axes_.get(0), indices.reshape(.{})); - } - var up_kind: std.BoundedArray(AxisKind, MAX_RANK) = .{}; - // Note: we assume the scatter_dims appear in the same order inside indices and inside self. - for (updates._shape.tags(), 0..) |t, up_ax| { - if (self._shape.hasTag(t)) |self_ax| { - if (self_kind.get(self_ax) == .batching) { - up_kind.appendAssumeCapacity(.batching); - } else { - meta.assert(updates.dim(up_ax) <= self.dim(self_ax), "scatterSlices expects the slices described in 'updates' to fit inside 'self', but along axis .{s} it doesn't. Got self={}, updates={}.", .{ t, self, updates }); - up_kind.appendAssumeCapacity(.update_window); - } - } else if (t == Shape.TagUnknown or indices._shape.hasTag(t) != null) { - up_kind.appendAssumeCapacity(.window_id); - } else { - std.debug.panic("scatterSlices expects 'updates' to be made of axes from 'self={}' and from 'indices={}', got unknown tag {s} in {}", .{ self, indices, t, updates }); - } - } - const n_indices_axes = updates.rank() - _collectAxes(AxisKind, up_kind, .update_window).len; - if (single_coord) { - meta.assert(n_indices_axes == indices.rank(), "scatterSlices({}, {any}) expects 'updates' to contain all axes from 'indices', got indices={}, updates={}", .{ self, coord_axes, indices, updates }); - } else { - meta.assert(n_indices_axes == indices.rank() - 1, "scatterSlices({}, {any}) expects 'updates' to contain all-but-last axes from 'indices', got indices={}, updates={}", .{ self, coord_axes, indices, updates }); - } - - const ctx = self.getContext(); - const mlir_ctx = ctx.mlirCtx(); - - const _scalar: Tensor = .{ ._shape = Shape.init(.{}, self.dtype()), ._id = undefined }; - const UpdateS = ops.BlockSign(ScatterOpts.increment); - const update_block = ctx.makeBlock(UpdateS, opts.update_fn, opts.update_fn_ctx, .{ _scalar, _scalar }); - - const op = dialect.stablehlo.scatter( - mlir_ctx, - &.{self.value()}, - &.{indices.value()}, - &.{updates.value()}, - update_block, - .{ - .update_window_dims = _collectAxes(AxisKind, up_kind, .update_window).constSlice(), - .inserted_window_dims = _collectAxes(AxisKind, self_kind, .inserted_window).constSlice(), - .input_batching_dims = _collectAxes(AxisKind, self_kind, .batching).constSlice(), - .scatter_indices_batching_dims = indices_batch_axes.constSlice(), - .scatter_dims_to_operand_dims = toI64(coord_axes_.constSlice()), - .index_vector_dim = index_coord_axis, - .indices_are_sorted = opts.indices_are_sorted, - .unique_indices = opts.indices_are_unique, - }, - mlir_ctx.location(loc), - ); - return _result(self._shape, op.result(0)); + return ops.scatter(.{self}, indices, .{updates}, Custom.inc, .{opts.update_fn}, opts)[0]; } test scatterSlices { @@ -2452,39 +3430,60 @@ pub const Tensor = struct { const platform = zml.testing.env(); const Local = struct { - pub fn scatter(self: Tensor, coord_axes: Shape.AxesArray, indices: Tensor, updates: Tensor) Tensor { + pub fn _scatter(self: Tensor, indices: []const Tensor, updates: Tensor) Tensor { return self.scatterSlices( - coord_axes.constSlice(), indices, updates, .{ .update_fn = ScatterOpts.increment }, ); } + + pub fn _scatterCB(self: Tensor, coords: Tensor, updates: Tensor) Tensor { + return self.scatterSlices( + .{ .c = coords.slice(.coord, .single(0)), .b = coords.slice(.coord, .single(1)) }, + updates, + .{ .update_fn = ScatterOpts.increment }, + ); + } + + pub fn _idx(idx_shape: anytype) Tensor { + return Tensor.constant(.{ .i32 = 0 }).broad(Shape.init(idx_shape, .i32)); + } }; { // Only test shapes - var comp = try zml.module.CompilationContext.init(std.heap.page_allocator, "test", platform); + var comp: zml.Compiler = .init(std.testing.allocator, std.testing.io, platform, .{}); defer comp.deinit(); comp.activate(); defer comp.deactivate(); - inline for (.{ - .{ .{ .a = 10 }, .a, .{}, .{ .a = 3 } }, - .{ .{ .a = 10, .b = 20 }, .b, .{ .a = 10, .n = 8 }, .{ .a = 10, .n = 8, .b = 2 } }, - // I'm not sure I like this variant, cause `b` is not mentionned in updates. - // So 'stablehlo.scatter' is implicitly broadcasting the updates along `b` axis. - // OTOH asking the user to do the broadcasting isn't trivial cause they will need to do shape wrangling and that's annoying. - .{ .{ .a = 10, .b = 20 }, .a, .{ .n = 8 }, .{ .n = 8, .a = 2 } }, - .{ .{ .a = 10, .b = 20 }, .{ .b, .a }, .{ .n = 8, ._ = 2 }, .{ .n = 8, .a = 3, .b = 2 } }, - .{ .{ .a = 10, .b = 20 }, .{ .a, .b }, .{ .n = 8, ._ = 2 }, .{ .a = 3, .n = 8, .b = 2 } }, - }) |testcase| { - const x_shape, const axes_, const idx_shape, const updates_shapes = testcase; - const x = Tensor.constant(x_shape, .{ .f16 = 0 }); - const idx = Tensor.constant(idx_shape, .{ .i32 = 0 }); - const updates = Tensor.constant(updates_shapes, .{ .f16 = 0 }); + const block = mlir.Block.init(&.{}, &.{}); + defer block.deinit(); + const scope = comp.pushBlock(block); + defer scope.pop(); - const y = scatterSlices(x, axes_, idx, updates, .{}); + const idx = Local._idx; + + inline for (.{ + // This is equivalent to a dynamic update slice, update 3 values at given offset of axis .a: + .{ .{ .a = 10 }, .{ .a = idx(.{}) }, .{ .a = 3 } }, + // Use .a as a batching axis with .a=10 x .n=8 updates of 2 elements of .b + .{ .{ .a = 10, .b = 20 }, .{ .b = idx(.{ .a = 10, .n = 8 }) }, .{ .a = 10, .n = 8, .b = 2 } }, + // Same but with update transposed + .{ .{ .a = 10, .b = 20 }, .{ .b = idx(.{ .a = 10, .n = 8 }) }, .{ .a = 10, .b = 2, .n = 8 } }, + // similar, but use the normalized form where a is no longer an explicit batching axis. + .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{ .a2 = 10, .n = 8 }), .b = idx(.{ .a2 = 10, .n = 8 }) }, .{ .a2 = 10, .n = 8, .b = 2 } }, + .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{ .a = 10, .n = 8 }), .b = idx(.{ .a = 10, .n = 8 }) }, .{ .a = 10, .n = 8, .b = 2 } }, + .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{ .n = 8 }) }, .{ .n = 8, .a = 2 } }, + .{ .{ .a = 10, .b = 20 }, .{ .b = idx(.{ .n = 8 }), .a = idx(.{ .n = 8 }) }, .{ .n = 8, .a = 3, .b = 2 } }, + .{ .{ .a = 10, .b = 20 }, .{ .a = idx(.{ .n = 8 }), .b = idx(.{ .n = 8 }) }, .{ .a = 3, .n = 8, .b = 2 } }, + }) |testcase| { + const x_shape, const indices, const updates_shapes = testcase; + const x = Tensor.constant(.{ .f16 = 0 }).broad(Shape.init(x_shape, .f16)); + const updates = Tensor.constant(.{ .f16 = 0 }).broad(Shape.init(updates_shapes, .f16)); + + const y = scatterSlices(x, indices, updates, .{}); // Shape doesn't change with scatterSlices try zml.testing.expectEqualShapes(x.shape(), y.shape()); try std.testing.expect(y.value().owner().verify()); @@ -2492,51 +3491,76 @@ pub const Tensor = struct { } // Test with actual values, no batching. { - const a_host = try zml.HostBuffer.arange(std.testing.allocator, .{ .end = 9 }, .i32); - const a = (try zml.Buffer.from(platform, a_host.reshape(.{ 3, 3 }))).withTags(.{ .a, .b }); - defer a.deinit(); - a_host.deinit(std.testing.allocator); + const a: Tensor = .init(.{ 3, 3 }, .i32); - const scatter_indices = try zml.Buffer.fromArray(platform, [2][1]i32{ .{0}, .{2} }); - const updates = try zml.Buffer.fromArray(platform, [2][3]i32{ .{ 10, 20, 30 }, .{ 70, 80, 90 } }); + const scatter_indices = Tensor.init(.{2}, .i32).withTags(.{.n}); + const updates = Tensor.init(.{ 2, 3 }, .i32).withTags(.{ .n, .b }); + + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._scatter, .{ a, &.{scatter_indices}, updates }, platform, .{}); + defer exe.deinit(); + + var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), std.mem.sliceAsBytes(&[9]i32{ 0, 1, 2, 3, 4, 5, 6, 7, 8 })); + defer a_buffer.deinit(); + var scatter_indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, scatter_indices.shape(), @ptrCast(&[2]i32{ 0, 2 })); + defer scatter_indices_buffer.deinit(); + var updates_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, updates.shape(), std.mem.sliceAsBytes(&[2][3]i32{ .{ 10, 20, 30 }, .{ 70, 80, 90 } })); + defer updates_buffer.deinit(); + var result = try exe.eval(std.testing.allocator, std.testing.io, .{ a_buffer, &.{scatter_indices_buffer}, updates_buffer }); + defer result.deinit(); const expected = [3][3]i32{ .{ 10, 21, 32 }, .{ 3, 4, 5 }, .{ 76, 87, 98 } }; - const result = try zml.testing.compileAndCall(platform, Local.scatter, .{ - a, - a.shape().axes(.{.a}), - scatter_indices.withTags(.{ .n, .coord }), - updates.withTags(.{ .n, .b }), - }); try std.testing.expect(a.shape().eql(result.shape())); - try std.testing.expectEqual(expected, result.getValue(@TypeOf(expected))); + try std.testing.expectEqual(expected, result.getValue(@TypeOf(expected), std.testing.io)); + } + // Test with setting individual values (no batching) + { + const a: Tensor = .init(.{9}, .i32); + + const scatter_indices = Tensor.init(.{2}, .i32).withTags(.{.n}); + const updates = Tensor.init(.{2}, .i32).withTags(.{.n}); + + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._scatter, .{ a, &.{scatter_indices}, updates }, platform, .{}); + defer exe.deinit(); + + var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), std.mem.sliceAsBytes(&[9]i32{ 0, 1, 2, 3, 4, 5, 6, 7, 8 })); + defer a_buffer.deinit(); + var scatter_indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, scatter_indices.shape(), std.mem.sliceAsBytes(&[2]i32{ 2, 7 })); + defer scatter_indices_buffer.deinit(); + var updates_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, updates.shape(), std.mem.sliceAsBytes(&[2]i32{ 20, 70 })); + defer updates_buffer.deinit(); + var result = try exe.eval(std.testing.allocator, std.testing.io, .{ a_buffer, &.{scatter_indices_buffer}, updates_buffer }); + defer result.deinit(); + + const expected = [9]i32{ 0, 1, 22, 3, 4, 5, 6, 77, 8 }; + try std.testing.expect(a.shape().eql(result.shape())); + try std.testing.expectEqual(expected, result.getValue(@TypeOf(expected), std.testing.io)); } { // Test with actual values and batching along axis .a - const operand = try zml.Buffer.constant(platform, Shape.init(.{ .a = 2, .b = 3, .c = 4, .d = 2 }, .u16), 0); - defer operand.deinit(); - const start_indices = (try zml.Buffer.fromArray( - platform, - [2][2][3][2]i32{ - .{ - .{ .{ 0, 0 }, .{ 1, 0 }, .{ 2, 1 } }, - .{ .{ 0, 1 }, .{ 1, 1 }, .{ 0, 9 } }, - }, - .{ - .{ .{ 0, 0 }, .{ 2, 1 }, .{ 2, 2 } }, - .{ .{ 1, 2 }, .{ 0, 1 }, .{ 1, 0 } }, - }, + const operand: Tensor = .init(.{ .a = 2, .b = 3, .c = 4, .d = 2 }, .u16); + const start_indices: Tensor = .init(.{ .n = 2, .a = 2, .m = 3, .coord = 2 }, .i32); + const values: Tensor = .init(.{ .n = 2, .a = 2, .m = 3, .c = 2, .d = 2 }, .u16); + + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._scatterCB, .{ operand, start_indices, values }, platform, .{}); + defer exe.deinit(); + + var operand_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, operand.shape(), std.mem.sliceAsBytes(&@as([2 * 3 * 4 * 2]u16, @splat(0)))); + defer operand_buffer.deinit(); + var start_indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, start_indices.shape(), std.mem.sliceAsBytes(&[2][2][3][2]i32{ + .{ + .{ .{ 0, 0 }, .{ 1, 0 }, .{ 2, 1 } }, + .{ .{ 0, 1 }, .{ 1, 1 }, .{ 0, 9 } }, }, - )).withTags(.{ .n, .a, .m, .coord }); - defer start_indices.deinit(); - - const values = try zml.Buffer.constant( - platform, - Shape.init(.{ .n = 2, .a = 2, .m = 3, .c = 2, .d = 2 }, .u16), - 1, - ); - defer values.deinit(); - - const result = try zml.testing.compileAndCall(platform, Local.scatter, .{ operand, operand.shape().axes(.{ .c, .b }), start_indices, values }); + .{ + .{ .{ 0, 0 }, .{ 2, 1 }, .{ 2, 2 } }, + .{ .{ 1, 2 }, .{ 0, 1 }, .{ 1, 0 } }, + }, + })); + defer start_indices_buffer.deinit(); + var values_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, values.shape(), std.mem.sliceAsBytes(&@as([2 * 2 * 3 * 2 * 2]u16, @splat(1)))); + defer values_buffer.deinit(); + var result = try exe.eval(std.testing.allocator, std.testing.io, .{ operand_buffer, start_indices_buffer, values_buffer }); + defer result.deinit(); const expected = [2][3][4][2]u16{ .{ @@ -2551,7 +3575,7 @@ pub const Tensor = struct { }, }; try std.testing.expect(operand.shape().eql(result.shape())); - try std.testing.expectEqual(expected, result.getValue(@TypeOf(expected))); + try std.testing.expectEqual(expected, result.getValue(@TypeOf(expected), std.testing.io)); } } @@ -2559,75 +3583,68 @@ pub const Tensor = struct { pub fn max(self: Tensor, axis_: anytype) Tensor { const a = self.axis(axis_); return ops.reduce( + .{self}, + .{Tensor.constant(self.dtype().minValue())}, + &.{a}, struct { - pub fn cmp(x: Tensor, res: Tensor) Tensor { - return res.maximum(x.convert(res.dtype())); + pub fn cmp(args: ops.ReduceArgs) struct { Tensor } { + return .{args.left.maximum(args.right)}; } }.cmp, - self, - Tensor.constant(&.{}, self.dtype().minValue()), - &.{a}, - ); + .{}, + )[0]; } /// Returns a Tensor containing the minimum over a given axis. pub fn min(self: Tensor, axis_: anytype) Tensor { const a = self.axis(axis_); return ops.reduce( + .{self}, + .{Tensor.constant(self.dtype().maxValue())}, + &.{a}, struct { - pub fn cmp(x: Tensor, res: Tensor) Tensor { - return res.minimum(x.convert(res.dtype())); + pub fn cmp(args: ops.ReduceArgs) struct { Tensor } { + return .{args.left.minimum(args.right)}; } }.cmp, - self, - Tensor.constant(&.{}, self.dtype().maxValue()), - &.{a}, - ); + .{}, + )[0]; } pub const ArgMaxRes = struct { values: Tensor, indices: Tensor, - fn cmp(left: ArgMaxRes, right: ArgMaxRes) ArgMaxRes { - const left_val = left.values; - const right_val = right.values; - const left_idx = left.indices; - const right_idx = right.indices; - - const left_gt_right = left_val.cmp(.GT, right_val); - const is_nan = left_val.cmp(.NE, left_val); + fn cmp(values: ops.ReduceArgs, indices: ops.ReduceArgs) [2]Tensor { + const left_gt_right = values.left.cmp(.GT, values.right); + const is_nan = values.left.cmp(.NE, values.left); const left_gt_or_nan = left_gt_right.logical(.OR, is_nan); - // we are bubbling up Nan. - const max_val = left_gt_or_nan.select(left_val, right_val); - // If left_val == right_val: keep the smallest idx. - const is_same = left_val.cmp(.EQ, right_val); - const is_first = left_idx.cmp(.LT, right_idx); + // If values.left == values.right: keep the smallest idx. + const is_same = values.left.cmp(.EQ, values.right); + const is_first = indices.left.cmp(.LT, indices.right); const is_same_but_first = is_same.logical(.AND, is_first); const keep_left_idx = left_gt_or_nan.logical(.OR, is_same_but_first); - const max_idx = keep_left_idx.select(left_idx, right_idx); + // we are bubbling up Nan. + const max_val = left_gt_or_nan.select(values.left, values.right); + const max_idx = keep_left_idx.select(indices.left, indices.right); - return .{ .values = max_val, .indices = max_idx }; + return .{ max_val, max_idx }; } }; - /// Returns two Tensors containing the maximum and the index of this maximum over a given axis. - /// - /// Stable argmax: - /// * bubbles up Nan - /// * in case of equality the smallest index matching the maximum - pub fn argMax(x: Tensor, axis_: anytype, index_dtype: DataType) ArgMaxRes { - meta.assert(index_dtype.isInteger(), "argMax expect index type to be an integer, got {}", .{index_dtype}); - + pub fn argMax(x: Tensor, axis_: anytype) ArgMaxRes { const a = x.axis(axis_); + const dt: DataType = if (x.dim(a) <= std.math.maxInt(i32)) .i32 else .i64; - return ops.reduce( - ArgMaxRes.cmp, - .{ .values = x, .indices = Tensor.arange(.{ .end = x.dim(a) }, index_dtype).broadcast(x._shape.withDtype(index_dtype), &.{a}) }, - .{ .values = Tensor.constant(&.{}, x.dtype().minValue()), .indices = Tensor.scalar(0, index_dtype) }, + const values, const indices = ops.reduce( + .{ x, Tensor.arange(.{ .end = x.dim(axis_) }, dt).broadcast(x.shape(), &.{a}) }, + .{ Tensor.constant(x.dtype().minValue()), Tensor.constant(dt.zero()) }, &.{a}, + ArgMaxRes.cmp, + .{}, ); + return .{ .values = values, .indices = indices }; } test argMax { @@ -2635,57 +3652,70 @@ pub const Tensor = struct { const platform = zml.testing.env(); const allocator = std.testing.allocator; const ArgMaxTest = struct { - pub fn forward(x: Tensor) Tensor.ArgMaxRes { - return x.argMax(1, .i32); + pub fn _fwd(x: Tensor) Tensor.ArgMaxRes { + return x.argMax(1); } }; - const argmax = try zml.compileFn(allocator, ArgMaxTest.forward, .{Shape.init(.{ 1, 5 }, .f32)}, platform); - defer argmax.deinit(); - // Test with tie + const x: Tensor = .init(.{ 1, 5 }, .f32); + var exe = try zml.module.compile(allocator, std.testing.io, ArgMaxTest._fwd, .{x}, platform, .{}); + defer exe.deinit(); + { - const x = try zml.Buffer.fromArray(platform, [1][5]f32{.{ 5.0, 4.1, 7.9, 0, 7.9 }}); - const res = argmax.call(.{x}); - const max_ = res.values.getValue(f32); - const max_idx = res.indices.getValue(i32); - try testing.expectEqual(max_, 7.9); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[1][5]f32{.{ 5.0, 4.1, 7.9, 0, 7.9 }})); + defer x_buffer.deinit(); + + var res: zml.Bufferized(Tensor.ArgMaxRes) = undefined; + var runner = try exe.runner(allocator); + defer runner.deinit(allocator); + runner.run(std.testing.io, .{x_buffer}, .{&res}, .{ .wait = true }); + defer res.values.deinit(); + defer res.indices.deinit(); + const max_ = res.values.getValue(f32, std.testing.io); + const max_idx = res.indices.getValue(i32, std.testing.io); + try std.testing.expectEqual(max_, 7.9); // We should always return the first max found. - try testing.expectEqual(max_idx, 2); + try std.testing.expectEqual(max_idx, 2); } - // Test with Nan { - const x = try zml.Buffer.fromArray(platform, [1][5]f32{.{ 5.0, std.math.nan(f32), 7.9, 0, 7.9 }}); - const res = argmax.call(.{x}); - const max_ = try res.values.getValue(f32); - const max_idx = try res.indices.getValue(i32); - try testing.expect(std.math.isNan(max_)); - try testing.expectEqual(max_idx, 1); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[1][5]f32{.{ 5.0, std.math.nan(f32), 7.9, 0, 7.9 }})); + defer x_buffer.deinit(); + + var res: zml.Bufferized(Tensor.ArgMaxRes) = undefined; + var runner = try exe.runner(allocator); + defer runner.deinit(allocator); + runner.run(std.testing.io, .{x_buffer}, .{&res}, .{ .wait = true }); + defer res.values.deinit(); + defer res.indices.deinit(); + const max_ = try res.values.getValue(f32, std.testing.io); + const max_idx = try res.indices.getValue(i32, std.testing.io); + try std.testing.expect(std.math.isNan(max_)); + try std.testing.expectEqual(max_idx, 1); } } pub const SortRes = ArgMaxRes; + pub const SortOpts = struct { descending: bool = false }; + /// Returns two Tensors. The first contains the sorted values and the second one contains the sorted indices. - pub fn sort(self: Tensor, axis_: i64, opts: struct { descending: bool = false }) SortRes { + pub fn sort(self: Tensor, axis_: anytype, opts: SortOpts) SortRes { const a = self.axis(axis_); const indices = Tensor.arange(.{ .end = self.dim(a) }, .i32).broadcast(self._shape, &.{a}); - const res = ops.sort( - struct { - fn call(direction: dialect.stablehlo.ComparisonDirection.Direction, lhs: Tensor, rhs: Tensor, _: Tensor, _: Tensor) Tensor { - return lhs.cmp(direction, rhs); - } - }.call, - if (opts.descending) .GT else .LT, - .{ self, indices }, - self.axis(axis_), - true, - ); + const direction: stablehlo.ComparisonDirectionAttr.Direction = if (opts.descending) .GT else .LT; + const res = ops.sort(.{ self, indices }, a, struct { + fn call(values: ops.SortArgs, _: ops.SortArgs, direction_: stablehlo.ComparisonDirectionAttr.Direction) Tensor { + return values.left.cmp(direction_, values.right); + } + }.call, .{direction}, true); return .{ .values = res[0], .indices = res[1] }; } + pub const ArgSortOpts = SortOpts; + /// Returns a Tensor containing the indices corresponding to the sorted values over the given axis. - pub fn argsort(self: Tensor, axis_: i64, opts: struct { descending: bool = false }) Tensor { + pub fn argsort(self: Tensor, axis_: anytype, opts: ArgSortOpts) Tensor { return self.sort(axis_, .{ .descending = opts.descending }).indices; } @@ -2693,38 +3723,70 @@ pub const Tensor = struct { const zml = @import("zml.zig"); const platform = zml.testing.env(); - var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena_state.deinit(); - const allocator = arena_state.allocator(); + const Local = struct { + pub fn _argsort(x: Tensor, axis_: u3, opts: ArgSortOpts) Tensor { + return x.argsort(axis_, opts); + } + }; + // 2D Tensor - dim = 1, ascending { - const x = try zml.Buffer.fromSlice(platform, .{ 2, 5 }, &[_]f32{ -0.9264, 0.7156, 1.0202, 0.3992, 1.2349, 1.0003, -0.1932, 1.3935, 0.7316, 0.0851 }); - const res = try zml.testing.compileAndCall(platform, Tensor.argsort, .{ x, 1, .{} }); - const res_cpu = try res.toHostAlloc(allocator); - try testing.expectEqualSlices(i32, &.{ 0, 3, 1, 2, 4, 1, 4, 3, 0, 2 }, res_cpu.items(i32)); + const x: Tensor = .init(.{ 2, 5 }, .f32); + + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._argsort, .{ x, 1, .{} }, platform, .{}); + defer exe.deinit(); + + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]f32{ -0.9264, 0.7156, 1.0202, 0.3992, 1.2349, 1.0003, -0.1932, 1.3935, 0.7316, 0.0851 })); + defer x_buffer.deinit(); + + var res = try exe.eval(std.testing.allocator, std.testing.io, .{x_buffer}); + defer res.deinit(); + + const res_cpu = try res.toSliceAlloc(std.testing.allocator, std.testing.io); + defer res_cpu.free(std.testing.allocator); + + try std.testing.expectEqualSlices(i32, &.{ 0, 3, 1, 2, 4, 1, 4, 3, 0, 2 }, res_cpu.items(i32)); } + // 3D Tensor, dim = 1, descending { - const x = try zml.Buffer.fromSlice(platform, .{ 1, 5, 10 }, &[_]f16{ + const x: Tensor = .init(.{ 1, 5, 10 }, .f16); + + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._argsort, .{ x, 1, .{ .descending = true } }, platform, .{}); + defer exe.deinit(); + + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]f16{ -0.2505, 1.2520, -0.7041, 0.1066, 1.2773, -1.7246, 0.8389, 1.1094, 0.0601, 1.0684, 0.9619, 1.3916, 1.2246, -0.1406, 0.3674, -1.2480, -1.7051, -0.0934, 0.3435, 0.4373, 1.3809, 0.5444, -0.6079, 1.2031, -0.6880, 1.2979, -0.1869, 0.2991, 0.0156, 0.1847, 0.6626, -0.3040, -0.8726, -1.4805, -1.6943, 1.1055, -2.0078, -0.5288, 0.8813, 0.8008, 2.0527, 1.1230, 0.5430, 0.2494, -0.9434, 0.7876, 0.1818, 0.9258, -2.4902, 1.5918, - }); - const res_dev = try zml.testing.compileAndCall(platform, Tensor.argsort, .{ x, 1, .{ .descending = true } }); - const res = try res_dev.toHostAlloc(allocator); - try testing.expectEqualSlices(i32, &.{ + })); + defer x_buffer.deinit(); + + var res = try exe.eval(std.testing.allocator, std.testing.io, .{x_buffer}); + defer res.deinit(); + + const res_cpu = try res.toSliceAlloc(std.testing.allocator, std.testing.io); + defer res_cpu.free(std.testing.allocator); + + try std.testing.expectEqualSlices(i32, &.{ 4, 1, 1, 2, 0, 2, 0, 0, 3, 4, 2, 0, 4, 4, 1, 3, 4, 4, 1, 0, 1, 4, 2, 0, 2, 4, 2, 2, 0, 3, 3, 2, 0, 1, 4, 1, 1, 1, 2, 1, 0, 3, 3, 3, 3, 0, 3, 3, 4, 2, - }, res.items(i32)); + }, res_cpu.items(i32)); } + // 4D Tensor, dim = 3, ascending { - const x = try zml.Buffer.fromSlice(platform, .{ 4, 2, 1, 4 }, &[_]i32{ + const x: Tensor = .init(.{ 4, 2, 1, 4 }, .i32); + + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._argsort, .{ x, 3, .{} }, platform, .{}); + defer exe.deinit(); + + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]i32{ 89, 31, 22, 42, 64, 39, 0, 30, 64, 71, 46, 31, @@ -2733,10 +3795,16 @@ pub const Tensor = struct { 93, 24, 45, 72, 64, 86, 62, 88, 57, 21, 19, 12, - }); - const res_dev = try zml.testing.compileAndCallWithTensors(platform, Tensor.argsort, .{ x.shape(), 3, .{} }, .{ x, 0, .{} }); - const res = try res_dev.toHostAlloc(allocator); - try testing.expectEqualSlices(i32, &.{ + })); + defer x_buffer.deinit(); + + var res = try exe.eval(std.testing.allocator, std.testing.io, .{x_buffer}); + defer res.deinit(); + + const res_cpu = try res.toSliceAlloc(std.testing.allocator, std.testing.io); + defer res_cpu.free(std.testing.allocator); + + try std.testing.expectEqualSlices(i32, &.{ 2, 1, 3, 0, 2, 3, 1, 0, 3, 2, 0, 1, @@ -2745,18 +3813,76 @@ pub const Tensor = struct { 1, 2, 3, 0, 2, 0, 1, 3, 3, 2, 1, 0, - }, res.items(i32)); + }, res_cpu.items(i32)); } } + pub const TopKOpts = struct { descending: bool = true }; + /// Returns a Tensor representing the result of Top-K over the given axis. - pub fn topK(self: Tensor, k: u32, axis_: anytype, opts: struct { descending: bool = true }) SortRes { - const a = self.axis(axis_); - const result = self.sort(a, .{ .descending = opts.descending }); - return .{ - .values = result.values.slice1d(a, .{ .end = k }), - .indices = result.indices.slice1d(a, .{ .end = k }), + pub fn topK(self: Tensor, named_axis_: anytype, k: u32, opts: TopKOpts) SortRes { + stdx.debug.assert(k > 0, "topK expects a k > 0, got 0", .{}); + const err_msg = "topK named axis should be an integer or a named axis, eg `x.topK(.{{ .best_token = .token }}, 16)` or `x.topK(-1, 16)`"; + const has_name: ?[:0]const u8, const a = switch (@typeInfo(@TypeOf(named_axis_))) { + .int, .comptime_int => .{ null, self.axis(@as(i64, @intCast(named_axis_))) }, + .@"struct" => |info| blk: { + stdx.debug.assertComptime(info.field_names.len == 1, err_msg, .{}); + break :blk .{ info.field_names[0], self.axis(@field(named_axis_, info.field_names[0])) }; + }, + else => stdx.debug.compileError(err_msg, .{}), }; + const ctx = Compiler.current(); + var result: SortRes = switch (ctx.platform.target) { + // Work around https://github.com/aws-neuron/aws-neuron-sdk/issues/1339 until Neuron's sort+slice rewrite uses the slice size as k. + .neuron => blk: { + stdx.debug.assert(self.dtype().isFloat(), "Neuron Tensor.topK only supports float tensors, got {}", .{self.dtype()}); + stdx.debug.assert(k <= 16384, "Neuron AwsNeuronTopK supports k <= 16384, got {}", .{k}); + + const last_axis: u4 = self.rank() - 1; + const topk_input = if (a == last_axis) self else self.swapAxes(a, last_axis); + const call_input = if (opts.descending) topk_input else topk_input.negate(); + const values_shape = topk_input.shape().setDim(last_axis, @intCast(k)); + const indices_shape = values_shape.withDtype(.i32); + const result_types = [_]*const mlir.Type{ + mlirx.Type.rankedTensor(ctx.mlir_ctx, values_shape), + mlirx.Type.rankedTensor(ctx.mlir_ctx, indices_shape), + }; + + var backend_config_buf: [32]u8 = undefined; + const backend_config = std.fmt.bufPrint(&backend_config_buf, "{d}", .{k}) catch unreachable; + const op = stablehlo.custom_call( + ctx.mlir_ctx, + &.{call_input.value()}, + &result_types, + .{ + .call_target_name = "AwsNeuronTopK", + .has_side_effect = false, + .backend_config = .{ .original = backend_config }, + }, + ctx.location, + ).appendTo(currentBlock()); + + var values = Tensor._result(values_shape, op.result(0)); + if (!opts.descending) values = values.negate(); + var indices = Tensor._result(indices_shape, op.result(1)); + if (a != last_axis) { + values = values.swapAxes(a, last_axis); + indices = indices.swapAxes(a, last_axis); + } + break :blk .{ .values = values, .indices = indices }; + }, + .cpu, .cuda, .rocm, .tpu, .oneapi, .metal, .furiosa => blk: { + var sorted = self.sort(a, .{ .descending = opts.descending }); + sorted.values = sorted.values.slice(a, .{ .end = k }); + sorted.indices = sorted.indices.slice(a, .{ .end = k }); + break :blk sorted; + }, + }; + if (has_name) |new_name| { + result.values._shape._tags.set(a, new_name.ptr); + result.indices._shape._tags.set(a, new_name.ptr); + } + return result; } pub const MaxPoolRes = ArgMaxRes; @@ -2767,7 +3893,7 @@ pub const Tensor = struct { window_strides: ?i64, base_dilations: i64 = 1, window_dilations: i64 = 1, - padding: []const i64 = &.{0}, + padding: [2]i64 = .{ 0, 0 }, }) MaxPoolRes { // TODO migrate to the following syntax. // maxPool(.{.a = .{ .stride = 5, .dilation = 2, .padding = .{0, 1} }, @@ -2778,74 +3904,76 @@ pub const Tensor = struct { // .padding = .{ .a = .{ 0, 2 }, .b = .{0, 2} // }) - meta.assert(opts.padding.len == 1 or opts.padding.len == 2 * self.rank(), "maxPool1d expects 'opts.padding' length to be a single integer or to be equal to the double of input tensor rank, got {} (input tensor rank is {})", .{ opts.padding.len, self.rank() }); - - // Note: the problem is initPoolArg assuming last axis // TODO: support maxPool on non last axis const a = self.axis(-1); + const ones: [constants.MAX_RANK]i64 = @splat(1); + var window_dimensions = ones; + window_dimensions[a] = opts.window_dimensions; + var window_strides = window_dimensions; + if (opts.window_strides) |stride| window_strides[a] = stride; - const window_dimensions = initPoolArg(self.rank(), &.{opts.window_dimensions}); - const window_strides = if (opts.window_strides) |ws| initPoolArg(self.rank(), &.{ws}) else window_dimensions; - const base_dilation = initPoolArg(self.rank(), &.{opts.base_dilations}); - const window_dilations = initPoolArg(self.rank(), &.{opts.window_dilations}); + var base_dilations = ones; + base_dilations[a] = opts.base_dilations; + var window_dilations = ones; + window_dilations[a] = opts.window_dilations; - return ops.reduceWindow( - MaxPoolRes.cmp, - .{ .values = self, .indices = iota(self._shape, .i32, a) }, - .{ .values = Tensor.constant(.{}, self.dtype().minValue()), .indices = Tensor.scalar(0, .i32) }, + var padding: [constants.MAX_RANK][2]i64 = @splat(.{ 0, 0 }); + padding[a] = opts.padding; + + const values, const indices = ops.reduceWindow( + 2, + .{ self, iota(self._shape, a) }, + .{ .constant(self.dtype().minValue()), .scalar(0, .i32) }, .{ .window_dimensions = window_dimensions[0..self.rank()], .window_strides = window_strides[0..self.rank()], - .base_dilations = base_dilation[0..self.rank()], + .base_dilations = base_dilations[0..self.rank()], .window_dilations = window_dilations[0..self.rank()], - .padding_values = opts.padding, - .padding_shape = &.{ @intCast(self.rank()), 2 }, + .padding = padding[0..self.rank()], }, + &MaxPoolRes.cmp, ); + return .{ .values = values, .indices = indices }; } /// Computes the 2d maxPool operation on the input Tensor. pub fn maxPool2d(self: Tensor, opts: struct { - window_dimensions: []const i64, - window_strides: ?[]const i64 = null, - base_dilations: []const i64 = &.{ 1, 1 }, - window_dilations: []const i64 = &.{ 1, 1 }, - padding: []const i64 = &.{0}, + window_dimensions: [2]i64, + window_strides: ?[2]i64 = null, + base_dilations: [2]i64 = .{ 1, 1 }, + window_dilations: [2]i64 = .{ 1, 1 }, + padding: [2][2]i64 = .{ .{ 0, 0 }, .{ 0, 0 } }, }) MaxPoolRes { - // TODO: rewrite using modern ZML, add ops.reduceWindow - meta.guard(self.rank() == 3 or self.rank() == 4, @src()); - meta.guard(opts.window_dimensions.len == 2, @src()); - meta.guard(opts.window_strides == null or opts.window_strides.?.len == 2, @src()); - meta.guard(opts.base_dilations.len == 2, @src()); - meta.guard(opts.window_dilations.len == 2, @src()); - meta.assert(opts.padding.len == 1 or opts.padding.len == 2 * self.rank(), "Padding needs to either be a single integer, or to be 2x time the number of input rank. In maxPool({}, .padding={d})", .{ self, opts.padding }); + // TODO: rewrite using modern ZML + stdx.debug.guard(self.rank() == 3 or self.rank() == 4, @src()); // TODO: support maxPool on non last axis // Note: the problem is initPoolArg assuming last axis const a = self.axis(-1); - const window_dimensions = initPoolArg(self.rank(), opts.window_dimensions); - const window_strides = if (opts.window_strides) |ws| initPoolArg(self.rank(), ws) else window_dimensions; - const base_dilation = initPoolArg(self.rank(), opts.base_dilations); - const window_dilations = initPoolArg(self.rank(), opts.window_dilations); + const window_dimensions = initPoolArg(self.rank(), &opts.window_dimensions); + const window_strides = if (opts.window_strides) |ws| initPoolArg(self.rank(), &ws) else window_dimensions; + const base_dilation = initPoolArg(self.rank(), &opts.base_dilations); + const window_dilations = initPoolArg(self.rank(), &opts.window_dilations); - return ops.reduceWindow( - MaxPoolRes.cmp, - .{ .values = self, .indices = iota(self._shape, .i32, a) }, - .{ .values = Tensor.constant(.{}, self.dtype().minValue()), .indices = Tensor.scalar(0, .i32) }, + var padding: [constants.MAX_RANK][2]i64 = @splat(.{ 0, 0 }); + padding[a - 1] = opts.padding[0]; + padding[a] = opts.padding[1]; + + const values, const indices = ops.reduceWindow( + 2, + .{ self, iota(self._shape, a) }, + .{ .constant(self.dtype().minValue()), .scalar(0, .i32) }, .{ .window_dimensions = window_dimensions[0..self.rank()], .window_strides = window_strides[0..self.rank()], .base_dilations = base_dilation[0..self.rank()], .window_dilations = window_dilations[0..self.rank()], - .padding_values = opts.padding, - .padding_shape = &.{ @intCast(self.rank()), 2 }, + .padding = padding[0..self.rank()], }, + &MaxPoolRes.cmp, ); - } - - pub inline fn axes(self: Tensor, axes_: anytype) std.BoundedArray(u3, Tensor.MAX_RANK) { - return self._shape.axes(axes_); + return .{ .values = values, .indices = indices }; } /// Chunk a given tensor into exactly n parts of equal shape. @@ -2857,7 +3985,7 @@ pub const Tensor = struct { var chunks: [n_chunks]Tensor = undefined; for (0..n_chunks) |i| { const start: i64 = @as(i64, @intCast(i)) * chunk_size; - chunks[i] = self.slice1d(a, .{ .start = start, .end = start + chunk_size }); + chunks[i] = self.slice(a, .{ .start = start, .end = start + chunk_size }); } return chunks; } @@ -2867,22 +3995,27 @@ pub const Tensor = struct { const platform = zml.testing.env(); // Only test shapes - var comp = try zml.module.CompilationContext.init(std.heap.page_allocator, "test", platform); + var comp: zml.Compiler = .init(std.testing.allocator, std.testing.io, platform, .{}); defer comp.deinit(); comp.activate(); defer comp.deactivate(); + const block = mlir.Block.init(&.{}, &.{}); + defer block.deinit(); + const scope = comp.pushBlock(block); + defer scope.pop(); + inline for (.{ .{ .{ .a = 12 }, .a, 3, .{ .a = 4 } }, .{ .{ .a = 12, .b = 2 }, .a, 3, .{ .a = 4, .b = 2 } }, .{ .{ 12, 2 }, 0, 3, .{ 4, 2 } }, }) |testcase| { const x_shape, const ax, const n_chunks, const res = testcase; - const x = Tensor.constant(x_shape, .{ .f16 = 0 }); + const x = Tensor.constant(.{ .f16 = 0 }).broad(Shape.init(x_shape, .f16)); const chunks = x.chunkExact(ax, n_chunks); const res_shape = Shape.init(res, .f16); - for (&chunks) |chk| { + for (chunks) |chk| { try zml.testing.expectEqualShapes(res_shape, chk.shape()); } } @@ -2893,23 +4026,19 @@ pub const Tensor = struct { pub fn chunkAllowTrailing( self: Tensor, axis_: i64, - n_chunks: comptime_int, - ) std.BoundedArray(Tensor, n_chunks + 1) { + n_chunks: usize, + ) []Tensor { const a = self.axis(axis_); - const d = self.dim(a); - const chunk_size: i64 = @divFloor(d, n_chunks); + const d: i64 = self.dim(a); + const chunk_size: i64 = @divFloor(d, @as(i64, @intCast(n_chunks))); const tail_chunk_size: i64 = @rem(d, chunk_size); - var chunks: std.BoundedArray(Tensor, n_chunks + 1) = .{}; - for (0..n_chunks) |i| { + const len: usize = if (tail_chunk_size == 0) n_chunks else n_chunks + 1; + const chunks = Compiler.current().alloc(Tensor, len); + + for (0.., chunks) |i, *chunk| { const start: i64 = @as(i64, @intCast(i)) * chunk_size; - chunks.appendAssumeCapacity( - self.slice1d(a, .{ .start = start, .end = start + chunk_size }), - ); - } - if (tail_chunk_size != 0) { - const start: i64 = n_chunks * chunk_size; - chunks.appendAssumeCapacity(self.slice1d(a, .{ .start = start })); + chunk.* = self.slice(a, .{ .start = start, .end = @min(start + chunk_size, d) }); } return chunks; } @@ -2919,11 +4048,16 @@ pub const Tensor = struct { const platform = zml.testing.env(); // Only test shapes - var comp = try zml.module.CompilationContext.init(std.heap.page_allocator, "test", platform); + var comp: zml.Compiler = .init(std.testing.allocator, std.testing.io, platform, .{}); defer comp.deinit(); comp.activate(); defer comp.deactivate(); + const block = mlir.Block.init(&.{}, &.{}); + defer block.deinit(); + const scope = comp.pushBlock(block); + defer scope.pop(); + inline for (.{ .{ .{ .a = 10 }, .a, 3, .{ .a = 3 }, .{ .a = 1 } }, .{ .{ .a = 10, .b = 2 }, .a, 3, .{ .a = 3, .b = 2 }, .{ .a = 1, .b = 2 } }, @@ -2931,152 +4065,85 @@ pub const Tensor = struct { .{ .{ 12, 2 }, 0, 3, .{ 4, 2 }, .{} }, }) |testcase| { const x_shape, const ax, const n_chunks, const res, const trailing = testcase; - const x = Tensor.constant(x_shape, .{ .f16 = 0 }); + const x = Tensor.zeroes(.init(x_shape, .f16)); const chunks = x.chunkAllowTrailing(x.axis(ax), n_chunks); const res_shape = Shape.init(res, .f16); - for (chunks.constSlice()[0..n_chunks]) |chk| { + for (chunks[0..n_chunks]) |chk| { try zml.testing.expectEqualShapes(res_shape, chk.shape()); } const trailing_shape = Shape.init(trailing, .f16); if (trailing_shape.rank() > 0) { try std.testing.expectEqual(n_chunks + 1, chunks.len); - try zml.testing.expectEqualShapes(trailing_shape, chunks.get(n_chunks).shape()); + try zml.testing.expectEqualShapes(trailing_shape, chunks[n_chunks].shape()); } else { try std.testing.expectEqual(n_chunks, chunks.len); } } } - pub fn split(self: Tensor, allocator: std.mem.Allocator, split_size_or_sections: []const i64, axis_: i64) ![]Tensor { - meta.assert(split_size_or_sections.len > 0, "split expects 'split_size_or_sections' length to be positive, got {}", .{split_size_or_sections.len}); + pub fn split(self: Tensor, axis_: anytype, split_sizes: []const i64) []Tensor { + stdx.debug.assert(split_sizes.len > 0, "split expects at least one 'split_sizes', got 0", .{}); const a = self.axis(axis_); - const length = self.dim(a); - if (split_size_or_sections.len != 1) { - var split_sum: i64 = 0; - for (split_size_or_sections) |n| split_sum += n; - meta.assert(split_sum == length, "split expects sum of 'split_size_or_sections' values and axis dimension to be equal, got {} and {}", .{ split_sum, length }); - } + const d = self.dim(a); + var split_sum: i64 = 0; + for (split_sizes) |n| split_sum += n; + stdx.debug.assert(split_sum == d, "split expects sum of 'split_sizes' values and axis dimension to be equal, got {} and {}", .{ split_sum, d }); - const res = try allocator.alloc(Tensor, split_size_or_sections.len); - errdefer allocator.dealloc(res); + const res = Compiler.current().alloc(Tensor, split_sizes.len); var start: i64 = 0; - for (split_size_or_sections, 0..) |n, i| { - res[i] = self.slice1d(a, .{ .start = start, .end = start + n }); + for (split_sizes, 0..) |n, i| { + res[i] = self.slice(a, .{ .start = start, .end = start + n }); start += n; } return res; } - /// Slices the input Tensor along a specific axis, with a start offset known at runtime. - pub fn dynamicSlice1d(self: Tensor, axis_: i8, len: u63, start_indices: Tensor) Tensor { - meta.assert(start_indices.rank() == 0, "dynamicSlice1d expects 'start_indices' tensor rank to be equal to 0, got {}", .{start_indices.rank()}); - - const a = self.axis(axis_); - const new_shape = self._shape.set(a, len); - const loc = self.getContext().mlirCtx().location(@src()).namedFmt(self.getContext().mlirCtx(), "axis={}, len={}", .{ axis_, len }); - var indices: [Tensor.MAX_RANK]mlir.Value = undefined; - for (0..self.rank()) |i| { - indices[i] = if (i == a) - start_indices.value() - else - constant(.{}, start_indices.dtype().zero()).value(); - } - - const op = dialect.stablehlo.dynamicSlice( - self.getContext().mlirCtx(), - self.value(), - new_shape.dims(), - indices[0..self.rank()], - loc, - ); - - return _result(new_shape, op.result(0)); - } - - pub const DynSlice = struct { start: Tensor, len: i64 }; - - /// Slices a Tensor across many axes, with runtime known offsets. - /// - /// Due to the nature of stablehlo, the length of the slices need to be known when compiling the IR. - /// When using the tagged API it is allowed to not specify some axes. - /// But with the non-tagged API all slices need to be specified. - /// Examples: - /// ``` - /// Tensor(.{.a=20,.b=30,.c=40 }).dynamicSlice(.{ .a = .{ .start = a_off, .len = 11}); - /// Tensor(.{.a=20,.b=30,.c=40 }).dynamicSlice(.{ - /// .a = .{ .start = a_off, .len = 11 }, - /// .b = .{ .start = b_off, .len = 12 }, - /// }); - /// Tensor(.{ 20,30,40}).dynamicSlice(.{.{ .start = scalar(0, .i32), .len = 20 }, .{ .start = b_off, .len = 12 }, .{ .start = scalar(0, .i32), .len = 40 }}); - /// ``` - pub fn dynamicSlice(self: Tensor, slices_: anytype) Tensor { - // TODO: the untagged api is a bit verbose. Should I allow: `Tensor(.{ 20,30,40}).dynamicSlice(.{.{}, .{ .start = b_off, .len = 12 }, .{}});` ?? - // - const slices, const slices_tags = Shape.parseStruct(DynSlice, slices_); - - // TODO use slices and slices_tags for the format. - // Currently this prints: "dynSlice(struct{q: struct{start: tensor.Tensor, comptime len: comptime_int = 1}}{ .q = struct{start: tensor.Tensor, comptime len: comptime_int = 1}{ .start = Tensor({1,10}, dtype=.i64), .len = 1 } })" - // which is kinda ugly. - const loc = self.getContext().mlirCtx().location(@src()).namedFmt(self.getContext().mlirCtx(), "dynSlice({any})", .{slices_}); - - const idx_dtype = if (slices.len > 0) slices.get(0).start.dtype() else .i32; - const zero = Tensor.scalar(0, idx_dtype).value(); - var offset_values = [_]mlir.Value{zero} ** MAX_RANK; - var res_shape = self._shape; - for (slices.constSlice(), 0..) |slice_, i| { - const offset = slice_.start; - const len = slice_.len; - if (slices_tags.len == 0) { - meta.assert(self.rank() == slices.len, "dynamicSlice expects tensor rank and 'slices_' length to be equal, got {} and {}", .{ self.rank(), slices.len }); - - offset_values[i] = offset.value(); - res_shape._dims.set(i, len); - - meta.assert(len <= self.dim(i), "dynamicSlice expects slices 'len' to be less than or equal to their corresponding dimension in input tensor, got {} and {} for index {}", .{ len, self.dim(i), i }); - } else { - const t = slices_tags.get(i); - const a = res_shape.hasTag(t) orelse meta.panic("dynamicSlice expects input tensor to have tags used in 'slices_' but {s} is missing (input shape is {})", .{ t, self._shape }); - - meta.assert(len <= self.dim(a), "dynamicSlice expects slices 'len' to be less than their corresponding dimension in input tensor, got {} and {} for axis {s}", .{ len, self.dim(a), t }); - - offset_values[a] = offset.value(); - res_shape._dims.set(a, len); - } - } - const op = dialect.stablehlo.dynamicSlice(self.getContext().mlirCtx(), self.value(), res_shape.dims(), offset_values[0..self.rank()], loc); - return _result(res_shape, op.result(0)); - } - - test dynamicSlice { + test split { const zml = @import("zml.zig"); const platform = zml.testing.env(); - const T = f32; - { - const x = try zml.Buffer.fromArray(platform, [10]T{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }); - const z = try zml.Buffer.scalar(platform, 4, .i32); - const res = try zml.testing.compileAndCall(platform, Tensor.dynamicSlice1d, .{ x, 0, 2, z }); + const Local = struct { + pub fn _split(x: Tensor, axis_: i8, split_sizes: []const i64) []Tensor { + return x.split(axis_, split_sizes); + } + }; - try testing.expectEqual([2]T{ 4, 5 }, try res.getValue([2]T)); - } + inline for (.{ + .{ [3]f32{ 0, 1, 2 }, Shape.init(.{3}, .f32), 0, [3]i64{ 1, 1, 1 }, &.{ [1]f32{0}, [1]f32{1}, [1]f32{2} } }, + .{ [10]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }, Shape.init(.{10}, .f32), 0, [2]i64{ 1, 9 }, &.{ [1]f32{0}, [9]f32{ 1, 2, 3, 4, 5, 6, 7, 8, 9 } } }, + .{ [4][3]f32{ .{ 0, 1, 2 }, .{ 3, 4, 5 }, .{ 6, 7, 8 }, .{ 9, 10, 11 } }, Shape.init(.{ 4, 3 }, .f32), 0, [2]i64{ 1, 3 }, &.{ [1][3]f32{.{ 0, 1, 2 }}, [3][3]f32{ .{ 3, 4, 5 }, .{ 6, 7, 8 }, .{ 9, 10, 11 } } } }, + .{ [2][6]f32{ .{ 0, 1, 2, 3, 4, 5 }, .{ 6, 7, 8, 9, 10, 11 } }, Shape.init(.{ 2, 6 }, .f32), 1, [2]i64{ 2, 4 }, &.{ [2][2]f32{ .{ 0, 1 }, .{ 6, 7 } }, [2][4]f32{ .{ 2, 3, 4, 5 }, .{ 8, 9, 10, 11 } } } }, + }) |testcase| { + const x_data, const x_shape, const ax, const args, const expectation = testcase; + const x: Tensor = .fromShape(x_shape); - { - // Strided - const x = try zml.Buffer.fromArray(platform, [2][5]T{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } }); - const z = try zml.Buffer.scalar(platform, 3, .i32); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._split, .{ x, ax, &args }, platform, .{}); + defer exe.deinit(); - const res = try zml.testing.compileAndCall(platform, Tensor.dynamicSlice1d, .{ x, 1, 2, z }); - try testing.expectEqual([4]T{ 3, 4, 8, 9 }, res.getValue([4]T)); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&x_data)); + defer x_buffer.deinit(); + + const res = try std.testing.allocator.alloc(zml.Buffer, expectation.len); + defer std.testing.allocator.free(res); + var runner = try exe.runner(std.testing.allocator); + defer runner.deinit(std.testing.allocator); + runner.run(std.testing.io, .{x_buffer}, res, .{ .wait = true }); + defer for (res) |*buffer| buffer.deinit(); + + inline for (expectation, 0..) |expected, i| { + try std.testing.expectEqual(expected, try res[i].getValue(@TypeOf(expected), std.testing.io)); + } } } /// Updates a slice of the input Tensor along a specific axis using the given 'update' Tensor, with a start offset known at runtime. + /// Note this is the untagged api, if you have tags, you should use dynamicUpdateSlice directly. pub fn dynamicUpdateSlice1d(self: Tensor, update: Tensor, axis_: i64, offset: Tensor) Tensor { const placeholder = Tensor.scalar(0, .i32); - var start_indices = [_]Tensor{placeholder} ** MAX_RANK; + var start_indices: [constants.MAX_RANK]Tensor = @splat(placeholder); start_indices[self.axis(axis_)] = offset; return self.dynamicUpdateSlice(start_indices[0..self.rank()], update); } @@ -3091,12 +4158,13 @@ pub const Tensor = struct { /// Tensor(.{ .a = 2, .b = 5 }).dynamicUpdateSlice(.{ .a = scalar(1, .i32) }, Tensor(.{ .b = 5 })); /// ``` pub fn dynamicUpdateSlice(self: Tensor, offset_: anytype, update_: Tensor) Tensor { - meta.assert(self.dtype() == update_.dtype(), "dynamicUpdateSlice expects input and 'update_' tensors to be of the same type, got {} and {}", .{ self.dtype(), update_.dtype() }); + // TODO: add updateSlice for when the offset isn't dynamic + stdx.debug.assert(self.dtype() == update_.dtype(), "dynamicUpdateSlice expects input and 'update_' tensors to be of the same type, got {} and {}", .{ self.dtype(), update_.dtype() }); const offset, const offset_tags = Shape.parseStruct(Tensor, offset_); // log.debug("offset: {any}, offset_tags: {any}", .{ offset, offset_tags }); for (offset.constSlice(), 0..) |start_idx, i| { - meta.assert(start_idx.rank() == 0, "dynamicUpdateSlice expects 'offset_' tensor ranks to be equal to 0, got {} at index {}", .{ start_idx.rank(), i }); + stdx.debug.assert(start_idx.rank() == 0, "dynamicUpdateSlice expects 'offset_' tensor ranks to be equal to 0, got {} at index {}", .{ start_idx.rank(), i }); } const tagged_api = update_._shape.isFullyTagged() and self._shape.isFullyTagged() and offset_tags.len > 0; @@ -3106,14 +4174,14 @@ pub const Tensor = struct { if (tagged_api) { // Check that all update tags are known. for (update._shape._tags.constSlice()) |t| { - meta.assert(self._shape.hasTag(t) != null, "dynamicUpdateSlice expects 'update_' tensor tags to be a subset of input tensor tags but {s} is missing (input shape is {})", .{ t, self._shape }); + stdx.debug.assert(self._shape.hasTag(t) != null, "dynamicUpdateSlice expects 'update_' tensor tags to be a subset of input tensor tags but {s} is missing (input shape is {f})", .{ t, self._shape }); } var update_shape = self._shape; var prev_ax: i8 = -1; for (self._shape.tags(), 0..) |t, self_ax| { if (update._shape.hasTag(t)) |up_ax| { - meta.assert(up_ax == prev_ax + 1, "dynamicUpdateSlice expects 'update_' and input tensor axis to have the same order, got {} and {}. (hint: you need to explicitly transpose 'update_')", .{ update_._shape, self._shape }); + stdx.debug.assert(up_ax == prev_ax + 1, "dynamicUpdateSlice expects 'update_' and input tensor axis to have the same order, got {f} and {f}. (hint: you need to explicitly transpose 'update_')", .{ update_, self }); update_shape._dims.set(self_ax, update.dim(up_ax)); prev_ax = up_ax; @@ -3124,25 +4192,25 @@ pub const Tensor = struct { update = update.reshape(update_shape); } - meta.assert(self.rank() == update.rank(), "dynamicUpdateSlice expects input and computed update tensors to have the same rank, got {} and {} (hint: it's probably an issue on our side)", .{ self.rank(), update.rank() }); + stdx.debug.assert(self.rank() == update.rank(), "dynamicUpdateSlice expects input and computed update tensors to have the same rank, got {f} and {f}", .{ self, update }); for (self.dims(), update.dims(), 0..) |self_d, up_d, ax| { const t = self._shape.debugTag(ax); - meta.assert(up_d <= self_d, "dynamicUpdateSlice expects 'update_' dimensions to be less than or equal to their corresponding dimension in input tensor, got {} and {} for axis .{s}", .{ up_d, self_d, t }); + stdx.debug.assert(up_d <= self_d, "dynamicUpdateSlice expects 'update_' dimensions to be less than or equal to their corresponding dimension in input tensor, got {} and {} for axis .{s}", .{ up_d, self_d, t }); if (tagged_api and up_d < self_d) { const axis_has_offset = std.mem.indexOfScalar(Shape.Tag, offset_tags.constSlice(), self._shape._tags.get(ax)) != null; - meta.assert(axis_has_offset, "dynamicUpdateSlice expects 'update_' dimensions to be equal to their corresponding dimension in input tensor, got {} and {} for axis .{s} (hint: you need to provide an offset)", .{ up_d, self_d, t }); + stdx.debug.assert(axis_has_offset, "dynamicUpdateSlice expects 'update_' dimensions to be equal to their corresponding dimension in input tensor, got {} and {} for axis .{s} (hint: you need to provide an offset)", .{ up_d, self_d, t }); } } const idx_dtype = if (offset.len > 0) offset.get(0).dtype() else .i32; const zero = Tensor.scalar(0, idx_dtype).value(); - var offset_values: [MAX_RANK]mlir.Value = undefined; + var offset_values: [constants.MAX_RANK]*const mlir.Value = undefined; if (offset_tags.len == 0) { // Without offset tags we need the same number of offset than rank. - meta.assert(self.rank() == offset.len, "dynamicUpdateSlice expects input tensor rank and 'offset_' length to be equal, got {} and {}", .{ self.rank(), offset.len }); + stdx.debug.assert(self.rank() == offset.len, "dynamicUpdateSlice expects input tensor rank and 'offset_' length to be equal, got {} and {}", .{ self.rank(), offset.len }); for (offset.constSlice(), 0..) |idx, i| { offset_values[i] = idx.value(); @@ -3150,21 +4218,20 @@ pub const Tensor = struct { } else { // If an axis isn't specified, update the full slice. // This is only allowed when using tagged sliced. - offset_values = .{zero} ** MAX_RANK; + offset_values = @splat(zero); for (offset.constSlice(), offset_tags.constSlice()) |start, t| { - const a = self._shape.hasTag(t) orelse meta.panic("dynamicUpdateSlice expects input tensor to have tags used in 'offset_' but {s} is missing (input shape is {})", .{ t, self._shape }); + const a = self._shape.hasTag(t) orelse stdx.debug.panic("dynamicUpdateSlice expects input tensor to have tags used in 'offset_' but {s} is missing (input shape is {f})", .{ t, self._shape }); offset_values[a] = start.value(); } } - const loc = self.getContext().mlirCtx().location(@src()); - const op = dialect.stablehlo.dynamic_update_slice( - self.getContext().mlirCtx(), + const op = stablehlo.dynamic_update_slice( + mlirCtx(), self.value(), update.value(), offset_values[0..self.rank()], - loc, - ); + currentLoc(), + ).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } @@ -3173,178 +4240,400 @@ pub const Tensor = struct { const platform = zml.testing.env(); { - const x = try zml.Buffer.fromArray(platform, [10]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }); - const y = try zml.Buffer.fromArray(platform, [2]f32{ -1, -1 }); - const idx = try zml.Buffer.scalar(platform, 4, .i32); - const res = try zml.testing.compileAndCall( - platform, - struct { - pub fn forward(x_: Tensor, idx_: struct { a: Tensor }, y_: Tensor) Tensor { - return x_.dynamicUpdateSlice(idx_, y_); - } - }.forward, - .{ x.withTags(.{.a}), .{ .a = idx }, y.withTags(.{.a}) }, - ); - try testing.expectEqual([10]f32{ 0, 1, 2, 3, -1, -1, 6, 7, 8, 9 }, try res.getValue([10]f32)); + const x = Tensor.init(.{10}, .f32).withTags(.{.a}); + const y = Tensor.init(.{2}, .f32).withTags(.{.a}); + const ids: Tensor = .init(.{}, .i32); + const forward = struct { + pub fn _fwd(x_: Tensor, idx_: struct { a: Tensor }, y_: Tensor) Tensor { + return x_.dynamicUpdateSlice(idx_, y_); + } + }._fwd; + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, .{ .a = ids }, y }, platform, .{}); + defer exe.deinit(); + + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[10]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 })); + defer x_buffer.deinit(); + var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), std.mem.sliceAsBytes(&[2]f32{ -1, -1 })); + defer y_buffer.deinit(); + var ids_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, ids.shape(), std.mem.sliceAsBytes(&[1]i32{4})); + defer ids_buffer.deinit(); + + var res = try exe.eval(std.testing.allocator, std.testing.io, .{ x_buffer, .{ .a = ids_buffer }, y_buffer }); + defer res.deinit(); + + try std.testing.expectEqual([10]f32{ 0, 1, 2, 3, -1, -1, 6, 7, 8, 9 }, try res.getValue([10]f32, std.testing.io)); } { // Updates 2D, tagged api - const x = try zml.Buffer.fromArray(platform, [2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } }); - const y = try zml.Buffer.fromArray(platform, [2]f32{ -1, -1 }); - const idx = try zml.Buffer.scalar(platform, 3, .i32); + const x = Tensor.init(.{ 2, 5 }, .f32).withTags(.{ .a, .b }); + const y = Tensor.init(.{2}, .f32).withTags(.{.a}); + const ids: Tensor = .init(.{}, .i32); + const forward = struct { + pub fn _fwd(x_: Tensor, idx_: Tensor, y_: Tensor) Tensor { + return x_.dynamicUpdateSlice(.{ .b = idx_ }, y_); + } + }._fwd; + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, ids, y }, platform, .{}); + defer exe.deinit(); - const res = try zml.testing.compileAndCall( - platform, - struct { - pub fn forward(x_: Tensor, idx_: Tensor, y_: Tensor) Tensor { - return x_.dynamicUpdateSlice(.{ .b = idx_ }, y_); - } - }.forward, - .{ x.withTags(.{ .a, .b }), idx, y.withTags(.{.a}) }, - ); - try testing.expectEqualDeep( + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } })); + defer x_buffer.deinit(); + var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), std.mem.sliceAsBytes(&[2]f32{ -1, -1 })); + defer y_buffer.deinit(); + var ids_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, ids.shape(), std.mem.sliceAsBytes(&[1]i32{3})); + defer ids_buffer.deinit(); + + var res = try exe.eval(std.testing.allocator, std.testing.io, .{ x_buffer, ids_buffer, y_buffer }); + defer res.deinit(); + + try std.testing.expectEqualDeep( [2][5]f32{ .{ 0, 1, 2, -1, 4 }, .{ 5, 6, 7, -1, 9 } }, - try res.getValue([2][5]f32), + try res.getValue([2][5]f32, std.testing.io), ); } { // Updates 2D slice, un-tagged api. Note that `y` needs to have a 1 dimension axis. - const x = try zml.Buffer.fromArray(platform, [2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } }); - const y = try zml.Buffer.fromArray(platform, [2][1]f32{ .{-1}, .{-1} }); - const idx = try zml.Buffer.scalar(platform, 3, .i32); - const res = try zml.testing.compileAndCall( - platform, - struct { - pub fn forward(x_: Tensor, idx_: Tensor, y_: Tensor) Tensor { - return x_.dynamicUpdateSlice(.{ zml.Tensor.scalar(0, .i32), idx_ }, y_); - } - }.forward, - .{ x, idx, y }, - ); - try testing.expectEqualDeep( + const x: Tensor = .init(.{ 2, 5 }, .f32); + const y: Tensor = .init(.{ 2, 1 }, .f32); + const ids: Tensor = .init(.{}, .i32); + const forward = struct { + pub fn _fwd(x_: Tensor, idx_: Tensor, y_: Tensor) Tensor { + return x_.dynamicUpdateSlice(.{ zml.Tensor.scalar(0, .i32), idx_ }, y_); + } + }._fwd; + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, ids, y }, platform, .{}); + defer exe.deinit(); + + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } })); + defer x_buffer.deinit(); + var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), std.mem.sliceAsBytes(&[2][1]f32{ .{-1}, .{-1} })); + defer y_buffer.deinit(); + var ids_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, ids.shape(), std.mem.sliceAsBytes(&[1]i32{3})); + defer ids_buffer.deinit(); + + var res = try exe.eval(std.testing.allocator, std.testing.io, .{ x_buffer, ids_buffer, y_buffer }); + defer res.deinit(); + + try std.testing.expectEqualDeep( [2][5]f32{ .{ 0, 1, 2, -1, 4 }, .{ 5, 6, 7, -1, 9 } }, - res.getValue([2][5]f32), + res.getValue([2][5]f32, std.testing.io), ); } { // Updates 2D, partial update - const x = try zml.Buffer.fromArray(platform, [2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } }); - const y = try zml.Buffer.fromArray(platform, [1]f32{-1}); - const idx_a = try zml.Buffer.scalar(platform, 1, .i32); - const idx_b = try zml.Buffer.scalar(platform, 3, .i32); - const res = try zml.testing.compileAndCall( - platform, - struct { - pub fn forward(x_: Tensor, idx_: struct { a: Tensor, b: Tensor }, y_: Tensor) Tensor { - return x_.dynamicUpdateSlice(idx_, y_); - } - }.forward, - .{ x.withTags(.{ .a, .b }), .{ .a = idx_a, .b = idx_b }, y.withTags(.{.a}) }, - ); - try testing.expectEqualDeep( + const x = Tensor.init(.{ 2, 5 }, .f32).withTags(.{ .a, .b }); + const y = Tensor.init(.{1}, .f32).withTags(.{.a}); + const idx_a: Tensor = .init(.{}, .i32); + const idx_b: Tensor = .init(.{}, .i32); + const forward = struct { + pub fn _fwd(x_: Tensor, idx_: struct { a: Tensor, b: Tensor }, y_: Tensor) Tensor { + return x_.dynamicUpdateSlice(idx_, y_); + } + }._fwd; + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, .{ .a = idx_a, .b = idx_b }, y }, platform, .{}); + defer exe.deinit(); + + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } })); + defer x_buffer.deinit(); + var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), std.mem.sliceAsBytes(&[1]f32{-1})); + defer y_buffer.deinit(); + var idx_a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, idx_a.shape(), std.mem.sliceAsBytes(&[1]i32{1})); + defer idx_a_buffer.deinit(); + var idx_b_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, idx_b.shape(), std.mem.sliceAsBytes(&[1]i32{3})); + defer idx_b_buffer.deinit(); + + var res = try exe.eval(std.testing.allocator, std.testing.io, .{ x_buffer, .{ .a = idx_a_buffer, .b = idx_b_buffer }, y_buffer }); + defer res.deinit(); + + try std.testing.expectEqualDeep( [2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, -1, 9 } }, - res.getValue([2][5]f32), + res.getValue([2][5]f32, std.testing.io), ); } { // Updates 2D, partial update, un-tagged api. - const x = try zml.Buffer.fromArray(platform, [2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } }); - const y = try zml.Buffer.fromArray(platform, [1][1]f32{.{-1}}); - const idx_a = try zml.Buffer.scalar(platform, 1, .i32); - const idx_b = try zml.Buffer.scalar(platform, 3, .i32); - const A = struct { - pub fn forward(x_: Tensor, idx_: [2]Tensor, y_: Tensor) Tensor { + const x: Tensor = .init(.{ 2, 5 }, .f32); + const y: Tensor = .init(.{ 1, 1 }, .f32); + const idx_a: Tensor = .init(.{}, .i32); + const idx_b: Tensor = .init(.{}, .i32); + const forward = struct { + pub fn _fwd(x_: Tensor, idx_: [2]Tensor, y_: Tensor) Tensor { return x_.dynamicUpdateSlice(&idx_, y_); } - }; - const res = try zml.testing.compileAndCall(platform, A.forward, .{ x, .{ idx_a, idx_b }, y }); - try testing.expectEqualDeep( + }._fwd; + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, .{ idx_a, idx_b }, y }, platform, .{}); + defer exe.deinit(); + + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } })); + defer x_buffer.deinit(); + var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), std.mem.sliceAsBytes(&[1][1]f32{.{-1}})); + defer y_buffer.deinit(); + var idx_a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, idx_a.shape(), std.mem.sliceAsBytes(&[1]i32{1})); + defer idx_a_buffer.deinit(); + var idx_b_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, idx_b.shape(), std.mem.sliceAsBytes(&[1]i32{3})); + defer idx_b_buffer.deinit(); + + var res = try exe.eval(std.testing.allocator, std.testing.io, .{ x_buffer, .{ idx_a_buffer, idx_b_buffer }, y_buffer }); + defer res.deinit(); + + try std.testing.expectEqualDeep( [2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, -1, 9 } }, - res.getValue([2][5]f32), + res.getValue([2][5]f32, std.testing.io), ); } } /// Returns a Tensor containing the element-wise result of the given 'cmp' comparison between the two input Tensors. - pub fn cmp(self: Tensor, direction: dialect.stablehlo.ComparisonDirection.Direction, other: Tensor) Tensor { - meta.assert(self.dtype() == other.dtype(), "cmp expects input tensors to be of the same type, got {} and {}", .{ self.dtype(), other.dtype() }); + pub fn cmp(self: Tensor, direction: stablehlo.ComparisonDirectionAttr.Direction, other: Tensor) Tensor { + stdx.debug.assert(self.dtype() == other.dtype(), "cmp expects input tensors to be of the same type, got {t} and {t}", .{ self.dtype(), other.dtype() }); if (self.rank() == 0 and other.rank() != 0) return self.broadcast(other._shape, &.{}).cmp(direction, other); if (self.rank() != 0 and other.rank() == 0) return self.cmp(direction, other.broadcast(self._shape, &.{})); - meta.assert(self._shape.eql(other._shape), "cmp expects input tensor shapes to match, got {} and {}", .{ self._shape, other._shape }); + stdx.debug.assert(self._shape.eql(other._shape), "cmp expects input tensor shapes to match, got {f} and {f}", .{ self._shape, other._shape }); - const loc = self.getContext().mlirCtx().location(@src()).namedFmt(self.getContext().mlirCtx(), "cmp(.{s})", .{@tagName(direction)}); - const op = dialect.stablehlo.compare( - self.getContext().mlirCtx(), + const op = stablehlo.compare( + mlirCtx(), self.value(), other.value(), - dialect.stablehlo.ComparisonDirection.init(self.getContext().mlirCtx(), direction), - getComparisonType(self.getContext().mlirCtx(), self.dtype()), - loc, - ); + stablehlo.ComparisonDirectionAttr.get(mlirCtx(), .{ .value = direction }).value(), + getComparisonType(mlirCtx(), self.dtype()).value(), + currentLoc(), + ).appendTo(currentBlock()); return _result(self._shape.withDtype(.bool), op.result(0)); } + /// For each vector in the input tensor, + /// creates a diagonal-matrix where diagonal values are set to the vector values. + pub fn toDiagonal(self: Tensor, axis_: anytype, new_tags: [2]@EnumLiteral()) Tensor { + stdx.debug.assert(self.rank() < constants.MAX_RANK - 1, "toDiagonal expects input up to {d} rank, got {f}", .{ constants.MAX_RANK - 1, self }); + const a = self.axis(axis_); + const d = self.dim(a); + const p = self.shape()._sharding.partition.get(a); + var res_shape = self._shape; + res_shape._dims.replaceRange(a, 1, &.{ d, d }) catch unreachable; + res_shape._tags.replaceRange(a, 1, &.{ @tagName(new_tags[0]), @tagName(new_tags[1]) }) catch unreachable; + + res_shape._sharding.partition = res_shape._sharding.partition.insert(a, p); + + const values = self.insertAxes(a + 1, .{new_tags[1]}).broad(res_shape); + const zeros = Tensor.constant(self.dtype().zero()).broad(res_shape); + + const x = Tensor.iota(res_shape, a); + const y = Tensor.iota(res_shape, a + 1); + var res = x.cmp(.EQ, y).select(values, zeros); + res._shape = res_shape; + return res; + } + + test toDiagonal { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + const Local = struct { + pub fn _toDiag(input: Tensor) Tensor { + const res = input.toDiagonal(-1, .{ .x, .y }); + std.debug.assert(res.dim(.x) == input.dim(-1)); + std.debug.assert(res.dim(.y) == input.dim(-1)); + return res; + } + }; + + const x: Tensor = .init(.{ 2, 2 }, .u8); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._toDiag, .{x}, platform, .{}); + defer exe.deinit(); + + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[2][2]u8{ .{ 1, 2 }, .{ 3, 4 } })); + defer x_buffer.deinit(); + + var res = try exe.eval(std.testing.allocator, std.testing.io, .{x_buffer}); + defer res.deinit(); + + try std.testing.expectEqual( + [2][2][2]u8{ + .{ .{ 1, 0 }, .{ 0, 2 } }, + .{ .{ 3, 0 }, .{ 0, 4 } }, + }, + try res.getValue([2][2][2]u8, std.testing.io), + ); + } + + /// For each matrix specified by the two axes, returns the lower triangular part of it. + /// The other elements are set to 0. + /// Usage: `.{ .b = 32, .w = 20, .h = 20 }.triangular(.{ .w, .h}, 0);` + /// + /// * if `num_diagonals` is set to 0, the diagonal is not modified. + /// * if set to -1, the diagonal is set to 0 + /// * if set to n, the n "quasi diagonal" above the diagonal are conserved. + /// + /// To get the upper triangular part, swap the order of axes: + /// `.{ .b = 32, .w = 20, .h = 20 }.triangular(.{ .h, .w }, 0);` + pub fn triangular(self: Tensor, axes_: anytype, num_diagonals: i32) Tensor { + stdx.debug.assertComptime(stdx.meta.isTuple(@TypeOf(axes_)) and axes_.len == 2, "triangular expects exactly two axes to work on.", .{}); + const _axes = self.axes(axes_); + + const x = Tensor.iota(self.shape(), _axes.get(0)); + const y = Tensor.iota(self.shape(), _axes.get(1)); + + const zeros = Tensor.constant(self.dtype().zero()).broad(self.shape()); + return x.addConstant(num_diagonals).cmp(.GE, y).select(self, zeros); + } + + test triangular { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + const Local = struct { + pub fn _tri(input: Tensor, num_diagonals: i32) Tensor { + return input.triangular(.{ -2, -1 }, num_diagonals); + } + }; + + const x: Tensor = .init(.{ 3, 3 }, .u8); + + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[3][3]u8{ + .{ 1, 1, 1 }, + .{ 1, 1, 1 }, + .{ 1, 1, 1 }, + })); + defer x_buffer.deinit(); + + { + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._tri, .{ x, 0 }, platform, .{}); + defer exe.deinit(); + + var res = try exe.eval(std.testing.allocator, std.testing.io, .{x_buffer}); + defer res.deinit(); + + try std.testing.expectEqual( + [3][3]u8{ + .{ 1, 0, 0 }, + .{ 1, 1, 0 }, + .{ 1, 1, 1 }, + }, + try res.getValue([3][3]u8, std.testing.io), + ); + } + { + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._tri, .{ x, 1 }, platform, .{}); + defer exe.deinit(); + + var res = try exe.eval(std.testing.allocator, std.testing.io, .{x_buffer}); + defer res.deinit(); + + try std.testing.expectEqual( + [3][3]u8{ + .{ 1, 1, 0 }, + .{ 1, 1, 1 }, + .{ 1, 1, 1 }, + }, + try res.getValue([3][3]u8, std.testing.io), + ); + } + { + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._tri, .{ x, -1 }, platform, .{}); + defer exe.deinit(); + + var res = try exe.eval(std.testing.allocator, std.testing.io, .{x_buffer}); + defer res.deinit(); + + try std.testing.expectEqual( + [3][3]u8{ + .{ 0, 0, 0 }, + .{ 1, 0, 0 }, + .{ 1, 1, 0 }, + }, + try res.getValue([3][3]u8, std.testing.io), + ); + } + } + /// For each element at index `i`, if `bool_tensor[i] == true`, `output[i] = on_true[i]` /// otherwise, if `bool_tensor[i] == false`, `output[i] = on_false[i]` pub fn select(bool_tensor: Tensor, on_true: Tensor, on_false: Tensor) Tensor { - meta.assert(bool_tensor.dtype() == .bool, "select expects input tensor type to be a boolean, got {}", .{bool_tensor.dtype()}); - meta.assert(on_true.dtype() == on_false.dtype(), "select expects 'on_true' and 'on_false' tensor types to be equal, got {} and {}", .{ on_true.dtype(), on_false.dtype() }); - meta.assert(bool_tensor._shape.eqlDims(on_true._shape), "select expects input tensor and 'on_true' tensor dimensions to match, got {} and {}", .{ bool_tensor._shape, on_true._shape }); - meta.assert(bool_tensor._shape.eqlDims(on_false._shape), "select expects input tensor and 'on_false' tensor dimensions to match, got {} and {}", .{ bool_tensor._shape, on_false._shape }); + stdx.debug.assert(bool_tensor.dtype() == .bool, "select expects input tensor type to be a boolean, got {}", .{bool_tensor.dtype()}); + stdx.debug.assert(on_true.dtype() == on_false.dtype(), "select expects 'on_true' and 'on_false' tensor types to be equal, got {} and {}", .{ on_true.dtype(), on_false.dtype() }); - const loc = bool_tensor.getContext().mlirCtx().location(@src()); - const op = dialect.stablehlo.select( - bool_tensor.getContext().mlirCtx(), + if (bool_tensor.rank() != 0 and on_true.rank() == 0) { + return bool_tensor.select(on_true.broad(bool_tensor.shape()), on_false); + } + if (bool_tensor.rank() != 0 and on_false.rank() == 0) { + return bool_tensor.select(on_true, on_false.broad(bool_tensor.shape())); + } + + stdx.debug.assert(bool_tensor._shape.eqlDims(on_true._shape), "select expects input tensor and 'on_true' tensor dimensions to match, got {f} and {f}", .{ bool_tensor._shape, on_true._shape }); + stdx.debug.assert(bool_tensor._shape.eqlDims(on_false._shape), "select expects input tensor and 'on_false' tensor dimensions to match, got {f} and {f}", .{ bool_tensor._shape, on_false._shape }); + + const op = stablehlo.select( + mlirCtx(), bool_tensor.value(), on_true.value(), on_false.value(), - loc, - ); + currentLoc(), + ).appendTo(currentBlock()); return _result(on_true._shape, op.result(0)); } + pub fn mask(x: Tensor, pred: Tensor, mask_value: anytype) Tensor { + return pred.broad(x.shape()).select(x, .scalar(mask_value, x.dtype())); + } + /// Returns a Tensor containing the element-wise not logical operation of the input Tensor. pub fn not(self: Tensor) Tensor { - const loc = self.getContext().mlirCtx().location(@src()); - const op = dialect.stablehlo.not(self.getContext().mlirCtx(), self.value(), loc); + const op = stablehlo.not(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } /// Returns a Tensor containing boolean indicating if there is a non-zero value over the given axis. - pub fn any(self: Tensor, axis_: i64) Tensor { + pub fn any(self: Tensor, axis_: anytype) Tensor { const pred = self.cmp(.NE, Tensor.constant(self.dims(), self.dtype().zero())); - const red = ops.reduce( + return ops.reduce( + .{pred}, + .{Tensor.scalar(false, .bool)}, + &.{self.axis(axis_)}, struct { - pub fn acc(x: Tensor, res: Tensor) Tensor { - return res.logical(.OR, x); + pub fn acc(args: ops.ReduceArgs) Tensor { + return args.left.logical(.OR, args.right); } }.acc, - pred, - Tensor.scalar(0, pred.dtype()), + .{}, + )[0]; + } + + /// Returns a Tensor containing boolean indicating if there is a non-zero value over the given axis. + pub fn all(self: Tensor, axis_: anytype) Tensor { + const pred = if (self.dtype() == .bool) self else self.cmp(.NE, Tensor.scalar(0, self.dtype())); + return ops.reduce( + .{pred}, + .{Tensor.scalar(true, .bool)}, &.{self.axis(axis_)}, - ); - return red; + struct { + pub fn acc(args: ops.ReduceArgs) Tensor { + return args.left.logical(.AND, args.right); + } + }.acc, + .{}, + )[0]; } /// Given a set of N vectors of lengths A, B, C, D, /// returns N tensors of rank N, and shape (A, B, C, D). - /// For any coordinate (a, b, c, d), - /// we have: + /// For any coordinate (a, b, c, d), we have: + /// /// - res[0][a, b, c, d] == A[a] /// - res[1][a, b, c, d] == B[b] /// - res[2][a, b, c, d] == C[c] /// - res[3][a, b, c, d] == D[d] + /// /// This is implemented with broadcasting, so typically it won't copy. /// In Pytorch/Numpy this is know as `meshgrid` with "ij" mode. - /// See torch.meshgrid for the "xy" mode. + /// See `zml.torch.meshgrid` for the "xy" mode. pub fn cartesianProduct(comptime N: u3, vectors: [N]Tensor) [N]Tensor { var out: @TypeOf(vectors) = undefined; _cartesianProduct(&vectors, &out); @@ -3352,11 +4641,11 @@ pub const Tensor = struct { } fn _cartesianProduct(vectors: []const Tensor, out: []Tensor) void { - meta.assert(vectors.len >= 1, "cartesianProduct expects at least one input.", .{}); - meta.assert(vectors.len < Tensor.MAX_RANK, "cartesianProduct expects at most {} input vectors, received {} !", .{ Tensor.MAX_RANK - 1, vectors.len }); + stdx.debug.assert(vectors.len >= 1, "cartesianProduct expects at least one input.", .{}); + stdx.debug.assert(vectors.len < constants.MAX_RANK, "cartesianProduct expects at most {} input vectors, received {} !", .{ constants.MAX_RANK - 1, vectors.len }); for (vectors) |x| { - meta.assert(x.rank() <= 1, "cartesianProduct expects 0 or 1 rank input vectors. Got: {any}", .{vectors}); - meta.assert(vectors[0].dtype() == x.dtype(), "cartesianProduct expects input vectors to have all the same dtype. Got: {any}", .{vectors}); + stdx.debug.assert(x.rank() <= 1, "cartesianProduct expects 0 or 1 rank input vectors. Got: {any}", .{vectors}); + stdx.debug.assert(vectors[0].dtype() == x.dtype(), "cartesianProduct expects input vectors to have all the same dtype. Got: {any}", .{vectors}); } var res_shape = Shape.init(.{}, vectors[0].dtype()); @@ -3375,53 +4664,77 @@ pub const Tensor = struct { test cartesianProduct { const zml = @import("zml.zig"); - const client = zml.testing.env(); + const platform = zml.testing.env(); - const x = try zml.Buffer.fromSlice(client, .{6}, &[_]i32{ 0, 1, 2, 3, 4, 5 }); - const y = try zml.Buffer.fromSlice(client, .{4}, &[_]i32{ 0, 1, 2, 3 }); + const x: Tensor = .init(.{6}, .i32); + const y: Tensor = .init(.{4}, .i32); const Local = struct { - pub fn cartesianProduct2(a: Tensor, b: Tensor) [2]Tensor { + pub fn _cartesianProduct2(a: Tensor, b: Tensor) [2]Tensor { return cartesianProduct(2, .{ a, b }); } }; - { - const xs, const ys = try zml.testing.compileAndCall(client, Local.cartesianProduct2, .{ x, y }); - try std.testing.expectEqualSlices(i64, &.{ 6, 4 }, xs.shape().dims()); - try std.testing.expectEqualSlices(i64, &.{ 6, 4 }, ys.shape().dims()); - try std.testing.expectEqualDeep( - [6][4]i32{ - .{ 0, 0, 0, 0 }, - .{ 1, 1, 1, 1 }, - .{ 2, 2, 2, 2 }, - .{ 3, 3, 3, 3 }, - .{ 4, 4, 4, 4 }, - .{ 5, 5, 5, 5 }, - }, - try xs.getValue([6][4]i32), - ); - try std.testing.expectEqualDeep( - [6][4]i32{ - .{ 0, 1, 2, 3 }, - .{ 0, 1, 2, 3 }, - .{ 0, 1, 2, 3 }, - .{ 0, 1, 2, 3 }, - .{ 0, 1, 2, 3 }, - .{ 0, 1, 2, 3 }, - }, - try ys.getValue([6][4]i32), - ); - } + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._cartesianProduct2, .{ x, y }, platform, .{}); + defer exe.deinit(); + + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3, 4, 5 })); + defer x_buffer.deinit(); + var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3 })); + defer y_buffer.deinit(); + + var xs: zml.Buffer = undefined; + var ys: zml.Buffer = undefined; + var runner = try exe.runner(std.testing.allocator); + defer runner.deinit(std.testing.allocator); + runner.run(std.testing.io, .{ x_buffer, y_buffer }, .{ &xs, &ys }, .{ .wait = true }); + defer xs.deinit(); + defer ys.deinit(); + + try std.testing.expectEqualSlices(i64, &.{ 6, 4 }, xs.shape().dims()); + try std.testing.expectEqualSlices(i64, &.{ 6, 4 }, ys.shape().dims()); + try std.testing.expectEqualDeep( + [6][4]i32{ + .{ 0, 0, 0, 0 }, + .{ 1, 1, 1, 1 }, + .{ 2, 2, 2, 2 }, + .{ 3, 3, 3, 3 }, + .{ 4, 4, 4, 4 }, + .{ 5, 5, 5, 5 }, + }, + try xs.getValue([6][4]i32, std.testing.io), + ); + try std.testing.expectEqualDeep( + [6][4]i32{ + .{ 0, 1, 2, 3 }, + .{ 0, 1, 2, 3 }, + .{ 0, 1, 2, 3 }, + .{ 0, 1, 2, 3 }, + .{ 0, 1, 2, 3 }, + .{ 0, 1, 2, 3 }, + }, + try ys.getValue([6][4]i32, std.testing.io), + ); + } + + pub fn rollRight1d(self: Tensor, axis_: anytype, shift: i64) Tensor { + const a = self.axis(axis_); + const n = self.dim(a); + const k = @mod(shift, n); // normalize shift to [0, n) + if (k == 0) return self; + return Tensor.concatenate(&.{ + self.slice(a, .{ .start = n - k, .end = n }), + self.slice(a, .{ .start = 0, .end = n - k }), + }, a); } /// Given a set of N vectors of lengths A, B, C, D, /// returns 1 tensors of rank N+1, and shape (A, B, C, D, N). - /// For any coordinate (a, b, c, d), - /// we have: + /// For any coordinate (a, b, c, d), we have: + /// /// - res[a, b, c, d] == (A[a], B[b], C[c], D[d]) pub fn cartesianProductStacked(vectors: []const Tensor) Tensor { - var out = std.BoundedArray(Tensor, Tensor.MAX_RANK).init(vectors.len) catch unreachable; + var out: stdx.BoundedArray(Tensor, constants.MAX_RANK) = .{ .buffer = undefined, .len = vectors.len }; _cartesianProduct(vectors, out.slice()); return Tensor.stack(out.constSlice(), .last, .coord); @@ -3430,16 +4743,26 @@ pub const Tensor = struct { test cartesianProductStacked { const zml = @import("zml.zig"); const platform = zml.testing.env(); - const x = try zml.Buffer.fromSlice(platform, .{6}, &[_]i32{ 0, 1, 2, 3, 4, 5 }); - const y = try zml.Buffer.fromSlice(platform, .{4}, &[_]i32{ 0, 1, 2, 3 }); + const x: Tensor = .init(.{6}, .i32); + const y: Tensor = .init(.{4}, .i32); const Local = struct { - pub fn cartesianProduct2(a: Tensor, b: Tensor) Tensor { + pub fn _fwd(a: Tensor, b: Tensor) Tensor { return cartesianProductStacked(&.{ a, b }); } }; - const z = try zml.testing.compileAndCall(platform, Local.cartesianProduct2, .{ x, y }); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._fwd, .{ x, y }, platform, .{}); + defer exe.deinit(); + + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3, 4, 5 })); + defer x_buffer.deinit(); + var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3 })); + defer y_buffer.deinit(); + + var z = try exe.eval(std.testing.allocator, std.testing.io, .{ x_buffer, y_buffer }); + defer z.deinit(); + try std.testing.expectEqualDeep( [6][4][2]i32{ .{ .{ 0, 0 }, .{ 0, 1 }, .{ 0, 2 }, .{ 0, 3 } }, @@ -3449,17 +4772,17 @@ pub const Tensor = struct { .{ .{ 4, 0 }, .{ 4, 1 }, .{ 4, 2 }, .{ 4, 3 } }, .{ .{ 5, 0 }, .{ 5, 1 }, .{ 5, 2 }, .{ 5, 3 } }, }, - try z.getValue([6][4][2]i32), + try z.getValue([6][4][2]i32, std.testing.io), ); } fn binaryOp( op_name: []const u8, - op_fn: fn (mlir.Context, mlir.Value, mlir.Value, mlir.Location) mlir.Operation, + op_fn: fn (*mlir.Context, *const mlir.Value, *const mlir.Value, *const mlir.Location) *mlir.Operation, ) fn (Tensor, Tensor) Tensor { return struct { pub fn binaryOpHelper(self: Tensor, other: Tensor) Tensor { - meta.assert(self.dtype() == other.dtype(), "{s} expects tensor to be of same type, got {} and {}", .{ op_name, self.dtype(), other.dtype() }); + stdx.debug.assert(self.dtype() == other.dtype(), "{s} expects tensor to be of same type, got {f} and {f}", .{ op_name, self, other }); if (self.rank() == 0 and other.rank() != 0) { return binaryOpHelper(self.broad(other._shape), other); @@ -3469,35 +4792,77 @@ pub const Tensor = struct { return binaryOpHelper(self, other.broad(self._shape)); } - meta.assert(self._shape.eql(other._shape), "{s} expects tensor shapes to match, got {} and {}", .{ op_name, self._shape, other._shape }); + var other_ = other; + var same_shape = self._shape.eql(other._shape); + if (!same_shape and std.mem.eql(Shape.Tag, self._shape.tags(), other._shape.tags()) and other._shape.canBroadcastTo(self._shape)) { + // Only a restrictive version of broadcasting is allowed here, where all the tags matches already. + // Typical use case: `x.div(x.sum(.a))` + same_shape = true; + other_ = other.broad(self._shape); + } - const mlirCtx = self.getContext().mlirCtx(); - const location = mlirCtx.location(@src()); - const ret = @call(.auto, op_fn, .{ mlirCtx, self.value(), other.value(), location }); + stdx.debug.assert(same_shape, "{s} expects tensor shapes to match, got {f} and {f}", .{ op_name, self._shape, other._shape }); + + const ret = @call(.auto, op_fn, .{ mlirCtx(), self.value(), other_.value(), mlir.Location.unknown(mlirCtx()) }).appendTo(currentBlock()); return _result(self._shape, ret.result(0)); } }.binaryOpHelper; } + + /// Insert code that will print the content of the given buffer at runtime. + /// Use the name parameter to differentiate different print calls in the output. + /// Only for debug purpose, it inserts device to host synchronization + /// so it will slow down the program execution. + pub fn print(input: Tensor, name: []const u8) void { + const ctx = Compiler.current(); + const full_name = std.fmt.allocPrint(ctx.arena.allocator(), "{s}: {f}", .{ name, input.shape() }) catch @panic("OOM"); + defer ctx.arena.allocator().free(full_name); + switch (ctx.platform.target) { + .cpu, .cuda, .rocm, .tpu, .metal => { + ops.manualComputation((struct { + input: Tensor, + name: []const u8, + + fn body(body_ctx: @This(), _: void) void { + ops.customCall("zml$print", body_ctx.input, {}, .{ .name = body_ctx.name }, .{ .has_side_effect = true }); + } + }).body, .{ .input = input, .name = full_name }, {}, ctx.platform.replicated_mesh, .{ .in_specs = .{ .input = .replicated } }); + }, + .oneapi, .neuron, .furiosa => {}, + } + } + + fn mlirCtx() *mlir.Context { + return Compiler.current().mlir_ctx; + } + + fn currentLoc() *const mlir.Location { + return Compiler.current().location; + } + + fn currentBlock() *mlir.Block { + return Compiler.current().currentScope().block; + } }; -fn initPoolArg(rank: usize, data: []const i64) [Tensor.MAX_RANK]i64 { +fn initPoolArg(rank: usize, data: []const i64) [constants.MAX_RANK]i64 { // TODO use shape - var result = [_]i64{1} ** Tensor.MAX_RANK; + var result: [constants.MAX_RANK]i64 = @splat(1); const start = rank - data.len; @memcpy(result[start .. start + data.len], data); return result; } -fn getPoolResDims(dt: DataType, in_dims: []const i64, base_dilations: @Vector(Tensor.MAX_RANK, i64), padding: []const i64, window_dimensions: @Vector(Tensor.MAX_RANK, i64), window_dilations: @Vector(Tensor.MAX_RANK, i64), window_strides: @Vector(Tensor.MAX_RANK, i64)) Shape { +fn getPoolResDims(dt: DataType, in_dims: []const i64, base_dilations: @Vector(constants.MAX_RANK, i64), padding: []const i64, window_dimensions: @Vector(constants.MAX_RANK, i64), window_dilations: @Vector(constants.MAX_RANK, i64), window_strides: @Vector(constants.MAX_RANK, i64)) Shape { // TODO use shape - var input_dims = [_]i64{1} ** Tensor.MAX_RANK; + var input_dims: [constants.MAX_RANK]i64 = @splat(1); @memcpy(input_dims[0..in_dims.len], in_dims); - const input_dims_: @Vector(Tensor.MAX_RANK, i64) = input_dims; - const splat_one: @Vector(Tensor.MAX_RANK, i64) = @splat(1); - const dilated_input_shape: @Vector(Tensor.MAX_RANK, i64) = (input_dims_ - splat_one) * base_dilations + splat_one; - var pad_slice0: @Vector(Tensor.MAX_RANK, i64) = @splat(padding[0]); - var pad_slice1: @Vector(Tensor.MAX_RANK, i64) = @splat(padding[0]); + const input_dims_: @Vector(constants.MAX_RANK, i64) = input_dims; + const splat_one: @Vector(constants.MAX_RANK, i64) = @splat(1); + const dilated_input_shape: @Vector(constants.MAX_RANK, i64) = (input_dims_ - splat_one) * base_dilations + splat_one; + var pad_slice0: @Vector(constants.MAX_RANK, i64) = @splat(padding[0]); + var pad_slice1: @Vector(constants.MAX_RANK, i64) = @splat(padding[0]); if (padding.len > 1) { var idx: usize = 0; while (idx < in_dims.len * 2) : (idx += 2) { @@ -3505,20 +4870,20 @@ fn getPoolResDims(dt: DataType, in_dims: []const i64, base_dilations: @Vector(Te pad_slice1[idx / 2] = padding[idx + 1]; } } - const padded_input_shape: @Vector(Tensor.MAX_RANK, i64) = pad_slice0 + dilated_input_shape + pad_slice1; + const padded_input_shape: @Vector(constants.MAX_RANK, i64) = pad_slice0 + dilated_input_shape + pad_slice1; const dilated_window_shape = (window_dimensions - splat_one) * window_dilations + splat_one; const dims = @divFloor(padded_input_shape - dilated_window_shape, window_strides) + splat_one; - const dims_arr: [Tensor.MAX_RANK]i64 = @bitCast(dims); + const dims_arr: [constants.MAX_RANK]i64 = @bitCast(dims); return Shape.init(dims_arr[0..in_dims.len], dt); } -fn getComparisonType(ctx: mlir.Context, dtype: DataType) dialect.stablehlo.CompareType { - return dialect.stablehlo.CompareType.init(ctx, switch (dtype) { - .i4, .i8, .i16, .i32, .i64 => .SIGNED, - .bool, .u4, .u8, .u16, .u32, .u64 => .UNSIGNED, - .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz, .bf16, .f16, .f32, .f64 => .FLOAT, - .c64, .c128 => @panic("Can't compare complex numbers"), - }); +fn getComparisonType(ctx: *mlir.Context, dtype: DataType) *const stablehlo.CompareTypeAttr { + return .get(ctx, .{ .value = switch (dtype.class()) { + .bool => .UNSIGNED, + .integer => if (dtype.isSignedInt()) .SIGNED else .UNSIGNED, + .float => .FLOAT, + .complex => @panic("Can't compare complex numbers"), + } }); } test "Tensor.maxPool1d" { @@ -3526,7 +4891,7 @@ test "Tensor.maxPool1d" { const platform = zml.testing.env(); const MaxPool = struct { - pub fn forward(x: zml.Tensor) Tensor.ArgMaxRes { + pub fn _fwd(x: zml.Tensor) Tensor.ArgMaxRes { return x.maxPool1d(.{ .window_dimensions = 3, .window_strides = 2, @@ -3537,23 +4902,29 @@ test "Tensor.maxPool1d" { var data: [20]f32 = undefined; for (&data, 0..) |*v, i| v.* = @floatFromInt(i); - const x = try zml.Buffer.fromSlice(platform, .{ 2, 2, 5 }, &data); - const result = try zml.testing.compileAndCall(platform, MaxPool.forward, .{x}); - try zml.testing.expectEqualShapes(Shape.init(.{ 2, 2, 2 }, .f32), result.values.shape()); - try zml.testing.expectEqualShapes(Shape.init(.{ 2, 2, 2 }, .i32), result.indices.shape()); - const buffer = result.values.getValue([2][2][2]f32); + const x: Tensor = .init(.{ 2, 2, 5 }, .f32); + + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, MaxPool._fwd, .{x}, platform, .{}); + defer exe.deinit(); + + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&data)); + defer x_buffer.deinit(); + + var result: zml.Bufferized(Tensor.ArgMaxRes) = undefined; + var runner = try exe.runner(std.testing.allocator); + defer runner.deinit(std.testing.allocator); + runner.run(std.testing.io, .{x_buffer}, .{&result}, .{ .wait = true }); + defer result.values.deinit(); + defer result.indices.deinit(); + + try zml.testing.expectEqualShapes(.init(.{ 2, 2, 2 }, .f32), result.values.shape()); + try zml.testing.expectEqualShapes(.init(.{ 2, 2, 2 }, .i32), result.indices.shape()); try std.testing.expectEqualDeep( [2][2][2]f32{ - [2][2]f32{ - [2]f32{ 2, 4 }, - [2]f32{ 7, 9 }, - }, - [2][2]f32{ - [2]f32{ 12, 14 }, - [2]f32{ 17, 19 }, - }, + .{ .{ 2, 4 }, .{ 7, 9 } }, + .{ .{ 12, 14 }, .{ 17, 19 } }, }, - buffer, + result.values.getValue([2][2][2]f32, std.testing.io), ); } @@ -3562,23 +4933,36 @@ test "Tensor.maxPool2d" { const platform = zml.testing.env(); const MaxPool = struct { - pub fn forward(x: Tensor) Tensor.ArgMaxRes { + pub fn _fwd(x: Tensor) Tensor.ArgMaxRes { return x.maxPool2d(.{ - .window_dimensions = &.{ 3, 2 }, - .window_strides = &.{ 2, 1 }, + .window_dimensions = .{ 3, 2 }, + .window_strides = .{ 2, 1 }, }); } }; var data: [100]f32 = undefined; for (&data, 0..) |*v, i| v.* = @floatFromInt(i); - const x = try zml.Buffer.fromSlice(platform, .{ 2, 2, 5, 5 }, &data); - const result = try zml.testing.compileAndCall(platform, MaxPool.forward, .{x}); + const x: Tensor = .init(.{ 2, 2, 5, 5 }, .f32); + + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, MaxPool._fwd, .{x}, platform, .{}); + defer exe.deinit(); + + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&data)); + defer x_buffer.deinit(); + + var result: zml.Bufferized(Tensor.ArgMaxRes) = undefined; + var runner = try exe.runner(std.testing.allocator); + defer runner.deinit(std.testing.allocator); + runner.run(std.testing.io, .{x_buffer}, .{&result}, .{ .wait = true }); + defer result.values.deinit(); + defer result.indices.deinit(); + try zml.testing.expectEqualShapes(Shape.init(.{ 2, 2, 2, 4 }, .f32), result.values.shape()); try zml.testing.expectEqualShapes(Shape.init(.{ 2, 2, 2, 4 }, .i32), result.indices.shape()); var buffer: [2][2][2][4]f32 = undefined; - _ = try result.values.toHost(std.mem.asBytes(&buffer)); + _ = try result.values.toSlice(std.testing.io, .init(result.values.shape(), std.mem.asBytes(&buffer))); try std.testing.expectEqualDeep( [2][2][2][4]f32{ .{ @@ -3594,139 +4978,84 @@ test "Tensor.maxPool2d" { ); } -/// Returns a mirrored version of T where each Tensor has been replaced by a Buffer. -pub fn Bufferized(comptime T: type) type { - return meta.MapType(Tensor, Buffer).map(T); -} - -/// Return a clone of a type with Tensors replaced by Shapes. -/// Recursively descends into the type. -/// See also: shapesOf() and its tests, and meta.MapType(). -pub fn ShapeOf(comptime T: type) type { - const M = meta.MapType(Tensor, Shape); - return M.map(T); -} - -/// Return a clone of the argument where each instance of a Tensor is replaced -/// by its Shape. This is similar to ShapeOf(), but with runtime values. -/// See also: meta.mapAlloc(). -pub fn shapesOf(model: anytype, allocator: std.mem.Allocator) !ShapeOf(@TypeOf(model)) { - var shapes: ShapeOf(@TypeOf(model)) = undefined; - try meta.mapAlloc(struct { - fn shapeFromTensorCallback(_: void, tensor: Tensor) Shape { - return tensor.shape(); - } - }.shapeFromTensorCallback, allocator, {}, model, &shapes); - return shapes; -} - -test shapesOf { - const alloc = std.testing.allocator; - - // Tensor in struct - { - const S = struct { - a: Tensor, - }; - const shape = Shape.init(.{ 28, 28 }, .f32); - const s: S = .{ - .a = Tensor{ ._shape = shape, ._id = undefined }, - }; - const shapes = try shapesOf(s, alloc); - try std.testing.expectEqual(shape, shapes.a); - } - - // single Tensor - { - const shape = Shape.init(.{ 28, 28 }, .f32); - const tensor = Tensor{ ._shape = shape, ._id = undefined }; - const shapes = try shapesOf(tensor, alloc); - try std.testing.expectEqual(shape, shapes); - } - - // nn linear layer, no bias - { - const nn = @import("nn.zig"); - const shape = Shape.init(.{ 28, 28 }, .f32); - const layer: nn.Linear = .{ - .weight = Tensor{ ._shape = shape, ._id = undefined }, - .bias = null, - }; - - const shapes = try shapesOf(layer, alloc); - try std.testing.expectEqual(shape, shapes.weight); - try std.testing.expectEqual(null, shapes.bias); - } - - // model - { - const Mnist = struct { - fc1: Layer, - fc2: Layer, - - const Layer = struct { - weight: Tensor, - bias: Tensor, - }; - }; - - const fc1_weight_shape = Shape.init(.{ 500, 784 }, .f32); - const fc1_bias_shape = Shape.init(.{500}, .f32); - const fc2_weight_shape = Shape.init(.{ 10, 500 }, .f32); - const fc2_bias_shape = Shape.init(.{10}, .f32); - const mnist: Mnist = .{ - .fc1 = .{ - .weight = Tensor{ ._shape = fc1_weight_shape, ._id = undefined }, - .bias = Tensor{ ._shape = fc1_bias_shape, ._id = undefined }, - }, - .fc2 = .{ - .weight = Tensor{ ._shape = fc2_weight_shape, ._id = undefined }, - .bias = Tensor{ ._shape = fc2_bias_shape, ._id = undefined }, - }, - }; - - const shapes = try shapesOf(mnist, alloc); - try std.testing.expectEqual(fc1_weight_shape, shapes.fc1.weight); - try std.testing.expectEqual(fc1_bias_shape, shapes.fc1.bias); - try std.testing.expectEqual(fc2_weight_shape, shapes.fc2.weight); - try std.testing.expectEqual(fc2_bias_shape, shapes.fc2.bias); - } -} - -fn _collectAxes(T: type, bounded_array: std.BoundedArray(T, Tensor.MAX_RANK), value: T) std.BoundedArray(i64, Tensor.MAX_RANK) { - var res: std.BoundedArray(i64, Tensor.MAX_RANK) = .{}; - for (bounded_array.constSlice(), 0..) |v, ax| { - if (v == value) { - res.appendAssumeCapacity(@intCast(ax)); - } - } - return res; -} - -fn _parseGatherCoord(self: Tensor, axes_: anytype) struct { bool, std.BoundedArray(u3, Tensor.MAX_RANK) } { +fn _parseGatherCoord(self: Tensor, axes_: anytype) struct { bool, stdx.BoundedArray(u3, constants.MAX_RANK) } { const AxesT = @TypeOf(axes_); - const axes_is_scalar = AxesT == EnumLiteral or AxesT == comptime_int or @typeInfo(AxesT) == .Int; + const axes_is_scalar = AxesT == @EnumLiteral() or AxesT == comptime_int or @typeInfo(AxesT) == .int; const coord_axes = if (axes_is_scalar) - std.BoundedArray(u3, Tensor.MAX_RANK).fromSlice(&.{self.axis(axes_)}) catch unreachable + stdx.BoundedArray(u3, constants.MAX_RANK).fromSlice(&.{self.axis(axes_)}) catch unreachable else self.axes(axes_); return .{ axes_is_scalar, coord_axes }; } -fn parseArrayInfo(T: type) Shape { - return switch (@typeInfo(T)) { - .Array => |arr| { - const s = parseArrayInfo(arr.child); - return s.insert(0, .{arr.len}); - }, - else => .{ ._dtype = DataType.fromZigType(T) }, - }; +fn toI64(values: anytype) stdx.BoundedArray(i64, constants.MAX_RANK) { + var res: stdx.BoundedArray(i64, constants.MAX_RANK) = .empty; + for (values) |val| res.appendAssumeCapacity(@intCast(val)); + return res; } -inline fn toI64(values: anytype) []i64 { - var res: [Tensor.MAX_RANK]i64 = undefined; - for (values, 0..) |val, i| res[i] = @intCast(val); - return res[0..values.len]; +fn transposeIsJustAReshape(x: Shape, permutation: []const i64) bool { + var perm: stdx.BoundedArray(struct { u8, bool }, constants.MAX_RANK) = .empty; + // Don't rewrite on invalid inputs. + if (permutation.len > x.rank()) return false; + for (permutation) |ax| { + const squeezable = x.dim(ax) == 1; + perm.appendAssumeCapacity(.{ @intCast(ax), squeezable }); + } + + var effective_ax: u8 = 0; + for (0..perm.len) |i| { + const ax, const squeezable = perm.get(i); + if (squeezable) { + // Effectively squeeze this axis by decrementing axes coming after by 1. + for (i..perm.len) |j| { + if (perm.buffer[j][0] > ax) { + perm.buffer[j][0] -= 1; + } + } + continue; + } + + if (ax != effective_ax) return false; + effective_ax += 1; + } + + return true; +} + +test transposeIsJustAReshape { + try std.testing.expect(transposeIsJustAReshape(Shape.init(.{ 5, 1, 3 }, .i32), &.{ 0, 1, 2 })); + try std.testing.expect(transposeIsJustAReshape(Shape.init(.{ 5, 1, 3 }, .i32), &.{ 1, 0, 2 })); + try std.testing.expect(!transposeIsJustAReshape(Shape.init(.{ 5, 1, 3 }, .i32), &.{ 2, 1, 0 })); + try std.testing.expect(transposeIsJustAReshape(Shape.init(.{ 64, 8, 1, 128 }, .bf16), &.{ 0, 2, 1, 3 })); + try std.testing.expect(!transposeIsJustAReshape(Shape.init(.{ 64, 8, 155, 128 }, .bf16), &.{ 0, 2, 1, 3 })); + try std.testing.expect(transposeIsJustAReshape(Shape.init(.{ 64, 1, 1, 128 }, .bf16), &.{ 1, 2, 0, 3 })); + try std.testing.expect(!transposeIsJustAReshape(Shape.init(.{ .b = 1, .h = 10, .q = 155, .hd = 1 }, .f32), &.{ 0, 2, 1, 3 })); + try std.testing.expect(!transposeIsJustAReshape(Shape.init(.{ 1, 10, 155, 1 }, .f32), &.{ 0, 2, 3, 1 })); + try std.testing.expect(transposeIsJustAReshape(Shape.init(.{ 1, 10, 155, 1 }, .f32), &.{ 0, 1, 3, 2 })); +} + +test "unused tensor" { + const zml = @import("zml.zig"); + const platform = zml.testing.env(); + + const Local = struct { + pub fn _fwd(x: Tensor) Tensor { + const y = x.addConstant(1); + _ = y; + return x; + } + }; + + var exe = try zml.module.compile( + std.testing.allocator, + std.testing.io, + Local._fwd, + .{Tensor.init(.{10}, .f32)}, + platform, + .{}, + ); + defer exe.deinit(); } diff --git a/zml/test_runner.zig b/zml/test_runner.zig deleted file mode 100644 index 7c0fe690..00000000 --- a/zml/test_runner.zig +++ /dev/null @@ -1,115 +0,0 @@ -//! Test runner for unit test based on https://github.com/ziglang/zig/blob/master/lib/compiler/test_runner.zig with async -const builtin = @import("builtin"); - -const std = @import("std"); -const asynk = @import("async"); -const io = std.io; -const testing = std.testing; -const assert = std.debug.assert; - -// note: std_options.log_level does not respect testing.log_level -// ref: https://github.com/ziglang/zig/issues/5738 -const log_level: std.log.Level = .warn; - -pub const std_options = .{ - .log_level = log_level, -}; - -var log_err_count: usize = 0; -var fba_buffer: [8192]u8 = undefined; -var fba = std.heap.FixedBufferAllocator.init(&fba_buffer); - -pub fn main() anyerror!void { - testing.log_level = log_level; - try asynk.AsyncThread.main(testing.allocator, asyncMain, .{}); -} - -pub fn asyncMain() void { - const test_fn_list: []const std.builtin.TestFn = builtin.test_functions; - var ok_count: usize = 0; - var skip_count: usize = 0; - var fail_count: usize = 0; - const root_node = std.Progress.start(.{ - .root_name = "Test", - .estimated_total_items = test_fn_list.len, - }); - const have_tty = std.io.getStdErr().isTty(); - - var args = std.process.args(); - // Skip executable path - _ = args.next().?; - - const identifier_query = if (args.next()) |arg| blk: { - std.debug.print("Only tests with identifiers that includes `{s}` will be run\n", .{arg}); - break :blk arg; - } else blk: { - break :blk ""; - }; - - var leaks: usize = 0; - - for (test_fn_list, 0..) |test_fn, i| { - if (std.mem.indexOf(u8, test_fn.name, identifier_query) == null) { - continue; - } - - testing.allocator_instance = .{}; - defer { - if (testing.allocator_instance.deinit() == .leak) { - leaks += 1; - } - } - - const test_node = root_node.start(test_fn.name, 0); - if (!have_tty) { - std.debug.print("{d}/{d} {s}...", .{ i + 1, test_fn_list.len, test_fn.name }); - } - if (test_fn.func()) |_| { - ok_count += 1; - test_node.end(); - if (!have_tty) std.debug.print("OK\n", .{}); - } else |err| switch (err) { - error.SkipZigTest => { - skip_count += 1; - if (have_tty) { - std.debug.print("{d}/{d} {s}...SKIP\n", .{ i + 1, test_fn_list.len, test_fn.name }); - } else { - std.debug.print("SKIP\n", .{}); - } - test_node.end(); - }, - else => { - fail_count += 1; - if (have_tty) { - std.debug.print("{d}/{d} {s}...FAIL ({s})\n", .{ - i + 1, test_fn_list.len, test_fn.name, @errorName(err), - }); - } else { - std.debug.print("FAIL ({s})\n", .{@errorName(err)}); - } - if (@errorReturnTrace()) |trace| { - std.debug.dumpStackTrace(trace.*); - } - test_node.end(); - }, - } - } - root_node.end(); - if (ok_count == test_fn_list.len) { - std.debug.print("All {d} tests passed.\n", .{ok_count}); - } else { - std.debug.print("{d} passed; {d} skipped; {d} failed.\n", .{ ok_count, skip_count, fail_count }); - } - if (log_err_count != 0) { - std.debug.print("{d} errors were logged.\n", .{log_err_count}); - } - if (leaks != 0) { - std.debug.print("{d} tests leaked memory.\n", .{leaks}); - } - if (leaks != 0 or log_err_count != 0 or fail_count != 0) { - std.process.exit(1); - } - - // Explicit exit 0 to exit from async main thread and main properly - std.process.exit(0); -} diff --git a/zml/testing.zig b/zml/testing.zig index df3c04e2..5e3d6333 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -1,36 +1,44 @@ const std = @import("std"); const builtin = @import("builtin"); +const stdx = @import("stdx"); + +const Platform = @import("platform.zig").Platform; +const Sharding = @import("Sharding.zig"); +const Slice = @import("slice.zig").Slice; const zml = @import("zml.zig"); -const meta = @import("meta.zig"); -const shapesOf = @import("tensor.zig").shapesOf; -const log = std.log.scoped(.zml_testing); +const log = std.log.scoped(.@"zml/testing"); -var _ctx: ?zml.Context = null; +var _platform: ?*const Platform = null; -pub fn env() zml.Platform { +pub fn env() *const Platform { if (!builtin.is_test) @compileError("Cannot use zml.testing.env outside of a test block"); - if (_ctx == null) { - _test_compile_opts = if (initCacheDir()) + if (_platform == null) { + const memory_fraction = 0.90; + const platform = Platform.auto( + std.heap.c_allocator, + std.testing.io, .{ - .cache_location = "/tmp/zml/tests/cache", - .xla_dump_to = "/tmp/zml/tests/", - } - else - .{}; + .cuda = .{ .allocator = .{ .vmm = .{ .memory_fraction = memory_fraction } } }, + .rocm = .{ .allocator = .{ .vmm = .{ .memory_fraction = memory_fraction } } }, + .metal = .{ .allocator = .{ .bfc = .{ .preallocate = true, .memory_fraction = memory_fraction } } }, + .oneapi = .{ .allocator = .{ .bfc = .{ .preallocate = true, .memory_fraction = memory_fraction } } }, + }, + ) catch @panic("Pjrt not available"); - _ctx = zml.Context.init() catch unreachable; + // Register meshes required by flash attention. + _ = platform.registerMesh("model", .mesh(.{ .model = .high_bandwidth })) catch unreachable; + _platform = platform; } - return _ctx.?.platforms.get(.cpu).?.withCompilationOptions(_test_compile_opts); + + return _platform.?; } -var _test_compile_opts: zml.CompilationOptions = .{}; +pub fn tpMesh() *const Sharding.Mesh { + if (!builtin.is_test) @compileError("Cannot use zml.testing.tpMesh() outside of a test block"); -fn initCacheDir() bool { - const tmp = std.fs.openDirAbsolute("/tmp", .{}) catch return false; - tmp.makePath("zml/tests/cache") catch return false; - return true; + return _platform.?.meshes.get("model").?; } /// In neural network we generally care about the relative precision, @@ -42,200 +50,462 @@ pub fn approxEq(comptime Float: type, l: Float, r: Float, tolerance: Float) bool return closeRel or closeAbs; } -/// Testing utility. Accepts both Tensor and HostBuffer but Tensor will be copied to the +fn toSlice(allocator: std.mem.Allocator, io: std.Io, slice_or_buffer: anytype) !struct { Slice, bool } { + return if (@TypeOf(slice_or_buffer) == zml.Buffer) b: { + const slice = try slice_or_buffer.toSliceAlloc(allocator, io); + break :b .{ slice, true }; + } else .{ slice_or_buffer, false }; +} + +/// Testing utility. Accepts both zml.Buffer and zml.HostBuffer but zml.Buffer will be copied to the /// host for comparison ! -pub fn expectClose(left_: anytype, right_: anytype, tolerance: f32) !void { - const allocator = if (builtin.is_test) std.testing.allocator else std.heap.page_allocator; - var left: zml.HostBuffer, const should_free_left = if (@TypeOf(left_) == zml.Buffer) - .{ try left_.toHostAlloc(allocator), true } - else - .{ left_, false }; +pub fn expectClose(io: std.Io, left_: anytype, right_: anytype, opts: CompareOpts) !void { + const allocator = if (builtin.is_test) std.testing.allocator else std.heap.smp_allocator; + var left: Slice, const should_free_left = try toSlice(allocator, io, left_); + defer if (should_free_left) left.free(allocator); - var right: zml.HostBuffer, const should_free_right = if (@TypeOf(right_) == zml.Buffer) - .{ try right_.toHostAlloc(allocator), true } - else - .{ right_, false }; + var right: Slice, const should_free_right = try toSlice(allocator, io, right_); + defer if (should_free_right) right.free(allocator); - defer { - if (should_free_left) left.deinit(allocator); - if (should_free_right) right.deinit(allocator); - } + const stderr = std.debug.lockStderr(&.{}); + defer std.debug.unlockStderr(); + const w = &stderr.file_writer.interface; - if (!std.mem.eql(i64, left.shape().dims(), right.shape().dims())) { - log.err("left.shape() {} != right.shape() {}", .{ left.shape(), right.shape() }); + //errdefer log.err("\n--> Left: {0d:24.3}\n--> Right: {1d:24.3}", .{ left, right }); + if (!std.mem.eql(i64, left.shape.dims(), right.shape.dims())) { + try w.print("left.shape() {f} != right.shape() {f}\n", .{ left.shape, right.shape }); return error.TestUnexpectedResult; } - if (left.dtype() != right.dtype() and !(left.dtype() == .f16 and right.dtype() == .bf16)) { - log.err("left.dtype ({}) != right.dtype ({})", .{ left.dtype(), right.dtype() }); + + if (left.dtype() != right.dtype()) { + try w.print("left.dtype ({}) != right.dtype ({})\n", .{ left.dtype(), right.dtype() }); return error.TestUnexpectedResult; } + switch (left.dtype()) { - inline .f16, .f32, .f64 => |t| { + inline .bf16, + .f16, + .f32, + .f64, + .f4e2m1, + .f8e3m4, + .f8e4m3, + .f8e4m3b11fnuz, + .f8e4m3fn, + .f8e4m3fnuz, + .f8e5m2, + .f8e5m2fnuz, + .f8e8m0, + => |t| { const L = t.toZigType(); - const left_data = left.items(L); switch (right.dtype()) { - inline .f16, .bf16, .f32, .f64, .f8e4m3fn => |rt| { + inline .bf16, + .f16, + .f32, + .f64, + .f8e4m3b11fnuz, + .f8e4m3fn, + .f8e4m3fnuz, + .f8e5m2, + .f8e5m2fnuz, + => |rt| { const R = rt.toZigType(); - const right_data = right.items(R); - for (left_data, right_data, 0..) |l, r, i| { - if (!approxEq(L, l, zml.floats.floatCast(L, r), @floatCast(tolerance))) { - log.err("left.data != right_data.\n < {d:.3} \n > {d:.3}\n error at idx {d}: {d:.3} != {d:.3}", .{ center(left_data, i), center(right_data, i), i, left_data[i], right_data[i] }); - return error.TestUnexpectedResult; - } + const report: CompareOpts.Report = try .compareSlices(true, allocator, L, R, left, right, opts); + if (!report.ok(opts)) { + try w.print("{f}\n", .{report}); + return error.TestUnexpectedResult; } + return; }, else => unreachable, } }, - inline .u8, .u16, .u32, .i16, .i32, .i64 => |t| { + inline .bool, .u2, .u4, .u8, .u16, .u32, .u64, .i2, .i4, .i8, .i16, .i32, .i64 => |t| { const T = t.toZigType(); - const left_data = left.items(T); - const right_data = right.items(T); - if (!std.mem.eql(T, left_data, right_data)) { - log.err("left.data ({d}) != right.data ({d})", .{ left_data[0..10], right_data[0..10] }); + const report: CompareOpts.Report = try .compareSlices(false, allocator, T, T, left, right, opts); + if (!report.ok(opts)) { + try w.print("{f}\n", .{report}); return error.TestUnexpectedResult; } + return; }, - else => unreachable, + .c64, .c128 => @panic("TODO: support comparison of complex"), } } +pub const CompareOpts = struct { + absolute_tolerance: f32 = 1e-3, + relative_tolerance: f32 = 1e-2, + epsilon_relative: f32 = 1e-6, + minimum_close_fraction: f32 = 0.999, + + pub const exact_match: CompareOpts = .{ + .absolute_tolerance = 0, + .relative_tolerance = 0, + .epsilon_relative = 0, + .minimum_close_fraction = 1.00, + }; + + pub const Report = struct { + nan_or_inf: bool, + max_absolute_error: f32, + mean_absolute_error: f32, + rmse: f32, + + close_fraction: f32, + + p50_absolute_error: f32, + p90_absolute_error: f32, + p99_absolute_error: f32, + p999_absolute_error: f32, + + pub fn ok(report: Report, opts: CompareOpts) bool { + return !report.nan_or_inf and report.close_fraction >= opts.minimum_close_fraction; + } + + fn compareSlices(comptime float_cmp: bool, allocator: std.mem.Allocator, comptime L: type, comptime R: type, left: Slice, right: Slice, opts: CompareOpts) !CompareOpts.Report { + var left_iter = (&left).contiguousItemsIterator(L); + var right_iter = (&right).contiguousItemsIterator(R); + var left_chunk: []const L = &.{}; + var right_chunk: []const R = &.{}; + var acc: Accumulator = if (comptime float_cmp) try .init(allocator, left.shape.count()) else .initNoQuantiles(left.shape.count()); + defer acc.deinit(allocator); + + while (true) { + if (left_chunk.len == 0) { + left_chunk = left_iter.next() orelse &.{}; + } + if (right_chunk.len == 0) { + right_chunk = right_iter.next() orelse &.{}; + } + + if (left_chunk.len == 0 or right_chunk.len == 0) { + std.debug.assert(left_chunk.len == right_chunk.len); + return acc.report(); + } + + const n = @min(left_chunk.len, right_chunk.len); + if (comptime float_cmp) { + acc.cmpFloatSlices(L, R, left_chunk[0..n], right_chunk[0..n], opts); + } else { + acc.cmpIntSlices(L, left_chunk[0..n], right_chunk[0..n]); + } + + left_chunk = left_chunk[n..]; + right_chunk = right_chunk[n..]; + } + } + + pub fn format( + self: Report, + writer: *std.Io.Writer, + ) std.Io.Writer.Error!void { + try writer.print( + \\ nan_or_inf: {} + \\ max_absolute_error: {d} + \\ mean_absolute_error: {d} + \\ rmse: {d} + \\ close_fraction: {d} + \\ p50_absolute_error: {d} + \\ p90_absolute_error: {d} + \\ p99_absolute_error: {d} + \\ p999_absolute_error: {d} + , .{ + self.nan_or_inf, + self.max_absolute_error, + self.mean_absolute_error, + self.rmse, + self.close_fraction, + self.p50_absolute_error, + self.p90_absolute_error, + self.p99_absolute_error, + self.p999_absolute_error, + }); + } + }; + + const Accumulator = struct { + absolute_errors: []f32, + relative_errors: []f32, + expected_total_count: usize, + processed_count: usize = 0, + nan_or_inf: bool = false, + max_absolute_error: f32 = 0, + sum_absolute_error: f64 = 0, + sum_squared_error: f64 = 0, + count_close: usize = 0, + + fn init(allocator: std.mem.Allocator, count: usize) !Accumulator { + const absolute_errors = try allocator.alloc(f32, count); + errdefer allocator.free(absolute_errors); + const relative_errors = try allocator.alloc(f32, count); + + @memset(absolute_errors, 0); + @memset(relative_errors, 0); + return .{ + .expected_total_count = count, + .absolute_errors = absolute_errors, + .relative_errors = relative_errors, + }; + } + + fn initNoQuantiles(count: usize) Accumulator { + return .{ + .expected_total_count = count, + .absolute_errors = &.{}, + .relative_errors = &.{}, + }; + } + + fn deinit(self: *Accumulator, allocator: std.mem.Allocator) void { + allocator.free(self.absolute_errors); + allocator.free(self.relative_errors); + } + + fn cmpFloatSlices(self: *Accumulator, comptime L: type, comptime R: type, left: []const L, right: []const R, opts: CompareOpts) void { + std.debug.assert(left.len == right.len); + std.debug.assert(self.processed_count + left.len <= self.absolute_errors.len); + + const start = self.processed_count; + for (left, right, 0..) |l, r, i| { + const l_f32 = zml.floats.floatCast(f32, l); + const r_f32 = zml.floats.floatCast(f32, r); + if (!std.math.isFinite(l_f32) or !std.math.isFinite(r_f32)) { + self.nan_or_inf = true; + continue; + } + const absolute_error: f32 = @abs(l_f32 - r_f32); + self.max_absolute_error = @max(self.max_absolute_error, absolute_error); + self.sum_absolute_error += absolute_error; + self.sum_squared_error += absolute_error * absolute_error; + + const scale = @max(@abs(l_f32), @abs(r_f32)); + const tolerance = opts.absolute_tolerance + opts.relative_tolerance * scale; + if (absolute_error <= tolerance) { + self.count_close += 1; + } + + self.absolute_errors[start + i] = absolute_error; + const denom = @max(opts.epsilon_relative, scale); + self.relative_errors[start + i] = absolute_error / denom; + } + self.processed_count += left.len; + } + + fn cmpIntSlices(self: *Accumulator, comptime T: type, left: []const T, right: []const T) void { + std.debug.assert(left.len == right.len); + + for (left, right) |l, r| { + switch (@typeInfo(T)) { + .int, .float => self.count_close += if (l == r) 1 else 0, + .vector => |vec| self.count_close += @popCount(@as(@Int(.unsigned, vec.len), @bitCast(l == r))), + else => @panic("cmpIntSlices expect int or vector of int"), + } + } + self.processed_count += left.len; + } + + fn report(self: *Accumulator) CompareOpts.Report { + std.debug.assert(self.processed_count == self.expected_total_count); + if (self.processed_count == 0) { + return .{ + .nan_or_inf = false, + .max_absolute_error = 0, + .mean_absolute_error = std.math.nan(f32), + .rmse = std.math.nan(f32), + .close_fraction = std.math.nan(f32), + .p50_absolute_error = 0, + .p90_absolute_error = 0, + .p99_absolute_error = 0, + .p999_absolute_error = 0, + }; + } + + std.sort.heap(f32, self.absolute_errors, {}, std.sort.asc(f32)); + std.sort.heap(f32, self.relative_errors, {}, std.sort.asc(f32)); + + const q = struct { + fn q(values: []const f32, frac: f32) f32 { + if (values.len == 0) return 0; + const idx: usize = @intFromFloat(std.math.round(@as(f32, @floatFromInt(values.len - 1)) * frac)); + return values[idx]; + } + }.q; + + const mean_absolute_error = stdx.math.divFloat(f64, self.sum_absolute_error, self.processed_count); + const rmse = std.math.sqrt(stdx.math.divFloat(f64, self.sum_squared_error, self.processed_count)); + const close_fraction = stdx.math.divFloat(f64, self.count_close, self.processed_count); + + return .{ + .nan_or_inf = self.nan_or_inf, + .max_absolute_error = self.max_absolute_error, + .mean_absolute_error = @floatCast(mean_absolute_error), + .rmse = @floatCast(rmse), + .close_fraction = @floatCast(close_fraction), + .p50_absolute_error = q(self.absolute_errors, 0.5), + .p90_absolute_error = q(self.absolute_errors, 0.9), + .p99_absolute_error = q(self.absolute_errors, 0.99), + .p999_absolute_error = q(self.absolute_errors, 0.999), + }; + } + }; +}; + +pub fn compareFloats(allocator: std.mem.Allocator, comptime L: type, comptime R: type, left: []const L, right: []const R, opts: CompareOpts) !CompareOpts.Report { + var acc: CompareOpts.Accumulator = try .init(allocator, left.len); + defer acc.deinit(allocator); + acc.cmpFloatSlices(true, L, R, left, right, opts); + return acc.report(); +} + pub fn expectEqualShapes(expected: zml.Shape, actual: zml.Shape) error{TestExpectedEqual}!void { if (expected.eqlWithTags(actual)) return; - std.debug.print("Expected {}, got {}", .{ expected, actual }); + std.debug.print("Expected {f}, got {f}", .{ expected, actual }); return error.TestExpectedEqual; } -/// Compile a function and immediatly call it with the given buffers. -/// The compiled module is discarded after the call. -/// Useful during testing when a module is typically called only once. -pub fn compileAndCall(platform: zml.Platform, func: anytype, buffer_args: zml.Bufferized(meta.FnParams(func))) !zml.Bufferized(zml.meta.FnResult(func)) { - // This simplify test API and also ensure this fn isn't used outside of tests. - const allocator = std.testing.allocator; - var arena = std.heap.ArenaAllocator.init(allocator); - defer arena.deinit(); +fn BufferizedWithArgs(comptime T: type) type { + return zml.meta.MapType(zml.Tensor, zml.Buffer).map(T); +} - const Local = struct { - pub fn bufferToShape(_: void, x: zml.Buffer) zml.Shape { - return x.shape(); +fn countWithPrefix(store: zml.io.TensorStore.View, prefix: []const u8) usize { + var count: usize = 0; + var it = store.store.registry.iterator(); + while (it.next()) |entry| { + if (std.mem.startsWith(u8, entry.key_ptr.*, prefix)) { + count += 1; } - }; - var shape_args: zml.ShapeOf(meta.FnParams(func)) = undefined; - try meta.mapAlloc(Local.bufferToShape, allocator, {}, buffer_args, &shape_args); + } - const mod = try zml.compileFn(allocator, func, shape_args, platform); - defer mod.deinit(); - - return mod.call(buffer_args); + return count; } -/// Compile a function and immediatly call it with the given buffers. -/// The compiled module is discarded after the call. -/// Useful during testing when a module is typically called only once. -pub fn compileAndCallWithTensors(platform: zml.Platform, func: anytype, shape_args: zml.ShapeOf(meta.FnParams(func)), buffer_args: zml.Bufferized(meta.FnParams(func))) !zml.Bufferized(zml.meta.FnResult(func)) { - // This simplify test API and also ensure this fn isn't used outside of tests. - const allocator = std.testing.allocator; - var arena = std.heap.ArenaAllocator.init(allocator); - defer arena.deinit(); - - const mod = try zml.compileFn(allocator, func, shape_args, platform); - defer mod.deinit(); - - return mod.call(buffer_args); -} - -pub fn testLayer(platform: zml.Platform, buffer_store: zml.aio.BufferStore, comptime name: []const u8, layer: anytype, layer_weights: zml.Bufferized(@TypeOf(layer)), tolerance: f32) !void { - try testLayerOut(platform, buffer_store, name, name ++ ".out", layer, layer_weights, tolerance); -} - -pub fn testLayerOut( - platform: zml.Platform, - activations: zml.aio.BufferStore, - comptime name: []const u8, - comptime out_name: []const u8, +/// Test a layer implementation against reference activations stored in a TensorStore. +/// The layer is expected to have a method with the name given by `func`. +/// The reference activations are expected to be stored under the keys: +/// - `{name}.in.0`, `{name}.in.1`, ... for inputs +/// - `{name}.out.0`, `{name}.out.1`, ... for outputs +/// +/// The layer weights are expected to be provided in `layer_weights`. +/// The comparison tolerance can be adjusted with `tolerance`. +/// +/// The function arguments need to be valid when initialized to undefined, meaning no slices, +/// no unions, no pointers. +pub fn testLayer( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, layer: anytype, + comptime func: std.meta.DeclEnum(@TypeOf(layer)), + activation_store: zml.io.TensorStore.View, + name: []const u8, layer_weights: zml.Bufferized(@TypeOf(layer)), - tolerance: f32, + meshes: []const *const zml.Mesh, + opts: CompareOpts, ) !void { - var arena = std.heap.ArenaAllocator.init(std.heap.c_allocator); + var arena = std.heap.ArenaAllocator.init(allocator); defer arena.deinit(); - const alloc = arena.allocator(); - log.info("Testing {s}", .{name}); - const fwd = @TypeOf(layer).forward; - const FwdSign = zml.module.ModuleSignature(fwd); + const forward = @field(@TypeOf(layer), @tagName(func)); + const ArgsT = stdx.meta.Tail(std.meta.ArgsTuple(@TypeOf(forward))); - const input_tensors = try zml.aio.populateModelWithPrefix(FwdSign.ArgsT, alloc, activations, name ++ ".in"); - const input_shapes = try shapesOf(input_tensors, alloc); + var args: ArgsT = undefined; - const n_in = zml.module.countTensors(&input_tensors); - const n_in_exp = activations.countLayers(name ++ ".in"); - if (n_in != n_in_exp) { - log.warn("Reference models uses {d} inputs, but implementation uses {d}", .{ n_in_exp, n_in }); - } - - const exe = try zml.compileModel(alloc, layer, .forward, input_shapes, platform); - - const n_out_exp = activations.countLayers(out_name); - if (exe.inner.result_buffer_count != n_out_exp) { - log.warn("Reference models produces {d} outputs, but implementation produces {d}", .{ n_out_exp, exe.inner.result_buffer_count }); - } - const mod = try exe.prepare(alloc, layer_weights); - - const FetchCtx = struct { - store: zml.aio.BufferStore, - index: u32, - prefix: std.ArrayListUnmanaged(u8), - platform: zml.Platform, - - fn fetch(ctx: *@This(), x: zml.Tensor) zml.Buffer { - _ = x; - defer ctx.index += 1; - var full_prefix = ctx.*.prefix; - _ = full_prefix.writer(undefined).print("{d}", .{ctx.index}) catch unreachable; - log.info("prefix: {s}", .{full_prefix.items}); - const host = ctx.store.get(full_prefix.items) orelse { - log.err("Didn't find test input: {s}", .{full_prefix.items}); - @panic("Missing test input"); - }; - return host.toDevice(ctx.platform) catch unreachable; - } + const LocalContext = struct { + activation_store: zml.io.TensorStore.View, + index: usize = 0, }; - // Note: zml.populateModelWithPrefix isn't enough, - // because it assumes we have the same structure in the activation file - // than in the function signature. - // But for sake of decoupling the reference implementation - // and ZML code that's not always the case. - { - var input_buffers: zml.Bufferized(FwdSign.ArgsT) = undefined; - var fetch_ctx: FetchCtx = .{ .store = activations, .index = 0, .prefix = .{}, .platform = platform }; - try fetch_ctx.prefix.ensureTotalCapacity(alloc, name.len + 32); - fetch_ctx.prefix.appendSliceAssumeCapacity(name ++ ".in."); - try zml.meta.mapAlloc(FetchCtx.fetch, alloc, &fetch_ctx, input_tensors, &input_buffers); - defer zml.aio.unloadBuffers(input_buffers); - _ = mod.call(input_buffers); + const input_name = try std.fmt.allocPrint(arena.allocator(), "{s}.in", .{name}); + const input_count = zml.meta.count(zml.Tensor, &args); + const expected_input_count = countWithPrefix(activation_store, input_name); + if (input_count != expected_input_count) { + log.warn("Reference models uses {d} inputs, but implementation uses {d}", .{ expected_input_count, input_count }); } - var buf: [1024]u8 = undefined; - for (mod.output_buffers, 0..) |out, i| { - const full_name = std.fmt.bufPrint(&buf, "{s}.{d}", .{ out_name, i }) catch unreachable; - const expected_out = activations.get(full_name) orelse { - log.warn("Output buffer not found: {s}", .{full_name}); - continue; + const store_input = activation_store.withPrefix(input_name); + var ctx = LocalContext{ .activation_store = store_input }; + try zml.meta.visit(struct { + fn cb(ctx_: *LocalContext, tensor: *zml.Tensor) !void { + var buffer: [256]u8 = undefined; + const subkey = std.fmt.bufPrint(&buffer, "{d}", .{ctx_.index}) catch unreachable; + + tensor.* = ctx_.activation_store.createReplicatedTensor(subkey, null); + ctx_.index += 1; + } + }.cb, &ctx, &args); + + const exe = try platform.compile(allocator, io, layer, func, args, .{ .meshes = if (meshes.len != 0) meshes else &.{} }); + defer exe.deinit(); + + const output_name = try std.fmt.allocPrint(arena.allocator(), "{s}.out", .{name}); + const output_count = exe.output_shapes.len; + const store_output = activation_store.withPrefix(output_name); + const expected_output_count = countWithPrefix(store_output, output_name); + if (output_count != expected_output_count) { + log.warn("Reference models produces {d} outputs, but implementation produces {d}", .{ expected_output_count, output_count }); + } + + var exe_args = try exe.args(allocator); + defer exe_args.deinit(allocator); + + var exe_results = try exe.results(allocator); + defer exe_results.deinit(allocator); + + var args_buffers = try zml.io.load(ArgsT, &args, allocator, io, platform, activation_store.store, .auto); + defer zml.meta.visit(struct { + fn cb(_: void, b: *zml.Buffer) void { + b.deinit(); + } + }.cb, {}, &args_buffers); + + exe_args.set(.{ layer_weights, args_buffers }); + + var results = try allocator.alloc(zml.Buffer, output_count); + defer allocator.free(results); + + exe.call(io, exe_args, &exe_results, .{ .wait = true }); + exe_results.fill(.{results}); + defer for (results) |*result| result.deinit(); + + var failed: bool = false; + var reader_buffer: [4096]u8 = undefined; + for (0..output_count) |i| { + const expected_slice = s: { + var buffer: [256]u8 = undefined; + const subkey = try std.fmt.bufPrint(&buffer, "{d}", .{i}); + var reader = try store_output.getReader(subkey, io, &reader_buffer); + defer reader.deinit(); + const shape = store_output.getShape(subkey).?; + const expected_slice: zml.Slice = try .alloc(allocator, shape); + errdefer expected_slice.free(allocator); + try reader.interface.readSliceAll(expected_slice.data()); + break :s expected_slice; }; - zml.testing.expectClose(expected_out, zml.Buffer.fromPjrtBuffer(platform, out), tolerance) catch |err| { - log.err("{s}.{d} doesn't match !", .{ out_name, i }); - return err; + defer expected_slice.free(allocator); + + const output_slice = try results[i].toSliceAlloc(allocator, io); + defer output_slice.free(allocator); + + zml.testing.expectClose(io, expected_slice, output_slice, opts) catch |err| switch (err) { + error.TestUnexpectedResult => { + const stderr = std.debug.lockStderr(&.{}); + defer std.debug.unlockStderr(); + const w = &stderr.file_writer.interface; + try w.print("{s}.{d} doesn't match !\n", .{ output_name, i }); + failed = true; + continue; + }, + else => return err, }; } - log.info("all good for {s} !", .{name}); -} - -pub inline fn expectEqual(expected: anytype, actual: @TypeOf(expected)) !void { - return std.testing.expectEqual(expected, actual); + if (failed) { + log.info("❌ check failed for {s} ! (absolute tolerance: {e} - relative tolerance: {e} - minimum_close_fraction: {d:0>3})", .{ name, opts.absolute_tolerance, opts.relative_tolerance, opts.minimum_close_fraction }); + } else { + log.info("✅ all good for {s} ! (absolute tolerance: {e} - relative tolerance: {e} - minimum_close_fraction: {d:0>3})", .{ name, opts.absolute_tolerance, opts.relative_tolerance, opts.minimum_close_fraction }); + } } fn center(slice: anytype, i: usize) @TypeOf(slice) { @@ -244,3 +514,7 @@ fn center(slice: anytype, i: usize) @TypeOf(slice) { const end = @min(start + 2 * c, slice.len); return slice[start..end]; } + +pub inline fn expectEqual(expected: anytype, actual: @TypeOf(expected)) !void { + return std.testing.expectEqual(expected, actual); +} diff --git a/zml/tokenizer.zig b/zml/tokenizer.zig deleted file mode 100644 index 2e211eac..00000000 --- a/zml/tokenizer.zig +++ /dev/null @@ -1,1025 +0,0 @@ -//! Text tokenizer implementations -const std = @import("std"); -const builtin = @import("builtin"); -const testing = std.testing; - -const log = std.log.scoped(.zml_tokenizer); - -const helpers = @import("helpers.zig"); -const meta = @import("meta.zig"); - -test { - std.testing.refAllDecls(@This()); - std.testing.refAllDecls(Normalizer); - std.testing.refAllDecls(Tokenizer); -} - -/// Byte Pair Encoding tokenizer generally used for LLM. -pub const Tokenizer = struct { - tokens: [][]const u8, - token_lookup: std.StringHashMapUnmanaged(u32), - special_tokens: SpecialTokens, - - scores: []f32, - max_token_len: u32, - normalizer: ?Normalizer, - // Allows to split unknown unicode characters into bytes. - byte_fallback: bool = false, - - arena_state: std.heap.ArenaAllocator, - vocab_size: u32, - next_token_id: u32 = 0, - - pub const SpecialTokens = struct { - eos: u32, - bos: u32, - unk: u32, - pad: u32 = std.math.maxInt(u32), - hard_space: u32 = std.math.maxInt(u32), - }; - - pub fn init( - allocator: std.mem.Allocator, - vocab_size: u32, - max_token_len: u32, - normalizer: ?Normalizer, - special_tokens: SpecialTokens, - alloc_tokens: bool, - ) !Tokenizer { - var arena_state = std.heap.ArenaAllocator.init(allocator); - errdefer arena_state.deinit(); - const arena = arena_state.allocator(); - - var token_lookup: std.StringHashMapUnmanaged(u32) = .{}; - errdefer token_lookup.deinit(arena); - - try token_lookup.ensureTotalCapacity(arena, @intCast(vocab_size)); - - const tokens: [][]const u8 = if (alloc_tokens) try arena.alloc([]u8, vocab_size) else &.{}; - errdefer if (alloc_tokens) arena.free(tokens); - - const scores: []f32 = if (alloc_tokens) try arena.alloc(f32, vocab_size) else &.{}; - errdefer if (alloc_tokens) arena.free(scores); - - return .{ - .tokens = tokens, - .scores = scores, - .max_token_len = max_token_len, - .token_lookup = token_lookup, - .arena_state = arena_state, - .normalizer = normalizer, - .vocab_size = vocab_size, - .special_tokens = special_tokens, - }; - } - - pub fn deinit(self: Tokenizer) void { - self.arena_state.deinit(); - } - - /// Reads a new word directly into the tokenizer arena. - pub fn readTokenInto(self: *Tokenizer, score: f32, len: usize, tok_reader: anytype) !void { - const arena = self.arena_state.allocator(); - - const token = try arena.alloc(u8, len); - const n = try tok_reader.read(token); - std.debug.assert(n == len); - - return self.addOwnedToken(score, token); - } - - /// Adds a new token (and copy it) - pub fn addToken(self: *Tokenizer, score: f32, token: []const u8) !void { - const arena = self.arena_state.allocator(); - return self.addOwnedToken(score, try arena.dupe(u8, token)); - } - - /// Adds a new token (without copying it) - pub fn addOwnedToken(self: *Tokenizer, score: f32, token: []const u8) void { - const i = self.next_token_id; - std.debug.assert(i < self.vocab_size); - self.next_token_id += 1; - - self.scores[i] = score; - self.tokens[i] = token; - const v = self.token_lookup.getOrPutAssumeCapacity(token); - if (!v.found_existing) { - v.value_ptr.* = i; - } - } - - pub fn addOwnedTokenByIndex(self: *Tokenizer, i: u32, score: f32, token: []const u8) void { - std.debug.assert(i < self.vocab_size); - self.next_token_id += 1; - self.scores[i] = score; - self.tokens[i] = token; - const v = self.token_lookup.getOrPutAssumeCapacity(token); - if (!v.found_existing) { - v.value_ptr.* = @intCast(i); - } - } - - fn lookup(self: *const Tokenizer, str: []const u8) ?u32 { - return self.token_lookup.get(str); - } - - pub const EncodeOptions = struct { - /// Should the beginning of sentence '' token be added. - add_bos: bool = true, - add_eos: bool = false, - pad_to: u32 = 0, - // Print tokenization intermediary steps. - debug: bool = false, - }; - - pub fn encode(self: *const Tokenizer, allocator: std.mem.Allocator, raw: []const u8, options: EncodeOptions) ![]u32 { - if (options.debug) log.debug("Tokenizer.encode('{s}')", .{raw}); - const input = if (self.normalizer) |n| try n.normalize(allocator, raw) else raw; - defer if (self.normalizer) |_| allocator.free(input); - if (options.debug) log.debug("Tokenizer.encode.normalize -> '{s}'", .{input}); - - // Allocate a buffer that can fit all indices as well as extra character if requested. - // We then slice it so that the token merging code doesn't see the bos token. - const tok_buff_alloc = try allocator.alloc(u32, @max(options.pad_to, input.len + 2)); - const tok_buff = if (options.add_bos) tok_buff_alloc[1..] else tok_buff_alloc; - - const MergeState = union(enum) { ready: u32, nope, hard_space, idk }; - const mergeable = try allocator.alloc(MergeState, tok_buff.len); - - var num_tokens: usize = 0; - var it: CharTokenIterator = .{ .input = input }; - while (try it.nextCodepointToken(self)) |token| : (num_tokens += 1) { - tok_buff[num_tokens] = token; - mergeable[num_tokens] = if (token == self.special_tokens.hard_space) - .hard_space - else - .idk; - } - - var stable_prefix: usize = 0; - var stable_off: usize = 0; - while (true) { - // Step by step visualization of the progress. - if (options.debug) { - var _debug_buf: [256]u8 = undefined; - var debug_progress: std.ArrayList(u8) = .{ .items = _debug_buf[0..0], .capacity = _debug_buf.len, .allocator = undefined }; - self.decodeWithOpts(&debug_progress, tok_buff[0..num_tokens], .{ .sep = "|" }) catch {}; - log.debug("tokens: {d} -> {s}", .{ tok_buff[0..num_tokens], debug_progress.items }); - } - var best_score: f32 = -1e10; - var best_token: u32 = 0; - var best_idx: ?usize = null; - var input_off: usize = stable_off; - - // Find best tokens to merge in all available tokens - for (stable_prefix..num_tokens - 1) |i| { - if (tok_buff[i] == self.special_tokens.unk) { - input_off += 1; - continue; - } - const cur_tok = self.tokens[tok_buff[i]]; - defer input_off += cur_tok.len; - - // Lookup merge for current token, if not already done. - switch (mergeable[i]) { - .nope => continue, - .ready => {}, - .hard_space => { - // Since tokens are not allowed to merge through hard sep, - // we don't need to merge the sentence-wide best token. - // We can just merge the best token since beginning. - if (best_idx != null) break; - // OTOH if there was no merge possible since beginning, - // we can skip the beginning in future iterations. - stable_prefix = i + 1; - stable_off = input_off + cur_tok.len; - continue; - }, - .idk => { - - // Special tokens can't be concatenated. - if (builtin.mode == .Debug and tok_buff[i] != self.special_tokens.unk) { - // Detects memory corruption of tokens. - if (cur_tok.len == 0 or cur_tok.len > self.max_token_len) @panic("Token looks corrupted !"); - - meta.assert(std.mem.eql(u8, cur_tok, input[input_off..][0..cur_tok.len]), "current token '{s}' not found in input string '{s}' !", .{ cur_tok, input[input_off..] }); - } - const next_tok = self.tokens[tok_buff[i + 1]]; - // if `next_tok` is `.unk`, length is 1; otherwise, it's the length of the token. - const next_tok_len = if (tok_buff[i + 1] == self.special_tokens.unk) 1 else next_tok.len; - const concat_tokens = input[input_off..][0 .. cur_tok.len + next_tok_len]; - // Save the result - mergeable[i] = if (self.lookup(concat_tokens)) |tok| - .{ .ready = tok } - else - .nope; - }, - } - - switch (mergeable[i]) { - .idk, .hard_space => unreachable, - .nope => continue, - .ready => |tok| { - if (self.scores[tok] > best_score) { - best_score = self.scores[tok]; - best_token = tok; - best_idx = i; - } - }, - } - } - - if (best_idx) |bidx| { - // Apply the merge. - tok_buff[bidx] = best_token; - std.mem.copyForwards(u32, tok_buff[bidx + 1 ..], tok_buff[bidx + 2 .. num_tokens]); - std.mem.copyForwards(MergeState, mergeable[bidx + 1 ..], mergeable[bidx + 2 .. num_tokens]); - num_tokens -= 1; - // We got two new merge lookups to do. - mergeable[bidx] = .idk; - if (bidx > 0 and mergeable[bidx - 1] != .hard_space) mergeable[bidx - 1] = .idk; - } else { - // No merge candidate => we are done ! - break; - } - } - - if (options.add_eos) { - tok_buff[num_tokens] = self.special_tokens.eos; - num_tokens += 1; - } - if (options.add_bos) { - tok_buff_alloc[0] = self.special_tokens.bos; - num_tokens += 1; - } - if (num_tokens < options.pad_to) { - for (num_tokens..options.pad_to) |i| { - tok_buff_alloc[i] = self.special_tokens.pad; - } - num_tokens = options.pad_to; - } - - // Release extra memory we don't need anymore. - allocator.free(mergeable); - _ = allocator.resize(tok_buff_alloc, num_tokens); - return tok_buff_alloc[0..num_tokens]; - } - - /// Returns a slice corresponding to the given id. Handles unknown ids and special ids. - pub fn lookupPiece(self: *const Tokenizer, id: usize) []const u8 { - return if (id == self.special_tokens.bos or id == self.special_tokens.eos or id == self.special_tokens.pad) - "" - else if (id == self.special_tokens.unk) - "" - else if (id > self.tokens.len) - std.debug.panic("Unexpected token id: {d}, vocab_size: {d}", .{ id, self.vocab_size }) - else - self.tokens[id]; - } - - /// Converts the given slice of tokens back into bytes. - /// Note that if the tokenizer allows sub-unicode bytes, it's possible - /// the output is not valid utf8. - pub fn decode(self: *const Tokenizer, allocator: std.mem.Allocator, input: []const u32) error{OutOfMemory}![]u8 { - var output = std.ArrayList(u8).init(allocator); - errdefer output.deinit(); - - try self.decodeWithOpts(&output, input, .{}); - return output.toOwnedSlice(); - } - - pub fn decodeWithOpts( - self: *const Tokenizer, - output: *std.ArrayList(u8), - input: []const u32, - opts: struct { sep: []const u8 = "" }, - ) error{OutOfMemory}!void { - const escaped = if (self.normalizer) |n| n.escapedSpace() else null; - // Flag used to indicate if the first dummy whitespace has been consumed. - for (input) |id| { - // Retrieve the slice corresponding to the id. - var piece = self.lookupPiece(id); - - // Convert `▁` to a regular space. - if (escaped) |escspc| { - // we modify piece inside the loop, so we can use it in the condition - while (std.mem.startsWith(u8, piece, escaped.?)) { - piece = piece[escspc.len..]; - // don't output a space at beginning of text. - if (output.items.len > 0) try output.append(' '); - } - } - - try output.appendSlice(piece); - if (opts.sep.len > 0) try output.appendSlice(opts.sep); - } - } - - /// Some tokenizers have bytes encoded in hex like this: "<0x40>". - /// This break the tokenization algorithm because the input text - /// will contain "@" not "<0x40>", - /// and if the input contains "<0x40>" it needs to not be treated as a single byte. - /// So we replace byte fallbacks strings, by their corresponding character. - /// This enables the normal tokenization algorithm to work. - pub fn rewriteByteFallbackTokens(tokenizer: *Tokenizer) !void { - tokenizer.byte_fallback = true; - var single_bytes = try tokenizer.arena_state.allocator().alloc(u8, 256); - var byte_fallback_buf = "<0x00>".*; - - for (0..256) |i| { - const c: u8 = @truncate(i); - single_bytes[i] = c; - - // First lookup the byte fallback entry. - // Note: we assume upper case, but we could try both upper and lower case if needed. - _ = std.fmt.bufPrintIntToSlice(byte_fallback_buf[3..5], c, 16, .upper, .{ .fill = '0', .width = 2 }); - const entry = tokenizer.token_lookup.getEntry(&byte_fallback_buf) orelse { - log.err("Tokenizer has \"byte_fallback\" = true, but doesn't contains the byte fallback token {s}", .{byte_fallback_buf}); - return error.InvalidInput; - }; - - // Check if the character is already present in the vocab. - // In that case, nothing to do, - // but note that the fallback token will be "unreachable", - // ie there is no way the tokenizer can produce it. - if (tokenizer.token_lookup.get(&.{c})) |_| continue; - - const idx: u32 = entry.value_ptr.*; - tokenizer.token_lookup.removeByPtr(entry.key_ptr); - tokenizer.addOwnedTokenByIndex(idx, tokenizer.scores[idx], single_bytes[i .. i + 1]); - } - } -}; - -test Tokenizer { - const allocator = std.testing.allocator; - const special_tokens: Tokenizer.SpecialTokens = .{ - .unk = 0, - .bos = 1, - .eos = 2, - }; - - var tokenizer = try Tokenizer.init(allocator, 10, 5, null, special_tokens, true); - defer tokenizer.deinit(); - - try tokenizer.addToken(10, "hello"); - try tokenizer.addToken(3.5, "world"); - - try testing.expect(tokenizer.lookup("hello") == 0); - try testing.expect(tokenizer.lookup("world") == 1); - - // TODO: test Tokenizer.decode, Tokenizer.encode, Tokenizer.readTokenInto -} - -/// Given a slice, split it in the most simple tokens using the given tokenizer tokens. -/// The output of this can be used to initialize the tokenization algorithm. -/// Normally we split the input text into utf8 codepoint, -/// but if we find an unknown codepoint we either split it in bytes, or use the special "unknown" token, -/// depending on the tokenizer configuration. -const CharTokenIterator = struct { - state: union(enum) { by_codepoint, by_byte: u8 } = .by_codepoint, - input: []const u8, - - fn nextCodepointToken(self: *CharTokenIterator, tokenizer: *const Tokenizer) error{ TruncatedInput, Utf8InvalidStartByte }!?u32 { - if (self.input.len == 0) return null; - return switch (self.state) { - .by_byte => |*byte_left| { - const idx = tokenizer.lookup(self.input[0..1]) orelse { - // Normally this has been caught when calling `rewriteByteFallbackTokens`. - std.debug.panic("Tokenizer has \"byte_fallback\" = true, but doesn't contains the byte fallback for token '<0x{X:02}>'", .{self.input[0]}); - }; - - self.input = self.input[1..]; - byte_left.* -|= 1; - if (byte_left.* == 0) self.state = .by_codepoint; - return idx; - }, - .by_codepoint => { - // Try to lookup valid utf8 codepoint first. - const utf8_len = try std.unicode.utf8ByteSequenceLength(self.input[0]); - if (self.input.len < utf8_len) return error.TruncatedInput; - if (tokenizer.lookup(self.input[0..utf8_len])) |idx| { - self.input = self.input[utf8_len..]; - return idx; - } - - // Otherwise split in bytes if it's allowed. - if (tokenizer.byte_fallback) { - // TODO: replace this by a continue statement next time we bump Zig. - self.state = .{ .by_byte = utf8_len }; - return self.nextCodepointToken(tokenizer); - } - - // Or mark the full utf8 codepoint as unknown. - log.debug("Token not found for char '{s}'", .{self.input[0..utf8_len]}); - self.input = self.input[utf8_len..]; - return tokenizer.special_tokens.unk; - }, - }; - } -}; - -test CharTokenIterator { - const special_tokens: Tokenizer.SpecialTokens = .{ .unk = 0, .bos = 1, .eos = 2 }; - var tokenizer = try Tokenizer.init(std.testing.allocator, 16, 4, null, special_tokens, true); - defer tokenizer.deinit(); - - tokenizer.addOwnedToken(1.0, ""); // 0 - tokenizer.addOwnedToken(1.0, ""); // 1 - tokenizer.addOwnedToken(1.0, ""); // 2 - tokenizer.addOwnedToken(1.0, "ζ"); // 3 - tokenizer.addOwnedToken(1.0, &.{0xE2}); // 4: ℳ, first byte - tokenizer.addOwnedToken(1.0, &.{0x84}); // 5: ℳ, second byte - tokenizer.addOwnedToken(1.0, &.{0xB3}); // 6: ℳ, third byte - tokenizer.addOwnedToken(1.0, "L"); // 7 - - // No byte fallback - { - tokenizer.byte_fallback = false; - var it: CharTokenIterator = .{ .input = "ζℳL" }; - var res: std.BoundedArray(u32, 8) = .{}; - while (try it.nextCodepointToken(&tokenizer)) |token| { - res.appendAssumeCapacity(token); - } - try std.testing.expectEqualSlices(u32, &[_]u32{ 3, 0, 7 }, res.constSlice()); - } - - // with byte fallback - { - tokenizer.byte_fallback = true; - var it: CharTokenIterator = .{ .input = "ζℳL" }; - var res: std.BoundedArray(u32, 8) = .{}; - while (try it.nextCodepointToken(&tokenizer)) |token| { - res.appendAssumeCapacity(token); - } - try std.testing.expectEqualSlices(u32, &[_]u32{ 3, 4, 5, 6, 7 }, res.constSlice()); - } -} - -/// Text normalizer. -/// Most tokenizer assumes the input text have been prepocessed with on of those. -pub const Normalizer = struct { - /// Space token used by sentencepiece derived tokenizer. - pub const sentencepiece_space = "▁"; // \xe2\x96\x81 - - _whitespace: std.BoundedArray(u8, 8) = .{}, - - flags: packed struct { - remove_extra_whitespaces: bool, - add_dummy_prefix: bool, - add_dummy_suffix: bool, - /// Cheap lower casing. - /// TODO: try to match Python "lower" - lower_case_ascii: bool, - /// cheap ascii punct splitting. - // doing this processing ahead of time simplifies the logic - split_on_punct_ascii: bool, - }, - - pub fn init(flags: std.meta.FieldType(Normalizer, .flags), escaped_whitespace: ?[]const u8) Normalizer { - var res: Normalizer = .{ .flags = flags }; - if (escaped_whitespace) |escaped| { - res._whitespace.appendSliceAssumeCapacity(escaped); - } - return res; - } - - pub inline fn escapedSpace(self: Normalizer) ?[]const u8 { - return if (self._whitespace.len > 1) self._whitespace.constSlice() else null; - } - - fn addSlice(data: []const u8, consumed: usize, normalized: *std.ArrayList(u8), normalized_to_origin: *std.ArrayList(usize)) !void { - try normalized.appendSlice(data); - for (data) |_| try normalized_to_origin.append(consumed); - } - - pub const Result = struct { - /// Normalized string - normalized: []const u8, - /// Mapping between chars in the original string and chars in the new string - normalized_to_origin: []const usize, - - pub fn deinit(self: Result, allocator: std.mem.Allocator) void { - allocator.free(self.normalized); - allocator.free(self.normalized_to_origin); - } - }; - - /// Simplifed version of Sentencepiece normalizer. - /// - /// Llama2 uses a normalizer called "identity" so this basically only handles trailing - /// whitespaces and replaces whitespace with the "▁" (U+2581) character. - pub fn normalize(self: Normalizer, allocator: std.mem.Allocator, input: []const u8) ![]const u8 { - const res = try self.normalizeWithMapping(allocator, input); - allocator.free(res.normalized_to_origin); - return res.normalized; - } - - /// Returns both the normalized string and a mapping between the normalized string and the original. - pub fn normalizeWithMapping(self: Normalizer, allocator: std.mem.Allocator, input: []const u8) !Result { - // Number of bytes consumed from the input. - var consumed: usize = 0; - var trimmed_input = input; - - // Skip leading whitespaces. - if (self.flags.remove_extra_whitespaces) { - while (trimmed_input.len != 0) { - if (trimmed_input[0] != ' ') break; - trimmed_input = trimmed_input[1..]; - consumed += 1; - } - } - - // If the trimmed input is empty, we are done. - if (trimmed_input.len == 0) { - return .{ .normalized = &.{}, .normalized_to_origin = &.{} }; - } - - // Pre-allocate outputs - const space = self.escapedSpace() orelse " "; - const overhead = if (self.flags.split_on_punct_ascii) space.len + 1 else space.len; - var normalized = try std.ArrayList(u8).initCapacity(allocator, trimmed_input.len * overhead + 2 * space.len); - errdefer normalized.deinit(); - var normalized_to_origin = try std.ArrayList(usize).initCapacity(allocator, normalized.capacity); - errdefer normalized_to_origin.deinit(); - - // If the spec asks for it, add a whitespace at the beginning. - if (self.flags.add_dummy_prefix) try addSlice(space, consumed, &normalized, &normalized_to_origin); - - var is_prev_space: bool = true; - var is_prev_word: bool = false; - - while (trimmed_input.len != 0) { - // NOTE(Corendos): This might feel weird but normally the slice we get comes from a normalizing process and can contain multiple codepoints. - // Since we have an "identity" normalizer, each slice is actually a unicode character. - const multibyte_length = try std.unicode.utf8ByteSequenceLength(trimmed_input[0]); - var slice = trimmed_input[0..multibyte_length]; - const origin = consumed; - consumed += multibyte_length; - trimmed_input = trimmed_input[multibyte_length..]; - - if (self.flags.remove_extra_whitespaces and is_prev_space) { - while (slice.len > 0 and slice[0] == ' ') { - slice = slice[1..]; - } - if (slice.len == 0) continue; - } - is_prev_space = slice[slice.len - 1] == ' '; - - if (slice.len == 1) ascii: { - // The more advanced logic only works with ascii atm - var byte = slice[0]; - if (self.escapedSpace() != null and byte == ' ') { - // replace the space token by the special token - try addSlice(space, origin, &normalized, &normalized_to_origin); - is_prev_word = false; - break :ascii; - } else if (self.flags.split_on_punct_ascii) { - if (is_prev_word and isPunct(slice)) { - // Insert a space, but continue handling the rest - try addSlice(space, origin, &normalized, &normalized_to_origin); - } - } - if (self.flags.lower_case_ascii) { - byte = std.ascii.toLower(byte); - } - try normalized.append(byte); - try normalized_to_origin.append(origin); - } else { - // we can safely copy to the output. - try addSlice(slice, origin, &normalized, &normalized_to_origin); - } - is_prev_word = !is_prev_space and !isPunct(slice); - } - - // Skip trailing whitespaces - if (self.flags.remove_extra_whitespaces) { - while (std.mem.endsWith(u8, normalized.items, space)) { - const length = normalized.items.len - space.len; - consumed = normalized_to_origin.items[length]; - try normalized.resize(length); - try normalized_to_origin.resize(length); - } - } - - try normalized_to_origin.append(consumed); - - std.debug.assert(normalized_to_origin.items.len == normalized.items.len + 1); - - if (self.flags.add_dummy_suffix) try addSlice(space, consumed, &normalized, &normalized_to_origin); - - return .{ - .normalized = try normalized.toOwnedSlice(), - .normalized_to_origin = try normalized_to_origin.toOwnedSlice(), - }; - } - - pub fn wellKnown(impl: KnownImplementation) Normalizer { - return switch (impl) { - .sentencepiece => init(.{ - .remove_extra_whitespaces = true, - .add_dummy_prefix = true, - .add_dummy_suffix = false, - .lower_case_ascii = false, - .split_on_punct_ascii = false, - }, sentencepiece_space), - .gpt2 => init(.{ - .remove_extra_whitespaces = true, - .add_dummy_prefix = true, - .add_dummy_suffix = false, - .lower_case_ascii = false, - .split_on_punct_ascii = false, - }, null), - }; - } - - pub fn fromHfJson(config: std.json.ObjectMap) error{InvalidNormalizerJson}!Normalizer { - var normalizer: Normalizer = .{ .flags = .{ - .remove_extra_whitespaces = false, - .add_dummy_suffix = false, - .add_dummy_prefix = false, - .lower_case_ascii = false, - .split_on_punct_ascii = false, - } }; - - // Normalizer config can be a single normalizer, or a sequence of normalizers. - const maybe_steps = object_get(config, .array, "normalizers"); - const steps = if (maybe_steps) |st| st.items else &.{std.json.Value{ .object = config }}; - - for (steps) |step_val| { - if (step_val != .object) { - return error.InvalidNormalizerJson; - } - const step = step_val.object; - - const step_type = object_get(step, .string, "type") orelse { - return error.InvalidNormalizerJson; - }; - if (std.mem.eql(u8, "Prepend", step_type)) { - normalizer.flags.add_dummy_prefix = true; - } else if (std.mem.eql(u8, "Append", step_type)) { - normalizer.flags.add_dummy_suffix = true; - } else if (std.mem.eql(u8, "Replace", step_type)) { - const pattern = object_get(step, .object, "pattern") orelse return error.InvalidNormalizerJson; - const str_pattern = object_get(pattern, .string, "String") orelse return error.InvalidNormalizerJson; - - if (std.mem.eql(u8, str_pattern, " ")) { - normalizer._whitespace.appendSliceAssumeCapacity( - object_get(step, .string, "content") orelse return error.InvalidNormalizerJson, - ); - } else { - log.warn("Normalizer Replace pattern not supported: '{s}' -> '{s}'", .{ str_pattern, object_get(pattern, .string, "content") orelse "" }); - } - } else { - log.warn("Unknown normalizer type: {s}", .{step_type}); - } - } - - return normalizer; - } - - test "Normalizer.fromHfJson" { - const config_json = - \\{ - \\ "type": "Sequence", - \\ "normalizers": [ - \\ { - \\ "type": "Prepend", - \\ "prepend": "▁" - \\ }, - \\ { - \\ "type": "Replace", - \\ "pattern": { - \\ "String": " " - \\ }, - \\ "content": "▁" - \\ } - \\ ] - \\} - ; - var arena = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena.deinit(); - const config = try std.json.parseFromSliceLeaky(std.json.Value, arena.allocator(), config_json, .{}); - const normalizer = try Normalizer.fromHfJson(config.object); - - const expected = Normalizer{ - ._whitespace = .{ .buffer = [_]u8{ 0xe2, 0x96, 0x81 } ++ [_]u8{0} ** 5, .len = 3 }, - .flags = .{ - .remove_extra_whitespaces = false, - .add_dummy_prefix = true, - .add_dummy_suffix = false, - .lower_case_ascii = false, - .split_on_punct_ascii = false, - }, - }; - try std.testing.expectEqual(expected.flags, normalizer.flags); - try std.testing.expectEqualStrings(expected.escapedSpace().?, normalizer.escapedSpace().?); - } -}; -pub const KnownImplementation = enum(u8) { - sentencepiece, - gpt2, -}; - -fn isPunct(unicode_char: []const u8) bool { - // TODO use unicode categories - if (unicode_char.len > 1) return false; - - return switch (unicode_char[0]) { - ' ', '\t' => false, - 0...8 => true, - 10...31 => true, - '!'...'/' => true, - ':'...'@' => true, - '['...'`' => true, - '{'...'~' => true, - else => false, - }; -} - -test Normalizer { - { - const n: Normalizer = .{ .flags = .{ - .remove_extra_whitespaces = true, - .add_dummy_prefix = true, - .add_dummy_suffix = false, - .lower_case_ascii = false, - .split_on_punct_ascii = false, - } }; - const res = try n.normalizeWithMapping(testing.allocator, "Hellŏ world!"); - defer res.deinit(testing.allocator); - - try testing.expectEqualSlices(u8, " Hellŏ world!", res.normalized); - try testing.expectEqualSlices( - usize, - // H e l l ŏ ␣ w o r l d ! - &.{ 0, 0, 1, 2, 3, 4, 4, 6, 8, 9, 10, 11, 12, 13, 14 }, - res.normalized_to_origin, - ); - } - - { - const n: Normalizer = .{ .flags = .{ - .remove_extra_whitespaces = true, - .add_dummy_prefix = true, - .add_dummy_suffix = true, - .lower_case_ascii = false, - .split_on_punct_ascii = false, - } }; - const res = try n.normalize(testing.allocator, "Hello world!"); - defer testing.allocator.free(res); - - try testing.expectEqualSlices(u8, " Hello world! ", res); - } - - { - const n = Normalizer.init( - .{ - .remove_extra_whitespaces = false, - .add_dummy_prefix = true, - .add_dummy_suffix = false, - .lower_case_ascii = false, - .split_on_punct_ascii = false, - }, - Normalizer.sentencepiece_space, - ); - const res = try n.normalize(testing.allocator, "Hello world!"); - defer testing.allocator.free(res); - - try testing.expectEqualSlices(u8, "▁Hello▁▁world!", res); - } - - { - const n: Normalizer = .{ .flags = .{ - .remove_extra_whitespaces = true, - .add_dummy_prefix = false, - .add_dummy_suffix = true, - .lower_case_ascii = true, - .split_on_punct_ascii = false, - } }; - const res = try n.normalize(testing.allocator, "Hello world!"); - defer testing.allocator.free(res); - - try testing.expectEqualSlices(u8, "hello world! ", res); - } - - { - const n: Normalizer = .{ .flags = .{ - .remove_extra_whitespaces = true, - .add_dummy_prefix = false, - .add_dummy_suffix = true, - .lower_case_ascii = false, - .split_on_punct_ascii = true, - } }; - const res = try n.normalize(testing.allocator, "Hello world!"); - defer testing.allocator.free(res); - - try testing.expectEqualSlices(u8, "Hello world ! ", res); - } -} - -/// gpt2 had their own way of storing text. -/// Unfortunately this has contaminated other models. -/// This implementation precompupte a mapping between bytes encoded with GPT2 algorithm, -/// into utf8 bytes, and do lookups at runtime. -pub const Gpt2TextDecoder = struct { - const Code = std.BoundedArray(u8, 2); - - // TODO: benchmark this is more efficient than doing the conversion at runtime. - code_to_byte: std.AutoArrayHashMap(Code, u8), - - pub fn init(allocator: std.mem.Allocator) !Gpt2TextDecoder { - var self = Gpt2TextDecoder{ - .code_to_byte = std.AutoArrayHashMap(Code, u8).init(allocator), - }; - try self.code_to_byte.ensureTotalCapacity(256); - errdefer unreachable; - - var n: usize = 0; - for (0..256) |index| { - var code: Code = .{ .buffer = .{ 0, 0 }, .len = 0 }; // 0-init - const i: u8 = @intCast(index); - if (isPrintableByte(i)) { - if (std.ascii.isASCII(i)) { - code.appendAssumeCapacity(i); - } else { - const codepoint: u21 = @as(u21, @intCast(i)); - code.len = @intCast(std.unicode.utf8Encode(codepoint, &code.buffer) catch unreachable); - } - } else { - const codepoint: u21 = 256 + @as(u21, @intCast(n)); - code.len = @intCast(std.unicode.utf8Encode(codepoint, &code.buffer) catch unreachable); - n += 1; - } - - self.code_to_byte.putAssumeCapacityNoClobber(code, i); - } - return self; - } - - pub fn deinit(self: *Gpt2TextDecoder) void { - self.code_to_byte.deinit(); - } - - /// Transform bytes representing text under the gpt2 encoding, - /// and write to the `unicode` buffer utf-8 bytes. - pub fn decode(self: Gpt2TextDecoder, unicode: *std.ArrayList(u8), bytes: []const u8) ![]const u8 { - const start = unicode.items.len; - var it = std.unicode.Utf8Iterator{ .i = 0, .bytes = bytes }; - while (it.nextCodepointSlice()) |codepoint| { - const code: Code = switch (codepoint.len) { - 1 => .{ .buffer = .{ codepoint[0], 0 }, .len = 1 }, // 0-init - 2 => .{ .buffer = .{ codepoint[0], codepoint[1] }, .len = 2 }, - else => return error.InvalidInput, - }; - const byte = self.code_to_byte.get(code) orelse return error.InvalidInput; - try unicode.append(byte); - } - return unicode.items[start..]; - } - - inline fn isPrintableByte(c: u8) bool { - return ('!' <= c and c <= '~') or (0xa1 <= c and c <= 0xac) or (0xae <= c and c <= 0xff); - } -}; - -test Gpt2TextDecoder { - var decoder = try Gpt2TextDecoder.init(testing.allocator); - defer decoder.deinit(); - - var out = std.ArrayList(u8).init(testing.allocator); - defer out.deinit(); - - // Ascii is not changed. - try testing.expectEqualStrings("getTitle", try decoder.decode(&out, "getTitle")); - // Leading space are represented with 'Ġ' - try testing.expectEqualStrings(" UINavigationController", try decoder.decode(&out, "ĠUINavigationController")); - // Russian is wild - try testing.expectEqualStrings(" работ", try decoder.decode(&out, "ĠÑĢабоÑĤ")); -} - -/// Open a json file in HF format and load the vocab from it. -pub fn fromHfJson(allocator: std.mem.Allocator, tokenizer_path: []const u8) !Tokenizer { - const file = try std.fs.cwd().openFile(tokenizer_path, .{}); - defer file.close(); - - var arena_state = std.heap.ArenaAllocator.init(allocator); - defer arena_state.deinit(); - const arena = arena_state.allocator(); - const file_content = try file.readToEndAlloc(arena, 32 * 1024 * 1024); - - const info = try std.json.parseFromSliceLeaky(std.json.Value, arena, file_content, .{ - .duplicate_field_behavior = .use_last, - }); - const main_object = switch (info) { - .object => |obj| if (obj.get("added_tokens") == null or obj.get("model") == null) { - return error.InvalidFormat; - } else obj, - else => return error.InvalidFormat, - }; - - // TODO: remove all panics - const added_tokens = main_object.get("added_tokens").?.array; - const vocab = main_object.get("model").?.object.get("vocab").?.object; - const vocab_size: u32 = @intCast(vocab.count() + added_tokens.items.len); - - const normalizer = if (object_get(main_object, .object, "normalizer")) |normalizer_config| - try Normalizer.fromHfJson(normalizer_config) - else - Normalizer.wellKnown(.gpt2); - - // delay init of special tokens. - var tokenizer = try Tokenizer.init(allocator, vocab_size, 256, normalizer, undefined, true); - errdefer tokenizer.deinit(); - - // Buffer containing all concatenated tokens. - // Reserve a big chunk, to avoid grow event, but release over-allocated memory. - var all_tokens = try std.ArrayList(u8).initCapacity(tokenizer.arena_state.allocator(), file_content.len); - const original_alloc = all_tokens.items.ptr; - // A re-alloc event here means we have invalidated all slices inside the tokenizer. - // If this is too annoying we could switch to a custom type instead of slices. - defer { - std.debug.assert(all_tokens.items.ptr == original_alloc); - } - - // gpt2 based tokenizer got a special way of encoding unicode. - // we don't know in advance if this will be used by this tokenizer or not. - // so we assume it is the case, but if we find some unicode character, - // outside of the range used by gpt2 we know it was wrong, and start over. - var is_gpt2_vocab: bool = true; - var gpt2_decoder = try Gpt2TextDecoder.init(allocator); - defer gpt2_decoder.deinit(); - var it = vocab.iterator(); - while (it.next()) |kv| { - const token = gpt2_decoder.decode(&all_tokens, kv.key_ptr.*) catch |err| { - switch (err) { - error.InvalidInput => { - is_gpt2_vocab = false; - break; - }, - else => return err, - } - }; - const idx: u32 = @intCast(kv.value_ptr.*.integer); - tokenizer.addOwnedTokenByIndex(idx, @floatFromInt(vocab_size - idx), token); - } - - if (!is_gpt2_vocab) { - // We where wrong, this is not a gpt2 vocab, start over, - // and reset the tokenizer state. - tokenizer.next_token_id = 0; - tokenizer.token_lookup.clearRetainingCapacity(); - all_tokens.clearRetainingCapacity(); - it = vocab.iterator(); - while (it.next()) |kv| { - const idx: u32 = @intCast(kv.value_ptr.*.integer); - const token = try dup(&all_tokens, kv.key_ptr.*); - tokenizer.addOwnedTokenByIndex(idx, @floatFromInt(vocab_size - idx), token); - } - } - - // More tokens, typically added during fine tuning of the model. - for (added_tokens.items) |token_obj| { - const v = token_obj.object.get("content").?.string; - const id: u32 = @intCast(token_obj.object.get("id").?.integer); - const token = try if (is_gpt2_vocab) - gpt2_decoder.decode(&all_tokens, v) - else - dup(&all_tokens, v); - - tokenizer.addOwnedTokenByIndex(id, 0, token); - } - // We won't add more tokens here, let release. - all_tokens.shrinkAndFree(all_tokens.items.len); - - tokenizer.special_tokens = .{ - .bos = tokenizer.lookup("") orelse tokenizer.lookup("<|begin_of_text|>") orelse @panic("bos token not found !"), - .eos = tokenizer.lookup("") orelse tokenizer.lookup("<|end_of_text|>") orelse @panic("eos token not found !"), - .unk = tokenizer.lookup("") orelse std.math.maxInt(u32), - }; - - if (main_object.get("model").?.object.get("byte_fallback")) |byte_fallback| { - if (byte_fallback == .bool and byte_fallback.bool) { - try tokenizer.rewriteByteFallbackTokens(); - } - } - return tokenizer; -} - -/// Returns a copy of the given string, stored inside the given ArrayList. -fn dup(buffer: *std.ArrayList(u8), str: []const u8) ![]const u8 { - const n = buffer.items.len; - try buffer.appendSlice(str); - return buffer.items[n..]; -} - -/// Returns the given entry in a json object only if it has the right type. -fn object_get( - object: std.json.ObjectMap, - comptime kind: std.meta.FieldEnum(std.json.Value), - key: []const u8, -) ?std.meta.FieldType(std.json.Value, kind) { - const val = object.get(key) orelse return null; - if (val != kind) return null; - return @field(val, @tagName(kind)); -} diff --git a/zml/tokenizer/BUILD.bazel b/zml/tokenizer/BUILD.bazel new file mode 100644 index 00000000..f2cbc132 --- /dev/null +++ b/zml/tokenizer/BUILD.bazel @@ -0,0 +1,39 @@ +load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library", "zig_test") +load("@zml//bazel:zig_srcs.bzl", "zig_srcs") + +zig_library( + name = "tokenizer", + import_name = "zml/tokenizer", + main = "tokenizer.zig", + tags = ["manual"], + visibility = ["//visibility:public"], + deps = [ + "//ffi:zig", + "//stdx", + "//zml/tokenizer/iree", + "//zml/tokenizer/sentencepiece", + ], +) + +zig_binary( + name = "main", + main = "main.zig", + visibility = ["//visibility:public"], + deps = [ + ":tokenizer", + "//stdx", + ], +) + +zig_test( + name = "test", + deps = [ + ":tokenizer", + ], +) + +# todo: reenable after fix +# zig_srcs( +# name = "sources", +# zig_lib = ":tokenizer", +# ) diff --git a/zml/tokenizer/iree/BUILD.bazel b/zml/tokenizer/iree/BUILD.bazel new file mode 100644 index 00000000..a49ccd61 --- /dev/null +++ b/zml/tokenizer/iree/BUILD.bazel @@ -0,0 +1,31 @@ +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") + +zig_library( + name = "iree", + srcs = ["tokenizer.zig"], + main = "tokenizer.zig", + visibility = ["//visibility:public"], + deps = [ + "@iree//runtime/src/iree/tokenizer", + "@iree//runtime/src/iree/tokenizer:normalizer", + "@iree//runtime/src/iree/tokenizer/format/huggingface:normalizer_json", + "@iree//runtime/src/iree/tokenizer/format/huggingface:tokenizer_json", + "@iree//runtime/src/iree/tokenizer/normalizer:lowercase", + "@iree//runtime/src/iree/tokenizer/normalizer:nfc", + "@iree//runtime/src/iree/tokenizer/normalizer:nfd", + "@iree//runtime/src/iree/tokenizer/normalizer:nfkd", + "@iree//runtime/src/iree/tokenizer/normalizer:regex_replace", + "@iree//runtime/src/iree/tokenizer/normalizer:replace", + "@iree//runtime/src/iree/tokenizer/normalizer:sequence", + "@iree//runtime/src/iree/tokenizer/normalizer:strip", + "@iree//runtime/src/iree/tokenizer/normalizer:strip_accents", + "@iree//runtime/src/iree/tokenizer/vocab", + ], +) + +zig_test( + name = "test", + deps = [ + ":iree", + ], +) diff --git a/zml/tokenizer/iree/tokenizer.zig b/zml/tokenizer/iree/tokenizer.zig new file mode 100644 index 00000000..b1586afb --- /dev/null +++ b/zml/tokenizer/iree/tokenizer.zig @@ -0,0 +1,1398 @@ +const std = @import("std"); + +const c = @import("c"); + +const log = std.log.scoped(.@"zml/tokenizer"); + +pub const Error = error{ + Cancelled, + Unknown, + InvalidArgument, + DeadlineExceeded, + NotFound, + AlreadyExists, + PermissionDenied, + ResourceExhausted, + FailedPrecondition, + Aborted, + OutOfRange, + Unimplemented, + Internal, + Unavailable, + DataLoss, + Unauthenticated, +}; + +const StatusCodeMask = 0x1F; + +inline fn stringView(s: []const u8) c.iree_string_view_t { + return .{ .data = s.ptr, .size = s.len }; +} + +inline fn mutableStringView(s: []u8) c.iree_mutable_string_view_t { + return .{ .data = s.ptr, .size = s.len }; +} + +inline fn byteSpan(bytes: []u8) c.iree_byte_span_t { + return .{ .data = bytes.ptr, .data_length = bytes.len }; +} + +inline fn tokenOutput(ids: []u32) c.iree_tokenizer_token_output_t { + comptime { + if (@sizeOf(c.iree_tokenizer_token_id_t) != @sizeOf(u32) or @alignOf(c.iree_tokenizer_token_id_t) != @alignOf(u32)) { + @compileError("iree_tokenizer_token_id_t must match u32 layout"); + } + } + return .{ + .capacity = ids.len, + .token_ids = @ptrCast(ids.ptr), + .token_offsets = null, + .type_ids = null, + }; +} + +inline fn tokenIdList(ids: []const u32) c.iree_tokenizer_token_id_list_t { + comptime { + if (@sizeOf(c.iree_tokenizer_token_id_t) != @sizeOf(u32) or @alignOf(c.iree_tokenizer_token_id_t) != @alignOf(u32)) { + @compileError("iree_tokenizer_token_id_t must match u32 layout"); + } + } + return .{ .count = ids.len, .values = @ptrCast(ids.ptr) }; +} + +inline fn statusCode(status: c.iree_status_t) u32 { + return @intCast(@intFromPtr(status.?) & StatusCodeMask); +} + +fn checkOk(status: c.iree_status_t) !void { + if (status == null) return; + const code = statusCode(status); + var buffer: [1024]u8 = undefined; + var buffer_length: usize = 0; + if (c.iree_status_format(status, buffer.len, &buffer, &buffer_length)) { + log.err("IREE tokenizer error: {s}", .{buffer[0..@min(buffer_length, buffer.len - 1)]}); + } + _ = c.iree_status_ignore(status); + return switch (code) { + c.IREE_STATUS_CANCELLED => Error.Cancelled, + c.IREE_STATUS_UNKNOWN => Error.Unknown, + c.IREE_STATUS_INVALID_ARGUMENT => Error.InvalidArgument, + c.IREE_STATUS_DEADLINE_EXCEEDED => Error.DeadlineExceeded, + c.IREE_STATUS_NOT_FOUND => Error.NotFound, + c.IREE_STATUS_ALREADY_EXISTS => Error.AlreadyExists, + c.IREE_STATUS_PERMISSION_DENIED => Error.PermissionDenied, + c.IREE_STATUS_RESOURCE_EXHAUSTED => Error.ResourceExhausted, + c.IREE_STATUS_FAILED_PRECONDITION => Error.FailedPrecondition, + c.IREE_STATUS_ABORTED => Error.Aborted, + c.IREE_STATUS_OUT_OF_RANGE => Error.OutOfRange, + c.IREE_STATUS_UNIMPLEMENTED => Error.Unimplemented, + c.IREE_STATUS_INTERNAL => Error.Internal, + c.IREE_STATUS_UNAVAILABLE => Error.Unavailable, + c.IREE_STATUS_DATA_LOSS => Error.DataLoss, + c.IREE_STATUS_UNAUTHENTICATED => Error.Unauthenticated, + else => Error.Internal, + }; +} + +pub const Tokenizer = struct { + // TODO: use the ArrayList pattern. Stop copying allocator everywhere + allocator: std.mem.Allocator, + inner: *c.iree_tokenizer_t, + + pub fn fromHuggingFaceJson(allocator: std.mem.Allocator, json: []const u8) !Tokenizer { + // TODO: wrap allocator into an iree allocator + var raw: ?*c.iree_tokenizer_t = null; + try checkOk(c.iree_tokenizer_from_huggingface_json(stringView(json), c.iree_allocator_system(), &raw)); + return .{ + .allocator = allocator, + .inner = raw.?, + }; + } + + pub fn fromBytes(allocator: std.mem.Allocator, bytes: []const u8) !Tokenizer { + return fromHuggingFaceJson(allocator, bytes); + } + + pub fn fromFile(allocator: std.mem.Allocator, io: std.Io, model: []const u8) !Tokenizer { + const json = try std.Io.Dir.cwd().readFileAlloc(io, model, allocator, .unlimited); + defer allocator.free(json); + return try fromHuggingFaceJson(allocator, json); + } + + pub fn deinit(self: *Tokenizer) void { + c.iree_tokenizer_free(self.inner); + } + + pub fn encoder(self: *const Tokenizer) !Encoder { + return Encoder.init(self); + } + + pub fn decoder(self: *const Tokenizer) !Decoder { + return Decoder.init(self, self.allocator); + } + + pub fn normalizer(self: *const Tokenizer, sequence: []const Normalizer.Kind) !Normalizer { + return Normalizer.init(self.allocator, sequence); + } + + pub fn normalizerFromHuggingFaceJson(_: *const Tokenizer, json: []const u8) !Normalizer { + return Normalizer.fromHuggingFaceJson(json); + } + + pub fn tokenId(self: *const Tokenizer, token: []const u8) ?u32 { + const vocab = c.iree_tokenizer_vocab(self.inner); + if (vocab == null) return null; + + // Looks up a string in the vocabulary. + // Returns the token ID if found, or -1 if not found. + const id = c.iree_tokenizer_vocab_lookup(vocab, stringView(token)); + return if (id < 0) null else @intCast(id); + } + + pub fn decode(self: *const Tokenizer, token_id: u32) ?[]const u8 { + const vocab = c.iree_tokenizer_vocab(self.inner); + if (vocab == null) return null; + + if (token_id > std.math.maxInt(i32)) return null; + + const text = c.iree_tokenizer_vocab_token_text(vocab, @intCast(token_id)); + if (text.data == null) return null; + + return text.data[0..text.size]; + } + + pub const Encoder = struct { + tokenizer: *const Tokenizer, + state_storage: []u8, + transform_buffer: []u8, + state: *c.iree_tokenizer_encode_state_t, + + fn checkEncodeStatus(status: c.iree_status_t) !void { + // Output exhaustion is normal streaming backpressure. The caller + // commits any produced tokens and retries with more output space. + if (status != null and statusCode(status) == c.IREE_STATUS_RESOURCE_EXHAUSTED) { + _ = c.iree_status_ignore(status); + return Error.ResourceExhausted; + } + try checkOk(status); + } + + fn init(tokenizer: *const Tokenizer) !Encoder { + var state_size: usize = undefined; + try checkOk(c.iree_tokenizer_encode_state_calculate_size(tokenizer.inner, &state_size)); + const state_storage = try tokenizer.allocator.alloc(u8, state_size); + errdefer tokenizer.allocator.free(state_storage); + + const transform_size: usize = c.IREE_TOKENIZER_TRANSFORM_BUFFER_MAX_SIZE; + const transform_buffer = try tokenizer.allocator.alloc(u8, transform_size); + errdefer tokenizer.allocator.free(transform_buffer); + + var state: ?*c.iree_tokenizer_encode_state_t = null; + try checkOk(c.iree_tokenizer_encode_state_initialize( + tokenizer.inner, + byteSpan(state_storage), + byteSpan(transform_buffer), + c.iree_tokenizer_offset_run_list_empty(), + c.IREE_TOKENIZER_ENCODE_FLAG_AT_INPUT_START, + &state, + )); + + return .{ + .tokenizer = tokenizer, + .state_storage = state_storage, + .transform_buffer = transform_buffer, + .state = state.?, + }; + } + + pub fn deinit(self: *Encoder) void { + c.iree_tokenizer_encode_state_deinitialize(self.state); + self.tokenizer.allocator.free(self.state_storage); + self.tokenizer.allocator.free(self.transform_buffer); + } + + pub fn reset(self: *Encoder) void { + c.iree_tokenizer_encode_state_reset(self.state, c.IREE_TOKENIZER_ENCODE_FLAG_AT_INPUT_START); + } + + pub fn encodeAlloc(self: *Encoder, allocator: std.mem.Allocator, text: []const u8) ![]u32 { + var token_ids = std.Io.Writer.Allocating.initAligned(allocator, .of(u32)); + errdefer token_ids.deinit(); + + try self.feed(text, &token_ids.writer); + try self.finalize(&token_ids.writer); + + const bytes = try token_ids.toOwnedSlice(); + const aligned_bytes: []align(4) u8 = @alignCast(bytes[0..bytes.len]); + return std.mem.bytesAsSlice(u32, aligned_bytes); + } + + /// /!\ Assumes the writer provides buffers with proper alignment. + /// /!\ Remember to call the finalize() method + pub fn writer(self: *Encoder, buffer: []u8, out: *std.Io.Writer) Writer { + return Writer.init(self, buffer, out); + } + + pub const Writer = struct { + encoder: *Encoder, + out: *std.Io.Writer, + interface: std.Io.Writer, + + fn init(encoder_: *Encoder, buffer: []u8, out: *std.Io.Writer) Writer { + return .{ + .encoder = encoder_, + .out = out, + .interface = .{ + .buffer = buffer, + .vtable = &.{ + .drain = drain, + .flush = flush, + .rebase = std.Io.Writer.failingRebase, + }, + }, + }; + } + + pub fn finalize(self: *Writer) std.Io.Writer.Error!void { + try self.interface.flush(); + self.encoder.finalize(self.out) catch |e| { + log.err("Finalize failed with {s}", .{@errorName(e)}); + return error.WriteFailed; + }; + } + + fn flush(w: *std.Io.Writer) std.Io.Writer.Error!void { + if (w.end == 0) return; + const self: *Writer = @alignCast(@fieldParentPtr("interface", w)); + try self.feed(w.buffer[0..w.end]); + w.end = 0; + } + + fn drain(w: *std.Io.Writer, data: []const []const u8, splat: usize) std.Io.Writer.Error!usize { + const self: *Writer = @alignCast(@fieldParentPtr("interface", w)); + + if (w.end != 0) { + try self.feed(w.buffer[0..w.end]); + w.end = 0; + } + + var total: usize = 0; + // Prefix chunks: written once each. + for (data[0 .. data.len - 1]) |chunk| { + try self.feed(chunk); + total += chunk.len; + } + // Last chunk is the splat pattern. + const pattern = data[data.len - 1]; + if (pattern.len == 0) return total; + for (0..splat) |_| { + try self.feed(pattern); + total += pattern.len; + } + return total; + } + + fn feed(self: *Writer, chunk: []const u8) std.Io.Writer.Error!void { + self.encoder.feed(chunk, self.out) catch |e| { + log.err("Feed failed with {s}", .{@errorName(e)}); + return error.WriteFailed; + }; + } + }; + + /// /!\ Assumes the writer provides buffers with proper alignment. + pub fn feed(self: *Encoder, text: []const u8, out: *std.Io.Writer) !void { + var min_tokens: usize = @max(text.len / 4, 32); + var remaining = text; + while (remaining.len > 0) { + var consumed: usize = 0; + var produced: usize = 0; + + checkEncodeStatus(c.iree_tokenizer_encode_state_feed( + self.state, + stringView(remaining), + tokenOutput(try writableTokenSliceGreedy(out, min_tokens)), + &consumed, + &produced, + )) catch |e| switch (e) { + error.ResourceExhausted => { + out.advance(produced * @sizeOf(u32)); + min_tokens = @max(32, min_tokens -| produced); + remaining = remaining[consumed..]; + produced = 0; + consumed = 0; + }, + else => return e, + }; + + if (produced == 0 and consumed == 0) { + min_tokens *= 2; + std.debug.assert(min_tokens < 1 << 20); + continue; + } + + out.advance(produced * @sizeOf(u32)); + min_tokens = @max(32, min_tokens -| produced); + remaining = remaining[consumed..]; + } + } + + /// /!\ Assumes the writer provides buffers with proper alignment. + pub fn finalize(self: *Encoder, out: *std.Io.Writer) !void { + var min_tokens: usize = 4; + while (true) { + var produced: usize = 0; + + checkEncodeStatus(c.iree_tokenizer_encode_state_finalize( + self.state, + tokenOutput(try writableTokenSliceGreedy(out, min_tokens)), + &produced, + )) catch |e| switch (e) { + error.ResourceExhausted => { + out.advance(produced * @sizeOf(u32)); + min_tokens *= 2; + std.debug.assert(min_tokens < 1 << 10); + continue; + }, + else => return e, + }; + + out.advance(produced * @sizeOf(u32)); + if (!c.iree_tokenizer_encode_state_has_pending(self.state)) break; + if (produced == 0) { + min_tokens *= 2; + std.debug.assert(min_tokens < 1 << 20); + } + } + } + + fn writableTokenSliceGreedy(out: *std.Io.Writer, min_tokens: usize) std.Io.Writer.Error![]u32 { + std.debug.assert(@mod(out.end, @sizeOf(u32)) == 0); + const bytes = try out.writableSliceGreedy(min_tokens * @sizeOf(u32)); + std.debug.assert(@mod(@intFromPtr(bytes.ptr), 4) == 0); + const aligned_len = bytes.len - @mod(bytes.len, @sizeOf(u32)); + const aligned_bytes: []align(4) u8 = @alignCast(bytes[0..aligned_len]); + return std.mem.bytesAsSlice(u32, aligned_bytes); + } + }; + + pub const Decoder = struct { + tokenizer: *const Tokenizer, + state_storage: []u8, + state: *c.iree_tokenizer_decode_state_t, + + fn init(tokenizer: *const Tokenizer, allocator: std.mem.Allocator) !Decoder { + var state_size: usize = undefined; + try checkOk(c.iree_tokenizer_decode_state_calculate_size(tokenizer.inner, &state_size)); + const state_storage = try allocator.alloc(u8, state_size); + errdefer allocator.free(state_storage); + + var state: ?*c.iree_tokenizer_decode_state_t = null; + try checkOk(c.iree_tokenizer_decode_state_initialize( + tokenizer.inner, + c.IREE_TOKENIZER_DECODE_FLAG_NONE, + byteSpan(state_storage), + &state, + )); + + return .{ + .tokenizer = tokenizer, + .state_storage = state_storage, + .state = state.?, + }; + } + + pub fn deinit(self: *Decoder) void { + c.iree_tokenizer_decode_state_deinitialize(self.state); + self.tokenizer.allocator.free(self.state_storage); + } + + pub fn reset(self: *Decoder) !void { + c.iree_tokenizer_decode_state_deinitialize(self.state); + var state: ?*c.iree_tokenizer_decode_state_t = null; + try checkOk(c.iree_tokenizer_decode_state_initialize( + self.tokenizer.inner, + c.IREE_TOKENIZER_DECODE_FLAG_NONE, + byteSpan(self.state_storage), + &state, + )); + self.state = state.?; + } + + pub fn feedOne(self: *Decoder, token_id: u32, out: []u8) ![]u8 { + std.debug.assert(out.len != 0); + var consumed: usize = 0; + var produced: usize = 0; + try checkOk(c.iree_tokenizer_decode_state_feed( + self.state, + tokenIdList(&.{token_id}), + mutableStringView(out), + &consumed, + &produced, + )); + std.debug.assert(consumed == 1); + return out[0..produced]; + } + + pub fn finalize(self: *Decoder, out: []u8) ![]u8 { + std.debug.assert(out.len != 0); + var produced: usize = 0; + try checkOk(c.iree_tokenizer_decode_state_finalize( + self.state, + mutableStringView(out), + &produced, + )); + return out[0..produced]; + } + + pub fn decode(self: *Decoder, token_ids: []const u32, out: *std.Io.Writer) !void { + var min_output: usize = @max(token_ids.len * 4, 128); + var remaining = token_ids; + while (remaining.len > 0) { + var consumed: usize = 0; + var produced: usize = 0; + + checkOk(c.iree_tokenizer_decode_state_feed( + self.state, + tokenIdList(remaining), + mutableStringView(try out.writableSliceGreedy(min_output)), + &consumed, + &produced, + )) catch |e| switch (e) { + error.ResourceExhausted => { + out.advance(produced); + min_output = @max(128, min_output -| produced); + remaining = remaining[consumed..]; + produced = 0; + consumed = 0; + }, + else => return e, + }; + + if (produced == 0 and consumed == 0) { + @branchHint(.unlikely); + min_output *= 2; + std.debug.assert(min_output < 1 << 20); + continue; + } + + out.advance(produced); + min_output = @max(128, min_output -| produced); + remaining = remaining[consumed..]; + } + + min_output = 32; + while (true) { + var produced: usize = 0; + + checkOk(c.iree_tokenizer_decode_state_finalize( + self.state, + mutableStringView(try out.writableSliceGreedy(min_output)), + &produced, + )) catch |e| switch (e) { + error.ResourceExhausted => { + out.advance(produced); + min_output *= 2; + std.debug.assert(min_output < 1 << 10); + continue; + }, + else => return e, + }; + + out.advance(produced); + break; + } + } + }; +}; + +pub const Normalizer = struct { + pub const Kind = union(enum) { + /// Unicode Normalization Form C. + NFC, + /// Unicode Normalization Form D. + NFD, + /// Unicode Normalization Form KD. + NFKD, + /// Removes Unicode combining marks. + /// filters out characters in the Unicode Mark category (Mn, Mc, Me). + StripAccents, + /// Converts text to lowercase. + Lowercase, + /// Substitutes occurrences of a literal or regular-expression pattern with the specified content. + Replace: struct { + kind: enum { + regex, + exact, + }, + pattern: []const u8, + content: []const u8, + }, + /// Removes leading and/or trailing whitespace. + Strip: enum(u8) { + left = 0b10, + right = 0b01, + both = 0b11, + }, + }; + + sequence: *c.iree_tokenizer_normalizer_t, + sequence_size: usize, + + // Supported normalizer types as of `4d4e97d00f099a21f38eeff26f82a6d9e3643a11`: + // - Sequence: Chains multiple normalizers in order + // - Lowercase: Unicode case folding + // - Strip: Leading/trailing whitespace removal + // - Prepend: Prefix string insertion + // - StripAccents: Removes combining marks (without NFD) + // - BertNormalizer: Combined BERT normalization pipeline + pub fn fromHuggingFaceJson(json: []const u8) !Normalizer { + var normalizer: ?*c.iree_tokenizer_normalizer_t = null; + errdefer if (normalizer) |ptr| c.iree_tokenizer_normalizer_free(ptr); + + try checkOk(c.iree_tokenizer_huggingface_parse_normalizer(.{ .data = json.ptr, .size = json.len }, c.iree_allocator_system(), &normalizer)); + if (normalizer == null) return error.NormalizerAllocationFailed; + + const sequence_size: usize = @intCast(c.iree_tokenizer_normalizer_state_size(normalizer.?)); + + return .{ + .sequence = normalizer.?, + .sequence_size = sequence_size, + }; + } + + fn createNormalizer(normalizer_kind: Kind) !?*c.iree_tokenizer_normalizer_t { + var norm: ?*c.iree_tokenizer_normalizer_t = null; + + const iree_allocator = c.iree_allocator_system(); + + const status = switch (normalizer_kind) { + .NFC => c.iree_tokenizer_normalizer_nfc_allocate(iree_allocator, &norm), + .NFD => c.iree_tokenizer_normalizer_nfd_allocate(iree_allocator, &norm), + .NFKD => c.iree_tokenizer_normalizer_nfkd_allocate(iree_allocator, &norm), + .StripAccents => c.iree_tokenizer_normalizer_strip_accents_allocate(iree_allocator, &norm), + .Lowercase => c.iree_tokenizer_normalizer_lowercase_allocate(iree_allocator, &norm), + .Replace => |rule| switch (rule.kind) { + .regex => c.iree_tokenizer_normalizer_regex_replace_allocate( + .{ .data = rule.pattern.ptr, .size = rule.pattern.len }, + .{ .data = rule.content.ptr, .size = rule.content.len }, + iree_allocator, + &norm, + ), + .exact => c.iree_tokenizer_normalizer_replace_allocate( + .{ .data = rule.pattern.ptr, .size = rule.pattern.len }, + .{ .data = rule.content.ptr, .size = rule.content.len }, + iree_allocator, + &norm, + ), + }, + .Strip => |direction| b: { + const strip_left = @backingInt(direction) & 0b10 != 0; + const strip_right = @backingInt(direction) & 0b01 != 0; + break :b c.iree_tokenizer_normalizer_strip_allocate(strip_left, strip_right, iree_allocator, &norm); + }, + }; + + try checkOk(status); + if (norm == null) return error.NormalizerAllocationFailed; + return norm; + } + + pub fn init(allocator: std.mem.Allocator, sequence: []const Kind) !Normalizer { + if (sequence.len == 0) return error.EmptySequence; + + if (sequence.len == 1) { + // IREE requires a sequence of at least two normalizer. + const norm = try createNormalizer(sequence[0]) orelse unreachable; + const sequence_size: usize = @intCast(c.iree_tokenizer_normalizer_state_size(norm)); + + return .{ + .sequence = norm, + .sequence_size = sequence_size, + }; + } + + var normalizers: std.ArrayList(*c.iree_tokenizer_normalizer_t) = try .initCapacity(allocator, sequence.len); + defer normalizers.deinit(allocator); + errdefer for (normalizers.items) |normalizer| c.iree_tokenizer_normalizer_free(normalizer); + + for (sequence) |normalizer| { + normalizers.appendAssumeCapacity(try createNormalizer(normalizer) orelse unreachable); + } + + var normalizer_sequence: ?*c.iree_tokenizer_normalizer_t = null; + // IREE takes ownership of all the normalizers. + try checkOk(c.iree_tokenizer_normalizer_sequence_allocate(normalizers.items.ptr, normalizers.items.len, c.iree_allocator_system(), &normalizer_sequence)); + errdefer c.iree_tokenizer_normalizer_free(normalizer_sequence); + normalizers.clearRetainingCapacity(); + + if (normalizer_sequence == null) return error.NormalizerAllocationFailed; + + const sequence_size: usize = @intCast(c.iree_tokenizer_normalizer_state_size(normalizer_sequence.?)); + if (sequence_size == 0) return error.NormalizerAllocationFailed; + + return .{ + .sequence = normalizer_sequence.?, + .sequence_size = sequence_size, + }; + } + + pub fn deinit(self: *Normalizer) void { + c.iree_tokenizer_normalizer_free(self.sequence); + } + + pub fn normalize(self: *Normalizer, allocator: std.mem.Allocator, text: []const u8) ![]const u8 { + const storage: []align(16) u8 = try allocator.alignedAlloc(u8, .@"16", self.sequence_size); + defer allocator.free(storage); + + var state: ?*c.iree_tokenizer_normalizer_state_t = null; + defer if (state) |ptr| c.iree_tokenizer_normalizer_state_deinitialize(ptr); + + try checkOk(c.iree_tokenizer_normalizer_state_initialize(self.sequence, storage.ptr, &state)); + if (state == null) return error.NormalizationFailed; + + var buffer: [2048]u8 = undefined; + var remaining = text; + + var writer: std.Io.Writer.Allocating = .init(allocator); + defer writer.deinit(); + + while (remaining.len > 0) { + var written: usize = 0; + var consumed: usize = 0; + + try checkOk(c.iree_tokenizer_normalizer_state_process( + state.?, + .{ .data = remaining.ptr, .size = remaining.len }, + .{ .data = &buffer, .size = buffer.len }, + c.IREE_TOKENIZER_NORMALIZER_FLAG_SEGMENT_END, + &consumed, + &written, + )); + + if (consumed == 0 and written == 0) return error.NormalizationFailed; + + remaining = remaining[consumed..]; + try writer.writer.writeAll(buffer[0..written]); + } + + var has_pending = true; + while (has_pending) { + var written: usize = 0; + + try checkOk(c.iree_tokenizer_normalizer_state_finalize(state.?, .{ .data = &buffer, .size = buffer.len }, &written)); + has_pending = c.iree_tokenizer_normalizer_state_has_pending(state.?); + + try writer.writer.writeAll(buffer[0..written]); + + if (written == 0 and has_pending) return error.NormalizationFailed; + } + + return writer.toOwnedSlice(); + } +}; + +test "normalizer from Hugging Face" { + const allocator = std.testing.allocator; + + const json = + \\{ + \\ "type": "Sequence", + \\ "normalizers": [ + \\ {"type":"NFKD"}, + \\ {"type":"StripAccents"}, + \\ {"type":"Lowercase"}, + \\ {"type":"Replace","pattern":{"Regex":"[ \\t\\r\\n]+"},"content":" "}, + \\ {"type":"Replace","pattern":{"Regex":"^ $"},"content":"\ue000"}, + \\ {"type":"Strip","strip_left":true,"strip_right":true}, + \\ {"type":"Replace","pattern":{"String":"\ue000"},"content":" "} + \\ ] + \\} + ; + + var normalizer = try Normalizer.fromHuggingFaceJson(json); + defer normalizer.deinit(); + + const TestCase = struct { + input: []const u8, + expected: []const u8, + }; + + for ([_]TestCase{ + .{ .input = " hello", .expected = "hello" }, + .{ .input = "hello ", .expected = "hello" }, + .{ .input = " hello ", .expected = "hello" }, + .{ .input = "\nworld\r\n", .expected = "world" }, + .{ .input = "ÉCOLE", .expected = "ecole" }, + .{ .input = " CAFÉ\tAU\nLAIT ", .expected = "cafe au lait" }, + .{ .input = "\t \r\n", .expected = " " }, + .{ .input = "", .expected = "" }, + }) |case| { + const res = try normalizer.normalize(allocator, case.input); + defer allocator.free(res); + + try std.testing.expectEqualSlices(u8, case.expected, res); + } +} + +test "normalizer sequence" { + const allocator = std.testing.allocator; + + var normalizer = try Normalizer.init(allocator, &.{ + .{ .Strip = .both }, + .Lowercase, + }); + defer normalizer.deinit(); + + const res = try normalizer.normalize(allocator, " helLO WoRLD "); + defer allocator.free(res); + + try std.testing.expectEqualSlices(u8, "hello world", res); +} + +test "normalizer preserves whitespace across output and sequence boundaries" { + const allocator = std.testing.allocator; + for ([_][]const Normalizer.Kind{ + &.{.{ .Strip = .both }}, + &.{ .Lowercase, .{ .Strip = .both } }, + &.{ .Lowercase, .{ .Strip = .both }, .NFC }, + }) |steps| { + var normalizer = try Normalizer.init(allocator, steps); + defer normalizer.deinit(); + for ([_]usize{ 63, 64, 2047, 2048, 4096 }) |prefix_len| { + const input = try allocator.alloc(u8, prefix_len + 4); + defer allocator.free(input); + @memset(input[0..prefix_len], 'a'); + @memcpy(input[prefix_len..], " b "); + const output = try normalizer.normalize(allocator, input); + defer allocator.free(output); + try std.testing.expectEqualSlices(u8, input[0 .. prefix_len + 2], output); + } + } +} + +test "normalizer sequence preserves Unicode across tiles" { + const allocator = std.testing.allocator; + var normalizer = try Normalizer.fromHuggingFaceJson( + \\{"type":"Sequence","normalizers":[{"type":"Lowercase"},{"type":"Strip","strip_left":true,"strip_right":true}]} + ); + defer normalizer.deinit(); + for ([_][]const u8{ "é", "€", "😀" }) |codepoint| { + for (61..65) |prefix_len| { + const input = try allocator.alloc(u8, prefix_len + codepoint.len + 1); + defer allocator.free(input); + @memset(input[0..prefix_len], 'a'); + @memcpy(input[prefix_len..][0..codepoint.len], codepoint); + input[input.len - 1] = 'b'; + const output = try normalizer.normalize(allocator, input); + defer allocator.free(output); + try std.testing.expectEqualSlices(u8, input, output); + } + } +} + +test "normalizer sequence preserves composition across tiles" { + const allocator = std.testing.allocator; + var normalizer = try Normalizer.init(allocator, &.{ .NFC, .Lowercase }); + defer normalizer.deinit(); + const input = @as([63]u8, @splat('a')) ++ "e\u{301}b"; + const output = try normalizer.normalize(allocator, input); + defer allocator.free(output); + try std.testing.expectEqualSlices(u8, @as([63]u8, @splat('a')) ++ "éb", output); +} + +test "normalizer sequence allocation failure retains child ownership" { + const FailAllocator = struct { + fn control(_: ?*anyopaque, _: c.iree_allocator_command_t, _: ?*const anyopaque, _: [*c]?*anyopaque) callconv(.c) c.iree_status_t { + return @ptrFromInt(c.IREE_STATUS_RESOURCE_EXHAUSTED); + } + }; + // Include a nested sequence: failure must preserve its shell and children. + var nested = try Normalizer.init(std.testing.allocator, &.{ .NFC, .Lowercase }); + defer nested.deinit(); + var strip = try Normalizer.init(std.testing.allocator, &.{.{ .Strip = .both }}); + defer strip.deinit(); + const children = [_]*c.iree_tokenizer_normalizer_t{ nested.sequence, strip.sequence }; + var sequence: ?*c.iree_tokenizer_normalizer_t = null; + const status = c.iree_tokenizer_normalizer_sequence_allocate(&children, children.len, .{ + .self = null, + .ctl = FailAllocator.control, + }, &sequence); + defer c.iree_status_free(status); + try std.testing.expect(status != null); + try std.testing.expectEqual(@as(u32, c.IREE_STATUS_RESOURCE_EXHAUSTED), statusCode(status)); + try std.testing.expect(sequence == null); + const output = try nested.normalize(std.testing.allocator, "HELLO"); + defer std.testing.allocator.free(output); + try std.testing.expectEqualStrings("hello", output); +} + +fn tokenOutputWriter(allocator: std.mem.Allocator) std.Io.Writer.Allocating { + return std.Io.Writer.Allocating.initAligned(allocator, .of(u32)); +} + +fn writtenTokenIds(out: *std.Io.Writer.Allocating) []const u32 { + const bytes = out.written(); + const aligned_bytes: []align(4) u8 = @alignCast(bytes[0..bytes.len]); + return std.mem.bytesAsSlice(u32, aligned_bytes); +} + +fn decodedTextWriter(allocator: std.mem.Allocator) std.Io.Writer.Allocating { + return std.Io.Writer.Allocating.init(allocator); +} + +test "huggingface json encode/decode" { + const allocator = std.testing.allocator; + const json = + \\{ + \\ "model": { + \\ "type": "WordPiece", + \\ "unk_token": "[UNK]", + \\ "continuing_subword_prefix": "##", + \\ "max_input_chars_per_word": 100, + \\ "vocab": { + \\ "[UNK]": 0, + \\ "hello": 1, + \\ "world": 2 + \\ } + \\ }, + \\ "pre_tokenizer": {"type": "Whitespace"}, + \\ "decoder": {"type": "WordPiece", "prefix": "##", "cleanup": false} + \\} + ; + + var tokenizer = try Tokenizer.fromHuggingFaceJson(allocator, json); + defer tokenizer.deinit(); + + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + const text = "hello world"; + + const token_ids = try encoder.encodeAlloc(allocator, text); + defer allocator.free(token_ids); + try std.testing.expectEqualSlices(u32, &.{ 1, 2 }, token_ids); + + const token_hello = tokenizer.tokenId("hello") orelse unreachable; + try std.testing.expectEqual(@as(u32, 1), token_hello); + + const unknown = tokenizer.tokenId("missing"); + try std.testing.expect(unknown == null); + + var decoder = try tokenizer.decoder(); + defer decoder.deinit(); + + var decoded = decodedTextWriter(allocator); + defer decoded.deinit(); + try decoder.decode(token_ids, &decoded.writer); + try std.testing.expectEqualStrings("hello world", decoded.written()); +} + +test "loads bpe tokenizer with newline lookahead split and added unk" { + const allocator = std.testing.allocator; + const json = + \\{ + \\ "model": { + \\ "type": "BPE", + \\ "unk_token": "[UNK]", + \\ "vocab": { + \\ "<|UNK|>": 0, + \\ "h": 1, + \\ "e": 2, + \\ "l": 3, + \\ "o": 4, + \\ "he": 5, + \\ "ll": 6, + \\ "lo": 7, + \\ "hel": 8, + \\ "hello": 9 + \\ }, + \\ "merges": ["h e", "l l", "l o", "he l", "hel lo"] + \\ }, + \\ "added_tokens": [ + \\ { + \\ "id": 0, + \\ "content": "<|UNK|>", + \\ "single_word": false, + \\ "lstrip": false, + \\ "rstrip": false, + \\ "normalized": false, + \\ "special": true + \\ } + \\ ], + \\ "pre_tokenizer": { + \\ "type": "Sequence", + \\ "pretokenizers": [ + \\ { + \\ "type": "Split", + \\ "pattern": {"Regex": "(?:\\r?\\n)+(?!\\r?\\n)"}, + \\ "behavior": "MergedWithNext", + \\ "invert": false + \\ }, + \\ {"type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": true} + \\ ] + \\ }, + \\ "decoder": {"type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true} + \\} + ; + + var tokenizer = try Tokenizer.fromBytes(allocator, json); + defer tokenizer.deinit(); + + try std.testing.expectEqual(@as(u32, 0), tokenizer.tokenId("<|UNK|>").?); + + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + const ids = try encoder.encodeAlloc(allocator, "hello"); + defer allocator.free(ids); + try std.testing.expectEqualSlices(u32, &.{ 5, 6, 4 }, ids); +} + +const test_mistral_tokenizer_json = + \\{ + \\ "model": { + \\ "type": "BPE", + \\ "vocab": { + \\ "1": 0, "2": 1, "3": 2, "4": 3, "5": 4, "6": 5, "7": 6, + \\ "12": 7, "23": 8, "34": 9, "45": 10, "56": 11, "67": 12, + \\ "123": 13, "234": 14, "345": 15, "456": 16, "567": 17, + \\ "1234": 18 + \\ }, + \\ "merges": ["1 2", "2 3", "3 4", "4 5", "5 6", "6 7", + \\ "12 3", "23 4", "34 5", "45 6", "56 7", "123 4"] + \\ }, + \\ "pre_tokenizer": { + \\ "type": "Sequence", + \\ "pretokenizers": [ + \\ { + \\ "type": "Split", + \\ "pattern": {"Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}{1,3}(?=(?:\\p{N}{3})*(?!\\p{N}))|\\p{N}{1,2}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"}, + \\ "behavior": "Isolated", + \\ "invert": false + \\ }, + \\ {"type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false} + \\ ] + \\ }, + \\ "decoder": {"type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true} + \\} +; + +test "Mistral right-aligned number lookahead" { + const allocator = std.testing.allocator; + var tokenizer = try Tokenizer.fromBytes(allocator, test_mistral_tokenizer_json); + defer tokenizer.deinit(); + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + const cases = .{ + .{ "1", &[_]u32{0} }, + .{ "12", &[_]u32{7} }, + .{ "123", &[_]u32{13} }, + .{ "1234", &[_]u32{ 0, 14 } }, + .{ "12345", &[_]u32{ 7, 15 } }, + .{ "123456", &[_]u32{ 13, 16 } }, + .{ "1234567", &[_]u32{ 0, 14, 17 } }, + }; + inline for (cases) |case| { + encoder.reset(); + const ids = try encoder.encodeAlloc(allocator, case[0]); + defer allocator.free(ids); + try std.testing.expectEqualSlices(u32, case[1], ids); + + encoder.reset(); + var streamed: std.Io.Writer.Allocating = .initAligned(allocator, .of(u32)); + defer streamed.deinit(); + for (0..case[0].len) |i| { + try encoder.feed(case[0][i .. i + 1], &streamed.writer); + } + try encoder.finalize(&streamed.writer); + try std.testing.expectEqualSlices(u8, std.mem.sliceAsBytes(case[1]), streamed.written()); + } +} + +test "Mistral Unicode number grouping and output overflow" { + const allocator = std.testing.allocator; + var parsed = try std.json.parseFromSlice(std.json.Value, allocator, test_mistral_tokenizer_json, .{}); + defer parsed.deinit(); + var model = try std.json.parseFromSlice(std.json.Value, allocator, + \\{"type": "WordPiece", "unk_token": "", "continuing_subword_prefix": "", "vocab": { + \\ "": 0, "١": 1, "٢٣٤": 2, "12": 3, "345": 4, + \\ "Ⅷ": 5, "²3४": 6, "1": 7, "111": 8 + \\}} + , .{}); + defer model.deinit(); + const arena = parsed.arena.allocator(); + try parsed.value.object.put(arena, "model", model.value); + // Test the Split segmenter directly, before ByteLevel maps UTF-8 bytes. + const split = parsed.value.object.get("pre_tokenizer").?.object.get("pretokenizers").?.array.items[0]; + try parsed.value.object.put(arena, "pre_tokenizer", split); + const json = try std.json.Stringify.valueAlloc(allocator, parsed.value, .{}); + defer allocator.free(json); + + var tokenizer = try Tokenizer.fromBytes(allocator, json); + defer tokenizer.deinit(); + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + const cases = .{ + .{ "١٢٣٤", &[_]u32{ 1, 2 } }, + .{ "12345", &[_]u32{ 3, 4 } }, + .{ "Ⅷ²3४", &[_]u32{ 5, 6 } }, + }; + inline for (cases) |case| { + encoder.reset(); + var streamed: std.Io.Writer.Allocating = .initAligned(allocator, .of(u32)); + defer streamed.deinit(); + for (0..case[0].len) |i| { + try encoder.feed(case[0][i .. i + 1], &streamed.writer); + } + try encoder.finalize(&streamed.writer); + try std.testing.expectEqualSlices(u8, std.mem.sliceAsBytes(case[1]), streamed.written()); + } + + encoder.reset(); + const ids = try encoder.encodeAlloc(allocator, &(@as([1000]u8, @splat('1')))); + defer allocator.free(ids); + try std.testing.expectEqual(@as(usize, 334), ids.len); + try std.testing.expectEqual(@as(u32, 7), ids[0]); + for (ids[1..]) |id| try std.testing.expectEqual(@as(u32, 8), id); +} + +test "writer" { + const allocator = std.testing.allocator; + const json = + \\{ + \\ "model": { + \\ "type": "WordPiece", + \\ "unk_token": "[UNK]", + \\ "continuing_subword_prefix": "##", + \\ "max_input_chars_per_word": 100, + \\ "vocab": { + \\ "[UNK]": 0, + \\ "hello": 1, + \\ "world": 2 + \\ } + \\ }, + \\ "pre_tokenizer": {"type": "Whitespace"}, + \\ "decoder": {"type": "WordPiece", "prefix": "##", "cleanup": false} + \\} + ; + + var tokenizer = try Tokenizer.fromBytes(allocator, json); + defer tokenizer.deinit(); + + const text = "hello world"; + + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + const token_ids_alloc = try encoder.encodeAlloc(allocator, text); + defer allocator.free(token_ids_alloc); + try std.testing.expectEqualSlices(u32, &.{ 1, 2 }, token_ids_alloc); + + encoder.reset(); + var buf: [1024]u8 = undefined; + var out = tokenOutputWriter(allocator); + defer out.deinit(); + var writer = encoder.writer(&buf, &out.writer); + try writer.interface.writeAll(text); + try writer.finalize(); + const token_ids = writtenTokenIds(&out); + + try std.testing.expectEqualSlices(u32, &.{ 1, 2 }, token_ids); + + var decoder = try tokenizer.decoder(); + defer decoder.deinit(); + + var decoded = decodedTextWriter(allocator); + defer decoded.deinit(); + try decoder.decode(token_ids, &decoded.writer); + try std.testing.expectEqualStrings("hello world", decoded.written()); +} + +/// Shared tokenizer JSON config used by multiple tests. +const test_tokenizer_json = + \\{ + \\ "model": { + \\ "type": "WordPiece", + \\ "unk_token": "[UNK]", + \\ "continuing_subword_prefix": "##", + \\ "max_input_chars_per_word": 100, + \\ "vocab": { + \\ "[UNK]": 0, + \\ "hello": 1, + \\ "world": 2, + \\ "foo": 3, + \\ "bar": 4, + \\ "a": 5, + \\ "b": 6, + \\ "c": 7 + \\ } + \\ }, + \\ "pre_tokenizer": {"type": "Whitespace"}, + \\ "decoder": {"type": "WordPiece", "prefix": "##", "cleanup": false} + \\} +; + +test "writer with tiny buffer forces drain" { + // A buffer smaller than the input forces the drain path (not just flush). + const allocator = std.testing.allocator; + + var tokenizer = try Tokenizer.fromBytes(allocator, test_tokenizer_json); + defer tokenizer.deinit(); + + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + // Buffer of 4 bytes; "hello world" is 11 bytes, so drain must be invoked. + var buf: [4]u8 = undefined; + var out = tokenOutputWriter(allocator); + defer out.deinit(); + var writer = encoder.writer(&buf, &out.writer); + try writer.interface.writeAll("hello world"); + try writer.finalize(); + + try std.testing.expectEqualSlices(u32, &.{ 1, 2 }, writtenTokenIds(&out)); +} + +test "writer byte-by-byte" { + // Feed one byte at a time through the writer to stress buffer/drain boundaries. + const allocator = std.testing.allocator; + + var tokenizer = try Tokenizer.fromBytes(allocator, test_tokenizer_json); + defer tokenizer.deinit(); + + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + var buf: [1]u8 = undefined; + var out = tokenOutputWriter(allocator); + defer out.deinit(); + var writer = encoder.writer(&buf, &out.writer); + for ("hello world") |byte| { + try writer.interface.writeByte(byte); + } + try writer.finalize(); + + try std.testing.expectEqualSlices(u32, &.{ 1, 2 }, writtenTokenIds(&out)); +} + +test "encoder reset between multiple encodes" { + // Ensure reset properly clears state so successive encodes are independent. + const allocator = std.testing.allocator; + + var tokenizer = try Tokenizer.fromBytes(allocator, test_tokenizer_json); + defer tokenizer.deinit(); + + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + { + const ids = try encoder.encodeAlloc(allocator, "foo bar"); + defer allocator.free(ids); + try std.testing.expectEqualSlices(u32, &.{ 3, 4 }, ids); + } + + encoder.reset(); + + { + const ids = try encoder.encodeAlloc(allocator, "hello world"); + defer allocator.free(ids); + try std.testing.expectEqualSlices(u32, &.{ 1, 2 }, ids); + } + + encoder.reset(); + + // Third encode to verify repeated resets work. + { + const ids = try encoder.encodeAlloc(allocator, "foo"); + defer allocator.free(ids); + try std.testing.expectEqualSlices(u32, &.{3}, ids); + } +} + +test "writer and encodeAlloc produce same results" { + const allocator = std.testing.allocator; + + var tokenizer = try Tokenizer.fromBytes(allocator, test_tokenizer_json); + defer tokenizer.deinit(); + + const inputs = [_][]const u8{ + "hello", + "hello world", + "foo bar", + "a b c", + "foo bar hello world a b c", + }; + + for (inputs) |text| { + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + const expected = try encoder.encodeAlloc(allocator, text); + defer allocator.free(expected); + + encoder.reset(); + + var buf: [8]u8 = undefined; + var out = tokenOutputWriter(allocator); + defer out.deinit(); + var writer = encoder.writer(&buf, &out.writer); + try writer.interface.writeAll(text); + try writer.finalize(); + + try std.testing.expectEqualSlices(u32, expected, writtenTokenIds(&out)); + } +} + +test "encode empty string" { + const allocator = std.testing.allocator; + + var tokenizer = try Tokenizer.fromBytes(allocator, test_tokenizer_json); + defer tokenizer.deinit(); + + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + // encodeAlloc with empty string should produce no tokens. + { + const ids = try encoder.encodeAlloc(allocator, ""); + defer allocator.free(ids); + try std.testing.expectEqual(@as(usize, 0), ids.len); + } + + encoder.reset(); + + // Writer with empty string should also produce no tokens. + { + var buf: [16]u8 = undefined; + var out = tokenOutputWriter(allocator); + defer out.deinit(); + var writer = encoder.writer(&buf, &out.writer); + try writer.interface.writeAll(""); + try writer.finalize(); + try std.testing.expectEqual(@as(usize, 0), writtenTokenIds(&out).len); + } +} + +test "unknown tokens" { + // Words not in the vocabulary should map to the [UNK] token (id 0). + const allocator = std.testing.allocator; + + var tokenizer = try Tokenizer.fromBytes(allocator, test_tokenizer_json); + defer tokenizer.deinit(); + + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + const ids = try encoder.encodeAlloc(allocator, "unknown"); + defer allocator.free(ids); + try std.testing.expectEqualSlices(u32, &.{0}, ids); +} + +test "decode round-trip" { + // Encode then decode several strings and verify round-trip fidelity. + const allocator = std.testing.allocator; + + var tokenizer = try Tokenizer.fromBytes(allocator, test_tokenizer_json); + defer tokenizer.deinit(); + + const inputs = [_][]const u8{ + "hello", + "hello world", + "foo bar", + "a b c", + }; + + for (inputs) |text| { + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + const ids = try encoder.encodeAlloc(allocator, text); + defer allocator.free(ids); + + var decoder = try tokenizer.decoder(); + defer decoder.deinit(); + + var decoded = decodedTextWriter(allocator); + defer decoded.deinit(); + try decoder.decode(ids, &decoded.writer); + + try std.testing.expectEqualStrings(text, decoded.written()); + } +} + +test "decoder reset between decodes" { + const allocator = std.testing.allocator; + + var tokenizer = try Tokenizer.fromBytes(allocator, test_tokenizer_json); + defer tokenizer.deinit(); + + var decoder = try tokenizer.decoder(); + defer decoder.deinit(); + + { + var text = decodedTextWriter(allocator); + defer text.deinit(); + try decoder.decode(&.{ 1, 2 }, &text.writer); + try std.testing.expectEqualStrings("hello world", text.written()); + } + + try decoder.reset(); + + { + var text = decodedTextWriter(allocator); + defer text.deinit(); + try decoder.decode(&.{ 3, 4 }, &text.writer); + try std.testing.expectEqualStrings("foo bar", text.written()); + } +} + +test "tokenId lookup" { + const allocator = std.testing.allocator; + + var tokenizer = try Tokenizer.fromBytes(allocator, test_tokenizer_json); + defer tokenizer.deinit(); + + try std.testing.expectEqual(@as(u32, 0), tokenizer.tokenId("[UNK]").?); + try std.testing.expectEqual(@as(u32, 1), tokenizer.tokenId("hello").?); + try std.testing.expectEqual(@as(u32, 2), tokenizer.tokenId("world").?); + try std.testing.expectEqual(@as(u32, 3), tokenizer.tokenId("foo").?); + try std.testing.expectEqual(@as(u32, 4), tokenizer.tokenId("bar").?); + try std.testing.expect(tokenizer.tokenId("nonexistent") == null); + try std.testing.expect(tokenizer.tokenId("") == null); +} + +test "tokenId <-> decode" { + const allocator = std.testing.allocator; + + var tokenizer = try Tokenizer.fromBytes(allocator, test_tokenizer_json); + defer tokenizer.deinit(); + + for ([_][]const u8{ "[UNK]", "hello", "world", "foo", "bar" }) |text| { + try std.testing.expectEqualSlices(u8, text, tokenizer.decode(tokenizer.tokenId(text).?).?); + } +} + +test "writer with pre-allocated output writer" { + // Pass an already-allocated aligned output writer to the encoder stream. + const allocator = std.testing.allocator; + + var tokenizer = try Tokenizer.fromBytes(allocator, test_tokenizer_json); + defer tokenizer.deinit(); + + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + var out = tokenOutputWriter(allocator); + defer out.deinit(); + try out.ensureUnusedCapacity(64 * @sizeOf(u32)); + + var buf: [32]u8 = undefined; + var writer = encoder.writer(&buf, &out.writer); + try writer.interface.writeAll("hello world"); + try writer.finalize(); + + try std.testing.expectEqualSlices(u32, &.{ 1, 2 }, writtenTokenIds(&out)); +} diff --git a/zml/tokenizer/main.zig b/zml/tokenizer/main.zig new file mode 100644 index 00000000..14eb1a87 --- /dev/null +++ b/zml/tokenizer/main.zig @@ -0,0 +1,67 @@ +const std = @import("std"); + +const stdx = @import("stdx"); +const zml_tokenizer = @import("zml/tokenizer"); + +const log = std.log.scoped(.@"//zml/tokenizer"); + +const Flags = struct { + tokenizer: []const u8, + prompt: []const u8, + expected: []const u8 = "", + verbose: bool = false, +}; + +pub fn main(init: std.process.Init) !void { + const allocator = init.gpa; + + var threaded: std.Io.Threaded = .init(allocator, .{}); + defer threaded.deinit(); + + const io = threaded.io(); + + const args = stdx.flags.parseProcessArgs(init.minimal, Flags); + + log.info("\tLoading tokenizer from {s}", .{args.tokenizer}); + var tokenizer = try zml_tokenizer.Tokenizer.fromFile(allocator, io, args.tokenizer); + log.info("✅\tLoaded tokenizer from {s}", .{args.tokenizer}); + defer tokenizer.deinit(); + + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + var decoder = try tokenizer.decoder(); + defer decoder.deinit(); + + const prompt_tok = try encoder.encodeAlloc(allocator, args.prompt); + defer allocator.free(prompt_tok); + + log.info("Input: {s}\nOutput: {any}", .{ args.prompt, prompt_tok }); + + var errors: u8 = 0; + { + var reconstructed = try decoder.decodeAlloc(allocator, prompt_tok); + defer reconstructed.deinit(allocator); + if (!std.mem.eql(u8, args.prompt, reconstructed.items)) { + log.err("Reconstructed string from tokens doesn't match source: {s}", .{reconstructed.items}); + errors += 1; + } + } + + if (args.expected.len > 0) { + var expected: std.ArrayList(u32) = try .initCapacity(allocator, args.prompt.len); + var it = std.mem.splitSequence(u8, args.expected, ","); + while (it.next()) |int_token| { + const tok = try std.fmt.parseInt(u32, int_token, 10); + try expected.append(allocator, tok); + } + if (!std.mem.eql(u32, expected.items, prompt_tok)) { + log.err("Doesn't match expected: {any}", .{expected.items}); + errors += 1; + } + } + + if (errors == 0) log.info("All good !", .{}); + + std.process.exit(errors); +} diff --git a/zml/tokenizer/sentencepiece/BUILD.bazel b/zml/tokenizer/sentencepiece/BUILD.bazel new file mode 100644 index 00000000..dd4d2e76 --- /dev/null +++ b/zml/tokenizer/sentencepiece/BUILD.bazel @@ -0,0 +1,34 @@ +load("//bazel:swig.bzl", "swig_cc_library") +load("//bazel:zig_srcs.bzl", "zig_srcs") +load("@rules_cc//cc:cc_static_library.bzl", "cc_static_library") +load("@rules_zig//zig:defs.bzl", "zig_library") + +swig_cc_library( + name = "sentencepiece_swig", + tags = ["manual"], + interface = "sentencepiece.i", + module = "sentencepiece", + deps = [ + "//ffi:cc", + "@com_google_sentencepiece//:sentencepiece_processor", + ], +) + +zig_library( + name = "sentencepiece", + tags = ["manual"], + import_name = "sentencepiece", + main = "sentencepiece.zig", + visibility = ["//visibility:public"], + deps = [ + ":sentencepiece_swig", + "//ffi:zig", + "//stdx", + ], +) + +# todo: reenable after fix +# zig_srcs( +# name = "sources", +# zig_lib = ":sentencepiece", +# ) diff --git a/zml/tokenizer/sentencepiece/sentencepiece.i b/zml/tokenizer/sentencepiece/sentencepiece.i new file mode 100644 index 00000000..d89489c7 --- /dev/null +++ b/zml/tokenizer/sentencepiece/sentencepiece.i @@ -0,0 +1,95 @@ +/* File : example.i */ +%module sentencepiece +%include +%include + +%{ +#include +#include +#include +#include +#include "ffi/zig_slice.h" +%} + +%insert("cheader") %{ +#include "ffi/zig_slice.h" +%} + +%typemap(in) absl::string_view { + $1 = absl::string_view((char *)$input.ptr, $input.len); +} + +%typemap(out) const std::string& %{ + $result.ptr = (void *)($1->data()); + $result.len = (size_t)($1->length()); +%} +%typemap(ctype) absl::string_view, const std::string& "zig_slice" + +%rename(std_string) std::string; +namespace std { + class string { + public: + void reserve(size_t n); + void clear(); + size_t capacity() const; + }; +} + +%extend std::string { + const std::string& data() const { + return *$self; + }; +} + +%extend std::vector { + T* data() { + return $self->data(); + }; +} + +%template(std_vector_int) std::vector; + +%typemap(out) sentencepiece::util::Status %{ + $result = $1.code(); +%} +%typemap(ctype) sentencepiece::util::Status "unsigned int" +%rename(sentencepiece_util_StatusCode, fullname=1) sentencepiece::util::StatusCode; + +namespace sentencepiece { + namespace util { + enum class StatusCode : int { + kOk = 0, + kCancelled = 1, + kUnknown = 2, + kInvalidArgument = 3, + kDeadlineExceeded = 4, + kNotFound = 5, + kAlreadyExists = 6, + kPermissionDenied = 7, + kResourceExhausted = 8, + kFailedPrecondition = 9, + kAborted = 10, + kOutOfRange = 11, + kUnimplemented = 12, + kInternal = 13, + kUnavailable = 14, + kDataLoss = 15, + kUnauthenticated = 16, + }; + } + + class SentencePieceProcessor { + public: + virtual sentencepiece::util::Status Load(absl::string_view filename); + virtual sentencepiece::util::Status LoadFromSerializedProto(absl::string_view serialized); + virtual sentencepiece::util::Status Encode(absl::string_view input, std::vector *ids) const; + virtual sentencepiece::util::Status Decode(const std::vector &ids, std::string *detokenized) const; + virtual int PieceToId(absl::string_view piece) const; + virtual const std::string &IdToPiece(int id) const; + virtual int GetPieceSize() const; + virtual int unk_id() const; + virtual int bos_id() const; + virtual int eos_id() const; + virtual int pad_id() const; + }; +} diff --git a/zml/tokenizer/sentencepiece/sentencepiece.zig b/zml/tokenizer/sentencepiece/sentencepiece.zig new file mode 100644 index 00000000..a09e0c0a --- /dev/null +++ b/zml/tokenizer/sentencepiece/sentencepiece.zig @@ -0,0 +1,203 @@ +const std = @import("std"); + +const c = @import("c"); +const ffi = @import("ffi"); +const stdx = @import("stdx"); + +const StringToTokenRatio = 3; + +pub const Error = error{ + Cancelled, + Unknown, + InvalidArgument, + DeadlineExceeded, + NotFound, + AlreadyExists, + PermissionDenied, + ResourceExhausted, + FailedPrecondition, + Aborted, + OutOfRange, + Unimplemented, + Internal, + Unavailable, + DataLoss, + Unauthenticated, +}; + +fn assertOk(code: c.sentencepiece_util_StatusCode) Error!void { + return switch (code) { + c.sentencepiece_util_StatusCode_kOk => {}, + c.sentencepiece_util_StatusCode_kCancelled => Error.Cancelled, + c.sentencepiece_util_StatusCode_kUnknown => Error.Unknown, + c.sentencepiece_util_StatusCode_kInvalidArgument => Error.InvalidArgument, + c.sentencepiece_util_StatusCode_kDeadlineExceeded => Error.DeadlineExceeded, + c.sentencepiece_util_StatusCode_kNotFound => Error.NotFound, + c.sentencepiece_util_StatusCode_kAlreadyExists => Error.AlreadyExists, + c.sentencepiece_util_StatusCode_kPermissionDenied => Error.PermissionDenied, + c.sentencepiece_util_StatusCode_kResourceExhausted => Error.ResourceExhausted, + c.sentencepiece_util_StatusCode_kFailedPrecondition => Error.FailedPrecondition, + c.sentencepiece_util_StatusCode_kAborted => Error.Aborted, + c.sentencepiece_util_StatusCode_kOutOfRange => Error.OutOfRange, + c.sentencepiece_util_StatusCode_kUnimplemented => Error.Unimplemented, + c.sentencepiece_util_StatusCode_kInternal => Error.Internal, + c.sentencepiece_util_StatusCode_kUnavailable => Error.Unavailable, + c.sentencepiece_util_StatusCode_kDataLoss => Error.DataLoss, + c.sentencepiece_util_StatusCode_kUnauthenticated => Error.Unauthenticated, + else => unreachable, + }; +} + +pub const Encoder = struct { + inner: *SentencePieceProcessor, + vec: *c.std_vector_int, + + fn init(inner: *SentencePieceProcessor) Encoder { + return .{ + .inner = inner, + .vec = c.std_vector_int_new() orelse unreachable, + }; + } + + pub fn deinit(self: *Encoder) void { + c.std_vector_int_delete(self.vec); + } + + pub fn reset(self: *Encoder) void { + c.std_vector_int_clear(self.vec); + } + + pub fn encode(self: *Encoder, input: []const u8) ![]const u32 { + c.std_vector_int_reserve(self.vec, input.len / StringToTokenRatio); + try assertOk(c.SentencePieceProcessor_Encode(@ptrCast(self.inner), ffi.ZigSlice.from(input), self.vec)); + return self.ids(); + } + + pub fn ids(self: *const Encoder) []const u32 { + return ffi.ZigSlice.to(u32, .{ + .ptr = c.std_vector_int_data(self.vec), + .len = c.std_vector_int_size(self.vec), + }); + } +}; + +pub const Decoder = struct { + const StringBufferSize = 64; + const StringBuffer = stdx.BoundedArray(u8, StringBufferSize); + const TokenIdsBufferSize = 4; + + inner: *SentencePieceProcessor, + vec: *c.std_vector_int, + str: *c.std_string, + last_string: StringBuffer = .empty, + + fn init(inner: *SentencePieceProcessor) !Decoder { + const vec = try (c.std_vector_int_new() orelse std.mem.Allocator.Error.OutOfMemory); + c.std_vector_int_reserve(vec, TokenIdsBufferSize); + errdefer c.std_vector_int_delete(vec); + + const str = try (c.std_string_new() orelse std.mem.Allocator.Error.OutOfMemory); + c.std_string_reserve(str, StringBufferSize); + errdefer c.std_string_delete(str); + + return .{ + .inner = inner, + .vec = vec, + .str = str, + }; + } + + pub fn deinit(self: *Decoder) void { + c.std_vector_int_delete(self.vec); + c.std_string_delete(self.str); + } + + pub fn reset(self: *Decoder) void { + c.std_vector_int_clear(self.vec); + c.std_string_clear(self.str); + } + + pub fn decode(self: *Decoder, ids_: []const u32) ![]const u8 { + c.std_vector_int_reserve(self.vec, ids_.len); + c.std_string_reserve(self.str, ids_.len * StringToTokenRatio); + for (ids_) |id| { + c.std_vector_int_push_back(self.vec, @intCast(id)); + } + try assertOk(c.SentencePieceProcessor_Decode(@ptrCast(self.inner), self.vec, self.str)); + return self.string(); + } + + pub fn string(self: *const Decoder) []const u8 { + const res = c.std_string_data(self.str); + return ffi.ZigSlice.to(u8, res); + } + + fn ids(self: *const Decoder) []u32 { + const ptr: [*c]u32 = @ptrCast(c.std_vector_int_data(self.vec)); + return ptr[0..c.std_vector_int_size(self.vec)]; + } + + pub fn next(self: *Decoder, token_id: u32) !?[]const u8 { + const current_ids = self.ids(); + if (current_ids.len >= c.std_vector_int_capacity(self.vec)) { + std.mem.copyForwards(u32, current_ids[0 .. current_ids.len - 1], current_ids[1..]); + current_ids[current_ids.len - 1] = token_id; + } else { + c.std_vector_int_push_back(self.vec, @intCast(token_id)); + } + try assertOk(c.SentencePieceProcessor_Decode(@ptrCast(self.inner), self.vec, self.str)); + const new_string = self.string(); + if (self.last_string.len == 0) { + self.last_string = try StringBuffer.fromSlice(new_string); + return new_string; + } + var view = try std.unicode.Utf8View.init(self.last_string.constSlice()); + var it = view.iterator(); + while (it.nextCodepointSlice()) |cp| { + const start = it.i - cp.len; + if (std.mem.startsWith(u8, new_string, self.last_string.constSlice()[start..])) { + const chunk = new_string[self.last_string.len - start ..]; + self.last_string = try StringBuffer.fromSlice(new_string); + return chunk; + } + } + return &.{}; + } +}; + +pub const SentencePieceProcessor = opaque { + pub fn fromFile(model: []const u8) !*SentencePieceProcessor { + const sp: *SentencePieceProcessor = @ptrCast(c.SentencePieceProcessor_new()); + errdefer sp.deinit(); + try assertOk(c.SentencePieceProcessor_Load(@ptrCast(sp), ffi.ZigSlice.from(model))); + return sp; + } + + pub fn fromBytes(bytes: []const u8) !*SentencePieceProcessor { + const sp: *SentencePieceProcessor = @ptrCast(c.SentencePieceProcessor_new()); + errdefer sp.deinit(); + try assertOk(c.SentencePieceProcessor_LoadFromSerializedProto(@ptrCast(sp), ffi.ZigSlice.from(bytes))); + return sp; + } + + pub fn deinit(self: *SentencePieceProcessor) void { + c.SentencePieceProcessor_delete(@ptrCast(self)); + } + + pub fn encoder(self: *SentencePieceProcessor) !Encoder { + return Encoder.init(self); + } + + pub fn decoder(self: *SentencePieceProcessor) !Decoder { + return try Decoder.init(self); + } + + pub fn tokenId(self: *SentencePieceProcessor, token: []const u8) u32 { + return @intCast(c.SentencePieceProcessor_PieceToId(@ptrCast(self), ffi.ZigSlice.from(token))); + } + + pub fn decode(self: *SentencePieceProcessor, token_id: u32) ?[]const u8 { + if (token_id >= @as(u32, @intCast(c.SentencePieceProcessor_GetPieceSize(@ptrCast(self))))) return null; + return ffi.ZigSlice.to(u8, c.SentencePieceProcessor_IdToPiece(@ptrCast(self), @intCast(token_id))); + } +}; diff --git a/zml/tokenizer/tokenizer.zig b/zml/tokenizer/tokenizer.zig new file mode 100644 index 00000000..fc304b81 --- /dev/null +++ b/zml/tokenizer/tokenizer.zig @@ -0,0 +1,189 @@ +const std = @import("std"); + +pub const iree = @import("iree"); +pub const sentencepiece = @import("sentencepiece"); + +const log = std.log.scoped(.@"zml/tokenizer"); + +const Tokenizers = enum { + iree, + sentencepiece, +}; + +pub const Tokenizer = union(Tokenizers) { + pub const Normalizer = union(Tokenizers) { + pub const Kind = iree.Normalizer.Kind; + + iree: iree.Normalizer, + sentencepiece: void, + + pub fn deinit(self: *Normalizer) void { + switch (self.*) { + .iree => |*n| n.deinit(), + else => {}, + } + } + + pub fn normalize(self: *Normalizer, allocator: std.mem.Allocator, text: []const u8) ![]const u8 { + return switch (self.*) { + .iree => |*n| n.normalize(allocator, text), + else => error.UnsupportedNormalizer, + }; + } + }; + + pub const Encoder = union(Tokenizers) { + iree: iree.Tokenizer.Encoder, + sentencepiece: sentencepiece.Encoder, + + pub fn deinit(self: *Encoder) void { + switch (self.*) { + inline else => |*v| v.deinit(), + } + } + + pub fn reset(self: *Encoder) void { + switch (self.*) { + inline else => |*v| v.reset(), + } + } + + pub fn encodeAlloc(self: *Encoder, allocator: std.mem.Allocator, text: []const u8) ![]u32 { + return switch (self.*) { + .iree => |*encoder_| encoder_.encodeAlloc(allocator, text), + inline else => |*encoder_| try allocator.dupe(u32, try encoder_.encode(text)), + }; + } + + pub fn encode(self: *Encoder, writer: *std.Io.Writer, text: []const u8) !void { + return switch (self.*) { + .iree => |*e| { + try e.feed(text, writer); + try e.finalize(writer); + }, + inline else => |*e| { + const encoded: []const u32 = try e.encode(text); + try writer.writeAll(std.mem.sliceAsBytes(encoded)); + }, + }; + } + + pub fn appendTokens(_: *Encoder, writer: *std.Io.Writer, tokens: []const u32) !void { + try writer.writeAll(@ptrCast(tokens)); + } + }; + + pub const Decoder = union(Tokenizers) { + iree: iree.Tokenizer.Decoder, + sentencepiece: sentencepiece.Decoder, + + pub fn deinit(self: *Decoder) void { + switch (self.*) { + inline else => |*v| v.deinit(), + } + } + + pub fn reset(self: *Decoder) void { + switch (self.*) { + inline else => |*v| v.reset(), + } + } + + pub fn decodeAlloc(self: *Decoder, allocator: std.mem.Allocator, token_ids: []const u32) !std.ArrayList(u8) { + return switch (self.*) { + .iree => |*d| { + var writer: std.Io.Writer.Allocating = .init(allocator); + errdefer writer.deinit(); + try d.decode(token_ids, &writer.writer); + return writer.toArrayList(); + }, + inline else => |*d| .fromOwnedSlice(try allocator.dupe(u8, try d.decode(token_ids))), + }; + } + + // Only valid until the next decode and while the decoder AND buffer is alive. + pub fn feedOne(self: *Decoder, token_id: u32, buffer: []u8) ![]const u8 { + return switch (self.*) { + .iree => |*d| d.feedOne(token_id, buffer), + inline else => |*d| try d.next(token_id) orelse &.{}, + }; + } + + /// Output is only valid until the next feed/finalize call and while the decoder AND buffer is alive. + pub fn finalize(self: *Decoder, buffer: []u8) ![]const u8 { + return switch (self.*) { + .iree => |*v| v.finalize(buffer), + inline else => &.{}, + }; + } + }; + + iree: iree.Tokenizer, + sentencepiece: *sentencepiece.SentencePieceProcessor, + + pub fn fromFile(allocator: std.mem.Allocator, io: std.Io, model: []const u8) !Tokenizer { + if (std.mem.endsWith(u8, model, ".pb")) { + return .{ .sentencepiece = try .fromFile(model) }; + } + if (std.mem.endsWith(u8, model, ".json")) { + return .{ .iree = try .fromFile(allocator, io, model) }; + } + + return error.InvalidArgument; + } + + pub fn fromBytes(allocator: std.mem.Allocator, bytes: []const u8) !Tokenizer { + if (bytes[0] == '{') { + return .{ .iree = try .fromBytes(allocator, bytes) }; + } + return .{ .sentencepiece = try .fromBytes(bytes) }; + } + + pub fn deinit(self: *Tokenizer) void { + switch (self.*) { + inline else => |*t| t.*.deinit(), + } + } + + pub fn encoder(self: *const Tokenizer) !Encoder { + return switch (self.*) { + inline else => |*v, tag| @unionInit(Encoder, @tagName(tag), try v.*.encoder()), + }; + } + + pub fn decoder(self: *const Tokenizer) !Decoder { + return switch (self.*) { + inline else => |*v, tag| @unionInit(Decoder, @tagName(tag), try v.*.decoder()), + }; + } + + pub fn normalizer(self: *const Tokenizer, sequence: []const Normalizer.Kind) !Normalizer { + return switch (self.*) { + .iree => |*backend| .{ .iree = try backend.normalizer(sequence) }, + else => error.UnsupportedNormalizer, + }; + } + + pub fn normalizerFromHuggingFaceJson(self: *const Tokenizer, json: []const u8) !Normalizer { + return switch (self.*) { + .iree => |*backend| .{ .iree = try backend.normalizerFromHuggingFaceJson(json) }, + else => error.UnsupportedNormalizer, + }; + } + + pub fn tokenId(self: *const Tokenizer, token: []const u8) ?u32 { + return switch (self.*) { + inline else => |v| v.tokenId(token), + }; + } + + pub fn decode(self: *const Tokenizer, token_id: u32) ?[]const u8 { + return switch (self.*) { + inline else => |v| v.decode(token_id), + }; + } +}; + +test { + std.testing.refAllDecls(@This()); +} diff --git a/zml/tools/BUILD.bazel b/zml/tools/BUILD.bazel deleted file mode 100644 index ab2396e1..00000000 --- a/zml/tools/BUILD.bazel +++ /dev/null @@ -1,21 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") - -cc_library( - name = "macos_c", - srcs = ["macos.c"], - hdrs = ["macos.h"], - visibility = ["//zml/tools:__subpackages__"], -) - -zig_library( - name = "tools", - srcs = ["tracer.zig"], - main = "tools.zig", - visibility = ["//visibility:public"], - deps = select({ - "@platforms//os:macos": [ - ":macos_c", - ], - "//conditions:default": [], - }), -) diff --git a/zml/tools/macos.c b/zml/tools/macos.c deleted file mode 100644 index e59dc638..00000000 --- a/zml/tools/macos.c +++ /dev/null @@ -1,26 +0,0 @@ -#include -#include - -void zml_os_signpost_event( - os_log_t log, - os_signpost_id_t signpost_id, - const char *message) -{ - os_signpost_event_emit(log, signpost_id, "zml", "%s", message); -} - -void zml_os_signpost_interval_begin( - os_log_t log, - os_signpost_id_t signpost_id, - const char *message) -{ - os_signpost_interval_begin(log, signpost_id, "zml", "%s", message); -} - -void zml_os_signpost_interval_end( - os_log_t log, - os_signpost_id_t signpost_id, - const char *message) -{ - os_signpost_interval_end(log, signpost_id, "zml", "%s", message); -} diff --git a/zml/tools/macos.h b/zml/tools/macos.h deleted file mode 100644 index 0d2f9b4f..00000000 --- a/zml/tools/macos.h +++ /dev/null @@ -1,17 +0,0 @@ -#include -#include - -void zml_os_signpost_event( - os_log_t log, - os_signpost_id_t signpost_id, - const char *message); - -void zml_os_signpost_interval_begin( - os_log_t log, - os_signpost_id_t signpost_id, - const char *message); - -void zml_os_signpost_interval_end( - os_log_t log, - os_signpost_id_t signpost_id, - const char *message); diff --git a/zml/tools/tools.zig b/zml/tools/tools.zig deleted file mode 100644 index 2bc03172..00000000 --- a/zml/tools/tools.zig +++ /dev/null @@ -1 +0,0 @@ -pub const Tracer = @import("tracer.zig").Tracer; diff --git a/zml/tools/tracer.zig b/zml/tools/tracer.zig deleted file mode 100644 index ed7d7fe1..00000000 --- a/zml/tools/tracer.zig +++ /dev/null @@ -1,61 +0,0 @@ -const builtin = @import("builtin"); - -pub const Tracer = switch (builtin.os.tag) { - .macos => MacOsTracer, - else => FakeTracer, -}; - -const MacOsTracer = struct { - const c = @import("c"); - - logger: c.os_log_t, - - pub fn init(name: [:0]const u8) MacOsTracer { - const logger = c.os_log_create(name.ptr, c.OS_LOG_CATEGORY_POINTS_OF_INTEREST); - return .{ - .logger = logger, - }; - } - - pub fn event(self: *const MacOsTracer, message: [:0]const u8) void { - const interval_id = c.os_signpost_id_generate(self.logger); - c.zml_os_signpost_event(self.logger, interval_id, message); - } - - pub fn frameStart(self: *const MacOsTracer, message: [:0]const u8) c.os_signpost_id_t { - const interval_id = c.os_signpost_id_generate(self.logger); - c.zml_os_signpost_interval_begin(self.logger, interval_id, message); - return interval_id; - } - - pub fn frameEnd(self: *const MacOsTracer, interval_id: c.os_signpost_id_t, message: [:0]const u8) void { - c.zml_os_signpost_interval_end(self.logger, interval_id, message); - } -}; - -/// Mock tracer for OS which don't have an impl. -const FakeTracer = struct { - pub fn init(name: [:0]const u8) FakeTracer { - _ = name; - return .{}; - } - - pub fn event(self: *const MacOsTracer, message: [:0]const u8) void { - _ = self; - _ = message; - return; - } - - pub fn frameStart(self: *const FakeTracer, message: [:0]const u8) u64 { - _ = self; - _ = message; - return 0; - } - - pub fn frameEnd(self: *const FakeTracer, interval_id: u64, message: [:0]const u8) void { - _ = self; - _ = interval_id; - _ = message; - return; - } -}; diff --git a/zml/torch.zig b/zml/torch.zig deleted file mode 100644 index c85cd230..00000000 --- a/zml/torch.zig +++ /dev/null @@ -1,278 +0,0 @@ -const std = @import("std"); -const log = std.log.scoped(.zml_torch); - -const zml = @import("zml.zig"); -const Tensor = zml.Tensor; -const meta = zml.meta; - -/// Multiplies a matrix or a vector with a tensor, -/// following the semantic of pytorch `@` operator. -/// When both sides are matrices, it's the textbook matrix multiplication : -/// `matmul(.{ 8, 9 }, .{ 9, 10 }) -> .{ 8, 10 }` -/// When one of the input is a tensor, it assumes the first dimensions are batches, -/// and the last two ones are used for the regular matmul. -/// * `matmul(.{10}, .{10}) -> .{}` -/// * `matmul(.{10}, .{10}) -> .{}` -pub fn matmul(lhs: Tensor, rhs: Tensor) Tensor { - meta.assert(lhs.rank() >= 1 and rhs.rank() >= 1, "Can't matmul({}, {}) ! The two tensors need to have at least rank 1.", .{ lhs, rhs }); - - const contracting = [_][2]i8{.{ -1, if (rhs.rank() >= 2) rhs.rank() - 2 else 0 }}; - if (lhs.rank() == 1 or rhs.rank() <= 2) { - // When lhs is a vector or rhs is small the torch semantics match the dot_general semantics and life is easy. - return lhs.dotGeneral(rhs, &contracting, &.{}); - } - - meta.assert(lhs.rank() == 2, "Can't matmul({}, {}) ! One of the two tensors need to have a rank less than 2.", .{ lhs, rhs }); - - // Pytorch treats the extra dimensions of rhs has batching dimensions, - // and implicitly broadcast lhs along those. - // We make this broadcasting explicit. - var left_shape = rhs.shape(); - left_shape._dims.set(left_shape.axis(-2), lhs.dim(-2)); - left_shape._tags.set(left_shape.axis(-2), lhs.shape().tag(-2)); - left_shape._dims.set(left_shape.axis(-1), lhs.dim(-1)); - left_shape._tags.set(left_shape.axis(-1), lhs.shape().tag(-1)); - const lhs_broad = lhs.broadcastLeft(left_shape); - - const n_batching_axes = rhs.rank() - lhs.rank(); - var batching: [Tensor.MAX_RANK][2]i8 = undefined; - for (0..n_batching_axes) |i| { - batching[i] = .{ @intCast(i), @intCast(i) }; - } - return lhs_broad.dotGeneral(rhs, &contracting, batching[0..n_batching_axes]); -} - -test matmul { - const platform = zml.testing.env(); - - var comp = try zml.module.CompilationContext.init(std.heap.page_allocator, "test", platform); - defer comp.deinit(); - - comp.activate(); - defer comp.deactivate(); - - // Generated with pytorch - inline for (.{ - .{ .{20}, .{20}, .{} }, - .{ .{20}, .{ 20, 15 }, .{15} }, - .{ .{20}, .{ 11, 20, 15 }, .{ 11, 15 } }, - .{ .{20}, .{ 9, 11, 20, 15 }, .{ 9, 11, 15 } }, - .{ .{20}, .{ 7, 9, 11, 20, 15 }, .{ 7, 9, 11, 15 } }, - .{ .{20}, .{ 5, 7, 9, 11, 20, 15 }, .{ 5, 7, 9, 11, 15 } }, - .{ .{ 12, 20 }, .{20}, .{12} }, - .{ .{ 12, 20 }, .{ 20, 15 }, .{ 12, 15 } }, - .{ .{ 12, 20 }, .{ 11, 20, 15 }, .{ 11, 12, 15 } }, - .{ .{ 12, 20 }, .{ 9, 11, 20, 15 }, .{ 9, 11, 12, 15 } }, - .{ .{ 12, 20 }, .{ 7, 9, 11, 20, 15 }, .{ 7, 9, 11, 12, 15 } }, - .{ .{ 12, 20 }, .{ 5, 7, 9, 11, 20, 15 }, .{ 5, 7, 9, 11, 12, 15 } }, - .{ .{ 10, 12, 20 }, .{20}, .{ 10, 12 } }, - .{ .{ 10, 12, 20 }, .{ 20, 15 }, .{ 10, 12, 15 } }, - .{ .{ 8, 10, 12, 20 }, .{20}, .{ 8, 10, 12 } }, - .{ .{ 8, 10, 12, 20 }, .{ 20, 15 }, .{ 8, 10, 12, 15 } }, - .{ .{ 6, 8, 10, 12, 20 }, .{20}, .{ 6, 8, 10, 12 } }, - .{ .{ 6, 8, 10, 12, 20 }, .{ 20, 15 }, .{ 6, 8, 10, 12, 15 } }, - .{ .{ 4, 6, 8, 10, 12, 20 }, .{20}, .{ 4, 6, 8, 10, 12 } }, - .{ .{ 4, 6, 8, 10, 12, 20 }, .{ 20, 15 }, .{ 4, 6, 8, 10, 12, 15 } }, - }) |testcase| { - const x_shape, const y_shape, const z_shape = testcase; - const x = Tensor.constant(x_shape, .{ .f32 = 0.0 }); - const y = Tensor.constant(y_shape, .{ .f32 = 0.0 }); - const z = matmul(x, y); - - try std.testing.expectEqualSlices(i64, &z_shape, z.dims()); - } -} - -/// Inserts a 1-dim axis at the given position. -/// Negative indexes are handled like pytorch, ie they are relative to the returned shaped: -/// - `.{5, 4}.unsqueeze(1)` returns `.{5, 1, 4}` -/// - `.{5, 4}.unsqueeze(-1)` returns `.{5, 4, 1}` -pub fn unsqueeze( - self: Tensor, - axis_: anytype, -) Tensor { - meta.assert(self.rank() < Tensor.MAX_RANK - 1, "Can't unsqueeze {}, it's already at max rank.", .{self}); - const a = switch (@typeInfo(@TypeOf(axis_))) { - .Int, .ComptimeInt => if (axis_ < 0) - @as(i8, self.rank()) + 1 + axis_ - else - self.axis(axis_), - else => self.axis(axis_), - }; - return self.insertAxes(a, .{._}); -} - -test unsqueeze { - const UnsqueezeTest = struct { - pub fn forward(x: Tensor) Tensor { - var y = x; - y = unsqueeze(y, 0); - y = unsqueeze(y, -1); - y = unsqueeze(y, -1); - return y; - } - }; - const platform = zml.testing.env(); - - const x = try zml.Buffer.fromArray(platform, @as([8]f16, undefined)); - const res = try zml.testing.compileAndCall(platform, UnsqueezeTest.forward, .{x}); - try zml.testing.expectEqualShapes(zml.Shape.init(.{ 1, 8, 1, 1 }, .f16), res.shape()); -} - -/// Given an input images with .{ .c, .w, .h } tags, -/// shuffle values between the channel (.c), width (.w) and height (.h) axes. -/// pixelShuffle(.{ .c, .w, .h }, u) -> .{ .c / u / u, .w * u, .h * u} -/// ref: https://pytorch.org/docs/stable/generated/torch.nn.PixelShuffle.html#pixelshuffle -pub fn pixelShuffle(tensor: Tensor, upscale_factor: u32) Tensor { - const shape = tensor.shape(); - meta.assert(shape.hasTags(.{ .c, .w, .h }), "pixelShuffle({}) is invalide. Missing tags {{.c, .w, .h}}", .{tensor}); - - meta.assert(@mod(shape.dim(.c), upscale_factor * upscale_factor) == 0, "pixelShuffle({}) is invalide. Number of channels {}, isn't divisible by upscale factor {}**2", .{ tensor, shape.dim(.c), upscale_factor }); - - const s = tensor.splitAxis(.c, .{ .c = -1, .upscale_h = upscale_factor, .upscale_w = upscale_factor }); - const perm = s.shape().contiguousPerm(.{ .h, .upscale_h, .w, .upscale_w }); - const cont = s.transpose(perm.constSlice()); - return cont.merge(.{ .h = .{ .h, .upscale_h }, .w = .{ .w, .upscale_w } }).transpose(tensor.shape()); -} - -test pixelShuffle { - const platform = zml.testing.env(); - - const upscale_factor = 3; - var digits: [9 * 4 * 4]i32 = undefined; - for (&digits, 0..) |*d, i| d.* = @intCast(i); - // TODO should we have tags in buffers ? - const input = try zml.Buffer.fromSlice(platform, .{ 1, 9, 4, 4 }, &digits); - const output = try zml.testing.compileAndCallWithTensors( - platform, - pixelShuffle, - .{ zml.Shape.init(.{ .batch_size = 1, .c = 9, .h = 4, .w = 4 }, .i32), upscale_factor }, - .{ input, upscale_factor }, - ); - - const exp = zml.HostBuffer.fromArray(&[1][1][12][12]i32{.{.{ - .{ 0, 16, 32, 1, 17, 33, 2, 18, 34, 3, 19, 35 }, - .{ 48, 64, 80, 49, 65, 81, 50, 66, 82, 51, 67, 83 }, - .{ 96, 112, 128, 97, 113, 129, 98, 114, 130, 99, 115, 131 }, - .{ 4, 20, 36, 5, 21, 37, 6, 22, 38, 7, 23, 39 }, - .{ 52, 68, 84, 53, 69, 85, 54, 70, 86, 55, 71, 87 }, - .{ 100, 116, 132, 101, 117, 133, 102, 118, 134, 103, 119, 135 }, - .{ 8, 24, 40, 9, 25, 41, 10, 26, 42, 11, 27, 43 }, - .{ 56, 72, 88, 57, 73, 89, 58, 74, 90, 59, 75, 91 }, - .{ 104, 120, 136, 105, 121, 137, 106, 122, 138, 107, 123, 139 }, - .{ 12, 28, 44, 13, 29, 45, 14, 30, 46, 15, 31, 47 }, - .{ 60, 76, 92, 61, 77, 93, 62, 78, 94, 63, 79, 95 }, - .{ 108, 124, 140, 109, 125, 141, 110, 126, 142, 111, 127, 143 }, - }}}); - try zml.testing.expectClose(exp, output, 0); -} - -/// Implementation of `torch.roll`. -/// -/// Note: at the difference of Pytorch, shifts need to be explicitly repeated, even if they are the same for all axes. -/// ref: https://pytorch.org/docs/stable/generated/torch.roll.html -pub fn roll(self: Tensor, shifts: []const i64, axes_: []const u8) Tensor { - // TODO(hugo) accept following syntax: x.roll(.{ .a = 5, .b = 8 }) - meta.assert(self.rank() > 0 and shifts.len == axes_.len, "Shifts length ({d}) and dims length ({d}) are not equal, we expect the same length.", .{ shifts.len, axes_.len }); - - if (shifts.len != 1 or axes_.len != 1) { - const tail_shifts = shifts[1..shifts.len]; - const tail_dims = axes_[1..axes_.len]; - const first_dim_rolled = roll(self, &.{shifts[0]}, &.{axes_[0]}); - return roll(first_dim_rolled, tail_shifts, tail_dims); - } - - const a = axes_[0]; - const start = @mod(self.dim(a) - shifts[0], self.dim(a)); - const idx = Tensor.arange(.{ .start = start, .end = start + self.dim(a) }, .f32); - const divisor: f32 = @floatFromInt(self.dim(a)); - return self.gatherValues(a, idx.fmod(divisor).convert(.i32), .{}); -} - -test roll { - const platform = zml.testing.env(); - - const input = try zml.Buffer.fromSlice(platform, .{ 4, 2 }, &[_]f32{ 2, 2, 3, 4, 5, 6, 7, 8 }); - const res = try zml.testing.compileAndCall( - platform, - roll, - .{ input, &[_]i64{ 2, 1 }, &[_]u8{ 0, 1 } }, - ); - - const expectation = zml.HostBuffer.fromSlice(.{ 4, 2 }, &[_]f32{ 6, 5, 8, 7, 2, 1, 4, 3 }); - try zml.testing.expectClose(expectation, res, 1e0); -} - -pub const MeshgridIndexing = enum { xy, ij }; - -/// Mimic Pytorch and Numpy api. -/// The .ij mode is just calling to `zml.nn.cartesianProduct` -/// and has simple semantics. -/// The .xy mode swap the role of the first two vectors, it's generally best -/// to rewrite the calling code to use .ij mode if possible. -/// See Numpy docs: -/// https://numpy.org/doc/stable/reference/generated/numpy.meshgrid.html#numpy.meshgrid -/// - In the 2-D case with vectors of length M and N: -/// * for ‘ij’ indexing, outputs are of shape (M, N) -/// * for ‘xy’ indexing, outputs are of shape (N, M) -/// - In the 3-D case with vectors of length M, N and P: -/// * for ‘ij’ indexing, outputs are of shape (M, N, P) -/// * for ‘xy’ indexing, outputs are of shape (N, M, P) -pub fn meshgrid(comptime N: u3, vectors: [N]Tensor, indexing: MeshgridIndexing) [N]Tensor { - meta.assertComptime(vectors.len >= 1, "Invalid meshgrid. No input.", .{}); - meta.assertComptime(vectors.len <= Tensor.MAX_RANK, "Invalid meshgrid(...). Too many inputs: {}", .{vectors.len}); - - if (vectors.len == 1) return vectors; - - return switch (indexing) { - .ij => zml.Tensor.cartesianProduct(N, vectors), - .xy => { - const x, const y = vectors[0..2].*; - var new_vectors = vectors; - new_vectors[0..2].* = .{ y, x }; - var res = zml.Tensor.cartesianProduct(N, new_vectors); - const y_res, const x_res = res[0..2].*; - res[0..2].* = .{ x_res, y_res }; - return res; - }, - }; -} - -test meshgrid { - const platform = zml.testing.env(); - - const x = try zml.Buffer.fromSlice(platform, .{6}, &[_]i32{ 0, 1, 2, 3, 4, 5 }); - const y = try zml.Buffer.fromSlice(platform, .{4}, &[_]i32{ 0, 1, 2, 3 }); - - const Local = struct { - pub fn meshgrid2(a: Tensor, b: Tensor, indexing: MeshgridIndexing) [2]Tensor { - return meshgrid(2, .{ a, b }, indexing); - } - }; - - // Only test .xy mode, sinc .ij is just calling cartesianProduct which - // got its own tests. - { - const xs, const ys = try zml.testing.compileAndCall(platform, Local.meshgrid2, .{ x, y, .xy }); - try std.testing.expectEqualSlices(i64, &.{ 4, 6 }, xs.dims()); - try std.testing.expectEqualSlices(i64, &.{ 4, 6 }, ys.dims()); - try std.testing.expectEqualDeep( - [4][6]i32{ - .{ 0, 1, 2, 3, 4, 5 }, - .{ 0, 1, 2, 3, 4, 5 }, - .{ 0, 1, 2, 3, 4, 5 }, - .{ 0, 1, 2, 3, 4, 5 }, - }, - try xs.getValue([4][6]i32), - ); - try std.testing.expectEqualDeep( - [4][6]i32{ - .{ 0, 0, 0, 0, 0, 0 }, - .{ 1, 1, 1, 1, 1, 1 }, - .{ 2, 2, 2, 2, 2, 2 }, - .{ 3, 3, 3, 3, 3, 3 }, - }, - try ys.getValue([4][6]i32), - ); - } -} diff --git a/zml/zml.zig b/zml/zml.zig index 484e4146..dafc3a0f 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -3,51 +3,87 @@ //! compiling it for various accelerators and targets, and executing it. //! -pub const Buffer = @import("buffer.zig").Buffer; -pub const Bufferized = @import("tensor.zig").Bufferized; -pub const CompilationOptions = @import("platform.zig").CompilationOptions; -pub const Context = @import("context.zig").Context; -pub const Data = @import("dtype.zig").Data; -pub const DataType = @import("dtype.zig").DataType; -pub const HostBuffer = @import("hostbuffer.zig").HostBuffer; -pub const Platform = @import("platform.zig").Platform; -pub const Shape = @import("shape.zig").Shape; -pub const ShapeOf = @import("tensor.zig").ShapeOf; -pub const Target = @import("platform.zig").Target; -pub const Tensor = @import("tensor.zig").Tensor; - // Namespaces -pub const context = @import("context.zig"); -pub const floats = @import("floats.zig"); -pub const helpers = @import("helpers.zig"); -pub const nn = @import("nn.zig"); -pub const module = @import("module.zig"); -pub const meta = @import("meta.zig"); -pub const platform = @import("platform.zig"); -pub const testing = @import("testing.zig"); -pub const torch = @import("torch.zig"); - -pub const tokenizer = @import("tokenizer.zig"); - -pub const call = ops.call; -pub const compile = module.compile; -pub const compileModel = module.compileModel; -pub const compileFn = module.compileFn; - -pub const ops = @import("ops.zig"); -pub const tools = struct { - pub const Tracer = @import("tools/tracer.zig").Tracer; -}; - -pub const aio = @import("aio.zig"); -pub const sentencepiece = @import("aio/sentencepiece.zig"); - -pub const log = std.log.scoped(.zml); - const std = @import("std"); -test { - // NOTE : testing entrypoint. - // Don't forget to import your module if you want to declare tests declarations that will be run by //zml:test +pub const bazel = @import("bazel"); +const c = @import("c"); +pub const pjrt = @import("pjrt"); +const runfiles = @import("runfiles"); +pub const stdx = @import("stdx"); +pub const tokenizer = @import("zml/tokenizer"); + +pub const attention = @import("attention/attention.zig"); +pub const paged_attention = @import("kernels/paged_attention/paged_attention.zig"); +pub const Buffer = @import("buffer.zig").Buffer; +pub const Compiler = @import("Compiler.zig"); +pub const constants = @import("constants.zig"); +pub const dtype = @import("dtype.zig"); +pub const Data = dtype.Data; +pub const DataType = dtype.DataType; +pub const exe = @import("exe.zig"); +pub const Exe = exe.Exe; +pub const FnExe = exe.FnExe; +pub const floats = @import("floats.zig"); +pub const io = @import("io.zig"); +pub const kernel = @import("kernel/kernel.zig"); +pub const logo = @import("logo.zig"); +pub const mem = @import("mem.zig"); +pub const Bufferized = mem.Bufferized; +pub const meta = @import("meta.zig"); +pub const mlir = @import("mlirx.zig"); +pub const moe = @import("kernels/moe/moe.zig"); +pub const nn = @import("nn.zig"); +pub const ops = @import("ops.zig"); +pub const pjrtx = @import("pjrtx.zig"); +pub const platform = @import("platform.zig"); +pub const Memory = platform.Memory; +pub const Platform = platform.Platform; +pub const Target = platform.Target; +pub const tracer = @import("profiling/tracer.zig"); +pub const quantization = @import("quantization.zig"); +pub const Quantization = quantization.Quantization; +pub const safetensors = @import("safetensors.zig"); +pub const shape = @import("shape.zig"); +pub const Shape = shape.Shape; +pub const Sharding = @import("Sharding.zig"); +pub const Mesh = Sharding.Mesh; +pub const slice = @import("slice.zig"); +pub const Slice = slice.Slice; +pub const tensor = @import("tensor.zig"); +pub const Tensor = tensor.Tensor; +pub const testing = @import("testing.zig"); + +/// Deprecated, use `zml.Compiler` instead +pub const module = struct { + pub const CompilationContext = Compiler; + pub const CompileError = Compiler.Error; + pub const CompilationOptions = Compiler.Options; + + /// Deprecated, use `platform.compileFn` or `zml.Compiler.compile` + pub fn compile( + allocator: std.mem.Allocator, + io_: std.Io, + comptime func: anytype, + args: std.meta.ArgsTuple(@TypeOf(func)), + platform_: *const Platform, + opts: Compiler.Options, + ) Compiler.Error!Exe { + return Compiler.compile(allocator, io_, platform_, func, args, opts); + } +}; + +test "zml" { std.testing.refAllDecls(@This()); } + +pub const KiB = 1024; +pub const MiB = 1024 * KiB; +pub const GiB = 1024 * MiB; +pub const TiB = 1024 * GiB; + +comptime { + // Forces zig to compile `@zml//bazel` module even when users don't depend on it even indirectly. + // Otherwise exported functions declared in `bazel` aren't compiled. + std.testing.refAllDecls(bazel); +}